feat(pose_3d): tag locator save, front-tag overlay, sidebar tabs, cloud params persist

- Sidebar a 3 tab verticali (POSE / CAMERAS / MISC) per ridurre lo scroll
- Tag locator: POST /api/pose_3d/tag_offset/{save,} salva crocetta nel
  viewport con snapshot giunti, current.json + archive rotation
- Front-camera tag overlay (MISC):
  - GET /api/pose_3d/front_tags scansiona ArUco sul D435i e per ogni tag
    misura i 4 corner 3D via depth sensor (indipendente da solvePnP);
    outlier rejection ±20% dalla mediana per ignorare corner che catturano
    background sul bordo del marker
  - Rendering quad cyan + label compatta (id, edge_mm, distanza camera)
    agganciato al d435_link, segue la camera anche in modalita' hand-eye
  - Trigger via pollState() quando il flag e' on, niente timer separato
- Cloud cross-camera: memorizza step/max_depth/sizes/scale_xy/flip/handeye
  in pose_3d.cloud_params al click di Acquisisci, ripopola UI da config
  al primo poll (state.cloud_params_populated guard)
- D435i front cloud: applica nuwa_to_cv_frame (il servizio ritorna native,
  non OpenCV come si pensava)
- Cache-bust per pose_3d.js e pose_3d.css

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
AdrianoandClaude Opus 4.7 committed 2026-06-01 16:53:26 +02:00
1 parent ae8e609b7e
commit fb0e9b9112
6 files changed
+816 -20

No files matched your search

+238 -2
View File
@@ -1,8 +1,12 @@
"""Endpoint per la tab "Pose 3D": editor visuale + jog dei polsi."""
from __future__ import annotations
import json
import logging
import os
import re
import time
from pathlib import Path
from typing import Any, Literal, Optional
import httpx
@@ -99,6 +103,7 @@ def get_state() -> dict[str, Any]:
"live_max_step_m": float(cfg.get("live_max_step_m", 0.15)),
"right_wrist_camera": cfg.get("right_wrist_camera") or {},
"clouds": cfg.get("clouds") or {},
"cloud_params": cfg.get("cloud_params") or {},
"timings_ms": sw.as_dict(),
}
@@ -229,11 +234,14 @@ def get_clouds(step: int = 8, max_depth_mm: int = 2000) -> dict[str, Any]:
except Exception as e:
result["nuwa"] = {"error": f"{type(e).__name__}: {e}"}
# D435i: stesso client NuwaClient ma backend RealSense, gia' in OpenCV
# D435i: stesso client NuwaClient (interfaccia uniforme), il servizio
# ritorna però Nuwa-native (X right, Y up, Z back) come per la Nuwa.
# Quindi: stessa conversione → OpenCV optical (X right, Y down, Z forward).
try:
with sw.step("d435_cloud"):
pts = _deps.front_camera().pointcloud(step=step, max_depth_mm=max_depth_mm)
result["d435"] = {"n": int(pts.shape[0]), "points": _pack(pts)}
pts_cv = nuwa_to_cv_frame(pts)
result["d435"] = {"n": int(pts_cv.shape[0]), "points": _pack(pts_cv)}
except Exception as e:
result["d435"] = {"error": f"{type(e).__name__}: {e}"}
@@ -241,6 +249,234 @@ def get_clouds(step: int = 8, max_depth_mm: int = 2000) -> dict[str, Any]:
return result
# --- Front-camera tag finder: detect + misura indipendente via depth ----
# Scansiona ArUco DICT_4X4_50 sulla RGB del D435i frontale e per ogni tag
# trovato misura i 4 corner 3D usando il depth sensor (independent da pnp).
# Output in frame ottico OpenCV (X right, Y down, Z forward) — il client
# lo agganciera' al link URDF del d435 nella scena 3D.
def _parse_id_range(spec: str) -> list[int]:
"""Accetta "0-49" o "1,10,20,33"."""
spec = (spec or "").strip()
if not spec:
return list(range(50))
out: list[int] = []
for part in spec.split(","):
part = part.strip()
if "-" in part:
a, b = part.split("-", 1)
out.extend(range(int(a), int(b) + 1))
else:
out.append(int(part))
return sorted(set(out))
@router.get("/front_tags")
def get_front_tags(ids: str = "0-49", probe_size_mm: float = 40.0) -> dict[str, Any]:
"""Detect ArUco markers via front D435i + misura geometria 3D reale.
Per ogni tag trovato chiama /api/measure ai 4 corner per ottenere i punti
3D direttamente dal depth sensor (indipendente da solvePnP). Cosi' la
dimensione misurata non dipende dalla size assunta in input.
`probe_size_mm` viene passato a marker-finder solo per soddisfare la sua
API (richiede un size per ogni id) ma non influisce sulla misura.
"""
sw = Stopwatch()
front = _deps.front_camera()
mk = _deps.marker()
# 1) intrinsics RGB del D435i → passate a marker-finder per pose corrette
with sw.step("intrinsics"):
intr = front.intrinsics()
rgb = (intr or {}).get("rgb") or {}
if not rgb or "fx" not in rgb:
raise HTTPException(500, detail=f"front camera intrinsics missing rgb: {intr!r}")
K = {
"camera_matrix": [
[float(rgb["fx"]), 0.0, float(rgb["cx"])],
[0.0, float(rgb["fy"]), float(rgb["cy"])],
[0.0, 0.0, 1.0],
],
"dist_coeffs": [
float((rgb.get("distortion") or {}).get(k, 0.0))
for k in ("k1", "k2", "p1", "p2", "k3")
],
}
# 2) detect (passiamo intrinsics override per non usare il default Nuwa)
id_list = _parse_id_range(ids)
markers = [{"id": int(i), "size_mm": float(probe_size_mm)} for i in id_list]
with sw.step("detect"):
det = mk.detect_url(markers=markers, image_url=front.rgb_url(), intrinsics=K)
# 3) per ogni found: depth ai 4 corner + center
tags_out: list[dict[str, Any]] = []
with sw.step("measure_corners"):
for m in det.get("found", []):
corners = m.get("corners") or []
if len(corners) != 4:
continue
pts3d: list[list[float]] = []
depths: list[Optional[int]] = []
try:
for (u, v) in corners:
md = front.measure(int(round(u)), int(round(v)), radius=2)
pt = (md or {}).get("point_m") or {}
d_mm = (md or {}).get("depth_mm")
if not pt or d_mm in (None, 0):
pts3d.append([float("nan")] * 3)
depths.append(None)
else:
pts3d.append([float(pt["x"]), float(pt["y"]), float(pt["z"])])
depths.append(int(d_mm))
cu = sum(c[0] for c in corners) / 4.0
cv = sum(c[1] for c in corners) / 4.0
cmd = front.measure(int(round(cu)), int(round(cv)), radius=4)
cpt = (cmd or {}).get("point_m") or {}
center3d = (
[float(cpt["x"]), float(cpt["y"]), float(cpt["z"])]
if cpt else [float("nan")] * 3
)
except Exception as e:
log.warning("front_tags: measure failed for id=%s: %s", m.get("id"), e)
continue
# Outlier rejection sulla depth: il pixel di un corner sul bordo
# del marker spesso prende la depth del background (es. 1358mm
# mentre gli altri 3 sono a 320mm). Rifiutiamo corner con depth
# > 20% off dalla mediana → li trattiamo come NaN nel calcolo edge.
valid_d = [d for d in depths if d is not None]
if len(valid_d) >= 3:
# mediana semplice (4 elementi → media dei 2 centrali)
sd = sorted(valid_d)
median_d = (sd[len(sd)//2 - 1] + sd[len(sd)//2]) / 2 if len(sd) % 2 == 0 else sd[len(sd)//2]
for i, d in enumerate(depths):
if d is None:
continue
if abs(d - median_d) / median_d > 0.20:
pts3d[i] = [float("nan")] * 3 # esclude da edge_mm
# 3D edge lengths (mm) — robusti a un corner senza depth (NaN salta)
def _L(a: list[float], b: list[float]) -> Optional[float]:
if any(x != x for x in a + b): # NaN check
return None
return (sum((a[i] - b[i]) ** 2 for i in range(3))) ** 0.5 * 1000.0
edges_mm = [_L(pts3d[i], pts3d[(i + 1) % 4]) for i in range(4)]
valid_edges = [e for e in edges_mm if e is not None]
edge_mean = sum(valid_edges) / len(valid_edges) if valid_edges else None
distance_m: Optional[float] = None
if all(x == x for x in center3d):
distance_m = (sum(c ** 2 for c in center3d)) ** 0.5
tags_out.append({
"id": int(m["id"]),
"corners_pixel": [[float(c[0]), float(c[1])] for c in corners],
"corners_3d_optical_m": pts3d,
"corner_depths_mm": depths,
"center_3d_optical_m": center3d,
"edge_mm_per_side": edges_mm,
"edge_mm": edge_mean,
"distance_m": distance_m,
})
result = {
"tags": tags_out,
"not_found_count": int(len(det.get("not_found", []))),
"frame": "d435_rgb_optical", # OpenCV: X right, Y down, Z forward
"timings_ms": sw.as_dict(),
}
return result
# --- Tag locator: piazza crocetta nella scena 3D e salva la posa in pelvis ---
# Workflow utente (vedi web/pose_3d.js sezione "tag locator"):
# 1) seleziona tag da dropdown (id+size+label)
# 2) click nel viewport → crocetta a quel punto
# 3) "Salva" → POST qui → scrive current.json + ruota archive
# TODO(orientation): per ora salviamo solo CdM; orientamento non catturato.
_TAG_OFFSET_DEFAULT_DIR = "/home/harry/robot/logging/wrist-tag-offset"
def _tag_offset_dir() -> Path:
"""Letto da config.yaml pose_3d.tag_offset.save_dir; ricaricato ad ogni call
cosi' un PUT /api/config a caldo prende effetto subito."""
cfg = (get_config().get("pose_3d", {}) or {}).get("tag_offset", {}) or {}
return Path(cfg.get("save_dir") or _TAG_OFFSET_DEFAULT_DIR)
class TagOffsetSaveReq(BaseModel):
tag_id: int
tag_size_mm: float
tag_label: Optional[str] = None
position_pelvis_m: list[float] = Field(min_length=3, max_length=3)
# Snapshot dei giunti reali al click (name → radianti). Null se G1 offline.
joints_at_save_rad: Optional[dict[str, float]] = None
note: Optional[str] = None
@router.post("/tag_offset/save")
def post_tag_offset_save(req: TagOffsetSaveReq) -> dict[str, Any]:
sw = Stopwatch()
ts = time.strftime("%Y%m%d_%H%M%S")
payload = {
"tag_id": int(req.tag_id),
"tag_size_mm": float(req.tag_size_mm),
"tag_label": req.tag_label,
"position_pelvis_m": [float(v) for v in req.position_pelvis_m],
"joints_at_save_rad": req.joints_at_save_rad,
"frame": "pelvis",
"method": "manual_3d_click",
"timestamp": ts,
"note": req.note,
}
base = _tag_offset_dir()
archive_dir = base / "archive"
current_path = base / "current.json"
archived_from: Optional[str] = None
try:
with sw.step("mkdir"):
archive_dir.mkdir(parents=True, exist_ok=True)
with sw.step("archive"):
if current_path.exists():
old = json.loads(current_path.read_text())
old_ts = old.get("timestamp") or ts
dest = archive_dir / f"{old_ts}.json"
# se collide, suffisso numerico
i = 1
while dest.exists():
dest = archive_dir / f"{old_ts}_{i}.json"
i += 1
current_path.rename(dest)
archived_from = str(dest)
with sw.step("write"):
current_path.write_text(json.dumps(payload, indent=2))
except OSError as e:
raise HTTPException(status_code=500, detail=f"tag_offset save fail: {e}")
return {
"ok": True,
"path": str(current_path),
"archived_from": archived_from,
"saved": payload,
"timings_ms": sw.as_dict(),
}
@router.get("/tag_offset")
def get_tag_offset() -> dict[str, Any]:
"""Restituisce il current.json se esiste, altrimenti 404."""
current_path = _tag_offset_dir() / "current.json"
if not current_path.exists():
raise HTTPException(status_code=404, detail="no tag offset saved yet")
try:
return json.loads(current_path.read_text())
except (OSError, json.JSONDecodeError) as e:
raise HTTPException(status_code=500, detail=f"read fail: {e}")
@router.post("/unlock")
def post_unlock() -> dict[str, Any]:
"""Proxy a POST {bridge}/unlock. Recovery rapido da arms: error_timeout."""
+2 -1
View File
@@ -107,7 +107,8 @@ def index() -> HTMLResponse:
idx = WEB_DIR / "index.html"
html = idx.read_text(encoding="utf-8")
# Cache-bust app.js and style.css with their mtime so a rebuild forces refetch.
for name in ("app.js", "work_plane.js", "right_arm.js", "style.css"):
for name in ("app.js", "work_plane.js", "right_arm.js", "pose_3d.js",
"style.css", "pose_3d.css"):
f = WEB_DIR / name
if f.exists():
v = int(f.stat().st_mtime)