feat(right_arm): /detect/wrist con YOLO + SAM21 + centroide camera

This commit is contained in:
Adriano Dal Pastro committed 2026-05-20 12:23:10 +02:00
1 parent 3e4e8055db
commit bd4eb8b929
2 files changed
+111

No files matched your search

+50
View File
@@ -8,11 +8,15 @@ from fastapi import APIRouter, HTTPException
from fastapi.responses import Response from fastapi.responses import Response
from pydantic import BaseModel from pydantic import BaseModel
import numpy as np
from ..clients import G1Unavailable from ..clients import G1Unavailable
from ..config import update_config from ..config import update_config
from ..geometry import quaternion_xyzw_to_rpy_deg from ..geometry import quaternion_xyzw_to_rpy_deg
from ..perception.sam21_bridge import Sam21Fallback, segment_with_sam21
from ..timing import Stopwatch from ..timing import Stopwatch
from . import _deps from . import _deps
from .perception_routes import _bbox_to_roi
router = APIRouter() router = APIRouter()
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -137,3 +141,49 @@ def detect_front(req: DetectFrontReq) -> dict[str, Any]:
"image_size": [int(intr.get("width", 0)), int(intr.get("height", 0))], "image_size": [int(intr.get("width", 0)), int(intr.get("height", 0))],
"timings_ms": sw.as_dict(), "timings_ms": sw.as_dict(),
} }
class DetectWristReq(BaseModel):
class_name: Optional[str] = None
conf: Optional[float] = None
def _detect_wrist_core(class_name: str | None, conf: float | None, sw: Stopwatch
) -> dict[str, Any]:
"""Funzione core riusata da /detect/wrist e da /move/toward."""
nuwa = _deps.nuwa()
image_url = nuwa.rgb_url()
try:
with sw.step("yolo"):
yres = _deps.yolo().detect_url(image_url=image_url, conf=conf)
except Exception as e:
raise HTTPException(status_code=502, detail=f"YOLO unreachable: {e}")
top = _top_detection(yres, class_name)
if top is None:
raise HTTPException(status_code=404, detail=f"no '{class_name or 'object'}' detected")
bbox = _bbox_to_roi(top["bbox"])
try:
seg = segment_with_sam21(nuwa, _deps.sam21(), bbox, sw)
except Sam21Fallback as e:
raise HTTPException(status_code=422, detail=f"SAM21 fallback: {e}")
pts = np.asarray(seg.points_cv_m, dtype=np.float64)
if pts.shape[0] < 10:
raise HTTPException(status_code=422, detail=f"too few sam21 points: {pts.shape[0]}")
with sw.step("centroid"):
centroid = pts.mean(axis=0)
distance = float(np.linalg.norm(centroid))
return {
"detection": top,
"bbox": list(bbox),
"centroid_camera_m": [float(centroid[0]), float(centroid[1]), float(centroid[2])],
"distance_m": distance,
"n_sam21_points": int(pts.shape[0]),
}
@router.post("/detect/wrist")
def detect_wrist(req: DetectWristReq) -> dict[str, Any]:
sw = Stopwatch()
body = _detect_wrist_core(req.class_name, req.conf, sw)
body["timings_ms"] = sw.as_dict()
return body
+61
View File
@@ -165,3 +165,64 @@ def test_detect_front_falls_back_to_proxy_url_on_yolo_failure():
assert len(attempts) == 2 assert len(attempts) == 2
assert "192.168" in attempts[0] assert "192.168" in attempts[0]
assert attempts[1].endswith("/api/right_arm/snapshot/front") assert attempts[1].endswith("/api/right_arm/snapshot/front")
def test_detect_wrist_returns_centroid_and_distance():
# YOLO → bbox → SAM21 → centroide
import numpy as np
fake_points = np.array([
[0.01, -0.02, 0.20],
[0.02, -0.01, 0.21],
[0.00, -0.03, 0.19],
] * 5, dtype=np.float64) # 15 punti, > 10 minimo
with patch("presa_mano.api.right_arm_routes._deps.nuwa") as nuwa_f, \
patch("presa_mano.api.right_arm_routes._deps.yolo") as yolo_f, \
patch("presa_mano.api.right_arm_routes.segment_with_sam21") as seg_f, \
patch("presa_mano.api.right_arm_routes._deps.sam21") as sam_f:
nuwa = MagicMock()
nuwa.rgb_url.return_value = "http://localhost:8877/api/rgb"
nuwa_f.return_value = nuwa
yolo = MagicMock()
yolo.detect_url.return_value = _fake_yolo_top()
yolo_f.return_value = yolo
sam_f.return_value = MagicMock()
seg = MagicMock()
seg.points_cv_m = fake_points
seg_f.return_value = seg
r = client.post("/api/right_arm/detect/wrist",
json={"class_name": "pestello"})
assert r.status_code == 200
body = r.json()
assert body["detection"]["class"] == "pestello"
assert "bbox" in body
c = body["centroid_camera_m"]
assert abs(c[2] - 0.20) < 0.05
assert body["distance_m"] > 0
assert body["n_sam21_points"] == 15
def test_detect_wrist_422_too_few_points():
import numpy as np
with patch("presa_mano.api.right_arm_routes._deps.nuwa") as nuwa_f, \
patch("presa_mano.api.right_arm_routes._deps.yolo") as yolo_f, \
patch("presa_mano.api.right_arm_routes.segment_with_sam21") as seg_f, \
patch("presa_mano.api.right_arm_routes._deps.sam21") as sam_f:
nuwa = MagicMock(); nuwa.rgb_url.return_value = "http://x"; nuwa_f.return_value = nuwa
yolo = MagicMock(); yolo.detect_url.return_value = _fake_yolo_top(); yolo_f.return_value = yolo
sam_f.return_value = MagicMock()
seg = MagicMock(); seg.points_cv_m = np.zeros((3, 3))
seg_f.return_value = seg
r = client.post("/api/right_arm/detect/wrist",
json={"class_name": "pestello"})
assert r.status_code == 422
def test_detect_wrist_404_when_yolo_no_match():
with patch("presa_mano.api.right_arm_routes._deps.nuwa") as nuwa_f, \
patch("presa_mano.api.right_arm_routes._deps.yolo") as yolo_f:
nuwa = MagicMock(); nuwa.rgb_url.return_value = "http://x"; nuwa_f.return_value = nuwa
yolo = MagicMock(); yolo.detect_url.return_value = {"detections": []}; yolo_f.return_value = yolo
r = client.post("/api/right_arm/detect/wrist", json={"class_name": "pestello"})
assert r.status_code == 404