From cc19f73b9672b3c3790bbf9f600d721d129a2f93 Mon Sep 17 00:00:00 2001 From: Adriano Dal Pastro Date: Mon, 11 May 2026 15:33:52 +0200 Subject: [PATCH] fix(sam21): inject rgb/ir width+height into intrinsics from decoded images Nuwa /api/intrinsics omits width/height from the rgb and ir blocks, so SAM21 rejected the payload with KeyError('width') and we silently fell back to bbox on every request. Derive the dimensions from the decoded BGR + IR depth shapes and augment intr_full before building CameraRegistration and calling SAM21. Verified end-to-end: source=sam21, ~2200 segmented points on Mortaio at 128 ms GPU inference. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/presa_mano/api/grasp_routes.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/presa_mano/api/grasp_routes.py b/src/presa_mano/api/grasp_routes.py index 4156575..38331b2 100644 --- a/src/presa_mano/api/grasp_routes.py +++ b/src/presa_mano/api/grasp_routes.py @@ -149,8 +149,7 @@ def _points_via_sam21(nuwa, bbox: tuple[int, int, int, int], sw: Stopwatch) -> n except (NuwaError, httpx.RequestError) as e: raise HTTPException(status_code=502, detail=f"Nuwa unreachable: {e}") - rgb_intr = intr_full.get("rgb") - if not rgb_intr: + if not intr_full.get("rgb"): log.warning("SAM21 path skipped: nuwa intrinsics missing 'rgb' block") raise _Sam21Fallback("nuwa intrinsics missing 'rgb' block") @@ -162,6 +161,7 @@ def _points_via_sam21(nuwa, bbox: tuple[int, int, int, int], sw: Stopwatch) -> n if bgr is None: log.warning("SAM21 path skipped: could not decode Nuwa RGB JPEG") raise _Sam21Fallback("could not decode Nuwa RGB JPEG") + h_rgb, w_rgb = bgr.shape[:2] ok, rgb_png_buf = cv2.imencode(".png", bgr) if not ok: log.warning("SAM21 path skipped: could not re-encode RGB as PNG") @@ -178,6 +178,15 @@ def _points_via_sam21(nuwa, bbox: tuple[int, int, int, int], sw: Stopwatch) -> n log.warning("SAM21 path skipped: depth PNG dtype %s (expected uint16)", depth_ir.dtype) raise _Sam21Fallback(f"depth PNG dtype expected uint16, got {depth_ir.dtype}") + h_ir, w_ir = depth_ir.shape + # Nuwa /api/intrinsics omits width/height — inject the real ones + # from the decoded images so CameraRegistration and SAM21 agree on + # frame sizes. + intr_full = { + **intr_full, + "rgb": {**intr_full["rgb"], "width": w_rgb, "height": h_rgb}, + "ir": {**intr_full.get("ir", {}), "width": w_ir, "height": h_ir}, + } reg = CameraRegistration.from_nuwa(intr_full) depth_aligned = align_depth_to_rgb(depth_ir, reg) ok, depth_png_buf = cv2.imencode(".png", depth_aligned) @@ -196,7 +205,7 @@ def _points_via_sam21(nuwa, bbox: tuple[int, int, int, int], sw: Stopwatch) -> n pts_cv = _deps.sam21().segment_pointcloud( rgb_png=rgb_png, depth_png=depth_png_out, - intrinsics=rgb_intr, + intrinsics=intr_full["rgb"], bbox=bbox, depth_scale_mm=1.0, )