fix(sam21): inject rgb/ir width+height into intrinsics from decoded images

Nuwa /api/intrinsics omits width/height from the rgb and ir blocks, so
SAM21 rejected the payload with KeyError('width') and we silently fell
back to bbox on every request. Derive the dimensions from the decoded
BGR + IR depth shapes and augment intr_full before building
CameraRegistration and calling SAM21. Verified end-to-end: source=sam21,
~2200 segmented points on Mortaio at 128 ms GPU inference.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adriano Dal PastroandClaude Opus 4.7 committed 2026-05-11 15:33:52 +02:00
1 parent b9bf878445
commit cc19f73b96
1 file changed
+12 -3
+12 -3
View File
@@ -149,8 +149,7 @@ def _points_via_sam21(nuwa, bbox: tuple[int, int, int, int], sw: Stopwatch) -> n
except (NuwaError, httpx.RequestError) as e:
raise HTTPException(status_code=502, detail=f"Nuwa unreachable: {e}")
rgb_intr = intr_full.get("rgb")
if not rgb_intr:
if not intr_full.get("rgb"):
log.warning("SAM21 path skipped: nuwa intrinsics missing 'rgb' block")
raise _Sam21Fallback("nuwa intrinsics missing 'rgb' block")
@@ -162,6 +161,7 @@ def _points_via_sam21(nuwa, bbox: tuple[int, int, int, int], sw: Stopwatch) -> n
if bgr is None:
log.warning("SAM21 path skipped: could not decode Nuwa RGB JPEG")
raise _Sam21Fallback("could not decode Nuwa RGB JPEG")
h_rgb, w_rgb = bgr.shape[:2]
ok, rgb_png_buf = cv2.imencode(".png", bgr)
if not ok:
log.warning("SAM21 path skipped: could not re-encode RGB as PNG")
@@ -178,6 +178,15 @@ def _points_via_sam21(nuwa, bbox: tuple[int, int, int, int], sw: Stopwatch) -> n
log.warning("SAM21 path skipped: depth PNG dtype %s (expected uint16)",
depth_ir.dtype)
raise _Sam21Fallback(f"depth PNG dtype expected uint16, got {depth_ir.dtype}")
h_ir, w_ir = depth_ir.shape
# Nuwa /api/intrinsics omits width/height — inject the real ones
# from the decoded images so CameraRegistration and SAM21 agree on
# frame sizes.
intr_full = {
**intr_full,
"rgb": {**intr_full["rgb"], "width": w_rgb, "height": h_rgb},
"ir": {**intr_full.get("ir", {}), "width": w_ir, "height": h_ir},
}
reg = CameraRegistration.from_nuwa(intr_full)
depth_aligned = align_depth_to_rgb(depth_ir, reg)
ok, depth_png_buf = cv2.imencode(".png", depth_aligned)
@@ -196,7 +205,7 @@ def _points_via_sam21(nuwa, bbox: tuple[int, int, int, int], sw: Stopwatch) -> n
pts_cv = _deps.sam21().segment_pointcloud(
rgb_png=rgb_png,
depth_png=depth_png_out,
intrinsics=rgb_intr,
intrinsics=intr_full["rgb"],
bbox=bbox,
depth_scale_mm=1.0,
)