From ae510e4387343e03a4a9eb12faadbac5e4381add Mon Sep 17 00:00:00 2001 From: Adriano Dal Pastro Date: Wed, 13 May 2026 11:10:25 +0200 Subject: [PATCH] feat(work_plane gui): overlay inliers used for fit as blue dots on RGB The fit endpoint now returns fit_points_px (the post-outlier-filter inliers projected to RGB pixels via Nuwa's RGB intrinsics). The GUI draws them on the same overlay as the ROI rectangle so the user can visually verify which depth pixels were fed to fit_plane_svd. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/presa_mano/api/work_plane_routes.py | 33 ++++++++++++++++++++++++- web/work_plane.js | 16 +++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/presa_mano/api/work_plane_routes.py b/src/presa_mano/api/work_plane_routes.py index 99f3382..edd0b1b 100644 --- a/src/presa_mano/api/work_plane_routes.py +++ b/src/presa_mano/api/work_plane_routes.py @@ -11,6 +11,7 @@ from fastapi import APIRouter, HTTPException from pydantic import BaseModel, Field from ..perception import filter_outliers, nuwa_to_cv_frame +from ..perception.projection import K_from_intrinsics, project_camera_to_pixels log = logging.getLogger(__name__) from ..perception.work_plane import ( @@ -66,7 +67,16 @@ def fit(req: FitReq) -> dict[str, Any]: ) pts_cv = nuwa_to_cv_frame(pc[:, :3]) with sw.step("filter_outliers"): - pts_cv = filter_outliers(pts_cv, n_std=2.0) + # Inline filter_outliers to keep the inlier mask, so we can project + # the SAME points back to RGB pixels for the GUI overlay. + if len(pts_cv) < 10: + inlier_mask = np.ones(len(pts_cv), dtype=bool) + else: + xyz = pts_cv[:, :3] + c_ = xyz.mean(axis=0) + d_ = np.linalg.norm(xyz - c_, axis=1) + inlier_mask = d_ < (d_.mean() + 2.0 * d_.std()) + pts_cv = pts_cv[inlier_mask] if len(pts_cv) < 10: raise HTTPException( status_code=422, @@ -103,10 +113,31 @@ def fit(req: FitReq) -> dict[str, Any]: idx = np.random.choice(len(scene_cv), req.max_preview_points, replace=False) scene_cv = scene_cv[idx] + # Project the inliers used for the fit back to RGB pixels so the GUI can + # overlay them on the live image (verify visually which depth pixels were + # actually fed to fit_plane_svd). pc is in OpenCV camera frame already + # (per pointcloud_roi docstring); use Nuwa's RGB intrinsics. + fit_points_px: list[list[int]] = [] + try: + intr_full = nuwa.intrinsics() + rgb_intr = intr_full.get("rgb") + if rgb_intr and rgb_intr.get("fx") and rgb_intr.get("cx"): + # Inject width/height so K_from_intrinsics has all required keys. + W = int(rgb_intr.get("width", 640)) + H = int(rgb_intr.get("height", 480)) + rgb_intr = {**rgb_intr, "width": W, "height": H} + K = K_from_intrinsics(rgb_intr) + inliers_cv = pc[inlier_mask, :3] # original CV-frame coords + with sw.step("project_inliers"): + fit_points_px = project_camera_to_pixels(inliers_cv, K, (W, H)).tolist() + except Exception as e: + log.warning("could not project fit inliers to RGB pixels: %s", e) + return { "plane": asdict(wp), "preview": True, "points_camera": scene_cv.tolist(), + "fit_points_px": fit_points_px, "timings_ms": sw.as_dict(), } diff --git a/web/work_plane.js b/web/work_plane.js index 3325b65..8a8b5ac 100644 --- a/web/work_plane.js +++ b/web/work_plane.js @@ -10,6 +10,7 @@ const RGB_REFRESH_MS = 1000; let roi = null; // {u0,v0,u1,v1} in image pixel coords let previewPlane = null; // last response.plane (preview, not saved) let previewPoints = null; // list of [x,y,z] in CV camera frame, meters +let fitPointsPx = null; // list of [u,v] pixels of inliers used for the fit let savedPlane = null; // last GET /api/work_plane response.plane function setChip(text, level) { @@ -89,6 +90,12 @@ function startRgbLoop() { function drawOverlay() { const ctx = overlay.getContext('2d'); ctx.clearRect(0, 0, overlay.width, overlay.height); + // Inliers used for the last fit, drawn first so the ROI rectangle stays + // on top. + if (fitPointsPx && fitPointsPx.length) { + ctx.fillStyle = '#4da3ff'; + for (const [u, v] of fitPointsPx) ctx.fillRect(u, v, 2, 2); + } if (roi) { ctx.strokeStyle = '#9b6dff'; ctx.lineWidth = 2; @@ -99,11 +106,15 @@ function startRgbLoop() { drawCross(ctx, firstCorner.u, firstCorner.v); } } + // Re-export so doFit can refresh the overlay after a successful fit. + window.__wpDrawOverlay = drawOverlay; viewport.addEventListener('click', (e) => { const p = imgPixelFromEvent(e); if (firstCorner === null && roi === null) { - // First click: place corner A. + // First click: place corner A. Drop any stale fit-points overlay so we + // don't suggest the new ROI is being analyzed yet. + fitPointsPx = null; firstCorner = p; drawOverlay(); setChip('punto 1/2 piazzato — clicca il secondo angolo'); @@ -132,6 +143,7 @@ function startRgbLoop() { } // ROI already set: start a new selection on the next click. roi = null; + fitPointsPx = null; firstCorner = p; drawOverlay(); setRoiInfo(); @@ -277,8 +289,10 @@ async function doFit() { if (r.detail) throw new Error(formatDetail(r.detail)); previewPlane = r.plane; previewPoints = r.points_camera; + fitPointsPx = r.fit_points_px || null; three.setPlane(previewPlane); three.setScatter(previewPoints, previewPlane, previewPlane.offset_mm); + if (window.__wpDrawOverlay) window.__wpDrawOverlay(); renderDetails(previewPlane); $('wpStateBlock').innerHTML = `preview · RMS ${previewPlane.fit_rms_mm.toFixed(1)} mm · n_inliers ${previewPlane.n_inliers} · offset ${previewPlane.offset_mm} mm · non salvato`; setChip('preview pronto', 'ok');