From 23299001f5067731f1224607862e492c068892b9 Mon Sep 17 00:00:00 2001 From: Olivia Casadei Date: Thu, 1 Oct 2026 10:14:21 +0200 Subject: [PATCH] feat: scripts/compare_host_align.py, Intel's align on PC2 vs an align on the host Read-only (GET only). For each frameset: /api/raw/latest and /api/capture of the same frameset_seq (retried until they match), the raw depth aligned on the host with pyrealsense2 (software_device built from /api/intrinsics: both streams' intrinsics, distortion model and coefficients, extrinsics depth->color; then syncer + rs.align), and a pixel-by-pixel comparison of the two aligned depths: pixels valid in both, only in one, exact-equal fraction, |diff| mean/median/p99/max in mm. One JSON line per frameset. The syncer may emit the first pair split, so a pair is pushed again (up to 5 times) until a complete frameset of that frame number comes out. Co-Authored-By: Claude Opus 5.5 --- scripts/compare_host_align.py | 213 ++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 scripts/compare_host_align.py diff --git a/scripts/compare_host_align.py b/scripts/compare_host_align.py new file mode 100644 index 0000000..873d2f9 --- /dev/null +++ b/scripts/compare_host_align.py @@ -0,0 +1,213 @@ +"""Compare Intel's align on PC2 with an align on the host, on the SAME frameset. Read-only. + + python scripts/compare_host_align.py --url http://192.168.123.99:8878 [--framesets 10] [--tries 20] + +For each frameset: GET /api/raw/latest (color + depth, not aligned) and +GET /api/capture (depth aligned on PC2) until both carry the same +frameset_seq; align the raw depth on the host with pyrealsense2 (a +software_device built from /api/intrinsics, then rs.align); report, pixel by +pixel, how the two aligned depths differ. Prints one JSON line per frameset +and a summary. Only GET requests; nothing on PC2 changes. + +The host's pyrealsense2 may be newer than PC2's (2.58 here, 2.50 there): the +two aligns are then not even the same code version. +""" + +import argparse +import base64 +import json +import sys + +import cv2 +import httpx +import numpy as np +import pyrealsense2 as rs + +SYNC_WAIT_MS = 200 # wait for the syncer's frameset after a push +SYNC_PUSHES_MAX = 5 # the syncer may emit the first pair split (color alone): push the same pair again + + +# Calibration ------------------------------------------------------------------ + +def rs_intrinsics(stream: dict) -> rs.intrinsics: + """rs.intrinsics from one stream of /api/intrinsics ("ir" or "rgb").""" + d = stream["distortion"] or {"k1": 0, "k2": 0, "k3": 0, "p1": 0, "p2": 0} + intr = rs.intrinsics() + intr.width, intr.height = stream["width"], stream["height"] + intr.fx, intr.fy, intr.ppx, intr.ppy = stream["fx"], stream["fy"], stream["cx"], stream["cy"] + intr.model = getattr(rs.distortion, stream["model"] or "none") + intr.coeffs = [d["k1"], d["k2"], d["p1"], d["p2"], d["k3"]] # the SDK's order + return intr + + +def rs_extrinsics(ext: dict) -> rs.extrinsics: + """rs.extrinsics depth->color from the API's (rotation row-major, translation mm).""" + r = ext["rotation"] + out = rs.extrinsics() + out.rotation = [r[0], r[3], r[6], r[1], r[4], r[7], r[2], r[5], r[8]] # back to column-major + out.translation = [t / 1000.0 for t in ext["translation"]] + return out + + +# Host align --------------------------------------------------------------------- + +class HostAligner: + """rs.align(color) on frames pushed into a software_device with PC2's calibration.""" + + def __init__(self, calibration: dict): + self._device = rs.software_device() + self._depth = self._device.add_sensor("Depth") + self._color = self._device.add_sensor("Color") + self._depth.add_read_only_option(rs.option.depth_units, 0.001) # we push millimeters + self._depth_profile = self._depth.add_video_stream( + _video_stream(rs.stream.depth, 0, rs.format.z16, 2, rs_intrinsics(calibration["ir"]))) + self._color_profile = self._color.add_video_stream( + _video_stream(rs.stream.color, 1, rs.format.bgr8, 3, rs_intrinsics(calibration["rgb"]))) + self._depth_profile.register_extrinsics_to(self._color_profile, rs_extrinsics(calibration["extrinsics_ir_to_rgb"])) + self._sync = rs.syncer() + self._align = rs.align(rs.stream.color) + self._number = 0 + self._started = [] + for sensor, profile in ((self._depth, self._depth_profile), (self._color, self._color_profile)): + sensor.open(profile) + sensor.start(self._sync) + self._started.append(sensor) + + def close(self) -> None: + for sensor in self._started: + sensor.stop() + sensor.close() + self._started = [] + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def align(self, depth_mm: np.ndarray, color_bgr: np.ndarray) -> np.ndarray: + """Depth (mm, depth grid) aligned onto the color grid by rs.align, as uint16 mm.""" + depth_mm, color_bgr = np.ascontiguousarray(depth_mm), np.ascontiguousarray(color_bgr) + for _ in range(SYNC_PUSHES_MAX): + self._number += 1 + stamp_ms = self._number * 1000.0 + _push(self._depth, self._depth_profile, depth_mm, 2, self._number, stamp_ms, depth=True) + _push(self._color, self._color_profile, color_bgr, 3, self._number, stamp_ms) + frames = self._complete_frameset(self._number) + if frames is not None: + aligned = self._align.process(frames) + return np.asanyarray(aligned.get_depth_frame().get_data()).copy() + raise RuntimeError(f"syncer gave no complete frameset in {SYNC_PUSHES_MAX} pushes") + + def _complete_frameset(self, number: int): + """The syncer's frameset holding depth AND color of frame `number`, or None.""" + for _ in range(2): # a split pair comes out as two framesets + try: + frames = self._sync.wait_for_frames(SYNC_WAIT_MS) + except RuntimeError: # "Frame did not arrive in time": try pushing again + return None + depth = frames.get_depth_frame() + if frames.size() == 2 and depth and depth.get_frame_number() == number: + return frames + return None + + +def _video_stream(kind, uid: int, fmt, bpp: int, intrinsics: rs.intrinsics) -> rs.video_stream: + vs = rs.video_stream() + vs.type, vs.index, vs.uid, vs.fmt, vs.bpp = kind, 0, uid, fmt, bpp + vs.width, vs.height, vs.fps, vs.intrinsics = intrinsics.width, intrinsics.height, 30, intrinsics + return vs + + +def _push(sensor, profile, pixels: np.ndarray, bpp: int, number: int, stamp_ms: float, depth: bool = False) -> None: + frame = rs.software_video_frame() + frame.pixels = pixels + frame.bpp = bpp + frame.stride = pixels.shape[1] * bpp + frame.timestamp = stamp_ms + frame.domain = rs.timestamp_domain.hardware_clock + frame.frame_number = number + frame.profile = profile.as_video_stream_profile() + if depth: + frame.depth_units = 0.001 + sensor.on_video_frame(frame) + + +# PC2 side (GET only) ------------------------------------------------------------ + +def _data(response: httpx.Response) -> dict: + response.raise_for_status() + body = response.json() + if not body.get("success"): + raise RuntimeError(f"{response.request.url}: {body.get('error')}") + return body["data"] + + +def _decode(b64: str, flags: int) -> np.ndarray: + return cv2.imdecode(np.frombuffer(base64.b64decode(b64), np.uint8), flags) + + +def fetch_same_frameset(client: httpx.Client, tries: int) -> tuple[dict, dict, int]: + """(raw, capture, attempts): /api/raw/latest and /api/capture of one frameset_seq.""" + for attempt in range(1, tries + 1): + raw = _data(client.get("/api/raw/latest")) + capture = _data(client.get("/api/capture")) + if capture.get("frameset_seq") == raw["frameset_seq"]: + return raw, capture, attempt + raise RuntimeError(f"no matching frameset in {tries} attempts (another client may keep the align busy)") + + +# Comparison (pure) ------------------------------------------------------------- + +def compare_depths(pc2: np.ndarray, host: np.ndarray) -> dict: + """Pixel-by-pixel difference of two aligned depth maps in mm (same shape).""" + if pc2.shape != host.shape: + raise ValueError(f"shapes differ: PC2 {pc2.shape}, host {host.shape}") + both = (pc2 > 0) & (host > 0) + diff = np.abs(pc2[both].astype(np.int32) - host[both].astype(np.int32)) + total = pc2.size + return { + "pixels": int(total), + "valid_both": int(both.sum()), + "only_pc2": int(((pc2 > 0) & (host == 0)).sum()), + "only_host": int(((host > 0) & (pc2 == 0)).sum()), + "equal_fraction_of_valid_both": float((diff == 0).mean()) if diff.size else None, + "abs_diff_mm": None if not diff.size else { + "mean": float(diff.mean()), "median": float(np.median(diff)), + "p99": float(np.percentile(diff, 99)), "max": int(diff.max()), + }, + } + + +def compare_one(client: httpx.Client, aligner: HostAligner, tries: int) -> dict: + raw, capture, attempts = fetch_same_frameset(client, tries) + host = aligner.align(_decode(raw["depth_png_b64"], cv2.IMREAD_UNCHANGED), + _decode(raw["color_jpeg_b64"], cv2.IMREAD_COLOR)) + pc2 = _decode(capture["depth_png_b64"], cv2.IMREAD_UNCHANGED) + return {"frameset_seq": raw["frameset_seq"], "attempts": attempts, **compare_depths(pc2, host)} + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--url", required=True, help="RealSense service, e.g. http://192.168.123.99:8878") + ap.add_argument("--framesets", type=int, default=10) + ap.add_argument("--tries", type=int, default=20, help="attempts to get raw and aligned of one frameset") + ap.add_argument("--api-key", default="") + args = ap.parse_args(argv) + headers = {"X-Api-Key": args.api_key} if args.api_key else {} + results = [] + with httpx.Client(base_url=args.url, headers=headers, timeout=10) as client: + calibration = _data(client.get("/api/intrinsics")) + with HostAligner(calibration) as aligner: + for _ in range(args.framesets): + result = compare_one(client, aligner, args.tries) + results.append(result) + print(json.dumps(result)) + means = [r["abs_diff_mm"]["mean"] for r in results if r["abs_diff_mm"]] + print(json.dumps({"summary": {"framesets": len(results), "pyrealsense2_host": rs.__version__, + "mean_abs_diff_mm_per_frameset": means}})) + return 0 + + +if __name__ == "__main__": + sys.exit(main())