Initial commit: RealSense D435 API server

Drop-in replacement for the Nuwa-HP60C camera container, exposing RGB +
aligned depth + intrinsics via REST/WebSocket so the existing
pestello-pose-detector pipeline can consume frames without changes.

Key adaptations for D4xx on USB 2.0 (this robot has no USB 3 port exposed
externally — see STATUS.md):
- pyrealsense2 built from source; the librealsense v2.55.1 install rule
  doesn't ship the main .so, so we copy it from the build dir and add an
  __init__.py that re-exports the submodule.
- Color stream uses YUYV format on USB 2.x (BGR8/RGB8 time out on this
  bus); converted to BGR via cv2.cvtColor in capture().
- rs.align is skipped on USB 2 (incompatible with YUYV); color is
  cv2-resized to depth shape for approximate per-pixel alignment.
- Driver auto-detects usb_type_descriptor and falls back to the
  full-resolution BGR8 + rs.align pipeline when on USB 3.

Tested: /api/{health,status,discover,rgb,depth,depth/color,capture,
intrinsics} all return 200 with valid payloads at 640x480.
This commit is contained in:
robot committed 2026-05-19 00:58:23 +02:00
commit 07d95792e3
58 files changed
+6987

No files matched your search

+94
View File
@@ -0,0 +1,94 @@
import asyncio
import logging
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from src.backend.api.routes import _frame_to_dict
from src.backend.services.image_saver import ImageSaver
try:
from src.backend.services.detector import Detector
except Exception:
Detector = None # onnxruntime is optional; YOLO features stay disabled
logger = logging.getLogger(__name__)
ws_router = APIRouter()
_camera = None
_saver: ImageSaver | None = None
_detector = None # Detector | None when onnxruntime is installed
def set_camera(camera):
global _camera
_camera = camera
def set_image_saver(saver: ImageSaver) -> None:
global _saver
_saver = saver
def set_detector(detector) -> None:
global _detector
_detector = detector
@ws_router.websocket("/ws")
async def websocket_stream(ws: WebSocket):
"""WebSocket stream of RGB+depth frames.
Client → server text messages:
'start' — begin streaming
'stop' — pause streaming
'close' — disconnect
'predict_on' — enable YOLO inference per frame (adds 'detections' to JSON)
'predict_off' — disable YOLO inference
'conf:0.55' — set YOLO confidence threshold (0.0–1.0)
"""
await ws.accept()
logger.info("WebSocket client connected")
streaming = False
predicting = False
try:
while True:
if streaming and _camera and _camera.is_open():
frame = _camera.capture(timeout_ms=500)
if frame:
data = _frame_to_dict(frame)
if predicting and _detector is not None:
data["detections"] = await asyncio.to_thread(_detector.detect, frame.rgb)
await ws.send_json(data)
if _saver and _saver.is_recording:
_saver.save_rec_frame(frame.rgb)
await asyncio.sleep(0.033)
try:
msg = await asyncio.wait_for(ws.receive_text(), timeout=0.01)
if msg == "start":
streaming = True
elif msg == "stop":
streaming = False
elif msg == "close":
break
elif msg == "predict_on":
predicting = _detector is not None
if not predicting:
logger.warning("predict_on requested but detector not loaded")
elif msg == "predict_off":
predicting = False
elif msg.startswith("conf:") and _detector is not None:
try:
_detector.set_conf(float(msg.split(":", 1)[1]))
except ValueError:
logger.warning(f"Invalid conf value: {msg!r}")
except asyncio.TimeoutError:
pass
if not streaming:
await asyncio.sleep(0.1)
except WebSocketDisconnect:
logger.info("WebSocket client disconnected")