Initial commit: RealSense D435 API server
Drop-in replacement for the Nuwa-HP60C camera container, exposing RGB +
aligned depth + intrinsics via REST/WebSocket so the existing
pestello-pose-detector pipeline can consume frames without changes.
Key adaptations for D4xx on USB 2.0 (this robot has no USB 3 port exposed
externally — see STATUS.md):
- pyrealsense2 built from source; the librealsense v2.55.1 install rule
doesn't ship the main .so, so we copy it from the build dir and add an
__init__.py that re-exports the submodule.
- Color stream uses YUYV format on USB 2.x (BGR8/RGB8 time out on this
bus); converted to BGR via cv2.cvtColor in capture().
- rs.align is skipped on USB 2 (incompatible with YUYV); color is
cv2-resized to depth shape for approximate per-pixel alignment.
- Driver auto-detects usb_type_descriptor and falls back to the
full-resolution BGR8 + rs.align pipeline when on USB 3.
Tested: /api/{health,status,discover,rgb,depth,depth/color,capture,
intrinsics} all return 200 with valid payloads at 640x480.
This commit is contained in:
robot committed
commit
07d95792e3
58 files changed
+6987
No files matched your search
@@ -0,0 +1,10 @@
|
||||
.git/
|
||||
.gitignore
|
||||
.claude/
|
||||
.omc/
|
||||
.pytest_cache/
|
||||
__pycache__/
|
||||
tests/
|
||||
docs/
|
||||
*.md
|
||||
src/libascam_bridge.so
|
||||
@@ -0,0 +1,26 @@
|
||||
# Server
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8878
|
||||
|
||||
# NATS
|
||||
NATS_URL=nats://nats:4222
|
||||
NATS_TOKEN=
|
||||
|
||||
# Service identity
|
||||
SERVICE_NAME=realsense-d435
|
||||
SERVICE_DOMAIN=vision
|
||||
|
||||
# Security — set to a non-empty string to require X-Api-Key on every REST call
|
||||
API_KEY=
|
||||
|
||||
# RealSense stream
|
||||
CAMERA_WIDTH=640
|
||||
CAMERA_HEIGHT=480
|
||||
CAMERA_FPS=30
|
||||
|
||||
# Auto-open the first discovered camera at container startup (1=on, 0=off).
|
||||
# With auto-open, the service is fully self-service: no /api/open call needed.
|
||||
CAMERA_AUTO_OPEN=1
|
||||
|
||||
# CORS — wildcard is OK for a LAN-only service. Tighten if exposed publicly.
|
||||
CORS_ORIGINS=*
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# uv
|
||||
.venv/
|
||||
|
||||
# Environment
|
||||
.env
|
||||
!.env.example
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Node
|
||||
node_modules/
|
||||
src/frontend/react/dist/
|
||||
|
||||
# Docker
|
||||
*.log
|
||||
|
||||
# Compiled bridge (rebuilt from source)
|
||||
src/backend/bridge/libascam_bridge.so
|
||||
|
||||
# Claude / OMC
|
||||
.claude/
|
||||
.omc/
|
||||
.pytest_cache/
|
||||
images/*
|
||||
models/*
|
||||
@@ -0,0 +1 @@
|
||||
3.11
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
# Multi-stage build for the RealSense D4xx API server on aarch64 (Jetson)
|
||||
# and x86_64. The Intel apt repo ships librealsense2 runtime for arm64
|
||||
# but no pyrealsense2 wheel, so the Python bindings are built from source
|
||||
# in a dedicated builder stage and copied into the runtime image.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 1: build pyrealsense2 (Python bindings) from source
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM ubuntu:22.04 AS librealsense-builder
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ARG RS_VERSION=v2.55.1
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
git cmake build-essential pkg-config ca-certificates \
|
||||
libssl-dev libusb-1.0-0-dev libudev-dev \
|
||||
python3 python3-dev python3-pip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /src
|
||||
RUN git clone --depth 1 --branch ${RS_VERSION} https://github.com/IntelRealSense/librealsense.git
|
||||
|
||||
WORKDIR /src/librealsense/build
|
||||
RUN cmake .. \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DBUILD_EXAMPLES=OFF \
|
||||
-DBUILD_GRAPHICAL_EXAMPLES=OFF \
|
||||
-DBUILD_TOOLS=OFF \
|
||||
-DBUILD_UNIT_TESTS=OFF \
|
||||
-DBUILD_NETWORK_DEVICE=OFF \
|
||||
-DBUILD_PYTHON_BINDINGS=ON \
|
||||
-DPYTHON_EXECUTABLE=/usr/bin/python3 \
|
||||
-DCMAKE_INSTALL_PREFIX=/opt/install \
|
||||
&& make -j$(nproc) \
|
||||
&& make install
|
||||
|
||||
# librealsense@v2.55.1 has a buggy install rule that doesn't install the
|
||||
# main pyrealsense2.so — only pyrsutils.so. We pick the main .so straight
|
||||
# from the build dir to work around it.
|
||||
RUN mkdir -p /opt/pyrealsense2 \
|
||||
&& cp -r /usr/lib/python3/dist-packages/pyrealsense2/. /opt/pyrealsense2/ \
|
||||
&& cp /src/librealsense/build/Release/pyrealsense2.cpython-310-aarch64-linux-gnu.so /opt/pyrealsense2/ \
|
||||
&& ls /opt/pyrealsense2/pyrealsense2.cpython-310-aarch64-linux-gnu.so > /dev/null \
|
||||
&& printf 'from .pyrealsense2 import *\n' > /opt/pyrealsense2/__init__.py \
|
||||
&& echo "=== /opt/pyrealsense2 contents ===" \
|
||||
&& ls -la /opt/pyrealsense2
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 2: build React frontend
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM node:20-alpine AS frontend-build
|
||||
WORKDIR /app
|
||||
COPY src/frontend/react/package*.json ./
|
||||
RUN npm ci
|
||||
COPY src/frontend/react/ .
|
||||
RUN npm run build
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 3: runtime image
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM ubuntu:22.04 AS runtime
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl gnupg \
|
||||
python3 python3-pip python3-venv \
|
||||
libusb-1.0-0 libudev1 \
|
||||
libgl1 libglib2.0-0 libsm6 libxext6 libxrender1 \
|
||||
usbutils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Intel RealSense apt repo for the librealsense2 native runtime (pinned to v2.55.1).
|
||||
# Intel rotated the signing key in 2025: the legacy librealsense.pgp file on
|
||||
# their server is the old 2018 key (C8B3A55A...), but the current `jammy`
|
||||
# Packages index is signed by the new key FB0B24895113F120. We fetch the new
|
||||
# key from Ubuntu's keyserver and dearmor it into the trusted keyring.
|
||||
RUN mkdir -p /etc/apt/keyrings \
|
||||
&& curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xFB0B24895113F120" \
|
||||
| gpg --dearmor -o /etc/apt/keyrings/librealsense.gpg \
|
||||
&& echo "deb [signed-by=/etc/apt/keyrings/librealsense.gpg] https://librealsense.intel.com/Debian/apt-repo jammy main" \
|
||||
> /etc/apt/sources.list.d/librealsense.list \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
librealsense2=2.55.1-0~realsense.3337 \
|
||||
librealsense2-udev-rules=2.55.1-0~realsense.3337 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Python bindings (built in stage 1). System Python 3.10 looks in
|
||||
# /usr/lib/python3/dist-packages, so install there.
|
||||
COPY --from=librealsense-builder /opt/pyrealsense2 /usr/lib/python3/dist-packages/pyrealsense2
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Python dependencies — keep this layer cacheable: install before copying source.
|
||||
RUN pip3 install --no-cache-dir \
|
||||
"fastapi>=0.115" \
|
||||
"uvicorn[standard]>=0.34" \
|
||||
"pydantic>=2.0" \
|
||||
"pydantic-settings>=2.0" \
|
||||
"python-dotenv>=1.0" \
|
||||
"nats-py>=2.9" \
|
||||
"numpy>=1.24,<2" \
|
||||
"opencv-python-headless>=4.8"
|
||||
|
||||
# Application source
|
||||
COPY src/backend/ src/backend/
|
||||
COPY --from=frontend-build /app/dist src/frontend/react/dist/
|
||||
|
||||
# Entrypoint
|
||||
COPY scripts/entrypoint.sh /app/entrypoint.sh
|
||||
RUN chmod +x /app/entrypoint.sh
|
||||
|
||||
# Optional .env (compose can also inject via env_file)
|
||||
COPY .env* ./
|
||||
|
||||
EXPOSE 8878
|
||||
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
@@ -0,0 +1,168 @@
|
||||
# Intel RealSense D435 — API Server
|
||||
|
||||
Server FastAPI per la depth camera Intel RealSense D4xx (testato su D435)
|
||||
con accesso remoto via REST/WebSocket. Esecuzione tramite Docker. Mantiene
|
||||
la stessa shape API del servizio Nuwa HP60C (envelope `ApiResponse`,
|
||||
endpoint `/api/...`, header `X-Api-Key`, WebSocket `/ws`) così che i client
|
||||
e il frontend React esistenti funzionano senza modifiche.
|
||||
|
||||
## Caratteristiche
|
||||
|
||||
- **RGB**: configurabile (default 640x480) BGR8, allineato al depth
|
||||
- **Depth**: 640x480 uint16 in millimetri (allineato al color via `rs.align`)
|
||||
- **Intrinsics**: parametri intrinseci color + depth e estrinseci depth→color
|
||||
- **Discovery**: enumerazione devices via `pyrealsense2`
|
||||
- **API Server**: REST + WebSocket con envelope `ApiResponse` standard
|
||||
- **Sicurezza**: API Key opzionale (`X-Api-Key`) su tutti gli endpoint REST
|
||||
- **NATS**: messaging asincrono opzionale (profilo `nats`)
|
||||
- **Frontend React**: viewer RGB, depth e 3D point cloud (riusato dal Nuwa)
|
||||
- **Swagger**: documentazione interattiva API a `/apidocs`
|
||||
|
||||
## Hardware
|
||||
|
||||
| Parametro | Valore |
|
||||
|-----------|--------|
|
||||
| Camera | Intel RealSense D435 |
|
||||
| Serial number (questo robot) | `933322070696` |
|
||||
| Firmware | `05.16.00.01` |
|
||||
| Range depth | 0.1m – 10m |
|
||||
| Interfaccia | USB 3.0 |
|
||||
| SDK | librealsense 2.55.1 + pyrealsense2 (built from source) |
|
||||
| Host | Jetson Xavier NX, Ubuntu 20.04 (JetPack 5.1.x) |
|
||||
|
||||
## Configurazione
|
||||
|
||||
Crea `.env` (vedi `.env.example`):
|
||||
|
||||
```env
|
||||
API_PORT=8878
|
||||
API_KEY=cambia-questa-chiave-segreta
|
||||
CAMERA_WIDTH=640
|
||||
CAMERA_HEIGHT=480
|
||||
CAMERA_FPS=30
|
||||
CAMERA_AUTO_OPEN=1 # apre la camera al boot, niente /api/open manuale
|
||||
CORS_ORIGINS=* # LAN-only OK
|
||||
```
|
||||
|
||||
## Accesso dalla LAN
|
||||
|
||||
Il container gira sul robot e binda la porta `8878` su tutte le interfacce
|
||||
(`0.0.0.0`). Dal vostro PC sulla stessa LAN raggiungete il servizio via:
|
||||
|
||||
```
|
||||
http://<ip-robot>:8878/api/health # health check
|
||||
http://<ip-robot>:8878/apidocs # Swagger UI
|
||||
http://<ip-robot>:8878/gui # frontend React
|
||||
ws://<ip-robot>:8878/ws # stream WebSocket
|
||||
```
|
||||
|
||||
Il robot ha Avahi attivo, quindi `ubuntu.local:8878` funziona se mDNS
|
||||
risolve sul vostro PC (`avahi-resolve -n ubuntu.local`). Altrimenti
|
||||
usate l'IP diretto (`ip a` sul robot per scoprirlo).
|
||||
|
||||
> Lo streaming **non richiede internet** — solo la build (~25 min al primo
|
||||
> avvio) deve raggiungere registry Docker/apt/PyPI/npm via il tinyproxy
|
||||
> locale del robot.
|
||||
|
||||
## Build & avvio
|
||||
|
||||
```bash
|
||||
# Backend (porta 8878)
|
||||
docker compose up --build -d
|
||||
|
||||
# Backend + NATS
|
||||
docker compose --profile nats up --build -d
|
||||
```
|
||||
|
||||
> Il primo build compila i bindings Python di `pyrealsense2` da source —
|
||||
> ~20–30 min su Jetson. I build successivi sono cached.
|
||||
|
||||
Il container:
|
||||
- non fa unbind UVC (a differenza del Nuwa) — RealSense usa direttamente uvcvideo
|
||||
- monta `/dev/bus/usb` in privileged mode per accesso diretto USB
|
||||
- riparte automaticamente (`restart: unless-stopped`)
|
||||
|
||||
## Endpoints
|
||||
|
||||
Identici al Nuwa per drop-in compatibility:
|
||||
|
||||
| Endpoint | Metodo | Descrizione |
|
||||
|----------|--------|-------------|
|
||||
| `/api/health` | GET | Health check (no auth) |
|
||||
| `/api/status` | GET | Stato SDK/camera e calibrazione |
|
||||
| `/api/discover` | GET | Rileva camere RealSense su USB |
|
||||
| `/api/open` | POST | Apre camera per indice (`device_index`) |
|
||||
| `/api/close` | POST | Chiude la camera |
|
||||
| `/api/capture` | GET | Frame RGB+depth (JSON, base64) |
|
||||
| `/api/rgb` | GET | Frame RGB come JPEG |
|
||||
| `/api/depth` | GET | Frame depth come PNG 16-bit (mm) |
|
||||
| `/api/depth/color` | GET | Depth colormap JET come JPEG |
|
||||
| `/api/intrinsics` | GET | Calibrazione completa |
|
||||
| `/api/pointcloud` | GET | Point cloud (subsampling `step`) |
|
||||
| `/api/pointcloud/roi` | GET | Point cloud su ROI pixel-space |
|
||||
| `/api/measure` | GET | Backprojection 3D di un pixel |
|
||||
| `/api/snapshot` | POST | Salva snapshot RGB |
|
||||
| `/api/recording/{start,stop,status}` | POST/GET | Registrazione continua |
|
||||
| `/ws` | WS | Streaming WebSocket (text msgs: `start`, `stop`, `close`) |
|
||||
|
||||
#### Payload calibrazione
|
||||
|
||||
`/api/intrinsics` ritorna i campi `ir` e `rgb` per compatibilità con il
|
||||
frontend Nuwa, ma in questo servizio:
|
||||
|
||||
- `ir` = intrinseci del **sensore depth** (post-rectification dello stereo IR)
|
||||
- `rgb` = intrinseci del **sensore color**
|
||||
- `extrinsics_ir_to_rgb` = trasformazione **depth → color**
|
||||
|
||||
```json
|
||||
{
|
||||
"ir": { "fx": ..., "fy": ..., "cx": ..., "cy": ...,
|
||||
"distortion": { "k1":..., "k2":..., "k3":..., "p1":..., "p2":... } },
|
||||
"rgb": { "fx": ..., "fy": ..., "cx": ..., "cy": ...,
|
||||
"distortion": { ... } },
|
||||
"extrinsics_ir_to_rgb": {
|
||||
"rotation": [r00, r01, r02, r10, r11, r12, r20, r21, r22],
|
||||
"translation": [tx, ty, tz]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- Rotation row-major 3x3
|
||||
- Translation in **millimetri** (la D4xx ritorna `extrinsics.translation` in
|
||||
metri; convertito qui per allinearsi alla convenzione Nuwa)
|
||||
- D4xx pubblica tipicamente coefficienti di distorsione tutti zero (immagini
|
||||
già rettificate dall'ASIC)
|
||||
|
||||
### Esempi curl
|
||||
|
||||
```bash
|
||||
# Discover
|
||||
curl -H "X-Api-Key: $API_KEY" http://localhost:8878/api/discover
|
||||
|
||||
# Apri (D435 in genere index 0)
|
||||
curl -X POST -H "X-Api-Key: $API_KEY" \
|
||||
"http://localhost:8878/api/open?device_index=0"
|
||||
|
||||
# Frame RGB
|
||||
curl -H "X-Api-Key: $API_KEY" http://localhost:8878/api/rgb -o rgb.jpg
|
||||
|
||||
# Frame depth (PNG 16-bit, valori in mm)
|
||||
curl -H "X-Api-Key: $API_KEY" http://localhost:8878/api/depth -o depth.png
|
||||
|
||||
# Intrinsics
|
||||
curl -H "X-Api-Key: $API_KEY" http://localhost:8878/api/intrinsics | jq
|
||||
|
||||
# Close
|
||||
curl -X POST -H "X-Api-Key: $API_KEY" http://localhost:8878/api/close
|
||||
```
|
||||
|
||||
## Note
|
||||
|
||||
- `librealsense2` runtime nel container è installato dal repo apt ufficiale
|
||||
Intel (pinned a `2.55.1-0~realsense.3337`). I bindings Python sono
|
||||
buildati da `IntelRealSense/librealsense@v2.55.1`.
|
||||
- Il frontend React originale (`/gui`) referenzia `ir` e `rgb` come label di
|
||||
display — su questo servizio quei nomi mappano depth/color (vedi sopra).
|
||||
- L'autostart è gestito da Docker (`restart: unless-stopped`). Per partire
|
||||
automaticamente al boot del robot serve `systemctl enable docker`.
|
||||
- Per disabilitare l'auth API setta `API_KEY=` (vuoto). Sconsigliato in produzione.
|
||||
@@ -0,0 +1,79 @@
|
||||
# Stato del progetto RealSense-D435 — 2026-05-19
|
||||
|
||||
## TL;DR
|
||||
|
||||
- ✅ Driver completo e funzionante su questo robot (Jetson Xavier NX, D435 su USB 2.1)
|
||||
- ✅ RGB 640x480 + Depth 640x480 simultanei via REST/WebSocket
|
||||
- ✅ Intrinsics, point cloud, measure, snapshot, recording, WebSocket stream — tutti operativi
|
||||
- ⚠️ Framerate effettivo ~4 fps (limitato dalla banda USB 2.0). Su porta USB 3 il driver riprende automaticamente 640x480@30.
|
||||
|
||||
## Cosa è stato risolto in questa sessione
|
||||
|
||||
1. **Bug install di librealsense v2.55.1**: il `.so` principale di `pyrealsense2` non viene installato dal CMake — solo `pyrsutils.so`. Workaround nel `Dockerfile`: si copia il `.so` direttamente da `/src/librealsense/build/Release/` e si aggiunge un `__init__.py` che riesporta il submodule (`from .pyrealsense2 import *`).
|
||||
2. **D4xx su USB 2 — formato color**: BGR8/RGB8 vanno in timeout su USB 2 anche da soli (firmware D4xx). Solo **YUYV** sostiene il transfer su USB 2. Il driver ora rileva `usb_type_descriptor` e su USB 2.x configura color = YUYV, convertendo in BGR via `cv2.cvtColor` in `capture()`.
|
||||
3. **`rs.align` incompatibile con YUYV**: il blocco di processing fallisce con "Error occurred during execution of the processing block". Su USB 2 saltiamo l'alignment hardware e facciamo un resize numpy del color per matchare la shape del depth (approximate alignment, OK per use case standard).
|
||||
4. **Risoluzione massima USB 2**: 640x480@15 (configurato) → ~4 fps reali. Testate tutte le combinazioni; 848x480 dà "Couldn't resolve requests" anche in YUYV.
|
||||
|
||||
## Hardware setup (constatato)
|
||||
|
||||
```
|
||||
Jetson Xavier NX
|
||||
├─ Bus 1 (USB 2.0, 480M)
|
||||
│ ├─ Hub interno → Inspire Hands (cdc_acm /dev/ttyACM0) + ch341
|
||||
│ └─ Hub esterno (collegato a porta USB-C frontale)
|
||||
│ ├─ Tastiera, mouse, ax88179 (Ethernet)
|
||||
│ └─ D435 ← qui
|
||||
└─ Bus 2 (USB 3.0, 10000M) — VUOTO, ma probabilmente non esposto esternamente
|
||||
```
|
||||
|
||||
- D435 serial: `933322070696`, fw `05.16.00.01`, USB `8086:0b07`, `usb_type = 2.1`
|
||||
- La porta USB-C frontale del robot H1 è cablata **USB 2 hardware** (confermato indirettamente: l'ax88179 sullo stesso hub gira a 480M)
|
||||
- Le Inspire Hands sono su hub interno separato — non vanno toccate
|
||||
|
||||
## Endpoints (verificati)
|
||||
|
||||
| Endpoint | HTTP | Note |
|
||||
|---|---|---|
|
||||
| `/api/health` | 200 | sdk_initialized=true, camera_open=true |
|
||||
| `/api/status` | 200 | include `color_available`, `usb_type` |
|
||||
| `/api/discover` | 200 | rileva D435 serial `933322070696` |
|
||||
| `/api/rgb` | 200 | JPEG 640x480 |
|
||||
| `/api/depth` | 200 | PNG 16-bit 640x480 |
|
||||
| `/api/depth/color` | 200 | JPEG colormap JET 640x480 |
|
||||
| `/api/capture` | 200 | JSON con tutte e tre base64 |
|
||||
| `/api/intrinsics` | 200 | ir + rgb + extrinsics_ir_to_rgb |
|
||||
| `/api/pointcloud`, `/api/pointcloud/roi`, `/api/measure`, `/api/snapshot` | 200 | da testare in scenario reale |
|
||||
|
||||
## File modificati in questa sessione
|
||||
|
||||
- `Dockerfile`: fix install pyrealsense2 (`.so` + `__init__.py`)
|
||||
- `src/backend/services/camera.py`:
|
||||
- rileva `usb_type_descriptor` in `open()`
|
||||
- profilo USB 2 (YUYV 640x480@15 color + Z16 640x480@15 depth)
|
||||
- conversione YUYV → BGR in `capture()` via `cv2.cvtColor` (manuale, dopo aver ricostruito lo shape (H,W,2))
|
||||
- skip `rs.align` su USB 2; resize numpy del color per matchare il depth
|
||||
- nuovi metodi `is_color_available()`, `usb_type()` per diagnostica
|
||||
- `src/backend/api/routes.py`: `/api/status` espone `color_available` + `usb_type`
|
||||
|
||||
## Da fare prossima volta
|
||||
|
||||
- Testare `/api/pointcloud` e `/api/measure` in un ambiente reale e verificare che l'approximate alignment (resize) sia accettabile per il caso d'uso. Se no, implementare alignment proprio con `cv2.remap` usando intrinsics + extrinsics.
|
||||
- Decidere se esporre il flag `usb_type` nel frontend React per warning utente
|
||||
- Eventualmente: profilo configurabile via `.env` per chi vuole controllare manualmente
|
||||
|
||||
## Modifiche al setup ambiente (persistono nei reboot)
|
||||
|
||||
1. `/etc/systemd/system/docker.service.d/http-proxy.conf` (proxy daemon)
|
||||
2. `unitree` nel gruppo `docker`
|
||||
3. `~/.docker/config.json` con proxy CLI + cli-plugins path
|
||||
4. `~/.docker/cli-plugins/docker-compose` (plugin v2)
|
||||
|
||||
## Avvio rapido
|
||||
|
||||
```bash
|
||||
cd ~/Robot_2026/RealSense-D435
|
||||
sg docker -c "docker compose up -d" # già buildato
|
||||
curl -s http://localhost:8878/api/health # check
|
||||
curl -s http://localhost:8878/api/rgb -o rgb.jpg
|
||||
curl -s http://localhost:8878/api/depth -o depth.png
|
||||
```
|
||||
@@ -0,0 +1,59 @@
|
||||
services:
|
||||
|
||||
# -- NATS Server ----------------------------
|
||||
nats:
|
||||
image: nats:latest
|
||||
profiles: ["nats"]
|
||||
ports:
|
||||
- "4222:4222"
|
||||
- "8222:8222"
|
||||
command: >
|
||||
--auth ${NATS_TOKEN:-realsense-dev-token}
|
||||
--http_port 8222
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "nats-server", "--help"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
|
||||
# -- Backend --------------------------------
|
||||
backend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
# network: host so that build-time apt/git/pip can reach the
|
||||
# tinyproxy running on the host at localhost:8888.
|
||||
network: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "${API_PORT:-8878}:8878"
|
||||
devices:
|
||||
- "/dev/bus/usb:/dev/bus/usb"
|
||||
volumes:
|
||||
- ./images:/app/images
|
||||
- ./models:/app/models:ro
|
||||
env_file: .env
|
||||
depends_on:
|
||||
nats:
|
||||
condition: service_healthy
|
||||
required: false
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8878/api/health')"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# -- Frontend React Dev (hot reload) --------
|
||||
frontend-dev:
|
||||
image: node:20-alpine
|
||||
working_dir: /app
|
||||
volumes:
|
||||
- ./src/frontend/react:/app
|
||||
- /app/node_modules
|
||||
command: sh -c "npm ci && npm run dev -- --host 0.0.0.0"
|
||||
ports:
|
||||
- "5173:5173"
|
||||
profiles: [dev]
|
||||
@@ -0,0 +1,29 @@
|
||||
[project]
|
||||
name = "realsense-d435"
|
||||
version = "1.0.0"
|
||||
description = "REST API + WebSocket server for Intel RealSense D4xx depth camera (pyrealsense2)"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"fastapi>=0.115",
|
||||
"uvicorn[standard]>=0.34",
|
||||
"pydantic>=2.0",
|
||||
"pydantic-settings>=2.0",
|
||||
"python-dotenv>=1.0",
|
||||
"nats-py>=2.9",
|
||||
"numpy>=1.24,<2",
|
||||
"opencv-python-headless>=4.8.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
"pytest-asyncio>=0.24",
|
||||
"httpx>=0.27",
|
||||
]
|
||||
yolo = [
|
||||
"onnxruntime>=1.16",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
start = "uvicorn src.backend.main:app --host 0.0.0.0 --port 8878"
|
||||
dev = "uvicorn src.backend.main:app --reload --host 0.0.0.0 --port 8878"
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
# RealSense D4xx uses uvcvideo via the librealsense2 backend, so no driver
|
||||
# unbind is needed — unlike the Nuwa HP60C entrypoint.
|
||||
#
|
||||
# librealsense expects the udev rules to be present on the host. The rules
|
||||
# package is installed inside the image too (librealsense2-udev-rules) but
|
||||
# udev itself runs on the host; if the camera is not visible to lsusb the
|
||||
# host udev rules are likely missing or stale.
|
||||
|
||||
set -e
|
||||
|
||||
PORT=${API_PORT:-8878}
|
||||
|
||||
echo "[entrypoint] Starting RealSense D4xx API server on port ${PORT}..."
|
||||
exec python3 -m uvicorn src.backend.main:app --host 0.0.0.0 --port "${PORT}"
|
||||
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,9 @@
|
||||
from fastapi import Header, HTTPException
|
||||
from src.backend.config import settings
|
||||
|
||||
|
||||
async def verify_api_key(x_api_key: str | None = Header(None)):
|
||||
if not settings.api_key:
|
||||
return
|
||||
if x_api_key != settings.api_key:
|
||||
raise HTTPException(status_code=403, detail="API key non valida")
|
||||
@@ -0,0 +1,537 @@
|
||||
"""REST API routes for the RealSense D435 camera server."""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import Response, JSONResponse
|
||||
|
||||
from src.backend.api.dependencies import verify_api_key
|
||||
from src.backend.models.api import ok, fail
|
||||
from src.backend.services.camera import RealSenseD435, CameraFrame
|
||||
from src.backend.services.image_saver import ImageSaver
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(dependencies=[Depends(verify_api_key)])
|
||||
|
||||
_camera: RealSenseD435 | None = None
|
||||
_saver: ImageSaver | None = None
|
||||
_frame_count = 0
|
||||
|
||||
|
||||
def set_camera(camera: RealSenseD435) -> None:
|
||||
global _camera
|
||||
_camera = camera
|
||||
|
||||
|
||||
def set_image_saver(saver: ImageSaver) -> None:
|
||||
global _saver
|
||||
_saver = saver
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Image encoding helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _encode_rgb_jpeg(rgb: np.ndarray, quality: int = 85) -> bytes:
|
||||
_, buf = cv2.imencode(".jpg", rgb, [cv2.IMWRITE_JPEG_QUALITY, quality])
|
||||
return buf.tobytes()
|
||||
|
||||
|
||||
def _encode_depth_png(depth: np.ndarray) -> bytes:
|
||||
_, buf = cv2.imencode(".png", depth)
|
||||
return buf.tobytes()
|
||||
|
||||
|
||||
def _depth_colormap(depth: np.ndarray) -> np.ndarray:
|
||||
valid = depth[depth > 0]
|
||||
if len(valid) == 0:
|
||||
return np.zeros((*depth.shape, 3), dtype=np.uint8)
|
||||
p5, p95 = np.percentile(valid, [5, 95])
|
||||
vis = np.zeros_like(depth, dtype=np.uint8)
|
||||
mask = depth > 0
|
||||
vis[mask] = np.clip(
|
||||
(depth[mask].astype(float) - p5) / max(p95 - p5, 1) * 255, 0, 255
|
||||
).astype(np.uint8)
|
||||
return cv2.applyColorMap(vis, cv2.COLORMAP_JET)
|
||||
|
||||
|
||||
def _frame_to_dict(frame: CameraFrame, include_images: bool = True) -> dict:
|
||||
global _frame_count
|
||||
_frame_count += 1
|
||||
|
||||
result = {
|
||||
"frame_id": _frame_count,
|
||||
"timestamp": frame.timestamp,
|
||||
"rgb_shape": list(frame.rgb.shape),
|
||||
"depth_shape": list(frame.depth.shape),
|
||||
"depth_min_mm": int(frame.depth[frame.depth > 0].min()) if (frame.depth > 0).any() else 0,
|
||||
"depth_max_mm": int(frame.depth.max()),
|
||||
"depth_valid_pixels": int((frame.depth > 0).sum()),
|
||||
}
|
||||
|
||||
if include_images:
|
||||
result["rgb_jpeg_b64"] = base64.b64encode(_encode_rgb_jpeg(frame.rgb)).decode()
|
||||
result["depth_png_b64"] = base64.b64encode(_encode_depth_png(frame.depth)).decode()
|
||||
colormap = _depth_colormap(frame.depth)
|
||||
result["depth_color_b64"] = base64.b64encode(_encode_rgb_jpeg(colormap, 90)).decode()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# /health is mounted as public (no API key) in main.py
|
||||
|
||||
|
||||
def _calibration_payload() -> dict | None:
|
||||
"""Full calibration (intrinsics + distortion + extrinsics) or None."""
|
||||
if not _camera or not _camera.intrinsics:
|
||||
return None
|
||||
i = _camera.intrinsics
|
||||
e = _camera.extrinsics
|
||||
dir_ = _camera.distortion_ir
|
||||
drgb = _camera.distortion_rgb
|
||||
|
||||
def _d(d):
|
||||
return None if d is None else {"k1": d.k1, "k2": d.k2, "k3": d.k3, "p1": d.p1, "p2": d.p2}
|
||||
|
||||
return {
|
||||
"ir": {
|
||||
"fx": i.fx_ir, "fy": i.fy_ir, "cx": i.cx_ir, "cy": i.cy_ir,
|
||||
"distortion": _d(dir_),
|
||||
},
|
||||
"rgb": {
|
||||
"fx": i.fx_rgb, "fy": i.fy_rgb, "cx": i.cx_rgb, "cy": i.cy_rgb,
|
||||
"distortion": _d(drgb),
|
||||
},
|
||||
"extrinsics_ir_to_rgb": None if e is None else {
|
||||
"rotation": list(e.rotation),
|
||||
"translation": list(e.translation),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def status():
|
||||
"""Camera status and intrinsics."""
|
||||
return ok({
|
||||
"sdk_initialized": _camera.is_initialized() if _camera else False,
|
||||
"camera_open": _camera.is_open() if _camera else False,
|
||||
"color_available": _camera.is_color_available() if (_camera and _camera.is_open()) else False,
|
||||
"usb_type": _camera.usb_type() if (_camera and _camera.is_open()) else None,
|
||||
"frame_count": _frame_count,
|
||||
"resolution": {"width": 640, "height": 480},
|
||||
"intrinsics": _calibration_payload(),
|
||||
})
|
||||
|
||||
|
||||
@router.get("/discover")
|
||||
async def discover(timeout_ms: int = Query(default=5000, ge=1000, le=30000)):
|
||||
"""Discover available RealSense cameras on USB bus."""
|
||||
if not _camera or not _camera.is_initialized():
|
||||
return fail("SDK_NOT_INITIALIZED", "SDK not initialized")
|
||||
|
||||
devices = _camera.discover(timeout_ms=timeout_ms)
|
||||
return ok({
|
||||
"cameras": [
|
||||
{
|
||||
"index": d.index,
|
||||
"vid": f"{d.vid:04x}",
|
||||
"pid": f"{d.pid:04x}",
|
||||
"serial": d.serial,
|
||||
"bus": d.bus,
|
||||
"device": d.device,
|
||||
}
|
||||
for d in devices
|
||||
],
|
||||
"count": len(devices),
|
||||
})
|
||||
|
||||
|
||||
@router.post("/open")
|
||||
async def open_camera(device_index: int = Query(default=0, ge=0)):
|
||||
"""Open a discovered camera by index and start streaming.
|
||||
|
||||
Idempotent: if a camera is already open, returns the current open state
|
||||
instead of failing — the server only supports one open camera at a time.
|
||||
"""
|
||||
if not _camera or not _camera.is_initialized():
|
||||
return fail("SDK_NOT_INITIALIZED", "SDK not initialized")
|
||||
|
||||
if _camera.is_open():
|
||||
return ok({
|
||||
"status": "open",
|
||||
"device_index": _camera.device_index,
|
||||
"intrinsics": _calibration_payload(),
|
||||
"already_open": True,
|
||||
})
|
||||
|
||||
if _camera.open(device_index):
|
||||
return ok({
|
||||
"status": "open",
|
||||
"device_index": device_index,
|
||||
"intrinsics": _calibration_payload(),
|
||||
"already_open": False,
|
||||
})
|
||||
else:
|
||||
return fail("OPEN_FAILED", "Failed to open camera")
|
||||
|
||||
|
||||
@router.post("/close")
|
||||
async def close_camera():
|
||||
"""Close the active camera and stop streaming."""
|
||||
if not _camera or not _camera.is_open():
|
||||
return fail("CAMERA_NOT_OPEN", "No camera is open")
|
||||
|
||||
_camera.close()
|
||||
return ok({"status": "closed"})
|
||||
|
||||
|
||||
@router.get("/capture")
|
||||
async def capture():
|
||||
"""Capture a single RGB+depth frame. Returns JSON with base64-encoded images."""
|
||||
if not _camera or not _camera.is_open():
|
||||
return fail("CAMERA_NOT_OPEN", "Camera not open")
|
||||
|
||||
frame = _camera.capture()
|
||||
if frame is None:
|
||||
return fail("CAPTURE_TIMEOUT", "Frame capture timeout")
|
||||
|
||||
return JSONResponse(ok(_frame_to_dict(frame)))
|
||||
|
||||
|
||||
@router.get("/rgb")
|
||||
async def get_rgb():
|
||||
"""Get RGB frame as JPEG image."""
|
||||
if not _camera or not _camera.is_open():
|
||||
raise HTTPException(503, "Camera not open")
|
||||
|
||||
rgb = _camera.get_rgb()
|
||||
if rgb is None:
|
||||
raise HTTPException(504, "No RGB frame")
|
||||
|
||||
return Response(content=_encode_rgb_jpeg(rgb), media_type="image/jpeg")
|
||||
|
||||
|
||||
@router.get("/depth")
|
||||
async def get_depth():
|
||||
"""Get depth frame as 16-bit PNG (values in mm)."""
|
||||
if not _camera or not _camera.is_open():
|
||||
raise HTTPException(503, "Camera not open")
|
||||
|
||||
depth = _camera.get_depth()
|
||||
if depth is None:
|
||||
raise HTTPException(504, "No depth frame")
|
||||
|
||||
return Response(content=_encode_depth_png(depth), media_type="image/png")
|
||||
|
||||
|
||||
@router.get("/depth/color")
|
||||
async def get_depth_colormap():
|
||||
"""Get depth frame as JET colormap JPEG."""
|
||||
if not _camera or not _camera.is_open():
|
||||
raise HTTPException(503, "Camera not open")
|
||||
|
||||
depth = _camera.get_depth()
|
||||
if depth is None:
|
||||
raise HTTPException(504, "No depth frame")
|
||||
|
||||
colormap = _depth_colormap(depth)
|
||||
return Response(content=_encode_rgb_jpeg(colormap, 90), media_type="image/jpeg")
|
||||
|
||||
|
||||
@router.get("/pointcloud", summary="3D Point cloud", tags=["capture"])
|
||||
async def get_pointcloud(
|
||||
step: int = Query(default=4, ge=1, le=16),
|
||||
max_depth_mm: int = Query(default=4000, ge=100, le=10000),
|
||||
):
|
||||
"""Get 3D point cloud as JSON array of [x,y,z,r,g,b] points.
|
||||
|
||||
Points are computed server-side using depth + RGB + intrinsics.
|
||||
step: subsampling factor (4 = every 4th pixel, faster)
|
||||
max_depth_mm: discard points beyond this depth
|
||||
"""
|
||||
if not _camera or not _camera.is_open():
|
||||
return fail("NO_CAMERA", "Camera not open")
|
||||
|
||||
frame = _camera.capture()
|
||||
if frame is None:
|
||||
return fail("TIMEOUT", "Frame capture timeout")
|
||||
|
||||
depth = frame.depth
|
||||
rgb = frame.rgb # BGR format
|
||||
|
||||
# Use intrinsics if available, otherwise use typical HP60C defaults
|
||||
if _camera.intrinsics:
|
||||
fx = _camera.intrinsics.fx_ir
|
||||
fy = _camera.intrinsics.fy_ir
|
||||
cx = _camera.intrinsics.cx_ir
|
||||
cy = _camera.intrinsics.cy_ir
|
||||
else:
|
||||
# Default intrinsics for 640x480 HP60C
|
||||
fx, fy = 445.0, 445.0
|
||||
cx, cy = 320.0, 240.0
|
||||
|
||||
h, w = depth.shape
|
||||
points = []
|
||||
|
||||
for y in range(0, h, step):
|
||||
for x in range(0, w, step):
|
||||
z_val = float(depth[y, x])
|
||||
if z_val <= 0 or z_val > max_depth_mm:
|
||||
continue
|
||||
z_m = z_val / 1000.0
|
||||
x_m = (x - cx) * z_m / fx
|
||||
y_m = (y - cy) * z_m / fy
|
||||
b_val, g_val, r_val = rgb[y, x] # BGR to RGB
|
||||
points.append([
|
||||
round(x_m, 4), round(-y_m, 4), round(-z_m, 4),
|
||||
int(r_val), int(g_val), int(b_val)
|
||||
])
|
||||
|
||||
return ok({
|
||||
"points": points,
|
||||
"count": len(points),
|
||||
"step": step,
|
||||
"intrinsics": {
|
||||
"fx": fx, "fy": fy, "cx": cx, "cy": cy
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@router.get("/pointcloud/roi", summary="3D Point cloud within ROI", tags=["capture"])
|
||||
async def get_pointcloud_roi(
|
||||
u_min: int = Query(..., ge=0, description="ROI left pixel (inclusive)"),
|
||||
v_min: int = Query(..., ge=0, description="ROI top pixel (inclusive)"),
|
||||
u_max: int = Query(..., ge=1, description="ROI right pixel (exclusive)"),
|
||||
v_max: int = Query(..., ge=1, description="ROI bottom pixel (exclusive)"),
|
||||
step: int = Query(default=1, ge=1, le=16),
|
||||
max_depth_mm: int = Query(default=4000, ge=100, le=10000),
|
||||
):
|
||||
"""Get 3D point cloud restricted to a pixel-space ROI.
|
||||
|
||||
Coordinates are RAW image pixels (no flip applied) in the depth/RGB
|
||||
frame (typically 640x480). Points are returned as [x, -y, -z, r, g, b]
|
||||
— same convention as /pointcloud so the frontend renderer is compatible.
|
||||
"""
|
||||
if not _camera or not _camera.is_open():
|
||||
return fail("NO_CAMERA", "Camera not open")
|
||||
|
||||
frame = _camera.capture()
|
||||
if frame is None:
|
||||
return fail("TIMEOUT", "Frame capture timeout")
|
||||
|
||||
depth = frame.depth
|
||||
rgb = frame.rgb # BGR format
|
||||
h, w = depth.shape
|
||||
|
||||
u0 = max(0, min(w, u_min))
|
||||
v0 = max(0, min(h, v_min))
|
||||
u1 = max(0, min(w, u_max))
|
||||
v1 = max(0, min(h, v_max))
|
||||
if u1 <= u0 or v1 <= v0:
|
||||
return fail("INVALID_ROI", f"Empty ROI ({u0},{v0})-({u1},{v1}) in {w}x{h}")
|
||||
|
||||
if _camera.intrinsics:
|
||||
fx = _camera.intrinsics.fx_ir
|
||||
fy = _camera.intrinsics.fy_ir
|
||||
cx = _camera.intrinsics.cx_ir
|
||||
cy = _camera.intrinsics.cy_ir
|
||||
else:
|
||||
fx, fy = 445.0, 445.0
|
||||
cx, cy = 320.0, 240.0
|
||||
|
||||
# Vectorized: restrict both arrays to the ROI and filter by depth mask.
|
||||
roi_depth = depth[v0:v1:step, u0:u1:step].astype(np.float32)
|
||||
roi_rgb = rgb[v0:v1:step, u0:u1:step]
|
||||
mask = (roi_depth > 0) & (roi_depth <= max_depth_mm)
|
||||
|
||||
if not mask.any():
|
||||
return ok({
|
||||
"points": [],
|
||||
"count": 0,
|
||||
"step": step,
|
||||
"roi": {"u_min": u0, "v_min": v0, "u_max": u1, "v_max": v1},
|
||||
"intrinsics": {"fx": fx, "fy": fy, "cx": cx, "cy": cy},
|
||||
})
|
||||
|
||||
ys_local, xs_local = np.where(mask)
|
||||
zs_mm = roi_depth[ys_local, xs_local]
|
||||
# Absolute pixel coordinates within the full image (needed for fx/cx back-projection).
|
||||
xs_abs = u0 + xs_local * step
|
||||
ys_abs = v0 + ys_local * step
|
||||
zs_m = zs_mm / 1000.0
|
||||
xs_m = (xs_abs - cx) * zs_m / fx
|
||||
ys_m = (ys_abs - cy) * zs_m / fy
|
||||
bgr = roi_rgb[ys_local, xs_local] # (N, 3) BGR
|
||||
|
||||
# Match /pointcloud convention: [x, -y, -z, r, g, b]
|
||||
points = np.stack([
|
||||
np.round(xs_m, 4),
|
||||
np.round(-ys_m, 4),
|
||||
np.round(-zs_m, 4),
|
||||
bgr[:, 2].astype(np.float32),
|
||||
bgr[:, 1].astype(np.float32),
|
||||
bgr[:, 0].astype(np.float32),
|
||||
], axis=1).tolist()
|
||||
|
||||
return ok({
|
||||
"points": points,
|
||||
"count": len(points),
|
||||
"step": step,
|
||||
"roi": {"u_min": u0, "v_min": v0, "u_max": u1, "v_max": v1},
|
||||
"intrinsics": {"fx": fx, "fy": fy, "cx": cx, "cy": cy},
|
||||
})
|
||||
|
||||
|
||||
@router.get("/measure", summary="3D measurement at pixel", tags=["capture"])
|
||||
async def measure_point(
|
||||
u: int = Query(..., ge=0, description="Pixel X in raw image coords (0..W-1)"),
|
||||
v: int = Query(..., ge=0, description="Pixel Y in raw image coords (0..H-1)"),
|
||||
radius: int = Query(default=6, ge=0, le=50, description="Search radius (px) for nearest valid depth"),
|
||||
):
|
||||
"""Find nearest valid-depth pixel around (u, v) and back-project to 3D.
|
||||
|
||||
Useful for testing RGB/depth alignment: click on RGB at a pixel, get the
|
||||
closest point-cloud point within `radius`. Coordinates are RAW image pixels
|
||||
(no flip applied) in the depth/RGB frame (typically 640x480).
|
||||
"""
|
||||
if not _camera or not _camera.is_open():
|
||||
return fail("CAMERA_NOT_OPEN", "Camera not open")
|
||||
|
||||
frame = _camera.capture()
|
||||
if frame is None:
|
||||
return fail("CAPTURE_TIMEOUT", "Frame capture timeout")
|
||||
|
||||
depth = frame.depth
|
||||
h, w = depth.shape
|
||||
if u >= w or v >= h:
|
||||
return fail("OUT_OF_RANGE", f"Pixel ({u},{v}) outside image ({w}x{h})")
|
||||
|
||||
u0 = max(0, u - radius)
|
||||
u1 = min(w, u + radius + 1)
|
||||
v0 = max(0, v - radius)
|
||||
v1 = min(h, v + radius + 1)
|
||||
window = depth[v0:v1, u0:u1]
|
||||
ys, xs = np.where(window > 0)
|
||||
if len(xs) == 0:
|
||||
return fail("NO_DEPTH", f"No valid depth within radius {radius}")
|
||||
|
||||
abs_xs = xs + u0
|
||||
abs_ys = ys + v0
|
||||
d2 = (abs_xs - u) ** 2 + (abs_ys - v) ** 2
|
||||
idx = int(np.argmin(d2))
|
||||
u_found = int(abs_xs[idx])
|
||||
v_found = int(abs_ys[idx])
|
||||
z_mm = int(depth[v_found, u_found])
|
||||
|
||||
if _camera.intrinsics:
|
||||
fx = _camera.intrinsics.fx_ir
|
||||
fy = _camera.intrinsics.fy_ir
|
||||
cx = _camera.intrinsics.cx_ir
|
||||
cy = _camera.intrinsics.cy_ir
|
||||
else:
|
||||
fx, fy = 445.0, 445.0
|
||||
cx, cy = 320.0, 240.0
|
||||
|
||||
z_m = z_mm / 1000.0
|
||||
x_m = (u_found - cx) * z_m / fx
|
||||
y_m = (v_found - cy) * z_m / fy
|
||||
dist_mm = float(np.sqrt(x_m * x_m + y_m * y_m + z_m * z_m) * 1000.0)
|
||||
|
||||
return ok({
|
||||
"u": u,
|
||||
"v": v,
|
||||
"u_found": u_found,
|
||||
"v_found": v_found,
|
||||
"pixel_offset": round(float(np.sqrt(float(d2[idx]))), 2),
|
||||
"radius": radius,
|
||||
"depth_mm": z_mm,
|
||||
"distance_mm": round(dist_mm, 1),
|
||||
"point_m": {
|
||||
"x": round(x_m, 4),
|
||||
"y": round(y_m, 4),
|
||||
"z": round(z_m, 4),
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@router.post("/snapshot")
|
||||
async def snapshot(
|
||||
flip_h: bool = Query(default=False),
|
||||
flip_v: bool = Query(default=False),
|
||||
):
|
||||
"""Capture a single RGB frame and save it as JPG in images_dir."""
|
||||
if not _camera or not _camera.is_open():
|
||||
return fail("CAMERA_NOT_OPEN", "Camera not open")
|
||||
if _saver is None:
|
||||
return fail("SAVER_NOT_READY", "Image saver not initialized")
|
||||
|
||||
frame = _camera.capture()
|
||||
if frame is None:
|
||||
return fail("CAPTURE_TIMEOUT", "Frame capture timeout")
|
||||
|
||||
filename = _saver.save_snapshot(frame.rgb, flip_h, flip_v)
|
||||
return ok({"filename": filename})
|
||||
|
||||
|
||||
@router.post("/recording/start")
|
||||
async def recording_start(
|
||||
flip_h: bool = Query(default=False),
|
||||
flip_v: bool = Query(default=False),
|
||||
interval: int = Query(default=1, ge=1, le=600),
|
||||
):
|
||||
"""Start continuous frame recording (applies to WebSocket stream loop).
|
||||
|
||||
interval: save 1 frame every N streamed frames (1 = every frame).
|
||||
"""
|
||||
if _saver is None:
|
||||
return fail("SAVER_NOT_READY", "Image saver not initialized")
|
||||
if _saver.is_recording:
|
||||
return fail("ALREADY_RECORDING", "Recording already in progress")
|
||||
|
||||
status = _saver.start_recording(flip_h, flip_v, interval)
|
||||
return ok(status)
|
||||
|
||||
|
||||
@router.post("/recording/stop")
|
||||
async def recording_stop():
|
||||
"""Stop continuous frame recording."""
|
||||
if _saver is None:
|
||||
return fail("SAVER_NOT_READY", "Image saver not initialized")
|
||||
if not _saver.is_recording:
|
||||
return fail("NOT_RECORDING", "No recording in progress")
|
||||
|
||||
status = _saver.stop_recording()
|
||||
return ok(status)
|
||||
|
||||
|
||||
@router.get("/recording/status")
|
||||
async def recording_status():
|
||||
"""Current recording state."""
|
||||
if _saver is None:
|
||||
return fail("SAVER_NOT_READY", "Image saver not initialized")
|
||||
return ok(_saver.status())
|
||||
|
||||
|
||||
@router.get("/intrinsics")
|
||||
async def get_intrinsics():
|
||||
"""Get camera intrinsic parameters."""
|
||||
if not _camera:
|
||||
return fail("INTRINSICS_NOT_AVAILABLE", "Camera driver not ready")
|
||||
if not _camera.is_open():
|
||||
return fail("INTRINSICS_NOT_AVAILABLE", "Camera not open")
|
||||
if not _camera.intrinsics:
|
||||
return fail(
|
||||
"INTRINSICS_NOT_AVAILABLE",
|
||||
"Camera open but SDK did not return calibration (AS_SDK_GetCamParameter). "
|
||||
"Check docker logs for 'GetCamParameter ... failed'.",
|
||||
)
|
||||
|
||||
return ok(_calibration_payload())
|
||||
@@ -0,0 +1,94 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
|
||||
|
||||
from src.backend.api.routes import _frame_to_dict
|
||||
from src.backend.services.image_saver import ImageSaver
|
||||
|
||||
try:
|
||||
from src.backend.services.detector import Detector
|
||||
except Exception:
|
||||
Detector = None # onnxruntime is optional; YOLO features stay disabled
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ws_router = APIRouter()
|
||||
|
||||
_camera = None
|
||||
_saver: ImageSaver | None = None
|
||||
_detector = None # Detector | None when onnxruntime is installed
|
||||
|
||||
|
||||
def set_camera(camera):
|
||||
global _camera
|
||||
_camera = camera
|
||||
|
||||
|
||||
def set_image_saver(saver: ImageSaver) -> None:
|
||||
global _saver
|
||||
_saver = saver
|
||||
|
||||
|
||||
def set_detector(detector) -> None:
|
||||
global _detector
|
||||
_detector = detector
|
||||
|
||||
|
||||
@ws_router.websocket("/ws")
|
||||
async def websocket_stream(ws: WebSocket):
|
||||
"""WebSocket stream of RGB+depth frames.
|
||||
|
||||
Client → server text messages:
|
||||
'start' — begin streaming
|
||||
'stop' — pause streaming
|
||||
'close' — disconnect
|
||||
'predict_on' — enable YOLO inference per frame (adds 'detections' to JSON)
|
||||
'predict_off' — disable YOLO inference
|
||||
'conf:0.55' — set YOLO confidence threshold (0.0–1.0)
|
||||
"""
|
||||
await ws.accept()
|
||||
logger.info("WebSocket client connected")
|
||||
streaming = False
|
||||
predicting = False
|
||||
|
||||
try:
|
||||
while True:
|
||||
if streaming and _camera and _camera.is_open():
|
||||
frame = _camera.capture(timeout_ms=500)
|
||||
if frame:
|
||||
data = _frame_to_dict(frame)
|
||||
if predicting and _detector is not None:
|
||||
data["detections"] = await asyncio.to_thread(_detector.detect, frame.rgb)
|
||||
await ws.send_json(data)
|
||||
if _saver and _saver.is_recording:
|
||||
_saver.save_rec_frame(frame.rgb)
|
||||
await asyncio.sleep(0.033)
|
||||
|
||||
try:
|
||||
msg = await asyncio.wait_for(ws.receive_text(), timeout=0.01)
|
||||
if msg == "start":
|
||||
streaming = True
|
||||
elif msg == "stop":
|
||||
streaming = False
|
||||
elif msg == "close":
|
||||
break
|
||||
elif msg == "predict_on":
|
||||
predicting = _detector is not None
|
||||
if not predicting:
|
||||
logger.warning("predict_on requested but detector not loaded")
|
||||
elif msg == "predict_off":
|
||||
predicting = False
|
||||
elif msg.startswith("conf:") and _detector is not None:
|
||||
try:
|
||||
_detector.set_conf(float(msg.split(":", 1)[1]))
|
||||
except ValueError:
|
||||
logger.warning(f"Invalid conf value: {msg!r}")
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
if not streaming:
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
except WebSocketDisconnect:
|
||||
logger.info("WebSocket client disconnected")
|
||||
@@ -0,0 +1,51 @@
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
# Server
|
||||
api_host: str = "0.0.0.0"
|
||||
api_port: int = 8878
|
||||
|
||||
# NATS
|
||||
nats_url: str = "nats://nats:4222"
|
||||
nats_token: str = ""
|
||||
|
||||
# Service identity
|
||||
service_name: str = "realsense-d435"
|
||||
service_domain: str = "vision"
|
||||
|
||||
# Security
|
||||
api_key: str = ""
|
||||
|
||||
# Storage
|
||||
images_dir: str = "/app/images"
|
||||
|
||||
# RealSense stream config
|
||||
camera_width: int = 640
|
||||
camera_height: int = 480
|
||||
camera_fps: int = 30
|
||||
camera_auto_open: bool = True
|
||||
|
||||
# YOLO detector (optional — disabled if model file or onnxruntime missing)
|
||||
yolo_onnx_path: str = "/app/models/best.onnx"
|
||||
yolo_classes: str = ""
|
||||
yolo_conf_default: float = 0.5
|
||||
|
||||
@property
|
||||
def yolo_class_list(self) -> list[str]:
|
||||
return [s.strip() for s in self.yolo_classes.split(",") if s.strip()]
|
||||
|
||||
# CORS (comma-separated string, parsed into list)
|
||||
cors_origins: str = "http://localhost:5173,http://localhost,http://localhost:80"
|
||||
|
||||
@property
|
||||
def cors_origins_list(self) -> list[str]:
|
||||
return [s.strip() for s in self.cors_origins.split(",")]
|
||||
|
||||
model_config = {
|
||||
"env_file": ".env",
|
||||
"env_file_encoding": "utf-8",
|
||||
}
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,147 @@
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from src.backend.config import settings
|
||||
from src.backend.api.routes import (
|
||||
router,
|
||||
set_camera as set_routes_camera,
|
||||
set_image_saver as set_routes_saver,
|
||||
)
|
||||
from src.backend.api.websocket import (
|
||||
ws_router,
|
||||
set_camera as set_ws_camera,
|
||||
set_image_saver as set_ws_saver,
|
||||
set_detector as set_ws_detector,
|
||||
)
|
||||
from src.backend.nats_client.connection import nats_manager
|
||||
from src.backend.nats_client.handlers import set_camera as set_nats_camera
|
||||
from src.backend.services.camera import RealSenseD435
|
||||
from src.backend.services.image_saver import ImageSaver
|
||||
|
||||
try:
|
||||
from src.backend.services.detector import Detector
|
||||
except Exception:
|
||||
Detector = None # YOLO/onnxruntime optional
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Startup
|
||||
camera = RealSenseD435(
|
||||
width=settings.camera_width,
|
||||
height=settings.camera_height,
|
||||
fps=settings.camera_fps,
|
||||
)
|
||||
sdk_ok = camera.init()
|
||||
if sdk_ok:
|
||||
logger.info("SDK initialized")
|
||||
else:
|
||||
logger.error(
|
||||
"SDK init failed — API will stay up so /api/health can be queried, "
|
||||
"but camera endpoints will return SDK_NOT_INITIALIZED"
|
||||
)
|
||||
|
||||
if sdk_ok and settings.camera_auto_open:
|
||||
try:
|
||||
devices = camera.discover()
|
||||
if devices:
|
||||
if camera.open(0):
|
||||
logger.info(
|
||||
f"Auto-opened camera 0 (serial={devices[0].serial})"
|
||||
)
|
||||
else:
|
||||
logger.warning("Auto-open requested but open(0) failed")
|
||||
else:
|
||||
logger.warning("Auto-open requested but no RealSense devices found")
|
||||
except Exception as exc:
|
||||
logger.warning(f"Auto-open failed: {exc}")
|
||||
|
||||
# Inject camera into all layers
|
||||
set_routes_camera(camera)
|
||||
set_ws_camera(camera)
|
||||
set_nats_camera(camera)
|
||||
|
||||
# Image saver (snapshot + continuous recording to images_dir)
|
||||
saver = ImageSaver(Path(settings.images_dir))
|
||||
set_routes_saver(saver)
|
||||
set_ws_saver(saver)
|
||||
logger.info(f"Image saver ready: {settings.images_dir}")
|
||||
|
||||
# YOLO detector (optional — disabled if onnxruntime or model file is missing)
|
||||
if Detector is not None:
|
||||
try:
|
||||
detector = Detector(
|
||||
settings.yolo_onnx_path,
|
||||
settings.yolo_class_list,
|
||||
settings.yolo_conf_default,
|
||||
)
|
||||
set_ws_detector(detector)
|
||||
except FileNotFoundError as exc:
|
||||
logger.warning(f"YOLO detector disabled (model not found): {exc}")
|
||||
except Exception as exc:
|
||||
logger.warning(f"YOLO detector disabled: {exc}")
|
||||
else:
|
||||
logger.info("YOLO detector disabled (onnxruntime not installed)")
|
||||
|
||||
# Connect NATS (non-blocking, graceful if unavailable)
|
||||
await nats_manager.connect()
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
await nats_manager.disconnect()
|
||||
camera.close()
|
||||
camera.deinit()
|
||||
logger.info("SDK deinitialized")
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title=settings.service_name,
|
||||
description="REST API + WebSocket for Intel RealSense D4xx depth camera (pyrealsense2)",
|
||||
version="1.0.0",
|
||||
docs_url="/apidocs",
|
||||
redoc_url="/redoc",
|
||||
openapi_url="/openapi.json",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins_list,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(router, prefix="/api")
|
||||
app.include_router(ws_router)
|
||||
|
||||
# Serve React frontend at /gui (built files in src/frontend/react/dist)
|
||||
_GUI_DIR = Path(__file__).parent.parent / "frontend" / "react" / "dist"
|
||||
if _GUI_DIR.is_dir():
|
||||
app.mount("/gui", StaticFiles(directory=str(_GUI_DIR), html=True), name="gui")
|
||||
logger.info(f"React GUI mounted at /gui from {_GUI_DIR}")
|
||||
|
||||
|
||||
# Public health endpoint (no API key required) for Docker healthchecks
|
||||
@app.get("/api/health", summary="Health check", tags=["system"])
|
||||
async def health():
|
||||
"""Service health check — no API key required."""
|
||||
from src.backend.models.api import ok
|
||||
from src.backend.nats_client.connection import nats_manager
|
||||
from src.backend.api.routes import _camera
|
||||
|
||||
return ok({
|
||||
"status": "ok",
|
||||
"sdk_initialized": _camera.is_initialized() if _camera else False,
|
||||
"camera_open": _camera.is_open() if _camera else False,
|
||||
"nats_connected": nats_manager.is_connected,
|
||||
})
|
||||
Whitespace-only changes.
@@ -0,0 +1,54 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ApiError(BaseModel):
|
||||
code: str
|
||||
message: str
|
||||
|
||||
|
||||
class ApiResponse(BaseModel):
|
||||
success: bool
|
||||
data: Any | None = None
|
||||
error: ApiError | None = None
|
||||
|
||||
|
||||
def ok(data: Any) -> dict:
|
||||
return {"success": True, "data": data, "error": None}
|
||||
|
||||
|
||||
def fail(code: str, message: str) -> dict:
|
||||
return {"success": False, "data": None, "error": {"code": code, "message": message}}
|
||||
|
||||
|
||||
class DeviceInfo(BaseModel):
|
||||
index: int
|
||||
vid: str
|
||||
pid: str
|
||||
serial: str
|
||||
bus: int
|
||||
device: int
|
||||
|
||||
|
||||
class IntrinsicsInfo(BaseModel):
|
||||
fx: float
|
||||
fy: float
|
||||
cx: float
|
||||
cy: float
|
||||
|
||||
|
||||
class CameraIntrinsicsInfo(BaseModel):
|
||||
ir: IntrinsicsInfo
|
||||
rgb: IntrinsicsInfo
|
||||
|
||||
|
||||
class FrameInfo(BaseModel):
|
||||
frame_id: int
|
||||
timestamp: float
|
||||
rgb_shape: list[int]
|
||||
depth_shape: list[int]
|
||||
depth_min_mm: int
|
||||
depth_max_mm: int
|
||||
depth_valid_pixels: int
|
||||
rgb_jpeg_b64: str | None = None
|
||||
depth_png_b64: str | None = None
|
||||
@@ -0,0 +1,26 @@
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class NatsResponse(BaseModel):
|
||||
success: bool
|
||||
data: dict | None = None
|
||||
error: dict | None = None
|
||||
|
||||
|
||||
class SnapshotRequest(BaseModel):
|
||||
camera_id: str = "default"
|
||||
include_depth: bool = True
|
||||
|
||||
|
||||
class SnapshotResponse(BaseModel):
|
||||
rgb_jpeg_b64: str
|
||||
depth_png_b64: str | None = None
|
||||
timestamp: float
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
class CameraStatusMessage(BaseModel):
|
||||
camera_open: bool
|
||||
sdk_initialized: bool
|
||||
frame_count: int
|
||||
Whitespace-only changes.
@@ -0,0 +1,58 @@
|
||||
import logging
|
||||
|
||||
import nats
|
||||
from nats.aio.client import Client
|
||||
from src.backend.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NatsManager:
|
||||
def __init__(self):
|
||||
self._nc: Client | None = None
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return self._nc is not None and self._nc.is_connected
|
||||
|
||||
@property
|
||||
def client(self) -> Client:
|
||||
if not self.is_connected:
|
||||
raise RuntimeError("NATS non connesso")
|
||||
return self._nc
|
||||
|
||||
async def connect(self):
|
||||
if not settings.nats_url:
|
||||
logger.info("NATS URL not configured, skipping connection")
|
||||
return
|
||||
try:
|
||||
self._nc = await nats.connect(
|
||||
settings.nats_url,
|
||||
token=settings.nats_token if settings.nats_token else None,
|
||||
name=settings.service_name,
|
||||
reconnect_time_wait=2,
|
||||
max_reconnect_attempts=-1,
|
||||
)
|
||||
logger.info(f"Connected to NATS at {settings.nats_url}")
|
||||
await self._register_handlers()
|
||||
await self._publish_discovery()
|
||||
except Exception as e:
|
||||
logger.warning(f"NATS connection failed: {e}. Running without NATS.")
|
||||
self._nc = None
|
||||
|
||||
async def disconnect(self):
|
||||
if self.is_connected:
|
||||
await self._nc.drain()
|
||||
self._nc = None
|
||||
logger.info("Disconnected from NATS")
|
||||
|
||||
async def _register_handlers(self):
|
||||
from src.backend.nats_client.handlers import register_all
|
||||
await register_all(self._nc)
|
||||
|
||||
async def _publish_discovery(self):
|
||||
from src.backend.nats_client.discovery import publish_discovery
|
||||
await publish_discovery(self._nc)
|
||||
|
||||
|
||||
nats_manager = NatsManager()
|
||||
@@ -0,0 +1,52 @@
|
||||
import json
|
||||
from nats.aio.client import Client
|
||||
from src.backend.config import settings
|
||||
from src.backend.nats_client.subjects import Subjects
|
||||
|
||||
CAPABILITIES = {
|
||||
"service": settings.service_name,
|
||||
"domain": settings.service_domain,
|
||||
"version": "1.0.0",
|
||||
"description": "Intel RealSense D4xx stereo depth camera server",
|
||||
"endpoints": {
|
||||
"rest": {
|
||||
"health": {"method": "GET", "path": "/api/health"},
|
||||
"discover": {"method": "GET", "path": "/api/discover"},
|
||||
"open": {"method": "POST", "path": "/api/open"},
|
||||
"close": {"method": "POST", "path": "/api/close"},
|
||||
"capture": {"method": "GET", "path": "/api/capture"},
|
||||
"rgb": {"method": "GET", "path": "/api/rgb"},
|
||||
"depth": {"method": "GET", "path": "/api/depth"},
|
||||
},
|
||||
},
|
||||
"nats": {
|
||||
"actions": [
|
||||
{
|
||||
"subject": Subjects.action("snapshot"),
|
||||
"pattern": "request/reply",
|
||||
"description": "Capture RGB+depth frame",
|
||||
"request_schema": {"include_depth": "boolean"},
|
||||
"response_schema": {"rgb_jpeg_b64": "string", "depth_png_b64": "string", "timestamp": "number"},
|
||||
},
|
||||
{
|
||||
"subject": Subjects.action("status"),
|
||||
"pattern": "request/reply",
|
||||
"description": "Get camera status",
|
||||
"response_schema": {"camera_open": "boolean", "sdk_initialized": "boolean"},
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def publish_discovery(nc: Client):
|
||||
await nc.publish(
|
||||
Subjects.DISCOVERY_ANNOUNCE,
|
||||
json.dumps(CAPABILITIES).encode(),
|
||||
)
|
||||
|
||||
async def handle_discovery_request(msg):
|
||||
await msg.respond(json.dumps(CAPABILITIES).encode())
|
||||
|
||||
await nc.subscribe(Subjects.DISCOVERY_REQUEST, cb=handle_discovery_request)
|
||||
await nc.subscribe(Subjects.SERVICE_DISCOVERY, cb=handle_discovery_request)
|
||||
@@ -0,0 +1,66 @@
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
|
||||
from nats.aio.client import Client
|
||||
from src.backend.nats_client.subjects import Subjects
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_camera = None
|
||||
|
||||
|
||||
def set_camera(camera):
|
||||
global _camera
|
||||
_camera = camera
|
||||
|
||||
|
||||
def _nats_response(success: bool, data=None, error=None) -> bytes:
|
||||
return json.dumps({
|
||||
"success": success,
|
||||
"data": data,
|
||||
"error": error,
|
||||
}).encode()
|
||||
|
||||
|
||||
async def register_all(nc: Client):
|
||||
async def handle_snapshot(msg):
|
||||
try:
|
||||
if not _camera or not _camera.is_open():
|
||||
await msg.respond(_nats_response(
|
||||
False, error={"code": "NO_CAMERA", "message": "Camera not open"}
|
||||
))
|
||||
return
|
||||
|
||||
import cv2
|
||||
frame = _camera.capture()
|
||||
if frame is None:
|
||||
await msg.respond(_nats_response(
|
||||
False, error={"code": "TIMEOUT", "message": "Frame capture timeout"}
|
||||
))
|
||||
return
|
||||
|
||||
_, rgb_buf = cv2.imencode(".jpg", frame.rgb, [cv2.IMWRITE_JPEG_QUALITY, 85])
|
||||
_, depth_buf = cv2.imencode(".png", frame.depth)
|
||||
|
||||
await msg.respond(_nats_response(True, data={
|
||||
"rgb_jpeg_b64": base64.b64encode(rgb_buf.tobytes()).decode(),
|
||||
"depth_png_b64": base64.b64encode(depth_buf.tobytes()).decode(),
|
||||
"timestamp": frame.timestamp,
|
||||
"width": frame.rgb.shape[1],
|
||||
"height": frame.rgb.shape[0],
|
||||
}))
|
||||
except Exception as e:
|
||||
logger.error(f"Snapshot handler error: {e}")
|
||||
await msg.respond(_nats_response(
|
||||
False, error={"code": "PROCESSING_ERROR", "message": str(e)}
|
||||
))
|
||||
|
||||
async def handle_status(msg):
|
||||
await msg.respond(_nats_response(True, data={
|
||||
"camera_open": _camera.is_open() if _camera else False,
|
||||
"sdk_initialized": _camera.is_initialized() if _camera else False,
|
||||
}))
|
||||
|
||||
await nc.subscribe(Subjects.action("snapshot"), cb=handle_snapshot)
|
||||
await nc.subscribe(Subjects.action("status"), cb=handle_status)
|
||||
@@ -0,0 +1,18 @@
|
||||
from src.backend.config import settings
|
||||
|
||||
DOMAIN = settings.service_domain
|
||||
SERVICE = settings.service_name
|
||||
|
||||
|
||||
class Subjects:
|
||||
DISCOVERY_REQUEST = "_discovery.request"
|
||||
DISCOVERY_ANNOUNCE = "_discovery.announce"
|
||||
SERVICE_DISCOVERY = f"_discovery.{SERVICE}"
|
||||
|
||||
@staticmethod
|
||||
def action(action: str) -> str:
|
||||
return f"{DOMAIN}.{SERVICE}.{action}"
|
||||
|
||||
@staticmethod
|
||||
def action_reply(action: str) -> str:
|
||||
return f"{DOMAIN}.{SERVICE}.{action}.reply"
|
||||
Whitespace-only changes.
@@ -0,0 +1,413 @@
|
||||
"""Intel RealSense D4xx depth camera driver via pyrealsense2.
|
||||
|
||||
Provides RGB (BGR8) and aligned Depth (uint16 mm) frames plus
|
||||
intrinsics/extrinsics through the same public surface as the
|
||||
Nuwa HP60C driver, so the routes/websocket/frontend stay unchanged.
|
||||
|
||||
In this driver the "ir_*" intrinsic fields hold the **depth** sensor
|
||||
intrinsics (the D4xx depth stream is rectified from the IR stereo pair),
|
||||
and "rgb_*" hold the **color** sensor intrinsics. After alignment both
|
||||
streams share the color intrinsics, but we still expose the raw depth
|
||||
intrinsics so client code that wants to back-project the unaligned depth
|
||||
can do so.
|
||||
|
||||
Usage:
|
||||
cam = RealSenseD435()
|
||||
cam.init()
|
||||
devs = cam.discover()
|
||||
cam.open(0)
|
||||
frame = cam.capture()
|
||||
cam.close()
|
||||
cam.deinit()
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import pyrealsense2 as rs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CameraFrame:
|
||||
"""Container for a single capture: aligned color + depth + timestamp."""
|
||||
rgb: np.ndarray # (H, W, 3) uint8 BGR
|
||||
depth: np.ndarray # (H, W) uint16 millimeters
|
||||
timestamp: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class CameraIntrinsics:
|
||||
"""Per-sensor intrinsics. `*_ir` = depth, `*_rgb` = color."""
|
||||
fx_ir: float
|
||||
fy_ir: float
|
||||
cx_ir: float
|
||||
cy_ir: float
|
||||
fx_rgb: float
|
||||
fy_rgb: float
|
||||
cx_rgb: float
|
||||
cy_rgb: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class CameraExtrinsics:
|
||||
"""Rigid transform depth → color. Row-major 3x3 rotation, translation in mm."""
|
||||
rotation: tuple
|
||||
translation: tuple
|
||||
|
||||
|
||||
@dataclass
|
||||
class Distortion:
|
||||
"""Brown-Conrady distortion (k1, k2, k3, p1, p2)."""
|
||||
k1: float
|
||||
k2: float
|
||||
k3: float
|
||||
p1: float
|
||||
p2: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class DiscoveredDevice:
|
||||
index: int
|
||||
vid: int
|
||||
pid: int
|
||||
serial: str
|
||||
bus: int
|
||||
device: int
|
||||
|
||||
|
||||
class RealSenseD435:
|
||||
"""Driver for Intel RealSense D4xx (tested on D435) via pyrealsense2.
|
||||
|
||||
Lifecycle: init() → discover() → open(index) → capture/get_rgb/get_depth → close() → deinit()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
width: int = 640,
|
||||
height: int = 480,
|
||||
fps: int = 30,
|
||||
frame_cache_ms: float = 100.0,
|
||||
):
|
||||
self._width = width
|
||||
self._height = height
|
||||
self._fps = fps
|
||||
self._frame_cache_ms = frame_cache_ms
|
||||
|
||||
self._ctx: Optional[rs.context] = None
|
||||
self._pipeline: Optional[rs.pipeline] = None
|
||||
self._profile = None
|
||||
self._align: Optional[rs.align] = None
|
||||
self._depth_scale: float = 0.001
|
||||
|
||||
self._initialized = False
|
||||
self._open = False
|
||||
self._discovered: list[DiscoveredDevice] = []
|
||||
self._last_frame: Optional[CameraFrame] = None
|
||||
self._color_available: bool = True
|
||||
self._color_format = rs.format.bgr8
|
||||
self._usb_type: str = "unknown"
|
||||
|
||||
self.device_index: Optional[int] = None
|
||||
self.intrinsics: Optional[CameraIntrinsics] = None
|
||||
self.extrinsics: Optional[CameraExtrinsics] = None
|
||||
self.distortion_ir: Optional[Distortion] = None
|
||||
self.distortion_rgb: Optional[Distortion] = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
def is_initialized(self) -> bool:
|
||||
return self._initialized
|
||||
|
||||
def is_open(self) -> bool:
|
||||
return self._open
|
||||
|
||||
def is_color_available(self) -> bool:
|
||||
"""False when the pipeline runs depth-only (USB 2.x fallback)."""
|
||||
return self._color_available
|
||||
|
||||
def usb_type(self) -> str:
|
||||
return self._usb_type
|
||||
|
||||
def init(self) -> bool:
|
||||
try:
|
||||
self._ctx = rs.context()
|
||||
self._initialized = True
|
||||
logger.info("RealSense SDK initialized")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"RealSense init failed: {e}")
|
||||
return False
|
||||
|
||||
def deinit(self) -> None:
|
||||
if self._open:
|
||||
self.close()
|
||||
self._ctx = None
|
||||
self._initialized = False
|
||||
logger.info("RealSense SDK deinitialized")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Discovery
|
||||
# ------------------------------------------------------------------
|
||||
def discover(self, timeout_ms: int = 5000) -> list[DiscoveredDevice]:
|
||||
"""Enumerate RealSense devices on USB. `timeout_ms` kept for API parity."""
|
||||
if not self._initialized:
|
||||
raise RuntimeError("SDK not initialized, call init() first")
|
||||
|
||||
devices = self._ctx.query_devices()
|
||||
self._discovered = []
|
||||
for i, dev in enumerate(devices):
|
||||
try:
|
||||
serial = dev.get_info(rs.camera_info.serial_number)
|
||||
try:
|
||||
pid_hex = dev.get_info(rs.camera_info.product_id)
|
||||
pid = int(pid_hex, 16) if pid_hex else 0
|
||||
except Exception:
|
||||
pid = 0
|
||||
bus = 0
|
||||
addr = 0
|
||||
try:
|
||||
phys = dev.get_info(rs.camera_info.physical_port)
|
||||
parts = phys.rsplit("/", 1)[-1].split("-")
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
bus = int(parts[0])
|
||||
except ValueError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
self._discovered.append(DiscoveredDevice(
|
||||
index=i, vid=0x8086, pid=pid, serial=serial, bus=bus, device=addr,
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to read device {i} info: {e}")
|
||||
|
||||
logger.info(f"Discovered {len(self._discovered)} RealSense device(s)")
|
||||
return self._discovered
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stream open/close
|
||||
# ------------------------------------------------------------------
|
||||
def open(self, device_index: int = 0) -> bool:
|
||||
if not self._initialized:
|
||||
logger.error("Cannot open: SDK not initialized")
|
||||
return False
|
||||
if self._open:
|
||||
logger.warning("Camera already open")
|
||||
return True
|
||||
|
||||
if not self._discovered:
|
||||
self.discover()
|
||||
if device_index < 0 or device_index >= len(self._discovered):
|
||||
logger.error(
|
||||
f"Invalid device_index {device_index} ({len(self._discovered)} devices found)"
|
||||
)
|
||||
return False
|
||||
|
||||
serial = self._discovered[device_index].serial
|
||||
|
||||
# Probe the USB type via the SDK. D4xx firmware can't sustain both
|
||||
# depth and color streams simultaneously on USB 2.x — wait_for_frames
|
||||
# times out indefinitely. We fall back to depth-only there.
|
||||
usb_type = "unknown"
|
||||
for dev in self._ctx.query_devices():
|
||||
try:
|
||||
if dev.get_info(rs.camera_info.serial_number) == serial:
|
||||
if dev.supports(rs.camera_info.usb_type_descriptor):
|
||||
usb_type = dev.get_info(rs.camera_info.usb_type_descriptor)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
self._usb_type = usb_type
|
||||
|
||||
is_usb2 = usb_type.startswith("2.")
|
||||
if is_usb2:
|
||||
# On USB 2.x the D4xx firmware only sustains the *native* YUYV
|
||||
# color format — BGR8/RGB8 trigger a wait_for_frames timeout. We
|
||||
# convert YUYV → BGR in capture() so callers stay format-agnostic.
|
||||
# 640x480@15 is the highest tested combination that holds without
|
||||
# dropping frames on this Jetson; actual delivery rate is ~4 fps
|
||||
# due to USB 2 bandwidth, but wait_for_frames stays under 300 ms.
|
||||
color_w, color_h, color_fps = 640, 480, 15
|
||||
depth_w, depth_h, depth_fps = 640, 480, 15
|
||||
color_fmt = rs.format.yuyv
|
||||
logger.warning(
|
||||
f"D4xx on USB {usb_type} — using USB2 profile "
|
||||
f"(color YUYV {color_w}x{color_h}@{color_fps}, "
|
||||
f"depth {depth_w}x{depth_h}@{depth_fps}, ~4 fps actual). "
|
||||
f"Move to a USB 3 port for full framerate."
|
||||
)
|
||||
else:
|
||||
color_w, color_h, color_fps = self._width, self._height, self._fps
|
||||
depth_w, depth_h, depth_fps = self._width, self._height, self._fps
|
||||
color_fmt = rs.format.bgr8
|
||||
|
||||
try:
|
||||
self._pipeline = rs.pipeline(self._ctx)
|
||||
config = rs.config()
|
||||
config.enable_device(serial)
|
||||
config.enable_stream(
|
||||
rs.stream.depth, depth_w, depth_h, rs.format.z16, depth_fps
|
||||
)
|
||||
config.enable_stream(
|
||||
rs.stream.color, color_w, color_h, color_fmt, color_fps
|
||||
)
|
||||
self._profile = self._pipeline.start(config)
|
||||
self._color_available = True
|
||||
self._color_format = color_fmt
|
||||
# rs.align fails when the color stream is YUYV (the only color
|
||||
# format the D4xx can sustain on USB 2). Skip hardware alignment
|
||||
# there; we fall back to a numpy resize in capture() so the rgb
|
||||
# and depth arrays share the same shape for downstream consumers.
|
||||
self._align = None if is_usb2 else rs.align(rs.stream.color)
|
||||
|
||||
depth_sensor = self._profile.get_device().first_depth_sensor()
|
||||
self._depth_scale = depth_sensor.get_depth_scale()
|
||||
|
||||
self._read_calibration()
|
||||
|
||||
self.device_index = device_index
|
||||
self._open = True
|
||||
logger.info(
|
||||
f"RealSense D4xx opened: serial={serial} usb={usb_type} "
|
||||
f"color={color_w}x{color_h}@{color_fps}({color_fmt}) "
|
||||
f"depth={depth_w}x{depth_h}@{depth_fps} depth_scale={self._depth_scale}"
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Open failed: {e}")
|
||||
self._teardown_pipeline()
|
||||
return False
|
||||
|
||||
def close(self) -> None:
|
||||
if not self._open:
|
||||
return
|
||||
self._teardown_pipeline()
|
||||
self.device_index = None
|
||||
self._last_frame = None
|
||||
logger.info("RealSense camera closed")
|
||||
|
||||
def _teardown_pipeline(self) -> None:
|
||||
if self._pipeline is not None:
|
||||
try:
|
||||
self._pipeline.stop()
|
||||
except Exception as e:
|
||||
logger.warning(f"pipeline.stop failed: {e}")
|
||||
self._pipeline = None
|
||||
self._profile = None
|
||||
self._align = None
|
||||
self._open = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Calibration
|
||||
# ------------------------------------------------------------------
|
||||
def _read_calibration(self) -> None:
|
||||
color_stream = self._profile.get_stream(rs.stream.color).as_video_stream_profile()
|
||||
depth_stream = self._profile.get_stream(rs.stream.depth).as_video_stream_profile()
|
||||
ci = color_stream.get_intrinsics()
|
||||
di = depth_stream.get_intrinsics()
|
||||
|
||||
self.intrinsics = CameraIntrinsics(
|
||||
fx_ir=di.fx, fy_ir=di.fy, cx_ir=di.ppx, cy_ir=di.ppy,
|
||||
fx_rgb=ci.fx, fy_rgb=ci.fy, cx_rgb=ci.ppx, cy_rgb=ci.ppy,
|
||||
)
|
||||
|
||||
try:
|
||||
ext = depth_stream.get_extrinsics_to(color_stream)
|
||||
r = list(ext.rotation)
|
||||
rotation = (
|
||||
r[0], r[3], r[6],
|
||||
r[1], r[4], r[7],
|
||||
r[2], r[5], r[8],
|
||||
)
|
||||
t = list(ext.translation)
|
||||
translation = (t[0] * 1000.0, t[1] * 1000.0, t[2] * 1000.0)
|
||||
self.extrinsics = CameraExtrinsics(rotation=rotation, translation=translation)
|
||||
except Exception as e:
|
||||
logger.warning(f"Extrinsics read failed: {e}")
|
||||
self.extrinsics = None
|
||||
|
||||
# rs intrinsics.coeffs order is [k1, k2, p1, p2, k3].
|
||||
def _to_distortion(coeffs) -> Distortion:
|
||||
if not coeffs or len(coeffs) < 5:
|
||||
return Distortion(0.0, 0.0, 0.0, 0.0, 0.0)
|
||||
return Distortion(
|
||||
k1=coeffs[0], k2=coeffs[1], k3=coeffs[4], p1=coeffs[2], p2=coeffs[3],
|
||||
)
|
||||
|
||||
self.distortion_ir = _to_distortion(list(di.coeffs))
|
||||
self.distortion_rgb = _to_distortion(list(ci.coeffs))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Frame capture
|
||||
# ------------------------------------------------------------------
|
||||
def capture(self, timeout_ms: int = 5000) -> Optional[CameraFrame]:
|
||||
if not self._open:
|
||||
return None
|
||||
try:
|
||||
frames = self._pipeline.wait_for_frames(timeout_ms)
|
||||
except Exception as e:
|
||||
logger.warning(f"wait_for_frames failed: {e}")
|
||||
return None
|
||||
|
||||
if self._align is not None:
|
||||
frames = self._align.process(frames)
|
||||
|
||||
color_frame = frames.get_color_frame()
|
||||
depth_frame = frames.get_depth_frame()
|
||||
if not color_frame or not depth_frame:
|
||||
return None
|
||||
|
||||
if self._color_format == rs.format.yuyv:
|
||||
# SDK exposes YUYV as a 1-channel view of the packed buffer;
|
||||
# rebuild it as (H, W, 2) uint8 so cv2 can decode it to BGR.
|
||||
h = color_frame.get_height()
|
||||
w = color_frame.get_width()
|
||||
buf = np.frombuffer(color_frame.get_data(), dtype=np.uint8)
|
||||
rgb = cv2.cvtColor(buf.reshape(h, w, 2), cv2.COLOR_YUV2BGR_YUYV)
|
||||
else:
|
||||
rgb = np.asanyarray(color_frame.get_data())
|
||||
|
||||
depth_raw = np.asanyarray(depth_frame.get_data())
|
||||
# On USB 2 (no hardware alignment) the color and depth streams have
|
||||
# different native resolutions. Resize color to depth shape so
|
||||
# downstream endpoints (/pointcloud, /measure) that index rgb[y, x]
|
||||
# and depth[y, x] in lockstep continue to work. This is an
|
||||
# *approximate* alignment — depth↔color extrinsics are ignored.
|
||||
if rgb.shape[:2] != depth_raw.shape[:2]:
|
||||
rgb = cv2.resize(
|
||||
rgb, (depth_raw.shape[1], depth_raw.shape[0]),
|
||||
interpolation=cv2.INTER_LINEAR,
|
||||
)
|
||||
|
||||
# depth_scale ~ 0.001 m/unit on D4xx → raw uint16 is already millimeters.
|
||||
if abs(self._depth_scale - 0.001) < 1e-6:
|
||||
depth_mm = depth_raw
|
||||
else:
|
||||
depth_mm = (depth_raw.astype(np.float32) * self._depth_scale * 1000.0).astype(np.uint16)
|
||||
|
||||
frame = CameraFrame(rgb=rgb, depth=depth_mm, timestamp=time.time())
|
||||
self._last_frame = frame
|
||||
return frame
|
||||
|
||||
def _cached_or_capture(self) -> Optional[CameraFrame]:
|
||||
if (
|
||||
self._last_frame is not None
|
||||
and (time.time() - self._last_frame.timestamp) * 1000.0 < self._frame_cache_ms
|
||||
):
|
||||
return self._last_frame
|
||||
return self.capture()
|
||||
|
||||
def get_rgb(self) -> Optional[np.ndarray]:
|
||||
f = self._cached_or_capture()
|
||||
return f.rgb if f else None
|
||||
|
||||
def get_depth(self) -> Optional[np.ndarray]:
|
||||
f = self._cached_or_capture()
|
||||
return f.depth if f else None
|
||||
@@ -0,0 +1,92 @@
|
||||
"""ONNX YOLO detector with letterbox preprocessing.
|
||||
|
||||
Assumes the exported ONNX model has end-to-end NMS embedded:
|
||||
output shape (1, max_det, 6) where the 6 columns are
|
||||
[x1, y1, x2, y2, conf, cls] in letterboxed input space.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _letterbox(img: np.ndarray, new_size: int = 640, color: tuple = (114, 114, 114)):
|
||||
"""Resize keeping aspect ratio and pad to (new_size, new_size)."""
|
||||
h, w = img.shape[:2]
|
||||
r = min(new_size / h, new_size / w)
|
||||
new_w, new_h = int(round(w * r)), int(round(h * r))
|
||||
pad_w = (new_size - new_w) / 2
|
||||
pad_h = (new_size - new_h) / 2
|
||||
if (w, h) != (new_w, new_h):
|
||||
img = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_LINEAR)
|
||||
top, bottom = int(round(pad_h - 0.1)), int(round(pad_h + 0.1))
|
||||
left, right = int(round(pad_w - 0.1)), int(round(pad_w + 0.1))
|
||||
img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)
|
||||
return img, r, left, top
|
||||
|
||||
|
||||
class Detector:
|
||||
def __init__(self, onnx_path: str | Path, class_names: list[str], conf_threshold: float = 0.5):
|
||||
path = Path(onnx_path)
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"ONNX model not found: {path}")
|
||||
|
||||
opts = ort.SessionOptions()
|
||||
opts.intra_op_num_threads = 4
|
||||
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
self.session = ort.InferenceSession(str(path), opts, providers=["CPUExecutionProvider"])
|
||||
|
||||
inp = self.session.get_inputs()[0]
|
||||
self.input_name = inp.name
|
||||
self.imgsz = int(inp.shape[-1])
|
||||
self.class_names = class_names
|
||||
self.conf_threshold = conf_threshold
|
||||
|
||||
# Warmup
|
||||
dummy = np.zeros((self.imgsz, self.imgsz, 3), dtype=np.uint8)
|
||||
self.detect(dummy)
|
||||
logger.info(f"Detector ready: model={path.name} imgsz={self.imgsz} classes={class_names}")
|
||||
|
||||
def set_conf(self, value: float) -> None:
|
||||
self.conf_threshold = float(np.clip(value, 0.0, 1.0))
|
||||
|
||||
def detect(self, bgr: np.ndarray) -> list[dict]:
|
||||
h, w = bgr.shape[:2]
|
||||
padded, scale, pad_x, pad_y = _letterbox(bgr, self.imgsz)
|
||||
rgb = cv2.cvtColor(padded, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0
|
||||
tensor = np.expand_dims(rgb.transpose(2, 0, 1), 0)
|
||||
|
||||
outputs = self.session.run(None, {self.input_name: tensor})
|
||||
dets = outputs[0][0] # (max_det, 6)
|
||||
|
||||
mask = dets[:, 4] >= self.conf_threshold
|
||||
dets = dets[mask]
|
||||
if len(dets) == 0:
|
||||
return []
|
||||
|
||||
best = int(np.argmax(dets[:, 4]))
|
||||
dets = dets[best:best + 1]
|
||||
|
||||
boxes = dets[:, :4].copy()
|
||||
boxes[:, [0, 2]] = (boxes[:, [0, 2]] - pad_x) / scale
|
||||
boxes[:, [1, 3]] = (boxes[:, [1, 3]] - pad_y) / scale
|
||||
boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, w - 1)
|
||||
boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, h - 1)
|
||||
|
||||
results = []
|
||||
for box, det in zip(boxes, dets):
|
||||
cls_idx = int(det[5])
|
||||
label = self.class_names[cls_idx] if 0 <= cls_idx < len(self.class_names) else str(cls_idx)
|
||||
results.append({
|
||||
"bbox": [round(float(box[0]), 1), round(float(box[1]), 1),
|
||||
round(float(box[2]), 1), round(float(box[3]), 1)],
|
||||
"conf": round(float(det[4]), 3),
|
||||
"cls": cls_idx,
|
||||
"label": label,
|
||||
})
|
||||
return results
|
||||
@@ -0,0 +1,105 @@
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _flip_code(flip_h: bool, flip_v: bool) -> int | None:
|
||||
if flip_h and flip_v:
|
||||
return -1
|
||||
if flip_h:
|
||||
return 1
|
||||
if flip_v:
|
||||
return 0
|
||||
return None
|
||||
|
||||
|
||||
def _timestamp() -> str:
|
||||
now = datetime.now()
|
||||
return now.strftime("%Y%m%d_%H%M%S_") + f"{now.microsecond // 1000:03d}"
|
||||
|
||||
|
||||
class ImageSaver:
|
||||
def __init__(self, images_dir: Path, jpeg_quality: int = 85):
|
||||
self._dir = images_dir
|
||||
self._dir.mkdir(parents=True, exist_ok=True)
|
||||
self._quality = jpeg_quality
|
||||
self._lock = Lock()
|
||||
self._recording = False
|
||||
self._rec_flip_h = False
|
||||
self._rec_flip_v = False
|
||||
self._rec_started_at: str | None = None
|
||||
self._rec_count = 0
|
||||
self._rec_interval = 1
|
||||
self._rec_seen = 0
|
||||
|
||||
@property
|
||||
def is_recording(self) -> bool:
|
||||
return self._recording
|
||||
|
||||
def _encode_jpeg(self, rgb: np.ndarray, flip_h: bool, flip_v: bool) -> bytes:
|
||||
img = rgb
|
||||
code = _flip_code(flip_h, flip_v)
|
||||
if code is not None:
|
||||
img = cv2.flip(img, code)
|
||||
_, buf = cv2.imencode(".jpg", img, [cv2.IMWRITE_JPEG_QUALITY, self._quality])
|
||||
return buf.tobytes()
|
||||
|
||||
def save_snapshot(self, rgb: np.ndarray, flip_h: bool, flip_v: bool) -> str:
|
||||
data = self._encode_jpeg(rgb, flip_h, flip_v)
|
||||
filename = f"snap_{_timestamp()}.jpg"
|
||||
(self._dir / filename).write_bytes(data)
|
||||
logger.info(f"Snapshot saved: {filename}")
|
||||
return filename
|
||||
|
||||
def start_recording(self, flip_h: bool, flip_v: bool, interval: int = 1) -> dict:
|
||||
if interval < 1:
|
||||
interval = 1
|
||||
with self._lock:
|
||||
if self._recording:
|
||||
raise RuntimeError("already recording")
|
||||
self._recording = True
|
||||
self._rec_flip_h = flip_h
|
||||
self._rec_flip_v = flip_v
|
||||
self._rec_started_at = datetime.now().isoformat(timespec="seconds")
|
||||
self._rec_count = 0
|
||||
self._rec_seen = 0
|
||||
self._rec_interval = interval
|
||||
logger.info(
|
||||
f"Recording started (flip_h={flip_h}, flip_v={flip_v}, interval={interval})"
|
||||
)
|
||||
return self.status()
|
||||
|
||||
def stop_recording(self) -> dict:
|
||||
with self._lock:
|
||||
snapshot = self.status()
|
||||
self._recording = False
|
||||
logger.info(f"Recording stopped: {snapshot['frames_written']} frames")
|
||||
return snapshot
|
||||
|
||||
def save_rec_frame(self, rgb: np.ndarray) -> None:
|
||||
if not self._recording:
|
||||
return
|
||||
self._rec_seen += 1
|
||||
if (self._rec_seen - 1) % self._rec_interval != 0:
|
||||
return
|
||||
data = self._encode_jpeg(rgb, self._rec_flip_h, self._rec_flip_v)
|
||||
filename = f"rec_{_timestamp()}.jpg"
|
||||
(self._dir / filename).write_bytes(data)
|
||||
self._rec_count += 1
|
||||
|
||||
def status(self) -> dict:
|
||||
return {
|
||||
"recording": self._recording,
|
||||
"frames_written": self._rec_count,
|
||||
"frames_seen": self._rec_seen,
|
||||
"interval": self._rec_interval,
|
||||
"started_at": self._rec_started_at,
|
||||
"flip_h": self._rec_flip_h,
|
||||
"flip_v": self._rec_flip_v,
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
node_modules
|
||||
dist
|
||||
dist-ssr
|
||||
*.local
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea
|
||||
.DS_Store
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
@@ -0,0 +1,16 @@
|
||||
# React + Vite
|
||||
|
||||
This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
|
||||
|
||||
Currently, two official plugins are available:
|
||||
|
||||
- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Oxc](https://oxc.rs)
|
||||
- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/)
|
||||
|
||||
## React Compiler
|
||||
|
||||
The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation).
|
||||
|
||||
## Expanding the ESLint configuration
|
||||
|
||||
If you are developing a production application, we recommend using TypeScript with type-aware lint rules enabled. Check out the [TS template](https://github.com/vitejs/vite/tree/main/packages/create-vite/template-react-ts) for information on how to integrate TypeScript and [`typescript-eslint`](https://typescript-eslint.io) in your project.
|
||||
@@ -0,0 +1,29 @@
|
||||
import js from '@eslint/js'
|
||||
import globals from 'globals'
|
||||
import reactHooks from 'eslint-plugin-react-hooks'
|
||||
import reactRefresh from 'eslint-plugin-react-refresh'
|
||||
import { defineConfig, globalIgnores } from 'eslint/config'
|
||||
|
||||
export default defineConfig([
|
||||
globalIgnores(['dist']),
|
||||
{
|
||||
files: ['**/*.{js,jsx}'],
|
||||
extends: [
|
||||
js.configs.recommended,
|
||||
reactHooks.configs.flat.recommended,
|
||||
reactRefresh.configs.vite,
|
||||
],
|
||||
languageOptions: {
|
||||
ecmaVersion: 2020,
|
||||
globals: globals.browser,
|
||||
parserOptions: {
|
||||
ecmaVersion: 'latest',
|
||||
ecmaFeatures: { jsx: true },
|
||||
sourceType: 'module',
|
||||
},
|
||||
},
|
||||
rules: {
|
||||
'no-unused-vars': ['error', { varsIgnorePattern: '^[A-Z_]' }],
|
||||
},
|
||||
},
|
||||
])
|
||||
@@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>react</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.jsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
Generated
+3232
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "react",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@react-three/drei": "^10.7.7",
|
||||
"@react-three/fiber": "^9.5.0",
|
||||
"react": "^19.2.4",
|
||||
"react-dom": "^19.2.4",
|
||||
"three": "^0.183.2",
|
||||
"zustand": "^5.0.12"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.39.4",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^6.0.1",
|
||||
"eslint": "^9.39.4",
|
||||
"eslint-plugin-react-hooks": "^7.0.1",
|
||||
"eslint-plugin-react-refresh": "^0.5.2",
|
||||
"globals": "^17.4.0",
|
||||
"vite": "^8.0.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="48" height="46" fill="none" viewBox="0 0 48 46"><path fill="#863bff" d="M25.946 44.938c-.664.845-2.021.375-2.021-.698V33.937a2.26 2.26 0 0 0-2.262-2.262H10.287c-.92 0-1.456-1.04-.92-1.788l7.48-10.471c1.07-1.497 0-3.578-1.842-3.578H1.237c-.92 0-1.456-1.04-.92-1.788L10.013.474c.214-.297.556-.474.92-.474h28.894c.92 0 1.456 1.04.92 1.788l-7.48 10.471c-1.07 1.498 0 3.579 1.842 3.579h11.377c.943 0 1.473 1.088.89 1.83L25.947 44.94z" style="fill:#863bff;fill:color(display-p3 .5252 .23 1);fill-opacity:1"/><mask id="a" width="48" height="46" x="0" y="0" maskUnits="userSpaceOnUse" style="mask-type:alpha"><path fill="#000" d="M25.842 44.938c-.664.844-2.021.375-2.021-.698V33.937a2.26 2.26 0 0 0-2.262-2.262H10.183c-.92 0-1.456-1.04-.92-1.788l7.48-10.471c1.07-1.498 0-3.579-1.842-3.579H1.133c-.92 0-1.456-1.04-.92-1.787L9.91.473c.214-.297.556-.474.92-.474h28.894c.92 0 1.456 1.04.92 1.788l-7.48 10.471c-1.07 1.498 0 3.578 1.842 3.578h11.377c.943 0 1.473 1.088.89 1.832L25.843 44.94z" style="fill:#000;fill-opacity:1"/></mask><g mask="url(#a)"><g filter="url(#b)"><ellipse cx="5.508" cy="14.704" fill="#ede6ff" rx="5.508" ry="14.704" style="fill:#ede6ff;fill:color(display-p3 .9275 .9033 1);fill-opacity:1" transform="matrix(.00324 1 1 -.00324 -4.47 31.516)"/></g><g filter="url(#c)"><ellipse cx="10.399" cy="29.851" fill="#ede6ff" rx="10.399" ry="29.851" style="fill:#ede6ff;fill:color(display-p3 .9275 .9033 1);fill-opacity:1" transform="matrix(.00324 1 1 -.00324 -39.328 7.883)"/></g><g filter="url(#d)"><ellipse cx="5.508" cy="30.487" fill="#7e14ff" rx="5.508" ry="30.487" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="rotate(89.814 -25.913 -14.639)scale(1 -1)"/></g><g filter="url(#e)"><ellipse cx="5.508" cy="30.599" fill="#7e14ff" rx="5.508" ry="30.599" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="rotate(89.814 -32.644 -3.334)scale(1 -1)"/></g><g filter="url(#f)"><ellipse cx="5.508" cy="30.599" fill="#7e14ff" rx="5.508" ry="30.599" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="matrix(.00324 1 1 -.00324 -34.34 30.47)"/></g><g filter="url(#g)"><ellipse cx="14.072" cy="22.078" fill="#ede6ff" rx="14.072" ry="22.078" style="fill:#ede6ff;fill:color(display-p3 .9275 .9033 1);fill-opacity:1" transform="rotate(93.35 24.506 48.493)scale(-1 1)"/></g><g filter="url(#h)"><ellipse cx="3.47" cy="21.501" fill="#7e14ff" rx="3.47" ry="21.501" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="rotate(89.009 28.708 47.59)scale(-1 1)"/></g><g filter="url(#i)"><ellipse cx="3.47" cy="21.501" fill="#7e14ff" rx="3.47" ry="21.501" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="rotate(89.009 28.708 47.59)scale(-1 1)"/></g><g filter="url(#j)"><ellipse cx=".387" cy="8.972" fill="#7e14ff" rx="4.407" ry="29.108" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="rotate(39.51 .387 8.972)"/></g><g filter="url(#k)"><ellipse cx="47.523" cy="-6.092" fill="#7e14ff" rx="4.407" ry="29.108" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="rotate(37.892 47.523 -6.092)"/></g><g filter="url(#l)"><ellipse cx="41.412" cy="6.333" fill="#47bfff" rx="5.971" ry="9.665" style="fill:#47bfff;fill:color(display-p3 .2799 .748 1);fill-opacity:1" transform="rotate(37.892 41.412 6.333)"/></g><g filter="url(#m)"><ellipse cx="-1.879" cy="38.332" fill="#7e14ff" rx="4.407" ry="29.108" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="rotate(37.892 -1.88 38.332)"/></g><g filter="url(#n)"><ellipse cx="-1.879" cy="38.332" fill="#7e14ff" rx="4.407" ry="29.108" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="rotate(37.892 -1.88 38.332)"/></g><g filter="url(#o)"><ellipse cx="35.651" cy="29.907" fill="#7e14ff" rx="4.407" ry="29.108" style="fill:#7e14ff;fill:color(display-p3 .4922 .0767 1);fill-opacity:1" transform="rotate(37.892 35.651 29.907)"/></g><g filter="url(#p)"><ellipse cx="38.418" cy="32.4" fill="#47bfff" rx="5.971" ry="15.297" style="fill:#47bfff;fill:color(display-p3 .2799 .748 1);fill-opacity:1" transform="rotate(37.892 38.418 32.4)"/></g></g><defs><filter id="b" width="60.045" height="41.654" x="-19.77" y="16.149" color-interpolation-filters="sRGB" filterUnits="userSpaceOnUse"><feFlood flood-opacity="0" result="BackgroundImageFix"/><feBlend in="SourceGraphic" in2="BackgroundImageFix" result="shape"/><feGaussianBlur result="effect1_foregroundBlur_2002_17158" stdDeviation="7.659"/></filter><filter id="c" width="90.34" height="51.437" x="-54.613" y="-7.533" color-interpolation-filters="sRGB" filterUnits="userSpaceOnUse"><feFlood flood-opacity="0" result="BackgroundImageFix"/><feBlend in="SourceGraphic" in2="BackgroundImageFix" result="shape"/><feGaussianBlur result="effect1_foregroundBlur_2002_17158" stdDeviation="7.659"/></filtLine truncated
|
||||
|
After Width: | Height: | Size: 9.3 KiB |
@@ -0,0 +1,24 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg">
|
||||
<symbol id="bluesky-icon" viewBox="0 0 16 17">
|
||||
<g clip-path="url(#bluesky-clip)"><path fill="#08060d" d="M7.75 7.735c-.693-1.348-2.58-3.86-4.334-5.097-1.68-1.187-2.32-.981-2.74-.79C.188 2.065.1 2.812.1 3.251s.241 3.602.398 4.13c.52 1.744 2.367 2.333 4.07 2.145-2.495.37-4.71 1.278-1.805 4.512 3.196 3.309 4.38-.71 4.987-2.746.608 2.036 1.307 5.91 4.93 2.746 2.72-2.746.747-4.143-1.747-4.512 1.702.189 3.55-.4 4.07-2.145.156-.528.397-3.691.397-4.13s-.088-1.186-.575-1.406c-.42-.19-1.06-.395-2.741.79-1.755 1.24-3.64 3.752-4.334 5.099"/></g>
|
||||
<defs><clipPath id="bluesky-clip"><path fill="#fff" d="M.1.85h15.3v15.3H.1z"/></clipPath></defs>
|
||||
</symbol>
|
||||
<symbol id="discord-icon" viewBox="0 0 20 19">
|
||||
<path fill="#08060d" d="M16.224 3.768a14.5 14.5 0 0 0-3.67-1.153c-.158.286-.343.67-.47.976a13.5 13.5 0 0 0-4.067 0c-.128-.306-.317-.69-.476-.976A14.4 14.4 0 0 0 3.868 3.77C1.546 7.28.916 10.703 1.231 14.077a14.7 14.7 0 0 0 4.5 2.306q.545-.748.965-1.587a9.5 9.5 0 0 1-1.518-.74q.191-.14.372-.293c2.927 1.369 6.107 1.369 8.999 0q.183.152.372.294-.723.437-1.52.74.418.838.963 1.588a14.6 14.6 0 0 0 4.504-2.308c.37-3.911-.63-7.302-2.644-10.309m-9.13 8.234c-.878 0-1.599-.82-1.599-1.82 0-.998.705-1.82 1.6-1.82.894 0 1.614.82 1.599 1.82.001 1-.705 1.82-1.6 1.82m5.91 0c-.878 0-1.599-.82-1.599-1.82 0-.998.705-1.82 1.6-1.82.893 0 1.614.82 1.599 1.82 0 1-.706 1.82-1.6 1.82"/>
|
||||
</symbol>
|
||||
<symbol id="documentation-icon" viewBox="0 0 21 20">
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="m15.5 13.333 1.533 1.322c.645.555.967.833.967 1.178s-.322.623-.967 1.179L15.5 18.333m-3.333-5-1.534 1.322c-.644.555-.966.833-.966 1.178s.322.623.966 1.179l1.534 1.321"/>
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="M17.167 10.836v-4.32c0-1.41 0-2.117-.224-2.68-.359-.906-1.118-1.621-2.08-1.96-.599-.21-1.349-.21-2.848-.21-2.623 0-3.935 0-4.983.369-1.684.591-3.013 1.842-3.641 3.428C3 6.449 3 7.684 3 10.154v2.122c0 2.558 0 3.838.706 4.726q.306.383.713.671c.76.536 1.79.64 3.581.66"/>
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="M3 10a2.78 2.78 0 0 1 2.778-2.778c.555 0 1.209.097 1.748-.047.48-.129.854-.503.982-.982.145-.54.048-1.194.048-1.749a2.78 2.78 0 0 1 2.777-2.777"/>
|
||||
</symbol>
|
||||
<symbol id="github-icon" viewBox="0 0 19 19">
|
||||
<path fill="#08060d" fill-rule="evenodd" d="M9.356 1.85C5.05 1.85 1.57 5.356 1.57 9.694a7.84 7.84 0 0 0 5.324 7.44c.387.079.528-.168.528-.376 0-.182-.013-.805-.013-1.454-2.165.467-2.616-.935-2.616-.935-.349-.91-.864-1.143-.864-1.143-.71-.48.051-.48.051-.48.787.051 1.2.805 1.2.805.695 1.194 1.817.857 2.268.649.064-.507.27-.857.49-1.052-1.728-.182-3.545-.857-3.545-3.87 0-.857.31-1.558.8-2.104-.078-.195-.349-1 .077-2.078 0 0 .657-.208 2.14.805a7.5 7.5 0 0 1 1.946-.26c.657 0 1.328.092 1.946.26 1.483-1.013 2.14-.805 2.14-.805.426 1.078.155 1.883.078 2.078.502.546.799 1.247.799 2.104 0 3.013-1.818 3.675-3.558 3.87.284.247.528.714.528 1.454 0 1.052-.012 1.896-.012 2.156 0 .208.142.455.528.377a7.84 7.84 0 0 0 5.324-7.441c.013-4.338-3.48-7.844-7.773-7.844" clip-rule="evenodd"/>
|
||||
</symbol>
|
||||
<symbol id="social-icon" viewBox="0 0 20 20">
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="M12.5 6.667a4.167 4.167 0 1 0-8.334 0 4.167 4.167 0 0 0 8.334 0"/>
|
||||
<path fill="none" stroke="#aa3bff" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.35" d="M2.5 16.667a5.833 5.833 0 0 1 8.75-5.053m3.837.474.513 1.035c.07.144.257.282.414.309l.93.155c.596.1.736.536.307.965l-.723.73a.64.64 0 0 0-.152.531l.207.903c.164.715-.213.991-.84.618l-.872-.52a.63.63 0 0 0-.577 0l-.872.52c-.624.373-1.003.094-.84-.618l.207-.903a.64.64 0 0 0-.152-.532l-.723-.729c-.426-.43-.289-.864.306-.964l.93-.156a.64.64 0 0 0 .412-.31l.513-1.034c.28-.562.735-.562 1.012 0"/>
|
||||
</symbol>
|
||||
<symbol id="x-icon" viewBox="0 0 19 19">
|
||||
<path fill="#08060d" fill-rule="evenodd" d="M1.893 1.98c.052.072 1.245 1.769 2.653 3.77l2.892 4.114c.183.261.333.48.333.486s-.068.089-.152.183l-.522.593-.765.867-3.597 4.087c-.375.426-.734.834-.798.905a1 1 0 0 0-.118.148c0 .01.236.017.664.017h.663l.729-.83c.4-.457.796-.906.879-.999a692 692 0 0 0 1.794-2.038c.034-.037.301-.34.594-.675l.551-.624.345-.392a7 7 0 0 1 .34-.374c.006 0 .93 1.306 2.052 2.903l2.084 2.965.045.063h2.275c1.87 0 2.273-.003 2.266-.021-.008-.02-1.098-1.572-3.894-5.547-2.013-2.862-2.28-3.246-2.273-3.266.008-.019.282-.332 2.085-2.38l2-2.274 1.567-1.782c.022-.028-.016-.03-.65-.03h-.674l-.3.342a871 871 0 0 1-1.782 2.025c-.067.075-.405.458-.75.852a100 100 0 0 1-.803.91c-.148.172-.299.344-.99 1.127-.304.343-.32.358-.345.327-.015-.019-.904-1.282-1.976-2.808L6.365 1.85H1.8zm1.782.91 8.078 11.294c.772 1.08 1.413 1.973 1.425 1.984.016.017.241.02 1.05.017l1.03-.004-2.694-3.766L7.796 5.75 5.722 2.852l-1.039-.004-1.039-.004z" clip-rule="evenodd"/>
|
||||
</symbol>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.9 KiB |
@@ -0,0 +1,117 @@
|
||||
import { useState, useCallback, useRef, useEffect } from 'react'
|
||||
import { useStore } from './store'
|
||||
import { useWebSocket } from './hooks/useWebSocket'
|
||||
import { useApi } from './hooks/useApi'
|
||||
import { Header } from './components/Header'
|
||||
import { CameraControls } from './components/CameraControls'
|
||||
import { StreamView } from './components/StreamView'
|
||||
import { DepthView } from './components/DepthView'
|
||||
import { PointCloudView } from './components/PointCloudView'
|
||||
import { SplitView } from './components/SplitView'
|
||||
import { StatusBar } from './components/StatusBar'
|
||||
import { CalibrationPanel } from './components/CalibrationPanel'
|
||||
|
||||
export default function App() {
|
||||
const [viewMode, setViewMode] = useState('rgb')
|
||||
const [showCalib, setShowCalib] = useState(false)
|
||||
const {
|
||||
setCurrentFrame, setFps, setStreaming, flipH, flipV, toggleFlipH, toggleFlipV,
|
||||
setConnected, setCameraOpen, setIntrinsics, setDevices, setSelectedDevice,
|
||||
} = useStore()
|
||||
const frame = useStore((s) => s.currentFrame)
|
||||
const fpsCounter = useRef({ count: 0, lastTime: Date.now() })
|
||||
const api = useApi()
|
||||
|
||||
const onFrame = useCallback((data) => {
|
||||
setCurrentFrame(data)
|
||||
fpsCounter.current.count++
|
||||
const now = Date.now()
|
||||
if (now - fpsCounter.current.lastTime >= 1000) {
|
||||
setFps(fpsCounter.current.count)
|
||||
fpsCounter.current.count = 0
|
||||
fpsCounter.current.lastTime = now
|
||||
}
|
||||
}, [setCurrentFrame, setFps])
|
||||
|
||||
const { start, stop, setPredict, setConf } = useWebSocket(onFrame)
|
||||
|
||||
const handleStart = () => { start(); setStreaming(true) }
|
||||
const handleStop = () => { stop(); setStreaming(false) }
|
||||
|
||||
// Sync GUI state with server on mount: if a camera is already open, reflect it
|
||||
// and auto-start the WebSocket stream so the user sees the live image.
|
||||
const syncedRef = useRef(false)
|
||||
useEffect(() => {
|
||||
if (syncedRef.current) return
|
||||
syncedRef.current = true
|
||||
;(async () => {
|
||||
const res = await api.status()
|
||||
if (!res?.success) return
|
||||
const { sdk_initialized, camera_open, intrinsics: sIntr } = res.data || {}
|
||||
if (sdk_initialized) setConnected(true)
|
||||
if (camera_open) {
|
||||
setCameraOpen(true)
|
||||
if (sIntr) setIntrinsics(sIntr)
|
||||
const disc = await api.discover()
|
||||
if (disc?.success) {
|
||||
setDevices(disc.data.cameras)
|
||||
if (disc.data.cameras.length > 0) {
|
||||
setSelectedDevice(disc.data.cameras[0].index)
|
||||
}
|
||||
}
|
||||
start()
|
||||
setStreaming(true)
|
||||
}
|
||||
})()
|
||||
}, [api, setConnected, setCameraOpen, setIntrinsics, setDevices, setSelectedDevice, setStreaming, start])
|
||||
|
||||
const tabStyle = (active) => ({
|
||||
padding: '6px 12px',
|
||||
background: active ? 'var(--accent)' : 'var(--bg-card)',
|
||||
color: 'var(--text)',
|
||||
border: 'none',
|
||||
borderRadius: 'var(--radius)',
|
||||
cursor: 'pointer',
|
||||
fontSize: '12px',
|
||||
})
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<Header />
|
||||
<div style={{ display: 'flex', flexDirection: 'column', overflow: 'hidden' }}>
|
||||
<CameraControls
|
||||
onStartStream={handleStart}
|
||||
onStopStream={handleStop}
|
||||
onSetPredict={setPredict}
|
||||
onSetConf={setConf}
|
||||
/>
|
||||
<div style={{
|
||||
display: 'flex', gap: '8px', padding: '8px 12px',
|
||||
background: 'var(--bg-panel)', borderBottom: '1px solid var(--border)',
|
||||
}}>
|
||||
<button style={tabStyle(viewMode === 'rgb')} onClick={() => setViewMode('rgb')}>RGB</button>
|
||||
<button style={tabStyle(viewMode === 'depth')} onClick={() => setViewMode('depth')}>Depth</button>
|
||||
<button style={tabStyle(viewMode === '3d')} onClick={() => setViewMode('3d')}>3D</button>
|
||||
<button style={tabStyle(viewMode === 'split')} onClick={() => setViewMode('split')}>RGB+3D</button>
|
||||
<div style={{ marginLeft: 'auto', display: 'flex', gap: '6px', alignItems: 'center' }}>
|
||||
<button
|
||||
style={tabStyle(false)}
|
||||
onClick={() => setShowCalib(true)}
|
||||
title="Mostra parametri di calibrazione (intrinsics, distortion, extrinsics)"
|
||||
>Calibration</button>
|
||||
<button style={tabStyle(flipH)} onClick={toggleFlipH} title="Mirror orizzontale">↔ H</button>
|
||||
<button style={tabStyle(flipV)} onClick={toggleFlipV} title="Mirror verticale">↕ V</button>
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ flex: 1, overflow: 'hidden' }}>
|
||||
{viewMode === 'rgb' && <StreamView frame={frame} type="rgb" flipH={flipH} flipV={flipV} />}
|
||||
{viewMode === 'depth' && <DepthView frame={frame} flipH={flipH} flipV={flipV} />}
|
||||
{viewMode === '3d' && <PointCloudView flipH={flipH} flipV={flipV} />}
|
||||
{viewMode === 'split' && <SplitView frame={frame} flipH={flipH} flipV={flipV} />}
|
||||
</div>
|
||||
</div>
|
||||
<StatusBar />
|
||||
{showCalib && <CalibrationPanel onClose={() => setShowCalib(false)} />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import { useStore } from '../store'
|
||||
|
||||
const fmt = (v, digits = 4) =>
|
||||
v === null || v === undefined || Number.isNaN(v) ? '—' : Number(v).toFixed(digits)
|
||||
|
||||
function Row({ label, value, mono = true }) {
|
||||
return (
|
||||
<div style={{
|
||||
display: 'flex', justifyContent: 'space-between',
|
||||
padding: '3px 0', gap: '16px', fontSize: '12px',
|
||||
}}>
|
||||
<span style={{ color: 'var(--text-muted)' }}>{label}</span>
|
||||
<span style={{
|
||||
color: 'var(--text)',
|
||||
fontFamily: mono ? 'var(--font-mono)' : 'inherit',
|
||||
}}>{value}</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Section({ title, children }) {
|
||||
return (
|
||||
<div style={{ marginBottom: '14px' }}>
|
||||
<div style={{
|
||||
fontSize: '11px', fontWeight: 600,
|
||||
color: 'var(--accent)', textTransform: 'uppercase',
|
||||
letterSpacing: '0.5px', marginBottom: '6px',
|
||||
}}>{title}</div>
|
||||
{children}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Sensor({ label, data }) {
|
||||
if (!data) return null
|
||||
const d = data.distortion
|
||||
return (
|
||||
<Section title={label}>
|
||||
<Row label="fx" value={fmt(data.fx, 2)} />
|
||||
<Row label="fy" value={fmt(data.fy, 2)} />
|
||||
<Row label="cx" value={fmt(data.cx, 2)} />
|
||||
<Row label="cy" value={fmt(data.cy, 2)} />
|
||||
{d ? (
|
||||
<>
|
||||
<Row label="k1" value={fmt(d.k1, 6)} />
|
||||
<Row label="k2" value={fmt(d.k2, 6)} />
|
||||
<Row label="k3" value={fmt(d.k3, 6)} />
|
||||
<Row label="p1" value={fmt(d.p1, 6)} />
|
||||
<Row label="p2" value={fmt(d.p2, 6)} />
|
||||
</>
|
||||
) : (
|
||||
<Row label="distortion" value="n/a" />
|
||||
)}
|
||||
</Section>
|
||||
)
|
||||
}
|
||||
|
||||
function Extrinsics({ data }) {
|
||||
if (!data) return null
|
||||
const R = data.rotation
|
||||
const T = data.translation
|
||||
const cell = {
|
||||
flex: 1, textAlign: 'right',
|
||||
fontFamily: 'var(--font-mono)', fontSize: '11px',
|
||||
padding: '2px 6px', color: 'var(--text)',
|
||||
}
|
||||
return (
|
||||
<Section title="Extrinsics IR → RGB">
|
||||
<div style={{ fontSize: '11px', color: 'var(--text-muted)', marginBottom: '4px' }}>
|
||||
Rotation (row-major)
|
||||
</div>
|
||||
<div style={{
|
||||
background: 'var(--bg-input)', borderRadius: 'var(--radius)',
|
||||
padding: '4px', marginBottom: '8px',
|
||||
}}>
|
||||
{[0, 1, 2].map((r) => (
|
||||
<div key={r} style={{ display: 'flex' }}>
|
||||
{[0, 1, 2].map((c) => (
|
||||
<span key={c} style={cell}>{fmt(R[r * 3 + c], 5)}</span>
|
||||
))}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<div style={{ fontSize: '11px', color: 'var(--text-muted)', marginBottom: '4px' }}>
|
||||
Translation (mm)
|
||||
</div>
|
||||
<Row label="tx" value={fmt(T[0], 3)} />
|
||||
<Row label="ty" value={fmt(T[1], 3)} />
|
||||
<Row label="tz" value={fmt(T[2], 3)} />
|
||||
</Section>
|
||||
)
|
||||
}
|
||||
|
||||
export function CalibrationPanel({ onClose }) {
|
||||
const intrinsics = useStore((s) => s.intrinsics)
|
||||
|
||||
const copyJson = () => {
|
||||
navigator.clipboard?.writeText(JSON.stringify(intrinsics, null, 2))
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
onClick={onClose}
|
||||
style={{
|
||||
position: 'fixed', inset: 0,
|
||||
background: 'rgba(0,0,0,0.5)',
|
||||
display: 'flex', alignItems: 'center', justifyContent: 'center',
|
||||
zIndex: 1000,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
style={{
|
||||
background: 'var(--bg-panel)',
|
||||
border: '1px solid var(--border)',
|
||||
borderRadius: 'var(--radius)',
|
||||
padding: '20px',
|
||||
width: '420px',
|
||||
maxHeight: '85vh',
|
||||
overflowY: 'auto',
|
||||
boxShadow: '0 10px 40px rgba(0,0,0,0.5)',
|
||||
}}
|
||||
>
|
||||
<div style={{
|
||||
display: 'flex', alignItems: 'center', justifyContent: 'space-between',
|
||||
marginBottom: '14px',
|
||||
}}>
|
||||
<h2 style={{ fontSize: '15px', fontWeight: 600, color: 'var(--text)' }}>
|
||||
Camera Calibration
|
||||
</h2>
|
||||
<div style={{ display: 'flex', gap: '6px' }}>
|
||||
<button
|
||||
onClick={copyJson}
|
||||
disabled={!intrinsics}
|
||||
style={{
|
||||
padding: '4px 10px', fontSize: '11px',
|
||||
background: 'var(--bg-card)', color: 'var(--text)',
|
||||
border: '1px solid var(--border)', borderRadius: 'var(--radius)',
|
||||
cursor: intrinsics ? 'pointer' : 'not-allowed',
|
||||
}}
|
||||
title="Copia JSON negli appunti"
|
||||
>Copy JSON</button>
|
||||
<button
|
||||
onClick={onClose}
|
||||
style={{
|
||||
padding: '4px 10px', fontSize: '11px',
|
||||
background: 'var(--bg-card)', color: 'var(--text)',
|
||||
border: '1px solid var(--border)', borderRadius: 'var(--radius)',
|
||||
cursor: 'pointer',
|
||||
}}
|
||||
>Close</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!intrinsics ? (
|
||||
<div style={{ color: 'var(--text-muted)', fontSize: '13px', padding: '20px 0' }}>
|
||||
Calibration non disponibile. Apri la camera per caricare i parametri.
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<Sensor label="RGB" data={intrinsics.rgb} />
|
||||
<Sensor label="IR / Depth" data={intrinsics.ir} />
|
||||
<Extrinsics data={intrinsics.extrinsics_ir_to_rgb} />
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
import { useState } from 'react'
|
||||
import { useStore } from '../store'
|
||||
import { useApi } from '../hooks/useApi'
|
||||
|
||||
export function CameraControls({ onStartStream, onStopStream, onSetPredict, onSetConf }) {
|
||||
const {
|
||||
devices, setDevices, selectedDevice, setSelectedDevice,
|
||||
cameraOpen, setCameraOpen, setConnected, setIntrinsics,
|
||||
flipH, flipV, recording, setRecording,
|
||||
recInterval, setRecInterval,
|
||||
predicting, setPredicting,
|
||||
confThreshold, setConfThreshold,
|
||||
} = useStore()
|
||||
const api = useApi()
|
||||
const [loading, setLoading] = useState(false)
|
||||
|
||||
const handleSnapshot = async () => {
|
||||
const res = await api.snapshot(flipH, flipV)
|
||||
if (res.success) {
|
||||
console.log('Snapshot saved:', res.data.filename)
|
||||
} else {
|
||||
alert(`Snapshot failed: ${res.error?.message ?? 'unknown error'}`)
|
||||
}
|
||||
}
|
||||
|
||||
const handleRecToggle = async () => {
|
||||
if (!recording) {
|
||||
const res = await api.startRec(flipH, flipV, recInterval)
|
||||
if (res.success) setRecording(true)
|
||||
else alert(`REC start failed: ${res.error?.message ?? 'unknown error'}`)
|
||||
} else {
|
||||
const res = await api.stopRec()
|
||||
if (res.success) {
|
||||
setRecording(false)
|
||||
console.log(`REC stopped: ${res.data.frames_written} frames`)
|
||||
} else {
|
||||
alert(`REC stop failed: ${res.error?.message ?? 'unknown error'}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const handleDiscover = async () => {
|
||||
setLoading(true)
|
||||
const res = await api.discover()
|
||||
if (res.success) {
|
||||
setDevices(res.data.cameras)
|
||||
setConnected(true)
|
||||
}
|
||||
setLoading(false)
|
||||
}
|
||||
|
||||
const handleOpen = async () => {
|
||||
setLoading(true)
|
||||
const res = await api.openCamera(selectedDevice)
|
||||
if (res.success) {
|
||||
setCameraOpen(true)
|
||||
if (res.data.intrinsics) setIntrinsics(res.data.intrinsics)
|
||||
if (typeof res.data.device_index === 'number') {
|
||||
setSelectedDevice(res.data.device_index)
|
||||
}
|
||||
} else {
|
||||
alert(`Open failed: ${res.error?.message ?? 'unknown error'}`)
|
||||
}
|
||||
setLoading(false)
|
||||
}
|
||||
|
||||
const handleClose = async () => {
|
||||
onStopStream()
|
||||
const res = await api.closeCamera()
|
||||
if (res.success) setCameraOpen(false)
|
||||
}
|
||||
|
||||
const handlePredictToggle = () => {
|
||||
const next = !predicting
|
||||
setPredicting(next)
|
||||
onSetPredict?.(next)
|
||||
}
|
||||
|
||||
const handleConfChange = (e) => {
|
||||
const v = Number(e.target.value)
|
||||
setConfThreshold(v)
|
||||
onSetConf?.(v)
|
||||
}
|
||||
|
||||
const btnStyle = (variant = 'default') => ({
|
||||
padding: '10px 20px',
|
||||
borderRadius: 'var(--radius)',
|
||||
border: 'none',
|
||||
cursor: loading ? 'wait' : 'pointer',
|
||||
fontSize: '14px',
|
||||
fontWeight: 600,
|
||||
display: 'inline-flex',
|
||||
alignItems: 'center',
|
||||
gap: '6px',
|
||||
background: variant === 'accent' ? 'var(--accent)' :
|
||||
variant === 'danger' ? 'var(--danger)' :
|
||||
variant === 'success' ? '#22c55e' :
|
||||
variant === 'open' ? '#3b82f6' : 'var(--bg-card)',
|
||||
color: '#fff',
|
||||
})
|
||||
|
||||
return (
|
||||
<div style={{
|
||||
background: 'var(--bg-panel)',
|
||||
padding: '12px',
|
||||
display: 'flex',
|
||||
gap: '8px',
|
||||
alignItems: 'center',
|
||||
borderBottom: '1px solid var(--border)',
|
||||
}}>
|
||||
<button onClick={handleDiscover} disabled={loading} style={btnStyle('accent')}>
|
||||
Discover
|
||||
</button>
|
||||
|
||||
{devices.length > 0 && (
|
||||
<select
|
||||
value={selectedDevice}
|
||||
onChange={(e) => setSelectedDevice(Number(e.target.value))}
|
||||
style={{
|
||||
background: 'var(--bg-input)', color: 'var(--text)',
|
||||
border: '1px solid var(--border)', borderRadius: 'var(--radius)',
|
||||
padding: '6px 8px', fontSize: '12px',
|
||||
}}
|
||||
>
|
||||
{devices.map((d) => (
|
||||
<option key={d.index} value={d.index}>
|
||||
[{d.index}] {d.serial} (bus {d.bus})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
|
||||
{!cameraOpen ? (
|
||||
<button onClick={handleOpen} disabled={loading || devices.length === 0} style={btnStyle('open')}>
|
||||
<span>⏻</span> Open
|
||||
</button>
|
||||
) : (
|
||||
<>
|
||||
<button onClick={onStartStream} style={btnStyle('success')}>
|
||||
<span>▶</span> Start
|
||||
</button>
|
||||
<button onClick={onStopStream} style={btnStyle()}>
|
||||
<span>■</span> Stop
|
||||
</button>
|
||||
<button onClick={handleSnapshot} style={btnStyle('accent')} title="Salva frame corrente">
|
||||
<span>📸</span> Snapshot
|
||||
</button>
|
||||
<label style={{
|
||||
display: 'inline-flex', alignItems: 'center', gap: '4px',
|
||||
color: 'var(--text)', fontSize: '12px',
|
||||
}} title="Salva un frame ogni N frame streamati">
|
||||
every
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={600}
|
||||
value={recInterval}
|
||||
disabled={recording}
|
||||
onChange={(e) => setRecInterval(Math.max(1, Number(e.target.value) || 1))}
|
||||
style={{
|
||||
width: '56px',
|
||||
background: 'var(--bg-input)', color: 'var(--text)',
|
||||
border: '1px solid var(--border)', borderRadius: 'var(--radius)',
|
||||
padding: '4px 6px', fontSize: '12px',
|
||||
}}
|
||||
/>
|
||||
frame
|
||||
</label>
|
||||
<button onClick={handleRecToggle} style={btnStyle('danger')} title="Registrazione continua">
|
||||
<span>{recording ? '\u25A0' : '\u25CF'}</span> {recording ? 'Stop REC' : 'REC'}
|
||||
</button>
|
||||
<button
|
||||
onClick={handlePredictToggle}
|
||||
style={btnStyle(predicting ? 'success' : 'accent')}
|
||||
title="Attiva/disattiva inferenza YOLO"
|
||||
>
|
||||
<span>🎯</span> {predicting ? 'Predict ON' : 'Predict'}
|
||||
</button>
|
||||
<label style={{
|
||||
display: 'inline-flex', alignItems: 'center', gap: '6px',
|
||||
color: 'var(--text)', fontSize: '12px',
|
||||
}} title="Soglia di confidenza YOLO">
|
||||
conf
|
||||
<input
|
||||
type="range"
|
||||
min={0.05}
|
||||
max={0.95}
|
||||
step={0.05}
|
||||
value={confThreshold}
|
||||
onChange={handleConfChange}
|
||||
style={{ width: '100px' }}
|
||||
/>
|
||||
<span style={{ width: '32px', textAlign: 'right' }}>{confThreshold.toFixed(2)}</span>
|
||||
</label>
|
||||
<button onClick={handleClose} style={btnStyle('danger')}>
|
||||
<span>✕</span> Close
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { useRef, useEffect } from 'react'
|
||||
import { useZoomPan } from '../hooks/useZoomPan'
|
||||
|
||||
export function DepthView({ frame, flipH = false, flipV = false }) {
|
||||
const canvasRef = useRef(null)
|
||||
const { zoom, pan, handlers } = useZoomPan()
|
||||
|
||||
useEffect(() => {
|
||||
if (!frame || !canvasRef.current) return
|
||||
const b64 = frame.depth_color_b64 || frame.depth_png_b64
|
||||
if (!b64) return
|
||||
const canvas = canvasRef.current
|
||||
const ctx = canvas.getContext('2d')
|
||||
const mime = frame.depth_color_b64 ? 'jpeg' : 'png'
|
||||
const img = new Image()
|
||||
img.onload = () => {
|
||||
canvas.width = img.width
|
||||
canvas.height = img.height
|
||||
ctx.drawImage(img, 0, 0)
|
||||
}
|
||||
img.src = `data:image/${mime};base64,${b64}`
|
||||
}, [frame])
|
||||
|
||||
return (
|
||||
<div
|
||||
{...handlers}
|
||||
style={{
|
||||
width: '100%', height: '100%', overflow: 'hidden', background: '#000',
|
||||
cursor: zoom > 1 ? 'grab' : 'default', touchAction: 'none',
|
||||
}}
|
||||
>
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
style={{
|
||||
width: '100%', height: '100%', objectFit: 'contain',
|
||||
transform: `translate(${pan.x}px, ${pan.y}px) scale(${zoom}) scaleX(${flipH ? -1 : 1}) scaleY(${flipV ? -1 : 1})`,
|
||||
transformOrigin: 'center center',
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { useStore } from '../store'
|
||||
|
||||
export function Header() {
|
||||
const { connected, cameraOpen, streaming, natsConnected } = useStore()
|
||||
|
||||
return (
|
||||
<header style={{
|
||||
background: 'var(--bg-panel)',
|
||||
borderBottom: '1px solid var(--border)',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
padding: '0 16px',
|
||||
gap: '16px',
|
||||
}}>
|
||||
<h1 style={{ fontSize: '15px', fontWeight: 600 }}>Nuwa HP60C</h1>
|
||||
<div style={{ display: 'flex', gap: '12px', marginLeft: 'auto', fontSize: '12px' }}>
|
||||
<Indicator label="SDK" active={connected} />
|
||||
<Indicator label="Camera" active={cameraOpen} />
|
||||
<Indicator label="Stream" active={streaming} />
|
||||
<Indicator label="NATS" active={natsConnected} />
|
||||
</div>
|
||||
</header>
|
||||
)
|
||||
}
|
||||
|
||||
function Indicator({ label, active }) {
|
||||
return (
|
||||
<span style={{ display: 'flex', alignItems: 'center', gap: '4px', color: 'var(--text-muted)' }}>
|
||||
<span style={{
|
||||
width: 8, height: 8, borderRadius: '50%',
|
||||
background: active ? 'var(--success)' : 'var(--text-dim)',
|
||||
}} />
|
||||
{label}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import { useRef, useState, useEffect, useCallback } from 'react'
|
||||
import { Canvas } from '@react-three/fiber'
|
||||
import { OrbitControls } from '@react-three/drei'
|
||||
import * as THREE from 'three'
|
||||
import { apiFetch } from '../hooks/useApi'
|
||||
import { useStore } from '../store'
|
||||
|
||||
function MeasurementMarker({ measurement }) {
|
||||
if (!measurement || measurement.error || !measurement.point_m) return null
|
||||
const { x, y, z } = measurement.point_m
|
||||
// /pointcloud stores points as [x, -y, -z] — match that convention here.
|
||||
return (
|
||||
<group position={[x, -y, -z]}>
|
||||
<mesh>
|
||||
<sphereGeometry args={[0.015, 16, 16]} />
|
||||
<meshBasicMaterial color="#ffcc00" />
|
||||
</mesh>
|
||||
<mesh>
|
||||
<sphereGeometry args={[0.025, 16, 16]} />
|
||||
<meshBasicMaterial color="#ffcc00" transparent opacity={0.25} />
|
||||
</mesh>
|
||||
</group>
|
||||
)
|
||||
}
|
||||
|
||||
function PointCloud({ points }) {
|
||||
const geometry = useRef(new THREE.BufferGeometry())
|
||||
|
||||
useEffect(() => {
|
||||
if (!points || points.length === 0) return
|
||||
|
||||
const pos = new Float32Array(points.length * 3)
|
||||
const col = new Float32Array(points.length * 3)
|
||||
|
||||
for (let i = 0; i < points.length; i++) {
|
||||
const p = points[i]
|
||||
pos[i * 3] = p[0]
|
||||
pos[i * 3 + 1] = p[1]
|
||||
pos[i * 3 + 2] = p[2]
|
||||
col[i * 3] = p[3] / 255
|
||||
col[i * 3 + 1] = p[4] / 255
|
||||
col[i * 3 + 2] = p[5] / 255
|
||||
}
|
||||
|
||||
geometry.current.setAttribute('position', new THREE.Float32BufferAttribute(pos, 3))
|
||||
geometry.current.setAttribute('color', new THREE.Float32BufferAttribute(col, 3))
|
||||
geometry.current.computeBoundingSphere()
|
||||
}, [points])
|
||||
|
||||
if (!points || points.length === 0) return null
|
||||
|
||||
return (
|
||||
<points>
|
||||
<primitive object={geometry.current} attach="geometry" />
|
||||
<pointsMaterial size={0.005} vertexColors sizeAttenuation />
|
||||
</points>
|
||||
)
|
||||
}
|
||||
|
||||
export function PointCloudView({ flipH = false, flipV = false }) {
|
||||
const [points, setPoints] = useState(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [autoRefresh, setAutoRefresh] = useState(false)
|
||||
const intervalRef = useRef(null)
|
||||
const measurement = useStore((s) => s.measurement)
|
||||
|
||||
const fetchPoints = useCallback(async () => {
|
||||
setLoading(true)
|
||||
try {
|
||||
const res = await apiFetch('/pointcloud?step=4&max_depth_mm=4000')
|
||||
if (res.success) {
|
||||
setPoints(res.data.points)
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Point cloud fetch error:', e)
|
||||
}
|
||||
setLoading(false)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
fetchPoints()
|
||||
}, [fetchPoints])
|
||||
|
||||
useEffect(() => {
|
||||
if (autoRefresh) {
|
||||
intervalRef.current = setInterval(fetchPoints, 500)
|
||||
} else {
|
||||
clearInterval(intervalRef.current)
|
||||
}
|
||||
return () => clearInterval(intervalRef.current)
|
||||
}, [autoRefresh, fetchPoints])
|
||||
|
||||
return (
|
||||
<div style={{ width: '100%', height: '100%', background: '#000', position: 'relative' }}>
|
||||
<div style={{
|
||||
position: 'absolute', top: 8, left: 8, zIndex: 10,
|
||||
display: 'flex', gap: '8px', alignItems: 'center',
|
||||
}}>
|
||||
<button
|
||||
onClick={fetchPoints}
|
||||
disabled={loading}
|
||||
style={{
|
||||
padding: '4px 10px', fontSize: '11px', borderRadius: '4px',
|
||||
border: 'none', background: 'var(--accent)', color: 'var(--text)',
|
||||
cursor: 'pointer', opacity: loading ? 0.5 : 1,
|
||||
}}
|
||||
>
|
||||
{loading ? '...' : 'Refresh'}
|
||||
</button>
|
||||
<label style={{ fontSize: '11px', color: 'var(--text-muted)', display: 'flex', alignItems: 'center', gap: '4px' }}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={autoRefresh}
|
||||
onChange={(e) => setAutoRefresh(e.target.checked)}
|
||||
/>
|
||||
Auto (2fps)
|
||||
</label>
|
||||
{points && (
|
||||
<span style={{ fontSize: '11px', color: 'var(--text-dim)' }}>
|
||||
{points.length.toLocaleString()} pts
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<Canvas camera={{ position: [0, 0, -2], fov: 60, near: 0.01, far: 100 }}>
|
||||
<ambientLight intensity={0.5} />
|
||||
<group scale={[flipH ? -1 : 1, flipV ? -1 : 1, 1]}>
|
||||
<PointCloud points={points} />
|
||||
<MeasurementMarker measurement={measurement} />
|
||||
</group>
|
||||
<OrbitControls enableDamping dampingFactor={0.1} />
|
||||
</Canvas>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { StreamView } from './StreamView'
|
||||
import { PointCloudView } from './PointCloudView'
|
||||
|
||||
export function SplitView({ frame, flipH = false, flipV = false }) {
|
||||
return (
|
||||
<div style={{
|
||||
width: '100%', height: '100%',
|
||||
display: 'grid', gridTemplateColumns: '1fr 1fr',
|
||||
gap: '2px', background: 'var(--border)',
|
||||
}}>
|
||||
<div style={{ position: 'relative', minWidth: 0, minHeight: 0 }}>
|
||||
<Label>RGB · click to measure</Label>
|
||||
<StreamView frame={frame} type="rgb" flipH={flipH} flipV={flipV} />
|
||||
</div>
|
||||
<div style={{ position: 'relative', minWidth: 0, minHeight: 0 }}>
|
||||
<Label>Point Cloud · yellow sphere = measured point</Label>
|
||||
<PointCloudView flipH={flipH} flipV={flipV} />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Label({ children }) {
|
||||
return (
|
||||
<div style={{
|
||||
position: 'absolute', top: 8, right: 8, zIndex: 11,
|
||||
padding: '3px 8px', background: 'rgba(0,0,0,0.65)',
|
||||
color: 'var(--text-dim)', fontSize: '10px',
|
||||
borderRadius: '4px', border: '1px solid var(--border)',
|
||||
fontFamily: 'monospace', pointerEvents: 'none',
|
||||
}}>{children}</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { useStore } from '../store'
|
||||
|
||||
export function StatusBar() {
|
||||
const { fps, frameCount, currentFrame } = useStore()
|
||||
|
||||
return (
|
||||
<footer style={{
|
||||
background: 'var(--bg-panel)',
|
||||
borderTop: '1px solid var(--border)',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
padding: '0 16px',
|
||||
gap: '16px',
|
||||
fontSize: '11px',
|
||||
color: 'var(--text-muted)',
|
||||
fontFamily: 'var(--font-mono)',
|
||||
}}>
|
||||
<span>FPS: {fps}</span>
|
||||
<span>Frames: {frameCount}</span>
|
||||
{currentFrame && (
|
||||
<>
|
||||
<span>Depth: {currentFrame.depth_min_mm}-{currentFrame.depth_max_mm}mm</span>
|
||||
<span>Valid px: {currentFrame.depth_valid_pixels}</span>
|
||||
</>
|
||||
)}
|
||||
</footer>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
import { useRef, useEffect, useState, useCallback } from 'react'
|
||||
import { useZoomPan } from '../hooks/useZoomPan'
|
||||
import { apiFetch } from '../hooks/useApi'
|
||||
import { useStore } from '../store'
|
||||
|
||||
const MEASURE_RADIUS = 6
|
||||
|
||||
function drawDetections(ctx, detections) {
|
||||
ctx.lineWidth = 2
|
||||
ctx.strokeStyle = '#22c55e'
|
||||
ctx.fillStyle = '#22c55e'
|
||||
ctx.font = '14px sans-serif'
|
||||
ctx.textBaseline = 'top'
|
||||
for (const d of detections) {
|
||||
const [x1, y1, x2, y2] = d.bbox
|
||||
const w = x2 - x1
|
||||
const h = y2 - y1
|
||||
ctx.strokeRect(x1, y1, w, h)
|
||||
const label = `${d.label} ${(d.conf * 100).toFixed(0)}%`
|
||||
const padding = 3
|
||||
const textW = ctx.measureText(label).width
|
||||
const tagH = 16
|
||||
const tagY = y1 - tagH >= 0 ? y1 - tagH : y1
|
||||
ctx.fillRect(x1, tagY, textW + padding * 2, tagH)
|
||||
ctx.fillStyle = '#000'
|
||||
ctx.fillText(label, x1 + padding, tagY + 1)
|
||||
ctx.fillStyle = '#22c55e'
|
||||
}
|
||||
}
|
||||
|
||||
export function StreamView({ frame, type = 'rgb', flipH = false, flipV = false }) {
|
||||
const canvasRef = useRef(null)
|
||||
const imgSizeRef = useRef({ w: 0, h: 0 })
|
||||
const [pending, setPending] = useState(false)
|
||||
const measurement = useStore((s) => s.measurement)
|
||||
const setMeasurement = useStore((s) => s.setMeasurement)
|
||||
|
||||
const canMeasure = type === 'rgb'
|
||||
|
||||
const handleTap = useCallback(async (e) => {
|
||||
if (!canMeasure || !canvasRef.current) return
|
||||
const canvas = canvasRef.current
|
||||
const cw = canvas.width
|
||||
const ch = canvas.height
|
||||
if (!cw || !ch) return
|
||||
const rect = canvas.getBoundingClientRect()
|
||||
if (rect.width === 0 || rect.height === 0) return
|
||||
|
||||
// objectFit: contain letterboxes the canvas content inside the CSS rect.
|
||||
// Compute the actual drawn content rect so clicks map to image pixels.
|
||||
const scale = Math.min(rect.width / cw, rect.height / ch)
|
||||
const dw = cw * scale
|
||||
const dh = ch * scale
|
||||
const ox = rect.left + (rect.width - dw) / 2
|
||||
const oy = rect.top + (rect.height - dh) / 2
|
||||
|
||||
let fx = (e.clientX - ox) / dw
|
||||
let fy = (e.clientY - oy) / dh
|
||||
if (fx < 0 || fx > 1 || fy < 0 || fy > 1) return // click in letterbox
|
||||
|
||||
// CSS flips invert the visual axis; undo to recover raw image coords.
|
||||
if (flipH) fx = 1 - fx
|
||||
if (flipV) fy = 1 - fy
|
||||
const u = Math.max(0, Math.min(cw - 1, Math.round(fx * cw)))
|
||||
const v = Math.max(0, Math.min(ch - 1, Math.round(fy * ch)))
|
||||
|
||||
setPending(true)
|
||||
try {
|
||||
const res = await apiFetch(`/measure?u=${u}&v=${v}&radius=${MEASURE_RADIUS}`)
|
||||
if (res.success) {
|
||||
setMeasurement({ ...res.data, ts: Date.now() })
|
||||
} else {
|
||||
setMeasurement({ error: res.error || 'ERR', u, v, ts: Date.now() })
|
||||
}
|
||||
} catch (err) {
|
||||
setMeasurement({ error: String(err), u, v, ts: Date.now() })
|
||||
}
|
||||
setPending(false)
|
||||
}, [canMeasure, flipH, flipV, setMeasurement])
|
||||
|
||||
const { zoom, pan, handlers } = useZoomPan({ onTap: canMeasure ? handleTap : undefined })
|
||||
|
||||
useEffect(() => {
|
||||
if (!frame || !canvasRef.current) return
|
||||
const canvas = canvasRef.current
|
||||
const ctx = canvas.getContext('2d')
|
||||
const b64 = type === 'rgb' ? frame.rgb_jpeg_b64 : frame.depth_png_b64
|
||||
if (!b64) return
|
||||
const img = new Image()
|
||||
img.onload = () => {
|
||||
canvas.width = img.width
|
||||
canvas.height = img.height
|
||||
imgSizeRef.current = { w: img.width, h: img.height }
|
||||
ctx.drawImage(img, 0, 0)
|
||||
if (type === 'rgb' && frame.detections?.length) {
|
||||
drawDetections(ctx, frame.detections)
|
||||
}
|
||||
if (canMeasure && measurement && !measurement.error && measurement.u_found !== undefined) {
|
||||
drawMarker(ctx, measurement)
|
||||
}
|
||||
}
|
||||
img.src = `data:image/${type === 'rgb' ? 'jpeg' : 'png'};base64,${b64}`
|
||||
}, [frame, type, canMeasure, measurement])
|
||||
|
||||
const label = (() => {
|
||||
if (pending) return 'Measuring...'
|
||||
if (!canMeasure || !measurement) return null
|
||||
if (measurement.error) return `(${measurement.u},${measurement.v}) — ${measurement.error}`
|
||||
const { u, v, u_found, v_found, depth_mm, distance_mm, point_m, pixel_offset } = measurement
|
||||
const offsetLbl = pixel_offset > 0 ? ` (Δ${pixel_offset}px)` : ''
|
||||
return (
|
||||
<>
|
||||
<div>pixel clicked: ({u}, {v}) → found: ({u_found}, {v_found}){offsetLbl}</div>
|
||||
<div>depth Z: <b>{depth_mm} mm</b> · distance: <b>{distance_mm} mm</b></div>
|
||||
<div>3D (m): x={point_m.x} y={point_m.y} z={point_m.z}</div>
|
||||
</>
|
||||
)
|
||||
})()
|
||||
|
||||
return (
|
||||
<div
|
||||
{...handlers}
|
||||
style={{
|
||||
width: '100%', height: '100%', overflow: 'hidden', background: '#000',
|
||||
cursor: canMeasure ? 'crosshair' : (zoom > 1 ? 'grab' : 'default'),
|
||||
touchAction: 'none', position: 'relative',
|
||||
}}
|
||||
>
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
style={{
|
||||
width: '100%', height: '100%', objectFit: 'contain',
|
||||
transform: `translate(${pan.x}px, ${pan.y}px) scale(${zoom}) scaleX(${flipH ? -1 : 1}) scaleY(${flipV ? -1 : 1})`,
|
||||
transformOrigin: 'center center',
|
||||
}}
|
||||
/>
|
||||
{canMeasure && label && (
|
||||
<div style={{
|
||||
position: 'absolute', left: 8, bottom: 8, zIndex: 10,
|
||||
padding: '6px 10px', background: 'rgba(0,0,0,0.72)',
|
||||
color: 'var(--text)', fontSize: '11px', lineHeight: 1.4,
|
||||
borderRadius: '6px', border: '1px solid var(--border)',
|
||||
fontFamily: 'monospace', maxWidth: '70%',
|
||||
}}>
|
||||
{label}
|
||||
{measurement && !measurement.error && (
|
||||
<button
|
||||
onClick={() => setMeasurement(null)}
|
||||
style={{
|
||||
marginTop: 4, padding: '2px 8px', fontSize: '10px',
|
||||
background: 'var(--bg-card)', color: 'var(--text)',
|
||||
border: '1px solid var(--border)', borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
}}
|
||||
>clear</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function drawMarker(ctx, m) {
|
||||
const { u, v, u_found, v_found } = m
|
||||
ctx.save()
|
||||
ctx.strokeStyle = 'rgba(255,255,255,0.95)'
|
||||
ctx.lineWidth = 1.5
|
||||
ctx.beginPath()
|
||||
ctx.moveTo(u - 6, v); ctx.lineTo(u + 6, v)
|
||||
ctx.moveTo(u, v - 6); ctx.lineTo(u, v + 6)
|
||||
ctx.stroke()
|
||||
ctx.strokeStyle = '#ffcc00'
|
||||
ctx.lineWidth = 2
|
||||
ctx.beginPath()
|
||||
ctx.arc(u_found, v_found, 5, 0, Math.PI * 2)
|
||||
ctx.stroke()
|
||||
ctx.fillStyle = 'rgba(255,204,0,0.35)'
|
||||
ctx.fill()
|
||||
if (u_found !== u || v_found !== v) {
|
||||
ctx.strokeStyle = 'rgba(255,204,0,0.7)'
|
||||
ctx.lineWidth = 1
|
||||
ctx.beginPath()
|
||||
ctx.moveTo(u, v); ctx.lineTo(u_found, v_found)
|
||||
ctx.stroke()
|
||||
}
|
||||
ctx.restore()
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
const API_URL = import.meta.env.VITE_API_URL || '/api'
|
||||
const API_KEY = import.meta.env.VITE_API_KEY || ''
|
||||
|
||||
export async function apiFetch(path, options = {}) {
|
||||
const headers = {
|
||||
'Content-Type': 'application/json',
|
||||
...options.headers,
|
||||
}
|
||||
if (API_KEY) headers['X-Api-Key'] = API_KEY
|
||||
|
||||
const res = await fetch(`${API_URL}${path}`, {
|
||||
...options,
|
||||
headers,
|
||||
})
|
||||
return res.json()
|
||||
}
|
||||
|
||||
export function useApi() {
|
||||
const discover = () => apiFetch('/discover')
|
||||
const openCamera = (index = 0) => apiFetch(`/open?device_index=${index}`, { method: 'POST' })
|
||||
const closeCamera = () => apiFetch('/close', { method: 'POST' })
|
||||
const health = () => apiFetch('/health')
|
||||
const status = () => apiFetch('/status')
|
||||
const capture = () => apiFetch('/capture')
|
||||
const intrinsics = () => apiFetch('/intrinsics')
|
||||
const snapshot = (flipH, flipV) =>
|
||||
apiFetch(`/snapshot?flip_h=${flipH}&flip_v=${flipV}`, { method: 'POST' })
|
||||
const startRec = (flipH, flipV, interval = 1) =>
|
||||
apiFetch(
|
||||
`/recording/start?flip_h=${flipH}&flip_v=${flipV}&interval=${interval}`,
|
||||
{ method: 'POST' },
|
||||
)
|
||||
const stopRec = () => apiFetch('/recording/stop', { method: 'POST' })
|
||||
const recStatus = () => apiFetch('/recording/status')
|
||||
const measure = (u, v, radius = 6) =>
|
||||
apiFetch(`/measure?u=${u}&v=${v}&radius=${radius}`)
|
||||
return {
|
||||
discover, openCamera, closeCamera, health, status, capture, intrinsics,
|
||||
snapshot, startRec, stopRec, recStatus, measure,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { useRef, useCallback, useEffect } from 'react'
|
||||
|
||||
function getWsUrl() {
|
||||
if (import.meta.env.VITE_WS_URL) return import.meta.env.VITE_WS_URL
|
||||
const proto = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
|
||||
return `${proto}//${window.location.host}/ws`
|
||||
}
|
||||
|
||||
export function useWebSocket(onFrame) {
|
||||
const wsRef = useRef(null)
|
||||
const streamingRef = useRef(false)
|
||||
const pendingStart = useRef(false)
|
||||
const predictRef = useRef(false)
|
||||
const confRef = useRef(null)
|
||||
|
||||
const connect = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) return
|
||||
if (wsRef.current?.readyState === WebSocket.CONNECTING) return
|
||||
const ws = new WebSocket(getWsUrl())
|
||||
ws.onopen = () => {
|
||||
if (predictRef.current) ws.send('predict_on')
|
||||
if (confRef.current != null) ws.send(`conf:${confRef.current.toFixed(2)}`)
|
||||
if (pendingStart.current) {
|
||||
ws.send('start')
|
||||
pendingStart.current = false
|
||||
}
|
||||
}
|
||||
ws.onmessage = (e) => {
|
||||
const data = JSON.parse(e.data)
|
||||
onFrame(data)
|
||||
}
|
||||
ws.onclose = () => { wsRef.current = null }
|
||||
ws.onerror = (e) => { console.error('WebSocket error:', e) }
|
||||
wsRef.current = ws
|
||||
}, [onFrame])
|
||||
|
||||
const start = useCallback(() => {
|
||||
if (!wsRef.current || wsRef.current.readyState === WebSocket.CLOSED) {
|
||||
pendingStart.current = true
|
||||
connect()
|
||||
} else if (wsRef.current.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send('start')
|
||||
} else {
|
||||
pendingStart.current = true
|
||||
}
|
||||
streamingRef.current = true
|
||||
}, [connect])
|
||||
|
||||
const stop = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send('stop')
|
||||
}
|
||||
streamingRef.current = false
|
||||
}, [])
|
||||
|
||||
const sendCommand = useCallback((cmd) => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(cmd)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}, [])
|
||||
|
||||
const setPredict = useCallback((on) => {
|
||||
predictRef.current = on
|
||||
sendCommand(on ? 'predict_on' : 'predict_off')
|
||||
}, [sendCommand])
|
||||
|
||||
const setConf = useCallback((value) => {
|
||||
confRef.current = value
|
||||
sendCommand(`conf:${value.toFixed(2)}`)
|
||||
}, [sendCommand])
|
||||
|
||||
const disconnect = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send('close')
|
||||
}
|
||||
wsRef.current?.close()
|
||||
wsRef.current = null
|
||||
streamingRef.current = false
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
return () => { wsRef.current?.close() }
|
||||
}, [])
|
||||
|
||||
return { connect, start, stop, disconnect, setPredict, setConf, isStreaming: streamingRef }
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { useState, useRef, useCallback } from 'react'
|
||||
|
||||
const TAP_THRESHOLD_PX = 4
|
||||
|
||||
export function useZoomPan({ onTap } = {}) {
|
||||
const [zoom, setZoom] = useState(1)
|
||||
const [pan, setPan] = useState({ x: 0, y: 0 })
|
||||
const dragging = useRef(false)
|
||||
const lastPos = useRef({ x: 0, y: 0 })
|
||||
const downPos = useRef(null)
|
||||
|
||||
const onWheel = useCallback((e) => {
|
||||
e.preventDefault()
|
||||
setZoom((z) => Math.min(10, Math.max(0.5, z * (e.deltaY < 0 ? 1.15 : 0.87))))
|
||||
}, [])
|
||||
|
||||
const onPointerDown = useCallback((e) => {
|
||||
dragging.current = true
|
||||
lastPos.current = { x: e.clientX, y: e.clientY }
|
||||
downPos.current = { x: e.clientX, y: e.clientY }
|
||||
e.currentTarget.setPointerCapture(e.pointerId)
|
||||
}, [])
|
||||
|
||||
const onPointerMove = useCallback((e) => {
|
||||
if (!dragging.current) return
|
||||
setPan((p) => ({
|
||||
x: p.x + e.clientX - lastPos.current.x,
|
||||
y: p.y + e.clientY - lastPos.current.y,
|
||||
}))
|
||||
lastPos.current = { x: e.clientX, y: e.clientY }
|
||||
}, [])
|
||||
|
||||
const onPointerUp = useCallback((e) => {
|
||||
dragging.current = false
|
||||
if (onTap && downPos.current) {
|
||||
const dx = e.clientX - downPos.current.x
|
||||
const dy = e.clientY - downPos.current.y
|
||||
if (Math.hypot(dx, dy) < TAP_THRESHOLD_PX) onTap(e)
|
||||
}
|
||||
downPos.current = null
|
||||
}, [onTap])
|
||||
|
||||
const onDoubleClick = useCallback(() => {
|
||||
setZoom(1)
|
||||
setPan({ x: 0, y: 0 })
|
||||
}, [])
|
||||
|
||||
const handlers = { onWheel, onPointerDown, onPointerMove, onPointerUp, onDoubleClick }
|
||||
|
||||
return { zoom, pan, handlers }
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
:root {
|
||||
--bg-dark: #0c0e14;
|
||||
--bg-panel: #141722;
|
||||
--bg-card: #1a1e2e;
|
||||
--bg-input: #0f1219;
|
||||
--border: #252a3a;
|
||||
--accent: #3b82f6;
|
||||
--accent-hover: #2563eb;
|
||||
--success: #10b981;
|
||||
--danger: #ef4444;
|
||||
--warning: #f59e0b;
|
||||
--text: #e2e8f0;
|
||||
--text-muted: #64748b;
|
||||
--text-dim: #475569;
|
||||
--radius: 8px;
|
||||
--font-ui: 'DM Sans', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
|
||||
--font-mono: 'JetBrains Mono', 'Fira Code', monospace;
|
||||
}
|
||||
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
|
||||
html, body, #root {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: var(--font-ui);
|
||||
background: var(--bg-dark);
|
||||
color: var(--text);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.app {
|
||||
display: grid;
|
||||
grid-template-rows: 44px 1fr 32px;
|
||||
height: 100dvh;
|
||||
overflow: hidden;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import React from 'react'
|
||||
import ReactDOM from 'react-dom/client'
|
||||
import App from './App'
|
||||
import './index.css'
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>,
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
import { create } from 'zustand'
|
||||
|
||||
export const useStore = create((set) => ({
|
||||
connected: false,
|
||||
cameraOpen: false,
|
||||
streaming: false,
|
||||
natsConnected: false,
|
||||
devices: [],
|
||||
selectedDevice: 0,
|
||||
currentFrame: null,
|
||||
fps: 0,
|
||||
frameCount: 0,
|
||||
intrinsics: null,
|
||||
flipH: false,
|
||||
flipV: false,
|
||||
recording: false,
|
||||
recFrameCount: 0,
|
||||
recInterval: 1,
|
||||
predicting: false,
|
||||
confThreshold: 0.5,
|
||||
measurement: null,
|
||||
setConnected: (v) => set({ connected: v }),
|
||||
setCameraOpen: (v) => set({ cameraOpen: v }),
|
||||
setStreaming: (v) => set({ streaming: v }),
|
||||
setNatsConnected: (v) => set({ natsConnected: v }),
|
||||
setDevices: (d) => set({ devices: d }),
|
||||
setSelectedDevice: (i) => set({ selectedDevice: i }),
|
||||
setCurrentFrame: (f) => set((s) => ({ currentFrame: f, frameCount: s.frameCount + 1 })),
|
||||
setFps: (v) => set({ fps: v }),
|
||||
setIntrinsics: (i) => set({ intrinsics: i }),
|
||||
setFlipH: (v) => set({ flipH: v }),
|
||||
setFlipV: (v) => set({ flipV: v }),
|
||||
toggleFlipH: () => set((s) => ({ flipH: !s.flipH })),
|
||||
toggleFlipV: () => set((s) => ({ flipV: !s.flipV })),
|
||||
setRecording: (v) => set({ recording: v }),
|
||||
setRecFrameCount: (n) => set({ recFrameCount: n }),
|
||||
setRecInterval: (n) => set({ recInterval: n }),
|
||||
setPredicting: (v) => set({ predicting: v }),
|
||||
setConfThreshold: (v) => set({ confThreshold: v }),
|
||||
setMeasurement: (m) => set({ measurement: m }),
|
||||
}))
|
||||
@@ -0,0 +1,16 @@
|
||||
import { defineConfig } from 'vite'
|
||||
import react from '@vitejs/plugin-react'
|
||||
|
||||
export default defineConfig({
|
||||
base: '/gui/',
|
||||
plugins: [react()],
|
||||
server: {
|
||||
proxy: {
|
||||
'/api': 'http://localhost:8877',
|
||||
'/ws': {
|
||||
target: 'ws://localhost:8877',
|
||||
ws: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
Whitespace-only changes.
@@ -0,0 +1,6 @@
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend():
|
||||
return "asyncio"
|
||||
@@ -0,0 +1,14 @@
|
||||
import pytest
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
from src.backend.config import settings
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_health_without_api_key():
|
||||
"""Request without API key should fail with 403 when key is configured."""
|
||||
from src.backend.main import app
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as ac:
|
||||
resp = await ac.get("/api/health")
|
||||
if settings.api_key:
|
||||
assert resp.status_code in (403, 422)
|
||||
@@ -0,0 +1,25 @@
|
||||
from src.backend.models.api import ok, fail, ApiResponse, ApiError
|
||||
|
||||
|
||||
class TestApiEnvelope:
|
||||
def test_ok_response(self):
|
||||
result = ok({"status": "running"})
|
||||
assert result["success"] is True
|
||||
assert result["data"]["status"] == "running"
|
||||
assert result["error"] is None
|
||||
|
||||
def test_fail_response(self):
|
||||
result = fail("TIMEOUT", "Frame capture timeout")
|
||||
assert result["success"] is False
|
||||
assert result["data"] is None
|
||||
assert result["error"]["code"] == "TIMEOUT"
|
||||
assert result["error"]["message"] == "Frame capture timeout"
|
||||
|
||||
def test_api_response_model(self):
|
||||
resp = ApiResponse(success=True, data={"key": "value"})
|
||||
assert resp.success is True
|
||||
assert resp.data == {"key": "value"}
|
||||
|
||||
def test_api_error_model(self):
|
||||
err = ApiError(code="NOT_FOUND", message="Resource not found")
|
||||
assert err.code == "NOT_FOUND"
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Smoke tests for the RealSense D435 service layer.
|
||||
|
||||
These tests exercise the dataclasses + the driver in its un-initialized
|
||||
state. Tests that need real hardware are skipped at runtime.
|
||||
"""
|
||||
import pytest
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _camera_module():
|
||||
try:
|
||||
return __import__("src.backend.services.camera", fromlist=["*"])
|
||||
except ImportError as exc:
|
||||
pytest.skip(f"pyrealsense2 not importable: {exc}")
|
||||
|
||||
|
||||
class TestCameraIntrinsics:
|
||||
def test_creation(self):
|
||||
mod = _camera_module()
|
||||
i = mod.CameraIntrinsics(
|
||||
fx_ir=383.0, fy_ir=383.0, cx_ir=320.0, cy_ir=240.0,
|
||||
fx_rgb=610.0, fy_rgb=610.0, cx_rgb=320.0, cy_rgb=240.0,
|
||||
)
|
||||
assert i.fx_ir == 383.0
|
||||
assert i.fx_rgb == 610.0
|
||||
|
||||
|
||||
class TestCameraFrame:
|
||||
def test_creation(self):
|
||||
mod = _camera_module()
|
||||
rgb = np.zeros((480, 640, 3), dtype=np.uint8)
|
||||
depth = np.zeros((480, 640), dtype=np.uint16)
|
||||
f = mod.CameraFrame(rgb=rgb, depth=depth, timestamp=1234.5)
|
||||
assert f.rgb.shape == (480, 640, 3)
|
||||
assert f.depth.shape == (480, 640)
|
||||
assert f.timestamp == 1234.5
|
||||
|
||||
|
||||
class TestDiscoveredDevice:
|
||||
def test_creation(self):
|
||||
mod = _camera_module()
|
||||
d = mod.DiscoveredDevice(index=0, vid=0x8086, pid=0x0B07, serial="ABC123", bus=1, device=5)
|
||||
assert d.vid == 0x8086
|
||||
assert d.serial == "ABC123"
|
||||
assert d.index == 0
|
||||
|
||||
|
||||
class TestRealSenseD435:
|
||||
def test_defaults(self):
|
||||
mod = _camera_module()
|
||||
cam = mod.RealSenseD435()
|
||||
assert not cam.is_open()
|
||||
assert not cam.is_initialized()
|
||||
assert cam.intrinsics is None
|
||||
|
||||
def test_capture_when_closed(self):
|
||||
mod = _camera_module()
|
||||
cam = mod.RealSenseD435()
|
||||
assert cam.capture() is None
|
||||
assert cam.get_rgb() is None
|
||||
assert cam.get_depth() is None
|
||||
|
||||
def test_discover_requires_init(self):
|
||||
mod = _camera_module()
|
||||
cam = mod.RealSenseD435()
|
||||
with pytest.raises(RuntimeError, match="not initialized"):
|
||||
cam.discover()
|
||||
|
||||
def test_open_requires_init(self):
|
||||
mod = _camera_module()
|
||||
cam = mod.RealSenseD435()
|
||||
# open() returns False rather than raising when not initialized.
|
||||
assert cam.open(0) is False
|
||||
Reference in new issue
Block a user