Compare commits

...
31 Commits
Author SHA1 Message Date
AdrianoandClaude Opus 4.8 c383fd3143 feat(version): mostra build time nel badge (non solo tooltip); bump 1.0.2
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 14:49:22 +02:00
AdrianoandClaude Opus 4.8 054ad103df chore: bump VERSION 1.0.0 -> 1.0.1 (deploy)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 14:44:25 +02:00
AdrianoandClaude Opus 4.8 7b4dc6ce8c docs: build.sh deploy workflow + badge versione (CLAUDE.md, README)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 14:43:07 +02:00
AdrianoandClaude Opus 4.8 cf726b479e feat: badge versione build in GUI + build.sh con auto-bump
- VERSION (1.0.0) + build.sh: bump patch/minor/major e deploy via docker compose
- Dockerfile: ARG/ENV APP_VERSION + BUILD_TIME (iniettati come build-arg)
- version.build_info(): env in Docker, fallback file VERSION + '(local)' in dev
- GET /api/version; badge #appVersionPill nell'header (tooltip = build time)
- test_version.py: env vs fallback

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 14:40:52 +02:00
AdrianoandClaude Opus 4.8 a88a064aa8 docs: spec badge versione build in GUI
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 14:40:52 +02:00
AdrianoandClaude Opus 4.8 8084b87261 Merge: allineamento cross-camera + overlay tag multi-camera (Pose 3D)
- umeyama_similarity (geometry)
- GET /api/pose_3d/tags (Nuwa + D435i), rimosso /front_tags
- POST /api/pose_3d/align: fit Umeyama + persistenza pose_3d.cross_camera
- /state espone cross_camera
- overlay tag su entrambe le camere (Nuwa rosso, D435i ciano)
- UI Allinea cross-camera + render Nuwa nel frame D435i
- docs

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 13:58:06 +02:00
AdrianoandClaude Opus 4.8 0ee3ef244b chore(pose_3d): tieni config.yaml allineato a main (drift runtime non versionato)
Il commit della Task 1 aveva incluso per errore (via git add -u) drift runtime
di config.yaml (cloud_params size/flip) persistito dall'app live. Lo rimuovo dal
branch: config.yaml resta uguale a main. I valori live restano come modifica
non committata nel working tree.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 12:43:34 +02:00
AdrianoandClaude Sonnet 4.6 d9213ca19a docs(pose_3d): tags multi-camera + calibrazione cross-camera
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-03 12:41:12 +02:00
AdrianoandClaude Opus 4.8 47ae8a6b2c feat(pose_3d): doAlign + persistenza/caricamento calibrazione cross-camera
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 12:38:25 +02:00
AdrianoandClaude Opus 4.8 746662d534 feat(pose_3d): piazzamento Nuwa allineato nel frame D435i (matrice)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 12:35:30 +02:00
AdrianoandClaude Sonnet 4.6 2f0dc69cdd feat(pose_3d): rinomina sezione Tags + controlli Allinea cross-camera
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-03 12:33:55 +02:00
AdrianoandClaude Sonnet 4.6 edc4430c67 feat(pose_3d): overlay tag su entrambe le camere (Nuwa rosso, D435i ciano)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-03 12:31:16 +02:00
AdrianoandClaude Sonnet 4.6 79f587567e feat(pose_3d): /state espone blocco cross_camera
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-03 12:28:52 +02:00
AdrianoandClaude Sonnet 4.6 5ab808accd feat(pose_3d): POST /align fit Umeyama cross-camera + persistenza
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-03 12:28:05 +02:00
AdrianoandClaude Sonnet 4.6 8b01ae123f feat(pose_3d): GET /tags multi-camera, rimuovi /front_tags
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-03 12:20:04 +02:00
AdrianoandClaude Sonnet 4.6 77fb0f8fc0 feat(geometry): umeyama_similarity (rot+trasl+scala uniforme)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-03 12:13:25 +02:00
AdrianoandClaude Opus 4.8 9189700f34 docs(pose_3d): piano implementazione allineamento cross-camera
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 12:06:25 +02:00
AdrianoandClaude Opus 4.8 80625cc281 docs(pose_3d): spec allineamento cross-camera + overlay tag multi-camera
Design per: detection tag su tutte le camere attive (Nuwa rosso + D435i),
rinomina "Tag camera frontale" -> "Tags", calibrazione persistente
Nuwa->frontale via tag condiviso (Umeyama similarity), flag+pulsante
"Allinea" nella sezione Cloud cross-camera.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 12:02:26 +02:00
AdrianoandClaude Opus 4.8 b4922fe01b fix(pose_3d): front-tag detection solo al cambio immagine, non in loop
L'overlay tag frontale (D435i) ri-eseguiva la detection ArUco ad ogni
tick di pollState() finche' il flag era attivo. Ora gira solo:
- al toggle on del checkbox
- a fine acquireClouds() (pulsante Acquisisci) se il flag e' attivo

Rimosso il piggyback in pollState() e lo stato front_tags.in_flight
ormai inutilizzato.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 10:05:51 +02:00
AdrianoandClaude Opus 4.7 fb0e9b9112 feat(pose_3d): tag locator save, front-tag overlay, sidebar tabs, cloud params persist
- Sidebar a 3 tab verticali (POSE / CAMERAS / MISC) per ridurre lo scroll
- Tag locator: POST /api/pose_3d/tag_offset/{save,} salva crocetta nel
  viewport con snapshot giunti, current.json + archive rotation
- Front-camera tag overlay (MISC):
  - GET /api/pose_3d/front_tags scansiona ArUco sul D435i e per ogni tag
    misura i 4 corner 3D via depth sensor (indipendente da solvePnP);
    outlier rejection ±20% dalla mediana per ignorare corner che catturano
    background sul bordo del marker
  - Rendering quad cyan + label compatta (id, edge_mm, distanza camera)
    agganciato al d435_link, segue la camera anche in modalita' hand-eye
  - Trigger via pollState() quando il flag e' on, niente timer separato
- Cloud cross-camera: memorizza step/max_depth/sizes/scale_xy/flip/handeye
  in pose_3d.cloud_params al click di Acquisisci, ripopola UI da config
  al primo poll (state.cloud_params_populated guard)
- D435i front cloud: applica nuwa_to_cv_frame (il servizio ritorna native,
  non OpenCV come si pensava)
- Cache-bust per pose_3d.js e pose_3d.css

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-06-01 16:53:26 +02:00
Adriano Dal PastroandClaude Opus 4.7 ae8e609b7e docs: aggiorna README.md e CLAUDE.md con tab Braccio DX, Pose 3D, cloud, diagnostics
Allineati alla rimozione del modulo agent e alle nuove funzionalità:
- Tab Braccio DX e Pose 3D documentate in GUI
- Servizi front_camera (D435i) e G1 /diagnostics nella tabella upstream
- Endpoint right_arm/* e pose_3d/* nella lista API
- Convenzioni Pose 3D (graceful offline, clouds, joints mirroring) in CLAUDE.md
- Router list e _deps aggiornati (front_camera(), 12 router modules)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-27 18:01:28 +02:00
Adriano Dal PastroandClaude Opus 4.7 6c2452fe20 refactor(pose_3d): rimuovi agent/chat, aggiungi cloud+diagnostics+camera polso
Rimosso il modulo agent (LLM chat) e la relativa config OpenRouter.
La tab Pose 3D ora: espone i giunti reali via G1 /diagnostics per
specchiare l'URDF senza IK; acquisisce point cloud Nuwa+D435i
(/api/pose_3d/clouds); mostra il modello 3D della camera sul polso;
degrada gracefully a 200 (wrist=null) quando il robot è offline.
Il vecchio renderer canvas-2D (app.js view3d + diagnostic3d.js) è
stato consolidato nel viewport Three.js di pose_3d.js.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-27 17:34:42 +02:00
Adriano Dal Pastro 9d30073d57 fix(pose_3d): zero scena al centro robot (pelvi), griglia sotto i piedi a Z=-0.793 2026-05-21 18:06:38 +02:00
Adriano Dal Pastro 38848ad53c feat(pose_3d): IK 2-bone su URDF dopo Apply/Jog/GoPose (polso modello segue il target) 2026-05-21 17:47:12 +02:00
Adriano Dal Pastro 7f60238be9 fix(pose_3d): URDF feet a terra (+0.793m) + sfere polso a zero-pose (Z=0.45) 2026-05-21 17:41:31 +02:00
Adriano Dal Pastro bfd4cc13fb fix(pose_3d): URDFLoader path resolution (rimuovi doppio prefisso /static/vendor/g1/) 2026-05-21 17:37:25 +02:00
Adriano Dal Pastro d5bb9cebd3 feat(pose_3d): workspace check client-side (ghost verde/rosso) + SHOULDER da URDF 2026-05-21 17:21:09 +02:00
Adriano Dal Pastro 8c09d79505 feat(pose_3d): modello reale G1 (URDF + STL meshes) al posto della silhouette finta
Sostituisce torso/head/hips/gambe/spalle primitive in tab Pose 3D con il
modello URDF G1 (g1_body29_hand14), caricato via urdf-loader 0.12.6.

Vendor:
- web/vendor/g1/g1.urdf + 64 mesh STL (~52 MB, copiati da
  G1_API_Server/assets/g1/)
- web/vendor/three/addons/loaders/URDFLoader.js (npm urdf-loader@0.12.6),
  import path adattato a 'three/addons/loaders/STLLoader.js' (importmap
  del sito) e ColladaLoader stubbato (G1 URDF è solo STL)
- web/vendor/three/addons/loaders/URDFClasses.js (companion del loader)

pose_3d.js:
- Aggiunge loadG1Urdf(scene) async che configura loader.loadMeshCb per
  servire le STL dal mount /static/vendor/g1/meshes/
- Best-effort: in caso di errore di parsing/fetch il body non viene
  aggiunto ma la tab continua a funzionare (polsi, UCS, gizmo)
2026-05-21 17:14:16 +02:00
Adriano Dal Pastro 18dce0d951 feat(pose_3d): bottone Sincronizza con robot (refetch state + reset target) 2026-05-21 17:08:09 +02:00
Adriano Dal Pastro b852976a4a feat(pose_3d): readout XYZ/RPY editabili (input number, drive il ghost) 2026-05-21 17:01:03 +02:00
Adriano Dal Pastro 8b722ca729 feat(pose_3d): silhouette robot completa (testa+gambe+spalle) + FRONT come marker terra 2026-05-21 16:46:44 +02:00
101 changed files with 158482 additions and 2759 deletions

No files matched your search

+15 -5
View File
@@ -9,6 +9,8 @@ Build & run (Docker):
docker compose up -d presa-mano # serves GUI + API on http://localhost:8890 docker compose up -d presa-mano # serves GUI + API on http://localhost:8890
``` ```
**Deploy changes — use `./build.sh`.** `src/` and `web/` are baked into the image (only `config.yaml` + `data/` are bind-mounted), so **code edits are invisible until the image is rebuilt**. `./build.sh` auto-bumps `VERSION` (patch by default; `./build.sh minor|major` for the others), injects version + build time as build-args, rebuilds, and restarts the container. The running version is shown as a badge in the GUI header (and at `GET /api/version`) so a stale deployment is visible at a glance. The `VERSION` bump is left uncommitted — commit it with your changes.
Local dev (uses `uv`, same as the Dockerfile): Local dev (uses `uv`, same as the Dockerfile):
```bash ```bash
uv sync # install deps from pyproject.toml uv sync # install deps from pyproject.toml
@@ -17,39 +19,43 @@ uv run uvicorn presa_mano.main:app --host 0.0.0.0 --port 8890 --reload
Useful endpoints (all under `http://localhost:8890`): Useful endpoints (all under `http://localhost:8890`):
- `GET /apidocs` — Swagger UI; `GET /redoc` — ReDoc. - `GET /apidocs` — Swagger UI; `GET /redoc` — ReDoc.
- `GET /health` — liveness; `GET /api/version` — `{version, build_time}` baked at build by `build.sh` (falls back to the `VERSION` file + `(local)` under `uv run`).
- `GET/PUT /api/config` — read/patch runtime config (deep-merged into `config.yaml` and persisted). - `GET/PUT /api/config` — read/patch runtime config (deep-merged into `config.yaml` and persisted).
- Calibration: `POST /api/calibration/sample`, `GET/DELETE /api/calibration/samples`, `POST /api/calibration/solve`, `GET /api/calibration/result`, `POST /api/calibration/preview`. - Calibration: `POST /api/calibration/sample`, `GET/DELETE /api/calibration/samples`, `POST /api/calibration/solve`, `GET /api/calibration/result`, `POST /api/calibration/preview`.
- Perception: `POST /api/perception/detect`, `POST /api/perception/object_points`, `GET /api/perception/measure`. - Perception: `POST /api/perception/detect`, `POST /api/perception/object_points`, `GET /api/perception/measure`.
- Grasp: `POST /api/grasp/plan`, `POST /api/grasp/execute` (set `dry_run` to gate G1 mutations). - Grasp: `POST /api/grasp/plan`, `POST /api/grasp/execute` (set `dry_run` to gate G1 mutations).
- Wrist/EE: `GET /api/wrist`, `POST /api/wrist/{absolute,relative,hand,emergency_stop,unlock}`. - Wrist/EE: `GET /api/wrist`, `POST /api/wrist/{absolute,relative,hand,emergency_stop,unlock}`.
- Right arm: `GET /api/right_arm/state`, `POST /api/right_arm/detect_{front,wrist}`, `POST /api/right_arm/move_toward`, `POST /api/right_arm/{home,custom_pose}`.
- Pose 3D: `GET /api/pose_3d/state` (wrist + joints + config, incl. `cross_camera` calibration), `POST /api/pose_3d/apply`, `GET /api/pose_3d/clouds` (Nuwa + D435i point clouds), `GET /api/pose_3d/tags` (ArUco su Nuwa polso + D435i frontale), `POST /api/pose_3d/align?align_id=N` (fit Umeyama Nuwa→frontale, persiste in `pose_3d.cross_camera`), `CRUD /api/pose_3d/poses/{name}`.
Tests live under `tests/` and run with `uv run pytest`. No linter is configured. Tests live under `tests/` and run with `uv run pytest`. No linter is configured.
## Architecture ## Architecture
This is a **FastAPI orchestrator** that glues together four external microservices to implement an eye-to-hand calibration + object-grasping pipeline for a Unitree G1 humanoid using a fixed Nuwa-HP60C depth camera. It owns no hardware itself — every action is an HTTP call. This is a **FastAPI orchestrator** that glues together external microservices to implement an eye-to-hand calibration + object-grasping + 3D visualization pipeline for a Unitree G1 humanoid. It owns no hardware itself — every action is an HTTP call.
Upstream services (defaults in `config.yaml > services`): Upstream services (defaults in `config.yaml > services`):
| Service | Default URL | Role | | Service | Default URL | Role |
|---------|------------|------| |---------|------------|------|
| Nuwa-HP60C | `:8877` | RGB, depth PNG, point cloud (full + ROI), intrinsics, pixel→3D measure | | Nuwa-HP60C | `:8877` | RGB, depth PNG, point cloud (full + ROI), intrinsics, pixel→3D measure. Eye-in-hand sul polso DX |
| Front camera (D435i) | `services.front_camera` | Point cloud frontale fissa sul torso; stessa interfaccia NuwaClient |
| marker-finder | `:8082` | ArUco detection (returns `corners`, `position_m`, `rotation_matrix`) | | marker-finder | `:8082` | ArUco detection (returns `corners`, `position_m`, `rotation_matrix`) |
| YOLO server | `:8879` | Object bbox detection from an image URL | | YOLO server | `:8879` | Object bbox detection from an image URL |
| G1 API | `:8765` | Wrist pose read/write, hand open/close, e-stop | | G1 API | `:8765` | Wrist pose read/write, hand open/close, e-stop, `/diagnostics` (29 joint positions) |
| SAM21 (NuwaSeg) | `:8002` | Promptable segmentation: rgb+depth+intrinsics+bbox → 3D point cloud of the object (replaces the bbox-only ROI cloud in the grasp pipeline) | | SAM21 (NuwaSeg) | `:8002` | Promptable segmentation: rgb+depth+intrinsics+bbox → 3D point cloud of the object (replaces the bbox-only ROI cloud in the grasp pipeline) |
### Layered package layout (`src/presa_mano/`) ### Layered package layout (`src/presa_mano/`)
- `main.py` — FastAPI app; mounts `web/` at `/static`, serves `web/index.html` at `/`, registers routers under `/api/*`. - `main.py` — FastAPI app; mounts `web/` at `/static`, serves `web/index.html` at `/`, registers routers under `/api/*`.
- `config.py` — Thread-safe singleton config loader. `load_config()` reads YAML once; `update_config(patch)` deep-merges and persists. Path comes from `PRESA_MANO_CONFIG` env (Dockerfile sets `/app/config.yaml`); data files live under `PRESA_MANO_DATA` (default `/app/data`). - `config.py` — Thread-safe singleton config loader. `load_config()` reads YAML once; `update_config(patch)` deep-merges and persists. Path comes from `PRESA_MANO_CONFIG` env (Dockerfile sets `/app/config.yaml`); data files live under `PRESA_MANO_DATA` (default `/app/data`).
- `api/` — Routers, one per concern. **Build clients per-request via `api/_deps.py`** (`nuwa()`, `marker()`, `yolo()`, `sam21()`, `g1(dry_run=...)`) so runtime `PUT /api/config` changes take effect without restart. Don't cache clients at module level. - `api/` — Routers, one per concern (calibration, config, grasp, home, models, perception, pose_3d, posture, right_arm, robot, work_plane, wrist). **Build clients per-request via `api/_deps.py`** (`nuwa()`, `front_camera()`, `marker()`, `yolo()`, `sam21()`, `g1(dry_run=...)`) so runtime `PUT /api/config` changes take effect without restart. Don't cache clients at module level.
- `clients/` — Thin `httpx` wrappers, one per upstream. Errors are normalized to service-specific exceptions (e.g. `G1Unavailable`, `NuwaError`). - `clients/` — Thin `httpx` wrappers, one per upstream. Errors are normalized to service-specific exceptions (e.g. `G1Unavailable`, `NuwaError`).
- `calibration/` — `sampler.py` builds one (T_base_wrist, T_cam_marker) sample; `handeye.py` runs `cv2.calibrateHandEye` and computes residuals; `store.py` persists samples (JSON) and the result (YAML). - `calibration/` — `sampler.py` builds one (T_base_wrist, T_cam_marker) sample; `handeye.py` runs `cv2.calibrateHandEye` and computes residuals; `store.py` persists samples (JSON) and the result (YAML).
- `perception/` — `detector.py` calls YOLO and normalizes responses across multiple envelope shapes; `pointcloud.py` handles frame conversion + outlier filtering. - `perception/` — `detector.py` calls YOLO and normalizes responses across multiple envelope shapes; `pointcloud.py` handles frame conversion + outlier filtering.
- `grasp/` — `axis.py` PCA → `ObjectAxes`; `planner.py` builds a wrist pose from those axes. - `grasp/` — `axis.py` PCA → `ObjectAxes`; `planner.py` builds a wrist pose from those axes.
- `geometry.py` — pure-numpy SE(3) helpers (quat↔R, rvec↔R, invert_T, transform_points). - `geometry.py` — pure-numpy SE(3) helpers (quat↔R, rvec↔R, invert_T, transform_points).
- `timing.py` — `Stopwatch` with `step(name)` context manager; every endpoint annotates its response with `timings_ms`. - `timing.py` — `Stopwatch` with `step(name)` context manager; every endpoint annotates its response with `timings_ms`.
- `web/` — vanilla HTML/JS/CSS GUI (3 tabs: Calibration, Grasp, Movimento EE), served as static files; talks only to `/api/*`. - `web/` — vanilla HTML/JS/CSS GUI served as static files; talks only to `/api/*`. Tab principali: Calibrazione, Calibrazione Piano, Cerca Modello, Movimento EE, Braccio DX, Pose 3D. Il viewport 3D (pose_3d.js) usa Three.js + URDFLoader per il modello robot.
### Core data flow: hand-eye calibration (eye-to-hand) ### Core data flow: hand-eye calibration (eye-to-hand)
@@ -88,6 +94,10 @@ Marker-finder caveat: it uses **D435i intrinsics baked into its config** that do
- **Dry-run semantics:** any G1-mutating endpoint takes a `dry_run` flag; `G1Client._post` short-circuits and logs `[DRY-RUN]` instead of hitting the network. Never silently strip dry-run from a mutating path. - **Dry-run semantics:** any G1-mutating endpoint takes a `dry_run` flag; `G1Client._post` short-circuits and logs `[DRY-RUN]` instead of hitting the network. Never silently strip dry-run from a mutating path.
- **Per-request client construction** (via `api/_deps.py`) is intentional — runtime `PUT /api/config` updates would otherwise be invisible. - **Per-request client construction** (via `api/_deps.py`) is intentional — runtime `PUT /api/config` updates would otherwise be invisible.
- **SAM21 vs bbox frame convention:** the grasp pipeline has two parallel paths in `_object_points_in_base`. The SAM21 path returns points already in the OpenCV RGB-camera frame and **must NOT call `nuwa_to_cv_frame`**; the legacy bbox path (via `_points_via_bbox`) still calls it because Nuwa's ROI cloud uses a different convention. Don't accidentally add the flip to the SAM21 branch — coordinates would double-flip. - **SAM21 vs bbox frame convention:** the grasp pipeline has two parallel paths in `_object_points_in_base`. The SAM21 path returns points already in the OpenCV RGB-camera frame and **must NOT call `nuwa_to_cv_frame`**; the legacy bbox path (via `_points_via_bbox`) still calls it because Nuwa's ROI cloud uses a different convention. Don't accidentally add the flip to the SAM21 branch — coordinates would double-flip.
- **Pose 3D graceful offline:** `GET /api/pose_3d/state` returns 200 even when G1 is unreachable — `wrist` and `joints` are `null`, `robot_error` carries the message. The tab must remain usable for editing config / named poses / viewing the URDF model without a live robot.
- **Pose 3D clouds:** `GET /api/pose_3d/clouds` acquires from Nuwa (eye-in-hand, polso DX) and D435i (front camera, torso) in parallel. Both are returned in **OpenCV optical frame** (X right, Y down, Z forward). The client attaches them to URDF links with configurable offset/rpy from `config.yaml > pose_3d.clouds`.
- **Pose 3D joints mirroring:** `G1Client.diagnostics()` → `motors[].{name, q}` gives real joint positions in radians. The frontend uses these to pose the URDF model directly, bypassing client-side IK.
- **Pose 3D cross-camera alignment:** `POST /api/pose_3d/align` fits an Umeyama similarity (rotation + translation + uniform scale) mapping Nuwa-optical → D435-optical from a tag (`align_id`) seen by both cameras, and persists it to `config.yaml > pose_3d.cross_camera`. The frontend renders the Nuwa cloud through the D435i transform pipeline (`d435LocalMatrix()` · alignMatrix) when `cross_camera.enabled`. If the tag isn't seen by both cameras, the saved matrix is NOT overwritten and the Nuwa cloud falls back to URDF placement. Tag detection (`GET /tags`) runs on both cameras; the GUI draws Nuwa tags red, D435i tags cyan.
- Endpoints return `{ ..., "timings_ms": {step: ms, ..., total: ms} }`. New endpoints should follow this with `Stopwatch().step(name)`. - Endpoints return `{ ..., "timings_ms": {step: ms, ..., total: ms} }`. New endpoints should follow this with `Stopwatch().step(name)`.
- Calibration paths in config (`calibration.file`, `calibration.samples_file`) are resolved against `PRESA_MANO_DATA` when relative — see `_deps.calibration_file_path()` / `samples_file_path()`. - Calibration paths in config (`calibration.file`, `calibration.samples_file`) are resolved against `PRESA_MANO_DATA` when relative — see `_deps.calibration_file_path()` / `samples_file_path()`.
- Italian is fine in user-facing copy (GUI, README, comments) — it's the project default. - Italian is fine in user-facing copy (GUI, README, comments) — it's the project default.
+8 -1
View File
@@ -12,13 +12,20 @@ COPY pyproject.toml README.md ./
COPY src/ src/ COPY src/ src/
RUN uv sync --no-dev RUN uv sync --no-dev
COPY config.yaml ./ COPY config.yaml VERSION ./
COPY web/ web/ COPY web/ web/
ENV PRESA_MANO_CONFIG=/app/config.yaml ENV PRESA_MANO_CONFIG=/app/config.yaml
ENV PRESA_MANO_DATA=/app/data ENV PRESA_MANO_DATA=/app/data
ENV PRESA_MANO_PORT=8890 ENV PRESA_MANO_PORT=8890
# Versione build: iniettata da build.sh come --build-arg. Cambia ad ogni build
# (il valore diverso invalida solo questi due layer finali, non le dipendenze).
ARG APP_VERSION=dev
ARG BUILD_TIME=unknown
ENV APP_VERSION=${APP_VERSION}
ENV BUILD_TIME=${BUILD_TIME}
EXPOSE 8890 EXPOSE 8890
# Shell form so $PRESA_MANO_PORT viene espanso a runtime: cambi la porta in # Shell form so $PRESA_MANO_PORT viene espanso a runtime: cambi la porta in
+32 -6
View File
@@ -8,17 +8,21 @@ Microservizio Python (FastAPI) che orchestra:
nuvola di punti dell'oggetto. nuvola di punti dell'oggetto.
3. **Calibrazione piano di lavoro** + filtro "sotto piano" applicato a ogni cloud. 3. **Calibrazione piano di lavoro** + filtro "sotto piano" applicato a ogni cloud.
4. **Pianificazione presa**: asse dell'oggetto (PCA) + posa polso per il robot attivo. 4. **Pianificazione presa**: asse dell'oggetto (PCA) + posa polso per il robot attivo.
5. **Visualizzazione 3D** (tab Pose 3D): viewport Three.js con URDF del robot
specchiato sui giunti reali, point cloud live (Nuwa eye-in-hand + D435i front
camera), gizmo di editing pose, named poses salvabili.
## Microservizi usati ## Microservizi usati
| Servizio | URL (default) | Scopo | | Servizio | URL (default) | Scopo |
|-------------------|-----------------------|----------------------------------------------------| |-------------------|-----------------------|----------------------------------------------------|
| Nuwa-HP60C | http://localhost:8877 | RGB, depth, point cloud, intrinsics | | Nuwa-HP60C | `:8877` | RGB, depth, point cloud, intrinsics (eye-in-hand polso DX) |
| Marker-finder | http://localhost:8082 | Detection 2D ArUco (corners + pose) | | Front camera (D435i) | config `services.front_camera` | Point cloud frontale fissa sul torso |
| YOLO Server | http://localhost:8879 | Bbox oggetto da prendere | | Marker-finder | `:8082` | Detection 2D ArUco (corners + pose) |
| SAM21 (NuwaSeg) | http://localhost:8002 | Segmentazione promptable: bbox → cloud 3D filtrata | | YOLO Server | `:8879` | Bbox oggetto da prendere |
| G1 API Server | http://localhost:8765 | Controllo polso/mano G1 (dry-run se down) | | SAM21 (NuwaSeg) | `:8002` | Segmentazione promptable: bbox → cloud 3D filtrata |
| H1 API Server | http://localhost:8767 | Controllo polso/mano H1 (network host, 4 DoF) | | G1 API Server | `:8765` | Controllo polso/mano G1, diagnostics giunti (dry-run se down) |
| H1 API Server | `:8767` | Controllo polso/mano H1 (network host, 4 DoF) |
Il servizio **presa-mano** espone GUI + API su **http://localhost:8890**. Il robot attivo Il servizio **presa-mano** espone GUI + API su **http://localhost:8890**. Il robot attivo
è scelto da `config.yaml > robot.active` (`g1` | `h1`); l'orchestrator fa routing è scelto da `config.yaml > robot.active` (`g1` | `h1`); l'orchestrator fa routing
@@ -31,6 +35,18 @@ docker compose up -d presa-mano
# Apri http://localhost:8890 # Apri http://localhost:8890
``` ```
## Build & deploy delle modifiche
`src/` e `web/` sono inclusi (baked) nell'immagine: le modifiche al codice **non si vedono finché non ricostruisci l'immagine**. Usa lo script:
```bash
./build.sh # patch++ (1.0.1 -> 1.0.2), rebuild + restart
./build.sh minor # 1.0.2 -> 1.1.0
./build.sh major # 1.1.0 -> 2.0.0
```
Lo script bumpa il file `VERSION`, inietta versione + ora di build nell'immagine e riavvia il container. La versione attiva è mostrata nel **badge in alto a destra** nella GUI (tooltip con l'ora di build) ed è esposta da `GET /api/version` — così capisci subito se il container gira l'ultima build. Il bump di `VERSION` resta non committato: committalo con le tue modifiche.
## GUI ## GUI
Tab principali: Tab principali:
@@ -47,6 +63,16 @@ Tab principali:
se SAM21 è down), visualizza bbox + asse, esegui (toggle dry-run). se SAM21 è down), visualizza bbox + asse, esegui (toggle dry-run).
- **Movimento EE**: lettura pose correnti, invio pose assolute/relative, apri/chiudi mano, - **Movimento EE**: lettura pose correnti, invio pose assolute/relative, apri/chiudi mano,
emergency stop. emergency stop.
- **Braccio DX**: detection via YOLO su front camera + camera polso, move-toward
iterativo, pose salvabili (home, custom), verifica tag ArUco.
- **Pose 3D**: viewport Three.js con URDF G1 specchiato sui giunti reali
(`/diagnostics`), point cloud Nuwa + D435i, modello 3D camera sul polso,
gizmo traslazione/rotazione, editor pose (Apply/Jog), named poses salvabili.
Overlay tag ArUco su entrambe le camere (Nuwa = rosso, D435i = ciano).
Calibrazione cross-camera: pulsante *Allinea* esegue fit Umeyama Nuwa→D435i
da un tag comune, persiste in `config.yaml > pose_3d.cross_camera`.
La tab resta utilizzabile anche senza robot (mostra il modello URDF, config,
pose salvate).
## Pipeline cloud-from-bbox ## Pipeline cloud-from-bbox
+1
View File
@@ -0,0 +1 @@
1.0.2
Executable
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
# Build + deploy di presa-mano con auto-bump della versione.
#
# ./build.sh → patch++ (1.0.1 -> 1.0.2)
# ./build.sh minor → minor++ (1.0.2 -> 1.1.0)
# ./build.sh major → major++ (1.1.0 -> 2.0.0)
#
# La nuova versione + l'ora di build vengono iniettate nell'immagine come
# build-arg e mostrate nel badge in GUI. Il file VERSION viene riscritto ma NON
# committato automaticamente (lo committi tu con le altre modifiche).
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
COMPOSE_DIR="$(dirname "$SCRIPT_DIR")" # docker-compose.yml sta un livello sopra
VERSION_FILE="$SCRIPT_DIR/VERSION"
[ -f "$VERSION_FILE" ] || echo "1.0.0" > "$VERSION_FILE"
cur="$(tr -d '[:space:]' < "$VERSION_FILE")"
IFS='.' read -r major minor patch <<< "$cur"
major="${major:-1}"; minor="${minor:-0}"; patch="${patch:-0}"
case "${1:-patch}" in
major) major=$((major + 1)); minor=0; patch=0 ;;
minor) minor=$((minor + 1)); patch=0 ;;
patch) patch=$((patch + 1)) ;;
*) echo "uso: $0 [major|minor|patch]" >&2; exit 1 ;;
esac
new="${major}.${minor}.${patch}"
echo "$new" > "$VERSION_FILE"
build_time="$(date -u +"%Y-%m-%d %H:%M UTC")"
echo "→ build V${new} (${build_time})"
cd "$COMPOSE_DIR"
docker compose build \
--build-arg APP_VERSION="$new" \
--build-arg BUILD_TIME="$build_time" \
presa-mano
docker compose up -d presa-mano
echo "✓ deployed V${new} (${build_time})"
+81 -15
View File
@@ -48,7 +48,10 @@ tab_right_arm:
x: 1 x: 1
y: -1 y: -1
z: -1 z: -1
cam_to_wrist_perm: [z, x, y] # wrist_X<-cam_Z, wrist_Y<-cam_X, wrist_Z<-cam_Y cam_to_wrist_perm:
- z
- x
- y
camera_offset_wrist_mm: camera_offset_wrist_mm:
x: 0.0 x: 0.0
y: 0.0 y: 0.0
@@ -79,7 +82,83 @@ tab_right_arm:
- 0 - 0
pose_3d: pose_3d:
live_max_step_m: 0.15 live_max_step_m: 0.15
named_poses: {} tag_offset:
save_dir: /home/harry/robot/logging/wrist-tag-offset
right_wrist_camera:
enabled: true
mesh_url: /static/assets/nuwa/nuwa-hp60c.stl
offset_m:
- 0.05
- 0
- 0.1
rpy_deg:
- 0
- -90
- -90
clouds:
nuwa:
use_nuwa_camera_transform: true
extra_offset_m:
- 0.0
- 0.0
- 0.0
extra_rpy_deg:
- 0
- 0
- 0
d435:
offset_m:
- -0.01
- 0.015
- -0.005
rpy_deg:
- -90
- 90
- 0
transform_pelvis_4x4:
- - 0.03273746934835532
- -0.9960477706629303
- -0.0825657111522212
- -0.00771413467718246
- - -0.7734929889294071
- 0.027067579810660346
- -0.633226612043623
- 0.37547513731967447
- - 0.6329588092267278
- 0.08459423550455474
- -0.7695498431821574
- 0.33337005199505626
- - 0.0
- 0.0
- 0.0
- 1.0
named_poses:
Base:
side: left
position:
- 0.2561414814483791
- 0.14648003458557432
- 0.10874469031742239
quaternion:
- -0.005339006082777725
- -0.047703617194205114
- -0.005110044879619669
- 0.9988341941293186
cloud_params:
step: 2
max_depth_mm: 1000
nuwa_size_mm: 5
d435_size_mm: 5
d435_scale_xy: 1
flip_nuwa:
- false
- false
- false
flip_d435:
- false
- false
- false
d435_use_handeye: false
grasp: grasp:
approach_offset_m: 0.1 approach_offset_m: 0.1
pre_grasp_clearance_m: 0.05 pre_grasp_clearance_m: 0.05
@@ -103,16 +182,3 @@ calibration:
method: DANIILIDIS method: DANIILIDIS
file: data/calibration.yaml file: data/calibration.yaml
samples_file: data/samples.json samples_file: data/samples.json
chat:
openrouter:
base_url: https://openrouter.ai/api/v1
model: anthropic/claude-sonnet-4.5
timeout_s: 60.0
max_tool_iterations: 8
temperature: 0.1
api_key_env: OPENROUTER_API_KEY
referer: http://localhost:8890
title: presa-mano
user_prompt_addendum: Misure sempre in mm interi. Sii conciso ma riporta i risultati
di OGNI tool che hai chiamato (anche solo in una riga compatta), non saltare
informazioni rilevanti.
+144
View File
@@ -0,0 +1,144 @@
# Bug report — D435i camera service: intrinsics mismatch nel back-projection 3D
**Servizio:** `front_camera` (RealSense D435i) — `http://192.168.123.99:8878`
**Severità:** alta — punti 3D ritornati con XY sovrastimati di ~54%
**Endpoint affetti:** almeno `/api/measure`; molto probabilmente anche `/api/pointcloud` e `/api/pointcloud/roi`
**Endpoint NON affetti:** `/api/intrinsics` (espone i parametri corretti)
**Data test:** 2026-05-27
**Reporter:** sviluppo presa-mano
---
## 1. Sintomo
Le coordinate 3D `(X, Y, Z)` ritornate dal back-projection lato server sono **stirate** lateralmente di un fattore **~1.54** sugli assi X e Y, mentre Z (depth) è plausibile.
Conseguenza: oggetti di dimensioni note appaiono ~50% più grandi del reale in qualsiasi misurazione 3D, e i point cloud non si sovrappongono ad altre camere correttamente calibrate.
---
## 2. Test di verifica
Marker ArUco fisico di **lato 103.0 mm** (`id=20`, dictionary DICT_4X4_50) posizionato davanti al D435i a ~735 mm.
### 2.1 Intrinsics riportate dal servizio (`GET /api/intrinsics`)
```json
{
"ir": { "fx": 392.30963134765625, "fy": 392.30963134765625,
"cx": 318.48004150390625, "cy": 242.43441772460938 },
"rgb": { "fx": 603.88037109375, "fy": 603.6130981445312,
"cx": 320.6182556152344, "cy": 250.35804748535156 },
"extrinsics_ir_to_rgb": { "rotation": [...], "translation": [14.68, -0.07, 0.29] }
}
```
### 2.2 Detect del marker (su RGB image del D435i)
Marker-finder ritorna le 4 corner del tag nel sistema **pixel del frame RGB**:
```
corners (RGB px): [(377, 68), (372, 152), (289, 147), (294, 63)]
```
### 2.3 `GET /api/measure?u=&v=&radius=3` per ciascuna corner
```
pixel (377, 68) → x=+0.1092 y=-0.3255 z=0.732 m
pixel (372, 152) → x=+0.0999 y=-0.1687 z=0.732 m
pixel (289, 147) → x=-0.0554 y=-0.1793 z=0.737 m
pixel (294, 63) → x=-0.0461 y=-0.3375 z=0.738 m
```
### 2.4 Misurazioni geometriche dai punti 3D ritornati
| Misura | Atteso | Ottenuto | Errore |
|-----------------|---------------|---------------|----------|
| Lato C0–C1 | 103.00 mm | 157.13 mm | **+52.6%** |
| Lato C1–C2 | 103.00 mm | 155.71 mm | **+51.2%** |
| Lato C2–C3 | 103.00 mm | 158.48 mm | **+53.9%** |
| Lato C3–C0 | 103.00 mm | 155.92 mm | **+51.4%** |
| Diagonale 0–2 | 145.66 mm | 220.19 mm | **+51.2%** |
| Diagonale 1–3 | 145.66 mm | 223.32 mm | **+53.3%** |
| Z medio (depth) | — | 735 mm | plausibile |
L'errore è **omogeneo ~52%** su tutte le coppie. Z (depth) è coerente — l'errore è confinato a X, Y.
---
## 3. Diagnosi (root cause)
Il rapporto fra le focali è esattamente:
```
fx_rgb / fx_ir = 603.88037 / 392.30963 = 1.5393
```
L'errore osservato sui lati è **1.524 ± 0.014**.
Verifica numerica della corner `(u=377, v=68)`, depth `z=0.732 m`:
```
Con RGB intrinsics (corretto): X = (377 − 320.62) × 0.732 / 603.88 = 0.0683 m
Con IR intrinsics (errato): X = (377 − 318.48) × 0.732 / 392.31 = 0.1091 m
Server ritorna: = 0.1092 m ← coincide con IR
```
**Conclusione:** il servizio sta usando le **intrinsics IR** (`fx=392.31, cx=318.48`) per back-projettare pixel che sono in **coordinate RGB** (allineamento depth-to-color attivo). Il risultato è X e Y inflati di `fx_rgb/fx_ir ≈ 1.54×`.
Il Z è corretto perché dipende solo dal valore di depth letto al pixel, non dalle intrinsics.
---
## 4. Fix consigliato
Nella routine di back-projection (probabilmente in `/api/measure`, `/api/pointcloud`, `/api/pointcloud/roi`):
```python
# ERRATO (sospetto attuale):
X = (u - cx_ir) * z / fx_ir
Y = (v - cy_ir) * z / fy_ir
# CORRETTO (depth allineata al frame color → usa RGB intrinsics):
X = (u - cx_rgb) * z / fx_rgb
Y = (v - cy_rgb) * z / fy_rgb
```
Se la pipeline NON sta facendo `align_to(color)` sul depth stream, allora il fix opposto: i pixel passati devono essere quelli del frame IR e si usano le intrinsics IR. Ma la semantica attuale (`/api/rgb` espone il frame color, `/api/measure` accetta `(u,v)` con range coerente col color frame ~640×480) suggerisce che l'API è progettata per RGB-aligned depth — quindi è il back-projection lato server che ha l'intrinsics sbagliata.
---
## 5. Test di non-regressione consigliati
Dopo il fix, verificare che:
1. **Tag di dimensione nota** (es. 103mm): le 4 corner back-proiettate formano un quadrato con lato `103.00 ± 5 mm` a 0.7m di distanza.
2. **Coerenza con `/api/intrinsics`**: per ogni pixel `(u, v)` con depth `z`, vale:
```
x_returned ≈ (u - cx_rgb) * z / fx_rgb (entro tolleranza distorsione)
y_returned ≈ (v - cy_rgb) * z / fy_rgb
```
3. **Cross-camera consistency**: il point cloud del D435i di un oggetto sovrapposto al cloud Nuwa (eye-in-hand polso DX) dello stesso oggetto deve avere errore residuo `< 30 mm` dopo hand-eye calibration.
---
## 6. Workaround temporaneo (lato client)
Nel viewport 3D di presa-mano è stato aggiunto un campo `D435i scale XY` con default **0.65** (`= fx_ir / fx_rgb`) che riscala il cloud per compensare il bug. Da rimuovere/portare a 1.0 quando il server è fixato.
---
## 7. Comandi per riprodurre
```bash
# 1) Verifica intrinsics
curl -sS http://192.168.123.99:8878/api/intrinsics
# 2) Detect marker (presuppone marker-finder a :8082 e tag 20 visibile)
curl -sS -X POST http://localhost:8082/detect \
-F "image_url=http://192.168.123.99:8878/api/rgb" \
-F 'markers=[{"id":20,"size_mm":103}]'
# 3) Per ogni corner (u,v) ritornata, misura 3D
curl -sS "http://192.168.123.99:8878/api/measure?u=377&v=68&radius=3"
```
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,153 @@
# Cross-camera tag overlay + Nuwa→frontale alignment calibration
**Date:** 2026-06-03
**Status:** approved (design)
**Area:** Pose 3D tab — `web/pose_3d.js`, `web/index.html`, `src/presa_mano/api/pose_3d_routes.py`, `src/presa_mano/geometry.py`
## Problem
The Pose 3D tab detects ArUco tags only on the **front D435i** camera (`GET /api/pose_3d/front_tags`) and draws them as an overlay. We want:
1. Tag detection on **all active cameras** (Nuwa wrist eye-in-hand + D435i front), not just the front one. Nuwa tags drawn in a **distinct red** color; D435i keeps its current color.
2. The "Tag camera frontale" UI block renamed to **"Tags"** (it now covers every active camera).
3. A **cross-camera alignment calibration**: a configurable `align_id` tag that must be seen by **both** cameras lets us fit a similarity transform that maps the Nuwa point cloud onto the D435i cloud (rotation + translation + uniform scale). This is a **persistent calibration** (saved, reused across sessions), gated by an enable/disable flag in the "Cloud cross-camera" section.
## Decisions (locked)
- **Recompute trigger:** dedicated **"Allinea"** button. The fitted matrix is frozen until the button is pressed again. Acquisisci does *not* recompute it.
- **Persistence:** the matrix is a **persistent calibration**, saved to `config.yaml > pose_3d.cross_camera` and reloaded on startup (like a hand-eye calibration). Session memory mirrors it.
- **Transform type:** **Umeyama similarity** — rotation + translation + single uniform scale.
- **Tag detection scope:** when "Mostra tag" is on, detection runs on **both** cameras always (independent of which clouds are shown).
- **Fallback when the align tag is not seen by both cameras at calibration time:** the `POST /align` call returns `found_in_both:false` and an error and **does not overwrite** the saved matrix. At render time, if the flag is ON but there is no valid saved matrix, the Nuwa cloud falls back to normal URDF/hand-eye placement and a warning is shown.
- **`align_id` default:** `0`. It must be a scene/table tag visible to both the wrist (Nuwa) and torso (D435i) cameras simultaneously — not a robot-mounted tag.
- **`/front_tags` removed** (the GUI is the only consumer) and replaced by the generalized endpoints below.
## Architecture
### Backend — `src/presa_mano/`
**`geometry.py` — new pure helper**
```python
def umeyama_similarity(src: np.ndarray, dst: np.ndarray
) -> tuple[float, np.ndarray, np.ndarray, float]:
"""Fit dst ≈ s·R·src + t (Umeyama 1991, with scale).
src, dst: (N,3) corresponding points. Returns (scale, R(3x3), t(3,), rmse_meters).
Raises ValueError if N < 3 or the point set is degenerate (rank-deficient covariance)."""
```
Numpy-only, unit-testable in isolation. Standard Umeyama: center both sets, `H = dst_c.T @ src_c / N`, SVD `U,S,Vt = svd(H)`, `D = diag(1,1,sign(det(U@Vt)))` to forbid reflections, `R = U @ D @ Vt`, `scale = trace(diag(S)@D) / var(src_c)`, `t = mean(dst) - scale·R·mean(src)`. `rmse` = RMS residual of `s·R·src + t` vs `dst`.
**`pose_3d_routes.py` — generalize detection**
Extract the per-camera body of the current `get_front_tags` into a camera-agnostic helper:
```python
def _detect_tags_on(client, frame_label: str, ids: str,
probe_size_mm: float = 40.0) -> list[dict]:
"""Detect ArUco on `client` RGB + measure 3D corners/center via `client.measure`.
`client` is any NuwaClient (Nuwa wrist cam or D435i). Returns the per-tag list
(id, corners_pixel, corners_3d_optical_m, corner_depths_mm, center_3d_optical_m,
edge_mm_per_side, edge_mm, distance_m) — identical schema to today's front_tags."""
```
The current intrinsics→K override, corner measure, depth-outlier rejection, edge/distance math move verbatim into this helper.
**`GET /api/pose_3d/tags?ids=0-49&probe_size_mm=40`** — detect on both cameras:
```json
{ "cameras": {
"nuwa": { "tags": [ ...tag schema... ], "frame": "nuwa_rgb_optical" },
"d435": { "tags": [ ... ], "frame": "d435_rgb_optical" } },
"timings_ms": { "nuwa": ..., "d435": ..., "total": ... } }
```
Each camera detected independently; a per-camera failure yields `{ "tags": [], "error": "<msg>", "frame": ... }` instead of a 500, so one offline camera doesn't kill the other (mirrors the `/clouds` graceful pattern).
**`POST /api/pose_3d/align?ids=0-49&align_id=0&probe_size_mm=40`** — compute + persist the calibration:
1. `_detect_tags_on` both cameras.
2. Find `align_id` in both `nuwa` and `d435` tag lists. If missing in either → return `{ "found_in_both": false, "error": "tag <id> non visto da <cam(s)>", ... }` **without touching the saved matrix**.
3. Build the 4 corner correspondences (`corners_3d_optical_m`), dropping any corner that is NaN in *either* camera. Require ≥3 valid corners else `found_in_both:false`, error.
4. `scale, R, t, rmse = umeyama_similarity(nuwa_corners, d435_corners)`.
5. Persist via `update_config({"pose_3d": {"cross_camera": {...}}})`:
```json
{ "align_id": 0, "scale": 0.98, "R": [[...3x3...]], "t": [x,y,z],
"rmse_mm": 3.2, "n_corners": 4 }
```
The `enabled` flag lives in the same `cross_camera` block but is written by the JS save path (see below), not overwritten here.
6. Return the same object plus `found_in_both:true`, `timings_ms`. Frame note: maps **Nuwa optical → D435 optical** (both OpenCV X-right/Y-down/Z-forward).
**`GET /api/pose_3d/state`** — extend the returned dict with `cross_camera`: the saved `{ enabled, align_id, scale, R, t, rmse_mm, n_corners }` (or `null` if never calibrated). The frontend uses it to repopulate the UI and apply the matrix on load.
### Frontend — `web/pose_3d.js` + `web/index.html`
**Tag overlay generalization.** `state.front_tags` → `state.tags = { nuwa: {group:null}, d435: {group:null} }`. Add `NUWA_TAG_COLOR = 0xff3b30` (red); D435i keeps `FRONT_TAG_COLOR`.
- `pollFrontTags()` → `pollTags()`: `GET /api/pose_3d/tags?ids=...`, then for each camera `renderTags(camKey, tags, color)`.
- `renderFrontTags(tags)` → `renderTags(camKey, tags, color)`: same geometry/label logic, but `clearTagsGroup(camKey)`, parent = `getCloudParent(camKey)` (so Nuwa tags ride the Nuwa cloud node, D435i tags the D435i node), color parameterized.
- `addOneFrontTag(grp, t)` → `addOneTag(grp, t, color)`.
- `clearFrontTagsGroup()` → `clearTagsGroup(camKey)` (per-camera).
- Trigger points unchanged in spirit: on the "Mostra tag" checkbox toggle-on, and at the end of `acquireClouds()` when the flag is on (the loop in `pollState()` stays removed). The IDs field `#p3FrontTagIds` is reused (the single shared range for both cameras).
**Cross-camera alignment.** New `state.align = { enabled:false, id:0, matrix:null /*THREE.Matrix4*/, info:null }`.
UI added to the existing **"Cloud cross-camera"** section in `index.html`:
```
[ ] Allinea Nuwa→frontale tag id [ 0 ] [Allinea]
<status line: scala / rmse / errore>
```
- `#p3AlignEnabled` (checkbox), `#p3AlignTagId` (number), `#p3AlignBtn` (button), `#p3AlignStatus` (status div).
Handlers:
- **`doAlign()`** (`#p3AlignBtn`): `POST /api/pose_3d/align?ids=<p3FrontTagIds>&align_id=<p3AlignTagId>`. On `found_in_both`: build `THREE.Matrix4` from `scale,R,t` → `state.align.matrix`, status shows `scala=… rmse=…mm`, then `applyNuwaCloudPlacement()`. On failure: leave `state.align.matrix` as-is, status shows the error in the warning color.
- **`#p3AlignEnabled` change**: update `state.align.enabled`, persist (see below), `applyNuwaCloudPlacement()`.
- On load, `applyCrossCameraFromCfg(s.cross_camera)` (called from `pollState`, guarded like `applyCloudParamsFromCfg` so it doesn't clobber in-progress edits): sets the checkbox/id inputs and, if `R/t/scale` present, builds `state.align.matrix`.
**Nuwa cloud placement.** Refactor the Nuwa branch of `applyCloudTransform`/`buildOrUpdateCloud` into `applyNuwaCloudPlacement()`:
- **Aligned mode** (`state.align.enabled && state.align.matrix`): re-parent the Nuwa `THREE.Points` to `getCloudParent('d435')` (same parent as the D435i cloud), set `matrixAutoUpdate=false`, and set
`nuwa.matrix = D435LocalMatrix · AlignMatrix`,
where `D435LocalMatrix` is exactly the matrix the D435i cloud uses relative to that parent (hand-eye `M·S`, or URDF `T(offset)·R(rpy)·S(scaleVec)` — extracted into a shared `d435LocalMatrix()` helper so both clouds stay in lockstep). A raw Nuwa optical point `p` then renders at `parent · D435Local · Align · p`, i.e. mapped into D435i optical space and pushed through D435i's own pipeline → guaranteed overlay regardless of URDF/hand-eye error.
- **Normal mode** (flag off, or no matrix): current behavior — parent `right_wrist_yaw_link`, `T·R·S` from `getCloudTransform('nuwa')` + `readCloudScaleVec('nuwa')`. If the flag is ON but `matrix` is null, also surface a warning in `#p3AlignStatus`.
`applyNuwaCloudPlacement()` is invoked from `buildOrUpdateCloud('nuwa')`, after `doAlign()`, and on the enable-flag toggle.
**Persistence (JS).** Extend `readCloudParamsFromUI`/`saveCloudParamsToConfig` path (or a sibling `saveCrossCameraFlag()`) to PUT `{ pose_3d: { cross_camera: { enabled: <bool>, align_id: <int> } } }`. The matrix fields are written server-side by `/align`; JS only owns `enabled` + `align_id`.
## Config schema
`config.yaml > pose_3d.cross_camera`:
```yaml
pose_3d:
cross_camera:
enabled: false
align_id: 0
scale: 1.0 # written by POST /align
R: [[1,0,0],[0,1,0],[0,0,1]]
t: [0.0, 0.0, 0.0]
rmse_mm: null
n_corners: 0
```
Reads tolerate a missing block (defaults). Writes go through `update_config` deep-merge.
## Error handling
- One camera offline → `/tags` and `/align` still report the other; `/align` with the dead camera missing the tag → `found_in_both:false`, no save.
- `umeyama_similarity` raises `ValueError` on <3 / degenerate corners → `/align` catches → `found_in_both:false`, error string.
- All new endpoints keep the `timings_ms` convention via `Stopwatch().step(...)`.
- Render-time: flag ON + null matrix → URDF fallback + visible warning (never a blank/crashed cloud).
## Testing
- **`umeyama_similarity`** (`tests/test_geometry.py` or new): apply a known `(s,R,t)` to random points, recover within tolerance (scale ~1e-6, R ~1e-6, t ~1e-6); collinear/<3 points → `ValueError`.
- **`POST /api/pose_3d/align`** (mock marker + both camera clients): tag in both → matrix returned + persisted; tag missing in one → `found_in_both:false`, saved matrix untouched. Use the **config snapshot/restore fixture** (conftest) because `update_config` reformats `config.yaml`.
- **`GET /api/pose_3d/tags`** (mock): both cameras' tags returned with correct `frame` labels; one camera raising → that camera reports `error`, the other still returns tags.
- **`GET /api/pose_3d/state`**: includes `cross_camera` (saved or null).
## Out of scope (YAGNI)
- Per-axis/anisotropic scale (rejected — uniform only).
- Continuous/live re-alignment (rejected — button-driven).
- Aligning D435i onto Nuwa (direction is fixed Nuwa→frontale).
- A clear/reset-calibration button (can be added later; not required now).
```
@@ -0,0 +1,95 @@
# GUI build-version badge
**Date:** 2026-06-03
**Status:** approved (design)
**Area:** `VERSION`, `build.sh`, `Dockerfile`, `src/presa_mano/version.py`, `src/presa_mano/main.py`, `web/index.html`, `web/app.js`
## Problem
`src/` and `web/` are baked into the Docker image (no bind-mount), so code changes only take effect after an image rebuild. There is no visible indicator of which build the running container serves, so a stale deployment is invisible — exactly the confusion behind "non vedo modifiche alla GUI". We want a version shown in the GUI header that increments on every build, plus the build time.
## Decisions (locked)
- **Version string:** `V<major>.<minor>.<patch>` (e.g. `V1.0.1`), **patch auto-increments every build**; build time shown too.
- **Injection:** a build wrapper script passes the version + build time as Docker build args.
- **Placement:** a pill badge in the GUI header, build time in its tooltip.
## Components
### `VERSION` (tracked file, repo root `presa-mano/VERSION`)
Plain text `major.minor.patch`, initialized to `1.0.0`. Single source of truth for the number; persists across builds via the file on disk. The build script rewrites it (left as an uncommitted working-tree edit — the script does NOT auto-commit).
### `build.sh` (repo root, executable)
```
1. cd to script dir; ensure VERSION exists (else seed 1.0.0)
2. parse current major.minor.patch
3. bump: default patch++; arg `minor` → minor++/patch=0; `major` → major++/minor=patch=0
4. write new VERSION
5. BUILD_TIME = $(date -u +"%Y-%m-%d %H:%M UTC")
6. (cd compose dir) docker compose build --build-arg APP_VERSION=<new> --build-arg BUILD_TIME="<ts>" presa-mano
7. docker compose up -d presa-mano
8. echo deployed version
```
The compose file lives one level up (`/opt/docker/robotserver/docker-compose.yml`); the script resolves that dir and runs compose there. Usage: `./build.sh [patch|minor|major]` (default patch).
### `Dockerfile`
Add late (before `CMD`), so changing them doesn't bust the dependency layers:
```
ARG APP_VERSION=dev
ARG BUILD_TIME=unknown
ENV APP_VERSION=${APP_VERSION}
ENV BUILD_TIME=${BUILD_TIME}
```
Also `COPY VERSION ./` (cheap, makes the file fallback work inside the container too).
### `src/presa_mano/version.py`
```python
def build_info() -> dict:
"""{'version', 'build_time'}. Prefer env (set at Docker build); fall back to
the VERSION file + '(local)' for `uv run` dev."""
```
- `version` = `APP_VERSION` env if set, else `<VERSION file contents> (local)` (or `0.0.0 (local)` if the file is unreadable).
- `build_time` = `BUILD_TIME` env if set, else `(local)`.
- `VERSION` file resolved as `Path(__file__).resolve().parents[2] / "VERSION"`.
### `src/presa_mano/main.py`
```python
from .version import build_info
@app.get("/api/version", tags=["meta"])
def version() -> dict:
return build_info()
```
### `web/index.html`
In the header, next to `#activeRobotPill`, add:
```html
<span class="active-robot-pill" id="appVersionPill" title="versione build">V—</span>
```
(reuses the existing pill styling).
### `web/app.js`
Add and call once at startup (alongside `refreshHomeRobotSelector()`):
```javascript
async function refreshAppVersion() {
try {
const r = await api('GET', '/api/version');
const pill = $('appVersionPill');
if (pill) { pill.textContent = `V${r.version}`; pill.title = `build ${r.build_time}`; }
} catch {}
}
refreshAppVersion();
```
## Testing
- Unit test `tests/test_version.py` for `build_info()`:
- env set (`APP_VERSION`, `BUILD_TIME` via monkeypatch) → returns exactly those.
- env unset (monkeypatch.delenv) → `version` ends with `(local)` and contains the `VERSION` file content; `build_time == "(local)"`.
- Frontend: manual — after `./build.sh` the badge increments and the tooltip shows the new build time.
## Out of scope (YAGNI)
- git SHA / dirty flag (user chose version+time only).
- Auto-commit of the VERSION bump.
- Frontend-vs-backend version mismatch warning.
-10
View File
@@ -1,10 +0,0 @@
"""Conversational AI assistant for presa-mano.
Talks to OpenRouter (OpenAI-compatible chat-completions API) and orchestrates
robot/perception actions via a curated set of tools defined in `tools.py`. The
LLM never sees the raw upstream services — it only sees high-level functions
that wrap our existing FastAPI endpoints, with safety guards (default dry-run
on motion, side validation, calibration-side awareness)."""
from .runner import AgentRunner # noqa: F401
from .tools import TOOL_DEFINITIONS, dispatch_tool # noqa: F401
-121
View File
@@ -1,121 +0,0 @@
"""System prompt for the presa-mano AI assistant.
Built dynamically so it always reflects the currently-active robot and the
real list of tools the model can call."""
from __future__ import annotations
from typing import Any
def build_system_prompt() -> str:
# Late imports to avoid the circular dependency:
# api/__init__.py imports agent_routes -> agent.runner -> agent.prompt,
# so agent.prompt must NOT import from `..api` at module load time.
from ..config import get_config
from ..api import _deps
cfg = get_config()
active = _deps.active_robot_key()
default_side = _deps.default_side()
aruco = cfg.get("aruco", {}).get("wrist_marker", {})
min_samples = int(cfg.get("calibration", {}).get("min_samples", 5))
return f"""Sei l'assistente di **presa-mano**, un orchestratore FastAPI che pilota
un robot umanoide (Unitree G1 o H1) con una camera depth Nuwa-HP60C fissa.
## Stile di risposta — REGOLE OBBLIGATORIE
- Rispondi in italiano, **diretto e conciso**: niente preamboli ("ecco i risultati…"),
niente paragrafi di chiusura ("se vuoi posso…"), niente emoji.
- Riporta SOLO i fatti che derivano dai tool eseguiti. **NON inventare valori
numerici, posizioni, dimensioni, spiegazioni di errori che il tool non ha
ritornato.** Se un tool ritorna `ok=false`, riporta letteralmente l'errore.
- Se manca un'informazione, dillo invece di indovinare. Non aggiungere
"possibili cause" speculative se non sono nel risultato del tool.
- Niente disclaimers o suggerimenti non richiesti. L'operatore ti dirà se
vuole un consiglio.
- Per dati numerici usa tabelle compatte; per istruzioni passi numerati brevi.
- Per tutto ciò che è azione su robot/camera/calibrazione DEVI usare i tool.
**Vietato simulare risultati.**
## Stato corrente
- Robot attivo: **{active.upper()}** (puoi cambiarlo con `set_active_robot`).
- Lato di default per le operazioni di braccio: **{default_side}**. Se l'utente
non specifica `right`/`left`, usa questo default.
- Marker ArUco al polso: id={aruco.get('id', 10)}, lato={aruco.get('size_mm', 40)} mm.
## Convenzioni
- Le posizioni sono in **metri** nel frame base del robot, gli orientamenti
in quaternioni `[x, y, z, w]`. Le rotazioni "in gradi" che l'utente cita
vanno convertite tu (le tool accettano sempre quaternioni).
- Movimenti **assoluti**: `move_wrist_absolute(side, position, quaternion)`.
- Movimenti **relativi**: `move_wrist_relative(side, delta_position)` —
delta in metri, frame base. Se l'utente dice "muovi di 200 mm in X+", passa
delta=[0.2, 0, 0]. Conferma sempre il risultato leggendo la nuova posa.
- **Modalità precise**: per default passa SEMPRE `precise=true` su
`move_wrist_absolute` e `move_wrist_relative`. Imposta `precise=false`
solo se l'utente lo chiede esplicitamente (es. "muovi veloce",
"avvicinati grossomodo"). Lo `precise=true` garantisce che il controller
raggiunga la posa target con accuratezza fine — necessario per
acquisizione sample di calibrazione, avvicinamento al grasp, e in generale
ogni movimento dove la posa finale conta.
- Default `dry_run=true` su tutte le operazioni di motion: NON fai mai partire
un movimento reale senza che l'utente l'abbia chiesto esplicitamente.
- Le mani: `hand_action(side, action)` con `action ∈
{{LEFT_OPEN, LEFT_CLOSE, RIGHT_OPEN, RIGHT_CLOSE}}`.
## Posizione di riposo (home) delle braccia
- Quando l'operatore dice **"torna a casa"**, **"home"**, **"posizione di
riposo"** o simili, chiama `arms_go_neutral()`. Agisce su entrambe le
braccia simultaneamente (è un comando del bridge, niente side da scegliere).
- Default `dry_run=true` come per gli altri movimenti — chiedi conferma
prima di mandarlo dal vivo.
## Calibrazione hand-eye (eye-to-hand)
La calibrazione lega `T_base_camera`. È **specifica per braccio**: dx e sx
hanno file separati. Procedura completa quando l'utente dice "calibra":
1. Determina `side` (chiedi se ambiguo).
2. `calibration_clear(side)` per partire da zero (chiedi prima conferma).
3. Ripeti almeno {min_samples} volte (consigliato 8–15):
a. `move_wrist_relative` o `move_wrist_absolute` per portare il polso in
una posa nuova ben visibile dalla camera (varia X, Y, Z e l'orientamento
del marker — più diversità = miglior calibrazione).
b. Aspetta l'assestamento (tool `sleep` non disponibile: passa al sample;
`move_wrist_*` di default attende).
c. `calibration_add_sample(side)` — questa cattura RGB+pose+marker.
d. Se il marker non è stato trovato (errore 422), riprova con una posa
diversa.
4. `calibration_solve(side, method='TSAI')` — richiede ≥{min_samples} sample.
5. Riporta all'utente i residui (`residual_translation_mm`, `residual_rotation_deg`).
Se i residui sono > 5 mm o > 1° suggerisci più sample.
## Percezione (per planning grasp)
- `detect_object(class_name, confidence)` → bbox + score.
- `get_point_cloud_roi(bbox)` → statistiche nuvola 3D (mean/min/max in **frame camera Nuwa**: X-right, Y-up, Z-back; Z<0 = davanti).
- `get_marker(marker_id, size_mm)` → posa marker **da PnP** nel **frame camera OpenCV** (X-right, Y-down, Z-forward; Z>0 = davanti). PnP eseguita da marker-finder con gli intrinseci veri della Nuwa. È il metodo più affidabile per **posizione assoluta** quando size_mm è dichiarata correttamente.
- `get_marker_3d(marker_id, size_mm)` → posa marker **dal sensore depth** (no PnP). Restituisce anche `measured_side_mm`. ATTENZIONE: se la correzione depth (affine, vedi sotto) è attiva, le dimensioni geometriche misurate vengono distorte — `measured_side_mm` può differire significativamente da `declared_size_mm` anche se la size dichiarata è corretta. **NON usare measured_side_mm come ground truth della size dichiarata.**
- Per validare se `size_mm` dichiarato è corretto, confronta `get_marker` Z con la distanza vera misurata col metro: il PnP scala linearmente con size_mm dichiarata.
- Per stimare la posa **nel frame base del robot** dopo la calibrazione, considera che la matrice `T_base_camera` salvata da `calibration_solve` è quella che mappa il frame camera OpenCV nel frame base.
## Sensore depth e correzione
Il sensore depth Nuwa ha un bias che è stato calibrato come trasformazione affine (`depth_corrected = (raw - offset) / scale`). Quando attiva, migliora l'accuratezza della **distanza** ma distorce le **dimensioni** misurate dai 4 corner 3D del marker. Lo stato della correzione è leggibile da `GET /api/perception/depth_correction` (non esposto come tool: chiedi all'utente se serve verificarla).
## Sicurezza
- Prima di QUALSIASI movimento reale (dry_run=false), conferma con l'utente
l'azione e il valore numerico.
- Non chiamare mai `emergency_stop` o `unlock` se non te lo chiede l'utente.
- Se l'utente ti dà un task vago ("muovi il braccio") chiedi precisazioni.
Comincia ascoltando la richiesta dell'operatore e usa i tool quando servono.
Se rispondi con testo soltanto (nessun tool call), assicurati che l'utente
abbia tutte le informazioni che ha chiesto.
""" + _addendum(cfg)
def _addendum(cfg: dict[str, Any]) -> str:
"""Optional extra instructions that the operator can edit from the GUI
without recompiling the base prompt. Stored in chat.openrouter.user_prompt_addendum."""
txt = (cfg.get("chat", {}).get("openrouter", {}).get("user_prompt_addendum") or "").strip()
if not txt:
return ""
return f"\n\n## Istruzioni aggiuntive dell'operatore\n{txt}\n"
-174
View File
@@ -1,174 +0,0 @@
"""OpenRouter chat runner with multi-turn tool calling.
OpenRouter speaks the OpenAI chat-completions API verbatim, so we use
`httpx` directly and re-implement the tool-call loop. We don't take a hard
dep on the openai SDK — the protocol is small enough to inline and this
keeps Docker layers slim.
"""
from __future__ import annotations
import json
import logging
import os
import time
from dataclasses import dataclass, field
from typing import Any
import httpx
from ..config import get_config
from .prompt import build_system_prompt
from .tools import TOOL_DEFINITIONS, dispatch_tool
log = logging.getLogger(__name__)
class OpenRouterError(RuntimeError):
pass
@dataclass
class ToolCallTrace:
name: str
arguments: dict[str, Any]
result: dict[str, Any]
latency_ms: float
@dataclass
class AgentResult:
reply: str
iterations: int
tool_calls: list[ToolCallTrace] = field(default_factory=list)
finish_reason: str | None = None
model: str | None = None
usage: dict[str, Any] | None = None
class AgentRunner:
"""Stateless: one runner per request. Caller is responsible for keeping
the conversation history and passing it back in via `messages`.
The run loop: send messages → if tool_calls in response, execute each,
append the assistant message + tool messages, repeat. Up to
`max_tool_iterations` rounds (configurable in chat.openrouter)."""
def __init__(self) -> None:
cfg = get_config().get("chat", {}).get("openrouter") or {}
self.base_url = str(cfg.get("base_url") or "https://openrouter.ai/api/v1").rstrip("/")
self.model = str(cfg.get("model") or "anthropic/claude-sonnet-4.5")
self.timeout_s = float(cfg.get("timeout_s") or 60.0)
self.max_tool_iterations = int(cfg.get("max_tool_iterations") or 8)
# Default low so the model doesn't invent values when summarising
# tool results. Override in chat.openrouter.temperature if needed.
self.temperature = float(cfg.get("temperature") if cfg.get("temperature") is not None else 0.1)
self.referer = cfg.get("referer") or "http://localhost:8890"
self.title = cfg.get("title") or "presa-mano"
env_var = str(cfg.get("api_key_env") or "OPENROUTER_API_KEY")
self.api_key = os.environ.get(env_var, "")
self.api_key_env = env_var
def has_credentials(self) -> bool:
return bool(self.api_key)
# ------------------------------------------------------------------
def run(self, user_message: str,
history: list[dict[str, Any]] | None = None,
) -> tuple[AgentResult, list[dict[str, Any]]]:
"""Returns (result, updated_history). `history` is the full message
list as it appears in the OpenAI API (excluding the system prompt;
we always inject it ourselves so it reflects current state)."""
if not self.has_credentials():
raise OpenRouterError(f"OpenRouter API key non impostata (env var {self.api_key_env}).")
system = {"role": "system", "content": build_system_prompt()}
messages: list[dict[str, Any]] = [system]
if history:
messages.extend(m for m in history if m.get("role") != "system")
messages.append({"role": "user", "content": user_message})
traces: list[ToolCallTrace] = []
iterations = 0
finish_reason: str | None = None
usage: dict[str, Any] | None = None
while iterations < self.max_tool_iterations:
iterations += 1
resp = self._chat_completion(messages)
usage = resp.get("usage") or usage
choices = resp.get("choices") or []
if not choices:
raise OpenRouterError(f"empty choices in response: {resp}")
msg = choices[0].get("message") or {}
finish_reason = choices[0].get("finish_reason")
messages.append(msg)
tool_calls = msg.get("tool_calls") or []
if not tool_calls:
# Final assistant turn — no further tools requested.
reply = msg.get("content") or ""
history_out = [m for m in messages if m.get("role") != "system"]
return AgentResult(reply=reply, iterations=iterations,
tool_calls=traces, finish_reason=finish_reason,
model=resp.get("model"), usage=usage), history_out
# Execute each tool call and append the result back.
for call in tool_calls:
fn = (call.get("function") or {})
name = fn.get("name") or "?"
raw_args = fn.get("arguments") or "{}"
try:
args = json.loads(raw_args) if isinstance(raw_args, str) else (raw_args or {})
except json.JSONDecodeError:
args = {}
t0 = time.perf_counter()
result = dispatch_tool(name, args)
latency = round((time.perf_counter() - t0) * 1000.0, 1)
traces.append(ToolCallTrace(name=name, arguments=args,
result=result, latency_ms=latency))
messages.append({
"role": "tool",
"tool_call_id": call.get("id"),
"name": name,
"content": json.dumps(result, ensure_ascii=False, default=str),
})
# Hit the iteration cap with tool calls still pending — degrade gracefully.
history_out = [m for m in messages if m.get("role") != "system"]
return AgentResult(
reply="Limite di iterazioni raggiunto: rivedi il task o chiedimi di continuare.",
iterations=iterations, tool_calls=traces, finish_reason="iteration_cap",
model=self.model, usage=usage,
), history_out
# ------------------------------------------------------------------
def _chat_completion(self, messages: list[dict[str, Any]]) -> dict[str, Any]:
url = f"{self.base_url}/chat/completions"
body = {
"model": self.model,
"messages": messages,
"tools": TOOL_DEFINITIONS,
"tool_choice": "auto",
"temperature": self.temperature,
}
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
# OpenRouter analytics — optional but recommended.
"HTTP-Referer": self.referer,
"X-Title": self.title,
}
try:
with httpx.Client(timeout=self.timeout_s) as c:
r = c.post(url, json=body, headers=headers)
except httpx.HTTPError as e:
raise OpenRouterError(f"OpenRouter unreachable: {e}") from e
if r.status_code >= 400:
try:
detail = r.json()
except Exception:
detail = r.text
raise OpenRouterError(f"OpenRouter HTTP {r.status_code}: {detail}")
return r.json()
-732
View File
@@ -1,732 +0,0 @@
"""Tools the AI assistant can invoke.
Each tool is defined twice:
1. As a JSON-schema entry in `TOOL_DEFINITIONS` (sent to OpenRouter / the LLM
so it knows the function signature).
2. As a Python handler keyed by tool name in `_HANDLERS` that wraps existing
internal endpoints/clients.
Handlers are intentionally thin and well-validated: they enforce side ∈
{right,left}, default dry_run=True for every motion call, and reuse the
existing per-request client construction (so a `set_active_robot` call mid-
conversation actually changes which robot subsequent tool calls hit).
Tool results are returned as dicts and serialised to JSON before being sent
back to the model. Errors are reported as `{"error": "..."}` rather than
raising, so the model can recover (e.g. retry with a different pose).
"""
from __future__ import annotations
import logging
import time
from dataclasses import asdict
from typing import Any, Callable
import numpy as np
from ..calibration import CalibrationSampler, CalibrationStore, HandEyeSolver, SamplesStore
from ..clients import G1StatusError, G1Unavailable
from ..config import get_config, update_config
from ..perception import ObjectDetector
log = logging.getLogger(__name__)
# Lazy proxy for `..api._deps` to break the import cycle:
# api/__init__.py loads agent_routes which loads agent.runner which loads us.
# Importing `..api._deps` at module load would trigger api/__init__.py while
# it's already mid-init. Attribute access via this proxy defers the import
# until the first tool actually runs, by which time everything is loaded.
class _LazyDeps:
def __getattr__(self, name: str) -> Any:
from ..api import _deps as _m
return getattr(_m, name)
_deps = _LazyDeps()
# ----- helpers --------------------------------------------------------------
def _ok(**kw: Any) -> dict[str, Any]:
out = {"ok": True}
out.update(kw)
return out
def _err(msg: str, **kw: Any) -> dict[str, Any]:
out = {"ok": False, "error": msg}
out.update(kw)
return out
def _validate_side(side: str | None) -> str:
s = (side or _deps.default_side()).lower()
if s not in {"right", "left"}:
raise ValueError(f"side must be 'right' or 'left', got: {side!r}")
return s
# ----- tool: home / robot selection ----------------------------------------
def _h_get_active_robot(_: dict[str, Any]) -> dict[str, Any]:
return _ok(active=_deps.active_robot_key(),
default_side=_deps.default_side())
def _h_set_active_robot(args: dict[str, Any]) -> dict[str, Any]:
target = str(args.get("robot", "")).lower()
if target not in {"g1", "h1"}:
return _err(f"robot must be 'g1' or 'h1', got: {target!r}")
services = (get_config().get("services") or {})
if not (services.get(target) or {}).get("base_url"):
return _err(f"services.{target}.base_url not configured")
update_config({"robot": {"active": target}})
return _ok(active=_deps.active_robot_key())
def _h_device_status(_: dict[str, Any]) -> dict[str, Any]:
"""Reuses the home_routes implementation to avoid duplicating the ping logic."""
from ..api.home_routes import status as _status
try:
return _ok(**_status())
except Exception as e:
return _err(str(e))
# ----- tool: wrist read/move ------------------------------------------------
def _h_get_wrist_pose(args: dict[str, Any]) -> dict[str, Any]:
try:
side = _validate_side(args.get("side"))
data = _deps.g1().wrist()
body = data.get("data") if isinstance(data.get("data"), dict) else data
pose = body.get(f"{side}_wrist") or body.get(side) or {}
return _ok(side=side,
position=pose.get("position"),
quaternion=pose.get("quaternion"))
except (G1Unavailable, G1StatusError) as e:
return _err(f"robot non raggiungibile: {e}")
except Exception as e:
return _err(str(e))
def _h_move_wrist_absolute(args: dict[str, Any]) -> dict[str, Any]:
try:
side = _validate_side(args.get("side"))
position = list(args.get("position") or [])
quaternion = list(args.get("quaternion") or [])
if len(position) != 3 or len(quaternion) != 4:
return _err("position deve avere 3 float (m), quaternion 4 float (xyzw)")
dry_run = bool(args.get("dry_run", True))
timeout_s = float(args.get("timeout_s", 5.0))
precise = bool(args.get("precise", True))
result = _deps.g1(dry_run=dry_run).wrist_absolute(
side=side, position=position, quaternion=quaternion,
wait=True, timeout_s=timeout_s, precise=precise,
)
return _ok(dry_run=dry_run, side=side, precise=precise, response=result)
except (G1Unavailable, G1StatusError) as e:
return _err(f"motion failed: {e}")
except Exception as e:
return _err(str(e))
def _h_move_wrist_relative(args: dict[str, Any]) -> dict[str, Any]:
try:
side = _validate_side(args.get("side"))
dp = args.get("delta_position")
dq = args.get("delta_quaternion")
if dp is not None and len(list(dp)) != 3:
return _err("delta_position deve avere 3 float (m) o essere null")
if dq is not None and len(list(dq)) != 4:
return _err("delta_quaternion deve avere 4 float (xyzw) o essere null")
dry_run = bool(args.get("dry_run", True))
timeout_s = float(args.get("timeout_s", 5.0))
precise = bool(args.get("precise", True))
result = _deps.g1(dry_run=dry_run).wrist_relative(
side=side,
delta_position=list(dp) if dp is not None else None,
delta_quaternion=list(dq) if dq is not None else None,
wait=True, timeout_s=timeout_s, precise=precise,
)
return _ok(dry_run=dry_run, side=side, precise=precise, response=result)
except (G1Unavailable, G1StatusError) as e:
return _err(f"motion failed: {e}")
except Exception as e:
return _err(str(e))
def _h_arms_go_neutral(args: dict[str, Any]) -> dict[str, Any]:
"""Riporta entrambe le braccia alla posa di home / riposo built-in del
bridge (POST /arms/go_neutral). È l'unico modo standard di "tornare a
casa" — non serve registrare pose custom."""
try:
dry_run = bool(args.get("dry_run", True))
result = _deps.g1(dry_run=dry_run).arms_go_neutral()
return _ok(dry_run=dry_run, response=result)
except (G1Unavailable, G1StatusError) as e:
return _err(f"go_neutral failed: {e}")
except Exception as e:
return _err(str(e))
def _h_hand_action(args: dict[str, Any]) -> dict[str, Any]:
try:
action = str(args.get("action", "")).upper()
valid = {"LEFT_OPEN", "LEFT_CLOSE", "RIGHT_OPEN", "RIGHT_CLOSE"}
if action not in valid:
return _err(f"action deve essere uno di {sorted(valid)}")
dry_run = bool(args.get("dry_run", True))
result = _deps.g1(dry_run=dry_run).hand_action(action)
return _ok(dry_run=dry_run, response=result)
except (G1Unavailable, G1StatusError) as e:
return _err(f"hand action failed: {e}")
except Exception as e:
return _err(str(e))
# ----- tool: perception -----------------------------------------------------
def _h_get_marker(args: dict[str, Any]) -> dict[str, Any]:
"""Detect ArUco marker and return its pose in the camera frame.
Da quando marker-finder accetta intrinseci OpenCV via API e presa-mano
gli pusha quelli veri della Nuwa al boot, possiamo fidarci direttamente
di `position_m`/`quaternion_xyzw` ritornati dal servizio. Niente più
re-PnP locale qui: per la diagnostica di ambiguità c'è il sampler
(CalibrationSampler) che la calcola sui sample reali."""
try:
cfg = get_config()
marker_id = int(args.get("marker_id") or cfg["aruco"]["wrist_marker"]["id"])
size_mm = float(args.get("size_mm") or cfg["aruco"]["wrist_marker"]["size_mm"])
result = _deps.marker().detect_url(
markers=[{"id": marker_id, "size_mm": size_mm}],
image_url=_deps.nuwa().rgb_url(),
)
found = result.get("found", [])
entry = next((m for m in found if int(m["id"]) == marker_id), None)
if entry is None:
return _err(f"marker id={marker_id} non trovato",
not_found=result.get("not_found"))
img_pts = np.asarray(entry["corners"], dtype=np.float64).reshape(-1, 2)
tvec = np.asarray(entry.get("position_m") or [0, 0, 0], dtype=np.float64).flatten()
return _ok(
marker_id=marker_id,
position_m=tvec.tolist(),
distance_m=float(np.linalg.norm(tvec)),
quaternion_xyzw=entry.get("quaternion_xyzw"),
rotation_matrix=entry.get("rotation_matrix"),
corners=entry.get("corners"),
center_px=img_pts.mean(axis=0).tolist(),
)
except Exception as e:
return _err(str(e))
def _h_detect_object(args: dict[str, Any]) -> dict[str, Any]:
try:
nuwa = _deps.nuwa()
yolo = _deps.yolo()
detector = ObjectDetector(yolo, nuwa)
conf = args.get("confidence")
conf = float(conf) if conf is not None else None
class_name = args.get("class_name")
dets = detector.detect(conf=conf)
if class_name:
dets = [d for d in dets if str(d.get("class_name", "")).lower() == str(class_name).lower()]
if not dets:
return _ok(found=False, detections=[])
return _ok(found=True, detections=dets[:5])
except Exception as e:
return _err(str(e))
def _h_get_marker_3d(args: dict[str, Any]) -> dict[str, Any]:
"""Pose marker da DEPTH (no PnP).
Detection 2D dei corner dal marker-finder, poi back-projection di
ciascun corner usando la depth con registrazione RGB↔IR. Centro = media
dei 4 corner 3D; orientamento = base ortonormale costruita dai lati del
marker. Più affidabile di PnP quando si conosce bene la depth e meno
sensibile a errori di size_mm dichiarata."""
try:
from ..perception.depth_to_rgb import (
CameraRegistration, back_project_rgb_pixel, fetch_depth_mm,
)
from ..perception.depth_correction import load as _load_dc_state
cfg = get_config()
marker_id = int(args.get("marker_id") or cfg["aruco"]["wrist_marker"]["id"])
size_mm = float(args.get("size_mm") or cfg["aruco"]["wrist_marker"]["size_mm"])
nuwa = _deps.nuwa()
result = _deps.marker().detect_url(
markers=[{"id": marker_id, "size_mm": size_mm}],
image_url=nuwa.rgb_url(),
)
entry = next((m for m in result.get("found", []) if int(m["id"]) == marker_id), None)
if entry is None:
return _err(f"marker id={marker_id} non trovato",
not_found=result.get("not_found"))
reg = CameraRegistration.from_nuwa(nuwa.intrinsics())
depth = fetch_depth_mm(nuwa)
dc_state = _load_dc_state(_deps.depth_correction_path())
corners_2d = np.asarray(entry["corners"], dtype=np.float64).reshape(-1, 2)
corners_3d = []
per_corner = []
for u, v in corners_2d:
r = back_project_rgb_pixel(u, v, depth, reg, sample_radius=4, n_iters=2,
depth_correction=dc_state)
per_corner.append({"rgb_pixel": r["rgb_pixel"], "ok": r["ok"],
"ir_pixel": r.get("ir_pixel"),
"depth_mm": r.get("depth_mm"),
"point_rgb_m": r.get("point_rgb_m"),
**({"error": r["error"]} if "error" in r else {})})
if r["ok"]:
corners_3d.append(r["point_rgb_m"])
if len(corners_3d) < 3:
return _err(f"depth mancante su {4 - len(corners_3d)} corner: marker fuori range o riflesso",
per_corner=per_corner)
pts = np.asarray(corners_3d, dtype=np.float64)
center = pts.mean(axis=0)
# Build an orthonormal frame from the marker plane:
# x-axis = (TR - TL) (top edge), y-axis = (BL - TL) (left edge),
# z-axis = x × y (outward normal).
out_pose: dict[str, Any] = {"center_m": center.tolist()}
if len(corners_3d) == 4:
tl, tr, br, bl = pts
x_axis = tr - tl
y_axis = bl - tl
x_axis /= (np.linalg.norm(x_axis) + 1e-12)
# Re-orthogonalize y against x:
y_axis = y_axis - (y_axis @ x_axis) * x_axis
y_axis /= (np.linalg.norm(y_axis) + 1e-12)
z_axis = np.cross(x_axis, y_axis)
R = np.column_stack([x_axis, y_axis, z_axis])
from ..geometry import R_to_quat_xyzw
out_pose["rotation_matrix"] = R.tolist()
out_pose["quaternion_xyzw"] = R_to_quat_xyzw(R).tolist()
# Side measured vs declared — sanity check
measured_side_mm = float(np.linalg.norm(tr - tl) * 1000)
out_pose["measured_side_mm"] = round(measured_side_mm, 1)
out_pose["declared_size_mm"] = size_mm
return _ok(
marker_id=marker_id,
distance_m=float(np.linalg.norm(center)),
n_corners_with_depth=len(corners_3d),
pose=out_pose,
per_corner=per_corner,
corners_2d=corners_2d.tolist(),
)
except Exception as e:
return _err(str(e))
def _h_get_point_cloud_roi(args: dict[str, Any]) -> dict[str, Any]:
try:
bbox = args.get("bbox")
if not bbox or len(list(bbox)) != 4:
return _err("bbox deve essere [x1,y1,x2,y2] in pixel")
x1, y1, x2, y2 = (int(v) for v in bbox)
nuwa = _deps.nuwa()
cloud = nuwa.pointcloud_roi(x1=x1, y1=y1, x2=x2, y2=y2)
# Cloud is potentially huge — return only summary stats so we don't
# blow up the LLM context.
pts = np.asarray(cloud, dtype=np.float64).reshape(-1, 6) if len(cloud) else np.zeros((0, 6))
if pts.size == 0:
return _ok(n_points=0, summary=None)
xyz = pts[:, :3]
return _ok(
n_points=int(pts.shape[0]),
mean=xyz.mean(axis=0).tolist(),
min=xyz.min(axis=0).tolist(),
max=xyz.max(axis=0).tolist(),
std_mm=(1000.0 * xyz.std(axis=0)).tolist(),
)
except Exception as e:
return _err(str(e))
# ----- tool: calibration ---------------------------------------------------
def _h_calibration_add_sample(args: dict[str, Any]) -> dict[str, Any]:
try:
side = _validate_side(args.get("side"))
cfg = get_config()
marker_id = int(args.get("marker_id") or cfg["aruco"]["wrist_marker"]["id"])
size_mm = float(args.get("size_mm") or cfg["aruco"]["wrist_marker"]["size_mm"])
sampler = CalibrationSampler(_deps.nuwa(), _deps.marker(), _deps.g1())
sample = sampler.acquire(marker_id=marker_id, size_mm=size_mm,
side=side, note=str(args.get("note", "")))
store = SamplesStore(_deps.samples_file_path(side))
items = store.append(sample)
return _ok(side=side, total=len(items), sample=asdict(sample))
except Exception as e:
return _err(str(e))
def _h_calibration_solve(args: dict[str, Any]) -> dict[str, Any]:
try:
side = _validate_side(args.get("side"))
cfg = get_config()
method = str(args.get("method") or cfg.get("calibration", {}).get("method") or "TSAI").upper()
min_n = int(cfg.get("calibration", {}).get("min_samples", 5))
samples = SamplesStore(_deps.samples_file_path(side)).load()
if len(samples) < min_n:
return _err(f"servono almeno {min_n} sample per side='{side}', ce ne sono {len(samples)}")
result = HandEyeSolver.solve(samples, method=method)
CalibrationStore(_deps.calibration_file_path(side)).save(result)
# Strip the heavy matrix from what we return to the model — keep summary.
slim = {
"side": side,
"method": result.get("method"),
"n_samples": result.get("n_samples"),
"residual_translation_mm": result.get("residual_translation_mm"),
"residual_rotation_deg": result.get("residual_rotation_deg"),
}
return _ok(**slim, saved=True)
except Exception as e:
return _err(str(e))
def _h_calibration_clear(args: dict[str, Any]) -> dict[str, Any]:
try:
side = _validate_side(args.get("side"))
SamplesStore(_deps.samples_file_path(side)).clear()
return _ok(side=side, total=0)
except Exception as e:
return _err(str(e))
def _h_calibration_get_result(args: dict[str, Any]) -> dict[str, Any]:
try:
side = _validate_side(args.get("side"))
doc = CalibrationStore(_deps.calibration_file_path(side)).load()
if doc is None:
return _err(f"nessuna calibrazione salvata per side='{side}'")
return _ok(side=side,
saved_at=doc.get("saved_at"),
method=doc.get("method"),
n_samples=doc.get("n_samples"),
residual_translation_mm=doc.get("residual_translation_mm"),
residual_rotation_deg=doc.get("residual_rotation_deg"))
except Exception as e:
return _err(str(e))
def _h_calibration_count_samples(args: dict[str, Any]) -> dict[str, Any]:
try:
side = _validate_side(args.get("side"))
samples = SamplesStore(_deps.samples_file_path(side)).load()
return _ok(side=side, total=len(samples))
except Exception as e:
return _err(str(e))
# ----- registry -------------------------------------------------------------
_HANDLERS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {
"get_active_robot": _h_get_active_robot,
"set_active_robot": _h_set_active_robot,
"device_status": _h_device_status,
"get_wrist_pose": _h_get_wrist_pose,
"move_wrist_absolute": _h_move_wrist_absolute,
"move_wrist_relative": _h_move_wrist_relative,
"hand_action": _h_hand_action,
"arms_go_neutral": _h_arms_go_neutral,
"get_marker": _h_get_marker,
"get_marker_3d": _h_get_marker_3d,
"detect_object": _h_detect_object,
"get_point_cloud_roi": _h_get_point_cloud_roi,
"calibration_add_sample": _h_calibration_add_sample,
"calibration_solve": _h_calibration_solve,
"calibration_clear": _h_calibration_clear,
"calibration_get_result": _h_calibration_get_result,
"calibration_count_samples": _h_calibration_count_samples,
}
# OpenRouter / OpenAI tool-calling format. Each entry is a 'function' tool.
TOOL_DEFINITIONS: list[dict[str, Any]] = [
{
"type": "function",
"function": {
"name": "get_active_robot",
"description": "Quale robot è attualmente collegato (g1 o h1) e quale lato è il default.",
"parameters": {"type": "object", "properties": {}},
},
},
{
"type": "function",
"function": {
"name": "set_active_robot",
"description": "Cambia il robot attivo (g1 o h1). Tutti i comandi successivi useranno questo robot.",
"parameters": {
"type": "object",
"required": ["robot"],
"properties": {"robot": {"type": "string", "enum": ["g1", "h1"]}},
},
},
},
{
"type": "function",
"function": {
"name": "device_status",
"description": "Stato di comunicazione (online/offline + latenza) dei microservizi: Nuwa, marker-finder, YOLO, robot.",
"parameters": {"type": "object", "properties": {}},
},
},
{
"type": "function",
"function": {
"name": "get_wrist_pose",
"description": "Posa attuale del polso (position xyz in metri, quaternion xyzw) nel frame base.",
"parameters": {
"type": "object",
"properties": {"side": {"type": "string", "enum": ["right", "left"]}},
},
},
},
{
"type": "function",
"function": {
"name": "move_wrist_absolute",
"description": (
"Sposta il polso a una posa assoluta nel frame base. Default dry_run=true: "
"il robot NON si muove davvero finché dry_run=false. Usa precise=true per "
"movimenti che richiedono accuratezza fine (es. acquisizione sample di "
"calibrazione, avvicinamento finale al grasp); precise=false è il default, "
"più veloce ma con tolleranza maggiore."
),
"parameters": {
"type": "object",
"required": ["side", "position", "quaternion"],
"properties": {
"side": {"type": "string", "enum": ["right", "left"]},
"position": {"type": "array", "items": {"type": "number"}, "minItems": 3, "maxItems": 3,
"description": "x,y,z in metri (frame base)"},
"quaternion": {"type": "array", "items": {"type": "number"}, "minItems": 4, "maxItems": 4,
"description": "x,y,z,w"},
"dry_run": {"type": "boolean", "default": True},
"precise": {"type": "boolean", "default": True,
"description": "true = controllo accurato (più lento, default)"},
"timeout_s": {"type": "number", "default": 5.0},
},
},
},
},
{
"type": "function",
"function": {
"name": "move_wrist_relative",
"description": (
"Sposta il polso di un delta rispetto alla posa attuale. delta_position "
"in metri (frame base), delta_quaternion opzionale. Default dry_run=true. "
"Usa precise=true se serve accuratezza fine."
),
"parameters": {
"type": "object",
"required": ["side"],
"properties": {
"side": {"type": "string", "enum": ["right", "left"]},
"delta_position": {"type": "array", "items": {"type": "number"}, "minItems": 3, "maxItems": 3},
"delta_quaternion": {"type": "array", "items": {"type": "number"}, "minItems": 4, "maxItems": 4},
"dry_run": {"type": "boolean", "default": True},
"precise": {"type": "boolean", "default": True},
"timeout_s": {"type": "number", "default": 5.0},
},
},
},
},
{
"type": "function",
"function": {
"name": "arms_go_neutral",
"description": (
"Riporta ENTRAMBE le braccia alla posa di home/riposo del robot "
"(POST /arms/go_neutral sul bridge). Usa quando l'utente dice "
"'torna a casa', 'home', 'posizione di riposo'. Default dry_run=true."
),
"parameters": {
"type": "object",
"properties": {"dry_run": {"type": "boolean", "default": True}},
},
},
},
{
"type": "function",
"function": {
"name": "hand_action",
"description": "Apre o chiude una mano. action ∈ {LEFT_OPEN, LEFT_CLOSE, RIGHT_OPEN, RIGHT_CLOSE}. Default dry_run=true.",
"parameters": {
"type": "object",
"required": ["action"],
"properties": {
"action": {"type": "string", "enum": ["LEFT_OPEN", "LEFT_CLOSE", "RIGHT_OPEN", "RIGHT_CLOSE"]},
"dry_run": {"type": "boolean", "default": True},
},
},
},
},
{
"type": "function",
"function": {
"name": "get_marker",
"description": (
"Rileva il marker ArUco nell'RGB corrente di Nuwa e ne ritorna la "
"posa nel frame camera OpenCV (X-right, Y-down, Z-forward, Z>0=davanti). "
"Restituisce position_m, quaternion_xyzw, rotation_matrix, distance_m, "
"corners 2D e center_px. La PnP è eseguita da marker-finder con gli "
"intrinseci veri della Nuwa (pushati al boot di presa-mano)."
),
"parameters": {
"type": "object",
"properties": {
"marker_id": {"type": "integer", "description": "default = id del marker polso da config"},
"size_mm": {"type": "number", "description": "default = size del marker polso da config"},
},
},
},
},
{
"type": "function",
"function": {
"name": "get_marker_3d",
"description": (
"Posa marker calcolata dal sensore depth (no PnP). Detection "
"2D dei 4 corner via marker-finder, poi per ogni corner "
"back-projection corretta usando depth IR + extrinsics IR↔RGB. "
"Centro = media 3D dei corner; orientamento da base ortonormale "
"del piano marker. Restituisce anche `measured_side_mm` (utile "
"per verificare che size_mm dichiarata sia giusta) e una entry "
"per_corner con qualità della depth a ciascuno dei 4 corner."
),
"parameters": {
"type": "object",
"properties": {
"marker_id": {"type": "integer"},
"size_mm": {"type": "number"},
},
},
},
},
{
"type": "function",
"function": {
"name": "detect_object",
"description": "Rileva oggetti via YOLO sull'RGB di Nuwa. class_name opzionale per filtrare; restituisce massimo 5 detection.",
"parameters": {
"type": "object",
"properties": {
"class_name": {"type": "string"},
"confidence": {"type": "number"},
},
},
},
},
{
"type": "function",
"function": {
"name": "get_point_cloud_roi",
"description": "Statistiche della nuvola punti (frame camera) all'interno di un bbox in pixel. Restituisce conteggio + media/min/max/std, NON i punti raw.",
"parameters": {
"type": "object",
"required": ["bbox"],
"properties": {
"bbox": {"type": "array", "items": {"type": "number"}, "minItems": 4, "maxItems": 4,
"description": "[x1,y1,x2,y2] in pixel"},
},
},
},
},
{
"type": "function",
"function": {
"name": "calibration_add_sample",
"description": "Cattura un sample di calibrazione hand-eye per il braccio indicato (RGB + posa polso + posa marker). Richiede il marker visibile.",
"parameters": {
"type": "object",
"properties": {
"side": {"type": "string", "enum": ["right", "left"]},
"marker_id": {"type": "integer"},
"size_mm": {"type": "number"},
"note": {"type": "string"},
},
},
},
},
{
"type": "function",
"function": {
"name": "calibration_solve",
"description": "Risolve hand-eye calibration per il braccio dato e SALVA la matrice T_base_camera_<side>.yaml. Riporta residui.",
"parameters": {
"type": "object",
"properties": {
"side": {"type": "string", "enum": ["right", "left"]},
"method": {"type": "string", "enum": ["TSAI", "PARK", "HORAUD", "ANDREFF", "DANIILIDIS"]},
},
},
},
},
{
"type": "function",
"function": {
"name": "calibration_clear",
"description": "Cancella tutti i sample di calibrazione per il braccio indicato.",
"parameters": {
"type": "object",
"properties": {"side": {"type": "string", "enum": ["right", "left"]}},
},
},
},
{
"type": "function",
"function": {
"name": "calibration_get_result",
"description": "Riporta la calibrazione salvata per il braccio indicato (residui + saved_at).",
"parameters": {
"type": "object",
"properties": {"side": {"type": "string", "enum": ["right", "left"]}},
},
},
},
{
"type": "function",
"function": {
"name": "calibration_count_samples",
"description": "Numero di sample attualmente memorizzati per il braccio indicato.",
"parameters": {
"type": "object",
"properties": {"side": {"type": "string", "enum": ["right", "left"]}},
},
},
},
]
def dispatch_tool(name: str, args: dict[str, Any]) -> dict[str, Any]:
"""Run the named tool with the provided arguments. Always returns a dict.
The runner serialises this to JSON and feeds it back to the model as a
`tool` role message. Any exception is converted to {"ok": false, ...}.
"""
h = _HANDLERS.get(name)
if h is None:
return _err(f"tool sconosciuto: {name}")
t0 = time.perf_counter()
try:
out = h(args or {})
except Exception as e:
log.exception("tool %s raised", name)
out = _err(f"{type(e).__name__}: {e}")
out["_latency_ms"] = round((time.perf_counter() - t0) * 1000.0, 1)
return out
+2 -4
View File
@@ -1,9 +1,8 @@
# Load `_deps` first so it is registered as an attribute of this package # Load `_deps` first so it is registered as an attribute of this package
# before any of the route modules (which reach into the agent module which # before any of the route modules (which import `from ..api import _deps`)
# in turn imports `from ..api import _deps`) start executing. # start executing.
from . import _deps # noqa: F401 from . import _deps # noqa: F401
from . import ( from . import (
agent_routes,
calibration_routes, calibration_routes,
config_routes, config_routes,
grasp_routes, grasp_routes,
@@ -16,7 +15,6 @@ from . import (
) )
__all__ = [ __all__ = [
"agent_routes",
"calibration_routes", "calibration_routes",
"config_routes", "config_routes",
"grasp_routes", "grasp_routes",
-203
View File
@@ -1,203 +0,0 @@
"""AI assistant routes.
- `POST /api/agent/chat` — send a user message, get back the assistant reply
and the trace of every tool the model invoked. Stateless on the server side:
the GUI keeps the rolling history and posts it back in `history`.
- `GET /api/agent/tools` — list of tools exposed to the model (debug aid).
- `GET/POST/DELETE /api/agent/tasks` — persistent registry of named prompt
templates so frequently-used tasks ("calibra braccio dx", "sposta a home
position", …) can be re-launched in one click without retyping the prompt.
"""
from __future__ import annotations
import json
import re
import time
from dataclasses import asdict
from typing import Any, Optional
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, Field
# Submodule imports (not `from ..agent import …`) to break a circular import:
# api/__init__.py is loaded first by main.py and pulls in agent_routes; if we
# imported via the agent package, agent/__init__.py would try to load
# agent.runner → agent.tools → ..api._deps while ..api is still mid-init.
from ..agent.runner import AgentRunner, OpenRouterError
from ..agent.tools import TOOL_DEFINITIONS
from ..config import data_dir, get_config
router = APIRouter()
# ---------- chat ----------
class ChatMessage(BaseModel):
role: str
content: Optional[str] = None
# Pass-through fields used by the OpenAI tool-calling protocol.
tool_calls: Optional[list[dict[str, Any]]] = None
tool_call_id: Optional[str] = None
name: Optional[str] = None
class ChatReq(BaseModel):
message: str = Field(..., min_length=1, max_length=4000)
history: list[ChatMessage] = Field(default_factory=list)
@router.get("/tools")
def list_tools() -> dict[str, Any]:
"""Tools exposed to the LLM. Useful for debugging the system prompt."""
return {"tools": TOOL_DEFINITIONS}
@router.get("/health")
def agent_health() -> dict[str, Any]:
"""Reports whether OpenRouter credentials are available and which model
is configured. The frontend uses this to grey-out the chat input when
the env var is missing."""
runner = AgentRunner()
return {
"configured": runner.has_credentials(),
"model": runner.model,
"api_key_env": runner.api_key_env,
"base_url": runner.base_url,
"temperature": runner.temperature,
"max_tool_iterations": runner.max_tool_iterations,
"timeout_s": runner.timeout_s,
}
@router.get("/system_prompt_preview")
def system_prompt_preview() -> dict[str, Any]:
"""Returns the exact system prompt the model will receive at the next
chat. Useful for the AI Settings tab so the operator can preview the
effect of editing chat.openrouter.user_prompt_addendum or any other
state that influences the prompt (active robot, marker config…)."""
from ..agent.prompt import build_system_prompt
return {"system_prompt": build_system_prompt()}
# Curated list shown in the AI Settings model selector. Plus a free-text
# field on the GUI side for anything not in the list. Keep the list short
# and focused on models that we've verified support OpenRouter's tool-use.
_KNOWN_MODELS = [
{"id": "anthropic/claude-sonnet-4.5", "label": "Claude Sonnet 4.5 (consigliato)"},
{"id": "anthropic/claude-opus-4.1", "label": "Claude Opus 4.1 (più capace, più caro)"},
{"id": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5 (veloce, economico)"},
{"id": "openai/gpt-4o", "label": "GPT-4o"},
{"id": "openai/gpt-4o-mini", "label": "GPT-4o mini"},
{"id": "google/gemini-2.5-pro", "label": "Gemini 2.5 Pro"},
{"id": "google/gemini-2.5-flash", "label": "Gemini 2.5 Flash"},
{"id": "meta-llama/llama-3.3-70b-instruct", "label": "Llama 3.3 70B"},
{"id": "mistralai/mistral-large-2411", "label": "Mistral Large 2411"},
]
@router.get("/known_models")
def known_models() -> dict[str, Any]:
"""Curated short-list of OpenRouter models that work with our tool-calling.
The GUI also offers a free-text input for anything not in this list."""
return {"models": _KNOWN_MODELS}
@router.post("/chat")
def chat(req: ChatReq) -> dict[str, Any]:
runner = AgentRunner()
if not runner.has_credentials():
raise HTTPException(status_code=412,
detail=f"OpenRouter API key non impostata (env var {runner.api_key_env}).")
history = [m.model_dump(exclude_none=True) for m in req.history]
try:
result, updated_history = runner.run(req.message, history=history)
except OpenRouterError as e:
raise HTTPException(status_code=502, detail=str(e))
return {
"reply": result.reply,
"iterations": result.iterations,
"finish_reason": result.finish_reason,
"model": result.model,
"usage": result.usage,
"tool_calls": [asdict(t) for t in result.tool_calls],
"history": updated_history,
}
# ---------- saved task templates ----------
_NAME_RE = re.compile(r"^[a-zA-Z0-9 _\-àèéìòù]{1,80}$")
class TaskItem(BaseModel):
name: str = Field(..., min_length=1, max_length=80)
prompt: str = Field(..., min_length=1, max_length=4000)
description: Optional[str] = Field(None, max_length=400)
def _tasks_path():
return data_dir() / "agent_tasks.json"
def _load_tasks() -> list[dict[str, Any]]:
p = _tasks_path()
if not p.exists():
return []
try:
with open(p, "r", encoding="utf-8") as f:
data = json.load(f)
return data if isinstance(data, list) else []
except json.JSONDecodeError:
return []
def _save_tasks(items: list[dict[str, Any]]) -> None:
p = _tasks_path()
p.parent.mkdir(parents=True, exist_ok=True)
with open(p, "w", encoding="utf-8") as f:
json.dump(items, f, ensure_ascii=False, indent=2)
@router.get("/tasks")
def list_tasks() -> dict[str, Any]:
return {"tasks": _load_tasks()}
@router.post("/tasks")
def save_task(req: TaskItem) -> dict[str, Any]:
if not _NAME_RE.match(req.name):
raise HTTPException(status_code=422, detail="nome non valido (max 80 char, alfanumerici/spazi/_-)")
items = _load_tasks()
now = time.time()
# Replace by name if already exists, otherwise append.
found = False
for it in items:
if it.get("name") == req.name:
it["prompt"] = req.prompt
it["description"] = req.description
it["updated_at"] = now
found = True
break
if not found:
items.append({
"name": req.name,
"prompt": req.prompt,
"description": req.description,
"created_at": now,
"updated_at": now,
})
_save_tasks(items)
return {"ok": True, "total": len(items), "name": req.name}
@router.delete("/tasks/{name}")
def delete_task(name: str) -> dict[str, Any]:
items = _load_tasks()
new_items = [it for it in items if it.get("name") != name]
if len(new_items) == len(items):
raise HTTPException(status_code=404, detail=f"task '{name}' non trovata")
_save_tasks(new_items)
return {"ok": True, "total": len(new_items)}
+379 -10
View File
@@ -1,17 +1,23 @@
"""Endpoint per la tab "Pose 3D": editor visuale + jog dei polsi.""" """Endpoint per la tab "Pose 3D": editor visuale + jog dei polsi."""
from __future__ import annotations from __future__ import annotations
import json
import logging import logging
import os
import re import re
import time
from pathlib import Path
from typing import Any, Literal, Optional from typing import Any, Literal, Optional
import httpx import httpx
import numpy as np
from fastapi import APIRouter, HTTPException from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from ..clients import G1Unavailable from ..clients import G1Unavailable
from ..config import get_config, set_config_value from ..config import get_config, set_config_value, update_config
from ..geometry import normalize_quaternion from ..geometry import normalize_quaternion, umeyama_similarity
from ..perception.pointcloud import nuwa_to_cv_frame
from ..timing import Stopwatch from ..timing import Stopwatch
from . import _deps from . import _deps
@@ -57,15 +63,14 @@ class GoReq(BaseModel):
@router.get("/state") @router.get("/state")
def get_state() -> dict[str, Any]: def get_state() -> dict[str, Any]:
sw = Stopwatch() sw = Stopwatch()
wrist_payload: dict[str, Any] | None = None
joints_payload: dict[str, float] | None = None
robot_error: str | None = None
try: try:
with sw.step("g1_wrist"): with sw.step("g1_wrist"):
w = _deps.g1().wrist() g1 = _deps.g1()
except G1Unavailable as e: w = g1.wrist()
raise HTTPException(status_code=502, detail=f"G1 unavailable: {e}") wrist_payload = {
cfg = get_config().get("pose_3d", {}) or {}
return {
"robot_active": _deps.active_robot_key(),
"wrist": {
"left": { "left": {
"position": list(w["left_wrist"]["position"]), "position": list(w["left_wrist"]["position"]),
"quaternion": list(w["left_wrist"]["quaternion"]), "quaternion": list(w["left_wrist"]["quaternion"]),
@@ -74,9 +79,33 @@ def get_state() -> dict[str, Any]:
"position": list(w["right_wrist"]["position"]), "position": list(w["right_wrist"]["position"]),
"quaternion": list(w["right_wrist"]["quaternion"]), "quaternion": list(w["right_wrist"]["quaternion"]),
}, },
}, }
# Tutti i giunti reali (mappa name → q in rad) per specchiare il robot
# nell'URDF senza ricorrere all'IK.
try:
with sw.step("g1_diagnostics"):
d = g1.diagnostics()
motors = d.get("motors") or []
joints_payload = {m["name"]: float(m["q"]) for m in motors
if isinstance(m, dict) and "name" in m and "q" in m}
except Exception:
joints_payload = None
except G1Unavailable as e:
# Robot offline: la tab deve restare utilizzabile per editare config /
# pose salvate e visualizzare il modello (camera marker compreso).
robot_error = f"G1 unavailable: {e}"
cfg = get_config().get("pose_3d", {}) or {}
return {
"robot_active": _deps.active_robot_key(),
"wrist": wrist_payload,
"joints": joints_payload,
"robot_error": robot_error,
"poses": cfg.get("named_poses") or {}, "poses": cfg.get("named_poses") or {},
"live_max_step_m": float(cfg.get("live_max_step_m", 0.15)), "live_max_step_m": float(cfg.get("live_max_step_m", 0.15)),
"right_wrist_camera": cfg.get("right_wrist_camera") or {},
"clouds": cfg.get("clouds") or {},
"cloud_params": cfg.get("cloud_params") or {},
"cross_camera": cfg.get("cross_camera") or {},
"timings_ms": sw.as_dict(), "timings_ms": sw.as_dict(),
} }
@@ -174,6 +203,346 @@ def delete_pose(name: str) -> dict[str, Any]:
return {"deleted": name} return {"deleted": name}
@router.get("/clouds")
def get_clouds(step: int = 8, max_depth_mm: int = 2000) -> dict[str, Any]:
"""Acquisisce cloud da Nuwa-HP60C (eye-in-hand polso DX) e D435i (front
camera fissa sul torso) in parallelo. Ritorna entrambi in **OpenCV optical
frame** (X right, Y down, Z forward) come array `[N, 6]` con XYZ+RGB
(RGB in 0-255). Il client li attacca ai link URDF `right_wrist_yaw_link`
e `d435_link` rispettivamente, con offset/rpy configurabili.
"""
sw = Stopwatch()
result: dict[str, Any] = {"timings_ms": {}}
def _pack(pts: "np.ndarray") -> list[list[float]]:
# Accetta sia (N,3) che (N,6). In caso di (N,3) aggiunge grigio neutro.
import numpy as np
if pts.ndim != 2:
return []
if pts.shape[1] >= 6:
out = pts[:, :6]
else:
n = pts.shape[0]
rgb = np.full((n, 3), 180.0, dtype=pts.dtype)
out = np.hstack([pts[:, :3], rgb])
return out.astype(float).tolist()
# Nuwa: native frame (X right, Y up, Z back) → OpenCV optical via Y/Z flip
try:
with sw.step("nuwa_cloud"):
pts = _deps.nuwa().pointcloud(step=step, max_depth_mm=max_depth_mm)
pts_cv = nuwa_to_cv_frame(pts)
result["nuwa"] = {"n": int(pts_cv.shape[0]), "points": _pack(pts_cv)}
except Exception as e:
result["nuwa"] = {"error": f"{type(e).__name__}: {e}"}
# D435i: stesso client NuwaClient (interfaccia uniforme), il servizio
# ritorna però Nuwa-native (X right, Y up, Z back) come per la Nuwa.
# Quindi: stessa conversione → OpenCV optical (X right, Y down, Z forward).
try:
with sw.step("d435_cloud"):
pts = _deps.front_camera().pointcloud(step=step, max_depth_mm=max_depth_mm)
pts_cv = nuwa_to_cv_frame(pts)
result["d435"] = {"n": int(pts_cv.shape[0]), "points": _pack(pts_cv)}
except Exception as e:
result["d435"] = {"error": f"{type(e).__name__}: {e}"}
result["timings_ms"] = sw.as_dict()
return result
# --- Multi-camera tag finder: detect + misura indipendente via depth ----
# Scansiona ArUco DICT_4X4_50 su tutte le camere attive (Nuwa polso DX e
# D435i frontale). Per ogni tag trovato misura i 4 corner 3D via depth sensor
# (indipendente da solvePnP). Output in frame ottico OpenCV per ciascuna camera.
def _edge_len_mm(a: list[float], b: list[float]) -> "Optional[float]":
"""Lunghezza 3D in mm tra due corner; None se un corner ha NaN."""
if any(x != x for x in a + b): # NaN check
return None
return (sum((a[i] - b[i]) ** 2 for i in range(3))) ** 0.5 * 1000.0
def _parse_id_range(spec: str) -> list[int]:
"""Accetta "0-49" o "1,10,20,33"."""
spec = (spec or "").strip()
if not spec:
return list(range(50))
out: list[int] = []
for part in spec.split(","):
part = part.strip()
if "-" in part:
a, b = part.split("-", 1)
out.extend(range(int(a), int(b) + 1))
else:
out.append(int(part))
return sorted(set(out))
def _detect_tags_on(client, frame_label: str, ids: str,
probe_size_mm: float = 40.0) -> list[dict]:
"""Detect ArUco markers su `client` (qualsiasi NuwaClient) e misura i 4
corner + centro in 3D via depth sensor (indipendente da solvePnP).
Ritorna una lista di dict per tag (id, corners_pixel, corners_3d_optical_m,
corner_depths_mm, center_3d_optical_m, edge_mm_per_side, edge_mm, distance_m).
`frame_label` è solo per i log; i punti sono nel frame ottico OpenCV della camera."""
mk = _deps.marker()
intr = client.intrinsics()
rgb = (intr or {}).get("rgb") or {}
if not rgb or "fx" not in rgb:
raise RuntimeError(f"intrinsics rgb mancanti: {intr!r}")
K = {
"camera_matrix": [
[float(rgb["fx"]), 0.0, float(rgb["cx"])],
[0.0, float(rgb["fy"]), float(rgb["cy"])],
[0.0, 0.0, 1.0],
],
"dist_coeffs": [
float((rgb.get("distortion") or {}).get(k, 0.0))
for k in ("k1", "k2", "p1", "p2", "k3")
],
}
id_list = _parse_id_range(ids)
markers = [{"id": int(i), "size_mm": float(probe_size_mm)} for i in id_list]
det = mk.detect_url(markers=markers, image_url=client.rgb_url(), intrinsics=K)
tags_out: list[dict[str, Any]] = []
for m in det.get("found", []):
corners = m.get("corners") or []
if len(corners) != 4:
continue
pts3d: list[list[float]] = []
depths: list[Optional[int]] = []
try:
for (u, v) in corners:
md = client.measure(int(round(u)), int(round(v)), radius=2)
pt = (md or {}).get("point_m") or {}
d_mm = (md or {}).get("depth_mm")
if not pt or d_mm in (None, 0):
pts3d.append([float("nan")] * 3)
depths.append(None)
else:
pts3d.append([float(pt["x"]), float(pt["y"]), float(pt["z"])])
depths.append(int(d_mm))
cu = sum(c[0] for c in corners) / 4.0
cv = sum(c[1] for c in corners) / 4.0
cmd = client.measure(int(round(cu)), int(round(cv)), radius=4)
cpt = (cmd or {}).get("point_m") or {}
center3d = ([float(cpt["x"]), float(cpt["y"]), float(cpt["z"])]
if cpt else [float("nan")] * 3)
except Exception as e:
log.warning("tags(%s): measure failed for id=%s: %s",
frame_label, m.get("id"), e)
continue
valid_d = [d for d in depths if d is not None]
if len(valid_d) >= 3:
sd = sorted(valid_d)
median_d = (sd[len(sd)//2 - 1] + sd[len(sd)//2]) / 2 if len(sd) % 2 == 0 else sd[len(sd)//2]
for i, d in enumerate(depths):
if d is None:
continue
if abs(d - median_d) / median_d > 0.20:
pts3d[i] = [float("nan")] * 3
edges_mm = [_edge_len_mm(pts3d[i], pts3d[(i + 1) % 4]) for i in range(4)]
valid_edges = [e for e in edges_mm if e is not None]
edge_mean = sum(valid_edges) / len(valid_edges) if valid_edges else None
distance_m: Optional[float] = None
if all(x == x for x in center3d):
distance_m = (sum(c ** 2 for c in center3d)) ** 0.5
tags_out.append({
"id": int(m["id"]),
"corners_pixel": [[float(c[0]), float(c[1])] for c in corners],
"corners_3d_optical_m": pts3d,
"corner_depths_mm": depths,
"center_3d_optical_m": center3d,
"edge_mm_per_side": edges_mm,
"edge_mm": edge_mean,
"distance_m": distance_m,
})
return tags_out
# Map: camera key → (frame_label, _deps attribute name).
# Usiamo il nome stringa dell'attributo su _deps e lo risolviamo via getattr
# a ogni richiesta: cosi' patch("...pose_3d_routes._deps.nuwa") viene visto.
_TAG_CAMERAS: dict[str, tuple[str, str]] = {
"nuwa": ("nuwa_rgb_optical", "nuwa"),
"d435": ("d435_rgb_optical", "front_camera"),
}
@router.get("/tags")
def get_tags(ids: str = "0-49", probe_size_mm: float = 40.0) -> dict[str, Any]:
"""Detect ArUco su tutte le camere attive (Nuwa polso + D435i frontale).
Una camera offline riporta tags=[] + error, non fa fallire l'altra."""
sw = Stopwatch()
cameras: dict[str, Any] = {}
for key, (frame_label, dep_name) in _TAG_CAMERAS.items():
try:
factory = getattr(_deps, dep_name)
with sw.step(key):
tags = _detect_tags_on(factory(), frame_label, ids, probe_size_mm)
cameras[key] = {"tags": tags, "frame": frame_label}
except Exception as e:
cameras[key] = {"tags": [], "frame": frame_label,
"error": f"{type(e).__name__}: {e}"}
return {"cameras": cameras, "timings_ms": sw.as_dict()}
@router.post("/align")
def post_align(ids: str = "0-49", align_id: int = 0,
probe_size_mm: float = 40.0) -> dict[str, Any]:
"""Calibrazione cross-camera: trova `align_id` su Nuwa e D435i, fitta una
similarita' (rot+trasl+scala) Nuwa-optical -> D435-optical e la PERSISTE in
config.yaml > pose_3d.cross_camera. Se il tag non e' visto da entrambe, NON
sovrascrive la matrice salvata."""
sw = Stopwatch()
found: dict[str, Optional[dict]] = {}
cam_errors: dict[str, str] = {}
for key, (frame_label, dep_name) in _TAG_CAMERAS.items():
try:
factory = getattr(_deps, dep_name)
with sw.step(key):
tags = _detect_tags_on(factory(), frame_label, ids, probe_size_mm)
found[key] = next((t for t in tags if int(t["id"]) == int(align_id)), None)
except Exception as e:
log.warning("align: camera %s failed: %s", key, e)
found[key] = None
cam_errors[key] = f"{type(e).__name__}: {e}"
missing = [k for k, v in found.items() if v is None]
if missing:
if cam_errors:
detail = "; ".join(f"{k}: {msg}" for k, msg in cam_errors.items())
err = f"camera non disponibile -> {detail}"
else:
err = f"tag {align_id} non visto da: {', '.join(missing)}"
return {"found_in_both": False, "align_id": int(align_id),
"error": err, "timings_ms": sw.as_dict()}
# I corner sono accoppiati posizionalmente: stesso marker-finder + stessa
# geometria ArUco -> ordine corner deterministico e identico tra le camere.
src, dst = [], []
for cn, cd in zip(found["nuwa"]["corners_3d_optical_m"],
found["d435"]["corners_3d_optical_m"]):
if any(x != x for x in cn) or any(x != x for x in cd):
continue
src.append(cn)
dst.append(cd)
if len(src) < 3:
return {"found_in_both": False, "align_id": int(align_id),
"error": f"corner validi insufficienti ({len(src)}/4)",
"timings_ms": sw.as_dict()}
try:
with sw.step("umeyama"):
scale, R, t, rmse = umeyama_similarity(np.array(src), np.array(dst))
except ValueError as e:
return {"found_in_both": False, "align_id": int(align_id),
"error": f"fit fallito: {e}", "timings_ms": sw.as_dict()}
payload = {
"align_id": int(align_id),
"scale": float(scale),
"R": [[float(x) for x in row] for row in R],
"t": [float(x) for x in t],
"rmse_mm": float(rmse * 1000.0),
"n_corners": len(src),
}
update_config({"pose_3d": {"cross_camera": payload}})
return {"found_in_both": True, **payload, "timings_ms": sw.as_dict()}
# --- Tag locator: piazza crocetta nella scena 3D e salva la posa in pelvis ---
# Workflow utente (vedi web/pose_3d.js sezione "tag locator"):
# 1) seleziona tag da dropdown (id+size+label)
# 2) click nel viewport → crocetta a quel punto
# 3) "Salva" → POST qui → scrive current.json + ruota archive
# TODO(orientation): per ora salviamo solo CdM; orientamento non catturato.
_TAG_OFFSET_DEFAULT_DIR = "/home/harry/robot/logging/wrist-tag-offset"
def _tag_offset_dir() -> Path:
"""Letto da config.yaml pose_3d.tag_offset.save_dir; ricaricato ad ogni call
cosi' un PUT /api/config a caldo prende effetto subito."""
cfg = (get_config().get("pose_3d", {}) or {}).get("tag_offset", {}) or {}
return Path(cfg.get("save_dir") or _TAG_OFFSET_DEFAULT_DIR)
class TagOffsetSaveReq(BaseModel):
tag_id: int
tag_size_mm: float
tag_label: Optional[str] = None
position_pelvis_m: list[float] = Field(min_length=3, max_length=3)
# Snapshot dei giunti reali al click (name → radianti). Null se G1 offline.
joints_at_save_rad: Optional[dict[str, float]] = None
note: Optional[str] = None
@router.post("/tag_offset/save")
def post_tag_offset_save(req: TagOffsetSaveReq) -> dict[str, Any]:
sw = Stopwatch()
ts = time.strftime("%Y%m%d_%H%M%S")
payload = {
"tag_id": int(req.tag_id),
"tag_size_mm": float(req.tag_size_mm),
"tag_label": req.tag_label,
"position_pelvis_m": [float(v) for v in req.position_pelvis_m],
"joints_at_save_rad": req.joints_at_save_rad,
"frame": "pelvis",
"method": "manual_3d_click",
"timestamp": ts,
"note": req.note,
}
base = _tag_offset_dir()
archive_dir = base / "archive"
current_path = base / "current.json"
archived_from: Optional[str] = None
try:
with sw.step("mkdir"):
archive_dir.mkdir(parents=True, exist_ok=True)
with sw.step("archive"):
if current_path.exists():
old = json.loads(current_path.read_text())
old_ts = old.get("timestamp") or ts
dest = archive_dir / f"{old_ts}.json"
# se collide, suffisso numerico
i = 1
while dest.exists():
dest = archive_dir / f"{old_ts}_{i}.json"
i += 1
current_path.rename(dest)
archived_from = str(dest)
with sw.step("write"):
current_path.write_text(json.dumps(payload, indent=2))
except OSError as e:
raise HTTPException(status_code=500, detail=f"tag_offset save fail: {e}")
return {
"ok": True,
"path": str(current_path),
"archived_from": archived_from,
"saved": payload,
"timings_ms": sw.as_dict(),
}
@router.get("/tag_offset")
def get_tag_offset() -> dict[str, Any]:
"""Restituisce il current.json se esiste, altrimenti 404."""
current_path = _tag_offset_dir() / "current.json"
if not current_path.exists():
raise HTTPException(status_code=404, detail="no tag offset saved yet")
try:
return json.loads(current_path.read_text())
except (OSError, json.JSONDecodeError) as e:
raise HTTPException(status_code=500, detail=f"read fail: {e}")
@router.post("/unlock") @router.post("/unlock")
def post_unlock() -> dict[str, Any]: def post_unlock() -> dict[str, Any]:
"""Proxy a POST {bridge}/unlock. Recovery rapido da arms: error_timeout.""" """Proxy a POST {bridge}/unlock. Recovery rapido da arms: error_timeout."""
+7
View File
@@ -58,6 +58,13 @@ class G1Client:
"""Hand finger state. Shape: {'left': {'fingers': {pinky,ring,middle,index,thumb_bend,thumb_rotation}, 'age_s'}, 'right': {...}}.""" """Hand finger state. Shape: {'left': {'fingers': {pinky,ring,middle,index,thumb_bend,thumb_rotation}, 'age_s'}, 'right': {...}}."""
return self._get("/hand") return self._get("/hand")
def diagnostics(self) -> dict[str, Any]:
"""Full motor state: {ok, demo, motors: [{name, q, dq, ...}, ...]}.
`q` is the joint position in radians. Use for live URDF mirroring.
"""
return self._get("/diagnostics")
# ----- mutating ----- # ----- mutating -----
def wrist_absolute(self, side: str, position: list[float], quaternion: list[float], def wrist_absolute(self, side: str, position: list[float], quaternion: list[float],
+38
View File
@@ -142,3 +142,41 @@ def quaternion_xyzw_to_rpy_deg(quat_xyzw) -> tuple[float, float, float]:
rpy_rad = R_to_euler_xyz(R) rpy_rad = R_to_euler_xyz(R)
rpy_deg = np.degrees(rpy_rad) rpy_deg = np.degrees(rpy_rad)
return float(rpy_deg[0]), float(rpy_deg[1]), float(rpy_deg[2]) return float(rpy_deg[0]), float(rpy_deg[1]), float(rpy_deg[2])
def umeyama_similarity(src: np.ndarray, dst: np.ndarray) -> tuple[float, np.ndarray, np.ndarray, float]:
"""Fit dst ≈ s·R·src + t (Umeyama 1991, con scala uniforme).
src, dst: (N,3) array di punti corrispondenti.
Ritorna (scale: float, R: (3,3), t: (3,), rmse: float in unità di dst).
Solleva ValueError se N<3 o il set è degenere (varianza nulla).
"""
src = np.asarray(src, dtype=np.float64).reshape(-1, 3)
dst = np.asarray(dst, dtype=np.float64).reshape(-1, 3)
if src.shape != dst.shape:
raise ValueError(f"shape mismatch: {src.shape} vs {dst.shape}")
n = src.shape[0]
if n < 3:
raise ValueError(f"servono almeno 3 punti, ricevuti {n}")
mu_src = src.mean(axis=0)
mu_dst = dst.mean(axis=0)
src_c = src - mu_src
dst_c = dst - mu_dst
var_src = (src_c ** 2).sum() / n
if var_src < 1e-12:
raise ValueError("punti src degeneri (varianza nulla)")
cov = (dst_c.T @ src_c) / n
U, D, Vt = np.linalg.svd(cov)
S = np.eye(3)
if np.linalg.det(U) * np.linalg.det(Vt) < 0:
S[2, 2] = -1.0 # vieta riflessioni
R = U @ S @ Vt
scale = float((D * np.diag(S)).sum() / var_src)
t = mu_dst - scale * (R @ mu_src)
pred = (scale * (R @ src.T)).T + t
rmse = float(np.sqrt(((pred - dst) ** 2).sum(axis=1).mean()))
return scale, R, t, rmse
+8 -3
View File
@@ -10,7 +10,6 @@ from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
from .api import ( from .api import (
agent_routes,
calibration_routes, calibration_routes,
config_routes, config_routes,
grasp_routes, grasp_routes,
@@ -26,6 +25,7 @@ from .api import (
) )
from .clients import G1StatusError from .clients import G1StatusError
from .config import load_config from .config import load_config
from .version import build_info
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
@@ -39,7 +39,6 @@ app = FastAPI(
app.include_router(config_routes.router, prefix="/api/config", tags=["config"]) app.include_router(config_routes.router, prefix="/api/config", tags=["config"])
app.include_router(home_routes.router, prefix="/api/home", tags=["home"]) app.include_router(home_routes.router, prefix="/api/home", tags=["home"])
app.include_router(agent_routes.router, prefix="/api/agent", tags=["agent"])
app.include_router(calibration_routes.router, prefix="/api/calibration", tags=["calibration"]) app.include_router(calibration_routes.router, prefix="/api/calibration", tags=["calibration"])
app.include_router(perception_routes.router, prefix="/api/perception", tags=["perception"]) app.include_router(perception_routes.router, prefix="/api/perception", tags=["perception"])
app.include_router(grasp_routes.router, prefix="/api/grasp", tags=["grasp"]) app.include_router(grasp_routes.router, prefix="/api/grasp", tags=["grasp"])
@@ -104,12 +103,18 @@ def health() -> dict:
return {"status": "ok"} return {"status": "ok"}
@app.get("/api/version", tags=["meta"])
def version() -> dict:
return build_info()
@app.get("/", include_in_schema=False) @app.get("/", include_in_schema=False)
def index() -> HTMLResponse: def index() -> HTMLResponse:
idx = WEB_DIR / "index.html" idx = WEB_DIR / "index.html"
html = idx.read_text(encoding="utf-8") html = idx.read_text(encoding="utf-8")
# Cache-bust app.js and style.css with their mtime so a rebuild forces refetch. # Cache-bust app.js and style.css with their mtime so a rebuild forces refetch.
for name in ("app.js", "work_plane.js", "right_arm.js", "style.css"): for name in ("app.js", "work_plane.js", "right_arm.js", "pose_3d.js",
"style.css", "pose_3d.css"):
f = WEB_DIR / name f = WEB_DIR / name
if f.exists(): if f.exists():
v = int(f.stat().st_mtime) v = int(f.stat().st_mtime)
+35
View File
@@ -0,0 +1,35 @@
"""Versione build dell'app, mostrata nel badge in GUI.
In Docker i valori arrivano da env (`APP_VERSION`, `BUILD_TIME`) iniettati come
build-arg da `build.sh`. In dev locale (`uv run`) gli env non ci sono: si
ripiega sul contenuto del file `VERSION` nella root del repo + suffisso
`(local)`.
"""
from __future__ import annotations
import os
from pathlib import Path
_VERSION_FILE = Path(__file__).resolve().parents[2] / "VERSION"
def _file_version() -> str:
try:
return _VERSION_FILE.read_text(encoding="utf-8").strip() or "0.0.0"
except OSError:
return "0.0.0"
def build_info() -> dict:
"""Ritorna {'version', 'build_time'}.
Preferisce gli env settati al build dell'immagine; in loro assenza ripiega
sul file VERSION marcando la versione come `(local)`.
"""
version = os.environ.get("APP_VERSION")
build_time = os.environ.get("BUILD_TIME")
if not version:
version = f"{_file_version()} (local)"
if not build_time:
build_time = "(local)"
return {"version": version, "build_time": build_time}
+49
View File
@@ -0,0 +1,49 @@
"""Unit test per umeyama_similarity (absolute orientation con scala)."""
from __future__ import annotations
import numpy as np
import pytest
from presa_mano.geometry import umeyama_similarity, rpy_deg_to_quaternion_xyzw, quat_xyzw_to_R
def _known_transform():
# scala 1.5, rotazione nota, traslazione nota
R = quat_xyzw_to_R(rpy_deg_to_quaternion_xyzw(10.0, -20.0, 35.0))
s = 1.5
t = np.array([0.3, -0.1, 0.2])
return s, R, t
def test_recovers_known_similarity():
s, R, t = _known_transform()
rng = np.random.default_rng(0)
src = rng.normal(size=(8, 3))
dst = (s * (R @ src.T)).T + t
s_hat, R_hat, t_hat, rmse = umeyama_similarity(src, dst)
assert abs(s_hat - s) < 1e-6
assert np.allclose(R_hat, R, atol=1e-6)
assert np.allclose(t_hat, t, atol=1e-6)
assert rmse < 1e-6
def test_forbids_reflection():
rng = np.random.default_rng(1)
src = rng.normal(size=(5, 3))
dst = rng.normal(size=(5, 3))
_, R_hat, _, _ = umeyama_similarity(src, dst)
assert np.linalg.det(R_hat) > 0
def test_raises_on_too_few_points():
with pytest.raises(ValueError):
umeyama_similarity(np.zeros((2, 3)), np.zeros((2, 3)))
def test_raises_on_degenerate_points():
src = np.ones((4, 3))
dst = np.ones((4, 3)) * 2.0
with pytest.raises(ValueError):
umeyama_similarity(src, dst)
+85
View File
@@ -0,0 +1,85 @@
"""Test POST /api/pose_3d/align: fit Umeyama + persistenza config."""
from __future__ import annotations
from unittest.mock import patch, MagicMock
import numpy as np
from fastapi.testclient import TestClient
from presa_mano.main import app
from presa_mano.config import get_config
client = TestClient(app)
_CORNERS_PX = [[100, 100], [140, 100], [140, 140], [100, 140]]
def _cam_with_points(points_by_pixel):
cam = MagicMock()
cam.intrinsics.return_value = {"rgb": {"fx": 600.0, "fy": 600.0, "cx": 320.0, "cy": 240.0}}
cam.rgb_url.return_value = "http://cam/rgb.png"
def _measure(u, v, radius=2):
key = (int(u), int(v))
if key in points_by_pixel:
x, y, z = points_by_pixel[key]
else:
xs = [p[0] for p in points_by_pixel.values()]
ys = [p[1] for p in points_by_pixel.values()]
zs = [p[2] for p in points_by_pixel.values()]
x, y, z = sum(xs)/len(xs), sum(ys)/len(ys), sum(zs)/len(zs)
return {"point_m": {"x": x, "y": y, "z": z}, "depth_mm": int(z * 1000)}
cam.measure.side_effect = _measure
return cam
def _marker(found_ids):
mk = MagicMock()
mk.detect_url.return_value = {
"found": [{"id": i, "corners": _CORNERS_PX} for i in found_ids],
"not_found": [],
}
return mk
def test_align_fits_and_persists():
nuwa_pts = {
(100, 100): (0.10, 0.10, 0.30),
(140, 100): (0.14, 0.10, 0.30),
(140, 140): (0.14, 0.14, 0.30),
(100, 140): (0.10, 0.14, 0.30),
}
d435_pts = {k: (v[0]*2, v[1]*2, v[2]*2) for k, v in nuwa_pts.items()}
with patch("presa_mano.api.pose_3d_routes._deps.nuwa") as nuwa_f, \
patch("presa_mano.api.pose_3d_routes._deps.front_camera") as front_f, \
patch("presa_mano.api.pose_3d_routes._deps.marker") as mk_f:
nuwa_f.return_value = _cam_with_points(nuwa_pts)
front_f.return_value = _cam_with_points(d435_pts)
mk_f.return_value = _marker([7])
r = client.post("/api/pose_3d/align?ids=0-9&align_id=7")
assert r.status_code == 200
body = r.json()
assert body["found_in_both"] is True
assert abs(body["scale"] - 2.0) < 1e-3
assert body["align_id"] == 7
cc = get_config()["pose_3d"]["cross_camera"]
assert abs(cc["scale"] - 2.0) < 1e-3
assert cc["align_id"] == 7
assert len(cc["R"]) == 3 and len(cc["t"]) == 3
def test_align_tag_missing_in_one_camera_does_not_save():
pts = {(100,100):(0.1,0.1,0.3),(140,100):(0.14,0.1,0.3),
(140,140):(0.14,0.14,0.3),(100,140):(0.1,0.14,0.3)}
with patch("presa_mano.api.pose_3d_routes._deps.nuwa") as nuwa_f, \
patch("presa_mano.api.pose_3d_routes._deps.front_camera") as front_f, \
patch("presa_mano.api.pose_3d_routes._deps.marker") as mk_f:
nuwa_f.return_value = _cam_with_points(pts)
front_f.return_value = _cam_with_points(pts)
mk_f.return_value = _marker([1]) # nessun tag id=99
r = client.post("/api/pose_3d/align?ids=0-9&align_id=99")
assert r.status_code == 200
body = r.json()
assert body["found_in_both"] is False
assert body.get("error")
assert "cross_camera" not in get_config().get("pose_3d", {})
+21 -2
View File
@@ -30,17 +30,26 @@ def test_state_returns_both_wrists():
assert body["wrist"]["right"]["position"] == [0.30, -0.20, 0.30] assert body["wrist"]["right"]["position"] == [0.30, -0.20, 0.30]
assert "poses" in body assert "poses" in body
assert "live_max_step_m" in body assert "live_max_step_m" in body
assert "right_wrist_camera" in body
assert body["robot_active"] in ("g1", "h1") assert body["robot_active"] in ("g1", "h1")
def test_state_502_when_g1_unavailable(): def test_state_returns_200_with_wrist_null_when_g1_unavailable():
# La tab deve restare utilizzabile per editare config / pose salvate
# e visualizzare il modello anche senza robot vivo.
from presa_mano.clients import G1Unavailable from presa_mano.clients import G1Unavailable
with patch("presa_mano.api.pose_3d_routes._deps.g1") as g1_factory: with patch("presa_mano.api.pose_3d_routes._deps.g1") as g1_factory:
g1 = MagicMock() g1 = MagicMock()
g1.wrist.side_effect = G1Unavailable("connection refused") g1.wrist.side_effect = G1Unavailable("connection refused")
g1_factory.return_value = g1 g1_factory.return_value = g1
r = client.get("/api/pose_3d/state") r = client.get("/api/pose_3d/state")
assert r.status_code == 502 assert r.status_code == 200
body = r.json()
assert body["wrist"] is None
assert "G1 unavailable" in (body.get("robot_error") or "")
# config esposta comunque
assert "right_wrist_camera" in body
assert "poses" in body
def test_apply_dry_run_calls_g1_with_dry_run_true(): def test_apply_dry_run_calls_g1_with_dry_run_true():
@@ -199,3 +208,13 @@ def test_unlock_proxies_to_bridge():
r = client.post("/api/pose_3d/unlock") r = client.post("/api/pose_3d/unlock")
assert r.status_code == 200 assert r.status_code == 200
assert r.json()["ok"] is True assert r.json()["ok"] is True
def test_state_includes_cross_camera_block():
with patch("presa_mano.api.pose_3d_routes._deps.g1") as g1_factory:
g1 = MagicMock()
g1.wrist.return_value = _fake_wrist_pair()
g1_factory.return_value = g1
r = client.get("/api/pose_3d/state")
assert r.status_code == 200
assert "cross_camera" in r.json() # {} se mai calibrato
+75
View File
@@ -0,0 +1,75 @@
"""Test del nuovo endpoint multi-camera GET /api/pose_3d/tags."""
from __future__ import annotations
from unittest.mock import patch, MagicMock
from fastapi.testclient import TestClient
from presa_mano.main import app
client = TestClient(app)
def _fake_camera(corner_depth_m=0.32):
cam = MagicMock()
cam.intrinsics.return_value = {
"rgb": {"fx": 600.0, "fy": 600.0, "cx": 320.0, "cy": 240.0}
}
cam.rgb_url.return_value = "http://cam/rgb.png"
def _measure(u, v, radius=2):
return {"point_m": {"x": u / 1000.0, "y": v / 1000.0, "z": corner_depth_m},
"depth_mm": int(corner_depth_m * 1000)}
cam.measure.side_effect = _measure
return cam
def _fake_marker(found_ids):
mk = MagicMock()
found = [{
"id": i,
"corners": [[100, 100], [140, 100], [140, 140], [100, 140]],
} for i in found_ids]
mk.detect_url.return_value = {"found": found, "not_found": []}
return mk
def test_tags_returns_both_cameras():
with patch("presa_mano.api.pose_3d_routes._deps.nuwa") as nuwa_f, \
patch("presa_mano.api.pose_3d_routes._deps.front_camera") as front_f, \
patch("presa_mano.api.pose_3d_routes._deps.marker") as mk_f:
nuwa_f.return_value = _fake_camera()
front_f.return_value = _fake_camera()
mk_f.return_value = _fake_marker([5])
r = client.get("/api/pose_3d/tags?ids=0-9")
assert r.status_code == 200
body = r.json()
assert set(body["cameras"].keys()) == {"nuwa", "d435"}
assert body["cameras"]["nuwa"]["frame"] == "nuwa_rgb_optical"
assert body["cameras"]["d435"]["frame"] == "d435_rgb_optical"
assert body["cameras"]["nuwa"]["tags"][0]["id"] == 5
assert body["cameras"]["d435"]["tags"][0]["id"] == 5
tag = body["cameras"]["nuwa"]["tags"][0]
assert tag["edge_mm"] is not None
assert 35.0 < tag["edge_mm"] < 45.0 # ~40mm lato del quad
assert tag["distance_m"] is not None
assert abs(tag["distance_m"] - 0.362) < 0.02 # sqrt(.12^2+.12^2+.32^2)
assert all(len(c) == 3 for c in tag["corners_3d_optical_m"])
def test_tags_one_camera_offline_reports_error():
with patch("presa_mano.api.pose_3d_routes._deps.nuwa") as nuwa_f, \
patch("presa_mano.api.pose_3d_routes._deps.front_camera") as front_f, \
patch("presa_mano.api.pose_3d_routes._deps.marker") as mk_f:
nuwa = MagicMock()
nuwa.intrinsics.side_effect = RuntimeError("nuwa down")
nuwa_f.return_value = nuwa
front_f.return_value = _fake_camera()
mk_f.return_value = _fake_marker([5])
r = client.get("/api/pose_3d/tags?ids=0-9")
assert r.status_code == 200
body = r.json()
assert body["cameras"]["nuwa"]["tags"] == []
assert "error" in body["cameras"]["nuwa"]
assert body["cameras"]["d435"]["tags"][0]["id"] == 5
+25
View File
@@ -0,0 +1,25 @@
"""Unit test per build_info(): env (Docker) vs fallback VERSION file (local)."""
from __future__ import annotations
from pathlib import Path
from presa_mano.version import build_info
_VERSION_FILE = Path(__file__).resolve().parents[1] / "VERSION"
def test_build_info_prefers_env(monkeypatch):
monkeypatch.setenv("APP_VERSION", "1.2.3")
monkeypatch.setenv("BUILD_TIME", "2026-06-03 14:22 UTC")
info = build_info()
assert info == {"version": "1.2.3", "build_time": "2026-06-03 14:22 UTC"}
def test_build_info_falls_back_to_version_file(monkeypatch):
monkeypatch.delenv("APP_VERSION", raising=False)
monkeypatch.delenv("BUILD_TIME", raising=False)
info = build_info()
file_ver = _VERSION_FILE.read_text(encoding="utf-8").strip()
assert file_ver in info["version"]
assert info["version"].endswith("(local)")
assert info["build_time"] == "(local)"
+10 -732
View File
@@ -642,392 +642,6 @@ $('unlockBtn').addEventListener('click', async () => {
} catch (e) { $('eeLog').textContent = e.message; } } catch (e) { $('eeLog').textContent = e.message; }
}); });
// ==================== VISTA 3D ====================
// Minimal canvas-2D 3D renderer: orthographic camera with yaw/pitch around a
// target point, painter's-algorithm depth sorting, mouse drag/zoom/pan.
const view3dCanvas = $('view3dCanvas');
const view3dCtx = view3dCanvas.getContext('2d');
const view3d = {
// camera
yaw: -0.6, // around base Z
pitch: 0.5, // 0 = horizontal, π/2 = top-down
scale: 350, // pixels per meter
target: [0.30, 0.0, 0.20], // world point at canvas center
// scene data
wrist: null, // { left:{position,quaternion}, right:{...} }
object: null, // { points:[[x,y,z],...], centroid:[x,y,z], n_points }
plan: null, // { grasp:{position,quaternion_xyzw}, pre_grasp:{position} }
calibrated: true, // false → object data unavailable
lastErr: null,
};
// ---- math helpers ----
function quatToR(q) {
const x=q[0], y=q[1], z=q[2], w=q[3];
const n = x*x + y*y + z*z + w*w;
if (n === 0) return [[1,0,0],[0,1,0],[0,0,1]];
const s = 2/n;
return [
[1 - s*(y*y + z*z), s*(x*y - z*w), s*(x*z + y*w)],
[s*(x*y + z*w), 1 - s*(x*x + z*z), s*(y*z - x*w)],
[s*(x*z - y*w), s*(y*z + x*w), 1 - s*(x*x + y*y)],
];
}
// Project world point → {sx, sy, depth}. depth = camera-axis distance for sorting.
function project(p) {
const dx = p[0] - view3d.target[0];
const dy = p[1] - view3d.target[1];
const dz = p[2] - view3d.target[2];
const cy = Math.cos(view3d.yaw), sy = Math.sin(view3d.yaw);
// Yaw around Z
const x1 = cy*dx + sy*dy;
const y1 = -sy*dx + cy*dy;
const z1 = dz;
// Pitch around X (camera looks roughly toward +Y after yaw, tilting down by pitch)
const cp = Math.cos(view3d.pitch), sp = Math.sin(view3d.pitch);
const y2 = cp*y1 + sp*z1;
const z2 = -sp*y1 + cp*z1;
// Orthographic: screen X = +x1; screen Y = -z2 (so +Z up on screen)
const sx = view3dCanvas.width / 2 + view3d.scale * x1;
const sxy = view3dCanvas.height / 2 - view3d.scale * z2;
// depth: y2 increases away from viewer → larger y2 = farther
return { sx, sy: sxy, depth: y2 };
}
function drawSegment(items, p1, p2, color, width = 2) {
const a = project(p1), b = project(p2);
items.push({ depth: (a.depth + b.depth) / 2, draw: () => {
view3dCtx.strokeStyle = color;
view3dCtx.lineWidth = width;
view3dCtx.beginPath();
view3dCtx.moveTo(a.sx, a.sy);
view3dCtx.lineTo(b.sx, b.sy);
view3dCtx.stroke();
}});
}
function drawDot(items, p, color, radius = 3) {
const a = project(p);
items.push({ depth: a.depth, draw: () => {
view3dCtx.fillStyle = color;
view3dCtx.beginPath();
view3dCtx.arc(a.sx, a.sy, radius, 0, 2 * Math.PI);
view3dCtx.fill();
}});
}
function drawLabel(items, p, text, color, dx = 8, dy = -8) {
const a = project(p);
items.push({ depth: a.depth - 1e-6, draw: () => {
view3dCtx.fillStyle = color;
view3dCtx.font = 'bold 11px ui-monospace, monospace';
view3dCtx.fillText(text, a.sx + dx, a.sy + dy);
}});
}
function drawTriad(items, position, R, len = 0.06) {
// Columns of R = axis directions in base frame.
const axes = [
{ col: 0, color: '#e74c3c' }, // X
{ col: 1, color: '#3ecf8e' }, // Y
{ col: 2, color: '#4da3ff' }, // Z
];
for (const a of axes) {
const tip = [
position[0] + R[0][a.col] * len,
position[1] + R[1][a.col] * len,
position[2] + R[2][a.col] * len,
];
drawSegment(items, position, tip, a.color, 2);
}
}
function drawGroundGrid(items) {
if (!$('view3dShowGrid').checked) return;
const half = 0.6, step = 0.05;
const minor = '#1a1f27', major = '#2a2f37';
for (let x = -half; x <= half + 1e-6; x += step) {
const isMajor = Math.abs(x % 0.10) < 1e-6;
drawSegment(items, [x, -half, 0], [x, half, 0], isMajor ? major : minor, 1);
}
for (let y = -half; y <= half + 1e-6; y += step) {
const isMajor = Math.abs(y % 0.10) < 1e-6;
drawSegment(items, [-half, y, 0], [half, y, 0], isMajor ? major : minor, 1);
}
}
function drawBaseAxes(items) {
drawTriad(items, [0,0,0], [[1,0,0],[0,1,0],[0,0,1]], 0.12);
drawLabel(items, [0.12, 0, 0], 'X', '#e74c3c');
drawLabel(items, [0, 0.12, 0], 'Y', '#3ecf8e');
drawLabel(items, [0, 0, 0.12], 'Z', '#4da3ff');
drawLabel(items, [0,0,0], 'BASE', '#9aa0a6', 8, 14);
}
function drawWrist(items, side, pose, color) {
if (!pose || !pose.position || !pose.quaternion) return;
const p = pose.position;
const R = quatToR(pose.quaternion);
drawTriad(items, p, R, 0.08);
drawDot(items, p, color, 5);
drawLabel(items, p, `polso ${side}`, color, 10, -6);
}
function drawObject(items) {
if (!view3d.object) return;
if ($('view3dShowCloud').checked && view3d.object.points) {
// Cap render to avoid huge per-frame work.
const max = 1500;
const pts = view3d.object.points;
const stride = Math.max(1, Math.ceil(pts.length / max));
for (let i = 0; i < pts.length; i += stride) {
drawDot(items, pts[i], '#aaaaaa', 1.4);
}
}
if (view3d.object.centroid) {
drawDot(items, view3d.object.centroid, '#ffd54f', 6);
drawLabel(items, view3d.object.centroid, 'oggetto', '#ffd54f', 10, -8);
}
}
function drawPlan(items) {
if (!view3d.plan) return;
const g = view3d.plan.grasp, pg = view3d.plan.pre_grasp;
if (!g || !g.position) return;
const R = quatToR(g.quaternion_xyzw);
// grasp pose triad + marker
drawTriad(items, g.position, R, 0.07);
drawDot(items, g.position, '#ff5e7e', 6);
drawLabel(items, g.position, 'grasp', '#ff5e7e', 10, -8);
// pre-grasp marker + approach line
if (pg && pg.position) {
drawDot(items, pg.position, '#ffffff', 4);
drawLabel(items, pg.position, 'pre-grasp', '#ffffff', 10, -8);
drawSegment(items, pg.position, g.position, '#ffffff', 1.5);
}
}
function renderView3d() {
const ctx = view3dCtx;
ctx.fillStyle = '#0d1014';
ctx.fillRect(0, 0, view3dCanvas.width, view3dCanvas.height);
const items = [];
drawGroundGrid(items);
drawBaseAxes(items);
if (view3d.wrist) {
drawWrist(items, 'right', view3d.wrist.right, '#f0ad4e');
drawWrist(items, 'left', view3d.wrist.left, '#9b59b6');
}
drawObject(items);
drawPlan(items);
// Painter's algorithm: far first.
items.sort((a, b) => b.depth - a.depth);
items.forEach(it => it.draw());
// HUD
ctx.fillStyle = '#9aa0a6';
ctx.font = '11px ui-monospace, monospace';
ctx.fillText(`yaw=${(view3d.yaw*180/Math.PI).toFixed(0)}° pitch=${(view3d.pitch*180/Math.PI).toFixed(0)}° scale=${view3d.scale.toFixed(0)} px/m`, 8, 14);
ctx.fillText(`target=[${view3d.target.map(v => v.toFixed(2)).join(', ')}]`, 8, 28);
if (!view3d.calibrated) {
ctx.fillStyle = '#f0ad4e';
ctx.fillText('⚠ calibrazione mancante: punti oggetto non disponibili in frame base', 8, view3dCanvas.height - 8);
}
if (view3d.lastErr) {
ctx.fillStyle = '#e74c3c';
ctx.fillText(view3d.lastErr, 8, view3dCanvas.height - 22);
}
}
// ---- mouse interaction ----
let drag3d = null;
view3dCanvas.addEventListener('mousedown', (ev) => {
drag3d = { x: ev.clientX, y: ev.clientY, pan: ev.shiftKey };
});
window.addEventListener('mouseup', () => { drag3d = null; });
window.addEventListener('mousemove', (ev) => {
if (!drag3d) return;
const dx = ev.clientX - drag3d.x;
const dy = ev.clientY - drag3d.y;
drag3d.x = ev.clientX; drag3d.y = ev.clientY;
if (drag3d.pan) {
// Pan in screen plane → world XY (approx, ignoring pitch foreshortening on Z)
const cy = Math.cos(view3d.yaw), sy = Math.sin(view3d.yaw);
const wx = cy*dx - sy*(-dy) * Math.cos(view3d.pitch);
const wy = sy*dx + cy*(-dy) * Math.cos(view3d.pitch);
view3d.target[0] -= wx / view3d.scale;
view3d.target[1] -= wy / view3d.scale;
// also let vertical drag move target Z when looking near horizontal
view3d.target[2] += (-dy) * Math.sin(view3d.pitch) / view3d.scale;
} else {
view3d.yaw += dx * 0.01;
view3d.pitch += dy * 0.01;
view3d.pitch = Math.max(-Math.PI/2 + 0.05, Math.min(Math.PI/2 - 0.05, view3d.pitch));
}
renderView3d();
});
view3dCanvas.addEventListener('wheel', (ev) => {
ev.preventDefault();
const f = Math.exp(-ev.deltaY * 0.001);
view3d.scale = Math.max(50, Math.min(3000, view3d.scale * f));
renderView3d();
}, { passive: false });
view3dCanvas.addEventListener('dblclick', () => resetView3d());
function resetView3d() {
view3d.yaw = -0.6;
view3d.pitch = 0.5;
view3d.scale = 350;
view3d.target = [0.30, 0.0, 0.20];
renderView3d();
}
$('view3dResetBtn').addEventListener('click', resetView3d);
function autoFitView3d() {
if (!$('view3dAutoFit').checked) return;
const pts = [];
if (view3d.wrist) {
if (view3d.wrist.right && view3d.wrist.right.position) pts.push(view3d.wrist.right.position);
if (view3d.wrist.left && view3d.wrist.left.position) pts.push(view3d.wrist.left.position);
}
if (view3d.object && view3d.object.centroid) pts.push(view3d.object.centroid);
if (view3d.plan && view3d.plan.grasp && view3d.plan.grasp.position) pts.push(view3d.plan.grasp.position);
if (pts.length === 0) return;
const min = [Infinity, Infinity, Infinity], max = [-Infinity, -Infinity, -Infinity];
for (const p of pts) for (let i = 0; i < 3; i++) {
if (p[i] < min[i]) min[i] = p[i];
if (p[i] > max[i]) max[i] = p[i];
}
view3d.target = [(min[0]+max[0])/2, (min[1]+max[1])/2, (min[2]+max[2])/2];
const span = Math.max(0.15, max[0]-min[0], max[1]-min[1], max[2]-min[2]);
// fit ~70% of canvas width
view3d.scale = Math.min(2000, (view3dCanvas.width * 0.7) / (span * 2.5));
}
// ---- data fetching ----
async function refreshView3d() {
view3d.lastErr = null;
const out = { wrist: null, object: null, plan: null, calibrated: true };
const summary = [];
// Wrist (always; doesn't need calibration — already in base frame).
try {
const w = await api('GET', '/api/wrist');
// Server returns {left_wrist, right_wrist}; tolerate {left,right} too.
const data = (w && w.data) || w || {};
out.wrist = {
left: data.left_wrist || data.left || null,
right: data.right_wrist || data.right || null,
};
summary.push(`wrist: ${out.wrist.right ? 'R✓' : 'R—'} ${out.wrist.left ? 'L✓' : 'L—'}`);
} catch (e) {
summary.push(`wrist: err (${e.message})`);
}
// Object points + centroid in base frame.
try {
const body = {
conf: +$('view3dConf').value || null,
class_name: $('view3dClass').value || null,
max_points: 3000,
};
const r = await api('POST', '/api/perception/object_points', body);
const pts = r.points_base || null;
if (!pts) {
out.calibrated = false;
summary.push('oggetto: serve calibrazione');
} else {
let cx=0, cy=0, cz=0;
for (const p of pts) { cx+=p[0]; cy+=p[1]; cz+=p[2]; }
const n = Math.max(1, pts.length);
out.object = {
points: pts,
centroid: [cx/n, cy/n, cz/n],
n_points: pts.length,
detection: r.detection,
};
summary.push(`oggetto: ${pts.length} pt`);
}
} catch (e) {
if (/^412\b/.test(e.message)) out.calibrated = false;
summary.push(`oggetto: ${e.message}`);
}
// Grasp plan (optional).
if ($('view3dShowPlan').checked && out.calibrated && out.object) {
try {
const body = {
conf: +$('view3dConf').value || null,
class_name: $('view3dClass').value || null,
};
const r = await api('POST', '/api/grasp/plan', body);
out.plan = r.plan || null;
if (out.plan && out.plan.grasp) {
const p = out.plan.grasp.position;
summary.push(`grasp: [${p.map(v => v.toFixed(2)).join(', ')}]`);
}
} catch (e) {
summary.push(`plan: ${e.message}`);
}
}
Object.assign(view3d, out);
autoFitView3d();
renderView3d();
$('view3dStatusLine').textContent = summary.join(' · ');
$('view3dStatusLine').style.color = view3d.lastErr ? 'var(--danger)' : 'var(--dim)';
$('view3dOut').textContent = JSON.stringify({
wrist: out.wrist,
object: out.object && {
n_points: out.object.n_points,
centroid: out.object.centroid,
detection: out.object.detection,
},
plan: out.plan,
}, null, 2);
}
$('view3dRefreshBtn').addEventListener('click', async () => {
setStatus('Aggiorno scena 3D…');
try { await refreshView3d(); setStatus('Scena 3D aggiornata', 'ok'); }
catch (e) { setStatus(e.message, 'err'); }
});
// loop
let view3dTimer = null;
let view3dInFlight = false;
async function view3dTick() {
if (view3dInFlight) return;
view3dInFlight = true;
try { await refreshView3d(); } catch {} finally { view3dInFlight = false; }
}
function startView3dLoop() {
if (view3dTimer) return;
const rate = Math.max(200, +$('view3dRateMs').value || 800);
view3dTick();
view3dTimer = setInterval(view3dTick, rate);
const b = $('view3dLoopBtn');
b.textContent = '■ Stop loop';
b.classList.add('running');
}
function stopView3dLoop() {
if (view3dTimer) { clearInterval(view3dTimer); view3dTimer = null; }
const b = $('view3dLoopBtn');
b.textContent = '▶ Loop';
b.classList.remove('running');
}
$('view3dLoopBtn').addEventListener('click', () => view3dTimer ? stopView3dLoop() : startView3dLoop());
// Pause loop when leaving the tab (mirrors the grasp-tab behavior).
document.querySelectorAll('.tab').forEach(btn => btn.addEventListener('click', () => {
if (btn.dataset.tab !== 'view3d' && view3dTimer) stopView3dLoop();
if (btn.dataset.tab === 'view3d') renderView3d();
}));
// Initial empty render so the canvas isn't blank when first shown.
renderView3d();
// ==================== HOME ==================== // ==================== HOME ====================
// Shows reachability + latency for every upstream microservice and lets the // Shows reachability + latency for every upstream microservice and lets the
// user switch the active robot (G1 ↔ H1). The orchestrator runs the actual // user switch the active robot (G1 ↔ H1). The orchestrator runs the actual
@@ -1128,6 +742,16 @@ async function refreshHomeRobotSelector() {
refreshHomeRobotSelector(); refreshHomeRobotSelector();
startHomePoll(); startHomePoll();
// Badge versione build (header). Si aggiorna ad ogni rebuild via build.sh.
async function refreshAppVersion() {
try {
const r = await api('GET', '/api/version');
const pill = $('appVersionPill');
if (pill) { pill.textContent = `V${r.version} · ${r.build_time}`; pill.title = `versione ${r.version} · build ${r.build_time}`; }
} catch {}
}
refreshAppVersion();
// ---------- init ---------- // ---------- init ----------
(async () => { (async () => {
try { try {
@@ -1136,349 +760,3 @@ startHomePoll();
} catch {} } catch {}
refreshWrist(); refreshWrist();
})(); })();
// ==================== ASSISTENTE AI ====================
// Curated tool-use chat: l'utente parla in linguaggio naturale, il backend
// passa la richiesta a OpenRouter con i tool definiti in agent/tools.py.
// La cronologia è tenuta lato browser e ribaltata indietro a ogni POST.
const agentState = {
history: [], // OpenAI-format messages (escluso system, ricostruito server-side)
visualMessages: [], // {role, content, traceIdx?} per il render
busy: false,
health: null,
};
function agentEscape(s) {
return String(s == null ? '' : s)
.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
}
function renderAgentChat() {
const host = $('agentChat');
host.innerHTML = '';
if (agentState.visualMessages.length === 0) {
const sys = document.createElement('div');
sys.className = 'agent-msg system';
sys.textContent = 'Conversazione vuota. Scrivi una richiesta sotto.';
host.appendChild(sys);
return;
}
for (const m of agentState.visualMessages) {
const div = document.createElement('div');
div.className = `agent-msg ${m.role}`;
div.textContent = m.content || '';
if (m.tools && m.tools.length) {
const meta = document.createElement('div');
meta.className = 'meta';
for (const t of m.tools) {
const pill = document.createElement('span');
pill.className = 'agent-tool-pill' + (t.ok ? '' : ' err');
pill.textContent = `${t.name} · ${t.latency_ms}ms`;
pill.title = JSON.stringify(t.args || {}, null, 2);
meta.appendChild(pill);
}
div.appendChild(meta);
}
host.appendChild(div);
}
host.scrollTop = host.scrollHeight;
}
async function refreshAgentHealth() {
try {
const h = await api('GET', '/api/agent/health');
agentState.health = h;
const el = $('agentHealth');
if (h.configured) {
el.textContent = `pronto · modello ${h.model}`;
el.style.color = 'var(--ok)';
$('agentSendBtn').disabled = false;
} else {
el.textContent = `API key mancante (env ${h.api_key_env}): chat disabilitata`;
el.style.color = 'var(--danger)';
$('agentSendBtn').disabled = true;
}
} catch (e) {
$('agentHealth').textContent = `errore: ${e.message}`;
$('agentHealth').style.color = 'var(--danger)';
}
// mirror del robot attivo nella pill della tab
try {
const r = await api('GET', '/api/home/robot');
const pill = $('agentActiveRobotPill');
if (pill) pill.textContent = `robot: ${(r.active || '—').toUpperCase()}`;
} catch {}
}
async function sendAgentMessage() {
if (agentState.busy) return;
const input = $('agentInput');
const message = input.value.trim();
if (!message) return;
agentState.busy = true;
$('agentSendBtn').disabled = true;
$('agentStatus').textContent = 'invio…';
$('agentStatus').style.color = 'var(--dim)';
agentState.visualMessages.push({ role: 'user', content: message });
renderAgentChat();
input.value = '';
try {
const r = await api('POST', '/api/agent/chat', {
message,
history: agentState.history,
});
agentState.history = r.history || [];
const tools = (r.tool_calls || []).map(tc => ({
name: tc.name, ok: tc.result && tc.result.ok !== false,
latency_ms: tc.latency_ms, args: tc.arguments,
}));
agentState.visualMessages.push({
role: 'assistant', content: r.reply || '(nessuna risposta)',
tools,
});
renderAgentChat();
// Tool trace dettagliata nel pannello di destra.
if (r.tool_calls && r.tool_calls.length) {
$('agentTrace').textContent = JSON.stringify(r.tool_calls, null, 2);
} else {
$('agentTrace').textContent = '(nessuna chiamata)';
}
const usage = r.usage || {};
const u = usage.total_tokens != null ? `· ${usage.total_tokens} tokens` : '';
$('agentStatus').textContent = `ok · ${r.iterations} iterazioni · ${tools.length} tool ${u}`;
$('agentStatus').style.color = 'var(--ok)';
// se la risposta ha cambiato robot attivo, aggiorna pill
refreshAgentHealth().catch(() => {});
} catch (e) {
agentState.visualMessages.push({
role: 'error', content: e.message,
});
renderAgentChat();
$('agentStatus').textContent = `errore: ${e.message}`;
$('agentStatus').style.color = 'var(--danger)';
} finally {
agentState.busy = false;
$('agentSendBtn').disabled = !(agentState.health && agentState.health.configured);
}
}
$('agentSendBtn').addEventListener('click', sendAgentMessage);
$('agentInput').addEventListener('keydown', (e) => {
if (e.ctrlKey && e.key === 'Enter') {
e.preventDefault();
sendAgentMessage();
}
});
$('agentClearBtn').addEventListener('click', () => {
if (agentState.visualMessages.length === 0 && agentState.history.length === 0) return;
if (!confirm('Resetta la conversazione corrente?')) return;
agentState.history = [];
agentState.visualMessages = [];
renderAgentChat();
$('agentTrace').textContent = '(nessuna chiamata)';
$('agentStatus').textContent = 'conversazione svuotata';
});
// ----- Task salvati -----
async function loadAgentTasks() {
const ul = $('agentTaskList');
try {
const r = await api('GET', '/api/agent/tasks');
const items = r.tasks || [];
ul.innerHTML = '';
if (items.length === 0) {
ul.innerHTML = '<li class="dim">(nessun task salvato)</li>';
return;
}
for (const it of items) {
const li = document.createElement('li');
const name = document.createElement('span');
name.className = 'task-name';
name.textContent = it.name;
const desc = document.createElement('span');
desc.className = 'task-desc';
desc.textContent = it.description || it.prompt;
desc.title = it.prompt;
const useBtn = document.createElement('button');
useBtn.className = 'link-btn';
useBtn.textContent = '↥ usa';
useBtn.title = 'Carica questo prompt nella casella di input';
useBtn.addEventListener('click', () => {
$('agentInput').value = it.prompt;
$('agentInput').focus();
});
const runBtn = document.createElement('button');
runBtn.className = 'link-btn';
runBtn.textContent = '▶ esegui';
runBtn.addEventListener('click', () => {
$('agentInput').value = it.prompt;
sendAgentMessage();
});
const delBtn = document.createElement('button');
delBtn.className = 'link-btn';
delBtn.textContent = '✕';
delBtn.title = 'Elimina';
delBtn.addEventListener('click', async () => {
if (!confirm(`Eliminare task "${it.name}"?`)) return;
try {
await api('DELETE', `/api/agent/tasks/${encodeURIComponent(it.name)}`);
loadAgentTasks();
} catch (e) { toast('Eliminazione fallita', e.message, false); }
});
li.appendChild(name);
li.appendChild(desc);
li.appendChild(useBtn);
li.appendChild(runBtn);
li.appendChild(delBtn);
ul.appendChild(li);
}
} catch (e) {
ul.innerHTML = `<li class="err">errore: ${agentEscape(e.message)}</li>`;
}
}
$('agentTaskSaveBtn').addEventListener('click', async (ev) => {
const name = $('agentTaskName').value.trim();
const prompt = $('agentInput').value.trim();
const description = $('agentTaskDesc').value.trim() || null;
if (!name) { toast('Manca il nome', 'Indica un nome per il task', false); return; }
if (!prompt) { toast('Prompt vuoto', 'Scrivi prima il prompt nella casella', false); return; }
try {
await api('POST', '/api/agent/tasks', { name, prompt, description });
flashBtn(ev.currentTarget, true);
toast('Task salvato', name, true);
$('agentTaskName').value = '';
$('agentTaskDesc').value = '';
loadAgentTasks();
} catch (e) {
flashBtn(ev.currentTarget, false);
toast('Salvataggio fallito', e.message, false);
}
});
$('agentTaskReloadBtn').addEventListener('click', loadAgentTasks);
// Carica health + task quando si apre la tab Assistente AI.
document.querySelectorAll('.tab').forEach(btn => btn.addEventListener('click', () => {
if (btn.dataset.tab === 'agent') {
refreshAgentHealth();
loadAgentTasks();
}
}));
// Bootstrap iniziale (anche se la tab non è attiva, così la pill si popola).
refreshAgentHealth();
loadAgentTasks();
// ==================== IMPOSTAZIONI AI ====================
async function loadAiSettings() {
const select = $('aiModelSelect');
// Lista modelli (statica con override custom): si carica una sola volta.
if (select && select.options.length <= 1) {
try {
const r = await api('GET', '/api/agent/known_models');
select.innerHTML = '';
for (const m of r.models) {
const opt = document.createElement('option');
opt.value = m.id;
opt.textContent = `${m.id} — ${m.label}`;
select.appendChild(opt);
}
} catch (e) {
select.innerHTML = `<option value="">errore: ${e.message}</option>`;
}
}
// Carica config corrente
try {
const cfg = await api('GET', '/api/config');
const o = (cfg.chat && cfg.chat.openrouter) || {};
const model = o.model || 'anthropic/claude-sonnet-4.5';
if ([...$('aiModelSelect').options].some(opt => opt.value === model)) {
$('aiModelSelect').value = model;
$('aiModelCustom').value = '';
} else {
$('aiModelCustom').value = model;
}
$('aiTemperature').value = (o.temperature != null) ? o.temperature : 0.1;
$('aiMaxIterations').value = o.max_tool_iterations || 8;
$('aiTimeout').value = o.timeout_s || 60;
$('aiPromptAddendum').value = o.user_prompt_addendum || '';
$('aiSettingsStatus').textContent = 'caricato da config';
$('aiSettingsStatus').style.color = 'var(--dim)';
} catch (e) {
$('aiSettingsStatus').textContent = `errore: ${e.message}`;
$('aiSettingsStatus').style.color = 'var(--danger)';
}
}
async function saveAiSettings(btn) {
const customModel = $('aiModelCustom').value.trim();
const model = customModel || $('aiModelSelect').value;
if (!model) { toast('Modello mancante', 'Seleziona o scrivi un modello', false); return; }
const temperature = parseFloat($('aiTemperature').value);
const max_tool_iterations = parseInt($('aiMaxIterations').value, 10);
const timeout_s = parseFloat($('aiTimeout').value);
if (Number.isNaN(temperature) || temperature < 0 || temperature > 2) {
toast('Temperature non valido', '0..2', false); return;
}
if (Number.isNaN(max_tool_iterations) || max_tool_iterations < 1) {
toast('Max iterations non valido', '>=1', false); return;
}
const patch = {
chat: { openrouter: {
model,
temperature,
max_tool_iterations,
timeout_s,
user_prompt_addendum: $('aiPromptAddendum').value,
}},
};
try {
await api('PUT', '/api/config', patch);
flashBtn(btn, true);
toast('Salvato', `model=${model}, T=${temperature}`, true);
$('aiSettingsStatus').textContent = `salvato · model=${model} · T=${temperature}`;
$('aiSettingsStatus').style.color = 'var(--ok)';
refreshAgentHealth().catch(() => {});
} catch (e) {
flashBtn(btn, false);
toast('Errore salvataggio', e.message, false);
$('aiSettingsStatus').textContent = e.message;
$('aiSettingsStatus').style.color = 'var(--danger)';
}
}
async function previewSystemPrompt() {
const pre = $('aiPromptPreview');
pre.textContent = 'caricamento…';
try {
const r = await api('GET', '/api/agent/system_prompt_preview');
pre.textContent = r.system_prompt || '(vuoto)';
} catch (e) {
pre.textContent = `errore: ${e.message}`;
}
}
$('aiSaveBtn').addEventListener('click', (ev) => saveAiSettings(ev.currentTarget));
$('aiReloadBtn').addEventListener('click', loadAiSettings);
$('aiResetTempBtn').addEventListener('click', () => { $('aiTemperature').value = 0.1; });
$('aiPromptPreviewBtn').addEventListener('click', previewSystemPrompt);
$('aiPromptCopyBtn').addEventListener('click', async () => {
try {
await navigator.clipboard.writeText($('aiPromptPreview').textContent);
toast('Copiato', 'Prompt nel clipboard', true);
} catch (e) { toast('Copia fallita', e.message, false); }
});
// Quando l'utente seleziona un modello dalla combo, svuota il custom (priorità chiara).
$('aiModelSelect').addEventListener('change', () => { $('aiModelCustom').value = ''; });
// Carica i valori quando si apre la tab.
document.querySelectorAll('.tab').forEach(btn => btn.addEventListener('click', () => {
if (btn.dataset.tab === 'ai-settings') {
loadAiSettings();
previewSystemPrompt();
}
}));
File diff suppressed because it is too large. Load diff
Binary file not shown.
-479
View File
@@ -1,479 +0,0 @@
/* G1 3D diagnostic viewer.
*
* Loads the URDF, builds the scene graph with two-level groups per joint
* (outer = <origin>, inner = motion driven by diagnostic q), renders meshes
* tinted by per-motor temperature and colored spheres at joint origins.
* Exposes a tiny global API consumed by app.js powerPollOnce:
* window.g1Diag3d.update(diagData)
*
* Deliberately avoids depending on any URDF library — the URDF subset we
* need (fixed/revolute/continuous/prismatic joints, single mesh per visual,
* relative mesh paths) is small enough to parse in ~50 lines.
*/
import * as THREE from 'three';
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
import { STLLoader } from 'three/addons/loaders/STLLoader.js';
const URDF_URL = '/static/assets/g1/g1_body29_hand14.urdf';
const MESH_BASE = '/static/assets/g1/';
const WARN_C_DEFAULT = 75;
const CRIT_C_DEFAULT = 85;
function parseOrigin(el) {
if (!el) return { xyz: [0, 0, 0], rpy: [0, 0, 0] };
const parse = (s) => s ? s.trim().split(/\s+/).map(Number) : [0, 0, 0];
return {
xyz: parse(el.getAttribute('xyz')),
rpy: parse(el.getAttribute('rpy')),
};
}
function parseAxis(el) {
if (!el) return [1, 0, 0];
const s = el.getAttribute('xyz');
return s ? s.trim().split(/\s+/).map(Number) : [1, 0, 0];
}
async function parseURDF(xmlText) {
const doc = new DOMParser().parseFromString(xmlText, 'application/xml');
const links = {};
for (const el of doc.querySelectorAll('link')) {
const name = el.getAttribute('name');
if (!name) continue;
const v = el.querySelector('visual');
let mesh = null, visualOrigin = { xyz: [0,0,0], rpy: [0,0,0] };
if (v) {
visualOrigin = parseOrigin(v.querySelector('origin'));
const me = v.querySelector('geometry mesh');
if (me) mesh = me.getAttribute('filename');
}
links[name] = { name, mesh, visualOrigin };
}
const joints = [];
for (const el of doc.querySelectorAll('joint')) {
const parent = el.querySelector('parent');
const child = el.querySelector('child');
if (!parent || !child) continue;
joints.push({
name: el.getAttribute('name'),
type: el.getAttribute('type') || 'fixed',
parent: parent.getAttribute('link'),
child: child.getAttribute('link'),
origin: parseOrigin(el.querySelector('origin')),
axis: parseAxis(el.querySelector('axis')),
});
}
return { links, joints };
}
function applyTransform(obj, o) {
obj.position.set(o.xyz[0], o.xyz[1], o.xyz[2]);
obj.rotation.order = 'ZYX'; // URDF RPY: intrinsic rotations yaw * pitch * roll
obj.rotation.set(o.rpy[0], o.rpy[1], o.rpy[2]);
}
function tempToColor(t, warn, crit) {
if (!Number.isFinite(t)) return 0x888888;
if (t >= crit) return 0xe74c3c;
if (t >= warn) return 0xf0ad4e;
// green->yellow interpolation below warn
const u = Math.max(0, Math.min(1, (t - 30) / (warn - 30)));
const r = Math.round(0x3e + u * (0xf0 - 0x3e));
const g = Math.round(0xcf - u * (0xcf - 0xad));
const b = Math.round(0x8e - u * (0x8e - 0x4e));
return (r << 16) | (g << 8) | b;
}
class G1Viewer {
constructor(canvas) {
this.canvas = canvas;
this.renderer = new THREE.WebGLRenderer({ canvas, antialias: true });
this.renderer.setPixelRatio(window.devicePixelRatio || 1);
this.scene = new THREE.Scene();
this.scene.background = new THREE.Color(0x0d1014);
// URDF is Z-up. Keep Z-up throughout for coherence with the robot base frame.
this.camera = new THREE.PerspectiveCamera(45, canvas.clientWidth / canvas.clientHeight, 0.05, 20);
this.camera.up.set(0, 0, 1);
this.camera.position.set(1.8, -1.8, 1.2);
this.camera.lookAt(0, 0, 0.4);
this.controls = new OrbitControls(this.camera, canvas);
this.controls.target.set(0, 0, 0.4);
this.controls.update();
const hemi = new THREE.HemisphereLight(0xffffff, 0x222233, 0.6);
this.scene.add(hemi);
const dir = new THREE.DirectionalLight(0xffffff, 0.8);
dir.position.set(2, 2, 4);
this.scene.add(dir);
const dir2 = new THREE.DirectionalLight(0xffffff, 0.3);
dir2.position.set(-2, -1, 2);
this.scene.add(dir2);
this.grid = new THREE.GridHelper(2, 20, 0x444444, 0x222233);
this.grid.rotation.x = Math.PI / 2; // grid is XY plane in Z-up
this.scene.add(this.grid);
this.robotRoot = new THREE.Group();
this.scene.add(this.robotRoot);
this.linkGroups = {};
this.linkMeshes = {};
this.jointGroups = {}; // static <origin> holders
this.jointMotions = {}; // inner motion-driven holders
this.jointIndicators = {}; // sphere at joint origin
this.motorByJointName = {}; // joint_name -> motor dict (from /diagnostics)
this.warnC = WARN_C_DEFAULT;
this.critC = CRIT_C_DEFAULT;
this.showMeshTint = true;
this.showSpheres = true;
this.pinnedJoint = null;
this.raycaster = new THREE.Raycaster();
this.mouseNDC = new THREE.Vector2();
this._onResize = () => this.resize();
window.addEventListener('resize', this._onResize);
this.canvas.addEventListener('pointermove', (ev) => this._onHover(ev));
this.canvas.addEventListener('click', (ev) => this._onClick(ev));
this._animate = this._animate.bind(this);
this.resize();
this._animate();
}
resize() {
const w = this.canvas.clientWidth || 720;
const h = this.canvas.clientHeight || 560;
this.renderer.setSize(w, h, false);
this.camera.aspect = w / h;
this.camera.updateProjectionMatrix();
}
_animate() {
this._raf = requestAnimationFrame(this._animate);
this.controls.update();
this.renderer.render(this.scene, this.camera);
}
async load() {
const res = await fetch(URDF_URL);
if (!res.ok) throw new Error(`URDF fetch ${res.status}`);
const xml = await res.text();
const urdf = await parseURDF(xml);
this._urdf = urdf;
this._build(urdf);
await this._loadMeshes(urdf);
this._placeFloor();
}
_build(urdf) {
const childSet = new Set(urdf.joints.map(j => j.child));
const rootName = Object.keys(urdf.links).find(n => !childSet.has(n));
const buildLink = (name) => {
const link = urdf.links[name];
const g = new THREE.Group();
g.name = `link:${name}`;
g.userData.linkName = name;
this.linkGroups[name] = g;
// child joints
for (const j of urdf.joints.filter(x => x.parent === name)) {
const origin = new THREE.Group();
applyTransform(origin, j.origin);
origin.userData.joint = j;
this.jointGroups[j.name] = origin;
const motion = new THREE.Group();
origin.add(motion);
this.jointMotions[j.name] = motion;
// sphere indicator at joint origin (in origin's local frame → origin(0,0,0))
const sphere = new THREE.Mesh(
new THREE.SphereGeometry(0.012, 16, 12),
new THREE.MeshBasicMaterial({ color: 0x888888, depthTest: false, transparent: true, opacity: 0.9 }),
);
sphere.renderOrder = 999;
sphere.userData.jointName = j.name;
origin.add(sphere);
this.jointIndicators[j.name] = sphere;
motion.add(buildLink(j.child));
g.add(origin);
}
return g;
};
this.robotRoot.add(buildLink(rootName));
}
async _loadMeshes(urdf) {
const loader = new STLLoader();
const tasks = [];
for (const name of Object.keys(urdf.links)) {
const link = urdf.links[name];
if (!link.mesh) continue;
const url = MESH_BASE + link.mesh;
tasks.push(new Promise((resolve) => {
loader.load(url, (geom) => {
geom.computeVertexNormals();
const mat = new THREE.MeshStandardMaterial({
color: 0x888888, metalness: 0.1, roughness: 0.6,
flatShading: false,
});
const mesh = new THREE.Mesh(geom, mat);
mesh.userData.linkName = name;
applyTransform(mesh, link.visualOrigin);
this.linkMeshes[name] = mesh;
this.linkGroups[name].add(mesh);
resolve();
}, undefined, (err) => {
console.warn('STL load failed:', url, err);
resolve();
});
}));
}
await Promise.all(tasks);
}
_placeFloor() {
// position grid at model's feet (min z of bounding box).
const box = new THREE.Box3().setFromObject(this.robotRoot);
if (isFinite(box.min.z)) this.grid.position.z = box.min.z;
}
setShowGrid(v) { this.grid.visible = v; }
setShowSpheres(v) {
this.showSpheres = v;
for (const s of Object.values(this.jointIndicators)) s.visible = v;
}
setShowMeshTint(v) {
this.showMeshTint = v;
this._applyColors();
}
fitView() {
const box = new THREE.Box3().setFromObject(this.robotRoot);
if (!isFinite(box.min.z)) return;
const c = new THREE.Vector3();
box.getCenter(c);
const s = new THREE.Vector3();
box.getSize(s);
const d = Math.max(s.x, s.y, s.z) * 1.8;
this.controls.target.copy(c);
const dir = new THREE.Vector3(1, -1, 0.4).normalize();
this.camera.position.copy(c).addScaledVector(dir, d);
this.controls.update();
}
resetView() {
this.controls.target.set(0, 0, 0.4);
this.camera.position.set(1.8, -1.8, 1.2);
this.controls.update();
}
update(diagData) {
if (!diagData || !Array.isArray(diagData.motors)) return;
this.warnC = Number(diagData.warn_temp_c ?? WARN_C_DEFAULT);
this.critC = Number(diagData.crit_temp_c ?? CRIT_C_DEFAULT);
this.motorByJointName = {};
for (const m of diagData.motors) {
const jn = m.name + '_joint';
this.motorByJointName[jn] = m;
// Apply joint angle.
const jg = this.jointMotions[jn];
if (jg) {
const joint = this.jointGroups[jn].userData.joint;
const ax = new THREE.Vector3(...joint.axis).normalize();
if (joint.type === 'revolute' || joint.type === 'continuous') {
jg.quaternion.setFromAxisAngle(ax, Number(m.q) || 0);
} else if (joint.type === 'prismatic') {
jg.position.copy(ax.multiplyScalar(Number(m.q) || 0));
}
}
}
this._applyColors();
if (this.pinnedJoint) this._renderPinned();
this._renderSummary(diagData);
}
_applyColors() {
for (const [jointName, motor] of Object.entries(this.motorByJointName)) {
const joint = this.jointGroups[jointName];
if (!joint) continue;
const temp = motor.temperature_motor;
const faulted = !!motor.faulted;
const enabled = motor.enabled !== false;
let color = tempToColor(temp, this.warnC, this.critC);
if (!enabled) color = 0x333333;
// sphere
const sph = this.jointIndicators[jointName];
if (sph) {
sph.material.color.setHex(color);
if (faulted) sph.material.color.setHex(0xe74c3c);
}
// mesh tint on child link
const childLink = joint.userData.joint.child;
const mesh = this.linkMeshes[childLink];
if (mesh) {
if (this.showMeshTint) {
mesh.material.color.setHex(color);
mesh.material.emissive.setHex(faulted ? 0x330000 : 0x000000);
mesh.material.emissiveIntensity = faulted ? 0.8 : 0;
} else {
mesh.material.color.setHex(0x888888);
mesh.material.emissiveIntensity = 0;
}
}
}
}
_pickJointAt(clientX, clientY) {
const rect = this.canvas.getBoundingClientRect();
this.mouseNDC.x = ((clientX - rect.left) / rect.width) * 2 - 1;
this.mouseNDC.y = -((clientY - rect.top) / rect.height) * 2 + 1;
this.raycaster.setFromCamera(this.mouseNDC, this.camera);
// Prefer hits on joint spheres first (small but always on top).
const sphHits = this.raycaster.intersectObjects(Object.values(this.jointIndicators), false);
if (sphHits.length) return sphHits[0].object.userData.jointName;
// Fall back to mesh hits (walk up the tree to find the joint whose motion group is the ancestor).
const meshHits = this.raycaster.intersectObjects(Object.values(this.linkMeshes), false);
if (!meshHits.length) return null;
const linkName = meshHits[0].object.userData.linkName;
// Find the joint whose child == linkName.
const joint = this._urdf.joints.find(j => j.child === linkName && this.motorByJointName[j.name]);
return joint ? joint.name : null;
}
_onHover(ev) {
const tt = document.getElementById('diag3dTooltip');
const joint = this._pickJointAt(ev.clientX, ev.clientY);
if (!joint) { tt.classList.add('hidden'); return; }
const m = this.motorByJointName[joint];
if (!m) { tt.classList.add('hidden'); return; }
const cls = m.faulted ? 'err' : (m.temperature_motor >= this.critC ? 'err' :
m.temperature_motor >= this.warnC ? 'warn' : '');
tt.className = 'diag3d-tooltip ' + cls;
tt.innerHTML = this._tooltipHTML(m);
const rect = this.canvas.getBoundingClientRect();
tt.style.left = (ev.clientX - rect.left + 14) + 'px';
tt.style.top = (ev.clientY - rect.top + 14) + 'px';
}
_onClick(ev) {
const joint = this._pickJointAt(ev.clientX, ev.clientY);
if (!joint) return;
this.pinnedJoint = joint;
this._renderPinned();
}
_tooltipHTML(m) {
const rows = [
['T motore', `${m.temperature_motor}°C`],
['T ctrl', `${m.temperature_ctrl}°C`],
['q', Number(m.q).toFixed(3) + ' rad'],
['dq', Number(m.dq).toFixed(3) + ' rad/s'],
['tau_est', Number(m.tau_est).toFixed(2) + ' Nm'],
['V', Number(m.voltage).toFixed(2) + ' V'],
['mode', m.mode + (m.enabled === false ? ' (OFF)' : '')],
['fault', m.faulted ? ('0x' + Number(m.motorstate).toString(16)) : 'no'],
];
const body = rows.map(([k, v]) => `<div class="tt-row"><span class="k">${k}</span><span>${v}</span></div>`).join('');
return `<div class="tt-title">#${m.index} · ${m.name}</div>${body}`;
}
_renderPinned() {
const host = document.getElementById('diag3dPinned');
if (!host) return;
if (!this.pinnedJoint) return;
const m = this.motorByJointName[this.pinnedJoint];
if (!m) {
host.innerHTML = `<div class="dim">giunto ${this.pinnedJoint}: nessun motore associato</div>`;
return;
}
const statusCls = m.faulted ? 'err' :
m.temperature_motor >= this.critC ? 'err' :
m.temperature_motor >= this.warnC ? 'warn' : '';
const rows = [
['nome', m.name],
['indice', m.index],
['T motore (°C)', m.temperature_motor],
['T ctrl (°C)', m.temperature_ctrl],
['q (rad)', Number(m.q).toFixed(4)],
['dq (rad/s)', Number(m.dq).toFixed(4)],
['tau_est (Nm)', Number(m.tau_est).toFixed(3)],
['voltaggio (V)', Number(m.voltage).toFixed(3)],
['mode', m.mode],
['enabled', m.enabled ? 'sì' : 'no'],
['motorstate raw', '0x' + Number(m.motorstate).toString(16)],
['faulted', m.faulted ? 'SÌ' : 'no'],
['derating', m.derating ? 'sì' : 'no'],
];
host.innerHTML =
`<div><strong>#${m.index} · ${m.name}</strong> <span class="dim">(giunto: ${this.pinnedJoint})</span></div>` +
`<table><tbody>${rows.map(([k,v]) =>
`<tr class="${statusCls}"><td>${k}</td><td>${v}</td></tr>`).join('')}</tbody></table>`;
}
_renderSummary(d) {
const host = document.getElementById('diag3dSummary');
if (!host) return;
const s = d.summary || {};
const fmt = (arr) => (arr && arr.length) ? arr.join(', ') : '—';
const rows = [
['lowstate', d.stale ? 'STALE' : (d.tick != null ? `tick ${d.tick}` : 'ok')],
['max temp (°C)', s.max_motor_temp_c != null ? `${s.max_motor_temp_c} @ ${s.max_motor_temp_joint}` : '—'],
['warn / crit (°C)', `${d.warn_temp_c} / ${d.crit_temp_c}`],
['derating', fmt(s.derating_joints)],
['critical', fmt(s.critical_joints)],
['faulted', fmt(s.faulted_joints)],
['disabled', fmt(s.disabled_joints)],
['min V', s.min_voltage != null ? Number(s.min_voltage).toFixed(2) + ' V' : '—'],
];
host.innerHTML = `<table><tbody>${rows.map(([k,v]) =>
`<tr><td>${k}</td><td>${v}</td></tr>`).join('')}</tbody></table>`;
}
}
// ---- Boot and expose global hook. ----
const canvas = document.getElementById('diag3dCanvas');
const statusEl = document.getElementById('diag3dStatus');
let viewer = null;
async function boot() {
if (!canvas) return;
try {
statusEl.textContent = 'caricamento URDF + mesh…';
viewer = new G1Viewer(canvas);
await viewer.load();
statusEl.textContent = 'URDF caricato · in attesa di /diagnostics…';
// Wire up toolbar controls.
const by = (id) => document.getElementById(id);
by('diag3dMeshTint').addEventListener('change', (e) => viewer.setShowMeshTint(e.target.checked));
by('diag3dSpheres').addEventListener('change', (e) => viewer.setShowSpheres(e.target.checked));
by('diag3dGrid').addEventListener('change', (e) => viewer.setShowGrid(e.target.checked));
by('diag3dResetBtn').addEventListener('click', () => viewer.resetView());
by('diag3dFitBtn').addEventListener('click', () => viewer.fitView());
// Re-size when the diagnostic tab becomes active (canvas may have 0 size while hidden).
document.querySelectorAll('.tab').forEach(btn => btn.addEventListener('click', () => {
if (btn.dataset.tab === 'diagnostic') setTimeout(() => viewer && viewer.resize(), 50);
}));
// Global hook for app.js to push diagnostic updates.
window.g1Diag3d = {
update: (d) => {
try { viewer.update(d); statusEl.textContent =
d.stale ? 'lowstate STALE' :
d.available === false ? (d.reason || '/diagnostics non disponibile') :
`motori aggiornati · tick ${d.tick ?? '?'}`; }
catch (e) { console.warn('g1Diag3d.update:', e); }
},
};
} catch (e) {
console.error('diag3d boot failed:', e);
statusEl.textContent = 'errore caricamento: ' + e.message;
}
}
// Defer boot until the tab is first opened — avoids eager CDN fetch and STL load
// for users who never visit this tab.
let booted = false;
document.querySelectorAll('.tab').forEach(btn => btn.addEventListener('click', () => {
if (btn.dataset.tab === 'diagnostic' && !booted) { booted = true; boot(); }
}));
+175 -168
View File
@@ -12,17 +12,15 @@
<h1>presa-mano</h1> <h1>presa-mano</h1>
<nav class="tabs"> <nav class="tabs">
<button class="tab active" data-tab="home">Home</button> <button class="tab active" data-tab="home">Home</button>
<button class="tab" data-tab="agent">Assistente AI</button>
<button class="tab" data-tab="ai-settings">Impostazioni AI</button>
<button class="tab" data-tab="workplane">Calibrazione Piano</button> <button class="tab" data-tab="workplane">Calibrazione Piano</button>
<button class="tab" data-tab="grasp">Cerca modello</button> <button class="tab" data-tab="grasp">Cerca modello</button>
<button class="tab" data-tab="view3d">Vista 3D</button>
<button class="tab" data-tab="right-arm">Braccio DX</button> <button class="tab" data-tab="right-arm">Braccio DX</button>
<button class="tab" data-tab="pose-3d">Pose 3D</button> <button class="tab" data-tab="pose-3d">Pose 3D</button>
<button class="tab" data-tab="ee">Movimento EE</button> <button class="tab" data-tab="ee">Movimento EE</button>
</nav> </nav>
<div class="status-strip" id="statusStrip"></div> <div class="status-strip" id="statusStrip"></div>
<span class="active-robot-pill" id="activeRobotPill" title="Robot attualmente collegato">robot: —</span> <span class="active-robot-pill" id="activeRobotPill" title="Robot attualmente collegato">robot: —</span>
<span class="active-robot-pill" id="appVersionPill" title="versione build">V—</span>
</header> </header>
<main> <main>
@@ -68,115 +66,6 @@
<pre id="homeLog" class="log small">—</pre> <pre id="homeLog" class="log small">—</pre>
</section> </section>
<!-- ============================== ASSISTENTE AI ============================== -->
<section class="panel" id="tab-agent">
<div class="posture-toolbar">
<span class="post-health" id="agentHealth">—</span>
<span class="active-robot-pill" id="agentActiveRobotPill" title="Robot attivo">robot: —</span>
<button id="agentClearBtn" title="Resetta la conversazione">Nuova conversazione</button>
</div>
<div class="grid-2 agent-layout">
<!-- LEFT: chat history + input -->
<div>
<h2>Chat</h2>
<div id="agentChat" class="agent-chat">
<div class="agent-msg system">Ciao. Dimmi cosa vuoi fare con il robot. Es: "muovi il braccio destro 200 mm in X+", "calibra il braccio destro", "mostra lo stato dei dispositivi".</div>
</div>
<div class="agent-input-row">
<textarea id="agentInput" rows="3" placeholder="Scrivi un comando o una richiesta in linguaggio naturale… (Ctrl+Invio per inviare)"></textarea>
<button id="agentSendBtn" class="primary">Invia ▸</button>
</div>
<div class="marker-info" id="agentStatus">—</div>
</div>
<!-- RIGHT: saved tasks + tool trace -->
<div>
<h2>Task salvati</h2>
<div class="info-box">
Salva un prompt usato spesso (es. "calibra braccio dx") per riutilizzarlo con un click. Il prompt corrente nella textarea viene salvato.
</div>
<div class="kv compact">
<label>Nome <input type="text" id="agentTaskName" placeholder="es. calibrazione_dx" maxlength="80"></label>
<label>Descrizione <input type="text" id="agentTaskDesc" placeholder="(opzionale)" maxlength="400"></label>
</div>
<div class="actions">
<button id="agentTaskSaveBtn" class="primary">💾 Salva prompt corrente</button>
<button id="agentTaskReloadBtn">⟳ Ricarica</button>
</div>
<ul id="agentTaskList" class="agent-task-list">
<li class="dim">caricamento…</li>
</ul>
<h2>Tool trace ultima risposta</h2>
<pre id="agentTrace" class="log small">(nessuna chiamata)</pre>
</div>
</div>
</section>
<!-- ============================== IMPOSTAZIONI AI ============================== -->
<section class="panel" id="tab-ai-settings">
<div class="info-box">
Parametri del client OpenRouter usato dall'Assistente AI. Le modifiche vengono persistite in <code>config.yaml</code> via <code>PUT /api/config</code> e applicate alla chat successiva (non c'è bisogno di restart). Il <strong>prompt aggiuntivo</strong> viene appeso al prompt di sistema dinamico — non sostituisce le regole base (frame, tool, sicurezza).
</div>
<div class="grid-2">
<div>
<h2>Modello e parametri</h2>
<div class="ctrl-block">
<div class="kv compact">
<label>Modello (combo)
<select id="aiModelSelect">
<option value="">caricamento…</option>
</select>
</label>
<label>Modello (custom — sovrascrive la combo)
<input type="text" id="aiModelCustom" placeholder="es. anthropic/claude-sonnet-4.5" />
</label>
<label title="0 = deterministico, 1 = creativo. Per orchestrazione robotica tieni basso (0.0–0.3).">
Temperature
<input type="number" id="aiTemperature" min="0" max="2" step="0.05" />
</label>
<label title="Massimo numero di iterazioni di tool calling per singolo messaggio">
Max tool iterations
<input type="number" id="aiMaxIterations" min="1" max="20" step="1" />
</label>
<label title="Timeout HTTP per la singola chiamata a OpenRouter (secondi)">
Timeout (s)
<input type="number" id="aiTimeout" min="5" max="300" step="5" />
</label>
</div>
<div class="actions">
<button id="aiSaveBtn" class="primary">💾 Salva impostazioni</button>
<button id="aiReloadBtn">⟳ Ricarica da config</button>
<button id="aiResetTempBtn" title="Riporta temperature a 0.1 (default safe)">Reset temp</button>
</div>
<div class="marker-info" id="aiSettingsStatus">—</div>
</div>
<h2>Prompt aggiuntivo</h2>
<div class="ctrl-block">
<div class="info-box">
Testo che viene appeso al system prompt dinamico (sezione "Istruzioni aggiuntive dell'operatore"). Usa per personalizzare lo stile, aggiungere contesto specifico al tuo setup, o regole custom. Lascia vuoto per non aggiungere nulla.
</div>
<textarea id="aiPromptAddendum" rows="10" style="width:100%;font-family:ui-monospace,monospace;font-size:12px;padding:8px;background:#0e1116;color:inherit;border:1px solid var(--border);border-radius:4px"></textarea>
</div>
</div>
<div>
<h2>Anteprima system prompt corrente</h2>
<div class="info-box">
Quello che il modello riceve davvero al prossimo messaggio (include lo stato corrente: robot attivo, marker, ecc., più l'eventuale addendum sopra dopo che lo salvi).
</div>
<div class="actions">
<button id="aiPromptPreviewBtn">⟳ Aggiorna anteprima</button>
<button id="aiPromptCopyBtn">📋 Copia</button>
</div>
<pre id="aiPromptPreview" class="log small" style="max-height:560px;overflow:auto">(premi 'Aggiorna anteprima')</pre>
</div>
</div>
</section>
<!-- ============================== CALIBRAZIONE PIANO ============================== --> <!-- ============================== CALIBRAZIONE PIANO ============================== -->
<section class="panel" id="tab-workplane"> <section class="panel" id="tab-workplane">
<div class="diag3d-toolbar"> <div class="diag3d-toolbar">
@@ -292,50 +181,6 @@
</div> </div>
</section> </section>
<!-- ============================== VISTA 3D ============================== -->
<section class="panel" id="tab-view3d">
<div class="grid-2">
<div>
<h2>Scena 3D — frame base robot</h2>
<div class="canvas-wrap">
<canvas id="view3dCanvas" width="720" height="540"></canvas>
</div>
<div class="marker-info" id="view3dHint">drag = ruota · wheel = zoom · doppio click = reset vista · shift+drag = pan</div>
<div class="marker-info" id="view3dStatusLine">—</div>
</div>
<div>
<h2>Dati</h2>
<div class="kv">
<label>Classe (filtro) <input type="text" id="view3dClass" placeholder="(tutte)" /></label>
<label>Confidenza <input type="number" id="view3dConf" value="0.5" step="0.05" min="0" max="1" /></label>
<label class="inline"><input type="checkbox" id="view3dShowCloud" checked /> Punti oggetto</label>
<label class="inline"><input type="checkbox" id="view3dShowGrid" checked /> Griglia tavolo</label>
<label class="inline"><input type="checkbox" id="view3dShowPlan" checked /> Pianifica grasp</label>
<label class="inline"><input type="checkbox" id="view3dAutoFit" checked /> Auto-fit vista</label>
</div>
<div class="actions">
<button id="view3dRefreshBtn" class="primary">Aggiorna scena</button>
<button id="view3dLoopBtn">▶ Loop</button>
<label class="inline" style="margin-left:8px"><input type="number" id="view3dRateMs" value="800" min="200" step="50" style="width:80px"> ms</label>
<button id="view3dResetBtn">Reset vista</button>
</div>
<div class="legend">
<div class="legend-row"><span class="dot" style="background:#e74c3c"></span> X (rosso) — base</div>
<div class="legend-row"><span class="dot" style="background:#3ecf8e"></span> Y (verde) — base</div>
<div class="legend-row"><span class="dot" style="background:#4da3ff"></span> Z (blu) — base</div>
<div class="legend-row"><span class="dot" style="background:#f0ad4e"></span> Polso destro</div>
<div class="legend-row"><span class="dot" style="background:#9b59b6"></span> Polso sinistro</div>
<div class="legend-row"><span class="dot" style="background:#aaaaaa"></span> Punti oggetto</div>
<div class="legend-row"><span class="dot" style="background:#ffd54f"></span> Centroide oggetto</div>
<div class="legend-row"><span class="dot" style="background:#ff5e7e"></span> Punto di presa</div>
<div class="legend-row"><span class="dot" style="background:#ffffff;border:1px solid #888"></span> Pre-grasp + traiettoria</div>
</div>
<h3>Stato</h3>
<pre id="view3dOut" class="log">(nessun dato — premi "Aggiorna scena")</pre>
</div>
</div>
</section>
<!-- ============================== BRACCIO DX ============================== --> <!-- ============================== BRACCIO DX ============================== -->
<section class="panel" id="tab-right-arm"> <section class="panel" id="tab-right-arm">
<div class="ra-grid"> <div class="ra-grid">
@@ -476,6 +321,7 @@
<div class="p3-help">click polso = seleziona · gizmo: T sposta / R ruota · drag = muovi · orbit = wheel/drag fondo · pan = shift+drag</div> <div class="p3-help">click polso = seleziona · gizmo: T sposta / R ruota · drag = muovi · orbit = wheel/drag fondo · pan = shift+drag</div>
</div> </div>
<aside class="p3-sidebar"> <aside class="p3-sidebar">
<div class="p3-group" data-group="pose">
<div class="p3-section"> <div class="p3-section">
<h3>Modalità</h3> <h3>Modalità</h3>
<div class="p3-row"> <div class="p3-row">
@@ -500,33 +346,188 @@
<div class="p3-section"> <div class="p3-section">
<h3>Posa target</h3> <h3>Posa target</h3>
<div class="p3-readout"> <div class="p3-readout">
<div>X <span id="p3X">—</span> mm</div> <label>X <input id="p3X" type="number" step="1" /> mm</label>
<div>Y <span id="p3Y">—</span> mm</div> <label>Y <input id="p3Y" type="number" step="1" /> mm</label>
<div>Z <span id="p3Z">—</span> mm</div> <label>Z <input id="p3Z" type="number" step="1" /> mm</label>
<div>R <span id="p3R">—</span>°</div> <label>R <input id="p3R" type="number" step="1" /> °</label>
<div>P <span id="p3P">—</span>°</div> <label>P <input id="p3P" type="number" step="1" /> °</label>
<div>W <span id="p3W">—</span>°</div> <label>W <input id="p3W" type="number" step="1" /> °</label>
</div> </div>
<div id="p3Feasibility" class="p3-feasibility"></div>
<div class="p3-status" id="p3Status">— polso fermo</div> <div class="p3-status" id="p3Status">— polso fermo</div>
<div class="p3-row" id="p3EditorActions"> <div class="p3-row" id="p3EditorActions">
<button id="p3ApplyBtn">Applica (Editor)</button> <button id="p3ApplyBtn">Applica (Editor)</button>
<button id="p3CancelBtn">Annulla modifica</button> <button id="p3CancelBtn">Annulla modifica</button>
</div> </div>
<div class="p3-row">
<button id="p3SyncBtn" title="Ricopia la posa attuale del robot negli input">⟳ Sincronizza con robot</button>
</div>
</div> </div>
<div class="p3-section"> <div class="p3-section">
<h3>Pose salvate</h3> <h3>Pose salvate — DX</h3>
<div class="p3-row"> <div class="p3-row">
<select id="p3PoseSelect"><option value="">(seleziona)</option></select> <select id="p3PoseSelectR"><option value="">(seleziona)</option></select>
</div> </div>
<div class="p3-row"> <div class="p3-row">
<button id="p3PoseGoBtn">Vai</button> <button id="p3PoseGoBtnR">Vai</button>
<input type="text" id="p3PoseName" placeholder="nome" maxlength="40"> <input type="text" id="p3PoseNameR" placeholder="nome" maxlength="40">
<button id="p3PoseSaveBtn">Salva</button> <button id="p3PoseSaveBtnR">Salva</button>
<button id="p3PoseDelBtn">✕</button> <button id="p3PoseDelBtnR">✕</button>
</div> </div>
</div> </div>
<div class="p3-section">
<h3>Pose salvate — SX</h3>
<div class="p3-row">
<select id="p3PoseSelectL"><option value="">(seleziona)</option></select>
</div>
<div class="p3-row">
<button id="p3PoseGoBtnL">Vai</button>
<input type="text" id="p3PoseNameL" placeholder="nome" maxlength="40">
<button id="p3PoseSaveBtnL">Salva</button>
<button id="p3PoseDelBtnL">✕</button>
</div>
</div>
</div><!-- /p3-group pose -->
<div class="p3-group" data-group="cameras" hidden>
<div class="p3-section">
<h3>Camera Nuwa (polso DX)</h3>
<div class="p3-row">
<label><input type="checkbox" id="p3NuwaEnabled"> Abilitata</label>
</div>
<div class="p3-readout">
<label>dX <input id="p3NuwaX" type="number" step="1" /> mm</label>
<label>dY <input id="p3NuwaY" type="number" step="1" /> mm</label>
<label>dZ <input id="p3NuwaZ" type="number" step="1" /> mm</label>
<label>R <input id="p3NuwaR" type="number" step="1" /> °</label>
<label>P <input id="p3NuwaP" type="number" step="1" /> °</label>
<label>W <input id="p3NuwaW" type="number" step="1" /> °</label>
</div>
<div class="p3-row">
<button id="p3NuwaApplyBtn">Applica</button>
<button id="p3NuwaResetBtn">Default</button>
</div>
</div>
<div class="p3-section">
<h3>Camera D435i (offset URDF)</h3>
<div class="p3-readout">
<label>dX <input id="p3D435X" type="number" step="1" /> mm</label>
<label>dY <input id="p3D435Y" type="number" step="1" /> mm</label>
<label>dZ <input id="p3D435Z" type="number" step="1" /> mm</label>
<label>R <input id="p3D435R" type="number" step="1" /> °</label>
<label>P <input id="p3D435P" type="number" step="1" /> °</label>
<label>W <input id="p3D435W" type="number" step="1" /> °</label>
</div>
<div class="p3-row">
<button id="p3D435ApplyBtn">Applica</button>
<button id="p3D435ResetBtn">Default</button>
</div>
</div>
<div class="p3-section">
<h3>Cloud cross-camera</h3>
<div class="p3-row">
<label><input type="checkbox" id="p3ShowNuwaCloud"> Nuwa</label>
<label><input type="checkbox" id="p3FlipNuwaX">↔X</label>
<label><input type="checkbox" id="p3FlipNuwaY">↔Y</label>
<label><input type="checkbox" id="p3FlipNuwaZ">↔Z</label>
</div>
<div class="p3-row">
<label><input type="checkbox" id="p3ShowD435Cloud"> D435i</label>
<label><input type="checkbox" id="p3FlipD435X">↔X</label>
<label><input type="checkbox" id="p3FlipD435Y">↔Y</label>
<label><input type="checkbox" id="p3FlipD435Z">↔Z</label>
</div>
<div class="p3-row">
<label><input type="checkbox" id="p3D435UseHandeye"> D435i: usa hand-eye (vs URDF)</label>
</div>
<div class="p3-row" title="Workaround per bug intrinsics-mismatch del servizio D435i (fx_ir/fx_rgb ≈ 0.65). Scala solo X e Y del cloud.">
<label>D435i scale XY
<input id="p3D435ScaleXY" type="number" min="0.1" max="3" step="0.01" value="0.65" style="width:60px">
</label>
</div>
<div class="p3-row" title="Calibrazione persistente Nuwa→frontale via tag condiviso (Umeyama). Premi Allinea con il tag visto da entrambe le camere.">
<label><input type="checkbox" id="p3AlignEnabled"> Allinea Nuwa→frontale</label>
<label>tag id <input id="p3AlignTagId" type="number" min="0" max="999" value="0" style="width:55px"></label>
<button id="p3AlignBtn">Allinea</button>
</div>
<div class="marker-info" id="p3AlignStatus">—</div>
<div class="p3-row">
<label>step <input id="p3CloudStep" type="number" min="1" max="32" value="8" style="width:50px"></label>
<label>max <input id="p3CloudMaxDepth" type="number" min="100" max="5000" step="100" value="2000" style="width:70px"> mm</label>
</div>
<div class="p3-row">
<label>size Nuwa <input id="p3NuwaCloudSize" type="number" min="0.5" max="100" step="0.5" value="5" style="width:55px"> mm</label>
<label>size D435i <input id="p3D435CloudSize" type="number" min="0.5" max="100" step="0.5" value="5" style="width:55px"> mm</label>
</div>
<div class="p3-row">
<button id="p3CloudAcquireBtn">Acquisisci</button>
<button id="p3CloudClearBtn">✕</button>
</div>
<div class="marker-info" id="p3CloudStatus">—</div>
</div>
</div><!-- /p3-group cameras -->
<div class="p3-group" data-group="misc" hidden>
<div class="p3-section">
<h3>Tag locator</h3>
<div class="p3-row">
<label><input type="checkbox" id="p3TagPlaceMode"> <b>Modalità tag click</b> (click sul modello = piazza crocetta)</label>
</div>
<div class="p3-row">
<label>Tag
<select id="p3TagSelect">
<option value="10:40:wrist_sx">wrist sx (id=10, 40mm)</option>
<option value="1:20:nuwa_dx">nuwa dx (id=1, 20mm)</option>
</select>
</label>
</div>
<div class="p3-row">
<label>x <input id="p3TagPosX" type="number" readonly style="width:70px"> mm</label>
<label>y <input id="p3TagPosY" type="number" readonly style="width:70px"> mm</label>
<label>z <input id="p3TagPosZ" type="number" readonly style="width:70px"> mm</label>
</div>
<div class="p3-row">
<button id="p3TagSaveBtn" disabled>Salva</button>
<button id="p3TagClearBtn" disabled>✕ rimuovi</button>
</div>
<div class="marker-info" id="p3TagStatus">attiva la modalità tag click sopra, poi clicca sul modello 3D</div>
</div>
<div class="p3-section">
<h3>Debug</h3>
<div class="p3-row">
<label><input type="checkbox" id="p3ShowJointUCS"> Mostra UCS giunti</label>
</div>
<div class="p3-row">
<label>Scala
<input id="p3JointUCSSize" type="number" min="10" max="200" step="5" value="40" style="width:60px"> mm
</label>
</div>
<div class="p3-row">
<label><input type="checkbox" id="p3ShowD435CalibUCS" checked> UCS D435i hand-eye (arancio)</label>
</div>
<div class="p3-row">
<label><input type="checkbox" id="p3ShowRobotBody" checked> Mostra robot (mesh)</label>
</div>
</div>
<div class="p3-section">
<h3>Tags</h3>
<div class="p3-row">
<label><input type="checkbox" id="p3ShowFrontTags"> Mostra tag visibili (quad + dim + dist)</label>
</div>
<div class="p3-row">
<label>IDs <input id="p3FrontTagIds" type="text" value="0-49" style="width:90px"></label>
</div>
<div class="marker-info" id="p3FrontTagStatus">—</div>
</div>
<div class="p3-section"> <div class="p3-section">
<button id="p3UnlockBtn">Sblocca braccia</button> <button id="p3UnlockBtn">Sblocca braccia</button>
</div> </div>
@@ -534,7 +535,13 @@
<div class="p3-section" id="p3ErrorBox" style="display:none"> <div class="p3-section" id="p3ErrorBox" style="display:none">
<div class="p3-error" id="p3ErrorText"></div> <div class="p3-error" id="p3ErrorText"></div>
</div> </div>
</div><!-- /p3-group misc -->
</aside> </aside>
<nav class="p3-tabrail" aria-label="Pose 3D groups">
<button type="button" class="p3-tab active" data-target="pose">POSE</button>
<button type="button" class="p3-tab" data-target="cameras">CAMERAS</button>
<button type="button" class="p3-tab" data-target="misc">MISC</button>
</nav>
</div> </div>
</section> </section>
+66 -2
View File
@@ -1,11 +1,40 @@
#tab-pose-3d .p3-layout { #tab-pose-3d .p3-layout {
display: grid; display: grid;
grid-template-columns: 1fr 360px; grid-template-columns: 1fr 360px 92px;
gap: 16px; gap: 16px;
height: calc(100vh - 140px); height: calc(100vh - 140px);
min-height: 600px; min-height: 600px;
} }
#tab-pose-3d .p3-tabrail {
display: flex;
flex-direction: column;
gap: 8px;
align-items: stretch;
}
#tab-pose-3d .p3-tab {
background: #1a1f26;
border: 1px solid #2a2f36;
color: #aab;
padding: 14px 8px;
border-radius: 6px;
font-size: 12px;
font-weight: bold;
letter-spacing: 1px;
cursor: pointer;
writing-mode: horizontal-tb;
text-align: center;
}
#tab-pose-3d .p3-tab:hover { background: #232a33; color: #e0e0e0; }
#tab-pose-3d .p3-tab.active {
background: #1f3556;
border-color: #4da3ff;
color: #4da3ff;
}
#tab-pose-3d .p3-group { display: contents; }
#tab-pose-3d .p3-group[hidden] { display: none; }
#tab-pose-3d .p3-viewport { #tab-pose-3d .p3-viewport {
position: relative; position: relative;
background: #0d1014; background: #0d1014;
@@ -80,13 +109,48 @@
color: #ddd; color: #ddd;
margin-bottom: 8px; margin-bottom: 8px;
} }
#tab-pose-3d .p3-readout span { color: #4da3ff; } #tab-pose-3d .p3-readout label {
display: flex;
align-items: center;
gap: 4px;
}
/* Label colorate come gli assi UCS: X/Y/Z = rosso/verde/blu (riga 1),
R/P/W = stessa tonalita' desaturata (riga 2, rotazione attorno a X/Y/Z). */
#tab-pose-3d .p3-readout label:nth-child(1) { color: #e74c3c; }
#tab-pose-3d .p3-readout label:nth-child(2) { color: #3ecf8e; }
#tab-pose-3d .p3-readout label:nth-child(3) { color: #4da3ff; }
#tab-pose-3d .p3-readout label:nth-child(4) { color: #c8857e; }
#tab-pose-3d .p3-readout label:nth-child(5) { color: #88c2a8; }
#tab-pose-3d .p3-readout label:nth-child(6) { color: #8aaedf; }
#tab-pose-3d .p3-readout input {
width: 60px;
background: #0d1014;
border: 1px solid #2a2f36;
color: #4da3ff;
padding: 2px 4px;
border-radius: 3px;
font-size: 12px;
font-family: monospace;
}
#tab-pose-3d .p3-readout input:focus {
outline: 1px solid #4da3ff;
border-color: #4da3ff;
}
#tab-pose-3d .p3-status { font-size: 12px; color: #888; margin-bottom: 8px; } #tab-pose-3d .p3-status { font-size: 12px; color: #888; margin-bottom: 8px; }
#tab-pose-3d .p3-status.modified { color: #f0ad4e; } #tab-pose-3d .p3-status.modified { color: #f0ad4e; }
#tab-pose-3d .p3-status.ok { color: #3ecf8e; } #tab-pose-3d .p3-status.ok { color: #3ecf8e; }
#tab-pose-3d .p3-status.error { color: #e74c3c; } #tab-pose-3d .p3-status.error { color: #e74c3c; }
#tab-pose-3d .p3-feasibility {
font-size: 12px;
margin-bottom: 6px;
font-family: monospace;
min-height: 1em;
}
#tab-pose-3d .p3-feasibility.ok { color: #3ecf8e; }
#tab-pose-3d .p3-feasibility.error { color: #e74c3c; }
#tab-pose-3d button { #tab-pose-3d button {
background: #2a3038; background: #2a3038;
border: 1px solid #3a4048; border: 1px solid #3a4048;
+1705 -61
View File
File diff suppressed because it is too large. Load diff
-31
View File
@@ -704,37 +704,6 @@ details.joint-temps summary { cursor: pointer; font-weight: 600; font-size: 12px
.device-pill.err { background: rgba(231,76,60,0.15); color: var(--danger); border: 1px solid var(--danger); } .device-pill.err { background: rgba(231,76,60,0.15); color: var(--danger); border: 1px solid var(--danger); }
.active-robot-pill { font-size: 12px; padding: 4px 10px; border-radius: 999px; background: rgba(77,163,255,0.15); color: var(--accent); border: 1px solid var(--accent); margin-left: 12px; } .active-robot-pill { font-size: 12px; padding: 4px 10px; border-radius: 999px; background: rgba(77,163,255,0.15); color: var(--accent); border: 1px solid var(--accent); margin-left: 12px; }
/* ---------- Assistente AI tab ---------- */
.agent-layout > div { display: flex; flex-direction: column; }
.agent-chat {
border: 1px solid var(--border);
border-radius: 4px;
padding: 10px;
height: 420px;
overflow-y: auto;
background: rgba(0,0,0,0.15);
display: flex;
flex-direction: column;
gap: 8px;
}
.agent-msg { padding: 8px 12px; border-radius: 6px; max-width: 90%; line-height: 1.4; font-size: 13px; white-space: pre-wrap; word-wrap: break-word; }
.agent-msg.user { align-self: flex-end; background: rgba(77,163,255,0.15); border: 1px solid var(--accent); }
.agent-msg.assistant { align-self: flex-start; background: rgba(255,255,255,0.04); border: 1px solid var(--border); }
.agent-msg.system { align-self: stretch; background: rgba(240,173,78,0.08); border: 1px dashed var(--warn); color: var(--dim); font-style: italic; }
.agent-msg.error { align-self: stretch; background: rgba(231,76,60,0.10); border: 1px solid var(--danger); color: var(--danger); }
.agent-msg .meta { font-size: 10px; color: var(--dim); margin-top: 4px; }
.agent-tool-pill { display: inline-block; padding: 1px 8px; margin: 2px 2px 0 0; font-size: 11px; border-radius: 999px; background: rgba(62,207,142,0.12); border: 1px solid var(--ok); color: var(--ok); }
.agent-tool-pill.err { background: rgba(231,76,60,0.12); border-color: var(--danger); color: var(--danger); }
.agent-input-row { display: flex; gap: 8px; align-items: stretch; margin-top: 8px; }
.agent-input-row textarea { flex: 1; resize: vertical; font-family: ui-monospace, monospace; font-size: 13px; padding: 8px; background: #0e1116; color: inherit; border: 1px solid var(--border); border-radius: 4px; }
.agent-input-row button { white-space: nowrap; }
.agent-task-list { list-style: none; padding: 0; margin: 8px 0; max-height: 260px; overflow-y: auto; }
.agent-task-list li { display: flex; gap: 6px; align-items: center; padding: 6px 8px; border-bottom: 1px solid var(--border); font-size: 12px; }
.agent-task-list li:hover { background: rgba(255,255,255,0.03); }
.agent-task-list .task-name { font-weight: 600; flex: 0 0 auto; }
.agent-task-list .task-desc { color: var(--dim); flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.agent-task-list button.link-btn { font-size: 11px; padding: 2px 6px; }
/* ----- Tab Braccio DX ----- */ /* ----- Tab Braccio DX ----- */
.ra-banner { .ra-banner {
padding: 8px 12px; padding: 8px 12px;
+1476
View File
File diff suppressed because it is too large. Load diff
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
View File
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+469
View File
@@ -0,0 +1,469 @@
import { Euler, Object3D, Vector3, Quaternion, Matrix4 } from 'three';
const _tempAxis = new Vector3();
const _tempEuler = new Euler();
const _tempTransform = new Matrix4();
const _tempOrigTransform = new Matrix4();
const _tempQuat = new Quaternion();
const _tempScale = new Vector3(1.0, 1.0, 1.0);
const _tempPosition = new Vector3();
class URDFBase extends Object3D {
constructor(...args) {
super(...args);
this.urdfNode = null;
this.urdfName = '';
}
copy(source, recursive) {
super.copy(source, recursive);
this.urdfNode = source.urdfNode;
this.urdfName = source.urdfName;
return this;
}
}
class URDFCollider extends URDFBase {
constructor(...args) {
super(...args);
this.isURDFCollider = true;
this.type = 'URDFCollider';
}
}
class URDFVisual extends URDFBase {
constructor(...args) {
super(...args);
this.isURDFVisual = true;
this.type = 'URDFVisual';
}
}
class URDFLink extends URDFBase {
constructor(...args) {
super(...args);
this.isURDFLink = true;
this.type = 'URDFLink';
}
}
class URDFJoint extends URDFBase {
get jointType() {
return this._jointType;
}
set jointType(v) {
if (this.jointType === v) return;
this._jointType = v;
this.matrixWorldNeedsUpdate = true;
switch (v) {
case 'fixed':
this.jointValue = [];
break;
case 'continuous':
case 'revolute':
case 'prismatic':
this.jointValue = new Array(1).fill(0);
break;
case 'planar':
// Planar joints are, 3dof: position XY and rotation Z.
this.jointValue = new Array(3).fill(0);
this.axis = new Vector3(0, 0, 1);
break;
case 'floating':
this.jointValue = new Array(6).fill(0);
break;
}
}
get angle() {
return this.jointValue[0];
}
constructor(...args) {
super(...args);
this.isURDFJoint = true;
this.type = 'URDFJoint';
this.jointValue = null;
this.jointType = 'fixed';
this.axis = new Vector3(1, 0, 0);
this.limit = { lower: 0, upper: 0 };
this.ignoreLimits = false;
this.origPosition = null;
this.origQuaternion = null;
this.mimicJoints = [];
}
/* Overrides */
copy(source, recursive) {
super.copy(source, recursive);
this.jointType = source.jointType;
this.axis = source.axis.clone();
this.limit.lower = source.limit.lower;
this.limit.upper = source.limit.upper;
this.ignoreLimits = false;
this.jointValue = [...source.jointValue];
this.origPosition = source.origPosition ? source.origPosition.clone() : null;
this.origQuaternion = source.origQuaternion ? source.origQuaternion.clone() : null;
this.mimicJoints = [...source.mimicJoints];
return this;
}
/* Public Functions */
/**
* @param {...number|null} values The joint value components to set, optionally null for no-op
* @returns {boolean} Whether the invocation of this function resulted in an actual change to the joint value
*/
setJointValue(...values) {
// Parse all incoming values into numbers except null, which we treat as a no-op for that value component.
values = values.map(v => v === null ? null : parseFloat(v));
if (!this.origPosition || !this.origQuaternion) {
this.origPosition = this.position.clone();
this.origQuaternion = this.quaternion.clone();
}
let didUpdate = false;
this.mimicJoints.forEach(joint => {
didUpdate = joint.updateFromMimickedJoint(...values) || didUpdate;
});
switch (this.jointType) {
case 'fixed': {
return didUpdate;
}
case 'continuous':
case 'revolute': {
let angle = values[0];
if (angle == null) return didUpdate;
if (angle === this.jointValue[0]) return didUpdate;
if (!this.ignoreLimits && this.jointType === 'revolute') {
angle = Math.min(this.limit.upper, angle);
angle = Math.max(this.limit.lower, angle);
}
this.quaternion
.setFromAxisAngle(this.axis, angle)
.premultiply(this.origQuaternion);
if (this.jointValue[0] !== angle) {
this.jointValue[0] = angle;
this.matrixWorldNeedsUpdate = true;
return true;
} else {
return didUpdate;
}
}
case 'prismatic': {
let pos = values[0];
if (pos == null) return didUpdate;
if (pos === this.jointValue[0]) return didUpdate;
if (!this.ignoreLimits) {
pos = Math.min(this.limit.upper, pos);
pos = Math.max(this.limit.lower, pos);
}
this.position.copy(this.origPosition);
_tempAxis.copy(this.axis).applyEuler(this.rotation);
this.position.addScaledVector(_tempAxis, pos);
if (this.jointValue[0] !== pos) {
this.jointValue[0] = pos;
this.matrixWorldNeedsUpdate = true;
return true;
} else {
return didUpdate;
}
}
case 'floating': {
// no-op if all values are identical to existing value or are null
if (this.jointValue.every((value, index) => values[index] === value || values[index] === null)) return didUpdate;
// Floating joints have six degrees of freedom: X, Y, Z, R, P, Y.
this.jointValue[0] = values[0] !== null ? values[0] : this.jointValue[0];
this.jointValue[1] = values[1] !== null ? values[1] : this.jointValue[1];
this.jointValue[2] = values[2] !== null ? values[2] : this.jointValue[2];
this.jointValue[3] = values[3] !== null ? values[3] : this.jointValue[3];
this.jointValue[4] = values[4] !== null ? values[4] : this.jointValue[4];
this.jointValue[5] = values[5] !== null ? values[5] : this.jointValue[5];
// Compose transform of joint origin and transform due to joint values
_tempOrigTransform.compose(this.origPosition, this.origQuaternion, _tempScale);
_tempQuat.setFromEuler(
_tempEuler.set(
this.jointValue[3],
this.jointValue[4],
this.jointValue[5],
'XYZ',
),
);
_tempPosition.set(this.jointValue[0], this.jointValue[1], this.jointValue[2]);
_tempTransform.compose(_tempPosition, _tempQuat, _tempScale);
// Calcualte new transform
_tempOrigTransform.premultiply(_tempTransform);
this.position.setFromMatrixPosition(_tempOrigTransform);
this.rotation.setFromRotationMatrix(_tempOrigTransform);
this.matrixWorldNeedsUpdate = true;
return true;
}
case 'planar': {
// no-op if all values are identical to existing value or are null
if (this.jointValue.every((value, index) => values[index] === value || values[index] === null)) return didUpdate;
this.jointValue[0] = values[0] !== null ? values[0] : this.jointValue[0];
this.jointValue[1] = values[1] !== null ? values[1] : this.jointValue[1];
this.jointValue[2] = values[2] !== null ? values[2] : this.jointValue[2];
// Compose transform of joint origin and transform due to joint values
_tempOrigTransform.compose(this.origPosition, this.origQuaternion, _tempScale);
_tempQuat.setFromAxisAngle(this.axis, this.jointValue[2]);
_tempPosition.set(this.jointValue[0], this.jointValue[1], 0.0);
_tempTransform.compose(_tempPosition, _tempQuat, _tempScale);
// Calculate new transform
_tempOrigTransform.premultiply(_tempTransform);
this.position.setFromMatrixPosition(_tempOrigTransform);
this.rotation.setFromRotationMatrix(_tempOrigTransform);
this.matrixWorldNeedsUpdate = true;
return true;
}
}
return didUpdate;
}
}
class URDFMimicJoint extends URDFJoint {
constructor(...args) {
super(...args);
this.type = 'URDFMimicJoint';
this.mimicJoint = null;
this.offset = 0;
this.multiplier = 1;
}
updateFromMimickedJoint(...values) {
const modifiedValues = values.map(x => x * this.multiplier + this.offset);
return super.setJointValue(...modifiedValues);
}
/* Overrides */
copy(source, recursive) {
super.copy(source, recursive);
this.mimicJoint = source.mimicJoint;
this.offset = source.offset;
this.multiplier = source.multiplier;
return this;
}
}
class URDFRobot extends URDFLink {
constructor(...args) {
super(...args);
this.isURDFRobot = true;
this.urdfNode = null;
this.urdfRobotNode = null;
this.robotName = null;
this.links = null;
this.joints = null;
this.colliders = null;
this.visual = null;
this.frames = null;
}
copy(source, recursive) {
super.copy(source, recursive);
this.urdfRobotNode = source.urdfRobotNode;
this.robotName = source.robotName;
this.links = {};
this.joints = {};
this.colliders = {};
this.visual = {};
this.traverse(c => {
if (c.isURDFJoint && c.urdfName in source.joints) {
this.joints[c.urdfName] = c;
}
if (c.isURDFLink && c.urdfName in source.links) {
this.links[c.urdfName] = c;
}
if (c.isURDFCollider && c.urdfName in source.colliders) {
this.colliders[c.urdfName] = c;
}
if (c.isURDFVisual && c.urdfName in source.visual) {
this.visual[c.urdfName] = c;
}
});
// Repair mimic joint references once we've re-accumulated all our joint data
for (const joint in this.joints) {
this.joints[joint].mimicJoints = this.joints[joint].mimicJoints.map((mimicJoint) => this.joints[mimicJoint.name]);
}
this.frames = {
...this.colliders,
...this.visual,
...this.links,
...this.joints,
};
return this;
}
getFrame(name) {
return this.frames[name];
}
setJointValue(jointName, ...angle) {
const joint = this.joints[jointName];
if (joint) {
return joint.setJointValue(...angle);
}
return false;
}
setJointValues(values) {
let didChange = false;
for (const name in values) {
const value = values[name];
if (Array.isArray(value)) {
didChange = this.setJointValue(name, ...value) || didChange;
} else {
didChange = this.setJointValue(name, value) || didChange;
}
}
return didChange;
}
}
export { URDFRobot, URDFLink, URDFJoint, URDFMimicJoint, URDFVisual, URDFCollider };
Loaded 100 of 101 files, more files were not shown because too many files have changed in this diff. Show more