feat: initial commit — orchestratore presa-mano con AI assistant + dual-robot
Pipeline completa per calibrazione hand-eye e grasp planning su robot Unitree G1/H1 con camera depth Nuwa-HP60C. Esposta come FastAPI + GUI vanilla JS, con assistente AI (OpenRouter + tool calling curato) per pilotare il sistema in linguaggio naturale. Architettura: - FastAPI orchestrator (porta da .env, default 8890) che coordina 4 microservizi upstream: Nuwa-HP60C, marker-finder, YOLO, robot bridge. - Dual-robot: services.g1 e services.h1 in config.yaml; switchabili a runtime via PUT /api/home/robot. - Calibrazione hand-eye eye-to-hand split per-side (right/left). - Marker pose: marker-finder riceve gli intrinseci RGB della Nuwa al boot, PnP server-side con quei dati. Nessun re-PnP locale richiesto. - Depth-to-RGB: registrazione iterativa IR↔RGB per back-projection 3D corretta (Nuwa /api/measure usa coords IR; il nostro /api/perception/ measure_rgb prende pixel RGB e ritorna 3D nel frame RGB OpenCV). - Correzione affine sul depth bias (calibrabile via marker come ground-truth) — riduce errore Z da ~6% a ~1.5% nei test. - AI assistant: 16 tool curati (move_wrist_*, hand_action, get_marker[_3d], detect_object, calibration_*, set_active_robot…), system prompt dinamico che riflette robot attivo + marker config + addendum customizzabile da GUI. Temperature di default 0.1 per evitare invenzioni. GUI (7 tab): - Home: stato comunicazione device + selettore robot attivo G1/H1 - Assistente AI: chat con OpenRouter, history persistita lato browser, task riutilizzabili (prompt salvati), trace tool calls - Impostazioni AI: modello da combo curata, temperature, max iterations, prompt addendum, anteprima system prompt - Calibrazione, Auto-Calibrazione, TrovaUtensile, Servo Visuale, Vista 3D, Movimento EE — invariate dalle versioni precedenti Setup: - Variabili in presa-mano/.env (template in .env.example) - OPENROUTER_API_KEY richiesta solo per il tab Assistente AI - PRESA_MANO_PORT configurabile Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
ac573505b7
121 files changed
+67199
No files matched your search
@@ -0,0 +1,136 @@
|
||||
"""Per-installation depth-bias correction for the Nuwa-HP60C.
|
||||
|
||||
The Nuwa depth sensor exhibits an offset on `depth_mm` that doesn't fit a
|
||||
simple multiplicative scale (see /api/perception/measure_rgb experiments
|
||||
on a known marker — bias varies with distance). We model the bias as an
|
||||
affine map:
|
||||
|
||||
depth_raw = scale * depth_true + offset_mm (sensor)
|
||||
depth_true = (depth_raw - offset_mm) / scale (correction)
|
||||
|
||||
and fit (scale, offset_mm) from N≥2 samples where `depth_true` is taken
|
||||
from a trusted source (typically a PnP-derived Z on a known-size ArUco
|
||||
marker held at varying distances).
|
||||
|
||||
State (params + samples) lives in a single JSON file so it can be
|
||||
inspected/edited by hand:
|
||||
|
||||
data/depth_correction.json
|
||||
{
|
||||
"enabled": true,
|
||||
"scale": 0.871,
|
||||
"offset_mm": 185.0,
|
||||
"fit_meta": {"n_samples": 4, "rms_residual_mm": 2.1, ...},
|
||||
"samples": [
|
||||
{"depth_raw_mm": 1030, "depth_true_mm": 970, "note": "...", "added_at": ...},
|
||||
...
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def default_state() -> dict[str, Any]:
|
||||
return {
|
||||
"enabled": False,
|
||||
"scale": 1.0,
|
||||
"offset_mm": 0.0,
|
||||
"fit_meta": None,
|
||||
"samples": [],
|
||||
}
|
||||
|
||||
|
||||
def load(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return default_state()
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
return default_state()
|
||||
# Fill in any missing keys with defaults.
|
||||
merged = default_state()
|
||||
merged.update(data)
|
||||
return merged
|
||||
except json.JSONDecodeError:
|
||||
return default_state()
|
||||
|
||||
|
||||
def save(path: Path, state: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def apply_correction(depth_raw_mm: int, state: dict[str, Any] | None) -> int:
|
||||
"""Convert a raw sensor depth (mm) to the corrected depth (mm). Returns
|
||||
the input unchanged if correction is disabled or state is None."""
|
||||
if not state or not state.get("enabled") or depth_raw_mm <= 0:
|
||||
return depth_raw_mm
|
||||
scale = float(state.get("scale") or 1.0)
|
||||
offset = float(state.get("offset_mm") or 0.0)
|
||||
if scale <= 0:
|
||||
return depth_raw_mm
|
||||
corrected = (depth_raw_mm - offset) / scale
|
||||
return max(0, int(round(corrected)))
|
||||
|
||||
|
||||
def fit_affine(samples: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Least-squares fit `depth_raw = scale * depth_true + offset_mm`.
|
||||
|
||||
Needs ≥2 samples with non-degenerate range of depth_true. With exactly
|
||||
2 samples the fit is exact and rms_residual_mm = 0. With more samples
|
||||
the residual is meaningful for assessing fit quality.
|
||||
"""
|
||||
if len(samples) < 2:
|
||||
raise ValueError("need at least 2 samples to fit affine model")
|
||||
raw = np.array([float(s["depth_raw_mm"]) for s in samples], dtype=np.float64)
|
||||
true = np.array([float(s["depth_true_mm"]) for s in samples], dtype=np.float64)
|
||||
if true.max() - true.min() < 50:
|
||||
raise ValueError("samples cover too narrow a range of depth_true (<50 mm)")
|
||||
A = np.column_stack([true, np.ones_like(true)])
|
||||
sol, *_ = np.linalg.lstsq(A, raw, rcond=None)
|
||||
scale = float(sol[0])
|
||||
offset = float(sol[1])
|
||||
pred = scale * true + offset
|
||||
residuals = pred - raw
|
||||
rms = float(np.sqrt(np.mean(residuals ** 2)))
|
||||
max_abs = float(np.max(np.abs(residuals)))
|
||||
return {
|
||||
"scale": scale,
|
||||
"offset_mm": offset,
|
||||
"n_samples": len(samples),
|
||||
"rms_residual_mm": rms,
|
||||
"max_abs_residual_mm": max_abs,
|
||||
"depth_true_min_mm": float(true.min()),
|
||||
"depth_true_max_mm": float(true.max()),
|
||||
"fitted_at": time.time(),
|
||||
}
|
||||
|
||||
|
||||
def add_sample(state: dict[str, Any], depth_raw_mm: int, depth_true_mm: int,
|
||||
note: str = "") -> dict[str, Any]:
|
||||
samples = list(state.get("samples") or [])
|
||||
samples.append({
|
||||
"depth_raw_mm": int(depth_raw_mm),
|
||||
"depth_true_mm": int(depth_true_mm),
|
||||
"note": note,
|
||||
"added_at": time.time(),
|
||||
})
|
||||
new_state = dict(state)
|
||||
new_state["samples"] = samples
|
||||
return new_state
|
||||
|
||||
|
||||
def clear_samples(state: dict[str, Any]) -> dict[str, Any]:
|
||||
new_state = dict(state)
|
||||
new_state["samples"] = []
|
||||
return new_state
|
||||
Reference in new issue
Block a user