feat: initial commit — orchestratore presa-mano con AI assistant + dual-robot
Pipeline completa per calibrazione hand-eye e grasp planning su robot Unitree G1/H1 con camera depth Nuwa-HP60C. Esposta come FastAPI + GUI vanilla JS, con assistente AI (OpenRouter + tool calling curato) per pilotare il sistema in linguaggio naturale. Architettura: - FastAPI orchestrator (porta da .env, default 8890) che coordina 4 microservizi upstream: Nuwa-HP60C, marker-finder, YOLO, robot bridge. - Dual-robot: services.g1 e services.h1 in config.yaml; switchabili a runtime via PUT /api/home/robot. - Calibrazione hand-eye eye-to-hand split per-side (right/left). - Marker pose: marker-finder riceve gli intrinseci RGB della Nuwa al boot, PnP server-side con quei dati. Nessun re-PnP locale richiesto. - Depth-to-RGB: registrazione iterativa IR↔RGB per back-projection 3D corretta (Nuwa /api/measure usa coords IR; il nostro /api/perception/ measure_rgb prende pixel RGB e ritorna 3D nel frame RGB OpenCV). - Correzione affine sul depth bias (calibrabile via marker come ground-truth) — riduce errore Z da ~6% a ~1.5% nei test. - AI assistant: 16 tool curati (move_wrist_*, hand_action, get_marker[_3d], detect_object, calibration_*, set_active_robot…), system prompt dinamico che riflette robot attivo + marker config + addendum customizzabile da GUI. Temperature di default 0.1 per evitare invenzioni. GUI (7 tab): - Home: stato comunicazione device + selettore robot attivo G1/H1 - Assistente AI: chat con OpenRouter, history persistita lato browser, task riutilizzabili (prompt salvati), trace tool calls - Impostazioni AI: modello da combo curata, temperature, max iterations, prompt addendum, anteprima system prompt - Calibrazione, Auto-Calibrazione, TrovaUtensile, Servo Visuale, Vista 3D, Movimento EE — invariate dalle versioni precedenti Setup: - Variabili in presa-mano/.env (template in .env.example) - OPENROUTER_API_KEY richiesta solo per il tab Assistente AI - PRESA_MANO_PORT configurabile Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
ac573505b7
121 files changed
+67199
No files matched your search
@@ -0,0 +1,174 @@
|
||||
"""OpenRouter chat runner with multi-turn tool calling.
|
||||
|
||||
OpenRouter speaks the OpenAI chat-completions API verbatim, so we use
|
||||
`httpx` directly and re-implement the tool-call loop. We don't take a hard
|
||||
dep on the openai SDK — the protocol is small enough to inline and this
|
||||
keeps Docker layers slim.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from ..config import get_config
|
||||
from .prompt import build_system_prompt
|
||||
from .tools import TOOL_DEFINITIONS, dispatch_tool
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OpenRouterError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallTrace:
|
||||
name: str
|
||||
arguments: dict[str, Any]
|
||||
result: dict[str, Any]
|
||||
latency_ms: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentResult:
|
||||
reply: str
|
||||
iterations: int
|
||||
tool_calls: list[ToolCallTrace] = field(default_factory=list)
|
||||
finish_reason: str | None = None
|
||||
model: str | None = None
|
||||
usage: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class AgentRunner:
|
||||
"""Stateless: one runner per request. Caller is responsible for keeping
|
||||
the conversation history and passing it back in via `messages`.
|
||||
|
||||
The run loop: send messages → if tool_calls in response, execute each,
|
||||
append the assistant message + tool messages, repeat. Up to
|
||||
`max_tool_iterations` rounds (configurable in chat.openrouter)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
cfg = get_config().get("chat", {}).get("openrouter") or {}
|
||||
self.base_url = str(cfg.get("base_url") or "https://openrouter.ai/api/v1").rstrip("/")
|
||||
self.model = str(cfg.get("model") or "anthropic/claude-sonnet-4.5")
|
||||
self.timeout_s = float(cfg.get("timeout_s") or 60.0)
|
||||
self.max_tool_iterations = int(cfg.get("max_tool_iterations") or 8)
|
||||
# Default low so the model doesn't invent values when summarising
|
||||
# tool results. Override in chat.openrouter.temperature if needed.
|
||||
self.temperature = float(cfg.get("temperature") if cfg.get("temperature") is not None else 0.1)
|
||||
self.referer = cfg.get("referer") or "http://localhost:8890"
|
||||
self.title = cfg.get("title") or "presa-mano"
|
||||
env_var = str(cfg.get("api_key_env") or "OPENROUTER_API_KEY")
|
||||
self.api_key = os.environ.get(env_var, "")
|
||||
self.api_key_env = env_var
|
||||
|
||||
def has_credentials(self) -> bool:
|
||||
return bool(self.api_key)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def run(self, user_message: str,
|
||||
history: list[dict[str, Any]] | None = None,
|
||||
) -> tuple[AgentResult, list[dict[str, Any]]]:
|
||||
"""Returns (result, updated_history). `history` is the full message
|
||||
list as it appears in the OpenAI API (excluding the system prompt;
|
||||
we always inject it ourselves so it reflects current state)."""
|
||||
if not self.has_credentials():
|
||||
raise OpenRouterError(f"OpenRouter API key non impostata (env var {self.api_key_env}).")
|
||||
|
||||
system = {"role": "system", "content": build_system_prompt()}
|
||||
messages: list[dict[str, Any]] = [system]
|
||||
if history:
|
||||
messages.extend(m for m in history if m.get("role") != "system")
|
||||
messages.append({"role": "user", "content": user_message})
|
||||
|
||||
traces: list[ToolCallTrace] = []
|
||||
iterations = 0
|
||||
finish_reason: str | None = None
|
||||
usage: dict[str, Any] | None = None
|
||||
|
||||
while iterations < self.max_tool_iterations:
|
||||
iterations += 1
|
||||
resp = self._chat_completion(messages)
|
||||
usage = resp.get("usage") or usage
|
||||
choices = resp.get("choices") or []
|
||||
if not choices:
|
||||
raise OpenRouterError(f"empty choices in response: {resp}")
|
||||
msg = choices[0].get("message") or {}
|
||||
finish_reason = choices[0].get("finish_reason")
|
||||
messages.append(msg)
|
||||
tool_calls = msg.get("tool_calls") or []
|
||||
if not tool_calls:
|
||||
# Final assistant turn — no further tools requested.
|
||||
reply = msg.get("content") or ""
|
||||
history_out = [m for m in messages if m.get("role") != "system"]
|
||||
return AgentResult(reply=reply, iterations=iterations,
|
||||
tool_calls=traces, finish_reason=finish_reason,
|
||||
model=resp.get("model"), usage=usage), history_out
|
||||
|
||||
# Execute each tool call and append the result back.
|
||||
for call in tool_calls:
|
||||
fn = (call.get("function") or {})
|
||||
name = fn.get("name") or "?"
|
||||
raw_args = fn.get("arguments") or "{}"
|
||||
try:
|
||||
args = json.loads(raw_args) if isinstance(raw_args, str) else (raw_args or {})
|
||||
except json.JSONDecodeError:
|
||||
args = {}
|
||||
t0 = time.perf_counter()
|
||||
result = dispatch_tool(name, args)
|
||||
latency = round((time.perf_counter() - t0) * 1000.0, 1)
|
||||
traces.append(ToolCallTrace(name=name, arguments=args,
|
||||
result=result, latency_ms=latency))
|
||||
messages.append({
|
||||
"role": "tool",
|
||||
"tool_call_id": call.get("id"),
|
||||
"name": name,
|
||||
"content": json.dumps(result, ensure_ascii=False, default=str),
|
||||
})
|
||||
|
||||
# Hit the iteration cap with tool calls still pending — degrade gracefully.
|
||||
history_out = [m for m in messages if m.get("role") != "system"]
|
||||
return AgentResult(
|
||||
reply="Limite di iterazioni raggiunto: rivedi il task o chiedimi di continuare.",
|
||||
iterations=iterations, tool_calls=traces, finish_reason="iteration_cap",
|
||||
model=self.model, usage=usage,
|
||||
), history_out
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _chat_completion(self, messages: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
body = {
|
||||
"model": self.model,
|
||||
"messages": messages,
|
||||
"tools": TOOL_DEFINITIONS,
|
||||
"tool_choice": "auto",
|
||||
"temperature": self.temperature,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
# OpenRouter analytics — optional but recommended.
|
||||
"HTTP-Referer": self.referer,
|
||||
"X-Title": self.title,
|
||||
}
|
||||
try:
|
||||
with httpx.Client(timeout=self.timeout_s) as c:
|
||||
r = c.post(url, json=body, headers=headers)
|
||||
except httpx.HTTPError as e:
|
||||
raise OpenRouterError(f"OpenRouter unreachable: {e}") from e
|
||||
if r.status_code >= 400:
|
||||
try:
|
||||
detail = r.json()
|
||||
except Exception:
|
||||
detail = r.text
|
||||
raise OpenRouterError(f"OpenRouter HTTP {r.status_code}: {detail}")
|
||||
return r.json()
|
||||
Reference in new issue
Block a user