From 541f13455016063631c0c30f3e20a2c319f067ec Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Tue, 26 May 2026 10:46:53 -0600 Subject: [PATCH 01/35] feat: PACK MIND conductor + dashboard skeleton Shared semantic memory for a Go2 team. Conductor (roster + append-only blackboard + deterministic mission state machine + movement lock) talks to each dog over MCP JSON-RPC; cross-dog handoff is by zone name only, no coordinates. Browser dashboard renders the causal chain. --mock runs the full Alpha->Bravo story with no hardware. dimos/experimental/pack_mind/ --- dimos/experimental/pack_mind/README.md | 63 +++ dimos/experimental/pack_mind/__init__.py | 15 + dimos/experimental/pack_mind/conductor.py | 479 ++++++++++++++++++ dimos/experimental/pack_mind/dashboard.html | 123 +++++ dimos/experimental/pack_mind/static/style.css | 130 +++++ 5 files changed, 810 insertions(+) create mode 100644 dimos/experimental/pack_mind/README.md create mode 100644 dimos/experimental/pack_mind/__init__.py create mode 100644 dimos/experimental/pack_mind/conductor.py create mode 100644 dimos/experimental/pack_mind/dashboard.html create mode 100644 dimos/experimental/pack_mind/static/style.css diff --git a/dimos/experimental/pack_mind/README.md b/dimos/experimental/pack_mind/README.md new file mode 100644 index 0000000000..fda3121fb1 --- /dev/null +++ b/dimos/experimental/pack_mind/README.md @@ -0,0 +1,63 @@ +# PACK MIND + +A shared semantic memory for a team of Unitree Go2s. +**Alpha sees. Bravo remembers. The pack acts.** + +Most multi-robot systems share a *map*. Pack Mind shares *meaning*: one dog finds +something, another dog answers from that shared memory and acts on it — without any +shared map, SLAM, or coordinate exchange. Cross-dog handoff is by **zone name** only. + +## What this is + +- **`conductor.py`** — the only shared layer. Roster + append-only event blackboard + + deterministic mission state machine + movement lock. Talks to each dog over MCP + JSON-RPC (HTTP). Serves the dashboard and a small action API. +- **`dashboard.html` / `static/style.css`** — the projector surface that makes the + shared memory visible: causal event cards, roster, mission state, operator controls. + +Each dog runs its own standard `unitree-go2-agentic` stack. The conductor is net-new. + +## Run it (no hardware) + +```bash +uv run python dimos/experimental/pack_mind/conductor.py --mock +# open http://localhost:8080 +``` + +Drive the whole Alpha → Bravo story from the dashboard buttons. `--mock` logs MCP +calls instead of making them, so the full demo runs on a laptop. + +## Run it (real dogs) + +Each dog, on its own compute box: + +```bash +dimos run unitree-go2-agentic --robot-ip --listen-host 0.0.0.0 \ + --mcp-port 9990 --viewer none --daemon +``` + +Then the conductor, from the control machine: + +```bash +uv run python dimos/experimental/pack_mind/conductor.py \ + --dog alpha= --dog bravo= +``` + +`--listen-host 0.0.0.0` on each dog is required — the MCP server binds `127.0.0.1` +by default and the conductor would get connection-refused otherwise. + +## Demo flow (the 6 controls) + +1. **Start Act 1** — Alpha begins scouting. +2. **Inject: Alpha found red backpack @ Zone B** — operator fallback, identical + blackboard effect to a real VLM detection. +3. **Ask Bravo where backpack is** — Bravo answers from shared memory, not its own. +4. **Send Bravo to Zone B** — Bravo navigates to its *own* `zone_b` (movement lock held). +5. **Verify success** — Bravo self-checks at the zone. +6. **Emergency stop** — release locks, all dogs hold. + +Hidden fallback buttons are not cheating. Broken live autonomy is. + +> NOTE: MCP tool argument names (`navigate_with_text`, `speak`, `look_out_for`) are +> passed through verbatim. Verify against the skill signatures in `dimos/agents/skills/` +> on the first real-hardware test. diff --git a/dimos/experimental/pack_mind/__init__.py b/dimos/experimental/pack_mind/__init__.py new file mode 100644 index 0000000000..ad1a54cf7e --- /dev/null +++ b/dimos/experimental/pack_mind/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — a shared semantic memory for a team of Unitree Go2s.""" diff --git a/dimos/experimental/pack_mind/conductor.py b/dimos/experimental/pack_mind/conductor.py new file mode 100644 index 0000000000..fd15e46932 --- /dev/null +++ b/dimos/experimental/pack_mind/conductor.py @@ -0,0 +1,479 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND conductor — a shared semantic memory for a team of Unitree Go2s. + +The conductor is the ONLY shared layer. Each dog runs its own full +``unitree-go2-agentic`` stack (own map, own memory) and exposes an MCP HTTP +endpoint. The conductor holds the roster, an append-only event blackboard, the +mission state machine, and a movement lock. It talks to each dog over MCP +JSON-RPC. It NEVER exchanges coordinates between dogs — cross-dog handoff is by +zone NAME only. + +Run without hardware (drives the whole story from the dashboard buttons):: + + uv run python dimos/experimental/pack_mind/conductor.py --mock + +Run against real dogs:: + + uv run python dimos/experimental/pack_mind/conductor.py \\ + --dog alpha=10.0.0.10 --dog bravo=10.0.0.11 --dog charlie=10.0.0.12 + +Then open http://localhost:8080 for the dashboard. + +NOTE: MCP tool argument names (``navigate_with_text``, ``look_out_for``, +``speak``) are passed through verbatim. Verify them against the skill +signatures in dimos/agents/skills/ on the first real-hardware test (G0). +""" + +from __future__ import annotations + +import argparse +import json +import threading +import uuid +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Literal, cast + +import requests + +DogId = Literal["alpha", "bravo", "charlie"] +ZoneId = Literal["zone_a", "zone_b", "zone_c"] +MissionState = Literal[ + "IDLE", + "SCOUT_ALPHA", + "FOUND_EVENT_RECORDED", + "QUERY_BRAVO", + "BRAVO_NAVIGATING", + "VERIFYING", + "DONE", +] + +# MCP call timeouts in seconds (plan section 5). Never block the dashboard on these. +TIMEOUT_SPEAK = 8.0 +TIMEOUT_LIGHT = 15.0 +TIMEOUT_NAV = 90.0 + +_HERE = Path(__file__).parent + + +@dataclass +class Dog: + """A pack member. ``zone`` is the named zone it was last sent to — never a coordinate.""" + + id: DogId + name: str + mcp_url: str + role: str + status: str = "idle" + zone: ZoneId | None = None + + +@dataclass(frozen=True) +class MemoryEvent: + """An immutable blackboard entry. The shared memory is an append-only list of these. + + There is deliberately no ``x``/``y``/``pose`` field. If you add one, you are + building the wrong project. + """ + + id: str + ts: str + robot: DogId + type: str + text: str + object: str | None = None + zone: ZoneId | None = None + source: str = "system" + confidence: float | None = None + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _new_id() -> str: + return f"evt-{uuid.uuid4().hex[:8]}" + + +class Conductor: + """Holds all shared state and orchestrates the deterministic mission state machine.""" + + def __init__(self, dogs: list[Dog], mock: bool) -> None: + self._dogs: dict[DogId, Dog] = {d.id: d for d in dogs} + self._events: list[MemoryEvent] = [] + self._mission: MissionState = "IDLE" + self._mock = mock + self._state_lock = threading.Lock() # guards roster + blackboard + mission + self._move_lock = threading.Lock() # only one dog moves at a time + self._acting: DogId | None = None + + # -- shared memory ------------------------------------------------------- + + def append_event( + self, + robot: DogId, + type_: str, + text: str, + *, + object_: str | None = None, + zone: ZoneId | None = None, + source: str = "system", + confidence: float | None = None, + ) -> MemoryEvent: + event = MemoryEvent( + id=_new_id(), + ts=_now(), + robot=robot, + type=type_, + text=text, + object=object_, + zone=zone, + source=source, + confidence=confidence, + ) + with self._state_lock: + self._events.append(event) + return event + + def _latest_zone_for(self, object_: str) -> ZoneId | None: + """The whole semantic-memory trick: any dog can answer from the blackboard.""" + with self._state_lock: + for event in reversed(self._events): + if event.type == "object_found" and event.object == object_ and event.zone: + return event.zone + return None + + def _set_mission(self, state: MissionState) -> None: + with self._state_lock: + self._mission = state + + # -- MCP transport ------------------------------------------------------- + + def call_tool( + self, dog_id: DogId, tool_name: str, args: dict[str, Any], timeout: float + ) -> str | None: + """Call one MCP tool on one dog. Returns text content, or None on failure. + + On timeout/error, appends a ``tool_timeout`` event and returns None so the + caller can fall back to a scripted beat. Never raises into the request thread. + """ + dog = self._dogs.get(dog_id) + if dog is None: + return None + + if self._mock: + self.append_event( + dog_id, "mock_call", f"[mock] {tool_name}({args})", source="mock" + ) + return f"[mock {dog_id}] {tool_name} ok" + + payload = { + "jsonrpc": "2.0", + "id": _new_id(), + "method": "tools/call", + "params": {"name": tool_name, "arguments": args}, + } + try: + resp = requests.post(dog.mcp_url, json=payload, timeout=timeout) + resp.raise_for_status() + data: Any = resp.json() + return _extract_text(data) + except requests.RequestException as exc: + self.append_event( + dog_id, "tool_timeout", f"{tool_name} failed: {exc}", source="system" + ) + return None + + def _speak(self, dog_id: DogId, text: str) -> None: + self.call_tool(dog_id, "speak", {"text": text, "blocking": False}, TIMEOUT_SPEAK) + + # -- mission state machine ---------------------------------------------- + + def start_act1(self) -> None: + self._set_mission("SCOUT_ALPHA") + with self._state_lock: + if "alpha" in self._dogs: + self._dogs["alpha"].status = "scouting" + self.append_event("alpha", "mission", "Alpha scouting for the red backpack.") + + def inject_found(self, dog_id: DogId, object_: str, zone: ZoneId) -> MemoryEvent: + """Operator fallback == real detection: identical blackboard effect.""" + event = self.append_event( + dog_id, + "object_found", + f"{dog_id.title()} found {object_} in {zone}.", + object_=object_, + zone=zone, + source="operator", + confidence=0.92, + ) + with self._state_lock: + if dog_id in self._dogs: + self._dogs[dog_id].status = "found_target" + self._set_mission("FOUND_EVENT_RECORDED") + self._run_async(lambda: self._speak(dog_id, f"Found {object_} in {zone}.")) + return event + + def ask_where(self, dog_id: DogId, object_: str) -> str: + """The winning moment: a dog answers from SHARED memory, not its own.""" + self._set_mission("QUERY_BRAVO") + zone = self._latest_zone_for(object_) + if zone is None: + answer = f"I have no pack memory of the {object_} yet." + else: + finder = self._finder_of(object_) + who = finder.title() if finder else "A teammate" + answer = f"{who} found it in {zone}. Follow me." + self.append_event(dog_id, "answer", answer, object_=object_, zone=zone) + self._run_async(lambda: self._speak(dog_id, answer)) + return answer + + def send_dog_to_memory(self, dog_id: DogId, object_: str) -> bool: + """Acquire the movement lock and navigate by zone NAME (the dog's own frame).""" + zone = self._latest_zone_for(object_) + if zone is None: + self.append_event(dog_id, "blocked", f"No memory of {object_}; cannot navigate.") + return False + if not self._move_lock.acquire(blocking=False): + self.append_event(dog_id, "blocked", "Another dog is moving; movement lock held.") + return False + self._acting = dog_id + self._set_mission("BRAVO_NAVIGATING") + with self._state_lock: + if dog_id in self._dogs: + self._dogs[dog_id].status = "navigating" + self._dogs[dog_id].zone = zone + self.append_event(dog_id, "navigating", f"{dog_id.title()} acting on pack memory -> {zone}.", zone=zone) + + def _go() -> None: + try: + self.call_tool(dog_id, "navigate_with_text", {"query": zone}, TIMEOUT_NAV) + with self._state_lock: + if dog_id in self._dogs: + self._dogs[dog_id].status = "arrived" + self.append_event(dog_id, "arrived", f"{dog_id.title()} reached {zone}.", zone=zone) + finally: + self._acting = None + self._move_lock.release() + + self._run_async(_go) + return True + + def verify_at_zone(self, dog_id: DogId, object_: str) -> None: + self._set_mission("VERIFYING") + zone = self._latest_zone_for(object_) + + def _verify() -> None: + # Primary path uses the VLM lookout; mock/fallback just confirms. + self.call_tool( + dog_id, "look_out_for", {"description_of_things": [object_]}, TIMEOUT_LIGHT + ) + self.append_event( + dog_id, "verified", "Confirmed. Pack memory was correct.", object_=object_, zone=zone + ) + self._run_async(lambda: self._speak(dog_id, "Confirmed. Pack memory was correct.")) + self._set_mission("DONE") + with self._state_lock: + if dog_id in self._dogs: + self._dogs[dog_id].status = "done" + + self._run_async(_verify) + + def emergency_stop(self) -> None: + if self._move_lock.locked(): + try: + self._move_lock.release() + except RuntimeError: + pass + self._acting = None + with self._state_lock: + for dog in self._dogs.values(): + dog.status = "idle" + self._set_mission("IDLE") + self.append_event("alpha", "estop", "Operator emergency stop. All dogs holding.", source="operator") + + # -- helpers ------------------------------------------------------------- + + def _finder_of(self, object_: str) -> DogId | None: + with self._state_lock: + for event in reversed(self._events): + if event.type == "object_found" and event.object == object_: + return event.robot + return None + + def _run_async(self, fn: Any) -> None: + threading.Thread(target=fn, daemon=True).start() + + def snapshot(self) -> dict[str, Any]: + with self._state_lock: + return { + "mission": self._mission, + "acting": self._acting, + "mock": self._mock, + "roster": [asdict(d) for d in self._dogs.values()], + "events": [asdict(e) for e in reversed(self._events)], + } + + +def _extract_text(data: Any) -> str | None: + """Pull text out of an MCP tools/call response, tolerating shape variation.""" + if not isinstance(data, dict): + return None + result = data.get("result", data) + if isinstance(result, dict): + content = result.get("content") + if isinstance(content, list): + parts = [c.get("text", "") for c in content if isinstance(c, dict)] + joined = " ".join(p for p in parts if p) + return joined or None + return None + + +# -- HTTP server ------------------------------------------------------------- + + +class _Server(ThreadingHTTPServer): + conductor: Conductor + + +class _Handler(BaseHTTPRequestHandler): + def log_message(self, fmt: str, *args: Any) -> None: # silence default logging + return + + @property + def _conductor(self) -> Conductor: + return cast(_Server, self.server).conductor + + def _send(self, code: int, body: bytes, content_type: str) -> None: + self.send_response(code) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_file(self, path: Path, content_type: str) -> None: + if not path.is_file(): + self._send(404, b"not found", "text/plain") + return + self._send(200, path.read_bytes(), content_type) + + def do_GET(self) -> None: # noqa: N802 (stdlib API) + if self.path in ("/", "/index.html"): + self._send_file(_HERE / "dashboard.html", "text/html; charset=utf-8") + elif self.path == "/static/style.css": + self._send_file(_HERE / "static" / "style.css", "text/css; charset=utf-8") + elif self.path == "/state": + body = json.dumps(self._conductor.snapshot()).encode() + self._send(200, body, "application/json") + else: + self._send(404, b"not found", "text/plain") + + def do_POST(self) -> None: # noqa: N802 (stdlib API) + if self.path != "/action": + self._send(404, b"not found", "text/plain") + return + length = int(self.headers.get("Content-Length", "0")) + raw = self.rfile.read(length) if length else b"{}" + try: + req: Any = json.loads(raw) + except json.JSONDecodeError: + self._send(400, b'{"ok": false, "error": "bad json"}', "application/json") + return + result = self._dispatch(req if isinstance(req, dict) else {}) + self._send(200, json.dumps(result).encode(), "application/json") + + def _dispatch(self, req: dict[str, Any]) -> dict[str, Any]: + action = req.get("action") + c = self._conductor + obj = cast(str, req.get("object", "red backpack")) + dog = cast(DogId, req.get("dog", "bravo")) + zone = cast(ZoneId, req.get("zone", "zone_b")) + if action == "start_act1": + c.start_act1() + elif action == "inject_found": + c.inject_found(cast(DogId, req.get("dog", "alpha")), obj, zone) + elif action == "ask_where": + return {"ok": True, "answer": c.ask_where(dog, obj)} + elif action == "send_dog": + return {"ok": c.send_dog_to_memory(dog, obj)} + elif action == "verify": + c.verify_at_zone(dog, obj) + elif action == "estop": + c.emergency_stop() + else: + return {"ok": False, "error": f"unknown action: {action}"} + return {"ok": True} + + +def _parse_dog(spec: str) -> Dog: + """Parse ``alpha=10.0.0.10`` or ``alpha=10.0.0.10:9990`` into a Dog.""" + name, _, addr = spec.partition("=") + name = name.strip().lower() + if name not in ("alpha", "bravo", "charlie"): + raise argparse.ArgumentTypeError(f"dog id must be alpha/bravo/charlie, got {name!r}") + host, _, port = addr.partition(":") + port = port or "9990" + roles = {"alpha": "scout", "bravo": "guide", "charlie": "guard"} + return Dog( + id=cast(DogId, name), + name=name.title(), + mcp_url=f"http://{host}:{port}/mcp", + role=roles[name], + ) + + +def _default_dogs() -> list[Dog]: + roles = {"alpha": "scout", "bravo": "guide"} + return [ + Dog(id=cast(DogId, n), name=n.title(), mcp_url=f"http://mock/{n}/mcp", role=r) + for n, r in roles.items() + ] + + +def main() -> None: + parser = argparse.ArgumentParser(description="PACK MIND conductor") + parser.add_argument( + "--dog", + action="append", + type=_parse_dog, + default=[], + metavar="ID=HOST[:PORT]", + help="A dog to control, e.g. alpha=10.0.0.10. Repeatable.", + ) + parser.add_argument("--mock", action="store_true", help="No hardware: log calls instead of HTTP.") + parser.add_argument("--port", type=int, default=8080, help="Dashboard/API port.") + args = parser.parse_args() + + dogs: list[Dog] = list(args.dog) or _default_dogs() + mock = args.mock or not args.dog # no real dogs given -> mock + conductor = Conductor(dogs, mock=mock) + + server = _Server(("0.0.0.0", args.port), _Handler) + server.conductor = conductor + mode = "MOCK (no hardware)" if mock else f"{len(dogs)} dog(s)" + print(f"PACK MIND conductor up on http://localhost:{args.port} [{mode}]") + print(f" roster: {', '.join(f'{d.id}->{d.mcp_url}' for d in dogs)}") + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nshutting down") + server.shutdown() + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/dashboard.html b/dimos/experimental/pack_mind/dashboard.html new file mode 100644 index 0000000000..e2935db364 --- /dev/null +++ b/dimos/experimental/pack_mind/dashboard.html @@ -0,0 +1,123 @@ + + + + + + PACK MIND + + + +
+
PACK MIND
+
Alpha sees. Bravo remembers. The pack acts.
+
IDLE
+
+
+ +
+
+

Pack

+
+ +

Operator

+
+ + + + + + +
+

Hidden fallback buttons are not cheating. Broken live autonomy is.

+
+ +
+

Shared semantic memory

+
semantic handoff — no shared map, no coordinates
+
+
+
+ + + + diff --git a/dimos/experimental/pack_mind/static/style.css b/dimos/experimental/pack_mind/static/style.css new file mode 100644 index 0000000000..901a7d1689 --- /dev/null +++ b/dimos/experimental/pack_mind/static/style.css @@ -0,0 +1,130 @@ +/* PACK MIND dashboard — projector-legible: big type, dark, high contrast. */ +:root { + --bg: #0b0e14; + --panel: #141925; + --line: #232a3a; + --text: #e6ebf5; + --muted: #8b95a8; + --accent: #3ddc97; + --warn: #ffb454; + --danger: #ff5c7c; + --memory: #6aa8ff; +} + +* { box-sizing: border-box; } + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font-family: ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; +} + +header { + padding: 1.2rem 2rem; + border-bottom: 1px solid var(--line); + display: grid; + grid-template-columns: auto 1fr auto auto; + align-items: center; + gap: 1.5rem; +} + +.brand { font-size: 2rem; font-weight: 800; letter-spacing: 0.08em; } +.tagline { color: var(--muted); font-size: 1.05rem; } +.mission { + font-size: 1.3rem; + font-weight: 700; + padding: 0.35rem 1rem; + border: 1px solid var(--accent); + border-radius: 999px; + color: var(--accent); +} +.mode { color: var(--warn); font-weight: 700; font-size: 0.85rem; letter-spacing: 0.1em; } + +main { + display: grid; + grid-template-columns: 360px 1fr; + gap: 1.5rem; + padding: 1.5rem 2rem; +} + +h2 { + font-size: 0.85rem; + text-transform: uppercase; + letter-spacing: 0.12em; + color: var(--muted); + margin: 0.5rem 0 0.75rem; +} + +.roster { display: flex; flex-direction: column; gap: 0.6rem; margin-bottom: 1.5rem; } +.dog { + background: var(--panel); + border: 1px solid var(--line); + border-left: 4px solid var(--muted); + border-radius: 10px; + padding: 0.7rem 0.9rem; +} +.dog-id { font-size: 1.2rem; font-weight: 700; } +.dog-role { color: var(--muted); font-size: 0.85rem; text-transform: uppercase; letter-spacing: 0.08em; } +.dog-status { margin-top: 0.25rem; font-size: 0.95rem; } +.dog.status-scouting { border-left-color: var(--warn); } +.dog.status-found_target { border-left-color: var(--accent); } +.dog.status-navigating { border-left-color: var(--memory); } +.dog.status-arrived, .dog.status-done { border-left-color: var(--accent); } + +.controls { display: flex; flex-direction: column; gap: 0.5rem; } +button { + font: inherit; + font-weight: 600; + text-align: left; + padding: 0.7rem 0.9rem; + border-radius: 10px; + border: 1px solid var(--line); + background: var(--panel); + color: var(--text); + cursor: pointer; + transition: transform 0.08s ease, border-color 0.15s ease; +} +button:hover { border-color: var(--muted); } +button:active { transform: translateY(1px); } +button.accent { border-color: var(--accent); color: var(--accent); } +button.danger { border-color: var(--danger); color: var(--danger); } +.hint { color: var(--muted); font-size: 0.8rem; margin-top: 0.75rem; } + +.handoff-note { + color: var(--memory); + font-size: 0.9rem; + margin-bottom: 1rem; + letter-spacing: 0.04em; +} +.events { display: flex; flex-direction: column; gap: 0.75rem; } +.empty { color: var(--muted); font-style: italic; padding: 2rem 0; } + +.card { + background: var(--panel); + border: 1px solid var(--line); + border-radius: 12px; + padding: 1rem 1.2rem; + animation: rise 0.25s ease; +} +@keyframes rise { from { opacity: 0; transform: translateY(6px); } to { opacity: 1; transform: none; } } +.card-top { + display: flex; + align-items: center; + gap: 0.6rem; + font-size: 1.3rem; + font-weight: 800; + letter-spacing: 0.02em; +} +.robot { text-transform: uppercase; color: var(--text); } +.label { color: var(--muted); } +.card-text { margin-top: 0.4rem; color: var(--muted); font-size: 1rem; } + +.card.type-object_found .label { color: var(--accent); } +.card.type-answer .label, +.card.type-navigating .label { color: var(--memory); } +.card.type-verified .label { color: var(--accent); } +.card.type-tool_timeout { border-color: var(--warn); } +.card.type-tool_timeout .label { color: var(--warn); } +.card.type-estop { border-color: var(--danger); } +.card.type-estop .label { color: var(--danger); } From 9c8b28ae3153d5ac5c8d01fd14ba365053e396fd Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Tue, 26 May 2026 19:37:23 -0600 Subject: [PATCH 02/35] feat: CPU-only MCP harness for PACK MIND G0 validation --- dimos/experimental/pack_mind/sim_harness.py | 52 +++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 dimos/experimental/pack_mind/sim_harness.py diff --git a/dimos/experimental/pack_mind/sim_harness.py b/dimos/experimental/pack_mind/sim_harness.py new file mode 100644 index 0000000000..c56af8dca2 --- /dev/null +++ b/dimos/experimental/pack_mind/sim_harness.py @@ -0,0 +1,52 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Minimal CUDA-free MCP harness for validating the PACK MIND conductor locally. + +The full ``unitree-go2-agentic`` stack needs a CUDA GPU (EdgeTAM perception), so +it cannot run on a CPU-only / Apple Silicon dev machine. PACK MIND's milestones +G0 (MCP reachability) and the conductor wire format do NOT need perception — they +need a live MCP server exposing ``speak``. This harness deploys exactly that: +``SpeakSkill`` behind an ``McpServer``, on CPU. + +Run:: + + uv run python dimos/experimental/pack_mind/sim_harness.py + +Then from another shell:: + + curl -s -X POST localhost:9990/mcp -H 'content-type: application/json' \\ + -d '{"jsonrpc":"2.0","id":"t","method":"tools/list"}' + +or point the conductor at it:: + + uv run python dimos/experimental/pack_mind/conductor.py --dog alpha=localhost:9990 +""" + +from __future__ import annotations + +from dimos.agents.mcp.mcp_server import McpServer +from dimos.agents.skills.speak_skill import SpeakSkill +from dimos.core.coordination.blueprints import autoconnect +from dimos.core.coordination.module_coordinator import ModuleCoordinator + + +def main() -> None: + blueprint = autoconnect(SpeakSkill.blueprint(), McpServer.blueprint()) + coordinator = ModuleCoordinator.build(blueprint) + coordinator.loop() + + +if __name__ == "__main__": + main() From 284888de158853a109be81558bb7013a3a0a6ca1 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Tue, 26 May 2026 19:50:28 -0600 Subject: [PATCH 03/35] test: conductor unit tests (21) + live-Go2 runbook --- dimos/experimental/pack_mind/RUNBOOK.md | 178 ++++++++++++++ .../experimental/pack_mind/test_conductor.py | 232 ++++++++++++++++++ 2 files changed, 410 insertions(+) create mode 100644 dimos/experimental/pack_mind/RUNBOOK.md create mode 100644 dimos/experimental/pack_mind/test_conductor.py diff --git a/dimos/experimental/pack_mind/RUNBOOK.md b/dimos/experimental/pack_mind/RUNBOOK.md new file mode 100644 index 0000000000..3f85c69fa1 --- /dev/null +++ b/dimos/experimental/pack_mind/RUNBOOK.md @@ -0,0 +1,178 @@ +# PACK MIND — Live Go2 Runbook + +> Run this top-to-bottom at the venue. Every command is copy-paste. Do **not** +> debug architecture on-site — that work is done; this is wiring + checks. + +**Pitch:** Most teams make robot dogs share *maps*. Pack Mind makes them share +*meaning*. Alpha sees → the pack remembers → Bravo acts on a teammate's memory, +by zone **name**, never coordinates. + +--- + +## 0. Topology (read once) + +``` +[Go2 alpha] <--robot-ip-- [GPU box A] dimos agentic stack --MCP 0.0.0.0:9990--> \ +[Go2 bravo] <--robot-ip-- [GPU box B] dimos agentic stack --MCP 0.0.0.0:9990--> > [operator laptop] + / conductor.py + dashboard :8080 +``` + +- Each dog needs a **CUDA** companion node (EdgeTAM perception). The agentic + stack runs on the node, not the dog. The dog is reached via `--robot-ip`. +- One GPU box per dog → both use MCP port `9990`. +- One GPU box running **two** stacks → give each a distinct `--mcp-port` + (`9990`, `9991`). +- The conductor runs on the **operator laptop** and talks to each node's MCP + over the LAN. + +### ⚠️ #1 KILLER — bind MCP to the LAN + +DimOS defaults `listen_host = 127.0.0.1`. The MCP server then only answers +localhost and the conductor gets **connection refused** over the LAN. Every +agentic stack MUST be launched with: + +``` +--listen-host 0.0.0.0 +``` + +If a G0 curl is refused, this is the cause 90% of the time. The other 10% is a +firewall on the GPU box. + +--- + +## 1. Per-dog stack bring-up (on each GPU node) + +```bash +# Node A → drives Go2 "alpha" +dimos run unitree-go2-agentic \ + --robot-ip \ + --listen-host 0.0.0.0 \ + --mcp-port 9990 \ + --daemon + +# Node B → drives Go2 "bravo" (or same box, --mcp-port 9991) +dimos run unitree-go2-agentic \ + --robot-ip \ + --listen-host 0.0.0.0 \ + --mcp-port 9990 \ + --daemon +``` + +Verify each node locally before involving the network: + +```bash +dimos status # run id, pid, uptime +dimos mcp status # PID, module list, skill list +dimos mcp list-tools # must include: speak, navigate_with_text, look_out_for, tag_location +``` + +--- + +## 2. G0 (h4) — remote MCP reachability over LAN + +From the **operator laptop**, for every node `:`: + +```bash +curl -s -X POST http://:/mcp \ + -H 'content-type: application/json' \ + -d '{"jsonrpc":"2.0","id":"1","method":"tools/list"}' | jq . +``` + +**PASS** = JSON with a `result.tools` array containing `speak`, +`navigate_with_text`, `look_out_for`. **G0 done.** + +> The MCP wire format itself is already validated CPU-side +> (`sim_harness.py`, multica WEB-5). At the venue you are only proving the +> *network path*, not the protocol. + +If refused: re-check `--listen-host 0.0.0.0`, then `ping `, then the +node's firewall (`sudo ufw allow 9990`). + +--- + +## 3. Launch the conductor (operator laptop) + +```bash +cd +source .venv/bin/activate +python dimos/experimental/pack_mind/conductor.py \ + --dog alpha=:9990 \ + --dog bravo=:9990 +# add --dog charlie=:9990 for the 3rd dog (gravy only) +``` + +Open **http://localhost:8080**. Six operator buttons drive the mission. Put this +on the projector — the causal event feed *is* the story. + +--- + +## 4. G1 (h12) — single-dog magic moment + +One dog, one human-legible beat. On the dashboard: + +1. **start_act1** → Alpha announces it is scouting. +2. **inject_found** (alpha / red backpack / zone_b) → Alpha speaks "Found red + backpack in zone_b" and the find lands on the shared blackboard. +3. **ask_where** (bravo / red backpack) → Bravo answers **from pack memory**: + *"Alpha found it in zone_b. Follow me."* + +If the dog physically speaks and the dashboard shows the causal chain, **G1 done.** + +--- + +## 5. G2 (h24) — two-dog MVP, **twice in a row** (THE gate) + +Full mission, both dogs, no crash, two consecutive clean runs: + +1. **start_act1** +2. **inject_found** alpha → zone_b +3. **ask_where** bravo → answers from shared memory +4. **send_dog** bravo → acquires movement lock, navigates to **zone_b by name** +5. **verify** bravo → VLM lookout confirms → "Pack memory was correct." + +Watch for: only one dog moving at a time (movement lock), no coordinates ever +exchanged. Run it, **estop**, reset, run it again. Two clean passes = **G2 done.** +This is the gate the judges buy. Everything before it is low-risk; the real risk +is multi-dog concurrency + venue WiFi. + +--- + +## 6. Fallback — the demo never hard-fails + +**Operator `inject_found` produces the identical blackboard effect as a real +detection.** If live perception is flaky, the operator injects the find and the +entire downstream story (memory → query → navigate → verify) runs unchanged. +Rehearse the demo driving inject_found by hand; a real detection is a bonus, not +a dependency. + +If a dog wanders or a nav call hangs: hit **estop** (releases the movement lock, +all dogs hold, mission → IDLE), then resume from start_act1. + +--- + +## 7. G3 (h36) / G4 (h48) + +- **G3:** Record a clean two-dog run end-to-end (screen + floor). Harden: rerun + G2 ~5× and note any flake. +- **G4:** Rehearse the 90s presentation 3×. Script the narration to the + dashboard event feed. Have the recorded G3 run ready as the fallback if the + live floor misbehaves. + +--- + +## 8. Quick reference + +| Thing | Command / value | +|---|---| +| MCP port (default) | `9990` (`--mcp-port`) | +| LAN bind (REQUIRED) | `--listen-host 0.0.0.0` | +| Dashboard | `http://localhost:8080` | +| Stop a stack | `dimos stop` (on the node) | +| Tail logs | `dimos log -f` (on the node) | +| Send raw agent text | `dimos agent-send "..."` | +| CPU dry-run (no dogs) | `conductor.py --mock` | +| MCP tool args | `speak(text, blocking)` · `navigate_with_text(query)` · `look_out_for(description_of_things)` · `tag_location(location_name)` | + +Zones are named (`zone_a/b/c`). Cross-dog handoff is **always** by zone name. +There is no coordinate field anywhere in the shared memory — by design. diff --git a/dimos/experimental/pack_mind/test_conductor.py b/dimos/experimental/pack_mind/test_conductor.py new file mode 100644 index 0000000000..dcc5e6adf9 --- /dev/null +++ b/dimos/experimental/pack_mind/test_conductor.py @@ -0,0 +1,232 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the PACK MIND conductor. + +All tests run on CPU with no hardware and no network: the conductor is driven in +``mock=True`` mode, where ``call_tool`` logs an event instead of issuing HTTP. The +state-machine code path is identical to real hardware, so these tests guard the +exact logic that runs at the venue. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import time + +import pytest + +from dimos.experimental.pack_mind.conductor import ( + Conductor, + Dog, + MemoryEvent, + _extract_text, + _parse_dog, +) + + +def _mock_conductor() -> Conductor: + dogs = [ + Dog(id="alpha", name="Alpha", mcp_url="http://mock/alpha/mcp", role="scout"), + Dog(id="bravo", name="Bravo", mcp_url="http://mock/bravo/mcp", role="guide"), + ] + return Conductor(dogs, mock=True) + + +def _wait_for_mission(c: Conductor, target: str, timeout: float = 3.0) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + if c.snapshot()["mission"] == target: + return True + time.sleep(0.02) + return False + + +# -- blackboard / immutability ---------------------------------------------- + + +def test_memory_event_is_frozen(): + event = MemoryEvent(id="e1", ts="t", robot="alpha", type="x", text="y") + + with pytest.raises(dataclasses.FrozenInstanceError): + event.text = "mutated" # type: ignore[misc] + + +def test_memory_event_carries_no_coordinate_fields(): + # The whole thesis: dogs share meaning, never geometry. + forbidden = {"x", "y", "z", "pose", "coordinate", "position"} + fields = set(MemoryEvent.__dataclass_fields__) + + assert forbidden.isdisjoint(fields) + + +def test_append_event_appends_and_returns_event(): + c = _mock_conductor() + + event = c.append_event("alpha", "object_found", "found it", object_="bag", zone="zone_a") + + assert event.robot == "alpha" + assert event.zone == "zone_a" + assert c.snapshot()["events"][0]["id"] == event.id + + +# -- semantic recall (the core trick) --------------------------------------- + + +def test_latest_zone_for_returns_most_recent_match(): + c = _mock_conductor() + c.append_event("alpha", "object_found", "a", object_="bag", zone="zone_a") + c.append_event("alpha", "object_found", "b", object_="bag", zone="zone_c") + + assert c._latest_zone_for("bag") == "zone_c" + + +def test_latest_zone_for_unknown_object_is_none(): + c = _mock_conductor() + + assert c._latest_zone_for("nonexistent") is None + + +def test_ask_where_answers_from_shared_memory(): + c = _mock_conductor() + # Alpha records the find; Bravo must answer from the SHARED blackboard. + c.inject_found("alpha", "red backpack", "zone_b") + + answer = c.ask_where("bravo", "red backpack") + + assert "Alpha" in answer + assert "zone_b" in answer + + +def test_ask_where_with_no_memory_admits_ignorance(): + c = _mock_conductor() + + answer = c.ask_where("bravo", "red backpack") + + assert "no pack memory" in answer.lower() + + +# -- movement lock (only one dog moves) ------------------------------------- + + +def test_send_dog_blocked_when_move_lock_held(): + c = _mock_conductor() + c.inject_found("alpha", "red backpack", "zone_b") + c._move_lock.acquire() + + try: + ok = c.send_dog_to_memory("bravo", "red backpack") + finally: + c._move_lock.release() + + assert ok is False + assert any(e.type == "blocked" for e in c._events) + + +def test_send_dog_without_memory_refuses(): + c = _mock_conductor() + + ok = c.send_dog_to_memory("bravo", "red backpack") + + assert ok is False + assert any(e.type == "blocked" for e in c._events) + + +def test_send_dog_releases_lock_after_navigation(): + c = _mock_conductor() + c.inject_found("alpha", "red backpack", "zone_b") + + assert c.send_dog_to_memory("bravo", "red backpack") is True + # async navigation finishes fast in mock; lock must be released for the next dog. + deadline = time.time() + 3.0 + while c._move_lock.locked() and time.time() < deadline: + time.sleep(0.02) + + assert not c._move_lock.locked() + assert any(e.type == "arrived" for e in c._events) + + +# -- full mission sequence --------------------------------------------------- + + +def test_full_mock_mission_reaches_done(): + c = _mock_conductor() + + c.start_act1() + assert c.snapshot()["mission"] == "SCOUT_ALPHA" + + c.inject_found("alpha", "red backpack", "zone_b") + assert c.snapshot()["mission"] == "FOUND_EVENT_RECORDED" + + c.ask_where("bravo", "red backpack") + assert c.snapshot()["mission"] == "QUERY_BRAVO" + + assert c.send_dog_to_memory("bravo", "red backpack") is True + assert c.snapshot()["mission"] == "BRAVO_NAVIGATING" + + c.verify_at_zone("bravo", "red backpack") + assert _wait_for_mission(c, "DONE") + + +def test_emergency_stop_resets_to_idle(): + c = _mock_conductor() + c.start_act1() + c.inject_found("alpha", "red backpack", "zone_b") + + c.emergency_stop() + + snap = c.snapshot() + assert snap["mission"] == "IDLE" + assert all(d["status"] == "idle" for d in snap["roster"]) + + +# -- arg parsing ------------------------------------------------------------- + + +def test_parse_dog_default_port(): + dog = _parse_dog("alpha=10.0.0.10") + + assert dog.mcp_url == "http://10.0.0.10:9990/mcp" + assert dog.role == "scout" + + +def test_parse_dog_explicit_port_and_case_insensitive(): + dog = _parse_dog("BRAVO=1.2.3.4:7000") + + assert dog.id == "bravo" + assert dog.mcp_url == "http://1.2.3.4:7000/mcp" + + +def test_parse_dog_rejects_unknown_id(): + with pytest.raises(argparse.ArgumentTypeError): + _parse_dog("zulu=1.2.3.4") + + +# -- MCP response shape tolerance -------------------------------------------- + + +@pytest.mark.parametrize( + "data,expected", + [ + ({"result": {"content": [{"text": "hi"}]}}, "hi"), + ({"content": [{"text": "a"}, {"text": "b"}]}, "a b"), + ({"result": {"content": []}}, None), + ({"result": {}}, None), + ("not a dict", None), + ({"result": {"content": [{"no_text": 1}]}}, None), + ], +) +def test_extract_text_tolerates_shape_variation(data, expected): + assert _extract_text(data) == expected From 9a9e7bc939479f90e9aa93b93c3d7772b1e7456b Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Tue, 26 May 2026 20:53:43 -0600 Subject: [PATCH 04/35] feat: live Go2 venue bring-up script (onboard-Jetson topology) --- dimos/experimental/pack_mind/venue_go2.sh | 80 +++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 dimos/experimental/pack_mind/venue_go2.sh diff --git a/dimos/experimental/pack_mind/venue_go2.sh b/dimos/experimental/pack_mind/venue_go2.sh new file mode 100755 index 0000000000..d57fc8318f --- /dev/null +++ b/dimos/experimental/pack_mind/venue_go2.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# PACK MIND — live Go2 venue bring-up (Go2 EDU onboard-Jetson topology). +# +# Run this from the operator laptop AFTER joining the dog's WiFi (dimair14). +# It probes the dog, guides the on-dog stack launch, then validates G0 and +# launches the conductor. Read-only probing; the on-dog launch is explicit. +# +# Override defaults via env: DOG=192.168.12.1 SSH_USER=unitree MCP_PORT=9990 ./venue_go2.sh +set -uo pipefail + +DOG="${DOG:-192.168.12.1}" +SSH_USER="${SSH_USER:-unitree}" +PORT="${MCP_PORT:-9990}" +SELF="$(cd "$(dirname "$0")" && pwd)" + +stage() { printf '\n=== %s ===\n' "$1"; } +ok() { printf ' PASS: %s\n' "$1"; } +bad() { printf ' FAIL: %s\n' "$1"; } + +stage "0 network — am I on dimair14 and can I see the dog?" +if ping -c 2 -t 3 "$DOG" >/dev/null 2>&1; then + ok "dog reachable at $DOG" +else + bad "cannot reach $DOG — join WiFi dimair14 (pw 88888888) first, then re-run." + exit 1 +fi + +stage "1 probe the dog (prompts for the dog's ssh password)" +echo " ssh ${SSH_USER}@${DOG} ..." +ssh -o ConnectTimeout=8 -o StrictHostKeyChecking=no "${SSH_USER}@${DOG}" ' + echo " host: $(hostname)"; + printf " cuda: "; (nvidia-smi -L 2>/dev/null | head -1 || echo "NO nvidia-smi — wrong host or no GPU"); + printf " dimos: "; (command -v dimos 2>/dev/null || ls -d ~/dimos 2>/dev/null || echo "NOT FOUND — DimOS not installed here"); + printf " port '"$PORT"' busy?: "; (ss -ltn 2>/dev/null | grep ":'"$PORT"'" || echo "free"); +' || { bad "ssh failed — check user/host/password. Default Unitree user is 'unitree'."; exit 1; } + +cat </dev/null | sed 's/^/ tool: /' || true) +else + bad "no tool list back. Got: ${RESP:0:200}" + echo " -> re-check --listen-host 0.0.0.0 on the dog, then dog firewall." + exit 1 +fi + +stage "4 G1 — make the REAL dog speak (direct MCP, bypasses conductor)" +curl -s --max-time 10 -X POST "http://${DOG}:${PORT}/mcp" \ + -H 'content-type: application/json' \ + -d '{"jsonrpc":"2.0","id":"2","method":"tools/call","params":{"name":"speak","arguments":{"text":"Pack mind online.","blocking":false}}}' \ + | (jq -r '.result.content[].text' 2>/dev/null || cat) +echo " (listen — the dog should have spoken)" + +stage "5 launch the conductor + dashboard" +cat < drive start_act1 / inject_found / ask_where / send_dog / verify + # second dog: add --dog bravo=:9990 +EOF +echo "Done. Single-dog path is live once the dashboard speaks through the dog." From 0ab0b659d630342fe6e13206675cf39e193b70ff Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 05:43:29 -0600 Subject: [PATCH 05/35] chore: ignore .scratch working dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index aedee04af7..351c408306 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,4 @@ htmlcov/ # Memory2 autorecord recording*.db +.scratch/ From 51a5e6d77598832f1671848a5d7617b38fdc6656 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 05:43:29 -0600 Subject: [PATCH 06/35] feat: PACK MIND coverage A/B sim + pack-mind-sim blueprint --- dimos/experimental/pack_mind/blueprint.py | 108 ++++++++ dimos/experimental/pack_mind/demo_spike.py | 110 ++++++++ dimos/experimental/pack_mind/render.py | 167 +++++++++++++ dimos/experimental/pack_mind/sim.py | 235 ++++++++++++++++++ .../experimental/pack_mind/sim_perception.py | 82 ++++++ dimos/experimental/pack_mind/sim_robot.py | 96 +++++++ .../pack_mind/test_pack_mind_sim.py | 65 +++++ dimos/experimental/pack_mind/view_rerun.py | 116 +++++++++ dimos/experimental/pack_mind/world.py | 90 +++++++ dimos/robot/all_blueprints.py | 3 + 10 files changed, 1072 insertions(+) create mode 100644 dimos/experimental/pack_mind/blueprint.py create mode 100644 dimos/experimental/pack_mind/demo_spike.py create mode 100644 dimos/experimental/pack_mind/render.py create mode 100644 dimos/experimental/pack_mind/sim.py create mode 100644 dimos/experimental/pack_mind/sim_perception.py create mode 100644 dimos/experimental/pack_mind/sim_robot.py create mode 100644 dimos/experimental/pack_mind/test_pack_mind_sim.py create mode 100644 dimos/experimental/pack_mind/view_rerun.py create mode 100644 dimos/experimental/pack_mind/world.py diff --git a/dimos/experimental/pack_mind/blueprint.py b/dimos/experimental/pack_mind/blueprint.py new file mode 100644 index 0000000000..837384b4ae --- /dev/null +++ b/dimos/experimental/pack_mind/blueprint.py @@ -0,0 +1,108 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native DimOS blueprint for the PACK MIND A/B coverage race. + + dimos run pack-mind-sim --viewer rerun + +Runs PACK (shared coverage memory) and INDEPENDENT (private) side by side and +publishes each one's live coverage as an OccupancyGrid. The Rerun bridge (added +automatically by ``--viewer rerun``) subscribes to the bus and renders both: +walls = OCCUPIED (dark), searched = FREE (light), unsearched = UNKNOWN (gray). +Watch the PACK map fill cleanly while INDEPENDENT re-walks the same corridors. +""" + +from __future__ import annotations + +import threading + +import numpy as np + +from dimos.core.coordination.blueprints import autoconnect +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.core.stream import Out +from dimos.experimental.pack_mind.sim import PackSim, build_sim +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid + +_TICK_DT = 0.005 # seconds between sim ticks (animation pace in the viewer) +_PUB_EVERY = 8 # publish a coverage grid every N ticks +_HOLD_S = 4.0 # hold the final frame before replaying the race +_SEED = 0 + + +def _coverage_grid(sim: PackSim, frame_id: str) -> OccupancyGrid: + """Encode coverage into OccupancyGrid semantics for the native colormap: + OCCUPIED=wall (dark), FREE=searched (light), UNKNOWN=unsearched (gray).""" + g = sim.world.grid + free = g == CostValues.FREE + out = np.full(g.shape, CostValues.UNKNOWN, dtype=np.int8) + out[g == CostValues.OCCUPIED] = CostValues.OCCUPIED + union: np.ndarray | None = None + for rb in sim.robots: + v = rb.router._visited + if v is None: + continue + union = v.copy() if union is None else (union | v) + if union is not None: + out[union & free] = CostValues.FREE + return OccupancyGrid(grid=out, resolution=sim.world.info.resolution, frame_id=frame_id) + + +class PackMindSimModule(Module): + """Steps both A/B sims in a background thread and publishes their coverage.""" + + pack_coverage: Out[OccupancyGrid] + indep_coverage: Out[OccupancyGrid] + + @rpc + def start(self) -> None: + super().start() + self._stop_evt = threading.Event() + self._thread = threading.Thread(target=self._loop, daemon=True) + self._thread.start() + + @rpc + def stop(self) -> None: + self._stop_evt.set() + if getattr(self, "_thread", None) is not None and self._thread.is_alive(): + self._thread.join(timeout=2.0) + super().stop() + + def _publish(self, pack: PackSim, indep: PackSim) -> None: + self.pack_coverage.publish(_coverage_grid(pack, "pack")) + self.indep_coverage.publish(_coverage_grid(indep, "independent")) + + def _loop(self) -> None: + while not self._stop_evt.is_set(): + pack = build_sim(shared=True, seed=_SEED) + indep = build_sim(shared=False, seed=_SEED) + self._publish(pack, indep) + while not self._stop_evt.is_set(): + done_p = all(rb.done for rb in pack.robots) + done_i = all(rb.done for rb in indep.robots) + if not done_p: + pack.step() + if not done_i: + indep.step() + if pack.tick_n % _PUB_EVERY == 0 or indep.tick_n % _PUB_EVERY == 0: + self._publish(pack, indep) + if done_p and done_i: + break + self._stop_evt.wait(_TICK_DT) + self._publish(pack, indep) + self._stop_evt.wait(_HOLD_S) # hold final frame, then replay + + +pack_mind_sim = autoconnect(PackMindSimModule.blueprint()) diff --git a/dimos/experimental/pack_mind/demo_spike.py b/dimos/experimental/pack_mind/demo_spike.py new file mode 100644 index 0000000000..2957526857 --- /dev/null +++ b/dimos/experimental/pack_mind/demo_spike.py @@ -0,0 +1,110 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND de-risk spike (PLAN §6). + +Answers the only question that decides feasibility of the video-only sim: + + Can we drive FrontierPatrolRouter + min_cost_astar on a hand-fabricated + OccupancyGrid in a plain loop, no robot stack, no CUDA, no ROS? + +Plus it proves the A/B knob: two routers sharing ONE VisitationHistory vs two +private ones. Run: + + uv run python dimos/experimental/pack_mind/demo_spike.py +""" + +from __future__ import annotations + +import numpy as np + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid +from dimos.navigation.patrolling.routers.frontier_patrol_router import FrontierPatrolRouter +from dimos.navigation.patrolling.routers.visitation_history import VisitationHistory +from dimos.navigation.replanning_a_star.min_cost_astar import min_cost_astar + +RES = 0.1 # m/cell +CLEARANCE_M = 0.3 + + +def make_world(w: int = 40, h: int = 40) -> OccupancyGrid: + """Free interior, OCCUPIED border + one interior wall. Fully known (no fog).""" + grid = np.full((h, w), CostValues.FREE, dtype=np.int8) + grid[0, :] = grid[-1, :] = CostValues.OCCUPIED + grid[:, 0] = grid[:, -1] = CostValues.OCCUPIED + grid[10:30, 20] = CostValues.OCCUPIED # vertical divider with a gap below row 30 + return OccupancyGrid(grid=grid, resolution=RES) + + +def pose_at(x: float, y: float) -> PoseStamped: + p = PoseStamped() + p.position.x = x + p.position.y = y + return p + + +def main() -> None: + world = make_world() + print(f"[spike] world: {world.width}x{world.height} @ {RES}m ({world})") + + # --- 1. router accepts a fabricated grid + odom, returns a goal --- + r = FrontierPatrolRouter(CLEARANCE_M) + r.handle_occupancy_grid(world) + r.handle_odom(pose_at(0.5, 0.5)) + goal = r.next_goal() + assert goal is not None, "FAIL: next_goal returned None on a fresh map" + print(f"[spike] PASS next_goal -> ({goal.position.x:.2f}, {goal.position.y:.2f})") + + # --- 2. A* plans start->goal with python fallback (use_cpp=False) --- + path = min_cost_astar( + world, (goal.position.x, goal.position.y), (0.5, 0.5), use_cpp=False + ) + assert path is not None and len(path.poses) > 0, "FAIL: A* found no path" + print(f"[spike] PASS min_cost_astar -> {len(path.poses)} waypoints") + + # --- 3. THE A/B KNOB: shared vs private VisitationHistory --- + # Private: two routers, independent visited masks. + a_priv, b_priv = FrontierPatrolRouter(CLEARANCE_M), FrontierPatrolRouter(CLEARANCE_M) + for rr in (a_priv, b_priv): + rr.handle_occupancy_grid(world) + a_priv.handle_odom(pose_at(0.5, 0.5)) + # A walks a bit; B should NOT see A's coverage. + for x in np.arange(0.5, 2.5, 0.1): + a_priv.handle_odom(pose_at(float(x), 0.5)) + b_sees_via_private = b_priv._visited.sum() if b_priv._visited is not None else 0 + + # Shared: B reuses A's VisitationHistory instance. + a_sh, b_sh = FrontierPatrolRouter(CLEARANCE_M), FrontierPatrolRouter(CLEARANCE_M) + shared = VisitationHistory(CLEARANCE_M) + a_sh._visitation = b_sh._visitation = shared + for rr in (a_sh, b_sh): + rr.handle_occupancy_grid(world) + for x in np.arange(0.5, 2.5, 0.1): + a_sh.handle_odom(pose_at(float(x), 0.5)) + b_sees_via_shared = b_sh._visited.sum() if b_sh._visited is not None else 0 + + print( + f"[spike] A/B knob: B sees {b_sees_via_private} visited cells (private) " + f"vs {b_sees_via_shared} (shared)" + ) + assert b_sees_via_private == 0, "FAIL: private leaked coverage" + assert b_sees_via_shared > 0, "FAIL: shared did not propagate A's coverage to B" + print("[spike] PASS shared VisitationHistory propagates; private does not") + + print("\n[spike] ALL GREEN — PLAN is buildable as written.") + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/render.py b/dimos/experimental/pack_mind/render.py new file mode 100644 index 0000000000..7533776ff4 --- /dev/null +++ b/dimos/experimental/pack_mind/render.py @@ -0,0 +1,167 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — G3 render: side-by-side A/B coverage race to mp4 (or GIF). + +Left = PACK (shared coverage memory), right = INDEPENDENT (private memory). Same +seed, same single deploy point, same survivors — the only difference is shared vs +private memory. The HUD shows coverage %, survivors found, and tick so the gap is +self-evident on screen. + + python -m dimos.experimental.pack_mind.render --out pack_mind_ab.mp4 --seed 0 +""" + +from __future__ import annotations + +import argparse +from typing import Any + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 +from matplotlib import animation # noqa: E402 + +from dimos.experimental.pack_mind.sim import PackSim, build_sim # noqa: E402 +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues # noqa: E402 + +_SNAP_EVERY = 15 # capture a frame every N ticks + + +def _union_visited(sim: PackSim) -> np.ndarray: + free = sim.world.grid == CostValues.FREE + union: np.ndarray | None = None + for rb in sim.robots: + v = rb.router._visited + if v is None: + continue + union = v.copy() if union is None else (union | v) + if union is None: + union = np.zeros_like(free) + return union & free + + +def _run_capture(shared: bool, seed: int, max_ticks: int) -> tuple[PackSim, list[dict[str, Any]]]: + sim = build_sim(shared=shared, seed=seed) + free_total = sim._free_total + frames: list[dict[str, Any]] = [] + + def snap() -> None: + vis = _union_visited(sim) + frames.append( + { + "tick": sim.tick_n, + "visited": vis, + "robots": [(rb.x, rb.y, rb.done) for rb in sim.robots], + "found": set(sim._found), + "cov": float(np.count_nonzero(vis)) / max(1, free_total), + } + ) + + snap() + for _ in range(max_ticks): + sim.step() + if sim.tick_n % _SNAP_EVERY == 0: + snap() + if all(rb.done for rb in sim.robots): + break + snap() + return sim, frames + + +def render_ab(seed: int = 0, out_path: str = "pack_mind_ab.mp4", max_ticks: int = 4000) -> str: + psim, pframes = _run_capture(True, seed, max_ticks) + _, iframes = _run_capture(False, seed, max_ticks) + n = max(len(pframes), len(iframes)) + + world = psim.world + grid = world.grid + h, w = grid.shape + survivors = psim.survivors + + base = np.ones((h, w, 3)) + base[grid == CostValues.OCCUPIED] = [0.05, 0.05, 0.08] + + fig, (axp, axi) = plt.subplots(1, 2, figsize=(12, 6.2)) + fig.suptitle("PACK MIND — same robots, same start, only memory differs", fontsize=13) + + def at(frames: list[dict[str, Any]], i: int) -> dict[str, Any]: + return frames[min(i, len(frames) - 1)] + + def draw(ax: Any, frame: dict[str, Any], title: str) -> None: + ax.clear() + img = base.copy() + img[frame["visited"]] = [0.55, 0.78, 1.0] + ax.imshow(img, origin="lower") + for si, (sx, sy) in enumerate(survivors): + gc = world.world_to_grid((sx, sy)) + color = "limegreen" if si in frame["found"] else "red" + ax.plot(gc.x, gc.y, marker="*", color=color, markersize=15, markeredgecolor="k") + for rx, ry, done in frame["robots"]: + gc = world.world_to_grid((rx, ry)) + ax.plot( + gc.x, + gc.y, + marker="o", + color="gray" if done else "orange", + markersize=9, + markeredgecolor="k", + ) + ax.set_title( + f"{title}\ncoverage {frame['cov']:.0%} " + f"found {len(frame['found'])}/{len(survivors)} t={frame['tick']}", + fontsize=11, + ) + ax.set_xticks([]) + ax.set_yticks([]) + + def update(i: int) -> list[Any]: + draw(axp, at(pframes, i), "PACK · shared memory") + draw(axi, at(iframes, i), "INDEPENDENT · private") + return [] + + anim = animation.FuncAnimation(fig, update, frames=n, interval=80) + saved = out_path + try: + anim.save(out_path, writer="ffmpeg", fps=12, dpi=100) + except Exception as exc: # ffmpeg missing → GIF fallback + saved = out_path.rsplit(".", 1)[0] + ".gif" + print(f"[render] ffmpeg unavailable ({exc}); writing GIF -> {saved}") + anim.save(saved, writer="pillow", fps=12) + plt.close(fig) + return saved + + +def query_report(result: Any) -> str: + """Scripted situational report from the findings pool (the 'memory' end card).""" + lines = [f"PACK MIND situational report — {result.found}/{result.total_survivors} survivors located:"] + for f in result.findings: + lines.append(f" • survivor at ({f.survivor_xy[0]:.1f}, {f.survivor_xy[1]:.1f}) — found by {f.by} at t={f.tick}") + lines.append(f"area coverage: {result.coverage:.0%} (mode={result.mode})") + return "\n".join(lines) + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND A/B render") + p.add_argument("--out", default="pack_mind_ab.mp4") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--max-ticks", type=int, default=4000) + return p.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + path = render_ab(seed=args.seed, out_path=args.out, max_ticks=args.max_ticks) + print(f"rendered -> {path}") diff --git a/dimos/experimental/pack_mind/sim.py b/dimos/experimental/pack_mind/sim.py new file mode 100644 index 0000000000..3847f0627b --- /dev/null +++ b/dimos/experimental/pack_mind/sim.py @@ -0,0 +1,235 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — PackSim orchestrator + SimResult (G1/G2). + +Honest A/B experiment: PACK and INDEPENDENT use the SAME robots, SAME start +point, SAME map and survivors. The ONLY difference is whether the robots share +one VisitationHistory (the coverage memory) or each keep a private one. So any +performance gap is attributable to shared memory alone — not to start positions. + +PACK (shared=True): one shared VisitationHistory. Each robot's CoveragePatrolRouter + sees teammates' coverage and steers toward still-uncovered area → the pack + fans out from the single entrance and divides the map with little overlap. + +INDEPENDENT (shared=False): private VisitationHistory per robot. Each robot + believes the whole map is uncovered → they redundantly re-walk the same + corridors → far more ticks to full coverage. + +CLI: + python -m dimos.experimental.pack_mind.sim --mode pack --seed 0 + python -m dimos.experimental.pack_mind.sim --mode independent --seed 0 +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass, field + +import numpy as np + +from dimos.experimental.pack_mind.sim_robot import SimRobot +from dimos.experimental.pack_mind.world import free_cell_count, make_maze_world, plant_survivors +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid +from dimos.navigation.patrolling.routers.coverage_patrol_router import CoveragePatrolRouter +from dimos.navigation.patrolling.routers.visitation_history import VisitationHistory + +CLEARANCE_M = 0.1 +DETECT_R = 0.4 +_STALL_TICKS = 200 # stop a run once coverage has not improved for this many ticks +_STALL_EPS = 0.001 + + +@dataclass +class Finding: + survivor_xy: tuple[float, float] + by: str + tick: int + + +@dataclass +class SimResult: + mode: str + ticks: int + coverage: float + found: int + total_survivors: int + ticks_to_target: int | None # ticks to reach coverage_target + ticks_to_all: int | None # ticks to find every survivor + coverage_target: float = 0.75 + findings: list[Finding] = field(default_factory=list) + + def __str__(self) -> str: + tt = self.ticks_to_target if self.ticks_to_target is not None else "never" + tall = self.ticks_to_all if self.ticks_to_all is not None else "never" + return ( + f"SimResult(mode={self.mode}, ticks={self.ticks}, " + f"coverage={self.coverage:.1%}, found={self.found}/{self.total_survivors}, " + f"ticks_to_{self.coverage_target:.0%}={tt}, ticks_to_all={tall})" + ) + + +class PackSim: + """Run N robots on a shared (pack) or private (independent) coverage mission.""" + + def __init__( + self, + world: OccupancyGrid, + survivors: list[tuple[float, float]], + n_robots: int, + shared: bool, + starts: list[tuple[float, float]], + seed: int = 0, + kill_at_tick: int | None = None, + ) -> None: + np.random.seed(seed) # CoveragePatrolRouter samples candidates via np.random + self.world = world + self.survivors = survivors + self.shared = shared + self.kill_at_tick = kill_at_tick + self._free_total = free_cell_count(world) + self.findings: list[Finding] = [] + self._found: set[int] = set() + self.tick_n = 0 + + routers = [CoveragePatrolRouter(CLEARANCE_M) for _ in range(n_robots)] + if shared: + # Shared coverage memory: set the SAME VisitationHistory on every router + # BEFORE SimRobot.__init__ calls handle_occupancy_grid (which initialises + # the mask). Robot B then immediately sees robot A's footprint. + shared_vis = VisitationHistory(CLEARANCE_M) + for r in routers: + r._visitation = shared_vis + # VisitationHistory is built for *patrolling*: at 50% saturation it prunes the + # oldest half of visited points (bounded memory), capping coverage at ~50%. + # A one-shot SAR search must remember everything — disable pruning. Also widen + # candidate sampling so the router targets the last uncovered pockets. + for r in routers: + r._visitation._saturation_threshold = 10.0 + r._candidates_to_consider = 24 + + self.robots = [ + SimRobot(f"r{i}", routers[i], world, starts[i]) for i in range(n_robots) + ] + + def _coverage(self) -> float: + """Fraction of free cells covered (union of robots' visited masks).""" + union: np.ndarray | None = None + free = self.world.grid == CostValues.FREE + for rb in self.robots: + v = rb.router._visited + if v is None: + continue + union = v.copy() if union is None else (union | v) + if union is None: + return 0.0 + return float(np.count_nonzero(union & free)) / max(1, self._free_total) + + def _detect(self) -> None: + for si, s in enumerate(self.survivors): + if si in self._found: + continue + for rb in self.robots: + if rb.done: + continue + if (rb.x - s[0]) ** 2 + (rb.y - s[1]) ** 2 <= DETECT_R**2: + self._found.add(si) + self.findings.append(Finding(s, rb.name, self.tick_n)) + break + + def step(self) -> None: + self.tick_n += 1 + # Death/persistence beat: kill the first robot at the given tick. Its + # coverage + findings remain (shared mask / findings pool outlive it). + if self.kill_at_tick is not None and self.tick_n == self.kill_at_tick: + self.robots[0].done = True + for rb in self.robots: + rb.tick() + self._detect() + + def run(self, max_ticks: int = 4000, coverage_target: float = 0.75) -> SimResult: + t_target: int | None = None + tall: int | None = None + best_cov = 0.0 + stall = 0 + for _ in range(max_ticks): + self.step() + cov = self._coverage() + if t_target is None and cov >= coverage_target: + t_target = self.tick_n + if tall is None and len(self._found) == len(self.survivors): + tall = self.tick_n + if cov > best_cov + _STALL_EPS: + best_cov = cov + stall = 0 + else: + stall += 1 + if all(rb.done for rb in self.robots): + break + if stall >= _STALL_TICKS: + break + return SimResult( + "pack" if self.shared else "independent", + self.tick_n, + self._coverage(), + len(self._found), + len(self.survivors), + t_target, + tall, + coverage_target, + list(self.findings), + ) + + +def build_sim( + shared: bool, + seed: int = 0, + n_robots: int = 3, + n_survivors: int = 4, + kill_at_tick: int | None = None, +) -> PackSim: + """Build a sim. Both modes share IDENTICAL starts (single SAR entry point); + the only variable is shared vs private memory.""" + world = make_maze_world() + survivors = plant_survivors(world, n_survivors, seed=seed) + # Single deploy point at the bottom-left entrance, tiny offsets to avoid overlap. + starts = [(0.5 + 0.2 * i, 0.4) for i in range(n_robots)] + return PackSim(world, survivors, n_robots, shared, starts, seed, kill_at_tick) + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND sim CLI") + p.add_argument("--mode", choices=["pack", "independent"], default="pack") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--render", metavar="PATH", default=None) + p.add_argument("--max-ticks", type=int, default=4000) + return p.parse_args() + + +if __name__ == "__main__": + import time + + args = _parse_args() + t0 = time.perf_counter() + sim = build_sim(shared=args.mode == "pack", seed=args.seed) + result = sim.run(max_ticks=args.max_ticks) + wall = time.perf_counter() - t0 + print(result) + print(f"wall time: {wall:.1f}s") + + if args.render: + from dimos.experimental.pack_mind.render import render_ab + + render_ab(seed=args.seed, out_path=args.render) + print(f"rendered -> {args.render}") diff --git a/dimos/experimental/pack_mind/sim_perception.py b/dimos/experimental/pack_mind/sim_perception.py new file mode 100644 index 0000000000..af0e8665cd --- /dev/null +++ b/dimos/experimental/pack_mind/sim_perception.py @@ -0,0 +1,82 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim backbone — step 1: VLM perception proof (CUDA-free). + +Replaces EdgeTAM with a VLM that *describes* the camera frame, exactly like the +robot-telephone reference (ouazmourad/dimos-21-days-sprint). Runs one G1 MuJoCo +sim (no perception stack, so no CUDA) plus a VLM describer, then asks the VLM to +describe what the simulated robot sees. If this prints a sensible description, +the VLM-perception -> semantic-memory bridge works on this machine. + +macOS needs mjpython for MuJoCo. Needs OPENAI_API_KEY + internet. + + mjpython dimos/experimental/pack_mind/sim_perception.py +""" + +from __future__ import annotations + +import time + +from dimos.agents.vlm_agent import VLMAgent +from dimos.core.coordination.blueprints import autoconnect +from dimos.core.coordination.module_coordinator import ModuleCoordinator +from dimos.core.core import rpc +from dimos.core.global_config import global_config +from dimos.robot.unitree.g1.blueprints.basic.unitree_g1_basic_sim import ( + unitree_g1_basic_sim, +) + +CAMERA_WARMUP_S = 20.0 + + +class DescriberVLM(VLMAgent): + """VLMAgent + a visual_query RPC that describes the latest camera frame. + + Local VLMAgent has query() (text-only) and query_image(img, q); it lacks the + fork's visual_query, so we add it here: query using the last frame received + on the color_image stream. + """ + + @rpc + def visual_query(self, query: str) -> str: + if self._latest_image is None: + return "No image yet — camera not streaming." + response = self._invoke_image(self._latest_image, query) + content = response.content + return content if isinstance(content, str) else str(content) + + +def main() -> None: + global_config.update(simulation="mujoco") + blueprint = autoconnect(unitree_g1_basic_sim, DescriberVLM.blueprint(model="gpt-4o")) + coordinator = ModuleCoordinator.build(blueprint) + try: + print(f"[pack-mind] sim up; warming camera {CAMERA_WARMUP_S:.0f}s...", flush=True) + time.sleep(CAMERA_WARMUP_S) + vlm = coordinator.get_instance(DescriberVLM) + prompt = ( + "Look at the scene and pick the most prominent object. Describe its " + "color, shape, size, and position relative to other objects in 2-3 " + "sentences. Do not name it directly — only its visual properties." + ) + print("[pack-mind] querying VLM with live sim frame...\n", flush=True) + print("VLM DESCRIPTION:\n" + vlm.visual_query(prompt), flush=True) + finally: + print("\n[pack-mind] shutting down sim...", flush=True) + coordinator.stop() + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/sim_robot.py b/dimos/experimental/pack_mind/sim_robot.py new file mode 100644 index 0000000000..03555de4cd --- /dev/null +++ b/dimos/experimental/pack_mind/sim_robot.py @@ -0,0 +1,96 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — SimRobot: one autonomous agent in the sim loop. + +Uses CoveragePatrolRouter (scores candidate goals by NEW coverage along the A* +path) instead of FrontierPatrolRouter (which thrashed: farthest-point goals laid +thin re-walked trails and plateaued ~41%). The router reads its `_visitation` +mask — shared across robots in PACK mode, private in INDEPENDENT mode — so the +only thing that changes between the two experiments is whether memory is shared. +""" + +from __future__ import annotations + +from math import hypot + +from dimos.experimental.pack_mind.world import pose_at +from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid +from dimos.navigation.patrolling.routers.coverage_patrol_router import CoveragePatrolRouter +from dimos.navigation.replanning_a_star.min_cost_astar import min_cost_astar + +_MAX_GOAL_FAILURES = 3 # consecutive next_goal/plan failures before the robot stops + + +class SimRobot: + """One simulated robot: coverage router + A* path follower.""" + + def __init__( + self, + name: str, + router: CoveragePatrolRouter, + world: OccupancyGrid, + start_xy: tuple[float, float], + step_m: float = 0.12, + ) -> None: + self.name = name + self.router = router + self.world = world + self.step_m = step_m + self.x, self.y = start_xy + self.path: list[tuple[float, float]] | None = None + self.idx = 0 + self.done = False + self._failures = 0 + # Throttle: handle_occupancy_grid is rate-limited to once/60s, so call + # it exactly once per router here at init. + self.router.handle_occupancy_grid(world) + self.router.handle_odom(pose_at(self.x, self.y)) + + def _request_goal(self) -> bool: + g = self.router.next_goal() + if g is None: + return False + p = min_cost_astar( + self.world, + (g.position.x, g.position.y), + (self.x, self.y), + use_cpp=False, + ) + if p is None or len(p.poses) == 0: + return False + self.path = [(po.position.x, po.position.y) for po in p.poses] + self.idx = 0 + return True + + def tick(self) -> None: + if self.done: + return + if self.path is None or self.idx >= len(self.path): + if not self._request_goal(): + self._failures += 1 + if self._failures >= _MAX_GOAL_FAILURES: + self.done = True + return + self._failures = 0 + tx, ty = self.path[self.idx] + dx, dy = tx - self.x, ty - self.y + d = hypot(dx, dy) + if d <= self.step_m or d == 0: + self.x, self.y = tx, ty + self.idx += 1 + else: + self.x += self.step_m * dx / d + self.y += self.step_m * dy / d + self.router.handle_odom(pose_at(self.x, self.y)) # stamps the visited mask diff --git a/dimos/experimental/pack_mind/test_pack_mind_sim.py b/dimos/experimental/pack_mind/test_pack_mind_sim.py new file mode 100644 index 0000000000..38aaad41f9 --- /dev/null +++ b/dimos/experimental/pack_mind/test_pack_mind_sim.py @@ -0,0 +1,65 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — G1+G2 tests. + +Run: bin/pytest-fast dimos/experimental/pack_mind/test_pack_mind_sim.py -v +""" + +import pytest + +from dimos.experimental.pack_mind.sim import build_sim +from dimos.experimental.pack_mind.world import free_cell_count, make_maze_world, plant_survivors + + +@pytest.mark.unit +def test_world_has_free_and_walls() -> None: + w = make_maze_world() + assert free_cell_count(w) > 0 + assert (w.grid == 100).any() # walls exist + + +@pytest.mark.unit +def test_survivors_on_free_cells() -> None: + w = make_maze_world() + s = plant_survivors(w, 4, seed=0) + assert len(s) == 4 + + +@pytest.mark.unit +def test_single_robot_substantial_coverage() -> None: + # GATE 0: the map is 100% coverable (no reachability wall), so one robot must + # cover the bulk of it. The coverage-patrol heuristic plateaus ~79% solo; 3 + # robots fill the nooks. We only need to prove there is no reachability wall. + sim = build_sim(shared=False, n_robots=1, n_survivors=1, seed=0) + r = sim.run(max_ticks=6000) + assert r.coverage > 0.7 + + +@pytest.mark.unit +def test_ab_pack_beats_independent_on_coverage() -> None: + """THE credibility test. Same robots, same single start, same map — the ONLY + difference is shared vs private coverage memory. Pack reaches the coverage + target in far fewer ticks and ends with higher coverage.""" + pack = build_sim(shared=True, seed=0).run() + indep = build_sim(shared=False, seed=0).run() + assert pack.ticks_to_target is not None + assert indep.ticks_to_target is None or pack.ticks_to_target < indep.ticks_to_target + assert pack.coverage > indep.coverage + # NOTE: we deliberately do NOT assert pack finds survivors faster. Independent + # robots blanket the map redundantly and can stumble onto victims sooner; the + # shared-memory win is *complete-area coverage* (certainty nothing is missed), + # not victim-stumble speed. Both find all survivors here; timing is reported, + # not asserted. + assert pack.found == pack.total_survivors diff --git a/dimos/experimental/pack_mind/view_rerun.py b/dimos/experimental/pack_mind/view_rerun.py new file mode 100644 index 0000000000..d974619457 --- /dev/null +++ b/dimos/experimental/pack_mind/view_rerun.py @@ -0,0 +1,116 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""View the PACK MIND A/B coverage race in the DimOS Viewer (Rerun). + +Streams both runs into one Rerun recording on a shared "tick" timeline so you can +scrub/playback and watch PACK (shared memory) pull ahead of INDEPENDENT (private). + + python -m dimos.experimental.pack_mind.view_rerun --seed 0 + python -m dimos.experimental.pack_mind.view_rerun --seed 0 --save run.rrd # no GUI +""" + +from __future__ import annotations + +import argparse +from typing import Any + +import numpy as np +import rerun as rr + +from dimos.experimental.pack_mind.render import _run_capture +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues +from dimos.visualization.rerun.init import rerun_init + +_WALL = np.array([13, 13, 20], dtype=np.uint8) +_FREE = np.array([245, 245, 245], dtype=np.uint8) +_VISITED = np.array([140, 200, 255], dtype=np.uint8) + + +def _set_tick(tick: int) -> None: + # rerun's time API name shifts across versions; support both. + try: + rr.set_time_sequence("tick", tick) # type: ignore[attr-defined] + except AttributeError: + rr.set_time("tick", sequence=tick) # type: ignore[attr-defined] + + +def _frame_rgb(grid: np.ndarray, visited: np.ndarray) -> np.ndarray: + h, w = grid.shape + img = np.empty((h, w, 3), dtype=np.uint8) + img[:] = _FREE + img[grid == CostValues.OCCUPIED] = _WALL + img[visited] = _VISITED + return np.flipud(img) # flip so +y points up in the viewer + + +def _log_run(root: str, sim: Any, frames: list[dict[str, Any]]) -> None: + grid = sim.world.grid + h = grid.shape[0] + world = sim.world + survivors = sim.survivors + + def to_img_xy(wx: float, wy: float) -> tuple[float, float]: + gc = world.world_to_grid((wx, wy)) + return (gc.x, (h - 1) - gc.y) # match the vertical flip + + for fr in frames: + _set_tick(fr["tick"]) + rr.log(f"{root}/map", rr.Image(_frame_rgb(grid, fr["visited"]))) + + spos = [to_img_xy(sx, sy) for sx, sy in survivors] + scol = [ + [60, 220, 60] if si in fr["found"] else [230, 40, 40] for si in range(len(survivors)) + ] + rr.log( + f"{root}/map/survivors", + rr.Points2D(spos, colors=scol, radii=1.6), + ) + + rpos = [to_img_xy(rx, ry) for rx, ry, _ in fr["robots"]] + rcol = [[150, 150, 150] if done else [255, 150, 0] for _, _, done in fr["robots"]] + rr.log(f"{root}/map/robots", rr.Points2D(rpos, colors=rcol, radii=1.3)) + + rr.log(f"{root}/stats", rr.TextLog(f"coverage {fr['cov']:.0%} found {len(fr['found'])}/{len(survivors)}")) + + +def view(seed: int = 0, max_ticks: int = 4000, save: str | None = None) -> None: + rerun_init(app_id="pack_mind") + if save: + rr.save(save) + else: + rr.spawn() # opens the Rerun (DimOS) viewer window + + psim, pframes = _run_capture(True, seed, max_ticks) + isim, iframes = _run_capture(False, seed, max_ticks) + _log_run("pack", psim, pframes) + _log_run("independent", isim, iframes) + + if save: + print(f"saved Rerun recording -> {save} (open with: rerun {save})") + else: + print("Rerun viewer launched. Scrub the 'tick' timeline; compare pack/ vs independent/.") + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND — view A/B race in Rerun") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--max-ticks", type=int, default=4000) + p.add_argument("--save", default=None, metavar="PATH", help="write .rrd instead of opening GUI") + return p.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + view(seed=args.seed, max_ticks=args.max_ticks, save=args.save) diff --git a/dimos/experimental/pack_mind/world.py b/dimos/experimental/pack_mind/world.py new file mode 100644 index 0000000000..35e463314e --- /dev/null +++ b/dimos/experimental/pack_mind/world.py @@ -0,0 +1,90 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — world fabrication (SAR floor as an OccupancyGrid). + +A collapsed-building floor: free space carved into rooms by walls with narrow +door gaps. Corridors/doors are kept wide enough to survive the router's safe-mask +erosion. Fully known map (no fog) — FrontierPatrolRouter sweeps unvisited *free* +cells, it does not need UNKNOWN. +""" + +from __future__ import annotations + +import numpy as np + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid + +RES = 0.1 # meters per cell +DOOR = 8 # door gap width in cells (>> erosion structure so it stays traversable) + + +def pose_at(x: float, y: float) -> PoseStamped: + """A PoseStamped at world (x, y).""" + p = PoseStamped() + p.position.x = x + p.position.y = y + return p + + +def make_maze_world(w: int = 60, h: int = 60, res: float = RES) -> OccupancyGrid: + """Free interior, OCCUPIED border + 3 vertical dividers with alternating doors. + + The alternating door placement forces a serpentine route between the four + columns — exactly where independent robots redundantly re-walk corridors and a + shared coverage memory pays off. + """ + grid = np.full((h, w), CostValues.FREE, dtype=np.int8) + grid[0, :] = grid[-1, :] = CostValues.OCCUPIED + grid[:, 0] = grid[:, -1] = CostValues.OCCUPIED + + # Vertical dividers at 1/4, 2/4, 3/4 width. Doors alternate top / bottom. + for i, x in enumerate((w // 4, w // 2, 3 * w // 4)): + grid[1:-1, x] = CostValues.OCCUPIED + if i % 2 == 0: + grid[1 : 1 + DOOR, x] = CostValues.FREE # door at top + else: + grid[-1 - DOOR : -1, x] = CostValues.FREE # door at bottom + return OccupancyGrid(grid=grid, resolution=res) + + +def free_cells_world(world: OccupancyGrid) -> list[tuple[float, float]]: + """World-coordinate centers of all FREE cells.""" + rows, cols = np.where(world.grid == CostValues.FREE) + out: list[tuple[float, float]] = [] + for r, c in zip(rows.tolist(), cols.tolist()): + wp = world.grid_to_world((int(c), int(r), 0)) + out.append((wp.x, wp.y)) + return out + + +def plant_survivors( + world: OccupancyGrid, n: int, seed: int, margin_m: float = 0.5 +) -> list[tuple[float, float]]: + """Plant n survivors on free cells, spread across the map (min pairwise dist).""" + rng = np.random.default_rng(seed) + cells = free_cells_world(world) + rng.shuffle(cells) + chosen: list[tuple[float, float]] = [] + for c in cells: + if all((c[0] - s[0]) ** 2 + (c[1] - s[1]) ** 2 >= margin_m**2 for s in chosen): + chosen.append(c) + if len(chosen) == n: + break + return chosen + + +def free_cell_count(world: OccupancyGrid) -> int: + return int(np.count_nonzero(world.grid == CostValues.FREE)) diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 3d101cca79..8792484743 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -70,6 +70,7 @@ "mid360-fastlio-voxels-native": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_voxels_native", "openarm-mock-planner-coordinator": "dimos.robot.manipulators.openarm.blueprints:openarm_mock_planner_coordinator", "openarm-planner-coordinator": "dimos.robot.manipulators.openarm.blueprints:openarm_planner_coordinator", + "pack-mind-sim": "dimos.experimental.pack_mind.blueprint:pack_mind_sim", "teleop-phone": "dimos.teleop.phone.blueprints:teleop_phone", "teleop-phone-go2": "dimos.teleop.phone.blueprints:teleop_phone_go2", "teleop-phone-go2-fleet": "dimos.teleop.phone.blueprints:teleop_phone_go2_fleet", @@ -133,6 +134,7 @@ "cost-mapper": "dimos.mapping.costmapper.CostMapper", "demo-calculator-skill": "dimos.agents.skills.demo_calculator_skill.DemoCalculatorSkill", "demo-robot": "dimos.agents.skills.demo_robot.DemoRobot", + "describer-vlm": "dimos.experimental.pack_mind.sim_perception.DescriberVLM", "desk-static-tf-module": "dimos.perception.fiducial.blueprints.desk_marker_tf.DeskStaticTfModule", "detection2-d-module": "dimos.perception.detection.module2D.Detection2DModule", "detection3-d-module": "dimos.perception.detection.module3D.Detection3DModule", @@ -181,6 +183,7 @@ "object-tracker3-d": "dimos.perception.object_tracker_3d.ObjectTracker3D", "object-tracking": "dimos.perception.object_tracker.ObjectTracking", "osm-skill": "dimos.agents.skills.osm.OsmSkill", + "pack-mind-sim-module": "dimos.experimental.pack_mind.blueprint.PackMindSimModule", "path-follower": "dimos.navigation.nav_stack.modules.path_follower.path_follower.PathFollower", "patrolling-module": "dimos.navigation.patrolling.module.PatrollingModule", "perceive-loop-skill": "dimos.perception.perceive_loop_skill.PerceiveLoopSkill", From 33d05561bc734c4e9e09253ab808c92b7de31246 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 05:43:30 -0600 Subject: [PATCH 07/35] feat: PACK MIND fog-of-war exploration engine + tests --- dimos/experimental/pack_mind/explore_sim.py | 296 ++++++++++++++++++ .../pack_mind/test_explore_sim.py | 64 ++++ 2 files changed, 360 insertions(+) create mode 100644 dimos/experimental/pack_mind/explore_sim.py create mode 100644 dimos/experimental/pack_mind/test_explore_sim.py diff --git a/dimos/experimental/pack_mind/explore_sim.py b/dimos/experimental/pack_mind/explore_sim.py new file mode 100644 index 0000000000..5dee317609 --- /dev/null +++ b/dimos/experimental/pack_mind/explore_sim.py @@ -0,0 +1,296 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — fog-of-war exploration engine (S1). + +A team of dogs explores an UNKNOWN maze. Each dog reveals cells via a raycast +sensor (walls block line of sight, RTS-style). Dogs navigate to frontiers (the +boundary between known-free and unknown). + +The A/B variable is the discovered map: + - SHARED : one known-map; every dog reveals into and plans against it. + - INDEPENDENT: each dog has its own known-map; the *team* fog is the union of + ONLINE dogs' maps — so taking a dog offline erases its territory from the + team's knowledge. In shared mode the map persists regardless. + +Pure numpy/scipy + DimOS min_cost_astar. No CUDA/ROS/sim deps. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from math import cos, hypot, sin + +import numpy as np +from numpy.typing import NDArray +from scipy.ndimage import binary_dilation, label + +from dimos.experimental.pack_mind.world import make_maze_world +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid +from dimos.navigation.replanning_a_star.min_cost_astar import min_cost_astar + +UNKNOWN = int(CostValues.UNKNOWN) +FREE = int(CostValues.FREE) +OCC = int(CostValues.OCCUPIED) + +SENSOR_R_CELLS = 11 # vision radius in cells +N_RAYS = 180 +STEP_M = 0.12 +_CROSS = np.ones((3, 3), dtype=bool) +_MIN_CLUSTER = 4 +_DOG_COLORS = ["#ff6b35", "#4dabf7", "#b197fc", "#69db7c", "#ffd43b"] + + +def _raycast_reveal( + known: NDArray[np.int8], + truth: NDArray[np.int8], + cx: int, + cy: int, + visible: NDArray[np.bool_] | None = None, +) -> None: + """Reveal cells within SENSOR_R_CELLS of (cx, cy); walls stop each ray. + + Writes discovered cells into ``known`` (persistent memory) and, if given, + marks currently-seen cells in ``visible`` (this-tick line of sight).""" + h, w = truth.shape + + def _see(gx: int, gy: int, val: int) -> None: + known[gy, gx] = val + if visible is not None: + visible[gy, gx] = True + + if 0 <= cy < h and 0 <= cx < w: + _see(cx, cy, OCC if truth[cy, cx] == OCC else FREE) + for k in range(N_RAYS): + a = 2.0 * np.pi * k / N_RAYS + dx, dy = cos(a), sin(a) + for r in range(1, SENSOR_R_CELLS + 1): + gx = int(round(cx + dx * r)) + gy = int(round(cy + dy * r)) + if not (0 <= gx < w and 0 <= gy < h): + break + if truth[gy, gx] == OCC: + _see(gx, gy, OCC) # see the wall, then vision stops + break + _see(gx, gy, FREE) + + +@dataclass +class Dog: + name: str + color: str + x: float + y: float + known: NDArray[np.int8] # shared array (pack) or private (independent) + online: bool = True + idle: bool = False + path: list[tuple[float, float]] | None = None + idx: int = 0 + trail: list[tuple[float, float]] = field(default_factory=list) + + +class ExploreSim: + def __init__( + self, + world: OccupancyGrid, + n_dogs: int, + shared: bool, + starts: list[tuple[float, float]], + seed: int = 0, + ) -> None: + np.random.seed(seed) + self.world = world + self.truth = world.grid + self.shared = shared + self.res = world.info.resolution + self.tick_n = 0 + self._total_free = int(np.count_nonzero(self.truth == FREE)) + + shape = self.truth.shape + self._visible = np.zeros(shape, dtype=bool) + if shared: + self._shared_known = np.full(shape, UNKNOWN, dtype=np.int8) + self.dogs: list[Dog] = [] + for i in range(n_dogs): + known = self._shared_known if shared else np.full(shape, UNKNOWN, dtype=np.int8) + self.dogs.append( + Dog(f"dog{i}", _DOG_COLORS[i % len(_DOG_COLORS)], starts[i][0], starts[i][1], known) + ) + + # ---- geometry helpers ---- + def _grid(self, x: float, y: float) -> tuple[int, int]: + g = self.world.world_to_grid((x, y)) + return int(g.x), int(g.y) + + # ---- fog / metrics ---- + def team_known(self) -> NDArray[np.int8]: + """What the operator sees. Shared: the one map. Independent: union over + ONLINE dogs (offline dogs' knowledge is lost to the team).""" + if self.shared: + return self._shared_known + merged = np.full(self.truth.shape, UNKNOWN, dtype=np.int8) + for d in self.dogs: + if not d.online: + continue + seen = d.known != UNKNOWN + merged[seen] = d.known[seen] + return merged + + def revealed_frac(self) -> float: + k = self.team_known() + return float(np.count_nonzero((k == FREE) & (self.truth == FREE))) / max(1, self._total_free) + + def state(self) -> dict: + """JSON-serializable snapshot for the web frontend. Per-cell code: + 0=unknown, 1=remembered-free, 2=visible-free, 3=remembered-wall, 4=visible-wall.""" + k = self.team_known() + vis = self._visible + codes = np.zeros(k.shape, dtype=np.uint8) + free, wall = k == FREE, k == OCC + codes[free] = 1 + codes[free & vis] = 2 + codes[wall] = 3 + codes[wall & vis] = 4 + h, w = k.shape + return { + "mode": "shared" if self.shared else "independent", + "tick": self.tick_n, + "revealed": round(self.revealed_frac(), 3), + "w": w, + "h": h, + "res": self.res, + "cells": codes.flatten().tolist(), + "dogs": [ + {"x": round(d.x, 3), "y": round(d.y, 3), "color": d.color, "online": d.online} + for d in self.dogs + ], + } + + # ---- exploration ---- + def _pick_goal(self, dog: Dog) -> tuple[float, float] | None: + known = dog.known + free = known == FREE + unknown = known == UNKNOWN + frontier = free & binary_dilation(unknown, structure=_CROSS) + if not frontier.any(): + return None + lab, n = label(frontier) + if n == 0: + return None + gx, gy = self._grid(dog.x, dog.y) + rows, cols = np.where(frontier) + ids = lab[rows, cols] + best_id, best_score = -1, -1.0 + for cid in range(1, n + 1): + m = ids == cid + size = int(np.count_nonzero(m)) + if size < _MIN_CLUSTER: + continue + cr, cc = rows[m].mean(), cols[m].mean() + dist = max(1.0, hypot(cc - gx, cr - gy)) + score = size / dist + if score > best_score: + best_score, best_id = score, cid + if best_id < 0: + return None + m = ids == best_id + cr, cc = rows[m], cols[m] + d2 = (cc - gx) ** 2 + (cr - gy) ** 2 + j = int(np.argmin(d2)) # nearest frontier cell in best cluster + wp = self.world.grid_to_world((int(cc[j]), int(cr[j]), 0)) + return (wp.x, wp.y) + + def _plan(self, dog: Dog, goal: tuple[float, float]) -> bool: + costmap = OccupancyGrid(grid=dog.known, resolution=self.res, frame_id="world") + path = min_cost_astar(costmap, goal, (dog.x, dog.y), unknown_penalty=1.0, use_cpp=False) + if path is None or len(path.poses) == 0: + return False + dog.path = [(p.position.x, p.position.y) for p in path.poses] + dog.idx = 0 + return True + + def _advance(self, dog: Dog) -> None: + if dog.path is None or dog.idx >= len(dog.path): + goal = self._pick_goal(dog) + if goal is None or not self._plan(dog, goal): + dog.idle = True + return + dog.idle = False + assert dog.path is not None + tx, ty = dog.path[dog.idx] + dx, dy = tx - dog.x, ty - dog.y + dist = hypot(dx, dy) + if dist <= STEP_M or dist == 0: + dog.x, dog.y = tx, ty + dog.idx += 1 + else: + dog.x += STEP_M * dx / dist + dog.y += STEP_M * dy / dist + + def set_online(self, dog_index: int, online: bool) -> None: + self.dogs[dog_index].online = online + + def step(self) -> None: + self.tick_n += 1 + self._visible = np.zeros(self.truth.shape, dtype=bool) + for d in self.dogs: + if d.online: + gx, gy = self._grid(d.x, d.y) + _raycast_reveal(d.known, self.truth, gx, gy, self._visible) + for d in self.dogs: + if d.online: + self._advance(d) + d.trail.append((d.x, d.y)) + + def all_done(self) -> bool: + return all((not d.online) or d.idle for d in self.dogs) + + def run(self, max_ticks: int = 4000, target: float = 0.95, kill: tuple[int, int] | None = None): + t_target: int | None = None + for _ in range(max_ticks): + if kill is not None and self.tick_n == kill[1]: + self.set_online(kill[0], False) + self.step() + if t_target is None and self.revealed_frac() >= target: + t_target = self.tick_n + if self.all_done(): + break + return { + "mode": "shared" if self.shared else "independent", + "ticks": self.tick_n, + "revealed": self.revealed_frac(), + "ticks_to_target": t_target, + } + + +_START_POOL = [(0.6, 0.5), (3.4, 0.5), (5.4, 0.5), (0.6, 5.4), (5.4, 5.4)] + + +def build_explore(shared: bool, seed: int = 0, n_dogs: int = 3) -> ExploreSim: + world = make_maze_world() + # Spread deployment (different entry points, realistic multi-dog SAR). Both + # modes use the SAME starts — only shared-vs-private memory differs. Spreading + # gives each dog distinct territory, so in independent mode losing a dog loses + # its discoveries (beat 4), while shared retains them. + starts = [_START_POOL[i % len(_START_POOL)] for i in range(n_dogs)] + return ExploreSim(world, n_dogs, shared, starts, seed) + + +if __name__ == "__main__": + import time + + for sh in (True, False): + t0 = time.perf_counter() + r = build_explore(shared=sh).run() + print(r, "wall=%.1fs" % (time.perf_counter() - t0)) diff --git a/dimos/experimental/pack_mind/test_explore_sim.py b/dimos/experimental/pack_mind/test_explore_sim.py new file mode 100644 index 0000000000..d0eb73b2a0 --- /dev/null +++ b/dimos/experimental/pack_mind/test_explore_sim.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — fog-of-war exploration engine tests (S1).""" + +import pytest + +from dimos.experimental.pack_mind.explore_sim import build_explore + + +@pytest.mark.unit +def test_reveal_grows() -> None: + sim = build_explore(shared=True, n_dogs=1) + for _ in range(50): + sim.step() + early = sim.revealed_frac() + for _ in range(150): + sim.step() + late = sim.revealed_frac() + assert late > early + 0.1 and late > 0.3 + + +@pytest.mark.unit +def test_shared_reveals_faster_than_independent() -> None: + s = build_explore(shared=True, seed=0).run() + i = build_explore(shared=False, seed=0).run() + assert s["ticks_to_target"] is not None + assert i["ticks_to_target"] is None or s["ticks_to_target"] < i["ticks_to_target"] + + +@pytest.mark.unit +def test_independent_loses_offline_dog_knowledge() -> None: + # INDEPENDENT: team fog = union of ONLINE dogs. Killing a dog that has + # explored unique territory drops the team's knowledge. + sim = build_explore(shared=False, n_dogs=3, seed=0) + for _ in range(140): + sim.step() + before = sim.revealed_frac() + sim.set_online(0, False) + after = sim.revealed_frac() + assert after < before - 0.1 # the dead dog's region leaves the team's knowledge + + +@pytest.mark.unit +def test_shared_keeps_offline_dog_knowledge() -> None: + # SHARED: the one map persists regardless of who is online. + sim = build_explore(shared=True, n_dogs=3, seed=0) + for _ in range(140): + sim.step() + before = sim.revealed_frac() + sim.set_online(0, False) + after = sim.revealed_frac() + assert after == before From 257d2d7d700fa0e29a75613a6106c567aa9a6f52 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 05:43:30 -0600 Subject: [PATCH 08/35] feat: PACK MIND web 3D fog-of-war demo (FastAPI + Three.js) --- dimos/experimental/pack_mind/server.py | 112 ++++++++++ .../pack_mind/static/explore.html | 194 ++++++++++++++++++ 2 files changed, 306 insertions(+) create mode 100644 dimos/experimental/pack_mind/server.py create mode 100644 dimos/experimental/pack_mind/static/explore.html diff --git a/dimos/experimental/pack_mind/server.py b/dimos/experimental/pack_mind/server.py new file mode 100644 index 0000000000..a1985998e7 --- /dev/null +++ b/dimos/experimental/pack_mind/server.py @@ -0,0 +1,112 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — web demo backend (S2). + +Streams two fog-of-war exploration sims (independent + shared) over a WebSocket to +the Three.js frontend, and accepts kill/reset controls. + + uv run python -m dimos.experimental.pack_mind.server + # then open http://localhost:8000 +""" + +from __future__ import annotations + +import asyncio +import json +import os +from pathlib import Path + +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles + +from dimos.experimental.pack_mind.explore_sim import build_explore + +_STATIC = Path(__file__).parent / "static" +_PORT = int(os.environ.get("PACK_MIND_PORT", "8000")) +_STEPS_PER_FRAME = 5 +_FPS = 18 +_HOLD_FRAMES = 70 # hold the finished maze before auto-replaying + +app = FastAPI(title="PACK MIND — Fog of War") +app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static") + + +@app.get("/") +def index() -> FileResponse: + return FileResponse(str(_STATIC / "explore.html")) + + +def _new_sims(seed: int, n_dogs: int) -> dict: + return { + "independent": build_explore(False, seed, n_dogs), + "shared": build_explore(True, seed, n_dogs), + } + + +@app.websocket("/ws") +async def ws(websocket: WebSocket) -> None: + await websocket.accept() + cfg = {"seed": 0, "n_dogs": 3} + sims = _new_sims(cfg["seed"], cfg["n_dogs"]) + controls: asyncio.Queue = asyncio.Queue() + + async def receiver() -> None: + try: + while True: + controls.put_nowait(json.loads(await websocket.receive_text())) + except Exception: + pass + + rt = asyncio.create_task(receiver()) + done_frames = 0 + try: + while True: + while not controls.empty(): + c = controls.get_nowait() + if c.get("cmd") == "reset": + cfg["seed"] = int(c.get("seed", cfg["seed"])) + sims = _new_sims(cfg["seed"], cfg["n_dogs"]) + done_frames = 0 + elif c.get("cmd") == "kill": + dog = int(c.get("dog", 0)) + for s in sims.values(): + if dog < len(s.dogs): + s.set_online(dog, False) + + if all(s.all_done() for s in sims.values()): + done_frames += 1 + if done_frames > _HOLD_FRAMES: + sims = _new_sims(cfg["seed"], cfg["n_dogs"]) + done_frames = 0 + else: + for s in sims.values(): + for _ in range(_STEPS_PER_FRAME): + if not s.all_done(): + s.step() + + await websocket.send_text(json.dumps({k: v.state() for k, v in sims.items()})) + await asyncio.sleep(1 / _FPS) + except WebSocketDisconnect: + pass + finally: + rt.cancel() + + +if __name__ == "__main__": + import uvicorn + + print(f"PACK MIND web demo → http://localhost:{_PORT}") + uvicorn.run(app, host="0.0.0.0", port=_PORT) diff --git a/dimos/experimental/pack_mind/static/explore.html b/dimos/experimental/pack_mind/static/explore.html new file mode 100644 index 0000000000..db50d72b4a --- /dev/null +++ b/dimos/experimental/pack_mind/static/explore.html @@ -0,0 +1,194 @@ + + + + + +PACK MIND — Fog of War + + + + +
+
PACK MIND · fog-of-war exploration
+ + +
connecting…
+
+
+
+
INDEPENDENT · private memory
+
+
+
+
+
PACK · shared memory
+
+
+
+
+ + + + From c41939b4ece69acc4880c016769fb7f063e555ad Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 05:43:30 -0600 Subject: [PATCH 09/35] docs: PACK MIND README for the exploration demo --- dimos/experimental/pack_mind/README.md | 107 ++++++++++++++----------- 1 file changed, 60 insertions(+), 47 deletions(-) diff --git a/dimos/experimental/pack_mind/README.md b/dimos/experimental/pack_mind/README.md index fda3121fb1..1ad73927b7 100644 --- a/dimos/experimental/pack_mind/README.md +++ b/dimos/experimental/pack_mind/README.md @@ -1,63 +1,76 @@ # PACK MIND -A shared semantic memory for a team of Unitree Go2s. -**Alpha sees. Bravo remembers. The pack acts.** +**One brain, many bodies, one memory that outlives any single dog.** -Most multi-robot systems share a *map*. Pack Mind shares *meaning*: one dog finds -something, another dog answers from that shared memory and acts on it — without any -shared map, SLAM, or coordinate exchange. Cross-dog handoff is by **zone name** only. +A team of robot dogs explores an unknown space. The question the demo answers: +**does *sharing memory* across the pack actually help?** We A/B it head-to-head — same +dogs, same start, same map; the *only* difference is whether they share one discovered +map or each keep a private one. -## What this is +## Thesis (and honest limits) -- **`conductor.py`** — the only shared layer. Roster + append-only event blackboard + - deterministic mission state machine + movement lock. Talks to each dog over MCP - JSON-RPC (HTTP). Serves the dashboard and a small action API. -- **`dashboard.html` / `static/style.css`** — the projector surface that makes the - shared memory visible: causal event cards, roster, mission state, operator controls. +- **Speed:** a pack sharing coverage/discovery memory avoids redundant re-searching, so + it searches the whole area faster. +- **Resilience (the strongest point):** when a dog goes offline, its discoveries persist + in shared memory and the survivors keep using them. With private memory, that dog's + knowledge dies with it. +- **Honest caveats:** this is a 2D-grid sim (rendered in 3D); point robots, scripted + sensing, no real SLAM/perception. The win is *complete-area search speed + knowledge + persistence* — **not** "finds victims faster" (redundant independent robots can stumble + on victims sooner). Keep the narration honest. -Each dog runs its own standard `unitree-go2-agentic` stack. The conductor is net-new. +## Two simulations -## Run it (no hardware) +### 1. Fog-of-war exploration (the current demo) — `explore_sim.py` +Unknown maze, revealed by a raycast sensor (walls block line of sight, RTS-style). Dogs +navigate to **frontiers** (known-free / unknown boundary). The discovered map is **shared** +(one map all dogs read+write) or **independent** (one per dog; the team only sees the union +of *online* dogs, so killing a dog erases its territory). Measured (seed 0, 3 dogs): +shared reaches 95% revealed at **tick 246** vs **335** independent; killing a dog drops +independent's team knowledge **0.80 → 0.32**, shared **drops 0**. -```bash -uv run python dimos/experimental/pack_mind/conductor.py --mock -# open http://localhost:8080 -``` - -Drive the whole Alpha → Bravo story from the dashboard buttons. `--mock` logs MCP -calls instead of making them, so the full demo runs on a laptop. - -## Run it (real dogs) - -Each dog, on its own compute box: +### 2. Coverage race (earlier model) — `sim.py` +Known map; dogs sweep with `CoveragePatrolRouter`; A/B = shared vs private +`VisitationHistory`. Shared hits 75% coverage at tick 1745 vs 3275; final 96.5% vs 88.9%. -```bash -dimos run unitree-go2-agentic --robot-ip --listen-host 0.0.0.0 \ - --mcp-port 9990 --viewer none --daemon -``` +Both reuse DimOS navigation primitives (`min_cost_astar`, patrol routers, +`VisitationHistory`) on a fabricated `OccupancyGrid` — pure numpy/scipy, **no CUDA/ROS/sim**. -Then the conductor, from the control machine: +## Run it ```bash -uv run python dimos/experimental/pack_mind/conductor.py \ - --dog alpha= --dog bravo= -``` +# Web 3D demo (FastAPI + Three.js) — the centerpiece. Side-by-side fog-of-war race, +# 3-level fog (visible / remembered / unknown), kill-a-dog + reset controls. +uv run python -m dimos.experimental.pack_mind.server # → http://localhost:8000 -`--listen-host 0.0.0.0` on each dog is required — the MCP server binds `127.0.0.1` -by default and the conductor would get connection-refused otherwise. +# Native DimOS blueprint — coverage A/B as two live OccupancyGrid streams in Rerun. +dimos --viewer rerun run pack-mind-sim -## Demo flow (the 6 controls) +# Standalone A/B video of the coverage race. +uv run python -m dimos.experimental.pack_mind.render --out pack_mind_ab.mp4 -1. **Start Act 1** — Alpha begins scouting. -2. **Inject: Alpha found red backpack @ Zone B** — operator fallback, identical - blackboard effect to a real VLM detection. -3. **Ask Bravo where backpack is** — Bravo answers from shared memory, not its own. -4. **Send Bravo to Zone B** — Bravo navigates to its *own* `zone_b` (movement lock held). -5. **Verify success** — Bravo self-checks at the zone. -6. **Emergency stop** — release locks, all dogs hold. - -Hidden fallback buttons are not cheating. Broken live autonomy is. +# Tests +bin/pytest-fast dimos/experimental/pack_mind/test_explore_sim.py -v +bin/pytest-fast dimos/experimental/pack_mind/test_pack_mind_sim.py -v +``` -> NOTE: MCP tool argument names (`navigate_with_text`, `speak`, `look_out_for`) are -> passed through verbatim. Verify against the skill signatures in `dimos/agents/skills/` -> on the first real-hardware test. +## File map + +| File | Role | +|---|---| +| `world.py` | Fabricate the maze `OccupancyGrid` + plant survivors | +| `explore_sim.py` | **Fog-of-war exploration engine** (raycast reveal, frontier, shared/private discovered map, offline persistence) | +| `server.py` | FastAPI WebSocket backend streaming both explore sims | +| `static/explore.html` | Three.js 3D fog-of-war frontend (side-by-side, kill/reset) | +| `sim.py` / `sim_robot.py` | Coverage-race sim (known map) | +| `blueprint.py` | `pack-mind-sim` native DimOS blueprint (publishes coverage as `OccupancyGrid`) | +| `render.py` | Standalone matplotlib → mp4 A/B render | +| `view_rerun.py` | Log the coverage A/B into a Rerun `.rrd` / live viewer | +| `demo_spike.py` | Feasibility spike (kept as a smoke test of the reused primitives) | +| `test_explore_sim.py` / `test_pack_mind_sim.py` | Engine tests | +| `sim_perception.py` | MuJoCo+VLM single-frame perception probe (optional, off the critical path) | + +### Legacy (the old "backpack handoff" demo — superseded, kept for reference) +`conductor.py`, `dashboard.html`, `static/style.css`, `sim_harness.py`, `venue_go2.sh`, +`RUNBOOK.md`, `test_conductor.py`. That demo was a single message relay dressed up as +"shared memory"; the exploration A/B above replaces it. Safe to delete in a cleanup pass. From a9b5895df389c54af7d959f461eeb96ae3e6adde Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 05:48:56 -0600 Subject: [PATCH 10/35] fix: keep PACK MIND web demo HUD/WS alive when WebGL is unavailable --- .../pack_mind/static/explore.html | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/dimos/experimental/pack_mind/static/explore.html b/dimos/experimental/pack_mind/static/explore.html index db50d72b4a..e36d6d5647 100644 --- a/dimos/experimental/pack_mind/static/explore.html +++ b/dimos/experimental/pack_mind/static/explore.html @@ -138,9 +138,22 @@ render(){ this.controls.update(); this.renderer.render(this.scene,this.camera); } } +function makeArena(id){ + try { return new Arena(document.getElementById(id)); } + catch(err){ + console.error('Arena init failed', err); + const el = document.getElementById(id); + const msg = document.createElement('div'); + msg.style.cssText = 'padding:24px;color:#ff8585;font-size:13px;line-height:1.5'; + msg.textContent = '3D unavailable on this device (needs hardware WebGL). ' + + 'Metrics still update below. Reason: ' + (err && err.message || err); + el.appendChild(msg); + return null; + } +} const arenas = { - independent: new Arena(document.getElementById('view-independent')), - shared: new Arena(document.getElementById('view-shared')), + independent: makeArena('view-independent'), + shared: makeArena('view-shared'), }; const huds = { independent: document.getElementById('hud-independent'), @@ -161,15 +174,19 @@ requestAnimationFrame(animate); if(latest){ for(const mode of ['independent','shared']){ - if(latest[mode]){ arenas[mode].update(latest[mode]); hud(mode, latest[mode]); } + if(latest[mode]){ if(arenas[mode]) arenas[mode].update(latest[mode]); hud(mode, latest[mode]); } } latest = null; } - arenas.independent.render(); arenas.shared.render(); + if(arenas.independent) arenas.independent.render(); + if(arenas.shared) arenas.shared.render(); } animate(); -addEventListener('resize', ()=>{ arenas.independent.resize(); arenas.shared.resize(); }); +addEventListener('resize', ()=>{ + if(arenas.independent) arenas.independent.resize(); + if(arenas.shared) arenas.shared.resize(); +}); // ---- websocket ---- let ws, killIdx = 0; From fde01d1536392a730b2879800075ac4db34d467b Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 06:04:34 -0600 Subject: [PATCH 11/35] feat: export PACK MIND maze to .npy for MuJoCo room-from-occupancy --- .../pack_mind/export_occupancy.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 dimos/experimental/pack_mind/export_occupancy.py diff --git a/dimos/experimental/pack_mind/export_occupancy.py b/dimos/experimental/pack_mind/export_occupancy.py new file mode 100644 index 0000000000..6f43dadce5 --- /dev/null +++ b/dimos/experimental/pack_mind/export_occupancy.py @@ -0,0 +1,50 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Export the PACK MIND maze as a .npy occupancy grid so MuJoCo can build the room. + + uv run python -m dimos.experimental.pack_mind.export_occupancy --out pack_mind_maze.npy + # then drive a Go2 through the photoreal 3D version of our maze: + dimos --simulation run unitree-go2 --mujoco-room-from-occupancy pack_mind_maze.npy + +`OccupancyGrid.from_path` loads .npy via np.load; `generate_mujoco_scene` +(dimos/mapping/occupancy/extrude_occupancy.py) extrudes occupied cells into 3D walls. +Our grid uses FREE=0 / OCCUPIED=100 / UNKNOWN=-1 (standard nav_msgs convention). +""" + +from __future__ import annotations + +import argparse + +import numpy as np + +from dimos.experimental.pack_mind.world import make_maze_world +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues + + +def main() -> None: + p = argparse.ArgumentParser(description="Export PACK MIND maze to .npy for MuJoCo") + p.add_argument("--out", default="pack_mind_maze.npy") + args = p.parse_args() + + grid = make_maze_world().grid + np.save(args.out, grid) + walls = int(np.count_nonzero(grid == CostValues.OCCUPIED)) + free = int(np.count_nonzero(grid == CostValues.FREE)) + print(f"saved {grid.shape} occupancy -> {args.out} (walls={walls}, free={free})") + print("next: dimos --simulation run unitree-go2 --mujoco-room-from-occupancy " + args.out) + + +if __name__ == "__main__": + main() From 97c55c66f857587fd874af0266ca4aff5916084e Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 06:37:05 -0600 Subject: [PATCH 12/35] feat: PACK MIND fog-of-war arena from real SLAM occupancy map Load big_office_simple_occupancy.png into a fog-of-war search arena: remap PNG values (15=free/105=wall/0=exterior), downsample, morphological close to bridge speckle, keep largest connected floor. spread_starts() picks deterministic spread deployment. build_explore_building() + server PACK_MIND_MAP=building switch. On the 27x37m office floor the shared-vs- independent gap widens (shared clears 100%, independent stalls ~39%). --- dimos/experimental/pack_mind/explore_sim.py | 25 ++++++- dimos/experimental/pack_mind/server.py | 9 ++- dimos/experimental/pack_mind/world.py | 78 +++++++++++++++++++++ 3 files changed, 107 insertions(+), 5 deletions(-) diff --git a/dimos/experimental/pack_mind/explore_sim.py b/dimos/experimental/pack_mind/explore_sim.py index 5dee317609..551030c9ed 100644 --- a/dimos/experimental/pack_mind/explore_sim.py +++ b/dimos/experimental/pack_mind/explore_sim.py @@ -29,6 +29,7 @@ from __future__ import annotations +import os from dataclasses import dataclass, field from math import cos, hypot, sin @@ -36,7 +37,11 @@ from numpy.typing import NDArray from scipy.ndimage import binary_dilation, label -from dimos.experimental.pack_mind.world import make_maze_world +from dimos.experimental.pack_mind.world import ( + load_building_world, + make_maze_world, + spread_starts, +) from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid from dimos.navigation.replanning_a_star.min_cost_astar import min_cost_astar @@ -287,10 +292,26 @@ def build_explore(shared: bool, seed: int = 0, n_dogs: int = 3) -> ExploreSim: return ExploreSim(world, n_dogs, shared, starts, seed) +def build_explore_building( + shared: bool, seed: int = 0, n_dogs: int = 3, downsample: int = 4 +) -> ExploreSim: + """ExploreSim on a real DimOS SLAM floor plan instead of the synthetic maze. + + A 27x37m office footprint (big_office_simple_occupancy.png) with long wings + and corridors. Starts are spread deterministically from the seed, so the + shared and independent runs share identical deployment — only the memory + model differs. The bigger, concave floor is where the shared-vs-independent + gap blows open (shared clears 100%, independent stalls redundantly).""" + world = load_building_world(downsample=downsample) + starts = spread_starts(world, n_dogs, seed=seed, min_dist_m=5.0) + return ExploreSim(world, n_dogs, shared, starts, seed) + + if __name__ == "__main__": import time + builder = build_explore_building if os.environ.get("PACK_MIND_MAP") == "building" else build_explore for sh in (True, False): t0 = time.perf_counter() - r = build_explore(shared=sh).run() + r = builder(shared=sh).run() print(r, "wall=%.1fs" % (time.perf_counter() - t0)) diff --git a/dimos/experimental/pack_mind/server.py b/dimos/experimental/pack_mind/server.py index a1985998e7..4f8937c3ba 100644 --- a/dimos/experimental/pack_mind/server.py +++ b/dimos/experimental/pack_mind/server.py @@ -32,7 +32,10 @@ from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles -from dimos.experimental.pack_mind.explore_sim import build_explore +from dimos.experimental.pack_mind.explore_sim import build_explore, build_explore_building + +# PACK_MIND_MAP=building swaps the synthetic maze for a real DimOS SLAM floor plan. +_BUILD = build_explore_building if os.environ.get("PACK_MIND_MAP") == "building" else build_explore _STATIC = Path(__file__).parent / "static" _PORT = int(os.environ.get("PACK_MIND_PORT", "8000")) @@ -51,8 +54,8 @@ def index() -> FileResponse: def _new_sims(seed: int, n_dogs: int) -> dict: return { - "independent": build_explore(False, seed, n_dogs), - "shared": build_explore(True, seed, n_dogs), + "independent": _BUILD(False, seed, n_dogs), + "shared": _BUILD(True, seed, n_dogs), } diff --git a/dimos/experimental/pack_mind/world.py b/dimos/experimental/pack_mind/world.py index 35e463314e..a4cf8d6dd9 100644 --- a/dimos/experimental/pack_mind/world.py +++ b/dimos/experimental/pack_mind/world.py @@ -23,13 +23,22 @@ from __future__ import annotations import numpy as np +from PIL import Image +from scipy.ndimage import binary_dilation, binary_erosion, label from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid +from dimos.utils.data import get_data RES = 0.1 # meters per cell DOOR = 8 # door gap width in cells (>> erosion structure so it stays traversable) +# Pixel-value semantics of the DimOS SLAM occupancy PNGs (e.g. +# big_office_simple_occupancy.png). `OccupancyGrid.from_path` does NOT remap +# these to CostValues, so we interpret them here. +_PNG_FREE_VAL = 15 # mapped interior floor +_PNG_WALL_VAL = 105 # walls / obstacles (the 0 background is the building exterior) + def pose_at(x: float, y: float) -> PoseStamped: """A PoseStamped at world (x, y).""" @@ -60,6 +69,75 @@ def make_maze_world(w: int = 60, h: int = 60, res: float = RES) -> OccupancyGrid return OccupancyGrid(grid=grid, resolution=res) +def load_building_world( + name: str = "big_office_simple_occupancy.png", + downsample: int = 4, + res_full: float = 0.05, + close_iters: int = 1, +) -> OccupancyGrid: + """Turn a real DimOS SLAM occupancy PNG into a fog-of-war search arena. + + The raw PNG is a *visualization* (pixel values 15=free, 105=wall, 0=exterior), + not a CostValues grid, so we remap by value. We downsample to keep the sim + fast and make the fixed sensor radius proportionate to the floor size. + + Structure comes from the *building footprint*, not interior pixels: the free + interior is majority-pooled per block, morphologically closed to bridge SLAM + speckle (which would otherwise sever real corridors), then reduced to its + largest connected component. Everything outside the footprint is OCCUPIED, so + the building's own concave outline — wings, corridors, dead-ends — is what + blocks line-of-sight. That concavity is what makes shared-vs-independent fog + diverge; the sparse interior wall dots are SLAM noise and are deliberately + not carved back in (dilating them disconnects the floor). + """ + raw = np.array(Image.open(get_data(name)).convert("L")) + ds = max(1, downsample) + h, w = raw.shape + hc, wc = h // ds, w // ds + raw = raw[: hc * ds, : wc * ds] + + free = (raw == _PNG_FREE_VAL).reshape(hc, ds, wc, ds).mean(axis=(1, 3)) > 0.5 + if close_iters > 0: # bridge speckle gaps so corridors stay connected + free = binary_dilation(free, iterations=close_iters) + free = binary_erosion(free, iterations=close_iters, border_value=1) + + lab, n = label(free) # keep the largest navigable floor, drop outliers + if n > 1: + sizes = np.bincount(lab.ravel()) + sizes[0] = 0 + free = lab == int(sizes.argmax()) + + grid = np.full((hc, wc), CostValues.OCCUPIED, dtype=np.int8) + grid[free] = CostValues.FREE + return OccupancyGrid(grid=grid, resolution=res_full * ds, frame_id="world") + + +def spread_starts( + world: OccupancyGrid, n: int, seed: int = 0, min_dist_m: float = 1.0 +) -> list[tuple[float, float]]: + """Pick n free-cell start positions, greedily maximizing pairwise spread. + + Replaces the hand-tuned _START_POOL when the arena geometry is data-driven.""" + rng = np.random.default_rng(seed) + cells = free_cells_world(world) + if not cells: + raise ValueError("world has no FREE cells") + rng.shuffle(cells) + chosen: list[tuple[float, float]] = [cells[0]] + for c in cells[1:]: + if all((c[0] - s[0]) ** 2 + (c[1] - s[1]) ** 2 >= min_dist_m**2 for s in chosen): + chosen.append(c) + if len(chosen) == n: + break + # If the arena was too cramped to find n spread points, pad with any free cells. + for c in cells: + if len(chosen) >= n: + break + if c not in chosen: + chosen.append(c) + return chosen[:n] + + def free_cells_world(world: OccupancyGrid) -> list[tuple[float, float]]: """World-coordinate centers of all FREE cells.""" rows, cols = np.where(world.grid == CostValues.FREE) From da227e62b3c47843156aff94c9c4c7bec70483a2 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 06:37:05 -0600 Subject: [PATCH 13/35] fix: scale PACK MIND web demo fog/camera to arena span Fixed Fog(60,140) blacked out the larger building floor (camera ~274u from target, beyond fog far=140). Scale fog near/far by span; read cell res from state instead of assuming 0.1m so dogs land on the right cells. --- dimos/experimental/pack_mind/static/explore.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dimos/experimental/pack_mind/static/explore.html b/dimos/experimental/pack_mind/static/explore.html index e36d6d5647..a3899f6b8f 100644 --- a/dimos/experimental/pack_mind/static/explore.html +++ b/dimos/experimental/pack_mind/static/explore.html @@ -99,11 +99,15 @@ this.walls = new THREE.InstancedMesh(new THREE.BoxGeometry(0.96,1.6,0.96), wmat, N); this.scene.add(this.walls); const cx=w/2, cz=h/2, span=Math.max(w,h); + // Fog must scale with arena span — fixed near/far (tuned for the 60-cell + // maze) fully obscured the larger building floor, blacking out the canvas. + this.scene.fog.near = span*0.55; this.scene.fog.far = span*2.6; this.camera.position.set(cx+span*0.05, span*1.05, cz+span*1.05); this.controls.target.set(cx,0,cz); this.controls.update(); } update(state){ if(!this.floor) this.initGrid(state.w, state.h); + this.res = state.res || this.res; // grid is data-driven; don't assume 0.1m cells const {w,cells} = state; let wi=0; for(let i=0;i Date: Wed, 27 May 2026 22:54:58 -0600 Subject: [PATCH 14/35] fix: select LocalAP WebRTC handshake for Go2 AP-mode IP When joined to the robot's own WiFi hotspot the gateway is always 192.168.12.1 and the peer expects the LocalAP handshake (empty SDP id). The previous hardcoded LocalSTA method sends id="STA_localNetwork", which some firmware rejects in AP mode, causing the WebRTC offer to hang with no answer. Auto-select LocalAP for the AP gateway IP, keep LocalSTA otherwise. --- dimos/robot/unitree/connection.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dimos/robot/unitree/connection.py b/dimos/robot/unitree/connection.py index 44101cc19d..f6aaacbc5e 100644 --- a/dimos/robot/unitree/connection.py +++ b/dimos/robot/unitree/connection.py @@ -93,12 +93,22 @@ def to_ndarray(self, format=None): # type: ignore[no-untyped-def] class UnitreeWebRTCConnection(Resource): _SPORT_API_ID_RAGEMODE: int = 2059 + _AP_GATEWAY_IP: str = "192.168.12.1" + def __init__(self, ip: str, mode: str = "ai") -> None: self.ip = ip self.mode = mode self.stop_timer: threading.Timer | None = None self.cmd_vel_timeout = 0.2 - self.conn = LegionConnection(WebRTCConnectionMethod.LocalSTA, ip=self.ip) + # When joined to the robot's own WiFi hotspot the gateway is always + # 192.168.12.1 and the peer expects the LocalAP handshake (empty SDP id). + # Any other IP means the robot is on a shared LAN → LocalSTA. + method = ( + WebRTCConnectionMethod.LocalAP + if self.ip == self._AP_GATEWAY_IP + else WebRTCConnectionMethod.LocalSTA + ) + self.conn = LegionConnection(method, ip=self.ip) self.connect() def connect(self) -> None: From 8621ca1dd35e3302d9b3d2023ca8725b96054bb7 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 22:55:14 -0600 Subject: [PATCH 15/35] fix: expose message classes in 'dimos topic send' eval namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dimos.msgs packages declare __all__ = [], so the previous getattr(module, "__all__", dir(module)) resolved to an empty list (present but falsy) and injected nothing into the eval context — every expression failed with "name 'Twist' is not defined". Walk same-named submodules (geometry_msgs.Twist.Twist, etc.) when __all__ is empty so message classes are available to the expression. --- dimos/robot/cli/topic.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/dimos/robot/cli/topic.py b/dimos/robot/cli/topic.py index e4dce1f139..3b8df8929c 100644 --- a/dimos/robot/cli/topic.py +++ b/dimos/robot/cli/topic.py @@ -15,6 +15,7 @@ from __future__ import annotations import importlib +import pkgutil import re import time @@ -120,14 +121,30 @@ def topic_send(topic: str, message_expr: str) -> None: for module_name in modules_to_import: try: module = importlib.import_module(module_name) - for name in getattr(module, "__all__", dir(module)): - if not name.startswith("_"): - obj = getattr(module, name, None) - if obj is not None: - eval_context[name] = obj except ImportError: continue + names = [n for n in getattr(module, "__all__", None) or [] if not n.startswith("_")] + for name in names: + obj = getattr(module, name, None) + if obj is not None: + eval_context[name] = obj + + # msgs packages declare __all__ = [] and keep each class in a same-named + # submodule (e.g. geometry_msgs.Twist.Twist). Walk submodules so the + # eval namespace gets the actual message classes. + if not names and hasattr(module, "__path__"): + for info in pkgutil.iter_modules(module.__path__): + if info.name.startswith("_"): + continue + try: + sub = importlib.import_module(f"{module_name}.{info.name}") + except ImportError: + continue + cls = getattr(sub, info.name, None) + if cls is not None: + eval_context[info.name] = cls + try: message = eval(message_expr, eval_context) except Exception as e: From 65c944bb8a922f2b57861ca36937e0b50b13f030 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Wed, 27 May 2026 22:55:14 -0600 Subject: [PATCH 16/35] test: add headless cmd_vel drive script for live Go2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keyboard teleop needs a pygame window (x11/cocoa), unavailable over SSH or headless. demo_drive_go2.py publishes Twist on /cmd_vel at a fixed rate so a running relay blueprint's ControlCoordinator forwards motion to the robot over WebRTC — no GUI required. Useful for live-hardware bring-up and demos. --- .../experimental/pack_mind/demo_drive_go2.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 dimos/experimental/pack_mind/demo_drive_go2.py diff --git a/dimos/experimental/pack_mind/demo_drive_go2.py b/dimos/experimental/pack_mind/demo_drive_go2.py new file mode 100644 index 0000000000..acb09c54e9 --- /dev/null +++ b/dimos/experimental/pack_mind/demo_drive_go2.py @@ -0,0 +1,80 @@ +# Copyright 2025 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Headless cmd_vel choreography for a live Go2 — no GUI / pygame required. + +The keyboard teleop blueprint needs an X11/Cocoa window for pygame, which is +unavailable over SSH or in a headless session. This script drives the dog by +publishing Twist messages straight onto the /cmd_vel LCM topic at a fixed rate, +exactly like KeyboardTeleop does — the running blueprint's ControlCoordinator +relays them to the robot over WebRTC. + +Run a relay blueprint in one terminal (it holds the WebRTC connection and the +ControlCoordinator; the pygame crash in its log is harmless): + + uv run dimos --robot-ip 192.168.12.1 run unitree-go2-webrtc-keyboard-teleop + +Then, once it logs "ControlCoordinator started", in a second terminal on the +SAME machine (LCM is host-local multicast): + + uv run python dimos/experimental/pack_mind/demo_drive_go2.py +""" + +from __future__ import annotations + +import time + +from dimos.core.transport import LCMTransport +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.geometry_msgs.Vector3 import Vector3 + +CMD_VEL_TOPIC = "/cmd_vel" +DEFAULT_RATE_HZ = 10.0 + + +def drive(transport: LCMTransport[Twist], vx: float, wz: float, secs: float, hz: float = DEFAULT_RATE_HZ) -> None: + """Hold a velocity for `secs` by republishing at `hz`. + + vx: forward m/s (positive forward, negative back). + wz: yaw rad/s (positive left, negative right). + Republishing is required: the connection layer auto-stops 0.2s after the + last command, so a single publish only produces a brief nudge. + """ + deadline = time.time() + secs + period = 1.0 / hz + while time.time() < deadline: + transport.broadcast( + None, + Twist( + linear=Vector3(x=vx, y=0.0, z=0.0), + angular=Vector3(x=0.0, y=0.0, z=wz), + ), + ) + time.sleep(period) + + +def main() -> None: + transport: LCMTransport[Twist] = LCMTransport(CMD_VEL_TOPIC, Twist) + + # Choreographed routine. Tune speeds/durations/order for your space. + # Start conservative: clear ~2m, keep the e-stop within reach. + drive(transport, vx=0.0, wz=0.0, secs=0.5) # settle + drive(transport, vx=0.3, wz=0.0, secs=2.0) # forward + drive(transport, vx=0.0, wz=0.5, secs=2.0) # turn left in place + drive(transport, vx=0.3, wz=0.0, secs=2.0) # forward + drive(transport, vx=0.0, wz=0.0, secs=0.5) # stop + + +if __name__ == "__main__": + main() From 63c9d4b03b582e9e930ba1745fbee3525333b1da Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 00:08:38 -0600 Subject: [PATCH 17/35] feat: PACK MIND frontier de-confliction + object search + Rerun viewer - explore_sim: shared-map frontier de-confliction so the pack fans out instead of re-walking each other's ground; plant a search target at the farthest free cell, detect-on-sight, and converge-on-found - view_explore_rerun: DimOS Viewer (Rerun) view of the maze/office search with shared-vs-independent A/B and kill-a-dog resilience - README: document the viewer commands --- .gitignore | 1 + dimos/experimental/pack_mind/README.md | 5 + dimos/experimental/pack_mind/explore_sim.py | 98 +++++- .../pack_mind/view_explore_rerun.py | 291 ++++++++++++++++++ 4 files changed, 390 insertions(+), 5 deletions(-) create mode 100644 dimos/experimental/pack_mind/view_explore_rerun.py diff --git a/.gitignore b/.gitignore index 351c408306..d645547c00 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,4 @@ htmlcov/ # Memory2 autorecord recording*.db .scratch/ +.gstack/ diff --git a/dimos/experimental/pack_mind/README.md b/dimos/experimental/pack_mind/README.md index 1ad73927b7..4de50ab4fa 100644 --- a/dimos/experimental/pack_mind/README.md +++ b/dimos/experimental/pack_mind/README.md @@ -43,6 +43,10 @@ Both reuse DimOS navigation primitives (`min_cost_astar`, patrol routers, # 3-level fog (visible / remembered / unknown), kill-a-dog + reset controls. uv run python -m dimos.experimental.pack_mind.server # → http://localhost:8000 +# DimOS Viewer (Rerun) — one maze, the pack exploring on the shared map. Scrub the +# "tick" timeline; add --kill DOG TICK for the resilience beat, --save run.rrd to skip the GUI. +uv run python -m dimos.experimental.pack_mind.view_explore_rerun --dogs 3 --seed 0 + # Native DimOS blueprint — coverage A/B as two live OccupancyGrid streams in Rerun. dimos --viewer rerun run pack-mind-sim @@ -62,6 +66,7 @@ bin/pytest-fast dimos/experimental/pack_mind/test_pack_mind_sim.py -v | `explore_sim.py` | **Fog-of-war exploration engine** (raycast reveal, frontier, shared/private discovered map, offline persistence) | | `server.py` | FastAPI WebSocket backend streaming both explore sims | | `static/explore.html` | Three.js 3D fog-of-war frontend (side-by-side, kill/reset) | +| `view_explore_rerun.py` | **DimOS Viewer (Rerun)** view of one shared-memory maze search (fog + dogs + trails + coverage scalar) | | `sim.py` / `sim_robot.py` | Coverage-race sim (known map) | | `blueprint.py` | `pack-mind-sim` native DimOS blueprint (publishes coverage as `OccupancyGrid`) | | `render.py` | Standalone matplotlib → mp4 A/B render | diff --git a/dimos/experimental/pack_mind/explore_sim.py b/dimos/experimental/pack_mind/explore_sim.py index 551030c9ed..06d8d7e708 100644 --- a/dimos/experimental/pack_mind/explore_sim.py +++ b/dimos/experimental/pack_mind/explore_sim.py @@ -31,13 +31,14 @@ import os from dataclasses import dataclass, field -from math import cos, hypot, sin +from math import cos, exp, hypot, sin import numpy as np from numpy.typing import NDArray from scipy.ndimage import binary_dilation, label from dimos.experimental.pack_mind.world import ( + free_cells_world, load_building_world, make_maze_world, spread_starts, @@ -102,6 +103,7 @@ class Dog: idle: bool = False path: list[tuple[float, float]] | None = None idx: int = 0 + goal: tuple[float, float] | None = None # current frontier target (for pack de-confliction) trail: list[tuple[float, float]] = field(default_factory=list) @@ -113,6 +115,9 @@ def __init__( shared: bool, starts: list[tuple[float, float]], seed: int = 0, + target: tuple[float, float] | None = None, + target_label: str = "target", + converge_on_found: bool = False, ) -> None: np.random.seed(seed) self.world = world @@ -122,6 +127,16 @@ def __init__( self.tick_n = 0 self._total_free = int(np.count_nonzero(self.truth == FREE)) + # Search target ("find me a red object"). Detected when its cell enters any + # online dog's line of sight; the find is written to shared memory (found/ + # found_by/found_tick) so the whole pack knows at once. + self.target_xy = target + self.target_label = target_label + self.converge_on_found = converge_on_found + self.found = False + self.found_by: str | None = None + self.found_tick: int | None = None + shape = self.truth.shape self._visible = np.zeros(shape, dtype=bool) if shared: @@ -180,10 +195,26 @@ def state(self) -> dict: {"x": round(d.x, 3), "y": round(d.y, 3), "color": d.color, "online": d.online} for d in self.dogs ], + "target": ( + { + "x": round(self.target_xy[0], 3), + "y": round(self.target_xy[1], 3), + "label": self.target_label, + "found": self.found, + "found_by": self.found_by, + "found_tick": self.found_tick, + } + if self.target_xy is not None + else None + ), } # ---- exploration ---- def _pick_goal(self, dog: Dog) -> tuple[float, float] | None: + # Object found: the pack already knows (shared memory) — converge on it + # instead of continuing to clear frontiers. "One sees it, all know." + if self.found and self.converge_on_found and self.target_xy is not None: + return self.target_xy known = dog.known free = known == FREE unknown = known == UNKNOWN @@ -194,6 +225,14 @@ def _pick_goal(self, dog: Dog) -> tuple[float, float] | None: if n == 0: return None gx, gy = self._grid(dog.x, dog.y) + # Frontiers other online dogs are already heading for. The shared map lets + # the pack de-conflict: a cluster near a teammate's target is down-weighted + # so the dogs fan out instead of all chasing the same boundary. + claimed = [ + self._grid(*o.goal) + for o in self.dogs + if o is not dog and o.online and o.goal is not None + ] rows, cols = np.where(frontier) ids = lab[rows, cols] best_id, best_score = -1, -1.0 @@ -205,6 +244,9 @@ def _pick_goal(self, dog: Dog) -> tuple[float, float] | None: cr, cc = rows[m].mean(), cols[m].mean() dist = max(1.0, hypot(cc - gx, cr - gy)) score = size / dist + if claimed: + d_other = min(hypot(cc - ox, cr - oy) for ox, oy in claimed) + score *= 1.0 - 0.85 * exp(-(d_other**2) / (2.0 * SENSOR_R_CELLS**2)) if score > best_score: best_score, best_id = score, cid if best_id < 0: @@ -230,7 +272,9 @@ def _advance(self, dog: Dog) -> None: goal = self._pick_goal(dog) if goal is None or not self._plan(dog, goal): dog.idle = True + dog.goal = None return + dog.goal = goal dog.idle = False assert dog.path is not None tx, ty = dog.path[dog.idx] @@ -253,11 +297,27 @@ def step(self) -> None: if d.online: gx, gy = self._grid(d.x, d.y) _raycast_reveal(d.known, self.truth, gx, gy, self._visible) + self._check_target() for d in self.dogs: if d.online: self._advance(d) d.trail.append((d.x, d.y)) + def _check_target(self) -> None: + """Mark the target found the tick its cell first enters any dog's sight.""" + if self.target_xy is None or self.found: + return + tgx, tgy = self._grid(*self.target_xy) + h, w = self._visible.shape + if not (0 <= tgy < h and 0 <= tgx < w) or not self._visible[tgy, tgx]: + return + self.found = True + self.found_tick = self.tick_n + online = [d for d in self.dogs if d.online] + if online: + tx, ty = self.target_xy + self.found_by = min(online, key=lambda d: (d.x - tx) ** 2 + (d.y - ty) ** 2).name + def all_done(self) -> bool: return all((not d.online) or d.idle for d in self.dogs) @@ -282,18 +342,41 @@ def run(self, max_ticks: int = 4000, target: float = 0.95, kill: tuple[int, int] _START_POOL = [(0.6, 0.5), (3.4, 0.5), (5.4, 0.5), (0.6, 5.4), (5.4, 5.4)] -def build_explore(shared: bool, seed: int = 0, n_dogs: int = 3) -> ExploreSim: +def _far_target(world: OccupancyGrid, starts: list[tuple[float, float]]) -> tuple[float, float]: + """Plant the object at the free cell farthest from every start, so it's only + found after real searching — not next to where a dog spawns.""" + cells = free_cells_world(world) + return max(cells, key=lambda c: min((c[0] - s[0]) ** 2 + (c[1] - s[1]) ** 2 for s in starts)) + + +def build_explore( + shared: bool, + seed: int = 0, + n_dogs: int = 3, + target_label: str | None = None, + converge_on_found: bool = False, +) -> ExploreSim: world = make_maze_world() # Spread deployment (different entry points, realistic multi-dog SAR). Both # modes use the SAME starts — only shared-vs-private memory differs. Spreading # gives each dog distinct territory, so in independent mode losing a dog loses # its discoveries (beat 4), while shared retains them. starts = [_START_POOL[i % len(_START_POOL)] for i in range(n_dogs)] - return ExploreSim(world, n_dogs, shared, starts, seed) + target = _far_target(world, starts) if target_label else None + return ExploreSim( + world, n_dogs, shared, starts, seed, + target=target, target_label=target_label or "target", + converge_on_found=converge_on_found, + ) def build_explore_building( - shared: bool, seed: int = 0, n_dogs: int = 3, downsample: int = 4 + shared: bool, + seed: int = 0, + n_dogs: int = 3, + downsample: int = 4, + target_label: str | None = None, + converge_on_found: bool = False, ) -> ExploreSim: """ExploreSim on a real DimOS SLAM floor plan instead of the synthetic maze. @@ -304,7 +387,12 @@ def build_explore_building( gap blows open (shared clears 100%, independent stalls redundantly).""" world = load_building_world(downsample=downsample) starts = spread_starts(world, n_dogs, seed=seed, min_dist_m=5.0) - return ExploreSim(world, n_dogs, shared, starts, seed) + target = _far_target(world, starts) if target_label else None + return ExploreSim( + world, n_dogs, shared, starts, seed, + target=target, target_label=target_label or "target", + converge_on_found=converge_on_found, + ) if __name__ == "__main__": diff --git a/dimos/experimental/pack_mind/view_explore_rerun.py b/dimos/experimental/pack_mind/view_explore_rerun.py new file mode 100644 index 0000000000..782fe8f6a8 --- /dev/null +++ b/dimos/experimental/pack_mind/view_explore_rerun.py @@ -0,0 +1,291 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Watch a pack of dogs search a maze with PACK MIND in the DimOS Viewer (Rerun). + +One maze, one shared discovered map. Every dog reveals into it via a raycast +sensor (walls block line of sight) and plans toward frontiers; the shared map +de-conflicts them so the pack fans out instead of re-walking each other's ground. +Scrub the "tick" timeline to watch the fog peel back, and drop ``maze/revealed`` +into a time-series view to see coverage climb to 100%. + + python -m dimos.experimental.pack_mind.view_explore_rerun + python -m dimos.experimental.pack_mind.view_explore_rerun --dogs 3 --seed 0 + python -m dimos.experimental.pack_mind.view_explore_rerun --kill 2 120 # offline dog2 at tick 120 + python -m dimos.experimental.pack_mind.view_explore_rerun --save run.rrd # headless, no GUI +""" + +from __future__ import annotations + +import argparse +from typing import Any + +import numpy as np +import rerun as rr + +from dimos.experimental.pack_mind.explore_sim import ( + ExploreSim, + build_explore, + build_explore_building, +) +from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid +from dimos.visualization.rerun.init import rerun_init + +# Per-cell fog palette, indexed by state() code: +# 0 unknown · 1 remembered-free · 2 visible-free · 3 remembered-wall · 4 visible-wall +_PALETTE = np.array( + [ + [12, 12, 20], # 0 unknown — the fog + [64, 80, 110], # 1 remembered free (out of current line of sight) + [205, 228, 255], # 2 visible free (currently sensed) + [38, 40, 58], # 3 remembered wall + [150, 130, 95], # 4 visible wall + ], + dtype=np.uint8, +) +_OFFLINE = [110, 110, 110] +_SNAP_EVERY = 4 # capture a frame every N ticks (keeps playback smooth + .rrd small) + + +def _set_tick(tick: int) -> None: + # rerun's time API name shifts across versions; support both. + try: + rr.set_time_sequence("tick", tick) # type: ignore[attr-defined] + except AttributeError: + rr.set_time("tick", sequence=tick) # type: ignore[attr-defined] + + +def _hex_rgb(hex_color: str) -> list[int]: + h = hex_color.lstrip("#") + return [int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16)] + + +def _codes_rgb(codes: np.ndarray) -> np.ndarray: + """Map an (h, w) array of fog codes to an RGB image, flipped so +y points up.""" + return np.flipud(_PALETTE[codes]) + + +def _build(map_name: str, n_dogs: int, seed: int, target_label: str, shared: bool) -> ExploreSim: + """Pack on the chosen arena, converging once the object is seen. + + shared=True is PACK MIND (one shared discovered map); shared=False gives each + dog a private map (the A/B baseline that re-walks each other's ground).""" + builder = build_explore_building if map_name == "office" else build_explore + return builder( + shared=shared, seed=seed, n_dogs=n_dogs, target_label=target_label, converge_on_found=True + ) + + +def _capture(sim: ExploreSim, max_ticks: int, kill: tuple[int, int] | None) -> list[dict[str, Any]]: + frames: list[dict[str, Any]] = [] + + def snap() -> None: + st = sim.state() + codes = np.array(st["cells"], dtype=np.uint8).reshape(st["h"], st["w"]) + frames.append( + { + "tick": sim.tick_n, + "codes": codes, + "revealed": st["revealed"], + "dogs": [(d.x, d.y, d.color, d.online) for d in sim.dogs], + "trails": [list(d.trail) for d in sim.dogs], + "target": st["target"], + } + ) + + snap() + for _ in range(max_ticks): + if kill is not None and sim.tick_n == kill[1]: + sim.set_online(kill[0], False) + sim.step() + if sim.tick_n % _SNAP_EVERY == 0: + snap() + if sim.all_done(): + break + snap() + return frames + + +def _log_run(root: str, metric: str, sim: ExploreSim, frames: list[dict[str, Any]]) -> None: + h = frames[0]["codes"].shape[0] + world = sim.world + label = sim.target_label + + def to_img_xy(wx: float, wy: float) -> tuple[float, float]: + g = world.world_to_grid((wx, wy)) + return (float(g.x), float((h - 1) - g.y)) # match the vertical flip + + for fr in frames: + _set_tick(fr["tick"]) + rr.log(f"{root}/fog", rr.Image(_codes_rgb(fr["codes"]))) + + pts = [to_img_xy(x, y) for x, y, _, _ in fr["dogs"]] + cols = [_hex_rgb(color) if online else _OFFLINE for _, _, color, online in fr["dogs"]] + rr.log(f"{root}/fog/dogs", rr.Points2D(pts, colors=cols, radii=1.6)) + + strips, scols = [], [] + for (_, _, color, _), trail in zip(fr["dogs"], fr["trails"]): + if len(trail) > 1: + strips.append([to_img_xy(px, py) for px, py in trail]) + scols.append(_hex_rgb(color)) + if strips: + rr.log(f"{root}/fog/trails", rr.LineStrips2D(strips, colors=scols, radii=0.35)) + + tgt = fr["target"] + if tgt is not None: + tcol = [[60, 220, 90]] if tgt["found"] else [[235, 40, 40]] + rr.log(f"{root}/fog/target", rr.Points2D([to_img_xy(tgt["x"], tgt["y"])], colors=tcol, radii=2.4)) + + rr.log(metric, rr.Scalars(fr["revealed"])) + if tgt is not None and tgt["found"]: + msg = f"FOUND {label} — {tgt['found_by']} @ tick {tgt['found_tick']} · searched {fr['revealed']:.0%}" + else: + msg = f"searching for {label} … searched {fr['revealed']:.0%} · tick {fr['tick']}" + rr.log(f"{root}/stats", rr.TextLog(msg)) + + +def _log_run_3d(root: str, metric: str, sim: ExploreSim, frames: list[dict[str, Any]]) -> None: + """Log the run as 3D world entities over the occupancy floor (SLAM-map look). + + Explored cells light up as Points3D on the z=0 floor (unknown stays dark, so + fog peels back in 3D); dogs and trails ride just above. Dog/trail/target + coordinates are already in world meters; floor cells map via origin + grid*res. + """ + world = sim.world + res = float(world.resolution) + origin = world.origin + label = sim.target_label + + for fr in frames: + _set_tick(fr["tick"]) + codes = fr["codes"] + # Reveal the real occupancy through the fog, rendered by the SAME textured + # -quad path the live SLAM map uses (OccupancyGrid.to_rerun) so it matches + # `dimos run unitree-go2`: unknown stays transparent, explored free -> 0, + # explored wall -> 100. The floor texture grows as the pack searches. + fog_cost = np.full(codes.shape, -1, dtype=np.int8) + fog_cost[(codes == 1) | (codes == 2)] = 0 + fog_cost[(codes == 3) | (codes == 4)] = 100 + revealed = OccupancyGrid(grid=fog_cost, resolution=res, origin=origin) + rr.log(f"{root}/map", revealed.to_rerun()) + + dpts = [[x, y, 0.25] for x, y, _, _ in fr["dogs"]] + dcols = [_hex_rgb(color) if online else _OFFLINE for _, _, color, online in fr["dogs"]] + rr.log(f"{root}/dogs", rr.Points3D(dpts, colors=dcols, radii=res * 2.5)) + + strips, scols = [], [] + for (_, _, color, _), trail in zip(fr["dogs"], fr["trails"]): + if len(trail) > 1: + strips.append([[px, py, 0.1] for px, py in trail]) + scols.append(_hex_rgb(color)) + if strips: + rr.log(f"{root}/trails", rr.LineStrips3D(strips, colors=scols, radii=res * 0.6)) + + tgt = fr["target"] + if tgt is not None: + tcol = [[60, 220, 90]] if tgt["found"] else [[235, 40, 40]] + rr.log(f"{root}/target", rr.Points3D([[tgt["x"], tgt["y"], 0.3]], colors=tcol, radii=res * 3.0)) + + rr.log(metric, rr.Scalars(fr["revealed"])) + if tgt is not None and tgt["found"]: + msg = f"FOUND {label} — {tgt['found_by']} @ tick {tgt['found_tick']} · searched {fr['revealed']:.0%}" + else: + msg = f"searching for {label} … searched {fr['revealed']:.0%} · tick {fr['tick']}" + rr.log(f"{root}/stats", rr.TextLog(msg)) + + +def view( + map_name: str = "maze", + n_dogs: int = 3, + seed: int = 0, + target_label: str = "red object", + max_ticks: int = 4000, + kill: tuple[int, int] | None = None, + save: str | None = None, + shared: bool = True, + threed: bool = False, +) -> None: + mode = "shared" if shared else "independent" + rerun_init(app_id=f"pack_mind_explore_{mode}") + if save: + rr.save(save) + else: + rr.spawn() # opens the DimOS Viewer (Rerun) window + + sim = _build(map_name, n_dogs, seed, target_label, shared) + frames = _capture(sim, max_ticks, kill) + logger = _log_run_3d if threed else _log_run + logger("maze", "maze/revealed", sim, frames) + + tgt = sim.state()["target"] + if tgt is not None and tgt["found"]: + print( + f"{n_dogs} dogs searched the {map_name}; {tgt['found_by']} saw the {target_label} " + f"at tick {tgt['found_tick']} (searched {sim.revealed_frac():.0%})." + ) + else: + print( + f"{n_dogs} dogs searched {sim.revealed_frac():.0%} of the {map_name} in " + f"{sim.tick_n} ticks; {target_label} not seen." + ) + if save: + print(f"saved Rerun recording -> {save} (open with: rerun {save})") + else: + print("DimOS Viewer launched. Scrub the 'tick' timeline to watch the pack search.") + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND — a pack searches for an object, viewed in Rerun") + p.add_argument("--map", choices=("maze", "office"), default="maze", help="search arena") + p.add_argument("--dogs", type=int, default=3, help="number of dogs in the pack") + p.add_argument("--target", default="red object", help="object descriptor to search for") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--max-ticks", type=int, default=4000) + p.add_argument( + "--kill", + nargs=2, + type=int, + default=None, + metavar=("DOG", "TICK"), + help="take DOG offline at TICK (resilience beat); shared keeps its searched area", + ) + p.add_argument("--save", default=None, metavar="PATH", help="write .rrd instead of opening GUI") + p.add_argument( + "--independent", + action="store_true", + help="give each dog a private map (A/B baseline); default is PACK MIND shared memory", + ) + p.add_argument( + "--3d", + dest="threed", + action="store_true", + help="render in 3D over the occupancy floor (SLAM-map look) instead of the flat 2D fog", + ) + return p.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + kill = (args.kill[0], args.kill[1]) if args.kill else None + view( + map_name=args.map, + n_dogs=args.dogs, + seed=args.seed, + target_label=args.target, + max_ticks=args.max_ticks, + kill=kill, + save=args.save, + shared=not args.independent, + threed=args.threed, + ) From 540b3d1ab33ff215803cde9752c692c10dd8f53f Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 00:08:52 -0600 Subject: [PATCH 18/35] =?UTF-8?q?feat:=20PACK=20MIND=20live=20coordinator?= =?UTF-8?q?=20=E2=80=94=20shared=20zone=20memory=20for=20a=202-dog=20searc?= =?UTF-8?q?h?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Laptop-side brain that shares MEANING (which zones are searched, what was found) over HTTP so two dogs on two laptops never re-search the same area, and the mission survives a dog going offline. Zone NAMES only — never coordinates — so two independent SLAM frames work together without map merging. - pack_coordinator: zone ledger (provably no double-assign), report_finding (stop-on-find + claimed-zone fallback), release_dog (survivor inherits a downed dog's unfinished zones; findings persist) - pack_coordinator_server: JSON/HTTP API + projector dashboard at / + fan-out prefs - pack_dashboard.html: live view (zones, finding, offline/inheritance, causal chain) - pack_search_skills: dog agent tools (start_search/next_zone/report_*/where_is) - pack_search_runner: RobotDriver-protocol search loop + MockDriver - live.py: unitree-go2-pack blueprint (one dog per laptop, env-configured) + prompt - mock_dog + demo_pack_scene: hardware-free test/rehearsal of handoff + inheritance - tests: 24 passing (coordinator, HTTP server, runner) --- .../experimental/pack_mind/demo_pack_scene.py | 77 +++++ dimos/experimental/pack_mind/live.py | 79 ++++++ dimos/experimental/pack_mind/mock_dog.py | 122 ++++++++ .../pack_mind/pack_coordinator.py | 220 ++++++++++++++ .../pack_mind/pack_coordinator_server.py | 268 ++++++++++++++++++ .../pack_mind/pack_dashboard.html | 176 ++++++++++++ .../pack_mind/pack_search_runner.py | 114 ++++++++ .../pack_mind/pack_search_skills.py | 228 +++++++++++++++ .../pack_mind/test_pack_coordinator.py | 153 ++++++++++ .../pack_mind/test_pack_coordinator_server.py | 181 ++++++++++++ .../pack_mind/test_pack_search_runner.py | 84 ++++++ 11 files changed, 1702 insertions(+) create mode 100644 dimos/experimental/pack_mind/demo_pack_scene.py create mode 100644 dimos/experimental/pack_mind/live.py create mode 100644 dimos/experimental/pack_mind/mock_dog.py create mode 100644 dimos/experimental/pack_mind/pack_coordinator.py create mode 100644 dimos/experimental/pack_mind/pack_coordinator_server.py create mode 100644 dimos/experimental/pack_mind/pack_dashboard.html create mode 100644 dimos/experimental/pack_mind/pack_search_runner.py create mode 100644 dimos/experimental/pack_mind/pack_search_skills.py create mode 100644 dimos/experimental/pack_mind/test_pack_coordinator.py create mode 100644 dimos/experimental/pack_mind/test_pack_coordinator_server.py create mode 100644 dimos/experimental/pack_mind/test_pack_search_runner.py diff --git a/dimos/experimental/pack_mind/demo_pack_scene.py b/dimos/experimental/pack_mind/demo_pack_scene.py new file mode 100644 index 0000000000..ebc9114f34 --- /dev/null +++ b/dimos/experimental/pack_mind/demo_pack_scene.py @@ -0,0 +1,77 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — hardware-free dry-run of the v4 demo's two magic beats. + +Prints the dashboard-style causal chain the audience will see, driven entirely by +the real coordinator + runner (only the robot bodies are mocked). Use it to +rehearse narration and to smoke-test the brain without dogs. + + uv run python -m dimos.experimental.pack_mind.demo_pack_scene + +`demo_` prefix keeps it out of pytest collection (it's a manual scene, not a test). +""" + +from __future__ import annotations + +from dimos.experimental.pack_mind.pack_coordinator import PackCoordinator +from dimos.experimental.pack_mind.pack_search_runner import MockDriver, act_on_finding, run_search + + +def _line(tag: str, msg: str) -> None: + print(f" {tag:<28} {msg}") + + +def beat_1_handoff() -> None: + print("\n=== BEAT 1 — THE HANDOFF (a body acts on memory it never gathered) ===") + c = PackCoordinator(["north", "east", "south", "west"]) + print(" ", c.start_search("red kit")) + res = run_search(c, MockDriver(target_zone="east"), "alpha", "red kit") + _line("ALPHA SEARCHED", f"{res.visited} → FOUND red kit in {res.zone}") + _line("PACK MEMORY UPDATED", f"finding by alpha @ {res.zone}") + bravo = MockDriver(target_zone="east") + print(" ", act_on_finding(c, bravo, "bravo")) + _line("BRAVO ACTED ON ALPHA'S MEMORY", f"went to {bravo.visited[-1]} — never searched it itself") + + +def beat_2_inheritance() -> None: + print("\n=== BEAT 2 — INHERITANCE (the mission doesn't die with the robot) ===") + prefs = {"alpha": ["south"], "bravo": ["north", "east", "west"]} + c = PackCoordinator(["north", "east", "south", "west"], preferences=prefs) + print(" ", c.start_search("red kit")) + bravo = MockDriver(target_zone="south") # the kit is in alpha's zone + _line("ALPHA CLAIMS", c.assign_zone("alpha") or "-") # "south", unfinished + for _ in range(3): + z = c.assign_zone("bravo") + if not z: + break + bravo.goto(z) + c.report_cleared("bravo", z) + _line("BRAVO CLEARED", z) + _line("ALPHA GOES OFFLINE", c.release_dog("alpha")) + res = run_search(c, bravo, "bravo", "red kit") + _line("BRAVO INHERITED + FINISHED", f"found red kit in {res.zone} (alpha started this zone)") + snap = c.snapshot() + _line("SHARED MEMORY", f"offline={snap['offline']} found={snap['found']} coverage={snap['coverage']}") + + +def main() -> None: + print("PACK MIND — shared operational memory (hardware-free scene)") + beat_1_handoff() + beat_2_inheritance() + print("\nAlpha saw it. Bravo remembered it. The pack acted. — PACK MIND\n") + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/live.py b/dimos/experimental/pack_mind/live.py new file mode 100644 index 0000000000..d4cd0adbbf --- /dev/null +++ b/dimos/experimental/pack_mind/live.py @@ -0,0 +1,79 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — live Go2 blueprint (one dog per laptop, shared memory over HTTP). + +Each laptop runs THIS blueprint against its own dog; the two processes share one +``PackCoordinator`` over the LAN. There is no in-process multi-instance problem +because the two dogs are two separate `dimos run` processes — they only meet at +the coordinator. Configure per laptop with env vars:: + + # Laptop A (also runs the coordinator): + python -m dimos.experimental.pack_mind.pack_coordinator_server --zones north,east,south,west & + PACK_DOG_NAME=alpha PACK_COORDINATOR_URL=http://127.0.0.1:8090 \\ + dimos run unitree-go2-pack --robot-ip --listen-host 0.0.0.0 + + # Laptop B: + PACK_DOG_NAME=bravo PACK_COORDINATOR_URL=http://:8090 \\ + dimos run unitree-go2-pack --robot-ip --listen-host 0.0.0.0 + +The detection→coordinator bridge needs NO custom module: the agent calls +``look_out_for([target], then={"name": "report_finding", "args": {...}})`` so a VLM +sighting auto-reports the finding to the pack (perceive_loop_skill dispatches the +``then`` continuation). + +NOTE: untested without hardware/GPU — bring up per the H0 gate in the runbook. +""" + +from __future__ import annotations + +from dimos.agents.mcp.mcp_client import McpClient +from dimos.agents.mcp.mcp_server import McpServer +from dimos.core.coordination.blueprints import autoconnect +from dimos.experimental.pack_mind.pack_search_skills import PackSearchSkills +from dimos.robot.unitree.go2.blueprints.agentic._common_agentic import _common_agentic +from dimos.robot.unitree.go2.blueprints.smart.unitree_go2_spatial import unitree_go2_spatial + +PACK_SYSTEM_PROMPT = """\ +You are one dog in a PACK that shares a single operational memory through tools. +You never command another dog and never exchange coordinates — you read and write +the shared memory by ZONE NAME only. + +MISSION LOOP (follow exactly): +1. When given a target (e.g. "find the red kit"), call start_search(target) ONCE. +2. Repeat: + a. Call next_zone to get YOUR assigned zone. If it returns empty, the search is + over — STOP and report status. + b. navigate_with_text("") to travel there. + c. look_out_for([""], then={"name": "report_finding", + "args": {"object": "", "zone": ""}}) so a sighting + auto-tells the whole pack. + d. Scan briefly. If you did not see it, call report_cleared(""). + e. Check should_stop — if true, a packmate found it; STOP. +3. If asked where the target is, call where_is(target) and act on the answer even + if you never saw it yourself — that is acting on the pack's shared memory. + +Keep moving. Be concise. Zones by name, never coordinates.""" + +# One dog per laptop. PackSearchSkills reads PACK_DOG_NAME + PACK_COORDINATOR_URL +# from the environment, so this single blueprint serves both laptops. +unitree_go2_pack = autoconnect( + unitree_go2_spatial, + McpServer.blueprint(), + McpClient.blueprint(system_prompt=PACK_SYSTEM_PROMPT), + _common_agentic, + PackSearchSkills.blueprint(), +) + +__all__ = ["unitree_go2_pack", "PACK_SYSTEM_PROMPT"] diff --git a/dimos/experimental/pack_mind/mock_dog.py b/dimos/experimental/pack_mind/mock_dog.py new file mode 100644 index 0000000000..8c2c4922b9 --- /dev/null +++ b/dimos/experimental/pack_mind/mock_dog.py @@ -0,0 +1,122 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND mock dog — a hardware-free client that drives one dog's search loop. + +No DimOS Module, no robot, just ``requests`` against a running coordinator. Run +the server plus two of these (one with ``--find-prob 1.0``) to demonstrate the +full no-overlap + stop-on-find story with zero robots:: + + uv run python dimos/experimental/pack_mind/pack_coordinator_server.py --port 8090 & + uv run python dimos/experimental/pack_mind/mock_dog.py --dog alpha --url http://localhost:8090 + uv run python dimos/experimental/pack_mind/mock_dog.py --dog bravo --url http://localhost:8090 --find-prob 1.0 + +Each dog loops: ask for a zone; stop if the pack already found the target; with +probability ``--find-prob`` report a finding (and stop), otherwise report the +zone cleared; repeat until no zone is left. +""" + +from __future__ import annotations + +import argparse +import random +import time +from typing import Any + +import requests + +DEFAULT_TARGET = "red object" +_REQUEST_TIMEOUT = 5.0 +_MIN_SLEEP = 0.05 +_MAX_SLEEP = 0.25 + + +def _post(url: str, path: str, payload: dict[str, Any]) -> dict[str, Any] | None: + try: + resp = requests.post(f"{url}{path}", json=payload, timeout=_REQUEST_TIMEOUT) + resp.raise_for_status() + data: Any = resp.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + print(f"[{path}] request failed: {exc}") + return None + + +def _get(url: str, path: str, params: dict[str, str]) -> dict[str, Any] | None: + try: + resp = requests.get(f"{url}{path}", params=params, timeout=_REQUEST_TIMEOUT) + resp.raise_for_status() + data: Any = resp.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + print(f"[{path}] request failed: {exc}") + return None + + +def run_dog( + dog: str, url: str, target: str, find_prob: float, rng: random.Random +) -> None: + url = url.rstrip("/") + while True: + stop = _get(url, "/should_stop", {"dog": dog}) + if stop is not None and stop.get("stop"): + print(f"[{dog}] pack already found the target — converging, stopping.") + return + + assigned = _post(url, "/assign_zone", {"dog": dog}) + zone = assigned.get("zone") if assigned else None + if not zone: + print(f"[{dog}] no zone assigned (search over or exhausted) — stopping.") + return + print(f"[{dog}] assigned zone: {zone}") + + time.sleep(rng.uniform(_MIN_SLEEP, _MAX_SLEEP)) + + if rng.random() < find_prob: + result = _post( + url, "/report_finding", {"dog": dog, "object": target, "zone": zone} + ) + finding = result.get("finding") if result else None + print(f"[{dog}] FOUND {target} in {zone} -> {finding} — stopping.") + return + + cleared = _post(url, "/report_cleared", {"dog": dog, "zone": zone}) + status = cleared.get("status") if cleared else "(unreachable)" + print(f"[{dog}] cleared {zone}: {status}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="PACK MIND mock dog client") + parser.add_argument("--dog", required=True, help="This dog's name, e.g. alpha.") + parser.add_argument( + "--url", + default="http://localhost:8090", + help="Coordinator base URL.", + ) + parser.add_argument("--target", default=DEFAULT_TARGET, help="What to 'find'.") + parser.add_argument( + "--find-prob", + type=float, + default=0.0, + help="Per-zone probability of reporting a finding (1.0 = always find).", + ) + parser.add_argument("--seed", type=int, default=None, help="RNG seed for repeatability.") + args = parser.parse_args() + + rng = random.Random(args.seed) + run_dog(args.dog, args.url, args.target, args.find_prob, rng) + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/pack_coordinator.py b/dimos/experimental/pack_mind/pack_coordinator.py new file mode 100644 index 0000000000..429940bbea --- /dev/null +++ b/dimos/experimental/pack_mind/pack_coordinator.py @@ -0,0 +1,220 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — laptop-side coordinator: the shared MEANING of the pack. + +This is the brain that lets two (or more) dogs search without overlapping. It +holds a **zone ledger** (which named regions are unsearched / claimed / cleared) +and a **target blackboard** (what we're looking for, whether it's been found and +where). It deliberately stores **no coordinates** — only zone NAMES — so it works +across dogs that each run their own SLAM frame (the honest "share meaning, not +maps" design, same invariant as conductor.py). + +The two guarantees that make the demo true: + 1. ``assign_zone`` never hands the same zone to two dogs (no overlap). + 2. once any dog reports a finding, ``found`` flips and assignment stops (the + whole pack knows and halts). + +Pure Python, thread-safe, no hardware deps — unit-testable in isolation and the +core of the HTTP coordinator (mockable like ``conductor.py --mock``). +""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from typing import Literal + +ZoneState = Literal["unsearched", "claimed", "cleared"] + + +@dataclass(frozen=True) +class Finding: + """An immutable record of an object sighting — by zone NAME, never coordinates.""" + + object: str + zone: str + by: str + ts: float + + +@dataclass +class Zone: + name: str + state: ZoneState = "unsearched" + by: str | None = None # the dog that claimed or cleared it + + +class PackCoordinator: + """Shared search ledger + target blackboard for a pack of dogs. + + Args: + zones: ordered zone names covering the search area. + preferences: optional per-dog ordering of zones, so dogs fan out to + different ends without any coordinate math (e.g. dog A prefers the + north zones, dog B the south). Falls back to ``zones`` order. + """ + + def __init__( + self, zones: list[str], preferences: dict[str, list[str]] | None = None + ) -> None: + if not zones: + raise ValueError("PackCoordinator needs at least one zone") + self._order = list(zones) + self._zones: dict[str, Zone] = {z: Zone(z) for z in zones} + self._preferences = preferences or {} + self._lock = threading.RLock() + self._target: str | None = None + self._finding: Finding | None = None + self._offline: set[str] = set() + self._log: list[str] = [] + + # -- mission ------------------------------------------------------------ + + def start_search(self, target: str) -> str: + """Begin a search for ``target``; resets the ledger so every zone is + unsearched again. Returns a status line.""" + with self._lock: + self._target = target + self._finding = None + self._offline = set() + self._zones = {z: Zone(z) for z in self._order} + self._log.append(f"search started: find '{target}'") + return f"Searching for {target}. {len(self._zones)} zones to cover." + + # -- allocation (the no-overlap guarantee) ------------------------------ + + def assign_zone(self, dog: str) -> str | None: + """Hand ``dog`` a zone no other dog has claimed or cleared, following + the dog's preference order. Returns None when the search is over (found + or nothing left) — the dog should then stop / converge.""" + with self._lock: + if self._finding is not None: + return None # object found → pack stops searching + for name in self._iter_order(dog): + zone = self._zones[name] + if zone.state == "unsearched": + self._zones[name] = Zone(name, "claimed", dog) + self._log.append(f"{dog} -> {name} (claimed)") + return name + return None + + def report_cleared(self, dog: str, zone: str) -> str: + """Mark ``zone`` fully searched by ``dog`` (object not here).""" + with self._lock: + if zone in self._zones: + self._zones[zone] = Zone(zone, "cleared", dog) + self._log.append(f"{dog} cleared {zone}") + return f"{zone} cleared." + + def report_finding(self, dog: str, object: str, zone: str = "") -> Finding: + """Record that ``dog`` saw ``object``. Flips ``found`` so the whole pack + stops. Idempotent: keeps the first finding. + + ``zone`` is best-effort: if it's empty or unknown (e.g. an LLM filled it in + wrong via a ``then=`` continuation), we fall back to the zone this dog + currently has claimed — the place it actually is.""" + with self._lock: + if self._finding is None: + if not zone or zone not in self._zones: + zone = self._claimed_zone_of(dog) or zone + self._finding = Finding(object, zone, dog, time.time()) + if zone in self._zones: + self._zones[zone] = Zone(zone, "cleared", dog) + self._log.append(f"{dog} FOUND {object} in {zone or '?'} — pack stop") + return self._finding + + def _claimed_zone_of(self, dog: str) -> str | None: + """The zone this dog currently holds claimed (where it is), if any.""" + for name, z in self._zones.items(): + if z.state == "claimed" and z.by == dog: + return name + return None + + def release_dog(self, dog: str) -> str: + """Take ``dog`` offline and hand its unfinished ground back to the pack. + + Its *findings persist* (the shared memory outlives the robot), but every + zone it had only **claimed** (not yet cleared) reverts to ``unsearched`` so + a surviving teammate inherits and finishes the mission. Cleared zones stay + cleared — that knowledge is not lost. This powers the resilience climax: + "the mission doesn't die with the robot." + """ + with self._lock: + self._offline.add(dog) + reclaimed = [] + for name, zone in self._zones.items(): + if zone.state == "claimed" and zone.by == dog: + self._zones[name] = Zone(name) # back to unsearched + reclaimed.append(name) + self._log.append( + f"{dog} OFFLINE — findings kept; reclaimed {reclaimed or 'no'} zone(s)" + ) + return f"{dog} offline. Reclaimed {len(reclaimed)} zone(s) for the pack." + + # -- queries ------------------------------------------------------------ + + @property + def found(self) -> bool: + with self._lock: + return self._finding is not None + + @property + def finding(self) -> Finding | None: + with self._lock: + return self._finding + + def should_stop(self, dog: str) -> bool: + """A dog polls this; True once the object is found by anyone.""" + return self.found + + def coverage(self) -> float: + """Fraction of zones cleared — the 'how much of the area is searched' metric.""" + with self._lock: + cleared = sum(1 for z in self._zones.values() if z.state == "cleared") + return cleared / len(self._zones) + + def snapshot(self) -> dict[str, object]: + with self._lock: + return { + "target": self._target, + "found": self._finding is not None, + "finding": ( + { + "object": self._finding.object, + "zone": self._finding.zone, + "by": self._finding.by, + } + if self._finding + else None + ), + "coverage": round(self.coverage(), 3), + "offline": sorted(self._offline), + "zones": [ + {"name": z.name, "state": z.state, "by": z.by} + for z in self._zones.values() + ], + "log": list(self._log), + } + + # -- helpers ------------------------------------------------------------ + + def _iter_order(self, dog: str) -> list[str]: + """The dog's preferred zone order, then any remaining zones.""" + pref = self._preferences.get(dog, []) + seen = set(pref) + return [z for z in pref if z in self._zones] + [ + z for z in self._order if z not in seen + ] diff --git a/dimos/experimental/pack_mind/pack_coordinator_server.py b/dimos/experimental/pack_mind/pack_coordinator_server.py new file mode 100644 index 0000000000..86cef0a2ad --- /dev/null +++ b/dimos/experimental/pack_mind/pack_coordinator_server.py @@ -0,0 +1,268 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND coordinator HTTP server — the shared MEANING, reachable over a LAN. + +Wraps a single ``PackCoordinator`` (the pure, unit-tested ledger) behind a tiny +JSON/HTTP API so each dog's laptop agent can call it with ``requests``. Mirrors +the stdlib ``ThreadingHTTPServer`` + ``BaseHTTPRequestHandler`` pattern in +``conductor.py``. The coordinator shares zone NAMES and findings only — never +coordinates — so two dogs running independent SLAM frames never re-search the +same area. + +Endpoints (all JSON; 200 on success, 400 on bad input):: + + GET /state -> snapshot() + POST /start_search {"target"} -> {"status"} + POST /assign_zone {"dog"} -> {"zone": str|null} + POST /report_cleared {"dog", "zone"} -> {"status"} + POST /report_finding {"dog", "object", "zone"} -> {"finding": {...}|null} + GET /should_stop?dog=NAME -> {"stop": bool} + +Run (binds 0.0.0.0 so dogs across the LAN can reach it):: + + uv run python dimos/experimental/pack_mind/pack_coordinator_server.py \\ + --port 8090 --zones north,east,south,west +""" + +from __future__ import annotations + +import argparse +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, cast +from urllib.parse import parse_qs, urlparse + +_DASHBOARD_HTML = Path(__file__).with_name("pack_dashboard.html") + +from dimos.experimental.pack_mind.pack_coordinator import PackCoordinator +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +# New named constant — the LAN port the coordinator listens on. No hardcoded +# ports elsewhere; clients import this to build their default URL. +DEFAULT_COORDINATOR_PORT = 8090 +DEFAULT_ZONES = ["north", "east", "south", "west"] + + +class _Server(ThreadingHTTPServer): + coordinator: PackCoordinator + + +class _Handler(BaseHTTPRequestHandler): + def log_message(self, fmt: str, *args: Any) -> None: # silence default logging + return + + @property + def _coordinator(self) -> PackCoordinator: + return cast(_Server, self.server).coordinator + + def _send_json(self, code: int, payload: Any) -> None: + body = json.dumps(payload).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _read_json(self) -> dict[str, Any] | None: + """Read a JSON object body. Returns None on malformed/non-object input.""" + length = int(self.headers.get("Content-Length", "0")) + raw = self.rfile.read(length) if length else b"{}" + try: + data: Any = json.loads(raw) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + def _serve_dashboard(self) -> None: + try: + body = _DASHBOARD_HTML.read_bytes() + except OSError: + self._send_json(500, {"error": "dashboard html not found"}) + return + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 (stdlib API) + parsed = urlparse(self.path) + if parsed.path in ("/", "/dashboard"): + self._serve_dashboard() + elif parsed.path == "/state": + self._send_json(200, self._coordinator.snapshot()) + elif parsed.path == "/where_is": + f = self._coordinator.finding + self._send_json( + 200, + { + "found": f is not None, + "object": f.object if f else None, + "zone": f.zone if f else None, + "by": f.by if f else None, + }, + ) + elif parsed.path == "/should_stop": + params = parse_qs(parsed.query) + dog_values = params.get("dog") + if not dog_values or not dog_values[0]: + self._send_json(400, {"error": "missing query param: dog"}) + return + self._send_json(200, {"stop": self._coordinator.should_stop(dog_values[0])}) + else: + self._send_json(404, {"error": f"unknown path: {parsed.path}"}) + + def do_POST(self) -> None: # noqa: N802 (stdlib API) + path = urlparse(self.path).path + req = self._read_json() + if req is None: + self._send_json(400, {"error": "bad json"}) + return + if path == "/start_search": + self._start_search(req) + elif path == "/assign_zone": + self._assign_zone(req) + elif path == "/report_cleared": + self._report_cleared(req) + elif path == "/report_finding": + self._report_finding(req) + elif path == "/release_dog": + self._release_dog(req) + else: + self._send_json(404, {"error": f"unknown path: {path}"}) + + # -- handlers ------------------------------------------------------------ + + def _start_search(self, req: dict[str, Any]) -> None: + target = req.get("target") + if not isinstance(target, str) or not target: + self._send_json(400, {"error": "missing or invalid field: target"}) + return + status = self._coordinator.start_search(target) + logger.info("start_search", target=target) + self._send_json(200, {"status": status}) + + def _assign_zone(self, req: dict[str, Any]) -> None: + dog = req.get("dog") + if not isinstance(dog, str) or not dog: + self._send_json(400, {"error": "missing or invalid field: dog"}) + return + zone = self._coordinator.assign_zone(dog) + logger.info("assign_zone", dog=dog, zone=zone) + self._send_json(200, {"zone": zone}) + + def _report_cleared(self, req: dict[str, Any]) -> None: + dog = req.get("dog") + zone = req.get("zone") + if not isinstance(dog, str) or not dog or not isinstance(zone, str) or not zone: + self._send_json(400, {"error": "missing or invalid fields: dog, zone"}) + return + status = self._coordinator.report_cleared(dog, zone) + logger.info("report_cleared", dog=dog, zone=zone) + self._send_json(200, {"status": status}) + + def _report_finding(self, req: dict[str, Any]) -> None: + dog = req.get("dog") + obj = req.get("object") + zone = req.get("zone", "") + if not isinstance(dog, str) or not dog or not isinstance(obj, str) or not obj: + self._send_json(400, {"error": "missing or invalid fields: dog, object"}) + return + # zone is best-effort: the coordinator falls back to the dog's claimed zone. + zone = zone if isinstance(zone, str) else "" + finding = self._coordinator.report_finding(dog, obj, zone) + logger.info("report_finding", dog=dog, object=obj, zone=zone) + self._send_json( + 200, + {"finding": {"object": finding.object, "zone": finding.zone, "by": finding.by}}, + ) + + def _release_dog(self, req: dict[str, Any]) -> None: + dog = req.get("dog") + if not isinstance(dog, str) or not dog: + self._send_json(400, {"error": "missing or invalid field: dog"}) + return + status = self._coordinator.release_dog(dog) + logger.info("release_dog", dog=dog) + self._send_json(200, {"status": status}) + + +def make_server( + host: str, + port: int, + zones: list[str], + preferences: dict[str, list[str]] | None = None, +) -> _Server: + """Build a ready-to-serve coordinator server. Pass port 0 for an ephemeral port.""" + server = _Server((host, port), _Handler) + server.coordinator = PackCoordinator(zones, preferences) + return server + + +def _parse_prefs(raw: str) -> dict[str, list[str]]: + """Parse "alpha:north,east;bravo:south,west" into per-dog zone orderings.""" + prefs: dict[str, list[str]] = {} + for chunk in raw.split(";"): + chunk = chunk.strip() + if not chunk or ":" not in chunk: + continue + dog, zone_csv = chunk.split(":", 1) + zones = [z.strip() for z in zone_csv.split(",") if z.strip()] + if dog.strip() and zones: + prefs[dog.strip()] = zones + return prefs + + +def main() -> None: + parser = argparse.ArgumentParser(description="PACK MIND coordinator HTTP server") + parser.add_argument("--host", default="0.0.0.0", help="Bind address (0.0.0.0 for LAN).") + parser.add_argument( + "--port", type=int, default=DEFAULT_COORDINATOR_PORT, help="Listen port." + ) + parser.add_argument( + "--zones", + default=",".join(DEFAULT_ZONES), + help="Comma-separated zone names.", + ) + parser.add_argument( + "--prefs", + default="", + help='Per-dog zone order so dogs fan out, e.g. "alpha:north,east;bravo:south,west".', + ) + args = parser.parse_args() + + zones = [z.strip() for z in args.zones.split(",") if z.strip()] + if not zones: + parser.error("--zones must contain at least one zone name") + + server = make_server(args.host, args.port, zones, _parse_prefs(args.prefs)) + actual_port = server.server_address[1] + print(f"PACK MIND coordinator up on http://{args.host}:{actual_port}") + print(f" reachable on the LAN at http://:{actual_port}") + print(f" zones: {', '.join(zones)}") + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nshutting down") + server.shutdown() + finally: + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/pack_dashboard.html b/dimos/experimental/pack_mind/pack_dashboard.html new file mode 100644 index 0000000000..3f74b3b96f --- /dev/null +++ b/dimos/experimental/pack_mind/pack_dashboard.html @@ -0,0 +1,176 @@ + + + + + + +PACK MIND + + + +
+

PACK MIND

+ shared operational memory + memory: empty +
+
+
+

Search zones — one shared memory, no overlap

+
+

Area searched

+
+
+
+
+

Finding

+
No sighting yet — the pack is searching.
+
+
+

Pack roster

+
+
+
+

Causal chain

+
+
+
+
+
ONE MEMORY · MANY BODIES · OUTLIVES ANY SINGLE ROBOT
+ + + diff --git a/dimos/experimental/pack_mind/pack_search_runner.py b/dimos/experimental/pack_mind/pack_search_runner.py new file mode 100644 index 0000000000..19318b970e --- /dev/null +++ b/dimos/experimental/pack_mind/pack_search_runner.py @@ -0,0 +1,114 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND search orchestration — the deterministic loop a dog runs. + +The loop is decoupled from *how* a dog moves and sees via the ``RobotDriver`` +Protocol, so the SAME orchestration runs: + - in tests / the hardware-free scene, with ``MockDriver``; + - on a real Go2, with a driver that wraps ``navigate_with_text`` + ``look_out_for``. + +It talks to the coordinator through the ``SearchClient`` Protocol — satisfied +directly by ``PackCoordinator`` (in-process tests) or by an HTTP-backed adapter +(live). Determinism here is deliberate: a scripted loop is far more demo-reliable +than asking the LLM to remember to loop. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Protocol + +from dimos.experimental.pack_mind.pack_coordinator import Finding + + +class RobotDriver(Protocol): + """How a body moves and perceives. Mock in tests; real skills on hardware.""" + + def goto(self, zone: str) -> bool: + """Travel to the named zone. Return True on arrival.""" + ... + + def look_for(self, target: str) -> bool: + """Scan the current location. Return True iff the target is seen here.""" + ... + + +class SearchClient(Protocol): + """The coordinator surface the loop needs. PackCoordinator satisfies this.""" + + def assign_zone(self, dog: str) -> str | None: ... + def report_cleared(self, dog: str, zone: str) -> str: ... + def report_finding(self, dog: str, object: str, zone: str) -> Finding: ... + def should_stop(self, dog: str) -> bool: ... + @property + def finding(self) -> Finding | None: ... + + +@dataclass +class SearchResult: + dog: str + found: bool + zone: str | None + visited: list[str] = field(default_factory=list) + + +def run_search( + client: SearchClient, driver: RobotDriver, dog: str, target: str, max_steps: int = 100 +) -> SearchResult: + """Pull a distinct zone, go, look, report — until found, stopped, or empty. + + Inheriting another dog's reclaimed zone is automatic: ``assign_zone`` simply + starts handing out the freed zones, so a survivor's loop continues seamlessly. + """ + visited: list[str] = [] + for _ in range(max_steps): + if client.should_stop(dog): + break + zone = client.assign_zone(dog) + if not zone: + break + if not driver.goto(zone): + continue # couldn't reach it; ask for another + visited.append(zone) + if driver.look_for(target): + client.report_finding(dog, target, zone) + return SearchResult(dog, True, zone, visited) + client.report_cleared(dog, zone) + f = client.finding + return SearchResult(dog, f is not None, f.zone if f else None, visited) + + +def act_on_finding(client: SearchClient, driver: RobotDriver, dog: str) -> str: + """The handoff: go to a teammate's reported find — memory you never gathered.""" + f = client.finding + if f is None: + return f"{dog}: nothing in pack memory to act on yet." + driver.goto(f.zone) + return f"{dog} acted on {f.by}'s memory: went to {f.zone}, confirmed {f.object}." + + +@dataclass +class MockDriver: + """Hardware-free driver: the target lives in ``target_zone``; travel always succeeds.""" + + target_zone: str + visited: list[str] = field(default_factory=list) + + def goto(self, zone: str) -> bool: + self.visited.append(zone) + return True + + def look_for(self, target: str) -> bool: + return bool(self.visited) and self.visited[-1] == self.target_zone diff --git a/dimos/experimental/pack_mind/pack_search_skills.py b/dimos/experimental/pack_mind/pack_search_skills.py new file mode 100644 index 0000000000..9f8b360a31 --- /dev/null +++ b/dimos/experimental/pack_mind/pack_search_skills.py @@ -0,0 +1,228 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND search skills — the bridge from a dog's LLM agent to the shared coordinator. + +A DimOS ``Module`` that turns the laptop coordinator's HTTP API into @skill tools +the robot's agent can call. The dog never learns the other dog's coordinates — it +only asks "what zone next?" and reports cleared zones / findings by NAME. Every +call is wrapped in a short-timeout try/except so a coordinator outage degrades to +a calm string and NEVER raises into the agent loop. + +Wire it into an agentic blueprint alongside the robot stack; point it at the +coordinator with the ``PACK_COORDINATOR_URL`` env var (or the constructor arg). +""" + +from __future__ import annotations + +import os +from typing import Any + +import requests + +from dimos.agents.annotation import skill +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.experimental.pack_mind.pack_coordinator_server import DEFAULT_COORDINATOR_PORT +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +# Built from localhost + the server's port constant — no hardcoded port literal. +DEFAULT_COORDINATOR_URL = f"http://localhost:{DEFAULT_COORDINATOR_PORT}" + +# Short timeout: the agent loop must never block on a flaky coordinator. +_REQUEST_TIMEOUT = 5.0 + + +class PackSearchSkills(Module): + """Exposes the PACK MIND coordinator to one dog's LLM agent as tools. + + Args: + dog_name: this dog's stable name (used as the ledger identity). + coordinator_url: base URL of the coordinator server; defaults to the + ``PACK_COORDINATOR_URL`` env var, else localhost + the default port. + """ + + def __init__( + self, + dog_name: str | None = None, + coordinator_url: str | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + # Per-laptop identity: explicit arg › PACK_DOG_NAME env › "alpha". This lets + # the SAME blueprint run on both laptops, differing only by env var. + self._dog_name = dog_name or os.environ.get("PACK_DOG_NAME") or "alpha" + self._url = ( + coordinator_url + or os.environ.get("PACK_COORDINATOR_URL") + or DEFAULT_COORDINATOR_URL + ).rstrip("/") + + @rpc + def start(self) -> None: + super().start() + logger.info("PackSearchSkills ready", dog=self._dog_name, coordinator=self._url) + + @rpc + def stop(self) -> None: + super().stop() + + # -- HTTP plumbing ------------------------------------------------------- + + def _post(self, path: str, payload: dict[str, Any]) -> dict[str, Any] | None: + """POST JSON; return the decoded dict, or None on any failure.""" + try: + resp = requests.post( + f"{self._url}{path}", json=payload, timeout=_REQUEST_TIMEOUT + ) + resp.raise_for_status() + data: Any = resp.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + logger.warning("coordinator POST failed", path=path, error=str(exc)) + return None + + def _get(self, path: str, params: dict[str, str]) -> dict[str, Any] | None: + try: + resp = requests.get( + f"{self._url}{path}", params=params, timeout=_REQUEST_TIMEOUT + ) + resp.raise_for_status() + data: Any = resp.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + logger.warning("coordinator GET failed", path=path, error=str(exc)) + return None + + # -- skills (exposed to the LLM) ----------------------------------------- + + @skill + def start_search(self, target: str) -> str: + """Begin a pack search for an object across the shared zones. + + Call this once at mission start. It resets the shared zone ledger so the + whole pack hunts for the same target without re-searching areas. + + Args: + target: A short description of what to find, e.g. "red backpack". + """ + data = self._post("/start_search", {"target": target}) + if data is None: + return f"Could not reach the pack coordinator to start searching for {target}." + return str(data.get("status", f"Searching for {target}.")) + + @skill + def report_cleared(self, zone: str) -> str: + """Tell the pack you finished searching a zone and the target was NOT there. + + Marks the zone cleared on the shared ledger so no teammate re-searches it. + + Args: + zone: The name of the zone you just finished searching, e.g. "north". + """ + data = self._post("/report_cleared", {"dog": self._dog_name, "zone": zone}) + if data is None: + return f"Could not reach the pack coordinator to report {zone} cleared." + return str(data.get("status", f"{zone} cleared.")) + + @skill + def report_finding(self, object: str, zone: str) -> str: + """Tell the whole pack you found the target so everyone stops searching. + + Records the sighting by zone NAME on the shared blackboard; the first + finding wins and halts all assignment. + + Args: + object: What you found, e.g. "red backpack". + zone: The zone name where you found it, e.g. "east". + """ + data = self._post( + "/report_finding", + {"dog": self._dog_name, "object": object, "zone": zone}, + ) + if data is None: + return f"Could not reach the pack coordinator to report finding {object}." + finding = data.get("finding") + if isinstance(finding, dict): + return f"Pack notified: {finding.get('object')} found in {finding.get('zone')}." + return f"Pack notified: {object} found in {zone}." + + @skill + def where_is(self, object: str) -> str: + """Ask the pack's shared memory where an object was found. + + Use this to act on a TEAMMATE's discovery — you may never have seen the + object yourself. Returns the zone a packmate reported it in, or that it + hasn't been found yet. + + Args: + object: What you're asking about, e.g. "red backpack". + """ + data = self._get("/where_is", {}) + if data is None: + return "Could not reach the pack memory." + if not data.get("found"): + return f"No packmate has found the {object} yet." + return ( + f"{data.get('by')} found the {data.get('object')} in {data.get('zone')}. " + f"I can take you there." + ) + + # -- rpc (internal control, not exposed to the LLM) ---------------------- + + @rpc + def finding_zone(self) -> str: + """The zone of the current finding (for navigation), or "" if none yet.""" + data = self._get("/where_is", {}) + if data is None or not data.get("found"): + return "" + zone = data.get("zone") + return zone if isinstance(zone, str) else "" + + @rpc + def release_dog(self, dog: str) -> bool: + """Mark a packmate offline so the pack reclaims its unfinished zones. + + Operator/coordinator action (e.g. a dog dropped). The downed dog's + findings persist; its claimed-but-uncleared zones return to the pool so a + survivor inherits them. Returns True on success. + """ + data = self._post("/release_dog", {"dog": dog}) + return data is not None + + @rpc + def next_zone(self) -> str: + """Ask the coordinator for the next unsearched zone for this dog. + + Returns the zone name, or "" when the search is over (found or no zones + left) — skills cannot return None cleanly, so "" signals "stop". + """ + data = self._post("/assign_zone", {"dog": self._dog_name}) + if data is None: + return "" + zone = data.get("zone") + return zone if isinstance(zone, str) else "" + + @rpc + def should_stop(self) -> bool: + """True once any dog has found the target and the pack should converge.""" + data = self._get("/should_stop", {"dog": self._dog_name}) + if data is None: + return False + return bool(data.get("stop", False)) + + +pack_search_skills = PackSearchSkills.blueprint diff --git a/dimos/experimental/pack_mind/test_pack_coordinator.py b/dimos/experimental/pack_mind/test_pack_coordinator.py new file mode 100644 index 0000000000..3183259899 --- /dev/null +++ b/dimos/experimental/pack_mind/test_pack_coordinator.py @@ -0,0 +1,153 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND coordinator tests — the no-overlap and stop-on-found guarantees.""" + +import pytest + +from dimos.experimental.pack_mind.pack_coordinator import PackCoordinator + +ZONES = ["north", "east", "south", "west"] + + +@pytest.mark.unit +def test_assign_never_double_assigns_a_zone() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + handed: list[str] = [] + for _ in range(len(ZONES)): + for dog in ("alpha", "bravo"): + z = c.assign_zone(dog) + if z is not None: + handed.append(z) + assert sorted(handed) == sorted(ZONES) # every zone handed exactly once + assert len(handed) == len(set(handed)) # no zone given twice → no overlap + + +@pytest.mark.unit +def test_assign_returns_none_when_exhausted() -> None: + c = PackCoordinator(["only"]) + c.start_search("red object") + assert c.assign_zone("alpha") == "only" + assert c.assign_zone("bravo") is None # nothing left for the second dog + + +@pytest.mark.unit +def test_finding_stops_the_pack() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + c.assign_zone("alpha") # claims a zone + assert not c.found + c.report_finding("alpha", "red object", "north") + assert c.found + assert c.should_stop("bravo") # the other dog learns instantly + assert c.assign_zone("bravo") is None # no more assignments after a find + + +@pytest.mark.unit +def test_finding_is_idempotent_keeps_first() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + first = c.report_finding("alpha", "red object", "north") + second = c.report_finding("bravo", "red object", "south") + assert second is first # first finding wins; pack already stopped + + +@pytest.mark.unit +def test_preferences_make_dogs_fan_out() -> None: + # Alpha prefers north end, Bravo the south end → first picks diverge. + prefs = {"alpha": ["north", "east"], "bravo": ["south", "west"]} + c = PackCoordinator(ZONES, preferences=prefs) + c.start_search("red object") + assert c.assign_zone("alpha") == "north" + assert c.assign_zone("bravo") == "south" + + +@pytest.mark.unit +def test_coverage_and_cleared() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + assert c.coverage() == 0.0 + c.report_cleared("alpha", "north") + c.report_cleared("bravo", "south") + assert c.coverage() == pytest.approx(0.5) + + +@pytest.mark.unit +def test_report_finding_falls_back_to_claimed_zone() -> None: + # The agent may report a finding with a wrong/empty zone (LLM-filled). The + # coordinator should fall back to the zone the dog actually holds claimed. + c = PackCoordinator(ZONES) + c.start_search("red object") + claimed = c.assign_zone("alpha") + f = c.report_finding("alpha", "red object", "") # empty zone + assert f.zone == claimed + c2 = PackCoordinator(ZONES) + c2.start_search("red object") + claimed2 = c2.assign_zone("alpha") + f2 = c2.report_finding("alpha", "red object", "not_a_real_zone") # bogus zone + assert f2.zone == claimed2 + + +@pytest.mark.unit +def test_release_dog_lets_survivor_inherit_unfinished_zone() -> None: + # The resilience climax: Alpha claims a zone, goes offline before clearing it; + # its zone returns to the pool so Bravo inherits and finishes the mission. + c = PackCoordinator(["north", "south"]) + c.start_search("red object") + alpha_zone = c.assign_zone("alpha") # alpha claims one + bravo_zone = c.assign_zone("bravo") # bravo claims the other + assert {alpha_zone, bravo_zone} == {"north", "south"} + # Bravo finishes its own zone, then would have nothing left... + c.report_cleared("bravo", bravo_zone) + assert c.assign_zone("bravo") is None # alpha still holds the last zone + # Alpha drops. Its claimed-but-uncleared zone must come back to the pack. + c.release_dog("alpha") + assert c.assign_zone("bravo") == alpha_zone # Bravo inherits alpha's ground + + +@pytest.mark.unit +def test_release_dog_keeps_findings_and_cleared_zones() -> None: + c = PackCoordinator(["north", "south", "east"]) + c.start_search("red object") + c.report_cleared("alpha", "north") # alpha already cleared one + z = c.assign_zone("alpha") # alpha claims another, unfinished + c.release_dog("alpha") + snap = c.snapshot() + states = {zz["name"]: zz["state"] for zz in snap["zones"]} # type: ignore[index] + assert states["north"] == "cleared" # cleared knowledge persists + assert states[z] == "unsearched" # unfinished zone reclaimed + assert "alpha" in snap["offline"] # type: ignore[operator] + + +@pytest.mark.unit +def test_finding_survives_finder_going_offline() -> None: + c = PackCoordinator(["north", "south"]) + c.start_search("red object") + c.report_finding("alpha", "red object", "north") + c.release_dog("alpha") # finder drops + assert c.found # the find outlives the robot + assert c.finding is not None and c.finding.zone == "north" + + +@pytest.mark.unit +def test_start_search_resets_ledger() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + c.report_cleared("alpha", "north") + c.report_finding("alpha", "red object", "east") + c.start_search("blue box") # new mission + assert not c.found + assert c.coverage() == 0.0 + assert c.assign_zone("alpha") is not None diff --git a/dimos/experimental/pack_mind/test_pack_coordinator_server.py b/dimos/experimental/pack_mind/test_pack_coordinator_server.py new file mode 100644 index 0000000000..14bf5b9103 --- /dev/null +++ b/dimos/experimental/pack_mind/test_pack_coordinator_server.py @@ -0,0 +1,181 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND coordinator server tests — drives the real HTTP server over requests. + +Starts the actual ``ThreadingHTTPServer`` on an ephemeral port (port 0) in a +background thread, exercising the no-overlap and stop-on-find guarantees through +the JSON API exactly as a dog's agent would. +""" + +from __future__ import annotations + +from collections.abc import Iterator +import threading + +import pytest +import requests + +from dimos.experimental.pack_mind.pack_coordinator_server import ( + _parse_prefs, + _Server, + make_server, +) + +ZONES = ["north", "east", "south", "west"] +_TIMEOUT = 5.0 + + +@pytest.fixture +def server_url() -> Iterator[str]: + server: _Server = make_server("127.0.0.1", 0, ZONES) + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{port}" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=_TIMEOUT) + + +@pytest.mark.unit +def test_two_dogs_get_distinct_zones(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red object"}, timeout=_TIMEOUT) + handed: list[str] = [] + for _ in range(len(ZONES)): + for dog in ("alpha", "bravo"): + resp = requests.post( + f"{server_url}/assign_zone", json={"dog": dog}, timeout=_TIMEOUT + ) + assert resp.status_code == 200 + zone = resp.json()["zone"] + if zone is not None: + handed.append(zone) + assert sorted(handed) == sorted(ZONES) # every zone handed exactly once + assert len(handed) == len(set(handed)) # no double-assign → no overlap + + +@pytest.mark.unit +def test_finding_stops_other_dog_and_assignment(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red object"}, timeout=_TIMEOUT) + requests.post(f"{server_url}/assign_zone", json={"dog": "alpha"}, timeout=_TIMEOUT) + + find = requests.post( + f"{server_url}/report_finding", + json={"dog": "alpha", "object": "red object", "zone": "north"}, + timeout=_TIMEOUT, + ) + assert find.status_code == 200 + assert find.json()["finding"]["zone"] == "north" + + stop = requests.get(f"{server_url}/should_stop", params={"dog": "bravo"}, timeout=_TIMEOUT) + assert stop.status_code == 200 + assert stop.json()["stop"] is True + + assigned = requests.post( + f"{server_url}/assign_zone", json={"dog": "bravo"}, timeout=_TIMEOUT + ) + assert assigned.json()["zone"] is None # no more zones handed after a find + + +@pytest.mark.unit +def test_state_reflects_coverage_after_cleared(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red object"}, timeout=_TIMEOUT) + assert requests.get(f"{server_url}/state", timeout=_TIMEOUT).json()["coverage"] == 0.0 + + requests.post( + f"{server_url}/report_cleared", json={"dog": "alpha", "zone": "north"}, timeout=_TIMEOUT + ) + requests.post( + f"{server_url}/report_cleared", json={"dog": "bravo", "zone": "south"}, timeout=_TIMEOUT + ) + state = requests.get(f"{server_url}/state", timeout=_TIMEOUT).json() + assert state["coverage"] == pytest.approx(0.5) + + +@pytest.mark.unit +def test_release_dog_reclaims_zone_over_http(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red object"}, timeout=_TIMEOUT) + alpha = requests.post( + f"{server_url}/assign_zone", json={"dog": "alpha"}, timeout=_TIMEOUT + ).json()["zone"] + bravo = requests.post( + f"{server_url}/assign_zone", json={"dog": "bravo"}, timeout=_TIMEOUT + ).json()["zone"] + requests.post( + f"{server_url}/report_cleared", json={"dog": "bravo", "zone": bravo}, timeout=_TIMEOUT + ) + # Alpha drops; its claimed zone must come back so a survivor can inherit it. + rel = requests.post(f"{server_url}/release_dog", json={"dog": "alpha"}, timeout=_TIMEOUT) + assert rel.status_code == 200 + inherited = requests.post( + f"{server_url}/assign_zone", json={"dog": "bravo"}, timeout=_TIMEOUT + ).json()["zone"] + assert inherited == alpha # bravo inherits alpha's reclaimed ground + assert "alpha" in requests.get(f"{server_url}/state", timeout=_TIMEOUT).json()["offline"] + + +@pytest.mark.unit +def test_where_is_returns_finding(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red kit"}, timeout=_TIMEOUT) + assert requests.get(f"{server_url}/where_is", timeout=_TIMEOUT).json()["found"] is False + requests.post( + f"{server_url}/report_finding", + json={"dog": "alpha", "object": "red kit", "zone": "east"}, + timeout=_TIMEOUT, + ) + w = requests.get(f"{server_url}/where_is", timeout=_TIMEOUT).json() + assert w["found"] is True and w["zone"] == "east" and w["by"] == "alpha" + + +@pytest.mark.unit +def test_report_finding_zone_optional_over_http(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red kit"}, timeout=_TIMEOUT) + claimed = requests.post( + f"{server_url}/assign_zone", json={"dog": "alpha"}, timeout=_TIMEOUT + ).json()["zone"] + # No zone field at all — coordinator falls back to alpha's claimed zone. + f = requests.post( + f"{server_url}/report_finding", + json={"dog": "alpha", "object": "red kit"}, + timeout=_TIMEOUT, + ) + assert f.status_code == 200 + assert f.json()["finding"]["zone"] == claimed + + +@pytest.mark.unit +def test_dashboard_served_at_root(server_url: str) -> None: + resp = requests.get(f"{server_url}/", timeout=_TIMEOUT) + assert resp.status_code == 200 + assert "text/html" in resp.headers.get("Content-Type", "") + assert "PACK" in resp.text and "/state" in resp.text # the polling dashboard + + +@pytest.mark.unit +def test_parse_prefs() -> None: + prefs = _parse_prefs("alpha:north,east;bravo:south,west") + assert prefs == {"alpha": ["north", "east"], "bravo": ["south", "west"]} + assert _parse_prefs("") == {} + assert _parse_prefs("garbage") == {} + + +@pytest.mark.unit +def test_bad_input_returns_400(server_url: str) -> None: + resp = requests.post(f"{server_url}/start_search", json={}, timeout=_TIMEOUT) + assert resp.status_code == 400 + missing = requests.get(f"{server_url}/should_stop", timeout=_TIMEOUT) + assert missing.status_code == 400 diff --git a/dimos/experimental/pack_mind/test_pack_search_runner.py b/dimos/experimental/pack_mind/test_pack_search_runner.py new file mode 100644 index 0000000000..2cd08ada6b --- /dev/null +++ b/dimos/experimental/pack_mind/test_pack_search_runner.py @@ -0,0 +1,84 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND runner tests — both magic beats end-to-end, hardware-free.""" + +import pytest + +from dimos.experimental.pack_mind.pack_coordinator import PackCoordinator +from dimos.experimental.pack_mind.pack_search_runner import ( + MockDriver, + act_on_finding, + run_search, +) + +ZONES = ["north", "east", "south", "west"] + + +@pytest.mark.unit +def test_single_dog_finds_target() -> None: + c = PackCoordinator(ZONES) + c.start_search("red kit") + res = run_search(c, MockDriver(target_zone="south"), "alpha", "red kit") + assert res.found and res.zone == "south" + assert c.found + + +@pytest.mark.unit +def test_two_dogs_split_work_and_one_finds() -> None: + # Preferences fan them out; only one zone holds the target. + prefs = {"alpha": ["north", "east"], "bravo": ["west", "south"]} + c = PackCoordinator(ZONES, preferences=prefs) + c.start_search("red kit") + # Interleave is unnecessary: whichever reaches "south" reports it and the other stops. + a = run_search(c, MockDriver(target_zone="south"), "alpha", "red kit") + b = run_search(c, MockDriver(target_zone="south"), "bravo", "red kit") + assert c.found + # No zone searched by both dogs (no overlap). + assert not (set(a.visited) & set(b.visited)) + + +@pytest.mark.unit +def test_handoff_acts_on_teammates_memory() -> None: + c = PackCoordinator(ZONES) + c.start_search("red kit") + run_search(c, MockDriver(target_zone="north"), "alpha", "red kit") # alpha finds + bravo_driver = MockDriver(target_zone="north") + msg = act_on_finding(c, bravo_driver, "bravo") # bravo never searched + assert "north" in msg and "alpha" in msg + assert bravo_driver.visited[-1] == "north" # bravo physically went on alpha's memory + + +@pytest.mark.unit +def test_inheritance_survivor_finishes_downed_dogs_mission() -> None: + # Alpha claims the target's zone but goes offline before clearing it; Bravo, + # already out of its own zones, inherits it and completes the mission. + prefs = {"alpha": ["south"], "bravo": ["north", "east", "west"]} + c = PackCoordinator(ZONES, preferences=prefs) + c.start_search("red kit") + alpha_zone = c.assign_zone("alpha") # alpha claims "south" (where the kit is) + assert alpha_zone == "south" + # Bravo clears its three zones, none have the kit. + bravo = MockDriver(target_zone="south") + for _ in range(3): + z = c.assign_zone("bravo") + assert z is not None + bravo.goto(z) + c.report_cleared("bravo", z) + assert c.assign_zone("bravo") is None # alpha still holds "south" + # Alpha drops. Bravo's loop resumes and inherits "south", finding the kit. + c.release_dog("alpha") + res = run_search(c, bravo, "bravo", "red kit") + assert res.found and res.zone == "south" + assert c.finding is not None and c.finding.by == "bravo" From e8127e42e1ff65e9511968f442d83901e6b77763 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 00:19:11 -0600 Subject: [PATCH 19/35] feat: PACK MIND paced live dashboard demo + deterministic mock dog - demo_pack_live: one command starts the coordinator + dashboard and plays the v4 inheritance climax with pauses, so the projector animates the full story (two dogs fan out with no overlap, a dog goes offline, the survivor inherits its unfinished zone and finds the object there) - mock_dog: add --target-zone (deterministic find in one place, like reality), --reset (call start_search first so re-runs reset the ledger), and --dwell (slow per-zone pacing for the dashboard) --- .../experimental/pack_mind/demo_pack_live.py | 113 ++++++++++++++++++ dimos/experimental/pack_mind/mock_dog.py | 36 +++++- 2 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 dimos/experimental/pack_mind/demo_pack_live.py diff --git a/dimos/experimental/pack_mind/demo_pack_live.py b/dimos/experimental/pack_mind/demo_pack_live.py new file mode 100644 index 0000000000..0d333001e0 --- /dev/null +++ b/dimos/experimental/pack_mind/demo_pack_live.py @@ -0,0 +1,113 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — paced LIVE narrative for the projector dashboard. + +One command: starts the coordinator + dashboard AND plays the full v4 story with +pauses, so you just open the browser and watch the shared memory animate. Drives +the real coordinator (the dashboard polls the same state), deterministically — no +hardware, no racing mock dogs. + + uv run python -m dimos.experimental.pack_mind.demo_pack_live --pace 2.0 + # then open http://localhost:8090 on the projector + +The story it plays (the inheritance climax — two dogs, no overlap, a dog drops, +the survivor inherits and finishes): + start_search → Alpha claims the target zone → Bravo clears the others (no + overlap) → Alpha goes OFFLINE (its zone reclaimed) → Bravo inherits it and finds + the object there. +""" + +from __future__ import annotations + +import argparse +import threading +import time + +from dimos.experimental.pack_mind.pack_coordinator_server import ( + DEFAULT_COORDINATOR_PORT, + make_server, +) + + +def _narrate(tag: str, msg: str) -> None: + print(f" {tag:<26} {msg}", flush=True) + + +def play(target: str, target_zone: str, zones: list[str], port: int, host: str, pace: float) -> None: + others = [z for z in zones if z != target_zone] + # Alpha prefers the target zone (so it holds it unfinished); Bravo prefers the rest. + prefs = {"alpha": [target_zone, *others], "bravo": [*others, target_zone]} + server = make_server(host, port, zones, prefs) + actual = server.server_address[1] + threading.Thread(target=server.serve_forever, daemon=True).start() + c = server.coordinator + + print(f"\nPACK MIND live demo on http://{host}:{actual}") + print(f" → open that in the projector browser now. Starting in {pace * 2:.0f}s…\n") + time.sleep(pace * 2) + + print("=== mission: find the red kit (two dogs, one shared memory) ===") + _narrate("MISSION", c.start_search(target)) + time.sleep(pace) + + _narrate("ALPHA CLAIMS", c.assign_zone("alpha") or "-") # the target zone, unfinished + time.sleep(pace) + + while True: + z = c.assign_zone("bravo") + if not z: + break + time.sleep(pace) + c.report_cleared("bravo", z) + _narrate("BRAVO CLEARED", f"{z} (no overlap — never alpha's ground)") + time.sleep(pace * 0.5) + + print("\n=== the real test: lose a dog mid-mission ===") + time.sleep(pace) + _narrate("ALPHA OFFLINE", c.release_dog("alpha")) + time.sleep(pace) + + inherited = c.assign_zone("bravo") + _narrate("BRAVO INHERITS", f"{inherited} — alpha started this zone, never finished") + time.sleep(pace) + f = c.report_finding("bravo", target, inherited or "") + _narrate("BRAVO FINDS IT", f"{f.object} in {f.zone} — mission complete") + + print("\nAlpha never finished. Bravo inherited its ground and found it. — PACK MIND") + print("(dashboard stays live; Ctrl-C to exit)\n") + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + server.shutdown() + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND paced live dashboard demo") + p.add_argument("--target", default="red kit") + p.add_argument("--target-zone", default="south", help="where the object is") + p.add_argument("--zones", default="north,east,south,west") + p.add_argument("--port", type=int, default=DEFAULT_COORDINATOR_PORT) + p.add_argument("--host", default="0.0.0.0") + p.add_argument("--pace", type=float, default=2.0, help="seconds between beats") + return p.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + zones = [z.strip() for z in args.zones.split(",") if z.strip()] + if args.target_zone not in zones: + raise SystemExit(f"--target-zone {args.target_zone!r} must be one of {zones}") + play(args.target, args.target_zone, zones, args.port, args.host, args.pace) diff --git a/dimos/experimental/pack_mind/mock_dog.py b/dimos/experimental/pack_mind/mock_dog.py index 8c2c4922b9..04a91b1686 100644 --- a/dimos/experimental/pack_mind/mock_dog.py +++ b/dimos/experimental/pack_mind/mock_dog.py @@ -65,7 +65,13 @@ def _get(url: str, path: str, params: dict[str, str]) -> dict[str, Any] | None: def run_dog( - dog: str, url: str, target: str, find_prob: float, rng: random.Random + dog: str, + url: str, + target: str, + find_prob: float, + rng: random.Random, + target_zone: str | None = None, + dwell: float | None = None, ) -> None: url = url.rstrip("/") while True: @@ -81,9 +87,12 @@ def run_dog( return print(f"[{dog}] assigned zone: {zone}") - time.sleep(rng.uniform(_MIN_SLEEP, _MAX_SLEEP)) + time.sleep(dwell if dwell is not None else rng.uniform(_MIN_SLEEP, _MAX_SLEEP)) - if rng.random() < find_prob: + # Deterministic find when --target-zone is set (the object lives in one + # place, like reality); otherwise fall back to the random per-zone chance. + hit = zone == target_zone if target_zone else rng.random() < find_prob + if hit: result = _post( url, "/report_finding", {"dog": dog, "object": target, "zone": zone} ) @@ -111,11 +120,30 @@ def main() -> None: default=0.0, help="Per-zone probability of reporting a finding (1.0 = always find).", ) + parser.add_argument( + "--target-zone", + default=None, + help="Deterministic: only 'find' the object in this zone (overrides --find-prob).", + ) + parser.add_argument( + "--dwell", type=float, default=None, help="Seconds per zone (slows the dashboard for demos)." + ) + parser.add_argument( + "--reset", + action="store_true", + help="Call start_search first to reset the ledger and set the target.", + ) parser.add_argument("--seed", type=int, default=None, help="RNG seed for repeatability.") args = parser.parse_args() + if args.reset: + _post(args.url.rstrip("/"), "/start_search", {"target": args.target}) + print(f"[{args.dog}] reset ledger; searching for {args.target}.") + rng = random.Random(args.seed) - run_dog(args.dog, args.url, args.target, args.find_prob, rng) + run_dog( + args.dog, args.url, args.target, args.find_prob, rng, args.target_zone, args.dwell + ) if __name__ == "__main__": From f81c59c3499eaf1552e900a347e257438ac6cc4a Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 00:29:18 -0600 Subject: [PATCH 20/35] chore: register unitree-go2-pack blueprint in all_blueprints.py --- dimos/robot/all_blueprints.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 8792484743..9bd664df27 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -104,6 +104,7 @@ "unitree-go2-keyboard-teleop": "dimos.robot.unitree.go2.blueprints.basic.unitree_go2_keyboard_teleop:unitree_go2_keyboard_teleop", "unitree-go2-markers": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2:unitree_go2_markers", "unitree-go2-memory": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2:unitree_go2_memory", + "unitree-go2-pack": "dimos.experimental.pack_mind.live:unitree_go2_pack", "unitree-go2-relocalization": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2:unitree_go2_relocalization", "unitree-go2-ros": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2_ros:unitree_go2_ros", "unitree-go2-security": "dimos.robot.unitree.go2.blueprints.agentic.unitree_go2_security:unitree_go2_security", @@ -184,6 +185,7 @@ "object-tracking": "dimos.perception.object_tracker.ObjectTracking", "osm-skill": "dimos.agents.skills.osm.OsmSkill", "pack-mind-sim-module": "dimos.experimental.pack_mind.blueprint.PackMindSimModule", + "pack-search-skills": "dimos.experimental.pack_mind.pack_search_skills.PackSearchSkills", "path-follower": "dimos.navigation.nav_stack.modules.path_follower.path_follower.PathFollower", "patrolling-module": "dimos.navigation.patrolling.module.PatrollingModule", "perceive-loop-skill": "dimos.perception.perceive_loop_skill.PerceiveLoopSkill", From a079e1d67868ff713f18c6cdc30974486747de29 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 00:33:17 -0600 Subject: [PATCH 21/35] feat: 3D arena view for the PACK MIND explore viewer (--3d) Render the fog floor via OccupancyGrid.to_rerun (matching the live SLAM map), extrude explored walls into Boxes3D for real volume, and aim an orbital camera at the arena centre so the scene auto-frames instead of landing edge-on. --- .../pack_mind/view_explore_rerun.py | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/dimos/experimental/pack_mind/view_explore_rerun.py b/dimos/experimental/pack_mind/view_explore_rerun.py index 782fe8f6a8..a492b68379 100644 --- a/dimos/experimental/pack_mind/view_explore_rerun.py +++ b/dimos/experimental/pack_mind/view_explore_rerun.py @@ -33,6 +33,7 @@ import numpy as np import rerun as rr +import rerun.blueprint as rrb from dimos.experimental.pack_mind.explore_sim import ( ExploreSim, @@ -165,21 +166,33 @@ def _log_run_3d(root: str, metric: str, sim: ExploreSim, frames: list[dict[str, world = sim.world res = float(world.resolution) origin = world.origin + ox = float(origin.position.x) + oy = float(origin.position.y) label = sim.target_label + wall_h = max(0.3, res * 4) # extrude walls so the scene has 3D volume to frame for fr in frames: _set_tick(fr["tick"]) codes = fr["codes"] - # Reveal the real occupancy through the fog, rendered by the SAME textured - # -quad path the live SLAM map uses (OccupancyGrid.to_rerun) so it matches - # `dimos run unitree-go2`: unknown stays transparent, explored free -> 0, - # explored wall -> 100. The floor texture grows as the pack searches. + # Floor: explored free space, rendered through the SAME textured-quad path + # the live SLAM map uses (OccupancyGrid.to_rerun) so it matches + # `dimos run unitree-go2`. Unexplored stays dark; the purple grows as the + # pack searches. fog_cost = np.full(codes.shape, -1, dtype=np.int8) fog_cost[(codes == 1) | (codes == 2)] = 0 - fog_cost[(codes == 3) | (codes == 4)] = 100 revealed = OccupancyGrid(grid=fog_cost, resolution=res, origin=origin) rr.log(f"{root}/map", revealed.to_rerun()) + # Walls: explored occupied cells extruded into standing boxes. Gives the + # camera real 3D structure to auto-frame (a flat floor alone reads edge-on). + wr, wc = np.nonzero((codes == 3) | (codes == 4)) + if wr.size: + centers = np.column_stack( + [ox + wc * res, oy + wr * res, np.full(wr.size, wall_h / 2, np.float32)] + ).astype(np.float32) + half = np.tile([res / 2, res / 2, wall_h / 2], (wr.size, 1)).astype(np.float32) + rr.log(f"{root}/walls", rr.Boxes3D(centers=centers, half_sizes=half, colors=[90, 100, 150])) + dpts = [[x, y, 0.25] for x, y, _, _ in fr["dogs"]] dcols = [_hex_rgb(color) if online else _OFFLINE for _, _, color, online in fr["dogs"]] rr.log(f"{root}/dogs", rr.Points3D(dpts, colors=dcols, radii=res * 2.5)) @@ -224,6 +237,33 @@ def view( rr.spawn() # opens the DimOS Viewer (Rerun) window sim = _build(map_name, n_dogs, seed, target_label, shared) + + if threed: + # Auto-framing lands the camera in-plane on a flat floor (you see only the + # grid). Explicitly aim an orbital eye down at the arena centre from above + # and to the side, sized to the arena span. + w_m = sim.world.grid.shape[1] * sim.world.resolution + h_m = sim.world.grid.shape[0] * sim.world.resolution + cx = sim.world.origin.position.x + w_m / 2.0 + cy = sim.world.origin.position.y + h_m / 2.0 + span = max(w_m, h_m) + rr.send_blueprint( + rrb.Blueprint( + rrb.Spatial3DView( + origin="maze", + background=rrb.Background(kind="SolidColor", color=[0, 0, 0]), + line_grid=rrb.LineGrid3D(plane=rr.components.Plane3D.XY.with_distance(0.0)), + eye_controls=rrb.EyeControls3D( + kind=rrb.Eye3DKind.Orbital, + position=[cx, cy - span * 1.1, span * 1.0], + look_target=[cx, cy, 0.0], + eye_up=[0.0, 0.0, 1.0], + ), + ), + collapse_panels=True, + ) + ) + frames = _capture(sim, max_ticks, kill) logger = _log_run_3d if threed else _log_run logger("maze", "maze/revealed", sim, frames) From fa8eda5429f11cfb34e7cccab596a042411044a4 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 01:40:05 -0600 Subject: [PATCH 22/35] refactor: drop incomplete 3D view from pack_mind explore viewer The --3d mode rendered the occupancy floor + extruded walls but never wired a working playback timeline in rerun 0.32 (recording showed no scrubbable tick axis), so it was a dead end for the demo. Real 3D map visuals come from LiDAR replay (`dimos run unitree-go2`), not the 2D coverage sim. Keep the working 2D fog A/B (--independent for the shared-vs-private baseline); that is the deliverable that proves the shared-memory thesis. --- .../pack_mind/view_explore_rerun.py | 102 +----------------- 1 file changed, 1 insertion(+), 101 deletions(-) diff --git a/dimos/experimental/pack_mind/view_explore_rerun.py b/dimos/experimental/pack_mind/view_explore_rerun.py index a492b68379..c29c1c7651 100644 --- a/dimos/experimental/pack_mind/view_explore_rerun.py +++ b/dimos/experimental/pack_mind/view_explore_rerun.py @@ -33,14 +33,12 @@ import numpy as np import rerun as rr -import rerun.blueprint as rrb from dimos.experimental.pack_mind.explore_sim import ( ExploreSim, build_explore, build_explore_building, ) -from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid from dimos.visualization.rerun.init import rerun_init # Per-cell fog palette, indexed by state() code: @@ -156,68 +154,6 @@ def to_img_xy(wx: float, wy: float) -> tuple[float, float]: rr.log(f"{root}/stats", rr.TextLog(msg)) -def _log_run_3d(root: str, metric: str, sim: ExploreSim, frames: list[dict[str, Any]]) -> None: - """Log the run as 3D world entities over the occupancy floor (SLAM-map look). - - Explored cells light up as Points3D on the z=0 floor (unknown stays dark, so - fog peels back in 3D); dogs and trails ride just above. Dog/trail/target - coordinates are already in world meters; floor cells map via origin + grid*res. - """ - world = sim.world - res = float(world.resolution) - origin = world.origin - ox = float(origin.position.x) - oy = float(origin.position.y) - label = sim.target_label - wall_h = max(0.3, res * 4) # extrude walls so the scene has 3D volume to frame - - for fr in frames: - _set_tick(fr["tick"]) - codes = fr["codes"] - # Floor: explored free space, rendered through the SAME textured-quad path - # the live SLAM map uses (OccupancyGrid.to_rerun) so it matches - # `dimos run unitree-go2`. Unexplored stays dark; the purple grows as the - # pack searches. - fog_cost = np.full(codes.shape, -1, dtype=np.int8) - fog_cost[(codes == 1) | (codes == 2)] = 0 - revealed = OccupancyGrid(grid=fog_cost, resolution=res, origin=origin) - rr.log(f"{root}/map", revealed.to_rerun()) - - # Walls: explored occupied cells extruded into standing boxes. Gives the - # camera real 3D structure to auto-frame (a flat floor alone reads edge-on). - wr, wc = np.nonzero((codes == 3) | (codes == 4)) - if wr.size: - centers = np.column_stack( - [ox + wc * res, oy + wr * res, np.full(wr.size, wall_h / 2, np.float32)] - ).astype(np.float32) - half = np.tile([res / 2, res / 2, wall_h / 2], (wr.size, 1)).astype(np.float32) - rr.log(f"{root}/walls", rr.Boxes3D(centers=centers, half_sizes=half, colors=[90, 100, 150])) - - dpts = [[x, y, 0.25] for x, y, _, _ in fr["dogs"]] - dcols = [_hex_rgb(color) if online else _OFFLINE for _, _, color, online in fr["dogs"]] - rr.log(f"{root}/dogs", rr.Points3D(dpts, colors=dcols, radii=res * 2.5)) - - strips, scols = [], [] - for (_, _, color, _), trail in zip(fr["dogs"], fr["trails"]): - if len(trail) > 1: - strips.append([[px, py, 0.1] for px, py in trail]) - scols.append(_hex_rgb(color)) - if strips: - rr.log(f"{root}/trails", rr.LineStrips3D(strips, colors=scols, radii=res * 0.6)) - - tgt = fr["target"] - if tgt is not None: - tcol = [[60, 220, 90]] if tgt["found"] else [[235, 40, 40]] - rr.log(f"{root}/target", rr.Points3D([[tgt["x"], tgt["y"], 0.3]], colors=tcol, radii=res * 3.0)) - - rr.log(metric, rr.Scalars(fr["revealed"])) - if tgt is not None and tgt["found"]: - msg = f"FOUND {label} — {tgt['found_by']} @ tick {tgt['found_tick']} · searched {fr['revealed']:.0%}" - else: - msg = f"searching for {label} … searched {fr['revealed']:.0%} · tick {fr['tick']}" - rr.log(f"{root}/stats", rr.TextLog(msg)) - - def view( map_name: str = "maze", n_dogs: int = 3, @@ -227,7 +163,6 @@ def view( kill: tuple[int, int] | None = None, save: str | None = None, shared: bool = True, - threed: bool = False, ) -> None: mode = "shared" if shared else "independent" rerun_init(app_id=f"pack_mind_explore_{mode}") @@ -237,36 +172,8 @@ def view( rr.spawn() # opens the DimOS Viewer (Rerun) window sim = _build(map_name, n_dogs, seed, target_label, shared) - - if threed: - # Auto-framing lands the camera in-plane on a flat floor (you see only the - # grid). Explicitly aim an orbital eye down at the arena centre from above - # and to the side, sized to the arena span. - w_m = sim.world.grid.shape[1] * sim.world.resolution - h_m = sim.world.grid.shape[0] * sim.world.resolution - cx = sim.world.origin.position.x + w_m / 2.0 - cy = sim.world.origin.position.y + h_m / 2.0 - span = max(w_m, h_m) - rr.send_blueprint( - rrb.Blueprint( - rrb.Spatial3DView( - origin="maze", - background=rrb.Background(kind="SolidColor", color=[0, 0, 0]), - line_grid=rrb.LineGrid3D(plane=rr.components.Plane3D.XY.with_distance(0.0)), - eye_controls=rrb.EyeControls3D( - kind=rrb.Eye3DKind.Orbital, - position=[cx, cy - span * 1.1, span * 1.0], - look_target=[cx, cy, 0.0], - eye_up=[0.0, 0.0, 1.0], - ), - ), - collapse_panels=True, - ) - ) - frames = _capture(sim, max_ticks, kill) - logger = _log_run_3d if threed else _log_run - logger("maze", "maze/revealed", sim, frames) + _log_run("maze", "maze/revealed", sim, frames) tgt = sim.state()["target"] if tgt is not None and tgt["found"]: @@ -306,12 +213,6 @@ def _parse_args() -> argparse.Namespace: action="store_true", help="give each dog a private map (A/B baseline); default is PACK MIND shared memory", ) - p.add_argument( - "--3d", - dest="threed", - action="store_true", - help="render in 3D over the occupancy floor (SLAM-map look) instead of the flat 2D fog", - ) return p.parse_args() @@ -327,5 +228,4 @@ def _parse_args() -> argparse.Namespace: kill=kill, save=args.save, shared=not args.independent, - threed=args.threed, ) From a922f8e9a577ef9ebd4106b46cb67a512c65a1d6 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 02:36:58 -0600 Subject: [PATCH 23/35] fix: PACK MIND live blueprint deploys on CPU (disable EdgeTAM/CUDA modules) EdgeTAM segmentation hard-requires a CUDA GPU and reaches the spatial stack two ways, both fatal on a CPU/CoreML ground station (e.g. a Mac): - unitree_go2_spatial -> SecurityModule.__init__ -> EdgeTAMProcessor() (deploy crash) - PersonFollowSkillContainer.follow_person -> EdgeTAMProcessor() (mid-demo crash) The pack demo uses neither, so disable both via .disabled_modules(). The blueprint then deploys clean on CPU with no --disable flag and no call-time landmine. Verified: active_blueprints = 19 modules, both excluded, PackSearchSkills + MCP present. --- dimos/experimental/pack_mind/live.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dimos/experimental/pack_mind/live.py b/dimos/experimental/pack_mind/live.py index d4cd0adbbf..0170809b96 100644 --- a/dimos/experimental/pack_mind/live.py +++ b/dimos/experimental/pack_mind/live.py @@ -40,8 +40,10 @@ from dimos.agents.mcp.mcp_client import McpClient from dimos.agents.mcp.mcp_server import McpServer +from dimos.agents.skills.person_follow import PersonFollowSkillContainer from dimos.core.coordination.blueprints import autoconnect from dimos.experimental.pack_mind.pack_search_skills import PackSearchSkills +from dimos.experimental.security_demo.security_module import SecurityModule from dimos.robot.unitree.go2.blueprints.agentic._common_agentic import _common_agentic from dimos.robot.unitree.go2.blueprints.smart.unitree_go2_spatial import unitree_go2_spatial @@ -68,12 +70,20 @@ # One dog per laptop. PackSearchSkills reads PACK_DOG_NAME + PACK_COORDINATOR_URL # from the environment, so this single blueprint serves both laptops. +# +# EdgeTAM (segmentation) HARD-requires a CUDA GPU. It reaches the spatial stack two +# ways, both fatal on a CPU/CoreML host (e.g. a Mac ground station): +# 1. unitree_go2_spatial -> SecurityModule.__init__ -> EdgeTAMProcessor() → deploy crash +# 2. PersonFollowSkillContainer.follow_person -> EdgeTAMProcessor() → mid-demo crash +# The pack demo uses neither (we move via skills + look_out_for, not security +# patrol or person-follow), so disable both. The blueprint then deploys clean on +# CPU — no --disable flag and no call-time landmine. unitree_go2_pack = autoconnect( unitree_go2_spatial, McpServer.blueprint(), McpClient.blueprint(system_prompt=PACK_SYSTEM_PROMPT), _common_agentic, PackSearchSkills.blueprint(), -) +).disabled_modules(SecurityModule, PersonFollowSkillContainer) __all__ = ["unitree_go2_pack", "PACK_SYSTEM_PROMPT"] From 1673ea40c02bda162059f7f9ea47c2586fa75dcb Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 05:54:24 -0600 Subject: [PATCH 24/35] docs: PACK MIND live 2-dog/2-laptop bring-up + model prefetch script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So a second laptop+dog can collaborate from a clean checkout: - prefetch_live_models.py: one command caches the runtime models the live stack needs (moondream2 for look_out_for, faster-whisper-base for STT) so the demo runs fully offline (HF_HUB_OFFLINE=1) on a dog's internet-less AP / flaky venue WiFi - README: 2-dog/2-laptop bring-up — per-laptop env + run, router/STA vs AP, dashboard, and the hard-won gotchas (no --robot-ip flag, EdgeTAM already disabled, offline mode, moondream first-call latency, bring-up gate order) --- dimos/experimental/pack_mind/README.md | 56 +++++++++++++++++ .../pack_mind/prefetch_live_models.py | 60 +++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 dimos/experimental/pack_mind/prefetch_live_models.py diff --git a/dimos/experimental/pack_mind/README.md b/dimos/experimental/pack_mind/README.md index 4de50ab4fa..e440463a2e 100644 --- a/dimos/experimental/pack_mind/README.md +++ b/dimos/experimental/pack_mind/README.md @@ -58,6 +58,55 @@ bin/pytest-fast dimos/experimental/pack_mind/test_explore_sim.py -v bin/pytest-fast dimos/experimental/pack_mind/test_pack_mind_sim.py -v ``` +## Live demo — 2 dogs / 2 laptops + +The intelligence (shared memory) runs on the laptops; the dogs are bodies. Movement +is teleop-assisted; the agent + coordinator own the memory layer. + +**1. One-time per laptop (with internet):** cache the runtime models so the demo runs +offline (a dog's WiFi AP has no internet, and venue WiFi is flaky): +```bash +uv run python -m dimos.experimental.pack_mind.prefetch_live_models # moondream2 + whisper +``` +CLIP/YOLO ship in `data/` (git-lfs). The agent brain (GPT-4o) is a hosted API — keep +the coordinator laptop on a router with WAN. + +**2. Network:** one travel router; put each Go2 in STA mode (Unitree app) onto it; both +laptops + both dogs on that subnet. DimOS picks the WebRTC method purely from +`ROBOT_IP` — `192.168.12.1` → the dog's own AP (single-laptop only), any other IP → +LocalSTA (shared router, required for 2 laptops). + +**3. Laptop A (Alpha + coordinator + dashboard):** +```bash +export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +export ROBOT_IP= LISTEN_HOST=0.0.0.0 OPENAI_API_KEY= +uv run python -m dimos.experimental.pack_mind.pack_coordinator_server \ + --zones north,east,south,west --prefs "alpha:north,east;bravo:south,west" & # dashboard: http://:8090 +PACK_DOG_NAME=alpha PACK_COORDINATOR_URL=http://127.0.0.1:8090 \ + uv run dimos run unitree-go2-pack --daemon +``` + +**4. Laptop B (Bravo):** +```bash +export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +export ROBOT_IP= LISTEN_HOST=0.0.0.0 OPENAI_API_KEY= +PACK_DOG_NAME=bravo PACK_COORDINATOR_URL=http://:8090 \ + uv run dimos run unitree-go2-pack --daemon +``` + +**Gotchas (hard-won on site):** +- `unitree-go2-pack` already disables EdgeTAM modules (SecurityModule, PersonFollow) — + they hard-require CUDA. No `--disable` flag needed. +- `dimos run` has **no `--robot-ip` flag** in this build — set the `ROBOT_IP` env and + verify with `dimos show-config`. +- `HF_HUB_OFFLINE=1` is required on a no-internet AP; run prefetch first. +- moondream2's **first** `look_out_for` is slow (~10–60s, CPU compile) — a lag, not a crash. +- Bring-up order: ping dog → `dimos show-config` → run to "running" → `mcp list-tools` + + `speak` → `look_out_for` → `start_search`/`next_zone` against the coordinator. + +**Hardware-free rehearsal (no dogs):** +`uv run python -m dimos.experimental.pack_mind.demo_pack_live --pace 2`, open http://localhost:8090. + ## File map | File | Role | @@ -67,6 +116,13 @@ bin/pytest-fast dimos/experimental/pack_mind/test_pack_mind_sim.py -v | `server.py` | FastAPI WebSocket backend streaming both explore sims | | `static/explore.html` | Three.js 3D fog-of-war frontend (side-by-side, kill/reset) | | `view_explore_rerun.py` | **DimOS Viewer (Rerun)** view of one shared-memory maze search (fog + dogs + trails + coverage scalar) | +| `live.py` | **Live blueprint** `unitree-go2-pack` (one dog per laptop, EdgeTAM disabled) + PACK system prompt | +| `pack_coordinator.py` / `pack_coordinator_server.py` | Shared zone ledger (no-overlap, find, **inheritance**) + JSON/HTTP API + dashboard route | +| `pack_dashboard.html` | Projector dashboard — zones, finding, offline/inheritance, causal chain | +| `pack_search_skills.py` | Dog agent tools: `start_search` / `next_zone` / `report_*` / `where_is` | +| `pack_search_runner.py` | `RobotDriver`-protocol search loop + `MockDriver` | +| `mock_dog.py` / `demo_pack_live.py` / `demo_pack_scene.py` | Hardware-free test + projector rehearsal of both magic beats | +| `prefetch_live_models.py` | Pre-cache moondream2 + whisper so the live stack runs offline | | `sim.py` / `sim_robot.py` | Coverage-race sim (known map) | | `blueprint.py` | `pack-mind-sim` native DimOS blueprint (publishes coverage as `OccupancyGrid`) | | `render.py` | Standalone matplotlib → mp4 A/B render | diff --git a/dimos/experimental/pack_mind/prefetch_live_models.py b/dimos/experimental/pack_mind/prefetch_live_models.py new file mode 100644 index 0000000000..4c72c7da87 --- /dev/null +++ b/dimos/experimental/pack_mind/prefetch_live_models.py @@ -0,0 +1,60 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pre-cache the HuggingFace models the live PACK MIND stack loads at runtime. + +Run this ONCE on each demo laptop while it has real internet. Afterwards the demo +runs fully offline with ``HF_HUB_OFFLINE=1`` — every model loads from the local +cache, so a flaky venue network (or the dog's internet-less WiFi AP) can't stall a +model load mid-demo. + + uv run python -m dimos.experimental.pack_mind.prefetch_live_models + +What these are (and why they bite if missing): + - moondream2: the VLM behind ``look_out_for`` (GlobalConfig.detection_model = + "moondream"). Loads lazily on the FIRST detection — uncached = crash mid-demo. + - faster-whisper-base: WebInput's voice STT, loaded at module start — uncached + + HF_HUB_OFFLINE=1 = the start-time LocalEntryNotFoundError. + +CLIP and YOLO ship in ``data/`` (git-lfs), so they are not fetched here. The +LLM/agent brain (GPT-4o / Qwen-VL) is a hosted API, not a local download — it needs +internet at demo time regardless (run the coordinator laptop on a router with WAN). +""" + +from __future__ import annotations + +import os + +# Force online for this one run, even if the shell exported HF_HUB_OFFLINE=1. +os.environ["HF_HUB_OFFLINE"] = "0" +os.environ["TRANSFORMERS_OFFLINE"] = "0" + +from huggingface_hub import snapshot_download # noqa: E402 + +MODELS: list[tuple[str, str]] = [ + ("vikhyatk/moondream2", "look_out_for detection VLM (detection_model=moondream)"), + ("Systran/faster-whisper-base", "WebInput voice STT (faster-whisper)"), +] + + +def main() -> None: + for repo, why in MODELS: + print(f"\n== {repo} — {why}") + path = snapshot_download(repo) + print(f" cached: {path}") + print("\nDone. Run the live demo with HF_HUB_OFFLINE=1 — models load from cache.") + + +if __name__ == "__main__": + main() From 1d47a9cd76a58a32aa226a61c9ed9e62ec935c6f Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 06:34:06 -0600 Subject: [PATCH 25/35] fix: prefetch moondream via full from_pretrained (pulls trust_remote_code modules + starmie-v1 tokenizer) --- .../pack_mind/prefetch_live_models.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/dimos/experimental/pack_mind/prefetch_live_models.py b/dimos/experimental/pack_mind/prefetch_live_models.py index 4c72c7da87..c3fe7d1747 100644 --- a/dimos/experimental/pack_mind/prefetch_live_models.py +++ b/dimos/experimental/pack_mind/prefetch_live_models.py @@ -42,17 +42,28 @@ from huggingface_hub import snapshot_download # noqa: E402 -MODELS: list[tuple[str, str]] = [ - ("vikhyatk/moondream2", "look_out_for detection VLM (detection_model=moondream)"), +# Plain snapshot is enough for these (loaded straight from the cached dir). +SNAPSHOT_MODELS: list[tuple[str, str]] = [ ("Systran/faster-whisper-base", "WebInput voice STT (faster-whisper)"), ] def main() -> None: - for repo, why in MODELS: + for repo, why in SNAPSHOT_MODELS: print(f"\n== {repo} — {why}") - path = snapshot_download(repo) - print(f" cached: {path}") + print(f" cached: {snapshot_download(repo)}") + + # moondream2 is loaded via trust_remote_code, which pulls THREE things a plain + # snapshot_download misses: the weights, the custom code modules + # (transformers_modules/...), and a SEPARATE tokenizer repo it references + # (moondream/starmie-v1). A full from_pretrained fetches all of them, so the + # live look_out_for then loads fully offline. + print("\n== vikhyatk/moondream2 — look_out_for VLM (full load for transitive deps)") + from transformers import AutoModelForCausalLM + + AutoModelForCausalLM.from_pretrained("vikhyatk/moondream2", trust_remote_code=True) + print(" moondream2 + code modules + starmie-v1 tokenizer cached") + print("\nDone. Run the live demo with HF_HUB_OFFLINE=1 — models load from cache.") From 2018919fe5910421fced63d05ef3875948ebd9c9 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 07:04:27 -0600 Subject: [PATCH 26/35] feat: PACK MIND fast GPU-free red-object detector for the live find beat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit moondream VLM on a CPU ground station is slow (tens of seconds, async) and fragile. The demo target is defined by COLOUR, so detect it with a colour filter: - red_detector.py: RedObjectDetector watches the camera; @skill look_for_red does a per-frame RGB ratio test (milliseconds, deterministic) and, on a hit, reports the finding to the coordinator (blank zone → coordinator fills the claimed zone) - wired into unitree-go2-pack; PACK prompt now uses look_for_red instead of look_out_for - 5 unit tests on the detection logic (hardware-free) --- dimos/experimental/pack_mind/live.py | 9 +- dimos/experimental/pack_mind/red_detector.py | 122 ++++++++++++++++++ .../pack_mind/test_red_detector.py | 58 +++++++++ 3 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 dimos/experimental/pack_mind/red_detector.py create mode 100644 dimos/experimental/pack_mind/test_red_detector.py diff --git a/dimos/experimental/pack_mind/live.py b/dimos/experimental/pack_mind/live.py index 0170809b96..5820703d72 100644 --- a/dimos/experimental/pack_mind/live.py +++ b/dimos/experimental/pack_mind/live.py @@ -43,6 +43,7 @@ from dimos.agents.skills.person_follow import PersonFollowSkillContainer from dimos.core.coordination.blueprints import autoconnect from dimos.experimental.pack_mind.pack_search_skills import PackSearchSkills +from dimos.experimental.pack_mind.red_detector import RedObjectDetector from dimos.experimental.security_demo.security_module import SecurityModule from dimos.robot.unitree.go2.blueprints.agentic._common_agentic import _common_agentic from dimos.robot.unitree.go2.blueprints.smart.unitree_go2_spatial import unitree_go2_spatial @@ -58,10 +59,9 @@ a. Call next_zone to get YOUR assigned zone. If it returns empty, the search is over — STOP and report status. b. navigate_with_text("") to travel there. - c. look_out_for([""], then={"name": "report_finding", - "args": {"object": "", "zone": ""}}) so a sighting - auto-tells the whole pack. - d. Scan briefly. If you did not see it, call report_cleared(""). + c. Call look_for_red to check the camera for the red object. If it reports one, + it has already told the whole pack — STOP. + d. If no red object, call report_cleared("") and continue. e. Check should_stop — if true, a packmate found it; STOP. 3. If asked where the target is, call where_is(target) and act on the answer even if you never saw it yourself — that is acting on the pack's shared memory. @@ -84,6 +84,7 @@ McpClient.blueprint(system_prompt=PACK_SYSTEM_PROMPT), _common_agentic, PackSearchSkills.blueprint(), + RedObjectDetector.blueprint(), # fast GPU-free "red object" find (vs slow moondream) ).disabled_modules(SecurityModule, PersonFollowSkillContainer) __all__ = ["unitree_go2_pack", "PACK_SYSTEM_PROMPT"] diff --git a/dimos/experimental/pack_mind/red_detector.py b/dimos/experimental/pack_mind/red_detector.py new file mode 100644 index 0000000000..799a76eb54 --- /dev/null +++ b/dimos/experimental/pack_mind/red_detector.py @@ -0,0 +1,122 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — fast, GPU-free "red object" detector for the live demo. + +The demo's target is defined by its COLOUR, so the right detector is a colour +filter on the live camera, not a 4GB VLM. moondream on a CPU ground station is +slow (tens of seconds, async) and fragile; this is a deterministic per-frame red +test that runs in milliseconds and is honest — the dog literally sees red. + +On a hit, it reports the finding to the pack coordinator (zone left blank, so the +coordinator fills in the dog's currently-claimed zone). Exposed as the @skill +``look_for_red`` so the operator (or agent) triggers it with one call. +""" + +from __future__ import annotations + +import os +from typing import Any + +import numpy as np +import requests +from reactivex.disposable import Disposable + +from dimos.agents.annotation import skill +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.core.stream import In +from dimos.experimental.pack_mind.pack_coordinator_server import DEFAULT_COORDINATOR_PORT +from dimos.msgs.sensor_msgs.Image import Image +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +DEFAULT_RED_THRESHOLD = 0.02 # ≥2% of the frame being red ⇒ a red object is in view +_REQUEST_TIMEOUT = 5.0 + + +def red_fraction(data: np.ndarray) -> float: + """Fraction of pixels that are convincingly red (RGB, lighting-tolerant ratio test). + + Red = a high red channel that clearly dominates green and blue. Using channel + *ratios* rather than absolute thresholds keeps it robust across lighting. + Assumes RGB channel order (DimOS camera frames); if it flags blue, the frame is + BGR — swap channels 0 and 2. + """ + if data.ndim != 3 or data.shape[2] < 3: + return 0.0 + r = data[..., 0].astype(np.int16) + g = data[..., 1].astype(np.int16) + b = data[..., 2].astype(np.int16) + mask = (r > 110) & ((r - g) > 50) & ((r - b) > 50) + return float(mask.mean()) + + +class RedObjectDetector(Module): + """Watches the camera; ``look_for_red`` checks the latest frame and, if a red + object is in view, tells the pack coordinator.""" + + color_image: In[Image] + + def __init__( + self, + dog_name: str | None = None, + coordinator_url: str | None = None, + threshold: float = DEFAULT_RED_THRESHOLD, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self._latest: Image | None = None + self._threshold = threshold + self._dog = dog_name or os.environ.get("PACK_DOG_NAME") or "alpha" + self._url = ( + coordinator_url + or os.environ.get("PACK_COORDINATOR_URL") + or f"http://127.0.0.1:{DEFAULT_COORDINATOR_PORT}" + ).rstrip("/") + + @rpc + def start(self) -> None: + super().start() + self.register_disposable(Disposable(self.color_image.subscribe(self._on_image))) + + def _on_image(self, image: Image) -> None: + self._latest = image + + @skill + def look_for_red(self) -> str: + """Look for a red object in the robot's current camera view. If one is + visible, report it to the pack's shared memory and say so. + """ + img = self._latest + if img is None: + return "No camera frame yet — is the robot streaming video?" + frac = red_fraction(np.asarray(img.data)) + if frac < self._threshold: + return f"No red object in view (red {frac:.1%})." + try: + requests.post( + f"{self._url}/report_finding", + json={"dog": self._dog, "object": "red object", "zone": ""}, + timeout=_REQUEST_TIMEOUT, + ) + except requests.RequestException as exc: + logger.warning("red object seen but coordinator unreachable", error=str(exc)) + return f"I see a red object (red {frac:.0%}) but could not reach the pack memory." + logger.info("red object detected", dog=self._dog, fraction=round(frac, 3)) + return f"I see a red object (red {frac:.0%}) — reported to the pack." + + +red_object_detector = RedObjectDetector.blueprint diff --git a/dimos/experimental/pack_mind/test_red_detector.py b/dimos/experimental/pack_mind/test_red_detector.py new file mode 100644 index 0000000000..2c4cd3ed55 --- /dev/null +++ b/dimos/experimental/pack_mind/test_red_detector.py @@ -0,0 +1,58 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the GPU-free red-object detector (the demo's reliable find path).""" + +import numpy as np +import pytest + +from dimos.experimental.pack_mind.red_detector import DEFAULT_RED_THRESHOLD, red_fraction + + +def _frame(rgb: tuple[int, int, int], h: int = 48, w: int = 64) -> np.ndarray: + img = np.zeros((h, w, 3), dtype=np.uint8) + img[:] = rgb + return img + + +@pytest.mark.unit +def test_all_red_frame_is_fully_red() -> None: + assert red_fraction(_frame((220, 20, 20))) == pytest.approx(1.0) + + +@pytest.mark.unit +def test_blue_and_green_and_gray_are_not_red() -> None: + for rgb in [(20, 20, 220), (20, 220, 20), (128, 128, 128), (0, 0, 0), (255, 255, 255)]: + assert red_fraction(_frame(rgb)) == 0.0 + + +@pytest.mark.unit +def test_partial_red_patch_crosses_threshold() -> None: + img = _frame((130, 130, 130)) # neutral background + img[:, :20] = (230, 30, 30) # a red object filling ~31% of the width + frac = red_fraction(img) + assert frac > DEFAULT_RED_THRESHOLD + assert frac == pytest.approx(20 / 64, abs=0.01) + + +@pytest.mark.unit +def test_tiny_red_speck_below_threshold() -> None: + img = _frame((130, 130, 130)) + img[0, 0] = (230, 30, 30) # a single red pixel + assert red_fraction(img) < DEFAULT_RED_THRESHOLD + + +@pytest.mark.unit +def test_grayscale_or_bad_shape_is_safe() -> None: + assert red_fraction(np.zeros((10, 10), dtype=np.uint8)) == 0.0 From 7da58d9bbe137766f9feceae0d71f930261028b0 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 07:10:36 -0600 Subject: [PATCH 27/35] test: keep red_detector tests to detection logic (Module construct leaks threads in-process) --- dimos/experimental/pack_mind/test_red_detector.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dimos/experimental/pack_mind/test_red_detector.py b/dimos/experimental/pack_mind/test_red_detector.py index 2c4cd3ed55..4ce984f535 100644 --- a/dimos/experimental/pack_mind/test_red_detector.py +++ b/dimos/experimental/pack_mind/test_red_detector.py @@ -12,7 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for the GPU-free red-object detector (the demo's reliable find path).""" +"""Tests for the GPU-free red-object detector (the demo's reliable find path). + +These cover the detection logic (``red_fraction``). The detector→coordinator glue +(``look_for_red`` posting a finding with the coordinator filling the claimed zone) +is covered by the coordinator server tests (report_finding + zone fallback); the +full chain was also verified manually end-to-end. We don't construct the +``RedObjectDetector`` Module here because instantiating a DimOS Module standalone +leaks framework threads that the suite's thread-leak guard rejects. +""" import numpy as np import pytest From 6e52dde359b01357e5c492d2cf270460a49df06a Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 07:11:50 -0600 Subject: [PATCH 28/35] docs: README live find path = look_for_red (fast, GPU-free); moondream is the slow fallback --- dimos/experimental/pack_mind/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dimos/experimental/pack_mind/README.md b/dimos/experimental/pack_mind/README.md index e440463a2e..bb15c99248 100644 --- a/dimos/experimental/pack_mind/README.md +++ b/dimos/experimental/pack_mind/README.md @@ -100,9 +100,13 @@ PACK_DOG_NAME=bravo PACK_COORDINATOR_URL=http://:8090 \ - `dimos run` has **no `--robot-ip` flag** in this build — set the `ROBOT_IP` env and verify with `dimos show-config`. - `HF_HUB_OFFLINE=1` is required on a no-internet AP; run prefetch first. -- moondream2's **first** `look_out_for` is slow (~10–60s, CPU compile) — a lag, not a crash. +- **Detection: use `look_for_red`** (RedObjectDetector) — a fast, GPU-free colour check + that reports the finding to the coordinator instantly. moondream/`look_out_for` also + works but is slow on a CPU host (~10–60s first call); keep it off the critical path. +- Movement: `relative_move` (skill) or keyboard teleop. `navigate_with_text` works but is + slow on CPU (Qwen). - Bring-up order: ping dog → `dimos show-config` → run to "running" → `mcp list-tools` + - `speak` → `look_out_for` → `start_search`/`next_zone` against the coordinator. + `speak` → `relative_move` → `look_for_red` → `start_search`/`next_zone` (dashboard at :8090). **Hardware-free rehearsal (no dogs):** `uv run python -m dimos.experimental.pack_mind.demo_pack_live --pace 2`, open http://localhost:8090. @@ -120,6 +124,7 @@ PACK_DOG_NAME=bravo PACK_COORDINATOR_URL=http://:8090 \ | `pack_coordinator.py` / `pack_coordinator_server.py` | Shared zone ledger (no-overlap, find, **inheritance**) + JSON/HTTP API + dashboard route | | `pack_dashboard.html` | Projector dashboard — zones, finding, offline/inheritance, causal chain | | `pack_search_skills.py` | Dog agent tools: `start_search` / `next_zone` / `report_*` / `where_is` | +| `red_detector.py` | **Fast GPU-free red-object detector** (`look_for_red`) — HSV-free colour test → auto-reports the find | | `pack_search_runner.py` | `RobotDriver`-protocol search loop + `MockDriver` | | `mock_dog.py` / `demo_pack_live.py` / `demo_pack_scene.py` | Hardware-free test + projector rehearsal of both magic beats | | `prefetch_live_models.py` | Pre-cache moondream2 + whisper so the live stack runs offline | From 24c46cec400f643f8a55bf69caae2cf0602a51e2 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 07:55:15 -0600 Subject: [PATCH 29/35] feat: PACK MIND keyboard demo driver (WASD teleop + auto look_for_red on stop + auto speak) --- dimos/experimental/pack_mind/demo_drive.py | 119 +++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 dimos/experimental/pack_mind/demo_drive.py diff --git a/dimos/experimental/pack_mind/demo_drive.py b/dimos/experimental/pack_mind/demo_drive.py new file mode 100644 index 0000000000..488ac31758 --- /dev/null +++ b/dimos/experimental/pack_mind/demo_drive.py @@ -0,0 +1,119 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — keyboard demo driver for one live Go2 running unitree-go2-pack. + +Drive the dog with the keyboard; after every move (when it stops) it AUTO-runs +``look_for_red`` and, on a find, AUTO-``speak``s the announcement. One operator, +hands on the keyboard, zero command typing during the demo. + + uv run python -m dimos.experimental.pack_mind.demo_drive + +Keys: + w / s forward / back (one step) + a / d turn left / right + space look for red now + f speak "red object found" + +/- bigger / smaller step + x quit + +Movement uses the ``relative_move`` skill, which blocks until the dog stops — so +"look on stop" is just: issue the move, then look the moment it returns. +""" + +from __future__ import annotations + +import argparse +import sys +import termios +import tty + +from dimos.agents.mcp.mcp_adapter import McpAdapter + +_FOUND_MARK = "i see a red object" +_FOUND_SPEECH = "red object found" + + +def _getch() -> str: + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + try: + tty.setraw(fd) + return sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old) + + +def _call(adapter: McpAdapter, name: str, args: dict[str, object] | None = None) -> str: + try: + return adapter.call_tool_text(name, args or {}) + except Exception as exc: # never let one flaky call kill the driver + return f"[{name} error: {exc}]" + + +def _look_and_announce(adapter: McpAdapter) -> None: + result = _call(adapter, "look_for_red") + print(f" 👁 {result}") + if _FOUND_MARK in result.lower(): + print(f" 📣 {_call(adapter, 'speak', {'text': _FOUND_SPEECH})}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="PACK MIND keyboard driver") + parser.add_argument("--url", default=None, help="MCP URL (default localhost:9990/mcp)") + parser.add_argument("--step", type=float, default=0.6, help="metres per forward/back press") + parser.add_argument("--turn", type=float, default=30.0, help="degrees per turn press") + parser.add_argument( + "--no-auto-look", action="store_true", help="don't auto look_for_red after each move" + ) + args = parser.parse_args() + + adapter = McpAdapter(args.url) + step, turn = args.step, args.turn + + print("PACK MIND driver — w/s move · a/d turn · space=look · f=speak · +/- step · x=quit") + print(f"connected to {adapter.url} (step={step}m turn={turn}°)\n") + + moves = { + "w": lambda: ("relative_move", {"forward": step, "left": 0.0, "degrees": 0.0}), + "s": lambda: ("relative_move", {"forward": -step, "left": 0.0, "degrees": 0.0}), + "a": lambda: ("relative_move", {"forward": 0.0, "left": 0.0, "degrees": turn}), + "d": lambda: ("relative_move", {"forward": 0.0, "left": 0.0, "degrees": -turn}), + } + + while True: + key = _getch().lower() + if key in ("x", "\x03", "\x04", "q"): # x, Ctrl-C, Ctrl-D, q + print("\nbye") + return + if key in moves: + name, margs = moves[key]() + arrow = {"w": "↑", "s": "↓", "a": "↰", "d": "↱"}[key] + print(f"{arrow} {_call(adapter, name, margs)}") + if not args.no_auto_look: + _look_and_announce(adapter) + elif key == " ": + _look_and_announce(adapter) + elif key == "f": + print(f" 📣 {_call(adapter, 'speak', {'text': _FOUND_SPEECH})}") + elif key == "+": + step += 0.2 + print(f" step = {step:.1f}m") + elif key == "-": + step = max(0.2, step - 0.2) + print(f" step = {step:.1f}m") + + +if __name__ == "__main__": + main() From 079a42b2505521f23135efebd1fe152b73bb39f8 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 08:14:59 -0600 Subject: [PATCH 30/35] fix: demo_drive short per-call timeout so it doesn't block on the flaky is_goal_reached LCM RPC --- dimos/experimental/pack_mind/demo_drive.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dimos/experimental/pack_mind/demo_drive.py b/dimos/experimental/pack_mind/demo_drive.py index 488ac31758..e698270711 100644 --- a/dimos/experimental/pack_mind/demo_drive.py +++ b/dimos/experimental/pack_mind/demo_drive.py @@ -77,9 +77,17 @@ def main() -> None: parser.add_argument( "--no-auto-look", action="store_true", help="don't auto look_for_red after each move" ) + parser.add_argument( + "--timeout", + type=int, + default=8, + help="per-call seconds. Short on purpose: the dog physically moves in a few " + "seconds, but relative_move then waits on a flaky LCM RPC (is_goal_reached) " + "that can hang for 120s — we don't block on it, the dog already moved.", + ) args = parser.parse_args() - adapter = McpAdapter(args.url) + adapter = McpAdapter(args.url, timeout=args.timeout) step, turn = args.step, args.turn print("PACK MIND driver — w/s move · a/d turn · space=look · f=speak · +/- step · x=quit") From a6ef2d1d185534fd364820a02b9eee7a33533131 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 08:20:48 -0600 Subject: [PATCH 31/35] feat: velocity teleop to bypass the flaky planner RPC (relative_move hang) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit relative_move runs the A* planner then waits on ReplanningAStarPlanner/is_goal_reached over LCM — flaky on macOS, hangs 120s even though the dog reached the goal. New VelocityTeleop publishes Twist straight to MovementManager's tele_cmd_vel lane: the dog moves instantly, no planner, no confirmation RPC. demo_drive now uses `drive` (velocity bursts) instead of relative_move; W/A/S/D map to forward/turn m/s·rad/s. --- dimos/experimental/pack_mind/demo_drive.py | 15 ++-- dimos/experimental/pack_mind/live.py | 2 + .../experimental/pack_mind/velocity_teleop.py | 83 +++++++++++++++++++ 3 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 dimos/experimental/pack_mind/velocity_teleop.py diff --git a/dimos/experimental/pack_mind/demo_drive.py b/dimos/experimental/pack_mind/demo_drive.py index e698270711..27cee6854f 100644 --- a/dimos/experimental/pack_mind/demo_drive.py +++ b/dimos/experimental/pack_mind/demo_drive.py @@ -72,8 +72,8 @@ def _look_and_announce(adapter: McpAdapter) -> None: def main() -> None: parser = argparse.ArgumentParser(description="PACK MIND keyboard driver") parser.add_argument("--url", default=None, help="MCP URL (default localhost:9990/mcp)") - parser.add_argument("--step", type=float, default=0.6, help="metres per forward/back press") - parser.add_argument("--turn", type=float, default=30.0, help="degrees per turn press") + parser.add_argument("--step", type=float, default=0.5, help="forward speed (m/s)") + parser.add_argument("--turn", type=float, default=0.8, help="turn speed (rad/s)") parser.add_argument( "--no-auto-look", action="store_true", help="don't auto look_for_red after each move" ) @@ -93,11 +93,14 @@ def main() -> None: print("PACK MIND driver — w/s move · a/d turn · space=look · f=speak · +/- step · x=quit") print(f"connected to {adapter.url} (step={step}m turn={turn}°)\n") + # Velocity teleop (drive) — instant, bypasses the planner + flaky is_goal_reached + # RPC. step = m/s forward burst; turn = rad/s; each press moves for `burst` seconds. + burst = 0.6 moves = { - "w": lambda: ("relative_move", {"forward": step, "left": 0.0, "degrees": 0.0}), - "s": lambda: ("relative_move", {"forward": -step, "left": 0.0, "degrees": 0.0}), - "a": lambda: ("relative_move", {"forward": 0.0, "left": 0.0, "degrees": turn}), - "d": lambda: ("relative_move", {"forward": 0.0, "left": 0.0, "degrees": -turn}), + "w": lambda: ("drive", {"forward": step, "turn": 0.0, "duration": burst}), + "s": lambda: ("drive", {"forward": -step, "turn": 0.0, "duration": burst}), + "a": lambda: ("drive", {"forward": 0.0, "turn": turn, "duration": burst}), + "d": lambda: ("drive", {"forward": 0.0, "turn": -turn, "duration": burst}), } while True: diff --git a/dimos/experimental/pack_mind/live.py b/dimos/experimental/pack_mind/live.py index 5820703d72..aad52bee90 100644 --- a/dimos/experimental/pack_mind/live.py +++ b/dimos/experimental/pack_mind/live.py @@ -44,6 +44,7 @@ from dimos.core.coordination.blueprints import autoconnect from dimos.experimental.pack_mind.pack_search_skills import PackSearchSkills from dimos.experimental.pack_mind.red_detector import RedObjectDetector +from dimos.experimental.pack_mind.velocity_teleop import VelocityTeleop from dimos.experimental.security_demo.security_module import SecurityModule from dimos.robot.unitree.go2.blueprints.agentic._common_agentic import _common_agentic from dimos.robot.unitree.go2.blueprints.smart.unitree_go2_spatial import unitree_go2_spatial @@ -85,6 +86,7 @@ _common_agentic, PackSearchSkills.blueprint(), RedObjectDetector.blueprint(), # fast GPU-free "red object" find (vs slow moondream) + VelocityTeleop.blueprint(), # direct cmd_vel teleop (vs planner-RPC relative_move) ).disabled_modules(SecurityModule, PersonFollowSkillContainer) __all__ = ["unitree_go2_pack", "PACK_SYSTEM_PROMPT"] diff --git a/dimos/experimental/pack_mind/velocity_teleop.py b/dimos/experimental/pack_mind/velocity_teleop.py new file mode 100644 index 0000000000..4434cdbf07 --- /dev/null +++ b/dimos/experimental/pack_mind/velocity_teleop.py @@ -0,0 +1,83 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — direct velocity teleop for the live demo. + +``relative_move`` is goal-based: it runs the A* planner and then waits on an +inter-module RPC (``ReplanningAStarPlanner/is_goal_reached``). On a CPU/macOS host +that RPC over LCM is flaky and can hang for 120s — so the dog reaches the goal but +the call never returns, freezing the driver. + +This bypasses all of that: publish a ``Twist`` straight to ``MovementManager``'s +``tele_cmd_vel`` (its keyboard-teleop lane, which takes priority and forwards to the +dog). The dog moves the instant the Twist lands — no goal, no confirmation round +trip. Drive in short bursts; ``drive`` auto-stops after ``duration``. +""" + +from __future__ import annotations + +import time + +from dimos.agents.annotation import skill +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.core.stream import Out +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +_MAX_DURATION = 3.0 # safety cap on a single burst + + +class VelocityTeleop(Module): + """Publishes velocity bursts to MovementManager's teleop lane (no planner).""" + + tele_cmd_vel: Out[Twist] + + @rpc + def start(self) -> None: + super().start() + + def _publish(self, forward: float, turn: float) -> None: + self.tele_cmd_vel.publish( + Twist(linear=Vector3(forward, 0.0, 0.0), angular=Vector3(0.0, 0.0, turn)) + ) + + @skill + def drive(self, forward: float = 0.0, turn: float = 0.0, duration: float = 0.6) -> str: + """Drive the robot by velocity for a short burst, then stop. Instant — no + path planner. Use this for keyboard teleop instead of relative_move. + + Args: + forward: Linear speed in m/s. Positive = forward, negative = back. + turn: Angular speed in rad/s. Positive = turn left, negative = right. + duration: Seconds to move before auto-stopping (capped at 3s). + """ + duration = max(0.0, min(duration, _MAX_DURATION)) + self._publish(forward, turn) + time.sleep(duration) + self._publish(0.0, 0.0) # stop + logger.info("teleop drive", forward=forward, turn=turn, duration=round(duration, 2)) + return f"drove forward={forward} turn={turn} for {duration:.1f}s" + + @skill + def halt(self) -> str: + """Stop the robot immediately.""" + self._publish(0.0, 0.0) + return "stopped" + + +velocity_teleop = VelocityTeleop.blueprint From ec6b00693bee1260543e827601abd967cf84f8e6 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 09:25:16 -0600 Subject: [PATCH 32/35] =?UTF-8?q?chore:=20PACK=20MIND=20cleanup=20for=20PR?= =?UTF-8?q?=20=E2=80=94=20drop=20superseded=20escort=20demo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - remove the old "backpack handoff" demo (conductor, dashboard, sim_harness, venue_go2, RUNBOOK, test_conductor) — superseded by the exploration A/B + live pack - recognize .disabled_modules() in the blueprint AST scanner so unitree-go2-pack (which disables the CUDA-only EdgeTAM modules) registers; regenerate all_blueprints --- dimos/experimental/pack_mind/README.md | 5 - dimos/experimental/pack_mind/RUNBOOK.md | 178 ------- dimos/experimental/pack_mind/conductor.py | 479 ------------------ dimos/experimental/pack_mind/dashboard.html | 123 ----- dimos/experimental/pack_mind/sim_harness.py | 52 -- dimos/experimental/pack_mind/static/style.css | 130 ----- .../experimental/pack_mind/test_conductor.py | 232 --------- dimos/experimental/pack_mind/venue_go2.sh | 80 --- dimos/robot/all_blueprints.py | 2 + dimos/robot/test_all_blueprints_generation.py | 9 +- 10 files changed, 10 insertions(+), 1280 deletions(-) delete mode 100644 dimos/experimental/pack_mind/RUNBOOK.md delete mode 100644 dimos/experimental/pack_mind/conductor.py delete mode 100644 dimos/experimental/pack_mind/dashboard.html delete mode 100644 dimos/experimental/pack_mind/sim_harness.py delete mode 100644 dimos/experimental/pack_mind/static/style.css delete mode 100644 dimos/experimental/pack_mind/test_conductor.py delete mode 100755 dimos/experimental/pack_mind/venue_go2.sh diff --git a/dimos/experimental/pack_mind/README.md b/dimos/experimental/pack_mind/README.md index bb15c99248..7aa65e0b79 100644 --- a/dimos/experimental/pack_mind/README.md +++ b/dimos/experimental/pack_mind/README.md @@ -135,8 +135,3 @@ PACK_DOG_NAME=bravo PACK_COORDINATOR_URL=http://:8090 \ | `demo_spike.py` | Feasibility spike (kept as a smoke test of the reused primitives) | | `test_explore_sim.py` / `test_pack_mind_sim.py` | Engine tests | | `sim_perception.py` | MuJoCo+VLM single-frame perception probe (optional, off the critical path) | - -### Legacy (the old "backpack handoff" demo — superseded, kept for reference) -`conductor.py`, `dashboard.html`, `static/style.css`, `sim_harness.py`, `venue_go2.sh`, -`RUNBOOK.md`, `test_conductor.py`. That demo was a single message relay dressed up as -"shared memory"; the exploration A/B above replaces it. Safe to delete in a cleanup pass. diff --git a/dimos/experimental/pack_mind/RUNBOOK.md b/dimos/experimental/pack_mind/RUNBOOK.md deleted file mode 100644 index 3f85c69fa1..0000000000 --- a/dimos/experimental/pack_mind/RUNBOOK.md +++ /dev/null @@ -1,178 +0,0 @@ -# PACK MIND — Live Go2 Runbook - -> Run this top-to-bottom at the venue. Every command is copy-paste. Do **not** -> debug architecture on-site — that work is done; this is wiring + checks. - -**Pitch:** Most teams make robot dogs share *maps*. Pack Mind makes them share -*meaning*. Alpha sees → the pack remembers → Bravo acts on a teammate's memory, -by zone **name**, never coordinates. - ---- - -## 0. Topology (read once) - -``` -[Go2 alpha] <--robot-ip-- [GPU box A] dimos agentic stack --MCP 0.0.0.0:9990--> \ -[Go2 bravo] <--robot-ip-- [GPU box B] dimos agentic stack --MCP 0.0.0.0:9990--> > [operator laptop] - / conductor.py - dashboard :8080 -``` - -- Each dog needs a **CUDA** companion node (EdgeTAM perception). The agentic - stack runs on the node, not the dog. The dog is reached via `--robot-ip`. -- One GPU box per dog → both use MCP port `9990`. -- One GPU box running **two** stacks → give each a distinct `--mcp-port` - (`9990`, `9991`). -- The conductor runs on the **operator laptop** and talks to each node's MCP - over the LAN. - -### ⚠️ #1 KILLER — bind MCP to the LAN - -DimOS defaults `listen_host = 127.0.0.1`. The MCP server then only answers -localhost and the conductor gets **connection refused** over the LAN. Every -agentic stack MUST be launched with: - -``` ---listen-host 0.0.0.0 -``` - -If a G0 curl is refused, this is the cause 90% of the time. The other 10% is a -firewall on the GPU box. - ---- - -## 1. Per-dog stack bring-up (on each GPU node) - -```bash -# Node A → drives Go2 "alpha" -dimos run unitree-go2-agentic \ - --robot-ip \ - --listen-host 0.0.0.0 \ - --mcp-port 9990 \ - --daemon - -# Node B → drives Go2 "bravo" (or same box, --mcp-port 9991) -dimos run unitree-go2-agentic \ - --robot-ip \ - --listen-host 0.0.0.0 \ - --mcp-port 9990 \ - --daemon -``` - -Verify each node locally before involving the network: - -```bash -dimos status # run id, pid, uptime -dimos mcp status # PID, module list, skill list -dimos mcp list-tools # must include: speak, navigate_with_text, look_out_for, tag_location -``` - ---- - -## 2. G0 (h4) — remote MCP reachability over LAN - -From the **operator laptop**, for every node `:`: - -```bash -curl -s -X POST http://:/mcp \ - -H 'content-type: application/json' \ - -d '{"jsonrpc":"2.0","id":"1","method":"tools/list"}' | jq . -``` - -**PASS** = JSON with a `result.tools` array containing `speak`, -`navigate_with_text`, `look_out_for`. **G0 done.** - -> The MCP wire format itself is already validated CPU-side -> (`sim_harness.py`, multica WEB-5). At the venue you are only proving the -> *network path*, not the protocol. - -If refused: re-check `--listen-host 0.0.0.0`, then `ping `, then the -node's firewall (`sudo ufw allow 9990`). - ---- - -## 3. Launch the conductor (operator laptop) - -```bash -cd -source .venv/bin/activate -python dimos/experimental/pack_mind/conductor.py \ - --dog alpha=:9990 \ - --dog bravo=:9990 -# add --dog charlie=:9990 for the 3rd dog (gravy only) -``` - -Open **http://localhost:8080**. Six operator buttons drive the mission. Put this -on the projector — the causal event feed *is* the story. - ---- - -## 4. G1 (h12) — single-dog magic moment - -One dog, one human-legible beat. On the dashboard: - -1. **start_act1** → Alpha announces it is scouting. -2. **inject_found** (alpha / red backpack / zone_b) → Alpha speaks "Found red - backpack in zone_b" and the find lands on the shared blackboard. -3. **ask_where** (bravo / red backpack) → Bravo answers **from pack memory**: - *"Alpha found it in zone_b. Follow me."* - -If the dog physically speaks and the dashboard shows the causal chain, **G1 done.** - ---- - -## 5. G2 (h24) — two-dog MVP, **twice in a row** (THE gate) - -Full mission, both dogs, no crash, two consecutive clean runs: - -1. **start_act1** -2. **inject_found** alpha → zone_b -3. **ask_where** bravo → answers from shared memory -4. **send_dog** bravo → acquires movement lock, navigates to **zone_b by name** -5. **verify** bravo → VLM lookout confirms → "Pack memory was correct." - -Watch for: only one dog moving at a time (movement lock), no coordinates ever -exchanged. Run it, **estop**, reset, run it again. Two clean passes = **G2 done.** -This is the gate the judges buy. Everything before it is low-risk; the real risk -is multi-dog concurrency + venue WiFi. - ---- - -## 6. Fallback — the demo never hard-fails - -**Operator `inject_found` produces the identical blackboard effect as a real -detection.** If live perception is flaky, the operator injects the find and the -entire downstream story (memory → query → navigate → verify) runs unchanged. -Rehearse the demo driving inject_found by hand; a real detection is a bonus, not -a dependency. - -If a dog wanders or a nav call hangs: hit **estop** (releases the movement lock, -all dogs hold, mission → IDLE), then resume from start_act1. - ---- - -## 7. G3 (h36) / G4 (h48) - -- **G3:** Record a clean two-dog run end-to-end (screen + floor). Harden: rerun - G2 ~5× and note any flake. -- **G4:** Rehearse the 90s presentation 3×. Script the narration to the - dashboard event feed. Have the recorded G3 run ready as the fallback if the - live floor misbehaves. - ---- - -## 8. Quick reference - -| Thing | Command / value | -|---|---| -| MCP port (default) | `9990` (`--mcp-port`) | -| LAN bind (REQUIRED) | `--listen-host 0.0.0.0` | -| Dashboard | `http://localhost:8080` | -| Stop a stack | `dimos stop` (on the node) | -| Tail logs | `dimos log -f` (on the node) | -| Send raw agent text | `dimos agent-send "..."` | -| CPU dry-run (no dogs) | `conductor.py --mock` | -| MCP tool args | `speak(text, blocking)` · `navigate_with_text(query)` · `look_out_for(description_of_things)` · `tag_location(location_name)` | - -Zones are named (`zone_a/b/c`). Cross-dog handoff is **always** by zone name. -There is no coordinate field anywhere in the shared memory — by design. diff --git a/dimos/experimental/pack_mind/conductor.py b/dimos/experimental/pack_mind/conductor.py deleted file mode 100644 index fd15e46932..0000000000 --- a/dimos/experimental/pack_mind/conductor.py +++ /dev/null @@ -1,479 +0,0 @@ -# Copyright 2025-2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""PACK MIND conductor — a shared semantic memory for a team of Unitree Go2s. - -The conductor is the ONLY shared layer. Each dog runs its own full -``unitree-go2-agentic`` stack (own map, own memory) and exposes an MCP HTTP -endpoint. The conductor holds the roster, an append-only event blackboard, the -mission state machine, and a movement lock. It talks to each dog over MCP -JSON-RPC. It NEVER exchanges coordinates between dogs — cross-dog handoff is by -zone NAME only. - -Run without hardware (drives the whole story from the dashboard buttons):: - - uv run python dimos/experimental/pack_mind/conductor.py --mock - -Run against real dogs:: - - uv run python dimos/experimental/pack_mind/conductor.py \\ - --dog alpha=10.0.0.10 --dog bravo=10.0.0.11 --dog charlie=10.0.0.12 - -Then open http://localhost:8080 for the dashboard. - -NOTE: MCP tool argument names (``navigate_with_text``, ``look_out_for``, -``speak``) are passed through verbatim. Verify them against the skill -signatures in dimos/agents/skills/ on the first real-hardware test (G0). -""" - -from __future__ import annotations - -import argparse -import json -import threading -import uuid -from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Any, Literal, cast - -import requests - -DogId = Literal["alpha", "bravo", "charlie"] -ZoneId = Literal["zone_a", "zone_b", "zone_c"] -MissionState = Literal[ - "IDLE", - "SCOUT_ALPHA", - "FOUND_EVENT_RECORDED", - "QUERY_BRAVO", - "BRAVO_NAVIGATING", - "VERIFYING", - "DONE", -] - -# MCP call timeouts in seconds (plan section 5). Never block the dashboard on these. -TIMEOUT_SPEAK = 8.0 -TIMEOUT_LIGHT = 15.0 -TIMEOUT_NAV = 90.0 - -_HERE = Path(__file__).parent - - -@dataclass -class Dog: - """A pack member. ``zone`` is the named zone it was last sent to — never a coordinate.""" - - id: DogId - name: str - mcp_url: str - role: str - status: str = "idle" - zone: ZoneId | None = None - - -@dataclass(frozen=True) -class MemoryEvent: - """An immutable blackboard entry. The shared memory is an append-only list of these. - - There is deliberately no ``x``/``y``/``pose`` field. If you add one, you are - building the wrong project. - """ - - id: str - ts: str - robot: DogId - type: str - text: str - object: str | None = None - zone: ZoneId | None = None - source: str = "system" - confidence: float | None = None - - -def _now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _new_id() -> str: - return f"evt-{uuid.uuid4().hex[:8]}" - - -class Conductor: - """Holds all shared state and orchestrates the deterministic mission state machine.""" - - def __init__(self, dogs: list[Dog], mock: bool) -> None: - self._dogs: dict[DogId, Dog] = {d.id: d for d in dogs} - self._events: list[MemoryEvent] = [] - self._mission: MissionState = "IDLE" - self._mock = mock - self._state_lock = threading.Lock() # guards roster + blackboard + mission - self._move_lock = threading.Lock() # only one dog moves at a time - self._acting: DogId | None = None - - # -- shared memory ------------------------------------------------------- - - def append_event( - self, - robot: DogId, - type_: str, - text: str, - *, - object_: str | None = None, - zone: ZoneId | None = None, - source: str = "system", - confidence: float | None = None, - ) -> MemoryEvent: - event = MemoryEvent( - id=_new_id(), - ts=_now(), - robot=robot, - type=type_, - text=text, - object=object_, - zone=zone, - source=source, - confidence=confidence, - ) - with self._state_lock: - self._events.append(event) - return event - - def _latest_zone_for(self, object_: str) -> ZoneId | None: - """The whole semantic-memory trick: any dog can answer from the blackboard.""" - with self._state_lock: - for event in reversed(self._events): - if event.type == "object_found" and event.object == object_ and event.zone: - return event.zone - return None - - def _set_mission(self, state: MissionState) -> None: - with self._state_lock: - self._mission = state - - # -- MCP transport ------------------------------------------------------- - - def call_tool( - self, dog_id: DogId, tool_name: str, args: dict[str, Any], timeout: float - ) -> str | None: - """Call one MCP tool on one dog. Returns text content, or None on failure. - - On timeout/error, appends a ``tool_timeout`` event and returns None so the - caller can fall back to a scripted beat. Never raises into the request thread. - """ - dog = self._dogs.get(dog_id) - if dog is None: - return None - - if self._mock: - self.append_event( - dog_id, "mock_call", f"[mock] {tool_name}({args})", source="mock" - ) - return f"[mock {dog_id}] {tool_name} ok" - - payload = { - "jsonrpc": "2.0", - "id": _new_id(), - "method": "tools/call", - "params": {"name": tool_name, "arguments": args}, - } - try: - resp = requests.post(dog.mcp_url, json=payload, timeout=timeout) - resp.raise_for_status() - data: Any = resp.json() - return _extract_text(data) - except requests.RequestException as exc: - self.append_event( - dog_id, "tool_timeout", f"{tool_name} failed: {exc}", source="system" - ) - return None - - def _speak(self, dog_id: DogId, text: str) -> None: - self.call_tool(dog_id, "speak", {"text": text, "blocking": False}, TIMEOUT_SPEAK) - - # -- mission state machine ---------------------------------------------- - - def start_act1(self) -> None: - self._set_mission("SCOUT_ALPHA") - with self._state_lock: - if "alpha" in self._dogs: - self._dogs["alpha"].status = "scouting" - self.append_event("alpha", "mission", "Alpha scouting for the red backpack.") - - def inject_found(self, dog_id: DogId, object_: str, zone: ZoneId) -> MemoryEvent: - """Operator fallback == real detection: identical blackboard effect.""" - event = self.append_event( - dog_id, - "object_found", - f"{dog_id.title()} found {object_} in {zone}.", - object_=object_, - zone=zone, - source="operator", - confidence=0.92, - ) - with self._state_lock: - if dog_id in self._dogs: - self._dogs[dog_id].status = "found_target" - self._set_mission("FOUND_EVENT_RECORDED") - self._run_async(lambda: self._speak(dog_id, f"Found {object_} in {zone}.")) - return event - - def ask_where(self, dog_id: DogId, object_: str) -> str: - """The winning moment: a dog answers from SHARED memory, not its own.""" - self._set_mission("QUERY_BRAVO") - zone = self._latest_zone_for(object_) - if zone is None: - answer = f"I have no pack memory of the {object_} yet." - else: - finder = self._finder_of(object_) - who = finder.title() if finder else "A teammate" - answer = f"{who} found it in {zone}. Follow me." - self.append_event(dog_id, "answer", answer, object_=object_, zone=zone) - self._run_async(lambda: self._speak(dog_id, answer)) - return answer - - def send_dog_to_memory(self, dog_id: DogId, object_: str) -> bool: - """Acquire the movement lock and navigate by zone NAME (the dog's own frame).""" - zone = self._latest_zone_for(object_) - if zone is None: - self.append_event(dog_id, "blocked", f"No memory of {object_}; cannot navigate.") - return False - if not self._move_lock.acquire(blocking=False): - self.append_event(dog_id, "blocked", "Another dog is moving; movement lock held.") - return False - self._acting = dog_id - self._set_mission("BRAVO_NAVIGATING") - with self._state_lock: - if dog_id in self._dogs: - self._dogs[dog_id].status = "navigating" - self._dogs[dog_id].zone = zone - self.append_event(dog_id, "navigating", f"{dog_id.title()} acting on pack memory -> {zone}.", zone=zone) - - def _go() -> None: - try: - self.call_tool(dog_id, "navigate_with_text", {"query": zone}, TIMEOUT_NAV) - with self._state_lock: - if dog_id in self._dogs: - self._dogs[dog_id].status = "arrived" - self.append_event(dog_id, "arrived", f"{dog_id.title()} reached {zone}.", zone=zone) - finally: - self._acting = None - self._move_lock.release() - - self._run_async(_go) - return True - - def verify_at_zone(self, dog_id: DogId, object_: str) -> None: - self._set_mission("VERIFYING") - zone = self._latest_zone_for(object_) - - def _verify() -> None: - # Primary path uses the VLM lookout; mock/fallback just confirms. - self.call_tool( - dog_id, "look_out_for", {"description_of_things": [object_]}, TIMEOUT_LIGHT - ) - self.append_event( - dog_id, "verified", "Confirmed. Pack memory was correct.", object_=object_, zone=zone - ) - self._run_async(lambda: self._speak(dog_id, "Confirmed. Pack memory was correct.")) - self._set_mission("DONE") - with self._state_lock: - if dog_id in self._dogs: - self._dogs[dog_id].status = "done" - - self._run_async(_verify) - - def emergency_stop(self) -> None: - if self._move_lock.locked(): - try: - self._move_lock.release() - except RuntimeError: - pass - self._acting = None - with self._state_lock: - for dog in self._dogs.values(): - dog.status = "idle" - self._set_mission("IDLE") - self.append_event("alpha", "estop", "Operator emergency stop. All dogs holding.", source="operator") - - # -- helpers ------------------------------------------------------------- - - def _finder_of(self, object_: str) -> DogId | None: - with self._state_lock: - for event in reversed(self._events): - if event.type == "object_found" and event.object == object_: - return event.robot - return None - - def _run_async(self, fn: Any) -> None: - threading.Thread(target=fn, daemon=True).start() - - def snapshot(self) -> dict[str, Any]: - with self._state_lock: - return { - "mission": self._mission, - "acting": self._acting, - "mock": self._mock, - "roster": [asdict(d) for d in self._dogs.values()], - "events": [asdict(e) for e in reversed(self._events)], - } - - -def _extract_text(data: Any) -> str | None: - """Pull text out of an MCP tools/call response, tolerating shape variation.""" - if not isinstance(data, dict): - return None - result = data.get("result", data) - if isinstance(result, dict): - content = result.get("content") - if isinstance(content, list): - parts = [c.get("text", "") for c in content if isinstance(c, dict)] - joined = " ".join(p for p in parts if p) - return joined or None - return None - - -# -- HTTP server ------------------------------------------------------------- - - -class _Server(ThreadingHTTPServer): - conductor: Conductor - - -class _Handler(BaseHTTPRequestHandler): - def log_message(self, fmt: str, *args: Any) -> None: # silence default logging - return - - @property - def _conductor(self) -> Conductor: - return cast(_Server, self.server).conductor - - def _send(self, code: int, body: bytes, content_type: str) -> None: - self.send_response(code) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def _send_file(self, path: Path, content_type: str) -> None: - if not path.is_file(): - self._send(404, b"not found", "text/plain") - return - self._send(200, path.read_bytes(), content_type) - - def do_GET(self) -> None: # noqa: N802 (stdlib API) - if self.path in ("/", "/index.html"): - self._send_file(_HERE / "dashboard.html", "text/html; charset=utf-8") - elif self.path == "/static/style.css": - self._send_file(_HERE / "static" / "style.css", "text/css; charset=utf-8") - elif self.path == "/state": - body = json.dumps(self._conductor.snapshot()).encode() - self._send(200, body, "application/json") - else: - self._send(404, b"not found", "text/plain") - - def do_POST(self) -> None: # noqa: N802 (stdlib API) - if self.path != "/action": - self._send(404, b"not found", "text/plain") - return - length = int(self.headers.get("Content-Length", "0")) - raw = self.rfile.read(length) if length else b"{}" - try: - req: Any = json.loads(raw) - except json.JSONDecodeError: - self._send(400, b'{"ok": false, "error": "bad json"}', "application/json") - return - result = self._dispatch(req if isinstance(req, dict) else {}) - self._send(200, json.dumps(result).encode(), "application/json") - - def _dispatch(self, req: dict[str, Any]) -> dict[str, Any]: - action = req.get("action") - c = self._conductor - obj = cast(str, req.get("object", "red backpack")) - dog = cast(DogId, req.get("dog", "bravo")) - zone = cast(ZoneId, req.get("zone", "zone_b")) - if action == "start_act1": - c.start_act1() - elif action == "inject_found": - c.inject_found(cast(DogId, req.get("dog", "alpha")), obj, zone) - elif action == "ask_where": - return {"ok": True, "answer": c.ask_where(dog, obj)} - elif action == "send_dog": - return {"ok": c.send_dog_to_memory(dog, obj)} - elif action == "verify": - c.verify_at_zone(dog, obj) - elif action == "estop": - c.emergency_stop() - else: - return {"ok": False, "error": f"unknown action: {action}"} - return {"ok": True} - - -def _parse_dog(spec: str) -> Dog: - """Parse ``alpha=10.0.0.10`` or ``alpha=10.0.0.10:9990`` into a Dog.""" - name, _, addr = spec.partition("=") - name = name.strip().lower() - if name not in ("alpha", "bravo", "charlie"): - raise argparse.ArgumentTypeError(f"dog id must be alpha/bravo/charlie, got {name!r}") - host, _, port = addr.partition(":") - port = port or "9990" - roles = {"alpha": "scout", "bravo": "guide", "charlie": "guard"} - return Dog( - id=cast(DogId, name), - name=name.title(), - mcp_url=f"http://{host}:{port}/mcp", - role=roles[name], - ) - - -def _default_dogs() -> list[Dog]: - roles = {"alpha": "scout", "bravo": "guide"} - return [ - Dog(id=cast(DogId, n), name=n.title(), mcp_url=f"http://mock/{n}/mcp", role=r) - for n, r in roles.items() - ] - - -def main() -> None: - parser = argparse.ArgumentParser(description="PACK MIND conductor") - parser.add_argument( - "--dog", - action="append", - type=_parse_dog, - default=[], - metavar="ID=HOST[:PORT]", - help="A dog to control, e.g. alpha=10.0.0.10. Repeatable.", - ) - parser.add_argument("--mock", action="store_true", help="No hardware: log calls instead of HTTP.") - parser.add_argument("--port", type=int, default=8080, help="Dashboard/API port.") - args = parser.parse_args() - - dogs: list[Dog] = list(args.dog) or _default_dogs() - mock = args.mock or not args.dog # no real dogs given -> mock - conductor = Conductor(dogs, mock=mock) - - server = _Server(("0.0.0.0", args.port), _Handler) - server.conductor = conductor - mode = "MOCK (no hardware)" if mock else f"{len(dogs)} dog(s)" - print(f"PACK MIND conductor up on http://localhost:{args.port} [{mode}]") - print(f" roster: {', '.join(f'{d.id}->{d.mcp_url}' for d in dogs)}") - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nshutting down") - server.shutdown() - - -if __name__ == "__main__": - main() diff --git a/dimos/experimental/pack_mind/dashboard.html b/dimos/experimental/pack_mind/dashboard.html deleted file mode 100644 index e2935db364..0000000000 --- a/dimos/experimental/pack_mind/dashboard.html +++ /dev/null @@ -1,123 +0,0 @@ - - - - - - PACK MIND - - - -
-
PACK MIND
-
Alpha sees. Bravo remembers. The pack acts.
-
IDLE
-
-
- -
-
-

Pack

-
- -

Operator

-
- - - - - - -
-

Hidden fallback buttons are not cheating. Broken live autonomy is.

-
- -
-

Shared semantic memory

-
semantic handoff — no shared map, no coordinates
-
-
-
- - - - diff --git a/dimos/experimental/pack_mind/sim_harness.py b/dimos/experimental/pack_mind/sim_harness.py deleted file mode 100644 index c56af8dca2..0000000000 --- a/dimos/experimental/pack_mind/sim_harness.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2025-2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Minimal CUDA-free MCP harness for validating the PACK MIND conductor locally. - -The full ``unitree-go2-agentic`` stack needs a CUDA GPU (EdgeTAM perception), so -it cannot run on a CPU-only / Apple Silicon dev machine. PACK MIND's milestones -G0 (MCP reachability) and the conductor wire format do NOT need perception — they -need a live MCP server exposing ``speak``. This harness deploys exactly that: -``SpeakSkill`` behind an ``McpServer``, on CPU. - -Run:: - - uv run python dimos/experimental/pack_mind/sim_harness.py - -Then from another shell:: - - curl -s -X POST localhost:9990/mcp -H 'content-type: application/json' \\ - -d '{"jsonrpc":"2.0","id":"t","method":"tools/list"}' - -or point the conductor at it:: - - uv run python dimos/experimental/pack_mind/conductor.py --dog alpha=localhost:9990 -""" - -from __future__ import annotations - -from dimos.agents.mcp.mcp_server import McpServer -from dimos.agents.skills.speak_skill import SpeakSkill -from dimos.core.coordination.blueprints import autoconnect -from dimos.core.coordination.module_coordinator import ModuleCoordinator - - -def main() -> None: - blueprint = autoconnect(SpeakSkill.blueprint(), McpServer.blueprint()) - coordinator = ModuleCoordinator.build(blueprint) - coordinator.loop() - - -if __name__ == "__main__": - main() diff --git a/dimos/experimental/pack_mind/static/style.css b/dimos/experimental/pack_mind/static/style.css deleted file mode 100644 index 901a7d1689..0000000000 --- a/dimos/experimental/pack_mind/static/style.css +++ /dev/null @@ -1,130 +0,0 @@ -/* PACK MIND dashboard — projector-legible: big type, dark, high contrast. */ -:root { - --bg: #0b0e14; - --panel: #141925; - --line: #232a3a; - --text: #e6ebf5; - --muted: #8b95a8; - --accent: #3ddc97; - --warn: #ffb454; - --danger: #ff5c7c; - --memory: #6aa8ff; -} - -* { box-sizing: border-box; } - -body { - margin: 0; - background: var(--bg); - color: var(--text); - font-family: ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; -} - -header { - padding: 1.2rem 2rem; - border-bottom: 1px solid var(--line); - display: grid; - grid-template-columns: auto 1fr auto auto; - align-items: center; - gap: 1.5rem; -} - -.brand { font-size: 2rem; font-weight: 800; letter-spacing: 0.08em; } -.tagline { color: var(--muted); font-size: 1.05rem; } -.mission { - font-size: 1.3rem; - font-weight: 700; - padding: 0.35rem 1rem; - border: 1px solid var(--accent); - border-radius: 999px; - color: var(--accent); -} -.mode { color: var(--warn); font-weight: 700; font-size: 0.85rem; letter-spacing: 0.1em; } - -main { - display: grid; - grid-template-columns: 360px 1fr; - gap: 1.5rem; - padding: 1.5rem 2rem; -} - -h2 { - font-size: 0.85rem; - text-transform: uppercase; - letter-spacing: 0.12em; - color: var(--muted); - margin: 0.5rem 0 0.75rem; -} - -.roster { display: flex; flex-direction: column; gap: 0.6rem; margin-bottom: 1.5rem; } -.dog { - background: var(--panel); - border: 1px solid var(--line); - border-left: 4px solid var(--muted); - border-radius: 10px; - padding: 0.7rem 0.9rem; -} -.dog-id { font-size: 1.2rem; font-weight: 700; } -.dog-role { color: var(--muted); font-size: 0.85rem; text-transform: uppercase; letter-spacing: 0.08em; } -.dog-status { margin-top: 0.25rem; font-size: 0.95rem; } -.dog.status-scouting { border-left-color: var(--warn); } -.dog.status-found_target { border-left-color: var(--accent); } -.dog.status-navigating { border-left-color: var(--memory); } -.dog.status-arrived, .dog.status-done { border-left-color: var(--accent); } - -.controls { display: flex; flex-direction: column; gap: 0.5rem; } -button { - font: inherit; - font-weight: 600; - text-align: left; - padding: 0.7rem 0.9rem; - border-radius: 10px; - border: 1px solid var(--line); - background: var(--panel); - color: var(--text); - cursor: pointer; - transition: transform 0.08s ease, border-color 0.15s ease; -} -button:hover { border-color: var(--muted); } -button:active { transform: translateY(1px); } -button.accent { border-color: var(--accent); color: var(--accent); } -button.danger { border-color: var(--danger); color: var(--danger); } -.hint { color: var(--muted); font-size: 0.8rem; margin-top: 0.75rem; } - -.handoff-note { - color: var(--memory); - font-size: 0.9rem; - margin-bottom: 1rem; - letter-spacing: 0.04em; -} -.events { display: flex; flex-direction: column; gap: 0.75rem; } -.empty { color: var(--muted); font-style: italic; padding: 2rem 0; } - -.card { - background: var(--panel); - border: 1px solid var(--line); - border-radius: 12px; - padding: 1rem 1.2rem; - animation: rise 0.25s ease; -} -@keyframes rise { from { opacity: 0; transform: translateY(6px); } to { opacity: 1; transform: none; } } -.card-top { - display: flex; - align-items: center; - gap: 0.6rem; - font-size: 1.3rem; - font-weight: 800; - letter-spacing: 0.02em; -} -.robot { text-transform: uppercase; color: var(--text); } -.label { color: var(--muted); } -.card-text { margin-top: 0.4rem; color: var(--muted); font-size: 1rem; } - -.card.type-object_found .label { color: var(--accent); } -.card.type-answer .label, -.card.type-navigating .label { color: var(--memory); } -.card.type-verified .label { color: var(--accent); } -.card.type-tool_timeout { border-color: var(--warn); } -.card.type-tool_timeout .label { color: var(--warn); } -.card.type-estop { border-color: var(--danger); } -.card.type-estop .label { color: var(--danger); } diff --git a/dimos/experimental/pack_mind/test_conductor.py b/dimos/experimental/pack_mind/test_conductor.py deleted file mode 100644 index dcc5e6adf9..0000000000 --- a/dimos/experimental/pack_mind/test_conductor.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright 2025-2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for the PACK MIND conductor. - -All tests run on CPU with no hardware and no network: the conductor is driven in -``mock=True`` mode, where ``call_tool`` logs an event instead of issuing HTTP. The -state-machine code path is identical to real hardware, so these tests guard the -exact logic that runs at the venue. -""" - -from __future__ import annotations - -import argparse -import dataclasses -import time - -import pytest - -from dimos.experimental.pack_mind.conductor import ( - Conductor, - Dog, - MemoryEvent, - _extract_text, - _parse_dog, -) - - -def _mock_conductor() -> Conductor: - dogs = [ - Dog(id="alpha", name="Alpha", mcp_url="http://mock/alpha/mcp", role="scout"), - Dog(id="bravo", name="Bravo", mcp_url="http://mock/bravo/mcp", role="guide"), - ] - return Conductor(dogs, mock=True) - - -def _wait_for_mission(c: Conductor, target: str, timeout: float = 3.0) -> bool: - deadline = time.time() + timeout - while time.time() < deadline: - if c.snapshot()["mission"] == target: - return True - time.sleep(0.02) - return False - - -# -- blackboard / immutability ---------------------------------------------- - - -def test_memory_event_is_frozen(): - event = MemoryEvent(id="e1", ts="t", robot="alpha", type="x", text="y") - - with pytest.raises(dataclasses.FrozenInstanceError): - event.text = "mutated" # type: ignore[misc] - - -def test_memory_event_carries_no_coordinate_fields(): - # The whole thesis: dogs share meaning, never geometry. - forbidden = {"x", "y", "z", "pose", "coordinate", "position"} - fields = set(MemoryEvent.__dataclass_fields__) - - assert forbidden.isdisjoint(fields) - - -def test_append_event_appends_and_returns_event(): - c = _mock_conductor() - - event = c.append_event("alpha", "object_found", "found it", object_="bag", zone="zone_a") - - assert event.robot == "alpha" - assert event.zone == "zone_a" - assert c.snapshot()["events"][0]["id"] == event.id - - -# -- semantic recall (the core trick) --------------------------------------- - - -def test_latest_zone_for_returns_most_recent_match(): - c = _mock_conductor() - c.append_event("alpha", "object_found", "a", object_="bag", zone="zone_a") - c.append_event("alpha", "object_found", "b", object_="bag", zone="zone_c") - - assert c._latest_zone_for("bag") == "zone_c" - - -def test_latest_zone_for_unknown_object_is_none(): - c = _mock_conductor() - - assert c._latest_zone_for("nonexistent") is None - - -def test_ask_where_answers_from_shared_memory(): - c = _mock_conductor() - # Alpha records the find; Bravo must answer from the SHARED blackboard. - c.inject_found("alpha", "red backpack", "zone_b") - - answer = c.ask_where("bravo", "red backpack") - - assert "Alpha" in answer - assert "zone_b" in answer - - -def test_ask_where_with_no_memory_admits_ignorance(): - c = _mock_conductor() - - answer = c.ask_where("bravo", "red backpack") - - assert "no pack memory" in answer.lower() - - -# -- movement lock (only one dog moves) ------------------------------------- - - -def test_send_dog_blocked_when_move_lock_held(): - c = _mock_conductor() - c.inject_found("alpha", "red backpack", "zone_b") - c._move_lock.acquire() - - try: - ok = c.send_dog_to_memory("bravo", "red backpack") - finally: - c._move_lock.release() - - assert ok is False - assert any(e.type == "blocked" for e in c._events) - - -def test_send_dog_without_memory_refuses(): - c = _mock_conductor() - - ok = c.send_dog_to_memory("bravo", "red backpack") - - assert ok is False - assert any(e.type == "blocked" for e in c._events) - - -def test_send_dog_releases_lock_after_navigation(): - c = _mock_conductor() - c.inject_found("alpha", "red backpack", "zone_b") - - assert c.send_dog_to_memory("bravo", "red backpack") is True - # async navigation finishes fast in mock; lock must be released for the next dog. - deadline = time.time() + 3.0 - while c._move_lock.locked() and time.time() < deadline: - time.sleep(0.02) - - assert not c._move_lock.locked() - assert any(e.type == "arrived" for e in c._events) - - -# -- full mission sequence --------------------------------------------------- - - -def test_full_mock_mission_reaches_done(): - c = _mock_conductor() - - c.start_act1() - assert c.snapshot()["mission"] == "SCOUT_ALPHA" - - c.inject_found("alpha", "red backpack", "zone_b") - assert c.snapshot()["mission"] == "FOUND_EVENT_RECORDED" - - c.ask_where("bravo", "red backpack") - assert c.snapshot()["mission"] == "QUERY_BRAVO" - - assert c.send_dog_to_memory("bravo", "red backpack") is True - assert c.snapshot()["mission"] == "BRAVO_NAVIGATING" - - c.verify_at_zone("bravo", "red backpack") - assert _wait_for_mission(c, "DONE") - - -def test_emergency_stop_resets_to_idle(): - c = _mock_conductor() - c.start_act1() - c.inject_found("alpha", "red backpack", "zone_b") - - c.emergency_stop() - - snap = c.snapshot() - assert snap["mission"] == "IDLE" - assert all(d["status"] == "idle" for d in snap["roster"]) - - -# -- arg parsing ------------------------------------------------------------- - - -def test_parse_dog_default_port(): - dog = _parse_dog("alpha=10.0.0.10") - - assert dog.mcp_url == "http://10.0.0.10:9990/mcp" - assert dog.role == "scout" - - -def test_parse_dog_explicit_port_and_case_insensitive(): - dog = _parse_dog("BRAVO=1.2.3.4:7000") - - assert dog.id == "bravo" - assert dog.mcp_url == "http://1.2.3.4:7000/mcp" - - -def test_parse_dog_rejects_unknown_id(): - with pytest.raises(argparse.ArgumentTypeError): - _parse_dog("zulu=1.2.3.4") - - -# -- MCP response shape tolerance -------------------------------------------- - - -@pytest.mark.parametrize( - "data,expected", - [ - ({"result": {"content": [{"text": "hi"}]}}, "hi"), - ({"content": [{"text": "a"}, {"text": "b"}]}, "a b"), - ({"result": {"content": []}}, None), - ({"result": {}}, None), - ("not a dict", None), - ({"result": {"content": [{"no_text": 1}]}}, None), - ], -) -def test_extract_text_tolerates_shape_variation(data, expected): - assert _extract_text(data) == expected diff --git a/dimos/experimental/pack_mind/venue_go2.sh b/dimos/experimental/pack_mind/venue_go2.sh deleted file mode 100755 index d57fc8318f..0000000000 --- a/dimos/experimental/pack_mind/venue_go2.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# PACK MIND — live Go2 venue bring-up (Go2 EDU onboard-Jetson topology). -# -# Run this from the operator laptop AFTER joining the dog's WiFi (dimair14). -# It probes the dog, guides the on-dog stack launch, then validates G0 and -# launches the conductor. Read-only probing; the on-dog launch is explicit. -# -# Override defaults via env: DOG=192.168.12.1 SSH_USER=unitree MCP_PORT=9990 ./venue_go2.sh -set -uo pipefail - -DOG="${DOG:-192.168.12.1}" -SSH_USER="${SSH_USER:-unitree}" -PORT="${MCP_PORT:-9990}" -SELF="$(cd "$(dirname "$0")" && pwd)" - -stage() { printf '\n=== %s ===\n' "$1"; } -ok() { printf ' PASS: %s\n' "$1"; } -bad() { printf ' FAIL: %s\n' "$1"; } - -stage "0 network — am I on dimair14 and can I see the dog?" -if ping -c 2 -t 3 "$DOG" >/dev/null 2>&1; then - ok "dog reachable at $DOG" -else - bad "cannot reach $DOG — join WiFi dimair14 (pw 88888888) first, then re-run." - exit 1 -fi - -stage "1 probe the dog (prompts for the dog's ssh password)" -echo " ssh ${SSH_USER}@${DOG} ..." -ssh -o ConnectTimeout=8 -o StrictHostKeyChecking=no "${SSH_USER}@${DOG}" ' - echo " host: $(hostname)"; - printf " cuda: "; (nvidia-smi -L 2>/dev/null | head -1 || echo "NO nvidia-smi — wrong host or no GPU"); - printf " dimos: "; (command -v dimos 2>/dev/null || ls -d ~/dimos 2>/dev/null || echo "NOT FOUND — DimOS not installed here"); - printf " port '"$PORT"' busy?: "; (ss -ltn 2>/dev/null | grep ":'"$PORT"'" || echo "free"); -' || { bad "ssh failed — check user/host/password. Default Unitree user is 'unitree'."; exit 1; } - -cat </dev/null | sed 's/^/ tool: /' || true) -else - bad "no tool list back. Got: ${RESP:0:200}" - echo " -> re-check --listen-host 0.0.0.0 on the dog, then dog firewall." - exit 1 -fi - -stage "4 G1 — make the REAL dog speak (direct MCP, bypasses conductor)" -curl -s --max-time 10 -X POST "http://${DOG}:${PORT}/mcp" \ - -H 'content-type: application/json' \ - -d '{"jsonrpc":"2.0","id":"2","method":"tools/call","params":{"name":"speak","arguments":{"text":"Pack mind online.","blocking":false}}}' \ - | (jq -r '.result.content[].text' 2>/dev/null || cat) -echo " (listen — the dog should have spoken)" - -stage "5 launch the conductor + dashboard" -cat < drive start_act1 / inject_found / ask_where / send_dog / verify - # second dog: add --dog bravo=:9990 -EOF -echo "Done. Single-dog path is live once the dashboard speaks through the dog." diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 9bd664df27..e4cc2ca130 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -199,6 +199,7 @@ "real-sense-camera": "dimos.hardware.sensors.camera.realsense.camera.RealSenseCamera", "receiver-module": "dimos.utils.demo_image_encoding.ReceiverModule", "recorder": "dimos.memory2.module.Recorder", + "red-object-detector": "dimos.experimental.pack_mind.red_detector.RedObjectDetector", "reid-module": "dimos.perception.detection.reid.module.ReidModule", "relocalization-module": "dimos.mapping.relocalization.module.RelocalizationModule", "replanning-a-star-planner": "dimos.navigation.replanning_a_star.module.ReplanningAStarPlanner", @@ -218,6 +219,7 @@ "unitree-g1-skill-container": "dimos.robot.unitree.g1.skill_container.UnitreeG1SkillContainer", "unitree-skill-container": "dimos.robot.unitree.unitree_skill_container.UnitreeSkillContainer", "unity-bridge-module": "dimos.simulation.unity.module.UnityBridgeModule", + "velocity-teleop": "dimos.experimental.pack_mind.velocity_teleop.VelocityTeleop", "vlm-agent": "dimos.agents.vlm_agent.VLMAgent", "vlm-stream-tester": "dimos.agents.vlm_stream_tester.VlmStreamTester", "voxel-grid-mapper": "dimos.mapping.voxels.VoxelGridMapper", diff --git a/dimos/robot/test_all_blueprints_generation.py b/dimos/robot/test_all_blueprints_generation.py index d8b0081d7f..d141f4e3ed 100644 --- a/dimos/robot/test_all_blueprints_generation.py +++ b/dimos/robot/test_all_blueprints_generation.py @@ -32,7 +32,14 @@ "dimos/core/blueprints.py", "dimos/core/test_blueprints.py", } -BLUEPRINT_METHODS = {"transports", "global_config", "remappings", "requirements", "configurators"} +BLUEPRINT_METHODS = { + "transports", + "global_config", + "remappings", + "requirements", + "configurators", + "disabled_modules", +} _EXCLUDED_MODULE_NAMES = {"Module", "ModuleBase", "StreamModule"} From ee14c400868425f0f75545a8dda2f123df24ca48 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 10:02:50 -0600 Subject: [PATCH 33/35] docs: add PACK MIND 90s demo video (hackathon deliverable, git-LFS) From 22b6ec2b3e3eed59bb2a039ffa604cc75b738437 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 10:51:39 -0600 Subject: [PATCH 34/35] docs: PACK MIND hosted pitch deck + architecture diagrams in README --- dimos/experimental/pack_mind/README.md | 18 ++++++++++++++++++ .../docs/pack_mind_partition_vs_memory.png | 3 +++ .../pack_mind/docs/packmind_how_it_works.png | 3 +++ .../pack_mind/docs/packmind_system_design.png | 3 +++ 4 files changed, 27 insertions(+) create mode 100644 dimos/experimental/pack_mind/docs/pack_mind_partition_vs_memory.png create mode 100644 dimos/experimental/pack_mind/docs/packmind_how_it_works.png create mode 100644 dimos/experimental/pack_mind/docs/packmind_system_design.png diff --git a/dimos/experimental/pack_mind/README.md b/dimos/experimental/pack_mind/README.md index 7aa65e0b79..bfc794f2c0 100644 --- a/dimos/experimental/pack_mind/README.md +++ b/dimos/experimental/pack_mind/README.md @@ -7,6 +7,24 @@ A team of robot dogs explores an unknown space. The question the demo answers: dogs, same start, same map; the *only* difference is whether they share one discovered map or each keep a private one. +## Pitch deck & diagrams + +**🎤 Pitch deck (browser):** — the idea, the two magic beats +(handoff + inheritance), the A/B proof, and the fleet-memory-layer business case. + +**Share meaning, not maps** — a static zone partition is just a brittle special case of +shared memory (it can't adapt to failure, discovery, or rebalancing): + +![Static partition vs shared memory](docs/pack_mind_partition_vs_memory.png) + +**How it works** — every dog, every tick: sense → remember → choose → plan → move, all +over one shared memory; the A/B knob is whether dogs share that memory or each keep their own: + +![PACK MIND search loop and shared memory](docs/packmind_how_it_works.png) + +System design — one coordination layer, two substrates (sim shares *cells*, live shares +*zones*, never coordinates): [`docs/packmind_system_design.png`](docs/packmind_system_design.png). + ## Thesis (and honest limits) - **Speed:** a pack sharing coverage/discovery memory avoids redundant re-searching, so diff --git a/dimos/experimental/pack_mind/docs/pack_mind_partition_vs_memory.png b/dimos/experimental/pack_mind/docs/pack_mind_partition_vs_memory.png new file mode 100644 index 0000000000..39b7068207 --- /dev/null +++ b/dimos/experimental/pack_mind/docs/pack_mind_partition_vs_memory.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62eca7ed015dd980a5ca2a42b8d7cb759ddd97346faef23b7e75d79cc31764db +size 293938 diff --git a/dimos/experimental/pack_mind/docs/packmind_how_it_works.png b/dimos/experimental/pack_mind/docs/packmind_how_it_works.png new file mode 100644 index 0000000000..037fd372ed --- /dev/null +++ b/dimos/experimental/pack_mind/docs/packmind_how_it_works.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa626a360469a02aea9cd0f28fb969d87a91cdba9acb70b10b2d5c2e0da9945 +size 570774 diff --git a/dimos/experimental/pack_mind/docs/packmind_system_design.png b/dimos/experimental/pack_mind/docs/packmind_system_design.png new file mode 100644 index 0000000000..846fc3f58c --- /dev/null +++ b/dimos/experimental/pack_mind/docs/packmind_system_design.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc3486c492fb6aa6934ba64a24cec02655a559d8ad120c50ca904154d90bb7c9 +size 930901 From 5f27bc4736411ac1652a5dc3506819a09e530502 Mon Sep 17 00:00:00 2001 From: Kai Xia Date: Thu, 28 May 2026 11:47:36 -0600 Subject: [PATCH 35/35] feat: PACK MIND live 2-dog runbook + preflight + scripted stable runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AP-per-dog + Tailscale topology for the live demo no existing doc covers (README assumes shared-router/STA; live.py's LAN_IP only works on one LAN). - LIVE_RUNBOOK.md: on-site page — night-before checklist, per-laptop env, bring-up order, stable/showcase demo paths, failure->fix table. - preflight.py: per-laptop chain checker (dog/route/internet/tailscale/ coordinator) with exact remediation per FAIL; pure stdlib. - scripted_pack_run.py: can't-miss stable path — one process per laptop drives this dog's identity against the real coordinator (no-overlap + inheritance) and optionally moves the real dog via /cmd_vel teleop; no LLM/nav-RPC. --- dimos/experimental/pack_mind/LIVE_RUNBOOK.md | 166 +++++++++++ dimos/experimental/pack_mind/preflight.py | 274 ++++++++++++++++++ .../pack_mind/scripted_pack_run.py | 195 +++++++++++++ 3 files changed, 635 insertions(+) create mode 100644 dimos/experimental/pack_mind/LIVE_RUNBOOK.md create mode 100644 dimos/experimental/pack_mind/preflight.py create mode 100644 dimos/experimental/pack_mind/scripted_pack_run.py diff --git a/dimos/experimental/pack_mind/LIVE_RUNBOOK.md b/dimos/experimental/pack_mind/LIVE_RUNBOOK.md new file mode 100644 index 0000000000..27d53d68f1 --- /dev/null +++ b/dimos/experimental/pack_mind/LIVE_RUNBOOK.md @@ -0,0 +1,166 @@ +# PACK MIND — LIVE RUNBOOK (2 dogs / 2 laptops / AP-per-dog + Tailscale) + +The on-site page for the 2-dog live demo. Documents the topology the README's "Live demo" +section does NOT cover: each Go2 stays in its own **AP mode**, each laptop is **dual-homed** +(Wi-Fi → its dog's AP, a USB-tether uplink → the internet), and the two laptops reach the +shared coordinator over a **Tailscale** tunnel across the internet. + +> README's live section assumes one shared router + dogs in STA (rejected: no trusted net on +> site) and its `--robot-ip` flag is stale. `live.py`'s LAN_IP only works on one LAN. Neither +> fits this topology — use THIS page. + +``` + Go2-A ──AP 192.168.12.1── Laptop-A(alpha) ── USB-tether ─→ INTERNET ─┐ + │ runs coordinator :8090 │ Tailscale + │ tailscale 100.x.A │ 100.64/10 + Go2-B ──AP 192.168.12.1── Laptop-B(bravo) ── USB-tether ─→ INTERNET ─┘ + PACK_COORDINATOR_URL=http://100.x.A:8090 +``` +Both dogs share IP `192.168.12.1` — two separate Wi-Fi L2 nets, each laptop sees only its +own dog. No conflict. The dog never touches Tailscale; only laptop↔laptop coordinator HTTP does. + +### Honesty red line (from PLAN §4 — Q&A will probe this) +The live demo proves **embodiment + coordination + memory-outlives-the-robot** (no-overlap, +inheritance). It does **NOT** prove the coverage-speed A/B claim — that's the sim's job. +Narrate: "two bodies, one shared memory; lose one, the mission survives." + +--- + +## 0. NIGHT BEFORE (cannot be debugged inside the demo window) + +1. **USB-tether uplink on both laptops** (Wi-Fi is taken by the dog AP). A single-Wi-Fi Mac + with no second uplink CANNOT do this topology. +2. **Tailscale on both, same tailnet.** From Laptop-B: `ping ` over + the internet (no dogs needed). +3. **Service order: uplink ABOVE Wi-Fi** (System Settings → Network → ⋯ → Set Service + Order). Else macOS routes WAN through the dog AP → Tailscale dies. Verify + `route -n get 8.8.8.8` shows the uplink iface. +4. **Cross-net smoke (no dogs)** — prove the ledger works over Tailscale end to end: + ```bash + # Laptop-A: + uv run python -m dimos.experimental.pack_mind.pack_coordinator_server \ + --zones north,east,south,west --prefs "alpha:south,north,east,west;bravo:north,east,west,south" + # Laptop-B (A's tailscale ip): + uv run python -m dimos.experimental.pack_mind.mock_dog --dog bravo \ + --url http://:8090 --reset --target "red kit" + ``` + Dashboard at `http://:8090` animates from Laptop-B's browser → brain works. +5. **Per-laptop `~/.packmind.env`** pre-written (§2). Confirm `OPENAI_API_KEY` valid on uplink. +6. **Prefetch models** (insurance): `uv run python -m dimos.experimental.pack_mind.prefetch_live_models`. + +--- + +## 1. MORNING — preflight each laptop FIRST (fix every FAIL before starting dogs) + +```bash +# Laptop-A: +uv run python -m dimos.experimental.pack_mind.preflight \ + --role alpha --peer --coordinator http://127.0.0.1:8090 +# Laptop-B: +uv run python -m dimos.experimental.pack_mind.preflight \ + --role bravo --peer --coordinator http://:8090 +``` + +--- + +## 2. Per-laptop env (`~/.packmind.env`, `source` before anything) + +```bash +# BOTH: +export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +export ROBOT_IP=192.168.12.1 # each laptop's OWN dog over its AP +export LISTEN_HOST=0.0.0.0 +export OPENAI_API_KEY= # agent brain (hosted) — needs the uplink +# Laptop-A only: +export PACK_DOG_NAME=alpha +export PACK_COORDINATOR_URL=http://127.0.0.1:8090 +# Laptop-B only: +export PACK_DOG_NAME=bravo +export PACK_COORDINATOR_URL=http://:8090 +``` +`dimos run` has **no `--robot-ip` flag** in this build — it reads `ROBOT_IP`; verify with +`dimos show-config`. `ROBOT_IP=192.168.12.1` = dog's-own-AP WebRTC path (one laptop per dog). + +--- + +## 3. Bring-up (Laptop-A first) + +**Laptop-A — coordinator + dashboard + dog alpha:** +```bash +source ~/.packmind.env +uv run python -m dimos.experimental.pack_mind.pack_coordinator_server \ + --zones north,east,south,west \ + --prefs "alpha:south,north,east,west;bravo:north,east,west,south" & +# → open http://:8090 on the projector NOW +uv run dimos run unitree-go2-pack --daemon +``` +**Laptop-B — dog bravo:** +```bash +source ~/.packmind.env +uv run dimos run unitree-go2-pack --daemon +``` +Per-dog gate before demo: ping dog → `dimos show-config` → run to "running" → `mcp +list-tools` + `speak` (proves WebRTC) → a `drive` burst → `look_for_red` → search skills. + +--- + +## 4. THE DEMO — STABLE first, SHOWCASE only if stable lands + +### 4a. STABLE (scripted, can't-miss) — `scripted_pack_run` +Real dogs move via velocity teleop; the **real coordinator** drives no-overlap + inheritance; +the dashboard animates. No LLM/nav-RPC on the critical path. +```bash +# Laptop-A (alpha = leader, holds target zone, you drop it on cue): +uv run python -m dimos.experimental.pack_mind.scripted_pack_run \ + --role alpha --leader --target "red kit" --target-zone south \ + --hold-zone --drop-on-enter --drive +# Laptop-B (bravo = sweeper, inherits + finds): +uv run python -m dimos.experimental.pack_mind.scripted_pack_run \ + --role bravo --target "red kit" --target-zone south \ + --find --keep-polling --drive +``` +Beat: alpha claims `south` and holds it → bravo clears `north/east/west` **zero overlap** → +on the host's cue press **Enter** on Laptop-A ("we lose Alpha") → alpha OFFLINE, `south` +returns to the pool → bravo **inherits** `south`, finds the kit → pack stops. Mission +outlived the robot. Drop `--drive` to run the pure-ledger story if the floor is tight. + +### 4b. SHOWCASE (full autonomy, higher risk) +Voice/`mcp` to each dog: *"find the red kit."* Agent runs `start_search → next_zone → +navigate → look_for_red → report_*`. Fragile: `navigate_with_text`/`relative_move` stall on +CPU/LCM-RPC. If a dog hangs, fall back to 4a — coordinator state is shared, so scripted and +autonomous dogs can even mix. + +--- + +## 5. FAILURE → FIX + +| Symptom | Cause | Fix | +|---|---|---| +| Laptop-B can't reach coordinator | NAT / wrong URL / service order | `tailscale ping `; uplink is default route; URL uses A's **tailscale** ip not 192.168.x | +| Internet dead on dog AP | Wi-Fi above uplink in service order | reorder uplink above Wi-Fi; `route -n get default` = uplink iface | +| `tailscale` not found (App Store) | CLI off PATH | `/Applications/Tailscale.app/Contents/MacOS/Tailscale` | +| Dog unreachable | not on its AP / wrong ROBOT_IP | join dog Wi-Fi; `ping 192.168.12.1`; `dimos show-config` | +| `dimos run` ignores robot ip | used a flag | unset; set `ROBOT_IP` env; `dimos show-config` | +| EdgeTAM/CUDA crash on deploy | GPU-only modules | none needed — `unitree-go2-pack` already disables SecurityModule + PersonFollow | +| Dog reaches goal but driver freezes | `relative_move` waits on flaky `is_goal_reached` RPC | use `scripted_pack_run --drive` (VelocityTeleop lane) | +| `look_out_for`/moondream slow 10–60s | VLM on CPU | use `look_for_red` (GPU-free) on the critical path | +| bravo stops before inheriting | next_zone empty while alpha still held south | run bravo with `--keep-polling` | +| projector state stale | dashboard cache | refresh `http://:8090` (it polls `/state`) | + +--- + +## 6. Rehearse with ZERO dogs (do the night before) +```bash +uv run python -m dimos.experimental.pack_mind.demo_pack_live --pace 2 # http://localhost:8090 +# OR the real two-process story locally (no --drive): +uv run python -m dimos.experimental.pack_mind.pack_coordinator_server \ + --prefs "alpha:south,north,east,west;bravo:north,east,west,south" & +uv run python -m dimos.experimental.pack_mind.scripted_pack_run --role alpha --leader \ + --target-zone south --hold-zone --drop-on-enter +uv run python -m dimos.experimental.pack_mind.scripted_pack_run --role bravo \ + --target-zone south --find --keep-polling +``` + +## 7. Teardown +`kill` the coordinator (holds the only shared state), Ctrl-C the `dimos run` daemons, +`tailscale down` to leave the tailnet. Nothing persists server-side. diff --git a/dimos/experimental/pack_mind/preflight.py b/dimos/experimental/pack_mind/preflight.py new file mode 100644 index 0000000000..ac03669568 --- /dev/null +++ b/dimos/experimental/pack_mind/preflight.py @@ -0,0 +1,274 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — live preflight for the 2-dog / 2-laptop demo. + +Run on EACH laptop the morning of the demo, BEFORE starting dogs. Walks the whole +AP-per-dog + Tailscale chain link by link and prints PASS/WARN/FAIL with the exact +fix for every failure. Pure stdlib (macOS-focused) — no dimos import, no extra deps. + + uv run python -m dimos.experimental.pack_mind.preflight \\ + --role alpha --peer --coordinator http://127.0.0.1:8090 + +Exit code 0 only when nothing is FAIL (WARN is allowed). See LIVE_RUNBOOK.md. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import os +import shutil +import socket +import subprocess +import sys +from typing import Literal +import urllib.request + +Status = Literal["PASS", "WARN", "FAIL"] + +_GREEN = "\033[32m" +_YELLOW = "\033[33m" +_RED = "\033[31m" +_DIM = "\033[2m" +_RESET = "\033[0m" +_COLOR: dict[Status, str] = {"PASS": _GREEN, "WARN": _YELLOW, "FAIL": _RED} + +_DOG_AP_SUBNET_PREFIX = "192.168.12." # Unitree Go2 AP default +_TS_APP_CLI = "/Applications/Tailscale.app/Contents/MacOS/Tailscale" + + +@dataclass +class Check: + status: Status + name: str + detail: str + fix: str = "" + + +def _run(cmd: list[str], timeout: float = 6.0) -> tuple[int, str]: + """Run a command; return (rc, combined stdout+stderr). rc=-1 on missing/timeout.""" + try: + p = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, check=False + ) + return p.returncode, (p.stdout + p.stderr).strip() + except (FileNotFoundError, subprocess.TimeoutExpired) as exc: + return -1, str(exc) + + +def _route_iface(target: str) -> str | None: + """macOS `route -n get ` -> the egress interface name, or None.""" + rc, out = _run(["route", "-n", "get", target]) + if rc != 0: + return None + for line in out.splitlines(): + stripped = line.strip() + if stripped.startswith("interface:"): + return stripped.split(":", 1)[1].strip() + return None + + +def _ifaddr(iface: str) -> str | None: + rc, out = _run(["ipconfig", "getifaddr", iface]) + return out.strip() if rc == 0 and out.strip() else None + + +def _tailscale_bin() -> str | None: + return shutil.which("tailscale") or ( + _TS_APP_CLI if os.path.exists(_TS_APP_CLI) else None + ) + + +# -- individual checks ------------------------------------------------------- + + +def check_env(role: str) -> list[Check]: + out: list[Check] = [] + dog = os.environ.get("PACK_DOG_NAME") + if dog and dog == role: + out.append(Check("PASS", "env PACK_DOG_NAME", dog)) + elif dog: + out.append( + Check("WARN", "env PACK_DOG_NAME", f"{dog!r} != --role {role!r}", + f"export PACK_DOG_NAME={role}") + ) + else: + out.append(Check("FAIL", "env PACK_DOG_NAME", "unset", f"export PACK_DOG_NAME={role}")) + + url = os.environ.get("PACK_COORDINATOR_URL") + if url: + out.append(Check("PASS", "env PACK_COORDINATOR_URL", url)) + else: + out.append(Check("FAIL", "env PACK_COORDINATOR_URL", "unset", + "export PACK_COORDINATOR_URL=http://:8090")) + + key = os.environ.get("OPENAI_API_KEY") + if key: + out.append(Check("PASS", "env OPENAI_API_KEY", "set")) + else: + out.append(Check("WARN", "env OPENAI_API_KEY", "unset (only needed for autonomy path 4b)", + "export OPENAI_API_KEY=")) + return out + + +def check_dog(robot_ip: str) -> list[Check]: + out: list[Check] = [] + rc, _ = _run(["ping", "-c1", "-t2", robot_ip], timeout=5) + if rc != 0: + out.append( + Check("FAIL", "dog reachable", f"{robot_ip} no reply", + "join the dog's Wi-Fi AP; confirm ROBOT_IP; power-cycle the dog") + ) + return out + out.append(Check("PASS", "dog reachable", robot_ip)) + + dog_iface = _route_iface(robot_ip) + if dog_iface: + out.append(Check("PASS", "route->dog", f"{robot_ip} via {dog_iface}")) + else: + out.append(Check("WARN", "route->dog", "no route entry", "reconnect the dog Wi-Fi")) + return out + + +def check_internet(robot_ip: str) -> list[Check]: + out: list[Check] = [] + dog_iface = _route_iface(robot_ip) + wan_iface = _route_iface("8.8.8.8") or _route_iface("default") + + if not wan_iface: + out.append(Check("FAIL", "default route", "none", + "connect the uplink (USB tether / Ethernet)")) + return out + + if dog_iface and wan_iface == dog_iface: + out.append( + Check("FAIL", "default route", f"WAN routes via dog AP iface {wan_iface}", + "System Settings->Network->...->Set Service Order: put the uplink ABOVE Wi-Fi") + ) + else: + out.append(Check("PASS", "default route", f"WAN via {wan_iface} (dog via {dog_iface})")) + + wan_ip = _ifaddr(wan_iface) + if wan_ip and wan_ip.startswith(_DOG_AP_SUBNET_PREFIX): + out.append( + Check("FAIL", "subnet collision", + f"uplink {wan_ip} collides with dog AP {_DOG_AP_SUBNET_PREFIX}x", + "change the uplink subnet or the dog AP subnet -- they must differ") + ) + elif wan_ip: + out.append(Check("PASS", "uplink subnet", wan_ip)) + + try: + with socket.create_connection(("1.1.1.1", 443), timeout=5): + out.append(Check("PASS", "internet", "1.1.1.1:443 reachable")) + except OSError as exc: + out.append(Check("FAIL", "internet", str(exc), + "fix the uplink / service order before Tailscale will work")) + return out + + +def check_tailscale(peer: str | None) -> list[Check]: + out: list[Check] = [] + ts = _tailscale_bin() + if not ts: + out.append( + Check("FAIL", "tailscale", "CLI not found", + f"install Tailscale; App Store build CLI at {_TS_APP_CLI}") + ) + return out + + rc, status_txt = _run([ts, "status"]) + if rc != 0: + out.append(Check("FAIL", "tailscale status", status_txt[:80] or "down", f"{ts} up")) + return out + + rc, self_ip_raw = _run([ts, "ip", "-4"]) + self_ip = self_ip_raw.splitlines()[0].strip() if rc == 0 and self_ip_raw.strip() else "" + if self_ip.startswith("100."): + out.append(Check("PASS", "tailscale up", f"self {self_ip}")) + else: + out.append(Check("WARN", "tailscale up", "no 100.x self ip", f"{ts} up --accept-routes")) + + if peer: + rc, pout = _run([ts, "ping", "--timeout=4s", "--c=1", peer], timeout=8) + if rc == 0 and "pong" in pout.lower(): + out.append(Check("PASS", "peer reachable", f"tailscale ping {peer}")) + else: + rc2, _ = _run(["ping", "-c1", "-t4", peer], timeout=6) + if rc2 == 0: + out.append(Check("PASS", "peer reachable", f"icmp {peer}")) + else: + out.append(Check("FAIL", "peer reachable", f"{peer} no pong/icmp", + "check peer laptop tailscale up + its service order/uplink")) + return out + + +def check_coordinator(url: str) -> list[Check]: + url = url.rstrip("/") + try: + with urllib.request.urlopen(f"{url}/state", timeout=5) as resp: + data = json.loads(resp.read().decode()) + zones = data.get("zones") if isinstance(data, dict) else None + if isinstance(zones, list) and zones: + names = ",".join(str(z.get("name", "?")) for z in zones) + return [Check("PASS", "coordinator /state", f"{url} zones=[{names}]")] + return [Check("WARN", "coordinator /state", f"{url} reachable but no zones", + "start it with --zones north,east,south,west")] + except Exception as exc: + return [Check("FAIL", "coordinator /state", f"{url}: {exc}", + "start pack_coordinator_server on Laptop-A; check the tailscale URL/port")] + + +# -- driver ------------------------------------------------------------------ + + +def main() -> int: + ap = argparse.ArgumentParser(description="PACK MIND live preflight (per laptop)") + ap.add_argument("--role", required=True, choices=["alpha", "bravo"]) + ap.add_argument("--peer", default=None, help="the OTHER laptop's tailscale IP") + ap.add_argument( + "--coordinator", + default=os.environ.get("PACK_COORDINATOR_URL", "http://127.0.0.1:8090"), + ) + ap.add_argument("--robot-ip", default=os.environ.get("ROBOT_IP", "192.168.12.1")) + args = ap.parse_args() + + checks: list[Check] = [] + checks += check_env(args.role) + checks += check_dog(args.robot_ip) + checks += check_internet(args.robot_ip) + checks += check_tailscale(args.peer) + checks += check_coordinator(args.coordinator) + + print(f"\n PACK MIND preflight -- role={args.role} dog={args.robot_ip}\n") + width = max(len(c.name) for c in checks) + for c in checks: + print(f" {_COLOR[c.status]}{c.status:<4}{_RESET} {c.name:<{width}} {c.detail}") + if c.status != "PASS" and c.fix: + print(f" {_DIM}fix: {c.fix}{_RESET}") + + fails = sum(c.status == "FAIL" for c in checks) + warns = sum(c.status == "WARN" for c in checks) + print(f"\n {fails} FAIL · {warns} WARN · {len(checks) - fails - warns} PASS") + if fails: + print(f" {_RED}NOT READY -- fix the FAILs above before starting dogs.{_RESET}\n") + return 1 + print(f" {_GREEN}READY.{_RESET}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dimos/experimental/pack_mind/scripted_pack_run.py b/dimos/experimental/pack_mind/scripted_pack_run.py new file mode 100644 index 0000000000..35b7e3070e --- /dev/null +++ b/dimos/experimental/pack_mind/scripted_pack_run.py @@ -0,0 +1,195 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — live STABLE path (the can't-miss demo runner, one process per laptop). + +Drives THIS dog's identity against the REAL coordinator (no LLM, no nav planner) and +optionally moves the REAL dog via velocity teleop, so the no-overlap + inheritance +story plays on the projector dashboard with zero autonomy fragility. The coordination +story is identical with or without ``--drive``. + + # Laptop-A (alpha holds the target zone, you drop it on cue): + uv run python -m dimos.experimental.pack_mind.scripted_pack_run --role alpha --leader \\ + --target "red kit" --target-zone south --hold-zone --drop-on-enter --drive + # Laptop-B (bravo sweeps, inherits, finds): + uv run python -m dimos.experimental.pack_mind.scripted_pack_run --role bravo \\ + --target "red kit" --target-zone south --find --keep-polling --drive + +Movement (``--drive``) publishes Twist to ``/cmd_vel`` exactly like ``demo_drive_go2``, +so a ``dimos run unitree-go2-pack`` daemon must already hold the WebRTC link on this +laptop (LCM is host-local). See LIVE_RUNBOOK.md §4a. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable +import time +from typing import Any + +import requests + +_TIMEOUT = 5.0 +_POLL_SECS = 1.0 # how often a sweeper re-asks while waiting to inherit + +Driver = Callable[[], None] + + +def _post(url: str, path: str, payload: dict[str, Any]) -> dict[str, Any] | None: + try: + r = requests.post(f"{url}{path}", json=payload, timeout=_TIMEOUT) + r.raise_for_status() + data: Any = r.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + print(f" [warn] {path} failed: {exc}") + return None + + +def _get(url: str, path: str, params: dict[str, str]) -> dict[str, Any] | None: + try: + r = requests.get(f"{url}{path}", params=params, timeout=_TIMEOUT) + r.raise_for_status() + data: Any = r.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + print(f" [warn] {path} failed: {exc}") + return None + + +def _make_driver(drive_secs: float) -> Driver: + """A /cmd_vel velocity-burst driver (host-local LCM), modelled on demo_drive_go2.""" + from dimos.core.transport import LCMTransport + from dimos.msgs.geometry_msgs.Twist import Twist + from dimos.msgs.geometry_msgs.Vector3 import Vector3 + + transport: LCMTransport[Twist] = LCMTransport("/cmd_vel", Twist) + + def burst(vx: float, wz: float, secs: float) -> None: + deadline = time.time() + secs + while time.time() < deadline: + transport.broadcast( + None, + Twist(linear=Vector3(x=vx, y=0.0, z=0.0), angular=Vector3(x=0.0, y=0.0, z=wz)), + ) + time.sleep(0.1) + transport.broadcast( + None, Twist(linear=Vector3(0.0, 0.0, 0.0), angular=Vector3(0.0, 0.0, 0.0)) + ) + + # a visible "I'm searching this zone" wiggle: forward, turn, forward. + def choreograph() -> None: + seg = max(0.4, drive_secs / 3) + burst(0.3, 0.0, seg) + burst(0.0, 0.5, seg) + burst(0.3, 0.0, seg) + + return choreograph + + +def run(args: argparse.Namespace) -> None: + url: str = args.url.rstrip("/") + dog: str = args.role + target: str = args.target + target_zone: str = args.target_zone + pace: float = args.pace + driver: Driver | None = _make_driver(args.drive_secs) if args.drive else None + + if args.leader: + status = _post(url, "/start_search", {"target": target}) + print(f" MISSION {status.get('status') if status else '(coordinator unreachable)'}") + time.sleep(pace) + + while True: + stop = _get(url, "/should_stop", {"dog": dog}) + if stop and stop.get("stop"): + print(f" [{dog}] pack found it — converging. STOP.") + return + + assigned = _post(url, "/assign_zone", {"dog": dog}) + zone = assigned.get("zone") if assigned else None + + if not zone: + if args.keep_polling: + stop = _get(url, "/should_stop", {"dog": dog}) + if stop and stop.get("stop"): + print(f" [{dog}] pack found it — STOP.") + return + print(f" [{dog}] no free zone — waiting to inherit a dropped teammate's ground…") + time.sleep(_POLL_SECS) + continue + print(f" [{dog}] no zone left — search over. STOP.") + return + + print(f" [{dog}] CLAIMS {zone}") + if driver: + driver() # visible search wiggle in place of real nav + else: + time.sleep(pace) + + # the dog that HOLDS the target zone: keep it claimed (still searching) and + # wait for the operator to drop us, so a teammate inherits it. + if args.hold_zone and zone == target_zone: + print(f" [{dog}] HOLDING {zone} (the object is here, still searching)…") + if args.drop_on_enter: + input(f" >>> press ENTER to DROP {dog} (simulate losing the robot) <<< ") + else: + time.sleep(pace * 2) + ok = _post(url, "/release_dog", {"dog": dog}) + print(f" [{dog}] OFFLINE — {ok.get('status') if ok else 'released'};" + " its ground returns to the pack.") + return + + # the finder: report the find when it reaches the target zone. + if args.find and zone == target_zone: + res = _post(url, "/report_finding", {"dog": dog, "object": target, "zone": zone}) + found = res.get("finding") if res else None + print(f" [{dog}] FOUND {target} in {zone} -> {found}. Pack STOP.") + return + + cleared = _post(url, "/report_cleared", {"dog": dog, "zone": zone}) + print(f" [{dog}] cleared {zone} (no overlap) — {cleared.get('status') if cleared else '?'}") + time.sleep(pace * 0.4) + + +def _parse() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND live stable per-laptop runner") + p.add_argument("--role", required=True, help="this dog's name, e.g. alpha") + p.add_argument("--url", default="http://127.0.0.1:8090", help="coordinator base URL") + p.add_argument("--target", default="red kit") + p.add_argument("--target-zone", default="south", help="where the object is") + p.add_argument("--leader", action="store_true", help="call start_search (resets ledger)") + p.add_argument("--hold-zone", action="store_true", + help="when assigned the target zone, hold it (don't clear) until dropped") + p.add_argument("--find", action="store_true", + help="when assigned the target zone, report the finding") + p.add_argument("--keep-polling", action="store_true", + help="when no free zone, keep re-asking (to inherit a dropped teammate's zone)") + p.add_argument("--drop-on-enter", action="store_true", + help="while holding, wait for ENTER then release this dog and exit") + p.add_argument("--drive", action="store_true", help="actually move the dog via /cmd_vel teleop") + p.add_argument("--drive-secs", type=float, default=1.5, help="seconds of motion per zone") + p.add_argument("--pace", type=float, default=2.0, help="seconds between beats (no-drive)") + return p.parse_args() + + +def main() -> None: + try: + run(_parse()) + except KeyboardInterrupt: + print("\n interrupted — coordinator state preserved.") + + +if __name__ == "__main__": + main()