diff --git a/.gitignore b/.gitignore index aedee04af7..d645547c00 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,5 @@ htmlcov/ # Memory2 autorecord recording*.db +.scratch/ +.gstack/ diff --git a/dimos/experimental/pack_mind/LIVE_RUNBOOK.md b/dimos/experimental/pack_mind/LIVE_RUNBOOK.md new file mode 100644 index 0000000000..27d53d68f1 --- /dev/null +++ b/dimos/experimental/pack_mind/LIVE_RUNBOOK.md @@ -0,0 +1,166 @@ +# PACK MIND — LIVE RUNBOOK (2 dogs / 2 laptops / AP-per-dog + Tailscale) + +The on-site page for the 2-dog live demo. Documents the topology the README's "Live demo" +section does NOT cover: each Go2 stays in its own **AP mode**, each laptop is **dual-homed** +(Wi-Fi → its dog's AP, a USB-tether uplink → the internet), and the two laptops reach the +shared coordinator over a **Tailscale** tunnel across the internet. + +> README's live section assumes one shared router + dogs in STA (rejected: no trusted net on +> site) and its `--robot-ip` flag is stale. `live.py`'s LAN_IP only works on one LAN. Neither +> fits this topology — use THIS page. + +``` + Go2-A ──AP 192.168.12.1── Laptop-A(alpha) ── USB-tether ─→ INTERNET ─┐ + │ runs coordinator :8090 │ Tailscale + │ tailscale 100.x.A │ 100.64/10 + Go2-B ──AP 192.168.12.1── Laptop-B(bravo) ── USB-tether ─→ INTERNET ─┘ + PACK_COORDINATOR_URL=http://100.x.A:8090 +``` +Both dogs share IP `192.168.12.1` — two separate Wi-Fi L2 nets, each laptop sees only its +own dog. No conflict. The dog never touches Tailscale; only laptop↔laptop coordinator HTTP does. + +### Honesty red line (from PLAN §4 — Q&A will probe this) +The live demo proves **embodiment + coordination + memory-outlives-the-robot** (no-overlap, +inheritance). It does **NOT** prove the coverage-speed A/B claim — that's the sim's job. +Narrate: "two bodies, one shared memory; lose one, the mission survives." + +--- + +## 0. NIGHT BEFORE (cannot be debugged inside the demo window) + +1. **USB-tether uplink on both laptops** (Wi-Fi is taken by the dog AP). A single-Wi-Fi Mac + with no second uplink CANNOT do this topology. +2. **Tailscale on both, same tailnet.** From Laptop-B: `ping ` over + the internet (no dogs needed). +3. **Service order: uplink ABOVE Wi-Fi** (System Settings → Network → ⋯ → Set Service + Order). Else macOS routes WAN through the dog AP → Tailscale dies. Verify + `route -n get 8.8.8.8` shows the uplink iface. +4. **Cross-net smoke (no dogs)** — prove the ledger works over Tailscale end to end: + ```bash + # Laptop-A: + uv run python -m dimos.experimental.pack_mind.pack_coordinator_server \ + --zones north,east,south,west --prefs "alpha:south,north,east,west;bravo:north,east,west,south" + # Laptop-B (A's tailscale ip): + uv run python -m dimos.experimental.pack_mind.mock_dog --dog bravo \ + --url http://:8090 --reset --target "red kit" + ``` + Dashboard at `http://:8090` animates from Laptop-B's browser → brain works. +5. **Per-laptop `~/.packmind.env`** pre-written (§2). Confirm `OPENAI_API_KEY` valid on uplink. +6. **Prefetch models** (insurance): `uv run python -m dimos.experimental.pack_mind.prefetch_live_models`. + +--- + +## 1. MORNING — preflight each laptop FIRST (fix every FAIL before starting dogs) + +```bash +# Laptop-A: +uv run python -m dimos.experimental.pack_mind.preflight \ + --role alpha --peer --coordinator http://127.0.0.1:8090 +# Laptop-B: +uv run python -m dimos.experimental.pack_mind.preflight \ + --role bravo --peer --coordinator http://:8090 +``` + +--- + +## 2. Per-laptop env (`~/.packmind.env`, `source` before anything) + +```bash +# BOTH: +export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +export ROBOT_IP=192.168.12.1 # each laptop's OWN dog over its AP +export LISTEN_HOST=0.0.0.0 +export OPENAI_API_KEY= # agent brain (hosted) — needs the uplink +# Laptop-A only: +export PACK_DOG_NAME=alpha +export PACK_COORDINATOR_URL=http://127.0.0.1:8090 +# Laptop-B only: +export PACK_DOG_NAME=bravo +export PACK_COORDINATOR_URL=http://:8090 +``` +`dimos run` has **no `--robot-ip` flag** in this build — it reads `ROBOT_IP`; verify with +`dimos show-config`. `ROBOT_IP=192.168.12.1` = dog's-own-AP WebRTC path (one laptop per dog). + +--- + +## 3. Bring-up (Laptop-A first) + +**Laptop-A — coordinator + dashboard + dog alpha:** +```bash +source ~/.packmind.env +uv run python -m dimos.experimental.pack_mind.pack_coordinator_server \ + --zones north,east,south,west \ + --prefs "alpha:south,north,east,west;bravo:north,east,west,south" & +# → open http://:8090 on the projector NOW +uv run dimos run unitree-go2-pack --daemon +``` +**Laptop-B — dog bravo:** +```bash +source ~/.packmind.env +uv run dimos run unitree-go2-pack --daemon +``` +Per-dog gate before demo: ping dog → `dimos show-config` → run to "running" → `mcp +list-tools` + `speak` (proves WebRTC) → a `drive` burst → `look_for_red` → search skills. + +--- + +## 4. THE DEMO — STABLE first, SHOWCASE only if stable lands + +### 4a. STABLE (scripted, can't-miss) — `scripted_pack_run` +Real dogs move via velocity teleop; the **real coordinator** drives no-overlap + inheritance; +the dashboard animates. No LLM/nav-RPC on the critical path. +```bash +# Laptop-A (alpha = leader, holds target zone, you drop it on cue): +uv run python -m dimos.experimental.pack_mind.scripted_pack_run \ + --role alpha --leader --target "red kit" --target-zone south \ + --hold-zone --drop-on-enter --drive +# Laptop-B (bravo = sweeper, inherits + finds): +uv run python -m dimos.experimental.pack_mind.scripted_pack_run \ + --role bravo --target "red kit" --target-zone south \ + --find --keep-polling --drive +``` +Beat: alpha claims `south` and holds it → bravo clears `north/east/west` **zero overlap** → +on the host's cue press **Enter** on Laptop-A ("we lose Alpha") → alpha OFFLINE, `south` +returns to the pool → bravo **inherits** `south`, finds the kit → pack stops. Mission +outlived the robot. Drop `--drive` to run the pure-ledger story if the floor is tight. + +### 4b. SHOWCASE (full autonomy, higher risk) +Voice/`mcp` to each dog: *"find the red kit."* Agent runs `start_search → next_zone → +navigate → look_for_red → report_*`. Fragile: `navigate_with_text`/`relative_move` stall on +CPU/LCM-RPC. If a dog hangs, fall back to 4a — coordinator state is shared, so scripted and +autonomous dogs can even mix. + +--- + +## 5. FAILURE → FIX + +| Symptom | Cause | Fix | +|---|---|---| +| Laptop-B can't reach coordinator | NAT / wrong URL / service order | `tailscale ping `; uplink is default route; URL uses A's **tailscale** ip not 192.168.x | +| Internet dead on dog AP | Wi-Fi above uplink in service order | reorder uplink above Wi-Fi; `route -n get default` = uplink iface | +| `tailscale` not found (App Store) | CLI off PATH | `/Applications/Tailscale.app/Contents/MacOS/Tailscale` | +| Dog unreachable | not on its AP / wrong ROBOT_IP | join dog Wi-Fi; `ping 192.168.12.1`; `dimos show-config` | +| `dimos run` ignores robot ip | used a flag | unset; set `ROBOT_IP` env; `dimos show-config` | +| EdgeTAM/CUDA crash on deploy | GPU-only modules | none needed — `unitree-go2-pack` already disables SecurityModule + PersonFollow | +| Dog reaches goal but driver freezes | `relative_move` waits on flaky `is_goal_reached` RPC | use `scripted_pack_run --drive` (VelocityTeleop lane) | +| `look_out_for`/moondream slow 10–60s | VLM on CPU | use `look_for_red` (GPU-free) on the critical path | +| bravo stops before inheriting | next_zone empty while alpha still held south | run bravo with `--keep-polling` | +| projector state stale | dashboard cache | refresh `http://:8090` (it polls `/state`) | + +--- + +## 6. Rehearse with ZERO dogs (do the night before) +```bash +uv run python -m dimos.experimental.pack_mind.demo_pack_live --pace 2 # http://localhost:8090 +# OR the real two-process story locally (no --drive): +uv run python -m dimos.experimental.pack_mind.pack_coordinator_server \ + --prefs "alpha:south,north,east,west;bravo:north,east,west,south" & +uv run python -m dimos.experimental.pack_mind.scripted_pack_run --role alpha --leader \ + --target-zone south --hold-zone --drop-on-enter +uv run python -m dimos.experimental.pack_mind.scripted_pack_run --role bravo \ + --target-zone south --find --keep-polling +``` + +## 7. Teardown +`kill` the coordinator (holds the only shared state), Ctrl-C the `dimos run` daemons, +`tailscale down` to leave the tailnet. Nothing persists server-side. diff --git a/dimos/experimental/pack_mind/README.md b/dimos/experimental/pack_mind/README.md new file mode 100644 index 0000000000..bfc794f2c0 --- /dev/null +++ b/dimos/experimental/pack_mind/README.md @@ -0,0 +1,155 @@ +# PACK MIND + +**One brain, many bodies, one memory that outlives any single dog.** + +A team of robot dogs explores an unknown space. The question the demo answers: +**does *sharing memory* across the pack actually help?** We A/B it head-to-head — same +dogs, same start, same map; the *only* difference is whether they share one discovered +map or each keep a private one. + +## Pitch deck & diagrams + +**🎤 Pitch deck (browser):** — the idea, the two magic beats +(handoff + inheritance), the A/B proof, and the fleet-memory-layer business case. + +**Share meaning, not maps** — a static zone partition is just a brittle special case of +shared memory (it can't adapt to failure, discovery, or rebalancing): + +![Static partition vs shared memory](docs/pack_mind_partition_vs_memory.png) + +**How it works** — every dog, every tick: sense → remember → choose → plan → move, all +over one shared memory; the A/B knob is whether dogs share that memory or each keep their own: + +![PACK MIND search loop and shared memory](docs/packmind_how_it_works.png) + +System design — one coordination layer, two substrates (sim shares *cells*, live shares +*zones*, never coordinates): [`docs/packmind_system_design.png`](docs/packmind_system_design.png). + +## Thesis (and honest limits) + +- **Speed:** a pack sharing coverage/discovery memory avoids redundant re-searching, so + it searches the whole area faster. +- **Resilience (the strongest point):** when a dog goes offline, its discoveries persist + in shared memory and the survivors keep using them. With private memory, that dog's + knowledge dies with it. +- **Honest caveats:** this is a 2D-grid sim (rendered in 3D); point robots, scripted + sensing, no real SLAM/perception. The win is *complete-area search speed + knowledge + persistence* — **not** "finds victims faster" (redundant independent robots can stumble + on victims sooner). Keep the narration honest. + +## Two simulations + +### 1. Fog-of-war exploration (the current demo) — `explore_sim.py` +Unknown maze, revealed by a raycast sensor (walls block line of sight, RTS-style). Dogs +navigate to **frontiers** (known-free / unknown boundary). The discovered map is **shared** +(one map all dogs read+write) or **independent** (one per dog; the team only sees the union +of *online* dogs, so killing a dog erases its territory). Measured (seed 0, 3 dogs): +shared reaches 95% revealed at **tick 246** vs **335** independent; killing a dog drops +independent's team knowledge **0.80 → 0.32**, shared **drops 0**. + +### 2. Coverage race (earlier model) — `sim.py` +Known map; dogs sweep with `CoveragePatrolRouter`; A/B = shared vs private +`VisitationHistory`. Shared hits 75% coverage at tick 1745 vs 3275; final 96.5% vs 88.9%. + +Both reuse DimOS navigation primitives (`min_cost_astar`, patrol routers, +`VisitationHistory`) on a fabricated `OccupancyGrid` — pure numpy/scipy, **no CUDA/ROS/sim**. + +## Run it + +```bash +# Web 3D demo (FastAPI + Three.js) — the centerpiece. Side-by-side fog-of-war race, +# 3-level fog (visible / remembered / unknown), kill-a-dog + reset controls. +uv run python -m dimos.experimental.pack_mind.server # → http://localhost:8000 + +# DimOS Viewer (Rerun) — one maze, the pack exploring on the shared map. Scrub the +# "tick" timeline; add --kill DOG TICK for the resilience beat, --save run.rrd to skip the GUI. +uv run python -m dimos.experimental.pack_mind.view_explore_rerun --dogs 3 --seed 0 + +# Native DimOS blueprint — coverage A/B as two live OccupancyGrid streams in Rerun. +dimos --viewer rerun run pack-mind-sim + +# Standalone A/B video of the coverage race. +uv run python -m dimos.experimental.pack_mind.render --out pack_mind_ab.mp4 + +# Tests +bin/pytest-fast dimos/experimental/pack_mind/test_explore_sim.py -v +bin/pytest-fast dimos/experimental/pack_mind/test_pack_mind_sim.py -v +``` + +## Live demo — 2 dogs / 2 laptops + +The intelligence (shared memory) runs on the laptops; the dogs are bodies. Movement +is teleop-assisted; the agent + coordinator own the memory layer. + +**1. One-time per laptop (with internet):** cache the runtime models so the demo runs +offline (a dog's WiFi AP has no internet, and venue WiFi is flaky): +```bash +uv run python -m dimos.experimental.pack_mind.prefetch_live_models # moondream2 + whisper +``` +CLIP/YOLO ship in `data/` (git-lfs). The agent brain (GPT-4o) is a hosted API — keep +the coordinator laptop on a router with WAN. + +**2. Network:** one travel router; put each Go2 in STA mode (Unitree app) onto it; both +laptops + both dogs on that subnet. DimOS picks the WebRTC method purely from +`ROBOT_IP` — `192.168.12.1` → the dog's own AP (single-laptop only), any other IP → +LocalSTA (shared router, required for 2 laptops). + +**3. Laptop A (Alpha + coordinator + dashboard):** +```bash +export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +export ROBOT_IP= LISTEN_HOST=0.0.0.0 OPENAI_API_KEY= +uv run python -m dimos.experimental.pack_mind.pack_coordinator_server \ + --zones north,east,south,west --prefs "alpha:north,east;bravo:south,west" & # dashboard: http://:8090 +PACK_DOG_NAME=alpha PACK_COORDINATOR_URL=http://127.0.0.1:8090 \ + uv run dimos run unitree-go2-pack --daemon +``` + +**4. Laptop B (Bravo):** +```bash +export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +export ROBOT_IP= LISTEN_HOST=0.0.0.0 OPENAI_API_KEY= +PACK_DOG_NAME=bravo PACK_COORDINATOR_URL=http://:8090 \ + uv run dimos run unitree-go2-pack --daemon +``` + +**Gotchas (hard-won on site):** +- `unitree-go2-pack` already disables EdgeTAM modules (SecurityModule, PersonFollow) — + they hard-require CUDA. No `--disable` flag needed. +- `dimos run` has **no `--robot-ip` flag** in this build — set the `ROBOT_IP` env and + verify with `dimos show-config`. +- `HF_HUB_OFFLINE=1` is required on a no-internet AP; run prefetch first. +- **Detection: use `look_for_red`** (RedObjectDetector) — a fast, GPU-free colour check + that reports the finding to the coordinator instantly. moondream/`look_out_for` also + works but is slow on a CPU host (~10–60s first call); keep it off the critical path. +- Movement: `relative_move` (skill) or keyboard teleop. `navigate_with_text` works but is + slow on CPU (Qwen). +- Bring-up order: ping dog → `dimos show-config` → run to "running" → `mcp list-tools` + + `speak` → `relative_move` → `look_for_red` → `start_search`/`next_zone` (dashboard at :8090). + +**Hardware-free rehearsal (no dogs):** +`uv run python -m dimos.experimental.pack_mind.demo_pack_live --pace 2`, open http://localhost:8090. + +## File map + +| File | Role | +|---|---| +| `world.py` | Fabricate the maze `OccupancyGrid` + plant survivors | +| `explore_sim.py` | **Fog-of-war exploration engine** (raycast reveal, frontier, shared/private discovered map, offline persistence) | +| `server.py` | FastAPI WebSocket backend streaming both explore sims | +| `static/explore.html` | Three.js 3D fog-of-war frontend (side-by-side, kill/reset) | +| `view_explore_rerun.py` | **DimOS Viewer (Rerun)** view of one shared-memory maze search (fog + dogs + trails + coverage scalar) | +| `live.py` | **Live blueprint** `unitree-go2-pack` (one dog per laptop, EdgeTAM disabled) + PACK system prompt | +| `pack_coordinator.py` / `pack_coordinator_server.py` | Shared zone ledger (no-overlap, find, **inheritance**) + JSON/HTTP API + dashboard route | +| `pack_dashboard.html` | Projector dashboard — zones, finding, offline/inheritance, causal chain | +| `pack_search_skills.py` | Dog agent tools: `start_search` / `next_zone` / `report_*` / `where_is` | +| `red_detector.py` | **Fast GPU-free red-object detector** (`look_for_red`) — HSV-free colour test → auto-reports the find | +| `pack_search_runner.py` | `RobotDriver`-protocol search loop + `MockDriver` | +| `mock_dog.py` / `demo_pack_live.py` / `demo_pack_scene.py` | Hardware-free test + projector rehearsal of both magic beats | +| `prefetch_live_models.py` | Pre-cache moondream2 + whisper so the live stack runs offline | +| `sim.py` / `sim_robot.py` | Coverage-race sim (known map) | +| `blueprint.py` | `pack-mind-sim` native DimOS blueprint (publishes coverage as `OccupancyGrid`) | +| `render.py` | Standalone matplotlib → mp4 A/B render | +| `view_rerun.py` | Log the coverage A/B into a Rerun `.rrd` / live viewer | +| `demo_spike.py` | Feasibility spike (kept as a smoke test of the reused primitives) | +| `test_explore_sim.py` / `test_pack_mind_sim.py` | Engine tests | +| `sim_perception.py` | MuJoCo+VLM single-frame perception probe (optional, off the critical path) | diff --git a/dimos/experimental/pack_mind/__init__.py b/dimos/experimental/pack_mind/__init__.py new file mode 100644 index 0000000000..ad1a54cf7e --- /dev/null +++ b/dimos/experimental/pack_mind/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — a shared semantic memory for a team of Unitree Go2s.""" diff --git a/dimos/experimental/pack_mind/blueprint.py b/dimos/experimental/pack_mind/blueprint.py new file mode 100644 index 0000000000..837384b4ae --- /dev/null +++ b/dimos/experimental/pack_mind/blueprint.py @@ -0,0 +1,108 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native DimOS blueprint for the PACK MIND A/B coverage race. + + dimos run pack-mind-sim --viewer rerun + +Runs PACK (shared coverage memory) and INDEPENDENT (private) side by side and +publishes each one's live coverage as an OccupancyGrid. The Rerun bridge (added +automatically by ``--viewer rerun``) subscribes to the bus and renders both: +walls = OCCUPIED (dark), searched = FREE (light), unsearched = UNKNOWN (gray). +Watch the PACK map fill cleanly while INDEPENDENT re-walks the same corridors. +""" + +from __future__ import annotations + +import threading + +import numpy as np + +from dimos.core.coordination.blueprints import autoconnect +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.core.stream import Out +from dimos.experimental.pack_mind.sim import PackSim, build_sim +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid + +_TICK_DT = 0.005 # seconds between sim ticks (animation pace in the viewer) +_PUB_EVERY = 8 # publish a coverage grid every N ticks +_HOLD_S = 4.0 # hold the final frame before replaying the race +_SEED = 0 + + +def _coverage_grid(sim: PackSim, frame_id: str) -> OccupancyGrid: + """Encode coverage into OccupancyGrid semantics for the native colormap: + OCCUPIED=wall (dark), FREE=searched (light), UNKNOWN=unsearched (gray).""" + g = sim.world.grid + free = g == CostValues.FREE + out = np.full(g.shape, CostValues.UNKNOWN, dtype=np.int8) + out[g == CostValues.OCCUPIED] = CostValues.OCCUPIED + union: np.ndarray | None = None + for rb in sim.robots: + v = rb.router._visited + if v is None: + continue + union = v.copy() if union is None else (union | v) + if union is not None: + out[union & free] = CostValues.FREE + return OccupancyGrid(grid=out, resolution=sim.world.info.resolution, frame_id=frame_id) + + +class PackMindSimModule(Module): + """Steps both A/B sims in a background thread and publishes their coverage.""" + + pack_coverage: Out[OccupancyGrid] + indep_coverage: Out[OccupancyGrid] + + @rpc + def start(self) -> None: + super().start() + self._stop_evt = threading.Event() + self._thread = threading.Thread(target=self._loop, daemon=True) + self._thread.start() + + @rpc + def stop(self) -> None: + self._stop_evt.set() + if getattr(self, "_thread", None) is not None and self._thread.is_alive(): + self._thread.join(timeout=2.0) + super().stop() + + def _publish(self, pack: PackSim, indep: PackSim) -> None: + self.pack_coverage.publish(_coverage_grid(pack, "pack")) + self.indep_coverage.publish(_coverage_grid(indep, "independent")) + + def _loop(self) -> None: + while not self._stop_evt.is_set(): + pack = build_sim(shared=True, seed=_SEED) + indep = build_sim(shared=False, seed=_SEED) + self._publish(pack, indep) + while not self._stop_evt.is_set(): + done_p = all(rb.done for rb in pack.robots) + done_i = all(rb.done for rb in indep.robots) + if not done_p: + pack.step() + if not done_i: + indep.step() + if pack.tick_n % _PUB_EVERY == 0 or indep.tick_n % _PUB_EVERY == 0: + self._publish(pack, indep) + if done_p and done_i: + break + self._stop_evt.wait(_TICK_DT) + self._publish(pack, indep) + self._stop_evt.wait(_HOLD_S) # hold final frame, then replay + + +pack_mind_sim = autoconnect(PackMindSimModule.blueprint()) diff --git a/dimos/experimental/pack_mind/demo_drive.py b/dimos/experimental/pack_mind/demo_drive.py new file mode 100644 index 0000000000..27cee6854f --- /dev/null +++ b/dimos/experimental/pack_mind/demo_drive.py @@ -0,0 +1,130 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — keyboard demo driver for one live Go2 running unitree-go2-pack. + +Drive the dog with the keyboard; after every move (when it stops) it AUTO-runs +``look_for_red`` and, on a find, AUTO-``speak``s the announcement. One operator, +hands on the keyboard, zero command typing during the demo. + + uv run python -m dimos.experimental.pack_mind.demo_drive + +Keys: + w / s forward / back (one step) + a / d turn left / right + space look for red now + f speak "red object found" + +/- bigger / smaller step + x quit + +Movement uses the ``relative_move`` skill, which blocks until the dog stops — so +"look on stop" is just: issue the move, then look the moment it returns. +""" + +from __future__ import annotations + +import argparse +import sys +import termios +import tty + +from dimos.agents.mcp.mcp_adapter import McpAdapter + +_FOUND_MARK = "i see a red object" +_FOUND_SPEECH = "red object found" + + +def _getch() -> str: + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + try: + tty.setraw(fd) + return sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old) + + +def _call(adapter: McpAdapter, name: str, args: dict[str, object] | None = None) -> str: + try: + return adapter.call_tool_text(name, args or {}) + except Exception as exc: # never let one flaky call kill the driver + return f"[{name} error: {exc}]" + + +def _look_and_announce(adapter: McpAdapter) -> None: + result = _call(adapter, "look_for_red") + print(f" 👁 {result}") + if _FOUND_MARK in result.lower(): + print(f" 📣 {_call(adapter, 'speak', {'text': _FOUND_SPEECH})}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="PACK MIND keyboard driver") + parser.add_argument("--url", default=None, help="MCP URL (default localhost:9990/mcp)") + parser.add_argument("--step", type=float, default=0.5, help="forward speed (m/s)") + parser.add_argument("--turn", type=float, default=0.8, help="turn speed (rad/s)") + parser.add_argument( + "--no-auto-look", action="store_true", help="don't auto look_for_red after each move" + ) + parser.add_argument( + "--timeout", + type=int, + default=8, + help="per-call seconds. Short on purpose: the dog physically moves in a few " + "seconds, but relative_move then waits on a flaky LCM RPC (is_goal_reached) " + "that can hang for 120s — we don't block on it, the dog already moved.", + ) + args = parser.parse_args() + + adapter = McpAdapter(args.url, timeout=args.timeout) + step, turn = args.step, args.turn + + print("PACK MIND driver — w/s move · a/d turn · space=look · f=speak · +/- step · x=quit") + print(f"connected to {adapter.url} (step={step}m turn={turn}°)\n") + + # Velocity teleop (drive) — instant, bypasses the planner + flaky is_goal_reached + # RPC. step = m/s forward burst; turn = rad/s; each press moves for `burst` seconds. + burst = 0.6 + moves = { + "w": lambda: ("drive", {"forward": step, "turn": 0.0, "duration": burst}), + "s": lambda: ("drive", {"forward": -step, "turn": 0.0, "duration": burst}), + "a": lambda: ("drive", {"forward": 0.0, "turn": turn, "duration": burst}), + "d": lambda: ("drive", {"forward": 0.0, "turn": -turn, "duration": burst}), + } + + while True: + key = _getch().lower() + if key in ("x", "\x03", "\x04", "q"): # x, Ctrl-C, Ctrl-D, q + print("\nbye") + return + if key in moves: + name, margs = moves[key]() + arrow = {"w": "↑", "s": "↓", "a": "↰", "d": "↱"}[key] + print(f"{arrow} {_call(adapter, name, margs)}") + if not args.no_auto_look: + _look_and_announce(adapter) + elif key == " ": + _look_and_announce(adapter) + elif key == "f": + print(f" 📣 {_call(adapter, 'speak', {'text': _FOUND_SPEECH})}") + elif key == "+": + step += 0.2 + print(f" step = {step:.1f}m") + elif key == "-": + step = max(0.2, step - 0.2) + print(f" step = {step:.1f}m") + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/demo_drive_go2.py b/dimos/experimental/pack_mind/demo_drive_go2.py new file mode 100644 index 0000000000..acb09c54e9 --- /dev/null +++ b/dimos/experimental/pack_mind/demo_drive_go2.py @@ -0,0 +1,80 @@ +# Copyright 2025 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Headless cmd_vel choreography for a live Go2 — no GUI / pygame required. + +The keyboard teleop blueprint needs an X11/Cocoa window for pygame, which is +unavailable over SSH or in a headless session. This script drives the dog by +publishing Twist messages straight onto the /cmd_vel LCM topic at a fixed rate, +exactly like KeyboardTeleop does — the running blueprint's ControlCoordinator +relays them to the robot over WebRTC. + +Run a relay blueprint in one terminal (it holds the WebRTC connection and the +ControlCoordinator; the pygame crash in its log is harmless): + + uv run dimos --robot-ip 192.168.12.1 run unitree-go2-webrtc-keyboard-teleop + +Then, once it logs "ControlCoordinator started", in a second terminal on the +SAME machine (LCM is host-local multicast): + + uv run python dimos/experimental/pack_mind/demo_drive_go2.py +""" + +from __future__ import annotations + +import time + +from dimos.core.transport import LCMTransport +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.geometry_msgs.Vector3 import Vector3 + +CMD_VEL_TOPIC = "/cmd_vel" +DEFAULT_RATE_HZ = 10.0 + + +def drive(transport: LCMTransport[Twist], vx: float, wz: float, secs: float, hz: float = DEFAULT_RATE_HZ) -> None: + """Hold a velocity for `secs` by republishing at `hz`. + + vx: forward m/s (positive forward, negative back). + wz: yaw rad/s (positive left, negative right). + Republishing is required: the connection layer auto-stops 0.2s after the + last command, so a single publish only produces a brief nudge. + """ + deadline = time.time() + secs + period = 1.0 / hz + while time.time() < deadline: + transport.broadcast( + None, + Twist( + linear=Vector3(x=vx, y=0.0, z=0.0), + angular=Vector3(x=0.0, y=0.0, z=wz), + ), + ) + time.sleep(period) + + +def main() -> None: + transport: LCMTransport[Twist] = LCMTransport(CMD_VEL_TOPIC, Twist) + + # Choreographed routine. Tune speeds/durations/order for your space. + # Start conservative: clear ~2m, keep the e-stop within reach. + drive(transport, vx=0.0, wz=0.0, secs=0.5) # settle + drive(transport, vx=0.3, wz=0.0, secs=2.0) # forward + drive(transport, vx=0.0, wz=0.5, secs=2.0) # turn left in place + drive(transport, vx=0.3, wz=0.0, secs=2.0) # forward + drive(transport, vx=0.0, wz=0.0, secs=0.5) # stop + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/demo_pack_live.py b/dimos/experimental/pack_mind/demo_pack_live.py new file mode 100644 index 0000000000..0d333001e0 --- /dev/null +++ b/dimos/experimental/pack_mind/demo_pack_live.py @@ -0,0 +1,113 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — paced LIVE narrative for the projector dashboard. + +One command: starts the coordinator + dashboard AND plays the full v4 story with +pauses, so you just open the browser and watch the shared memory animate. Drives +the real coordinator (the dashboard polls the same state), deterministically — no +hardware, no racing mock dogs. + + uv run python -m dimos.experimental.pack_mind.demo_pack_live --pace 2.0 + # then open http://localhost:8090 on the projector + +The story it plays (the inheritance climax — two dogs, no overlap, a dog drops, +the survivor inherits and finishes): + start_search → Alpha claims the target zone → Bravo clears the others (no + overlap) → Alpha goes OFFLINE (its zone reclaimed) → Bravo inherits it and finds + the object there. +""" + +from __future__ import annotations + +import argparse +import threading +import time + +from dimos.experimental.pack_mind.pack_coordinator_server import ( + DEFAULT_COORDINATOR_PORT, + make_server, +) + + +def _narrate(tag: str, msg: str) -> None: + print(f" {tag:<26} {msg}", flush=True) + + +def play(target: str, target_zone: str, zones: list[str], port: int, host: str, pace: float) -> None: + others = [z for z in zones if z != target_zone] + # Alpha prefers the target zone (so it holds it unfinished); Bravo prefers the rest. + prefs = {"alpha": [target_zone, *others], "bravo": [*others, target_zone]} + server = make_server(host, port, zones, prefs) + actual = server.server_address[1] + threading.Thread(target=server.serve_forever, daemon=True).start() + c = server.coordinator + + print(f"\nPACK MIND live demo on http://{host}:{actual}") + print(f" → open that in the projector browser now. Starting in {pace * 2:.0f}s…\n") + time.sleep(pace * 2) + + print("=== mission: find the red kit (two dogs, one shared memory) ===") + _narrate("MISSION", c.start_search(target)) + time.sleep(pace) + + _narrate("ALPHA CLAIMS", c.assign_zone("alpha") or "-") # the target zone, unfinished + time.sleep(pace) + + while True: + z = c.assign_zone("bravo") + if not z: + break + time.sleep(pace) + c.report_cleared("bravo", z) + _narrate("BRAVO CLEARED", f"{z} (no overlap — never alpha's ground)") + time.sleep(pace * 0.5) + + print("\n=== the real test: lose a dog mid-mission ===") + time.sleep(pace) + _narrate("ALPHA OFFLINE", c.release_dog("alpha")) + time.sleep(pace) + + inherited = c.assign_zone("bravo") + _narrate("BRAVO INHERITS", f"{inherited} — alpha started this zone, never finished") + time.sleep(pace) + f = c.report_finding("bravo", target, inherited or "") + _narrate("BRAVO FINDS IT", f"{f.object} in {f.zone} — mission complete") + + print("\nAlpha never finished. Bravo inherited its ground and found it. — PACK MIND") + print("(dashboard stays live; Ctrl-C to exit)\n") + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + server.shutdown() + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND paced live dashboard demo") + p.add_argument("--target", default="red kit") + p.add_argument("--target-zone", default="south", help="where the object is") + p.add_argument("--zones", default="north,east,south,west") + p.add_argument("--port", type=int, default=DEFAULT_COORDINATOR_PORT) + p.add_argument("--host", default="0.0.0.0") + p.add_argument("--pace", type=float, default=2.0, help="seconds between beats") + return p.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + zones = [z.strip() for z in args.zones.split(",") if z.strip()] + if args.target_zone not in zones: + raise SystemExit(f"--target-zone {args.target_zone!r} must be one of {zones}") + play(args.target, args.target_zone, zones, args.port, args.host, args.pace) diff --git a/dimos/experimental/pack_mind/demo_pack_scene.py b/dimos/experimental/pack_mind/demo_pack_scene.py new file mode 100644 index 0000000000..ebc9114f34 --- /dev/null +++ b/dimos/experimental/pack_mind/demo_pack_scene.py @@ -0,0 +1,77 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — hardware-free dry-run of the v4 demo's two magic beats. + +Prints the dashboard-style causal chain the audience will see, driven entirely by +the real coordinator + runner (only the robot bodies are mocked). Use it to +rehearse narration and to smoke-test the brain without dogs. + + uv run python -m dimos.experimental.pack_mind.demo_pack_scene + +`demo_` prefix keeps it out of pytest collection (it's a manual scene, not a test). +""" + +from __future__ import annotations + +from dimos.experimental.pack_mind.pack_coordinator import PackCoordinator +from dimos.experimental.pack_mind.pack_search_runner import MockDriver, act_on_finding, run_search + + +def _line(tag: str, msg: str) -> None: + print(f" {tag:<28} {msg}") + + +def beat_1_handoff() -> None: + print("\n=== BEAT 1 — THE HANDOFF (a body acts on memory it never gathered) ===") + c = PackCoordinator(["north", "east", "south", "west"]) + print(" ", c.start_search("red kit")) + res = run_search(c, MockDriver(target_zone="east"), "alpha", "red kit") + _line("ALPHA SEARCHED", f"{res.visited} → FOUND red kit in {res.zone}") + _line("PACK MEMORY UPDATED", f"finding by alpha @ {res.zone}") + bravo = MockDriver(target_zone="east") + print(" ", act_on_finding(c, bravo, "bravo")) + _line("BRAVO ACTED ON ALPHA'S MEMORY", f"went to {bravo.visited[-1]} — never searched it itself") + + +def beat_2_inheritance() -> None: + print("\n=== BEAT 2 — INHERITANCE (the mission doesn't die with the robot) ===") + prefs = {"alpha": ["south"], "bravo": ["north", "east", "west"]} + c = PackCoordinator(["north", "east", "south", "west"], preferences=prefs) + print(" ", c.start_search("red kit")) + bravo = MockDriver(target_zone="south") # the kit is in alpha's zone + _line("ALPHA CLAIMS", c.assign_zone("alpha") or "-") # "south", unfinished + for _ in range(3): + z = c.assign_zone("bravo") + if not z: + break + bravo.goto(z) + c.report_cleared("bravo", z) + _line("BRAVO CLEARED", z) + _line("ALPHA GOES OFFLINE", c.release_dog("alpha")) + res = run_search(c, bravo, "bravo", "red kit") + _line("BRAVO INHERITED + FINISHED", f"found red kit in {res.zone} (alpha started this zone)") + snap = c.snapshot() + _line("SHARED MEMORY", f"offline={snap['offline']} found={snap['found']} coverage={snap['coverage']}") + + +def main() -> None: + print("PACK MIND — shared operational memory (hardware-free scene)") + beat_1_handoff() + beat_2_inheritance() + print("\nAlpha saw it. Bravo remembered it. The pack acted. — PACK MIND\n") + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/demo_spike.py b/dimos/experimental/pack_mind/demo_spike.py new file mode 100644 index 0000000000..2957526857 --- /dev/null +++ b/dimos/experimental/pack_mind/demo_spike.py @@ -0,0 +1,110 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND de-risk spike (PLAN §6). + +Answers the only question that decides feasibility of the video-only sim: + + Can we drive FrontierPatrolRouter + min_cost_astar on a hand-fabricated + OccupancyGrid in a plain loop, no robot stack, no CUDA, no ROS? + +Plus it proves the A/B knob: two routers sharing ONE VisitationHistory vs two +private ones. Run: + + uv run python dimos/experimental/pack_mind/demo_spike.py +""" + +from __future__ import annotations + +import numpy as np + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid +from dimos.navigation.patrolling.routers.frontier_patrol_router import FrontierPatrolRouter +from dimos.navigation.patrolling.routers.visitation_history import VisitationHistory +from dimos.navigation.replanning_a_star.min_cost_astar import min_cost_astar + +RES = 0.1 # m/cell +CLEARANCE_M = 0.3 + + +def make_world(w: int = 40, h: int = 40) -> OccupancyGrid: + """Free interior, OCCUPIED border + one interior wall. Fully known (no fog).""" + grid = np.full((h, w), CostValues.FREE, dtype=np.int8) + grid[0, :] = grid[-1, :] = CostValues.OCCUPIED + grid[:, 0] = grid[:, -1] = CostValues.OCCUPIED + grid[10:30, 20] = CostValues.OCCUPIED # vertical divider with a gap below row 30 + return OccupancyGrid(grid=grid, resolution=RES) + + +def pose_at(x: float, y: float) -> PoseStamped: + p = PoseStamped() + p.position.x = x + p.position.y = y + return p + + +def main() -> None: + world = make_world() + print(f"[spike] world: {world.width}x{world.height} @ {RES}m ({world})") + + # --- 1. router accepts a fabricated grid + odom, returns a goal --- + r = FrontierPatrolRouter(CLEARANCE_M) + r.handle_occupancy_grid(world) + r.handle_odom(pose_at(0.5, 0.5)) + goal = r.next_goal() + assert goal is not None, "FAIL: next_goal returned None on a fresh map" + print(f"[spike] PASS next_goal -> ({goal.position.x:.2f}, {goal.position.y:.2f})") + + # --- 2. A* plans start->goal with python fallback (use_cpp=False) --- + path = min_cost_astar( + world, (goal.position.x, goal.position.y), (0.5, 0.5), use_cpp=False + ) + assert path is not None and len(path.poses) > 0, "FAIL: A* found no path" + print(f"[spike] PASS min_cost_astar -> {len(path.poses)} waypoints") + + # --- 3. THE A/B KNOB: shared vs private VisitationHistory --- + # Private: two routers, independent visited masks. + a_priv, b_priv = FrontierPatrolRouter(CLEARANCE_M), FrontierPatrolRouter(CLEARANCE_M) + for rr in (a_priv, b_priv): + rr.handle_occupancy_grid(world) + a_priv.handle_odom(pose_at(0.5, 0.5)) + # A walks a bit; B should NOT see A's coverage. + for x in np.arange(0.5, 2.5, 0.1): + a_priv.handle_odom(pose_at(float(x), 0.5)) + b_sees_via_private = b_priv._visited.sum() if b_priv._visited is not None else 0 + + # Shared: B reuses A's VisitationHistory instance. + a_sh, b_sh = FrontierPatrolRouter(CLEARANCE_M), FrontierPatrolRouter(CLEARANCE_M) + shared = VisitationHistory(CLEARANCE_M) + a_sh._visitation = b_sh._visitation = shared + for rr in (a_sh, b_sh): + rr.handle_occupancy_grid(world) + for x in np.arange(0.5, 2.5, 0.1): + a_sh.handle_odom(pose_at(float(x), 0.5)) + b_sees_via_shared = b_sh._visited.sum() if b_sh._visited is not None else 0 + + print( + f"[spike] A/B knob: B sees {b_sees_via_private} visited cells (private) " + f"vs {b_sees_via_shared} (shared)" + ) + assert b_sees_via_private == 0, "FAIL: private leaked coverage" + assert b_sees_via_shared > 0, "FAIL: shared did not propagate A's coverage to B" + print("[spike] PASS shared VisitationHistory propagates; private does not") + + print("\n[spike] ALL GREEN — PLAN is buildable as written.") + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/docs/pack_mind_partition_vs_memory.png b/dimos/experimental/pack_mind/docs/pack_mind_partition_vs_memory.png new file mode 100644 index 0000000000..39b7068207 --- /dev/null +++ b/dimos/experimental/pack_mind/docs/pack_mind_partition_vs_memory.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62eca7ed015dd980a5ca2a42b8d7cb759ddd97346faef23b7e75d79cc31764db +size 293938 diff --git a/dimos/experimental/pack_mind/docs/packmind_how_it_works.png b/dimos/experimental/pack_mind/docs/packmind_how_it_works.png new file mode 100644 index 0000000000..037fd372ed --- /dev/null +++ b/dimos/experimental/pack_mind/docs/packmind_how_it_works.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa626a360469a02aea9cd0f28fb969d87a91cdba9acb70b10b2d5c2e0da9945 +size 570774 diff --git a/dimos/experimental/pack_mind/docs/packmind_system_design.png b/dimos/experimental/pack_mind/docs/packmind_system_design.png new file mode 100644 index 0000000000..846fc3f58c --- /dev/null +++ b/dimos/experimental/pack_mind/docs/packmind_system_design.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc3486c492fb6aa6934ba64a24cec02655a559d8ad120c50ca904154d90bb7c9 +size 930901 diff --git a/dimos/experimental/pack_mind/explore_sim.py b/dimos/experimental/pack_mind/explore_sim.py new file mode 100644 index 0000000000..06d8d7e708 --- /dev/null +++ b/dimos/experimental/pack_mind/explore_sim.py @@ -0,0 +1,405 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — fog-of-war exploration engine (S1). + +A team of dogs explores an UNKNOWN maze. Each dog reveals cells via a raycast +sensor (walls block line of sight, RTS-style). Dogs navigate to frontiers (the +boundary between known-free and unknown). + +The A/B variable is the discovered map: + - SHARED : one known-map; every dog reveals into and plans against it. + - INDEPENDENT: each dog has its own known-map; the *team* fog is the union of + ONLINE dogs' maps — so taking a dog offline erases its territory from the + team's knowledge. In shared mode the map persists regardless. + +Pure numpy/scipy + DimOS min_cost_astar. No CUDA/ROS/sim deps. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from math import cos, exp, hypot, sin + +import numpy as np +from numpy.typing import NDArray +from scipy.ndimage import binary_dilation, label + +from dimos.experimental.pack_mind.world import ( + free_cells_world, + load_building_world, + make_maze_world, + spread_starts, +) +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid +from dimos.navigation.replanning_a_star.min_cost_astar import min_cost_astar + +UNKNOWN = int(CostValues.UNKNOWN) +FREE = int(CostValues.FREE) +OCC = int(CostValues.OCCUPIED) + +SENSOR_R_CELLS = 11 # vision radius in cells +N_RAYS = 180 +STEP_M = 0.12 +_CROSS = np.ones((3, 3), dtype=bool) +_MIN_CLUSTER = 4 +_DOG_COLORS = ["#ff6b35", "#4dabf7", "#b197fc", "#69db7c", "#ffd43b"] + + +def _raycast_reveal( + known: NDArray[np.int8], + truth: NDArray[np.int8], + cx: int, + cy: int, + visible: NDArray[np.bool_] | None = None, +) -> None: + """Reveal cells within SENSOR_R_CELLS of (cx, cy); walls stop each ray. + + Writes discovered cells into ``known`` (persistent memory) and, if given, + marks currently-seen cells in ``visible`` (this-tick line of sight).""" + h, w = truth.shape + + def _see(gx: int, gy: int, val: int) -> None: + known[gy, gx] = val + if visible is not None: + visible[gy, gx] = True + + if 0 <= cy < h and 0 <= cx < w: + _see(cx, cy, OCC if truth[cy, cx] == OCC else FREE) + for k in range(N_RAYS): + a = 2.0 * np.pi * k / N_RAYS + dx, dy = cos(a), sin(a) + for r in range(1, SENSOR_R_CELLS + 1): + gx = int(round(cx + dx * r)) + gy = int(round(cy + dy * r)) + if not (0 <= gx < w and 0 <= gy < h): + break + if truth[gy, gx] == OCC: + _see(gx, gy, OCC) # see the wall, then vision stops + break + _see(gx, gy, FREE) + + +@dataclass +class Dog: + name: str + color: str + x: float + y: float + known: NDArray[np.int8] # shared array (pack) or private (independent) + online: bool = True + idle: bool = False + path: list[tuple[float, float]] | None = None + idx: int = 0 + goal: tuple[float, float] | None = None # current frontier target (for pack de-confliction) + trail: list[tuple[float, float]] = field(default_factory=list) + + +class ExploreSim: + def __init__( + self, + world: OccupancyGrid, + n_dogs: int, + shared: bool, + starts: list[tuple[float, float]], + seed: int = 0, + target: tuple[float, float] | None = None, + target_label: str = "target", + converge_on_found: bool = False, + ) -> None: + np.random.seed(seed) + self.world = world + self.truth = world.grid + self.shared = shared + self.res = world.info.resolution + self.tick_n = 0 + self._total_free = int(np.count_nonzero(self.truth == FREE)) + + # Search target ("find me a red object"). Detected when its cell enters any + # online dog's line of sight; the find is written to shared memory (found/ + # found_by/found_tick) so the whole pack knows at once. + self.target_xy = target + self.target_label = target_label + self.converge_on_found = converge_on_found + self.found = False + self.found_by: str | None = None + self.found_tick: int | None = None + + shape = self.truth.shape + self._visible = np.zeros(shape, dtype=bool) + if shared: + self._shared_known = np.full(shape, UNKNOWN, dtype=np.int8) + self.dogs: list[Dog] = [] + for i in range(n_dogs): + known = self._shared_known if shared else np.full(shape, UNKNOWN, dtype=np.int8) + self.dogs.append( + Dog(f"dog{i}", _DOG_COLORS[i % len(_DOG_COLORS)], starts[i][0], starts[i][1], known) + ) + + # ---- geometry helpers ---- + def _grid(self, x: float, y: float) -> tuple[int, int]: + g = self.world.world_to_grid((x, y)) + return int(g.x), int(g.y) + + # ---- fog / metrics ---- + def team_known(self) -> NDArray[np.int8]: + """What the operator sees. Shared: the one map. Independent: union over + ONLINE dogs (offline dogs' knowledge is lost to the team).""" + if self.shared: + return self._shared_known + merged = np.full(self.truth.shape, UNKNOWN, dtype=np.int8) + for d in self.dogs: + if not d.online: + continue + seen = d.known != UNKNOWN + merged[seen] = d.known[seen] + return merged + + def revealed_frac(self) -> float: + k = self.team_known() + return float(np.count_nonzero((k == FREE) & (self.truth == FREE))) / max(1, self._total_free) + + def state(self) -> dict: + """JSON-serializable snapshot for the web frontend. Per-cell code: + 0=unknown, 1=remembered-free, 2=visible-free, 3=remembered-wall, 4=visible-wall.""" + k = self.team_known() + vis = self._visible + codes = np.zeros(k.shape, dtype=np.uint8) + free, wall = k == FREE, k == OCC + codes[free] = 1 + codes[free & vis] = 2 + codes[wall] = 3 + codes[wall & vis] = 4 + h, w = k.shape + return { + "mode": "shared" if self.shared else "independent", + "tick": self.tick_n, + "revealed": round(self.revealed_frac(), 3), + "w": w, + "h": h, + "res": self.res, + "cells": codes.flatten().tolist(), + "dogs": [ + {"x": round(d.x, 3), "y": round(d.y, 3), "color": d.color, "online": d.online} + for d in self.dogs + ], + "target": ( + { + "x": round(self.target_xy[0], 3), + "y": round(self.target_xy[1], 3), + "label": self.target_label, + "found": self.found, + "found_by": self.found_by, + "found_tick": self.found_tick, + } + if self.target_xy is not None + else None + ), + } + + # ---- exploration ---- + def _pick_goal(self, dog: Dog) -> tuple[float, float] | None: + # Object found: the pack already knows (shared memory) — converge on it + # instead of continuing to clear frontiers. "One sees it, all know." + if self.found and self.converge_on_found and self.target_xy is not None: + return self.target_xy + known = dog.known + free = known == FREE + unknown = known == UNKNOWN + frontier = free & binary_dilation(unknown, structure=_CROSS) + if not frontier.any(): + return None + lab, n = label(frontier) + if n == 0: + return None + gx, gy = self._grid(dog.x, dog.y) + # Frontiers other online dogs are already heading for. The shared map lets + # the pack de-conflict: a cluster near a teammate's target is down-weighted + # so the dogs fan out instead of all chasing the same boundary. + claimed = [ + self._grid(*o.goal) + for o in self.dogs + if o is not dog and o.online and o.goal is not None + ] + rows, cols = np.where(frontier) + ids = lab[rows, cols] + best_id, best_score = -1, -1.0 + for cid in range(1, n + 1): + m = ids == cid + size = int(np.count_nonzero(m)) + if size < _MIN_CLUSTER: + continue + cr, cc = rows[m].mean(), cols[m].mean() + dist = max(1.0, hypot(cc - gx, cr - gy)) + score = size / dist + if claimed: + d_other = min(hypot(cc - ox, cr - oy) for ox, oy in claimed) + score *= 1.0 - 0.85 * exp(-(d_other**2) / (2.0 * SENSOR_R_CELLS**2)) + if score > best_score: + best_score, best_id = score, cid + if best_id < 0: + return None + m = ids == best_id + cr, cc = rows[m], cols[m] + d2 = (cc - gx) ** 2 + (cr - gy) ** 2 + j = int(np.argmin(d2)) # nearest frontier cell in best cluster + wp = self.world.grid_to_world((int(cc[j]), int(cr[j]), 0)) + return (wp.x, wp.y) + + def _plan(self, dog: Dog, goal: tuple[float, float]) -> bool: + costmap = OccupancyGrid(grid=dog.known, resolution=self.res, frame_id="world") + path = min_cost_astar(costmap, goal, (dog.x, dog.y), unknown_penalty=1.0, use_cpp=False) + if path is None or len(path.poses) == 0: + return False + dog.path = [(p.position.x, p.position.y) for p in path.poses] + dog.idx = 0 + return True + + def _advance(self, dog: Dog) -> None: + if dog.path is None or dog.idx >= len(dog.path): + goal = self._pick_goal(dog) + if goal is None or not self._plan(dog, goal): + dog.idle = True + dog.goal = None + return + dog.goal = goal + dog.idle = False + assert dog.path is not None + tx, ty = dog.path[dog.idx] + dx, dy = tx - dog.x, ty - dog.y + dist = hypot(dx, dy) + if dist <= STEP_M or dist == 0: + dog.x, dog.y = tx, ty + dog.idx += 1 + else: + dog.x += STEP_M * dx / dist + dog.y += STEP_M * dy / dist + + def set_online(self, dog_index: int, online: bool) -> None: + self.dogs[dog_index].online = online + + def step(self) -> None: + self.tick_n += 1 + self._visible = np.zeros(self.truth.shape, dtype=bool) + for d in self.dogs: + if d.online: + gx, gy = self._grid(d.x, d.y) + _raycast_reveal(d.known, self.truth, gx, gy, self._visible) + self._check_target() + for d in self.dogs: + if d.online: + self._advance(d) + d.trail.append((d.x, d.y)) + + def _check_target(self) -> None: + """Mark the target found the tick its cell first enters any dog's sight.""" + if self.target_xy is None or self.found: + return + tgx, tgy = self._grid(*self.target_xy) + h, w = self._visible.shape + if not (0 <= tgy < h and 0 <= tgx < w) or not self._visible[tgy, tgx]: + return + self.found = True + self.found_tick = self.tick_n + online = [d for d in self.dogs if d.online] + if online: + tx, ty = self.target_xy + self.found_by = min(online, key=lambda d: (d.x - tx) ** 2 + (d.y - ty) ** 2).name + + def all_done(self) -> bool: + return all((not d.online) or d.idle for d in self.dogs) + + def run(self, max_ticks: int = 4000, target: float = 0.95, kill: tuple[int, int] | None = None): + t_target: int | None = None + for _ in range(max_ticks): + if kill is not None and self.tick_n == kill[1]: + self.set_online(kill[0], False) + self.step() + if t_target is None and self.revealed_frac() >= target: + t_target = self.tick_n + if self.all_done(): + break + return { + "mode": "shared" if self.shared else "independent", + "ticks": self.tick_n, + "revealed": self.revealed_frac(), + "ticks_to_target": t_target, + } + + +_START_POOL = [(0.6, 0.5), (3.4, 0.5), (5.4, 0.5), (0.6, 5.4), (5.4, 5.4)] + + +def _far_target(world: OccupancyGrid, starts: list[tuple[float, float]]) -> tuple[float, float]: + """Plant the object at the free cell farthest from every start, so it's only + found after real searching — not next to where a dog spawns.""" + cells = free_cells_world(world) + return max(cells, key=lambda c: min((c[0] - s[0]) ** 2 + (c[1] - s[1]) ** 2 for s in starts)) + + +def build_explore( + shared: bool, + seed: int = 0, + n_dogs: int = 3, + target_label: str | None = None, + converge_on_found: bool = False, +) -> ExploreSim: + world = make_maze_world() + # Spread deployment (different entry points, realistic multi-dog SAR). Both + # modes use the SAME starts — only shared-vs-private memory differs. Spreading + # gives each dog distinct territory, so in independent mode losing a dog loses + # its discoveries (beat 4), while shared retains them. + starts = [_START_POOL[i % len(_START_POOL)] for i in range(n_dogs)] + target = _far_target(world, starts) if target_label else None + return ExploreSim( + world, n_dogs, shared, starts, seed, + target=target, target_label=target_label or "target", + converge_on_found=converge_on_found, + ) + + +def build_explore_building( + shared: bool, + seed: int = 0, + n_dogs: int = 3, + downsample: int = 4, + target_label: str | None = None, + converge_on_found: bool = False, +) -> ExploreSim: + """ExploreSim on a real DimOS SLAM floor plan instead of the synthetic maze. + + A 27x37m office footprint (big_office_simple_occupancy.png) with long wings + and corridors. Starts are spread deterministically from the seed, so the + shared and independent runs share identical deployment — only the memory + model differs. The bigger, concave floor is where the shared-vs-independent + gap blows open (shared clears 100%, independent stalls redundantly).""" + world = load_building_world(downsample=downsample) + starts = spread_starts(world, n_dogs, seed=seed, min_dist_m=5.0) + target = _far_target(world, starts) if target_label else None + return ExploreSim( + world, n_dogs, shared, starts, seed, + target=target, target_label=target_label or "target", + converge_on_found=converge_on_found, + ) + + +if __name__ == "__main__": + import time + + builder = build_explore_building if os.environ.get("PACK_MIND_MAP") == "building" else build_explore + for sh in (True, False): + t0 = time.perf_counter() + r = builder(shared=sh).run() + print(r, "wall=%.1fs" % (time.perf_counter() - t0)) diff --git a/dimos/experimental/pack_mind/export_occupancy.py b/dimos/experimental/pack_mind/export_occupancy.py new file mode 100644 index 0000000000..6f43dadce5 --- /dev/null +++ b/dimos/experimental/pack_mind/export_occupancy.py @@ -0,0 +1,50 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Export the PACK MIND maze as a .npy occupancy grid so MuJoCo can build the room. + + uv run python -m dimos.experimental.pack_mind.export_occupancy --out pack_mind_maze.npy + # then drive a Go2 through the photoreal 3D version of our maze: + dimos --simulation run unitree-go2 --mujoco-room-from-occupancy pack_mind_maze.npy + +`OccupancyGrid.from_path` loads .npy via np.load; `generate_mujoco_scene` +(dimos/mapping/occupancy/extrude_occupancy.py) extrudes occupied cells into 3D walls. +Our grid uses FREE=0 / OCCUPIED=100 / UNKNOWN=-1 (standard nav_msgs convention). +""" + +from __future__ import annotations + +import argparse + +import numpy as np + +from dimos.experimental.pack_mind.world import make_maze_world +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues + + +def main() -> None: + p = argparse.ArgumentParser(description="Export PACK MIND maze to .npy for MuJoCo") + p.add_argument("--out", default="pack_mind_maze.npy") + args = p.parse_args() + + grid = make_maze_world().grid + np.save(args.out, grid) + walls = int(np.count_nonzero(grid == CostValues.OCCUPIED)) + free = int(np.count_nonzero(grid == CostValues.FREE)) + print(f"saved {grid.shape} occupancy -> {args.out} (walls={walls}, free={free})") + print("next: dimos --simulation run unitree-go2 --mujoco-room-from-occupancy " + args.out) + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/live.py b/dimos/experimental/pack_mind/live.py new file mode 100644 index 0000000000..aad52bee90 --- /dev/null +++ b/dimos/experimental/pack_mind/live.py @@ -0,0 +1,92 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — live Go2 blueprint (one dog per laptop, shared memory over HTTP). + +Each laptop runs THIS blueprint against its own dog; the two processes share one +``PackCoordinator`` over the LAN. There is no in-process multi-instance problem +because the two dogs are two separate `dimos run` processes — they only meet at +the coordinator. Configure per laptop with env vars:: + + # Laptop A (also runs the coordinator): + python -m dimos.experimental.pack_mind.pack_coordinator_server --zones north,east,south,west & + PACK_DOG_NAME=alpha PACK_COORDINATOR_URL=http://127.0.0.1:8090 \\ + dimos run unitree-go2-pack --robot-ip --listen-host 0.0.0.0 + + # Laptop B: + PACK_DOG_NAME=bravo PACK_COORDINATOR_URL=http://:8090 \\ + dimos run unitree-go2-pack --robot-ip --listen-host 0.0.0.0 + +The detection→coordinator bridge needs NO custom module: the agent calls +``look_out_for([target], then={"name": "report_finding", "args": {...}})`` so a VLM +sighting auto-reports the finding to the pack (perceive_loop_skill dispatches the +``then`` continuation). + +NOTE: untested without hardware/GPU — bring up per the H0 gate in the runbook. +""" + +from __future__ import annotations + +from dimos.agents.mcp.mcp_client import McpClient +from dimos.agents.mcp.mcp_server import McpServer +from dimos.agents.skills.person_follow import PersonFollowSkillContainer +from dimos.core.coordination.blueprints import autoconnect +from dimos.experimental.pack_mind.pack_search_skills import PackSearchSkills +from dimos.experimental.pack_mind.red_detector import RedObjectDetector +from dimos.experimental.pack_mind.velocity_teleop import VelocityTeleop +from dimos.experimental.security_demo.security_module import SecurityModule +from dimos.robot.unitree.go2.blueprints.agentic._common_agentic import _common_agentic +from dimos.robot.unitree.go2.blueprints.smart.unitree_go2_spatial import unitree_go2_spatial + +PACK_SYSTEM_PROMPT = """\ +You are one dog in a PACK that shares a single operational memory through tools. +You never command another dog and never exchange coordinates — you read and write +the shared memory by ZONE NAME only. + +MISSION LOOP (follow exactly): +1. When given a target (e.g. "find the red kit"), call start_search(target) ONCE. +2. Repeat: + a. Call next_zone to get YOUR assigned zone. If it returns empty, the search is + over — STOP and report status. + b. navigate_with_text("") to travel there. + c. Call look_for_red to check the camera for the red object. If it reports one, + it has already told the whole pack — STOP. + d. If no red object, call report_cleared("") and continue. + e. Check should_stop — if true, a packmate found it; STOP. +3. If asked where the target is, call where_is(target) and act on the answer even + if you never saw it yourself — that is acting on the pack's shared memory. + +Keep moving. Be concise. Zones by name, never coordinates.""" + +# One dog per laptop. PackSearchSkills reads PACK_DOG_NAME + PACK_COORDINATOR_URL +# from the environment, so this single blueprint serves both laptops. +# +# EdgeTAM (segmentation) HARD-requires a CUDA GPU. It reaches the spatial stack two +# ways, both fatal on a CPU/CoreML host (e.g. a Mac ground station): +# 1. unitree_go2_spatial -> SecurityModule.__init__ -> EdgeTAMProcessor() → deploy crash +# 2. PersonFollowSkillContainer.follow_person -> EdgeTAMProcessor() → mid-demo crash +# The pack demo uses neither (we move via skills + look_out_for, not security +# patrol or person-follow), so disable both. The blueprint then deploys clean on +# CPU — no --disable flag and no call-time landmine. +unitree_go2_pack = autoconnect( + unitree_go2_spatial, + McpServer.blueprint(), + McpClient.blueprint(system_prompt=PACK_SYSTEM_PROMPT), + _common_agentic, + PackSearchSkills.blueprint(), + RedObjectDetector.blueprint(), # fast GPU-free "red object" find (vs slow moondream) + VelocityTeleop.blueprint(), # direct cmd_vel teleop (vs planner-RPC relative_move) +).disabled_modules(SecurityModule, PersonFollowSkillContainer) + +__all__ = ["unitree_go2_pack", "PACK_SYSTEM_PROMPT"] diff --git a/dimos/experimental/pack_mind/mock_dog.py b/dimos/experimental/pack_mind/mock_dog.py new file mode 100644 index 0000000000..04a91b1686 --- /dev/null +++ b/dimos/experimental/pack_mind/mock_dog.py @@ -0,0 +1,150 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND mock dog — a hardware-free client that drives one dog's search loop. + +No DimOS Module, no robot, just ``requests`` against a running coordinator. Run +the server plus two of these (one with ``--find-prob 1.0``) to demonstrate the +full no-overlap + stop-on-find story with zero robots:: + + uv run python dimos/experimental/pack_mind/pack_coordinator_server.py --port 8090 & + uv run python dimos/experimental/pack_mind/mock_dog.py --dog alpha --url http://localhost:8090 + uv run python dimos/experimental/pack_mind/mock_dog.py --dog bravo --url http://localhost:8090 --find-prob 1.0 + +Each dog loops: ask for a zone; stop if the pack already found the target; with +probability ``--find-prob`` report a finding (and stop), otherwise report the +zone cleared; repeat until no zone is left. +""" + +from __future__ import annotations + +import argparse +import random +import time +from typing import Any + +import requests + +DEFAULT_TARGET = "red object" +_REQUEST_TIMEOUT = 5.0 +_MIN_SLEEP = 0.05 +_MAX_SLEEP = 0.25 + + +def _post(url: str, path: str, payload: dict[str, Any]) -> dict[str, Any] | None: + try: + resp = requests.post(f"{url}{path}", json=payload, timeout=_REQUEST_TIMEOUT) + resp.raise_for_status() + data: Any = resp.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + print(f"[{path}] request failed: {exc}") + return None + + +def _get(url: str, path: str, params: dict[str, str]) -> dict[str, Any] | None: + try: + resp = requests.get(f"{url}{path}", params=params, timeout=_REQUEST_TIMEOUT) + resp.raise_for_status() + data: Any = resp.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + print(f"[{path}] request failed: {exc}") + return None + + +def run_dog( + dog: str, + url: str, + target: str, + find_prob: float, + rng: random.Random, + target_zone: str | None = None, + dwell: float | None = None, +) -> None: + url = url.rstrip("/") + while True: + stop = _get(url, "/should_stop", {"dog": dog}) + if stop is not None and stop.get("stop"): + print(f"[{dog}] pack already found the target — converging, stopping.") + return + + assigned = _post(url, "/assign_zone", {"dog": dog}) + zone = assigned.get("zone") if assigned else None + if not zone: + print(f"[{dog}] no zone assigned (search over or exhausted) — stopping.") + return + print(f"[{dog}] assigned zone: {zone}") + + time.sleep(dwell if dwell is not None else rng.uniform(_MIN_SLEEP, _MAX_SLEEP)) + + # Deterministic find when --target-zone is set (the object lives in one + # place, like reality); otherwise fall back to the random per-zone chance. + hit = zone == target_zone if target_zone else rng.random() < find_prob + if hit: + result = _post( + url, "/report_finding", {"dog": dog, "object": target, "zone": zone} + ) + finding = result.get("finding") if result else None + print(f"[{dog}] FOUND {target} in {zone} -> {finding} — stopping.") + return + + cleared = _post(url, "/report_cleared", {"dog": dog, "zone": zone}) + status = cleared.get("status") if cleared else "(unreachable)" + print(f"[{dog}] cleared {zone}: {status}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="PACK MIND mock dog client") + parser.add_argument("--dog", required=True, help="This dog's name, e.g. alpha.") + parser.add_argument( + "--url", + default="http://localhost:8090", + help="Coordinator base URL.", + ) + parser.add_argument("--target", default=DEFAULT_TARGET, help="What to 'find'.") + parser.add_argument( + "--find-prob", + type=float, + default=0.0, + help="Per-zone probability of reporting a finding (1.0 = always find).", + ) + parser.add_argument( + "--target-zone", + default=None, + help="Deterministic: only 'find' the object in this zone (overrides --find-prob).", + ) + parser.add_argument( + "--dwell", type=float, default=None, help="Seconds per zone (slows the dashboard for demos)." + ) + parser.add_argument( + "--reset", + action="store_true", + help="Call start_search first to reset the ledger and set the target.", + ) + parser.add_argument("--seed", type=int, default=None, help="RNG seed for repeatability.") + args = parser.parse_args() + + if args.reset: + _post(args.url.rstrip("/"), "/start_search", {"target": args.target}) + print(f"[{args.dog}] reset ledger; searching for {args.target}.") + + rng = random.Random(args.seed) + run_dog( + args.dog, args.url, args.target, args.find_prob, rng, args.target_zone, args.dwell + ) + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/pack_coordinator.py b/dimos/experimental/pack_mind/pack_coordinator.py new file mode 100644 index 0000000000..429940bbea --- /dev/null +++ b/dimos/experimental/pack_mind/pack_coordinator.py @@ -0,0 +1,220 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — laptop-side coordinator: the shared MEANING of the pack. + +This is the brain that lets two (or more) dogs search without overlapping. It +holds a **zone ledger** (which named regions are unsearched / claimed / cleared) +and a **target blackboard** (what we're looking for, whether it's been found and +where). It deliberately stores **no coordinates** — only zone NAMES — so it works +across dogs that each run their own SLAM frame (the honest "share meaning, not +maps" design, same invariant as conductor.py). + +The two guarantees that make the demo true: + 1. ``assign_zone`` never hands the same zone to two dogs (no overlap). + 2. once any dog reports a finding, ``found`` flips and assignment stops (the + whole pack knows and halts). + +Pure Python, thread-safe, no hardware deps — unit-testable in isolation and the +core of the HTTP coordinator (mockable like ``conductor.py --mock``). +""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from typing import Literal + +ZoneState = Literal["unsearched", "claimed", "cleared"] + + +@dataclass(frozen=True) +class Finding: + """An immutable record of an object sighting — by zone NAME, never coordinates.""" + + object: str + zone: str + by: str + ts: float + + +@dataclass +class Zone: + name: str + state: ZoneState = "unsearched" + by: str | None = None # the dog that claimed or cleared it + + +class PackCoordinator: + """Shared search ledger + target blackboard for a pack of dogs. + + Args: + zones: ordered zone names covering the search area. + preferences: optional per-dog ordering of zones, so dogs fan out to + different ends without any coordinate math (e.g. dog A prefers the + north zones, dog B the south). Falls back to ``zones`` order. + """ + + def __init__( + self, zones: list[str], preferences: dict[str, list[str]] | None = None + ) -> None: + if not zones: + raise ValueError("PackCoordinator needs at least one zone") + self._order = list(zones) + self._zones: dict[str, Zone] = {z: Zone(z) for z in zones} + self._preferences = preferences or {} + self._lock = threading.RLock() + self._target: str | None = None + self._finding: Finding | None = None + self._offline: set[str] = set() + self._log: list[str] = [] + + # -- mission ------------------------------------------------------------ + + def start_search(self, target: str) -> str: + """Begin a search for ``target``; resets the ledger so every zone is + unsearched again. Returns a status line.""" + with self._lock: + self._target = target + self._finding = None + self._offline = set() + self._zones = {z: Zone(z) for z in self._order} + self._log.append(f"search started: find '{target}'") + return f"Searching for {target}. {len(self._zones)} zones to cover." + + # -- allocation (the no-overlap guarantee) ------------------------------ + + def assign_zone(self, dog: str) -> str | None: + """Hand ``dog`` a zone no other dog has claimed or cleared, following + the dog's preference order. Returns None when the search is over (found + or nothing left) — the dog should then stop / converge.""" + with self._lock: + if self._finding is not None: + return None # object found → pack stops searching + for name in self._iter_order(dog): + zone = self._zones[name] + if zone.state == "unsearched": + self._zones[name] = Zone(name, "claimed", dog) + self._log.append(f"{dog} -> {name} (claimed)") + return name + return None + + def report_cleared(self, dog: str, zone: str) -> str: + """Mark ``zone`` fully searched by ``dog`` (object not here).""" + with self._lock: + if zone in self._zones: + self._zones[zone] = Zone(zone, "cleared", dog) + self._log.append(f"{dog} cleared {zone}") + return f"{zone} cleared." + + def report_finding(self, dog: str, object: str, zone: str = "") -> Finding: + """Record that ``dog`` saw ``object``. Flips ``found`` so the whole pack + stops. Idempotent: keeps the first finding. + + ``zone`` is best-effort: if it's empty or unknown (e.g. an LLM filled it in + wrong via a ``then=`` continuation), we fall back to the zone this dog + currently has claimed — the place it actually is.""" + with self._lock: + if self._finding is None: + if not zone or zone not in self._zones: + zone = self._claimed_zone_of(dog) or zone + self._finding = Finding(object, zone, dog, time.time()) + if zone in self._zones: + self._zones[zone] = Zone(zone, "cleared", dog) + self._log.append(f"{dog} FOUND {object} in {zone or '?'} — pack stop") + return self._finding + + def _claimed_zone_of(self, dog: str) -> str | None: + """The zone this dog currently holds claimed (where it is), if any.""" + for name, z in self._zones.items(): + if z.state == "claimed" and z.by == dog: + return name + return None + + def release_dog(self, dog: str) -> str: + """Take ``dog`` offline and hand its unfinished ground back to the pack. + + Its *findings persist* (the shared memory outlives the robot), but every + zone it had only **claimed** (not yet cleared) reverts to ``unsearched`` so + a surviving teammate inherits and finishes the mission. Cleared zones stay + cleared — that knowledge is not lost. This powers the resilience climax: + "the mission doesn't die with the robot." + """ + with self._lock: + self._offline.add(dog) + reclaimed = [] + for name, zone in self._zones.items(): + if zone.state == "claimed" and zone.by == dog: + self._zones[name] = Zone(name) # back to unsearched + reclaimed.append(name) + self._log.append( + f"{dog} OFFLINE — findings kept; reclaimed {reclaimed or 'no'} zone(s)" + ) + return f"{dog} offline. Reclaimed {len(reclaimed)} zone(s) for the pack." + + # -- queries ------------------------------------------------------------ + + @property + def found(self) -> bool: + with self._lock: + return self._finding is not None + + @property + def finding(self) -> Finding | None: + with self._lock: + return self._finding + + def should_stop(self, dog: str) -> bool: + """A dog polls this; True once the object is found by anyone.""" + return self.found + + def coverage(self) -> float: + """Fraction of zones cleared — the 'how much of the area is searched' metric.""" + with self._lock: + cleared = sum(1 for z in self._zones.values() if z.state == "cleared") + return cleared / len(self._zones) + + def snapshot(self) -> dict[str, object]: + with self._lock: + return { + "target": self._target, + "found": self._finding is not None, + "finding": ( + { + "object": self._finding.object, + "zone": self._finding.zone, + "by": self._finding.by, + } + if self._finding + else None + ), + "coverage": round(self.coverage(), 3), + "offline": sorted(self._offline), + "zones": [ + {"name": z.name, "state": z.state, "by": z.by} + for z in self._zones.values() + ], + "log": list(self._log), + } + + # -- helpers ------------------------------------------------------------ + + def _iter_order(self, dog: str) -> list[str]: + """The dog's preferred zone order, then any remaining zones.""" + pref = self._preferences.get(dog, []) + seen = set(pref) + return [z for z in pref if z in self._zones] + [ + z for z in self._order if z not in seen + ] diff --git a/dimos/experimental/pack_mind/pack_coordinator_server.py b/dimos/experimental/pack_mind/pack_coordinator_server.py new file mode 100644 index 0000000000..86cef0a2ad --- /dev/null +++ b/dimos/experimental/pack_mind/pack_coordinator_server.py @@ -0,0 +1,268 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND coordinator HTTP server — the shared MEANING, reachable over a LAN. + +Wraps a single ``PackCoordinator`` (the pure, unit-tested ledger) behind a tiny +JSON/HTTP API so each dog's laptop agent can call it with ``requests``. Mirrors +the stdlib ``ThreadingHTTPServer`` + ``BaseHTTPRequestHandler`` pattern in +``conductor.py``. The coordinator shares zone NAMES and findings only — never +coordinates — so two dogs running independent SLAM frames never re-search the +same area. + +Endpoints (all JSON; 200 on success, 400 on bad input):: + + GET /state -> snapshot() + POST /start_search {"target"} -> {"status"} + POST /assign_zone {"dog"} -> {"zone": str|null} + POST /report_cleared {"dog", "zone"} -> {"status"} + POST /report_finding {"dog", "object", "zone"} -> {"finding": {...}|null} + GET /should_stop?dog=NAME -> {"stop": bool} + +Run (binds 0.0.0.0 so dogs across the LAN can reach it):: + + uv run python dimos/experimental/pack_mind/pack_coordinator_server.py \\ + --port 8090 --zones north,east,south,west +""" + +from __future__ import annotations + +import argparse +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, cast +from urllib.parse import parse_qs, urlparse + +_DASHBOARD_HTML = Path(__file__).with_name("pack_dashboard.html") + +from dimos.experimental.pack_mind.pack_coordinator import PackCoordinator +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +# New named constant — the LAN port the coordinator listens on. No hardcoded +# ports elsewhere; clients import this to build their default URL. +DEFAULT_COORDINATOR_PORT = 8090 +DEFAULT_ZONES = ["north", "east", "south", "west"] + + +class _Server(ThreadingHTTPServer): + coordinator: PackCoordinator + + +class _Handler(BaseHTTPRequestHandler): + def log_message(self, fmt: str, *args: Any) -> None: # silence default logging + return + + @property + def _coordinator(self) -> PackCoordinator: + return cast(_Server, self.server).coordinator + + def _send_json(self, code: int, payload: Any) -> None: + body = json.dumps(payload).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _read_json(self) -> dict[str, Any] | None: + """Read a JSON object body. Returns None on malformed/non-object input.""" + length = int(self.headers.get("Content-Length", "0")) + raw = self.rfile.read(length) if length else b"{}" + try: + data: Any = json.loads(raw) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + def _serve_dashboard(self) -> None: + try: + body = _DASHBOARD_HTML.read_bytes() + except OSError: + self._send_json(500, {"error": "dashboard html not found"}) + return + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 (stdlib API) + parsed = urlparse(self.path) + if parsed.path in ("/", "/dashboard"): + self._serve_dashboard() + elif parsed.path == "/state": + self._send_json(200, self._coordinator.snapshot()) + elif parsed.path == "/where_is": + f = self._coordinator.finding + self._send_json( + 200, + { + "found": f is not None, + "object": f.object if f else None, + "zone": f.zone if f else None, + "by": f.by if f else None, + }, + ) + elif parsed.path == "/should_stop": + params = parse_qs(parsed.query) + dog_values = params.get("dog") + if not dog_values or not dog_values[0]: + self._send_json(400, {"error": "missing query param: dog"}) + return + self._send_json(200, {"stop": self._coordinator.should_stop(dog_values[0])}) + else: + self._send_json(404, {"error": f"unknown path: {parsed.path}"}) + + def do_POST(self) -> None: # noqa: N802 (stdlib API) + path = urlparse(self.path).path + req = self._read_json() + if req is None: + self._send_json(400, {"error": "bad json"}) + return + if path == "/start_search": + self._start_search(req) + elif path == "/assign_zone": + self._assign_zone(req) + elif path == "/report_cleared": + self._report_cleared(req) + elif path == "/report_finding": + self._report_finding(req) + elif path == "/release_dog": + self._release_dog(req) + else: + self._send_json(404, {"error": f"unknown path: {path}"}) + + # -- handlers ------------------------------------------------------------ + + def _start_search(self, req: dict[str, Any]) -> None: + target = req.get("target") + if not isinstance(target, str) or not target: + self._send_json(400, {"error": "missing or invalid field: target"}) + return + status = self._coordinator.start_search(target) + logger.info("start_search", target=target) + self._send_json(200, {"status": status}) + + def _assign_zone(self, req: dict[str, Any]) -> None: + dog = req.get("dog") + if not isinstance(dog, str) or not dog: + self._send_json(400, {"error": "missing or invalid field: dog"}) + return + zone = self._coordinator.assign_zone(dog) + logger.info("assign_zone", dog=dog, zone=zone) + self._send_json(200, {"zone": zone}) + + def _report_cleared(self, req: dict[str, Any]) -> None: + dog = req.get("dog") + zone = req.get("zone") + if not isinstance(dog, str) or not dog or not isinstance(zone, str) or not zone: + self._send_json(400, {"error": "missing or invalid fields: dog, zone"}) + return + status = self._coordinator.report_cleared(dog, zone) + logger.info("report_cleared", dog=dog, zone=zone) + self._send_json(200, {"status": status}) + + def _report_finding(self, req: dict[str, Any]) -> None: + dog = req.get("dog") + obj = req.get("object") + zone = req.get("zone", "") + if not isinstance(dog, str) or not dog or not isinstance(obj, str) or not obj: + self._send_json(400, {"error": "missing or invalid fields: dog, object"}) + return + # zone is best-effort: the coordinator falls back to the dog's claimed zone. + zone = zone if isinstance(zone, str) else "" + finding = self._coordinator.report_finding(dog, obj, zone) + logger.info("report_finding", dog=dog, object=obj, zone=zone) + self._send_json( + 200, + {"finding": {"object": finding.object, "zone": finding.zone, "by": finding.by}}, + ) + + def _release_dog(self, req: dict[str, Any]) -> None: + dog = req.get("dog") + if not isinstance(dog, str) or not dog: + self._send_json(400, {"error": "missing or invalid field: dog"}) + return + status = self._coordinator.release_dog(dog) + logger.info("release_dog", dog=dog) + self._send_json(200, {"status": status}) + + +def make_server( + host: str, + port: int, + zones: list[str], + preferences: dict[str, list[str]] | None = None, +) -> _Server: + """Build a ready-to-serve coordinator server. Pass port 0 for an ephemeral port.""" + server = _Server((host, port), _Handler) + server.coordinator = PackCoordinator(zones, preferences) + return server + + +def _parse_prefs(raw: str) -> dict[str, list[str]]: + """Parse "alpha:north,east;bravo:south,west" into per-dog zone orderings.""" + prefs: dict[str, list[str]] = {} + for chunk in raw.split(";"): + chunk = chunk.strip() + if not chunk or ":" not in chunk: + continue + dog, zone_csv = chunk.split(":", 1) + zones = [z.strip() for z in zone_csv.split(",") if z.strip()] + if dog.strip() and zones: + prefs[dog.strip()] = zones + return prefs + + +def main() -> None: + parser = argparse.ArgumentParser(description="PACK MIND coordinator HTTP server") + parser.add_argument("--host", default="0.0.0.0", help="Bind address (0.0.0.0 for LAN).") + parser.add_argument( + "--port", type=int, default=DEFAULT_COORDINATOR_PORT, help="Listen port." + ) + parser.add_argument( + "--zones", + default=",".join(DEFAULT_ZONES), + help="Comma-separated zone names.", + ) + parser.add_argument( + "--prefs", + default="", + help='Per-dog zone order so dogs fan out, e.g. "alpha:north,east;bravo:south,west".', + ) + args = parser.parse_args() + + zones = [z.strip() for z in args.zones.split(",") if z.strip()] + if not zones: + parser.error("--zones must contain at least one zone name") + + server = make_server(args.host, args.port, zones, _parse_prefs(args.prefs)) + actual_port = server.server_address[1] + print(f"PACK MIND coordinator up on http://{args.host}:{actual_port}") + print(f" reachable on the LAN at http://:{actual_port}") + print(f" zones: {', '.join(zones)}") + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nshutting down") + server.shutdown() + finally: + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/pack_dashboard.html b/dimos/experimental/pack_mind/pack_dashboard.html new file mode 100644 index 0000000000..3f74b3b96f --- /dev/null +++ b/dimos/experimental/pack_mind/pack_dashboard.html @@ -0,0 +1,176 @@ + + + + + + +PACK MIND + + + +
+

PACK MIND

+ shared operational memory + memory: empty +
+
+
+

Search zones — one shared memory, no overlap

+
+

Area searched

+
+
+
+
+

Finding

+
No sighting yet — the pack is searching.
+
+
+

Pack roster

+
+
+
+

Causal chain

+
+
+
+
+
ONE MEMORY · MANY BODIES · OUTLIVES ANY SINGLE ROBOT
+ + + diff --git a/dimos/experimental/pack_mind/pack_search_runner.py b/dimos/experimental/pack_mind/pack_search_runner.py new file mode 100644 index 0000000000..19318b970e --- /dev/null +++ b/dimos/experimental/pack_mind/pack_search_runner.py @@ -0,0 +1,114 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND search orchestration — the deterministic loop a dog runs. + +The loop is decoupled from *how* a dog moves and sees via the ``RobotDriver`` +Protocol, so the SAME orchestration runs: + - in tests / the hardware-free scene, with ``MockDriver``; + - on a real Go2, with a driver that wraps ``navigate_with_text`` + ``look_out_for``. + +It talks to the coordinator through the ``SearchClient`` Protocol — satisfied +directly by ``PackCoordinator`` (in-process tests) or by an HTTP-backed adapter +(live). Determinism here is deliberate: a scripted loop is far more demo-reliable +than asking the LLM to remember to loop. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Protocol + +from dimos.experimental.pack_mind.pack_coordinator import Finding + + +class RobotDriver(Protocol): + """How a body moves and perceives. Mock in tests; real skills on hardware.""" + + def goto(self, zone: str) -> bool: + """Travel to the named zone. Return True on arrival.""" + ... + + def look_for(self, target: str) -> bool: + """Scan the current location. Return True iff the target is seen here.""" + ... + + +class SearchClient(Protocol): + """The coordinator surface the loop needs. PackCoordinator satisfies this.""" + + def assign_zone(self, dog: str) -> str | None: ... + def report_cleared(self, dog: str, zone: str) -> str: ... + def report_finding(self, dog: str, object: str, zone: str) -> Finding: ... + def should_stop(self, dog: str) -> bool: ... + @property + def finding(self) -> Finding | None: ... + + +@dataclass +class SearchResult: + dog: str + found: bool + zone: str | None + visited: list[str] = field(default_factory=list) + + +def run_search( + client: SearchClient, driver: RobotDriver, dog: str, target: str, max_steps: int = 100 +) -> SearchResult: + """Pull a distinct zone, go, look, report — until found, stopped, or empty. + + Inheriting another dog's reclaimed zone is automatic: ``assign_zone`` simply + starts handing out the freed zones, so a survivor's loop continues seamlessly. + """ + visited: list[str] = [] + for _ in range(max_steps): + if client.should_stop(dog): + break + zone = client.assign_zone(dog) + if not zone: + break + if not driver.goto(zone): + continue # couldn't reach it; ask for another + visited.append(zone) + if driver.look_for(target): + client.report_finding(dog, target, zone) + return SearchResult(dog, True, zone, visited) + client.report_cleared(dog, zone) + f = client.finding + return SearchResult(dog, f is not None, f.zone if f else None, visited) + + +def act_on_finding(client: SearchClient, driver: RobotDriver, dog: str) -> str: + """The handoff: go to a teammate's reported find — memory you never gathered.""" + f = client.finding + if f is None: + return f"{dog}: nothing in pack memory to act on yet." + driver.goto(f.zone) + return f"{dog} acted on {f.by}'s memory: went to {f.zone}, confirmed {f.object}." + + +@dataclass +class MockDriver: + """Hardware-free driver: the target lives in ``target_zone``; travel always succeeds.""" + + target_zone: str + visited: list[str] = field(default_factory=list) + + def goto(self, zone: str) -> bool: + self.visited.append(zone) + return True + + def look_for(self, target: str) -> bool: + return bool(self.visited) and self.visited[-1] == self.target_zone diff --git a/dimos/experimental/pack_mind/pack_search_skills.py b/dimos/experimental/pack_mind/pack_search_skills.py new file mode 100644 index 0000000000..9f8b360a31 --- /dev/null +++ b/dimos/experimental/pack_mind/pack_search_skills.py @@ -0,0 +1,228 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND search skills — the bridge from a dog's LLM agent to the shared coordinator. + +A DimOS ``Module`` that turns the laptop coordinator's HTTP API into @skill tools +the robot's agent can call. The dog never learns the other dog's coordinates — it +only asks "what zone next?" and reports cleared zones / findings by NAME. Every +call is wrapped in a short-timeout try/except so a coordinator outage degrades to +a calm string and NEVER raises into the agent loop. + +Wire it into an agentic blueprint alongside the robot stack; point it at the +coordinator with the ``PACK_COORDINATOR_URL`` env var (or the constructor arg). +""" + +from __future__ import annotations + +import os +from typing import Any + +import requests + +from dimos.agents.annotation import skill +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.experimental.pack_mind.pack_coordinator_server import DEFAULT_COORDINATOR_PORT +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +# Built from localhost + the server's port constant — no hardcoded port literal. +DEFAULT_COORDINATOR_URL = f"http://localhost:{DEFAULT_COORDINATOR_PORT}" + +# Short timeout: the agent loop must never block on a flaky coordinator. +_REQUEST_TIMEOUT = 5.0 + + +class PackSearchSkills(Module): + """Exposes the PACK MIND coordinator to one dog's LLM agent as tools. + + Args: + dog_name: this dog's stable name (used as the ledger identity). + coordinator_url: base URL of the coordinator server; defaults to the + ``PACK_COORDINATOR_URL`` env var, else localhost + the default port. + """ + + def __init__( + self, + dog_name: str | None = None, + coordinator_url: str | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + # Per-laptop identity: explicit arg › PACK_DOG_NAME env › "alpha". This lets + # the SAME blueprint run on both laptops, differing only by env var. + self._dog_name = dog_name or os.environ.get("PACK_DOG_NAME") or "alpha" + self._url = ( + coordinator_url + or os.environ.get("PACK_COORDINATOR_URL") + or DEFAULT_COORDINATOR_URL + ).rstrip("/") + + @rpc + def start(self) -> None: + super().start() + logger.info("PackSearchSkills ready", dog=self._dog_name, coordinator=self._url) + + @rpc + def stop(self) -> None: + super().stop() + + # -- HTTP plumbing ------------------------------------------------------- + + def _post(self, path: str, payload: dict[str, Any]) -> dict[str, Any] | None: + """POST JSON; return the decoded dict, or None on any failure.""" + try: + resp = requests.post( + f"{self._url}{path}", json=payload, timeout=_REQUEST_TIMEOUT + ) + resp.raise_for_status() + data: Any = resp.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + logger.warning("coordinator POST failed", path=path, error=str(exc)) + return None + + def _get(self, path: str, params: dict[str, str]) -> dict[str, Any] | None: + try: + resp = requests.get( + f"{self._url}{path}", params=params, timeout=_REQUEST_TIMEOUT + ) + resp.raise_for_status() + data: Any = resp.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + logger.warning("coordinator GET failed", path=path, error=str(exc)) + return None + + # -- skills (exposed to the LLM) ----------------------------------------- + + @skill + def start_search(self, target: str) -> str: + """Begin a pack search for an object across the shared zones. + + Call this once at mission start. It resets the shared zone ledger so the + whole pack hunts for the same target without re-searching areas. + + Args: + target: A short description of what to find, e.g. "red backpack". + """ + data = self._post("/start_search", {"target": target}) + if data is None: + return f"Could not reach the pack coordinator to start searching for {target}." + return str(data.get("status", f"Searching for {target}.")) + + @skill + def report_cleared(self, zone: str) -> str: + """Tell the pack you finished searching a zone and the target was NOT there. + + Marks the zone cleared on the shared ledger so no teammate re-searches it. + + Args: + zone: The name of the zone you just finished searching, e.g. "north". + """ + data = self._post("/report_cleared", {"dog": self._dog_name, "zone": zone}) + if data is None: + return f"Could not reach the pack coordinator to report {zone} cleared." + return str(data.get("status", f"{zone} cleared.")) + + @skill + def report_finding(self, object: str, zone: str) -> str: + """Tell the whole pack you found the target so everyone stops searching. + + Records the sighting by zone NAME on the shared blackboard; the first + finding wins and halts all assignment. + + Args: + object: What you found, e.g. "red backpack". + zone: The zone name where you found it, e.g. "east". + """ + data = self._post( + "/report_finding", + {"dog": self._dog_name, "object": object, "zone": zone}, + ) + if data is None: + return f"Could not reach the pack coordinator to report finding {object}." + finding = data.get("finding") + if isinstance(finding, dict): + return f"Pack notified: {finding.get('object')} found in {finding.get('zone')}." + return f"Pack notified: {object} found in {zone}." + + @skill + def where_is(self, object: str) -> str: + """Ask the pack's shared memory where an object was found. + + Use this to act on a TEAMMATE's discovery — you may never have seen the + object yourself. Returns the zone a packmate reported it in, or that it + hasn't been found yet. + + Args: + object: What you're asking about, e.g. "red backpack". + """ + data = self._get("/where_is", {}) + if data is None: + return "Could not reach the pack memory." + if not data.get("found"): + return f"No packmate has found the {object} yet." + return ( + f"{data.get('by')} found the {data.get('object')} in {data.get('zone')}. " + f"I can take you there." + ) + + # -- rpc (internal control, not exposed to the LLM) ---------------------- + + @rpc + def finding_zone(self) -> str: + """The zone of the current finding (for navigation), or "" if none yet.""" + data = self._get("/where_is", {}) + if data is None or not data.get("found"): + return "" + zone = data.get("zone") + return zone if isinstance(zone, str) else "" + + @rpc + def release_dog(self, dog: str) -> bool: + """Mark a packmate offline so the pack reclaims its unfinished zones. + + Operator/coordinator action (e.g. a dog dropped). The downed dog's + findings persist; its claimed-but-uncleared zones return to the pool so a + survivor inherits them. Returns True on success. + """ + data = self._post("/release_dog", {"dog": dog}) + return data is not None + + @rpc + def next_zone(self) -> str: + """Ask the coordinator for the next unsearched zone for this dog. + + Returns the zone name, or "" when the search is over (found or no zones + left) — skills cannot return None cleanly, so "" signals "stop". + """ + data = self._post("/assign_zone", {"dog": self._dog_name}) + if data is None: + return "" + zone = data.get("zone") + return zone if isinstance(zone, str) else "" + + @rpc + def should_stop(self) -> bool: + """True once any dog has found the target and the pack should converge.""" + data = self._get("/should_stop", {"dog": self._dog_name}) + if data is None: + return False + return bool(data.get("stop", False)) + + +pack_search_skills = PackSearchSkills.blueprint diff --git a/dimos/experimental/pack_mind/prefetch_live_models.py b/dimos/experimental/pack_mind/prefetch_live_models.py new file mode 100644 index 0000000000..c3fe7d1747 --- /dev/null +++ b/dimos/experimental/pack_mind/prefetch_live_models.py @@ -0,0 +1,71 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pre-cache the HuggingFace models the live PACK MIND stack loads at runtime. + +Run this ONCE on each demo laptop while it has real internet. Afterwards the demo +runs fully offline with ``HF_HUB_OFFLINE=1`` — every model loads from the local +cache, so a flaky venue network (or the dog's internet-less WiFi AP) can't stall a +model load mid-demo. + + uv run python -m dimos.experimental.pack_mind.prefetch_live_models + +What these are (and why they bite if missing): + - moondream2: the VLM behind ``look_out_for`` (GlobalConfig.detection_model = + "moondream"). Loads lazily on the FIRST detection — uncached = crash mid-demo. + - faster-whisper-base: WebInput's voice STT, loaded at module start — uncached + + HF_HUB_OFFLINE=1 = the start-time LocalEntryNotFoundError. + +CLIP and YOLO ship in ``data/`` (git-lfs), so they are not fetched here. The +LLM/agent brain (GPT-4o / Qwen-VL) is a hosted API, not a local download — it needs +internet at demo time regardless (run the coordinator laptop on a router with WAN). +""" + +from __future__ import annotations + +import os + +# Force online for this one run, even if the shell exported HF_HUB_OFFLINE=1. +os.environ["HF_HUB_OFFLINE"] = "0" +os.environ["TRANSFORMERS_OFFLINE"] = "0" + +from huggingface_hub import snapshot_download # noqa: E402 + +# Plain snapshot is enough for these (loaded straight from the cached dir). +SNAPSHOT_MODELS: list[tuple[str, str]] = [ + ("Systran/faster-whisper-base", "WebInput voice STT (faster-whisper)"), +] + + +def main() -> None: + for repo, why in SNAPSHOT_MODELS: + print(f"\n== {repo} — {why}") + print(f" cached: {snapshot_download(repo)}") + + # moondream2 is loaded via trust_remote_code, which pulls THREE things a plain + # snapshot_download misses: the weights, the custom code modules + # (transformers_modules/...), and a SEPARATE tokenizer repo it references + # (moondream/starmie-v1). A full from_pretrained fetches all of them, so the + # live look_out_for then loads fully offline. + print("\n== vikhyatk/moondream2 — look_out_for VLM (full load for transitive deps)") + from transformers import AutoModelForCausalLM + + AutoModelForCausalLM.from_pretrained("vikhyatk/moondream2", trust_remote_code=True) + print(" moondream2 + code modules + starmie-v1 tokenizer cached") + + print("\nDone. Run the live demo with HF_HUB_OFFLINE=1 — models load from cache.") + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/preflight.py b/dimos/experimental/pack_mind/preflight.py new file mode 100644 index 0000000000..ac03669568 --- /dev/null +++ b/dimos/experimental/pack_mind/preflight.py @@ -0,0 +1,274 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — live preflight for the 2-dog / 2-laptop demo. + +Run on EACH laptop the morning of the demo, BEFORE starting dogs. Walks the whole +AP-per-dog + Tailscale chain link by link and prints PASS/WARN/FAIL with the exact +fix for every failure. Pure stdlib (macOS-focused) — no dimos import, no extra deps. + + uv run python -m dimos.experimental.pack_mind.preflight \\ + --role alpha --peer --coordinator http://127.0.0.1:8090 + +Exit code 0 only when nothing is FAIL (WARN is allowed). See LIVE_RUNBOOK.md. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import os +import shutil +import socket +import subprocess +import sys +from typing import Literal +import urllib.request + +Status = Literal["PASS", "WARN", "FAIL"] + +_GREEN = "\033[32m" +_YELLOW = "\033[33m" +_RED = "\033[31m" +_DIM = "\033[2m" +_RESET = "\033[0m" +_COLOR: dict[Status, str] = {"PASS": _GREEN, "WARN": _YELLOW, "FAIL": _RED} + +_DOG_AP_SUBNET_PREFIX = "192.168.12." # Unitree Go2 AP default +_TS_APP_CLI = "/Applications/Tailscale.app/Contents/MacOS/Tailscale" + + +@dataclass +class Check: + status: Status + name: str + detail: str + fix: str = "" + + +def _run(cmd: list[str], timeout: float = 6.0) -> tuple[int, str]: + """Run a command; return (rc, combined stdout+stderr). rc=-1 on missing/timeout.""" + try: + p = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, check=False + ) + return p.returncode, (p.stdout + p.stderr).strip() + except (FileNotFoundError, subprocess.TimeoutExpired) as exc: + return -1, str(exc) + + +def _route_iface(target: str) -> str | None: + """macOS `route -n get ` -> the egress interface name, or None.""" + rc, out = _run(["route", "-n", "get", target]) + if rc != 0: + return None + for line in out.splitlines(): + stripped = line.strip() + if stripped.startswith("interface:"): + return stripped.split(":", 1)[1].strip() + return None + + +def _ifaddr(iface: str) -> str | None: + rc, out = _run(["ipconfig", "getifaddr", iface]) + return out.strip() if rc == 0 and out.strip() else None + + +def _tailscale_bin() -> str | None: + return shutil.which("tailscale") or ( + _TS_APP_CLI if os.path.exists(_TS_APP_CLI) else None + ) + + +# -- individual checks ------------------------------------------------------- + + +def check_env(role: str) -> list[Check]: + out: list[Check] = [] + dog = os.environ.get("PACK_DOG_NAME") + if dog and dog == role: + out.append(Check("PASS", "env PACK_DOG_NAME", dog)) + elif dog: + out.append( + Check("WARN", "env PACK_DOG_NAME", f"{dog!r} != --role {role!r}", + f"export PACK_DOG_NAME={role}") + ) + else: + out.append(Check("FAIL", "env PACK_DOG_NAME", "unset", f"export PACK_DOG_NAME={role}")) + + url = os.environ.get("PACK_COORDINATOR_URL") + if url: + out.append(Check("PASS", "env PACK_COORDINATOR_URL", url)) + else: + out.append(Check("FAIL", "env PACK_COORDINATOR_URL", "unset", + "export PACK_COORDINATOR_URL=http://:8090")) + + key = os.environ.get("OPENAI_API_KEY") + if key: + out.append(Check("PASS", "env OPENAI_API_KEY", "set")) + else: + out.append(Check("WARN", "env OPENAI_API_KEY", "unset (only needed for autonomy path 4b)", + "export OPENAI_API_KEY=")) + return out + + +def check_dog(robot_ip: str) -> list[Check]: + out: list[Check] = [] + rc, _ = _run(["ping", "-c1", "-t2", robot_ip], timeout=5) + if rc != 0: + out.append( + Check("FAIL", "dog reachable", f"{robot_ip} no reply", + "join the dog's Wi-Fi AP; confirm ROBOT_IP; power-cycle the dog") + ) + return out + out.append(Check("PASS", "dog reachable", robot_ip)) + + dog_iface = _route_iface(robot_ip) + if dog_iface: + out.append(Check("PASS", "route->dog", f"{robot_ip} via {dog_iface}")) + else: + out.append(Check("WARN", "route->dog", "no route entry", "reconnect the dog Wi-Fi")) + return out + + +def check_internet(robot_ip: str) -> list[Check]: + out: list[Check] = [] + dog_iface = _route_iface(robot_ip) + wan_iface = _route_iface("8.8.8.8") or _route_iface("default") + + if not wan_iface: + out.append(Check("FAIL", "default route", "none", + "connect the uplink (USB tether / Ethernet)")) + return out + + if dog_iface and wan_iface == dog_iface: + out.append( + Check("FAIL", "default route", f"WAN routes via dog AP iface {wan_iface}", + "System Settings->Network->...->Set Service Order: put the uplink ABOVE Wi-Fi") + ) + else: + out.append(Check("PASS", "default route", f"WAN via {wan_iface} (dog via {dog_iface})")) + + wan_ip = _ifaddr(wan_iface) + if wan_ip and wan_ip.startswith(_DOG_AP_SUBNET_PREFIX): + out.append( + Check("FAIL", "subnet collision", + f"uplink {wan_ip} collides with dog AP {_DOG_AP_SUBNET_PREFIX}x", + "change the uplink subnet or the dog AP subnet -- they must differ") + ) + elif wan_ip: + out.append(Check("PASS", "uplink subnet", wan_ip)) + + try: + with socket.create_connection(("1.1.1.1", 443), timeout=5): + out.append(Check("PASS", "internet", "1.1.1.1:443 reachable")) + except OSError as exc: + out.append(Check("FAIL", "internet", str(exc), + "fix the uplink / service order before Tailscale will work")) + return out + + +def check_tailscale(peer: str | None) -> list[Check]: + out: list[Check] = [] + ts = _tailscale_bin() + if not ts: + out.append( + Check("FAIL", "tailscale", "CLI not found", + f"install Tailscale; App Store build CLI at {_TS_APP_CLI}") + ) + return out + + rc, status_txt = _run([ts, "status"]) + if rc != 0: + out.append(Check("FAIL", "tailscale status", status_txt[:80] or "down", f"{ts} up")) + return out + + rc, self_ip_raw = _run([ts, "ip", "-4"]) + self_ip = self_ip_raw.splitlines()[0].strip() if rc == 0 and self_ip_raw.strip() else "" + if self_ip.startswith("100."): + out.append(Check("PASS", "tailscale up", f"self {self_ip}")) + else: + out.append(Check("WARN", "tailscale up", "no 100.x self ip", f"{ts} up --accept-routes")) + + if peer: + rc, pout = _run([ts, "ping", "--timeout=4s", "--c=1", peer], timeout=8) + if rc == 0 and "pong" in pout.lower(): + out.append(Check("PASS", "peer reachable", f"tailscale ping {peer}")) + else: + rc2, _ = _run(["ping", "-c1", "-t4", peer], timeout=6) + if rc2 == 0: + out.append(Check("PASS", "peer reachable", f"icmp {peer}")) + else: + out.append(Check("FAIL", "peer reachable", f"{peer} no pong/icmp", + "check peer laptop tailscale up + its service order/uplink")) + return out + + +def check_coordinator(url: str) -> list[Check]: + url = url.rstrip("/") + try: + with urllib.request.urlopen(f"{url}/state", timeout=5) as resp: + data = json.loads(resp.read().decode()) + zones = data.get("zones") if isinstance(data, dict) else None + if isinstance(zones, list) and zones: + names = ",".join(str(z.get("name", "?")) for z in zones) + return [Check("PASS", "coordinator /state", f"{url} zones=[{names}]")] + return [Check("WARN", "coordinator /state", f"{url} reachable but no zones", + "start it with --zones north,east,south,west")] + except Exception as exc: + return [Check("FAIL", "coordinator /state", f"{url}: {exc}", + "start pack_coordinator_server on Laptop-A; check the tailscale URL/port")] + + +# -- driver ------------------------------------------------------------------ + + +def main() -> int: + ap = argparse.ArgumentParser(description="PACK MIND live preflight (per laptop)") + ap.add_argument("--role", required=True, choices=["alpha", "bravo"]) + ap.add_argument("--peer", default=None, help="the OTHER laptop's tailscale IP") + ap.add_argument( + "--coordinator", + default=os.environ.get("PACK_COORDINATOR_URL", "http://127.0.0.1:8090"), + ) + ap.add_argument("--robot-ip", default=os.environ.get("ROBOT_IP", "192.168.12.1")) + args = ap.parse_args() + + checks: list[Check] = [] + checks += check_env(args.role) + checks += check_dog(args.robot_ip) + checks += check_internet(args.robot_ip) + checks += check_tailscale(args.peer) + checks += check_coordinator(args.coordinator) + + print(f"\n PACK MIND preflight -- role={args.role} dog={args.robot_ip}\n") + width = max(len(c.name) for c in checks) + for c in checks: + print(f" {_COLOR[c.status]}{c.status:<4}{_RESET} {c.name:<{width}} {c.detail}") + if c.status != "PASS" and c.fix: + print(f" {_DIM}fix: {c.fix}{_RESET}") + + fails = sum(c.status == "FAIL" for c in checks) + warns = sum(c.status == "WARN" for c in checks) + print(f"\n {fails} FAIL · {warns} WARN · {len(checks) - fails - warns} PASS") + if fails: + print(f" {_RED}NOT READY -- fix the FAILs above before starting dogs.{_RESET}\n") + return 1 + print(f" {_GREEN}READY.{_RESET}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dimos/experimental/pack_mind/red_detector.py b/dimos/experimental/pack_mind/red_detector.py new file mode 100644 index 0000000000..799a76eb54 --- /dev/null +++ b/dimos/experimental/pack_mind/red_detector.py @@ -0,0 +1,122 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — fast, GPU-free "red object" detector for the live demo. + +The demo's target is defined by its COLOUR, so the right detector is a colour +filter on the live camera, not a 4GB VLM. moondream on a CPU ground station is +slow (tens of seconds, async) and fragile; this is a deterministic per-frame red +test that runs in milliseconds and is honest — the dog literally sees red. + +On a hit, it reports the finding to the pack coordinator (zone left blank, so the +coordinator fills in the dog's currently-claimed zone). Exposed as the @skill +``look_for_red`` so the operator (or agent) triggers it with one call. +""" + +from __future__ import annotations + +import os +from typing import Any + +import numpy as np +import requests +from reactivex.disposable import Disposable + +from dimos.agents.annotation import skill +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.core.stream import In +from dimos.experimental.pack_mind.pack_coordinator_server import DEFAULT_COORDINATOR_PORT +from dimos.msgs.sensor_msgs.Image import Image +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +DEFAULT_RED_THRESHOLD = 0.02 # ≥2% of the frame being red ⇒ a red object is in view +_REQUEST_TIMEOUT = 5.0 + + +def red_fraction(data: np.ndarray) -> float: + """Fraction of pixels that are convincingly red (RGB, lighting-tolerant ratio test). + + Red = a high red channel that clearly dominates green and blue. Using channel + *ratios* rather than absolute thresholds keeps it robust across lighting. + Assumes RGB channel order (DimOS camera frames); if it flags blue, the frame is + BGR — swap channels 0 and 2. + """ + if data.ndim != 3 or data.shape[2] < 3: + return 0.0 + r = data[..., 0].astype(np.int16) + g = data[..., 1].astype(np.int16) + b = data[..., 2].astype(np.int16) + mask = (r > 110) & ((r - g) > 50) & ((r - b) > 50) + return float(mask.mean()) + + +class RedObjectDetector(Module): + """Watches the camera; ``look_for_red`` checks the latest frame and, if a red + object is in view, tells the pack coordinator.""" + + color_image: In[Image] + + def __init__( + self, + dog_name: str | None = None, + coordinator_url: str | None = None, + threshold: float = DEFAULT_RED_THRESHOLD, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self._latest: Image | None = None + self._threshold = threshold + self._dog = dog_name or os.environ.get("PACK_DOG_NAME") or "alpha" + self._url = ( + coordinator_url + or os.environ.get("PACK_COORDINATOR_URL") + or f"http://127.0.0.1:{DEFAULT_COORDINATOR_PORT}" + ).rstrip("/") + + @rpc + def start(self) -> None: + super().start() + self.register_disposable(Disposable(self.color_image.subscribe(self._on_image))) + + def _on_image(self, image: Image) -> None: + self._latest = image + + @skill + def look_for_red(self) -> str: + """Look for a red object in the robot's current camera view. If one is + visible, report it to the pack's shared memory and say so. + """ + img = self._latest + if img is None: + return "No camera frame yet — is the robot streaming video?" + frac = red_fraction(np.asarray(img.data)) + if frac < self._threshold: + return f"No red object in view (red {frac:.1%})." + try: + requests.post( + f"{self._url}/report_finding", + json={"dog": self._dog, "object": "red object", "zone": ""}, + timeout=_REQUEST_TIMEOUT, + ) + except requests.RequestException as exc: + logger.warning("red object seen but coordinator unreachable", error=str(exc)) + return f"I see a red object (red {frac:.0%}) but could not reach the pack memory." + logger.info("red object detected", dog=self._dog, fraction=round(frac, 3)) + return f"I see a red object (red {frac:.0%}) — reported to the pack." + + +red_object_detector = RedObjectDetector.blueprint diff --git a/dimos/experimental/pack_mind/render.py b/dimos/experimental/pack_mind/render.py new file mode 100644 index 0000000000..7533776ff4 --- /dev/null +++ b/dimos/experimental/pack_mind/render.py @@ -0,0 +1,167 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — G3 render: side-by-side A/B coverage race to mp4 (or GIF). + +Left = PACK (shared coverage memory), right = INDEPENDENT (private memory). Same +seed, same single deploy point, same survivors — the only difference is shared vs +private memory. The HUD shows coverage %, survivors found, and tick so the gap is +self-evident on screen. + + python -m dimos.experimental.pack_mind.render --out pack_mind_ab.mp4 --seed 0 +""" + +from __future__ import annotations + +import argparse +from typing import Any + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 +from matplotlib import animation # noqa: E402 + +from dimos.experimental.pack_mind.sim import PackSim, build_sim # noqa: E402 +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues # noqa: E402 + +_SNAP_EVERY = 15 # capture a frame every N ticks + + +def _union_visited(sim: PackSim) -> np.ndarray: + free = sim.world.grid == CostValues.FREE + union: np.ndarray | None = None + for rb in sim.robots: + v = rb.router._visited + if v is None: + continue + union = v.copy() if union is None else (union | v) + if union is None: + union = np.zeros_like(free) + return union & free + + +def _run_capture(shared: bool, seed: int, max_ticks: int) -> tuple[PackSim, list[dict[str, Any]]]: + sim = build_sim(shared=shared, seed=seed) + free_total = sim._free_total + frames: list[dict[str, Any]] = [] + + def snap() -> None: + vis = _union_visited(sim) + frames.append( + { + "tick": sim.tick_n, + "visited": vis, + "robots": [(rb.x, rb.y, rb.done) for rb in sim.robots], + "found": set(sim._found), + "cov": float(np.count_nonzero(vis)) / max(1, free_total), + } + ) + + snap() + for _ in range(max_ticks): + sim.step() + if sim.tick_n % _SNAP_EVERY == 0: + snap() + if all(rb.done for rb in sim.robots): + break + snap() + return sim, frames + + +def render_ab(seed: int = 0, out_path: str = "pack_mind_ab.mp4", max_ticks: int = 4000) -> str: + psim, pframes = _run_capture(True, seed, max_ticks) + _, iframes = _run_capture(False, seed, max_ticks) + n = max(len(pframes), len(iframes)) + + world = psim.world + grid = world.grid + h, w = grid.shape + survivors = psim.survivors + + base = np.ones((h, w, 3)) + base[grid == CostValues.OCCUPIED] = [0.05, 0.05, 0.08] + + fig, (axp, axi) = plt.subplots(1, 2, figsize=(12, 6.2)) + fig.suptitle("PACK MIND — same robots, same start, only memory differs", fontsize=13) + + def at(frames: list[dict[str, Any]], i: int) -> dict[str, Any]: + return frames[min(i, len(frames) - 1)] + + def draw(ax: Any, frame: dict[str, Any], title: str) -> None: + ax.clear() + img = base.copy() + img[frame["visited"]] = [0.55, 0.78, 1.0] + ax.imshow(img, origin="lower") + for si, (sx, sy) in enumerate(survivors): + gc = world.world_to_grid((sx, sy)) + color = "limegreen" if si in frame["found"] else "red" + ax.plot(gc.x, gc.y, marker="*", color=color, markersize=15, markeredgecolor="k") + for rx, ry, done in frame["robots"]: + gc = world.world_to_grid((rx, ry)) + ax.plot( + gc.x, + gc.y, + marker="o", + color="gray" if done else "orange", + markersize=9, + markeredgecolor="k", + ) + ax.set_title( + f"{title}\ncoverage {frame['cov']:.0%} " + f"found {len(frame['found'])}/{len(survivors)} t={frame['tick']}", + fontsize=11, + ) + ax.set_xticks([]) + ax.set_yticks([]) + + def update(i: int) -> list[Any]: + draw(axp, at(pframes, i), "PACK · shared memory") + draw(axi, at(iframes, i), "INDEPENDENT · private") + return [] + + anim = animation.FuncAnimation(fig, update, frames=n, interval=80) + saved = out_path + try: + anim.save(out_path, writer="ffmpeg", fps=12, dpi=100) + except Exception as exc: # ffmpeg missing → GIF fallback + saved = out_path.rsplit(".", 1)[0] + ".gif" + print(f"[render] ffmpeg unavailable ({exc}); writing GIF -> {saved}") + anim.save(saved, writer="pillow", fps=12) + plt.close(fig) + return saved + + +def query_report(result: Any) -> str: + """Scripted situational report from the findings pool (the 'memory' end card).""" + lines = [f"PACK MIND situational report — {result.found}/{result.total_survivors} survivors located:"] + for f in result.findings: + lines.append(f" • survivor at ({f.survivor_xy[0]:.1f}, {f.survivor_xy[1]:.1f}) — found by {f.by} at t={f.tick}") + lines.append(f"area coverage: {result.coverage:.0%} (mode={result.mode})") + return "\n".join(lines) + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND A/B render") + p.add_argument("--out", default="pack_mind_ab.mp4") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--max-ticks", type=int, default=4000) + return p.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + path = render_ab(seed=args.seed, out_path=args.out, max_ticks=args.max_ticks) + print(f"rendered -> {path}") diff --git a/dimos/experimental/pack_mind/scripted_pack_run.py b/dimos/experimental/pack_mind/scripted_pack_run.py new file mode 100644 index 0000000000..35b7e3070e --- /dev/null +++ b/dimos/experimental/pack_mind/scripted_pack_run.py @@ -0,0 +1,195 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — live STABLE path (the can't-miss demo runner, one process per laptop). + +Drives THIS dog's identity against the REAL coordinator (no LLM, no nav planner) and +optionally moves the REAL dog via velocity teleop, so the no-overlap + inheritance +story plays on the projector dashboard with zero autonomy fragility. The coordination +story is identical with or without ``--drive``. + + # Laptop-A (alpha holds the target zone, you drop it on cue): + uv run python -m dimos.experimental.pack_mind.scripted_pack_run --role alpha --leader \\ + --target "red kit" --target-zone south --hold-zone --drop-on-enter --drive + # Laptop-B (bravo sweeps, inherits, finds): + uv run python -m dimos.experimental.pack_mind.scripted_pack_run --role bravo \\ + --target "red kit" --target-zone south --find --keep-polling --drive + +Movement (``--drive``) publishes Twist to ``/cmd_vel`` exactly like ``demo_drive_go2``, +so a ``dimos run unitree-go2-pack`` daemon must already hold the WebRTC link on this +laptop (LCM is host-local). See LIVE_RUNBOOK.md §4a. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable +import time +from typing import Any + +import requests + +_TIMEOUT = 5.0 +_POLL_SECS = 1.0 # how often a sweeper re-asks while waiting to inherit + +Driver = Callable[[], None] + + +def _post(url: str, path: str, payload: dict[str, Any]) -> dict[str, Any] | None: + try: + r = requests.post(f"{url}{path}", json=payload, timeout=_TIMEOUT) + r.raise_for_status() + data: Any = r.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + print(f" [warn] {path} failed: {exc}") + return None + + +def _get(url: str, path: str, params: dict[str, str]) -> dict[str, Any] | None: + try: + r = requests.get(f"{url}{path}", params=params, timeout=_TIMEOUT) + r.raise_for_status() + data: Any = r.json() + return data if isinstance(data, dict) else None + except (requests.RequestException, ValueError) as exc: + print(f" [warn] {path} failed: {exc}") + return None + + +def _make_driver(drive_secs: float) -> Driver: + """A /cmd_vel velocity-burst driver (host-local LCM), modelled on demo_drive_go2.""" + from dimos.core.transport import LCMTransport + from dimos.msgs.geometry_msgs.Twist import Twist + from dimos.msgs.geometry_msgs.Vector3 import Vector3 + + transport: LCMTransport[Twist] = LCMTransport("/cmd_vel", Twist) + + def burst(vx: float, wz: float, secs: float) -> None: + deadline = time.time() + secs + while time.time() < deadline: + transport.broadcast( + None, + Twist(linear=Vector3(x=vx, y=0.0, z=0.0), angular=Vector3(x=0.0, y=0.0, z=wz)), + ) + time.sleep(0.1) + transport.broadcast( + None, Twist(linear=Vector3(0.0, 0.0, 0.0), angular=Vector3(0.0, 0.0, 0.0)) + ) + + # a visible "I'm searching this zone" wiggle: forward, turn, forward. + def choreograph() -> None: + seg = max(0.4, drive_secs / 3) + burst(0.3, 0.0, seg) + burst(0.0, 0.5, seg) + burst(0.3, 0.0, seg) + + return choreograph + + +def run(args: argparse.Namespace) -> None: + url: str = args.url.rstrip("/") + dog: str = args.role + target: str = args.target + target_zone: str = args.target_zone + pace: float = args.pace + driver: Driver | None = _make_driver(args.drive_secs) if args.drive else None + + if args.leader: + status = _post(url, "/start_search", {"target": target}) + print(f" MISSION {status.get('status') if status else '(coordinator unreachable)'}") + time.sleep(pace) + + while True: + stop = _get(url, "/should_stop", {"dog": dog}) + if stop and stop.get("stop"): + print(f" [{dog}] pack found it — converging. STOP.") + return + + assigned = _post(url, "/assign_zone", {"dog": dog}) + zone = assigned.get("zone") if assigned else None + + if not zone: + if args.keep_polling: + stop = _get(url, "/should_stop", {"dog": dog}) + if stop and stop.get("stop"): + print(f" [{dog}] pack found it — STOP.") + return + print(f" [{dog}] no free zone — waiting to inherit a dropped teammate's ground…") + time.sleep(_POLL_SECS) + continue + print(f" [{dog}] no zone left — search over. STOP.") + return + + print(f" [{dog}] CLAIMS {zone}") + if driver: + driver() # visible search wiggle in place of real nav + else: + time.sleep(pace) + + # the dog that HOLDS the target zone: keep it claimed (still searching) and + # wait for the operator to drop us, so a teammate inherits it. + if args.hold_zone and zone == target_zone: + print(f" [{dog}] HOLDING {zone} (the object is here, still searching)…") + if args.drop_on_enter: + input(f" >>> press ENTER to DROP {dog} (simulate losing the robot) <<< ") + else: + time.sleep(pace * 2) + ok = _post(url, "/release_dog", {"dog": dog}) + print(f" [{dog}] OFFLINE — {ok.get('status') if ok else 'released'};" + " its ground returns to the pack.") + return + + # the finder: report the find when it reaches the target zone. + if args.find and zone == target_zone: + res = _post(url, "/report_finding", {"dog": dog, "object": target, "zone": zone}) + found = res.get("finding") if res else None + print(f" [{dog}] FOUND {target} in {zone} -> {found}. Pack STOP.") + return + + cleared = _post(url, "/report_cleared", {"dog": dog, "zone": zone}) + print(f" [{dog}] cleared {zone} (no overlap) — {cleared.get('status') if cleared else '?'}") + time.sleep(pace * 0.4) + + +def _parse() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND live stable per-laptop runner") + p.add_argument("--role", required=True, help="this dog's name, e.g. alpha") + p.add_argument("--url", default="http://127.0.0.1:8090", help="coordinator base URL") + p.add_argument("--target", default="red kit") + p.add_argument("--target-zone", default="south", help="where the object is") + p.add_argument("--leader", action="store_true", help="call start_search (resets ledger)") + p.add_argument("--hold-zone", action="store_true", + help="when assigned the target zone, hold it (don't clear) until dropped") + p.add_argument("--find", action="store_true", + help="when assigned the target zone, report the finding") + p.add_argument("--keep-polling", action="store_true", + help="when no free zone, keep re-asking (to inherit a dropped teammate's zone)") + p.add_argument("--drop-on-enter", action="store_true", + help="while holding, wait for ENTER then release this dog and exit") + p.add_argument("--drive", action="store_true", help="actually move the dog via /cmd_vel teleop") + p.add_argument("--drive-secs", type=float, default=1.5, help="seconds of motion per zone") + p.add_argument("--pace", type=float, default=2.0, help="seconds between beats (no-drive)") + return p.parse_args() + + +def main() -> None: + try: + run(_parse()) + except KeyboardInterrupt: + print("\n interrupted — coordinator state preserved.") + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/server.py b/dimos/experimental/pack_mind/server.py new file mode 100644 index 0000000000..4f8937c3ba --- /dev/null +++ b/dimos/experimental/pack_mind/server.py @@ -0,0 +1,115 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — web demo backend (S2). + +Streams two fog-of-war exploration sims (independent + shared) over a WebSocket to +the Three.js frontend, and accepts kill/reset controls. + + uv run python -m dimos.experimental.pack_mind.server + # then open http://localhost:8000 +""" + +from __future__ import annotations + +import asyncio +import json +import os +from pathlib import Path + +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles + +from dimos.experimental.pack_mind.explore_sim import build_explore, build_explore_building + +# PACK_MIND_MAP=building swaps the synthetic maze for a real DimOS SLAM floor plan. +_BUILD = build_explore_building if os.environ.get("PACK_MIND_MAP") == "building" else build_explore + +_STATIC = Path(__file__).parent / "static" +_PORT = int(os.environ.get("PACK_MIND_PORT", "8000")) +_STEPS_PER_FRAME = 5 +_FPS = 18 +_HOLD_FRAMES = 70 # hold the finished maze before auto-replaying + +app = FastAPI(title="PACK MIND — Fog of War") +app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static") + + +@app.get("/") +def index() -> FileResponse: + return FileResponse(str(_STATIC / "explore.html")) + + +def _new_sims(seed: int, n_dogs: int) -> dict: + return { + "independent": _BUILD(False, seed, n_dogs), + "shared": _BUILD(True, seed, n_dogs), + } + + +@app.websocket("/ws") +async def ws(websocket: WebSocket) -> None: + await websocket.accept() + cfg = {"seed": 0, "n_dogs": 3} + sims = _new_sims(cfg["seed"], cfg["n_dogs"]) + controls: asyncio.Queue = asyncio.Queue() + + async def receiver() -> None: + try: + while True: + controls.put_nowait(json.loads(await websocket.receive_text())) + except Exception: + pass + + rt = asyncio.create_task(receiver()) + done_frames = 0 + try: + while True: + while not controls.empty(): + c = controls.get_nowait() + if c.get("cmd") == "reset": + cfg["seed"] = int(c.get("seed", cfg["seed"])) + sims = _new_sims(cfg["seed"], cfg["n_dogs"]) + done_frames = 0 + elif c.get("cmd") == "kill": + dog = int(c.get("dog", 0)) + for s in sims.values(): + if dog < len(s.dogs): + s.set_online(dog, False) + + if all(s.all_done() for s in sims.values()): + done_frames += 1 + if done_frames > _HOLD_FRAMES: + sims = _new_sims(cfg["seed"], cfg["n_dogs"]) + done_frames = 0 + else: + for s in sims.values(): + for _ in range(_STEPS_PER_FRAME): + if not s.all_done(): + s.step() + + await websocket.send_text(json.dumps({k: v.state() for k, v in sims.items()})) + await asyncio.sleep(1 / _FPS) + except WebSocketDisconnect: + pass + finally: + rt.cancel() + + +if __name__ == "__main__": + import uvicorn + + print(f"PACK MIND web demo → http://localhost:{_PORT}") + uvicorn.run(app, host="0.0.0.0", port=_PORT) diff --git a/dimos/experimental/pack_mind/sim.py b/dimos/experimental/pack_mind/sim.py new file mode 100644 index 0000000000..3847f0627b --- /dev/null +++ b/dimos/experimental/pack_mind/sim.py @@ -0,0 +1,235 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — PackSim orchestrator + SimResult (G1/G2). + +Honest A/B experiment: PACK and INDEPENDENT use the SAME robots, SAME start +point, SAME map and survivors. The ONLY difference is whether the robots share +one VisitationHistory (the coverage memory) or each keep a private one. So any +performance gap is attributable to shared memory alone — not to start positions. + +PACK (shared=True): one shared VisitationHistory. Each robot's CoveragePatrolRouter + sees teammates' coverage and steers toward still-uncovered area → the pack + fans out from the single entrance and divides the map with little overlap. + +INDEPENDENT (shared=False): private VisitationHistory per robot. Each robot + believes the whole map is uncovered → they redundantly re-walk the same + corridors → far more ticks to full coverage. + +CLI: + python -m dimos.experimental.pack_mind.sim --mode pack --seed 0 + python -m dimos.experimental.pack_mind.sim --mode independent --seed 0 +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass, field + +import numpy as np + +from dimos.experimental.pack_mind.sim_robot import SimRobot +from dimos.experimental.pack_mind.world import free_cell_count, make_maze_world, plant_survivors +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid +from dimos.navigation.patrolling.routers.coverage_patrol_router import CoveragePatrolRouter +from dimos.navigation.patrolling.routers.visitation_history import VisitationHistory + +CLEARANCE_M = 0.1 +DETECT_R = 0.4 +_STALL_TICKS = 200 # stop a run once coverage has not improved for this many ticks +_STALL_EPS = 0.001 + + +@dataclass +class Finding: + survivor_xy: tuple[float, float] + by: str + tick: int + + +@dataclass +class SimResult: + mode: str + ticks: int + coverage: float + found: int + total_survivors: int + ticks_to_target: int | None # ticks to reach coverage_target + ticks_to_all: int | None # ticks to find every survivor + coverage_target: float = 0.75 + findings: list[Finding] = field(default_factory=list) + + def __str__(self) -> str: + tt = self.ticks_to_target if self.ticks_to_target is not None else "never" + tall = self.ticks_to_all if self.ticks_to_all is not None else "never" + return ( + f"SimResult(mode={self.mode}, ticks={self.ticks}, " + f"coverage={self.coverage:.1%}, found={self.found}/{self.total_survivors}, " + f"ticks_to_{self.coverage_target:.0%}={tt}, ticks_to_all={tall})" + ) + + +class PackSim: + """Run N robots on a shared (pack) or private (independent) coverage mission.""" + + def __init__( + self, + world: OccupancyGrid, + survivors: list[tuple[float, float]], + n_robots: int, + shared: bool, + starts: list[tuple[float, float]], + seed: int = 0, + kill_at_tick: int | None = None, + ) -> None: + np.random.seed(seed) # CoveragePatrolRouter samples candidates via np.random + self.world = world + self.survivors = survivors + self.shared = shared + self.kill_at_tick = kill_at_tick + self._free_total = free_cell_count(world) + self.findings: list[Finding] = [] + self._found: set[int] = set() + self.tick_n = 0 + + routers = [CoveragePatrolRouter(CLEARANCE_M) for _ in range(n_robots)] + if shared: + # Shared coverage memory: set the SAME VisitationHistory on every router + # BEFORE SimRobot.__init__ calls handle_occupancy_grid (which initialises + # the mask). Robot B then immediately sees robot A's footprint. + shared_vis = VisitationHistory(CLEARANCE_M) + for r in routers: + r._visitation = shared_vis + # VisitationHistory is built for *patrolling*: at 50% saturation it prunes the + # oldest half of visited points (bounded memory), capping coverage at ~50%. + # A one-shot SAR search must remember everything — disable pruning. Also widen + # candidate sampling so the router targets the last uncovered pockets. + for r in routers: + r._visitation._saturation_threshold = 10.0 + r._candidates_to_consider = 24 + + self.robots = [ + SimRobot(f"r{i}", routers[i], world, starts[i]) for i in range(n_robots) + ] + + def _coverage(self) -> float: + """Fraction of free cells covered (union of robots' visited masks).""" + union: np.ndarray | None = None + free = self.world.grid == CostValues.FREE + for rb in self.robots: + v = rb.router._visited + if v is None: + continue + union = v.copy() if union is None else (union | v) + if union is None: + return 0.0 + return float(np.count_nonzero(union & free)) / max(1, self._free_total) + + def _detect(self) -> None: + for si, s in enumerate(self.survivors): + if si in self._found: + continue + for rb in self.robots: + if rb.done: + continue + if (rb.x - s[0]) ** 2 + (rb.y - s[1]) ** 2 <= DETECT_R**2: + self._found.add(si) + self.findings.append(Finding(s, rb.name, self.tick_n)) + break + + def step(self) -> None: + self.tick_n += 1 + # Death/persistence beat: kill the first robot at the given tick. Its + # coverage + findings remain (shared mask / findings pool outlive it). + if self.kill_at_tick is not None and self.tick_n == self.kill_at_tick: + self.robots[0].done = True + for rb in self.robots: + rb.tick() + self._detect() + + def run(self, max_ticks: int = 4000, coverage_target: float = 0.75) -> SimResult: + t_target: int | None = None + tall: int | None = None + best_cov = 0.0 + stall = 0 + for _ in range(max_ticks): + self.step() + cov = self._coverage() + if t_target is None and cov >= coverage_target: + t_target = self.tick_n + if tall is None and len(self._found) == len(self.survivors): + tall = self.tick_n + if cov > best_cov + _STALL_EPS: + best_cov = cov + stall = 0 + else: + stall += 1 + if all(rb.done for rb in self.robots): + break + if stall >= _STALL_TICKS: + break + return SimResult( + "pack" if self.shared else "independent", + self.tick_n, + self._coverage(), + len(self._found), + len(self.survivors), + t_target, + tall, + coverage_target, + list(self.findings), + ) + + +def build_sim( + shared: bool, + seed: int = 0, + n_robots: int = 3, + n_survivors: int = 4, + kill_at_tick: int | None = None, +) -> PackSim: + """Build a sim. Both modes share IDENTICAL starts (single SAR entry point); + the only variable is shared vs private memory.""" + world = make_maze_world() + survivors = plant_survivors(world, n_survivors, seed=seed) + # Single deploy point at the bottom-left entrance, tiny offsets to avoid overlap. + starts = [(0.5 + 0.2 * i, 0.4) for i in range(n_robots)] + return PackSim(world, survivors, n_robots, shared, starts, seed, kill_at_tick) + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND sim CLI") + p.add_argument("--mode", choices=["pack", "independent"], default="pack") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--render", metavar="PATH", default=None) + p.add_argument("--max-ticks", type=int, default=4000) + return p.parse_args() + + +if __name__ == "__main__": + import time + + args = _parse_args() + t0 = time.perf_counter() + sim = build_sim(shared=args.mode == "pack", seed=args.seed) + result = sim.run(max_ticks=args.max_ticks) + wall = time.perf_counter() - t0 + print(result) + print(f"wall time: {wall:.1f}s") + + if args.render: + from dimos.experimental.pack_mind.render import render_ab + + render_ab(seed=args.seed, out_path=args.render) + print(f"rendered -> {args.render}") diff --git a/dimos/experimental/pack_mind/sim_perception.py b/dimos/experimental/pack_mind/sim_perception.py new file mode 100644 index 0000000000..af0e8665cd --- /dev/null +++ b/dimos/experimental/pack_mind/sim_perception.py @@ -0,0 +1,82 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim backbone — step 1: VLM perception proof (CUDA-free). + +Replaces EdgeTAM with a VLM that *describes* the camera frame, exactly like the +robot-telephone reference (ouazmourad/dimos-21-days-sprint). Runs one G1 MuJoCo +sim (no perception stack, so no CUDA) plus a VLM describer, then asks the VLM to +describe what the simulated robot sees. If this prints a sensible description, +the VLM-perception -> semantic-memory bridge works on this machine. + +macOS needs mjpython for MuJoCo. Needs OPENAI_API_KEY + internet. + + mjpython dimos/experimental/pack_mind/sim_perception.py +""" + +from __future__ import annotations + +import time + +from dimos.agents.vlm_agent import VLMAgent +from dimos.core.coordination.blueprints import autoconnect +from dimos.core.coordination.module_coordinator import ModuleCoordinator +from dimos.core.core import rpc +from dimos.core.global_config import global_config +from dimos.robot.unitree.g1.blueprints.basic.unitree_g1_basic_sim import ( + unitree_g1_basic_sim, +) + +CAMERA_WARMUP_S = 20.0 + + +class DescriberVLM(VLMAgent): + """VLMAgent + a visual_query RPC that describes the latest camera frame. + + Local VLMAgent has query() (text-only) and query_image(img, q); it lacks the + fork's visual_query, so we add it here: query using the last frame received + on the color_image stream. + """ + + @rpc + def visual_query(self, query: str) -> str: + if self._latest_image is None: + return "No image yet — camera not streaming." + response = self._invoke_image(self._latest_image, query) + content = response.content + return content if isinstance(content, str) else str(content) + + +def main() -> None: + global_config.update(simulation="mujoco") + blueprint = autoconnect(unitree_g1_basic_sim, DescriberVLM.blueprint(model="gpt-4o")) + coordinator = ModuleCoordinator.build(blueprint) + try: + print(f"[pack-mind] sim up; warming camera {CAMERA_WARMUP_S:.0f}s...", flush=True) + time.sleep(CAMERA_WARMUP_S) + vlm = coordinator.get_instance(DescriberVLM) + prompt = ( + "Look at the scene and pick the most prominent object. Describe its " + "color, shape, size, and position relative to other objects in 2-3 " + "sentences. Do not name it directly — only its visual properties." + ) + print("[pack-mind] querying VLM with live sim frame...\n", flush=True) + print("VLM DESCRIPTION:\n" + vlm.visual_query(prompt), flush=True) + finally: + print("\n[pack-mind] shutting down sim...", flush=True) + coordinator.stop() + + +if __name__ == "__main__": + main() diff --git a/dimos/experimental/pack_mind/sim_robot.py b/dimos/experimental/pack_mind/sim_robot.py new file mode 100644 index 0000000000..03555de4cd --- /dev/null +++ b/dimos/experimental/pack_mind/sim_robot.py @@ -0,0 +1,96 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — SimRobot: one autonomous agent in the sim loop. + +Uses CoveragePatrolRouter (scores candidate goals by NEW coverage along the A* +path) instead of FrontierPatrolRouter (which thrashed: farthest-point goals laid +thin re-walked trails and plateaued ~41%). The router reads its `_visitation` +mask — shared across robots in PACK mode, private in INDEPENDENT mode — so the +only thing that changes between the two experiments is whether memory is shared. +""" + +from __future__ import annotations + +from math import hypot + +from dimos.experimental.pack_mind.world import pose_at +from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid +from dimos.navigation.patrolling.routers.coverage_patrol_router import CoveragePatrolRouter +from dimos.navigation.replanning_a_star.min_cost_astar import min_cost_astar + +_MAX_GOAL_FAILURES = 3 # consecutive next_goal/plan failures before the robot stops + + +class SimRobot: + """One simulated robot: coverage router + A* path follower.""" + + def __init__( + self, + name: str, + router: CoveragePatrolRouter, + world: OccupancyGrid, + start_xy: tuple[float, float], + step_m: float = 0.12, + ) -> None: + self.name = name + self.router = router + self.world = world + self.step_m = step_m + self.x, self.y = start_xy + self.path: list[tuple[float, float]] | None = None + self.idx = 0 + self.done = False + self._failures = 0 + # Throttle: handle_occupancy_grid is rate-limited to once/60s, so call + # it exactly once per router here at init. + self.router.handle_occupancy_grid(world) + self.router.handle_odom(pose_at(self.x, self.y)) + + def _request_goal(self) -> bool: + g = self.router.next_goal() + if g is None: + return False + p = min_cost_astar( + self.world, + (g.position.x, g.position.y), + (self.x, self.y), + use_cpp=False, + ) + if p is None or len(p.poses) == 0: + return False + self.path = [(po.position.x, po.position.y) for po in p.poses] + self.idx = 0 + return True + + def tick(self) -> None: + if self.done: + return + if self.path is None or self.idx >= len(self.path): + if not self._request_goal(): + self._failures += 1 + if self._failures >= _MAX_GOAL_FAILURES: + self.done = True + return + self._failures = 0 + tx, ty = self.path[self.idx] + dx, dy = tx - self.x, ty - self.y + d = hypot(dx, dy) + if d <= self.step_m or d == 0: + self.x, self.y = tx, ty + self.idx += 1 + else: + self.x += self.step_m * dx / d + self.y += self.step_m * dy / d + self.router.handle_odom(pose_at(self.x, self.y)) # stamps the visited mask diff --git a/dimos/experimental/pack_mind/static/explore.html b/dimos/experimental/pack_mind/static/explore.html new file mode 100644 index 0000000000..a3899f6b8f --- /dev/null +++ b/dimos/experimental/pack_mind/static/explore.html @@ -0,0 +1,215 @@ + + + + + +PACK MIND — Fog of War + + + + +
+
PACK MIND · fog-of-war exploration
+ + +
connecting…
+
+
+
+
INDEPENDENT · private memory
+
+
+
+
+
PACK · shared memory
+
+
+
+
+ + + + diff --git a/dimos/experimental/pack_mind/test_explore_sim.py b/dimos/experimental/pack_mind/test_explore_sim.py new file mode 100644 index 0000000000..d0eb73b2a0 --- /dev/null +++ b/dimos/experimental/pack_mind/test_explore_sim.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — fog-of-war exploration engine tests (S1).""" + +import pytest + +from dimos.experimental.pack_mind.explore_sim import build_explore + + +@pytest.mark.unit +def test_reveal_grows() -> None: + sim = build_explore(shared=True, n_dogs=1) + for _ in range(50): + sim.step() + early = sim.revealed_frac() + for _ in range(150): + sim.step() + late = sim.revealed_frac() + assert late > early + 0.1 and late > 0.3 + + +@pytest.mark.unit +def test_shared_reveals_faster_than_independent() -> None: + s = build_explore(shared=True, seed=0).run() + i = build_explore(shared=False, seed=0).run() + assert s["ticks_to_target"] is not None + assert i["ticks_to_target"] is None or s["ticks_to_target"] < i["ticks_to_target"] + + +@pytest.mark.unit +def test_independent_loses_offline_dog_knowledge() -> None: + # INDEPENDENT: team fog = union of ONLINE dogs. Killing a dog that has + # explored unique territory drops the team's knowledge. + sim = build_explore(shared=False, n_dogs=3, seed=0) + for _ in range(140): + sim.step() + before = sim.revealed_frac() + sim.set_online(0, False) + after = sim.revealed_frac() + assert after < before - 0.1 # the dead dog's region leaves the team's knowledge + + +@pytest.mark.unit +def test_shared_keeps_offline_dog_knowledge() -> None: + # SHARED: the one map persists regardless of who is online. + sim = build_explore(shared=True, n_dogs=3, seed=0) + for _ in range(140): + sim.step() + before = sim.revealed_frac() + sim.set_online(0, False) + after = sim.revealed_frac() + assert after == before diff --git a/dimos/experimental/pack_mind/test_pack_coordinator.py b/dimos/experimental/pack_mind/test_pack_coordinator.py new file mode 100644 index 0000000000..3183259899 --- /dev/null +++ b/dimos/experimental/pack_mind/test_pack_coordinator.py @@ -0,0 +1,153 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND coordinator tests — the no-overlap and stop-on-found guarantees.""" + +import pytest + +from dimos.experimental.pack_mind.pack_coordinator import PackCoordinator + +ZONES = ["north", "east", "south", "west"] + + +@pytest.mark.unit +def test_assign_never_double_assigns_a_zone() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + handed: list[str] = [] + for _ in range(len(ZONES)): + for dog in ("alpha", "bravo"): + z = c.assign_zone(dog) + if z is not None: + handed.append(z) + assert sorted(handed) == sorted(ZONES) # every zone handed exactly once + assert len(handed) == len(set(handed)) # no zone given twice → no overlap + + +@pytest.mark.unit +def test_assign_returns_none_when_exhausted() -> None: + c = PackCoordinator(["only"]) + c.start_search("red object") + assert c.assign_zone("alpha") == "only" + assert c.assign_zone("bravo") is None # nothing left for the second dog + + +@pytest.mark.unit +def test_finding_stops_the_pack() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + c.assign_zone("alpha") # claims a zone + assert not c.found + c.report_finding("alpha", "red object", "north") + assert c.found + assert c.should_stop("bravo") # the other dog learns instantly + assert c.assign_zone("bravo") is None # no more assignments after a find + + +@pytest.mark.unit +def test_finding_is_idempotent_keeps_first() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + first = c.report_finding("alpha", "red object", "north") + second = c.report_finding("bravo", "red object", "south") + assert second is first # first finding wins; pack already stopped + + +@pytest.mark.unit +def test_preferences_make_dogs_fan_out() -> None: + # Alpha prefers north end, Bravo the south end → first picks diverge. + prefs = {"alpha": ["north", "east"], "bravo": ["south", "west"]} + c = PackCoordinator(ZONES, preferences=prefs) + c.start_search("red object") + assert c.assign_zone("alpha") == "north" + assert c.assign_zone("bravo") == "south" + + +@pytest.mark.unit +def test_coverage_and_cleared() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + assert c.coverage() == 0.0 + c.report_cleared("alpha", "north") + c.report_cleared("bravo", "south") + assert c.coverage() == pytest.approx(0.5) + + +@pytest.mark.unit +def test_report_finding_falls_back_to_claimed_zone() -> None: + # The agent may report a finding with a wrong/empty zone (LLM-filled). The + # coordinator should fall back to the zone the dog actually holds claimed. + c = PackCoordinator(ZONES) + c.start_search("red object") + claimed = c.assign_zone("alpha") + f = c.report_finding("alpha", "red object", "") # empty zone + assert f.zone == claimed + c2 = PackCoordinator(ZONES) + c2.start_search("red object") + claimed2 = c2.assign_zone("alpha") + f2 = c2.report_finding("alpha", "red object", "not_a_real_zone") # bogus zone + assert f2.zone == claimed2 + + +@pytest.mark.unit +def test_release_dog_lets_survivor_inherit_unfinished_zone() -> None: + # The resilience climax: Alpha claims a zone, goes offline before clearing it; + # its zone returns to the pool so Bravo inherits and finishes the mission. + c = PackCoordinator(["north", "south"]) + c.start_search("red object") + alpha_zone = c.assign_zone("alpha") # alpha claims one + bravo_zone = c.assign_zone("bravo") # bravo claims the other + assert {alpha_zone, bravo_zone} == {"north", "south"} + # Bravo finishes its own zone, then would have nothing left... + c.report_cleared("bravo", bravo_zone) + assert c.assign_zone("bravo") is None # alpha still holds the last zone + # Alpha drops. Its claimed-but-uncleared zone must come back to the pack. + c.release_dog("alpha") + assert c.assign_zone("bravo") == alpha_zone # Bravo inherits alpha's ground + + +@pytest.mark.unit +def test_release_dog_keeps_findings_and_cleared_zones() -> None: + c = PackCoordinator(["north", "south", "east"]) + c.start_search("red object") + c.report_cleared("alpha", "north") # alpha already cleared one + z = c.assign_zone("alpha") # alpha claims another, unfinished + c.release_dog("alpha") + snap = c.snapshot() + states = {zz["name"]: zz["state"] for zz in snap["zones"]} # type: ignore[index] + assert states["north"] == "cleared" # cleared knowledge persists + assert states[z] == "unsearched" # unfinished zone reclaimed + assert "alpha" in snap["offline"] # type: ignore[operator] + + +@pytest.mark.unit +def test_finding_survives_finder_going_offline() -> None: + c = PackCoordinator(["north", "south"]) + c.start_search("red object") + c.report_finding("alpha", "red object", "north") + c.release_dog("alpha") # finder drops + assert c.found # the find outlives the robot + assert c.finding is not None and c.finding.zone == "north" + + +@pytest.mark.unit +def test_start_search_resets_ledger() -> None: + c = PackCoordinator(ZONES) + c.start_search("red object") + c.report_cleared("alpha", "north") + c.report_finding("alpha", "red object", "east") + c.start_search("blue box") # new mission + assert not c.found + assert c.coverage() == 0.0 + assert c.assign_zone("alpha") is not None diff --git a/dimos/experimental/pack_mind/test_pack_coordinator_server.py b/dimos/experimental/pack_mind/test_pack_coordinator_server.py new file mode 100644 index 0000000000..14bf5b9103 --- /dev/null +++ b/dimos/experimental/pack_mind/test_pack_coordinator_server.py @@ -0,0 +1,181 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND coordinator server tests — drives the real HTTP server over requests. + +Starts the actual ``ThreadingHTTPServer`` on an ephemeral port (port 0) in a +background thread, exercising the no-overlap and stop-on-find guarantees through +the JSON API exactly as a dog's agent would. +""" + +from __future__ import annotations + +from collections.abc import Iterator +import threading + +import pytest +import requests + +from dimos.experimental.pack_mind.pack_coordinator_server import ( + _parse_prefs, + _Server, + make_server, +) + +ZONES = ["north", "east", "south", "west"] +_TIMEOUT = 5.0 + + +@pytest.fixture +def server_url() -> Iterator[str]: + server: _Server = make_server("127.0.0.1", 0, ZONES) + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{port}" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=_TIMEOUT) + + +@pytest.mark.unit +def test_two_dogs_get_distinct_zones(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red object"}, timeout=_TIMEOUT) + handed: list[str] = [] + for _ in range(len(ZONES)): + for dog in ("alpha", "bravo"): + resp = requests.post( + f"{server_url}/assign_zone", json={"dog": dog}, timeout=_TIMEOUT + ) + assert resp.status_code == 200 + zone = resp.json()["zone"] + if zone is not None: + handed.append(zone) + assert sorted(handed) == sorted(ZONES) # every zone handed exactly once + assert len(handed) == len(set(handed)) # no double-assign → no overlap + + +@pytest.mark.unit +def test_finding_stops_other_dog_and_assignment(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red object"}, timeout=_TIMEOUT) + requests.post(f"{server_url}/assign_zone", json={"dog": "alpha"}, timeout=_TIMEOUT) + + find = requests.post( + f"{server_url}/report_finding", + json={"dog": "alpha", "object": "red object", "zone": "north"}, + timeout=_TIMEOUT, + ) + assert find.status_code == 200 + assert find.json()["finding"]["zone"] == "north" + + stop = requests.get(f"{server_url}/should_stop", params={"dog": "bravo"}, timeout=_TIMEOUT) + assert stop.status_code == 200 + assert stop.json()["stop"] is True + + assigned = requests.post( + f"{server_url}/assign_zone", json={"dog": "bravo"}, timeout=_TIMEOUT + ) + assert assigned.json()["zone"] is None # no more zones handed after a find + + +@pytest.mark.unit +def test_state_reflects_coverage_after_cleared(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red object"}, timeout=_TIMEOUT) + assert requests.get(f"{server_url}/state", timeout=_TIMEOUT).json()["coverage"] == 0.0 + + requests.post( + f"{server_url}/report_cleared", json={"dog": "alpha", "zone": "north"}, timeout=_TIMEOUT + ) + requests.post( + f"{server_url}/report_cleared", json={"dog": "bravo", "zone": "south"}, timeout=_TIMEOUT + ) + state = requests.get(f"{server_url}/state", timeout=_TIMEOUT).json() + assert state["coverage"] == pytest.approx(0.5) + + +@pytest.mark.unit +def test_release_dog_reclaims_zone_over_http(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red object"}, timeout=_TIMEOUT) + alpha = requests.post( + f"{server_url}/assign_zone", json={"dog": "alpha"}, timeout=_TIMEOUT + ).json()["zone"] + bravo = requests.post( + f"{server_url}/assign_zone", json={"dog": "bravo"}, timeout=_TIMEOUT + ).json()["zone"] + requests.post( + f"{server_url}/report_cleared", json={"dog": "bravo", "zone": bravo}, timeout=_TIMEOUT + ) + # Alpha drops; its claimed zone must come back so a survivor can inherit it. + rel = requests.post(f"{server_url}/release_dog", json={"dog": "alpha"}, timeout=_TIMEOUT) + assert rel.status_code == 200 + inherited = requests.post( + f"{server_url}/assign_zone", json={"dog": "bravo"}, timeout=_TIMEOUT + ).json()["zone"] + assert inherited == alpha # bravo inherits alpha's reclaimed ground + assert "alpha" in requests.get(f"{server_url}/state", timeout=_TIMEOUT).json()["offline"] + + +@pytest.mark.unit +def test_where_is_returns_finding(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red kit"}, timeout=_TIMEOUT) + assert requests.get(f"{server_url}/where_is", timeout=_TIMEOUT).json()["found"] is False + requests.post( + f"{server_url}/report_finding", + json={"dog": "alpha", "object": "red kit", "zone": "east"}, + timeout=_TIMEOUT, + ) + w = requests.get(f"{server_url}/where_is", timeout=_TIMEOUT).json() + assert w["found"] is True and w["zone"] == "east" and w["by"] == "alpha" + + +@pytest.mark.unit +def test_report_finding_zone_optional_over_http(server_url: str) -> None: + requests.post(f"{server_url}/start_search", json={"target": "red kit"}, timeout=_TIMEOUT) + claimed = requests.post( + f"{server_url}/assign_zone", json={"dog": "alpha"}, timeout=_TIMEOUT + ).json()["zone"] + # No zone field at all — coordinator falls back to alpha's claimed zone. + f = requests.post( + f"{server_url}/report_finding", + json={"dog": "alpha", "object": "red kit"}, + timeout=_TIMEOUT, + ) + assert f.status_code == 200 + assert f.json()["finding"]["zone"] == claimed + + +@pytest.mark.unit +def test_dashboard_served_at_root(server_url: str) -> None: + resp = requests.get(f"{server_url}/", timeout=_TIMEOUT) + assert resp.status_code == 200 + assert "text/html" in resp.headers.get("Content-Type", "") + assert "PACK" in resp.text and "/state" in resp.text # the polling dashboard + + +@pytest.mark.unit +def test_parse_prefs() -> None: + prefs = _parse_prefs("alpha:north,east;bravo:south,west") + assert prefs == {"alpha": ["north", "east"], "bravo": ["south", "west"]} + assert _parse_prefs("") == {} + assert _parse_prefs("garbage") == {} + + +@pytest.mark.unit +def test_bad_input_returns_400(server_url: str) -> None: + resp = requests.post(f"{server_url}/start_search", json={}, timeout=_TIMEOUT) + assert resp.status_code == 400 + missing = requests.get(f"{server_url}/should_stop", timeout=_TIMEOUT) + assert missing.status_code == 400 diff --git a/dimos/experimental/pack_mind/test_pack_mind_sim.py b/dimos/experimental/pack_mind/test_pack_mind_sim.py new file mode 100644 index 0000000000..38aaad41f9 --- /dev/null +++ b/dimos/experimental/pack_mind/test_pack_mind_sim.py @@ -0,0 +1,65 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — G1+G2 tests. + +Run: bin/pytest-fast dimos/experimental/pack_mind/test_pack_mind_sim.py -v +""" + +import pytest + +from dimos.experimental.pack_mind.sim import build_sim +from dimos.experimental.pack_mind.world import free_cell_count, make_maze_world, plant_survivors + + +@pytest.mark.unit +def test_world_has_free_and_walls() -> None: + w = make_maze_world() + assert free_cell_count(w) > 0 + assert (w.grid == 100).any() # walls exist + + +@pytest.mark.unit +def test_survivors_on_free_cells() -> None: + w = make_maze_world() + s = plant_survivors(w, 4, seed=0) + assert len(s) == 4 + + +@pytest.mark.unit +def test_single_robot_substantial_coverage() -> None: + # GATE 0: the map is 100% coverable (no reachability wall), so one robot must + # cover the bulk of it. The coverage-patrol heuristic plateaus ~79% solo; 3 + # robots fill the nooks. We only need to prove there is no reachability wall. + sim = build_sim(shared=False, n_robots=1, n_survivors=1, seed=0) + r = sim.run(max_ticks=6000) + assert r.coverage > 0.7 + + +@pytest.mark.unit +def test_ab_pack_beats_independent_on_coverage() -> None: + """THE credibility test. Same robots, same single start, same map — the ONLY + difference is shared vs private coverage memory. Pack reaches the coverage + target in far fewer ticks and ends with higher coverage.""" + pack = build_sim(shared=True, seed=0).run() + indep = build_sim(shared=False, seed=0).run() + assert pack.ticks_to_target is not None + assert indep.ticks_to_target is None or pack.ticks_to_target < indep.ticks_to_target + assert pack.coverage > indep.coverage + # NOTE: we deliberately do NOT assert pack finds survivors faster. Independent + # robots blanket the map redundantly and can stumble onto victims sooner; the + # shared-memory win is *complete-area coverage* (certainty nothing is missed), + # not victim-stumble speed. Both find all survivors here; timing is reported, + # not asserted. + assert pack.found == pack.total_survivors diff --git a/dimos/experimental/pack_mind/test_pack_search_runner.py b/dimos/experimental/pack_mind/test_pack_search_runner.py new file mode 100644 index 0000000000..2cd08ada6b --- /dev/null +++ b/dimos/experimental/pack_mind/test_pack_search_runner.py @@ -0,0 +1,84 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND runner tests — both magic beats end-to-end, hardware-free.""" + +import pytest + +from dimos.experimental.pack_mind.pack_coordinator import PackCoordinator +from dimos.experimental.pack_mind.pack_search_runner import ( + MockDriver, + act_on_finding, + run_search, +) + +ZONES = ["north", "east", "south", "west"] + + +@pytest.mark.unit +def test_single_dog_finds_target() -> None: + c = PackCoordinator(ZONES) + c.start_search("red kit") + res = run_search(c, MockDriver(target_zone="south"), "alpha", "red kit") + assert res.found and res.zone == "south" + assert c.found + + +@pytest.mark.unit +def test_two_dogs_split_work_and_one_finds() -> None: + # Preferences fan them out; only one zone holds the target. + prefs = {"alpha": ["north", "east"], "bravo": ["west", "south"]} + c = PackCoordinator(ZONES, preferences=prefs) + c.start_search("red kit") + # Interleave is unnecessary: whichever reaches "south" reports it and the other stops. + a = run_search(c, MockDriver(target_zone="south"), "alpha", "red kit") + b = run_search(c, MockDriver(target_zone="south"), "bravo", "red kit") + assert c.found + # No zone searched by both dogs (no overlap). + assert not (set(a.visited) & set(b.visited)) + + +@pytest.mark.unit +def test_handoff_acts_on_teammates_memory() -> None: + c = PackCoordinator(ZONES) + c.start_search("red kit") + run_search(c, MockDriver(target_zone="north"), "alpha", "red kit") # alpha finds + bravo_driver = MockDriver(target_zone="north") + msg = act_on_finding(c, bravo_driver, "bravo") # bravo never searched + assert "north" in msg and "alpha" in msg + assert bravo_driver.visited[-1] == "north" # bravo physically went on alpha's memory + + +@pytest.mark.unit +def test_inheritance_survivor_finishes_downed_dogs_mission() -> None: + # Alpha claims the target's zone but goes offline before clearing it; Bravo, + # already out of its own zones, inherits it and completes the mission. + prefs = {"alpha": ["south"], "bravo": ["north", "east", "west"]} + c = PackCoordinator(ZONES, preferences=prefs) + c.start_search("red kit") + alpha_zone = c.assign_zone("alpha") # alpha claims "south" (where the kit is) + assert alpha_zone == "south" + # Bravo clears its three zones, none have the kit. + bravo = MockDriver(target_zone="south") + for _ in range(3): + z = c.assign_zone("bravo") + assert z is not None + bravo.goto(z) + c.report_cleared("bravo", z) + assert c.assign_zone("bravo") is None # alpha still holds "south" + # Alpha drops. Bravo's loop resumes and inherits "south", finding the kit. + c.release_dog("alpha") + res = run_search(c, bravo, "bravo", "red kit") + assert res.found and res.zone == "south" + assert c.finding is not None and c.finding.by == "bravo" diff --git a/dimos/experimental/pack_mind/test_red_detector.py b/dimos/experimental/pack_mind/test_red_detector.py new file mode 100644 index 0000000000..4ce984f535 --- /dev/null +++ b/dimos/experimental/pack_mind/test_red_detector.py @@ -0,0 +1,66 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the GPU-free red-object detector (the demo's reliable find path). + +These cover the detection logic (``red_fraction``). The detector→coordinator glue +(``look_for_red`` posting a finding with the coordinator filling the claimed zone) +is covered by the coordinator server tests (report_finding + zone fallback); the +full chain was also verified manually end-to-end. We don't construct the +``RedObjectDetector`` Module here because instantiating a DimOS Module standalone +leaks framework threads that the suite's thread-leak guard rejects. +""" + +import numpy as np +import pytest + +from dimos.experimental.pack_mind.red_detector import DEFAULT_RED_THRESHOLD, red_fraction + + +def _frame(rgb: tuple[int, int, int], h: int = 48, w: int = 64) -> np.ndarray: + img = np.zeros((h, w, 3), dtype=np.uint8) + img[:] = rgb + return img + + +@pytest.mark.unit +def test_all_red_frame_is_fully_red() -> None: + assert red_fraction(_frame((220, 20, 20))) == pytest.approx(1.0) + + +@pytest.mark.unit +def test_blue_and_green_and_gray_are_not_red() -> None: + for rgb in [(20, 20, 220), (20, 220, 20), (128, 128, 128), (0, 0, 0), (255, 255, 255)]: + assert red_fraction(_frame(rgb)) == 0.0 + + +@pytest.mark.unit +def test_partial_red_patch_crosses_threshold() -> None: + img = _frame((130, 130, 130)) # neutral background + img[:, :20] = (230, 30, 30) # a red object filling ~31% of the width + frac = red_fraction(img) + assert frac > DEFAULT_RED_THRESHOLD + assert frac == pytest.approx(20 / 64, abs=0.01) + + +@pytest.mark.unit +def test_tiny_red_speck_below_threshold() -> None: + img = _frame((130, 130, 130)) + img[0, 0] = (230, 30, 30) # a single red pixel + assert red_fraction(img) < DEFAULT_RED_THRESHOLD + + +@pytest.mark.unit +def test_grayscale_or_bad_shape_is_safe() -> None: + assert red_fraction(np.zeros((10, 10), dtype=np.uint8)) == 0.0 diff --git a/dimos/experimental/pack_mind/velocity_teleop.py b/dimos/experimental/pack_mind/velocity_teleop.py new file mode 100644 index 0000000000..4434cdbf07 --- /dev/null +++ b/dimos/experimental/pack_mind/velocity_teleop.py @@ -0,0 +1,83 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND — direct velocity teleop for the live demo. + +``relative_move`` is goal-based: it runs the A* planner and then waits on an +inter-module RPC (``ReplanningAStarPlanner/is_goal_reached``). On a CPU/macOS host +that RPC over LCM is flaky and can hang for 120s — so the dog reaches the goal but +the call never returns, freezing the driver. + +This bypasses all of that: publish a ``Twist`` straight to ``MovementManager``'s +``tele_cmd_vel`` (its keyboard-teleop lane, which takes priority and forwards to the +dog). The dog moves the instant the Twist lands — no goal, no confirmation round +trip. Drive in short bursts; ``drive`` auto-stops after ``duration``. +""" + +from __future__ import annotations + +import time + +from dimos.agents.annotation import skill +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.core.stream import Out +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +_MAX_DURATION = 3.0 # safety cap on a single burst + + +class VelocityTeleop(Module): + """Publishes velocity bursts to MovementManager's teleop lane (no planner).""" + + tele_cmd_vel: Out[Twist] + + @rpc + def start(self) -> None: + super().start() + + def _publish(self, forward: float, turn: float) -> None: + self.tele_cmd_vel.publish( + Twist(linear=Vector3(forward, 0.0, 0.0), angular=Vector3(0.0, 0.0, turn)) + ) + + @skill + def drive(self, forward: float = 0.0, turn: float = 0.0, duration: float = 0.6) -> str: + """Drive the robot by velocity for a short burst, then stop. Instant — no + path planner. Use this for keyboard teleop instead of relative_move. + + Args: + forward: Linear speed in m/s. Positive = forward, negative = back. + turn: Angular speed in rad/s. Positive = turn left, negative = right. + duration: Seconds to move before auto-stopping (capped at 3s). + """ + duration = max(0.0, min(duration, _MAX_DURATION)) + self._publish(forward, turn) + time.sleep(duration) + self._publish(0.0, 0.0) # stop + logger.info("teleop drive", forward=forward, turn=turn, duration=round(duration, 2)) + return f"drove forward={forward} turn={turn} for {duration:.1f}s" + + @skill + def halt(self) -> str: + """Stop the robot immediately.""" + self._publish(0.0, 0.0) + return "stopped" + + +velocity_teleop = VelocityTeleop.blueprint diff --git a/dimos/experimental/pack_mind/view_explore_rerun.py b/dimos/experimental/pack_mind/view_explore_rerun.py new file mode 100644 index 0000000000..c29c1c7651 --- /dev/null +++ b/dimos/experimental/pack_mind/view_explore_rerun.py @@ -0,0 +1,231 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Watch a pack of dogs search a maze with PACK MIND in the DimOS Viewer (Rerun). + +One maze, one shared discovered map. Every dog reveals into it via a raycast +sensor (walls block line of sight) and plans toward frontiers; the shared map +de-conflicts them so the pack fans out instead of re-walking each other's ground. +Scrub the "tick" timeline to watch the fog peel back, and drop ``maze/revealed`` +into a time-series view to see coverage climb to 100%. + + python -m dimos.experimental.pack_mind.view_explore_rerun + python -m dimos.experimental.pack_mind.view_explore_rerun --dogs 3 --seed 0 + python -m dimos.experimental.pack_mind.view_explore_rerun --kill 2 120 # offline dog2 at tick 120 + python -m dimos.experimental.pack_mind.view_explore_rerun --save run.rrd # headless, no GUI +""" + +from __future__ import annotations + +import argparse +from typing import Any + +import numpy as np +import rerun as rr + +from dimos.experimental.pack_mind.explore_sim import ( + ExploreSim, + build_explore, + build_explore_building, +) +from dimos.visualization.rerun.init import rerun_init + +# Per-cell fog palette, indexed by state() code: +# 0 unknown · 1 remembered-free · 2 visible-free · 3 remembered-wall · 4 visible-wall +_PALETTE = np.array( + [ + [12, 12, 20], # 0 unknown — the fog + [64, 80, 110], # 1 remembered free (out of current line of sight) + [205, 228, 255], # 2 visible free (currently sensed) + [38, 40, 58], # 3 remembered wall + [150, 130, 95], # 4 visible wall + ], + dtype=np.uint8, +) +_OFFLINE = [110, 110, 110] +_SNAP_EVERY = 4 # capture a frame every N ticks (keeps playback smooth + .rrd small) + + +def _set_tick(tick: int) -> None: + # rerun's time API name shifts across versions; support both. + try: + rr.set_time_sequence("tick", tick) # type: ignore[attr-defined] + except AttributeError: + rr.set_time("tick", sequence=tick) # type: ignore[attr-defined] + + +def _hex_rgb(hex_color: str) -> list[int]: + h = hex_color.lstrip("#") + return [int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16)] + + +def _codes_rgb(codes: np.ndarray) -> np.ndarray: + """Map an (h, w) array of fog codes to an RGB image, flipped so +y points up.""" + return np.flipud(_PALETTE[codes]) + + +def _build(map_name: str, n_dogs: int, seed: int, target_label: str, shared: bool) -> ExploreSim: + """Pack on the chosen arena, converging once the object is seen. + + shared=True is PACK MIND (one shared discovered map); shared=False gives each + dog a private map (the A/B baseline that re-walks each other's ground).""" + builder = build_explore_building if map_name == "office" else build_explore + return builder( + shared=shared, seed=seed, n_dogs=n_dogs, target_label=target_label, converge_on_found=True + ) + + +def _capture(sim: ExploreSim, max_ticks: int, kill: tuple[int, int] | None) -> list[dict[str, Any]]: + frames: list[dict[str, Any]] = [] + + def snap() -> None: + st = sim.state() + codes = np.array(st["cells"], dtype=np.uint8).reshape(st["h"], st["w"]) + frames.append( + { + "tick": sim.tick_n, + "codes": codes, + "revealed": st["revealed"], + "dogs": [(d.x, d.y, d.color, d.online) for d in sim.dogs], + "trails": [list(d.trail) for d in sim.dogs], + "target": st["target"], + } + ) + + snap() + for _ in range(max_ticks): + if kill is not None and sim.tick_n == kill[1]: + sim.set_online(kill[0], False) + sim.step() + if sim.tick_n % _SNAP_EVERY == 0: + snap() + if sim.all_done(): + break + snap() + return frames + + +def _log_run(root: str, metric: str, sim: ExploreSim, frames: list[dict[str, Any]]) -> None: + h = frames[0]["codes"].shape[0] + world = sim.world + label = sim.target_label + + def to_img_xy(wx: float, wy: float) -> tuple[float, float]: + g = world.world_to_grid((wx, wy)) + return (float(g.x), float((h - 1) - g.y)) # match the vertical flip + + for fr in frames: + _set_tick(fr["tick"]) + rr.log(f"{root}/fog", rr.Image(_codes_rgb(fr["codes"]))) + + pts = [to_img_xy(x, y) for x, y, _, _ in fr["dogs"]] + cols = [_hex_rgb(color) if online else _OFFLINE for _, _, color, online in fr["dogs"]] + rr.log(f"{root}/fog/dogs", rr.Points2D(pts, colors=cols, radii=1.6)) + + strips, scols = [], [] + for (_, _, color, _), trail in zip(fr["dogs"], fr["trails"]): + if len(trail) > 1: + strips.append([to_img_xy(px, py) for px, py in trail]) + scols.append(_hex_rgb(color)) + if strips: + rr.log(f"{root}/fog/trails", rr.LineStrips2D(strips, colors=scols, radii=0.35)) + + tgt = fr["target"] + if tgt is not None: + tcol = [[60, 220, 90]] if tgt["found"] else [[235, 40, 40]] + rr.log(f"{root}/fog/target", rr.Points2D([to_img_xy(tgt["x"], tgt["y"])], colors=tcol, radii=2.4)) + + rr.log(metric, rr.Scalars(fr["revealed"])) + if tgt is not None and tgt["found"]: + msg = f"FOUND {label} — {tgt['found_by']} @ tick {tgt['found_tick']} · searched {fr['revealed']:.0%}" + else: + msg = f"searching for {label} … searched {fr['revealed']:.0%} · tick {fr['tick']}" + rr.log(f"{root}/stats", rr.TextLog(msg)) + + +def view( + map_name: str = "maze", + n_dogs: int = 3, + seed: int = 0, + target_label: str = "red object", + max_ticks: int = 4000, + kill: tuple[int, int] | None = None, + save: str | None = None, + shared: bool = True, +) -> None: + mode = "shared" if shared else "independent" + rerun_init(app_id=f"pack_mind_explore_{mode}") + if save: + rr.save(save) + else: + rr.spawn() # opens the DimOS Viewer (Rerun) window + + sim = _build(map_name, n_dogs, seed, target_label, shared) + frames = _capture(sim, max_ticks, kill) + _log_run("maze", "maze/revealed", sim, frames) + + tgt = sim.state()["target"] + if tgt is not None and tgt["found"]: + print( + f"{n_dogs} dogs searched the {map_name}; {tgt['found_by']} saw the {target_label} " + f"at tick {tgt['found_tick']} (searched {sim.revealed_frac():.0%})." + ) + else: + print( + f"{n_dogs} dogs searched {sim.revealed_frac():.0%} of the {map_name} in " + f"{sim.tick_n} ticks; {target_label} not seen." + ) + if save: + print(f"saved Rerun recording -> {save} (open with: rerun {save})") + else: + print("DimOS Viewer launched. Scrub the 'tick' timeline to watch the pack search.") + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND — a pack searches for an object, viewed in Rerun") + p.add_argument("--map", choices=("maze", "office"), default="maze", help="search arena") + p.add_argument("--dogs", type=int, default=3, help="number of dogs in the pack") + p.add_argument("--target", default="red object", help="object descriptor to search for") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--max-ticks", type=int, default=4000) + p.add_argument( + "--kill", + nargs=2, + type=int, + default=None, + metavar=("DOG", "TICK"), + help="take DOG offline at TICK (resilience beat); shared keeps its searched area", + ) + p.add_argument("--save", default=None, metavar="PATH", help="write .rrd instead of opening GUI") + p.add_argument( + "--independent", + action="store_true", + help="give each dog a private map (A/B baseline); default is PACK MIND shared memory", + ) + return p.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + kill = (args.kill[0], args.kill[1]) if args.kill else None + view( + map_name=args.map, + n_dogs=args.dogs, + seed=args.seed, + target_label=args.target, + max_ticks=args.max_ticks, + kill=kill, + save=args.save, + shared=not args.independent, + ) diff --git a/dimos/experimental/pack_mind/view_rerun.py b/dimos/experimental/pack_mind/view_rerun.py new file mode 100644 index 0000000000..d974619457 --- /dev/null +++ b/dimos/experimental/pack_mind/view_rerun.py @@ -0,0 +1,116 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""View the PACK MIND A/B coverage race in the DimOS Viewer (Rerun). + +Streams both runs into one Rerun recording on a shared "tick" timeline so you can +scrub/playback and watch PACK (shared memory) pull ahead of INDEPENDENT (private). + + python -m dimos.experimental.pack_mind.view_rerun --seed 0 + python -m dimos.experimental.pack_mind.view_rerun --seed 0 --save run.rrd # no GUI +""" + +from __future__ import annotations + +import argparse +from typing import Any + +import numpy as np +import rerun as rr + +from dimos.experimental.pack_mind.render import _run_capture +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues +from dimos.visualization.rerun.init import rerun_init + +_WALL = np.array([13, 13, 20], dtype=np.uint8) +_FREE = np.array([245, 245, 245], dtype=np.uint8) +_VISITED = np.array([140, 200, 255], dtype=np.uint8) + + +def _set_tick(tick: int) -> None: + # rerun's time API name shifts across versions; support both. + try: + rr.set_time_sequence("tick", tick) # type: ignore[attr-defined] + except AttributeError: + rr.set_time("tick", sequence=tick) # type: ignore[attr-defined] + + +def _frame_rgb(grid: np.ndarray, visited: np.ndarray) -> np.ndarray: + h, w = grid.shape + img = np.empty((h, w, 3), dtype=np.uint8) + img[:] = _FREE + img[grid == CostValues.OCCUPIED] = _WALL + img[visited] = _VISITED + return np.flipud(img) # flip so +y points up in the viewer + + +def _log_run(root: str, sim: Any, frames: list[dict[str, Any]]) -> None: + grid = sim.world.grid + h = grid.shape[0] + world = sim.world + survivors = sim.survivors + + def to_img_xy(wx: float, wy: float) -> tuple[float, float]: + gc = world.world_to_grid((wx, wy)) + return (gc.x, (h - 1) - gc.y) # match the vertical flip + + for fr in frames: + _set_tick(fr["tick"]) + rr.log(f"{root}/map", rr.Image(_frame_rgb(grid, fr["visited"]))) + + spos = [to_img_xy(sx, sy) for sx, sy in survivors] + scol = [ + [60, 220, 60] if si in fr["found"] else [230, 40, 40] for si in range(len(survivors)) + ] + rr.log( + f"{root}/map/survivors", + rr.Points2D(spos, colors=scol, radii=1.6), + ) + + rpos = [to_img_xy(rx, ry) for rx, ry, _ in fr["robots"]] + rcol = [[150, 150, 150] if done else [255, 150, 0] for _, _, done in fr["robots"]] + rr.log(f"{root}/map/robots", rr.Points2D(rpos, colors=rcol, radii=1.3)) + + rr.log(f"{root}/stats", rr.TextLog(f"coverage {fr['cov']:.0%} found {len(fr['found'])}/{len(survivors)}")) + + +def view(seed: int = 0, max_ticks: int = 4000, save: str | None = None) -> None: + rerun_init(app_id="pack_mind") + if save: + rr.save(save) + else: + rr.spawn() # opens the Rerun (DimOS) viewer window + + psim, pframes = _run_capture(True, seed, max_ticks) + isim, iframes = _run_capture(False, seed, max_ticks) + _log_run("pack", psim, pframes) + _log_run("independent", isim, iframes) + + if save: + print(f"saved Rerun recording -> {save} (open with: rerun {save})") + else: + print("Rerun viewer launched. Scrub the 'tick' timeline; compare pack/ vs independent/.") + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="PACK MIND — view A/B race in Rerun") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--max-ticks", type=int, default=4000) + p.add_argument("--save", default=None, metavar="PATH", help="write .rrd instead of opening GUI") + return p.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + view(seed=args.seed, max_ticks=args.max_ticks, save=args.save) diff --git a/dimos/experimental/pack_mind/world.py b/dimos/experimental/pack_mind/world.py new file mode 100644 index 0000000000..a4cf8d6dd9 --- /dev/null +++ b/dimos/experimental/pack_mind/world.py @@ -0,0 +1,168 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PACK MIND sim — world fabrication (SAR floor as an OccupancyGrid). + +A collapsed-building floor: free space carved into rooms by walls with narrow +door gaps. Corridors/doors are kept wide enough to survive the router's safe-mask +erosion. Fully known map (no fog) — FrontierPatrolRouter sweeps unvisited *free* +cells, it does not need UNKNOWN. +""" + +from __future__ import annotations + +import numpy as np +from PIL import Image +from scipy.ndimage import binary_dilation, binary_erosion, label + +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.nav_msgs.OccupancyGrid import CostValues, OccupancyGrid +from dimos.utils.data import get_data + +RES = 0.1 # meters per cell +DOOR = 8 # door gap width in cells (>> erosion structure so it stays traversable) + +# Pixel-value semantics of the DimOS SLAM occupancy PNGs (e.g. +# big_office_simple_occupancy.png). `OccupancyGrid.from_path` does NOT remap +# these to CostValues, so we interpret them here. +_PNG_FREE_VAL = 15 # mapped interior floor +_PNG_WALL_VAL = 105 # walls / obstacles (the 0 background is the building exterior) + + +def pose_at(x: float, y: float) -> PoseStamped: + """A PoseStamped at world (x, y).""" + p = PoseStamped() + p.position.x = x + p.position.y = y + return p + + +def make_maze_world(w: int = 60, h: int = 60, res: float = RES) -> OccupancyGrid: + """Free interior, OCCUPIED border + 3 vertical dividers with alternating doors. + + The alternating door placement forces a serpentine route between the four + columns — exactly where independent robots redundantly re-walk corridors and a + shared coverage memory pays off. + """ + grid = np.full((h, w), CostValues.FREE, dtype=np.int8) + grid[0, :] = grid[-1, :] = CostValues.OCCUPIED + grid[:, 0] = grid[:, -1] = CostValues.OCCUPIED + + # Vertical dividers at 1/4, 2/4, 3/4 width. Doors alternate top / bottom. + for i, x in enumerate((w // 4, w // 2, 3 * w // 4)): + grid[1:-1, x] = CostValues.OCCUPIED + if i % 2 == 0: + grid[1 : 1 + DOOR, x] = CostValues.FREE # door at top + else: + grid[-1 - DOOR : -1, x] = CostValues.FREE # door at bottom + return OccupancyGrid(grid=grid, resolution=res) + + +def load_building_world( + name: str = "big_office_simple_occupancy.png", + downsample: int = 4, + res_full: float = 0.05, + close_iters: int = 1, +) -> OccupancyGrid: + """Turn a real DimOS SLAM occupancy PNG into a fog-of-war search arena. + + The raw PNG is a *visualization* (pixel values 15=free, 105=wall, 0=exterior), + not a CostValues grid, so we remap by value. We downsample to keep the sim + fast and make the fixed sensor radius proportionate to the floor size. + + Structure comes from the *building footprint*, not interior pixels: the free + interior is majority-pooled per block, morphologically closed to bridge SLAM + speckle (which would otherwise sever real corridors), then reduced to its + largest connected component. Everything outside the footprint is OCCUPIED, so + the building's own concave outline — wings, corridors, dead-ends — is what + blocks line-of-sight. That concavity is what makes shared-vs-independent fog + diverge; the sparse interior wall dots are SLAM noise and are deliberately + not carved back in (dilating them disconnects the floor). + """ + raw = np.array(Image.open(get_data(name)).convert("L")) + ds = max(1, downsample) + h, w = raw.shape + hc, wc = h // ds, w // ds + raw = raw[: hc * ds, : wc * ds] + + free = (raw == _PNG_FREE_VAL).reshape(hc, ds, wc, ds).mean(axis=(1, 3)) > 0.5 + if close_iters > 0: # bridge speckle gaps so corridors stay connected + free = binary_dilation(free, iterations=close_iters) + free = binary_erosion(free, iterations=close_iters, border_value=1) + + lab, n = label(free) # keep the largest navigable floor, drop outliers + if n > 1: + sizes = np.bincount(lab.ravel()) + sizes[0] = 0 + free = lab == int(sizes.argmax()) + + grid = np.full((hc, wc), CostValues.OCCUPIED, dtype=np.int8) + grid[free] = CostValues.FREE + return OccupancyGrid(grid=grid, resolution=res_full * ds, frame_id="world") + + +def spread_starts( + world: OccupancyGrid, n: int, seed: int = 0, min_dist_m: float = 1.0 +) -> list[tuple[float, float]]: + """Pick n free-cell start positions, greedily maximizing pairwise spread. + + Replaces the hand-tuned _START_POOL when the arena geometry is data-driven.""" + rng = np.random.default_rng(seed) + cells = free_cells_world(world) + if not cells: + raise ValueError("world has no FREE cells") + rng.shuffle(cells) + chosen: list[tuple[float, float]] = [cells[0]] + for c in cells[1:]: + if all((c[0] - s[0]) ** 2 + (c[1] - s[1]) ** 2 >= min_dist_m**2 for s in chosen): + chosen.append(c) + if len(chosen) == n: + break + # If the arena was too cramped to find n spread points, pad with any free cells. + for c in cells: + if len(chosen) >= n: + break + if c not in chosen: + chosen.append(c) + return chosen[:n] + + +def free_cells_world(world: OccupancyGrid) -> list[tuple[float, float]]: + """World-coordinate centers of all FREE cells.""" + rows, cols = np.where(world.grid == CostValues.FREE) + out: list[tuple[float, float]] = [] + for r, c in zip(rows.tolist(), cols.tolist()): + wp = world.grid_to_world((int(c), int(r), 0)) + out.append((wp.x, wp.y)) + return out + + +def plant_survivors( + world: OccupancyGrid, n: int, seed: int, margin_m: float = 0.5 +) -> list[tuple[float, float]]: + """Plant n survivors on free cells, spread across the map (min pairwise dist).""" + rng = np.random.default_rng(seed) + cells = free_cells_world(world) + rng.shuffle(cells) + chosen: list[tuple[float, float]] = [] + for c in cells: + if all((c[0] - s[0]) ** 2 + (c[1] - s[1]) ** 2 >= margin_m**2 for s in chosen): + chosen.append(c) + if len(chosen) == n: + break + return chosen + + +def free_cell_count(world: OccupancyGrid) -> int: + return int(np.count_nonzero(world.grid == CostValues.FREE)) diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 3d101cca79..e4cc2ca130 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -70,6 +70,7 @@ "mid360-fastlio-voxels-native": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_voxels_native", "openarm-mock-planner-coordinator": "dimos.robot.manipulators.openarm.blueprints:openarm_mock_planner_coordinator", "openarm-planner-coordinator": "dimos.robot.manipulators.openarm.blueprints:openarm_planner_coordinator", + "pack-mind-sim": "dimos.experimental.pack_mind.blueprint:pack_mind_sim", "teleop-phone": "dimos.teleop.phone.blueprints:teleop_phone", "teleop-phone-go2": "dimos.teleop.phone.blueprints:teleop_phone_go2", "teleop-phone-go2-fleet": "dimos.teleop.phone.blueprints:teleop_phone_go2_fleet", @@ -103,6 +104,7 @@ "unitree-go2-keyboard-teleop": "dimos.robot.unitree.go2.blueprints.basic.unitree_go2_keyboard_teleop:unitree_go2_keyboard_teleop", "unitree-go2-markers": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2:unitree_go2_markers", "unitree-go2-memory": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2:unitree_go2_memory", + "unitree-go2-pack": "dimos.experimental.pack_mind.live:unitree_go2_pack", "unitree-go2-relocalization": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2:unitree_go2_relocalization", "unitree-go2-ros": "dimos.robot.unitree.go2.blueprints.smart.unitree_go2_ros:unitree_go2_ros", "unitree-go2-security": "dimos.robot.unitree.go2.blueprints.agentic.unitree_go2_security:unitree_go2_security", @@ -133,6 +135,7 @@ "cost-mapper": "dimos.mapping.costmapper.CostMapper", "demo-calculator-skill": "dimos.agents.skills.demo_calculator_skill.DemoCalculatorSkill", "demo-robot": "dimos.agents.skills.demo_robot.DemoRobot", + "describer-vlm": "dimos.experimental.pack_mind.sim_perception.DescriberVLM", "desk-static-tf-module": "dimos.perception.fiducial.blueprints.desk_marker_tf.DeskStaticTfModule", "detection2-d-module": "dimos.perception.detection.module2D.Detection2DModule", "detection3-d-module": "dimos.perception.detection.module3D.Detection3DModule", @@ -181,6 +184,8 @@ "object-tracker3-d": "dimos.perception.object_tracker_3d.ObjectTracker3D", "object-tracking": "dimos.perception.object_tracker.ObjectTracking", "osm-skill": "dimos.agents.skills.osm.OsmSkill", + "pack-mind-sim-module": "dimos.experimental.pack_mind.blueprint.PackMindSimModule", + "pack-search-skills": "dimos.experimental.pack_mind.pack_search_skills.PackSearchSkills", "path-follower": "dimos.navigation.nav_stack.modules.path_follower.path_follower.PathFollower", "patrolling-module": "dimos.navigation.patrolling.module.PatrollingModule", "perceive-loop-skill": "dimos.perception.perceive_loop_skill.PerceiveLoopSkill", @@ -194,6 +199,7 @@ "real-sense-camera": "dimos.hardware.sensors.camera.realsense.camera.RealSenseCamera", "receiver-module": "dimos.utils.demo_image_encoding.ReceiverModule", "recorder": "dimos.memory2.module.Recorder", + "red-object-detector": "dimos.experimental.pack_mind.red_detector.RedObjectDetector", "reid-module": "dimos.perception.detection.reid.module.ReidModule", "relocalization-module": "dimos.mapping.relocalization.module.RelocalizationModule", "replanning-a-star-planner": "dimos.navigation.replanning_a_star.module.ReplanningAStarPlanner", @@ -213,6 +219,7 @@ "unitree-g1-skill-container": "dimos.robot.unitree.g1.skill_container.UnitreeG1SkillContainer", "unitree-skill-container": "dimos.robot.unitree.unitree_skill_container.UnitreeSkillContainer", "unity-bridge-module": "dimos.simulation.unity.module.UnityBridgeModule", + "velocity-teleop": "dimos.experimental.pack_mind.velocity_teleop.VelocityTeleop", "vlm-agent": "dimos.agents.vlm_agent.VLMAgent", "vlm-stream-tester": "dimos.agents.vlm_stream_tester.VlmStreamTester", "voxel-grid-mapper": "dimos.mapping.voxels.VoxelGridMapper", diff --git a/dimos/robot/cli/topic.py b/dimos/robot/cli/topic.py index e4dce1f139..3b8df8929c 100644 --- a/dimos/robot/cli/topic.py +++ b/dimos/robot/cli/topic.py @@ -15,6 +15,7 @@ from __future__ import annotations import importlib +import pkgutil import re import time @@ -120,14 +121,30 @@ def topic_send(topic: str, message_expr: str) -> None: for module_name in modules_to_import: try: module = importlib.import_module(module_name) - for name in getattr(module, "__all__", dir(module)): - if not name.startswith("_"): - obj = getattr(module, name, None) - if obj is not None: - eval_context[name] = obj except ImportError: continue + names = [n for n in getattr(module, "__all__", None) or [] if not n.startswith("_")] + for name in names: + obj = getattr(module, name, None) + if obj is not None: + eval_context[name] = obj + + # msgs packages declare __all__ = [] and keep each class in a same-named + # submodule (e.g. geometry_msgs.Twist.Twist). Walk submodules so the + # eval namespace gets the actual message classes. + if not names and hasattr(module, "__path__"): + for info in pkgutil.iter_modules(module.__path__): + if info.name.startswith("_"): + continue + try: + sub = importlib.import_module(f"{module_name}.{info.name}") + except ImportError: + continue + cls = getattr(sub, info.name, None) + if cls is not None: + eval_context[info.name] = cls + try: message = eval(message_expr, eval_context) except Exception as e: diff --git a/dimos/robot/test_all_blueprints_generation.py b/dimos/robot/test_all_blueprints_generation.py index d8b0081d7f..d141f4e3ed 100644 --- a/dimos/robot/test_all_blueprints_generation.py +++ b/dimos/robot/test_all_blueprints_generation.py @@ -32,7 +32,14 @@ "dimos/core/blueprints.py", "dimos/core/test_blueprints.py", } -BLUEPRINT_METHODS = {"transports", "global_config", "remappings", "requirements", "configurators"} +BLUEPRINT_METHODS = { + "transports", + "global_config", + "remappings", + "requirements", + "configurators", + "disabled_modules", +} _EXCLUDED_MODULE_NAMES = {"Module", "ModuleBase", "StreamModule"} diff --git a/dimos/robot/unitree/connection.py b/dimos/robot/unitree/connection.py index 44101cc19d..f6aaacbc5e 100644 --- a/dimos/robot/unitree/connection.py +++ b/dimos/robot/unitree/connection.py @@ -93,12 +93,22 @@ def to_ndarray(self, format=None): # type: ignore[no-untyped-def] class UnitreeWebRTCConnection(Resource): _SPORT_API_ID_RAGEMODE: int = 2059 + _AP_GATEWAY_IP: str = "192.168.12.1" + def __init__(self, ip: str, mode: str = "ai") -> None: self.ip = ip self.mode = mode self.stop_timer: threading.Timer | None = None self.cmd_vel_timeout = 0.2 - self.conn = LegionConnection(WebRTCConnectionMethod.LocalSTA, ip=self.ip) + # When joined to the robot's own WiFi hotspot the gateway is always + # 192.168.12.1 and the peer expects the LocalAP handshake (empty SDP id). + # Any other IP means the robot is on a shared LAN → LocalSTA. + method = ( + WebRTCConnectionMethod.LocalAP + if self.ip == self._AP_GATEWAY_IP + else WebRTCConnectionMethod.LocalSTA + ) + self.conn = LegionConnection(method, ip=self.ip) self.connect() def connect(self) -> None: