feat: fresh Ubuntu testing

.gitignore

@@ -89,3 +89,5 @@ recording*.db
 
 # Rerun recordings
 *.rrd
+
+/misc/fresh-ubuntu-tests/cache

dimos/utils/docs/doclinks.py

@@ -661,9 +661,7 @@ def main() -> None:
     def process_file(md_path: Path, quiet: bool = False) -> tuple[bool, list[str]]:
         """Process a single markdown file. Returns (changed, errors)."""
         md_path = md_path.resolve()
-        if not quiet:
-            rel = md_path.relative_to(root) if md_path.is_relative_to(root) else md_path
-            print(f"\nProcessing {rel}...")
+        rel = md_path.relative_to(root) if md_path.is_relative_to(root) else md_path
 
         content = md_path.read_text()
         new_content, changes, errors = process_markdown(
@@ -677,6 +675,10 @@ def process_file(md_path: Path, quiet: bool = False) -> tuple[bool, list[str]]:
             doc_index=doc_index,
         )
 
+        # Only announce the file when there's something to report.
+        if not quiet and (changes or errors):
+            print(f"\nProcessing {rel}...")
+
         if errors:
             for err in errors:
                 print(f"  Error: {err}", file=sys.stderr)
@@ -691,10 +693,8 @@ def process_file(md_path: Path, quiet: bool = False) -> tuple[bool, list[str]]:
                 if not quiet:
                     print("  Updated")
             return True, errors
-        else:
-            if not quiet:
-                print("  No changes needed")
-            return False, errors
+
+        return False, errors
 
     # Watch mode
     if args.watch:

docs/development/testing.md

@@ -63,6 +63,27 @@ When writing or debugging a specific self-hosted test, override `-m` yourself to
 pytest -m self_hosted dimos/path/to/test_something.py
 ```
 
+## Testing on a fresh Ubuntu install
+
+CI tests dimos with pre-built images and cached deps, so it can't catch gaps
+between what [`installation/ubuntu.md`](/docs/installation/ubuntu.md) tells a new user to
+do and what a clean machine actually needs (e.g. a system package we require but
+forgot to document).
+
+The [misc/fresh-ubuntu-tests/](/misc/fresh-ubuntu-tests/) harness closes that
+gap. It replays the documented install + test flow inside a fresh, official,
+**unmodified** Ubuntu Desktop 24.04 VM (VirtualBox).
+
+It's intended to be executed locally.
+
+```sh skip
+cd misc/fresh-ubuntu-tests
+
+./vmtest.sh build   # download + verify the official ISO, install, snapshot "golden" (once, ~15-30 min)
+./vmtest.sh run     # clone golden, run the doc flow, report PASS/FAIL
+./vmtest.sh clean   # delete leftover run clones and logs (keeps the ISO + golden VM)
+```
+
 ## Writing tests
 
 Test files live next to the code they test. If you have `dimos/core/pubsub.py`, its tests go in `dimos/core/test_pubsub.py`.

docs/usage/transports/dds.md

@@ -45,13 +45,10 @@ sudo ln -sf /usr/lib/x86_64-linux-gnu/libcycloneddsidl.so* /opt/cyclonedds/lib/
 sudo ln -sf /usr/bin/idlc /opt/cyclonedds/bin/
 sudo ln -sf /usr/bin/ddsperf /opt/cyclonedds/bin/
 sudo ln -sf /usr/include/dds /opt/cyclonedds/include/
-
-# Install with the dds extra
-CYCLONEDDS_HOME=/opt/cyclonedds uv pip install -e '.[dds]'
 ```
 
 To install all extras including DDS:
 
 ```bash
-CYCLONEDDS_HOME=/opt/cyclonedds uv sync --extra dds
+CYCLONEDDS_HOME=/opt/cyclonedds uv sync --all-extras --all-groups
 ```

misc/fresh-ubuntu-tests/suite/run.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Runs the fresh-ubuntu test suite INSIDE the VM (invoked by vmtest.sh).
+#
+# setup.sh must pass, or the suite stops. Then every tests/*.sh runs in turn,
+# each to its own log in ./logs/, and a failing test does NOT stop the others.
+# Add a test by dropping another tests/NN-name.sh -- NN sets the order, which can
+# matter since the tests share one VM (e.g. they reuse the same .venv).
+#
+# run.sh provides the environment so each test can be just the command(s): uv is
+# on PATH, and tests run with the cwd set to the cloned repo.
+
+set -uo pipefail
+shopt -s nullglob
+
+SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; readonly SUITE_DIR
+readonly LOGS="$SUITE_DIR/logs"
+readonly REPO="$HOME/dimos"
+export PATH="$HOME/.local/bin:$PATH"
+
+rm -rf "$LOGS"; mkdir -p "$LOGS"
+hdr() { printf '\n\033[1m== %s ==\033[0m\n' "$*"; }
+
+# setup is gating: if it fails, run nothing else.
+hdr "setup"
+if ! bash "$SUITE_DIR/setup.sh" 2>&1 | tee "$LOGS/setup.log"; then
+  echo "setup failed -- aborting suite"
+  exit 1
+fi
+
+# each test is independent: run it in the repo, log it, keep going on failure.
+names=(); codes=()
+for t in "$SUITE_DIR"/tests/*.sh; do
+  name="$(basename "$t" .sh)"
+  hdr "test: $name"
+  if (cd "$REPO" && bash "$t") 2>&1 | tee "$LOGS/$name.log"; then
+    codes+=(0)
+  else
+    codes+=("${PIPESTATUS[0]}")
+  fi
+  names+=("$name")
+done
+
+hdr "summary"
+fail=0
+for i in "${!names[@]}"; do
+  if [[ "${codes[$i]}" -eq 0 ]]; then
+    printf '  PASS  %s\n' "${names[$i]}"
+  else
+    printf '  FAIL  %s (exit %s)\n' "${names[$i]}" "${codes[$i]}"
+    fail=1
+  fi
+done
+exit "$fail"

misc/fresh-ubuntu-tests/suite/setup.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Sets up the whole system in the fresh VM: the documented dimos install flow
+# (docs/installation/ubuntu.md). Run first by run.sh; if it fails, no tests run.
+# uv is already on PATH (run.sh sets it).
+
+set -euxo pipefail
+export GIT_LFS_SKIP_SMUDGE=1
+
+# system dependencies (docs/installation/ubuntu.md)
+sudo apt-get update
+sudo apt-get install -y curl g++ portaudio19-dev git-lfs libturbojpeg python3-dev pre-commit
+
+# uv
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# clone + base sync
+git clone https://github.com/dimensionalOS/dimos.git "$HOME/dimos"
+cd "$HOME/dimos"
+uv sync --all-groups

misc/fresh-ubuntu-tests/suite/tests/01-mypy.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+uv run mypy dimos

misc/fresh-ubuntu-tests/suite/tests/02-pytest.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+uv run pytest --numprocesses=auto dimos

misc/fresh-ubuntu-tests/suite/tests/03-cyclonedds.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# Install the CycloneDDS development library
+sudo apt-get install -y cyclonedds-dev
+
+# Create a compatibility directory structure
+# (required because Ubuntu's multiarch layout doesn't match the expected CMake layout)
+sudo mkdir -p /opt/cyclonedds/{lib,bin,include}
+sudo ln -sf /usr/lib/x86_64-linux-gnu/libddsc.so* /opt/cyclonedds/lib/
+sudo ln -sf /usr/lib/x86_64-linux-gnu/libcycloneddsidl.so* /opt/cyclonedds/lib/
+sudo ln -sf /usr/bin/idlc /opt/cyclonedds/bin/
+sudo ln -sf /usr/bin/ddsperf /opt/cyclonedds/bin/
+sudo ln -sf /usr/include/dds /opt/cyclonedds/include/
+
+rm -fr .venv
+CYCLONEDDS_HOME=/opt/cyclonedds uv sync --all-extras --all-groups