From d92291b37051658a12db9e23c6ad77c5354483ac Mon Sep 17 00:00:00 2001 From: Paul Nechifor Date: Sat, 30 May 2026 02:01:22 +0300 Subject: [PATCH] feat: fresh ubuntu testing --- .gitignore | 2 + dimos/utils/docs/doclinks.py | 14 +- docs/development/testing.md | 21 ++ docs/usage/transports/dds.md | 5 +- misc/fresh-ubuntu-tests/suite/run.sh | 54 ++++ misc/fresh-ubuntu-tests/suite/setup.sh | 20 ++ .../fresh-ubuntu-tests/suite/tests/01-mypy.sh | 3 + .../suite/tests/02-pytest.sh | 3 + .../suite/tests/03-cyclonedds.sh | 17 ++ misc/fresh-ubuntu-tests/vmtest.sh | 241 ++++++++++++++++++ 10 files changed, 369 insertions(+), 11 deletions(-) create mode 100755 misc/fresh-ubuntu-tests/suite/run.sh create mode 100755 misc/fresh-ubuntu-tests/suite/setup.sh create mode 100755 misc/fresh-ubuntu-tests/suite/tests/01-mypy.sh create mode 100755 misc/fresh-ubuntu-tests/suite/tests/02-pytest.sh create mode 100755 misc/fresh-ubuntu-tests/suite/tests/03-cyclonedds.sh create mode 100755 misc/fresh-ubuntu-tests/vmtest.sh diff --git a/.gitignore b/.gitignore index ffc1c4f31f..8b40cff286 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,5 @@ recording*.db # Rerun recordings *.rrd + +/misc/fresh-ubuntu-tests/cache diff --git a/dimos/utils/docs/doclinks.py b/dimos/utils/docs/doclinks.py index 122abb8f72..fba42c6678 100644 --- a/dimos/utils/docs/doclinks.py +++ b/dimos/utils/docs/doclinks.py @@ -661,9 +661,7 @@ def main() -> None: def process_file(md_path: Path, quiet: bool = False) -> tuple[bool, list[str]]: """Process a single markdown file. Returns (changed, errors).""" md_path = md_path.resolve() - if not quiet: - rel = md_path.relative_to(root) if md_path.is_relative_to(root) else md_path - print(f"\nProcessing {rel}...") + rel = md_path.relative_to(root) if md_path.is_relative_to(root) else md_path content = md_path.read_text() new_content, changes, errors = process_markdown( @@ -677,6 +675,10 @@ def process_file(md_path: Path, quiet: bool = False) -> tuple[bool, list[str]]: doc_index=doc_index, ) + # Only announce the file when there's something to report. + if not quiet and (changes or errors): + print(f"\nProcessing {rel}...") + if errors: for err in errors: print(f" Error: {err}", file=sys.stderr) @@ -691,10 +693,8 @@ def process_file(md_path: Path, quiet: bool = False) -> tuple[bool, list[str]]: if not quiet: print(" Updated") return True, errors - else: - if not quiet: - print(" No changes needed") - return False, errors + + return False, errors # Watch mode if args.watch: diff --git a/docs/development/testing.md b/docs/development/testing.md index 85a98fe075..40e429797c 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -63,6 +63,27 @@ When writing or debugging a specific self-hosted test, override `-m` yourself to pytest -m self_hosted dimos/path/to/test_something.py ``` +## Testing on a fresh Ubuntu install + +CI tests dimos with pre-built images and cached deps, so it can't catch gaps +between what [`installation/ubuntu.md`](/docs/installation/ubuntu.md) tells a new user to +do and what a clean machine actually needs (e.g. a system package we require but +forgot to document). + +The [misc/fresh-ubuntu-tests/](/misc/fresh-ubuntu-tests/) harness closes that +gap. It replays the documented install + test flow inside a fresh, official, +**unmodified** Ubuntu Desktop 24.04 VM (VirtualBox). + +It's intended to be executed locally. + +```sh skip +cd misc/fresh-ubuntu-tests + +./vmtest.sh build # download + verify the official ISO, install, snapshot "golden" (once, ~15-30 min) +./vmtest.sh run # clone golden, run the doc flow, report PASS/FAIL +./vmtest.sh clean # delete leftover run clones and logs (keeps the ISO + golden VM) +``` + ## Writing tests Test files live next to the code they test. If you have `dimos/core/pubsub.py`, its tests go in `dimos/core/test_pubsub.py`. diff --git a/docs/usage/transports/dds.md b/docs/usage/transports/dds.md index 924b9d43e8..7f08eedcac 100644 --- a/docs/usage/transports/dds.md +++ b/docs/usage/transports/dds.md @@ -45,13 +45,10 @@ sudo ln -sf /usr/lib/x86_64-linux-gnu/libcycloneddsidl.so* /opt/cyclonedds/lib/ sudo ln -sf /usr/bin/idlc /opt/cyclonedds/bin/ sudo ln -sf /usr/bin/ddsperf /opt/cyclonedds/bin/ sudo ln -sf /usr/include/dds /opt/cyclonedds/include/ - -# Install with the dds extra -CYCLONEDDS_HOME=/opt/cyclonedds uv pip install -e '.[dds]' ``` To install all extras including DDS: ```bash -CYCLONEDDS_HOME=/opt/cyclonedds uv sync --extra dds +CYCLONEDDS_HOME=/opt/cyclonedds uv sync --all-extras --all-groups ``` diff --git a/misc/fresh-ubuntu-tests/suite/run.sh b/misc/fresh-ubuntu-tests/suite/run.sh new file mode 100755 index 0000000000..7637b52ddb --- /dev/null +++ b/misc/fresh-ubuntu-tests/suite/run.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +# Runs the fresh-ubuntu test suite INSIDE the VM (invoked by vmtest.sh). +# +# setup.sh must pass, or the suite stops. Then every tests/*.sh runs in turn, +# each to its own log in ./logs/, and a failing test does NOT stop the others. +# Add a test by dropping another tests/NN-name.sh -- NN sets the order, which can +# matter since the tests share one VM (e.g. they reuse the same .venv). +# +# run.sh provides the environment so each test can be just the command(s): uv is +# on PATH, and tests run with the cwd set to the cloned repo. + +set -uo pipefail +shopt -s nullglob + +SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; readonly SUITE_DIR +readonly LOGS="$SUITE_DIR/logs" +readonly REPO="$HOME/dimos" +export PATH="$HOME/.local/bin:$PATH" + +rm -rf "$LOGS"; mkdir -p "$LOGS" +hdr() { printf '\n\033[1m== %s ==\033[0m\n' "$*"; } + +# setup is gating: if it fails, run nothing else. +hdr "setup" +if ! bash "$SUITE_DIR/setup.sh" 2>&1 | tee "$LOGS/setup.log"; then + echo "setup failed -- aborting suite" + exit 1 +fi + +# each test is independent: run it in the repo, log it, keep going on failure. +names=(); codes=() +for t in "$SUITE_DIR"/tests/*.sh; do + name="$(basename "$t" .sh)" + hdr "test: $name" + if (cd "$REPO" && bash "$t") 2>&1 | tee "$LOGS/$name.log"; then + codes+=(0) + else + codes+=("${PIPESTATUS[0]}") + fi + names+=("$name") +done + +hdr "summary" +fail=0 +for i in "${!names[@]}"; do + if [[ "${codes[$i]}" -eq 0 ]]; then + printf ' PASS %s\n' "${names[$i]}" + else + printf ' FAIL %s (exit %s)\n' "${names[$i]}" "${codes[$i]}" + fail=1 + fi +done +exit "$fail" diff --git a/misc/fresh-ubuntu-tests/suite/setup.sh b/misc/fresh-ubuntu-tests/suite/setup.sh new file mode 100755 index 0000000000..9761acb5fd --- /dev/null +++ b/misc/fresh-ubuntu-tests/suite/setup.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Sets up the whole system in the fresh VM: the documented dimos install flow +# (docs/installation/ubuntu.md). Run first by run.sh; if it fails, no tests run. +# uv is already on PATH (run.sh sets it). + +set -euxo pipefail +export GIT_LFS_SKIP_SMUDGE=1 + +# system dependencies (docs/installation/ubuntu.md) +sudo apt-get update +sudo apt-get install -y curl g++ portaudio19-dev git-lfs libturbojpeg python3-dev pre-commit + +# uv +curl -LsSf https://astral.sh/uv/install.sh | sh + +# clone + base sync +git clone https://github.com/dimensionalOS/dimos.git "$HOME/dimos" +cd "$HOME/dimos" +uv sync --all-groups diff --git a/misc/fresh-ubuntu-tests/suite/tests/01-mypy.sh b/misc/fresh-ubuntu-tests/suite/tests/01-mypy.sh new file mode 100755 index 0000000000..d1da5f49f7 --- /dev/null +++ b/misc/fresh-ubuntu-tests/suite/tests/01-mypy.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euxo pipefail +uv run mypy dimos diff --git a/misc/fresh-ubuntu-tests/suite/tests/02-pytest.sh b/misc/fresh-ubuntu-tests/suite/tests/02-pytest.sh new file mode 100755 index 0000000000..39db542d52 --- /dev/null +++ b/misc/fresh-ubuntu-tests/suite/tests/02-pytest.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euxo pipefail +uv run pytest --numprocesses=auto dimos diff --git a/misc/fresh-ubuntu-tests/suite/tests/03-cyclonedds.sh b/misc/fresh-ubuntu-tests/suite/tests/03-cyclonedds.sh new file mode 100755 index 0000000000..0caf6b62d5 --- /dev/null +++ b/misc/fresh-ubuntu-tests/suite/tests/03-cyclonedds.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Install the CycloneDDS development library +sudo apt-get install -y cyclonedds-dev + +# Create a compatibility directory structure +# (required because Ubuntu's multiarch layout doesn't match the expected CMake layout) +sudo mkdir -p /opt/cyclonedds/{lib,bin,include} +sudo ln -sf /usr/lib/x86_64-linux-gnu/libddsc.so* /opt/cyclonedds/lib/ +sudo ln -sf /usr/lib/x86_64-linux-gnu/libcycloneddsidl.so* /opt/cyclonedds/lib/ +sudo ln -sf /usr/bin/idlc /opt/cyclonedds/bin/ +sudo ln -sf /usr/bin/ddsperf /opt/cyclonedds/bin/ +sudo ln -sf /usr/include/dds /opt/cyclonedds/include/ + +rm -fr .venv +CYCLONEDDS_HOME=/opt/cyclonedds uv sync --all-extras --all-groups diff --git a/misc/fresh-ubuntu-tests/vmtest.sh b/misc/fresh-ubuntu-tests/vmtest.sh new file mode 100755 index 0000000000..fd9a4fa867 --- /dev/null +++ b/misc/fresh-ubuntu-tests/vmtest.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +# +# vmtest.sh -- run the documented dimos install + tests on a fresh, official, +# unmodified Ubuntu Desktop 24.04 VM (VirtualBox). +# +# Why: CI tests dimos from a maintainer's angle (pre-built images, cached deps). +# It does not verify that a brand-new user following docs/installation/ubuntu.md +# on a clean machine can actually install and run dimos. This tool does, by +# replaying that flow (the suite/ next to this script) in a fresh VM. +# +# Run by hand, occasionally. It is slow; that's fine. +# +# ./vmtest.sh build download + verify the ISO, install the OS, snapshot "golden" (once) +# ./vmtest.sh run clone golden, run the doc flow, report PASS/FAIL +# ./vmtest.sh clean delete leftover run clones and logs (keeps the ISO + golden VM) +# +# Everything lives in a gitignored ./cache subdir next to this script. To change +# what is tested or the VM size, edit the constants below -- there are no flags. + +set -euo pipefail + +# VBoxHeadless otherwise drops a -VBoxHeadless-.log in the cwd on +# every VM start; send that default process log to nowhere. The VM's own +# VBox.log (in the VM folder) is unaffected. +export VBOX_RELEASE_LOG_DEST=nofile + +# --- constants --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; readonly SCRIPT_DIR +readonly RELEASE="24.04" +readonly VM="dimos-vmtest-base" +readonly MEM=12288 # MiB +readonly CPUS=8 +readonly DISK=40960 # MiB (dynamic; only grows as used) +readonly SSH_PORT=2222 +readonly VM_USER="tester" +readonly VM_PASS="tester" + +readonly CACHE="$SCRIPT_DIR/cache" +readonly ISO_DIR="$CACHE/iso" +readonly VMS_DIR="$CACHE/vms" +readonly SSH_DIR="$CACHE/ssh" +readonly LOG_DIR="$CACHE/logs" +readonly SSH_KEY="$SSH_DIR/id_ed25519" + +# --- helpers --- +log() { printf '\033[1;34m[vmtest]\033[0m %s\n' "$*" >&2; } +warn() { printf '\033[1;33m[vmtest]\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31m[vmtest] error:\033[0m %s\n' "$*" >&2; exit 1; } +need() { command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"; } + +SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 -o LogLevel=ERROR) +ssh_vm() { ssh "${SSH_OPTS[@]}" -p "$SSH_PORT" "$VM_USER@127.0.0.1" "$@"; } +scp_to() { scp "${SSH_OPTS[@]}" -P "$SSH_PORT" "$1" "$VM_USER@127.0.0.1:$2"; } + +vm_exists() { VBoxManage showvminfo "$1" >/dev/null 2>&1; } +vm_state() { VBoxManage showvminfo "$1" --machinereadable 2>/dev/null | sed -n 's/^VMState="\(.*\)"/\1/p'; } +run_clones() { VBoxManage list vms 2>/dev/null | sed -n 's/^"\(dimos-vmtest-run-[^"]*\)".*/\1/p'; } + +golden_exists() { + vm_exists "$VM" && VBoxManage snapshot "$VM" list --machinereadable 2>/dev/null \ + | grep -q 'SnapshotName[^=]*="golden"' +} + +wait_for_ssh() { # $1 = timeout seconds + local deadline=$(( SECONDS + $1 )) + while (( SECONDS < deadline )); do + ssh_vm true >/dev/null 2>&1 && return 0 + sleep 5 + done + return 1 +} + +wait_poweroff() { # $1 = vm, $2 = timeout + local deadline=$(( SECONDS + $2 )) s + while (( SECONDS < deadline )); do + s="$(vm_state "$1")" + [[ "$s" == "poweroff" || "$s" == "aborted" || -z "$s" ]] && return 0 + sleep 3 + done + return 1 +} + +power_off() { # graceful, then hard + local vm="$1" + [[ "$(vm_state "$vm")" == "running" ]] || return 0 + VBoxManage controlvm "$vm" acpipowerbutton >/dev/null 2>&1 || true + wait_poweroff "$vm" 60 && return 0 + VBoxManage controlvm "$vm" poweroff >/dev/null 2>&1 || true + wait_poweroff "$vm" 30 || true +} + +remove_vm() { # power off + delete + vm_exists "$1" || return 0 + power_off "$1" + VBoxManage unregistervm "$1" --delete >/dev/null 2>&1 || true +} + +ensure_iso() { + need wget + mkdir -p "$ISO_DIR" + local fname="ubuntu-24.04.4-desktop-amd64.iso" + ISO_PATH="$ISO_DIR/$fname" + + if [[ -f "$ISO_PATH" ]]; then + return + fi + + wget --tries=3 --continue -O "$ISO_PATH" "https://releases.ubuntu.com/$RELEASE/$fname" \ + || die "download failed" +} + +ensure_sshkey() { + need ssh-keygen + mkdir -p "$SSH_DIR"; chmod 700 "$SSH_DIR" + [[ -f "$SSH_KEY" ]] || ssh-keygen -t ed25519 -N '' -C 'dimos-vmtest' -f "$SSH_KEY" >/dev/null +} + +# Bootstrap run as root inside the installed system by VBox's autoinstall late- +# command. Adds only what we need to drive the VM headlessly (openssh + our key + +# passwordless sudo) -- none are dimos deps, so they don't mask a dep gap. Passed +# base64-encoded: the only quoting-safe way through VBox's template substitution. +post_install_command() { + local script + script="$(cat < /home/$VM_USER/.ssh/authorized_keys < /etc/sudoers.d/90-$VM_USER +chmod 440 /etc/sudoers.d/90-$VM_USER +touch /etc/dimos-vmtest-ready +EOF +)" + printf 'bash -c "echo %s | base64 -d | bash"' "$(printf '%s' "$script" | base64 -w0)" +} + +# --- commands --- +cmd_build() { + need VBoxManage + golden_exists && die "golden image already exists; delete VM '$VM' to rebuild" + vm_exists "$VM" && die "VM '$VM' exists without a golden snapshot; delete it to rebuild" + + ensure_iso + ensure_sshkey + mkdir -p "$VMS_DIR" + + log "creating VM '$VM'" + VBoxManage createvm --name "$VM" --basefolder "$VMS_DIR" --ostype Ubuntu_64 --register + VBoxManage modifyvm "$VM" --memory "$MEM" --cpus "$CPUS" --ioapic on --rtcuseutc on \ + --graphicscontroller vmsvga --vram 64 --audio-driver none --firmware bios + VBoxManage modifyvm "$VM" --nic1 nat --natpf1 "ssh,tcp,127.0.0.1,$SSH_PORT,,22" + VBoxManage createmedium disk --filename "$VMS_DIR/$VM/$VM.vdi" --size "$DISK" --format VDI + VBoxManage storagectl "$VM" --name SATA --add sata --controller IntelAhci --portcount 2 + VBoxManage storageattach "$VM" --storagectl SATA --port 0 --device 0 --type hdd \ + --medium "$VMS_DIR/$VM/$VM.vdi" + VBoxManage storagectl "$VM" --name IDE --add ide + + log "starting unattended install (slow: ~15-30 min)" + VBoxManage unattended install "$VM" \ + --iso="$ISO_PATH" \ + --user="$VM_USER" --user-password="$VM_PASS" --full-user-name="Dimos Tester" \ + --hostname="dimos-vmtest.local" \ + --no-install-additions \ + --post-install-command="$(post_install_command)" \ + --start-vm=headless + + log "waiting for the installed system over SSH (up to 60 min)" + wait_for_ssh 3600 || die "VM never became reachable over SSH" + ssh_vm 'test -f /etc/dimos-vmtest-ready' || die "post-install marker missing; install incomplete" + + log "shutting down to snapshot a clean base" + power_off "$VM" + wait_poweroff "$VM" 120 || die "VM did not power off" + VBoxManage snapshot "$VM" take golden --description "fresh Ubuntu $RELEASE desktop install" +} + +CLONE="" +cleanup_clone() { [[ -n "$CLONE" ]] && remove_vm "$CLONE"; } + +cmd_run() { + need VBoxManage + golden_exists || die "no golden image; run '$0 build' first" + mkdir -p "$LOG_DIR" + + local stamp; stamp="$(date +%Y%m%d-%H%M%S)-$$" + CLONE="dimos-vmtest-run-$stamp" + local rundir="$LOG_DIR/$stamp" + mkdir -p "$rundir" + trap cleanup_clone EXIT + + log "cloning golden -> $CLONE" + VBoxManage clonevm "$VM" --snapshot golden --options link \ + --name "$CLONE" --basefolder "$VMS_DIR" --register >/dev/null + VBoxManage startvm "$CLONE" --type headless >/dev/null + + log "waiting for SSH" + wait_for_ssh 300 || die "clone never became reachable over SSH" + + log "running suite (per-script logs -> $rundir/logs)" + scp -r "${SSH_OPTS[@]}" -P "$SSH_PORT" "$SCRIPT_DIR/suite" "$VM_USER@127.0.0.1:suite" + local rc=0 + ssh_vm "bash suite/run.sh" 2>&1 | tee "$rundir/run.log" || rc=${PIPESTATUS[0]} + # pull the individual per-script logs back to the host before the clone is deleted + scp -r "${SSH_OPTS[@]}" -P "$SSH_PORT" "$VM_USER@127.0.0.1:suite/logs" "$rundir/" 2>/dev/null || true + + if [[ "$rc" -eq 0 ]]; then + printf '\033[1;32m[vmtest] PASS\033[0m logs=%s\n' "$rundir" >&2 + else + printf '\033[1;31m[vmtest] FAIL (exit %s)\033[0m logs=%s\n' "$rc" "$rundir" >&2 + fi + return "$rc" +} + +# Remove leftover run clones and logs; keep the ISO and golden VM (the slow-to- +# rebuild caches). The SSH key is kept too -- it is paired with the golden VM. +cmd_clean() { + need VBoxManage + local name + while read -r name; do + [[ -n "$name" ]] || continue + log "removing clone $name" + remove_vm "$name" + done < <(run_clones) + rm -rf "$LOG_DIR" + log "clean done (kept ISO + golden VM)" +} + +case "${1:-}" in + build) cmd_build ;; + run) cmd_run ;; + clean) cmd_clean ;; + *) die "usage: $0 {build|run|clean}" ;; +esac