Review BitBox all-initiatives audit findings

.github/actions/bitbox-flake-poster/action.yml

@@ -0,0 +1,136 @@
+name: BitBox flake-rate poster
+description: |
+  Updates the trailing-30-day per-flow flake-rate in
+  bitbox-testkit/coverage_report.md. Designed to run from realunit-app
+  but writing to a sibling repo via the GITHUB_TOKEN.
+
+  Behaviour:
+    - On every run, appends one row to a per-flow rolling log
+      .maestro/bitbox/flake-log.jsonl (committed back to the realunit-app
+      repo on `push: develop` runs ONLY -- PR runs do not commit).
+    - Posts a comment on the PR (if `pull_request` event) with the
+      single-flow attempt count + outcome.
+    - On scheduled / push runs, also computes the trailing-30-day green
+      rate and writes it to bitbox-testkit/coverage_report.md via a
+      cross-repo dispatch (NOT implemented yet -- this action stubs the
+      cross-repo write; the audit BL-100 cross-repo workflow finalises
+      that wire-up).
+
+inputs:
+  flow:
+    description: "Flow short ID (M-1, M-2, ...)"
+    required: true
+  attempts:
+    description: "Attempts taken in this run"
+    required: true
+  outcome:
+    description: "Step outcome: success | failure | skipped"
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Append flake log
+      shell: bash
+      env:
+        FLOW: ${{ inputs.flow }}
+        ATTEMPTS: ${{ inputs.attempts }}
+        OUTCOME: ${{ inputs.outcome }}
+      run: |
+        set -euo pipefail
+        LOG=".maestro/bitbox/flake-log.jsonl"
+        mkdir -p "$(dirname "$LOG")"
+        ts="$(date -u +%FT%TZ)"
+        entry='{"ts":"'"$ts"'","flow":"'"$FLOW"'","attempts":'"$ATTEMPTS"',"outcome":"'"$OUTCOME"'","sha":"'"${GITHUB_SHA:-}"'","run":"'"${GITHUB_RUN_ID:-}"'"}'
+        echo "$entry" >> "$LOG"
+        echo "appended: $entry"
+
+    - name: Compute trailing-30-day flake-rate
+      id: rate
+      shell: bash
+      env:
+        FLOW: ${{ inputs.flow }}
+      run: |
+        set -euo pipefail
+        LOG=".maestro/bitbox/flake-log.jsonl"
+        if [ ! -f "$LOG" ]; then
+          echo "rate=unknown" >> "$GITHUB_OUTPUT"
+          echo "n=0" >> "$GITHUB_OUTPUT"
+          exit 0
+        fi
+        # Last 30 d window. Rate = greens / total (entries with attempts == 1)
+        # are unambiguous greens; > 1 attempts is a flake-but-recovered.
+        cutoff="$(date -u -v-30d +%FT%TZ 2>/dev/null || date -u -d '30 days ago' +%FT%TZ)"
+        python3 - <<PY
+        import json, os, sys
+        cutoff = "$cutoff"
+        flow = "${FLOW}"
+        path = "$LOG"
+        total = 0; green = 0; flake = 0; fail = 0
+        with open(path) as f:
+            for line in f:
+                e = json.loads(line)
+                if e["flow"] != flow:
+                    continue
+                if e["ts"] < cutoff:
+                    continue
+                total += 1
+                if e["outcome"] == "success" and int(e["attempts"]) == 1:
+                    green += 1
+                elif e["outcome"] == "success":
+                    green += 1; flake += 1
+                elif e["outcome"] == "skipped":
+                    total -= 1  # don't count skipped
+                else:
+                    fail += 1
+        rate = "n/a" if total == 0 else "%.1f%%" % (100.0 * green / total)
+        flake_rate = "n/a" if total == 0 else "%.1f%%" % (100.0 * flake / total)
+        gh_out = os.environ["GITHUB_OUTPUT"]
+        with open(gh_out, "a") as o:
+            o.write("rate=%s\n" % rate)
+            o.write("flake_rate=%s\n" % flake_rate)
+            o.write("n=%d\n" % total)
+        print("flow=%s total=%d green=%d flake=%d fail=%d rate=%s flake_rate=%s" %
+              (flow, total, green, flake, fail, rate, flake_rate))
+        PY
+
+    - name: PR-comment with flake rate
+      if: github.event_name == 'pull_request'
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const flow = '${{ inputs.flow }}';
+          const attempts = '${{ inputs.attempts }}';
+          const outcome = '${{ inputs.outcome }}';
+          const rate = '${{ steps.rate.outputs.rate }}';
+          const flake = '${{ steps.rate.outputs.flake_rate }}';
+          const n = '${{ steps.rate.outputs.n }}';
+          const body = [
+            '### Tier-3 Maestro flow `' + flow + '`',
+            '',
+            '| Metric | Value |',
+            '|---|---|',
+            '| This run outcome | `' + outcome + '` |',
+            '| Attempts this run | `' + attempts + '` |',
+            '| Trailing-30-day green rate | `' + rate + '` (n=' + n + ') |',
+            '| Trailing-30-day flake rate | `' + flake + '` |',
+            '',
+            'Source: `.maestro/bitbox/flake-log.jsonl` (this PR).',
+          ].join('\n');
+          await github.rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+            body
+          });
+
+    - name: Note cross-repo coverage update (stub)
+      if: github.event_name == 'push' || github.event_name == 'schedule'
+      shell: bash
+      run: |
+        # The audit BL-100 cross-repo-audit workflow + the
+        # coverage-honesty.yaml workflow in bitbox-testkit pull this
+        # per-flow flake-rate when they next run. We do NOT push to
+        # bitbox-testkit directly from here -- cross-repo writes are
+        # gated through that workflow's own permissions.
+        echo "::notice::flake log updated; bitbox-testkit/coverage_report.md will pick this up on its next run"

.github/actions/bitbox-maestro-flow/action.yml

@@ -0,0 +1,100 @@
+name: BitBox Maestro flow runner
+description: |
+  Runs one .maestro/bitbox/M-*.yaml flow against the real BitBox 02 Nova
+  hardware on the self-hosted runner. Retries up to `retries` times if
+  the failure mode is recoverable (IOSDriverTimeoutException, XCTest
+  driver crash). Assertion failures are NEVER retried -- those are
+  real regressions and must surface as red checks.
+
+inputs:
+  flow:
+    description: "Filename under .maestro/bitbox/ (e.g. M-1-happy-path.yaml)"
+    required: true
+  retries:
+    description: "Max attempts (default 3)"
+    required: false
+    default: "3"
+  background:
+    description: "Run in background (for the M-5 two-phase case)"
+    required: false
+    default: "false"
+
+outputs:
+  attempts:
+    description: "Number of attempts taken"
+    value: ${{ steps.run.outputs.attempts }}
+
+runs:
+  using: composite
+  steps:
+    - name: Run Maestro flow
+      id: run
+      shell: bash
+      env:
+        FLOW: ${{ inputs.flow }}
+        MAX_ATTEMPTS: ${{ inputs.retries }}
+        BACKGROUND: ${{ inputs.background }}
+      run: |
+        set -euo pipefail
+        MAESTRO="${MAESTRO:-$HOME/.maestro/bin/maestro}"
+        FLOW_PATH=".maestro/bitbox/${FLOW}"
+        if [ ! -f "$FLOW_PATH" ]; then
+          echo "::error::flow file not found: $FLOW_PATH"
+          exit 1
+        fi
+
+        run_once () {
+          local attempt="$1"
+          local log="/tmp/maestro-${FLOW%.yaml}-attempt-${attempt}.log"
+          echo "=== ATTEMPT $attempt :: $FLOW ==="
+          if [ "$BACKGROUND" = "true" ]; then
+            "$MAESTRO" test "$FLOW_PATH" --debug-output "/tmp/maestro-debug-${attempt}" >"$log" 2>&1 &
+            echo $! > "/tmp/maestro-${FLOW%.yaml}-bg.pid"
+            sleep 2
+            return 0
+          fi
+          if "$MAESTRO" test "$FLOW_PATH" --debug-output "/tmp/maestro-debug-${attempt}" 2>&1 | tee "$log"; then
+            return 0
+          fi
+          # Recoverable failure modes: only IOSDriverTimeoutException +
+          # XCTestCase initialisation failures get a retry. Assertion
+          # failures fall through.
+          if grep -qE "IOSDriverTimeoutException|XCTestCase init failed|driver startup timeout" "$log"; then
+            echo "::warning::recoverable Maestro failure on attempt $attempt; will retry"
+            return 1
+          fi
+          # Assertion / other failures: hard-fail, do NOT retry.
+          echo "::error::non-recoverable failure (assertion or unknown) on attempt $attempt"
+          return 2
+        }
+
+        attempts=0
+        while [ $attempts -lt $MAX_ATTEMPTS ]; do
+          attempts=$((attempts+1))
+          rc=0
+          run_once "$attempts" || rc=$?
+          if [ $rc -eq 0 ]; then
+            echo "attempts=$attempts" >> "$GITHUB_OUTPUT"
+            echo "::notice::$FLOW green on attempt $attempts"
+            exit 0
+          fi
+          if [ $rc -eq 2 ]; then
+            echo "attempts=$attempts" >> "$GITHUB_OUTPUT"
+            exit 1
+          fi
+          # rc == 1 -> recoverable, loop
+        done
+        echo "attempts=$attempts" >> "$GITHUB_OUTPUT"
+        echo "::error::$FLOW exhausted $MAX_ATTEMPTS attempts"
+        exit 1
+
+    - name: Upload Maestro artefacts
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: maestro-${{ inputs.flow }}-${{ github.run_id }}-${{ github.run_attempt }}
+        path: |
+          /tmp/maestro-debug-*
+          /tmp/maestro-*.log
+        if-no-files-found: warn
+        retention-days: 14

.github/actions/bitbox-maestro-setup/action.yml

@@ -0,0 +1,97 @@
+name: BitBox Maestro setup
+description: |
+  Pre-flight setup for Tier-3 BitBox Maestro flows on the self-hosted
+  Apple Silicon runner. Builds the iOS Runner.app (and the Android APK
+  if requested), boots the primary iPhone simulator (and the secondary
+  if requested), installs the app, and verifies Maestro version pin.
+
+  Does NOT physically reset the BitBox -- that is per-flow responsibility
+  in the flow's docblock.
+
+inputs:
+  android:
+    description: "Build + install the Android APK in addition to iOS"
+    required: false
+    default: "false"
+  two-phone:
+    description: "Boot + install on the secondary iPhone for M-5"
+    required: false
+    default: "false"
+
+runs:
+  using: composite
+  steps:
+    - name: Flutter setup
+      uses: subosito/flutter-action@v2
+      with:
+        flutter-version: "3.41.6"
+        channel: stable
+        cache: true
+
+    - name: Flutter pub get + generators
+      shell: bash
+      run: |
+        set -euo pipefail
+        flutter pub get
+        dart run tool/generate_localization.dart
+        dart run tool/generate_release_info.dart
+        flutter pub run build_runner build
+
+    - name: Cache iOS DerivedData + Pods
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/Library/Developer/Xcode/DerivedData
+          ios/Pods
+        key: ios-derived-data-${{ runner.os }}-bitbox-${{ hashFiles('pubspec.lock', 'ios/Podfile.lock') }}
+        restore-keys: |
+          ios-derived-data-${{ runner.os }}-bitbox-
+
+    - name: Build iOS simulator app
+      shell: bash
+      run: flutter build ios --simulator --debug
+
+    - name: Build Android APK
+      if: inputs.android == 'true'
+      shell: bash
+      run: flutter build apk --debug
+
+    - name: Verify Maestro pin
+      shell: bash
+      run: |
+        set -euo pipefail
+        MAESTRO_VERSION="$(cat .maestro-version)"
+        if [ -z "$MAESTRO_VERSION" ]; then
+          echo "::error::.maestro-version missing"; exit 1
+        fi
+        INSTALLED="$($HOME/.maestro/bin/maestro --version 2>&1 | tail -n1 || echo none)"
+        echo "Maestro: installed=$INSTALLED expected=$MAESTRO_VERSION"
+        if [ "$INSTALLED" != "$MAESTRO_VERSION" ]; then
+          echo "::warning::Maestro version drift; runner may need provisioning"
+        fi
+
+    - name: Verify BitBox hardware reachable
+      shell: bash
+      run: |
+        set +e
+        # On the self-hosted runner the BitBox CLI (if installed) lists
+        # the device. If absent, skip the check with a warning rather
+        # than fail -- the per-flow logic surfaces the real failure.
+        if command -v bitbox-cli >/dev/null 2>&1; then
+          bitbox-cli ls 2>&1 || echo "::warning::bitbox-cli ls failed -- check device power"
+        else
+          echo "::notice::bitbox-cli not installed on runner; relying on per-flow detection"
+        fi
+        # iOS sims booted?
+        xcrun simctl list devices booted
+
+    - name: Verify Android device (if requested)
+      if: inputs.android == 'true'
+      shell: bash
+      run: |
+        set -euo pipefail
+        adb devices
+        if ! adb devices | grep -q "device$"; then
+          echo "::error::Android device not reachable; M-7 will fail PRECONDITION"
+          exit 1
+        fi