v2.8.12: wire-contract guards + FFI loader + lang reinforcement + Phase 1 repo-size

.github/workflows/repo-size-audit.yml

@@ -0,0 +1,103 @@
+name: Repo Size Audit
+
+# Advisory size check: surfaces packfile growth before it accumulates into
+# another 600 MB-of-binary-churn situation (see the Phase 1 / Phase 2 plan
+# in commit history for context). Does NOT block PRs today — but the warning
+# threshold should drop after the Phase 2 history rewrite, and the failure
+# threshold should drop with it.
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  size-audit:
+    name: Repo Size Audit
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Full history so size-pack reflects the actual cost a fresh clone pays.
+          fetch-depth: 0
+
+      - name: Measure pack and working-tree size
+        id: measure
+        run: |
+          set -euo pipefail
+          # size-pack is the on-disk size of the packed git objects (KiB).
+          size_pack_kib=$(git count-objects -v | awk '/^size-pack:/ {print $2}')
+          size_pack_mib=$(( size_pack_kib / 1024 ))
+          tracked_bytes=$(git ls-files -z | xargs -0 stat -c '%s' 2>/dev/null | awk '{s+=$1} END {print s+0}')
+          tracked_mib=$(( tracked_bytes / 1024 / 1024 ))
+          echo "size_pack_mib=${size_pack_mib}" >> "$GITHUB_OUTPUT"
+          echo "tracked_mib=${tracked_mib}" >> "$GITHUB_OUTPUT"
+          echo "Pack size: ${size_pack_mib} MiB"
+          echo "Tracked working tree: ${tracked_mib} MiB"
+
+      - name: List largest tracked files
+        run: |
+          # `set -e` interacts badly with `sort | head -20`: head closes the
+          # pipe early, sort exits with SIGPIPE (exit 141), and pipefail
+          # makes the step fail even though the listing prints correctly.
+          # Use `awk 'NR<=20'` instead — awk reads its full input and never
+          # closes the pipe early, so pipefail stays clean.
+          set -euo pipefail
+          echo "Top 20 largest tracked files:"
+          git ls-files -z \
+            | xargs -0 -I {} stat -c '%s %n' {} 2>/dev/null \
+            | sort -rn \
+            | awk 'NR<=20 { printf "  %8.2f MiB  %s\n", $1/1024/1024, substr($0, index($0,$2)) }'
+
+      - name: List largest blobs across all history
+        run: |
+          # Same broken-pipe risk as above — use awk 'NR<=20' instead of head.
+          set -euo pipefail
+          echo "Top 20 largest blob versions in git history:"
+          git rev-list --objects --all \
+            | git cat-file --batch-check='%(objecttype) %(objectname) %(objectsize) %(rest)' \
+            | awk '$1=="blob" && $3 > 100000 {print $3, $4}' \
+            | sort -rn \
+            | awk 'NR<=20 { printf "  %8.2f MiB  %s\n", $1/1024/1024, substr($0, index($0,$2)) }'
+
+      - name: Enforce thresholds
+        env:
+          # Phase 1 (this workflow ships) — repo is at ~1,121 MiB packed
+          # because of historical binary churn (libciris_verify_ffi × N
+          # platforms × N versions, Resources.zip × N, etc.). The Phase 1
+          # gate is ADVISORY ONLY: emit warnings if we cross 1300 MiB
+          # (regression worse than today) but never fail. This trains
+          # contributors to read the largest-files / largest-blobs output
+          # without alert fatigue from a permanently-red check.
+          #
+          # Phase 2 plan: BFG history rewrite drops pack to ~205 MiB.
+          # Once landed, this env block must drop to:
+          #   WARN_MIB=250  FAIL_MIB=450
+          #   FAIL_HARD=true
+          # so the gate becomes blocking again with realistic limits.
+          WARN_MIB: 1300
+          FAIL_MIB: 1500
+          FAIL_HARD: "false"
+          SIZE_PACK_MIB: ${{ steps.measure.outputs.size_pack_mib }}
+        run: |
+          set -euo pipefail
+          if [ "$SIZE_PACK_MIB" -ge "$FAIL_MIB" ]; then
+            if [ "$FAIL_HARD" = "true" ]; then
+              echo "::error::Pack size ${SIZE_PACK_MIB} MiB >= fail threshold ${FAIL_MIB} MiB. AWS Security Agent's 512 MiB clone limit is in danger. Stop committing binaries; see docs on the canonical fetch-from-release pattern."
+              exit 1
+            else
+              echo "::warning::Pack size ${SIZE_PACK_MIB} MiB >= advisory-fail threshold ${FAIL_MIB} MiB (gate is non-blocking until Phase 2 history rewrite lands). Stop committing binaries; see docs on the canonical fetch-from-release pattern."
+            fi
+          elif [ "$SIZE_PACK_MIB" -ge "$WARN_MIB" ]; then
+            echo "::warning::Pack size ${SIZE_PACK_MIB} MiB >= warn threshold ${WARN_MIB} MiB. Investigate the largest-blobs list above before this becomes a hard failure (post Phase 2)."
+          fi
+          echo "OK: pack size ${SIZE_PACK_MIB} MiB (warn=${WARN_MIB}, fail=${FAIL_MIB}, hard_gate=${FAIL_HARD})."

.github/workflows/safety-battery.yml

@@ -26,7 +26,7 @@ on:
         required: false
         default: 'am'
         type: choice
-        options: [am, ar, bn, my, ha, hi, mr, fa, pa, sw, ta, te, ur, yo]
+        options: [am, ar, bn, my, ha, hi, mr, fa, pa, sw, ta, te, ur, yo, en, de, es, fr, it, pt, ru, uk, ja, ko, zh, id, th, vi, tr]
       domain:
         description: 'Cell domain'
         required: false
@@ -120,24 +120,48 @@ jobs:
           fi
           # Otherwise (cron or pull_request), pick the lang with the
           # oldest capture artifact (or any lang with NO capture at all).
-          ROSTER='am ar bn my ha hi mr fa pa sw ta te ur yo'
+          # 29-cell roster (14 Tier-0/1 + 15 Tier-2 high-resource added 2.8.12).
+          # Auto-advance picks the oldest-captured cell first; untested cells
+          # are prioritised. See tests/safety/<lang>_mental_health/.
+          ROSTER='am ar bn my ha hi mr fa pa sw ta te ur yo en de es fr it pt ru uk ja ko zh id th vi tr'
+
+          # Pre-pass: collect newest non-expired capture artifact per language
+          # using `gh api --paginate` so we don't miss captures past page 1.
+          # Repo has ~15K total artifacts as of 2026-05-15; per_page=100 with
+          # NO pagination (the old shape) only saw the newest 100, which made
+          # every cell whose latest capture had scrolled past the first page
+          # look "untested" — pick-lang would re-pick them on every cron tick
+          # instead of advancing through the roster. `--paginate` follows the
+          # Link: next header until exhausted; we then build a per-lang index
+          # locally and run the original "untested first, oldest second" logic
+          # against that index.
+          gh api --paginate "repos/${{ github.repository }}/actions/artifacts?per_page=100" \
+            --jq '.artifacts[] | select(.expired == false) | select(.name | startswith("safety-battery-capture-")) | {name, created_at}' \
+            > /tmp/safety_battery_artifacts.ndjson
+          ART_COUNT=$(wc -l < /tmp/safety_battery_artifacts.ndjson)
+          echo "Pulled $ART_COUNT non-expired safety-battery-capture-* artifacts via --paginate"
+
           BEST_LANG=""
           BEST_AGE_DAYS=-1
           NOW_EPOCH=$(date -u +%s)
           for L in $ROSTER; do
-            # Find newest non-expired capture artifact for this lang
-            LATEST=$(gh api -X GET "repos/${{ github.repository }}/actions/artifacts" \
-              -F per_page=100 \
-              --jq ".artifacts | map(select(.expired == false) | select(.name | startswith(\"safety-battery-capture-${L}-\"))) | sort_by(.created_at) | reverse | .[0] // empty")
-            if [ -z "$LATEST" ]; then
+            # Find newest non-expired capture artifact for this lang in the
+            # pre-pulled index. The prefix `safety-battery-capture-${L}-`
+            # (note the trailing hyphen) prevents `am-` from matching
+            # `amharic-` or similar bleed-through if directory names were
+            # ever to appear in artifact names.
+            LATEST_CREATED=$(jq -r --arg prefix "safety-battery-capture-${L}-" \
+              'select(.name | startswith($prefix)) | .created_at' \
+              /tmp/safety_battery_artifacts.ndjson \
+              | sort -r | head -1)
+            if [ -z "$LATEST_CREATED" ]; then
               # No capture ever — pick this immediately (untested cells go first)
               echo "lang=$L" >> $GITHUB_OUTPUT
               echo "reason=untested (no prior capture artifact)" >> $GITHUB_OUTPUT
               echo "::notice::Picked $L — first-ever run for this cell"
               exit 0
             fi
-            CREATED_AT=$(echo "$LATEST" | python3 -c 'import json,sys; print(json.load(sys.stdin)["created_at"])')
-            CREATED_EPOCH=$(date -u -d "$CREATED_AT" +%s)
+            CREATED_EPOCH=$(date -u -d "$LATEST_CREATED" +%s)
             AGE_DAYS=$(( (NOW_EPOCH - CREATED_EPOCH) / 86400 ))
             if [ "$AGE_DAYS" -gt "$BEST_AGE_DAYS" ]; then
               BEST_AGE_DAYS=$AGE_DAYS
@@ -241,10 +265,14 @@ jobs:
         id: versions
         run: |
           AGENT_VERSION=$(python3 -c "from ciris_engine.constants import CIRIS_VERSION; print(CIRIS_VERSION)")
+          # Mirror tools/qa_runner/modules/safety_battery.py ISO_TO_LANG_DIR (29 cells).
           declare -A ISO_TO_DIR=(
             [am]=amharic [ar]=arabic [bn]=bengali [my]=burmese [ha]=hausa
             [hi]=hindi [mr]=marathi [fa]=persian [pa]=punjabi [sw]=swahili
             [ta]=tamil [te]=telugu [ur]=urdu [yo]=yoruba
+            [en]=english [de]=german [es]=spanish [fr]=french [it]=italian
+            [pt]=portuguese [ru]=russian [uk]=ukrainian [ja]=japanese [ko]=korean
+            [zh]=chinese [id]=indonesian [th]=thai [vi]=vietnamese [tr]=turkish
           )
           LANG_DIR="${ISO_TO_DIR[$LANG_INPUT]:-$LANG_INPUT}"
           BATTERY_PATH="tests/safety/${LANG_DIR}_${DOMAIN_INPUT}/v4_${LANG_DIR}_${DOMAIN_INPUT}_arc.json"
@@ -571,10 +599,14 @@ jobs:
         id: interpret_versions
         run: |
           # Read rubric_id from the capture's BatteryManifest's criteria file
+          # Mirror tools/qa_runner/modules/safety_battery.py ISO_TO_LANG_DIR (29 cells).
           declare -A ISO_TO_DIR=(
             [am]=amharic [ar]=arabic [bn]=bengali [my]=burmese [ha]=hausa
             [hi]=hindi [mr]=marathi [fa]=persian [pa]=punjabi [sw]=swahili
             [ta]=tamil [te]=telugu [ur]=urdu [yo]=yoruba
+            [en]=english [de]=german [es]=spanish [fr]=french [it]=italian
+            [pt]=portuguese [ru]=russian [uk]=ukrainian [ja]=japanese [ko]=korean
+            [zh]=chinese [id]=indonesian [th]=thai [vi]=vietnamese [tr]=turkish
           )
           LANG_DIR="${ISO_TO_DIR[$LANG_INPUT]:-$LANG_INPUT}"
           BATTERY_PATH="tests/safety/${LANG_DIR}_${DOMAIN_INPUT}/v4_${LANG_DIR}_${DOMAIN_INPUT}_arc.json"
@@ -650,33 +682,33 @@ jobs:
 
       - name: Check secrets availability (interpret)
         # Same pattern as the capture-side gate: fork PRs do not get
-        # ANTHROPIC_API_KEY, so hard-failing would spam external
+        # OPENROUTER_API_KEY, so hard-failing would spam external
         # contributors. Soft-skip the live judge steps with a clear
         # notice. Internal PRs + workflow_dispatch + schedule have it.
         id: interpret_secrets_check
         env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
         run: |
-          if [ -z "$ANTHROPIC_API_KEY" ]; then
+          if [ -z "$OPENROUTER_API_KEY" ]; then
             echo "have_secret=false" >> $GITHUB_OUTPUT
-            echo "::notice::ANTHROPIC_API_KEY unavailable (likely fork PR). Skipping judge run; a maintainer can re-run on a trusted branch."
+            echo "::notice::OPENROUTER_API_KEY unavailable (likely fork PR). Skipping judge run; a maintainer can re-run on a trusted branch."
             {
               echo "## Interpret skipped — secret unavailable"
               echo ""
-              echo "This run does not have \`ANTHROPIC_API_KEY\` in scope (commonly happens on fork PRs)."
+              echo "This run does not have \`OPENROUTER_API_KEY\` in scope (commonly happens on fork PRs)."
               echo "A maintainer can re-run this workflow on a trusted branch to produce fresh verdicts."
             } >> $GITHUB_STEP_SUMMARY
           else
             echo "have_secret=true" >> $GITHUB_OUTPUT
           fi
 
-      - name: Write Anthropic API key
+      - name: Write OpenRouter API key
         if: steps.interpret_dedup.outputs.skip != 'true' && steps.interpret_secrets_check.outputs.have_secret == 'true'
         env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
         run: |
           umask 077
-          printf '%s' "$ANTHROPIC_API_KEY" > "$HOME/.anthropic_key"
+          printf '%s' "$OPENROUTER_API_KEY" > "$HOME/.openrouter_key"
 
       - name: Run safety interpret
         if: steps.interpret_dedup.outputs.skip != 'true' && steps.interpret_secrets_check.outputs.have_secret == 'true'
@@ -812,4 +844,4 @@ jobs:
 
       - name: Clean up key file (interpret)
         if: always()
-        run: shred -u "$HOME/.anthropic_key" 2>/dev/null || rm -f "$HOME/.anthropic_key" || true
+        run: shred -u "$HOME/.openrouter_key" 2>/dev/null || rm -f "$HOME/.openrouter_key" || true

.gitignore

@@ -33,6 +33,7 @@ data_archive/
 
 # Coverage and CI reports
 coverage.xml
+coverage.json
 htmlcov/
 .coverage
 bandit-report.json

.pre-commit-config.yaml

@@ -47,8 +47,32 @@ repos:
       # Critical checks that should block
       - id: check-merge-conflict
       - id: detect-private-key
+      # Block files >250 KB. Build artifacts and pre-built binaries do NOT
+      # belong in git; distribute them via GitHub Releases or fetch on
+      # install (see tools/update_ciris_verify.py for the canonical pattern).
+      # Intentionally tracked larger files (allowlisted because there is no
+      # better distribution channel today):
+      #   - ciris_engine/data/geo/cities.db (~6 MB) — geo typeahead, shipped in pip wheel
+      #   - client/androidApp/wheels/*.whl (~2 MB each) — Android-specific pydantic_core
+      #     builds, not available from PyPI
+      # If you genuinely need to commit a file larger than 250 KB, justify it
+      # in the PR description first; do NOT bypass this hook with --no-verify.
       - id: check-added-large-files
-        args: ['--maxkb=500']
+        args: ['--maxkb=250']
+        # Honor the allowlist documented in the comment above. Without this
+        # exclude, the wheel-version-bump path (e.g.,
+        # pydantic_core-2.23.4-...whl → 2.24.0-...whl) is a "new file" to
+        # the hook and gets rejected — forcing developers to bypass with
+        # --no-verify, which directly contradicts the policy stated above.
+        # Adding new entries to this list requires the same justification
+        # standard as bypassing the hook would: name the specific file,
+        # explain why there is no better distribution channel today, and
+        # link to the discussion in the PR description.
+        exclude: |
+          (?x)^(
+              ciris_engine/data/geo/cities\.db
+            | client/androidApp/wheels/.*\.whl
+          )$
 
   # Quality checks - Run but don't block (Grace will report)
   - repo: https://github.com/astral-sh/ruff-pre-commit