feat: evidence-grade NIM discovery + all-modality cost-quality benchmark

.github/workflows/fuzz.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
   schedule:
     # Weekly deeper run (longer per-target budget via FUZZ_SECONDS).
     - cron: "41 4 * * 2"
@@ -26,6 +25,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python
@@ -50,6 +50,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python
@@ -83,6 +84,9 @@ jobs:
       - name: Fuzz orchestration engine
         run: python fuzz/fuzz_orchestration.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/orchestration
 
+      - name: Fuzz NIM model-catalog parser
+        run: python fuzz/fuzz_nim_catalog.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/nim_catalog
+
       - name: Upload crash artifacts
         if: failure()
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5

.github/workflows/nim-benchmark.yml

@@ -0,0 +1,149 @@
+name: NIM benchmark
+
+# Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark.
+# Manual dispatch defaults to a deterministic dry run. Monthly scheduled runs
+# are live but use a conservative hard request cap. Dry execution receives no
+# provider credential; only the live step can read NVIDIA_NIM_API_KEY.
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run without contacting NVIDIA"
+        type: boolean
+        default: true
+      max_total_requests:
+        description: "Hard cap on provider requests for this run"
+        type: number
+        default: 2000
+      pricing_scenario:
+        description: "Optional reviewed pricing-scenario JSON path"
+        type: string
+        default: ""
+  schedule:
+    - cron: "23 3 5 * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: nim-benchmark
+  cancel-in-progress: false
+
+jobs:
+  dry_run_benchmark:
+    name: Deterministic NIM benchmark dry run
+    if: github.event_name == 'workflow_dispatch' && inputs.dry_run == true
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install pinned runtime
+        run: |
+          python -m pip install --require-hashes -r requirements.lock
+          python -m pip install --no-deps -e .
+
+      - name: Run dry benchmark
+        env:
+          MAX_REQUESTS: ${{ inputs.max_total_requests }}
+          PRICING_SCENARIO: ${{ inputs.pricing_scenario }}
+          PROVENANCE_GIT_SHA: ${{ github.sha }}
+          PROVENANCE_RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          extra_args=()
+          if [ -n "$PRICING_SCENARIO" ]; then
+            extra_args+=(--pricing-scenario "$PRICING_SCENARIO")
+          fi
+          python -m contextual_orchestrator nim-benchmark \
+            --dry-run \
+            "${extra_args[@]}" \
+            --task-manifest examples/nim_task_manifest.json \
+            --output-dir benchmark_artifacts \
+            --max-total-requests "$MAX_REQUESTS" \
+            --git-sha "$PROVENANCE_GIT_SHA" \
+            --workflow-run-id "$PROVENANCE_RUN_ID"
+
+      - name: Upload dry-run artifacts
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5
+        with:
+          name: nim-benchmark-dry-${{ github.run_id }}
+          path: benchmark_artifacts/
+          retention-days: 30
+          if-no-files-found: error
+
+  live_benchmark:
+    name: Live NIM catalog benchmark
+    if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.dry_run != true)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install pinned runtime
+        run: |
+          python -m pip install --require-hashes -r requirements.lock
+          python -m pip install --no-deps -e .
+
+      - name: Resolve live parameters
+        id: live_params
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_MAX_REQUESTS: ${{ inputs.max_total_requests }}
+          INPUT_PRICING: ${{ inputs.pricing_scenario }}
+        run: |
+          set -euo pipefail
+          if [ "$EVENT_NAME" = "schedule" ]; then
+            echo "max_requests=2000" >> "$GITHUB_OUTPUT"
+            echo "pricing_scenario=" >> "$GITHUB_OUTPUT"
+          else
+            echo "max_requests=${INPUT_MAX_REQUESTS}" >> "$GITHUB_OUTPUT"
+            echo "pricing_scenario=${INPUT_PRICING}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run live benchmark
+        env:
+          NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}
+          MAX_REQUESTS: ${{ steps.live_params.outputs.max_requests }}
+          PRICING_SCENARIO: ${{ steps.live_params.outputs.pricing_scenario }}
+          PROVENANCE_GIT_SHA: ${{ github.sha }}
+          PROVENANCE_RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          extra_args=()
+          if [ -n "$PRICING_SCENARIO" ]; then
+            extra_args+=(--pricing-scenario "$PRICING_SCENARIO")
+          fi
+          python -m contextual_orchestrator nim-benchmark \
+            "${extra_args[@]}" \
+            --task-manifest examples/nim_task_manifest.json \
+            --output-dir benchmark_artifacts \
+            --max-total-requests "$MAX_REQUESTS" \
+            --git-sha "$PROVENANCE_GIT_SHA" \
+            --workflow-run-id "$PROVENANCE_RUN_ID"
+
+      - name: Upload live benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5
+        with:
+          name: nim-benchmark-live-${{ github.run_id }}
+          path: benchmark_artifacts/
+          retention-days: 90
+          if-no-files-found: error

.github/workflows/security.yml

@@ -9,7 +9,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
   schedule:
     - cron: "17 3 * * 1"
   workflow_dispatch:
@@ -34,6 +33,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Initialize CodeQL
@@ -54,6 +54,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python

.github/workflows/tests.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
 
 permissions:
   contents: read
@@ -21,6 +20,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Set up Python
@@ -36,3 +36,57 @@ jobs:
 
       - name: Run full test suite
         run: python -m pytest -q
+
+  nim_benchmark_quality:
+    name: NIM benchmark coverage, docstrings, and package smoke
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install hash-locked quality tools
+        run: python -m pip install --require-hashes -r requirements-opencode-review-ci.txt
+
+      - name: Prove complete benchmark coverage and public docstrings
+        run: |
+          set -euo pipefail
+          python -m coverage erase
+          python -m coverage run --branch \
+            --source=contextual_orchestrator.nim_benchmark,contextual_orchestrator.nim_csv_evidence \
+            -m pytest \
+            tests/test_nim_benchmark.py \
+            tests/test_nim_benchmark_budget_view.py \
+            tests/test_nim_benchmark_release_acceptance.py \
+            tests/test_nim_benchmark_review_regressions.py \
+            tests/test_nim_benchmark_workflow_contract.py \
+            tests/test_nim_artifact_publication.py \
+            tests/test_nim_artifact_publication_edges.py \
+            tests/test_nim_csv_evidence.py \
+            tests/test_nim_csv_evidence_edges.py \
+            -q
+          python -m coverage report \
+            --include=contextual_orchestrator/nim_benchmark.py,contextual_orchestrator/nim_csv_evidence.py \
+            --show-missing \
+            --fail-under=100
+          python -m interrogate -f 100 contextual_orchestrator/nim_benchmark.py
+          python -m interrogate -f 100 contextual_orchestrator/nim_csv_evidence.py
+
+      - name: Build, install, and import the wheel
+        run: |
+          set -euo pipefail
+          rm -rf dist "$RUNNER_TEMP/nim-wheel-site"
+          python -m pip wheel --no-deps . --wheel-dir dist
+          python -m pip install --no-deps \
+            --target "$RUNNER_TEMP/nim-wheel-site" \
+            dist/contextual_orchestrator-*.whl
+          cd "$RUNNER_TEMP"
+          PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \
+            python -c "import contextual_orchestrator; import contextual_orchestrator.nim_benchmark; import contextual_orchestrator.nim_csv_evidence"

.gitignore

@@ -12,3 +12,9 @@ tempcred.txt
 
 # hypothesis fuzzing DB
 .hypothesis/
+
+# coverage measurement data
+.coverage
+
+# local benchmark artifacts (uploaded by CI, not committed)
+benchmark_artifacts/

CHANGELOG.md

@@ -6,19 +6,43 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and
 
 ## [Unreleased]
 
+### Added
+
+- Add an optional provider-neutral NVIDIA NIM benchmark harness that dynamically discovers the live `/v1/models` catalog, probes every discovered model under bounded concurrency and a hard request cap, records machine-readable capability outcomes, and compares direct, route-once, bounded-conduct, and explicit pricing-scenario policies over a locked task manifest.
+- Add deterministic no-egress benchmark dry runs, secret-redacted JSON/CSV/Markdown evidence artifacts, paired bootstrap uncertainty, quality-latency and quality-hypothetical-cost Pareto frontiers, all-modality catalog fuzzing, and a manually gated benchmark workflow.
+- Add a validated deterministic one-frame H.264 MP4 probe fixture, complete preflight reservation for every discovered model-capability cell plus the full evaluation envelope, and a thirty-task locked manifest that reaches the declared paired-evidence floor without creating an automatic routing recommendation.
+- Add direct benchmark quality gates for 100% production statement/branch coverage, 100% public docstrings, wheel build/install/import smoke testing, and optional-import isolation.
+
 ### Security
 
 - Restrict the private plain-HTTP provider seam to `localhost` or literal loopback IP addresses, reject URL userinfo before connection, dial directly without ambient proxy lookup, reject all redirect responses, and close failed resources deterministically.
 - Pin each HTTPS provider connection to the exact public addresses approved during validation, preserve the original hostname for TLS verification, bypass environment proxy resolution, and reject redirects to close DNS-rebinding and credential-forwarding SSRF paths.
 - Integrate DNS-pinned provider dispatch directly into `ModelClient` so package import performs no optional-adapter monkey-patching or order-dependent class mutation.
 - Reject provider hosts that resolve to any non-globally-routable address, including RFC 6598 shared address space, while retaining explicit multicast, private, loopback, link-local, and reserved-address protections.
-- Document narrowly scoped Semgrep suppressions for parameter-bound database queries, the explicit development-only TLS verification opt-out, and provider URLs that pass the egress guard.
+- Document narrowly scoped Semgrep suppressions for parameter-bound database queries and the explicit development-only TLS verification opt-out.
+- Remove the NIM benchmark's dynamic `urllib.request.urlopen` sink and compatibility monkeypatch path; live discovery, probes, and evaluation now use direct validation-time-address-pinned TLS with original-host SNI/certificate verification, no proxy lookup, no redirect following, and deterministic cleanup.
+- Bound every NIM provider response to 8 MiB and fail closed before an oversized catalog, probe, or evaluation body can exhaust benchmark-runner memory.
+- Split scheduled dry and live benchmark jobs so zero-egress dry runs never receive `NVIDIA_NIM_API_KEY`; only the bounded live benchmark step receives the GitHub Secret.
+- Remove temporary branch-writing/source-export repair mechanisms and the optional benchmark monkeypatch module from the mergeable tree.
+- Import the NIM catalog parser while Atheris import instrumentation is active, with an AST regression contract that prevents parser branches from silently losing coverage guidance.
 
 ### Changed
 
+- Make every repository-local pull-request checkout in Tests, Fuzz, and Security select `github.event.pull_request.head.sha` rather than GitHub's synthetic merge ref, while preserving `github.sha` for push, schedule, and manual runs and retaining non-persisted checkout credentials.
+- Preserve every benchmark cell's step, role, agent, and model assignment in CLI and scheduled-workflow CSV artifacts as deterministic JSON; fail closed before publishing success when JSON/CSV identities are duplicated, malformed, or incomplete, and replace the enriched CSV atomically.
+- Exclude zero-success benchmark policies from quality Pareto frontiers with explicit evidence labels, validate every Markdown-consumed report field before artifact writes, normalize excessive catalog JSON depth to the catalog domain error, preserve positive sub-second provider timeouts, make the standalone NIM test runner fixture-safe, and execute these review regressions inside the 100% branch-coverage gate.
+- Expose a stable complete-run request planning view and align API, CLI, manual-workflow, and deterministic test caps with the locked thirty-task evidence floor, preserving fail-before-probe behavior.
+- Fail closed after catalog discovery but before capability egress when the complete all-model probe and equal-budget evaluation plan cannot fit the configured hard request cap; the monthly 2,000-request ceiling covers the representative 127-model, thirty-task, seven-worker plan requiring 1,564 requests and still rejects larger plans before partial probing.
 - Pin Atheris by Python interpreter so the Python 3.11 fuzz job and the newer central coverage-evidence image both install a published, hash-locked wheel.
+- Record the reviewed current NVIDIA NIM General FAQ as expiring evidence for free Developer Program hosted-endpoint prototyping access, while keeping NVIDIA AI Enterprise production licensing and every hypothetical model rate explicitly separate.
+- Require live hypothetical pricing scenarios to carry reviewed source, reviewer, review date, validity horizon, rate basis, uncertainty, and explicit rates; reject unreviewed, incomplete, future-dated, or expired price evidence before provider egress.
+- Give direct, route-once, conduct, and reviewed cheapest-worker cells one equal total prompt-plus-completion token budget and one common five-call envelope, with configured-versus-observed evidence in every cell.
+- Keep the optional NIM adapter lazy: importing the runtime package no longer imports the benchmark or mutates benchmark globals.
+- Record immutable source-artifact digests and exact Git tree identity in the integration evidence so buyers and reviewers can reproduce the accepted benchmark source independently of transient workflow state.
 
 ### Documentation
 
-- Add APA 7 doctoring for Python environment-marker semantics, Atheris artifact availability and hashes, and the supported-platform uncertainty boundary.
+- Add APA 7 doctoring for Python environment-marker semantics, Atheris artifact availability and hashes, the NIM benchmark validity boundary, the thirty-task evidence floor, and supported-platform uncertainty.
 - Record the CI trust boundary between generic coverage and native fuzz execution, including the evidence-preserving retry rule for branch-referenced reusable workflows.
+- Make the NIM security-integration receipt head-stable: GitHub pull-request metadata and exact-head Checks are authoritative, while older commit and workflow identifiers remain explicitly historical only.
+- Clarify that the bundled thirty-task NIM manifest may reach `evidence_review_required` only when the paired-task and 90% completion floors are met; otherwise it remains `insufficient_evidence`, and no artifact changes production routing automatically.