From bb31ac172cd17388e850f0b9fba84e8d4b6b73ee Mon Sep 17 00:00:00 2001 From: Hardonian <118695431+Hardonian@users.noreply.github.com> Date: Fri, 29 May 2026 04:58:41 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Optimize=20sequential=20embedding?= =?UTF-8?q?=20generation=20to=20run=20in=20parallel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/verify-governance.ts | 41 ++++++----- scripts/verify-status-truth.ts | 115 ++++++++++++++++++++----------- scripts/verify-threat-seams.ts | 18 +++-- scripts/verify-version-drift.ts | 30 +++++--- src/lib/control-plane/meshrag.ts | 11 +-- 5 files changed, 133 insertions(+), 82 deletions(-) diff --git a/scripts/verify-governance.ts b/scripts/verify-governance.ts index e22200c8ab..bbeb0099e7 100644 --- a/scripts/verify-governance.ts +++ b/scripts/verify-governance.ts @@ -1,35 +1,40 @@ // SPDX-License-Identifier: Apache-2.0 -import { mkdirSync, writeFileSync } from 'node:fs'; -import { spawnSync } from 'node:child_process'; +import { mkdirSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; const checks = [ - 'npm run verify:status-truth', - 'npm run verify:threat-seams', - 'npm run verify:version-drift', + "npm run verify:status-truth", + "npm run verify:threat-seams", + "npm run verify:version-drift", ]; -const results: Array<{command:string;status:'pass'|'fail';exitCode:number|null}> = []; +const results: Array<{ command: string; status: "pass" | "fail"; exitCode: number | null }> = []; for (const command of checks) { - const run = spawnSync(command, { stdio: 'inherit', shell: true }); - results.push({ command, status: run.status === 0 ? 'pass' : 'fail', exitCode: run.status }); + const run = spawnSync(command, { stdio: "inherit", shell: true }); + results.push({ command, status: run.status === 0 ? "pass" : "fail", exitCode: run.status }); if (run.status !== 0) break; } -const overall = results.every((r)=>r.status==='pass') ? 'pass' : 'degraded'; +const overall = results.every((r) => r.status === "pass") ? "pass" : "degraded"; const scoreboard = { generatedAt: new Date().toISOString(), overall, categories: { - contractBoundaryIntegrity: results.find((r)=>r.command.includes('status-truth'))?.status ?? 'unknown', - threatSeamStatus: results.find((r)=>r.command.includes('threat-seams'))?.status ?? 'unknown', - versionDriftStatus: results.find((r)=>r.command.includes('version-drift'))?.status ?? 'unknown', + contractBoundaryIntegrity: + results.find((r) => r.command.includes("status-truth"))?.status ?? "unknown", + threatSeamStatus: results.find((r) => r.command.includes("threat-seams"))?.status ?? "unknown", + versionDriftStatus: + results.find((r) => r.command.includes("version-drift"))?.status ?? "unknown", }, verification: results, }; -mkdirSync('artifacts/governance', { recursive: true }); -writeFileSync('artifacts/governance/governance-scoreboard.json', JSON.stringify(scoreboard, null, 2) + '\n'); -const md = `# Governance Scoreboard\n\n- Overall: **${overall}**\n\n## Verification Commands\n${results.map((r)=>`- ${r.status.toUpperCase()}: \`${r.command}\``).join('\n')}\n`; -writeFileSync('artifacts/governance/governance-scoreboard.md', md); +mkdirSync("artifacts/governance", { recursive: true }); +writeFileSync( + "artifacts/governance/governance-scoreboard.json", + JSON.stringify(scoreboard, null, 2) + "\n", +); +const md = `# Governance Scoreboard\n\n- Overall: **${overall}**\n\n## Verification Commands\n${results.map((r) => `- ${r.status.toUpperCase()}: \`${r.command}\``).join("\n")}\n`; +writeFileSync("artifacts/governance/governance-scoreboard.md", md); -if (overall !== 'pass') process.exit(1); -console.log('verify:governance PASS'); +if (overall !== "pass") process.exit(1); +console.log("verify:governance PASS"); diff --git a/scripts/verify-status-truth.ts b/scripts/verify-status-truth.ts index 4354ef9f41..68041e85d0 100644 --- a/scripts/verify-status-truth.ts +++ b/scripts/verify-status-truth.ts @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import { execSync } from 'node:child_process'; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { execSync } from "node:child_process"; -type State = 'implemented' | 'partial' | 'planned' | 'deprecated' | 'unknown'; +type State = "implemented" | "partial" | "planned" | "deprecated" | "unknown"; type Component = { name: string; @@ -19,83 +19,114 @@ type Component = { }; const docs = { - current: readFileSync('docs/architecture/current-state.md', 'utf8'), - target: readFileSync('docs/architecture/target-state.md', 'utf8'), + current: readFileSync("docs/architecture/current-state.md", "utf8"), + target: readFileSync("docs/architecture/target-state.md", "utf8"), }; -const rawFiles = execSync('rg --files src test docs .github/workflows scripts').toString('utf8').trim(); -const files = rawFiles ? rawFiles.split('\n') : []; +const rawFiles = execSync("rg --files src test docs .github/workflows scripts") + .toString("utf8") + .trim(); +const files = rawFiles ? rawFiles.split("\n") : []; const hasFile = (pattern: RegExp) => files.some((f) => pattern.test(f)); const components: Component[] = [ { - name: 'control-plane', - state: hasFile(/src\/lib\/control-plane\/control-plane\.test\.ts$/) ? 'implemented' : 'partial', - implementationEvidence: ['src/lib/control-plane/orchestration.ts', 'src/lib/control-plane/scheduler.ts'], - testEvidence: ['src/lib/control-plane/control-plane.test.ts', 'src/lib/control-plane/runtime-seams.test.ts'], - docsEvidence: ['docs/architecture/current-state.md', 'docs/architecture/target-state.md'], - ciEvidence: ['.github/workflows/ci.yml'], - exportEvidence: ['src/lib/control-plane/evidence-export.ts'], + name: "control-plane", + state: hasFile(/src\/lib\/control-plane\/control-plane\.test\.ts$/) ? "implemented" : "partial", + implementationEvidence: [ + "src/lib/control-plane/orchestration.ts", + "src/lib/control-plane/scheduler.ts", + ], + testEvidence: [ + "src/lib/control-plane/control-plane.test.ts", + "src/lib/control-plane/runtime-seams.test.ts", + ], + docsEvidence: ["docs/architecture/current-state.md", "docs/architecture/target-state.md"], + ciEvidence: [".github/workflows/ci.yml"], + exportEvidence: ["src/lib/control-plane/evidence-export.ts"], driftFindings: [], - reasoning: 'Control-plane modules and tests exist and are wired into verification scripts.', + reasoning: "Control-plane modules and tests exist and are wired into verification scripts.", }, { - name: 'policy-egress-enforcement', - state: hasFile(/test\/ssrf-parity\.test\.ts$/) ? 'implemented' : 'partial', - implementationEvidence: ['src/lib/policies.ts', 'src/lib/private-networks.ts'], - testEvidence: ['test/ssrf-parity.test.ts', 'test/config-set-nested-ssrf.test.ts'], - docsEvidence: ['docs/verification/release-readiness.md'], - ciEvidence: ['.github/workflows/ci.yml'], + name: "policy-egress-enforcement", + state: hasFile(/test\/ssrf-parity\.test\.ts$/) ? "implemented" : "partial", + implementationEvidence: ["src/lib/policies.ts", "src/lib/private-networks.ts"], + testEvidence: ["test/ssrf-parity.test.ts", "test/config-set-nested-ssrf.test.ts"], + docsEvidence: ["docs/verification/release-readiness.md"], + ciEvidence: [".github/workflows/ci.yml"], exportEvidence: [], driftFindings: [], - reasoning: 'SSRF/policy tests and policy modules exist in repository truth paths.', + reasoning: "SSRF/policy tests and policy modules exist in repository truth paths.", }, { - name: 'proofpack-replay-governance', - state: hasFile(/scripts\/verify-proofpack\.ts$/) && hasFile(/scripts\/verify-replay\.ts$/) ? 'implemented' : 'partial', - implementationEvidence: ['src/lib/control-plane/proofpack-ledger.ts', 'src/lib/control-plane/replay.ts'], - testEvidence: ['src/lib/control-plane/proofpack-ledger.test.ts'], - docsEvidence: ['docs/verification/release-readiness.md'], - ciEvidence: ['.github/workflows/ci.yml'], - exportEvidence: ['scripts/verify-proofpack.ts', 'scripts/verify-replay.ts'], + name: "proofpack-replay-governance", + state: + hasFile(/scripts\/verify-proofpack\.ts$/) && hasFile(/scripts\/verify-replay\.ts$/) + ? "implemented" + : "partial", + implementationEvidence: [ + "src/lib/control-plane/proofpack-ledger.ts", + "src/lib/control-plane/replay.ts", + ], + testEvidence: ["src/lib/control-plane/proofpack-ledger.test.ts"], + docsEvidence: ["docs/verification/release-readiness.md"], + ciEvidence: [".github/workflows/ci.yml"], + exportEvidence: ["scripts/verify-proofpack.ts", "scripts/verify-replay.ts"], driftFindings: [], - reasoning: 'Proofpack and replay verification surfaces are present and script-addressable.', + reasoning: "Proofpack and replay verification surfaces are present and script-addressable.", }, ]; for (const component of components) { - for (const path of [...component.implementationEvidence, ...component.testEvidence, ...component.docsEvidence]) { + for (const path of [ + ...component.implementationEvidence, + ...component.testEvidence, + ...component.docsEvidence, + ]) { if (!files.includes(path)) { component.driftFindings.push(`Missing evidence file: ${path}`); - component.state = component.state === 'implemented' ? 'partial' : component.state; + component.state = component.state === "implemented" ? "partial" : component.state; } } } const staleClaims: string[] = []; if (/does not yet implement a dedicated deterministic control plane/i.test(docs.current)) { - staleClaims.push('current-state.md still claims deterministic control plane is not implemented.'); + staleClaims.push("current-state.md still claims deterministic control plane is not implemented."); } -if (/target state/i.test(docs.target) && /planned/i.test(docs.target) && components.some((c) => c.state === 'implemented')) { - staleClaims.push('target-state.md contains planned-only claims for implemented capabilities.'); +if ( + /target state/i.test(docs.target) && + /planned/i.test(docs.target) && + components.some((c) => c.state === "implemented") +) { + staleClaims.push("target-state.md contains planned-only claims for implemented capabilities."); } const conflicting = components.filter( - (c) => c.state === 'implemented' && (c.testEvidence.length === 0 || c.implementationEvidence.length === 0), + (c) => + c.state === "implemented" && + (c.testEvidence.length === 0 || c.implementationEvidence.length === 0), ); const payload = { generatedAt: new Date().toISOString(), - schemaVersion: '1.0.0', + schemaVersion: "1.0.0", components, staleClaims, }; -mkdirSync('artifacts/governance', { recursive: true }); -writeFileSync('artifacts/governance/status-truth-map.json', `${JSON.stringify(payload, null, 2)}\n`); +mkdirSync("artifacts/governance", { recursive: true }); +writeFileSync( + "artifacts/governance/status-truth-map.json", + `${JSON.stringify(payload, null, 2)}\n`, +); -if (staleClaims.length > 0 || conflicting.length > 0 || components.some((c) => c.driftFindings.length > 0)) { - console.error('verify:status-truth FAILED'); +if ( + staleClaims.length > 0 || + conflicting.length > 0 || + components.some((c) => c.driftFindings.length > 0) +) { + console.error("verify:status-truth FAILED"); for (const message of staleClaims) console.error(`- ${message}`); for (const c of components.filter((x) => x.driftFindings.length > 0)) { for (const finding of c.driftFindings) console.error(`- ${c.name}: ${finding}`); @@ -103,4 +134,4 @@ if (staleClaims.length > 0 || conflicting.length > 0 || components.some((c) => c process.exit(1); } -console.log('verify:status-truth PASS'); +console.log("verify:status-truth PASS"); diff --git a/scripts/verify-threat-seams.ts b/scripts/verify-threat-seams.ts index 11ce1aa5eb..4fd55b7d96 100644 --- a/scripts/verify-threat-seams.ts +++ b/scripts/verify-threat-seams.ts @@ -1,17 +1,23 @@ // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from 'node:child_process'; +import { spawnSync } from "node:child_process"; const suites = [ - ['npx', 'vitest', 'run', 'test/ssrf-parity.test.ts', 'test/config-set-nested-ssrf.test.ts'], - ['npx', 'vitest', 'run', 'src/lib/control-plane/runtime-seams.test.ts', 'src/lib/control-plane/degraded-taxonomy.test.ts'], + ["npx", "vitest", "run", "test/ssrf-parity.test.ts", "test/config-set-nested-ssrf.test.ts"], + [ + "npx", + "vitest", + "run", + "src/lib/control-plane/runtime-seams.test.ts", + "src/lib/control-plane/degraded-taxonomy.test.ts", + ], ]; for (const [cmd, ...args] of suites) { - const result = spawnSync(cmd, args, { stdio: 'inherit', shell: process.platform === 'win32' }); + const result = spawnSync(cmd, args, { stdio: "inherit", shell: process.platform === "win32" }); if (result.status !== 0) { - console.error(`verify:threat-seams FAILED at ${cmd} ${args.join(' ')}`); + console.error(`verify:threat-seams FAILED at ${cmd} ${args.join(" ")}`); process.exit(result.status ?? 1); } } -console.log('verify:threat-seams PASS'); +console.log("verify:threat-seams PASS"); diff --git a/scripts/verify-version-drift.ts b/scripts/verify-version-drift.ts index c00418c7c2..132a107506 100644 --- a/scripts/verify-version-drift.ts +++ b/scripts/verify-version-drift.ts @@ -1,28 +1,36 @@ // SPDX-License-Identifier: Apache-2.0 -import { readFileSync } from 'node:fs'; +import { readFileSync } from "node:fs"; -type Allow = { key:string; root:string; nemoclaw:string; reason:string }; +type Allow = { key: string; root: string; nemoclaw: string; reason: string }; -type Pkg = { version: string; dependencies?: Record; devDependencies?: Record }; -const root = JSON.parse(readFileSync('package.json','utf8')) as Pkg; -const nested = JSON.parse(readFileSync('nemoclaw/package.json','utf8')) as Pkg; +type Pkg = { + version: string; + dependencies?: Record; + devDependencies?: Record; +}; +const root = JSON.parse(readFileSync("package.json", "utf8")) as Pkg; +const nested = JSON.parse(readFileSync("nemoclaw/package.json", "utf8")) as Pkg; const issues: string[] = []; -const allowlist = JSON.parse(readFileSync('docs/verification/version-drift-allowlist.json','utf8')) as {allowed: Allow[]}; -const isAllowed=(key:string,rootV:string,nestedV:string)=>allowlist.allowed.some((a)=>a.key===key&&a.root===rootV&&a.nemoclaw===nestedV); +const allowlist = JSON.parse( + readFileSync("docs/verification/version-drift-allowlist.json", "utf8"), +) as { allowed: Allow[] }; +const isAllowed = (key: string, rootV: string, nestedV: string) => + allowlist.allowed.some((a) => a.key === key && a.root === rootV && a.nemoclaw === nestedV); if (root.version !== nested.version) { issues.push(`Version drift: root=${root.version} nemoclaw=${nested.version}`); } -for (const dep of ['typescript','vitest']) { +for (const dep of ["typescript", "vitest"]) { const a = root.devDependencies?.[dep]; const b = nested.devDependencies?.[dep]; - if (a && b && a !== b && !isAllowed(`devDependency:${dep}`, a, b)) issues.push(`Tooling drift for ${dep}: root=${a} nemoclaw=${b}`); + if (a && b && a !== b && !isAllowed(`devDependency:${dep}`, a, b)) + issues.push(`Tooling drift for ${dep}: root=${a} nemoclaw=${b}`); } if (issues.length) { - console.error('verify:version-drift FAILED'); + console.error("verify:version-drift FAILED"); for (const issue of issues) console.error(`- ${issue}`); process.exit(1); } -console.log('verify:version-drift PASS'); +console.log("verify:version-drift PASS"); diff --git a/src/lib/control-plane/meshrag.ts b/src/lib/control-plane/meshrag.ts index 7e39000d39..3a3ea917c4 100644 --- a/src/lib/control-plane/meshrag.ts +++ b/src/lib/control-plane/meshrag.ts @@ -121,11 +121,12 @@ export async function ingestRecords(input: { records: RetrievalIngestionRecord[] if (input.embedding.status() === "unavailable") return { accepted: [], degraded: ["embedding_adapter_unavailable"] }; if (input.vectorStore.status() === "unavailable") return { accepted: [], degraded: ["vector_store_unavailable"] }; if (input.vectorStore.status() === "degraded") degraded.push("vector_store_degraded"); - const rows = []; - for (const record of input.records) { - const embedded = await input.embedding.embed({ text: record.content }); - rows.push({ recordId: record.recordId, vector: embedded.vector, record }); - } + const rows = await Promise.all( + input.records.map(async (record) => { + const embedded = await input.embedding.embed({ text: record.content }); + return { recordId: record.recordId, vector: embedded.vector, record }; + }) + ); await input.vectorStore.upsert(rows); return { accepted: rows.map((r) => r.recordId), degraded }; }