diff --git a/.gitignore b/.gitignore index 87c64cc..79bfee2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ site/dist/ .playwright-cli/ .worktrees/ output/ +packages/codex-plugin/dist/ diff --git a/package.json b/package.json index 635cdd4..1c2e32f 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,9 @@ }, "scripts": { "benchmark": "node evals/benchmark.mjs", + "codex-plugin:build": "node scripts/build-codex-plugin.mjs", + "codex-plugin:check": "node scripts/verify-codex-plugin.mjs", + "codex-plugin:sync:local": "node scripts/sync-codex-plugin.mjs", "capture:model-ui": "node scripts/capture-model-ui-matrix.mjs", "capture:model-ui:screenshots": "node scripts/capture-model-ui-screenshots.mjs", "demo:comparison": "node scripts/demo-comparison.mjs", @@ -36,7 +39,7 @@ "site:serve": "node scripts/site-local-server.mjs", "smoke:icons": "node scripts/lucide-icon-catalog-smoke.mjs", "smoke:openai-ui-workflow": "node scripts/smoke-openai-ui-workflow.mjs", - "test": "node tests/lucide-icon-catalog.test.mjs && node tests/lucide-icon-catalog-smoke.test.mjs && node tests/kernel-contract.test.mjs && node tests/analyze-implementation-brief.test.mjs && node tests/activity-model-review.test.mjs && node tests/surface-types.test.mjs && node tests/ui-generation-handoff.test.mjs && node tests/cognitive-dimensions-review.test.mjs && node tests/openai-responses-provider.test.mjs && node tests/cli.test.mjs && node tests/demo-one-shot.test.mjs && node tests/demo-modal-actions.test.mjs && node tests/demo-comparison.test.mjs && node tests/demo-music-comparison.test.mjs && node tests/demo-model-ui-matrix.test.mjs && node tests/ui-generation-evals.test.mjs && node tests/mcp-pilot-evals.test.mjs && node tests/mcp-pilot-llm-judge.test.mjs && node tests/eval-rerun-guardrails.test.mjs && node tests/mcp.test.mjs && node tests/mcp-stdio.test.mjs && node tests/mcp-http.test.mjs && node tests/install-mcp.test.mjs && node tests/ai-native-design-system-release.test.mjs && node tests/er-flow-dashboard.test.mjs && node tests/site.test.mjs && node tests/site-local-server.test.mjs && node tests/skills.test.mjs && node evals/run-evals.mjs" + "test": "node tests/lucide-icon-catalog.test.mjs && node tests/lucide-icon-catalog-smoke.test.mjs && node tests/kernel-contract.test.mjs && node tests/analyze-implementation-brief.test.mjs && node tests/activity-model-review.test.mjs && node tests/surface-types.test.mjs && node tests/ui-generation-handoff.test.mjs && node tests/cognitive-dimensions-review.test.mjs && node tests/openai-responses-provider.test.mjs && node tests/cli.test.mjs && node tests/demo-one-shot.test.mjs && node tests/demo-modal-actions.test.mjs && node tests/demo-comparison.test.mjs && node tests/demo-music-comparison.test.mjs && node tests/demo-model-ui-matrix.test.mjs && node tests/ui-generation-evals.test.mjs && node tests/mcp-pilot-evals.test.mjs && node tests/mcp-pilot-llm-judge.test.mjs && node tests/eval-rerun-guardrails.test.mjs && node tests/mcp.test.mjs && node tests/mcp-stdio.test.mjs && node tests/mcp-http.test.mjs && node tests/install-mcp.test.mjs && node tests/codex-plugin.test.mjs && node tests/ai-native-design-system-release.test.mjs && node tests/er-flow-dashboard.test.mjs && node tests/site.test.mjs && node tests/site-local-server.test.mjs && node tests/skills.test.mjs && node evals/run-evals.mjs" }, "dependencies": { "@emotion/cache": "^11.14.0", diff --git a/packages/codex-plugin/.codex-plugin/plugin.json b/packages/codex-plugin/.codex-plugin/plugin.json new file mode 100644 index 0000000..12491be --- /dev/null +++ b/packages/codex-plugin/.codex-plugin/plugin.json @@ -0,0 +1,39 @@ +{ + "name": "judgmentkit", + "version": "0.1.0", + "description": "AI UI review and handoff guidance that helps Codex preserve design intent, catch workflow mismatches, and enforce hard design-system acceptance gates.", + "author": { + "name": "JudgmentKit", + "url": "https://judgmentkit.ai" + }, + "homepage": "https://judgmentkit.ai", + "repository": "https://github.com/mikeylong/judgmentkit", + "keywords": [ + "ai-ui-review", + "design-handoff", + "design-system", + "figma", + "workflow-fit", + "ui-critique" + ], + "skills": "./skills/", + "interface": { + "displayName": "JudgmentKit", + "shortDescription": "AI UI review with hard design-system gates", + "longDescription": "JudgmentKit helps product designers and design teams review AI-generated UI before screens are planned, built, or accepted. It turns product briefs, Figma or Adobe handoffs, design-system guidance, and generated screens into clear decisions, states, terminology, workflow checks, implementation evidence, and hard design-system acceptance gates. If generated UI does not pass the active design system, it is a failed candidate, not an artifact. The hosted endpoint is for allowed or sanitized work; use a local or self-hosted JudgmentKit path for confidential briefs, unreleased designs, proprietary design-system details, source code, or customer data.", + "developerName": "JudgmentKit", + "category": "Productivity", + "capabilities": [ + "Interactive", + "Read" + ], + "websiteURL": "https://judgmentkit.ai", + "brandColor": "#255C99", + "defaultPrompt": [ + "Turn this product brief into UI decisions and review criteria.", + "Review this sanitized Figma or generated screen for workflow fit and disclosure issues.", + "Create hard design-system acceptance criteria for this interface." + ] + }, + "mcpServers": "./.mcp.json" +} diff --git a/packages/codex-plugin/.mcp.json b/packages/codex-plugin/.mcp.json new file mode 100644 index 0000000..777abf8 --- /dev/null +++ b/packages/codex-plugin/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "judgmentkit": { + "type": "http", + "url": "https://judgmentkit.ai/mcp" + } + } +} diff --git a/packages/codex-plugin/CHANGELOG.md b/packages/codex-plugin/CHANGELOG.md new file mode 100644 index 0000000..4fd401f --- /dev/null +++ b/packages/codex-plugin/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +## 0.1.0 - 2026-07-04 + +- Added versioned source package for the JudgmentKit Codex plugin under `packages/codex-plugin`. +- Bootstrapped plugin manifest, hosted MCP configuration, hosted MCP skill, and OpenAI agent metadata from the local personal plugin source. +- Normalized the committed plugin version to base semver `0.1.0` with no Codex cachebuster suffix. +- Documented the package-local version policy and hosted privacy/local boundary. diff --git a/packages/codex-plugin/README.md b/packages/codex-plugin/README.md new file mode 100644 index 0000000..83d8f81 --- /dev/null +++ b/packages/codex-plugin/README.md @@ -0,0 +1,33 @@ +# JudgmentKit Codex Plugin Source + +This package is the versioned source for the JudgmentKit Codex plugin. It was bootstrapped from the local personal plugin source, not from an installed Codex cache. + +## Version Policy + +- `package.json` is the committed plugin version source. +- `.codex-plugin/plugin.json` mirrors the same base semver version for Codex plugin metadata. +- Do not commit Codex timestamp or cachebuster suffixes in this source package. +- Cachebuster versions, if needed, belong only in local sync or install output outside this checked-in package. + +Current committed plugin version: `0.1.0`. + +## Package Contents + +- `package.json`: package metadata and authoritative plugin version. +- `.codex-plugin/plugin.json`: Codex plugin manifest. +- `.mcp.json`: hosted JudgmentKit MCP endpoint configuration. +- `skills/judgmentkit-hosted-mcp/SKILL.md`: agent-facing hosted MCP usage policy and workflow. +- `skills/judgmentkit-hosted-mcp/agents/openai.yaml`: OpenAI agent metadata for the skill. +- `CHANGELOG.md`: package-local release history. + +## Hosted Privacy Boundary + +The included MCP configuration points to `https://judgmentkit.ai/mcp`. Hosted requests leave the local environment and are appropriate only for allowed or sanitized work. + +Do not send confidential briefs, unreleased designs, proprietary design-system details, source code, customer data, or internal roadmaps to the hosted endpoint unless the active workspace policy allows it. For that work, use a local checkout, local stdio server, or self-hosted JudgmentKit MCP endpoint instead. + +## Source Boundary + +Keep this package limited to plugin source files. Do not use it for generated install caches, root project scripts, root tests, repository-wide docs, or installed Codex cache contents. + +Build output belongs under `dist/` and is ignored by git. Local personal plugin copies are sync targets, not source. diff --git a/packages/codex-plugin/package.json b/packages/codex-plugin/package.json new file mode 100644 index 0000000..043a35f --- /dev/null +++ b/packages/codex-plugin/package.json @@ -0,0 +1,18 @@ +{ + "name": "@judgmentkit/codex-plugin", + "version": "0.1.0", + "private": true, + "description": "Versioned source package for the JudgmentKit Codex plugin.", + "type": "module", + "files": [ + ".codex-plugin/", + ".mcp.json", + "skills/", + "README.md", + "CHANGELOG.md" + ], + "judgmentkit": { + "pluginManifest": "./.codex-plugin/plugin.json", + "versionSource": "./package.json" + } +} diff --git a/packages/codex-plugin/skills/judgmentkit-hosted-mcp/SKILL.md b/packages/codex-plugin/skills/judgmentkit-hosted-mcp/SKILL.md new file mode 100644 index 0000000..007c90d --- /dev/null +++ b/packages/codex-plugin/skills/judgmentkit-hosted-mcp/SKILL.md @@ -0,0 +1,149 @@ +--- +name: judgmentkit-hosted-mcp +description: Use JudgmentKit when the user explicitly asks or the project opts in to clarify what a UI must support, choose the right surface type, review workflow fit, set disclosure boundaries, enforce design-system acceptance gates, or prepare design-system handoff criteria before product UI is planned, generated, reviewed, or accepted. +--- + +# JudgmentKit Hosted MCP + +## Overview + +Use this skill to help product designers working with Codex clarify the job a product UI must do before screens, components, or styling are chosen. Start from a brief, Figma or Adobe handoff, design mockup, generated screen, or local source context and make the design decision set explicit: participants, outcomes, workflow, surface type, domain language, disclosure boundaries, required states, and evidence needed for handoff or acceptance. + +MCP and server details belong only in setup, diagnostics, audit, or integration work. Keep them out of product-facing UI and copy. + +## Design-System Acceptance Gate + +Treat the active design system as an existential gate for generated UI: + +- A generated UI that does not pass the active design system is not an artifact. It is a failed candidate. +- If no complete external design-system adapter is supplied and intended, the active authority is the internal JudgmentKit design system returned by `create_ui_implementation_contract`. +- Do not accept, render, publish, summarize as successful, or preserve a generated UI candidate until `review_ui_implementation_candidate` passes against the active implementation contract. +- Do not repair design-system failures in the renderer or downstream glue code. Send the failed candidate back through JudgmentKit MCP review and use the returned hard constraints and repair instructions. +- Do not treat "mostly uses tokens", wrapper normalization, fallback styling, visual cleanup, or post-hoc token rewriting as design-system compliance. +- Reject strict JudgmentKit candidates that carry visual authority outside the active design system, including inline `style` attributes, model-defined visual custom properties, local aliases, literal colors, literal shadows, undeclared token names, external fonts, or unsupported component semantics. +- Legacy, stale, or previously accepted captures must be re-reviewed against the active implementation contract before they can be used as current artifacts. +- If the active design-system review fails, the next action is repair or regeneration against the MCP-returned constraints, not acceptance with caveats. + +## Privacy Boundary + +The hosted endpoint processes the MCP request payload and records sanitized usage events such as event type and tool name. It does not intentionally store submitted briefs, design context, generated code, or review packets, but hosted requests still leave the local environment. + +Use sanitized inputs for confidential work. For unreleased designs, proprietary design-system details, source code, customer data, or internal roadmaps, prefer a local checkout, local stdio server, or self-hosted JudgmentKit MCP endpoint instead of `https://judgmentkit.ai/mcp`. + +## Policy Check + +Before calling MCP tools, check the active user, project, and AGENTS instructions. + +- Use JudgmentKit only when the user explicitly asks for it or the current project instructions opt into JudgmentKit for the task. +- Do not send confidential briefs, unreleased designs, proprietary design-system details, source code, customer data, or internal roadmap context to the hosted endpoint unless workspace policy allows it. +- When allowed, use JudgmentKit for UI generation, UI critique, implementation planning, workflow review, handoff review, and generated UI acceptance. +- Do not use JudgmentKit for unrelated backend work, factual answers, docs-only edits, or general code review unless the task depends on UI activity fit. +- If the activity is setup, debugging, auditing, or integration work, implementation details may be visible. Otherwise, keep prompts, schemas, resource ids, MCP servers, tool names, traces, and model configuration out of product UI. +- When ranking findings, use `High`, `Medium`, and `Low` labels. + +## Default Workflow + +1. Read the user's brief and available local source context. +2. Call `create_activity_model_review` with the source brief. +3. If the packet needs source context, ask only the targeted blocking questions unless local context answers them. +4. Call `recommend_surface_types({ brief, activity_review })` before workflow or frontend implementation guidance. +5. When a model or agent proposes an activity model, call `review_activity_model_candidate` before trusting it. +6. When a UI workflow is proposed, call `recommend_ui_workflow_profiles` for specialized review fit when useful, then call `review_ui_workflow_candidate`. +7. Call `review_cognitive_dimensions_candidate` when mapping, visibility, hidden dependencies, premature commitment, progressive evaluation, change cost, mental operations, or disclosure risks matter. +8. Call `create_ui_implementation_contract`. Use the default JudgmentKit design-system source unless a complete external design-system adapter is available and intended. +9. Call `create_ui_generation_handoff` with the reviewed workflow and implementation contract before generating UI. +10. Call `create_frontend_generation_context` when implementation needs project runtime, UI library, approved component families, visual requirements, verification commands, or browser checks. +11. Call `create_frontend_implementation_skill_context` when the implementing agent needs portable frontend guidance. +12. After implementation, call `review_ui_implementation_candidate` with the generated code or evidence and the active implementation contract before accepting the result. +13. If design-system authority fails, stop treating the output as generated UI. Use the MCP review result to issue hard constraints and regenerate or repair before any acceptance step. + +Use the MCP response `structuredContent` for follow-up tool calls and implementation decisions. Use `content[0].text` as the concise human-facing planning card. + +## Activity Contract + +Before suggesting screens, components, or styling, establish: + +- activity being supported +- participants +- objective and outcomes +- existing tools and artifacts +- rules and rituals +- division of labor +- domain vocabulary +- implementation concepts that must stay hidden +- diagnostics that may be revealed only when useful + +Then produce or verify the interaction contract: + +- what the user is trying to do +- how they think about the work +- primary decisions +- what should be easy or harder +- terms to use and avoid +- state changes that matter to the activity +- what the user should leave knowing or having done + +## Surface Types + +Treat surface type as activity-purpose guidance, not visual theme guidance: + +- `marketing`: persuade, orient, convert, or explain an offer +- `workbench`: inspect, compare, decide, and act across work items +- `operator_review`: review AI- or system-produced work, evidence, risk, and handoff +- `form_flow`: collect or change structured information with validation +- `dashboard_monitor`: track status, exceptions, trends, or operational health +- `content_report`: read, understand, cite, or share information +- `setup_debug_tool`: configure, inspect, test, or troubleshoot machinery +- `conversation`: support open-ended exchange where the thread is primary + +If two surface types are plausible, choose from the user's completion state before choosing components. For example, a monitor stays a monitor when tickets are downstream drill-in, while an operator review surface is appropriate when a human reviews AI- or system-produced work before it advances. + +## Disclosure Rules + +Product UI should use domain language and hide implementation machinery unless the user is doing setup, debugging, auditing, integration, or explicit source inspection. + +Avoid copying these terms into product UI: + +- `ready_for_review` +- `activity_model` +- `review_status` +- `Primary user` +- `Main decision` +- prompt template +- resource id +- schema +- tool call +- MCP server +- trace +- model configuration + +Translate implementation language into activity language: + +- Use `refund request`, not `database table`. +- Use `handoff reason`, not `tool call result`. +- Use `policy review`, not `prompt template`. + +## Implementation Gate + +Generate or accept frontend work only after the activity, target participant, primary decision, outcome, domain language, disclosure boundary, workflow candidate, surface type, and implementation contract are explicit enough for review. + +Design-system compliance is mandatory: + +- The generated UI must pass the active implementation contract's design-system source. +- If no external adapter is active, generated UI must use the internal JudgmentKit design-system authority exactly as returned by the contract. +- A failed design-system check blocks acceptance, publishing, screenshots, examples, release evidence, and handoff completion. +- Renderer-side cleanup may make failed work easier to inspect, but it must not convert failed work into an accepted artifact. + +Require evidence appropriate to the surface: + +- approved primitives and required states from the implementation contract +- static checks +- desktop and mobile browser QA +- accessibility evidence for semantics, keyboard navigation, focus order, focus-visible, responsive reflow/no-overflow, and automated checks +- conditional evidence for meaningful non-text visuals, custom widgets, forms, status messages, overlays, motion, dense controls, hover or focus content, and text over generated or rendered visuals + +When substantive visuals are required, use image generation, Three.js/WebGL, or D3-style visualization rather than rudimentary CSS/SVG/JS geometry as the final visual asset. + +## Fallback + +If the hosted MCP server is unavailable, say that JudgmentKit results could not be created and ask whether to continue with the deterministic activity-first checklist. Do not label checklist output as a JudgmentKit packet. diff --git a/packages/codex-plugin/skills/judgmentkit-hosted-mcp/agents/openai.yaml b/packages/codex-plugin/skills/judgmentkit-hosted-mcp/agents/openai.yaml new file mode 100644 index 0000000..5ff869f --- /dev/null +++ b/packages/codex-plugin/skills/judgmentkit-hosted-mcp/agents/openai.yaml @@ -0,0 +1,13 @@ +interface: + display_name: "JudgmentKit Hosted MCP" + short_description: "UI workflow review and hard design-system gates" + default_prompt: "Use $judgmentkit-hosted-mcp to review this UI brief for workflow fit, surface choice, disclosure boundaries, and the hard design-system acceptance gate. If the generated UI does not pass the active design system, treat it as a failed candidate and send it back through JudgmentKit for hard repair constraints." +dependencies: + tools: + - type: "mcp" + value: "judgmentkit" + description: "Hosted JudgmentKit MCP server" + transport: "streamable_http" + url: "https://judgmentkit.ai/mcp" +policy: + allow_implicit_invocation: false diff --git a/scripts/build-codex-plugin.mjs b/scripts/build-codex-plugin.mjs new file mode 100644 index 0000000..2ad63c1 --- /dev/null +++ b/scripts/build-codex-plugin.mjs @@ -0,0 +1,178 @@ +#!/usr/bin/env node +import fs from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; + +import { + CODEX_PLUGIN_MCP_URL, + DEFAULT_PACKAGE_JSON_PATH, + DEFAULT_SOURCE_DIR, + PROJECT_ROOT, + verifyCodexPlugin, +} from "./verify-codex-plugin.mjs"; + +export const DEFAULT_DIST_DIR = path.join(DEFAULT_SOURCE_DIR, "dist", "judgmentkit"); +export const PLUGIN_BUILD_ENTRIES = [ + "package.json", + ".codex-plugin", + ".mcp.json", + "skills", +]; + +function assertDistinctPaths(sourceDir, outDir) { + const resolvedSourceDir = path.resolve(sourceDir); + const resolvedOutDir = path.resolve(outDir); + const sourceRelativeToOut = path.relative(resolvedOutDir, resolvedSourceDir); + const outRelativeToSource = path.relative(resolvedSourceDir, resolvedOutDir); + + if (resolvedSourceDir === resolvedOutDir || !sourceRelativeToOut.startsWith("..")) { + throw new Error(`Build output must not contain, or be contained by, source directory: ${resolvedOutDir}`); + } + + if (!outRelativeToSource.startsWith("..") && !outRelativeToSource.split(path.sep).includes("dist")) { + throw new Error(`Build output inside source directory must be under dist/: ${resolvedOutDir}`); + } +} + +async function copyIfPresent(sourcePath, outPath) { + try { + await fs.cp(sourcePath, outPath, { + recursive: true, + force: true, + errorOnExist: false, + }); + return true; + } catch (error) { + if (error.code === "ENOENT") { + return false; + } + throw error; + } +} + +export async function buildCodexPlugin({ + sourceDir = DEFAULT_SOURCE_DIR, + outDir = DEFAULT_DIST_DIR, + packageJsonPath = DEFAULT_PACKAGE_JSON_PATH, + mcpUrl = CODEX_PLUGIN_MCP_URL, + clean = true, +} = {}) { + const resolvedSourceDir = path.resolve(sourceDir); + const resolvedOutDir = path.resolve(outDir); + const resolvedPackageJsonPath = path.resolve(packageJsonPath); + + assertDistinctPaths(resolvedSourceDir, resolvedOutDir); + + const sourceVerification = await verifyCodexPlugin({ + sourceDir: resolvedSourceDir, + packageJsonPath: resolvedPackageJsonPath, + mcpUrl, + }); + + await fs.rm(resolvedOutDir, { recursive: true, force: true }); + await fs.mkdir(resolvedOutDir, { recursive: true }); + + const copiedEntries = []; + for (const entry of PLUGIN_BUILD_ENTRIES) { + const didCopy = await copyIfPresent(path.join(resolvedSourceDir, entry), path.join(resolvedOutDir, entry)); + if (didCopy) copiedEntries.push(entry); + } + + const outputVerification = await verifyCodexPlugin({ + sourceDir: resolvedOutDir, + packageJsonPath: path.join(resolvedOutDir, "package.json"), + mcpUrl, + }); + + return { + status: "built", + source_dir: resolvedSourceDir, + out_dir: resolvedOutDir, + copied_entries: copiedEntries, + source_verification: sourceVerification, + output_verification: outputVerification, + }; +} + +function parseArgs(argv) { + const options = { + sourceDir: DEFAULT_SOURCE_DIR, + outDir: DEFAULT_DIST_DIR, + packageJsonPath: DEFAULT_PACKAGE_JSON_PATH, + mcpUrl: CODEX_PLUGIN_MCP_URL, + clean: true, + json: false, + help: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + + if (arg === "--source") { + options.sourceDir = argv[++index]; + } else if (arg === "--out") { + options.outDir = argv[++index]; + } else if (arg === "--package") { + options.packageJsonPath = argv[++index]; + } else if (arg === "--mcp-url") { + options.mcpUrl = argv[++index]; + } else if (arg === "--no-clean") { + throw new Error("--no-clean is not supported for Codex plugin builds."); + } else if (arg === "--json") { + options.json = true; + } else if (arg === "--help" || arg === "-h") { + options.help = true; + } else { + throw new Error(`Unsupported argument: ${arg}`); + } + } + + return options; +} + +function printUsage() { + process.stderr.write( + [ + "Usage:", + " node scripts/build-codex-plugin.mjs [--source ] [--out ] [--package ] [--mcp-url ] [--no-clean] [--json]", + "", + `Defaults to ${path.relative(PROJECT_ROOT, DEFAULT_SOURCE_DIR)} -> ${path.relative(PROJECT_ROOT, DEFAULT_DIST_DIR)}.`, + "", + ].join("\n"), + ); +} + +export async function runBuildCodexPluginCli(argv = process.argv.slice(2)) { + const options = parseArgs(argv); + + if (options.help) { + printUsage(); + return; + } + + const result = await buildCodexPlugin(options); + + if (options.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + return; + } + + process.stdout.write( + [ + "Codex plugin build complete.", + `Source: ${result.source_dir}`, + `Output: ${result.out_dir}`, + `Version: ${result.output_verification.plugin_version}`, + "", + ].join("\n"), + ); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + try { + await runBuildCodexPluginCli(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`Codex plugin build failed: ${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/sync-codex-plugin.mjs b/scripts/sync-codex-plugin.mjs new file mode 100644 index 0000000..2aa6bfa --- /dev/null +++ b/scripts/sync-codex-plugin.mjs @@ -0,0 +1,228 @@ +#!/usr/bin/env node +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; + +import { buildCodexPlugin, DEFAULT_DIST_DIR } from "./build-codex-plugin.mjs"; +import { + CODEX_PLUGIN_MCP_URL, + DEFAULT_PACKAGE_JSON_PATH, + DEFAULT_SOURCE_DIR, + PROJECT_ROOT, +} from "./verify-codex-plugin.mjs"; + +export const DEFAULT_LOCAL_PLUGIN_DIR = path.join(os.homedir(), "plugins", "judgmentkit"); + +function isSameOrInside(parentDir, childDir) { + const relative = path.relative(parentDir, childDir); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function pathsOverlap(firstDir, secondDir) { + return isSameOrInside(firstDir, secondDir) || isSameOrInside(secondDir, firstDir); +} + +function assertSafeSyncTarget(targetDir, protectedDirs = []) { + const resolvedTargetDir = path.resolve(targetDir); + const parsed = path.parse(resolvedTargetDir); + + if (resolvedTargetDir === parsed.root || resolvedTargetDir === os.homedir()) { + throw new Error(`Refusing to sync Codex plugin into unsafe target: ${resolvedTargetDir}`); + } + + for (const protectedDir of protectedDirs) { + const resolvedProtectedDir = path.resolve(protectedDir); + if (pathsOverlap(resolvedTargetDir, resolvedProtectedDir)) { + throw new Error( + `Refusing to sync Codex plugin into path that overlaps protected directory ${resolvedProtectedDir}: ${resolvedTargetDir}`, + ); + } + } +} + +function codexSuffix() { + return `+${"codex"}.`; +} + +export function createCodexSyncVersion(baseVersion, timestamp = new Date()) { + const stamp = timestamp.toISOString().replace(/\D/g, "").slice(0, 14); + return `${baseVersion}${codexSuffix()}${stamp}`; +} + +async function readJson(filePath) { + return JSON.parse(await fs.readFile(filePath, "utf8")); +} + +async function writeJson(filePath, value) { + await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`); +} + +export async function rewriteSyncedPluginVersion(pluginDir, version) { + const pluginJsonPath = path.join(pluginDir, ".codex-plugin", "plugin.json"); + const pluginJson = await readJson(pluginJsonPath); + pluginJson.version = version; + await writeJson(pluginJsonPath, pluginJson); + return { + plugin_json_path: pluginJsonPath, + version, + }; +} + +export async function syncCodexPlugin({ + sourceDir = DEFAULT_SOURCE_DIR, + outDir = DEFAULT_DIST_DIR, + targetDir = DEFAULT_LOCAL_PLUGIN_DIR, + packageJsonPath = DEFAULT_PACKAGE_JSON_PATH, + mcpUrl = CODEX_PLUGIN_MCP_URL, + cachebuster = true, + timestamp = new Date(), + cleanBuild = true, + dryRun = false, +} = {}) { + const resolvedTargetDir = path.resolve(targetDir); + const resolvedSourceDir = path.resolve(sourceDir); + const requestedOutDir = path.resolve(outDir); + const resolvedOutDir = dryRun + ? await fs.mkdtemp(path.join(os.tmpdir(), "judgmentkit-codex-plugin-dry-run-")) + : requestedOutDir; + assertSafeSyncTarget(resolvedTargetDir, [PROJECT_ROOT, resolvedSourceDir, requestedOutDir, resolvedOutDir]); + + const build = await buildCodexPlugin({ + sourceDir: resolvedSourceDir, + outDir: resolvedOutDir, + packageJsonPath, + mcpUrl, + clean: cleanBuild, + }); + + if (!dryRun) { + await fs.rm(resolvedTargetDir, { recursive: true, force: true }); + await fs.mkdir(path.dirname(resolvedTargetDir), { recursive: true }); + await fs.cp(build.out_dir, resolvedTargetDir, { + recursive: true, + force: true, + }); + } + + let syncedVersion = build.output_verification.plugin_version; + let versionRewrite = null; + + if (cachebuster) { + syncedVersion = createCodexSyncVersion(build.output_verification.plugin_version, timestamp); + if (!dryRun) { + versionRewrite = await rewriteSyncedPluginVersion(resolvedTargetDir, syncedVersion); + } + } + + return { + status: dryRun ? "dry_run" : "synced", + source_dir: build.source_dir, + build_dir: build.out_dir, + target_dir: resolvedTargetDir, + source_version: build.output_verification.plugin_version, + synced_version: syncedVersion, + cachebuster, + dry_run: dryRun, + version_rewrite: versionRewrite, + }; +} + +function parseArgs(argv) { + const options = { + sourceDir: DEFAULT_SOURCE_DIR, + outDir: DEFAULT_DIST_DIR, + targetDir: DEFAULT_LOCAL_PLUGIN_DIR, + packageJsonPath: DEFAULT_PACKAGE_JSON_PATH, + mcpUrl: CODEX_PLUGIN_MCP_URL, + cachebuster: true, + cleanBuild: true, + dryRun: false, + timestamp: undefined, + json: false, + help: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + + if (arg === "--source") { + options.sourceDir = argv[++index]; + } else if (arg === "--out") { + options.outDir = argv[++index]; + } else if (arg === "--target") { + options.targetDir = argv[++index]; + } else if (arg === "--package") { + options.packageJsonPath = argv[++index]; + } else if (arg === "--mcp-url") { + options.mcpUrl = argv[++index]; + } else if (arg === "--timestamp") { + options.timestamp = new Date(argv[++index]); + } else if (arg === "--no-cachebuster") { + options.cachebuster = false; + } else if (arg === "--no-clean-build") { + options.cleanBuild = false; + } else if (arg === "--dry-run") { + options.dryRun = true; + } else if (arg === "--json") { + options.json = true; + } else if (arg === "--help" || arg === "-h") { + options.help = true; + } else { + throw new Error(`Unsupported argument: ${arg}`); + } + } + + if (options.timestamp && Number.isNaN(options.timestamp.getTime())) { + throw new Error("--timestamp must be a valid date or ISO timestamp."); + } + + return options; +} + +function printUsage() { + process.stderr.write( + [ + "Usage:", + " node scripts/sync-codex-plugin.mjs [--source ] [--out ] [--target ] [--package ] [--mcp-url ] [--timestamp ] [--no-cachebuster] [--dry-run] [--json]", + "", + `Defaults to ${path.relative(PROJECT_ROOT, DEFAULT_SOURCE_DIR)} -> ${path.relative(PROJECT_ROOT, DEFAULT_DIST_DIR)} -> ${DEFAULT_LOCAL_PLUGIN_DIR}.`, + "", + ].join("\n"), + ); +} + +export async function runSyncCodexPluginCli(argv = process.argv.slice(2)) { + const options = parseArgs(argv); + + if (options.help) { + printUsage(); + return; + } + + const result = await syncCodexPlugin(options); + + if (options.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + return; + } + + process.stdout.write( + [ + result.dry_run ? "Codex plugin sync dry run complete." : "Codex plugin sync complete.", + `Target: ${result.target_dir}`, + `Source version: ${result.source_version}`, + `Synced version: ${result.synced_version}`, + "", + ].join("\n"), + ); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + try { + await runSyncCodexPluginCli(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`Codex plugin sync failed: ${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/verify-codex-plugin.mjs b/scripts/verify-codex-plugin.mjs new file mode 100644 index 0000000..f10593d --- /dev/null +++ b/scripts/verify-codex-plugin.mjs @@ -0,0 +1,539 @@ +#!/usr/bin/env node +import fs from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +export const PROJECT_ROOT = path.resolve(__dirname, ".."); +export const DEFAULT_SOURCE_DIR = path.join(PROJECT_ROOT, "packages", "codex-plugin"); +export const DEFAULT_PACKAGE_JSON_PATH = path.join(DEFAULT_SOURCE_DIR, "package.json"); +export const CODEX_PLUGIN_MCP_URL = "https://judgmentkit.ai/mcp"; +export const STRICT_BASE_SEMVER_PATTERN = /^\d+\.\d+\.\d+$/; + +export const REQUIRED_PLUGIN_FILES = [ + "package.json", + ".codex-plugin/plugin.json", + ".mcp.json", + "skills/judgmentkit-hosted-mcp/SKILL.md", + "skills/judgmentkit-hosted-mcp/agents/openai.yaml", +]; + +export const OLD_V1_TOOL_NAMES = [ + "list_resources", + "get_resource", + "get_workflow_bundle", + "get_page_markdown", + "get_example", + "resolve_related", +]; + +export const REQUIRED_PRIVACY_WORDING = [ + "The hosted endpoint processes the MCP request payload and records sanitized usage events", + "It does not intentionally store submitted briefs, design context, generated code, or review packets", + "Use sanitized inputs for confidential work.", + "For unreleased designs, proprietary design-system details, source code, customer data, or internal roadmaps, prefer a local checkout, local stdio server, or self-hosted JudgmentKit MCP endpoint instead of `https://judgmentkit.ai/mcp`.", +]; + +export const REQUIRED_DESIGN_SYSTEM_GATE_WORDING = [ + "A generated UI that does not pass the active design system is not an artifact. It is a failed candidate.", + "Do not accept, render, publish, summarize as successful, or preserve a generated UI candidate until `review_ui_implementation_candidate` passes against the active implementation contract.", + 'Do not treat "mostly uses tokens", wrapper normalization, fallback styling, visual cleanup, or post-hoc token rewriting as design-system compliance.', + "If the active design-system review fails, the next action is repair or regeneration against the MCP-returned constraints, not acceptance with caveats.", +]; + +const CODEX_SUFFIX_PATTERN = new RegExp(`\\+${"codex"}\\b`, "i"); +const LOCAL_HOME_PATTERN = new RegExp(`/Users/${"mike"}\\b`); +const CODEX_CACHE_PATTERN = new RegExp(`\\.codex/${"plugins"}/cache`); +const TODO_PATTERN = new RegExp(`\\b${["TO", "DO"].join("")}\\b`, "i"); + +const FORBIDDEN_TEXT_PATTERNS = [ + { + label: "Codex cachebuster version marker", + pattern: CODEX_SUFFIX_PATTERN, + }, + { + label: "absolute local home path", + pattern: LOCAL_HOME_PATTERN, + }, + { + label: "Codex plugin cache path", + pattern: CODEX_CACHE_PATTERN, + }, + { + label: "stdio MCP install instruction", + pattern: /\bmcp:stdio\b/i, + }, + { + label: "git clone install instruction", + pattern: /\bgit\s+clone\b/i, + }, + { + label: "npm install instruction", + pattern: /\bnpm\s+install\b/i, + }, + { + label: "unfinished placeholder marker", + pattern: TODO_PATTERN, + }, + ...OLD_V1_TOOL_NAMES.map((toolName) => ({ + label: `old v1 tool name ${toolName}`, + pattern: new RegExp(`\\b${escapeRegExp(toolName)}\\b`), + })), +]; + +export class CodexPluginVerificationError extends Error { + constructor(failures) { + super(`Codex plugin verification failed with ${failures.length} issue${failures.length === 1 ? "" : "s"}.`); + this.name = "CodexPluginVerificationError"; + this.failures = failures; + } +} + +function escapeRegExp(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function normalizeText(value) { + return String(value).replace(/\s+/g, " ").trim(); +} + +function formatRelative(rootDir, filePath) { + return path.relative(rootDir, filePath).split(path.sep).join("/"); +} + +async function pathExists(filePath) { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +async function readJsonFile(filePath, failures, label) { + try { + return JSON.parse(await fs.readFile(filePath, "utf8")); + } catch (error) { + failures.push({ + check: "valid_json", + file: filePath, + message: `${label} must be readable JSON: ${error.message}`, + }); + return null; + } +} + +async function listFiles(rootDir) { + const files = []; + + async function visit(currentDir) { + const entries = await fs.readdir(currentDir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(currentDir, entry.name); + + if (entry.isDirectory()) { + if (currentDir === rootDir && entry.name === "dist") { + continue; + } + await visit(fullPath); + } else if (entry.isFile()) { + files.push(fullPath); + } + } + } + + await visit(rootDir); + return files.sort(); +} + +async function readTextFiles(rootDir, files, failures) { + const entries = []; + + for (const filePath of files) { + try { + const text = await fs.readFile(filePath, "utf8"); + entries.push({ + filePath, + relativePath: formatRelative(rootDir, filePath), + text, + }); + } catch (error) { + failures.push({ + check: "read_source_file", + file: filePath, + message: `Could not read ${filePath}: ${error.message}`, + }); + } + } + + return entries; +} + +function collectForbiddenTextFailures(textEntries) { + const failures = []; + + for (const entry of textEntries) { + for (const rule of FORBIDDEN_TEXT_PATTERNS) { + if (rule.pattern.test(entry.text)) { + failures.push({ + check: "forbidden_source_text", + file: entry.relativePath, + message: `${entry.relativePath} contains ${rule.label}.`, + }); + } + } + } + + return failures; +} + +function collectMissingWordingFailures(skillText) { + const normalizedSkillText = normalizeText(skillText); + const failures = []; + + for (const phrase of REQUIRED_PRIVACY_WORDING) { + if (!normalizedSkillText.includes(normalizeText(phrase))) { + failures.push({ + check: "privacy_wording", + file: "skills/judgmentkit-hosted-mcp/SKILL.md", + message: `Missing required privacy wording from hosted MCP skill: ${phrase}`, + }); + } + } + + for (const phrase of REQUIRED_DESIGN_SYSTEM_GATE_WORDING) { + if (!normalizedSkillText.includes(normalizeText(phrase))) { + failures.push({ + check: "design_system_gate_wording", + file: "skills/judgmentkit-hosted-mcp/SKILL.md", + message: `Missing required design-system gate wording from hosted MCP skill: ${phrase}`, + }); + } + } + + return failures; +} + +function collectManifestFailures(pluginJson) { + const failures = []; + + if (pluginJson?.name !== "judgmentkit") { + failures.push({ + check: "plugin_manifest", + file: ".codex-plugin/plugin.json", + message: "Plugin manifest name must be judgmentkit.", + }); + } + + if (pluginJson?.skills !== "./skills/") { + failures.push({ + check: "plugin_manifest", + file: ".codex-plugin/plugin.json", + message: "Plugin manifest skills path must be ./skills/.", + }); + } + + if (pluginJson?.mcpServers !== "./.mcp.json") { + failures.push({ + check: "plugin_manifest", + file: ".codex-plugin/plugin.json", + message: "Plugin manifest mcpServers path must be ./.mcp.json.", + }); + } + + for (const [field, value] of [ + ["interface.displayName", pluginJson?.interface?.displayName], + ["interface.shortDescription", pluginJson?.interface?.shortDescription], + ["interface.longDescription", pluginJson?.interface?.longDescription], + ["interface.developerName", pluginJson?.interface?.developerName], + ["interface.category", pluginJson?.interface?.category], + ["interface.websiteURL", pluginJson?.interface?.websiteURL], + ]) { + if (typeof value !== "string" || value.trim().length === 0) { + failures.push({ + check: "plugin_manifest", + file: ".codex-plugin/plugin.json", + message: `Plugin manifest must define ${field}.`, + }); + } + } + + if (!Array.isArray(pluginJson?.interface?.capabilities) || !pluginJson.interface.capabilities.includes("Read")) { + failures.push({ + check: "plugin_manifest", + file: ".codex-plugin/plugin.json", + message: "Plugin manifest interface.capabilities must include Read.", + }); + } + + if (!Array.isArray(pluginJson?.interface?.defaultPrompt) || pluginJson.interface.defaultPrompt.length === 0) { + failures.push({ + check: "plugin_manifest", + file: ".codex-plugin/plugin.json", + message: "Plugin manifest interface.defaultPrompt must include at least one starter prompt.", + }); + } + + return failures; +} + +function collectAgentYamlFailures(agentYamlText, mcpUrl) { + const normalizedAgentYaml = normalizeText(agentYamlText); + const requiredSnippets = [ + 'display_name: "JudgmentKit Hosted MCP"', + 'type: "mcp"', + 'value: "judgmentkit"', + 'transport: "streamable_http"', + `url: "${mcpUrl}"`, + "allow_implicit_invocation: false", + ]; + const failures = []; + + for (const snippet of requiredSnippets) { + if (!normalizedAgentYaml.includes(normalizeText(snippet))) { + failures.push({ + check: "agent_yaml", + file: "skills/judgmentkit-hosted-mcp/agents/openai.yaml", + message: `OpenAI agent metadata must include ${snippet}.`, + }); + } + } + + if (normalizedAgentYaml.includes("allow_implicit_invocation: true")) { + failures.push({ + check: "agent_policy", + file: "skills/judgmentkit-hosted-mcp/agents/openai.yaml", + message: "Hosted JudgmentKit plugin must not allow implicit invocation.", + }); + } + + return failures; +} + +function assertNoFailures(failures) { + if (failures.length > 0) { + throw new CodexPluginVerificationError(failures); + } +} + +export async function verifyCodexPlugin({ + sourceDir = DEFAULT_SOURCE_DIR, + packageJsonPath = DEFAULT_PACKAGE_JSON_PATH, + mcpUrl = CODEX_PLUGIN_MCP_URL, +} = {}) { + const resolvedSourceDir = path.resolve(sourceDir); + const resolvedPackageJsonPath = path.resolve(packageJsonPath); + const failures = []; + + let sourceStat = null; + try { + sourceStat = await fs.stat(resolvedSourceDir); + } catch (error) { + if (error.code !== "ENOENT") { + throw error; + } + } + + if (!sourceStat?.isDirectory()) { + failures.push({ + check: "source_directory", + file: resolvedSourceDir, + message: `Codex plugin source directory does not exist: ${resolvedSourceDir}`, + }); + assertNoFailures(failures); + } + + const packageJson = await readJsonFile(resolvedPackageJsonPath, failures, "root package.json"); + const packageVersion = packageJson?.version; + + for (const relativePath of REQUIRED_PLUGIN_FILES) { + const filePath = path.join(resolvedSourceDir, relativePath); + if (!(await pathExists(filePath))) { + failures.push({ + check: "required_file", + file: relativePath, + message: `Missing required Codex plugin file: ${relativePath}`, + }); + } + } + + const pluginJsonPath = path.join(resolvedSourceDir, ".codex-plugin", "plugin.json"); + const pluginJson = (await pathExists(pluginJsonPath)) + ? await readJsonFile(pluginJsonPath, failures, "plugin manifest") + : null; + const pluginVersion = pluginJson?.version; + + if (typeof packageVersion !== "string" || packageVersion.trim().length === 0) { + failures.push({ + check: "package_version", + file: resolvedPackageJsonPath, + message: "Root package.json must define a non-empty version.", + }); + } + + if (typeof pluginVersion !== "string" || pluginVersion.trim().length === 0) { + failures.push({ + check: "plugin_version", + file: ".codex-plugin/plugin.json", + message: "Plugin manifest must define a non-empty version.", + }); + } else { + if (!STRICT_BASE_SEMVER_PATTERN.test(pluginVersion)) { + failures.push({ + check: "strict_base_semver", + file: ".codex-plugin/plugin.json", + message: `Plugin version must be strict base semver, got ${pluginVersion}.`, + }); + } + + if (CODEX_SUFFIX_PATTERN.test(pluginVersion)) { + failures.push({ + check: "no_codex_version_suffix", + file: ".codex-plugin/plugin.json", + message: "Plugin source version must not include Codex cachebuster build metadata.", + }); + } + } + + if (packageVersion && pluginVersion && packageVersion !== pluginVersion) { + failures.push({ + check: "package_plugin_version_equality", + file: ".codex-plugin/plugin.json", + message: `Plugin version ${pluginVersion} must equal package version ${packageVersion}.`, + }); + } + failures.push(...collectManifestFailures(pluginJson)); + + const mcpJsonPath = path.join(resolvedSourceDir, ".mcp.json"); + const mcpJson = (await pathExists(mcpJsonPath)) + ? await readJsonFile(mcpJsonPath, failures, "Codex MCP manifest") + : null; + const actualMcpUrl = mcpJson?.mcpServers?.judgmentkit?.url; + + if (actualMcpUrl !== mcpUrl) { + failures.push({ + check: "mcp_url", + file: ".mcp.json", + message: `mcpServers.judgmentkit.url must equal ${mcpUrl}, got ${actualMcpUrl ?? "(missing)"}.`, + }); + } + + if (mcpJson?.mcpServers?.judgmentkit?.type !== "http") { + failures.push({ + check: "mcp_transport", + file: ".mcp.json", + message: "mcpServers.judgmentkit.type must be http.", + }); + } + + const sourceFiles = await listFiles(resolvedSourceDir); + const textEntries = await readTextFiles(resolvedSourceDir, sourceFiles, failures); + failures.push(...collectForbiddenTextFailures(textEntries)); + + const skillEntry = textEntries.find((entry) => entry.relativePath === "skills/judgmentkit-hosted-mcp/SKILL.md"); + const agentYamlEntry = textEntries.find( + (entry) => entry.relativePath === "skills/judgmentkit-hosted-mcp/agents/openai.yaml", + ); + failures.push(...collectMissingWordingFailures(skillEntry?.text ?? "")); + failures.push(...collectAgentYamlFailures(agentYamlEntry?.text ?? "", mcpUrl)); + + assertNoFailures(failures); + + return { + status: "passed", + source_dir: resolvedSourceDir, + package_json_path: resolvedPackageJsonPath, + package_version: packageVersion, + plugin_version: pluginVersion, + mcp_url: actualMcpUrl, + files_checked: textEntries.map((entry) => entry.relativePath), + required_files: REQUIRED_PLUGIN_FILES, + }; +} + +function parseArgs(argv) { + const options = { + sourceDir: DEFAULT_SOURCE_DIR, + packageJsonPath: DEFAULT_PACKAGE_JSON_PATH, + mcpUrl: CODEX_PLUGIN_MCP_URL, + json: false, + help: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + + if (arg === "--source") { + options.sourceDir = argv[++index]; + } else if (arg === "--package") { + options.packageJsonPath = argv[++index]; + } else if (arg === "--mcp-url") { + options.mcpUrl = argv[++index]; + } else if (arg === "--json") { + options.json = true; + } else if (arg === "--help" || arg === "-h") { + options.help = true; + } else { + throw new Error(`Unsupported argument: ${arg}`); + } + } + + return options; +} + +function printUsage() { + process.stderr.write( + [ + "Usage:", + " node scripts/verify-codex-plugin.mjs [--source ] [--package ] [--mcp-url ] [--json]", + "", + `Defaults to ${path.relative(PROJECT_ROOT, DEFAULT_SOURCE_DIR)} with ${path.relative(PROJECT_ROOT, DEFAULT_PACKAGE_JSON_PATH)} and ${CODEX_PLUGIN_MCP_URL}.`, + "", + ].join("\n"), + ); +} + +export async function runVerifyCodexPluginCli(argv = process.argv.slice(2)) { + const options = parseArgs(argv); + + if (options.help) { + printUsage(); + return; + } + + const result = await verifyCodexPlugin(options); + + if (options.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + return; + } + + process.stdout.write( + [ + "Codex plugin verification passed.", + `Source: ${result.source_dir}`, + `Version: ${result.plugin_version}`, + `Files checked: ${result.files_checked.length}`, + "", + ].join("\n"), + ); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + try { + await runVerifyCodexPluginCli(process.argv.slice(2)); + } catch (error) { + if (error instanceof CodexPluginVerificationError) { + process.stderr.write(`${error.message}\n`); + for (const failure of error.failures) { + process.stderr.write(`- [${failure.check}] ${failure.message}\n`); + } + } else { + process.stderr.write(`Codex plugin verification failed: ${error.message}\n`); + } + process.exitCode = 1; + } +} diff --git a/tests/codex-plugin.test.mjs b/tests/codex-plugin.test.mjs new file mode 100644 index 0000000..5cd3599 --- /dev/null +++ b/tests/codex-plugin.test.mjs @@ -0,0 +1,452 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { buildCodexPlugin } from "../scripts/build-codex-plugin.mjs"; +import { syncCodexPlugin } from "../scripts/sync-codex-plugin.mjs"; +import { + CodexPluginVerificationError, + DEFAULT_SOURCE_DIR, + verifyCodexPlugin, +} from "../scripts/verify-codex-plugin.mjs"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(__dirname, ".."); +const DEFAULT_MCP_URL = "https://judgmentkit.ai/mcp"; + +function codexSuffix() { + return `+${"codex"}`; +} + +async function writeJson(filePath, value) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`); +} + +async function readJson(filePath) { + return JSON.parse(await fs.readFile(filePath, "utf8")); +} + +async function createFixture(overrides = {}) { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "judgmentkit-codex-plugin-")); + const sourceDir = path.join(tempDir, "packages", "codex-plugin"); + const packageJsonPath = path.join(sourceDir, "package.json"); + const version = overrides.packageVersion ?? "1.2.3"; + const pluginVersion = overrides.pluginVersion ?? version; + const skillText = overrides.skillText ?? validSkillText(); + const agentYaml = overrides.agentYaml ?? validAgentYaml(); + const pluginJson = { + name: "judgmentkit", + version: pluginVersion, + description: "AI UI review and handoff guidance that enforces hard design-system acceptance gates.", + skills: "./skills/", + mcpServers: "./.mcp.json", + interface: { + displayName: "JudgmentKit", + shortDescription: "AI UI review with hard design-system gates", + longDescription: "The hosted endpoint is for allowed or sanitized work; use a local or self-hosted JudgmentKit path for confidential briefs, unreleased designs, proprietary design-system details, source code, or customer data.", + developerName: "JudgmentKit", + category: "Productivity", + capabilities: ["Interactive", "Read"], + websiteURL: "https://judgmentkit.ai", + defaultPrompt: [ + "Turn this product brief into UI decisions and review criteria.", + ], + }, + }; + const mcpJson = { + mcpServers: { + judgmentkit: { + type: "http", + url: overrides.mcpUrl ?? DEFAULT_MCP_URL, + }, + }, + }; + + await writeJson(packageJsonPath, { + name: "judgmentkit", + version, + type: "module", + }); + await writeJson(path.join(sourceDir, ".codex-plugin", "plugin.json"), pluginJson); + await writeJson(path.join(sourceDir, ".mcp.json"), mcpJson); + await fs.mkdir(path.join(sourceDir, "skills", "judgmentkit-hosted-mcp", "agents"), { recursive: true }); + await fs.writeFile(path.join(sourceDir, "skills", "judgmentkit-hosted-mcp", "SKILL.md"), skillText); + + if (overrides.writeAgentYaml !== false) { + await fs.writeFile( + path.join(sourceDir, "skills", "judgmentkit-hosted-mcp", "agents", "openai.yaml"), + agentYaml, + ); + } + + if (overrides.extraFiles) { + for (const [relativePath, text] of Object.entries(overrides.extraFiles)) { + const filePath = path.join(sourceDir, relativePath); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, text); + } + } + + return { + tempDir, + packageJsonPath, + sourceDir, + version, + }; +} + +function validSkillText() { + return [ + "---", + "name: judgmentkit-hosted-mcp", + "description: Use JudgmentKit when the user explicitly asks or the project opts in to clarify what a UI must support, choose the right surface type, review workflow fit, set disclosure boundaries, enforce design-system acceptance gates, or prepare design-system handoff criteria before product UI is planned, generated, reviewed, or accepted.", + "---", + "", + "# JudgmentKit Hosted MCP", + "", + "## Design-System Acceptance Gate", + "", + "A generated UI that does not pass the active design system is not an artifact. It is a failed candidate.", + "", + "Do not accept, render, publish, summarize as successful, or preserve a generated UI candidate until `review_ui_implementation_candidate` passes against the active implementation contract.", + "", + 'Do not treat "mostly uses tokens", wrapper normalization, fallback styling, visual cleanup, or post-hoc token rewriting as design-system compliance.', + "", + "If the active design-system review fails, the next action is repair or regeneration against the MCP-returned constraints, not acceptance with caveats.", + "", + "## Privacy Boundary", + "", + "The hosted endpoint processes the MCP request payload and records sanitized usage events such as event type and tool name.", + "", + "It does not intentionally store submitted briefs, design context, generated code, or review packets, but hosted requests still leave the local environment.", + "", + "Use sanitized inputs for confidential work.", + "", + "For unreleased designs, proprietary design-system details, source code, customer data, or internal roadmaps, prefer a local checkout, local stdio server, or self-hosted JudgmentKit MCP endpoint instead of `https://judgmentkit.ai/mcp`.", + "", + ].join("\n"); +} + +function validAgentYaml() { + return [ + "interface:", + ' display_name: "JudgmentKit Hosted MCP"', + ' short_description: "UI workflow review and hard design-system gates"', + ' default_prompt: "Use $judgmentkit-hosted-mcp to review this UI brief for workflow fit, surface choice, disclosure boundaries, and the hard design-system acceptance gate."', + "dependencies:", + " tools:", + ' - type: "mcp"', + ' value: "judgmentkit"', + ' description: "Hosted JudgmentKit MCP server"', + ' transport: "streamable_http"', + ` url: "${DEFAULT_MCP_URL}"`, + "policy:", + " allow_implicit_invocation: false", + "", + ].join("\n"); +} + +async function assertVerifyRejects(options, expectedChecks) { + await assert.rejects( + () => verifyCodexPlugin(options), + (error) => { + assert.ok(error instanceof CodexPluginVerificationError); + const checks = new Set(error.failures.map((failure) => failure.check)); + for (const expectedCheck of expectedChecks) { + assert.ok(checks.has(expectedCheck), `expected failure check ${expectedCheck}, got ${[...checks].join(", ")}`); + } + return true; + }, + ); +} + +{ + const fixture = await createFixture(); + const result = await verifyCodexPlugin({ + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }); + + assert.equal(result.status, "passed"); + assert.equal(result.package_version, fixture.version); + assert.equal(result.plugin_version, fixture.version); + assert.equal(result.mcp_url, DEFAULT_MCP_URL); + assert.ok(result.files_checked.includes(".codex-plugin/plugin.json")); + assert.ok(result.files_checked.includes("skills/judgmentkit-hosted-mcp/agents/openai.yaml")); +} + +{ + const fixture = await createFixture({ pluginVersion: "1.2.4" }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["package_plugin_version_equality"], + ); +} + +{ + const fixture = await createFixture({ pluginVersion: `1.2.3${codexSuffix()}.20260704010203` }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["strict_base_semver", "no_codex_version_suffix", "forbidden_source_text"], + ); +} + +{ + const fixture = await createFixture({ pluginVersion: "1.2.3-beta.1" }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["strict_base_semver"], + ); +} + +{ + const fixture = await createFixture({ mcpUrl: "https://example.test/mcp" }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["mcp_url"], + ); +} + +{ + const fixture = await createFixture({ + agentYaml: validAgentYaml().replace("allow_implicit_invocation: false", "allow_implicit_invocation: true"), + }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["agent_yaml", "agent_policy"], + ); +} + +{ + const fixture = await createFixture({ + agentYaml: validAgentYaml().replace('transport: "streamable_http"', 'transport: "stdio"'), + }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["agent_yaml"], + ); +} + +{ + const fixture = await createFixture({ writeAgentYaml: false }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["required_file"], + ); +} + +{ + const forbiddenText = [ + `Do not keep ${"/Users/" + "mike"}/work paths.`, + `Do not keep ${".codex/" + "plugins/cache"}/personal paths.`, + "Avoid get_workflow_bundle.", + "Avoid mcp:stdio.", + "Avoid git clone.", + "Avoid npm install.", + `Avoid ${["TO", "DO"].join("")} markers.`, + ].join("\n"); + const fixture = await createFixture({ + extraFiles: { + "skills/judgmentkit-hosted-mcp/forbidden.md": forbiddenText, + }, + }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["forbidden_source_text"], + ); +} + +{ + const fixture = await createFixture({ skillText: "# JudgmentKit Hosted MCP\n" }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["privacy_wording", "design_system_gate_wording"], + ); +} + +{ + const fixture = await createFixture({ + skillText: "# JudgmentKit Hosted MCP\n", + extraFiles: { + "README.md": validSkillText(), + }, + }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["privacy_wording", "design_system_gate_wording"], + ); +} + +{ + const fixture = await createFixture({ + extraFiles: { + "skills/judgmentkit-hosted-mcp/dist/forbidden.md": `Avoid ${".codex/" + "plugins/cache"}/personal paths.`, + }, + }); + await assertVerifyRejects( + { + sourceDir: fixture.sourceDir, + packageJsonPath: fixture.packageJsonPath, + }, + ["forbidden_source_text"], + ); +} + +{ + const fixture = await createFixture({ + extraFiles: { + "private-notes.md": "This file should not be copied by the build whitelist.\n", + }, + }); + const outDir = path.join(fixture.tempDir, "dist", "codex-plugin"); + await fs.mkdir(outDir, { recursive: true }); + await fs.writeFile(path.join(outDir, "stale.md"), "stale file"); + const result = await buildCodexPlugin({ + sourceDir: fixture.sourceDir, + outDir, + packageJsonPath: fixture.packageJsonPath, + clean: false, + }); + + assert.equal(result.status, "built"); + assert.deepEqual(result.copied_entries, ["package.json", ".codex-plugin", ".mcp.json", "skills"]); + assert.equal((await readJson(path.join(outDir, ".codex-plugin", "plugin.json"))).version, fixture.version); + await assert.rejects( + () => fs.access(path.join(outDir, "private-notes.md")), + (error) => error.code === "ENOENT", + ); + await assert.rejects( + () => fs.access(path.join(outDir, "stale.md")), + (error) => error.code === "ENOENT", + ); +} + +{ + const fixture = await createFixture(); + const outDir = path.join(fixture.tempDir, "build", "codex-plugin"); + const targetDir = path.join(fixture.tempDir, "synced", "judgmentkit"); + const result = await syncCodexPlugin({ + sourceDir: fixture.sourceDir, + outDir, + targetDir, + packageJsonPath: fixture.packageJsonPath, + timestamp: new Date("2026-07-04T01:02:03Z"), + }); + const targetPluginJson = await readJson(path.join(targetDir, ".codex-plugin", "plugin.json")); + const sourcePluginJson = await readJson(path.join(fixture.sourceDir, ".codex-plugin", "plugin.json")); + + assert.equal(result.status, "synced"); + assert.equal(result.source_version, fixture.version); + assert.equal(result.synced_version, `1.2.3${codexSuffix()}.20260704010203`); + assert.equal(targetPluginJson.version, result.synced_version); + assert.equal(sourcePluginJson.version, fixture.version); +} + +{ + const fixture = await createFixture(); + const unsafeTargets = [ + root, + path.join(root, "packages", "codex-plugin"), + path.join(fixture.sourceDir), + path.join(fixture.sourceDir, "nested"), + path.dirname(fixture.sourceDir), + path.join(fixture.tempDir, "build", "codex-plugin"), + ]; + + for (const targetDir of unsafeTargets) { + await assert.rejects( + () => + syncCodexPlugin({ + sourceDir: fixture.sourceDir, + outDir: path.join(fixture.tempDir, "build", "codex-plugin"), + targetDir, + packageJsonPath: fixture.packageJsonPath, + dryRun: true, + }), + /Refusing to sync Codex plugin/, + ); + } +} + +{ + const fixture = await createFixture(); + const outDir = path.join(fixture.tempDir, "build", "codex-plugin"); + const targetDir = path.join(fixture.tempDir, "dry-run", "judgmentkit"); + const result = await syncCodexPlugin({ + sourceDir: fixture.sourceDir, + outDir, + targetDir, + packageJsonPath: fixture.packageJsonPath, + timestamp: new Date("2026-07-04T01:02:03Z"), + dryRun: true, + }); + + assert.equal(result.status, "dry_run"); + assert.equal(result.synced_version, `1.2.3${codexSuffix()}.20260704010203`); + await assert.rejects( + () => fs.access(targetDir), + (error) => error.code === "ENOENT", + ); +} + +{ + await fs.access(DEFAULT_SOURCE_DIR); + await verifyCodexPlugin({ + sourceDir: DEFAULT_SOURCE_DIR, + }); +} + +{ + const installerSource = await fs.readFile(path.join(root, "scripts", "install-mcp.mjs"), "utf8"); + + for (const forbiddenText of [ + "packages/codex-plugin", + "codex plugin add", + "plugin marketplace", + ".codex/plugins/cache", + "mcp:stdio", + "git clone", + "npm install", + ]) { + assert.equal( + installerSource.includes(forbiddenText), + false, + `install:mcp should stay hosted-MCP-only and not reference ${forbiddenText}`, + ); + } +} + +console.log("Codex plugin checks passed.");