From 6f4c5f5674cc268d0bdb1f5b6ce56f950503ccdd Mon Sep 17 00:00:00 2001
From: Ahmed Sayeed Wasif <ahmed.wasif@automattic.com>
Date: Thu, 16 Jul 2026 12:32:57 +0600
Subject: [PATCH 1/6] Add integration conformance checker library

Static checks that map the VIP integration conformance checklist to an
objective pass/fail/warn/n-a verdict per rule, plus human-readable and
JSON report rendering. Security review and the plugin/platform
config-schema match stay in the human-review layer, never an automated
pass.
---
 src/lib/validate-integration/report.ts   |  75 +++
 src/lib/validate-integration/validate.ts | 755 +++++++++++++++++++++++
 2 files changed, 830 insertions(+)
 create mode 100644 src/lib/validate-integration/report.ts
 create mode 100644 src/lib/validate-integration/validate.ts

diff --git a/src/lib/validate-integration/report.ts b/src/lib/validate-integration/report.ts
new file mode 100644
index 000000000..3d83fe8e6
--- /dev/null
+++ b/src/lib/validate-integration/report.ts
@@ -0,0 +1,75 @@
+/**
+ * Rendering for the integration conformance report — a human-readable summary
+ * for developers and a machine-readable JSON payload for CI.
+ */
+
+import chalk from 'chalk';
+
+import type { CheckResult, CheckStatus, ValidationReport } from './validate';
+
+const STATUS_LABEL: Record< CheckStatus, string > = {
+	pass: 'PASS',
+	fail: 'FAIL',
+	warn: 'WARN',
+	not_applicable: 'N/A ',
+};
+
+function colorForStatus( status: CheckStatus, text: string ): string {
+	switch ( status ) {
+		case 'pass':
+			return chalk.green( text );
+		case 'fail':
+			return chalk.red( text );
+		case 'warn':
+			return chalk.yellow( text );
+		default:
+			return chalk.gray( text );
+	}
+}
+
+function formatCheck( result: CheckResult ): string {
+	const label = colorForStatus( result.status, STATUS_LABEL[ result.status ] );
+	const details = ( result.details ?? [] ).map( detail => chalk.gray( `        - ${ detail }` ) );
+	return [
+		`  ${ label }  Rule ${ result.rule }: ${ result.title }`,
+		`        ${ result.message }`,
+		...details,
+	].join( '\n' );
+}
+
+export function formatHumanReport( report: ValidationReport ): string {
+	const counts = report.results.reduce< Record< CheckStatus, number > >(
+		( acc, result ) => {
+			acc[ result.status ] += 1;
+			return acc;
+		},
+		{ pass: 0, fail: 0, warn: 0, not_applicable: 0 }
+	);
+
+	const humanReviewLines = report.humanReview.flatMap( item => [
+		`  ${ chalk.cyan( '•' ) } ${ item.title }`,
+		chalk.gray( `    ${ item.reason }` ),
+	] );
+
+	const verdict = report.conformant
+		? chalk.green.bold( '✓ Conformant — no automated checks failed.' )
+		: chalk.red.bold( '✗ Not conformant — one or more automated checks failed.' );
+
+	return [
+		'',
+		chalk.bold( `Integration conformance check: ${ report.path }` ),
+		'',
+		...report.results.map( formatCheck ),
+		'',
+		chalk.bold( 'Human review required (not automated):' ),
+		...humanReviewLines,
+		'',
+		`Summary: ${ counts.pass } passed, ${ counts.fail } failed, ${ counts.warn } warnings, ${ counts.not_applicable } n/a.`,
+		verdict,
+		'',
+	].join( '\n' );
+}
+
+export function formatJsonReport( report: ValidationReport ): string {
+	return JSON.stringify( report, null, 2 );
+}
diff --git a/src/lib/validate-integration/validate.ts b/src/lib/validate-integration/validate.ts
new file mode 100644
index 000000000..749e51d62
--- /dev/null
+++ b/src/lib/validate-integration/validate.ts
@@ -0,0 +1,755 @@
+/**
+ * Conformance checker for WordPress VIP partner integrations.
+ *
+ * Encodes the VIP integration conformance checklist as automated, static checks
+ * a partner can run locally and in CI to get an objective
+ * conformant / not-conformant answer before submitting.
+ *
+ * The checks here are deliberately static (file and config inspection). Some
+ * rules can only be partially verified this way; those return `warn` and say
+ * so rather than pretending a clean pass. Two items — the plugin/platform
+ * config-schema match and the security review — are not automatable at all and
+ * are surfaced separately as human-review, never as an automated pass/fail.
+ */
+
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { extname, join } from 'node:path';
+
+export type CheckStatus = 'pass' | 'fail' | 'warn' | 'not_applicable';
+
+export interface CheckResult {
+	/** Stable machine identifier for the rule. */
+	id: string;
+	/** Checklist number (1-9) this rule maps to. */
+	rule: number;
+	title: string;
+	status: CheckStatus;
+	/** One-line explanation of the verdict. */
+	message: string;
+	/** Optional supporting evidence lines. */
+	details?: string[];
+}
+
+export interface HumanReviewItem {
+	title: string;
+	reason: string;
+}
+
+export interface ValidationReport {
+	path: string;
+	results: CheckResult[];
+	humanReview: HumanReviewItem[];
+	/** True when no check failed. Warnings do not break conformance. */
+	conformant: boolean;
+}
+
+interface ComposerJson {
+	type?: string;
+	autoload?: Record< string, unknown >;
+	scripts?: Record< string, unknown >;
+	require?: Record< string, string >;
+}
+
+interface Context {
+	root: string;
+	composer: ComposerJson | null;
+	/** True when composer.json is present, even if it failed to parse. */
+	composerExists: boolean;
+	/** npm scripts, so a `composer test` that shells to `npm test` can be resolved. */
+	packageScripts: Record< string, unknown > | null;
+	/** Concatenated markdown from README plus every file under docs/. */
+	docsText: string;
+	/** Concatenated PHP source, excluding vendor/ and node_modules/. */
+	phpSource: string;
+	/** Concatenated YAML from .github/workflows/. */
+	workflowsText: string;
+	/** The runtime config constant referenced by the integration, if any. */
+	configConstant: string | null;
+	/** Root-level plugin entry file (with a "Plugin Name:" header), if any. */
+	entryFile: string | null;
+}
+
+const SKIP_DIRS = new Set( [ 'vendor', 'node_modules', '.git', 'dist', 'coverage' ] );
+
+function readFileSafe( filePath: string ): string {
+	try {
+		return readFileSync( filePath, 'utf8' );
+	} catch {
+		return '';
+	}
+}
+
+function parseComposer( root: string ): ComposerJson | null {
+	const composerPath = join( root, 'composer.json' );
+	if ( ! existsSync( composerPath ) ) {
+		return null;
+	}
+	try {
+		return JSON.parse( readFileSafe( composerPath ) ) as ComposerJson;
+	} catch {
+		return null;
+	}
+}
+
+function parsePackageScripts( root: string ): Record< string, unknown > | null {
+	const packagePath = join( root, 'package.json' );
+	if ( ! existsSync( packagePath ) ) {
+		return null;
+	}
+	try {
+		const parsed = JSON.parse( readFileSafe( packagePath ) ) as {
+			scripts?: Record< string, unknown >;
+		};
+		return parsed.scripts ?? null;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Collect files with one of the given extensions, skipping dependency and build
+ * directories. Bounded to a shallow-ish walk so a stray large tree can't hang.
+ */
+function collectFiles( root: string, extensions: string[], maxDepth = 6 ): string[] {
+	const found: string[] = [];
+
+	const walk = ( dir: string, depth: number ): void => {
+		if ( depth > maxDepth ) {
+			return;
+		}
+		let entries: string[];
+		try {
+			entries = readdirSync( dir );
+		} catch {
+			return;
+		}
+		for ( const entry of entries ) {
+			const full = join( dir, entry );
+			let isDir = false;
+			try {
+				isDir = statSync( full ).isDirectory();
+			} catch {
+				continue;
+			}
+			if ( isDir ) {
+				if ( ! SKIP_DIRS.has( entry ) && ! entry.startsWith( '.' ) ) {
+					walk( full, depth + 1 );
+				}
+				continue;
+			}
+			if ( extensions.includes( extname( entry ).toLowerCase() ) ) {
+				found.push( full );
+			}
+		}
+	};
+
+	walk( root, 0 );
+	return found;
+}
+
+function collectDocsText( root: string ): string {
+	const parts: string[] = [];
+	const readme = join( root, 'README.md' );
+	if ( existsSync( readme ) ) {
+		parts.push( readFileSafe( readme ) );
+	}
+	const docsDir = join( root, 'docs' );
+	if ( existsSync( docsDir ) ) {
+		for ( const file of collectFiles( docsDir, [ '.md' ] ) ) {
+			parts.push( readFileSafe( file ) );
+		}
+	}
+	return parts.join( '\n\n' );
+}
+
+function collectWorkflowsText( root: string ): string {
+	const dir = join( root, '.github', 'workflows' );
+	if ( ! existsSync( dir ) ) {
+		return '';
+	}
+	return collectFiles( dir, [ '.yml', '.yaml' ], 2 ).map( readFileSafe ).join( '\n\n' );
+}
+
+function detectConfigConstant( phpSource: string ): string | null {
+	// Prefer the Starter Kit pattern: a Config class declaring
+	// `const CONSTANT_NAME = '<CONSTANT>'`. This catches integrations that follow
+	// the convention even if they don't use the VIP_*_CONFIG suffix.
+	const declared = /CONSTANT_NAME\s*=\s*'([A-Z_][A-Z0-9_]*)'/.exec( phpSource );
+	if ( declared ) {
+		return declared[ 1 ];
+	}
+	// Fall back to a VIP_<NAME>_CONFIG constant referenced anywhere.
+	const referenced = /\bVIP_[A-Z0-9_]+_CONFIG\b/.exec( phpSource );
+	return referenced ? referenced[ 0 ] : null;
+}
+
+function detectEntryFile( root: string ): string | null {
+	// The plugin entry file lives at the repo root and carries a plugin header.
+	let entries: string[];
+	try {
+		entries = readdirSync( root );
+	} catch {
+		return null;
+	}
+	for ( const entry of entries ) {
+		if ( extname( entry ).toLowerCase() !== '.php' ) {
+			continue;
+		}
+		const contents = readFileSafe( join( root, entry ) );
+		if ( /Plugin Name:/i.test( contents ) ) {
+			return entry;
+		}
+	}
+	return null;
+}
+
+function buildContext( root: string ): Context {
+	const composer = parseComposer( root );
+	const phpFiles = collectFiles( root, [ '.php' ] );
+	const phpSource = phpFiles.map( readFileSafe ).join( '\n\n' );
+
+	return {
+		root,
+		composer,
+		composerExists: existsSync( join( root, 'composer.json' ) ),
+		packageScripts: parsePackageScripts( root ),
+		docsText: collectDocsText( root ),
+		phpSource,
+		workflowsText: collectWorkflowsText( root ),
+		configConstant: detectConfigConstant( phpSource ),
+		entryFile: detectEntryFile( root ),
+	};
+}
+
+/**
+ * Expand a Composer script into the concrete commands it runs, following
+ * `@other-script` references. `@php ...` and other non-script `@` calls are
+ * kept as literal commands.
+ */
+function resolveComposerScript(
+	scripts: Record< string, unknown >,
+	name: string,
+	seen: Set< string > = new Set()
+): string[] {
+	if ( seen.has( name ) ) {
+		return [];
+	}
+	seen.add( name );
+
+	const raw = scripts[ name ];
+	if ( raw === undefined || raw === null ) {
+		return [];
+	}
+
+	// Composer script values are a command string or an array of them.
+	const rawEntries: unknown[] = Array.isArray( raw ) ? raw : [ raw ];
+	const entries = rawEntries.filter( ( item ): item is string => typeof item === 'string' );
+	const commands: string[] = [];
+
+	for ( const entry of entries ) {
+		const trimmed = entry.trim();
+		if ( trimmed.startsWith( '@' ) ) {
+			const refName = trimmed.slice( 1 ).split( /\s+/ )[ 0 ];
+			if ( Object.hasOwn( scripts, refName ) ) {
+				commands.push( ...resolveComposerScript( scripts, refName, seen ) );
+				continue;
+			}
+		}
+		commands.push( trimmed );
+	}
+
+	return commands;
+}
+
+/**
+ * Expand `npm test` / `npm run <script>` commands into the package.json script
+ * body they run, so a `composer test` that delegates to npm is judged by what
+ * npm actually runs (e.g. `playwright test`), not by the word "test".
+ */
+function expandNpmDelegations(
+	commands: string[],
+	packageScripts: Record< string, unknown > | null
+): string[] {
+	if ( ! packageScripts ) {
+		return commands;
+	}
+	const expanded: string[] = [];
+	for ( const cmd of commands ) {
+		expanded.push( cmd );
+		const match = /\bnpm (?:run )?([a-z0-9:_-]+)/i.exec( cmd );
+		if ( match ) {
+			const body = packageScripts[ match[ 1 ] ];
+			if ( typeof body === 'string' ) {
+				expanded.push( body );
+			}
+		}
+	}
+	return expanded;
+}
+
+/** Drop no-op commands (echo/comment/true) that do not actually run anything. */
+function realCommands( commands: string[] ): string[] {
+	return commands.filter( cmd => ! /^(echo|:|true|#)\b/.test( cmd.trim() ) );
+}
+
+/** Fenced code blocks (```...```) that mention the given needle. */
+function codeBlocksMentioning( markdown: string, needle: string ): string[] {
+	const blocks = markdown.match( /```[\s\S]*?```/g ) ?? [];
+	return blocks.filter( block => block.includes( needle ) );
+}
+
+// Precondition stated out loud when the config checks are skipped: detection
+// keys off a `Config::CONSTANT_NAME` declaration or a VIP_*_CONFIG constant. An
+// integration that uses runtime config under a different pattern is not exempt —
+// it must follow the convention or be flagged for human review.
+const CONFIG_DETECTION_NOTE =
+	'No runtime config constant detected (looked for a `Config::CONSTANT_NAME` declaration or a `VIP_*_CONFIG` constant). If this integration uses runtime config under another name, adopt the convention or flag it for human review — these config checks were skipped, not passed.';
+
+// --- Individual checks -----------------------------------------------------
+
+function checkLoadsThroughStarterKit( ctx: Context ): CheckResult {
+	const base = {
+		id: 'loads-through-starter-kit',
+		rule: 1,
+		title: 'Loads through the Starter Kit workflow',
+	};
+	const details: string[] = [];
+
+	if ( ! ctx.composer ) {
+		return {
+			...base,
+			status: 'fail',
+			message: ctx.composerExists
+				? 'composer.json is present but is not valid JSON.'
+				: 'No composer.json found — VIP loads integrations as Composer wordpress-plugin packages.',
+		};
+	}
+	if ( ! ctx.entryFile ) {
+		return {
+			...base,
+			status: 'fail',
+			message: 'No root-level plugin entry file with a "Plugin Name:" header was found.',
+		};
+	}
+	details.push( `Entry file: ${ ctx.entryFile }` );
+
+	// A wrong Composer "type" or a missing autoload both stop the integration
+	// from loading through the Starter Kit workflow. These are deterministic, so
+	// they fail the gate rather than merely warn.
+	const problems: string[] = [];
+	if ( ctx.composer.type !== 'wordpress-plugin' ) {
+		problems.push(
+			`composer.json "type" is "${
+				ctx.composer.type ?? 'unset'
+			}"; it must be "wordpress-plugin" so VIP loads it as a plugin`
+		);
+	}
+	if ( ! ctx.composer.autoload ) {
+		problems.push(
+			'composer.json has no "autoload" section, so the integration\'s classes will not load'
+		);
+	}
+
+	if ( problems.length > 0 ) {
+		return { ...base, status: 'fail', message: `${ problems.join( '; ' ) }.`, details };
+	}
+
+	details.push( 'composer.json type is "wordpress-plugin" with an autoload section.' );
+	return {
+		...base,
+		status: 'pass',
+		message: 'Plugin entry file and Composer wordpress-plugin package are present.',
+		details,
+	};
+}
+
+function checkComposerTest( ctx: Context ): CheckResult {
+	const base = {
+		id: 'composer-test',
+		rule: 2,
+		title: '`composer test` runs PHPUnit and e2e tests',
+	};
+	const scripts = ctx.composer?.scripts;
+	if ( ! scripts || ! Object.hasOwn( scripts, 'test' ) ) {
+		return { ...base, status: 'fail', message: 'composer.json has no "test" script.' };
+	}
+
+	// Resolve @script references, then follow any npm delegation into package.json,
+	// then drop no-op (echo/comment) commands so the check reflects what runs.
+	const resolved = expandNpmDelegations(
+		resolveComposerScript( scripts, 'test' ),
+		ctx.packageScripts
+	);
+	const combined = realCommands( resolved ).join( ' • ' );
+	const hasUnit = /\bphpunit\b/i.test( combined );
+	const hasE2e = /\b(playwright|cypress|codeception|puppeteer)\b/i.test( combined );
+
+	if ( hasUnit && hasE2e ) {
+		return {
+			...base,
+			status: 'pass',
+			message: 'composer test declares a PHPUnit run and an e2e runner (Playwright/Cypress).',
+			details: [
+				`Resolved commands: ${ combined }`,
+				'Static check: it verifies the test commands are wired, not that the tests pass.',
+			],
+		};
+	}
+
+	const missing: string[] = [];
+	if ( ! hasUnit ) {
+		missing.push( 'PHPUnit (no `phpunit` invocation)' );
+	}
+	if ( ! hasE2e ) {
+		missing.push( 'an e2e runner (no Playwright/Cypress invocation)' );
+	}
+	return {
+		...base,
+		status: 'fail',
+		message: `composer test does not wire up: ${ missing.join( ' and ' ) }.`,
+		details: [ `Resolved commands: ${ combined || '(none)' }` ],
+	};
+}
+
+function checkValidateIntegrationScript( ctx: Context ): CheckResult {
+	const base = {
+		id: 'validate-integration-script',
+		rule: 3,
+		title: '`composer run validate-integration` exists',
+	};
+	const scripts = ctx.composer?.scripts;
+	if ( scripts && Object.hasOwn( scripts, 'validate-integration' ) ) {
+		return {
+			...base,
+			status: 'pass',
+			message: 'composer.json defines a "validate-integration" script.',
+		};
+	}
+	return {
+		...base,
+		status: 'fail',
+		message: 'composer.json has no "validate-integration" script.',
+	};
+}
+
+function checkConfigConstantDocumented( ctx: Context ): CheckResult {
+	const base = {
+		id: 'config-constant-documented',
+		rule: 4,
+		title: 'Config constant is documented and referenced in code',
+	};
+	if ( ! ctx.configConstant ) {
+		return { ...base, status: 'not_applicable', message: CONFIG_DETECTION_NOTE };
+	}
+	if ( ctx.docsText.includes( ctx.configConstant ) ) {
+		return {
+			...base,
+			status: 'pass',
+			message: `Config constant ${ ctx.configConstant } is referenced in code and documented.`,
+		};
+	}
+	return {
+		...base,
+		status: 'fail',
+		message: `Config constant ${ ctx.configConstant } is used in code but not documented in README/docs.`,
+	};
+}
+
+function checkGracefulConfigHandling( ctx: Context ): CheckResult {
+	const base = {
+		id: 'graceful-config-handling',
+		rule: 5,
+		title: 'Missing/invalid config is handled without fataling',
+	};
+	if ( ! ctx.configConstant ) {
+		return { ...base, status: 'not_applicable', message: CONFIG_DETECTION_NOTE };
+	}
+
+	const guards = [
+		{ re: /is_ready\s*\(/, label: 'is_ready()' },
+		{ re: /missing_fields\s*\(/, label: 'missing_fields()' },
+		{ re: /is_available\s*\(/, label: 'is_available()' },
+		{
+			// configConstant is matched from source as [A-Z0-9_]+ only, so it is
+			// safe to interpolate into a pattern here.
+			// eslint-disable-next-line security/detect-non-literal-regexp
+			re: new RegExp( String.raw`defined\s*\(\s*(self::CONSTANT_NAME|'${ ctx.configConstant }')` ),
+			label: 'defined() guard',
+		},
+		{ re: /is_array\s*\(/, label: 'is_array() guard' },
+	];
+	const present = guards
+		.filter( guard => guard.re.test( ctx.phpSource ) )
+		.map( guard => guard.label );
+
+	if ( present.length > 0 ) {
+		return {
+			...base,
+			status: 'pass',
+			message: 'Config access is guarded against a missing or invalid constant.',
+			details: [
+				`Guards found: ${ present.join( ', ' ) }`,
+				"Static signal only — behavioral proof comes from the integration's own tests (rule 2).",
+			],
+		};
+	}
+
+	return {
+		...base,
+		status: 'warn',
+		message:
+			'Could not find a static guard (is_ready()/defined()/is_array()) around config access.',
+		details: [
+			'Verify via tests that missing or invalid config degrades gracefully instead of fataling.',
+		],
+	};
+}
+
+function checkConfigExamplesInDocs( ctx: Context ): CheckResult {
+	const base = {
+		id: 'config-examples-in-docs',
+		rule: 6,
+		title: 'Docs include valid and incomplete config examples',
+	};
+	if ( ! ctx.configConstant ) {
+		return { ...base, status: 'not_applicable', message: CONFIG_DETECTION_NOTE };
+	}
+
+	const blocks = codeBlocksMentioning( ctx.docsText, ctx.configConstant );
+	if ( blocks.length === 0 ) {
+		return {
+			...base,
+			status: 'fail',
+			message: `No documented config example references ${ ctx.configConstant }.`,
+		};
+	}
+	if ( blocks.length < 2 ) {
+		return {
+			...base,
+			status: 'fail',
+			message:
+				'Only one config example is documented; both a valid and an incomplete example are required.',
+		};
+	}
+
+	// Distinguish a "valid" from an "incomplete" example: either the docs call
+	// it out in prose, or one example carries strictly fewer keys than another.
+	const keyCounts = blocks.map(
+		block => ( block.match( /['"][a-z0-9_]+['"]\s*=>/gi ) ?? [] ).length
+	);
+	const hasSmallerExample = Math.min( ...keyCounts ) < Math.max( ...keyCounts );
+	const mentionsIncomplete = /incomplete|missing (a )?required|setup in progress/i.test(
+		ctx.docsText
+	);
+
+	if ( hasSmallerExample || mentionsIncomplete ) {
+		return {
+			...base,
+			status: 'pass',
+			message: 'Docs include both a valid and an incomplete config example.',
+		};
+	}
+	return {
+		...base,
+		status: 'warn',
+		message:
+			'Multiple config examples are documented, but none is clearly an incomplete/missing-field example.',
+	};
+}
+
+function checkCompatibilityMatrix( ctx: Context ): CheckResult {
+	const base = {
+		id: 'compatibility-matrix',
+		rule: 7,
+		title: 'Compatibility evidence covers WP 6.9/7.0 and PHP 8.2-8.5',
+	};
+	// Evidence must be a real CI matrix (.github/workflows) or an explicit,
+	// approved exception note — not a version number that happens to appear in a
+	// changelog or prose. Scanning arbitrary docs lets coincidental substrings
+	// pass, so only workflows and an explicit exception note count here.
+	if (
+		/compatibility exception|approved exception/i.test(
+			`${ ctx.workflowsText }\n${ ctx.docsText }`
+		)
+	) {
+		return {
+			...base,
+			status: 'pass',
+			message: 'An approved compatibility exception note is documented.',
+		};
+	}
+
+	if ( ctx.workflowsText.trim() === '' ) {
+		return {
+			...base,
+			status: 'fail',
+			message:
+				'No CI workflows found to evidence the compatibility matrix (WP 6.9 + 7.0, PHP 8.2-8.5).',
+			details: [
+				'Add a CI matrix under .github/workflows, or document an approved compatibility exception note.',
+			],
+		};
+	}
+
+	const missing: string[] = [];
+	if ( ! /\b6\.9\b/.test( ctx.workflowsText ) ) {
+		missing.push( 'WordPress 6.9' );
+	}
+	// WP 7.0 in CI is often expressed as "latest"; accept either, but only from
+	// the workflow matrix.
+	if ( ! /\b7\.0\b/.test( ctx.workflowsText ) && ! /\blatest\b/.test( ctx.workflowsText ) ) {
+		missing.push( 'WordPress 7.0' );
+	}
+	for ( const php of [ '8.2', '8.3', '8.4', '8.5' ] ) {
+		if ( ! ctx.workflowsText.includes( php ) ) {
+			missing.push( `PHP ${ php }` );
+		}
+	}
+
+	if ( missing.length === 0 ) {
+		return {
+			...base,
+			status: 'pass',
+			message: 'CI matrix covers WP 6.9 + 7.0 and PHP 8.2-8.5.',
+		};
+	}
+	return {
+		...base,
+		status: 'fail',
+		message: `CI compatibility matrix is missing: ${ missing.join( ', ' ) }.`,
+		details: [
+			'Cover the matrix in CI (.github/workflows) or add an approved compatibility exception note.',
+		],
+	};
+}
+
+function checkBuildTestCommandsDocumented( ctx: Context ): CheckResult {
+	const base = {
+		id: 'build-test-commands-documented',
+		rule: 8,
+		title: 'Build and test commands are documented',
+	};
+	const hasTest = /composer (run )?test|phpunit|playwright test/i.test( ctx.docsText );
+	const hasBuild = /npm run build|npm ci|composer install|npm install/i.test( ctx.docsText );
+
+	if ( hasTest && hasBuild ) {
+		return {
+			...base,
+			status: 'pass',
+			message: 'Docs document both build/install and test commands.',
+		};
+	}
+
+	const missing: string[] = [];
+	if ( ! hasBuild ) {
+		missing.push( 'build/install commands' );
+	}
+	if ( ! hasTest ) {
+		missing.push( 'test commands' );
+	}
+	return { ...base, status: 'fail', message: `Docs are missing: ${ missing.join( ' and ' ) }.` };
+}
+
+function checkTelemetryTracksOnly( ctx: Context ): CheckResult {
+	const base = {
+		id: 'telemetry-tracks-only',
+		rule: 9,
+		title: 'Telemetry uses the Starter Kit pattern (Tracks only, no secrets)',
+	};
+	const usesTelemetry = /Telemetry|record_event\s*\(/.test( ctx.phpSource );
+	if ( ! usesTelemetry ) {
+		return {
+			...base,
+			status: 'not_applicable',
+			message: 'The integration does not record telemetry.',
+		};
+	}
+
+	const usesVipTelemetryApi = /Automattic\\VIP\\Telemetry/.test( ctx.phpSource );
+	const guarded = /class_exists\s*\(/.test( ctx.phpSource );
+	const usesStats = /Automattic\\VIP\\Stats|record_pixel|->pixel\b/.test( ctx.phpSource );
+
+	if ( usesStats ) {
+		return {
+			...base,
+			status: 'fail',
+			message: 'Telemetry uses Stats/Pixel; VIP integrations must use Tracks events only.',
+		};
+	}
+
+	// Best-effort scan for obvious secret/PII keys in event properties. This is
+	// advisory: real secret review stays in the human-review layer.
+	const suspicious = (
+		ctx.phpSource.match(
+			/record_event\s*\([\s\S]{0,400}?['"](password|secret|api_token|token|credential|email)['"]/gi
+		) ?? []
+	).length;
+
+	if ( ! usesVipTelemetryApi || ! guarded ) {
+		return {
+			...base,
+			status: 'warn',
+			message:
+				'Telemetry is recorded, but the VIP Telemetry (Tracks) helper pattern with a class_exists guard was not detected.',
+		};
+	}
+	if ( suspicious > 0 ) {
+		return {
+			...base,
+			status: 'warn',
+			message:
+				'Telemetry uses the Tracks helper, but event properties may include secret/PII keys — review before submitting.',
+		};
+	}
+
+	return {
+		...base,
+		status: 'pass',
+		message: 'Telemetry uses the guarded VIP Tracks helper with no obvious secrets in properties.',
+	};
+}
+
+const HUMAN_REVIEW: HumanReviewItem[] = [
+	{
+		title: 'Plugin - platform config-schema match',
+		reason:
+			"Whether the plugin's expected config matches the platform schema is not fully deterministic and is confirmed in human review, not by this checker.",
+	},
+	{
+		title: 'Security review',
+		reason:
+			'Security posture (input handling, secret storage, capability checks) is assessed in human review, not by this checker.',
+	},
+];
+
+/**
+ * Run every conformance check against an integration directory and return a
+ * structured report. `conformant` is false when any check has status `fail`.
+ */
+export function validateIntegration( root: string ): ValidationReport {
+	const ctx = buildContext( root );
+
+	const results: CheckResult[] = [
+		checkLoadsThroughStarterKit( ctx ),
+		checkComposerTest( ctx ),
+		checkValidateIntegrationScript( ctx ),
+		checkConfigConstantDocumented( ctx ),
+		checkGracefulConfigHandling( ctx ),
+		checkConfigExamplesInDocs( ctx ),
+		checkCompatibilityMatrix( ctx ),
+		checkBuildTestCommandsDocumented( ctx ),
+		checkTelemetryTracksOnly( ctx ),
+	];
+
+	return {
+		path: root,
+		results,
+		humanReview: HUMAN_REVIEW,
+		conformant: ! results.some( result => result.status === 'fail' ),
+	};
+}
+
+/** Whether the given path looks like an integration we can check at all. */
+export function looksLikeIntegration( root: string ): boolean {
+	return existsSync( join( root, 'composer.json' ) ) || detectEntryFile( root ) !== null;
+}

From b9d70ffcb4eaf2e9dbdfc661e8b4c811752d249a Mon Sep 17 00:00:00 2001
From: Ahmed Sayeed Wasif <ahmed.wasif@automattic.com>
Date: Thu, 16 Jul 2026 12:32:57 +0600
Subject: [PATCH 2/6] Add `vip integration validate` command

Register the no-auth integration command and its validate subcommand,
wire the bin entries and internal-bin loader, and gate CI with a
non-zero exit when an integration is not conformant.
---
 package.json                        |  2 +
 src/bin/vip-integration-validate.ts | 82 +++++++++++++++++++++++++++++
 src/bin/vip-integration.js          | 10 ++++
 src/bin/vip.js                      |  5 ++
 src/lib/cli/internal-bin-loader.js  |  2 +
 5 files changed, 101 insertions(+)
 create mode 100644 src/bin/vip-integration-validate.ts
 create mode 100644 src/bin/vip-integration.js

diff --git a/package.json b/package.json
index 96c9207b7..5c903a7aa 100644
--- a/package.json
+++ b/package.json
@@ -64,6 +64,8 @@
     "vip-import-sql-status": "dist/bin/vip-import-sql-status.js",
     "vip-import-validate-files": "dist/bin/vip-import-validate-files.js",
     "vip-import-validate-sql": "dist/bin/vip-import-validate-sql.js",
+    "vip-integration": "dist/bin/vip-integration.js",
+    "vip-integration-validate": "dist/bin/vip-integration-validate.js",
     "vip-logs": "dist/bin/vip-logs.js",
     "vip-search-replace": "dist/bin/vip-search-replace.js",
     "vip-slowlogs": "dist/bin/vip-slowlogs.js",
diff --git a/src/bin/vip-integration-validate.ts b/src/bin/vip-integration-validate.ts
new file mode 100644
index 000000000..2c9612bea
--- /dev/null
+++ b/src/bin/vip-integration-validate.ts
@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+
+/**
+ * External dependencies
+ */
+import chalk from 'chalk';
+import { resolve } from 'node:path';
+
+/**
+ * Internal dependencies
+ */
+import command from '../lib/cli/command';
+import { trackEvent } from '../lib/tracker';
+import { formatHumanReport, formatJsonReport } from '../lib/validate-integration/report';
+import { looksLikeIntegration, validateIntegration } from '../lib/validate-integration/validate';
+
+const usage = 'vip integration validate';
+
+const examples = [
+	{
+		usage: 'vip integration validate',
+		description: 'Check the integration in the current directory for conformance.',
+	},
+	{
+		usage: 'vip integration validate ./my-integration',
+		description: 'Check the integration in the given directory.',
+	},
+	{
+		usage: 'vip integration validate --format json',
+		description: 'Emit a machine-readable JSON report (for CI).',
+	},
+];
+
+export async function integrationValidateCommand(
+	arg: string[] = [],
+	opts: Record< string, unknown > = {}
+): Promise< void > {
+	const root = resolve( arg[ 0 ] ?? process.cwd() );
+
+	const format = typeof opts.format === 'string' ? opts.format : 'human';
+	if ( format !== 'human' && format !== 'json' ) {
+		console.error( chalk.red( `Unknown --format "${ format }". Use "human" or "json".` ) );
+		process.exitCode = 1;
+		return;
+	}
+	const asJson = format === 'json';
+
+	await trackEvent( 'integration_validate_command_execute' );
+
+	if ( ! looksLikeIntegration( root ) ) {
+		const message = `No integration found at ${ root } (expected a composer.json or a plugin entry file).`;
+		if ( asJson ) {
+			console.log( JSON.stringify( { path: root, error: message }, null, 2 ) );
+		} else {
+			console.error( chalk.red( message ) );
+		}
+		await trackEvent( 'integration_validate_command_error', { reason: 'not_an_integration' } );
+		process.exitCode = 1;
+		return;
+	}
+
+	const report = validateIntegration( root );
+
+	console.log( asJson ? formatJsonReport( report ) : formatHumanReport( report ) );
+
+	await trackEvent( 'integration_validate_command_success', {
+		conformant: report.conformant,
+		failed: report.results.filter( result => result.status === 'fail' ).length,
+	} );
+
+	if ( ! report.conformant ) {
+		process.exitCode = 1;
+	}
+}
+
+void command( {
+	requiredArgs: 0,
+	usage,
+} )
+	.option( 'format', 'Output format: "human" (default) or "json".', 'human' )
+	.examples( examples )
+	.argv( process.argv, integrationValidateCommand );
diff --git a/src/bin/vip-integration.js b/src/bin/vip-integration.js
new file mode 100644
index 000000000..ec8c6599c
--- /dev/null
+++ b/src/bin/vip-integration.js
@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+
+import command from '../lib/cli/command';
+
+command( {
+	requiredArgs: 0,
+	usage: 'vip integration',
+} )
+	.command( 'validate', 'Check a VIP integration for conformance before submitting it.' )
+	.argv( process.argv );
diff --git a/src/bin/vip.js b/src/bin/vip.js
index 713f7eaa8..19127bef1 100755
--- a/src/bin/vip.js
+++ b/src/bin/vip.js
@@ -76,6 +76,7 @@ const runCmd = async function () {
 		.command( 'db', "Access an environment's database." )
 		.command( 'defensive-mode', 'Manage VIP defensive mode for an environment.' )
 		.command( 'sync', 'Sync the database from production to a non-production environment.' )
+		.command( 'integration', 'Check a VIP integration for conformance before submitting it.' )
 		.command( 'whoami', 'Retrieve details about the current authenticated VIP-CLI user.' )
 		.command( 'wp', 'Execute a WP-CLI command against an environment.' );
 
@@ -202,6 +203,9 @@ const rootCmd = async function () {
 	const isDevEnvCommandWithoutEnv =
 		doesArgvHaveAtLeastOneParam( process.argv, [ 'dev-env' ] ) &&
 		! containsAppEnvArgument( process.argv );
+	// `integration` inspects local files only and never calls the API, so it
+	// does not require authentication (same as `dev-env`).
+	const isIntegrationCommand = doesArgvHaveAtLeastOneParam( process.argv, [ 'integration' ] );
 	const isCustomDeployCmdWithKey =
 		doesArgvHaveAtLeastOneParam( process.argv, [ 'deploy' ] ) && Boolean( customDeployToken );
 
@@ -213,6 +217,7 @@ const rootCmd = async function () {
 			isHelpCommand ||
 			isVersionCommand ||
 			isDevEnvCommandWithoutEnv ||
+			isIntegrationCommand ||
 			token?.valid() ||
 			isCustomDeployCmdWithKey )
 	) {
diff --git a/src/lib/cli/internal-bin-loader.js b/src/lib/cli/internal-bin-loader.js
index 6acda2269..0a13cffca 100644
--- a/src/lib/cli/internal-bin-loader.js
+++ b/src/lib/cli/internal-bin-loader.js
@@ -59,6 +59,8 @@ const internalBinLoaders = {
 	'vip-import-sql-status': () => import( '../../bin/vip-import-sql-status' ),
 	'vip-import-validate-files': () => import( '../../bin/vip-import-validate-files' ),
 	'vip-import-validate-sql': () => import( '../../bin/vip-import-validate-sql' ),
+	'vip-integration': () => import( '../../bin/vip-integration' ),
+	'vip-integration-validate': () => import( '../../bin/vip-integration-validate' ),
 	'vip-logout': () => import( '../../bin/vip-logout' ),
 	'vip-logs': () => import( '../../bin/vip-logs' ),
 	'vip-search-replace': () => import( '../../bin/vip-search-replace' ),

From d994000eb343cf9cf16dcdab1995293519214d43 Mon Sep 17 00:00:00 2001
From: Ahmed Sayeed Wasif <ahmed.wasif@automattic.com>
Date: Thu, 16 Jul 2026 12:32:57 +0600
Subject: [PATCH 3/6] Test the integration conformance checker

Cover conformant and non-conformant integrations, the load/test/config
edge cases, and the report rendering.
---
 .../lib/validate-integration-report.test.ts   |  51 ++++
 __tests__/lib/validate-integration.test.ts    | 251 ++++++++++++++++++
 2 files changed, 302 insertions(+)
 create mode 100644 __tests__/lib/validate-integration-report.test.ts
 create mode 100644 __tests__/lib/validate-integration.test.ts

diff --git a/__tests__/lib/validate-integration-report.test.ts b/__tests__/lib/validate-integration-report.test.ts
new file mode 100644
index 000000000..1237a3a43
--- /dev/null
+++ b/__tests__/lib/validate-integration-report.test.ts
@@ -0,0 +1,51 @@
+import { describe, expect, it } from '@jest/globals';
+
+import { formatHumanReport, formatJsonReport } from '../../src/lib/validate-integration/report';
+
+import type { ValidationReport } from '../../src/lib/validate-integration/validate';
+
+function sampleReport( conformant: boolean ): ValidationReport {
+	return {
+		path: '/tmp/acme',
+		results: [
+			{ id: 'a', rule: 1, title: 'First rule', status: 'pass', message: 'looks good' },
+			{
+				id: 'b',
+				rule: 2,
+				title: 'Second rule',
+				status: conformant ? 'warn' : 'fail',
+				message: conformant ? 'heads up' : 'broken',
+				details: [ 'evidence line' ],
+			},
+			{ id: 'c', rule: 3, title: 'Third rule', status: 'not_applicable', message: 'skipped' },
+		],
+		humanReview: [ { title: 'Security review', reason: 'human only' } ],
+		conformant,
+	};
+}
+
+describe( 'validate-integration report', () => {
+	it( 'renders a human report with per-rule verdicts and the human-review section', () => {
+		const text = formatHumanReport( sampleReport( true ) );
+		expect( text ).toContain( 'Rule 1: First rule' );
+		expect( text ).toContain( 'evidence line' );
+		expect( text ).toContain( 'Human review required' );
+		expect( text ).toContain( 'Security review' );
+		expect( text ).toContain( 'Conformant' );
+		expect( text ).toContain( '1 passed' );
+	} );
+
+	it( 'marks a non-conformant report clearly', () => {
+		const text = formatHumanReport( sampleReport( false ) );
+		expect( text ).toContain( 'Not conformant' );
+		expect( text ).toContain( '1 failed' );
+	} );
+
+	it( 'emits valid, round-trippable JSON', () => {
+		const report = sampleReport( false );
+		const parsed = JSON.parse( formatJsonReport( report ) ) as ValidationReport;
+		expect( parsed.conformant ).toBe( false );
+		expect( parsed.results ).toHaveLength( 3 );
+		expect( parsed.humanReview[ 0 ].title ).toBe( 'Security review' );
+	} );
+} );
diff --git a/__tests__/lib/validate-integration.test.ts b/__tests__/lib/validate-integration.test.ts
new file mode 100644
index 000000000..0aefb1b54
--- /dev/null
+++ b/__tests__/lib/validate-integration.test.ts
@@ -0,0 +1,251 @@
+import { afterAll, beforeAll, describe, expect, it } from '@jest/globals';
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+import {
+	looksLikeIntegration,
+	validateIntegration,
+} from '../../src/lib/validate-integration/validate';
+
+import type { CheckStatus } from '../../src/lib/validate-integration/validate';
+
+/** Write a minimal but fully conformant integration into `root`. */
+function scaffoldConformant( root: string ): void {
+	mkdirSync( join( root, 'docs' ), { recursive: true } );
+	mkdirSync( join( root, 'inc' ), { recursive: true } );
+	mkdirSync( join( root, '.github', 'workflows' ), { recursive: true } );
+
+	writeFileSync(
+		join( root, 'composer.json' ),
+		JSON.stringify( {
+			name: 'acme/widget',
+			type: 'wordpress-plugin',
+			autoload: { classmap: [ 'inc/' ] },
+			scripts: {
+				test: [ '@test:unit', '@test:e2e' ],
+				'test:unit': 'phpunit',
+				'test:e2e': 'npm test',
+				'validate-integration': '@php bin/validate-integration.php',
+			},
+		} )
+	);
+
+	writeFileSync(
+		join( root, 'package.json' ),
+		JSON.stringify( { name: 'e2e', scripts: { test: 'playwright test', build: 'echo none' } } )
+	);
+
+	writeFileSync(
+		join( root, 'acme-widget.php' ),
+		`<?php\n/**\n * Plugin Name: Acme Widget\n */\nrequire_once __DIR__ . '/vendor/autoload.php';\n`
+	);
+
+	writeFileSync(
+		join( root, 'inc', 'class-config.php' ),
+		`<?php\nfinal class Config {\n\tpublic const CONSTANT_NAME = 'VIP_ACME_WIDGET_CONFIG';\n\tpublic function __construct( $raw ) { if ( is_array( $raw ) ) {} }\n\tpublic function is_ready(): bool { return true; }\n\tpublic function missing_fields(): array { return []; }\n\tpublic function is_available(): bool { return defined( self::CONSTANT_NAME ); }\n}\n`
+	);
+
+	writeFileSync(
+		join( root, 'inc', 'class-telemetry.php' ),
+		`<?php\nfinal class Telemetry {\n\tprivate function __construct() {\n\t\tif ( class_exists( \\Automattic\\VIP\\Telemetry\\Telemetry::class ) ) {}\n\t}\n\tpublic function record_event( string $name, array $props = [] ): void {}\n}\n`
+	);
+
+	writeFileSync(
+		join( root, 'docs', 'integration.md' ),
+		[
+			'# Integration',
+			'',
+			'## Build and test',
+			'Run `composer install` and `npm ci`, then `composer test`. `npm run build` ships no assets.',
+			'',
+			'## Runtime config',
+			'Config constant: `VIP_ACME_WIDGET_CONFIG`.',
+			'',
+			'Valid config:',
+			'```php',
+			"define( 'VIP_ACME_WIDGET_CONFIG', [ 'api_base_url' => 'x', 'api_token' => 'y' ] );",
+			'```',
+			'',
+			'Incomplete config (a required value is missing):',
+			'```php',
+			"define( 'VIP_ACME_WIDGET_CONFIG', [ 'api_base_url' => 'x' ] );",
+			'```',
+		].join( '\n' )
+	);
+
+	writeFileSync(
+		join( root, '.github', 'workflows', 'unit-tests.yml' ),
+		[
+			'jobs:',
+			'  test:',
+			'    strategy:',
+			'      matrix:',
+			'        config:',
+			"          - { wp: 6.9.x, php: '8.2' }",
+			"          - { wp: latest, php: '8.3' }",
+			"          - { wp: latest, php: '8.4' }",
+			"          - { wp: 7.0, php: '8.5' }",
+		].join( '\n' )
+	);
+}
+
+function statusById( root: string ): Record< string, CheckStatus > {
+	const report = validateIntegration( root );
+	return Object.fromEntries( report.results.map( result => [ result.id, result.status ] ) );
+}
+
+describe( 'validateIntegration', () => {
+	let dir: string;
+
+	beforeAll( () => {
+		dir = mkdtempSync( join( tmpdir(), 'vip-validate-' ) );
+	} );
+
+	afterAll( () => {
+		rmSync( dir, { recursive: true, force: true } );
+	} );
+
+	it( 'passes every rule for a conformant integration', () => {
+		const root = join( dir, 'conformant' );
+		mkdirSync( root, { recursive: true } );
+		scaffoldConformant( root );
+
+		const report = validateIntegration( root );
+
+		expect( report.conformant ).toBe( true );
+		expect( report.results.every( result => result.status !== 'fail' ) ).toBe( true );
+		expect( report.humanReview ).toHaveLength( 2 );
+	} );
+
+	it( 'fails and reports which rules broke for a non-conformant integration', () => {
+		const root = join( dir, 'broken' );
+		mkdirSync( root, { recursive: true } );
+		writeFileSync(
+			join( root, 'composer.json' ),
+			JSON.stringify( { name: 'acme/broken', type: 'library' } )
+		);
+
+		const report = validateIntegration( root );
+		const status = statusById( root );
+
+		expect( report.conformant ).toBe( false );
+		expect( status[ 'loads-through-starter-kit' ] ).toBe( 'fail' );
+		expect( status[ 'composer-test' ] ).toBe( 'fail' );
+		expect( status[ 'validate-integration-script' ] ).toBe( 'fail' );
+		expect( status[ 'compatibility-matrix' ] ).toBe( 'fail' );
+		// No config constant and no telemetry -> not applicable, not a failure.
+		expect( status[ 'config-constant-documented' ] ).toBe( 'not_applicable' );
+		expect( status[ 'telemetry-tracks-only' ] ).toBe( 'not_applicable' );
+	} );
+
+	it( 'fails rule 2 when composer test omits the e2e suite', () => {
+		const root = join( dir, 'unit-only' );
+		mkdirSync( root, { recursive: true } );
+		scaffoldConformant( root );
+		writeFileSync(
+			join( root, 'composer.json' ),
+			JSON.stringify( {
+				type: 'wordpress-plugin',
+				autoload: { classmap: [ 'inc/' ] },
+				scripts: { test: 'phpunit', 'validate-integration': 'echo ok' },
+			} )
+		);
+
+		expect( statusById( root )[ 'composer-test' ] ).toBe( 'fail' );
+	} );
+
+	it( 'fails rule 1 when the composer type is not wordpress-plugin', () => {
+		const root = join( dir, 'wrong-type' );
+		mkdirSync( root, { recursive: true } );
+		scaffoldConformant( root );
+		writeFileSync(
+			join( root, 'composer.json' ),
+			JSON.stringify( {
+				type: 'library',
+				autoload: { classmap: [ 'inc/' ] },
+				scripts: { test: [ 'phpunit', 'playwright test' ], 'validate-integration': 'x' },
+			} )
+		);
+
+		expect( statusById( root )[ 'loads-through-starter-kit' ] ).toBe( 'fail' );
+	} );
+
+	it( 'resolves npm-delegated e2e through package.json for rule 2', () => {
+		const root = join( dir, 'npm-delegated' );
+		mkdirSync( root, { recursive: true } );
+		scaffoldConformant( root );
+		// composer test -> `npm test`, and package.json test -> `playwright test`.
+		writeFileSync(
+			join( root, 'composer.json' ),
+			JSON.stringify( {
+				type: 'wordpress-plugin',
+				autoload: { classmap: [ 'inc/' ] },
+				scripts: { test: [ 'phpunit', 'npm test' ], 'validate-integration': 'x' },
+			} )
+		);
+
+		expect( statusById( root )[ 'composer-test' ] ).toBe( 'pass' );
+	} );
+
+	it( 'does not accept no-op echo commands as tests for rule 2', () => {
+		const root = join( dir, 'echo-tests' );
+		mkdirSync( root, { recursive: true } );
+		scaffoldConformant( root );
+		writeFileSync(
+			join( root, 'composer.json' ),
+			JSON.stringify( {
+				type: 'wordpress-plugin',
+				autoload: { classmap: [ 'inc/' ] },
+				scripts: { test: [ 'echo phpunit', 'echo npm test' ], 'validate-integration': 'x' },
+			} )
+		);
+
+		expect( statusById( root )[ 'composer-test' ] ).toBe( 'fail' );
+	} );
+
+	it( 'fails rule 7 when compatibility is only prose, with no CI matrix', () => {
+		const root = join( dir, 'no-ci' );
+		mkdirSync( join( root, 'docs' ), { recursive: true } );
+		scaffoldConformant( root );
+		rmSync( join( root, '.github' ), { recursive: true, force: true } );
+		writeFileSync(
+			join( root, 'docs', 'compat.md' ),
+			'Tested against WordPress 6.9 and 7.0, PHP 8.2, 8.3, 8.4, 8.5. Install the latest release.'
+		);
+
+		expect( statusById( root )[ 'compatibility-matrix' ] ).toBe( 'fail' );
+	} );
+
+	it( 'distinguishes a malformed composer.json from a missing one', () => {
+		const root = join( dir, 'malformed' );
+		mkdirSync( root, { recursive: true } );
+		writeFileSync( join( root, 'composer.json' ), '{ "type": "wordpress-plugin", ' );
+		writeFileSync( join( root, 'plugin.php' ), '<?php\n/** Plugin Name: M */\n' );
+
+		const rule1 = validateIntegration( root ).results.find(
+			result => result.id === 'loads-through-starter-kit'
+		);
+		expect( rule1?.status ).toBe( 'fail' );
+		expect( rule1?.message ).toMatch( /not valid JSON/ );
+	} );
+
+	it( 'warns on rule 5 when config access is not guarded', () => {
+		const root = join( dir, 'unguarded' );
+		mkdirSync( join( root, 'docs' ), { recursive: true } );
+		writeFileSync(
+			join( root, 'widget.php' ),
+			`<?php\n/** Plugin Name: W */\n$c = constant( 'VIP_WIDGET_CONFIG' );\necho $c['api_token'];\n`
+		);
+		writeFileSync( join( root, 'composer.json' ), JSON.stringify( { type: 'wordpress-plugin' } ) );
+		writeFileSync( join( root, 'docs', 'x.md' ), 'Config: `VIP_WIDGET_CONFIG`' );
+
+		expect( statusById( root )[ 'graceful-config-handling' ] ).toBe( 'warn' );
+	} );
+
+	it( 'recognizes an integration directory', () => {
+		const root = join( dir, 'conformant' );
+		expect( looksLikeIntegration( root ) ).toBe( true );
+		expect( looksLikeIntegration( join( dir, 'does-not-exist' ) ) ).toBe( false );
+	} );
+} );

From f87c5e9d1acca474173c5e503804f0922df5ed2e Mon Sep 17 00:00:00 2001
From: Ahmed Sayeed Wasif <ahmed.wasif@automattic.com>
Date: Thu, 16 Jul 2026 12:32:57 +0600
Subject: [PATCH 4/6] Document the `vip integration validate` command

---
 docs/integration-validate.md | 47 ++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 docs/integration-validate.md

diff --git a/docs/integration-validate.md b/docs/integration-validate.md
new file mode 100644
index 000000000..d33880c44
--- /dev/null
+++ b/docs/integration-validate.md
@@ -0,0 +1,47 @@
+# `vip integration validate`
+
+Checks a WordPress VIP partner integration for conformance and returns an
+objective **conformant / not conformant** verdict — the gate a partner (or an
+internal team) runs locally and in CI before submitting an integration.
+
+```sh
+vip integration validate [path]        # defaults to the current directory
+vip integration validate --format json # machine-readable output for CI
+```
+
+- **Exit code.** `0` when conformant, `1` when any automated rule fails (or on a
+  bad path argument / unknown `--format`), so it can gate a CI job directly.
+- **`--format`.** `human` (default) or `json`. Any other value is an error.
+
+## What it decides — and what it does not
+
+The checker is a **static** analysis of the integration's files (composer.json,
+package.json, docs, PHP source, CI workflows). Each rule reports `pass`, `fail`,
+`warn`, or `n/a`, and only a `fail` breaks conformance. A `warn` flags something
+the tool cannot verify statically (it does not run the integration's tests), so
+a green result means "the integration is wired correctly," not "its tests pass."
+
+Two things are **never** decided here and are printed as a _Human review
+required_ section instead of a fake pass:
+
+- **Plugin - platform config-schema match** — not fully deterministic.
+- **Security review** — assessed by a human, not this checker.
+
+## Automated rules
+
+| #   | Rule                                                 | Fails when                                                                                                                                              |
+| --- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 1   | Loads through the Starter Kit workflow               | No composer.json / invalid JSON, no plugin entry file, wrong Composer `type`, or missing `autoload`                                                     |
+| 2   | `composer test` runs PHPUnit + e2e                   | `composer test` does not wire up `phpunit` and an e2e runner (Playwright/Cypress), resolving `npm`-delegated scripts and ignoring no-op `echo` commands |
+| 3   | `composer run validate-integration` exists           | The script is absent                                                                                                                                    |
+| 4   | Config constant documented + referenced              | The detected config constant is used in code but not documented                                                                                         |
+| 5   | Missing/invalid config handled without fataling      | _(warn-only — static analysis cannot prove runtime behavior; verify via the integration's own tests)_                                                   |
+| 6   | Docs include valid + incomplete config examples      | Fewer than two config examples, or no incomplete example                                                                                                |
+| 7   | Compatibility matrix covers WP 6.9/7.0 + PHP 8.2–8.5 | The CI workflows do not cover the matrix and there is no approved exception note                                                                        |
+| 8   | Build and test commands documented                   | Docs omit build/install or test commands                                                                                                                |
+| 9   | Telemetry uses the Tracks helper, no secrets         | Telemetry uses Stats/Pixel _(warn if the guarded Tracks helper is not detected or property keys look sensitive)_                                        |
+
+Rules 4–6 apply only when a runtime config constant is detected (a
+`Config::CONSTANT_NAME` declaration or a `VIP_*_CONFIG` constant). If an
+integration uses runtime config under a different name, adopt the convention or
+flag it for human review — those checks are reported as skipped, not passed.

From 9e12594ad29711fdeb4bd237c77d700dd43189e9 Mon Sep 17 00:00:00 2001
From: Ahmed Sayeed Wasif <ahmed.wasif@automattic.com>
Date: Thu, 16 Jul 2026 20:24:50 +0600
Subject: [PATCH 5/6] Require login when integration is only a command argument

Skipping auth on any argv containing "integration" let a logged-out
call like `vip @app.env -- wp option get integration` bypass the login
prompt and fail. Gate the no-auth path on the absence of an app/env
argument, mirroring dev-env.
---
 src/bin/vip.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/bin/vip.js b/src/bin/vip.js
index 19127bef1..3dd7b1bdc 100755
--- a/src/bin/vip.js
+++ b/src/bin/vip.js
@@ -204,8 +204,13 @@ const rootCmd = async function () {
 		doesArgvHaveAtLeastOneParam( process.argv, [ 'dev-env' ] ) &&
 		! containsAppEnvArgument( process.argv );
 	// `integration` inspects local files only and never calls the API, so it
-	// does not require authentication (same as `dev-env`).
-	const isIntegrationCommand = doesArgvHaveAtLeastOneParam( process.argv, [ 'integration' ] );
+	// does not require authentication (same as `dev-env`). Require the absence
+	// of an app/env argument so that `integration` appearing as an argument to
+	// another command (e.g. `vip @app.env -- wp option get integration`) still
+	// prompts for login instead of silently skipping it.
+	const isIntegrationCommand =
+		doesArgvHaveAtLeastOneParam( process.argv, [ 'integration' ] ) &&
+		! containsAppEnvArgument( process.argv );
 	const isCustomDeployCmdWithKey =
 		doesArgvHaveAtLeastOneParam( process.argv, [ 'deploy' ] ) && Boolean( customDeployToken );
 

From e1b0ca1dbd5c41c368bfea1eee48ec654d7c3fd3 Mon Sep 17 00:00:00 2001
From: Ahmed Sayeed Wasif <ahmed.wasif@automattic.com>
Date: Thu, 16 Jul 2026 20:25:01 +0600
Subject: [PATCH 6/6] Scope the compat-matrix latest match to WordPress
 versions

The rule 7 check treated any `latest` token as WP 7.0 evidence, so a
`runs-on: ubuntu-latest` runner label wrongly satisfied it. Match
`latest` only when tied to a WordPress version key (e.g. `wp: latest`).
---
 __tests__/lib/validate-integration.test.ts | 52 ++++++++++++++++++++++
 src/lib/validate-integration/validate.ts   |  9 ++--
 2 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/__tests__/lib/validate-integration.test.ts b/__tests__/lib/validate-integration.test.ts
index 0aefb1b54..a8df3b1f5 100644
--- a/__tests__/lib/validate-integration.test.ts
+++ b/__tests__/lib/validate-integration.test.ts
@@ -217,6 +217,58 @@ describe( 'validateIntegration', () => {
 		expect( statusById( root )[ 'compatibility-matrix' ] ).toBe( 'fail' );
 	} );
 
+	it( 'does not count `runs-on: ubuntu-latest` as WordPress 7.0 evidence', () => {
+		const root = join( dir, 'ubuntu-latest' );
+		mkdirSync( root, { recursive: true } );
+		scaffoldConformant( root );
+		// A matrix that covers 6.9 and the PHP range but expresses no WP 7.0 or
+		// `wp: latest` — only an unrelated `runs-on: ubuntu-latest` runner label.
+		writeFileSync(
+			join( root, '.github', 'workflows', 'unit-tests.yml' ),
+			[
+				'jobs:',
+				'  test:',
+				'    runs-on: ubuntu-latest',
+				'    strategy:',
+				'      matrix:',
+				'        config:',
+				"          - { wp: 6.9.x, php: '8.2' }",
+				"          - { wp: 6.9.x, php: '8.3' }",
+				"          - { wp: 6.9.x, php: '8.4' }",
+				"          - { wp: 6.9.x, php: '8.5' }",
+			].join( '\n' )
+		);
+
+		const rule7 = validateIntegration( root ).results.find(
+			result => result.id === 'compatibility-matrix'
+		);
+		expect( rule7?.status ).toBe( 'fail' );
+		expect( rule7?.message ).toMatch( /WordPress 7\.0/ );
+	} );
+
+	it( 'accepts `wp: latest` in the matrix as WordPress 7.0 evidence', () => {
+		const root = join( dir, 'wp-latest' );
+		mkdirSync( root, { recursive: true } );
+		scaffoldConformant( root );
+		writeFileSync(
+			join( root, '.github', 'workflows', 'unit-tests.yml' ),
+			[
+				'jobs:',
+				'  test:',
+				'    runs-on: ubuntu-latest',
+				'    strategy:',
+				'      matrix:',
+				'        config:',
+				"          - { wp: 6.9.x, php: '8.2' }",
+				"          - { wp: latest, php: '8.3' }",
+				"          - { wp: latest, php: '8.4' }",
+				"          - { wp: latest, php: '8.5' }",
+			].join( '\n' )
+		);
+
+		expect( statusById( root )[ 'compatibility-matrix' ] ).toBe( 'pass' );
+	} );
+
 	it( 'distinguishes a malformed composer.json from a missing one', () => {
 		const root = join( dir, 'malformed' );
 		mkdirSync( root, { recursive: true } );
diff --git a/src/lib/validate-integration/validate.ts b/src/lib/validate-integration/validate.ts
index 749e51d62..131fdab1e 100644
--- a/src/lib/validate-integration/validate.ts
+++ b/src/lib/validate-integration/validate.ts
@@ -595,9 +595,12 @@ function checkCompatibilityMatrix( ctx: Context ): CheckResult {
 	if ( ! /\b6\.9\b/.test( ctx.workflowsText ) ) {
 		missing.push( 'WordPress 6.9' );
 	}
-	// WP 7.0 in CI is often expressed as "latest"; accept either, but only from
-	// the workflow matrix.
-	if ( ! /\b7\.0\b/.test( ctx.workflowsText ) && ! /\blatest\b/.test( ctx.workflowsText ) ) {
+	// WP 7.0 in CI is often expressed as "latest" against a WordPress version
+	// key (e.g. `wp: latest`). Match that form specifically so unrelated
+	// `latest` tokens — `runs-on: ubuntu-latest`, `mariadb:latest` — are not
+	// mistaken for WordPress version evidence.
+	const wpLatest = /\bw(?:p|ordpress)(?:[-_]version)?['":= ]+latest\b/i;
+	if ( ! /\b7\.0\b/.test( ctx.workflowsText ) && ! wpLatest.test( ctx.workflowsText ) ) {
 		missing.push( 'WordPress 7.0' );
 	}
 	for ( const php of [ '8.2', '8.3', '8.4', '8.5' ] ) {