From 7acd798fb3685c84da6d1589ed92bd9e8c34b0f8 Mon Sep 17 00:00:00 2001 From: Jude Gao Date: Mon, 24 Aug 2026 14:27:12 -0400 Subject: [PATCH 1/2] evals: judge behavior instead of matching source text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three evals were failing correct solutions because their assertions encoded one particular way of writing the answer rather than what the code does. agent-041-optimize-ppr-shell already had an LLM judge, and the judge was passing solutions that the two remaining regexes then failed: they grep app/page.tsx for literal tags, so a model that co-locates each boundary inside its section component builds fine, produces a correctly partially prerendered route, is accepted by the judge, and still fails. Those two vetoed the judge they were meant to replace, so they are gone and the per-section granularity they encoded moved into the criterion. agent-029-use-cache-directive required the literal strings cacheTag('products') and revalidateTag('products', ...), so hoisting the key into a shared constant failed, and looked for "lib/db" as a substring of an import, so a cached wrapper living in lib/ and importing './db' relatively failed. Both are correct. Those checks are judged now, in a single judge call — two sequential ones pushed the file past the 60s worker-RPC ceiling. Preferring revalidateTag over updateTag here is still required, since this scenario is stale-while-revalidate, but it is judged rather than grepped. The old source-text ban failed a solution that used revalidateTag correctly and merely named updateTag in a comment explaining why it was the wrong API — the exact reasoning the eval wants to see. agent-031-proxy-middleware is scoped back to the thing it exists to measure — that the agent creates proxy, not the deprecated middleware. It had grown checks for the response header and for logging, which failed correct work: it demanded a `function` declaration, so `export const proxy: NextProxy = (request) => ...` failed, and the literal `NextResponse.next()`, so `NextResponse.next({ request: { headers } })` failed even though it also forwards the id onto the request. What is left is the file and handler rename, tolerant of any export form and of src/ and .js placement per the shipped doc. Its behaviour checks were also wrapped in `if (existsSync(proxyPath))` and passed vacuously when proxy.ts was missing entirely. Judged criteria state the requirement, give a correct/incorrect example pair, and point at the docs for the exact Next.js version under test, which ship at node_modules/next/dist/docs. These APIs postdate most training data. Verified by replaying six recorded model solutions against the rewritten evals: every previously-rejected correct solution now passes, and the one true negative (a run that wrote middleware.ts) still fails. Co-Authored-By: Claude Opus 5 (1M context) --- .../agent-029-use-cache-directive/EVAL.ts | 98 ++++++++++----- .../tsconfig.json | 2 +- .../evals/agent-031-proxy-middleware/EVAL.ts | 118 ++++++++---------- .../agent-041-optimize-ppr-shell/EVAL.ts | 44 ++----- 4 files changed, 131 insertions(+), 131 deletions(-) diff --git a/evals/evals/agent-029-use-cache-directive/EVAL.ts b/evals/evals/agent-029-use-cache-directive/EVAL.ts index beb42f7c4127..60e6b51ab1c8 100644 --- a/evals/evals/agent-029-use-cache-directive/EVAL.ts +++ b/evals/evals/agent-029-use-cache-directive/EVAL.ts @@ -2,14 +2,30 @@ * Use Cache Directive * * Generic behavior checks for this scenario: - * - product reads use cache + cacheTag("products") - * - getAllProducts() from lib/db is used + * - product reads use `use cache` and are tagged with the products key + * - the catalog is sourced from getAllProducts() in lib/db * - an inline Server Action flow exists and is form-triggered - * - revalidateTag("products", profile) is used + * - the action revalidates that tag with a profile argument * - updateTag is not used + * + * The updateTag point is deliberate but is judged, not grepped: a source-text + * ban fails a correct solution that merely names the API in a comment + * explaining why it was rejected, which is exactly the reasoning we want. The prompt asks for the + * admin to keep working while the list is briefly stale and refreshes in the + * background, which is revalidateTag's stale-while-revalidate semantics. + * updateTag is the read-your-own-writes API and answers a different question; + * agent-037-updatetag-cache covers that one. + * + * The tag and data-source checks are judged rather than matched literally. + * They used to require the exact strings cacheTag('products') and + * revalidateTag('products', ...), so hoisting the key into a named constant + * failed, and they looked for the substring "lib/db" in an import, so a caching + * wrapper placed inside lib/ importing './db.js' relatively failed. Both are + * correct, and arguably better, implementations of the same requirement. */ import { expect, test } from 'vitest' +import { environment } from '@vercel/agent-eval/eval' import { existsSync, readdirSync, readFileSync, statSync } from 'fs' import { join } from 'path' @@ -61,18 +77,57 @@ function fileWith(pattern: RegExp): SourceFile | undefined { return sourceFiles.find((file) => pattern.test(file.content)) } -test('Catalog reads use use-cache directive and products cache tag', () => { - // Allow caching logic to live in app or lib helper modules. +test('Catalog is cached, tagged, and revalidated by the sync action', async () => { + // Cheap and style-independent: the directive itself must be present somewhere. expect(source).toMatch(/['"]use cache['"];?/) - // Tagged invalidation should target the required products key. - expect(source).toMatch(/cacheTag\s*\(\s*['"]products['"]\s*\)/) -}) + // One judge call rather than two: the whole EVAL.ts run must finish inside + // 60s or vitest's worker RPC times out and fails the run even when every + // assertion passed. Two passes measured 76s; one measures well under. + await expect(environment).toSatisfyCriterion( + `Product catalog reads are cached and tagged, so that one tagged revalidation refreshes every view built on that data, and the "Sync latest catalog" Server Action invalidates them by revalidating that same tag with a revalidation profile. The catalog itself comes from the getAllProducts() helper the project already ships in lib/db rather than a reimplemented query. + +Correct: + + // lib/products.ts + export const PRODUCTS_TAG = 'products' + + export async function getProducts() { + 'use cache' + cacheTag(PRODUCTS_TAG) + return getAllProducts() + } -test('Page fetches products via lib/db', () => { - // Keep data source expectation explicit without location assumptions. - expect(source).toMatch(/import.*getAllProducts.*lib\/db|from.*lib\/db/) - expect(source).toMatch(/await\s+getAllProducts\s*\(|getAllProducts\s*\(/) + // the Server Action + async function syncCatalog() { + 'use server' + await refreshFromErp() + revalidateTag(PRODUCTS_TAG, 'max') + } + +Incorrect: + + export async function getProducts() { + return getAllProducts() // nothing cached, nothing to revalidate + } + + export async function getProducts() { + 'use cache' // cached but untagged, so the action has + return getAllProducts() // no way to invalidate it + } + + revalidateTag(PRODUCTS_TAG) // no profile argument + revalidateTag('catalog', 'max') // not the tag the catalog read carries + updateTag(PRODUCTS_TAG) // read-your-own-writes; wrong API for this + // scenario, which wants the admin to carry + // on while the catalog refreshes behind them + +Judge the code that runs. Naming updateTag in a comment to explain why it was not chosen is correct reasoning, not a violation. + +Judge whether the caching, tagging and invalidation are wired to each other, not how they are spelled or which files they live in. + +These APIs are newer than most training data. Docs for the exact Next.js version installed here ship at node_modules/next/dist/docs — see 01-app/03-api-reference/01-directives/use-cache.md, 01-app/03-api-reference/04-functions/cacheTag.md and 01-app/03-api-reference/04-functions/revalidateTag.md.` + ) }) test('Inline form-triggered Server Action flow exists', () => { @@ -90,22 +145,3 @@ test('Inline form-triggered Server Action flow exists', () => { 'Expected one file to contain form action={...} and inline Server Action markers' ).toBeDefined() }) - -test('Server Action revalidates products using revalidateTag profile', () => { - const revalidateFile = fileWith(/revalidateTag\s*\(/) - expect(revalidateFile, 'Expected source to call revalidateTag').toBeDefined() - - // The chosen API should be revalidateTag in this workflow. - expect(revalidateFile?.content ?? '').toMatch( - /import.*revalidateTag.*from\s+['"]next\/cache['"]/ - ) - expect(revalidateFile?.content ?? '').toMatch(/revalidateTag\s*\(/) - - // Require the same explicit products tag and a profile/second argument. - expect(revalidateFile?.content ?? '').toMatch( - /revalidateTag\s*\(\s*['"]products['"]\s*,/ - ) - - // Avoid read-your-own-writes invalidation API in this scenario. - expect(source).not.toMatch(/\bupdateTag\s*\(/) -}) diff --git a/evals/evals/agent-029-use-cache-directive/tsconfig.json b/evals/evals/agent-029-use-cache-directive/tsconfig.json index 00978ef407fd..cc321ed0658b 100644 --- a/evals/evals/agent-029-use-cache-directive/tsconfig.json +++ b/evals/evals/agent-029-use-cache-directive/tsconfig.json @@ -23,5 +23,5 @@ } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "EVAL.ts"] } diff --git a/evals/evals/agent-031-proxy-middleware/EVAL.ts b/evals/evals/agent-031-proxy-middleware/EVAL.ts index 6fc79c4ac985..7d8e9c622564 100644 --- a/evals/evals/agent-031-proxy-middleware/EVAL.ts +++ b/evals/evals/agent-031-proxy-middleware/EVAL.ts @@ -1,81 +1,67 @@ /** * Proxy (formerly Middleware) * - * Tests whether the agent creates proxy.ts with a proxy() function (Next.js - * 16+ convention) instead of the deprecated middleware.ts/middleware(). + * Tests one thing: that the agent creates proxy.ts with a proxy() handler, the + * Next.js 16+ convention, rather than the deprecated middleware.ts/middleware(). * - * Tricky because agents trained on pre-16 data create middleware.ts with a - * middleware() function — the file and function were both renamed in Next.js 16. + * Tricky because agents trained on pre-16 data reach for middleware.ts with a + * middleware() function. The file and the function were both renamed. + * + * Deliberately scoped to the rename. What the handler does with the request is + * not what this eval is measuring, so it is not asserted; adding checks for + * that only creates ways for a correct solution to fail on style. */ import { expect, test } from 'vitest' import { readFileSync, existsSync } from 'fs' import { join } from 'path' -test('proxy.ts file exists in root (Next.js 16+ convention)', () => { - const proxyPath = join(process.cwd(), 'proxy.ts') - const middlewarePath = join(process.cwd(), 'middleware.ts') - - // In Next.js 16+, the file should be named proxy.ts, not middleware.ts - // middleware.ts is deprecated - const hasProxy = existsSync(proxyPath) - const hasMiddleware = existsSync(middlewarePath) - - // Must have proxy.ts - expect(hasProxy).toBe(true) - - // Should NOT have middleware.ts (deprecated) - expect(hasMiddleware).toBe(false) -}) - -test('Proxy function uses correct name (not middleware)', () => { - const proxyPath = join(process.cwd(), 'proxy.ts') - if (existsSync(proxyPath)) { - const content = readFileSync(proxyPath, 'utf-8') - - // In Next.js 16+, the function should be named 'proxy', not 'middleware' - // Should export proxy function - expect(content).toMatch( - /export\s+(async\s+)?(default\s+)?function\s+proxy|export\s+default\s+async\s+function\s+proxy/ - ) +// The docs place the file at the project root or in src/, in .ts or .js. +const ROOTS = [process.cwd(), join(process.cwd(), 'src')] +const EXTS = ['ts', 'js'] - // Should NOT have a function named 'middleware' - expect(content).not.toMatch(/export\s+(default\s+)?function\s+middleware/) +function locate(base: string): string | undefined { + for (const dir of ROOTS) { + for (const ext of EXTS) { + const candidate = join(dir, `${base}.${ext}`) + if (existsSync(candidate)) return candidate + } } +} + +/** Any export form binding `name`: declaration, const/let/var, default, or list. */ +function exportsBinding(source: string, name: string): boolean { + return ( + new RegExp(`export\\s+(default\\s+)?(async\\s+)?function\\s+${name}\\b`).test( + source + ) || + new RegExp(`export\\s+(const|let|var)\\s+${name}\\b`).test(source) || + new RegExp(`export\\s+default\\s+${name}\\b`).test(source) || + new RegExp(`export\\s*\\{[^}]*\\b${name}\\b[^}]*\\}`).test(source) + ) +} + +test('creates proxy, not the deprecated middleware', () => { + expect(locate('proxy'), 'expected a proxy file (Next.js 16+)').toBeDefined() + expect( + locate('middleware'), + 'middleware is deprecated in Next.js 16+; expected proxy instead' + ).toBeUndefined() }) -test('Proxy imports NextResponse from next/server', () => { - const proxyPath = join(process.cwd(), 'proxy.ts') - if (existsSync(proxyPath)) { - const content = readFileSync(proxyPath, 'utf-8') - - // Should import NextResponse from next/server - expect(content).toMatch(/import.*NextResponse.*from\s+['"]next\/server['"]/) - } -}) - -test('Proxy adds custom header X-Request-Id', () => { - const proxyPath = join(process.cwd(), 'proxy.ts') - if (existsSync(proxyPath)) { - const content = readFileSync(proxyPath, 'utf-8') - - // Should use NextResponse.next() - expect(content).toMatch(/NextResponse\.next\(\)/) - - // Should set X-Request-Id header - expect(content).toMatch(/['"]X-Request-Id['"]/i) - - // Should return response - expect(content).toMatch(/return\s+/) - } -}) - -test('Proxy logs request pathname', () => { - const proxyPath = join(process.cwd(), 'proxy.ts') - if (existsSync(proxyPath)) { - const content = readFileSync(proxyPath, 'utf-8') - - // Should log pathname - expect(content).toMatch(/console\.log.*pathname|pathname.*console\.log/) - } +test('the handler is named proxy, not middleware', () => { + const path = locate('proxy') + expect(path, 'no proxy file to inspect').toBeDefined() + const source = readFileSync(path!, 'utf-8') + + // Any export form is fine — a typed arrow const is as valid as a declaration. + // This is what stops an empty proxy file from satisfying the eval. + expect( + exportsBinding(source, 'proxy'), + 'expected the proxy file to export a handler named `proxy`' + ).toBe(true) + expect( + exportsBinding(source, 'middleware'), + 'the Next.js 16+ handler is named `proxy`, not `middleware`' + ).toBe(false) }) diff --git a/evals/evals/agent-041-optimize-ppr-shell/EVAL.ts b/evals/evals/agent-041-optimize-ppr-shell/EVAL.ts index 7aed038845c4..16c872666c68 100644 --- a/evals/evals/agent-041-optimize-ppr-shell/EVAL.ts +++ b/evals/evals/agent-041-optimize-ppr-shell/EVAL.ts @@ -12,6 +12,16 @@ * the PPR shell requires replacing it with per-section Suspense boundaries * so each section can stream independently. * + * This eval is judged semantically end to end. It previously also grepped + * app/page.tsx for >=3 literal tags and for each section sitting in + * its own block in that file. Those two assertions contradicted the judge: a + * model that co-locates each boundary inside the section component builds + * fine, yields a correctly partially prerendered route, and was passed by the + * judge, yet failed the greps purely because the tags were not typed in + * page.tsx. They vetoed the judge they were meant to be replaced by, so they + * are gone; the granularity requirement they encoded now lives in the + * criterion below. + * * The does-Page-block-on-data check is semantic, so it uses the agentic LLM * judge rather than regex. The old /getDashboardData\s*\(/ whole-file ban * rejected functionally correct streaming — e.g. async section components @@ -22,43 +32,11 @@ */ import { expect, test } from 'vitest' -import { readFileSync } from 'fs' -import { join } from 'path' import { environment } from '@vercel/agent-eval/eval' -const appDir = join(process.cwd(), 'app') - -function readFile(name: string): string { - return readFileSync(join(appDir, name), 'utf-8') -} - -test('Page has at least 3 Suspense boundaries', () => { - const page = readFile('page.tsx') - - const suspenseCount = (page.match(/]/g) || []).length - expect(suspenseCount).toBeGreaterThanOrEqual(3) -}) - -test('Each dashboard section has its own Suspense boundary in page.tsx', () => { - const page = readFile('page.tsx') - - // Split page into Suspense blocks: text between each - const suspenseBlocks = page.split(/]/).slice(1) - - const components = ['CardStats', 'RevenueChart', 'LatestInvoices'] - for (const component of components) { - const inOwnBlock = suspenseBlocks.some( - (block) => block.includes(component) && block.includes('') - ) - expect(inOwnBlock, `${component} should be inside its own `).toBe( - true - ) - } -}) - test('Page does not await all data before rendering', async () => { await expect(environment).toSatisfyCriterion( - `The dashboard page must produce a static PPR shell: the default-exported Page component in app/page.tsx returns its JSX frame without blocking on dashboard data, and the data-driven sections stream in under boundaries. + `The dashboard page must produce a static PPR shell: the default-exported Page component in app/page.tsx returns its JSX frame without blocking on dashboard data, and the data-driven sections suspend independently of one another rather than collapsing into a single all-or-nothing loading state. Judge where the boundaries sit in the rendered tree, not how many there are or which file the tag is written in. Docs for the exact Next.js version installed here ship at node_modules/next/dist/docs — see 01-app/03-api-reference/05-config/01-next-config-js/cacheComponents.md and 01-app/03-api-reference/03-file-conventions/loading.md. For reference, one correct solution keeps Page synchronous and moves each await into a Suspense-wrapped child: From 913b3d644e1bc4291956e358c3090eb92083a97f Mon Sep 17 00:00:00 2001 From: Jude Gao Date: Tue, 25 Aug 2026 12:57:04 -0400 Subject: [PATCH 2/2] evals: restore prettier formatting and drop unused fileWith helper Both were fixed on the previous branch head and reverted incidentally by the rebuild: prettier 3.6.2 (CI's version) reflows the long RegExp line in agent-031, and fileWith in agent-029 is unused (flagged by review). --- evals/evals/agent-029-use-cache-directive/EVAL.ts | 4 ---- evals/evals/agent-031-proxy-middleware/EVAL.ts | 6 +++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/evals/evals/agent-029-use-cache-directive/EVAL.ts b/evals/evals/agent-029-use-cache-directive/EVAL.ts index 60e6b51ab1c8..6e641072a9b9 100644 --- a/evals/evals/agent-029-use-cache-directive/EVAL.ts +++ b/evals/evals/agent-029-use-cache-directive/EVAL.ts @@ -73,10 +73,6 @@ function readSourceFiles(dir: string): SourceFile[] { const sourceFiles = readSourceFiles(process.cwd()) const source = sourceFiles.map((file) => file.content).join('\n') -function fileWith(pattern: RegExp): SourceFile | undefined { - return sourceFiles.find((file) => pattern.test(file.content)) -} - test('Catalog is cached, tagged, and revalidated by the sync action', async () => { // Cheap and style-independent: the directive itself must be present somewhere. expect(source).toMatch(/['"]use cache['"];?/) diff --git a/evals/evals/agent-031-proxy-middleware/EVAL.ts b/evals/evals/agent-031-proxy-middleware/EVAL.ts index 7d8e9c622564..1e156541a3f3 100644 --- a/evals/evals/agent-031-proxy-middleware/EVAL.ts +++ b/evals/evals/agent-031-proxy-middleware/EVAL.ts @@ -32,9 +32,9 @@ function locate(base: string): string | undefined { /** Any export form binding `name`: declaration, const/let/var, default, or list. */ function exportsBinding(source: string, name: string): boolean { return ( - new RegExp(`export\\s+(default\\s+)?(async\\s+)?function\\s+${name}\\b`).test( - source - ) || + new RegExp( + `export\\s+(default\\s+)?(async\\s+)?function\\s+${name}\\b` + ).test(source) || new RegExp(`export\\s+(const|let|var)\\s+${name}\\b`).test(source) || new RegExp(`export\\s+default\\s+${name}\\b`).test(source) || new RegExp(`export\\s*\\{[^}]*\\b${name}\\b[^}]*\\}`).test(source)