From e9144775f37af1425f8063e428de61af28eb9894 Mon Sep 17 00:00:00 2001 From: pitoi Date: Sat, 11 Jul 2026 16:12:56 +0000 Subject: [PATCH 1/2] Generated with Hive: Refine PR fix agent prompt and enhance CI log assertion extraction --- scripts/test-ci-logs.ts | 72 ++++++++- src/__tests__/unit/lib/github/pr-ci.test.ts | 149 ++++++++++++++++++ .../unit/lib/github/pr-monitor.test.ts | 49 +++++- src/lib/github/pr-ci.ts | 95 +++++++++-- src/lib/github/pr-monitor.ts | 7 +- 5 files changed, 351 insertions(+), 21 deletions(-) create mode 100644 src/__tests__/unit/lib/github/pr-ci.test.ts diff --git a/scripts/test-ci-logs.ts b/scripts/test-ci-logs.ts index c1acf6e59a..98d1becdd3 100644 --- a/scripts/test-ci-logs.ts +++ b/scripts/test-ci-logs.ts @@ -68,6 +68,21 @@ function isGroupLine(line: string): boolean { return line.includes("##[group]") || line.includes("::group::"); } +/** + * Check if a line contains test framework output (Expected/Received blocks, assertion errors, etc.). + * All patterns must be unanchored — every GitHub Actions log line is prefixed with an ISO timestamp + * (e.g. "2024-01-15T10:30:00.0000000Z"), so ^\\s+-anchored patterns will never match real lines. + */ +function isTestFrameworkLine(line: string): boolean { + return ( + /\bExpected\b/.test(line) || + /\bReceived\b/.test(line) || + /●\s+\S/.test(line) || // Jest/Vitest describe › test bullet + /\bAssertionError\b/.test(line) || + /FAIL\s+\S+\.(test|spec)\.\S+/.test(line) // Jest FAIL + ); +} + /** * NEW IMPLEMENTATION: Extract logs for a failed step * @@ -195,20 +210,25 @@ function extractStepLogs( // ============================================================ const MAX_LINES = 150; + const MAX_SECONDARY_LINES = 100; + const SECONDARY_CONTEXT = 15; + let extractedLogs: string | null = null; + let extractedStart = -1; + let extractedEnd = -1; if (errorMarkers.length > 0) { // Take lines from (firstError - MAX_LINES) through (lastError + 5) const firstErrorLine = errorMarkers[0].lineNum; const lastErrorLine = errorMarkers[errorMarkers.length - 1].lineNum; - const extractStart = Math.max(stepStartLine >= 0 ? stepStartLine : 0, firstErrorLine - MAX_LINES); - const extractEnd = Math.min(lines.length, lastErrorLine + 5); + extractedStart = Math.max(stepStartLine >= 0 ? stepStartLine : 0, firstErrorLine - MAX_LINES); + extractedEnd = Math.min(lines.length, lastErrorLine + 5); - debug.push(`STEP 4: Extracting lines ${extractStart} to ${extractEnd} (${extractEnd - extractStart} lines)`); + debug.push(`STEP 4: Extracting lines ${extractedStart} to ${extractedEnd} (${extractedEnd - extractedStart} lines)`); debug.push(` -> ${MAX_LINES} lines before first ##[error], through last ##[error] + 5`); - extractedLogs = lines.slice(extractStart, extractEnd).join("\n"); + extractedLogs = lines.slice(extractedStart, extractedEnd).join("\n"); } else if (stepStartLine >= 0) { // No ##[error] found - take last N lines of the step matchMethod += " + fallback to last N lines (no ##[error])"; @@ -227,6 +247,50 @@ function extractStepLogs( extractedLogs = "...(truncated)\n" + lines.slice(-MAX_LINES).join("\n"); } + // ============================================================ + // STEP 5: Secondary pass — find test framework output + // ============================================================ + // Captures assertion diffs (Expected/Received) that appear before ##[error] markers + // or in logs with no markers. Appended (not prepended) so it survives tail-truncation. + + const scanStart = stepStartLine >= 0 ? stepStartLine : 0; + const scanEnd = stepStartLine >= 0 ? stepEndLine : lines.length; + + let firstHit = -1; + let lastHit = -1; + for (let i = scanStart; i < scanEnd; i++) { + if (isTestFrameworkLine(lines[i])) { + if (firstHit === -1) firstHit = i; + lastHit = i; + } + } + + if (firstHit !== -1) { + const secStart = Math.max(scanStart, firstHit - SECONDARY_CONTEXT); + const secEnd = Math.min(scanEnd, lastHit + SECONDARY_CONTEXT); + + // Deduplicate: exclude lines already in the primary extraction window + let secLines: string[]; + if (extractedStart !== -1 && secEnd > extractedStart && secStart < extractedEnd) { + const beforeOverlap = secStart < extractedStart ? lines.slice(secStart, extractedStart) : []; + const afterOverlap = secEnd > extractedEnd ? lines.slice(extractedEnd, secEnd) : []; + secLines = [...beforeOverlap, ...afterOverlap]; + } else { + secLines = lines.slice(secStart, secEnd); + } + + if (secLines.length > MAX_SECONDARY_LINES) { + secLines = secLines.slice(0, MAX_SECONDARY_LINES); + } + + if (secLines.length > 0) { + debug.push(`STEP 5: Secondary pass found ${secLines.length} test framework lines`); + extractedLogs = (extractedLogs ?? "") + "\n### Test output\n" + secLines.join("\n"); + } + } else { + debug.push(`STEP 5: Secondary pass found no test framework lines`); + } + return { logs: extractedLogs, method: matchMethod, debug }; } diff --git a/src/__tests__/unit/lib/github/pr-ci.test.ts b/src/__tests__/unit/lib/github/pr-ci.test.ts new file mode 100644 index 0000000000..7b0b1a0fc5 --- /dev/null +++ b/src/__tests__/unit/lib/github/pr-ci.test.ts @@ -0,0 +1,149 @@ +import { describe, it, expect } from "vitest"; +import { extractStepLogs } from "@/lib/github/pr-ci"; + +// All mock log lines use ISO timestamp prefixes to match real GitHub Actions log format. +const TS = "2024-01-15T10:30:00.0000000Z "; + +function makeLine(content: string): string { + return `${TS}${content}`; +} + +function makeLog(lines: string[]): string { + return lines.join("\n"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Secondary pass: Expected/Received 200 lines before ##[error] +// +// Structure: +// line 0 : ##[group]Run npm test (no name match → Method B finds via error) +// lines 1-3: assertion block (● / Expected / Received) +// lines 4-203: 200 filler lines +// line 204 : ##[error]... (Method B picks up this section) +// line 205 : ##[endgroup] +// +// Primary window (MAX_LINES=150): extractStart = max(0, 204-150) = 54 +// → assertion block at lines 1-3 is OUTSIDE the window (before line 54) +// Secondary pass finds hits at 1-3, secStart=0, secEnd=18 → no overlap → appended +// ───────────────────────────────────────────────────────────────────────────── + +describe("extractStepLogs — secondary pass with error markers", () => { + it("appends Expected/Received block that appears 200 lines before ##[error]", () => { + // Assertion block comes FIRST so it is 200+ lines before the error marker + const assertionBlock = [ + makeLine("● describe › test name"), + makeLine(" Expected: 1"), + makeLine(" Received: 2"), + ]; + + // 200 filler lines between assertion and error + const filler = Array.from({ length: 200 }, (_, i) => makeLine(`filler line ${i}`)); + + const lines = [ + makeLine("##[group]Run npm test"), + ...assertionBlock, + ...filler, + makeLine("##[error]Process completed with exit code 1"), + makeLine("##[endgroup]"), + ]; + + const log = makeLog(lines); + // Step name does not match group content → Method B finds section via error marker + const result = extractStepLogs(log, 1, "Run tests"); + + expect(result).not.toBeNull(); + // Primary block must contain the error marker + expect(result).toContain("##[error]Process completed with exit code 1"); + // Secondary pass must have appended the assertion block AFTER the primary content + expect(result).toContain("### Test output"); + const primaryEnd = result!.indexOf("##[error]"); + const secondaryStart = result!.indexOf("### Test output"); + expect(secondaryStart).toBeGreaterThan(primaryEnd); + expect(result).toContain("Expected: 1"); + expect(result).toContain("Received: 2"); + }); +}); + +// ───────────────────────────────────────────────────────────────────────────── +// Secondary pass: no ##[error], has ● describe › test block +// +// Structure: +// line 0: ##[group]Run npm test (step name "npm test" matches → Method A) +// lines 1-6: short log with test framework output +// line 7: ##[endgroup] +// +// No error markers → primary is the full step section (< MAX_LINES). +// extractedStart stays -1 (no error-marker window computed). +// Secondary pass finds ●/Expected/Received → appended. +// ───────────────────────────────────────────────────────────────────────────── + +describe("extractStepLogs — secondary pass without error markers", () => { + it("captures test framework output when no ##[error] is present", () => { + const lines = [ + makeLine("##[group]Run npm test"), + makeLine("some output"), + makeLine("● describe › test name"), + makeLine(" Expected: true"), + makeLine(" Received: false"), + makeLine("more output"), + makeLine("##[endgroup]"), + ]; + + const log = makeLog(lines); + // "npm test" is contained in the group marker → Method A finds the step section + const result = extractStepLogs(log, 1, "npm test"); + + expect(result).not.toBeNull(); + expect(result).toContain("### Test output"); + expect(result).toContain("Expected: true"); + expect(result).toContain("Received: false"); + expect(result).toContain("● describe › test name"); + }); +}); + +// ───────────────────────────────────────────────────────────────────────────── +// Secondary pass: capped at 100 lines +// +// Structure: +// line 0 : ##[group]Run npm test (step name "npm test" → Method A) +// lines 1-200 : "Expected: value N" (all match isTestFrameworkLine) +// lines 201-250: regular filler +// line 251 : ##[endgroup] +// +// No error markers → primary = "...(truncated)\n" + last 150 lines (lines 102-251). +// extractedStart stays -1. +// Secondary: firstHit=1, lastHit=200; secStart=0, secEnd=215 (200+15). +// No deduplication (extractedStart=-1). 215 candidate lines → capped to 100. +// ───────────────────────────────────────────────────────────────────────────── + +describe("extractStepLogs — secondary pass line cap", () => { + it("caps secondary content at 100 lines when there are many test framework matches", () => { + // 200 lines each matching isTestFrameworkLine via "Expected" + const testLines = Array.from({ length: 200 }, (_, i) => makeLine(`Expected: value ${i}`)); + // 50 plain filler lines after the test output + const filler = Array.from({ length: 50 }, (_, i) => makeLine(`filler ${i}`)); + + const lines = [ + makeLine("##[group]Run npm test"), + ...testLines, + ...filler, + makeLine("##[endgroup]"), + ]; + + const log = makeLog(lines); + // "npm test" matches group marker → Method A finds the step + const result = extractStepLogs(log, 1, "npm test"); + + expect(result).not.toBeNull(); + expect(result).toContain("### Test output"); + + // Extract and count only the secondary section's lines + const secondaryMarker = "### Test output\n"; + const secondaryIdx = result!.indexOf(secondaryMarker); + expect(secondaryIdx).toBeGreaterThanOrEqual(0); + const secondaryContent = result!.slice(secondaryIdx + secondaryMarker.length); + const secondaryLines = secondaryContent.split("\n").filter((l) => l.length > 0); + + expect(secondaryLines.length).toBeLessThanOrEqual(100); + }); +}); diff --git a/src/__tests__/unit/lib/github/pr-monitor.test.ts b/src/__tests__/unit/lib/github/pr-monitor.test.ts index 56dfbdf323..eb8a20682b 100644 --- a/src/__tests__/unit/lib/github/pr-monitor.test.ts +++ b/src/__tests__/unit/lib/github/pr-monitor.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; -import { mergeBaseBranch, rebaseOntoBaseBranch, triggerAgentModeFix, triggerLiveModeFix, createPrLogger, monitorSinglePR } from "@/lib/github/pr-monitor"; +import { mergeBaseBranch, rebaseOntoBaseBranch, triggerAgentModeFix, triggerLiveModeFix, createPrLogger, monitorSinglePR, buildFixPrompt } from "@/lib/github/pr-monitor"; import type { Octokit } from "@octokit/rest"; import { ChatRole, ChatStatus } from "@prisma/client"; @@ -1935,3 +1935,50 @@ describe("monitorSinglePR", () => { }); }); }); + +// ───────────────────────────────────────────────────────────────────────────── +// buildFixPrompt +// ───────────────────────────────────────────────────────────────────────────── + +describe("buildFixPrompt", () => { + const base = { + artifactId: "artifact-1", + taskId: "task-1", + prNumber: 42, + owner: "acme", + repo: "app", + headBranch: "feat/x", + baseBranch: "main", + state: "ci_failure" as const, + mergeable: true, + ciStatus: "failure" as const, + prState: "open" as const, + merged: false, + failedChecks: ["CI / test"], + problemDetails: "", + failedCheckLogs: {}, + }; + + it("does NOT contain the playwright timeout hint in ci_failure output", () => { + const result = buildFixPrompt(base); + expect(result).not.toContain("playwright"); + expect(result).not.toContain("increase playwright timeouts"); + expect(result).not.toContain("github CI can be slow sometimes"); + }); + + it("contains all three diagnostic checklist points in ci_failure output", () => { + const result = buildFixPrompt(base); + expect(result).toContain("Read the full failing test file"); + expect(result).toContain("Determine whether the failure is a broken test assertion"); + expect(result).toContain("Only modify the test if the assertion itself is genuinely incorrect"); + }); + + it("diagnostic checklist appears BEFORE the Please: action directive", () => { + const result = buildFixPrompt(base); + const diagnosticIdx = result.indexOf("Before making any changes:"); + const pleaseIdx = result.indexOf("Please:"); + expect(diagnosticIdx).toBeGreaterThanOrEqual(0); + expect(pleaseIdx).toBeGreaterThanOrEqual(0); + expect(diagnosticIdx).toBeLessThan(pleaseIdx); + }); +}); diff --git a/src/lib/github/pr-ci.ts b/src/lib/github/pr-ci.ts index 4dae3432a2..3bdf5142fb 100644 --- a/src/lib/github/pr-ci.ts +++ b/src/lib/github/pr-ci.ts @@ -84,6 +84,21 @@ function isGroupLine(line: string): boolean { return line.includes("##[group]") || line.includes("::group::"); } +/** + * Check if a line contains test framework output (Expected/Received blocks, assertion errors, etc.). + * All patterns must be unanchored — every GitHub Actions log line is prefixed with an ISO timestamp + * (e.g. "2024-01-15T10:30:00.0000000Z"), so ^\\s+-anchored patterns will never match real lines. + */ +function isTestFrameworkLine(line: string): boolean { + return ( + /\bExpected\b/.test(line) || + /\bReceived\b/.test(line) || + /●\s+\S/.test(line) || // Jest/Vitest describe › test bullet + /\bAssertionError\b/.test(line) || + /FAIL\s+\S+\.(test|spec)\.\S+/.test(line) // Jest FAIL + ); +} + /** * Extract logs for a specific step from the full job logs. * @@ -98,9 +113,11 @@ function isGroupLine(line: string): boolean { * 2. Find error markers within section (GitHub adds these for any failure) * 3. Extract N lines before error marker (error details appear before the marker) */ -function extractStepLogs(fullLogs: string, stepNumber: number, stepName: string): string | null { +export function extractStepLogs(fullLogs: string, stepNumber: number, stepName: string): string | null { const lines = fullLogs.split("\n"); const MAX_LINES = 150; + const MAX_SECONDARY_LINES = 100; + const SECONDARY_CONTEXT = 15; // Step 1: Find all group markers (##[group] or ::group::) const groupMarkers: { lineNum: number; content: string }[] = []; @@ -164,30 +181,80 @@ function extractStepLogs(fullLogs: string, stepNumber: number, stepName: string) } } - // Step 4: Extract logs + // Track extraction window for deduplication in the secondary pass + let extractedStart = -1; + let extractedEnd = -1; + + // Step 4: Extract primary logs — collect into variable instead of returning early + // so the secondary pass can always run after. + let primaryResult: string; + if (errorMarkers.length > 0) { // Take MAX_LINES before first error marker, through last error marker + 5 const firstErrorLine = errorMarkers[0].lineNum; const lastErrorLine = errorMarkers[errorMarkers.length - 1].lineNum; - const extractStart = Math.max(stepStartLine >= 0 ? stepStartLine : 0, firstErrorLine - MAX_LINES); - const extractEnd = Math.min(lines.length, lastErrorLine + 5); - - return lines.slice(extractStart, extractEnd).join("\n"); - } + extractedStart = Math.max(stepStartLine >= 0 ? stepStartLine : 0, firstErrorLine - MAX_LINES); + extractedEnd = Math.min(lines.length, lastErrorLine + 5); - // No error markers found - take last MAX_LINES of step (or entire log as last resort) - if (stepStartLine >= 0) { + primaryResult = lines.slice(extractedStart, extractedEnd).join("\n"); + } else if (stepStartLine >= 0) { + // No error markers found - take last MAX_LINES of step const stepLines = lines.slice(stepStartLine, stepEndLine); if (stepLines.length > MAX_LINES) { - return "...(truncated)\n" + stepLines.slice(-MAX_LINES).join("\n"); + primaryResult = "...(truncated)\n" + stepLines.slice(-MAX_LINES).join("\n"); + } else { + primaryResult = stepLines.join("\n"); + } + } else { + // Last resort: end of entire log — return immediately, no bounded section to scan + log.warn("Could not find step section, using end of log", { stepName, stepNumber }); + return "...(truncated)\n" + lines.slice(-MAX_LINES).join("\n"); + } + + // Step 5: Secondary pass — find test framework output (Expected/Received, assertion errors, etc.) + // This captures assertion diffs that appear before ##[error] markers or in logs with no markers. + // Runs universally across all primary branches so nothing is missed. + const scanStart = stepStartLine >= 0 ? stepStartLine : 0; + const scanEnd = stepStartLine >= 0 ? stepEndLine : lines.length; + + let firstHit = -1; + let lastHit = -1; + for (let i = scanStart; i < scanEnd; i++) { + if (isTestFrameworkLine(lines[i])) { + if (firstHit === -1) firstHit = i; + lastHit = i; + } + } + + if (firstHit !== -1) { + const secStart = Math.max(scanStart, firstHit - SECONDARY_CONTEXT); + const secEnd = Math.min(scanEnd, lastHit + SECONDARY_CONTEXT); + + // Deduplicate: exclude lines already in the primary extraction window + let secLines: string[]; + if (extractedStart !== -1 && secEnd > extractedStart && secStart < extractedEnd) { + // Ranges overlap — take non-overlapping portions + const beforeOverlap = secStart < extractedStart ? lines.slice(secStart, extractedStart) : []; + const afterOverlap = secEnd > extractedEnd ? lines.slice(extractedEnd, secEnd) : []; + secLines = [...beforeOverlap, ...afterOverlap]; + } else { + secLines = lines.slice(secStart, secEnd); + } + + // Cap at MAX_SECONDARY_LINES + if (secLines.length > MAX_SECONDARY_LINES) { + secLines = secLines.slice(0, MAX_SECONDARY_LINES); + } + + if (secLines.length > 0) { + // Append (not prepend) — fetchFailedStepLogs truncates via slice(-maxLogSize), + // keeping the tail; content at the front is the first material silently discarded. + primaryResult += "\n### Test output\n" + secLines.join("\n"); } - return stepLines.join("\n"); } - // Last resort: end of entire log - log.warn("Could not find step section, using end of log", { stepName, stepNumber }); - return "...(truncated)\n" + lines.slice(-MAX_LINES).join("\n"); + return primaryResult; } /** diff --git a/src/lib/github/pr-monitor.ts b/src/lib/github/pr-monitor.ts index 2cc231a959..f0f2459c1e 100644 --- a/src/lib/github/pr-monitor.ts +++ b/src/lib/github/pr-monitor.ts @@ -61,13 +61,16 @@ ${result.problemDetails || ""}`; Failed checks: ${result.failedChecks?.join(", ") || "unknown"} +Before making any changes: +1. Read the full failing test file and understand exactly what the test is asserting. +2. Determine whether the failure is a broken test assertion (the test expectation is wrong) or a code regression (the implementation broke something the test correctly captures). +3. Only modify the test if the assertion itself is genuinely incorrect — never change a test just to make CI green. If the code is broken, fix the code. + Please: 1. Review the CI failure logs below 2. Fix the issues causing the failures 3. Changes will be pushed to the PR automatically, you don't need to push manually. -(if the error is from playwright, you might just need to increase playwright timeouts a bit... github CI can be slow sometimes) - ${result.problemDetails || ""}`; // Append log excerpts if available From e2515cfd445e95ad935a5b240d35e1e28ff46ea2 Mon Sep 17 00:00:00 2001 From: pitoi Date: Wed, 22 Jul 2026 19:49:17 +0000 Subject: [PATCH 2/2] Generated with Hive: Increase integration test hook timeout to 60s for CI stability --- vitest.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/vitest.config.ts b/vitest.config.ts index ad6f09d3d1..1bd4771395 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -30,6 +30,7 @@ export default defineConfig({ // --max-old-space-size in execArgv, causing ERR_WORKER_INVALID_EXEC_ARGV. // A single fork also ensures Prisma's native library engine is initialised // once, avoiding the "Engine is not yet connected" race seen with multiple forks. + hookTimeout: testSuite === "integration" ? 60000 : 10000, pool: testSuite === "integration" ? "forks" : "threads", poolOptions: testSuite === "integration" ? { forks: {