Generated with Hive: Refine PR fix agent prompt and enhance CI log assertion extraction

scripts/test-ci-logs.ts

@@ -68,6 +68,21 @@ function isGroupLine(line: string): boolean {
   return line.includes("##[group]") || line.includes("::group::");
 }
 
+/**
+ * Check if a line contains test framework output (Expected/Received blocks, assertion errors, etc.).
+ * All patterns must be unanchored — every GitHub Actions log line is prefixed with an ISO timestamp
+ * (e.g. "2024-01-15T10:30:00.0000000Z"), so ^\\s+-anchored patterns will never match real lines.
+ */
+function isTestFrameworkLine(line: string): boolean {
+  return (
+    /\bExpected\b/.test(line) ||
+    /\bReceived\b/.test(line) ||
+    /●\s+\S/.test(line) || // Jest/Vitest describe › test bullet
+    /\bAssertionError\b/.test(line) ||
+    /FAIL\s+\S+\.(test|spec)\.\S+/.test(line) // Jest FAIL <file>
+  );
+}
+
 /**
  * NEW IMPLEMENTATION: Extract logs for a failed step
  *
@@ -195,20 +210,25 @@ function extractStepLogs(
   // ============================================================
 
   const MAX_LINES = 150;
+  const MAX_SECONDARY_LINES = 100;
+  const SECONDARY_CONTEXT = 15;
+
   let extractedLogs: string | null = null;
+  let extractedStart = -1;
+  let extractedEnd = -1;
 
   if (errorMarkers.length > 0) {
     // Take lines from (firstError - MAX_LINES) through (lastError + 5)
     const firstErrorLine = errorMarkers[0].lineNum;
     const lastErrorLine = errorMarkers[errorMarkers.length - 1].lineNum;
 
-    const extractStart = Math.max(stepStartLine >= 0 ? stepStartLine : 0, firstErrorLine - MAX_LINES);
-    const extractEnd = Math.min(lines.length, lastErrorLine + 5);
+    extractedStart = Math.max(stepStartLine >= 0 ? stepStartLine : 0, firstErrorLine - MAX_LINES);
+    extractedEnd = Math.min(lines.length, lastErrorLine + 5);
 
-    debug.push(`STEP 4: Extracting lines ${extractStart} to ${extractEnd} (${extractEnd - extractStart} lines)`);
+    debug.push(`STEP 4: Extracting lines ${extractedStart} to ${extractedEnd} (${extractedEnd - extractedStart} lines)`);
     debug.push(`  -> ${MAX_LINES} lines before first ##[error], through last ##[error] + 5`);
 
-    extractedLogs = lines.slice(extractStart, extractEnd).join("\n");
+    extractedLogs = lines.slice(extractedStart, extractedEnd).join("\n");
   } else if (stepStartLine >= 0) {
     // No ##[error] found - take last N lines of the step
     matchMethod += " + fallback to last N lines (no ##[error])";
@@ -227,6 +247,50 @@ function extractStepLogs(
     extractedLogs = "...(truncated)\n" + lines.slice(-MAX_LINES).join("\n");
   }
 
+  // ============================================================
+  // STEP 5: Secondary pass — find test framework output
+  // ============================================================
+  // Captures assertion diffs (Expected/Received) that appear before ##[error] markers
+  // or in logs with no markers. Appended (not prepended) so it survives tail-truncation.
+
+  const scanStart = stepStartLine >= 0 ? stepStartLine : 0;
+  const scanEnd = stepStartLine >= 0 ? stepEndLine : lines.length;
+
+  let firstHit = -1;
+  let lastHit = -1;
+  for (let i = scanStart; i < scanEnd; i++) {
+    if (isTestFrameworkLine(lines[i])) {
+      if (firstHit === -1) firstHit = i;
+      lastHit = i;
+    }
+  }
+
+  if (firstHit !== -1) {
+    const secStart = Math.max(scanStart, firstHit - SECONDARY_CONTEXT);
+    const secEnd = Math.min(scanEnd, lastHit + SECONDARY_CONTEXT);
+
+    // Deduplicate: exclude lines already in the primary extraction window
+    let secLines: string[];
+    if (extractedStart !== -1 && secEnd > extractedStart && secStart < extractedEnd) {
+      const beforeOverlap = secStart < extractedStart ? lines.slice(secStart, extractedStart) : [];
+      const afterOverlap = secEnd > extractedEnd ? lines.slice(extractedEnd, secEnd) : [];
+      secLines = [...beforeOverlap, ...afterOverlap];
+    } else {
+      secLines = lines.slice(secStart, secEnd);
+    }
+
+    if (secLines.length > MAX_SECONDARY_LINES) {
+      secLines = secLines.slice(0, MAX_SECONDARY_LINES);
+    }
+
+    if (secLines.length > 0) {
+      debug.push(`STEP 5: Secondary pass found ${secLines.length} test framework lines`);
+      extractedLogs = (extractedLogs ?? "") + "\n### Test output\n" + secLines.join("\n");
+    }
+  } else {
+    debug.push(`STEP 5: Secondary pass found no test framework lines`);
+  }
+
   return { logs: extractedLogs, method: matchMethod, debug };
 }
 

src/__tests__/unit/lib/github/pr-ci.test.ts

@@ -0,0 +1,149 @@
+import { describe, it, expect } from "vitest";
+import { extractStepLogs } from "@/lib/github/pr-ci";
+
+// All mock log lines use ISO timestamp prefixes to match real GitHub Actions log format.
+const TS = "2024-01-15T10:30:00.0000000Z ";
+
+function makeLine(content: string): string {
+  return `${TS}${content}`;
+}
+
+function makeLog(lines: string[]): string {
+  return lines.join("\n");
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Secondary pass: Expected/Received 200 lines before ##[error]
+//
+// Structure:
+//   line 0  : ##[group]Run npm test  (no name match → Method B finds via error)
+//   lines 1-3: assertion block (● / Expected / Received)
+//   lines 4-203: 200 filler lines
+//   line 204 : ##[error]...          (Method B picks up this section)
+//   line 205 : ##[endgroup]
+//
+// Primary window (MAX_LINES=150): extractStart = max(0, 204-150) = 54
+//   → assertion block at lines 1-3 is OUTSIDE the window (before line 54)
+// Secondary pass finds hits at 1-3, secStart=0, secEnd=18 → no overlap → appended
+// ─────────────────────────────────────────────────────────────────────────────
+
+describe("extractStepLogs — secondary pass with error markers", () => {
+  it("appends Expected/Received block that appears 200 lines before ##[error]", () => {
+    // Assertion block comes FIRST so it is 200+ lines before the error marker
+    const assertionBlock = [
+      makeLine("● describe › test name"),
+      makeLine("  Expected: 1"),
+      makeLine("  Received: 2"),
+    ];
+
+    // 200 filler lines between assertion and error
+    const filler = Array.from({ length: 200 }, (_, i) => makeLine(`filler line ${i}`));
+
+    const lines = [
+      makeLine("##[group]Run npm test"),
+      ...assertionBlock,
+      ...filler,
+      makeLine("##[error]Process completed with exit code 1"),
+      makeLine("##[endgroup]"),
+    ];
+
+    const log = makeLog(lines);
+    // Step name does not match group content → Method B finds section via error marker
+    const result = extractStepLogs(log, 1, "Run tests");
+
+    expect(result).not.toBeNull();
+    // Primary block must contain the error marker
+    expect(result).toContain("##[error]Process completed with exit code 1");
+    // Secondary pass must have appended the assertion block AFTER the primary content
+    expect(result).toContain("### Test output");
+    const primaryEnd = result!.indexOf("##[error]");
+    const secondaryStart = result!.indexOf("### Test output");
+    expect(secondaryStart).toBeGreaterThan(primaryEnd);
+    expect(result).toContain("Expected: 1");
+    expect(result).toContain("Received: 2");
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Secondary pass: no ##[error], has ● describe › test block
+//
+// Structure:
+//   line 0: ##[group]Run npm test  (step name "npm test" matches → Method A)
+//   lines 1-6: short log with test framework output
+//   line 7: ##[endgroup]
+//
+// No error markers → primary is the full step section (< MAX_LINES).
+// extractedStart stays -1 (no error-marker window computed).
+// Secondary pass finds ●/Expected/Received → appended.
+// ─────────────────────────────────────────────────────────────────────────────
+
+describe("extractStepLogs — secondary pass without error markers", () => {
+  it("captures test framework output when no ##[error] is present", () => {
+    const lines = [
+      makeLine("##[group]Run npm test"),
+      makeLine("some output"),
+      makeLine("● describe › test name"),
+      makeLine("  Expected: true"),
+      makeLine("  Received: false"),
+      makeLine("more output"),
+      makeLine("##[endgroup]"),
+    ];
+
+    const log = makeLog(lines);
+    // "npm test" is contained in the group marker → Method A finds the step section
+    const result = extractStepLogs(log, 1, "npm test");
+
+    expect(result).not.toBeNull();
+    expect(result).toContain("### Test output");
+    expect(result).toContain("Expected: true");
+    expect(result).toContain("Received: false");
+    expect(result).toContain("● describe › test name");
+  });
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Secondary pass: capped at 100 lines
+//
+// Structure:
+//   line 0       : ##[group]Run npm test  (step name "npm test" → Method A)
+//   lines 1-200  : "Expected: value N"    (all match isTestFrameworkLine)
+//   lines 201-250: regular filler
+//   line 251     : ##[endgroup]
+//
+// No error markers → primary = "...(truncated)\n" + last 150 lines (lines 102-251).
+// extractedStart stays -1.
+// Secondary: firstHit=1, lastHit=200; secStart=0, secEnd=215 (200+15).
+// No deduplication (extractedStart=-1). 215 candidate lines → capped to 100.
+// ─────────────────────────────────────────────────────────────────────────────
+
+describe("extractStepLogs — secondary pass line cap", () => {
+  it("caps secondary content at 100 lines when there are many test framework matches", () => {
+    // 200 lines each matching isTestFrameworkLine via "Expected"
+    const testLines = Array.from({ length: 200 }, (_, i) => makeLine(`Expected: value ${i}`));
+    // 50 plain filler lines after the test output
+    const filler = Array.from({ length: 50 }, (_, i) => makeLine(`filler ${i}`));
+
+    const lines = [
+      makeLine("##[group]Run npm test"),
+      ...testLines,
+      ...filler,
+      makeLine("##[endgroup]"),
+    ];
+
+    const log = makeLog(lines);
+    // "npm test" matches group marker → Method A finds the step
+    const result = extractStepLogs(log, 1, "npm test");
+
+    expect(result).not.toBeNull();
+    expect(result).toContain("### Test output");
+
+    // Extract and count only the secondary section's lines
+    const secondaryMarker = "### Test output\n";
+    const secondaryIdx = result!.indexOf(secondaryMarker);
+    expect(secondaryIdx).toBeGreaterThanOrEqual(0);
+    const secondaryContent = result!.slice(secondaryIdx + secondaryMarker.length);
+    const secondaryLines = secondaryContent.split("\n").filter((l) => l.length > 0);
+
+    expect(secondaryLines.length).toBeLessThanOrEqual(100);
+  });
+});

src/__tests__/unit/lib/github/pr-monitor.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, beforeEach, vi } from "vitest";
-import { mergeBaseBranch, rebaseOntoBaseBranch, triggerAgentModeFix, triggerLiveModeFix, createPrLogger, monitorSinglePR, claimPRFixInProgress } from "@/lib/github/pr-monitor";
+import { mergeBaseBranch, rebaseOntoBaseBranch, triggerAgentModeFix, triggerLiveModeFix, createPrLogger, monitorSinglePR, buildFixPrompt, claimPRFixInProgress } from "@/lib/github/pr-monitor";
 import type { Octokit } from "@octokit/rest";
 import { ChatRole, ChatStatus } from "@prisma/client";
 
@@ -1940,6 +1940,53 @@ describe("monitorSinglePR", () => {
   });
 });
 
+// ─────────────────────────────────────────────────────────────────────────────
+// buildFixPrompt
+// ─────────────────────────────────────────────────────────────────────────────
+
+describe("buildFixPrompt", () => {
+  const base = {
+    artifactId: "artifact-1",
+    taskId: "task-1",
+    prNumber: 42,
+    owner: "acme",
+    repo: "app",
+    headBranch: "feat/x",
+    baseBranch: "main",
+    state: "ci_failure" as const,
+    mergeable: true,
+    ciStatus: "failure" as const,
+    prState: "open" as const,
+    merged: false,
+    failedChecks: ["CI / test"],
+    problemDetails: "",
+    failedCheckLogs: {},
+  };
+
+  it("does NOT contain the playwright timeout hint in ci_failure output", () => {
+    const result = buildFixPrompt(base);
+    expect(result).not.toContain("playwright");
+    expect(result).not.toContain("increase playwright timeouts");
+    expect(result).not.toContain("github CI can be slow sometimes");
+  });
+
+  it("contains all three diagnostic checklist points in ci_failure output", () => {
+    const result = buildFixPrompt(base);
+    expect(result).toContain("Read the full failing test file");
+    expect(result).toContain("Determine whether the failure is a broken test assertion");
+    expect(result).toContain("Only modify the test if the assertion itself is genuinely incorrect");
+  });
+
+  it("diagnostic checklist appears BEFORE the Please: action directive", () => {
+    const result = buildFixPrompt(base);
+    const diagnosticIdx = result.indexOf("Before making any changes:");
+    const pleaseIdx = result.indexOf("Please:");
+    expect(diagnosticIdx).toBeGreaterThanOrEqual(0);
+    expect(pleaseIdx).toBeGreaterThanOrEqual(0);
+    expect(diagnosticIdx).toBeLessThan(pleaseIdx);
+  });
+});
+
 // ─── CAS claim: claimPRFixInProgress unit tests ────────────────────────────────
 
 describe("claimPRFixInProgress", () => {