From e9e22fec13501ecc25181a9a59d3ec4ffdad75f2 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Sat, 4 Jul 2026 06:05:03 +0200
Subject: [PATCH] fix(results): keep raw transcripts valid jsonl

---
 .../commands/eval/artifact-writer.test.ts     | 26 +++++++++-
 packages/core/src/evaluation/run-artifacts.ts | 48 ++++++++++++++++++-
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts
index 63ba0233e..b739185ab 100644
--- a/apps/cli/test/commands/eval/artifact-writer.test.ts
+++ b/apps/cli/test/commands/eval/artifact-writer.test.ts
@@ -1938,7 +1938,31 @@ describe('writeArtifactsFromResults', () => {
     await expect(readFile(copiedRawLogPath, 'utf8')).rejects.toThrow();
 
     const transcriptPath = runArtifactPath(testDir, indexLine, 'sample-1', 'transcript-raw.jsonl');
-    await expect(readFile(transcriptPath, 'utf8')).resolves.toBe(rawLog);
+    const rawTranscriptLines = (await readFile(transcriptPath, 'utf8'))
+      .trim()
+      .split('\n')
+      .map((line) => JSON.parse(line));
+    expect(rawTranscriptLines).toEqual([
+      {
+        schema_version: 'agentv.raw_provider_log_line.v1',
+        source: {
+          kind: 'provider_log',
+          format: 'text',
+          line_index: 0,
+        },
+        content: '# provider-native stream log',
+      },
+      {
+        schema_version: 'agentv.raw_provider_log_line.v1',
+        source: {
+          kind: 'provider_log',
+          format: 'text',
+          line_index: 1,
+        },
+        content:
+          '{"time":"00:00","data":{"camelCaseProviderKey":true,"toolInput":{"filePath":"src/index.ts"}}}',
+      },
+    ]);
     await expect(readFile(rawLogPath, 'utf8')).resolves.toBe(rawLog);
     await expect(
       readFile(path.join(testDir, rowDir, 'sample-1', 'transcript.jsonl'), 'utf8'),
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index b5a30f1db..9eafb7983 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -2471,13 +2471,59 @@ async function writeRawTranscriptJsonl(
 ): Promise<void> {
   const rawSource = rawProviderLogSourcePath(result);
   if (rawSource) {
-    await copyFile(rawSource, filePath);
+    await copyRawProviderLogAsJsonl(rawSource, filePath);
     await cleanupProviderStagingFile(rawSource).catch(() => undefined);
     return;
   }
   await writeGeneratedRawTranscriptJsonl(filePath, result, envelope);
 }
 
+async function copyRawProviderLogAsJsonl(sourcePath: string, filePath: string): Promise<void> {
+  const content = await readFile(sourcePath, 'utf8');
+  if (isJsonlContent(content)) {
+    await copyFile(sourcePath, filePath);
+    return;
+  }
+
+  const lines = splitLogLines(content);
+  const records = lines.map((line, index) => ({
+    schema_version: 'agentv.raw_provider_log_line.v1',
+    source: {
+      kind: 'provider_log',
+      format: 'text',
+      line_index: index,
+    },
+    content: line,
+  }));
+  await writeJsonlFile(filePath, records);
+}
+
+function isJsonlContent(content: string): boolean {
+  const lines = content.split(/\r?\n/).filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return true;
+  }
+  return lines.every((line) => {
+    try {
+      JSON.parse(line);
+      return true;
+    } catch {
+      return false;
+    }
+  });
+}
+
+function splitLogLines(content: string): string[] {
+  if (content.length === 0) {
+    return [];
+  }
+  const lines = content.replace(/\r\n/g, '\n').split('\n');
+  if (lines.at(-1) === '') {
+    lines.pop();
+  }
+  return lines;
+}
+
 function buildMetricsArtifactPayload(params: {
   readonly result: EvaluationResult;
   readonly envelope: TraceEnvelope;