[DIT-13447] Introduce escape decoding on extractor level

lib/src/scan/lang/extractors/android-resources.test.ts

@@ -65,3 +65,69 @@ describe("androidResourceExtractor", () => {
     expect(welcome?.context.identifiers).toEqual(["welcome"]);
   });
 });
+
+// The main <string> path. An earlier version of this fix only reached the CDATA
+// sweep and silently did nothing here.
+describe("androidResourceExtractor escape decoding", () => {
+  const extract = (source: string) =>
+    androidResourceExtractor.extract({ source, kind: "android_resources" });
+
+  it("decodes escapes in a <string>", async () => {
+    const hits = await extract(
+      `<resources><string name="a">line one\\nline two</string></resources>`
+    );
+    expect(hits.map((h) => h.value)).toEqual(["line one\nline two"]);
+  });
+
+  it("decodes escapes inside <plurals> items", async () => {
+    const hits = await extract(
+      `<resources>
+         <plurals name="n">
+           <item quantity="one">one\\nfile</item>
+           <item quantity="other">many\\nfiles</item>
+         </plurals>
+       </resources>`
+    );
+    expect(hits.map((h) => h.value)).toEqual(["one\nfile", "many\nfiles"]);
+  });
+});
+
+// Ditto wraps placeholders in <xliff:g>. Only the first text child used to survive,
+// so the copy stopped at the first placeholder and every variable disappeared.
+describe("androidResourceExtractor inline markup", () => {
+  const extract = (source: string) => androidResourceExtractor.extract({ source, kind: "android_resources" });
+
+  it("keeps the text and the specifiers around an <xliff:g>", async () => {
+    const hits = await extract(
+      `<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+         <string name="hint">We sent it to <xliff:g id="phone" example="(555) 555-555">%1$s</xliff:g>. It expires in <xliff:g id="mins" example="10">%2$s</xliff:g> minutes.</string>
+       </resources>`
+    );
+    expect(hits.map((h) => h.value)).toEqual(["We sent it to %1$s. It expires in %2$s minutes."]);
+  });
+
+  it("keeps each plural item distinct", async () => {
+    const hits = await extract(
+      `<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+         <plurals name="hint">
+           <item quantity="one">Expires in <xliff:g id="m">%1$s</xliff:g> minute.</item>
+           <item quantity="other">Expires in <xliff:g id="m">%1$s</xliff:g> minutes.</item>
+         </plurals>
+       </resources>`
+    );
+    expect(hits.map((h) => ({ v: h.value, ids: h.context.identifiers }))).toEqual([
+      { v: "Expires in %1$s minute.", ids: ["hint", "one"] },
+      { v: "Expires in %1$s minutes.", ids: ["hint", "other"] },
+    ]);
+  });
+
+  it("keeps the words inside an inline style tag", async () => {
+    const hits = await extract(`<resources><string name="a">Please <b>do not</b> share it</string></resources>`);
+    expect(hits.map((h) => h.value)).toEqual(["Please do not share it"]);
+  });
+
+  it("decodes XML entities", async () => {
+    const hits = await extract(`<resources><string name="a">&lt;- Back &amp; forth</string></resources>`);
+    expect(hits.map((h) => h.value)).toEqual(["<- Back & forth"]);
+  });
+});

lib/src/scan/lang/extractors/android-resources.ts

@@ -1,7 +1,7 @@
 import { Lang, parse, type SgNode } from "@ast-grep/napi";
 
 import type { ExtractedHit, LanguageExtractor } from "../types";
-import { offsetToLineCol } from "./util";
+import { decodeEscapes, offsetToLineCol } from "./util";
 import { elementAttribute, emitTextHit, findCdataElements, tagName } from "./xml";
 
 /**
@@ -38,7 +38,7 @@ export const androidResourceExtractor: LanguageExtractor = {
           if (child.kind() !== "element" || tagName(child) !== "item") continue;
           const variant = tag === "plurals" ? elementAttribute(child, "quantity") ?? "" : String(index);
           // The resource name is the lookup key; quantity/index are selectors.
-          emitTextHit(child, [parentName, variant], out, source, parentName || undefined);
+          emitTextHit(child, [parentName, variant], out, source, parentName || undefined, decodeEscapes);
           index++;
         }
       }
@@ -61,7 +61,7 @@ function emitCdataValues(source: string, out: ExtractedHit[]): void {
     if (m.tag === "item" && !parent) continue;
     const { line, column } = offsetToLineCol(source, m.valueOffset);
     out.push({
-      value: m.value,
+      value: decodeEscapes(m.value),
       location: { line, column },
       context: {
         parentRole: "resource_value",
@@ -108,5 +108,5 @@ function findEnclosingResourceParent(source: string, before: number): string | n
 
 function emitStringElement(el: SgNode, out: ExtractedHit[], source: string): void {
   const name = elementAttribute(el, "name");
-  emitTextHit(el, name ? [name] : [], out, source, name ?? undefined);
+  emitTextHit(el, name ? [name] : [], out, source, name ?? undefined, decodeEscapes);
 }

lib/src/scan/lang/extractors/javascript.test.ts

@@ -74,3 +74,34 @@ describe("javascriptExtractor", () => {
     expect(inner?.context.parentTag).toBeUndefined();
   });
 });
+
+// Escapes are decoded for real JS literals only. Getting this wrong either ships
+// a literal `\n` to users or mangles JSX text, so each branch is pinned.
+describe("javascriptExtractor escape decoding", () => {
+  test("decodes escapes in a plain string literal", async () => {
+    const hits = await ts.extract({ source: `const m = "line one\\nline two";\n`, kind: "typescript" });
+    expect(hits[0].value).toBe('"line one\nline two"');
+  });
+
+  test("decodes escapes in a template literal", async () => {
+    const hits = await ts.extract({ source: "const m = `line one\\nline two`;\n", kind: "typescript" });
+    expect(hits[0].value).toBe("`line one\nline two`");
+  });
+
+  test("leaves jsx text alone — a backslash there is two literal characters", async () => {
+    const hits = await tsx.extract({ source: `const C = () => <p>line one\\nline two</p>;\n`, kind: "tsx" });
+    expect(hits.find((h) => h.context.parentRole === "markup_text")?.value).toBe("line one\\nline two");
+  });
+
+  test("leaves a bare jsx attribute alone but decodes the braced form", async () => {
+    const bare = await tsx.extract({ source: `const C = () => <input placeholder="a\\nb" />;\n`, kind: "tsx" });
+    expect(bare[0].value).toBe('"a\\nb"');
+    const braced = await tsx.extract({ source: `const C = () => <input placeholder={"a\\nb"} />;\n`, kind: "tsx" });
+    expect(braced[0].value).toBe('"a\nb"');
+  });
+
+  test("does not convert format specifiers — a bare %s in JS is usually copy", async () => {
+    const hits = await ts.extract({ source: `const m = "Battery at %s";\n`, kind: "typescript" });
+    expect(hits[0].value).toBe('"Battery at %s"');
+  });
+});

lib/src/scan/lang/extractors/javascript.ts

@@ -2,6 +2,7 @@ import { Lang, parse, type SgNode } from "@ast-grep/napi";
 
 import type { DittoScanEnclosingContext } from "../../types";
 import type { ExtractedHit, LanguageExtractor } from "../types";
+import { decodeEscapes } from "./util";
 import { extractHtmlMarkup } from "./html-markup";
 
 /**
@@ -99,20 +100,18 @@ function keyName(node: SgNode | undefined): string | null {
 }
 
 /**
- * Returns the raw source text for the node, or null if it should be
- * skipped. The only thing we skip here is whitespace-only `jsx_text`.
- * tree-sitter emits a `jsx_text` node for the literal whitespace between
- * elements (the newline and indent between `<p>` and `<span>` in
- * pretty-printed JSX), and those are not real candidates.
- *
- * We deliberately do not strip quotes, unescape `\n`, or normalize
- * `${...}` substitutions. The LLM reads `source_context` and handles raw
- * source fine, and a clean post-processing pass would just be lossy.
+ * Source text for the node, or null for whitespace-only `jsx_text`. Escapes are
+ * decoded (left raw, an exporter escapes them again) but only for real JS
+ * literals — a backslash isn't an escape in `jsx_text` or a bare JSX attribute.
  */
 function nodeValue(node: SgNode): string | null {
   const text = node.text();
-  if (node.kind() === "jsx_text" && text.trim().length === 0) return null;
-  return text;
+  if (node.kind() === "jsx_text") {
+    return text.trim().length === 0 ? null : text;
+  }
+  // A bare JSX attribute is HTML-shaped: a backslash isn't an escape there.
+  if (node.parent()?.kind() === "jsx_attribute") return text;
+  return decodeEscapes(text);
 }
 
 /**

lib/src/scan/lang/extractors/kotlin.test.ts

@@ -53,3 +53,21 @@ describe("kotlinExtractor", () => {
     expect(c?.context.callee).toBeUndefined();
   });
 });
+
+// From a pilot bug report: the escape shipped to users as the literal text `\n`.
+describe("kotlinExtractor escape decoding", () => {
+  const extract = (source: string) => kotlinExtractor.extract({ source, kind: "kotlin" });
+
+  it("decodes escapes in a Compose string", async () => {
+    const hits = await extract(
+      'fun S() { Text(text = "We need your phone number to provide\\nstatus updates") }'
+    );
+    expect(hits[0].value).toBe('"We need your phone number to provide\nstatus updates"');
+  });
+
+  // Raw strings don't process escapes, so a `\n` in one is two real characters.
+  it("leaves a raw string's escapes alone", async () => {
+    const hits = await extract('val s = """a\\nb"""');
+    expect(hits[0].value).toBe('"""a\\nb"""');
+  });
+});

lib/src/scan/lang/extractors/kotlin.ts

@@ -2,6 +2,7 @@ import { parse, type SgNode } from "@ast-grep/napi";
 
 import type { DittoScanEnclosingContext } from "../../types";
 import type { ExtractedHit, LanguageExtractor } from "../types";
+import { decodeEscapes } from "./util";
 
 /**
  * Kotlin string extractor. Tree-sitter's Kotlin grammar models all
@@ -22,8 +23,11 @@ export const kotlinExtractor: LanguageExtractor = {
       // visited as their own top-level hit by the same `findAll` scan.
       if (isInsideInterpolation(node)) continue;
       const { line, column } = node.range().start;
+      const text = node.text();
+      // Raw strings (`"""..."""`) don't process escapes.
+      const decoded = text.startsWith('"""') ? text : decodeEscapes(text);
       out.push({
-        value: node.text(),
+        value: decoded,
         location: { line: line + 1, column: column + 1 },
         context: classifyKtParent(node),
       });

lib/src/scan/lang/extractors/resx.test.ts

@@ -0,0 +1,16 @@
+import { resxExtractor } from "./resx";
+
+const extract = (source: string) =>
+  resxExtractor.extract({ source, kind: "resx" });
+
+// .resx shares `emitTextHit` with the Android extractor, which decodes escapes.
+// XML has no backslash escapes, so .resx passes no transform and keeps values
+// exactly as written.
+describe("resxExtractor leaves values verbatim", () => {
+  test("does not decode a backslash sequence", async () => {
+    const hits = await extract(
+      `<root><data name="a"><value>line one\\nline two</value></data></root>`
+    );
+    expect(hits[0].value).toBe("line one\\nline two");
+  });
+});

lib/src/scan/lang/extractors/strings.test.ts

@@ -36,10 +36,11 @@ describe("stringsExtractor (.strings)", () => {
     expect(hits.map((h) => h.context.identifiers[0])).toEqual(["a", "b"]);
   });
 
-  test("preserves escape sequences verbatim in value", async () => {
+  // Decoded, not verbatim: left raw, an exporter escapes it again and ships `\n`.
+  test("decodes escape sequences in value", async () => {
     const hits = await extract(`"k" = "line1\\nline2 with \\"quote\\"";\n`);
     expect(hits).toHaveLength(1);
-    expect(hits[0].value).toBe('line1\\nline2 with \\"quote\\"');
+    expect(hits[0].value).toBe('line1\nline2 with "quote"');
   });
 
   test("returns no hits for malformed input", async () => {

lib/src/scan/lang/extractors/strings.ts

@@ -1,5 +1,5 @@
 import type { ExtractedHit, LanguageExtractor } from "../types";
-import { offsetToLineCol } from "./util";
+import { decodeEscapes, offsetToLineCol } from "./util";
 
 /**
  * iOS `.strings` files (Localizable.strings, InfoPlist.strings, etc.).
@@ -17,7 +17,7 @@ export const stringsExtractor: LanguageExtractor = {
     const out: ExtractedHit[] = [];
     for (const pair of parseStringsFile(source)) {
       out.push({
-        value: pair.value,
+        value: decodeEscapes(pair.value),
         location: pair.location,
         context: { parentRole: "resource_value", identifiers: [pair.key] },
         i18nKey: pair.key,

lib/src/scan/lang/extractors/swift.test.ts

@@ -53,3 +53,16 @@ describe("swiftExtractor", () => {
     });
   });
 });
+
+describe("swiftExtractor escape decoding", () => {
+  test("decodes escapes", async () => {
+    const hits = await extract(`let a = "line one\\nline two"\n`);
+    expect(hits[0].value).toBe('"line one\nline two"');
+  });
+
+  // Raw strings (`#"..."#`) don't process escapes.
+  test("leaves a raw string's escapes alone", async () => {
+    const hits = await extract(`let a = #"a\\nb"#\n`);
+    expect(hits[0].value).toBe('#"a\\nb"#');
+  });
+});

lib/src/scan/lang/extractors/swift.ts

@@ -2,6 +2,7 @@ import { parse, type SgNode } from "@ast-grep/napi";
 
 import type { DittoScanEnclosingContext } from "../../types";
 import type { ExtractedHit, LanguageExtractor } from "../types";
+import { decodeEscapes } from "./util";
 
 /**
  * Swift string-literal node kinds.
@@ -28,8 +29,11 @@ export const swiftExtractor: LanguageExtractor = {
         // inner shows up as part of its source text.
         if (isInsideInterpolation(node)) continue;
         const { line, column } = node.range().start;
+        const text = node.text();
+        // Raw strings (`#"..."#`) don't process escapes.
+        const decoded = text.startsWith("#") ? text : decodeEscapes(text);
         out.push({
-          value: node.text(),
+          value: decoded,
           location: { line: line + 1, column: column + 1 },
           context: classifySwiftParent(node),
         });

lib/src/scan/lang/extractors/util.test.ts

@@ -0,0 +1,32 @@
+import { decodeEscapes } from "./util";
+
+describe("decodeEscapes", () => {
+  it("decodes newlines, tabs and quotes", () => {
+    expect(decodeEscapes("line one\\nline two")).toBe("line one\nline two");
+    expect(decodeEscapes("a\\tb")).toBe("a\tb");
+    expect(decodeEscapes('say \\"hi\\"')).toBe('say "hi"');
+    expect(decodeEscapes("it\\'s")).toBe("it's");
+  });
+
+  it("decodes unicode escapes", () => {
+    expect(decodeEscapes("5 \\u2605")).toBe("5 ★");
+  });
+
+  // A doubled backslash means the author wanted a real backslash, so it must not
+  // become a newline.
+  it("keeps an intentional literal backslash-n", () => {
+    expect(decodeEscapes("use \\\\n for a break")).toBe("use \\n for a break");
+  });
+
+  // Android escapes `?` and `@` on export, because a resource value that starts
+  // with one is a reference. The copy holds the bare character.
+  it("drops the backslash before escaped punctuation", () => {
+    expect(decodeEscapes("Forgot Password\\?")).toBe("Forgot Password?");
+    expect(decodeEscapes("\\@ home")).toBe("@ home");
+    expect(decodeEscapes("100\\% sure")).toBe("100% sure");
+  });
+
+  it("leaves an escaped letter alone", () => {
+    expect(decodeEscapes("match \\d digits")).toBe("match \\d digits");
+  });
+});

lib/src/scan/lang/extractors/util.ts

@@ -1,3 +1,22 @@
+// Only for formats that process escapes. A doubled backslash resolves last, so
+// `\\n` stays a backslash and an `n`.
+export function decodeEscapes(text: string): string {
+  return text.replace(/\\(u[0-9a-fA-F]{4}|[nrt]|[^A-Za-z0-9])/g, (_, seq: string) => {
+    switch (seq[0]) {
+      case "n":
+        return "\n";
+      case "r":
+        return "\r";
+      case "t":
+        return "\t";
+      case "u":
+        return String.fromCharCode(parseInt(seq.slice(1), 16));
+      default:
+        return seq; // \\ ' " ` ? @ % …
+    }
+  });
+}
+
 // 1-based (line, column) from a byte offset into a UTF-16 source string.
 // Treats LF as the line break; CR-only sources will report a single line.
 export function offsetToLineCol(source: string, offset: number): { line: number; column: number } {