diff --git a/lib/src/scan/lang/extractors/android-resources.test.ts b/lib/src/scan/lang/extractors/android-resources.test.ts index e880325..86dc7e1 100644 --- a/lib/src/scan/lang/extractors/android-resources.test.ts +++ b/lib/src/scan/lang/extractors/android-resources.test.ts @@ -65,3 +65,69 @@ describe("androidResourceExtractor", () => { expect(welcome?.context.identifiers).toEqual(["welcome"]); }); }); + +// The main path. An earlier version of this fix only reached the CDATA +// sweep and silently did nothing here. +describe("androidResourceExtractor escape decoding", () => { + const extract = (source: string) => + androidResourceExtractor.extract({ source, kind: "android_resources" }); + + it("decodes escapes in a ", async () => { + const hits = await extract( + `line one\\nline two` + ); + expect(hits.map((h) => h.value)).toEqual(["line one\nline two"]); + }); + + it("decodes escapes inside items", async () => { + const hits = await extract( + ` + + one\\nfile + many\\nfiles + + ` + ); + expect(hits.map((h) => h.value)).toEqual(["one\nfile", "many\nfiles"]); + }); +}); + +// Ditto wraps placeholders in . Only the first text child used to survive, +// so the copy stopped at the first placeholder and every variable disappeared. +describe("androidResourceExtractor inline markup", () => { + const extract = (source: string) => androidResourceExtractor.extract({ source, kind: "android_resources" }); + + it("keeps the text and the specifiers around an ", async () => { + const hits = await extract( + ` + We sent it to %1$s. It expires in %2$s minutes. + ` + ); + expect(hits.map((h) => h.value)).toEqual(["We sent it to %1$s. It expires in %2$s minutes."]); + }); + + it("keeps each plural item distinct", async () => { + const hits = await extract( + ` + + Expires in %1$s minute. + Expires in %1$s minutes. + + ` + ); + expect(hits.map((h) => ({ v: h.value, ids: h.context.identifiers }))).toEqual([ + { v: "Expires in %1$s minute.", ids: ["hint", "one"] }, + { v: "Expires in %1$s minutes.", ids: ["hint", "other"] }, + ]); + }); + + it("keeps the words inside an inline style tag", async () => { + const hits = await extract(`Please do not share it`); + expect(hits.map((h) => h.value)).toEqual(["Please do not share it"]); + }); + + it("decodes XML entities", async () => { + const hits = await extract(`<- Back & forth`); + expect(hits.map((h) => h.value)).toEqual(["<- Back & forth"]); + }); +}); diff --git a/lib/src/scan/lang/extractors/android-resources.ts b/lib/src/scan/lang/extractors/android-resources.ts index ae874ff..0284d94 100644 --- a/lib/src/scan/lang/extractors/android-resources.ts +++ b/lib/src/scan/lang/extractors/android-resources.ts @@ -1,7 +1,7 @@ import { Lang, parse, type SgNode } from "@ast-grep/napi"; import type { ExtractedHit, LanguageExtractor } from "../types"; -import { offsetToLineCol } from "./util"; +import { decodeEscapes, offsetToLineCol } from "./util"; import { elementAttribute, emitTextHit, findCdataElements, tagName } from "./xml"; /** @@ -38,7 +38,7 @@ export const androidResourceExtractor: LanguageExtractor = { if (child.kind() !== "element" || tagName(child) !== "item") continue; const variant = tag === "plurals" ? elementAttribute(child, "quantity") ?? "" : String(index); // The resource name is the lookup key; quantity/index are selectors. - emitTextHit(child, [parentName, variant], out, source, parentName || undefined); + emitTextHit(child, [parentName, variant], out, source, parentName || undefined, decodeEscapes); index++; } } @@ -61,7 +61,7 @@ function emitCdataValues(source: string, out: ExtractedHit[]): void { if (m.tag === "item" && !parent) continue; const { line, column } = offsetToLineCol(source, m.valueOffset); out.push({ - value: m.value, + value: decodeEscapes(m.value), location: { line, column }, context: { parentRole: "resource_value", @@ -108,5 +108,5 @@ function findEnclosingResourceParent(source: string, before: number): string | n function emitStringElement(el: SgNode, out: ExtractedHit[], source: string): void { const name = elementAttribute(el, "name"); - emitTextHit(el, name ? [name] : [], out, source, name ?? undefined); + emitTextHit(el, name ? [name] : [], out, source, name ?? undefined, decodeEscapes); } diff --git a/lib/src/scan/lang/extractors/javascript.test.ts b/lib/src/scan/lang/extractors/javascript.test.ts index 2fd4468..c4d1ffe 100644 --- a/lib/src/scan/lang/extractors/javascript.test.ts +++ b/lib/src/scan/lang/extractors/javascript.test.ts @@ -74,3 +74,34 @@ describe("javascriptExtractor", () => { expect(inner?.context.parentTag).toBeUndefined(); }); }); + +// Escapes are decoded for real JS literals only. Getting this wrong either ships +// a literal `\n` to users or mangles JSX text, so each branch is pinned. +describe("javascriptExtractor escape decoding", () => { + test("decodes escapes in a plain string literal", async () => { + const hits = await ts.extract({ source: `const m = "line one\\nline two";\n`, kind: "typescript" }); + expect(hits[0].value).toBe('"line one\nline two"'); + }); + + test("decodes escapes in a template literal", async () => { + const hits = await ts.extract({ source: "const m = `line one\\nline two`;\n", kind: "typescript" }); + expect(hits[0].value).toBe("`line one\nline two`"); + }); + + test("leaves jsx text alone — a backslash there is two literal characters", async () => { + const hits = await tsx.extract({ source: `const C = () =>

line one\\nline two

;\n`, kind: "tsx" }); + expect(hits.find((h) => h.context.parentRole === "markup_text")?.value).toBe("line one\\nline two"); + }); + + test("leaves a bare jsx attribute alone but decodes the braced form", async () => { + const bare = await tsx.extract({ source: `const C = () => ;\n`, kind: "tsx" }); + expect(bare[0].value).toBe('"a\\nb"'); + const braced = await tsx.extract({ source: `const C = () => ;\n`, kind: "tsx" }); + expect(braced[0].value).toBe('"a\nb"'); + }); + + test("does not convert format specifiers — a bare %s in JS is usually copy", async () => { + const hits = await ts.extract({ source: `const m = "Battery at %s";\n`, kind: "typescript" }); + expect(hits[0].value).toBe('"Battery at %s"'); + }); +}); diff --git a/lib/src/scan/lang/extractors/javascript.ts b/lib/src/scan/lang/extractors/javascript.ts index 36b781c..aeb6180 100644 --- a/lib/src/scan/lang/extractors/javascript.ts +++ b/lib/src/scan/lang/extractors/javascript.ts @@ -2,6 +2,7 @@ import { Lang, parse, type SgNode } from "@ast-grep/napi"; import type { DittoScanEnclosingContext } from "../../types"; import type { ExtractedHit, LanguageExtractor } from "../types"; +import { decodeEscapes } from "./util"; import { extractHtmlMarkup } from "./html-markup"; /** @@ -99,20 +100,18 @@ function keyName(node: SgNode | undefined): string | null { } /** - * Returns the raw source text for the node, or null if it should be - * skipped. The only thing we skip here is whitespace-only `jsx_text`. - * tree-sitter emits a `jsx_text` node for the literal whitespace between - * elements (the newline and indent between `

` and `` in - * pretty-printed JSX), and those are not real candidates. - * - * We deliberately do not strip quotes, unescape `\n`, or normalize - * `${...}` substitutions. The LLM reads `source_context` and handles raw - * source fine, and a clean post-processing pass would just be lossy. + * Source text for the node, or null for whitespace-only `jsx_text`. Escapes are + * decoded (left raw, an exporter escapes them again) but only for real JS + * literals — a backslash isn't an escape in `jsx_text` or a bare JSX attribute. */ function nodeValue(node: SgNode): string | null { const text = node.text(); - if (node.kind() === "jsx_text" && text.trim().length === 0) return null; - return text; + if (node.kind() === "jsx_text") { + return text.trim().length === 0 ? null : text; + } + // A bare JSX attribute is HTML-shaped: a backslash isn't an escape there. + if (node.parent()?.kind() === "jsx_attribute") return text; + return decodeEscapes(text); } /** diff --git a/lib/src/scan/lang/extractors/kotlin.test.ts b/lib/src/scan/lang/extractors/kotlin.test.ts index 77fb628..cb680f3 100644 --- a/lib/src/scan/lang/extractors/kotlin.test.ts +++ b/lib/src/scan/lang/extractors/kotlin.test.ts @@ -53,3 +53,21 @@ describe("kotlinExtractor", () => { expect(c?.context.callee).toBeUndefined(); }); }); + +// From a pilot bug report: the escape shipped to users as the literal text `\n`. +describe("kotlinExtractor escape decoding", () => { + const extract = (source: string) => kotlinExtractor.extract({ source, kind: "kotlin" }); + + it("decodes escapes in a Compose string", async () => { + const hits = await extract( + 'fun S() { Text(text = "We need your phone number to provide\\nstatus updates") }' + ); + expect(hits[0].value).toBe('"We need your phone number to provide\nstatus updates"'); + }); + + // Raw strings don't process escapes, so a `\n` in one is two real characters. + it("leaves a raw string's escapes alone", async () => { + const hits = await extract('val s = """a\\nb"""'); + expect(hits[0].value).toBe('"""a\\nb"""'); + }); +}); diff --git a/lib/src/scan/lang/extractors/kotlin.ts b/lib/src/scan/lang/extractors/kotlin.ts index 0411fb9..3823150 100644 --- a/lib/src/scan/lang/extractors/kotlin.ts +++ b/lib/src/scan/lang/extractors/kotlin.ts @@ -2,6 +2,7 @@ import { parse, type SgNode } from "@ast-grep/napi"; import type { DittoScanEnclosingContext } from "../../types"; import type { ExtractedHit, LanguageExtractor } from "../types"; +import { decodeEscapes } from "./util"; /** * Kotlin string extractor. Tree-sitter's Kotlin grammar models all @@ -22,8 +23,11 @@ export const kotlinExtractor: LanguageExtractor = { // visited as their own top-level hit by the same `findAll` scan. if (isInsideInterpolation(node)) continue; const { line, column } = node.range().start; + const text = node.text(); + // Raw strings (`"""..."""`) don't process escapes. + const decoded = text.startsWith('"""') ? text : decodeEscapes(text); out.push({ - value: node.text(), + value: decoded, location: { line: line + 1, column: column + 1 }, context: classifyKtParent(node), }); diff --git a/lib/src/scan/lang/extractors/resx.test.ts b/lib/src/scan/lang/extractors/resx.test.ts new file mode 100644 index 0000000..b8d0a5e --- /dev/null +++ b/lib/src/scan/lang/extractors/resx.test.ts @@ -0,0 +1,16 @@ +import { resxExtractor } from "./resx"; + +const extract = (source: string) => + resxExtractor.extract({ source, kind: "resx" }); + +// .resx shares `emitTextHit` with the Android extractor, which decodes escapes. +// XML has no backslash escapes, so .resx passes no transform and keeps values +// exactly as written. +describe("resxExtractor leaves values verbatim", () => { + test("does not decode a backslash sequence", async () => { + const hits = await extract( + `line one\\nline two` + ); + expect(hits[0].value).toBe("line one\\nline two"); + }); +}); diff --git a/lib/src/scan/lang/extractors/strings.test.ts b/lib/src/scan/lang/extractors/strings.test.ts index a14a788..126294b 100644 --- a/lib/src/scan/lang/extractors/strings.test.ts +++ b/lib/src/scan/lang/extractors/strings.test.ts @@ -36,10 +36,11 @@ describe("stringsExtractor (.strings)", () => { expect(hits.map((h) => h.context.identifiers[0])).toEqual(["a", "b"]); }); - test("preserves escape sequences verbatim in value", async () => { + // Decoded, not verbatim: left raw, an exporter escapes it again and ships `\n`. + test("decodes escape sequences in value", async () => { const hits = await extract(`"k" = "line1\\nline2 with \\"quote\\"";\n`); expect(hits).toHaveLength(1); - expect(hits[0].value).toBe('line1\\nline2 with \\"quote\\"'); + expect(hits[0].value).toBe('line1\nline2 with "quote"'); }); test("returns no hits for malformed input", async () => { diff --git a/lib/src/scan/lang/extractors/strings.ts b/lib/src/scan/lang/extractors/strings.ts index 38226eb..9274e49 100644 --- a/lib/src/scan/lang/extractors/strings.ts +++ b/lib/src/scan/lang/extractors/strings.ts @@ -1,5 +1,5 @@ import type { ExtractedHit, LanguageExtractor } from "../types"; -import { offsetToLineCol } from "./util"; +import { decodeEscapes, offsetToLineCol } from "./util"; /** * iOS `.strings` files (Localizable.strings, InfoPlist.strings, etc.). @@ -17,7 +17,7 @@ export const stringsExtractor: LanguageExtractor = { const out: ExtractedHit[] = []; for (const pair of parseStringsFile(source)) { out.push({ - value: pair.value, + value: decodeEscapes(pair.value), location: pair.location, context: { parentRole: "resource_value", identifiers: [pair.key] }, i18nKey: pair.key, diff --git a/lib/src/scan/lang/extractors/swift.test.ts b/lib/src/scan/lang/extractors/swift.test.ts index e272f0d..0b26c38 100644 --- a/lib/src/scan/lang/extractors/swift.test.ts +++ b/lib/src/scan/lang/extractors/swift.test.ts @@ -53,3 +53,16 @@ describe("swiftExtractor", () => { }); }); }); + +describe("swiftExtractor escape decoding", () => { + test("decodes escapes", async () => { + const hits = await extract(`let a = "line one\\nline two"\n`); + expect(hits[0].value).toBe('"line one\nline two"'); + }); + + // Raw strings (`#"..."#`) don't process escapes. + test("leaves a raw string's escapes alone", async () => { + const hits = await extract(`let a = #"a\\nb"#\n`); + expect(hits[0].value).toBe('#"a\\nb"#'); + }); +}); diff --git a/lib/src/scan/lang/extractors/swift.ts b/lib/src/scan/lang/extractors/swift.ts index b5bc4cb..a1c9c10 100644 --- a/lib/src/scan/lang/extractors/swift.ts +++ b/lib/src/scan/lang/extractors/swift.ts @@ -2,6 +2,7 @@ import { parse, type SgNode } from "@ast-grep/napi"; import type { DittoScanEnclosingContext } from "../../types"; import type { ExtractedHit, LanguageExtractor } from "../types"; +import { decodeEscapes } from "./util"; /** * Swift string-literal node kinds. @@ -28,8 +29,11 @@ export const swiftExtractor: LanguageExtractor = { // inner shows up as part of its source text. if (isInsideInterpolation(node)) continue; const { line, column } = node.range().start; + const text = node.text(); + // Raw strings (`#"..."#`) don't process escapes. + const decoded = text.startsWith("#") ? text : decodeEscapes(text); out.push({ - value: node.text(), + value: decoded, location: { line: line + 1, column: column + 1 }, context: classifySwiftParent(node), }); diff --git a/lib/src/scan/lang/extractors/util.test.ts b/lib/src/scan/lang/extractors/util.test.ts new file mode 100644 index 0000000..630f94a --- /dev/null +++ b/lib/src/scan/lang/extractors/util.test.ts @@ -0,0 +1,32 @@ +import { decodeEscapes } from "./util"; + +describe("decodeEscapes", () => { + it("decodes newlines, tabs and quotes", () => { + expect(decodeEscapes("line one\\nline two")).toBe("line one\nline two"); + expect(decodeEscapes("a\\tb")).toBe("a\tb"); + expect(decodeEscapes('say \\"hi\\"')).toBe('say "hi"'); + expect(decodeEscapes("it\\'s")).toBe("it's"); + }); + + it("decodes unicode escapes", () => { + expect(decodeEscapes("5 \\u2605")).toBe("5 ★"); + }); + + // A doubled backslash means the author wanted a real backslash, so it must not + // become a newline. + it("keeps an intentional literal backslash-n", () => { + expect(decodeEscapes("use \\\\n for a break")).toBe("use \\n for a break"); + }); + + // Android escapes `?` and `@` on export, because a resource value that starts + // with one is a reference. The copy holds the bare character. + it("drops the backslash before escaped punctuation", () => { + expect(decodeEscapes("Forgot Password\\?")).toBe("Forgot Password?"); + expect(decodeEscapes("\\@ home")).toBe("@ home"); + expect(decodeEscapes("100\\% sure")).toBe("100% sure"); + }); + + it("leaves an escaped letter alone", () => { + expect(decodeEscapes("match \\d digits")).toBe("match \\d digits"); + }); +}); diff --git a/lib/src/scan/lang/extractors/util.ts b/lib/src/scan/lang/extractors/util.ts index 7c73a95..3035d9d 100644 --- a/lib/src/scan/lang/extractors/util.ts +++ b/lib/src/scan/lang/extractors/util.ts @@ -1,3 +1,22 @@ +// Only for formats that process escapes. A doubled backslash resolves last, so +// `\\n` stays a backslash and an `n`. +export function decodeEscapes(text: string): string { + return text.replace(/\\(u[0-9a-fA-F]{4}|[nrt]|[^A-Za-z0-9])/g, (_, seq: string) => { + switch (seq[0]) { + case "n": + return "\n"; + case "r": + return "\r"; + case "t": + return "\t"; + case "u": + return String.fromCharCode(parseInt(seq.slice(1), 16)); + default: + return seq; // \\ ' " ` ? @ % … + } + }); +} + // 1-based (line, column) from a byte offset into a UTF-16 source string. // Treats LF as the line break; CR-only sources will report a single line. export function offsetToLineCol(source: string, offset: number): { line: number; column: number } { diff --git a/lib/src/scan/lang/extractors/xml.ts b/lib/src/scan/lang/extractors/xml.ts index 983abcb..5c89ee1 100644 --- a/lib/src/scan/lang/extractors/xml.ts +++ b/lib/src/scan/lang/extractors/xml.ts @@ -35,34 +35,60 @@ export function elementAttribute(element: SgNode, name: string): string | null { return null; } -// Emit a `resource_value` hit from `element`'s first text child. No-op when -// the element has no direct text node, the text is whitespace-only, or the +// Emit a `resource_value` hit from all of `element`'s inner text. No-op when +// the element has no inner text, the text is whitespace-only, or the // element contains a CDATA section (handled by `findCdataElements`). The // HTML grammar parses CDATA inconsistently when its body looks like markup // — `x]]>` can surface `]]` as a stray text node. // Skipping any element whose source range contains ` string ): void { if (source !== undefined && elementContainsCdata(element, source)) return; - const text = element.children().find((c) => c.kind() === "text"); - if (!text) return; - const value = text.text(); - if (value.trim().length === 0) return; - const range = text.range(); + const inner = innerText(element); + if (!inner || inner.value.length === 0) return; + const { value, line, column } = inner; out.push({ - value, - location: { line: range.start.line + 1, column: range.start.column + 1 }, + value: transformValue ? transformValue(value) : value, + location: { line, column }, context: { parentRole: "resource_value", identifiers }, i18nKey, }); } +// All of an element's inner text, with nested markup (``, ``) stripped. +// Reads the source span: the grammar drops the whitespace next to a nested tag. +function innerText(element: SgNode): { value: string; line: number; column: number } | null { + const children = element.children(); + const start = children.find((c) => c.kind() === "start_tag"); + const end = children.find((c) => c.kind() === "end_tag"); + if (!start || !end) return null; + const offset = element.range().start.index; + const raw = element.text().slice(start.range().end.index - offset, end.range().start.index - offset); + const { line, column } = start.range().end; + return { value: decodeXmlEntities(raw.replace(/<[^>]*>/g, "")).trim(), line: line + 1, column: column + 1 }; +} + +// `&` resolves last, so an escaped entity like `&lt;` stays `<`. +function decodeXmlEntities(text: string): string { + return text + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) => String.fromCodePoint(parseInt(hex, 16))) + .replace(/&#(\d+);/g, (_, dec: string) => String.fromCodePoint(parseInt(dec, 10))) + .replace(/&/g, "&"); +} + function elementContainsCdata(element: SgNode, source: string): boolean { const range = element.range(); const idx = source.indexOf("