From bbfb7175c7f66a90167277787df1984ce9a3a8db Mon Sep 17 00:00:00 2001 From: Joey Holiga Date: Wed, 12 Aug 2026 14:30:31 -0400 Subject: [PATCH 1/2] Address decoding issues for extracted escaped values --- .../lang/extractors/android-resources.test.ts | 26 ++++++++++++++++ .../scan/lang/extractors/android-resources.ts | 8 ++--- .../scan/lang/extractors/javascript.test.ts | 31 +++++++++++++++++++ lib/src/scan/lang/extractors/javascript.ts | 21 ++++++------- lib/src/scan/lang/extractors/kotlin.test.ts | 18 +++++++++++ lib/src/scan/lang/extractors/kotlin.ts | 6 +++- lib/src/scan/lang/extractors/resx.test.ts | 16 ++++++++++ lib/src/scan/lang/extractors/strings.test.ts | 5 +-- lib/src/scan/lang/extractors/strings.ts | 4 +-- lib/src/scan/lang/extractors/swift.test.ts | 13 ++++++++ lib/src/scan/lang/extractors/swift.ts | 6 +++- lib/src/scan/lang/extractors/util.test.ts | 24 ++++++++++++++ lib/src/scan/lang/extractors/util.ts | 21 +++++++++++++ lib/src/scan/lang/extractors/xml.ts | 7 +++-- package.json | 2 +- 15 files changed, 184 insertions(+), 24 deletions(-) create mode 100644 lib/src/scan/lang/extractors/resx.test.ts create mode 100644 lib/src/scan/lang/extractors/util.test.ts diff --git a/lib/src/scan/lang/extractors/android-resources.test.ts b/lib/src/scan/lang/extractors/android-resources.test.ts index e8803254..e82296b0 100644 --- a/lib/src/scan/lang/extractors/android-resources.test.ts +++ b/lib/src/scan/lang/extractors/android-resources.test.ts @@ -65,3 +65,29 @@ describe("androidResourceExtractor", () => { expect(welcome?.context.identifiers).toEqual(["welcome"]); }); }); + +// The main path. An earlier version of this fix only reached the CDATA +// sweep and silently did nothing here. +describe("androidResourceExtractor escape decoding", () => { + const extract = (source: string) => + androidResourceExtractor.extract({ source, kind: "android_resources" }); + + it("decodes escapes in a ", async () => { + const hits = await extract( + `line one\\nline two` + ); + expect(hits.map((h) => h.value)).toEqual(["line one\nline two"]); + }); + + it("decodes escapes inside items", async () => { + const hits = await extract( + ` + + one\\nfile + many\\nfiles + + ` + ); + expect(hits.map((h) => h.value)).toEqual(["one\nfile", "many\nfiles"]); + }); +}); diff --git a/lib/src/scan/lang/extractors/android-resources.ts b/lib/src/scan/lang/extractors/android-resources.ts index ae874ff8..0284d947 100644 --- a/lib/src/scan/lang/extractors/android-resources.ts +++ b/lib/src/scan/lang/extractors/android-resources.ts @@ -1,7 +1,7 @@ import { Lang, parse, type SgNode } from "@ast-grep/napi"; import type { ExtractedHit, LanguageExtractor } from "../types"; -import { offsetToLineCol } from "./util"; +import { decodeEscapes, offsetToLineCol } from "./util"; import { elementAttribute, emitTextHit, findCdataElements, tagName } from "./xml"; /** @@ -38,7 +38,7 @@ export const androidResourceExtractor: LanguageExtractor = { if (child.kind() !== "element" || tagName(child) !== "item") continue; const variant = tag === "plurals" ? elementAttribute(child, "quantity") ?? "" : String(index); // The resource name is the lookup key; quantity/index are selectors. - emitTextHit(child, [parentName, variant], out, source, parentName || undefined); + emitTextHit(child, [parentName, variant], out, source, parentName || undefined, decodeEscapes); index++; } } @@ -61,7 +61,7 @@ function emitCdataValues(source: string, out: ExtractedHit[]): void { if (m.tag === "item" && !parent) continue; const { line, column } = offsetToLineCol(source, m.valueOffset); out.push({ - value: m.value, + value: decodeEscapes(m.value), location: { line, column }, context: { parentRole: "resource_value", @@ -108,5 +108,5 @@ function findEnclosingResourceParent(source: string, before: number): string | n function emitStringElement(el: SgNode, out: ExtractedHit[], source: string): void { const name = elementAttribute(el, "name"); - emitTextHit(el, name ? [name] : [], out, source, name ?? undefined); + emitTextHit(el, name ? [name] : [], out, source, name ?? undefined, decodeEscapes); } diff --git a/lib/src/scan/lang/extractors/javascript.test.ts b/lib/src/scan/lang/extractors/javascript.test.ts index 2fd44683..c4d1ffe5 100644 --- a/lib/src/scan/lang/extractors/javascript.test.ts +++ b/lib/src/scan/lang/extractors/javascript.test.ts @@ -74,3 +74,34 @@ describe("javascriptExtractor", () => { expect(inner?.context.parentTag).toBeUndefined(); }); }); + +// Escapes are decoded for real JS literals only. Getting this wrong either ships +// a literal `\n` to users or mangles JSX text, so each branch is pinned. +describe("javascriptExtractor escape decoding", () => { + test("decodes escapes in a plain string literal", async () => { + const hits = await ts.extract({ source: `const m = "line one\\nline two";\n`, kind: "typescript" }); + expect(hits[0].value).toBe('"line one\nline two"'); + }); + + test("decodes escapes in a template literal", async () => { + const hits = await ts.extract({ source: "const m = `line one\\nline two`;\n", kind: "typescript" }); + expect(hits[0].value).toBe("`line one\nline two`"); + }); + + test("leaves jsx text alone — a backslash there is two literal characters", async () => { + const hits = await tsx.extract({ source: `const C = () =>

line one\\nline two

;\n`, kind: "tsx" }); + expect(hits.find((h) => h.context.parentRole === "markup_text")?.value).toBe("line one\\nline two"); + }); + + test("leaves a bare jsx attribute alone but decodes the braced form", async () => { + const bare = await tsx.extract({ source: `const C = () => ;\n`, kind: "tsx" }); + expect(bare[0].value).toBe('"a\\nb"'); + const braced = await tsx.extract({ source: `const C = () => ;\n`, kind: "tsx" }); + expect(braced[0].value).toBe('"a\nb"'); + }); + + test("does not convert format specifiers — a bare %s in JS is usually copy", async () => { + const hits = await ts.extract({ source: `const m = "Battery at %s";\n`, kind: "typescript" }); + expect(hits[0].value).toBe('"Battery at %s"'); + }); +}); diff --git a/lib/src/scan/lang/extractors/javascript.ts b/lib/src/scan/lang/extractors/javascript.ts index 36b781c2..aeb6180e 100644 --- a/lib/src/scan/lang/extractors/javascript.ts +++ b/lib/src/scan/lang/extractors/javascript.ts @@ -2,6 +2,7 @@ import { Lang, parse, type SgNode } from "@ast-grep/napi"; import type { DittoScanEnclosingContext } from "../../types"; import type { ExtractedHit, LanguageExtractor } from "../types"; +import { decodeEscapes } from "./util"; import { extractHtmlMarkup } from "./html-markup"; /** @@ -99,20 +100,18 @@ function keyName(node: SgNode | undefined): string | null { } /** - * Returns the raw source text for the node, or null if it should be - * skipped. The only thing we skip here is whitespace-only `jsx_text`. - * tree-sitter emits a `jsx_text` node for the literal whitespace between - * elements (the newline and indent between `

` and `` in - * pretty-printed JSX), and those are not real candidates. - * - * We deliberately do not strip quotes, unescape `\n`, or normalize - * `${...}` substitutions. The LLM reads `source_context` and handles raw - * source fine, and a clean post-processing pass would just be lossy. + * Source text for the node, or null for whitespace-only `jsx_text`. Escapes are + * decoded (left raw, an exporter escapes them again) but only for real JS + * literals — a backslash isn't an escape in `jsx_text` or a bare JSX attribute. */ function nodeValue(node: SgNode): string | null { const text = node.text(); - if (node.kind() === "jsx_text" && text.trim().length === 0) return null; - return text; + if (node.kind() === "jsx_text") { + return text.trim().length === 0 ? null : text; + } + // A bare JSX attribute is HTML-shaped: a backslash isn't an escape there. + if (node.parent()?.kind() === "jsx_attribute") return text; + return decodeEscapes(text); } /** diff --git a/lib/src/scan/lang/extractors/kotlin.test.ts b/lib/src/scan/lang/extractors/kotlin.test.ts index 77fb6280..cb680f3a 100644 --- a/lib/src/scan/lang/extractors/kotlin.test.ts +++ b/lib/src/scan/lang/extractors/kotlin.test.ts @@ -53,3 +53,21 @@ describe("kotlinExtractor", () => { expect(c?.context.callee).toBeUndefined(); }); }); + +// From a pilot bug report: the escape shipped to users as the literal text `\n`. +describe("kotlinExtractor escape decoding", () => { + const extract = (source: string) => kotlinExtractor.extract({ source, kind: "kotlin" }); + + it("decodes escapes in a Compose string", async () => { + const hits = await extract( + 'fun S() { Text(text = "We need your phone number to provide\\nstatus updates") }' + ); + expect(hits[0].value).toBe('"We need your phone number to provide\nstatus updates"'); + }); + + // Raw strings don't process escapes, so a `\n` in one is two real characters. + it("leaves a raw string's escapes alone", async () => { + const hits = await extract('val s = """a\\nb"""'); + expect(hits[0].value).toBe('"""a\\nb"""'); + }); +}); diff --git a/lib/src/scan/lang/extractors/kotlin.ts b/lib/src/scan/lang/extractors/kotlin.ts index 0411fb92..38231500 100644 --- a/lib/src/scan/lang/extractors/kotlin.ts +++ b/lib/src/scan/lang/extractors/kotlin.ts @@ -2,6 +2,7 @@ import { parse, type SgNode } from "@ast-grep/napi"; import type { DittoScanEnclosingContext } from "../../types"; import type { ExtractedHit, LanguageExtractor } from "../types"; +import { decodeEscapes } from "./util"; /** * Kotlin string extractor. Tree-sitter's Kotlin grammar models all @@ -22,8 +23,11 @@ export const kotlinExtractor: LanguageExtractor = { // visited as their own top-level hit by the same `findAll` scan. if (isInsideInterpolation(node)) continue; const { line, column } = node.range().start; + const text = node.text(); + // Raw strings (`"""..."""`) don't process escapes. + const decoded = text.startsWith('"""') ? text : decodeEscapes(text); out.push({ - value: node.text(), + value: decoded, location: { line: line + 1, column: column + 1 }, context: classifyKtParent(node), }); diff --git a/lib/src/scan/lang/extractors/resx.test.ts b/lib/src/scan/lang/extractors/resx.test.ts new file mode 100644 index 00000000..b8d0a5e4 --- /dev/null +++ b/lib/src/scan/lang/extractors/resx.test.ts @@ -0,0 +1,16 @@ +import { resxExtractor } from "./resx"; + +const extract = (source: string) => + resxExtractor.extract({ source, kind: "resx" }); + +// .resx shares `emitTextHit` with the Android extractor, which decodes escapes. +// XML has no backslash escapes, so .resx passes no transform and keeps values +// exactly as written. +describe("resxExtractor leaves values verbatim", () => { + test("does not decode a backslash sequence", async () => { + const hits = await extract( + `line one\\nline two` + ); + expect(hits[0].value).toBe("line one\\nline two"); + }); +}); diff --git a/lib/src/scan/lang/extractors/strings.test.ts b/lib/src/scan/lang/extractors/strings.test.ts index a14a7883..126294bc 100644 --- a/lib/src/scan/lang/extractors/strings.test.ts +++ b/lib/src/scan/lang/extractors/strings.test.ts @@ -36,10 +36,11 @@ describe("stringsExtractor (.strings)", () => { expect(hits.map((h) => h.context.identifiers[0])).toEqual(["a", "b"]); }); - test("preserves escape sequences verbatim in value", async () => { + // Decoded, not verbatim: left raw, an exporter escapes it again and ships `\n`. + test("decodes escape sequences in value", async () => { const hits = await extract(`"k" = "line1\\nline2 with \\"quote\\"";\n`); expect(hits).toHaveLength(1); - expect(hits[0].value).toBe('line1\\nline2 with \\"quote\\"'); + expect(hits[0].value).toBe('line1\nline2 with "quote"'); }); test("returns no hits for malformed input", async () => { diff --git a/lib/src/scan/lang/extractors/strings.ts b/lib/src/scan/lang/extractors/strings.ts index 38226eb8..9274e491 100644 --- a/lib/src/scan/lang/extractors/strings.ts +++ b/lib/src/scan/lang/extractors/strings.ts @@ -1,5 +1,5 @@ import type { ExtractedHit, LanguageExtractor } from "../types"; -import { offsetToLineCol } from "./util"; +import { decodeEscapes, offsetToLineCol } from "./util"; /** * iOS `.strings` files (Localizable.strings, InfoPlist.strings, etc.). @@ -17,7 +17,7 @@ export const stringsExtractor: LanguageExtractor = { const out: ExtractedHit[] = []; for (const pair of parseStringsFile(source)) { out.push({ - value: pair.value, + value: decodeEscapes(pair.value), location: pair.location, context: { parentRole: "resource_value", identifiers: [pair.key] }, i18nKey: pair.key, diff --git a/lib/src/scan/lang/extractors/swift.test.ts b/lib/src/scan/lang/extractors/swift.test.ts index e272f0dc..0b26c388 100644 --- a/lib/src/scan/lang/extractors/swift.test.ts +++ b/lib/src/scan/lang/extractors/swift.test.ts @@ -53,3 +53,16 @@ describe("swiftExtractor", () => { }); }); }); + +describe("swiftExtractor escape decoding", () => { + test("decodes escapes", async () => { + const hits = await extract(`let a = "line one\\nline two"\n`); + expect(hits[0].value).toBe('"line one\nline two"'); + }); + + // Raw strings (`#"..."#`) don't process escapes. + test("leaves a raw string's escapes alone", async () => { + const hits = await extract(`let a = #"a\\nb"#\n`); + expect(hits[0].value).toBe('#"a\\nb"#'); + }); +}); diff --git a/lib/src/scan/lang/extractors/swift.ts b/lib/src/scan/lang/extractors/swift.ts index b5bc4cb4..a1c9c10b 100644 --- a/lib/src/scan/lang/extractors/swift.ts +++ b/lib/src/scan/lang/extractors/swift.ts @@ -2,6 +2,7 @@ import { parse, type SgNode } from "@ast-grep/napi"; import type { DittoScanEnclosingContext } from "../../types"; import type { ExtractedHit, LanguageExtractor } from "../types"; +import { decodeEscapes } from "./util"; /** * Swift string-literal node kinds. @@ -28,8 +29,11 @@ export const swiftExtractor: LanguageExtractor = { // inner shows up as part of its source text. if (isInsideInterpolation(node)) continue; const { line, column } = node.range().start; + const text = node.text(); + // Raw strings (`#"..."#`) don't process escapes. + const decoded = text.startsWith("#") ? text : decodeEscapes(text); out.push({ - value: node.text(), + value: decoded, location: { line: line + 1, column: column + 1 }, context: classifySwiftParent(node), }); diff --git a/lib/src/scan/lang/extractors/util.test.ts b/lib/src/scan/lang/extractors/util.test.ts new file mode 100644 index 00000000..939f68b7 --- /dev/null +++ b/lib/src/scan/lang/extractors/util.test.ts @@ -0,0 +1,24 @@ +import { decodeEscapes } from "./util"; + +describe("decodeEscapes", () => { + it("decodes newlines, tabs and quotes", () => { + expect(decodeEscapes("line one\\nline two")).toBe("line one\nline two"); + expect(decodeEscapes("a\\tb")).toBe("a\tb"); + expect(decodeEscapes('say \\"hi\\"')).toBe('say "hi"'); + expect(decodeEscapes("it\\'s")).toBe("it's"); + }); + + it("decodes unicode escapes", () => { + expect(decodeEscapes("5 \\u2605")).toBe("5 ★"); + }); + + // A doubled backslash means the author wanted a real backslash, so it must not + // become a newline. + it("keeps an intentional literal backslash-n", () => { + expect(decodeEscapes("use \\\\n for a break")).toBe("use \\n for a break"); + }); + + it("leaves unrecognized escapes alone", () => { + expect(decodeEscapes("100\\% sure")).toBe("100\\% sure"); + }); +}); diff --git a/lib/src/scan/lang/extractors/util.ts b/lib/src/scan/lang/extractors/util.ts index 7c73a95e..e2574348 100644 --- a/lib/src/scan/lang/extractors/util.ts +++ b/lib/src/scan/lang/extractors/util.ts @@ -1,3 +1,24 @@ +// Only for formats that process escapes. A doubled backslash resolves last, so +// `\\n` stays a backslash and an `n`. +export function decodeEscapes(text: string): string { + return text.replace(/\\(u[0-9a-fA-F]{4}|[nrt'"`\\])/g, (_, seq: string) => { + switch (seq[0]) { + case "n": + return "\n"; + case "r": + return "\r"; + case "t": + return "\t"; + case "u": + return String.fromCharCode(parseInt(seq.slice(1), 16)); + case "\\": + return "\\"; + default: + return seq; // ' " ` + } + }); +} + // 1-based (line, column) from a byte offset into a UTF-16 source string. // Treats LF as the line break; CR-only sources will report a single line. export function offsetToLineCol(source: string, offset: number): { line: number; column: number } { diff --git a/lib/src/scan/lang/extractors/xml.ts b/lib/src/scan/lang/extractors/xml.ts index 983abcbf..67eb85ac 100644 --- a/lib/src/scan/lang/extractors/xml.ts +++ b/lib/src/scan/lang/extractors/xml.ts @@ -42,12 +42,15 @@ export function elementAttribute(element: SgNode, name: string): string | null { // — `x]]>` can surface `]]` as a stray text node. // Skipping any element whose source range contains ` string ): void { if (source !== undefined && elementContainsCdata(element, source)) return; const text = element.children().find((c) => c.kind() === "text"); @@ -56,7 +59,7 @@ export function emitTextHit( if (value.trim().length === 0) return; const range = text.range(); out.push({ - value, + value: transformValue ? transformValue(value) : value, location: { line: range.start.line + 1, column: range.start.column + 1 }, context: { parentRole: "resource_value", identifiers }, i18nKey, diff --git a/package.json b/package.json index cfc72466..dafd4795 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dittowords/cli", - "version": "5.7.0", + "version": "5.7.1", "description": "Command Line Interface for Ditto (dittowords.com).", "license": "MIT", "main": "bin/ditto.js", From 725e7b62ef7f0c076897eb1de9b677b1a6df2468 Mon Sep 17 00:00:00 2001 From: Joey Holiga Date: Wed, 12 Aug 2026 21:05:41 -0400 Subject: [PATCH 2/2] improvements to escaping punctuation / xml specifically for Android repos --- .../lang/extractors/android-resources.test.ts | 40 +++++++++++++++++++ lib/src/scan/lang/extractors/util.test.ts | 12 +++++- lib/src/scan/lang/extractors/util.ts | 6 +-- lib/src/scan/lang/extractors/xml.ts | 39 ++++++++++++++---- 4 files changed, 83 insertions(+), 14 deletions(-) diff --git a/lib/src/scan/lang/extractors/android-resources.test.ts b/lib/src/scan/lang/extractors/android-resources.test.ts index e82296b0..86dc7e1b 100644 --- a/lib/src/scan/lang/extractors/android-resources.test.ts +++ b/lib/src/scan/lang/extractors/android-resources.test.ts @@ -91,3 +91,43 @@ describe("androidResourceExtractor escape decoding", () => { expect(hits.map((h) => h.value)).toEqual(["one\nfile", "many\nfiles"]); }); }); + +// Ditto wraps placeholders in . Only the first text child used to survive, +// so the copy stopped at the first placeholder and every variable disappeared. +describe("androidResourceExtractor inline markup", () => { + const extract = (source: string) => androidResourceExtractor.extract({ source, kind: "android_resources" }); + + it("keeps the text and the specifiers around an ", async () => { + const hits = await extract( + ` + We sent it to %1$s. It expires in %2$s minutes. + ` + ); + expect(hits.map((h) => h.value)).toEqual(["We sent it to %1$s. It expires in %2$s minutes."]); + }); + + it("keeps each plural item distinct", async () => { + const hits = await extract( + ` + + Expires in %1$s minute. + Expires in %1$s minutes. + + ` + ); + expect(hits.map((h) => ({ v: h.value, ids: h.context.identifiers }))).toEqual([ + { v: "Expires in %1$s minute.", ids: ["hint", "one"] }, + { v: "Expires in %1$s minutes.", ids: ["hint", "other"] }, + ]); + }); + + it("keeps the words inside an inline style tag", async () => { + const hits = await extract(`Please do not share it`); + expect(hits.map((h) => h.value)).toEqual(["Please do not share it"]); + }); + + it("decodes XML entities", async () => { + const hits = await extract(`<- Back & forth`); + expect(hits.map((h) => h.value)).toEqual(["<- Back & forth"]); + }); +}); diff --git a/lib/src/scan/lang/extractors/util.test.ts b/lib/src/scan/lang/extractors/util.test.ts index 939f68b7..630f94af 100644 --- a/lib/src/scan/lang/extractors/util.test.ts +++ b/lib/src/scan/lang/extractors/util.test.ts @@ -18,7 +18,15 @@ describe("decodeEscapes", () => { expect(decodeEscapes("use \\\\n for a break")).toBe("use \\n for a break"); }); - it("leaves unrecognized escapes alone", () => { - expect(decodeEscapes("100\\% sure")).toBe("100\\% sure"); + // Android escapes `?` and `@` on export, because a resource value that starts + // with one is a reference. The copy holds the bare character. + it("drops the backslash before escaped punctuation", () => { + expect(decodeEscapes("Forgot Password\\?")).toBe("Forgot Password?"); + expect(decodeEscapes("\\@ home")).toBe("@ home"); + expect(decodeEscapes("100\\% sure")).toBe("100% sure"); + }); + + it("leaves an escaped letter alone", () => { + expect(decodeEscapes("match \\d digits")).toBe("match \\d digits"); }); }); diff --git a/lib/src/scan/lang/extractors/util.ts b/lib/src/scan/lang/extractors/util.ts index e2574348..3035d9d0 100644 --- a/lib/src/scan/lang/extractors/util.ts +++ b/lib/src/scan/lang/extractors/util.ts @@ -1,7 +1,7 @@ // Only for formats that process escapes. A doubled backslash resolves last, so // `\\n` stays a backslash and an `n`. export function decodeEscapes(text: string): string { - return text.replace(/\\(u[0-9a-fA-F]{4}|[nrt'"`\\])/g, (_, seq: string) => { + return text.replace(/\\(u[0-9a-fA-F]{4}|[nrt]|[^A-Za-z0-9])/g, (_, seq: string) => { switch (seq[0]) { case "n": return "\n"; @@ -11,10 +11,8 @@ export function decodeEscapes(text: string): string { return "\t"; case "u": return String.fromCharCode(parseInt(seq.slice(1), 16)); - case "\\": - return "\\"; default: - return seq; // ' " ` + return seq; // \\ ' " ` ? @ % … } }); } diff --git a/lib/src/scan/lang/extractors/xml.ts b/lib/src/scan/lang/extractors/xml.ts index 67eb85ac..5c89ee13 100644 --- a/lib/src/scan/lang/extractors/xml.ts +++ b/lib/src/scan/lang/extractors/xml.ts @@ -35,8 +35,8 @@ export function elementAttribute(element: SgNode, name: string): string | null { return null; } -// Emit a `resource_value` hit from `element`'s first text child. No-op when -// the element has no direct text node, the text is whitespace-only, or the +// Emit a `resource_value` hit from all of `element`'s inner text. No-op when +// the element has no inner text, the text is whitespace-only, or the // element contains a CDATA section (handled by `findCdataElements`). The // HTML grammar parses CDATA inconsistently when its body looks like markup // — `x]]>` can surface `]]` as a stray text node. @@ -53,19 +53,42 @@ export function emitTextHit( transformValue?: (value: string) => string ): void { if (source !== undefined && elementContainsCdata(element, source)) return; - const text = element.children().find((c) => c.kind() === "text"); - if (!text) return; - const value = text.text(); - if (value.trim().length === 0) return; - const range = text.range(); + const inner = innerText(element); + if (!inner || inner.value.length === 0) return; + const { value, line, column } = inner; out.push({ value: transformValue ? transformValue(value) : value, - location: { line: range.start.line + 1, column: range.start.column + 1 }, + location: { line, column }, context: { parentRole: "resource_value", identifiers }, i18nKey, }); } +// All of an element's inner text, with nested markup (``, ``) stripped. +// Reads the source span: the grammar drops the whitespace next to a nested tag. +function innerText(element: SgNode): { value: string; line: number; column: number } | null { + const children = element.children(); + const start = children.find((c) => c.kind() === "start_tag"); + const end = children.find((c) => c.kind() === "end_tag"); + if (!start || !end) return null; + const offset = element.range().start.index; + const raw = element.text().slice(start.range().end.index - offset, end.range().start.index - offset); + const { line, column } = start.range().end; + return { value: decodeXmlEntities(raw.replace(/<[^>]*>/g, "")).trim(), line: line + 1, column: column + 1 }; +} + +// `&` resolves last, so an escaped entity like `&lt;` stays `<`. +function decodeXmlEntities(text: string): string { + return text + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) => String.fromCodePoint(parseInt(hex, 16))) + .replace(/&#(\d+);/g, (_, dec: string) => String.fromCodePoint(parseInt(dec, 10))) + .replace(/&/g, "&"); +} + function elementContainsCdata(element: SgNode, source: string): boolean { const range = element.range(); const idx = source.indexOf("