From eee7bb3168a2d6e9de365084101abc83bb0ea034 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Fri, 16 Feb 2024 15:18:11 -0800 Subject: [PATCH 01/30] Add more test cases --- assembly/__tests__/as-json.spec.ts | 236 +++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) diff --git a/assembly/__tests__/as-json.spec.ts b/assembly/__tests__/as-json.spec.ts index 0b16c691..9a9cb836 100644 --- a/assembly/__tests__/as-json.spec.ts +++ b/assembly/__tests__/as-json.spec.ts @@ -73,9 +73,20 @@ describe("Ser/de Numbers", () => { canSerde(10e2, "1000.0"); canSerde(123456e-5, "1.23456"); + canSerde(0.0, "0.0"); canSerde(0.0, "0.0"); canSerde(7.23, "7.23"); + + canSerde(1e-6, "0.000001"); + canSerde(1e-7, "1e-7"); + canDeser("1E-7", 1e-7); + + canSerde(1e20, "100000000000000000000.0"); + canSerde(1e21, "1e+21"); + canDeser("1E+21", 1e21); + canDeser("1e21", 1e21); + canDeser("1E21", 1e21); }); it("should ser/de booleans", () => { @@ -97,6 +108,11 @@ describe("Ser/de Array", () => { it("should ser/de float arrays", () => { canSerde([7.23, 10e2, 10e2, 123456e-5, 123456e-5, 0.0, 7.23]); + + canSerde([1e21,1e22,1e-7,1e-8,1e-9], "[1e+21,1e+22,1e-7,1e-8,1e-9]"); + canDeser("[1E+21,1E+22,1E-7,1E-8,1E-9]", [1e21,1e22,1e-7,1e-8,1e-9]); + canDeser("[1e21,1e22,1e-7,1e-8,1e-9]", [1e21,1e22,1e-7,1e-8,1e-9]); + canDeser("[1E21,1E22,1E-7,1E-8,1E-9]", [1e21,1e22,1e-7,1e-8,1e-9]); }); it("should ser/de boolean arrays", () => { @@ -167,6 +183,38 @@ describe("Ser/de Objects", () => { isVerified: true, }, '{"firstName":"Emmet","lastName":"West","lastActive":[8,27,2022],"age":23,"pos":{"x":3.4,"y":1.2,"z":8.3},"isVerified":true}'); }); + + it("should ser/de object with floats", () => { + canSerde({ f: 7.23 }, '{"f":7.23}'); + canSerde({ f: 0.000001 }, '{"f":0.000001}'); + + canSerde({ f: 1e-7 }, '{"f":1e-7}'); + canDeser('{"f":1E-7}', { f: 1e-7 }); + + canSerde({ f: 1e20 }, '{"f":100000000000000000000.0}'); + canSerde({ f: 1e21 }, '{"f":1e+21}'); + canDeser('{"f":1E+21}', { f: 1e21 }); + canDeser('{"f":1e21}', { f: 1e21 }); + }); + + it("should ser/de object with float arrays", () => { + canSerde( + { fa: [1e21,1e22,1e-7,1e-8,1e-9] }, + '{"fa":[1e+21,1e+22,1e-7,1e-8,1e-9]}'); + + canDeser( + '{"fa":[1E+21,1E+22,1E-7,1E-8,1E-9]}', + { fa: [1e21,1e22,1e-7,1e-8,1e-9] }); + + canDeser( + '{"fa":[1e21,1e22,1e-7,1e-8,1e-9]}', + { fa: [1e21,1e22,1e-7,1e-8,1e-9] }); + + canDeser( + '{"fa":[1E21,1E22,1E-7,1E-8,1E-9]}', + { fa: [1e21,1e22,1e-7,1e-8,1e-9] }); + + }); }); describe("Ser externals", () => { @@ -343,3 +391,191 @@ describe("Ser/de Maps", () => { }); }); + +describe("Ser/de escape sequences in strings", () => { + it("should encode short escape sequences", () => { + canSer("\\", '"\\\\"'); + canSer('"', '"\\""'); + canSer("\n", '"\\n"'); + canSer("\r", '"\\r"'); + canSer("\t", '"\\t"'); + canSer("\b", '"\\b"'); + canSer("\f", '"\\f"'); + }); + + it("should decode short escape sequences", () => { + canDeser('"\\\\"', "\\"); + canDeser('"\\""', '"'); + canDeser('"\\n"', "\n"); + canDeser('"\\r"', "\r"); + canDeser('"\\t"', "\t"); + canDeser('"\\b"', "\b"); + canDeser('"\\f"', "\f"); + }); + + it("should decode escaped forward slash but not encode", () => { + canSer("/", '"/"'); + canDeser('"/"', "/"); + canDeser('"\\/"', "/"); // allowed + }); + + // 0x00 - 0x1f, excluding characters that have short escape sequences + it("should encode long escape sequences", () => { + const singles = ["\n", "\r", "\t", "\b", "\f"]; + for (let i = 0; i < 0x1F; i++) { + const c = String.fromCharCode(i); + if (singles.includes(c)) continue; + const actual = JSON.stringify(c); + const expected = `"\\u${i.toString(16).padStart(4, "0")}"`; + expect(actual).toBe(expected, `Failed to encode '\\x${i.toString(16).padStart(2, "0")}'`); + } + }); + + // \u0000 - \u001f + it("should decode long escape sequences (lower cased)", () => { + for (let i = 0; i <= 0x1f; i++) { + const s = `"\\u${i.toString(16).padStart(4, "0").toLowerCase()}"`; + const actual = JSON.parse(s); + const expected = String.fromCharCode(i); + expect(actual).toBe(expected, `Failed to decode ${s}`); + } + }); + + // \u0000 - \u001F + it("should decode long escape sequences (upper cased)", () => { + for (let i = 0; i <= 0x1f; i++) { + const s = `"\\u${i.toString(16).padStart(4, "0").toUpperCase()}"`; + const actual = JSON.parse(s); + const expected = String.fromCharCode(i); + expect(actual).toBe(expected, `Failed to decode ${s}`); + } + }); + + // See https://datatracker.ietf.org/doc/html/rfc8259#section-7 + it("should decode UTF-16 surrogate pairs", () => { + const s = '"\\uD834\\uDD1E"'; + const actual = JSON.parse(s); + const expected = "𝄞"; + expect(actual).toBe(expected); + }); + + // Just because we can decode UTF-16 surrogate pairs, doesn't mean we should encode them. + it("should not encode UTF-16 surrogate pairs", () => { + const s = "𝄞"; + const actual = JSON.stringify(s); + const expected = '"𝄞"'; + expect(actual).toBe(expected); + }); + + it("should encode multiple escape sequences", () => { + canSer('"""', '"\\"\\"\\""'); + canSer('\\\\\\', '"\\\\\\\\\\\\"'); + }); + +}); + +describe("Ser/de special strings in object", () => { + it("should serialize quotes in string in object", () => { + const o: ObjWithString = { s: '"""' }; + const s = '{"s":"\\"\\"\\""}'; + canSer(o, s); + }); + it("should deserialize quotes in string in object", () => { + const o: ObjWithString = { s: '"""' }; + const s = '{"s":"\\"\\"\\""}'; + canDeser(s, o); + }); + it("should serialize backslashes in string in object", () => { + const o: ObjWithString = { s: "\\\\\\" }; + const s = '{"s":"\\\\\\\\\\\\"}'; + canSer(o, s); + }); + it("should deserialize backslashes in string in object", () => { + const o: ObjWithString = { s: "\\\\\\" }; + const s = '{"s":"\\\\\\\\\\\\"}'; + canDeser(s, o); + }); + + it("should ser/de short escape sequences in strings in objects", () => { + const o: ObjWithString = { s: "\n\r\t\b\f" }; + const s = '{"s":"\\n\\r\\t\\b\\f"}'; + canSerde(o, s); + }); + + it("should ser/de short escape sequences in string arrays", () => { + const a = ["\n", "\r", "\t", "\b", "\f"]; + const s = '["\\n","\\r","\\t","\\b","\\f"]'; + canSerde(a, s); + }); + + it("should ser/de short escape sequences in string arrays in objects", () => { + const o: ObjectWithStringArray = { sa: ["\n", "\r", "\t", "\b", "\f"] }; + const s = '{"sa":["\\n","\\r","\\t","\\b","\\f"]}'; + canSerde(o, s); + }); + + it("should ser/de long escape sequences in strings in objects", () => { + const singles = ["\n", "\r", "\t", "\b", "\f"]; + let x = ""; + let y = ""; + for (let i = 0; i < 0x1F; i++) { + const c = String.fromCharCode(i); + if (singles.includes(c)) continue; + x += c; + y += `\\u${i.toString(16).padStart(4, "0")}`; + } + const o: ObjWithString = { s: x }; + const s = `{"s":"${y}"}`; + canSerde(o, s); + }); + + it("should ser/de long escape sequences in strings in arrays", () => { + const singles = ["\n", "\r", "\t", "\b", "\f"]; + let x: string[] = []; + let y: string[] = []; + for (let i = 0; i < 0x1F; i++) { + const c = String.fromCharCode(i); + if (singles.includes(c)) continue; + x.push(c); + y.push(`\\u${i.toString(16).padStart(4, "0")}`); + } + const a = x; + const s = `["${y.join('","')}"]`; + canSerde(a, s); + }); + + it("should ser/de long escape sequences in string arrays in objects", () => { + const singles = ["\n", "\r", "\t", "\b", "\f"]; + let x: string[] = []; + let y: string[] = []; + for (let i = 0; i < 0x1F; i++) { + const c = String.fromCharCode(i); + if (singles.includes(c)) continue; + x.push(c); + y.push(`\\u${i.toString(16).padStart(4, "0")}`); + } + const o: ObjectWithStringArray = { sa: x }; + const s = `{"sa":["${y.join('","')}"]}`; + canSerde(o, s); + }); +}); + +@json +class ObjWithString { + s!: string; +} + +@json +class ObjectWithStringArray { + sa!: string[]; +} + +@json +class ObjectWithFloat { + f!: f64; +} + +@json +class ObjectWithFloatArray { + fa!: f64[]; +} From 68d78befc5501491ec5a9769a7f7bb606277c886 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 11:41:42 -0800 Subject: [PATCH 02/30] Fix negative exponential in number array parsing --- assembly/src/json.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 11fb1ade..82fa8dea 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -830,7 +830,7 @@ export namespace JSON { let i = 1; for (; i < data.length - 1; i++) { const char = unsafeCharCodeAt(data, i); - if ((lastPos === 0 && char >= 48 && char <= 57) || char === 45) { + if (lastPos === 0 && ((char >= 48 && char <= 57) || char === 45)) { lastPos = i; } else if ((isSpace(char) || char == commaCode) && lastPos > 0) { result.push(parseNumber>(data.slice(lastPos, i))); From 89387131ca0b7359b15921fe6544d3ad5ec6d4af Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 12:30:37 -0800 Subject: [PATCH 03/30] Add more tests --- assembly/__tests__/as-json.spec.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/assembly/__tests__/as-json.spec.ts b/assembly/__tests__/as-json.spec.ts index 9a9cb836..0d9a1a0f 100644 --- a/assembly/__tests__/as-json.spec.ts +++ b/assembly/__tests__/as-json.spec.ts @@ -472,6 +472,12 @@ describe("Ser/de escape sequences in strings", () => { canSer('\\\\\\', '"\\\\\\\\\\\\"'); }); + it("cannot parse invalid escape sequences", () => { + expect(() => { + JSON.parse('"\\z"'); + }).toThrow(); + }); + }); describe("Ser/de special strings in object", () => { @@ -496,6 +502,17 @@ describe("Ser/de special strings in object", () => { canDeser(s, o); }); + it("should deserialize slashes in string in object", () => { + const o: ObjWithString = { s: "//" }; + const s = '{"s":"/\\/"}'; + canDeser(s, o); + }); + it("should deserialize slashes in string in array", () => { + const a = ["/", "/"]; + const s = '["/","\/"]'; + canDeser(s, a); + }); + it("should ser/de short escape sequences in strings in objects", () => { const o: ObjWithString = { s: "\n\r\t\b\f" }; const s = '{"s":"\\n\\r\\t\\b\\f"}'; From 1c7aac57491bb284e6e62f5f24aecbcbf6d40229 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 12:34:29 -0800 Subject: [PATCH 04/30] refactoring --- assembly/src/chars.ts | 2 ++ assembly/src/json.ts | 23 +++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/assembly/src/chars.ts b/assembly/src/chars.ts index 1bdf32d9..b237efd1 100644 --- a/assembly/src/chars.ts +++ b/assembly/src/chars.ts @@ -35,6 +35,8 @@ @inline export const sCode = 115; // @ts-ignore = Decorator is valid here @inline export const nCode = 110; +// @ts-ignore = Decorator is valid here +@inline export const bCode = 98; // Strings // @ts-ignore: Decorator is valid here @inline export const trueWord = "true"; diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 82fa8dea..6be6dea6 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -2,6 +2,7 @@ import { StringSink } from "as-string-sink/assembly"; import { isSpace } from "util/string"; import { aCode, + bCode, eCode, fCode, lCode, @@ -14,6 +15,7 @@ import { backSlashCode, colonCode, commaCode, + forwardSlashCode, leftBraceCode, leftBracketCode, newLineCode, @@ -327,12 +329,12 @@ export namespace JSON { // @ts-ignore: Decorator @inline function serializeString(data: string): string { - let result = new StringSink('"'); + let result = new StringSink(quoteWord); let last: i32 = 0; for (let i = 0; i < data.length; i++) { const char = unsafeCharCodeAt(data, i); - if (char === 34 || char === 92) { + if (char === quoteCode || char === backSlashCode) { result.write(data, last, i); result.writeCodePoint(backSlashCode); last = i; @@ -380,7 +382,6 @@ export namespace JSON { let result = new StringSink(); let last = 1; for (let i = 1; i < data.length - 1; i++) { - // \\" if (unsafeCharCodeAt(data, i) === backSlashCode) { const char = unsafeCharCodeAt(data, ++i); result.write(data, last, i - 1); @@ -389,32 +390,32 @@ export namespace JSON { last = i + 1; } else if (char >= 92 && char <= 117) { switch (char) { - case 92: { + case backSlashCode: { result.writeCodePoint(backSlashCode); last = i + 1; break; } - case 98: { + case bCode: { result.write("\b"); last = i + 1; break; } - case 102: { + case fCode: { result.write("\f"); last = i + 1; break; } - case 110: { + case nCode: { result.writeCodePoint(newLineCode); last = i + 1; break; } - case 114: { + case rCode: { result.write("\r"); last = i + 1; break; } - case 116: { + case tCode: { result.write("\t"); last = i + 1; break; @@ -530,9 +531,7 @@ export namespace JSON { if (char === backSlashCode && !escaping) { escaping = true; } else { - if ( - char === quoteCode && !escaping - ) { + if (char === quoteCode && !escaping) { if (isKey === false) { key.reinst(data, outerLoopIndex, stringValueIndex); isKey = true; From bdfb4ef629d624ba86cfea923ae39ababce6a76e Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 12:38:04 -0800 Subject: [PATCH 05/30] improve zero length string shortcut --- assembly/src/json.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 6be6dea6..ed3fd833 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -329,6 +329,10 @@ export namespace JSON { // @ts-ignore: Decorator @inline function serializeString(data: string): string { + if (data.length === 0) { + return quoteWord + quoteWord; + } + let result = new StringSink(quoteWord); let last: i32 = 0; @@ -369,9 +373,6 @@ export namespace JSON { } } } - if (result.length === 1) { - return quoteWord + data + quoteWord; - } result.write(data, last); result.write(quoteWord); return result.toString(); From fa458d11fab818de8fd743a2348d1e33c7edfcd0 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 12:39:14 -0800 Subject: [PATCH 06/30] Encode all control characters --- assembly/src/json.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index ed3fd833..08f7ba68 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -342,7 +342,7 @@ export namespace JSON { result.write(data, last, i); result.writeCodePoint(backSlashCode); last = i; - } else if (char <= 13 && char >= 8) { + } else if (char < 16) { result.write(data, last, i); last = i + 1; switch (char) { @@ -358,10 +358,6 @@ export namespace JSON { result.write("\\n"); break; } - case 11: { - result.write("\\x0B"); // \\u000b - break; - } case 12: { result.write("\\f"); break; @@ -370,7 +366,19 @@ export namespace JSON { result.write("\\r"); break; } + default: { + // all chars 0-31 must be encoded as a four digit unicode escape sequence + // \u0000 to \u000f handled here + result.write("\\u000" + char.toString(16)); + break; + } } + } else if (char < 32) { + result.write(data, last, i); + last = i + 1; + // all chars 0-31 must be encoded as a four digit unicode escape sequence + // \u0010 to \u001f handled here + result.write("\\u00" + char.toString(16)); } } result.write(data, last); From e8c21a2b66b8922255a2d8d9adf8fe9d6692ae49 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 12:39:55 -0800 Subject: [PATCH 07/30] Decode all escaped codepoints --- assembly/src/json.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 08f7ba68..46b7f896 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -397,7 +397,7 @@ export namespace JSON { if (char === 34) { result.writeCodePoint(quoteCode); last = i + 1; - } else if (char >= 92 && char <= 117) { + } else { switch (char) { case backSlashCode: { result.writeCodePoint(backSlashCode); @@ -429,18 +429,16 @@ export namespace JSON { last = i + 1; break; } - default: { - if ( - char === 117 && - load(changetype(data) + ((i + 1) << 1)) === - 27584753879220272 - ) { - result.write("\u000b"); - i += 4; - last = i + 1; - } + case uCode: { + const code = u16.parse(data.slice(i + 1, i + 5), 16); + result.writeCodePoint(code); + i += 4; + last = i + 1; break; } + default: { + throw new Error(`JSON: Cannot parse "${data}" as string. Invalid escape sequence: \\${data.charAt(i)}`); + } } } } From 81a691fdb1d2ad7f908ca1f8daaf7d777e26a654 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 12:40:10 -0800 Subject: [PATCH 08/30] Decode escaped forward slash --- assembly/src/json.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 46b7f896..81ef3e21 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -404,6 +404,11 @@ export namespace JSON { last = i + 1; break; } + case forwardSlashCode: { + result.writeCodePoint(forwardSlashCode); + last = i + 1; + break; + } case bCode: { result.write("\b"); last = i + 1; From 762bc0e33b3ffdec6239be3d517fcf5fc0153314 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 14:32:24 -0800 Subject: [PATCH 09/30] fix tsc errors on transform build --- transform/lib/index.js | 2 ++ transform/src/index.ts | 3 +++ 2 files changed, 5 insertions(+) diff --git a/transform/lib/index.js b/transform/lib/index.js index 15517456..18a98ba8 100644 --- a/transform/lib/index.js +++ b/transform/lib/index.js @@ -77,8 +77,10 @@ class AsJSONTransform extends BaseVisitor { let type = toString(member.type); const name = member.name.text; let aliasName = name; + // @ts-ignore if (member.decorators && ((_d = member.decorators[0]) === null || _d === void 0 ? void 0 : _d.name.text) === "alias") { if (member.decorators[0] && member.decorators[0].args[0]) { + // @ts-ignore aliasName = member.decorators[0].args[0].value; } } diff --git a/transform/src/index.ts b/transform/src/index.ts index 24141196..06c40939 100644 --- a/transform/src/index.ts +++ b/transform/src/index.ts @@ -89,8 +89,11 @@ class AsJSONTransform extends BaseVisitor { const name = member.name.text; let aliasName = name; + + // @ts-ignore if (member.decorators && member.decorators[0]?.name.text === "alias") { if (member.decorators[0] && member.decorators[0].args![0]) { + // @ts-ignore aliasName = member.decorators[0].args![0].value; } } From e236dac7aa1cde063dc87867d57175fcce074504 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 14:33:44 -0800 Subject: [PATCH 10/30] Add more tests --- assembly/__tests__/as-json.spec.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/assembly/__tests__/as-json.spec.ts b/assembly/__tests__/as-json.spec.ts index 0d9a1a0f..522aaf76 100644 --- a/assembly/__tests__/as-json.spec.ts +++ b/assembly/__tests__/as-json.spec.ts @@ -575,6 +575,12 @@ describe("Ser/de special strings in object", () => { const s = `{"sa":["${y.join('","')}"]}`; canSerde(o, s); }); + + it("should ser/de escape sequences in key of object", () => { + const o: ObjWithStrangeKey = { n: 123 }; + const s = '{"a\\\\\\t\\"\\u0002b":123}'; + canSerde(o, s); + }); }); @json @@ -596,3 +602,9 @@ class ObjectWithFloat { class ObjectWithFloatArray { fa!: f64[]; } + +@json +class ObjWithStrangeKey { + @alias('a\\\t"\x02b') + n!: i32; +} From eedef610296ee7945afc8f75ae56ab718ce9d6cb Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 14:34:36 -0800 Subject: [PATCH 11/30] fix encodings of strings in object keys and values --- assembly/src/json.ts | 5 +++-- transform/lib/index.js | 4 ++-- transform/src/index.ts | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 81ef3e21..95ddf48a 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -544,12 +544,13 @@ export namespace JSON { escaping = true; } else { if (char === quoteCode && !escaping) { + const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); if (isKey === false) { - key.reinst(data, outerLoopIndex, stringValueIndex); + key.reinst(value); isKey = true; } else { // @ts-ignore - schema.__JSON_Set_Key>(key, data, outerLoopIndex, stringValueIndex, initializeDefaultValues); + schema.__JSON_Set_Key>(key, value, 0, value.length, initializeDefaultValues); isKey = false; } outerLoopIndex = ++stringValueIndex; diff --git a/transform/lib/index.js b/transform/lib/index.js index 18a98ba8..13953b23 100644 --- a/transform/lib/index.js +++ b/transform/lib/index.js @@ -98,9 +98,9 @@ class AsJSONTransform extends BaseVisitor { "u64", "i64", ].includes(type.toLowerCase())) { - this.currentClass.encodeStmts.push(`"${aliasName}":\${this.${name}.toString()},`); + this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}.toString()},`); // @ts-ignore - this.currentClass.setDataStmts.push(`if (key.equals("${aliasName}")) { + this.currentClass.setDataStmts.push(`if (key.equals("${aliasName.replace(/\\/g, '\\\\')}")) { this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); return; } diff --git a/transform/src/index.ts b/transform/src/index.ts index 06c40939..ec832388 100644 --- a/transform/src/index.ts +++ b/transform/src/index.ts @@ -114,11 +114,11 @@ class AsJSONTransform extends BaseVisitor { ].includes(type.toLowerCase()) ) { this.currentClass.encodeStmts.push( - `"${aliasName}":\${this.${name}.toString()},` + `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${this.${name}.toString()},` ); // @ts-ignore this.currentClass.setDataStmts.push( - `if (key.equals("${aliasName}")) { + `if (key.equals("${aliasName.replace(/\\/g,'\\\\')}")) { this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); return; } From bcfabe01ba96431fec58e9f28ba1bcf7b8a698e2 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 15:54:30 -0800 Subject: [PATCH 12/30] More tests --- assembly/__tests__/as-json.spec.ts | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/assembly/__tests__/as-json.spec.ts b/assembly/__tests__/as-json.spec.ts index 522aaf76..b02484b0 100644 --- a/assembly/__tests__/as-json.spec.ts +++ b/assembly/__tests__/as-json.spec.ts @@ -480,7 +480,7 @@ describe("Ser/de escape sequences in strings", () => { }); -describe("Ser/de special strings in object", () => { +describe("Ser/de special strings in object values", () => { it("should serialize quotes in string in object", () => { const o: ObjWithString = { s: '"""' }; const s = '{"s":"\\"\\"\\""}'; @@ -576,11 +576,27 @@ describe("Ser/de special strings in object", () => { canSerde(o, s); }); - it("should ser/de escape sequences in key of object", () => { - const o: ObjWithStrangeKey = { n: 123 }; +}); + +describe("Ser/de special strings in object keys", () => { + + it("should ser/de escape sequences in key of object with int value", () => { + const o: ObjWithStrangeKey = { data: 123 }; const s = '{"a\\\\\\t\\"\\u0002b":123}'; canSerde(o, s); }); + + it("should ser/de escape sequences in key of object with float value", () => { + const o: ObjWithStrangeKey = { data: 123.4 }; + const s = '{"a\\\\\\t\\"\\u0002b":123.4}'; + canSerde(o, s); + }); + + it("should ser/de escape sequences in key of object with string value", () => { + const o: ObjWithStrangeKey = { data: "abc" }; + const s = '{"a\\\\\\t\\"\\u0002b":"abc"}'; + canSerde(o, s); + }); }); @json @@ -604,7 +620,7 @@ class ObjectWithFloatArray { } @json -class ObjWithStrangeKey { +class ObjWithStrangeKey { @alias('a\\\t"\x02b') - n!: i32; + data!: T; } From 1a9ff98f944c8cfdc11dc54f9ef587f0e24333c4 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 15:55:28 -0800 Subject: [PATCH 13/30] Additional fixes to encoding in object keys --- transform/lib/index.js | 10 +++++----- transform/src/index.ts | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/transform/lib/index.js b/transform/lib/index.js index 13953b23..af25e7ce 100644 --- a/transform/lib/index.js +++ b/transform/lib/index.js @@ -100,7 +100,7 @@ class AsJSONTransform extends BaseVisitor { ].includes(type.toLowerCase())) { this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}.toString()},`); // @ts-ignore - this.currentClass.setDataStmts.push(`if (key.equals("${aliasName.replace(/\\/g, '\\\\')}")) { + this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); return; } @@ -114,9 +114,9 @@ class AsJSONTransform extends BaseVisitor { "f32", "f64", ].includes(type.toLowerCase())) { - this.currentClass.encodeStmts.push(`"${aliasName}":\${this.${name}.toString()},`); + this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}.toString()},`); // @ts-ignore - this.currentClass.setDataStmts.push(`if (key.equals("${aliasName}")) { + this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __parseObjectValue<${type}>(data.slice(val_start, val_end), initializeDefaultValues); return; } @@ -126,9 +126,9 @@ class AsJSONTransform extends BaseVisitor { } } else { - this.currentClass.encodeStmts.push(`"${aliasName}":\${JSON.stringify<${type}>(this.${name})},`); + this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${JSON.stringify<${type}>(this.${name})},`); // @ts-ignore - this.currentClass.setDataStmts.push(`if (key.equals("${aliasName}")) { + this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __parseObjectValue<${type}>(val_start ? data.slice(val_start, val_end) : data, initializeDefaultValues); return; } diff --git a/transform/src/index.ts b/transform/src/index.ts index ec832388..199c44f8 100644 --- a/transform/src/index.ts +++ b/transform/src/index.ts @@ -118,7 +118,7 @@ class AsJSONTransform extends BaseVisitor { ); // @ts-ignore this.currentClass.setDataStmts.push( - `if (key.equals("${aliasName.replace(/\\/g,'\\\\')}")) { + `if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); return; } @@ -137,11 +137,11 @@ class AsJSONTransform extends BaseVisitor { ].includes(type.toLowerCase()) ) { this.currentClass.encodeStmts.push( - `"${aliasName}":\${this.${name}.toString()},` + `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${this.${name}.toString()},` ); // @ts-ignore this.currentClass.setDataStmts.push( - `if (key.equals("${aliasName}")) { + `if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __parseObjectValue<${type}>(data.slice(val_start, val_end), initializeDefaultValues); return; } @@ -154,11 +154,11 @@ class AsJSONTransform extends BaseVisitor { } } else { this.currentClass.encodeStmts.push( - `"${aliasName}":\${JSON.stringify<${type}>(this.${name})},` + `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${JSON.stringify<${type}>(this.${name})},` ); // @ts-ignore this.currentClass.setDataStmts.push( - `if (key.equals("${aliasName}")) { + `if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __parseObjectValue<${type}>(val_start ? data.slice(val_start, val_end) : data, initializeDefaultValues); return; } From d7dd2967dc8a3c9c473795b43b598615ca9e9255 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 15:56:03 -0800 Subject: [PATCH 14/30] Fix extraneous encoding in object values. --- assembly/src/json.ts | 45 +------------------------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 95ddf48a..6742e8e7 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -256,51 +256,8 @@ export namespace JSON { @global @inline function __parseObjectValue(data: string, initializeDefaultValues: boolean): T { let type: T; if (isString()) { - let result = ""; - let last = 0; - for (let i = 0; i < data.length; i++) { - // \\" - if (unsafeCharCodeAt(data, i) === backSlashCode) { - const char = unsafeCharCodeAt(data, ++i); - result += data.slice(last, i - 1); - if (char === 34) { - result += '"'; - last = ++i; - } else if (char === 110) { - result += "\n"; - last = ++i; - // 92 98 114 116 102 117 - } else if (char >= 92 && char <= 117) { - if (char === 92) { - result += "\\"; - last = ++i; - } else if (char === 98) { - result += "\b"; - last = ++i; - } else if (char === 102) { - result += "\f"; - last = ++i; - } else if (char === 114) { - result += "\r"; - last = ++i; - } else if (char === 116) { - result += "\t"; - last = ++i; - } else if ( - char === 117 && - load(changetype(data) + ((i + 1) << 1)) === - 27584753879220272 - ) { - result += "\u000b"; - i += 4; - last = ++i; - } - } - } - } - result += data.slice(last); // @ts-ignore - return result; + return data; } else if (isBoolean()) { // @ts-ignore return parseBoolean(data); From 66d8334a54afb16278b60ce2710aaf0ee31df10a Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 17:00:16 -0800 Subject: [PATCH 15/30] update tests --- assembly/__tests__/as-json.spec.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/assembly/__tests__/as-json.spec.ts b/assembly/__tests__/as-json.spec.ts index b02484b0..e97d1c8e 100644 --- a/assembly/__tests__/as-json.spec.ts +++ b/assembly/__tests__/as-json.spec.ts @@ -74,9 +74,7 @@ describe("Ser/de Numbers", () => { canSerde(123456e-5, "1.23456"); canSerde(0.0, "0.0"); - - canSerde(0.0, "0.0"); - canSerde(7.23, "7.23"); + canSerde(-7.23, "-7.23"); canSerde(1e-6, "0.000001"); canSerde(1e-7, "1e-7"); From 9550cf457b2cf172646497c2c0f823cd62c56fd9 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 18:09:36 -0800 Subject: [PATCH 16/30] Also handle encoding in map keys and values --- assembly/__tests__/as-json.spec.ts | 13 +++++++++++++ assembly/src/json.ts | 5 +++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/assembly/__tests__/as-json.spec.ts b/assembly/__tests__/as-json.spec.ts index e97d1c8e..9df2bdf3 100644 --- a/assembly/__tests__/as-json.spec.ts +++ b/assembly/__tests__/as-json.spec.ts @@ -595,6 +595,19 @@ describe("Ser/de special strings in object keys", () => { const s = '{"a\\\\\\t\\"\\u0002b":"abc"}'; canSerde(o, s); }); + + it("should ser/de escape sequences in map key", () => { + const m = new Map(); + m.set('a\\\t"\x02b', 'abc'); + const s = '{"a\\\\\\t\\"\\u0002b":"abc"}'; + canSerde(m, s); + }); + it("should ser/de escape sequences in map value", () => { + const m = new Map(); + m.set('abc', 'a\\\t"\x02b'); + const s = '{"abc":"a\\\\\\t\\"\\u0002b"}'; + canSerde(m, s); + }); }); @json diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 6742e8e7..39fc0dc7 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -632,12 +632,13 @@ export namespace JSON { if ( char === quoteCode && !escaping ) { + const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); if (isKey === false) { - key.reinst(data, outerLoopIndex, stringValueIndex); + key.reinst(value); isKey = true; } else { if (isString>()) { - map.set(parseMapKey>(key), data.slice(outerLoopIndex, stringValueIndex)); + map.set(parseMapKey>(key), value); } isKey = false; } From 3a39521c51af346d9fd946ae4a5437111aabe782 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 18:43:45 -0800 Subject: [PATCH 17/30] optimization / cleanup --- assembly/src/chars.ts | 11 +++- assembly/src/json.ts | 145 ++++++++++++++++++++---------------------- assembly/src/util.ts | 6 +- 3 files changed, 82 insertions(+), 80 deletions(-) diff --git a/assembly/src/chars.ts b/assembly/src/chars.ts index b237efd1..1d58c91f 100644 --- a/assembly/src/chars.ts +++ b/assembly/src/chars.ts @@ -60,6 +60,15 @@ @inline export const rightBracketWord = "]"; // @ts-ignore: Decorator is valid here @inline export const quoteWord = "\""; + // Escape Codes // @ts-ignore: Decorator is valid here -@inline export const newLineCode = 10; +@inline export const backspaceCode = 8; // \b +// @ts-ignore: Decorator is valid here +@inline export const tabCode = 9; // \t +// @ts-ignore: Decorator is valid here +@inline export const newLineCode = 10; // \n +// @ts-ignore: Decorator is valid here +@inline export const formFeedCode = 12; // \f +// @ts-ignore: Decorator is valid here +@inline export const carriageReturnCode = 13; // \r diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 39fc0dc7..d46f67ba 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -18,18 +18,21 @@ import { forwardSlashCode, leftBraceCode, leftBracketCode, - newLineCode, quoteCode, rightBraceCode, rightBracketCode, - colonWord, + backspaceCode, + carriageReturnCode, + tabCode, + formFeedCode, + newLineCode, + commaWord, quoteWord, leftBraceWord, leftBracketWord, - rightBraceWord, rightBracketWord, emptyArrayWord, @@ -102,11 +105,11 @@ export namespace JSON { for (let i = 0; i < data.length - 1; i++) { // @ts-ignore result.write(JSON.stringify(unchecked(data[i]))); - result.write(commaWord); + result.writeCodePoint(commaCode); } // @ts-ignore result.write(JSON.stringify(unchecked(data[data.length - 1]))); - result.write(rightBracketWord); + result.writeCodePoint(rightBracketCode); return result.toString(); } } else if (data instanceof Map) { @@ -115,13 +118,13 @@ export namespace JSON { let values = data.values(); for (let i = 0; i < data.size; i++) { result.write(serializeString(keys[i].toString())); - result.write(colonWord); + result.writeCodePoint(colonCode); result.write(JSON.stringify(values[i])); if (i < data.size - 1) { - result.write(commaWord); + result.writeCodePoint(commaCode); } } - result.write(rightBraceWord); + result.writeCodePoint(rightBraceCode); return result.toString(); } else { throw new Error( @@ -196,11 +199,11 @@ export namespace JSON { for (let i = 0; i < data.length - 1; i++) { // @ts-ignore result.write(JSON.stringify(unchecked(data[i]))); - result.write(commaWord); + result.writeCodePoint(commaCode); } // @ts-ignore result.write(JSON.stringify(unchecked(data[data.length - 1]))); - result.write(rightBracketWord); + result.writeCodePoint(rightBracketCode); out = result.toString(); return; } @@ -303,23 +306,23 @@ export namespace JSON { result.write(data, last, i); last = i + 1; switch (char) { - case 8: { + case backspaceCode: { result.write("\\b"); break; } - case 9: { + case tabCode: { result.write("\\t"); break; } - case 10: { + case newLineCode: { result.write("\\n"); break; } - case 12: { + case formFeedCode: { result.write("\\f"); break; } - case 13: { + case carriageReturnCode: { result.write("\\r"); break; } @@ -339,7 +342,7 @@ export namespace JSON { } } result.write(data, last); - result.write(quoteWord); + result.writeCodePoint(quoteCode); return result.toString(); } @@ -351,56 +354,56 @@ export namespace JSON { if (unsafeCharCodeAt(data, i) === backSlashCode) { const char = unsafeCharCodeAt(data, ++i); result.write(data, last, i - 1); - if (char === 34) { - result.writeCodePoint(quoteCode); - last = i + 1; - } else { - switch (char) { - case backSlashCode: { - result.writeCodePoint(backSlashCode); - last = i + 1; - break; - } - case forwardSlashCode: { - result.writeCodePoint(forwardSlashCode); - last = i + 1; - break; - } - case bCode: { - result.write("\b"); - last = i + 1; - break; - } - case fCode: { - result.write("\f"); - last = i + 1; - break; - } - case nCode: { - result.writeCodePoint(newLineCode); - last = i + 1; - break; - } - case rCode: { - result.write("\r"); - last = i + 1; - break; - } - case tCode: { - result.write("\t"); - last = i + 1; - break; - } - case uCode: { - const code = u16.parse(data.slice(i + 1, i + 5), 16); - result.writeCodePoint(code); - i += 4; - last = i + 1; - break; - } - default: { - throw new Error(`JSON: Cannot parse "${data}" as string. Invalid escape sequence: \\${data.charAt(i)}`); - } + switch (char) { + case quoteCode: { + result.writeCodePoint(quoteCode); + last = i + 1; + break; + } + case backSlashCode: { + result.writeCodePoint(backSlashCode); + last = i + 1; + break; + } + case forwardSlashCode: { + result.writeCodePoint(forwardSlashCode); + last = i + 1; + break; + } + case bCode: { + result.writeCodePoint(backspaceCode); + last = i + 1; + break; + } + case fCode: { + result.writeCodePoint(formFeedCode); + last = i + 1; + break; + } + case nCode: { + result.writeCodePoint(newLineCode); + last = i + 1; + break; + } + case rCode: { + result.writeCodePoint(carriageReturnCode); + last = i + 1; + break; + } + case tCode: { + result.writeCodePoint(tabCode); + last = i + 1; + break; + } + case uCode: { + const code = u16.parse(data.slice(i + 1, i + 5), 16); + result.writeCodePoint(code); + i += 4; + last = i + 1; + break; + } + default: { + throw new Error(`JSON: Cannot parse "${data}" as string. Invalid escape sequence: \\${data.charAt(i)}`); } } } @@ -773,16 +776,6 @@ export namespace JSON { let lastPos = 1; for (let i = 1; i < data.length - 1; i++) { const char = unsafeCharCodeAt(data, i); - /*// if char == "t" && i+3 == "e" - if (char === tCode && data.charCodeAt(i + 3) === eCode) { - //i += 3; - result.push(parseBoolean>(data.slice(lastPos, i+2))); - //i++; - } else if (char === fCode && data.charCodeAt(i + 4) === eCode) { - //i += 4; - result.push(parseBoolean>(data.slice(lastPos, i+3))); - //i++; - }*/ if (char === tCode || char === fCode) { lastPos = i; } else if (char === eCode) { diff --git a/assembly/src/util.ts b/assembly/src/util.ts index 44c12c44..90e0cba7 100644 --- a/assembly/src/util.ts +++ b/assembly/src/util.ts @@ -12,11 +12,11 @@ import { backSlashCode, quoteCode } from "./chars"; const result = new StringSink(); let instr = false; for (let i = 0; i < data.length; i++) { - const char = data.charCodeAt(i); + const char = unsafeCharCodeAt(data, i); if (instr === false && char === quoteCode) instr = true; else if ( instr === true && char === quoteCode - && data.charCodeAt(i - 1) !== backSlashCode + && unsafeCharCodeAt(data, i - 1) !== backSlashCode ) instr = false; if (instr === false) { @@ -347,4 +347,4 @@ import { backSlashCode, quoteCode } from "./chars"; return load(changetype(p1_data) + p1_start) == load(changetype(p2_data) + p2_start) } return memory.compare(changetype(p1_data) + p1_start, changetype(p2_data) + p2_start, p1_len) === 0; -} \ No newline at end of file +} From 03006adb5f4dcea417499f876beca747ea66602d Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Tue, 20 Feb 2024 19:16:41 -0800 Subject: [PATCH 18/30] perf improvement --- assembly/src/json.ts | 12 +++++++++--- assembly/src/util.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index d46f67ba..ee70d431 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -40,7 +40,7 @@ import { falseWord, nullWord, } from "./chars"; -import { snip_fast, unsafeCharCodeAt } from "./util"; +import { snip_fast, unsafeCharCodeAt, containsCodePoint } from "./util"; import { Virtual } from "as-virtual/assembly"; /** @@ -504,11 +504,17 @@ export namespace JSON { escaping = true; } else { if (char === quoteCode && !escaping) { - const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); if (isKey === false) { - key.reinst(value); + // perf: we can avoid creating a new string here if the key doesn't contain any escape sequences + if (containsCodePoint(data, backSlashCode, outerLoopIndex, stringValueIndex)) { + const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); + key.reinst(value); + } else { + key.reinst(data, outerLoopIndex, stringValueIndex); + } isKey = true; } else { + const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); // @ts-ignore schema.__JSON_Set_Key>(key, value, 0, value.length, initializeDefaultValues); isKey = false; diff --git a/assembly/src/util.ts b/assembly/src/util.ts index 90e0cba7..ca2714c1 100644 --- a/assembly/src/util.ts +++ b/assembly/src/util.ts @@ -348,3 +348,11 @@ import { backSlashCode, quoteCode } from "./chars"; } return memory.compare(changetype(p1_data) + p1_start, changetype(p2_data) + p2_start, p1_len) === 0; } + +// @ts-ignore +@inline export function containsCodePoint(str: string, code: u32, start: i32, end: i32): bool { + for (let i = start; i <= end; i++) { + if (unsafeCharCodeAt(str, i) == code) return true; + } + return false; +} From 5fd01f84bfa56833014bf8c536b7eea46c7c62e6 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 10:19:33 -0800 Subject: [PATCH 19/30] perf improvement for map parsing --- assembly/src/json.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index ee70d431..730266aa 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -641,12 +641,18 @@ export namespace JSON { if ( char === quoteCode && !escaping ) { - const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); if (isKey === false) { - key.reinst(value); + // perf: we can avoid creating a new string here if the key doesn't contain any escape sequences + if (containsCodePoint(data, backSlashCode, outerLoopIndex, stringValueIndex)) { + const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); + key.reinst(value); + } else { + key.reinst(data, outerLoopIndex, stringValueIndex); + } isKey = true; } else { if (isString>()) { + const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); map.set(parseMapKey>(key), value); } isKey = false; From 4c83347cce07b7230029c22f24be7d7f9557fe83 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 10:20:36 -0800 Subject: [PATCH 20/30] invert condition to reduce nesting --- assembly/src/json.ts | 109 ++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 730266aa..bc8cbc47 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -351,60 +351,61 @@ export namespace JSON { let result = new StringSink(); let last = 1; for (let i = 1; i < data.length - 1; i++) { - if (unsafeCharCodeAt(data, i) === backSlashCode) { - const char = unsafeCharCodeAt(data, ++i); - result.write(data, last, i - 1); - switch (char) { - case quoteCode: { - result.writeCodePoint(quoteCode); - last = i + 1; - break; - } - case backSlashCode: { - result.writeCodePoint(backSlashCode); - last = i + 1; - break; - } - case forwardSlashCode: { - result.writeCodePoint(forwardSlashCode); - last = i + 1; - break; - } - case bCode: { - result.writeCodePoint(backspaceCode); - last = i + 1; - break; - } - case fCode: { - result.writeCodePoint(formFeedCode); - last = i + 1; - break; - } - case nCode: { - result.writeCodePoint(newLineCode); - last = i + 1; - break; - } - case rCode: { - result.writeCodePoint(carriageReturnCode); - last = i + 1; - break; - } - case tCode: { - result.writeCodePoint(tabCode); - last = i + 1; - break; - } - case uCode: { - const code = u16.parse(data.slice(i + 1, i + 5), 16); - result.writeCodePoint(code); - i += 4; - last = i + 1; - break; - } - default: { - throw new Error(`JSON: Cannot parse "${data}" as string. Invalid escape sequence: \\${data.charAt(i)}`); - } + if (unsafeCharCodeAt(data, i) !== backSlashCode) { + continue; + } + const char = unsafeCharCodeAt(data, ++i); + result.write(data, last, i - 1); + switch (char) { + case quoteCode: { + result.writeCodePoint(quoteCode); + last = i + 1; + break; + } + case backSlashCode: { + result.writeCodePoint(backSlashCode); + last = i + 1; + break; + } + case forwardSlashCode: { + result.writeCodePoint(forwardSlashCode); + last = i + 1; + break; + } + case bCode: { + result.writeCodePoint(backspaceCode); + last = i + 1; + break; + } + case fCode: { + result.writeCodePoint(formFeedCode); + last = i + 1; + break; + } + case nCode: { + result.writeCodePoint(newLineCode); + last = i + 1; + break; + } + case rCode: { + result.writeCodePoint(carriageReturnCode); + last = i + 1; + break; + } + case tCode: { + result.writeCodePoint(tabCode); + last = i + 1; + break; + } + case uCode: { + const code = u16.parse(data.slice(i + 1, i + 5), 16); + result.writeCodePoint(code); + i += 4; + last = i + 1; + break; + } + default: { + throw new Error(`JSON: Cannot parse "${data}" as string. Invalid escape sequence: \\${data.charAt(i)}`); } } } From 99d78887438a27c13a4006a341d5fe8a480f6c18 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 10:42:16 -0800 Subject: [PATCH 21/30] perf: init capacity when parsing string --- assembly/src/json.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index bc8cbc47..a43cd32c 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -348,7 +348,7 @@ export namespace JSON { // @ts-ignore: Decorator @inline function parseString(data: string): string { - let result = new StringSink(); + let result = StringSink.withCapacity(data.length); let last = 1; for (let i = 1; i < data.length - 1; i++) { if (unsafeCharCodeAt(data, i) !== backSlashCode) { From e66b28abee50fc87579b3e568bf350f7d85dfc08 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 11:36:26 -0800 Subject: [PATCH 22/30] perf: avoid slice before parsing string --- assembly/src/json.ts | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index a43cd32c..23e81c49 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -347,10 +347,11 @@ export namespace JSON { } // @ts-ignore: Decorator -@inline function parseString(data: string): string { - let result = StringSink.withCapacity(data.length); - let last = 1; - for (let i = 1; i < data.length - 1; i++) { +@inline function parseString(data: string, start: i32 = 0, end: i32 = 0): string { + end = end || data.length - 1; + let result = StringSink.withCapacity(end - start - 1); + let last = start + 1; + for (let i = last; i < end; i++) { if (unsafeCharCodeAt(data, i) !== backSlashCode) { continue; } @@ -409,10 +410,10 @@ export namespace JSON { } } } - if ((data.length - 1) > last) { - result.write(data, last, data.length - 1); + if (end > last) { + result.write(data, last, end); } - return result.toString(); + return result.toString() } // @ts-ignore: Decorator @@ -508,14 +509,13 @@ export namespace JSON { if (isKey === false) { // perf: we can avoid creating a new string here if the key doesn't contain any escape sequences if (containsCodePoint(data, backSlashCode, outerLoopIndex, stringValueIndex)) { - const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); - key.reinst(value); + key.reinst(parseString(data, outerLoopIndex - 1, stringValueIndex)); } else { key.reinst(data, outerLoopIndex, stringValueIndex); } isKey = true; } else { - const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); + const value = parseString(data, outerLoopIndex - 1, stringValueIndex); // @ts-ignore schema.__JSON_Set_Key>(key, value, 0, value.length, initializeDefaultValues); isKey = false; @@ -645,15 +645,14 @@ export namespace JSON { if (isKey === false) { // perf: we can avoid creating a new string here if the key doesn't contain any escape sequences if (containsCodePoint(data, backSlashCode, outerLoopIndex, stringValueIndex)) { - const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); - key.reinst(value); + key.reinst(parseString(data, outerLoopIndex - 1, stringValueIndex)); } else { key.reinst(data, outerLoopIndex, stringValueIndex); } isKey = true; } else { if (isString>()) { - const value = parseString(data.slice(outerLoopIndex-1, stringValueIndex+1)); + const value = parseString(data, outerLoopIndex - 1, stringValueIndex); map.set(parseMapKey>(key), value); } isKey = false; @@ -774,7 +773,7 @@ export namespace JSON { lastPos = i; } else { instr = false; - result.push(parseString(data.slice(lastPos, i + 1))); + result.push(parseString(data, lastPos, i)); } } escaping = false; From c98fcaada10a54fc2a5974ee3f6ae2e6eb1ccba0 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 16:20:27 -0800 Subject: [PATCH 23/30] Remove redundant toString --- transform/lib/index.js | 4 ++-- transform/src/index.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/transform/lib/index.js b/transform/lib/index.js index af25e7ce..60550759 100644 --- a/transform/lib/index.js +++ b/transform/lib/index.js @@ -98,7 +98,7 @@ class AsJSONTransform extends BaseVisitor { "u64", "i64", ].includes(type.toLowerCase())) { - this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}.toString()},`); + this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}},`); // @ts-ignore this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); @@ -114,7 +114,7 @@ class AsJSONTransform extends BaseVisitor { "f32", "f64", ].includes(type.toLowerCase())) { - this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}.toString()},`); + this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}},`); // @ts-ignore this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __parseObjectValue<${type}>(data.slice(val_start, val_end), initializeDefaultValues); diff --git a/transform/src/index.ts b/transform/src/index.ts index 199c44f8..70a0a9fb 100644 --- a/transform/src/index.ts +++ b/transform/src/index.ts @@ -114,7 +114,7 @@ class AsJSONTransform extends BaseVisitor { ].includes(type.toLowerCase()) ) { this.currentClass.encodeStmts.push( - `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${this.${name}.toString()},` + `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${this.${name}},` ); // @ts-ignore this.currentClass.setDataStmts.push( @@ -137,7 +137,7 @@ class AsJSONTransform extends BaseVisitor { ].includes(type.toLowerCase()) ) { this.currentClass.encodeStmts.push( - `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${this.${name}.toString()},` + `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${this.${name}},` ); // @ts-ignore this.currentClass.setDataStmts.push( From 17c3ede5d7437f6572a5e10959e912cd34c4faf3 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 16:25:22 -0800 Subject: [PATCH 24/30] Cleanup indentation of generated code --- transform/lib/index.js | 31 ++++++++++++------------------- transform/src/index.ts | 32 ++++++++++++-------------------- 2 files changed, 24 insertions(+), 39 deletions(-) diff --git a/transform/lib/index.js b/transform/lib/index.js index 60550759..e597e8ad 100644 --- a/transform/lib/index.js +++ b/transform/lib/index.js @@ -101,10 +101,9 @@ class AsJSONTransform extends BaseVisitor { this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}},`); // @ts-ignore this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { - this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); - return; - } - `); + this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); + return; + }`); if (member.initializer) { this.currentClass.initializeStmts.push(`this.${name} = ${toString(member.initializer)}`); } @@ -117,10 +116,9 @@ class AsJSONTransform extends BaseVisitor { this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}},`); // @ts-ignore this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { - this.${name} = __parseObjectValue<${type}>(data.slice(val_start, val_end), initializeDefaultValues); - return; - } - `); + this.${name} = __parseObjectValue<${type}>(data.slice(val_start, val_end), initializeDefaultValues); + return; + }`); if (member.initializer) { this.currentClass.initializeStmts.push(`this.${name} = ${toString(member.initializer)}`); } @@ -129,10 +127,9 @@ class AsJSONTransform extends BaseVisitor { this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${JSON.stringify<${type}>(this.${name})},`); // @ts-ignore this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { - this.${name} = __parseObjectValue<${type}>(val_start ? data.slice(val_start, val_end) : data, initializeDefaultValues); - return; - } - `); + this.${name} = __parseObjectValue<${type}>(val_start ? data.slice(val_start, val_end) : data, initializeDefaultValues); + return; + }`); if (member.initializer) { this.currentClass.initializeStmts.push(`this.${name} = ${toString(member.initializer)}`); } @@ -147,23 +144,19 @@ class AsJSONTransform extends BaseVisitor { serializeFunc = ` @inline __JSON_Serialize(): string { return \`{${this.currentClass.encodeStmts.join("")}}\`; - } - `; + }`; } else { serializeFunc = ` @inline __JSON_Serialize(): string { return "{}"; - } - `; + }`; } // Odd behavior here... When pairing this transform with asyncify, having @inline on __JSON_Set_Key with a generic will cause it to freeze. // Binaryen cannot predict and add/mangle code when it is genericed. const setKeyFunc = ` __JSON_Set_Key<__JSON_Key_Type>(key: __JSON_Key_Type, data: string, val_start: i32, val_end: i32, initializeDefaultValues: boolean): void { - ${ - // @ts-ignore - this.currentClass.setDataStmts.join("")} + ${this.currentClass.setDataStmts.join("\n ")} } `; let initializeFunc = ""; diff --git a/transform/src/index.ts b/transform/src/index.ts index 70a0a9fb..1c3fd733 100644 --- a/transform/src/index.ts +++ b/transform/src/index.ts @@ -119,10 +119,9 @@ class AsJSONTransform extends BaseVisitor { // @ts-ignore this.currentClass.setDataStmts.push( `if (key.equals(${JSON.stringify(aliasName)})) { - this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); - return; - } - ` + this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); + return; + }` ); if (member.initializer) { this.currentClass.initializeStmts.push( @@ -142,10 +141,9 @@ class AsJSONTransform extends BaseVisitor { // @ts-ignore this.currentClass.setDataStmts.push( `if (key.equals(${JSON.stringify(aliasName)})) { - this.${name} = __parseObjectValue<${type}>(data.slice(val_start, val_end), initializeDefaultValues); - return; - } - ` + this.${name} = __parseObjectValue<${type}>(data.slice(val_start, val_end), initializeDefaultValues); + return; + }` ); if (member.initializer) { this.currentClass.initializeStmts.push( @@ -159,10 +157,9 @@ class AsJSONTransform extends BaseVisitor { // @ts-ignore this.currentClass.setDataStmts.push( `if (key.equals(${JSON.stringify(aliasName)})) { - this.${name} = __parseObjectValue<${type}>(val_start ? data.slice(val_start, val_end) : data, initializeDefaultValues); - return; - } - ` + this.${name} = __parseObjectValue<${type}>(val_start ? data.slice(val_start, val_end) : data, initializeDefaultValues); + return; + }` ); if (member.initializer) { this.currentClass.initializeStmts.push( @@ -185,24 +182,19 @@ class AsJSONTransform extends BaseVisitor { serializeFunc = ` @inline __JSON_Serialize(): string { return \`{${this.currentClass.encodeStmts.join("")}}\`; - } - `; + }`; } else { serializeFunc = ` @inline __JSON_Serialize(): string { return "{}"; - } - `; + }`; } // Odd behavior here... When pairing this transform with asyncify, having @inline on __JSON_Set_Key with a generic will cause it to freeze. // Binaryen cannot predict and add/mangle code when it is genericed. const setKeyFunc = ` __JSON_Set_Key<__JSON_Key_Type>(key: __JSON_Key_Type, data: string, val_start: i32, val_end: i32, initializeDefaultValues: boolean): void { - ${ - // @ts-ignore - this.currentClass.setDataStmts.join("") - } + ${this.currentClass.setDataStmts.join("\n ")} } `; From 48b87df7dadfc3881085b27df60fd67da62cd471 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 16:27:06 -0800 Subject: [PATCH 25/30] minor --- transform/lib/index.js | 5 ++++- transform/src/index.ts | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/transform/lib/index.js b/transform/lib/index.js index e597e8ad..c5779110 100644 --- a/transform/lib/index.js +++ b/transform/lib/index.js @@ -179,7 +179,10 @@ class AsJSONTransform extends BaseVisitor { const initializeMethod = SimpleParser.parseClassMember(initializeFunc, node); node.members.push(initializeMethod); this.schemasList.push(this.currentClass); - //console.log(toString(node)); + // Uncomment to see the generated code for debugging. + // console.log(serializeFunc); + // console.log(setKeyFunc); + // console.log(initializeFunc); } visitSource(node) { super.visitSource(node); diff --git a/transform/src/index.ts b/transform/src/index.ts index 1c3fd733..d3ddeb2b 100644 --- a/transform/src/index.ts +++ b/transform/src/index.ts @@ -221,7 +221,11 @@ class AsJSONTransform extends BaseVisitor { node.members.push(initializeMethod); this.schemasList.push(this.currentClass); - //console.log(toString(node)); + + // Uncomment to see the generated code for debugging. + // console.log(serializeFunc); + // console.log(setKeyFunc); + // console.log(initializeFunc); } visitSource(node: Source): void { super.visitSource(node); From c64ea00bbc2f6960cebf12ff49043277a9167bee Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 16:54:11 -0800 Subject: [PATCH 26/30] perf: remove generic and inline function --- assembly/src/json.ts | 14 +++++++------- transform/lib/index.js | 24 ++++++++++++++++++++---- transform/src/index.ts | 35 +++++++++++++++++++++++++++++------ 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 23e81c49..68f9e3b8 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -466,7 +466,7 @@ export namespace JSON { if (depth === 0) { ++arrayValueIndex; // @ts-ignore - schema.__JSON_Set_Key>(key, data, outerLoopIndex, arrayValueIndex, initializeDefaultValues); + schema.__JSON_Set_Key(key, data, outerLoopIndex, arrayValueIndex, initializeDefaultValues); outerLoopIndex = arrayValueIndex; isKey = false; break; @@ -487,7 +487,7 @@ export namespace JSON { if (depth === 0) { ++objectValueIndex; // @ts-ignore - schema.__JSON_Set_Key>(key, data, outerLoopIndex, objectValueIndex, initializeDefaultValues); + schema.__JSON_Set_Key(key, data, outerLoopIndex, objectValueIndex, initializeDefaultValues); outerLoopIndex = objectValueIndex; isKey = false; break; @@ -517,7 +517,7 @@ export namespace JSON { } else { const value = parseString(data, outerLoopIndex - 1, stringValueIndex); // @ts-ignore - schema.__JSON_Set_Key>(key, value, 0, value.length, initializeDefaultValues); + schema.__JSON_Set_Key(key, value, 0, value.length, initializeDefaultValues); isKey = false; } outerLoopIndex = ++stringValueIndex; @@ -533,7 +533,7 @@ export namespace JSON { unsafeCharCodeAt(data, ++outerLoopIndex) === lCode ) { // @ts-ignore - schema.__JSON_Set_Key>(key, nullWord, 0, 4, initializeDefaultValues); + schema.__JSON_Set_Key(key, nullWord, 0, 4, initializeDefaultValues); isKey = false; } else if ( char === tCode && @@ -542,7 +542,7 @@ export namespace JSON { unsafeCharCodeAt(data, ++outerLoopIndex) === eCode ) { // @ts-ignore - schema.__JSON_Set_Key>(key, trueWord, 0, 4, initializeDefaultValues); + schema.__JSON_Set_Key(key, trueWord, 0, 4, initializeDefaultValues); isKey = false; } else if ( char === fCode && @@ -552,7 +552,7 @@ export namespace JSON { unsafeCharCodeAt(data, ++outerLoopIndex) === eCode ) { // @ts-ignore - schema.__JSON_Set_Key>(key, falseWord, 0, 5, initializeDefaultValues); + schema.__JSON_Set_Key(key, falseWord, 0, 5, initializeDefaultValues); isKey = false; } else if ((char >= 48 && char <= 57) || char === 45) { let numberValueIndex = ++outerLoopIndex; @@ -560,7 +560,7 @@ export namespace JSON { const char = unsafeCharCodeAt(data, numberValueIndex); if (char === commaCode || char === rightBraceCode || isSpace(char)) { // @ts-ignore - schema.__JSON_Set_Key>(key, data, outerLoopIndex - 1, numberValueIndex, initializeDefaultValues); + schema.__JSON_Set_Key(key, data, outerLoopIndex - 1, numberValueIndex, initializeDefaultValues); outerLoopIndex = numberValueIndex; isKey = false; break; diff --git a/transform/lib/index.js b/transform/lib/index.js index c5779110..08d2ab22 100644 --- a/transform/lib/index.js +++ b/transform/lib/index.js @@ -1,3 +1,4 @@ +import { Parser, Source, Tokenizer, } from "assemblyscript/dist/assemblyscript.js"; import { toString, isStdlib } from "visitor-as/dist/utils.js"; import { BaseVisitor, SimpleParser } from "visitor-as/dist/index.js"; import { Transform } from "assemblyscript/dist/transform.js"; @@ -17,7 +18,7 @@ class AsJSONTransform extends BaseVisitor { constructor() { super(...arguments); this.schemasList = []; - this.sources = []; + this.sources = new Set(); } visitMethodDeclaration() { } visitClassDeclaration(node) { @@ -152,10 +153,8 @@ class AsJSONTransform extends BaseVisitor { return "{}"; }`; } - // Odd behavior here... When pairing this transform with asyncify, having @inline on __JSON_Set_Key with a generic will cause it to freeze. - // Binaryen cannot predict and add/mangle code when it is genericed. const setKeyFunc = ` - __JSON_Set_Key<__JSON_Key_Type>(key: __JSON_Key_Type, data: string, val_start: i32, val_end: i32, initializeDefaultValues: boolean): void { + @inline __JSON_Set_Key(key: __Virtual, data: string, val_start: i32, val_end: i32, initializeDefaultValues: boolean): void { ${this.currentClass.setDataStmts.join("\n ")} } `; @@ -179,6 +178,7 @@ class AsJSONTransform extends BaseVisitor { const initializeMethod = SimpleParser.parseClassMember(initializeFunc, node); node.members.push(initializeMethod); this.schemasList.push(this.currentClass); + this.sources.add(node.name.range.source); // Uncomment to see the generated code for debugging. // console.log(serializeFunc); // console.log(setKeyFunc); @@ -186,6 +186,22 @@ class AsJSONTransform extends BaseVisitor { } visitSource(node) { super.visitSource(node); + // Only add the import statement to sources that have JSON decorated classes. + if (!this.sources.has(node)) { + return; + } + // Note, the following one liner would be easier, but it fails with an assertion error + // because as-virtual's SimpleParser doesn't set the parser.currentSource correctly. + // + // const stmt = SimpleParser.parseTopLevelStatement('import { Virtual as __Virtual } from "as-virtual/assembly";'); + // ... So we have to do it the long way: + const s = 'import { Virtual as __Virtual } from "as-virtual/assembly";'; + const t = new Tokenizer(new Source(0 /* SourceKind.User */, "index.ts", s)); + const p = new Parser(); + p.currentSource = t.source; + const stmt = p.parseTopLevelStatement(t); + // Add the import statement to the top of the source. + node.statements.unshift(stmt); } } export default class Transformer extends Transform { diff --git a/transform/src/index.ts b/transform/src/index.ts index d3ddeb2b..c3d1cfc0 100644 --- a/transform/src/index.ts +++ b/transform/src/index.ts @@ -1,9 +1,12 @@ import { ClassDeclaration, FieldDeclaration, - Source, Parser, -} from "assemblyscript/dist/assemblyscript"; + Source, + SourceKind, + Tokenizer, +} from "assemblyscript/dist/assemblyscript.js"; + import { toString, isStdlib } from "visitor-as/dist/utils.js"; import { BaseVisitor, SimpleParser } from "visitor-as/dist/index.js"; import { Transform } from "assemblyscript/dist/transform.js"; @@ -23,7 +26,7 @@ class SchemaData { class AsJSONTransform extends BaseVisitor { public schemasList: SchemaData[] = []; public currentClass!: SchemaData; - public sources: Source[] = []; + public sources = new Set(); visitMethodDeclaration(): void { } visitClassDeclaration(node: ClassDeclaration): void { @@ -190,10 +193,8 @@ class AsJSONTransform extends BaseVisitor { }`; } - // Odd behavior here... When pairing this transform with asyncify, having @inline on __JSON_Set_Key with a generic will cause it to freeze. - // Binaryen cannot predict and add/mangle code when it is genericed. const setKeyFunc = ` - __JSON_Set_Key<__JSON_Key_Type>(key: __JSON_Key_Type, data: string, val_start: i32, val_end: i32, initializeDefaultValues: boolean): void { + @inline __JSON_Set_Key(key: __Virtual, data: string, val_start: i32, val_end: i32, initializeDefaultValues: boolean): void { ${this.currentClass.setDataStmts.join("\n ")} } `; @@ -221,14 +222,36 @@ class AsJSONTransform extends BaseVisitor { node.members.push(initializeMethod); this.schemasList.push(this.currentClass); + this.sources.add(node.name.range.source); // Uncomment to see the generated code for debugging. // console.log(serializeFunc); // console.log(setKeyFunc); // console.log(initializeFunc); } + visitSource(node: Source): void { super.visitSource(node); + + // Only add the import statement to sources that have JSON decorated classes. + if (!this.sources.has(node)) { + return; + } + + // Note, the following one liner would be easier, but it fails with an assertion error + // because as-virtual's SimpleParser doesn't set the parser.currentSource correctly. + // + // const stmt = SimpleParser.parseTopLevelStatement('import { Virtual as __Virtual } from "as-virtual/assembly";'); + + // ... So we have to do it the long way: + const s = 'import { Virtual as __Virtual } from "as-virtual/assembly";' + const t = new Tokenizer(new Source(SourceKind.User, "index.ts", s)); + const p = new Parser(); + p.currentSource = t.source; + const stmt = p.parseTopLevelStatement(t)!; + + // Add the import statement to the top of the source. + node.statements.unshift(stmt); } } From 01f57fa7144dc33e03b4a59d5bed92ff6f66a8db Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 16:54:24 -0800 Subject: [PATCH 27/30] workaround as-pect bug --- assembly/__tests__/as-json.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/assembly/__tests__/as-json.spec.ts b/assembly/__tests__/as-json.spec.ts index 9df2bdf3..4aa0cd23 100644 --- a/assembly/__tests__/as-json.spec.ts +++ b/assembly/__tests__/as-json.spec.ts @@ -596,6 +596,10 @@ describe("Ser/de special strings in object keys", () => { canSerde(o, s); }); + // Something buggy in as-pect needs a dummy value reflected here + // or the subsequent test fails. It's not used in any test. + Reflect.toReflectedValue(0); + it("should ser/de escape sequences in map key", () => { const m = new Map(); m.set('a\\\t"\x02b', 'abc'); From b2de43309b6b8ff9c4414dc83595d1620e897c30 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 17:44:25 -0800 Subject: [PATCH 28/30] perf: unchecked array access --- assembly/src/json.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index 68f9e3b8..b0732518 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -117,9 +117,9 @@ export namespace JSON { let keys = data.keys(); let values = data.values(); for (let i = 0; i < data.size; i++) { - result.write(serializeString(keys[i].toString())); + result.write(serializeString(unchecked(keys[i]).toString())); result.writeCodePoint(colonCode); - result.write(JSON.stringify(values[i])); + result.write(JSON.stringify(unchecked(values[i]))); if (i < data.size - 1) { result.writeCodePoint(commaCode); } From fbcdc865712795011b2e9d55f606b5871fd6a1d7 Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 17:51:14 -0800 Subject: [PATCH 29/30] Avoid string concatenation --- assembly/src/json.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/assembly/src/json.ts b/assembly/src/json.ts index b0732518..1a4e3cc8 100644 --- a/assembly/src/json.ts +++ b/assembly/src/json.ts @@ -73,7 +73,7 @@ export namespace JSON { // @ts-ignore: Hidden function return data.__JSON_Serialize(); } else if (data instanceof Date) { - return "\"" + data.toISOString() + "\""; + return `"${data.toISOString()}"`; } else if (isArrayLike()) { // @ts-ignore if (data.length == 0) { @@ -329,7 +329,8 @@ export namespace JSON { default: { // all chars 0-31 must be encoded as a four digit unicode escape sequence // \u0000 to \u000f handled here - result.write("\\u000" + char.toString(16)); + result.write("\\u000"); + result.write(char.toString(16)); break; } } @@ -338,7 +339,8 @@ export namespace JSON { last = i + 1; // all chars 0-31 must be encoded as a four digit unicode escape sequence // \u0010 to \u001f handled here - result.write("\\u00" + char.toString(16)); + result.write("\\u00"); + result.write(char.toString(16)); } } result.write(data, last); From a7056853b27bb9b638157654272820afc547be9e Mon Sep 17 00:00:00 2001 From: Matt Johnson-Pint Date: Wed, 21 Feb 2024 18:09:24 -0800 Subject: [PATCH 30/30] Fix error with ` in object key alias --- assembly/__tests__/as-json.spec.ts | 8 ++++---- transform/lib/index.js | 11 ++++++++--- transform/src/index.ts | 12 +++++++++--- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/assembly/__tests__/as-json.spec.ts b/assembly/__tests__/as-json.spec.ts index 4aa0cd23..f2bc840f 100644 --- a/assembly/__tests__/as-json.spec.ts +++ b/assembly/__tests__/as-json.spec.ts @@ -580,19 +580,19 @@ describe("Ser/de special strings in object keys", () => { it("should ser/de escape sequences in key of object with int value", () => { const o: ObjWithStrangeKey = { data: 123 }; - const s = '{"a\\\\\\t\\"\\u0002b":123}'; + const s = '{"a\\\\\\t\\"\\u0002b`c":123}'; canSerde(o, s); }); it("should ser/de escape sequences in key of object with float value", () => { const o: ObjWithStrangeKey = { data: 123.4 }; - const s = '{"a\\\\\\t\\"\\u0002b":123.4}'; + const s = '{"a\\\\\\t\\"\\u0002b`c":123.4}'; canSerde(o, s); }); it("should ser/de escape sequences in key of object with string value", () => { const o: ObjWithStrangeKey = { data: "abc" }; - const s = '{"a\\\\\\t\\"\\u0002b":"abc"}'; + const s = '{"a\\\\\\t\\"\\u0002b`c":"abc"}'; canSerde(o, s); }); @@ -636,6 +636,6 @@ class ObjectWithFloatArray { @json class ObjWithStrangeKey { - @alias('a\\\t"\x02b') + @alias('a\\\t"\x02b`c') data!: T; } diff --git a/transform/lib/index.js b/transform/lib/index.js index 08d2ab22..4e43b209 100644 --- a/transform/lib/index.js +++ b/transform/lib/index.js @@ -99,7 +99,7 @@ class AsJSONTransform extends BaseVisitor { "u64", "i64", ].includes(type.toLowerCase())) { - this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}},`); + this.currentClass.encodeStmts.push(`${encodeKey(aliasName)}:\${this.${name}},`); // @ts-ignore this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __atoi_fast<${type}>(data, val_start << 1, val_end << 1); @@ -114,7 +114,7 @@ class AsJSONTransform extends BaseVisitor { "f32", "f64", ].includes(type.toLowerCase())) { - this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${this.${name}},`); + this.currentClass.encodeStmts.push(`${encodeKey(aliasName)}:\${this.${name}},`); // @ts-ignore this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __parseObjectValue<${type}>(data.slice(val_start, val_end), initializeDefaultValues); @@ -125,7 +125,7 @@ class AsJSONTransform extends BaseVisitor { } } else { - this.currentClass.encodeStmts.push(`${JSON.stringify(aliasName).replace(/\\/g, '\\\\')}:\${JSON.stringify<${type}>(this.${name})},`); + this.currentClass.encodeStmts.push(`${encodeKey(aliasName)}:\${JSON.stringify<${type}>(this.${name})},`); // @ts-ignore this.currentClass.setDataStmts.push(`if (key.equals(${JSON.stringify(aliasName)})) { this.${name} = __parseObjectValue<${type}>(val_start ? data.slice(val_start, val_end) : data, initializeDefaultValues); @@ -204,6 +204,11 @@ class AsJSONTransform extends BaseVisitor { node.statements.unshift(stmt); } } +function encodeKey(aliasName) { + return JSON.stringify(aliasName) + .replace(/\\/g, "\\\\") + .replace(/\`/g, '\\`'); +} export default class Transformer extends Transform { // Trigger the transform after parse. afterParse(parser) { diff --git a/transform/src/index.ts b/transform/src/index.ts index c3d1cfc0..c84ba2ee 100644 --- a/transform/src/index.ts +++ b/transform/src/index.ts @@ -117,7 +117,7 @@ class AsJSONTransform extends BaseVisitor { ].includes(type.toLowerCase()) ) { this.currentClass.encodeStmts.push( - `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${this.${name}},` + `${encodeKey(aliasName)}:\${this.${name}},` ); // @ts-ignore this.currentClass.setDataStmts.push( @@ -139,7 +139,7 @@ class AsJSONTransform extends BaseVisitor { ].includes(type.toLowerCase()) ) { this.currentClass.encodeStmts.push( - `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${this.${name}},` + `${encodeKey(aliasName)}:\${this.${name}},` ); // @ts-ignore this.currentClass.setDataStmts.push( @@ -155,7 +155,7 @@ class AsJSONTransform extends BaseVisitor { } } else { this.currentClass.encodeStmts.push( - `${JSON.stringify(aliasName).replace(/\\/g,'\\\\')}:\${JSON.stringify<${type}>(this.${name})},` + `${encodeKey(aliasName)}:\${JSON.stringify<${type}>(this.${name})},` ); // @ts-ignore this.currentClass.setDataStmts.push( @@ -255,6 +255,12 @@ class AsJSONTransform extends BaseVisitor { } } +function encodeKey(aliasName: string): string { + return JSON.stringify(aliasName) + .replace(/\\/g, "\\\\") + .replace(/\`/g, '\\`'); +} + export default class Transformer extends Transform { // Trigger the transform after parse. afterParse(parser: Parser): void {