From 76c0618771daea2eaa382c32b973366c972ab686 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 8 Oct 2023 18:55:07 +0200 Subject: [PATCH 01/11] feat: sync back-references --- benchmark/index.js | 51 ++++++++++++++++++++++--------- benchmark/package.json | 2 +- src/index.test.ts | 67 ++++++++++++++++++++++++++++++++++++++--- src/sync/deserialize.ts | 43 +++++++++++++++++++++++--- src/sync/serialize.ts | 40 ++++++++++++------------ 5 files changed, 158 insertions(+), 45 deletions(-) diff --git a/benchmark/index.js b/benchmark/index.js index 87f72ba1..a04443a5 100644 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -1,11 +1,15 @@ // taken from https://github.com/Rich-Harris/superjson-and-devalue import ARSON from "arson"; -import { parse, stringify, uneval } from "devalue"; +import * as devalue from "devalue"; import c from "kleur"; import * as superjson from "superjson"; import { createTson, tsonDate, tsonRegExp, tsonSet } from "tupleson"; +const time_formatter = new Intl.NumberFormat('en-US', { unit: 'millisecond', style: 'unit' }); +const size_formatter = new Intl.NumberFormat('en-US', { unit: 'byte', style: 'unit' }); +const number_formatter = new Intl.NumberFormat('en-US'); + const obj = { array: [{ foo: 1 }, { bar: 2 }, { baz: 3 }], date: new Date(), @@ -16,35 +20,37 @@ const obj = { }; // circular references are not supported by tupleson -// obj.self = obj; +obj.self = obj; const tson = createTson({ types: [tsonDate, tsonRegExp, tsonSet], }); const superjson_serialized = superjson.stringify(obj); -const devalue_unevaled = uneval(obj); -const devalue_stringified = stringify(obj); +const devalue_unevaled = devalue.uneval(obj); +const devalue_stringified = devalue.stringify(obj); const arson_stringified = ARSON.stringify(obj); const tson_serialized = tson.stringify(obj); +console.log('-- SERIALIZED SIZE --\n') + console.log( - `superjson output: ${c.bold().cyan(superjson_serialized.length)} bytes`, + `superjson output: ${c.bold().cyan(size_formatter.format(superjson_serialized.length))}`, ); -console.log(`tson output: ${c.bold().cyan(tson_serialized.length)} bytes`); +console.log(`tson output: ${c.bold().cyan(size_formatter.format(tson_serialized.length))}`); // console.log(superjson_serialized); console.log( - `devalue.uneval output: ${c.bold().cyan(devalue_unevaled.length)} bytes`, + `devalue.uneval output: ${c.bold().cyan(size_formatter.format(devalue_unevaled.length))}`, ); // console.log(devalue_unevaled); console.log( `devalue.stringify output: ${c .bold() - .cyan(devalue_stringified.length)} bytes`, + .cyan(size_formatter.format(devalue_stringified.length))}`, ); // console.log(devalue_stringified); -console.log(`arson output: ${c.bold().cyan(arson_stringified.length)} bytes`); +console.log(`arson output: ${c.bold().cyan(size_formatter.format(arson_stringified.length))}`); // console.log(arson_stringified); // const superjson_deserialized = superjson.parse(superjson_serialized); @@ -53,31 +59,46 @@ console.log(`arson output: ${c.bold().cyan(arson_stringified.length)} bytes`); const iterations = 1e6; function test(fn, label = fn.toString()) { - const start = Date.now(); console.log(); console.log(c.bold(label)); + global.gc(); // force garbage collection before each test let i = iterations; + const before_snap = process.memoryUsage(); + const start = Date.now(); while (i--) { fn(); } - + const delta = Date.now() - start; + const after_snap = process.memoryUsage(); console.log( - `${iterations} iterations in ${c.bold().cyan(Date.now() - start)}ms`, + `${number_formatter.format(iterations)} iterations in ${c.bold().cyan(time_formatter.format(delta))}`, ); + // log memory usage delta + for (const key in after_snap) { + const before = before_snap[key]; + const after = after_snap[key]; + const diff = after - before; + const color = diff < 0 ? c.green : c.red; + console.log(` ${key}: ${color(size_formatter.format(diff))}`); + } } +console.log('\n-- SERIALIZATION DURATION --') + // serialization test(() => superjson.stringify(obj)); test(() => tson.stringify(obj)); -test(() => uneval(obj)); -test(() => stringify(obj)); +test(() => devalue.uneval(obj)); +test(() => devalue.stringify(obj)); test(() => ARSON.stringify(obj)); +console.log('\n-- DESERIALIZATION DURATION --') + // deserialization test(() => superjson.parse(superjson_serialized)); test(() => tson.parse(tson_serialized)); test(() => eval(`(${devalue_unevaled})`)); test(() => ARSON.parse(arson_stringified)); -test(() => parse(devalue_stringified)); +test(() => devalue.parse(devalue_stringified)); console.log(); diff --git a/benchmark/package.json b/benchmark/package.json index 9431843b..447710bb 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -7,7 +7,7 @@ "type": "module", "scripts": { "postinstall": "cd ../ && pnpm run build", - "start": "node index.js" + "start": "node --expose-gc --max-old-space-size=8192 index.js" }, "dependencies": { "arson": "^0.2.6", diff --git a/src/index.test.ts b/src/index.test.ts index f17c20d4..d7aa4b1a 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "vitest"; -import { TsonOptions, TsonType, createTson, createTsonAsync } from "./index.js"; +import { TsonOptions, TsonType, createTson, createTsonAsync, tsonDate, tsonPromise } from "./index.js"; import { expectError, waitError } from "./internals/testUtils.js"; test("multiple handlers for primitive string found", () => { @@ -33,20 +33,77 @@ test("duplicate keys", () => { ); }); -test("no max call stack", () => { +test("back-reference: circular object reference", () => { const t = createTson({ types: [], }); const expected: Record = {}; expected["a"] = expected; + expected["b"] = expected; - // stringify should fail b/c of JSON limitations - const err = expectError(() => t.stringify(expected)); + const str = t.stringify(expected) + const res = t.parse(str); - expect(err.message).toMatchInlineSnapshot('"Circular reference detected"'); + expect(res).toEqual(expected); }); +test("back-reference: circular array reference", () => { + const t = createTson({ + types: [], + }); + + const expected: unknown[] = []; + expected[0] = expected; + expected[1] = expected; + + const str = t.stringify(expected) + const res = t.parse(str); + + expect(res).toEqual(expected); +}); + +test("back-reference: non-circular complex reference", () => { + const t = createTson({ + types: [tsonDate], + }); + + const expected: Record = {}; + expected["a"] = {} + expected["b"] = expected["a"] + expected["c"] = new Date() + expected["d"] = expected["c"] + + const str = t.stringify(expected) + const res = t.parse(str); + + expect(res["b"]).toBe(res["a"]); + expect(res["d"]).toBe(res["c"]); +}); + +/** + * WILL NOT WORK: the async serialize/deserialize functions haven't + * been adapted to handle back-references yet + */ +// test("async: back-reference", async () => { +// const t = createTsonAsync({ +// types: [tsonPromise], +// }); + +// const needle = {} + +// const expected = { +// a: needle, +// b: Promise.resolve(needle), +// }; + +// const str = await t.stringify(expected); +// const res = await t.parse(str); + +// expect(res).toEqual(expected); +// expect(res.a).toBe(await res.b); +// }) + test("allow duplicate objects", () => { const t = createTson({ types: [], diff --git a/src/sync/deserialize.ts b/src/sync/deserialize.ts index da27568a..bc5857db 100644 --- a/src/sync/deserialize.ts +++ b/src/sync/deserialize.ts @@ -9,7 +9,7 @@ import { TsonTransformerSerializeDeserialize, } from "./syncTypes.js"; -type WalkFn = (value: unknown) => unknown; +type WalkFn = (value: unknown, path?: (string|number)[]) => unknown; type WalkerFactory = (nonce: TsonNonce) => WalkFn; type AnyTsonTransformerSerializeDeserialize = @@ -30,17 +30,52 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { } const walker: WalkerFactory = (nonce) => { - const walk: WalkFn = (value) => { + const seen = new Map(); + const backrefs: [circular_key: string, origin_key: string][] = []; + + const coreWalk: WalkFn = (value, path = []) => { + const key = path.join(nonce); if (isTsonTuple(value, nonce)) { const [type, serializedValue] = value; + if (type === 'CIRCULAR') { + backrefs.push([key, serializedValue as string]); + return; + } // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const transformer = typeByKey[type]!; - return transformer.deserialize(walk(serializedValue)); + const parsed = transformer.deserialize(coreWalk(serializedValue, path)); + seen.set(key, parsed); + return parsed; } - return mapOrReturn(value, walk); + const parsed = mapOrReturn(value, (value, key) => coreWalk(value, [...path, key])); + if (parsed && typeof parsed === 'object') { + seen.set(key, parsed) + } + return parsed; }; + const walk: WalkFn = (value) => { + const res = coreWalk(value); + for (const [key, ref] of backrefs) { + const prev = seen.get(ref); + if (!prev) { + throw new Error(`Back-reference ${ref.split(nonce).join('.')} not found`); + } + const path = key.split(nonce); + let insertAt = res as any + try { + while (path.length > 1) { + insertAt = insertAt[path.shift()!]; + } + insertAt[path[0]!] = prev + } catch (cause) { + throw new Error(`Invalid path to back-reference ${ref.split(nonce).join('.')}`, { cause }); + } + } + return res + } + return walk; }; diff --git a/src/sync/serialize.ts b/src/sync/serialize.ts index bc1d2d6f..4f67af2c 100644 --- a/src/sync/serialize.ts +++ b/src/sync/serialize.ts @@ -1,4 +1,4 @@ -import { TsonCircularReferenceError } from "../errors.js"; +// import { TsonCircularReferenceError } from "../errors.js"; import { GetNonce, getDefaultNonce } from "../internals/getNonce.js"; import { mapOrReturn } from "../internals/mapOrReturn.js"; import { @@ -13,7 +13,7 @@ import { TsonTypeTesterPrimitive, } from "./syncTypes.js"; -type WalkFn = (value: unknown) => unknown; +type WalkFn = (value: unknown, path?: (string|number)[]) => unknown; type WalkerFactory = (nonce: TsonNonce) => WalkFn; function getHandlers(opts: TsonOptions) { @@ -56,26 +56,13 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { const [getNonce, nonPrimitive, byPrimitive] = getHandlers(opts); const walker: WalkerFactory = (nonce) => { - const seen = new WeakSet(); + const seen = new WeakMap(); const cache = new WeakMap(); - const walk: WalkFn = (value) => { + const walk: WalkFn = (value, path = []) => { const type = typeof value; const isComplex = !!value && type === "object"; - if (isComplex) { - if (seen.has(value)) { - const cached = cache.get(value); - if (!cached) { - throw new TsonCircularReferenceError(value); - } - - return cached; - } - - seen.add(value); - } - const cacheAndReturn = (result: unknown) => { if (isComplex) { cache.set(value, result); @@ -84,6 +71,19 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { return result; }; + if (isComplex) { + const prev = seen.get(value); + if (prev) { + return [ + "CIRCULAR", + prev.join(nonce), + nonce, + ] as TsonTuple; + } + + seen.set(value, path) + } + const primitiveHandler = byPrimitive[type]; if ( primitiveHandler && @@ -92,7 +92,7 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { return cacheAndReturn([ primitiveHandler.key, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - walk(primitiveHandler.serialize!(value)), + walk(primitiveHandler.serialize!(value), path), nonce, ] as TsonTuple); } @@ -102,13 +102,13 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { return cacheAndReturn([ handler.key, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - walk(handler.serialize!(value)), + walk(handler.serialize!(value), path), nonce, ] as TsonTuple); } } - return cacheAndReturn(mapOrReturn(value, walk)); + return cacheAndReturn(mapOrReturn(value, (value, key) => walk(value, [...path, key]))); }; return walk; From ccd992520d23d984461e657b00546b1c3fb3761c Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 8 Oct 2023 19:09:58 +0200 Subject: [PATCH 02/11] lint --- benchmark/index.js | 43 +++++++++++++++++++++++++++++----------- src/index.test.ts | 24 +++++++++++++--------- src/sync/deserialize.ts | 44 ++++++++++++++++++++++++++++------------- src/sync/serialize.ts | 16 +++++++-------- 4 files changed, 84 insertions(+), 43 deletions(-) diff --git a/benchmark/index.js b/benchmark/index.js index a04443a5..c64ea20a 100644 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -6,9 +6,15 @@ import c from "kleur"; import * as superjson from "superjson"; import { createTson, tsonDate, tsonRegExp, tsonSet } from "tupleson"; -const time_formatter = new Intl.NumberFormat('en-US', { unit: 'millisecond', style: 'unit' }); -const size_formatter = new Intl.NumberFormat('en-US', { unit: 'byte', style: 'unit' }); -const number_formatter = new Intl.NumberFormat('en-US'); +const time_formatter = new Intl.NumberFormat("en-US", { + style: "unit", + unit: "millisecond", +}); +const size_formatter = new Intl.NumberFormat("en-US", { + style: "unit", + unit: "byte", +}); +const number_formatter = new Intl.NumberFormat("en-US"); const obj = { array: [{ foo: 1 }, { bar: 2 }, { baz: 3 }], @@ -32,16 +38,24 @@ const devalue_stringified = devalue.stringify(obj); const arson_stringified = ARSON.stringify(obj); const tson_serialized = tson.stringify(obj); -console.log('-- SERIALIZED SIZE --\n') +console.log("-- SERIALIZED SIZE --\n"); console.log( - `superjson output: ${c.bold().cyan(size_formatter.format(superjson_serialized.length))}`, + `superjson output: ${c + .bold() + .cyan(size_formatter.format(superjson_serialized.length))}`, ); -console.log(`tson output: ${c.bold().cyan(size_formatter.format(tson_serialized.length))}`); +console.log( + `tson output: ${c + .bold() + .cyan(size_formatter.format(tson_serialized.length))}`, +); // console.log(superjson_serialized); console.log( - `devalue.uneval output: ${c.bold().cyan(size_formatter.format(devalue_unevaled.length))}`, + `devalue.uneval output: ${c + .bold() + .cyan(size_formatter.format(devalue_unevaled.length))}`, ); // console.log(devalue_unevaled); console.log( @@ -50,7 +64,11 @@ console.log( .cyan(size_formatter.format(devalue_stringified.length))}`, ); // console.log(devalue_stringified); -console.log(`arson output: ${c.bold().cyan(size_formatter.format(arson_stringified.length))}`); +console.log( + `arson output: ${c + .bold() + .cyan(size_formatter.format(arson_stringified.length))}`, +); // console.log(arson_stringified); // const superjson_deserialized = superjson.parse(superjson_serialized); @@ -68,10 +86,13 @@ function test(fn, label = fn.toString()) { while (i--) { fn(); } + const delta = Date.now() - start; const after_snap = process.memoryUsage(); console.log( - `${number_formatter.format(iterations)} iterations in ${c.bold().cyan(time_formatter.format(delta))}`, + `${number_formatter.format(iterations)} iterations in ${c + .bold() + .cyan(time_formatter.format(delta))}`, ); // log memory usage delta for (const key in after_snap) { @@ -83,7 +104,7 @@ function test(fn, label = fn.toString()) { } } -console.log('\n-- SERIALIZATION DURATION --') +console.log("\n-- SERIALIZATION DURATION --"); // serialization test(() => superjson.stringify(obj)); @@ -92,7 +113,7 @@ test(() => devalue.uneval(obj)); test(() => devalue.stringify(obj)); test(() => ARSON.stringify(obj)); -console.log('\n-- DESERIALIZATION DURATION --') +console.log("\n-- DESERIALIZATION DURATION --"); // deserialization test(() => superjson.parse(superjson_serialized)); diff --git a/src/index.test.ts b/src/index.test.ts index d7aa4b1a..caebe3bd 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,7 +1,13 @@ import { expect, test } from "vitest"; -import { TsonOptions, TsonType, createTson, createTsonAsync, tsonDate, tsonPromise } from "./index.js"; -import { expectError, waitError } from "./internals/testUtils.js"; +import { + TsonOptions, + TsonType, + createTson, + createTsonAsync, + tsonDate, +} from "./index.js"; +import { waitError } from "./internals/testUtils.js"; test("multiple handlers for primitive string found", () => { const stringHandler: TsonType = { @@ -42,7 +48,7 @@ test("back-reference: circular object reference", () => { expected["a"] = expected; expected["b"] = expected; - const str = t.stringify(expected) + const str = t.stringify(expected); const res = t.parse(str); expect(res).toEqual(expected); @@ -57,7 +63,7 @@ test("back-reference: circular array reference", () => { expected[0] = expected; expected[1] = expected; - const str = t.stringify(expected) + const str = t.stringify(expected); const res = t.parse(str); expect(res).toEqual(expected); @@ -69,12 +75,12 @@ test("back-reference: non-circular complex reference", () => { }); const expected: Record = {}; - expected["a"] = {} - expected["b"] = expected["a"] - expected["c"] = new Date() - expected["d"] = expected["c"] + expected["a"] = {}; + expected["b"] = expected["a"]; + expected["c"] = new Date(); + expected["d"] = expected["c"]; - const str = t.stringify(expected) + const str = t.stringify(expected); const res = t.parse(str); expect(res["b"]).toBe(res["a"]); diff --git a/src/sync/deserialize.ts b/src/sync/deserialize.ts index bc5857db..2b1766aa 100644 --- a/src/sync/deserialize.ts +++ b/src/sync/deserialize.ts @@ -9,7 +9,7 @@ import { TsonTransformerSerializeDeserialize, } from "./syncTypes.js"; -type WalkFn = (value: unknown, path?: (string|number)[]) => unknown; +type WalkFn = (value: unknown, path?: (number | string)[]) => unknown; type WalkerFactory = (nonce: TsonNonce) => WalkFn; type AnyTsonTransformerSerializeDeserialize = @@ -32,26 +32,32 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { const walker: WalkerFactory = (nonce) => { const seen = new Map(); const backrefs: [circular_key: string, origin_key: string][] = []; - + const coreWalk: WalkFn = (value, path = []) => { const key = path.join(nonce); if (isTsonTuple(value, nonce)) { const [type, serializedValue] = value; - if (type === 'CIRCULAR') { + if (type === "CIRCULAR") { backrefs.push([key, serializedValue as string]); return; } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const transformer = typeByKey[type]!; - const parsed = transformer.deserialize(coreWalk(serializedValue, path)); + const parsed = transformer.deserialize( + coreWalk(serializedValue, path), + ) as unknown; seen.set(key, parsed); return parsed; } - const parsed = mapOrReturn(value, (value, key) => coreWalk(value, [...path, key])); - if (parsed && typeof parsed === 'object') { - seen.set(key, parsed) + const parsed = mapOrReturn(value, (value, key) => + coreWalk(value, [...path, key]), + ); + if (parsed && typeof parsed === "object") { + seen.set(key, parsed); } + return parsed; }; @@ -60,21 +66,31 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { for (const [key, ref] of backrefs) { const prev = seen.get(ref); if (!prev) { - throw new Error(`Back-reference ${ref.split(nonce).join('.')} not found`); + throw new Error( + `Back-reference ${ref.split(nonce).join(".")} not found`, + ); } + const path = key.split(nonce); - let insertAt = res as any + let insertAt = res; try { while (path.length > 1) { - insertAt = insertAt[path.shift()!]; + //@ts-expect-error -- insertAt is unknown and not checked, but if it passed serialization, it should be an object + insertAt = insertAt[path.shift()]; } - insertAt[path[0]!] = prev + + //@ts-expect-error -- see above, + if it passed serialization, path should be length 1 at this point + insertAt[path[0]] = prev; } catch (cause) { - throw new Error(`Invalid path to back-reference ${ref.split(nonce).join('.')}`, { cause }); + throw new Error( + `Invalid path to back-reference ${ref.split(nonce).join(".")}`, + { cause }, + ); } } - return res - } + + return res; + }; return walk; }; diff --git a/src/sync/serialize.ts b/src/sync/serialize.ts index 4f67af2c..f0798d6a 100644 --- a/src/sync/serialize.ts +++ b/src/sync/serialize.ts @@ -13,7 +13,7 @@ import { TsonTypeTesterPrimitive, } from "./syncTypes.js"; -type WalkFn = (value: unknown, path?: (string|number)[]) => unknown; +type WalkFn = (value: unknown, path?: (number | string)[]) => unknown; type WalkerFactory = (nonce: TsonNonce) => WalkFn; function getHandlers(opts: TsonOptions) { @@ -56,7 +56,7 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { const [getNonce, nonPrimitive, byPrimitive] = getHandlers(opts); const walker: WalkerFactory = (nonce) => { - const seen = new WeakMap(); + const seen = new WeakMap(); const cache = new WeakMap(); const walk: WalkFn = (value, path = []) => { @@ -74,14 +74,10 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { if (isComplex) { const prev = seen.get(value); if (prev) { - return [ - "CIRCULAR", - prev.join(nonce), - nonce, - ] as TsonTuple; + return ["CIRCULAR", prev.join(nonce), nonce] as TsonTuple; } - seen.set(value, path) + seen.set(value, path); } const primitiveHandler = byPrimitive[type]; @@ -108,7 +104,9 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { } } - return cacheAndReturn(mapOrReturn(value, (value, key) => walk(value, [...path, key]))); + return cacheAndReturn( + mapOrReturn(value, (value, key) => walk(value, [...path, key])), + ); }; return walk; From 852d9c7bc4c2a61a6cf25b1b7f0c1f0ef87e64d7 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 8 Oct 2023 19:16:41 +0200 Subject: [PATCH 03/11] minor: better tests --- benchmark/index.js | 2 -- src/index.test.ts | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/benchmark/index.js b/benchmark/index.js index c64ea20a..2a5c4887 100644 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -24,8 +24,6 @@ const obj = { set: new Set([1, 2, 3]), xss: '', }; - -// circular references are not supported by tupleson obj.self = obj; const tson = createTson({ diff --git a/src/index.test.ts b/src/index.test.ts index caebe3bd..5ca36fe2 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -52,6 +52,8 @@ test("back-reference: circular object reference", () => { const res = t.parse(str); expect(res).toEqual(expected); + expect(res).toBe(res["a"]); + expect(res["b"]).toBe(res["a"]); }); test("back-reference: circular array reference", () => { @@ -67,6 +69,8 @@ test("back-reference: circular array reference", () => { const res = t.parse(str); expect(res).toEqual(expected); + expect(res).toBe(res[0]); + expect(res[1]).toBe(res[0]); }); test("back-reference: non-circular complex reference", () => { @@ -83,6 +87,7 @@ test("back-reference: non-circular complex reference", () => { const str = t.stringify(expected); const res = t.parse(str); + expect(res).toEqual(expected); expect(res["b"]).toBe(res["a"]); expect(res["d"]).toBe(res["c"]); }); From 47e2ef0d8924e9390e2c247f89ab0d474e7385e7 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 8 Oct 2023 19:22:49 +0200 Subject: [PATCH 04/11] spelling --- src/sync/deserialize.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/sync/deserialize.ts b/src/sync/deserialize.ts index 2b1766aa..1c89ed35 100644 --- a/src/sync/deserialize.ts +++ b/src/sync/deserialize.ts @@ -31,14 +31,14 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { const walker: WalkerFactory = (nonce) => { const seen = new Map(); - const backrefs: [circular_key: string, origin_key: string][] = []; + const references: [copyKey: string, originKey: string][] = []; const coreWalk: WalkFn = (value, path = []) => { const key = path.join(nonce); if (isTsonTuple(value, nonce)) { const [type, serializedValue] = value; if (type === "CIRCULAR") { - backrefs.push([key, serializedValue as string]); + references.push([key, serializedValue as string]); return; } @@ -63,15 +63,15 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { const walk: WalkFn = (value) => { const res = coreWalk(value); - for (const [key, ref] of backrefs) { - const prev = seen.get(ref); + for (const [copyKey, originKey] of references) { + const prev = seen.get(originKey); if (!prev) { throw new Error( - `Back-reference ${ref.split(nonce).join(".")} not found`, + `Back-reference ${originKey.split(nonce).join(".")} not found`, ); } - const path = key.split(nonce); + const path = copyKey.split(nonce); let insertAt = res; try { while (path.length > 1) { @@ -83,7 +83,7 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { insertAt[path[0]] = prev; } catch (cause) { throw new Error( - `Invalid path to back-reference ${ref.split(nonce).join(".")}`, + `Invalid path to reference insertion ${copyKey.split(nonce).join(".")}`, { cause }, ); } From 84d8925deac2cdc6c75eb8fb7efb2427c703fdfe Mon Sep 17 00:00:00 2001 From: KATT Date: Sun, 8 Oct 2023 20:05:51 +0200 Subject: [PATCH 05/11] added some tests and snapshots to look at --- src/index.test.ts | 116 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 4 deletions(-) diff --git a/src/index.test.ts b/src/index.test.ts index 5ca36fe2..fe235490 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -41,6 +41,7 @@ test("duplicate keys", () => { test("back-reference: circular object reference", () => { const t = createTson({ + nonce: () => "__tson", types: [], }); @@ -48,16 +49,37 @@ test("back-reference: circular object reference", () => { expected["a"] = expected; expected["b"] = expected; - const str = t.stringify(expected); + const str = t.stringify(expected, 2); const res = t.parse(str); expect(res).toEqual(expected); expect(res).toBe(res["a"]); expect(res["b"]).toBe(res["a"]); + + expect(str).toMatchInlineSnapshot( + ` + "{ + \\"json\\": { + \\"a\\": [ + \\"CIRCULAR\\", + \\"\\", + \\"__tson\\" + ], + \\"b\\": [ + \\"CIRCULAR\\", + \\"\\", + \\"__tson\\" + ] + }, + \\"nonce\\": \\"__tson\\" + }" + `, + ); }); test("back-reference: circular array reference", () => { const t = createTson({ + nonce: () => "__tson", types: [], }); @@ -65,31 +87,117 @@ test("back-reference: circular array reference", () => { expected[0] = expected; expected[1] = expected; - const str = t.stringify(expected); + const str = t.stringify(expected, 2); const res = t.parse(str); expect(res).toEqual(expected); expect(res).toBe(res[0]); expect(res[1]).toBe(res[0]); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + [ + \\"CIRCULAR\\", + \\"\\", + \\"__tson\\" + ], + [ + \\"CIRCULAR\\", + \\"\\", + \\"__tson\\" + ] + ], + \\"nonce\\": \\"__tson\\" + }" + `); }); test("back-reference: non-circular complex reference", () => { const t = createTson({ + nonce: () => "__tson", types: [tsonDate], }); const expected: Record = {}; expected["a"] = {}; expected["b"] = expected["a"]; - expected["c"] = new Date(); + expected["c"] = new Date(0); expected["d"] = expected["c"]; - const str = t.stringify(expected); + const str = t.stringify(expected, 2); const res = t.parse(str); expect(res).toEqual(expected); expect(res["b"]).toBe(res["a"]); expect(res["d"]).toBe(res["c"]); + + expect(str).toMatchInlineSnapshot( + ` + "{ + \\"json\\": { + \\"a\\": {}, + \\"b\\": [ + \\"CIRCULAR\\", + \\"a\\", + \\"__tson\\" + ], + \\"c\\": [ + \\"Date\\", + \\"1970-01-01T00:00:00.000Z\\", + \\"__tson\\" + ], + \\"d\\": [ + \\"CIRCULAR\\", + \\"c\\", + \\"__tson\\" + ] + }, + \\"nonce\\": \\"__tson\\" + }" + `, + ); +}); + +test("back-reference: grandparent reference", () => { + const t = createTson({ + nonce: () => "__tson", + types: [tsonDate], + }); + + const expected: Record = { + a: { + a: { + b: { + a: null, + }, + }, + }, + }; + expected["a"].a.b.a = expected["a"].a; + + const str = t.stringify(expected, 2); + const res = t.parse(str); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": { + \\"a\\": { + \\"a\\": { + \\"b\\": { + \\"a\\": [ + \\"CIRCULAR\\", + \\"a__tsona\\", + \\"__tson\\" + ] + } + } + } + }, + \\"nonce\\": \\"__tson\\" + }" + `); + expect(res).toEqual(expected); }); /** From e056d0a5a67b8b65c785d657cb86586473000131 Mon Sep 17 00:00:00 2001 From: KATT Date: Sun, 8 Oct 2023 20:06:23 +0200 Subject: [PATCH 06/11] tweak test snap --- src/index.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/index.test.ts b/src/index.test.ts index fe235490..cdf2129d 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -161,7 +161,7 @@ test("back-reference: non-circular complex reference", () => { test("back-reference: grandparent reference", () => { const t = createTson({ - nonce: () => "__tson", + nonce: () => "__tson__", types: [tsonDate], }); @@ -187,14 +187,14 @@ test("back-reference: grandparent reference", () => { \\"b\\": { \\"a\\": [ \\"CIRCULAR\\", - \\"a__tsona\\", - \\"__tson\\" + \\"a__tson__a\\", + \\"__tson__\\" ] } } } }, - \\"nonce\\": \\"__tson\\" + \\"nonce\\": \\"__tson__\\" }" `); expect(res).toEqual(expected); From 220d52959c75846ea59db26464a723aeb294284f Mon Sep 17 00:00:00 2001 From: KATT Date: Sun, 8 Oct 2023 20:09:00 +0200 Subject: [PATCH 07/11] ok - found a bug --- src/index.test.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/index.test.ts b/src/index.test.ts index cdf2129d..1d9c5fe2 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -6,6 +6,7 @@ import { createTson, createTsonAsync, tsonDate, + tsonMap, } from "./index.js"; import { waitError } from "./internals/testUtils.js"; @@ -200,6 +201,40 @@ test("back-reference: grandparent reference", () => { expect(res).toEqual(expected); }); +test("back-reference: self-referencing Map", () => { + const t = createTson({ + nonce: () => "__tson__", + types: [tsonMap], + }); + + const expected = new Map(); + expected.set("a", expected); + + const str = t.stringify(expected, 2); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + \\"Map\\", + [ + [ + \\"a\\", + [ + \\"CIRCULAR\\", + \\"\\", + \\"__tson__\\" + ] + ] + ], + \\"__tson__\\" + ], + \\"nonce\\": \\"__tson__\\" + }" + `); + const res = t.parse(str); + + expect(res).toEqual(expected); +}); /** * WILL NOT WORK: the async serialize/deserialize functions haven't * been adapted to handle back-references yet From d83bc7e600de74608b3ff16b99f462aa20f86021 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 8 Oct 2023 21:04:42 +0200 Subject: [PATCH 08/11] handle Map and Set --- src/index.test.ts | 110 ++++++++++++++++++++++++++++++++++++++++ src/sync/deserialize.ts | 49 +++++++++++++++--- 2 files changed, 153 insertions(+), 6 deletions(-) diff --git a/src/index.test.ts b/src/index.test.ts index 1d9c5fe2..9f73a4fe 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -7,6 +7,7 @@ import { createTsonAsync, tsonDate, tsonMap, + tsonSet, } from "./index.js"; import { waitError } from "./internals/testUtils.js"; @@ -234,6 +235,115 @@ test("back-reference: self-referencing Map", () => { const res = t.parse(str); expect(res).toEqual(expected); + expect(res.get("a")).toBe(res); +}); + +test("back-reference: self-referencing Map deep", () => { + const t = createTson({ + nonce: () => "__tson__", + types: [tsonMap], + }); + + const expected = new Map(); + expected.set("a", { + foo: expected, + }); + + const str = t.stringify(expected, 2); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + \\"Map\\", + [ + [ + \\"a\\", + { + \\"foo\\": [ + \\"CIRCULAR\\", + \\"\\", + \\"__tson__\\" + ] + } + ] + ], + \\"__tson__\\" + ], + \\"nonce\\": \\"__tson__\\" + }" + `); + const res = t.parse(str); + + expect(res).toEqual(expected); + expect(res.get("a").foo).toBe(res); +}); + +test("back-reference: self-referencing Set", () => { + const t = createTson({ + nonce: () => "__tson__", + types: [tsonSet], + }); + + const expected = new Set(); + expected.add(expected); + + const str = t.stringify(expected, 2); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + \\"Set\\", + [ + [ + \\"CIRCULAR\\", + \\"\\", + \\"__tson__\\" + ] + ], + \\"__tson__\\" + ], + \\"nonce\\": \\"__tson__\\" + }" + `); + const res = t.parse(str); + + expect(res).toEqual(expected); + expect(res.has(res)).toBe(true); +}); + +test("back-reference: self-referencing Set deep", () => { + const t = createTson({ + nonce: () => "__tson__", + types: [tsonSet], + }); + + const expected = new Set(); + expected.add({ foo: expected }); + + const str = t.stringify(expected, 2); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + \\"Set\\", + [ + { + \\"foo\\": [ + \\"CIRCULAR\\", + \\"\\", + \\"__tson__\\" + ] + } + ], + \\"__tson__\\" + ], + \\"nonce\\": \\"__tson__\\" + }" + `); + const res = t.parse(str); + + expect(res).toEqual(expected); + expect(res.values().next().value.foo).toBe(res); }); /** * WILL NOT WORK: the async serialize/deserialize functions haven't diff --git a/src/sync/deserialize.ts b/src/sync/deserialize.ts index 1c89ed35..d96d85b2 100644 --- a/src/sync/deserialize.ts +++ b/src/sync/deserialize.ts @@ -39,7 +39,7 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { const [type, serializedValue] = value; if (type === "CIRCULAR") { references.push([key, serializedValue as string]); - return; + return nonce; } // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -75,15 +75,52 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { let insertAt = res; try { while (path.length > 1) { - //@ts-expect-error -- insertAt is unknown and not checked, but if it passed serialization, it should be an object - insertAt = insertAt[path.shift()]; + if (insertAt instanceof Map) { + if (path.length <= 2) { + break; + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-non-null-assertion -- if it passed serialization, path will have an index at this point + const key = Array.from(insertAt.keys())[Number(path.shift()!)]; + insertAt = insertAt.get(key); + path.shift(); + } else if (insertAt instanceof Set) { + //@ts-expect-error -- if it passed serialization, path will have an index at this point + insertAt = Array.from(insertAt)[path.shift()]; + } else { + //@ts-expect-error -- insertAt is unknown and not checked, but if it passed serialization, it should be an object + insertAt = insertAt[path.shift()]; + } } - //@ts-expect-error -- see above, + if it passed serialization, path should be length 1 at this point - insertAt[path[0]] = prev; + if (insertAt instanceof Map) { + if (path.length !== 2) { + throw new Error( + `Invalid path to Map insertion ${copyKey + .split(nonce) + .join(".")}`, + ); + } + + const mapKeys = Array.from(insertAt.keys()); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-non-null-assertion -- if it passed serialization, path will have an index at this point + const key = mapKeys[Number(path[0]!)]; + insertAt.set(key, prev); + } else if (insertAt instanceof Set) { + /** + * WARNING: this doesn't preserve order in the Set + */ + insertAt.delete(nonce); + insertAt.add(prev); + } else { + //@ts-expect-error -- see above, + if it passed serialization, path should be length 1 at this point + insertAt[path[0]] = prev; + } } catch (cause) { throw new Error( - `Invalid path to reference insertion ${copyKey.split(nonce).join(".")}`, + `Invalid path to reference insertion ${copyKey + .split(nonce) + .join(".")}`, { cause }, ); } From fa3c0b6dbdad3e03820efacddebe71f8e0a22ef8 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 8 Oct 2023 21:10:32 +0200 Subject: [PATCH 09/11] rename --- src/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.test.ts b/src/index.test.ts index 9f73a4fe..a37a15d9 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -115,7 +115,7 @@ test("back-reference: circular array reference", () => { `); }); -test("back-reference: non-circular complex reference", () => { +test("back-reference: referential equality", () => { const t = createTson({ nonce: () => "__tson", types: [tsonDate], From 30d85f26817e58e5831cc26e53c90d12e7efdf2c Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 8 Oct 2023 21:29:02 +0200 Subject: [PATCH 10/11] fix: stronger grandparent test --- src/index.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.test.ts b/src/index.test.ts index a37a15d9..bf2c9206 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -200,6 +200,7 @@ test("back-reference: grandparent reference", () => { }" `); expect(res).toEqual(expected); + expect(res["a"].a).toBe(res["a"].a.b.a); }); test("back-reference: self-referencing Map", () => { From f62151bc6c4a8ff6385e23a03845f2063ed836e5 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 8 Oct 2023 21:29:26 +0200 Subject: [PATCH 11/11] rename 'CIRCULAR' tson key to 'Reference' --- src/index.test.ts | 22 +++++++++++----------- src/sync/deserialize.ts | 2 +- src/sync/serialize.ts | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/index.test.ts b/src/index.test.ts index bf2c9206..854687cd 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -63,12 +63,12 @@ test("back-reference: circular object reference", () => { "{ \\"json\\": { \\"a\\": [ - \\"CIRCULAR\\", + \\"Reference\\", \\"\\", \\"__tson\\" ], \\"b\\": [ - \\"CIRCULAR\\", + \\"Reference\\", \\"\\", \\"__tson\\" ] @@ -100,12 +100,12 @@ test("back-reference: circular array reference", () => { "{ \\"json\\": [ [ - \\"CIRCULAR\\", + \\"Reference\\", \\"\\", \\"__tson\\" ], [ - \\"CIRCULAR\\", + \\"Reference\\", \\"\\", \\"__tson\\" ] @@ -140,7 +140,7 @@ test("back-reference: referential equality", () => { \\"json\\": { \\"a\\": {}, \\"b\\": [ - \\"CIRCULAR\\", + \\"Reference\\", \\"a\\", \\"__tson\\" ], @@ -150,7 +150,7 @@ test("back-reference: referential equality", () => { \\"__tson\\" ], \\"d\\": [ - \\"CIRCULAR\\", + \\"Reference\\", \\"c\\", \\"__tson\\" ] @@ -188,7 +188,7 @@ test("back-reference: grandparent reference", () => { \\"a\\": { \\"b\\": { \\"a\\": [ - \\"CIRCULAR\\", + \\"Reference\\", \\"a__tson__a\\", \\"__tson__\\" ] @@ -222,7 +222,7 @@ test("back-reference: self-referencing Map", () => { [ \\"a\\", [ - \\"CIRCULAR\\", + \\"Reference\\", \\"\\", \\"__tson__\\" ] @@ -261,7 +261,7 @@ test("back-reference: self-referencing Map deep", () => { \\"a\\", { \\"foo\\": [ - \\"CIRCULAR\\", + \\"Reference\\", \\"\\", \\"__tson__\\" ] @@ -296,7 +296,7 @@ test("back-reference: self-referencing Set", () => { \\"Set\\", [ [ - \\"CIRCULAR\\", + \\"Reference\\", \\"\\", \\"__tson__\\" ] @@ -330,7 +330,7 @@ test("back-reference: self-referencing Set deep", () => { [ { \\"foo\\": [ - \\"CIRCULAR\\", + \\"Reference\\", \\"\\", \\"__tson__\\" ] diff --git a/src/sync/deserialize.ts b/src/sync/deserialize.ts index d96d85b2..8448181b 100644 --- a/src/sync/deserialize.ts +++ b/src/sync/deserialize.ts @@ -37,7 +37,7 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { const key = path.join(nonce); if (isTsonTuple(value, nonce)) { const [type, serializedValue] = value; - if (type === "CIRCULAR") { + if (type === "Reference") { references.push([key, serializedValue as string]); return nonce; } diff --git a/src/sync/serialize.ts b/src/sync/serialize.ts index f0798d6a..25d406d8 100644 --- a/src/sync/serialize.ts +++ b/src/sync/serialize.ts @@ -74,7 +74,7 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { if (isComplex) { const prev = seen.get(value); if (prev) { - return ["CIRCULAR", prev.join(nonce), nonce] as TsonTuple; + return ["Reference", prev.join(nonce), nonce] as TsonTuple; } seen.set(value, path);