diff --git a/benchmark/index.js b/benchmark/index.js index 87f72ba1..2a5c4887 100644 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -1,11 +1,21 @@ // taken from https://github.com/Rich-Harris/superjson-and-devalue import ARSON from "arson"; -import { parse, stringify, uneval } from "devalue"; +import * as devalue from "devalue"; import c from "kleur"; import * as superjson from "superjson"; import { createTson, tsonDate, tsonRegExp, tsonSet } from "tupleson"; +const time_formatter = new Intl.NumberFormat("en-US", { + style: "unit", + unit: "millisecond", +}); +const size_formatter = new Intl.NumberFormat("en-US", { + style: "unit", + unit: "byte", +}); +const number_formatter = new Intl.NumberFormat("en-US"); + const obj = { array: [{ foo: 1 }, { bar: 2 }, { baz: 3 }], date: new Date(), @@ -14,37 +24,49 @@ const obj = { set: new Set([1, 2, 3]), xss: '', }; - -// circular references are not supported by tupleson -// obj.self = obj; +obj.self = obj; const tson = createTson({ types: [tsonDate, tsonRegExp, tsonSet], }); const superjson_serialized = superjson.stringify(obj); -const devalue_unevaled = uneval(obj); -const devalue_stringified = stringify(obj); +const devalue_unevaled = devalue.uneval(obj); +const devalue_stringified = devalue.stringify(obj); const arson_stringified = ARSON.stringify(obj); const tson_serialized = tson.stringify(obj); +console.log("-- SERIALIZED SIZE --\n"); + console.log( - `superjson output: ${c.bold().cyan(superjson_serialized.length)} bytes`, + `superjson output: ${c + .bold() + .cyan(size_formatter.format(superjson_serialized.length))}`, ); -console.log(`tson output: ${c.bold().cyan(tson_serialized.length)} bytes`); +console.log( + `tson output: ${c + .bold() + .cyan(size_formatter.format(tson_serialized.length))}`, +); // console.log(superjson_serialized); console.log( - `devalue.uneval output: ${c.bold().cyan(devalue_unevaled.length)} bytes`, + `devalue.uneval output: ${c + .bold() + .cyan(size_formatter.format(devalue_unevaled.length))}`, ); // console.log(devalue_unevaled); console.log( `devalue.stringify output: ${c .bold() - .cyan(devalue_stringified.length)} bytes`, + .cyan(size_formatter.format(devalue_stringified.length))}`, ); // console.log(devalue_stringified); -console.log(`arson output: ${c.bold().cyan(arson_stringified.length)} bytes`); +console.log( + `arson output: ${c + .bold() + .cyan(size_formatter.format(arson_stringified.length))}`, +); // console.log(arson_stringified); // const superjson_deserialized = superjson.parse(superjson_serialized); @@ -53,31 +75,49 @@ console.log(`arson output: ${c.bold().cyan(arson_stringified.length)} bytes`); const iterations = 1e6; function test(fn, label = fn.toString()) { - const start = Date.now(); console.log(); console.log(c.bold(label)); + global.gc(); // force garbage collection before each test let i = iterations; + const before_snap = process.memoryUsage(); + const start = Date.now(); while (i--) { fn(); } + const delta = Date.now() - start; + const after_snap = process.memoryUsage(); console.log( - `${iterations} iterations in ${c.bold().cyan(Date.now() - start)}ms`, + `${number_formatter.format(iterations)} iterations in ${c + .bold() + .cyan(time_formatter.format(delta))}`, ); + // log memory usage delta + for (const key in after_snap) { + const before = before_snap[key]; + const after = after_snap[key]; + const diff = after - before; + const color = diff < 0 ? c.green : c.red; + console.log(` ${key}: ${color(size_formatter.format(diff))}`); + } } +console.log("\n-- SERIALIZATION DURATION --"); + // serialization test(() => superjson.stringify(obj)); test(() => tson.stringify(obj)); -test(() => uneval(obj)); -test(() => stringify(obj)); +test(() => devalue.uneval(obj)); +test(() => devalue.stringify(obj)); test(() => ARSON.stringify(obj)); +console.log("\n-- DESERIALIZATION DURATION --"); + // deserialization test(() => superjson.parse(superjson_serialized)); test(() => tson.parse(tson_serialized)); test(() => eval(`(${devalue_unevaled})`)); test(() => ARSON.parse(arson_stringified)); -test(() => parse(devalue_stringified)); +test(() => devalue.parse(devalue_stringified)); console.log(); diff --git a/benchmark/package.json b/benchmark/package.json index 9431843b..447710bb 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -7,7 +7,7 @@ "type": "module", "scripts": { "postinstall": "cd ../ && pnpm run build", - "start": "node index.js" + "start": "node --expose-gc --max-old-space-size=8192 index.js" }, "dependencies": { "arson": "^0.2.6", diff --git a/src/index.test.ts b/src/index.test.ts index f17c20d4..854687cd 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,7 +1,15 @@ import { expect, test } from "vitest"; -import { TsonOptions, TsonType, createTson, createTsonAsync } from "./index.js"; -import { expectError, waitError } from "./internals/testUtils.js"; +import { + TsonOptions, + TsonType, + createTson, + createTsonAsync, + tsonDate, + tsonMap, + tsonSet, +} from "./index.js"; +import { waitError } from "./internals/testUtils.js"; test("multiple handlers for primitive string found", () => { const stringHandler: TsonType = { @@ -33,20 +41,334 @@ test("duplicate keys", () => { ); }); -test("no max call stack", () => { +test("back-reference: circular object reference", () => { const t = createTson({ + nonce: () => "__tson", types: [], }); const expected: Record = {}; expected["a"] = expected; + expected["b"] = expected; - // stringify should fail b/c of JSON limitations - const err = expectError(() => t.stringify(expected)); + const str = t.stringify(expected, 2); + const res = t.parse(str); - expect(err.message).toMatchInlineSnapshot('"Circular reference detected"'); + expect(res).toEqual(expected); + expect(res).toBe(res["a"]); + expect(res["b"]).toBe(res["a"]); + + expect(str).toMatchInlineSnapshot( + ` + "{ + \\"json\\": { + \\"a\\": [ + \\"Reference\\", + \\"\\", + \\"__tson\\" + ], + \\"b\\": [ + \\"Reference\\", + \\"\\", + \\"__tson\\" + ] + }, + \\"nonce\\": \\"__tson\\" + }" + `, + ); +}); + +test("back-reference: circular array reference", () => { + const t = createTson({ + nonce: () => "__tson", + types: [], + }); + + const expected: unknown[] = []; + expected[0] = expected; + expected[1] = expected; + + const str = t.stringify(expected, 2); + const res = t.parse(str); + + expect(res).toEqual(expected); + expect(res).toBe(res[0]); + expect(res[1]).toBe(res[0]); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + [ + \\"Reference\\", + \\"\\", + \\"__tson\\" + ], + [ + \\"Reference\\", + \\"\\", + \\"__tson\\" + ] + ], + \\"nonce\\": \\"__tson\\" + }" + `); +}); + +test("back-reference: referential equality", () => { + const t = createTson({ + nonce: () => "__tson", + types: [tsonDate], + }); + + const expected: Record = {}; + expected["a"] = {}; + expected["b"] = expected["a"]; + expected["c"] = new Date(0); + expected["d"] = expected["c"]; + + const str = t.stringify(expected, 2); + const res = t.parse(str); + + expect(res).toEqual(expected); + expect(res["b"]).toBe(res["a"]); + expect(res["d"]).toBe(res["c"]); + + expect(str).toMatchInlineSnapshot( + ` + "{ + \\"json\\": { + \\"a\\": {}, + \\"b\\": [ + \\"Reference\\", + \\"a\\", + \\"__tson\\" + ], + \\"c\\": [ + \\"Date\\", + \\"1970-01-01T00:00:00.000Z\\", + \\"__tson\\" + ], + \\"d\\": [ + \\"Reference\\", + \\"c\\", + \\"__tson\\" + ] + }, + \\"nonce\\": \\"__tson\\" + }" + `, + ); +}); + +test("back-reference: grandparent reference", () => { + const t = createTson({ + nonce: () => "__tson__", + types: [tsonDate], + }); + + const expected: Record = { + a: { + a: { + b: { + a: null, + }, + }, + }, + }; + expected["a"].a.b.a = expected["a"].a; + + const str = t.stringify(expected, 2); + const res = t.parse(str); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": { + \\"a\\": { + \\"a\\": { + \\"b\\": { + \\"a\\": [ + \\"Reference\\", + \\"a__tson__a\\", + \\"__tson__\\" + ] + } + } + } + }, + \\"nonce\\": \\"__tson__\\" + }" + `); + expect(res).toEqual(expected); + expect(res["a"].a).toBe(res["a"].a.b.a); +}); + +test("back-reference: self-referencing Map", () => { + const t = createTson({ + nonce: () => "__tson__", + types: [tsonMap], + }); + + const expected = new Map(); + expected.set("a", expected); + + const str = t.stringify(expected, 2); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + \\"Map\\", + [ + [ + \\"a\\", + [ + \\"Reference\\", + \\"\\", + \\"__tson__\\" + ] + ] + ], + \\"__tson__\\" + ], + \\"nonce\\": \\"__tson__\\" + }" + `); + const res = t.parse(str); + + expect(res).toEqual(expected); + expect(res.get("a")).toBe(res); }); +test("back-reference: self-referencing Map deep", () => { + const t = createTson({ + nonce: () => "__tson__", + types: [tsonMap], + }); + + const expected = new Map(); + expected.set("a", { + foo: expected, + }); + + const str = t.stringify(expected, 2); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + \\"Map\\", + [ + [ + \\"a\\", + { + \\"foo\\": [ + \\"Reference\\", + \\"\\", + \\"__tson__\\" + ] + } + ] + ], + \\"__tson__\\" + ], + \\"nonce\\": \\"__tson__\\" + }" + `); + const res = t.parse(str); + + expect(res).toEqual(expected); + expect(res.get("a").foo).toBe(res); +}); + +test("back-reference: self-referencing Set", () => { + const t = createTson({ + nonce: () => "__tson__", + types: [tsonSet], + }); + + const expected = new Set(); + expected.add(expected); + + const str = t.stringify(expected, 2); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + \\"Set\\", + [ + [ + \\"Reference\\", + \\"\\", + \\"__tson__\\" + ] + ], + \\"__tson__\\" + ], + \\"nonce\\": \\"__tson__\\" + }" + `); + const res = t.parse(str); + + expect(res).toEqual(expected); + expect(res.has(res)).toBe(true); +}); + +test("back-reference: self-referencing Set deep", () => { + const t = createTson({ + nonce: () => "__tson__", + types: [tsonSet], + }); + + const expected = new Set(); + expected.add({ foo: expected }); + + const str = t.stringify(expected, 2); + + expect(str).toMatchInlineSnapshot(` + "{ + \\"json\\": [ + \\"Set\\", + [ + { + \\"foo\\": [ + \\"Reference\\", + \\"\\", + \\"__tson__\\" + ] + } + ], + \\"__tson__\\" + ], + \\"nonce\\": \\"__tson__\\" + }" + `); + const res = t.parse(str); + + expect(res).toEqual(expected); + expect(res.values().next().value.foo).toBe(res); +}); +/** + * WILL NOT WORK: the async serialize/deserialize functions haven't + * been adapted to handle back-references yet + */ +// test("async: back-reference", async () => { +// const t = createTsonAsync({ +// types: [tsonPromise], +// }); + +// const needle = {} + +// const expected = { +// a: needle, +// b: Promise.resolve(needle), +// }; + +// const str = await t.stringify(expected); +// const res = await t.parse(str); + +// expect(res).toEqual(expected); +// expect(res.a).toBe(await res.b); +// }) + test("allow duplicate objects", () => { const t = createTson({ types: [], diff --git a/src/sync/deserialize.ts b/src/sync/deserialize.ts index da27568a..8448181b 100644 --- a/src/sync/deserialize.ts +++ b/src/sync/deserialize.ts @@ -9,7 +9,7 @@ import { TsonTransformerSerializeDeserialize, } from "./syncTypes.js"; -type WalkFn = (value: unknown) => unknown; +type WalkFn = (value: unknown, path?: (number | string)[]) => unknown; type WalkerFactory = (nonce: TsonNonce) => WalkFn; type AnyTsonTransformerSerializeDeserialize = @@ -30,15 +30,103 @@ export function createTsonDeserialize(opts: TsonOptions): TsonDeserializeFn { } const walker: WalkerFactory = (nonce) => { - const walk: WalkFn = (value) => { + const seen = new Map(); + const references: [copyKey: string, originKey: string][] = []; + + const coreWalk: WalkFn = (value, path = []) => { + const key = path.join(nonce); if (isTsonTuple(value, nonce)) { const [type, serializedValue] = value; + if (type === "Reference") { + references.push([key, serializedValue as string]); + return nonce; + } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const transformer = typeByKey[type]!; - return transformer.deserialize(walk(serializedValue)); + const parsed = transformer.deserialize( + coreWalk(serializedValue, path), + ) as unknown; + seen.set(key, parsed); + return parsed; + } + + const parsed = mapOrReturn(value, (value, key) => + coreWalk(value, [...path, key]), + ); + if (parsed && typeof parsed === "object") { + seen.set(key, parsed); + } + + return parsed; + }; + + const walk: WalkFn = (value) => { + const res = coreWalk(value); + for (const [copyKey, originKey] of references) { + const prev = seen.get(originKey); + if (!prev) { + throw new Error( + `Back-reference ${originKey.split(nonce).join(".")} not found`, + ); + } + + const path = copyKey.split(nonce); + let insertAt = res; + try { + while (path.length > 1) { + if (insertAt instanceof Map) { + if (path.length <= 2) { + break; + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-non-null-assertion -- if it passed serialization, path will have an index at this point + const key = Array.from(insertAt.keys())[Number(path.shift()!)]; + insertAt = insertAt.get(key); + path.shift(); + } else if (insertAt instanceof Set) { + //@ts-expect-error -- if it passed serialization, path will have an index at this point + insertAt = Array.from(insertAt)[path.shift()]; + } else { + //@ts-expect-error -- insertAt is unknown and not checked, but if it passed serialization, it should be an object + insertAt = insertAt[path.shift()]; + } + } + + if (insertAt instanceof Map) { + if (path.length !== 2) { + throw new Error( + `Invalid path to Map insertion ${copyKey + .split(nonce) + .join(".")}`, + ); + } + + const mapKeys = Array.from(insertAt.keys()); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-non-null-assertion -- if it passed serialization, path will have an index at this point + const key = mapKeys[Number(path[0]!)]; + insertAt.set(key, prev); + } else if (insertAt instanceof Set) { + /** + * WARNING: this doesn't preserve order in the Set + */ + insertAt.delete(nonce); + insertAt.add(prev); + } else { + //@ts-expect-error -- see above, + if it passed serialization, path should be length 1 at this point + insertAt[path[0]] = prev; + } + } catch (cause) { + throw new Error( + `Invalid path to reference insertion ${copyKey + .split(nonce) + .join(".")}`, + { cause }, + ); + } } - return mapOrReturn(value, walk); + return res; }; return walk; diff --git a/src/sync/serialize.ts b/src/sync/serialize.ts index bc1d2d6f..25d406d8 100644 --- a/src/sync/serialize.ts +++ b/src/sync/serialize.ts @@ -1,4 +1,4 @@ -import { TsonCircularReferenceError } from "../errors.js"; +// import { TsonCircularReferenceError } from "../errors.js"; import { GetNonce, getDefaultNonce } from "../internals/getNonce.js"; import { mapOrReturn } from "../internals/mapOrReturn.js"; import { @@ -13,7 +13,7 @@ import { TsonTypeTesterPrimitive, } from "./syncTypes.js"; -type WalkFn = (value: unknown) => unknown; +type WalkFn = (value: unknown, path?: (number | string)[]) => unknown; type WalkerFactory = (nonce: TsonNonce) => WalkFn; function getHandlers(opts: TsonOptions) { @@ -56,26 +56,13 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { const [getNonce, nonPrimitive, byPrimitive] = getHandlers(opts); const walker: WalkerFactory = (nonce) => { - const seen = new WeakSet(); + const seen = new WeakMap(); const cache = new WeakMap(); - const walk: WalkFn = (value) => { + const walk: WalkFn = (value, path = []) => { const type = typeof value; const isComplex = !!value && type === "object"; - if (isComplex) { - if (seen.has(value)) { - const cached = cache.get(value); - if (!cached) { - throw new TsonCircularReferenceError(value); - } - - return cached; - } - - seen.add(value); - } - const cacheAndReturn = (result: unknown) => { if (isComplex) { cache.set(value, result); @@ -84,6 +71,15 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { return result; }; + if (isComplex) { + const prev = seen.get(value); + if (prev) { + return ["Reference", prev.join(nonce), nonce] as TsonTuple; + } + + seen.set(value, path); + } + const primitiveHandler = byPrimitive[type]; if ( primitiveHandler && @@ -92,7 +88,7 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { return cacheAndReturn([ primitiveHandler.key, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - walk(primitiveHandler.serialize!(value)), + walk(primitiveHandler.serialize!(value), path), nonce, ] as TsonTuple); } @@ -102,13 +98,15 @@ export function createTsonSerialize(opts: TsonOptions): TsonSerializeFn { return cacheAndReturn([ handler.key, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - walk(handler.serialize!(value)), + walk(handler.serialize!(value), path), nonce, ] as TsonTuple); } } - return cacheAndReturn(mapOrReturn(value, walk)); + return cacheAndReturn( + mapOrReturn(value, (value, key) => walk(value, [...path, key])), + ); }; return walk;