From b41ccd7ad67a3741ac292d9109eebeb9d5574d42 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Wed, 5 Feb 2025 21:39:16 +0900 Subject: [PATCH 1/3] add a regression test based on #195 --- test/reuse-instances-with-extensions.test.ts | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 test/reuse-instances-with-extensions.test.ts diff --git a/test/reuse-instances-with-extensions.test.ts b/test/reuse-instances-with-extensions.test.ts new file mode 100644 index 0000000..a275a1c --- /dev/null +++ b/test/reuse-instances-with-extensions.test.ts @@ -0,0 +1,41 @@ +// https://github.com/msgpack/msgpack-javascript/issues/195 + +import { deepStrictEqual } from "assert"; +import { Encoder, Decoder, ExtensionCodec } from "../src/index"; + +const MSGPACK_EXT_TYPE_BIGINT = 0; + +function registerCodecs(context: MsgPackContext) { + const { extensionCodec, encode, decode } = context; + + extensionCodec.register({ + type: MSGPACK_EXT_TYPE_BIGINT, + encode: (value) => (typeof value === "bigint" ? encode(value.toString()) : null), + decode: (data) => BigInt(decode(data) as string), + }); +} + +class MsgPackContext { + readonly encode: (value: unknown) => Uint8Array; + readonly decode: (buffer: BufferSource | ArrayLike) => unknown; + readonly extensionCodec = new ExtensionCodec(); + + constructor() { + const encoder = new Encoder({ extensionCodec: this.extensionCodec, context: this }); + const decoder = new Decoder({ extensionCodec: this.extensionCodec, context: this }); + + this.encode = encoder.encode.bind(encoder); + this.decode = decoder.decode.bind(decoder); + + registerCodecs(this); + } +} + +describe("reuse instances with extensions", () => { + it("should encode and decode a bigint", () => { + const context = new MsgPackContext(); + const buf = context.encode(BigInt(42)); + const data = context.decode(buf); + deepStrictEqual(data, BigInt(42)); + }); +}); From 70c21a95bd8267ff83ebf4a9610a4baf0d1b13cf Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Thu, 6 Feb 2025 21:37:48 +0900 Subject: [PATCH 2/3] make encode() and decode() re-entrant --- src/Decoder.ts | 60 +++++++++++++++++++- src/Encoder.ts | 42 ++++++++++++++ test/reuse-instances-with-extensions.test.ts | 11 +++- 3 files changed, 110 insertions(+), 3 deletions(-) diff --git a/src/Decoder.ts b/src/Decoder.ts index 05ded6d..55109d2 100644 --- a/src/Decoder.ts +++ b/src/Decoder.ts @@ -221,6 +221,8 @@ export class Decoder { private headByte = HEAD_BYTE_REQUIRED; private readonly stack = new StackPool(); + private entered = false; + public constructor(options?: DecoderOptions) { this.extensionCodec = options?.extensionCodec ?? (ExtensionCodec.defaultCodec as ExtensionCodecType); this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined @@ -235,6 +237,22 @@ export class Decoder { this.keyDecoder = options?.keyDecoder !== undefined ? options.keyDecoder : sharedCachedKeyDecoder; } + private clone(): Decoder { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + return new Decoder({ + extensionCodec: this.extensionCodec, + context: this.context, + useBigInt64: this.useBigInt64, + rawStrings: this.rawStrings, + maxStrLength: this.maxStrLength, + maxBinLength: this.maxBinLength, + maxArrayLength: this.maxArrayLength, + maxMapLength: this.maxMapLength, + maxExtLength: this.maxExtLength, + keyDecoder: this.keyDecoder, + } as any); + } + private reinitializeState() { this.totalPos = 0; this.headByte = HEAD_BYTE_REQUIRED; @@ -274,11 +292,27 @@ export class Decoder { return new RangeError(`Extra ${view.byteLength - pos} of ${view.byteLength} byte(s) found at buffer[${posToShow}]`); } + private enteringGuard(): Disposable { + this.entered = true; + return { + [Symbol.dispose]: () => { + this.entered = false; + }, + }; + } + /** * @throws {@link DecodeError} * @throws {@link RangeError} */ public decode(buffer: ArrayLike | ArrayBufferView | ArrayBufferLike): unknown { + if (this.entered) { + const instance = this.clone(); + return instance.decode(buffer); + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + using _guard = this.enteringGuard(); + this.reinitializeState(); this.setBuffer(buffer); @@ -290,6 +324,14 @@ export class Decoder { } public *decodeMulti(buffer: ArrayLike | ArrayBufferView | ArrayBufferLike): Generator { + if (this.entered) { + const instance = this.clone(); + yield* instance.decodeMulti(buffer); + return; + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + using _guard = this.enteringGuard(); + this.reinitializeState(); this.setBuffer(buffer); @@ -299,10 +341,18 @@ export class Decoder { } public async decodeAsync(stream: AsyncIterable | ArrayBufferView | ArrayBufferLike>): Promise { + if (this.entered) { + const instance = this.clone(); + return instance.decodeAsync(stream); + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + using _guard = this.enteringGuard(); + let decoded = false; let object: unknown; for await (const buffer of stream) { if (decoded) { + this.entered = false; throw this.createExtraByteError(this.totalPos); } @@ -343,7 +393,15 @@ export class Decoder { return this.decodeMultiAsync(stream, false); } - private async *decodeMultiAsync(stream: AsyncIterable | ArrayBufferView | ArrayBufferLike>, isArray: boolean) { + private async *decodeMultiAsync(stream: AsyncIterable | ArrayBufferView | ArrayBufferLike>, isArray: boolean): AsyncGenerator { + if (this.entered) { + const instance = this.clone(); + yield* instance.decodeMultiAsync(stream, isArray); + return; + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + using _guard = this.enteringGuard(); + let isArrayHeaderRequired = isArray; let arrayItemsLeft = -1; diff --git a/src/Encoder.ts b/src/Encoder.ts index 614d1a7..af32b33 100644 --- a/src/Encoder.ts +++ b/src/Encoder.ts @@ -86,6 +86,8 @@ export class Encoder { private view: DataView; private bytes: Uint8Array; + private entered = false; + public constructor(options?: EncoderOptions) { this.extensionCodec = options?.extensionCodec ?? (ExtensionCodec.defaultCodec as ExtensionCodecType); this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined @@ -103,16 +105,49 @@ export class Encoder { this.bytes = new Uint8Array(this.view.buffer); } + private clone() { + // Because of slightly special argument `context`, + // type assertion is needed. + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + return new Encoder({ + extensionCodec: this.extensionCodec, + context: this.context, + useBigInt64: this.useBigInt64, + maxDepth: this.maxDepth, + initialBufferSize: this.initialBufferSize, + sortKeys: this.sortKeys, + forceFloat32: this.forceFloat32, + ignoreUndefined: this.ignoreUndefined, + forceIntegerToFloat: this.forceIntegerToFloat, + } as any); + } + private reinitializeState() { this.pos = 0; } + private enteringGuard(): Disposable { + this.entered = true; + return { + [Symbol.dispose]: () => { + this.entered = false; + }, + }; + } + /** * This is almost equivalent to {@link Encoder#encode}, but it returns an reference of the encoder's internal buffer and thus much faster than {@link Encoder#encode}. * * @returns Encodes the object and returns a shared reference the encoder's internal buffer. */ public encodeSharedRef(object: unknown): Uint8Array { + if (this.entered) { + const instance = this.clone(); + return instance.encodeSharedRef(object); + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + using _guard = this.enteringGuard(); + this.reinitializeState(); this.doEncode(object, 1); return this.bytes.subarray(0, this.pos); @@ -122,6 +157,13 @@ export class Encoder { * @returns Encodes the object and returns a copy of the encoder's internal buffer. */ public encode(object: unknown): Uint8Array { + if (this.entered) { + const instance = this.clone(); + return instance.encode(object); + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + using _guard = this.enteringGuard(); + this.reinitializeState(); this.doEncode(object, 1); return this.bytes.slice(0, this.pos); diff --git a/test/reuse-instances-with-extensions.test.ts b/test/reuse-instances-with-extensions.test.ts index a275a1c..39cb456 100644 --- a/test/reuse-instances-with-extensions.test.ts +++ b/test/reuse-instances-with-extensions.test.ts @@ -21,8 +21,8 @@ class MsgPackContext { readonly extensionCodec = new ExtensionCodec(); constructor() { - const encoder = new Encoder({ extensionCodec: this.extensionCodec, context: this }); - const decoder = new Decoder({ extensionCodec: this.extensionCodec, context: this }); + const encoder = new Encoder({ extensionCodec: this.extensionCodec, context: this }); + const decoder = new Decoder({ extensionCodec: this.extensionCodec, context: this }); this.encode = encoder.encode.bind(encoder); this.decode = decoder.decode.bind(decoder); @@ -38,4 +38,11 @@ describe("reuse instances with extensions", () => { const data = context.decode(buf); deepStrictEqual(data, BigInt(42)); }); + + it("should encode and decode bigints", () => { + const context = new MsgPackContext(); + const buf = context.encode([BigInt(1), BigInt(2), BigInt(3)]); + const data = context.decode(buf); + deepStrictEqual(data, [BigInt(1), BigInt(2), BigInt(3)]); + }); }); From 4fc612b5b68888fe0e0878aab6dc93ed44ecdd74 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Thu, 6 Feb 2025 21:44:06 +0900 Subject: [PATCH 3/3] need downlevelIteration for ES5 --- tsconfig.dist.es5+esm.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tsconfig.dist.es5+esm.json b/tsconfig.dist.es5+esm.json index 0a35997..9a281c6 100644 --- a/tsconfig.dist.es5+esm.json +++ b/tsconfig.dist.es5+esm.json @@ -3,6 +3,7 @@ "compilerOptions": { "target": "es5", "module": "es2015", + "downlevelIteration": true, "outDir": "./dist.es5+esm", "declaration": false, "noEmitOnError": true,