diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 2035884..79d6c27 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,10 +1,6 @@ { - // List of extensions which should be recommended for users of this workspace. - "recommendations": [ - "dbaeumer.vscode-eslint", - "yzhang.markdown-all-in-one" - ], - // List of extensions recommended by VS Code that should not be recommended for users of this workspace. - "unwantedRecommendations": [ - ] + // List of extensions which should be recommended for users of this workspace. + "recommendations": ["dbaeumer.vscode-eslint", "yzhang.markdown-all-in-one", "hbenl.vscode-mocha-test-adapter"], + // List of extensions recommended by VS Code that should not be recommended for users of this workspace. + "unwantedRecommendations": [] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 30b6d8a..02f527c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,13 +2,13 @@ "typescript.tsdk": "node_modules/typescript/lib", "files.eol": "\n", "editor.tabSize": 2, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.fixAll.eslint": true, + "source.fixAll": "always" }, - "cSpell.words": [ - "instanceof", - "tsdoc", - "typeof", - "whatwg" - ] + "cSpell.words": ["instanceof", "tsdoc", "typeof", "whatwg"], + "mochaExplorer.files": "test/**/*.test.{ts,js}", + "mochaExplorer.require": ["ts-node/register", "tsconfig-paths/register"] } diff --git a/README.md b/README.md index 5cebb1f..5984ec1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# MessagePack for JavaScript/ECMA-262 +# MessagePack for JavaScript/ECMA-262 [![npm version](https://img.shields.io/npm/v/@msgpack/msgpack.svg)](https://www.npmjs.com/package/@msgpack/msgpack) ![CI](https://github.com/msgpack/msgpack-javascript/workflows/CI/badge.svg) [![codecov](https://codecov.io/gh/msgpack/msgpack-javascript/branch/master/graphs/badge.svg)](https://codecov.io/gh/msgpack/msgpack-javascript) [![minzip](https://badgen.net/bundlephobia/minzip/@msgpack/msgpack)](https://bundlephobia.com/result?p=@msgpack/msgpack) [![tree-shaking](https://badgen.net/bundlephobia/tree-shaking/@msgpack/msgpack)](https://bundlephobia.com/result?p=@msgpack/msgpack) @@ -10,7 +10,7 @@ This library serves as a comprehensive reference implementation of MessagePack f Additionally, this is also a universal JavaScript library. It is compatible not only with browsers, but with Node.js or other JavaScript engines that implement ES2015+ standards. As it is written in [TypeScript](https://www.typescriptlang.org/), this library bundles up-to-date type definition files (`d.ts`). -*Note that this is the second edition of "MessagePack for JavaScript". The first edition, which was implemented in ES5 and never released to npmjs.com, is tagged as [`classic`](https://github.com/msgpack/msgpack-javascript/tree/classic). +\*Note that this is the second edition of "MessagePack for JavaScript". The first edition, which was implemented in ES5 and never released to npmjs.com, is tagged as [`classic`](https://github.com/msgpack/msgpack-javascript/tree/classic). ## Synopsis @@ -44,15 +44,16 @@ deepStrictEqual(decode(encoded), object); - [`EncoderOptions`](#encoderoptions) - [`decode(buffer: ArrayLike | BufferSource, options?: DecoderOptions): unknown`](#decodebuffer-arraylikenumber--buffersource-options-decoderoptions-unknown) - [`DecoderOptions`](#decoderoptions) + - [`IntMode`](#intmode) - [`decodeMulti(buffer: ArrayLike | BufferSource, options?: DecoderOptions): Generator`](#decodemultibuffer-arraylikenumber--buffersource-options-decoderoptions-generatorunknown-void-unknown) - [`decodeAsync(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): Promise`](#decodeasyncstream-readablestreamlikearraylikenumber--buffersource-options-decoderoptions-promiseunknown) - [`decodeArrayStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable`](#decodearraystreamstream-readablestreamlikearraylikenumber--buffersource-options-decoderoptions-asynciterableunknown) - [`decodeMultiStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable`](#decodemultistreamstream-readablestreamlikearraylikenumber--buffersource-options-decoderoptions-asynciterableunknown) - [Reusing Encoder and Decoder instances](#reusing-encoder-and-decoder-instances) - [Extension Types](#extension-types) - - [ExtensionCodec context](#extensioncodec-context) - - [Handling BigInt with ExtensionCodec](#handling-bigint-with-extensioncodec) - - [The temporal module as timestamp extensions](#the-temporal-module-as-timestamp-extensions) + - [ExtensionCodec context](#extensioncodec-context) + - [Handling BigInt](#handling-bigint) + - [The temporal module as timestamp extensions](#the-temporal-module-as-timestamp-extensions) - [Decoding a Blob](#decoding-a-blob) - [MessagePack Specification](#messagepack-specification) - [MessagePack Mapping Table](#messagepack-mapping-table) @@ -109,17 +110,17 @@ console.log(buffer); #### `EncoderOptions` -Name|Type|Default -----|----|---- -extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` -context | user-defined | - -useBigInt64 | boolean | false -maxDepth | number | `100` -initialBufferSize | number | `2048` -sortKeys | boolean | false -forceFloat32 | boolean | false -forceIntegerToFloat | boolean | false -ignoreUndefined | boolean | false +| Name | Type | Default | +| ------------------- | -------------- | ----------------------------- | +| extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` | +| context | user-defined | - | +| forceBigIntToInt64 | boolean | false | +| maxDepth | number | `100` | +| initialBufferSize | number | `2048` | +| sortKeys | boolean | false | +| forceFloat32 | boolean | false | +| forceIntegerToFloat | boolean | false | +| ignoreUndefined | boolean | false | ### `decode(buffer: ArrayLike | BufferSource, options?: DecoderOptions): unknown` @@ -143,19 +144,32 @@ NodeJS `Buffer` is also acceptable because it is a subclass of `Uint8Array`. #### `DecoderOptions` -Name|Type|Default -----|----|---- -extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` -context | user-defined | - -useBigInt64 | boolean | false -maxStrLength | number | `4_294_967_295` (UINT32_MAX) -maxBinLength | number | `4_294_967_295` (UINT32_MAX) -maxArrayLength | number | `4_294_967_295` (UINT32_MAX) -maxMapLength | number | `4_294_967_295` (UINT32_MAX) -maxExtLength | number | `4_294_967_295` (UINT32_MAX) +| Name | Type | Default | +| -------------- | -------------- | ------------------------------------------------------------------------------------ | +| extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` | +| context | user-defined | - | +| useBigInt64 | boolean | false | +| intMode | IntMode | `IntMode.AS_ENCODED` if `useBigInt64` is `true` or `IntMode.UNSAFE_NUMBER` otherwise | +| maxStrLength | number | `4_294_967_295` (UINT32_MAX) | +| maxBinLength | number | `4_294_967_295` (UINT32_MAX) | +| maxArrayLength | number | `4_294_967_295` (UINT32_MAX) | +| maxMapLength | number | `4_294_967_295` (UINT32_MAX) | +| maxExtLength | number | `4_294_967_295` (UINT32_MAX) | You can use `max${Type}Length` to limit the length of each type decoded. +`intMode` determines whether decoded integers should be returned as numbers or bigints in different circumstances. The possible values are [described below](#intmode). + +##### `IntMode` + +The `IntMode` enum defines different options for decoding integers. They are described below: + +- `IntMode.UNSAFE_NUMBER`: Always returns the value as a number. Be aware that there will be a loss of precision if the value is outside the range of `Number.MIN_SAFE_INTEGER` to `Number.MAX_SAFE_INTEGER`. +- `IntMode.SAFE_NUMBER`: Always returns the value as a number, but throws an error if the value is outside of the range of `Number.MIN_SAFE_INTEGER` to `Number.MAX_SAFE_INTEGER`. +- `IntMode.MIXED`: Returns all values inside the range of `Number.MIN_SAFE_INTEGER` to `Number.MAX_SAFE_INTEGER` as numbers and all values outside that range as bigints. +- `IntMode.AS_ENCODED`: Returns all values encoded as int64/uint64 as bigints and all other integers as numbers. +- `IntMode.BIGINT`: Always returns the value as a bigint, even if it is small enough to safely fit in a number. + ### `decodeMulti(buffer: ArrayLike | BufferSource, options?: DecoderOptions): Generator` It decodes `buffer` that includes multiple MessagePack-encoded objects, and returns decoded objects as a generator. See also `decodeMultiStream()`, which is an asynchronous variant of this function. @@ -192,7 +206,9 @@ const contentType = response.headers.get("Content-Type"); if (contentType && contentType.startsWith(MSGPACK_TYPE) && response.body != null) { const object = await decodeAsync(response.body); // do something with object -} else { /* handle errors */ } +} else { + /* handle errors */ +} ``` ### `decodeArrayStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable` @@ -267,7 +283,7 @@ import { encode, decode, ExtensionCodec } from "@msgpack/msgpack"; const extensionCodec = new ExtensionCodec(); // Set -const SET_EXT_TYPE = 0 // Any in 0-127 +const SET_EXT_TYPE = 0; // Any in 0-127 extensionCodec.register({ type: SET_EXT_TYPE, encode: (object: unknown): Uint8Array | null => { @@ -352,19 +368,28 @@ const encoded = = encode({myType: new MyType()}, { extensionCodec, context const decoded = decode(encoded, { extensionCodec, context }); ``` -#### Handling BigInt with ExtensionCodec +#### Handling BigInt + +**Decoding** -This library does not handle BigInt by default, but you have two options to handle it: +This library does not handle decoding BigInt by default, but you have three options to decode using BigInt's: -* Set `useBigInt64: true` to map bigint to MessagePack's int64/uint64 -* Define a custom `ExtensionCodec` to map bigint to a MessagePack's extension type +- Set `useBigInt64: true` to decode MessagePack's `int64`/`uint64` into a BigInt +- Set `intMode` to exert [greater control](#intmode) over BigInt handling +- Define a custom `ExtensionCodec` to map bigint to a MessagePack extension type -`useBigInt64: true` is the simplest way to handle bigint, but it has limitations: +**Encoding** -* A bigint is encoded in 8 byte binaries even if it's a small integer -* A bigint must be smaller than the max value of the uint64 and larger than the min value of the int64. Otherwise the behavior is undefined. +This library will encode a BigInt into a MessagePack int64/uint64 if it is > 32-bit OR you set `forceBigIntToInt64` to `true`. This library will encode a `number` that is > 32-bit into a MessagePack int64/uint64 if it is > 32-bit. -So you might want to define a custom codec to handle bigint like this: +If you set `forceBigIntToInt64` to `true` note: + +- A bigint is encoded in 8 byte binaries even if it's a small integer +- A bigint must be smaller than the max value of the uint64 and larger than the min value of the int64. Otherwise the behavior is undefined. + +**Custom codec** + +Alternatively, you can define a custom codec to handle bigint like this: ```typescript import { deepStrictEqual } from "assert"; @@ -405,7 +430,7 @@ deepStrictEqual(decode(encoded, { extensionCodec }), value); There is a proposal for a new date/time representations in JavaScript: -* https://github.com/tc39/proposal-temporal +- https://github.com/tc39/proposal-temporal This library maps `Date` to the MessagePack timestamp extension by default, but you can re-map the temporal module (or [Temporal Polyfill](https://github.com/tc39/proposal-temporal/tree/main/polyfill)) to the timestamp extension like this: @@ -474,9 +499,9 @@ async function decodeFromBlob(blob: Blob): unknown { This library is compatible with the "August 2017" revision of MessagePack specification at the point where timestamp ext was added: -* [x] str/bin separation, added at August 2013 -* [x] extension types, added at August 2013 -* [x] timestamp ext type, added at August 2017 +- [x] str/bin separation, added at August 2013 +- [x] extension types, added at August 2013 +- [x] timestamp ext type, added at August 2017 The living specification is here: @@ -488,46 +513,44 @@ Note that as of June 2019 there're no official "version" on the MessagePack spec The following table shows how JavaScript values are mapped to [MessagePack formats](https://github.com/msgpack/msgpack/blob/master/spec.md) and vice versa. -The mapping of integers varies on the setting of `useBigInt64`. - -The default, `useBigInt64: false` is: - -Source Value|MessagePack Format|Value Decoded -----|----|---- -null, undefined|nil|null (*1) -boolean (true, false)|bool family|boolean (true, false) -number (53-bit int)|int family|number -number (64-bit float)|float family|number -string|str family|string -ArrayBufferView |bin family|Uint8Array (*2) -Array|array family|Array -Object|map family|Object (*3) -Date|timestamp ext family|Date (*4) -bigint|N/A|N/A (*5) - -* *1 Both `null` and `undefined` are mapped to `nil` (`0xC0`) type, and are decoded into `null` -* *2 Any `ArrayBufferView`s including NodeJS's `Buffer` are mapped to `bin` family, and are decoded into `Uint8Array` -* *3 In handling `Object`, it is regarded as `Record` in terms of TypeScript -* *4 MessagePack timestamps may have nanoseconds, which will lost when it is decoded into JavaScript `Date`. This behavior can be overridden by registering `-1` for the extension codec. -* *5 bigint is not supported in `useBigInt64: false` mode, but you can define an extension codec for it. +The mapping of integers varies on the setting of `intMode`. + +| Source Value | MessagePack Format | Value Decoded | +| --------------------- | -------------------- | ---------------------- | +| null, undefined | nil | null (\*1) | +| boolean (true, false) | bool family | boolean (true, false) | +| number (53-bit int) | int family | number or bigint (\*2) | +| number (64-bit float) | float family | number (64-bit float) | +| bigint | int family | number or bigint (\*2) | +| string | str family | string | +| ArrayBufferView | bin family | Uint8Array (\*3) | +| Array | array family | Array | +| Object | map family | Object (\*4) | +| Date | timestamp ext family | Date (\*5) | +| bigint | int family | bigint | + +- \*1 Both `null` and `undefined` are mapped to `nil` (`0xC0`) type, and are decoded into `null` +- \*2 MessagePack ints are decoded as either numbers or bigints depending on the [IntMode](#intmode) used during decoding. +- \*3 Any `ArrayBufferView`s including NodeJS's `Buffer` are mapped to `bin` family, and are decoded into `Uint8Array` +- \*4 In handling `Object`, it is regarded as `Record` in terms of TypeScript +- \*5 MessagePack timestamps may have nanoseconds, which will lost when it is decoded into JavaScript `Date`. This behavior can be overridden by registering `-1` for the extension codec. If you set `useBigInt64: true`, the following mapping is used: -Source Value|MessagePack Format|Value Decoded -----|----|---- -null, undefined|nil|null -boolean (true, false)|bool family|boolean (true, false) -**number (32-bit int)**|int family|number -**number (except for the above)**|float family|number -**bigint**|int64 / uint64|bigint (*6) -string|str family|string -ArrayBufferView |bin family|Uint8Array -Array|array family|Array -Object|map family|Object -Date|timestamp ext family|Date - - -* *6 If the bigint is larger than the max value of uint64 or smaller than the min value of int64, then the behavior is undefined. +| Source Value | MessagePack Format | Value Decoded | +| --------------------------------- | -------------------- | --------------------- | +| null, undefined | nil | null | +| boolean (true, false) | bool family | boolean (true, false) | +| **number (32-bit int)** | int family | number | +| **number (except for the above)** | float family | number | +| **bigint** | int64 / uint64 | bigint (\*5) | +| string | str family | string | +| ArrayBufferView | bin family | Uint8Array | +| Array | array family | Array | +| Object | map family | Object | +| Date | timestamp ext family | Date | + +- \*5 If the bigint is larger than the max value of uint64 or smaller than the min value of int64, then the behavior is undefined. ## Prerequisites @@ -535,12 +558,12 @@ This is a universal JavaScript library that supports major browsers and NodeJS. ### ECMA-262 -* ES2015 language features -* ES2018 standard library, including: - * Typed arrays (ES2015) - * Async iterations (ES2018) - * Features added in ES2015-ES2022 -* whatwg encodings (`TextEncoder` and `TextDecoder`) +- ES2015 language features +- ES2018 standard library, including: + - Typed arrays (ES2015) + - Async iterations (ES2018) + - Features added in ES2015-ES2022 +- whatwg encodings (`TextEncoder` and `TextDecoder`) ES2022 standard library used in this library can be polyfilled with [core-js](https://github.com/zloirock/core-js). @@ -566,16 +589,16 @@ However, MessagePack can handles binary data effectively, actual performance dep Benchmark on NodeJS/v18.1.0 (V8/10.1) -operation | op | ms | op/s ------------------------------------------------------------------ | ------: | ----: | ------: -buf = Buffer.from(JSON.stringify(obj)); | 902100 | 5000 | 180420 -obj = JSON.parse(buf.toString("utf-8")); | 898700 | 5000 | 179740 -buf = require("msgpack-lite").encode(obj); | 411000 | 5000 | 82200 -obj = require("msgpack-lite").decode(buf); | 246200 | 5001 | 49230 -buf = require("@msgpack/msgpack").encode(obj); | 843300 | 5000 | 168660 -obj = require("@msgpack/msgpack").decode(buf); | 489300 | 5000 | 97860 -buf = /* @msgpack/msgpack */ encoder.encode(obj); | 1154200 | 5000 | 230840 -obj = /* @msgpack/msgpack */ decoder.decode(buf); | 448900 | 5000 | 89780 +| operation | op | ms | op/s | +| ------------------------------------------------- | ------: | ---: | -----: | +| buf = Buffer.from(JSON.stringify(obj)); | 902100 | 5000 | 180420 | +| obj = JSON.parse(buf.toString("utf-8")); | 898700 | 5000 | 179740 | +| buf = require("msgpack-lite").encode(obj); | 411000 | 5000 | 82200 | +| obj = require("msgpack-lite").decode(buf); | 246200 | 5001 | 49230 | +| buf = require("@msgpack/msgpack").encode(obj); | 843300 | 5000 | 168660 | +| obj = require("@msgpack/msgpack").decode(buf); | 489300 | 5000 | 97860 | +| buf = /_ @msgpack/msgpack _/ encoder.encode(obj); | 1154200 | 5000 | 230840 | +| obj = /_ @msgpack/msgpack _/ decoder.decode(buf); | 448900 | 5000 | 89780 | Note that `JSON` cases use `Buffer` to emulate I/O where a JavaScript string must be converted into a byte array encoded in UTF-8, whereas MessagePack modules deal with byte arrays. @@ -585,11 +608,11 @@ Note that `JSON` cases use `Buffer` to emulate I/O where a JavaScript string mus The NPM package distributed in npmjs.com includes both ES2015+ and ES5 files: -* `dist/` is compiled into ES2019 with CommomJS, provided for NodeJS v10 -* `dist.es5+umd/` is compiled into ES5 with UMD - * `dist.es5+umd/msgpack.min.js` - the minified file - * `dist.es5+umd/msgpack.js` - the non-minified file -* `dist.es5+esm/` is compiled into ES5 with ES modules, provided for webpack-like bundlers and NodeJS's ESM-mode +- `dist/` is compiled into ES2019 with CommomJS, provided for NodeJS v10 +- `dist.es5+umd/` is compiled into ES5 with UMD + - `dist.es5+umd/msgpack.min.js` - the minified file + - `dist.es5+umd/msgpack.js` - the non-minified file +- `dist.es5+esm/` is compiled into ES5 with ES modules, provided for webpack-like bundlers and NodeJS's ESM-mode If you use NodeJS and/or webpack, their module resolvers use the suitable one automatically. @@ -603,7 +626,6 @@ This library is available via CDN: It loads `MessagePack` module to the global object. - ## Deno Support You can use this module on Deno. @@ -628,12 +650,12 @@ This library uses Travis CI. test matrix: -* TypeScript targets - * `target=es2019` / `target=es5` -* JavaScript engines - * NodeJS, browsers (Chrome, Firefox, Safari, IE11, and so on) +- TypeScript targets + - `target=es2019` / `target=es5` +- JavaScript engines + - NodeJS, browsers (Chrome, Firefox, Safari, IE11, and so on) -See [test:* in package.json](./package.json) and [.travis.yml](./.travis.yml) for details. +See [test:\* in package.json](./package.json) and [.travis.yml](./.travis.yml) for details. ### Release Engineering diff --git a/package-lock.json b/package-lock.json index ef3486f..80f832a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,9 @@ "karma-webpack": "latest", "lodash": "latest", "mocha": "latest", - "msgpack-test-js": "latest", + "msg-ext": "^1.0.1", + "msg-int64": "^0.1.1", + "msg-timestamp": "^1.0.1", "prettier": "latest", "rimraf": "latest", "ts-loader": "latest", @@ -4071,24 +4073,6 @@ "node": ">= 4.5.0" } }, - "node_modules/msgpack-test-js": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/msgpack-test-js/-/msgpack-test-js-1.0.0.tgz", - "integrity": "sha512-qdg9whLj+MnZkjoINb9E8ndzRKPLNl9gBhA1rcLYniisEcwMGM4P601ErgxNbyJgiDQ5fYzwQtWgHyGYEQfN3g==", - "dev": true, - "dependencies": { - "msg-ext": "^1.0.0", - "msg-int64": "^0.1.0", - "msg-timestamp": "^1.0.0", - "msgpack-test-suite": "^1.0.0" - } - }, - "node_modules/msgpack-test-suite": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/msgpack-test-suite/-/msgpack-test-suite-1.0.0.tgz", - "integrity": "sha512-GTuEtmh0KMcuxvRKmlYraapxjaTGw9+kOGvkprf1mgBoP9hNPAyyn/4sf6lmqvFSm9sFt1cnJ5WVsyIbZRfzDQ==", - "dev": true - }, "node_modules/nanoid": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", diff --git a/package.json b/package.json index 78a64b7..6d2b630 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,9 @@ "karma-webpack": "latest", "lodash": "latest", "mocha": "latest", - "msgpack-test-js": "latest", + "msg-ext": "^1.0.1", + "msg-int64": "^0.1.1", + "msg-timestamp": "^1.0.1", "prettier": "latest", "rimraf": "latest", "ts-loader": "latest", diff --git a/src/Decoder.ts b/src/Decoder.ts index eedb0fb..a369959 100644 --- a/src/Decoder.ts +++ b/src/Decoder.ts @@ -1,6 +1,6 @@ import { prettyByte } from "./utils/prettyByte"; import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec"; -import { getInt64, getUint64, UINT32_MAX } from "./utils/int"; +import { IntMode, getInt64, getUint64, convertSafeIntegerToMode, UINT32_MAX } from "./utils/int"; import { utf8Decode } from "./utils/utf8"; import { createDataView, ensureUint8Array } from "./utils/typedArrays"; import { CachedKeyDecoder, KeyDecoder } from "./CachedKeyDecoder"; @@ -16,10 +16,17 @@ export type DecoderOptions = Readonly< * Depends on ES2020's {@link DataView#getBigInt64} and * {@link DataView#getBigUint64}. * - * Defaults to false. + * Defaults to false. If true, equivalent to intMode: IntMode.AS_ENCODED. */ useBigInt64: boolean; + /** + * Allows for more fine-grained control of BigInt handling, overrides useBigInt64. + * + * Defaults to IntMode.AS_ENCODED if useBigInt64 is true or IntMode.UNSAFE_NUMBER otherwise. + */ + intMode?: IntMode; + /** * Maximum string length. * @@ -194,7 +201,7 @@ const sharedCachedKeyDecoder = new CachedKeyDecoder(); export class Decoder { private readonly extensionCodec: ExtensionCodecType; private readonly context: ContextType; - private readonly useBigInt64: boolean; + private readonly intMode: IntMode; private readonly maxStrLength: number; private readonly maxBinLength: number; private readonly maxArrayLength: number; @@ -214,7 +221,7 @@ export class Decoder { this.extensionCodec = options?.extensionCodec ?? (ExtensionCodec.defaultCodec as ExtensionCodecType); this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined - this.useBigInt64 = options?.useBigInt64 ?? false; + this.intMode = options?.intMode ?? (options?.useBigInt64 ? IntMode.AS_ENCODED : IntMode.UNSAFE_NUMBER); this.maxStrLength = options?.maxStrLength ?? UINT32_MAX; this.maxBinLength = options?.maxBinLength ?? UINT32_MAX; this.maxArrayLength = options?.maxArrayLength ?? UINT32_MAX; @@ -371,11 +378,11 @@ export class Decoder { if (headByte >= 0xe0) { // negative fixint (111x xxxx) 0xe0 - 0xff - object = headByte - 0x100; + object = this.convertNumber(headByte - 0x100); } else if (headByte < 0xc0) { if (headByte < 0x80) { // positive fixint (0xxx xxxx) 0x00 - 0x7f - object = headByte; + object = this.convertNumber(headByte); } else if (headByte < 0x90) { // fixmap (1000 xxxx) 0x80 - 0x8f const size = headByte - 0x80; @@ -418,36 +425,28 @@ export class Decoder { object = this.readF64(); } else if (headByte === 0xcc) { // uint 8 - object = this.readU8(); + object = this.convertNumber(this.readU8()); } else if (headByte === 0xcd) { // uint 16 - object = this.readU16(); + object = this.convertNumber(this.readU16()); } else if (headByte === 0xce) { // uint 32 - object = this.readU32(); + object = this.convertNumber(this.readU32()); } else if (headByte === 0xcf) { // uint 64 - if (this.useBigInt64) { - object = this.readU64AsBigInt(); - } else { - object = this.readU64(); - } + object = this.readU64(); } else if (headByte === 0xd0) { // int 8 - object = this.readI8(); + object = this.convertNumber(this.readI8()); } else if (headByte === 0xd1) { // int 16 - object = this.readI16(); + object = this.convertNumber(this.readI16()); } else if (headByte === 0xd2) { // int 32 - object = this.readI32(); + object = this.convertNumber(this.readI32()); } else if (headByte === 0xd3) { // int 64 - if (this.useBigInt64) { - object = this.readI64AsBigInt(); - } else { - object = this.readI64(); - } + object = this.readI64(); } else if (headByte === 0xd9) { // str 8 const byteLength = this.lookU8(); @@ -692,6 +691,10 @@ export class Decoder { return this.extensionCodec.decode(data, extType, this.context); } + private convertNumber(value: number): number | bigint { + return convertSafeIntegerToMode(value, this.intMode); + } + private lookU8() { return this.view.getUint8(this.pos); } @@ -740,26 +743,14 @@ export class Decoder { return value; } - private readU64(): number { - const value = getUint64(this.view, this.pos); - this.pos += 8; - return value; - } - - private readI64(): number { - const value = getInt64(this.view, this.pos); - this.pos += 8; - return value; - } - - private readU64AsBigInt(): bigint { - const value = this.view.getBigUint64(this.pos); + private readU64(): number | bigint { + const value = getUint64(this.view, this.pos, this.intMode); this.pos += 8; return value; } - private readI64AsBigInt(): bigint { - const value = this.view.getBigInt64(this.pos); + private readI64(): number | bigint { + const value = getInt64(this.view, this.pos, this.intMode); this.pos += 8; return value; } diff --git a/src/Encoder.ts b/src/Encoder.ts index cda0822..8c774cd 100644 --- a/src/Encoder.ts +++ b/src/Encoder.ts @@ -13,14 +13,14 @@ export type EncoderOptions = Partial< extensionCodec: ExtensionCodecType; /** - * Encodes bigint as Int64 or Uint64 if it's set to true. - * {@link forceIntegerToFloat} does not affect bigint. + * Encodes bigint as Int64 or Uint64 if it's set to true, regardless of the size of bigint number. + * {@link forceIntegerToFloat} does not affect bigint if this is enabled. * Depends on ES2020's {@link DataView#setBigInt64} and * {@link DataView#setBigUint64}. * * Defaults to false. */ - useBigInt64: boolean; + forceBigIntToInt64: boolean; /** * The maximum depth in nested objects and arrays. @@ -43,6 +43,7 @@ export type EncoderOptions = Partial< * Defaults to `false`. If enabled, it spends more time in encoding objects. */ sortKeys: boolean; + /** * If `true`, non-integer numbers are encoded in float32, not in float64 (the default). * @@ -74,7 +75,7 @@ export type EncoderOptions = Partial< export class Encoder { private readonly extensionCodec: ExtensionCodecType; private readonly context: ContextType; - private readonly useBigInt64: boolean; + private readonly forceBigIntToInt64: boolean; private readonly maxDepth: number; private readonly initialBufferSize: number; private readonly sortKeys: boolean; @@ -90,7 +91,7 @@ export class Encoder { this.extensionCodec = options?.extensionCodec ?? (ExtensionCodec.defaultCodec as ExtensionCodecType); this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined - this.useBigInt64 = options?.useBigInt64 ?? false; + this.forceBigIntToInt64 = options?.forceBigIntToInt64 ?? false; this.maxDepth = options?.maxDepth ?? DEFAULT_MAX_DEPTH; this.initialBufferSize = options?.initialBufferSize ?? DEFAULT_INITIAL_BUFFER_SIZE; this.sortKeys = options?.sortKeys ?? false; @@ -137,15 +138,9 @@ export class Encoder { } else if (typeof object === "boolean") { this.encodeBoolean(object); } else if (typeof object === "number") { - if (!this.forceIntegerToFloat) { - this.encodeNumber(object); - } else { - this.encodeNumberAsFloat(object); - } + this.encodeNumber(object); } else if (typeof object === "string") { this.encodeString(object); - } else if (this.useBigInt64 && typeof object === "bigint") { - this.encodeBigInt64(object); } else { this.encodeObject(object, depth); } @@ -200,12 +195,10 @@ export class Encoder { // uint 32 this.writeU8(0xce); this.writeU32(object); - } else if (!this.useBigInt64) { + } else { // uint 64 this.writeU8(0xcf); this.writeU64(object); - } else { - this.encodeNumberAsFloat(object); } } else { if (object >= -0x20) { @@ -223,12 +216,10 @@ export class Encoder { // int 32 this.writeU8(0xd2); this.writeI32(object); - } else if (!this.useBigInt64) { + } else { // int 64 this.writeU8(0xd3); this.writeI64(object); - } else { - this.encodeNumberAsFloat(object); } } } else { @@ -248,7 +239,33 @@ export class Encoder { } } - private encodeBigInt64(object: bigint): void { + private encodeBigInt(object: bigint) { + if (this.forceBigIntToInt64) { + this.encodeBigIntAsInt64(object); + } else if (object >= 0) { + if (object < 0x100000000 || this.forceIntegerToFloat) { + // uint 32 or lower, or force to float + this.encodeNumber(Number(object)); + } else if (object < BigInt("0x10000000000000000")) { + // uint 64 + this.encodeBigIntAsInt64(object); + } else { + throw new Error(`Bigint is too large for uint64: ${object}`); + } + } else { + if (object >= -0x80000000 || this.forceIntegerToFloat) { + // int 32 or lower, or force to float + this.encodeNumber(Number(object)); + } else if (object >= BigInt(-1) * BigInt("0x8000000000000000")) { + // int 64 + this.encodeBigIntAsInt64(object); + } else { + throw new Error(`Bigint is too small for int64: ${object}`); + } + } + } + + private encodeBigIntAsInt64(object: bigint): void { if (object >= BigInt(0)) { // uint 64 this.writeU8(0xcf); @@ -300,6 +317,10 @@ export class Encoder { this.encodeArray(object, depth); } else if (ArrayBuffer.isView(object)) { this.encodeBinary(object); + } else if (typeof object === "bigint") { + // this is here instead of in doEncode so that we can try encoding with an extension first, + // otherwise we would break existing extensions for bigints + this.encodeBigInt(object); } else if (typeof object === "object") { this.encodeMap(object as Record, depth); } else { diff --git a/src/index.ts b/src/index.ts index 4141ad4..9b58b9e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,8 @@ import { decode, decodeMulti } from "./decode"; export { decode, decodeMulti }; import type { DecodeOptions } from "./decode"; export type { DecodeOptions }; +import { IntMode } from './utils/int'; +export { IntMode }; import { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream } from "./decodeAsync"; export { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream }; diff --git a/src/timestamp.ts b/src/timestamp.ts index e3fe015..2687361 100644 --- a/src/timestamp.ts +++ b/src/timestamp.ts @@ -1,6 +1,6 @@ // https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type import { DecodeError } from "./DecodeError"; -import { getInt64, setInt64 } from "./utils/int"; +import { IntMode, getInt64, setInt64 } from "./utils/int"; export const EXT_TIMESTAMP = -1; @@ -87,7 +87,7 @@ export function decodeTimestampToTimeSpec(data: Uint8Array): TimeSpec { case 12: { // timestamp 96 = { nsec32 (unsigned), sec64 (signed) } - const sec = getInt64(view, 4); + const sec = getInt64(view, 4, IntMode.UNSAFE_NUMBER); const nsec = view.getUint32(0); return { sec, nsec }; } diff --git a/src/utils/int.ts b/src/utils/int.ts index 7fa93fb..12461de 100644 --- a/src/utils/int.ts +++ b/src/utils/int.ts @@ -1,5 +1,34 @@ // Integer Utility +/** + * An enum of different options for decoding integers. + */ +export enum IntMode { + /** + * Always returns the value as a number. Be aware that there will be a loss of precision if the + * value is outside the range of Number.MIN_SAFE_INTEGER to Number.MAX_SAFE_INTEGER. + */ + UNSAFE_NUMBER, + /** + * Always returns the value as a number, but throws an error if the value is outside of the range + * of Number.MIN_SAFE_INTEGER to Number.MAX_SAFE_INTEGER. + */ + SAFE_NUMBER, + /** + * Returns all values encoded as int64/uint64 as bigints and all other integers as numbers. + */ + AS_ENCODED, + /** + * Returns all values inside the range of Number.MIN_SAFE_INTEGER to Number.MAX_SAFE_INTEGER as + * numbers and all values outside that range as bigints. + */ + MIXED, + /** + * Always returns the value as a bigint, even if it is small enough to safely fit in a number. + */ + BIGINT, +} + export const UINT32_MAX = 0xffff_ffff; // DataView extension to handle int64 / uint64, @@ -19,14 +48,72 @@ export function setInt64(view: DataView, offset: number, value: number): void { view.setUint32(offset + 4, low); } -export function getInt64(view: DataView, offset: number): number { - const high = view.getInt32(offset); - const low = view.getUint32(offset + 4); - return high * 0x1_0000_0000 + low; +export function getInt64(view: DataView, offset: number, mode: IntMode.UNSAFE_NUMBER | IntMode.SAFE_NUMBER): number; +export function getInt64(view: DataView, offset: number, mode: IntMode.BIGINT): bigint; +export function getInt64(view: DataView, offset: number, mode: IntMode): number | bigint; +export function getInt64(view: DataView, offset: number, mode: IntMode): number | bigint { + if (mode === IntMode.UNSAFE_NUMBER || mode === IntMode.SAFE_NUMBER) { + // for compatibility, don't use view.getBigInt64 if the user hasn't told us to use BigInts + const high = view.getInt32(offset); + const low = view.getUint32(offset + 4); + + if ( + mode === IntMode.SAFE_NUMBER && + (high < Math.floor(Number.MIN_SAFE_INTEGER / 0x1_0000_0000) || + (high === Math.floor(Number.MIN_SAFE_INTEGER / 0x1_0000_0000) && low === 0) || + high > (Number.MAX_SAFE_INTEGER - low) / 0x1_0000_0000) + ) { + const hexValue = `${high < 0 ? "-" : ""}0x${Math.abs(high).toString(16)}${low.toString(16).padStart(8, "0")}`; + throw new Error(`Mode is IntMode.SAFE_NUMBER and value is not a safe integer: ${hexValue}`); + } + + return high * 0x1_0000_0000 + low; + } + + const value = view.getBigInt64(offset); + + if (mode === IntMode.MIXED && value >= Number.MIN_SAFE_INTEGER && value <= Number.MAX_SAFE_INTEGER) { + return Number(value); + } + + return value; } -export function getUint64(view: DataView, offset: number): number { - const high = view.getUint32(offset); - const low = view.getUint32(offset + 4); - return high * 0x1_0000_0000 + low; +export function getUint64(view: DataView, offset: number, mode: IntMode.UNSAFE_NUMBER | IntMode.SAFE_NUMBER): number; +export function getUint64(view: DataView, offset: number, mode: IntMode.BIGINT): bigint; +export function getUint64(view: DataView, offset: number, mode: IntMode): number | bigint; +export function getUint64(view: DataView, offset: number, mode: IntMode): number | bigint { + if (mode === IntMode.UNSAFE_NUMBER || mode === IntMode.SAFE_NUMBER) { + // for compatibility, don't use view.getBigUint64 if the user hasn't told us to use BigInts + const high = view.getUint32(offset); + const low = view.getUint32(offset + 4); + + if (mode === IntMode.SAFE_NUMBER && high > (Number.MAX_SAFE_INTEGER - low) / 0x1_0000_0000) { + const hexValue = `0x${high.toString(16)}${low.toString(16).padStart(8, "0")}`; + throw new Error(`Mode is IntMode.SAFE_NUMBER and value is not a safe integer: ${hexValue}`); + } + + return high * 0x1_0000_0000 + low; + } + + const value = view.getBigUint64(offset); + + if (mode === IntMode.MIXED && value <= Number.MAX_SAFE_INTEGER) { + return Number(value); + } + + return value; +} + +/** + * Convert a safe integer Number (i.e. in the range Number.MIN_SAFE_INTEGER to Number.MAX_SAFE_INTEGER) + * with respect to the given IntMode. For all modes except IntMode.BIGINT, this returns the original + * Number unmodified. + */ +export function convertSafeIntegerToMode(value: number, mode: IntMode): number | bigint { + if (mode === IntMode.BIGINT) { + return BigInt(value); + } + + return Number(value); } diff --git a/test/bigint64.test.ts b/test/bigint64.test.ts index fabf0f5..b09ccdb 100644 --- a/test/bigint64.test.ts +++ b/test/bigint64.test.ts @@ -1,5 +1,6 @@ import assert from "assert"; -import { encode, decode } from "../src"; +import { encode, decode, IntMode, ExtensionCodec, DecodeError } from "../src"; +import { getInt64, getUint64 } from "../src/utils/int"; describe("useBigInt64: true", () => { before(function () { @@ -10,29 +11,41 @@ describe("useBigInt64: true", () => { it("encodes and decodes 0n", () => { const value = BigInt(0); - const encoded = encode(value, { useBigInt64: true }); + const encoded = encode(value, { forceBigIntToInt64: true }); assert.deepStrictEqual(decode(encoded, { useBigInt64: true }), value); }); it("encodes and decodes MAX_SAFE_INTEGER+1", () => { const value = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); - const encoded = encode(value, { useBigInt64: true }); + const encoded = encode(value, { forceBigIntToInt64: true }); assert.deepStrictEqual(decode(encoded, { useBigInt64: true }), value); }); it("encodes and decodes MIN_SAFE_INTEGER-1", () => { const value = BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1); - const encoded = encode(value, { useBigInt64: true }); + const encoded = encode(value, { forceBigIntToInt64: true }); assert.deepStrictEqual(decode(encoded, { useBigInt64: true }), value); }); - it("encodes and decodes values with numbers and bigints", () => { + it("encodes and decodes values with numbers and bigints - MIXED", () => { const value = { ints: [0, Number.MAX_SAFE_INTEGER, Number.MIN_SAFE_INTEGER], nums: [Number.NaN, Math.PI, Math.E, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY], + bigints: [BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1)], + }; + const encoded = encode(value, { forceBigIntToInt64: true }); + const decoded = decode(encoded, { intMode: IntMode.MIXED }); + assert.deepStrictEqual(decoded, value); + }); + + it("encodes and decodes values with numbers and bigints - AS_ENCODED", () => { + const value = { + ints: [0, Math.pow(2, 32) - 1, -1 * Math.pow(2, 31)], + nums: [Number.NaN, Math.PI, Math.E, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY], bigints: [BigInt(0), BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1)], }; - const encoded = encode(value, { useBigInt64: true }); - assert.deepStrictEqual(decode(encoded, { useBigInt64: true }), value); + const encoded = encode(value, { forceBigIntToInt64: true }); + const decoded = decode(encoded, { intMode: IntMode.AS_ENCODED }); + assert.deepStrictEqual(decoded, value); }); }); diff --git a/test/codec-bigint.test.ts b/test/codec-bigint.test.ts index 4ed6aff..52b8f41 100644 --- a/test/codec-bigint.test.ts +++ b/test/codec-bigint.test.ts @@ -1,5 +1,6 @@ import assert from "assert"; import { encode, decode, ExtensionCodec, DecodeError } from "../src"; +import { IntMode, getInt64, getUint64 } from "../src/utils/int"; // There's a built-in `useBigInt64: true` option, but a custom codec might be // better if you'd like to encode bigint to reduce the size of binaries. @@ -29,11 +30,229 @@ extensionCodec.register({ }, }); +interface TestCase { + input: bigint; + expected: Map; +} + +// declared as a function to delay referencing the BigInt constructor +function BIGINTSPECS(): Record { + return { + ZERO: { + input: BigInt(0), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0], + [IntMode.SAFE_NUMBER, 0], + [IntMode.MIXED, 0], + [IntMode.BIGINT, BigInt(0)], + ]), + }, + ONE: { + input: BigInt(1), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 1], + [IntMode.SAFE_NUMBER, 1], + [IntMode.MIXED, 1], + [IntMode.BIGINT, BigInt(1)], + ]), + }, + MINUS_ONE: { + input: BigInt(-1), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, -1], + [IntMode.SAFE_NUMBER, -1], + [IntMode.MIXED, -1], + [IntMode.BIGINT, BigInt(-1)], + ]), + }, + X_FF: { + input: BigInt(0xff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0xff], + [IntMode.SAFE_NUMBER, 0xff], + [IntMode.MIXED, 0xff], + [IntMode.BIGINT, BigInt(0xff)], + ]), + }, + MINUS_X_FF: { + input: BigInt(-0xff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, -0xff], + [IntMode.SAFE_NUMBER, -0xff], + [IntMode.MIXED, -0xff], + [IntMode.BIGINT, BigInt(-0xff)], + ]), + }, + INT32_MAX: { + input: BigInt(0x7fffffff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0x7fffffff], + [IntMode.SAFE_NUMBER, 0x7fffffff], + [IntMode.MIXED, 0x7fffffff], + [IntMode.BIGINT, BigInt(0x7fffffff)], + ]), + }, + INT32_MIN: { + input: BigInt(-0x7fffffff - 1), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, -0x7fffffff - 1], + [IntMode.SAFE_NUMBER, -0x7fffffff - 1], + [IntMode.MIXED, -0x7fffffff - 1], + [IntMode.BIGINT, BigInt(-0x7fffffff - 1)], + ]), + }, + MAX_SAFE_INTEGER: { + input: BigInt(Number.MAX_SAFE_INTEGER), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, Number.MAX_SAFE_INTEGER], + [IntMode.SAFE_NUMBER, Number.MAX_SAFE_INTEGER], + [IntMode.MIXED, Number.MAX_SAFE_INTEGER], + [IntMode.BIGINT, BigInt(Number.MAX_SAFE_INTEGER)], + ]), + }, + MAX_SAFE_INTEGER_PLUS_ONE: { + input: BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)], + [IntMode.BIGINT, BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)], + ]), + }, + MIN_SAFE_INTEGER: { + input: BigInt(Number.MIN_SAFE_INTEGER), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, Number.MIN_SAFE_INTEGER], + [IntMode.SAFE_NUMBER, Number.MIN_SAFE_INTEGER], + [IntMode.MIXED, Number.MIN_SAFE_INTEGER], + [IntMode.BIGINT, BigInt(Number.MIN_SAFE_INTEGER)], + ]), + }, + MIN_SAFE_INTEGER_MINUS_ONE: { + input: BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1)], + [IntMode.BIGINT, BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1)], + ]), + }, + INT64_MAX: { + input: BigInt("0x7fffffffffffffff"), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt("0x7fffffffffffffff")], + [IntMode.BIGINT, BigInt("0x7fffffffffffffff")], + ]), + }, + INT64_MIN: { + input: BigInt(-1) * BigInt("0x8000000000000000"), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt(-1) * BigInt("0x8000000000000000")], + [IntMode.BIGINT, BigInt(-1) * BigInt("0x8000000000000000")], + ]), + }, + }; +} + +// declared as a function to delay referencing the BigInt constructor +function BIGUINTSPECS(): Record { + return { + ZERO: { + input: BigInt(0), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0], + [IntMode.SAFE_NUMBER, 0], + [IntMode.MIXED, 0], + [IntMode.BIGINT, BigInt(0)], + ]), + }, + ONE: { + input: BigInt(1), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 1], + [IntMode.SAFE_NUMBER, 1], + [IntMode.MIXED, 1], + [IntMode.BIGINT, BigInt(1)], + ]), + }, + X_FF: { + input: BigInt(0xff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0xff], + [IntMode.SAFE_NUMBER, 0xff], + [IntMode.MIXED, 0xff], + [IntMode.BIGINT, BigInt(0xff)], + ]), + }, + UINT32_MAX: { + input: BigInt(0xffffffff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0xffffffff], + [IntMode.SAFE_NUMBER, 0xffffffff], + [IntMode.MIXED, 0xffffffff], + [IntMode.BIGINT, BigInt(0xffffffff)], + ]), + }, + MAX_SAFE_INTEGER: { + input: BigInt(Number.MAX_SAFE_INTEGER), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, Number.MAX_SAFE_INTEGER], + [IntMode.SAFE_NUMBER, Number.MAX_SAFE_INTEGER], + [IntMode.MIXED, Number.MAX_SAFE_INTEGER], + [IntMode.BIGINT, BigInt(Number.MAX_SAFE_INTEGER)], + ]), + }, + MAX_SAFE_INTEGER_PLUS_ONE: { + input: BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)], + [IntMode.BIGINT, BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)], + ]), + }, + UINT64_MAX: { + input: BigInt("0xffffffffffffffff"), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt("0xffffffffffffffff")], + [IntMode.BIGINT, BigInt("0xffffffffffffffff")], + ]), + }, + }; +} + +function abs(value: bigint): bigint { + if (value < 0) { + return BigInt(-1) * value; + } + return value; +} + describe("codec BigInt", () => { - it("encodes and decodes 0n", () => { - const value = BigInt(0); - const encoded = encode(value, { extensionCodec }); - assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + context("extension", () => { + it("encodes and decodes 0n", () => { + const value = BigInt(0); + const encoded = encode(value, { extensionCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + }); + + it("encodes and decodes MAX_SAFE_INTEGER+1", () => { + const value = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); + const encoded = encode(value, { extensionCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + }); + + it("encodes and decodes MIN_SAFE_INTEGER-1", () => { + const value = BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1); + const encoded = encode(value, { extensionCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + }); }); it("encodes and decodes 100n", () => { @@ -54,9 +273,91 @@ describe("codec BigInt", () => { assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); }); - it("encodes and decodes MIN_SAFE_INTEGER-1", () => { - const value = BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1); - const encoded = encode(value, { extensionCodec }); - assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + context("native", () => { + context("int 64", () => { + const specs = BIGINTSPECS(); + + for (const name of Object.keys(specs)) { + const testCase = specs[name]!; + + it(`sets and gets ${testCase.input} (${testCase.input < 0 ? "-" : ""}0x${abs(testCase.input).toString( + 16, + )})`, () => { + const b = new Uint8Array(8); + const view = new DataView(b.buffer); + view.setBigInt64(0, testCase.input); + for (const [mode, expected] of testCase.expected) { + if (expected === "error") { + assert.throws( + () => getInt64(view, 0, mode), + new RegExp( + `Mode is IntMode\\.SAFE_NUMBER and value is not a safe integer: ${ + testCase.input < 0 ? "-" : "" + }0x${abs(testCase.input).toString(16)}$`, + ), + ); + continue; + } + assert.deepStrictEqual(getInt64(view, 0, mode), expected); + } + }); + } + }); + + context("uint 64", () => { + const specs = BIGUINTSPECS(); + + for (const name of Object.keys(specs)) { + const testCase = specs[name]!; + + it(`sets and gets ${testCase.input} (0x${testCase.input.toString(16)})`, () => { + const b = new Uint8Array(8); + const view = new DataView(b.buffer); + view.setBigUint64(0, testCase.input); + for (const [mode, expected] of testCase.expected) { + if (expected === "error") { + assert.throws( + () => getUint64(view, 0, mode), + new RegExp( + `Mode is IntMode\\.SAFE_NUMBER and value is not a safe integer: 0x${testCase.input.toString(16)}$`, + ), + ); + continue; + } + assert.deepStrictEqual(getUint64(view, 0, mode), expected); + } + }); + } + }); + }); + + context("IntMode.AS_ENCODED vs IntMode.Mixed", () => { + it("decodes 64-bit integers properly", () => { + let input = Uint8Array.from([0xcf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3]); + assert.deepStrictEqual(decode(input, { intMode: IntMode.AS_ENCODED }), BigInt(3)); + assert.deepStrictEqual(decode(input, { intMode: IntMode.MIXED }), 3); + + input = Uint8Array.from([0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf6]); + assert.deepStrictEqual(decode(input, { intMode: IntMode.AS_ENCODED }), BigInt(-10)); + assert.deepStrictEqual(decode(input, { intMode: IntMode.MIXED }), -10); + + input = Uint8Array.from([0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); + assert.deepStrictEqual(decode(input, { intMode: IntMode.AS_ENCODED }), BigInt("0xffffffffffffffff")); + assert.deepStrictEqual(decode(input, { intMode: IntMode.MIXED }), BigInt("0xffffffffffffffff")); + }); + + it("decodes smaller integers properly", () => { + let input = Uint8Array.from([0x03]); + assert.deepStrictEqual(decode(input, { intMode: IntMode.AS_ENCODED }), 3); + assert.deepStrictEqual(decode(input, { intMode: IntMode.MIXED }), 3); + + input = Uint8Array.from([0xf6]); + assert.deepStrictEqual(decode(input, { intMode: IntMode.AS_ENCODED }), -10); + assert.deepStrictEqual(decode(input, { intMode: IntMode.MIXED }), -10); + + input = Uint8Array.from([0xce, 0xff, 0xff, 0xff, 0xff]); + assert.deepStrictEqual(decode(input, { intMode: IntMode.AS_ENCODED }), 0xffffffff); + assert.deepStrictEqual(decode(input, { intMode: IntMode.MIXED }), 0xffffffff); + }); }); }); diff --git a/test/codec-int.test.ts b/test/codec-int.test.ts index 486f93b..7a1f106 100644 --- a/test/codec-int.test.ts +++ b/test/codec-int.test.ts @@ -1,5 +1,5 @@ import assert from "assert"; -import { setInt64, getInt64, getUint64, setUint64 } from "../src/utils/int"; +import { IntMode, setInt64, getInt64, getUint64, setUint64 } from "../src/utils/int"; const INT64SPECS = { ZERO: 0, @@ -22,7 +22,12 @@ describe("codec: int64 / uint64", () => { const b = new Uint8Array(8); const view = new DataView(b.buffer); setInt64(view, 0, value); - assert.deepStrictEqual(getInt64(view, 0), value); + assert.deepStrictEqual(getInt64(view, 0, IntMode.UNSAFE_NUMBER), value); + assert.deepStrictEqual(getInt64(view, 0, IntMode.SAFE_NUMBER), value); + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(getInt64(view, 0, IntMode.MIXED), value); + assert.deepStrictEqual(getInt64(view, 0, IntMode.BIGINT), BigInt(value)); + } }); } }); @@ -32,14 +37,24 @@ describe("codec: int64 / uint64", () => { const b = new Uint8Array(8); const view = new DataView(b.buffer); setUint64(view, 0, 0); - assert.deepStrictEqual(getUint64(view, 0), 0); + assert.deepStrictEqual(getUint64(view, 0, IntMode.UNSAFE_NUMBER), 0); + assert.deepStrictEqual(getUint64(view, 0, IntMode.SAFE_NUMBER), 0); + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(getUint64(view, 0, IntMode.MIXED), 0); + assert.deepStrictEqual(getUint64(view, 0, IntMode.BIGINT), BigInt(0)); + } }); it(`sets and gets MAX_SAFE_INTEGER`, () => { const b = new Uint8Array(8); const view = new DataView(b.buffer); setUint64(view, 0, Number.MAX_SAFE_INTEGER); - assert.deepStrictEqual(getUint64(view, 0), Number.MAX_SAFE_INTEGER); + assert.deepStrictEqual(getUint64(view, 0, IntMode.UNSAFE_NUMBER), Number.MAX_SAFE_INTEGER); + assert.deepStrictEqual(getUint64(view, 0, IntMode.SAFE_NUMBER), Number.MAX_SAFE_INTEGER); + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(getUint64(view, 0, IntMode.MIXED), Number.MAX_SAFE_INTEGER); + assert.deepStrictEqual(getUint64(view, 0, IntMode.BIGINT), BigInt(Number.MAX_SAFE_INTEGER)); + } }); }); }); diff --git a/test/encode.test.ts b/test/encode.test.ts index d2679d5..6f88e4d 100644 --- a/test/encode.test.ts +++ b/test/encode.test.ts @@ -28,6 +28,10 @@ describe("encode", () => { context("forceFloat", () => { it("encodes integers as integers without forceIntegerToFloat", () => { assert.deepStrictEqual(encode(3), Uint8Array.from([0x3])); + + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(encode(BigInt(3)), Uint8Array.from([0x3])); + } }); it("encodes integers as floating point when forceIntegerToFloat=true", () => { @@ -35,6 +39,13 @@ describe("encode", () => { encode(3, { forceIntegerToFloat: true }), Uint8Array.from([0xcb, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), ); + + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual( + encode(BigInt(3), { forceIntegerToFloat: true }), + Uint8Array.from([0xcb, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), + ); + } }); it("encodes integers as float32 when forceIntegerToFloat=true and forceFloat32=true", () => { @@ -42,10 +53,21 @@ describe("encode", () => { encode(3, { forceIntegerToFloat: true, forceFloat32: true }), Uint8Array.from([0xca, 0x40, 0x40, 0x00, 0x00]), ); + + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual( + encode(BigInt(3), { forceIntegerToFloat: true, forceFloat32: true }), + Uint8Array.from([0xca, 0x40, 0x40, 0x00, 0x00]), + ); + } }); it("encodes integers as integers when forceIntegerToFloat=false", () => { assert.deepStrictEqual(encode(3, { forceIntegerToFloat: false }), Uint8Array.from([0x3])); + + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(encode(BigInt(3), { forceIntegerToFloat: false }), Uint8Array.from([0x3])); + } }); }); @@ -71,4 +93,46 @@ describe("encode", () => { const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteLength); assert.deepStrictEqual(decode(arrayBuffer), decode(buffer)); }); + + context("forceBigIntToInt64", () => { + if (typeof BigInt !== "undefined") { + it("encodes bigints as integers without forceBigIntToInt64", () => { + let input = BigInt(3); + let expected = Uint8Array.from([0x03]); + assert.deepStrictEqual(encode(input), expected); + + input = BigInt(-10); + expected = Uint8Array.from([0xf6]); + assert.deepStrictEqual(encode(input), expected); + + input = BigInt("0xffffffffffffffff"); + expected = Uint8Array.from([0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); + assert.deepStrictEqual(encode(input), expected); + }); + + it("encodes bigints as int64 when forceBigIntToInt64=true", () => { + let input = BigInt(3); + let expected = Uint8Array.from([0xcf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03]); + assert.deepStrictEqual(encode(input, { forceBigIntToInt64: true }), expected); + + input = BigInt(-10); + expected = Uint8Array.from([0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf6]); + assert.deepStrictEqual(encode(input, { forceBigIntToInt64: true }), expected); + + input = BigInt("0xffffffffffffffff"); + expected = Uint8Array.from([0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); + assert.deepStrictEqual(encode(input), expected); + }); + } + }); + + context("Bigint that exceeds 64 bits", () => { + if (typeof BigInt !== "undefined") { + const MAX_UINT64_PLUS_ONE = BigInt("0x10000000000000000"); + assert.throws(() => encode(MAX_UINT64_PLUS_ONE), /Bigint is too large for uint64: 18446744073709551616$/); + + const MIN_INT64_MINUS_ONE = BigInt(-1) * BigInt("0x8000000000000001"); + assert.throws(() => encode(MIN_INT64_MINUS_ONE), /Bigint is too small for int64: -9223372036854775809$/); + } + }); }); diff --git a/test/msgpack-test-suite.test.ts b/test/msgpack-test-suite.test.ts index 6800973..fcdbcfc 100644 --- a/test/msgpack-test-suite.test.ts +++ b/test/msgpack-test-suite.test.ts @@ -1,8 +1,10 @@ import assert from "assert"; import util from "util"; -import { Exam } from "msgpack-test-js"; import { MsgTimestamp } from "msg-timestamp"; -import { encode, decode, ExtensionCodec, EXT_TIMESTAMP, encodeTimeSpecToTimestamp } from "@msgpack/msgpack"; +import { MsgUInt64, MsgInt64 } from "msg-int64"; +import { encode, decode, ExtensionCodec, EXT_TIMESTAMP, encodeTimeSpecToTimestamp, IntMode } from "@msgpack/msgpack"; +import { getExams } from "./utils/group"; +import { TypeKey } from "./utils/type"; const extensionCodec = new ExtensionCodec(); extensionCodec.register({ @@ -22,26 +24,43 @@ extensionCodec.register({ }, }); -const TEST_TYPES = { - array: 1, - bignum: 0, // TODO - binary: 1, - bool: 1, - map: 1, - nil: 1, - number: 1, - string: 1, - timestamp: 1, +const TEST_TYPES: Record = { + array: true, + bignum: typeof BigInt !== "undefined", + binary: true, + bool: true, + map: true, + nil: true, + number: true, + string: true, + timestamp: true, + ext: false, }; +function convertValueForEncoding(value: unknown): unknown { + if (value instanceof MsgInt64 || value instanceof MsgUInt64) { + return BigInt(value.toString()); + } + + return value; +} + +function convertValueForDecoding(value: unknown): unknown { + if (typeof value === "bigint") { + return value.toString(); + } + + return value; +} + describe("msgpack-test-suite", () => { - Exam.getExams(TEST_TYPES).forEach((exam) => { + getExams(TEST_TYPES).forEach((exam) => { const types = exam.getTypes(TEST_TYPES); const first = types[0]!; const title = `${first}: ${exam.stringify(first)}`; it(`encodes ${title}`, () => { types.forEach((type) => { - const value = exam.getValue(type); + const value = convertValueForEncoding(exam.getValue(type)); const buffer = Buffer.from(encode(value, { extensionCodec })); if (exam.matchMsgpack(buffer)) { @@ -58,7 +77,7 @@ describe("msgpack-test-suite", () => { it(`decodes ${title}`, () => { const msgpacks = exam.getMsgpacks(); msgpacks.forEach((encoded, idx) => { - const value = decode(encoded, { extensionCodec }); + const value = convertValueForDecoding(decode(encoded, { extensionCodec, intMode: IntMode.MIXED })); if (exam.matchValue(value)) { assert(true, exam.stringify(idx)); } else { diff --git a/test/utils/exam.ts b/test/utils/exam.ts new file mode 100644 index 0000000..ed7eef3 --- /dev/null +++ b/test/utils/exam.ts @@ -0,0 +1,75 @@ +/** + * From https://github.com/kawanet/msgpack-test-js/blob/master/lib/exam.js + */ + +import { Type, TypeKey } from "./type"; + +export type Suite = Record & { msgpack: Array }; + +const binary = Type.getType("binary"); + +export class Exam { + src: Suite; + msgpack?: Array; + + constructor(src: Suite) { + this.src = src ?? ({} as Suite); + } + + getMsgpacks(): Array { + return this.msgpack || (this.msgpack = this.parseAllMsgpack(this.src)); + } + + getTypes(filter?: Record): Array { + const src = this.src; + + return ( + (Object.keys(src) as Array) + .filter((type) => { + return !filter || filter[type]; + }) + .map(function (type) { + return Type.getType(type); + }) + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + .filter((x) => !!x) + ); + } + + getValue(type: Type | TypeKey): any { + if (!(type instanceof Type)) { + type = Type.getType(type); + } + return type.parse(this.src[type.name]); + } + + matchMsgpack(encoded: Buffer): boolean { + return this.getMsgpacks().some(function (check) { + return binary.compare(encoded, check); + }); + } + + matchValue(value: any): boolean { + return this.getTypes().some((type) => { + return type.compare(value, this.getValue(type)); + }); + } + + stringify(idx: number | Type | TypeKey): string { + if (typeof idx === "number") { + return this.src.msgpack[idx]!; + } + + const type = idx instanceof Type ? idx : Type.getType(idx); + + if (type) { + return JSON.stringify(this.src[type.name]); + } + + throw new Error(`${idx} not supported`); + } + + private parseAllMsgpack(src: Suite): Array { + return src.msgpack.map((r) => binary.parse(r)); + } +} diff --git a/test/utils/group.ts b/test/utils/group.ts new file mode 100644 index 0000000..1f4e6b1 --- /dev/null +++ b/test/utils/group.ts @@ -0,0 +1,38 @@ +/** + * From https://github.com/kawanet/msgpack-test-js/blob/master/lib/group.js + */ + +import suite from "./msgpack-test-suite"; +import { Exam, Suite } from "./exam"; +import { TypeKey } from "./type"; + +export class Group { + name: keyof typeof suite; + + constructor(name: keyof typeof suite) { + this.name = name; + } + + static getGroups(): Array { + return (Object.keys(suite) as Array).sort().map((s) => new Group(s)); + } + + getExams(filter?: Record): Array { + const name = this.name; + const array = suite[name]; + + return array + .map((x) => new Exam(x as Suite)) + .filter(function (exam) { + return !filter || exam.getTypes(filter).length; + }); + } + + toString(): string { + return this.name; + } +} + +export function getExams(filter?: Record): Array { + return Group.getGroups().flatMap((group) => group.getExams(filter)); +} diff --git a/test/utils/msgpack-test-suite.ts b/test/utils/msgpack-test-suite.ts new file mode 100644 index 0000000..33d0883 --- /dev/null +++ b/test/utils/msgpack-test-suite.ts @@ -0,0 +1,502 @@ +// From https://rawgit.com/kawanet/msgpack-test-suite/master/dist/msgpack-test-suite.json +export default { + "10.nil.yaml": [ + { + "nil": null, + "msgpack": ["c0"], + }, + ], + "11.bool.yaml": [ + { + "bool": false, + "msgpack": ["c2"], + }, + { + "bool": true, + "msgpack": ["c3"], + }, + ], + "12.binary.yaml": [ + { + "binary": "", + "msgpack": ["c4-00", "c5-00-00", "c6-00-00-00-00"], + }, + { + "binary": "01", + "msgpack": ["c4-01-01", "c5-00-01-01", "c6-00-00-00-01-01"], + }, + { + "binary": "00-ff", + "msgpack": ["c4-02-00-ff", "c5-00-02-00-ff", "c6-00-00-00-02-00-ff"], + }, + ], + "20.number-positive.yaml": [ + { + "number": 0, + "msgpack": [ + "00", + "cc-00", + "cd-00-00", + "ce-00-00-00-00", + "cf-00-00-00-00-00-00-00-00", + "d0-00", + "d1-00-00", + "d2-00-00-00-00", + "d3-00-00-00-00-00-00-00-00", + "ca-00-00-00-00", + "cb-00-00-00-00-00-00-00-00", + ], + }, + { + "number": 1, + "msgpack": [ + "01", + "cc-01", + "cd-00-01", + "ce-00-00-00-01", + "cf-00-00-00-00-00-00-00-01", + "d0-01", + "d1-00-01", + "d2-00-00-00-01", + "d3-00-00-00-00-00-00-00-01", + "ca-3f-80-00-00", + "cb-3f-f0-00-00-00-00-00-00", + ], + }, + { + "number": 127, + "msgpack": [ + "7f", + "cc-7f", + "cd-00-7f", + "ce-00-00-00-7f", + "cf-00-00-00-00-00-00-00-7f", + "d0-7f", + "d1-00-7f", + "d2-00-00-00-7f", + "d3-00-00-00-00-00-00-00-7f", + ], + }, + { + "number": 128, + "msgpack": [ + "cc-80", + "cd-00-80", + "ce-00-00-00-80", + "cf-00-00-00-00-00-00-00-80", + "d1-00-80", + "d2-00-00-00-80", + "d3-00-00-00-00-00-00-00-80", + ], + }, + { + "number": 255, + "msgpack": [ + "cc-ff", + "cd-00-ff", + "ce-00-00-00-ff", + "cf-00-00-00-00-00-00-00-ff", + "d1-00-ff", + "d2-00-00-00-ff", + "d3-00-00-00-00-00-00-00-ff", + ], + }, + { + "number": 256, + "msgpack": [ + "cd-01-00", + "ce-00-00-01-00", + "cf-00-00-00-00-00-00-01-00", + "d1-01-00", + "d2-00-00-01-00", + "d3-00-00-00-00-00-00-01-00", + ], + }, + { + "number": 65535, + "msgpack": [ + "cd-ff-ff", + "ce-00-00-ff-ff", + "cf-00-00-00-00-00-00-ff-ff", + "d2-00-00-ff-ff", + "d3-00-00-00-00-00-00-ff-ff", + ], + }, + { + "number": 65536, + "msgpack": ["ce-00-01-00-00", "cf-00-00-00-00-00-01-00-00", "d2-00-01-00-00", "d3-00-00-00-00-00-01-00-00"], + }, + { + "number": 2147483647, + "msgpack": ["ce-7f-ff-ff-ff", "cf-00-00-00-00-7f-ff-ff-ff", "d2-7f-ff-ff-ff", "d3-00-00-00-00-7f-ff-ff-ff"], + }, + { + "number": 2147483648, + "msgpack": [ + "ce-80-00-00-00", + "cf-00-00-00-00-80-00-00-00", + "d3-00-00-00-00-80-00-00-00", + "ca-4f-00-00-00", + "cb-41-e0-00-00-00-00-00-00", + ], + }, + { + "number": 4294967295, + "msgpack": [ + "ce-ff-ff-ff-ff", + "cf-00-00-00-00-ff-ff-ff-ff", + "d3-00-00-00-00-ff-ff-ff-ff", + "cb-41-ef-ff-ff-ff-e0-00-00", + ], + }, + ], + "21.number-negative.yaml": [ + { + "number": -1, + "msgpack": [ + "ff", + "d0-ff", + "d1-ff-ff", + "d2-ff-ff-ff-ff", + "d3-ff-ff-ff-ff-ff-ff-ff-ff", + "ca-bf-80-00-00", + "cb-bf-f0-00-00-00-00-00-00", + ], + }, + { + "number": -32, + "msgpack": [ + "e0", + "d0-e0", + "d1-ff-e0", + "d2-ff-ff-ff-e0", + "d3-ff-ff-ff-ff-ff-ff-ff-e0", + "ca-c2-00-00-00", + "cb-c0-40-00-00-00-00-00-00", + ], + }, + { + "number": -33, + "msgpack": ["d0-df", "d1-ff-df", "d2-ff-ff-ff-df", "d3-ff-ff-ff-ff-ff-ff-ff-df"], + }, + { + "number": -128, + "msgpack": ["d0-80", "d1-ff-80", "d2-ff-ff-ff-80", "d3-ff-ff-ff-ff-ff-ff-ff-80"], + }, + { + "number": -256, + "msgpack": ["d1-ff-00", "d2-ff-ff-ff-00", "d3-ff-ff-ff-ff-ff-ff-ff-00"], + }, + { + "number": -32768, + "msgpack": ["d1-80-00", "d2-ff-ff-80-00", "d3-ff-ff-ff-ff-ff-ff-80-00"], + }, + { + "number": -65536, + "msgpack": ["d2-ff-ff-00-00", "d3-ff-ff-ff-ff-ff-ff-00-00"], + }, + { + "number": -2147483648, + "msgpack": ["d2-80-00-00-00", "d3-ff-ff-ff-ff-80-00-00-00", "cb-c1-e0-00-00-00-00-00-00"], + }, + ], + "22.number-float.yaml": [ + { + "number": 0.5, + "msgpack": ["ca-3f-00-00-00", "cb-3f-e0-00-00-00-00-00-00"], + }, + { + "number": -0.5, + "msgpack": ["ca-bf-00-00-00", "cb-bf-e0-00-00-00-00-00-00"], + }, + ], + "23.number-bignum.yaml": [ + { + "number": 4294967296, + "bignum": "4294967296", + "msgpack": [ + "cf-00-00-00-01-00-00-00-00", + "d3-00-00-00-01-00-00-00-00", + "ca-4f-80-00-00", + "cb-41-f0-00-00-00-00-00-00", + ], + }, + { + "number": -4294967296, + "bignum": "-4294967296", + "msgpack": ["d3-ff-ff-ff-ff-00-00-00-00", "cb-c1-f0-00-00-00-00-00-00"], + }, + { + "number": 281474976710656, + "bignum": "281474976710656", + "msgpack": [ + "cf-00-01-00-00-00-00-00-00", + "d3-00-01-00-00-00-00-00-00", + "ca-57-80-00-00", + "cb-42-f0-00-00-00-00-00-00", + ], + }, + { + "number": -281474976710656, + "bignum": "-281474976710656", + "msgpack": ["d3-ff-ff-00-00-00-00-00-00", "ca-d7-80-00-00", "cb-c2-f0-00-00-00-00-00-00"], + }, + { + "bignum": "9223372036854775807", + "msgpack": ["d3-7f-ff-ff-ff-ff-ff-ff-ff", "cf-7f-ff-ff-ff-ff-ff-ff-ff"], + }, + { + "bignum": "-9223372036854775807", + "msgpack": ["d3-80-00-00-00-00-00-00-01"], + }, + { + "bignum": "9223372036854775808", + "msgpack": ["cf-80-00-00-00-00-00-00-00"], + }, + { + "bignum": "-9223372036854775808", + "msgpack": ["d3-80-00-00-00-00-00-00-00"], + }, + { + "bignum": "18446744073709551615", + "msgpack": ["cf-ff-ff-ff-ff-ff-ff-ff-ff"], + }, + ], + "30.string-ascii.yaml": [ + { + "string": "", + "msgpack": ["a0", "d9-00", "da-00-00", "db-00-00-00-00"], + }, + { + "string": "a", + "msgpack": ["a1-61", "d9-01-61", "da-00-01-61", "db-00-00-00-01-61"], + }, + { + "string": "1234567890123456789012345678901", + "msgpack": [ + "bf-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31", + "d9-1f-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31", + "da-00-1f-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31", + ], + }, + { + "string": "12345678901234567890123456789012", + "msgpack": [ + "d9-20-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31-32", + "da-00-20-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31-32-33-34-35-36-37-38-39-30-31-32", + ], + }, + ], + "31.string-utf8.yaml": [ + { + "string": "Кириллица", + "msgpack": [ + "b2-d0-9a-d0-b8-d1-80-d0-b8-d0-bb-d0-bb-d0-b8-d1-86-d0-b0", + "d9-12-d0-9a-d0-b8-d1-80-d0-b8-d0-bb-d0-bb-d0-b8-d1-86-d0-b0", + ], + }, + { + "string": "ひらがな", + "msgpack": ["ac-e3-81-b2-e3-82-89-e3-81-8c-e3-81-aa", "d9-0c-e3-81-b2-e3-82-89-e3-81-8c-e3-81-aa"], + }, + { + "string": "한글", + "msgpack": ["a6-ed-95-9c-ea-b8-80", "d9-06-ed-95-9c-ea-b8-80"], + }, + { + "string": "汉字", + "msgpack": ["a6-e6-b1-89-e5-ad-97", "d9-06-e6-b1-89-e5-ad-97"], + }, + { + "string": "漢字", + "msgpack": ["a6-e6-bc-a2-e5-ad-97", "d9-06-e6-bc-a2-e5-ad-97"], + }, + ], + "32.string-emoji.yaml": [ + { + "string": "❤", + "msgpack": ["a3-e2-9d-a4", "d9-03-e2-9d-a4"], + }, + { + "string": "🍺", + "msgpack": ["a4-f0-9f-8d-ba", "d9-04-f0-9f-8d-ba"], + }, + ], + "40.array.yaml": [ + { + "array": [], + "msgpack": ["90", "dc-00-00", "dd-00-00-00-00"], + }, + { + "array": [1], + "msgpack": ["91-01", "dc-00-01-01", "dd-00-00-00-01-01"], + }, + { + "array": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + "msgpack": [ + "9f-01-02-03-04-05-06-07-08-09-0a-0b-0c-0d-0e-0f", + "dc-00-0f-01-02-03-04-05-06-07-08-09-0a-0b-0c-0d-0e-0f", + "dd-00-00-00-0f-01-02-03-04-05-06-07-08-09-0a-0b-0c-0d-0e-0f", + ], + }, + { + "array": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + "msgpack": [ + "dc-00-10-01-02-03-04-05-06-07-08-09-0a-0b-0c-0d-0e-0f-10", + "dd-00-00-00-10-01-02-03-04-05-06-07-08-09-0a-0b-0c-0d-0e-0f-10", + ], + }, + { + "array": ["a"], + "msgpack": ["91-a1-61", "dc-00-01-a1-61", "dd-00-00-00-01-a1-61"], + }, + ], + "41.map.yaml": [ + { + "map": {}, + "msgpack": ["80", "de-00-00", "df-00-00-00-00"], + }, + { + "map": { + "a": 1, + }, + "msgpack": ["81-a1-61-01", "de-00-01-a1-61-01", "df-00-00-00-01-a1-61-01"], + }, + { + "map": { + "a": "A", + }, + "msgpack": ["81-a1-61-a1-41", "de-00-01-a1-61-a1-41", "df-00-00-00-01-a1-61-a1-41"], + }, + ], + "42.nested.yaml": [ + { + "array": [[]], + "msgpack": ["91-90", "dc-00-01-dc-00-00", "dd-00-00-00-01-dd-00-00-00-00"], + }, + { + "array": [{}], + "msgpack": ["91-80", "dc-00-01-80", "dd-00-00-00-01-80"], + }, + { + "map": { + "a": {}, + }, + "msgpack": ["81-a1-61-80", "de-00-01-a1-61-de-00-00", "df-00-00-00-01-a1-61-df-00-00-00-00"], + }, + { + "map": { + "a": [], + }, + "msgpack": ["81-a1-61-90", "de-00-01-a1-61-90", "df-00-00-00-01-a1-61-90"], + }, + ], + "50.timestamp.yaml": [ + { + "timestamp": [1514862245, 0], + "msgpack": ["d6-ff-5a-4a-f6-a5"], + }, + { + "timestamp": [1514862245, 678901234], + "msgpack": ["d7-ff-a1-dc-d7-c8-5a-4a-f6-a5"], + }, + { + "timestamp": [2147483647, 999999999], + "msgpack": ["d7-ff-ee-6b-27-fc-7f-ff-ff-ff"], + }, + { + "timestamp": [2147483648, 0], + "msgpack": ["d6-ff-80-00-00-00"], + }, + { + "timestamp": [2147483648, 1], + "msgpack": ["d7-ff-00-00-00-04-80-00-00-00"], + }, + { + "timestamp": [4294967295, 0], + "msgpack": ["d6-ff-ff-ff-ff-ff"], + }, + { + "timestamp": [4294967295, 999999999], + "msgpack": ["d7-ff-ee-6b-27-fc-ff-ff-ff-ff"], + }, + { + "timestamp": [4294967296, 0], + "msgpack": ["d7-ff-00-00-00-01-00-00-00-00"], + }, + { + "timestamp": [17179869183, 999999999], + "msgpack": ["d7-ff-ee-6b-27-ff-ff-ff-ff-ff"], + }, + { + "timestamp": [17179869184, 0], + "msgpack": ["c7-0c-ff-00-00-00-00-00-00-00-04-00-00-00-00"], + }, + { + "timestamp": [-1, 0], + "msgpack": ["c7-0c-ff-00-00-00-00-ff-ff-ff-ff-ff-ff-ff-ff"], + }, + { + "timestamp": [-1, 999999999], + "msgpack": ["c7-0c-ff-3b-9a-c9-ff-ff-ff-ff-ff-ff-ff-ff-ff"], + }, + { + "timestamp": [0, 0], + "msgpack": ["d6-ff-00-00-00-00"], + }, + { + "timestamp": [0, 1], + "msgpack": ["d7-ff-00-00-00-04-00-00-00-00"], + }, + { + "timestamp": [1, 0], + "msgpack": ["d6-ff-00-00-00-01"], + }, + { + "timestamp": [-2208988801, 999999999], + "msgpack": ["c7-0c-ff-3b-9a-c9-ff-ff-ff-ff-ff-7c-55-81-7f"], + }, + { + "timestamp": [-2208988800, 0], + "msgpack": ["c7-0c-ff-00-00-00-00-ff-ff-ff-ff-7c-55-81-80"], + }, + { + "timestamp": [-62167219200, 0], + "msgpack": ["c7-0c-ff-00-00-00-00-ff-ff-ff-f1-86-8b-84-00"], + }, + { + "timestamp": [253402300799, 999999999], + "msgpack": ["c7-0c-ff-3b-9a-c9-ff-00-00-00-3a-ff-f4-41-7f"], + }, + ], + "60.ext.yaml": [ + { + "ext": [1, "10"], + "msgpack": ["d4-01-10"], + }, + { + "ext": [2, "20-21"], + "msgpack": ["d5-02-20-21"], + }, + { + "ext": [3, "30-31-32-33"], + "msgpack": ["d6-03-30-31-32-33"], + }, + { + "ext": [4, "40-41-42-43-44-45-46-47"], + "msgpack": ["d7-04-40-41-42-43-44-45-46-47"], + }, + { + "ext": [5, "50-51-52-53-54-55-56-57-58-59-5a-5b-5c-5d-5e-5f"], + "msgpack": ["d8-05-50-51-52-53-54-55-56-57-58-59-5a-5b-5c-5d-5e-5f"], + }, + { + "ext": [6, ""], + "msgpack": ["c7-00-06", "c8-00-00-06", "c9-00-00-00-00-06"], + }, + { + "ext": [7, "70-71-72"], + "msgpack": ["c7-03-07-70-71-72", "c8-00-03-07-70-71-72", "c9-00-00-00-03-07-70-71-72"], + }, + ], +}; diff --git a/test/utils/type.ts b/test/utils/type.ts new file mode 100644 index 0000000..0bf12b9 --- /dev/null +++ b/test/utils/type.ts @@ -0,0 +1,159 @@ +/** + * From: https://github.com/kawanet/msgpack-test-js/blob/master/lib/type.js + */ + +import Int64 from "msg-int64"; +import { MsgTimestamp } from "msg-timestamp"; +import { MsgExt } from "msg-ext"; + +export type TypeKey = + | "array" + | "bignum" + | "binary" + | "bool" + | "ext" + | "map" + | "nil" + | "number" + | "string" + | "timestamp"; + +function parseBignum(str: string) { + let value = str; + const orig = (value += ""); + const parser = value.startsWith("-") ? Int64.MsgInt64 : Int64.MsgUInt64; + value = value.replace(/0x/, ""); + const radix = value !== orig ? 16 : 10; + return new parser(value, radix); +} + +function parseBinary(str: string) { + const array = str ? str.split(/[^0-9a-fA-F]+/g).map(parseHex) : []; + return Buffer.from ? Buffer.from(array) : new Buffer(array); +} + +function parseExt(array: any) { + const type = array[0]; + const buffer = parseBinary(array[1]); + return new MsgExt(buffer, type); +} + +function parseHex(str: string) { + return parseInt(str, 16) || 0; +} + +function parseTimestamp(array: any) { + return MsgTimestamp.from(array[0], array[1]); +} + +function compareBinary(a: any, b: any) { + if (!a) { + return false; + } + if (!b) { + return false; + } + + const aLen = a.length; + const bLen = b.length; + if (aLen !== bLen) { + return false; + } + + return [].every.call(a, function (value, idx) { + return value === b[idx]; + }); +} + +function compareExt(a: any, b: any) { + if (!a) { + return false; + } + if (!b) { + return false; + } + + return a.type === b.type && compareBinary(a.buffer, b.buffer); +} + +function compareString(a: any, b: any) { + return "" + a === "" + b; +} + +function compareNumber(a: any, b: any) { + return +a === +b; +} + +function compareStrict(a: any, b: any) { + return a === b; +} + +function compareDeep(a: any, b: any) { + return ( + JSON.stringify(a, ((_obj: any, _key: string, value: any) => + typeof value === "bigint" ? `BIGINT:${value}` : value) as (this: any, key: string, value: any) => any) === + JSON.stringify(b, ((_obj: any, _key: string, value: any) => + typeof value === "bigint" ? `BIGINT:${value}` : value) as (this: any, key: string, value: any) => any) + ); +} + +function compareMap(a: any, b: any) { + if (!a) { + return false; + } + if (!b) { + return false; + } + + const aKeys = Object.keys(a); + const bKeys = Object.keys(b); + if (aKeys.length !== bKeys.length) { + return false; + } + + return [].every.call(aKeys, function (key) { + return key in b && compareDeep(a[key], b[key]); + }); +} + +export class Type { + static types: Record = { + array: new Type("array", compareDeep), + bignum: new Type("bignum", compareString, parseBignum), + binary: new Type("binary", compareBinary, parseBinary), + bool: new Type("bool"), + ext: new Type("ext", compareExt, parseExt), + map: new Type("map", compareMap), + nil: new Type("nil"), + number: new Type("number", compareNumber), + string: new Type("string", compareString), + timestamp: new Type("timestamp", compareString, parseTimestamp), + }; + + name: TypeKey; + comparer?: (a: any, b: any) => boolean; + parser?: (value: any) => any; + + constructor(name: TypeKey, comparer?: (a: any, b: any) => boolean, parser?: (value: any) => any) { + this.name = name; + this.comparer = comparer; + this.parser = parser; + } + + static getType(type: TypeKey): Type { + return Type.types[type]; + } + + // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types + compare(a: any, b: any): boolean { + return this.comparer?.(a, b) ?? compareStrict(a, b); + } + + parse(value: any): any { + return this.parser ? this.parser(value) : value; + } + + toString(): string { + return this.name; + } +}