diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index f9d19809..58f3e3f5 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -2,9 +2,9 @@ name: test-nodejs on: push: - branches: [ master ] + branches: [ encoder] pull_request: - branches: [ master ] + branches: [ encoder ] jobs: build: diff --git a/.gitignore b/.gitignore index d1d42235..a4a4c932 100644 --- a/.gitignore +++ b/.gitignore @@ -18,5 +18,5 @@ node_modules coverage/ .nyc_output/ dist/ - +.vscode/ .cache/ diff --git a/README.md b/README.md index 54a14101..1b91ebe2 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,18 @@ [![npm](https://img.shields.io/npm/v/binary-parser)](https://www.npmjs.com/package/binary-parser) [![license](https://img.shields.io/github/license/keichi/binary-parser)](https://github.com/keichi/binary-parser/blob/master/LICENSE) -Binary-parser is a parser builder for JavaScript that enables you to write -efficient binary parsers in a simple and declarative manner. +Until the *encoding* feature is merged in baseline of original project, +this branch is published under the name: **binary-parser-encoder** in [npm](https://npmjs.org/). + +[![build](https://github.com/Ericbla/binary-parser/workflows/build/badge.svg)](https://github.com/Ericbla/binary-parser/actions?query=workflow%3Abuild) +[![npm](https://img.shields.io/npm/v/binary-parser-encoder)](https://www.npmjs.com/package/binary-parser-encoder) + +Binary-parser is a parser/encoder builder for JavaScript that enables you to write +efficient binary parsers/encoders in a simple and declarative manner. It supports all common data types required to analyze a structured binary -data. Binary-parser dynamically generates and compiles the parser code -on-the-fly, which runs as fast as a hand-written parser (which takes much more +data. Binary-parser dynamically generates and compiles the parser and encoder code +on-the-fly, which runs as fast as a hand-written parser/encoder (which takes much more time and effort to write). Supported data types are: - [Integers](#uint8-16-32-64le-bename-options) (8, 16, 32 and 64 bit signed @@ -32,12 +38,14 @@ and [binary](https://github.com/substack/node-binary). ## Quick Start 1. Create an empty `Parser` object with `new Parser()` or `Parser.start()`. -2. Chain methods to build your desired parser. (See [API](#api) for detailed +2. Chain methods to build your desired parser and/or encoder. (See [API](#api) for detailed documentation of each method) 3. Call `Parser.prototype.parse` with a `Buffer`/`Uint8Array` object passed as its only argument. 4. The parsed result will be returned as an object. - If parsing failed, an exception will be thrown. +5. Or call `Parser.prototype.encode` with an object passed as argument. +6. Encoded result will be returned as a `Buffer` object. ```javascript // Module import @@ -73,6 +81,23 @@ const buf = Buffer.from("450002c5939900002c06ef98adc24f6c850186d1", "hex"); // Parse buffer and show result console.log(ipHeader.parse(buf)); + +var anIpHeader = { + version: 4, + headerLength: 5, + tos: 0, + packetLength: 709, + id: 37785, + offset: 0, + fragOffset: 0, + ttl: 44, + protocol: 6, + checksum: 61336, + src: [ 173, 194, 79, 108 ], + dst: [ 133, 1, 134, 209 ] }; + +// Encode an IP header object and show result as hex string +console.log(ipHeader.encode(anIpHeader).toString("hex")); ``` ## Installation @@ -87,14 +112,22 @@ The npm package provides entry points for both CommonJS and ES modules. ## API -### new Parser() +### new Parser([options]) Create an empty parser object that parses nothing. +`options` is an optional object to pass options to this declarative +parser. + - `smartBufferSize` The chunk size of the encoding (smart)buffer (when encoding is used) (default is 256 bytes). ### parse(buffer) Parse a `Buffer`/`Uint8Array` object `buffer` with this parser and return the resulting object. When `parse(buffer)` is called for the first time, the associated parser code is compiled on-the-fly and internally cached. +### encode(obj) +Encode an `Object` object `obj` with this parser and return the resulting +`Buffer`. When `encode(obj)` is called for the first time, encoder code is +compiled on-the-fly and internally cached. + ### create(constructorFunction) Set the constructor function that should be called to create the object returned from the `parse` method. @@ -151,12 +184,23 @@ the following keys: - `length ` - (Optional) Length of the string. Can be a number, string or a function. Use number for statically sized arrays, string to reference another variable and function to do some calculation. + Note: When encoding the string is padded with a `padd` charecter to fit the length requirement. - `zeroTerminated` - (Optional, defaults to `false`) If true, then this parser - reads until it reaches zero. + reads until it reaches zero (or the specified `length`). When encoding, a *null* character is inserted at end of + the string (if the optional `length` allows it). - `greedy` - (Optional, defaults to `false`) If true, then this parser reads - until it reaches the end of the buffer. Will consume zero-bytes. + until it reaches the end of the buffer. Will consume zero-bytes. (Note: has + no effect on encoding function) - `stripNull` - (Optional, must be used with `length`) If true, then strip - null characters from end of the string. + null characters from end of the string. (Note: When encoding, this will also set the **default** `padd` character + to null instead of space) +- `trim` - (Optional, default to `false`) If true, then trim() (remove leading and trailing spaces) + the parsed string. +- `padding` - (Optional, Only used for encoding, default to `right`) If `left` then the string + will be right aligned (padding left with `padd` char or space) depending of the `length` option +- `padd` - (Optional, Only used for encoding with `length` specified) A string from which first character (1 Byte) + is used as a padding char if necessary (provided string length is less than `length` option). Note: Only 'ascii' + or utf8 < 0x80 are alowed. Note: The default padd character is *space* (or *null* when `stripNull` is used). ### buffer(name[, options]) Parse bytes as a buffer. Its type will be the same as the input to @@ -175,7 +219,8 @@ keys: calculation. - `readUntil` - (either `length` or `readUntil` is required) If `"eof"`, then this parser will read till it reaches the end of the `Buffer`/`Uint8Array` - object. If it is a function, this parser will read the buffer until the + object. (Note: has no effect on encoding.) + If it is a function, this parser will read the buffer until the function returns true. ### array(name, options) @@ -194,6 +239,12 @@ keys: - `readUntil` - (either `length`, `lengthInBytes`, or `readUntil` is required) If `"eof"`, then this parser reads until the end of the `Buffer`/`Uint8Array` object. If function it reads until the function returns true. + **Note**: When encoding, + the `buffer` second parameter of `readUntil` function is the buffer already encoded + before this array. So no *read-ahead* is possible. +- `encodeUntil` - a function (item, object), only used when encoding, that replaces + the `readUntil` function when present and allow limit the number of encoded items + by returning true based on *item* values or other *object* properies. ```javascript const parser = new Parser() @@ -331,7 +382,7 @@ const parser = new Parser() ### seek(relOffset) Move the buffer offset for `relOffset` bytes from the current position. Use a negative `relOffset` value to rewind the offset. This method was previously -named `skip(length)`. +named `skip(length)`. (Note: when encoding, the skipped bytes will be filled with zeros) ### endianness(endianness) Define what endianness to use in this parser. `endianness` can be either @@ -348,6 +399,21 @@ const parser = new Parser() .int32("c"); ``` +### encoderSetOptions(opts) +Set specific options for encoding. +Current supported `opts` object may contain: + - bitEndianess: true|false (default false) When true, tell the encoder to respect endianess BITs order, so that + encoding is exactly the reverse of the parsing process for bits fields. + +```javascript +var parser = new Parser() + .endianess("little") + .encoderSetOptions({bitEndianess: true}) // Use BITs endianess for bits fields + .bit4("a") + .bit4("b") + .uint16("c"); +``` + ### namely(alias) Set an alias to this parser, so that it can be referred to by name in methods like `.array`, `.nest` and `.choice`, without the requirement to have an @@ -495,26 +561,44 @@ mainParser.parse(buffer); Returns how many bytes this parser consumes. If the size of the parser cannot be statically determined, a `NaN` is returned. -### compile() +### compile() and compileEncode() Compile this parser on-the-fly and cache its result. Usually, there is no need -to call this method directly, since it's called when `parse(buffer)` is +to call this method directly, since it's called when `parse(buffer)` or `encode(obj)` is executed for the first time. -### getCode() -Dynamically generates the code for this parser and returns it as a string. +### getCode() and getCodeEncode() +Dynamically generates the code for this parser/encoder and returns it as a string. Useful for debugging the generated code. ### Common options These options can be used in all parsers. - `formatter` - Function that transforms the parsed value into a more desired - form. + form. *formatter*(value, obj, buffer, offset) → *new value* \ + where `value` is the value to be formatted, `obj` is the current object being generated, `buffer` is the buffer currently beeing parsed and `offset` is the current offset in that buffer. + ```javascript + const parser = new Parser().array("ipv4", { + type: uint8, + length: "4", + formatter: function(arr, obj, buffer, offset) { + return arr.join("."); + } + }); + ``` + +- `encoder` - Function that transforms an object property into a more desired + form for encoding. This is the opposite of the above `formatter` function. \ + *encoder*(value) → *new value* \ + where `value` is the value to be encoded (de-formatted) and `obj` is the object currently being encoded. ```javascript const parser = new Parser().array("ipv4", { type: uint8, length: "4", - formatter: function(arr) { + formatter: function(arr, obj, buffer, offset) { return arr.join("."); + }, + encoder: function(str, obj) { + return str.split("."); } }); ``` diff --git a/lib/binary_parser.ts b/lib/binary_parser.ts index b9afa25d..b49b70e5 100644 --- a/lib/binary_parser.ts +++ b/lib/binary_parser.ts @@ -1,3 +1,5 @@ +import { SmartBuffer } from "smart-buffer"; + class Context { code = ""; scopes = [["vars"]]; @@ -109,6 +111,7 @@ class Context { const aliasRegistry = new Map(); const FUNCTION_PREFIX = "___parser_"; +const FUNCTION_ENCODE_PREFIX = "___encoder_"; interface ParserOptions { length?: number | string | ((item: any) => number); @@ -116,7 +119,9 @@ interface ParserOptions { lengthInBytes?: number | string | ((item: any) => number); type?: string | Parser; formatter?: (item: any) => any; + encoder?: (item: any) => any; encoding?: string; + encodeUntil?: "eof" | ((item: any, buffer: Buffer) => number); readUntil?: "eof" | ((item: any, buffer: Buffer) => boolean); greedy?: boolean; choices?: { [key: number]: string | Parser }; @@ -125,11 +130,18 @@ interface ParserOptions { clone?: boolean; stripNull?: boolean; key?: string; + trim?: boolean; + padding?: string; + padd?: string; tag?: string | ((item: any) => number); offset?: number | string | ((item: any) => number); wrapper?: (buffer: Buffer) => Buffer; } +interface EncoderOptions { + bitEndianess?: boolean; +} + type Types = PrimitiveTypes | ComplexTypes; type ComplexTypes = @@ -274,6 +286,38 @@ const PRIMITIVE_LITTLE_ENDIANS: { [key in PrimitiveTypes]: boolean } = { doublebe: false, }; +const CAPITILIZED_TYPE_NAMES: { [key in Types]: string } = { + uint8: "UInt8", + uint16le: "UInt16LE", + uint16be: "UInt16BE", + uint32le: "UInt32LE", + uint32be: "UInt32BE", + int8: "Int8", + int16le: "Int16LE", + int16be: "Int16BE", + int32le: "Int32LE", + int32be: "Int32BE", + int64be: "BigInt64BE", + int64le: "BigInt64LE", + uint64be: "BigUInt64BE", + uint64le: "BigUInt64LE", + floatle: "FloatLE", + floatbe: "FloatBE", + doublele: "DoubleLE", + doublebe: "DoubleBE", + bit: "Bit", + string: "String", + buffer: "Buffer", + array: "Array", + choice: "Choice", + nest: "Nest", + seek: "Seek", + pointer: "Pointer", + saveOffset: "SaveOffset", + "": "", + wrapper: "Wrapper", +}; + export class Parser { varName = ""; type: Types = ""; @@ -285,11 +329,22 @@ export class Parser { constructorFn?: Function; alias?: string; useContextVariables = false; + compiledEncode: Function | null = null; + smartBufferSize: number; + encoderOpts: EncoderOptions; - constructor() {} + constructor(opts?: any) { + this.smartBufferSize = + opts && typeof opts === "object" && opts.smartBufferSize + ? opts.smartBufferSize + : 256; + this.encoderOpts = { + bitEndianess: false, + }; + } - static start() { - return new Parser(); + static start(opts?: any) { + return new Parser(opts); } private primitiveGenerateN(type: PrimitiveTypes, ctx: Context) { @@ -303,6 +358,14 @@ export class Parser { ctx.pushCode(`offset += ${PRIMITIVE_SIZES[type]};`); } + private primitiveGenerate_encodeN(type: PrimitiveTypes, ctx: Context) { + const typeName = CAPITILIZED_TYPE_NAMES[type]; + + ctx.pushCode( + `smartBuffer.write${typeName}(${ctx.generateVariable(this.varName)});`, + ); + } + private primitiveN( type: PrimitiveTypes, varName: string, @@ -744,6 +807,12 @@ export class Parser { return this; } + encoderSetOptions(opts: EncoderOptions) { + Object.assign(this.encoderOpts, opts); + + return this; + } + endianess(endianess: "little" | "big"): this { return this.endianness(endianess); } @@ -786,6 +855,27 @@ export class Parser { return this.getContext(importPath).code; } + private getContextEncode(importPath: string) { + const ctx = new Context(importPath, this.useContextVariables); + + ctx.pushCode('if (!obj || typeof obj !== "object") {'); + ctx.generateError('"argument obj is not an object"'); + ctx.pushCode("}"); + + if (!this.alias) { + this.addRawCodeEncode(ctx); + } else { + this.addAliasedCodeEncode(ctx); + ctx.pushCode(`return ${FUNCTION_ENCODE_PREFIX + this.alias}(obj);`); + } + + return ctx; + } + + getCodeEncode() { + return this.getContextEncode("").code; // TODO: Not sure "" is a valid input here + } + private addRawCode(ctx: Context) { ctx.pushCode("var offset = 0;"); ctx.pushCode( @@ -804,6 +894,25 @@ export class Parser { ctx.pushCode("return vars;"); } + private addRawCodeEncode(ctx: Context) { + ctx.pushCode("var vars = obj || {};"); + ctx.pushCode("vars.$parent = null;"); + ctx.pushCode("vars.$root = vars;"); + + ctx.pushCode( + `var smartBuffer = SmartBuffer.fromOptions({size: ${this.smartBufferSize}, encoding: "utf8"});`, + ); + + this.generateEncode(ctx); + + ctx.pushCode("delete vars.$parent;"); + ctx.pushCode("delete vars.$root;"); + + this.resolveReferences(ctx, "encode"); + + ctx.pushCode("return smartBuffer.toBuffer();"); + } + private addAliasedCode(ctx: Context) { ctx.pushCode(`function ${FUNCTION_PREFIX + this.alias}(offset, context) {`); ctx.pushCode( @@ -828,11 +937,41 @@ export class Parser { return ctx; } - private resolveReferences(ctx: Context) { + private addAliasedCodeEncode(ctx: Context) { + ctx.pushCode( + `function ${FUNCTION_ENCODE_PREFIX + this.alias}(obj, context) {`, + ); + + ctx.pushCode("var vars = obj || {};"); + ctx.pushCode( + "var ctx = Object.assign({$parent: null, $root: vars}, context || {});", + ); + ctx.pushCode(`vars = Object.assign(vars, ctx);`); + ctx.pushCode( + `var smartBuffer = SmartBuffer.fromOptions({size: ${this.smartBufferSize}, encoding: "utf8"});`, + ); + + this.generateEncode(ctx); + + ctx.markResolved(this.alias!); + this.resolveReferences(ctx, "encode"); + + ctx.pushCode("return { result: smartBuffer.toBuffer() };"); + ctx.pushCode("}"); + + return ctx; + } + + private resolveReferences(ctx: Context, encode?: string) { const references = ctx.getUnresolvedReferences(); ctx.markRequested(references); references.forEach((alias) => { - aliasRegistry.get(alias)?.addAliasedCode(ctx); + const parser = aliasRegistry.get(alias); + if (encode) { + parser?.addAliasedCodeEncode(ctx); + } else { + parser?.addAliasedCode(ctx); + } }); } @@ -846,6 +985,25 @@ export class Parser { )(ctx.imports, TextDecoder); } + compileEncode() { + const importPath = "imports"; + const ctx = this.getContextEncode(importPath); + this.compiledEncode = new Function( + importPath, + "TextDecoder", + "SmartBuffer", + `return function (obj) { ${ctx.code} };`, + )( + ctx.imports, + typeof TextDecoder === "undefined" + ? require("util").TextDecoder + : TextDecoder, + typeof SmartBuffer === "undefined" + ? require("smart-buffer").SmartBuffer + : SmartBuffer, + ); + } + sizeOf(): number { let size = NaN; @@ -906,6 +1064,19 @@ export class Parser { return this.compiled!(buffer, this.constructorFn); } + // Follow the parser chain till the root and start encoding from there + encode(obj: any) { + if (!this.compiledEncode) { + this.compileEncode(); + } + + const encoded = this.compiledEncode!(obj); + if (encoded.result) { + return encoded.result; + } + return encoded; + } + private setNextParser( type: Types, varName: string, @@ -917,6 +1088,7 @@ export class Parser { parser.varName = varName; parser.options = options; parser.endian = this.endian; + parser.encoderOpts = this.encoderOpts; if (this.head) { this.head.next = parser; @@ -994,6 +1166,78 @@ export class Parser { return this.generateNext(ctx); } + generateEncode(ctx: Context) { + var savVarName = ctx.generateTmpVariable(); + const varName = ctx.generateVariable(this.varName); + + // Transform with the possibly provided encoder before encoding + if (this.options.encoder) { + ctx.pushCode(`var ${savVarName} = ${varName}`); + this.generateEncoder(ctx, varName, this.options.encoder); + } + + if (this.type) { + switch (this.type) { + case "uint8": + case "uint16le": + case "uint16be": + case "uint32le": + case "uint32be": + case "int8": + case "int16le": + case "int16be": + case "int32le": + case "int32be": + case "int64be": + case "int64le": + case "uint64be": + case "uint64le": + case "floatle": + case "floatbe": + case "doublele": + case "doublebe": + this.primitiveGenerate_encodeN(this.type, ctx); + break; + case "bit": + this.generate_encodeBit(ctx); + break; + case "string": + this.generate_encodeString(ctx); + break; + case "buffer": + this.generate_encodeBuffer(ctx); + break; + case "seek": + this.generate_encodeSeek(ctx); + break; + case "nest": + this.generate_encodeNest(ctx); + break; + case "array": + this.generate_encodeArray(ctx); + break; + case "choice": + this.generate_encodeChoice(ctx); + break; + case "pointer": + this.generate_encodePointer(ctx); + break; + case "saveOffset": + this.generate_encodeSaveOffset(ctx); + break; + } + this.generateAssert(ctx); + } + + if (this.options.encoder) { + // Restore varName after encoder transformation so that next parsers will + // have access to original field value (but not nested ones) + ctx.pushCode(`${varName} = ${savVarName};`); + } + + return this.generateEncodeNext(ctx); + } + private generateAssert(ctx: Context) { if (!this.options.assert) { return; @@ -1038,6 +1282,15 @@ export class Parser { return ctx; } + // Recursively call code generators and append results + private generateEncodeNext(ctx: Context) { + if (this.next) { + ctx = this.next.generateEncode(ctx); + } + + return ctx; + } + private nextNotBit() { // Used to test if next type is a bitN or not if (this.next) { @@ -1161,11 +1414,87 @@ export class Parser { } } + private generate_encodeBit(ctx: Context) { + // TODO find better method to handle nested bit fields + const parser = JSON.parse(JSON.stringify(this)); + parser.varName = ctx.generateVariable(parser.varName); + ctx.bitFields.push(parser); + + if ( + !this.next || + (this.next && ["bit", "nest"].indexOf(this.next.type) < 0) + ) { + let sum = 0; + ctx.bitFields.forEach((parser) => { + sum += parser.options.length as number; + }); + + if (sum <= 8) { + sum = 8; + } else if (sum <= 16) { + sum = 16; + } else if (sum <= 24) { + sum = 24; + } else if (sum <= 32) { + sum = 32; + } else { + throw new Error( + "Currently, bit field sequences longer than 4-bytes is not supported.", + ); + } + + const isBitLittleEndian = + this.endian === "le" && this.encoderOpts.bitEndianess; + const tmpVal = ctx.generateTmpVariable(); + const boundVal = ctx.generateTmpVariable(); + ctx.pushCode(`var ${tmpVal} = 0;`); + ctx.pushCode(`var ${boundVal} = 0;`); + let bitOffset = 0; + ctx.bitFields.forEach((parser) => { + ctx.pushCode( + `${boundVal} = (${parser.varName} & ${ + (1 << (parser.options.length as number)) - 1 + });`, + ); + ctx.pushCode( + `${tmpVal} |= (${boundVal} << ${ + isBitLittleEndian + ? bitOffset + : sum - (parser.options.length as number) - bitOffset + });`, + ); + ctx.pushCode(`${tmpVal} = ${tmpVal} >>> 0;`); + bitOffset += parser.options.length as number; + }); + if (sum == 8) { + ctx.pushCode(`smartBuffer.writeUInt8(${tmpVal});`); + } else if (sum == 16) { + ctx.pushCode(`smartBuffer.writeUInt16BE(${tmpVal});`); + } else if (sum == 24) { + const val1 = ctx.generateTmpVariable(); + const val2 = ctx.generateTmpVariable(); + ctx.pushCode(`var ${val1} = (${tmpVal} >>> 8);`); + ctx.pushCode(`var ${val2} = (${tmpVal} & 0x0ff);`); + ctx.pushCode(`smartBuffer.writeUInt16BE(${val1});`); + ctx.pushCode(`smartBuffer.writeUInt8(${val2});`); + } else if (sum == 32) { + ctx.pushCode(`smartBuffer.writeUInt32BE(${tmpVal});`); + } + + ctx.bitFields = []; + } + } + private generateSeek(ctx: Context) { const length = ctx.generateOption(this.options.length!); ctx.pushCode(`offset += ${length};`); } + private generate_encodeSeek(ctx: Context) { + const length = ctx.generateOption(this.options.length!); + ctx.pushCode(`smartBuffer.writeBuffer(Buffer.alloc(${length}));`); + } + private generateString(ctx: Context) { const name = ctx.generateVariable(this.varName); const start = ctx.generateTmpVariable(); @@ -1179,7 +1508,8 @@ export class Parser { ctx.pushCode( `while(dataView.getUint8(offset++) !== 0 && offset - ${start} < ${len});`, ); - const end = `offset - ${start} < ${len} ? offset - 1 : offset`; + //const end = `offset - ${start} < ${len} ? offset - 1 : offset`; + const end = "dataView.getUint8(offset -1) == 0 ? offset - 1 : offset"; ctx.pushCode( isHex ? `${name} = Array.from(buffer.subarray(${start}, ${end}), ${toHex}).join('');` @@ -1213,6 +1543,63 @@ export class Parser { if (this.options.stripNull) { ctx.pushCode(`${name} = ${name}.replace(/\\x00+$/g, '')`); } + if (this.options.trim) { + ctx.pushCode(`${name} = ${name}.trim()`); + } + } + + private generate_encodeString(ctx: Context) { + const name = ctx.generateVariable(this.varName); + + // Get the length of string to encode + if (this.options.length) { + const optLength = ctx.generateOption(this.options.length); + // Encode the string to a temporary buffer + const tmpBuf = ctx.generateTmpVariable(); + ctx.pushCode( + `var ${tmpBuf} = Buffer.from(${name}, "${this.options.encoding}");`, + ); + // Truncate the buffer to specified (Bytes) length + ctx.pushCode(`${tmpBuf} = ${tmpBuf}.slice(0, ${optLength});`); + // Compute padding length + const padLen = ctx.generateTmpVariable(); + ctx.pushCode(`${padLen} = ${optLength} - ${tmpBuf}.length;`); + if (this.options.zeroTerminated) { + ctx.pushCode(`smartBuffer.writeBuffer(${tmpBuf});`); + ctx.pushCode(`if (${padLen} > 0) { smartBuffer.writeUInt8(0x00); }`); + } else { + const padCharVar = ctx.generateTmpVariable(); + let padChar = this.options.stripNull ? "\u0000" : " "; + if (this.options.padd && typeof this.options.padd === "string") { + const code = this.options.padd.charCodeAt(0); + if (code < 0x80) { + padChar = String.fromCharCode(code); + } + } + ctx.pushCode(`${padCharVar} = "${padChar}";`); + if (this.options.padding === "left") { + // Add heading padding spaces + ctx.pushCode( + `if (${padLen} > 0) {smartBuffer.writeString(${padCharVar}.repeat(${padLen}));}`, + ); + } + // Copy the temporary string buffer to current smartBuffer + ctx.pushCode(`smartBuffer.writeBuffer(${tmpBuf});`); + if (this.options.padding !== "left") { + // Add trailing padding spaces + ctx.pushCode( + `if (${padLen} > 0) {smartBuffer.writeString(${padCharVar}.repeat(${padLen}));}`, + ); + } + } + } else { + ctx.pushCode( + `smartBuffer.writeString(${name}, "${this.options.encoding}");`, + ); + if (this.options.zeroTerminated) { + ctx.pushCode("smartBuffer.writeUInt8(0x00);"); + } + } } private generateBuffer(ctx: Context) { @@ -1248,6 +1635,12 @@ export class Parser { } } + private generate_encodeBuffer(ctx: Context) { + ctx.pushCode( + `smartBuffer.writeBuffer(${ctx.generateVariable(this.varName)});`, + ); + } + private generateArray(ctx: Context) { const length = ctx.generateOption(this.options.length!); const lengthInBytes = ctx.generateOption(this.options.lengthInBytes!); @@ -1344,6 +1737,103 @@ export class Parser { } } + private generate_encodeArray(ctx: Context) { + const length = ctx.generateOption(this.options.length!); + const lengthInBytes = ctx.generateOption(this.options.lengthInBytes!); + const type = this.options.type; + const name = ctx.generateVariable(this.varName); + const item = ctx.generateTmpVariable(); + const itemCounter = ctx.generateTmpVariable(); + const maxItems = ctx.generateTmpVariable(); + const isHash = typeof this.options.key === "string"; + + if (isHash) { + ctx.generateError('"Encoding associative array not supported"'); + } + + ctx.pushCode(`var ${maxItems} = 0;`); + + // Get default array length (if defined) + ctx.pushCode(`if(${name}) {${maxItems} = ${name}.length;}`); + + // Compute the desired count of array items to encode (min of array size + // and length option) + if (length !== undefined) { + ctx.pushCode( + `${maxItems} = ${maxItems} > ${length} ? ${length} : ${maxItems}`, + ); + } + + // Save current encoding smartBuffer and allocate a new one + const savSmartBuffer = ctx.generateTmpVariable(); + ctx.pushCode( + `var ${savSmartBuffer} = smartBuffer; ` + + `smartBuffer = SmartBuffer.fromOptions({size: ${this.smartBufferSize}, encoding: "utf8"});`, + ); + + ctx.pushCode(`if(${maxItems} > 0) {`); + + ctx.pushCode(`var ${itemCounter} = 0;`); + if ( + typeof this.options.encodeUntil === "function" || + typeof this.options.readUntil === "function" + ) { + ctx.pushCode("do {"); + } else { + ctx.pushCode(`for ( ; ${itemCounter} < ${maxItems}; ) {`); + } + + ctx.pushCode(`var ${item} = ${name}[${itemCounter}];`); + ctx.pushCode(`${itemCounter}++;`); + + if (typeof type === "string") { + if (!aliasRegistry.get(type)) { + ctx.pushCode( + `smartBuffer.write${ + CAPITILIZED_TYPE_NAMES[type as PrimitiveTypes] + }(${item});`, + ); + } else { + ctx.pushCode( + `smartBuffer.writeBuffer(${ + FUNCTION_ENCODE_PREFIX + type + }(${item}).result);`, + ); + if (type !== this.alias) { + ctx.addReference(type); + } + } + } else if (type instanceof Parser) { + ctx.pushScope(item); + type.generateEncode(ctx); + ctx.popScope(); + } + + ctx.pushCode("}"); // End of 'do {' or 'for (...) {' + + if (typeof this.options.encodeUntil === "function") { + ctx.pushCode( + ` while (${itemCounter} < ${maxItems} && !(${this.options.encodeUntil}).call(this, ${item}, vars));`, + ); + } else if (typeof this.options.readUntil === "function") { + ctx.pushCode( + ` while (${itemCounter} < ${maxItems} && !(${this.options.readUntil}).call(this, ${item}, ${savSmartBuffer}.toBuffer()));`, + ); + } + ctx.pushCode("}"); // End of 'if(...) {' + + const tmpBuffer = ctx.generateTmpVariable(); + ctx.pushCode(`var ${tmpBuffer} = smartBuffer.toBuffer()`); + if (lengthInBytes !== undefined) { + // Truncate the tmpBuffer so that it will respect the lengthInBytes option + ctx.pushCode(`${tmpBuffer} = ${tmpBuffer}.slice(0, ${lengthInBytes});`); + } + // Copy tmp Buffer to saved smartBuffer + ctx.pushCode(`${savSmartBuffer}.writeBuffer(${tmpBuffer});`); + // Restore current smartBuffer + ctx.pushCode(`smartBuffer = ${savSmartBuffer};`); + } + private generateChoiceCase( ctx: Context, varName: string, @@ -1378,6 +1868,42 @@ export class Parser { } } + private generate_encodeChoiceCase( + ctx: Context, + varName: string, + type: string | Parser, + ) { + if (typeof type === "string") { + if (!aliasRegistry.has(type)) { + ctx.pushCode( + `smartBuffer.write${ + CAPITILIZED_TYPE_NAMES[type as Types] + }(${ctx.generateVariable(this.varName)});`, + ); + } else { + var tempVar = ctx.generateTmpVariable(); + ctx.pushCode( + `var ${tempVar} = ${ + FUNCTION_ENCODE_PREFIX + type + }(${ctx.generateVariable(this.varName)}, {`, + ); + if (ctx.useContextVariables) { + const parentVar = ctx.generateVariable(); + ctx.pushCode(`$parent: ${parentVar},`); + ctx.pushCode(`$root: ${parentVar}.$root,`); + } + ctx.pushCode(`});`); + + ctx.pushCode(`smartBuffer.writeBuffer(${tempVar}.result);`); + if (type !== this.alias) ctx.addReference(type); + } + } else if (type instanceof Parser) { + ctx.pushPath(varName); + type.generateEncode(ctx); + ctx.popPath(varName); + } + } + private generateChoice(ctx: Context) { const tag = ctx.generateOption(this.options.tag!); const nestVar = ctx.generateVariable(this.varName); @@ -1414,6 +1940,42 @@ export class Parser { } } + private generate_encodeChoice(ctx: Context) { + const tag = ctx.generateOption(this.options.tag!); + const nestVar = ctx.generateVariable(this.varName); + + if (this.varName && ctx.useContextVariables) { + const parentVar = ctx.generateVariable(); + ctx.pushCode(`${nestVar}.$parent = ${parentVar};`); + ctx.pushCode(`${nestVar}.$root = ${parentVar}.$root;`); + } + ctx.pushCode(`switch(${tag}) {`); + for (const tagString in this.options.choices) { + const tag = parseInt(tagString, 10); + const type = this.options.choices[tag]; + + ctx.pushCode(`case ${tag}:`); + this.generate_encodeChoiceCase(ctx, this.varName, type); + ctx.pushCode("break;"); + } + ctx.pushCode("default:"); + if (this.options.defaultChoice) { + this.generate_encodeChoiceCase( + ctx, + this.varName, + this.options.defaultChoice, + ); + } else { + ctx.generateError(`"Met undefined tag value " + ${tag} + " at choice"`); + } + ctx.pushCode("}"); + + if (this.varName && ctx.useContextVariables) { + ctx.pushCode(`delete ${nestVar}.$parent;`); + ctx.pushCode(`delete ${nestVar}.$root;`); + } + } + private generateNest(ctx: Context) { const nestVar = ctx.generateVariable(this.varName); @@ -1458,6 +2020,47 @@ export class Parser { } } + private generate_encodeNest(ctx: Context) { + const nestVar = ctx.generateVariable(this.varName); + + if (this.options.type instanceof Parser) { + if (this.varName && ctx.useContextVariables) { + const parentVar = ctx.generateVariable(); + ctx.pushCode(`${nestVar}.$parent = ${parentVar};`); + ctx.pushCode(`${nestVar}.$root = ${parentVar}.$root;`); + } + + ctx.pushPath(this.varName); + this.options.type.generateEncode(ctx); + ctx.popPath(this.varName); + + if (this.varName && ctx.useContextVariables) { + if (ctx.useContextVariables) { + ctx.pushCode(`delete ${nestVar}.$parent;`); + ctx.pushCode(`delete ${nestVar}.$root;`); + } + } + } else if (aliasRegistry.has(this.options.type!)) { + var tempVar = ctx.generateTmpVariable(); + ctx.pushCode( + `var ${tempVar} = ${ + FUNCTION_ENCODE_PREFIX + this.options.type + }(${nestVar}, {`, + ); + if (ctx.useContextVariables) { + const parentVar = ctx.generateVariable(); + ctx.pushCode(`$parent: ${parentVar},`); + ctx.pushCode(`$root: ${parentVar}.$root,`); + } + ctx.pushCode(`});`); + + ctx.pushCode(`smartBuffer.writeBuffer(${tempVar}.result);`); + if (this.options.type !== this.alias) { + ctx.addReference(this.options.type!); + } + } + } + private generateWrapper(ctx: Context) { const wrapperVar = ctx.generateVariable(this.varName); const wrappedBuf = ctx.generateTmpVariable(); @@ -1539,6 +2142,14 @@ export class Parser { } } + private generateEncoder(ctx: Context, varName: string, encoder?: Function) { + if (typeof encoder === "function") { + ctx.pushCode( + `${varName} = (${encoder}).call(${ctx.generateVariable()}, ${varName}, vars);`, + ); + } + } + private generatePointer(ctx: Context) { const type = this.options.type; const offset = ctx.generateOption(this.options.offset!); @@ -1598,8 +2209,18 @@ export class Parser { ctx.pushCode(`offset = ${tempVar};`); } + // @ts-ignore TS6133 + private generate_encodePointer(ctx: Context) { + // TODO + } + private generateSaveOffset(ctx: Context) { const varName = ctx.generateVariable(this.varName); ctx.pushCode(`${varName} = offset`); } + + // @ts-ignore TS6133 + private generate_encodeSaveOffset(ctx: Context) { + // TODO + } } diff --git a/package-lock.json b/package-lock.json index b8b7b497..daa0873d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "binary-parser", "version": "2.2.1", "license": "MIT", + "dependencies": { + "smart-buffer": "~4.2.0" + }, "devDependencies": { "@types/mocha": "^10.0.1", "@types/node": "^20.4.2", @@ -21,7 +24,7 @@ "typescript": "^5.1.6" }, "engines": { - "node": ">=12" + "node": ">=14" } }, "node_modules/@babel/code-frame": { @@ -3545,6 +3548,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/socket.io": { "version": "4.7.1", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.7.1.tgz", @@ -6908,6 +6920,11 @@ "object-inspect": "^1.9.0" } }, + "smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==" + }, "socket.io": { "version": "4.7.1", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.7.1.tgz", diff --git a/package.json b/package.json index 9755ca50..23879911 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "2.2.1", "description": "Blazing-fast binary parser builder", "main": "dist/binary_parser.js", + "types": "dist/binary_parser.d.ts", "module": "dist/esm/binary_parser.mjs", "devDependencies": { "@types/mocha": "^10.0.1", @@ -29,7 +30,7 @@ "build:esm": "tsc --module esnext --outDir dist/esm && mv dist/esm/binary_parser.js dist/esm/binary_parser.mjs", "format": "prettier --list-different \"{lib,example,test,benchmark}/**/*.{ts,js}\"", "format:fix": "prettier --write \"{lib,example,test,benchmark}/**/*.{ts,js}\"", - "test": "mocha --require ts-node/register test/*.ts", + "test": "mocha --require ts-node/register test/*.{js,ts}", "test:browser": "karma start --single-run --browsers ChromeHeadless karma.conf.js", "prepare": "npm run build" }, @@ -41,6 +42,8 @@ "binary", "parser", "decode", + "encoder", + "encode", "unpack", "struct", "buffer", @@ -51,13 +54,22 @@ "email": "hello@keichi.dev", "url": "https://keichi.dev/" }, + "contributors": [ + { + "name": "Eric Blanchard", + "email": "ericbla@gmail.com" + } + ], "license": "MIT", "repository": { "type": "git", - "url": "http://github.com/keichi/binary-parser.git" + "url": "http://github.com/Ericbla/binary-parser.git" }, - "bugs": "http://github.com/keichi/binary-parser/issues", + "bugs": "http://github.com/Ericbla/binary-parser/issues", "engines": { "node": ">=14" + }, + "dependencies": { + "smart-buffer": "~4.2.0" } } diff --git a/test/yy_primitive_encoder.js b/test/yy_primitive_encoder.js new file mode 100644 index 00000000..ecdd4ba7 --- /dev/null +++ b/test/yy_primitive_encoder.js @@ -0,0 +1,461 @@ +var assert = require("assert"); +var util = require("util"); +var Parser = require("../dist/binary_parser").Parser; + +describe("Primitive encoder", function () { + describe("Primitive encoders", function () { + it("should nothing", function () { + var parser = Parser.start(); + + var buffer = parser.encode({ a: 0, b: 1 }); + assert.deepEqual(buffer.length, 0); + }); + it("should encode integer types", function () { + var parser = Parser.start().uint8("a").int16le("b").uint32be("c"); + + var buffer = Buffer.from([0x00, 0xd2, 0x04, 0x00, 0xbc, 0x61, 0x4e]); + var parsed = parser.parse(buffer); + var encoded = parser.encode(parsed); + assert.deepEqual( + { a: parsed.a, b: parsed.b, c: parsed.c }, + { a: 0, b: 1234, c: 12345678 }, + ); + assert.deepEqual(encoded, buffer); + }); + describe("BigInt64 encoders", () => { + const [major] = process.version.replace("v", "").split("."); + if (Number(major) >= 12) { + it("should encode biguints64", () => { + const parser = Parser.start().uint64be("a").uint64le("b"); + // from https://nodejs.org/api/buffer.html#buffer_buf_readbiguint64le_offset + const buf = Buffer.from([ + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, + ]); + let parsed = parser.parse(buf); + assert.deepEqual(parsed, { + a: BigInt("4294967295"), + b: BigInt("18446744069414584320"), + }); + let encoded = parser.encode(parsed); + assert.deepEqual(encoded, buf); + }); + + it("should encode bigints64", () => { + const parser = Parser.start() + .int64be("a") + .int64le("b") + .int64be("c") + .int64le("d"); + // from https://nodejs.org/api/buffer.html#buffer_buf_readbiguint64le_offset + const buf = Buffer.from([ + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + ]); + let parsed = parser.parse(buf); + assert.deepEqual(parsed, { + a: BigInt("4294967295"), + b: BigInt("-4294967295"), + c: BigInt("4294967295"), + d: BigInt("-4294967295"), + }); + let encoded = parser.encode(parsed); + assert.deepEqual(encoded, buf); + }); + } else { + it("should throw when run under not v12", () => { + assert.throws(() => Parser.start().bigint64("a")); + }); + } + }); + it("should use encoder to transform to integer", function () { + var parser = Parser.start() + .uint8("a", { + formatter: function (val) { + return val * 2; + }, + encoder: function (val) { + return val / 2; + }, + }) + .int16le("b", { + formatter: function (val) { + return "test" + String(val); + }, + encoder: function (val) { + return parseInt(val.substr("test".length)); + }, + }); + + var buffer = Buffer.from([0x01, 0xd2, 0x04]); + var parsed = parser.parse(buffer); + var parsedClone = Object.assign({}, parsed); + var encoded = parser.encode(parsedClone); + assert.deepEqual(parsed, { a: 2, b: "test1234" }); + assert.deepEqual(encoded, buffer); + }); + it("should encode floating point types", function () { + var parser = Parser.start().floatbe("a").doublele("b"); + + var FLT_EPSILON = 0.00001; + var buffer = Buffer.from([ + 0x41, 0x45, 0x85, 0x1f, 0x7a, 0x36, 0xab, 0x3e, 0x57, 0x5b, 0xb1, 0xbf, + ]); + var result = parser.parse(buffer); + + assert(Math.abs(result.a - 12.345) < FLT_EPSILON); + assert(Math.abs(result.b - -0.0678) < FLT_EPSILON); + var encoded = parser.encode(result); + assert.deepEqual(encoded, buffer); + }); + it("should handle endianess", function () { + var parser = Parser.start().int32le("little").int32be("big"); + + var buffer = Buffer.from([ + 0x4e, 0x61, 0xbc, 0x00, 0x00, 0xbc, 0x61, 0x4e, + ]); + var parsed = parser.parse(buffer); + assert.deepEqual(parsed, { + little: 12345678, + big: 12345678, + }); + var encoded = parser.encode(parsed); + assert.deepEqual(encoded, buffer); + }); + it("should skip when specified", function () { + var parser = Parser.start() + .uint8("a") + .skip(3) + .uint16le("b") + .uint32be("c"); + + var buffer = Buffer.from([ + 0x00, + 0x00, // Skipped will be encoded as Null + 0x00, // Skipped will be encoded as Null + 0x00, // Skipped will be encoded as Null + 0xd2, + 0x04, + 0x00, + 0xbc, + 0x61, + 0x4e, + ]); + var parsed = parser.parse(buffer); + assert.deepEqual(parsed, { a: 0, b: 1234, c: 12345678 }); + var encoded = parser.encode(parsed); + assert.deepEqual(encoded, buffer); + }); + }); + + describe("Bit field encoders", function () { + var binaryLiteral = function (s) { + var i; + var bytes = []; + + s = s.replace(/\s/g, ""); + for (i = 0; i < s.length; i += 8) { + bytes.push(parseInt(s.slice(i, i + 8), 2)); + } + + return Buffer.from(bytes); + }; + + it("binary literal helper should work", function () { + assert.deepEqual(binaryLiteral("11110000"), Buffer.from([0xf0])); + assert.deepEqual( + binaryLiteral("11110000 10100101"), + Buffer.from([0xf0, 0xa5]), + ); + }); + + it("should encode 1-byte-length 8 bit field", function () { + var parser = new Parser().bit8("a"); + + var buf = binaryLiteral("11111111"); + + assert.deepEqual(parser.parse(buf), { a: 255 }); + assert.deepEqual(parser.encode({ a: 255 }), buf); + }); + + it("should encode 1-byte-length 2x 4 bit fields", function () { + var parser = new Parser().bit4("a").bit4("b"); + + var buf = binaryLiteral("1111 1111"); + + assert.deepEqual(parser.parse(buf), { a: 15, b: 15 }); + assert.deepEqual(parser.encode({ a: 15, b: 15 }), buf); + }); + + it("should encode 1-byte-length bit field sequence", function () { + var parser = new Parser().bit1("a").bit2("b").bit4("c").bit1("d"); + + var buf = binaryLiteral("1 10 1010 0"); + var decoded = parser.parse(buf); + assert.deepEqual(decoded, { + a: 1, + b: 2, + c: 10, + d: 0, + }); + + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buf); + + // Endianess will change nothing you still specify bits for left to right + parser = new Parser() + .endianess("little") + .bit1("a") + .bit2("b") + .bit4("c") + .bit1("d"); + + encoded = parser.encode({ + a: 1, + b: 2, + c: 10, + d: 0, + }); + assert.deepEqual(encoded, buf); + }); + it("should parse 2-byte-length bit field sequence", function () { + var parser = new Parser().bit3("a").bit9("b").bit4("c"); + + var buf = binaryLiteral("101 111000111 0111"); + var decoded = parser.parse(buf); + assert.deepEqual(decoded, { + a: 5, + b: 455, + c: 7, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buf); + }); + it("should parse 4-byte-length bit field sequence", function () { + var parser = new Parser() + .bit1("a") + .bit24("b") + .bit4("c") + .bit2("d") + .bit1("e"); + var buf = binaryLiteral("1 101010101010101010101010 1111 01 1"); + var decoded = parser.parse(buf); + assert.deepEqual(decoded, { + a: 1, + b: 11184810, + c: 15, + d: 1, + e: 1, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buf); + }); + it("should parse nested bit fields", function () { + var parser = new Parser().bit1("a").nest("x", { + type: new Parser().bit2("b").bit4("c").bit1("d"), + }); + + var buf = binaryLiteral("1 10 1010 0"); + var decoded = parser.parse(buf); + assert.deepEqual(decoded, { + a: 1, + x: { + b: 2, + c: 10, + d: 0, + }, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buf); + }); + }); + + describe("String encoder", function () { + it("should encode ASCII encoded string", function () { + var text = "hello, world"; + var buffer = Buffer.from(text, "utf8"); + var parser = Parser.start().string("msg", { + length: buffer.length, + encoding: "utf8", + }); + + var decoded = parser.parse(buffer); + assert.equal(decoded.msg, text); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode UTF8 encoded string", function () { + var text = "こんにちは、せかい。"; + var buffer = Buffer.from(text, "utf8"); + var parser = Parser.start().string("msg", { + length: buffer.length, + encoding: "utf8", + }); + + var decoded = parser.parse(buffer); + assert.equal(decoded.msg, text); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode HEX encoded string", function () { + var text = "cafebabe"; + var buffer = Buffer.from(text, "hex"); + var parser = Parser.start().string("msg", { + length: buffer.length, + encoding: "hex", + }); + + var decoded = parser.parse(buffer); + assert.equal(decoded.msg, text); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode variable length string", function () { + var buffer = Buffer.from("0c68656c6c6f2c20776f726c64", "hex"); + var parser = Parser.start() + .uint8("length") + .string("msg", { length: "length", encoding: "utf8" }); + + var decoded = parser.parse(buffer); + assert.equal(decoded.msg, "hello, world"); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode zero terminated string", function () { + var buffer = Buffer.from("68656c6c6f2c20776f726c6400", "hex"); + var parser = Parser.start().string("msg", { + zeroTerminated: true, + encoding: "utf8", + }); + + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { msg: "hello, world" }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode zero terminated fixed-length string", function () { + var buffer = Buffer.from("abc\u0000defghij\u0000"); + var parser = Parser.start() + .string("a", { length: 5, zeroTerminated: true }) + .string("b", { length: 5, zeroTerminated: true }) + .string("c", { length: 5, zeroTerminated: true }); + + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + a: "abc", + b: "defgh", + c: "ij", + }); + let encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + + encoded = parser.encode({ + a: "a234", + b: "b2345", + c: "c2345678", + }); + assert.deepEqual(encoded, Buffer.from("a234\u0000b2345c2345")); + }); + it("should strip trailing null characters", function () { + var buffer = Buffer.from("746573740000", "hex"); + var parser1 = Parser.start().string("str", { + length: 6, + stripNull: false, + }); + var parser2 = Parser.start().string("str", { + length: 6, + stripNull: true, + }); + + var decoded1 = parser1.parse(buffer); + assert.equal(decoded1.str, "test\u0000\u0000"); + var encoded1 = parser1.encode(decoded1); + assert.deepEqual(encoded1, buffer); + + var decoded2 = parser2.parse(buffer); + assert.equal(decoded2.str, "test"); + var encoded2 = parser2.encode(decoded2); + assert.deepEqual(encoded2, buffer); + }); + it("should encode string with zero-bytes internally", function () { + var buffer = Buffer.from("abc\u0000defghij\u0000"); + var parser = Parser.start().string("a", { greedy: true }); + + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + a: "abc\u0000defghij\u0000", + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode string with default right padding", function () { + var parser = Parser.start().string("a", { length: 6 }); + var encoded = parser.encode({ a: "abcd" }); + assert.deepEqual(encoded, Buffer.from("abcd ")); + encoded = parser.encode({ a: "abcdefgh" }); + assert.deepEqual(encoded, Buffer.from("abcdef")); + }); + it("should encode string with left padding", function () { + var parser = Parser.start().string("a", { length: 6, padding: "left" }); + var encoded = parser.encode({ a: "abcd" }); + assert.deepEqual(encoded, Buffer.from(" abcd")); + encoded = parser.encode({ a: "abcdefgh" }); + assert.deepEqual(encoded, Buffer.from("abcdef")); + }); + it("should encode string with right padding and provided padding char", function () { + var parser = Parser.start().string("a", { length: 6, padd: "x" }); + var encoded = parser.encode({ a: "abcd" }); + assert.deepEqual(encoded, Buffer.from("abcdxx")); + encoded = parser.encode({ a: "abcdefgh" }); + assert.deepEqual(encoded, Buffer.from("abcdef")); + }); + it("should encode string with left padding and provided padding char", function () { + var parser = Parser.start().string("a", { + length: 6, + padding: "left", + padd: ".", + }); + var encoded = parser.encode({ a: "abcd" }); + assert.deepEqual(encoded, Buffer.from("..abcd")); + encoded = parser.encode({ a: "abcdefgh" }); + assert.deepEqual(encoded, Buffer.from("abcdef")); + }); + it("should encode string with padding and padding char 0", function () { + var parser = Parser.start().string("a", { length: 6, padd: "\u0000" }); + var encoded = parser.encode({ a: "abcd" }); + assert.deepEqual(encoded, Buffer.from("abcd\u0000\u0000")); + }); + it("should encode string with padding and first byte of padding char", function () { + var parser = Parser.start().string("a", { length: 6, padd: "1234" }); + var encoded = parser.encode({ a: "abcd" }); + assert.deepEqual(encoded, Buffer.from("abcd11")); + }); + it("should encode string with space padding when padd char is not encoded on 1 Byte", function () { + var parser = Parser.start().string("a", { length: 6, padd: "こ" }); + var encoded = parser.encode({ a: "abcd" }); + assert.deepEqual(encoded, Buffer.from("abcd ")); + }); + }); + + describe("Buffer encoder", function () { + it("should encode buffer", function () { + var parser = new Parser().uint8("len").buffer("raw", { + length: "len", + }); + + var buf = Buffer.from("deadbeefdeadbeef", "hex"); + var result = parser.parse( + Buffer.concat([Buffer.from([8]), buf, Buffer.from("garbage at end")]), + ); + + assert.deepEqual(result, { + len: 8, + raw: buf, + }); + + var encoded = parser.encode(result); + assert.deepEqual(encoded, Buffer.concat([Buffer.from([8]), buf])); + }); + }); +}); diff --git a/test/zz_composite_encoder.js b/test/zz_composite_encoder.js new file mode 100644 index 00000000..2c0e3d97 --- /dev/null +++ b/test/zz_composite_encoder.js @@ -0,0 +1,889 @@ +var assert = require("assert"); +var util = require("util"); +var Parser = require("../dist/binary_parser").Parser; + +describe("Composite encoder", function () { + describe("Array encoder", function () { + it("should encode array of primitive types", function () { + var parser = Parser.start().uint8("length").array("message", { + length: "length", + type: "uint8", + }); + + var buffer = Buffer.from([12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + length: 12, + message: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode array of primitive types with lengthInBytes", function () { + var parser = Parser.start().uint8("length").array("message", { + lengthInBytes: "length", + type: "uint8", + }); + + var buffer = Buffer.from([12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + length: 12, + message: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode array of primitive types with lengthInBytes as a maximum but not minimum", function () { + var parser = Parser.start().uint8("length").array("message", { + lengthInBytes: "length", + type: "uint8", + }); + var encoded = parser.encode({ + length: 5, + message: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], // Extra items in array than encoding limit + }); + assert.deepEqual(encoded, Buffer.from([5, 1, 2, 3, 4, 5])); + encoded = parser.encode({ + length: 5, + message: [1, 2, 3], // Less items in array than encoding limit + }); + assert.deepEqual(encoded, Buffer.from([5, 1, 2, 3])); + }); + it("should encode array of user defined types", function () { + var elementParser = new Parser().uint8("key").int16le("value"); + + var parser = Parser.start().uint16le("length").array("message", { + length: "length", + type: elementParser, + }); + + var buffer = Buffer.from([ + 0x02, 0x00, 0xca, 0xd2, 0x04, 0xbe, 0xd3, 0x04, + ]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + length: 0x02, + message: [ + { key: 0xca, value: 1234 }, + { key: 0xbe, value: 1235 }, + ], + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode array of user defined types with lengthInBytes", function () { + var elementParser = new Parser().uint8("key").int16le("value"); + + var parser = Parser.start().uint16le("length").array("message", { + lengthInBytes: "length", + type: elementParser, + }); + + var buffer = Buffer.from([ + 0x06, 0x00, 0xca, 0xd2, 0x04, 0xbe, 0xd3, 0x04, + ]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + length: 0x06, + message: [ + { key: 0xca, value: 1234 }, + { key: 0xbe, value: 1235 }, + ], + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode array of user defined types with length function", function () { + var elementParser = new Parser().uint8("key").int16le("value"); + + var parser = Parser.start() + .uint16le("length") + .array("message", { + length: function () { + return this.length; + }, + type: elementParser, + }); + + var buffer = Buffer.from([ + 0x02, 0x00, 0xca, 0xd2, 0x04, 0xbe, 0xd3, 0x04, + ]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + length: 0x02, + message: [ + { key: 0xca, value: 1234 }, + { key: 0xbe, value: 1235 }, + ], + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode array of arrays", function () { + var rowParser = Parser.start().uint8("length").array("cols", { + length: "length", + type: "int32le", + }); + + var parser = Parser.start().uint8("length").array("rows", { + length: "length", + type: rowParser, + }); + + var buffer = Buffer.alloc(1 + 10 * (1 + 5 * 4)); + var i, j; + + iterator = 0; + buffer.writeUInt8(10, iterator); + iterator += 1; + for (i = 0; i < 10; i++) { + buffer.writeUInt8(5, iterator); + iterator += 1; + for (j = 0; j < 5; j++) { + buffer.writeInt32LE(i * j, iterator); + iterator += 4; + } + } + + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + length: 10, + rows: [ + { length: 5, cols: [0, 0, 0, 0, 0] }, + { length: 5, cols: [0, 1, 2, 3, 4] }, + { length: 5, cols: [0, 2, 4, 6, 8] }, + { length: 5, cols: [0, 3, 6, 9, 12] }, + { length: 5, cols: [0, 4, 8, 12, 16] }, + { length: 5, cols: [0, 5, 10, 15, 20] }, + { length: 5, cols: [0, 6, 12, 18, 24] }, + { length: 5, cols: [0, 7, 14, 21, 28] }, + { length: 5, cols: [0, 8, 16, 24, 32] }, + { length: 5, cols: [0, 9, 18, 27, 36] }, + ], + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode until function returns true when readUntil is function", function () { + var parser = Parser.start().array("data", { + readUntil: function (item, buf) { + return item === 0; + }, + type: "uint8", + }); + + var buffer = Buffer.from([ + 0xff, 0xff, 0xff, 0x01, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + ]); + assert.deepEqual(parser.parse(buffer), { + data: [0xff, 0xff, 0xff, 0x01, 0x00], + }); + var encoded = parser.encode({ + ignore1: [0x00, 0x00], + data: [0xff, 0xff, 0xff, 0x01, 0x00, 0xff, 0xff, 0x00, 0xff], + ignore2: [0x01, 0x00, 0xff], + }); + assert.deepEqual(encoded, Buffer.from([0xff, 0xff, 0xff, 0x01, 0x00])); + }); + it("should not support associative arrays", function () { + var parser = Parser.start() + .int8("numlumps") + .array("lumps", { + type: Parser.start() + .int32le("filepos") + .int32le("size") + .string("name", { length: 8, encoding: "ascii" }), + length: "numlumps", + key: "name", + }); + + assert.throws(function () { + parser.encode({ + numlumps: 2, + lumps: { + AAAAAAAA: { + filepos: 1234, + size: 5678, + name: "AAAAAAAA", + }, + bbbbbbbb: { + filepos: 5678, + size: 1234, + name: "bbbbbbbb", + }, + }, + }); + }, /Encoding associative array not supported/); + }); + it("should use encoder to transform encoded array", function () { + var parser = Parser.start().array("data", { + type: "uint8", + length: 4, + formatter: function (arr) { + return arr.join("."); + }, + encoder: function (str) { + return str.split("."); + }, + }); + + var buffer = Buffer.from([0x0a, 0x0a, 0x01, 0x6e]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + data: "10.10.1.110", + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to go into recursion", function () { + var parser = Parser.start().namely("self").uint8("length").array("data", { + type: "self", + length: "length", + }); + + var buffer = Buffer.from([1, 1, 1, 0]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + length: 1, + data: [ + { + length: 1, + data: [ + { + length: 1, + data: [{ length: 0, data: [] }], + }, + ], + }, + ], + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to go into even deeper recursion", function () { + var parser = Parser.start().namely("self").uint8("length").array("data", { + type: "self", + length: "length", + }); + + // 2 + // / \ + // 3 1 + // / | \ \ + // 1 0 2 0 + // / / \ + // 0 1 0 + // / + // 0 + + var buffer = Buffer.from([ + 2, /* 0 */ 3, /* 0 */ 1, /* 0 */ 0, /* 1 */ 0, /* 2 */ 2, /* 0 */ 1, + /* 0 */ 0, /* 1 */ 0, /* 1 */ 1, /* 0 */ 0, + ]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + length: 2, + data: [ + { + length: 3, + data: [ + { length: 1, data: [{ length: 0, data: [] }] }, + { length: 0, data: [] }, + { + length: 2, + data: [ + { length: 1, data: [{ length: 0, data: [] }] }, + { length: 0, data: [] }, + ], + }, + ], + }, + { + length: 1, + data: [{ length: 0, data: [] }], + }, + ], + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + + it("should allow parent parser attributes as choice key", function () { + var ChildParser = Parser.start().choice("data", { + tag: function (vars) { + return vars.version; + }, + choices: { + 1: Parser.start().uint8("v1"), + 2: Parser.start().uint16("v2"), + }, + }); + + var ParentParser = Parser.start() + .uint8("version") + .nest("child", { type: ChildParser }); + + var buffer = Buffer.from([0x1, 0x2]); + var decoded = ParentParser.parse(buffer); + assert.deepEqual(decoded, { + version: 1, + child: { data: { v1: 2 } }, + }); + var encoded = ParentParser.encode(decoded); + assert.deepEqual(encoded, buffer); + + buffer = Buffer.from([0x2, 0x3, 0x4]); + decoded = ParentParser.parse(buffer); + assert.deepEqual(decoded, { + version: 2, + child: { data: { v2: 0x0304 } }, + }); + encoded = ParentParser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + }); + + describe("Choice encoder", function () { + it("should encode choices of primitive types", function () { + var parser = Parser.start() + .uint8("tag1") + .choice("data1", { + tag: "tag1", + choices: { + 0: "int32le", + 1: "int16le", + }, + }) + .uint8("tag2") + .choice("data2", { + tag: "tag2", + choices: { + 0: "int32le", + 1: "int16le", + }, + }); + + var buffer = Buffer.from([0x0, 0x4e, 0x61, 0xbc, 0x00, 0x01, 0xd2, 0x04]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + tag1: 0, + data1: 12345678, + tag2: 1, + data2: 1234, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should encode default choice", function () { + var parser = Parser.start() + .uint8("tag") + .choice("data", { + tag: "tag", + choices: { + 0: "int32le", + 1: "int16le", + }, + defaultChoice: "uint8", + }) + .int32le("test"); + + buffer = Buffer.from([0x03, 0xff, 0x2f, 0xcb, 0x04, 0x0]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + tag: 3, + data: 0xff, + test: 314159, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should parse choices of user defied types", function () { + var parser = Parser.start() + .uint8("tag") + .choice("data", { + tag: "tag", + choices: { + 1: Parser.start() + .uint8("length") + .string("message", { length: "length" }), + 3: Parser.start().int32le("number"), + }, + }); + + var buffer = Buffer.from([ + 0x1, 0xc, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x77, 0x6f, 0x72, + 0x6c, 0x64, + ]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + tag: 1, + data: { + length: 12, + message: "hello, world", + }, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + buffer = Buffer.from([0x03, 0x4e, 0x61, 0xbc, 0x00]); + decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + tag: 3, + data: { + number: 12345678, + }, + }); + encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to go into recursion", function () { + var stop = Parser.start(); + + var parser = Parser.start() + .namely("self") + .uint8("type") + .choice("data", { + tag: "type", + choices: { + 0: stop, + 1: "self", + }, + }); + + var buffer = Buffer.from([1, 1, 1, 0]); + var decoded = parser.parse(buffer); + assert.deepEqual(parser.parse(buffer), { + type: 1, + data: { + type: 1, + data: { + type: 1, + data: { type: 0, data: {} }, + }, + }, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to go into recursion with simple nesting", function () { + var stop = Parser.start(); + + var parser = Parser.start() + .namely("self") + .uint8("type") + .choice("data", { + tag: "type", + choices: { + 0: stop, + 1: "self", + 2: Parser.start() + .nest("left", { type: "self" }) + .nest("right", { type: stop }), + }, + }); + + var buffer = Buffer.from([2, /* left */ 1, 1, 0 /* right */]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + type: 2, + data: { + left: { + type: 1, + data: { + type: 1, + data: { + type: 0, + data: {}, + }, + }, + }, + right: {}, + }, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to refer to other parsers by name", function () { + var parser = Parser.start().namely("self"); + + var stop = Parser.start().namely("stop"); + + var twoCells = Parser.start() + .namely("twoCells") + .nest("left", { type: "self" }) + .nest("right", { type: "stop" }); + + parser.uint8("type").choice("data", { + tag: "type", + choices: { + 0: "stop", + 1: "self", + 2: "twoCells", + }, + }); + + var buffer = Buffer.from([2, /* left */ 1, 1, 0 /* right */]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + type: 2, + data: { + left: { + type: 1, + data: { type: 1, data: { type: 0, data: {} } }, + }, + right: {}, + }, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to refer to other parsers both directly and by name", function () { + var parser = Parser.start().namely("self"); + + var stop = Parser.start(); + + var twoCells = Parser.start() + .nest("left", { type: "self" }) + .nest("right", { type: stop }); + + parser.uint8("type").choice("data", { + tag: "type", + choices: { + 0: stop, + 1: "self", + 2: twoCells, + }, + }); + + var buffer = Buffer.from([2, /* left */ 1, 1, 0 /* right */]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + type: 2, + data: { + left: { + type: 1, + data: { type: 1, data: { type: 0, data: {} } }, + }, + right: {}, + }, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to go into recursion with complex nesting", function () { + var stop = Parser.start(); + + var parser = Parser.start() + .namely("self") + .uint8("type") + .choice("data", { + tag: "type", + choices: { + 0: stop, + 1: "self", + 2: Parser.start() + .nest("left", { type: "self" }) + .nest("right", { type: "self" }), + 3: Parser.start() + .nest("one", { type: "self" }) + .nest("two", { type: "self" }) + .nest("three", { type: "self" }), + }, + }); + + // 2 + // / \ + // 3 1 + // / | \ \ + // 1 0 2 0 + // / / \ + // 0 1 0 + // / + // 0 + + var buffer = Buffer.from([ + 2, /* left -> */ 3, /* one -> */ 1, /* -> */ 0, /* two -> */ 0, + /* three -> */ 2, /* left -> */ 1, /* -> */ 0, /* right -> */ 0, + /* right -> */ 1, /* -> */ 0, + ]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + type: 2, + data: { + left: { + type: 3, + data: { + one: { type: 1, data: { type: 0, data: {} } }, + two: { type: 0, data: {} }, + three: { + type: 2, + data: { + left: { type: 1, data: { type: 0, data: {} } }, + right: { type: 0, data: {} }, + }, + }, + }, + }, + right: { + type: 1, + data: { type: 0, data: {} }, + }, + }, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to 'flatten' choices when using null varName", function () { + var parser = Parser.start() + .uint8("tag") + .choice(null, { + tag: "tag", + choices: { + 1: Parser.start() + .uint8("length") + .string("message", { length: "length" }), + 3: Parser.start().int32le("number"), + }, + }); + + var buffer = Buffer.from([ + 0x1, 0xc, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x77, 0x6f, 0x72, + 0x6c, 0x64, + ]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + tag: 1, + length: 12, + message: "hello, world", + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + buffer = Buffer.from([0x03, 0x4e, 0x61, 0xbc, 0x00]); + decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + tag: 3, + number: 12345678, + }); + encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to 'flatten' choices when omitting varName paramater", function () { + var parser = Parser.start() + .uint8("tag") + .choice({ + tag: "tag", + choices: { + 1: Parser.start() + .uint8("length") + .string("message", { length: "length" }), + 3: Parser.start().int32le("number"), + }, + }); + + var buffer = Buffer.from([ + 0x1, 0xc, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x77, 0x6f, 0x72, + 0x6c, 0x64, + ]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + tag: 1, + length: 12, + message: "hello, world", + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + buffer = Buffer.from([0x03, 0x4e, 0x61, 0xbc, 0x00]); + decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + tag: 3, + number: 12345678, + }); + encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + it("should be able to use function as the choice selector", function () { + var parser = Parser.start() + .string("selector", { length: 4 }) + .choice(null, { + tag: function () { + return parseInt(this.selector, 2); // string base 2 to integer decimal + }, + choices: { + 2: Parser.start() + .uint8("length") + .string("message", { length: "length" }), + 7: Parser.start().int32le("number"), + }, + }); + + var buffer = Buffer.from([ + 48, 48, 49, 48, 0xc, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x77, + 0x6f, 0x72, 0x6c, 0x64, + ]); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + selector: "0010", // -> choice 2 + length: 12, + message: "hello, world", + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + buffer = Buffer.from([48, 49, 49, 49, 0x4e, 0x61, 0xbc, 0x00]); + decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + selector: "0111", // -> choice 7 + number: 12345678, + }); + encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + }); + + describe("Nest parser", function () { + it("should encode nested parsers", function () { + var nameParser = new Parser() + .string("firstName", { + zeroTerminated: true, + }) + .string("lastName", { + zeroTerminated: true, + }); + var infoParser = new Parser().uint8("age"); + var personParser = new Parser() + .nest("name", { + type: nameParser, + }) + .nest("info", { + type: infoParser, + }); + + var buffer = Buffer.concat([ + Buffer.from("John\0Doe\0"), + Buffer.from([0x20]), + ]); + var person = personParser.parse(buffer); + assert.deepEqual(person, { + name: { + firstName: "John", + lastName: "Doe", + }, + info: { + age: 0x20, + }, + }); + var encoded = personParser.encode(person); + assert.deepEqual(encoded, buffer); + }); + + it("should format parsed nested parser", function () { + var nameParser = new Parser() + .string("firstName", { + zeroTerminated: true, + }) + .string("lastName", { + zeroTerminated: true, + }); + var personParser = new Parser().nest("name", { + type: nameParser, + formatter: function (name) { + return name.firstName + " " + name.lastName; + }, + encoder: function (name) { + // Reverse of aboce formatter + var names = name.split(" "); + return { firstName: names[0], lastName: names[1] }; + }, + }); + + var buffer = Buffer.from("John\0Doe\0"); + var person = personParser.parse(buffer); + assert.deepEqual(person, { + name: "John Doe", + }); + var encoded = personParser.encode(person); + assert.deepEqual(encoded, buffer); + }); + + it("should 'flatten' output when using null varName", function () { + var parser = new Parser() + .string("s1", { zeroTerminated: true }) + .nest(null, { + type: new Parser().string("s2", { zeroTerminated: true }), + }); + + var buf = Buffer.from("foo\0bar\0"); + var decoded = parser.parse(buf); + assert.deepEqual(decoded, { s1: "foo", s2: "bar" }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buf); + }); + + it("should 'flatten' output when omitting varName", function () { + var parser = new Parser().string("s1", { zeroTerminated: true }).nest({ + type: new Parser().string("s2", { zeroTerminated: true }), + }); + + var buf = Buffer.from("foo\0bar\0"); + var decoded = parser.parse(buf); + assert.deepEqual(decoded, { s1: "foo", s2: "bar" }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buf); + }); + }); + + describe("Buffer encoder", function () { + //this is a test for testing a fix of a bug, that removed the last byte of the + //buffer parser + it("should return a buffer with same size", function () { + var bufferParser = new Parser().buffer("buf", { + readUntil: "eof", + formatter: function (buffer) { + return buffer; + }, + }); + + var buffer = Buffer.from("John\0Doe\0"); + var decoded = bufferParser.parse(buffer); + assert.deepEqual(decoded, { buf: buffer }); + var encoded = bufferParser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + }); + + describe("Constructors", function () { + it("should create a custom object type", function () { + function Person() { + this.name = ""; + } + Person.prototype.toString = function () { + return "[object Person]"; + }; + var parser = Parser.start().create(Person).string("name", { + zeroTerminated: true, + }); + + var buffer = Buffer.from("John Doe\0"); + var person = parser.parse(buffer); + assert.ok(person instanceof Person); + assert.equal(person.name, "John Doe"); + var encoded = parser.encode(person); + assert.deepEqual(encoded, buffer); + }); + }); + + describe("encode other fields after bit", function () { + it("Encode uint8", function () { + var buffer = Buffer.from([0, 1, 0, 4]); + for (var i = 17; i <= 24; i++) { + var parser = Parser.start()["bit" + i]("a").uint8("b"); + var decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + a: 1 << (i - 16), + b: 4, + }); + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + } + }); + }); +}); diff --git a/test/zz_encoder_bugs.js b/test/zz_encoder_bugs.js new file mode 100644 index 00000000..3dec7010 --- /dev/null +++ b/test/zz_encoder_bugs.js @@ -0,0 +1,421 @@ +var assert = require("assert"); +const { SmartBuffer } = require("smart-buffer/build/smartbuffer"); +var Parser = require("../dist/binary_parser").Parser; + +describe("Specific bugs testing", function () { + describe("Array encoder with readUntil", function () { + it("should limit to array length even if readUntil is never true", function () { + var parser = Parser.start() + .uint16("len") + .array("payloads", { + type: new Parser().uint8("cmd").array("params", { + type: new Parser().uint8("param"), + readUntil: function (item, buffer) { + return buffer.length == 2; // Stop when 2 bytes left in parsed buffer + }, + }), + lengthInBytes: function () { + return this.len - 4; + }, + }) + .uint16("crc"); + + var buffer = Buffer.from("0008AAB1B2B3FFFF", "hex"); + var decoded = parser.parse(buffer); + + assert.deepEqual(decoded, { + len: 8, + payloads: [ + { + cmd: 170, + params: [ + { + param: 177, + }, + { + param: 178, + }, + { + param: 179, + }, + ], + }, + ], + crc: 65535, + }); + + var encoded; + // Although readUntil is never true here, the encoding will be good + assert.doesNotThrow(function () { + encoded = parser.encode(decoded); + }); + assert.deepEqual(encoded, buffer); + }); + + it("is not the reverse of parsing when readUntil gives false information", function () { + var parser = Parser.start() + .uint16("len") + .array("payloads", { + type: new Parser().uint8("cmd").array("params", { + type: new Parser().uint8("param"), + readUntil: function (item, buffer) { + return buffer.length <= 2; // Stop when 2 bytes left in buffer + }, + }), + lengthInBytes: function () { + return this.len - 4; + }, + }) + .uint16("crc"); + + var buffer = Buffer.from("0008AAB1B2B3FFFF", "hex"); + var decoded = parser.parse(buffer); + + assert.deepEqual(decoded, { + len: 8, + payloads: [ + { + cmd: 170, + params: [ + { + param: 177, + }, + { + param: 178, + }, + { + param: 179, + }, + ], + }, + ], + crc: 0xffff, + }); + + var encoded = parser.encode(decoded); + // Missing parms 178 and 179 as readUntil will be true at first run + assert.deepEqual(encoded, Buffer.from("0008AAB1FFFF", "hex")); + }); + + it("should ignore readUntil when encodeUntil is provided", function () { + var parser = Parser.start() + .uint16("len") + .array("payloads", { + type: new Parser().uint8("cmd").array("params", { + type: new Parser().uint8("param"), + readUntil: function (item, buffer) { + return buffer.length == 2; // Stop when 2 bytes left in buffer + }, + encodeUntil: function (item, obj) { + return item.param === 178; // Stop encoding when value 178 is reached + }, + }), + lengthInBytes: function () { + return this.len - 4; + }, + }) + .uint16("crc"); + + var buffer = Buffer.from("0008AAB1B2B3FFFF", "hex"); + var decoded = parser.parse(buffer); + + assert.deepEqual(decoded, { + len: 8, + payloads: [ + { + cmd: 170, + params: [ + { + param: 177, + }, + { + param: 178, + }, + { + param: 179, + }, + ], + }, + ], + crc: 0xffff, + }); + + var encoded = parser.encode(decoded); + // Missing parms 179 as encodeUntil stops at 178 + assert.deepEqual(encoded, Buffer.from("0008AAB1B2FFFF", "hex")); + }); + + it("should accept readUntil=eof and no encodeUntil provided", function () { + var parser = Parser.start().array("arr", { + type: "uint8", + readUntil: "eof", // Read until end of buffer + }); + + var buffer = Buffer.from("01020304050607", "hex"); + var decoded = parser.parse(buffer); + + assert.deepEqual(decoded, { + arr: [1, 2, 3, 4, 5, 6, 7], + }); + + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, Buffer.from("01020304050607", "hex")); + }); + + it("should accept empty array to encode", function () { + var parser = Parser.start().array("arr", { + type: "uint8", + readUntil: "eof", // Read until end of buffer + }); + + var buffer = Buffer.from("", "hex"); + var decoded = parser.parse(buffer); + + assert.deepEqual(decoded, { + arr: [], + }); + + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, Buffer.from("", "hex")); + }); + + it("should accept empty array to encode and encodeUntil function", function () { + var parser = Parser.start().array("arr", { + type: "uint8", + readUntil: "eof", // Read until end of buffer + encodeUntil: function (item, obj) { + return false; // Never stop on content value + }, + }); + + var buffer = Buffer.from("", "hex"); + var decoded = parser.parse(buffer); + + assert.deepEqual(decoded, { + arr: [], + }); + + var encoded = parser.encode(decoded); + assert.deepEqual(encoded, Buffer.from("", "hex")); + }); + + it("should accept undefined or null array", function () { + var parser = Parser.start().array("arr", { + type: "uint8", + readUntil: "eof", // Read until end of buffer + }); + + var buffer = Buffer.from("", "hex"); + var decoded = parser.parse(buffer); + + // Decode an empty buffer as an empty array + assert.deepEqual(decoded, { + arr: [], + }); + + // Encode undefined, null or empty array as an empty buffer + [{}, { arr: undefined }, { arr: null }, { arr: [] }].forEach((data) => { + let encoded = parser.encode(data); + assert.deepEqual(encoded, Buffer.from("", "hex")); + }); + }); + }); + + describe("Issue #19 Little endianess incorrect", function () { + let binaryLiteral = function (s) { + var i; + var bytes = []; + + s = s.replace(/\s/g, ""); + for (i = 0; i < s.length; i += 8) { + bytes.push(parseInt(s.slice(i, i + 8), 2)); + } + + return Buffer.from(bytes); + }; + it("should parse 4-byte-length bit field sequence wit little endian", function () { + let buf = binaryLiteral("0000000000001111 1010000110100010"); // 000F A1A2 + + // Parsed as two uint16 with little-endian (BYTES order) + let parser1 = new Parser().uint16le("a").uint16le("b"); + + // Parsed as two 16 bits fields with little-endian + let parser2 = new Parser().endianess("little").bit16("a").bit16("b"); + + let parsed1 = parser1.parse(buf); + let parsed2 = parser2.parse(buf); + + assert.deepEqual(parsed1, { + a: 0x0f00, // 000F + b: 0xa2a1, // A1A2 + }); + + assert.deepEqual(parsed2, { + a: 0xa1a2, // last 16 bits (but value coded as BE) + b: 0x000f, // first 16 bits (but value coded as BE) + }); + + /* This is a little confusing. The endianess with bits fields affect the order of fields */ + }); + it("should encode bit ranges with little endian correctly", function () { + let bigParser = Parser.start() + .endianess("big") + .bit4("a") + .bit1("b") + .bit1("c") + .bit1("d") + .bit1("e") + .uint16("f") + .array("g", { type: "uint8", readUntil: "eof" }); + let littleParser = Parser.start() + .endianess("little") + .bit4("a") + .bit1("b") + .bit1("c") + .bit1("d") + .bit1("e") + .uint16("f") + .array("g", { type: "uint8", readUntil: "eof" }); + // Parser definition for a symetric encoding/decoding of little-endian bit fields + let little2Parser = Parser.start() + .endianess("little") + .encoderSetOptions({ bitEndianess: true }) + .bit4("a") + .bit1("b") + .bit1("c") + .bit1("d") + .bit1("e") + .uint16("f") + .array("g", { type: "uint8", readUntil: "eof" }); + + let data = binaryLiteral( + "0011 0 1 0 1 0000000011111111 00000001 00000010 00000011", + ); // 35 00FF 01 02 03 + // in big endian: 3 0 1 0 1 00FF 1 2 3 + // in little endian: 3 0 1 0 1 FF00 1 2 3 + // LE with encoderBitEndianess option: + // 5 1 1 0 0 FF00 1 2 3 + + //let bigDecoded = bigParser.parse(data); + //let littleDecoded = littleParser.parse(data); + let little2Decoded = little2Parser.parse(data); + + //console.log(bigDecoded); + //console.log(littleDecoded); + //console.log(little2Decoded); + + let big = { + a: 3, + b: 0, + c: 1, + d: 0, + e: 1, + f: 0x00ff, + g: [1, 2, 3], + }; + let little = { + a: 3, + b: 0, + c: 1, + d: 0, + e: 1, + f: 0xff00, + g: [1, 2, 3], + }; + let little2 = { + a: 5, + b: 1, + c: 1, + d: 0, + e: 0, + f: 0xff00, + g: [1, 2, 3], + }; + + assert.deepEqual(little2Decoded, little2); + + let bigEncoded = bigParser.encode(big); + let littleEncoded = littleParser.encode(little); + let little2Encoded = little2Parser.encode(little2); + + //console.log(bigEncoded); + //console.log(littleEncoded); + //console.log(little2Encoded); + + assert.deepEqual(bigEncoded, data); + assert.deepEqual(littleEncoded, data); + assert.deepEqual(little2Encoded, data); + }); + }); + + describe("Issue #20 Encoding fixed length null terminated or strip null strings", function () { + it("should encode zero terminated fixed-length string", function () { + // In that case parsing and encoding are not the exact oposite + let buffer = Buffer.from( + "\u0000A\u0000AB\u0000ABC\u0000ABCD\u0000ABCDE\u0000", + ); + let parser = Parser.start() + .string("a", { length: 4, zeroTerminated: true }) + .string("b", { length: 4, zeroTerminated: true }) + .string("c", { length: 4, zeroTerminated: true }) + .string("d", { length: 4, zeroTerminated: true }) + .string("e", { length: 4, zeroTerminated: true }) + .string("f", { length: 4, zeroTerminated: true }) + .string("g", { length: 4, zeroTerminated: true }) + .string("h", { length: 4, zeroTerminated: true }); + + let decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + a: "", + b: "A", + c: "AB", + d: "ABC", + e: "ABCD", + f: "", + g: "ABCD", + h: "E", + }); + + let encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + + it("should encode fixed-length string with stripNull", function () { + let parser = Parser.start() + .string("a", { length: 8, zeroTerminated: false, stripNull: true }) + .string("b", { length: 8, zeroTerminated: false, stripNull: true }) + .string("z", { length: 2, zeroTerminated: false, stripNull: true }); + let buffer = Buffer.from("ABCD\u0000\u0000\u0000\u000012345678ZZ"); + let decoded = parser.parse(buffer); + assert.deepEqual(decoded, { + a: "ABCD", + b: "12345678", + z: "ZZ", + }); + let encoded = parser.encode(decoded); + assert.deepEqual(encoded, buffer); + }); + }); + + describe("Issue #23 Unable to encode uint64", function () { + it("should not fail when encoding uint64", function () { + let ipHeader = new Parser() + .uint16("fragment_id") + .uint16("fragment_total") + .uint64("datetime"); + + let anIpHeader = { + fragment_id: 1, + fragment_total: 1, + datetime: 4744430483355899789n, + }; + + try { + let result = ipHeader.encode(anIpHeader).toString("hex"); + assert.ok(true, "No exception"); + } catch (ex) { + assert.fail(ex); + } + }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 938d0b40..f35ea426 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,9 +10,8 @@ "noImplicitReturns": true, "noFallthroughCasesInSwitch": true, "types": ["node", "mocha"], - "esModuleInterop": true + "esModuleInterop": true, + "moduleResolution": "NodeNext" }, - "files": [ - "lib/binary_parser.ts" - ] + "files": ["lib/binary_parser.ts"] }