From 9fdaf0cc37e10afdb9b61228a67cd5dafd9a1312 Mon Sep 17 00:00:00 2001 From: djm <35852337+dmaccormack@users.noreply.github.com> Date: Wed, 22 Nov 2023 15:42:55 -0500 Subject: [PATCH] fix: Fix bug stringifying number-like keys and values A string like "1e 6" will currently be stringified as 1e+6, which is ambiguous because it looks like a number literal and it will be parsed as one. This patch fixes the stringify code so that these strings are quoted. AQF strings have a similar issue and this patch addresses them as well. --- src/JsonURL.js | 76 ++++++++++++++++++++++---------- test/parse.test.js | 92 ++++++++++++++++++++++++++++++++++++++- test/parseLiteral.test.js | 35 +++++++++------ test/stringify.test.js | 77 +++++++++++++++++++++++++++++++- 4 files changed, 240 insertions(+), 40 deletions(-) diff --git a/src/JsonURL.js b/src/JsonURL.js index 9a5a578..7c7b181 100644 --- a/src/JsonURL.js +++ b/src/JsonURL.js @@ -33,7 +33,7 @@ import { setupToJsonURLText, toJsonURLText } from "./proto.js"; const RX_DECODE_SPACE = /\+/g; const RX_ENCODE_SPACE = / /g; -const RX_AQF_DECODE = /(![\s\S]?)/g; +const RX_AQF_DECODE_ESCAPE = /(![\s\S]?)/g; // // patterns for use with RegEx.test(). @@ -42,7 +42,9 @@ const RX_AQF_DECODE = /(![\s\S]?)/g; const RX_ENCODE_STRING_SAFE = /^[-A-Za-z0-9._~!$*;=@?/ ][-A-Za-z0-9._~!$*;=@?/' ]*$/; const RX_ENCODE_STRING_QSAFE = /^[-A-Za-z0-9._~!$*,;=@?/(): ]+$/; -const RX_ENCODE_NUMBER = /^-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$/; +const RX_ENCODE_NUMBER = /^-?\d+(?:\.\d+)?(?:[eE][-]?\d+)?$/; +const RX_ENCODE_NUMBER_PLUS = /^-?\d+(?:\.\d+)?[eE]\+\d+$/; +const RX_ENCODE_NUMBER_SPACE = /^-?\d+(?:\.\d+)?[eE] \d+$/; const RX_ENCODE_BASE = /[(),:]|%2[04]|%3B/gi; const RX_ENCODE_BASE_MAP = { @@ -55,7 +57,7 @@ const RX_ENCODE_BASE_MAP = { "%3B": ";", }; -const RX_ENCODE_AQF = /[!(),:]|%2[01489C]|%3[AB]/gi; +const RX_ENCODE_AQF = /[!(),:]|%2[01489BC]|%3[AB]/gi; const RX_ENCODE_AQF_MAP = { "%20": "+", "%21": "!!", @@ -66,6 +68,7 @@ const RX_ENCODE_AQF_MAP = { "%29": "!)", ")": "!)", "+": "!+", + "%2B": "!+", "%2C": "!,", ",": "!,", "%3A": "!:", @@ -131,6 +134,7 @@ UNESCAPE[CHAR_n] = "n"; const EMPTY_STRING = ""; const EMPTY_STRING_AQF = "!e"; const SPACE = " "; +const PLUS = "+"; function newEmptyString(pos, emptyOK) { if (emptyOK) { @@ -205,7 +209,17 @@ function hexDecode(pos, c) { } } -function isBoolNullNumber(s) { +function isBang(s, offset) { + return ( + s.charCodeAt(offset - 1) === CHAR_BANG || + (offset > 2 && + s.charCodeAt(offset - 3) === CHAR_PERCENT && + s.charCodeAt(offset - 2) === CHAR_0 + 2 && + s.charCodeAt(offset - 1) === CHAR_0 + 1) + ); +} + +function isBoolNullNoPlusNumber(s) { if (s === "true" || s === "false" || s === "null") { return true; } @@ -269,30 +283,41 @@ function toJsonURLText_String(options, depth, isKey) { return encodeStringLiteral(this, options.AQF); } - if (isBoolNullNumber(this)) { + if (isBoolNullNoPlusNumber(this)) { // - // if this string looks like a Boolean, Number, or ``null'' literal - // then it must be quoted + // this string looks like a boolean, `null`, or number literal without + // a plus char // if (isKey === true) { + // keys are assumed to be strings return this; } if (options.AQF) { - if (this.indexOf("+") == -1) { - return "!" + this; - } - return this.replace("+", "!+"); - } - if (this.indexOf("+") == -1) { - return "'" + this + "'"; + return "!" + this; } + return "'" + this + "'"; + } + if (RX_ENCODE_NUMBER_PLUS.test(this)) { + // + // this string looks like a number with an exponent that includes a `+` + // + if (options.AQF) { + return this.replace(PLUS, "!+"); + } + return this.replace(PLUS, "%2B"); + } + if (RX_ENCODE_NUMBER_SPACE.test(this)) { // - // if the string needs to be encoded then it no longer looks like a - // literal and does not needs to be quoted. + // this string would look like a number if it were allowed to have a + // space represented as a plus // - return encodeURIComponent(this); + if (options.AQF) { + return "!" + this.replace(SPACE, "+"); + } + return "'" + this.replace(SPACE, "+") + "'"; } + if (options.AQF) { return encodeStringLiteral(this, true); } @@ -957,10 +982,6 @@ class ParserAQF extends Parser { return ret; } - acceptPlus() { - return this.accept(CHAR_PLUS); - } - findLiteralEnd() { const end = this.end; const pos = this.pos; @@ -1025,7 +1046,16 @@ class ParserAQF extends Parser { const text = this.text; const pos = this.pos; const ret = decodeURIComponent( - text.substring(pos, litend).replace(RX_DECODE_SPACE, SPACE) + text + .substring(pos, litend) + .replace(RX_DECODE_SPACE, function (match, offset) { + if (offset === 0 || !isBang(text, pos + offset)) { + return SPACE; + } + return PLUS; + // const c = text.charCodeAt(pos + offset - 1); + // return c === CHAR_BANG ? PLUS : SPACE; + }) ); this.pos = litend; @@ -1034,7 +1064,7 @@ class ParserAQF extends Parser { return EMPTY_STRING; } - return ret.replace(RX_AQF_DECODE, function name(match, _p, offset) { + return ret.replace(RX_AQF_DECODE_ESCAPE, function (match, _p, offset) { if (match.length === 2) { const c = match.charCodeAt(1); const uc = UNESCAPE[c]; diff --git a/test/parse.test.js b/test/parse.test.js index 72340dc..00aaca7 100644 --- a/test/parse.test.js +++ b/test/parse.test.js @@ -96,7 +96,6 @@ test.each([ ["('Hello,+(World)!')", ["Hello, (World)!"]], ["('','')", ["", ""]], ["('qkey':g)", { qkey: "g" }], - ["1e%2B1", "1e+1"], ])("JsonURL.parse(%s)", (text, expected) => { expect(u.parse(text)).toEqual(expected); expect(JsonURL.parse(text)).toEqual(expected); @@ -116,7 +115,6 @@ test.each([ ["('Hello!,+!(World!)!!')", ["'Hello, (World)!'"]], ["(!e,!e)", ["", ""]], ["(!e:g)", { "": "g" }], - ["1e%2B1", 10], ["%48%45%4C%4C%4F%21,+%57%4F%52%4C%44!!", "HELLO, WORLD!"], ["%28%61%3A%62%2C%63%3A%64%29", { a: "b", c: "d" }], ["%28%61%2C%62%2C%63%2c%64%29", ["a", "b", "c", "d"]], @@ -150,6 +148,96 @@ test.each([ expect(parseAQF(text, options)).toEqual(expected); }); +// +// Test edge cases for number-like strings combined with various options +// +test.each([ + [ + "1e+1", + "1e%2B1", + "1e!+1", + "1e%2B1", + "1e!+1", + "1e%2B1", + "1e%2B1", + "1e!+1", + "1e!+1", + ], + [ + "1e+3", + "1e+3", + "1e+3", + "1e%2B3", + "1e!+3", + "1e%2B3", + "1e%2B3", + "1e!+3", + "1e!+3", + ], + [ + "1e 3", + "'1e+3'", + "!1e+3", + "1e+3", + "1e+3", + "'1e+3'", + "1e+3", + "!1e+3", + "1e+3", + ], +])( + "JsonURL.parseNumberLikeString(%p)", + ( + expected, + inputKey, + inputKeyAqf, + inputKeyIsl, + inputKeyIslAqf, + inputBase, + inputImpliedStringLiteral, + inputAqf, + inputImpliedStringAqf + ) => { + function makeObject(s) { + const ret = {}; + ret[s] = "a"; + return ret; + } + function makeText(s) { + return "(" + s + ":a)"; + } + + expect(JsonURL.parse(makeText(inputKey))).toEqual(makeObject(expected)); + expect(JsonURL.parse(makeText(inputKeyAqf), { AQF: true })).toEqual( + makeObject(expected) + ); + expect( + JsonURL.parse(makeText(inputKeyIsl), { impliedStringLiterals: true }) + ).toEqual(makeObject(expected)); + + expect( + JsonURL.parse(makeText(inputKeyIslAqf), { + AQF: true, + impliedStringLiterals: true, + }) + ).toEqual(makeObject(expected)); + + expect(u.parse(inputBase)).toBe(expected); + expect( + u.parse(inputImpliedStringLiteral, { impliedStringLiterals: true }) + ).toBe(expected); + expect(u.parse(inputAqf, { AQF: true })).toBe(expected); + expect( + u.parse(inputImpliedStringAqf, { AQF: true, impliedStringLiterals: true }) + ).toBe(expected); + } +); + +/* +(text, value, aqfValue, impliedStrValue) => { + +*/ + test.each([undefined])("JsonURL.parse(%p)", (text) => { expect(u.parse(text)).toBeUndefined(); expect(JsonURL.parse(text)).toBeUndefined(); diff --git a/test/parseLiteral.test.js b/test/parseLiteral.test.js index b84c61f..c7db340 100644 --- a/test/parseLiteral.test.js +++ b/test/parseLiteral.test.js @@ -45,13 +45,12 @@ function escapeStringAQF(s) { .replace(/:/, "!:"); } -function runTest(text, value, keyValue, strLitValue) { +function runTest(text, value, keyValue, impliedStrValue) { expect(u.parseLiteral(text)).toBe(value); - expect(JsonURL.parse(text)).toBe(value); expect(u.parseLiteral(text, 0, text.length, true)).toBe(keyValue); expect( u.parseLiteral(text, 0, text.length, true, { impliedStringLiterals: true }) - ).toBe(strLitValue); + ).toBe(impliedStrValue); // // verify that parseLiteral() and parse() return the same thing (as @@ -61,10 +60,10 @@ function runTest(text, value, keyValue, strLitValue) { expect(JsonURL.parse(text)).toBe(value); } -function runTestAQF(text, value, strLitValue) { +function runTestAQF(text, value, impliedStrValue) { expect(u.parse(text, { AQF: true })).toBe(value); expect(u.parse(text, { AQF: true, impliedStringLiterals: true })).toBe( - strLitValue + impliedStrValue ); } @@ -147,7 +146,6 @@ test.each([ runTestAQF(textAQF, value, keyValue); }); -// eslint-disable-next-line jest/expect-expect test.each([ // // fixed point @@ -155,6 +153,8 @@ test.each([ ["-3e0", -3, undefined, "-3e0"], ["1e+2", 1e2, undefined, "1e 2"], ["-2e+1", -2e1, undefined, "-2e 1"], + ["1e-2", 1e-2, undefined, "1e-2"], + ["1e+2", 1e2, undefined, "1e 2"], // // floating point @@ -164,25 +164,32 @@ test.each([ // // string // - ["'hello'", "hello", "'hello'", undefined], + ["'hello'", "hello", "'hello'", "'hello'"], ["hello%2Bworld", "hello+world", undefined, undefined], ["y+%3D+mx+%2B+b", "y = mx + b", undefined, undefined], ["a%3Db%26c%3Dd", "a=b&c=d", undefined, undefined], ["hello%F0%9F%8D%95world", "hello\uD83C\uDF55world", undefined, undefined], ["-e+", "-e ", undefined, undefined], ["-e+1", "-e 1", undefined, undefined], - ["1e%2B1", "1e+1", 10, "1e+1"], + ["1e%2B1", "1e+1", "1e+1", "1e+1"], ["%26true", "&true", undefined, undefined], -])("JsonURL.parseLiteral(%p)", (text, value, aqfValue, strLitValue) => { +])("JsonURL.parseLiteral(%p)", (text, value, aqfValue, impliedStrValue) => { let keyValue = typeof value === "string" ? value : text; if (aqfValue === undefined) { aqfValue = value; } - if (strLitValue === undefined) { - strLitValue = aqfValue; - } - runTest(text, value, keyValue, strLitValue); - runTestAQF(text, aqfValue, strLitValue); + + runTest( + text, + value, + keyValue, + impliedStrValue === undefined ? keyValue : impliedStrValue + ); + + expect(u.parse(text, { AQF: true })).toBe(aqfValue); + expect(u.parse(text, { AQF: true, impliedStringLiterals: true })).toBe( + impliedStrValue === undefined ? aqfValue : impliedStrValue + ); }); test("JsonURL.parseLiteral('null')", () => { diff --git a/test/stringify.test.js b/test/stringify.test.js index 9ae98ab..3582f44 100644 --- a/test/stringify.test.js +++ b/test/stringify.test.js @@ -82,7 +82,7 @@ test.each([ ["2e1", "'2e1'", "2e1", "!2e1"], ["-4", "'-4'", "-4", "!-4"], ["5a", undefined, undefined, undefined], - ["'1+2'", "%271%2B2'", "'1%2B2'", undefined], + ["'1+2'", "%271%2B2'", "'1%2B2'", "'1!+2'"], ["1e+1", "1e%2B1", undefined, "1e!+1"], ["a b c", "a+b+c", undefined, undefined], ["a,b", "'a,b'", "a%2Cb", "a!,b"], @@ -200,3 +200,78 @@ test("JsonURL.stringify(instance specific toJsonURLText function)", () => { expect(JsonURL.stringify(filter)).toBe("(filter:color,value:red)"); }); + +test.each([ + // + // string + // + [ + "1e+3", + "1e%2B3", + "1e!+3", + "1e%2B3", + "1e!+3", + "1e%2B3", + "1e%2B3", + "1e!+3", + "1e!+3", + ], + [ + "1e 3", + "'1e+3'", + "!1e+3", + "1e+3", + "1e+3", + "'1e+3'", + "1e+3", + "!1e+3", + "1e+3", + ], +])( + "JsonURL.stringifyLiteralAndKey(%p)", + ( + text, + expectedKey, + expectedKeyAqf, + expectedKeyIsl, + expectedKeyIslAqf, + expected, + expectedImpliedStringLiteral, + expectedAqf, + expectedImpliedStringAqf + ) => { + function makeObject(s) { + const ret = {}; + ret[s] = 1; + return ret; + } + function makeText(s) { + return "(" + s + ":1)"; + } + + expect(JsonURL.stringify(makeObject(text))).toEqual(makeText(expectedKey)); + expect(JsonURL.stringify(makeObject(text), { AQF: true })).toEqual( + makeText(expectedKeyAqf) + ); + expect( + JsonURL.stringify(makeObject(text), { + impliedStringLiterals: true, + }) + ).toEqual(makeText(expectedKeyIsl)); + expect( + JsonURL.stringify(makeObject(text), { + AQF: true, + impliedStringLiterals: true, + }) + ).toEqual(makeText(expectedKeyIslAqf)); + + expect(JsonURL.stringify(text)).toBe(expected); + expect(JsonURL.stringify(text, { impliedStringLiterals: true })).toBe( + expectedImpliedStringLiteral + ); + expect(JsonURL.stringify(text, { AQF: true })).toBe(expectedAqf); + expect( + JsonURL.stringify(text, { AQF: true, impliedStringLiterals: true }) + ).toBe(expectedImpliedStringAqf); + } +);