From 5b15e754798d6d2552faf2f68413734cf1ebd505 Mon Sep 17 00:00:00 2001 From: Jon King Date: Sat, 27 May 2023 18:17:31 -0700 Subject: [PATCH 1/6] feat: add jsonata query operation --- package-lock.json | 9 +++++ package.json | 1 + src/core/operations/Jsonata.mjs | 65 +++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 src/core/operations/Jsonata.mjs diff --git a/package-lock.json b/package-lock.json index 50639ea8..9cdad930 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,6 +53,7 @@ "js-sha3": "^0.9.3", "jsesc": "^3.0.2", "json5": "^2.2.3", + "jsonata": "^2.0.3", "jsonpath-plus": "^9.0.0", "jsonwebtoken": "8.5.1", "jsqr": "^1.4.0", @@ -12447,6 +12448,14 @@ "node": ">=6" } }, + "node_modules/jsonata": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/jsonata/-/jsonata-2.0.6.tgz", + "integrity": "sha512-WhQB5tXQ32qjkx2GYHFw2XbL90u+LLzjofAYwi+86g6SyZeXHz9F1Q0amy3dWRYczshOC3Haok9J4pOCgHtwyQ==", + "engines": { + "node": ">= 8" + } + }, "node_modules/jsonpath-plus": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-9.0.0.tgz", diff --git a/package.json b/package.json index 20180e05..fd89c425 100644 --- a/package.json +++ b/package.json @@ -139,6 +139,7 @@ "js-sha3": "^0.9.3", "jsesc": "^3.0.2", "json5": "^2.2.3", + "jsonata": "^2.0.3", "jsonpath-plus": "^9.0.0", "jsonwebtoken": "8.5.1", "jsqr": "^1.4.0", diff --git a/src/core/operations/Jsonata.mjs b/src/core/operations/Jsonata.mjs new file mode 100644 index 00000000..41cf2751 --- /dev/null +++ b/src/core/operations/Jsonata.mjs @@ -0,0 +1,65 @@ +/** + * @author Jon K (jon@ajarsoftware.com) + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + */ + +import jsonata from "jsonata"; +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; + +/** + * Jsonata Query operation + */ +class JsonataQuery extends Operation { + /** + * JsonataQuery constructor + */ + constructor() { + super(); + + this.name = "Jsonata Query"; + this.module = "Code"; + this.description = + "Query and transform JSON data with a jsonata query."; + this.infoURL = "https://docs.jsonata.org/overview.html"; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + name: "Query", + type: "text", + value: "string", + }, + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {JSON} + */ + async run(input, args) { + const [query] = args; + let result, jsonObj; + + try { + jsonObj = JSON.parse(input); + } catch (err) { + throw new OperationError(`Invalid input JSON: ${err.message}`); + } + + try { + const expression = jsonata(query); + result = await expression.evaluate(jsonObj); + } catch (err) { + throw new OperationError( + `Invalid Jsonata Expression: ${err.message}` + ); + } + + return JSON.stringify(result); + } +} + +export default JsonataQuery; From 4a428e802e47a13fa9ce52f5241922c28c63b1e6 Mon Sep 17 00:00:00 2001 From: Jon King Date: Sat, 27 May 2023 18:39:33 -0700 Subject: [PATCH 2/6] feat: add to category --- src/core/config/Categories.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index de3ea882..022d41b1 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -365,6 +365,7 @@ "Regular expression", "XPath expression", "JPath expression", + "Jsonata Query", "CSS selector", "Extract EXIF", "Extract ID3", From 2b2435210b3cae39518dd82356d75ff1965e01ab Mon Sep 17 00:00:00 2001 From: Jon King Date: Thu, 13 Jul 2023 11:33:07 -0700 Subject: [PATCH 3/6] chore: update return type from JSON to string --- src/core/operations/Jsonata.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Jsonata.mjs b/src/core/operations/Jsonata.mjs index 41cf2751..1651e093 100644 --- a/src/core/operations/Jsonata.mjs +++ b/src/core/operations/Jsonata.mjs @@ -37,7 +37,7 @@ class JsonataQuery extends Operation { /** * @param {string} input * @param {Object[]} args - * @returns {JSON} + * @returns {string} */ async run(input, args) { const [query] = args; From c4574ff042f27246300d0eac98cf3330134bb6ff Mon Sep 17 00:00:00 2001 From: Jon King Date: Mon, 17 Feb 2025 10:15:13 -0700 Subject: [PATCH 4/6] fix: properly handle undefined results --- src/core/operations/Jsonata.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Jsonata.mjs b/src/core/operations/Jsonata.mjs index 1651e093..82cc4d39 100644 --- a/src/core/operations/Jsonata.mjs +++ b/src/core/operations/Jsonata.mjs @@ -58,7 +58,7 @@ class JsonataQuery extends Operation { ); } - return JSON.stringify(result); + return JSON.stringify(result === undefined ? "" : result); } } From 4c6b4cd4e76dd19301c77b8670690b8e3419a996 Mon Sep 17 00:00:00 2001 From: Jon King Date: Mon, 17 Feb 2025 10:15:35 -0700 Subject: [PATCH 5/6] added Jsonata Query tests --- tests/operations/index.mjs | 10 +- tests/operations/tests/Jsonata.mjs | 551 +++++++++++++++++++++++++++++ 2 files changed, 555 insertions(+), 6 deletions(-) create mode 100644 tests/operations/tests/Jsonata.mjs diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index a82bc874..ba131f84 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -11,10 +11,7 @@ * @license Apache-2.0 */ -import { - setLongTestFailure, - logTestReport, -} from "../lib/utils.mjs"; +import { setLongTestFailure, logTestReport } from "../lib/utils.mjs"; import TestRegister from "../lib/TestRegister.mjs"; import "./tests/AESKeyWrap.mjs"; @@ -88,6 +85,7 @@ import "./tests/IndexOfCoincidence.mjs"; import "./tests/JA3Fingerprint.mjs"; import "./tests/JA4.mjs"; import "./tests/JA3SFingerprint.mjs"; +import "./tests/Jsonata.mjs"; import "./tests/JSONBeautify.mjs"; import "./tests/JSONMinify.mjs"; import "./tests/JSONtoCSV.mjs"; @@ -168,14 +166,14 @@ const testStatus = { allTestsPassing: true, counts: { total: 0, - } + }, }; setLongTestFailure(); const logOpsTestReport = logTestReport.bind(null, testStatus); -(async function() { +(async function () { const results = await TestRegister.runTests(); logOpsTestReport(results); })(); diff --git a/tests/operations/tests/Jsonata.mjs b/tests/operations/tests/Jsonata.mjs new file mode 100644 index 00000000..d664137f --- /dev/null +++ b/tests/operations/tests/Jsonata.mjs @@ -0,0 +1,551 @@ +/** + * JSON to CSV tests. + * + * @author Jon King [jon@ajarsoftware.com] + * + * @copyright Crown Copyright 2025 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +const INPUT_JSON_OBJECT_WITH_ARRAYS = `{ + "FirstName": "Fred", + "Surname": "Smith", + "Age": 28, + "Address": { + "Street": "Hursley Park", + "City": "Winchester", + "Postcode": "SO21 2JN" + }, + "Phone": [ + { + "type": "home", + "number": "0203 544 1234" + }, + { + "type": "office", + "number": "01962 001234" + }, + { + "type": "office", + "number": "01962 001235" + }, + { + "type": "mobile", + "number": "077 7700 1234" + } + ], + "Email": [ + { + "type": "work", + "address": ["fred.smith@my-work.com", "fsmith@my-work.com"] + }, + { + "type": "home", + "address": ["freddy@my-social.com", "frederic.smith@very-serious.com"] + } + ], + "Other": { + "Over 18 ?": true, + "Misc": null, + "Alternative.Address": { + "Street": "Brick Lane", + "City": "London", + "Postcode": "E1 6RF" + } + } +}`; + +const INPUT_ARRAY_OF_OBJECTS = `[ + { "ref": [ 1,2 ] }, + { "ref": [ 3,4 ] } +]`; + +const INPUT_NUMBER_ARRAY = `{ + "Numbers": [1, 2.4, 3.5, 10, 20.9, 30] +}`; + +TestRegister.addTests([ + { + name: "Jsonata: Returns a JSON string (double quoted)", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '"Smith"', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Surname"], + }, + ], + }, + { + name: "Jsonata: Returns a JSON number", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: "28", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Age"], + }, + ], + }, + { + name: "Jsonata: Field references are separated by '.'", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '"Winchester"', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Address.City"], + }, + ], + }, + { + name: "Jsonata: Matched the path and returns the null value", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: "null", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Other.Misc"], + }, + ], + }, + { + name: "Jsonata: Path not found. Returns nothing", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '""', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Other.DoesntExist"], + }, + ], + }, + { + name: "Jsonata: Field references containing whitespace or reserved tokens can be enclosed in backticks", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: "true", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Other.`Over 18 ?`"], + }, + ], + }, + { + name: "Jsonata: Returns the first item (an object)", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '{"type":"home","number":"0203 544 1234"}', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[0]"], + }, + ], + }, + { + name: "Jsonata: Returns the second item", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '{"type":"office","number":"01962 001234"}', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[1]"], + }, + ], + }, + { + name: "Jsonata: Returns the last item", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '{"type":"mobile","number":"077 7700 1234"}', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[-1]"], + }, + ], + }, + { + name: "Jsonata: Negative indexed count from the end", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '{"type":"office","number":"01962 001235"}', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[-2]"], + }, + ], + }, + { + name: "Jsonata: Doesn't exist - returns nothing", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '""', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[8]"], + }, + ], + }, + { + name: "Jsonata: Selects the number field in the first item", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '"0203 544 1234"', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[0].number"], + }, + ], + }, + { + name: "Jsonata: No index is given to Phone so it selects all of them (the whole array), then it selects all the number fields for each of them", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: + '["0203 544 1234","01962 001234","01962 001235","077 7700 1234"]', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone.number"], + }, + ], + }, + { + name: "Jsonata: Might expect it to just return the first number, but it returns the first number of each of the items selected by Phone", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: + '["0203 544 1234","01962 001234","01962 001235","077 7700 1234"]', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone.number[0]"], + }, + ], + }, + { + name: "Jsonata: Applies the index to the array returned by Phone.number.", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '"0203 544 1234"', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["(Phone.number)[0]"], + }, + ], + }, + { + name: "Jsonata: Returns a range of items by creating an array of indexes", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: + '[{"type":"home","number":"0203 544 1234"},{"type":"office","number":"01962 001234"}]', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[[0..1]]"], + }, + ], + }, + // Predicates + { + name: "Jsonata: Select the Phone items that have a type field that equals 'mobile'", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '{"type":"mobile","number":"077 7700 1234"}', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[type='mobile']"], + }, + ], + }, + { + name: "Jsonata: Select the mobile phone number", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '"077 7700 1234"', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[type='mobile'].number"], + }, + ], + }, + { + name: "Jsonata: Select the office phone numbers - there are two of them", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '["01962 001234","01962 001235"]', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Phone[type='office'].number"], + }, + ], + }, + // Wildcards + { + name: "Jsonata: Select the values of all the fields of 'Address'", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '["Hursley Park","Winchester","SO21 2JN"]', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Address.*"], + }, + ], + }, + { + name: "Jsonata: Select the 'Postcode' value of any child object", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '"SO21 2JN"', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["*.Postcode"], + }, + ], + }, + { + name: "Jsonata: Select all Postcode values, regardless of how deeply nested they are in the structure", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '["SO21 2JN","E1 6RF"]', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["**.Postcode"], + }, + ], + }, + // String Expressions + { + name: "Jsonata: Concatenate 'FirstName' followed by space followed by 'Surname'", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '"Fred Smith"', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["FirstName & ' ' & Surname"], + }, + ], + }, + { + name: "Jsonata: Concatenates the 'Street' and 'City' from the 'Address' object with a comma separator", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '"Hursley Park, Winchester"', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Address.(Street & ', ' & City)"], + }, + ], + }, + { + name: "Jsonata: Casts the operands to strings, if necessary", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: '"50true"', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["5&0&true"], + }, + ], + }, + // Numeric Expressions + { + name: "Jsonata: Addition", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "3.4", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[0] + Numbers[1]"], + }, + ], + }, + { + name: "Jsonata: Subtraction", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "-19.9", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[0] - Numbers[4]"], + }, + ], + }, + { + name: "Jsonata: Multiplication", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "30", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[0] * Numbers[5]"], + }, + ], + }, + { + name: "Jsonata: Division", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "0.04784688995215311", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[0] / Numbers[4]"], + }, + ], + }, + { + name: "Jsonata: Modulus", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "3.5", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[2] % Numbers[5]"], + }, + ], + }, + { + name: "Jsonata: Equality", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "false", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[0] = Numbers[5]"], + }, + ], + }, + { + name: "Jsonata: Inequality", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "true", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[0] != Numbers[4]"], + }, + ], + }, + { + name: "Jsonata: Less than", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "true", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[0] < Numbers[4]"], + }, + ], + }, + { + name: "Jsonata: Less than or equal to", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "true", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[0] <= Numbers[4]"], + }, + ], + }, + { + name: "Jsonata: Greater than", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "false", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[0] > Numbers[4]"], + }, + ], + }, + { + name: "Jsonata: Greater than or equal to", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "false", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["Numbers[2] >= Numbers[4]"], + }, + ], + }, + { + name: "Jsonata: Value is contained in", + input: INPUT_JSON_OBJECT_WITH_ARRAYS, + expectedOutput: "true", + recipeConfig: [ + { + op: "Jsonata Query", + args: ['"01962 001234" in Phone.number'], + }, + ], + }, + // Boolean Expressions + { + name: "Jsonata: and", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "true", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["(Numbers[2] != 0) and (Numbers[5] != Numbers[1])"], + }, + ], + }, + { + name: "Jsonata: or", + input: INPUT_NUMBER_ARRAY, + expectedOutput: "true", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["(Numbers[2] != 0) or (Numbers[5] = Numbers[1])"], + }, + ], + }, + // Array tests + { + name: "Jsonata: $ at the start of an expression refers to the entire input document, subscripting it with 0 selects the first item", + input: INPUT_ARRAY_OF_OBJECTS, + expectedOutput: '{"ref":[1,2]}', + recipeConfig: [ + { + op: "Jsonata Query", + args: ["$[0]"], + }, + ], + }, + { + name: "Jsonata: .ref here returns the entire internal array", + input: INPUT_ARRAY_OF_OBJECTS, + expectedOutput: "[1,2]", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["$[0].ref"], + }, + ], + }, + { + name: "Jsonata: returns element on first position of the internal array", + input: INPUT_ARRAY_OF_OBJECTS, + expectedOutput: "1", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["$[0].ref[0]"], + }, + ], + }, + { + name: "Jsonata: $.field_reference flattens the result into a single array", + input: INPUT_ARRAY_OF_OBJECTS, + expectedOutput: "[1,2,3,4]", + recipeConfig: [ + { + op: "Jsonata Query", + args: ["$.ref"], + }, + ], + }, +]); From 91125f00e7abf09298e251ff7106a5ff0e356767 Mon Sep 17 00:00:00 2001 From: Jon King Date: Mon, 17 Feb 2025 10:19:38 -0700 Subject: [PATCH 6/6] updated Jsonata Query tests header title --- tests/operations/tests/Jsonata.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/operations/tests/Jsonata.mjs b/tests/operations/tests/Jsonata.mjs index d664137f..fb46a961 100644 --- a/tests/operations/tests/Jsonata.mjs +++ b/tests/operations/tests/Jsonata.mjs @@ -1,5 +1,5 @@ /** - * JSON to CSV tests. + * Jsonata Query tests. * * @author Jon King [jon@ajarsoftware.com] *