From 283d7f2159db862f0b17d2840251775f6c762c6b Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 10:40:18 +0000 Subject: [PATCH 1/6] Add Output Filter option to Magic operation --- src/core/lib/Magic.mjs | 9 ++++++--- src/core/operations/Magic.mjs | 9 +++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index b4d5a7b0..52908eb3 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -267,7 +267,7 @@ class Magic { * @param {boolean} [useful=false] - Whether the current recipe should be scored highly * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ - async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false) { + async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, filter=null) { if (depth < 0) return []; // Find any operations that can be run on this data @@ -305,7 +305,7 @@ class Magic { const magic = new Magic(output, this.opPatterns), speculativeResults = await magic.speculativeExecution( - depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful); + depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, filter); results = results.concat(speculativeResults); })); @@ -331,7 +331,10 @@ class Magic { r.fileType || // A file was found r.isUTF8 || // UTF-8 was found r.matchingOps.length // A matching op was found - ) + ) && + ( + filter == null || // Either no filter was passed, or + new RegExp(filter).test(r.data)) // the filter matches the result data ); // Return a sorted list of possible recipes along with their properties diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index b3e15e63..bb419bab 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -43,6 +43,11 @@ class Magic extends Operation { "name": "Extensive language support", "type": "boolean", "value": false + }, + { + "name": "Output Filter (Regex)", + "type": "string", + "value": "" } ]; } @@ -56,10 +61,10 @@ class Magic extends Operation { */ async run(state) { const ings = state.opList[state.progress].ingValues, - [depth, intensive, extLang] = ings, + [depth, intensive, extLang, filter] = ings, dish = state.dish, magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)), - options = await magic.speculativeExecution(depth, extLang, intensive); + options = await magic.speculativeExecution(depth, extLang, intensive, [], false, filter); // Record the current state for use when presenting this.state = state; From 367d79e82049ea6200f807f547129de72998b2a7 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 11:55:49 +0000 Subject: [PATCH 2/6] Fix filtering to work on all the data and not just the result snippet --- src/core/lib/Magic.mjs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 52908eb3..50c632b8 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -276,17 +276,18 @@ class Magic { let results = []; // Record the properties of the current data - results.push({ - recipe: recipeConfig, - data: this.inputStr.slice(0, 100), - languageScores: this.detectLanguage(extLang), - fileType: this.detectFileType(), - isUTF8: this.isUTF8(), - entropy: this.calcEntropy(), - matchingOps: matchingOps, - useful: useful - }); - + if (filter == null || new RegExp(filter).test(this.inputStr)){ + results.push({ + recipe: recipeConfig, + data: this.inputStr.slice(0, 100), + languageScores: this.detectLanguage(extLang), + fileType: this.detectFileType(), + isUTF8: this.isUTF8(), + entropy: this.calcEntropy(), + matchingOps: matchingOps, + useful: useful + }); + } const prevOp = recipeConfig[recipeConfig.length - 1]; // Execute each of the matching operations, then recursively call the speculativeExecution() @@ -331,10 +332,7 @@ class Magic { r.fileType || // A file was found r.isUTF8 || // UTF-8 was found r.matchingOps.length // A matching op was found - ) && - ( - filter == null || // Either no filter was passed, or - new RegExp(filter).test(r.data)) // the filter matches the result data + ) ); // Return a sorted list of possible recipes along with their properties From c8eab5d2185cf89fbfd0b7b2065dc842de15905f Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 14:06:39 +0000 Subject: [PATCH 3/6] Commenting --- src/core/lib/Magic.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 50c632b8..eeda155a 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -276,6 +276,8 @@ class Magic { let results = []; // Record the properties of the current data + // Only if there either wasn't a filter provided, + // or the filter matches in the data if (filter == null || new RegExp(filter).test(this.inputStr)){ results.push({ recipe: recipeConfig, From b6bdcaa71f8f5b689c7a76e6840c351d9a983c6a Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 14:19:52 +0000 Subject: [PATCH 4/6] Rename output filter argument to Crib --- src/core/operations/Magic.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index bb419bab..4575acf6 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -45,7 +45,7 @@ class Magic extends Operation { "value": false }, { - "name": "Output Filter (Regex)", + "name": "Crib (known plaintext string or regex)", "type": "string", "value": "" } From 5d4c7244e197956e967dd5ae9626309ccf9cef2c Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 16:52:09 +0000 Subject: [PATCH 5/6] Add note about crib to operation description --- src/core/operations/Magic.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index 4575acf6..8bfdb067 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -23,7 +23,7 @@ class Magic extends Operation { this.name = "Magic"; this.flowControl = true; this.module = "Default"; - this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar."; + this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.

Optionally enter a regular expression to match a string you expect to find to filter results (crib)"; this.infoURL = "https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic"; this.inputType = "ArrayBuffer"; this.outputType = "JSON"; From f600571c6d743e573be2b7966f01f67bcf6b63a1 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 19 Dec 2018 09:42:09 +0000 Subject: [PATCH 6/6] Fix to make the filter work when intensive mode was turned on. --- src/core/lib/Magic.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index eeda155a..640979cf 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -265,6 +265,7 @@ class Magic { * performance) * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point * @param {boolean} [useful=false] - Whether the current recipe should be scored highly + * @param {string} [filter=null] - The regex crib provided by the user, to filter the operation output * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, filter=null) { @@ -320,7 +321,7 @@ class Magic { await Promise.all(bfEncodings.map(async enc => { const magic = new Magic(enc.data, this.opPatterns), bfResults = await magic.speculativeExecution( - depth-1, extLang, false, [...recipeConfig, enc.conf]); + depth-1, extLang, false, [...recipeConfig, enc.conf], useful, filter); results = results.concat(bfResults); }));