From 283d7f2159db862f0b17d2840251775f6c762c6b Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 10:40:18 +0000 Subject: [PATCH 01/11] Add Output Filter option to Magic operation --- src/core/lib/Magic.mjs | 9 ++++++--- src/core/operations/Magic.mjs | 9 +++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index b4d5a7b0..52908eb3 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -267,7 +267,7 @@ class Magic { * @param {boolean} [useful=false] - Whether the current recipe should be scored highly * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ - async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false) { + async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, filter=null) { if (depth < 0) return []; // Find any operations that can be run on this data @@ -305,7 +305,7 @@ class Magic { const magic = new Magic(output, this.opPatterns), speculativeResults = await magic.speculativeExecution( - depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful); + depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, filter); results = results.concat(speculativeResults); })); @@ -331,7 +331,10 @@ class Magic { r.fileType || // A file was found r.isUTF8 || // UTF-8 was found r.matchingOps.length // A matching op was found - ) + ) && + ( + filter == null || // Either no filter was passed, or + new RegExp(filter).test(r.data)) // the filter matches the result data ); // Return a sorted list of possible recipes along with their properties diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index b3e15e63..bb419bab 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -43,6 +43,11 @@ class Magic extends Operation { "name": "Extensive language support", "type": "boolean", "value": false + }, + { + "name": "Output Filter (Regex)", + "type": "string", + "value": "" } ]; } @@ -56,10 +61,10 @@ class Magic extends Operation { */ async run(state) { const ings = state.opList[state.progress].ingValues, - [depth, intensive, extLang] = ings, + [depth, intensive, extLang, filter] = ings, dish = state.dish, magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)), - options = await magic.speculativeExecution(depth, extLang, intensive); + options = await magic.speculativeExecution(depth, extLang, intensive, [], false, filter); // Record the current state for use when presenting this.state = state; From 367d79e82049ea6200f807f547129de72998b2a7 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 11:55:49 +0000 Subject: [PATCH 02/11] Fix filtering to work on all the data and not just the result snippet --- src/core/lib/Magic.mjs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 52908eb3..50c632b8 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -276,17 +276,18 @@ class Magic { let results = []; // Record the properties of the current data - results.push({ - recipe: recipeConfig, - data: this.inputStr.slice(0, 100), - languageScores: this.detectLanguage(extLang), - fileType: this.detectFileType(), - isUTF8: this.isUTF8(), - entropy: this.calcEntropy(), - matchingOps: matchingOps, - useful: useful - }); - + if (filter == null || new RegExp(filter).test(this.inputStr)){ + results.push({ + recipe: recipeConfig, + data: this.inputStr.slice(0, 100), + languageScores: this.detectLanguage(extLang), + fileType: this.detectFileType(), + isUTF8: this.isUTF8(), + entropy: this.calcEntropy(), + matchingOps: matchingOps, + useful: useful + }); + } const prevOp = recipeConfig[recipeConfig.length - 1]; // Execute each of the matching operations, then recursively call the speculativeExecution() @@ -331,10 +332,7 @@ class Magic { r.fileType || // A file was found r.isUTF8 || // UTF-8 was found r.matchingOps.length // A matching op was found - ) && - ( - filter == null || // Either no filter was passed, or - new RegExp(filter).test(r.data)) // the filter matches the result data + ) ); // Return a sorted list of possible recipes along with their properties From c8eab5d2185cf89fbfd0b7b2065dc842de15905f Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 14:06:39 +0000 Subject: [PATCH 03/11] Commenting --- src/core/lib/Magic.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 50c632b8..eeda155a 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -276,6 +276,8 @@ class Magic { let results = []; // Record the properties of the current data + // Only if there either wasn't a filter provided, + // or the filter matches in the data if (filter == null || new RegExp(filter).test(this.inputStr)){ results.push({ recipe: recipeConfig, From b6bdcaa71f8f5b689c7a76e6840c351d9a983c6a Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 14:19:52 +0000 Subject: [PATCH 04/11] Rename output filter argument to Crib --- src/core/operations/Magic.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index bb419bab..4575acf6 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -45,7 +45,7 @@ class Magic extends Operation { "value": false }, { - "name": "Output Filter (Regex)", + "name": "Crib (known plaintext string or regex)", "type": "string", "value": "" } From 5d4c7244e197956e967dd5ae9626309ccf9cef2c Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 16:52:09 +0000 Subject: [PATCH 05/11] Add note about crib to operation description --- src/core/operations/Magic.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index 4575acf6..8bfdb067 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -23,7 +23,7 @@ class Magic extends Operation { this.name = "Magic"; this.flowControl = true; this.module = "Default"; - this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar."; + this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.

Optionally enter a regular expression to match a string you expect to find to filter results (crib)"; this.infoURL = "https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic"; this.inputType = "ArrayBuffer"; this.outputType = "JSON"; From f600571c6d743e573be2b7966f01f67bcf6b63a1 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 19 Dec 2018 09:42:09 +0000 Subject: [PATCH 06/11] Fix to make the filter work when intensive mode was turned on. --- src/core/lib/Magic.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index eeda155a..640979cf 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -265,6 +265,7 @@ class Magic { * performance) * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point * @param {boolean} [useful=false] - Whether the current recipe should be scored highly + * @param {string} [filter=null] - The regex crib provided by the user, to filter the operation output * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, filter=null) { @@ -320,7 +321,7 @@ class Magic { await Promise.all(bfEncodings.map(async enc => { const magic = new Magic(enc.data, this.opPatterns), bfResults = await magic.speculativeExecution( - depth-1, extLang, false, [...recipeConfig, enc.conf]); + depth-1, extLang, false, [...recipeConfig, enc.conf], useful, filter); results = results.concat(bfResults); })); From 866dd546c8bddc20a6678b0cd87d52f99714cda9 Mon Sep 17 00:00:00 2001 From: Anthony Arnold Date: Thu, 20 Dec 2018 00:05:10 +1000 Subject: [PATCH 07/11] Add the Play Media operation and place it under a new "Multimedia" category. Move the Render Image to this new category as well. --- src/core/config/Categories.json | 8 ++- src/core/operations/PlayMedia.mjs | 101 ++++++++++++++++++++++++++++++ test/index.mjs | 1 + test/tests/operations/Media.mjs | 43 +++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/PlayMedia.mjs create mode 100644 test/tests/operations/Media.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 3998d7db..690fea57 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -341,6 +341,13 @@ "From MessagePack" ] }, + { + "name": "Multimedia", + "ops": [ + "Render Image", + "Play Media" + ] + }, { "name": "Other", "ops": [ @@ -355,7 +362,6 @@ "Generate TOTP", "Generate HOTP", "Haversine distance", - "Render Image", "Remove EXIF", "Extract EXIF", "Numberwang", diff --git a/src/core/operations/PlayMedia.mjs b/src/core/operations/PlayMedia.mjs new file mode 100644 index 00000000..e6ac53e4 --- /dev/null +++ b/src/core/operations/PlayMedia.mjs @@ -0,0 +1,101 @@ +/** + * @author anthony-arnold [anthony.arnold@uqconnect.edu.au] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import { fromBase64, toBase64 } from "../lib/Base64"; +import { fromHex } from "../lib/Hex"; +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Utils from "../Utils"; +import Magic from "../lib/Magic"; + +/** + * PlayMedia operation + */ +class PlayMedia extends Operation { + + /** + * PlayMedia constructor + */ + constructor() { + super(); + + this.name = "Play Media"; + this.module = "Media"; + this.description = "Plays the input as sound or video depending on the type."; + this.infoURL = ""; + this.inputType = "string"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + "name": "Input format", + "type": "option", + "value": ["Raw", "Base64", "Hex"] + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {byteArray} The multimedia data as bytes. + */ + run(input, args) { + const inputFormat = args[0]; + + if (!input.length) return []; + + // Convert input to raw bytes + switch (inputFormat) { + case "Hex": + input = fromHex(input); + break; + case "Base64": + // Don't trust the Base64 entered by the user. + // Unwrap it first, then re-encode later. + input = fromBase64(input, undefined, "byteArray"); + break; + case "Raw": + default: + input = Utils.strToByteArray(input); + break; + } + + + // Determine file type + const type = Magic.magicFileType(input); + if (!(type && /^audio|video/.test(type.mime))) { + throw new OperationError("Invalid file type"); + } + + return input; + } + + /** + * Displays an audio or video element that may be able to play the media + * file. + * @param data {byteArray} Data containing an audio or video file. + * @returns {string} Markup to display a media player. + */ + async present(data) { + if (!data.length) return ""; + + const type = Magic.magicFileType(data); + const matches = /^audio|video/.exec(type.mime); + if (!matches) { + throw new OperationError("Invalid file type"); + } + const dataURI = `data:${type.mime};base64,${toBase64(data)}`; + const element = matches[0]; + + let html = `<${element} src='${dataURI}' type='${type.mime}' controls>`; + html += "

Unsupported media type.

"; + html += ``; + return html; + } +} + +export default PlayMedia; diff --git a/test/index.mjs b/test/index.mjs index a08d9b32..c944b765 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -80,6 +80,7 @@ import "./tests/operations/ToGeohash.mjs"; import "./tests/operations/TranslateDateTimeFormat"; import "./tests/operations/Magic"; import "./tests/operations/ParseTLV"; +import "./tests/operations/Media"; let allTestsPassing = true; const testStatusCounts = { diff --git a/test/tests/operations/Media.mjs b/test/tests/operations/Media.mjs new file mode 100644 index 00000000..4d285e8d --- /dev/null +++ b/test/tests/operations/Media.mjs @@ -0,0 +1,43 @@ +/** + * Media operation tests. + * @author anthony-arnold [anthony.arnold@uqconnect.edu.au] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ +import TestRegister from "../../TestRegister"; + +TestRegister.addTests([ + { + name: "Play Media: nothing", + input: "", + expectedOutput: "", + recipeConfig: [ + { op: "Play Media", args: ["Raw"] } + ] + }, + { + name: "Play Media: raw wav", + input: "52494646bcaf010057415645666d74201000000001000100401f0000401f0000010008006461746198af0100818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081818281807f82807f817e81808280827d8086817d80828184817f80807d847f7e7e8582837b81857e7e82867f7c7f857e7d838182837f83847b7c838082807c7d838481827e83827c7f848383807c7f7c80848085808180827e83827f7c82827c7f858183807c837b847f7f82827a85837a8287817f7882827f89837e7f877b7e808281817f7e807c827b7e7d8383838486838883837d7d7682797d847f7d868287867d83847c81807d7e818080828082828380807e7f827d8181807e828082858581808081807f7a7f81808183807d7c7f7f858180848484857f8082817e827b7e7f7f8385807f7f7d858183808081807f7f8183847f7e7d81828284857d7f7e8084868182837d7e7d7a7e7e8085837f80807b7f7b7a80827d7f817e8386838588868386868689868586838386868584807c7a78787875706e6c6a6c6d70757a7c7e7f80858b94a1a29a9996989a9e9ca09ea19caa8e554129243e4d475e7184a0ada2a4a0918179646167696f7b7a8390939ca4aab2af9c9691909aa2a3ae74321e162a4d5c506c88a5babca6a4a7a18c7a5e5c646f6a6b6d7f929a8f858182827e79849299a1abb2c7cc975b3826324447496788a5b8b6a9aaa9977b60494c586467707c949fa195938f8f847a717c88a0aab7b7d6a46045362f3c483d6089a1acb4a3a4a89570615652676f6d788f97a29c8e8992898381808393999da9bf8e515e504c4e5d4879929a90a1909e9c855c5b5f717679768b9b9f958c838c8d8c7f8782969ca797a963585f63425d60687e93818b99938c84786f7e7a7b7f86838a867f8486858899939697a7a6be6e465f6c4d465d678e948275958a867063688e8b828fa2a0988b7a7f7c737185898a8e90989c9b8c8c6e5f5a655159646f727f737382857c7f898a9aa4b1b6cac5d4957d695b403c4152", + expectedOutput: "", + recipeConfig: [ + { op: "From Hex", args: ["Space"] }, + { op: "Play Media", args: ["Raw"] } + ] + }, + { + name: "Play Media: hex ogg", + input: "4f676753000200000000000000003129000000000000642493e3011e01766f72626973000000000244ac0000000000008138010000000000b8014f676753000000000000000000003129000001000000a3565ae9102dffffffffffffffffffffffffffff2403766f726269731d000000586970682e4f7267206c6962566f726269732049203230303230373137000000000105766f726269732242435601000001009c739a318799629452892194de3968196394526929a55a4aa9a183166babbdf7de7befbdf7de7bef1d739431469552524aa99d739631471563524a89a5945642682184d662abbdf7de6befb5f6de7bef99424c29a41442084a281d538c29a494424a4a0825640e3ac61c538c52093dd65e6bccbdb6d87beda163ce39e61c534c4a6821740e3ae69c534c4a68a984524206a153d05289adf7de62ebb9a5da7bef81d0905500000100c040101ab20a00500000108aa1188a028486ac020032000004e0288ee3388ee23892623916101ab20a00000200100000c0900c4bb114cdd1244dd22ccf134dd3377dd3366d55d7755dd7755dd77520346415000001004040a719a61a20c28c6416080d590500200000004420c3140342435601000001000052243949a2e4a494520e836431492ae5a494521ec5e4514d3206a594524a29a594524a29a594520a8364394a2ae5a4945212a364314aaad4a494521ee5e4", + expectedOutput: "", + recipeConfig: [ + { op: "Play Media", args: ["Hex"] } + ] + }, + { + name: "Play Media: base64 webm", + input: "GkXfo6NChoEBQveBAULygQRC84EIQoKEd2VibeyCAABCh4EBQoWBARhTgGcQIQmHEU2bdLtNu4tTq4QVSalmU6yBQE27i1OrhBZUrmtTrIGsTbuNU6uEEU2bdFOsgyEJc027jFOrhBxTu2tTrIINQRVJqWbnc6SQRsadRaGFqSlNPQovdQBWvSrXsYMPQkBEiYRG/cAARGGIBBu7mlIesABNgKVodHRwOi8vc291cmNlZm9yZ2UubmV0L3Byb2plY3RzL3lhbWthV0GQU29yZW5zb24gU3F1ZWV6ZRZUrmtMj66414EBc8WHiBmgyaYxwoOBASPjg4QCYloAIzFPhD+AAACGhVZfVlA4JYaIg1ZQOOCIsIICgLqCAWiuTFLXgQJzxYgBiP65XI76uoOBAiMxT4Q/gAAAhohBX1ZPUkJJU2OiTBkCHjoBdm9yYmlzAAAAAAFErAAA/////wD6AAD/////uAEDdm9yYmlzKgAAAFhpcGguT3JnIGxpYlZvcmJpcyBJIDIwMTAwMzI1IChFdmVyeXdoZXJlKQAAAAABBXZvcmJpcx9CQ1YBAAABABhjVClGmVLSSokZc5QxRplikkqJpYQWQkidcxRTqTnXnGusubUghBAaU1ApBZlSjlJpGWOQKQWZUhBLSSV0EjonnWMQW0nB1phri0G2HIQNmlJMKcSUUopCCBlTjCnFlFJKQgcldA465hxTjkooQbicc6u1lpZji6l0kkrnJGRMQkgphZJKB6VTTkJINZbWUikdc1JSakHoIIQQQrYghA2C0JBVAAABAMBAEBqyCgBQAAAQiqEYigKEhqwCADIAAASgKI7iKI4jOZJjSRYQGrIKAAACABAAAMBwFEmRFMmxJEvSLEvTRFFVfdU2VVX2dV3XdV3XdSA0ZBUAAAEAQEinmaUaIMIMZBgIDVkFACAAAABGKMIQA0JDVgEAAAEAAGIoOYgmtOZ8c46DZjloKsXmdHAi1eZJbirm5pxzzjknm3PGOOecc4pyZjFoJrTmnHMSg2YpaCa05pxznsTmQWuqtOacc8Y5p4NxRhjnnHOatOZBajbW5pxzFrSmOWouxeaccyLl5kltLtXmnHPOOeecc84555xzqhenc3BOOOecc6L25lpuQhfnnHM+Gad7c0I455xzzjnnnHPOOeecc4LQkFUAABAAAEEYNoZxpyBIn6OBGEWIacikB92jwyRoDHIKqUejo5FS6iCUVMZJKZ0gNGQVAAAIAAAhhBRSSCGFFFJIIYUUUoghhhhiyCmnnIIKKqmkoooyyiyzzDLLLLPMMuuws8467DDEEEMMrbQSS0211VhjrbnnnGsO0lpprbXWSimllFJKKQgNWQUAgAAAEAgZZJBBRiGFFFKIIaaccsopqKACQkNWAQCAAAACAAAAPMlzREd0REd0REd0REd0RMdzPEeUREmUREm0TMvUTE8VVdWVXVvWZd32bWEXdt33dd/3dePXhWFZlmVZlmVZlmVZlmVZlmVZgtCQVQAACAAAgBBCCCGFFFJIIaUYY8wx56CTUEIgNGQVAAAIACAAAADAURzFcSRHciTJkixJkzRLszzN0zxN9ERRFE3TVEVXdEXdtEXZlE3XdE3ZdFVZtV1Ztm3Z1m1flm3f933f933f933f933f93UdCA1ZBQBIAADoSI6kSIqkSI7jOJIkAaEhqwAAGQAAAQAoiqM4juNIkiRJlqRJnuVZomZqpmd6qqgCoSGrAABAAAABAAAAAAAomuIppuIpouI5oiNKomVaoqZqriibsuu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6LhAasgoAkAAA0JEcyZEcSZEUSZEcyQFCQ1YBADIAAAIAcAzHkBTJsSxL0zzN0zxN9ERP9ExPFV3RBUJDVgEAgAAAAgAAAAAAMCTDUixHczRJlFRLtVRNtVRLFVVPVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVNU3TNE0gNGQlAAAEAMBijcHlICElJeXeEMIQk54xJiG1XiEEkZLeMQYVg54yogxy3kLjEIMeCA1ZEQBEAQAAxiDHEHPIOUepkxI556h0lBrnHKWOUmcpxZhizSiV2FKsjXOOUketo5RiLC12lFKNqcYCAAACHAAAAiyEQkNWBABRAACEMUgppBRijDmnnEOMKeeYc4Yx5hxzjjnnoHRSKuecdE5KxBhzjjmnnHNSOieVc05KJ6EAAIAABwCAAAuh0JAVAUCcAIBBkjxP8jRRlDRPFEVTdF1RNF3X8jzV9ExTVT3RVFVTVW3ZVFVZljzPND3TVFXPNFXVVFVZNlVVlkVV1W3TdXXbdFXdlm3b911bFnZRVW3dVF3bN1XX9l3Z9n1Z1nVj8jxV9UzTdT3TdGXVdW1bdV1d90xTlk3XlWXTdW3blWVdd2XZ9zXTdF3TVWXZdF3ZdmVXt11Z9n3TdYXflWVfV2VZGHZd94Vb15XldF3dV2VXN1ZZ9n1b14Xh1nVhmTxPVT3TdF3PNF1XdV1fV13X1jXTlGXTdW3ZVF1ZdmXZ911X1nXPNGXZdF3bNl1Xll1Z9n1XlnXddF1fV2VZ+FVX9nVZ15Xh1m3hN13X91VZ9oVXlnXh1nVhuXVdGD5V9X1TdoXhdGXf14XfWW5dOJbRdX1hlW3hWGVZOX7hWJbd95VldF1fWG3ZGFZZFoZf+J3l9n3jeHVdGW7d58y67wzH76T7ytPVbWOZfd1ZZl93juEYOr/w46mqr5uuKwynLAu/7evGs/u+soyu6/uqLAu/KtvCseu+8/y+sCyj7PrCasvCsNq2Mdy+biy/cBzLa+vKMeu+UbZ1fF94CsPzdHVdeWZdx/Z1dONHOH7KAACAAQcAgAATykChISsCgDgBAI8kiaJkWaIoWZYoiqbouqJouq6kaaapaZ5pWppnmqZpqrIpmq4saZppWp5mmpqnmaZomq5rmqasiqYpy6ZqyrJpmrLsurJtu65s26JpyrJpmrJsmqYsu7Kr267s6rqkWaapeZ5pap5nmqZqyrJpmq6reZ5qep5oqp4oqqpqqqqtqqosW55nmproqaYniqpqqqatmqoqy6aq2rJpqrZsqqptu6rs+rJt67ppqrJtqqYtm6pq267s6rIs27ovaZppap5nmprnmaZpmrJsmqorW56nmp4oqqrmiaZqqqosm6aqypbnmaoniqrqiZ5rmqoqy6Zq2qppmrZsqqotm6Yqy65t+77ryrJuqqpsm6pq66ZqyrJsy77vyqruiqYpy6aq2rJpqrIt27Lvy7Ks+6JpyrJpqrJtqqouy7JtG7Ns+7pomrJtqqYtm6oq27It+7os27rvyq5vq6qs67It+7ru+q5w67owvLJs+6qs+ror27pv6zLb9n1E05RlUzVt21RVWXZl2fZl2/Z90TRtW1VVWzZN1bZlWfZ9WbZtYTRN2TZVVdZN1bRtWZZtYbZl4XZl2bdlW/Z115V1X9d949dl3ea6su3Lsq37qqv6tu77wnDrrvAKAAAYcAAACDChDBQashIAiAIAAIxhjDEIjVLOOQehUco55yBkzkEIIZXMOQghlJI5B6GUlDLnIJSSUgihlJRaCyGUlFJrBQAAFDgAAATYoCmxOEChISsBgFQAAIPjWJbnmaJq2rJjSZ4niqqpqrbtSJbniaJpqqptW54niqapqq7r65rniaJpqqrr6rpomqapqq7ruroumqKpqqrrurKum6aqqq4ru7Ls66aqqqrryq4s+8Kquq4ry7Jt68Kwqq7ryrJs27Zv3Lqu677v+8KRreu6LvzCMQxHAQDgCQ4AQAU2rI5wUjQWWGjISgAgAwCAMAYhgxBCBiGEkFJKIaWUEgAAMOAAABBgQhkoNGQlABADAAAQASGDEEIIIYQQQgghhBBCCCGEEELnnHPOOeecc84JANiPcACQejAxMYWFhqwEAFIBAABjlFKKMecgRIw5xhh0EkqKGHOOMQelpFQ5ByGEVFrLrXIOQggptVRb5pyU1mKMOcbMOSkpxVZzzqGU1GKsueaaOymt1ZprzbmW1mrNNedccy6txZprzjXn3HLMNeecc845xpxzzjnnnHMBADgNDgCgBzasjnBSNBZYaMhKACAVAIBARinGnHMOOoQUY845ByGESCHGnHMOQggVY845Bx2EECrGHHMOQgghZM45ByGEEELInIMOOgghhNBBByGEEEIopXMQQgghhBJKCCGEEEIIIYQOQgghhBBCCCGEEEIIoZQSQgghhFBCKCUUAABY4AAAEGDD6ggnRWOBhYasBACAAAAghyWolDNhkGPQY0OQctRMgxBTTnSmmJPaTMUUZA5EJ51EhlpQtpfMAgAAIAgACDABBAYICr4QAmIMAEAQIjNEQmEVLDAogwaHeQDwABEhEQAkJijSLi6gywAXdHHXgRCCEIQgFgdQQAIOTrjhiTc84QYn6BSVOggAAAAAAAMAeAAAOCiAiIjmKiwuMDI0Njg6PAIAAAAAAAYAPgAAjg8gIqK5CosLjAyNDY4OjwAAAAAAAAAAACAgIAAAAAAAEAAAACAgJYaIhlZvcmJpc+GGtYRHLEQAHFO7a0IAu4yzgQC3h/eBAfGCD0e7kbOCAli3i/eBAfGCD0dTeIEqu5Gzgg3At4v3gQHxgg9HU3iB8buSs4Ib0LeM94EB8YIPR1N4ggHmu5KzgiN4t4z3gQHxgg9HU3iCAnW7krOCK8C3jPeBAfGCD0dTeIIDBruSs4IzkLeM94EB8YIPR1N4ggOXu5KzgjY4t4z3gQHxgg9HU3iCA8W7krOCOpi3jPeBAfGCD0dTeIIEIbuSs4I+gLeM94EB8YIPR1N4ggRlu5KzgkDYt4z3gQHxgg9HU3iCBI67krOCSNC3jPeBAfGCD0dTeIIFIbuSs4JJwLeM94EB8YIPR1N4ggU2u5Kzgk3Qt4z3gQHxgg9HU3iCBYO7krOCUZC3jPeBAfGCD0dTeIIFxLuSs4JR4LeM94EB8YIPR1N4ggXKu5KzglXwt4z3gQHxgg9HU3iCBhe7krOCWYi3jPeBAfGCD0dTeIIGXLuSs4JhWLeM94EB8YIPR1N4ggblu5KzgmWQt4z3gQHxgg9HU3iCBy67krOCaDi3jPeBAfGCD0dTeIIHXLuSs4JosLeM94EB8YIPR1N4ggdku5KzgnUIt4z3gQHxgg9HU3iCCEK7krOCddC3jPeBAfGCD0dTeIIIUruSs4J2ILeM94EB8YIPR1N4gghdu5Kzgn5At4z3gQHxgg9HU3iCCPgfQ7Z1ECD6JOeBAKeCD0ejQbOBAACAEjQAnQEqgAJoATkPAEEcIhYWIhYSIAYAABhYE9d0hkrLkkLy9LukMlZckheXpd0hkrLkkLy9LukMlZckheXpd0hkrLkkLy9LukMlZckheXpd0hkrLkkLy9LukMlZckheXpd0hkrLkkLy9LukMlZckheXpQ==", + expectedOutput: "", + recipeConfig: [ + { op: "Play Media", args: ["Base64"] } + ] + } +]); From ed2c886359669ee090c5e795668510c9f8a19f14 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 19 Dec 2018 17:24:38 +0000 Subject: [PATCH 08/11] Magic crib now only applies after all branches have been explored. --- src/core/lib/Magic.mjs | 33 +++++++++++++++------------------ src/core/operations/Magic.mjs | 12 +++++++++--- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 640979cf..00132d5d 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -265,10 +265,10 @@ class Magic { * performance) * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point * @param {boolean} [useful=false] - Whether the current recipe should be scored highly - * @param {string} [filter=null] - The regex crib provided by the user, to filter the operation output + * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation output * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ - async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, filter=null) { + async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, crib=null) { if (depth < 0) return []; // Find any operations that can be run on this data @@ -277,20 +277,17 @@ class Magic { let results = []; // Record the properties of the current data - // Only if there either wasn't a filter provided, - // or the filter matches in the data - if (filter == null || new RegExp(filter).test(this.inputStr)){ - results.push({ - recipe: recipeConfig, - data: this.inputStr.slice(0, 100), - languageScores: this.detectLanguage(extLang), - fileType: this.detectFileType(), - isUTF8: this.isUTF8(), - entropy: this.calcEntropy(), - matchingOps: matchingOps, - useful: useful - }); - } + results.push({ + recipe: recipeConfig, + data: this.inputStr.slice(0, 100), + languageScores: this.detectLanguage(extLang), + fileType: this.detectFileType(), + isUTF8: this.isUTF8(), + entropy: this.calcEntropy(), + matchingOps: matchingOps, + useful: useful, + matchesCrib: crib && crib.test(this.inputStr) + }); const prevOp = recipeConfig[recipeConfig.length - 1]; // Execute each of the matching operations, then recursively call the speculativeExecution() @@ -309,7 +306,7 @@ class Magic { const magic = new Magic(output, this.opPatterns), speculativeResults = await magic.speculativeExecution( - depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, filter); + depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib); results = results.concat(speculativeResults); })); @@ -321,7 +318,7 @@ class Magic { await Promise.all(bfEncodings.map(async enc => { const magic = new Magic(enc.data, this.opPatterns), bfResults = await magic.speculativeExecution( - depth-1, extLang, false, [...recipeConfig, enc.conf], useful, filter); + depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib); results = results.concat(bfResults); })); diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index 8bfdb067..1555ebad 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -23,7 +23,7 @@ class Magic extends Operation { this.name = "Magic"; this.flowControl = true; this.module = "Default"; - this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.

Optionally enter a regular expression to match a string you expect to find to filter results (crib)"; + this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.

Optionally enter a regular expression to match a string you expect to find to filter results (crib)."; this.infoURL = "https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic"; this.inputType = "ArrayBuffer"; this.outputType = "JSON"; @@ -61,10 +61,16 @@ class Magic extends Operation { */ async run(state) { const ings = state.opList[state.progress].ingValues, - [depth, intensive, extLang, filter] = ings, + [depth, intensive, extLang, crib] = ings, dish = state.dish, magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)), - options = await magic.speculativeExecution(depth, extLang, intensive, [], false, filter); + cribRegex = (crib && crib.length) ? new RegExp(crib, "i") : null; + let options = await magic.speculativeExecution(depth, extLang, intensive, [], false, cribRegex); + + // Filter down to results which matched the crib + if (cribRegex) { + options = options.filter(option => option.matchesCrib); + } // Record the current state for use when presenting this.state = state; From f367c1f78bbcecd9f841efa31622627758bbe041 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 19 Dec 2018 17:25:05 +0000 Subject: [PATCH 09/11] 8.15.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5e20c8bc..8ea5bd80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.15.0", + "version": "8.15.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 3505cfa2..e5cd12e9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.15.0", + "version": "8.15.1", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 01c4cfdc8d6fbeb9589e820aa2414f5736a9eeb2 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 19 Dec 2018 17:58:38 +0000 Subject: [PATCH 10/11] Tidied up 'Play Media' operation --- CHANGELOG.md | 6 ++++++ src/core/config/Categories.json | 10 +++++----- src/core/operations/PlayMedia.mjs | 9 +++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6355831e..751ac287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.16.0] - 2018-12-19 +- 'Play Media' operation added [@anthony-arnold] | [#446] + ### [8.15.0] - 2018-12-18 - 'Text Encoding Brute Force' operation added [@Cynser] | [#439] @@ -76,6 +79,7 @@ All major and minor version changes will be documented in this file. Details of +[8.16.0]: https://github.com/gchq/CyberChef/releases/tag/v8.16.0 [8.15.0]: https://github.com/gchq/CyberChef/releases/tag/v8.15.0 [8.14.0]: https://github.com/gchq/CyberChef/releases/tag/v8.14.0 [8.13.0]: https://github.com/gchq/CyberChef/releases/tag/v8.13.0 @@ -112,6 +116,7 @@ All major and minor version changes will be documented in this file. Details of [@jarmovanlenthe]: https://github.com/jarmovanlenthe [@tcode2k16]: https://github.com/tcode2k16 [@Cynser]: https://github.com/Cynser +[@anthony-arnold]: https://github.com/anthony-arnold [#95]: https://github.com/gchq/CyberChef/pull/299 [#173]: https://github.com/gchq/CyberChef/pull/173 @@ -138,3 +143,4 @@ All major and minor version changes will be documented in this file. Details of [#439]: https://github.com/gchq/CyberChef/pull/439 [#441]: https://github.com/gchq/CyberChef/pull/441 [#443]: https://github.com/gchq/CyberChef/pull/443 +[#446]: https://github.com/gchq/CyberChef/pull/446 diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 690fea57..2e6bfe2d 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -342,8 +342,12 @@ ] }, { - "name": "Multimedia", + "name": "Forensics", "ops": [ + "Detect File Type", + "Scan for Embedded Files", + "Remove EXIF", + "Extract EXIF", "Render Image", "Play Media" ] @@ -354,16 +358,12 @@ "Entropy", "Frequency distribution", "Chi Square", - "Detect File Type", - "Scan for Embedded Files", "Disassemble x86", "Pseudo-Random Number Generator", "Generate UUID", "Generate TOTP", "Generate HOTP", "Haversine distance", - "Remove EXIF", - "Extract EXIF", "Numberwang", "XKCD Random Number" ] diff --git a/src/core/operations/PlayMedia.mjs b/src/core/operations/PlayMedia.mjs index e6ac53e4..81328a73 100644 --- a/src/core/operations/PlayMedia.mjs +++ b/src/core/operations/PlayMedia.mjs @@ -23,8 +23,8 @@ class PlayMedia extends Operation { super(); this.name = "Play Media"; - this.module = "Media"; - this.description = "Plays the input as sound or video depending on the type."; + this.module = "Default"; + this.description = "Plays the input as audio or video depending on the type.

Tags: sound, movie, mp3, mp4, mov, webm, wav, ogg"; this.infoURL = ""; this.inputType = "string"; this.outputType = "byteArray"; @@ -44,7 +44,7 @@ class PlayMedia extends Operation { * @returns {byteArray} The multimedia data as bytes. */ run(input, args) { - const inputFormat = args[0]; + const [inputFormat] = args; if (!input.length) return []; @@ -68,7 +68,7 @@ class PlayMedia extends Operation { // Determine file type const type = Magic.magicFileType(input); if (!(type && /^audio|video/.test(type.mime))) { - throw new OperationError("Invalid file type"); + throw new OperationError("Invalid or unrecognised file type"); } return input; @@ -77,6 +77,7 @@ class PlayMedia extends Operation { /** * Displays an audio or video element that may be able to play the media * file. + * * @param data {byteArray} Data containing an audio or video file. * @returns {string} Markup to display a media player. */ From 8c3569ea63728e1ccd8690f87a0eacde97385cc5 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 19 Dec 2018 17:59:55 +0000 Subject: [PATCH 11/11] 8.16.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8ea5bd80..f146cc0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.15.1", + "version": "8.16.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index e5cd12e9..cfa83d39 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.15.1", + "version": "8.16.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef",