Merge branch 'master' into feature-extract-files

CHANGELOG.md

@@ -2,6 +2,9 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
+### [8.16.0] - 2018-12-19
+- 'Play Media' operation added [@anthony-arnold] | [#446]
+
 ### [8.15.0] - 2018-12-18
 - 'Text Encoding Brute Force' operation added [@Cynser] | [#439]
 
@@ -76,6 +79,7 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[8.16.0]: https://github.com/gchq/CyberChef/releases/tag/v8.16.0
 [8.15.0]: https://github.com/gchq/CyberChef/releases/tag/v8.15.0
 [8.14.0]: https://github.com/gchq/CyberChef/releases/tag/v8.14.0
 [8.13.0]: https://github.com/gchq/CyberChef/releases/tag/v8.13.0
@@ -112,6 +116,7 @@ All major and minor version changes will be documented in this file. Details of
 [@jarmovanlenthe]: https://github.com/jarmovanlenthe
 [@tcode2k16]: https://github.com/tcode2k16
 [@Cynser]: https://github.com/Cynser
+[@anthony-arnold]: https://github.com/anthony-arnold
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -138,3 +143,4 @@ All major and minor version changes will be documented in this file. Details of
 [#439]: https://github.com/gchq/CyberChef/pull/439
 [#441]: https://github.com/gchq/CyberChef/pull/441
 [#443]: https://github.com/gchq/CyberChef/pull/443
+[#446]: https://github.com/gchq/CyberChef/pull/446

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.15.0",
+  "version": "8.16.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.15.0",
+  "version": "8.16.0",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

src/core/config/Categories.json

@@ -350,7 +350,8 @@
             "Extract Files",
             "Remove EXIF",
             "Extract EXIF",
-            "Render Image"
+            "Render Image",
+            "Play Media"
         ]
     },
     {
@@ -365,7 +366,6 @@
             "Generate TOTP",
             "Generate HOTP",
             "Haversine distance",
-            "Render Image",
             "Numberwang",
             "XKCD Random Number"
         ]

src/core/lib/Magic.mjs

@@ -273,9 +273,10 @@ class Magic {
      *                                      performance)
      * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point
      * @param {boolean} [useful=false] - Whether the current recipe should be scored highly
+     * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation output
      * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding
      */
-    async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false) {
+    async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, crib=null) {
         if (depth < 0) return [];
 
         // Find any operations that can be run on this data
@@ -292,9 +293,9 @@ class Magic {
             isUTF8: this.isUTF8(),
             entropy: this.calcEntropy(),
             matchingOps: matchingOps,
-            useful: useful
+            useful: useful,
+            matchesCrib: crib && crib.test(this.inputStr)
         });
-
         const prevOp = recipeConfig[recipeConfig.length - 1];
 
         // Execute each of the matching operations, then recursively call the speculativeExecution()
@@ -313,7 +314,7 @@ class Magic {
 
             const magic = new Magic(output, this.opPatterns),
                 speculativeResults = await magic.speculativeExecution(
-                    depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
+                    depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);
 
             results = results.concat(speculativeResults);
         }));
@@ -325,7 +326,7 @@ class Magic {
             await Promise.all(bfEncodings.map(async enc => {
                 const magic = new Magic(enc.data, this.opPatterns),
                     bfResults = await magic.speculativeExecution(
-                        depth-1, extLang, false, [...recipeConfig, enc.conf]);
+                        depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib);
 
                 results = results.concat(bfResults);
             }));

src/core/operations/Magic.mjs

@@ -23,7 +23,7 @@ class Magic extends Operation {
         this.name = "Magic";
         this.flowControl = true;
         this.module = "Default";
-        this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.";
+        this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.<br><br>Optionally enter a regular expression to match a string you expect to find to filter results (crib).";
         this.infoURL = "https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic";
         this.inputType = "ArrayBuffer";
         this.outputType = "JSON";
@@ -43,6 +43,11 @@ class Magic extends Operation {
                 "name": "Extensive language support",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Crib (known plaintext string or regex)",
+                "type": "string",
+                "value": ""
             }
         ];
     }
@@ -56,10 +61,16 @@ class Magic extends Operation {
      */
     async run(state) {
         const ings = state.opList[state.progress].ingValues,
-            [depth, intensive, extLang] = ings,
+            [depth, intensive, extLang, crib] = ings,
             dish = state.dish,
             magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)),
-            options = await magic.speculativeExecution(depth, extLang, intensive);
+            cribRegex = (crib && crib.length) ? new RegExp(crib, "i") : null;
+        let options = await magic.speculativeExecution(depth, extLang, intensive, [], false, cribRegex);
+
+        // Filter down to results which matched the crib
+        if (cribRegex) {
+            options = options.filter(option => option.matchesCrib);
+        }
 
         // Record the current state for use when presenting
         this.state = state;

src/core/operations/PlayMedia.mjs

@@ -0,0 +1,102 @@
+/**
+ * @author anthony-arnold [anthony.arnold@uqconnect.edu.au]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import { fromBase64, toBase64 } from "../lib/Base64";
+import { fromHex } from "../lib/Hex";
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Utils from "../Utils";
+import Magic from "../lib/Magic";
+
+/**
+ * PlayMedia operation
+ */
+class PlayMedia extends Operation {
+
+    /**
+     * PlayMedia constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Play Media";
+        this.module = "Default";
+        this.description = "Plays the input as audio or video depending on the type.<br><br>Tags: sound, movie, mp3, mp4, mov, webm, wav, ogg";
+        this.infoURL = "";
+        this.inputType = "string";
+        this.outputType = "byteArray";
+        this.presentType = "html";
+        this.args = [
+            {
+                "name": "Input format",
+                "type": "option",
+                "value": ["Raw", "Base64", "Hex"]
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {byteArray} The multimedia data as bytes.
+     */
+    run(input, args) {
+        const [inputFormat] = args;
+
+        if (!input.length) return [];
+
+        // Convert input to raw bytes
+        switch (inputFormat) {
+            case "Hex":
+                input = fromHex(input);
+                break;
+            case "Base64":
+                // Don't trust the Base64 entered by the user.
+                // Unwrap it first, then re-encode later.
+                input = fromBase64(input, undefined, "byteArray");
+                break;
+            case "Raw":
+            default:
+                input = Utils.strToByteArray(input);
+                break;
+        }
+
+
+        // Determine file type
+        const type = Magic.magicFileType(input);
+        if (!(type && /^audio|video/.test(type.mime))) {
+            throw new OperationError("Invalid or unrecognised file type");
+        }
+
+        return input;
+    }
+
+    /**
+     * Displays an audio or video element that may be able to play the media
+     * file.
+     *
+     * @param data {byteArray} Data containing an audio or video file.
+     * @returns {string} Markup to display a media player.
+     */
+    async present(data) {
+        if (!data.length) return "";
+
+        const type = Magic.magicFileType(data);
+        const matches = /^audio|video/.exec(type.mime);
+        if (!matches) {
+            throw new OperationError("Invalid file type");
+        }
+        const dataURI = `data:${type.mime};base64,${toBase64(data)}`;
+        const element = matches[0];
+
+        let html = `<${element} src='${dataURI}' type='${type.mime}' controls>`;
+        html += "<p>Unsupported media type.</p>";
+        html += `</${element}>`;
+        return html;
+    }
+}
+
+export default PlayMedia;

test/index.mjs

@@ -80,6 +80,7 @@ import "./tests/operations/ToGeohash.mjs";
 import "./tests/operations/TranslateDateTimeFormat";
 import "./tests/operations/Magic";
 import "./tests/operations/ParseTLV";
+import "./tests/operations/Media";
 
 let allTestsPassing = true;
 const testStatusCounts = {