Merge pull request #2 from gchq/master

Current

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "5.7.3",
+  "version": "5.10.2",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

src/core/Utils.js

@@ -259,6 +259,22 @@ const Utils = {
     },
 
 
+    /**
+     * Escape a string containing regex control characters so that it can be safely
+     * used in a regex without causing unintended behaviours.
+     *
+     * @param {string} str
+     * @returns {string}
+     *
+     * @example
+     * // returns "\[example\]"
+     * Utils.escapeRegex("[example]");
+     */
+    escapeRegex: function(str) {
+        return str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
+    },
+
+
     /**
      * Expand an alphabet range string into a list of the characters in that range.
      *

src/core/config/Categories.js

@@ -126,6 +126,7 @@ const Categories = [
     {
         name: "Networking",
         ops: [
+            "HTTP request",
             "Strip HTTP headers",
             "Parse User Agent",
             "Parse IP range",
@@ -290,6 +291,8 @@ const Categories = [
             "Scan for Embedded Files",
             "Generate UUID",
             "Render Image",
+            "Remove EXIF",
+            "Extract EXIF",
             "Numberwang",
         ]
     },

src/core/config/OperationConfig.js

@@ -1919,7 +1919,7 @@ const OperationConfig = {
         args: []
     },
     "Find / Replace": {
-        description: "Replaces all occurrences of the first string with the second.<br><br>The three match options are only relevant to regex search strings.",
+        description: "Replaces all occurrences of the first string with the second.<br><br> Includes support for regular expressions (regex), simple strings and extended strings (which support \\n, \\r, \\t, \\b, \\f and escaped hex bytes using \\x notation, e.g. \\x00 for a null byte).",
         run: StrUtils.runFindReplace,
         manualBake: true,
         inputType: "string",
@@ -2231,7 +2231,7 @@ const OperationConfig = {
         ]
     },
     "From UNIX Timestamp": {
-        description: "Converts a UNIX timestamp to a datetime string.<br><br>e.g. <code>978346800</code> becomes <code>Mon 1 January 2001 11:00:00 UTC</code>",
+        description: "Converts a UNIX timestamp to a datetime string.<br><br>e.g. <code>978346800</code> becomes <code>Mon 1 January 2001 11:00:00 UTC</code><br><br>A UNIX timestamp is a 32-bit value representing the number of seconds since January 1, 1970 UTC (the UNIX epoch).",
         run: DateTime.runFromUnixTimestamp,
         inputType: "number",
         outputType: "string",
@@ -2244,7 +2244,7 @@ const OperationConfig = {
         ]
     },
     "To UNIX Timestamp": {
-        description: "Parses a datetime string in UTC and returns the corresponding UNIX timestamp.<br><br>e.g. <code>Mon 1 January 2001 11:00:00</code> becomes <code>978346800</code>",
+        description: "Parses a datetime string in UTC and returns the corresponding UNIX timestamp.<br><br>e.g. <code>Mon 1 January 2001 11:00:00</code> becomes <code>978346800</code><br><br>A UNIX timestamp is a 32-bit value representing the number of seconds since January 1, 1970 UTC (the UNIX epoch).",
         run: DateTime.runToUnixTimestamp,
         inputType: "string",
         outputType: "number",
@@ -2262,27 +2262,27 @@ const OperationConfig = {
         ]
     },
     "Windows Filetime to UNIX Timestamp":{
-        description: "Converts a Windows Filetime timestamp to a datetime format",
+        description: "Converts a Windows Filetime value to a UNIX timestamp.<br><br>A Windows Filetime is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 UTC.<br><br>A UNIX timestamp is a 32-bit value representing the number of seconds since January 1, 1970 UTC (the UNIX epoch).<br><br>This operation also supports UNIX timestamps in milliseconds, microseconds and nanoseconds.",
         run: DateTime.runFromFiletimeToUnix,
         inputType: "string",
         outputType: "string",
         args: [
             {
-                name: "Output Units",
+                name: "Output units",
-                type: "Option",
+                type: "option",
                 value: DateTime.UNITS
             }
         ]
     },
     "UNIX Timestamp to Windows Filetime":{
-        description: "Parses a datetime string in UTC and returns the corresponding Windows Filetime timestamp",
+        description: "Converts a UNIX timestamp to a Windows Filetime value.<br><br>A Windows Filetime is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 UTC.<br><br>A UNIX timestamp is a 32-bit value representing the number of seconds since January 1, 1970 UTC (the UNIX epoch).<br><br>This operation also supports UNIX timestamps in milliseconds, microseconds and nanoseconds.",
         run: DateTime.runToFiletimeFromUnix,
         inputType: "string",
         outputType: "string",
         args: [
             {
-                name: "Input Units",
+                name: "Input units",
-                type: "Option",
+                type: "option",
                 value: DateTime.UNITS
             }
         ]
@@ -3396,7 +3396,7 @@ const OperationConfig = {
             "<br><br>",
             "EXIF data from photos usually contains information about the image file itself as well as the device used to create it.",
         ].join("\n"),
-        run: Image.runEXIF,
+        run: Image.runExtractEXIF,
         inputType: "byteArray",
         outputType: "string",
         args: [],
@@ -3414,6 +3414,59 @@ const OperationConfig = {
             }
         ]
     },
+    "Remove EXIF": {
+        description: [
+            "Removes EXIF data from a JPEG image.",
+            "<br><br>",
+            "EXIF data embedded in photos usually contains information about the image file itself as well as the device used to create it.",
+        ].join("\n"),
+        run: Image.runRemoveEXIF,
+        inputType: "byteArray",
+        outputType: "byteArray",
+        args: []
+    },
+    "HTTP request": {
+        description: [
+            "Makes an HTTP request and returns the response.",
+            "<br><br>",
+            "This operation supports different HTTP verbs like GET, POST, PUT, etc.",
+            "<br><br>",
+            "You can add headers line by line in the format <code>Key: Value</code>",
+            "<br><br>",
+            "The status code of the response, along with a limited selection of exposed headers, can be viewed by checking the 'Show response metadata' option. Only a limited set of response headers are exposed by the browser for security reasons.",
+        ].join("\n"),
+        run: HTTP.runHTTPRequest,
+        inputType: "string",
+        outputType: "string",
+        manualBake: true,
+        args: [
+            {
+                name: "Method",
+                type: "option",
+                value: HTTP.METHODS,
+            },
+            {
+                name: "URL",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Headers",
+                type: "text",
+                value: "",
+            },
+            {
+                name: "Mode",
+                type: "option",
+                value: HTTP.MODE,
+            },
+            {
+                name: "Show response metadata",
+                type: "boolean",
+                value: false,
+            }
+        ]
+    },
 };
 
 export default OperationConfig;

src/core/lib/remove-exif.js

@@ -0,0 +1,153 @@
+/* piexifjs
+The MIT License (MIT)
+Copyright (c) 2014, 2015 hMatoba(https://github.com/hMatoba)
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+import Utils from "../Utils.js";
+
+// Param jpeg should be a binaryArray
+function removeEXIF(jpeg) {
+  // Convert binaryArray to char string
+  jpeg = Utils.byteArrayToChars(jpeg);
+  if (jpeg.slice(0, 2) != "\xff\xd8") {
+    throw ("Given data is not jpeg.");
+  }
+
+  var segments = splitIntoSegments(jpeg);
+  if (segments[1].slice(0, 2) == "\xff\xe1" &&
+    segments[1].slice(4, 10) == "Exif\x00\x00") {
+    segments = [segments[0]].concat(segments.slice(2));
+  } else if (segments[2].slice(0, 2) == "\xff\xe1" &&
+    segments[2].slice(4, 10) == "Exif\x00\x00") {
+    segments = segments.slice(0, 2).concat(segments.slice(3));
+  } else {
+    throw ("Exif not found.");
+  }
+
+  var new_data = segments.join("");
+
+  // Convert back to binaryArray
+  new_data = Utils.strToCharcode(new_data);
+
+  return new_data;
+};
+
+function splitIntoSegments(data) {
+  if (data.slice(0, 2) != "\xff\xd8") {
+    throw ("Given data isn't JPEG.");
+  }
+
+  var head = 2;
+  var segments = ["\xff\xd8"];
+  while (true) {
+    if (data.slice(head, head + 2) == "\xff\xda") {
+      segments.push(data.slice(head));
+      break;
+    } else {
+      var length = unpack(">H", data.slice(head + 2, head + 4))[0];
+      var endPoint = head + length + 2;
+      segments.push(data.slice(head, endPoint));
+      head = endPoint;
+    }
+
+    if (head >= data.length) {
+      throw ("Wrong JPEG data.");
+    }
+  }
+  return segments;
+}
+
+function unpack(mark, str) {
+  if (typeof(str) != "string") {
+    throw ("'unpack' error. Got invalid type argument.");
+  }
+  var l = 0;
+  for (var markPointer = 1; markPointer < mark.length; markPointer++) {
+    if (mark[markPointer].toLowerCase() == "b") {
+      l += 1;
+    } else if (mark[markPointer].toLowerCase() == "h") {
+      l += 2;
+    } else if (mark[markPointer].toLowerCase() == "l") {
+      l += 4;
+    } else {
+      throw ("'unpack' error. Got invalid mark.");
+    }
+  }
+
+  if (l != str.length) {
+    throw ("'unpack' error. Mismatch between symbol and string length. " + l + ":" + str.length);
+  }
+
+  var littleEndian;
+  if (mark[0] == "<") {
+    littleEndian = true;
+  } else if (mark[0] == ">") {
+    littleEndian = false;
+  } else {
+    throw ("'unpack' error.");
+  }
+  var unpacked = [];
+  var strPointer = 0;
+  var p = 1;
+  var val = null;
+  var c = null;
+  var length = null;
+  var sliced = "";
+
+  while (c = mark[p]) {
+    if (c.toLowerCase() == "b") {
+      length = 1;
+      sliced = str.slice(strPointer, strPointer + length);
+      val = sliced.charCodeAt(0);
+      if ((c == "b") && (val >= 0x80)) {
+        val -= 0x100;
+      }
+    } else if (c == "H") {
+      length = 2;
+      sliced = str.slice(strPointer, strPointer + length);
+      if (littleEndian) {
+        sliced = sliced.split("").reverse().join("");
+      }
+      val = sliced.charCodeAt(0) * 0x100 +
+        sliced.charCodeAt(1);
+    } else if (c.toLowerCase() == "l") {
+      length = 4;
+      sliced = str.slice(strPointer, strPointer + length);
+      if (littleEndian) {
+        sliced = sliced.split("").reverse().join("");
+      }
+      val = sliced.charCodeAt(0) * 0x1000000 +
+        sliced.charCodeAt(1) * 0x10000 +
+        sliced.charCodeAt(2) * 0x100 +
+        sliced.charCodeAt(3);
+      if ((c == "l") && (val >= 0x80000000)) {
+        val -= 0x100000000;
+      }
+    } else {
+      throw ("'unpack' error. " + c);
+    }
+
+    unpacked.push(val);
+    strPointer += length;
+    p += 1;
+  }
+
+  return unpacked;
+}
+
+export default removeEXIF;

src/core/operations/DateTime.js

@@ -81,11 +81,11 @@ const DateTime = {
 
 
     /**
-     * Converts a Windows FILETIME to Unix Epoch time.
+     * Windows Filetime to Unix Timestamp operation.
      *
      * @author bwhitn [brian.m.whitney@outlook.com]
      * @param {string} input
-     * @param {Object[]} args (not used)
+     * @param {Object[]} args
      * @returns {string}
      */
     runFromFiletimeToUnix: function(input, args) {
@@ -107,11 +107,11 @@ const DateTime = {
 
 
     /**
-     * Converts a Unix Epoch time to Windows FILETIME.
+     * Unix Timestamp to Windows Filetime operation.
      *
      * @author bwhitn [brian.m.whitney@outlook.com]
      * @param {string} input
-     * @param {Object[]} args (not used)
+     * @param {Object[]} args
      * @returns {string}
      */
     runToFiletimeFromUnix: function(input, args) {

src/core/operations/HTTP.js

@@ -12,6 +12,17 @@ import {UAS_parser as UAParser} from "../lib/uas_parser.js";
  */
 const HTTP = {
 
+    /**
+     * @constant
+     * @default
+     */
+    METHODS: [
+        "GET", "POST", "HEAD",
+        "PUT", "PATCH", "DELETE",
+        "CONNECT", "TRACE", "OPTIONS"
+    ],
+
+
     /**
      * Strip HTTP headers operation.
      *
@@ -51,6 +62,95 @@ const HTTP = {
             "Device Type: " + ua.deviceType + "\n";
     },
 
+
+    /**
+     * @constant
+     * @default
+     */
+    MODE: [
+        "Cross-Origin Resource Sharing",
+        "No CORS (limited to HEAD, GET or POST)",
+    ],
+
+    /**
+     * Lookup table for HTTP modes
+     *
+     * @private
+     * @constant
+     */
+    _modeLookup: {
+        "Cross-Origin Resource Sharing": "cors",
+        "No CORS (limited to HEAD, GET or POST)": "no-cors",
+    },
+
+    /**
+     * HTTP request operation.
+     *
+     * @author tlwr [toby@toby.codes]
+     * @author n1474335 [n1474335@gmail.com]
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    runHTTPRequest(input, args) {
+        const method = args[0],
+            url = args[1],
+            headersText = args[2],
+            mode = args[3],
+            showResponseMetadata = args[4];
+
+        if (url.length === 0) return "";
+
+        let headers = new Headers();
+        headersText.split(/\r?\n/).forEach(line => {
+            line = line.trim();
+
+            if (line.length === 0) return;
+
+            let split = line.split(":");
+            if (split.length !== 2) throw `Could not parse header in line: ${line}`;
+
+            headers.set(split[0].trim(), split[1].trim());
+        });
+
+        let config = {
+            method: method,
+            headers: headers,
+            mode: HTTP._modeLookup[mode],
+            cache: "no-cache",
+        };
+
+        if (method !== "GET" && method !== "HEAD") {
+            config.body = input;
+        }
+
+        return fetch(url, config)
+        .then(r => {
+            if (r.status === 0 && r.type === "opaque") {
+                return "Error: Null response. Try setting the connection mode to CORS.";
+            }
+
+            if (showResponseMetadata) {
+                let headers = "";
+                for (let pair of r.headers.entries()) {
+                    headers += "    " + pair[0] + ": " + pair[1] + "\n";
+                }
+                return r.text().then(b => {
+                    return "####\n  Status: " + r.status + " " + r.statusText +
+                        "\n  Exposed headers:\n" + headers + "####\n\n" + b;
+                });
+            }
+            return r.text();
+        })
+        .catch(e => {
+            return e.toString() +
+                "\n\nThis error could be caused by one of the following:\n" +
+                " - An invalid URL\n" +
+                " - Making a request to an insecure resource (HTTP) from a secure source (HTTPS)\n" +
+                " - Making a cross-origin request to a server which does not support CORS\n";
+        });
+    },
+
 };
 
 export default HTTP;

src/core/operations/Image.js

@@ -1,4 +1,5 @@
 import * as ExifParser from "exif-parser";
+import removeEXIF from "../lib/remove-exif.js";
 import Utils from "../Utils.js";
 import FileType from "./FileType.js";
 
@@ -23,7 +24,7 @@ const Image = {
      * @param {Object[]} args
      * @returns {string}
      */
-    runEXIF(input, args) {
+    runExtractEXIF(input, args) {
         try {
             const bytes = Uint8Array.from(input);
             const parser = ExifParser.create(bytes.buffer);
@@ -44,6 +45,30 @@ const Image = {
     },
 
 
+    /**
+     * Remove EXIF operation.
+     *
+     * Removes EXIF data from a byteArray, representing a JPG.
+     *
+     * @author David Moodie [davidmoodie12@gmail.com]
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    runRemoveEXIF(input, args) {
+        // Do nothing if input is empty
+        if (input.length === 0) return input;
+
+        try {
+            return removeEXIF(input);
+        } catch (err) {
+            // Simply return input if no EXIF data is found
+            if (err === "Exif not found.") return input;
+            throw "Could not remove EXIF data from image: " + err;
+        }
+    },
+
+
     /**
      * @constant
      * @default

src/core/operations/StrUtils.js

@@ -227,14 +227,16 @@ const StrUtils = {
 
         if (type === "Regex") {
             find = new RegExp(find, modifiers);
-        } else if (type.indexOf("Extended") === 0) {
+            return input.replace(find, replace);
+        }
+
+        if (type.indexOf("Extended") === 0) {
             find = Utils.parseEscapedChars(find);
         }
 
-        return input.replace(find, replace, modifiers);
+        find = new RegExp(Utils.escapeRegex(find), modifiers);
-        // Non-standard addition of flags in the third argument. This will work in Firefox but
+
-        // probably nowhere else. The purpose is to allow global matching when the `find` parameter
+        return input.replace(find, replace);
-        // is just a string.
     },
 
 

src/web/App.js

@@ -21,21 +21,22 @@ import Split from "split.js";
  * @param {Object} options - Default setting for app options.
  */
 const App = function(categories, operations, defaultFavourites, defaultOptions) {
-    this.categories  = categories;
+    this.categories    = categories;
-    this.operations  = operations;
+    this.operations    = operations;
-    this.dfavourites = defaultFavourites;
+    this.dfavourites   = defaultFavourites;
-    this.doptions    = defaultOptions;
+    this.doptions      = defaultOptions;
-    this.options     = Utils.extend({}, defaultOptions);
+    this.options       = Utils.extend({}, defaultOptions);
 
-    this.chef        = new Chef();
+    this.chef          = new Chef();
-    this.manager     = new Manager(this);
+    this.manager       = new Manager(this);
 
-    this.baking      = false;
+    this.baking        = false;
-    this.autoBake_   = false;
+    this.autoBake_     = false;
-    this.progress    = 0;
+    this.autoBakePause = false;
-    this.ingId       = 0;
+    this.progress      = 0;
+    this.ingId         = 0;
 
-    window.chef      = this.chef;
+    window.chef        = this.chef;
 };
 
 
@@ -166,7 +167,7 @@ App.prototype.bake = async function(step) {
  * Runs Auto Bake if it is set.
  */
 App.prototype.autoBake = function() {
-    if (this.autoBake_) {
+    if (this.autoBake_ && !this.autoBakePause) {
         this.bake();
     }
 };
@@ -413,9 +414,9 @@ App.prototype.loadURIParams = function() {
         return b;
     })(window.location.search.substr(1).split("&"));
 
-    // Turn off auto-bake while loading
+    // Pause auto-bake while loading but don't modify `this.autoBake_`
-    const autoBakeVal = this.autoBake_;
+    // otherwise `manualBake` cannot trigger.
-    this.autoBake_ = false;
+    this.autoBakePause = true;
 
     // Read in recipe from query string
     if (this.queryString.recipe) {
@@ -451,8 +452,8 @@ App.prototype.loadURIParams = function() {
         } catch (err) {}
     }
 
-    // Restore auto-bake state
+    // Unpause auto-bake
-    this.autoBake_ = autoBakeVal;
+    this.autoBakePause = false;
     this.autoBake();
 };
 

src/web/RecipeWaiter.js

@@ -354,8 +354,7 @@ RecipeWaiter.prototype.buildRecipeOperation = function(el) {
         el.classList.add("flow-control-op");
     }
 
-    // Disable auto-bake if this is a manual op - this should be moved to the 'operationadd'
+    // Disable auto-bake if this is a manual op
-    // handler after event restructuring
     if (op.manualBake && this.app.autoBake_) {
         this.manager.controls.setAutoBake(false);
         this.app.alert("Auto-Bake is disabled by default when using this operation.", "info", 5000);

src/web/html/index.html

@@ -35,7 +35,7 @@
             // Load theme before the preloader is shown
             document.querySelector(":root").className = JSON.parse(localStorage.getItem("options")).theme;
 
-            // Cycle loading messages
+            // Define loading messages
             const loadingMsgs = [
                 "Proving P = NP...",
                 "Computing 6 x 9...",
@@ -49,15 +49,28 @@
                 "Navigating neural network...",
                 "Importing machine learning..."
             ];
+
+            // Shuffle array using Durstenfeld algorithm
+            for (let i = loadingMsgs.length - 1; i > 0; --i) {
+                const j = Math.floor(Math.random() * (i + 1));
+                const temp = loadingMsgs[i];
+                loadingMsgs[i] = loadingMsgs[j];
+                loadingMsgs[j] = temp;
+            }
+
+            // Show next loading message then move it to the end of the array
             function changeLoadingMsg() {
+                const msg = loadingMsgs.shift();
                 try {
                     const el = document.getElementById("preloader-msg");
                     el.className = "loading"; // Causes CSS transition on first message
-                    el.innerHTML = loadingMsgs[Math.floor(Math.random()*loadingMsgs.length)];
+                    el.innerHTML = msg;
-                } catch (err) {}
+                } catch (err) {} // Ignore errors if DOM not yet ready
+                loadingMsgs.push(msg);
             }
+
             changeLoadingMsg();
-            window.loadingMsgsInt = setInterval(changeLoadingMsg, (Math.random()*500) + 500);
+            window.loadingMsgsInt = setInterval(changeLoadingMsg, (Math.random() * 1000) + 1000);
         </script>
     </head>
     <body>