Merge remote-tracking branch 'upstream/master' into features/yara

src/core/operations/FromCaseInsensitiveRegex.mjs

@@ -0,0 +1,39 @@
+/**
+ * @author masq [github.cyberchef@masq.cc]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+
+/**
+ * From Case Insensitive Regex operation
+ */
+class FromCaseInsensitiveRegex extends Operation {
+
+    /**
+     * FromCaseInsensitiveRegex constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "From Case Insensitive Regex";
+        this.module = "Default";
+        this.description = "Converts a case-insensitive regex string to a case sensitive regex string (no guarantee on it being the proper original casing) in case the i flag wasn't available at the time but now is, or you need it to be case-sensitive again.<br><br>e.g. <code>[mM][oO][zZ][iI][lL][lL][aA]/[0-9].[0-9] .*</code> becomes <code>Mozilla/[0-9].[0-9] .*</code>";
+        this.infoURL = "https://wikipedia.org/wiki/Regular_expression";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        return input.replace(/\[[a-z]{2}\]/ig, m => m[1].toUpperCase() === m[2].toUpperCase() ? m[1] : m);
+    }
+}
+
+export default FromCaseInsensitiveRegex;

src/core/operations/GenerateLoremIpsum.mjs

@@ -0,0 +1,70 @@
+/**
+ * @author klaxon [klaxon@veyr.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import { GenerateParagraphs, GenerateSentences, GenerateWords, GenerateBytes } from "../lib/LoremIpsum";
+
+/**
+ * Generate Lorem Ipsum operation
+ */
+class GenerateLoremIpsum extends Operation {
+
+    /**
+     * GenerateLoremIpsum constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Generate Lorem Ipsum";
+        this.module = "Default";
+        this.description = "Generate varying length lorem ipsum placeholder text.";
+        this.infoURL = "https://wikipedia.org/wiki/Lorem_ipsum";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Length",
+                "type": "number",
+                "value": "3"
+            },
+            {
+                "name": "Length in",
+                "type": "option",
+                "value": ["Paragraphs", "Sentences", "Words", "Bytes"]
+            }
+
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [length, lengthType] = args;
+        if (length < 1){
+            throw new OperationError("Length must be greater than 0");
+        }
+        switch (lengthType) {
+            case "Paragraphs":
+                return GenerateParagraphs(length);
+            case "Sentences":
+                return GenerateSentences(length);
+            case "Words":
+                return GenerateWords(length);
+            case "Bytes":
+                return GenerateBytes(length);
+            default:
+                throw new OperationError("Invalid length type");
+
+        }
+    }
+
+}
+
+export default GenerateLoremIpsum;

src/core/operations/RegularExpression.mjs

@@ -228,40 +228,29 @@ function regexList (input, regex, displayTotal, matches, captureGroups) {
 function regexHighlight (input, regex, displayTotal) {
     let output = "",
         title = "",
-        m,
         hl = 1,
-        i = 0,
         total = 0;
 
-    while ((m = regex.exec(input))) {
-        // Moves pointer when an empty string is matched (prevents infinite loop)
-        if (m.index === regex.lastIndex) {
-            regex.lastIndex++;
-        }
+    output = input.replace(regex, (match, ...args) => {
+        args.pop(); // Throw away full string
+        const offset = args.pop(),
+            groups = args;
 
-        // Add up to match
-        output += Utils.escapeHtml(input.slice(i, m.index));
-
-        title = `Offset: ${m.index}\n`;
-        if (m.length > 1) {
+        title = `Offset: ${offset}\n`;
+        if (groups.length) {
             title += "Groups:\n";
-            for (let n = 1; n < m.length; ++n) {
-                title += `\t${n}: ${m[n]}\n`;
+            for (let i = 0; i < groups.length; i++) {
+                title += `\t${i+1}: ${Utils.escapeHtml(groups[i])}\n`;
             }
         }
 
-        // Add match with highlighting
-        output += "<span class='hl"+hl+"' title='"+title+"'>" + Utils.escapeHtml(m[0]) + "</span>";
-
         // Switch highlight
         hl = hl === 1 ? 2 : 1;
 
-        i = regex.lastIndex;
         total++;
-    }
 
-    // Add all after final match
-    output += Utils.escapeHtml(input.slice(i, input.length));
+        return `<span class='hl${hl}' title='${title}'>${Utils.escapeHtml(match)}</span>`;
+    });
 
     if (displayTotal)
         output = "Total found: " + total + "\n\n" + output;

src/core/operations/Subsection.mjs

@@ -0,0 +1,153 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import XRegExp from "xregexp";
+import Operation from "../Operation";
+import Recipe from "../Recipe";
+import Dish from "../Dish";
+
+/**
+ * Subsection operation
+ */
+class Subsection extends Operation {
+
+    /**
+     * Subsection constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Subsection";
+        this.flowControl = true;
+        this.module = "Regex";
+        this.description = "Select a part of the input data using a regular expression (regex), and run all subsequent operations on each match separately.<br><br>You can use up to one capture group, where the recipe will only be run on the data in the capture group. If there's more than one capture group, only the first one will be operated on.";
+        this.infoURL = "";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Section (regex)",
+                "type": "string",
+                "value": ""
+            },
+            {
+                "name": "Case sensitive matching",
+                "type": "boolean",
+                "value": true
+            },
+            {
+                "name": "Global matching",
+                "type": "boolean",
+                "value": true
+            },
+            {
+                "name": "Ignore errors",
+                "type": "boolean",
+                "value": false
+            }
+        ];
+    }
+
+    /**
+     * @param {Object} state - The current state of the recipe.
+     * @param {number} state.progress - The current position in the recipe.
+     * @param {Dish} state.dish - The Dish being operated on
+     * @param {Operation[]} state.opList - The list of operations in the recipe
+     * @returns {Object} - The updated state of the recipe
+     */
+    async run(state) {
+        const opList    = state.opList,
+            inputType   = opList[state.progress].inputType,
+            outputType  = opList[state.progress].outputType,
+            input       = await state.dish.get(inputType),
+            ings        = opList[state.progress].ingValues,
+            [section, caseSensitive, global, ignoreErrors] = ings,
+            subOpList   = [];
+
+        if (input && section !== "") {
+            // Create subOpList for each tranche to operate on
+            // all remaining operations unless we encounter a Merge
+            for (let i = state.progress + 1; i < opList.length; i++) {
+                if (opList[i].name === "Merge" && !opList[i].disabled) {
+                    break;
+                } else {
+                    subOpList.push(opList[i]);
+                }
+            }
+
+            let flags = "",
+                inOffset = 0,
+                output = "",
+                m,
+                progress = 0;
+
+            if (!caseSensitive) flags += "i";
+            if (global) flags += "g";
+
+            const regex = new XRegExp(section, flags),
+                recipe = new Recipe();
+
+            recipe.addOperations(subOpList);
+            state.forkOffset += state.progress + 1;
+
+            // Take a deep(ish) copy of the ingredient values
+            const ingValues = subOpList.map(op => JSON.parse(JSON.stringify(op.ingValues)));
+            let matched = false;
+
+            // Run recipe over each match
+            while ((m = regex.exec(input))) {
+                matched = true;
+                // Add up to match
+                let matchStr = m[0];
+
+                if (m.length === 1) { // No capture groups
+                    output += input.slice(inOffset, m.index);
+                    inOffset = m.index + m[0].length;
+                } else if (m.length >= 2) {
+                    matchStr = m[1];
+
+                    // Need to add some of the matched string that isn't in the capture group
+                    output += input.slice(inOffset, m.index + m[0].indexOf(m[1]));
+                    // Set i to be after the end of the first capture group
+                    inOffset = m.index + m[0].indexOf(m[1]) + m[1].length;
+                }
+
+                // Baseline ing values for each tranche so that registers are reset
+                subOpList.forEach((op, i) => {
+                    op.ingValues = JSON.parse(JSON.stringify(ingValues[i]));
+                });
+
+                const dish = new Dish();
+                dish.set(matchStr, inputType);
+
+                try {
+                    progress = await recipe.execute(dish, 0, state);
+                } catch (err) {
+                    if (!ignoreErrors) {
+                        throw err;
+                    }
+                    progress = err.progress + 1;
+                }
+                output += await dish.get(outputType);
+                if (!regex.global) break;
+            }
+
+            // If no matches were found, advance progress to after a Merge op
+            // Otherwise, the operations below Subsection will be run on all the input data
+            if (!matched) {
+                state.progress += subOpList.length + 1;
+            }
+
+            output += input.slice(inOffset);
+            state.progress += progress;
+            state.dish.set(output, outputType);
+        }
+        return state;
+    }
+
+}
+
+export default Subsection;

src/core/operations/ToCaseInsensitiveRegex.mjs

@@ -0,0 +1,39 @@
+/**
+ * @author masq [github.cyberchef@masq.cc]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+
+/**
+ * To Case Insensitive Regex operation
+ */
+class ToCaseInsensitiveRegex extends Operation {
+
+    /**
+     * ToCaseInsensitiveRegex constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "To Case Insensitive Regex";
+        this.module = "Default";
+        this.description = "Converts a case-sensitive regex string into a case-insensitive regex string in case the i flag is unavailable to you.<br><br>e.g. <code>Mozilla/[0-9].[0-9] .*</code> becomes <code>[mM][oO][zZ][iI][lL][lL][aA]/[0-9].[0-9] .*</code>";
+        this.infoURL = "https://wikipedia.org/wiki/Regular_expression";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        return input.replace(/[a-z]/ig, m => `[${m.toLowerCase()}${m.toUpperCase()}]`);
+    }
+}
+
+export default ToCaseInsensitiveRegex;