Merge branch 'n1073645-newMagic'

src/core/Utils.mjs

@@ -1182,6 +1182,7 @@ class Utils {
             "CRLF":          /\r\n/g,
             "Forward slash": /\//g,
             "Backslash":     /\\/g,
+            "0x with comma": /,?0x/g,
             "0x":            /0x/g,
             "\\x":           /\\x/g,
             "None":          /\s+/g // Included here to remove whitespace when there shouldn't be any

src/core/config/scripts/generateConfig.mjs

@@ -42,13 +42,10 @@ for (const opObj in Ops) {
         outputType:  op.presentType,
         flowControl: op.flowControl,
         manualBake:  op.manualBake,
-        args:        op.args
+        args:        op.args,
+        checks:      op.checks
     };
 
-    if ("patterns" in op) {
-        operationConfig[op.name].patterns = op.patterns;
-    }
-
     if (!(op.module in modules))
         modules[op.module] = {};
     modules[op.module][op.name] = opObj;

src/core/lib/IP.mjs

@@ -26,7 +26,7 @@ export function ipv4CidrRange(cidr, includeNetworkInfo, enumerateAddresses, allo
     let output = "";
 
     if (cidrRange < 0 || cidrRange > 31) {
-        return "IPv4 CIDR must be less than 32";
+        throw new OperationError("IPv4 CIDR must be less than 32");
     }
 
     const mask = ~(0xFFFFFFFF >>> cidrRange),
@@ -64,7 +64,7 @@ export function ipv6CidrRange(cidr, includeNetworkInfo) {
         cidrRange = parseInt(cidr[cidr.length-1], 10);
 
     if (cidrRange < 0 || cidrRange > 127) {
-        return "IPv6 CIDR must be less than 128";
+        throw new OperationError("IPv6 CIDR must be less than 128");
     }
 
     const ip1 = new Array(8),
@@ -211,7 +211,7 @@ export function ipv4ListedRange(match, includeNetworkInfo, enumerateAddresses, a
         const network = strToIpv4(ipv4CidrList[i].split("/")[0]);
         const cidrRange = parseInt(ipv4CidrList[i].split("/")[1], 10);
         if (cidrRange < 0 || cidrRange > 31) {
-            return "IPv4 CIDR must be less than 32";
+            throw new OperationError("IPv4 CIDR must be less than 32");
         }
         const mask = ~(0xFFFFFFFF >>> cidrRange),
             cidrIp1 = network & mask,
@@ -254,7 +254,7 @@ export function ipv6ListedRange(match, includeNetworkInfo) {
         const cidrRange = parseInt(ipv6CidrList[i].split("/")[1], 10);
 
         if (cidrRange < 0 || cidrRange > 127) {
-            return "IPv6 CIDR must be less than 128";
+            throw new OperationError("IPv6 CIDR must be less than 128");
         }
 
         const cidrIp1 = new Array(8),

src/core/lib/Magic.mjs

@@ -2,7 +2,7 @@ import OperationConfig from "../config/OperationConfig.json";
 import Utils, { isWorkerEnvironment } from "../Utils.mjs";
 import Recipe from "../Recipe.mjs";
 import Dish from "../Dish.mjs";
-import {detectFileType} from "./FileType.mjs";
+import {detectFileType, isType} from "./FileType.mjs";
 import chiSquared from "chi-squared";
 
 /**
@@ -19,31 +19,38 @@ class Magic {
      * Magic constructor.
      *
      * @param {ArrayBuffer} buf
-     * @param {Object[]} [opPatterns]
+     * @param {Object[]} [opCriteria]
+     * @param {Object} [prevOp]
      */
-    constructor(buf, opPatterns) {
+    constructor(buf, opCriteria=Magic._generateOpCriteria(), prevOp=null) {
         this.inputBuffer = new Uint8Array(buf);
         this.inputStr = Utils.arrayBufferToStr(buf);
-        this.opPatterns = opPatterns || Magic._generateOpPatterns();
+        this.opCriteria = opCriteria;
+        this.prevOp = prevOp;
     }
 
     /**
-     * Finds operations that claim to be able to decode the input based on regular
+     * Finds operations that claim to be able to decode the input based on various criteria.
-     * expression matches.
      *
      * @returns {Object[]}
      */
-    findMatchingOps() {
+    findMatchingInputOps() {
-        const matches = [];
+        const matches = [],
+            inputEntropy = this.calcEntropy();
 
-        for (let i = 0; i < this.opPatterns.length; i++) {
+        this.opCriteria.forEach(check => {
-            const pattern = this.opPatterns[i],
+            // If the input doesn't lie in the required entropy range, move on
-                regex = new RegExp(pattern.match, pattern.flags);
+            if (check.entropyRange &&
+                (inputEntropy < check.entropyRange[0] ||
+                inputEntropy > check.entropyRange[1]))
+                return;
+            // If the input doesn't match the pattern, move on
+            if (check.pattern &&
+                !check.pattern.test(this.inputStr))
+                return;
 
-            if (regex.test(this.inputStr)) {
+            matches.push(check);
-                matches.push(pattern);
+        });
-            }
-        }
 
         return matches;
     }
@@ -185,8 +192,10 @@ class Magic {
      *
      * @returns {number}
      */
-    calcEntropy() {
+    calcEntropy(data=this.inputBuffer, standalone=false) {
-        const prob = this._freqDist();
+        if (!standalone && this.inputEntropy) return this.inputEntropy;
+
+        const prob = this._freqDist(data, standalone);
         let entropy = 0,
             p;
 
@@ -195,6 +204,8 @@ class Magic {
             if (p === 0) continue;
             entropy += p * Math.log(p) / Math.log(2);
         }
+
+        if (!standalone) this.inputEntropy = -entropy;
         return -entropy;
     }
 
@@ -264,6 +275,32 @@ class Magic {
         return results;
     }
 
+    /**
+     * Checks whether the data passes output criteria for an operation check
+     *
+     * @param {ArrayBuffer} data
+     * @param {Object} criteria
+     * @returns {boolean}
+     */
+    outputCheckPasses(data, criteria) {
+        if (criteria.pattern) {
+            const dataStr = Utils.arrayBufferToStr(data),
+                regex = new RegExp(criteria.pattern, criteria.flags);
+            if (!regex.test(dataStr))
+                return false;
+        }
+        if (criteria.entropyRange) {
+            const dataEntropy = this.calcEntropy(data, true);
+            if (dataEntropy < criteria.entropyRange[0] || dataEntropy > criteria.entropyRange[1])
+                return false;
+        }
+        if (criteria.mime &&
+            !isType(criteria.mime, data))
+            return false;
+
+        return true;
+    }
+
     /**
      * Speculatively executes matching operations, recording metadata of each result.
      *
@@ -274,15 +311,23 @@ class Magic {
      *     performance)
      * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point
      * @param {boolean} [useful=false] - Whether the current recipe should be scored highly
-     * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation output
+     * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation
+     *     output
      * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding
      */
-    async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, crib=null) {
+    async speculativeExecution(
+        depth=0,
+        extLang=false,
+        intensive=false,
+        recipeConfig=[],
+        useful=false,
+        crib=null) {
+
+        // If we have reached the recursion depth, return
         if (depth < 0) return [];
 
         // Find any operations that can be run on this data
-        const matchingOps = this.findMatchingOps();
+        const matchingOps = this.findMatchingInputOps();
-
         let results = [];
 
         // Record the properties of the current data
@@ -308,17 +353,21 @@ class Magic {
                 },
                 output = await this._runRecipe([opConfig]);
 
-            // If the recipe is repeating and returning the same data, do not continue
-            if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
-                return;
-            }
-
             // If the recipe returned an empty buffer, do not continue
             if (_buffersEqual(output, new ArrayBuffer())) {
                 return;
             }
 
-            const magic = new Magic(output, this.opPatterns),
+            // If the recipe is repeating and returning the same data, do not continue
+            if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
+                return;
+            }
+
+            // If the output criteria for this op doesn't match the output, do not continue
+            if (op.output && !this.outputCheckPasses(output, op.output))
+                return;
+
+            const magic = new Magic(output, this.opCriteria, OperationConfig[op.op]),
                 speculativeResults = await magic.speculativeExecution(
                     depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);
 
@@ -330,7 +379,7 @@ class Magic {
             const bfEncodings = await this.bruteForce();
 
             await Promise.all(bfEncodings.map(async enc => {
-                const magic = new Magic(enc.data, this.opPatterns),
+                const magic = new Magic(enc.data, this.opCriteria, undefined),
                     bfResults = await magic.speculativeExecution(
                         depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib);
 
@@ -345,7 +394,8 @@ class Magic {
                 r.languageScores[0].probability > 0 ||    // Some kind of language was found
                 r.fileType ||                             // A file was found
                 r.isUTF8 ||                               // UTF-8 was found
-                r.matchingOps.length                      // A matching op was found
+                r.matchingOps.length ||                   // A matching op was found
+                r.matchesCrib                             // The crib matches
             )
         );
 
@@ -376,9 +426,10 @@ class Magic {
             bScore += b.entropy;
 
             // A result with no recipe but matching ops suggests there are better options
-            if ((!a.recipe.length && a.matchingOps.length) &&
+            if ((!a.recipe.length && a.matchingOps.length) && b.recipe.length)
-                b.recipe.length)
                 return 1;
+            if ((!b.recipe.length && b.matchingOps.length) && a.recipe.length)
+                return -1;
 
             return aScore - bScore;
         });
@@ -417,14 +468,16 @@ class Magic {
      * Calculates the number of times each byte appears in the input as a percentage
      *
      * @private
+     * @param {ArrayBuffer} [data]
+     * @param {boolean} [standalone]
      * @returns {number[]}
      */
-    _freqDist() {
+    _freqDist(data=this.inputBuffer, standalone=false) {
-        if (this.freqDist) return this.freqDist;
+        if (!standalone && this.freqDist) return this.freqDist;
 
-        const len = this.inputBuffer.length;
+        const len = data.length,
+            counts = new Array(256).fill(0);
         let i = len;
-        const counts = new Array(256).fill(0);
 
         if (!len) {
             this.freqDist = counts;
@@ -432,13 +485,15 @@ class Magic {
         }
 
         while (i--) {
-            counts[this.inputBuffer[i]]++;
+            counts[data[i]]++;
         }
 
-        this.freqDist = counts.map(c => {
+        const result = counts.map(c => {
             return c / len * 100;
         });
-        return this.freqDist;
+
+        if (!standalone) this.freqDist = result;
+        return result;
     }
 
     /**
@@ -447,24 +502,29 @@ class Magic {
      * @private
      * @returns {Object[]}
      */
-    static _generateOpPatterns() {
+    static _generateOpCriteria() {
-        const opPatterns = [];
+        const opCriteria = [];
 
         for (const op in OperationConfig) {
-            if (!("patterns" in OperationConfig[op])) continue;
+            if (!("checks" in OperationConfig[op]))
+                continue;
 
-            OperationConfig[op].patterns.forEach(pattern => {
+            OperationConfig[op].checks.forEach(check => {
-                opPatterns.push({
+                // Add to the opCriteria list.
+                // Compile the regex here and cache the compiled version so we
+                // don't have to keep calculating it.
+                opCriteria.push({
                     op: op,
-                    match: pattern.match,
+                    pattern: check.pattern ? new RegExp(check.pattern, check.flags) : null,
-                    flags: pattern.flags,
+                    args: check.args,
-                    args: pattern.args,
+                    useful: check.useful,
-                    useful: pattern.useful || false
+                    entropyRange: check.entropyRange,
+                    output: check.output
                 });
             });
         }
 
-        return opPatterns;
+        return opCriteria;
     }
 
     /**

src/core/operations/A1Z26CipherDecode.mjs

@@ -33,6 +33,38 @@ class A1Z26CipherDecode extends Operation {
                 value: DELIM_OPTIONS
             }
         ];
+        this.checks = [
+            {
+                pattern:  "^\\s*([12]?[0-9] )+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Space"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9],)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Comma"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9];)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Semi-colon"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9]:)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Colon"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9]\\n)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Line feed"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9]\\r\\n)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["CRLF"]
+            }
+        ];
     }
 
     /**

src/core/operations/BaconCipherDecode.mjs

@@ -44,6 +44,48 @@ class BaconCipherDecode extends Operation {
                 "value": false
             }
         ];
+        this.checks = [
+            {
+                pattern:  "^\\s*([01]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Standard (I=J and U=V)", "0/1", false]
+            },
+            {
+                pattern:  "^\\s*([01]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Standard (I=J and U=V)", "0/1", true]
+            },
+            {
+                pattern:  "^\\s*([AB]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Standard (I=J and U=V)", "A/B", false]
+            },
+            {
+                pattern:  "^\\s*([AB]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Standard (I=J and U=V)", "A/B", true]
+            },
+            {
+                pattern:  "^\\s*([01]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Complete", "0/1", false]
+            },
+            {
+                pattern:  "^\\s*([01]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Complete", "0/1", true]
+            },
+            {
+                pattern:  "^\\s*([AB]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Complete", "A/B", false]
+            },
+            {
+                pattern:  "^\\s*([AB]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Complete", "A/B", true]
+            }
+        ];
     }
 
     /**

src/core/operations/Bzip2Decompress.mjs

@@ -33,9 +33,9 @@ class Bzip2Decompress extends Operation {
                 value: false
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                "match": "^\\x42\\x5a\\x68",
+                "pattern": "^\\x42\\x5a\\x68",
                 "flags": "",
                 "args": []
             }

src/core/operations/DechunkHTTPResponse.mjs

@@ -24,6 +24,13 @@ class DechunkHTTPResponse extends Operation {
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];
+        this.checks = [
+            {
+                pattern:  "^[0-9A-F]+\r\n",
+                flags:  "i",
+                args:   []
+            }
+        ];
     }
 
     /**

src/core/operations/DecodeNetBIOSName.mjs

@@ -30,6 +30,13 @@ class DecodeNetBIOSName extends Operation {
                 "value": 65
             }
         ];
+        this.checks = [
+            {
+                pattern:  "^\\s*\\S{32}$",
+                flags:  "",
+                args:   [65]
+            }
+        ];
     }
 
     /**

src/core/operations/DefangIPAddresses.mjs

@@ -25,7 +25,17 @@ class DefangIPAddresses extends Operation {
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];
-
+        this.checks = [
+            {
+                pattern: "^\\s*(([0-9]{1,3}\\.){3}[0-9]{1,3}|([0-9a-f]{4}:){7}[0-9a-f]{4})\\s*$",
+                flags: "i",
+                args: [],
+                output: {
+                    pattern: "^\\s*(([0-9]{1,3}\\[\\.\\]){3}[0-9]{1,3}|([0-9a-f]{4}\\[\\:\\]){7}[0-9a-f]{4})\\s*$",
+                    flags: "i"
+                }
+            }
+        ];
     }
 
     /**

src/core/operations/EscapeUnicodeCharacters.mjs

@@ -44,22 +44,22 @@ class EscapeUnicodeCharacters extends Operation {
                 "value": true
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "\\\\u(?:[\\da-f]{4,6})",
+                pattern: "\\\\u(?:[\\da-f]{4,6})",
                 flags: "i",
                 args: ["\\u"]
             },
             {
-                match: "%u(?:[\\da-f]{4,6})",
+                pattern: "%u(?:[\\da-f]{4,6})",
                 flags: "i",
                 args: ["%u"]
             },
             {
-                match: "U\\+(?:[\\da-f]{4,6})",
+                pattern: "U\\+(?:[\\da-f]{4,6})",
                 flags: "i",
                 args: ["U+"]
-            },
+            }
         ];
     }
 

src/core/operations/FromBCD.mjs

@@ -49,9 +49,9 @@ class FromBCD extends Operation {
                 "value": FORMAT
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^(?:\\d{4} ){3,}\\d{4}$",
+                pattern: "^(?:\\d{4} ){3,}\\d{4}$",
                 flags: "",
                 args: ["8 4 2 1", true, false, "Nibbles"]
             },

src/core/operations/FromBase32.mjs

@@ -36,12 +36,12 @@ class FromBase32 extends Operation {
                 value: true
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$",
+                pattern: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$",
                 flags: "",
                 args: ["A-Z2-7=", false]
-            },
+            }
         ];
     }
 

src/core/operations/FromBase58.mjs

@@ -38,14 +38,14 @@ class FromBase58 extends Operation {
                 "value": true
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
+                pattern: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
                 flags: "",
                 args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", false]
             },
             {
-                match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
+                pattern: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
                 flags: "",
                 args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", false]
             },

src/core/operations/FromBase64.mjs

@@ -36,69 +36,69 @@ class FromBase64 extends Operation {
                 value: true
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^\\s*(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                 flags: "i",
                 args: ["A-Za-z0-9+/=", true]
             },
             {
-                match: "^\\s*[A-Z\\d\\-_]{20,}\\s*$",
+                pattern: "^\\s*[A-Z\\d\\-_]{20,}\\s*$",
                 flags: "i",
                 args: ["A-Za-z0-9-_", true]
             },
             {
-                match: "^\\s*(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?\\s*$",
                 flags: "i",
                 args: ["A-Za-z0-9+\\-=", true]
             },
             {
-                match: "^\\s*(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?\\s*$",
                 flags: "i",
                 args: ["./0-9A-Za-z=", true]
             },
             {
-                match: "^\\s*[A-Z\\d_.]{20,}\\s*$",
+                pattern: "^\\s*[A-Z\\d_.]{20,}\\s*$",
                 flags: "i",
                 args: ["A-Za-z0-9_.", true]
             },
             {
-                match: "^\\s*(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?\\s*$",
                 flags: "i",
                 args: ["A-Za-z0-9._-", true]
             },
             {
-                match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                 flags: "i",
                 args: ["0-9a-zA-Z+/=", true]
             },
             {
-                match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                 flags: "i",
                 args: ["0-9A-Za-z+/=", true]
             },
             {
-                match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
+                pattern: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
                 flags: "",
                 args: [" -_", false]
             },
             {
-                match: "^\\s*[A-Z\\d+\\-]{20,}\\s*$",
+                pattern: "^\\s*[A-Z\\d+\\-]{20,}\\s*$",
                 flags: "i",
                 args: ["+\\-0-9A-Za-z", true]
             },
             {
-                match: "^\\s*[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}\\s*$",
+                pattern: "^\\s*[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}\\s*$",
                 flags: "",
                 args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", true]
             },
             {
-                match: "^\\s*(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?\\s*$",
                 flags: "i",
                 args: ["N-ZA-Mn-za-m0-9+/=", true]
             },
             {
-                match: "^\\s*[A-Z\\d./]{20,}\\s*$",
+                pattern: "^\\s*[A-Z\\d./]{20,}\\s*$",
                 flags: "i",
                 args: ["./0-9A-Za-z", true]
             },

src/core/operations/FromBinary.mjs

@@ -33,39 +33,39 @@ class FromBinary extends Operation {
                 "value": BIN_DELIM_OPTIONS
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^(?:[01]{8})+$",
+                pattern: "^(?:[01]{8})+$",
                 flags: "",
                 args: ["None"]
             },
             {
-                match: "^(?:[01]{8})(?: [01]{8})*$",
+                pattern: "^(?:[01]{8})(?: [01]{8})*$",
                 flags: "",
                 args: ["Space"]
             },
             {
-                match: "^(?:[01]{8})(?:,[01]{8})*$",
+                pattern: "^(?:[01]{8})(?:,[01]{8})*$",
                 flags: "",
                 args: ["Comma"]
             },
             {
-                match: "^(?:[01]{8})(?:;[01]{8})*$",
+                pattern: "^(?:[01]{8})(?:;[01]{8})*$",
                 flags: "",
                 args: ["Semi-colon"]
             },
             {
-                match: "^(?:[01]{8})(?::[01]{8})*$",
+                pattern: "^(?:[01]{8})(?::[01]{8})*$",
                 flags: "",
                 args: ["Colon"]
             },
             {
-                match: "^(?:[01]{8})(?:\\n[01]{8})*$",
+                pattern: "^(?:[01]{8})(?:\\n[01]{8})*$",
                 flags: "",
                 args: ["Line feed"]
             },
             {
-                match: "^(?:[01]{8})(?:\\r\\n[01]{8})*$",
+                pattern: "^(?:[01]{8})(?:\\r\\n[01]{8})*$",
                 flags: "",
                 args: ["CRLF"]
             },

src/core/operations/FromDecimal.mjs

@@ -36,37 +36,37 @@ class FromDecimal extends Operation {
                 "value": false
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
                 args: ["Space", false]
             },
             {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
                 args: ["Comma", false]
             },
             {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
                 args: ["Semi-colon", false]
             },
             {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
                 args: ["Colon", false]
             },
             {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
                 args: ["Line feed", false]
             },
             {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                 flags: "",
                 args: ["CRLF", false]
-            },
+            }
         ];
     }
 

src/core/operations/FromHTMLEntity.mjs

@@ -25,12 +25,12 @@ class FromHTMLEntity extends Operation {
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];
-        this.patterns = [
+        this.checks = [
             {
-                match: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});",
+                pattern: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});",
                 flags: "i",
                 args: []
-            },
+            }
         ];
     }
 

src/core/operations/FromHex.mjs

@@ -32,49 +32,54 @@ class FromHex extends Operation {
                 value: FROM_HEX_DELIM_OPTIONS
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^(?:[\\dA-F]{2})+$",
+                pattern: "^(?:[\\dA-F]{2})+$",
                 flags: "i",
                 args: ["None"]
             },
             {
-                match: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$",
                 flags: "i",
                 args: ["Space"]
             },
             {
-                match: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$",
                 flags: "i",
                 args: ["Comma"]
             },
             {
-                match: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$",
                 flags: "i",
                 args: ["Semi-colon"]
             },
             {
-                match: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$",
                 flags: "i",
                 args: ["Colon"]
             },
             {
-                match: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$",
                 flags: "i",
                 args: ["Line feed"]
             },
             {
-                match: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$",
                 flags: "i",
                 args: ["CRLF"]
             },
             {
-                match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$",
+                pattern: "^(?:0x[\\dA-F]{2})+$",
                 flags: "i",
                 args: ["0x"]
             },
             {
-                match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$",
+                pattern: "^0x[\\dA-F]{2}(?:,0x[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["0x with comma"]
+            },
+            {
+                pattern: "^(?:\\\\x[\\dA-F]{2})+$",
                 flags: "i",
                 args: ["\\x"]
             }

src/core/operations/FromHexContent.mjs

@@ -26,6 +26,13 @@ class FromHexContent extends Operation {
         this.inputType = "string";
         this.outputType = "byteArray";
         this.args = [];
+        this.checks = [
+            {
+                pattern:  "\\|([\\da-f]{2} ?)+\\|",
+                flags:  "i",
+                args:   []
+            }
+        ];
     }
 
     /**

src/core/operations/FromHexdump.mjs

@@ -27,9 +27,9 @@ class FromHexdump extends Operation {
         this.inputType = "string";
         this.outputType = "byteArray";
         this.args = [];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?){2,}$",
+                pattern: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?){2,}$",
                 flags: "i",
                 args: []
             },

src/core/operations/FromMorseCode.mjs

@@ -37,12 +37,12 @@ class FromMorseCode extends Operation {
                 "value": WORD_DELIM_OPTIONS
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)",
+                pattern: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)",
                 flags: "i",
                 args: ["Space", "Line feed"]
-            },
+            }
         ];
     }
 

src/core/operations/FromOctal.mjs

@@ -32,37 +32,37 @@ class FromOctal extends Operation {
                 "value": DELIM_OPTIONS
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$",
                 flags: "",
                 args: ["Space"]
             },
             {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                 flags: "",
                 args: ["Comma"]
             },
             {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                 flags: "",
                 args: ["Semi-colon"]
             },
             {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                 flags: "",
                 args: ["Colon"]
             },
             {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                 flags: "",
                 args: ["Line feed"]
             },
             {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                 flags: "",
                 args: ["CRLF"]
-            },
+            }
         ];
     }
 

src/core/operations/FromQuotedPrintable.mjs

@@ -28,9 +28,9 @@ class FromQuotedPrintable extends Operation {
         this.inputType = "string";
         this.outputType = "byteArray";
         this.args = [];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]{0,76}(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
+                pattern: "^[\\x21-\\x3d\\x3f-\\x7e \\t]{0,76}(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
                 flags: "i",
                 args: []
             },

src/core/operations/FromUNIXTimestamp.mjs

@@ -33,27 +33,27 @@ class FromUNIXTimestamp extends Operation {
                 "value": UNITS
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^1?\\d{9}$",
+                pattern: "^1?\\d{9}$",
                 flags: "",
                 args: ["Seconds (s)"]
             },
             {
-                match: "^1?\\d{12}$",
+                pattern: "^1?\\d{12}$",
                 flags: "",
                 args: ["Milliseconds (ms)"]
             },
             {
-                match: "^1?\\d{15}$",
+                pattern: "^1?\\d{15}$",
                 flags: "",
                 args: ["Microseconds (μs)"]
             },
             {
-                match: "^1?\\d{18}$",
+                pattern: "^1?\\d{18}$",
                 flags: "",
                 args: ["Nanoseconds (ns)"]
-            },
+            }
         ];
     }
 

src/core/operations/Gunzip.mjs

@@ -27,12 +27,12 @@ class Gunzip extends Operation {
         this.inputType = "ArrayBuffer";
         this.outputType = "ArrayBuffer";
         this.args = [];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^\\x1f\\x8b\\x08",
+                pattern: "^\\x1f\\x8b\\x08",
                 flags: "",
                 args: []
-            },
+            }
         ];
     }
 

src/core/operations/ParseQRCode.mjs

@@ -33,9 +33,9 @@ class ParseQRCode extends Operation {
                 "value": false
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                "match": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
+                "pattern": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
                 "flags": "",
                 "args": [false],
                 "useful": true

src/core/operations/ParseSSHHostKey.mjs

@@ -38,6 +38,13 @@ class ParseSSHHostKey extends Operation {
                 ]
             }
         ];
+        this.checks = [
+            {
+                pattern:  "^\\s*([A-F\\d]{2}[,;:]){15,}[A-F\\d]{2}\\s*$",
+                flags:  "i",
+                args:   ["Hex"]
+            }
+        ];
     }
 
     /**

src/core/operations/ParseUNIXFilePermissions.mjs

@@ -25,6 +25,13 @@ class ParseUNIXFilePermissions extends Operation {
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];
+        this.checks = [
+            {
+                pattern:  "^\\s*d[rxw-]{9}\\s*$",
+                flags:  "",
+                args:   []
+            }
+        ];
     }
 
     /**

src/core/operations/ParseUserAgent.mjs

@@ -25,6 +25,13 @@ class ParseUserAgent extends Operation {
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];
+        this.checks = [
+            {
+                pattern:  "^(User-Agent:|Mozilla\\/)[^\\n\\r]+\\s*$",
+                flags:  "i",
+                args:   []
+            }
+        ];
     }
 
     /**

src/core/operations/ParseX509Certificate.mjs

@@ -35,13 +35,11 @@ class ParseX509Certificate extends Operation {
                 "value": ["PEM", "DER Hex", "Base64", "Raw"]
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                "match": "^-+BEGIN CERTIFICATE-+\\r?\\n[\\da-z+/\\n\\r]+-+END CERTIFICATE-+\\r?\\n?$",
+                "pattern": "^-+BEGIN CERTIFICATE-+\\r?\\n[\\da-z+/\\n\\r]+-+END CERTIFICATE-+\\r?\\n?$",
                 "flags": "i",
-                "args": [
+                "args": ["PEM"]
-                    "PEM"
-                ]
             }
         ];
     }

src/core/operations/RawInflate.mjs

@@ -60,6 +60,12 @@ class RawInflate extends Operation {
                 value: false
             }
         ];
+        this.checks = [
+            {
+                entropyRange: [7.5, 8],
+                args: [0, 0, INFLATE_BUFFER_TYPE, false, false]
+            }
+        ];
     }
 
     /**

src/core/operations/RegularExpression.mjs

@@ -163,7 +163,7 @@ class RegularExpression extends Operation {
                     case "List matches with capture groups":
                         return Utils.escapeHtml(regexList(input, regex, displayTotal, true, true));
                     default:
-                        return "Error: Invalid output format";
+                        throw new OperationError("Error: Invalid output format");
                 }
             } catch (err) {
                 throw new OperationError("Invalid regex. Details: " + err.message);

src/core/operations/RenderImage.mjs

@@ -35,12 +35,15 @@ class RenderImage extends Operation {
                 "value": ["Raw", "Base64", "Hex"]
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                "match": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
+                pattern: "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
-                "flags": "",
+                flags: "",
-                "args": ["Raw"],
+                args: ["Raw"],
-                "useful": true
+                useful: true,
+                output: {
+                    mime: "image"
+                }
             }
         ];
     }

src/core/operations/StripHTMLTags.mjs

@@ -35,6 +35,13 @@ class StripHTMLTags extends Operation {
                 "value": true
             }
         ];
+        this.checks = [
+            {
+                pattern:  "(</html>|</div>|</body>)",
+                flags:  "i",
+                args:   [true, true]
+            }
+        ];
     }
 
     /**

src/core/operations/StripHTTPHeaders.mjs

@@ -24,6 +24,13 @@ class StripHTTPHeaders extends Operation {
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];
+        this.checks = [
+            {
+                pattern:  "^HTTP(.|\\s)+?(\\r?\\n){2}",
+                flags:  "",
+                args:   []
+            }
+        ];
     }
 
     /**

src/core/operations/URLDecode.mjs

@@ -24,9 +24,9 @@ class URLDecode extends Operation {
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];
-        this.patterns = [
+        this.checks = [
             {
-                match: ".*(?:%[\\da-f]{2}.*){4}",
+                pattern: ".*(?:%[\\da-f]{2}.*){4}",
                 flags: "i",
                 args: []
             },

src/core/operations/Untar.mjs

@@ -27,9 +27,9 @@ class Untar extends Operation {
         this.outputType = "List<File>";
         this.presentType = "html";
         this.args = [];
-        this.patterns = [
+        this.checks = [
             {
-                "match": "^.{257}\\x75\\x73\\x74\\x61\\x72",
+                "pattern": "^.{257}\\x75\\x73\\x74\\x61\\x72",
                 "flags": "",
                 "args": []
             }

src/core/operations/Unzip.mjs

@@ -40,12 +40,12 @@ class Unzip extends Operation {
                 value: false
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)",
+                pattern: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)",
                 flags: "",
                 args: ["", false]
-            },
+            }
         ];
     }
 

src/core/operations/ZlibInflate.mjs

@@ -59,9 +59,9 @@ class ZlibInflate extends Operation {
                 value: false
             }
         ];
-        this.patterns = [
+        this.checks = [
             {
-                match: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)",
+                pattern: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)",
                 flags: "",
                 args: [0, 0, "Adaptive", false, false]
             },

tests/lib/TestRegister.mjs

@@ -97,10 +97,14 @@ class TestRegister {
                     ret.status = "passing";
                 } else if ("expectedMatch" in test && test.expectedMatch.test(result.result)) {
                     ret.status = "passing";
+                } else if ("unexpectedMatch" in test && !test.unexpectedMatch.test(result.result)) {
+                    ret.status = "passing";
                 } else {
                     ret.status = "failing";
                     const expected = test.expectedOutput ? test.expectedOutput :
-                        test.expectedMatch ? test.expectedMatch.toString() : "unknown";
+                        test.expectedMatch ? test.expectedMatch.toString() :
+                            test.unexpectedMatch ? "to not find " + test.unexpectedMatch.toString() :
+                                "unknown";
                     ret.output = [
                         "Expected",
                         "\t" + expected.replace(/\n/g, "\n\t"),

tests/operations/tests/Hex.mjs

@@ -92,6 +92,19 @@ TestRegister.addTests([
                 ]
             }
         ]
-
+    },
+    {
+        name: "0x with Comma to Ascii",
+        input: "0x74,0x65,0x73,0x74,0x20,0x73,0x74,0x72,0x69,0x6e,0x67",
+        expectedOutput: "test string",
+        recipeConfig: [
+            {
+                "op": "From Hex",
+                "args": [
+                    "0x with comma"
+                ]
             }
+        ]
+
+    },
 ]);