Improvements and basic tests for Magic op and addition of deep cloning for dishes

src/core/Chef.mjs

@@ -96,7 +96,7 @@ class Chef {
         const returnType = this.dish.size > threshold ? Dish.ARRAY_BUFFER : Dish.STRING;
 
         // Create a raw version of the dish, unpresented
-        const rawDish = new Dish(this.dish);
+        const rawDish = this.dish.clone();
 
         // Present the raw result
         await recipe.present(this.dish);

src/core/Dish.mjs

@@ -300,6 +300,69 @@ class Dish {
         }
     }
 
+
+    /**
+     * Returns a deep clone of the current Dish.
+     *
+     * @returns {Dish}
+     */
+    clone() {
+        const newDish = new Dish();
+
+        switch (this.type) {
+            case Dish.STRING:
+            case Dish.HTML:
+            case Dish.NUMBER:
+            case Dish.BIG_NUMBER:
+                // These data types are immutable so it is acceptable to copy them by reference
+                newDish.set(
+                    this.value,
+                    this.type
+                );
+                break;
+            case Dish.BYTE_ARRAY:
+            case Dish.JSON:
+                // These data types are mutable so they need to be copied by value
+                newDish.set(
+                    JSON.parse(JSON.stringify(this.value)),
+                    this.type
+                );
+                break;
+            case Dish.ARRAY_BUFFER:
+                // Slicing an ArrayBuffer returns a new ArrayBuffer with a copy its contents
+                newDish.set(
+                    this.value.slice(0),
+                    this.type
+                );
+                break;
+            case Dish.FILE:
+                // A new file can be created by copying over all the values from the original
+                newDish.set(
+                    new File([this.value], this.value.name, {
+                        "type": this.value.type,
+                        "lastModified": this.value.lastModified
+                    }),
+                    this.type
+                );
+                break;
+            case Dish.LIST_FILE:
+                newDish.set(
+                    this.value.map(f =>
+                        new File([f], f.name, {
+                            "type": f.type,
+                            "lastModified": f.lastModified
+                        })
+                    ),
+                    this.type
+                );
+                break;
+            default:
+                throw new Error("Cannot clone Dish, unknown type");
+        }
+
+        return newDish;
+    }
+
 }
 
 

src/core/lib/Magic.mjs

@@ -287,6 +287,8 @@ class Magic {
             useful: useful
         });
 
+        const prevOp = recipeConfig[recipeConfig.length - 1];
+
         // Execute each of the matching operations, then recursively call the speculativeExecution()
         // method on the resulting data, recording the properties of each option.
         await Promise.all(matchingOps.map(async op => {
@@ -294,8 +296,14 @@ class Magic {
                     op: op.op,
                     args: op.args
                 },
-                output = await this._runRecipe([opConfig]),
+                output = await this._runRecipe([opConfig]);
-                magic = new Magic(output, this.opPatterns),
+
+            // If the recipe is repeating and returning the same data, do not continue
+            if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
+                return;
+            }
+
+            const magic = new Magic(output, this.opPatterns),
                 speculativeResults = await magic.speculativeExecution(
                     depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
 
@@ -315,13 +323,16 @@ class Magic {
             }));
         }
 
-        // Prune branches that do not match anything
+        // Prune branches that result in unhelpful outputs
         results = results.filter(r =>
-            r.languageScores[0].probability > 0 ||
+            (r.useful || r.data.length > 0) &&          // The operation resulted in ""
-            r.fileType ||
+            (                                           // One of the following must be true
-            r.isUTF8 ||
+                r.languageScores[0].probability > 0 ||    // Some kind of language was found
-            r.matchingOps.length ||
+                r.fileType ||                             // A file was found
-            r.useful);
+                r.isUTF8 ||                               // UTF-8 was found
+                r.matchingOps.length                      // A matching op was found
+            )
+        );
 
         // Return a sorted list of possible recipes along with their properties
         return results.sort((a, b) => {
@@ -374,7 +385,7 @@ class Magic {
 
         const recipe = new Recipe(recipeConfig);
         try {
-            await recipe.execute(dish, 0);
+            await recipe.execute(dish);
             return dish.get(Dish.ARRAY_BUFFER);
         } catch (err) {
             // If there are errors, return an empty buffer
@@ -395,7 +406,10 @@ class Magic {
         let i = len;
         const counts = new Array(256).fill(0);
 
-        if (!len) return counts;
+        if (!len) {
+            this.freqDist = counts;
+            return this.freqDist;
+        }
 
         while (i--) {
             counts[this.inputBuffer[i]]++;

src/core/operations/FromHexdump.mjs

@@ -26,7 +26,7 @@ class FromHexdump extends Operation {
         this.args = [];
         this.patterns = [
             {
-                match: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?)+$",
+                match: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?){2,}$",
                 flags: "i",
                 args: []
             },

src/core/operations/RenderImage.mjs

@@ -26,7 +26,8 @@ class RenderImage extends Operation {
         this.module = "Image";
         this.description = "Displays the input as an image. Supports the following formats:<br><br><ul><li>jpg/jpeg</li><li>png</li><li>gif</li><li>webp</li><li>bmp</li><li>ico</li></ul>";
         this.inputType = "string";
-        this.outputType = "html";
+        this.outputType = "byteArray";
+        this.presentType = "html";
         this.args = [
             {
                 "name": "Input format",
@@ -51,9 +52,8 @@ class RenderImage extends Operation {
      */
     run(input, args) {
         const inputFormat = args[0];
-        let dataURI = "data:";
 
-        if (!input.length) return "";
+        if (!input.length) return [];
 
         // Convert input to raw bytes
         switch (inputFormat) {
@@ -73,6 +73,26 @@ class RenderImage extends Operation {
 
         // Determine file type
         const type = Magic.magicFileType(input);
+        if (!(type && type.mime.indexOf("image") === 0)) {
+            throw new OperationError("Invalid file type");
+        }
+
+        return input;
+    }
+
+    /**
+     * Displays the image using HTML for web apps.
+     *
+     * @param {byteArray} data
+     * @returns {html}
+     */
+    async present(data) {
+        if (!data.length) return "";
+
+        let dataURI = "data:";
+
+        // Determine file type
+        const type = Magic.magicFileType(data);
         if (type && type.mime.indexOf("image") === 0) {
             dataURI += type.mime + ";";
         } else {
@@ -80,7 +100,7 @@ class RenderImage extends Operation {
         }
 
         // Add image data to URI
-        dataURI += "base64," + toBase64(input);
+        dataURI += "base64," + toBase64(data);
 
         return "<img src='" + dataURI + "'>";
     }

src/test.mjs

@@ -0,0 +1,20 @@
+import Dish from "./core/Dish";
+
+const a = new Dish();
+const i = "original";
+a.set(i, Dish.STRING);
+
+console.log(a);
+
+const b = a.clone();
+
+console.log(b);
+
+console.log("changing a");
+
+a.value.toUpperCase();
+// const c = new Uint8Array([1,2,3,4,5,6,7,8,9,0]).buffer;
+// a.set(c, Dish.ARRAY_BUFFER);
+
+console.log(a);
+console.log(b);

src/web/BackgroundWorkerWaiter.mjs

@@ -120,7 +120,7 @@ class BackgroundWorkerWaiter {
      * @param {string|ArrayBuffer} input
      */
     magic(input) {
-        // If we're still working on the previous bake, cancel it before stating a new one.
+        // If we're still working on the previous bake, cancel it before starting a new one.
         if (this.completedCallback + 1 < this.callbackID) {
             clearTimeout(this.timeout);
             this.cancelBake();

test/TestRegister.mjs

@@ -66,11 +66,15 @@ import Chef from "../src/core/Chef";
                             ret.output = "Expected an error but did not receive one.";
                         } else if (result.result === test.expectedOutput) {
                             ret.status = "passing";
+                        } else if (test.hasOwnProperty("expectedMatch") && test.expectedMatch.test(result.result)) {
+                            ret.status = "passing";
                         } else {
                             ret.status = "failing";
+                            const expected = test.expectedOutput ? test.expectedOutput :
+                                test.expectedMatch ? test.expectedMatch.toString() : "unknown";
                             ret.output = [
                                 "Expected",
-                                "\t" + test.expectedOutput.replace(/\n/g, "\n\t"),
+                                "\t" + expected.replace(/\n/g, "\n\t"),
                                 "Received",
                                 "\t" + result.result.replace(/\n/g, "\n\t"),
                             ].join("\n");

test/index.mjs

@@ -62,6 +62,7 @@ import "./tests/operations/SetDifference";
 import "./tests/operations/SetIntersection";
 import "./tests/operations/SetUnion";
 import "./tests/operations/SymmetricDifference";
+import "./tests/operations/Magic";
 
 let allTestsPassing = true;
 const testStatusCounts = {

test/tests/operations/Magic.mjs

@@ -0,0 +1,57 @@
+/**
+ * Magic tests.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ *
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+import TestRegister from "../../TestRegister";
+
+
+TestRegister.addTests([
+    {
+        name: "Magic: nothing",
+        input: "",
+        expectedOutput: "Nothing of interest could be detected about the input data.\nHave you tried modifying the operation arguments?",
+        recipeConfig: [
+            {
+                op: "Magic",
+                args: [3, false, false]
+            }
+        ],
+    },
+    {
+        name: "Magic: hex",
+        input: "41 42 43 44 45",
+        expectedMatch: /"#recipe=From_Hex\('Space'\)"/,
+        recipeConfig: [
+            {
+                op: "Magic",
+                args: [3, false, false]
+            }
+        ],
+    },
+    {
+        name: "Magic: jpeg",
+        input: "\xFF\xD8\xFF",
+        expectedMatch: /Render_Image\('Raw'\)/,
+        recipeConfig: [
+            {
+                op: "Magic",
+                args: [3, false, false]
+            }
+        ],
+    },
+    {
+        name: "Magic: mojibake",
+        input: "d091d18bd100d182d180d0b0d10020d0bad0bed180d0b8d187d0bdd0b5d0b2d0b0d10020d0bbd0b8d100d0b020d0bfd180d18bd0b3d0b0d0b5d18220d187d0b5d180d0b5d0b720d0bbd0b5d0bdd0b8d0b2d183d18e20d100d0bed0b1d0b0d0bad1832e",
+        expectedMatch: /Быртрар коричневар лира прыгает через ленивую робаку./,
+        recipeConfig: [
+            {
+                op: "Magic",
+                args: [3, true, false]
+            }
+        ],
+    },
+]);