Merge branch 'master' into feature-async-ops

.gitignore

@@ -1,6 +1,7 @@
 node_modules
 npm-debug.log
 build/dev
+build/test
 docs/*
 !docs/*.conf.json
 !docs/*.ico

Gruntfile.js

@@ -9,10 +9,14 @@ module.exports = function(grunt) {
         "A persistent task which creates a development build whenever source files are modified.",
         ["clean:dev", "concat:css", "concat:js", "copy:htmlDev", "copy:staticDev", "chmod:build", "watch"]);
 
+    grunt.registerTask("test",
+        "A task which runs all the tests in test/tests.",
+        ["clean:test", "concat:jsTest", "copy:htmlTest", "chmod:build", "execute:test"]);
+
     grunt.registerTask("prod",
         "Creates a production-ready build. Use the --msg flag to add a compile message.",
         ["eslint", "exec:stats", "clean", "jsdoc", "concat", "copy:htmlDev", "copy:htmlProd", "copy:htmlInline",
-         "copy:staticDev", "copy:staticProd", "cssmin", "uglify:prod", "inline", "htmlmin", "chmod"]);
+         "copy:staticDev", "copy:staticProd", "cssmin", "uglify:prod", "inline", "htmlmin", "chmod", "test"]);
 
     grunt.registerTask("docs",
         "Compiles documentation in the /docs directory.",
@@ -31,6 +35,7 @@ module.exports = function(grunt) {
         ["eslint", "exec:stats", "exec:displayStats"]);
 
     grunt.registerTask("doc", "docs");
+    grunt.registerTask("tests", "test");
     grunt.registerTask("lint", "eslint");
 
 
@@ -46,11 +51,12 @@ module.exports = function(grunt) {
     grunt.loadNpmTasks("grunt-inline-alt");
     grunt.loadNpmTasks("grunt-chmod");
     grunt.loadNpmTasks("grunt-exec");
+    grunt.loadNpmTasks("grunt-execute");
     grunt.loadNpmTasks("grunt-contrib-watch");
 
 
     // JS includes
-    var jsFiles = [
+    var jsIncludes = [
         // Third party framework libraries
         "src/js/lib/jquery-2.1.1.js",
         "src/js/lib/bootstrap-3.3.6.js",
@@ -134,6 +140,7 @@ module.exports = function(grunt) {
         "src/js/lib/vkbeautify.js",
         "src/js/lib/Sortable.js",
         "src/js/lib/bootstrap-colorpicker.js",
+        "src/js/lib/es6-promise.auto.js",
         "src/js/lib/xpath.js",
 
         // Custom libraries
@@ -154,10 +161,19 @@ module.exports = function(grunt) {
         "src/js/views/html/*.js",
         "!src/js/views/html/main.js",
 
-        // Start the app!
-        "src/js/views/html/main.js",
     ];
 
+    var jsAppFiles = jsIncludes.concat([
+        // Start the main app!
+        "src/js/views/html/main.js",
+    ]);
+
+    var jsTestFiles = jsIncludes.concat([
+        "test/TestRegister.js",
+        "test/tests/**/*.js",
+        "test/TestRunner.js",
+    ]);
+
     var banner = '/**\n\
  * CyberChef - The Cyber Swiss Army Knife\n\
  *\n\
@@ -198,6 +214,7 @@ module.exports = function(grunt) {
             config: ["src/js/config/**/*.js"],
             views: ["src/js/views/**/*.js"],
             operations: ["src/js/operations/**/*.js"],
+            tests: ["test/**/*.js"],
         },
         jsdoc: {
             options: {
@@ -217,6 +234,7 @@ module.exports = function(grunt) {
         clean: {
             dev: ["build/dev/*"],
             prod: ["build/prod/*"],
+            test: ["build/test/*"],
             docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico"],
         },
         concat: {
@@ -243,8 +261,15 @@ module.exports = function(grunt) {
                 options: {
                     banner: '"use strict";\n'
                 },
-                src: jsFiles,
+                src: jsAppFiles,
                 dest: "build/dev/scripts.js"
+            },
+            jsTest: {
+                options: {
+                    banner: '"use strict";\n'
+                },
+                src: jsTestFiles,
+                dest: "build/test/tests.js"
             }
         },
         copy: {
@@ -257,6 +282,10 @@ module.exports = function(grunt) {
                 src: "src/html/index.html",
                 dest: "build/dev/index.html"
             },
+            htmlTest: {
+                src: "test/test.html",
+                dest: "build/test/index.html"
+            },
             htmlProd: {
                 options: {
                     process: function(content, srcpath) {
@@ -461,6 +490,9 @@ module.exports = function(grunt) {
                 ].join(";")
             }
         },
+        execute: {
+            test: "test/NodeRunner.js"
+        },
         watch: {
             css: {
                 files: "src/css/**/*.css",

README.md

@@ -79,6 +79,11 @@ An installation walkthrough, how-to guides for adding new operations, descriptio
  - Submit a pull request.
 
 
+## Licencing
+
+CyberChef is released under the [Apache 2.0 Licence](https://www.apache.org/licenses/LICENSE-2.0) and is covered by [Crown Copyright](https://www.nationalarchives.gov.uk/information-management/re-using-public-sector-information/copyright-and-re-use/crown-copyright/).
+
+
   [1]: https://gchq.github.io/CyberChef
   [2]: https://gchq.github.io/CyberChef/?recipe=%5B%7B%22op%22%3A%22From%20Base64%22%2C%22args%22%3A%5B%22A-Za-z0-9%2B%2F%3D%22%2Ctrue%5D%7D%5D&input=VTI4Z2JHOXVaeUJoYm1RZ2RHaGhibXR6SUdadmNpQmhiR3dnZEdobElHWnBjMmd1
   [3]: https://gchq.github.io/CyberChef/?recipe=%5B%7B%22op%22%3A%22Translate%20DateTime%20Format%22%2C%22args%22%3A%5B%22Standard%20date%20and%20time%22%2C%22DD%2FMM%2FYYYY%20HH%3Amm%3Ass%22%2C%22UTC%22%2C%22dddd%20Do%20MMMM%20YYYY%20HH%3Amm%3Ass%20Z%20z%22%2C%22Australia%2FQueensland%22%5D%7D%5D&input=MTUvMDYvMjAxNSAyMDo0NTowMA

package.json

@@ -37,8 +37,10 @@
     "grunt-contrib-watch": "~1.0.0",
     "grunt-eslint": "^19.0.0",
     "grunt-exec": "~1.0.1",
+    "grunt-execute": "^0.2.2",
     "grunt-inline-alt": "~0.3.10",
     "grunt-jsdoc": "^2.1.0",
-    "ink-docstrap": "^1.1.4"
+    "ink-docstrap": "^1.1.4",
+    "phantomjs-prebuilt": "^2.1.14"
   }
 }

src/js/.eslintrc.json

@@ -109,6 +109,10 @@
         "OutputWaiter": false,
         "RecipeWaiter": false,
         "SeasonalWaiter": false,
-        "WindowWaiter": false
+        "WindowWaiter": false,
+
+        /* tests */
+        "TestRegister": false,
+        "TestRunner": false
     }
-}
+}

src/js/core/Dish.js

@@ -139,7 +139,7 @@ Dish.prototype.translate = function(toType) {
             this.type = Dish.BYTE_ARRAY;
             break;
         case Dish.HTML:
-            this.value = this.value ? Utils.strToByteArray(Utils.stripHtmlTags(this.value, true)) : [];
+            this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : [];
             this.type = Dish.BYTE_ARRAY;
             break;
         default:

src/js/core/FlowControl.js

@@ -152,7 +152,6 @@ var FlowControl = {
             maxJumps = ings[1];
 
         if (state.numJumps >= maxJumps) {
-            state.progress++;
             return state;
         }
 
@@ -180,7 +179,6 @@ var FlowControl = {
             maxJumps = ings[2];
 
         if (state.numJumps >= maxJumps) {
-            state.progress++;
             return state;
         }
 

src/js/core/Utils.js

@@ -928,6 +928,33 @@ var Utils = {
     },
 
 
+    /**
+     * Unescapes HTML tags in a string to make them render again.
+     *
+     * @param {string} str
+     * @returns string
+     *
+     * @example
+     * // return "A <script> tag"
+     * Utils.unescapeHtml("A &lt;script&gt; tag");
+     */
+    unescapeHtml: function(str) {
+        var HTML_CHARS = {
+            "&amp;":  "&",
+            "&lt;":   "<",
+            "&gt;":   ">",
+            "&quot;": '"',
+            "&#x27;": "'",
+            "&#x2F;": "/",
+            "&#x60;": "`"
+        };
+
+        return str.replace(/&#?x?[a-z0-9]{2,4};/ig, function (match) {
+            return HTML_CHARS[match] || match;
+        });
+    },
+
+
     /**
      * Expresses a number of milliseconds in a human readable format.
      *

test/NodeRunner.js

@@ -0,0 +1,24 @@
+/* eslint-env node */
+
+/**
+ * NodeRunner.js
+ *
+ * The purpose of this file is to execute via PhantomJS the file
+ * PhantomRunner.js, because PhantomJS is managed by node.
+ *
+ * @author tlwr [toby@toby.codes]
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+
+var path = require("path"),
+    phantomjs = require("phantomjs-prebuilt"),
+    phantomEntryPoint = path.join(__dirname, "PhantomRunner.js"),
+    program = phantomjs.exec(phantomEntryPoint);
+
+program.stdout.pipe(process.stdout);
+program.stderr.pipe(process.stderr);
+
+program.on("exit", function(status) {
+    process.exit(status);
+});

test/PhantomRunner.js

@@ -0,0 +1,99 @@
+/* eslint-env node */
+/* globals phantom */
+
+/**
+ * PhantomRunner.js
+ *
+ * This file navigates to build/test/index.html and logs the test results.
+ *
+ * @author tlwr [toby@toby.codes]
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+
+var page = require("webpage").create(),
+    allTestsPassing = true,
+    testStatusCounts = {
+        total: 0,
+    };
+
+
+/**
+ * Helper function to convert a status to an icon.
+ *
+ * @param {string} status
+ * @returns {string}
+ */
+function statusToIcon(status) {
+    var icons = {
+        erroring: "🔥",
+        failing: "❌",
+        passing: "✔️️",
+    };
+    return icons[status] || "?";
+}
+
+
+/**
+ * Callback function to handle test results.
+ */
+page.onCallback = function(messageType) {
+    if (messageType === "testResult") {
+        var testResult = arguments[1];
+
+        allTestsPassing = allTestsPassing && testResult.status === "passing";
+        var newCount = (testStatusCounts[testResult.status] || 0) + 1;
+        testStatusCounts[testResult.status] = newCount;
+        testStatusCounts.total += 1;
+
+        console.log([
+            statusToIcon(testResult.status),
+            testResult.test.name
+        ].join(" "));
+
+        if (testResult.output) {
+            console.log(
+                testResult.output
+                    .trim()
+                    .replace(/^/, "\t")
+                    .replace(/\n/g, "\n\t")
+            );
+        }
+    } else if (messageType === "exit") {
+
+        console.log("\n");
+
+        for (var testStatus in testStatusCounts) {
+            var count = testStatusCounts[testStatus];
+            if (count > 0) {
+                console.log(testStatus.toUpperCase(), count);
+            }
+        }
+
+        if (!allTestsPassing) {
+            console.log("\nNot all tests are passing");
+        }
+
+        phantom.exit(allTestsPassing ? 0 : 1);
+    }
+};
+
+
+/**
+ * Open the test webpage in PhantomJS.
+ */
+page.open("build/test/index.html", function(status) {
+    if (status !== "success") {
+        console.log("STATUS: ", status);
+        phantom.exit(1);
+    }
+});
+
+
+/**
+ * Fail if the process takes longer than 10 seconds.
+ */
+setTimeout(function() {
+    console.log("Tests took longer than 10 seconds to run, returning.");
+    phantom.exit(1);
+}, 10 * 1000);

test/TestRegister.js

@@ -0,0 +1,89 @@
+/**
+ * TestRegister.js
+ *
+ * This is so individual files can register their tests in one place, and
+ * ensure that they will get run by the frontend.
+ *
+ * @author tlwr [toby@toby.codes]
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+
+(function() {
+    /**
+     * Object to store and run the list of tests.
+     *
+     * @class
+     * @constructor
+     */
+    function TestRegister() {
+        this.tests = [];
+    }
+
+
+    /**
+     * Add a list of tests to the register.
+     *
+     * @param {Object[]} tests
+     */
+    TestRegister.prototype.addTests = function(tests) {
+        this.tests = this.tests.concat(tests);
+    };
+
+
+    /**
+     * Runs all the tests in the register.
+     */
+    TestRegister.prototype.runTests = function() {
+        return Promise.all(
+            this.tests.map(function(test, i) {
+                var chef = new Chef();
+
+                return Promise.resolve(chef.bake(
+                    test.input,
+                    test.recipeConfig,
+                    {},
+                    0,
+                    false
+                ))
+                .then(function(result) {
+                    var ret = {
+                        test: test,
+                        status: null,
+                        output: null,
+                    };
+
+                    if (result.error) {
+                        if (test.expectedError) {
+                            ret.status = "passing";
+                        } else {
+                            ret.status = "erroring";
+                            ret.output = result.error.displayStr;
+                        }
+                    } else {
+                        if (test.expectedError) {
+                            ret.status = "failing";
+                            ret.output = "Expected an error but did not receive one.";
+                        } else if (result.result === test.expectedOutput) {
+                            ret.status = "passing";
+                        } else {
+                            ret.status = "failing";
+                            ret.output = [
+                                "Expected",
+                                "\t" + test.expectedOutput.replace(/\n/g, "\n\t"),
+                                "Received",
+                                "\t" + result.result.replace(/\n/g, "\n\t"),
+                            ].join("\n");
+                        }
+                    }
+
+                    return ret;
+                });
+            })
+        );
+    };
+
+
+    // Singleton TestRegister, keeping things simple and obvious.
+    window.TestRegister = new TestRegister();
+})();

test/TestRunner.js

@@ -0,0 +1,38 @@
+/**
+ * TestRunner.js
+ *
+ * This is for actually running the tests in the test register.
+ *
+ * @author tlwr [toby@toby.codes]
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+
+document.addEventListener("DOMContentLoaded", function() {
+    TestRegister.runTests()
+    .then(function(results) {
+        results.forEach(function(testResult) {
+
+            if (typeof window.callPhantom === "function") {
+                // If we're running this in PhantomJS
+                window.callPhantom(
+                    "testResult",
+                    testResult
+                );
+            } else {
+                // If we're just viewing this in a normal browser
+                var output = [
+                    "----------",
+                    testResult.test.name,
+                    testResult.status,
+                    testResult.output,
+                ].join("<br>");
+                document.querySelector("main").innerHTML += output;
+            }
+        });
+
+        if (typeof window.callPhantom === "function") {
+            window.callPhantom("exit");
+        }
+    });
+});

test/test.html

@@ -0,0 +1,33 @@
+<!--
+    CyberChef test suite
+    
+    @author tlwr [toby@toby.codes]
+
+    @copyright Crown Copyright 2017
+    @license Apache-2.0
+    
+      Copyright 2017 Crown Copyright
+    
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+    
+        http://www.apache.org/licenses/LICENSE-2.0
+    
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta charset="UTF-8">
+        <title>CyberChef test suite</title>
+    </head>
+    <body>
+        <main style="white-space: pre"></main>
+        <script type="application/javascript" src="tests.js"></script>
+    </body>
+</html>

test/tests/operations/Base58.js

@@ -0,0 +1,76 @@
+/**
+ * Base58 tests.
+ *
+ * @author tlwr [toby@toby.codes]
+ *
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+TestRegister.addTests([
+    {
+        name: "To Base58 (Bitcoin): nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                op: "To Base58",
+                args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"],
+            },
+        ],
+    },
+    {
+        name: "To Base58 (Ripple): nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                op: "To Base58",
+                args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz"],
+            },
+        ],
+    },
+    {
+        name: "To Base58 (Bitcoin): 'hello world'",
+        input: "hello world",
+        expectedOutput: "StV1DL6CwTryKyV",
+        recipeConfig: [
+            {
+                op: "To Base58",
+                args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"],
+            },
+        ],
+    },
+    {
+        name: "To Base58 (Ripple): 'hello world'",
+        input: "hello world",
+        expectedOutput: "StVrDLaUATiyKyV",
+        recipeConfig: [
+            {
+                op: "To Base58",
+                args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz"],
+            },
+        ],
+    },
+    {
+        name: "From Base58 (Bitcoin): 'StV1DL6CwTryKyV'",
+        input: "StV1DL6CwTryKyV",
+        expectedOutput: "hello world",
+        recipeConfig: [
+            {
+                op: "From Base58",
+                args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"],
+            },
+        ],
+    },
+    {
+        name: "From Base58 (Ripple): 'StVrDLaUATiyKyV'",
+        input: "StVrDLaUATiyKyV",
+        expectedOutput: "hello world",
+        recipeConfig: [
+            {
+                op: "From Base58",
+                args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz"],
+            },
+        ],
+    },
+]);

test/tests/operations/FlowControl.js

@@ -0,0 +1,112 @@
+/**
+ * Flow Control tests.
+ *
+ * @author tlwr [toby@toby.codes]
+ *
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+TestRegister.addTests([
+    {
+        name: "Fork: nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                op: "Fork",
+                args: ["\n", "\n", false],
+            },
+        ],
+    },
+    {
+        name: "Fork, Merge: nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                op: "Fork",
+                args: ["\n", "\n", false],
+            },
+            {
+                op: "Merge",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "Fork, (expect) Error, Merge",
+        input: "1\n2\na\n4",
+        expectedError: true,
+        recipeConfig: [
+            {
+                op: "Fork",
+                args: ["\n", "\n", false],
+            },
+            {
+                op: "To Base",
+                args: [16],
+            },
+            {
+                op: "Merge",
+                args: [],
+            },
+        ],
+    },
+    {
+        name: "Fork, Conditional Jump, Encodings",
+        input: "Some data with a 1 in it\nSome data with a 2 in it",
+        expectedOutput: "U29tZSBkYXRhIHdpdGggYSAxIGluIGl0\n53 6f 6d 65 20 64 61 74 61 20 77 69 74 68 20 61 20 32 20 69 6e 20 69 74\n",
+        recipeConfig: [
+            {"op":"Fork", "args":["\\n", "\\n", false]},
+            {"op":"Conditional Jump", "args":["1", "2", "10"]},
+            {"op":"To Hex", "args":["Space"]},
+            {"op":"Return", "args":[]},
+            {"op":"To Base64", "args":["A-Za-z0-9+/="]}
+        ]
+    },
+    {
+        name: "Conditional Jump: Skips 0",
+        input: [
+            "match",
+            "should be changed 1",
+            "should be changed 2",
+        ].join("\n"),
+        expectedOutput: [
+            "match",
+            "should be changed 1 was changed",
+            "should be changed 2 was changed"
+        ].join("\n"),
+        recipeConfig: [
+            {
+                op: "Conditional Jump",
+                args: ["match", 0, 0],
+            },
+            {
+                op: "Find / Replace",
+                args: [
+                    {
+                        "option": "Regex",
+                        "string": "should be changed 1"
+                    },
+                    "should be changed 1 was changed",
+                    true,
+                    true,
+                    true,
+                ],
+            },
+            {
+                op: "Find / Replace",
+                args: [
+                    {
+                        "option": "Regex",
+                        "string": "should be changed 2"
+                    },
+                    "should be changed 2 was changed",
+                    true,
+                    true,
+                    true,
+                ],
+            },
+        ],
+    },
+]);

test/tests/operations/MorseCode.js

@@ -0,0 +1,32 @@
+/**
+ * Base58 tests.
+ *
+ * @author tlwr [toby@toby.codes]
+ *
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+TestRegister.addTests([
+    {
+        name: "To Morse Code: 'SOS'",
+        input: "SOS",
+        expectedOutput: "... --- ...",
+        recipeConfig: [
+            {
+                op: "To Morse Code",
+                args: ["-/.", "Space", "Line feed"],
+            },
+        ],
+    },
+    {
+        name: "From Morse Code '... --- ...'",
+        input: "... --- ...",
+        expectedOutput: "SOS",
+        recipeConfig: [
+            {
+                op: "From Morse Code",
+                args: ["Space", "Line feed"],
+            },
+        ],
+    },
+]);

test/tests/operations/StrUtils.js

@@ -0,0 +1,24 @@
+/**
+ * StrUtils tests.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+TestRegister.addTests([
+    {
+        name: "Regex, non-HTML op",
+        input: "/<>",
+        expectedOutput: "/<>",
+        recipeConfig: [
+            {
+                "op": "Regular expression",
+                "args": ["User defined", "", true, true, false, "Highlight matches"]
+            },
+            {
+                "op": "Remove whitespace",
+                "args": [true, true, true, true, true, false]
+            }
+        ],
+    },
+]);