diff --git a/src/core/Chef.mjs b/src/core/Chef.mjs index 9c52695e..a935d75c 100755 --- a/src/core/Chef.mjs +++ b/src/core/Chef.mjs @@ -91,8 +91,8 @@ class Chef { return { result: this.dish.type === Dish.HTML ? - this.dish.get(Dish.HTML, notUTF8) : - this.dish.get(returnType, notUTF8), + await this.dish.get(Dish.HTML, notUTF8) : + await this.dish.get(returnType, notUTF8), type: Dish.enumLookup(this.dish.type), progress: progress, duration: new Date().getTime() - startTime, diff --git a/src/core/Dish.mjs b/src/core/Dish.mjs index 792395c1..6aeaf3e9 100755 --- a/src/core/Dish.mjs +++ b/src/core/Dish.mjs @@ -51,6 +51,8 @@ class Dish { case "bignumber": case "big number": return Dish.BIG_NUMBER; + case "list": + return Dish.LIST_FILE; default: throw "Invalid data type string. No matching enum."; } @@ -77,6 +79,8 @@ class Dish { return "ArrayBuffer"; case Dish.BIG_NUMBER: return "BigNumber"; + case Dish.LIST_FILE: + return "List"; default: throw "Invalid data type enum. No matching type."; } @@ -86,7 +90,7 @@ class Dish { /** * Sets the data value and type and then validates them. * - * @param {byteArray|string|number|ArrayBuffer|BigNumber} value + * @param {*} value * - The value of the input data. * @param {number} type * - The data type of value, see Dish enums. @@ -112,15 +116,14 @@ class Dish { * * @param {number} type - The data type of value, see Dish enums. * @param {boolean} [notUTF8=false] - Do not treat strings as UTF8. - * @returns {byteArray|string|number|ArrayBuffer|BigNumber} - * The value of the output data. + * @returns {*} - The value of the output data. */ - get(type, notUTF8=false) { + async get(type, notUTF8=false) { if (typeof type === "string") { type = Dish.typeEnum(type); } if (this.type !== type) { - this.translate(type, notUTF8); + await this._translate(type, notUTF8); } return this.value; } @@ -132,7 +135,7 @@ class Dish { * @param {number} toType - The data type of value, see Dish enums. * @param {boolean} [notUTF8=false] - Do not treat strings as UTF8. */ - translate(toType, notUTF8=false) { + async _translate(toType, notUTF8=false) { log.debug(`Translating Dish from ${Dish.enumLookup(this.type)} to ${Dish.enumLookup(toType)}`); const byteArrayToStr = notUTF8 ? Utils.byteArrayToChars : Utils.byteArrayToUtf8; @@ -142,7 +145,7 @@ class Dish { this.value = this.value ? Utils.strToByteArray(this.value) : []; break; case Dish.NUMBER: - this.value = typeof this.value == "number" ? Utils.strToByteArray(this.value.toString()) : []; + this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : []; break; case Dish.HTML: this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : []; @@ -154,6 +157,11 @@ class Dish { case Dish.BIG_NUMBER: this.value = this.value instanceof BigNumber ? Utils.strToByteArray(this.value.toFixed()) : []; break; + case Dish.LIST_FILE: + this.value = await Promise.all(this.value.map(async f => Utils.readFile(f))); + this.value = this.value.map(b => Array.prototype.slice.call(b)); + this.value = [].concat.apply([], this.value); + break; default: break; } @@ -183,6 +191,10 @@ class Dish { } this.type = Dish.BIG_NUMBER; break; + case Dish.LIST_FILE: + this.value = new File(this.value, "unknown"); + this.type = Dish.LIST_FILE; + break; default: break; } @@ -220,6 +232,9 @@ class Dish { return this.value instanceof ArrayBuffer; case Dish.BIG_NUMBER: return this.value instanceof BigNumber; + case Dish.LIST_FILE: + return this.value instanceof Array && + this.value.reduce((acc, curr) => acc && curr instanceof File, true); default: return false; } @@ -244,6 +259,8 @@ class Dish { return this.value.toString().length; case Dish.ARRAY_BUFFER: return this.value.byteLength; + case Dish.LIST_FILE: + return this.value.reduce((acc, curr) => acc + curr.size, 0); default: return -1; } @@ -288,6 +305,12 @@ Dish.ARRAY_BUFFER = 4; * @enum */ Dish.BIG_NUMBER = 5; +/** + * Dish data type enum for lists of files. + * @readonly + * @enum + */ +Dish.LIST_FILE = 6; export default Dish; diff --git a/src/core/FlowControl.js b/src/core/FlowControl.js index 73f29d78..92440c49 100755 --- a/src/core/FlowControl.js +++ b/src/core/FlowControl.js @@ -26,7 +26,7 @@ const FlowControl = { const opList = state.opList, inputType = opList[state.progress].inputType, outputType = opList[state.progress].outputType, - input = state.dish.get(inputType), + input = await state.dish.get(inputType), ings = opList[state.progress].ingValues, splitDelim = ings[0], mergeDelim = ings[1], @@ -77,7 +77,7 @@ const FlowControl = { } progress = err.progress + 1; } - output += dish.get(outputType) + mergeDelim; + output += await dish.get(outputType) + mergeDelim; } state.dish.set(output, outputType); @@ -111,7 +111,7 @@ const FlowControl = { * @param {Operation[]} state.opList - The list of operations in the recipe. * @returns {Object} The updated state of the recipe. */ - runRegister: function(state) { + runRegister: async function(state) { const ings = state.opList[state.progress].ingValues, extractorStr = ings[0], i = ings[1], @@ -122,7 +122,7 @@ const FlowControl = { if (m) modifiers += "m"; const extractor = new RegExp(extractorStr, modifiers), - input = state.dish.get(Dish.STRING), + input = await state.dish.get(Dish.STRING), registers = input.match(extractor); if (!registers) return state; @@ -208,7 +208,7 @@ const FlowControl = { * @param {number} state.numJumps - The number of jumps taken so far. * @returns {Object} The updated state of the recipe. */ - runCondJump: function(state) { + runCondJump: async function(state) { const ings = state.opList[state.progress].ingValues, dish = state.dish, regexStr = ings[0], @@ -223,7 +223,7 @@ const FlowControl = { } if (regexStr !== "") { - const strMatch = dish.get(Dish.STRING).search(regexStr) > -1; + const strMatch = await dish.get(Dish.STRING).search(regexStr) > -1; if (!invert && strMatch || invert && !strMatch) { state.progress = jmpIndex; state.numJumps++; diff --git a/src/core/Operation.mjs b/src/core/Operation.mjs index 30ad71e4..e87c31f4 100755 --- a/src/core/Operation.mjs +++ b/src/core/Operation.mjs @@ -19,6 +19,7 @@ class Operation { // Private fields this._inputType = -1; this._outputType = -1; + this._presentType = -1; this._breakpoint = false; this._disabled = false; this._flowControl = false; @@ -71,6 +72,22 @@ class Operation { } + /** + * Method to be called when displaying the result of an operation in a human-readable + * format. This allows operations to return usable data from their run() method and + * only format them when this method is called. + * + * The default action is to return the data unchanged, but child classes can override + * this behaviour. + * + * @param {*} data - The result of the run() function + * @returns {*} - A human-readable version of the data + */ + present(data) { + return data; + } + + /** * Sets the input type as a Dish enum. * @@ -98,6 +115,7 @@ class Operation { */ set outputType(typeStr) { this._outputType = Dish.typeEnum(typeStr); + if (this._presentType < 0) this._presentType = this._outputType; } @@ -111,6 +129,26 @@ class Operation { } + /** + * Sets the presentation type as a Dish enum. + * + * @param {string} typeStr + */ + set presentType(typeStr) { + this._presentType = Dish.typeEnum(typeStr); + } + + + /** + * Gets the presentation type as a readable string. + * + * @returns {string} + */ + get presentType() { + return Dish.enumLookup(this._presentType); + } + + /** * Sets the args for the current operation. * diff --git a/src/core/Recipe.mjs b/src/core/Recipe.mjs index 7afaeab9..14792157 100755 --- a/src/core/Recipe.mjs +++ b/src/core/Recipe.mjs @@ -130,7 +130,7 @@ class Recipe { * - The final progress through the recipe */ async execute(dish, startFrom=0, forkState={}) { - let op, input, output, + let op, input, output, lastRunOp, numJumps = 0, numRegisters = forkState.numRegisters || 0; @@ -149,7 +149,7 @@ class Recipe { } try { - input = dish.get(op.inputType); + input = await dish.get(op.inputType); log.debug("Executing operation"); if (op.flowControl) { @@ -169,6 +169,7 @@ class Recipe { numRegisters = state.numRegisters; } else { output = await op.run(input, op.ingValues); + lastRunOp = op; dish.set(output, op.outputType); } } catch (err) { @@ -187,6 +188,11 @@ class Recipe { } } + // Present the results of the final operation + // TODO try/catch + output = await lastRunOp.present(output); + dish.set(output, lastRunOp.presentType); + log.debug("Recipe complete"); return this.opList.length; } diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index 2be207eb..88cfa52e 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -812,35 +812,30 @@ class Utils { /** * Formats a list of files or directories. - * A File is an object with a "fileName" and optionally a "contents". - * If the fileName ends with "/" and the contents is of length 0 then - * it is considered a directory. * * @author tlwr [toby@toby.codes] + * @author n1474335 [n1474335@gmail.com] * - * @param {Object[]} files + * @param {File[]} files * @returns {html} */ - static displayFilesAsHTML(files) { - /* and used to denote newlines and spaces in HTML markup. - * If a non-html operation is used, all markup will be removed but these - * whitespace chars will remain for formatting purposes. - */ - + static async displayFilesAsHTML(files) { const formatDirectory = function(file) { const html = `
`; return html; }; - const formatFile = function(file, i) { + const formatFile = async function(file, i) { + const buff = await Utils.readFile(file); + const fileStr = Utils.arrayBufferToStr(buff.buffer); const blob = new Blob( - [new Uint8Array(file.bytes)], + [buff], {type: "octet/stream"} ); const blobUrl = URL.createObjectURL(blob); @@ -850,13 +845,13 @@ class Utils { data-toggle='collapse' aria-expanded='true' aria-controls='collapse${i}' - title="Show/hide contents of '${Utils.escapeHtml(file.fileName)}'">👁️`; + title="Show/hide contents of '${Utils.escapeHtml(file.name)}'">👁️`; const downloadFileElem = `💾`; + title='Download ${Utils.escapeHtml(file.name)}' + download='${Utils.escapeHtml(file.name)}'>💾`; - const hexFileData = toHexFast(new Uint8Array(file.bytes)); + const hexFileData = toHexFast(buff); const switchToInputElem = `

- ${Utils.escapeHtml(file.fileName)} - ${viewFileElem} - ${downloadFileElem} - ${switchToInputElem} + ${Utils.escapeHtml(file.name)} + ${viewFileElem} + ${downloadFileElem} + ${switchToInputElem} - ${file.size.toLocaleString()} bytes + ${file.size.toLocaleString()} bytes

@@ -880,7 +875,7 @@ class Utils {
- 
${Utils.escapeHtml(file.contents)}
+ 
${Utils.escapeHtml(fileStr)}
`; @@ -891,17 +886,15 @@ class Utils { ${files.length} file(s) found `; - files.forEach(function(file, i) { - if (typeof file.contents !== "undefined") { - html += formatFile(file, i); + for (let i = 0; i < files.length; i++) { + if (files[i].name.endsWith("/")) { + html += formatDirectory(files[i]); } else { - html += formatDirectory(file); + html += await formatFile(files[i], i); } - }); + } - return html.replace(/(?:(
(?:\n|.)*<\/pre>)|\s{2,})/g, "$1") // Remove whitespace from markup
-            .replace(//g, "\n") // Replace  with newlines
-            .replace(//g, " "); // Replace  with spaces
+        return html;
     }
 
 
@@ -941,6 +934,47 @@ class Utils {
     }
 
 
+    /**
+     * Reads a File and returns the data as a Uint8Array.
+     *
+     * @param {File} file
+     * @returns {Uint8Array}
+     *
+     * @example
+     * // returns Uint8Array(5) [104, 101, 108, 108, 111]
+     * await Utils.readFile(new File(["hello"], "test"))
+     */
+    static readFile(file) {
+        return new Promise((resolve, reject) => {
+            const reader = new FileReader();
+            const data = new Uint8Array(file.size);
+            let offset = 0;
+            const CHUNK_SIZE = 10485760; // 10MiB
+
+            const seek = function() {
+                if (offset >= file.size) {
+                    resolve(data);
+                    return;
+                }
+                const slice = file.slice(offset, offset + CHUNK_SIZE);
+                reader.readAsArrayBuffer(slice);
+            };
+
+            reader.onload = function(e) {
+                data.set(new Uint8Array(reader.result), offset);
+                offset += CHUNK_SIZE;
+                seek();
+            };
+
+            reader.onerror = function(e) {
+                reject(reader.error.message);
+            };
+
+            seek();
+        });
+    }
+
+
     /**
      * Actual modulo function, since % is actually the remainder function in JS.
      *
diff --git a/src/core/config/scripts/generateConfig.mjs b/src/core/config/scripts/generateConfig.mjs
index b8905d50..51af293a 100644
--- a/src/core/config/scripts/generateConfig.mjs
+++ b/src/core/config/scripts/generateConfig.mjs
@@ -38,7 +38,7 @@ for (const opObj in Ops) {
         module: op.module,
         description: op.description,
         inputType: op.inputType,
-        outputType: op.outputType,
+        outputType: op.presentType,
         flowControl: op.flowControl,
         args: op.args
     };
diff --git a/src/core/operations/Unzip.mjs b/src/core/operations/Unzip.mjs
index 1444551a..07f2a65e 100644
--- a/src/core/operations/Unzip.mjs
+++ b/src/core/operations/Unzip.mjs
@@ -25,7 +25,8 @@ class Unzip extends Operation {
         this.module = "Compression";
         this.description = "Decompresses data using the PKZIP algorithm and displays it per file, with support for passwords.";
         this.inputType = "byteArray";
-        this.outputType = "html";
+        this.outputType = "List";
+        this.presentType = "html";
         this.args = [
             {
                 name: "Password",
@@ -43,7 +44,7 @@ class Unzip extends Operation {
     /**
      * @param {byteArray} input
      * @param {Object[]} args
-     * @returns {html}
+     * @returns {File[]}
      */
     run(input, args) {
         const options = {
@@ -51,28 +52,22 @@ class Unzip extends Operation {
                 verify: args[1]
             },
             unzip = new Zlib.Unzip(input, options),
-            filenames = unzip.getFilenames(),
-            files = [];
+            filenames = unzip.getFilenames();
 
-        filenames.forEach(function(fileName) {
+        return filenames.map(fileName => {
             const bytes = unzip.decompress(fileName);
-            const contents = Utils.byteArrayToUtf8(bytes);
-
-            const file = {
-                fileName: fileName,
-                size: contents.length,
-            };
-
-            const isDir = contents.length === 0 && fileName.endsWith("/");
-            if (!isDir) {
-                file.bytes = bytes;
-                file.contents = contents;
-            }
-
-            files.push(file);
+            return new File([bytes], fileName);
         });
+    }
 
-        return Utils.displayFilesAsHTML(files);
+    /**
+     * Displays the files in HTML for web apps.
+     *
+     * @param {File[]} files
+     * @returns {html}
+     */
+    async present(files) {
+        return await Utils.displayFilesAsHTML(files);
     }
 
 }