From 33a1d0173aa73a389a4d139e70a6a916bc379a29 Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 5 Jun 2020 10:27:05 +0100 Subject: [PATCH] simplify Recipe API, add flow control checks to node API --- src/core/Chef.mjs | 28 +-- src/core/Recipe.mjs | 307 ++++++++++++++--------------- src/core/lib/Magic.mjs | 4 +- src/core/operations/Fork.mjs | 4 +- src/core/operations/Magic.mjs | 4 +- src/core/operations/Subsection.mjs | 4 +- src/node/api.mjs | 1 + tests/node/tests/operations.mjs | 1 - tests/operations/index.mjs | 1 - 9 files changed, 167 insertions(+), 187 deletions(-) diff --git a/src/core/Chef.mjs b/src/core/Chef.mjs index 36998cec..5eb07517 100755 --- a/src/core/Chef.mjs +++ b/src/core/Chef.mjs @@ -39,12 +39,14 @@ class Chef { */ async bake(input, recipeConfig, options) { log.debug("Chef baking"); - const startTime = Date.now(), - recipe = new Recipe(recipeConfig), - containsFc = recipe.containsFlowControl(), - notUTF8 = options && "treatAsUtf8" in options && !options.treatAsUtf8; - let error = false, - progress = 0; + + const startTime = Date.now(); + const recipe = await Recipe.buildRecipe(recipeConfig); + const containsFc = recipe.state.containsFlowControl(); + const notUTF8 = options && "treatAsUtf8" in options && !options.treatAsUtf8; + + let error = false; + let progress = 0; if (containsFc && isWorkerEnvironment()) self.setOption("attemptHighlight", false); @@ -53,7 +55,7 @@ class Chef { this.dish.set(input, type); try { - progress = await recipe.execute(this.dish, progress); + progress = await recipe.execute(this.dish); } catch (err) { log.error(err); error = { @@ -107,12 +109,12 @@ class Chef { * @param {Object[]} recipeConfig - The recipe configuration object * @returns {number} The time it took to run the silent bake in milliseconds. */ - silentBake(recipeConfig) { + async silentBake(recipeConfig) { log.debug("Running silent bake"); - const startTime = Date.now(), - recipe = new Recipe(recipeConfig), - dish = new Dish(); + const startTime = Date.now(); + const recipe = await Recipe.buildRecipe(recipeConfig); + const dish = new Dish(); try { recipe.execute(dish); @@ -134,8 +136,8 @@ class Chef { * @returns {Object} */ async calculateHighlights(recipeConfig, direction, pos) { - const recipe = new Recipe(recipeConfig); - const highlights = await recipe.generateHighlightList(); + const recipe = await Recipe.buildRecipe(recipeConfig); + const highlights = recipe.generateHighlightList(); if (!highlights) return false; diff --git a/src/core/Recipe.mjs b/src/core/Recipe.mjs index 14cb39b7..42c58af1 100755 --- a/src/core/Recipe.mjs +++ b/src/core/Recipe.mjs @@ -24,69 +24,18 @@ class Recipe { * * @param {Object} recipeConfig */ - constructor(recipeConfig) { - this.opList = []; - - if (recipeConfig) { - this._parseConfig(recipeConfig); - } + constructor(operations=[]) { + this.state = new RecipeState(); + this.state.opList = operations; } - - /** - * Reads and parses the given config. - * - * @private - * @param {Object} recipeConfig - */ - _parseConfig(recipeConfig) { - recipeConfig.forEach(c => { - this.opList.push({ - name: c.op, - module: OperationConfig[c.op].module, - ingValues: c.args, - breakpoint: c.breakpoint, - disabled: c.disabled, - }); - }); - } - - - /** - * Populate elements of opList with operation instances. - * Dynamic import here removes top-level cyclic dependency issue. - * - * @private - */ - async _hydrateOpList() { - if (!modules) { - // Using Webpack Magic Comments to force the dynamic import to be included in the main chunk - // https://webpack.js.org/api/module-methods/ - modules = await import(/* webpackMode: "eager" */ "./config/modules/OpModules.mjs"); - modules = modules.default; - } - - this.opList = this.opList.map(o => { - if (o instanceof Operation) { - return o; - } else { - const op = new modules[o.module][o.name](); - op.ingValues = o.ingValues; - op.breakpoint = o.breakpoint; - op.disabled = o.disabled; - return op; - } - }); - } - - /** * Returns the value of the Recipe as it should be displayed in a recipe config. * * @returns {Object[]} */ get config() { - return this.opList.map(op => ({ + return this.state.opList.map(op => ({ op: op.name, args: op.ingValues, })); @@ -99,7 +48,7 @@ class Recipe { * @param {Operation} operation */ addOperation(operation) { - this.opList.push(operation); + this.state.addOperation(operation); } @@ -110,85 +59,31 @@ class Recipe { */ addOperations(operations) { operations.forEach(o => { - if (o instanceof Operation) { - this.opList.push(o); - } else { - this.opList.push({ - name: o.name, - module: o.module, - ingValues: o.args, - breakpoint: o.breakpoint, - disabled: o.disabled, - }); - } + this.state.addOperation(o); }); } - /** - * Set a breakpoint on a specified Operation. - * - * @param {number} position - The index of the Operation - * @param {boolean} value - */ - setBreakpoint(position, value) { - try { - this.opList[position].breakpoint = value; - } catch (err) { - // Ignore index error - } - } - - - /** - * Remove breakpoints on all Operations in the Recipe up to the specified position. Used by Flow - * Control Fork operation. - * - * @param {number} pos - */ - removeBreaksUpTo(pos) { - for (let i = 0; i < pos; i++) { - this.opList[i].breakpoint = false; - } - } - - - /** - * Returns true if there is a Flow Control Operation in this Recipe. - * - * @returns {boolean} - */ - containsFlowControl() { - return this.opList.reduce((acc, curr) => { - return acc || curr.flowControl; - }, false); - } - - /** * Executes each operation in the recipe over the given Dish. * * @param {Dish} dish - * @param {number} [startFrom=0] - * - The index of the Operation to start executing from * @param {number} [forkState={}] * - If this is a forked recipe, the state of the recipe up to this point * @returns {number} * - The final progress through the recipe */ - async execute(dish, startFrom=0, forkState={}) { - let op, input, output, - numJumps = 0, - numRegisters = forkState.numRegisters || 0; + async execute(dish, forkState={}) { + let op, input, output; + this.state.dish = dish; + this.state.updateForkState(forkState); + this.lastRunOp = null; - if (startFrom === 0) this.lastRunOp = null; + log.debug(`[*] Executing recipe of ${this.state.opList.length} operations`); - await this._hydrateOpList(); - - log.debug(`[*] Executing recipe of ${this.opList.length} operations, starting at ${startFrom}`); - - for (let i = startFrom; i < this.opList.length; i++) { - op = this.opList[i]; + while (this.state.progress < this.state.opList.length) { + const i = this.state.progress; + op = this.state.currentOp; log.debug(`[${i}] ${op.name} ${JSON.stringify(op.ingValues)}`); if (op.disabled) { log.debug("Operation is disabled, skipping"); @@ -200,32 +95,21 @@ class Recipe { } try { - input = await dish.get(op.inputType); + input = await this.state.dish.get(op.inputType); log.debug(`Executing operation '${op.name}'`); if (isWorkerEnvironment()) { - self.sendStatusMessage(`Baking... (${i+1}/${this.opList.length})`); - self.sendProgressMessage(i + 1, this.opList.length); + self.sendStatusMessage(`Baking... (${i+1}/${this.state.opList.length})`); + self.sendProgressMessage(i + 1, this.state.opList.length); } if (op.flowControl) { - // Package up the current state - let state = { - "progress": i, - "dish": dish, - "opList": this.opList, - "numJumps": numJumps, - "numRegisters": numRegisters, - "forkOffset": forkState.forkOffset || 0 - }; - - state = await op.run(state); - i = state.progress; - numJumps = state.numJumps; - numRegisters = state.numRegisters; + this.state = await op.run(this.state); + this.state.progress++; } else { output = await op.run(input, op.ingValues); - dish.set(output, op.outputType); + this.state.dish.set(output, op.outputType); + this.state.progress++; } this.lastRunOp = op; } catch (err) { @@ -234,11 +118,11 @@ class Recipe { (err.type && err.type === "OperationError")) { // Cannot rely on `err instanceof OperationError` here as extending // native types is not fully supported yet. - dish.set(err.message, "string"); + this.state.dish.set(err.message, "string"); return i; } else if (err instanceof DishError || (err.type && err.type === "DishError")) { - dish.set(err.message, "string"); + this.state.dish.set(err.message, "string"); return i; } else { const e = typeof err == "string" ? { message: err } : err; @@ -257,7 +141,7 @@ class Recipe { } log.debug("Recipe complete"); - return this.opList.length; + return this.state.opList.length; } @@ -287,17 +171,6 @@ class Recipe { } - /** - * Creates a Recipe from a given configuration string. - * - * @param {string} recipeStr - */ - fromString(recipeStr) { - const recipeConfig = JSON.parse(recipeStr); - this._parseConfig(recipeConfig); - } - - /** * Generates a list of all the highlight functions assigned to operations in the recipe, if the * entire recipe supports highlighting. @@ -307,13 +180,15 @@ class Recipe { * @returns {function} highlights[].b * @returns {Object[]} highlights[].args */ - async generateHighlightList() { - await this._hydrateOpList(); + generateHighlightList() { const highlights = []; - - for (let i = 0; i < this.opList.length; i++) { - const op = this.opList[i]; - if (op.disabled) continue; + while (this.state.progress < this.state.opList.length) { + // for (let i = 0; i < this.state.opList.length; i++) { + const op = this.state.currentOp; + if (op.disabled) { + this.state.progress++; + continue; + } // If any breakpoints are set, do not attempt to highlight if (op.breakpoint) return false; @@ -333,16 +208,120 @@ class Recipe { /** - * Determines whether the previous operation has a different presentation type to its normal output. + * Build a recipe using a recipeConfig * - * @param {number} progress - * @returns {boolean} + * Hydrate the recipeConfig before using the hydrated operations + * in the Recipe constructor. This decouples the hydration of + * the operations from the Recipe logic. + * + * @param {recipeConfig} recipeConfig */ - lastOpPresented(progress) { - if (progress < 1) return false; - return this.opList[progress-1].presentType !== this.opList[progress-1].outputType; + static async buildRecipe(recipeConfig) { + const operations = []; + recipeConfig.forEach(c => { + operations.push({ + name: c.op, + module: OperationConfig[c.op].module, + ingValues: c.args, + breakpoint: c.breakpoint, + disabled: c.disabled, + }); + }); + + if (!modules) { + // Using Webpack Magic Comments to force the dynamic import to be included in the main chunk + // https://webpack.js.org/api/module-methods/ + modules = await import(/* webpackMode: "eager" */ "./config/modules/OpModules.mjs"); + modules = modules.default; + } + + const hydratedOperations = operations.map(o => { + if (o instanceof Operation) { + return o; + } else { + const op = new modules[o.module][o.name](); + op.ingValues = o.ingValues; + op.breakpoint = o.breakpoint; + op.disabled = o.disabled; + return op; + } + }); + + return new Recipe(hydratedOperations); } } + +/** + * Encapsulate the state of a Recipe + * + * Encapsulating the state makes it cleaner when passing the state + * between operations. + */ +class RecipeState { + + /** + * initialise a RecipeState + */ + constructor() { + this.dish = null; + this.opList = []; + this.progress = 0; + this.forkOffset = 0; + this.numRegisters = 0; + this.numJumps = 0; + } + + /** + * get the next operation due to be run. + * @return {Operation} + */ + get currentOp() { + return this.opList[this.progress]; + } + + /** + * add an operation to the end of RecipeState's opList + * @param {Operation} operation + */ + addOperation(operation) { + this.opList.push(operation); + } + + /** + * @returns {boolean} whether there's a flowControl operation in + * the RecipeState's opList + */ + containsFlowControl() { + return this.opList.reduce((p, c) => { + return p || c.flowControl; + }, false); + } + + /** + * Update the RecipeState with state from a fork. Used at the end + * of the Fork operation. + * @param {Object} forkState + */ + updateForkState(forkState) { + if (forkState.progress || forkState.progress === 0) { + this.progress = forkState.progress; + } + + if (forkState.numRegisters || forkState.numRegisters === 0) { + this.numRegisters = forkState.numRegisters; + } + + if (forkState.numJumps || forkState.numJumps === 0) { + this.numJumps = forkState.numJumps; + } + + if (forkState.forkOffset || forkState.forkOffset === 0) { + this.forkOffset = forkState.forkOffset; + } + } +} + export default Recipe; +export { RecipeState }; diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 983b9aed..1f0c0d26 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -449,11 +449,11 @@ class Magic { if (isWorkerEnvironment()) self.loadRequiredModules(recipeConfig); - const recipe = new Recipe(recipeConfig); + const recipe = await Recipe.buildRecipe(recipeConfig); try { await recipe.execute(dish); // Return an empty buffer if the recipe did not run to completion - if (recipe.lastRunOp === recipe.opList[recipe.opList.length - 1]) { + if (recipe.lastRunOp === recipe.state.opList[recipe.state.opList.length - 1]) { return await dish.get(Dish.ARRAY_BUFFER); } else { return new ArrayBuffer(); diff --git a/src/core/operations/Fork.mjs b/src/core/operations/Fork.mjs index 6e961990..0d012410 100644 --- a/src/core/operations/Fork.mjs +++ b/src/core/operations/Fork.mjs @@ -89,7 +89,7 @@ class Fork extends Operation { // Run recipe over each tranche for (i = 0; i < inputs.length; i++) { // Baseline ing values for each tranche so that registers are reset - recipe.opList.forEach((op, i) => { + recipe.state.opList.forEach((op, i) => { op.ingValues = JSON.parse(JSON.stringify(ingValues[i])); }); @@ -97,7 +97,7 @@ class Fork extends Operation { dish.set(inputs[i], inputType); try { - progress = await recipe.execute(dish, 0, state); + progress = await recipe.execute(dish, state); } catch (err) { if (!ignoreErrors) { throw err; diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index d5357d95..576bdcc4 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -112,9 +112,9 @@ class Magic extends Operation { options.forEach(option => { // Construct recipe URL // Replace this Magic op with the generated recipe - const recipeConfig = currentRecipeConfig.slice(0, this.state.progress) + const recipeConfig = currentRecipeConfig.slice(0, this.state.progress - 1) .concat(option.recipe) - .concat(currentRecipeConfig.slice(this.state.progress + 1)), + .concat(currentRecipeConfig.slice(this.state.progress)), recipeURL = "recipe=" + Utils.encodeURIFragment(Utils.generatePrettyRecipe(recipeConfig)); let language = "", diff --git a/src/core/operations/Subsection.mjs b/src/core/operations/Subsection.mjs index 86980e40..e668f172 100644 --- a/src/core/operations/Subsection.mjs +++ b/src/core/operations/Subsection.mjs @@ -115,7 +115,7 @@ class Subsection extends Operation { } // Baseline ing values for each tranche so that registers are reset - recipe.opList.forEach((op, i) => { + recipe.state.opList.forEach((op, i) => { op.ingValues = JSON.parse(JSON.stringify(ingValues[i])); }); @@ -123,7 +123,7 @@ class Subsection extends Operation { dish.set(matchStr, inputType); try { - progress = await recipe.execute(dish, 0, state); + progress = await recipe.execute(dish, state); } catch (err) { if (!ignoreErrors) { throw err; diff --git a/src/node/api.mjs b/src/node/api.mjs index 5733fb1d..b1f913ed 100644 --- a/src/node/api.mjs +++ b/src/node/api.mjs @@ -192,6 +192,7 @@ export function _wrap(OpClass) { wrapped = async (input, args=null) => { const {transformedInput, transformedArgs} = prepareOp(opInstance, input, args); const result = await opInstance.run(transformedInput, transformedArgs); + return new NodeDish({ value: result, type: opInstance.outputType, diff --git a/tests/node/tests/operations.mjs b/tests/node/tests/operations.mjs index baf3f238..2c71cdc9 100644 --- a/tests/node/tests/operations.mjs +++ b/tests/node/tests/operations.mjs @@ -1075,6 +1075,5 @@ ExifImageHeight: 57`); assert.equal(output, res.value); }), - ]); diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 8d3cd623..c8b08651 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -79,7 +79,6 @@ import "./tests/StrUtils.mjs"; import "./tests/SymmetricDifference.mjs"; import "./tests/TextEncodingBruteForce.mjs"; import "./tests/TranslateDateTimeFormat.mjs"; -import "./tests/Magic.mjs"; import "./tests/ParseTLV.mjs"; import "./tests/Media.mjs"; import "./tests/ToFromInsensitiveRegex.mjs";