From 40c7e6304664358bb33a336f181ca7d778a25bc2 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Mon, 2 Dec 2019 15:47:02 +0000 Subject: [PATCH] Magic progress --- src/core/Operation.mjs | 3 + src/core/config/scripts/generateConfig.mjs | 10 +- src/core/lib/Magic.mjs | 117 ++++++++----------- src/core/lib/MagicObject.mjs | 15 +++ src/core/lib/Test.mjs | 127 +++++++++++++++++++++ 5 files changed, 199 insertions(+), 73 deletions(-) create mode 100644 src/core/lib/MagicObject.mjs create mode 100644 src/core/lib/Test.mjs diff --git a/src/core/Operation.mjs b/src/core/Operation.mjs index 32ecff07..1da3b3ea 100755 --- a/src/core/Operation.mjs +++ b/src/core/Operation.mjs @@ -25,6 +25,9 @@ class Operation { this._flowControl = false; this._manualBake = false; this._ingList = []; + // this._checks = []; + this._inputRegexes = []; + this._outputRegexes = []; // Public fields this.name = ""; diff --git a/src/core/config/scripts/generateConfig.mjs b/src/core/config/scripts/generateConfig.mjs index e8f99e40..492a000c 100644 --- a/src/core/config/scripts/generateConfig.mjs +++ b/src/core/config/scripts/generateConfig.mjs @@ -45,8 +45,14 @@ for (const opObj in Ops) { args: op.args }; - if ("patterns" in op) { - operationConfig[op.name].patterns = op.patterns; + if ("checks" in op) { + if ("outRegexes" in op.checks) { + operationConfig[op.name].outputRegexes = op.checks.outRegexes; + } + + if ("inRegexes" in op.checks) { + operationConfig[op.name].inputRegexes = op.checks.inRegexes; + } } if (!(op.module in modules)) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 79c64452..050772e3 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -4,6 +4,7 @@ import Recipe from "../Recipe.mjs"; import Dish from "../Dish.mjs"; import {detectFileType} from "./FileType.mjs"; import chiSquared from "chi-squared"; +import potentialOps from "./Test.mjs"; /** * A class for detecting encodings, file types and byte frequencies and @@ -24,28 +25,7 @@ class Magic { constructor(buf, opPatterns) { this.inputBuffer = new Uint8Array(buf); this.inputStr = Utils.arrayBufferToStr(buf); - this.opPatterns = opPatterns || Magic._generateOpPatterns(); - } - - /** - * Finds operations that claim to be able to decode the input based on regular - * expression matches. - * - * @returns {Object[]} - */ - findMatchingOps() { - const matches = []; - - for (let i = 0; i < this.opPatterns.length; i++) { - const pattern = this.opPatterns[i], - regex = new RegExp(pattern.match, pattern.flags); - - if (regex.test(this.inputStr)) { - matches.push(pattern); - } - } - - return matches; + this.opPatterns = new potentialOps(opPatterns); } /** @@ -281,7 +261,7 @@ class Magic { if (depth < 0) return []; // Find any operations that can be run on this data - const matchingOps = this.findMatchingOps(); + const matchingOps = this.opPatterns.findMatchingInputRegexes(this.inputStr); let results = []; @@ -299,32 +279,49 @@ class Magic { }); const prevOp = recipeConfig[recipeConfig.length - 1]; - // Execute each of the matching operations, then recursively call the speculativeExecution() - // method on the resulting data, recording the properties of each option. - await Promise.all(matchingOps.map(async op => { - const opConfig = { - op: op.op, - args: op.args - }, - output = await this._runRecipe([opConfig]); + async function regexesTests(flag, sensible) { - // If the recipe is repeating and returning the same data, do not continue - if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) { - return; - } + // Execute each of the matching operations, then recursively call the speculativeExecution() + // method on the resulting data, recording the properties of each option. + await Promise.all(sensible.map(async op => { + const opConfig = { + op: op.op, + args: op.args + }, + output = await this._runRecipe([opConfig]); - // If the recipe returned an empty buffer, do not continue - if (_buffersEqual(output, new ArrayBuffer())) { - return; - } + // If the recipe is repeating and returning the same data, do not continue + if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) { + return; + } - const magic = new Magic(output, this.opPatterns), - speculativeResults = await magic.speculativeExecution( - depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib); - - results = results.concat(speculativeResults); - })); + // If the recipe returned an empty buffer, do not continue + if (_buffersEqual(output, new ArrayBuffer())) { + return; + } + if(flag) { + const outputRegexes = OperationConfig[op.op].outputRegexes; + if (outputRegexes) + for (const pattern of outputRegexes) + if (!(new RegExp(pattern.match, pattern.flags).test(Utils.arrayBufferToStr(output)))) + return; + } else { + if (!(op.match.test(output))){ + return; + } + } + const magic = new Magic(output, this.opPatterns), + speculativeResults = await magic.speculativeExecution( + depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib); + results = results.concat(speculativeResults); + })); + } + + regexesTests(1, matchingOps); + regexesTests(0, this.opPatterns.getOutputRegexes()); + //console.log("haha",this.opPatterns.getOutputRegexes()); + if (intensive) { // Run brute forcing of various types on the data and create a new branch for each option const bfEncodings = await this.bruteForce(); @@ -338,6 +335,8 @@ class Magic { })); } + // console.log(results); + // Prune branches that result in unhelpful outputs const prunedResults = results.filter(r => (r.useful || r.data.length > 0) && // The operation resulted in "" @@ -349,6 +348,8 @@ class Magic { ) ); + // console.log(prunedResults); + // Return a sorted list of possible recipes along with their properties return prunedResults.sort((a, b) => { // Each option is sorted based on its most likely language (lower is better) @@ -441,32 +442,6 @@ class Magic { return this.freqDist; } - /** - * Generates a list of all patterns that operations claim to be able to decode. - * - * @private - * @returns {Object[]} - */ - static _generateOpPatterns() { - const opPatterns = []; - - for (const op in OperationConfig) { - if (!("patterns" in OperationConfig[op])) continue; - - OperationConfig[op].patterns.forEach(pattern => { - opPatterns.push({ - op: op, - match: pattern.match, - flags: pattern.flags, - args: pattern.args, - useful: pattern.useful || false - }); - }); - } - - return opPatterns; - } - /** * Calculates Pearson's Chi-Squared test for two frequency arrays. * https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test diff --git a/src/core/lib/MagicObject.mjs b/src/core/lib/MagicObject.mjs new file mode 100644 index 00000000..0c605e76 --- /dev/null +++ b/src/core/lib/MagicObject.mjs @@ -0,0 +1,15 @@ +/** + * + */ +class magicObject { + /** + * + * @param inRegexes + * @param outRegexes + */ + constructor (inRegexes = null, outRegexes = null) { + this.inRegexes = inRegexes; + this.outRegexes = outRegexes; + } + +} export default magicObject; diff --git a/src/core/lib/Test.mjs b/src/core/lib/Test.mjs new file mode 100644 index 00000000..70d7878b --- /dev/null +++ b/src/core/lib/Test.mjs @@ -0,0 +1,127 @@ +import OperationConfig from "../config/OperationConfig.json"; + +/** + * + */ +class potentialOps { + + /** + * + * @param prevOp + */ + constructor (prevOp) { + if (typeof prevOp === "undefined") { + this.inputRegexes = this.generateInputOpPatterns(); + this.hasInputRegexes = true; + this.outputRegexes = this.generateOutputOpPatterns(); + this.hasOutputRegexes = true; + } else { + this.inputRegexes = prevOp.getInputRegexes(); + this.hasInputRegexes = true; + this.outputRegexes = prevOp.getOutputRegexes(); + this.hasOutputRegexes = true; + } + } + + /** + * + * @param outputRegexes + */ + setOutputRegexes (outputRegexes) { + this.outputRegexes = [...outputRegexes]; + this.hasOutputRegexes = true; + } + + /** + * + * @param inputRegexes + */ + setInputRegexes (inputRegexes) { + this.inputRegexes = [...inputRegexes]; + this.hasInputRegexes = true; + } + + /** + * + */ + getInputRegexes () { + return this.inputRegexes; + } + + /** + * + */ + getOutputRegexes () { + return this.outputRegexes; + } + + /** + * Finds operations that claim to be able to decode the input based on regular + * expression matches. + * + * @returns {Object[]} + */ + findMatchingInputRegexes (inputStr) { + const matches = []; + + for (const pattern of this.inputRegexes) { + const regex = new RegExp(pattern.match, pattern.flags); + + if (regex.test(inputStr)) { + matches.push(pattern); + } + } + + return matches; + } + + + /** + * Generates a list of all patterns that operations claim to be able to decode. + * + * @private + * @returns {Object[]} + */ + generateInputOpPatterns() { + const opPatterns = []; + for (const op in OperationConfig) { + if (!("inputRegexes" in OperationConfig[op])) continue; + + OperationConfig[op].inputRegexes.forEach(pattern => { + opPatterns.push({ + op: op, + match: pattern.match, + flags: pattern.flags, + args: pattern.args, + useful: pattern.useful || false + }); + }); + } + + return opPatterns; + } + + /** + * + */ + generateOutputOpPatterns() { + const opPatterns = []; + for (const op in OperationConfig) { + if (!(OperationConfig[op].outputRegexes)) continue; + + OperationConfig[op].outputRegexes.forEach(pattern => { + opPatterns.push({ + op: op, + match: pattern.match, + flags: pattern.flags, + shouldMatch: pattern.shouldMatch, + args: pattern.args, + useful: pattern.useful || false + }); + }); + } + + return opPatterns; + } + +} export default potentialOps;