From ad0c04884dca146e57c72e02831ee9e3f0d239fd Mon Sep 17 00:00:00 2001 From: n1073645 Date: Fri, 20 Dec 2019 15:17:28 +0000 Subject: [PATCH] Needs tiding but it does function --- src/core/operations/ExtractEntropies.mjs | 400 +++++++++++++++++++---- 1 file changed, 339 insertions(+), 61 deletions(-) diff --git a/src/core/operations/ExtractEntropies.mjs b/src/core/operations/ExtractEntropies.mjs index 32a9ee6c..4289a3fb 100644 --- a/src/core/operations/ExtractEntropies.mjs +++ b/src/core/operations/ExtractEntropies.mjs @@ -6,7 +6,7 @@ import Operation from "../Operation.mjs"; import OperationError from "../errors/OperationError.mjs"; -//import Entropy from "./Entropy.mjs"; +import Utils from "../Utils.mjs"; /** * Extract Entropies operation @@ -24,7 +24,8 @@ class ExtractEntropies extends Operation { this.description = ""; this.infoURL = ""; this.inputType = "ArrayBuffer"; - this.outputType = "List"; + this.outputType = "JSON"; + this.presentType = "html"; this.args = [ { name: "Entropies", @@ -43,7 +44,7 @@ class ExtractEntropies extends Operation { off: [2, 3] }, { - name: "Ascii Range", + name: "English Text", off: [2, 3] }, { @@ -54,8 +55,8 @@ class ExtractEntropies extends Operation { }, { name: "Block Size", - type: "number", - value: "8" + type: "string", + value: "10" }, { name: "Lower Entropy Bound", @@ -66,11 +67,21 @@ class ExtractEntropies extends Operation { name: "Upper Entropy Bound", type: "number", value: 0 + }, + { + name: "Algorithm To Use", + type: "option", + value: ["Absolute Mean Deviation", "Standard Deviation"] + }, + { + name: "Flip Output", + type: "boolean", + value: false } ]; } - /** + /** * Calculates the frequency of bytes in the input. * * @param {Uint8Array} input @@ -106,7 +117,7 @@ class ExtractEntropies extends Operation { } /** - * Calculates the scanning entropy of the input + * Calculates the scanning entropy of the input. * * @param {Uint8Array} inputBytes * @returns {Object} @@ -123,94 +134,361 @@ class ExtractEntropies extends Operation { return { entropyData, binWidth }; } + /** + * Calculates the average of a list of entropies. + * + * @param {Array} entropies + * @returns {number} + */ generateAverage(entropies) { return entropies.reduce((previous, current) => current += previous) / entropies.length; } + /** + * Caculates the mean of the data above the original mean. + * + * @param {Array} entropies + * @param {number} meanEntropy + * @returns {number} + */ getAboveMean(entropies, meanEntropy) { - const result = []; - entropies.forEach((element) =>{ - if (element > meanEntropy) - result.push(element - meanEntropy); + let total = 0, count = 0; + entropies.forEach((element) => { + if (element > meanEntropy) { + total += (element - meanEntropy); + count++; + } }); - return this.generateAverage(result); - } - - getBelowMean(entropies, meanEntropy) { - const result = []; - entropies.forEach((element) =>{ - if (element < meanEntropy) - result.push(meanEntropy - element); - }); - return this.generateAverage(result); + return total/count; } + /** + * Caculates the mean of the data below the original mean. + * + * @param {Array} entropies + * @param {number} meanEntropy + * @returns {number} + */ + getBelowMean(entropies, meanEntropy) { + let total = 0, count = 0; + entropies.forEach((element) => { + if (element < meanEntropy) { + total += (meanEntropy - element); + count++; + } + }); + return total/count; + } + + /** + * Retrieves all the blocks with entropy higher than the high mean. + * + * @param {Array} entropies + * @param {number} highEntropy + * @returns {Array} + */ getAllAbove(entropies, highEntropy) { const result = []; entropies.forEach((element, index) => { - if(element > highEntropy) - result.push([element, index]); - }); - return result; - } - - getAllBelow(entropies, lowEntropy) { - const result = []; - entropies.forEach((element, index) => { - if(element < lowEntropy) + if (element > highEntropy) result.push([element, index]); }); return result; } + /** + * Retrieves all the blocks with entropy lower than the low mean. + * + * @param {Array} entropies + * @param {number} highEntropy + * @returns {Array} + */ + getAllBelow(entropies, lowEntropy) { + const result = []; + entropies.forEach((element, index) => { + if (element < lowEntropy) + result.push([element, index]); + }); + return result; + } + + /** + * Calculates the standard deviation of all of the entropies. + * + * @param {Array} entropies + * @param {number} meanEntropy + * @returns {number} + */ + calculateStandardDeviation(entropies, meanEntropy) { + let total = 0; + entropies.forEach((element) => { + total += Math.pow((element-meanEntropy), 2); + }); + return Math.sqrt(total/(entropies.length-1)); + } + + /** + * Calculates the standard deviation of all of the entropies below the standard deviation of the whole data. + * + * @param {Array} entropies + * @param {number} meanEntropy + * @returns {number} + */ + getBelowStandardDeviation(entropies, meanEntropy) { + const result = []; + entropies.forEach((element) => { + if (element < meanEntropy) { + result.push(element); + } + }); + return this.calculateStandardDeviation(result, meanEntropy); + } + + /** + * Calculates the standard deviation of all of the entropies above the standard deviation of the whole data. + * + * @param {Array} entropies + * @param {number} meanEntropy + * @returns {number} + */ + getAboveStandardDeviation(entropies, meanEntropy) { + const result = []; + entropies.forEach((element) => { + if (element > meanEntropy) { + result.push(element); + } + }); + return this.calculateStandardDeviation(result, meanEntropy); + } + + /** + * Determines which algorithm to use for calculating the deviation. + * + * @param {string} algorithm + * @param {Array} entropies + * @param {number} meanEntropy + * @returns {number} + */ + algorithmTypeLow(algorithm, entropies, meanEntropy) { + switch (algorithm) { + case "Absolute Mean Deviation": + return this.getBelowMean(entropies.entropyData, meanEntropy); + case "Standard Deviation": + return this.getBelowStandardDeviation(entropies.entropyData, meanEntropy); + } + } + + /** + * Determines which algorithm to use for calculating the deviation. + * + * @param {string} algorithm + * @param {Array} entropies + * @param {number} meanEntropy + * @returns {number} + */ + algorithmTypeHigh(algorithm, entropies, meanEntropy) { + switch (algorithm) { + case "Absolute Mean Deviation": + return this.getAboveMean(entropies.entropyData, meanEntropy); + case "Standard Deviation": + return this.getAboveStandardDeviation(entropies.entropyData, meanEntropy); + } + } + + /** + * Retrieves all of the blocks with entropy between the lower bound and upper bound. + * + * @param {Array} entropies + * @param {number} lbound + * @param {number} ubound + * @param {ArrayBuffer} input + * @param {boolean} flipGroupings + * @param {number} binWidth + * @returns {string} + */ + getRange(entropies, lbound, ubound, input, flipGroupings, binWidth) { + const result = []; + entropies.forEach((element, index) => { + if (element > lbound && element < ubound) + result.push([element, index]); + }); + return this.generateOutput(result, input, flipGroupings, binWidth); + } + + /** + * Determines the data ranges for blocks with similar entropies. + * + * @param {Array} data + * @returns {Array} + */ + groupings(data) { + const result = [data[0][1]]; + for (let i = 0; i < data.length - 1; i++) { + if (Math.abs(data[i][1] - data[i+1][1]) > 3) // I think this needs to be scaled on the bock size rather than a hardcoded value. + result.push(data[i][1], data[i+1][1]); + } + result.push(data[data.length-1][1]); + return result; + } + + /** + * Flips the groups to cover the remaining data. + * + * @param {Array} data + * @returns {Array} + */ + flipGroupings(data) { + const result = []; + if (data[0]) { + result.push(0, data[0] - 1); + } + for (let i = 1; i < data.length-1; i+=2) { + result.push(data[i]+1, data[i+1]); + } + return result; + } + + /** + * Frames the data into a html format. + * + * @param {Array} ranges + * @param {ArrayBuffer} input + * @param {boolean} flipOutput + * @param {number} binWidth + * @returns {string} + */ + generateOutput(ranges, input, flipOutput, binWidth) { + let output = ""; + if (!(ranges.length)) + return output; + ranges = this.groupings(ranges); + if (flipOutput) + ranges = this.flipGroupings(ranges); + + for (let i = 0; i < ranges.length; i+=2) { + if (i === 0) + output +=` + ${binWidth} + `; + else + output += ` + + `; + output += `${(ranges[i]*binWidth).toString(16)}-${(((ranges[i+1]+1)*binWidth)-1).toString(16)} + ${Utils.escapeHtml(Utils.printable(Utils.truncate(Utils.arrayBufferToStr(input.slice(ranges[i]*binWidth, ((ranges[i+1]+1)*binWidth)-1)), 99)))} + `; + } + return output; + } + + /** + * Puts the remaining data into the table html format. + * + * @param {string} data + * @returns {string} + */ + generateCompleteOutput(data) { + if (!(data)) + return "Nothing of interest could be found in the input data.\nHave you tried changing the block size?"; + + return ` + + + + + `+ data + "
Block SizeOffsetResult snippet
"; + } + + /** + * Dispatches on the type of output we want. + * + * @param {Array} entropies + * @param {string} input + * @param {string} algorithm + * @param {boolean} flipOutput + * @param {number} binWidth + * @param {ArrayBuffer} data + * @returns {string} + */ + entropyCalculations(entropies, input, algorithm, flipOutput, binWidth, data) { + let aboveScale = 0; + let belowScale = 0; + let highResults = []; + let lowResults = []; + + const meanEntropy = this.generateAverage(entropies.entropyData); + switch (input) { + case "High": + aboveScale = this.algorithmTypeHigh(algorithm, entropies, meanEntropy); + highResults = this.getAllAbove(entropies.entropyData, meanEntropy+aboveScale); + return this.generateOutput(highResults, data, flipOutput, binWidth); + case "Low": + belowScale = this.algorithmTypeLow(algorithm, entropies, meanEntropy); + lowResults = this.getAllBelow(entropies.entropyData, meanEntropy-belowScale); + return this.generateOutput(lowResults, data, flipOutput, binWidth); + case "Low and High": + aboveScale = this.algorithmTypeHigh(algorithm, entropies, meanEntropy); + highResults = this.getAllAbove(entropies.entropyData, meanEntropy+aboveScale); + belowScale = this.algorithmTypeLow(algorithm, entropies, meanEntropy); + lowResults = this.getAllBelow(entropies.entropyData, meanEntropy-belowScale); + return this.generateOutput(lowResults, data, flipOutput, binWidth) + this.generateOutput(highResults, data, flipOutput, binWidth); + } + } + /** * @param {ArrayBuffer} input * @param {Object[]} args - * @returns {List} + * @returns {JSON} */ - run(input, args) { - args[1] = parseInt(args[1]); + main(input, args) { if (args[2] < 0) throw new OperationError("Cannot have a lower bound entropy lower than 0"); if (args[3] > 8) throw new OperationError("Cannot have an upper bound entropy greater than 8"); if (args[1] >= input.byteLength) throw new OperationError("Block size is larger than the input"); - - let highMean = 0; - let lowMean = 0; - let highResults = []; - let lowResults = []; + if (args[1] < 0) + throw new OperationError("Cannot have a negative block size"); + + let result = []; + const entropies = this.calculateScanningEntropy(new Uint8Array(input), args[1]); switch (args[0]) { - case "Ascii Range": + case "English Text": + result = this.getRange(entropies.entropyData, 3.5, 5, input, args[5], args[1]); break; case "Enter Value Range": if (args[2] === args[3]) throw new OperationError("Should not have the upper bound entropy and the lower bound entropy equal to one another"); + if (args[2] > args[3]) + throw new OperationError("Should not have lower bound entropy higher than the highere bound entropy"); + result = this.getRange(entropies.entropyData, args[2], args[3], input, args[5], args[1]); break; default: - const entropies = this.calculateScanningEntropy(new Uint8Array(input), args[1]); - const meanEntropy = this.generateAverage(entropies.entropyData); - switch (args[0]) { - case "High": - highMean = this.getAboveMean(entropies.entropyData, meanEntropy); - highResults = this.getAllAbove(entropies.entropyData, meanEntropy+highMean); - break; - case "Low": - lowMean = this.getBelowMean(entropies.entropyData, meanEntropy); - lowResults = this.getAllBelow(entropies.entropyData, meanEntropy-lowMean); - console.log(lowResults); - console.log("All entropies: ", entropies); - break; - case "Low and High": - highMean = this.getAboveMean(entropies.entropyData, meanEntropy); - highResults = this.getAllAbove(entropies.entropyData, meanEntropy+highMean); - lowMean = this.getBelowMean(entropies.entropyData, meanEntropy); - lowResults = this.getAllBelow(entropies.entropyData, meanEntropy-lowMean); - break; - } + result = this.entropyCalculations(entropies, args[0], args[4], args[5], args[1], input); } - return " "; + return result; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {JSON} + */ + run(input, args) { + let result = ""; + if (args[1].indexOf("-") !== -1) { + const temp = args[1].split("-"); + for (let i = parseInt(temp[0], 10); i