mirror of
https://github.com/gchq/CyberChef.git
synced 2025-05-11 00:31:31 -04:00
Needs tiding but it does function
This commit is contained in:
n1073645
parent
a5f071dd36
commit
ad0c04884d
1 changed files with 339 additions and 61 deletions
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
import Operation from "../Operation.mjs";
|
||||
import OperationError from "../errors/OperationError.mjs";
|
||||
//import Entropy from "./Entropy.mjs";
|
||||
import Utils from "../Utils.mjs";
|
||||
|
||||
/**
|
||||
* Extract Entropies operation
|
||||
|
|
@ -24,7 +24,8 @@ class ExtractEntropies extends Operation {
|
|||
this.description = "";
|
||||
this.infoURL = "";
|
||||
this.inputType = "ArrayBuffer";
|
||||
this.outputType = "List<File>";
|
||||
this.outputType = "JSON";
|
||||
this.presentType = "html";
|
||||
this.args = [
|
||||
{
|
||||
name: "Entropies",
|
||||
|
|
@ -43,7 +44,7 @@ class ExtractEntropies extends Operation {
|
|||
off: [2, 3]
|
||||
},
|
||||
{
|
||||
name: "Ascii Range",
|
||||
name: "English Text",
|
||||
off: [2, 3]
|
||||
},
|
||||
{
|
||||
|
|
@ -54,8 +55,8 @@ class ExtractEntropies extends Operation {
|
|||
},
|
||||
{
|
||||
name: "Block Size",
|
||||
type: "number",
|
||||
value: "8"
|
||||
type: "string",
|
||||
value: "10"
|
||||
},
|
||||
{
|
||||
name: "Lower Entropy Bound",
|
||||
|
|
@ -66,11 +67,21 @@ class ExtractEntropies extends Operation {
|
|||
name: "Upper Entropy Bound",
|
||||
type: "number",
|
||||
value: 0
|
||||
},
|
||||
{
|
||||
name: "Algorithm To Use",
|
||||
type: "option",
|
||||
value: ["Absolute Mean Deviation", "Standard Deviation"]
|
||||
},
|
||||
{
|
||||
name: "Flip Output",
|
||||
type: "boolean",
|
||||
value: false
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
/**
|
||||
* Calculates the frequency of bytes in the input.
|
||||
*
|
||||
* @param {Uint8Array} input
|
||||
|
|
@ -106,7 +117,7 @@ class ExtractEntropies extends Operation {
|
|||
}
|
||||
|
||||
/**
|
||||
* Calculates the scanning entropy of the input
|
||||
* Calculates the scanning entropy of the input.
|
||||
*
|
||||
* @param {Uint8Array} inputBytes
|
||||
* @returns {Object}
|
||||
|
|
@ -123,94 +134,361 @@ class ExtractEntropies extends Operation {
|
|||
return { entropyData, binWidth };
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the average of a list of entropies.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @returns {number}
|
||||
*/
|
||||
generateAverage(entropies) {
|
||||
return entropies.reduce((previous, current) => current += previous) / entropies.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caculates the mean of the data above the original mean.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @param {number} meanEntropy
|
||||
* @returns {number}
|
||||
*/
|
||||
getAboveMean(entropies, meanEntropy) {
|
||||
const result = [];
|
||||
entropies.forEach((element) =>{
|
||||
if (element > meanEntropy)
|
||||
result.push(element - meanEntropy);
|
||||
let total = 0, count = 0;
|
||||
entropies.forEach((element) => {
|
||||
if (element > meanEntropy) {
|
||||
total += (element - meanEntropy);
|
||||
count++;
|
||||
}
|
||||
});
|
||||
return this.generateAverage(result);
|
||||
return total/count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caculates the mean of the data below the original mean.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @param {number} meanEntropy
|
||||
* @returns {number}
|
||||
*/
|
||||
getBelowMean(entropies, meanEntropy) {
|
||||
const result = [];
|
||||
entropies.forEach((element) =>{
|
||||
if (element < meanEntropy)
|
||||
result.push(meanEntropy - element);
|
||||
let total = 0, count = 0;
|
||||
entropies.forEach((element) => {
|
||||
if (element < meanEntropy) {
|
||||
total += (meanEntropy - element);
|
||||
count++;
|
||||
}
|
||||
});
|
||||
return this.generateAverage(result);
|
||||
return total/count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves all the blocks with entropy higher than the high mean.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @param {number} highEntropy
|
||||
* @returns {Array}
|
||||
*/
|
||||
getAllAbove(entropies, highEntropy) {
|
||||
const result = [];
|
||||
entropies.forEach((element, index) => {
|
||||
if(element > highEntropy)
|
||||
result.push([element, index]);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
getAllBelow(entropies, lowEntropy) {
|
||||
const result = [];
|
||||
entropies.forEach((element, index) => {
|
||||
if(element < lowEntropy)
|
||||
if (element > highEntropy)
|
||||
result.push([element, index]);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves all the blocks with entropy lower than the low mean.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @param {number} highEntropy
|
||||
* @returns {Array}
|
||||
*/
|
||||
getAllBelow(entropies, lowEntropy) {
|
||||
const result = [];
|
||||
entropies.forEach((element, index) => {
|
||||
if (element < lowEntropy)
|
||||
result.push([element, index]);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the standard deviation of all of the entropies.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @param {number} meanEntropy
|
||||
* @returns {number}
|
||||
*/
|
||||
calculateStandardDeviation(entropies, meanEntropy) {
|
||||
let total = 0;
|
||||
entropies.forEach((element) => {
|
||||
total += Math.pow((element-meanEntropy), 2);
|
||||
});
|
||||
return Math.sqrt(total/(entropies.length-1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the standard deviation of all of the entropies below the standard deviation of the whole data.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @param {number} meanEntropy
|
||||
* @returns {number}
|
||||
*/
|
||||
getBelowStandardDeviation(entropies, meanEntropy) {
|
||||
const result = [];
|
||||
entropies.forEach((element) => {
|
||||
if (element < meanEntropy) {
|
||||
result.push(element);
|
||||
}
|
||||
});
|
||||
return this.calculateStandardDeviation(result, meanEntropy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the standard deviation of all of the entropies above the standard deviation of the whole data.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @param {number} meanEntropy
|
||||
* @returns {number}
|
||||
*/
|
||||
getAboveStandardDeviation(entropies, meanEntropy) {
|
||||
const result = [];
|
||||
entropies.forEach((element) => {
|
||||
if (element > meanEntropy) {
|
||||
result.push(element);
|
||||
}
|
||||
});
|
||||
return this.calculateStandardDeviation(result, meanEntropy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines which algorithm to use for calculating the deviation.
|
||||
*
|
||||
* @param {string} algorithm
|
||||
* @param {Array} entropies
|
||||
* @param {number} meanEntropy
|
||||
* @returns {number}
|
||||
*/
|
||||
algorithmTypeLow(algorithm, entropies, meanEntropy) {
|
||||
switch (algorithm) {
|
||||
case "Absolute Mean Deviation":
|
||||
return this.getBelowMean(entropies.entropyData, meanEntropy);
|
||||
case "Standard Deviation":
|
||||
return this.getBelowStandardDeviation(entropies.entropyData, meanEntropy);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines which algorithm to use for calculating the deviation.
|
||||
*
|
||||
* @param {string} algorithm
|
||||
* @param {Array} entropies
|
||||
* @param {number} meanEntropy
|
||||
* @returns {number}
|
||||
*/
|
||||
algorithmTypeHigh(algorithm, entropies, meanEntropy) {
|
||||
switch (algorithm) {
|
||||
case "Absolute Mean Deviation":
|
||||
return this.getAboveMean(entropies.entropyData, meanEntropy);
|
||||
case "Standard Deviation":
|
||||
return this.getAboveStandardDeviation(entropies.entropyData, meanEntropy);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves all of the blocks with entropy between the lower bound and upper bound.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @param {number} lbound
|
||||
* @param {number} ubound
|
||||
* @param {ArrayBuffer} input
|
||||
* @param {boolean} flipGroupings
|
||||
* @param {number} binWidth
|
||||
* @returns {string}
|
||||
*/
|
||||
getRange(entropies, lbound, ubound, input, flipGroupings, binWidth) {
|
||||
const result = [];
|
||||
entropies.forEach((element, index) => {
|
||||
if (element > lbound && element < ubound)
|
||||
result.push([element, index]);
|
||||
});
|
||||
return this.generateOutput(result, input, flipGroupings, binWidth);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines the data ranges for blocks with similar entropies.
|
||||
*
|
||||
* @param {Array} data
|
||||
* @returns {Array}
|
||||
*/
|
||||
groupings(data) {
|
||||
const result = [data[0][1]];
|
||||
for (let i = 0; i < data.length - 1; i++) {
|
||||
if (Math.abs(data[i][1] - data[i+1][1]) > 3) // I think this needs to be scaled on the bock size rather than a hardcoded value.
|
||||
result.push(data[i][1], data[i+1][1]);
|
||||
}
|
||||
result.push(data[data.length-1][1]);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flips the groups to cover the remaining data.
|
||||
*
|
||||
* @param {Array} data
|
||||
* @returns {Array}
|
||||
*/
|
||||
flipGroupings(data) {
|
||||
const result = [];
|
||||
if (data[0]) {
|
||||
result.push(0, data[0] - 1);
|
||||
}
|
||||
for (let i = 1; i < data.length-1; i+=2) {
|
||||
result.push(data[i]+1, data[i+1]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Frames the data into a html format.
|
||||
*
|
||||
* @param {Array} ranges
|
||||
* @param {ArrayBuffer} input
|
||||
* @param {boolean} flipOutput
|
||||
* @param {number} binWidth
|
||||
* @returns {string}
|
||||
*/
|
||||
generateOutput(ranges, input, flipOutput, binWidth) {
|
||||
let output = "";
|
||||
if (!(ranges.length))
|
||||
return output;
|
||||
ranges = this.groupings(ranges);
|
||||
if (flipOutput)
|
||||
ranges = this.flipGroupings(ranges);
|
||||
|
||||
for (let i = 0; i < ranges.length; i+=2) {
|
||||
if (i === 0)
|
||||
output +=`
|
||||
<td>${binWidth}</td>
|
||||
`;
|
||||
else
|
||||
output += `
|
||||
<td></td>
|
||||
`;
|
||||
output += `<td>${(ranges[i]*binWidth).toString(16)}-${(((ranges[i+1]+1)*binWidth)-1).toString(16)}</td>
|
||||
<td>${Utils.escapeHtml(Utils.printable(Utils.truncate(Utils.arrayBufferToStr(input.slice(ranges[i]*binWidth, ((ranges[i+1]+1)*binWidth)-1)), 99)))}</td>
|
||||
</tr>`;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Puts the remaining data into the table html format.
|
||||
*
|
||||
* @param {string} data
|
||||
* @returns {string}
|
||||
*/
|
||||
generateCompleteOutput(data) {
|
||||
if (!(data))
|
||||
return "Nothing of interest could be found in the input data.\nHave you tried changing the block size?";
|
||||
|
||||
return `<table
|
||||
class='table table-hover table-sm table-bordered'
|
||||
style='table-layout: fixed;'>
|
||||
<tr>
|
||||
<th>Block Size</th>
|
||||
<th>Offset</th>
|
||||
<th>Result snippet</th>
|
||||
</tr>`+ data + "</table><script type='application/javascript'>$('[data-toggle=\"tooltip\"]').tooltip()</script>";
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatches on the type of output we want.
|
||||
*
|
||||
* @param {Array} entropies
|
||||
* @param {string} input
|
||||
* @param {string} algorithm
|
||||
* @param {boolean} flipOutput
|
||||
* @param {number} binWidth
|
||||
* @param {ArrayBuffer} data
|
||||
* @returns {string}
|
||||
*/
|
||||
entropyCalculations(entropies, input, algorithm, flipOutput, binWidth, data) {
|
||||
let aboveScale = 0;
|
||||
let belowScale = 0;
|
||||
let highResults = [];
|
||||
let lowResults = [];
|
||||
|
||||
const meanEntropy = this.generateAverage(entropies.entropyData);
|
||||
switch (input) {
|
||||
case "High":
|
||||
aboveScale = this.algorithmTypeHigh(algorithm, entropies, meanEntropy);
|
||||
highResults = this.getAllAbove(entropies.entropyData, meanEntropy+aboveScale);
|
||||
return this.generateOutput(highResults, data, flipOutput, binWidth);
|
||||
case "Low":
|
||||
belowScale = this.algorithmTypeLow(algorithm, entropies, meanEntropy);
|
||||
lowResults = this.getAllBelow(entropies.entropyData, meanEntropy-belowScale);
|
||||
return this.generateOutput(lowResults, data, flipOutput, binWidth);
|
||||
case "Low and High":
|
||||
aboveScale = this.algorithmTypeHigh(algorithm, entropies, meanEntropy);
|
||||
highResults = this.getAllAbove(entropies.entropyData, meanEntropy+aboveScale);
|
||||
belowScale = this.algorithmTypeLow(algorithm, entropies, meanEntropy);
|
||||
lowResults = this.getAllBelow(entropies.entropyData, meanEntropy-belowScale);
|
||||
return this.generateOutput(lowResults, data, flipOutput, binWidth) + this.generateOutput(highResults, data, flipOutput, binWidth);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ArrayBuffer} input
|
||||
* @param {Object[]} args
|
||||
* @returns {List<File>}
|
||||
* @returns {JSON}
|
||||
*/
|
||||
run(input, args) {
|
||||
args[1] = parseInt(args[1]);
|
||||
main(input, args) {
|
||||
if (args[2] < 0)
|
||||
throw new OperationError("Cannot have a lower bound entropy lower than 0");
|
||||
if (args[3] > 8)
|
||||
throw new OperationError("Cannot have an upper bound entropy greater than 8");
|
||||
if (args[1] >= input.byteLength)
|
||||
throw new OperationError("Block size is larger than the input");
|
||||
if (args[1] < 0)
|
||||
throw new OperationError("Cannot have a negative block size");
|
||||
|
||||
let highMean = 0;
|
||||
let lowMean = 0;
|
||||
let highResults = [];
|
||||
let lowResults = [];
|
||||
let result = [];
|
||||
const entropies = this.calculateScanningEntropy(new Uint8Array(input), args[1]);
|
||||
switch (args[0]) {
|
||||
case "Ascii Range":
|
||||
case "English Text":
|
||||
result = this.getRange(entropies.entropyData, 3.5, 5, input, args[5], args[1]);
|
||||
break;
|
||||
case "Enter Value Range":
|
||||
if (args[2] === args[3])
|
||||
throw new OperationError("Should not have the upper bound entropy and the lower bound entropy equal to one another");
|
||||
if (args[2] > args[3])
|
||||
throw new OperationError("Should not have lower bound entropy higher than the highere bound entropy");
|
||||
result = this.getRange(entropies.entropyData, args[2], args[3], input, args[5], args[1]);
|
||||
break;
|
||||
default:
|
||||
const entropies = this.calculateScanningEntropy(new Uint8Array(input), args[1]);
|
||||
const meanEntropy = this.generateAverage(entropies.entropyData);
|
||||
switch (args[0]) {
|
||||
case "High":
|
||||
highMean = this.getAboveMean(entropies.entropyData, meanEntropy);
|
||||
highResults = this.getAllAbove(entropies.entropyData, meanEntropy+highMean);
|
||||
break;
|
||||
case "Low":
|
||||
lowMean = this.getBelowMean(entropies.entropyData, meanEntropy);
|
||||
lowResults = this.getAllBelow(entropies.entropyData, meanEntropy-lowMean);
|
||||
console.log(lowResults);
|
||||
console.log("All entropies: ", entropies);
|
||||
break;
|
||||
case "Low and High":
|
||||
highMean = this.getAboveMean(entropies.entropyData, meanEntropy);
|
||||
highResults = this.getAllAbove(entropies.entropyData, meanEntropy+highMean);
|
||||
lowMean = this.getBelowMean(entropies.entropyData, meanEntropy);
|
||||
lowResults = this.getAllBelow(entropies.entropyData, meanEntropy-lowMean);
|
||||
break;
|
||||
}
|
||||
result = this.entropyCalculations(entropies, args[0], args[4], args[5], args[1], input);
|
||||
}
|
||||
return " ";
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ArrayBuffer} input
|
||||
* @param {Object[]} args
|
||||
* @returns {JSON}
|
||||
*/
|
||||
run(input, args) {
|
||||
let result = "";
|
||||
if (args[1].indexOf("-") !== -1) {
|
||||
const temp = args[1].split("-");
|
||||
for (let i = parseInt(temp[0], 10); i <parseInt(temp[1], 10); i++) {
|
||||
args[1] = i;
|
||||
result += this.main(input, args);
|
||||
}
|
||||
} else {
|
||||
args[1] = parseInt(args[1], 10);
|
||||
result = this.main(input, args);
|
||||
}
|
||||
return this.generateCompleteOutput(result);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue