mirrors/CyberChef
1
0
Fork 0
mirror of https://github.com/gchq/CyberChef.git synced 2025-05-10 16:25:01 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Magic progress

Browse source
This commit is contained in:
n1073645 2019-12-02 15:47:02 +00:00
parent 9f3b185f77
commit 40c7e63046
5 changed files with 199 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/core/Operation.mjs
View file

@ -25,6 +25,9 @@ class Operation {
this._flowControl = false;
this._manualBake = false;
this._ingList = [];
// this._checks = [];
this._inputRegexes = [];
this._outputRegexes = [];
// Public fields
this.name = "";

10
src/core/config/scripts/generateConfig.mjs
View file

@ -45,8 +45,14 @@ for (const opObj in Ops) {
args: op.args
};
if ("patterns" in op) {
operationConfig[op.name].patterns = op.patterns;
if ("checks" in op) {
if ("outRegexes" in op.checks) {
operationConfig[op.name].outputRegexes = op.checks.outRegexes;
}
if ("inRegexes" in op.checks) {
operationConfig[op.name].inputRegexes = op.checks.inRegexes;
}
}
if (!(op.module in modules))

117
src/core/lib/Magic.mjs
View file

@ -4,6 +4,7 @@ import Recipe from "../Recipe.mjs";
import Dish from "../Dish.mjs";
import {detectFileType} from "./FileType.mjs";
import chiSquared from "chi-squared";
import potentialOps from "./Test.mjs";
/**
* A class for detecting encodings, file types and byte frequencies and
@ -24,28 +25,7 @@ class Magic {
constructor(buf, opPatterns) {
this.inputBuffer = new Uint8Array(buf);
this.inputStr = Utils.arrayBufferToStr(buf);
this.opPatterns = opPatterns || Magic._generateOpPatterns();
}
/**
* Finds operations that claim to be able to decode the input based on regular
* expression matches.
*
* @returns {Object[]}
*/
findMatchingOps() {
const matches = [];
for (let i = 0; i < this.opPatterns.length; i++) {
const pattern = this.opPatterns[i],
regex = new RegExp(pattern.match, pattern.flags);
if (regex.test(this.inputStr)) {
matches.push(pattern);
}
}
return matches;
this.opPatterns = new potentialOps(opPatterns);
}
/**
@ -281,7 +261,7 @@ class Magic {
if (depth < 0) return [];
// Find any operations that can be run on this data
const matchingOps = this.findMatchingOps();
const matchingOps = this.opPatterns.findMatchingInputRegexes(this.inputStr);
let results = [];
@ -299,32 +279,49 @@ class Magic {
});
const prevOp = recipeConfig[recipeConfig.length - 1];
// Execute each of the matching operations, then recursively call the speculativeExecution()
// method on the resulting data, recording the properties of each option.
await Promise.all(matchingOps.map(async op => {
const opConfig = {
op: op.op,
args: op.args
},
output = await this._runRecipe([opConfig]);
async function regexesTests(flag, sensible) {
// If the recipe is repeating and returning the same data, do not continue
if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
return;
}
// Execute each of the matching operations, then recursively call the speculativeExecution()
// method on the resulting data, recording the properties of each option.
await Promise.all(sensible.map(async op => {
const opConfig = {
op: op.op,
args: op.args
},
output = await this._runRecipe([opConfig]);
// If the recipe returned an empty buffer, do not continue
if (_buffersEqual(output, new ArrayBuffer())) {
return;
}
// If the recipe is repeating and returning the same data, do not continue
if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
return;
}
const magic = new Magic(output, this.opPatterns),
speculativeResults = await magic.speculativeExecution(
depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);
results = results.concat(speculativeResults);
}));
// If the recipe returned an empty buffer, do not continue
if (_buffersEqual(output, new ArrayBuffer())) {
return;
}
if(flag) {
const outputRegexes = OperationConfig[op.op].outputRegexes;
if (outputRegexes)
for (const pattern of outputRegexes)
if (!(new RegExp(pattern.match, pattern.flags).test(Utils.arrayBufferToStr(output))))
return;
} else {
if (!(op.match.test(output))){
return;
}
}
const magic = new Magic(output, this.opPatterns),
speculativeResults = await magic.speculativeExecution(
depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);
results = results.concat(speculativeResults);
}));
}
regexesTests(1, matchingOps);
regexesTests(0, this.opPatterns.getOutputRegexes());
//console.log("haha",this.opPatterns.getOutputRegexes());
if (intensive) {
// Run brute forcing of various types on the data and create a new branch for each option
const bfEncodings = await this.bruteForce();
@ -338,6 +335,8 @@ class Magic {
}));
}
// console.log(results);
// Prune branches that result in unhelpful outputs
const prunedResults = results.filter(r =>
(r.useful || r.data.length > 0) && // The operation resulted in ""
@ -349,6 +348,8 @@ class Magic {
)
);
// console.log(prunedResults);
// Return a sorted list of possible recipes along with their properties
return prunedResults.sort((a, b) => {
// Each option is sorted based on its most likely language (lower is better)
@ -441,32 +442,6 @@ class Magic {
return this.freqDist;
}
/**
* Generates a list of all patterns that operations claim to be able to decode.
*
* @private
* @returns {Object[]}
*/
static _generateOpPatterns() {
const opPatterns = [];
for (const op in OperationConfig) {
if (!("patterns" in OperationConfig[op])) continue;
OperationConfig[op].patterns.forEach(pattern => {
opPatterns.push({
op: op,
match: pattern.match,
flags: pattern.flags,
args: pattern.args,
useful: pattern.useful || false
});
});
}
return opPatterns;
}
/**
* Calculates Pearson's Chi-Squared test for two frequency arrays.
* https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test

15
src/core/lib/MagicObject.mjs Normal file
View file

@ -0,0 +1,15 @@
/**
*
*/
class magicObject {
/**
*
* @param inRegexes
* @param outRegexes
*/
constructor (inRegexes = null, outRegexes = null) {
this.inRegexes = inRegexes;
this.outRegexes = outRegexes;
}
} export default magicObject;

127
src/core/lib/Test.mjs Normal file
View file

@ -0,0 +1,127 @@
import OperationConfig from "../config/OperationConfig.json";
/**
*
*/
class potentialOps {
/**
*
* @param prevOp
*/
constructor (prevOp) {
if (typeof prevOp === "undefined") {
this.inputRegexes = this.generateInputOpPatterns();
this.hasInputRegexes = true;
this.outputRegexes = this.generateOutputOpPatterns();
this.hasOutputRegexes = true;
} else {
this.inputRegexes = prevOp.getInputRegexes();
this.hasInputRegexes = true;
this.outputRegexes = prevOp.getOutputRegexes();
this.hasOutputRegexes = true;
}
}
/**
*
* @param outputRegexes
*/
setOutputRegexes (outputRegexes) {
this.outputRegexes = [...outputRegexes];
this.hasOutputRegexes = true;
}
/**
*
* @param inputRegexes
*/
setInputRegexes (inputRegexes) {
this.inputRegexes = [...inputRegexes];
this.hasInputRegexes = true;
}
/**
*
*/
getInputRegexes () {
return this.inputRegexes;
}
/**
*
*/
getOutputRegexes () {
return this.outputRegexes;
}
/**
* Finds operations that claim to be able to decode the input based on regular
* expression matches.
*
* @returns {Object[]}
*/
findMatchingInputRegexes (inputStr) {
const matches = [];
for (const pattern of this.inputRegexes) {
const regex = new RegExp(pattern.match, pattern.flags);
if (regex.test(inputStr)) {
matches.push(pattern);
}
}
return matches;
}
/**
* Generates a list of all patterns that operations claim to be able to decode.
*
* @private
* @returns {Object[]}
*/
generateInputOpPatterns() {
const opPatterns = [];
for (const op in OperationConfig) {
if (!("inputRegexes" in OperationConfig[op])) continue;
OperationConfig[op].inputRegexes.forEach(pattern => {
opPatterns.push({
op: op,
match: pattern.match,
flags: pattern.flags,
args: pattern.args,
useful: pattern.useful || false
});
});
}
return opPatterns;
}
/**
*
*/
generateOutputOpPatterns() {
const opPatterns = [];
for (const op in OperationConfig) {
if (!(OperationConfig[op].outputRegexes)) continue;
OperationConfig[op].outputRegexes.forEach(pattern => {
opPatterns.push({
op: op,
match: pattern.match,
flags: pattern.flags,
shouldMatch: pattern.shouldMatch,
args: pattern.args,
useful: pattern.useful || false
});
});
}
return opPatterns;
}
} export default potentialOps;