mirror of
https://github.com/gchq/CyberChef.git
synced 2025-04-22 07:46:16 -04:00
Merge branch 'esm' into port-flowcontrol
This commit is contained in:
d98762625
commit
046e1ebad9
37 changed files with 2726 additions and 56 deletions
54
src/core/operations/DetectFileType.mjs
Normal file
54
src/core/operations/DetectFileType.mjs
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
/**
|
||||
* @author n1474335 [n1474335@gmail.com]
|
||||
* @copyright Crown Copyright 2016
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
|
||||
import Operation from "../Operation";
|
||||
import Magic from "../lib/Magic";
|
||||
|
||||
/**
|
||||
* Detect File Type operation
|
||||
*/
|
||||
class DetectFileType extends Operation {
|
||||
|
||||
/**
|
||||
* DetectFileType constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.name = "Detect File Type";
|
||||
this.module = "Default";
|
||||
this.description = "Attempts to guess the MIME (Multipurpose Internet Mail Extensions) type of the data based on 'magic bytes'.<br><br>Currently supports the following file types: 7z, amr, avi, bmp, bz2, class, cr2, crx, dex, dmg, doc, elf, eot, epub, exe, flac, flv, gif, gz, ico, iso, jpg, jxr, m4a, m4v, mid, mkv, mov, mp3, mp4, mpg, ogg, otf, pdf, png, ppt, ps, psd, rar, rtf, sqlite, swf, tar, tar.z, tif, ttf, utf8, vmdk, wav, webm, webp, wmv, woff, woff2, xls, xz, zip.";
|
||||
this.inputType = "ArrayBuffer";
|
||||
this.outputType = "string";
|
||||
this.args = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ArrayBuffer} input
|
||||
* @param {Object[]} args
|
||||
* @returns {string}
|
||||
*/
|
||||
run(input, args) {
|
||||
const data = new Uint8Array(input),
|
||||
type = Magic.magicFileType(data);
|
||||
|
||||
if (!type) {
|
||||
return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?";
|
||||
} else {
|
||||
let output = "File extension: " + type.ext + "\n" +
|
||||
"MIME type: " + type.mime;
|
||||
|
||||
if (type.desc && type.desc.length) {
|
||||
output += "\nDescription: " + type.desc;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default DetectFileType;
|
||||
|
|
@ -44,6 +44,23 @@ class EscapeUnicodeCharacters extends Operation {
|
|||
"value": true
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "\\\\u(?:[\\da-f]{4,6})",
|
||||
flags: "i",
|
||||
args: ["\\u"]
|
||||
},
|
||||
{
|
||||
match: "%u(?:[\\da-f]{4,6})",
|
||||
flags: "i",
|
||||
args: ["%u"]
|
||||
},
|
||||
{
|
||||
match: "U\\+(?:[\\da-f]{4,6})",
|
||||
flags: "i",
|
||||
args: ["U+"]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -48,6 +48,13 @@ class FromBCD extends Operation {
|
|||
"value": FORMAT
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^(?:\\d{4} ){3,}\\d{4}$",
|
||||
flags: "",
|
||||
args: ["8 4 2 1", true, false, "Nibbles"]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -35,6 +35,13 @@ class FromBase32 extends Operation {
|
|||
value: true
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$",
|
||||
flags: "",
|
||||
args: ["A-Z2-7=", false]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -37,6 +37,18 @@ class FromBase58 extends Operation {
|
|||
"value": true
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
|
||||
flags: "",
|
||||
args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", false]
|
||||
},
|
||||
{
|
||||
match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
|
||||
flags: "",
|
||||
args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", false]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -35,6 +35,73 @@ class FromBase64 extends Operation {
|
|||
value: true
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
|
||||
flags: "i",
|
||||
args: ["A-Za-z0-9+/=", false]
|
||||
},
|
||||
{
|
||||
match: "^[A-Z\\d\\-_]{20,}$",
|
||||
flags: "i",
|
||||
args: ["A-Za-z0-9-_", false]
|
||||
},
|
||||
{
|
||||
match: "^(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?$",
|
||||
flags: "i",
|
||||
args: ["A-Za-z0-9+\\-=", false]
|
||||
},
|
||||
{
|
||||
match: "^(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?$",
|
||||
flags: "i",
|
||||
args: ["./0-9A-Za-z=", false]
|
||||
},
|
||||
{
|
||||
match: "^[A-Z\\d_.]{20,}$",
|
||||
flags: "i",
|
||||
args: ["A-Za-z0-9_.", false]
|
||||
},
|
||||
{
|
||||
match: "^(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?$",
|
||||
flags: "i",
|
||||
args: ["A-Za-z0-9._-", false]
|
||||
},
|
||||
{
|
||||
match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
|
||||
flags: "i",
|
||||
args: ["0-9a-zA-Z+/=", false]
|
||||
},
|
||||
{
|
||||
match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
|
||||
flags: "i",
|
||||
args: ["0-9A-Za-z+/=", false]
|
||||
},
|
||||
{
|
||||
match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
|
||||
flags: "",
|
||||
args: [" -_", false]
|
||||
},
|
||||
{
|
||||
match: "^[A-Z\\d+\\-]{20,}$",
|
||||
flags: "i",
|
||||
args: ["+\\-0-9A-Za-z", false]
|
||||
},
|
||||
{
|
||||
match: "^[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}$",
|
||||
flags: "",
|
||||
args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", false]
|
||||
},
|
||||
{
|
||||
match: "^(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?$",
|
||||
flags: "i",
|
||||
args: ["N-ZA-Mn-za-m0-9+/=", false]
|
||||
},
|
||||
{
|
||||
match: "^[A-Z\\d./]{20,}$",
|
||||
flags: "i",
|
||||
args: ["./0-9A-Za-z", false]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -31,6 +31,43 @@ class FromBinary extends Operation {
|
|||
"value": BIN_DELIM_OPTIONS
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^(?:[01]{8})+$",
|
||||
flags: "",
|
||||
args: ["None"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[01]{8})(?: [01]{8})*$",
|
||||
flags: "",
|
||||
args: ["Space"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[01]{8})(?:,[01]{8})*$",
|
||||
flags: "",
|
||||
args: ["Comma"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[01]{8})(?:;[01]{8})*$",
|
||||
flags: "",
|
||||
args: ["Semi-colon"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[01]{8})(?::[01]{8})*$",
|
||||
flags: "",
|
||||
args: ["Colon"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[01]{8})(?:\\n[01]{8})*$",
|
||||
flags: "",
|
||||
args: ["Line feed"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[01]{8})(?:\\r\\n[01]{8})*$",
|
||||
flags: "",
|
||||
args: ["CRLF"]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -31,6 +31,38 @@ class FromDecimal extends Operation {
|
|||
"value": DELIM_OPTIONS
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
|
||||
flags: "",
|
||||
args: ["Space"]
|
||||
},
|
||||
{
|
||||
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
|
||||
flags: "",
|
||||
args: ["Comma"]
|
||||
},
|
||||
{
|
||||
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
|
||||
flags: "",
|
||||
args: ["Semi-colon"]
|
||||
},
|
||||
{
|
||||
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
|
||||
flags: "",
|
||||
args: ["Colon"]
|
||||
},
|
||||
{
|
||||
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
|
||||
flags: "",
|
||||
args: ["Line feed"]
|
||||
},
|
||||
{
|
||||
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
|
||||
flags: "",
|
||||
args: ["CRLF"]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -24,6 +24,13 @@ class FromHTMLEntity extends Operation {
|
|||
this.inputType = "string";
|
||||
this.outputType = "string";
|
||||
this.args = [];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});",
|
||||
flags: "i",
|
||||
args: []
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -31,6 +31,53 @@ class FromHex extends Operation {
|
|||
value: FROM_HEX_DELIM_OPTIONS
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^(?:[\\dA-F]{2})+$",
|
||||
flags: "i",
|
||||
args: ["None"]
|
||||
},
|
||||
{
|
||||
match: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$",
|
||||
flags: "i",
|
||||
args: ["Space"]
|
||||
},
|
||||
{
|
||||
match: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$",
|
||||
flags: "i",
|
||||
args: ["Comma"]
|
||||
},
|
||||
{
|
||||
match: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$",
|
||||
flags: "i",
|
||||
args: ["Semi-colon"]
|
||||
},
|
||||
{
|
||||
match: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$",
|
||||
flags: "i",
|
||||
args: ["Colon"]
|
||||
},
|
||||
{
|
||||
match: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$",
|
||||
flags: "i",
|
||||
args: ["Line feed"]
|
||||
},
|
||||
{
|
||||
match: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$",
|
||||
flags: "i",
|
||||
args: ["CRLF"]
|
||||
},
|
||||
{
|
||||
match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$",
|
||||
flags: "i",
|
||||
args: ["0x"]
|
||||
},
|
||||
{
|
||||
match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$",
|
||||
flags: "i",
|
||||
args: ["\\x"]
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -24,6 +24,13 @@ class FromHexdump extends Operation {
|
|||
this.inputType = "string";
|
||||
this.outputType = "byteArray";
|
||||
this.args = [];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^(?:(?:[\\dA-F]{4,16}:?)?\\s*((?:[\\dA-F]{2}\\s){1,8}(?:\\s|[\\dA-F]{2}-)(?:[\\dA-F]{2}\\s){1,8}|(?:[\\dA-F]{2}\\s|[\\dA-F]{4}\\s)+)[^\\n]*\\n?)+$",
|
||||
flags: "i",
|
||||
args: []
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -36,6 +36,13 @@ class FromMorseCode extends Operation {
|
|||
"value": WORD_DELIM_OPTIONS
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)",
|
||||
flags: "i",
|
||||
args: ["Space", "Line feed"]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -31,6 +31,38 @@ class FromOctal extends Operation {
|
|||
"value": DELIM_OPTIONS
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$",
|
||||
flags: "",
|
||||
args: ["Space"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$",
|
||||
flags: "",
|
||||
args: ["Comma"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$",
|
||||
flags: "",
|
||||
args: ["Semi-colon"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$",
|
||||
flags: "",
|
||||
args: ["Colon"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
|
||||
flags: "",
|
||||
args: ["Line feed"]
|
||||
},
|
||||
{
|
||||
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
|
||||
flags: "",
|
||||
args: ["CRLF"]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -27,6 +27,13 @@ class FromQuotedPrintable extends Operation {
|
|||
this.inputType = "string";
|
||||
this.outputType = "byteArray";
|
||||
this.args = [];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]*(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
|
||||
flags: "i",
|
||||
args: []
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -32,6 +32,28 @@ class FromUNIXTimestamp extends Operation {
|
|||
"value": UNITS
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^1?\\d{9}$",
|
||||
flags: "",
|
||||
args: ["Seconds (s)"]
|
||||
},
|
||||
{
|
||||
match: "^1?\\d{12}$",
|
||||
flags: "",
|
||||
args: ["Milliseconds (ms)"]
|
||||
},
|
||||
{
|
||||
match: "^1?\\d{15}$",
|
||||
flags: "",
|
||||
args: ["Microseconds (μs)"]
|
||||
},
|
||||
{
|
||||
match: "^1?\\d{18}$",
|
||||
flags: "",
|
||||
args: ["Nanoseconds (ns)"]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -26,6 +26,13 @@ class Gunzip extends Operation {
|
|||
this.inputType = "ArrayBuffer";
|
||||
this.outputType = "ArrayBuffer";
|
||||
this.args = [];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^\\x1f\\x8b\\x08",
|
||||
flags: "",
|
||||
args: []
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
140
src/core/operations/Magic.mjs
Normal file
140
src/core/operations/Magic.mjs
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
/**
|
||||
* @author n1474335 [n1474335@gmail.com]
|
||||
* @copyright Crown Copyright 2016
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
|
||||
import Operation from "../Operation";
|
||||
import Utils from "../Utils";
|
||||
import Dish from "../Dish";
|
||||
import MagicLib from "../lib/Magic";
|
||||
|
||||
/**
|
||||
* Magic operation
|
||||
*/
|
||||
class Magic extends Operation {
|
||||
|
||||
/**
|
||||
* Magic constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.name = "Magic";
|
||||
this.flowControl = true;
|
||||
this.module = "Default";
|
||||
this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.";
|
||||
this.inputType = "ArrayBuffer";
|
||||
this.outputType = "html";
|
||||
this.args = [
|
||||
{
|
||||
"name": "Depth",
|
||||
"type": "number",
|
||||
"value": 3
|
||||
},
|
||||
{
|
||||
"name": "Intensive mode",
|
||||
"type": "boolean",
|
||||
"value": false
|
||||
},
|
||||
{
|
||||
"name": "Extensive language support",
|
||||
"type": "boolean",
|
||||
"value": false
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Object} state - The current state of the recipe.
|
||||
* @param {number} state.progress - The current position in the recipe.
|
||||
* @param {Dish} state.dish - The Dish being operated on.
|
||||
* @param {Operation[]} state.opList - The list of operations in the recipe.
|
||||
* @returns {Object} The updated state of the recipe.
|
||||
*/
|
||||
async run(state) {
|
||||
const ings = state.opList[state.progress].ingValues,
|
||||
[depth, intensive, extLang] = ings,
|
||||
dish = state.dish,
|
||||
currentRecipeConfig = state.opList.map(op => op.config),
|
||||
magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)),
|
||||
options = await magic.speculativeExecution(depth, extLang, intensive);
|
||||
|
||||
let output = `<table
|
||||
class='table table-hover table-condensed table-bordered'
|
||||
style='table-layout: fixed;'>
|
||||
<tr>
|
||||
<th>Recipe (click to load)</th>
|
||||
<th>Result snippet</th>
|
||||
<th>Properties</th>
|
||||
</tr>`;
|
||||
|
||||
/**
|
||||
* Returns a CSS colour value based on an integer input.
|
||||
*
|
||||
* @param {number} val
|
||||
* @returns {string}
|
||||
*/
|
||||
function chooseColour(val) {
|
||||
if (val < 3) return "green";
|
||||
if (val < 5) return "goldenrod";
|
||||
return "red";
|
||||
}
|
||||
|
||||
options.forEach(option => {
|
||||
// Construct recipe URL
|
||||
// Replace this Magic op with the generated recipe
|
||||
const recipeConfig = currentRecipeConfig.slice(0, state.progress)
|
||||
.concat(option.recipe)
|
||||
.concat(currentRecipeConfig.slice(state.progress + 1)),
|
||||
recipeURL = "recipe=" + Utils.encodeURIFragment(Utils.generatePrettyRecipe(recipeConfig));
|
||||
|
||||
let language = "",
|
||||
fileType = "",
|
||||
matchingOps = "",
|
||||
useful = "";
|
||||
const entropy = `<span data-toggle="tooltip" data-container="body" title="Shannon Entropy is measured from 0 to 8. High entropy suggests encrypted or compressed data. Normal text is usually around 3.5 to 5.">Entropy: <span style="color: ${chooseColour(option.entropy)}">${option.entropy.toFixed(2)}</span></span>`,
|
||||
validUTF8 = option.isUTF8 ? "<span data-toggle='tooltip' data-container='body' title='The data could be a valid UTF8 string based on its encoding.'>Valid UTF8</span>\n" : "";
|
||||
|
||||
if (option.languageScores[0].probability > 0) {
|
||||
let likelyLangs = option.languageScores.filter(l => l.probability > 0);
|
||||
if (likelyLangs.length < 1) likelyLangs = [option.languageScores[0]];
|
||||
language = "<span data-toggle='tooltip' data-container='body' title='Based on a statistical comparison of the frequency of bytes in various languages. Ordered by likelihood.'>" +
|
||||
"Possible languages:\n " +
|
||||
likelyLangs.map(lang => {
|
||||
return MagicLib.codeToLanguage(lang.lang);
|
||||
}).join("\n ") +
|
||||
"</span>\n";
|
||||
}
|
||||
|
||||
if (option.fileType) {
|
||||
fileType = `<span data-toggle="tooltip" data-container="body" title="Based on the presence of magic bytes.">File type: ${option.fileType.mime} (${option.fileType.ext})</span>\n`;
|
||||
}
|
||||
|
||||
if (option.matchingOps.length) {
|
||||
matchingOps = `Matching ops: ${[...new Set(option.matchingOps.map(op => op.op))].join(", ")}\n`;
|
||||
}
|
||||
|
||||
if (option.useful) {
|
||||
useful = "<span data-toggle='tooltip' data-container='body' title='This could be an operation that displays data in a useful way, such as rendering an image.'>Useful op detected</span>\n";
|
||||
}
|
||||
|
||||
output += `<tr>
|
||||
<td><a href="#${recipeURL}">${Utils.generatePrettyRecipe(option.recipe, true)}</a></td>
|
||||
<td>${Utils.escapeHtml(Utils.printable(Utils.truncate(option.data, 99)))}</td>
|
||||
<td>${language}${fileType}${matchingOps}${useful}${validUTF8}${entropy}</td>
|
||||
</tr>`;
|
||||
});
|
||||
|
||||
output += "</table><script type='application/javascript'>$('[data-toggle=\"tooltip\"]').tooltip()</script>";
|
||||
|
||||
if (!options.length) {
|
||||
output = "Nothing of interest could be detected about the input data.\nHave you tried modifying the operation arguments?";
|
||||
}
|
||||
dish.set(output, Dish.HTML);
|
||||
return state;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default Magic;
|
||||
85
src/core/operations/ScanForEmbeddedFiles.mjs
Normal file
85
src/core/operations/ScanForEmbeddedFiles.mjs
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
/**
|
||||
* @author n1474335 [n1474335@gmail.com]
|
||||
* @copyright Crown Copyright 2016
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
|
||||
import Operation from "../Operation";
|
||||
import Utils from "../Utils";
|
||||
import Magic from "../lib/Magic";
|
||||
|
||||
/**
|
||||
* Scan for Embedded Files operation
|
||||
*/
|
||||
class ScanForEmbeddedFiles extends Operation {
|
||||
|
||||
/**
|
||||
* ScanForEmbeddedFiles constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.name = "Scan for Embedded Files";
|
||||
this.module = "Default";
|
||||
this.description = "Scans the data for potential embedded files by looking for magic bytes at all offsets. This operation is prone to false positives.<br><br>WARNING: Files over about 100KB in size will take a VERY long time to process.";
|
||||
this.inputType = "ArrayBuffer";
|
||||
this.outputType = "string";
|
||||
this.args = [
|
||||
{
|
||||
"name": "Ignore common byte sequences",
|
||||
"type": "boolean",
|
||||
"value": true
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ArrayBuffer} input
|
||||
* @param {Object[]} args
|
||||
* @returns {string}
|
||||
*/
|
||||
run(input, args) {
|
||||
let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n",
|
||||
type,
|
||||
numFound = 0,
|
||||
numCommonFound = 0;
|
||||
const ignoreCommon = args[0],
|
||||
commonExts = ["ico", "ttf", ""],
|
||||
data = new Uint8Array(input);
|
||||
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
type = Magic.magicFileType(data.slice(i));
|
||||
if (type) {
|
||||
if (ignoreCommon && commonExts.indexOf(type.ext) > -1) {
|
||||
numCommonFound++;
|
||||
continue;
|
||||
}
|
||||
numFound++;
|
||||
output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" +
|
||||
" File extension: " + type.ext + "\n" +
|
||||
" MIME type: " + type.mime + "\n";
|
||||
|
||||
if (type.desc && type.desc.length) {
|
||||
output += " Description: " + type.desc + "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (numFound === 0) {
|
||||
output += "\nNo embedded files were found.";
|
||||
}
|
||||
|
||||
if (numCommonFound > 0) {
|
||||
output += "\n\n" + numCommonFound;
|
||||
output += numCommonFound === 1 ?
|
||||
" file type was detected that has a common byte sequence. This is likely to be a false positive." :
|
||||
" file types were detected that have common byte sequences. These are likely to be false positives.";
|
||||
output += " Run this operation with the 'Ignore common byte sequences' option unchecked to see details.";
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default ScanForEmbeddedFiles;
|
||||
|
|
@ -23,6 +23,13 @@ class URLDecode extends Operation {
|
|||
this.inputType = "string";
|
||||
this.outputType = "string";
|
||||
this.args = [];
|
||||
this.patterns = [
|
||||
{
|
||||
match: ".*(?:%[\\da-f]{2}.*){4}",
|
||||
flags: "i",
|
||||
args: []
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -39,6 +39,13 @@ class Unzip extends Operation {
|
|||
value: false
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)",
|
||||
flags: "",
|
||||
args: ["", false]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -58,6 +58,13 @@ class ZlibInflate extends Operation {
|
|||
value: false
|
||||
}
|
||||
];
|
||||
this.patterns = [
|
||||
{
|
||||
match: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)",
|
||||
flags: "",
|
||||
args: [0, 0, "Adaptive", false, false]
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -225,8 +225,8 @@ const FileType = {
|
|||
|
||||
if (buf[0] === 0x78 && buf[1] === 0x01) {
|
||||
return {
|
||||
ext: "dmg",
|
||||
mime: "application/x-apple-diskimage"
|
||||
ext: "dmg, zlib",
|
||||
mime: "application/x-apple-diskimage, application/x-deflate"
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -434,8 +434,11 @@ const FileType = {
|
|||
};
|
||||
}
|
||||
|
||||
// Added by n1474335 [n1474335@gmail.com] from here on
|
||||
// ################################################################## //
|
||||
/**
|
||||
*
|
||||
* Added by n1474335 [n1474335@gmail.com] from here on
|
||||
*
|
||||
*/
|
||||
if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) {
|
||||
return {
|
||||
ext: "z, tar.z",
|
||||
|
|
@ -469,16 +472,16 @@ const FileType = {
|
|||
// Must be before Little-endian UTF-16 BOM
|
||||
if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) {
|
||||
return {
|
||||
ext: "",
|
||||
mime: "",
|
||||
ext: "UTF32LE",
|
||||
mime: "charset/utf32le",
|
||||
desc: "Little-endian UTF-32 encoded Unicode byte order mark detected."
|
||||
};
|
||||
}
|
||||
|
||||
if (buf[0] === 0xFF && buf[1] === 0xFE) {
|
||||
return {
|
||||
ext: "",
|
||||
mime: "",
|
||||
ext: "UTF16LE",
|
||||
mime: "charset/utf16le",
|
||||
desc: "Little-endian UTF-16 encoded Unicode byte order mark detected."
|
||||
};
|
||||
}
|
||||
|
|
@ -524,6 +527,13 @@ const FileType = {
|
|||
};
|
||||
}
|
||||
|
||||
if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) {
|
||||
return {
|
||||
ext: "zlib",
|
||||
mime: "application/x-deflate"
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import Recipe from "./Recipe";
|
||||
import Dish from "./Dish";
|
||||
import Recipe from "./Recipe.js";
|
||||
import Dish from "./Dish.js";
|
||||
import Magic from "./lib/Magic.js";
|
||||
import Utils from "./Utils.js";
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -183,7 +185,7 @@ const FlowControl = {
|
|||
* @returns {Object} The updated state of the recipe.
|
||||
*/
|
||||
runJump: function(state) {
|
||||
const ings = state.opList[state.progress].ingValues,
|
||||
const ings = state.opList[state.progress].ingValues,
|
||||
label = ings[0],
|
||||
maxJumps = ings[1],
|
||||
jmpIndex = FlowControl._getLabelIndex(label, state);
|
||||
|
|
@ -211,7 +213,7 @@ const FlowControl = {
|
|||
* @returns {Object} The updated state of the recipe.
|
||||
*/
|
||||
runCondJump: async function(state) {
|
||||
const ings = state.opList[state.progress].ingValues,
|
||||
const ings = state.opList[state.progress].ingValues,
|
||||
dish = state.dish,
|
||||
regexStr = ings[0],
|
||||
invert = ings[1],
|
||||
|
|
@ -266,6 +268,99 @@ const FlowControl = {
|
|||
},
|
||||
|
||||
|
||||
/**
|
||||
* Magic operation.
|
||||
*
|
||||
* @param {Object} state - The current state of the recipe.
|
||||
* @param {number} state.progress - The current position in the recipe.
|
||||
* @param {Dish} state.dish - The Dish being operated on.
|
||||
* @param {Operation[]} state.opList - The list of operations in the recipe.
|
||||
* @returns {Object} The updated state of the recipe.
|
||||
*/
|
||||
runMagic: async function(state) {
|
||||
const ings = state.opList[state.progress].ingValues,
|
||||
depth = ings[0],
|
||||
intensive = ings[1],
|
||||
extLang = ings[2],
|
||||
dish = state.dish,
|
||||
currentRecipeConfig = state.opList.map(op => op.getConfig()),
|
||||
magic = new Magic(dish.get(Dish.ARRAY_BUFFER)),
|
||||
options = await magic.speculativeExecution(depth, extLang, intensive);
|
||||
|
||||
let output = `<table
|
||||
class='table table-hover table-condensed table-bordered'
|
||||
style='table-layout: fixed;'>
|
||||
<tr>
|
||||
<th>Recipe (click to load)</th>
|
||||
<th>Result snippet</th>
|
||||
<th>Properties</th>
|
||||
</tr>`;
|
||||
|
||||
/**
|
||||
* Returns a CSS colour value based on an integer input.
|
||||
*
|
||||
* @param {number} val
|
||||
* @returns {string}
|
||||
*/
|
||||
function chooseColour(val) {
|
||||
if (val < 3) return "green";
|
||||
if (val < 5) return "goldenrod";
|
||||
return "red";
|
||||
}
|
||||
|
||||
options.forEach(option => {
|
||||
// Construct recipe URL
|
||||
// Replace this Magic op with the generated recipe
|
||||
const recipeConfig = currentRecipeConfig.slice(0, state.progress)
|
||||
.concat(option.recipe)
|
||||
.concat(currentRecipeConfig.slice(state.progress + 1)),
|
||||
recipeURL = "recipe=" + Utils.encodeURIFragment(Utils.generatePrettyRecipe(recipeConfig));
|
||||
|
||||
let language = "",
|
||||
fileType = "",
|
||||
matchingOps = "",
|
||||
useful = "",
|
||||
entropy = `<span data-toggle="tooltip" data-container="body" title="Shannon Entropy is measured from 0 to 8. High entropy suggests encrypted or compressed data. Normal text is usually around 3.5 to 5.">Entropy: <span style="color: ${chooseColour(option.entropy)}">${option.entropy.toFixed(2)}</span></span>`,
|
||||
validUTF8 = option.isUTF8 ? "<span data-toggle='tooltip' data-container='body' title='The data could be a valid UTF8 string based on its encoding.'>Valid UTF8</span>\n" : "";
|
||||
|
||||
if (option.languageScores[0].probability > 0) {
|
||||
let likelyLangs = option.languageScores.filter(l => l.probability > 0);
|
||||
if (likelyLangs.length < 1) likelyLangs = [option.languageScores[0]];
|
||||
language = "<span data-toggle='tooltip' data-container='body' title='Based on a statistical comparison of the frequency of bytes in various languages. Ordered by likelihood.'>" +
|
||||
"Possible languages:\n " + likelyLangs.map(lang => {
|
||||
return Magic.codeToLanguage(lang.lang);
|
||||
}).join("\n ") + "</span>\n";
|
||||
}
|
||||
|
||||
if (option.fileType) {
|
||||
fileType = `<span data-toggle="tooltip" data-container="body" title="Based on the presence of magic bytes.">File type: ${option.fileType.mime} (${option.fileType.ext})</span>\n`;
|
||||
}
|
||||
|
||||
if (option.matchingOps.length) {
|
||||
matchingOps = `Matching ops: ${[...new Set(option.matchingOps.map(op => op.op))].join(", ")}\n`;
|
||||
}
|
||||
|
||||
if (option.useful) {
|
||||
useful = "<span data-toggle='tooltip' data-container='body' title='This could be an operation that displays data in a useful way, such as rendering an image.'>Useful op detected</span>\n";
|
||||
}
|
||||
|
||||
output += `<tr>
|
||||
<td><a href="#${recipeURL}">${Utils.generatePrettyRecipe(option.recipe, true)}</a></td>
|
||||
<td>${Utils.escapeHtml(Utils.printable(Utils.truncate(option.data, 99)))}</td>
|
||||
<td>${language}${fileType}${matchingOps}${useful}${validUTF8}${entropy}</td>
|
||||
</tr>`;
|
||||
});
|
||||
|
||||
output += "</table><script type='application/javascript'>$('[data-toggle=\"tooltip\"]').tooltip()</script>";
|
||||
|
||||
if (!options.length) {
|
||||
output = "Nothing of interest could be detected about the input data.\nHave you tried modifying the operation arguments?";
|
||||
}
|
||||
dish.set(output, Dish.HTML);
|
||||
return state;
|
||||
},
|
||||
|
||||
|
||||
/**
|
||||
* Returns the index of a label.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ const PublicKey = {
|
|||
pkStr = "",
|
||||
sig = cert.getSignatureValueHex(),
|
||||
sigStr = "",
|
||||
extensions = cert.getInfo().split("X509v3 Extensions:\n")[1].split("signature")[0];
|
||||
extensions = "";
|
||||
|
||||
// Public Key fields
|
||||
pkFields.push({
|
||||
|
|
@ -144,6 +144,10 @@ const PublicKey = {
|
|||
sigStr = " Signature: " + PublicKey._formatByteStr(sig, 16, 18);
|
||||
}
|
||||
|
||||
// Extensions
|
||||
try {
|
||||
extensions = cert.getInfo().split("X509v3 Extensions:\n")[1].split("signature")[0];
|
||||
} catch (err) {}
|
||||
|
||||
let issuerStr = PublicKey._formatDnStr(issuer, 2),
|
||||
nbDate = PublicKey._formatDate(cert.getNotBefore()),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue