mirrors/CyberChef
1
0
Fork 0
mirror of https://github.com/gchq/CyberChef.git synced 2025-04-22 07:46:16 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'feature-extract-files'

Browse source
This commit is contained in:
n1474335 2019-03-09 06:26:36 +00:00
commit dcb59edb79
26 changed files with 2581 additions and 639 deletions
Download patch file Download diff file Expand all files Collapse all files

35
src/core/operations/DetectFileType.mjs
View file

@ -5,7 +5,8 @@
*/
import Operation from "../Operation";
import Magic from "../lib/Magic";
import {detectFileType} from "../lib/FileType";
import {FILE_SIGNATURES} from "../lib/FileSignatures";
/**
* Detect File Type operation
@ -24,7 +25,13 @@ class DetectFileType extends Operation {
this.infoURL = "https://wikipedia.org/wiki/List_of_file_signatures";
this.inputType = "ArrayBuffer";
this.outputType = "string";
this.args = [];
this.args = Object.keys(FILE_SIGNATURES).map(cat => {
return {
name: cat,
type: "boolean",
value: true
};
});
}
/**
@ -34,17 +41,27 @@ class DetectFileType extends Operation {
*/
run(input, args) {
const data = new Uint8Array(input),
type = Magic.magicFileType(data);
categories = [];
if (!type) {
args.forEach((cat, i) => {
if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]);
});
const types = detectFileType(data, categories);
if (!types.length) {
return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?";
} else {
let output = "File extension: " + type.ext + "\n" +
"MIME type: " + type.mime;
let output = "";
if (type.desc && type.desc.length) {
output += "\nDescription: " + type.desc;
}
types.forEach(type => {
output += "File extension: " + type.extension + "\n" +
"MIME type: " + type.mime + "\n";
if (type.description && type.description.length) {
output += "\nDescription: " + type.description + "\n";
}
});
return output;
}

100
src/core/operations/ExtractFiles.mjs Normal file
View file

@ -0,0 +1,100 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2018
* @license Apache-2.0
*/
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
import {scanForFileTypes, extractFile} from "../lib/FileType";
import {FILE_SIGNATURES} from "../lib/FileSignatures";
/**
* Extract Files operation
*/
class ExtractFiles extends Operation {
/**
* ExtractFiles constructor
*/
constructor() {
super();
this.name = "Extract Files";
this.module = "Default";
this.description = "TODO";
this.infoURL = "https://forensicswiki.org/wiki/File_Carving";
this.inputType = "ArrayBuffer";
this.outputType = "List<File>";
this.presentType = "html";
this.args = Object.keys(FILE_SIGNATURES).map(cat => {
return {
name: cat,
type: "boolean",
value: cat === "Miscellaneous" ? false : true
};
}).concat([
{
name: "Ignore failed extractions",
type: "boolean",
value: "true"
}
]);
}
/**
* @param {ArrayBuffer} input
* @param {Object[]} args
* @returns {List<File>}
*/
run(input, args) {
const bytes = new Uint8Array(input),
categories = [],
ignoreFailedExtractions = args.pop(1);
args.forEach((cat, i) => {
if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]);
});
// Scan for embedded files
const detectedFiles = scanForFileTypes(bytes, categories);
// Extract each file that we support
const files = [];
const errors = [];
detectedFiles.forEach(detectedFile => {
try {
files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset));
} catch (err) {
if (!ignoreFailedExtractions && err.message.indexOf("No extraction algorithm available") < 0) {
errors.push(
`Error while attempting to extract ${detectedFile.fileDetails.name} ` +
`at offset ${detectedFile.offset}:\n` +
`${err.message}`
);
}
}
});
if (errors.length) {
throw new OperationError(errors.join("\n\n"));
}
return files;
}
/**
* Displays the files in HTML for web apps.
*
* @param {File[]} files
* @returns {html}
*/
async present(files) {
return await Utils.displayFilesAsHTML(files);
}
}
export default ExtractFiles;

8
src/core/operations/GenerateQRCode.mjs
View file

@ -8,7 +8,7 @@ import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import qr from "qr-image";
import { toBase64 } from "../lib/Base64";
import Magic from "../lib/Magic";
import { isImage } from "../lib/FileType";
import Utils from "../Utils";
/**
@ -100,9 +100,9 @@ class GenerateQRCode extends Operation {
if (format === "PNG") {
let dataURI = "data:";
const type = Magic.magicFileType(data);
if (type && type.mime.indexOf("image") === 0){
dataURI += type.mime + ";";
const mime = isImage(data);
if (mime){
dataURI += mime + ";";
} else {
throw new OperationError("Invalid PNG file generated by QR image");
}

89
src/core/operations/ParseQRCode.mjs
View file

@ -6,7 +6,7 @@
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Magic from "../lib/Magic";
import { isImage } from "../lib/FileType";
import jsqr from "jsqr";
import jimp from "jimp";
@ -42,64 +42,61 @@ class ParseQRCode extends Operation {
* @returns {string}
*/
async run(input, args) {
const type = Magic.magicFileType(input);
const [normalise] = args;
// Make sure that the input is an image
if (type && type.mime.indexOf("image") === 0) {
let image = input;
if (!isImage(input)) throw new OperationError("Invalid file type.");
if (normalise) {
// Process the image to be easier to read by jsqr
// Disables the alpha channel
// Sets the image default background to white
// Normalises the image colours
// Makes the image greyscale
// Converts image to a JPEG
image = await new Promise((resolve, reject) => {
jimp.read(Buffer.from(input))
.then(image => {
image
.rgba(false)
.background(0xFFFFFFFF)
.normalize()
.greyscale()
.getBuffer(jimp.MIME_JPEG, (error, result) => {
resolve(result);
});
})
.catch(err => {
reject(new OperationError("Error reading the image file."));
});
});
}
let image = input;
if (image instanceof OperationError) {
throw image;
}
return new Promise((resolve, reject) => {
jimp.read(Buffer.from(image))
if (normalise) {
// Process the image to be easier to read by jsqr
// Disables the alpha channel
// Sets the image default background to white
// Normalises the image colours
// Makes the image greyscale
// Converts image to a JPEG
image = await new Promise((resolve, reject) => {
jimp.read(Buffer.from(input))
.then(image => {
if (image.bitmap != null) {
const qrData = jsqr(image.bitmap.data, image.getWidth(), image.getHeight());
if (qrData != null) {
resolve(qrData.data);
} else {
reject(new OperationError("Couldn't read a QR code from the image."));
}
} else {
reject(new OperationError("Error reading the image file."));
}
image
.rgba(false)
.background(0xFFFFFFFF)
.normalize()
.greyscale()
.getBuffer(jimp.MIME_JPEG, (error, result) => {
resolve(result);
});
})
.catch(err => {
reject(new OperationError("Error reading the image file."));
});
});
} else {
throw new OperationError("Invalid file type.");
}
if (image instanceof OperationError) {
throw image;
}
return new Promise((resolve, reject) => {
jimp.read(Buffer.from(image))
.then(image => {
if (image.bitmap != null) {
const qrData = jsqr(image.bitmap.data, image.getWidth(), image.getHeight());
if (qrData != null) {
resolve(qrData.data);
} else {
reject(new OperationError("Couldn't read a QR code from the image."));
}
} else {
reject(new OperationError("Error reading the image file."));
}
})
.catch(err => {
reject(new OperationError("Error reading the image file."));
});
});
}
}

13
src/core/operations/PlayMedia.mjs
View file

@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex";
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
import Magic from "../lib/Magic";
import { isType, detectFileType } from "../lib/FileType";
/**
* PlayMedia operation
@ -66,8 +66,7 @@ class PlayMedia extends Operation {
// Determine file type
const type = Magic.magicFileType(input);
if (!(type && /^audio|video/.test(type.mime))) {
if (!isType(/^(audio|video)/, input)) {
throw new OperationError("Invalid or unrecognised file type");
}
@ -84,15 +83,15 @@ class PlayMedia extends Operation {
async present(data) {
if (!data.length) return "";
const type = Magic.magicFileType(data);
const matches = /^audio|video/.exec(type.mime);
const types = detectFileType(data);
const matches = /^audio|video/.exec(types[0].mime);
if (!matches) {
throw new OperationError("Invalid file type");
}
const dataURI = `data:${type.mime};base64,${toBase64(data)}`;
const dataURI = `data:${types[0].mime};base64,${toBase64(data)}`;
const element = matches[0];
let html = `<${element} src='${dataURI}' type='${type.mime}' controls>`;
let html = `<${element} src='${dataURI}' type='${types[0].mime}' controls>`;
html += "<p>Unsupported media type.</p>";
html += `</${element}>`;
return html;

11
src/core/operations/RenderImage.mjs
View file

@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex";
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
import Magic from "../lib/Magic";
import {isImage} from "../lib/FileType";
/**
* Render Image operation
@ -72,8 +72,7 @@ class RenderImage extends Operation {
}
// Determine file type
const type = Magic.magicFileType(input);
if (!(type && type.mime.indexOf("image") === 0)) {
if (!isImage(input)) {
throw new OperationError("Invalid file type");
}
@ -92,9 +91,9 @@ class RenderImage extends Operation {
let dataURI = "data:";
// Determine file type
const type = Magic.magicFileType(data);
if (type && type.mime.indexOf("image") === 0) {
dataURI += type.mime + ";";
const mime = isImage(data);
if (mime) {
dataURI += mime + ";";
} else {
throw new OperationError("Invalid file type");
}

61
src/core/operations/ScanForEmbeddedFiles.mjs
View file

@ -6,7 +6,8 @@
import Operation from "../Operation";
import Utils from "../Utils";
import Magic from "../lib/Magic";
import {scanForFileTypes} from "../lib/FileType";
import {FILE_SIGNATURES} from "../lib/FileSignatures";
/**
* Scan for Embedded Files operation
@ -25,13 +26,13 @@ class ScanForEmbeddedFiles extends Operation {
this.infoURL = "https://wikipedia.org/wiki/List_of_file_signatures";
this.inputType = "ArrayBuffer";
this.outputType = "string";
this.args = [
{
"name": "Ignore common byte sequences",
"type": "boolean",
"value": true
}
];
this.args = Object.keys(FILE_SIGNATURES).map(cat => {
return {
name: cat,
type: "boolean",
value: cat === "Miscellaneous" ? false : true
};
});
}
/**
@ -41,43 +42,33 @@ class ScanForEmbeddedFiles extends Operation {
*/
run(input, args) {
let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n",
type,
numFound = 0,
numCommonFound = 0;
const ignoreCommon = args[0],
commonExts = ["ico", "ttf", ""],
numFound = 0;
const categories = [],
data = new Uint8Array(input);
for (let i = 0; i < data.length; i++) {
type = Magic.magicFileType(data.slice(i));
if (type) {
if (ignoreCommon && commonExts.indexOf(type.ext) > -1) {
numCommonFound++;
continue;
}
numFound++;
output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" +
" File extension: " + type.ext + "\n" +
" MIME type: " + type.mime + "\n";
args.forEach((cat, i) => {
if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]);
});
if (type.desc && type.desc.length) {
output += " Description: " + type.desc + "\n";
const types = scanForFileTypes(data, categories);
if (types.length) {
types.forEach(type => {
numFound++;
output += "\nOffset " + type.offset + " (0x" + Utils.hex(type.offset) + "):\n" +
" File extension: " + type.fileDetails.extension + "\n" +
" MIME type: " + type.fileDetails.mime + "\n";
if (type.fileDetails.description && type.fileDetails.description.length) {
output += " Description: " + type.fileDetails.description + "\n";
}
}
});
}
if (numFound === 0) {
output += "\nNo embedded files were found.";
}
if (numCommonFound > 0) {
output += "\n\n" + numCommonFound;
output += numCommonFound === 1 ?
" file type was detected that has a common byte sequence. This is likely to be a false positive." :
" file types were detected that have common byte sequences. These are likely to be false positives.";
output += " Run this operation with the 'Ignore common byte sequences' option unchecked to see details.";
}
return output;
}

89
src/core/operations/SplitColourChannels.mjs
View file

@ -7,7 +7,7 @@
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
import Magic from "../lib/Magic";
import {isImage} from "../lib/FileType";
import jimp from "jimp";
@ -38,56 +38,53 @@ class SplitColourChannels extends Operation {
* @returns {List<File>}
*/
async run(input, args) {
const type = Magic.magicFileType(input);
// Make sure that the input is an image
if (type && type.mime.indexOf("image") === 0) {
const parsedImage = await jimp.read(Buffer.from(input));
if (!isImage(input)) throw new OperationError("Invalid file type.");
const red = new Promise(async (resolve, reject) => {
try {
const split = parsedImage
.clone()
.color([
{apply: "blue", params: [-255]},
{apply: "green", params: [-255]}
])
.getBufferAsync(jimp.MIME_PNG);
resolve(new File([new Uint8Array((await split).values())], "red.png", {type: "image/png"}));
} catch (err) {
reject(new OperationError(`Could not split red channel: ${err}`));
}
});
const parsedImage = await jimp.read(Buffer.from(input));
const green = new Promise(async (resolve, reject) => {
try {
const split = parsedImage.clone()
.color([
{apply: "red", params: [-255]},
{apply: "blue", params: [-255]},
]).getBufferAsync(jimp.MIME_PNG);
resolve(new File([new Uint8Array((await split).values())], "green.png", {type: "image/png"}));
} catch (err) {
reject(new OperationError(`Could not split green channel: ${err}`));
}
});
const red = new Promise(async (resolve, reject) => {
try {
const split = parsedImage
.clone()
.color([
{apply: "blue", params: [-255]},
{apply: "green", params: [-255]}
])
.getBufferAsync(jimp.MIME_PNG);
resolve(new File([new Uint8Array((await split).values())], "red.png", {type: "image/png"}));
} catch (err) {
reject(new OperationError(`Could not split red channel: ${err}`));
}
});
const blue = new Promise(async (resolve, reject) => {
try {
const split = parsedImage
.color([
{apply: "red", params: [-255]},
{apply: "green", params: [-255]},
]).getBufferAsync(jimp.MIME_PNG);
resolve(new File([new Uint8Array((await split).values())], "blue.png", {type: "image/png"}));
} catch (err) {
reject(new OperationError(`Could not split blue channel: ${err}`));
}
});
const green = new Promise(async (resolve, reject) => {
try {
const split = parsedImage.clone()
.color([
{apply: "red", params: [-255]},
{apply: "blue", params: [-255]},
]).getBufferAsync(jimp.MIME_PNG);
resolve(new File([new Uint8Array((await split).values())], "green.png", {type: "image/png"}));
} catch (err) {
reject(new OperationError(`Could not split green channel: ${err}`));
}
});
return await Promise.all([red, green, blue]);
} else {
throw new OperationError("Invalid file type.");
}
const blue = new Promise(async (resolve, reject) => {
try {
const split = parsedImage
.color([
{apply: "red", params: [-255]},
{apply: "green", params: [-255]},
]).getBufferAsync(jimp.MIME_PNG);
resolve(new File([new Uint8Array((await split).values())], "blue.png", {type: "image/png"}));
} catch (err) {
reject(new OperationError(`Could not split blue channel: ${err}`));
}
});
return await Promise.all([red, green, blue]);
}
/**

35
src/core/operations/Untar.mjs
View file

@ -6,6 +6,7 @@
import Operation from "../Operation";
import Utils from "../Utils";
import Stream from "../lib/Stream";
/**
* Untar operation
@ -41,38 +42,6 @@ class Untar extends Operation {
* @returns {List<File>}
*/
run(input, args) {
const Stream = function(input) {
this.bytes = input;
this.position = 0;
};
Stream.prototype.getBytes = function(bytesToGet) {
const newPosition = this.position + bytesToGet;
const bytes = this.bytes.slice(this.position, newPosition);
this.position = newPosition;
return bytes;
};
Stream.prototype.readString = function(numBytes) {
let result = "";
for (let i = this.position; i < this.position + numBytes; i++) {
const currentByte = this.bytes[i];
if (currentByte === 0) break;
result += String.fromCharCode(currentByte);
}
this.position += numBytes;
return result;
};
Stream.prototype.readInt = function(numBytes, base) {
const string = this.readString(numBytes);
return parseInt(string, base);
};
Stream.prototype.hasMore = function() {
return this.position < this.bytes.length;
};
const stream = new Stream(input),
files = [];
@ -85,7 +54,7 @@ class Untar extends Operation {
ownerUID: stream.readString(8),
ownerGID: stream.readString(8),
size: parseInt(stream.readString(12), 8), // Octal
lastModTime: new Date(1000 * stream.readInt(12, 8)), // Octal
lastModTime: new Date(1000 * parseInt(stream.readString(12), 8)), // Octal
checksum: stream.readString(8),
type: stream.readString(1),
linkedFileName: stream.readString(100),