merge 8.0.1 release -> node-lib

This commit is contained in:
d98762625 2018-08-11 22:15:09 +01:00
commit 7c1ac4392e
63 changed files with 4608 additions and 4331 deletions

View file

@ -96,7 +96,7 @@ class Chef {
const returnType = this.dish.size > threshold ? Dish.ARRAY_BUFFER : Dish.STRING;
// Create a raw version of the dish, unpresented
const rawDish = new Dish(this.dish);
const rawDish = this.dish.clone();
// Present the raw result
await recipe.present(this.dish);

View file

@ -301,6 +301,69 @@ class Dish {
}
}
/**
* Returns a deep clone of the current Dish.
*
* @returns {Dish}
*/
clone() {
const newDish = new Dish();
switch (this.type) {
case Dish.STRING:
case Dish.HTML:
case Dish.NUMBER:
case Dish.BIG_NUMBER:
// These data types are immutable so it is acceptable to copy them by reference
newDish.set(
this.value,
this.type
);
break;
case Dish.BYTE_ARRAY:
case Dish.JSON:
// These data types are mutable so they need to be copied by value
newDish.set(
JSON.parse(JSON.stringify(this.value)),
this.type
);
break;
case Dish.ARRAY_BUFFER:
// Slicing an ArrayBuffer returns a new ArrayBuffer with a copy its contents
newDish.set(
this.value.slice(0),
this.type
);
break;
case Dish.FILE:
// A new file can be created by copying over all the values from the original
newDish.set(
new File([this.value], this.value.name, {
"type": this.value.type,
"lastModified": this.value.lastModified
}),
this.type
);
break;
case Dish.LIST_FILE:
newDish.set(
this.value.map(f =>
new File([f], f.name, {
"type": f.type,
"lastModified": f.lastModified
})
),
this.type
);
break;
default:
throw new Error("Cannot clone Dish, unknown type");
}
return newDish;
}
}

View file

@ -21,7 +21,10 @@ class Ingredient {
this.name = "";
this.type = "";
this._value = null;
this.disabled = false;
this.hint = "";
this.toggleValues = [];
this.target = null;
if (ingredientConfig) {
this._parseConfig(ingredientConfig);
@ -39,7 +42,10 @@ class Ingredient {
this.name = ingredientConfig.name;
this.type = ingredientConfig.type;
this.defaultValue = ingredientConfig.value;
this.disabled = !!ingredientConfig.disabled;
this.hint = ingredientConfig.hint || false;
this.toggleValues = ingredientConfig.toggleValues;
this.target = typeof ingredientConfig.target !== "undefined" ? ingredientConfig.target : null;
}

View file

@ -170,12 +170,17 @@ class Operation {
*/
get args() {
return this._ingList.map(ing => {
return {
const conf = {
name: ing.name,
type: ing.type,
value: ing.defaultValue,
toggleValues: ing.toggleValues || []
value: ing.defaultValue
};
if (ing.toggleValues) conf.toggleValues = ing.toggleValues;
if (ing.hint) conf.hint = ing.hint;
if (ing.disabled) conf.disabled = ing.disabled;
if (ing.target) conf.target = ing.target;
return conf;
});
}

View file

@ -828,11 +828,11 @@ class Utils {
*/
static async displayFilesAsHTML(files) {
const formatDirectory = function(file) {
const html = `<div class='panel panel-default' style='white-space: normal;'>
<div class='panel-heading' role='tab'>
<h4 class='panel-title'>
const html = `<div class='card' style='white-space: normal;'>
<div class='card-header'>
<h6 class="mb-0">
${Utils.escapeHtml(file.name)}
</h4>
</h6>
</div>
</div>`;
return html;
@ -845,29 +845,29 @@ class Utils {
{type: "octet/stream"}
);
const html = `<div class='panel panel-default' style='white-space: normal;'>
<div class='panel-heading' role='tab' id='heading${i}'>
<h4 class='panel-title'>
<div>
<a href='#collapse${i}'
class='collapsed'
data-toggle='collapse'
aria-expanded='true'
aria-controls='collapse${i}'
title="Show/hide contents of '${Utils.escapeHtml(file.name)}'">${Utils.escapeHtml(file.name)}</a>
<a href='${URL.createObjectURL(blob)}'
title='Download ${Utils.escapeHtml(file.name)}'
download='${Utils.escapeHtml(file.name)}'>&#x1f4be;</a>
<span class='pull-right'>
${file.size.toLocaleString()} bytes
</span>
</div>
</h4>
const html = `<div class='card' style='white-space: normal;'>
<div class='card-header' id='heading${i}'>
<h6 class='mb-0'>
<a class='collapsed'
data-toggle='collapse'
href='#collapse${i}'
aria-expanded='false'
aria-controls='collapse${i}'
title="Show/hide contents of '${Utils.escapeHtml(file.name)}'">
${Utils.escapeHtml(file.name)}</a>
<span class='float-right' style="margin-top: -3px">
${file.size.toLocaleString()} bytes
<a title="Download ${Utils.escapeHtml(file.name)}"
href='${URL.createObjectURL(blob)}'
download='${Utils.escapeHtml(file.name)}'>
<i class="material-icons" style="vertical-align: bottom">save</i>
</a>
</span>
</h6>
</div>
<div id='collapse${i}' class='panel-collapse collapse'
role='tabpanel' aria-labelledby='heading${i}'>
<div class='panel-body'>
<pre><code>${Utils.escapeHtml(Utils.arrayBufferToStr(buff.buffer))}</code></pre>
<div id='collapse${i}' class='collapse' aria-labelledby='heading${i}' data-parent="#files">
<div class='card-body'>
<pre>${Utils.escapeHtml(Utils.arrayBufferToStr(buff.buffer))}</pre>
</div>
</div>
</div>`;
@ -875,8 +875,8 @@ class Utils {
};
let html = `<div style='padding: 5px; white-space: normal;'>
${files.length} file(s) found<NL>
</div>`;
${files.length} file(s) found
</div><div id="files" style="padding: 20px">`;
for (let i = 0; i < files.length; i++) {
if (files[i].name.endsWith("/")) {
@ -886,7 +886,7 @@ class Utils {
}
}
return html;
return html += "</div>";
}

View file

@ -328,6 +328,7 @@
"Generate UUID",
"Generate TOTP",
"Generate HOTP",
"Haversine distance",
"Render Image",
"Remove EXIF",
"Extract EXIF",

View file

@ -48,7 +48,7 @@ export function drawBarChart(canvas, scores, xAxisLabel, yAxisLabel, numXLabels,
leftPadding = canvas.width * 0.08,
rightPadding = canvas.width * 0.03,
topPadding = canvas.height * 0.08,
bottomPadding = canvas.height * 0.15,
bottomPadding = canvas.height * 0.2,
graphHeight = canvas.height - topPadding - bottomPadding,
graphWidth = canvas.width - leftPadding - rightPadding,
base = topPadding + graphHeight,
@ -146,7 +146,7 @@ export function drawScaleBar(canvas, score, max, markings) {
leftPadding = canvas.width * 0.01,
rightPadding = canvas.width * 0.01,
topPadding = canvas.height * 0.1,
bottomPadding = canvas.height * 0.3,
bottomPadding = canvas.height * 0.35,
barHeight = canvas.height - topPadding - bottomPadding,
barWidth = canvas.width - leftPadding - rightPadding;

View file

@ -49,8 +49,8 @@ export const DATETIME_FORMATS = [
* MomentJS DateTime formatting examples.
*/
export const FORMAT_EXAMPLES = `Format string tokens:
<table class="table table-striped table-hover table-condensed table-bordered" style="font-family: sans-serif">
<thead>
<table class="table table-striped table-hover table-sm table-bordered" style="font-family: sans-serif">
<thead class="thead-dark">
<tr>
<th>Category</th>
<th>Token</th>

View file

@ -287,6 +287,8 @@ class Magic {
useful: useful
});
const prevOp = recipeConfig[recipeConfig.length - 1];
// Execute each of the matching operations, then recursively call the speculativeExecution()
// method on the resulting data, recording the properties of each option.
await Promise.all(matchingOps.map(async op => {
@ -294,8 +296,14 @@ class Magic {
op: op.op,
args: op.args
},
output = await this._runRecipe([opConfig]),
magic = new Magic(output, this.opPatterns),
output = await this._runRecipe([opConfig]);
// If the recipe is repeating and returning the same data, do not continue
if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
return;
}
const magic = new Magic(output, this.opPatterns),
speculativeResults = await magic.speculativeExecution(
depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
@ -315,13 +323,16 @@ class Magic {
}));
}
// Prune branches that do not match anything
// Prune branches that result in unhelpful outputs
results = results.filter(r =>
r.languageScores[0].probability > 0 ||
r.fileType ||
r.isUTF8 ||
r.matchingOps.length ||
r.useful);
(r.useful || r.data.length > 0) && // The operation resulted in ""
( // One of the following must be true
r.languageScores[0].probability > 0 || // Some kind of language was found
r.fileType || // A file was found
r.isUTF8 || // UTF-8 was found
r.matchingOps.length // A matching op was found
)
);
// Return a sorted list of possible recipes along with their properties
return results.sort((a, b) => {
@ -374,7 +385,7 @@ class Magic {
const recipe = new Recipe(recipeConfig);
try {
await recipe.execute(dish, 0);
await recipe.execute(dish);
return dish.get(Dish.ARRAY_BUFFER);
} catch (err) {
// If there are errors, return an empty buffer
@ -395,7 +406,10 @@ class Magic {
let i = len;
const counts = new Array(256).fill(0);
if (!len) return counts;
if (!len) {
this.freqDist = counts;
return this.freqDist;
}
while (i--) {
counts[this.inputBuffer[i]]++;

View file

@ -47,9 +47,10 @@ class Diff extends Operation {
"value": true
},
{
"name": "Ignore whitespace (relevant for word and line)",
"name": "Ignore whitespace",
"type": "boolean",
"value": false
"value": false,
"hint": "Relevant for word and line"
}
];
}

View file

@ -20,7 +20,7 @@ class Entropy extends Operation {
this.name = "Entropy";
this.module = "Default";
this.description = "Calculates the Shannon entropy of the input data which gives an idea of its randomness. 8 is the maximum.";
this.description = "Shannon Entropy, in the context of information theory, is a measure of the rate at which information is produced by a source of data. It can be used, in a broad sense, to detect whether data is likely to be structured or unstructured. 8 is the maximum, representing highly unstructured, 'random' data. English language text usually falls somewhere between 3.5 and 5. Properly encrypted or compressed data should have an entropy of over 7.5.";
this.inputType = "byteArray";
this.outputType = "number";
this.presentType = "html";

View file

@ -26,7 +26,7 @@ class FromHexdump extends Operation {
this.args = [];
this.patterns = [
{
match: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?)+$",
match: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?){2,}$",
flags: "i",
args: []
},

View file

@ -0,0 +1,58 @@
/**
* @author Dachande663 [dachande663@gmail.com]
* @copyright Crown Copyright 2018
* @license Apache-2.0
*/
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
/**
* HaversineDistance operation
*/
class HaversineDistance extends Operation {
/**
* HaversineDistance constructor
*/
constructor() {
super();
this.name = "Haversine distance";
this.module = "Default";
this.description = "Returns the distance between two pairs of GPS latitude and longitude co-ordinates in metres.<br><br>e.g. <code>51.487263,-0.124323, 38.9517,-77.1467</code>";
this.inputType = "string";
this.outputType = "number";
this.args = [];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {number}
*/
run(input, args) {
const values = input.match(/^(-?\d+(\.\d+)?), ?(-?\d+(\.\d+)?), ?(-?\d+(\.\d+)?), ?(-?\d+(\.\d+)?)$/);
if (!values) {
throw new OperationError("Input must in the format lat1, lng1, lat2, lng2");
}
const lat1 = parseFloat(values[1]);
const lng1 = parseFloat(values[3]);
const lat2 = parseFloat(values[6]);
const lng2 = parseFloat(values[8]);
const TO_RAD = Math.PI / 180;
const dLat = (lat2-lat1) * TO_RAD;
const dLng = (lng2-lng1) * TO_RAD;
const a = Math.sin(dLat/2) * Math.sin(dLat/2) + Math.cos(lat1 * TO_RAD) * Math.cos(lat2 * TO_RAD) * Math.sin(dLng/2) * Math.sin(dLng/2);
const metres = 6371000 * 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a));
return metres;
}
}
export default HaversineDistance;

View file

@ -23,7 +23,7 @@ class Magic extends Operation {
this.name = "Magic";
this.flowControl = true;
this.module = "Default";
this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.";
this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.<br><br>A technical explanation of the Magic operation can be found <a href='https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic'>here</a>.";
this.inputType = "ArrayBuffer";
this.outputType = "JSON";
this.presentType = "html";
@ -77,7 +77,7 @@ class Magic extends Operation {
const currentRecipeConfig = this.state.opList.map(op => op.config);
let output = `<table
class='table table-hover table-condensed table-bordered'
class='table table-hover table-sm table-bordered'
style='table-layout: fixed;'>
<tr>
<th>Recipe (click to load)</th>

View file

@ -113,7 +113,7 @@ CMYK: ${cmyk}
document.getElementById('input-text').value = 'rgba(' +
color.r + ', ' + color.g + ', ' + color.b + ', ' + color.a + ')';
window.app.autoBake();
});
}).children(".colorpicker").removeClass('dropdown-menu');
</script>`;
}

View file

@ -7,7 +7,6 @@
import Operation from "../Operation";
import moment from "moment-timezone";
import {DATETIME_FORMATS, FORMAT_EXAMPLES} from "../lib/DateTime";
import OperationError from "../errors/OperationError";
/**
* Parse DateTime operation
@ -60,7 +59,7 @@ class ParseDateTime extends Operation {
date = moment.tz(input, inputFormat, inputTimezone);
if (!date || date.format() === "Invalid date") throw Error;
} catch (err) {
throw new OperationError(`Invalid format.\n\n${FORMAT_EXAMPLES}`);
return `Invalid format.\n\n${FORMAT_EXAMPLES}`;
}
output += "Date: " + date.format("dddd Do MMMM YYYY") +

View file

@ -97,7 +97,7 @@ class ParseIPv4Header extends Operation {
checksumResult = givenChecksum + " (incorrect, should be " + correctChecksum + ")";
}
output = `<table class='table table-hover table-condensed table-bordered table-nonfluid'><tr><th>Field</th><th>Value</th></tr>
output = `<table class='table table-hover table-sm table-bordered table-nonfluid'><tr><th>Field</th><th>Value</th></tr>
<tr><td>Version</td><td>${version}</td></tr>
<tr><td>Internet Header Length (IHL)</td><td>${ihl} (${ihl * 4} bytes)</td></tr>
<tr><td>Differentiated Services Code Point (DSCP)</td><td>${dscp}</td></tr>

View file

@ -26,7 +26,8 @@ class RenderImage extends Operation {
this.module = "Image";
this.description = "Displays the input as an image. Supports the following formats:<br><br><ul><li>jpg/jpeg</li><li>png</li><li>gif</li><li>webp</li><li>bmp</li><li>ico</li></ul>";
this.inputType = "string";
this.outputType = "html";
this.outputType = "byteArray";
this.presentType = "html";
this.args = [
{
"name": "Input format",
@ -51,9 +52,8 @@ class RenderImage extends Operation {
*/
run(input, args) {
const inputFormat = args[0];
let dataURI = "data:";
if (!input.length) return "";
if (!input.length) return [];
// Convert input to raw bytes
switch (inputFormat) {
@ -73,6 +73,26 @@ class RenderImage extends Operation {
// Determine file type
const type = Magic.magicFileType(input);
if (!(type && type.mime.indexOf("image") === 0)) {
throw new OperationError("Invalid file type");
}
return input;
}
/**
* Displays the image using HTML for web apps.
*
* @param {byteArray} data
* @returns {html}
*/
async present(data) {
if (!data.length) return "";
let dataURI = "data:";
// Determine file type
const type = Magic.magicFileType(data);
if (type && type.mime.indexOf("image") === 0) {
dataURI += type.mime + ";";
} else {
@ -80,7 +100,7 @@ class RenderImage extends Operation {
}
// Add image data to URI
dataURI += "base64," + toBase64(input);
dataURI += "base64," + toBase64(data);
return "<img src='" + dataURI + "'>";
}

View file

@ -147,12 +147,12 @@ class ToTable extends Operation {
*/
function htmlOutput(tableData) {
// Start the HTML output with suitable classes for styling.
let output = "<table class='table table-hover table-condensed table-bordered table-nonfluid'>";
let output = "<table class='table table-hover table-sm table-bordered table-nonfluid'>";
// If the first row is a header then put it in <thead> with <th> cells.
if (firstRowHeader) {
const row = tableData.shift();
output += "<thead>";
output += "<thead class='thead-light'>";
output += outputRow(row, "th");
output += "</thead>";
}

View file

@ -7,7 +7,6 @@
import Operation from "../Operation";
import moment from "moment-timezone";
import {DATETIME_FORMATS, FORMAT_EXAMPLES} from "../lib/DateTime";
import OperationError from "../errors/OperationError";
/**
* Translate DateTime Format operation
@ -68,7 +67,7 @@ class TranslateDateTimeFormat extends Operation {
date = moment.tz(input, inputFormat, inputTimezone);
if (!date || date.format() === "Invalid date") throw Error;
} catch (err) {
throw new OperationError(`Invalid format.\n\n${FORMAT_EXAMPLES}`);
return `Invalid format.\n\n${FORMAT_EXAMPLES}`;
}
return date.tz(outputTimezone).format(outputFormat);