mirrors/CyberChef
1
0
Fork 0
mirror of https://github.com/gchq/CyberChef.git synced 2025-04-21 23:36:16 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into master

Browse source
This commit is contained in:
VirtualColossus 2019-10-31 15:33:54 +00:00 • committed by GitHub
commit b9571db9f1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 225 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

6
CHANGELOG.md
View file

@ -2,6 +2,9 @@
All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
### [9.8.0] - 2019-10-31
- 'Avro to JSON' operation added [@jarrodconnolly] | [#865]
### [9.7.0] - 2019-09-13
- 'Optical Character Recognition' operation added [@MShwed] [@n1474335] | [#632]
@ -185,6 +188,7 @@ All major and minor version changes will be documented in this file. Details of
[9.8.0]: https://github.com/gchq/CyberChef/releases/tag/v9.8.0
[9.7.0]: https://github.com/gchq/CyberChef/releases/tag/v9.7.0
[9.6.0]: https://github.com/gchq/CyberChef/releases/tag/v9.6.0
[9.5.0]: https://github.com/gchq/CyberChef/releases/tag/v9.5.0
@ -262,6 +266,7 @@ All major and minor version changes will be documented in this file. Details of
[@Ge0rg3]: https://github.com/Ge0rg3
[@MShwed]: https://github.com/MShwed
[@kassi]: https://github.com/kassi
[@jarrodconnolly]: https://github.com/jarrodconnolly
[#95]: https://github.com/gchq/CyberChef/pull/299
[#173]: https://github.com/gchq/CyberChef/pull/173
@ -322,3 +327,4 @@ All major and minor version changes will be documented in this file. Details of
[#625]: https://github.com/gchq/CyberChef/pull/625
[#627]: https://github.com/gchq/CyberChef/pull/627
[#632]: https://github.com/gchq/CyberChef/pull/632
[#865]: https://github.com/gchq/CyberChef/pull/865

7
package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "cyberchef",
"version": "9.7.19",
"version": "9.8.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -2149,6 +2149,11 @@
}
}
},
"avsc": {
"version": "5.4.16",
"resolved": "https://registry.npmjs.org/avsc/-/avsc-5.4.16.tgz",
"integrity": "sha512-Z85B8ZaEU2PWNPRJYuMSp5Hg7Nw3KPKW47lW/Kus7AcwV7fr6uJG3UckagqIPLydIeO/Cm+yjnJG7g0tliICOg=="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",

3
package.json
View file

@ -1,6 +1,6 @@
{
"name": "cyberchef",
"version": "9.7.19",
"version": "9.8.0",
"description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
"author": "n1474335 <n1474335@gmail.com>",
"homepage": "https://gchq.github.io/CyberChef",
@ -86,6 +86,7 @@
"@babel/polyfill": "^7.4.4",
"@babel/runtime": "^7.5.5",
"arrive": "^2.4.1",
"avsc": "^5.4.16",
"babel-plugin-transform-builtin-extend": "1.1.2",
"bcryptjs": "^2.4.3",
"bignumber.js": "^9.0.0",

3
src/core/config/Categories.json
View file

@ -59,7 +59,8 @@
"From Braille",
"Parse TLV",
"CSV to JSON",
"JSON to CSV"
"JSON to CSV",
"Avro to JSON"
]
},
{

76
src/core/operations/AvroToJSON.mjs Normal file
View file

@ -0,0 +1,76 @@
/**
* @author jarrodconnolly [jarrod@nestedquotes.ca]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation.mjs";
import OperationError from "../errors/OperationError.mjs";
import avro from "avsc";
/**
* Avro to JSON operation
*/
class AvroToJSON extends Operation {
/**
* AvroToJSON constructor
*/
constructor() {
super();
this.name = "Avro to JSON";
this.module = "Serialise";
this.description = "Converts Avro encoded data into JSON.";
this.infoURL = "https://wikipedia.org/wiki/Apache_Avro";
this.inputType = "ArrayBuffer";
this.outputType = "string";
this.args = [
{
name: "Force Valid JSON",
type: "boolean",
value: true
}
];
}
/**
* @param {ArrayBuffer} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
if (input.byteLength <= 0) {
throw new OperationError("Please provide an input.");
}
const forceJSON = args[0];
return new Promise((resolve, reject) => {
const result = [];
const inpArray = new Uint8Array(input);
const decoder = new avro.streams.BlockDecoder();
decoder
.on("data", function (obj) {
result.push(obj);
})
.on("error", function () {
reject(new OperationError("Error parsing Avro file."));
})
.on("end", function () {
if (forceJSON) {
resolve(result.length === 1 ? JSON.stringify(result[0], null, 4) : JSON.stringify(result, null, 4));
} else {
const data = result.reduce((result, current) => result + JSON.stringify(current) + "\n", "");
resolve(data);
}
});
decoder.write(inpArray);
decoder.end();
});
}
}
export default AvroToJSON;

2
src/core/operations/BSONDeserialise.mjs
View file

@ -20,7 +20,7 @@ class BSONDeserialise extends Operation {
super();
this.name = "BSON deserialise";
this.module = "BSON";
this.module = "Serialise";
this.description = "BSON is a computer data interchange format used mainly as a data storage and network transfer format in the MongoDB database. It is a binary form for representing simple data structures, associative arrays (called objects or documents in MongoDB), and various data types of specific interest to MongoDB. The name 'BSON' is based on the term JSON and stands for 'Binary JSON'.<br><br>Input data should be in a raw bytes format.";
this.infoURL = "https://wikipedia.org/wiki/BSON";
this.inputType = "ArrayBuffer";

2
src/core/operations/BSONSerialise.mjs
View file

@ -20,7 +20,7 @@ class BSONSerialise extends Operation {
super();
this.name = "BSON serialise";
this.module = "BSON";
this.module = "Serialise";
this.description = "BSON is a computer data interchange format used mainly as a data storage and network transfer format in the MongoDB database. It is a binary form for representing simple data structures, associative arrays (called objects or documents in MongoDB), and various data types of specific interest to MongoDB. The name 'BSON' is based on the term JSON and stands for 'Binary JSON'.<br><br>Input data should be valid JSON.";
this.infoURL = "https://wikipedia.org/wiki/BSON";
this.inputType = "string";

8
src/core/operations/Diff.mjs
View file

@ -47,6 +47,11 @@ class Diff extends Operation {
"type": "boolean",
"value": true
},
{
"name": "Show subtraction",
"type": "boolean",
"value": false
},
{
"name": "Ignore whitespace",
"type": "boolean",
@ -67,6 +72,7 @@ class Diff extends Operation {
diffBy,
showAdded,
showRemoved,
showSubtraction,
ignoreWhitespace
] = args,
samples = input.split(sampleDelim);
@ -116,7 +122,7 @@ class Diff extends Operation {
if (showAdded) output += "<span class='hl5'>" + Utils.escapeHtml(diff[i].value) + "</span>";
} else if (diff[i].removed) {
if (showRemoved) output += "<span class='hl3'>" + Utils.escapeHtml(diff[i].value) + "</span>";
} else {
} else if (!showSubtraction) {
output += Utils.escapeHtml(diff[i].value);
}
}

53
src/web/HTMLIngredient.mjs
View file

@ -54,7 +54,9 @@ class HTMLIngredient {
case "binaryString":
case "byteArray":
html += `<div class="form-group">
<label for="${this.id}" class="bmd-label-floating">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating">${this.name}</label>
<input type="text"
class="form-control arg"
id="${this.id}"
@ -62,13 +64,14 @@ class HTMLIngredient {
arg-name="${this.name}"
value="${this.value}"
${this.disabled ? "disabled" : ""}>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
</div>`;
break;
case "shortString":
case "binaryShortString":
html += `<div class="form-group inline">
<label for="${this.id}" class="bmd-label-floating inline">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating inline">${this.name}</label>
<input type="text"
class="form-control arg inline"
id="${this.id}"
@ -76,13 +79,14 @@ class HTMLIngredient {
arg-name="${this.name}"
value="${this.value}"
${this.disabled ? "disabled" : ""}>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
</div>`;
break;
case "toggleString":
html += `<div class="form-group input-group">
<div class="toggle-string">
<label for="${this.id}" class="bmd-label-floating toggle-string">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating toggle-string">${this.name}</label>
<input type="text"
class="form-control arg toggle-string"
id="${this.id}"
@ -90,7 +94,6 @@ class HTMLIngredient {
arg-name="${this.name}"
value="${this.value}"
${this.disabled ? "disabled" : ""}>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
</div>
<div class="input-group-append">
<button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">${this.toggleValues[0]}</button>
@ -105,7 +108,9 @@ class HTMLIngredient {
break;
case "number":
html += `<div class="form-group inline">
<label for="${this.id}" class="bmd-label-floating inline">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating inline">${this.name}</label>
<input type="number"
class="form-control arg inline"
id="${this.id}"
@ -116,13 +121,12 @@ class HTMLIngredient {
max="${this.max}"
step="${this.step}"
${this.disabled ? "disabled" : ""}>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
</div>`;
break;
case "boolean":
html += `<div class="form-group inline boolean-arg">
<div class="checkbox">
<label>
<label ${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}>
<input type="checkbox"
class="arg"
id="${this.id}"
@ -132,13 +136,14 @@ class HTMLIngredient {
${this.disabled ? " disabled" : ""}
value="${this.name}"> ${this.name}
</label>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
</div>
</div>`;
break;
case "option":
html += `<div class="form-group inline">
<label for="${this.id}" class="bmd-label-floating inline">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating inline">${this.name}</label>
<select
class="form-control arg inline"
id="${this.id}"
@ -155,13 +160,14 @@ class HTMLIngredient {
}
}
html += `</select>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
</div>`;
break;
case "populateOption":
case "populateMultiOption":
html += `<div class="form-group">
<label for="${this.id}" class="bmd-label-floating">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating">${this.name}</label>
<select
class="form-control arg no-state-change populate-option"
id="${this.id}"
@ -181,7 +187,6 @@ class HTMLIngredient {
}
}
html += `</select>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
</div>`;
eventFn = this.type === "populateMultiOption" ?
@ -191,7 +196,9 @@ class HTMLIngredient {
break;
case "editableOption":
html += `<div class="form-group input-group">
<label for="${this.id}" class="bmd-label-floating">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating">${this.name}</label>
<input type="text"
class="form-control arg"
id="${this.id}"
@ -199,7 +206,6 @@ class HTMLIngredient {
arg-name="${this.name}"
value="${this.value[this.defaultIndex].value}"
${this.disabled ? "disabled" : ""}>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
<div class="input-group-append">
<button type="button"
class="btn btn-secondary dropdown-toggle dropdown-toggle-split"
@ -221,7 +227,9 @@ class HTMLIngredient {
break;
case "editableOptionShort":
html += `<div class="form-group input-group inline">
<label for="${this.id}" class="bmd-label-floating inline">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating inline">${this.name}</label>
<input type="text"
class="form-control arg inline"
id="${this.id}"
@ -229,7 +237,6 @@ class HTMLIngredient {
arg-name="${this.name}"
value="${this.value[this.defaultIndex].value}"
${this.disabled ? "disabled" : ""}>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
<div class="input-group-append inline">
<button type="button"
class="btn btn-secondary dropdown-toggle dropdown-toggle-split"
@ -251,7 +258,9 @@ class HTMLIngredient {
break;
case "text":
html += `<div class="form-group">
<label for="${this.id}" class="bmd-label-floating">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating">${this.name}</label>
<textarea
class="form-control arg"
id="${this.id}"
@ -259,12 +268,13 @@ class HTMLIngredient {
arg-name="${this.name}"
rows="${this.rows ? this.rows : 3}"
${this.disabled ? "disabled" : ""}>${this.value}</textarea>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
</div>`;
break;
case "argSelector":
html += `<div class="form-group inline">
<label for="${this.id}" class="bmd-label-floating inline">${this.name}</label>
<label for="${this.id}"
${this.hint ? `data-toggle="tooltip" title="${this.hint}"` : ""}
class="bmd-label-floating inline">${this.name}</label>
<select
class="form-control arg inline arg-selector"
id="${this.id}"
@ -279,7 +289,6 @@ class HTMLIngredient {
</option>`;
}
html += `</select>
${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
</div>`;
this.manager.addDynamicListener(".arg-selector", "change", this.argSelectorChange, this);

4
src/web/stylesheets/components/_operation.css
View file

@ -159,10 +159,6 @@ div.toggle-string {
width: calc(100% - 13px);
}
.operation .bmd-form-group .bmd-help {
margin-top: -17px;
}
.input-group .form-control {
border-top-left-radius: 4px !important;
}

2
src/web/waiters/RecipeWaiter.mjs
View file

@ -399,6 +399,8 @@ class RecipeWaiter {
this.buildRecipeOperation(item);
document.getElementById("rec-list").appendChild(item);
$(item).find("[data-toggle='tooltip']").tooltip();
item.dispatchEvent(this.manager.operationadd);
return item;
}

2
tests/operations/index.mjs
View file

@ -91,8 +91,10 @@ import "./tests/Protobuf.mjs";
import "./tests/ParseSSHHostKey.mjs";
import "./tests/DefangIP.mjs";
import "./tests/ParseUDP.mjs";
import "./tests/AvroToJSON.mjs";
import "./tests/Lorenz.mjs";
// Cannot test operations that use the File type yet
// import "./tests/SplitColourChannels.mjs";

66
tests/operations/tests/AvroToJSON.mjs Normal file
View file

@ -0,0 +1,66 @@
/**
* Avro to JSON tests.
*
* @author jarrodconnolly [jarrod@nestedquotes.ca]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../../lib/TestRegister";
TestRegister.addTests([
{
name: "Avro to JSON: no input (force JSON true)",
input: "",
expectedOutput: "Please provide an input.",
recipeConfig: [
{
op: "Avro to JSON",
args: [true]
}
],
},
{
name: "Avro to JSON: no input (force JSON false)",
input: "",
expectedOutput: "Please provide an input.",
recipeConfig: [
{
op: "Avro to JSON",
args: [false]
}
],
},
{
name: "Avro to JSON: small (force JSON true)",
input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
"\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" +
"\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" +
"\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" +
"\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" +
"\x15\x41",
expectedOutput: "{\n \"name\": \"myname\"\n}",
recipeConfig: [
{
op: "Avro to JSON",
args: [true]
}
],
},
{
name: "Avro to JSON: small (force JSON false)",
input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
"\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" +
"\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" +
"\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" +
"\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" +
"\x15\x41",
expectedOutput: "{\"name\":\"myname\"}\n",
recipeConfig: [
{
op: "Avro to JSON",
args: [false]
}
],
}
]);

24
tests/operations/tests/StrUtils.mjs
View file

@ -15,7 +15,29 @@ TestRegister.addTests([
recipeConfig: [
{
"op": "Diff",
"args": ["\\n\\n", "Character", true, true, false]
"args": ["\\n\\n", "Character", true, true, false, false]
}
],
},
{
name: "Diff added with subtraction, basic usage",
input: "testing23\n\ntesting123",
expectedOutput: "<span class='hl5'>1</span>",
recipeConfig: [
{
"op": "Diff",
"args": ["\\n\\n", "Character", true, true, true, false]
}
],
},
{
name: "Diff removed with subtraction, basic usage",
input: "testing123\n\ntesting3",
expectedOutput: "<span class='hl3'>12</span>",
recipeConfig: [
{
"op": "Diff",
"args": ["\\n\\n", "Character", true, true, true, false]
}
],
},