From 362755b22ff993aff60cd508690f3dd8849641be Mon Sep 17 00:00:00 2001 From: BlacAmDK Date: Tue, 5 Dec 2023 14:40:37 +0800 Subject: [PATCH 1/6] Fix ExtractIPAddresses IPv6 regexp IPv6 regexp shouldn't match IPv4 address. --- src/core/operations/ExtractIPAddresses.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/ExtractIPAddresses.mjs b/src/core/operations/ExtractIPAddresses.mjs index 95e0a50f..97b52478 100644 --- a/src/core/operations/ExtractIPAddresses.mjs +++ b/src/core/operations/ExtractIPAddresses.mjs @@ -66,7 +66,7 @@ class ExtractIPAddresses extends Operation { run(input, args) { const [includeIpv4, includeIpv6, removeLocal, displayTotal, sort, unique] = args, ipv4 = "(?:(?:\\d|[01]?\\d\\d|2[0-4]\\d|25[0-5])\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d|\\d)(?:\\/\\d{1,2})?", - ipv6 = "((?=.*::)(?!.*::.+::)(::)?([\\dA-F]{1,4}:(:|\\b)|){5}|([\\dA-F]{1,4}:){6})((([\\dA-F]{1,4}((?!\\3)::|:\\b|(?![\\dA-F])))|(?!\\2\\3)){2}|(((2[0-4]|1\\d|[1-9])?\\d|25[0-5])\\.?\\b){4})"; + ipv6 = "((?=.*::)(?!.*::.+::)(::)?([\\dA-F]{1,4}:(:|\\b)|){5}|([\\dA-F]{1,4}:){6})(([\\dA-F]{1,4}((?!\\3)::|:\\b|(?![\\dA-F])))|(?!\\2\\3)){2}"; let ips = ""; if (includeIpv4 && includeIpv6) { From 77042abc2348fcb19e4f25d89d356f4d2697382e Mon Sep 17 00:00:00 2001 From: Maxime THIEBAUT <46688461+0xThiebaut@users.noreply.github.com> Date: Mon, 25 Dec 2023 01:17:16 +0100 Subject: [PATCH 2/6] Add support for LZNT1 decompression. --- .gitignore | 1 + src/core/config/Categories.json | 3 +- src/core/lib/LZNT1.mjs | 88 ++++++++++++++++++++++ src/core/operations/LZNT1Decompress.mjs | 41 ++++++++++ src/web/html/index.html | 3 +- tests/node/tests/operations.mjs | 4 + tests/operations/index.mjs | 1 + tests/operations/tests/LZNT1Decompress.mjs | 22 ++++++ 8 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 src/core/lib/LZNT1.mjs create mode 100644 src/core/operations/LZNT1Decompress.mjs create mode 100644 tests/operations/tests/LZNT1Decompress.mjs diff --git a/.gitignore b/.gitignore index 3b7449c4..42923f5d 100755 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ npm-debug.log travis.log build .vscode +.idea .*.swp src/core/config/modules/* src/core/config/OperationConfig.json diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index cf4d91be..c8f83f95 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -351,7 +351,8 @@ "LZMA Decompress", "LZMA Compress", "LZ4 Decompress", - "LZ4 Compress" + "LZ4 Compress", + "LZNT1 Decompress" ] }, { diff --git a/src/core/lib/LZNT1.mjs b/src/core/lib/LZNT1.mjs new file mode 100644 index 00000000..9a1c7fab --- /dev/null +++ b/src/core/lib/LZNT1.mjs @@ -0,0 +1,88 @@ +/** + * + * LZNT1 Decompress. + * + * @author 0xThiebaut [thiebaut.dev] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + * + * https://github.com/Velocidex/go-ntfs/blob/master/parser%2Flznt1.go + */ + +import Utils from "../Utils.mjs"; +import OperationError from "../errors/OperationError.mjs"; + +const COMPRESSED_MASK = 1 << 15, + SIZE_MASK = (1 << 12) - 1; + +/** + * @param {number} offset + * @returns {number} + */ +function getDisplacement(offset) { + let result = 0; + while (offset >= 0x10) { + offset >>= 1; + result += 1; + } + return result; +} + +/** + * @param {byteArray} compressed + * @returns {byteArray} + */ +export function decompress(compressed) { + const decompressed = Array(); + let coffset = 0; + + while (coffset + 2 <= compressed.length) { + const doffset = decompressed.length; + + const blockHeader = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little"); + coffset += 2; + + const size = blockHeader & SIZE_MASK; + const blockEnd = coffset + size + 1; + + if (size === 0) { + break; + } else if (compressed.length < coffset + size) { + throw new OperationError("Malformed LZNT1 stream: Block too small! Has the stream been truncated?"); + } + + if ((blockHeader & COMPRESSED_MASK) !== 0) { + while (coffset < blockEnd) { + let header = compressed[coffset++]; + + for (let i = 0; i < 8 && coffset < blockEnd; i++) { + if ((header & 1) === 0) { + decompressed.push(compressed[coffset++]); + } else { + const pointer = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little"); + coffset += 2; + + const displacement = getDisplacement(decompressed.length - doffset - 1); + const symbolOffset = (pointer >> (12 - displacement)) + 1; + const symbolLength = (pointer & (0xFFF >> displacement)) + 2; + const shiftOffset = decompressed.length - symbolOffset; + + for (let shiftDelta = 0; shiftDelta < symbolLength + 1; shiftDelta++) { + const shift = shiftOffset + shiftDelta; + if (shift < 0 || decompressed.length <= shift) { + throw new OperationError("Malformed LZNT1 stream: Invalid shift!"); + } + decompressed.push(decompressed[shift]); + } + } + header >>= 1; + } + } + } else { + decompressed.push(...compressed.slice(coffset, coffset + size + 1)); + coffset += size + 1; + } + } + + return decompressed; +} diff --git a/src/core/operations/LZNT1Decompress.mjs b/src/core/operations/LZNT1Decompress.mjs new file mode 100644 index 00000000..b5308e77 --- /dev/null +++ b/src/core/operations/LZNT1Decompress.mjs @@ -0,0 +1,41 @@ +/** + * @author 0xThiebaut [thiebaut.dev] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import {decompress} from "../lib/LZNT1.mjs"; + +/** + * LZNT1 Decompress operation + */ +class LZNT1Decompress extends Operation { + + /** + * LZNT1 Decompress constructor + */ + constructor() { + super(); + + this.name = "LZNT1 Decompress"; + this.module = "Compression"; + this.description = "Decompresses data using the LZNT1 algorithm.

Similar to the Windows API RtlDecompressBuffer."; + this.infoURL = "https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15"; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.args = []; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + return decompress(input); + } + +} + +export default LZNT1Decompress; diff --git a/src/web/html/index.html b/src/web/html/index.html index c602c275..5c3c3263 100755 --- a/src/web/html/index.html +++ b/src/web/html/index.html @@ -62,7 +62,8 @@ "Training branch predictor...", "Timing cache hits...", "Speculatively executing recipes...", - "Adding LLM hallucinations..." + "Adding LLM hallucinations...", + "Decompressing malware..." ]; // Shuffle array using Durstenfeld algorithm diff --git a/tests/node/tests/operations.mjs b/tests/node/tests/operations.mjs index 8611ecb4..86dbee50 100644 --- a/tests/node/tests/operations.mjs +++ b/tests/node/tests/operations.mjs @@ -635,6 +635,10 @@ WWFkYSBZYWRh\r assert.strictEqual(chef.keccak("Flea Market").toString(), "c2a06880b19e453ee5440e8bd4c2024bedc15a6630096aa3f609acfd2b8f15f27cd293e1cc73933e81432269129ce954a6138889ce87831179d55dcff1cc7587"); }), + it("LZNT1 Decompress", () => { + assert.strictEqual(chef.LZNT1Decompress("\x1a\xb0\x00compress\x00edtestda\x04ta\x07\x88alot").toString(), "compressedtestdatacompressedalot"); + }), + it("MD6", () => { assert.strictEqual(chef.MD6("Head Over Heels", {key: "arty"}).toString(), "d8f7fe4931fbaa37316f76283d5f615f50ddd54afdc794b61da522556aee99ad"); }), diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 570fbb6f..e61f542d 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -62,6 +62,7 @@ import "./tests/JSONtoCSV.mjs"; import "./tests/JWTDecode.mjs"; import "./tests/JWTSign.mjs"; import "./tests/JWTVerify.mjs"; +import "./tests/LZNT1Decompress.mjs"; import "./tests/MS.mjs"; import "./tests/Magic.mjs"; import "./tests/MorseCode.mjs"; diff --git a/tests/operations/tests/LZNT1Decompress.mjs b/tests/operations/tests/LZNT1Decompress.mjs new file mode 100644 index 00000000..dcfad01a --- /dev/null +++ b/tests/operations/tests/LZNT1Decompress.mjs @@ -0,0 +1,22 @@ +/** + * LZNT1 Decompress tests. + * + * @author 0xThiebaut [thiebaut.dev] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "LZNT1 Decompress", + input: "\x1a\xb0\x00compress\x00edtestda\x04ta\x07\x88alot", + expectedOutput: "compressedtestdatacompressedalot", + recipeConfig: [ + { + op: "LZNT1 Decompress", + args: [] + } + ], + } +]); From 0bf7852e83fa998e7e336f1c0d3ba6cae9a73ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Notin?= Date: Tue, 26 Dec 2023 15:59:32 +0100 Subject: [PATCH 3/6] Add UUID regex to 'Regular expression' operation I use this one often so I'm sure others will like it too :) --- src/core/operations/RegularExpression.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/operations/RegularExpression.mjs b/src/core/operations/RegularExpression.mjs index 1d8de9c4..18d3fda9 100644 --- a/src/core/operations/RegularExpression.mjs +++ b/src/core/operations/RegularExpression.mjs @@ -83,6 +83,10 @@ class RegularExpression extends Operation { name: "Strings", value: "[A-Za-z\\d/\\-:.,_$%\\x27\"()<>= !\\[\\]{}@]{4,}" }, + { + name: "UUID (any version)", + value: "[0-9a-fA-F]{8}\\b-[0-9a-fA-F]{4}\\b-[0-9a-fA-F]{4}\\b-[0-9a-fA-F]{4}\\b-[0-9a-fA-F]{12}" + }, ], "target": 1 }, From b78533bb02399a9d46ca43295ac39749d14287ed Mon Sep 17 00:00:00 2001 From: a3957273 <89583054+a3957273@users.noreply.github.com> Date: Sat, 3 Feb 2024 02:02:13 +0000 Subject: [PATCH 4/6] Update forensics wiki address --- src/core/Utils.mjs | 17 +++++++++++++++++ src/core/operations/CTPH.mjs | 2 +- src/core/operations/CompareCTPHHashes.mjs | 2 +- src/core/operations/CompareSSDEEPHashes.mjs | 2 +- src/core/operations/SSDEEP.mjs | 2 +- src/web/HTMLOperation.mjs | 4 ++-- 6 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index 787ef270..18b0e688 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -892,6 +892,23 @@ class Utils { } + /** + * Converts a string to it's title case equivalent. + * + * @param {string} str + * @returns string + * + * @example + * // return "A Tiny String" + * Utils.toTitleCase("a tIny String"); + */ + static toTitleCase(str) { + return str.replace(/\w\S*/g, function(txt) { + return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase(); + }); + } + + /** * Encodes a URI fragment (#) or query (?) using a minimal amount of percent-encoding. * diff --git a/src/core/operations/CTPH.mjs b/src/core/operations/CTPH.mjs index 7394faaa..6b6a487d 100644 --- a/src/core/operations/CTPH.mjs +++ b/src/core/operations/CTPH.mjs @@ -21,7 +21,7 @@ class CTPH extends Operation { this.name = "CTPH"; this.module = "Crypto"; this.description = "Context Triggered Piecewise Hashing, also called Fuzzy Hashing, can match inputs that have homologies. Such inputs have sequences of identical bytes in the same order, although bytes in between these sequences may be different in both content and length.

CTPH was originally based on the work of Dr. Andrew Tridgell and a spam email detector called SpamSum. This method was adapted by Jesse Kornblum and published at the DFRWS conference in 2006 in a paper 'Identifying Almost Identical Files Using Context Triggered Piecewise Hashing'."; - this.infoURL = "https://forensicswiki.xyz/wiki/index.php?title=Context_Triggered_Piecewise_Hashing"; + this.infoURL = "https://forensics.wiki/context_triggered_piecewise_hashing/"; this.inputType = "string"; this.outputType = "string"; this.args = []; diff --git a/src/core/operations/CompareCTPHHashes.mjs b/src/core/operations/CompareCTPHHashes.mjs index 40fec472..91956220 100644 --- a/src/core/operations/CompareCTPHHashes.mjs +++ b/src/core/operations/CompareCTPHHashes.mjs @@ -24,7 +24,7 @@ class CompareCTPHHashes extends Operation { this.name = "Compare CTPH hashes"; this.module = "Crypto"; this.description = "Compares two Context Triggered Piecewise Hashing (CTPH) fuzzy hashes to determine the similarity between them on a scale of 0 to 100."; - this.infoURL = "https://forensicswiki.xyz/wiki/index.php?title=Context_Triggered_Piecewise_Hashing"; + this.infoURL = "https://forensics.wiki/context_triggered_piecewise_hashing/"; this.inputType = "string"; this.outputType = "Number"; this.args = [ diff --git a/src/core/operations/CompareSSDEEPHashes.mjs b/src/core/operations/CompareSSDEEPHashes.mjs index 806b048e..9937d7e6 100644 --- a/src/core/operations/CompareSSDEEPHashes.mjs +++ b/src/core/operations/CompareSSDEEPHashes.mjs @@ -24,7 +24,7 @@ class CompareSSDEEPHashes extends Operation { this.name = "Compare SSDEEP hashes"; this.module = "Crypto"; this.description = "Compares two SSDEEP fuzzy hashes to determine the similarity between them on a scale of 0 to 100."; - this.infoURL = "https://forensicswiki.xyz/wiki/index.php?title=Ssdeep"; + this.infoURL = "https://forensics.wiki/ssdeep/"; this.inputType = "string"; this.outputType = "Number"; this.args = [ diff --git a/src/core/operations/SSDEEP.mjs b/src/core/operations/SSDEEP.mjs index df1557d5..6a76a68b 100644 --- a/src/core/operations/SSDEEP.mjs +++ b/src/core/operations/SSDEEP.mjs @@ -21,7 +21,7 @@ class SSDEEP extends Operation { this.name = "SSDEEP"; this.module = "Crypto"; this.description = "SSDEEP is a program for computing context triggered piecewise hashes (CTPH). Also called fuzzy hashes, CTPH can match inputs that have homologies. Such inputs have sequences of identical bytes in the same order, although bytes in between these sequences may be different in both content and length.

SSDEEP hashes are now widely used for simple identification purposes (e.g. the 'Basic Properties' section in VirusTotal). Although 'better' fuzzy hashes are available, SSDEEP is still one of the primary choices because of its speed and being a de facto standard.

This operation is fundamentally the same as the CTPH operation, however their outputs differ in format."; - this.infoURL = "https://forensicswiki.xyz/wiki/index.php?title=Ssdeep"; + this.infoURL = "https://forensics.wiki/ssdeep"; this.inputType = "string"; this.outputType = "string"; this.args = []; diff --git a/src/web/HTMLOperation.mjs b/src/web/HTMLOperation.mjs index d0523aa4..ae61b58d 100755 --- a/src/web/HTMLOperation.mjs +++ b/src/web/HTMLOperation.mjs @@ -157,9 +157,9 @@ function titleFromWikiLink(urlStr) { pageTitle = ""; switch (urlObj.host) { - case "forensicswiki.xyz": + case "forensics.wiki": wikiName = "Forensics Wiki"; - pageTitle = urlObj.query.substr(6).replace(/_/g, " "); // Chop off 'title=' + pageTitle = Utils.toTitleCase(urlObj.path.replace(/\//g, "").replace(/_/g, " ")); break; case "wikipedia.org": wikiName = "Wikipedia"; From 22a873c73e849f64a404ee12fed28acf0f582ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Notin?= Date: Tue, 28 Nov 2023 16:34:51 +0100 Subject: [PATCH 5/6] Describe that "Parse ASN.1 hex string" operation requires an hex string input Just in case the title of the operation doesn't make it clear enough --- src/core/operations/ParseASN1HexString.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/ParseASN1HexString.mjs b/src/core/operations/ParseASN1HexString.mjs index 14890186..35fd5ba4 100644 --- a/src/core/operations/ParseASN1HexString.mjs +++ b/src/core/operations/ParseASN1HexString.mjs @@ -20,7 +20,7 @@ class ParseASN1HexString extends Operation { this.name = "Parse ASN.1 hex string"; this.module = "PublicKey"; - this.description = "Abstract Syntax Notation One (ASN.1) is a standard and notation that describes rules and structures for representing, encoding, transmitting, and decoding data in telecommunications and computer networking.

This operation parses arbitrary ASN.1 data and presents the resulting tree."; + this.description = "Abstract Syntax Notation One (ASN.1) is a standard and notation that describes rules and structures for representing, encoding, transmitting, and decoding data in telecommunications and computer networking.

This operation parses arbitrary ASN.1 data (encoded as an hex string: use the 'To Hex' operation if necessary) and presents the resulting tree."; this.infoURL = "https://wikipedia.org/wiki/Abstract_Syntax_Notation_One"; this.inputType = "string"; this.outputType = "string"; From df151eabf91cdb31292cfa336844bfdc24afba19 Mon Sep 17 00:00:00 2001 From: a3957273 <89583054+a3957273@users.noreply.github.com> Date: Sat, 3 Feb 2024 14:19:50 +0000 Subject: [PATCH 6/6] 10.6.0 --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93944764..c8854d89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ All major and minor version changes will be documented in this file. Details of ## Details +### [10.6.0] - 2024-02-03 +- Updated 'Forensics Wiki' URLs to new domain [@a3957273] | [#1703] +- Added 'LZNT1 Decompress' operation [@0xThiebaut] | [#1675] +- Updated 'Regex Expression' UUID matcher [@cnotin] | [#1678] +- Removed duplicate 'hover' message within baking info [#KevinSJ] | [#1541] + ### [10.5.0] - 2023-07-14 - Added GOST Encrypt, Decrypt, Sign, Verify, Key Wrap, and Key Unwrap operations [@n1474335] | [#592] diff --git a/package-lock.json b/package-lock.json index 8b484b79..ae8e5889 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cyberchef", - "version": "10.5.2", + "version": "10.6.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cyberchef", - "version": "10.5.2", + "version": "10.6.0", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index 5259e1bb..55b65312 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "10.5.2", + "version": "10.6.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef",