From cc8665078640cf7f0c36c1d5803a704f8695972f Mon Sep 17 00:00:00 2001 From: bee-san Date: Wed, 27 Dec 2023 12:41:09 +0000 Subject: [PATCH 1/3] Add support for LZNT1 decompression Introduces support for LZNT1 decompression, commonly leveraged by malware through RtlDecompressBuffer (closes #534). The decompression logic is ported from go-ntfs, the test data is similar to malduck's. from: https://github.com/gchq/CyberChef/pull/1675 --- src/core/config/Categories.json | 3 +- src/core/lib/LZNT1.mjs | 88 ++++++++++++++++++++++ src/core/operations/LZNT1Decompress.mjs | 41 ++++++++++ tests/node/tests/operations.mjs | 4 + tests/operations/index.mjs | 1 + tests/operations/tests/LZNT1Decompress.mjs | 22 ++++++ 6 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 src/core/lib/LZNT1.mjs create mode 100644 src/core/operations/LZNT1Decompress.mjs create mode 100644 tests/operations/tests/LZNT1Decompress.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 3d9f12e2..c214c781 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -378,7 +378,8 @@ "LZMA Decompress", "LZMA Compress", "LZ4 Decompress", - "LZ4 Compress" + "LZ4 Compress", + "LZNT1 Decompress" ] }, { diff --git a/src/core/lib/LZNT1.mjs b/src/core/lib/LZNT1.mjs new file mode 100644 index 00000000..85f8b34f --- /dev/null +++ b/src/core/lib/LZNT1.mjs @@ -0,0 +1,88 @@ +/** + * + * LZNT1 Decompress. + * + * @author 0xThiebaut [thiebaut.dev] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + * + * https://github.com/Velocidex/go-ntfs/blob/master/parser%2Flznt1.go + */ + +import Utils from "../Utils.mjs"; +import OperationError from "../errors/OperationError.mjs"; + +const COMPRESSED_MASK = 1 << 15, + SIZE_MASK = (1 << 12) - 1; + +/** + * @param {number} offset + * @returns {number} + */ +function getDisplacement(offset) { + let result = 0; + while (offset >= 0x10) { + offset >>= 1; + result += 1; + } + return result; +} + +/** + * @param {byteArray} compressed + * @returns {byteArray} + */ +export function decompress(compressed) { + const decompressed = Array(); + let coffset = 0; + + while (coffset + 2 <= compressed.length) { + const doffset = decompressed.length; + + const blockHeader = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little"); + coffset += 2; + + const size = blockHeader & SIZE_MASK; + const blockEnd = coffset + size + 1; + + if (size === 0) { + break; + } else if (compressed.length < coffset + size) { + throw new OperationError("Malformed LZNT1 stream: Block too small! Has the stream been truncated?"); + } + + if ((blockHeader & COMPRESSED_MASK) !== 0) { + while (coffset < blockEnd) { + let header = compressed[coffset++]; + + for (let i = 0; i < 8 && coffset < blockEnd; i++) { + if ((header & 1) === 0) { + decompressed.push(compressed[coffset++]); + } else { + const pointer = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little"); + coffset += 2; + + const displacement = getDisplacement(decompressed.length - doffset - 1); + const symbolOffset = (pointer >> (12 - displacement)) + 1; + const symbolLength = (pointer & (0xFFF >> displacement)) + 2; + const shiftOffset = decompressed.length - symbolOffset; + + for (let shiftDelta = 0; shiftDelta < symbolLength + 1; shiftDelta++) { + const shift = shiftOffset + shiftDelta; + if (shift < 0 || decompressed.length <= shift) { + throw new OperationError("Malformed LZNT1 stream: Invalid shift!"); + } + decompressed.push(decompressed[shift]); + } + } + header >>= 1; + } + } + } else { + decompressed.push(...compressed.slice(coffset, coffset + size + 1)); + coffset += size + 1; + } + } + + return decompressed; +} diff --git a/src/core/operations/LZNT1Decompress.mjs b/src/core/operations/LZNT1Decompress.mjs new file mode 100644 index 00000000..3c793c3f --- /dev/null +++ b/src/core/operations/LZNT1Decompress.mjs @@ -0,0 +1,41 @@ +/** + * @author 0xThiebaut [thiebaut.dev] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import {decompress} from "../lib/LZNT1.mjs"; + +/** + * LZNT1 Decompress operation + */ +class LZNT1Decompress extends Operation { + + /** + * LZNT1 Decompress constructor + */ + constructor() { + super(); + + this.name = "LZNT1 Decompress"; + this.module = "Compression"; + this.description = "Decompresses data using the LZNT1 algorithm.

Similar to the Windows API RtlDecompressBuffer."; + this.infoURL = "https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15"; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.args = []; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + return decompress(input); + } + +} + +export default LZNT1Decompress; diff --git a/tests/node/tests/operations.mjs b/tests/node/tests/operations.mjs index 8611ecb4..86dbee50 100644 --- a/tests/node/tests/operations.mjs +++ b/tests/node/tests/operations.mjs @@ -635,6 +635,10 @@ WWFkYSBZYWRh\r assert.strictEqual(chef.keccak("Flea Market").toString(), "c2a06880b19e453ee5440e8bd4c2024bedc15a6630096aa3f609acfd2b8f15f27cd293e1cc73933e81432269129ce954a6138889ce87831179d55dcff1cc7587"); }), + it("LZNT1 Decompress", () => { + assert.strictEqual(chef.LZNT1Decompress("\x1a\xb0\x00compress\x00edtestda\x04ta\x07\x88alot").toString(), "compressedtestdatacompressedalot"); + }), + it("MD6", () => { assert.strictEqual(chef.MD6("Head Over Heels", {key: "arty"}).toString(), "d8f7fe4931fbaa37316f76283d5f615f50ddd54afdc794b61da522556aee99ad"); }), diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 3cf698de..90fa5a24 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -64,6 +64,7 @@ import "./tests/JSONtoCSV.mjs"; import "./tests/JWTDecode.mjs"; import "./tests/JWTSign.mjs"; import "./tests/JWTVerify.mjs"; +import "./tests/LZNT1Decompress.mjs"; import "./tests/MS.mjs"; import "./tests/Magic.mjs"; import "./tests/Modhex.mjs"; diff --git a/tests/operations/tests/LZNT1Decompress.mjs b/tests/operations/tests/LZNT1Decompress.mjs new file mode 100644 index 00000000..ba0f855b --- /dev/null +++ b/tests/operations/tests/LZNT1Decompress.mjs @@ -0,0 +1,22 @@ +/** + * LZNT1 Decompress tests. + * + * @author 0xThiebaut [thiebaut.dev] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "LZNT1 Decompress", + input: "\x1a\xb0\x00compress\x00edtestda\x04ta\x07\x88alot", + expectedOutput: "compressedtestdatacompressedalot", + recipeConfig: [ + { + op: "LZNT1 Decompress", + args: [] + } + ], + } +]); \ No newline at end of file From 963c58ed2488046fba729aef683d7933b98f66d3 Mon Sep 17 00:00:00 2001 From: bee-san Date: Wed, 27 Dec 2023 12:44:38 +0000 Subject: [PATCH 2/3] updated linebreaks --- src/core/lib/LZNT1.mjs | 176 ++++++++++----------- src/core/operations/LZNT1Decompress.mjs | 82 +++++----- tests/operations/tests/LZNT1Decompress.mjs | 42 ++--- 3 files changed, 150 insertions(+), 150 deletions(-) diff --git a/src/core/lib/LZNT1.mjs b/src/core/lib/LZNT1.mjs index 85f8b34f..9a1c7fab 100644 --- a/src/core/lib/LZNT1.mjs +++ b/src/core/lib/LZNT1.mjs @@ -1,88 +1,88 @@ -/** - * - * LZNT1 Decompress. - * - * @author 0xThiebaut [thiebaut.dev] - * @copyright Crown Copyright 2023 - * @license Apache-2.0 - * - * https://github.com/Velocidex/go-ntfs/blob/master/parser%2Flznt1.go - */ - -import Utils from "../Utils.mjs"; -import OperationError from "../errors/OperationError.mjs"; - -const COMPRESSED_MASK = 1 << 15, - SIZE_MASK = (1 << 12) - 1; - -/** - * @param {number} offset - * @returns {number} - */ -function getDisplacement(offset) { - let result = 0; - while (offset >= 0x10) { - offset >>= 1; - result += 1; - } - return result; -} - -/** - * @param {byteArray} compressed - * @returns {byteArray} - */ -export function decompress(compressed) { - const decompressed = Array(); - let coffset = 0; - - while (coffset + 2 <= compressed.length) { - const doffset = decompressed.length; - - const blockHeader = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little"); - coffset += 2; - - const size = blockHeader & SIZE_MASK; - const blockEnd = coffset + size + 1; - - if (size === 0) { - break; - } else if (compressed.length < coffset + size) { - throw new OperationError("Malformed LZNT1 stream: Block too small! Has the stream been truncated?"); - } - - if ((blockHeader & COMPRESSED_MASK) !== 0) { - while (coffset < blockEnd) { - let header = compressed[coffset++]; - - for (let i = 0; i < 8 && coffset < blockEnd; i++) { - if ((header & 1) === 0) { - decompressed.push(compressed[coffset++]); - } else { - const pointer = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little"); - coffset += 2; - - const displacement = getDisplacement(decompressed.length - doffset - 1); - const symbolOffset = (pointer >> (12 - displacement)) + 1; - const symbolLength = (pointer & (0xFFF >> displacement)) + 2; - const shiftOffset = decompressed.length - symbolOffset; - - for (let shiftDelta = 0; shiftDelta < symbolLength + 1; shiftDelta++) { - const shift = shiftOffset + shiftDelta; - if (shift < 0 || decompressed.length <= shift) { - throw new OperationError("Malformed LZNT1 stream: Invalid shift!"); - } - decompressed.push(decompressed[shift]); - } - } - header >>= 1; - } - } - } else { - decompressed.push(...compressed.slice(coffset, coffset + size + 1)); - coffset += size + 1; - } - } - - return decompressed; -} +/** + * + * LZNT1 Decompress. + * + * @author 0xThiebaut [thiebaut.dev] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + * + * https://github.com/Velocidex/go-ntfs/blob/master/parser%2Flznt1.go + */ + +import Utils from "../Utils.mjs"; +import OperationError from "../errors/OperationError.mjs"; + +const COMPRESSED_MASK = 1 << 15, + SIZE_MASK = (1 << 12) - 1; + +/** + * @param {number} offset + * @returns {number} + */ +function getDisplacement(offset) { + let result = 0; + while (offset >= 0x10) { + offset >>= 1; + result += 1; + } + return result; +} + +/** + * @param {byteArray} compressed + * @returns {byteArray} + */ +export function decompress(compressed) { + const decompressed = Array(); + let coffset = 0; + + while (coffset + 2 <= compressed.length) { + const doffset = decompressed.length; + + const blockHeader = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little"); + coffset += 2; + + const size = blockHeader & SIZE_MASK; + const blockEnd = coffset + size + 1; + + if (size === 0) { + break; + } else if (compressed.length < coffset + size) { + throw new OperationError("Malformed LZNT1 stream: Block too small! Has the stream been truncated?"); + } + + if ((blockHeader & COMPRESSED_MASK) !== 0) { + while (coffset < blockEnd) { + let header = compressed[coffset++]; + + for (let i = 0; i < 8 && coffset < blockEnd; i++) { + if ((header & 1) === 0) { + decompressed.push(compressed[coffset++]); + } else { + const pointer = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little"); + coffset += 2; + + const displacement = getDisplacement(decompressed.length - doffset - 1); + const symbolOffset = (pointer >> (12 - displacement)) + 1; + const symbolLength = (pointer & (0xFFF >> displacement)) + 2; + const shiftOffset = decompressed.length - symbolOffset; + + for (let shiftDelta = 0; shiftDelta < symbolLength + 1; shiftDelta++) { + const shift = shiftOffset + shiftDelta; + if (shift < 0 || decompressed.length <= shift) { + throw new OperationError("Malformed LZNT1 stream: Invalid shift!"); + } + decompressed.push(decompressed[shift]); + } + } + header >>= 1; + } + } + } else { + decompressed.push(...compressed.slice(coffset, coffset + size + 1)); + coffset += size + 1; + } + } + + return decompressed; +} diff --git a/src/core/operations/LZNT1Decompress.mjs b/src/core/operations/LZNT1Decompress.mjs index 3c793c3f..b5308e77 100644 --- a/src/core/operations/LZNT1Decompress.mjs +++ b/src/core/operations/LZNT1Decompress.mjs @@ -1,41 +1,41 @@ -/** - * @author 0xThiebaut [thiebaut.dev] - * @copyright Crown Copyright 2023 - * @license Apache-2.0 - */ - -import Operation from "../Operation.mjs"; -import {decompress} from "../lib/LZNT1.mjs"; - -/** - * LZNT1 Decompress operation - */ -class LZNT1Decompress extends Operation { - - /** - * LZNT1 Decompress constructor - */ - constructor() { - super(); - - this.name = "LZNT1 Decompress"; - this.module = "Compression"; - this.description = "Decompresses data using the LZNT1 algorithm.

Similar to the Windows API RtlDecompressBuffer."; - this.infoURL = "https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15"; - this.inputType = "byteArray"; - this.outputType = "byteArray"; - this.args = []; - } - - /** - * @param {byteArray} input - * @param {Object[]} args - * @returns {byteArray} - */ - run(input, args) { - return decompress(input); - } - -} - -export default LZNT1Decompress; +/** + * @author 0xThiebaut [thiebaut.dev] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import {decompress} from "../lib/LZNT1.mjs"; + +/** + * LZNT1 Decompress operation + */ +class LZNT1Decompress extends Operation { + + /** + * LZNT1 Decompress constructor + */ + constructor() { + super(); + + this.name = "LZNT1 Decompress"; + this.module = "Compression"; + this.description = "Decompresses data using the LZNT1 algorithm.

Similar to the Windows API RtlDecompressBuffer."; + this.infoURL = "https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15"; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.args = []; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + return decompress(input); + } + +} + +export default LZNT1Decompress; diff --git a/tests/operations/tests/LZNT1Decompress.mjs b/tests/operations/tests/LZNT1Decompress.mjs index ba0f855b..cda7f675 100644 --- a/tests/operations/tests/LZNT1Decompress.mjs +++ b/tests/operations/tests/LZNT1Decompress.mjs @@ -1,22 +1,22 @@ -/** - * LZNT1 Decompress tests. - * - * @author 0xThiebaut [thiebaut.dev] - * @copyright Crown Copyright 2023 - * @license Apache-2.0 - */ -import TestRegister from "../../lib/TestRegister.mjs"; - -TestRegister.addTests([ - { - name: "LZNT1 Decompress", - input: "\x1a\xb0\x00compress\x00edtestda\x04ta\x07\x88alot", - expectedOutput: "compressedtestdatacompressedalot", - recipeConfig: [ - { - op: "LZNT1 Decompress", - args: [] - } - ], - } +/** + * LZNT1 Decompress tests. + * + * @author 0xThiebaut [thiebaut.dev] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "LZNT1 Decompress", + input: "\x1a\xb0\x00compress\x00edtestda\x04ta\x07\x88alot", + expectedOutput: "compressedtestdatacompressedalot", + recipeConfig: [ + { + op: "LZNT1 Decompress", + args: [] + } + ], + } ]); \ No newline at end of file From 84d019103b8da15ed28ebefc5522ed3f8bcd5843 Mon Sep 17 00:00:00 2001 From: bee-san Date: Wed, 27 Dec 2023 12:46:38 +0000 Subject: [PATCH 3/3] new line at end of file --- tests/operations/tests/LZNT1Decompress.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/operations/tests/LZNT1Decompress.mjs b/tests/operations/tests/LZNT1Decompress.mjs index cda7f675..dcfad01a 100644 --- a/tests/operations/tests/LZNT1Decompress.mjs +++ b/tests/operations/tests/LZNT1Decompress.mjs @@ -19,4 +19,4 @@ TestRegister.addTests([ } ], } -]); \ No newline at end of file +]);