From 3412296a768d2e587088244d8212fc8ef33fb821 Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 10:37:11 +0200 Subject: [PATCH 1/5] feat: working decompression --- package-lock.json | 6 ++ package.json | 1 + src/core/operations/ZStandardDecompress.mjs | 90 +++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 src/core/operations/ZStandardDecompress.mjs diff --git a/package-lock.json b/package-lock.json index 3904f840..70cf3227 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "fernet": "^0.4.0", "file-saver": "^2.0.5", "flat": "^6.0.1", + "fzstd": "^0.1.1", "geodesy": "1.1.3", "highlight.js": "^11.9.0", "ieee754": "^1.2.1", @@ -7916,6 +7917,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/fzstd": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/fzstd/-/fzstd-0.1.1.tgz", + "integrity": "sha512-dkuVSOKKwh3eas5VkJy1AW1vFpet8TA/fGmVA5krThl8YcOVE/8ZIoEA1+U1vEn5ckxxhLirSdY837azmbaNHA==" + }, "node_modules/gamma": { "version": "1.0.0", "license": "MIT" diff --git a/package.json b/package.json index cc3517d3..e092ad49 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "fernet": "^0.4.0", "file-saver": "^2.0.5", "flat": "^6.0.1", + "fzstd": "^0.1.1", "geodesy": "1.1.3", "highlight.js": "^11.9.0", "ieee754": "^1.2.1", diff --git a/src/core/operations/ZStandardDecompress.mjs b/src/core/operations/ZStandardDecompress.mjs new file mode 100644 index 00000000..9d68a401 --- /dev/null +++ b/src/core/operations/ZStandardDecompress.mjs @@ -0,0 +1,90 @@ +/** + * @author Scarjit [ferdinand@linnenberg.dev] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ + +global.document = {}; +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import {isWorkerEnvironment} from "../Utils.mjs"; +import * as fzstd from "fzstd"; + +/** + * ZStandard Decompress operation + */ +class ZStandardDecompress extends Operation { + + /** + * ZStandardDecompress constructor + */ + constructor() { + super(); + + this.name = "ZStandard Decompress"; + this.module = "Compress"; + this.description = "ZStandard is a compression algorithm focused on fast decompression."; + this.infoURL = "https://wikipedia.org/wiki/Zstd"; // Usually a Wikipedia link. Remember to remove localisation (i.e. https://wikipedia.org/etc rather than https://en.wikipedia.org/etc) + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = [ + { + "name": "Chunk Size (bytes)", + "type": "number", + "value": 65536 + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {ArrayBuffer} + */ + run(input, args) { + const chunkSize = args[0]; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); + } + // Validate input starts with ZStandard magic number + const magicNumber = new Uint8Array(input, 0, 4); + if (magicNumber[0] !== 0x28 || magicNumber[1] !== 0xb5 || magicNumber[2] !== 0x2f || magicNumber[3] !== 0xfd) { + throw new OperationError("Invalid ZStandard input: does not start with magic number."); + } + + if (isWorkerEnvironment()) self.sendStatusMessage("Loading ZStandard..."); + return new Promise(async (resolve, reject) => { + const compressed = new Uint8Array(input); + try { + const outChunks = []; // Array of Uint8Array chunks + const stream = new fzstd.Decompress((chunk, isLast) => { + // Add to list of output chunks + outChunks.push(chunk); + // Log after all chunks decompressed + if (isLast) { + // Combine all chunks into a single Uint8Array + const totalLength = outChunks.reduce((sum, chunk) => sum + chunk.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of outChunks) { + result.set(chunk, offset); + offset += chunk.length; + } + resolve(result.buffer); + } + }); + for (let i = 0; i < compressed.length; i += chunkSize) { + if (isWorkerEnvironment()) self.sendStatusMessage(`Decompressing chunk ${i / chunkSize + 1} of ${Math.ceil(compressed.length / chunkSize)}`); + const chunk = compressed.subarray(i, i + chunkSize); + stream.push(chunk); + } + stream.push(new Uint8Array(0), true); // Signal end of stream + } catch (error) { + reject(new OperationError("Decompression failed: " + error.message)); + } + }); + } + +} + +export default ZStandardDecompress; From adca9f3e400ac3f7256c133737f7d6c7248807c0 Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 10:46:37 +0200 Subject: [PATCH 2/5] chore: cleanup --- src/core/operations/ZStandardDecompress.mjs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core/operations/ZStandardDecompress.mjs b/src/core/operations/ZStandardDecompress.mjs index 9d68a401..a6a53768 100644 --- a/src/core/operations/ZStandardDecompress.mjs +++ b/src/core/operations/ZStandardDecompress.mjs @@ -4,7 +4,6 @@ * @license Apache-2.0 */ -global.document = {}; import Operation from "../Operation.mjs"; import OperationError from "../errors/OperationError.mjs"; import {isWorkerEnvironment} from "../Utils.mjs"; @@ -47,20 +46,21 @@ class ZStandardDecompress extends Operation { throw new OperationError("Please provide an input."); } // Validate input starts with ZStandard magic number + const ZSTD_MAGIC_NUMBER = [0x28, 0xb5, 0x2f, 0xfd]; const magicNumber = new Uint8Array(input, 0, 4); - if (magicNumber[0] !== 0x28 || magicNumber[1] !== 0xb5 || magicNumber[2] !== 0x2f || magicNumber[3] !== 0xfd) { + if (!ZSTD_MAGIC_NUMBER.every((val, index) => val === magicNumber[index])) { throw new OperationError("Invalid ZStandard input: does not start with magic number."); } + if (isWorkerEnvironment()) self.sendStatusMessage("Loading ZStandard..."); - return new Promise(async (resolve, reject) => { + return new Promise( (resolve, reject) => { const compressed = new Uint8Array(input); try { const outChunks = []; // Array of Uint8Array chunks const stream = new fzstd.Decompress((chunk, isLast) => { - // Add to list of output chunks + // Add to the list of output chunks outChunks.push(chunk); - // Log after all chunks decompressed if (isLast) { // Combine all chunks into a single Uint8Array const totalLength = outChunks.reduce((sum, chunk) => sum + chunk.length, 0); @@ -73,8 +73,9 @@ class ZStandardDecompress extends Operation { resolve(result.buffer); } }); + const chunks = Math.ceil(compressed.length / chunkSize); for (let i = 0; i < compressed.length; i += chunkSize) { - if (isWorkerEnvironment()) self.sendStatusMessage(`Decompressing chunk ${i / chunkSize + 1} of ${Math.ceil(compressed.length / chunkSize)}`); + if (isWorkerEnvironment()) self.sendStatusMessage(`Decompressing chunk ${i / chunkSize + 1} of ${chunks}...`); const chunk = compressed.subarray(i, i + chunkSize); stream.push(chunk); } From b6944b7ab6c249c2fe760b285e35aded33a62700 Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 10:58:58 +0200 Subject: [PATCH 3/5] feat: added test --- src/core/config/Categories.json | 3 ++- tests/operations/tests/Compress.mjs | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index bebdd6a5..aa3095bb 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -381,7 +381,8 @@ "LZMA Compress", "LZ4 Decompress", "LZ4 Compress", - "LZNT1 Decompress" + "LZNT1 Decompress", + "ZStandard Decompress" ] }, { diff --git a/tests/operations/tests/Compress.mjs b/tests/operations/tests/Compress.mjs index 60117c67..88e17db4 100644 --- a/tests/operations/tests/Compress.mjs +++ b/tests/operations/tests/Compress.mjs @@ -105,4 +105,19 @@ TestRegister.addTests([ } ], }, + { + name: "ZStandard Decompress", + input: "KLUv/QRYwQAAVGhlIGNhdCBzYXQgb24gdGhlIG1hdC4KpvSd8w==", + expectedOutput: "The cat sat on the mat.", + recipeConfig: [ + { + "op": "From Base64", + "args": [] + }, + { + "op": "ZStandard Decompress", + "args": [] + } + ], + } ]); From e767a89b6113e975f253d6ab30c48e282ddb1ebe Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 11:20:37 +0200 Subject: [PATCH 4/5] fix: lint --- src/core/operations/ZStandardDecompress.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/ZStandardDecompress.mjs b/src/core/operations/ZStandardDecompress.mjs index a6a53768..3bdcca9b 100644 --- a/src/core/operations/ZStandardDecompress.mjs +++ b/src/core/operations/ZStandardDecompress.mjs @@ -54,7 +54,7 @@ class ZStandardDecompress extends Operation { if (isWorkerEnvironment()) self.sendStatusMessage("Loading ZStandard..."); - return new Promise( (resolve, reject) => { + return new Promise((resolve, reject) => { const compressed = new Uint8Array(input); try { const outChunks = []; // Array of Uint8Array chunks From 158641c92128200089a20bca5d6321e388ca2f94 Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 11:31:15 +0200 Subject: [PATCH 5/5] fix: removed newline from input and set default arg --- tests/operations/tests/Compress.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/operations/tests/Compress.mjs b/tests/operations/tests/Compress.mjs index 88e17db4..d1d451fd 100644 --- a/tests/operations/tests/Compress.mjs +++ b/tests/operations/tests/Compress.mjs @@ -107,7 +107,7 @@ TestRegister.addTests([ }, { name: "ZStandard Decompress", - input: "KLUv/QRYwQAAVGhlIGNhdCBzYXQgb24gdGhlIG1hdC4KpvSd8w==", + input: "KLUv/QRYuQAAVGhlIGNhdCBzYXQgb24gdGhlIG1hdC4tJ481", expectedOutput: "The cat sat on the mat.", recipeConfig: [ { @@ -116,7 +116,7 @@ TestRegister.addTests([ }, { "op": "ZStandard Decompress", - "args": [] + "args": [65536] } ], }