mirror of
https://github.com/gchq/CyberChef.git
synced 2025-04-20 14:56:19 -04:00
Add new worker for zipping outputs.
Use bakeId to track which outputs are stale.
This commit is contained in:
parent
df20196201
commit
07021b8dd5
5 changed files with 222 additions and 14 deletions
|
@ -93,7 +93,6 @@ self.addEventListener("message", function(e) {
|
||||||
async function bake(data) {
|
async function bake(data) {
|
||||||
// Ensure the relevant modules are loaded
|
// Ensure the relevant modules are loaded
|
||||||
self.loadRequiredModules(data.recipeConfig);
|
self.loadRequiredModules(data.recipeConfig);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
self.inputNum = parseInt(data.inputNum, 10);
|
self.inputNum = parseInt(data.inputNum, 10);
|
||||||
const response = await self.chef.bake(
|
const response = await self.chef.bake(
|
||||||
|
@ -119,7 +118,8 @@ async function bake(data) {
|
||||||
action: "bakeComplete",
|
action: "bakeComplete",
|
||||||
data: Object.assign(response, {
|
data: Object.assign(response, {
|
||||||
id: data.id,
|
id: data.id,
|
||||||
inputNum: data.inputNum
|
inputNum: data.inputNum,
|
||||||
|
bakeId: data.bakeId
|
||||||
})
|
})
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -128,7 +128,8 @@ async function bake(data) {
|
||||||
action: "bakeComplete",
|
action: "bakeComplete",
|
||||||
data: Object.assign(response, {
|
data: Object.assign(response, {
|
||||||
id: data.id,
|
id: data.id,
|
||||||
inputNum: data.inputNum
|
inputNum: data.inputNum,
|
||||||
|
bakeId: data.bakeId
|
||||||
})
|
})
|
||||||
}, [response.result]);
|
}, [response.result]);
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,6 +8,7 @@
|
||||||
import Utils from "../core/Utils";
|
import Utils from "../core/Utils";
|
||||||
import FileSaver from "file-saver";
|
import FileSaver from "file-saver";
|
||||||
import zip from "zlibjs/bin/zip.min";
|
import zip from "zlibjs/bin/zip.min";
|
||||||
|
import ZipWorker from "worker-loader?inline&fallback=false!./ZipWorker";
|
||||||
|
|
||||||
const Zlib = zip.Zlib;
|
const Zlib = zip.Zlib;
|
||||||
|
|
||||||
|
@ -29,6 +30,8 @@ class OutputWaiter {
|
||||||
this.outputs = {};
|
this.outputs = {};
|
||||||
this.activeTab = -1;
|
this.activeTab = -1;
|
||||||
|
|
||||||
|
this.zipWorker = null;
|
||||||
|
|
||||||
this.maxTabs = 4; // Calculate this
|
this.maxTabs = 4; // Calculate this
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -85,7 +88,8 @@ class OutputWaiter {
|
||||||
inputNum: inputNum,
|
inputNum: inputNum,
|
||||||
statusMessage: `Input ${inputNum} has not been baked yet.`,
|
statusMessage: `Input ${inputNum} has not been baked yet.`,
|
||||||
error: null,
|
error: null,
|
||||||
status: "inactive"
|
status: "inactive",
|
||||||
|
bakeId: -1
|
||||||
};
|
};
|
||||||
|
|
||||||
this.outputs[inputNum] = newOutput;
|
this.outputs[inputNum] = newOutput;
|
||||||
|
@ -163,6 +167,17 @@ class OutputWaiter {
|
||||||
this.set(inputNum);
|
this.set(inputNum);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the stored bake ID for the output in the ouptut array
|
||||||
|
*
|
||||||
|
* @param {number} bakeId
|
||||||
|
* @param {number} inputNum
|
||||||
|
*/
|
||||||
|
updateOutputBakeId(bakeId, inputNum) {
|
||||||
|
if (this.getOutput(inputNum) === -1) return;
|
||||||
|
this.outputs[inputNum].bakeId = bakeId;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Removes an output from the output array.
|
* Removes an output from the output array.
|
||||||
*
|
*
|
||||||
|
@ -202,11 +217,13 @@ class OutputWaiter {
|
||||||
const outputFile = document.getElementById("output-file");
|
const outputFile = document.getElementById("output-file");
|
||||||
const outputHighlighter = document.getElementById("output-highlighter");
|
const outputHighlighter = document.getElementById("output-highlighter");
|
||||||
const inputHighlighter = document.getElementById("input-highlighter");
|
const inputHighlighter = document.getElementById("input-highlighter");
|
||||||
// If inactive, show blank
|
|
||||||
// If pending or baking, show loader and status message
|
// If pending or baking, show loader and status message
|
||||||
// If error, style the tab and handle the error
|
// If error, style the tab and handle the error
|
||||||
// If done, display the output if it's the active tab
|
// If done, display the output if it's the active tab
|
||||||
if (output.status === "inactive" || output.status === "stale") {
|
// If inactive, show the last bake value (or blank)
|
||||||
|
if (output.status === "inactive" ||
|
||||||
|
output.status === "stale" ||
|
||||||
|
(output.status === "baked" && output.bakeId < this.manager.worker.bakeId)) {
|
||||||
this.manager.controls.showStaleIndicator();
|
this.manager.controls.showStaleIndicator();
|
||||||
} else {
|
} else {
|
||||||
this.manager.controls.hideStaleIndicator();
|
this.manager.controls.hideStaleIndicator();
|
||||||
|
@ -421,10 +438,111 @@ class OutputWaiter {
|
||||||
this.downloadAllFiles();
|
this.downloadAllFiles();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Spawns a new ZipWorker and sends it the outputs so that they can
|
||||||
|
* be zipped for download
|
||||||
|
*/
|
||||||
|
downloadAllFiles() {
|
||||||
|
|
||||||
|
const inputNums = Object.keys(this.outputs);
|
||||||
|
for (let i = 0; i < inputNums.length; i++) {
|
||||||
|
const iNum = inputNums[i];
|
||||||
|
if (this.outputs[iNum].status !== "baked" ||
|
||||||
|
this.outputs[iNum].bakeId !== this.manager.worker.bakeId) {
|
||||||
|
if (window.confirm("Not all outputs have been baked yet. Continue downloading outputs?")) {
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileName = window.prompt("Please enter a filename: ", "download.zip");
|
||||||
|
|
||||||
|
if (fileName === null || fileName === "") {
|
||||||
|
// Don't zip the files if there isn't a filename
|
||||||
|
this.app.alert("No filename was specified.", 3000);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let fileExt = window.prompt("Please enter a file extension for the files: ", ".txt");
|
||||||
|
|
||||||
|
if (fileExt === null) {
|
||||||
|
// Use .dat as the default file extension
|
||||||
|
fileExt = ".dat";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.zipWorker !== null) {
|
||||||
|
this.terminateZipWorker();
|
||||||
|
}
|
||||||
|
|
||||||
|
const downloadButton = document.getElementById("save-all-to-file");
|
||||||
|
|
||||||
|
downloadButton.disabled = true;
|
||||||
|
downloadButton.classList.add("spin");
|
||||||
|
$("[data-toggle='tooltip']").tooltip("hide");
|
||||||
|
|
||||||
|
downloadButton.firstElementChild.innerHTML = "autorenew";
|
||||||
|
|
||||||
|
log.debug("Creating ZipWorker");
|
||||||
|
this.zipWorker = new ZipWorker();
|
||||||
|
this.zipWorker.postMessage({
|
||||||
|
outputs: this.outputs,
|
||||||
|
filename: fileName,
|
||||||
|
fileExtension: fileExt
|
||||||
|
});
|
||||||
|
this.zipWorker.addEventListener("message", this.handleZipWorkerMessage.bind(this));
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Terminate the ZipWorker
|
||||||
|
*/
|
||||||
|
terminateZipWorker() {
|
||||||
|
if (this.zipWorker === null) return; // Already terminated
|
||||||
|
|
||||||
|
log.debug("Terminating ZipWorker.");
|
||||||
|
|
||||||
|
this.zipWorker.terminate();
|
||||||
|
this.zipWorker = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle messages sent back by the ZipWorker
|
||||||
|
*/
|
||||||
|
handleZipWorkerMessage(e) {
|
||||||
|
const r = e.data;
|
||||||
|
if (!r.hasOwnProperty("zippedFile")) {
|
||||||
|
log.error("No zipped file was sent in the message.");
|
||||||
|
this.terminateZipWorker();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!r.hasOwnProperty("filename")) {
|
||||||
|
log.error("No filename was sent in the message.");
|
||||||
|
this.terminateZipWorker();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const file = new File([r.zippedFile], r.filename);
|
||||||
|
FileSaver.saveAs(file, r.filename, false);
|
||||||
|
|
||||||
|
this.terminateZipWorker();
|
||||||
|
|
||||||
|
const downloadButton = document.getElementById("save-all-to-file");
|
||||||
|
|
||||||
|
downloadButton.disabled = false;
|
||||||
|
downloadButton.classList.remove("spin");
|
||||||
|
|
||||||
|
downloadButton.firstElementChild.innerHTML = "archive";
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handler for download all files events.
|
* Handler for download all files events.
|
||||||
*/
|
*/
|
||||||
async downloadAllFiles() {
|
async downloadAllFilesOld() {
|
||||||
const fileName = window.prompt("Please enter a filename: ", "download.zip");
|
const fileName = window.prompt("Please enter a filename: ", "download.zip");
|
||||||
const fileExt = window.prompt("Please enter a file extension for the files: ", ".txt");
|
const fileExt = window.prompt("Please enter a file extension for the files: ", ".txt");
|
||||||
const zip = new Zlib.Zip();
|
const zip = new Zlib.Zip();
|
||||||
|
@ -931,7 +1049,11 @@ class OutputWaiter {
|
||||||
* Copies the output to the clipboard
|
* Copies the output to the clipboard
|
||||||
*/
|
*/
|
||||||
copyClick() {
|
copyClick() {
|
||||||
const output = this.getActive();
|
let output = this.getActive();
|
||||||
|
|
||||||
|
if (typeof output !== "string") {
|
||||||
|
output = Utils.arrayBufferToStr(output);
|
||||||
|
}
|
||||||
|
|
||||||
// Create invisible textarea to populate with the raw dish string (not the printable version that
|
// Create invisible textarea to populate with the raw dish string (not the printable version that
|
||||||
// contains dots instead of the actual bytes)
|
// contains dots instead of the actual bytes)
|
||||||
|
|
|
@ -26,6 +26,7 @@ class WorkerWaiter {
|
||||||
this.maxWorkers = navigator.hardwareConcurrency || 4;
|
this.maxWorkers = navigator.hardwareConcurrency || 4;
|
||||||
this.inputs = [];
|
this.inputs = [];
|
||||||
this.totalOutputs = 0;
|
this.totalOutputs = 0;
|
||||||
|
this.bakeId = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -141,7 +142,7 @@ class WorkerWaiter {
|
||||||
switch (r.action) {
|
switch (r.action) {
|
||||||
case "bakeComplete":
|
case "bakeComplete":
|
||||||
log.debug(`Bake ${inputNum} complete.`);
|
log.debug(`Bake ${inputNum} complete.`);
|
||||||
this.updateOutput(r.data, r.data.inputNum);
|
this.updateOutput(r.data, r.data.inputNum, r.data.bakeId);
|
||||||
this.workerFinished(currentWorker);
|
this.workerFinished(currentWorker);
|
||||||
break;
|
break;
|
||||||
case "bakeError":
|
case "bakeError":
|
||||||
|
@ -187,11 +188,12 @@ class WorkerWaiter {
|
||||||
*
|
*
|
||||||
* @param {Object} data
|
* @param {Object} data
|
||||||
* @param {number} inputNum
|
* @param {number} inputNum
|
||||||
|
* @param {number} bakeId
|
||||||
*/
|
*/
|
||||||
updateOutput(data, inputNum) {
|
updateOutput(data, inputNum, bakeId) {
|
||||||
|
this.manager.output.updateOutputBakeId(bakeId, inputNum);
|
||||||
this.manager.output.updateOutputValue(data, inputNum);
|
this.manager.output.updateOutputValue(data, inputNum);
|
||||||
this.manager.output.updateOutputStatus("baked", inputNum);
|
this.manager.output.updateOutputStatus("baked", inputNum);
|
||||||
|
|
||||||
// this.manager.recipe.updateBreakpointIndicator(this.app.progress);
|
// this.manager.recipe.updateBreakpointIndicator(this.app.progress);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -337,7 +339,8 @@ class WorkerWaiter {
|
||||||
options: this.options,
|
options: this.options,
|
||||||
progress: this.progress,
|
progress: this.progress,
|
||||||
step: this.step,
|
step: this.step,
|
||||||
inputNum: nextInput.inputNum
|
inputNum: nextInput.inputNum,
|
||||||
|
bakeId: this.bakeId
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
|
@ -349,7 +352,8 @@ class WorkerWaiter {
|
||||||
options: this.options,
|
options: this.options,
|
||||||
progress: this.progress,
|
progress: this.progress,
|
||||||
step: this.step,
|
step: this.step,
|
||||||
inputNum: nextInput.inputNum
|
inputNum: nextInput.inputNum,
|
||||||
|
bakeId: this.bakeId
|
||||||
}
|
}
|
||||||
}, [nextInput.input]);
|
}, [nextInput.input]);
|
||||||
}
|
}
|
||||||
|
@ -366,7 +370,7 @@ class WorkerWaiter {
|
||||||
bake(recipeConfig, options, progress, step) {
|
bake(recipeConfig, options, progress, step) {
|
||||||
this.setBakingStatus(true);
|
this.setBakingStatus(true);
|
||||||
this.bakeStartTime = new Date().getTime();
|
this.bakeStartTime = new Date().getTime();
|
||||||
|
this.bakeId++;
|
||||||
this.recipeConfig = recipeConfig;
|
this.recipeConfig = recipeConfig;
|
||||||
this.options = options;
|
this.options = options;
|
||||||
this.progress = progress;
|
this.progress = progress;
|
||||||
|
|
69
src/web/ZipWorker.mjs
Normal file
69
src/web/ZipWorker.mjs
Normal file
|
@ -0,0 +1,69 @@
|
||||||
|
/**
|
||||||
|
* Web Worker to handle zipping the outputs for download.
|
||||||
|
*
|
||||||
|
* @author j433866 [j433866@gmail.com]
|
||||||
|
* @copyright Crown Copyright 2019
|
||||||
|
* @license Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
import zip from "zlibjs/bin/zip.min";
|
||||||
|
import Utils from "../core/Utils";
|
||||||
|
|
||||||
|
const Zlib = zip.Zlib;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Respond to message from parent thread.
|
||||||
|
*/
|
||||||
|
self.addEventListener("message", function(e) {
|
||||||
|
const r = e.data;
|
||||||
|
if (!r.hasOwnProperty("outputs")) {
|
||||||
|
log.error("No files were passed to the ZipWorker.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!r.hasOwnProperty("filename")) {
|
||||||
|
log.error("No filename was passed to the ZipWorker");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!r.hasOwnProperty("fileExtension")) {
|
||||||
|
log.error("No file extension was passed to the ZipWorker");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.zipFiles(r.outputs, r.filename, r.fileExtension);
|
||||||
|
});
|
||||||
|
|
||||||
|
self.setOption = function(...args) {};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compress the files into a zip file and send the zip back
|
||||||
|
* to the OutputWaiter.
|
||||||
|
*
|
||||||
|
* @param {object} outputs
|
||||||
|
* @param {string} filename
|
||||||
|
* @param {string} fileExtension
|
||||||
|
*/
|
||||||
|
self.zipFiles = function(outputs, filename, fileExtension) {
|
||||||
|
const zip = new Zlib.Zip();
|
||||||
|
const inputNums = Object.keys(outputs);
|
||||||
|
|
||||||
|
for (let i = 0; i < inputNums.length; i++) {
|
||||||
|
const iNum = inputNums[i];
|
||||||
|
const name = Utils.strToByteArray(iNum + fileExtension);
|
||||||
|
|
||||||
|
let output;
|
||||||
|
if (outputs[iNum].data === null) {
|
||||||
|
output = new Uint8Array(0);
|
||||||
|
} else if (typeof outputs[iNum].data.result === "string") {
|
||||||
|
output = new Uint8Array(Utils.strToArrayBuffer(outputs[iNum].data.result));
|
||||||
|
} else {
|
||||||
|
output = new Uint8Array(outputs[iNum].data.result);
|
||||||
|
}
|
||||||
|
zip.addFile(output, {filename: name});
|
||||||
|
}
|
||||||
|
|
||||||
|
const zippedFile = zip.compress();
|
||||||
|
self.postMessage({
|
||||||
|
zippedFile: zippedFile.buffer,
|
||||||
|
filename: filename
|
||||||
|
}, [zippedFile.buffer]);
|
||||||
|
};
|
|
@ -49,3 +49,15 @@
|
||||||
#controls .btn {
|
#controls .btn {
|
||||||
border-radius: 30px;
|
border-radius: 30px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.spin {
|
||||||
|
animation-name: spin;
|
||||||
|
animation-duration: 3s;
|
||||||
|
animation-iteration-count: infinite;
|
||||||
|
animation-timing-function: linear;
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyframes spin {
|
||||||
|
0% {transform: rotate(0deg);}
|
||||||
|
100% {transform: rotate(360deg);}
|
||||||
|
}
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue