diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..92ebd43c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,41 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/javascript-node +{ + "name": "CyberChef", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/javascript-node:1-18-bookworm", + + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers/features/github-cli": "latest" + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [8080], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": { + "npm": "bash -c \"sudo chown node node_modules && npm install\"" + }, + + "containerEnv": { + "DISPLAY": ":99" + }, + + "mounts": [ + "source=${localWorkspaceFolderBasename}-node_modules,target=${containerWorkspaceFolder}/node_modules,type=volume" + ], + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + "dbaeumer.vscode-eslint", + "GitHub.vscode-github-actions" + ] + } + } + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..dd87e2d7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +node_modules +build diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6313b56c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml index b485edad..daa59490 100644 --- a/.github/workflows/pull_requests.yml +++ b/.github/workflows/pull_requests.yml @@ -33,6 +33,20 @@ jobs: if: success() run: npx grunt prod + - name: Production Image Build + if: success() + id: build-image + uses: redhat-actions/buildah-build@v2 + with: + # Not being uploaded to any registry, use a simple name to allow Buildah to build correctly. + image: cyberchef + containerfiles: ./Dockerfile + platforms: linux/amd64 + oci: true + # Webpack seems to use a lot of open files, increase the max open file limit to accomodate. + extra-args: | + --ulimit nofile=10000 + - name: UI Tests if: success() run: | diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index b1be78f9..a068ffbb 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -6,6 +6,12 @@ on: tags: - 'v*' +env: + REGISTRY: ghcr.io + REGISTRY_USER: ${{ github.actor }} + REGISTRY_PASSWORD: ${{ github.token }} + IMAGE_NAME: ${{ github.repository }} + jobs: main: runs-on: ubuntu-latest @@ -19,7 +25,7 @@ jobs: - name: Install run: | - npm install + npm ci npm run setheapsize - name: Lint @@ -31,17 +37,38 @@ jobs: npm run testnodeconsumer - name: Production Build - if: success() run: npx grunt prod - name: UI Tests - if: success() run: | sudo apt-get install xvfb xvfb-run --server-args="-screen 0 1200x800x24" npx grunt testui + - name: Image Metadata + id: image-metadata + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + + - name: Production Image Build + id: build-image + uses: redhat-actions/buildah-build@v2 + with: + tags: ${{ steps.image-metadata.outputs.tags }} + labels: ${{ steps.image-metadata.outputs.labels }} + containerfiles: ./Dockerfile + platforms: linux/amd64 + oci: true + # Webpack seems to use a lot of open files, increase the max open file limit to accomodate. + extra-args: | + --ulimit nofile=10000 + + - name: Upload Release Assets - if: success() id: upload-release-assets uses: svenstaro/upload-release-action@v2 with: @@ -53,7 +80,14 @@ jobs: body: "See the [CHANGELOG](https://github.com/gchq/CyberChef/blob/master/CHANGELOG.md) and [commit messages](https://github.com/gchq/CyberChef/commits/master) for details." - name: Publish to NPM - if: success() uses: JS-DevTools/npm-publish@v1 with: token: ${{ secrets.NPM_TOKEN }} + + - name: Publish to GHCR + uses: redhat-actions/push-to-registry@v2 + with: + tags: ${{ steps.build-image.outputs.tags }} + registry: ${{ env.REGISTRY }} + username: ${{ env.REGISTRY_USER }} + password: ${{ env.REGISTRY_PASSWORD }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dabea66..145d1b14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ All major and minor version changes will be documented in this file. Details of ## Details +### [10.8.0] - 2024-02-13 +- Add official Docker images [@AshCorr] | [#1699] + ### [10.7.0] - 2024-02-09 - Added 'File Tree' operation [@sw5678] | [#1667] - Added 'RISON' operation [@sg5506844] | [#1555] @@ -383,6 +386,7 @@ All major and minor version changes will be documented in this file. Details of ## [4.0.0] - 2016-11-28 - Initial open source commit [@n1474335] | [b1d73a72](https://github.com/gchq/CyberChef/commit/b1d73a725dc7ab9fb7eb789296efd2b7e4b08306) +[10.8.0]: https://github.com/gchq/CyberChef/releases/tag/v10.7.0 [10.7.0]: https://github.com/gchq/CyberChef/releases/tag/v10.7.0 [10.6.0]: https://github.com/gchq/CyberChef/releases/tag/v10.6.0 [10.5.0]: https://github.com/gchq/CyberChef/releases/tag/v10.5.0 @@ -546,6 +550,7 @@ All major and minor version changes will be documented in this file. Details of [@sw5678]: https://github.com/sw5678 [@sg5506844]: https://github.com/sg5506844 [@AliceGrey]: https://github.com/AliceGrey +[@AshCorr]: https://github.com/AshCorr [8ad18b]: https://github.com/gchq/CyberChef/commit/8ad18bc7db6d9ff184ba3518686293a7685bf7b7 @@ -672,3 +677,4 @@ All major and minor version changes will be documented in this file. Details of [#1667]: https://github.com/gchq/CyberChef/issues/1667 [#1555]: https://github.com/gchq/CyberChef/issues/1555 [#1694]: https://github.com/gchq/CyberChef/issues/1694 +[#1699]: https://github.com/gchq/CyberChef/issues/1694 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..be4c8bad --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM node:18-alpine AS build + +COPY . . +RUN npm ci +RUN npm run build + +FROM nginx:1.25-alpine3.18 AS cyberchef + +COPY --from=build ./build/prod /usr/share/nginx/html/ diff --git a/README.md b/README.md index 24f56e77..5549bda2 100755 --- a/README.md +++ b/README.md @@ -20,6 +20,22 @@ Cryptographic operations in CyberChef should not be relied upon to provide secur [A live demo can be found here][1] - have fun! +## Containers + +If you would like to try out CyberChef locally you can either build it yourself: + +```bash +docker build --tag cyberchef --ulimit nofile=10000 . +docker run -it -p 8080:80 cyberchef +``` + +Or you can use our image directly: + +```bash +docker run -it -p 8080:80 ghcr.io/gchq/cyberchef:latest +``` + +This image is built and published through our [GitHub Workflows](.github/workflows/releases.yml) ## How it works diff --git a/package-lock.json b/package-lock.json index 1d1c6333..29717727 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cyberchef", - "version": "10.7.0", + "version": "10.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cyberchef", - "version": "10.7.0", + "version": "10.8.0", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index 0ab4990e..e0c3b136 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "10.7.0", + "version": "10.8.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index ac1f81d6..4f1b3328 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -70,6 +70,7 @@ "Avro to JSON", "CBOR Encode", "CBOR Decode", + "Caret/M-decode", "Rison Encode", "Rison Decode" ] @@ -333,7 +334,8 @@ "CSS selector", "Extract EXIF", "Extract ID3", - "Extract Files" + "Extract Files", + "RAKE" ] }, { diff --git a/src/core/operations/CaretMdecode.mjs b/src/core/operations/CaretMdecode.mjs new file mode 100644 index 00000000..68c6dacb --- /dev/null +++ b/src/core/operations/CaretMdecode.mjs @@ -0,0 +1,98 @@ +/** + * @author tedk [tedk@ted.do] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; + +/** + * Caret/M-decode operation + * + * https://gist.githubusercontent.com/JaHIY/3c91bbf7bea5661e6abfbd1349ee81a2/raw/c7b480e9ff24bcb8f5287a8a8a2dcb9bf5628506/decode_m_notation.cpp + */ +class CaretMdecode extends Operation { + + /** + * CaretMdecode constructor + */ + constructor() { + super(); + + this.name = "Caret/M-decode"; + this.module = "Default"; + this.description = "Decodes caret or M-encoded strings, i.e. ^M turns into a newline, M-^] turns into 0x9d. Sources such as `cat -v`.\n\nPlease be aware that when using `cat -v` ^_ (caret-underscore) will not be encoded, but represents a valid encoding (namely that of 0x1f)."; + this.infoURL = "https://en.wikipedia.org/wiki/Caret_notation"; + this.inputType = "string"; + this.outputType = "byteArray"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + + const bytes = []; + + let prev = ""; + + for (let i = 0; i < input.length; i++) { + + const charCode = input.charCodeAt(i); + const curChar = input.charAt(i); + + if (prev === "M-^") { + if (charCode > 63 && charCode <= 95) { + bytes.push(charCode + 64); + } else if (charCode === 63) { + bytes.push(255); + } else { + bytes.push(77, 45, 94, charCode); + } + prev = ""; + } else if (prev === "M-") { + if (curChar === "^") { + prev = prev + "^"; + } else if (charCode >= 32 && charCode <= 126) { + bytes.push(charCode + 128); + prev = ""; + } else { + bytes.push(77, 45, charCode); + prev = ""; + } + } else if (prev === "M") { + if (curChar === "-") { + prev = prev + "-"; + } else { + bytes.push(77, charCode); + prev = ""; + } + } else if (prev === "^") { + if (charCode > 63 && charCode <= 126) { + bytes.push(charCode - 64); + } else if (charCode === 63) { + bytes.push(127); + } else { + bytes.push(94, charCode); + } + prev = ""; + } else { + if (curChar === "M") { + prev = "M"; + } else if (curChar === "^") { + prev = "^"; + } else { + bytes.push(charCode); + } + } + + } + return bytes; + } + +} + +export default CaretMdecode; diff --git a/src/core/operations/Diff.mjs b/src/core/operations/Diff.mjs index 84107245..9f180f86 100644 --- a/src/core/operations/Diff.mjs +++ b/src/core/operations/Diff.mjs @@ -119,9 +119,9 @@ class Diff extends Operation { for (let i = 0; i < diff.length; i++) { if (diff[i].added) { - if (showAdded) output += "" + Utils.escapeHtml(diff[i].value) + ""; + if (showAdded) output += "" + Utils.escapeHtml(diff[i].value) + ""; } else if (diff[i].removed) { - if (showRemoved) output += "" + Utils.escapeHtml(diff[i].value) + ""; + if (showRemoved) output += "" + Utils.escapeHtml(diff[i].value) + ""; } else if (!showSubtraction) { output += Utils.escapeHtml(diff[i].value); } diff --git a/src/core/operations/FileTree.mjs b/src/core/operations/FileTree.mjs index c01401c6..8321f8f5 100644 --- a/src/core/operations/FileTree.mjs +++ b/src/core/operations/FileTree.mjs @@ -21,7 +21,7 @@ class FileTree extends Operation { this.name = "File Tree"; this.module = "Default"; - this.description = "Creates file tree from list of file paths (Similar too tree linux command)"; + this.description = "Creates file tree from list of file paths (similar to the tree command in Linux)"; this.inputType = "string"; this.outputType = "string"; this.args = [ diff --git a/src/core/operations/RAKE.mjs b/src/core/operations/RAKE.mjs new file mode 100644 index 00000000..d1165b51 --- /dev/null +++ b/src/core/operations/RAKE.mjs @@ -0,0 +1,149 @@ +/** + * @author sw5678 + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; + +/** + * RAKE operation + */ +class RAKE extends Operation { + + /** + * RAKE constructor + */ + constructor() { + super(); + + this.name = "RAKE"; + this.module = "Default"; + this.description = [ + "Rapid Keyword Extraction (RAKE)", + "

", + "RAKE is a domain-independent keyword extraction algorithm in Natural Language Processing.", + "

", + "The list of stop words are from the NLTK python package", + ].join("\n"); + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + name: "Word Delimiter (Regex)", + type: "text", + value: "\\s" + }, + { + name: "Sentence Delimiter (Regex)", + type: "text", + value: "\\.\\s|\\n" + }, + { + name: "Stop Words", + type: "text", + value: "i,me,my,myself,we,our,ours,ourselves,you,you're,you've,you'll,you'd,your,yours,yourself,yourselves,he,him,his,himself,she,she's,her,hers,herself,it,it's,its,itsef,they,them,their,theirs,themselves,what,which,who,whom,this,that,that'll,these,those,am,is,are,was,were,be,been,being,have,has,had,having,do,does',did,doing,a,an,the,and,but,if,or,because,as,until,while,of,at,by,for,with,about,against,between,into,through,during,before,after,above,below,to,from,up,down,in,out,on,off,over,under,again,further,then,once,here,there,when,where,why,how,all,any,both,each,few,more,most,other,some,such,no,nor,not,only,own,same,so,than,too,very,s,t,can,will,just,don,don't,should,should've,now,d,ll,m,o,re,ve,y,ain,aren,aren't,couldn,couldn't,didn,didn't,doesn,doesn't,hadn,hadn't,hasn,hasn't,haven,haven't,isn,isn't,ma,mightn,mightn't,mustn,mustn't,needn,needn't,shan,shan't,shouldn,shouldn't,wasn,wasn't,weren,weren't,won,won't,wouldn,wouldn't" + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + + // Get delimiter regexs + const wordDelim = new RegExp(args[0], "g"); + const sentDelim = new RegExp(args[1], "g"); + + // Deduplicate the stop words and add the empty string + const stopWords = args[2].toLowerCase().replace(/ /g, "").split(",").unique(); + stopWords.push(""); + + // Lower case input and remove start and ending whitespace + input = input.toLowerCase().trim(); + + // Get tokens, token count, and phrases + const tokens = []; + const wordFrequencies = []; + let phrases = []; + + // Build up list of phrases and token counts + const sentences = input.split(sentDelim); + for (const sent of sentences) { + + // Split sentence into words + const splitSent = sent.split(wordDelim); + let startIndex = 0; + + for (let i = 0; i < splitSent.length; i++) { + const token = splitSent[i]; + if (stopWords.includes(token)) { + // If token is stop word then split to create phrase + phrases.push(splitSent.slice(startIndex, i)); + startIndex = i + 1; + } else { + // If token is not a stop word add to the count of the list of words + if (tokens.includes(token)) { + wordFrequencies[tokens.indexOf(token)]+=1; + } else { + tokens.push(token); + wordFrequencies.push(1); + } + } + } + phrases.push(splitSent.slice(startIndex)); + } + + // remove empty phrases + phrases = phrases.filter(subArray => subArray.length > 0); + + // Remove duplicate phrases + const uniquePhrases = [...new Set(phrases.map(function (phrase) { + return phrase.join(" "); + }))]; + phrases = uniquePhrases.map(function (phrase) { + return phrase.split(" "); + }); + + // Generate word_degree_matrix and populate + const wordDegreeMatrix = Array.from(Array(tokens.length), _ => Array(tokens.length).fill(0)); + phrases.forEach(function (phrase) { + phrase.forEach(function (word1) { + phrase.forEach(function (word2) { + wordDegreeMatrix[tokens.indexOf(word1)][tokens.indexOf(word2)]++; + }); + }); + }); + + // Calculate degree score for each token + const degreeScores = Array(tokens.length).fill(0); + for (let i=0; i b[0] - a[0]); + scores.unshift(new Array("Scores: ", "Keywords: ")); + + // Output works with the 'To Table' functionality already built into CC + return scores.map(function (score) { + return score.join(", "); + }).join("\n"); + } +} + +export default RAKE; diff --git a/src/web/stylesheets/index.css b/src/web/stylesheets/index.css index 960c7006..0565399f 100755 --- a/src/web/stylesheets/index.css +++ b/src/web/stylesheets/index.css @@ -36,4 +36,5 @@ @import "./layout/_structure.css"; /* Operations */ +@import "./operations/diff.css"; @import "./operations/json.css"; diff --git a/src/web/stylesheets/operations/diff.css b/src/web/stylesheets/operations/diff.css new file mode 100644 index 00000000..008cbbf5 --- /dev/null +++ b/src/web/stylesheets/operations/diff.css @@ -0,0 +1,8 @@ +del { + background-color: var(--hl3); +} + +ins { + text-decoration: underline; /* shouldn't be needed, but Chromium doesn't copy to clipboard without it */ + background-color: var(--hl5); +} diff --git a/src/web/stylesheets/operations/json.css b/src/web/stylesheets/operations/json.css index 22c07128..27861c8f 100644 --- a/src/web/stylesheets/operations/json.css +++ b/src/web/stylesheets/operations/json.css @@ -44,7 +44,8 @@ ul.json-dict, ol.json-array { display: contents; } .json-summary { - display: contents; + display: inline; + list-style: none; } /* Display object and array brackets when closed */ diff --git a/tests/browser/02_ops.js b/tests/browser/02_ops.js index e2c8a219..7cb0e941 100644 --- a/tests/browser/02_ops.js +++ b/tests/browser/02_ops.js @@ -108,7 +108,7 @@ module.exports = { // testOp(browser, "Derive EVP key", "test input", "test_output"); // testOp(browser, "Derive PBKDF2 key", "test input", "test_output"); // testOp(browser, "Detect File Type", "test input", "test_output"); - testOpHtml(browser, "Diff", "The cat sat on the mat\n\nThe mat cat on the sat", ".hl5:first-child", "mat", ["\\n\\n", "Word", true, true, false, false]); + testOpHtml(browser, "Diff", "The cat sat on the mat\n\nThe mat cat on the sat", "ins:first-child", "mat", ["\\n\\n", "Word", true, true, false, false]); // testOp(browser, "Disassemble x86", "test input", "test_output"); testOpImage(browser, "Dither Image", "files/Hitchhikers_Guide.jpeg"); // testOp(browser, "Divide", "test input", "test_output"); diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index aa120ba0..e85b6ad3 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -17,131 +17,132 @@ import { } from "../lib/utils.mjs"; import TestRegister from "../lib/TestRegister.mjs"; -import "./tests/BCD.mjs"; -import "./tests/BSON.mjs"; +import "./tests/AESKeyWrap.mjs"; +import "./tests/AvroToJSON.mjs"; import "./tests/BaconCipher.mjs"; import "./tests/Base45.mjs"; import "./tests/Base58.mjs"; -import "./tests/Base64.mjs"; import "./tests/Base62.mjs"; +import "./tests/Base64.mjs"; import "./tests/Base85.mjs"; import "./tests/Base92.mjs"; +import "./tests/BCD.mjs"; import "./tests/BitwiseOp.mjs"; +import "./tests/BLAKE2b.mjs"; +import "./tests/BLAKE2s.mjs"; +import "./tests/Bombe.mjs"; +import "./tests/BSON.mjs"; import "./tests/ByteRepr.mjs"; +import "./tests/CaesarBoxCipher.mjs"; +import "./tests/CaretMdecode.mjs"; import "./tests/CartesianProduct.mjs"; -import "./tests/CetaceanCipherEncode.mjs"; +import "./tests/CBORDecode.mjs"; +import "./tests/CBOREncode.mjs"; import "./tests/CetaceanCipherDecode.mjs"; +import "./tests/CetaceanCipherEncode.mjs"; import "./tests/ChaCha.mjs"; -import "./tests/CharEnc.mjs"; import "./tests/ChangeIPFormat.mjs"; +import "./tests/CharEnc.mjs"; import "./tests/Charts.mjs"; import "./tests/Checksum.mjs"; import "./tests/Ciphers.mjs"; +import "./tests/CipherSaber2.mjs"; +import "./tests/CMAC.mjs"; import "./tests/Code.mjs"; +import "./tests/Colossus.mjs"; import "./tests/Comment.mjs"; import "./tests/Compress.mjs"; import "./tests/ConditionalJump.mjs"; +import "./tests/ConvertCoordinateFormat.mjs"; +import "./tests/ConvertToNATOAlphabet.mjs"; import "./tests/Crypt.mjs"; import "./tests/CSV.mjs"; import "./tests/DateTime.mjs"; +import "./tests/DefangIP.mjs"; +import "./tests/ELFInfo.mjs"; +import "./tests/Enigma.mjs"; import "./tests/ExtractEmailAddresses.mjs"; +import "./tests/FileTree.mjs"; +import "./tests/FletcherChecksum.mjs"; import "./tests/Fork.mjs"; import "./tests/FromDecimal.mjs"; import "./tests/GenerateAllHashes.mjs"; -import "./tests/Gzip.mjs"; +import "./tests/GenerateDeBruijnSequence.mjs"; +import "./tests/GetAllCasings.mjs"; +import "./tests/GOST.mjs"; import "./tests/Gunzip.mjs"; +import "./tests/Gzip.mjs"; import "./tests/Hash.mjs"; +import "./tests/HASSH.mjs"; import "./tests/HaversineDistance.mjs"; import "./tests/Hex.mjs"; import "./tests/Hexdump.mjs"; +import "./tests/HKDF.mjs"; import "./tests/Image.mjs"; import "./tests/IndexOfCoincidence.mjs"; -import "./tests/Jump.mjs"; +import "./tests/JA3Fingerprint.mjs"; +import "./tests/JA3SFingerprint.mjs"; import "./tests/JSONBeautify.mjs"; import "./tests/JSONMinify.mjs"; import "./tests/JSONtoCSV.mjs"; +import "./tests/Jump.mjs"; import "./tests/JWTDecode.mjs"; import "./tests/JWTSign.mjs"; import "./tests/JWTVerify.mjs"; +import "./tests/LevenshteinDistance.mjs"; +import "./tests/Lorenz.mjs"; +import "./tests/LS47.mjs"; +import "./tests/LuhnChecksum.mjs"; import "./tests/LZNT1Decompress.mjs"; -import "./tests/MS.mjs"; +import "./tests/LZString.mjs"; import "./tests/Magic.mjs"; +import "./tests/Media.mjs"; import "./tests/MorseCode.mjs"; +import "./tests/MS.mjs"; +import "./tests/MultipleBombe.mjs"; import "./tests/MurmurHash3.mjs"; import "./tests/NetBIOS.mjs"; import "./tests/NormaliseUnicode.mjs"; +import "./tests/NTLM.mjs"; import "./tests/OTP.mjs"; +import "./tests/ParseIPRange.mjs"; +import "./tests/ParseObjectIDTimestamp.mjs"; +import "./tests/ParseQRCode.mjs"; +import "./tests/ParseSSHHostKey.mjs"; +import "./tests/ParseTCP.mjs"; +import "./tests/ParseTLV.mjs"; +import "./tests/ParseUDP.mjs"; +import "./tests/PEMtoHex.mjs"; import "./tests/PGP.mjs"; import "./tests/PHP.mjs"; -import "./tests/ParseIPRange.mjs"; -import "./tests/ParseQRCode.mjs"; -import "./tests/PEMtoHex.mjs"; import "./tests/PowerSet.mjs"; +import "./tests/Protobuf.mjs"; +import "./tests/Rabbit.mjs"; +import "./tests/RAKE.mjs"; import "./tests/Regex.mjs"; import "./tests/Register.mjs"; +import "./tests/RisonEncodeDecode.mjs"; import "./tests/Rotate.mjs"; +import "./tests/RSA.mjs"; import "./tests/SeqUtils.mjs"; import "./tests/SetDifference.mjs"; import "./tests/SetIntersection.mjs"; import "./tests/SetUnion.mjs"; +import "./tests/Shuffle.mjs"; +import "./tests/SIGABA.mjs"; import "./tests/SM4.mjs"; +// import "./tests/SplitColourChannels.mjs"; // Cannot test operations that use the File type yet import "./tests/StrUtils.mjs"; +import "./tests/Subsection.mjs"; +import "./tests/SwapCase.mjs"; import "./tests/SymmetricDifference.mjs"; import "./tests/TextEncodingBruteForce.mjs"; -import "./tests/TranslateDateTimeFormat.mjs"; -import "./tests/Magic.mjs"; -import "./tests/ParseTLV.mjs"; -import "./tests/Media.mjs"; import "./tests/ToFromInsensitiveRegex.mjs"; -import "./tests/YARA.mjs"; -import "./tests/ConvertCoordinateFormat.mjs"; -import "./tests/Enigma.mjs"; -import "./tests/Bombe.mjs"; -import "./tests/MultipleBombe.mjs"; +import "./tests/TranslateDateTimeFormat.mjs"; import "./tests/Typex.mjs"; -import "./tests/BLAKE2b.mjs"; -import "./tests/BLAKE2s.mjs"; -import "./tests/Protobuf.mjs"; -import "./tests/ParseSSHHostKey.mjs"; -import "./tests/DefangIP.mjs"; -import "./tests/ParseUDP.mjs"; -import "./tests/ParseTCP.mjs"; -import "./tests/AvroToJSON.mjs"; -import "./tests/Lorenz.mjs"; -import "./tests/LuhnChecksum.mjs"; -import "./tests/CipherSaber2.mjs"; -import "./tests/Colossus.mjs"; -import "./tests/ParseObjectIDTimestamp.mjs"; -import "./tests/Unicode.mjs"; -import "./tests/RSA.mjs"; -import "./tests/CBOREncode.mjs"; -import "./tests/CBORDecode.mjs"; -import "./tests/RisonEncodeDecode.mjs"; -import "./tests/JA3Fingerprint.mjs"; -import "./tests/JA3SFingerprint.mjs"; -import "./tests/HASSH.mjs"; -import "./tests/GetAllCasings.mjs"; -import "./tests/SIGABA.mjs"; -import "./tests/ELFInfo.mjs"; -import "./tests/Subsection.mjs"; -import "./tests/CaesarBoxCipher.mjs"; import "./tests/UnescapeString.mjs"; -import "./tests/LS47.mjs"; -import "./tests/LZString.mjs"; -import "./tests/NTLM.mjs"; -import "./tests/Shuffle.mjs"; -import "./tests/FletcherChecksum.mjs"; -import "./tests/CMAC.mjs"; -import "./tests/AESKeyWrap.mjs"; -import "./tests/Rabbit.mjs"; -import "./tests/LevenshteinDistance.mjs"; -import "./tests/SwapCase.mjs"; -import "./tests/HKDF.mjs"; -import "./tests/GenerateDeBruijnSequence.mjs"; -import "./tests/GOST.mjs"; - -// Cannot test operations that use the File type yet -// import "./tests/SplitColourChannels.mjs"; +import "./tests/Unicode.mjs"; +import "./tests/YARA.mjs"; const testStatus = { allTestsPassing: true, diff --git a/tests/operations/tests/CaretMdecode.mjs b/tests/operations/tests/CaretMdecode.mjs new file mode 100644 index 00000000..7c4f228e --- /dev/null +++ b/tests/operations/tests/CaretMdecode.mjs @@ -0,0 +1,39 @@ +/** + * Caesar Box Cipher tests. + * + * @author tedk [tedk@ted.do] + * + * @copyright Crown Copyright 2020 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "Caret/M-decode: nothing", + input: "", + expectedOutput: "", + recipeConfig: [ + { + op: "Caret/M-decode", + args: [], + }, + ], + }, + { + /* + * Tests the full range. + * Everything except "^_" (\x5e\x5f) will decode correctly. + */ + name: "Caret/M-decode: Full set", + input: "^@^A^B^C^D^E^F^G^H^I^J^K^L^M^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\\^]^^^_ !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~^?M-^@M-^AM-^BM-^CM-^DM-^EM-^FM-^GM-^HM-^IM-^JM-^KM-^LM-^MM-^NM-^OM-^PM-^QM-^RM-^SM-^TM-^UM-^VM-^WM-^XM-^YM-^ZM-^[M-^\\M-^]M-^^M-^_M- M-!M-\"M-#M-$M-%M-&M-'M-(M-)M-*M-+M-,M--M-.M-/M-0M-1M-2M-3M-4M-5M-6M-7M-8M-9M-:M-;M-M-?M-@M-AM-BM-CM-DM-EM-FM-GM-HM-IM-JM-KM-LM-MM-NM-OM-PM-QM-RM-SM-TM-UM-VM-WM-XM-YM-ZM-[M-\\M-]M-^M-_M-`M-aM-bM-cM-dM-eM-fM-gM-hM-iM-jM-kM-lM-mM-nM-oM-pM-qM-rM-sM-tM-uM-vM-wM-xM-yM-zM-{M-|M-}M-~M-^?", + expectedOutput: "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x1f\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\x8d\x2d\x5f\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff", + recipeConfig: [ + { + op: "Caret/M-decode", + args: [], + }, + ], + }, +]); + diff --git a/tests/operations/tests/FileTree.mjs b/tests/operations/tests/FileTree.mjs index 34249e3c..fc97678f 100644 --- a/tests/operations/tests/FileTree.mjs +++ b/tests/operations/tests/FileTree.mjs @@ -1,4 +1,6 @@ /** + * File tree tests. + * * @author sw5678 * @copyright Crown Copyright 2023 * @license Apache-2.0 @@ -7,14 +9,13 @@ import TestRegister from "../../lib/TestRegister.mjs"; TestRegister.addTests([ { - "name": "Swap Case: basic example", + "name": "File Tree: basic example", "input": "/test_dir1/test_file1.txt\n/test_dir1/test_file2.txt\n/test_dir2/test_file1.txt", "expectedOutput": "test_dir1\n|---test_file1.txt\n|---test_file2.txt\ntest_dir2\n|---test_file1.txt", "recipeConfig": [ { "op": "File Tree", - "args": [ - ], + "args": ["/", "Line feed"], }, ], } diff --git a/tests/operations/tests/FromGeohash.mjs b/tests/operations/tests/FromGeohash.mjs deleted file mode 100644 index dec58687..00000000 --- a/tests/operations/tests/FromGeohash.mjs +++ /dev/null @@ -1,55 +0,0 @@ -/** - * To Geohash tests - * - * @author gchq77703 - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - */ -import TestRegister from "../../lib/TestRegister.mjs"; - -TestRegister.addTests([ - { - name: "From Geohash", - input: "ww8p1r4t8", - expectedOutput: "37.83238649368286,112.55838632583618", - recipeConfig: [ - { - op: "From Geohash", - args: [], - }, - ], - }, - { - name: "From Geohash", - input: "ww8p1r", - expectedOutput: "37.83416748046875,112.5604248046875", - recipeConfig: [ - { - op: "From Geohash", - args: [], - }, - ], - }, - { - name: "From Geohash", - input: "ww8", - expectedOutput: "37.265625,113.203125", - recipeConfig: [ - { - op: "From Geohash", - args: [], - }, - ], - }, - { - name: "From Geohash", - input: "w", - expectedOutput: "22.5,112.5", - recipeConfig: [ - { - op: "From Geohash", - args: [], - }, - ], - }, -]); diff --git a/tests/operations/tests/RAKE.mjs b/tests/operations/tests/RAKE.mjs new file mode 100644 index 00000000..8164ca01 --- /dev/null +++ b/tests/operations/tests/RAKE.mjs @@ -0,0 +1,22 @@ +/** + * RAKE, Rapid Automatic Keyword Extraction tests. + * + * @author sw5678 + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + "name": "RAKE: Basic Example", + "input": "test1 test2. test2", + "expectedOutput": "Scores: , Keywords: \n3.5, test1 test2\n1.5, test2", + "recipeConfig": [ + { + "op": "RAKE", + "args": ["\\s", "\\.\\s|\\n", "i,me,my,myself,we,our"] + }, + ], + } +]); diff --git a/tests/operations/tests/StrUtils.mjs b/tests/operations/tests/StrUtils.mjs index c78b6b23..70e9c362 100644 --- a/tests/operations/tests/StrUtils.mjs +++ b/tests/operations/tests/StrUtils.mjs @@ -11,7 +11,7 @@ TestRegister.addTests([ { name: "Diff, basic usage", input: "testing23\n\ntesting123", - expectedOutput: "testing123", + expectedOutput: "testing123", recipeConfig: [ { "op": "Diff", @@ -22,7 +22,7 @@ TestRegister.addTests([ { name: "Diff added with subtraction, basic usage", input: "testing23\n\ntesting123", - expectedOutput: "1", + expectedOutput: "1", recipeConfig: [ { "op": "Diff", @@ -33,7 +33,7 @@ TestRegister.addTests([ { name: "Diff removed with subtraction, basic usage", input: "testing123\n\ntesting3", - expectedOutput: "12", + expectedOutput: "12", recipeConfig: [ { "op": "Diff", diff --git a/tests/operations/tests/ToGeohash.mjs b/tests/operations/tests/ToGeohash.mjs deleted file mode 100644 index 96dece85..00000000 --- a/tests/operations/tests/ToGeohash.mjs +++ /dev/null @@ -1,55 +0,0 @@ -/** - * To Geohash tests - * - * @author gchq77703 - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - */ -import TestRegister from "../../lib/TestRegister.mjs"; - -TestRegister.addTests([ - { - name: "To Geohash", - input: "37.8324,112.5584", - expectedOutput: "ww8p1r4t8", - recipeConfig: [ - { - op: "To Geohash", - args: [9], - }, - ], - }, - { - name: "To Geohash", - input: "37.9324,-112.2584", - expectedOutput: "9w8pv3ruj", - recipeConfig: [ - { - op: "To Geohash", - args: [9], - }, - ], - }, - { - name: "To Geohash", - input: "37.8324,112.5584", - expectedOutput: "ww8", - recipeConfig: [ - { - op: "To Geohash", - args: [3], - }, - ], - }, - { - name: "To Geohash", - input: "37.9324,-112.2584", - expectedOutput: "9w8pv3rujxy5b99", - recipeConfig: [ - { - op: "To Geohash", - args: [15], - }, - ], - }, -]);