From eda9a40aa27c89a64bf66d514303367b585fbddf Mon Sep 17 00:00:00 2001 From: Oliver Rahner Date: Thu, 21 Jul 2022 13:46:15 +0200 Subject: [PATCH 1/3] add Protobuf Stream Decode operation --- src/core/config/Categories.json | 1 + src/core/lib/Protobuf.mjs | 18 +++++++ src/core/operations/ProtobufStreamDecode.mjs | 54 ++++++++++++++++++++ tests/operations/tests/Protobuf.mjs | 27 ++++++++++ 4 files changed, 100 insertions(+) create mode 100644 src/core/operations/ProtobufStreamDecode.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 8ac60048..852c6e64 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -207,6 +207,7 @@ "URL Decode", "Protobuf Decode", "Protobuf Encode", + "Protobuf Stream Decode", "VarInt Encode", "VarInt Decode", "JA3 Fingerprint", diff --git a/src/core/lib/Protobuf.mjs b/src/core/lib/Protobuf.mjs index e131d3a5..6e7b5b6b 100644 --- a/src/core/lib/Protobuf.mjs +++ b/src/core/lib/Protobuf.mjs @@ -116,6 +116,24 @@ class Protobuf { return this.mergeDecodes(input); } + static decodeStream(input, args) { + this.updateProtoRoot(args[0]); + this.showUnknownFields = args[1]; + this.showTypes = args[2]; + + let streams = new Protobuf(input); + let output = []; + let objLength = streams._varInt(); + while (!isNaN(objLength) && objLength > 0) { + let subData = streams.data.slice(streams.offset, streams.offset + objLength); + output.push(this.mergeDecodes(subData)); + streams.offset += objLength; + objLength = streams._varInt(); + } + + return output; + } + /** * Update the parsedProto, throw parsing errors * diff --git a/src/core/operations/ProtobufStreamDecode.mjs b/src/core/operations/ProtobufStreamDecode.mjs new file mode 100644 index 00000000..01243b54 --- /dev/null +++ b/src/core/operations/ProtobufStreamDecode.mjs @@ -0,0 +1,54 @@ +/** + * @author GCHQ Contributor [3] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import Protobuf from "../lib/Protobuf.mjs"; + +/** + * Protobuf Stream Decode operation + */ +class ProtobufStreamDecode extends Operation { + + /** + * ProtobufStreamDecode constructor + */ + constructor() { + super(); + + this.name = "Protobuf Stream Decode"; + this.module = "Protobuf"; + this.description = "Decodes Protobuf encoded data from streams to a JSON array representation of the data using the field number as the field key.

If a .proto schema is defined, the encoded data will be decoded with reference to the schema. Only one message instance will be decoded.

Show Unknown Fields
When a schema is used, this option shows fields that are present in the input data but not defined in the schema.

Show Types
Show the type of a field next to its name. For undefined fields, the wiretype and example types are shown instead."; + this.infoURL = "https://developers.google.com/protocol-buffers/docs/techniques#streaming"; + this.inputType = "ArrayBuffer"; + this.outputType = "JSON"; + this.args = [ + { + name: "Show Types", + type: "boolean", + value: false + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {JSON} + */ + run(input, args) { + input = new Uint8Array(input); + try { + // provide standard values for currently removed arguments + return Protobuf.decodeStream(input, ["", false, ...args]); + } catch (err) { + throw new OperationError(err); + } + } + +} + +export default ProtobufStreamDecode; diff --git a/tests/operations/tests/Protobuf.mjs b/tests/operations/tests/Protobuf.mjs index 2131e723..5e90ec4c 100644 --- a/tests/operations/tests/Protobuf.mjs +++ b/tests/operations/tests/Protobuf.mjs @@ -303,4 +303,31 @@ TestRegister.addTests([ } ] }, + { + name: "Protobuf Stream Decode: no schema", + input: "0d081c1203596f751a024d65202b0c0a0a0a066162633132331200", + expectedOutput: JSON.stringify([{ + "1": 28, + "2": "You", + "3": "Me", + "4": 43 + }, + { + "1": { + "1": "abc123", + "2": {} + } + }], null, 4), + recipeConfig: [ + { + "op": "From Hex", + "args": ["Auto"] + }, + { + "op": "Protobuf Stream Decode", + "args": ["", false, false] + } + ] + }, + ]); From a0546c8a3d414e87483af7ddd968e7977141d92f Mon Sep 17 00:00:00 2001 From: Oliver Rahner Date: Thu, 21 Jul 2022 14:00:37 +0200 Subject: [PATCH 2/3] fix linting --- src/core/lib/Protobuf.mjs | 12 +++++++++--- tests/operations/tests/Protobuf.mjs | 24 +++++++++++++----------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/core/lib/Protobuf.mjs b/src/core/lib/Protobuf.mjs index 6e7b5b6b..e5cae4fc 100644 --- a/src/core/lib/Protobuf.mjs +++ b/src/core/lib/Protobuf.mjs @@ -116,16 +116,22 @@ class Protobuf { return this.mergeDecodes(input); } + /** + * Parse Protobuf stream data + * @param {byteArray} input + * @param {any[]} args + * @returns {any[]} + */ static decodeStream(input, args) { this.updateProtoRoot(args[0]); this.showUnknownFields = args[1]; this.showTypes = args[2]; - let streams = new Protobuf(input); - let output = []; + const streams = new Protobuf(input); + const output = []; let objLength = streams._varInt(); while (!isNaN(objLength) && objLength > 0) { - let subData = streams.data.slice(streams.offset, streams.offset + objLength); + const subData = streams.data.slice(streams.offset, streams.offset + objLength); output.push(this.mergeDecodes(subData)); streams.offset += objLength; objLength = streams._varInt(); diff --git a/tests/operations/tests/Protobuf.mjs b/tests/operations/tests/Protobuf.mjs index 5e90ec4c..fd3c36e7 100644 --- a/tests/operations/tests/Protobuf.mjs +++ b/tests/operations/tests/Protobuf.mjs @@ -306,18 +306,20 @@ TestRegister.addTests([ { name: "Protobuf Stream Decode: no schema", input: "0d081c1203596f751a024d65202b0c0a0a0a066162633132331200", - expectedOutput: JSON.stringify([{ - "1": 28, - "2": "You", - "3": "Me", - "4": 43 - }, - { - "1": { - "1": "abc123", - "2": {} + expectedOutput: JSON.stringify([ + { + "1": 28, + "2": "You", + "3": "Me", + "4": 43 + }, + { + "1": { + "1": "abc123", + "2": {} + } } - }], null, 4), + ], null, 4), recipeConfig: [ { "op": "From Hex", From 2316b77c9e409b55724ee888a3d4f0be13f6c774 Mon Sep 17 00:00:00 2001 From: Oliver Rahner Date: Thu, 21 Jul 2022 16:02:26 +0200 Subject: [PATCH 3/3] Fix test --- tests/operations/tests/Protobuf.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/operations/tests/Protobuf.mjs b/tests/operations/tests/Protobuf.mjs index fd3c36e7..3ac20518 100644 --- a/tests/operations/tests/Protobuf.mjs +++ b/tests/operations/tests/Protobuf.mjs @@ -327,7 +327,7 @@ TestRegister.addTests([ }, { "op": "Protobuf Stream Decode", - "args": ["", false, false] + "args": [false] } ] },