Merge branch 'feature-protobuf'

src/core/config/Categories.json

@@ -169,6 +169,9 @@
             "Parse URI",
             "URL Encode",
             "URL Decode",
+            "Protobuf Decode",
+            "VarInt Encode",
+            "VarInt Decode",
             "Format MAC addresses",
             "Change IP format",
             "Group IP addresses",

src/core/lib/Protobuf.mjs

@@ -0,0 +1,285 @@
+import Utils from "../Utils";
+
+/**
+ * Protobuf lib. Contains functions to decode protobuf serialised
+ * data without a schema or .proto file.
+ *
+ * Provides utility functions to encode and decode variable length
+ * integers (varint).
+ *
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+class Protobuf {
+
+    /**
+     * Protobuf constructor
+     *
+     * @param {byteArray} data
+     */
+    constructor(data) {
+        // Check we have a byteArray
+        if (data instanceof Array) {
+            this.data = data;
+        } else {
+            throw new Error("Protobuf input must be a byteArray");
+        }
+
+        // Set up masks
+        this.TYPE = 0x07;
+        this.NUMBER = 0x78;
+        this.MSB = 0x80;
+        this.VALUE = 0x7f;
+
+        // Declare offset and length
+        this.offset = 0;
+        this.LENGTH = data.length;
+    }
+
+    // Public Functions
+
+    /**
+     * Encode a varint from a number
+     *
+     * @param {number} number
+     * @returns {byteArray}
+     */
+    static varIntEncode(number) {
+        const MSB = 0x80,
+            VALUE = 0x7f,
+            MSBALL = ~VALUE,
+            INT = Math.pow(2, 31);
+        const out = [];
+        let offset = 0;
+
+        while (number >= INT) {
+            out[offset++] = (number & 0xff) | MSB;
+            number /= 128;
+        }
+        while (number & MSBALL) {
+            out[offset++] = (number & 0xff) | MSB;
+            number >>>= 7;
+        }
+        out[offset] = number | 0;
+        return out;
+    }
+
+    /**
+     * Decode a varint from the byteArray
+     *
+     * @param {byteArray} input
+     * @returns {number}
+     */
+    static varIntDecode(input) {
+        const pb = new Protobuf(input);
+        return pb._varInt();
+    }
+
+    /**
+     * Parse Protobuf data
+     *
+     * @param {byteArray} input
+     * @returns {Object}
+     */
+    static decode(input) {
+        const pb = new Protobuf(input);
+        return pb._parse();
+    }
+
+    // Private Class Functions
+
+    /**
+     * Main private parsing function
+     *
+     * @private
+     * @returns {Object}
+     */
+    _parse() {
+        let object = {};
+        // Continue reading whilst we still have data
+        while (this.offset < this.LENGTH) {
+            const field = this._parseField();
+            object = this._addField(field, object);
+        }
+        // Throw an error if we have gone beyond the end of the data
+        if (this.offset > this.LENGTH) {
+            throw new Error("Exhausted Buffer");
+        }
+        return object;
+    }
+
+    /**
+     * Add a field read from the protobuf data into the Object. As
+     * protobuf fields can appear multiple times, if the field already
+     * exists we need to add the new field into an array of fields
+     * for that key.
+     *
+     * @private
+     * @param {Object} field
+     * @param {Object} object
+     * @returns {Object}
+     */
+    _addField(field, object) {
+        // Get the field key/values
+        const key = field.key;
+        const value = field.value;
+        object[key] = object.hasOwnProperty(key) ?
+            object[key] instanceof Array ?
+                object[key].concat([value]) :
+                [object[key], value] :
+            value;
+        return object;
+    }
+
+    /**
+     * Parse a field and return the Object read from the record
+     *
+     * @private
+     * @returns {Object}
+     */
+    _parseField() {
+        // Get the field headers
+        const header = this._fieldHeader();
+        const type = header.type;
+        const key = header.key;
+        switch (type) {
+            // varint
+            case 0:
+                return { "key": key, "value": this._varInt() };
+            // fixed 64
+            case 1:
+                return { "key": key, "value": this._uint64() };
+            // length delimited
+            case 2:
+                return { "key": key, "value": this._lenDelim() };
+            // fixed 32
+            case 5:
+                return { "key": key, "value": this._uint32() };
+            // unknown type
+            default:
+                throw new Error("Unknown type 0x" + type.toString(16));
+        }
+    }
+
+    /**
+     * Parse the field header and return the type and key
+     *
+     * @private
+     * @returns {Object}
+     */
+    _fieldHeader() {
+        // Make sure we call type then number to preserve offset
+        return { "type": this._fieldType(), "key": this._fieldNumber() };
+    }
+
+    /**
+     * Parse the field type from the field header. Type is stored in the
+     * lower 3 bits of the tag byte. This does not move the offset on as
+     * we need to read the field number from the tag byte too.
+     *
+     * @private
+     * @returns {number}
+     */
+    _fieldType() {
+        // Field type stored in lower 3 bits of tag byte
+        return this.data[this.offset] & this.TYPE;
+    }
+
+    /**
+     * Parse the field number (i.e. the key) from the field header. The
+     * field number is stored in the upper 5 bits of the tag byte - but
+     * is also varint encoded so the follow on bytes may need to be read
+     * when field numbers are > 15.
+     *
+     * @private
+     * @returns {number}
+     */
+    _fieldNumber() {
+        let shift = -3;
+        let fieldNumber = 0;
+        do {
+            fieldNumber += shift < 28 ?
+                shift === -3 ?
+                    (this.data[this.offset] & this.NUMBER) >> -shift :
+                    (this.data[this.offset] & this.VALUE) << shift :
+                (this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
+            shift += 7;
+        } while ((this.data[this.offset++] & this.MSD) === this.MSB);
+        return fieldNumber;
+    }
+
+    // Field Parsing Functions
+
+    /**
+     * Read off a varint from the data
+     *
+     * @private
+     * @returns {number}
+     */
+    _varInt() {
+        let value = 0;
+        let shift = 0;
+        // Keep reading while upper bit set
+        do {
+            value += shift < 28 ?
+                (this.data[this.offset] & this.VALUE) << shift :
+                (this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
+            shift += 7;
+        } while ((this.data[this.offset++] & this.MSB) === this.MSB);
+        return value;
+    }
+
+    /**
+     * Read off a 64 bit unsigned integer from the data
+     *
+     * @private
+     * @returns {number}
+     */
+    _uint64() {
+        // Read off a Uint64
+        let num = this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
+        num = num * 0x100000000 + this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
+        return num;
+    }
+
+    /**
+     * Read off a length delimited field from the data
+     *
+     * @private
+     * @returns {Object|string}
+     */
+    _lenDelim() {
+        // Read off the field length
+        const length = this._varInt();
+        const fieldBytes = this.data.slice(this.offset, this.offset + length);
+        let field;
+        try {
+            // Attempt to parse as a new Protobuf Object
+            const pbObject = new Protobuf(fieldBytes);
+            field = pbObject._parse();
+        } catch (err) {
+            // Otherwise treat as bytes
+            field = Utils.byteArrayToChars(fieldBytes);
+        }
+        // Move the offset and return the field
+        this.offset += length;
+        return field;
+    }
+
+    /**
+     * Read a 32 bit unsigned integer from the data
+     *
+     * @private
+     * @returns {number}
+     */
+    _uint32() {
+        // Use a dataview to read off the integer
+        const dataview = new DataView(new Uint8Array(this.data.slice(this.offset, this.offset + 4)).buffer);
+        const value = dataview.getUint32(0);
+        this.offset += 4;
+        return value;
+    }
+}
+
+export default Protobuf;

src/core/operations/ProtobufDecode.mjs

@@ -0,0 +1,46 @@
+/**
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Protobuf from "../lib/Protobuf";
+
+/**
+ * Protobuf Decode operation
+ */
+class ProtobufDecode extends Operation {
+
+    /**
+     * ProtobufDecode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Protobuf Decode";
+        this.module = "Default";
+        this.description = "Decodes any Protobuf encoded data to a JSON representation of the data using the field number as the field key.";
+        this.infoURL = "https://wikipedia.org/wiki/Protocol_Buffers";
+        this.inputType = "byteArray";
+        this.outputType = "JSON";
+        this.args = [];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        try {
+            return Protobuf.decode(input);
+        } catch (err) {
+            throw new OperationError(err);
+        }
+    }
+
+}
+
+export default ProtobufDecode;

src/core/operations/VarIntDecode.mjs

@@ -0,0 +1,46 @@
+/**
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Protobuf from "../lib/Protobuf";
+
+/**
+ * VarInt Decode operation
+ */
+class VarIntDecode extends Operation {
+
+    /**
+     * VarIntDecode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "VarInt Decode";
+        this.module = "Default";
+        this.description = "Decodes a VarInt encoded integer. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf.";
+        this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints";
+        this.inputType = "byteArray";
+        this.outputType = "number";
+        this.args = [];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {number}
+     */
+    run(input, args) {
+        try {
+            return Protobuf.varIntDecode(input);
+        } catch (err) {
+            throw new OperationError(err);
+        }
+    }
+
+}
+
+export default VarIntDecode;

src/core/operations/VarIntEncode.mjs

@@ -0,0 +1,46 @@
+/**
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Protobuf from "../lib/Protobuf";
+
+/**
+ * VarInt Encode operation
+ */
+class VarIntEncode extends Operation {
+
+    /**
+     * VarIntEncode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "VarInt Encode";
+        this.module = "Default";
+        this.description = "Encodes a Vn integer as a VarInt. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf.";
+        this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints";
+        this.inputType = "number";
+        this.outputType = "byteArray";
+        this.args = [];
+    }
+
+    /**
+     * @param {number} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        try {
+            return Protobuf.varIntEncode(input);
+        } catch (err) {
+            throw new OperationError(err);
+        }
+    }
+
+}
+
+export default VarIntEncode;

tests/operations/index.mjs

@@ -89,6 +89,7 @@ import "./tests/MultipleBombe";
 import "./tests/Typex";
 import "./tests/BLAKE2b";
 import "./tests/BLAKE2s";
+import "./tests/Protobuf";
 
 // Cannot test operations that use the File type yet
 //import "./tests/SplitColourChannels";

tests/operations/tests/Protobuf.mjs

@@ -0,0 +1,36 @@
+/**
+ * Protobuf tests.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "Protobuf Decode",
+        input: "0d1c0000001203596f751a024d65202b2a0a0a066162633132331200",
+        expectedOutput: JSON.stringify({
+            "1": 469762048,
+            "2": "You",
+            "3": "Me",
+            "4": 43,
+            "5": {
+                "1": "abc123",
+                "2": {}
+            }
+        }, null, 4),
+        recipeConfig: [
+            {
+                "op": "From Hex",
+                "args": ["Auto"]
+            },
+            {
+                "op": "Protobuf Decode",
+                "args": []
+            }
+        ]
+    },
+]);