mirror of
https://github.com/gchq/CyberChef.git
synced 2025-04-20 23:06:16 -04:00
Protobuf operations improved to enable full and partial schema support
This commit is contained in:
n1474335
parent
a4a13666e6
commit
dd18e52993
7 changed files with 723 additions and 19 deletions
|
|
@ -1,4 +1,5 @@
|
|||
import Utils from "../Utils.mjs";
|
||||
import protobuf from "protobufjs";
|
||||
|
||||
/**
|
||||
* Protobuf lib. Contains functions to decode protobuf serialised
|
||||
|
|
@ -32,9 +33,10 @@ class Protobuf {
|
|||
this.MSB = 0x80;
|
||||
this.VALUE = 0x7f;
|
||||
|
||||
// Declare offset and length
|
||||
// Declare offset, length, and field type object
|
||||
this.offset = 0;
|
||||
this.LENGTH = data.length;
|
||||
this.fieldTypes = {};
|
||||
}
|
||||
|
||||
// Public Functions
|
||||
|
|
@ -76,15 +78,281 @@ class Protobuf {
|
|||
return pb._varInt();
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode input JSON according to the given schema
|
||||
*
|
||||
* @param {Object} input
|
||||
* @param {Object []} args
|
||||
* @returns {Object}
|
||||
*/
|
||||
static encode(input, args) {
|
||||
this.updateProtoRoot(args[0]);
|
||||
if (!this.mainMessageName) {
|
||||
throw new Error("Schema Error: Schema not defined");
|
||||
}
|
||||
const message = this.parsedProto.root.nested[this.mainMessageName];
|
||||
|
||||
// Convert input into instance of message, and verify instance
|
||||
input = message.fromObject(input);
|
||||
const error = message.verify(input);
|
||||
if (error) {
|
||||
throw new Error("Input Error: " + error);
|
||||
}
|
||||
// Encode input
|
||||
const output = message.encode(input).finish();
|
||||
return new Uint8Array(output).buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Protobuf data
|
||||
*
|
||||
* @param {byteArray} input
|
||||
* @returns {Object}
|
||||
*/
|
||||
static decode(input) {
|
||||
static decode(input, args) {
|
||||
this.updateProtoRoot(args[0]);
|
||||
this.showUnknownFields = args[1];
|
||||
this.showTypes = args[2];
|
||||
return this.mergeDecodes(input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the parsedProto, throw parsing errors
|
||||
*
|
||||
* @param {string} protoText
|
||||
*/
|
||||
static updateProtoRoot(protoText) {
|
||||
try {
|
||||
this.parsedProto = protobuf.parse(protoText);
|
||||
if (this.parsedProto.package) {
|
||||
this.parsedProto.root = this.parsedProto.root.nested[this.parsedProto.package];
|
||||
}
|
||||
this.updateMainMessageName();
|
||||
} catch (error) {
|
||||
throw new Error("Schema " + error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set mainMessageName to the first instance of a message defined in the schema that is not a submessage
|
||||
*
|
||||
*/
|
||||
static updateMainMessageName() {
|
||||
const messageNames = [];
|
||||
const fieldTypes = [];
|
||||
this.parsedProto.root.nestedArray.forEach(block => {
|
||||
if (block.constructor.name === "Type") {
|
||||
messageNames.push(block.name);
|
||||
this.parsedProto.root.nested[block.name].fieldsArray.forEach(field => {
|
||||
fieldTypes.push(field.type);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (messageNames.length === 0) {
|
||||
this.mainMessageName = null;
|
||||
} else {
|
||||
for (const name of messageNames) {
|
||||
if (!fieldTypes.includes(name)) {
|
||||
this.mainMessageName = name;
|
||||
break;
|
||||
}
|
||||
}
|
||||
this.mainMessageName = messageNames[0];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode input using Protobufjs package and raw methods, compare, and merge results
|
||||
*
|
||||
* @param {byteArray} input
|
||||
* @returns {Object}
|
||||
*/
|
||||
static mergeDecodes(input) {
|
||||
const pb = new Protobuf(input);
|
||||
return pb._parse();
|
||||
let rawDecode = pb._parse();
|
||||
let message;
|
||||
|
||||
if (this.showTypes) {
|
||||
rawDecode = this.showRawTypes(rawDecode, pb.fieldTypes);
|
||||
this.parsedProto.root = this.appendTypesToFieldNames(this.parsedProto.root);
|
||||
}
|
||||
|
||||
try {
|
||||
message = this.parsedProto.root.nested[this.mainMessageName];
|
||||
const packageDecode = message.toObject(message.decode(input), {
|
||||
bytes: String,
|
||||
longs: Number,
|
||||
enums: String,
|
||||
defualts: true
|
||||
});
|
||||
const output = {};
|
||||
|
||||
if (this.showUnknownFields) {
|
||||
output[message.name] = packageDecode;
|
||||
output["Unknown Fields"] = this.compareFields(rawDecode, message);
|
||||
return output;
|
||||
} else {
|
||||
return packageDecode;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
if (message) {
|
||||
throw new Error("Input " + error);
|
||||
} else {
|
||||
return rawDecode;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace fieldnames with fieldname and type
|
||||
*
|
||||
* @param {Object} schemaRoot
|
||||
* @returns {Object}
|
||||
*/
|
||||
static appendTypesToFieldNames(schemaRoot) {
|
||||
for (const block of schemaRoot.nestedArray) {
|
||||
if (block.constructor.name === "Type") {
|
||||
for (const [fieldName, fieldData] of Object.entries(block.fields)) {
|
||||
schemaRoot.nested[block.name].remove(block.fields[fieldName]);
|
||||
schemaRoot.nested[block.name].add(new protobuf.Field(`${fieldName} (${fieldData.type})`, fieldData.id, fieldData.type, fieldData.rule));
|
||||
}
|
||||
}
|
||||
}
|
||||
return schemaRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add field type to field name for fields in the raw decoded output
|
||||
*
|
||||
* @param {Object} rawDecode
|
||||
* @param {Object} fieldTypes
|
||||
* @returns {Object}
|
||||
*/
|
||||
static showRawTypes(rawDecode, fieldTypes) {
|
||||
for (const [fieldNum, value] of Object.entries(rawDecode)) {
|
||||
const fieldType = fieldTypes[fieldNum];
|
||||
let outputFieldValue;
|
||||
let outputFieldType;
|
||||
|
||||
// Submessages
|
||||
if (isNaN(fieldType)) {
|
||||
outputFieldType = 2;
|
||||
|
||||
// Repeated submessages
|
||||
if (Array.isArray(value)) {
|
||||
const fieldInstances = [];
|
||||
for (const instance of Object.keys(value)) {
|
||||
if (typeof(value[instance]) !== "string") {
|
||||
fieldInstances.push(this.showRawTypes(value[instance], fieldType));
|
||||
} else {
|
||||
fieldInstances.push(value[instance]);
|
||||
}
|
||||
}
|
||||
outputFieldValue = fieldInstances;
|
||||
|
||||
// Single submessage
|
||||
} else {
|
||||
outputFieldValue = this.showRawTypes(value, fieldType);
|
||||
}
|
||||
|
||||
// Non-submessage field
|
||||
} else {
|
||||
outputFieldType = fieldType;
|
||||
outputFieldValue = value;
|
||||
}
|
||||
|
||||
// Substitute fieldNum with field number and type
|
||||
rawDecode[`field #${fieldNum}: ${this.getTypeInfo(outputFieldType)}`] = outputFieldValue;
|
||||
delete rawDecode[fieldNum];
|
||||
}
|
||||
return rawDecode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare raw decode to package decode and return discrepancies
|
||||
*
|
||||
* @param rawDecodedMessage
|
||||
* @param schemaMessage
|
||||
* @returns {Object}
|
||||
*/
|
||||
static compareFields(rawDecodedMessage, schemaMessage) {
|
||||
// Define message data using raw decode output and schema
|
||||
const schemaFieldProperties = {};
|
||||
const schemaFieldNames = Object.keys(schemaMessage.fields);
|
||||
schemaFieldNames.forEach(field => schemaFieldProperties[schemaMessage.fields[field].id] = field);
|
||||
|
||||
// Loop over each field present in the raw decode output
|
||||
for (const fieldName in rawDecodedMessage) {
|
||||
let fieldId;
|
||||
if (isNaN(fieldName)) {
|
||||
fieldId = fieldName.match(/^field #(\d+)/)[1];
|
||||
} else {
|
||||
fieldId = fieldName;
|
||||
}
|
||||
|
||||
// Check if this field is defined in the schema
|
||||
if (fieldId in schemaFieldProperties) {
|
||||
const schemaFieldName = schemaFieldProperties[fieldId];
|
||||
|
||||
// Extract the current field data from the raw decode and schema
|
||||
const rawFieldData = rawDecodedMessage[fieldName];
|
||||
const schemaField = schemaMessage.fields[schemaFieldName];
|
||||
|
||||
// Check for repeated fields
|
||||
if (Array.isArray(rawFieldData) && !schemaField.repeated) {
|
||||
rawDecodedMessage[`(${schemaMessage.name}) ${schemaFieldName} is a repeated field`] = rawFieldData;
|
||||
}
|
||||
|
||||
// Check for submessage fields
|
||||
if (schemaField.resolvedType !== null && schemaField.resolvedType.constructor.name === "Type") {
|
||||
const subMessageType = schemaMessage.fields[schemaFieldName].type;
|
||||
const schemaSubMessage = this.parsedProto.root.nested[subMessageType];
|
||||
const rawSubMessages = rawDecodedMessage[fieldName];
|
||||
let rawDecodedSubMessage = {};
|
||||
|
||||
// Squash multiple submessage instances into one submessage
|
||||
if (Array.isArray(rawSubMessages)) {
|
||||
rawSubMessages.forEach(subMessageInstance => {
|
||||
const instanceFields = Object.entries(subMessageInstance);
|
||||
instanceFields.forEach(subField => {
|
||||
rawDecodedSubMessage[subField[0]] = subField[1];
|
||||
});
|
||||
});
|
||||
} else {
|
||||
rawDecodedSubMessage = rawSubMessages;
|
||||
}
|
||||
|
||||
// Treat submessage as own message and compare its fields
|
||||
rawDecodedSubMessage = Protobuf.compareFields(rawDecodedSubMessage, schemaSubMessage);
|
||||
if (Object.entries(rawDecodedSubMessage).length !== 0) {
|
||||
rawDecodedMessage[`${schemaFieldName} (${subMessageType}) has missing fields`] = rawDecodedSubMessage;
|
||||
}
|
||||
}
|
||||
delete rawDecodedMessage[fieldName];
|
||||
}
|
||||
}
|
||||
return rawDecodedMessage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns wiretype information for input wiretype number
|
||||
*
|
||||
* @param {number} wireType
|
||||
* @returns {string}
|
||||
*/
|
||||
static getTypeInfo(wireType) {
|
||||
switch (wireType) {
|
||||
case 0:
|
||||
return "VarInt (e.g. int32, bool)";
|
||||
case 1:
|
||||
return "64-Bit (e.g. fixed64, double)";
|
||||
case 2:
|
||||
return "L-delim (e.g. string, message)";
|
||||
case 5:
|
||||
return "32-Bit (e.g. fixed32, float)";
|
||||
}
|
||||
}
|
||||
|
||||
// Private Class Functions
|
||||
|
|
@ -143,6 +411,11 @@ class Protobuf {
|
|||
const header = this._fieldHeader();
|
||||
const type = header.type;
|
||||
const key = header.key;
|
||||
|
||||
if (typeof(this.fieldTypes[key]) !== "object") {
|
||||
this.fieldTypes[key] = type;
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
// varint
|
||||
case 0:
|
||||
|
|
@ -152,7 +425,7 @@ class Protobuf {
|
|||
return { "key": key, "value": this._uint64() };
|
||||
// length delimited
|
||||
case 2:
|
||||
return { "key": key, "value": this._lenDelim() };
|
||||
return { "key": key, "value": this._lenDelim(key) };
|
||||
// fixed 32
|
||||
case 5:
|
||||
return { "key": key, "value": this._uint32() };
|
||||
|
|
@ -237,10 +510,10 @@ class Protobuf {
|
|||
* @returns {number}
|
||||
*/
|
||||
_uint64() {
|
||||
// Read off a Uint64
|
||||
let num = this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
|
||||
num = num * 0x100000000 + this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
|
||||
return num;
|
||||
// Read off a Uint64 with little-endian
|
||||
const lowerHalf = this.data[this.offset++] + (this.data[this.offset++] * 0x100) + (this.data[this.offset++] * 0x10000) + this.data[this.offset++] * 0x1000000;
|
||||
const upperHalf = this.data[this.offset++] + (this.data[this.offset++] * 0x100) + (this.data[this.offset++] * 0x10000) + this.data[this.offset++] * 0x1000000;
|
||||
return upperHalf * 0x100000000 + lowerHalf;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -249,7 +522,7 @@ class Protobuf {
|
|||
* @private
|
||||
* @returns {Object|string}
|
||||
*/
|
||||
_lenDelim() {
|
||||
_lenDelim(fieldNum) {
|
||||
// Read off the field length
|
||||
const length = this._varInt();
|
||||
const fieldBytes = this.data.slice(this.offset, this.offset + length);
|
||||
|
|
@ -258,6 +531,10 @@ class Protobuf {
|
|||
// Attempt to parse as a new Protobuf Object
|
||||
const pbObject = new Protobuf(fieldBytes);
|
||||
field = pbObject._parse();
|
||||
|
||||
// Set field types object
|
||||
this.fieldTypes[fieldNum] = {...this.fieldTypes[fieldNum], ...pbObject.fieldTypes};
|
||||
|
||||
} catch (err) {
|
||||
// Otherwise treat as bytes
|
||||
field = Utils.byteArrayToChars(fieldBytes);
|
||||
|
|
@ -276,7 +553,7 @@ class Protobuf {
|
|||
_uint32() {
|
||||
// Use a dataview to read off the integer
|
||||
const dataview = new DataView(new Uint8Array(this.data.slice(this.offset, this.offset + 4)).buffer);
|
||||
const value = dataview.getUint32(0);
|
||||
const value = dataview.getUint32(0, true);
|
||||
this.offset += 4;
|
||||
return value;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue