From 53bf52c989639826d51edb11a14dd8d4c3e9430e Mon Sep 17 00:00:00 2001 From: sevzero Date: Thu, 24 May 2018 13:09:04 +0000 Subject: [PATCH 1/4] Added decoder for chunked HTTP encoding This decoder will join up a HTTP response sent using chunked transfer encoding, raised in issue #168. This is useful when attempting to extract files or gzipped responses sent using chunked transfer encoding, particularly when combined with the gunzip operation. --- src/core/config/Categories.json | 1 + src/core/config/scripts/portOperation.mjs | 7 ++++ src/core/operations/DechunkHTTP.mjs | 50 +++++++++++++++++++++++ src/core/operations/legacy/HTTP.js | 21 ++++++++++ 4 files changed, 79 insertions(+) create mode 100644 src/core/operations/DechunkHTTP.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 9078e250..9bd2977f 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -137,6 +137,7 @@ "ops": [ "HTTP request", "Strip HTTP headers", + "Dechunk HTTP response", "Parse User Agent", "Parse IP range", "Parse IPv6 address", diff --git a/src/core/config/scripts/portOperation.mjs b/src/core/config/scripts/portOperation.mjs index cac7b1eb..8ac322ec 100644 --- a/src/core/config/scripts/portOperation.mjs +++ b/src/core/config/scripts/portOperation.mjs @@ -2239,6 +2239,13 @@ const OP_CONFIG = { outputType: "string", args: [] }, + "Dechunk HTTP response": { + module: "HTTP", + description: "Parses a HTTP response transferred using transfer-encoding:chunked", + inputType: "string", + outputType: "string", + args: [] + }, "Parse User Agent": { module: "HTTP", description: "Attempts to identify and categorise information contained in a user-agent string.", diff --git a/src/core/operations/DechunkHTTP.mjs b/src/core/operations/DechunkHTTP.mjs new file mode 100644 index 00000000..1a428e3a --- /dev/null +++ b/src/core/operations/DechunkHTTP.mjs @@ -0,0 +1,50 @@ +/** + * @author sevzero [sevzero@protonmail.com] + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; + +/** + * Dechunk HTTP response operation + */ +class DechunkHTTP extends Operation { + + /** + * DechunkHTTP constructor + */ + constructor() { + super(); + + this.name = "Dechunk HTTP response"; + this.module = "Default"; + this.description = "Parses a HTTP response transferred using transfer-encoding:chunked"; + this.inputType = "string"; + this.outputType = "string"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + var chunks = []; + var chunkSizeEnd = input.indexOf("\n") + 1; + var lineEndings = input.charAt(chunkSizeEnd - 2) == "\r" ? "\r\n" : "\n"; + var lineEndingsLength = lineEndings.length; + var chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); + while (!isNaN(chunkSize)) { + chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); + input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength); + chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength; + chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); + } + return chunks.join('') + input; + } + +} + +export default DechunkHTTP; diff --git a/src/core/operations/legacy/HTTP.js b/src/core/operations/legacy/HTTP.js index dcb1cc50..b41e118c 100755 --- a/src/core/operations/legacy/HTTP.js +++ b/src/core/operations/legacy/HTTP.js @@ -37,6 +37,27 @@ const HTTP = { return (headerEnd < 2) ? input : input.slice(headerEnd, input.length); }, + /** + * Dechunk response operation + * + * @param {string} input + * @param {Object[]} args} + * @returns {string} + */ + runDechunk: function(input, args) { + var chunks = []; + var chunkSizeEnd = input.indexOf("\n") + 1; + var lineEndings = input.charAt(chunkSizeEnd - 2) == "\r" ? "\r\n" : "\n"; + var lineEndingsLength = lineEndings.length; + var chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); + while (!isNaN(chunkSize)) { + chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); + input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength); + chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength; + chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); + } + return chunks.join('') + input; + }, /** * Parse User Agent operation. From 83c145c2ac60849b96e2b593b16eee8c09212cbe Mon Sep 17 00:00:00 2001 From: sevzero Date: Thu, 24 May 2018 13:17:46 +0000 Subject: [PATCH 2/4] Reduced variable scopes --- src/core/operations/DechunkHTTP.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/operations/DechunkHTTP.mjs b/src/core/operations/DechunkHTTP.mjs index 1a428e3a..310d595a 100644 --- a/src/core/operations/DechunkHTTP.mjs +++ b/src/core/operations/DechunkHTTP.mjs @@ -31,18 +31,18 @@ class DechunkHTTP extends Operation { * @returns {string} */ run(input, args) { - var chunks = []; - var chunkSizeEnd = input.indexOf("\n") + 1; - var lineEndings = input.charAt(chunkSizeEnd - 2) == "\r" ? "\r\n" : "\n"; - var lineEndingsLength = lineEndings.length; - var chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); + let chunks = []; + let chunkSizeEnd = input.indexOf("\n") + 1; + let lineEndings = input.charAt(chunkSizeEnd - 2) === "\r" ? "\r\n" : "\n"; + let lineEndingsLength = lineEndings.length; + let chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); while (!isNaN(chunkSize)) { chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength); chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength; chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); } - return chunks.join('') + input; + return chunks.join("") + input; } } From 67bd2605c05fcf86ab70fad1f73701bcf168a27f Mon Sep 17 00:00:00 2001 From: sevzero Date: Thu, 24 May 2018 13:21:50 +0000 Subject: [PATCH 3/4] Updated variable scope in legacy HTTP.js file --- src/core/operations/legacy/HTTP.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/operations/legacy/HTTP.js b/src/core/operations/legacy/HTTP.js index b41e118c..e0d40528 100755 --- a/src/core/operations/legacy/HTTP.js +++ b/src/core/operations/legacy/HTTP.js @@ -45,18 +45,18 @@ const HTTP = { * @returns {string} */ runDechunk: function(input, args) { - var chunks = []; - var chunkSizeEnd = input.indexOf("\n") + 1; - var lineEndings = input.charAt(chunkSizeEnd - 2) == "\r" ? "\r\n" : "\n"; - var lineEndingsLength = lineEndings.length; - var chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); + let chunks = []; + let chunkSizeEnd = input.indexOf("\n") + 1; + let lineEndings = input.charAt(chunkSizeEnd - 2) === "\r" ? "\r\n" : "\n"; + let lineEndingsLength = lineEndings.length; + let chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); while (!isNaN(chunkSize)) { chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength); chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength; chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); } - return chunks.join('') + input; + return chunks.join("") + input; }, /** From ed84614389e65d4755053a27588cbf984e720b3b Mon Sep 17 00:00:00 2001 From: sevzero Date: Thu, 24 May 2018 13:39:14 +0000 Subject: [PATCH 4/4] Fixed issues raised by Travis --- src/core/operations/DechunkHTTP.mjs | 8 ++++---- src/core/operations/legacy/HTTP.js | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/core/operations/DechunkHTTP.mjs b/src/core/operations/DechunkHTTP.mjs index 310d595a..b63bd590 100644 --- a/src/core/operations/DechunkHTTP.mjs +++ b/src/core/operations/DechunkHTTP.mjs @@ -31,13 +31,13 @@ class DechunkHTTP extends Operation { * @returns {string} */ run(input, args) { - let chunks = []; + const chunks = []; let chunkSizeEnd = input.indexOf("\n") + 1; - let lineEndings = input.charAt(chunkSizeEnd - 2) === "\r" ? "\r\n" : "\n"; - let lineEndingsLength = lineEndings.length; + const lineEndings = input.charAt(chunkSizeEnd - 2) === "\r" ? "\r\n" : "\n"; + const lineEndingsLength = lineEndings.length; let chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); while (!isNaN(chunkSize)) { - chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); + chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength); chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength; chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); diff --git a/src/core/operations/legacy/HTTP.js b/src/core/operations/legacy/HTTP.js index e0d40528..70e3f8a1 100755 --- a/src/core/operations/legacy/HTTP.js +++ b/src/core/operations/legacy/HTTP.js @@ -45,13 +45,13 @@ const HTTP = { * @returns {string} */ runDechunk: function(input, args) { - let chunks = []; + const chunks = []; let chunkSizeEnd = input.indexOf("\n") + 1; - let lineEndings = input.charAt(chunkSizeEnd - 2) === "\r" ? "\r\n" : "\n"; - let lineEndingsLength = lineEndings.length; + const lineEndings = input.charAt(chunkSizeEnd - 2) === "\r" ? "\r\n" : "\n"; + const lineEndingsLength = lineEndings.length; let chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16); while (!isNaN(chunkSize)) { - chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); + chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength); chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength; chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16);