From 37a86b58e5df8d24247efdd1e367639db39a1327 Mon Sep 17 00:00:00 2001
From: Russell <russell.hole@gmail.com>
Date: Wed, 23 May 2018 12:51:37 +0000
Subject: [PATCH] Added decoder for chunked HTTP encoding

This decoder will join up a HTTP response sent using chunked transfer encoding, raised in issue #168.

This is useful when attempting to extract files or gzipped responses sent using chunked transfer encoding, particularly when combined with the gunzip operation.
---
 src/core/config/Categories.js      |  1 +
 src/core/config/OperationConfig.js |  7 +++++++
 src/core/config/modules/HTTP.js    |  1 +
 src/core/operations/HTTP.js        | 22 ++++++++++++++++++++++
 4 files changed, 31 insertions(+)

diff --git a/src/core/config/Categories.js b/src/core/config/Categories.js
index 9d531f8d..ee7de5bd 100755
--- a/src/core/config/Categories.js
+++ b/src/core/config/Categories.js
@@ -150,6 +150,7 @@ const Categories = [
         ops: [
             "HTTP request",
             "Strip HTTP headers",
+            "Dechunk HTTP response",
             "Parse User Agent",
             "Parse IP range",
             "Parse IPv6 address",
diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js
index 500a8803..0149c91d 100644
--- a/src/core/config/OperationConfig.js
+++ b/src/core/config/OperationConfig.js
@@ -1846,6 +1846,13 @@ const OperationConfig = {
         outputType: "string",
         args: []
     },
+    "Dechunk HTTP response": {
+        module: "HTTP",
+        description: "Parses a HTTP response transferred using transfer-encoding:chunked",
+        inputType: "string",
+        outputType: "string",
+        args: []
+    },
     "Parse User Agent": {
         module: "HTTP",
         description: "Attempts to identify and categorise information contained in a user-agent string.",
diff --git a/src/core/config/modules/HTTP.js b/src/core/config/modules/HTTP.js
index 8972a3e6..d8700a60 100644
--- a/src/core/config/modules/HTTP.js
+++ b/src/core/config/modules/HTTP.js
@@ -16,6 +16,7 @@ let OpModules = typeof self === "undefined" ? {} : self.OpModules || {};
 OpModules.HTTP = {
     "HTTP request":       HTTP.runHTTPRequest,
     "Strip HTTP headers": HTTP.runStripHeaders,
+    "Dechunk HTTP response": HTTP.runDechunk,
     "Parse User Agent":   HTTP.runParseUserAgent,
 };
 
diff --git a/src/core/operations/HTTP.js b/src/core/operations/HTTP.js
index dcb1cc50..0fae21de 100755
--- a/src/core/operations/HTTP.js
+++ b/src/core/operations/HTTP.js
@@ -37,6 +37,28 @@ const HTTP = {
         return (headerEnd < 2) ? input : input.slice(headerEnd, input.length);
     },
 
+    /**
+     * Dechunk response operation
+     *
+     * @param {string} input
+     * @param {Object[]} args}
+     * @returns {string}
+    */
+    runDechunk: function(input, args) {
+        var chunks = [];
+        var chunkSizeEnd = input.indexOf("\n") + 1;
+        var lineEndings = input.charAt(chunkSizeEnd - 2) == "\r" ? "\r\n" : "\n";
+        var lineEndingsLength = lineEndings.length;
+        var chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16);
+        while (!isNaN(chunkSize)) {
+            chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); 
+            input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength);
+            chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength;
+            chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16);
+        }
+        return chunks.join('') + input;
+    },
+
 
     /**
      * Parse User Agent operation.