From e069ae4991b5f3f7dd028d17a3023f2f37442a3e Mon Sep 17 00:00:00 2001 From: Vimal-Raghubir Date: Sun, 25 Mar 2018 19:49:17 -0400 Subject: [PATCH 1/8] Added Parsing to and from CSV --- src/core/config/Categories.js | 2 + src/core/config/OperationConfig.js | 34 +++++++- src/core/config/modules/CSVParser.js | 21 +++++ src/core/config/modules/OpModules.js | 4 +- src/core/operations/CSVParser.js | 48 +++++++++++ test/tests/operations/CSVParser.js | 123 +++++++++++++++++++++++++++ 6 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 src/core/config/modules/CSVParser.js create mode 100644 src/core/operations/CSVParser.js create mode 100644 test/tests/operations/CSVParser.js diff --git a/src/core/config/Categories.js b/src/core/config/Categories.js index 6f04267f..a36d52aa 100755 --- a/src/core/config/Categories.js +++ b/src/core/config/Categories.js @@ -331,6 +331,8 @@ const Categories = [ "Extract EXIF", "Numberwang", "XKCD Random Number", + "Parse CSV to string", + "Parse string to CSV" ] }, { diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index 7eed4876..760380b9 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -38,6 +38,7 @@ import StrUtils from "../operations/StrUtils.js"; import Tidy from "../operations/Tidy.js"; import Unicode from "../operations/Unicode.js"; import URL_ from "../operations/URL.js"; +import CSVParser from "../operations/CSVParser.js"; /** @@ -4018,7 +4019,38 @@ const OperationConfig = { inputType: "string", outputType: "number", args: [] - } + }, + "CSV To String": { + module: "CSVParser", + description: "Function used to parse CSV string to a regular string.", + inputType: "string", + outputType: "string", + args: [ + { + name: "delimeter", + type: "string", + value: "," + }, + { + name: "quotes", + type: "string", + value: '"' + } + ] + }, + "String to CSV": { + module: "CSVParser", + description: "Function used to parse a string array to a CSV string.", + inputType: "string", + outputType: "string", + args: [ + { + name: "delimeter", + type: "string", + value: "," + } + ] + } }; diff --git a/src/core/config/modules/CSVParser.js b/src/core/config/modules/CSVParser.js new file mode 100644 index 00000000..a378be27 --- /dev/null +++ b/src/core/config/modules/CSVParser.js @@ -0,0 +1,21 @@ +import CSVParser from "../../operations/CSVParser.js"; + + +/** + * CSVParser module. + * + * Libraries: + * - csv-string + * + * @author VimalRaghubir [vraghubir0418@gmail.com] + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + */ +let OpModules = typeof self === "undefined" ? {} : self.OpModules || {}; + +OpModules.CSVParser = { + "Parse from CSV": CSVParser.csvToString, + "Parse to CSV": CSVParser.stringToCSV, +}; + +export default OpModules; \ No newline at end of file diff --git a/src/core/config/modules/OpModules.js b/src/core/config/modules/OpModules.js index 9a5e3ff5..c9d6d227 100644 --- a/src/core/config/modules/OpModules.js +++ b/src/core/config/modules/OpModules.js @@ -21,6 +21,7 @@ import PublicKeyModule from "./PublicKey.js"; import RegexModule from "./Regex.js"; import ShellcodeModule from "./Shellcode.js"; import URLModule from "./URL.js"; +import CSVModule from "./CSVParser.js" Object.assign( OpModules, @@ -37,7 +38,8 @@ Object.assign( PublicKeyModule, RegexModule, ShellcodeModule, - URLModule + URLModule, + CSVModule ); export default OpModules; diff --git a/src/core/operations/CSVParser.js b/src/core/operations/CSVParser.js new file mode 100644 index 00000000..bad0780c --- /dev/null +++ b/src/core/operations/CSVParser.js @@ -0,0 +1,48 @@ +import { CSV } from 'csv-string'; + +/** +* @author VimalRaghubir [vraghubir0418@gmail.com] +* @copyright Crown Copyright 2016 +* @license Apache-2.0 +* @namespace +*/ + + +const CSVParser = { + /** + * Parse from CSV + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + csvToString: function(input, args) { + var array = ""; + if (input) { + var detectedDelimeter = CSV.detect(input); + if (detectedDelimeter != args[0]) { + args[0] = detectedDelimeter; + } + array = CSV.parse(input, args[0], args[1]); + } else { + array = "The passed in data is not a csv string. Please pass in a csv string."; + } + return array; + }, + /** + * Parse to CSV + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + stringToCSV: function(input, args) { + var string = ""; + if (input) { + string = CSV.stringify(input, args[0]); + } else { + string = "The passed in data is not a string that can be converted to a CSV."; + } + return string; + } +} + +export default CSVParser; \ No newline at end of file diff --git a/test/tests/operations/CSVParser.js b/test/tests/operations/CSVParser.js new file mode 100644 index 00000000..7401d3bd --- /dev/null +++ b/test/tests/operations/CSVParser.js @@ -0,0 +1,123 @@ +/** + * CSVParser tests. + * + * @author Vimal Raghubir + * + * @copyright Crown Copyright 2017 + * @license Apache-2.0 + */ + +import TestRegister from "../../TestRegister.js"; + +TestRegister.addTests([ + { + name: "Testing hello parsed to CSV", + input: "hello", + expectedOutput: "hello", + recipeConfig: [ + { + op: "String to CSV", + args: [','], + }, + ], + }, + { + name: "Testing hello world parsed to CSV", + input: "['hello', 'world']", + expectedOutput: "hello;world", + recipeConfig: [ + { + op: "String to CSV", + args: [';'], + }, + ], + }, + { + name: "Testing false parsed to CSV", + input: false, + expectedOutput: "The passed in data is not a string that can be converted to a CSV.", + recipeConfig: [ + { + op: "String to CSV", + args: [','], + }, + ], + }, + { + name: "Testing ||| parsed to CSV", + input: "|||", + expectedOutput: "|""|""|""|""|", + recipeConfig: [ + { + op: "String to CSV", + args: ['|'], + }, + ], + }, + { + name: "Testing 0 parsed to CSV", + input: 0, + expectedOutput: "The passed in data is not a string that can be converted to a CSV.", + recipeConfig: [ + { + op: "String to CSV", + args: [','], + }, + ], + }, + { + name: "Testing 1,2,3,\n1,2, parsed to String", + input: "1,2,3,\n,1,2,", + expectedOutput: "[["1","2","3"],["1","2"]]", + recipeConfig: [ + { + op: "CSV to String", + args: [','], + }, + ], + }, + { + name: "Testing \n\n\n parsed to String", + input: "\n\n\n", + expectedOutput: "[[""],[""],[""]]", + recipeConfig: [ + { + op: "CSV to String", + args: [','], + }, + ], + }, + { + name: "Testing 1,2,3,4,5 parsed to String", + input: "1,2,3,4,5", + expectedOutput: "[['1,'2','3','4','5']]", + recipeConfig: [ + { + op: "CSV to String", + args: ['|'], + }, + ], + }, + { + name: "Testing 0 parsed to CSV", + input: 0, + expectedOutput: "The passed in data is not a string that can be converted to a CSV.", + recipeConfig: [ + { + op: "CSV to String", + args: [','], + }, + ], + }, + { + name: "Testing false parsed to CSV", + input: false, + expectedOutput: "The passed in data is not a string that can be converted to a CSV.", + recipeConfig: [ + { + op: "CSV to String", + args: [','], + }, + ], + }, +]); \ No newline at end of file From 5bab9b88e8ae901df459482c98b6a8332a91060c Mon Sep 17 00:00:00 2001 From: Vimal-Raghubir Date: Sun, 25 Mar 2018 20:07:57 -0400 Subject: [PATCH 2/8] Fixed travis checks --- src/core/config/Categories.js | 4 +- src/core/config/OperationConfig.js | 62 ++++++++++++++-------------- src/core/config/modules/CSVParser.js | 2 +- src/core/config/modules/OpModules.js | 4 +- src/core/operations/CSVParser.js | 52 +++++++++++------------ 5 files changed, 62 insertions(+), 62 deletions(-) diff --git a/src/core/config/Categories.js b/src/core/config/Categories.js index a36d52aa..2328659f 100755 --- a/src/core/config/Categories.js +++ b/src/core/config/Categories.js @@ -331,8 +331,8 @@ const Categories = [ "Extract EXIF", "Numberwang", "XKCD Random Number", - "Parse CSV to string", - "Parse string to CSV" + "Parse CSV to string", + "Parse string to CSV" ] }, { diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index 760380b9..cc5a2427 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -4020,37 +4020,37 @@ const OperationConfig = { outputType: "number", args: [] }, - "CSV To String": { - module: "CSVParser", - description: "Function used to parse CSV string to a regular string.", - inputType: "string", - outputType: "string", - args: [ - { - name: "delimeter", - type: "string", - value: "," - }, - { - name: "quotes", - type: "string", - value: '"' - } - ] - }, - "String to CSV": { - module: "CSVParser", - description: "Function used to parse a string array to a CSV string.", - inputType: "string", - outputType: "string", - args: [ - { - name: "delimeter", - type: "string", - value: "," - } - ] - } + "CSV To String": { + module: "CSVParser", + description: "Function used to parse CSV string to a regular string.", + inputType: "string", + outputType: "string", + args: [ + { + name: "delimeter", + type: "string", + value: "," + }, + { + name: "quotes", + type: "string", + value: '"' + } + ] + }, + "String to CSV": { + module: "CSVParser", + description: "Function used to parse a string array to a CSV string.", + inputType: "string", + outputType: "string", + args: [ + { + name: "delimeter", + type: "string", + value: "," + } + ] + } }; diff --git a/src/core/config/modules/CSVParser.js b/src/core/config/modules/CSVParser.js index a378be27..4ca120ae 100644 --- a/src/core/config/modules/CSVParser.js +++ b/src/core/config/modules/CSVParser.js @@ -18,4 +18,4 @@ OpModules.CSVParser = { "Parse to CSV": CSVParser.stringToCSV, }; -export default OpModules; \ No newline at end of file +export default OpModules; diff --git a/src/core/config/modules/OpModules.js b/src/core/config/modules/OpModules.js index c9d6d227..f8e45506 100644 --- a/src/core/config/modules/OpModules.js +++ b/src/core/config/modules/OpModules.js @@ -21,7 +21,7 @@ import PublicKeyModule from "./PublicKey.js"; import RegexModule from "./Regex.js"; import ShellcodeModule from "./Shellcode.js"; import URLModule from "./URL.js"; -import CSVModule from "./CSVParser.js" +import CSVModule from "./CSVParser.js"; Object.assign( OpModules, @@ -39,7 +39,7 @@ Object.assign( RegexModule, ShellcodeModule, URLModule, - CSVModule + CSVModule ); export default OpModules; diff --git a/src/core/operations/CSVParser.js b/src/core/operations/CSVParser.js index bad0780c..2684896e 100644 --- a/src/core/operations/CSVParser.js +++ b/src/core/operations/CSVParser.js @@ -1,4 +1,4 @@ -import { CSV } from 'csv-string'; +import { CSV } from "csv-string"; /** * @author VimalRaghubir [vraghubir0418@gmail.com] @@ -15,34 +15,34 @@ const CSVParser = { * @param {Object[]} args * @returns {string} */ - csvToString: function(input, args) { - var array = ""; - if (input) { - var detectedDelimeter = CSV.detect(input); - if (detectedDelimeter != args[0]) { - args[0] = detectedDelimeter; - } - array = CSV.parse(input, args[0], args[1]); - } else { - array = "The passed in data is not a csv string. Please pass in a csv string."; - } - return array; - }, - /** + csvToString: function(input, args) { + let array = ""; + if (input) { + let detectedDelimeter = CSV.detect(input); + if (detectedDelimeter !== args[0]) { + args[0] = detectedDelimeter; + } + array = CSV.parse(input, args[0], args[1]); + } else { + array = "The passed in data is not a csv string. Please pass in a csv string."; + } + return array; + }, + /** * Parse to CSV * @param {string} input * @param {Object[]} args * @returns {string} */ - stringToCSV: function(input, args) { - var string = ""; - if (input) { - string = CSV.stringify(input, args[0]); - } else { - string = "The passed in data is not a string that can be converted to a CSV."; - } - return string; - } -} + stringToCSV: function(input, args) { + let string = ""; + if (input) { + string = CSV.stringify(input, args[0]); + } else { + string = "The passed in data is not a string that can be converted to a CSV."; + } + return string; + } +}; -export default CSVParser; \ No newline at end of file +export default CSVParser; From e8a2040dd62dbd6b1fee4df30064ba5017a34259 Mon Sep 17 00:00:00 2001 From: Vimal-Raghubir Date: Sun, 25 Mar 2018 20:17:05 -0400 Subject: [PATCH 3/8] Fixed travis second error --- package-lock.json | 5 +++++ package.json | 1 + src/core/operations/CSVParser.js | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 70628a32..44f4a95f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2419,6 +2419,11 @@ "cssom": "0.3.2" } }, + "csv-string": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/csv-string/-/csv-string-3.1.2.tgz", + "integrity": "sha512-JzjYuRSU/5y6H9BqwaJTNsF8dYlFql14n1GIOgu2QHFCoU20U2DaeMShyIZuvU7WS0n7u+AMhgUOuqRZ9eFhog==" + }, "ctph.js": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/ctph.js/-/ctph.js-0.0.5.tgz", diff --git a/package.json b/package.json index b9ef681a..04a9391c 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "bootstrap-switch": "^3.3.4", "crypto-api": "^0.8.0", "crypto-js": "^3.1.9-1", + "csv-string": "^3.1.2", "ctph.js": "0.0.5", "diff": "^3.4.0", "escodegen": "^1.9.1", diff --git a/src/core/operations/CSVParser.js b/src/core/operations/CSVParser.js index 2684896e..0ffb9303 100644 --- a/src/core/operations/CSVParser.js +++ b/src/core/operations/CSVParser.js @@ -22,7 +22,7 @@ const CSVParser = { if (detectedDelimeter !== args[0]) { args[0] = detectedDelimeter; } - array = CSV.parse(input, args[0], args[1]); + array = CSV.parse(input, args[0], args[1]); } else { array = "The passed in data is not a csv string. Please pass in a csv string."; } @@ -42,7 +42,7 @@ const CSVParser = { string = "The passed in data is not a string that can be converted to a CSV."; } return string; - } + } }; export default CSVParser; From 20d866bcd109445239e8e4c0be97724ed1c132dc Mon Sep 17 00:00:00 2001 From: Vimal-Raghubir Date: Sun, 25 Mar 2018 20:23:06 -0400 Subject: [PATCH 4/8] Removed unused variable --- src/core/config/OperationConfig.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index cc5a2427..95cad160 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -38,7 +38,6 @@ import StrUtils from "../operations/StrUtils.js"; import Tidy from "../operations/Tidy.js"; import Unicode from "../operations/Unicode.js"; import URL_ from "../operations/URL.js"; -import CSVParser from "../operations/CSVParser.js"; /** From 640d60d6448a66a889e978b9fa52e6936ed032bb Mon Sep 17 00:00:00 2001 From: Vimal-Raghubir Date: Sun, 25 Mar 2018 20:28:15 -0400 Subject: [PATCH 5/8] Fixed parsing error --- test/tests/operations/CSVParser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/operations/CSVParser.js b/test/tests/operations/CSVParser.js index 7401d3bd..f5b0a2b5 100644 --- a/test/tests/operations/CSVParser.js +++ b/test/tests/operations/CSVParser.js @@ -46,7 +46,7 @@ TestRegister.addTests([ { name: "Testing ||| parsed to CSV", input: "|||", - expectedOutput: "|""|""|""|""|", + expectedOutput: "|||||", recipeConfig: [ { op: "String to CSV", From 3ee09deb90a9071c90ec105f72939379eb35d2b4 Mon Sep 17 00:00:00 2001 From: Vimal-Raghubir Date: Sun, 25 Mar 2018 20:34:27 -0400 Subject: [PATCH 6/8] Fixed missing identifier --- test/tests/operations/CSVParser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/operations/CSVParser.js b/test/tests/operations/CSVParser.js index f5b0a2b5..9a3871cd 100644 --- a/test/tests/operations/CSVParser.js +++ b/test/tests/operations/CSVParser.js @@ -68,7 +68,7 @@ TestRegister.addTests([ { name: "Testing 1,2,3,\n1,2, parsed to String", input: "1,2,3,\n,1,2,", - expectedOutput: "[["1","2","3"],["1","2"]]", + expectedOutput: "[[1,2,3],[1,2]]", recipeConfig: [ { op: "CSV to String", From ad47b42a6c8455fa0da2b48aedeadd76095350f6 Mon Sep 17 00:00:00 2001 From: Vimal-Raghubir Date: Sun, 25 Mar 2018 20:39:00 -0400 Subject: [PATCH 7/8] Fixed testing --- test/tests/operations/CSVParser.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests/operations/CSVParser.js b/test/tests/operations/CSVParser.js index 9a3871cd..78685bf1 100644 --- a/test/tests/operations/CSVParser.js +++ b/test/tests/operations/CSVParser.js @@ -79,7 +79,7 @@ TestRegister.addTests([ { name: "Testing \n\n\n parsed to String", input: "\n\n\n", - expectedOutput: "[[""],[""],[""]]", + expectedOutput: "[[],[],[]]", recipeConfig: [ { op: "CSV to String", @@ -90,7 +90,7 @@ TestRegister.addTests([ { name: "Testing 1,2,3,4,5 parsed to String", input: "1,2,3,4,5", - expectedOutput: "[['1,'2','3','4','5']]", + expectedOutput: "[[1,2,3,4,5]]", recipeConfig: [ { op: "CSV to String", From 2a2d8e9e67f8679bf97aa206793aa5e9a63f773f Mon Sep 17 00:00:00 2001 From: Vimal-Raghubir Date: Sun, 25 Mar 2018 20:44:41 -0400 Subject: [PATCH 8/8] Fixed formatting --- test/tests/operations/CSVParser.js | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/tests/operations/CSVParser.js b/test/tests/operations/CSVParser.js index 78685bf1..ddf7ef77 100644 --- a/test/tests/operations/CSVParser.js +++ b/test/tests/operations/CSVParser.js @@ -17,107 +17,107 @@ TestRegister.addTests([ recipeConfig: [ { op: "String to CSV", - args: [','], + args: [","], }, ], }, - { + { name: "Testing hello world parsed to CSV", input: "['hello', 'world']", expectedOutput: "hello;world", recipeConfig: [ { op: "String to CSV", - args: [';'], + args: [";"], }, ], }, - { + { name: "Testing false parsed to CSV", input: false, expectedOutput: "The passed in data is not a string that can be converted to a CSV.", recipeConfig: [ { op: "String to CSV", - args: [','], + args: [","], }, ], }, - { + { name: "Testing ||| parsed to CSV", input: "|||", expectedOutput: "|||||", recipeConfig: [ { op: "String to CSV", - args: ['|'], + args: ["|"], }, ], }, - { + { name: "Testing 0 parsed to CSV", input: 0, expectedOutput: "The passed in data is not a string that can be converted to a CSV.", recipeConfig: [ { op: "String to CSV", - args: [','], + args: [","], }, ], }, - { + { name: "Testing 1,2,3,\n1,2, parsed to String", input: "1,2,3,\n,1,2,", expectedOutput: "[[1,2,3],[1,2]]", recipeConfig: [ { op: "CSV to String", - args: [','], + args: [","], }, ], }, - { + { name: "Testing \n\n\n parsed to String", input: "\n\n\n", expectedOutput: "[[],[],[]]", recipeConfig: [ { op: "CSV to String", - args: [','], + args: [","], }, ], }, - { + { name: "Testing 1,2,3,4,5 parsed to String", input: "1,2,3,4,5", expectedOutput: "[[1,2,3,4,5]]", recipeConfig: [ { op: "CSV to String", - args: ['|'], + args: ["|"], }, ], }, - { + { name: "Testing 0 parsed to CSV", input: 0, expectedOutput: "The passed in data is not a string that can be converted to a CSV.", recipeConfig: [ { op: "CSV to String", - args: [','], + args: [","], }, ], }, - { + { name: "Testing false parsed to CSV", input: false, expectedOutput: "The passed in data is not a string that can be converted to a CSV.", recipeConfig: [ { op: "CSV to String", - args: [','], + args: [","], }, ], }, -]); \ No newline at end of file +]);