WIP HAD to move NodeDish out - NONE of it is async!

This commit is contained in:
d98762625 2019-02-15 15:20:05 +00:00
parent aafde8986d
commit 04b7f2fa8c
16 changed files with 442 additions and 411 deletions

View file

@ -74,6 +74,7 @@ class Dish {
case "list<file>":
return Dish.LIST_FILE;
default:
console.log(typeStr);
throw new DishError("Invalid data type string. No matching enum.");
}
}
@ -383,7 +384,6 @@ class Dish {
return newDish;
}
}

View file

@ -9,6 +9,7 @@ import {fromBase64, toBase64} from "./lib/Base64";
import {fromHex} from "./lib/Hex";
import {fromDecimal} from "./lib/Decimal";
import {fromBinary} from "./lib/Binary";
import { fstat } from "fs";
/**
@ -919,7 +920,7 @@ class Utils {
/**
* Reads a File and returns the data as a Uint8Array.
*
* @param {File} file
* @param {File | for node: array|arrayBuffer|buffer|string} file
* @returns {Uint8Array}
*
* @example
@ -927,33 +928,49 @@ class Utils {
* await Utils.readFile(new File(["hello"], "test"))
*/
static readFile(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
const data = new Uint8Array(file.size);
let offset = 0;
const CHUNK_SIZE = 10485760; // 10MiB
if (Utils.isBrowser()) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
const data = new Uint8Array(file.size);
let offset = 0;
const CHUNK_SIZE = 10485760; // 10MiB
const seek = function() {
if (offset >= file.size) {
resolve(data);
return;
}
const slice = file.slice(offset, offset + CHUNK_SIZE);
reader.readAsArrayBuffer(slice);
};
const seek = function() {
if (offset >= file.size) {
resolve(data);
return;
}
const slice = file.slice(offset, offset + CHUNK_SIZE);
reader.readAsArrayBuffer(slice);
};
reader.onload = function(e) {
data.set(new Uint8Array(reader.result), offset);
offset += CHUNK_SIZE;
seek();
};
reader.onerror = function(e) {
reject(reader.error.message);
};
reader.onload = function(e) {
data.set(new Uint8Array(reader.result), offset);
offset += CHUNK_SIZE;
seek();
};
});
reader.onerror = function(e) {
reject(reader.error.message);
};
} else if (Utils.isNode()) {
return Buffer.from(file).buffer;
}
seek();
});
throw new Error("Unkown environment!");
}
/** */
static readFileSync(file) {
if (Utils.isBrowser()) {
throw new TypeError("Browser environment cannot support readFileSync");
}
return Buffer.from(file).buffer;
}
@ -1050,6 +1067,20 @@ class Utils {
}[token];
}
/**
* Check if code is running in a browser environment
*/
static isBrowser() {
return typeof window !== "undefined" && typeof window.document !== "undefined";
}
/**
* Check if code is running in a Node environment
*/
static isNode() {
return typeof process !== "undefined" && process.versions != null && process.versions.node != null;
}
}
export default Utils;

View file

@ -132,6 +132,8 @@ class Tar extends Operation {
tarball.writeBytes(input);
tarball.writeEndBlocks();
console.log("here");
return new File([new Uint8Array(tarball.bytes)], args[0]);
}

View file

@ -131,6 +131,7 @@ class Untar extends Operation {
* @returns {html}
*/
async present(files) {
console.log("err....");
return await Utils.displayFilesAsHTML(files);
}

41
src/node/File.mjs Normal file
View file

@ -0,0 +1,41 @@
/**
* @author d98762625 [d98762625@gmail.com]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import mime from "mime";
/**
* FileShim
*
* Create a class that behaves like the File object in the Browser so that
* operations that use the File object still work.
*
* File doesn't write to disk, but it would be easy to do so with e.gfs.writeFile.
*/
class File {
/**
* Constructor
*
* @param {String|Array|ArrayBuffer|Buffer} bits - file content
* @param {String} name (optional) - file name
* @param {Object} stats (optional) - file stats e.g. lastModified
*/
constructor(data, name="", stats={}) {
this.data = Buffer.from(data);
this.name = name;
this.lastModified = stats.lastModified || Date.now();
this.type = stats.type || mime.getType(this.name);
}
/**
* size property
*/
get size() {
return this.data.length;
}
}
export default File;

186
src/node/NodeDish.mjs Normal file
View file

@ -0,0 +1,186 @@
/**
* @author d98762625 [d98762625@gmail.com]
* @copyright Crown Copyright 2018
* @license Apache-2.0
*/
import util from "util";
import Dish from "../core/Dish";
import Utils from "../core/Utils";
import DishError from "../core/errors/DishError";
import BigNumber from "bignumber.js";
/**
* Subclass of Dish where `get` and `_translate` are synchronous.
* Also define functions to improve coercion behaviour.
*/
class NodeDish extends Dish {
/**
* Create a Dish
* @param {any} inputOrDish - The dish input
* @param {String|Number} - The dish type, as enum or string
*/
constructor(inputOrDish=null, type=null) {
// Allow `fs` file input:
// Any node fs Buffers transformed to array buffer
// NOT Buffer.buff, as this makes a buffer of the whole object.
if (Buffer.isBuffer(inputOrDish)) {
inputOrDish = new Uint8Array(inputOrDish).buffer;
}
super(inputOrDish, type);
}
/**
* Returns the value of the data in the type format specified.
*
* @param {number} type - The data type of value, see Dish enums.
* @param {boolean} [notUTF8=false] - Do not treat strings as UTF8.
* @returns {*} - The value of the output data.
*/
get(type, notUTF8=false) {
if (typeof type === "string") {
type = Dish.typeEnum(type);
}
if (this.type !== type) {
this._translate(type, notUTF8);
}
return this.value;
}
/**
* alias for get
* @param args see get args
*/
to(...args) {
return this.get(...args);
}
/**
* Avoid coercion to a String primitive.
*/
toString() {
return this.get(Dish.typeEnum("string"));
}
/**
* What we want to log to the console.
*/
[util.inspect.custom](depth, options) {
return this.get(Dish.typeEnum("string"));
}
/**
* Backwards compatibility for node v6
* Log only the value to the console in node.
*/
inspect() {
return this.get(Dish.typeEnum("string"));
}
/**
* Avoid coercion to a Number primitive.
*/
valueOf() {
return this.get(Dish.typeEnum("number"));
}
/**
* Translates the data to the given type format.
*
* @param {number} toType - The data type of value, see Dish enums.
* @param {boolean} [notUTF8=false] - Do not treat strings as UTF8.
*/
_translate(toType, notUTF8=false) {
log.debug(`Translating Dish from ${Dish.enumLookup(this.type)} to ${Dish.enumLookup(toType)}`);
const byteArrayToStr = notUTF8 ? Utils.byteArrayToChars : Utils.byteArrayToUtf8;
// Convert data to intermediate byteArray type
try {
switch (this.type) {
case Dish.STRING:
this.value = this.value ? Utils.strToByteArray(this.value) : [];
break;
case Dish.NUMBER:
this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : [];
break;
case Dish.HTML:
this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : [];
break;
case Dish.ARRAY_BUFFER:
// Array.from() would be nicer here, but it's slightly slower
this.value = Array.prototype.slice.call(new Uint8Array(this.value));
break;
case Dish.BIG_NUMBER:
this.value = BigNumber.isBigNumber(this.value) ? Utils.strToByteArray(this.value.toFixed()) : [];
break;
case Dish.JSON:
this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value, null, 4)) : [];
break;
case Dish.FILE:
this.value = Utils.readFileSync(this.value);
this.value = Array.prototype.slice.call(this.value);
break;
case Dish.LIST_FILE:
this.value = this.value.map(f => Utils.readFileSync(f));
this.value = this.value.map(b => Array.prototype.slice.call(b));
this.value = [].concat.apply([], this.value);
break;
default:
break;
}
} catch (err) {
throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to byteArray: ${err}`);
}
this.type = Dish.BYTE_ARRAY;
// Convert from byteArray to toType
try {
switch (toType) {
case Dish.STRING:
case Dish.HTML:
this.value = this.value ? byteArrayToStr(this.value) : "";
this.type = Dish.STRING;
break;
case Dish.NUMBER:
this.value = this.value ? parseFloat(byteArrayToStr(this.value)) : 0;
this.type = Dish.NUMBER;
break;
case Dish.ARRAY_BUFFER:
this.value = new Uint8Array(this.value).buffer;
this.type = Dish.ARRAY_BUFFER;
break;
case Dish.BIG_NUMBER:
try {
this.value = new BigNumber(byteArrayToStr(this.value));
} catch (err) {
this.value = new BigNumber(NaN);
}
this.type = Dish.BIG_NUMBER;
break;
case Dish.JSON:
this.value = JSON.parse(byteArrayToStr(this.value));
this.type = Dish.JSON;
break;
case Dish.FILE:
this.value = new File(this.value, "unknown");
break;
case Dish.LIST_FILE:
this.value = [new File(this.value, "unknown")];
this.type = Dish.LIST_FILE;
break;
default:
break;
}
} catch (err) {
throw new DishError(`Error translating from byteArray to ${Dish.enumLookup(toType)}: ${err}`);
}
}
}
export default NodeDish;

View file

@ -9,7 +9,7 @@ import { sanitise } from "./apiUtils";
/**
* Similar to core/Recipe, Recipe controls a list of operations and
* the SyncDish the operate on. However, this Recipe is for the node
* the NodeDish the operate on. However, this Recipe is for the node
* environment.
*/
class NodeRecipe {
@ -73,17 +73,17 @@ class NodeRecipe {
/**
* Run the dish through each operation, one at a time.
* @param {SyncDish} dish
* @returns {SyncDish}
* @param {NodeDish} dish
* @returns {NodeDish}
*/
execute(dish) {
return this.opList.reduce((prev, curr) => {
async execute(dish) {
return await this.opList.reduce(async (prev, curr) => {
// CASE where opLis item is op and args
if (curr.hasOwnProperty("op") && curr.hasOwnProperty("args")) {
return curr.op(prev, curr.args);
return await curr.op(prev, curr.args);
}
// CASE opList item is just op.
return curr(prev);
return await curr(prev);
}, dish);
}
}

View file

@ -1,196 +0,0 @@
/**
* @author d98762625 [d98762625@gmail.com]
* @copyright Crown Copyright 2018
* @license Apache-2.0
*/
import util from "util";
import Utils from "../core/Utils";
import Dish from "../core/Dish";
import BigNumber from "bignumber.js";
import log from "loglevel";
/**
* Subclass of Dish where `get` and `_translate` are synchronous.
* Also define functions to improve coercion behaviour.
*/
class SyncDish extends Dish {
/**
* Create a Dish
* @param {any} inputOrDish - The dish input
* @param {String|Number} - The dish type, as enum or string
*/
constructor(inputOrDish=null, type=null) {
// Allow `fs` file input:
// Any node fs Buffers transformed to array buffer
// NOT Buffer.buff, as this makes a buffer of the whole object.
if (Buffer.isBuffer(inputOrDish)) {
inputOrDish = new Uint8Array(inputOrDish).buffer;
}
super(inputOrDish, type);
}
/**
* Apply the inputted operation to the dish.
*
* @param {WrappedOperation} operation the operation to perform
* @param {*} args - any arguments for the operation
* @returns {Dish} a new dish with the result of the operation.
*/
apply(operation, args=null) {
return operation(this.value, args);
}
/**
* Synchronously returns the value of the data in the type format specified.
*
* @param {number} type - The data type of value, see Dish enums.
* @param {boolean} [notUTF8=false] - Do not treat strings as UTF8.
* @returns {*} - The value of the output data.
*/
get(type, notUTF8=false) {
if (typeof type === "string") {
type = Dish.typeEnum(type);
}
if (this.type !== type) {
this._translate(type, notUTF8);
}
return this.value;
}
/**
* alias for get
* @param args see get args
*/
to(...args) {
return this.get(...args);
}
/**
* Avoid coercion to a String primitive.
*/
toString() {
return this.get(Dish.typeEnum("string"));
}
/**
* What we want to log to the console.
*/
[util.inspect.custom](depth, options) {
return this.get(Dish.typeEnum("string"));
}
/**
* Backwards compatibility for node v6
* Log only the value to the console in node.
*/
inspect() {
return this.get(Dish.typeEnum("string"));
}
/**
* Avoid coercion to a Number primitive.
*/
valueOf() {
return this.get(Dish.typeEnum("number"));
}
/**
* Synchronously translates the data to the given type format.
*
* @param {number} toType - The data type of value, see Dish enums.
* @param {boolean} [notUTF8=false] - Do not treat strings as UTF8.
*/
_translate(toType, notUTF8=false) {
log.debug(`Translating Dish from ${Dish.enumLookup(this.type)} to ${Dish.enumLookup(toType)}`);
const byteArrayToStr = notUTF8 ? Utils.byteArrayToChars : Utils.byteArrayToUtf8;
// Convert data to intermediate byteArray type
switch (this.type) {
case Dish.STRING:
this.value = this.value ? Utils.strToByteArray(this.value) : [];
break;
case Dish.NUMBER:
this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : [];
break;
case Dish.HTML:
this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : [];
break;
case Dish.ARRAY_BUFFER:
// Array.from() would be nicer here, but it's slightly slower
this.value = Array.prototype.slice.call(new Uint8Array(this.value));
break;
case Dish.BIG_NUMBER:
this.value = this.value instanceof BigNumber ? Utils.strToByteArray(this.value.toFixed()) : [];
break;
case Dish.JSON:
this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value)) : [];
break;
case Dish.BUFFER:
this.value = this.value instanceof Buffer ? this.value.buffer : [];
break;
// No such API in Node.js.
// case Dish.FILE:
// this.value = Utils.readFileSync(this.value);
// this.value = Array.prototype.slice.call(this.value);
// break;
// case Dish.LIST_FILE:
// this.value = this.value.map(f => Utils.readFileSync(f));
// this.value = this.value.map(b => Array.prototype.slice.call(b));
// this.value = [].concat.apply([], this.value);
// break;
default:
break;
}
this.type = Dish.BYTE_ARRAY;
// Convert from byteArray to toType
switch (toType) {
case Dish.STRING:
case Dish.HTML:
this.value = this.value ? byteArrayToStr(this.value) : "";
this.type = Dish.STRING;
break;
case Dish.NUMBER:
this.value = this.value ? parseFloat(byteArrayToStr(this.value)) : 0;
this.type = Dish.NUMBER;
break;
case Dish.ARRAY_BUFFER:
this.value = new Uint8Array(this.value).buffer;
this.type = Dish.ARRAY_BUFFER;
break;
case Dish.BIG_NUMBER:
try {
this.value = new BigNumber(byteArrayToStr(this.value));
} catch (err) {
this.value = new BigNumber(NaN);
}
this.type = Dish.BIG_NUMBER;
break;
case Dish.JSON:
this.value = JSON.parse(byteArrayToStr(this.value));
this.type = Dish.JSON;
break;
case Dish.BUFFER:
this.value = Buffer.from(new Uint8Array(this.value));
this.type = Dish.BUFFER;
break;
// No such API in Node.js.
// case Dish.FILE:
// this.value = new File(this.value, "unknown");
// break;
// case Dish.LIST_FILE:
// this.value = [new File(this.value, "unknown")];
// this.type = Dish.LIST_FILE;
// break;
default:
break;
}
}
}
export default SyncDish;

View file

@ -8,37 +8,13 @@
/*eslint no-console: ["off"] */
import SyncDish from "./SyncDish";
import NodeDish from "./NodeDish";
import NodeRecipe from "./NodeRecipe";
import OperationConfig from "../core/config/OperationConfig.json";
import { sanitise, removeSubheadingsFromArray, sentenceToCamelCase } from "./apiUtils";
import ExludedOperationError from "../core/errors/ExcludedOperationError";
/**
* Extract default arg value from operation argument
* @param {Object} arg - an arg from an operation
*/
function extractArg(arg) {
if (arg.type === "option") {
// pick default option if not already chosen
return typeof arg.value === "string" ? arg.value : arg.value[0];
}
if (arg.type === "editableOption") {
return typeof arg.value === "string" ? arg.value : arg.value[0].value;
}
if (arg.type === "toggleString") {
// ensure string and option exist when user hasn't defined
arg.string = arg.string || "";
arg.option = arg.option || arg.toggleValues[0];
return arg;
}
return arg.value;
}
/**
* transformArgs
*
@ -52,84 +28,87 @@ function extractArg(arg) {
* @param {Object[]} originalArgs - the operation-s args list
* @param {Object} newArgs - any inputted args
*/
function transformArgs(originalArgs, newArgs) {
function reconcileOpArgs(operationArgs, objectStyleArgs) {
if (Array.isArray(objectStyleArgs)) {
return objectStyleArgs;
}
// Filter out arg values that are list subheadings - they are surrounded in [].
// See Strings op for example.
const allArgs = Object.assign([], originalArgs).map((a) => {
const opArgs = Object.assign([], operationArgs).map((a) => {
if (Array.isArray(a.value)) {
a.value = removeSubheadingsFromArray(a.value);
}
return a;
});
if (newArgs) {
Object.keys(newArgs).map((key) => {
const index = allArgs.findIndex((arg) => {
// transform object style arg objects to the same shape as op's args
if (objectStyleArgs) {
Object.keys(objectStyleArgs).map((key) => {
const index = opArgs.findIndex((arg) => {
return arg.name.toLowerCase().replace(/ /g, "") ===
key.toLowerCase().replace(/ /g, "");
});
if (index > -1) {
const argument = allArgs[index];
const argument = opArgs[index];
if (argument.type === "toggleString") {
if (typeof newArgs[key] === "string") {
argument.string = newArgs[key];
if (typeof objectStyleArgs[key] === "string") {
argument.string = objectStyleArgs[key];
} else {
argument.string = newArgs[key].string;
argument.option = newArgs[key].option;
argument.string = objectStyleArgs[key].string;
argument.option = objectStyleArgs[key].option;
}
} else if (argument.type === "editableOption") {
// takes key: "option", key: {name, val: "string"}, key: {name, val: [...]}
argument.value = typeof newArgs[key] === "string" ? newArgs[key]: newArgs[key].value;
argument.value = typeof objectStyleArgs[key] === "string" ? objectStyleArgs[key]: objectStyleArgs[key].value;
} else {
argument.value = newArgs[key];
argument.value = objectStyleArgs[key];
}
}
});
}
return allArgs.map(extractArg);
return opArgs.map((arg) => {
if (arg.type === "option") {
// pick default option if not already chosen
return typeof arg.value === "string" ? arg.value : arg.value[0];
}
if (arg.type === "editableOption") {
return typeof arg.value === "string" ? arg.value : arg.value[0].value;
}
if (arg.type === "toggleString") {
// ensure string and option exist when user hasn't defined
arg.string = arg.string || "";
arg.option = arg.option || arg.toggleValues[0];
return arg;
}
return arg.value;
});
}
/**
* Ensure an input is a SyncDish object.
* Ensure an input is a NodeDish object.
* @param input
*/
function ensureIsDish(input) {
if (!input) {
return new SyncDish();
return new NodeDish();
}
if (input instanceof SyncDish) {
if (input instanceof NodeDish) {
return input;
} else {
return new SyncDish(input);
return new NodeDish(input);
}
}
/**
* prepareOp: transform args, make input the right type.
* Also convert any Buffers to ArrayBuffers.
* @param opInstance - instance of the operation
* @param input - operation input
* @param args - operation args
*/
function prepareOp(opInstance, input, args) {
const dish = ensureIsDish(input);
let transformedArgs;
// Transform object-style args to original args array
if (!Array.isArray(args)) {
transformedArgs = transformArgs(opInstance.args, args);
} else {
transformedArgs = args;
}
const transformedInput = dish.get(opInstance.inputType);
return {transformedInput, transformedArgs};
}
/**
* createArgOptions
*
@ -154,7 +133,6 @@ function createArgOptions(op) {
return result;
}
/**
* Wrap an operation to be consumed by node API.
* Checks to see if run function is async or not.
@ -169,44 +147,29 @@ export function wrap(OpClass) {
// Check to see if class's run function is async.
const opInstance = new OpClass();
const isAsync = opInstance.run.constructor.name === "AsyncFunction";
let wrapped;
/**
* Async wrapped operation run function
* @param {*} input
* @param {Object | String[]} args - either in Object or normal args array
* @returns {Promise<NodeDish>} operation's output, on a Dish.
* @throws {OperationError} if the operation throws one.
*/
const wrapped = async (input, args=null) => {
const dish = ensureIsDish(input);
// If async, wrap must be async.
if (isAsync) {
/**
* Async wrapped operation run function
* @param {*} input
* @param {Object | String[]} args - either in Object or normal args array
* @returns {Promise<SyncDish>} operation's output, on a Dish.
* @throws {OperationError} if the operation throws one.
*/
wrapped = async (input, args=null) => {
const {transformedInput, transformedArgs} = prepareOp(opInstance, input, args);
const result = await opInstance.run(transformedInput, transformedArgs);
return new SyncDish({
value: result,
type: opInstance.outputType
});
};
} else {
/**
* wrapped operation run function
* @param {*} input
* @param {Object | String[]} args - either in Object or normal args array
* @returns {SyncDish} operation's output, on a Dish.
* @throws {OperationError} if the operation throws one.
*/
wrapped = (input, args=null) => {
const {transformedInput, transformedArgs} = prepareOp(opInstance, input, args);
const result = opInstance.run(transformedInput, transformedArgs);
return new SyncDish({
value: result,
type: opInstance.outputType
});
};
}
// Transform object-style args to original args array
const transformedArgs = reconcileOpArgs(opInstance.args, args);
// ensure the input is the correct type
const transformedInput = await dish.get(opInstance.inputType);
const result = await opInstance.run(transformedInput, transformedArgs);
return new NodeDish({
value: result,
type: opInstance.outputType
});
};
// used in chef.help
wrapped.opName = OpClass.name;
@ -288,7 +251,7 @@ export function bake(operations){
* @param {*} input - some input for a recipe.
* @param {String | Function | String[] | Function[] | [String | Function]} recipeConfig -
* An operation, operation name, or an array of either.
* @returns {SyncDish} of the result
* @returns {NodeDish} of the result
* @throws {TypeError} if invalid recipe given.
*/
return function(input, recipeConfig) {

View file

@ -14,8 +14,8 @@ export default [
"Comment",
// Exclude file ops until HTML5 File Object can be mimicked
"Tar",
"Untar",
// "Tar",
// "Untar",
"Unzip",
"Zip",

View file

@ -39,7 +39,7 @@ let code = `/**
import "babel-polyfill";
import SyncDish from "./SyncDish";
import NodeDish from "./NodeDish";
import { wrap, help, bake, explainExludedFunction } from "./api";
import {
// import as core_ to avoid name clashes after wrap.
@ -87,7 +87,7 @@ code += ` };
const chef = generateChef();
// Add some additional features to chef object.
chef.help = help;
chef.Dish = SyncDish;
chef.Dish = NodeDish;
// Define consts here so we can add to top-level export - wont allow
// export of chef property.
@ -121,7 +121,7 @@ Object.keys(operations).forEach((op) => {
code += ` ${decapitalise(op)},\n`;
});
code += " SyncDish as Dish,\n";
code += " NodeDish as Dish,\n";
code += " prebaked as bake,\n";
code += " help,\n";
code += "};\n";

View file

@ -9,6 +9,7 @@
import chef from "./index";
import repl from "repl";
import File from "./File";
import "babel-polyfill";
/*eslint no-console: ["off"] */
@ -26,6 +27,8 @@ const replServer = repl.start({
prompt: "chef > ",
});
global.File = File;
Object.keys(chef).forEach((key) => {
if (key !== "operations") {
replServer.context[key] = chef[key];