From 6ebbd15ab280655a39b6e2c104a45bcab6206b2d Mon Sep 17 00:00:00 2001 From: Brunon Blok <43315279+brun0ne@users.noreply.github.com> Date: Mon, 10 Apr 2023 16:18:29 +0000 Subject: [PATCH 1/4] add operation: generate spectrogram --- package-lock.json | 28 ++ package.json | 2 + src/core/config/Categories.json | 3 +- src/core/operations/GenerateSpectrogram.mjs | 317 ++++++++++++++++++++ 4 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/GenerateSpectrogram.mjs diff --git a/package-lock.json b/package-lock.json index 9b939b88..c447c662 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "d3": "7.8.2", "d3-hexbin": "^0.2.2", "diff": "^5.1.0", + "dsp.js": "^1.0.1", "es6-promisify": "^7.0.0", "escodegen": "^2.0.0", "esprima": "^4.0.1", @@ -89,6 +90,7 @@ "unorm": "^1.6.0", "utf8": "^3.0.0", "vkbeautify": "^0.99.3", + "wavefile": "^11.0.0", "xmldom": "^0.6.0", "xpath": "0.0.32", "xregexp": "^5.1.1", @@ -5710,6 +5712,11 @@ "node": ">=10" } }, + "node_modules/dsp.js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dsp.js/-/dsp.js-1.0.1.tgz", + "integrity": "sha512-5ols1j2i2tdbtd38DMR6xGSmE8dE/qFWgqrNN1ZNnIZnOEAFjzoVq5hM57nbQS9u0VmxqCUw8YccgsS9+zhtNg==" + }, "node_modules/duplexer": { "version": "0.1.2", "dev": true, @@ -13079,6 +13086,17 @@ "node": ">=10.13.0" } }, + "node_modules/wavefile": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/wavefile/-/wavefile-11.0.0.tgz", + "integrity": "sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng==", + "bin": { + "wavefile": "bin/wavefile.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wbuf": { "version": "1.7.3", "dev": true, @@ -17733,6 +17751,11 @@ "version": "10.0.0", "dev": true }, + "dsp.js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dsp.js/-/dsp.js-1.0.1.tgz", + "integrity": "sha512-5ols1j2i2tdbtd38DMR6xGSmE8dE/qFWgqrNN1ZNnIZnOEAFjzoVq5hM57nbQS9u0VmxqCUw8YccgsS9+zhtNg==" + }, "duplexer": { "version": "0.1.2", "dev": true @@ -22722,6 +22745,11 @@ "graceful-fs": "^4.1.2" } }, + "wavefile": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/wavefile/-/wavefile-11.0.0.tgz", + "integrity": "sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng==" + }, "wbuf": { "version": "1.7.3", "dev": true, diff --git a/package.json b/package.json index f5d9a7f9..e9345c41 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ "d3": "7.8.2", "d3-hexbin": "^0.2.2", "diff": "^5.1.0", + "dsp.js": "^1.0.1", "es6-promisify": "^7.0.0", "escodegen": "^2.0.0", "esprima": "^4.0.1", @@ -171,6 +172,7 @@ "unorm": "^1.6.0", "utf8": "^3.0.0", "vkbeautify": "^0.99.3", + "wavefile": "^11.0.0", "xmldom": "^0.6.0", "xpath": "0.0.32", "xregexp": "^5.1.1", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index ce2f01f5..fbc98ea5 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -477,7 +477,8 @@ "Hex Density chart", "Scatter chart", "Series chart", - "Heatmap chart" + "Heatmap chart", + "Generate Spectrogram" ] }, { diff --git a/src/core/operations/GenerateSpectrogram.mjs b/src/core/operations/GenerateSpectrogram.mjs new file mode 100644 index 00000000..69835d9f --- /dev/null +++ b/src/core/operations/GenerateSpectrogram.mjs @@ -0,0 +1,317 @@ +/** + * @author brun0ne [brunonblok@gmail.com] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; + +import Utils from "../Utils.mjs"; +import { isType, detectFileType } from "../lib/FileType.mjs"; + +import jimp from "jimp"; +import {isWorkerEnvironment} from "../Utils.mjs"; +import { toBase64 } from "../lib/Base64.mjs"; + +import * as wavefile from "wavefile"; +import { RFFT } from "dsp.js"; + +/** + * Generate Spectrogram operation + */ +class GenerateSpectrogram extends Operation { + + /** + * GenerateSpectrogram constructor + */ + constructor() { + super(); + + this.name = "Generate Spectrogram"; + this.module = "Default"; + this.description = "Generates a spectrogram from a wave file"; + this.infoURL = "https://en.wikipedia.org/wiki/Spectrogram"; + this.inputType = "string"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + "name": "Frame size", + "type": "option", + "value": [ + "256", + "64", + "128", + "512", + "1024", + "2048" + ] + }, + { + "name": "Overlap", + "type": "option", + "value": [ + "0%", + "50%" + ] + }, + { + "name": "Color scheme", + "type": "option", + "value": [ + "Green-Black", + "Yellow-Blue", + "Greyscale", + "Neon" + ] + }, + { + "name": "Gain (dB)", + "type": "number", + "value": 0, + "min": -100, + "max": 100 + }, + { + "name": "Channel", + "type": "number", + "value": 0, + "min": 0 + }, + { + "name": "Include axes", + "type": "boolean", + "value": true + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {byteArray} the sound file as bytes + */ + run(input, args) { + input = Utils.strToByteArray(input); + + // Determine file type + if (!isType(/^(audio)/, input)) { + throw new OperationError("Invalid or unrecognised file type"); + } + + return input; + } + + /** + * Returns the spectrogram html + * @param {array} array of arrays of samples + * @param {number} sampleRate + * @param {Object[]} args + * @returns {string} HTML + */ + async getSpectrogram(sampleChannels, sampleRate, args) { + /* args */ + const frameSize = parseInt(args[0]); + const overlap = (parseInt(args[1]) / 100) * frameSize; + const colorScheme = args[2]; + const gain = args[3]; + const channelNum = args[4]; + const include_axes = args[5]; + + if (sampleChannels[channelNum] == null) { + throw new OperationError(`Invalid channel number: ${channelNum}`); + } + + const channel = sampleChannels[channelNum]; + const MAX_FREQ = sampleRate / 2; + + /* positions and sizes */ + let padding_left = 0; + let padding_right = 0; + let padding_bottom = 0; + if (include_axes){ + padding_left = Math.log10(MAX_FREQ) * 10 + 10 * 3; // space for the text (10px per digit) + padding_right = 50; + padding_bottom = 20; + } + + const width = channel.length / (frameSize - overlap); + const height = frameSize / 2; + + const image_width = width + padding_left + padding_right; + const image_height = height + padding_bottom; + + /* create an image */ + const image = new jimp(image_width, image_height, (err, image) => {}); + if (isWorkerEnvironment()) + self.sendStatusMessage("Generating a spectrogram from data..."); + + function fill(rgba) { + return function(x, y, idx) { + this.bitmap.data[idx + 0] = rgba[0]; + this.bitmap.data[idx + 1] = rgba[1]; + this.bitmap.data[idx + 2] = rgba[2]; + this.bitmap.data[idx + 3] = rgba[3]; + } + } + + /* fill with black */ + image.scan(0, 0, image.bitmap.width, image.bitmap.height, fill([0, 0, 0, 255]).bind(image)); + + if (include_axes) { + /* draw border */ + image.scan(padding_left - 1, height + 1, width + 2, 1, fill([255, 255, 255, 255]).bind(image)); // horizontal + image.scan(padding_left - 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (left) + image.scan(padding_left + width + 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (right) + + /* set font */ + const font = await import(/* webpackMode: "eager" */ "../../web/static/fonts/bmfonts/Roboto72White.fnt"); + await import(/* webpackMode: "eager" */ "../../web/static/fonts/bmfonts/Roboto72White.png"); + + /* LoadFont needs an absolute url, so append the font name to self.docURL */ + const jimpFont = await jimp.loadFont(self.docURL + "/" + font.default); + + /* create a temporary scaled image */ + const textScale = 72 / 10; + const textImage = new jimp(image.bitmap.width * textScale, image.bitmap.height * textScale, (err, image) => {}); + + /* write min and max frequency */ + textImage.print(jimpFont, /* align to right */ (padding_left - 30) * textScale, (height - 10) * textScale, "0 Hz"); + textImage.print(jimpFont, /* align to right */ (padding_left - (Math.log10(MAX_FREQ) + 1) * 10) * textScale, 5 * textScale, `${(MAX_FREQ).toFixed(0)} Hz`); + + /* write min and max time */ + textImage.print(jimpFont, (padding_left - 5) * textScale, (height + 5) * textScale, "0 s"); + textImage.print(jimpFont, (padding_left + width) * textScale, (height + 5) * textScale, `${(channel.length / sampleRate).toFixed(0)} s`); + + /* draw the scaled image on the original image */ + textImage.scaleToFit(image.bitmap.width, image.bitmap.height); + image.blit(textImage, 0, 0); + } + + const rfft = new RFFT(frameSize, sampleRate); + + let frames = []; + for (let start = 0; start < channel.length; start += (frameSize - overlap)){ + let chunk = channel.slice(start, start+frameSize); + + if (chunk.length < frameSize) { + /* pad with zeros */ + const pad = new Float64Array(frameSize - chunk.length).fill(0); + chunk = Float64Array.from([...chunk, ...pad]); + + console.log(chunk); + } + + /* get frequency spectrum */ + const freq = rfft.forward(chunk); + let frame = []; + for (let i = 0; i < freq.length; i++) { + /* convert to decibels */ + let strength = 10 * Math.log10(Math.abs(freq[i])) + gain; + + if (freq[i] == 0) // avoid -Infinity + strength = 0; + + if (strength < 0) + strength = 0; + + frame.push(strength); + } + frames.push(frame); + } + + /* normalize */ + let max = 0; + for (let i = 0; i < frames.length; i++) { + for (let j = 0; j < frames[i].length; j++) { + if (frames[i][j] > max) + max = frames[i][j]; + } + } + + /* draw */ + let pos_x = padding_left; + for (let i = 0; i < frames.length; i++) { + for (let j = 0; j < frames[i].length; j++) { + const colorStrength = (frames[i][j] / max) * 255; + let color; + + switch (colorScheme) { + case "Green-Black": + color = [0, colorStrength, 0, 255]; + break; + case "Greyscale": + color = [colorStrength, colorStrength, colorStrength, 255]; + break; + case "Yellow-Blue": + color = [colorStrength, colorStrength, Math.abs(50 - colorStrength), 255]; + break; + case "Neon": + color = [Math.sin(colorStrength/255) * 255, 0, Math.tanh(colorStrength) * 255, 255]; + break; + default: + throw new OperationError(`Unknown color scheme: ${colorScheme}`); + } + + image.scan(pos_x, height - j, 1, 1, fill(color).bind(image)); + } + pos_x++; + } + + /* get image data */ + let imageBuffer; + try { + imageBuffer = await image.getBufferAsync(jimp.MIME_PNG); + } catch (err) { + throw new OperationError(`Error generating image. (${err})`); + } + + return `` + } + + /** + * Displays a spectrogram + * + * @param {byteArray} data containing the audio file + * @returns {string} HTML + */ + async present(data, args) { + console.log(args); + + if (!data.length) return ""; + + // check file type + const types = detectFileType(data); + if (types[0].mime !== "audio/wav" && types[0].mime !== "audio/x-wav") { + throw new OperationError("Only WAV files are supported"); + } + + // parse wave + let wave; + try { + wave = new wavefile.WaveFile(data); + } + catch (err) { + throw new OperationError("Invalid WAV file"); + } + + // get properties + const numChannels = wave.fmt.numChannels; + const sampleRate = wave.fmt.sampleRate; + + // get samples + let sampleChannels = []; + if (numChannels > 1) { + sampleChannels = wave.getSamples(); // returns an array of arrays + } + else { + sampleChannels.push(wave.getSamples()); // returns an array + } + + // get the spectrogram html + return this.getSpectrogram(sampleChannels, sampleRate, args); + } +} + +export default GenerateSpectrogram; From eb94c6f35b3f73739c8a5d6cf73f8898c1be2784 Mon Sep 17 00:00:00 2001 From: Brunon Blok <43315279+brun0ne@users.noreply.github.com> Date: Mon, 10 Apr 2023 16:37:02 +0000 Subject: [PATCH 2/4] comply with lint --- src/core/operations/GenerateSpectrogram.mjs | 77 +++++++++++---------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/src/core/operations/GenerateSpectrogram.mjs b/src/core/operations/GenerateSpectrogram.mjs index 69835d9f..eef3979f 100644 --- a/src/core/operations/GenerateSpectrogram.mjs +++ b/src/core/operations/GenerateSpectrogram.mjs @@ -112,12 +112,12 @@ class GenerateSpectrogram extends Operation { */ async getSpectrogram(sampleChannels, sampleRate, args) { /* args */ - const frameSize = parseInt(args[0]); - const overlap = (parseInt(args[1]) / 100) * frameSize; + const frameSize = parseInt(args[0], 10); + const overlap = (parseInt(args[1], 10) / 100) * frameSize; const colorScheme = args[2]; const gain = args[3]; const channelNum = args[4]; - const include_axes = args[5]; + const includeAxes = args[5]; if (sampleChannels[channelNum] == null) { throw new OperationError(`Invalid channel number: ${channelNum}`); @@ -127,44 +127,49 @@ class GenerateSpectrogram extends Operation { const MAX_FREQ = sampleRate / 2; /* positions and sizes */ - let padding_left = 0; - let padding_right = 0; - let padding_bottom = 0; - if (include_axes){ - padding_left = Math.log10(MAX_FREQ) * 10 + 10 * 3; // space for the text (10px per digit) - padding_right = 50; - padding_bottom = 20; + let paddingLeft = 0; + let paddingRight = 0; + let paddingBottom = 0; + if (includeAxes) { + paddingLeft = Math.log10(MAX_FREQ) * 10 + 10 * 3; // space for the frequency labels (10px per digit) + paddingRight = 50; + paddingBottom = 20; } const width = channel.length / (frameSize - overlap); const height = frameSize / 2; - const image_width = width + padding_left + padding_right; - const image_height = height + padding_bottom; + const imageWidth = width + paddingLeft + paddingRight; + const imageHeight = height + paddingBottom; /* create an image */ - const image = new jimp(image_width, image_height, (err, image) => {}); + const image = new jimp(imageWidth, imageHeight, (err, image) => {}); if (isWorkerEnvironment()) self.sendStatusMessage("Generating a spectrogram from data..."); + /** + * Returns a function filling a pixel with given color + * @param {array} rgba + * @returns function + */ function fill(rgba) { return function(x, y, idx) { this.bitmap.data[idx + 0] = rgba[0]; this.bitmap.data[idx + 1] = rgba[1]; this.bitmap.data[idx + 2] = rgba[2]; this.bitmap.data[idx + 3] = rgba[3]; - } + }; } /* fill with black */ image.scan(0, 0, image.bitmap.width, image.bitmap.height, fill([0, 0, 0, 255]).bind(image)); - if (include_axes) { + if (includeAxes) { /* draw border */ - image.scan(padding_left - 1, height + 1, width + 2, 1, fill([255, 255, 255, 255]).bind(image)); // horizontal - image.scan(padding_left - 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (left) - image.scan(padding_left + width + 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (right) - + image.scan(paddingLeft - 1, height + 1, width + 2, 1, fill([255, 255, 255, 255]).bind(image)); // horizontal + image.scan(paddingLeft - 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (left) + image.scan(paddingLeft + width + 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (right) + /* set font */ const font = await import(/* webpackMode: "eager" */ "../../web/static/fonts/bmfonts/Roboto72White.fnt"); await import(/* webpackMode: "eager" */ "../../web/static/fonts/bmfonts/Roboto72White.png"); @@ -177,12 +182,12 @@ class GenerateSpectrogram extends Operation { const textImage = new jimp(image.bitmap.width * textScale, image.bitmap.height * textScale, (err, image) => {}); /* write min and max frequency */ - textImage.print(jimpFont, /* align to right */ (padding_left - 30) * textScale, (height - 10) * textScale, "0 Hz"); - textImage.print(jimpFont, /* align to right */ (padding_left - (Math.log10(MAX_FREQ) + 1) * 10) * textScale, 5 * textScale, `${(MAX_FREQ).toFixed(0)} Hz`); + textImage.print(jimpFont, /* align to right */ (paddingLeft - 30) * textScale, (height - 10) * textScale, "0 Hz"); + textImage.print(jimpFont, /* align to right */ (paddingLeft - (Math.log10(MAX_FREQ) + 1) * 10) * textScale, 5 * textScale, `${(MAX_FREQ).toFixed(0)} Hz`); /* write min and max time */ - textImage.print(jimpFont, (padding_left - 5) * textScale, (height + 5) * textScale, "0 s"); - textImage.print(jimpFont, (padding_left + width) * textScale, (height + 5) * textScale, `${(channel.length / sampleRate).toFixed(0)} s`); + textImage.print(jimpFont, (paddingLeft - 5) * textScale, (height + 5) * textScale, "0 s"); + textImage.print(jimpFont, (paddingLeft + width) * textScale, (height + 5) * textScale, `${(channel.length / sampleRate).toFixed(0)} s`); /* draw the scaled image on the original image */ textImage.scaleToFit(image.bitmap.width, image.bitmap.height); @@ -191,8 +196,8 @@ class GenerateSpectrogram extends Operation { const rfft = new RFFT(frameSize, sampleRate); - let frames = []; - for (let start = 0; start < channel.length; start += (frameSize - overlap)){ + const frames = []; + for (let start = 0; start < channel.length; start += (frameSize - overlap)) { let chunk = channel.slice(start, start+frameSize); if (chunk.length < frameSize) { @@ -205,12 +210,12 @@ class GenerateSpectrogram extends Operation { /* get frequency spectrum */ const freq = rfft.forward(chunk); - let frame = []; + const frame = []; for (let i = 0; i < freq.length; i++) { /* convert to decibels */ let strength = 10 * Math.log10(Math.abs(freq[i])) + gain; - - if (freq[i] == 0) // avoid -Infinity + + if (freq[i] === 0) // avoid -Infinity strength = 0; if (strength < 0) @@ -220,7 +225,7 @@ class GenerateSpectrogram extends Operation { } frames.push(frame); } - + /* normalize */ let max = 0; for (let i = 0; i < frames.length; i++) { @@ -231,7 +236,7 @@ class GenerateSpectrogram extends Operation { } /* draw */ - let pos_x = padding_left; + let posX = paddingLeft; for (let i = 0; i < frames.length; i++) { for (let j = 0; j < frames[i].length; j++) { const colorStrength = (frames[i][j] / max) * 255; @@ -254,9 +259,9 @@ class GenerateSpectrogram extends Operation { throw new OperationError(`Unknown color scheme: ${colorScheme}`); } - image.scan(pos_x, height - j, 1, 1, fill(color).bind(image)); + image.scan(posX, height - j, 1, 1, fill(color).bind(image)); } - pos_x++; + posX++; } /* get image data */ @@ -267,7 +272,7 @@ class GenerateSpectrogram extends Operation { throw new OperationError(`Error generating image. (${err})`); } - return `` + return ``; } /** @@ -291,8 +296,7 @@ class GenerateSpectrogram extends Operation { let wave; try { wave = new wavefile.WaveFile(data); - } - catch (err) { + } catch (err) { throw new OperationError("Invalid WAV file"); } @@ -304,8 +308,7 @@ class GenerateSpectrogram extends Operation { let sampleChannels = []; if (numChannels > 1) { sampleChannels = wave.getSamples(); // returns an array of arrays - } - else { + } else { sampleChannels.push(wave.getSamples()); // returns an array } From c182f06e1435609b0ec565d519680c0ffee75080 Mon Sep 17 00:00:00 2001 From: Brunon Blok <43315279+brun0ne@users.noreply.github.com> Date: Mon, 10 Apr 2023 16:57:51 +0000 Subject: [PATCH 3/4] update description --- src/core/operations/GenerateSpectrogram.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/GenerateSpectrogram.mjs b/src/core/operations/GenerateSpectrogram.mjs index eef3979f..069bb892 100644 --- a/src/core/operations/GenerateSpectrogram.mjs +++ b/src/core/operations/GenerateSpectrogram.mjs @@ -30,7 +30,7 @@ class GenerateSpectrogram extends Operation { this.name = "Generate Spectrogram"; this.module = "Default"; - this.description = "Generates a spectrogram from a wave file"; + this.description = "Generates a spectrogram from a wave file.

Frame size - the number of samples process at once by FFT.
Overlap - the number of samples to overlap between frames.
Color scheme - the color scheme to use.
Gain (dB) - gain in decibels.
Channel - the channel to use.
Include axes - whether to include axes and labels on the rendered image."; this.infoURL = "https://en.wikipedia.org/wiki/Spectrogram"; this.inputType = "string"; this.outputType = "byteArray"; From 6d51f53829d175e67f697ae662c62ea33ed706a1 Mon Sep 17 00:00:00 2001 From: Brunon Blok <43315279+brun0ne@users.noreply.github.com> Date: Mon, 10 Apr 2023 17:31:56 +0000 Subject: [PATCH 4/4] remove debug logging --- src/core/operations/GenerateSpectrogram.mjs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/core/operations/GenerateSpectrogram.mjs b/src/core/operations/GenerateSpectrogram.mjs index 069bb892..f6e749fb 100644 --- a/src/core/operations/GenerateSpectrogram.mjs +++ b/src/core/operations/GenerateSpectrogram.mjs @@ -204,8 +204,6 @@ class GenerateSpectrogram extends Operation { /* pad with zeros */ const pad = new Float64Array(frameSize - chunk.length).fill(0); chunk = Float64Array.from([...chunk, ...pad]); - - console.log(chunk); } /* get frequency spectrum */ @@ -282,8 +280,6 @@ class GenerateSpectrogram extends Operation { * @returns {string} HTML */ async present(data, args) { - console.log(args); - if (!data.length) return ""; // check file type