From 6ebbd15ab280655a39b6e2c104a45bcab6206b2d Mon Sep 17 00:00:00 2001 From: Brunon Blok <43315279+brun0ne@users.noreply.github.com> Date: Mon, 10 Apr 2023 16:18:29 +0000 Subject: [PATCH] add operation: generate spectrogram --- package-lock.json | 28 ++ package.json | 2 + src/core/config/Categories.json | 3 +- src/core/operations/GenerateSpectrogram.mjs | 317 ++++++++++++++++++++ 4 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/GenerateSpectrogram.mjs diff --git a/package-lock.json b/package-lock.json index 9b939b88..c447c662 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "d3": "7.8.2", "d3-hexbin": "^0.2.2", "diff": "^5.1.0", + "dsp.js": "^1.0.1", "es6-promisify": "^7.0.0", "escodegen": "^2.0.0", "esprima": "^4.0.1", @@ -89,6 +90,7 @@ "unorm": "^1.6.0", "utf8": "^3.0.0", "vkbeautify": "^0.99.3", + "wavefile": "^11.0.0", "xmldom": "^0.6.0", "xpath": "0.0.32", "xregexp": "^5.1.1", @@ -5710,6 +5712,11 @@ "node": ">=10" } }, + "node_modules/dsp.js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dsp.js/-/dsp.js-1.0.1.tgz", + "integrity": "sha512-5ols1j2i2tdbtd38DMR6xGSmE8dE/qFWgqrNN1ZNnIZnOEAFjzoVq5hM57nbQS9u0VmxqCUw8YccgsS9+zhtNg==" + }, "node_modules/duplexer": { "version": "0.1.2", "dev": true, @@ -13079,6 +13086,17 @@ "node": ">=10.13.0" } }, + "node_modules/wavefile": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/wavefile/-/wavefile-11.0.0.tgz", + "integrity": "sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng==", + "bin": { + "wavefile": "bin/wavefile.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wbuf": { "version": "1.7.3", "dev": true, @@ -17733,6 +17751,11 @@ "version": "10.0.0", "dev": true }, + "dsp.js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dsp.js/-/dsp.js-1.0.1.tgz", + "integrity": "sha512-5ols1j2i2tdbtd38DMR6xGSmE8dE/qFWgqrNN1ZNnIZnOEAFjzoVq5hM57nbQS9u0VmxqCUw8YccgsS9+zhtNg==" + }, "duplexer": { "version": "0.1.2", "dev": true @@ -22722,6 +22745,11 @@ "graceful-fs": "^4.1.2" } }, + "wavefile": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/wavefile/-/wavefile-11.0.0.tgz", + "integrity": "sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng==" + }, "wbuf": { "version": "1.7.3", "dev": true, diff --git a/package.json b/package.json index f5d9a7f9..e9345c41 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ "d3": "7.8.2", "d3-hexbin": "^0.2.2", "diff": "^5.1.0", + "dsp.js": "^1.0.1", "es6-promisify": "^7.0.0", "escodegen": "^2.0.0", "esprima": "^4.0.1", @@ -171,6 +172,7 @@ "unorm": "^1.6.0", "utf8": "^3.0.0", "vkbeautify": "^0.99.3", + "wavefile": "^11.0.0", "xmldom": "^0.6.0", "xpath": "0.0.32", "xregexp": "^5.1.1", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index ce2f01f5..fbc98ea5 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -477,7 +477,8 @@ "Hex Density chart", "Scatter chart", "Series chart", - "Heatmap chart" + "Heatmap chart", + "Generate Spectrogram" ] }, { diff --git a/src/core/operations/GenerateSpectrogram.mjs b/src/core/operations/GenerateSpectrogram.mjs new file mode 100644 index 00000000..69835d9f --- /dev/null +++ b/src/core/operations/GenerateSpectrogram.mjs @@ -0,0 +1,317 @@ +/** + * @author brun0ne [brunonblok@gmail.com] + * @copyright Crown Copyright 2023 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; + +import Utils from "../Utils.mjs"; +import { isType, detectFileType } from "../lib/FileType.mjs"; + +import jimp from "jimp"; +import {isWorkerEnvironment} from "../Utils.mjs"; +import { toBase64 } from "../lib/Base64.mjs"; + +import * as wavefile from "wavefile"; +import { RFFT } from "dsp.js"; + +/** + * Generate Spectrogram operation + */ +class GenerateSpectrogram extends Operation { + + /** + * GenerateSpectrogram constructor + */ + constructor() { + super(); + + this.name = "Generate Spectrogram"; + this.module = "Default"; + this.description = "Generates a spectrogram from a wave file"; + this.infoURL = "https://en.wikipedia.org/wiki/Spectrogram"; + this.inputType = "string"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + "name": "Frame size", + "type": "option", + "value": [ + "256", + "64", + "128", + "512", + "1024", + "2048" + ] + }, + { + "name": "Overlap", + "type": "option", + "value": [ + "0%", + "50%" + ] + }, + { + "name": "Color scheme", + "type": "option", + "value": [ + "Green-Black", + "Yellow-Blue", + "Greyscale", + "Neon" + ] + }, + { + "name": "Gain (dB)", + "type": "number", + "value": 0, + "min": -100, + "max": 100 + }, + { + "name": "Channel", + "type": "number", + "value": 0, + "min": 0 + }, + { + "name": "Include axes", + "type": "boolean", + "value": true + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {byteArray} the sound file as bytes + */ + run(input, args) { + input = Utils.strToByteArray(input); + + // Determine file type + if (!isType(/^(audio)/, input)) { + throw new OperationError("Invalid or unrecognised file type"); + } + + return input; + } + + /** + * Returns the spectrogram html + * @param {array} array of arrays of samples + * @param {number} sampleRate + * @param {Object[]} args + * @returns {string} HTML + */ + async getSpectrogram(sampleChannels, sampleRate, args) { + /* args */ + const frameSize = parseInt(args[0]); + const overlap = (parseInt(args[1]) / 100) * frameSize; + const colorScheme = args[2]; + const gain = args[3]; + const channelNum = args[4]; + const include_axes = args[5]; + + if (sampleChannels[channelNum] == null) { + throw new OperationError(`Invalid channel number: ${channelNum}`); + } + + const channel = sampleChannels[channelNum]; + const MAX_FREQ = sampleRate / 2; + + /* positions and sizes */ + let padding_left = 0; + let padding_right = 0; + let padding_bottom = 0; + if (include_axes){ + padding_left = Math.log10(MAX_FREQ) * 10 + 10 * 3; // space for the text (10px per digit) + padding_right = 50; + padding_bottom = 20; + } + + const width = channel.length / (frameSize - overlap); + const height = frameSize / 2; + + const image_width = width + padding_left + padding_right; + const image_height = height + padding_bottom; + + /* create an image */ + const image = new jimp(image_width, image_height, (err, image) => {}); + if (isWorkerEnvironment()) + self.sendStatusMessage("Generating a spectrogram from data..."); + + function fill(rgba) { + return function(x, y, idx) { + this.bitmap.data[idx + 0] = rgba[0]; + this.bitmap.data[idx + 1] = rgba[1]; + this.bitmap.data[idx + 2] = rgba[2]; + this.bitmap.data[idx + 3] = rgba[3]; + } + } + + /* fill with black */ + image.scan(0, 0, image.bitmap.width, image.bitmap.height, fill([0, 0, 0, 255]).bind(image)); + + if (include_axes) { + /* draw border */ + image.scan(padding_left - 1, height + 1, width + 2, 1, fill([255, 255, 255, 255]).bind(image)); // horizontal + image.scan(padding_left - 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (left) + image.scan(padding_left + width + 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (right) + + /* set font */ + const font = await import(/* webpackMode: "eager" */ "../../web/static/fonts/bmfonts/Roboto72White.fnt"); + await import(/* webpackMode: "eager" */ "../../web/static/fonts/bmfonts/Roboto72White.png"); + + /* LoadFont needs an absolute url, so append the font name to self.docURL */ + const jimpFont = await jimp.loadFont(self.docURL + "/" + font.default); + + /* create a temporary scaled image */ + const textScale = 72 / 10; + const textImage = new jimp(image.bitmap.width * textScale, image.bitmap.height * textScale, (err, image) => {}); + + /* write min and max frequency */ + textImage.print(jimpFont, /* align to right */ (padding_left - 30) * textScale, (height - 10) * textScale, "0 Hz"); + textImage.print(jimpFont, /* align to right */ (padding_left - (Math.log10(MAX_FREQ) + 1) * 10) * textScale, 5 * textScale, `${(MAX_FREQ).toFixed(0)} Hz`); + + /* write min and max time */ + textImage.print(jimpFont, (padding_left - 5) * textScale, (height + 5) * textScale, "0 s"); + textImage.print(jimpFont, (padding_left + width) * textScale, (height + 5) * textScale, `${(channel.length / sampleRate).toFixed(0)} s`); + + /* draw the scaled image on the original image */ + textImage.scaleToFit(image.bitmap.width, image.bitmap.height); + image.blit(textImage, 0, 0); + } + + const rfft = new RFFT(frameSize, sampleRate); + + let frames = []; + for (let start = 0; start < channel.length; start += (frameSize - overlap)){ + let chunk = channel.slice(start, start+frameSize); + + if (chunk.length < frameSize) { + /* pad with zeros */ + const pad = new Float64Array(frameSize - chunk.length).fill(0); + chunk = Float64Array.from([...chunk, ...pad]); + + console.log(chunk); + } + + /* get frequency spectrum */ + const freq = rfft.forward(chunk); + let frame = []; + for (let i = 0; i < freq.length; i++) { + /* convert to decibels */ + let strength = 10 * Math.log10(Math.abs(freq[i])) + gain; + + if (freq[i] == 0) // avoid -Infinity + strength = 0; + + if (strength < 0) + strength = 0; + + frame.push(strength); + } + frames.push(frame); + } + + /* normalize */ + let max = 0; + for (let i = 0; i < frames.length; i++) { + for (let j = 0; j < frames[i].length; j++) { + if (frames[i][j] > max) + max = frames[i][j]; + } + } + + /* draw */ + let pos_x = padding_left; + for (let i = 0; i < frames.length; i++) { + for (let j = 0; j < frames[i].length; j++) { + const colorStrength = (frames[i][j] / max) * 255; + let color; + + switch (colorScheme) { + case "Green-Black": + color = [0, colorStrength, 0, 255]; + break; + case "Greyscale": + color = [colorStrength, colorStrength, colorStrength, 255]; + break; + case "Yellow-Blue": + color = [colorStrength, colorStrength, Math.abs(50 - colorStrength), 255]; + break; + case "Neon": + color = [Math.sin(colorStrength/255) * 255, 0, Math.tanh(colorStrength) * 255, 255]; + break; + default: + throw new OperationError(`Unknown color scheme: ${colorScheme}`); + } + + image.scan(pos_x, height - j, 1, 1, fill(color).bind(image)); + } + pos_x++; + } + + /* get image data */ + let imageBuffer; + try { + imageBuffer = await image.getBufferAsync(jimp.MIME_PNG); + } catch (err) { + throw new OperationError(`Error generating image. (${err})`); + } + + return `` + } + + /** + * Displays a spectrogram + * + * @param {byteArray} data containing the audio file + * @returns {string} HTML + */ + async present(data, args) { + console.log(args); + + if (!data.length) return ""; + + // check file type + const types = detectFileType(data); + if (types[0].mime !== "audio/wav" && types[0].mime !== "audio/x-wav") { + throw new OperationError("Only WAV files are supported"); + } + + // parse wave + let wave; + try { + wave = new wavefile.WaveFile(data); + } + catch (err) { + throw new OperationError("Invalid WAV file"); + } + + // get properties + const numChannels = wave.fmt.numChannels; + const sampleRate = wave.fmt.sampleRate; + + // get samples + let sampleChannels = []; + if (numChannels > 1) { + sampleChannels = wave.getSamples(); // returns an array of arrays + } + else { + sampleChannels.push(wave.getSamples()); // returns an array + } + + // get the spectrogram html + return this.getSpectrogram(sampleChannels, sampleRate, args); + } +} + +export default GenerateSpectrogram;