From 7bfc0a04507084628d08b5531c6a2a5586c9dc6c Mon Sep 17 00:00:00 2001
From: Brunon Blok <43315279+brun0ne@users.noreply.github.com>
Date: Mon, 10 Apr 2023 16:18:29 +0000
Subject: [PATCH] add operation: generate spectrogram
---
package-lock.json | 20 ++
package.json | 2 +
src/core/config/Categories.json | 3 +-
src/core/operations/GenerateSpectrogram.mjs | 317 ++++++++++++++++++++
4 files changed, 341 insertions(+), 1 deletion(-)
create mode 100644 src/core/operations/GenerateSpectrogram.mjs
diff --git a/package-lock.json b/package-lock.json
index b374df4b..258a5eb0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -38,6 +38,7 @@
"d3-hexbin": "^0.2.2",
"diff": "^5.2.0",
"dompurify": "^3.2.5",
+ "dsp.js": "^1.0.1",
"es6-promisify": "^7.0.0",
"escodegen": "^2.1.0",
"esprima": "^4.0.1",
@@ -99,6 +100,7 @@
"utf8": "^3.0.0",
"uuid": "^11.1.0",
"vkbeautify": "^0.99.3",
+ "wavefile": "^11.0.0",
"xpath": "0.0.34",
"xregexp": "^5.1.1",
"zlibjs": "^0.3.1"
@@ -8470,6 +8472,12 @@
"url": "https://github.com/motdotla/dotenv?sponsor=1"
}
},
+ "node_modules/dsp.js": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/dsp.js/-/dsp.js-1.0.1.tgz",
+ "integrity": "sha512-5ols1j2i2tdbtd38DMR6xGSmE8dE/qFWgqrNN1ZNnIZnOEAFjzoVq5hM57nbQS9u0VmxqCUw8YccgsS9+zhtNg==",
+ "license": "MIT"
+ },
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -18254,6 +18262,18 @@
"node": ">=10.13.0"
}
},
+ "node_modules/wavefile": {
+ "version": "11.0.0",
+ "resolved": "https://registry.npmjs.org/wavefile/-/wavefile-11.0.0.tgz",
+ "integrity": "sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng==",
+ "license": "MIT",
+ "bin": {
+ "wavefile": "bin/wavefile.js"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/wbuf": {
"version": "1.7.3",
"resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz",
diff --git a/package.json b/package.json
index 9191ab6f..7f6e9fa6 100644
--- a/package.json
+++ b/package.json
@@ -124,6 +124,7 @@
"d3-hexbin": "^0.2.2",
"diff": "^5.2.0",
"dompurify": "^3.2.5",
+ "dsp.js": "^1.0.1",
"es6-promisify": "^7.0.0",
"escodegen": "^2.1.0",
"esprima": "^4.0.1",
@@ -185,6 +186,7 @@
"utf8": "^3.0.0",
"uuid": "^11.1.0",
"vkbeautify": "^0.99.3",
+ "wavefile": "^11.0.0",
"xpath": "0.0.34",
"xregexp": "^5.1.1",
"zlibjs": "^0.3.1"
diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json
index 434c8bb6..1b645969 100644
--- a/src/core/config/Categories.json
+++ b/src/core/config/Categories.json
@@ -536,7 +536,8 @@
"Hex Density chart",
"Scatter chart",
"Series chart",
- "Heatmap chart"
+ "Heatmap chart",
+ "Generate Spectrogram"
]
},
{
diff --git a/src/core/operations/GenerateSpectrogram.mjs b/src/core/operations/GenerateSpectrogram.mjs
new file mode 100644
index 00000000..ca828515
--- /dev/null
+++ b/src/core/operations/GenerateSpectrogram.mjs
@@ -0,0 +1,317 @@
+/**
+ * @author brun0ne [brunonblok@gmail.com]
+ * @copyright Crown Copyright 2023
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
+
+import Utils from "../Utils.mjs";
+import { isType, detectFileType } from "../lib/FileType.mjs";
+
+import Jimp from "jimp/es/index.js";
+
+import {isWorkerEnvironment} from "../Utils.mjs";
+import { toBase64 } from "../lib/Base64.mjs";
+
+import * as wavefile from "wavefile";
+import { RFFT } from "dsp.js";
+
+/**
+ * Generate Spectrogram operation
+ */
+class GenerateSpectrogram extends Operation {
+
+ /**
+ * GenerateSpectrogram constructor
+ */
+ constructor() {
+ super();
+
+ this.name = "Generate Spectrogram";
+ this.module = "Default";
+ this.description = "Generates a spectrogram from a wave file.
Frame size - the number of samples process at once by FFT.
Overlap - the number of samples to overlap between frames.
Color scheme - the color scheme to use.
Gain (dB) - gain in decibels.
Channel - the channel to use.
Include axes - whether to include axes and labels on the rendered image.";
+ this.infoURL = "https://en.wikipedia.org/wiki/Spectrogram";
+ this.inputType = "string";
+ this.outputType = "byteArray";
+ this.presentType = "html";
+ this.args = [
+ {
+ "name": "Frame size",
+ "type": "option",
+ "value": [
+ "256",
+ "64",
+ "128",
+ "512",
+ "1024",
+ "2048"
+ ]
+ },
+ {
+ "name": "Overlap",
+ "type": "option",
+ "value": [
+ "0%",
+ "50%"
+ ]
+ },
+ {
+ "name": "Color scheme",
+ "type": "option",
+ "value": [
+ "Green-Black",
+ "Yellow-Blue",
+ "Greyscale",
+ "Neon"
+ ]
+ },
+ {
+ "name": "Gain (dB)",
+ "type": "number",
+ "value": 0,
+ "min": -100,
+ "max": 100
+ },
+ {
+ "name": "Channel",
+ "type": "number",
+ "value": 0,
+ "min": 0
+ },
+ {
+ "name": "Include axes",
+ "type": "boolean",
+ "value": true
+ }
+ ];
+ }
+
+ /**
+ * @param {string} input
+ * @param {Object[]} args
+ * @returns {byteArray} the sound file as bytes
+ */
+ run(input, args) {
+ input = Utils.strToByteArray(input);
+
+ // Determine file type
+ if (!isType(/^(audio)/, input)) {
+ throw new OperationError("Invalid or unrecognised file type");
+ }
+
+ return input;
+ }
+
+ /**
+ * Returns the spectrogram html
+ * @param {array} array of arrays of samples
+ * @param {number} sampleRate
+ * @param {Object[]} args
+ * @returns {string} HTML
+ */
+ async getSpectrogram(sampleChannels, sampleRate, args) {
+ /* args */
+ const frameSize = parseInt(args[0], 10);
+ const overlap = (parseInt(args[1], 10) / 100) * frameSize;
+ const colorScheme = args[2];
+ const gain = args[3];
+ const channelNum = args[4];
+ const includeAxes = args[5];
+
+ if (sampleChannels[channelNum] == null) {
+ throw new OperationError(`Invalid channel number: ${channelNum}`);
+ }
+
+ const channel = sampleChannels[channelNum];
+ const MAX_FREQ = sampleRate / 2;
+
+ /* positions and sizes */
+ let paddingLeft = 0;
+ let paddingRight = 0;
+ let paddingBottom = 0;
+ if (includeAxes) {
+ paddingLeft = Math.log10(MAX_FREQ) * 10 + 10 * 3; // space for the frequency labels (10px per digit)
+ paddingRight = 50;
+ paddingBottom = 20;
+ }
+
+ const width = channel.length / (frameSize - overlap);
+ const height = frameSize / 2;
+
+ const imageWidth = width + paddingLeft + paddingRight;
+ const imageHeight = height + paddingBottom;
+
+ /* create an image */
+ const image = new Jimp(imageWidth, imageHeight, (err, image) => {});
+ if (isWorkerEnvironment())
+ self.sendStatusMessage("Generating a spectrogram from data...");
+
+ /**
+ * Returns a function filling a pixel with given color
+ * @param {array} rgba
+ * @returns function
+ */
+ function fill(rgba) {
+ return function(x, y, idx) {
+ this.bitmap.data[idx + 0] = rgba[0];
+ this.bitmap.data[idx + 1] = rgba[1];
+ this.bitmap.data[idx + 2] = rgba[2];
+ this.bitmap.data[idx + 3] = rgba[3];
+ };
+ }
+
+ /* fill with black */
+ image.scan(0, 0, image.bitmap.width, image.bitmap.height, fill([0, 0, 0, 255]).bind(image));
+
+ if (includeAxes) {
+ /* draw border */
+ image.scan(paddingLeft - 1, height + 1, width + 2, 1, fill([255, 255, 255, 255]).bind(image)); // horizontal
+ image.scan(paddingLeft - 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (left)
+ image.scan(paddingLeft + width + 1, 0, 1, height + 1, fill([255, 255, 255, 255]).bind(image)); // vertical (right)
+
+ /* set font */
+ const font = await import(/* webpackMode: "eager" */ "../../web/static/fonts/bmfonts/Roboto72White.fnt");
+ await import(/* webpackMode: "eager" */ "../../web/static/fonts/bmfonts/Roboto72White.png");
+
+ /* LoadFont needs an absolute url, so append the font name to self.docURL */
+ const jimpFont = await Jimp.loadFont(self.docURL + "/" + font.default);
+
+ /* create a temporary scaled image */
+ const textScale = 72 / 10;
+ const textImage = new Jimp(image.bitmap.width * textScale, image.bitmap.height * textScale, (err, image) => {});
+
+ /* write min and max frequency */
+ textImage.print(jimpFont, /* align to right */ (paddingLeft - 30) * textScale, (height - 10) * textScale, "0 Hz");
+ textImage.print(jimpFont, /* align to right */ (paddingLeft - (Math.log10(MAX_FREQ) + 1) * 10) * textScale, 5 * textScale, `${(MAX_FREQ).toFixed(0)} Hz`);
+
+ /* write min and max time */
+ textImage.print(jimpFont, (paddingLeft - 5) * textScale, (height + 5) * textScale, "0 s");
+ textImage.print(jimpFont, (paddingLeft + width) * textScale, (height + 5) * textScale, `${(channel.length / sampleRate).toFixed(0)} s`);
+
+ /* draw the scaled image on the original image */
+ textImage.scaleToFit(image.bitmap.width, image.bitmap.height);
+ image.blit(textImage, 0, 0);
+ }
+
+ const rfft = new RFFT(frameSize, sampleRate);
+
+ const frames = [];
+ for (let start = 0; start < channel.length; start += (frameSize - overlap)) {
+ let chunk = channel.slice(start, start+frameSize);
+
+ if (chunk.length < frameSize) {
+ /* pad with zeros */
+ const pad = new Float64Array(frameSize - chunk.length).fill(0);
+ chunk = Float64Array.from([...chunk, ...pad]);
+ }
+
+ /* get frequency spectrum */
+ const freq = rfft.forward(chunk);
+ const frame = [];
+ for (let i = 0; i < freq.length; i++) {
+ /* convert to decibels */
+ let strength = 10 * Math.log10(Math.abs(freq[i])) + gain;
+
+ if (freq[i] === 0) // avoid -Infinity
+ strength = 0;
+
+ if (strength < 0)
+ strength = 0;
+
+ frame.push(strength);
+ }
+ frames.push(frame);
+ }
+
+ /* normalize */
+ let max = 0;
+ for (let i = 0; i < frames.length; i++) {
+ for (let j = 0; j < frames[i].length; j++) {
+ if (frames[i][j] > max)
+ max = frames[i][j];
+ }
+ }
+
+ /* draw */
+ let posX = paddingLeft;
+ for (let i = 0; i < frames.length; i++) {
+ for (let j = 0; j < frames[i].length; j++) {
+ const colorStrength = (frames[i][j] / max) * 255;
+ let color;
+
+ switch (colorScheme) {
+ case "Green-Black":
+ color = [0, colorStrength, 0, 255];
+ break;
+ case "Greyscale":
+ color = [colorStrength, colorStrength, colorStrength, 255];
+ break;
+ case "Yellow-Blue":
+ color = [colorStrength, colorStrength, Math.abs(50 - colorStrength), 255];
+ break;
+ case "Neon":
+ color = [Math.sin(colorStrength/255) * 255, 0, Math.tanh(colorStrength) * 255, 255];
+ break;
+ default:
+ throw new OperationError(`Unknown color scheme: ${colorScheme}`);
+ }
+
+ image.scan(posX, height - j, 1, 1, fill(color).bind(image));
+ }
+ posX++;
+ }
+
+ /* get image data */
+ let imageBuffer;
+ try {
+ imageBuffer = await image.getBufferAsync(Jimp.MIME_PNG);
+ } catch (err) {
+ throw new OperationError(`Error generating image. (${err})`);
+ }
+
+ return `})
`;
+ }
+
+ /**
+ * Displays a spectrogram
+ *
+ * @param {byteArray} data containing the audio file
+ * @returns {string} HTML
+ */
+ async present(data, args) {
+ if (!data.length) return "";
+
+ // check file type
+ const types = detectFileType(data);
+ if (types[0].mime !== "audio/wav" && types[0].mime !== "audio/x-wav") {
+ throw new OperationError("Only WAV files are supported");
+ }
+
+ // parse wave
+ let wave;
+ try {
+ wave = new wavefile.WaveFile(data);
+ } catch (err) {
+ throw new OperationError("Invalid WAV file");
+ }
+
+ // get properties
+ const numChannels = wave.fmt.numChannels;
+ const sampleRate = wave.fmt.sampleRate;
+
+ // get samples
+ let sampleChannels = [];
+ if (numChannels > 1) {
+ sampleChannels = wave.getSamples(); // returns an array of arrays
+ } else {
+ sampleChannels.push(wave.getSamples()); // returns an array
+ }
+
+ // get the spectrogram html
+ return this.getSpectrogram(sampleChannels, sampleRate, args);
+ }
+}
+
+export default GenerateSpectrogram;