browser audio detection phase 3

This commit is contained in:
Joren de Graaf
2026-10-02 21:34:38 +02:00
parent 5c0ca7bc47
commit 20bdabdd8f
50 changed files with 24755 additions and 144 deletions
@@ -0,0 +1,70 @@
const PREFIX = "[BoomCast][YAMNet]";
export class BrowserYamnetClassifier {
static async classify(waveform, { modelUrl, classMapUrl, signal, loadTensorFlow } = {}) {
let tf;
let model;
let input;
let output;
const started = performance.now();
try {
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log(`${PREFIX} Lazy-loading TensorFlow.js.`);
tf = loadTensorFlow ? await loadTensorFlow() : await import("@tensorflow/tfjs");
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log(`${PREFIX} TensorFlow.js loaded.`);
await tf.ready();
console.log(`${PREFIX} Backend:`, tf.getBackend());
console.log(`${PREFIX} Loading model.`);
const [loaded, mappingResponse] = await Promise.all([
tf.loadGraphModel(modelUrl),
fetch(classMapUrl, { cache: "force-cache", signal }),
]);
model = loaded;
if (!mappingResponse.ok) throw new Error("YAMNet class map unavailable");
const classMap = await mappingResponse.json();
if (!Array.isArray(classMap) || classMap.length !== 521) throw new Error("Invalid YAMNet class map");
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log(`${PREFIX} Model ready.`);
// YAMNet needs at least one 0.96 s patch, including STFT padding.
const samples = waveform.length < 15600 ? Float32Array.from({ length: 15600 }, (_, i) => waveform[i] || 0) : waveform;
input = tf.tensor1d(samples, "float32");
console.log(`${PREFIX} Starting inference.`);
const result = await model.executeAsync(input);
output = Array.isArray(result) ? result : [result];
const scores = output.find((tensor) => tensor.shape.length === 2 && tensor.shape[1] === classMap.length);
if (!scores) throw new Error("YAMNet score output unavailable");
const values = await scores.data();
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
const frames = scores.shape[0];
const predictions = classMap.map((name, index) => {
let score = -1;
let frame = 0;
for (let row = 0; row < frames; row += 1) {
const value = values[row * classMap.length + index];
if (value > score) { score = value; frame = row; }
}
return { class: name, score, frame, timeSeconds: Number((frame * 0.48).toFixed(2)) };
});
predictions.sort((a, b) => b.score - a.score);
console.log(`${PREFIX} Inference completed in`, Math.round(performance.now() - started), "ms.");
console.groupCollapsed(`${PREFIX} Top local predictions`);
console.table(predictions.slice(0, 10));
console.groupEnd();
return predictions;
} catch (error) {
console.error(`${PREFIX} Local inference failed:`, error);
throw error;
} finally {
if (tf) console.log(`${PREFIX} TensorFlow memory before cleanup:`, tf.memory());
console.log("[BoomCast][Cleanup] Disposing YAMNet tensors.");
input?.dispose();
output?.forEach((tensor) => tensor?.dispose());
console.log("[BoomCast][Cleanup] Disposing browser YAMNet model.");
model?.dispose();
console.log("[BoomCast][Cleanup] Releasing waveform buffers.");
if (tf) console.log(`${PREFIX} TensorFlow memory after cleanup:`, tf.memory());
console.log("[BoomCast][Cleanup] Browser classification cleanup complete.");
}
}
}