Files
FireworksApp/frontend/src/scripts/browser-yamnet-classifier.js
T

71 lines
3.8 KiB
JavaScript

const PREFIX = "[BoomCast][YAMNet]";
export class BrowserYamnetClassifier {
static async classify(waveform, { modelUrl, classMapUrl, signal, loadTensorFlow } = {}) {
let tf;
let model;
let input;
let output;
const started = performance.now();
try {
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log(`${PREFIX} Lazy-loading TensorFlow.js.`);
tf = loadTensorFlow ? await loadTensorFlow() : await import("@tensorflow/tfjs");
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log(`${PREFIX} TensorFlow.js loaded.`);
await tf.ready();
console.log(`${PREFIX} Backend:`, tf.getBackend());
console.log(`${PREFIX} Loading model.`);
const [loaded, mappingResponse] = await Promise.all([
tf.loadGraphModel(modelUrl),
fetch(classMapUrl, { cache: "force-cache", signal }),
]);
model = loaded;
if (!mappingResponse.ok) throw new Error("YAMNet class map unavailable");
const classMap = await mappingResponse.json();
if (!Array.isArray(classMap) || classMap.length !== 521) throw new Error("Invalid YAMNet class map");
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log(`${PREFIX} Model ready.`);
// YAMNet needs at least one 0.96 s patch, including STFT padding.
const samples = waveform.length < 15600 ? Float32Array.from({ length: 15600 }, (_, i) => waveform[i] || 0) : waveform;
input = tf.tensor1d(samples, "float32");
console.log(`${PREFIX} Starting inference.`);
const result = await model.executeAsync(input);
output = Array.isArray(result) ? result : [result];
const scores = output.find((tensor) => tensor.shape.length === 2 && tensor.shape[1] === classMap.length);
if (!scores) throw new Error("YAMNet score output unavailable");
const values = await scores.data();
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
const frames = scores.shape[0];
const predictions = classMap.map((name, index) => {
let score = -1;
let frame = 0;
for (let row = 0; row < frames; row += 1) {
const value = values[row * classMap.length + index];
if (value > score) { score = value; frame = row; }
}
return { class: name, score, frame, timeSeconds: Number((frame * 0.48).toFixed(2)) };
});
predictions.sort((a, b) => b.score - a.score);
console.log(`${PREFIX} Inference completed in`, Math.round(performance.now() - started), "ms.");
console.groupCollapsed(`${PREFIX} Top local predictions`);
console.table(predictions.slice(0, 10));
console.groupEnd();
return predictions;
} catch (error) {
console.error(`${PREFIX} Local inference failed:`, error);
throw error;
} finally {
if (tf) console.log(`${PREFIX} TensorFlow memory before cleanup:`, tf.memory());
console.log("[BoomCast][Cleanup] Disposing YAMNet tensors.");
input?.dispose();
output?.forEach((tensor) => tensor?.dispose());
console.log("[BoomCast][Cleanup] Disposing browser YAMNet model.");
model?.dispose();
console.log("[BoomCast][Cleanup] Releasing waveform buffers.");
if (tf) console.log(`${PREFIX} TensorFlow memory after cleanup:`, tf.memory());
console.log("[BoomCast][Cleanup] Browser classification cleanup complete.");
}
}
}