browser audio detection phase 3
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
const PREFIX = "[BoomCast][YAMNet]";
|
||||
|
||||
export class BrowserYamnetClassifier {
|
||||
static async classify(waveform, { modelUrl, classMapUrl, signal, loadTensorFlow } = {}) {
|
||||
let tf;
|
||||
let model;
|
||||
let input;
|
||||
let output;
|
||||
const started = performance.now();
|
||||
try {
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
console.log(`${PREFIX} Lazy-loading TensorFlow.js.`);
|
||||
tf = loadTensorFlow ? await loadTensorFlow() : await import("@tensorflow/tfjs");
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
console.log(`${PREFIX} TensorFlow.js loaded.`);
|
||||
await tf.ready();
|
||||
console.log(`${PREFIX} Backend:`, tf.getBackend());
|
||||
console.log(`${PREFIX} Loading model.`);
|
||||
const [loaded, mappingResponse] = await Promise.all([
|
||||
tf.loadGraphModel(modelUrl),
|
||||
fetch(classMapUrl, { cache: "force-cache", signal }),
|
||||
]);
|
||||
model = loaded;
|
||||
if (!mappingResponse.ok) throw new Error("YAMNet class map unavailable");
|
||||
const classMap = await mappingResponse.json();
|
||||
if (!Array.isArray(classMap) || classMap.length !== 521) throw new Error("Invalid YAMNet class map");
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
console.log(`${PREFIX} Model ready.`);
|
||||
// YAMNet needs at least one 0.96 s patch, including STFT padding.
|
||||
const samples = waveform.length < 15600 ? Float32Array.from({ length: 15600 }, (_, i) => waveform[i] || 0) : waveform;
|
||||
input = tf.tensor1d(samples, "float32");
|
||||
console.log(`${PREFIX} Starting inference.`);
|
||||
const result = await model.executeAsync(input);
|
||||
output = Array.isArray(result) ? result : [result];
|
||||
const scores = output.find((tensor) => tensor.shape.length === 2 && tensor.shape[1] === classMap.length);
|
||||
if (!scores) throw new Error("YAMNet score output unavailable");
|
||||
const values = await scores.data();
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
const frames = scores.shape[0];
|
||||
const predictions = classMap.map((name, index) => {
|
||||
let score = -1;
|
||||
let frame = 0;
|
||||
for (let row = 0; row < frames; row += 1) {
|
||||
const value = values[row * classMap.length + index];
|
||||
if (value > score) { score = value; frame = row; }
|
||||
}
|
||||
return { class: name, score, frame, timeSeconds: Number((frame * 0.48).toFixed(2)) };
|
||||
});
|
||||
predictions.sort((a, b) => b.score - a.score);
|
||||
console.log(`${PREFIX} Inference completed in`, Math.round(performance.now() - started), "ms.");
|
||||
console.groupCollapsed(`${PREFIX} Top local predictions`);
|
||||
console.table(predictions.slice(0, 10));
|
||||
console.groupEnd();
|
||||
return predictions;
|
||||
} catch (error) {
|
||||
console.error(`${PREFIX} Local inference failed:`, error);
|
||||
throw error;
|
||||
} finally {
|
||||
if (tf) console.log(`${PREFIX} TensorFlow memory before cleanup:`, tf.memory());
|
||||
console.log("[BoomCast][Cleanup] Disposing YAMNet tensors.");
|
||||
input?.dispose();
|
||||
output?.forEach((tensor) => tensor?.dispose());
|
||||
console.log("[BoomCast][Cleanup] Disposing browser YAMNet model.");
|
||||
model?.dispose();
|
||||
console.log("[BoomCast][Cleanup] Releasing waveform buffers.");
|
||||
if (tf) console.log(`${PREFIX} TensorFlow memory after cleanup:`, tf.memory());
|
||||
console.log("[BoomCast][Cleanup] Browser classification cleanup complete.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user