browser audio detection phase 3

This commit is contained in:
Joren de Graaf
2026-10-02 21:34:38 +02:00
parent c68e4b95ba
commit 1de4b8a0b3
50 changed files with 24755 additions and 144 deletions
@@ -0,0 +1,31 @@
const RATE = 16000;
export class AudioPreprocessor {
static async prepare(file, { signal } = {}) {
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log("[BoomCast][YAMNet] Preparing local classification.");
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
if (!AudioContextClass || !(window.OfflineAudioContext || window.webkitOfflineAudioContext)) throw new Error("Browser audio decoding unavailable");
const context = new AudioContextClass();
try {
const decoded = await context.decodeAudioData(await file.arrayBuffer());
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
if (!decoded.duration || !Number.isFinite(decoded.duration)) throw new Error("Invalid audio duration");
const OfflineContextClass = window.OfflineAudioContext || window.webkitOfflineAudioContext;
const offline = new OfflineContextClass(1, Math.ceil(decoded.duration * RATE), RATE);
const source = offline.createBufferSource();
source.buffer = decoded;
source.connect(offline.destination);
source.start();
const rendered = await offline.startRendering();
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
const waveform = rendered.getChannelData(0).slice();
console.log("[BoomCast][YAMNet] Waveform prepared.", {
sampleRate: RATE, samples: waveform.length, durationSeconds: waveform.length / RATE,
});
return waveform;
} finally {
await context.close().catch(() => {});
}
}
}
+96 -68
View File
@@ -12,6 +12,11 @@ if (form) {
const submit = form.querySelector('button[type="submit"]');
const cancel = form.querySelector(".upload-cancel");
const results = form.querySelector("#classification-results");
const gatePanel = form.querySelector("#browser-gate");
const gateTitle = form.querySelector("#browser-gate-title");
const gateMessage = form.querySelector("#browser-gate-message");
const gateChoose = form.querySelector("#browser-gate-choose");
const gateOverride = form.querySelector("#browser-gate-override");
const status = document.querySelector("#upload-status");
const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)");
const states = ["idle", "file-selected", "uploading", "processing", "success", "error", "cancelled"];
@@ -23,8 +28,9 @@ if (form) {
let selectionId = 0;
let selectionController = null;
let selectedResult = null;
let extractionPending = false;
let extractionFailed = false;
let preparationPending = false;
let preparationFailed = false;
let gateBlocked = false;
const browserProcessor = new BrowserSubmissionProcessor();
if (!browserProcessor.isEnabled()) {
console.log("[BoomCast][BrowserBeta] Feature disabled; using legacy submission flow.");
@@ -130,41 +136,68 @@ if (form) {
selectionController?.abort();
selectionController = null;
selectedResult = null;
extractionPending = false;
extractionFailed = false;
preparationPending = false;
preparationFailed = false;
gateBlocked = false;
gatePanel.hidden = true;
showFile(file);
if (!busy) { submit.disabled = false; cancel.hidden = true; }
if (!file || !browserProcessor.isEnabled() || !BrowserSubmissionProcessor.isVideo(file)) return;
if (!file || !browserProcessor.canProcess(file)) return;
const id = selectionId;
const controller = new AbortController();
selectionController = controller;
extractionPending = true;
preparationPending = true;
submit.disabled = true;
cancel.hidden = false;
status.textContent = t.extractingAudio || "Extracting audio from video…";
status.textContent = BrowserSubmissionProcessor.isVideo(file)
? (t.extractingAudio || "Extracting audio from video…")
: (t.preparingRecording || "Preparing recording…");
browserProcessor.prepare(file, { signal: controller.signal, onState: (state) => {
if (id === selectionId) console.log("[BoomCast][FFmpeg] State:", state);
if (id !== selectionId) return;
status.textContent = state === "checking" ? (t.checkingRecording || "Checking recording…")
: (t.extractingAudio || "Extracting audio from video…");
} }).then((result) => {
if (id !== selectionId || controller.signal.aborted) return;
selectedResult = result;
extractionPending = false;
submit.disabled = false;
preparationPending = false;
cancel.hidden = true;
status.textContent = result.file === file
? (t.videoExtractionUnavailable || "Video processing is unavailable. Please choose an audio recording.")
: (t.onlyAudioSent || "Audio extracted on this device. Only audio will be sent to BoomCast.");
extractionFailed = result.file === file;
if (extractionFailed) submit.disabled = true;
if (BrowserSubmissionProcessor.isVideo(file) && result.file === file) {
preparationFailed = true;
submit.disabled = true;
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
return;
}
selectedResult = result;
gateBlocked = result.gate === "speech_only" || result.gate === "no_match";
if (gateBlocked) {
gateTitle.textContent = result.gate === "speech_only"
? (t.speechGateTitle || "This recording appears to contain speech, but no relevant sound was detected.")
: (t.irrelevantGateTitle || "No relevant sound was detected.");
gateMessage.textContent = result.gate === "speech_only"
? (t.speechGateMessage || "To protect people’s privacy, this recording will not be uploaded automatically. You can still submit it for manual review.")
: (t.irrelevantGateMessage || "BoomCast could not identify a sound this service currently looks for. You can still submit it for manual review.");
gatePanel.hidden = false;
submit.disabled = true;
status.textContent = (t.gateChooseMessage || "Recording checked. Choose another recording or submit for manual review.");
} else {
submit.disabled = false;
status.textContent = BrowserSubmissionProcessor.isVideo(file)
? (t.videoAudioOnly || "Video processed on this device. Only extracted audio will be sent if you submit.")
: (t.readyToSubmit || "Ready to submit.");
}
}).catch((error) => {
if (id !== selectionId) return;
console.error("[BoomCast][FFmpeg] Falling back safely.", error);
extractionPending = false;
extractionFailed = true;
submit.disabled = true;
console.error("[BoomCast][BrowserBeta] Preparation failed:", error);
preparationPending = false;
cancel.hidden = true;
status.textContent = controller.signal.aborted
? (t.extractionCancelled || "Audio extraction cancelled. Choose a recording to continue.")
: (t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.");
if (BrowserSubmissionProcessor.isVideo(file)) {
preparationFailed = true;
submit.disabled = true;
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
} else {
selectedResult = { file, metadata: null, gate: "unavailable" };
submit.disabled = false;
status.textContent = t.readyToSubmit || "Ready to submit.";
}
});
}
@@ -184,6 +217,22 @@ if (form) {
});
function startUpload(file, metadata = null) {
// HTMX's later Pending action also reads the native file input. Replace a
// video with its extracted WAV so no subsequent form action can send video.
if (fileInput.files[0] !== file) {
try {
const transfer = new DataTransfer();
transfer.items.add(file);
fileInput.files = transfer.files;
if (fileInput.files[0] !== file) throw new Error("Prepared audio could not replace video in form");
} catch (error) {
console.error("[BoomCast][BrowserBeta] Could not secure native form upload:", error);
preparationFailed = true;
submit.disabled = true;
status.textContent = (t.secureVideoFailed || "Could not prepare this video for upload. Please choose an audio recording.");
return;
}
}
const formData = new FormData(form);
formData.set("audio", file);
if (metadata) formData.set("browser_processing_metadata", JSON.stringify(metadata));
@@ -226,59 +275,36 @@ if (form) {
form.addEventListener("submit", (event) => {
event.preventDefault();
event.stopImmediatePropagation();
if (busy || preparing || extractionPending || extractionFailed) return;
if (busy || preparationPending || preparationFailed || gateBlocked) return;
if (!form.reportValidity()) return;
const file = fileInput.files[0];
if (!file) return;
if (selectedResult && BrowserSubmissionProcessor.isVideo(file)) {
startUpload(selectedResult.file, selectedResult.metadata);
return;
}
if (!browserProcessor.isEnabled()) {
startUpload(file);
return;
}
const id = selectionId;
preparing = true;
submit.disabled = true;
status.textContent = t.preparingRecording || "Preparing recording…";
browserProcessor.prepare(file).then(({ file: preparedFile, metadata }) => {
preparing = false;
submit.disabled = false;
if (fileInput.files[0] !== file || selectionId !== id) return;
if (BrowserSubmissionProcessor.isVideo(file) && preparedFile === file) {
extractionFailed = true;
submit.disabled = true;
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
return;
}
startUpload(preparedFile || file, metadata);
}).catch((error) => {
console.error("[BoomCast][BrowserBeta] Initialisation failed:", error);
browserProcessor.cleanup();
console.log("[BoomCast][BrowserBeta] Restoring legacy submission flow.");
preparing = false;
submit.disabled = false;
if (fileInput.files[0] !== file || selectionId !== id) return;
if (BrowserSubmissionProcessor.isVideo(file)) {
extractionFailed = true;
submit.disabled = true;
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
return;
}
if (fileInput.files[0] === file && selectionId === id) startUpload(file);
});
if (selectedResult) { startUpload(selectedResult.file, selectedResult.metadata); return; }
if (!browserProcessor.isEnabled()) { startUpload(file); return; }
// Unknown types retain the existing server flow.
startUpload(file);
}, true);
gateChoose.addEventListener("click", () => fileInput.click());
gateOverride.addEventListener("click", () => {
if (!gateBlocked || !selectedResult || busy || preparationPending) return;
console.warn("[BoomCast][Gate] User explicitly overrode local gate.");
console.log("[BoomCast][Gate] Submission will enter manual review.");
const metadata = { ...selectedResult.metadata, browser_override: true };
selectedResult.metadata = metadata;
gateBlocked = false;
gatePanel.hidden = true;
startUpload(selectedResult.file, metadata);
});
cancel.addEventListener("click", () => {
if (extractionPending) {
if (preparationPending) {
selectionController?.abort();
selectionId += 1;
extractionPending = false;
extractionFailed = true;
preparationPending = false;
preparationFailed = true;
submit.disabled = true;
cancel.hidden = true;
status.textContent = t.extractionCancelled || "Audio extraction cancelled. Choose a recording to continue.";
status.textContent = t.extractionCancelled || (t.preparationCancelled || "Preparation cancelled. Choose a recording to continue.");
} else request?.abort();
});
window.addEventListener("pageshow", (event) => {
@@ -287,8 +313,10 @@ if (form) {
selectionController?.abort();
selectionId += 1;
selectedResult = null;
extractionPending = false;
extractionFailed = false;
preparationPending = false;
preparationFailed = false;
gateBlocked = false;
gatePanel.hidden = true;
submit.disabled = false;
fileInput.value = "";
results.replaceChildren();
@@ -0,0 +1,17 @@
export class BrowserSubmissionGate {
static evaluate(predictions, policy) {
console.log("[BoomCast][Gate] Evaluating browser classification.");
const relevant = new Set((policy.whitelist || []).map((name) => name.toLowerCase()));
const speechNames = new Set((policy.privacy_sensitive_speech || []).map((name) => name.toLowerCase()));
const matchThreshold = Number(policy.thresholds?.relevance ?? 0.5);
const speechThreshold = Number(policy.thresholds?.speech ?? 0.5);
const matches = predictions.filter((item) => relevant.has(item.class.toLowerCase()) && item.score >= matchThreshold);
const speechDetected = predictions.some((item) => speechNames.has(item.class.toLowerCase()) && item.score >= speechThreshold);
console.log("[BoomCast][Gate] Relevant whitelist matches:", matches);
console.log("[BoomCast][Gate] Speech detected:", speechDetected);
const result = matches.length ? "whitelist_match" : speechDetected ? "speech_only" : "no_match";
console.log(`[BoomCast][Gate] Decision: ${result.toUpperCase()}`);
if (result !== "whitelist_match") console.log("[BoomCast][Gate] Automatic upload suppressed.");
return { result, speechDetected, matches };
}
}
@@ -3,82 +3,104 @@ const PREFIX = "[BoomCast][BrowserBeta]";
export function readBrowserProcessingConfig() {
const element = document.querySelector("#boomcast-browser-config");
if (!element) return { browserProcessingBeta: false };
try {
return JSON.parse(element.textContent);
} catch (error) {
try { return JSON.parse(element.textContent); }
catch (error) {
console.error(`${PREFIX} Configuration unavailable:`, error);
return { browserProcessingBeta: false };
}
}
export class BrowserSubmissionProcessor {
constructor(config = readBrowserProcessingConfig(), extractVideo = null) {
constructor(config = readBrowserProcessingConfig(), extractVideo = null, dependencies = {}) {
this.config = config;
this.extractVideo = extractVideo;
this.dependencies = dependencies;
}
isEnabled() {
return this.config.browserProcessingBeta === true;
}
isEnabled() { return this.config.browserProcessingBeta === true; }
canProcess(file) {
return this.isEnabled() && Boolean(file) && /^(audio|video)\//.test(file.type);
return this.isEnabled() && Boolean(file) && (BrowserSubmissionProcessor.isAudio(file) || BrowserSubmissionProcessor.isVideo(file));
}
static isAudio(file) {
return Boolean(file) && (file.type.startsWith("audio/") || (!file.type && /\.(aac|aif|aiff|flac|m4a|mp3|ogg|opus|wav|weba)$/i.test(file.name || "")));
}
static isVideo(file) {
return Boolean(file) && (file.type.startsWith("video/") || (!file.type && /\.(mp4|mov|m4v|webm|mkv|avi)$/i.test(file.name || "")));
}
async prepare(file, { signal, onState } = {}) {
if (!this.isEnabled()) return { file, metadata: null };
async prepare(file, { signal, onState = () => {} } = {}) {
if (!this.canProcess(file)) return { file, metadata: null, gate: "unavailable" };
console.log(`${PREFIX} Initialising beta submission pipeline.`);
console.log(`${PREFIX} File selected:`, { type: file?.type || "", size: file?.size || 0 });
if (!this.canProcess(file) && !BrowserSubmissionProcessor.isVideo(file)) {
console.log(`${PREFIX} Browser preprocessing unavailable; falling back.`);
return { file, metadata: null };
}
let policyTimeout;
const video = BrowserSubmissionProcessor.isVideo(file);
const controller = new AbortController();
const abort = () => controller.abort();
signal?.addEventListener("abort", abort, { once: true });
const timeout = setTimeout(() => controller.abort(), 5000);
let policy;
try {
const controller = new AbortController();
policyTimeout = setTimeout(() => controller.abort(), 5000);
const response = await fetch(this.config.policyUrl, {
cache: "no-store", credentials: "same-origin", signal: controller.signal,
});
if (!response.ok) throw new Error(`Policy HTTP ${response.status}`);
const policy = await response.json();
policy = await response.json();
if (!policy.browser_processing_beta_enabled || policy.version !== this.config.policyVersion) {
console.log(`${PREFIX} Browser beta disabled or policy changed; using legacy submission flow.`);
return { file, metadata: null };
return { file, metadata: null, gate: "unavailable" };
}
if (BrowserSubmissionProcessor.isVideo(file)) {
const extract = this.extractVideo || (await import("./ffmpeg-video-extractor.js")).extractVideoAudio;
const wav = await extract(file, { signal, onState });
return { file: wav, metadata: { browser_processing_used: true, browser_processing_version: policy.version, browser_gate_result: "unavailable", browser_override: false } };
}
// Audio continues through the existing submission path.
console.log(`${PREFIX} Browser processing shell ready; using original recording.`);
return {
file,
metadata: {
browser_processing_used: false,
browser_processing_version: policy.version,
browser_gate_result: "unavailable",
browser_override: false,
},
};
} catch (error) {
console.error(`${PREFIX} Initialisation failed:`, error);
this.cleanup();
if (BrowserSubmissionProcessor.isVideo(file)) throw error;
console.log(`${PREFIX} Restoring legacy submission flow.`);
return { file, metadata: null };
if (signal?.aborted) throw error;
console.error(`${PREFIX} Policy unavailable:`, error);
if (video) throw error;
return { file, metadata: null, gate: "unavailable" };
} finally {
clearTimeout(policyTimeout);
clearTimeout(timeout);
signal?.removeEventListener("abort", abort);
}
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
let preparedFile = file;
if (video) {
const extract = this.extractVideo || (await import("./ffmpeg-video-extractor.js")).extractVideoAudio;
preparedFile = await extract(file, { signal, onState });
}
const metadata = {
browser_processing_used: video,
browser_processing_version: policy.version,
browser_gate_result: "unavailable",
browser_override: false,
};
try {
onState("checking");
const prepareAudio = this.dependencies.prepareAudio || (await import("./audio-preprocessor.js")).AudioPreprocessor.prepare;
let waveform = await prepareAudio(preparedFile, { signal });
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
try {
const classify = this.dependencies.classify || (await import("./browser-yamnet-classifier.js")).BrowserYamnetClassifier.classify;
const predictions = await classify(waveform, {
modelUrl: policy.model_url, classMapUrl: policy.class_map_url, signal,
});
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
const evaluate = this.dependencies.evaluate || (await import("./browser-submission-gate.js")).BrowserSubmissionGate.evaluate;
const decision = evaluate(predictions, policy);
metadata.browser_processing_used = true;
metadata.browser_gate_result = decision.result;
metadata.browser_speech_detected = decision.speechDetected;
return { file: preparedFile, metadata, gate: decision.result, predictions: predictions.slice(0, 5) };
} finally {
waveform = null;
console.log("[BoomCast][Cleanup] Releasing waveform buffers.");
}
} catch (error) {
if (signal?.aborted) throw error;
console.error("[BoomCast][YAMNet] Local classification unavailable:", error);
console.log("[BoomCast][Gate] Falling back to server classification.");
metadata.browser_gate_result = "unavailable";
return { file: preparedFile, metadata, gate: "unavailable" };
}
}
cleanup() {
// Future passes can release workers, object URLs, and model tensors here.
console.log("[BoomCast][Cleanup] Browser beta resources released.");
}
cleanup() { console.log("[BoomCast][Cleanup] Browser beta resources released."); }
}
@@ -0,0 +1,70 @@
const PREFIX = "[BoomCast][YAMNet]";
export class BrowserYamnetClassifier {
static async classify(waveform, { modelUrl, classMapUrl, signal, loadTensorFlow } = {}) {
let tf;
let model;
let input;
let output;
const started = performance.now();
try {
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log(`${PREFIX} Lazy-loading TensorFlow.js.`);
tf = loadTensorFlow ? await loadTensorFlow() : await import("@tensorflow/tfjs");
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log(`${PREFIX} TensorFlow.js loaded.`);
await tf.ready();
console.log(`${PREFIX} Backend:`, tf.getBackend());
console.log(`${PREFIX} Loading model.`);
const [loaded, mappingResponse] = await Promise.all([
tf.loadGraphModel(modelUrl),
fetch(classMapUrl, { cache: "force-cache", signal }),
]);
model = loaded;
if (!mappingResponse.ok) throw new Error("YAMNet class map unavailable");
const classMap = await mappingResponse.json();
if (!Array.isArray(classMap) || classMap.length !== 521) throw new Error("Invalid YAMNet class map");
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
console.log(`${PREFIX} Model ready.`);
// YAMNet needs at least one 0.96 s patch, including STFT padding.
const samples = waveform.length < 15600 ? Float32Array.from({ length: 15600 }, (_, i) => waveform[i] || 0) : waveform;
input = tf.tensor1d(samples, "float32");
console.log(`${PREFIX} Starting inference.`);
const result = await model.executeAsync(input);
output = Array.isArray(result) ? result : [result];
const scores = output.find((tensor) => tensor.shape.length === 2 && tensor.shape[1] === classMap.length);
if (!scores) throw new Error("YAMNet score output unavailable");
const values = await scores.data();
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
const frames = scores.shape[0];
const predictions = classMap.map((name, index) => {
let score = -1;
let frame = 0;
for (let row = 0; row < frames; row += 1) {
const value = values[row * classMap.length + index];
if (value > score) { score = value; frame = row; }
}
return { class: name, score, frame, timeSeconds: Number((frame * 0.48).toFixed(2)) };
});
predictions.sort((a, b) => b.score - a.score);
console.log(`${PREFIX} Inference completed in`, Math.round(performance.now() - started), "ms.");
console.groupCollapsed(`${PREFIX} Top local predictions`);
console.table(predictions.slice(0, 10));
console.groupEnd();
return predictions;
} catch (error) {
console.error(`${PREFIX} Local inference failed:`, error);
throw error;
} finally {
if (tf) console.log(`${PREFIX} TensorFlow memory before cleanup:`, tf.memory());
console.log("[BoomCast][Cleanup] Disposing YAMNet tensors.");
input?.dispose();
output?.forEach((tensor) => tensor?.dispose());
console.log("[BoomCast][Cleanup] Disposing browser YAMNet model.");
model?.dispose();
console.log("[BoomCast][Cleanup] Releasing waveform buffers.");
if (tf) console.log(`${PREFIX} TensorFlow memory after cleanup:`, tf.memory());
console.log("[BoomCast][Cleanup] Browser classification cleanup complete.");
}
}
}
@@ -509,3 +509,9 @@
transition-duration: .01ms !important;
}
}
.browser-gate { margin: 1rem 0; padding: 1rem; border: 1px solid #e5bb7a; border-radius: .75rem; background: #4d392950; }
.browser-gate[hidden] { display: none; }
.browser-gate p { margin: .45rem 0 1rem; }
.browser-gate-actions { display: flex; gap: .65rem; flex-wrap: wrap; }
.browser-gate-actions button { cursor: pointer; }
@@ -0,0 +1,64 @@
import assert from "node:assert/strict";
import test from "node:test";
import { BrowserSubmissionGate } from "../src/scripts/browser-submission-gate.js";
import { BrowserSubmissionProcessor } from "../src/scripts/browser-submission-processor.js";
const policy = { whitelist: ["Fireworks"], privacy_sensitive_speech: ["Speech"], thresholds: { relevance: .5, speech: .5 } };
test("whitelist match allows normal submission even when speech is present", () => {
const decision = BrowserSubmissionGate.evaluate([{ class: "Fireworks", score: .8 }, { class: "Speech", score: .9 }], policy);
assert.equal(decision.result, "whitelist_match");
assert.equal(decision.speechDetected, true);
});
test("speech without relevance and irrelevant audio require explicit override", () => {
assert.equal(BrowserSubmissionGate.evaluate([{ class: "Speech", score: .9 }], policy).result, "speech_only");
assert.equal(BrowserSubmissionGate.evaluate([{ class: "Music", score: .9 }], policy).result, "no_match");
});
test("audio and video-derived WAV use the same gate; audio never calls FFmpeg", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => ({ ok: true, json: async () => ({ ...policy, browser_processing_beta_enabled: true, version: 1 }) });
let extractionCalls = 0;
let classificationCalls = 0;
const processor = new BrowserSubmissionProcessor(
{ browserProcessingBeta: true, policyUrl: "/policy", policyVersion: 1 },
async () => { extractionCalls += 1; return new File(["wav"], "clip.wav", { type: "audio/wav" }); },
{ prepareAudio: async () => new Float32Array(16000), classify: async () => { classificationCalls += 1; return [{ class: "Speech", score: .9 }]; } },
);
try {
assert.equal((await processor.prepare(new File(["audio"], "audio.wav", { type: "audio/wav" }))).gate, "speech_only");
assert.equal(extractionCalls, 0);
assert.equal((await processor.prepare(new File(["video"], "clip.mp4", { type: "video/mp4" }))).gate, "speech_only");
assert.equal(extractionCalls, 1);
assert.equal(classificationCalls, 2);
} finally { globalThis.fetch = originalFetch; }
});
test("local classifier failure restores ordinary server upload", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => ({ ok: true, json: async () => ({ ...policy, browser_processing_beta_enabled: true, version: 1 }) });
const processor = new BrowserSubmissionProcessor(
{ browserProcessingBeta: true, policyUrl: "/policy", policyVersion: 1 }, null,
{ prepareAudio: async () => new Float32Array(16000), classify: async () => { throw new Error("offline"); } },
);
const audio = new File(["audio"], "audio.wav", { type: "audio/wav" });
try {
const result = await processor.prepare(audio);
assert.equal(result.file, audio);
assert.equal(result.gate, "unavailable");
} finally { globalThis.fetch = originalFetch; }
});
test("aborted classification never yields a gate decision", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => ({ ok: true, json: async () => ({ ...policy, browser_processing_beta_enabled: true, version: 1 }) });
const controller = new AbortController();
const processor = new BrowserSubmissionProcessor(
{ browserProcessingBeta: true, policyUrl: "/policy", policyVersion: 1 }, null,
{ prepareAudio: async () => new Float32Array(16000), classify: async () => { controller.abort(); return [{ class: "Fireworks", score: .9 }]; } },
);
try {
await assert.rejects(processor.prepare(new File(["audio"], "audio.wav", { type: "audio/wav" }), { signal: controller.signal }));
} finally { globalThis.fetch = originalFetch; }
});
@@ -12,7 +12,7 @@ test("disabled beta returns the original file without loading policy", async ()
const processor = new BrowserSubmissionProcessor({ browserProcessingBeta: false });
assert.equal(processor.isEnabled(), false);
assert.equal(processor.canProcess(file), false);
assert.deepEqual(await processor.prepare(file), { file, metadata: null });
assert.deepEqual(await processor.prepare(file), { file, metadata: null, gate: "unavailable" });
} finally {
globalThis.fetch = originalFetch;
}
@@ -37,9 +37,9 @@ test("admin-disabled policy and fetch failure both fall back to the original fil
const processor = new BrowserSubmissionProcessor({ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 });
try {
globalThis.fetch = async () => ({ ok: true, json: async () => ({ browser_processing_beta_enabled: false, version: 1 }) });
assert.deepEqual(await processor.prepare(file), { file, metadata: null });
assert.deepEqual(await processor.prepare(file), { file, metadata: null, gate: "unavailable" });
globalThis.fetch = async () => { throw new Error("offline"); };
assert.deepEqual(await processor.prepare(file), { file, metadata: null });
assert.deepEqual(await processor.prepare(file), { file, metadata: null, gate: "unavailable" });
} finally {
globalThis.fetch = originalFetch;
}
@@ -53,6 +53,7 @@ test("video routing is gated and audio never invokes FFmpeg", async () => {
const processor = new BrowserSubmissionProcessor(
{ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 },
async () => { calls += 1; return new File(["wav"], "clip.wav", { type: "audio/wav" }); },
{ prepareAudio: async () => new Float32Array(16000), classify: async () => [{ class: "Fireworks", score: .9 }] },
);
await processor.prepare(file);
assert.equal(calls, 0);
@@ -62,7 +63,7 @@ test("video routing is gated and audio never invokes FFmpeg", async () => {
assert.equal(result.file.type, "audio/wav");
assert.equal(result.metadata.browser_processing_used, true);
const disabled = new BrowserSubmissionProcessor({ browserProcessingBeta: false }, async () => { throw new Error("must not run"); });
assert.deepEqual(await disabled.prepare(video), { file: video, metadata: null });
assert.deepEqual(await disabled.prepare(video), { file: video, metadata: null, gate: "unavailable" });
} finally {
globalThis.fetch = originalFetch;
}
@@ -0,0 +1,45 @@
import assert from "node:assert/strict";
import test from "node:test";
import { BrowserYamnetClassifier } from "../src/scripts/browser-yamnet-classifier.js";
test("YAMNet peak scores are mapped and all owned resources are disposed", async () => {
const originalFetch = globalThis.fetch;
const disposed = [];
const values = new Float32Array(2 * 521);
values[0] = .4;
values[521] = .9;
const output = { shape: [2, 521], data: async () => values, dispose: () => disposed.push("output") };
const model = { executeAsync: async () => [output], dispose: () => disposed.push("model") };
const tf = {
ready: async () => {}, getBackend: () => "cpu", memory: () => ({ numTensors: 0 }),
loadGraphModel: async () => model,
tensor1d: () => ({ dispose: () => disposed.push("input") }),
};
globalThis.fetch = async () => ({ ok: true, json: async () => ["Fireworks", ...Array(520).fill("Other")] });
try {
const predictions = await BrowserYamnetClassifier.classify(new Float32Array(16000), {
modelUrl: "/static/browser-yamnet/model.json", classMapUrl: "/static/browser-yamnet/yamnet_class_map.json",
loadTensorFlow: async () => tf,
});
assert.equal(predictions[0].class, "Fireworks");
assert.equal(predictions[0].score, values[521]);
assert.equal(predictions[0].timeSeconds, .48);
assert.deepEqual(disposed, ["input", "output", "model"]);
} finally { globalThis.fetch = originalFetch; }
});
test("model failure still disposes the loaded model", async () => {
const originalFetch = globalThis.fetch;
let disposed = false;
const tf = {
ready: async () => {}, getBackend: () => "cpu", memory: () => ({ numTensors: 0 }),
loadGraphModel: async () => ({ dispose: () => { disposed = true; } }),
};
globalThis.fetch = async () => ({ ok: false });
try {
await assert.rejects(BrowserYamnetClassifier.classify(new Float32Array(16000), {
modelUrl: "/model.json", classMapUrl: "/map.json", loadTensorFlow: async () => tf,
}));
assert.equal(disposed, true);
} finally { globalThis.fetch = originalFetch; }
});