browser audio detection phase 4

This commit is contained in:
Joren de Graaf
2026-10-02 21:45:24 +02:00
parent 1de4b8a0b3
commit 46ad514a70
20 changed files with 903 additions and 31 deletions
+111 -28
View File
@@ -1,5 +1,3 @@
import { BrowserSubmissionProcessor } from "./browser-submission-processor.js";
const form = document.querySelector(".upload-form");
if (form) {
@@ -19,21 +17,49 @@ if (form) {
const gateOverride = form.querySelector("#browser-gate-override");
const status = document.querySelector("#upload-status");
const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)");
const browserToggle = form.querySelector("#browser-processing-toggle");
const configElement = document.querySelector("#boomcast-browser-config");
let browserConfig = { browserProcessingBeta: false };
try { browserConfig = JSON.parse(configElement?.textContent || "{}"); }
catch (error) { console.error("[BoomCast][BrowserBeta] Configuration unavailable:", error); }
const browserAvailable = browserConfig.browserProcessingBeta === true && Boolean(browserToggle);
if (browserToggle) browserToggle.checked = false;
const audioAccept = fileInput.accept;
const videoAccept = ",video/*,.mp4,.mov,.m4v,.webm,.mkv,.avi";
const browserModeEnabled = () => browserAvailable && browserToggle.checked;
const isVideoFile = (file) => Boolean(file) &&
(file.type.startsWith("video/") || (!file.type && /\.(mp4|mov|m4v|webm|mkv|avi)$/i.test(file.name || "")));
let browserProcessor = null;
let browserProcessorPromise = null;
let sourceFile = null;
async function getBrowserProcessor() {
if (browserProcessor) return browserProcessor;
browserProcessorPromise ||= import("./browser-submission-processor.js").then(({ BrowserSubmissionProcessor }) => {
browserProcessor = new BrowserSubmissionProcessor(browserConfig);
console.log("[BoomCast][BrowserBeta] Coordinator initialised.");
return browserProcessor;
}).catch((error) => {
browserProcessorPromise = null;
throw error;
});
return browserProcessorPromise;
}
const states = ["idle", "file-selected", "uploading", "processing", "success", "error", "cancelled"];
let request = null;
let busy = false;
let uploadComplete = false;
let processingTimer = null;
let preparing = false;
let selectionId = 0;
let selectionController = null;
let selectedResult = null;
let preparationPending = false;
let preparationFailed = false;
let gateBlocked = false;
const browserProcessor = new BrowserSubmissionProcessor();
if (!browserProcessor.isEnabled()) {
console.log("[BoomCast][BrowserBeta] Feature disabled; using legacy submission flow.");
if (!browserAvailable) {
console.log("[BoomCast][BrowserBeta] Feature disabled globally; using existing server flow.");
} else {
console.log("[BoomCast][BrowserBeta] Browser processing disabled by user; using existing server flow.");
}
function setUploadState(state, { progress = null, message = "" } = {}) {
@@ -42,6 +68,7 @@ if (form) {
busy = active;
submit.disabled = active;
fileInput.disabled = active;
if (browserToggle) browserToggle.disabled = active;
cancel.hidden = !active;
if (active) form.setAttribute("aria-busy", "true");
else form.removeAttribute("aria-busy");
@@ -134,33 +161,50 @@ if (form) {
function selectFile(file) {
selectionId += 1;
selectionController?.abort();
browserProcessor?.cancel();
selectionController = null;
selectedResult = null;
preparationPending = false;
preparationFailed = false;
gateBlocked = false;
gatePanel.hidden = true;
if (!busy) results.replaceChildren();
showFile(file);
if (!busy) { submit.disabled = false; cancel.hidden = true; }
if (!file || !browserProcessor.canProcess(file)) return;
if (!file) return;
console.log("[BoomCast][BrowserBeta] File selected.", { mimeType: file.type, sizeBytes: file.size });
if (!browserModeEnabled()) {
console.log("[BoomCast][BrowserBeta] Server mode selected.");
if (isVideoFile(file)) {
preparationFailed = true;
submit.disabled = true;
status.textContent = t.serverAudioOnly || "Server processing currently accepts audio recordings. Choose an audio file or enable browser processing.";
}
return;
}
console.log("[BoomCast][BrowserBeta] Browser mode selected.");
const id = selectionId;
const controller = new AbortController();
selectionController = controller;
preparationPending = true;
submit.disabled = true;
cancel.hidden = false;
status.textContent = BrowserSubmissionProcessor.isVideo(file)
status.textContent = isVideoFile(file)
? (t.extractingAudio || "Extracting audio from video…")
: (t.preparingRecording || "Preparing recording…");
browserProcessor.prepare(file, { signal: controller.signal, onState: (state) => {
if (id !== selectionId) return;
status.textContent = state === "checking" ? (t.checkingRecording || "Checking recording…")
: (t.extractingAudio || "Extracting audio from video…");
} }).then((result) => {
if (id !== selectionId || controller.signal.aborted) return;
getBrowserProcessor().then((processor) => {
if (id !== selectionId || controller.signal.aborted || !browserModeEnabled()) return null;
processor.setEnabled(true);
return processor.process(file, { signal: controller.signal, onState: (state) => {
if (id !== selectionId) return;
status.textContent = state === "checking" ? (t.checkingRecording || "Checking recording…")
: (t.extractingAudio || "Extracting audio from video…");
} });
}).then((result) => {
if (!result || id !== selectionId || controller.signal.aborted || !browserModeEnabled()) return;
preparationPending = false;
cancel.hidden = true;
if (BrowserSubmissionProcessor.isVideo(file) && result.file === file) {
if (isVideoFile(file) && result.file === file) {
preparationFailed = true;
submit.disabled = true;
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
@@ -177,31 +221,64 @@ if (form) {
: (t.irrelevantGateMessage || "BoomCast could not identify a sound this service currently looks for. You can still submit it for manual review.");
gatePanel.hidden = false;
submit.disabled = true;
status.textContent = (t.gateChooseMessage || "Recording checked. Choose another recording or submit for manual review.");
status.textContent = t.gateChooseMessage || "Recording checked. Choose another recording or submit for manual review.";
} else {
submit.disabled = false;
status.textContent = BrowserSubmissionProcessor.isVideo(file)
? (t.videoAudioOnly || "Video processed on this device. Only extracted audio will be sent if you submit.")
: (t.readyToSubmit || "Ready to submit.");
status.textContent = result.gate === "unavailable"
? (t.localUnavailable || "Local checking was unavailable. BoomCast will use normal server processing.")
: isVideoFile(file)
? (t.videoAudioOnly || "Video processed on this device. Only extracted audio will be sent if you submit.")
: (t.readyToSubmit || "Ready to submit.");
}
}).catch((error) => {
if (id !== selectionId) return;
console.error("[BoomCast][BrowserBeta] Preparation failed:", error);
if (id !== selectionId || !browserModeEnabled()) return;
console.error("[BoomCast][BrowserBeta] Unexpected browser pipeline error:", error);
preparationPending = false;
cancel.hidden = true;
if (BrowserSubmissionProcessor.isVideo(file)) {
if (isVideoFile(file)) {
preparationFailed = true;
submit.disabled = true;
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
} else {
console.warn("[BoomCast][BrowserBeta] Browser pipeline unavailable; falling back to server.");
selectedResult = { file, metadata: null, gate: "unavailable" };
submit.disabled = false;
status.textContent = t.readyToSubmit || "Ready to submit.";
status.textContent = t.localUnavailable || "Local checking was unavailable. BoomCast will use normal server processing.";
}
});
}
fileInput.addEventListener("change", () => selectFile(fileInput.files[0]));
function restoreOriginalFile() {
if (!sourceFile || fileInput.files[0] === sourceFile) return;
try {
const transfer = new DataTransfer();
transfer.items.add(sourceFile);
fileInput.files = transfer.files;
} catch (error) {
console.error("[BoomCast][Cleanup] Could not restore original selection:", error);
fileInput.value = "";
sourceFile = null;
}
}
browserToggle?.addEventListener("change", () => {
if (browserModeEnabled()) {
console.log("[BoomCast][BrowserBeta] Browser processing enabled by user.");
fileInput.accept = audioAccept + videoAccept;
selectFile(sourceFile || fileInput.files[0]);
} else {
console.log("[BoomCast][BrowserBeta] Switching to server processing.");
console.log("[BoomCast][Cleanup] Cancelling active browser processing job.");
selectionController?.abort();
browserProcessor?.setEnabled(false);
browserProcessor?.cleanup();
fileInput.accept = audioAccept;
restoreOriginalFile();
selectFile(sourceFile || fileInput.files[0]);
}
});
fileInput.addEventListener("change", () => { sourceFile = fileInput.files[0] || null; selectFile(sourceFile); });
["dragenter", "dragover"].forEach((name) => dropZone.addEventListener(name, (event) => {
event.preventDefault();
if (!busy) dropZone.classList.add("dragging");
@@ -213,7 +290,8 @@ if (form) {
dropZone.addEventListener("drop", (event) => {
if (busy || !event.dataTransfer.files.length) return;
fileInput.files = event.dataTransfer.files;
selectFile(fileInput.files[0]);
sourceFile = fileInput.files[0] || null;
selectFile(sourceFile);
});
function startUpload(file, metadata = null) {
@@ -279,15 +357,15 @@ if (form) {
if (!form.reportValidity()) return;
const file = fileInput.files[0];
if (!file) return;
if (!browserModeEnabled()) { startUpload(file); return; }
if (selectedResult) { startUpload(selectedResult.file, selectedResult.metadata); return; }
if (!browserProcessor.isEnabled()) { startUpload(file); return; }
// Unknown types retain the existing server flow.
// Unknown audio types retain the existing server flow.
startUpload(file);
}, true);
gateChoose.addEventListener("click", () => fileInput.click());
gateOverride.addEventListener("click", () => {
if (!gateBlocked || !selectedResult || busy || preparationPending) return;
if (!browserModeEnabled() || !gateBlocked || !selectedResult || busy || preparationPending) return;
console.warn("[BoomCast][Gate] User explicitly overrode local gate.");
console.log("[BoomCast][Gate] Submission will enter manual review.");
const metadata = { ...selectedResult.metadata, browser_override: true };
@@ -299,6 +377,7 @@ if (form) {
cancel.addEventListener("click", () => {
if (preparationPending) {
selectionController?.abort();
browserProcessor?.cancel();
selectionId += 1;
preparationPending = false;
preparationFailed = true;
@@ -311,8 +390,12 @@ if (form) {
if (event.persisted) {
request?.abort();
selectionController?.abort();
browserProcessor?.cancel();
selectionId += 1;
selectedResult = null;
sourceFile = null;
if (browserToggle) { browserToggle.checked = false; fileInput.accept = audioAccept; }
browserProcessor?.setEnabled(false);
preparationPending = false;
preparationFailed = false;
gateBlocked = false;
@@ -15,9 +15,39 @@ export class BrowserSubmissionProcessor {
this.config = config;
this.extractVideo = extractVideo;
this.dependencies = dependencies;
this.userEnabled = false;
this.activeController = null;
}
isEnabled() { return this.config.browserProcessingBeta === true; }
isAvailable() { return this.config.browserProcessingBeta === true; }
isEnabled() { return this.isAvailable() && this.userEnabled; }
setEnabled(enabled) {
if (!enabled) this.cancel();
this.userEnabled = this.isAvailable() && enabled === true;
console.log(`${PREFIX} ${this.userEnabled ? "Browser processing enabled by user." : "Browser processing disabled by user; using existing server flow."}`);
}
cancel() {
this.activeController?.abort();
this.activeController = null;
}
async process(file, { signal, onState } = {}) {
this.cancel();
const controller = new AbortController();
this.activeController = controller;
const abort = () => controller.abort();
signal?.addEventListener("abort", abort, { once: true });
if (signal?.aborted) controller.abort();
try {
return await this.prepare(file, { signal: controller.signal, onState });
} finally {
signal?.removeEventListener("abort", abort);
if (this.activeController === controller) this.activeController = null;
}
}
canProcess(file) {
return this.isEnabled() && Boolean(file) && (BrowserSubmissionProcessor.isAudio(file) || BrowserSubmissionProcessor.isVideo(file));
@@ -102,5 +132,8 @@ export class BrowserSubmissionProcessor {
}
}
cleanup() { console.log("[BoomCast][Cleanup] Browser beta resources released."); }
cleanup() {
this.cancel();
console.log("[BoomCast][Cleanup] Browser beta resources released.");
}
}
@@ -515,3 +515,8 @@
.browser-gate p { margin: .45rem 0 1rem; }
.browser-gate-actions { display: flex; gap: .65rem; flex-wrap: wrap; }
.browser-gate-actions button { cursor: pointer; }
.browser-mode { margin: 1rem 0; }
.browser-mode__label { display: inline-flex; align-items: center; gap: .55rem; font-weight: 700; cursor: pointer; }
.browser-mode__label input { width: 1.1rem; height: 1.1rem; margin: 0; accent-color: #91baff; }
.browser-mode p { margin: .35rem 0 0 1.65rem; font-size: .9rem; opacity: .85; }
+84
View File
@@ -0,0 +1,84 @@
import assert from "node:assert/strict";
import test from "node:test";
import { JSDOM } from "jsdom";
function fixture() {
const dom = new JSDOM(`<!doctype html><html><body>
<script id="boomcast-browser-config" type="application/json">{"browserProcessingBeta":true,"policyUrl":"/policy","policyVersion":1}</script>
<form class="upload-form" action="/classify">
<label id="drop-zone"><input id="audio-file" name="audio" type="file" accept="audio/*,.wav" required><span id="upload-progress-circle"><span id="upload-icon-content"></span></span><span id="file-name"></span></label>
<input id="browser-processing-toggle" type="checkbox">
<button type="submit">Submit</button><button class="upload-cancel" type="button" hidden>Cancel</button>
<section id="browser-gate" hidden><strong id="browser-gate-title"></strong><p id="browser-gate-message"></p><button id="browser-gate-choose" type="button"></button><button id="browser-gate-override" type="button"></button></section>
<section id="classification-results"></section>
</form><p id="upload-status"></p></body></html>`, { url: "https://example.test/" });
const { window } = dom;
window.matchMedia = () => ({ matches: false });
window.BOOMCAST_I18N = {};
const requests = [];
class MockXHR {
upload = { addEventListener() {} };
addEventListener() {}
open() {}
send(data) { requests.push(data); }
}
const originals = Object.fromEntries(["window", "document", "FormData", "XMLHttpRequest", "DOMParser", "fetch"].map((name) => [name, globalThis[name]]));
Object.assign(globalThis, { window, document: window.document, FormData: window.FormData, XMLHttpRequest: MockXHR, DOMParser: window.DOMParser });
const form = window.document.querySelector("form");
form.reportValidity = () => true;
const input = window.document.querySelector("#audio-file");
const audio = new window.File(["audio"], "clip.wav", { type: "audio/wav" });
Object.defineProperty(input, "files", { configurable: true, value: [audio] });
const toggle = window.document.querySelector("#browser-processing-toggle");
return { window, form, input, audio, toggle, requests, restore: () => {
for (const [name, value] of Object.entries(originals)) {
if (value === undefined) delete globalThis[name];
else globalThis[name] = value;
}
dom.window.close();
} };
}
test("admin ON with user toggle OFF uploads the original audio without policy fetch", async () => {
const ui = fixture();
let fetches = 0;
globalThis.fetch = () => { fetches += 1; throw new Error("browser policy must not load"); };
try {
await import("../src/scripts/audio-upload.js?mode=server");
assert.equal(ui.toggle.checked, false);
ui.input.dispatchEvent(new ui.window.Event("change", { bubbles: true }));
assert.equal(ui.form.querySelector('button[type="submit"]').disabled, false);
ui.form.dispatchEvent(new ui.window.Event("submit", { bubbles: true, cancelable: true }));
assert.equal(fetches, 0);
assert.equal(ui.requests.length, 1);
assert.equal(ui.requests[0].get("audio").name, "clip.wav");
assert.equal(ui.requests[0].has("browser_processing_metadata"), false);
} finally { ui.restore(); }
});
test("switching browser mode off invalidates a policy job and restores server submission", async () => {
const ui = fixture();
let resolveFetch;
let fetches = 0;
globalThis.fetch = () => { fetches += 1; return new Promise((resolve) => { resolveFetch = resolve; }); };
try {
await import("../src/scripts/audio-upload.js?mode=switch");
ui.input.dispatchEvent(new ui.window.Event("change", { bubbles: true }));
ui.toggle.checked = true;
ui.toggle.dispatchEvent(new ui.window.Event("change", { bubbles: true }));
for (let i = 0; i < 30 && !resolveFetch; i += 1) await new Promise((resolve) => setTimeout(resolve, 5));
assert.equal(fetches, 1);
assert.equal(ui.form.querySelector('button[type="submit"]').disabled, true);
ui.toggle.checked = false;
ui.toggle.dispatchEvent(new ui.window.Event("change", { bubbles: true }));
resolveFetch({ ok: true, json: async () => ({ browser_processing_beta_enabled: true, version: 1 }) });
await new Promise((resolve) => setTimeout(resolve, 0));
assert.equal(ui.input.accept, "audio/*,.wav");
assert.equal(ui.form.querySelector('button[type="submit"]').disabled, false);
assert.equal(ui.window.document.querySelector("#browser-gate").hidden, true);
ui.form.dispatchEvent(new ui.window.Event("submit", { bubbles: true, cancelable: true }));
assert.equal(ui.requests.length, 1);
assert.equal(ui.requests[0].get("audio").name, "clip.wav");
assert.equal(ui.requests[0].has("browser_processing_metadata"), false);
} finally { ui.restore(); }
});
@@ -26,6 +26,7 @@ test("audio and video-derived WAV use the same gate; audio never calls FFmpeg",
async () => { extractionCalls += 1; return new File(["wav"], "clip.wav", { type: "audio/wav" }); },
{ prepareAudio: async () => new Float32Array(16000), classify: async () => { classificationCalls += 1; return [{ class: "Speech", score: .9 }]; } },
);
processor.setEnabled(true);
try {
assert.equal((await processor.prepare(new File(["audio"], "audio.wav", { type: "audio/wav" }))).gate, "speech_only");
assert.equal(extractionCalls, 0);
@@ -43,6 +44,7 @@ test("local classifier failure restores ordinary server upload", async () => {
{ prepareAudio: async () => new Float32Array(16000), classify: async () => { throw new Error("offline"); } },
);
const audio = new File(["audio"], "audio.wav", { type: "audio/wav" });
processor.setEnabled(true);
try {
const result = await processor.prepare(audio);
assert.equal(result.file, audio);
@@ -58,7 +60,40 @@ test("aborted classification never yields a gate decision", async () => {
{ browserProcessingBeta: true, policyUrl: "/policy", policyVersion: 1 }, null,
{ prepareAudio: async () => new Float32Array(16000), classify: async () => { controller.abort(); return [{ class: "Fireworks", score: .9 }]; } },
);
processor.setEnabled(true);
try {
await assert.rejects(processor.prepare(new File(["audio"], "audio.wav", { type: "audio/wav" }), { signal: controller.signal }));
} finally { globalThis.fetch = originalFetch; }
});
test("replacing a classification cancels the old result and keeps the new file", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => ({ ok: true, json: async () => ({ ...policy, browser_processing_beta_enabled: true, version: 1 }) });
let firstStarted;
const started = new Promise((resolve) => { firstStarted = resolve; });
let finishFirst;
let calls = 0;
const processor = new BrowserSubmissionProcessor(
{ browserProcessingBeta: true, policyUrl: "/policy", policyVersion: 1 }, null,
{ prepareAudio: async () => new Float32Array(16000), classify: async () => {
calls += 1;
if (calls === 1) {
firstStarted();
return new Promise((resolve) => { finishFirst = resolve; });
}
return [{ class: "Fireworks", score: .9 }];
} },
);
processor.setEnabled(true);
const firstFile = new File(["a"], "first.wav", { type: "audio/wav" });
const secondFile = new File(["b"], "second.wav", { type: "audio/wav" });
try {
const first = processor.process(firstFile);
await started;
const second = await processor.process(secondFile);
assert.equal(second.file, secondFile);
assert.equal(second.gate, "whitelist_match");
finishFirst([{ class: "Speech", score: .9 }]);
await assert.rejects(first);
} finally { globalThis.fetch = originalFetch; }
});
@@ -10,6 +10,7 @@ test("disabled beta returns the original file without loading policy", async ()
globalThis.fetch = () => { throw new Error("fetch must not run"); };
try {
const processor = new BrowserSubmissionProcessor({ browserProcessingBeta: false });
processor.setEnabled(true);
assert.equal(processor.isEnabled(), false);
assert.equal(processor.canProcess(file), false);
assert.deepEqual(await processor.prepare(file), { file, metadata: null, gate: "unavailable" });
@@ -23,6 +24,7 @@ test("enabled shell checks current server policy and never claims processing occ
globalThis.fetch = async () => ({ ok: true, json: async () => ({ browser_processing_beta_enabled: true, version: 1 }) });
try {
const processor = new BrowserSubmissionProcessor({ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 });
processor.setEnabled(true);
const result = await processor.prepare(file);
assert.equal(result.file, file);
assert.equal(result.metadata.browser_processing_used, false);
@@ -35,6 +37,7 @@ test("enabled shell checks current server policy and never claims processing occ
test("admin-disabled policy and fetch failure both fall back to the original file", async () => {
const originalFetch = globalThis.fetch;
const processor = new BrowserSubmissionProcessor({ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 });
processor.setEnabled(true);
try {
globalThis.fetch = async () => ({ ok: true, json: async () => ({ browser_processing_beta_enabled: false, version: 1 }) });
assert.deepEqual(await processor.prepare(file), { file, metadata: null, gate: "unavailable" });
@@ -55,6 +58,7 @@ test("video routing is gated and audio never invokes FFmpeg", async () => {
async () => { calls += 1; return new File(["wav"], "clip.wav", { type: "audio/wav" }); },
{ prepareAudio: async () => new Float32Array(16000), classify: async () => [{ class: "Fireworks", score: .9 }] },
);
processor.setEnabled(true);
await processor.prepare(file);
assert.equal(calls, 0);
const video = { type: "video/mp4", size: 123, name: "clip.mp4" };
@@ -77,8 +81,32 @@ test("video conversion failure rejects instead of falling back to original video
{ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 },
async () => { throw new Error("decode failed"); },
);
processor.setEnabled(true);
await assert.rejects(processor.prepare({ type: "video/mp4", name: "bad.mp4" }), /decode failed/);
} finally {
globalThis.fetch = originalFetch;
}
});
test("admin ON and user toggle OFF bypasses policy, FFmpeg and TensorFlow", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = () => { throw new Error("policy must not load"); };
const audio = new File(["audio"], "clip.wav", { type: "audio/wav" });
const processor = new BrowserSubmissionProcessor(
{ browserProcessingBeta: true, policyUrl: "/policy", policyVersion: 1 },
async () => { throw new Error("FFmpeg must not load"); },
{ prepareAudio: async () => { throw new Error("audio preparation must not run"); },
classify: async () => { throw new Error("TensorFlow must not load"); } },
);
try {
assert.equal(processor.isAvailable(), true);
assert.equal(processor.isEnabled(), false);
assert.deepEqual(await processor.process(audio), { file: audio, metadata: null, gate: "unavailable" });
processor.setEnabled(true);
assert.equal(processor.isEnabled(), true);
processor.setEnabled(false);
assert.equal(processor.isEnabled(), false);
assert.deepEqual(await processor.process(audio), { file: audio, metadata: null, gate: "unavailable" });
} finally { globalThis.fetch = originalFetch; }
});