browser audio detection phase 3

This commit is contained in:
Joren de Graaf
2026-10-02 21:34:38 +02:00
parent 5c0ca7bc47
commit 20bdabdd8f
50 changed files with 24755 additions and 144 deletions
@@ -3,82 +3,104 @@ const PREFIX = "[BoomCast][BrowserBeta]";
export function readBrowserProcessingConfig() {
const element = document.querySelector("#boomcast-browser-config");
if (!element) return { browserProcessingBeta: false };
try {
return JSON.parse(element.textContent);
} catch (error) {
try { return JSON.parse(element.textContent); }
catch (error) {
console.error(`${PREFIX} Configuration unavailable:`, error);
return { browserProcessingBeta: false };
}
}
export class BrowserSubmissionProcessor {
constructor(config = readBrowserProcessingConfig(), extractVideo = null) {
constructor(config = readBrowserProcessingConfig(), extractVideo = null, dependencies = {}) {
this.config = config;
this.extractVideo = extractVideo;
this.dependencies = dependencies;
}
isEnabled() {
return this.config.browserProcessingBeta === true;
}
isEnabled() { return this.config.browserProcessingBeta === true; }
canProcess(file) {
return this.isEnabled() && Boolean(file) && /^(audio|video)\//.test(file.type);
return this.isEnabled() && Boolean(file) && (BrowserSubmissionProcessor.isAudio(file) || BrowserSubmissionProcessor.isVideo(file));
}
static isAudio(file) {
return Boolean(file) && (file.type.startsWith("audio/") || (!file.type && /\.(aac|aif|aiff|flac|m4a|mp3|ogg|opus|wav|weba)$/i.test(file.name || "")));
}
static isVideo(file) {
return Boolean(file) && (file.type.startsWith("video/") || (!file.type && /\.(mp4|mov|m4v|webm|mkv|avi)$/i.test(file.name || "")));
}
async prepare(file, { signal, onState } = {}) {
if (!this.isEnabled()) return { file, metadata: null };
async prepare(file, { signal, onState = () => {} } = {}) {
if (!this.canProcess(file)) return { file, metadata: null, gate: "unavailable" };
console.log(`${PREFIX} Initialising beta submission pipeline.`);
console.log(`${PREFIX} File selected:`, { type: file?.type || "", size: file?.size || 0 });
if (!this.canProcess(file) && !BrowserSubmissionProcessor.isVideo(file)) {
console.log(`${PREFIX} Browser preprocessing unavailable; falling back.`);
return { file, metadata: null };
}
let policyTimeout;
const video = BrowserSubmissionProcessor.isVideo(file);
const controller = new AbortController();
const abort = () => controller.abort();
signal?.addEventListener("abort", abort, { once: true });
const timeout = setTimeout(() => controller.abort(), 5000);
let policy;
try {
const controller = new AbortController();
policyTimeout = setTimeout(() => controller.abort(), 5000);
const response = await fetch(this.config.policyUrl, {
cache: "no-store", credentials: "same-origin", signal: controller.signal,
});
if (!response.ok) throw new Error(`Policy HTTP ${response.status}`);
const policy = await response.json();
policy = await response.json();
if (!policy.browser_processing_beta_enabled || policy.version !== this.config.policyVersion) {
console.log(`${PREFIX} Browser beta disabled or policy changed; using legacy submission flow.`);
return { file, metadata: null };
return { file, metadata: null, gate: "unavailable" };
}
if (BrowserSubmissionProcessor.isVideo(file)) {
const extract = this.extractVideo || (await import("./ffmpeg-video-extractor.js")).extractVideoAudio;
const wav = await extract(file, { signal, onState });
return { file: wav, metadata: { browser_processing_used: true, browser_processing_version: policy.version, browser_gate_result: "unavailable", browser_override: false } };
}
// Audio continues through the existing submission path.
console.log(`${PREFIX} Browser processing shell ready; using original recording.`);
return {
file,
metadata: {
browser_processing_used: false,
browser_processing_version: policy.version,
browser_gate_result: "unavailable",
browser_override: false,
},
};
} catch (error) {
console.error(`${PREFIX} Initialisation failed:`, error);
this.cleanup();
if (BrowserSubmissionProcessor.isVideo(file)) throw error;
console.log(`${PREFIX} Restoring legacy submission flow.`);
return { file, metadata: null };
if (signal?.aborted) throw error;
console.error(`${PREFIX} Policy unavailable:`, error);
if (video) throw error;
return { file, metadata: null, gate: "unavailable" };
} finally {
clearTimeout(policyTimeout);
clearTimeout(timeout);
signal?.removeEventListener("abort", abort);
}
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
let preparedFile = file;
if (video) {
const extract = this.extractVideo || (await import("./ffmpeg-video-extractor.js")).extractVideoAudio;
preparedFile = await extract(file, { signal, onState });
}
const metadata = {
browser_processing_used: video,
browser_processing_version: policy.version,
browser_gate_result: "unavailable",
browser_override: false,
};
try {
onState("checking");
const prepareAudio = this.dependencies.prepareAudio || (await import("./audio-preprocessor.js")).AudioPreprocessor.prepare;
let waveform = await prepareAudio(preparedFile, { signal });
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
try {
const classify = this.dependencies.classify || (await import("./browser-yamnet-classifier.js")).BrowserYamnetClassifier.classify;
const predictions = await classify(waveform, {
modelUrl: policy.model_url, classMapUrl: policy.class_map_url, signal,
});
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
const evaluate = this.dependencies.evaluate || (await import("./browser-submission-gate.js")).BrowserSubmissionGate.evaluate;
const decision = evaluate(predictions, policy);
metadata.browser_processing_used = true;
metadata.browser_gate_result = decision.result;
metadata.browser_speech_detected = decision.speechDetected;
return { file: preparedFile, metadata, gate: decision.result, predictions: predictions.slice(0, 5) };
} finally {
waveform = null;
console.log("[BoomCast][Cleanup] Releasing waveform buffers.");
}
} catch (error) {
if (signal?.aborted) throw error;
console.error("[BoomCast][YAMNet] Local classification unavailable:", error);
console.log("[BoomCast][Gate] Falling back to server classification.");
metadata.browser_gate_result = "unavailable";
return { file: preparedFile, metadata, gate: "unavailable" };
}
}
cleanup() {
// Future passes can release workers, object URLs, and model tensors here.
console.log("[BoomCast][Cleanup] Browser beta resources released.");
}
cleanup() { console.log("[BoomCast][Cleanup] Browser beta resources released."); }
}