browser audio detection phase 2

This commit is contained in:
Joren de Graaf
2026-10-02 16:57:12 +02:00
parent beeb2b5e0e
commit c68e4b95ba
17 changed files with 334 additions and 10 deletions
+88 -5
View File
@@ -20,6 +20,11 @@ if (form) {
let uploadComplete = false;
let processingTimer = null;
let preparing = false;
let selectionId = 0;
let selectionController = null;
let selectedResult = null;
let extractionPending = false;
let extractionFailed = false;
const browserProcessor = new BrowserSubmissionProcessor();
if (!browserProcessor.isEnabled()) {
console.log("[BoomCast][BrowserBeta] Feature disabled; using legacy submission flow.");
@@ -120,7 +125,50 @@ if (form) {
}
}
fileInput.addEventListener("change", () => showFile(fileInput.files[0]));
function selectFile(file) {
selectionId += 1;
selectionController?.abort();
selectionController = null;
selectedResult = null;
extractionPending = false;
extractionFailed = false;
showFile(file);
if (!busy) { submit.disabled = false; cancel.hidden = true; }
if (!file || !browserProcessor.isEnabled() || !BrowserSubmissionProcessor.isVideo(file)) return;
const id = selectionId;
const controller = new AbortController();
selectionController = controller;
extractionPending = true;
submit.disabled = true;
cancel.hidden = false;
status.textContent = t.extractingAudio || "Extracting audio from video…";
browserProcessor.prepare(file, { signal: controller.signal, onState: (state) => {
if (id === selectionId) console.log("[BoomCast][FFmpeg] State:", state);
} }).then((result) => {
if (id !== selectionId || controller.signal.aborted) return;
selectedResult = result;
extractionPending = false;
submit.disabled = false;
cancel.hidden = true;
status.textContent = result.file === file
? (t.videoExtractionUnavailable || "Video processing is unavailable. Please choose an audio recording.")
: (t.onlyAudioSent || "Audio extracted on this device. Only audio will be sent to BoomCast.");
extractionFailed = result.file === file;
if (extractionFailed) submit.disabled = true;
}).catch((error) => {
if (id !== selectionId) return;
console.error("[BoomCast][FFmpeg] Falling back safely.", error);
extractionPending = false;
extractionFailed = true;
submit.disabled = true;
cancel.hidden = true;
status.textContent = controller.signal.aborted
? (t.extractionCancelled || "Audio extraction cancelled. Choose a recording to continue.")
: (t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.");
});
}
fileInput.addEventListener("change", () => selectFile(fileInput.files[0]));
["dragenter", "dragover"].forEach((name) => dropZone.addEventListener(name, (event) => {
event.preventDefault();
if (!busy) dropZone.classList.add("dragging");
@@ -132,7 +180,7 @@ if (form) {
dropZone.addEventListener("drop", (event) => {
if (busy || !event.dataTransfer.files.length) return;
fileInput.files = event.dataTransfer.files;
showFile(fileInput.files[0]);
selectFile(fileInput.files[0]);
});
function startUpload(file, metadata = null) {
@@ -178,20 +226,32 @@ if (form) {
form.addEventListener("submit", (event) => {
event.preventDefault();
event.stopImmediatePropagation();
if (busy || preparing) return;
if (busy || preparing || extractionPending || extractionFailed) return;
if (!form.reportValidity()) return;
const file = fileInput.files[0];
if (!file) return;
if (selectedResult && BrowserSubmissionProcessor.isVideo(file)) {
startUpload(selectedResult.file, selectedResult.metadata);
return;
}
if (!browserProcessor.isEnabled()) {
startUpload(file);
return;
}
const id = selectionId;
preparing = true;
submit.disabled = true;
status.textContent = t.preparingRecording || "Preparing recording…";
browserProcessor.prepare(file).then(({ file: preparedFile, metadata }) => {
preparing = false;
submit.disabled = false;
if (fileInput.files[0] !== file || selectionId !== id) return;
if (BrowserSubmissionProcessor.isVideo(file) && preparedFile === file) {
extractionFailed = true;
submit.disabled = true;
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
return;
}
startUpload(preparedFile || file, metadata);
}).catch((error) => {
console.error("[BoomCast][BrowserBeta] Initialisation failed:", error);
@@ -199,14 +259,37 @@ if (form) {
console.log("[BoomCast][BrowserBeta] Restoring legacy submission flow.");
preparing = false;
submit.disabled = false;
startUpload(file);
if (fileInput.files[0] !== file || selectionId !== id) return;
if (BrowserSubmissionProcessor.isVideo(file)) {
extractionFailed = true;
submit.disabled = true;
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
return;
}
if (fileInput.files[0] === file && selectionId === id) startUpload(file);
});
}, true);
cancel.addEventListener("click", () => request?.abort());
cancel.addEventListener("click", () => {
if (extractionPending) {
selectionController?.abort();
selectionId += 1;
extractionPending = false;
extractionFailed = true;
submit.disabled = true;
cancel.hidden = true;
status.textContent = t.extractionCancelled || "Audio extraction cancelled. Choose a recording to continue.";
} else request?.abort();
});
window.addEventListener("pageshow", (event) => {
if (event.persisted) {
request?.abort();
selectionController?.abort();
selectionId += 1;
selectedResult = null;
extractionPending = false;
extractionFailed = false;
submit.disabled = false;
fileInput.value = "";
results.replaceChildren();
showFile();
@@ -12,8 +12,9 @@ export function readBrowserProcessingConfig() {
}
export class BrowserSubmissionProcessor {
constructor(config = readBrowserProcessingConfig()) {
constructor(config = readBrowserProcessingConfig(), extractVideo = null) {
this.config = config;
this.extractVideo = extractVideo;
}
isEnabled() {
@@ -24,11 +25,15 @@ export class BrowserSubmissionProcessor {
return this.isEnabled() && Boolean(file) && /^(audio|video)\//.test(file.type);
}
async prepare(file) {
static isVideo(file) {
return Boolean(file) && (file.type.startsWith("video/") || (!file.type && /\.(mp4|mov|m4v|webm|mkv|avi)$/i.test(file.name || "")));
}
async prepare(file, { signal, onState } = {}) {
if (!this.isEnabled()) return { file, metadata: null };
console.log(`${PREFIX} Initialising beta submission pipeline.`);
console.log(`${PREFIX} File selected:`, { type: file?.type || "", size: file?.size || 0 });
if (!this.canProcess(file)) {
if (!this.canProcess(file) && !BrowserSubmissionProcessor.isVideo(file)) {
console.log(`${PREFIX} Browser preprocessing unavailable; falling back.`);
return { file, metadata: null };
}
@@ -45,7 +50,12 @@ export class BrowserSubmissionProcessor {
console.log(`${PREFIX} Browser beta disabled or policy changed; using legacy submission flow.`);
return { file, metadata: null };
}
// Pass 1 has no local classifier or transcoder. Keep the original file.
if (BrowserSubmissionProcessor.isVideo(file)) {
const extract = this.extractVideo || (await import("./ffmpeg-video-extractor.js")).extractVideoAudio;
const wav = await extract(file, { signal, onState });
return { file: wav, metadata: { browser_processing_used: true, browser_processing_version: policy.version, browser_gate_result: "unavailable", browser_override: false } };
}
// Audio continues through the existing submission path.
console.log(`${PREFIX} Browser processing shell ready; using original recording.`);
return {
file,
@@ -59,6 +69,7 @@ export class BrowserSubmissionProcessor {
} catch (error) {
console.error(`${PREFIX} Initialisation failed:`, error);
this.cleanup();
if (BrowserSubmissionProcessor.isVideo(file)) throw error;
console.log(`${PREFIX} Restoring legacy submission flow.`);
return { file, metadata: null };
} finally {
@@ -0,0 +1,82 @@
const PREFIX = "[BoomCast][FFmpeg]";
export class VideoExtractionError extends Error {
constructor(code, cause) {
super(code, { cause });
this.name = "VideoExtractionError";
this.code = code;
}
}
export async function extractVideoAudio(file, { signal, onState = () => {}, createFFmpeg } = {}) {
let ffmpeg;
let inputWritten = false;
let outputWritten = false;
let stage = "module_load_failed";
const started = performance.now();
const inputName = "input";
const outputName = "audio.wav";
const abort = () => ffmpeg?.terminate();
signal?.addEventListener("abort", abort, { once: true });
console.log(`${PREFIX} Video input detected.`, { mimeType: file.type, sizeBytes: file.size });
try {
if (signal?.aborted) throw new VideoExtractionError("cancelled");
onState("loading_ffmpeg");
console.log(`${PREFIX} Starting lazy import.`);
const factory = createFFmpeg || (async () => {
const [{ FFmpeg }, { default: coreURL }, { default: wasmURL }] = await Promise.all([
import("@ffmpeg/ffmpeg"),
import("@ffmpeg/core?url"),
import("@ffmpeg/core/wasm?url"),
]);
console.log(`${PREFIX} ffmpeg.wasm module imported.`);
return { ffmpeg: new FFmpeg(), coreURL, wasmURL };
});
const runtime = await factory();
ffmpeg = runtime.ffmpeg;
if (signal?.aborted) throw new VideoExtractionError("cancelled");
ffmpeg.on?.("log", ({ message }) => console.log(`${PREFIX}[core]`, message));
stage = "core_load_failed";
console.log(`${PREFIX} Loading FFmpeg core.`);
await ffmpeg.load({ coreURL: runtime.coreURL, wasmURL: runtime.wasmURL });
console.log(`${PREFIX} FFmpeg core ready.`);
if (signal?.aborted) throw new VideoExtractionError("cancelled");
onState("extracting_audio");
stage = "resource_failed";
console.log(`${PREFIX} Writing input to virtual filesystem.`);
// One input ArrayBuffer is unavoidable for the FFmpeg virtual filesystem.
inputWritten = true;
await ffmpeg.writeFile(inputName, new Uint8Array(await file.arrayBuffer()));
if (signal?.aborted) throw new VideoExtractionError("cancelled");
stage = "extraction_failed";
console.log(`${PREFIX} Starting audio extraction: mono / 16kHz WAV.`);
outputWritten = true;
const exitCode = await ffmpeg.exec(["-i", inputName, "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", outputName]);
if (exitCode !== 0) throw new VideoExtractionError("invalid_media");
if (signal?.aborted) throw new VideoExtractionError("cancelled");
const data = await ffmpeg.readFile(outputName);
if (!data?.byteLength) throw new VideoExtractionError("invalid_media");
if (data.byteLength > 25 * 1024 * 1024) throw new VideoExtractionError("output_too_large");
const wav = new File([data], `${file.name.replace(/\.[^.]+$/, "") || "recording"}.wav`, { type: "audio/wav" });
console.log(`${PREFIX} Audio extraction completed.`, { outputSizeBytes: wav.size });
console.log(`${PREFIX} Extraction finished in ${Math.round(performance.now() - started)} ms.`);
return wav;
} catch (error) {
const code = signal?.aborted ? "cancelled" : error instanceof VideoExtractionError ? error.code
: error instanceof RangeError ? "resource_failed" : stage;
if (code === "invalid_media") console.warn(`${PREFIX} Unsupported or undecodable video.`);
console.error(`${PREFIX} Audio extraction failed:`, error);
throw new VideoExtractionError(code, error);
} finally {
console.log(`${PREFIX} Removing temporary files.`);
if (ffmpeg?.loaded) {
if (inputWritten) await ffmpeg.deleteFile(inputName).catch(() => {});
if (outputWritten) await ffmpeg.deleteFile(outputName).catch(() => {});
}
console.log(`${PREFIX} Terminating worker.`);
ffmpeg?.terminate();
ffmpeg = null;
signal?.removeEventListener("abort", abort);
console.log(`${PREFIX} Cleanup complete.`);
}
}
@@ -44,3 +44,40 @@ test("admin-disabled policy and fetch failure both fall back to the original fil
globalThis.fetch = originalFetch;
}
});
test("video routing is gated and audio never invokes FFmpeg", async () => {
const originalFetch = globalThis.fetch;
let calls = 0;
globalThis.fetch = async () => ({ ok: true, json: async () => ({ browser_processing_beta_enabled: true, version: 1 }) });
try {
const processor = new BrowserSubmissionProcessor(
{ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 },
async () => { calls += 1; return new File(["wav"], "clip.wav", { type: "audio/wav" }); },
);
await processor.prepare(file);
assert.equal(calls, 0);
const video = { type: "video/mp4", size: 123, name: "clip.mp4" };
const result = await processor.prepare(video);
assert.equal(calls, 1);
assert.equal(result.file.type, "audio/wav");
assert.equal(result.metadata.browser_processing_used, true);
const disabled = new BrowserSubmissionProcessor({ browserProcessingBeta: false }, async () => { throw new Error("must not run"); });
assert.deepEqual(await disabled.prepare(video), { file: video, metadata: null });
} finally {
globalThis.fetch = originalFetch;
}
});
test("video conversion failure rejects instead of falling back to original video", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => ({ ok: true, json: async () => ({ browser_processing_beta_enabled: true, version: 1 }) });
try {
const processor = new BrowserSubmissionProcessor(
{ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 },
async () => { throw new Error("decode failed"); },
);
await assert.rejects(processor.prepare({ type: "video/mp4", name: "bad.mp4" }), /decode failed/);
} finally {
globalThis.fetch = originalFetch;
}
});
@@ -0,0 +1,38 @@
import assert from "node:assert/strict";
import test from "node:test";
import { extractVideoAudio } from "../src/scripts/ffmpeg-video-extractor.js";
function mockRuntime(exitCode = 0) {
const calls = [];
const ffmpeg = {
loaded: false,
on: () => {},
load: async () => { calls.push("load"); ffmpeg.loaded = true; },
writeFile: async () => { calls.push("write"); },
exec: async (args) => { calls.push(args); return exitCode; },
readFile: async () => { calls.push("read"); return new Uint8Array([82, 73, 70, 70]); },
deleteFile: async (path) => { calls.push(`delete:${path}`); },
terminate: () => { calls.push("terminate"); ffmpeg.loaded = false; },
};
return { ffmpeg, calls };
}
const video = new File(["video"], "clip.mp4", { type: "video/mp4" });
test("successful extraction emits WAV and cleans up virtual files and worker", async () => {
const runtime = mockRuntime();
const wav = await extractVideoAudio(video, { createFFmpeg: async () => runtime });
assert.equal(wav.type, "audio/wav");
assert.equal(wav.name, "clip.wav");
assert.deepEqual(runtime.calls.find(Array.isArray), ["-i", "input", "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", "audio.wav"]);
assert.ok(runtime.calls.includes("delete:input"));
assert.ok(runtime.calls.includes("delete:audio.wav"));
assert.equal(runtime.calls.at(-1), "terminate");
});
test("failed extraction removes input and terminates worker", async () => {
const runtime = mockRuntime(1);
await assert.rejects(extractVideoAudio(video, { createFFmpeg: async () => runtime }), { code: "invalid_media" });
assert.ok(runtime.calls.includes("delete:input"));
assert.equal(runtime.calls.at(-1), "terminate");
});