browser audio detection phase 2
This commit is contained in:
@@ -20,6 +20,11 @@ if (form) {
|
||||
let uploadComplete = false;
|
||||
let processingTimer = null;
|
||||
let preparing = false;
|
||||
let selectionId = 0;
|
||||
let selectionController = null;
|
||||
let selectedResult = null;
|
||||
let extractionPending = false;
|
||||
let extractionFailed = false;
|
||||
const browserProcessor = new BrowserSubmissionProcessor();
|
||||
if (!browserProcessor.isEnabled()) {
|
||||
console.log("[BoomCast][BrowserBeta] Feature disabled; using legacy submission flow.");
|
||||
@@ -120,7 +125,50 @@ if (form) {
|
||||
}
|
||||
}
|
||||
|
||||
fileInput.addEventListener("change", () => showFile(fileInput.files[0]));
|
||||
function selectFile(file) {
|
||||
selectionId += 1;
|
||||
selectionController?.abort();
|
||||
selectionController = null;
|
||||
selectedResult = null;
|
||||
extractionPending = false;
|
||||
extractionFailed = false;
|
||||
showFile(file);
|
||||
if (!busy) { submit.disabled = false; cancel.hidden = true; }
|
||||
if (!file || !browserProcessor.isEnabled() || !BrowserSubmissionProcessor.isVideo(file)) return;
|
||||
const id = selectionId;
|
||||
const controller = new AbortController();
|
||||
selectionController = controller;
|
||||
extractionPending = true;
|
||||
submit.disabled = true;
|
||||
cancel.hidden = false;
|
||||
status.textContent = t.extractingAudio || "Extracting audio from video…";
|
||||
browserProcessor.prepare(file, { signal: controller.signal, onState: (state) => {
|
||||
if (id === selectionId) console.log("[BoomCast][FFmpeg] State:", state);
|
||||
} }).then((result) => {
|
||||
if (id !== selectionId || controller.signal.aborted) return;
|
||||
selectedResult = result;
|
||||
extractionPending = false;
|
||||
submit.disabled = false;
|
||||
cancel.hidden = true;
|
||||
status.textContent = result.file === file
|
||||
? (t.videoExtractionUnavailable || "Video processing is unavailable. Please choose an audio recording.")
|
||||
: (t.onlyAudioSent || "Audio extracted on this device. Only audio will be sent to BoomCast.");
|
||||
extractionFailed = result.file === file;
|
||||
if (extractionFailed) submit.disabled = true;
|
||||
}).catch((error) => {
|
||||
if (id !== selectionId) return;
|
||||
console.error("[BoomCast][FFmpeg] Falling back safely.", error);
|
||||
extractionPending = false;
|
||||
extractionFailed = true;
|
||||
submit.disabled = true;
|
||||
cancel.hidden = true;
|
||||
status.textContent = controller.signal.aborted
|
||||
? (t.extractionCancelled || "Audio extraction cancelled. Choose a recording to continue.")
|
||||
: (t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.");
|
||||
});
|
||||
}
|
||||
|
||||
fileInput.addEventListener("change", () => selectFile(fileInput.files[0]));
|
||||
["dragenter", "dragover"].forEach((name) => dropZone.addEventListener(name, (event) => {
|
||||
event.preventDefault();
|
||||
if (!busy) dropZone.classList.add("dragging");
|
||||
@@ -132,7 +180,7 @@ if (form) {
|
||||
dropZone.addEventListener("drop", (event) => {
|
||||
if (busy || !event.dataTransfer.files.length) return;
|
||||
fileInput.files = event.dataTransfer.files;
|
||||
showFile(fileInput.files[0]);
|
||||
selectFile(fileInput.files[0]);
|
||||
});
|
||||
|
||||
function startUpload(file, metadata = null) {
|
||||
@@ -178,20 +226,32 @@ if (form) {
|
||||
form.addEventListener("submit", (event) => {
|
||||
event.preventDefault();
|
||||
event.stopImmediatePropagation();
|
||||
if (busy || preparing) return;
|
||||
if (busy || preparing || extractionPending || extractionFailed) return;
|
||||
if (!form.reportValidity()) return;
|
||||
const file = fileInput.files[0];
|
||||
if (!file) return;
|
||||
if (selectedResult && BrowserSubmissionProcessor.isVideo(file)) {
|
||||
startUpload(selectedResult.file, selectedResult.metadata);
|
||||
return;
|
||||
}
|
||||
if (!browserProcessor.isEnabled()) {
|
||||
startUpload(file);
|
||||
return;
|
||||
}
|
||||
const id = selectionId;
|
||||
preparing = true;
|
||||
submit.disabled = true;
|
||||
status.textContent = t.preparingRecording || "Preparing recording…";
|
||||
browserProcessor.prepare(file).then(({ file: preparedFile, metadata }) => {
|
||||
preparing = false;
|
||||
submit.disabled = false;
|
||||
if (fileInput.files[0] !== file || selectionId !== id) return;
|
||||
if (BrowserSubmissionProcessor.isVideo(file) && preparedFile === file) {
|
||||
extractionFailed = true;
|
||||
submit.disabled = true;
|
||||
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
|
||||
return;
|
||||
}
|
||||
startUpload(preparedFile || file, metadata);
|
||||
}).catch((error) => {
|
||||
console.error("[BoomCast][BrowserBeta] Initialisation failed:", error);
|
||||
@@ -199,14 +259,37 @@ if (form) {
|
||||
console.log("[BoomCast][BrowserBeta] Restoring legacy submission flow.");
|
||||
preparing = false;
|
||||
submit.disabled = false;
|
||||
startUpload(file);
|
||||
if (fileInput.files[0] !== file || selectionId !== id) return;
|
||||
if (BrowserSubmissionProcessor.isVideo(file)) {
|
||||
extractionFailed = true;
|
||||
submit.disabled = true;
|
||||
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
|
||||
return;
|
||||
}
|
||||
if (fileInput.files[0] === file && selectionId === id) startUpload(file);
|
||||
});
|
||||
}, true);
|
||||
|
||||
cancel.addEventListener("click", () => request?.abort());
|
||||
cancel.addEventListener("click", () => {
|
||||
if (extractionPending) {
|
||||
selectionController?.abort();
|
||||
selectionId += 1;
|
||||
extractionPending = false;
|
||||
extractionFailed = true;
|
||||
submit.disabled = true;
|
||||
cancel.hidden = true;
|
||||
status.textContent = t.extractionCancelled || "Audio extraction cancelled. Choose a recording to continue.";
|
||||
} else request?.abort();
|
||||
});
|
||||
window.addEventListener("pageshow", (event) => {
|
||||
if (event.persisted) {
|
||||
request?.abort();
|
||||
selectionController?.abort();
|
||||
selectionId += 1;
|
||||
selectedResult = null;
|
||||
extractionPending = false;
|
||||
extractionFailed = false;
|
||||
submit.disabled = false;
|
||||
fileInput.value = "";
|
||||
results.replaceChildren();
|
||||
showFile();
|
||||
|
||||
@@ -12,8 +12,9 @@ export function readBrowserProcessingConfig() {
|
||||
}
|
||||
|
||||
export class BrowserSubmissionProcessor {
|
||||
constructor(config = readBrowserProcessingConfig()) {
|
||||
constructor(config = readBrowserProcessingConfig(), extractVideo = null) {
|
||||
this.config = config;
|
||||
this.extractVideo = extractVideo;
|
||||
}
|
||||
|
||||
isEnabled() {
|
||||
@@ -24,11 +25,15 @@ export class BrowserSubmissionProcessor {
|
||||
return this.isEnabled() && Boolean(file) && /^(audio|video)\//.test(file.type);
|
||||
}
|
||||
|
||||
async prepare(file) {
|
||||
static isVideo(file) {
|
||||
return Boolean(file) && (file.type.startsWith("video/") || (!file.type && /\.(mp4|mov|m4v|webm|mkv|avi)$/i.test(file.name || "")));
|
||||
}
|
||||
|
||||
async prepare(file, { signal, onState } = {}) {
|
||||
if (!this.isEnabled()) return { file, metadata: null };
|
||||
console.log(`${PREFIX} Initialising beta submission pipeline.`);
|
||||
console.log(`${PREFIX} File selected:`, { type: file?.type || "", size: file?.size || 0 });
|
||||
if (!this.canProcess(file)) {
|
||||
if (!this.canProcess(file) && !BrowserSubmissionProcessor.isVideo(file)) {
|
||||
console.log(`${PREFIX} Browser preprocessing unavailable; falling back.`);
|
||||
return { file, metadata: null };
|
||||
}
|
||||
@@ -45,7 +50,12 @@ export class BrowserSubmissionProcessor {
|
||||
console.log(`${PREFIX} Browser beta disabled or policy changed; using legacy submission flow.`);
|
||||
return { file, metadata: null };
|
||||
}
|
||||
// Pass 1 has no local classifier or transcoder. Keep the original file.
|
||||
if (BrowserSubmissionProcessor.isVideo(file)) {
|
||||
const extract = this.extractVideo || (await import("./ffmpeg-video-extractor.js")).extractVideoAudio;
|
||||
const wav = await extract(file, { signal, onState });
|
||||
return { file: wav, metadata: { browser_processing_used: true, browser_processing_version: policy.version, browser_gate_result: "unavailable", browser_override: false } };
|
||||
}
|
||||
// Audio continues through the existing submission path.
|
||||
console.log(`${PREFIX} Browser processing shell ready; using original recording.`);
|
||||
return {
|
||||
file,
|
||||
@@ -59,6 +69,7 @@ export class BrowserSubmissionProcessor {
|
||||
} catch (error) {
|
||||
console.error(`${PREFIX} Initialisation failed:`, error);
|
||||
this.cleanup();
|
||||
if (BrowserSubmissionProcessor.isVideo(file)) throw error;
|
||||
console.log(`${PREFIX} Restoring legacy submission flow.`);
|
||||
return { file, metadata: null };
|
||||
} finally {
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
const PREFIX = "[BoomCast][FFmpeg]";
|
||||
|
||||
export class VideoExtractionError extends Error {
|
||||
constructor(code, cause) {
|
||||
super(code, { cause });
|
||||
this.name = "VideoExtractionError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
export async function extractVideoAudio(file, { signal, onState = () => {}, createFFmpeg } = {}) {
|
||||
let ffmpeg;
|
||||
let inputWritten = false;
|
||||
let outputWritten = false;
|
||||
let stage = "module_load_failed";
|
||||
const started = performance.now();
|
||||
const inputName = "input";
|
||||
const outputName = "audio.wav";
|
||||
const abort = () => ffmpeg?.terminate();
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
console.log(`${PREFIX} Video input detected.`, { mimeType: file.type, sizeBytes: file.size });
|
||||
try {
|
||||
if (signal?.aborted) throw new VideoExtractionError("cancelled");
|
||||
onState("loading_ffmpeg");
|
||||
console.log(`${PREFIX} Starting lazy import.`);
|
||||
const factory = createFFmpeg || (async () => {
|
||||
const [{ FFmpeg }, { default: coreURL }, { default: wasmURL }] = await Promise.all([
|
||||
import("@ffmpeg/ffmpeg"),
|
||||
import("@ffmpeg/core?url"),
|
||||
import("@ffmpeg/core/wasm?url"),
|
||||
]);
|
||||
console.log(`${PREFIX} ffmpeg.wasm module imported.`);
|
||||
return { ffmpeg: new FFmpeg(), coreURL, wasmURL };
|
||||
});
|
||||
const runtime = await factory();
|
||||
ffmpeg = runtime.ffmpeg;
|
||||
if (signal?.aborted) throw new VideoExtractionError("cancelled");
|
||||
ffmpeg.on?.("log", ({ message }) => console.log(`${PREFIX}[core]`, message));
|
||||
stage = "core_load_failed";
|
||||
console.log(`${PREFIX} Loading FFmpeg core.`);
|
||||
await ffmpeg.load({ coreURL: runtime.coreURL, wasmURL: runtime.wasmURL });
|
||||
console.log(`${PREFIX} FFmpeg core ready.`);
|
||||
if (signal?.aborted) throw new VideoExtractionError("cancelled");
|
||||
onState("extracting_audio");
|
||||
stage = "resource_failed";
|
||||
console.log(`${PREFIX} Writing input to virtual filesystem.`);
|
||||
// One input ArrayBuffer is unavoidable for the FFmpeg virtual filesystem.
|
||||
inputWritten = true;
|
||||
await ffmpeg.writeFile(inputName, new Uint8Array(await file.arrayBuffer()));
|
||||
if (signal?.aborted) throw new VideoExtractionError("cancelled");
|
||||
stage = "extraction_failed";
|
||||
console.log(`${PREFIX} Starting audio extraction: mono / 16kHz WAV.`);
|
||||
outputWritten = true;
|
||||
const exitCode = await ffmpeg.exec(["-i", inputName, "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", outputName]);
|
||||
if (exitCode !== 0) throw new VideoExtractionError("invalid_media");
|
||||
if (signal?.aborted) throw new VideoExtractionError("cancelled");
|
||||
const data = await ffmpeg.readFile(outputName);
|
||||
if (!data?.byteLength) throw new VideoExtractionError("invalid_media");
|
||||
if (data.byteLength > 25 * 1024 * 1024) throw new VideoExtractionError("output_too_large");
|
||||
const wav = new File([data], `${file.name.replace(/\.[^.]+$/, "") || "recording"}.wav`, { type: "audio/wav" });
|
||||
console.log(`${PREFIX} Audio extraction completed.`, { outputSizeBytes: wav.size });
|
||||
console.log(`${PREFIX} Extraction finished in ${Math.round(performance.now() - started)} ms.`);
|
||||
return wav;
|
||||
} catch (error) {
|
||||
const code = signal?.aborted ? "cancelled" : error instanceof VideoExtractionError ? error.code
|
||||
: error instanceof RangeError ? "resource_failed" : stage;
|
||||
if (code === "invalid_media") console.warn(`${PREFIX} Unsupported or undecodable video.`);
|
||||
console.error(`${PREFIX} Audio extraction failed:`, error);
|
||||
throw new VideoExtractionError(code, error);
|
||||
} finally {
|
||||
console.log(`${PREFIX} Removing temporary files.`);
|
||||
if (ffmpeg?.loaded) {
|
||||
if (inputWritten) await ffmpeg.deleteFile(inputName).catch(() => {});
|
||||
if (outputWritten) await ffmpeg.deleteFile(outputName).catch(() => {});
|
||||
}
|
||||
console.log(`${PREFIX} Terminating worker.`);
|
||||
ffmpeg?.terminate();
|
||||
ffmpeg = null;
|
||||
signal?.removeEventListener("abort", abort);
|
||||
console.log(`${PREFIX} Cleanup complete.`);
|
||||
}
|
||||
}
|
||||
@@ -44,3 +44,40 @@ test("admin-disabled policy and fetch failure both fall back to the original fil
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("video routing is gated and audio never invokes FFmpeg", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let calls = 0;
|
||||
globalThis.fetch = async () => ({ ok: true, json: async () => ({ browser_processing_beta_enabled: true, version: 1 }) });
|
||||
try {
|
||||
const processor = new BrowserSubmissionProcessor(
|
||||
{ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 },
|
||||
async () => { calls += 1; return new File(["wav"], "clip.wav", { type: "audio/wav" }); },
|
||||
);
|
||||
await processor.prepare(file);
|
||||
assert.equal(calls, 0);
|
||||
const video = { type: "video/mp4", size: 123, name: "clip.mp4" };
|
||||
const result = await processor.prepare(video);
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(result.file.type, "audio/wav");
|
||||
assert.equal(result.metadata.browser_processing_used, true);
|
||||
const disabled = new BrowserSubmissionProcessor({ browserProcessingBeta: false }, async () => { throw new Error("must not run"); });
|
||||
assert.deepEqual(await disabled.prepare(video), { file: video, metadata: null });
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("video conversion failure rejects instead of falling back to original video", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = async () => ({ ok: true, json: async () => ({ browser_processing_beta_enabled: true, version: 1 }) });
|
||||
try {
|
||||
const processor = new BrowserSubmissionProcessor(
|
||||
{ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 },
|
||||
async () => { throw new Error("decode failed"); },
|
||||
);
|
||||
await assert.rejects(processor.prepare({ type: "video/mp4", name: "bad.mp4" }), /decode failed/);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { extractVideoAudio } from "../src/scripts/ffmpeg-video-extractor.js";
|
||||
|
||||
function mockRuntime(exitCode = 0) {
|
||||
const calls = [];
|
||||
const ffmpeg = {
|
||||
loaded: false,
|
||||
on: () => {},
|
||||
load: async () => { calls.push("load"); ffmpeg.loaded = true; },
|
||||
writeFile: async () => { calls.push("write"); },
|
||||
exec: async (args) => { calls.push(args); return exitCode; },
|
||||
readFile: async () => { calls.push("read"); return new Uint8Array([82, 73, 70, 70]); },
|
||||
deleteFile: async (path) => { calls.push(`delete:${path}`); },
|
||||
terminate: () => { calls.push("terminate"); ffmpeg.loaded = false; },
|
||||
};
|
||||
return { ffmpeg, calls };
|
||||
}
|
||||
|
||||
const video = new File(["video"], "clip.mp4", { type: "video/mp4" });
|
||||
|
||||
test("successful extraction emits WAV and cleans up virtual files and worker", async () => {
|
||||
const runtime = mockRuntime();
|
||||
const wav = await extractVideoAudio(video, { createFFmpeg: async () => runtime });
|
||||
assert.equal(wav.type, "audio/wav");
|
||||
assert.equal(wav.name, "clip.wav");
|
||||
assert.deepEqual(runtime.calls.find(Array.isArray), ["-i", "input", "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", "audio.wav"]);
|
||||
assert.ok(runtime.calls.includes("delete:input"));
|
||||
assert.ok(runtime.calls.includes("delete:audio.wav"));
|
||||
assert.equal(runtime.calls.at(-1), "terminate");
|
||||
});
|
||||
|
||||
test("failed extraction removes input and terminates worker", async () => {
|
||||
const runtime = mockRuntime(1);
|
||||
await assert.rejects(extractVideoAudio(video, { createFFmpeg: async () => runtime }), { code: "invalid_media" });
|
||||
assert.ok(runtime.calls.includes("delete:input"));
|
||||
assert.equal(runtime.calls.at(-1), "terminate");
|
||||
});
|
||||
Reference in New Issue
Block a user