browser audio detection phase 3
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
const RATE = 16000;
|
||||
|
||||
export class AudioPreprocessor {
|
||||
static async prepare(file, { signal } = {}) {
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
console.log("[BoomCast][YAMNet] Preparing local classification.");
|
||||
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
|
||||
if (!AudioContextClass || !(window.OfflineAudioContext || window.webkitOfflineAudioContext)) throw new Error("Browser audio decoding unavailable");
|
||||
const context = new AudioContextClass();
|
||||
try {
|
||||
const decoded = await context.decodeAudioData(await file.arrayBuffer());
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
if (!decoded.duration || !Number.isFinite(decoded.duration)) throw new Error("Invalid audio duration");
|
||||
const OfflineContextClass = window.OfflineAudioContext || window.webkitOfflineAudioContext;
|
||||
const offline = new OfflineContextClass(1, Math.ceil(decoded.duration * RATE), RATE);
|
||||
const source = offline.createBufferSource();
|
||||
source.buffer = decoded;
|
||||
source.connect(offline.destination);
|
||||
source.start();
|
||||
const rendered = await offline.startRendering();
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
const waveform = rendered.getChannelData(0).slice();
|
||||
console.log("[BoomCast][YAMNet] Waveform prepared.", {
|
||||
sampleRate: RATE, samples: waveform.length, durationSeconds: waveform.length / RATE,
|
||||
});
|
||||
return waveform;
|
||||
} finally {
|
||||
await context.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,11 @@ if (form) {
|
||||
const submit = form.querySelector('button[type="submit"]');
|
||||
const cancel = form.querySelector(".upload-cancel");
|
||||
const results = form.querySelector("#classification-results");
|
||||
const gatePanel = form.querySelector("#browser-gate");
|
||||
const gateTitle = form.querySelector("#browser-gate-title");
|
||||
const gateMessage = form.querySelector("#browser-gate-message");
|
||||
const gateChoose = form.querySelector("#browser-gate-choose");
|
||||
const gateOverride = form.querySelector("#browser-gate-override");
|
||||
const status = document.querySelector("#upload-status");
|
||||
const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)");
|
||||
const states = ["idle", "file-selected", "uploading", "processing", "success", "error", "cancelled"];
|
||||
@@ -23,8 +28,9 @@ if (form) {
|
||||
let selectionId = 0;
|
||||
let selectionController = null;
|
||||
let selectedResult = null;
|
||||
let extractionPending = false;
|
||||
let extractionFailed = false;
|
||||
let preparationPending = false;
|
||||
let preparationFailed = false;
|
||||
let gateBlocked = false;
|
||||
const browserProcessor = new BrowserSubmissionProcessor();
|
||||
if (!browserProcessor.isEnabled()) {
|
||||
console.log("[BoomCast][BrowserBeta] Feature disabled; using legacy submission flow.");
|
||||
@@ -130,41 +136,68 @@ if (form) {
|
||||
selectionController?.abort();
|
||||
selectionController = null;
|
||||
selectedResult = null;
|
||||
extractionPending = false;
|
||||
extractionFailed = false;
|
||||
preparationPending = false;
|
||||
preparationFailed = false;
|
||||
gateBlocked = false;
|
||||
gatePanel.hidden = true;
|
||||
showFile(file);
|
||||
if (!busy) { submit.disabled = false; cancel.hidden = true; }
|
||||
if (!file || !browserProcessor.isEnabled() || !BrowserSubmissionProcessor.isVideo(file)) return;
|
||||
if (!file || !browserProcessor.canProcess(file)) return;
|
||||
const id = selectionId;
|
||||
const controller = new AbortController();
|
||||
selectionController = controller;
|
||||
extractionPending = true;
|
||||
preparationPending = true;
|
||||
submit.disabled = true;
|
||||
cancel.hidden = false;
|
||||
status.textContent = t.extractingAudio || "Extracting audio from video…";
|
||||
status.textContent = BrowserSubmissionProcessor.isVideo(file)
|
||||
? (t.extractingAudio || "Extracting audio from video…")
|
||||
: (t.preparingRecording || "Preparing recording…");
|
||||
browserProcessor.prepare(file, { signal: controller.signal, onState: (state) => {
|
||||
if (id === selectionId) console.log("[BoomCast][FFmpeg] State:", state);
|
||||
if (id !== selectionId) return;
|
||||
status.textContent = state === "checking" ? (t.checkingRecording || "Checking recording…")
|
||||
: (t.extractingAudio || "Extracting audio from video…");
|
||||
} }).then((result) => {
|
||||
if (id !== selectionId || controller.signal.aborted) return;
|
||||
selectedResult = result;
|
||||
extractionPending = false;
|
||||
submit.disabled = false;
|
||||
preparationPending = false;
|
||||
cancel.hidden = true;
|
||||
status.textContent = result.file === file
|
||||
? (t.videoExtractionUnavailable || "Video processing is unavailable. Please choose an audio recording.")
|
||||
: (t.onlyAudioSent || "Audio extracted on this device. Only audio will be sent to BoomCast.");
|
||||
extractionFailed = result.file === file;
|
||||
if (extractionFailed) submit.disabled = true;
|
||||
if (BrowserSubmissionProcessor.isVideo(file) && result.file === file) {
|
||||
preparationFailed = true;
|
||||
submit.disabled = true;
|
||||
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
|
||||
return;
|
||||
}
|
||||
selectedResult = result;
|
||||
gateBlocked = result.gate === "speech_only" || result.gate === "no_match";
|
||||
if (gateBlocked) {
|
||||
gateTitle.textContent = result.gate === "speech_only"
|
||||
? (t.speechGateTitle || "This recording appears to contain speech, but no relevant sound was detected.")
|
||||
: (t.irrelevantGateTitle || "No relevant sound was detected.");
|
||||
gateMessage.textContent = result.gate === "speech_only"
|
||||
? (t.speechGateMessage || "To protect people’s privacy, this recording will not be uploaded automatically. You can still submit it for manual review.")
|
||||
: (t.irrelevantGateMessage || "BoomCast could not identify a sound this service currently looks for. You can still submit it for manual review.");
|
||||
gatePanel.hidden = false;
|
||||
submit.disabled = true;
|
||||
status.textContent = (t.gateChooseMessage || "Recording checked. Choose another recording or submit for manual review.");
|
||||
} else {
|
||||
submit.disabled = false;
|
||||
status.textContent = BrowserSubmissionProcessor.isVideo(file)
|
||||
? (t.videoAudioOnly || "Video processed on this device. Only extracted audio will be sent if you submit.")
|
||||
: (t.readyToSubmit || "Ready to submit.");
|
||||
}
|
||||
}).catch((error) => {
|
||||
if (id !== selectionId) return;
|
||||
console.error("[BoomCast][FFmpeg] Falling back safely.", error);
|
||||
extractionPending = false;
|
||||
extractionFailed = true;
|
||||
submit.disabled = true;
|
||||
console.error("[BoomCast][BrowserBeta] Preparation failed:", error);
|
||||
preparationPending = false;
|
||||
cancel.hidden = true;
|
||||
status.textContent = controller.signal.aborted
|
||||
? (t.extractionCancelled || "Audio extraction cancelled. Choose a recording to continue.")
|
||||
: (t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.");
|
||||
if (BrowserSubmissionProcessor.isVideo(file)) {
|
||||
preparationFailed = true;
|
||||
submit.disabled = true;
|
||||
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
|
||||
} else {
|
||||
selectedResult = { file, metadata: null, gate: "unavailable" };
|
||||
submit.disabled = false;
|
||||
status.textContent = t.readyToSubmit || "Ready to submit.";
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -184,6 +217,22 @@ if (form) {
|
||||
});
|
||||
|
||||
function startUpload(file, metadata = null) {
|
||||
// HTMX's later Pending action also reads the native file input. Replace a
|
||||
// video with its extracted WAV so no subsequent form action can send video.
|
||||
if (fileInput.files[0] !== file) {
|
||||
try {
|
||||
const transfer = new DataTransfer();
|
||||
transfer.items.add(file);
|
||||
fileInput.files = transfer.files;
|
||||
if (fileInput.files[0] !== file) throw new Error("Prepared audio could not replace video in form");
|
||||
} catch (error) {
|
||||
console.error("[BoomCast][BrowserBeta] Could not secure native form upload:", error);
|
||||
preparationFailed = true;
|
||||
submit.disabled = true;
|
||||
status.textContent = (t.secureVideoFailed || "Could not prepare this video for upload. Please choose an audio recording.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
const formData = new FormData(form);
|
||||
formData.set("audio", file);
|
||||
if (metadata) formData.set("browser_processing_metadata", JSON.stringify(metadata));
|
||||
@@ -226,59 +275,36 @@ if (form) {
|
||||
form.addEventListener("submit", (event) => {
|
||||
event.preventDefault();
|
||||
event.stopImmediatePropagation();
|
||||
if (busy || preparing || extractionPending || extractionFailed) return;
|
||||
if (busy || preparationPending || preparationFailed || gateBlocked) return;
|
||||
if (!form.reportValidity()) return;
|
||||
const file = fileInput.files[0];
|
||||
if (!file) return;
|
||||
if (selectedResult && BrowserSubmissionProcessor.isVideo(file)) {
|
||||
startUpload(selectedResult.file, selectedResult.metadata);
|
||||
return;
|
||||
}
|
||||
if (!browserProcessor.isEnabled()) {
|
||||
startUpload(file);
|
||||
return;
|
||||
}
|
||||
const id = selectionId;
|
||||
preparing = true;
|
||||
submit.disabled = true;
|
||||
status.textContent = t.preparingRecording || "Preparing recording…";
|
||||
browserProcessor.prepare(file).then(({ file: preparedFile, metadata }) => {
|
||||
preparing = false;
|
||||
submit.disabled = false;
|
||||
if (fileInput.files[0] !== file || selectionId !== id) return;
|
||||
if (BrowserSubmissionProcessor.isVideo(file) && preparedFile === file) {
|
||||
extractionFailed = true;
|
||||
submit.disabled = true;
|
||||
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
|
||||
return;
|
||||
}
|
||||
startUpload(preparedFile || file, metadata);
|
||||
}).catch((error) => {
|
||||
console.error("[BoomCast][BrowserBeta] Initialisation failed:", error);
|
||||
browserProcessor.cleanup();
|
||||
console.log("[BoomCast][BrowserBeta] Restoring legacy submission flow.");
|
||||
preparing = false;
|
||||
submit.disabled = false;
|
||||
if (fileInput.files[0] !== file || selectionId !== id) return;
|
||||
if (BrowserSubmissionProcessor.isVideo(file)) {
|
||||
extractionFailed = true;
|
||||
submit.disabled = true;
|
||||
status.textContent = t.videoExtractionFailed || "Could not extract audio from this video. Please choose an audio recording.";
|
||||
return;
|
||||
}
|
||||
if (fileInput.files[0] === file && selectionId === id) startUpload(file);
|
||||
});
|
||||
if (selectedResult) { startUpload(selectedResult.file, selectedResult.metadata); return; }
|
||||
if (!browserProcessor.isEnabled()) { startUpload(file); return; }
|
||||
// Unknown types retain the existing server flow.
|
||||
startUpload(file);
|
||||
}, true);
|
||||
|
||||
gateChoose.addEventListener("click", () => fileInput.click());
|
||||
gateOverride.addEventListener("click", () => {
|
||||
if (!gateBlocked || !selectedResult || busy || preparationPending) return;
|
||||
console.warn("[BoomCast][Gate] User explicitly overrode local gate.");
|
||||
console.log("[BoomCast][Gate] Submission will enter manual review.");
|
||||
const metadata = { ...selectedResult.metadata, browser_override: true };
|
||||
selectedResult.metadata = metadata;
|
||||
gateBlocked = false;
|
||||
gatePanel.hidden = true;
|
||||
startUpload(selectedResult.file, metadata);
|
||||
});
|
||||
cancel.addEventListener("click", () => {
|
||||
if (extractionPending) {
|
||||
if (preparationPending) {
|
||||
selectionController?.abort();
|
||||
selectionId += 1;
|
||||
extractionPending = false;
|
||||
extractionFailed = true;
|
||||
preparationPending = false;
|
||||
preparationFailed = true;
|
||||
submit.disabled = true;
|
||||
cancel.hidden = true;
|
||||
status.textContent = t.extractionCancelled || "Audio extraction cancelled. Choose a recording to continue.";
|
||||
status.textContent = t.extractionCancelled || (t.preparationCancelled || "Preparation cancelled. Choose a recording to continue.");
|
||||
} else request?.abort();
|
||||
});
|
||||
window.addEventListener("pageshow", (event) => {
|
||||
@@ -287,8 +313,10 @@ if (form) {
|
||||
selectionController?.abort();
|
||||
selectionId += 1;
|
||||
selectedResult = null;
|
||||
extractionPending = false;
|
||||
extractionFailed = false;
|
||||
preparationPending = false;
|
||||
preparationFailed = false;
|
||||
gateBlocked = false;
|
||||
gatePanel.hidden = true;
|
||||
submit.disabled = false;
|
||||
fileInput.value = "";
|
||||
results.replaceChildren();
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
export class BrowserSubmissionGate {
|
||||
static evaluate(predictions, policy) {
|
||||
console.log("[BoomCast][Gate] Evaluating browser classification.");
|
||||
const relevant = new Set((policy.whitelist || []).map((name) => name.toLowerCase()));
|
||||
const speechNames = new Set((policy.privacy_sensitive_speech || []).map((name) => name.toLowerCase()));
|
||||
const matchThreshold = Number(policy.thresholds?.relevance ?? 0.5);
|
||||
const speechThreshold = Number(policy.thresholds?.speech ?? 0.5);
|
||||
const matches = predictions.filter((item) => relevant.has(item.class.toLowerCase()) && item.score >= matchThreshold);
|
||||
const speechDetected = predictions.some((item) => speechNames.has(item.class.toLowerCase()) && item.score >= speechThreshold);
|
||||
console.log("[BoomCast][Gate] Relevant whitelist matches:", matches);
|
||||
console.log("[BoomCast][Gate] Speech detected:", speechDetected);
|
||||
const result = matches.length ? "whitelist_match" : speechDetected ? "speech_only" : "no_match";
|
||||
console.log(`[BoomCast][Gate] Decision: ${result.toUpperCase()}`);
|
||||
if (result !== "whitelist_match") console.log("[BoomCast][Gate] Automatic upload suppressed.");
|
||||
return { result, speechDetected, matches };
|
||||
}
|
||||
}
|
||||
@@ -3,82 +3,104 @@ const PREFIX = "[BoomCast][BrowserBeta]";
|
||||
export function readBrowserProcessingConfig() {
|
||||
const element = document.querySelector("#boomcast-browser-config");
|
||||
if (!element) return { browserProcessingBeta: false };
|
||||
try {
|
||||
return JSON.parse(element.textContent);
|
||||
} catch (error) {
|
||||
try { return JSON.parse(element.textContent); }
|
||||
catch (error) {
|
||||
console.error(`${PREFIX} Configuration unavailable:`, error);
|
||||
return { browserProcessingBeta: false };
|
||||
}
|
||||
}
|
||||
|
||||
export class BrowserSubmissionProcessor {
|
||||
constructor(config = readBrowserProcessingConfig(), extractVideo = null) {
|
||||
constructor(config = readBrowserProcessingConfig(), extractVideo = null, dependencies = {}) {
|
||||
this.config = config;
|
||||
this.extractVideo = extractVideo;
|
||||
this.dependencies = dependencies;
|
||||
}
|
||||
|
||||
isEnabled() {
|
||||
return this.config.browserProcessingBeta === true;
|
||||
}
|
||||
isEnabled() { return this.config.browserProcessingBeta === true; }
|
||||
|
||||
canProcess(file) {
|
||||
return this.isEnabled() && Boolean(file) && /^(audio|video)\//.test(file.type);
|
||||
return this.isEnabled() && Boolean(file) && (BrowserSubmissionProcessor.isAudio(file) || BrowserSubmissionProcessor.isVideo(file));
|
||||
}
|
||||
|
||||
static isAudio(file) {
|
||||
return Boolean(file) && (file.type.startsWith("audio/") || (!file.type && /\.(aac|aif|aiff|flac|m4a|mp3|ogg|opus|wav|weba)$/i.test(file.name || "")));
|
||||
}
|
||||
|
||||
static isVideo(file) {
|
||||
return Boolean(file) && (file.type.startsWith("video/") || (!file.type && /\.(mp4|mov|m4v|webm|mkv|avi)$/i.test(file.name || "")));
|
||||
}
|
||||
|
||||
async prepare(file, { signal, onState } = {}) {
|
||||
if (!this.isEnabled()) return { file, metadata: null };
|
||||
async prepare(file, { signal, onState = () => {} } = {}) {
|
||||
if (!this.canProcess(file)) return { file, metadata: null, gate: "unavailable" };
|
||||
console.log(`${PREFIX} Initialising beta submission pipeline.`);
|
||||
console.log(`${PREFIX} File selected:`, { type: file?.type || "", size: file?.size || 0 });
|
||||
if (!this.canProcess(file) && !BrowserSubmissionProcessor.isVideo(file)) {
|
||||
console.log(`${PREFIX} Browser preprocessing unavailable; falling back.`);
|
||||
return { file, metadata: null };
|
||||
}
|
||||
let policyTimeout;
|
||||
const video = BrowserSubmissionProcessor.isVideo(file);
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort();
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
const timeout = setTimeout(() => controller.abort(), 5000);
|
||||
let policy;
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
policyTimeout = setTimeout(() => controller.abort(), 5000);
|
||||
const response = await fetch(this.config.policyUrl, {
|
||||
cache: "no-store", credentials: "same-origin", signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) throw new Error(`Policy HTTP ${response.status}`);
|
||||
const policy = await response.json();
|
||||
policy = await response.json();
|
||||
if (!policy.browser_processing_beta_enabled || policy.version !== this.config.policyVersion) {
|
||||
console.log(`${PREFIX} Browser beta disabled or policy changed; using legacy submission flow.`);
|
||||
return { file, metadata: null };
|
||||
return { file, metadata: null, gate: "unavailable" };
|
||||
}
|
||||
if (BrowserSubmissionProcessor.isVideo(file)) {
|
||||
const extract = this.extractVideo || (await import("./ffmpeg-video-extractor.js")).extractVideoAudio;
|
||||
const wav = await extract(file, { signal, onState });
|
||||
return { file: wav, metadata: { browser_processing_used: true, browser_processing_version: policy.version, browser_gate_result: "unavailable", browser_override: false } };
|
||||
}
|
||||
// Audio continues through the existing submission path.
|
||||
console.log(`${PREFIX} Browser processing shell ready; using original recording.`);
|
||||
return {
|
||||
file,
|
||||
metadata: {
|
||||
browser_processing_used: false,
|
||||
browser_processing_version: policy.version,
|
||||
browser_gate_result: "unavailable",
|
||||
browser_override: false,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`${PREFIX} Initialisation failed:`, error);
|
||||
this.cleanup();
|
||||
if (BrowserSubmissionProcessor.isVideo(file)) throw error;
|
||||
console.log(`${PREFIX} Restoring legacy submission flow.`);
|
||||
return { file, metadata: null };
|
||||
if (signal?.aborted) throw error;
|
||||
console.error(`${PREFIX} Policy unavailable:`, error);
|
||||
if (video) throw error;
|
||||
return { file, metadata: null, gate: "unavailable" };
|
||||
} finally {
|
||||
clearTimeout(policyTimeout);
|
||||
clearTimeout(timeout);
|
||||
signal?.removeEventListener("abort", abort);
|
||||
}
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
let preparedFile = file;
|
||||
if (video) {
|
||||
const extract = this.extractVideo || (await import("./ffmpeg-video-extractor.js")).extractVideoAudio;
|
||||
preparedFile = await extract(file, { signal, onState });
|
||||
}
|
||||
const metadata = {
|
||||
browser_processing_used: video,
|
||||
browser_processing_version: policy.version,
|
||||
browser_gate_result: "unavailable",
|
||||
browser_override: false,
|
||||
};
|
||||
try {
|
||||
onState("checking");
|
||||
const prepareAudio = this.dependencies.prepareAudio || (await import("./audio-preprocessor.js")).AudioPreprocessor.prepare;
|
||||
let waveform = await prepareAudio(preparedFile, { signal });
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
try {
|
||||
const classify = this.dependencies.classify || (await import("./browser-yamnet-classifier.js")).BrowserYamnetClassifier.classify;
|
||||
const predictions = await classify(waveform, {
|
||||
modelUrl: policy.model_url, classMapUrl: policy.class_map_url, signal,
|
||||
});
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
const evaluate = this.dependencies.evaluate || (await import("./browser-submission-gate.js")).BrowserSubmissionGate.evaluate;
|
||||
const decision = evaluate(predictions, policy);
|
||||
metadata.browser_processing_used = true;
|
||||
metadata.browser_gate_result = decision.result;
|
||||
metadata.browser_speech_detected = decision.speechDetected;
|
||||
return { file: preparedFile, metadata, gate: decision.result, predictions: predictions.slice(0, 5) };
|
||||
} finally {
|
||||
waveform = null;
|
||||
console.log("[BoomCast][Cleanup] Releasing waveform buffers.");
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
console.error("[BoomCast][YAMNet] Local classification unavailable:", error);
|
||||
console.log("[BoomCast][Gate] Falling back to server classification.");
|
||||
metadata.browser_gate_result = "unavailable";
|
||||
return { file: preparedFile, metadata, gate: "unavailable" };
|
||||
}
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
// Future passes can release workers, object URLs, and model tensors here.
|
||||
console.log("[BoomCast][Cleanup] Browser beta resources released.");
|
||||
}
|
||||
cleanup() { console.log("[BoomCast][Cleanup] Browser beta resources released."); }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
const PREFIX = "[BoomCast][YAMNet]";
|
||||
|
||||
export class BrowserYamnetClassifier {
|
||||
static async classify(waveform, { modelUrl, classMapUrl, signal, loadTensorFlow } = {}) {
|
||||
let tf;
|
||||
let model;
|
||||
let input;
|
||||
let output;
|
||||
const started = performance.now();
|
||||
try {
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
console.log(`${PREFIX} Lazy-loading TensorFlow.js.`);
|
||||
tf = loadTensorFlow ? await loadTensorFlow() : await import("@tensorflow/tfjs");
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
console.log(`${PREFIX} TensorFlow.js loaded.`);
|
||||
await tf.ready();
|
||||
console.log(`${PREFIX} Backend:`, tf.getBackend());
|
||||
console.log(`${PREFIX} Loading model.`);
|
||||
const [loaded, mappingResponse] = await Promise.all([
|
||||
tf.loadGraphModel(modelUrl),
|
||||
fetch(classMapUrl, { cache: "force-cache", signal }),
|
||||
]);
|
||||
model = loaded;
|
||||
if (!mappingResponse.ok) throw new Error("YAMNet class map unavailable");
|
||||
const classMap = await mappingResponse.json();
|
||||
if (!Array.isArray(classMap) || classMap.length !== 521) throw new Error("Invalid YAMNet class map");
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
console.log(`${PREFIX} Model ready.`);
|
||||
// YAMNet needs at least one 0.96 s patch, including STFT padding.
|
||||
const samples = waveform.length < 15600 ? Float32Array.from({ length: 15600 }, (_, i) => waveform[i] || 0) : waveform;
|
||||
input = tf.tensor1d(samples, "float32");
|
||||
console.log(`${PREFIX} Starting inference.`);
|
||||
const result = await model.executeAsync(input);
|
||||
output = Array.isArray(result) ? result : [result];
|
||||
const scores = output.find((tensor) => tensor.shape.length === 2 && tensor.shape[1] === classMap.length);
|
||||
if (!scores) throw new Error("YAMNet score output unavailable");
|
||||
const values = await scores.data();
|
||||
if (signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
const frames = scores.shape[0];
|
||||
const predictions = classMap.map((name, index) => {
|
||||
let score = -1;
|
||||
let frame = 0;
|
||||
for (let row = 0; row < frames; row += 1) {
|
||||
const value = values[row * classMap.length + index];
|
||||
if (value > score) { score = value; frame = row; }
|
||||
}
|
||||
return { class: name, score, frame, timeSeconds: Number((frame * 0.48).toFixed(2)) };
|
||||
});
|
||||
predictions.sort((a, b) => b.score - a.score);
|
||||
console.log(`${PREFIX} Inference completed in`, Math.round(performance.now() - started), "ms.");
|
||||
console.groupCollapsed(`${PREFIX} Top local predictions`);
|
||||
console.table(predictions.slice(0, 10));
|
||||
console.groupEnd();
|
||||
return predictions;
|
||||
} catch (error) {
|
||||
console.error(`${PREFIX} Local inference failed:`, error);
|
||||
throw error;
|
||||
} finally {
|
||||
if (tf) console.log(`${PREFIX} TensorFlow memory before cleanup:`, tf.memory());
|
||||
console.log("[BoomCast][Cleanup] Disposing YAMNet tensors.");
|
||||
input?.dispose();
|
||||
output?.forEach((tensor) => tensor?.dispose());
|
||||
console.log("[BoomCast][Cleanup] Disposing browser YAMNet model.");
|
||||
model?.dispose();
|
||||
console.log("[BoomCast][Cleanup] Releasing waveform buffers.");
|
||||
if (tf) console.log(`${PREFIX} TensorFlow memory after cleanup:`, tf.memory());
|
||||
console.log("[BoomCast][Cleanup] Browser classification cleanup complete.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -509,3 +509,9 @@
|
||||
transition-duration: .01ms !important;
|
||||
}
|
||||
}
|
||||
|
||||
.browser-gate { margin: 1rem 0; padding: 1rem; border: 1px solid #e5bb7a; border-radius: .75rem; background: #4d392950; }
|
||||
.browser-gate[hidden] { display: none; }
|
||||
.browser-gate p { margin: .45rem 0 1rem; }
|
||||
.browser-gate-actions { display: flex; gap: .65rem; flex-wrap: wrap; }
|
||||
.browser-gate-actions button { cursor: pointer; }
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { BrowserSubmissionGate } from "../src/scripts/browser-submission-gate.js";
|
||||
import { BrowserSubmissionProcessor } from "../src/scripts/browser-submission-processor.js";
|
||||
|
||||
const policy = { whitelist: ["Fireworks"], privacy_sensitive_speech: ["Speech"], thresholds: { relevance: .5, speech: .5 } };
|
||||
|
||||
test("whitelist match allows normal submission even when speech is present", () => {
|
||||
const decision = BrowserSubmissionGate.evaluate([{ class: "Fireworks", score: .8 }, { class: "Speech", score: .9 }], policy);
|
||||
assert.equal(decision.result, "whitelist_match");
|
||||
assert.equal(decision.speechDetected, true);
|
||||
});
|
||||
|
||||
test("speech without relevance and irrelevant audio require explicit override", () => {
|
||||
assert.equal(BrowserSubmissionGate.evaluate([{ class: "Speech", score: .9 }], policy).result, "speech_only");
|
||||
assert.equal(BrowserSubmissionGate.evaluate([{ class: "Music", score: .9 }], policy).result, "no_match");
|
||||
});
|
||||
|
||||
test("audio and video-derived WAV use the same gate; audio never calls FFmpeg", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = async () => ({ ok: true, json: async () => ({ ...policy, browser_processing_beta_enabled: true, version: 1 }) });
|
||||
let extractionCalls = 0;
|
||||
let classificationCalls = 0;
|
||||
const processor = new BrowserSubmissionProcessor(
|
||||
{ browserProcessingBeta: true, policyUrl: "/policy", policyVersion: 1 },
|
||||
async () => { extractionCalls += 1; return new File(["wav"], "clip.wav", { type: "audio/wav" }); },
|
||||
{ prepareAudio: async () => new Float32Array(16000), classify: async () => { classificationCalls += 1; return [{ class: "Speech", score: .9 }]; } },
|
||||
);
|
||||
try {
|
||||
assert.equal((await processor.prepare(new File(["audio"], "audio.wav", { type: "audio/wav" }))).gate, "speech_only");
|
||||
assert.equal(extractionCalls, 0);
|
||||
assert.equal((await processor.prepare(new File(["video"], "clip.mp4", { type: "video/mp4" }))).gate, "speech_only");
|
||||
assert.equal(extractionCalls, 1);
|
||||
assert.equal(classificationCalls, 2);
|
||||
} finally { globalThis.fetch = originalFetch; }
|
||||
});
|
||||
|
||||
test("local classifier failure restores ordinary server upload", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = async () => ({ ok: true, json: async () => ({ ...policy, browser_processing_beta_enabled: true, version: 1 }) });
|
||||
const processor = new BrowserSubmissionProcessor(
|
||||
{ browserProcessingBeta: true, policyUrl: "/policy", policyVersion: 1 }, null,
|
||||
{ prepareAudio: async () => new Float32Array(16000), classify: async () => { throw new Error("offline"); } },
|
||||
);
|
||||
const audio = new File(["audio"], "audio.wav", { type: "audio/wav" });
|
||||
try {
|
||||
const result = await processor.prepare(audio);
|
||||
assert.equal(result.file, audio);
|
||||
assert.equal(result.gate, "unavailable");
|
||||
} finally { globalThis.fetch = originalFetch; }
|
||||
});
|
||||
|
||||
test("aborted classification never yields a gate decision", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = async () => ({ ok: true, json: async () => ({ ...policy, browser_processing_beta_enabled: true, version: 1 }) });
|
||||
const controller = new AbortController();
|
||||
const processor = new BrowserSubmissionProcessor(
|
||||
{ browserProcessingBeta: true, policyUrl: "/policy", policyVersion: 1 }, null,
|
||||
{ prepareAudio: async () => new Float32Array(16000), classify: async () => { controller.abort(); return [{ class: "Fireworks", score: .9 }]; } },
|
||||
);
|
||||
try {
|
||||
await assert.rejects(processor.prepare(new File(["audio"], "audio.wav", { type: "audio/wav" }), { signal: controller.signal }));
|
||||
} finally { globalThis.fetch = originalFetch; }
|
||||
});
|
||||
@@ -12,7 +12,7 @@ test("disabled beta returns the original file without loading policy", async ()
|
||||
const processor = new BrowserSubmissionProcessor({ browserProcessingBeta: false });
|
||||
assert.equal(processor.isEnabled(), false);
|
||||
assert.equal(processor.canProcess(file), false);
|
||||
assert.deepEqual(await processor.prepare(file), { file, metadata: null });
|
||||
assert.deepEqual(await processor.prepare(file), { file, metadata: null, gate: "unavailable" });
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
@@ -37,9 +37,9 @@ test("admin-disabled policy and fetch failure both fall back to the original fil
|
||||
const processor = new BrowserSubmissionProcessor({ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 });
|
||||
try {
|
||||
globalThis.fetch = async () => ({ ok: true, json: async () => ({ browser_processing_beta_enabled: false, version: 1 }) });
|
||||
assert.deepEqual(await processor.prepare(file), { file, metadata: null });
|
||||
assert.deepEqual(await processor.prepare(file), { file, metadata: null, gate: "unavailable" });
|
||||
globalThis.fetch = async () => { throw new Error("offline"); };
|
||||
assert.deepEqual(await processor.prepare(file), { file, metadata: null });
|
||||
assert.deepEqual(await processor.prepare(file), { file, metadata: null, gate: "unavailable" });
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
@@ -53,6 +53,7 @@ test("video routing is gated and audio never invokes FFmpeg", async () => {
|
||||
const processor = new BrowserSubmissionProcessor(
|
||||
{ browserProcessingBeta: true, policyUrl: "/browser-policy/", policyVersion: 1 },
|
||||
async () => { calls += 1; return new File(["wav"], "clip.wav", { type: "audio/wav" }); },
|
||||
{ prepareAudio: async () => new Float32Array(16000), classify: async () => [{ class: "Fireworks", score: .9 }] },
|
||||
);
|
||||
await processor.prepare(file);
|
||||
assert.equal(calls, 0);
|
||||
@@ -62,7 +63,7 @@ test("video routing is gated and audio never invokes FFmpeg", async () => {
|
||||
assert.equal(result.file.type, "audio/wav");
|
||||
assert.equal(result.metadata.browser_processing_used, true);
|
||||
const disabled = new BrowserSubmissionProcessor({ browserProcessingBeta: false }, async () => { throw new Error("must not run"); });
|
||||
assert.deepEqual(await disabled.prepare(video), { file: video, metadata: null });
|
||||
assert.deepEqual(await disabled.prepare(video), { file: video, metadata: null, gate: "unavailable" });
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { BrowserYamnetClassifier } from "../src/scripts/browser-yamnet-classifier.js";
|
||||
|
||||
test("YAMNet peak scores are mapped and all owned resources are disposed", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const disposed = [];
|
||||
const values = new Float32Array(2 * 521);
|
||||
values[0] = .4;
|
||||
values[521] = .9;
|
||||
const output = { shape: [2, 521], data: async () => values, dispose: () => disposed.push("output") };
|
||||
const model = { executeAsync: async () => [output], dispose: () => disposed.push("model") };
|
||||
const tf = {
|
||||
ready: async () => {}, getBackend: () => "cpu", memory: () => ({ numTensors: 0 }),
|
||||
loadGraphModel: async () => model,
|
||||
tensor1d: () => ({ dispose: () => disposed.push("input") }),
|
||||
};
|
||||
globalThis.fetch = async () => ({ ok: true, json: async () => ["Fireworks", ...Array(520).fill("Other")] });
|
||||
try {
|
||||
const predictions = await BrowserYamnetClassifier.classify(new Float32Array(16000), {
|
||||
modelUrl: "/static/browser-yamnet/model.json", classMapUrl: "/static/browser-yamnet/yamnet_class_map.json",
|
||||
loadTensorFlow: async () => tf,
|
||||
});
|
||||
assert.equal(predictions[0].class, "Fireworks");
|
||||
assert.equal(predictions[0].score, values[521]);
|
||||
assert.equal(predictions[0].timeSeconds, .48);
|
||||
assert.deepEqual(disposed, ["input", "output", "model"]);
|
||||
} finally { globalThis.fetch = originalFetch; }
|
||||
});
|
||||
|
||||
test("model failure still disposes the loaded model", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let disposed = false;
|
||||
const tf = {
|
||||
ready: async () => {}, getBackend: () => "cpu", memory: () => ({ numTensors: 0 }),
|
||||
loadGraphModel: async () => ({ dispose: () => { disposed = true; } }),
|
||||
};
|
||||
globalThis.fetch = async () => ({ ok: false });
|
||||
try {
|
||||
await assert.rejects(BrowserYamnetClassifier.classify(new Float32Array(16000), {
|
||||
modelUrl: "/model.json", classMapUrl: "/map.json", loadTensorFlow: async () => tf,
|
||||
}));
|
||||
assert.equal(disposed, true);
|
||||
} finally { globalThis.fetch = originalFetch; }
|
||||
});
|
||||
Reference in New Issue
Block a user