browser audio detection phase 3

This commit is contained in:
Joren de Graaf
2026-10-02 21:34:38 +02:00
parent 5c0ca7bc47
commit 20bdabdd8f
50 changed files with 24755 additions and 144 deletions
+18
View File
@@ -0,0 +1,18 @@
# Browser YAMNet model
The browser GraphModel uses the [official TensorFlow YAMNet implementation](https://github.com/tensorflow/models/tree/master/research/audioset/yamnet) and [published weights](https://storage.googleapis.com/audioset/yamnet.h5). The official 521-class AudioSet map is included beside it. Runtime assets live in `fireworks_detect/static/browser-yamnet/`; browsers fetch them only from BoomCast. `VERSION` records the pinned source revision and TensorFlow version.
To regenerate, use a dedicated conversion environment. The conversion script downloads the pinned source and weights, exports a SavedModel, converts it to a TensorFlow.js GraphModel, and writes the class map and version file. It supplies scoped empty modules for the converter's unused TF Decision Forests, TF Hub, and JAX imports; YAMNet does not use those features. The following package set was used for this artifact on Python 3.13:
```sh
mkdir -p .yamnet-build-tmp
python3 -m venv .yamnet-convert
TMPDIR="$PWD/.yamnet-build-tmp" .yamnet-convert/bin/pip install --no-cache-dir tensorflow-cpu tensorflowjs tensorflow-hub tf-keras
TMPDIR="$PWD/.yamnet-build-tmp" .yamnet-convert/bin/pip install --no-cache-dir --no-deps tensorflowjs==4.22.0
TF_USE_LEGACY_KERAS=1 TMPDIR="$PWD/.yamnet-build-tmp" .yamnet-convert/bin/python scripts/build_browser_yamnet.py
node scripts/verify_browser_yamnet.mjs
npm run build
python manage.py collectstatic --noinput
```
The verifier runs an actual TensorFlow.js inference and checks the 521-class score output. Keep the class map aligned with the model weights. The browser's relevance and speech thresholds are triage settings in `browser_policy.py`; the server classifier remains authoritative. Compare changed model outputs with representative clips before enabling the beta after an upgrade.
+80
View File
@@ -0,0 +1,80 @@
"""Regenerate the self-hosted YAMNet TensorFlow.js GraphModel.
Run in a dedicated Python environment with tensorflow-cpu, tensorflowjs,
TensorFlow Hub, and tf-keras installed. Uses the official TensorFlow YAMNet
implementation and published weights; normal site traffic never contacts them.
"""
import csv
import json
import os
import subprocess
import sys
import tempfile
import urllib.request
from pathlib import Path
SOURCE_REVISION = "c651073789719ff39dcf51406733bbcaaafb43f2"
SOURCE_BASE = f"https://raw.githubusercontent.com/tensorflow/models/{SOURCE_REVISION}/research/audioset/yamnet"
WEIGHTS_URL = "https://storage.googleapis.com/audioset/yamnet.h5"
OUTPUT = Path(__file__).resolve().parents[1] / "fireworks_detect/static/browser-yamnet"
def download(url, target):
urllib.request.urlretrieve(url, target)
def main():
os.environ.setdefault("TF_USE_LEGACY_KERAS", "1")
with tempfile.TemporaryDirectory() as directory:
directory = Path(directory)
for name in ("yamnet.py", "features.py", "params.py", "yamnet_class_map.csv"):
download(f"{SOURCE_BASE}/{name}", directory / name)
download(WEIGHTS_URL, directory / "yamnet.h5")
sys.path.insert(0, str(directory))
import tensorflow as tf
import params
import yamnet
class Export(tf.Module):
def __init__(self):
super().__init__()
self.model = yamnet.yamnet_frames_model(params.Params(tflite_compatible=True))
self.model.load_weights(str(directory / "yamnet.h5"))
@tf.function(input_signature=[tf.TensorSpec([None], tf.float32, name="waveform")])
def classify(self, waveform):
scores, embeddings, spectrogram = self.model(waveform)
return {"predictions": scores, "embeddings": embeddings, "log_mel_spectrogram": spectrogram}
export = Export()
with tempfile.TemporaryDirectory() as saved_dir:
tf.saved_model.save(export, saved_dir, signatures={"serving_default": export.classify})
OUTPUT.mkdir(parents=True, exist_ok=True)
# The converter imports TF Decision Forests unconditionally even
# though YAMNet contains no decision forest ops. Keep this stub
# scoped to the conversion subprocess rather than installing TFDF.
(directory / "tensorflow_decision_forests.py").write_text("")
(directory / "tensorflow_hub.py").write_text("")
(directory / "jax/experimental").mkdir(parents=True)
(directory / "jax/__init__.py").write_text("")
(directory / "jax/experimental/__init__.py").write_text("")
(directory / "jax/experimental/jax2tf.py").write_text("")
converter_env = {**os.environ, "PYTHONPATH": str(directory)}
subprocess.run([
sys.executable, "-m", "tensorflowjs.converters.converter",
"--input_format=tf_saved_model", "--output_format=tfjs_graph_model",
"--signature_name=serving_default", saved_dir, str(OUTPUT),
], check=True, env=converter_env)
with (directory / "yamnet_class_map.csv").open(newline="") as source:
rows = list(csv.DictReader(source))
assert len(rows) == 521
(OUTPUT / "yamnet_class_map.csv").write_bytes((directory / "yamnet_class_map.csv").read_bytes())
(OUTPUT / "yamnet_class_map.json").write_text(json.dumps([row["display_name"] for row in rows]))
(OUTPUT / "VERSION").write_text(
f"google-yamnet-1\nsource_revision={SOURCE_REVISION}\ntensorflow={tf.__version__}\n"
)
print(f"Browser YAMNet GraphModel written to {OUTPUT}")
if __name__ == "__main__":
main()
+29
View File
@@ -0,0 +1,29 @@
import { readFileSync } from "node:fs";
import { join, dirname } from "node:path";
import * as tf from "@tensorflow/tfjs";
const directory = new URL("../fireworks_detect/static/browser-yamnet/", import.meta.url).pathname;
const manifest = JSON.parse(readFileSync(join(directory, "model.json"), "utf8"));
const mapping = JSON.parse(readFileSync(join(directory, "yamnet_class_map.json"), "utf8"));
if (manifest.format !== "graph-model" || mapping.length !== 521) throw new Error("Invalid browser YAMNet assets");
const specs = manifest.weightsManifest.flatMap((group) => group.weights);
const chunks = manifest.weightsManifest.flatMap((group) => group.paths.map((path) => readFileSync(join(directory, path))));
const combined = Buffer.concat(chunks);
const model = await tf.loadGraphModel({ load: async () => ({
modelTopology: manifest.modelTopology,
weightSpecs: specs,
weightData: combined.buffer.slice(combined.byteOffset, combined.byteOffset + combined.byteLength),
}) });
const input = tf.zeros([16000]);
let outputs;
try {
outputs = await model.executeAsync(input);
const tensors = Array.isArray(outputs) ? outputs : [outputs];
const scores = tensors.find((tensor) => tensor.shape.length === 2 && tensor.shape[1] === 521);
if (!scores || scores.shape[0] < 1) throw new Error("YAMNet did not produce scores");
console.log("Verified browser YAMNet score tensor:", scores.shape);
} finally {
input.dispose();
(Array.isArray(outputs) ? outputs : [outputs]).forEach((tensor) => tensor?.dispose());
model.dispose();
}