first commit
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
# Browser YAMNet model
|
||||
|
||||
The browser GraphModel uses the [official TensorFlow YAMNet implementation](https://github.com/tensorflow/models/tree/master/research/audioset/yamnet) and [published weights](https://storage.googleapis.com/audioset/yamnet.h5). The official 521-class AudioSet map is included beside it. Runtime assets live in `fireworks_detect/static/browser-yamnet/`; browsers fetch them only from BoomCast. `VERSION` records the pinned source revision and TensorFlow version.
|
||||
|
||||
To regenerate, use a dedicated conversion environment. The conversion script downloads the pinned source and weights, exports a SavedModel, converts it to a TensorFlow.js GraphModel, and writes the class map and version file. It supplies scoped empty modules for the converter's unused TF Decision Forests, TF Hub, and JAX imports; YAMNet does not use those features. The following package set was used for this artifact on Python 3.13:
|
||||
|
||||
```sh
|
||||
mkdir -p .yamnet-build-tmp
|
||||
python3 -m venv .yamnet-convert
|
||||
TMPDIR="$PWD/.yamnet-build-tmp" .yamnet-convert/bin/pip install --no-cache-dir tensorflow-cpu tensorflowjs tensorflow-hub tf-keras
|
||||
TMPDIR="$PWD/.yamnet-build-tmp" .yamnet-convert/bin/pip install --no-cache-dir --no-deps tensorflowjs==4.22.0
|
||||
TF_USE_LEGACY_KERAS=1 TMPDIR="$PWD/.yamnet-build-tmp" .yamnet-convert/bin/python scripts/build_browser_yamnet.py
|
||||
node scripts/verify_browser_yamnet.mjs
|
||||
npm run build
|
||||
python manage.py collectstatic --noinput
|
||||
```
|
||||
|
||||
The verifier runs an actual TensorFlow.js inference and checks the 521-class score output. Keep the class map aligned with the model weights. The browser's relevance and speech thresholds are triage settings in `browser_policy.py`; the server classifier remains authoritative. Compare changed model outputs with representative clips before enabling the beta after an upgrade.
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Regenerate the self-hosted YAMNet TensorFlow.js GraphModel.
|
||||
|
||||
Run in a dedicated Python environment with tensorflow-cpu, tensorflowjs,
|
||||
TensorFlow Hub, and tf-keras installed. Uses the official TensorFlow YAMNet
|
||||
implementation and published weights; normal site traffic never contacts them.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
SOURCE_REVISION = "c651073789719ff39dcf51406733bbcaaafb43f2"
|
||||
SOURCE_BASE = f"https://raw.githubusercontent.com/tensorflow/models/{SOURCE_REVISION}/research/audioset/yamnet"
|
||||
WEIGHTS_URL = "https://storage.googleapis.com/audioset/yamnet.h5"
|
||||
OUTPUT = Path(__file__).resolve().parents[1] / "fireworks_detect/static/browser-yamnet"
|
||||
|
||||
|
||||
def download(url, target):
|
||||
urllib.request.urlretrieve(url, target)
|
||||
|
||||
|
||||
def main():
|
||||
os.environ.setdefault("TF_USE_LEGACY_KERAS", "1")
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
directory = Path(directory)
|
||||
for name in ("yamnet.py", "features.py", "params.py", "yamnet_class_map.csv"):
|
||||
download(f"{SOURCE_BASE}/{name}", directory / name)
|
||||
download(WEIGHTS_URL, directory / "yamnet.h5")
|
||||
sys.path.insert(0, str(directory))
|
||||
import tensorflow as tf
|
||||
import params
|
||||
import yamnet
|
||||
|
||||
class Export(tf.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.model = yamnet.yamnet_frames_model(params.Params(tflite_compatible=True))
|
||||
self.model.load_weights(str(directory / "yamnet.h5"))
|
||||
|
||||
@tf.function(input_signature=[tf.TensorSpec([None], tf.float32, name="waveform")])
|
||||
def classify(self, waveform):
|
||||
scores, embeddings, spectrogram = self.model(waveform)
|
||||
return {"predictions": scores, "embeddings": embeddings, "log_mel_spectrogram": spectrogram}
|
||||
|
||||
export = Export()
|
||||
with tempfile.TemporaryDirectory() as saved_dir:
|
||||
tf.saved_model.save(export, saved_dir, signatures={"serving_default": export.classify})
|
||||
OUTPUT.mkdir(parents=True, exist_ok=True)
|
||||
# The converter imports TF Decision Forests unconditionally even
|
||||
# though YAMNet contains no decision forest ops. Keep this stub
|
||||
# scoped to the conversion subprocess rather than installing TFDF.
|
||||
(directory / "tensorflow_decision_forests.py").write_text("")
|
||||
(directory / "tensorflow_hub.py").write_text("")
|
||||
(directory / "jax/experimental").mkdir(parents=True)
|
||||
(directory / "jax/__init__.py").write_text("")
|
||||
(directory / "jax/experimental/__init__.py").write_text("")
|
||||
(directory / "jax/experimental/jax2tf.py").write_text("")
|
||||
converter_env = {**os.environ, "PYTHONPATH": str(directory)}
|
||||
subprocess.run([
|
||||
sys.executable, "-m", "tensorflowjs.converters.converter",
|
||||
"--input_format=tf_saved_model", "--output_format=tfjs_graph_model",
|
||||
"--signature_name=serving_default", saved_dir, str(OUTPUT),
|
||||
], check=True, env=converter_env)
|
||||
with (directory / "yamnet_class_map.csv").open(newline="") as source:
|
||||
rows = list(csv.DictReader(source))
|
||||
assert len(rows) == 521
|
||||
(OUTPUT / "yamnet_class_map.csv").write_bytes((directory / "yamnet_class_map.csv").read_bytes())
|
||||
(OUTPUT / "yamnet_class_map.json").write_text(json.dumps([row["display_name"] for row in rows]))
|
||||
(OUTPUT / "VERSION").write_text(
|
||||
f"google-yamnet-1\nsource_revision={SOURCE_REVISION}\ntensorflow={tf.__version__}\n"
|
||||
)
|
||||
print(f"Browser YAMNet GraphModel written to {OUTPUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const host = process.env.DJANGO_HOST ?? "0.0.0.0";
|
||||
const port = process.env.DJANGO_PORT ?? "8000";
|
||||
const command = process.env.DJANGO_DEV_COMMAND ?? `.venv/bin/python manage.py runserver ${host}:${port}`;
|
||||
const child = spawn(command, { shell: true, stdio: "inherit", env: { ...process.env, VITE_USE_DEV_SERVER: "1", DJANGO_DEBUG: "1" } });
|
||||
|
||||
child.on("exit", (code, signal) => {
|
||||
if (signal) {
|
||||
process.kill(process.pid, signal);
|
||||
}
|
||||
process.exit(code ?? 0);
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import * as tf from "@tensorflow/tfjs";
|
||||
|
||||
const directory = new URL("../fireworks_detect/static/browser-yamnet/", import.meta.url).pathname;
|
||||
const manifest = JSON.parse(readFileSync(join(directory, "model.json"), "utf8"));
|
||||
const mapping = JSON.parse(readFileSync(join(directory, "yamnet_class_map.json"), "utf8"));
|
||||
if (manifest.format !== "graph-model" || mapping.length !== 521) throw new Error("Invalid browser YAMNet assets");
|
||||
const specs = manifest.weightsManifest.flatMap((group) => group.weights);
|
||||
const chunks = manifest.weightsManifest.flatMap((group) => group.paths.map((path) => readFileSync(join(directory, path))));
|
||||
const combined = Buffer.concat(chunks);
|
||||
const model = await tf.loadGraphModel({ load: async () => ({
|
||||
modelTopology: manifest.modelTopology,
|
||||
weightSpecs: specs,
|
||||
weightData: combined.buffer.slice(combined.byteOffset, combined.byteOffset + combined.byteLength),
|
||||
}) });
|
||||
const input = tf.zeros([16000]);
|
||||
let outputs;
|
||||
try {
|
||||
outputs = await model.executeAsync(input);
|
||||
const tensors = Array.isArray(outputs) ? outputs : [outputs];
|
||||
const scores = tensors.find((tensor) => tensor.shape.length === 2 && tensor.shape[1] === 521);
|
||||
if (!scores || scores.shape[0] < 1) throw new Error("YAMNet did not produce scores");
|
||||
console.log("Verified browser YAMNet score tensor:", scores.shape);
|
||||
} finally {
|
||||
input.dispose();
|
||||
(Array.isArray(outputs) ? outputs : [outputs]).forEach((tensor) => tensor?.dispose());
|
||||
model.dispose();
|
||||
}
|
||||
Reference in New Issue
Block a user