browser audio processing phase 1

This commit is contained in:
Joren de Graaf
2026-10-02 16:44:15 +02:00
parent c93bead437
commit beeb2b5e0e
43 changed files with 684 additions and 33 deletions
+28 -1
View File
@@ -10,7 +10,7 @@ from django.urls import path, reverse
from django.utils.html import format_html
from django.utils.translation import gettext as _, gettext_lazy, ngettext
from .models import ClassificationRule, Event, PendingEvent, Sensor
from .models import BrowserProcessingSettings, ClassificationRule, Event, PendingEvent, Sensor
from . import views as classifier_views
@@ -21,6 +21,33 @@ class ClassificationRuleAdmin(admin.ModelAdmin):
search_fields = ("value",)
@admin.register(BrowserProcessingSettings)
class BrowserProcessingSettingsAdmin(admin.ModelAdmin):
list_display = ("beta_status",)
readonly_fields = ("beta_status",)
fieldsets = (
(gettext_lazy("Browser-side media processing (Beta)"), {
"fields": ("beta_status", "browser_processing_beta_enabled"),
"description": gettext_lazy(
"Enables experimental client-side video audio extraction and YAMNet pre-classification. "
"Browser compatibility may vary. Disabling this restores the normal server-side submission flow."
),
}),
)
@admin.display(description=gettext_lazy("Beta status"))
def beta_status(self, obj):
if obj and obj.browser_processing_beta_enabled:
return format_html('<strong style="color:#166534">{}</strong>', _("Enabled"))
return format_html('<strong style="color:#991b1b">{}</strong>', _("Disabled — normal submission"))
def has_add_permission(self, request):
return super().has_add_permission(request) and not BrowserProcessingSettings.objects.exists()
def has_delete_permission(self, request, obj=None):
return False
class ClassificationAdmin(admin.ModelAdmin):
list_display = ("original_filename", "source_type", "sensor", "occurrences", "location", "visually_confirmed", "primary_label", "primary_score", "duration", "created_at")
list_filter = ("source_type", "visually_confirmed", "created_at")
+67
View File
@@ -0,0 +1,67 @@
"""Public, advisory configuration for the optional browser processing beta."""
import json
from django.urls import reverse
from .models import BrowserProcessingSettings, ClassificationRule
POLICY_VERSION = 1
SPEECH_LABEL_TERMS = (
"speech", "conversation", "narration", "monologue", "babbling", "whisper",
"speaking", "speech synthesizer",
)
GATE_RESULTS = {"whitelist_match", "no_match", "unavailable", "error"}
def browser_config():
"""Keep template configuration in one small, non-secret object."""
return {
"browserProcessingBeta": BrowserProcessingSettings.enabled(),
"policyUrl": reverse("fireworks_detect:browser_policy"),
"policyVersion": POLICY_VERSION,
}
def browser_policy():
enabled = BrowserProcessingSettings.enabled()
policy = {
"version": POLICY_VERSION,
"browser_processing_beta_enabled": enabled,
"whitelist": [],
"privacy_sensitive_speech": [],
"thresholds": {},
}
if not enabled:
return policy
rules = ClassificationRule.objects.filter(enabled=True).order_by("value")
policy["whitelist"] = list(rules.filter(rule_type=ClassificationRule.RuleType.WHITELIST).values_list("value", flat=True))
policy["privacy_sensitive_speech"] = [
value for value in rules.filter(rule_type=ClassificationRule.RuleType.BLACKLIST).values_list("value", flat=True)
if any(term in value.lower() for term in SPEECH_LABEL_TERMS)
]
return policy
def advisory_browser_metadata(post_data):
"""Parse bounded diagnostics; never use this to authorize or classify audio."""
raw = post_data.get("browser_processing_metadata", "")
if not raw or len(raw) > 1024:
return {}
try:
submitted = json.loads(raw)
except (TypeError, ValueError):
return {}
if not isinstance(submitted, dict):
return {}
metadata = {}
for key in ("browser_processing_used", "browser_override"):
if type(submitted.get(key)) is bool:
metadata[key] = submitted[key]
version = submitted.get("browser_processing_version")
if type(version) is int and 0 < version <= POLICY_VERSION:
metadata["browser_processing_version"] = version
gate_result = submitted.get("browser_gate_result")
if isinstance(gate_result, str) and gate_result in GATE_RESULTS:
metadata["browser_gate_result"] = gate_result
return metadata
@@ -0,0 +1,24 @@
# Generated by Django 6.1.1 on 2026-10-02 14:31
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('fireworks_detect', '0008_sensor_owner_sensor_archived_at'),
]
operations = [
migrations.CreateModel(
name='BrowserProcessingSettings',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('browser_processing_beta_enabled', models.BooleanField(default=False)),
],
options={
'verbose_name': 'Browser-side media processing (Beta)',
'verbose_name_plural': 'Browser-side media processing (Beta)',
},
),
]
@@ -0,0 +1,16 @@
from django.db import migrations
def create_disabled_settings(apps, schema_editor):
Settings = apps.get_model("fireworks_detect", "BrowserProcessingSettings")
Settings.objects.get_or_create(pk=1, defaults={"browser_processing_beta_enabled": False})
class Migration(migrations.Migration):
dependencies = [
("fireworks_detect", "0009_browserprocessingsettings"),
]
operations = [
migrations.RunPython(create_disabled_settings, migrations.RunPython.noop),
]
+21
View File
@@ -88,6 +88,27 @@ class ClassificationRule(models.Model):
return f"{self.value} ({self.get_rule_type_display()})"
class BrowserProcessingSettings(models.Model):
"""Single site-wide switch for experimental browser processing."""
browser_processing_beta_enabled = models.BooleanField(default=False)
class Meta:
verbose_name = _("Browser-side media processing (Beta)")
verbose_name_plural = _("Browser-side media processing (Beta)")
def save(self, *args, **kwargs):
self.pk = 1
return super().save(*args, **kwargs)
@classmethod
def enabled(cls):
return bool(cls.objects.filter(pk=1).values_list("browser_processing_beta_enabled", flat=True).first())
def __str__(self):
return str(_("Browser-side media processing (Beta)"))
class ClassifiedAudio(models.Model):
"""Shared fields for a classifier response and its original audio upload."""
@@ -9,6 +9,7 @@
<script src="https://cdn.jsdelivr.net/npm/htmx.org@2.0.4/dist/htmx.min.js"></script>
{% vite_asset "frontend/src/entries/dashboard.js" %}
{% include "js_translations.html" %}
{% include "fireworks_detect/includes/browser_processing_config.html" %}
</head>
<body class="dashboard-page">
{% if user.is_authenticated %}<a class="profile-circle" href="{% url "user_settings" %}" aria-label="{% blocktrans with username=user.get_username %}Open settings for {{ username }}{% endblocktrans %}" title="{% trans "Settings" %}">{{ user.get_username|first|upper }}</a>{% endif %}
@@ -9,6 +9,7 @@
<script src="https://cdn.jsdelivr.net/npm/htmx.org@2.0.4/dist/htmx.min.js"></script>
{% vite_asset "frontend/src/entries/fireworks-home.js" %}
{% include "js_translations.html" %}
{% include "fireworks_detect/includes/browser_processing_config.html" %}
</head>
<body>
@@ -30,4 +31,4 @@
</main>
</body>
</html>
</html>
@@ -1,10 +1,10 @@
{% load i18n %}
<form class="upload-form" action="{% url "fireworks_detect:classify_upload" %}" method="post" enctype="multipart/form-data" hx-post="{% url "fireworks_detect:classify_upload" %}" hx-target="#classification-results" hx-swap="innerHTML" hx-encoding="multipart/form-data" hx-indicator="#upload-indicator">
<form class="upload-form audio-upload--idle" action="{% url "fireworks_detect:classify_upload" %}" method="post" enctype="multipart/form-data" hx-post="{% url "fireworks_detect:classify_upload" %}" hx-target="#classification-results" hx-swap="innerHTML" hx-encoding="multipart/form-data" hx-indicator="#upload-indicator" data-scroll-to-results="true">
{% csrf_token %}
<label class="drop-zone" id="drop-zone">
<input class="file-input" id="audio-file" type="file" name="audio" accept="audio/*,.wav,.mp3,.m4a,.ogg,.flac,.aac,.opus" required>
<span>
<span class="upload-icon">↑</span>
<span class="upload-icon" id="upload-progress-circle"><span class="upload-icon__content" id="upload-icon-content" aria-hidden="true">↑</span></span>
<strong>{% trans "Drop an audio file here" %}</strong>
<span>{% trans "or click to browse your device" %}</span>
</span>
@@ -13,6 +13,7 @@
<div class="form-footer">
<p class="file-note">{% trans "WAV, MP3, M4A, OGG and more · 25 MiB maximum" %}</p>
<button type="submit"><span class="spinner htmx-indicator" id="upload-indicator" aria-hidden="true"></span>{% trans "Analyse audio" %}</button>
<button class="upload-cancel" type="button" hidden>{% trans "Cancel upload" %}</button>
</div>
<section id="classification-results" aria-live="polite" aria-label="{% trans "Classification results" %}"></section>
</form>
@@ -0,0 +1 @@
{{ browser_processing_config|json_script:"boomcast-browser-config" }}
+101
View File
@@ -0,0 +1,101 @@
import json
from tempfile import TemporaryDirectory
from unittest.mock import MagicMock, patch
from django.contrib.auth import get_user_model
from django.core import signing
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.files.storage import FileSystemStorage
from django.test import TestCase, override_settings
from django.urls import reverse
from .browser_policy import advisory_browser_metadata
from .models import BrowserProcessingSettings, ClassificationRule, PendingEvent
class BrowserProcessingBetaTests(TestCase):
def test_setting_defaults_off_and_admin_can_toggle_it(self):
self.assertFalse(BrowserProcessingSettings.enabled())
settings = BrowserProcessingSettings.objects.get(pk=1)
self.assertFalse(settings.browser_processing_beta_enabled)
staff = get_user_model().objects.create_superuser("beta-admin", "beta@example.com", "password")
self.client.force_login(staff)
changelist = self.client.get(reverse("admin:fireworks_detect_browserprocessingsettings_changelist"))
self.assertContains(changelist, "Disabled — normal submission")
url = reverse("admin:fireworks_detect_browserprocessingsettings_change", args=[settings.pk])
self.assertContains(self.client.get(url), "Browser-side media processing (Beta)")
self.assertEqual(self.client.post(url, {"browser_processing_beta_enabled": "on", "_save": "Save"}).status_code, 302)
self.assertTrue(BrowserProcessingSettings.enabled())
self.assertEqual(self.client.post(url, {"_save": "Save"}).status_code, 302)
self.assertFalse(BrowserProcessingSettings.enabled())
self.assertEqual(BrowserProcessingSettings.objects.count(), 1)
def test_policy_and_page_config_are_empty_while_disabled(self):
ClassificationRule.objects.create(value="Fireworks", rule_type=ClassificationRule.RuleType.WHITELIST)
ClassificationRule.objects.create(value="Speech", rule_type=ClassificationRule.RuleType.BLACKLIST)
response = self.client.get(reverse("fireworks_detect:browser_policy"))
self.assertEqual(response.json(), {
"version": 1, "browser_processing_beta_enabled": False,
"whitelist": [], "privacy_sensitive_speech": [], "thresholds": {},
})
self.assertIn("no-cache", response["Cache-Control"])
for url in (reverse("hello"), reverse("fireworks_detect:fireworks")):
page = self.client.get(url)
self.assertContains(page, '"browserProcessingBeta": false')
self.assertContains(page, 'id="boomcast-browser-config"')
def test_enabled_policy_reuses_current_rules_and_exposes_no_secrets(self):
BrowserProcessingSettings.objects.update(browser_processing_beta_enabled=True)
ClassificationRule.objects.create(value="Fireworks", rule_type=ClassificationRule.RuleType.WHITELIST)
ClassificationRule.objects.create(value="Disabled sound", rule_type=ClassificationRule.RuleType.WHITELIST, enabled=False)
ClassificationRule.objects.create(value="Speech", rule_type=ClassificationRule.RuleType.BLACKLIST)
ClassificationRule.objects.create(value="Music", rule_type=ClassificationRule.RuleType.BLACKLIST)
policy = self.client.get(reverse("fireworks_detect:browser_policy")).json()
self.assertTrue(policy["browser_processing_beta_enabled"])
self.assertEqual(policy["whitelist"], ["Fireworks"])
self.assertEqual(policy["privacy_sensitive_speech"], ["Speech"])
self.assertEqual(policy["thresholds"], {})
self.assertNotIn("token", json.dumps(policy).lower())
BrowserProcessingSettings.objects.update(browser_processing_beta_enabled=False)
self.assertFalse(self.client.get(reverse("fireworks_detect:browser_policy")).json()["browser_processing_beta_enabled"])
@patch("fireworks_detect.views.urlopen")
def test_old_upload_and_advisory_metadata_keep_server_classifier_authoritative(self, urlopen):
reply = MagicMock()
reply.read.return_value = b'{"predictions": [{"class": "Speech", "score": 0.9}]}'
urlopen.return_value.__enter__.return_value = reply
url = reverse("fireworks_detect:classify_upload")
audio = lambda: SimpleUploadedFile("sample.wav", b"RIFF0000WAVEdata", content_type="audio/wav")
old = self.client.post(url, {"audio": audio()})
self.assertEqual(old.status_code, 200)
self.assertContains(old, "Speech")
metadata = json.dumps({
"browser_processing_used": True, "browser_processing_version": 1,
"browser_gate_result": "whitelist_match", "browser_override": False,
"secret": "ignored",
})
future = self.client.post(url, {"audio": audio(), "browser_processing_metadata": metadata})
self.assertEqual(future.status_code, 200)
self.assertContains(future, "Speech")
self.assertNotContains(future, "whitelist_match")
self.assertEqual(urlopen.call_count, 2)
self.assertNotIn(b"whitelist_match", urlopen.call_args.args[0].data)
self.assertEqual(advisory_browser_metadata({"browser_processing_metadata": metadata}), {
"browser_processing_used": True, "browser_processing_version": 1,
"browser_gate_result": "whitelist_match", "browser_override": False,
})
self.assertEqual(advisory_browser_metadata({"browser_processing_metadata": '{"browser_gate_result": []}'}), {})
def test_pending_submission_accepts_metadata_without_storing_it_as_classification(self):
payload = {"predictions": [{"class": "Fireworks", "score": 0.8}]}
token = signing.dumps(payload, salt="fireworks_detect.classification")
with TemporaryDirectory() as media_root, override_settings(MEDIA_ROOT=media_root), patch.object(
PendingEvent._meta.get_field("audio_file"), "storage", FileSystemStorage(location=media_root)
):
response = self.client.post(reverse("fireworks_detect:submit_pending"), {
"audio": SimpleUploadedFile("sample.wav", b"RIFF0000WAVEdata", content_type="audio/wav"),
"classification_token": token,
"browser_processing_metadata": '{"browser_processing_used": true, "browser_gate_result": "whitelist_match"}',
})
self.assertEqual(response.status_code, 200)
self.assertEqual(PendingEvent.objects.get().classifier_response, payload)
+2
View File
@@ -13,6 +13,7 @@ from .views import (
submit_manual_pending_event,
submit_pending_event,
sensor_heartbeat,
browser_processing_policy,
)
app_name = "fireworks_detect"
@@ -22,6 +23,7 @@ urlpatterns = [
path("map/", fireworks_map_view, name="map"),
path("classify/", classify_audio, name="classify"),
path("classify/upload/", classify_upload, name="classify_upload"),
path("browser-policy/", browser_processing_policy, name="browser_policy"),
path("heartbeat/", sensor_heartbeat, name="sensor_heartbeat"),
path("submit-pending/", submit_pending_event, name="submit_pending"),
path("submit-manual-pending/", submit_manual_pending_event, name="submit_manual_pending"),
+18 -1
View File
@@ -2,6 +2,7 @@ import math
from datetime import timedelta
from decimal import Decimal, InvalidOperation
import json
import logging
import mimetypes
import uuid
from pathlib import Path
@@ -20,6 +21,7 @@ from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from .h3_utils import cell_boundary, cell_center, cell_from_coordinates, h3_resolution
from .browser_policy import advisory_browser_metadata, browser_config, browser_policy
from .models import ClassificationRule, Event, PendingEvent, Sensor
@@ -35,11 +37,18 @@ SPEECH_TERMS = (
"speaking", "speech synthesizer",
)
AUDIO_EXTENSIONS = {".aac", ".aif", ".aiff", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav", ".weba"}
logger = logging.getLogger(__name__)
@require_http_methods(["GET"])
@never_cache
def browser_processing_policy(request):
return JsonResponse(browser_policy())
@never_cache
def fireworks_view(request):
return render(request, "fireworks_detect/home.html")
return render(request, "fireworks_detect/home.html", {"browser_processing_config": browser_config()})
MAP_DATE_RANGES = {
@@ -157,6 +166,7 @@ def dashboard_view(request):
"h3_resolution": h3_resolution(),
"h3_cells": h3_cells,
"map_events": map_events,
"browser_processing_config": browser_config(),
})
@@ -248,6 +258,10 @@ def classify_upload(request):
if uploaded_file.size > MAX_AUDIO_SIZE:
return _render_error(request, _("This demo accepts audio files up to 25 MiB."))
metadata = advisory_browser_metadata(request.POST)
if metadata:
logger.info("Browser beta diagnostics (advisory only): %s", metadata)
body, content_type = _encode_audio_multipart(uploaded_file)
classifier_request = Request(
CLASSIFIER_URL,
@@ -279,6 +293,9 @@ def classify_upload(request):
@require_http_methods(["POST"])
@never_cache
def submit_pending_event(request):
metadata = advisory_browser_metadata(request.POST)
if metadata:
logger.info("Browser beta submission diagnostics (advisory only): %s", metadata)
uploaded_file = request.FILES.get("audio")
token = request.POST.get("classification_token")
if not uploaded_file: