browser audio processing phase 1
This commit is contained in:
@@ -10,7 +10,7 @@ from django.urls import path, reverse
|
||||
from django.utils.html import format_html
|
||||
from django.utils.translation import gettext as _, gettext_lazy, ngettext
|
||||
|
||||
from .models import ClassificationRule, Event, PendingEvent, Sensor
|
||||
from .models import BrowserProcessingSettings, ClassificationRule, Event, PendingEvent, Sensor
|
||||
from . import views as classifier_views
|
||||
|
||||
|
||||
@@ -21,6 +21,33 @@ class ClassificationRuleAdmin(admin.ModelAdmin):
|
||||
search_fields = ("value",)
|
||||
|
||||
|
||||
@admin.register(BrowserProcessingSettings)
|
||||
class BrowserProcessingSettingsAdmin(admin.ModelAdmin):
|
||||
list_display = ("beta_status",)
|
||||
readonly_fields = ("beta_status",)
|
||||
fieldsets = (
|
||||
(gettext_lazy("Browser-side media processing (Beta)"), {
|
||||
"fields": ("beta_status", "browser_processing_beta_enabled"),
|
||||
"description": gettext_lazy(
|
||||
"Enables experimental client-side video audio extraction and YAMNet pre-classification. "
|
||||
"Browser compatibility may vary. Disabling this restores the normal server-side submission flow."
|
||||
),
|
||||
}),
|
||||
)
|
||||
|
||||
@admin.display(description=gettext_lazy("Beta status"))
|
||||
def beta_status(self, obj):
|
||||
if obj and obj.browser_processing_beta_enabled:
|
||||
return format_html('<strong style="color:#166534">{}</strong>', _("Enabled"))
|
||||
return format_html('<strong style="color:#991b1b">{}</strong>', _("Disabled — normal submission"))
|
||||
|
||||
def has_add_permission(self, request):
|
||||
return super().has_add_permission(request) and not BrowserProcessingSettings.objects.exists()
|
||||
|
||||
def has_delete_permission(self, request, obj=None):
|
||||
return False
|
||||
|
||||
|
||||
class ClassificationAdmin(admin.ModelAdmin):
|
||||
list_display = ("original_filename", "source_type", "sensor", "occurrences", "location", "visually_confirmed", "primary_label", "primary_score", "duration", "created_at")
|
||||
list_filter = ("source_type", "visually_confirmed", "created_at")
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Public, advisory configuration for the optional browser processing beta."""
|
||||
|
||||
import json
|
||||
|
||||
from django.urls import reverse
|
||||
|
||||
from .models import BrowserProcessingSettings, ClassificationRule
|
||||
|
||||
POLICY_VERSION = 1
|
||||
SPEECH_LABEL_TERMS = (
|
||||
"speech", "conversation", "narration", "monologue", "babbling", "whisper",
|
||||
"speaking", "speech synthesizer",
|
||||
)
|
||||
GATE_RESULTS = {"whitelist_match", "no_match", "unavailable", "error"}
|
||||
|
||||
|
||||
def browser_config():
|
||||
"""Keep template configuration in one small, non-secret object."""
|
||||
return {
|
||||
"browserProcessingBeta": BrowserProcessingSettings.enabled(),
|
||||
"policyUrl": reverse("fireworks_detect:browser_policy"),
|
||||
"policyVersion": POLICY_VERSION,
|
||||
}
|
||||
|
||||
|
||||
def browser_policy():
|
||||
enabled = BrowserProcessingSettings.enabled()
|
||||
policy = {
|
||||
"version": POLICY_VERSION,
|
||||
"browser_processing_beta_enabled": enabled,
|
||||
"whitelist": [],
|
||||
"privacy_sensitive_speech": [],
|
||||
"thresholds": {},
|
||||
}
|
||||
if not enabled:
|
||||
return policy
|
||||
rules = ClassificationRule.objects.filter(enabled=True).order_by("value")
|
||||
policy["whitelist"] = list(rules.filter(rule_type=ClassificationRule.RuleType.WHITELIST).values_list("value", flat=True))
|
||||
policy["privacy_sensitive_speech"] = [
|
||||
value for value in rules.filter(rule_type=ClassificationRule.RuleType.BLACKLIST).values_list("value", flat=True)
|
||||
if any(term in value.lower() for term in SPEECH_LABEL_TERMS)
|
||||
]
|
||||
return policy
|
||||
|
||||
|
||||
def advisory_browser_metadata(post_data):
|
||||
"""Parse bounded diagnostics; never use this to authorize or classify audio."""
|
||||
raw = post_data.get("browser_processing_metadata", "")
|
||||
if not raw or len(raw) > 1024:
|
||||
return {}
|
||||
try:
|
||||
submitted = json.loads(raw)
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
if not isinstance(submitted, dict):
|
||||
return {}
|
||||
metadata = {}
|
||||
for key in ("browser_processing_used", "browser_override"):
|
||||
if type(submitted.get(key)) is bool:
|
||||
metadata[key] = submitted[key]
|
||||
version = submitted.get("browser_processing_version")
|
||||
if type(version) is int and 0 < version <= POLICY_VERSION:
|
||||
metadata["browser_processing_version"] = version
|
||||
gate_result = submitted.get("browser_gate_result")
|
||||
if isinstance(gate_result, str) and gate_result in GATE_RESULTS:
|
||||
metadata["browser_gate_result"] = gate_result
|
||||
return metadata
|
||||
@@ -0,0 +1,24 @@
|
||||
# Generated by Django 6.1.1 on 2026-10-02 14:31
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('fireworks_detect', '0008_sensor_owner_sensor_archived_at'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='BrowserProcessingSettings',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('browser_processing_beta_enabled', models.BooleanField(default=False)),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'Browser-side media processing (Beta)',
|
||||
'verbose_name_plural': 'Browser-side media processing (Beta)',
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,16 @@
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
def create_disabled_settings(apps, schema_editor):
|
||||
Settings = apps.get_model("fireworks_detect", "BrowserProcessingSettings")
|
||||
Settings.objects.get_or_create(pk=1, defaults={"browser_processing_beta_enabled": False})
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("fireworks_detect", "0009_browserprocessingsettings"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(create_disabled_settings, migrations.RunPython.noop),
|
||||
]
|
||||
@@ -88,6 +88,27 @@ class ClassificationRule(models.Model):
|
||||
return f"{self.value} ({self.get_rule_type_display()})"
|
||||
|
||||
|
||||
class BrowserProcessingSettings(models.Model):
|
||||
"""Single site-wide switch for experimental browser processing."""
|
||||
|
||||
browser_processing_beta_enabled = models.BooleanField(default=False)
|
||||
|
||||
class Meta:
|
||||
verbose_name = _("Browser-side media processing (Beta)")
|
||||
verbose_name_plural = _("Browser-side media processing (Beta)")
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
self.pk = 1
|
||||
return super().save(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def enabled(cls):
|
||||
return bool(cls.objects.filter(pk=1).values_list("browser_processing_beta_enabled", flat=True).first())
|
||||
|
||||
def __str__(self):
|
||||
return str(_("Browser-side media processing (Beta)"))
|
||||
|
||||
|
||||
class ClassifiedAudio(models.Model):
|
||||
"""Shared fields for a classifier response and its original audio upload."""
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
<script src="https://cdn.jsdelivr.net/npm/htmx.org@2.0.4/dist/htmx.min.js"></script>
|
||||
{% vite_asset "frontend/src/entries/dashboard.js" %}
|
||||
{% include "js_translations.html" %}
|
||||
{% include "fireworks_detect/includes/browser_processing_config.html" %}
|
||||
</head>
|
||||
<body class="dashboard-page">
|
||||
{% if user.is_authenticated %}<a class="profile-circle" href="{% url "user_settings" %}" aria-label="{% blocktrans with username=user.get_username %}Open settings for {{ username }}{% endblocktrans %}" title="{% trans "Settings" %}">{{ user.get_username|first|upper }}</a>{% endif %}
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
<script src="https://cdn.jsdelivr.net/npm/htmx.org@2.0.4/dist/htmx.min.js"></script>
|
||||
{% vite_asset "frontend/src/entries/fireworks-home.js" %}
|
||||
{% include "js_translations.html" %}
|
||||
{% include "fireworks_detect/includes/browser_processing_config.html" %}
|
||||
</head>
|
||||
|
||||
<body>
|
||||
@@ -30,4 +31,4 @@
|
||||
</main>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
</html>
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
{% load i18n %}
|
||||
<form class="upload-form" action="{% url "fireworks_detect:classify_upload" %}" method="post" enctype="multipart/form-data" hx-post="{% url "fireworks_detect:classify_upload" %}" hx-target="#classification-results" hx-swap="innerHTML" hx-encoding="multipart/form-data" hx-indicator="#upload-indicator">
|
||||
<form class="upload-form audio-upload--idle" action="{% url "fireworks_detect:classify_upload" %}" method="post" enctype="multipart/form-data" hx-post="{% url "fireworks_detect:classify_upload" %}" hx-target="#classification-results" hx-swap="innerHTML" hx-encoding="multipart/form-data" hx-indicator="#upload-indicator" data-scroll-to-results="true">
|
||||
{% csrf_token %}
|
||||
<label class="drop-zone" id="drop-zone">
|
||||
<input class="file-input" id="audio-file" type="file" name="audio" accept="audio/*,.wav,.mp3,.m4a,.ogg,.flac,.aac,.opus" required>
|
||||
<span>
|
||||
<span class="upload-icon">↑</span>
|
||||
<span class="upload-icon" id="upload-progress-circle"><span class="upload-icon__content" id="upload-icon-content" aria-hidden="true">↑</span></span>
|
||||
<strong>{% trans "Drop an audio file here" %}</strong>
|
||||
<span>{% trans "or click to browse your device" %}</span>
|
||||
</span>
|
||||
@@ -13,6 +13,7 @@
|
||||
<div class="form-footer">
|
||||
<p class="file-note">{% trans "WAV, MP3, M4A, OGG and more · 25 MiB maximum" %}</p>
|
||||
<button type="submit"><span class="spinner htmx-indicator" id="upload-indicator" aria-hidden="true"></span>{% trans "Analyse audio" %}</button>
|
||||
<button class="upload-cancel" type="button" hidden>{% trans "Cancel upload" %}</button>
|
||||
</div>
|
||||
<section id="classification-results" aria-live="polite" aria-label="{% trans "Classification results" %}"></section>
|
||||
</form>
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{{ browser_processing_config|json_script:"boomcast-browser-config" }}
|
||||
@@ -0,0 +1,101 @@
|
||||
import json
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core import signing
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.core.files.storage import FileSystemStorage
|
||||
from django.test import TestCase, override_settings
|
||||
from django.urls import reverse
|
||||
|
||||
from .browser_policy import advisory_browser_metadata
|
||||
from .models import BrowserProcessingSettings, ClassificationRule, PendingEvent
|
||||
|
||||
|
||||
class BrowserProcessingBetaTests(TestCase):
|
||||
def test_setting_defaults_off_and_admin_can_toggle_it(self):
|
||||
self.assertFalse(BrowserProcessingSettings.enabled())
|
||||
settings = BrowserProcessingSettings.objects.get(pk=1)
|
||||
self.assertFalse(settings.browser_processing_beta_enabled)
|
||||
staff = get_user_model().objects.create_superuser("beta-admin", "beta@example.com", "password")
|
||||
self.client.force_login(staff)
|
||||
changelist = self.client.get(reverse("admin:fireworks_detect_browserprocessingsettings_changelist"))
|
||||
self.assertContains(changelist, "Disabled — normal submission")
|
||||
url = reverse("admin:fireworks_detect_browserprocessingsettings_change", args=[settings.pk])
|
||||
self.assertContains(self.client.get(url), "Browser-side media processing (Beta)")
|
||||
self.assertEqual(self.client.post(url, {"browser_processing_beta_enabled": "on", "_save": "Save"}).status_code, 302)
|
||||
self.assertTrue(BrowserProcessingSettings.enabled())
|
||||
self.assertEqual(self.client.post(url, {"_save": "Save"}).status_code, 302)
|
||||
self.assertFalse(BrowserProcessingSettings.enabled())
|
||||
self.assertEqual(BrowserProcessingSettings.objects.count(), 1)
|
||||
|
||||
def test_policy_and_page_config_are_empty_while_disabled(self):
|
||||
ClassificationRule.objects.create(value="Fireworks", rule_type=ClassificationRule.RuleType.WHITELIST)
|
||||
ClassificationRule.objects.create(value="Speech", rule_type=ClassificationRule.RuleType.BLACKLIST)
|
||||
response = self.client.get(reverse("fireworks_detect:browser_policy"))
|
||||
self.assertEqual(response.json(), {
|
||||
"version": 1, "browser_processing_beta_enabled": False,
|
||||
"whitelist": [], "privacy_sensitive_speech": [], "thresholds": {},
|
||||
})
|
||||
self.assertIn("no-cache", response["Cache-Control"])
|
||||
for url in (reverse("hello"), reverse("fireworks_detect:fireworks")):
|
||||
page = self.client.get(url)
|
||||
self.assertContains(page, '"browserProcessingBeta": false')
|
||||
self.assertContains(page, 'id="boomcast-browser-config"')
|
||||
|
||||
def test_enabled_policy_reuses_current_rules_and_exposes_no_secrets(self):
|
||||
BrowserProcessingSettings.objects.update(browser_processing_beta_enabled=True)
|
||||
ClassificationRule.objects.create(value="Fireworks", rule_type=ClassificationRule.RuleType.WHITELIST)
|
||||
ClassificationRule.objects.create(value="Disabled sound", rule_type=ClassificationRule.RuleType.WHITELIST, enabled=False)
|
||||
ClassificationRule.objects.create(value="Speech", rule_type=ClassificationRule.RuleType.BLACKLIST)
|
||||
ClassificationRule.objects.create(value="Music", rule_type=ClassificationRule.RuleType.BLACKLIST)
|
||||
policy = self.client.get(reverse("fireworks_detect:browser_policy")).json()
|
||||
self.assertTrue(policy["browser_processing_beta_enabled"])
|
||||
self.assertEqual(policy["whitelist"], ["Fireworks"])
|
||||
self.assertEqual(policy["privacy_sensitive_speech"], ["Speech"])
|
||||
self.assertEqual(policy["thresholds"], {})
|
||||
self.assertNotIn("token", json.dumps(policy).lower())
|
||||
BrowserProcessingSettings.objects.update(browser_processing_beta_enabled=False)
|
||||
self.assertFalse(self.client.get(reverse("fireworks_detect:browser_policy")).json()["browser_processing_beta_enabled"])
|
||||
|
||||
@patch("fireworks_detect.views.urlopen")
|
||||
def test_old_upload_and_advisory_metadata_keep_server_classifier_authoritative(self, urlopen):
|
||||
reply = MagicMock()
|
||||
reply.read.return_value = b'{"predictions": [{"class": "Speech", "score": 0.9}]}'
|
||||
urlopen.return_value.__enter__.return_value = reply
|
||||
url = reverse("fireworks_detect:classify_upload")
|
||||
audio = lambda: SimpleUploadedFile("sample.wav", b"RIFF0000WAVEdata", content_type="audio/wav")
|
||||
old = self.client.post(url, {"audio": audio()})
|
||||
self.assertEqual(old.status_code, 200)
|
||||
self.assertContains(old, "Speech")
|
||||
metadata = json.dumps({
|
||||
"browser_processing_used": True, "browser_processing_version": 1,
|
||||
"browser_gate_result": "whitelist_match", "browser_override": False,
|
||||
"secret": "ignored",
|
||||
})
|
||||
future = self.client.post(url, {"audio": audio(), "browser_processing_metadata": metadata})
|
||||
self.assertEqual(future.status_code, 200)
|
||||
self.assertContains(future, "Speech")
|
||||
self.assertNotContains(future, "whitelist_match")
|
||||
self.assertEqual(urlopen.call_count, 2)
|
||||
self.assertNotIn(b"whitelist_match", urlopen.call_args.args[0].data)
|
||||
self.assertEqual(advisory_browser_metadata({"browser_processing_metadata": metadata}), {
|
||||
"browser_processing_used": True, "browser_processing_version": 1,
|
||||
"browser_gate_result": "whitelist_match", "browser_override": False,
|
||||
})
|
||||
self.assertEqual(advisory_browser_metadata({"browser_processing_metadata": '{"browser_gate_result": []}'}), {})
|
||||
|
||||
def test_pending_submission_accepts_metadata_without_storing_it_as_classification(self):
|
||||
payload = {"predictions": [{"class": "Fireworks", "score": 0.8}]}
|
||||
token = signing.dumps(payload, salt="fireworks_detect.classification")
|
||||
with TemporaryDirectory() as media_root, override_settings(MEDIA_ROOT=media_root), patch.object(
|
||||
PendingEvent._meta.get_field("audio_file"), "storage", FileSystemStorage(location=media_root)
|
||||
):
|
||||
response = self.client.post(reverse("fireworks_detect:submit_pending"), {
|
||||
"audio": SimpleUploadedFile("sample.wav", b"RIFF0000WAVEdata", content_type="audio/wav"),
|
||||
"classification_token": token,
|
||||
"browser_processing_metadata": '{"browser_processing_used": true, "browser_gate_result": "whitelist_match"}',
|
||||
})
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(PendingEvent.objects.get().classifier_response, payload)
|
||||
@@ -13,6 +13,7 @@ from .views import (
|
||||
submit_manual_pending_event,
|
||||
submit_pending_event,
|
||||
sensor_heartbeat,
|
||||
browser_processing_policy,
|
||||
)
|
||||
|
||||
app_name = "fireworks_detect"
|
||||
@@ -22,6 +23,7 @@ urlpatterns = [
|
||||
path("map/", fireworks_map_view, name="map"),
|
||||
path("classify/", classify_audio, name="classify"),
|
||||
path("classify/upload/", classify_upload, name="classify_upload"),
|
||||
path("browser-policy/", browser_processing_policy, name="browser_policy"),
|
||||
path("heartbeat/", sensor_heartbeat, name="sensor_heartbeat"),
|
||||
path("submit-pending/", submit_pending_event, name="submit_pending"),
|
||||
path("submit-manual-pending/", submit_manual_pending_event, name="submit_manual_pending"),
|
||||
|
||||
@@ -2,6 +2,7 @@ import math
|
||||
from datetime import timedelta
|
||||
from decimal import Decimal, InvalidOperation
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
@@ -20,6 +21,7 @@ from django.views.decorators.csrf import csrf_exempt
|
||||
from django.views.decorators.http import require_http_methods
|
||||
|
||||
from .h3_utils import cell_boundary, cell_center, cell_from_coordinates, h3_resolution
|
||||
from .browser_policy import advisory_browser_metadata, browser_config, browser_policy
|
||||
from .models import ClassificationRule, Event, PendingEvent, Sensor
|
||||
|
||||
|
||||
@@ -35,11 +37,18 @@ SPEECH_TERMS = (
|
||||
"speaking", "speech synthesizer",
|
||||
)
|
||||
AUDIO_EXTENSIONS = {".aac", ".aif", ".aiff", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav", ".weba"}
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@require_http_methods(["GET"])
|
||||
@never_cache
|
||||
def browser_processing_policy(request):
|
||||
return JsonResponse(browser_policy())
|
||||
|
||||
|
||||
@never_cache
|
||||
def fireworks_view(request):
|
||||
return render(request, "fireworks_detect/home.html")
|
||||
return render(request, "fireworks_detect/home.html", {"browser_processing_config": browser_config()})
|
||||
|
||||
|
||||
MAP_DATE_RANGES = {
|
||||
@@ -157,6 +166,7 @@ def dashboard_view(request):
|
||||
"h3_resolution": h3_resolution(),
|
||||
"h3_cells": h3_cells,
|
||||
"map_events": map_events,
|
||||
"browser_processing_config": browser_config(),
|
||||
})
|
||||
|
||||
|
||||
@@ -248,6 +258,10 @@ def classify_upload(request):
|
||||
if uploaded_file.size > MAX_AUDIO_SIZE:
|
||||
return _render_error(request, _("This demo accepts audio files up to 25 MiB."))
|
||||
|
||||
metadata = advisory_browser_metadata(request.POST)
|
||||
if metadata:
|
||||
logger.info("Browser beta diagnostics (advisory only): %s", metadata)
|
||||
|
||||
body, content_type = _encode_audio_multipart(uploaded_file)
|
||||
classifier_request = Request(
|
||||
CLASSIFIER_URL,
|
||||
@@ -279,6 +293,9 @@ def classify_upload(request):
|
||||
@require_http_methods(["POST"])
|
||||
@never_cache
|
||||
def submit_pending_event(request):
|
||||
metadata = advisory_browser_metadata(request.POST)
|
||||
if metadata:
|
||||
logger.info("Browser beta submission diagnostics (advisory only): %s", metadata)
|
||||
uploaded_file = request.FILES.get("audio")
|
||||
token = request.POST.get("classification_token")
|
||||
if not uploaded_file:
|
||||
|
||||
Reference in New Issue
Block a user