505 lines
21 KiB
Python
505 lines
21 KiB
Python
import math
|
|
import secrets
|
|
import uuid
|
|
|
|
from django.conf import settings
|
|
from django.contrib.auth.hashers import check_password, make_password
|
|
from django.core.exceptions import ValidationError
|
|
from django.core.files.storage import default_storage
|
|
from django.db import models, transaction
|
|
from django.utils import timezone
|
|
from django.utils.translation import gettext_lazy as _
|
|
|
|
|
|
class Sensor(models.Model):
|
|
"""A physical recorder authorised to submit audio to the device API."""
|
|
|
|
name = models.CharField(max_length=100, unique=True, help_text=_("A recognisable device name, e.g. Garden ESP32."))
|
|
owner = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True, on_delete=models.SET_NULL, related_name="sensors")
|
|
is_active = models.BooleanField(default=True)
|
|
archived_at = models.DateTimeField(null=True, blank=True, editable=False)
|
|
token_prefix = models.CharField(max_length=15, blank=True, editable=False)
|
|
token_hash = models.CharField(max_length=256, blank=True, editable=False)
|
|
last_heartbeat = models.DateTimeField(null=True, blank=True, editable=False)
|
|
last_submission = models.DateTimeField(null=True, blank=True, editable=False)
|
|
created_at = models.DateTimeField(auto_now_add=True)
|
|
|
|
class Meta:
|
|
ordering = ["name"]
|
|
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
def issue_token(self):
|
|
"""Return a secret once; only a salted password hash is persisted."""
|
|
token = f"ss_{secrets.token_urlsafe(32)}"
|
|
self.token_prefix = token[:15]
|
|
self.token_hash = make_password(token)
|
|
self.save(update_fields=["token_prefix", "token_hash"])
|
|
return token
|
|
|
|
def token_matches(self, token):
|
|
return bool(self.token_hash) and check_password(token, self.token_hash)
|
|
|
|
def heartbeat_status(self):
|
|
if not self.last_heartbeat:
|
|
return "never", _("Never seen")
|
|
age = (timezone.now() - self.last_heartbeat).total_seconds()
|
|
if age <= 300:
|
|
return "online", _("Online")
|
|
if age <= 900:
|
|
return "delayed", _("Delayed")
|
|
return "offline", _("Offline")
|
|
|
|
def archive(self):
|
|
self.archived_at = timezone.now()
|
|
self.is_active = False
|
|
self.token_hash = ""
|
|
self.token_prefix = ""
|
|
self.save(update_fields=["archived_at", "is_active", "token_hash", "token_prefix"])
|
|
|
|
def record_heartbeat(self, submission=False):
|
|
now = timezone.now()
|
|
self.last_heartbeat = now
|
|
fields = ["last_heartbeat"]
|
|
if submission:
|
|
self.last_submission = now
|
|
fields.append("last_submission")
|
|
self.save(update_fields=fields)
|
|
|
|
|
|
class ClassificationRule(models.Model):
|
|
"""A configurable inclusion or exclusion rule for a classification label."""
|
|
|
|
class RuleType(models.TextChoices):
|
|
WHITELIST = "whitelist", _("Whitelist")
|
|
BLACKLIST = "blacklist", _("Blacklist")
|
|
|
|
value = models.CharField(max_length=255)
|
|
rule_type = models.CharField(max_length=10, choices=RuleType.choices)
|
|
enabled = models.BooleanField(default=True)
|
|
|
|
class Meta:
|
|
ordering = ["value"]
|
|
constraints = [
|
|
models.UniqueConstraint(fields=["value", "rule_type"], name="unique_classification_rule"),
|
|
]
|
|
|
|
def __str__(self):
|
|
return f"{self.value} ({self.get_rule_type_display()})"
|
|
|
|
|
|
class BrowserProcessingSettings(models.Model):
|
|
"""Single site-wide switch for experimental browser processing."""
|
|
|
|
browser_processing_beta_enabled = models.BooleanField(default=False)
|
|
|
|
class Meta:
|
|
verbose_name = _("Browser-side media processing (Beta)")
|
|
verbose_name_plural = _("Browser-side media processing (Beta)")
|
|
|
|
def save(self, *args, **kwargs):
|
|
self.pk = 1
|
|
return super().save(*args, **kwargs)
|
|
|
|
@classmethod
|
|
def enabled(cls):
|
|
return bool(cls.objects.filter(pk=1).values_list("browser_processing_beta_enabled", flat=True).first())
|
|
|
|
def __str__(self):
|
|
return str(_("Browser-side media processing (Beta)"))
|
|
|
|
|
|
class ClassifiedAudio(models.Model):
|
|
"""Shared fields for a classifier response and its original audio upload."""
|
|
|
|
class SourceType(models.TextChoices):
|
|
SENSOR = "sensor", _("Sensor")
|
|
MANUAL = "manual", _("Manual report")
|
|
|
|
class DurationKind(models.TextChoices):
|
|
ESTIMATED = "estimated", _("Estimated")
|
|
MEASURED = "measured", _("Measured")
|
|
|
|
audio_file = models.FileField(upload_to="fireworks_audio/%Y/%m/%d", blank=True)
|
|
sensor = models.ForeignKey(Sensor, null=True, blank=True, on_delete=models.SET_NULL, related_name="%(class)s_records")
|
|
original_filename = models.CharField(max_length=255)
|
|
classifier_response = models.JSONField()
|
|
source_type = models.CharField(max_length=10, choices=SourceType.choices, default=SourceType.SENSOR)
|
|
occurrences = models.PositiveIntegerField(default=1)
|
|
observation_duration_minutes = models.PositiveSmallIntegerField(null=True, blank=True)
|
|
observation_duration_kind = models.CharField(max_length=10, choices=DurationKind.choices, blank=True)
|
|
occurrence_time_approximate = models.BooleanField(default=False)
|
|
location = models.CharField(max_length=255, blank=True)
|
|
latitude = models.DecimalField(max_digits=9, decimal_places=6, null=True, blank=True)
|
|
longitude = models.DecimalField(max_digits=9, decimal_places=6, null=True, blank=True)
|
|
h3_cell = models.CharField(max_length=20, blank=True, db_index=True)
|
|
visually_confirmed = models.BooleanField(default=False)
|
|
direction_degrees = models.FloatField(null=True, blank=True)
|
|
direction_accuracy_degrees = models.FloatField(null=True, blank=True)
|
|
direction_source = models.CharField(max_length=32, blank=True)
|
|
created_at = models.DateTimeField(auto_now_add=True)
|
|
captured_at = models.DateTimeField(null=True, blank=True)
|
|
firmware_version = models.CharField(max_length=64, blank=True)
|
|
device_metadata = models.JSONField(default=dict, blank=True)
|
|
|
|
class Meta:
|
|
abstract = True
|
|
ordering = ["-created_at"]
|
|
|
|
def __str__(self):
|
|
return self.original_filename
|
|
|
|
def clean(self):
|
|
super().clean()
|
|
errors = {}
|
|
if self.direction_degrees is not None and (
|
|
not math.isfinite(self.direction_degrees) or not 0 <= self.direction_degrees < 360
|
|
):
|
|
errors["direction_degrees"] = _("Direction must be a finite value from 0 up to, but not including, 360 degrees.")
|
|
if self.direction_accuracy_degrees is not None and (
|
|
not math.isfinite(self.direction_accuracy_degrees) or self.direction_accuracy_degrees < 0
|
|
):
|
|
errors["direction_accuracy_degrees"] = _("Compass accuracy must be a finite, non-negative value.")
|
|
if self.observation_duration_minutes is not None and not 1 <= self.observation_duration_minutes <= 1440:
|
|
errors["observation_duration_minutes"] = _("Observation duration must be between 1 and 1,440 minutes.")
|
|
if bool(self.observation_duration_minutes) != bool(self.observation_duration_kind):
|
|
errors["observation_duration_kind"] = _("A duration needs an estimated or measured source.")
|
|
if errors:
|
|
raise ValidationError(errors)
|
|
|
|
|
|
@property
|
|
def direction_cardinal(self):
|
|
if self.direction_degrees is None:
|
|
return None
|
|
directions = ("N", "NE", "E", "SE", "S", "SW", "W", "NW")
|
|
return directions[int((self.direction_degrees + 22.5) % 360 // 45)]
|
|
|
|
|
|
@property
|
|
def direction_display(self):
|
|
if self.direction_degrees is None:
|
|
return "—"
|
|
return f"{self.direction_degrees:g}° ({self.direction_cardinal})"
|
|
|
|
@property
|
|
def is_burst(self):
|
|
from .bursts import BURST_MIN_BANGS
|
|
return self.occurrences >= BURST_MIN_BANGS
|
|
|
|
@property
|
|
def event_kind(self):
|
|
return "burst" if self.is_burst else "ordinary"
|
|
|
|
@property
|
|
def bangs_per_15_minutes(self):
|
|
from .bursts import burst_rate
|
|
if not self.observation_duration_kind:
|
|
return None
|
|
return burst_rate(self.occurrences, self.observation_duration_minutes)
|
|
|
|
@property
|
|
def burst_intensity(self):
|
|
from .bursts import burst_intensity
|
|
if not self.observation_duration_kind:
|
|
return None
|
|
return burst_intensity(self.occurrences, self.observation_duration_minutes)
|
|
|
|
@property
|
|
def burst_intensity_display(self):
|
|
return {"light": _("Light"), "moderate": _("Moderate"), "intense": _("Intense")}.get(self.burst_intensity)
|
|
|
|
|
|
@property
|
|
def predictions(self):
|
|
predictions = self.classifier_response.get("predictions", [])
|
|
return predictions if isinstance(predictions, list) else []
|
|
|
|
@property
|
|
def primary_prediction(self):
|
|
return self.predictions[0] if self.predictions else {}
|
|
|
|
@property
|
|
def primary_label(self):
|
|
return self.primary_prediction.get("class", _("No prediction"))
|
|
|
|
@property
|
|
def primary_score(self):
|
|
try:
|
|
return f"{float(self.primary_prediction.get('score', 0)) * 100:.1f}%"
|
|
except (TypeError, ValueError):
|
|
return "—"
|
|
|
|
@property
|
|
def duration(self):
|
|
return self.classifier_response.get("duration_seconds", "—")
|
|
|
|
|
|
def _preview_items(self, key):
|
|
preview = []
|
|
for item in self.classifier_response.get(key, []):
|
|
label = str(item.get("class", "Unknown"))
|
|
lowered = label.lower()
|
|
kind = "speech" if any(term in lowered for term in ("speech", "conversation", "narration", "monologue", "whisper")) else "firework" if any(term in lowered for term in ("firework", "firecracker", "explosion", "bang", "boom", "hiss", "crackle")) else "default"
|
|
preview.append({**item, "label": label, "kind": kind, "score_display": f"{float(item.get('score', 0)) * 100:.1f}%"})
|
|
return preview
|
|
|
|
@property
|
|
def admin_predictions(self):
|
|
return self._preview_items("predictions")
|
|
|
|
@property
|
|
def admin_peaks(self):
|
|
return self._preview_items("peak_predictions")
|
|
|
|
@property
|
|
def admin_audio_route(self):
|
|
return "fireworks_detect:pending_audio" if self._meta.model_name == "pendingevent" else "fireworks_detect:event_audio"
|
|
|
|
class PendingEvent(ClassifiedAudio):
|
|
"""A submitted classification awaiting a human review in Django admin."""
|
|
|
|
class Meta(ClassifiedAudio.Meta):
|
|
verbose_name = _("Pending event")
|
|
verbose_name_plural = _("Pending")
|
|
|
|
@transaction.atomic
|
|
def approve(self, actor=None):
|
|
event = Event.objects.create(
|
|
submitted_at=self.created_at,
|
|
audio_file=self.audio_file.name,
|
|
sensor=self.sensor,
|
|
captured_at=self.captured_at,
|
|
firmware_version=self.firmware_version,
|
|
device_metadata=self.device_metadata,
|
|
original_filename=self.original_filename,
|
|
classifier_response=self.classifier_response,
|
|
source_type=self.source_type,
|
|
occurrences=self.occurrences,
|
|
observation_duration_minutes=self.observation_duration_minutes,
|
|
observation_duration_kind=self.observation_duration_kind,
|
|
occurrence_time_approximate=self.occurrence_time_approximate,
|
|
location=self.location,
|
|
latitude=self.latitude,
|
|
longitude=self.longitude,
|
|
h3_cell=self.h3_cell,
|
|
visually_confirmed=self.visually_confirmed,
|
|
direction_degrees=self.direction_degrees,
|
|
direction_accuracy_degrees=self.direction_accuracy_degrees,
|
|
direction_source=self.direction_source,
|
|
)
|
|
from .field_notes import associate
|
|
if hasattr(self, "burst_session"):
|
|
session = self.burst_session
|
|
session.pending_event = None
|
|
session.event = event
|
|
session.save(update_fields=["pending_event", "event"])
|
|
for context in self.note_contexts.select_for_update():
|
|
associate(context, event=event, actor=actor, reason="approved")
|
|
self.delete()
|
|
return event
|
|
|
|
def deny(self):
|
|
file_name = self.audio_file.name
|
|
if hasattr(self, "burst_session"):
|
|
for capture in self.burst_session.captures.all():
|
|
if capture.audio_file.name and capture.audio_file.name != file_name:
|
|
default_storage.delete(capture.audio_file.name)
|
|
self.burst_session.captures.all().delete()
|
|
self.delete()
|
|
if file_name:
|
|
default_storage.delete(file_name)
|
|
|
|
|
|
class Event(ClassifiedAudio):
|
|
"""A reviewed classification that is safe to appear in the event feed."""
|
|
|
|
approved_at = models.DateTimeField(auto_now_add=True)
|
|
submitted_at = models.DateTimeField(null=True, blank=True)
|
|
|
|
class Meta(ClassifiedAudio.Meta):
|
|
verbose_name = _("Event")
|
|
verbose_name_plural = _("Events")
|
|
|
|
|
|
class BurstSession(models.Model):
|
|
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
|
owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="burst_sessions")
|
|
pending_event = models.OneToOneField(PendingEvent, null=True, blank=True, on_delete=models.SET_NULL, related_name="burst_session")
|
|
event = models.OneToOneField(Event, null=True, blank=True, on_delete=models.SET_NULL, related_name="burst_session")
|
|
started_at = models.DateTimeField(default=timezone.now)
|
|
ended_at = models.DateTimeField(null=True, blank=True)
|
|
maximum_minutes = models.PositiveSmallIntegerField(default=15)
|
|
status = models.CharField(max_length=24, default="listening")
|
|
last_seen_at = models.DateTimeField(null=True, blank=True)
|
|
last_audio_at = models.DateTimeField(null=True, blank=True)
|
|
last_audio_frame = models.PositiveBigIntegerField(default=0)
|
|
last_sequence = models.PositiveIntegerField(default=0)
|
|
timeout_at = models.DateTimeField(null=True, blank=True)
|
|
diagnostic_events = models.JSONField(default=list)
|
|
|
|
|
|
class BurstCapture(models.Model):
|
|
session = models.ForeignKey(BurstSession, on_delete=models.CASCADE, related_name="captures")
|
|
captured_at = models.DateTimeField()
|
|
duration_seconds = models.FloatField()
|
|
audio_file = models.FileField(upload_to="fireworks_audio/%Y/%m/%d")
|
|
is_primary = models.BooleanField(default=False)
|
|
|
|
class Meta:
|
|
ordering = ["captured_at", "pk"]
|
|
|
|
|
|
class ManualReportRevision(models.Model):
|
|
"""Record edits to a session-owned manual Pending report."""
|
|
|
|
pending_event = models.ForeignKey(PendingEvent, null=True, blank=True, on_delete=models.SET_NULL)
|
|
original_pending_id = models.PositiveBigIntegerField()
|
|
previous_occurrences = models.PositiveIntegerField()
|
|
new_occurrences = models.PositiveIntegerField()
|
|
previous_duration_minutes = models.PositiveSmallIntegerField(null=True, blank=True)
|
|
new_duration_minutes = models.PositiveSmallIntegerField(null=True, blank=True)
|
|
changed_at = models.DateTimeField(auto_now_add=True)
|
|
|
|
|
|
class SensorSubmission(models.Model):
|
|
"""Durable idempotency receipt, retained after moderation."""
|
|
|
|
sensor = models.ForeignKey(Sensor, on_delete=models.CASCADE, related_name="submissions")
|
|
event_id = models.CharField(max_length=100)
|
|
audio_sha256 = models.CharField(max_length=64)
|
|
submission_id = models.UUIDField(default=uuid.uuid4, unique=True, editable=False)
|
|
pending_event = models.ForeignKey(PendingEvent, null=True, blank=True, on_delete=models.SET_NULL)
|
|
created_at = models.DateTimeField(auto_now_add=True)
|
|
|
|
class Meta:
|
|
constraints = [models.UniqueConstraint(fields=["sensor", "event_id"], name="unique_sensor_event_id")]
|
|
|
|
|
|
class FieldNoteContext(models.Model):
|
|
"""An author's note association, including drafts before a report exists.
|
|
|
|
This stores no detection/incident data; Event/PendingEvent remain canonical.
|
|
"""
|
|
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
|
author = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT)
|
|
event = models.ForeignKey(Event, null=True, blank=True, on_delete=models.SET_NULL, related_name="note_contexts")
|
|
pending_event = models.ForeignKey(PendingEvent, null=True, blank=True, on_delete=models.SET_NULL, related_name="note_contexts")
|
|
created_at = models.DateTimeField(auto_now_add=True)
|
|
|
|
class Meta:
|
|
constraints = [models.CheckConstraint(condition=~(models.Q(event__isnull=False) & models.Q(pending_event__isnull=False)), name="note_context_one_target")]
|
|
|
|
|
|
class FieldNoteQuerySet(models.QuerySet):
|
|
def visible_to(self, user):
|
|
if not user.is_authenticated or not user.is_active:
|
|
return self.none()
|
|
if user.is_active and user.has_perm("fireworks_detect.view_all_fieldnotes"):
|
|
return self
|
|
return self.filter(author=user)
|
|
|
|
def reportable_to(self, user):
|
|
if not user.is_authenticated or not user.is_active:
|
|
return self.none()
|
|
notes = self.filter(visibility="reportable")
|
|
if user.is_active and user.has_perm("fireworks_detect.export_reportable_fieldnotes"):
|
|
return notes
|
|
return notes.filter(author=user)
|
|
|
|
|
|
class FieldNote(models.Model):
|
|
class Category(models.TextChoices):
|
|
GENERAL = "general", _("General")
|
|
WITNESS = "witness", _("Witness")
|
|
PEOPLE = "people_vehicles", _("People / vehicles")
|
|
CCTV = "cctv", _("CCTV")
|
|
LOCATION = "location_direction", _("Location / direction")
|
|
FOLLOW_UP = "follow_up", _("Follow-up")
|
|
OTHER = "other", _("Other")
|
|
|
|
class Source(models.TextChoices):
|
|
FIRST_HAND = "first_hand", _("First-hand")
|
|
SECOND_HAND = "second_hand", _("Second-hand")
|
|
UNKNOWN = "unknown", _("Other / unknown")
|
|
|
|
class Verification(models.TextChoices):
|
|
UNVERIFIED = "unverified", _("Unverified")
|
|
CORROBORATED = "corroborated", _("Corroborated")
|
|
DISPUTED = "disputed", _("Disputed")
|
|
|
|
class Visibility(models.TextChoices):
|
|
PRIVATE = "private", _("Private")
|
|
REPORTABLE = "reportable", _("Reportable")
|
|
|
|
author = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT, related_name="field_notes")
|
|
context = models.ForeignKey(FieldNoteContext, on_delete=models.PROTECT, related_name="notes")
|
|
content = models.TextField(max_length=10000)
|
|
category = models.CharField(max_length=32, choices=Category.choices, default=Category.GENERAL)
|
|
source_type = models.CharField(max_length=24, choices=Source.choices, default=Source.UNKNOWN)
|
|
verification_status = models.CharField(max_length=24, choices=Verification.choices, default=Verification.UNVERIFIED)
|
|
visibility = models.CharField(max_length=16, choices=Visibility.choices, default=Visibility.PRIVATE)
|
|
observed_at = models.DateTimeField(null=True, blank=True)
|
|
created_at = models.DateTimeField(auto_now_add=True)
|
|
updated_at = models.DateTimeField(auto_now=True)
|
|
objects = FieldNoteQuerySet.as_manager()
|
|
|
|
class Meta:
|
|
ordering = ["created_at", "pk"]
|
|
indexes = [models.Index(fields=["author", "created_at"], name="fieldnote_author_time")]
|
|
permissions = [
|
|
("view_all_fieldnotes", "Read all private field notes"),
|
|
("export_reportable_fieldnotes", "Export reportable field notes from all authors"),
|
|
]
|
|
constraints = [
|
|
models.CheckConstraint(condition=~models.Q(content=""), name="fieldnote_nonempty"),
|
|
models.CheckConstraint(condition=models.Q(visibility__in=["private", "reportable"]), name="fieldnote_visibility"),
|
|
models.CheckConstraint(condition=models.Q(source_type__in=["first_hand", "second_hand", "unknown"]), name="fieldnote_source"),
|
|
models.CheckConstraint(condition=models.Q(verification_status__in=["unverified", "corroborated", "disputed"]), name="fieldnote_verification"),
|
|
]
|
|
|
|
def clean(self):
|
|
super().clean()
|
|
if not self.content.strip():
|
|
raise ValidationError({"content": _("Enter a note.")})
|
|
if self.context_id and self.author_id != self.context.author_id:
|
|
raise ValidationError(_("The note and its context must have the same author."))
|
|
|
|
def save(self, *args, **kwargs):
|
|
if not self._state.adding:
|
|
raise ValidationError(_("Field notes are append-only. Add a follow-up to correct an earlier note."))
|
|
self.full_clean()
|
|
return super().save(*args, **kwargs)
|
|
|
|
|
|
class FieldNoteAssociation(models.Model):
|
|
"""Immutable provenance retained even when the original event is deleted."""
|
|
context = models.ForeignKey(FieldNoteContext, on_delete=models.PROTECT, related_name="associations")
|
|
actor = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL)
|
|
previous = models.JSONField(default=dict)
|
|
target = models.JSONField(default=dict)
|
|
reason = models.CharField(max_length=32)
|
|
created_at = models.DateTimeField(auto_now_add=True)
|
|
|
|
class Meta:
|
|
ordering = ["created_at", "pk"]
|
|
|
|
def save(self, *args, **kwargs):
|
|
if not self._state.adding:
|
|
raise ValidationError("Association history is append-only.")
|
|
return super().save(*args, **kwargs)
|
|
|
|
|
|
class McpAccessToken(models.Model):
|
|
"""Revocable credential for read-only MCP access; never stores the raw token."""
|
|
|
|
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="boomcast_mcp_tokens")
|
|
name = models.CharField(max_length=100)
|
|
token_hash = models.CharField(max_length=64, unique=True)
|
|
created_at = models.DateTimeField(auto_now_add=True)
|
|
revoked_at = models.DateTimeField(null=True, blank=True)
|
|
|
|
def __str__(self):
|
|
return f"{self.user}: {self.name}"
|