diff --git a/main/deploy/systemd/tasknest-inventory-scan-worker.service b/main/deploy/systemd/tasknest-inventory-scan-worker.service new file mode 100644 index 0000000..e1ce002 --- /dev/null +++ b/main/deploy/systemd/tasknest-inventory-scan-worker.service @@ -0,0 +1,14 @@ +[Unit] +Description=Process Tasknest kitchen inventory scan jobs +After=network.target + +[Service] +Type=simple +User=jorenmartijn +WorkingDirectory=/home/jorenmartijn/django/main +ExecStart=/home/jorenmartijn/django/tasksnest-host/bin/python manage.py process_inventory_scans --poll-seconds 2 +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/main/food/management/commands/process_inventory_scans.py b/main/food/management/commands/process_inventory_scans.py new file mode 100644 index 0000000..bfc8bdd --- /dev/null +++ b/main/food/management/commands/process_inventory_scans.py @@ -0,0 +1,27 @@ +import time + +from django.core.management.base import BaseCommand + +from food.services.inventory_scanner import process_next_scan_item + + +class Command(BaseCommand): + help = 'Process due persisted kitchen inventory scan enrichment jobs.' + + def add_arguments(self, parser): + parser.add_argument('--once', action='store_true', help='Process at most one job and exit.') + parser.add_argument('--poll-seconds', type=float, default=2.0) + parser.add_argument('--max-items', type=int, default=0, help='Stop after this many jobs; 0 runs continuously.') + + def handle(self, *args, **options): + processed = 0 + while True: + did_work = process_next_scan_item() + if did_work: + processed += 1 + if options['once'] or (options['max_items'] and processed >= options['max_items']): + return + elif options['once']: + return + else: + time.sleep(max(0.25, options['poll_seconds'])) diff --git a/main/food/migrations/0015_openfoodfactsrequest_inventoryreviewpreference_and_more.py b/main/food/migrations/0015_openfoodfactsrequest_inventoryreviewpreference_and_more.py new file mode 100644 index 0000000..e0b428b --- /dev/null +++ b/main/food/migrations/0015_openfoodfactsrequest_inventoryreviewpreference_and_more.py @@ -0,0 +1,127 @@ +# Generated by Django 6.1.1 on 2026-10-25 23:32 + +import django.core.validators +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('food', '0014_alter_inventoryconsumption_unit'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='OpenFoodFactsRequest', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('requested_at', models.DateTimeField(db_index=True)), + ('endpoint', models.CharField(blank=True, max_length=24)), + ], + ), + migrations.CreateModel( + name='OpenFoodFactsRateLimit', + fields=[ + ('id', models.PositiveSmallIntegerField(default=1, editable=False, primary_key=True, serialize=False)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='InventoryReviewPreference', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('mode', models.CharField(choices=[('individual', 'Individual'), ('bulk', 'Bulk')], default='individual', max_length=12)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='inventory_review_preference', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='InventoryScanRun', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('api_id', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('status', models.CharField(choices=[('active', 'Active'), ('processing', 'Processing'), ('complete', 'Complete'), ('complete_with_review', 'Complete with review'), ('cancelled', 'Cancelled')], db_index=True, default='active', max_length=24)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('submitted_at', models.DateTimeField(blank=True, null=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('notification_attempted_at', models.DateTimeField(blank=True, null=True)), + ('notification_sent_at', models.DateTimeField(blank=True, null=True)), + ('notification_error', models.CharField(blank=True, max_length=255)), + ('selected_location', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='active_scan_runs', to='food.inventorylocation')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='inventory_scan_runs', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='InventoryScanBatch', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('sequence', models.PositiveIntegerField()), + ('status', models.CharField(choices=[('draft', 'Draft'), ('queued', 'Queued'), ('complete', 'Complete')], db_index=True, default='draft', max_length=12)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('submitted_at', models.DateTimeField(blank=True, null=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('location', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='scan_batches', to='food.inventorylocation')), + ('run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='batches', to='food.inventoryscanrun')), + ], + options={ + 'ordering': ('sequence',), + }, + ), + migrations.CreateModel( + name='InventoryScanItem', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('barcode', models.CharField(db_index=True, max_length=64)), + ('status', models.CharField(choices=[('active', 'Active scan'), ('confirmed', 'Confirmed'), ('queued', 'Queued'), ('processing', 'Processing'), ('needs_review', 'Needs review'), ('complete', 'Complete'), ('deferred', 'Deferred'), ('failed', 'Failed')], db_index=True, default='active', max_length=16)), + ('package_count', models.PositiveIntegerField(default=1, validators=[django.core.validators.MinValueValidator(1)])), + ('package_amount', models.DecimalField(blank=True, decimal_places=3, max_digits=10, null=True, validators=[django.core.validators.MinValueValidator(0.001)])), + ('package_unit', models.CharField(blank=True, max_length=8)), + ('pieces_per_package', models.PositiveIntegerField(blank=True, null=True)), + ('piece_amount', models.DecimalField(blank=True, decimal_places=3, max_digits=10, null=True, validators=[django.core.validators.MinValueValidator(0.001)])), + ('inner_packs_per_package', models.PositiveIntegerField(blank=True, null=True)), + ('pieces_per_inner_pack', models.PositiveIntegerField(blank=True, null=True)), + ('inner_pack_amount', models.DecimalField(blank=True, decimal_places=3, max_digits=10, null=True, validators=[django.core.validators.MinValueValidator(0.001)])), + ('inner_pack_label', models.CharField(blank=True, max_length=40)), + ('piece_label', models.CharField(blank=True, max_length=40)), + ('product_name', models.CharField(blank=True, max_length=255)), + ('brand', models.CharField(blank=True, max_length=255)), + ('category_name', models.CharField(blank=True, max_length=100)), + ('notes', models.TextField(blank=True)), + ('best_before', models.DateField(blank=True, null=True)), + ('opened', models.BooleanField(default=False)), + ('source_data', models.JSONField(blank=True, default=dict)), + ('user_data', models.JSONField(blank=True, default=dict)), + ('attempts', models.PositiveSmallIntegerField(default=0)), + ('next_attempt_at', models.DateTimeField(blank=True, db_index=True, null=True)), + ('lease_until', models.DateTimeField(blank=True, db_index=True, null=True)), + ('last_error', models.CharField(blank=True, max_length=255)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('confirmed_at', models.DateTimeField(blank=True, null=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('batch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='food.inventoryscanbatch')), + ('location', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='scan_items', to='food.inventorylocation')), + ('inventory_item', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='scan_items', to='food.kitcheninventoryitem')), + ('product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='scan_items', to='food.foodproduct')), + ], + options={ + 'ordering': ('created_at', 'pk'), + 'indexes': [models.Index(fields=['status', 'next_attempt_at'], name='food_scan_item_due')], + }, + ), + migrations.AddIndex( + model_name='inventoryscanrun', + index=models.Index(fields=['user', 'status', '-created_at'], name='food_scan_run_user_status'), + ), + migrations.AddConstraint( + model_name='inventoryscanbatch', + constraint=models.UniqueConstraint(fields=('run', 'sequence'), name='food_unique_scan_batch_sequence'), + ), + migrations.AddConstraint( + model_name='inventoryscanbatch', + constraint=models.UniqueConstraint(condition=models.Q(('status', 'draft')), fields=('run',), name='food_one_draft_scan_batch'), + ), + ] diff --git a/main/food/migrations/0016_inventoryscanrun_food_one_active_scan_run.py b/main/food/migrations/0016_inventoryscanrun_food_one_active_scan_run.py new file mode 100644 index 0000000..4c292d5 --- /dev/null +++ b/main/food/migrations/0016_inventoryscanrun_food_one_active_scan_run.py @@ -0,0 +1,19 @@ +# Generated by Django 6.1.1 on 2026-10-25 23:36 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('food', '0015_openfoodfactsrequest_inventoryreviewpreference_and_more'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddConstraint( + model_name='inventoryscanrun', + constraint=models.UniqueConstraint(condition=models.Q(('status', 'active')), fields=('user',), name='food_one_active_scan_run'), + ), + ] diff --git a/main/food/models.py b/main/food/models.py index 3f6df9b..3b2326f 100644 --- a/main/food/models.py +++ b/main/food/models.py @@ -380,3 +380,127 @@ class ApiIdempotencyRecord(models.Model): class Meta: constraints = [models.UniqueConstraint(fields=('token', 'key'), name='food_unique_api_idempotency_key')] + + +class InventoryScanRun(models.Model): + """A resumable scan-in session, including batches submitted while scanning continues.""" + class Status(models.TextChoices): + ACTIVE = 'active', 'Active' + PROCESSING = 'processing', 'Processing' + COMPLETE = 'complete', 'Complete' + COMPLETE_WITH_REVIEW = 'complete_with_review', 'Complete with review' + CANCELLED = 'cancelled', 'Cancelled' + + api_id = models.UUIDField(default=uuid.uuid4, unique=True, editable=False) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='inventory_scan_runs') + status = models.CharField(max_length=24, choices=Status.choices, default=Status.ACTIVE, db_index=True) + selected_location = models.ForeignKey(InventoryLocation, on_delete=models.SET_NULL, null=True, blank=True, + related_name='active_scan_runs') + created_at = models.DateTimeField(auto_now_add=True) + submitted_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + notification_attempted_at = models.DateTimeField(null=True, blank=True) + notification_sent_at = models.DateTimeField(null=True, blank=True) + notification_error = models.CharField(max_length=255, blank=True) + + class Meta: + indexes = [models.Index(fields=('user', 'status', '-created_at'), name='food_scan_run_user_status')] + constraints = [models.UniqueConstraint(fields=('user',), condition=Q(status='active'), + name='food_one_active_scan_run')] + + +class InventoryScanBatch(models.Model): + """A location-specific portion of a run. Save & next closes one and opens another.""" + class Status(models.TextChoices): + DRAFT = 'draft', 'Draft' + QUEUED = 'queued', 'Queued' + COMPLETE = 'complete', 'Complete' + + run = models.ForeignKey(InventoryScanRun, on_delete=models.CASCADE, related_name='batches') + location = models.ForeignKey(InventoryLocation, on_delete=models.PROTECT, related_name='scan_batches') + sequence = models.PositiveIntegerField() + status = models.CharField(max_length=12, choices=Status.choices, default=Status.DRAFT, db_index=True) + created_at = models.DateTimeField(auto_now_add=True) + submitted_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + + class Meta: + ordering = ('sequence',) + constraints = [ + models.UniqueConstraint(fields=('run', 'sequence'), name='food_unique_scan_batch_sequence'), + models.UniqueConstraint(fields=('run',), condition=Q(status='draft'), name='food_one_draft_scan_batch'), + ] + + +class InventoryScanItem(models.Model): + """A persisted scan entry that becomes its own retryable enrichment job on submit.""" + class Status(models.TextChoices): + ACTIVE = 'active', 'Active scan' + CONFIRMED = 'confirmed', 'Confirmed' + QUEUED = 'queued', 'Queued' + PROCESSING = 'processing', 'Processing' + NEEDS_REVIEW = 'needs_review', 'Needs review' + COMPLETE = 'complete', 'Complete' + DEFERRED = 'deferred', 'Deferred' + FAILED = 'failed', 'Failed' + + batch = models.ForeignKey(InventoryScanBatch, on_delete=models.CASCADE, related_name='items') + location = models.ForeignKey(InventoryLocation, on_delete=models.PROTECT, related_name='scan_items') + barcode = models.CharField(max_length=64, db_index=True) + status = models.CharField(max_length=16, choices=Status.choices, default=Status.ACTIVE, db_index=True) + package_count = models.PositiveIntegerField(default=1, validators=[MinValueValidator(1)]) + package_amount = models.DecimalField(max_digits=10, decimal_places=3, null=True, blank=True, validators=POSITIVE) + package_unit = models.CharField(max_length=8, blank=True) + pieces_per_package = models.PositiveIntegerField(null=True, blank=True) + piece_amount = models.DecimalField(max_digits=10, decimal_places=3, null=True, blank=True, validators=POSITIVE) + inner_packs_per_package = models.PositiveIntegerField(null=True, blank=True) + pieces_per_inner_pack = models.PositiveIntegerField(null=True, blank=True) + inner_pack_amount = models.DecimalField(max_digits=10, decimal_places=3, null=True, blank=True, validators=POSITIVE) + inner_pack_label = models.CharField(max_length=40, blank=True) + piece_label = models.CharField(max_length=40, blank=True) + product_name = models.CharField(max_length=255, blank=True) + brand = models.CharField(max_length=255, blank=True) + category_name = models.CharField(max_length=100, blank=True) + notes = models.TextField(blank=True) + best_before = models.DateField(null=True, blank=True) + opened = models.BooleanField(default=False) + source_data = models.JSONField(default=dict, blank=True) + user_data = models.JSONField(default=dict, blank=True) + product = models.ForeignKey(FoodProduct, on_delete=models.SET_NULL, null=True, blank=True, + related_name='scan_items') + inventory_item = models.ForeignKey(KitchenInventoryItem, on_delete=models.SET_NULL, null=True, blank=True, + related_name='scan_items') + attempts = models.PositiveSmallIntegerField(default=0) + next_attempt_at = models.DateTimeField(null=True, blank=True, db_index=True) + lease_until = models.DateTimeField(null=True, blank=True, db_index=True) + last_error = models.CharField(max_length=255, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + confirmed_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + + class Meta: + ordering = ('created_at', 'pk') + indexes = [models.Index(fields=('status', 'next_attempt_at'), name='food_scan_item_due')] + + +class OpenFoodFactsRequest(models.Model): + """A durable timestamp for each actual OFF HTTP request; shared by web and worker processes.""" + requested_at = models.DateTimeField(db_index=True) + endpoint = models.CharField(max_length=24, blank=True) + + +class OpenFoodFactsRateLimit(models.Model): + """A single database lock row coordinates request reservations across processes.""" + id = models.PositiveSmallIntegerField(primary_key=True, default=1, editable=False) + updated_at = models.DateTimeField(auto_now=True) + + +class InventoryReviewPreference(models.Model): + class Mode(models.TextChoices): + INDIVIDUAL = 'individual', 'Individual' + BULK = 'bulk', 'Bulk' + + user = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, + related_name='inventory_review_preference') + mode = models.CharField(max_length=12, choices=Mode.choices, default=Mode.INDIVIDUAL) + updated_at = models.DateTimeField(auto_now=True) diff --git a/main/food/services/inventory_scanner.py b/main/food/services/inventory_scanner.py new file mode 100644 index 0000000..e390776 --- /dev/null +++ b/main/food/services/inventory_scanner.py @@ -0,0 +1,520 @@ +"""Persistent scan-in session and enrichment queue services.""" +from datetime import timedelta +from decimal import Decimal, InvalidOperation +import random + +from django.core.exceptions import ValidationError +from django.db import IntegrityError, connections, transaction +from django.db.models import F, Q +from django.utils import timezone + +from food.models import (FoodProduct, InventoryCategory, InventoryLocation, + InventoryReviewPreference, InventoryScanBatch, + InventoryScanItem, InventoryScanRun, + KitchenInventoryItem) +from food.services.openfoodfacts import (OpenFoodFactsError, + OpenFoodFactsTransientError, + cached_product) + + +class ScanWorkflowError(ValueError): + pass + + +def get_or_create_active_run(user, location_id): + location = InventoryLocation.objects.filter(user=user, pk=location_id).first() + if not location: + raise ScanWorkflowError('Choose one of your existing inventory locations.') + try: + with transaction.atomic(): + run = (InventoryScanRun.objects.select_for_update().filter( + user=user, status=InventoryScanRun.Status.ACTIVE).order_by('-created_at').first()) + if run: + draft = run.batches.filter(status=InventoryScanBatch.Status.DRAFT).first() + if not draft: + raise ScanWorkflowError('This scan session is being saved. Refresh and try again.') + run.selected_location = draft.location + return run + run = InventoryScanRun.objects.create(user=user, selected_location=location) + InventoryScanBatch.objects.create(run=run, location=location, sequence=1) + return run + except IntegrityError: + run = resume_active_run(user) + if run: + return run + raise + + +def resume_active_run(user): + return (InventoryScanRun.objects.filter(user=user, status=InventoryScanRun.Status.ACTIVE) + .prefetch_related('batches__items').order_by('-created_at').first()) + + +def create_active_scan(run, user, barcode): + if not barcode.isascii() or not barcode.isdigit() or not 6 <= len(barcode) <= 64: + raise ScanWorkflowError('Enter a valid numeric barcode.') + with transaction.atomic(): + locked = InventoryScanRun.objects.select_for_update().filter( + pk=run.pk, user=user, status=InventoryScanRun.Status.ACTIVE).first() + if not locked: + raise ScanWorkflowError('This scan session is no longer active.') + batch = locked.batches.select_for_update().filter(status=InventoryScanBatch.Status.DRAFT).first() + if not batch: + raise ScanWorkflowError('Choose a location before scanning.') + if batch.items.filter(status=InventoryScanItem.Status.ACTIVE).exists(): + raise ScanWorkflowError('Confirm or remove the active scan before scanning another item.') + product = FoodProduct.objects.filter(owner=user, barcode=barcode).first() + values = {} + if product: + values = { + 'product_name': product.product_name, 'brand': product.brand, + 'package_amount': product.package_amount, 'package_unit': product.package_unit, + 'source_data': {'cached': True, 'source': product.source, + 'nutrition': _product_nutrients(product)}, + } + return InventoryScanItem.objects.create(batch=batch, location=batch.location, + barcode=barcode, **values) + + +def confirm_scan_item(user, item_id, values): + with transaction.atomic(): + item = (InventoryScanItem.objects.select_for_update().select_related('batch__run') + .filter(pk=item_id, batch__run__user=user).first()) + if not item or item.status != InventoryScanItem.Status.ACTIVE or item.batch.status != InventoryScanBatch.Status.DRAFT: + raise ScanWorkflowError('This scan can no longer be edited.') + location = InventoryLocation.objects.filter(user=user, pk=values.get('location_id')).first() + if not location: + raise ScanWorkflowError('Choose one of your existing inventory locations.') + item.location = location + item.product_name = str(values.get('product_name', item.product_name)).strip()[:255] + item.brand = str(values.get('brand', item.brand)).strip()[:255] + item.category_name = str(values.get('category_name', '')).strip()[:100] + item.notes = str(values.get('notes', '')).strip()[:4000] + item.package_count = _integer(values.get('package_count', 1), 'Enter a valid number of packages.', minimum=1, maximum=10000) + item.package_amount = _decimal_or_none(values.get('package_amount'), 'Enter a valid package size.') + item.package_unit = str(values.get('package_unit', '')).strip()[:8] + item.pieces_per_package = _integer_or_none(values.get('pieces_per_package'), 'Enter a valid number of pieces.', maximum=100000) + item.piece_amount = _decimal_or_none(values.get('piece_amount'), 'Enter a valid amount per piece.') + item.inner_packs_per_package = _integer_or_none(values.get('inner_packs_per_package'), 'Enter a valid number of inner packs.', maximum=100000) + item.pieces_per_inner_pack = _integer_or_none(values.get('pieces_per_inner_pack'), 'Enter a valid number of pieces per inner pack.', maximum=100000) + item.inner_pack_amount = _decimal_or_none(values.get('inner_pack_amount'), 'Enter a valid inner pack amount.') + item.inner_pack_label = str(values.get('inner_pack_label', '')).strip()[:40] + item.piece_label = str(values.get('piece_label', '')).strip()[:40] + item.best_before = values.get('best_before') or None + item.opened = bool(values.get('opened')) + item.user_data = _item_data(item) + item.status = InventoryScanItem.Status.CONFIRMED + item.confirmed_at = timezone.now() + item.save() + item.batch.run.selected_location = location + item.batch.run.save(update_fields=('selected_location',)) + return item + + +def discard_active_scan(user, item_id): + deleted, _ = InventoryScanItem.objects.filter(pk=item_id, batch__run__user=user, + status=InventoryScanItem.Status.ACTIVE, batch__status=InventoryScanBatch.Status.DRAFT).delete() + if not deleted: + raise ScanWorkflowError('Only an unconfirmed active scan can be removed.') + + +def adjust_confirmed_count(user, item_id, delta): + if delta not in (-1, 1): + raise ScanWorkflowError('Invalid package adjustment.') + with transaction.atomic(): + item = InventoryScanItem.objects.select_for_update().filter(pk=item_id, batch__run__user=user, + batch__status=InventoryScanBatch.Status.DRAFT, status=InventoryScanItem.Status.CONFIRMED).first() + if not item: + raise ScanWorkflowError('This queued item can no longer be changed.') + if item.package_count + delta < 1: + item.delete() + return None + item.package_count += delta + item.user_data = _item_data(item) + item.save(update_fields=('package_count', 'user_data')) + return item + + +def submit_batch(user, *, batch_id, next_location_id=None, finish=False): + with transaction.atomic(): + batch = (InventoryScanBatch.objects.select_for_update().select_related('run') + .filter(pk=batch_id, run__user=user).first()) + if not batch: + raise ScanWorkflowError('Scan batch not found.') + run = InventoryScanRun.objects.select_for_update().get(pk=batch.run_id) + if batch.status == InventoryScanBatch.Status.QUEUED: + return run + if run.status != InventoryScanRun.Status.ACTIVE or batch.status != InventoryScanBatch.Status.DRAFT: + raise ScanWorkflowError('This scan batch has already been submitted.') + if batch.items.filter(status=InventoryScanItem.Status.ACTIVE).exists(): + raise ScanWorkflowError('Confirm or remove the active item before saving this batch.') + items = batch.items.filter(status=InventoryScanItem.Status.CONFIRMED) + if not items.exists(): + raise ScanWorkflowError('Confirm at least one item before saving this batch.') + now = timezone.now() + items.update(status=InventoryScanItem.Status.QUEUED, next_attempt_at=now) + batch.status = InventoryScanBatch.Status.QUEUED + batch.submitted_at = now + batch.save(update_fields=('status', 'submitted_at')) + if finish: + run.status = InventoryScanRun.Status.PROCESSING + run.submitted_at = now + run.save(update_fields=('status', 'submitted_at')) + else: + location = InventoryLocation.objects.filter(user=user, pk=next_location_id).first() + if not location: + raise ScanWorkflowError('Choose an existing location for the next batch.') + run.selected_location = location + next_sequence = (run.batches.order_by('-sequence').values_list('sequence', flat=True).first() or 0) + 1 + InventoryScanBatch.objects.create(run=run, location=location, sequence=next_sequence) + run.save(update_fields=('selected_location',)) + return run + + +def _claim_next_item(): + now = timezone.now() + with transaction.atomic(): + skip_locked = connections['default'].features.has_select_for_update_skip_locked + qs = InventoryScanItem.objects.select_for_update(skip_locked=skip_locked).filter( + Q(status=InventoryScanItem.Status.QUEUED, next_attempt_at__lte=now) | + Q(status=InventoryScanItem.Status.PROCESSING, lease_until__lt=now), + batch__status=InventoryScanBatch.Status.QUEUED, + batch__run__status__in=(InventoryScanRun.Status.ACTIVE, InventoryScanRun.Status.PROCESSING)) + item = qs.select_related('batch__run__user').order_by('next_attempt_at', 'created_at', 'pk').first() + if not item: + return None + item.status = InventoryScanItem.Status.PROCESSING + item.lease_until = now + timedelta(minutes=15) + item.last_error = '' + item.save(update_fields=('status', 'lease_until', 'last_error')) + return item.pk + + +def process_next_scan_item(): + item_id = _claim_next_item() + if not item_id: + return False + item = InventoryScanItem.objects.select_related('batch__run__user', 'location').get(pk=item_id) + product = FoodProduct.objects.filter(owner=item.batch.run.user, barcode=item.barcode).first() + try: + if not product: + def record_http_attempt(): + InventoryScanItem.objects.filter(pk=item_id, status=InventoryScanItem.Status.PROCESSING).update( + attempts=F('attempts') + 1) + product = cached_product(item.batch.run.user, item.barcode, wait_for_slot=True, + on_request_start=record_http_attempt) + except OpenFoodFactsTransientError as error: + _retry_or_review(item_id, str(error), retry_after=error.retry_after) + _refresh_completion(item.batch.run_id) + return True + except OpenFoodFactsError as error: + _mark_needs_review(item_id, str(error)) + _refresh_completion(item.batch.run_id) + return True + if not product: + _mark_needs_review(item_id, 'No product was found for this barcode. Enter product details to approve it.') + _refresh_completion(item.batch.run_id) + return True + with transaction.atomic(): + current = InventoryScanItem.objects.select_for_update().select_related('batch__run').get(pk=item_id) + if current.status != InventoryScanItem.Status.PROCESSING: + return True + _populate_from_product(current, product) + current.product = product + if not current.product_name.strip() or not current.location_id or not _valid_stock_input(current): + current.status = InventoryScanItem.Status.NEEDS_REVIEW + current.last_error = 'Product identity or package details need review.' + current.completed_at = timezone.now() + current.lease_until = None + current.save() + else: + try: + stock = _create_inventory_stock(current, product) + except (ValidationError, ScanWorkflowError) as error: + current.status = InventoryScanItem.Status.NEEDS_REVIEW + current.last_error = str(error)[:255] + current.completed_at = timezone.now() + current.lease_until = None + current.save() + else: + current.status = InventoryScanItem.Status.COMPLETE + current.inventory_item = stock + current.completed_at = timezone.now() + current.lease_until = None + current.last_error = '' + current.save() + _refresh_completion(item.batch.run_id) + return True + + +def _retry_or_review(item_id, message, *, retry_after=None): + with transaction.atomic(): + item = InventoryScanItem.objects.select_for_update().get(pk=item_id) + item.lease_until = None + item.last_error = message[:255] + item.refresh_from_db(fields=('attempts',)) + if item.attempts >= 10: + item.status = InventoryScanItem.Status.NEEDS_REVIEW + item.completed_at = timezone.now() + else: + backoff = min(900, 5 * (2 ** max(0, item.attempts - 1))) + delay = max(backoff + random.uniform(0, backoff * 0.25), retry_after or 0) + item.status = InventoryScanItem.Status.QUEUED + item.next_attempt_at = timezone.now() + timedelta(seconds=delay) + item.save(update_fields=('status', 'lease_until', 'last_error', 'completed_at', 'next_attempt_at')) + + +def _mark_needs_review(item_id, message): + InventoryScanItem.objects.filter(pk=item_id, status=InventoryScanItem.Status.PROCESSING).update( + status=InventoryScanItem.Status.NEEDS_REVIEW, last_error=message[:255], completed_at=timezone.now(), lease_until=None) + + +def _populate_from_product(item, product): + if not item.product_name: + item.product_name = product.product_name + if not item.brand: + item.brand = product.brand + if item.package_amount is None and product.package_amount is not None: + item.package_amount = product.package_amount + item.package_unit = product.package_unit + item.source_data = { + 'source': product.source, + 'fetched_at': product.fetched_at.isoformat() if product.fetched_at else None, + 'product_name': product.product_name, + 'brand': product.brand, + 'package_quantity': product.package_quantity, + 'image_url': product.image_url, + 'nutrition': _product_nutrients(product), + 'raw_data': product.raw_data if isinstance(product.raw_data, dict) else {}, + } + item.user_data = _item_data(item) + + +def _create_inventory_stock(item, product): + package_amount = item.package_amount + package_unit = item.package_unit + pieces = item.pieces_per_package + if pieces and package_amount is None and not package_unit: + package_amount, package_unit = Decimal(pieces), 'pieces' + if pieces and item.inner_packs_per_package and item.pieces_per_inner_pack: + if package_amount is None and item.inner_pack_amount is not None: + package_amount = item.inner_pack_amount * item.inner_packs_per_package + if package_unit and package_unit not in {'mg', 'g', 'kg', 'ml', 'L', 'pieces', 'packs', 'portions'}: + raise ScanWorkflowError('Choose a supported package unit.') + category = None + if item.category_name: + category, _ = InventoryCategory.objects.get_or_create(user=item.batch.run.user, + name__iexact=item.category_name, defaults={'name': item.category_name}) + base = KitchenInventoryItem( + user=item.batch.run.user, product=product, location=item.location, + category=category, quantity=item.package_count if package_amount is None else 1, + notes=item.notes, opened=item.opened, best_before=item.best_before, + original_amount=package_amount, remaining_amount=package_amount, + inventory_unit=package_unit, pieces_per_package=pieces, + remaining_pieces=pieces, piece_amount=item.piece_amount, + inner_packs_per_package=item.inner_packs_per_package, + pieces_per_inner_pack=item.pieces_per_inner_pack, + inner_pack_amount=item.inner_pack_amount, inner_pack_label=item.inner_pack_label, + piece_label=item.piece_label, + ) + base.full_clean() + base.save() + if package_amount is None: + return base + for _ in range(item.package_count - 1): + package = KitchenInventoryItem( + user=base.user, product=base.product, location=base.location, + category=base.category, quantity=1, notes=base.notes, opened=base.opened, + best_before=base.best_before, original_amount=base.original_amount, + remaining_amount=base.original_amount, inventory_unit=base.inventory_unit, + pieces_per_package=base.pieces_per_package, remaining_pieces=base.pieces_per_package, + piece_amount=base.piece_amount, inner_packs_per_package=base.inner_packs_per_package, + pieces_per_inner_pack=base.pieces_per_inner_pack, inner_pack_amount=base.inner_pack_amount, + inner_pack_label=base.inner_pack_label, piece_label=base.piece_label, + ) + package.full_clean() + package.save() + return base + + +def _valid_stock_input(item): + if item.package_amount is None and not item.pieces_per_package: + return True # Count-only packages have unknown size by design. + if item.package_amount is not None and item.package_unit: + return True + return bool(item.inner_packs_per_package and item.pieces_per_inner_pack and + item.inner_pack_amount and item.package_unit) + + +def _refresh_completion(run_id): + with transaction.atomic(): + run = InventoryScanRun.objects.select_for_update().get(pk=run_id) + for batch in run.batches.filter(status=InventoryScanBatch.Status.QUEUED): + if not batch.items.filter(status__in=(InventoryScanItem.Status.QUEUED, InventoryScanItem.Status.PROCESSING)).exists(): + batch.status = InventoryScanBatch.Status.COMPLETE + batch.completed_at = timezone.now() + batch.save(update_fields=('status', 'completed_at')) + if run.status == InventoryScanRun.Status.PROCESSING and not run.batches.filter( + status=InventoryScanBatch.Status.QUEUED).exists(): + review = run.batches.filter(items__status__in=(InventoryScanItem.Status.NEEDS_REVIEW, + InventoryScanItem.Status.DEFERRED, + InventoryScanItem.Status.FAILED)).exists() + run.status = (InventoryScanRun.Status.COMPLETE_WITH_REVIEW if review + else InventoryScanRun.Status.COMPLETE) + run.completed_at = timezone.now() + run.save(update_fields=('status', 'completed_at')) + if run.status in (InventoryScanRun.Status.COMPLETE, InventoryScanRun.Status.COMPLETE_WITH_REVIEW): + _send_completion_notification(run_id) + + +def _send_completion_notification(run_id): + from planner.services.notifications import NotificationError, send_notification + + now = timezone.now() + with transaction.atomic(): + run = InventoryScanRun.objects.select_for_update().select_related('user').get(pk=run_id) + if run.notification_attempted_at or run.status not in ( + InventoryScanRun.Status.COMPLETE, InventoryScanRun.Status.COMPLETE_WITH_REVIEW): + return + from django.db.models import Count + tallies = dict(run.batches.values('items__status').annotate(total=Count('items')).values_list('items__status', 'total')) + succeeded = tallies.get(InventoryScanItem.Status.COMPLETE, 0) + review = sum(tallies.get(status, 0) for status in (InventoryScanItem.Status.NEEDS_REVIEW, + InventoryScanItem.Status.DEFERRED, + InventoryScanItem.Status.FAILED)) + failed = tallies.get(InventoryScanItem.Status.FAILED, 0) + run.notification_attempted_at = now # durable at-most-once delivery guard for worker restarts + run.save(update_fields=('notification_attempted_at',)) + user = run.user + url = '/food/inventory/#inventory-review' + message = f'Bulk scan finished: {succeeded} added, {review} need review, {failed} failed. {url}' + try: + sent = send_notification(user, 'Kitchen inventory scan complete', message) + except NotificationError as error: + InventoryScanRun.objects.filter(pk=run_id).update(notification_error=str(error)[:255]) + return + if sent: + InventoryScanRun.objects.filter(pk=run_id).update(notification_sent_at=timezone.now(), notification_error='') + + +def update_review_preference(user, mode): + if mode not in InventoryReviewPreference.Mode.values: + raise ScanWorkflowError('Choose Individual or Bulk review.') + preference, _ = InventoryReviewPreference.objects.update_or_create(user=user, defaults={'mode': mode}) + return preference + + +def approve_scan_item(user, item_id, values): + with transaction.atomic(): + item = (InventoryScanItem.objects.select_for_update().select_related('batch__run__user') + .filter(pk=item_id, batch__run__user=user, + status__in=(InventoryScanItem.Status.NEEDS_REVIEW, InventoryScanItem.Status.DEFERRED)).first()) + if not item: + raise ScanWorkflowError('This staged item is no longer awaiting review.') + if values.get('product_name'): + item.product_name = str(values['product_name']).strip()[:255] + if values.get('brand') is not None: + item.brand = str(values['brand']).strip()[:255] + if values.get('category_name') is not None: + item.category_name = str(values['category_name']).strip()[:100] + if values.get('notes') is not None: + item.notes = str(values['notes']).strip()[:4000] + if values.get('location_id'): + item.location = InventoryLocation.objects.get(user=user, pk=values['location_id']) + item.package_count = _integer(values.get('package_count', item.package_count), 'Enter a valid number of packages.', 1, 10000) + if 'package_amount' in values: + item.package_amount = _decimal_or_none(values['package_amount'], 'Enter a valid package size.') + if 'package_unit' in values: + item.package_unit = str(values['package_unit']).strip()[:8] + if not item.product_name: + raise ScanWorkflowError('Enter a product name before approving.') + product = item.product or FoodProduct.objects.filter(owner=user, barcode=item.barcode).first() + if not product: + product = FoodProduct.objects.create(owner=user, barcode=item.barcode, + product_name=item.product_name, brand=item.brand, + package_quantity=(f'{item.package_amount} {item.package_unit}' if item.package_amount else ''), + package_amount=item.package_amount, package_unit=item.package_unit, + source=FoodProduct.Source.MANUAL, raw_data={'scan_item_id': item.pk, 'manual_nutrition_override': True}) + elif product.source == FoodProduct.Source.MANUAL: + product.product_name = item.product_name + product.brand = item.brand + product.save(update_fields=('product_name', 'brand', 'updated_at')) + else: + _apply_scan_product_corrections(product, item) + _populate_from_product(item, product) + item.product = product + if not _valid_stock_input(item): + raise ScanWorkflowError('Add valid package details or choose count-only stock before approving.') + stock = _create_inventory_stock(item, product) + item.status = InventoryScanItem.Status.COMPLETE + item.inventory_item = stock + item.last_error = '' + item.completed_at = timezone.now() + item.save() + _refresh_completion(item.batch.run_id) + return item + + +def defer_scan_item(user, item_id): + updated = InventoryScanItem.objects.filter(pk=item_id, batch__run__user=user, + status=InventoryScanItem.Status.NEEDS_REVIEW).update(status=InventoryScanItem.Status.DEFERRED) + if not updated: + raise ScanWorkflowError('This item can no longer be deferred.') + + +def _apply_scan_product_corrections(product, item): + # Metadata corrections are limited to product identity and packaging. Nutrition remains source-owned. + if (product.source == FoodProduct.Source.OPEN_FOOD_FACTS and isinstance(product.raw_data, dict) and + product.raw_data.get('manual_nutrition_override')): + return + product.product_name = item.product_name or product.product_name + product.brand = item.brand or product.brand + product.package_amount = item.package_amount or product.package_amount + product.package_unit = item.package_unit or product.package_unit + product.save(update_fields=('product_name', 'brand', 'package_amount', 'package_unit', 'updated_at')) + + +def _product_nutrients(product): + return {field: str(getattr(product, field)) if getattr(product, field) is not None else None + for field in ('energy_kcal_100', 'fat_100', 'saturated_fat_100', 'carbohydrates_100', + 'sugars_100', 'protein_100', 'salt_100', 'fibre_100')} + + +def _item_data(item): + return {'product_name': item.product_name, 'brand': item.brand, 'category_name': item.category_name, + 'notes': item.notes, 'location_id': item.location_id, 'package_count': item.package_count, + 'package_amount': str(item.package_amount) if item.package_amount is not None else None, + 'package_unit': item.package_unit, 'pieces_per_package': item.pieces_per_package, + 'piece_amount': str(item.piece_amount) if item.piece_amount is not None else None, + 'inner_packs_per_package': item.inner_packs_per_package, + 'pieces_per_inner_pack': item.pieces_per_inner_pack, + 'inner_pack_amount': str(item.inner_pack_amount) if item.inner_pack_amount is not None else None, + 'best_before': item.best_before.isoformat() if item.best_before else None, 'opened': item.opened} + + +def _integer(value, message, minimum=0, maximum=100000): + try: + result = int(value) + except (TypeError, ValueError): + raise ScanWorkflowError(message) from None + if str(result) != str(value).strip() and not isinstance(value, int): + raise ScanWorkflowError(message) + if not minimum <= result <= maximum: + raise ScanWorkflowError(message) + return result + + +def _integer_or_none(value, message, maximum=100000): + return None if value in (None, '') else _integer(value, message, 1, maximum) + + +def _decimal_or_none(value, message): + if value in (None, ''): + return None + try: + result = Decimal(str(value)) + except (InvalidOperation, TypeError, ValueError): + raise ScanWorkflowError(message) from None + if not result.is_finite() or result <= 0 or result > Decimal('9999999.999'): + raise ScanWorkflowError(message) + return result.quantize(Decimal('.001')) diff --git a/main/food/services/off_rate_limit.py b/main/food/services/off_rate_limit.py new file mode 100644 index 0000000..f8090a6 --- /dev/null +++ b/main/food/services/off_rate_limit.py @@ -0,0 +1,59 @@ +"""Database-coordinated cap for actual Open Food Facts HTTP requests.""" +from contextlib import contextmanager +from datetime import timedelta +import time + +from django.db import transaction +from django.utils import timezone + +from food.models import OpenFoodFactsRateLimit, OpenFoodFactsRequest + + +REQUESTS_PER_MINUTE = 5 +WINDOW_SECONDS = 60 + + +@contextmanager +def off_http_request(endpoint, *, wait=True, before_request=None): + """Reserve and serialize one real request across web and worker processes. + + The singleton row lock is held through the HTTP call, so simultaneous workers + cannot race through the budget check. Failed HTTP requests still count because + the reservation commits before the exception is re-raised. + """ + while True: + delay = 0 + failure = None + traceback = None + with transaction.atomic(): + OpenFoodFactsRateLimit.objects.get_or_create(pk=1) + OpenFoodFactsRateLimit.objects.select_for_update().get(pk=1) + now = timezone.now() + cutoff = now - timedelta(seconds=WINDOW_SECONDS) + oldest = OpenFoodFactsRequest.objects.filter(requested_at__gte=cutoff).order_by('requested_at').first() + if OpenFoodFactsRequest.objects.filter(requested_at__gte=cutoff).count() >= REQUESTS_PER_MINUTE: + delay = max(0.05, (oldest.requested_at + timedelta(seconds=WINDOW_SECONDS + 0.01) - now).total_seconds()) + else: + OpenFoodFactsRequest.objects.create(requested_at=now, endpoint=endpoint[:24]) + OpenFoodFactsRequest.objects.filter(requested_at__lt=now - timedelta(minutes=10)).delete() + try: + if before_request: + before_request() + yield + except BaseException as error: + failure = error + traceback = error.__traceback__ + if delay: + if not wait: + raise OpenFoodFactsRateLimited(delay) + time.sleep(delay) + continue + if failure is not None: + raise failure.with_traceback(traceback) + return + + +class OpenFoodFactsRateLimited(Exception): + def __init__(self, retry_after): + self.retry_after = max(1, int(retry_after + 0.999)) + super().__init__(f'Open Food Facts request budget is full; retry in {self.retry_after} seconds.') diff --git a/main/food/services/openfoodfacts.py b/main/food/services/openfoodfacts.py index 5c559f4..3b5c1ea 100644 --- a/main/food/services/openfoodfacts.py +++ b/main/food/services/openfoodfacts.py @@ -2,6 +2,7 @@ import json import logging import re +from email.utils import parsedate_to_datetime from hashlib import sha256 from datetime import timedelta from decimal import Decimal, InvalidOperation @@ -15,6 +16,7 @@ from django.core.cache import cache from django.utils import timezone from food.models import FoodProduct, Unit +from .off_rate_limit import OpenFoodFactsRateLimited, off_http_request logger = logging.getLogger(__name__) FIELDS = 'code,product_name,brands,quantity,serving_size,serving_quantity,image_front_small_url,image_front_url,nutriments,categories,categories_tags' @@ -35,7 +37,16 @@ class OpenFoodFactsError(Exception): class OpenFoodFactsRateLimit(OpenFoodFactsError): - pass + def __init__(self, message, retry_after=None): + self.retry_after = retry_after + super().__init__(message) + + +class OpenFoodFactsTransientError(OpenFoodFactsError): + def __init__(self, message, *, retry_after=None, status_code=None): + self.retry_after = retry_after + self.status_code = status_code + super().__init__(message) def normalise_query(query): @@ -108,41 +119,59 @@ def _rate_limit(user_id, kind, limit): class OpenFoodFactsClient: - def _get_json(self, url, params=None): + def _get_json(self, url, params=None, *, wait_for_slot=False, on_request_start=None): if not settings.OPENFOODFACTS_ENABLED: raise OpenFoodFactsError('Open Food Facts is unavailable.') if params: url = f'{url}?{urlencode(params)}' request = Request(url, headers={'User-Agent': settings.OPENFOODFACTS_USER_AGENT, 'Accept': 'application/json'}) + endpoint = 'product' if '/product/' in url else 'search' try: - with urlopen(request, timeout=settings.OPENFOODFACTS_TIMEOUT_SECONDS) as response: - data = json.load(response) - except (HTTPError, URLError, TimeoutError, ValueError, OSError) as error: - logger.warning('Open Food Facts request failed: %s', error) - raise OpenFoodFactsError('Open Food Facts is temporarily unavailable.') from error + with off_http_request(endpoint, wait=wait_for_slot, before_request=on_request_start): + try: + with urlopen(request, timeout=settings.OPENFOODFACTS_TIMEOUT_SECONDS) as response: + data = json.load(response) + except HTTPError as error: + retry_after = _retry_after(error.headers.get('Retry-After') if error.headers else None) + logger.warning('Open Food Facts HTTP %s on %s request', error.code, endpoint) + if error.code == 429 or error.code >= 500: + raise OpenFoodFactsTransientError( + 'Open Food Facts is temporarily unavailable.', retry_after=retry_after, + status_code=error.code) from error + raise OpenFoodFactsError(f'Open Food Facts rejected the request (HTTP {error.code}).') from error + except (URLError, TimeoutError, OSError) as error: + logger.warning('Open Food Facts request failed: %s', error) + raise OpenFoodFactsTransientError('Open Food Facts is temporarily unavailable.') from error + except ValueError as error: + logger.warning('Open Food Facts returned invalid JSON: %s', error) + raise OpenFoodFactsError('Open Food Facts returned an invalid response.') from error + except OpenFoodFactsRateLimited as error: + raise OpenFoodFactsRateLimit(str(error), retry_after=error.retry_after) from error if not isinstance(data, dict): raise OpenFoodFactsError('Open Food Facts returned an invalid response.') return data - def get_product(self, barcode): + def get_product(self, barcode, *, wait_for_slot=False, on_request_start=None): if not re.fullmatch(r'[0-9]{1,64}', barcode): raise ValueError('Enter a numeric barcode.') - data = self._get_json(settings.OPENFOODFACTS_PRODUCT_URL.format(barcode=barcode), {'fields': FIELDS}) + data = self._get_json(settings.OPENFOODFACTS_PRODUCT_URL.format(barcode=barcode), {'fields': FIELDS}, + wait_for_slot=wait_for_slot, on_request_start=on_request_start) return data.get('product') if data.get('status') == 1 else None - def search(self, query, page=1): + def search(self, query, page=1, *, wait_for_slot=False): query = normalise_query(query) if not query or len(query) > 100 or page < 1 or page > 20: raise ValueError('Enter a search query and valid page.') data = self._get_json(settings.OPENFOODFACTS_SEARCH_URL, {'action': 'process', 'search_simple': 1, - 'json': 1, 'search_terms': query, 'page': page, 'page_size': 20, 'fields': FIELDS}) + 'json': 1, 'search_terms': query, 'page': page, 'page_size': 20, 'fields': FIELDS}, + wait_for_slot=wait_for_slot) products = data.get('products') if not isinstance(products, list): raise OpenFoodFactsError('Open Food Facts returned an invalid response.') return [mapped for item in products if (mapped := map_product(item))] -def cached_product(user, barcode, *, refresh=False): +def cached_product(user, barcode, *, refresh=False, wait_for_slot=False, on_request_start=None): if not re.fullmatch(r'[0-9]{1,64}', barcode): raise ValueError('Enter a numeric barcode.') product = FoodProduct.objects.filter(owner=user, barcode=barcode).first() @@ -154,7 +183,8 @@ def cached_product(user, barcode, *, refresh=False): return product _rate_limit(user.pk, 'product', 10) try: - data = OpenFoodFactsClient().get_product(barcode) + data = OpenFoodFactsClient().get_product(barcode, wait_for_slot=wait_for_slot, + on_request_start=on_request_start) except OpenFoodFactsError: if product: return product @@ -173,6 +203,21 @@ def cached_product(user, barcode, *, refresh=False): return product +def _retry_after(value): + if not value: + return None + try: + return max(0, int(value.strip())) + except (AttributeError, TypeError, ValueError): + try: + requested_at = parsedate_to_datetime(value) + if requested_at.tzinfo is None: + requested_at = requested_at.replace(tzinfo=timezone.get_current_timezone()) + return max(0, int((requested_at - timezone.now()).total_seconds())) + except (TypeError, ValueError, OverflowError): + return None + + def cached_search(user, query, page=1): normalized = normalise_query(query) if not normalized or len(normalized) > 100 or page < 1 or page > 20: diff --git a/main/food/test_inventory_scanner.py b/main/food/test_inventory_scanner.py new file mode 100644 index 0000000..15df235 --- /dev/null +++ b/main/food/test_inventory_scanner.py @@ -0,0 +1,120 @@ +from datetime import timedelta +from decimal import Decimal +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.utils import timezone + +from food.models import (FoodProduct, InventoryLocation, InventoryScanBatch, + InventoryScanItem, InventoryScanRun, + KitchenInventoryItem, OpenFoodFactsRequest) +from food.services.inventory_scanner import (ScanWorkflowError, + confirm_scan_item, create_active_scan, get_or_create_active_run, + process_next_scan_item, submit_batch) +from food.services.off_rate_limit import OpenFoodFactsRateLimited, off_http_request +from food.services.openfoodfacts import OpenFoodFactsTransientError + + +class InventoryScanQueueTests(TestCase): + def setUp(self): + self.user = get_user_model().objects.create_user(username='scanner-owner', password='test-password') + self.other = get_user_model().objects.create_user(username='scanner-other', password='test-password') + self.fridge = InventoryLocation.objects.create(user=self.user, name='Fridge') + self.shelf = InventoryLocation.objects.create(user=self.user, name='Shelf') + self.product = FoodProduct.objects.create(owner=self.user, barcode='12345678', + product_name='Oat drink', package_amount=Decimal('1000'), package_unit='ml') + + def _confirmed_item(self, location=None, barcode='12345678'): + run = get_or_create_active_run(self.user, (location or self.fridge).pk) + item = create_active_scan(run, self.user, barcode) + confirm_scan_item(self.user, item.pk, { + 'location_id': (location or self.fridge).pk, 'product_name': 'Oat drink', + 'package_count': 2, 'package_amount': '1000', 'package_unit': 'ml', + }) + return run, item + + def test_location_batches_are_atomic_and_keep_original_locations(self): + run, item = self._confirmed_item() + batch = item.batch + updated = submit_batch(self.user, batch_id=batch.pk, next_location_id=self.shelf.pk) + batch.refresh_from_db() + new_batch = updated.batches.get(status=InventoryScanBatch.Status.DRAFT) + self.assertEqual((batch.status, batch.location), (InventoryScanBatch.Status.QUEUED, self.fridge)) + self.assertEqual(new_batch.location, self.shelf) + self.assertEqual(item.location, self.fridge) + item.refresh_from_db() + self.assertEqual(item.status, InventoryScanItem.Status.QUEUED) + + def test_batch_rejects_other_users_location(self): + run, item = self._confirmed_item() + foreign = InventoryLocation.objects.create(user=self.other, name='Other') + with self.assertRaises(ScanWorkflowError): + submit_batch(self.user, batch_id=item.batch_id, next_location_id=foreign.pk) + item.refresh_from_db() + self.assertEqual(item.status, InventoryScanItem.Status.CONFIRMED) + + def test_active_run_resumes_and_rejects_second_active_scan(self): + run = get_or_create_active_run(self.user, self.fridge.pk) + item = create_active_scan(run, self.user, '12345678') + self.assertEqual(get_or_create_active_run(self.user, self.shelf.pk).pk, run.pk) + with self.assertRaises(ScanWorkflowError): + create_active_scan(run, self.user, '87654321') + + def test_done_enqueues_and_worker_creates_separate_tracked_packages(self): + run, item = self._confirmed_item() + submit_batch(self.user, batch_id=item.batch_id, finish=True) + with patch('food.services.inventory_scanner._send_completion_notification') as notify: + self.assertTrue(process_next_scan_item()) + item.refresh_from_db() + run.refresh_from_db() + self.assertEqual(item.status, InventoryScanItem.Status.COMPLETE) + self.assertEqual(item.inventory_item.product, self.product) + stock = KitchenInventoryItem.objects.filter(user=self.user, product=self.product) + self.assertEqual(stock.count(), 2) + self.assertTrue(all(row.original_amount == Decimal('1000') and row.inventory_unit == 'ml' for row in stock)) + self.assertEqual(run.status, InventoryScanRun.Status.COMPLETE) + notify.assert_called_once_with(run.pk) + + def test_transient_error_is_scheduled_and_bounded_by_ten_attempts(self): + run, item = self._confirmed_item(barcode='87654321') + submit_batch(self.user, batch_id=item.batch_id, finish=True) + def transient_request(*args, **kwargs): + kwargs['on_request_start']() + raise OpenFoodFactsTransientError('temporary failure', retry_after=30) + with patch('food.services.inventory_scanner.cached_product', side_effect=transient_request): + process_next_scan_item() + item.refresh_from_db() + self.assertEqual(item.attempts, 1) + self.assertEqual(item.status, InventoryScanItem.Status.QUEUED) + self.assertGreaterEqual(item.next_attempt_at, timezone.now() + timedelta(seconds=29)) + + def test_unknown_product_moves_to_review_without_losing_user_fields(self): + run, item = self._confirmed_item(barcode='87654321') + submit_batch(self.user, batch_id=item.batch_id, finish=True) + with patch('food.services.inventory_scanner.cached_product', return_value=None): + process_next_scan_item() + item.refresh_from_db() + self.assertEqual(item.status, InventoryScanItem.Status.NEEDS_REVIEW) + self.assertEqual(item.product_name, 'Oat drink') + self.assertEqual(item.location, self.fridge) + self.assertIsNone(item.inventory_item) + + +class OpenFoodFactsGlobalLimitTests(TestCase): + def test_non_waiting_request_is_refused_when_shared_budget_is_full(self): + now = timezone.now() + OpenFoodFactsRequest.objects.bulk_create([ + OpenFoodFactsRequest(requested_at=now - timedelta(seconds=10 * index), endpoint='product') + for index in range(5) + ]) + with self.assertRaises(OpenFoodFactsRateLimited): + with off_http_request('search', wait=False): + self.fail('The sixth actual request must not start.') + self.assertEqual(OpenFoodFactsRequest.objects.count(), 5) + + def test_failed_actual_request_still_consumes_budget(self): + with self.assertRaises(RuntimeError): + with off_http_request('product', wait=False): + raise RuntimeError('network error after request start') + self.assertEqual(OpenFoodFactsRequest.objects.count(), 1)