Add persistent inventory scan job foundation
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Process Tasknest kitchen inventory scan jobs
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=jorenmartijn
|
||||
WorkingDirectory=/home/jorenmartijn/django/main
|
||||
ExecStart=/home/jorenmartijn/django/tasksnest-host/bin/python manage.py process_inventory_scans --poll-seconds 2
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,27 @@
|
||||
import time
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from food.services.inventory_scanner import process_next_scan_item
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Process due persisted kitchen inventory scan enrichment jobs.'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('--once', action='store_true', help='Process at most one job and exit.')
|
||||
parser.add_argument('--poll-seconds', type=float, default=2.0)
|
||||
parser.add_argument('--max-items', type=int, default=0, help='Stop after this many jobs; 0 runs continuously.')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
processed = 0
|
||||
while True:
|
||||
did_work = process_next_scan_item()
|
||||
if did_work:
|
||||
processed += 1
|
||||
if options['once'] or (options['max_items'] and processed >= options['max_items']):
|
||||
return
|
||||
elif options['once']:
|
||||
return
|
||||
else:
|
||||
time.sleep(max(0.25, options['poll_seconds']))
|
||||
@@ -0,0 +1,127 @@
|
||||
# Generated by Django 6.1.1 on 2026-10-25 23:32
|
||||
|
||||
import django.core.validators
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('food', '0014_alter_inventoryconsumption_unit'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='OpenFoodFactsRequest',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('requested_at', models.DateTimeField(db_index=True)),
|
||||
('endpoint', models.CharField(blank=True, max_length=24)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='OpenFoodFactsRateLimit',
|
||||
fields=[
|
||||
('id', models.PositiveSmallIntegerField(default=1, editable=False, primary_key=True, serialize=False)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='InventoryReviewPreference',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('mode', models.CharField(choices=[('individual', 'Individual'), ('bulk', 'Bulk')], default='individual', max_length=12)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='inventory_review_preference', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='InventoryScanRun',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('api_id', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)),
|
||||
('status', models.CharField(choices=[('active', 'Active'), ('processing', 'Processing'), ('complete', 'Complete'), ('complete_with_review', 'Complete with review'), ('cancelled', 'Cancelled')], db_index=True, default='active', max_length=24)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('submitted_at', models.DateTimeField(blank=True, null=True)),
|
||||
('completed_at', models.DateTimeField(blank=True, null=True)),
|
||||
('notification_attempted_at', models.DateTimeField(blank=True, null=True)),
|
||||
('notification_sent_at', models.DateTimeField(blank=True, null=True)),
|
||||
('notification_error', models.CharField(blank=True, max_length=255)),
|
||||
('selected_location', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='active_scan_runs', to='food.inventorylocation')),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='inventory_scan_runs', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='InventoryScanBatch',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('sequence', models.PositiveIntegerField()),
|
||||
('status', models.CharField(choices=[('draft', 'Draft'), ('queued', 'Queued'), ('complete', 'Complete')], db_index=True, default='draft', max_length=12)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('submitted_at', models.DateTimeField(blank=True, null=True)),
|
||||
('completed_at', models.DateTimeField(blank=True, null=True)),
|
||||
('location', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='scan_batches', to='food.inventorylocation')),
|
||||
('run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='batches', to='food.inventoryscanrun')),
|
||||
],
|
||||
options={
|
||||
'ordering': ('sequence',),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='InventoryScanItem',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('barcode', models.CharField(db_index=True, max_length=64)),
|
||||
('status', models.CharField(choices=[('active', 'Active scan'), ('confirmed', 'Confirmed'), ('queued', 'Queued'), ('processing', 'Processing'), ('needs_review', 'Needs review'), ('complete', 'Complete'), ('deferred', 'Deferred'), ('failed', 'Failed')], db_index=True, default='active', max_length=16)),
|
||||
('package_count', models.PositiveIntegerField(default=1, validators=[django.core.validators.MinValueValidator(1)])),
|
||||
('package_amount', models.DecimalField(blank=True, decimal_places=3, max_digits=10, null=True, validators=[django.core.validators.MinValueValidator(0.001)])),
|
||||
('package_unit', models.CharField(blank=True, max_length=8)),
|
||||
('pieces_per_package', models.PositiveIntegerField(blank=True, null=True)),
|
||||
('piece_amount', models.DecimalField(blank=True, decimal_places=3, max_digits=10, null=True, validators=[django.core.validators.MinValueValidator(0.001)])),
|
||||
('inner_packs_per_package', models.PositiveIntegerField(blank=True, null=True)),
|
||||
('pieces_per_inner_pack', models.PositiveIntegerField(blank=True, null=True)),
|
||||
('inner_pack_amount', models.DecimalField(blank=True, decimal_places=3, max_digits=10, null=True, validators=[django.core.validators.MinValueValidator(0.001)])),
|
||||
('inner_pack_label', models.CharField(blank=True, max_length=40)),
|
||||
('piece_label', models.CharField(blank=True, max_length=40)),
|
||||
('product_name', models.CharField(blank=True, max_length=255)),
|
||||
('brand', models.CharField(blank=True, max_length=255)),
|
||||
('category_name', models.CharField(blank=True, max_length=100)),
|
||||
('notes', models.TextField(blank=True)),
|
||||
('best_before', models.DateField(blank=True, null=True)),
|
||||
('opened', models.BooleanField(default=False)),
|
||||
('source_data', models.JSONField(blank=True, default=dict)),
|
||||
('user_data', models.JSONField(blank=True, default=dict)),
|
||||
('attempts', models.PositiveSmallIntegerField(default=0)),
|
||||
('next_attempt_at', models.DateTimeField(blank=True, db_index=True, null=True)),
|
||||
('lease_until', models.DateTimeField(blank=True, db_index=True, null=True)),
|
||||
('last_error', models.CharField(blank=True, max_length=255)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('confirmed_at', models.DateTimeField(blank=True, null=True)),
|
||||
('completed_at', models.DateTimeField(blank=True, null=True)),
|
||||
('batch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='food.inventoryscanbatch')),
|
||||
('location', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='scan_items', to='food.inventorylocation')),
|
||||
('inventory_item', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='scan_items', to='food.kitcheninventoryitem')),
|
||||
('product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='scan_items', to='food.foodproduct')),
|
||||
],
|
||||
options={
|
||||
'ordering': ('created_at', 'pk'),
|
||||
'indexes': [models.Index(fields=['status', 'next_attempt_at'], name='food_scan_item_due')],
|
||||
},
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name='inventoryscanrun',
|
||||
index=models.Index(fields=['user', 'status', '-created_at'], name='food_scan_run_user_status'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='inventoryscanbatch',
|
||||
constraint=models.UniqueConstraint(fields=('run', 'sequence'), name='food_unique_scan_batch_sequence'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='inventoryscanbatch',
|
||||
constraint=models.UniqueConstraint(condition=models.Q(('status', 'draft')), fields=('run',), name='food_one_draft_scan_batch'),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,19 @@
|
||||
# Generated by Django 6.1.1 on 2026-10-25 23:36
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('food', '0015_openfoodfactsrequest_inventoryreviewpreference_and_more'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddConstraint(
|
||||
model_name='inventoryscanrun',
|
||||
constraint=models.UniqueConstraint(condition=models.Q(('status', 'active')), fields=('user',), name='food_one_active_scan_run'),
|
||||
),
|
||||
]
|
||||
@@ -380,3 +380,127 @@ class ApiIdempotencyRecord(models.Model):
|
||||
|
||||
class Meta:
|
||||
constraints = [models.UniqueConstraint(fields=('token', 'key'), name='food_unique_api_idempotency_key')]
|
||||
|
||||
|
||||
class InventoryScanRun(models.Model):
|
||||
"""A resumable scan-in session, including batches submitted while scanning continues."""
|
||||
class Status(models.TextChoices):
|
||||
ACTIVE = 'active', 'Active'
|
||||
PROCESSING = 'processing', 'Processing'
|
||||
COMPLETE = 'complete', 'Complete'
|
||||
COMPLETE_WITH_REVIEW = 'complete_with_review', 'Complete with review'
|
||||
CANCELLED = 'cancelled', 'Cancelled'
|
||||
|
||||
api_id = models.UUIDField(default=uuid.uuid4, unique=True, editable=False)
|
||||
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='inventory_scan_runs')
|
||||
status = models.CharField(max_length=24, choices=Status.choices, default=Status.ACTIVE, db_index=True)
|
||||
selected_location = models.ForeignKey(InventoryLocation, on_delete=models.SET_NULL, null=True, blank=True,
|
||||
related_name='active_scan_runs')
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
submitted_at = models.DateTimeField(null=True, blank=True)
|
||||
completed_at = models.DateTimeField(null=True, blank=True)
|
||||
notification_attempted_at = models.DateTimeField(null=True, blank=True)
|
||||
notification_sent_at = models.DateTimeField(null=True, blank=True)
|
||||
notification_error = models.CharField(max_length=255, blank=True)
|
||||
|
||||
class Meta:
|
||||
indexes = [models.Index(fields=('user', 'status', '-created_at'), name='food_scan_run_user_status')]
|
||||
constraints = [models.UniqueConstraint(fields=('user',), condition=Q(status='active'),
|
||||
name='food_one_active_scan_run')]
|
||||
|
||||
|
||||
class InventoryScanBatch(models.Model):
|
||||
"""A location-specific portion of a run. Save & next closes one and opens another."""
|
||||
class Status(models.TextChoices):
|
||||
DRAFT = 'draft', 'Draft'
|
||||
QUEUED = 'queued', 'Queued'
|
||||
COMPLETE = 'complete', 'Complete'
|
||||
|
||||
run = models.ForeignKey(InventoryScanRun, on_delete=models.CASCADE, related_name='batches')
|
||||
location = models.ForeignKey(InventoryLocation, on_delete=models.PROTECT, related_name='scan_batches')
|
||||
sequence = models.PositiveIntegerField()
|
||||
status = models.CharField(max_length=12, choices=Status.choices, default=Status.DRAFT, db_index=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
submitted_at = models.DateTimeField(null=True, blank=True)
|
||||
completed_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ('sequence',)
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=('run', 'sequence'), name='food_unique_scan_batch_sequence'),
|
||||
models.UniqueConstraint(fields=('run',), condition=Q(status='draft'), name='food_one_draft_scan_batch'),
|
||||
]
|
||||
|
||||
|
||||
class InventoryScanItem(models.Model):
|
||||
"""A persisted scan entry that becomes its own retryable enrichment job on submit."""
|
||||
class Status(models.TextChoices):
|
||||
ACTIVE = 'active', 'Active scan'
|
||||
CONFIRMED = 'confirmed', 'Confirmed'
|
||||
QUEUED = 'queued', 'Queued'
|
||||
PROCESSING = 'processing', 'Processing'
|
||||
NEEDS_REVIEW = 'needs_review', 'Needs review'
|
||||
COMPLETE = 'complete', 'Complete'
|
||||
DEFERRED = 'deferred', 'Deferred'
|
||||
FAILED = 'failed', 'Failed'
|
||||
|
||||
batch = models.ForeignKey(InventoryScanBatch, on_delete=models.CASCADE, related_name='items')
|
||||
location = models.ForeignKey(InventoryLocation, on_delete=models.PROTECT, related_name='scan_items')
|
||||
barcode = models.CharField(max_length=64, db_index=True)
|
||||
status = models.CharField(max_length=16, choices=Status.choices, default=Status.ACTIVE, db_index=True)
|
||||
package_count = models.PositiveIntegerField(default=1, validators=[MinValueValidator(1)])
|
||||
package_amount = models.DecimalField(max_digits=10, decimal_places=3, null=True, blank=True, validators=POSITIVE)
|
||||
package_unit = models.CharField(max_length=8, blank=True)
|
||||
pieces_per_package = models.PositiveIntegerField(null=True, blank=True)
|
||||
piece_amount = models.DecimalField(max_digits=10, decimal_places=3, null=True, blank=True, validators=POSITIVE)
|
||||
inner_packs_per_package = models.PositiveIntegerField(null=True, blank=True)
|
||||
pieces_per_inner_pack = models.PositiveIntegerField(null=True, blank=True)
|
||||
inner_pack_amount = models.DecimalField(max_digits=10, decimal_places=3, null=True, blank=True, validators=POSITIVE)
|
||||
inner_pack_label = models.CharField(max_length=40, blank=True)
|
||||
piece_label = models.CharField(max_length=40, blank=True)
|
||||
product_name = models.CharField(max_length=255, blank=True)
|
||||
brand = models.CharField(max_length=255, blank=True)
|
||||
category_name = models.CharField(max_length=100, blank=True)
|
||||
notes = models.TextField(blank=True)
|
||||
best_before = models.DateField(null=True, blank=True)
|
||||
opened = models.BooleanField(default=False)
|
||||
source_data = models.JSONField(default=dict, blank=True)
|
||||
user_data = models.JSONField(default=dict, blank=True)
|
||||
product = models.ForeignKey(FoodProduct, on_delete=models.SET_NULL, null=True, blank=True,
|
||||
related_name='scan_items')
|
||||
inventory_item = models.ForeignKey(KitchenInventoryItem, on_delete=models.SET_NULL, null=True, blank=True,
|
||||
related_name='scan_items')
|
||||
attempts = models.PositiveSmallIntegerField(default=0)
|
||||
next_attempt_at = models.DateTimeField(null=True, blank=True, db_index=True)
|
||||
lease_until = models.DateTimeField(null=True, blank=True, db_index=True)
|
||||
last_error = models.CharField(max_length=255, blank=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
confirmed_at = models.DateTimeField(null=True, blank=True)
|
||||
completed_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ('created_at', 'pk')
|
||||
indexes = [models.Index(fields=('status', 'next_attempt_at'), name='food_scan_item_due')]
|
||||
|
||||
|
||||
class OpenFoodFactsRequest(models.Model):
|
||||
"""A durable timestamp for each actual OFF HTTP request; shared by web and worker processes."""
|
||||
requested_at = models.DateTimeField(db_index=True)
|
||||
endpoint = models.CharField(max_length=24, blank=True)
|
||||
|
||||
|
||||
class OpenFoodFactsRateLimit(models.Model):
|
||||
"""A single database lock row coordinates request reservations across processes."""
|
||||
id = models.PositiveSmallIntegerField(primary_key=True, default=1, editable=False)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
|
||||
class InventoryReviewPreference(models.Model):
|
||||
class Mode(models.TextChoices):
|
||||
INDIVIDUAL = 'individual', 'Individual'
|
||||
BULK = 'bulk', 'Bulk'
|
||||
|
||||
user = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE,
|
||||
related_name='inventory_review_preference')
|
||||
mode = models.CharField(max_length=12, choices=Mode.choices, default=Mode.INDIVIDUAL)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
@@ -0,0 +1,520 @@
|
||||
"""Persistent scan-in session and enrichment queue services."""
|
||||
from datetime import timedelta
|
||||
from decimal import Decimal, InvalidOperation
|
||||
import random
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import IntegrityError, connections, transaction
|
||||
from django.db.models import F, Q
|
||||
from django.utils import timezone
|
||||
|
||||
from food.models import (FoodProduct, InventoryCategory, InventoryLocation,
|
||||
InventoryReviewPreference, InventoryScanBatch,
|
||||
InventoryScanItem, InventoryScanRun,
|
||||
KitchenInventoryItem)
|
||||
from food.services.openfoodfacts import (OpenFoodFactsError,
|
||||
OpenFoodFactsTransientError,
|
||||
cached_product)
|
||||
|
||||
|
||||
class ScanWorkflowError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def get_or_create_active_run(user, location_id):
|
||||
location = InventoryLocation.objects.filter(user=user, pk=location_id).first()
|
||||
if not location:
|
||||
raise ScanWorkflowError('Choose one of your existing inventory locations.')
|
||||
try:
|
||||
with transaction.atomic():
|
||||
run = (InventoryScanRun.objects.select_for_update().filter(
|
||||
user=user, status=InventoryScanRun.Status.ACTIVE).order_by('-created_at').first())
|
||||
if run:
|
||||
draft = run.batches.filter(status=InventoryScanBatch.Status.DRAFT).first()
|
||||
if not draft:
|
||||
raise ScanWorkflowError('This scan session is being saved. Refresh and try again.')
|
||||
run.selected_location = draft.location
|
||||
return run
|
||||
run = InventoryScanRun.objects.create(user=user, selected_location=location)
|
||||
InventoryScanBatch.objects.create(run=run, location=location, sequence=1)
|
||||
return run
|
||||
except IntegrityError:
|
||||
run = resume_active_run(user)
|
||||
if run:
|
||||
return run
|
||||
raise
|
||||
|
||||
|
||||
def resume_active_run(user):
|
||||
return (InventoryScanRun.objects.filter(user=user, status=InventoryScanRun.Status.ACTIVE)
|
||||
.prefetch_related('batches__items').order_by('-created_at').first())
|
||||
|
||||
|
||||
def create_active_scan(run, user, barcode):
|
||||
if not barcode.isascii() or not barcode.isdigit() or not 6 <= len(barcode) <= 64:
|
||||
raise ScanWorkflowError('Enter a valid numeric barcode.')
|
||||
with transaction.atomic():
|
||||
locked = InventoryScanRun.objects.select_for_update().filter(
|
||||
pk=run.pk, user=user, status=InventoryScanRun.Status.ACTIVE).first()
|
||||
if not locked:
|
||||
raise ScanWorkflowError('This scan session is no longer active.')
|
||||
batch = locked.batches.select_for_update().filter(status=InventoryScanBatch.Status.DRAFT).first()
|
||||
if not batch:
|
||||
raise ScanWorkflowError('Choose a location before scanning.')
|
||||
if batch.items.filter(status=InventoryScanItem.Status.ACTIVE).exists():
|
||||
raise ScanWorkflowError('Confirm or remove the active scan before scanning another item.')
|
||||
product = FoodProduct.objects.filter(owner=user, barcode=barcode).first()
|
||||
values = {}
|
||||
if product:
|
||||
values = {
|
||||
'product_name': product.product_name, 'brand': product.brand,
|
||||
'package_amount': product.package_amount, 'package_unit': product.package_unit,
|
||||
'source_data': {'cached': True, 'source': product.source,
|
||||
'nutrition': _product_nutrients(product)},
|
||||
}
|
||||
return InventoryScanItem.objects.create(batch=batch, location=batch.location,
|
||||
barcode=barcode, **values)
|
||||
|
||||
|
||||
def confirm_scan_item(user, item_id, values):
|
||||
with transaction.atomic():
|
||||
item = (InventoryScanItem.objects.select_for_update().select_related('batch__run')
|
||||
.filter(pk=item_id, batch__run__user=user).first())
|
||||
if not item or item.status != InventoryScanItem.Status.ACTIVE or item.batch.status != InventoryScanBatch.Status.DRAFT:
|
||||
raise ScanWorkflowError('This scan can no longer be edited.')
|
||||
location = InventoryLocation.objects.filter(user=user, pk=values.get('location_id')).first()
|
||||
if not location:
|
||||
raise ScanWorkflowError('Choose one of your existing inventory locations.')
|
||||
item.location = location
|
||||
item.product_name = str(values.get('product_name', item.product_name)).strip()[:255]
|
||||
item.brand = str(values.get('brand', item.brand)).strip()[:255]
|
||||
item.category_name = str(values.get('category_name', '')).strip()[:100]
|
||||
item.notes = str(values.get('notes', '')).strip()[:4000]
|
||||
item.package_count = _integer(values.get('package_count', 1), 'Enter a valid number of packages.', minimum=1, maximum=10000)
|
||||
item.package_amount = _decimal_or_none(values.get('package_amount'), 'Enter a valid package size.')
|
||||
item.package_unit = str(values.get('package_unit', '')).strip()[:8]
|
||||
item.pieces_per_package = _integer_or_none(values.get('pieces_per_package'), 'Enter a valid number of pieces.', maximum=100000)
|
||||
item.piece_amount = _decimal_or_none(values.get('piece_amount'), 'Enter a valid amount per piece.')
|
||||
item.inner_packs_per_package = _integer_or_none(values.get('inner_packs_per_package'), 'Enter a valid number of inner packs.', maximum=100000)
|
||||
item.pieces_per_inner_pack = _integer_or_none(values.get('pieces_per_inner_pack'), 'Enter a valid number of pieces per inner pack.', maximum=100000)
|
||||
item.inner_pack_amount = _decimal_or_none(values.get('inner_pack_amount'), 'Enter a valid inner pack amount.')
|
||||
item.inner_pack_label = str(values.get('inner_pack_label', '')).strip()[:40]
|
||||
item.piece_label = str(values.get('piece_label', '')).strip()[:40]
|
||||
item.best_before = values.get('best_before') or None
|
||||
item.opened = bool(values.get('opened'))
|
||||
item.user_data = _item_data(item)
|
||||
item.status = InventoryScanItem.Status.CONFIRMED
|
||||
item.confirmed_at = timezone.now()
|
||||
item.save()
|
||||
item.batch.run.selected_location = location
|
||||
item.batch.run.save(update_fields=('selected_location',))
|
||||
return item
|
||||
|
||||
|
||||
def discard_active_scan(user, item_id):
|
||||
deleted, _ = InventoryScanItem.objects.filter(pk=item_id, batch__run__user=user,
|
||||
status=InventoryScanItem.Status.ACTIVE, batch__status=InventoryScanBatch.Status.DRAFT).delete()
|
||||
if not deleted:
|
||||
raise ScanWorkflowError('Only an unconfirmed active scan can be removed.')
|
||||
|
||||
|
||||
def adjust_confirmed_count(user, item_id, delta):
|
||||
if delta not in (-1, 1):
|
||||
raise ScanWorkflowError('Invalid package adjustment.')
|
||||
with transaction.atomic():
|
||||
item = InventoryScanItem.objects.select_for_update().filter(pk=item_id, batch__run__user=user,
|
||||
batch__status=InventoryScanBatch.Status.DRAFT, status=InventoryScanItem.Status.CONFIRMED).first()
|
||||
if not item:
|
||||
raise ScanWorkflowError('This queued item can no longer be changed.')
|
||||
if item.package_count + delta < 1:
|
||||
item.delete()
|
||||
return None
|
||||
item.package_count += delta
|
||||
item.user_data = _item_data(item)
|
||||
item.save(update_fields=('package_count', 'user_data'))
|
||||
return item
|
||||
|
||||
|
||||
def submit_batch(user, *, batch_id, next_location_id=None, finish=False):
|
||||
with transaction.atomic():
|
||||
batch = (InventoryScanBatch.objects.select_for_update().select_related('run')
|
||||
.filter(pk=batch_id, run__user=user).first())
|
||||
if not batch:
|
||||
raise ScanWorkflowError('Scan batch not found.')
|
||||
run = InventoryScanRun.objects.select_for_update().get(pk=batch.run_id)
|
||||
if batch.status == InventoryScanBatch.Status.QUEUED:
|
||||
return run
|
||||
if run.status != InventoryScanRun.Status.ACTIVE or batch.status != InventoryScanBatch.Status.DRAFT:
|
||||
raise ScanWorkflowError('This scan batch has already been submitted.')
|
||||
if batch.items.filter(status=InventoryScanItem.Status.ACTIVE).exists():
|
||||
raise ScanWorkflowError('Confirm or remove the active item before saving this batch.')
|
||||
items = batch.items.filter(status=InventoryScanItem.Status.CONFIRMED)
|
||||
if not items.exists():
|
||||
raise ScanWorkflowError('Confirm at least one item before saving this batch.')
|
||||
now = timezone.now()
|
||||
items.update(status=InventoryScanItem.Status.QUEUED, next_attempt_at=now)
|
||||
batch.status = InventoryScanBatch.Status.QUEUED
|
||||
batch.submitted_at = now
|
||||
batch.save(update_fields=('status', 'submitted_at'))
|
||||
if finish:
|
||||
run.status = InventoryScanRun.Status.PROCESSING
|
||||
run.submitted_at = now
|
||||
run.save(update_fields=('status', 'submitted_at'))
|
||||
else:
|
||||
location = InventoryLocation.objects.filter(user=user, pk=next_location_id).first()
|
||||
if not location:
|
||||
raise ScanWorkflowError('Choose an existing location for the next batch.')
|
||||
run.selected_location = location
|
||||
next_sequence = (run.batches.order_by('-sequence').values_list('sequence', flat=True).first() or 0) + 1
|
||||
InventoryScanBatch.objects.create(run=run, location=location, sequence=next_sequence)
|
||||
run.save(update_fields=('selected_location',))
|
||||
return run
|
||||
|
||||
|
||||
def _claim_next_item():
|
||||
now = timezone.now()
|
||||
with transaction.atomic():
|
||||
skip_locked = connections['default'].features.has_select_for_update_skip_locked
|
||||
qs = InventoryScanItem.objects.select_for_update(skip_locked=skip_locked).filter(
|
||||
Q(status=InventoryScanItem.Status.QUEUED, next_attempt_at__lte=now) |
|
||||
Q(status=InventoryScanItem.Status.PROCESSING, lease_until__lt=now),
|
||||
batch__status=InventoryScanBatch.Status.QUEUED,
|
||||
batch__run__status__in=(InventoryScanRun.Status.ACTIVE, InventoryScanRun.Status.PROCESSING))
|
||||
item = qs.select_related('batch__run__user').order_by('next_attempt_at', 'created_at', 'pk').first()
|
||||
if not item:
|
||||
return None
|
||||
item.status = InventoryScanItem.Status.PROCESSING
|
||||
item.lease_until = now + timedelta(minutes=15)
|
||||
item.last_error = ''
|
||||
item.save(update_fields=('status', 'lease_until', 'last_error'))
|
||||
return item.pk
|
||||
|
||||
|
||||
def process_next_scan_item():
|
||||
item_id = _claim_next_item()
|
||||
if not item_id:
|
||||
return False
|
||||
item = InventoryScanItem.objects.select_related('batch__run__user', 'location').get(pk=item_id)
|
||||
product = FoodProduct.objects.filter(owner=item.batch.run.user, barcode=item.barcode).first()
|
||||
try:
|
||||
if not product:
|
||||
def record_http_attempt():
|
||||
InventoryScanItem.objects.filter(pk=item_id, status=InventoryScanItem.Status.PROCESSING).update(
|
||||
attempts=F('attempts') + 1)
|
||||
product = cached_product(item.batch.run.user, item.barcode, wait_for_slot=True,
|
||||
on_request_start=record_http_attempt)
|
||||
except OpenFoodFactsTransientError as error:
|
||||
_retry_or_review(item_id, str(error), retry_after=error.retry_after)
|
||||
_refresh_completion(item.batch.run_id)
|
||||
return True
|
||||
except OpenFoodFactsError as error:
|
||||
_mark_needs_review(item_id, str(error))
|
||||
_refresh_completion(item.batch.run_id)
|
||||
return True
|
||||
if not product:
|
||||
_mark_needs_review(item_id, 'No product was found for this barcode. Enter product details to approve it.')
|
||||
_refresh_completion(item.batch.run_id)
|
||||
return True
|
||||
with transaction.atomic():
|
||||
current = InventoryScanItem.objects.select_for_update().select_related('batch__run').get(pk=item_id)
|
||||
if current.status != InventoryScanItem.Status.PROCESSING:
|
||||
return True
|
||||
_populate_from_product(current, product)
|
||||
current.product = product
|
||||
if not current.product_name.strip() or not current.location_id or not _valid_stock_input(current):
|
||||
current.status = InventoryScanItem.Status.NEEDS_REVIEW
|
||||
current.last_error = 'Product identity or package details need review.'
|
||||
current.completed_at = timezone.now()
|
||||
current.lease_until = None
|
||||
current.save()
|
||||
else:
|
||||
try:
|
||||
stock = _create_inventory_stock(current, product)
|
||||
except (ValidationError, ScanWorkflowError) as error:
|
||||
current.status = InventoryScanItem.Status.NEEDS_REVIEW
|
||||
current.last_error = str(error)[:255]
|
||||
current.completed_at = timezone.now()
|
||||
current.lease_until = None
|
||||
current.save()
|
||||
else:
|
||||
current.status = InventoryScanItem.Status.COMPLETE
|
||||
current.inventory_item = stock
|
||||
current.completed_at = timezone.now()
|
||||
current.lease_until = None
|
||||
current.last_error = ''
|
||||
current.save()
|
||||
_refresh_completion(item.batch.run_id)
|
||||
return True
|
||||
|
||||
|
||||
def _retry_or_review(item_id, message, *, retry_after=None):
|
||||
with transaction.atomic():
|
||||
item = InventoryScanItem.objects.select_for_update().get(pk=item_id)
|
||||
item.lease_until = None
|
||||
item.last_error = message[:255]
|
||||
item.refresh_from_db(fields=('attempts',))
|
||||
if item.attempts >= 10:
|
||||
item.status = InventoryScanItem.Status.NEEDS_REVIEW
|
||||
item.completed_at = timezone.now()
|
||||
else:
|
||||
backoff = min(900, 5 * (2 ** max(0, item.attempts - 1)))
|
||||
delay = max(backoff + random.uniform(0, backoff * 0.25), retry_after or 0)
|
||||
item.status = InventoryScanItem.Status.QUEUED
|
||||
item.next_attempt_at = timezone.now() + timedelta(seconds=delay)
|
||||
item.save(update_fields=('status', 'lease_until', 'last_error', 'completed_at', 'next_attempt_at'))
|
||||
|
||||
|
||||
def _mark_needs_review(item_id, message):
|
||||
InventoryScanItem.objects.filter(pk=item_id, status=InventoryScanItem.Status.PROCESSING).update(
|
||||
status=InventoryScanItem.Status.NEEDS_REVIEW, last_error=message[:255], completed_at=timezone.now(), lease_until=None)
|
||||
|
||||
|
||||
def _populate_from_product(item, product):
|
||||
if not item.product_name:
|
||||
item.product_name = product.product_name
|
||||
if not item.brand:
|
||||
item.brand = product.brand
|
||||
if item.package_amount is None and product.package_amount is not None:
|
||||
item.package_amount = product.package_amount
|
||||
item.package_unit = product.package_unit
|
||||
item.source_data = {
|
||||
'source': product.source,
|
||||
'fetched_at': product.fetched_at.isoformat() if product.fetched_at else None,
|
||||
'product_name': product.product_name,
|
||||
'brand': product.brand,
|
||||
'package_quantity': product.package_quantity,
|
||||
'image_url': product.image_url,
|
||||
'nutrition': _product_nutrients(product),
|
||||
'raw_data': product.raw_data if isinstance(product.raw_data, dict) else {},
|
||||
}
|
||||
item.user_data = _item_data(item)
|
||||
|
||||
|
||||
def _create_inventory_stock(item, product):
|
||||
package_amount = item.package_amount
|
||||
package_unit = item.package_unit
|
||||
pieces = item.pieces_per_package
|
||||
if pieces and package_amount is None and not package_unit:
|
||||
package_amount, package_unit = Decimal(pieces), 'pieces'
|
||||
if pieces and item.inner_packs_per_package and item.pieces_per_inner_pack:
|
||||
if package_amount is None and item.inner_pack_amount is not None:
|
||||
package_amount = item.inner_pack_amount * item.inner_packs_per_package
|
||||
if package_unit and package_unit not in {'mg', 'g', 'kg', 'ml', 'L', 'pieces', 'packs', 'portions'}:
|
||||
raise ScanWorkflowError('Choose a supported package unit.')
|
||||
category = None
|
||||
if item.category_name:
|
||||
category, _ = InventoryCategory.objects.get_or_create(user=item.batch.run.user,
|
||||
name__iexact=item.category_name, defaults={'name': item.category_name})
|
||||
base = KitchenInventoryItem(
|
||||
user=item.batch.run.user, product=product, location=item.location,
|
||||
category=category, quantity=item.package_count if package_amount is None else 1,
|
||||
notes=item.notes, opened=item.opened, best_before=item.best_before,
|
||||
original_amount=package_amount, remaining_amount=package_amount,
|
||||
inventory_unit=package_unit, pieces_per_package=pieces,
|
||||
remaining_pieces=pieces, piece_amount=item.piece_amount,
|
||||
inner_packs_per_package=item.inner_packs_per_package,
|
||||
pieces_per_inner_pack=item.pieces_per_inner_pack,
|
||||
inner_pack_amount=item.inner_pack_amount, inner_pack_label=item.inner_pack_label,
|
||||
piece_label=item.piece_label,
|
||||
)
|
||||
base.full_clean()
|
||||
base.save()
|
||||
if package_amount is None:
|
||||
return base
|
||||
for _ in range(item.package_count - 1):
|
||||
package = KitchenInventoryItem(
|
||||
user=base.user, product=base.product, location=base.location,
|
||||
category=base.category, quantity=1, notes=base.notes, opened=base.opened,
|
||||
best_before=base.best_before, original_amount=base.original_amount,
|
||||
remaining_amount=base.original_amount, inventory_unit=base.inventory_unit,
|
||||
pieces_per_package=base.pieces_per_package, remaining_pieces=base.pieces_per_package,
|
||||
piece_amount=base.piece_amount, inner_packs_per_package=base.inner_packs_per_package,
|
||||
pieces_per_inner_pack=base.pieces_per_inner_pack, inner_pack_amount=base.inner_pack_amount,
|
||||
inner_pack_label=base.inner_pack_label, piece_label=base.piece_label,
|
||||
)
|
||||
package.full_clean()
|
||||
package.save()
|
||||
return base
|
||||
|
||||
|
||||
def _valid_stock_input(item):
|
||||
if item.package_amount is None and not item.pieces_per_package:
|
||||
return True # Count-only packages have unknown size by design.
|
||||
if item.package_amount is not None and item.package_unit:
|
||||
return True
|
||||
return bool(item.inner_packs_per_package and item.pieces_per_inner_pack and
|
||||
item.inner_pack_amount and item.package_unit)
|
||||
|
||||
|
||||
def _refresh_completion(run_id):
|
||||
with transaction.atomic():
|
||||
run = InventoryScanRun.objects.select_for_update().get(pk=run_id)
|
||||
for batch in run.batches.filter(status=InventoryScanBatch.Status.QUEUED):
|
||||
if not batch.items.filter(status__in=(InventoryScanItem.Status.QUEUED, InventoryScanItem.Status.PROCESSING)).exists():
|
||||
batch.status = InventoryScanBatch.Status.COMPLETE
|
||||
batch.completed_at = timezone.now()
|
||||
batch.save(update_fields=('status', 'completed_at'))
|
||||
if run.status == InventoryScanRun.Status.PROCESSING and not run.batches.filter(
|
||||
status=InventoryScanBatch.Status.QUEUED).exists():
|
||||
review = run.batches.filter(items__status__in=(InventoryScanItem.Status.NEEDS_REVIEW,
|
||||
InventoryScanItem.Status.DEFERRED,
|
||||
InventoryScanItem.Status.FAILED)).exists()
|
||||
run.status = (InventoryScanRun.Status.COMPLETE_WITH_REVIEW if review
|
||||
else InventoryScanRun.Status.COMPLETE)
|
||||
run.completed_at = timezone.now()
|
||||
run.save(update_fields=('status', 'completed_at'))
|
||||
if run.status in (InventoryScanRun.Status.COMPLETE, InventoryScanRun.Status.COMPLETE_WITH_REVIEW):
|
||||
_send_completion_notification(run_id)
|
||||
|
||||
|
||||
def _send_completion_notification(run_id):
|
||||
from planner.services.notifications import NotificationError, send_notification
|
||||
|
||||
now = timezone.now()
|
||||
with transaction.atomic():
|
||||
run = InventoryScanRun.objects.select_for_update().select_related('user').get(pk=run_id)
|
||||
if run.notification_attempted_at or run.status not in (
|
||||
InventoryScanRun.Status.COMPLETE, InventoryScanRun.Status.COMPLETE_WITH_REVIEW):
|
||||
return
|
||||
from django.db.models import Count
|
||||
tallies = dict(run.batches.values('items__status').annotate(total=Count('items')).values_list('items__status', 'total'))
|
||||
succeeded = tallies.get(InventoryScanItem.Status.COMPLETE, 0)
|
||||
review = sum(tallies.get(status, 0) for status in (InventoryScanItem.Status.NEEDS_REVIEW,
|
||||
InventoryScanItem.Status.DEFERRED,
|
||||
InventoryScanItem.Status.FAILED))
|
||||
failed = tallies.get(InventoryScanItem.Status.FAILED, 0)
|
||||
run.notification_attempted_at = now # durable at-most-once delivery guard for worker restarts
|
||||
run.save(update_fields=('notification_attempted_at',))
|
||||
user = run.user
|
||||
url = '/food/inventory/#inventory-review'
|
||||
message = f'Bulk scan finished: {succeeded} added, {review} need review, {failed} failed. {url}'
|
||||
try:
|
||||
sent = send_notification(user, 'Kitchen inventory scan complete', message)
|
||||
except NotificationError as error:
|
||||
InventoryScanRun.objects.filter(pk=run_id).update(notification_error=str(error)[:255])
|
||||
return
|
||||
if sent:
|
||||
InventoryScanRun.objects.filter(pk=run_id).update(notification_sent_at=timezone.now(), notification_error='')
|
||||
|
||||
|
||||
def update_review_preference(user, mode):
|
||||
if mode not in InventoryReviewPreference.Mode.values:
|
||||
raise ScanWorkflowError('Choose Individual or Bulk review.')
|
||||
preference, _ = InventoryReviewPreference.objects.update_or_create(user=user, defaults={'mode': mode})
|
||||
return preference
|
||||
|
||||
|
||||
def approve_scan_item(user, item_id, values):
|
||||
with transaction.atomic():
|
||||
item = (InventoryScanItem.objects.select_for_update().select_related('batch__run__user')
|
||||
.filter(pk=item_id, batch__run__user=user,
|
||||
status__in=(InventoryScanItem.Status.NEEDS_REVIEW, InventoryScanItem.Status.DEFERRED)).first())
|
||||
if not item:
|
||||
raise ScanWorkflowError('This staged item is no longer awaiting review.')
|
||||
if values.get('product_name'):
|
||||
item.product_name = str(values['product_name']).strip()[:255]
|
||||
if values.get('brand') is not None:
|
||||
item.brand = str(values['brand']).strip()[:255]
|
||||
if values.get('category_name') is not None:
|
||||
item.category_name = str(values['category_name']).strip()[:100]
|
||||
if values.get('notes') is not None:
|
||||
item.notes = str(values['notes']).strip()[:4000]
|
||||
if values.get('location_id'):
|
||||
item.location = InventoryLocation.objects.get(user=user, pk=values['location_id'])
|
||||
item.package_count = _integer(values.get('package_count', item.package_count), 'Enter a valid number of packages.', 1, 10000)
|
||||
if 'package_amount' in values:
|
||||
item.package_amount = _decimal_or_none(values['package_amount'], 'Enter a valid package size.')
|
||||
if 'package_unit' in values:
|
||||
item.package_unit = str(values['package_unit']).strip()[:8]
|
||||
if not item.product_name:
|
||||
raise ScanWorkflowError('Enter a product name before approving.')
|
||||
product = item.product or FoodProduct.objects.filter(owner=user, barcode=item.barcode).first()
|
||||
if not product:
|
||||
product = FoodProduct.objects.create(owner=user, barcode=item.barcode,
|
||||
product_name=item.product_name, brand=item.brand,
|
||||
package_quantity=(f'{item.package_amount} {item.package_unit}' if item.package_amount else ''),
|
||||
package_amount=item.package_amount, package_unit=item.package_unit,
|
||||
source=FoodProduct.Source.MANUAL, raw_data={'scan_item_id': item.pk, 'manual_nutrition_override': True})
|
||||
elif product.source == FoodProduct.Source.MANUAL:
|
||||
product.product_name = item.product_name
|
||||
product.brand = item.brand
|
||||
product.save(update_fields=('product_name', 'brand', 'updated_at'))
|
||||
else:
|
||||
_apply_scan_product_corrections(product, item)
|
||||
_populate_from_product(item, product)
|
||||
item.product = product
|
||||
if not _valid_stock_input(item):
|
||||
raise ScanWorkflowError('Add valid package details or choose count-only stock before approving.')
|
||||
stock = _create_inventory_stock(item, product)
|
||||
item.status = InventoryScanItem.Status.COMPLETE
|
||||
item.inventory_item = stock
|
||||
item.last_error = ''
|
||||
item.completed_at = timezone.now()
|
||||
item.save()
|
||||
_refresh_completion(item.batch.run_id)
|
||||
return item
|
||||
|
||||
|
||||
def defer_scan_item(user, item_id):
|
||||
updated = InventoryScanItem.objects.filter(pk=item_id, batch__run__user=user,
|
||||
status=InventoryScanItem.Status.NEEDS_REVIEW).update(status=InventoryScanItem.Status.DEFERRED)
|
||||
if not updated:
|
||||
raise ScanWorkflowError('This item can no longer be deferred.')
|
||||
|
||||
|
||||
def _apply_scan_product_corrections(product, item):
|
||||
# Metadata corrections are limited to product identity and packaging. Nutrition remains source-owned.
|
||||
if (product.source == FoodProduct.Source.OPEN_FOOD_FACTS and isinstance(product.raw_data, dict) and
|
||||
product.raw_data.get('manual_nutrition_override')):
|
||||
return
|
||||
product.product_name = item.product_name or product.product_name
|
||||
product.brand = item.brand or product.brand
|
||||
product.package_amount = item.package_amount or product.package_amount
|
||||
product.package_unit = item.package_unit or product.package_unit
|
||||
product.save(update_fields=('product_name', 'brand', 'package_amount', 'package_unit', 'updated_at'))
|
||||
|
||||
|
||||
def _product_nutrients(product):
|
||||
return {field: str(getattr(product, field)) if getattr(product, field) is not None else None
|
||||
for field in ('energy_kcal_100', 'fat_100', 'saturated_fat_100', 'carbohydrates_100',
|
||||
'sugars_100', 'protein_100', 'salt_100', 'fibre_100')}
|
||||
|
||||
|
||||
def _item_data(item):
|
||||
return {'product_name': item.product_name, 'brand': item.brand, 'category_name': item.category_name,
|
||||
'notes': item.notes, 'location_id': item.location_id, 'package_count': item.package_count,
|
||||
'package_amount': str(item.package_amount) if item.package_amount is not None else None,
|
||||
'package_unit': item.package_unit, 'pieces_per_package': item.pieces_per_package,
|
||||
'piece_amount': str(item.piece_amount) if item.piece_amount is not None else None,
|
||||
'inner_packs_per_package': item.inner_packs_per_package,
|
||||
'pieces_per_inner_pack': item.pieces_per_inner_pack,
|
||||
'inner_pack_amount': str(item.inner_pack_amount) if item.inner_pack_amount is not None else None,
|
||||
'best_before': item.best_before.isoformat() if item.best_before else None, 'opened': item.opened}
|
||||
|
||||
|
||||
def _integer(value, message, minimum=0, maximum=100000):
|
||||
try:
|
||||
result = int(value)
|
||||
except (TypeError, ValueError):
|
||||
raise ScanWorkflowError(message) from None
|
||||
if str(result) != str(value).strip() and not isinstance(value, int):
|
||||
raise ScanWorkflowError(message)
|
||||
if not minimum <= result <= maximum:
|
||||
raise ScanWorkflowError(message)
|
||||
return result
|
||||
|
||||
|
||||
def _integer_or_none(value, message, maximum=100000):
|
||||
return None if value in (None, '') else _integer(value, message, 1, maximum)
|
||||
|
||||
|
||||
def _decimal_or_none(value, message):
|
||||
if value in (None, ''):
|
||||
return None
|
||||
try:
|
||||
result = Decimal(str(value))
|
||||
except (InvalidOperation, TypeError, ValueError):
|
||||
raise ScanWorkflowError(message) from None
|
||||
if not result.is_finite() or result <= 0 or result > Decimal('9999999.999'):
|
||||
raise ScanWorkflowError(message)
|
||||
return result.quantize(Decimal('.001'))
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Database-coordinated cap for actual Open Food Facts HTTP requests."""
|
||||
from contextlib import contextmanager
|
||||
from datetime import timedelta
|
||||
import time
|
||||
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from food.models import OpenFoodFactsRateLimit, OpenFoodFactsRequest
|
||||
|
||||
|
||||
REQUESTS_PER_MINUTE = 5
|
||||
WINDOW_SECONDS = 60
|
||||
|
||||
|
||||
@contextmanager
|
||||
def off_http_request(endpoint, *, wait=True, before_request=None):
|
||||
"""Reserve and serialize one real request across web and worker processes.
|
||||
|
||||
The singleton row lock is held through the HTTP call, so simultaneous workers
|
||||
cannot race through the budget check. Failed HTTP requests still count because
|
||||
the reservation commits before the exception is re-raised.
|
||||
"""
|
||||
while True:
|
||||
delay = 0
|
||||
failure = None
|
||||
traceback = None
|
||||
with transaction.atomic():
|
||||
OpenFoodFactsRateLimit.objects.get_or_create(pk=1)
|
||||
OpenFoodFactsRateLimit.objects.select_for_update().get(pk=1)
|
||||
now = timezone.now()
|
||||
cutoff = now - timedelta(seconds=WINDOW_SECONDS)
|
||||
oldest = OpenFoodFactsRequest.objects.filter(requested_at__gte=cutoff).order_by('requested_at').first()
|
||||
if OpenFoodFactsRequest.objects.filter(requested_at__gte=cutoff).count() >= REQUESTS_PER_MINUTE:
|
||||
delay = max(0.05, (oldest.requested_at + timedelta(seconds=WINDOW_SECONDS + 0.01) - now).total_seconds())
|
||||
else:
|
||||
OpenFoodFactsRequest.objects.create(requested_at=now, endpoint=endpoint[:24])
|
||||
OpenFoodFactsRequest.objects.filter(requested_at__lt=now - timedelta(minutes=10)).delete()
|
||||
try:
|
||||
if before_request:
|
||||
before_request()
|
||||
yield
|
||||
except BaseException as error:
|
||||
failure = error
|
||||
traceback = error.__traceback__
|
||||
if delay:
|
||||
if not wait:
|
||||
raise OpenFoodFactsRateLimited(delay)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if failure is not None:
|
||||
raise failure.with_traceback(traceback)
|
||||
return
|
||||
|
||||
|
||||
class OpenFoodFactsRateLimited(Exception):
|
||||
def __init__(self, retry_after):
|
||||
self.retry_after = max(1, int(retry_after + 0.999))
|
||||
super().__init__(f'Open Food Facts request budget is full; retry in {self.retry_after} seconds.')
|
||||
@@ -2,6 +2,7 @@
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from email.utils import parsedate_to_datetime
|
||||
from hashlib import sha256
|
||||
from datetime import timedelta
|
||||
from decimal import Decimal, InvalidOperation
|
||||
@@ -15,6 +16,7 @@ from django.core.cache import cache
|
||||
from django.utils import timezone
|
||||
|
||||
from food.models import FoodProduct, Unit
|
||||
from .off_rate_limit import OpenFoodFactsRateLimited, off_http_request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
FIELDS = 'code,product_name,brands,quantity,serving_size,serving_quantity,image_front_small_url,image_front_url,nutriments,categories,categories_tags'
|
||||
@@ -35,7 +37,16 @@ class OpenFoodFactsError(Exception):
|
||||
|
||||
|
||||
class OpenFoodFactsRateLimit(OpenFoodFactsError):
|
||||
pass
|
||||
def __init__(self, message, retry_after=None):
|
||||
self.retry_after = retry_after
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class OpenFoodFactsTransientError(OpenFoodFactsError):
|
||||
def __init__(self, message, *, retry_after=None, status_code=None):
|
||||
self.retry_after = retry_after
|
||||
self.status_code = status_code
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
def normalise_query(query):
|
||||
@@ -108,41 +119,59 @@ def _rate_limit(user_id, kind, limit):
|
||||
|
||||
|
||||
class OpenFoodFactsClient:
|
||||
def _get_json(self, url, params=None):
|
||||
def _get_json(self, url, params=None, *, wait_for_slot=False, on_request_start=None):
|
||||
if not settings.OPENFOODFACTS_ENABLED:
|
||||
raise OpenFoodFactsError('Open Food Facts is unavailable.')
|
||||
if params:
|
||||
url = f'{url}?{urlencode(params)}'
|
||||
request = Request(url, headers={'User-Agent': settings.OPENFOODFACTS_USER_AGENT, 'Accept': 'application/json'})
|
||||
endpoint = 'product' if '/product/' in url else 'search'
|
||||
try:
|
||||
with urlopen(request, timeout=settings.OPENFOODFACTS_TIMEOUT_SECONDS) as response:
|
||||
data = json.load(response)
|
||||
except (HTTPError, URLError, TimeoutError, ValueError, OSError) as error:
|
||||
logger.warning('Open Food Facts request failed: %s', error)
|
||||
raise OpenFoodFactsError('Open Food Facts is temporarily unavailable.') from error
|
||||
with off_http_request(endpoint, wait=wait_for_slot, before_request=on_request_start):
|
||||
try:
|
||||
with urlopen(request, timeout=settings.OPENFOODFACTS_TIMEOUT_SECONDS) as response:
|
||||
data = json.load(response)
|
||||
except HTTPError as error:
|
||||
retry_after = _retry_after(error.headers.get('Retry-After') if error.headers else None)
|
||||
logger.warning('Open Food Facts HTTP %s on %s request', error.code, endpoint)
|
||||
if error.code == 429 or error.code >= 500:
|
||||
raise OpenFoodFactsTransientError(
|
||||
'Open Food Facts is temporarily unavailable.', retry_after=retry_after,
|
||||
status_code=error.code) from error
|
||||
raise OpenFoodFactsError(f'Open Food Facts rejected the request (HTTP {error.code}).') from error
|
||||
except (URLError, TimeoutError, OSError) as error:
|
||||
logger.warning('Open Food Facts request failed: %s', error)
|
||||
raise OpenFoodFactsTransientError('Open Food Facts is temporarily unavailable.') from error
|
||||
except ValueError as error:
|
||||
logger.warning('Open Food Facts returned invalid JSON: %s', error)
|
||||
raise OpenFoodFactsError('Open Food Facts returned an invalid response.') from error
|
||||
except OpenFoodFactsRateLimited as error:
|
||||
raise OpenFoodFactsRateLimit(str(error), retry_after=error.retry_after) from error
|
||||
if not isinstance(data, dict):
|
||||
raise OpenFoodFactsError('Open Food Facts returned an invalid response.')
|
||||
return data
|
||||
|
||||
def get_product(self, barcode):
|
||||
def get_product(self, barcode, *, wait_for_slot=False, on_request_start=None):
|
||||
if not re.fullmatch(r'[0-9]{1,64}', barcode):
|
||||
raise ValueError('Enter a numeric barcode.')
|
||||
data = self._get_json(settings.OPENFOODFACTS_PRODUCT_URL.format(barcode=barcode), {'fields': FIELDS})
|
||||
data = self._get_json(settings.OPENFOODFACTS_PRODUCT_URL.format(barcode=barcode), {'fields': FIELDS},
|
||||
wait_for_slot=wait_for_slot, on_request_start=on_request_start)
|
||||
return data.get('product') if data.get('status') == 1 else None
|
||||
|
||||
def search(self, query, page=1):
|
||||
def search(self, query, page=1, *, wait_for_slot=False):
|
||||
query = normalise_query(query)
|
||||
if not query or len(query) > 100 or page < 1 or page > 20:
|
||||
raise ValueError('Enter a search query and valid page.')
|
||||
data = self._get_json(settings.OPENFOODFACTS_SEARCH_URL, {'action': 'process', 'search_simple': 1,
|
||||
'json': 1, 'search_terms': query, 'page': page, 'page_size': 20, 'fields': FIELDS})
|
||||
'json': 1, 'search_terms': query, 'page': page, 'page_size': 20, 'fields': FIELDS},
|
||||
wait_for_slot=wait_for_slot)
|
||||
products = data.get('products')
|
||||
if not isinstance(products, list):
|
||||
raise OpenFoodFactsError('Open Food Facts returned an invalid response.')
|
||||
return [mapped for item in products if (mapped := map_product(item))]
|
||||
|
||||
|
||||
def cached_product(user, barcode, *, refresh=False):
|
||||
def cached_product(user, barcode, *, refresh=False, wait_for_slot=False, on_request_start=None):
|
||||
if not re.fullmatch(r'[0-9]{1,64}', barcode):
|
||||
raise ValueError('Enter a numeric barcode.')
|
||||
product = FoodProduct.objects.filter(owner=user, barcode=barcode).first()
|
||||
@@ -154,7 +183,8 @@ def cached_product(user, barcode, *, refresh=False):
|
||||
return product
|
||||
_rate_limit(user.pk, 'product', 10)
|
||||
try:
|
||||
data = OpenFoodFactsClient().get_product(barcode)
|
||||
data = OpenFoodFactsClient().get_product(barcode, wait_for_slot=wait_for_slot,
|
||||
on_request_start=on_request_start)
|
||||
except OpenFoodFactsError:
|
||||
if product:
|
||||
return product
|
||||
@@ -173,6 +203,21 @@ def cached_product(user, barcode, *, refresh=False):
|
||||
return product
|
||||
|
||||
|
||||
def _retry_after(value):
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return max(0, int(value.strip()))
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
try:
|
||||
requested_at = parsedate_to_datetime(value)
|
||||
if requested_at.tzinfo is None:
|
||||
requested_at = requested_at.replace(tzinfo=timezone.get_current_timezone())
|
||||
return max(0, int((requested_at - timezone.now()).total_seconds()))
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def cached_search(user, query, page=1):
|
||||
normalized = normalise_query(query)
|
||||
if not normalized or len(normalized) > 100 or page < 1 or page > 20:
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
from datetime import timedelta
|
||||
from decimal import Decimal
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
from food.models import (FoodProduct, InventoryLocation, InventoryScanBatch,
|
||||
InventoryScanItem, InventoryScanRun,
|
||||
KitchenInventoryItem, OpenFoodFactsRequest)
|
||||
from food.services.inventory_scanner import (ScanWorkflowError,
|
||||
confirm_scan_item, create_active_scan, get_or_create_active_run,
|
||||
process_next_scan_item, submit_batch)
|
||||
from food.services.off_rate_limit import OpenFoodFactsRateLimited, off_http_request
|
||||
from food.services.openfoodfacts import OpenFoodFactsTransientError
|
||||
|
||||
|
||||
class InventoryScanQueueTests(TestCase):
|
||||
def setUp(self):
|
||||
self.user = get_user_model().objects.create_user(username='scanner-owner', password='test-password')
|
||||
self.other = get_user_model().objects.create_user(username='scanner-other', password='test-password')
|
||||
self.fridge = InventoryLocation.objects.create(user=self.user, name='Fridge')
|
||||
self.shelf = InventoryLocation.objects.create(user=self.user, name='Shelf')
|
||||
self.product = FoodProduct.objects.create(owner=self.user, barcode='12345678',
|
||||
product_name='Oat drink', package_amount=Decimal('1000'), package_unit='ml')
|
||||
|
||||
def _confirmed_item(self, location=None, barcode='12345678'):
|
||||
run = get_or_create_active_run(self.user, (location or self.fridge).pk)
|
||||
item = create_active_scan(run, self.user, barcode)
|
||||
confirm_scan_item(self.user, item.pk, {
|
||||
'location_id': (location or self.fridge).pk, 'product_name': 'Oat drink',
|
||||
'package_count': 2, 'package_amount': '1000', 'package_unit': 'ml',
|
||||
})
|
||||
return run, item
|
||||
|
||||
def test_location_batches_are_atomic_and_keep_original_locations(self):
|
||||
run, item = self._confirmed_item()
|
||||
batch = item.batch
|
||||
updated = submit_batch(self.user, batch_id=batch.pk, next_location_id=self.shelf.pk)
|
||||
batch.refresh_from_db()
|
||||
new_batch = updated.batches.get(status=InventoryScanBatch.Status.DRAFT)
|
||||
self.assertEqual((batch.status, batch.location), (InventoryScanBatch.Status.QUEUED, self.fridge))
|
||||
self.assertEqual(new_batch.location, self.shelf)
|
||||
self.assertEqual(item.location, self.fridge)
|
||||
item.refresh_from_db()
|
||||
self.assertEqual(item.status, InventoryScanItem.Status.QUEUED)
|
||||
|
||||
def test_batch_rejects_other_users_location(self):
|
||||
run, item = self._confirmed_item()
|
||||
foreign = InventoryLocation.objects.create(user=self.other, name='Other')
|
||||
with self.assertRaises(ScanWorkflowError):
|
||||
submit_batch(self.user, batch_id=item.batch_id, next_location_id=foreign.pk)
|
||||
item.refresh_from_db()
|
||||
self.assertEqual(item.status, InventoryScanItem.Status.CONFIRMED)
|
||||
|
||||
def test_active_run_resumes_and_rejects_second_active_scan(self):
|
||||
run = get_or_create_active_run(self.user, self.fridge.pk)
|
||||
item = create_active_scan(run, self.user, '12345678')
|
||||
self.assertEqual(get_or_create_active_run(self.user, self.shelf.pk).pk, run.pk)
|
||||
with self.assertRaises(ScanWorkflowError):
|
||||
create_active_scan(run, self.user, '87654321')
|
||||
|
||||
def test_done_enqueues_and_worker_creates_separate_tracked_packages(self):
|
||||
run, item = self._confirmed_item()
|
||||
submit_batch(self.user, batch_id=item.batch_id, finish=True)
|
||||
with patch('food.services.inventory_scanner._send_completion_notification') as notify:
|
||||
self.assertTrue(process_next_scan_item())
|
||||
item.refresh_from_db()
|
||||
run.refresh_from_db()
|
||||
self.assertEqual(item.status, InventoryScanItem.Status.COMPLETE)
|
||||
self.assertEqual(item.inventory_item.product, self.product)
|
||||
stock = KitchenInventoryItem.objects.filter(user=self.user, product=self.product)
|
||||
self.assertEqual(stock.count(), 2)
|
||||
self.assertTrue(all(row.original_amount == Decimal('1000') and row.inventory_unit == 'ml' for row in stock))
|
||||
self.assertEqual(run.status, InventoryScanRun.Status.COMPLETE)
|
||||
notify.assert_called_once_with(run.pk)
|
||||
|
||||
def test_transient_error_is_scheduled_and_bounded_by_ten_attempts(self):
|
||||
run, item = self._confirmed_item(barcode='87654321')
|
||||
submit_batch(self.user, batch_id=item.batch_id, finish=True)
|
||||
def transient_request(*args, **kwargs):
|
||||
kwargs['on_request_start']()
|
||||
raise OpenFoodFactsTransientError('temporary failure', retry_after=30)
|
||||
with patch('food.services.inventory_scanner.cached_product', side_effect=transient_request):
|
||||
process_next_scan_item()
|
||||
item.refresh_from_db()
|
||||
self.assertEqual(item.attempts, 1)
|
||||
self.assertEqual(item.status, InventoryScanItem.Status.QUEUED)
|
||||
self.assertGreaterEqual(item.next_attempt_at, timezone.now() + timedelta(seconds=29))
|
||||
|
||||
def test_unknown_product_moves_to_review_without_losing_user_fields(self):
|
||||
run, item = self._confirmed_item(barcode='87654321')
|
||||
submit_batch(self.user, batch_id=item.batch_id, finish=True)
|
||||
with patch('food.services.inventory_scanner.cached_product', return_value=None):
|
||||
process_next_scan_item()
|
||||
item.refresh_from_db()
|
||||
self.assertEqual(item.status, InventoryScanItem.Status.NEEDS_REVIEW)
|
||||
self.assertEqual(item.product_name, 'Oat drink')
|
||||
self.assertEqual(item.location, self.fridge)
|
||||
self.assertIsNone(item.inventory_item)
|
||||
|
||||
|
||||
class OpenFoodFactsGlobalLimitTests(TestCase):
|
||||
def test_non_waiting_request_is_refused_when_shared_budget_is_full(self):
|
||||
now = timezone.now()
|
||||
OpenFoodFactsRequest.objects.bulk_create([
|
||||
OpenFoodFactsRequest(requested_at=now - timedelta(seconds=10 * index), endpoint='product')
|
||||
for index in range(5)
|
||||
])
|
||||
with self.assertRaises(OpenFoodFactsRateLimited):
|
||||
with off_http_request('search', wait=False):
|
||||
self.fail('The sixth actual request must not start.')
|
||||
self.assertEqual(OpenFoodFactsRequest.objects.count(), 5)
|
||||
|
||||
def test_failed_actual_request_still_consumes_budget(self):
|
||||
with self.assertRaises(RuntimeError):
|
||||
with off_http_request('product', wait=False):
|
||||
raise RuntimeError('network error after request start')
|
||||
self.assertEqual(OpenFoodFactsRequest.objects.count(), 1)
|
||||
Reference in New Issue
Block a user