60 lines
2.4 KiB
Python
60 lines
2.4 KiB
Python
"""Database-coordinated cap for actual Open Food Facts HTTP requests."""
|
|
from contextlib import contextmanager
|
|
from datetime import timedelta
|
|
import time
|
|
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
|
|
from food.models import OpenFoodFactsRateLimit, OpenFoodFactsRequest
|
|
|
|
|
|
REQUESTS_PER_MINUTE = 5
|
|
WINDOW_SECONDS = 60
|
|
|
|
|
|
@contextmanager
|
|
def off_http_request(endpoint, *, wait=True, before_request=None):
|
|
"""Reserve and serialize one real request across web and worker processes.
|
|
|
|
The singleton row lock is held through the HTTP call, so simultaneous workers
|
|
cannot race through the budget check. Failed HTTP requests still count because
|
|
the reservation commits before the exception is re-raised.
|
|
"""
|
|
while True:
|
|
delay = 0
|
|
failure = None
|
|
traceback = None
|
|
with transaction.atomic():
|
|
OpenFoodFactsRateLimit.objects.get_or_create(pk=1)
|
|
OpenFoodFactsRateLimit.objects.select_for_update().get(pk=1)
|
|
now = timezone.now()
|
|
cutoff = now - timedelta(seconds=WINDOW_SECONDS)
|
|
oldest = OpenFoodFactsRequest.objects.filter(requested_at__gte=cutoff).order_by('requested_at').first()
|
|
if OpenFoodFactsRequest.objects.filter(requested_at__gte=cutoff).count() >= REQUESTS_PER_MINUTE:
|
|
delay = max(0.05, (oldest.requested_at + timedelta(seconds=WINDOW_SECONDS + 0.01) - now).total_seconds())
|
|
else:
|
|
OpenFoodFactsRequest.objects.create(requested_at=now, endpoint=endpoint[:24])
|
|
OpenFoodFactsRequest.objects.filter(requested_at__lt=now - timedelta(minutes=10)).delete()
|
|
try:
|
|
if before_request:
|
|
before_request()
|
|
yield
|
|
except BaseException as error:
|
|
failure = error
|
|
traceback = error.__traceback__
|
|
if delay:
|
|
if not wait:
|
|
raise OpenFoodFactsRateLimited(delay)
|
|
time.sleep(delay)
|
|
continue
|
|
if failure is not None:
|
|
raise failure.with_traceback(traceback)
|
|
return
|
|
|
|
|
|
class OpenFoodFactsRateLimited(Exception):
|
|
def __init__(self, retry_after):
|
|
self.retry_after = max(1, int(retry_after + 0.999))
|
|
super().__init__(f'Open Food Facts request budget is full; retry in {self.retry_after} seconds.')
|