Files
Django_TasksApp/main/food/services/off_rate_limit.py
T

60 lines
2.4 KiB
Python

"""Database-coordinated cap for actual Open Food Facts HTTP requests."""
from contextlib import contextmanager
from datetime import timedelta
import time
from django.db import transaction
from django.utils import timezone
from food.models import OpenFoodFactsRateLimit, OpenFoodFactsRequest
REQUESTS_PER_MINUTE = 5
WINDOW_SECONDS = 60
@contextmanager
def off_http_request(endpoint, *, wait=True, before_request=None):
"""Reserve and serialize one real request across web and worker processes.
The singleton row lock is held through the HTTP call, so simultaneous workers
cannot race through the budget check. Failed HTTP requests still count because
the reservation commits before the exception is re-raised.
"""
while True:
delay = 0
failure = None
traceback = None
with transaction.atomic():
OpenFoodFactsRateLimit.objects.get_or_create(pk=1)
OpenFoodFactsRateLimit.objects.select_for_update().get(pk=1)
now = timezone.now()
cutoff = now - timedelta(seconds=WINDOW_SECONDS)
oldest = OpenFoodFactsRequest.objects.filter(requested_at__gte=cutoff).order_by('requested_at').first()
if OpenFoodFactsRequest.objects.filter(requested_at__gte=cutoff).count() >= REQUESTS_PER_MINUTE:
delay = max(0.05, (oldest.requested_at + timedelta(seconds=WINDOW_SECONDS + 0.01) - now).total_seconds())
else:
OpenFoodFactsRequest.objects.create(requested_at=now, endpoint=endpoint[:24])
OpenFoodFactsRequest.objects.filter(requested_at__lt=now - timedelta(minutes=10)).delete()
try:
if before_request:
before_request()
yield
except BaseException as error:
failure = error
traceback = error.__traceback__
if delay:
if not wait:
raise OpenFoodFactsRateLimited(delay)
time.sleep(delay)
continue
if failure is not None:
raise failure.with_traceback(traceback)
return
class OpenFoodFactsRateLimited(Exception):
def __init__(self, retry_after):
self.retry_after = max(1, int(retry_after + 0.999))
super().__init__(f'Open Food Facts request budget is full; retry in {self.retry_after} seconds.')