41 lines
2.0 KiB
Python
41 lines
2.0 KiB
Python
from django.core.management.base import BaseCommand
|
|
from django.db.models import Q
|
|
from django.utils import timezone
|
|
from datetime import timedelta
|
|
from knowledge.metadata import MetadataError, fetch_metadata
|
|
from knowledge.models import SavedItem
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = 'Fetch safe, bounded metadata for pending saved links.'
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument('--limit', type=int, default=20)
|
|
|
|
def handle(self, *args, **options):
|
|
total = 0
|
|
now = timezone.now()
|
|
candidates = SavedItem.objects.filter(metadata_attempts__lt=5).filter(
|
|
Q(metadata_status=SavedItem.MetadataStatus.PENDING) |
|
|
Q(metadata_status=SavedItem.MetadataStatus.ERROR, metadata_retry_at__lte=now)).order_by('created_at')
|
|
for item in candidates[:max(0, min(options['limit'], 100))]:
|
|
item.metadata_attempts += 1
|
|
try:
|
|
fields = fetch_metadata(item.original_url)
|
|
for name, value in fields.items():
|
|
setattr(item, name, value)
|
|
item.metadata_status = SavedItem.MetadataStatus.READY
|
|
item.metadata_error = ''
|
|
item.metadata_fetched_at = timezone.now()
|
|
item.metadata_retry_at = None
|
|
item.save(update_fields=[*fields, 'metadata_status', 'metadata_error', 'metadata_fetched_at',
|
|
'metadata_retry_at', 'metadata_attempts', 'updated_at'])
|
|
except MetadataError as exc:
|
|
item.metadata_status = SavedItem.MetadataStatus.ERROR
|
|
item.metadata_error = str(exc)[:200]
|
|
item.metadata_retry_at = timezone.now() + timedelta(minutes=min(5 * 2 ** (item.metadata_attempts - 1), 1440))
|
|
item.save(update_fields=['metadata_status', 'metadata_error', 'metadata_retry_at',
|
|
'metadata_attempts', 'updated_at'])
|
|
total += 1
|
|
self.stdout.write(f'Processed {total} link(s).')
|