81 lines
3.5 KiB
Python
81 lines
3.5 KiB
Python
import ipaddress
|
|
import re
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
|
|
from django.core.exceptions import ValidationError
|
|
from django.db.models import Q
|
|
from django.utils import timezone
|
|
from datetime import timedelta
|
|
|
|
from .models import SavedItem
|
|
|
|
|
|
def clean_url(value):
|
|
value = (value or '').strip()
|
|
if len(value) > 2048 or any(ord(c) < 32 for c in value):
|
|
raise ValidationError('Enter a valid URL under 2048 characters.')
|
|
try:
|
|
parts = urlsplit(value)
|
|
host = parts.hostname
|
|
port = parts.port
|
|
except ValueError as exc:
|
|
raise ValidationError('Enter a valid URL.') from exc
|
|
if parts.scheme.lower() not in ('http', 'https') or not host or parts.username or parts.password:
|
|
raise ValidationError('Only public HTTP(S) URLs are accepted.')
|
|
host = host.lower().rstrip('.')
|
|
if '.' not in host and ':' not in host:
|
|
raise ValidationError('A public hostname is required.')
|
|
try:
|
|
address = ipaddress.ip_address(host.strip('[]'))
|
|
except ValueError:
|
|
address = None
|
|
if address and not address.is_global:
|
|
raise ValidationError('Local and private addresses are not accepted.')
|
|
if host in ('localhost', 'localhost.localdomain') or host.endswith(('.local', '.internal', '.localhost')):
|
|
raise ValidationError('Local hostnames are not accepted.')
|
|
authority = host if ':' not in host else f'[{host}]'
|
|
if port and port != (443 if parts.scheme.lower() == 'https' else 80):
|
|
authority += f':{port}'
|
|
key = urlunsplit((parts.scheme.lower(), authority, parts.path or '/', parts.query, ''))
|
|
if len(key) > 2048:
|
|
raise ValidationError('URL is too long.')
|
|
return key
|
|
|
|
|
|
def save_item(owner, url, **fields):
|
|
key = clean_url(url)
|
|
allowed = {'title', 'notes', 'saved_reason', 'deferral_reason', 'tags'}
|
|
defaults = {name: value for name, value in fields.items() if name in allowed}
|
|
item, created = SavedItem.objects.get_or_create(owner=owner, url_key=key,
|
|
defaults={'original_url': url.strip(), **defaults})
|
|
return item, created
|
|
|
|
|
|
def search_items(owner, query, limit=20):
|
|
words = re.findall(r'[\w-]{2,}', (query or '').lower())[:8]
|
|
qs = SavedItem.objects.filter(owner=owner).exclude(status=SavedItem.Status.ARCHIVED)
|
|
for word in words:
|
|
qs = qs.filter(Q(title__icontains=word) | Q(extracted_title__icontains=word) |
|
|
Q(original_url__icontains=word) | Q(description__icontains=word) |
|
|
Q(notes__icontains=word) | Q(saved_reason__icontains=word) | Q(tags__icontains=word))
|
|
return list(qs[:max(1, min(limit, 100))])
|
|
|
|
|
|
def relevant_items(owner, context):
|
|
words = set(re.findall(r'[\w-]{4,}', (context or '').lower()))
|
|
words -= {'this', 'that', 'with', 'from', 'about', 'what', 'would', 'could', 'should', 'there', 'their'}
|
|
if not words:
|
|
return []
|
|
now = timezone.now()
|
|
candidates = SavedItem.objects.filter(owner=owner, status=SavedItem.Status.SAVED).filter(
|
|
Q(snoozed_until__isnull=True) | Q(snoozed_until__lte=now)).filter(
|
|
Q(last_surfaced_at__isnull=True) | Q(last_surfaced_at__lte=now - timedelta(days=14)))
|
|
scored = []
|
|
for item in candidates[:500]:
|
|
text = ' '.join([item.title, item.extracted_title, item.description, item.saved_reason, ' '.join(item.tags)]).lower()
|
|
hits = words & set(re.findall(r'[\w-]{4,}', text))
|
|
if len(hits) >= 2 or (len(hits) == 1 and len(next(iter(hits))) >= 6):
|
|
scored.append((len(hits), item, sorted(hits)))
|
|
scored.sort(key=lambda row: (-row[0], -row[1].created_at.timestamp()))
|
|
return scored[:3]
|