Files
2026-10-21 15:25:33 +02:00

213 lines
9.1 KiB
Python

"""Bounded, DNS-pinned preview metadata for public web pages and YouTube."""
import http.client
import ipaddress
import json
import re
import socket
import ssl
import time
from html.parser import HTMLParser
from urllib.parse import parse_qs, urlencode, urljoin, urlsplit
from django.core.exceptions import ValidationError
from .services import clean_url
MAX_HTML_BYTES = 512 * 1024
MAX_JSON_BYTES = 16 * 1024
MAX_REDIRECTS = 3
VIDEO_ID = re.compile(r'^[A-Za-z0-9_-]{11}$')
class MetadataError(Exception):
pass
class MetadataParser(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.title_parts = []
self.in_title = False
self.meta = {}
self.icons = []
self.canonical = ''
def handle_starttag(self, tag, attrs):
props = dict(attrs)
if tag == 'title':
self.in_title = True
elif tag == 'meta':
name = (props.get('property') or props.get('name') or '').lower().strip()
value = (props.get('content') or '').strip()
if name and value and name not in self.meta:
self.meta[name] = value
elif tag == 'link':
rel = (props.get('rel') or '').lower().split()
if 'canonical' in rel and not self.canonical:
self.canonical = props.get('href') or ''
if 'icon' in rel and props.get('href'):
self.icons.append(props['href'])
def handle_endtag(self, tag):
if tag == 'title':
self.in_title = False
def handle_data(self, data):
if self.in_title and sum(map(len, self.title_parts)) < 300:
self.title_parts.append(data)
class PinnedHTTPConnection(http.client.HTTPConnection):
def __init__(self, host, port, address, secure):
super().__init__(host, port, timeout=5)
self.address = address
self.secure = secure
def connect(self):
raw = socket.create_connection((self.address, self.port), timeout=self.timeout)
self.sock = ssl.create_default_context().wrap_socket(raw, server_hostname=self.host) if self.secure else raw
def public_address(host, port):
try:
answers = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
except OSError as exc:
raise MetadataError('DNS lookup failed.') from exc
if not answers:
raise MetadataError('No DNS address found.')
addresses = [answer[4][0] for answer in answers]
if any(not ipaddress.ip_address(address).is_global for address in addresses):
raise MetadataError('DNS resolved to a non-public address.')
return addresses[0]
def _public_url(value, base):
try:
candidate = clean_url(urljoin(base, value))
except (ValidationError, ValueError, TypeError):
return ''
return candidate if urlsplit(candidate).scheme == 'https' else ''
def _fetch_bytes(url, accepted_type, limit, max_redirects=MAX_REDIRECTS):
try:
current = clean_url(url)
except ValidationError as exc:
raise MetadataError('Invalid page URL.') from exc
deadline = time.monotonic() + 20
for _ in range(max_redirects + 1):
if time.monotonic() >= deadline:
raise MetadataError('Preview fetch timed out.')
parts = urlsplit(current)
port = parts.port or (443 if parts.scheme == 'https' else 80)
if port not in (80, 443):
raise MetadataError('Only standard web ports are fetched.')
address = public_address(parts.hostname, port)
connection = PinnedHTTPConnection(parts.hostname, port, address, parts.scheme == 'https')
try:
target = (parts.path or '/') + (f'?{parts.query}' if parts.query else '')
connection.request('GET', target, headers={'Host': parts.netloc, 'User-Agent': 'CommonGroundKnowledge/1.0',
'Accept': accepted_type, 'Accept-Encoding': 'identity', 'Connection': 'close'})
response = connection.getresponse()
if response.status in (301, 302, 303, 307, 308):
location = response.getheader('Location')
if not location:
raise MetadataError('Redirect has no location.')
try:
current = clean_url(urljoin(current, location))
except ValidationError as exc:
raise MetadataError('Redirect points to a disallowed URL.') from exc
continue
if response.status != 200:
raise MetadataError(f'HTTP {response.status}.')
content_type = (response.getheader('Content-Type') or '').lower().split(';', 1)[0].strip()
if content_type != accepted_type:
raise MetadataError('Unexpected content type.')
length = response.getheader('Content-Length')
if length and int(length) > limit:
raise MetadataError('Response is too large.')
chunks = []
size = 0
while True:
if time.monotonic() >= deadline:
raise MetadataError('Preview fetch timed out.')
chunk = response.read(min(8192, limit + 1 - size))
if not chunk:
return current, b''.join(chunks)
size += len(chunk)
if size > limit:
raise MetadataError('Response is too large.')
chunks.append(chunk)
except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc:
raise MetadataError('Could not fetch preview metadata.') from exc
finally:
connection.close()
raise MetadataError('Too many redirects.')
def youtube_video_id(url):
parts = urlsplit(url)
host = (parts.hostname or '').lower()
if host in ('youtube.com', 'www.youtube.com', 'm.youtube.com'):
if parts.path == '/watch':
value = parse_qs(parts.query).get('v', [''])[0]
elif parts.path.startswith(('/shorts/', '/live/', '/embed/')):
value = parts.path.split('/')[2]
else:
return ''
elif host == 'youtu.be':
value = parts.path.strip('/')
else:
return ''
return value if VIDEO_ID.fullmatch(value or '') else ''
def _youtube_metadata(url, video_id):
endpoint = 'https://www.youtube.com/oembed?' + urlencode({'url': url, 'format': 'json'})
_, body = _fetch_bytes(endpoint, 'application/json', MAX_JSON_BYTES, max_redirects=0)
try:
data = json.loads(body)
if not isinstance(data, dict):
raise ValueError()
except (ValueError, UnicodeError) as exc:
raise MetadataError('Invalid YouTube preview.') from exc
image = _public_url(data.get('thumbnail_url'), url) if isinstance(data.get('thumbnail_url'), str) else ''
return {'extracted_title': str(data.get('title') or '')[:300], 'description': '',
'thumbnail_url': image or f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg',
'image_alt': f'Thumbnail for {str(data.get("title") or "YouTube video")[:260]}',
'site_name': str(data.get('author_name') or 'YouTube')[:120], 'favicon_url': '',
'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}',
'canonical_url': f'https://www.youtube.com/watch?v={video_id}'}
def fetch_metadata(url):
video_id = youtube_video_id(url)
if video_id:
try:
return _youtube_metadata(url, video_id)
except MetadataError:
# A known video still gets a useful card when oEmbed is unavailable.
return {'extracted_title': '', 'description': '',
'thumbnail_url': f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg',
'image_alt': 'YouTube video thumbnail', 'site_name': 'YouTube', 'favicon_url': '',
'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}',
'canonical_url': f'https://www.youtube.com/watch?v={video_id}'}
final_url, body = _fetch_bytes(url, 'text/html', MAX_HTML_BYTES)
parser = MetadataParser()
parser.feed(body.decode('utf-8', errors='replace'))
meta = parser.meta
title = meta.get('og:title') or meta.get('twitter:title') or ''.join(parser.title_parts)
description = meta.get('og:description') or meta.get('twitter:description') or meta.get('description') or ''
image = meta.get('og:image') or meta.get('twitter:image') or ''
canonical = _public_url(parser.canonical, final_url) if parser.canonical else ''
if canonical and urlsplit(canonical).hostname != urlsplit(final_url).hostname:
canonical = ''
icon = _public_url(parser.icons[0], final_url) if parser.icons else ''
return {'extracted_title': ' '.join(title.split())[:300], 'description': description[:2000],
'thumbnail_url': _public_url(image, final_url) if image else '',
'image_alt': (meta.get('og:image:alt') or meta.get('twitter:image:alt') or '')[:300],
'site_name': (meta.get('og:site_name') or urlsplit(final_url).hostname or '')[:120],
'favicon_url': icon or _public_url('/favicon.ico', final_url),
'provider': 'web', 'embed_url': '', 'canonical_url': canonical}