"""Bounded, DNS-pinned preview metadata for public web pages and YouTube.""" import http.client import ipaddress import json import re import socket import ssl import time from html.parser import HTMLParser from urllib.parse import parse_qs, urlencode, urljoin, urlsplit from django.core.exceptions import ValidationError from .services import clean_url MAX_HTML_BYTES = 512 * 1024 MAX_JSON_BYTES = 16 * 1024 MAX_REDIRECTS = 3 VIDEO_ID = re.compile(r'^[A-Za-z0-9_-]{11}$') class MetadataError(Exception): pass class MetadataParser(HTMLParser): def __init__(self): super().__init__(convert_charrefs=True) self.title_parts = [] self.in_title = False self.meta = {} self.icons = [] self.canonical = '' def handle_starttag(self, tag, attrs): props = dict(attrs) if tag == 'title': self.in_title = True elif tag == 'meta': name = (props.get('property') or props.get('name') or '').lower().strip() value = (props.get('content') or '').strip() if name and value and name not in self.meta: self.meta[name] = value elif tag == 'link': rel = (props.get('rel') or '').lower().split() if 'canonical' in rel and not self.canonical: self.canonical = props.get('href') or '' if 'icon' in rel and props.get('href'): self.icons.append(props['href']) def handle_endtag(self, tag): if tag == 'title': self.in_title = False def handle_data(self, data): if self.in_title and sum(map(len, self.title_parts)) < 300: self.title_parts.append(data) class PinnedHTTPConnection(http.client.HTTPConnection): def __init__(self, host, port, address, secure): super().__init__(host, port, timeout=5) self.address = address self.secure = secure def connect(self): raw = socket.create_connection((self.address, self.port), timeout=self.timeout) self.sock = ssl.create_default_context().wrap_socket(raw, server_hostname=self.host) if self.secure else raw def public_address(host, port): try: answers = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) except OSError as exc: raise MetadataError('DNS lookup failed.') from exc if not answers: raise MetadataError('No DNS address found.') addresses = [answer[4][0] for answer in answers] if any(not ipaddress.ip_address(address).is_global for address in addresses): raise MetadataError('DNS resolved to a non-public address.') return addresses[0] def _public_url(value, base): try: candidate = clean_url(urljoin(base, value)) except (ValidationError, ValueError, TypeError): return '' return candidate if urlsplit(candidate).scheme == 'https' else '' def _fetch_bytes(url, accepted_type, limit, max_redirects=MAX_REDIRECTS): try: current = clean_url(url) except ValidationError as exc: raise MetadataError('Invalid page URL.') from exc deadline = time.monotonic() + 20 for _ in range(max_redirects + 1): if time.monotonic() >= deadline: raise MetadataError('Preview fetch timed out.') parts = urlsplit(current) port = parts.port or (443 if parts.scheme == 'https' else 80) if port not in (80, 443): raise MetadataError('Only standard web ports are fetched.') address = public_address(parts.hostname, port) connection = PinnedHTTPConnection(parts.hostname, port, address, parts.scheme == 'https') try: target = (parts.path or '/') + (f'?{parts.query}' if parts.query else '') connection.request('GET', target, headers={'Host': parts.netloc, 'User-Agent': 'CommonGroundKnowledge/1.0', 'Accept': accepted_type, 'Accept-Encoding': 'identity', 'Connection': 'close'}) response = connection.getresponse() if response.status in (301, 302, 303, 307, 308): location = response.getheader('Location') if not location: raise MetadataError('Redirect has no location.') try: current = clean_url(urljoin(current, location)) except ValidationError as exc: raise MetadataError('Redirect points to a disallowed URL.') from exc continue if response.status != 200: raise MetadataError(f'HTTP {response.status}.') content_type = (response.getheader('Content-Type') or '').lower().split(';', 1)[0].strip() if content_type != accepted_type: raise MetadataError('Unexpected content type.') length = response.getheader('Content-Length') if length and int(length) > limit: raise MetadataError('Response is too large.') chunks = [] size = 0 while True: if time.monotonic() >= deadline: raise MetadataError('Preview fetch timed out.') chunk = response.read(min(8192, limit + 1 - size)) if not chunk: return current, b''.join(chunks) size += len(chunk) if size > limit: raise MetadataError('Response is too large.') chunks.append(chunk) except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc: raise MetadataError('Could not fetch preview metadata.') from exc finally: connection.close() raise MetadataError('Too many redirects.') def youtube_video_id(url): parts = urlsplit(url) host = (parts.hostname or '').lower() if host in ('youtube.com', 'www.youtube.com', 'm.youtube.com'): if parts.path == '/watch': value = parse_qs(parts.query).get('v', [''])[0] elif parts.path.startswith(('/shorts/', '/live/', '/embed/')): value = parts.path.split('/')[2] else: return '' elif host == 'youtu.be': value = parts.path.strip('/') else: return '' return value if VIDEO_ID.fullmatch(value or '') else '' def _youtube_metadata(url, video_id): endpoint = 'https://www.youtube.com/oembed?' + urlencode({'url': url, 'format': 'json'}) _, body = _fetch_bytes(endpoint, 'application/json', MAX_JSON_BYTES, max_redirects=0) try: data = json.loads(body) if not isinstance(data, dict): raise ValueError() except (ValueError, UnicodeError) as exc: raise MetadataError('Invalid YouTube preview.') from exc image = _public_url(data.get('thumbnail_url'), url) if isinstance(data.get('thumbnail_url'), str) else '' return {'extracted_title': str(data.get('title') or '')[:300], 'description': '', 'thumbnail_url': image or f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg', 'image_alt': f'Thumbnail for {str(data.get("title") or "YouTube video")[:260]}', 'site_name': str(data.get('author_name') or 'YouTube')[:120], 'favicon_url': '', 'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}', 'canonical_url': f'https://www.youtube.com/watch?v={video_id}'} def fetch_metadata(url): video_id = youtube_video_id(url) if video_id: try: return _youtube_metadata(url, video_id) except MetadataError: # A known video still gets a useful card when oEmbed is unavailable. return {'extracted_title': '', 'description': '', 'thumbnail_url': f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg', 'image_alt': 'YouTube video thumbnail', 'site_name': 'YouTube', 'favicon_url': '', 'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}', 'canonical_url': f'https://www.youtube.com/watch?v={video_id}'} final_url, body = _fetch_bytes(url, 'text/html', MAX_HTML_BYTES) parser = MetadataParser() parser.feed(body.decode('utf-8', errors='replace')) meta = parser.meta title = meta.get('og:title') or meta.get('twitter:title') or ''.join(parser.title_parts) description = meta.get('og:description') or meta.get('twitter:description') or meta.get('description') or '' image = meta.get('og:image') or meta.get('twitter:image') or '' canonical = _public_url(parser.canonical, final_url) if parser.canonical else '' if canonical and urlsplit(canonical).hostname != urlsplit(final_url).hostname: canonical = '' icon = _public_url(parser.icons[0], final_url) if parser.icons else '' return {'extracted_title': ' '.join(title.split())[:300], 'description': description[:2000], 'thumbnail_url': _public_url(image, final_url) if image else '', 'image_alt': (meta.get('og:image:alt') or meta.get('twitter:image:alt') or '')[:300], 'site_name': (meta.get('og:site_name') or urlsplit(final_url).hostname or '')[:120], 'favicon_url': icon or _public_url('/favicon.ico', final_url), 'provider': 'web', 'embed_url': '', 'canonical_url': canonical}