213 lines
9.1 KiB
Python
213 lines
9.1 KiB
Python
"""Bounded, DNS-pinned preview metadata for public web pages and YouTube."""
|
|
import http.client
|
|
import ipaddress
|
|
import json
|
|
import re
|
|
import socket
|
|
import ssl
|
|
import time
|
|
from html.parser import HTMLParser
|
|
from urllib.parse import parse_qs, urlencode, urljoin, urlsplit
|
|
|
|
from django.core.exceptions import ValidationError
|
|
|
|
from .services import clean_url
|
|
|
|
|
|
MAX_HTML_BYTES = 512 * 1024
|
|
MAX_JSON_BYTES = 16 * 1024
|
|
MAX_REDIRECTS = 3
|
|
VIDEO_ID = re.compile(r'^[A-Za-z0-9_-]{11}$')
|
|
|
|
|
|
class MetadataError(Exception):
|
|
pass
|
|
|
|
|
|
class MetadataParser(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.title_parts = []
|
|
self.in_title = False
|
|
self.meta = {}
|
|
self.icons = []
|
|
self.canonical = ''
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
props = dict(attrs)
|
|
if tag == 'title':
|
|
self.in_title = True
|
|
elif tag == 'meta':
|
|
name = (props.get('property') or props.get('name') or '').lower().strip()
|
|
value = (props.get('content') or '').strip()
|
|
if name and value and name not in self.meta:
|
|
self.meta[name] = value
|
|
elif tag == 'link':
|
|
rel = (props.get('rel') or '').lower().split()
|
|
if 'canonical' in rel and not self.canonical:
|
|
self.canonical = props.get('href') or ''
|
|
if 'icon' in rel and props.get('href'):
|
|
self.icons.append(props['href'])
|
|
|
|
def handle_endtag(self, tag):
|
|
if tag == 'title':
|
|
self.in_title = False
|
|
|
|
def handle_data(self, data):
|
|
if self.in_title and sum(map(len, self.title_parts)) < 300:
|
|
self.title_parts.append(data)
|
|
|
|
|
|
class PinnedHTTPConnection(http.client.HTTPConnection):
|
|
def __init__(self, host, port, address, secure):
|
|
super().__init__(host, port, timeout=5)
|
|
self.address = address
|
|
self.secure = secure
|
|
|
|
def connect(self):
|
|
raw = socket.create_connection((self.address, self.port), timeout=self.timeout)
|
|
self.sock = ssl.create_default_context().wrap_socket(raw, server_hostname=self.host) if self.secure else raw
|
|
|
|
|
|
def public_address(host, port):
|
|
try:
|
|
answers = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
|
except OSError as exc:
|
|
raise MetadataError('DNS lookup failed.') from exc
|
|
if not answers:
|
|
raise MetadataError('No DNS address found.')
|
|
addresses = [answer[4][0] for answer in answers]
|
|
if any(not ipaddress.ip_address(address).is_global for address in addresses):
|
|
raise MetadataError('DNS resolved to a non-public address.')
|
|
return addresses[0]
|
|
|
|
|
|
def _public_url(value, base):
|
|
try:
|
|
candidate = clean_url(urljoin(base, value))
|
|
except (ValidationError, ValueError, TypeError):
|
|
return ''
|
|
return candidate if urlsplit(candidate).scheme == 'https' else ''
|
|
|
|
|
|
def _fetch_bytes(url, accepted_type, limit, max_redirects=MAX_REDIRECTS):
|
|
try:
|
|
current = clean_url(url)
|
|
except ValidationError as exc:
|
|
raise MetadataError('Invalid page URL.') from exc
|
|
deadline = time.monotonic() + 20
|
|
for _ in range(max_redirects + 1):
|
|
if time.monotonic() >= deadline:
|
|
raise MetadataError('Preview fetch timed out.')
|
|
parts = urlsplit(current)
|
|
port = parts.port or (443 if parts.scheme == 'https' else 80)
|
|
if port not in (80, 443):
|
|
raise MetadataError('Only standard web ports are fetched.')
|
|
address = public_address(parts.hostname, port)
|
|
connection = PinnedHTTPConnection(parts.hostname, port, address, parts.scheme == 'https')
|
|
try:
|
|
target = (parts.path or '/') + (f'?{parts.query}' if parts.query else '')
|
|
connection.request('GET', target, headers={'Host': parts.netloc, 'User-Agent': 'CommonGroundKnowledge/1.0',
|
|
'Accept': accepted_type, 'Accept-Encoding': 'identity', 'Connection': 'close'})
|
|
response = connection.getresponse()
|
|
if response.status in (301, 302, 303, 307, 308):
|
|
location = response.getheader('Location')
|
|
if not location:
|
|
raise MetadataError('Redirect has no location.')
|
|
try:
|
|
current = clean_url(urljoin(current, location))
|
|
except ValidationError as exc:
|
|
raise MetadataError('Redirect points to a disallowed URL.') from exc
|
|
continue
|
|
if response.status != 200:
|
|
raise MetadataError(f'HTTP {response.status}.')
|
|
content_type = (response.getheader('Content-Type') or '').lower().split(';', 1)[0].strip()
|
|
if content_type != accepted_type:
|
|
raise MetadataError('Unexpected content type.')
|
|
length = response.getheader('Content-Length')
|
|
if length and int(length) > limit:
|
|
raise MetadataError('Response is too large.')
|
|
chunks = []
|
|
size = 0
|
|
while True:
|
|
if time.monotonic() >= deadline:
|
|
raise MetadataError('Preview fetch timed out.')
|
|
chunk = response.read(min(8192, limit + 1 - size))
|
|
if not chunk:
|
|
return current, b''.join(chunks)
|
|
size += len(chunk)
|
|
if size > limit:
|
|
raise MetadataError('Response is too large.')
|
|
chunks.append(chunk)
|
|
except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc:
|
|
raise MetadataError('Could not fetch preview metadata.') from exc
|
|
finally:
|
|
connection.close()
|
|
raise MetadataError('Too many redirects.')
|
|
|
|
|
|
def youtube_video_id(url):
|
|
parts = urlsplit(url)
|
|
host = (parts.hostname or '').lower()
|
|
if host in ('youtube.com', 'www.youtube.com', 'm.youtube.com'):
|
|
if parts.path == '/watch':
|
|
value = parse_qs(parts.query).get('v', [''])[0]
|
|
elif parts.path.startswith(('/shorts/', '/live/', '/embed/')):
|
|
value = parts.path.split('/')[2]
|
|
else:
|
|
return ''
|
|
elif host == 'youtu.be':
|
|
value = parts.path.strip('/')
|
|
else:
|
|
return ''
|
|
return value if VIDEO_ID.fullmatch(value or '') else ''
|
|
|
|
|
|
def _youtube_metadata(url, video_id):
|
|
endpoint = 'https://www.youtube.com/oembed?' + urlencode({'url': url, 'format': 'json'})
|
|
_, body = _fetch_bytes(endpoint, 'application/json', MAX_JSON_BYTES, max_redirects=0)
|
|
try:
|
|
data = json.loads(body)
|
|
if not isinstance(data, dict):
|
|
raise ValueError()
|
|
except (ValueError, UnicodeError) as exc:
|
|
raise MetadataError('Invalid YouTube preview.') from exc
|
|
image = _public_url(data.get('thumbnail_url'), url) if isinstance(data.get('thumbnail_url'), str) else ''
|
|
return {'extracted_title': str(data.get('title') or '')[:300], 'description': '',
|
|
'thumbnail_url': image or f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg',
|
|
'image_alt': f'Thumbnail for {str(data.get("title") or "YouTube video")[:260]}',
|
|
'site_name': str(data.get('author_name') or 'YouTube')[:120], 'favicon_url': '',
|
|
'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}',
|
|
'canonical_url': f'https://www.youtube.com/watch?v={video_id}'}
|
|
|
|
|
|
def fetch_metadata(url):
|
|
video_id = youtube_video_id(url)
|
|
if video_id:
|
|
try:
|
|
return _youtube_metadata(url, video_id)
|
|
except MetadataError:
|
|
# A known video still gets a useful card when oEmbed is unavailable.
|
|
return {'extracted_title': '', 'description': '',
|
|
'thumbnail_url': f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg',
|
|
'image_alt': 'YouTube video thumbnail', 'site_name': 'YouTube', 'favicon_url': '',
|
|
'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}',
|
|
'canonical_url': f'https://www.youtube.com/watch?v={video_id}'}
|
|
final_url, body = _fetch_bytes(url, 'text/html', MAX_HTML_BYTES)
|
|
parser = MetadataParser()
|
|
parser.feed(body.decode('utf-8', errors='replace'))
|
|
meta = parser.meta
|
|
title = meta.get('og:title') or meta.get('twitter:title') or ''.join(parser.title_parts)
|
|
description = meta.get('og:description') or meta.get('twitter:description') or meta.get('description') or ''
|
|
image = meta.get('og:image') or meta.get('twitter:image') or ''
|
|
canonical = _public_url(parser.canonical, final_url) if parser.canonical else ''
|
|
if canonical and urlsplit(canonical).hostname != urlsplit(final_url).hostname:
|
|
canonical = ''
|
|
icon = _public_url(parser.icons[0], final_url) if parser.icons else ''
|
|
return {'extracted_title': ' '.join(title.split())[:300], 'description': description[:2000],
|
|
'thumbnail_url': _public_url(image, final_url) if image else '',
|
|
'image_alt': (meta.get('og:image:alt') or meta.get('twitter:image:alt') or '')[:300],
|
|
'site_name': (meta.get('og:site_name') or urlsplit(final_url).hostname or '')[:120],
|
|
'favicon_url': icon or _public_url('/favicon.ico', final_url),
|
|
'provider': 'web', 'embed_url': '', 'canonical_url': canonical}
|