link preview v1

This commit is contained in:
Joren de Graaf
2026-10-21 15:25:33 +02:00
parent 4948d7645e
commit 9a5210a48f
30 changed files with 1173 additions and 4 deletions
View File
+6
View File
@@ -0,0 +1,6 @@
from django.apps import AppConfig
class KnowledgeConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'knowledge'
View File
@@ -0,0 +1,40 @@
from django.core.management.base import BaseCommand
from django.db.models import Q
from django.utils import timezone
from datetime import timedelta
from knowledge.metadata import MetadataError, fetch_metadata
from knowledge.models import SavedItem
class Command(BaseCommand):
help = 'Fetch safe, bounded metadata for pending saved links.'
def add_arguments(self, parser):
parser.add_argument('--limit', type=int, default=20)
def handle(self, *args, **options):
total = 0
now = timezone.now()
candidates = SavedItem.objects.filter(metadata_attempts__lt=5).filter(
Q(metadata_status=SavedItem.MetadataStatus.PENDING) |
Q(metadata_status=SavedItem.MetadataStatus.ERROR, metadata_retry_at__lte=now)).order_by('created_at')
for item in candidates[:max(0, min(options['limit'], 100))]:
item.metadata_attempts += 1
try:
fields = fetch_metadata(item.original_url)
for name, value in fields.items():
setattr(item, name, value)
item.metadata_status = SavedItem.MetadataStatus.READY
item.metadata_error = ''
item.metadata_fetched_at = timezone.now()
item.metadata_retry_at = None
item.save(update_fields=[*fields, 'metadata_status', 'metadata_error', 'metadata_fetched_at',
'metadata_retry_at', 'metadata_attempts', 'updated_at'])
except MetadataError as exc:
item.metadata_status = SavedItem.MetadataStatus.ERROR
item.metadata_error = str(exc)[:200]
item.metadata_retry_at = timezone.now() + timedelta(minutes=min(5 * 2 ** (item.metadata_attempts - 1), 1440))
item.save(update_fields=['metadata_status', 'metadata_error', 'metadata_retry_at',
'metadata_attempts', 'updated_at'])
total += 1
self.stdout.write(f'Processed {total} link(s).')
@@ -0,0 +1,12 @@
from django.core.management.base import BaseCommand, CommandError
from knowledge.notifications import NotificationError, rediscover_once
class Command(BaseCommand):
help = 'Send at most one eligible Knowledge Library ntfy reminder, subject to opt-in, quiet hours and weekly limit.'
def handle(self, *args, **options):
try:
self.stdout.write(rediscover_once())
except NotificationError as exc:
raise CommandError(str(exc)) from exc
+212
View File
@@ -0,0 +1,212 @@
"""Bounded, DNS-pinned preview metadata for public web pages and YouTube."""
import http.client
import ipaddress
import json
import re
import socket
import ssl
import time
from html.parser import HTMLParser
from urllib.parse import parse_qs, urlencode, urljoin, urlsplit
from django.core.exceptions import ValidationError
from .services import clean_url
MAX_HTML_BYTES = 512 * 1024
MAX_JSON_BYTES = 16 * 1024
MAX_REDIRECTS = 3
VIDEO_ID = re.compile(r'^[A-Za-z0-9_-]{11}$')
class MetadataError(Exception):
pass
class MetadataParser(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.title_parts = []
self.in_title = False
self.meta = {}
self.icons = []
self.canonical = ''
def handle_starttag(self, tag, attrs):
props = dict(attrs)
if tag == 'title':
self.in_title = True
elif tag == 'meta':
name = (props.get('property') or props.get('name') or '').lower().strip()
value = (props.get('content') or '').strip()
if name and value and name not in self.meta:
self.meta[name] = value
elif tag == 'link':
rel = (props.get('rel') or '').lower().split()
if 'canonical' in rel and not self.canonical:
self.canonical = props.get('href') or ''
if 'icon' in rel and props.get('href'):
self.icons.append(props['href'])
def handle_endtag(self, tag):
if tag == 'title':
self.in_title = False
def handle_data(self, data):
if self.in_title and sum(map(len, self.title_parts)) < 300:
self.title_parts.append(data)
class PinnedHTTPConnection(http.client.HTTPConnection):
def __init__(self, host, port, address, secure):
super().__init__(host, port, timeout=5)
self.address = address
self.secure = secure
def connect(self):
raw = socket.create_connection((self.address, self.port), timeout=self.timeout)
self.sock = ssl.create_default_context().wrap_socket(raw, server_hostname=self.host) if self.secure else raw
def public_address(host, port):
try:
answers = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
except OSError as exc:
raise MetadataError('DNS lookup failed.') from exc
if not answers:
raise MetadataError('No DNS address found.')
addresses = [answer[4][0] for answer in answers]
if any(not ipaddress.ip_address(address).is_global for address in addresses):
raise MetadataError('DNS resolved to a non-public address.')
return addresses[0]
def _public_url(value, base):
try:
candidate = clean_url(urljoin(base, value))
except (ValidationError, ValueError, TypeError):
return ''
return candidate if urlsplit(candidate).scheme == 'https' else ''
def _fetch_bytes(url, accepted_type, limit, max_redirects=MAX_REDIRECTS):
try:
current = clean_url(url)
except ValidationError as exc:
raise MetadataError('Invalid page URL.') from exc
deadline = time.monotonic() + 20
for _ in range(max_redirects + 1):
if time.monotonic() >= deadline:
raise MetadataError('Preview fetch timed out.')
parts = urlsplit(current)
port = parts.port or (443 if parts.scheme == 'https' else 80)
if port not in (80, 443):
raise MetadataError('Only standard web ports are fetched.')
address = public_address(parts.hostname, port)
connection = PinnedHTTPConnection(parts.hostname, port, address, parts.scheme == 'https')
try:
target = (parts.path or '/') + (f'?{parts.query}' if parts.query else '')
connection.request('GET', target, headers={'Host': parts.netloc, 'User-Agent': 'CommonGroundKnowledge/1.0',
'Accept': accepted_type, 'Accept-Encoding': 'identity', 'Connection': 'close'})
response = connection.getresponse()
if response.status in (301, 302, 303, 307, 308):
location = response.getheader('Location')
if not location:
raise MetadataError('Redirect has no location.')
try:
current = clean_url(urljoin(current, location))
except ValidationError as exc:
raise MetadataError('Redirect points to a disallowed URL.') from exc
continue
if response.status != 200:
raise MetadataError(f'HTTP {response.status}.')
content_type = (response.getheader('Content-Type') or '').lower().split(';', 1)[0].strip()
if content_type != accepted_type:
raise MetadataError('Unexpected content type.')
length = response.getheader('Content-Length')
if length and int(length) > limit:
raise MetadataError('Response is too large.')
chunks = []
size = 0
while True:
if time.monotonic() >= deadline:
raise MetadataError('Preview fetch timed out.')
chunk = response.read(min(8192, limit + 1 - size))
if not chunk:
return current, b''.join(chunks)
size += len(chunk)
if size > limit:
raise MetadataError('Response is too large.')
chunks.append(chunk)
except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc:
raise MetadataError('Could not fetch preview metadata.') from exc
finally:
connection.close()
raise MetadataError('Too many redirects.')
def youtube_video_id(url):
parts = urlsplit(url)
host = (parts.hostname or '').lower()
if host in ('youtube.com', 'www.youtube.com', 'm.youtube.com'):
if parts.path == '/watch':
value = parse_qs(parts.query).get('v', [''])[0]
elif parts.path.startswith(('/shorts/', '/live/', '/embed/')):
value = parts.path.split('/')[2]
else:
return ''
elif host == 'youtu.be':
value = parts.path.strip('/')
else:
return ''
return value if VIDEO_ID.fullmatch(value or '') else ''
def _youtube_metadata(url, video_id):
endpoint = 'https://www.youtube.com/oembed?' + urlencode({'url': url, 'format': 'json'})
_, body = _fetch_bytes(endpoint, 'application/json', MAX_JSON_BYTES, max_redirects=0)
try:
data = json.loads(body)
if not isinstance(data, dict):
raise ValueError()
except (ValueError, UnicodeError) as exc:
raise MetadataError('Invalid YouTube preview.') from exc
image = _public_url(data.get('thumbnail_url'), url) if isinstance(data.get('thumbnail_url'), str) else ''
return {'extracted_title': str(data.get('title') or '')[:300], 'description': '',
'thumbnail_url': image or f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg',
'image_alt': f'Thumbnail for {str(data.get("title") or "YouTube video")[:260]}',
'site_name': str(data.get('author_name') or 'YouTube')[:120], 'favicon_url': '',
'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}',
'canonical_url': f'https://www.youtube.com/watch?v={video_id}'}
def fetch_metadata(url):
video_id = youtube_video_id(url)
if video_id:
try:
return _youtube_metadata(url, video_id)
except MetadataError:
# A known video still gets a useful card when oEmbed is unavailable.
return {'extracted_title': '', 'description': '',
'thumbnail_url': f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg',
'image_alt': 'YouTube video thumbnail', 'site_name': 'YouTube', 'favicon_url': '',
'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}',
'canonical_url': f'https://www.youtube.com/watch?v={video_id}'}
final_url, body = _fetch_bytes(url, 'text/html', MAX_HTML_BYTES)
parser = MetadataParser()
parser.feed(body.decode('utf-8', errors='replace'))
meta = parser.meta
title = meta.get('og:title') or meta.get('twitter:title') or ''.join(parser.title_parts)
description = meta.get('og:description') or meta.get('twitter:description') or meta.get('description') or ''
image = meta.get('og:image') or meta.get('twitter:image') or ''
canonical = _public_url(parser.canonical, final_url) if parser.canonical else ''
if canonical and urlsplit(canonical).hostname != urlsplit(final_url).hostname:
canonical = ''
icon = _public_url(parser.icons[0], final_url) if parser.icons else ''
return {'extracted_title': ' '.join(title.split())[:300], 'description': description[:2000],
'thumbnail_url': _public_url(image, final_url) if image else '',
'image_alt': (meta.get('og:image:alt') or meta.get('twitter:image:alt') or '')[:300],
'site_name': (meta.get('og:site_name') or urlsplit(final_url).hostname or '')[:120],
'favicon_url': icon or _public_url('/favicon.ico', final_url),
'provider': 'web', 'embed_url': '', 'canonical_url': canonical}
+83
View File
@@ -0,0 +1,83 @@
# Generated by Django 6.1.2 on 2026-10-20 17:27
import django.db.models.deletion
import uuid
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='CaptureToken',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=80)),
('token_hash', models.CharField(max_length=64, unique=True)),
('created_at', models.DateTimeField(auto_now_add=True)),
('revoked_at', models.DateTimeField(blank=True, null=True)),
('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='knowledge_tokens', to=settings.AUTH_USER_MODEL)),
],
),
migrations.CreateModel(
name='SavedItem',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('original_url', models.URLField(max_length=2048)),
('url_key', models.CharField(max_length=2048)),
('canonical_url', models.URLField(blank=True, max_length=2048)),
('title', models.CharField(blank=True, max_length=300)),
('extracted_title', models.CharField(blank=True, max_length=300)),
('description', models.TextField(blank=True)),
('thumbnail_url', models.URLField(blank=True, max_length=2048)),
('notes', models.TextField(blank=True)),
('tags', models.JSONField(blank=True, default=list)),
('saved_reason', models.CharField(blank=True, max_length=300)),
('deferral_reason', models.CharField(blank=True, max_length=120)),
('status', models.CharField(choices=[('saved', 'Saved'), ('read', 'Read'), ('archived', 'Archived')], default='saved', max_length=12)),
('metadata_status', models.CharField(choices=[('pending', 'Pending'), ('ready', 'Ready'), ('error', 'Error')], default='pending', max_length=12)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('read_at', models.DateTimeField(blank=True, null=True)),
('last_surfaced_at', models.DateTimeField(blank=True, null=True)),
('snoozed_until', models.DateTimeField(blank=True, null=True)),
('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='knowledge_items', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['-created_at'],
},
),
migrations.CreateModel(
name='RediscoveryEvent',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('channel', models.CharField(choices=[('mcp', 'MCP'), ('ntfy', 'ntfy')], max_length=12)),
('outcome', models.CharField(max_length=20)),
('created_at', models.DateTimeField(auto_now_add=True)),
('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rediscovery_events', to='knowledge.saveditem')),
],
),
migrations.AddIndex(
model_name='saveditem',
index=models.Index(fields=['owner', 'status', '-created_at'], name='knowledge_s_owner_i_909a54_idx'),
),
migrations.AddIndex(
model_name='saveditem',
index=models.Index(fields=['owner', 'last_surfaced_at'], name='knowledge_s_owner_i_89f98e_idx'),
),
migrations.AddConstraint(
model_name='saveditem',
constraint=models.UniqueConstraint(fields=('owner', 'url_key'), name='knowledge_owner_url_unique'),
),
migrations.AddIndex(
model_name='rediscoveryevent',
index=models.Index(fields=['item', '-created_at'], name='knowledge_r_item_id_0f6d7f_idx'),
),
]
@@ -0,0 +1,68 @@
# Generated by Django 6.1.2 on 2026-10-20 17:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('knowledge', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='saveditem',
name='embed_url',
field=models.URLField(blank=True, max_length=2048),
),
migrations.AddField(
model_name='saveditem',
name='favicon_url',
field=models.URLField(blank=True, max_length=2048),
),
migrations.AddField(
model_name='saveditem',
name='image_alt',
field=models.CharField(blank=True, max_length=300),
),
migrations.AddField(
model_name='saveditem',
name='metadata_attempts',
field=models.PositiveSmallIntegerField(default=0),
),
migrations.AddField(
model_name='saveditem',
name='metadata_error',
field=models.CharField(blank=True, max_length=200),
),
migrations.AddField(
model_name='saveditem',
name='metadata_fetched_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='saveditem',
name='metadata_retry_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='saveditem',
name='preview_mode',
field=models.CharField(choices=[('preview', 'Preview'), ('plain', 'Plain link'), ('embed', 'Playable embed')], default='preview', max_length=12),
),
migrations.AddField(
model_name='saveditem',
name='provider',
field=models.CharField(blank=True, max_length=40),
),
migrations.AddField(
model_name='saveditem',
name='site_name',
field=models.CharField(blank=True, max_length=120),
),
migrations.AddField(
model_name='saveditem',
name='user_description',
field=models.TextField(blank=True),
),
]
View File
+75
View File
@@ -0,0 +1,75 @@
import uuid
from urllib.parse import urlsplit
from django.conf import settings
from django.db import models
class SavedItem(models.Model):
class Status(models.TextChoices):
SAVED = 'saved', 'Saved'
READ = 'read', 'Read'
ARCHIVED = 'archived', 'Archived'
class MetadataStatus(models.TextChoices):
PENDING = 'pending', 'Pending'
READY = 'ready', 'Ready'
ERROR = 'error', 'Error'
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='knowledge_items')
original_url = models.URLField(max_length=2048)
url_key = models.CharField(max_length=2048)
canonical_url = models.URLField(max_length=2048, blank=True)
title = models.CharField(max_length=300, blank=True)
user_description = models.TextField(blank=True)
extracted_title = models.CharField(max_length=300, blank=True)
description = models.TextField(blank=True)
thumbnail_url = models.URLField(max_length=2048, blank=True)
image_alt = models.CharField(max_length=300, blank=True)
site_name = models.CharField(max_length=120, blank=True)
favicon_url = models.URLField(max_length=2048, blank=True)
provider = models.CharField(max_length=40, blank=True)
embed_url = models.URLField(max_length=2048, blank=True)
preview_mode = models.CharField(max_length=12, choices=[('preview', 'Preview'), ('plain', 'Plain link'), ('embed', 'Playable embed')], default='preview')
notes = models.TextField(blank=True)
tags = models.JSONField(default=list, blank=True)
saved_reason = models.CharField(max_length=300, blank=True)
deferral_reason = models.CharField(max_length=120, blank=True)
status = models.CharField(max_length=12, choices=Status.choices, default=Status.SAVED)
metadata_status = models.CharField(max_length=12, choices=MetadataStatus.choices, default=MetadataStatus.PENDING)
metadata_error = models.CharField(max_length=200, blank=True)
metadata_fetched_at = models.DateTimeField(null=True, blank=True)
metadata_retry_at = models.DateTimeField(null=True, blank=True)
metadata_attempts = models.PositiveSmallIntegerField(default=0)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
read_at = models.DateTimeField(null=True, blank=True)
last_surfaced_at = models.DateTimeField(null=True, blank=True)
snoozed_until = models.DateTimeField(null=True, blank=True)
class Meta:
constraints = [models.UniqueConstraint(fields=['owner', 'url_key'], name='knowledge_owner_url_unique')]
indexes = [models.Index(fields=['owner', 'status', '-created_at']), models.Index(fields=['owner', 'last_surfaced_at'])]
ordering = ['-created_at']
@property
def domain(self):
return urlsplit(self.original_url).hostname or self.original_url
class CaptureToken(models.Model):
owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='knowledge_tokens')
name = models.CharField(max_length=80)
token_hash = models.CharField(max_length=64, unique=True)
created_at = models.DateTimeField(auto_now_add=True)
revoked_at = models.DateTimeField(null=True, blank=True)
class RediscoveryEvent(models.Model):
item = models.ForeignKey(SavedItem, on_delete=models.CASCADE, related_name='rediscovery_events')
channel = models.CharField(max_length=12, choices=[('mcp', 'MCP'), ('ntfy', 'ntfy')])
outcome = models.CharField(max_length=20)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
indexes = [models.Index(fields=['item', '-created_at'])]
+83
View File
@@ -0,0 +1,83 @@
import random
import socket
from datetime import timedelta
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from django.conf import settings
from django.contrib.auth import get_user_model
from django.db import transaction
from django.db import models
from django.utils import timezone
from .models import RediscoveryEvent, SavedItem
class NotificationError(Exception):
pass
def send_ntfy_notification(url, title, message, timeout=10):
# Matches TasksNest's UTF-8 POST + Title header convention. URL comes from server environment.
request = Request(url, data=message.encode('utf-8'),
headers={'Content-Type': 'text/plain; charset=utf-8', 'Title': title}, method='POST')
try:
with urlopen(request, timeout=timeout) as response:
status = response.getcode()
except HTTPError as error:
raise NotificationError(f'ntfy responded with HTTP {error.code}.') from error
except (TimeoutError, socket.timeout, URLError, OSError) as error:
raise NotificationError('Could not connect to ntfy.') from error
if not 200 <= status < 300:
raise NotificationError(f'ntfy responded with HTTP {status}.')
def choose_item(user, mode='forgotten', now=None):
now = now or timezone.now()
cutoff = now - timedelta(days=14)
items = list(SavedItem.objects.filter(owner=user, status=SavedItem.Status.SAVED,
created_at__lte=now - timedelta(days=2)).filter(
models.Q(snoozed_until__isnull=True) | models.Q(snoozed_until__lte=now)).filter(
models.Q(last_surfaced_at__isnull=True) | models.Q(last_surfaced_at__lte=cutoff))[:500])
if not items:
return None
if mode == 'random':
return random.choice(items)
# "Timely" is deliberately recency based, not an unsupported claim about price or availability.
if mode == 'timely':
return max(items, key=lambda item: item.created_at)
return min(items, key=lambda item: item.last_surfaced_at or item.created_at)
def rediscover_once(now=None):
now = now or timezone.now()
if not settings.KNOWLEDGE_NTFY_ENABLED:
return 'disabled'
if not settings.KNOWLEDGE_NTFY_URL or not settings.KNOWLEDGE_NTFY_USER_ID:
return 'unconfigured'
local_hour = timezone.localtime(now).hour
start, end = settings.KNOWLEDGE_NTFY_QUIET_START, settings.KNOWLEDGE_NTFY_QUIET_END
in_quiet = start <= local_hour < end if start < end else local_hour >= start or local_hour < end
if in_quiet:
return 'quiet_hours'
user = get_user_model().objects.filter(pk=settings.KNOWLEDGE_NTFY_USER_ID).first()
if user is None:
return 'unknown_user'
week_start = now - timedelta(days=7)
delivered = RediscoveryEvent.objects.filter(item__owner=user, channel='ntfy', outcome='delivered', created_at__gte=week_start).count()
if delivered >= max(0, min(settings.KNOWLEDGE_NTFY_WEEKLY_LIMIT, 7)):
return 'weekly_limit'
mode = settings.KNOWLEDGE_NTFY_MODE if settings.KNOWLEDGE_NTFY_MODE in ('random', 'forgotten', 'timely') else 'forgotten'
item = choose_item(user, mode, now)
if item is None:
return 'no_eligible_items'
title = item.title or item.extracted_title or 'Saved link to revisit'
message = f'{title}\n{item.original_url}'
if item.saved_reason:
message += f'\nYou saved it because: {item.saved_reason}'
send_ntfy_notification(settings.KNOWLEDGE_NTFY_URL, 'Knowledge Library', message)
with transaction.atomic():
item.last_surfaced_at = now
item.save(update_fields=['last_surfaced_at', 'updated_at'])
RediscoveryEvent.objects.create(item=item, channel='ntfy', outcome='delivered')
return 'delivered'
+80
View File
@@ -0,0 +1,80 @@
import ipaddress
import re
from urllib.parse import urlsplit, urlunsplit
from django.core.exceptions import ValidationError
from django.db.models import Q
from django.utils import timezone
from datetime import timedelta
from .models import SavedItem
def clean_url(value):
value = (value or '').strip()
if len(value) > 2048 or any(ord(c) < 32 for c in value):
raise ValidationError('Enter a valid URL under 2048 characters.')
try:
parts = urlsplit(value)
host = parts.hostname
port = parts.port
except ValueError as exc:
raise ValidationError('Enter a valid URL.') from exc
if parts.scheme.lower() not in ('http', 'https') or not host or parts.username or parts.password:
raise ValidationError('Only public HTTP(S) URLs are accepted.')
host = host.lower().rstrip('.')
if '.' not in host and ':' not in host:
raise ValidationError('A public hostname is required.')
try:
address = ipaddress.ip_address(host.strip('[]'))
except ValueError:
address = None
if address and not address.is_global:
raise ValidationError('Local and private addresses are not accepted.')
if host in ('localhost', 'localhost.localdomain') or host.endswith(('.local', '.internal', '.localhost')):
raise ValidationError('Local hostnames are not accepted.')
authority = host if ':' not in host else f'[{host}]'
if port and port != (443 if parts.scheme.lower() == 'https' else 80):
authority += f':{port}'
key = urlunsplit((parts.scheme.lower(), authority, parts.path or '/', parts.query, ''))
if len(key) > 2048:
raise ValidationError('URL is too long.')
return key
def save_item(owner, url, **fields):
key = clean_url(url)
allowed = {'title', 'notes', 'saved_reason', 'deferral_reason', 'tags'}
defaults = {name: value for name, value in fields.items() if name in allowed}
item, created = SavedItem.objects.get_or_create(owner=owner, url_key=key,
defaults={'original_url': url.strip(), **defaults})
return item, created
def search_items(owner, query, limit=20):
words = re.findall(r'[\w-]{2,}', (query or '').lower())[:8]
qs = SavedItem.objects.filter(owner=owner).exclude(status=SavedItem.Status.ARCHIVED)
for word in words:
qs = qs.filter(Q(title__icontains=word) | Q(extracted_title__icontains=word) |
Q(original_url__icontains=word) | Q(description__icontains=word) |
Q(notes__icontains=word) | Q(saved_reason__icontains=word) | Q(tags__icontains=word))
return list(qs[:max(1, min(limit, 100))])
def relevant_items(owner, context):
words = set(re.findall(r'[\w-]{4,}', (context or '').lower()))
words -= {'this', 'that', 'with', 'from', 'about', 'what', 'would', 'could', 'should', 'there', 'their'}
if not words:
return []
now = timezone.now()
candidates = SavedItem.objects.filter(owner=owner, status=SavedItem.Status.SAVED).filter(
Q(snoozed_until__isnull=True) | Q(snoozed_until__lte=now)).filter(
Q(last_surfaced_at__isnull=True) | Q(last_surfaced_at__lte=now - timedelta(days=14)))
scored = []
for item in candidates[:500]:
text = ' '.join([item.title, item.extracted_title, item.description, item.saved_reason, ' '.join(item.tags)]).lower()
hits = words & set(re.findall(r'[\w-]{4,}', text))
if len(hits) >= 2 or (len(hits) == 1 and len(next(iter(hits))) >= 6):
scored.append((len(hits), item, sorted(hits)))
scored.sort(key=lambda row: (-row[0], -row[1].created_at.timestamp()))
return scored[:3]
+4
View File
@@ -0,0 +1,4 @@
.knowledge-form{margin-bottom:20px}.knowledge-fields{display:grid;gap:6px;margin-bottom:14px;max-width:680px}.knowledge-fields label,.knowledge-search label{font-weight:700}.knowledge-fields input,.knowledge-search input,.knowledge-actions input,.knowledge-actions textarea,.knowledge-actions select{width:100%;padding:9px;border:1px solid var(--border);border-radius:8px;background:var(--surface);color:var(--text)}.knowledge-search{max-width:680px}.knowledge-search>div{display:flex;gap:8px}.knowledge-list{display:grid;gap:14px;margin-top:20px}.knowledge-item{padding:0;overflow:hidden;display:grid;grid-template-columns:minmax(0,220px) minmax(0,1fr)}.knowledge-item:not(:has(.knowledge-image-wrap)){grid-template-columns:1fr}.knowledge-image-wrap{background:var(--surface-secondary);min-height:180px;display:grid;place-items:center}.knowledge-image{width:100%;height:100%;max-height:300px;object-fit:cover}.knowledge-image-fallback{color:var(--text-secondary)}.knowledge-body{padding:20px;min-width:0}.knowledge-site{font-size:11px;color:var(--text-secondary);font-weight:700}.knowledge-body h2{font-size:19px;overflow-wrap:anywhere}.knowledge-description,.knowledge-reason{color:var(--text-secondary)}.knowledge-state{font-size:11px;color:var(--text-muted)}.knowledge-actions{display:flex;gap:12px;align-items:start;flex-wrap:wrap;margin-top:14px}.knowledge-actions button,.knowledge-actions summary,.knowledge-play{border:1px solid var(--border);border-radius:7px;padding:7px 10px;background:var(--surface-secondary);color:var(--text);cursor:pointer}.knowledge-actions details{min-width:min(100%,250px)}.knowledge-actions details[open]{width:min(100%,450px)}.knowledge-actions details form{display:grid;gap:10px;padding-top:12px}.knowledge-actions label{display:grid;gap:4px;font-weight:700}.knowledge-actions textarea{min-height:75px}.knowledge-player{margin-top:12px}.knowledge-third-party{font-size:11px;color:var(--text-muted);margin:5px 0}.knowledge-player iframe{width:100%;aspect-ratio:16/9;border:0;border-radius:8px}@media(max-width:680px){.knowledge-item{grid-template-columns:1fr}.knowledge-image-wrap{height:190px;min-height:0}.knowledge-body{padding:16px}}@media(prefers-reduced-motion:reduce){.knowledge-item{transition:none}}
.knowledge-favicon{width:16px;height:16px;object-fit:contain;vertical-align:middle;margin-right:5px}
.knowledge-item--embed{grid-template-columns:1fr}.knowledge-item--embed .knowledge-image-wrap{max-height:330px}.knowledge-item--embed .knowledge-player{min-height:54px}
.knowledge-image-wrap{overflow:hidden}.knowledge-image{display:block}@media(max-width:680px){.knowledge-image-wrap{height:190px}.knowledge-image{height:190px;max-height:190px}}
+40
View File
@@ -0,0 +1,40 @@
document.addEventListener('DOMContentLoaded', () => {
document.addEventListener('error', (event) => {
if (event.target.matches?.('.knowledge-favicon')) { event.target.hidden = true; return; }
if (!event.target.matches?.('.knowledge-image')) return;
event.target.hidden = true;
const fallback = event.target.parentElement.querySelector('.knowledge-image-fallback');
if (fallback) fallback.hidden = false;
}, true);
document.addEventListener('click', (event) => {
const button = event.target.closest?.('.knowledge-play');
if (!button) return;
const player = button.closest('.knowledge-player');
const url = player?.dataset.embedUrl || '';
if (!/^https:\/\/www\.youtube-nocookie\.com\/embed\/[A-Za-z0-9_-]{11}$/.test(url)) return;
const frame = document.createElement('iframe');
frame.src = url;
frame.title = 'YouTube video player';
frame.referrerPolicy = 'no-referrer';
frame.allow = 'accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture; web-share';
frame.allowFullscreen = true;
player.replaceChildren(frame);
frame.focus();
});
const started = Date.now();
const poll = async () => {
if (document.hidden || Date.now() - started > 120000) return;
const cards = [...document.querySelectorAll('.knowledge-item[data-metadata-status="pending"]')].slice(0, 20);
for (const card of cards) {
try {
const response = await fetch(card.dataset.cardUrl, {credentials: 'same-origin', cache: 'no-store'});
if (!response.ok) continue;
const html = await response.text();
const documentFragment = new DOMParser().parseFromString(html, 'text/html');
const replacement = documentFragment.querySelector('.knowledge-item');
if (replacement && replacement.dataset.metadataStatus !== 'pending') card.replaceWith(replacement);
} catch (_) { /* Preserve the current card on a transient polling failure. */ }
}
};
const timer = setInterval(() => { if (Date.now() - started > 120000) clearInterval(timer); else poll(); }, 10000);
});
@@ -0,0 +1,12 @@
<article class="widget-card knowledge-item{% if item.preview_mode == 'embed' %} knowledge-item--embed{% endif %}" id="knowledge-item-{{ item.id }}" data-card-url="{% url 'knowledge:item_card' item.id %}" data-metadata-status="{{ item.metadata_status }}">
{% if item.preview_mode != 'plain' and item.thumbnail_url %}<div class="knowledge-image-wrap"><img class="knowledge-image" src="{{ item.thumbnail_url }}" alt="{{ item.image_alt|default:'Preview image for saved link' }}" loading="lazy" referrerpolicy="no-referrer"><span class="knowledge-image-fallback" hidden>Image unavailable</span></div>{% endif %}
<div class="knowledge-body">
<p class="knowledge-site">{% if item.favicon_url and item.preview_mode != 'plain' %}<img class="knowledge-favicon" src="{{ item.favicon_url }}" alt="" loading="lazy" referrerpolicy="no-referrer">{% endif %}{% if item.site_name %}{{ item.site_name }}{% else %}{{ item.domain }}{% endif %}{% if item.provider == 'youtube' %} · YouTube{% endif %}</p>
<h2><a href="{{ item.original_url }}" target="_blank" rel="noopener noreferrer">{{ item.title|default:item.extracted_title|default:item.original_url }}</a></h2>
{% if item.preview_mode != 'plain' %}{% if item.user_description or item.description %}<p class="knowledge-description">{{ item.user_description|default:item.description }}</p>{% endif %}{% endif %}
{% if item.saved_reason %}<p class="knowledge-reason">Saved because: {{ item.saved_reason }}</p>{% endif %}
<p class="knowledge-state"><small>Saved {{ item.created_at|date:'j M Y' }} · {% if item.metadata_status == 'pending' %}Preview loading{% elif item.metadata_status == 'error' %}Preview unavailable{% if item.metadata_error %}: {{ item.metadata_error }}{% endif %}{% else %}Preview updated {{ item.metadata_fetched_at|date:'j M Y' }}{% endif %}</small></p>
{% if item.provider == 'youtube' and item.preview_mode != 'plain' and item.embed_url %}<div class="knowledge-player" data-embed-url="{{ item.embed_url }}"><button type="button" class="knowledge-play">Play video</button><p class="knowledge-third-party">Playing loads YouTube content.</p></div>{% endif %}
<div class="knowledge-actions"><details><summary>Edit preview</summary><form method="post" action="{% url 'knowledge:edit' item.id %}">{% csrf_token %}<label>Your title<input name="title" value="{{ item.title }}" maxlength="300"></label><label>Your description<textarea name="description" maxlength="2000">{{ item.user_description }}</textarea></label><label>Display mode<select name="preview_mode"><option value="preview" {% if item.preview_mode == 'preview' %}selected{% endif %}>Preview</option><option value="plain" {% if item.preview_mode == 'plain' %}selected{% endif %}>Plain link</option>{% if item.provider == 'youtube' %}<option value="embed" {% if item.preview_mode == 'embed' %}selected{% endif %}>Playable embed</option>{% endif %}</select></label><button type="submit">Save changes</button></form></details><form method="post" action="{% url 'knowledge:refresh' item.id %}">{% csrf_token %}<button type="submit">Refresh preview</button></form><form method="post" action="{% url 'knowledge:archive' item.id %}">{% csrf_token %}<button type="submit">Archive</button></form></div>
</div>
</article>
+10
View File
@@ -0,0 +1,10 @@
{% extends 'core/base.html' %}{% load static %}
{% block extra_head %}<link rel="stylesheet" href="{% static 'knowledge/previews.css' %}"><script src="{% static 'knowledge/previews.js' %}" defer></script>{% endblock %}
{% block content %}
<div class="page-heading"><div><h1>Knowledge Library</h1><p>Save things you want to revisit when the time is right.</p></div><a class="button" href="{% url 'knowledge:tokens' %}">Capture tokens</a></div>
<section class="widget-card knowledge-form"><h2>Save a link</h2>
{% if error %}<p role="alert">{{ error }}</p>{% endif %}
<form method="post" action="{% url 'knowledge:add' %}">{% csrf_token %}<div class="knowledge-fields"><label for="save-url">URL</label><input id="save-url" name="url" type="url" required maxlength="2048" value="{{ form_url }}"><label for="save-title">Your title (optional)</label><input id="save-title" name="title" maxlength="300"><label for="save-reason">Why save it? (optional)</label><input id="save-reason" name="saved_reason" maxlength="300"></div><button class="button" type="submit">Save link</button></form></section>
<form method="get" role="search" class="knowledge-search"><label for="knowledge-search">Search saved links</label><div><input id="knowledge-search" type="search" name="q" value="{{ query }}"><button class="button" type="submit">Search</button></div></form>
<div class="knowledge-list">{% for item in items %}{% include 'knowledge/_item_card.html' %}{% empty %}<p>No saved links found.</p>{% endfor %}</div>
{% endblock %}
@@ -0,0 +1,5 @@
{% extends 'core/base.html' %}
{% block content %}<h1>Capture tokens</h1><p>Create a token for an Apple Shortcut that sends a URL to this library. Tokens only permit saving links.</p>
{% if secret %}<div class="widget-card" role="status"><strong>Copy this token now; it will not be shown again.</strong><p><code style="overflow-wrap:anywhere">{{ secret }}</code></p></div>{% endif %}
<form method="post">{% csrf_token %}<label for="token-name">Token name</label><input id="token-name" name="name" value="Apple Shortcut" maxlength="80"><button class="button" type="submit">Create token</button></form>
<h2>Existing tokens</h2><ul>{% for token in tokens %}<li>{{ token.name }} · {{ token.created_at|date:'j M Y' }} {% if token.revoked_at %}(revoked){% else %}<form method="post" style="display:inline">{% csrf_token %}<input type="hidden" name="action" value="revoke"><input type="hidden" name="token_id" value="{{ token.id }}"><button type="submit">Revoke</button></form>{% endif %}</li>{% empty %}<li>No tokens yet.</li>{% endfor %}</ul><a href="{% url 'knowledge:index' %}">Back to library</a>{% endblock %}
+183
View File
@@ -0,0 +1,183 @@
import hashlib
import json
from django.contrib.auth import get_user_model
from django.test import TestCase
from django.urls import reverse
from django.core.management import call_command
from django.test import override_settings
from django.utils import timezone
from datetime import timedelta
from unittest.mock import patch
from .models import CaptureToken, SavedItem
from .services import relevant_items, save_item
from .notifications import rediscover_once
from .metadata import MetadataError, fetch_metadata, public_address, youtube_video_id
class KnowledgeTests(TestCase):
def setUp(self):
self.alice = get_user_model().objects.create_user('alice', password='test-password')
self.bob = get_user_model().objects.create_user('bob', password='test-password')
def test_capture_is_idempotent_and_owner_scoped(self):
token = 'capture-secret'
CaptureToken.objects.create(owner=self.alice, name='Shortcut', token_hash=hashlib.sha256(token.encode()).hexdigest())
url = reverse('knowledge_api_add')
headers = {'HTTP_AUTHORIZATION': f'Bearer {token}', 'content_type': 'application/json'}
first = self.client.post(url, data=json.dumps({'url': 'https://example.org/esp32#top', 'title': 'ESP32'}), **headers)
second = self.client.post(url, data=json.dumps({'url': 'https://EXAMPLE.org/esp32'}), **headers)
self.assertEqual(first.status_code, 201)
self.assertEqual(second.status_code, 200)
self.assertTrue(second.json()['duplicate'])
self.assertEqual(SavedItem.objects.filter(owner=self.alice).count(), 1)
self.assertEqual(SavedItem.objects.filter(owner=self.bob).count(), 0)
self.assertEqual(self.client.post(url, data='{}', content_type='application/json').status_code, 401)
def test_private_url_and_cross_user_visibility(self):
token = 'capture-secret'
CaptureToken.objects.create(owner=self.alice, name='Shortcut', token_hash=hashlib.sha256(token.encode()).hexdigest())
response = self.client.post(reverse('knowledge_api_add'), data=json.dumps({'url': 'http://127.0.0.1/admin'}),
content_type='application/json', HTTP_AUTHORIZATION=f'Bearer {token}')
self.assertEqual(response.status_code, 400)
item, _ = save_item(self.alice, 'https://example.org/sensor', title='ESP32 sensor microphone')
self.assertEqual(relevant_items(self.bob, 'ESP32 sensor microphone'), [])
self.assertEqual(relevant_items(self.alice, 'ESP32 sensor microphone')[0][1], item)
self.client.force_login(self.bob)
self.assertNotContains(self.client.get(reverse('knowledge:index')), 'ESP32 sensor microphone')
self.assertEqual(self.client.post(reverse('knowledge:archive', args=[item.pk])).status_code, 404)
def test_token_revocation(self):
self.client.force_login(self.alice)
self.client.post(reverse('knowledge:tokens'), {'name': 'Shortcut'})
token = CaptureToken.objects.get(owner=self.alice)
self.client.post(reverse('knowledge:tokens'), {'action': 'revoke', 'token_id': token.pk})
token.refresh_from_db()
self.assertIsNotNone(token.revoked_at)
@override_settings(KNOWLEDGE_NTFY_ENABLED=True, KNOWLEDGE_NTFY_URL='https://ntfy.example.org/topic',
KNOWLEDGE_NTFY_WEEKLY_LIMIT=1, KNOWLEDGE_NTFY_QUIET_START=23, KNOWLEDGE_NTFY_QUIET_END=6)
@patch('knowledge.notifications.send_ntfy_notification')
def test_ntfy_limit_and_cooldown(self, sender):
item, _ = save_item(self.alice, 'https://example.org/esp32', title='ESP32 sensors')
SavedItem.objects.filter(pk=item.pk).update(created_at=timezone.now() - timedelta(days=5))
with override_settings(KNOWLEDGE_NTFY_USER_ID=str(self.alice.pk)):
now = timezone.now().replace(hour=12)
self.assertEqual(rediscover_once(now), 'delivered')
self.assertEqual(rediscover_once(now), 'weekly_limit')
sender.assert_called_once()
@patch('knowledge.metadata.socket.getaddrinfo')
def test_metadata_rejects_private_dns_answer(self, resolver):
resolver.return_value = [(None, None, None, None, ('192.168.1.1', 443))]
with self.assertRaises(MetadataError):
public_address('example.org', 443)
@patch('knowledge.metadata.PinnedHTTPConnection')
@patch('knowledge.metadata.public_address', return_value='93.184.215.14')
def test_metadata_reads_only_bounded_html(self, resolver, connection_type):
response = connection_type.return_value.getresponse.return_value
response.status = 200
response.getheader.side_effect = lambda name: {'Content-Type': 'text/html; charset=utf-8', 'Content-Length': '90'}.get(name)
response.read.side_effect = [b'<title>Useful ESP32 page</title><meta name="description" content="A sensor guide">', b'']
result = fetch_metadata('https://example.org/esp32')
self.assertEqual(result['extracted_title'], 'Useful ESP32 page')
self.assertEqual(result['description'], 'A sensor guide')
connection_type.assert_called_once_with('example.org', 443, '93.184.215.14', True)
@patch('knowledge.metadata._fetch_bytes')
def test_open_graph_precedence_and_fallback(self, fetch):
html = b'''<title>HTML title</title><meta name="description" content="HTML description">
<meta name="twitter:title" content="Twitter title"><meta name="twitter:image" content="https://example.org/tw.jpg">
<meta property="og:title" content="OG title"><meta property="og:description" content="OG description">
<meta property="og:image" content="https://example.org/og.jpg"><meta property="og:image:alt" content="A flower">
<meta property="og:site_name" content="Example Journal"><link rel="icon" href="/favicon.png">'''
fetch.return_value = ('https://example.org/article', html)
result = fetch_metadata('https://example.org/article')
self.assertEqual(result['extracted_title'], 'OG title')
self.assertEqual(result['description'], 'OG description')
self.assertEqual(result['thumbnail_url'], 'https://example.org/og.jpg')
self.assertEqual(result['image_alt'], 'A flower')
self.assertEqual(result['site_name'], 'Example Journal')
self.assertEqual(result['favicon_url'], 'https://example.org/favicon.png')
fetch.return_value = ('https://example.org/article', b'<title>HTML only</title>')
self.assertEqual(fetch_metadata('https://example.org/article')['extracted_title'], 'HTML only')
fetch.return_value = ('https://example.org/article', b'<html><head></head><body>Nothing to extract</body></html>')
self.assertEqual(fetch_metadata('https://example.org/article')['extracted_title'], '')
@patch('knowledge.metadata._fetch_bytes')
def test_youtube_oembed_uses_allowlisted_embed(self, fetch):
fetch.return_value = ('https://www.youtube.com/oembed', json.dumps({
'title': 'Circuit demo', 'author_name': 'Maker channel',
'thumbnail_url': 'https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg'}).encode())
result = fetch_metadata('https://youtu.be/dQw4w9WgXcQ')
self.assertEqual(result['site_name'], 'Maker channel')
self.assertEqual(result['embed_url'], 'https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ')
self.assertEqual(youtube_video_id('https://youtube.com.evil.test/watch?v=dQw4w9WgXcQ'), '')
@patch('knowledge.metadata.PinnedHTTPConnection')
@patch('knowledge.metadata.public_address', return_value='93.184.215.14')
def test_redirect_to_private_address_is_blocked(self, resolver, connection_type):
response = connection_type.return_value.getresponse.return_value
response.status = 302
response.getheader.return_value = 'http://127.0.0.1/admin'
with self.assertRaises(MetadataError):
fetch_metadata('https://example.org/article')
@patch('knowledge.metadata.PinnedHTTPConnection')
@patch('knowledge.metadata.public_address', return_value='93.184.215.14')
def test_timeout_is_a_safe_failure(self, resolver, connection_type):
import socket
connection_type.return_value.request.side_effect = socket.timeout()
with self.assertRaises(MetadataError):
fetch_metadata('https://example.org/article')
@patch('knowledge.management.commands.fetch_knowledge_metadata.fetch_metadata')
def test_metadata_refresh_preserves_user_edits_and_cache(self, fetch):
item, _ = save_item(self.alice, 'https://example.org/article')
self.client.force_login(self.alice)
self.client.post(reverse('knowledge:edit', args=[item.pk]), {
'title': 'My title', 'description': 'My notes on this', 'preview_mode': 'preview'})
fetch.return_value = {'extracted_title': 'Publisher title', 'description': 'Publisher description',
'thumbnail_url': 'https://example.org/image.jpg', 'image_alt': 'Image', 'site_name': 'Example',
'favicon_url': '', 'provider': 'web', 'embed_url': '', 'canonical_url': ''}
call_command('fetch_knowledge_metadata')
item.refresh_from_db()
self.assertEqual(item.title, 'My title')
self.assertEqual(item.user_description, 'My notes on this')
self.assertEqual(item.description, 'Publisher description')
self.assertEqual(item.metadata_status, SavedItem.MetadataStatus.READY)
call_command('fetch_knowledge_metadata')
self.assertEqual(fetch.call_count, 1)
self.assertContains(self.client.get(reverse('knowledge:index')), 'My title')
self.assertContains(self.client.get(reverse('knowledge:index')), 'My notes on this')
def test_preview_escapes_metadata_and_defers_youtube_embed(self):
item, _ = save_item(self.alice, 'https://youtu.be/dQw4w9WgXcQ')
item.provider = 'youtube'
item.embed_url = 'https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ'
item.extracted_title = '<script>alert(1)</script>'
item.thumbnail_url = 'https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg'
item.save()
self.client.force_login(self.alice)
response = self.client.get(reverse('knowledge:index'))
self.assertContains(response, '&lt;script&gt;alert(1)&lt;/script&gt;')
self.assertNotContains(response, '<iframe')
self.assertContains(response, 'Play video')
self.assertContains(response, 'knowledge-image-fallback')
self.client.force_login(self.bob)
self.assertEqual(self.client.get(reverse('knowledge:item_card', args=[item.pk])).status_code, 404)
@patch('knowledge.management.commands.fetch_knowledge_metadata.fetch_metadata', side_effect=MetadataError('HTTP 503.'))
def test_worker_retries_with_backoff_and_keeps_saved_link(self, fetch):
item, _ = save_item(self.alice, 'https://example.org/broken')
call_command('fetch_knowledge_metadata')
item.refresh_from_db()
self.assertEqual(item.metadata_status, SavedItem.MetadataStatus.ERROR)
self.assertEqual(item.metadata_attempts, 1)
self.assertGreater(item.metadata_retry_at, timezone.now())
call_command('fetch_knowledge_metadata')
self.assertEqual(fetch.call_count, 1)
self.client.force_login(self.alice)
self.assertContains(self.client.get(reverse('knowledge:index')), 'Preview unavailable')
self.assertContains(self.client.get(reverse('knowledge:index')), 'https://example.org/broken')
+13
View File
@@ -0,0 +1,13 @@
from django.urls import path
from . import views
app_name = 'knowledge'
urlpatterns = [
path('', views.index, name='index'),
path('add/', views.add, name='add'),
path('items/<uuid:item_id>/archive/', views.archive, name='archive'),
path('items/<uuid:item_id>/card/', views.item_card, name='item_card'),
path('items/<uuid:item_id>/edit/', views.edit, name='edit'),
path('items/<uuid:item_id>/refresh/', views.refresh, name='refresh'),
path('tokens/', views.tokens, name='tokens'),
]
+129
View File
@@ -0,0 +1,129 @@
import hashlib
import json
import secrets
from datetime import timedelta
from django.contrib.auth.decorators import login_required
from django.core.exceptions import ValidationError
from django.core.cache import cache
from django.http import JsonResponse
from django.shortcuts import get_object_or_404, redirect, render
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from .models import CaptureToken, SavedItem
from .services import save_item, search_items
@login_required
def index(request):
query = request.GET.get('q', '').strip()[:200]
items = search_items(request.user, query, 100) if query else SavedItem.objects.filter(owner=request.user).exclude(status=SavedItem.Status.ARCHIVED)[:100]
return render(request, 'knowledge/index.html', {'title': 'Knowledge Library', 'section': 'knowledge', 'items': items, 'query': query})
@login_required
def item_card(request, item_id):
item = get_object_or_404(SavedItem, pk=item_id, owner=request.user)
response = render(request, 'knowledge/_item_card.html', {'item': item})
response['Cache-Control'] = 'private, no-store'
return response
@login_required
@require_POST
def add(request):
try:
item, created = save_item(request.user, request.POST.get('url'),
title=request.POST.get('title', '').strip()[:300],
notes=request.POST.get('notes', '').strip()[:3000],
saved_reason=request.POST.get('saved_reason', '').strip()[:300])
except ValidationError as exc:
return render(request, 'knowledge/index.html', {'title': 'Knowledge Library', 'section': 'knowledge',
'items': SavedItem.objects.filter(owner=request.user)[:100], 'error': exc.messages[0], 'form_url': request.POST.get('url', '')}, status=400)
return redirect('knowledge:index')
@login_required
@require_POST
def archive(request, item_id):
item = get_object_or_404(SavedItem, pk=item_id, owner=request.user)
item.status = SavedItem.Status.ARCHIVED
item.save(update_fields=['status', 'updated_at'])
return redirect('knowledge:index')
@login_required
@require_POST
def edit(request, item_id):
item = get_object_or_404(SavedItem, pk=item_id, owner=request.user)
mode = request.POST.get('preview_mode', 'preview')
if mode not in ('preview', 'plain', 'embed') or (mode == 'embed' and item.provider != 'youtube'):
return JsonResponse({'error': 'Invalid preview mode'}, status=400)
item.title = request.POST.get('title', '').strip()[:300]
item.user_description = request.POST.get('description', '').strip()[:2000]
item.preview_mode = mode
item.save(update_fields=['title', 'user_description', 'preview_mode', 'updated_at'])
return redirect('knowledge:index')
@login_required
@require_POST
def refresh(request, item_id):
item = get_object_or_404(SavedItem, pk=item_id, owner=request.user)
if item.metadata_fetched_at and item.metadata_fetched_at > timezone.now() - timedelta(minutes=5):
return redirect('knowledge:index')
item.metadata_status = SavedItem.MetadataStatus.PENDING
item.metadata_retry_at = None
item.metadata_error = ''
item.metadata_attempts = 0
item.save(update_fields=['metadata_status', 'metadata_retry_at', 'metadata_error', 'metadata_attempts', 'updated_at'])
return redirect('knowledge:index')
@login_required
def tokens(request):
secret = None
if request.method == 'POST':
if request.POST.get('action') == 'revoke':
token = get_object_or_404(CaptureToken, pk=request.POST.get('token_id'), owner=request.user)
token.revoked_at = timezone.now()
token.save(update_fields=['revoked_at'])
else:
secret = secrets.token_urlsafe(32)
CaptureToken.objects.create(owner=request.user, name=request.POST.get('name', 'Apple Shortcut')[:80],
token_hash=hashlib.sha256(secret.encode()).hexdigest())
return render(request, 'knowledge/tokens.html', {'title': 'Capture tokens', 'section': 'knowledge',
'tokens': CaptureToken.objects.filter(owner=request.user).order_by('-created_at'), 'secret': secret})
@csrf_exempt
@require_POST
def api_add(request):
if len(request.body) > 16_384:
return JsonResponse({'error': 'Request too large'}, status=413)
header = request.headers.get('Authorization', '')
if not header.startswith('Bearer ') or len(header) > 180:
return JsonResponse({'error': 'Bearer capture token required'}, status=401)
token_hash = hashlib.sha256(header[7:].encode()).hexdigest()
token = CaptureToken.objects.select_related('owner').filter(token_hash=token_hash, revoked_at__isnull=True).first()
if token is None:
return JsonResponse({'error': 'Invalid capture token'}, status=401)
rate_key = f'knowledge-capture:{token.pk}:{timezone.now():%Y%m%d%H%M}'
if not cache.add(rate_key, 1, 90):
try:
if cache.incr(rate_key) > 30:
return JsonResponse({'error': 'Too many captures; try again shortly'}, status=429)
except ValueError:
cache.set(rate_key, 1, 90)
try:
payload = json.loads(request.body)
if not isinstance(payload, dict):
raise ValueError()
item, created = save_item(token.owner, payload.get('url'),
title=str(payload.get('title') or '')[:300], notes=str(payload.get('notes') or '')[:3000],
saved_reason=str(payload.get('saved_reason') or '')[:300])
except (ValueError, TypeError, ValidationError) as exc:
return JsonResponse({'error': str(exc) or 'Invalid JSON'}, status=400)
return JsonResponse({'id': str(item.pk), 'created': created, 'duplicate': not created,
'url': item.original_url, 'metadata_status': item.metadata_status}, status=201 if created else 200)