link preview v1
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class KnowledgeConfig(AppConfig):
|
||||
default_auto_field = 'django.db.models.BigAutoField'
|
||||
name = 'knowledge'
|
||||
@@ -0,0 +1,40 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
from datetime import timedelta
|
||||
from knowledge.metadata import MetadataError, fetch_metadata
|
||||
from knowledge.models import SavedItem
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Fetch safe, bounded metadata for pending saved links.'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('--limit', type=int, default=20)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
total = 0
|
||||
now = timezone.now()
|
||||
candidates = SavedItem.objects.filter(metadata_attempts__lt=5).filter(
|
||||
Q(metadata_status=SavedItem.MetadataStatus.PENDING) |
|
||||
Q(metadata_status=SavedItem.MetadataStatus.ERROR, metadata_retry_at__lte=now)).order_by('created_at')
|
||||
for item in candidates[:max(0, min(options['limit'], 100))]:
|
||||
item.metadata_attempts += 1
|
||||
try:
|
||||
fields = fetch_metadata(item.original_url)
|
||||
for name, value in fields.items():
|
||||
setattr(item, name, value)
|
||||
item.metadata_status = SavedItem.MetadataStatus.READY
|
||||
item.metadata_error = ''
|
||||
item.metadata_fetched_at = timezone.now()
|
||||
item.metadata_retry_at = None
|
||||
item.save(update_fields=[*fields, 'metadata_status', 'metadata_error', 'metadata_fetched_at',
|
||||
'metadata_retry_at', 'metadata_attempts', 'updated_at'])
|
||||
except MetadataError as exc:
|
||||
item.metadata_status = SavedItem.MetadataStatus.ERROR
|
||||
item.metadata_error = str(exc)[:200]
|
||||
item.metadata_retry_at = timezone.now() + timedelta(minutes=min(5 * 2 ** (item.metadata_attempts - 1), 1440))
|
||||
item.save(update_fields=['metadata_status', 'metadata_error', 'metadata_retry_at',
|
||||
'metadata_attempts', 'updated_at'])
|
||||
total += 1
|
||||
self.stdout.write(f'Processed {total} link(s).')
|
||||
@@ -0,0 +1,12 @@
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from knowledge.notifications import NotificationError, rediscover_once
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Send at most one eligible Knowledge Library ntfy reminder, subject to opt-in, quiet hours and weekly limit.'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
try:
|
||||
self.stdout.write(rediscover_once())
|
||||
except NotificationError as exc:
|
||||
raise CommandError(str(exc)) from exc
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Bounded, DNS-pinned preview metadata for public web pages and YouTube."""
|
||||
import http.client
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import ssl
|
||||
import time
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import parse_qs, urlencode, urljoin, urlsplit
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
|
||||
from .services import clean_url
|
||||
|
||||
|
||||
MAX_HTML_BYTES = 512 * 1024
|
||||
MAX_JSON_BYTES = 16 * 1024
|
||||
MAX_REDIRECTS = 3
|
||||
VIDEO_ID = re.compile(r'^[A-Za-z0-9_-]{11}$')
|
||||
|
||||
|
||||
class MetadataError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class MetadataParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title_parts = []
|
||||
self.in_title = False
|
||||
self.meta = {}
|
||||
self.icons = []
|
||||
self.canonical = ''
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
props = dict(attrs)
|
||||
if tag == 'title':
|
||||
self.in_title = True
|
||||
elif tag == 'meta':
|
||||
name = (props.get('property') or props.get('name') or '').lower().strip()
|
||||
value = (props.get('content') or '').strip()
|
||||
if name and value and name not in self.meta:
|
||||
self.meta[name] = value
|
||||
elif tag == 'link':
|
||||
rel = (props.get('rel') or '').lower().split()
|
||||
if 'canonical' in rel and not self.canonical:
|
||||
self.canonical = props.get('href') or ''
|
||||
if 'icon' in rel and props.get('href'):
|
||||
self.icons.append(props['href'])
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == 'title':
|
||||
self.in_title = False
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.in_title and sum(map(len, self.title_parts)) < 300:
|
||||
self.title_parts.append(data)
|
||||
|
||||
|
||||
class PinnedHTTPConnection(http.client.HTTPConnection):
|
||||
def __init__(self, host, port, address, secure):
|
||||
super().__init__(host, port, timeout=5)
|
||||
self.address = address
|
||||
self.secure = secure
|
||||
|
||||
def connect(self):
|
||||
raw = socket.create_connection((self.address, self.port), timeout=self.timeout)
|
||||
self.sock = ssl.create_default_context().wrap_socket(raw, server_hostname=self.host) if self.secure else raw
|
||||
|
||||
|
||||
def public_address(host, port):
|
||||
try:
|
||||
answers = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||
except OSError as exc:
|
||||
raise MetadataError('DNS lookup failed.') from exc
|
||||
if not answers:
|
||||
raise MetadataError('No DNS address found.')
|
||||
addresses = [answer[4][0] for answer in answers]
|
||||
if any(not ipaddress.ip_address(address).is_global for address in addresses):
|
||||
raise MetadataError('DNS resolved to a non-public address.')
|
||||
return addresses[0]
|
||||
|
||||
|
||||
def _public_url(value, base):
|
||||
try:
|
||||
candidate = clean_url(urljoin(base, value))
|
||||
except (ValidationError, ValueError, TypeError):
|
||||
return ''
|
||||
return candidate if urlsplit(candidate).scheme == 'https' else ''
|
||||
|
||||
|
||||
def _fetch_bytes(url, accepted_type, limit, max_redirects=MAX_REDIRECTS):
|
||||
try:
|
||||
current = clean_url(url)
|
||||
except ValidationError as exc:
|
||||
raise MetadataError('Invalid page URL.') from exc
|
||||
deadline = time.monotonic() + 20
|
||||
for _ in range(max_redirects + 1):
|
||||
if time.monotonic() >= deadline:
|
||||
raise MetadataError('Preview fetch timed out.')
|
||||
parts = urlsplit(current)
|
||||
port = parts.port or (443 if parts.scheme == 'https' else 80)
|
||||
if port not in (80, 443):
|
||||
raise MetadataError('Only standard web ports are fetched.')
|
||||
address = public_address(parts.hostname, port)
|
||||
connection = PinnedHTTPConnection(parts.hostname, port, address, parts.scheme == 'https')
|
||||
try:
|
||||
target = (parts.path or '/') + (f'?{parts.query}' if parts.query else '')
|
||||
connection.request('GET', target, headers={'Host': parts.netloc, 'User-Agent': 'CommonGroundKnowledge/1.0',
|
||||
'Accept': accepted_type, 'Accept-Encoding': 'identity', 'Connection': 'close'})
|
||||
response = connection.getresponse()
|
||||
if response.status in (301, 302, 303, 307, 308):
|
||||
location = response.getheader('Location')
|
||||
if not location:
|
||||
raise MetadataError('Redirect has no location.')
|
||||
try:
|
||||
current = clean_url(urljoin(current, location))
|
||||
except ValidationError as exc:
|
||||
raise MetadataError('Redirect points to a disallowed URL.') from exc
|
||||
continue
|
||||
if response.status != 200:
|
||||
raise MetadataError(f'HTTP {response.status}.')
|
||||
content_type = (response.getheader('Content-Type') or '').lower().split(';', 1)[0].strip()
|
||||
if content_type != accepted_type:
|
||||
raise MetadataError('Unexpected content type.')
|
||||
length = response.getheader('Content-Length')
|
||||
if length and int(length) > limit:
|
||||
raise MetadataError('Response is too large.')
|
||||
chunks = []
|
||||
size = 0
|
||||
while True:
|
||||
if time.monotonic() >= deadline:
|
||||
raise MetadataError('Preview fetch timed out.')
|
||||
chunk = response.read(min(8192, limit + 1 - size))
|
||||
if not chunk:
|
||||
return current, b''.join(chunks)
|
||||
size += len(chunk)
|
||||
if size > limit:
|
||||
raise MetadataError('Response is too large.')
|
||||
chunks.append(chunk)
|
||||
except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc:
|
||||
raise MetadataError('Could not fetch preview metadata.') from exc
|
||||
finally:
|
||||
connection.close()
|
||||
raise MetadataError('Too many redirects.')
|
||||
|
||||
|
||||
def youtube_video_id(url):
|
||||
parts = urlsplit(url)
|
||||
host = (parts.hostname or '').lower()
|
||||
if host in ('youtube.com', 'www.youtube.com', 'm.youtube.com'):
|
||||
if parts.path == '/watch':
|
||||
value = parse_qs(parts.query).get('v', [''])[0]
|
||||
elif parts.path.startswith(('/shorts/', '/live/', '/embed/')):
|
||||
value = parts.path.split('/')[2]
|
||||
else:
|
||||
return ''
|
||||
elif host == 'youtu.be':
|
||||
value = parts.path.strip('/')
|
||||
else:
|
||||
return ''
|
||||
return value if VIDEO_ID.fullmatch(value or '') else ''
|
||||
|
||||
|
||||
def _youtube_metadata(url, video_id):
|
||||
endpoint = 'https://www.youtube.com/oembed?' + urlencode({'url': url, 'format': 'json'})
|
||||
_, body = _fetch_bytes(endpoint, 'application/json', MAX_JSON_BYTES, max_redirects=0)
|
||||
try:
|
||||
data = json.loads(body)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError()
|
||||
except (ValueError, UnicodeError) as exc:
|
||||
raise MetadataError('Invalid YouTube preview.') from exc
|
||||
image = _public_url(data.get('thumbnail_url'), url) if isinstance(data.get('thumbnail_url'), str) else ''
|
||||
return {'extracted_title': str(data.get('title') or '')[:300], 'description': '',
|
||||
'thumbnail_url': image or f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg',
|
||||
'image_alt': f'Thumbnail for {str(data.get("title") or "YouTube video")[:260]}',
|
||||
'site_name': str(data.get('author_name') or 'YouTube')[:120], 'favicon_url': '',
|
||||
'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}',
|
||||
'canonical_url': f'https://www.youtube.com/watch?v={video_id}'}
|
||||
|
||||
|
||||
def fetch_metadata(url):
|
||||
video_id = youtube_video_id(url)
|
||||
if video_id:
|
||||
try:
|
||||
return _youtube_metadata(url, video_id)
|
||||
except MetadataError:
|
||||
# A known video still gets a useful card when oEmbed is unavailable.
|
||||
return {'extracted_title': '', 'description': '',
|
||||
'thumbnail_url': f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg',
|
||||
'image_alt': 'YouTube video thumbnail', 'site_name': 'YouTube', 'favicon_url': '',
|
||||
'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}',
|
||||
'canonical_url': f'https://www.youtube.com/watch?v={video_id}'}
|
||||
final_url, body = _fetch_bytes(url, 'text/html', MAX_HTML_BYTES)
|
||||
parser = MetadataParser()
|
||||
parser.feed(body.decode('utf-8', errors='replace'))
|
||||
meta = parser.meta
|
||||
title = meta.get('og:title') or meta.get('twitter:title') or ''.join(parser.title_parts)
|
||||
description = meta.get('og:description') or meta.get('twitter:description') or meta.get('description') or ''
|
||||
image = meta.get('og:image') or meta.get('twitter:image') or ''
|
||||
canonical = _public_url(parser.canonical, final_url) if parser.canonical else ''
|
||||
if canonical and urlsplit(canonical).hostname != urlsplit(final_url).hostname:
|
||||
canonical = ''
|
||||
icon = _public_url(parser.icons[0], final_url) if parser.icons else ''
|
||||
return {'extracted_title': ' '.join(title.split())[:300], 'description': description[:2000],
|
||||
'thumbnail_url': _public_url(image, final_url) if image else '',
|
||||
'image_alt': (meta.get('og:image:alt') or meta.get('twitter:image:alt') or '')[:300],
|
||||
'site_name': (meta.get('og:site_name') or urlsplit(final_url).hostname or '')[:120],
|
||||
'favicon_url': icon or _public_url('/favicon.ico', final_url),
|
||||
'provider': 'web', 'embed_url': '', 'canonical_url': canonical}
|
||||
@@ -0,0 +1,83 @@
|
||||
# Generated by Django 6.1.2 on 2026-10-20 17:27
|
||||
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='CaptureToken',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('name', models.CharField(max_length=80)),
|
||||
('token_hash', models.CharField(max_length=64, unique=True)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('revoked_at', models.DateTimeField(blank=True, null=True)),
|
||||
('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='knowledge_tokens', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='SavedItem',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('original_url', models.URLField(max_length=2048)),
|
||||
('url_key', models.CharField(max_length=2048)),
|
||||
('canonical_url', models.URLField(blank=True, max_length=2048)),
|
||||
('title', models.CharField(blank=True, max_length=300)),
|
||||
('extracted_title', models.CharField(blank=True, max_length=300)),
|
||||
('description', models.TextField(blank=True)),
|
||||
('thumbnail_url', models.URLField(blank=True, max_length=2048)),
|
||||
('notes', models.TextField(blank=True)),
|
||||
('tags', models.JSONField(blank=True, default=list)),
|
||||
('saved_reason', models.CharField(blank=True, max_length=300)),
|
||||
('deferral_reason', models.CharField(blank=True, max_length=120)),
|
||||
('status', models.CharField(choices=[('saved', 'Saved'), ('read', 'Read'), ('archived', 'Archived')], default='saved', max_length=12)),
|
||||
('metadata_status', models.CharField(choices=[('pending', 'Pending'), ('ready', 'Ready'), ('error', 'Error')], default='pending', max_length=12)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('read_at', models.DateTimeField(blank=True, null=True)),
|
||||
('last_surfaced_at', models.DateTimeField(blank=True, null=True)),
|
||||
('snoozed_until', models.DateTimeField(blank=True, null=True)),
|
||||
('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='knowledge_items', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-created_at'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='RediscoveryEvent',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('channel', models.CharField(choices=[('mcp', 'MCP'), ('ntfy', 'ntfy')], max_length=12)),
|
||||
('outcome', models.CharField(max_length=20)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rediscovery_events', to='knowledge.saveditem')),
|
||||
],
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name='saveditem',
|
||||
index=models.Index(fields=['owner', 'status', '-created_at'], name='knowledge_s_owner_i_909a54_idx'),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name='saveditem',
|
||||
index=models.Index(fields=['owner', 'last_surfaced_at'], name='knowledge_s_owner_i_89f98e_idx'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='saveditem',
|
||||
constraint=models.UniqueConstraint(fields=('owner', 'url_key'), name='knowledge_owner_url_unique'),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name='rediscoveryevent',
|
||||
index=models.Index(fields=['item', '-created_at'], name='knowledge_r_item_id_0f6d7f_idx'),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,68 @@
|
||||
# Generated by Django 6.1.2 on 2026-10-20 17:46
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('knowledge', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='embed_url',
|
||||
field=models.URLField(blank=True, max_length=2048),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='favicon_url',
|
||||
field=models.URLField(blank=True, max_length=2048),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='image_alt',
|
||||
field=models.CharField(blank=True, max_length=300),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='metadata_attempts',
|
||||
field=models.PositiveSmallIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='metadata_error',
|
||||
field=models.CharField(blank=True, max_length=200),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='metadata_fetched_at',
|
||||
field=models.DateTimeField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='metadata_retry_at',
|
||||
field=models.DateTimeField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='preview_mode',
|
||||
field=models.CharField(choices=[('preview', 'Preview'), ('plain', 'Plain link'), ('embed', 'Playable embed')], default='preview', max_length=12),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='provider',
|
||||
field=models.CharField(blank=True, max_length=40),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='site_name',
|
||||
field=models.CharField(blank=True, max_length=120),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='saveditem',
|
||||
name='user_description',
|
||||
field=models.TextField(blank=True),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,75 @@
|
||||
import uuid
|
||||
from urllib.parse import urlsplit
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
|
||||
|
||||
class SavedItem(models.Model):
|
||||
class Status(models.TextChoices):
|
||||
SAVED = 'saved', 'Saved'
|
||||
READ = 'read', 'Read'
|
||||
ARCHIVED = 'archived', 'Archived'
|
||||
|
||||
class MetadataStatus(models.TextChoices):
|
||||
PENDING = 'pending', 'Pending'
|
||||
READY = 'ready', 'Ready'
|
||||
ERROR = 'error', 'Error'
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='knowledge_items')
|
||||
original_url = models.URLField(max_length=2048)
|
||||
url_key = models.CharField(max_length=2048)
|
||||
canonical_url = models.URLField(max_length=2048, blank=True)
|
||||
title = models.CharField(max_length=300, blank=True)
|
||||
user_description = models.TextField(blank=True)
|
||||
extracted_title = models.CharField(max_length=300, blank=True)
|
||||
description = models.TextField(blank=True)
|
||||
thumbnail_url = models.URLField(max_length=2048, blank=True)
|
||||
image_alt = models.CharField(max_length=300, blank=True)
|
||||
site_name = models.CharField(max_length=120, blank=True)
|
||||
favicon_url = models.URLField(max_length=2048, blank=True)
|
||||
provider = models.CharField(max_length=40, blank=True)
|
||||
embed_url = models.URLField(max_length=2048, blank=True)
|
||||
preview_mode = models.CharField(max_length=12, choices=[('preview', 'Preview'), ('plain', 'Plain link'), ('embed', 'Playable embed')], default='preview')
|
||||
notes = models.TextField(blank=True)
|
||||
tags = models.JSONField(default=list, blank=True)
|
||||
saved_reason = models.CharField(max_length=300, blank=True)
|
||||
deferral_reason = models.CharField(max_length=120, blank=True)
|
||||
status = models.CharField(max_length=12, choices=Status.choices, default=Status.SAVED)
|
||||
metadata_status = models.CharField(max_length=12, choices=MetadataStatus.choices, default=MetadataStatus.PENDING)
|
||||
metadata_error = models.CharField(max_length=200, blank=True)
|
||||
metadata_fetched_at = models.DateTimeField(null=True, blank=True)
|
||||
metadata_retry_at = models.DateTimeField(null=True, blank=True)
|
||||
metadata_attempts = models.PositiveSmallIntegerField(default=0)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
read_at = models.DateTimeField(null=True, blank=True)
|
||||
last_surfaced_at = models.DateTimeField(null=True, blank=True)
|
||||
snoozed_until = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
constraints = [models.UniqueConstraint(fields=['owner', 'url_key'], name='knowledge_owner_url_unique')]
|
||||
indexes = [models.Index(fields=['owner', 'status', '-created_at']), models.Index(fields=['owner', 'last_surfaced_at'])]
|
||||
ordering = ['-created_at']
|
||||
|
||||
@property
|
||||
def domain(self):
|
||||
return urlsplit(self.original_url).hostname or self.original_url
|
||||
|
||||
|
||||
class CaptureToken(models.Model):
|
||||
owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='knowledge_tokens')
|
||||
name = models.CharField(max_length=80)
|
||||
token_hash = models.CharField(max_length=64, unique=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
revoked_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
|
||||
class RediscoveryEvent(models.Model):
|
||||
item = models.ForeignKey(SavedItem, on_delete=models.CASCADE, related_name='rediscovery_events')
|
||||
channel = models.CharField(max_length=12, choices=[('mcp', 'MCP'), ('ntfy', 'ntfy')])
|
||||
outcome = models.CharField(max_length=20)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
indexes = [models.Index(fields=['item', '-created_at'])]
|
||||
@@ -0,0 +1,83 @@
|
||||
import random
|
||||
import socket
|
||||
from datetime import timedelta
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.db import transaction
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
|
||||
from .models import RediscoveryEvent, SavedItem
|
||||
|
||||
|
||||
class NotificationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def send_ntfy_notification(url, title, message, timeout=10):
|
||||
# Matches TasksNest's UTF-8 POST + Title header convention. URL comes from server environment.
|
||||
request = Request(url, data=message.encode('utf-8'),
|
||||
headers={'Content-Type': 'text/plain; charset=utf-8', 'Title': title}, method='POST')
|
||||
try:
|
||||
with urlopen(request, timeout=timeout) as response:
|
||||
status = response.getcode()
|
||||
except HTTPError as error:
|
||||
raise NotificationError(f'ntfy responded with HTTP {error.code}.') from error
|
||||
except (TimeoutError, socket.timeout, URLError, OSError) as error:
|
||||
raise NotificationError('Could not connect to ntfy.') from error
|
||||
if not 200 <= status < 300:
|
||||
raise NotificationError(f'ntfy responded with HTTP {status}.')
|
||||
|
||||
|
||||
def choose_item(user, mode='forgotten', now=None):
|
||||
now = now or timezone.now()
|
||||
cutoff = now - timedelta(days=14)
|
||||
items = list(SavedItem.objects.filter(owner=user, status=SavedItem.Status.SAVED,
|
||||
created_at__lte=now - timedelta(days=2)).filter(
|
||||
models.Q(snoozed_until__isnull=True) | models.Q(snoozed_until__lte=now)).filter(
|
||||
models.Q(last_surfaced_at__isnull=True) | models.Q(last_surfaced_at__lte=cutoff))[:500])
|
||||
if not items:
|
||||
return None
|
||||
if mode == 'random':
|
||||
return random.choice(items)
|
||||
# "Timely" is deliberately recency based, not an unsupported claim about price or availability.
|
||||
if mode == 'timely':
|
||||
return max(items, key=lambda item: item.created_at)
|
||||
return min(items, key=lambda item: item.last_surfaced_at or item.created_at)
|
||||
|
||||
|
||||
def rediscover_once(now=None):
|
||||
now = now or timezone.now()
|
||||
if not settings.KNOWLEDGE_NTFY_ENABLED:
|
||||
return 'disabled'
|
||||
if not settings.KNOWLEDGE_NTFY_URL or not settings.KNOWLEDGE_NTFY_USER_ID:
|
||||
return 'unconfigured'
|
||||
local_hour = timezone.localtime(now).hour
|
||||
start, end = settings.KNOWLEDGE_NTFY_QUIET_START, settings.KNOWLEDGE_NTFY_QUIET_END
|
||||
in_quiet = start <= local_hour < end if start < end else local_hour >= start or local_hour < end
|
||||
if in_quiet:
|
||||
return 'quiet_hours'
|
||||
user = get_user_model().objects.filter(pk=settings.KNOWLEDGE_NTFY_USER_ID).first()
|
||||
if user is None:
|
||||
return 'unknown_user'
|
||||
week_start = now - timedelta(days=7)
|
||||
delivered = RediscoveryEvent.objects.filter(item__owner=user, channel='ntfy', outcome='delivered', created_at__gte=week_start).count()
|
||||
if delivered >= max(0, min(settings.KNOWLEDGE_NTFY_WEEKLY_LIMIT, 7)):
|
||||
return 'weekly_limit'
|
||||
mode = settings.KNOWLEDGE_NTFY_MODE if settings.KNOWLEDGE_NTFY_MODE in ('random', 'forgotten', 'timely') else 'forgotten'
|
||||
item = choose_item(user, mode, now)
|
||||
if item is None:
|
||||
return 'no_eligible_items'
|
||||
title = item.title or item.extracted_title or 'Saved link to revisit'
|
||||
message = f'{title}\n{item.original_url}'
|
||||
if item.saved_reason:
|
||||
message += f'\nYou saved it because: {item.saved_reason}'
|
||||
send_ntfy_notification(settings.KNOWLEDGE_NTFY_URL, 'Knowledge Library', message)
|
||||
with transaction.atomic():
|
||||
item.last_surfaced_at = now
|
||||
item.save(update_fields=['last_surfaced_at', 'updated_at'])
|
||||
RediscoveryEvent.objects.create(item=item, channel='ntfy', outcome='delivered')
|
||||
return 'delivered'
|
||||
@@ -0,0 +1,80 @@
|
||||
import ipaddress
|
||||
import re
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
from datetime import timedelta
|
||||
|
||||
from .models import SavedItem
|
||||
|
||||
|
||||
def clean_url(value):
|
||||
value = (value or '').strip()
|
||||
if len(value) > 2048 or any(ord(c) < 32 for c in value):
|
||||
raise ValidationError('Enter a valid URL under 2048 characters.')
|
||||
try:
|
||||
parts = urlsplit(value)
|
||||
host = parts.hostname
|
||||
port = parts.port
|
||||
except ValueError as exc:
|
||||
raise ValidationError('Enter a valid URL.') from exc
|
||||
if parts.scheme.lower() not in ('http', 'https') or not host or parts.username or parts.password:
|
||||
raise ValidationError('Only public HTTP(S) URLs are accepted.')
|
||||
host = host.lower().rstrip('.')
|
||||
if '.' not in host and ':' not in host:
|
||||
raise ValidationError('A public hostname is required.')
|
||||
try:
|
||||
address = ipaddress.ip_address(host.strip('[]'))
|
||||
except ValueError:
|
||||
address = None
|
||||
if address and not address.is_global:
|
||||
raise ValidationError('Local and private addresses are not accepted.')
|
||||
if host in ('localhost', 'localhost.localdomain') or host.endswith(('.local', '.internal', '.localhost')):
|
||||
raise ValidationError('Local hostnames are not accepted.')
|
||||
authority = host if ':' not in host else f'[{host}]'
|
||||
if port and port != (443 if parts.scheme.lower() == 'https' else 80):
|
||||
authority += f':{port}'
|
||||
key = urlunsplit((parts.scheme.lower(), authority, parts.path or '/', parts.query, ''))
|
||||
if len(key) > 2048:
|
||||
raise ValidationError('URL is too long.')
|
||||
return key
|
||||
|
||||
|
||||
def save_item(owner, url, **fields):
|
||||
key = clean_url(url)
|
||||
allowed = {'title', 'notes', 'saved_reason', 'deferral_reason', 'tags'}
|
||||
defaults = {name: value for name, value in fields.items() if name in allowed}
|
||||
item, created = SavedItem.objects.get_or_create(owner=owner, url_key=key,
|
||||
defaults={'original_url': url.strip(), **defaults})
|
||||
return item, created
|
||||
|
||||
|
||||
def search_items(owner, query, limit=20):
|
||||
words = re.findall(r'[\w-]{2,}', (query or '').lower())[:8]
|
||||
qs = SavedItem.objects.filter(owner=owner).exclude(status=SavedItem.Status.ARCHIVED)
|
||||
for word in words:
|
||||
qs = qs.filter(Q(title__icontains=word) | Q(extracted_title__icontains=word) |
|
||||
Q(original_url__icontains=word) | Q(description__icontains=word) |
|
||||
Q(notes__icontains=word) | Q(saved_reason__icontains=word) | Q(tags__icontains=word))
|
||||
return list(qs[:max(1, min(limit, 100))])
|
||||
|
||||
|
||||
def relevant_items(owner, context):
|
||||
words = set(re.findall(r'[\w-]{4,}', (context or '').lower()))
|
||||
words -= {'this', 'that', 'with', 'from', 'about', 'what', 'would', 'could', 'should', 'there', 'their'}
|
||||
if not words:
|
||||
return []
|
||||
now = timezone.now()
|
||||
candidates = SavedItem.objects.filter(owner=owner, status=SavedItem.Status.SAVED).filter(
|
||||
Q(snoozed_until__isnull=True) | Q(snoozed_until__lte=now)).filter(
|
||||
Q(last_surfaced_at__isnull=True) | Q(last_surfaced_at__lte=now - timedelta(days=14)))
|
||||
scored = []
|
||||
for item in candidates[:500]:
|
||||
text = ' '.join([item.title, item.extracted_title, item.description, item.saved_reason, ' '.join(item.tags)]).lower()
|
||||
hits = words & set(re.findall(r'[\w-]{4,}', text))
|
||||
if len(hits) >= 2 or (len(hits) == 1 and len(next(iter(hits))) >= 6):
|
||||
scored.append((len(hits), item, sorted(hits)))
|
||||
scored.sort(key=lambda row: (-row[0], -row[1].created_at.timestamp()))
|
||||
return scored[:3]
|
||||
@@ -0,0 +1,4 @@
|
||||
.knowledge-form{margin-bottom:20px}.knowledge-fields{display:grid;gap:6px;margin-bottom:14px;max-width:680px}.knowledge-fields label,.knowledge-search label{font-weight:700}.knowledge-fields input,.knowledge-search input,.knowledge-actions input,.knowledge-actions textarea,.knowledge-actions select{width:100%;padding:9px;border:1px solid var(--border);border-radius:8px;background:var(--surface);color:var(--text)}.knowledge-search{max-width:680px}.knowledge-search>div{display:flex;gap:8px}.knowledge-list{display:grid;gap:14px;margin-top:20px}.knowledge-item{padding:0;overflow:hidden;display:grid;grid-template-columns:minmax(0,220px) minmax(0,1fr)}.knowledge-item:not(:has(.knowledge-image-wrap)){grid-template-columns:1fr}.knowledge-image-wrap{background:var(--surface-secondary);min-height:180px;display:grid;place-items:center}.knowledge-image{width:100%;height:100%;max-height:300px;object-fit:cover}.knowledge-image-fallback{color:var(--text-secondary)}.knowledge-body{padding:20px;min-width:0}.knowledge-site{font-size:11px;color:var(--text-secondary);font-weight:700}.knowledge-body h2{font-size:19px;overflow-wrap:anywhere}.knowledge-description,.knowledge-reason{color:var(--text-secondary)}.knowledge-state{font-size:11px;color:var(--text-muted)}.knowledge-actions{display:flex;gap:12px;align-items:start;flex-wrap:wrap;margin-top:14px}.knowledge-actions button,.knowledge-actions summary,.knowledge-play{border:1px solid var(--border);border-radius:7px;padding:7px 10px;background:var(--surface-secondary);color:var(--text);cursor:pointer}.knowledge-actions details{min-width:min(100%,250px)}.knowledge-actions details[open]{width:min(100%,450px)}.knowledge-actions details form{display:grid;gap:10px;padding-top:12px}.knowledge-actions label{display:grid;gap:4px;font-weight:700}.knowledge-actions textarea{min-height:75px}.knowledge-player{margin-top:12px}.knowledge-third-party{font-size:11px;color:var(--text-muted);margin:5px 0}.knowledge-player iframe{width:100%;aspect-ratio:16/9;border:0;border-radius:8px}@media(max-width:680px){.knowledge-item{grid-template-columns:1fr}.knowledge-image-wrap{height:190px;min-height:0}.knowledge-body{padding:16px}}@media(prefers-reduced-motion:reduce){.knowledge-item{transition:none}}
|
||||
.knowledge-favicon{width:16px;height:16px;object-fit:contain;vertical-align:middle;margin-right:5px}
|
||||
.knowledge-item--embed{grid-template-columns:1fr}.knowledge-item--embed .knowledge-image-wrap{max-height:330px}.knowledge-item--embed .knowledge-player{min-height:54px}
|
||||
.knowledge-image-wrap{overflow:hidden}.knowledge-image{display:block}@media(max-width:680px){.knowledge-image-wrap{height:190px}.knowledge-image{height:190px;max-height:190px}}
|
||||
@@ -0,0 +1,40 @@
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
document.addEventListener('error', (event) => {
|
||||
if (event.target.matches?.('.knowledge-favicon')) { event.target.hidden = true; return; }
|
||||
if (!event.target.matches?.('.knowledge-image')) return;
|
||||
event.target.hidden = true;
|
||||
const fallback = event.target.parentElement.querySelector('.knowledge-image-fallback');
|
||||
if (fallback) fallback.hidden = false;
|
||||
}, true);
|
||||
document.addEventListener('click', (event) => {
|
||||
const button = event.target.closest?.('.knowledge-play');
|
||||
if (!button) return;
|
||||
const player = button.closest('.knowledge-player');
|
||||
const url = player?.dataset.embedUrl || '';
|
||||
if (!/^https:\/\/www\.youtube-nocookie\.com\/embed\/[A-Za-z0-9_-]{11}$/.test(url)) return;
|
||||
const frame = document.createElement('iframe');
|
||||
frame.src = url;
|
||||
frame.title = 'YouTube video player';
|
||||
frame.referrerPolicy = 'no-referrer';
|
||||
frame.allow = 'accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture; web-share';
|
||||
frame.allowFullscreen = true;
|
||||
player.replaceChildren(frame);
|
||||
frame.focus();
|
||||
});
|
||||
const started = Date.now();
|
||||
const poll = async () => {
|
||||
if (document.hidden || Date.now() - started > 120000) return;
|
||||
const cards = [...document.querySelectorAll('.knowledge-item[data-metadata-status="pending"]')].slice(0, 20);
|
||||
for (const card of cards) {
|
||||
try {
|
||||
const response = await fetch(card.dataset.cardUrl, {credentials: 'same-origin', cache: 'no-store'});
|
||||
if (!response.ok) continue;
|
||||
const html = await response.text();
|
||||
const documentFragment = new DOMParser().parseFromString(html, 'text/html');
|
||||
const replacement = documentFragment.querySelector('.knowledge-item');
|
||||
if (replacement && replacement.dataset.metadataStatus !== 'pending') card.replaceWith(replacement);
|
||||
} catch (_) { /* Preserve the current card on a transient polling failure. */ }
|
||||
}
|
||||
};
|
||||
const timer = setInterval(() => { if (Date.now() - started > 120000) clearInterval(timer); else poll(); }, 10000);
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
<article class="widget-card knowledge-item{% if item.preview_mode == 'embed' %} knowledge-item--embed{% endif %}" id="knowledge-item-{{ item.id }}" data-card-url="{% url 'knowledge:item_card' item.id %}" data-metadata-status="{{ item.metadata_status }}">
|
||||
{% if item.preview_mode != 'plain' and item.thumbnail_url %}<div class="knowledge-image-wrap"><img class="knowledge-image" src="{{ item.thumbnail_url }}" alt="{{ item.image_alt|default:'Preview image for saved link' }}" loading="lazy" referrerpolicy="no-referrer"><span class="knowledge-image-fallback" hidden>Image unavailable</span></div>{% endif %}
|
||||
<div class="knowledge-body">
|
||||
<p class="knowledge-site">{% if item.favicon_url and item.preview_mode != 'plain' %}<img class="knowledge-favicon" src="{{ item.favicon_url }}" alt="" loading="lazy" referrerpolicy="no-referrer">{% endif %}{% if item.site_name %}{{ item.site_name }}{% else %}{{ item.domain }}{% endif %}{% if item.provider == 'youtube' %} · YouTube{% endif %}</p>
|
||||
<h2><a href="{{ item.original_url }}" target="_blank" rel="noopener noreferrer">{{ item.title|default:item.extracted_title|default:item.original_url }}</a></h2>
|
||||
{% if item.preview_mode != 'plain' %}{% if item.user_description or item.description %}<p class="knowledge-description">{{ item.user_description|default:item.description }}</p>{% endif %}{% endif %}
|
||||
{% if item.saved_reason %}<p class="knowledge-reason">Saved because: {{ item.saved_reason }}</p>{% endif %}
|
||||
<p class="knowledge-state"><small>Saved {{ item.created_at|date:'j M Y' }} · {% if item.metadata_status == 'pending' %}Preview loading{% elif item.metadata_status == 'error' %}Preview unavailable{% if item.metadata_error %}: {{ item.metadata_error }}{% endif %}{% else %}Preview updated {{ item.metadata_fetched_at|date:'j M Y' }}{% endif %}</small></p>
|
||||
{% if item.provider == 'youtube' and item.preview_mode != 'plain' and item.embed_url %}<div class="knowledge-player" data-embed-url="{{ item.embed_url }}"><button type="button" class="knowledge-play">Play video</button><p class="knowledge-third-party">Playing loads YouTube content.</p></div>{% endif %}
|
||||
<div class="knowledge-actions"><details><summary>Edit preview</summary><form method="post" action="{% url 'knowledge:edit' item.id %}">{% csrf_token %}<label>Your title<input name="title" value="{{ item.title }}" maxlength="300"></label><label>Your description<textarea name="description" maxlength="2000">{{ item.user_description }}</textarea></label><label>Display mode<select name="preview_mode"><option value="preview" {% if item.preview_mode == 'preview' %}selected{% endif %}>Preview</option><option value="plain" {% if item.preview_mode == 'plain' %}selected{% endif %}>Plain link</option>{% if item.provider == 'youtube' %}<option value="embed" {% if item.preview_mode == 'embed' %}selected{% endif %}>Playable embed</option>{% endif %}</select></label><button type="submit">Save changes</button></form></details><form method="post" action="{% url 'knowledge:refresh' item.id %}">{% csrf_token %}<button type="submit">Refresh preview</button></form><form method="post" action="{% url 'knowledge:archive' item.id %}">{% csrf_token %}<button type="submit">Archive</button></form></div>
|
||||
</div>
|
||||
</article>
|
||||
@@ -0,0 +1,10 @@
|
||||
{% extends 'core/base.html' %}{% load static %}
|
||||
{% block extra_head %}<link rel="stylesheet" href="{% static 'knowledge/previews.css' %}"><script src="{% static 'knowledge/previews.js' %}" defer></script>{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-heading"><div><h1>Knowledge Library</h1><p>Save things you want to revisit when the time is right.</p></div><a class="button" href="{% url 'knowledge:tokens' %}">Capture tokens</a></div>
|
||||
<section class="widget-card knowledge-form"><h2>Save a link</h2>
|
||||
{% if error %}<p role="alert">{{ error }}</p>{% endif %}
|
||||
<form method="post" action="{% url 'knowledge:add' %}">{% csrf_token %}<div class="knowledge-fields"><label for="save-url">URL</label><input id="save-url" name="url" type="url" required maxlength="2048" value="{{ form_url }}"><label for="save-title">Your title (optional)</label><input id="save-title" name="title" maxlength="300"><label for="save-reason">Why save it? (optional)</label><input id="save-reason" name="saved_reason" maxlength="300"></div><button class="button" type="submit">Save link</button></form></section>
|
||||
<form method="get" role="search" class="knowledge-search"><label for="knowledge-search">Search saved links</label><div><input id="knowledge-search" type="search" name="q" value="{{ query }}"><button class="button" type="submit">Search</button></div></form>
|
||||
<div class="knowledge-list">{% for item in items %}{% include 'knowledge/_item_card.html' %}{% empty %}<p>No saved links found.</p>{% endfor %}</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,5 @@
|
||||
{% extends 'core/base.html' %}
|
||||
{% block content %}<h1>Capture tokens</h1><p>Create a token for an Apple Shortcut that sends a URL to this library. Tokens only permit saving links.</p>
|
||||
{% if secret %}<div class="widget-card" role="status"><strong>Copy this token now; it will not be shown again.</strong><p><code style="overflow-wrap:anywhere">{{ secret }}</code></p></div>{% endif %}
|
||||
<form method="post">{% csrf_token %}<label for="token-name">Token name</label><input id="token-name" name="name" value="Apple Shortcut" maxlength="80"><button class="button" type="submit">Create token</button></form>
|
||||
<h2>Existing tokens</h2><ul>{% for token in tokens %}<li>{{ token.name }} · {{ token.created_at|date:'j M Y' }} {% if token.revoked_at %}(revoked){% else %}<form method="post" style="display:inline">{% csrf_token %}<input type="hidden" name="action" value="revoke"><input type="hidden" name="token_id" value="{{ token.id }}"><button type="submit">Revoke</button></form>{% endif %}</li>{% empty %}<li>No tokens yet.</li>{% endfor %}</ul><a href="{% url 'knowledge:index' %}">Back to library</a>{% endblock %}
|
||||
@@ -0,0 +1,183 @@
|
||||
import hashlib
|
||||
import json
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.test import TestCase
|
||||
from django.urls import reverse
|
||||
from django.core.management import call_command
|
||||
from django.test import override_settings
|
||||
from django.utils import timezone
|
||||
from datetime import timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from .models import CaptureToken, SavedItem
|
||||
from .services import relevant_items, save_item
|
||||
from .notifications import rediscover_once
|
||||
from .metadata import MetadataError, fetch_metadata, public_address, youtube_video_id
|
||||
|
||||
|
||||
class KnowledgeTests(TestCase):
|
||||
def setUp(self):
|
||||
self.alice = get_user_model().objects.create_user('alice', password='test-password')
|
||||
self.bob = get_user_model().objects.create_user('bob', password='test-password')
|
||||
|
||||
def test_capture_is_idempotent_and_owner_scoped(self):
|
||||
token = 'capture-secret'
|
||||
CaptureToken.objects.create(owner=self.alice, name='Shortcut', token_hash=hashlib.sha256(token.encode()).hexdigest())
|
||||
url = reverse('knowledge_api_add')
|
||||
headers = {'HTTP_AUTHORIZATION': f'Bearer {token}', 'content_type': 'application/json'}
|
||||
first = self.client.post(url, data=json.dumps({'url': 'https://example.org/esp32#top', 'title': 'ESP32'}), **headers)
|
||||
second = self.client.post(url, data=json.dumps({'url': 'https://EXAMPLE.org/esp32'}), **headers)
|
||||
self.assertEqual(first.status_code, 201)
|
||||
self.assertEqual(second.status_code, 200)
|
||||
self.assertTrue(second.json()['duplicate'])
|
||||
self.assertEqual(SavedItem.objects.filter(owner=self.alice).count(), 1)
|
||||
self.assertEqual(SavedItem.objects.filter(owner=self.bob).count(), 0)
|
||||
self.assertEqual(self.client.post(url, data='{}', content_type='application/json').status_code, 401)
|
||||
|
||||
def test_private_url_and_cross_user_visibility(self):
|
||||
token = 'capture-secret'
|
||||
CaptureToken.objects.create(owner=self.alice, name='Shortcut', token_hash=hashlib.sha256(token.encode()).hexdigest())
|
||||
response = self.client.post(reverse('knowledge_api_add'), data=json.dumps({'url': 'http://127.0.0.1/admin'}),
|
||||
content_type='application/json', HTTP_AUTHORIZATION=f'Bearer {token}')
|
||||
self.assertEqual(response.status_code, 400)
|
||||
item, _ = save_item(self.alice, 'https://example.org/sensor', title='ESP32 sensor microphone')
|
||||
self.assertEqual(relevant_items(self.bob, 'ESP32 sensor microphone'), [])
|
||||
self.assertEqual(relevant_items(self.alice, 'ESP32 sensor microphone')[0][1], item)
|
||||
self.client.force_login(self.bob)
|
||||
self.assertNotContains(self.client.get(reverse('knowledge:index')), 'ESP32 sensor microphone')
|
||||
self.assertEqual(self.client.post(reverse('knowledge:archive', args=[item.pk])).status_code, 404)
|
||||
|
||||
def test_token_revocation(self):
|
||||
self.client.force_login(self.alice)
|
||||
self.client.post(reverse('knowledge:tokens'), {'name': 'Shortcut'})
|
||||
token = CaptureToken.objects.get(owner=self.alice)
|
||||
self.client.post(reverse('knowledge:tokens'), {'action': 'revoke', 'token_id': token.pk})
|
||||
token.refresh_from_db()
|
||||
self.assertIsNotNone(token.revoked_at)
|
||||
|
||||
@override_settings(KNOWLEDGE_NTFY_ENABLED=True, KNOWLEDGE_NTFY_URL='https://ntfy.example.org/topic',
|
||||
KNOWLEDGE_NTFY_WEEKLY_LIMIT=1, KNOWLEDGE_NTFY_QUIET_START=23, KNOWLEDGE_NTFY_QUIET_END=6)
|
||||
@patch('knowledge.notifications.send_ntfy_notification')
|
||||
def test_ntfy_limit_and_cooldown(self, sender):
|
||||
item, _ = save_item(self.alice, 'https://example.org/esp32', title='ESP32 sensors')
|
||||
SavedItem.objects.filter(pk=item.pk).update(created_at=timezone.now() - timedelta(days=5))
|
||||
with override_settings(KNOWLEDGE_NTFY_USER_ID=str(self.alice.pk)):
|
||||
now = timezone.now().replace(hour=12)
|
||||
self.assertEqual(rediscover_once(now), 'delivered')
|
||||
self.assertEqual(rediscover_once(now), 'weekly_limit')
|
||||
sender.assert_called_once()
|
||||
|
||||
@patch('knowledge.metadata.socket.getaddrinfo')
|
||||
def test_metadata_rejects_private_dns_answer(self, resolver):
|
||||
resolver.return_value = [(None, None, None, None, ('192.168.1.1', 443))]
|
||||
with self.assertRaises(MetadataError):
|
||||
public_address('example.org', 443)
|
||||
|
||||
@patch('knowledge.metadata.PinnedHTTPConnection')
|
||||
@patch('knowledge.metadata.public_address', return_value='93.184.215.14')
|
||||
def test_metadata_reads_only_bounded_html(self, resolver, connection_type):
|
||||
response = connection_type.return_value.getresponse.return_value
|
||||
response.status = 200
|
||||
response.getheader.side_effect = lambda name: {'Content-Type': 'text/html; charset=utf-8', 'Content-Length': '90'}.get(name)
|
||||
response.read.side_effect = [b'<title>Useful ESP32 page</title><meta name="description" content="A sensor guide">', b'']
|
||||
result = fetch_metadata('https://example.org/esp32')
|
||||
self.assertEqual(result['extracted_title'], 'Useful ESP32 page')
|
||||
self.assertEqual(result['description'], 'A sensor guide')
|
||||
connection_type.assert_called_once_with('example.org', 443, '93.184.215.14', True)
|
||||
|
||||
@patch('knowledge.metadata._fetch_bytes')
|
||||
def test_open_graph_precedence_and_fallback(self, fetch):
|
||||
html = b'''<title>HTML title</title><meta name="description" content="HTML description">
|
||||
<meta name="twitter:title" content="Twitter title"><meta name="twitter:image" content="https://example.org/tw.jpg">
|
||||
<meta property="og:title" content="OG title"><meta property="og:description" content="OG description">
|
||||
<meta property="og:image" content="https://example.org/og.jpg"><meta property="og:image:alt" content="A flower">
|
||||
<meta property="og:site_name" content="Example Journal"><link rel="icon" href="/favicon.png">'''
|
||||
fetch.return_value = ('https://example.org/article', html)
|
||||
result = fetch_metadata('https://example.org/article')
|
||||
self.assertEqual(result['extracted_title'], 'OG title')
|
||||
self.assertEqual(result['description'], 'OG description')
|
||||
self.assertEqual(result['thumbnail_url'], 'https://example.org/og.jpg')
|
||||
self.assertEqual(result['image_alt'], 'A flower')
|
||||
self.assertEqual(result['site_name'], 'Example Journal')
|
||||
self.assertEqual(result['favicon_url'], 'https://example.org/favicon.png')
|
||||
fetch.return_value = ('https://example.org/article', b'<title>HTML only</title>')
|
||||
self.assertEqual(fetch_metadata('https://example.org/article')['extracted_title'], 'HTML only')
|
||||
fetch.return_value = ('https://example.org/article', b'<html><head></head><body>Nothing to extract</body></html>')
|
||||
self.assertEqual(fetch_metadata('https://example.org/article')['extracted_title'], '')
|
||||
|
||||
@patch('knowledge.metadata._fetch_bytes')
|
||||
def test_youtube_oembed_uses_allowlisted_embed(self, fetch):
|
||||
fetch.return_value = ('https://www.youtube.com/oembed', json.dumps({
|
||||
'title': 'Circuit demo', 'author_name': 'Maker channel',
|
||||
'thumbnail_url': 'https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg'}).encode())
|
||||
result = fetch_metadata('https://youtu.be/dQw4w9WgXcQ')
|
||||
self.assertEqual(result['site_name'], 'Maker channel')
|
||||
self.assertEqual(result['embed_url'], 'https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ')
|
||||
self.assertEqual(youtube_video_id('https://youtube.com.evil.test/watch?v=dQw4w9WgXcQ'), '')
|
||||
|
||||
@patch('knowledge.metadata.PinnedHTTPConnection')
|
||||
@patch('knowledge.metadata.public_address', return_value='93.184.215.14')
|
||||
def test_redirect_to_private_address_is_blocked(self, resolver, connection_type):
|
||||
response = connection_type.return_value.getresponse.return_value
|
||||
response.status = 302
|
||||
response.getheader.return_value = 'http://127.0.0.1/admin'
|
||||
with self.assertRaises(MetadataError):
|
||||
fetch_metadata('https://example.org/article')
|
||||
|
||||
@patch('knowledge.metadata.PinnedHTTPConnection')
|
||||
@patch('knowledge.metadata.public_address', return_value='93.184.215.14')
|
||||
def test_timeout_is_a_safe_failure(self, resolver, connection_type):
|
||||
import socket
|
||||
connection_type.return_value.request.side_effect = socket.timeout()
|
||||
with self.assertRaises(MetadataError):
|
||||
fetch_metadata('https://example.org/article')
|
||||
|
||||
@patch('knowledge.management.commands.fetch_knowledge_metadata.fetch_metadata')
|
||||
def test_metadata_refresh_preserves_user_edits_and_cache(self, fetch):
|
||||
item, _ = save_item(self.alice, 'https://example.org/article')
|
||||
self.client.force_login(self.alice)
|
||||
self.client.post(reverse('knowledge:edit', args=[item.pk]), {
|
||||
'title': 'My title', 'description': 'My notes on this', 'preview_mode': 'preview'})
|
||||
fetch.return_value = {'extracted_title': 'Publisher title', 'description': 'Publisher description',
|
||||
'thumbnail_url': 'https://example.org/image.jpg', 'image_alt': 'Image', 'site_name': 'Example',
|
||||
'favicon_url': '', 'provider': 'web', 'embed_url': '', 'canonical_url': ''}
|
||||
call_command('fetch_knowledge_metadata')
|
||||
item.refresh_from_db()
|
||||
self.assertEqual(item.title, 'My title')
|
||||
self.assertEqual(item.user_description, 'My notes on this')
|
||||
self.assertEqual(item.description, 'Publisher description')
|
||||
self.assertEqual(item.metadata_status, SavedItem.MetadataStatus.READY)
|
||||
call_command('fetch_knowledge_metadata')
|
||||
self.assertEqual(fetch.call_count, 1)
|
||||
self.assertContains(self.client.get(reverse('knowledge:index')), 'My title')
|
||||
self.assertContains(self.client.get(reverse('knowledge:index')), 'My notes on this')
|
||||
|
||||
def test_preview_escapes_metadata_and_defers_youtube_embed(self):
|
||||
item, _ = save_item(self.alice, 'https://youtu.be/dQw4w9WgXcQ')
|
||||
item.provider = 'youtube'
|
||||
item.embed_url = 'https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ'
|
||||
item.extracted_title = '<script>alert(1)</script>'
|
||||
item.thumbnail_url = 'https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg'
|
||||
item.save()
|
||||
self.client.force_login(self.alice)
|
||||
response = self.client.get(reverse('knowledge:index'))
|
||||
self.assertContains(response, '<script>alert(1)</script>')
|
||||
self.assertNotContains(response, '<iframe')
|
||||
self.assertContains(response, 'Play video')
|
||||
self.assertContains(response, 'knowledge-image-fallback')
|
||||
self.client.force_login(self.bob)
|
||||
self.assertEqual(self.client.get(reverse('knowledge:item_card', args=[item.pk])).status_code, 404)
|
||||
|
||||
@patch('knowledge.management.commands.fetch_knowledge_metadata.fetch_metadata', side_effect=MetadataError('HTTP 503.'))
|
||||
def test_worker_retries_with_backoff_and_keeps_saved_link(self, fetch):
|
||||
item, _ = save_item(self.alice, 'https://example.org/broken')
|
||||
call_command('fetch_knowledge_metadata')
|
||||
item.refresh_from_db()
|
||||
self.assertEqual(item.metadata_status, SavedItem.MetadataStatus.ERROR)
|
||||
self.assertEqual(item.metadata_attempts, 1)
|
||||
self.assertGreater(item.metadata_retry_at, timezone.now())
|
||||
call_command('fetch_knowledge_metadata')
|
||||
self.assertEqual(fetch.call_count, 1)
|
||||
self.client.force_login(self.alice)
|
||||
self.assertContains(self.client.get(reverse('knowledge:index')), 'Preview unavailable')
|
||||
self.assertContains(self.client.get(reverse('knowledge:index')), 'https://example.org/broken')
|
||||
@@ -0,0 +1,13 @@
|
||||
from django.urls import path
|
||||
from . import views
|
||||
|
||||
app_name = 'knowledge'
|
||||
urlpatterns = [
|
||||
path('', views.index, name='index'),
|
||||
path('add/', views.add, name='add'),
|
||||
path('items/<uuid:item_id>/archive/', views.archive, name='archive'),
|
||||
path('items/<uuid:item_id>/card/', views.item_card, name='item_card'),
|
||||
path('items/<uuid:item_id>/edit/', views.edit, name='edit'),
|
||||
path('items/<uuid:item_id>/refresh/', views.refresh, name='refresh'),
|
||||
path('tokens/', views.tokens, name='tokens'),
|
||||
]
|
||||
@@ -0,0 +1,129 @@
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from datetime import timedelta
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.core.cache import cache
|
||||
from django.http import JsonResponse
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.utils import timezone
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
from django.views.decorators.http import require_POST
|
||||
|
||||
from .models import CaptureToken, SavedItem
|
||||
from .services import save_item, search_items
|
||||
|
||||
|
||||
@login_required
|
||||
def index(request):
|
||||
query = request.GET.get('q', '').strip()[:200]
|
||||
items = search_items(request.user, query, 100) if query else SavedItem.objects.filter(owner=request.user).exclude(status=SavedItem.Status.ARCHIVED)[:100]
|
||||
return render(request, 'knowledge/index.html', {'title': 'Knowledge Library', 'section': 'knowledge', 'items': items, 'query': query})
|
||||
|
||||
|
||||
@login_required
|
||||
def item_card(request, item_id):
|
||||
item = get_object_or_404(SavedItem, pk=item_id, owner=request.user)
|
||||
response = render(request, 'knowledge/_item_card.html', {'item': item})
|
||||
response['Cache-Control'] = 'private, no-store'
|
||||
return response
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def add(request):
|
||||
try:
|
||||
item, created = save_item(request.user, request.POST.get('url'),
|
||||
title=request.POST.get('title', '').strip()[:300],
|
||||
notes=request.POST.get('notes', '').strip()[:3000],
|
||||
saved_reason=request.POST.get('saved_reason', '').strip()[:300])
|
||||
except ValidationError as exc:
|
||||
return render(request, 'knowledge/index.html', {'title': 'Knowledge Library', 'section': 'knowledge',
|
||||
'items': SavedItem.objects.filter(owner=request.user)[:100], 'error': exc.messages[0], 'form_url': request.POST.get('url', '')}, status=400)
|
||||
return redirect('knowledge:index')
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def archive(request, item_id):
|
||||
item = get_object_or_404(SavedItem, pk=item_id, owner=request.user)
|
||||
item.status = SavedItem.Status.ARCHIVED
|
||||
item.save(update_fields=['status', 'updated_at'])
|
||||
return redirect('knowledge:index')
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def edit(request, item_id):
|
||||
item = get_object_or_404(SavedItem, pk=item_id, owner=request.user)
|
||||
mode = request.POST.get('preview_mode', 'preview')
|
||||
if mode not in ('preview', 'plain', 'embed') or (mode == 'embed' and item.provider != 'youtube'):
|
||||
return JsonResponse({'error': 'Invalid preview mode'}, status=400)
|
||||
item.title = request.POST.get('title', '').strip()[:300]
|
||||
item.user_description = request.POST.get('description', '').strip()[:2000]
|
||||
item.preview_mode = mode
|
||||
item.save(update_fields=['title', 'user_description', 'preview_mode', 'updated_at'])
|
||||
return redirect('knowledge:index')
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def refresh(request, item_id):
|
||||
item = get_object_or_404(SavedItem, pk=item_id, owner=request.user)
|
||||
if item.metadata_fetched_at and item.metadata_fetched_at > timezone.now() - timedelta(minutes=5):
|
||||
return redirect('knowledge:index')
|
||||
item.metadata_status = SavedItem.MetadataStatus.PENDING
|
||||
item.metadata_retry_at = None
|
||||
item.metadata_error = ''
|
||||
item.metadata_attempts = 0
|
||||
item.save(update_fields=['metadata_status', 'metadata_retry_at', 'metadata_error', 'metadata_attempts', 'updated_at'])
|
||||
return redirect('knowledge:index')
|
||||
|
||||
|
||||
@login_required
|
||||
def tokens(request):
|
||||
secret = None
|
||||
if request.method == 'POST':
|
||||
if request.POST.get('action') == 'revoke':
|
||||
token = get_object_or_404(CaptureToken, pk=request.POST.get('token_id'), owner=request.user)
|
||||
token.revoked_at = timezone.now()
|
||||
token.save(update_fields=['revoked_at'])
|
||||
else:
|
||||
secret = secrets.token_urlsafe(32)
|
||||
CaptureToken.objects.create(owner=request.user, name=request.POST.get('name', 'Apple Shortcut')[:80],
|
||||
token_hash=hashlib.sha256(secret.encode()).hexdigest())
|
||||
return render(request, 'knowledge/tokens.html', {'title': 'Capture tokens', 'section': 'knowledge',
|
||||
'tokens': CaptureToken.objects.filter(owner=request.user).order_by('-created_at'), 'secret': secret})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
@require_POST
|
||||
def api_add(request):
|
||||
if len(request.body) > 16_384:
|
||||
return JsonResponse({'error': 'Request too large'}, status=413)
|
||||
header = request.headers.get('Authorization', '')
|
||||
if not header.startswith('Bearer ') or len(header) > 180:
|
||||
return JsonResponse({'error': 'Bearer capture token required'}, status=401)
|
||||
token_hash = hashlib.sha256(header[7:].encode()).hexdigest()
|
||||
token = CaptureToken.objects.select_related('owner').filter(token_hash=token_hash, revoked_at__isnull=True).first()
|
||||
if token is None:
|
||||
return JsonResponse({'error': 'Invalid capture token'}, status=401)
|
||||
rate_key = f'knowledge-capture:{token.pk}:{timezone.now():%Y%m%d%H%M}'
|
||||
if not cache.add(rate_key, 1, 90):
|
||||
try:
|
||||
if cache.incr(rate_key) > 30:
|
||||
return JsonResponse({'error': 'Too many captures; try again shortly'}, status=429)
|
||||
except ValueError:
|
||||
cache.set(rate_key, 1, 90)
|
||||
try:
|
||||
payload = json.loads(request.body)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError()
|
||||
item, created = save_item(token.owner, payload.get('url'),
|
||||
title=str(payload.get('title') or '')[:300], notes=str(payload.get('notes') or '')[:3000],
|
||||
saved_reason=str(payload.get('saved_reason') or '')[:300])
|
||||
except (ValueError, TypeError, ValidationError) as exc:
|
||||
return JsonResponse({'error': str(exc) or 'Invalid JSON'}, status=400)
|
||||
return JsonResponse({'id': str(item.pk), 'created': created, 'duplicate': not created,
|
||||
'url': item.original_url, 'metadata_status': item.metadata_status}, status=201 if created else 200)
|
||||
Reference in New Issue
Block a user