diff --git a/README.md b/README.md index 630e216..9d38ec6 100644 --- a/README.md +++ b/README.md @@ -60,3 +60,7 @@ Run checks and tests with: ``` For compiled frontend assets, run `npm run build`. Django uses Vite in development, the build manifest when present, and checked-in preview assets otherwise. + +## Knowledge Library + +`/knowledge/` is a separate personal link library with its own menu, data and capture API. It does not use Project Manager cards. See [Knowledge Library setup and limitations](docs/knowledge-library.md) and the [Pocket source review](docs/research/knowledge-library-pocket.md). The separate MCP `knowledge:read` scope grants contextual discovery; opt-in ntfy scheduling is available through a management command. diff --git a/config/settings.py b/config/settings.py index e9acaa1..f55d743 100644 --- a/config/settings.py +++ b/config/settings.py @@ -17,6 +17,7 @@ INSTALLED_APPS = [ "django.contrib.staticfiles", "core", "dashboard", + "knowledge", 'projects', 'oauth2_provider', ] @@ -57,6 +58,13 @@ PROJECT_DOCS_MAX_FILES = int(os.environ.get('PROJECT_DOCS_MAX_FILES', '500')) PROJECTS_ATTACHMENT_MAX_BYTES = int(os.environ.get('PROJECTS_ATTACHMENT_MAX_BYTES', 10 * 1024 * 1024)) PROJECTS_ATTACHMENT_MAX_FILES = int(os.environ.get('PROJECTS_ATTACHMENT_MAX_FILES', 50)) PROJECTS_ATTACHMENT_MAX_CARD_BYTES = int(os.environ.get('PROJECTS_ATTACHMENT_MAX_CARD_BYTES', 100 * 1024 * 1024)) +KNOWLEDGE_NTFY_ENABLED = os.environ.get('KNOWLEDGE_NTFY_ENABLED', '0') == '1' +KNOWLEDGE_NTFY_URL = os.environ.get('KNOWLEDGE_NTFY_URL', '') +KNOWLEDGE_NTFY_USER_ID = os.environ.get('KNOWLEDGE_NTFY_USER_ID', '') +KNOWLEDGE_NTFY_MODE = os.environ.get('KNOWLEDGE_NTFY_MODE', 'forgotten') +KNOWLEDGE_NTFY_WEEKLY_LIMIT = int(os.environ.get('KNOWLEDGE_NTFY_WEEKLY_LIMIT', '2')) +KNOWLEDGE_NTFY_QUIET_START = int(os.environ.get('KNOWLEDGE_NTFY_QUIET_START', '22')) +KNOWLEDGE_NTFY_QUIET_END = int(os.environ.get('KNOWLEDGE_NTFY_QUIET_END', '8')) DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': BASE_DIR / 'db.sqlite3'}} CACHES = { @@ -81,7 +89,7 @@ if not DEBUG and (not MCP_PUBLIC_URL.startswith('https://') or not OAUTH_ISSUER_ raise ImproperlyConfigured('MCP_PUBLIC_URL and OAUTH_ISSUER_URL must use HTTPS in production.') OAUTH2_PROVIDER = { 'OIDC_ISS_ENDPOINT': f'{OAUTH_ISSUER_URL}/', - 'SCOPES': {'projects:read': 'Read projects and cards', 'projects:write': 'Create and edit projects and cards'}, + 'SCOPES': {'projects:read': 'Read projects and cards', 'projects:write': 'Create and edit projects and cards', 'knowledge:read': 'Find saved knowledge links'}, 'DEFAULT_SCOPES': ['projects:read'], 'ALLOWED_REDIRECT_URI_SCHEMES': ['http', 'https'] if DEBUG else ['https'], 'PKCE_REQUIRED': True, diff --git a/config/urls.py b/config/urls.py index 006c218..f558f54 100644 --- a/config/urls.py +++ b/config/urls.py @@ -5,6 +5,7 @@ from django.conf import settings from oauth2_provider.urls import metadata_urlpatterns from core import views from projects import views as project_views +from knowledge import views as knowledge_views urlpatterns = [ *metadata_urlpatterns, @@ -14,6 +15,8 @@ urlpatterns = [ path('accounts/logout/', auth_views.LogoutView.as_view(), name='logout'), path('accounts/password/change/', auth_views.PasswordChangeView.as_view(template_name='projects/form.html', success_url='/projects/profile/'), name='password_change'), path('projects/', include('projects.urls')), + path('knowledge/', include('knowledge.urls')), + path('api/bookmarks/', knowledge_views.api_add, name='knowledge_api_add'), path("", include('dashboard.urls')), path("examples/today/", views.today, name="today"), path("health/", views.health, name="health"), diff --git a/core/templates/core/base.html b/core/templates/core/base.html index 0761521..7db9bbd 100644 --- a/core/templates/core/base.html +++ b/core/templates/core/base.html @@ -12,6 +12,7 @@ {% vite_assets %} + {% block extra_head %}{% endblock %} @@ -21,7 +22,7 @@ - {% if user.is_authenticated %}{% endif %} + {% if user.is_authenticated %}{% endif %} @@ -31,5 +32,5 @@
{% block content %}{% endblock %}
- + diff --git a/deploy/systemd/common-ground-knowledge-metadata.service b/deploy/systemd/common-ground-knowledge-metadata.service new file mode 100644 index 0000000..a1c1cf4 --- /dev/null +++ b/deploy/systemd/common-ground-knowledge-metadata.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fetch bounded Knowledge Library preview metadata +After=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/jorenmartijn/Desktop/django_dashboard +EnvironmentFile=-%h/.config/common-ground/env +ExecStart=/home/jorenmartijn/Desktop/django_dashboard/.venv-host/bin/python manage.py fetch_knowledge_metadata --limit 5 +TimeoutStartSec=120 + diff --git a/deploy/systemd/common-ground-knowledge-metadata.timer b/deploy/systemd/common-ground-knowledge-metadata.timer new file mode 100644 index 0000000..24c374d --- /dev/null +++ b/deploy/systemd/common-ground-knowledge-metadata.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Process Knowledge Library preview metadata each minute + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +AccuracySec=15s +Unit=common-ground-knowledge-metadata.service + +[Install] +WantedBy=timers.target diff --git a/docs/knowledge-library.md b/docs/knowledge-library.md new file mode 100644 index 0000000..e560fb5 --- /dev/null +++ b/docs/knowledge-library.md @@ -0,0 +1,43 @@ +# Knowledge Library + +Knowledge Library is a separate Django app under `/knowledge/`, with its own models, routes and menu entry. It does not create or modify Project Manager cards. See [Pocket source review and schema rationale](research/knowledge-library-pocket.md). + +## Capture + +Run migrations, sign in, and open **Knowledge Library → Capture tokens**. The token is displayed only when created and can be revoked there. An Apple Shortcut can send a JSON POST to `/api/bookmarks/` with `Authorization: Bearer ` and `Content-Type: application/json`: + +```json +{"url":"https://example.org/article","title":"Optional title","saved_reason":"Why this matters"} +``` + +The endpoint returns `201` for a new item or `200` with `duplicate: true` for an existing URL. URLs are normalized conservatively: scheme/host case and fragment are ignored, while path and query are preserved. Capture is owner-scoped and limited to 30 posts per token per minute. Token values are stored only as SHA-256 hashes. Browser forms use Django session authentication and CSRF; bearer tokens are accepted only at the capture endpoint. A saved link remains available when metadata has not been fetched. + +New items show `pending` until `.venv/bin/python manage.py fetch_knowledge_metadata --limit 20` runs. A one-minute user-systemd timer is supplied in `deploy/systemd/common-ground-knowledge-metadata.{service,timer}`; link and enable it after the application and migration are deployed. The timer processes at most five links per run. The page polls pending cards for two minutes and replaces them when the worker completes. On this host, link both files from `deploy/systemd/` into `~/.config/systemd/user/`, run `systemctl --user daemon-reload`, then `systemctl --user enable --now common-ground-knowledge-metadata.timer` after migrations. Use `flatpak-spawn --host` before `systemctl` when running inside the Codex Flatpak shell. It checks every DNS answer and redirect target, pins the outbound connection to a validated public IP, restricts ports, limits redirects, total time and response bytes, and extracts Open Graph metadata first, then Twitter Card metadata, then HTML title/description and favicon. It stores no page HTML. YouTube URLs use the fixed YouTube oEmbed JSON endpoint; returned embed HTML is ignored. Failed fetches show `error` and use exponential retry, capped at five attempts. No arbitrary URL is fetched during the save request. Fetched HTML is never rendered or stored as article content. + +## MCP discovery + +Authorize the MCP client with `knowledge:read` to expose `find_relevant_saved_interests(context)`. It returns at most three links belonging to the OAuth user. The current matching is intentionally simple and deterministic; it searches saved titles, descriptions, reasons and tags, and can return no matches. It does not inspect conversations on its own. Clients must choose to call it, so automatic invocation cannot be guaranteed. Existing `projects:read` scope does not grant library access. + +## ntfy prototype + +TasksNest's `planner/services/notifications.py` at commit `743785c2e0e41514c8dd18c035b7ea21716d87de` uses a UTF-8 HTTP POST with a `Title` header; this app follows that transport convention. Configure the server environment, outside Git: + +```sh +KNOWLEDGE_NTFY_ENABLED=1 +KNOWLEDGE_NTFY_URL=https://your-ntfy-host/private-topic +KNOWLEDGE_NTFY_USER_ID= +KNOWLEDGE_NTFY_MODE=forgotten +KNOWLEDGE_NTFY_WEEKLY_LIMIT=2 +KNOWLEDGE_NTFY_QUIET_START=22 +KNOWLEDGE_NTFY_QUIET_END=8 +``` + +The feature is off by default. Run `.venv/bin/python manage.py rediscover_knowledge` from a scheduled service (for example, once daily); no timer is installed by the app. Modes are `forgotten`, `random`, and `timely` (recent saved item, not a claim that its price or availability changed). Items must be at least two days old, are cooled down for 14 days after delivery, and the weekly limit defaults to two. Quiet hours use Django's configured local timezone. The notification includes the original source URL and saved reason. Delivery events are recorded after success; failed sends do not consume a slot. Snooze and dismissal controls are future work. + +## Current limits + +This checkout uses SQLite. Search is scoped by owner but does not yet use PostgreSQL full-text search or semantic matching. The browser page supports save, search, edit of user title/description, preview/plain/embed modes, manual refresh and archive. Publisher metadata remains separate from user edits, so refresh preserves overrides. YouTube playback is opt-in and uses only a constructed `youtube-nocookie.com` URL after a click; no provider HTML or iframe loads automatically. Preview images load directly from their publisher with no referrer; broken images fall back to text. Editing tags, reasons, and snooze needs a later UI pass. No article body is stored. No project conversion happens automatically. + +## Preview implementation choices + +The current project uses SQLite, so preview metadata is cached on the owner-scoped `SavedItem` row rather than in PostgreSQL. A later PostgreSQL deployment can use the same migration fields and add a full-text index. `micawber` offers general oEmbed provider discovery, Beautiful Soup offers broader HTML parsing, and `httpx` offers a higher-level HTTP client. This feature needs only one allowlisted oEmbed provider and a narrow set of head tags. The standard-library parser and DNS-pinned connection avoid extra dependencies and make SSRF checks explicit. [Open Graph image alt and site name](https://ogp.me/) and the [YouTube oEmbed endpoint](https://oembed.org/providers/youtube) informed field mapping; [YouTube Help](https://support.google.com/youtube/answer/171780) documents the privacy-enhanced embed domain. diff --git a/docs/research/knowledge-library-pocket.md b/docs/research/knowledge-library-pocket.md new file mode 100644 index 0000000..f29e1cb --- /dev/null +++ b/docs/research/knowledge-library-pocket.md @@ -0,0 +1,22 @@ +# Knowledge library: Pocket source review and design + +Reviewed 2026-10-20 before implementing the library data layer. Source inspected at [Pocket/pocket-monorepo](https://github.com/Pocket/pocket-monorepo/tree/5e5268fd92e997bb3350eaeafb54285238178a8e), commit `5e5268fd92e997bb3350eaeafb54285238178a8e`. + +## Verified upstream patterns + +- [`servers/list-api/schema.graphql`](https://github.com/Pocket/pocket-monorepo/blob/5e5268fd92e997bb3350eaeafb54285238178a8e/servers/list-api/schema.graphql) defines `SavedItem` as a user's saved URL with title, tags, archive/favorite state, timestamps and a linked underlying item. That separation is useful: the saved reason and state belong to the user, while fetched page metadata is fallible. +- [`servers/list-api/src/dataService/savedItemsService.ts`](https://github.com/Pocket/pocket-monorepo/blob/5e5268fd92e997bb3350eaeafb54285238178a8e/servers/list-api/src/dataService/savedItemsService.ts) scopes saved-item queries by `user_id` and looks up `given_url` separately from the resolved item. [`servers/list-api/src/models/SavedItem.ts`](https://github.com/Pocket/pocket-monorepo/blob/5e5268fd92e997bb3350eaeafb54285238178a8e/servers/list-api/src/models/SavedItem.ts) handles archive, tags and soft deletion. The published monorepo does not provide a complete migration history for the legacy list database, so its exact indexes and constraints cannot be inferred from the API alone. +- [`servers/list-api/src/externalCaller/parserCaller.ts`](https://github.com/Pocket/pocket-monorepo/blob/5e5268fd92e997bb3350eaeafb54285238178a8e/servers/list-api/src/externalCaller/parserCaller.ts) calls a separate parser. [`servers/parser-graphql-wrapper/src/databases/pocketMetadataStore.ts`](https://github.com/Pocket/pocket-monorepo/blob/5e5268fd92e997bb3350eaeafb54285238178a8e/servers/parser-graphql-wrapper/src/databases/pocketMetadataStore.ts) illustrates metadata storage. The many service and cache layers are excessive for this personal Django app. +- [`lambdas/user-list-search-corpus-parser-hydration`](https://github.com/Pocket/pocket-monorepo/tree/5e5268fd92e997bb3350eaeafb54285238178a8e/lambdas/user-list-search-corpus-parser-hydration) shows separate search hydration. Local title, URL, description, notes and tag search should come first. + +## License and reuse boundary + +The monorepo root [`LICENSE`](https://github.com/Pocket/pocket-monorepo/blob/5e5268fd92e997bb3350eaeafb54285238178a8e/LICENSE) is Apache-2.0, but [`servers/client-api/LICENSE`](https://github.com/Pocket/pocket-monorepo/blob/5e5268fd92e997bb3350eaeafb54285238178a8e/servers/client-api/LICENSE) is an MIT notice credited to Apollo Graph, Inc. Dependencies retain their own licenses. Apache-2.0 copying would require preserving license and applicable notices, marking changes, and observing its patent terms. Third-party article text and images are not granted by a source-code license. Pocket names and marks are not licensed as product branding. No upstream code or assets are copied here; the implementation uses independently written Django models and services based on general patterns. The [Pocket iOS repository](https://github.com/Pocket/pocket-ios) contains a Share extension, but that native code is unnecessary for an Apple Shortcut sending a POST. + +## Proposed local schema and pipeline + +Use a separate `knowledge` app. `SavedItem` has owner, original URL, conservative URL key, optional canonical URL, user title, extracted title/description/thumbnail, notes, tags, saved and deferral reasons, status, read date, metadata state, and exposure dates. Enforce `(owner, url_key)` uniqueness and index `(owner, status, created_at)` and `(owner, last_surfaced_at)`. `CaptureToken` stores only a hash, owner, scope, creation/revocation dates. `RediscoveryEvent` records attempted/delivered resurfacing and feedback. Do not model project cards as saved items. Optional project association can come later through an explicit link. + +Save must commit without network access. A separate bounded worker may fetch metadata after validating each DNS resolution and redirect target against private, loopback, link-local and reserved addresses; cap redirects, time, response bytes and content type. Never render fetched HTML. Keep metadata pending until the separately invoked bounded worker runs. A failed fetch becomes an explicit error state. No arbitrary URL is fetched in the capture request. The current checkout uses SQLite, so local search uses indexed owner filtering plus simple text matching; PostgreSQL `SearchVector` and a GIN index are a deployment upgrade, not an assumed current capability. Do not add pgvector until relevance tests show a need. + +MCP contextual retrieval uses only the authenticated owner's records, a relevance threshold, a maximum of three results, and a recent-exposure cooldown. A tool description can invite proactive calls, but the MCP server cannot compel ChatGPT to invoke it. ntfy scheduling must be opt-in with quiet hours, a low weekly budget, delivery history and duplicate suppression. The TasksNest ntfy implementation is outside this checkout and must be inspected before reusing its exact transport conventions. diff --git a/knowledge/__init__.py b/knowledge/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knowledge/apps.py b/knowledge/apps.py new file mode 100644 index 0000000..cd70322 --- /dev/null +++ b/knowledge/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class KnowledgeConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'knowledge' diff --git a/knowledge/management/__init__.py b/knowledge/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knowledge/management/commands/__init__.py b/knowledge/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knowledge/management/commands/fetch_knowledge_metadata.py b/knowledge/management/commands/fetch_knowledge_metadata.py new file mode 100644 index 0000000..45723ef --- /dev/null +++ b/knowledge/management/commands/fetch_knowledge_metadata.py @@ -0,0 +1,40 @@ +from django.core.management.base import BaseCommand +from django.db.models import Q +from django.utils import timezone +from datetime import timedelta +from knowledge.metadata import MetadataError, fetch_metadata +from knowledge.models import SavedItem + + +class Command(BaseCommand): + help = 'Fetch safe, bounded metadata for pending saved links.' + + def add_arguments(self, parser): + parser.add_argument('--limit', type=int, default=20) + + def handle(self, *args, **options): + total = 0 + now = timezone.now() + candidates = SavedItem.objects.filter(metadata_attempts__lt=5).filter( + Q(metadata_status=SavedItem.MetadataStatus.PENDING) | + Q(metadata_status=SavedItem.MetadataStatus.ERROR, metadata_retry_at__lte=now)).order_by('created_at') + for item in candidates[:max(0, min(options['limit'], 100))]: + item.metadata_attempts += 1 + try: + fields = fetch_metadata(item.original_url) + for name, value in fields.items(): + setattr(item, name, value) + item.metadata_status = SavedItem.MetadataStatus.READY + item.metadata_error = '' + item.metadata_fetched_at = timezone.now() + item.metadata_retry_at = None + item.save(update_fields=[*fields, 'metadata_status', 'metadata_error', 'metadata_fetched_at', + 'metadata_retry_at', 'metadata_attempts', 'updated_at']) + except MetadataError as exc: + item.metadata_status = SavedItem.MetadataStatus.ERROR + item.metadata_error = str(exc)[:200] + item.metadata_retry_at = timezone.now() + timedelta(minutes=min(5 * 2 ** (item.metadata_attempts - 1), 1440)) + item.save(update_fields=['metadata_status', 'metadata_error', 'metadata_retry_at', + 'metadata_attempts', 'updated_at']) + total += 1 + self.stdout.write(f'Processed {total} link(s).') diff --git a/knowledge/management/commands/rediscover_knowledge.py b/knowledge/management/commands/rediscover_knowledge.py new file mode 100644 index 0000000..85eb2ea --- /dev/null +++ b/knowledge/management/commands/rediscover_knowledge.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand, CommandError +from knowledge.notifications import NotificationError, rediscover_once + + +class Command(BaseCommand): + help = 'Send at most one eligible Knowledge Library ntfy reminder, subject to opt-in, quiet hours and weekly limit.' + + def handle(self, *args, **options): + try: + self.stdout.write(rediscover_once()) + except NotificationError as exc: + raise CommandError(str(exc)) from exc diff --git a/knowledge/metadata.py b/knowledge/metadata.py new file mode 100644 index 0000000..6da253e --- /dev/null +++ b/knowledge/metadata.py @@ -0,0 +1,212 @@ +"""Bounded, DNS-pinned preview metadata for public web pages and YouTube.""" +import http.client +import ipaddress +import json +import re +import socket +import ssl +import time +from html.parser import HTMLParser +from urllib.parse import parse_qs, urlencode, urljoin, urlsplit + +from django.core.exceptions import ValidationError + +from .services import clean_url + + +MAX_HTML_BYTES = 512 * 1024 +MAX_JSON_BYTES = 16 * 1024 +MAX_REDIRECTS = 3 +VIDEO_ID = re.compile(r'^[A-Za-z0-9_-]{11}$') + + +class MetadataError(Exception): + pass + + +class MetadataParser(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.title_parts = [] + self.in_title = False + self.meta = {} + self.icons = [] + self.canonical = '' + + def handle_starttag(self, tag, attrs): + props = dict(attrs) + if tag == 'title': + self.in_title = True + elif tag == 'meta': + name = (props.get('property') or props.get('name') or '').lower().strip() + value = (props.get('content') or '').strip() + if name and value and name not in self.meta: + self.meta[name] = value + elif tag == 'link': + rel = (props.get('rel') or '').lower().split() + if 'canonical' in rel and not self.canonical: + self.canonical = props.get('href') or '' + if 'icon' in rel and props.get('href'): + self.icons.append(props['href']) + + def handle_endtag(self, tag): + if tag == 'title': + self.in_title = False + + def handle_data(self, data): + if self.in_title and sum(map(len, self.title_parts)) < 300: + self.title_parts.append(data) + + +class PinnedHTTPConnection(http.client.HTTPConnection): + def __init__(self, host, port, address, secure): + super().__init__(host, port, timeout=5) + self.address = address + self.secure = secure + + def connect(self): + raw = socket.create_connection((self.address, self.port), timeout=self.timeout) + self.sock = ssl.create_default_context().wrap_socket(raw, server_hostname=self.host) if self.secure else raw + + +def public_address(host, port): + try: + answers = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except OSError as exc: + raise MetadataError('DNS lookup failed.') from exc + if not answers: + raise MetadataError('No DNS address found.') + addresses = [answer[4][0] for answer in answers] + if any(not ipaddress.ip_address(address).is_global for address in addresses): + raise MetadataError('DNS resolved to a non-public address.') + return addresses[0] + + +def _public_url(value, base): + try: + candidate = clean_url(urljoin(base, value)) + except (ValidationError, ValueError, TypeError): + return '' + return candidate if urlsplit(candidate).scheme == 'https' else '' + + +def _fetch_bytes(url, accepted_type, limit, max_redirects=MAX_REDIRECTS): + try: + current = clean_url(url) + except ValidationError as exc: + raise MetadataError('Invalid page URL.') from exc + deadline = time.monotonic() + 20 + for _ in range(max_redirects + 1): + if time.monotonic() >= deadline: + raise MetadataError('Preview fetch timed out.') + parts = urlsplit(current) + port = parts.port or (443 if parts.scheme == 'https' else 80) + if port not in (80, 443): + raise MetadataError('Only standard web ports are fetched.') + address = public_address(parts.hostname, port) + connection = PinnedHTTPConnection(parts.hostname, port, address, parts.scheme == 'https') + try: + target = (parts.path or '/') + (f'?{parts.query}' if parts.query else '') + connection.request('GET', target, headers={'Host': parts.netloc, 'User-Agent': 'CommonGroundKnowledge/1.0', + 'Accept': accepted_type, 'Accept-Encoding': 'identity', 'Connection': 'close'}) + response = connection.getresponse() + if response.status in (301, 302, 303, 307, 308): + location = response.getheader('Location') + if not location: + raise MetadataError('Redirect has no location.') + try: + current = clean_url(urljoin(current, location)) + except ValidationError as exc: + raise MetadataError('Redirect points to a disallowed URL.') from exc + continue + if response.status != 200: + raise MetadataError(f'HTTP {response.status}.') + content_type = (response.getheader('Content-Type') or '').lower().split(';', 1)[0].strip() + if content_type != accepted_type: + raise MetadataError('Unexpected content type.') + length = response.getheader('Content-Length') + if length and int(length) > limit: + raise MetadataError('Response is too large.') + chunks = [] + size = 0 + while True: + if time.monotonic() >= deadline: + raise MetadataError('Preview fetch timed out.') + chunk = response.read(min(8192, limit + 1 - size)) + if not chunk: + return current, b''.join(chunks) + size += len(chunk) + if size > limit: + raise MetadataError('Response is too large.') + chunks.append(chunk) + except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc: + raise MetadataError('Could not fetch preview metadata.') from exc + finally: + connection.close() + raise MetadataError('Too many redirects.') + + +def youtube_video_id(url): + parts = urlsplit(url) + host = (parts.hostname or '').lower() + if host in ('youtube.com', 'www.youtube.com', 'm.youtube.com'): + if parts.path == '/watch': + value = parse_qs(parts.query).get('v', [''])[0] + elif parts.path.startswith(('/shorts/', '/live/', '/embed/')): + value = parts.path.split('/')[2] + else: + return '' + elif host == 'youtu.be': + value = parts.path.strip('/') + else: + return '' + return value if VIDEO_ID.fullmatch(value or '') else '' + + +def _youtube_metadata(url, video_id): + endpoint = 'https://www.youtube.com/oembed?' + urlencode({'url': url, 'format': 'json'}) + _, body = _fetch_bytes(endpoint, 'application/json', MAX_JSON_BYTES, max_redirects=0) + try: + data = json.loads(body) + if not isinstance(data, dict): + raise ValueError() + except (ValueError, UnicodeError) as exc: + raise MetadataError('Invalid YouTube preview.') from exc + image = _public_url(data.get('thumbnail_url'), url) if isinstance(data.get('thumbnail_url'), str) else '' + return {'extracted_title': str(data.get('title') or '')[:300], 'description': '', + 'thumbnail_url': image or f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg', + 'image_alt': f'Thumbnail for {str(data.get("title") or "YouTube video")[:260]}', + 'site_name': str(data.get('author_name') or 'YouTube')[:120], 'favicon_url': '', + 'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}', + 'canonical_url': f'https://www.youtube.com/watch?v={video_id}'} + + +def fetch_metadata(url): + video_id = youtube_video_id(url) + if video_id: + try: + return _youtube_metadata(url, video_id) + except MetadataError: + # A known video still gets a useful card when oEmbed is unavailable. + return {'extracted_title': '', 'description': '', + 'thumbnail_url': f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg', + 'image_alt': 'YouTube video thumbnail', 'site_name': 'YouTube', 'favicon_url': '', + 'provider': 'youtube', 'embed_url': f'https://www.youtube-nocookie.com/embed/{video_id}', + 'canonical_url': f'https://www.youtube.com/watch?v={video_id}'} + final_url, body = _fetch_bytes(url, 'text/html', MAX_HTML_BYTES) + parser = MetadataParser() + parser.feed(body.decode('utf-8', errors='replace')) + meta = parser.meta + title = meta.get('og:title') or meta.get('twitter:title') or ''.join(parser.title_parts) + description = meta.get('og:description') or meta.get('twitter:description') or meta.get('description') or '' + image = meta.get('og:image') or meta.get('twitter:image') or '' + canonical = _public_url(parser.canonical, final_url) if parser.canonical else '' + if canonical and urlsplit(canonical).hostname != urlsplit(final_url).hostname: + canonical = '' + icon = _public_url(parser.icons[0], final_url) if parser.icons else '' + return {'extracted_title': ' '.join(title.split())[:300], 'description': description[:2000], + 'thumbnail_url': _public_url(image, final_url) if image else '', + 'image_alt': (meta.get('og:image:alt') or meta.get('twitter:image:alt') or '')[:300], + 'site_name': (meta.get('og:site_name') or urlsplit(final_url).hostname or '')[:120], + 'favicon_url': icon or _public_url('/favicon.ico', final_url), + 'provider': 'web', 'embed_url': '', 'canonical_url': canonical} diff --git a/knowledge/migrations/0001_initial.py b/knowledge/migrations/0001_initial.py new file mode 100644 index 0000000..59bde10 --- /dev/null +++ b/knowledge/migrations/0001_initial.py @@ -0,0 +1,83 @@ +# Generated by Django 6.1.2 on 2026-10-20 17:27 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='CaptureToken', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=80)), + ('token_hash', models.CharField(max_length=64, unique=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('revoked_at', models.DateTimeField(blank=True, null=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='knowledge_tokens', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='SavedItem', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('original_url', models.URLField(max_length=2048)), + ('url_key', models.CharField(max_length=2048)), + ('canonical_url', models.URLField(blank=True, max_length=2048)), + ('title', models.CharField(blank=True, max_length=300)), + ('extracted_title', models.CharField(blank=True, max_length=300)), + ('description', models.TextField(blank=True)), + ('thumbnail_url', models.URLField(blank=True, max_length=2048)), + ('notes', models.TextField(blank=True)), + ('tags', models.JSONField(blank=True, default=list)), + ('saved_reason', models.CharField(blank=True, max_length=300)), + ('deferral_reason', models.CharField(blank=True, max_length=120)), + ('status', models.CharField(choices=[('saved', 'Saved'), ('read', 'Read'), ('archived', 'Archived')], default='saved', max_length=12)), + ('metadata_status', models.CharField(choices=[('pending', 'Pending'), ('ready', 'Ready'), ('error', 'Error')], default='pending', max_length=12)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('read_at', models.DateTimeField(blank=True, null=True)), + ('last_surfaced_at', models.DateTimeField(blank=True, null=True)), + ('snoozed_until', models.DateTimeField(blank=True, null=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='knowledge_items', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + migrations.CreateModel( + name='RediscoveryEvent', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('channel', models.CharField(choices=[('mcp', 'MCP'), ('ntfy', 'ntfy')], max_length=12)), + ('outcome', models.CharField(max_length=20)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rediscovery_events', to='knowledge.saveditem')), + ], + ), + migrations.AddIndex( + model_name='saveditem', + index=models.Index(fields=['owner', 'status', '-created_at'], name='knowledge_s_owner_i_909a54_idx'), + ), + migrations.AddIndex( + model_name='saveditem', + index=models.Index(fields=['owner', 'last_surfaced_at'], name='knowledge_s_owner_i_89f98e_idx'), + ), + migrations.AddConstraint( + model_name='saveditem', + constraint=models.UniqueConstraint(fields=('owner', 'url_key'), name='knowledge_owner_url_unique'), + ), + migrations.AddIndex( + model_name='rediscoveryevent', + index=models.Index(fields=['item', '-created_at'], name='knowledge_r_item_id_0f6d7f_idx'), + ), + ] diff --git a/knowledge/migrations/0002_saveditem_embed_url_saveditem_favicon_url_and_more.py b/knowledge/migrations/0002_saveditem_embed_url_saveditem_favicon_url_and_more.py new file mode 100644 index 0000000..8b1d8e2 --- /dev/null +++ b/knowledge/migrations/0002_saveditem_embed_url_saveditem_favicon_url_and_more.py @@ -0,0 +1,68 @@ +# Generated by Django 6.1.2 on 2026-10-20 17:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('knowledge', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='saveditem', + name='embed_url', + field=models.URLField(blank=True, max_length=2048), + ), + migrations.AddField( + model_name='saveditem', + name='favicon_url', + field=models.URLField(blank=True, max_length=2048), + ), + migrations.AddField( + model_name='saveditem', + name='image_alt', + field=models.CharField(blank=True, max_length=300), + ), + migrations.AddField( + model_name='saveditem', + name='metadata_attempts', + field=models.PositiveSmallIntegerField(default=0), + ), + migrations.AddField( + model_name='saveditem', + name='metadata_error', + field=models.CharField(blank=True, max_length=200), + ), + migrations.AddField( + model_name='saveditem', + name='metadata_fetched_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='saveditem', + name='metadata_retry_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='saveditem', + name='preview_mode', + field=models.CharField(choices=[('preview', 'Preview'), ('plain', 'Plain link'), ('embed', 'Playable embed')], default='preview', max_length=12), + ), + migrations.AddField( + model_name='saveditem', + name='provider', + field=models.CharField(blank=True, max_length=40), + ), + migrations.AddField( + model_name='saveditem', + name='site_name', + field=models.CharField(blank=True, max_length=120), + ), + migrations.AddField( + model_name='saveditem', + name='user_description', + field=models.TextField(blank=True), + ), + ] diff --git a/knowledge/migrations/__init__.py b/knowledge/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knowledge/models.py b/knowledge/models.py new file mode 100644 index 0000000..0b3a636 --- /dev/null +++ b/knowledge/models.py @@ -0,0 +1,75 @@ +import uuid +from urllib.parse import urlsplit +from django.conf import settings +from django.db import models + + +class SavedItem(models.Model): + class Status(models.TextChoices): + SAVED = 'saved', 'Saved' + READ = 'read', 'Read' + ARCHIVED = 'archived', 'Archived' + + class MetadataStatus(models.TextChoices): + PENDING = 'pending', 'Pending' + READY = 'ready', 'Ready' + ERROR = 'error', 'Error' + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='knowledge_items') + original_url = models.URLField(max_length=2048) + url_key = models.CharField(max_length=2048) + canonical_url = models.URLField(max_length=2048, blank=True) + title = models.CharField(max_length=300, blank=True) + user_description = models.TextField(blank=True) + extracted_title = models.CharField(max_length=300, blank=True) + description = models.TextField(blank=True) + thumbnail_url = models.URLField(max_length=2048, blank=True) + image_alt = models.CharField(max_length=300, blank=True) + site_name = models.CharField(max_length=120, blank=True) + favicon_url = models.URLField(max_length=2048, blank=True) + provider = models.CharField(max_length=40, blank=True) + embed_url = models.URLField(max_length=2048, blank=True) + preview_mode = models.CharField(max_length=12, choices=[('preview', 'Preview'), ('plain', 'Plain link'), ('embed', 'Playable embed')], default='preview') + notes = models.TextField(blank=True) + tags = models.JSONField(default=list, blank=True) + saved_reason = models.CharField(max_length=300, blank=True) + deferral_reason = models.CharField(max_length=120, blank=True) + status = models.CharField(max_length=12, choices=Status.choices, default=Status.SAVED) + metadata_status = models.CharField(max_length=12, choices=MetadataStatus.choices, default=MetadataStatus.PENDING) + metadata_error = models.CharField(max_length=200, blank=True) + metadata_fetched_at = models.DateTimeField(null=True, blank=True) + metadata_retry_at = models.DateTimeField(null=True, blank=True) + metadata_attempts = models.PositiveSmallIntegerField(default=0) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + read_at = models.DateTimeField(null=True, blank=True) + last_surfaced_at = models.DateTimeField(null=True, blank=True) + snoozed_until = models.DateTimeField(null=True, blank=True) + + class Meta: + constraints = [models.UniqueConstraint(fields=['owner', 'url_key'], name='knowledge_owner_url_unique')] + indexes = [models.Index(fields=['owner', 'status', '-created_at']), models.Index(fields=['owner', 'last_surfaced_at'])] + ordering = ['-created_at'] + + @property + def domain(self): + return urlsplit(self.original_url).hostname or self.original_url + + +class CaptureToken(models.Model): + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='knowledge_tokens') + name = models.CharField(max_length=80) + token_hash = models.CharField(max_length=64, unique=True) + created_at = models.DateTimeField(auto_now_add=True) + revoked_at = models.DateTimeField(null=True, blank=True) + + +class RediscoveryEvent(models.Model): + item = models.ForeignKey(SavedItem, on_delete=models.CASCADE, related_name='rediscovery_events') + channel = models.CharField(max_length=12, choices=[('mcp', 'MCP'), ('ntfy', 'ntfy')]) + outcome = models.CharField(max_length=20) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + indexes = [models.Index(fields=['item', '-created_at'])] diff --git a/knowledge/notifications.py b/knowledge/notifications.py new file mode 100644 index 0000000..21956d4 --- /dev/null +++ b/knowledge/notifications.py @@ -0,0 +1,83 @@ +import random +import socket +from datetime import timedelta +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.db import transaction +from django.db import models +from django.utils import timezone + +from .models import RediscoveryEvent, SavedItem + + +class NotificationError(Exception): + pass + + +def send_ntfy_notification(url, title, message, timeout=10): + # Matches TasksNest's UTF-8 POST + Title header convention. URL comes from server environment. + request = Request(url, data=message.encode('utf-8'), + headers={'Content-Type': 'text/plain; charset=utf-8', 'Title': title}, method='POST') + try: + with urlopen(request, timeout=timeout) as response: + status = response.getcode() + except HTTPError as error: + raise NotificationError(f'ntfy responded with HTTP {error.code}.') from error + except (TimeoutError, socket.timeout, URLError, OSError) as error: + raise NotificationError('Could not connect to ntfy.') from error + if not 200 <= status < 300: + raise NotificationError(f'ntfy responded with HTTP {status}.') + + +def choose_item(user, mode='forgotten', now=None): + now = now or timezone.now() + cutoff = now - timedelta(days=14) + items = list(SavedItem.objects.filter(owner=user, status=SavedItem.Status.SAVED, + created_at__lte=now - timedelta(days=2)).filter( + models.Q(snoozed_until__isnull=True) | models.Q(snoozed_until__lte=now)).filter( + models.Q(last_surfaced_at__isnull=True) | models.Q(last_surfaced_at__lte=cutoff))[:500]) + if not items: + return None + if mode == 'random': + return random.choice(items) + # "Timely" is deliberately recency based, not an unsupported claim about price or availability. + if mode == 'timely': + return max(items, key=lambda item: item.created_at) + return min(items, key=lambda item: item.last_surfaced_at or item.created_at) + + +def rediscover_once(now=None): + now = now or timezone.now() + if not settings.KNOWLEDGE_NTFY_ENABLED: + return 'disabled' + if not settings.KNOWLEDGE_NTFY_URL or not settings.KNOWLEDGE_NTFY_USER_ID: + return 'unconfigured' + local_hour = timezone.localtime(now).hour + start, end = settings.KNOWLEDGE_NTFY_QUIET_START, settings.KNOWLEDGE_NTFY_QUIET_END + in_quiet = start <= local_hour < end if start < end else local_hour >= start or local_hour < end + if in_quiet: + return 'quiet_hours' + user = get_user_model().objects.filter(pk=settings.KNOWLEDGE_NTFY_USER_ID).first() + if user is None: + return 'unknown_user' + week_start = now - timedelta(days=7) + delivered = RediscoveryEvent.objects.filter(item__owner=user, channel='ntfy', outcome='delivered', created_at__gte=week_start).count() + if delivered >= max(0, min(settings.KNOWLEDGE_NTFY_WEEKLY_LIMIT, 7)): + return 'weekly_limit' + mode = settings.KNOWLEDGE_NTFY_MODE if settings.KNOWLEDGE_NTFY_MODE in ('random', 'forgotten', 'timely') else 'forgotten' + item = choose_item(user, mode, now) + if item is None: + return 'no_eligible_items' + title = item.title or item.extracted_title or 'Saved link to revisit' + message = f'{title}\n{item.original_url}' + if item.saved_reason: + message += f'\nYou saved it because: {item.saved_reason}' + send_ntfy_notification(settings.KNOWLEDGE_NTFY_URL, 'Knowledge Library', message) + with transaction.atomic(): + item.last_surfaced_at = now + item.save(update_fields=['last_surfaced_at', 'updated_at']) + RediscoveryEvent.objects.create(item=item, channel='ntfy', outcome='delivered') + return 'delivered' diff --git a/knowledge/services.py b/knowledge/services.py new file mode 100644 index 0000000..d8485c2 --- /dev/null +++ b/knowledge/services.py @@ -0,0 +1,80 @@ +import ipaddress +import re +from urllib.parse import urlsplit, urlunsplit + +from django.core.exceptions import ValidationError +from django.db.models import Q +from django.utils import timezone +from datetime import timedelta + +from .models import SavedItem + + +def clean_url(value): + value = (value or '').strip() + if len(value) > 2048 or any(ord(c) < 32 for c in value): + raise ValidationError('Enter a valid URL under 2048 characters.') + try: + parts = urlsplit(value) + host = parts.hostname + port = parts.port + except ValueError as exc: + raise ValidationError('Enter a valid URL.') from exc + if parts.scheme.lower() not in ('http', 'https') or not host or parts.username or parts.password: + raise ValidationError('Only public HTTP(S) URLs are accepted.') + host = host.lower().rstrip('.') + if '.' not in host and ':' not in host: + raise ValidationError('A public hostname is required.') + try: + address = ipaddress.ip_address(host.strip('[]')) + except ValueError: + address = None + if address and not address.is_global: + raise ValidationError('Local and private addresses are not accepted.') + if host in ('localhost', 'localhost.localdomain') or host.endswith(('.local', '.internal', '.localhost')): + raise ValidationError('Local hostnames are not accepted.') + authority = host if ':' not in host else f'[{host}]' + if port and port != (443 if parts.scheme.lower() == 'https' else 80): + authority += f':{port}' + key = urlunsplit((parts.scheme.lower(), authority, parts.path or '/', parts.query, '')) + if len(key) > 2048: + raise ValidationError('URL is too long.') + return key + + +def save_item(owner, url, **fields): + key = clean_url(url) + allowed = {'title', 'notes', 'saved_reason', 'deferral_reason', 'tags'} + defaults = {name: value for name, value in fields.items() if name in allowed} + item, created = SavedItem.objects.get_or_create(owner=owner, url_key=key, + defaults={'original_url': url.strip(), **defaults}) + return item, created + + +def search_items(owner, query, limit=20): + words = re.findall(r'[\w-]{2,}', (query or '').lower())[:8] + qs = SavedItem.objects.filter(owner=owner).exclude(status=SavedItem.Status.ARCHIVED) + for word in words: + qs = qs.filter(Q(title__icontains=word) | Q(extracted_title__icontains=word) | + Q(original_url__icontains=word) | Q(description__icontains=word) | + Q(notes__icontains=word) | Q(saved_reason__icontains=word) | Q(tags__icontains=word)) + return list(qs[:max(1, min(limit, 100))]) + + +def relevant_items(owner, context): + words = set(re.findall(r'[\w-]{4,}', (context or '').lower())) + words -= {'this', 'that', 'with', 'from', 'about', 'what', 'would', 'could', 'should', 'there', 'their'} + if not words: + return [] + now = timezone.now() + candidates = SavedItem.objects.filter(owner=owner, status=SavedItem.Status.SAVED).filter( + Q(snoozed_until__isnull=True) | Q(snoozed_until__lte=now)).filter( + Q(last_surfaced_at__isnull=True) | Q(last_surfaced_at__lte=now - timedelta(days=14))) + scored = [] + for item in candidates[:500]: + text = ' '.join([item.title, item.extracted_title, item.description, item.saved_reason, ' '.join(item.tags)]).lower() + hits = words & set(re.findall(r'[\w-]{4,}', text)) + if len(hits) >= 2 or (len(hits) == 1 and len(next(iter(hits))) >= 6): + scored.append((len(hits), item, sorted(hits))) + scored.sort(key=lambda row: (-row[0], -row[1].created_at.timestamp())) + return scored[:3] diff --git a/knowledge/static/knowledge/previews.css b/knowledge/static/knowledge/previews.css new file mode 100644 index 0000000..992cca7 --- /dev/null +++ b/knowledge/static/knowledge/previews.css @@ -0,0 +1,4 @@ +.knowledge-form{margin-bottom:20px}.knowledge-fields{display:grid;gap:6px;margin-bottom:14px;max-width:680px}.knowledge-fields label,.knowledge-search label{font-weight:700}.knowledge-fields input,.knowledge-search input,.knowledge-actions input,.knowledge-actions textarea,.knowledge-actions select{width:100%;padding:9px;border:1px solid var(--border);border-radius:8px;background:var(--surface);color:var(--text)}.knowledge-search{max-width:680px}.knowledge-search>div{display:flex;gap:8px}.knowledge-list{display:grid;gap:14px;margin-top:20px}.knowledge-item{padding:0;overflow:hidden;display:grid;grid-template-columns:minmax(0,220px) minmax(0,1fr)}.knowledge-item:not(:has(.knowledge-image-wrap)){grid-template-columns:1fr}.knowledge-image-wrap{background:var(--surface-secondary);min-height:180px;display:grid;place-items:center}.knowledge-image{width:100%;height:100%;max-height:300px;object-fit:cover}.knowledge-image-fallback{color:var(--text-secondary)}.knowledge-body{padding:20px;min-width:0}.knowledge-site{font-size:11px;color:var(--text-secondary);font-weight:700}.knowledge-body h2{font-size:19px;overflow-wrap:anywhere}.knowledge-description,.knowledge-reason{color:var(--text-secondary)}.knowledge-state{font-size:11px;color:var(--text-muted)}.knowledge-actions{display:flex;gap:12px;align-items:start;flex-wrap:wrap;margin-top:14px}.knowledge-actions button,.knowledge-actions summary,.knowledge-play{border:1px solid var(--border);border-radius:7px;padding:7px 10px;background:var(--surface-secondary);color:var(--text);cursor:pointer}.knowledge-actions details{min-width:min(100%,250px)}.knowledge-actions details[open]{width:min(100%,450px)}.knowledge-actions details form{display:grid;gap:10px;padding-top:12px}.knowledge-actions label{display:grid;gap:4px;font-weight:700}.knowledge-actions textarea{min-height:75px}.knowledge-player{margin-top:12px}.knowledge-third-party{font-size:11px;color:var(--text-muted);margin:5px 0}.knowledge-player iframe{width:100%;aspect-ratio:16/9;border:0;border-radius:8px}@media(max-width:680px){.knowledge-item{grid-template-columns:1fr}.knowledge-image-wrap{height:190px;min-height:0}.knowledge-body{padding:16px}}@media(prefers-reduced-motion:reduce){.knowledge-item{transition:none}} +.knowledge-favicon{width:16px;height:16px;object-fit:contain;vertical-align:middle;margin-right:5px} +.knowledge-item--embed{grid-template-columns:1fr}.knowledge-item--embed .knowledge-image-wrap{max-height:330px}.knowledge-item--embed .knowledge-player{min-height:54px} +.knowledge-image-wrap{overflow:hidden}.knowledge-image{display:block}@media(max-width:680px){.knowledge-image-wrap{height:190px}.knowledge-image{height:190px;max-height:190px}} diff --git a/knowledge/static/knowledge/previews.js b/knowledge/static/knowledge/previews.js new file mode 100644 index 0000000..2053916 --- /dev/null +++ b/knowledge/static/knowledge/previews.js @@ -0,0 +1,40 @@ +document.addEventListener('DOMContentLoaded', () => { + document.addEventListener('error', (event) => { + if (event.target.matches?.('.knowledge-favicon')) { event.target.hidden = true; return; } + if (!event.target.matches?.('.knowledge-image')) return; + event.target.hidden = true; + const fallback = event.target.parentElement.querySelector('.knowledge-image-fallback'); + if (fallback) fallback.hidden = false; + }, true); + document.addEventListener('click', (event) => { + const button = event.target.closest?.('.knowledge-play'); + if (!button) return; + const player = button.closest('.knowledge-player'); + const url = player?.dataset.embedUrl || ''; + if (!/^https:\/\/www\.youtube-nocookie\.com\/embed\/[A-Za-z0-9_-]{11}$/.test(url)) return; + const frame = document.createElement('iframe'); + frame.src = url; + frame.title = 'YouTube video player'; + frame.referrerPolicy = 'no-referrer'; + frame.allow = 'accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture; web-share'; + frame.allowFullscreen = true; + player.replaceChildren(frame); + frame.focus(); + }); + const started = Date.now(); + const poll = async () => { + if (document.hidden || Date.now() - started > 120000) return; + const cards = [...document.querySelectorAll('.knowledge-item[data-metadata-status="pending"]')].slice(0, 20); + for (const card of cards) { + try { + const response = await fetch(card.dataset.cardUrl, {credentials: 'same-origin', cache: 'no-store'}); + if (!response.ok) continue; + const html = await response.text(); + const documentFragment = new DOMParser().parseFromString(html, 'text/html'); + const replacement = documentFragment.querySelector('.knowledge-item'); + if (replacement && replacement.dataset.metadataStatus !== 'pending') card.replaceWith(replacement); + } catch (_) { /* Preserve the current card on a transient polling failure. */ } + } + }; + const timer = setInterval(() => { if (Date.now() - started > 120000) clearInterval(timer); else poll(); }, 10000); +}); diff --git a/knowledge/templates/knowledge/_item_card.html b/knowledge/templates/knowledge/_item_card.html new file mode 100644 index 0000000..192ba18 --- /dev/null +++ b/knowledge/templates/knowledge/_item_card.html @@ -0,0 +1,12 @@ +
+ {% if item.preview_mode != 'plain' and item.thumbnail_url %}
{{ item.image_alt|default:'Preview image for saved link' }}
{% endif %} +
+

{% if item.favicon_url and item.preview_mode != 'plain' %}{% endif %}{% if item.site_name %}{{ item.site_name }}{% else %}{{ item.domain }}{% endif %}{% if item.provider == 'youtube' %} · YouTube{% endif %}

+

{{ item.title|default:item.extracted_title|default:item.original_url }}

+ {% if item.preview_mode != 'plain' %}{% if item.user_description or item.description %}

{{ item.user_description|default:item.description }}

{% endif %}{% endif %} + {% if item.saved_reason %}

Saved because: {{ item.saved_reason }}

{% endif %} +

Saved {{ item.created_at|date:'j M Y' }} · {% if item.metadata_status == 'pending' %}Preview loading{% elif item.metadata_status == 'error' %}Preview unavailable{% if item.metadata_error %}: {{ item.metadata_error }}{% endif %}{% else %}Preview updated {{ item.metadata_fetched_at|date:'j M Y' }}{% endif %}

+ {% if item.provider == 'youtube' and item.preview_mode != 'plain' and item.embed_url %}

Playing loads YouTube content.

{% endif %} +
Edit preview
{% csrf_token %}
{% csrf_token %}
{% csrf_token %}
+
+
diff --git a/knowledge/templates/knowledge/index.html b/knowledge/templates/knowledge/index.html new file mode 100644 index 0000000..8f98c50 --- /dev/null +++ b/knowledge/templates/knowledge/index.html @@ -0,0 +1,10 @@ +{% extends 'core/base.html' %}{% load static %} +{% block extra_head %}{% endblock %} +{% block content %} +

Knowledge Library

Save things you want to revisit when the time is right.

Capture tokens
+

Save a link

+{% if error %}

{{ error }}

{% endif %} +
{% csrf_token %}
+ +
{% for item in items %}{% include 'knowledge/_item_card.html' %}{% empty %}

No saved links found.

{% endfor %}
+{% endblock %} diff --git a/knowledge/templates/knowledge/tokens.html b/knowledge/templates/knowledge/tokens.html new file mode 100644 index 0000000..276c285 --- /dev/null +++ b/knowledge/templates/knowledge/tokens.html @@ -0,0 +1,5 @@ +{% extends 'core/base.html' %} +{% block content %}

Capture tokens

Create a token for an Apple Shortcut that sends a URL to this library. Tokens only permit saving links.

+{% if secret %}
Copy this token now; it will not be shown again.

{{ secret }}

{% endif %} +
{% csrf_token %}
+

Existing tokens

Back to library{% endblock %} diff --git a/knowledge/tests.py b/knowledge/tests.py new file mode 100644 index 0000000..a46a53c --- /dev/null +++ b/knowledge/tests.py @@ -0,0 +1,183 @@ +import hashlib +import json +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.urls import reverse +from django.core.management import call_command +from django.test import override_settings +from django.utils import timezone +from datetime import timedelta +from unittest.mock import patch + +from .models import CaptureToken, SavedItem +from .services import relevant_items, save_item +from .notifications import rediscover_once +from .metadata import MetadataError, fetch_metadata, public_address, youtube_video_id + + +class KnowledgeTests(TestCase): + def setUp(self): + self.alice = get_user_model().objects.create_user('alice', password='test-password') + self.bob = get_user_model().objects.create_user('bob', password='test-password') + + def test_capture_is_idempotent_and_owner_scoped(self): + token = 'capture-secret' + CaptureToken.objects.create(owner=self.alice, name='Shortcut', token_hash=hashlib.sha256(token.encode()).hexdigest()) + url = reverse('knowledge_api_add') + headers = {'HTTP_AUTHORIZATION': f'Bearer {token}', 'content_type': 'application/json'} + first = self.client.post(url, data=json.dumps({'url': 'https://example.org/esp32#top', 'title': 'ESP32'}), **headers) + second = self.client.post(url, data=json.dumps({'url': 'https://EXAMPLE.org/esp32'}), **headers) + self.assertEqual(first.status_code, 201) + self.assertEqual(second.status_code, 200) + self.assertTrue(second.json()['duplicate']) + self.assertEqual(SavedItem.objects.filter(owner=self.alice).count(), 1) + self.assertEqual(SavedItem.objects.filter(owner=self.bob).count(), 0) + self.assertEqual(self.client.post(url, data='{}', content_type='application/json').status_code, 401) + + def test_private_url_and_cross_user_visibility(self): + token = 'capture-secret' + CaptureToken.objects.create(owner=self.alice, name='Shortcut', token_hash=hashlib.sha256(token.encode()).hexdigest()) + response = self.client.post(reverse('knowledge_api_add'), data=json.dumps({'url': 'http://127.0.0.1/admin'}), + content_type='application/json', HTTP_AUTHORIZATION=f'Bearer {token}') + self.assertEqual(response.status_code, 400) + item, _ = save_item(self.alice, 'https://example.org/sensor', title='ESP32 sensor microphone') + self.assertEqual(relevant_items(self.bob, 'ESP32 sensor microphone'), []) + self.assertEqual(relevant_items(self.alice, 'ESP32 sensor microphone')[0][1], item) + self.client.force_login(self.bob) + self.assertNotContains(self.client.get(reverse('knowledge:index')), 'ESP32 sensor microphone') + self.assertEqual(self.client.post(reverse('knowledge:archive', args=[item.pk])).status_code, 404) + + def test_token_revocation(self): + self.client.force_login(self.alice) + self.client.post(reverse('knowledge:tokens'), {'name': 'Shortcut'}) + token = CaptureToken.objects.get(owner=self.alice) + self.client.post(reverse('knowledge:tokens'), {'action': 'revoke', 'token_id': token.pk}) + token.refresh_from_db() + self.assertIsNotNone(token.revoked_at) + + @override_settings(KNOWLEDGE_NTFY_ENABLED=True, KNOWLEDGE_NTFY_URL='https://ntfy.example.org/topic', + KNOWLEDGE_NTFY_WEEKLY_LIMIT=1, KNOWLEDGE_NTFY_QUIET_START=23, KNOWLEDGE_NTFY_QUIET_END=6) + @patch('knowledge.notifications.send_ntfy_notification') + def test_ntfy_limit_and_cooldown(self, sender): + item, _ = save_item(self.alice, 'https://example.org/esp32', title='ESP32 sensors') + SavedItem.objects.filter(pk=item.pk).update(created_at=timezone.now() - timedelta(days=5)) + with override_settings(KNOWLEDGE_NTFY_USER_ID=str(self.alice.pk)): + now = timezone.now().replace(hour=12) + self.assertEqual(rediscover_once(now), 'delivered') + self.assertEqual(rediscover_once(now), 'weekly_limit') + sender.assert_called_once() + + @patch('knowledge.metadata.socket.getaddrinfo') + def test_metadata_rejects_private_dns_answer(self, resolver): + resolver.return_value = [(None, None, None, None, ('192.168.1.1', 443))] + with self.assertRaises(MetadataError): + public_address('example.org', 443) + + @patch('knowledge.metadata.PinnedHTTPConnection') + @patch('knowledge.metadata.public_address', return_value='93.184.215.14') + def test_metadata_reads_only_bounded_html(self, resolver, connection_type): + response = connection_type.return_value.getresponse.return_value + response.status = 200 + response.getheader.side_effect = lambda name: {'Content-Type': 'text/html; charset=utf-8', 'Content-Length': '90'}.get(name) + response.read.side_effect = [b'Useful ESP32 page', b''] + result = fetch_metadata('https://example.org/esp32') + self.assertEqual(result['extracted_title'], 'Useful ESP32 page') + self.assertEqual(result['description'], 'A sensor guide') + connection_type.assert_called_once_with('example.org', 443, '93.184.215.14', True) + + @patch('knowledge.metadata._fetch_bytes') + def test_open_graph_precedence_and_fallback(self, fetch): + html = b'''HTML title + + + + ''' + fetch.return_value = ('https://example.org/article', html) + result = fetch_metadata('https://example.org/article') + self.assertEqual(result['extracted_title'], 'OG title') + self.assertEqual(result['description'], 'OG description') + self.assertEqual(result['thumbnail_url'], 'https://example.org/og.jpg') + self.assertEqual(result['image_alt'], 'A flower') + self.assertEqual(result['site_name'], 'Example Journal') + self.assertEqual(result['favicon_url'], 'https://example.org/favicon.png') + fetch.return_value = ('https://example.org/article', b'HTML only') + self.assertEqual(fetch_metadata('https://example.org/article')['extracted_title'], 'HTML only') + fetch.return_value = ('https://example.org/article', b'Nothing to extract') + self.assertEqual(fetch_metadata('https://example.org/article')['extracted_title'], '') + + @patch('knowledge.metadata._fetch_bytes') + def test_youtube_oembed_uses_allowlisted_embed(self, fetch): + fetch.return_value = ('https://www.youtube.com/oembed', json.dumps({ + 'title': 'Circuit demo', 'author_name': 'Maker channel', + 'thumbnail_url': 'https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg'}).encode()) + result = fetch_metadata('https://youtu.be/dQw4w9WgXcQ') + self.assertEqual(result['site_name'], 'Maker channel') + self.assertEqual(result['embed_url'], 'https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ') + self.assertEqual(youtube_video_id('https://youtube.com.evil.test/watch?v=dQw4w9WgXcQ'), '') + + @patch('knowledge.metadata.PinnedHTTPConnection') + @patch('knowledge.metadata.public_address', return_value='93.184.215.14') + def test_redirect_to_private_address_is_blocked(self, resolver, connection_type): + response = connection_type.return_value.getresponse.return_value + response.status = 302 + response.getheader.return_value = 'http://127.0.0.1/admin' + with self.assertRaises(MetadataError): + fetch_metadata('https://example.org/article') + + @patch('knowledge.metadata.PinnedHTTPConnection') + @patch('knowledge.metadata.public_address', return_value='93.184.215.14') + def test_timeout_is_a_safe_failure(self, resolver, connection_type): + import socket + connection_type.return_value.request.side_effect = socket.timeout() + with self.assertRaises(MetadataError): + fetch_metadata('https://example.org/article') + + @patch('knowledge.management.commands.fetch_knowledge_metadata.fetch_metadata') + def test_metadata_refresh_preserves_user_edits_and_cache(self, fetch): + item, _ = save_item(self.alice, 'https://example.org/article') + self.client.force_login(self.alice) + self.client.post(reverse('knowledge:edit', args=[item.pk]), { + 'title': 'My title', 'description': 'My notes on this', 'preview_mode': 'preview'}) + fetch.return_value = {'extracted_title': 'Publisher title', 'description': 'Publisher description', + 'thumbnail_url': 'https://example.org/image.jpg', 'image_alt': 'Image', 'site_name': 'Example', + 'favicon_url': '', 'provider': 'web', 'embed_url': '', 'canonical_url': ''} + call_command('fetch_knowledge_metadata') + item.refresh_from_db() + self.assertEqual(item.title, 'My title') + self.assertEqual(item.user_description, 'My notes on this') + self.assertEqual(item.description, 'Publisher description') + self.assertEqual(item.metadata_status, SavedItem.MetadataStatus.READY) + call_command('fetch_knowledge_metadata') + self.assertEqual(fetch.call_count, 1) + self.assertContains(self.client.get(reverse('knowledge:index')), 'My title') + self.assertContains(self.client.get(reverse('knowledge:index')), 'My notes on this') + + def test_preview_escapes_metadata_and_defers_youtube_embed(self): + item, _ = save_item(self.alice, 'https://youtu.be/dQw4w9WgXcQ') + item.provider = 'youtube' + item.embed_url = 'https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ' + item.extracted_title = '' + item.thumbnail_url = 'https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg' + item.save() + self.client.force_login(self.alice) + response = self.client.get(reverse('knowledge:index')) + self.assertContains(response, '<script>alert(1)</script>') + self.assertNotContains(response, '/archive/', views.archive, name='archive'), + path('items//card/', views.item_card, name='item_card'), + path('items//edit/', views.edit, name='edit'), + path('items//refresh/', views.refresh, name='refresh'), + path('tokens/', views.tokens, name='tokens'), +] diff --git a/knowledge/views.py b/knowledge/views.py new file mode 100644 index 0000000..b1e34c2 --- /dev/null +++ b/knowledge/views.py @@ -0,0 +1,129 @@ +import hashlib +import json +import secrets +from datetime import timedelta +from django.contrib.auth.decorators import login_required +from django.core.exceptions import ValidationError +from django.core.cache import cache +from django.http import JsonResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.utils import timezone +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST + +from .models import CaptureToken, SavedItem +from .services import save_item, search_items + + +@login_required +def index(request): + query = request.GET.get('q', '').strip()[:200] + items = search_items(request.user, query, 100) if query else SavedItem.objects.filter(owner=request.user).exclude(status=SavedItem.Status.ARCHIVED)[:100] + return render(request, 'knowledge/index.html', {'title': 'Knowledge Library', 'section': 'knowledge', 'items': items, 'query': query}) + + +@login_required +def item_card(request, item_id): + item = get_object_or_404(SavedItem, pk=item_id, owner=request.user) + response = render(request, 'knowledge/_item_card.html', {'item': item}) + response['Cache-Control'] = 'private, no-store' + return response + + +@login_required +@require_POST +def add(request): + try: + item, created = save_item(request.user, request.POST.get('url'), + title=request.POST.get('title', '').strip()[:300], + notes=request.POST.get('notes', '').strip()[:3000], + saved_reason=request.POST.get('saved_reason', '').strip()[:300]) + except ValidationError as exc: + return render(request, 'knowledge/index.html', {'title': 'Knowledge Library', 'section': 'knowledge', + 'items': SavedItem.objects.filter(owner=request.user)[:100], 'error': exc.messages[0], 'form_url': request.POST.get('url', '')}, status=400) + return redirect('knowledge:index') + + +@login_required +@require_POST +def archive(request, item_id): + item = get_object_or_404(SavedItem, pk=item_id, owner=request.user) + item.status = SavedItem.Status.ARCHIVED + item.save(update_fields=['status', 'updated_at']) + return redirect('knowledge:index') + + +@login_required +@require_POST +def edit(request, item_id): + item = get_object_or_404(SavedItem, pk=item_id, owner=request.user) + mode = request.POST.get('preview_mode', 'preview') + if mode not in ('preview', 'plain', 'embed') or (mode == 'embed' and item.provider != 'youtube'): + return JsonResponse({'error': 'Invalid preview mode'}, status=400) + item.title = request.POST.get('title', '').strip()[:300] + item.user_description = request.POST.get('description', '').strip()[:2000] + item.preview_mode = mode + item.save(update_fields=['title', 'user_description', 'preview_mode', 'updated_at']) + return redirect('knowledge:index') + + +@login_required +@require_POST +def refresh(request, item_id): + item = get_object_or_404(SavedItem, pk=item_id, owner=request.user) + if item.metadata_fetched_at and item.metadata_fetched_at > timezone.now() - timedelta(minutes=5): + return redirect('knowledge:index') + item.metadata_status = SavedItem.MetadataStatus.PENDING + item.metadata_retry_at = None + item.metadata_error = '' + item.metadata_attempts = 0 + item.save(update_fields=['metadata_status', 'metadata_retry_at', 'metadata_error', 'metadata_attempts', 'updated_at']) + return redirect('knowledge:index') + + +@login_required +def tokens(request): + secret = None + if request.method == 'POST': + if request.POST.get('action') == 'revoke': + token = get_object_or_404(CaptureToken, pk=request.POST.get('token_id'), owner=request.user) + token.revoked_at = timezone.now() + token.save(update_fields=['revoked_at']) + else: + secret = secrets.token_urlsafe(32) + CaptureToken.objects.create(owner=request.user, name=request.POST.get('name', 'Apple Shortcut')[:80], + token_hash=hashlib.sha256(secret.encode()).hexdigest()) + return render(request, 'knowledge/tokens.html', {'title': 'Capture tokens', 'section': 'knowledge', + 'tokens': CaptureToken.objects.filter(owner=request.user).order_by('-created_at'), 'secret': secret}) + + +@csrf_exempt +@require_POST +def api_add(request): + if len(request.body) > 16_384: + return JsonResponse({'error': 'Request too large'}, status=413) + header = request.headers.get('Authorization', '') + if not header.startswith('Bearer ') or len(header) > 180: + return JsonResponse({'error': 'Bearer capture token required'}, status=401) + token_hash = hashlib.sha256(header[7:].encode()).hexdigest() + token = CaptureToken.objects.select_related('owner').filter(token_hash=token_hash, revoked_at__isnull=True).first() + if token is None: + return JsonResponse({'error': 'Invalid capture token'}, status=401) + rate_key = f'knowledge-capture:{token.pk}:{timezone.now():%Y%m%d%H%M}' + if not cache.add(rate_key, 1, 90): + try: + if cache.incr(rate_key) > 30: + return JsonResponse({'error': 'Too many captures; try again shortly'}, status=429) + except ValueError: + cache.set(rate_key, 1, 90) + try: + payload = json.loads(request.body) + if not isinstance(payload, dict): + raise ValueError() + item, created = save_item(token.owner, payload.get('url'), + title=str(payload.get('title') or '')[:300], notes=str(payload.get('notes') or '')[:3000], + saved_reason=str(payload.get('saved_reason') or '')[:300]) + except (ValueError, TypeError, ValidationError) as exc: + return JsonResponse({'error': str(exc) or 'Invalid JSON'}, status=400) + return JsonResponse({'id': str(item.pk), 'created': created, 'duplicate': not created, + 'url': item.original_url, 'metadata_status': item.metadata_status}, status=201 if created else 200) diff --git a/mcp_server/server.py b/mcp_server/server.py index 008b82d..9a2ff13 100644 --- a/mcp_server/server.py +++ b/mcp_server/server.py @@ -34,6 +34,7 @@ from projects import services from projects.models import Activity, Card, CardComment, CardRevision, CardAttachment, Column, Project, Tag, ProjectDocument, DocumentationSync from projects import attachments from projects import documentation +from knowledge import services as knowledge_services def _verified(raw): @@ -150,11 +151,21 @@ def _safe(operation): def create_server(): - server = MCPServer('Common Ground Projects', description='Projects, Kanban cards, and Markdown specifications.', + server = MCPServer('Common Ground', description='Projects and a separate personal knowledge library.', token_verifier=OAuthTokenVerifier(), auth=AuthSettings( issuer_url=AnyHttpUrl(settings.OAUTH_ISSUER_URL), resource_server_url=AnyHttpUrl(settings.MCP_PUBLIC_URL), required_scopes=[], validate_token_resource=False)) + @server.tool(structured_output=True, description='Find up to three relevant links in the connected user’s private Knowledge Library when the conversation overlaps their saved interests. Call opportunistically for concrete topics; an empty result means do not interrupt the user. This tool cannot observe conversations unless called by the client.') + async def find_relevant_saved_interests(context: str) -> dict[str, Any]: + def run(): + user = _actor('knowledge:read') + matches = knowledge_services.relevant_items(user, context[:2000]) + return {'items': [{'id': str(item.pk), 'title': item.title or item.extracted_title or item.original_url, + 'url': item.original_url, 'saved_reason': item.saved_reason, + 'why_relevant': 'Matches ' + ', '.join(hits)} for _, item, hits in matches]} + return await sync_to_async(lambda: _safe(run))() + @server.tool(structured_output=True, description='List projects accessible to the connected user, with counts and repository metadata.') async def projects_list(limit: int = 50, offset: int = 0) -> dict[str, Any]: def run():