git documentation feature
This commit is contained in:
+14
-1
@@ -6,11 +6,24 @@ class ProjectsConfig(AppConfig):
|
||||
name = 'projects'
|
||||
|
||||
def ready(self):
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.db.models.signals import post_delete
|
||||
from .models import CardAttachment
|
||||
from .models import CardAttachment, Project
|
||||
|
||||
def remove_stored_file(sender, instance, **kwargs):
|
||||
if instance.file:
|
||||
instance.file.delete(save=False)
|
||||
|
||||
post_delete.connect(remove_stored_file, sender=CardAttachment, weak=False, dispatch_uid='projects.remove_attachment_file')
|
||||
|
||||
def remove_project_mirror(sender, instance, **kwargs):
|
||||
root = Path(settings.PROJECT_DOCS_MIRROR_ROOT)
|
||||
path = root / str(instance.pk)
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
return
|
||||
transaction.on_commit(lambda: shutil.rmtree(path, ignore_errors=True))
|
||||
|
||||
post_delete.connect(remove_project_mirror, sender=Project, weak=False, dispatch_uid='projects.remove_documentation_mirror')
|
||||
|
||||
@@ -0,0 +1,463 @@
|
||||
"""Read-only, project-scoped Git documentation index."""
|
||||
|
||||
import base64
|
||||
import difflib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from time import monotonic
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import transaction
|
||||
from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
|
||||
from . import services
|
||||
from .models import DocumentationSync, ProjectDocument
|
||||
|
||||
|
||||
MAX_RESULTS = 100
|
||||
MAX_READ_LINES = 250
|
||||
MAX_READ_CHARS = 30000
|
||||
REF_RE = re.compile(r'^refs/heads/[A-Za-z0-9][A-Za-z0-9._/-]*$')
|
||||
HASH_RE = re.compile(r'^[0-9a-f]{40}([0-9a-f]{24})?$')
|
||||
HEADING_RE = re.compile(r'^#{1,6}\s+(.+?)\s*#*\s*$')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DocumentationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def normal_path(value):
|
||||
if not isinstance(value, str) or not value or len(value) > 500 or '\x00' in value:
|
||||
raise ValidationError('Invalid document path.')
|
||||
if value.startswith('/') or '\\' in value or any(part in ('', '.', '..') for part in value.split('/')):
|
||||
raise ValidationError('Invalid document path.')
|
||||
if any(ord(c) < 32 for c in value):
|
||||
raise ValidationError('Invalid document path.')
|
||||
return value
|
||||
|
||||
|
||||
def indexed_path(value):
|
||||
try:
|
||||
normal_path(value)
|
||||
except ValidationError:
|
||||
return False
|
||||
return value in ('README.md', 'AGENTS.md') or (value.startswith('docs/') and value.lower().endswith('.md'))
|
||||
|
||||
|
||||
def _repository_url(project):
|
||||
raw = project.git_repository_url
|
||||
try:
|
||||
parsed = urlsplit(raw)
|
||||
hostname = parsed.hostname
|
||||
port = parsed.port
|
||||
except ValueError as exc:
|
||||
raise DocumentationError('Repository URL is malformed.') from exc
|
||||
allowed = {item.strip().lower() for item in settings.PROJECT_DOCS_ALLOWED_GIT_HOSTS.split(',') if item.strip()}
|
||||
authority = hostname.lower() if hostname else ''
|
||||
if port:
|
||||
authority += f':{port}'
|
||||
if (parsed.scheme != 'https' or not authority or authority not in allowed or
|
||||
parsed.username or parsed.password or parsed.query or parsed.fragment or
|
||||
not parsed.path.startswith('/') or '..' in parsed.path.split('/') or
|
||||
'%' in raw or '\\' in raw or any(c.isspace() for c in raw)):
|
||||
raise DocumentationError('Repository URL is not an approved HTTPS Git endpoint.')
|
||||
return raw
|
||||
|
||||
|
||||
def _git_env(project):
|
||||
env = {key: value for key, value in os.environ.items()
|
||||
if not key.startswith('GIT_') and key not in ('SSH_ASKPASS', 'SSH_ASKPASS_REQUIRE')}
|
||||
env.update({'GIT_TERMINAL_PROMPT': '0', 'GIT_CONFIG_NOSYSTEM': '1',
|
||||
'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_SYSTEM': os.devnull,
|
||||
'GIT_NO_REPLACE_OBJECTS': '1'})
|
||||
try:
|
||||
secrets = json.loads(settings.PROJECT_DOCS_CREDENTIALS_JSON or '{}')
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise DocumentationError('Git credentials are not configured correctly.') from exc
|
||||
if not isinstance(secrets, dict):
|
||||
raise DocumentationError('Git credentials are not configured correctly.')
|
||||
entry = secrets.get(str(project.pk), {})
|
||||
if not isinstance(entry, dict):
|
||||
raise DocumentationError('Project Git credentials are not configured correctly.')
|
||||
if entry:
|
||||
username, token = entry.get('username'), entry.get('token')
|
||||
if entry.get('repository_url') != _repository_url(project):
|
||||
raise DocumentationError('Project Git credential is not bound to this repository URL.')
|
||||
if not isinstance(username, str) or not isinstance(token, str) or not username or not token or '\n' in username + token or '\r' in username + token:
|
||||
raise DocumentationError('Project Git credentials are incomplete.')
|
||||
encoded = base64.b64encode(f'{username}:{token}'.encode()).decode()
|
||||
env['GIT_CONFIG_COUNT'] = '1'
|
||||
env['GIT_CONFIG_KEY_0'] = f'http.{_repository_url(project)}.extraHeader'
|
||||
env['GIT_CONFIG_VALUE_0'] = f'Authorization: Basic {encoded}'
|
||||
return env
|
||||
|
||||
|
||||
def _git(args, *, env, timeout=30, max_output=20 * 1024 * 1024):
|
||||
limiter = shutil.which('prlimit')
|
||||
if not limiter:
|
||||
raise DocumentationError('Git resource limiter is unavailable on this server.')
|
||||
command = [limiter, f'--fsize={settings.PROJECT_DOCS_MAX_MIRROR_BYTES}', '--cpu=65', '--',
|
||||
'git', '-c', 'credential.helper=', '-c', 'core.hooksPath=/dev/null',
|
||||
'-c', 'protocol.file.allow=never', '-c', 'protocol.ext.allow=never',
|
||||
'-c', 'http.followRedirects=false', '-c', 'transfer.fsckObjects=true',
|
||||
'-c', 'gc.auto=0', *args]
|
||||
try:
|
||||
with tempfile.TemporaryFile() as output:
|
||||
result = subprocess.run(command, env=env, stdout=output,
|
||||
stderr=subprocess.DEVNULL, timeout=timeout, check=False)
|
||||
if result.returncode or output.tell() > max_output:
|
||||
raise DocumentationError('Git fetch or index failed; check repository access and size limits.')
|
||||
output.seek(0)
|
||||
return output.read(max_output + 1)
|
||||
except (OSError, subprocess.TimeoutExpired, subprocess.SubprocessError) as exc:
|
||||
raise DocumentationError('Git fetch or index timed out or could not start.') from exc
|
||||
|
||||
|
||||
def _mirror_path(project, *, create=True):
|
||||
root = Path(settings.PROJECT_DOCS_MIRROR_ROOT)
|
||||
if create:
|
||||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
path = root / str(project.pk)
|
||||
if path.is_symlink() or (path.exists() and not path.is_dir()):
|
||||
raise DocumentationError('Repository storage is unavailable.')
|
||||
return path
|
||||
|
||||
|
||||
def _mirror_size(path):
|
||||
return sum(item.stat().st_size for item in path.rglob('*') if item.is_file() and not item.is_symlink())
|
||||
|
||||
|
||||
def _fetch_snapshot(project, observed_head=None):
|
||||
url = _repository_url(project)
|
||||
env = _git_env(project)
|
||||
deadline = monotonic() + 90
|
||||
|
||||
def run_git(args, *, timeout=30, max_output=20 * 1024 * 1024):
|
||||
remaining = deadline - monotonic()
|
||||
if remaining <= 0:
|
||||
raise DocumentationError('Git documentation refresh exceeded its time limit.')
|
||||
return _git(args, env=env, timeout=min(timeout, remaining), max_output=max_output)
|
||||
|
||||
path = _mirror_path(project)
|
||||
if not path.exists():
|
||||
run_git(['init', '--bare', str(path)])
|
||||
path.chmod(0o700)
|
||||
if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES:
|
||||
raise DocumentationError('Repository mirror exceeds the configured size limit.')
|
||||
remote = run_git(['ls-remote', '--symref', url, 'HEAD'], max_output=4096).decode('ascii', 'replace')
|
||||
match = re.search(r'^ref: (refs/heads/\S+)\s+HEAD$', remote, re.M)
|
||||
if not match or not REF_RE.fullmatch(match.group(1)) or '..' in match.group(1) or '@{' in match.group(1):
|
||||
raise DocumentationError('Repository does not advertise a safe default branch.')
|
||||
branch = match.group(1)
|
||||
head_match = re.search(r'^([0-9a-f]{40}(?:[0-9a-f]{24})?)\s+HEAD$', remote, re.M)
|
||||
if not head_match:
|
||||
raise DocumentationError('Repository did not provide a valid HEAD commit.')
|
||||
if observed_head:
|
||||
observed_head(branch, head_match.group(1))
|
||||
run_git(['--git-dir', str(path), 'fetch', '--no-tags', '--depth=50', url,
|
||||
f'+{branch}:refs/remotes/origin/{branch.removeprefix("refs/heads/")}'], timeout=60, max_output=1024)
|
||||
if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES:
|
||||
raise DocumentationError('Repository mirror exceeds the configured size limit.')
|
||||
commit = run_git(['--git-dir', str(path), 'rev-parse', f'refs/remotes/origin/{branch.removeprefix("refs/heads/")}'],
|
||||
max_output=128).decode().strip()
|
||||
if not HASH_RE.fullmatch(commit):
|
||||
raise DocumentationError('Repository returned an invalid commit.')
|
||||
committed_at = run_git(['--git-dir', str(path), 'show', '-s', '--format=%cI', commit], max_output=128).decode().strip()
|
||||
try:
|
||||
commit_time = datetime.fromisoformat(committed_at)
|
||||
if commit_time.tzinfo is None:
|
||||
raise ValueError
|
||||
except ValueError as exc:
|
||||
raise DocumentationError('Repository returned an invalid commit time.') from exc
|
||||
listing = run_git(['--git-dir', str(path), 'ls-tree', '-r', '-z', commit],
|
||||
max_output=settings.PROJECT_DOCS_MAX_TREE_BYTES)
|
||||
rows = []
|
||||
audit_blob = None
|
||||
total = 0
|
||||
for raw in listing.split(b'\x00'):
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
metadata, name = raw.split(b'\t', 1)
|
||||
mode, kind, blob = metadata.decode('ascii').split(' ')
|
||||
name = name.decode('utf-8')
|
||||
except (ValueError, UnicodeDecodeError) as exc:
|
||||
if b'\tdocs/' in raw or raw.endswith((b'\tREADME.md', b'\tAGENTS.md')):
|
||||
raise DocumentationError('Repository tree contains an unsupported documentation path.') from exc
|
||||
continue
|
||||
if name == 'docs/.audit-state.json' and mode == '100644' and kind == 'blob':
|
||||
audit_blob = blob
|
||||
if not indexed_path(name):
|
||||
continue
|
||||
if mode != '100644' or kind != 'blob' or not HASH_RE.fullmatch(blob):
|
||||
continue
|
||||
rows.append((name, blob))
|
||||
if len(rows) > settings.PROJECT_DOCS_MAX_FILES:
|
||||
raise DocumentationError('Repository has too many documentation files.')
|
||||
documents = []
|
||||
for name, blob in rows:
|
||||
content = run_git(['--git-dir', str(path), 'cat-file', 'blob', blob],
|
||||
max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES)
|
||||
total += len(content)
|
||||
if total > settings.PROJECT_DOCS_MAX_CONTENT_BYTES:
|
||||
raise DocumentationError('Documentation exceeds the configured content limit.')
|
||||
try:
|
||||
markdown = content.decode('utf-8')
|
||||
except UnicodeDecodeError as exc:
|
||||
raise DocumentationError(f'Document {name} is not UTF-8.') from exc
|
||||
headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))]
|
||||
documents.append(ProjectDocument(project=project, path=name,
|
||||
title=(headings[0] if headings else posixpath.basename(name))[:255], headings=headings[:100],
|
||||
markdown=markdown, content_hash=hashlib.sha256(content).hexdigest(), source_commit=commit))
|
||||
audit = {}
|
||||
if audit_blob and HASH_RE.fullmatch(audit_blob):
|
||||
raw = run_git(['--git-dir', str(path), 'cat-file', 'blob', audit_blob], max_output=65536)
|
||||
try:
|
||||
candidate = json.loads(raw)
|
||||
if isinstance(candidate, dict) and candidate.get('schema_version') == 1:
|
||||
limits = {'last_successful_audit': 40, 'last_audited_commit': 64,
|
||||
'last_report_path': 300, 'last_attempt': 40, 'last_status': 40}
|
||||
audit = {key: candidate[key] for key, size in limits.items()
|
||||
if isinstance(candidate.get(key), str) and len(candidate[key]) <= size}
|
||||
if isinstance(candidate.get('audit_interval_days'), int):
|
||||
audit['audit_interval_days'] = candidate['audit_interval_days']
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
pass
|
||||
return url, branch, commit, commit_time, documents, audit
|
||||
|
||||
|
||||
def sync_project(user, project, *, force=False):
|
||||
services.project_for(user, project.pk, write=force)
|
||||
if not project.git_repository_url:
|
||||
raise DocumentationError('Configure a Git repository URL before refreshing documentation.')
|
||||
state, _ = DocumentationSync.objects.get_or_create(project=project)
|
||||
now = timezone.now()
|
||||
interval = timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS))
|
||||
if not force and state.last_attempt_at and now - state.last_attempt_at < interval:
|
||||
return state
|
||||
if force and state.last_attempt_at and now - state.last_attempt_at < timedelta(seconds=30):
|
||||
return state
|
||||
token = str(uuid.uuid4())
|
||||
claimed = DocumentationSync.objects.filter(pk=state.pk).filter(
|
||||
Q(sync_token='') | Q(sync_started_at__lt=now - timedelta(minutes=3))).update(
|
||||
sync_token=token, sync_started_at=now, last_attempt_at=now)
|
||||
if not claimed:
|
||||
return DocumentationSync.objects.get(pk=state.pk)
|
||||
try:
|
||||
def record_head(branch, head):
|
||||
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(branch=branch, remote_head=head)
|
||||
|
||||
url, branch, commit, commit_time, documents, audit = _fetch_snapshot(project, observed_head=record_head)
|
||||
with transaction.atomic():
|
||||
state = DocumentationSync.objects.select_for_update().get(pk=state.pk)
|
||||
if state.sync_token != token:
|
||||
return state
|
||||
if state.indexed_commit != commit or state.indexed_repository_url != url:
|
||||
ProjectDocument.objects.filter(project=project).delete()
|
||||
ProjectDocument.objects.bulk_create(documents, batch_size=100)
|
||||
state.indexed_commit = commit
|
||||
state.remote_head = commit
|
||||
state.indexed_repository_url = url
|
||||
state.branch = branch
|
||||
state.commit_time = commit_time
|
||||
state.last_success_at = timezone.now()
|
||||
state.last_error = ''
|
||||
state.sync_token = ''
|
||||
state.sync_started_at = None
|
||||
state.audit_state = audit
|
||||
state.save()
|
||||
except DocumentationError as exc:
|
||||
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(
|
||||
last_error=str(exc)[:300], sync_token='', sync_started_at=None)
|
||||
except Exception as exc:
|
||||
logger.error('Documentation indexing failed for project %s (%s).', project.pk, type(exc).__name__)
|
||||
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(
|
||||
last_error='Documentation indexing failed; retry after checking server logs.',
|
||||
sync_token='', sync_started_at=None)
|
||||
return DocumentationSync.objects.get(pk=state.pk)
|
||||
|
||||
|
||||
def maybe_sync(user, project):
|
||||
"""On-demand polling: one bounded refresh per interval for authorized readers."""
|
||||
services.project_for(user, project.pk)
|
||||
if project.git_repository_url:
|
||||
return sync_project(user, project)
|
||||
return DocumentationSync.objects.filter(project=project).first()
|
||||
|
||||
|
||||
def sync_status(project, state):
|
||||
if not project.git_repository_url:
|
||||
return 'No repository configured'
|
||||
if not state or not state.indexed_commit:
|
||||
if state and state.sync_token:
|
||||
return 'Refreshing from Git; no indexed documentation yet'
|
||||
return 'No indexed documentation' if not state or not state.last_error else f'Sync error: {state.last_error}'
|
||||
if state.sync_token:
|
||||
return 'Refreshing from Git; showing the last indexed version'
|
||||
if state.last_error:
|
||||
return f'Stale after sync error: {state.last_error}'
|
||||
if state.indexed_repository_url != project.git_repository_url:
|
||||
return 'Stale: repository URL changed'
|
||||
if state.remote_head and state.remote_head != state.indexed_commit:
|
||||
return 'Stale: remote HEAD differs from indexed commit'
|
||||
if state.last_success_at and timezone.now() - state.last_success_at >= timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS)):
|
||||
return 'Refresh due; indexed commit may lag remote'
|
||||
return 'Indexed from Git; document accuracy is not guaranteed'
|
||||
|
||||
|
||||
def audit_status(state):
|
||||
audit = state.audit_state if state else {}
|
||||
date_value = audit.get('last_successful_audit')
|
||||
commit = audit.get('last_audited_commit')
|
||||
partial = audit.get('last_status') in ('partial', 'error')
|
||||
try:
|
||||
last = datetime.fromisoformat(date_value.replace('Z', '+00:00'))
|
||||
days = int(audit.get('audit_interval_days', 14))
|
||||
if last.tzinfo is None or not HASH_RE.fullmatch(commit) or not 1 <= days <= 365:
|
||||
raise ValueError
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return 'Partial or failed / due' if partial else 'Unknown / due'
|
||||
if last > timezone.now() + timedelta(minutes=5):
|
||||
return 'Unknown / due (audit timestamp is in the future)'
|
||||
if timezone.now() >= last + timedelta(days=days):
|
||||
return f'Overdue (last successful audit {last.date()})' + ('; latest attempt partial or failed' if partial else '')
|
||||
if state.indexed_commit != commit:
|
||||
return f'Last audited {last.date()}; indexed commit differs' + ('; latest attempt partial or failed' if partial else '')
|
||||
return f'Last audited {last.date()} at this commit' + ('; latest attempt partial or failed' if partial else '')
|
||||
|
||||
|
||||
def list_documents(user, project, *, path='', limit=50, offset=0):
|
||||
services.project_for(user, project.pk)
|
||||
if path:
|
||||
normal_path(path)
|
||||
size = min(max(int(limit), 1), MAX_RESULTS)
|
||||
offset = max(int(offset), 0)
|
||||
qs = ProjectDocument.objects.filter(project=project)
|
||||
if path:
|
||||
qs = qs.filter(path__startswith=path.rstrip('/') + '/')
|
||||
qs = qs.order_by('path')
|
||||
return list(qs[offset:offset + size]), qs.count()
|
||||
|
||||
|
||||
def search_documents(user, query, *, project=None, limit=30, offset=0):
|
||||
query = query.strip() if isinstance(query, str) else ''
|
||||
if not query or len(query) > 200:
|
||||
raise ValidationError('Search query must be 1–200 characters.')
|
||||
qs = ProjectDocument.objects.select_related('project').filter(project__in=services.accessible(user))
|
||||
if project:
|
||||
services.project_for(user, project.pk)
|
||||
qs = qs.filter(project=project)
|
||||
qs = qs.filter(Q(path__icontains=query) | Q(title__icontains=query) | Q(markdown__icontains=query)).order_by('project_id', 'path')
|
||||
size = min(max(int(limit), 1), MAX_RESULTS)
|
||||
offset = max(int(offset), 0)
|
||||
return list(qs[offset:offset + size]), qs.count()
|
||||
|
||||
|
||||
def read_document(user, project, path):
|
||||
services.project_for(user, project.pk)
|
||||
return ProjectDocument.objects.get(project=project, path=normal_path(path))
|
||||
|
||||
|
||||
def excerpt(document, query, radius=120):
|
||||
content = document.markdown
|
||||
at = content.lower().find(query.lower())
|
||||
if at < 0:
|
||||
at = 0
|
||||
start = max(0, at - radius)
|
||||
end = min(len(content), at + len(query) + radius)
|
||||
return ('…' if start else '') + content[start:end].replace('\n', ' ') + ('…' if end < len(content) else '')
|
||||
|
||||
|
||||
def line_window(document, start_line=1, end_line=None):
|
||||
lines = document.markdown.splitlines(keepends=True)
|
||||
start = max(int(start_line), 1)
|
||||
if start > len(lines) + 1:
|
||||
raise ValidationError('Start line is beyond the document.')
|
||||
requested_end = int(end_line) if end_line is not None else start + MAX_READ_LINES - 1
|
||||
if requested_end < start:
|
||||
raise ValidationError('Invalid line range.')
|
||||
end = min(requested_end, start + MAX_READ_LINES - 1, len(lines))
|
||||
output = ''
|
||||
partial_line = False
|
||||
actual_end = start - 1
|
||||
for number in range(start, end + 1):
|
||||
line = lines[number - 1]
|
||||
remaining = MAX_READ_CHARS - len(output)
|
||||
if remaining <= 0:
|
||||
break
|
||||
if len(line) > remaining:
|
||||
output += line[:remaining]
|
||||
partial_line = True
|
||||
actual_end = number
|
||||
break
|
||||
output += line
|
||||
actual_end = number
|
||||
return {'markdown': output, 'start_line': start, 'end_line': actual_end,
|
||||
'total_lines': len(lines), 'truncated': actual_end < len(lines) or partial_line,
|
||||
'partial_line': partial_line}
|
||||
|
||||
|
||||
def document_history(user, project, path, limit=20):
|
||||
services.project_for(user, project.pk)
|
||||
normal_path(path)
|
||||
state = DocumentationSync.objects.filter(project=project).first()
|
||||
if not state or not state.indexed_commit:
|
||||
return []
|
||||
mirror = _mirror_path(project, create=False)
|
||||
if not mirror.exists():
|
||||
return []
|
||||
output = _git(['--git-dir', str(mirror), 'log', f'--max-count={min(max(int(limit), 1), 50)}',
|
||||
'--format=%H%x09%cI', state.indexed_commit, '--', path],
|
||||
env=_git_env(project), max_output=8192)
|
||||
rows = []
|
||||
for line in output.decode('ascii', 'replace').splitlines():
|
||||
try:
|
||||
commit, committed_at = line.split('\t', 1)
|
||||
except ValueError:
|
||||
continue
|
||||
if HASH_RE.fullmatch(commit):
|
||||
rows.append({'commit': commit, 'committed_at': committed_at})
|
||||
return rows
|
||||
|
||||
|
||||
def historical_document(user, project, path, revision):
|
||||
current = read_document(user, project, path)
|
||||
if revision == current.source_commit:
|
||||
return current
|
||||
if not isinstance(revision, str) or not HASH_RE.fullmatch(revision):
|
||||
raise ValidationError('Invalid Git revision.')
|
||||
if revision not in {row['commit'] for row in document_history(user, project, path, limit=50)}:
|
||||
raise ValidationError('Revision is unavailable in the retained Git history.')
|
||||
mirror = _mirror_path(project, create=False)
|
||||
content = _git(['--git-dir', str(mirror), 'show', f'{revision}:{path}'],
|
||||
env=_git_env(project), max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES)
|
||||
try:
|
||||
markdown = content.decode('utf-8')
|
||||
except UnicodeDecodeError as exc:
|
||||
raise DocumentationError('Historical document is not UTF-8.') from exc
|
||||
headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))]
|
||||
return ProjectDocument(project=project, path=path, title=(headings[0] if headings else posixpath.basename(path))[:255],
|
||||
headings=headings[:100], markdown=markdown,
|
||||
content_hash=hashlib.sha256(content).hexdigest(), source_commit=revision)
|
||||
|
||||
|
||||
def document_diff(user, project, path, before, after):
|
||||
first = historical_document(user, project, path, before)
|
||||
second = historical_document(user, project, path, after)
|
||||
diff = ''.join(difflib.unified_diff(first.markdown.splitlines(keepends=True), second.markdown.splitlines(keepends=True),
|
||||
fromfile=before, tofile=after))
|
||||
return diff[:MAX_READ_CHARS], len(diff) > MAX_READ_CHARS
|
||||
@@ -0,0 +1,50 @@
|
||||
# Generated by Django 6.1.2 on 2026-10-20 12:20
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('projects', '0005_cardattachment'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='DocumentationSync',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('branch', models.CharField(blank=True, max_length=255)),
|
||||
('indexed_repository_url', models.URLField(blank=True)),
|
||||
('indexed_commit', models.CharField(blank=True, max_length=64)),
|
||||
('remote_head', models.CharField(blank=True, max_length=64)),
|
||||
('commit_time', models.DateTimeField(blank=True, null=True)),
|
||||
('last_success_at', models.DateTimeField(blank=True, null=True)),
|
||||
('last_attempt_at', models.DateTimeField(blank=True, null=True)),
|
||||
('last_error', models.CharField(blank=True, max_length=300)),
|
||||
('sync_token', models.CharField(blank=True, max_length=36)),
|
||||
('sync_started_at', models.DateTimeField(blank=True, null=True)),
|
||||
('audit_state', models.JSONField(blank=True, default=dict)),
|
||||
('project', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='documentation_sync', to='projects.project')),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='ProjectDocument',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('path', models.CharField(max_length=500)),
|
||||
('title', models.CharField(max_length=255)),
|
||||
('headings', models.JSONField(default=list)),
|
||||
('markdown', models.TextField()),
|
||||
('content_hash', models.CharField(max_length=64)),
|
||||
('source_commit', models.CharField(max_length=64)),
|
||||
('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='documents', to='projects.project')),
|
||||
],
|
||||
options={
|
||||
'ordering': ['path'],
|
||||
'indexes': [models.Index(fields=['project', 'path'], name='projects_pr_project_424af7_idx')],
|
||||
'constraints': [models.UniqueConstraint(fields=('project', 'path'), name='unique_project_document_path')],
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -1,4 +1,5 @@
|
||||
import uuid
|
||||
from urllib.parse import urlsplit
|
||||
from django.conf import settings
|
||||
from django.core.validators import RegexValidator
|
||||
from django.core.exceptions import ValidationError
|
||||
@@ -39,6 +40,49 @@ class Project(models.Model):
|
||||
class Meta:
|
||||
constraints = [models.UniqueConstraint(fields=['owner', 'slug'], name='unique_owner_project_slug'), models.UniqueConstraint(fields=['owner', 'name'], name='unique_owner_project_name')]
|
||||
|
||||
def clean(self):
|
||||
if self.git_repository_url:
|
||||
try:
|
||||
parsed = urlsplit(self.git_repository_url)
|
||||
_ = parsed.port
|
||||
valid = parsed.scheme == 'https' and bool(parsed.hostname)
|
||||
except ValueError:
|
||||
valid = False
|
||||
parsed = None
|
||||
if (not valid or parsed.username or parsed.password or parsed.query or parsed.fragment or
|
||||
'\\' in self.git_repository_url or any(c.isspace() for c in self.git_repository_url)):
|
||||
raise ValidationError({'git_repository_url': 'Use an HTTPS Git URL without credentials or query parameters.'})
|
||||
|
||||
|
||||
class DocumentationSync(models.Model):
|
||||
project = models.OneToOneField(Project, on_delete=models.CASCADE, related_name='documentation_sync')
|
||||
branch = models.CharField(max_length=255, blank=True)
|
||||
indexed_repository_url = models.URLField(blank=True)
|
||||
indexed_commit = models.CharField(max_length=64, blank=True)
|
||||
remote_head = models.CharField(max_length=64, blank=True)
|
||||
commit_time = models.DateTimeField(null=True, blank=True)
|
||||
last_success_at = models.DateTimeField(null=True, blank=True)
|
||||
last_attempt_at = models.DateTimeField(null=True, blank=True)
|
||||
last_error = models.CharField(max_length=300, blank=True)
|
||||
sync_token = models.CharField(max_length=36, blank=True)
|
||||
sync_started_at = models.DateTimeField(null=True, blank=True)
|
||||
audit_state = models.JSONField(default=dict, blank=True)
|
||||
|
||||
|
||||
class ProjectDocument(models.Model):
|
||||
project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='documents')
|
||||
path = models.CharField(max_length=500)
|
||||
title = models.CharField(max_length=255)
|
||||
headings = models.JSONField(default=list)
|
||||
markdown = models.TextField()
|
||||
content_hash = models.CharField(max_length=64)
|
||||
source_commit = models.CharField(max_length=64)
|
||||
|
||||
class Meta:
|
||||
ordering = ['path']
|
||||
constraints = [models.UniqueConstraint(fields=['project', 'path'], name='unique_project_document_path')]
|
||||
indexes = [models.Index(fields=['project', 'path'])]
|
||||
|
||||
|
||||
class Membership(models.Model):
|
||||
ROLE = [('editor', 'Editor'), ('viewer', 'Viewer')]
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{% if nodes %}<ul class="list-unstyled ms-3">{% for node in nodes %}<li><a href="{% url 'projects:docs_home' project.id %}?path={{ node.path|urlencode }}"{% if node.path == prefix %} aria-current="page"{% endif %}>{{ node.name }}/</a>{% include 'projects/_docs_folder_tree.html' with nodes=node.children %}</li>{% endfor %}</ul>{% endif %}
|
||||
@@ -2,7 +2,7 @@
|
||||
{% block content %}
|
||||
<style>.htmx-indicator{display:none}.htmx-request.htmx-indicator,.htmx-request .htmx-indicator{display:inline}</style>
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<div><a href="{% url 'projects:list' %}">All projects</a><h1>{{ project.name }}</h1><p>{{ project.description }}</p>{% if project.git_repository_url %}<a href="{{ project.git_repository_url }}" rel="noopener noreferrer">Repository</a>{% endif %}<p>{{ done_count }} of {{ total_count }} cards done</p></div>
|
||||
<div><a href="{% url 'projects:list' %}">All projects</a><h1>{{ project.name }}</h1><p>{{ project.description }}</p>{% if project.git_repository_url %}<a href="{{ project.git_repository_url }}" rel="noopener noreferrer">Repository</a>{% endif %} <a href="{% url 'projects:docs_home' project.id %}">Documentation</a><p>{{ done_count }} of {{ total_count }} cards done</p></div>
|
||||
{% if can_edit %}<div><a class="btn btn-outline-secondary" href="{% url 'projects:edit' project.id %}">Edit project</a> {% if project.archived %}<form class="d-inline" method="post" action="{% url 'projects:restore' project.id %}">{% csrf_token %}<button class="btn btn-outline-success">Restore project</button></form>{% else %}<form class="d-inline" method="post" action="{% url 'projects:archive' project.id %}">{% csrf_token %}<button class="btn btn-outline-danger">Archive project</button></form> <a class="btn btn-success" href="{% url 'projects:card_create' project.id %}">New card</a>{% endif %}</div>{% endif %}
|
||||
</div>
|
||||
<div class="d-flex gap-3 flex-wrap mb-3">{% for label,count in priority_counts %}<span class="badge text-bg-light border">{{ label }}: {{ count }} open</span>{% endfor %}</div>
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{% extends 'core/base.html' %}
|
||||
{% block content %}
|
||||
<nav aria-label="Breadcrumb" class="mb-3"><a href="{% url 'projects:board' project.id %}">{{ project.name }}</a> / <a href="{% url 'projects:docs_home' project.id %}">Documentation</a> / {% for crumb in breadcrumbs %}<a href="{% url 'projects:docs_home' project.id %}?path={{ crumb.path|urlencode }}">{{ crumb.name }}</a> / {% endfor %}{{ file_name }}</nav>
|
||||
<div class="d-flex justify-content-between align-items-start gap-3 flex-wrap"><div><h1>{{ document.title }}</h1><p><code>{{ document.path }}</code></p></div>{% if source_url %}<a class="btn btn-outline-secondary" href="{{ source_url }}" rel="noopener noreferrer">View in Git</a>{% endif %}</div>
|
||||
<div class="alert {% if sync.last_error %}alert-warning{% else %}alert-secondary{% endif %}" role="status"><strong>{{ sync_status }}</strong><span class="d-block">Source commit: <code>{{ document.source_commit }}</code> · SHA-256: <code>{{ document.content_hash }}</code></span>{% if document_commit_time %}<span class="d-block">Commit time: {{ document_commit_time }}</span>{% endif %}{% if sync.remote_head and sync.remote_head != sync.indexed_commit %}<span class="d-block">Last observed remote HEAD: <code>{{ sync.remote_head }}</code></span>{% endif %}{% if sync.last_success_at %}<span class="d-block">Fetched {{ sync.last_success_at }}</span>{% endif %}<span class="d-block">Documentation accuracy audit: {{ audit_label }}.</span></div>
|
||||
{% if is_historical %}<p class="alert alert-info" role="status">Viewing a historical Git version. <a href="{% url 'projects:docs_document' project.id document.path %}">Return to indexed version</a>.</p>{% endif %}
|
||||
<article class="card card-body markdown-body">{{ rendered }}</article>
|
||||
<details class="mt-3"><summary>Raw Markdown</summary><pre class="card card-body overflow-auto">{{ document.markdown }}</pre></details>
|
||||
{% if history %}<section class="mt-4" aria-labelledby="doc-history-heading"><h2 class="h5" id="doc-history-heading">Git history</h2><p class="text-muted">Recent commits for this path in the retained shallow mirror. Renames and deleted documents are not followed yet.</p><ol>{% for version in history %}<li><a href="?revision={{ version.commit }}"><code>{{ version.commit|slice:':12' }}</code></a> · {{ version.committed_at }}</li>{% endfor %}</ol></section>{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,28 @@
|
||||
{% extends 'core/base.html' %}
|
||||
{% block content %}
|
||||
<div class="d-flex justify-content-between align-items-start gap-3 flex-wrap mb-3">
|
||||
<div><a href="{% url 'projects:board' project.id %}">← {{ project.name }}</a><h1 class="mt-2">Documentation</h1><p class="text-muted">Read-only Markdown from the project's Git repository.</p></div>
|
||||
{% if can_edit and project.git_repository_url %}<form method="post" action="{% url 'projects:docs_refresh' project.id %}">{% csrf_token %}<button class="btn btn-outline-primary">Refresh from Git</button></form>{% endif %}
|
||||
</div>
|
||||
<div class="alert {% if sync.last_error %}alert-warning{% else %}alert-secondary{% endif %}" role="status">
|
||||
<strong>{{ sync_status }}</strong>
|
||||
{% if sync.indexed_commit %}<span class="d-block">Indexed commit: <code>{{ sync.indexed_commit }}</code>{% if sync.branch %} · {{ sync.branch }}{% endif %}{% if sync.commit_time %} · committed {{ sync.commit_time }}{% endif %}{% if sync.last_success_at %} · fetched {{ sync.last_success_at }}{% endif %}</span>{% endif %}
|
||||
{% if sync.remote_head and sync.remote_head != sync.indexed_commit %}<span class="d-block">Last observed remote HEAD: <code>{{ sync.remote_head }}</code></span>{% endif %}
|
||||
{% if sync.last_attempt_at %}<span class="d-block">Last fetch attempt: {{ sync.last_attempt_at }}</span>{% endif %}
|
||||
<span class="d-block">Documentation accuracy audit: {{ audit_label }}.</span>
|
||||
</div>
|
||||
{% if not project.git_repository_url %}<p>No repository is configured. {% if can_edit %}Add its HTTPS Git URL in <a href="{% url 'projects:edit' project.id %}">project settings</a>.{% else %}Ask a project editor to configure its Git URL.{% endif %}</p>{% endif %}
|
||||
<form method="get" action="{% url 'projects:docs_home' project.id %}" role="search" class="mb-4">
|
||||
<label for="docs-search" class="form-label">Search documentation</label>
|
||||
<div class="input-group"><input id="docs-search" class="form-control" name="q" value="{{ query }}" maxlength="200" placeholder="Path, heading or text"><button class="btn btn-outline-secondary">Search</button></div>
|
||||
</form>
|
||||
<div class="row g-4">
|
||||
<nav class="col-md-4 col-lg-3" aria-label="Documentation folders"><h2 class="h5">Folders</h2><ul class="list-unstyled"><li><a href="{% url 'projects:docs_home' project.id %}">All documents</a></li></ul>{% include 'projects/_docs_folder_tree.html' with nodes=folder_tree %}</nav>
|
||||
<section class="col-md-8 col-lg-9" aria-label="Document results">
|
||||
<h2 class="h5">{% if query %}Search results{% elif prefix %}{{ prefix }}/{% else %}Documents{% endif %} <small class="text-muted">({{ total }})</small></h2>
|
||||
{% if documents %}<ul class="list-group">{% if query %}{% for result in results %}<li class="list-group-item"><a href="{% url 'projects:docs_document' project.id result.document.path %}"><strong>{{ result.document.title }}</strong></a><br><small class="text-muted">{{ result.document.path }}</small><p class="mb-0 mt-1">{{ result.snippet }}</p></li>{% endfor %}{% else %}{% for document in documents %}<li class="list-group-item"><a href="{% url 'projects:docs_document' project.id document.path %}"><strong>{{ document.title }}</strong></a><br><small class="text-muted">{{ document.path }}</small></li>{% endfor %}{% endif %}</ul>
|
||||
{% else %}<p>{% if query %}No matching documents.{% elif sync.last_error %}No indexed documents are available. The latest Git fetch failed.{% elif project.git_repository_url %}No Markdown documentation has been indexed yet.{% else %}Add a repository to begin.{% endif %}</p>{% endif %}
|
||||
{% if has_previous or has_next %}<nav class="mt-3" aria-label="Document pages">{% if has_previous %}<a class="btn btn-outline-secondary btn-sm" href="?q={{ query|urlencode }}&path={{ prefix|urlencode }}&page={{ page|add:'-1' }}">Previous</a>{% endif %} <span>Page {{ page }}</span> {% if has_next %}<a class="btn btn-outline-secondary btn-sm" href="?q={{ query|urlencode }}&path={{ prefix|urlencode }}&page={{ page|add:'1' }}">Next</a>{% endif %}</nav>{% endif %}
|
||||
</section>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -1,12 +1,62 @@
|
||||
import markdown
|
||||
import nh3
|
||||
import posixpath
|
||||
from urllib.parse import unquote, urlsplit
|
||||
from django import template
|
||||
from django.urls import reverse
|
||||
from django.utils.safestring import mark_safe
|
||||
from markdown.extensions import Extension
|
||||
from markdown.treeprocessors import Treeprocessor
|
||||
|
||||
register = template.Library()
|
||||
|
||||
TAGS = {'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'li', 'pre', 'code', 'blockquote', 'strong', 'em', 'a', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'hr', 'br', 'input'}
|
||||
ATTRIBUTES = {'a': {'href', 'title'}, 'input': {'type', 'checked', 'disabled'}}
|
||||
|
||||
|
||||
def _clean(value, extensions, attributes=ATTRIBUTES):
|
||||
html = markdown.markdown(value or '', extensions=extensions)
|
||||
return mark_safe(nh3.clean(html, tags=TAGS, attributes=attributes, url_schemes={'https', 'http', 'mailto'}))
|
||||
|
||||
|
||||
@register.filter
|
||||
def safe_markdown(value):
|
||||
html = markdown.markdown(value or '', extensions=['extra', 'fenced_code', 'tables', 'sane_lists', 'pymdownx.tasklist'])
|
||||
return mark_safe(nh3.clean(html, tags={'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'li', 'pre', 'code', 'blockquote', 'strong', 'em', 'a', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'hr', 'br', 'input'}, attributes={'a': {'href', 'title'}, 'input': {'type', 'checked', 'disabled'}}, url_schemes={'https', 'http', 'mailto'}))
|
||||
return _clean(value, ['extra', 'fenced_code', 'tables', 'sane_lists', 'pymdownx.tasklist'])
|
||||
|
||||
|
||||
class _DocumentLinks(Treeprocessor):
|
||||
def __init__(self, md, project_id, current_path, paths):
|
||||
super().__init__(md)
|
||||
self.project_id = project_id
|
||||
self.current_path = current_path
|
||||
self.paths = paths
|
||||
|
||||
def run(self, root):
|
||||
for element in root.iter('a'):
|
||||
href = element.get('href', '')
|
||||
parts = urlsplit(href)
|
||||
if parts.scheme or parts.netloc or not parts.path:
|
||||
continue
|
||||
target = posixpath.normpath(posixpath.join(posixpath.dirname(self.current_path), unquote(parts.path)))
|
||||
if target.endswith('/'):
|
||||
target += 'README.md'
|
||||
if target in self.paths:
|
||||
url = reverse('projects:docs_document', kwargs={'project_id': self.project_id, 'path': target})
|
||||
element.set('href', url + (f'#{parts.fragment}' if parts.fragment else ''))
|
||||
else:
|
||||
element.attrib.pop('href', None)
|
||||
|
||||
|
||||
class _DocumentLinkExtension(Extension):
|
||||
def __init__(self, project_id, current_path, paths):
|
||||
super().__init__()
|
||||
self.project_id, self.current_path, self.paths = project_id, current_path, paths
|
||||
|
||||
def extendMarkdown(self, md):
|
||||
md.treeprocessors.register(_DocumentLinks(md, self.project_id, self.current_path, self.paths), 'project_document_links', 12)
|
||||
|
||||
|
||||
def render_document_markdown(value, project_id, current_path, paths):
|
||||
doc_attributes = {**ATTRIBUTES, **{tag: {'id'} for tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6')}}
|
||||
return _clean(value, ['extra', 'fenced_code', 'tables', 'sane_lists', 'pymdownx.tasklist', 'toc',
|
||||
_DocumentLinkExtension(project_id, current_path, paths)], attributes=doc_attributes)
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from datetime import timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from asgiref.sync import async_to_sync
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.test import Client, TestCase, override_settings
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
from mcp.server.auth.provider import AccessToken
|
||||
from mcp.server.mcpserver.exceptions import ToolError
|
||||
|
||||
from mcp_server.server import create_server
|
||||
from .documentation import (_fetch_snapshot, _git_env, _repository_url, audit_status,
|
||||
document_diff, document_history, historical_document,
|
||||
normal_path, sync_project)
|
||||
from .models import DocumentationSync, Membership, ProjectDocument
|
||||
from .services import create_project
|
||||
from .templatetags.project_markdown import render_document_markdown
|
||||
|
||||
|
||||
class DocumentationTests(TestCase):
|
||||
def setUp(self):
|
||||
users = get_user_model()
|
||||
self.owner = users.objects.create_user('docs-owner', password='password')
|
||||
self.viewer = users.objects.create_user('docs-viewer', password='password')
|
||||
self.outsider = users.objects.create_user('docs-outsider', password='password')
|
||||
self.project = create_project(self.owner, name='Documentation test',
|
||||
git_repository_url='https://git.jorendegraaf.eu/owner/project.git')
|
||||
Membership.objects.create(project=self.project, user=self.viewer, role='viewer')
|
||||
|
||||
def snapshot(self, path='docs/guide.md', markdown='# Guide\n\n[Next](next.md)\n', commit='a' * 40):
|
||||
item = ProjectDocument(project=self.project, path=path, title='Guide', headings=['Guide'],
|
||||
markdown=markdown, content_hash=hashlib.sha256(markdown.encode()).hexdigest(), source_commit=commit)
|
||||
return (self.project.git_repository_url, 'refs/heads/main', commit, timezone.now(), [item], {})
|
||||
|
||||
def import_snapshot(self):
|
||||
with patch('projects.documentation._fetch_snapshot', return_value=self.snapshot()):
|
||||
return sync_project(self.owner, self.project)
|
||||
|
||||
def test_sync_replaces_deleted_documents_and_retains_last_good_on_error(self):
|
||||
state = self.import_snapshot()
|
||||
self.assertEqual(ProjectDocument.objects.filter(project=self.project).count(), 1)
|
||||
self.assertEqual(state.indexed_commit, 'a' * 40)
|
||||
DocumentationSync.objects.filter(project=self.project).update(last_attempt_at=timezone.now() - timedelta(minutes=2))
|
||||
with patch('projects.documentation._fetch_snapshot', return_value=self.snapshot('README.md', '# New\n', 'b' * 40)):
|
||||
sync_project(self.owner, self.project, force=True)
|
||||
self.assertFalse(ProjectDocument.objects.filter(project=self.project, path='docs/guide.md').exists())
|
||||
self.assertTrue(ProjectDocument.objects.filter(project=self.project, path='README.md').exists())
|
||||
DocumentationSync.objects.filter(project=self.project).update(last_attempt_at=timezone.now() - timedelta(minutes=2))
|
||||
from .documentation import DocumentationError
|
||||
with patch('projects.documentation._fetch_snapshot', side_effect=DocumentationError('fetch unavailable')):
|
||||
state = sync_project(self.owner, self.project, force=True)
|
||||
self.assertEqual(state.indexed_commit, 'b' * 40)
|
||||
self.assertIn('fetch unavailable', state.last_error)
|
||||
self.assertTrue(ProjectDocument.objects.filter(project=self.project, path='README.md').exists())
|
||||
|
||||
def test_permissions_views_and_refresh(self):
|
||||
self.import_snapshot()
|
||||
home = reverse('projects:docs_home', args=[self.project.pk])
|
||||
detail = reverse('projects:docs_document', args=[self.project.pk, 'docs/guide.md'])
|
||||
refresh = reverse('projects:docs_refresh', args=[self.project.pk])
|
||||
self.assertEqual(self.client.get(home).status_code, 302)
|
||||
self.client.force_login(self.outsider)
|
||||
self.assertEqual(self.client.get(home).status_code, 404)
|
||||
self.assertEqual(self.client.get(detail).status_code, 404)
|
||||
self.client.force_login(self.viewer)
|
||||
self.assertContains(self.client.get(home), 'docs/guide.md')
|
||||
self.assertContains(self.client.get(detail), 'Guide')
|
||||
self.assertEqual(self.client.post(refresh).status_code, 403)
|
||||
self.client.force_login(self.owner)
|
||||
self.assertEqual(self.client.get(reverse('projects:docs_document', args=[self.project.pk, 'docs/../secret.md'])).status_code, 404)
|
||||
csrf_client = Client(enforce_csrf_checks=True, HTTP_HOST='localhost')
|
||||
csrf_client.force_login(self.owner)
|
||||
self.assertEqual(csrf_client.post(refresh).status_code, 403)
|
||||
csrf_client.get(home)
|
||||
self.assertEqual(csrf_client.post(refresh, HTTP_X_CSRFTOKEN=csrf_client.cookies['csrftoken'].value).status_code, 302)
|
||||
|
||||
def test_markdown_links_and_untrusted_html(self):
|
||||
html = str(render_document_markdown('# Guide\n\n[Next](next.md) [Missing](absent.md)\n\n<script>bad()</script>',
|
||||
self.project.pk, 'docs/guide.md', {'docs/guide.md', 'docs/next.md'}))
|
||||
self.assertIn(f'/projects/{self.project.pk}/docs/docs/next.md/', html)
|
||||
self.assertIn('<h1 id="guide">Guide</h1>', html)
|
||||
self.assertNotIn('href="absent.md"', html)
|
||||
self.assertNotIn('<script>', html)
|
||||
for path in ('../secret.md', '/etc/passwd', 'docs//file.md', 'docs\\file.md'):
|
||||
with self.assertRaises(ValidationError):
|
||||
normal_path(path)
|
||||
|
||||
def test_mcp_list_search_read_and_isolation(self):
|
||||
self.import_snapshot()
|
||||
ProjectDocument.objects.create(project=self.project, path='README.md', title='Read me',
|
||||
markdown='# Read me\n', headings=['Read me'], content_hash=hashlib.sha256(b'# Read me\n').hexdigest(),
|
||||
source_commit='a' * 40)
|
||||
server = create_server()
|
||||
token = AccessToken(token='test', client_id='test', scopes=['projects:read'], subject=str(self.viewer.pk))
|
||||
with patch('mcp_server.server.get_access_token', return_value=token):
|
||||
listed = async_to_sync(server.call_tool)('docs_list', {'project': str(self.project.pk), 'limit': 1}).structured_content
|
||||
self.assertEqual(listed['items'][0]['path'], 'README.md')
|
||||
following = async_to_sync(server.call_tool)('docs_list', {'project': str(self.project.pk), 'limit': 1, 'cursor': listed['next_cursor']}).structured_content
|
||||
self.assertEqual(following['items'][0]['path'], 'docs/guide.md')
|
||||
found = async_to_sync(server.call_tool)('docs_search', {'query': 'Guide'}).structured_content
|
||||
self.assertEqual(found['total'], 1)
|
||||
self.assertNotIn('markdown', found['items'][0])
|
||||
read = async_to_sync(server.call_tool)('docs_read', {'project': str(self.project.pk), 'path': 'docs/guide.md', 'start_line': 1, 'end_line': 1}).structured_content
|
||||
self.assertEqual(read['markdown'], '# Guide\n')
|
||||
self.assertTrue(read['truncated'])
|
||||
with self.assertRaises(ToolError):
|
||||
async_to_sync(server.call_tool)('docs_list', {'project': str(self.project.pk), 'cursor': 'stale:1'})
|
||||
with self.assertRaises(ToolError):
|
||||
async_to_sync(server.call_tool)('docs_read', {'project': str(self.project.pk), 'path': '../secret.md'})
|
||||
denied = AccessToken(token='test', client_id='test', scopes=['projects:read'], subject=str(self.outsider.pk))
|
||||
with patch('mcp_server.server.get_access_token', return_value=denied):
|
||||
self.assertEqual(async_to_sync(server.call_tool)('docs_search', {'query': 'Guide'}).structured_content['total'], 0)
|
||||
with self.assertRaises(ToolError):
|
||||
async_to_sync(server.call_tool)('docs_read', {'project': str(self.project.pk), 'path': 'docs/guide.md'})
|
||||
|
||||
def test_url_allowlist_and_per_project_credentials(self):
|
||||
valid_url = self.project.git_repository_url
|
||||
self.assertEqual(_repository_url(self.project), valid_url)
|
||||
for url in ('http://git.jorendegraaf.eu/x', 'https://evil.example/x',
|
||||
'https://user:secret@git.jorendegraaf.eu/x', 'https://git.jorendegraaf.eu/a/../b'):
|
||||
self.project.git_repository_url = url
|
||||
from .documentation import DocumentationError
|
||||
with self.assertRaises(DocumentationError):
|
||||
_repository_url(self.project)
|
||||
self.project.git_repository_url = 'https://user:secret@git.jorendegraaf.eu/x'
|
||||
with self.assertRaises(ValidationError):
|
||||
self.project.full_clean()
|
||||
self.project.git_repository_url = valid_url
|
||||
credentials = {str(self.project.pk): {'repository_url': valid_url,
|
||||
'username': 'readonly', 'token': 'secret-token'}}
|
||||
with override_settings(PROJECT_DOCS_CREDENTIALS_JSON=json.dumps(credentials)):
|
||||
env = _git_env(self.project)
|
||||
self.assertIn('Authorization: Basic ', env['GIT_CONFIG_VALUE_0'])
|
||||
other = create_project(self.owner, name='Other')
|
||||
self.assertNotIn('GIT_CONFIG_VALUE_0', _git_env(other))
|
||||
|
||||
def test_invalid_audit_state_and_concurrent_claim(self):
|
||||
state = self.import_snapshot()
|
||||
self.assertEqual(audit_status(state), 'Unknown / due')
|
||||
state.audit_state = {'last_successful_audit': 'broken', 'last_audited_commit': 'a' * 40}
|
||||
self.assertEqual(audit_status(state), 'Unknown / due')
|
||||
DocumentationSync.objects.filter(pk=state.pk).update(
|
||||
sync_token='held', sync_started_at=timezone.now(), last_attempt_at=timezone.now() - timedelta(hours=1))
|
||||
with patch('projects.documentation._fetch_snapshot') as fetch:
|
||||
sync_project(self.owner, self.project)
|
||||
fetch.assert_not_called()
|
||||
|
||||
def test_symlink_document_is_not_indexed(self):
|
||||
blob = 'b' * 40
|
||||
listing = f'120000 blob {blob}\tdocs/link.md\0'.encode() + f'100644 blob {blob}\tdocs/real.md\0'.encode()
|
||||
outputs = [f'ref: refs/heads/main\tHEAD\n{"a" * 40}\tHEAD\n'.encode(), b'',
|
||||
('a' * 40 + '\n').encode(), b'2026-10-20T12:00:00+00:00\n', listing, b'# Real\n']
|
||||
with tempfile.TemporaryDirectory() as root, override_settings(PROJECT_DOCS_MIRROR_ROOT=root):
|
||||
(Path(root) / str(self.project.pk)).mkdir()
|
||||
with patch('projects.documentation._git', side_effect=outputs) as git_call:
|
||||
_, _, _, _, docs, _ = _fetch_snapshot(self.project)
|
||||
self.assertEqual([item.path for item in docs], ['docs/real.md'])
|
||||
self.assertEqual(sum(1 for call in git_call.call_args_list if 'cat-file' in call.args[0]), 1)
|
||||
|
||||
def test_retained_git_history_and_diff_are_scoped(self):
|
||||
def git(*args, cwd=None):
|
||||
return subprocess.check_output(['git', *args], cwd=cwd, stderr=subprocess.DEVNULL).decode().strip()
|
||||
|
||||
with tempfile.TemporaryDirectory() as root, override_settings(PROJECT_DOCS_MIRROR_ROOT=Path(root) / 'mirrors'):
|
||||
work = Path(root) / 'work'
|
||||
work.mkdir()
|
||||
git('init', '-q', cwd=work)
|
||||
git('config', 'user.name', 'Test', cwd=work)
|
||||
git('config', 'user.email', 'test@example.com', cwd=work)
|
||||
(work / 'docs').mkdir()
|
||||
path = work / 'docs' / 'guide.md'
|
||||
path.write_text('# Old\n')
|
||||
git('add', '.', cwd=work)
|
||||
git('commit', '-qm', 'first', cwd=work)
|
||||
first = git('rev-parse', 'HEAD', cwd=work)
|
||||
path.write_text('# New\n')
|
||||
git('commit', '-qam', 'second', cwd=work)
|
||||
second = git('rev-parse', 'HEAD', cwd=work)
|
||||
mirror = Path(root) / 'mirrors' / str(self.project.pk)
|
||||
mirror.parent.mkdir()
|
||||
git('clone', '--bare', '-q', str(work), str(mirror))
|
||||
DocumentationSync.objects.create(project=self.project, indexed_commit=second,
|
||||
indexed_repository_url=self.project.git_repository_url)
|
||||
ProjectDocument.objects.create(project=self.project, path='docs/guide.md', title='New',
|
||||
markdown='# New\n', headings=['New'], content_hash=hashlib.sha256(b'# New\n').hexdigest(),
|
||||
source_commit=second)
|
||||
history = document_history(self.viewer, self.project, 'docs/guide.md')
|
||||
self.assertEqual([item['commit'] for item in history], [second, first])
|
||||
old = historical_document(self.viewer, self.project, 'docs/guide.md', first)
|
||||
self.assertEqual(old.markdown, '# Old\n')
|
||||
diff, truncated = document_diff(self.viewer, self.project, 'docs/guide.md', first, second)
|
||||
self.assertIn('-# Old', diff)
|
||||
self.assertIn('+# New', diff)
|
||||
self.assertFalse(truncated)
|
||||
from django.http import Http404
|
||||
with self.assertRaises(Http404):
|
||||
document_history(self.outsider, self.project, 'docs/guide.md')
|
||||
|
||||
def test_project_delete_cleans_private_mirror_after_commit(self):
|
||||
with tempfile.TemporaryDirectory() as root, override_settings(PROJECT_DOCS_MIRROR_ROOT=root):
|
||||
mirror = Path(root) / str(self.project.pk)
|
||||
mirror.mkdir()
|
||||
(mirror / 'marker').write_text('private')
|
||||
with self.captureOnCommitCallbacks(execute=True):
|
||||
self.project.delete()
|
||||
self.assertFalse(mirror.exists())
|
||||
@@ -9,6 +9,9 @@ urlpatterns = [
|
||||
path('profile/', views.profile, name='profile'),
|
||||
path('profile/connections/<int:grant_id>/revoke/', views.revoke_connection, name='revoke_connection'),
|
||||
path('<uuid:project_id>/', views.board, name='board'),
|
||||
path('<uuid:project_id>/docs/', views.docs_home, name='docs_home'),
|
||||
path('<uuid:project_id>/docs/refresh/', views.docs_refresh, name='docs_refresh'),
|
||||
path('<uuid:project_id>/docs/<path:path>/', views.docs_document, name='docs_document'),
|
||||
path('<uuid:project_id>/edit/', views.project_edit, name='edit'),
|
||||
path('<uuid:project_id>/archive/', views.project_archive, name='archive'),
|
||||
path('<uuid:project_id>/restore/', views.project_restore, name='restore'),
|
||||
|
||||
@@ -4,6 +4,7 @@ from django.contrib.auth.views import LoginView
|
||||
from django.core.cache import cache
|
||||
from django.core import signing
|
||||
from django.http import HttpResponse, FileResponse
|
||||
from django.http import Http404
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
@@ -22,7 +23,11 @@ from .forms import CardForm, ColumnForm, ProfileForm, ProjectForm, TagForm
|
||||
from .models import Activity, Card, CardComment, CardAttachment, Column, Profile, Membership, Tag
|
||||
from . import services
|
||||
from . import attachments
|
||||
from . import documentation
|
||||
from .templatetags.project_markdown import safe_markdown
|
||||
from .templatetags.project_markdown import render_document_markdown
|
||||
from .models import DocumentationSync, ProjectDocument
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
class ThrottledLoginView(LoginView):
|
||||
@@ -303,6 +308,83 @@ def markdown_preview(request):
|
||||
return HttpResponse(safe_markdown(request.POST.get('description_markdown', '')))
|
||||
|
||||
|
||||
def _documentation_context(request, project):
|
||||
state = documentation.maybe_sync(request.user, project)
|
||||
can_edit = project.owner_id == request.user.pk or Membership.objects.filter(project=project, user=request.user, role='editor').exists()
|
||||
return {'project': project, 'sync': state, 'sync_status': documentation.sync_status(project, state),
|
||||
'can_edit': can_edit, 'audit_label': documentation.audit_status(state),
|
||||
'section': 'projects', 'title': f'{project.name} documentation'}
|
||||
|
||||
|
||||
@login_required
|
||||
def docs_home(request, project_id):
|
||||
project = services.project_for(request.user, project_id)
|
||||
context = _documentation_context(request, project)
|
||||
query = request.GET.get('q', '').strip()[:200]
|
||||
prefix = request.GET.get('path', '').strip()
|
||||
if prefix:
|
||||
try:
|
||||
documentation.normal_path(prefix)
|
||||
except ValidationError as exc:
|
||||
raise Http404('Folder unavailable') from exc
|
||||
page = max(1, int(request.GET.get('page', '1'))) if request.GET.get('page', '1').isdigit() else 1
|
||||
if query:
|
||||
documents, total = documentation.search_documents(request.user, query, project=project, limit=30, offset=(page - 1) * 30)
|
||||
else:
|
||||
documents, total = documentation.list_documents(request.user, project, path=prefix, limit=30, offset=(page - 1) * 30)
|
||||
paths = ProjectDocument.objects.filter(project=project).values_list('path', flat=True)
|
||||
folders = sorted({part for path in paths for part in ('/'.join(path.split('/')[:i]) for i in range(1, len(path.split('/'))))})
|
||||
folder_tree = []
|
||||
folder_nodes = {}
|
||||
for folder in folders:
|
||||
node = {'name': folder.rsplit('/', 1)[-1], 'path': folder, 'children': []}
|
||||
folder_nodes[folder] = node
|
||||
parent = folder.rpartition('/')[0]
|
||||
(folder_nodes[parent]['children'] if parent else folder_tree).append(node)
|
||||
context.update({'documents': documents, 'query': query, 'page': page, 'has_next': page * 30 < total,
|
||||
'has_previous': page > 1, 'total': total, 'prefix': prefix, 'folder_tree': folder_tree,
|
||||
'results': [{'document': item, 'snippet': documentation.excerpt(item, query)} for item in documents] if query else []})
|
||||
return render(request, 'projects/docs_home.html', context)
|
||||
|
||||
|
||||
@login_required
|
||||
def docs_document(request, project_id, path):
|
||||
project = services.project_for(request.user, project_id)
|
||||
context = _documentation_context(request, project)
|
||||
try:
|
||||
current = documentation.read_document(request.user, project, path)
|
||||
revision = request.GET.get('revision', '')
|
||||
document = documentation.historical_document(request.user, project, path, revision) if revision else current
|
||||
except (ProjectDocument.DoesNotExist, ValidationError, documentation.DocumentationError) as exc:
|
||||
raise Http404('Document unavailable') from exc
|
||||
paths = set(ProjectDocument.objects.filter(project=project).values_list('path', flat=True))
|
||||
context['document'] = document
|
||||
context['file_name'] = path.rsplit('/', 1)[-1]
|
||||
context['breadcrumbs'] = [{'name': part, 'path': '/'.join(path.split('/')[:index + 1])}
|
||||
for index, part in enumerate(path.split('/')[:-1])]
|
||||
context['is_historical'] = document.source_commit != current.source_commit
|
||||
try:
|
||||
context['history'] = documentation.document_history(request.user, project, path)
|
||||
except documentation.DocumentationError:
|
||||
context['history'] = []
|
||||
context['document_commit_time'] = next((item['committed_at'] for item in context['history']
|
||||
if item['commit'] == document.source_commit),
|
||||
context['sync'].commit_time if context['sync'] and not context['is_historical'] else None)
|
||||
context['rendered'] = render_document_markdown(document.markdown, project.pk, document.path, paths)
|
||||
if context['sync'] and context['sync'].indexed_repository_url:
|
||||
base = context['sync'].indexed_repository_url.rstrip('/').removesuffix('.git')
|
||||
context['source_url'] = f'{base}/src/commit/{document.source_commit}/{quote(document.path)}'
|
||||
return render(request, 'projects/docs_document.html', context)
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def docs_refresh(request, project_id):
|
||||
project = services.project_for(request.user, project_id, write=True)
|
||||
documentation.sync_project(request.user, project, force=True)
|
||||
return redirect('projects:docs_home', project_id=project.pk)
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def card_move(request, card_id):
|
||||
|
||||
Reference in New Issue
Block a user