git documentation feature

This commit is contained in:
Joren de Graaf
2026-10-20 19:10:00 +02:00
parent 03d7afc6a1
commit a15c1f4c94
21 changed files with 1242 additions and 9 deletions
+14 -1
View File
@@ -6,11 +6,24 @@ class ProjectsConfig(AppConfig):
name = 'projects'
def ready(self):
import shutil
from pathlib import Path
from django.conf import settings
from django.db import transaction
from django.db.models.signals import post_delete
from .models import CardAttachment
from .models import CardAttachment, Project
def remove_stored_file(sender, instance, **kwargs):
if instance.file:
instance.file.delete(save=False)
post_delete.connect(remove_stored_file, sender=CardAttachment, weak=False, dispatch_uid='projects.remove_attachment_file')
def remove_project_mirror(sender, instance, **kwargs):
root = Path(settings.PROJECT_DOCS_MIRROR_ROOT)
path = root / str(instance.pk)
if path.is_symlink() or not path.is_dir():
return
transaction.on_commit(lambda: shutil.rmtree(path, ignore_errors=True))
post_delete.connect(remove_project_mirror, sender=Project, weak=False, dispatch_uid='projects.remove_documentation_mirror')
+463
View File
@@ -0,0 +1,463 @@
"""Read-only, project-scoped Git documentation index."""
import base64
import difflib
import hashlib
import json
import logging
import os
import posixpath
import re
import shutil
import subprocess
import tempfile
from time import monotonic
import uuid
from datetime import datetime, timedelta
from pathlib import Path
from urllib.parse import urlsplit
from django.conf import settings
from django.core.exceptions import ValidationError
from django.db import transaction
from django.db.models import Q
from django.utils import timezone
from . import services
from .models import DocumentationSync, ProjectDocument
MAX_RESULTS = 100
MAX_READ_LINES = 250
MAX_READ_CHARS = 30000
REF_RE = re.compile(r'^refs/heads/[A-Za-z0-9][A-Za-z0-9._/-]*$')
HASH_RE = re.compile(r'^[0-9a-f]{40}([0-9a-f]{24})?$')
HEADING_RE = re.compile(r'^#{1,6}\s+(.+?)\s*#*\s*$')
logger = logging.getLogger(__name__)
class DocumentationError(Exception):
pass
def normal_path(value):
if not isinstance(value, str) or not value or len(value) > 500 or '\x00' in value:
raise ValidationError('Invalid document path.')
if value.startswith('/') or '\\' in value or any(part in ('', '.', '..') for part in value.split('/')):
raise ValidationError('Invalid document path.')
if any(ord(c) < 32 for c in value):
raise ValidationError('Invalid document path.')
return value
def indexed_path(value):
try:
normal_path(value)
except ValidationError:
return False
return value in ('README.md', 'AGENTS.md') or (value.startswith('docs/') and value.lower().endswith('.md'))
def _repository_url(project):
raw = project.git_repository_url
try:
parsed = urlsplit(raw)
hostname = parsed.hostname
port = parsed.port
except ValueError as exc:
raise DocumentationError('Repository URL is malformed.') from exc
allowed = {item.strip().lower() for item in settings.PROJECT_DOCS_ALLOWED_GIT_HOSTS.split(',') if item.strip()}
authority = hostname.lower() if hostname else ''
if port:
authority += f':{port}'
if (parsed.scheme != 'https' or not authority or authority not in allowed or
parsed.username or parsed.password or parsed.query or parsed.fragment or
not parsed.path.startswith('/') or '..' in parsed.path.split('/') or
'%' in raw or '\\' in raw or any(c.isspace() for c in raw)):
raise DocumentationError('Repository URL is not an approved HTTPS Git endpoint.')
return raw
def _git_env(project):
env = {key: value for key, value in os.environ.items()
if not key.startswith('GIT_') and key not in ('SSH_ASKPASS', 'SSH_ASKPASS_REQUIRE')}
env.update({'GIT_TERMINAL_PROMPT': '0', 'GIT_CONFIG_NOSYSTEM': '1',
'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_SYSTEM': os.devnull,
'GIT_NO_REPLACE_OBJECTS': '1'})
try:
secrets = json.loads(settings.PROJECT_DOCS_CREDENTIALS_JSON or '{}')
except (TypeError, ValueError) as exc:
raise DocumentationError('Git credentials are not configured correctly.') from exc
if not isinstance(secrets, dict):
raise DocumentationError('Git credentials are not configured correctly.')
entry = secrets.get(str(project.pk), {})
if not isinstance(entry, dict):
raise DocumentationError('Project Git credentials are not configured correctly.')
if entry:
username, token = entry.get('username'), entry.get('token')
if entry.get('repository_url') != _repository_url(project):
raise DocumentationError('Project Git credential is not bound to this repository URL.')
if not isinstance(username, str) or not isinstance(token, str) or not username or not token or '\n' in username + token or '\r' in username + token:
raise DocumentationError('Project Git credentials are incomplete.')
encoded = base64.b64encode(f'{username}:{token}'.encode()).decode()
env['GIT_CONFIG_COUNT'] = '1'
env['GIT_CONFIG_KEY_0'] = f'http.{_repository_url(project)}.extraHeader'
env['GIT_CONFIG_VALUE_0'] = f'Authorization: Basic {encoded}'
return env
def _git(args, *, env, timeout=30, max_output=20 * 1024 * 1024):
limiter = shutil.which('prlimit')
if not limiter:
raise DocumentationError('Git resource limiter is unavailable on this server.')
command = [limiter, f'--fsize={settings.PROJECT_DOCS_MAX_MIRROR_BYTES}', '--cpu=65', '--',
'git', '-c', 'credential.helper=', '-c', 'core.hooksPath=/dev/null',
'-c', 'protocol.file.allow=never', '-c', 'protocol.ext.allow=never',
'-c', 'http.followRedirects=false', '-c', 'transfer.fsckObjects=true',
'-c', 'gc.auto=0', *args]
try:
with tempfile.TemporaryFile() as output:
result = subprocess.run(command, env=env, stdout=output,
stderr=subprocess.DEVNULL, timeout=timeout, check=False)
if result.returncode or output.tell() > max_output:
raise DocumentationError('Git fetch or index failed; check repository access and size limits.')
output.seek(0)
return output.read(max_output + 1)
except (OSError, subprocess.TimeoutExpired, subprocess.SubprocessError) as exc:
raise DocumentationError('Git fetch or index timed out or could not start.') from exc
def _mirror_path(project, *, create=True):
root = Path(settings.PROJECT_DOCS_MIRROR_ROOT)
if create:
root.mkdir(mode=0o700, parents=True, exist_ok=True)
path = root / str(project.pk)
if path.is_symlink() or (path.exists() and not path.is_dir()):
raise DocumentationError('Repository storage is unavailable.')
return path
def _mirror_size(path):
return sum(item.stat().st_size for item in path.rglob('*') if item.is_file() and not item.is_symlink())
def _fetch_snapshot(project, observed_head=None):
url = _repository_url(project)
env = _git_env(project)
deadline = monotonic() + 90
def run_git(args, *, timeout=30, max_output=20 * 1024 * 1024):
remaining = deadline - monotonic()
if remaining <= 0:
raise DocumentationError('Git documentation refresh exceeded its time limit.')
return _git(args, env=env, timeout=min(timeout, remaining), max_output=max_output)
path = _mirror_path(project)
if not path.exists():
run_git(['init', '--bare', str(path)])
path.chmod(0o700)
if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES:
raise DocumentationError('Repository mirror exceeds the configured size limit.')
remote = run_git(['ls-remote', '--symref', url, 'HEAD'], max_output=4096).decode('ascii', 'replace')
match = re.search(r'^ref: (refs/heads/\S+)\s+HEAD$', remote, re.M)
if not match or not REF_RE.fullmatch(match.group(1)) or '..' in match.group(1) or '@{' in match.group(1):
raise DocumentationError('Repository does not advertise a safe default branch.')
branch = match.group(1)
head_match = re.search(r'^([0-9a-f]{40}(?:[0-9a-f]{24})?)\s+HEAD$', remote, re.M)
if not head_match:
raise DocumentationError('Repository did not provide a valid HEAD commit.')
if observed_head:
observed_head(branch, head_match.group(1))
run_git(['--git-dir', str(path), 'fetch', '--no-tags', '--depth=50', url,
f'+{branch}:refs/remotes/origin/{branch.removeprefix("refs/heads/")}'], timeout=60, max_output=1024)
if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES:
raise DocumentationError('Repository mirror exceeds the configured size limit.')
commit = run_git(['--git-dir', str(path), 'rev-parse', f'refs/remotes/origin/{branch.removeprefix("refs/heads/")}'],
max_output=128).decode().strip()
if not HASH_RE.fullmatch(commit):
raise DocumentationError('Repository returned an invalid commit.')
committed_at = run_git(['--git-dir', str(path), 'show', '-s', '--format=%cI', commit], max_output=128).decode().strip()
try:
commit_time = datetime.fromisoformat(committed_at)
if commit_time.tzinfo is None:
raise ValueError
except ValueError as exc:
raise DocumentationError('Repository returned an invalid commit time.') from exc
listing = run_git(['--git-dir', str(path), 'ls-tree', '-r', '-z', commit],
max_output=settings.PROJECT_DOCS_MAX_TREE_BYTES)
rows = []
audit_blob = None
total = 0
for raw in listing.split(b'\x00'):
if not raw:
continue
try:
metadata, name = raw.split(b'\t', 1)
mode, kind, blob = metadata.decode('ascii').split(' ')
name = name.decode('utf-8')
except (ValueError, UnicodeDecodeError) as exc:
if b'\tdocs/' in raw or raw.endswith((b'\tREADME.md', b'\tAGENTS.md')):
raise DocumentationError('Repository tree contains an unsupported documentation path.') from exc
continue
if name == 'docs/.audit-state.json' and mode == '100644' and kind == 'blob':
audit_blob = blob
if not indexed_path(name):
continue
if mode != '100644' or kind != 'blob' or not HASH_RE.fullmatch(blob):
continue
rows.append((name, blob))
if len(rows) > settings.PROJECT_DOCS_MAX_FILES:
raise DocumentationError('Repository has too many documentation files.')
documents = []
for name, blob in rows:
content = run_git(['--git-dir', str(path), 'cat-file', 'blob', blob],
max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES)
total += len(content)
if total > settings.PROJECT_DOCS_MAX_CONTENT_BYTES:
raise DocumentationError('Documentation exceeds the configured content limit.')
try:
markdown = content.decode('utf-8')
except UnicodeDecodeError as exc:
raise DocumentationError(f'Document {name} is not UTF-8.') from exc
headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))]
documents.append(ProjectDocument(project=project, path=name,
title=(headings[0] if headings else posixpath.basename(name))[:255], headings=headings[:100],
markdown=markdown, content_hash=hashlib.sha256(content).hexdigest(), source_commit=commit))
audit = {}
if audit_blob and HASH_RE.fullmatch(audit_blob):
raw = run_git(['--git-dir', str(path), 'cat-file', 'blob', audit_blob], max_output=65536)
try:
candidate = json.loads(raw)
if isinstance(candidate, dict) and candidate.get('schema_version') == 1:
limits = {'last_successful_audit': 40, 'last_audited_commit': 64,
'last_report_path': 300, 'last_attempt': 40, 'last_status': 40}
audit = {key: candidate[key] for key, size in limits.items()
if isinstance(candidate.get(key), str) and len(candidate[key]) <= size}
if isinstance(candidate.get('audit_interval_days'), int):
audit['audit_interval_days'] = candidate['audit_interval_days']
except (ValueError, UnicodeDecodeError):
pass
return url, branch, commit, commit_time, documents, audit
def sync_project(user, project, *, force=False):
services.project_for(user, project.pk, write=force)
if not project.git_repository_url:
raise DocumentationError('Configure a Git repository URL before refreshing documentation.')
state, _ = DocumentationSync.objects.get_or_create(project=project)
now = timezone.now()
interval = timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS))
if not force and state.last_attempt_at and now - state.last_attempt_at < interval:
return state
if force and state.last_attempt_at and now - state.last_attempt_at < timedelta(seconds=30):
return state
token = str(uuid.uuid4())
claimed = DocumentationSync.objects.filter(pk=state.pk).filter(
Q(sync_token='') | Q(sync_started_at__lt=now - timedelta(minutes=3))).update(
sync_token=token, sync_started_at=now, last_attempt_at=now)
if not claimed:
return DocumentationSync.objects.get(pk=state.pk)
try:
def record_head(branch, head):
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(branch=branch, remote_head=head)
url, branch, commit, commit_time, documents, audit = _fetch_snapshot(project, observed_head=record_head)
with transaction.atomic():
state = DocumentationSync.objects.select_for_update().get(pk=state.pk)
if state.sync_token != token:
return state
if state.indexed_commit != commit or state.indexed_repository_url != url:
ProjectDocument.objects.filter(project=project).delete()
ProjectDocument.objects.bulk_create(documents, batch_size=100)
state.indexed_commit = commit
state.remote_head = commit
state.indexed_repository_url = url
state.branch = branch
state.commit_time = commit_time
state.last_success_at = timezone.now()
state.last_error = ''
state.sync_token = ''
state.sync_started_at = None
state.audit_state = audit
state.save()
except DocumentationError as exc:
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(
last_error=str(exc)[:300], sync_token='', sync_started_at=None)
except Exception as exc:
logger.error('Documentation indexing failed for project %s (%s).', project.pk, type(exc).__name__)
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(
last_error='Documentation indexing failed; retry after checking server logs.',
sync_token='', sync_started_at=None)
return DocumentationSync.objects.get(pk=state.pk)
def maybe_sync(user, project):
"""On-demand polling: one bounded refresh per interval for authorized readers."""
services.project_for(user, project.pk)
if project.git_repository_url:
return sync_project(user, project)
return DocumentationSync.objects.filter(project=project).first()
def sync_status(project, state):
if not project.git_repository_url:
return 'No repository configured'
if not state or not state.indexed_commit:
if state and state.sync_token:
return 'Refreshing from Git; no indexed documentation yet'
return 'No indexed documentation' if not state or not state.last_error else f'Sync error: {state.last_error}'
if state.sync_token:
return 'Refreshing from Git; showing the last indexed version'
if state.last_error:
return f'Stale after sync error: {state.last_error}'
if state.indexed_repository_url != project.git_repository_url:
return 'Stale: repository URL changed'
if state.remote_head and state.remote_head != state.indexed_commit:
return 'Stale: remote HEAD differs from indexed commit'
if state.last_success_at and timezone.now() - state.last_success_at >= timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS)):
return 'Refresh due; indexed commit may lag remote'
return 'Indexed from Git; document accuracy is not guaranteed'
def audit_status(state):
audit = state.audit_state if state else {}
date_value = audit.get('last_successful_audit')
commit = audit.get('last_audited_commit')
partial = audit.get('last_status') in ('partial', 'error')
try:
last = datetime.fromisoformat(date_value.replace('Z', '+00:00'))
days = int(audit.get('audit_interval_days', 14))
if last.tzinfo is None or not HASH_RE.fullmatch(commit) or not 1 <= days <= 365:
raise ValueError
except (AttributeError, TypeError, ValueError):
return 'Partial or failed / due' if partial else 'Unknown / due'
if last > timezone.now() + timedelta(minutes=5):
return 'Unknown / due (audit timestamp is in the future)'
if timezone.now() >= last + timedelta(days=days):
return f'Overdue (last successful audit {last.date()})' + ('; latest attempt partial or failed' if partial else '')
if state.indexed_commit != commit:
return f'Last audited {last.date()}; indexed commit differs' + ('; latest attempt partial or failed' if partial else '')
return f'Last audited {last.date()} at this commit' + ('; latest attempt partial or failed' if partial else '')
def list_documents(user, project, *, path='', limit=50, offset=0):
services.project_for(user, project.pk)
if path:
normal_path(path)
size = min(max(int(limit), 1), MAX_RESULTS)
offset = max(int(offset), 0)
qs = ProjectDocument.objects.filter(project=project)
if path:
qs = qs.filter(path__startswith=path.rstrip('/') + '/')
qs = qs.order_by('path')
return list(qs[offset:offset + size]), qs.count()
def search_documents(user, query, *, project=None, limit=30, offset=0):
query = query.strip() if isinstance(query, str) else ''
if not query or len(query) > 200:
raise ValidationError('Search query must be 1–200 characters.')
qs = ProjectDocument.objects.select_related('project').filter(project__in=services.accessible(user))
if project:
services.project_for(user, project.pk)
qs = qs.filter(project=project)
qs = qs.filter(Q(path__icontains=query) | Q(title__icontains=query) | Q(markdown__icontains=query)).order_by('project_id', 'path')
size = min(max(int(limit), 1), MAX_RESULTS)
offset = max(int(offset), 0)
return list(qs[offset:offset + size]), qs.count()
def read_document(user, project, path):
services.project_for(user, project.pk)
return ProjectDocument.objects.get(project=project, path=normal_path(path))
def excerpt(document, query, radius=120):
content = document.markdown
at = content.lower().find(query.lower())
if at < 0:
at = 0
start = max(0, at - radius)
end = min(len(content), at + len(query) + radius)
return ('…' if start else '') + content[start:end].replace('\n', ' ') + ('…' if end < len(content) else '')
def line_window(document, start_line=1, end_line=None):
lines = document.markdown.splitlines(keepends=True)
start = max(int(start_line), 1)
if start > len(lines) + 1:
raise ValidationError('Start line is beyond the document.')
requested_end = int(end_line) if end_line is not None else start + MAX_READ_LINES - 1
if requested_end < start:
raise ValidationError('Invalid line range.')
end = min(requested_end, start + MAX_READ_LINES - 1, len(lines))
output = ''
partial_line = False
actual_end = start - 1
for number in range(start, end + 1):
line = lines[number - 1]
remaining = MAX_READ_CHARS - len(output)
if remaining <= 0:
break
if len(line) > remaining:
output += line[:remaining]
partial_line = True
actual_end = number
break
output += line
actual_end = number
return {'markdown': output, 'start_line': start, 'end_line': actual_end,
'total_lines': len(lines), 'truncated': actual_end < len(lines) or partial_line,
'partial_line': partial_line}
def document_history(user, project, path, limit=20):
services.project_for(user, project.pk)
normal_path(path)
state = DocumentationSync.objects.filter(project=project).first()
if not state or not state.indexed_commit:
return []
mirror = _mirror_path(project, create=False)
if not mirror.exists():
return []
output = _git(['--git-dir', str(mirror), 'log', f'--max-count={min(max(int(limit), 1), 50)}',
'--format=%H%x09%cI', state.indexed_commit, '--', path],
env=_git_env(project), max_output=8192)
rows = []
for line in output.decode('ascii', 'replace').splitlines():
try:
commit, committed_at = line.split('\t', 1)
except ValueError:
continue
if HASH_RE.fullmatch(commit):
rows.append({'commit': commit, 'committed_at': committed_at})
return rows
def historical_document(user, project, path, revision):
current = read_document(user, project, path)
if revision == current.source_commit:
return current
if not isinstance(revision, str) or not HASH_RE.fullmatch(revision):
raise ValidationError('Invalid Git revision.')
if revision not in {row['commit'] for row in document_history(user, project, path, limit=50)}:
raise ValidationError('Revision is unavailable in the retained Git history.')
mirror = _mirror_path(project, create=False)
content = _git(['--git-dir', str(mirror), 'show', f'{revision}:{path}'],
env=_git_env(project), max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES)
try:
markdown = content.decode('utf-8')
except UnicodeDecodeError as exc:
raise DocumentationError('Historical document is not UTF-8.') from exc
headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))]
return ProjectDocument(project=project, path=path, title=(headings[0] if headings else posixpath.basename(path))[:255],
headings=headings[:100], markdown=markdown,
content_hash=hashlib.sha256(content).hexdigest(), source_commit=revision)
def document_diff(user, project, path, before, after):
first = historical_document(user, project, path, before)
second = historical_document(user, project, path, after)
diff = ''.join(difflib.unified_diff(first.markdown.splitlines(keepends=True), second.markdown.splitlines(keepends=True),
fromfile=before, tofile=after))
return diff[:MAX_READ_CHARS], len(diff) > MAX_READ_CHARS
@@ -0,0 +1,50 @@
# Generated by Django 6.1.2 on 2026-10-20 12:20
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('projects', '0005_cardattachment'),
]
operations = [
migrations.CreateModel(
name='DocumentationSync',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('branch', models.CharField(blank=True, max_length=255)),
('indexed_repository_url', models.URLField(blank=True)),
('indexed_commit', models.CharField(blank=True, max_length=64)),
('remote_head', models.CharField(blank=True, max_length=64)),
('commit_time', models.DateTimeField(blank=True, null=True)),
('last_success_at', models.DateTimeField(blank=True, null=True)),
('last_attempt_at', models.DateTimeField(blank=True, null=True)),
('last_error', models.CharField(blank=True, max_length=300)),
('sync_token', models.CharField(blank=True, max_length=36)),
('sync_started_at', models.DateTimeField(blank=True, null=True)),
('audit_state', models.JSONField(blank=True, default=dict)),
('project', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='documentation_sync', to='projects.project')),
],
),
migrations.CreateModel(
name='ProjectDocument',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('path', models.CharField(max_length=500)),
('title', models.CharField(max_length=255)),
('headings', models.JSONField(default=list)),
('markdown', models.TextField()),
('content_hash', models.CharField(max_length=64)),
('source_commit', models.CharField(max_length=64)),
('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='documents', to='projects.project')),
],
options={
'ordering': ['path'],
'indexes': [models.Index(fields=['project', 'path'], name='projects_pr_project_424af7_idx')],
'constraints': [models.UniqueConstraint(fields=('project', 'path'), name='unique_project_document_path')],
},
),
]
+44
View File
@@ -1,4 +1,5 @@
import uuid
from urllib.parse import urlsplit
from django.conf import settings
from django.core.validators import RegexValidator
from django.core.exceptions import ValidationError
@@ -39,6 +40,49 @@ class Project(models.Model):
class Meta:
constraints = [models.UniqueConstraint(fields=['owner', 'slug'], name='unique_owner_project_slug'), models.UniqueConstraint(fields=['owner', 'name'], name='unique_owner_project_name')]
def clean(self):
if self.git_repository_url:
try:
parsed = urlsplit(self.git_repository_url)
_ = parsed.port
valid = parsed.scheme == 'https' and bool(parsed.hostname)
except ValueError:
valid = False
parsed = None
if (not valid or parsed.username or parsed.password or parsed.query or parsed.fragment or
'\\' in self.git_repository_url or any(c.isspace() for c in self.git_repository_url)):
raise ValidationError({'git_repository_url': 'Use an HTTPS Git URL without credentials or query parameters.'})
class DocumentationSync(models.Model):
project = models.OneToOneField(Project, on_delete=models.CASCADE, related_name='documentation_sync')
branch = models.CharField(max_length=255, blank=True)
indexed_repository_url = models.URLField(blank=True)
indexed_commit = models.CharField(max_length=64, blank=True)
remote_head = models.CharField(max_length=64, blank=True)
commit_time = models.DateTimeField(null=True, blank=True)
last_success_at = models.DateTimeField(null=True, blank=True)
last_attempt_at = models.DateTimeField(null=True, blank=True)
last_error = models.CharField(max_length=300, blank=True)
sync_token = models.CharField(max_length=36, blank=True)
sync_started_at = models.DateTimeField(null=True, blank=True)
audit_state = models.JSONField(default=dict, blank=True)
class ProjectDocument(models.Model):
project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='documents')
path = models.CharField(max_length=500)
title = models.CharField(max_length=255)
headings = models.JSONField(default=list)
markdown = models.TextField()
content_hash = models.CharField(max_length=64)
source_commit = models.CharField(max_length=64)
class Meta:
ordering = ['path']
constraints = [models.UniqueConstraint(fields=['project', 'path'], name='unique_project_document_path')]
indexes = [models.Index(fields=['project', 'path'])]
class Membership(models.Model):
ROLE = [('editor', 'Editor'), ('viewer', 'Viewer')]
@@ -0,0 +1 @@
{% if nodes %}<ul class="list-unstyled ms-3">{% for node in nodes %}<li><a href="{% url 'projects:docs_home' project.id %}?path={{ node.path|urlencode }}"{% if node.path == prefix %} aria-current="page"{% endif %}>{{ node.name }}/</a>{% include 'projects/_docs_folder_tree.html' with nodes=node.children %}</li>{% endfor %}</ul>{% endif %}
+1 -1
View File
@@ -2,7 +2,7 @@
{% block content %}
<style>.htmx-indicator{display:none}.htmx-request.htmx-indicator,.htmx-request .htmx-indicator{display:inline}</style>
<div class="d-flex justify-content-between align-items-center mb-3">
<div><a href="{% url 'projects:list' %}">All projects</a><h1>{{ project.name }}</h1><p>{{ project.description }}</p>{% if project.git_repository_url %}<a href="{{ project.git_repository_url }}" rel="noopener noreferrer">Repository</a>{% endif %}<p>{{ done_count }} of {{ total_count }} cards done</p></div>
<div><a href="{% url 'projects:list' %}">All projects</a><h1>{{ project.name }}</h1><p>{{ project.description }}</p>{% if project.git_repository_url %}<a href="{{ project.git_repository_url }}" rel="noopener noreferrer">Repository</a>{% endif %} <a href="{% url 'projects:docs_home' project.id %}">Documentation</a><p>{{ done_count }} of {{ total_count }} cards done</p></div>
{% if can_edit %}<div><a class="btn btn-outline-secondary" href="{% url 'projects:edit' project.id %}">Edit project</a> {% if project.archived %}<form class="d-inline" method="post" action="{% url 'projects:restore' project.id %}">{% csrf_token %}<button class="btn btn-outline-success">Restore project</button></form>{% else %}<form class="d-inline" method="post" action="{% url 'projects:archive' project.id %}">{% csrf_token %}<button class="btn btn-outline-danger">Archive project</button></form> <a class="btn btn-success" href="{% url 'projects:card_create' project.id %}">New card</a>{% endif %}</div>{% endif %}
</div>
<div class="d-flex gap-3 flex-wrap mb-3">{% for label,count in priority_counts %}<span class="badge text-bg-light border">{{ label }}: {{ count }} open</span>{% endfor %}</div>
@@ -0,0 +1,10 @@
{% extends 'core/base.html' %}
{% block content %}
<nav aria-label="Breadcrumb" class="mb-3"><a href="{% url 'projects:board' project.id %}">{{ project.name }}</a> / <a href="{% url 'projects:docs_home' project.id %}">Documentation</a> / {% for crumb in breadcrumbs %}<a href="{% url 'projects:docs_home' project.id %}?path={{ crumb.path|urlencode }}">{{ crumb.name }}</a> / {% endfor %}{{ file_name }}</nav>
<div class="d-flex justify-content-between align-items-start gap-3 flex-wrap"><div><h1>{{ document.title }}</h1><p><code>{{ document.path }}</code></p></div>{% if source_url %}<a class="btn btn-outline-secondary" href="{{ source_url }}" rel="noopener noreferrer">View in Git</a>{% endif %}</div>
<div class="alert {% if sync.last_error %}alert-warning{% else %}alert-secondary{% endif %}" role="status"><strong>{{ sync_status }}</strong><span class="d-block">Source commit: <code>{{ document.source_commit }}</code> · SHA-256: <code>{{ document.content_hash }}</code></span>{% if document_commit_time %}<span class="d-block">Commit time: {{ document_commit_time }}</span>{% endif %}{% if sync.remote_head and sync.remote_head != sync.indexed_commit %}<span class="d-block">Last observed remote HEAD: <code>{{ sync.remote_head }}</code></span>{% endif %}{% if sync.last_success_at %}<span class="d-block">Fetched {{ sync.last_success_at }}</span>{% endif %}<span class="d-block">Documentation accuracy audit: {{ audit_label }}.</span></div>
{% if is_historical %}<p class="alert alert-info" role="status">Viewing a historical Git version. <a href="{% url 'projects:docs_document' project.id document.path %}">Return to indexed version</a>.</p>{% endif %}
<article class="card card-body markdown-body">{{ rendered }}</article>
<details class="mt-3"><summary>Raw Markdown</summary><pre class="card card-body overflow-auto">{{ document.markdown }}</pre></details>
{% if history %}<section class="mt-4" aria-labelledby="doc-history-heading"><h2 class="h5" id="doc-history-heading">Git history</h2><p class="text-muted">Recent commits for this path in the retained shallow mirror. Renames and deleted documents are not followed yet.</p><ol>{% for version in history %}<li><a href="?revision={{ version.commit }}"><code>{{ version.commit|slice:':12' }}</code></a> · {{ version.committed_at }}</li>{% endfor %}</ol></section>{% endif %}
{% endblock %}
@@ -0,0 +1,28 @@
{% extends 'core/base.html' %}
{% block content %}
<div class="d-flex justify-content-between align-items-start gap-3 flex-wrap mb-3">
<div><a href="{% url 'projects:board' project.id %}">← {{ project.name }}</a><h1 class="mt-2">Documentation</h1><p class="text-muted">Read-only Markdown from the project's Git repository.</p></div>
{% if can_edit and project.git_repository_url %}<form method="post" action="{% url 'projects:docs_refresh' project.id %}">{% csrf_token %}<button class="btn btn-outline-primary">Refresh from Git</button></form>{% endif %}
</div>
<div class="alert {% if sync.last_error %}alert-warning{% else %}alert-secondary{% endif %}" role="status">
<strong>{{ sync_status }}</strong>
{% if sync.indexed_commit %}<span class="d-block">Indexed commit: <code>{{ sync.indexed_commit }}</code>{% if sync.branch %} · {{ sync.branch }}{% endif %}{% if sync.commit_time %} · committed {{ sync.commit_time }}{% endif %}{% if sync.last_success_at %} · fetched {{ sync.last_success_at }}{% endif %}</span>{% endif %}
{% if sync.remote_head and sync.remote_head != sync.indexed_commit %}<span class="d-block">Last observed remote HEAD: <code>{{ sync.remote_head }}</code></span>{% endif %}
{% if sync.last_attempt_at %}<span class="d-block">Last fetch attempt: {{ sync.last_attempt_at }}</span>{% endif %}
<span class="d-block">Documentation accuracy audit: {{ audit_label }}.</span>
</div>
{% if not project.git_repository_url %}<p>No repository is configured. {% if can_edit %}Add its HTTPS Git URL in <a href="{% url 'projects:edit' project.id %}">project settings</a>.{% else %}Ask a project editor to configure its Git URL.{% endif %}</p>{% endif %}
<form method="get" action="{% url 'projects:docs_home' project.id %}" role="search" class="mb-4">
<label for="docs-search" class="form-label">Search documentation</label>
<div class="input-group"><input id="docs-search" class="form-control" name="q" value="{{ query }}" maxlength="200" placeholder="Path, heading or text"><button class="btn btn-outline-secondary">Search</button></div>
</form>
<div class="row g-4">
<nav class="col-md-4 col-lg-3" aria-label="Documentation folders"><h2 class="h5">Folders</h2><ul class="list-unstyled"><li><a href="{% url 'projects:docs_home' project.id %}">All documents</a></li></ul>{% include 'projects/_docs_folder_tree.html' with nodes=folder_tree %}</nav>
<section class="col-md-8 col-lg-9" aria-label="Document results">
<h2 class="h5">{% if query %}Search results{% elif prefix %}{{ prefix }}/{% else %}Documents{% endif %} <small class="text-muted">({{ total }})</small></h2>
{% if documents %}<ul class="list-group">{% if query %}{% for result in results %}<li class="list-group-item"><a href="{% url 'projects:docs_document' project.id result.document.path %}"><strong>{{ result.document.title }}</strong></a><br><small class="text-muted">{{ result.document.path }}</small><p class="mb-0 mt-1">{{ result.snippet }}</p></li>{% endfor %}{% else %}{% for document in documents %}<li class="list-group-item"><a href="{% url 'projects:docs_document' project.id document.path %}"><strong>{{ document.title }}</strong></a><br><small class="text-muted">{{ document.path }}</small></li>{% endfor %}{% endif %}</ul>
{% else %}<p>{% if query %}No matching documents.{% elif sync.last_error %}No indexed documents are available. The latest Git fetch failed.{% elif project.git_repository_url %}No Markdown documentation has been indexed yet.{% else %}Add a repository to begin.{% endif %}</p>{% endif %}
{% if has_previous or has_next %}<nav class="mt-3" aria-label="Document pages">{% if has_previous %}<a class="btn btn-outline-secondary btn-sm" href="?q={{ query|urlencode }}&amp;path={{ prefix|urlencode }}&amp;page={{ page|add:'-1' }}">Previous</a>{% endif %} <span>Page {{ page }}</span> {% if has_next %}<a class="btn btn-outline-secondary btn-sm" href="?q={{ query|urlencode }}&amp;path={{ prefix|urlencode }}&amp;page={{ page|add:'1' }}">Next</a>{% endif %}</nav>{% endif %}
</section>
</div>
{% endblock %}
+52 -2
View File
@@ -1,12 +1,62 @@
import markdown
import nh3
import posixpath
from urllib.parse import unquote, urlsplit
from django import template
from django.urls import reverse
from django.utils.safestring import mark_safe
from markdown.extensions import Extension
from markdown.treeprocessors import Treeprocessor
register = template.Library()
TAGS = {'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'li', 'pre', 'code', 'blockquote', 'strong', 'em', 'a', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'hr', 'br', 'input'}
ATTRIBUTES = {'a': {'href', 'title'}, 'input': {'type', 'checked', 'disabled'}}
def _clean(value, extensions, attributes=ATTRIBUTES):
html = markdown.markdown(value or '', extensions=extensions)
return mark_safe(nh3.clean(html, tags=TAGS, attributes=attributes, url_schemes={'https', 'http', 'mailto'}))
@register.filter
def safe_markdown(value):
html = markdown.markdown(value or '', extensions=['extra', 'fenced_code', 'tables', 'sane_lists', 'pymdownx.tasklist'])
return mark_safe(nh3.clean(html, tags={'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'li', 'pre', 'code', 'blockquote', 'strong', 'em', 'a', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'hr', 'br', 'input'}, attributes={'a': {'href', 'title'}, 'input': {'type', 'checked', 'disabled'}}, url_schemes={'https', 'http', 'mailto'}))
return _clean(value, ['extra', 'fenced_code', 'tables', 'sane_lists', 'pymdownx.tasklist'])
class _DocumentLinks(Treeprocessor):
def __init__(self, md, project_id, current_path, paths):
super().__init__(md)
self.project_id = project_id
self.current_path = current_path
self.paths = paths
def run(self, root):
for element in root.iter('a'):
href = element.get('href', '')
parts = urlsplit(href)
if parts.scheme or parts.netloc or not parts.path:
continue
target = posixpath.normpath(posixpath.join(posixpath.dirname(self.current_path), unquote(parts.path)))
if target.endswith('/'):
target += 'README.md'
if target in self.paths:
url = reverse('projects:docs_document', kwargs={'project_id': self.project_id, 'path': target})
element.set('href', url + (f'#{parts.fragment}' if parts.fragment else ''))
else:
element.attrib.pop('href', None)
class _DocumentLinkExtension(Extension):
def __init__(self, project_id, current_path, paths):
super().__init__()
self.project_id, self.current_path, self.paths = project_id, current_path, paths
def extendMarkdown(self, md):
md.treeprocessors.register(_DocumentLinks(md, self.project_id, self.current_path, self.paths), 'project_document_links', 12)
def render_document_markdown(value, project_id, current_path, paths):
doc_attributes = {**ATTRIBUTES, **{tag: {'id'} for tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6')}}
return _clean(value, ['extra', 'fenced_code', 'tables', 'sane_lists', 'pymdownx.tasklist', 'toc',
_DocumentLinkExtension(project_id, current_path, paths)], attributes=doc_attributes)
+213
View File
@@ -0,0 +1,213 @@
import hashlib
import json
import subprocess
import tempfile
from pathlib import Path
from datetime import timedelta
from unittest.mock import patch
from asgiref.sync import async_to_sync
from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError
from django.test import Client, TestCase, override_settings
from django.urls import reverse
from django.utils import timezone
from mcp.server.auth.provider import AccessToken
from mcp.server.mcpserver.exceptions import ToolError
from mcp_server.server import create_server
from .documentation import (_fetch_snapshot, _git_env, _repository_url, audit_status,
document_diff, document_history, historical_document,
normal_path, sync_project)
from .models import DocumentationSync, Membership, ProjectDocument
from .services import create_project
from .templatetags.project_markdown import render_document_markdown
class DocumentationTests(TestCase):
def setUp(self):
users = get_user_model()
self.owner = users.objects.create_user('docs-owner', password='password')
self.viewer = users.objects.create_user('docs-viewer', password='password')
self.outsider = users.objects.create_user('docs-outsider', password='password')
self.project = create_project(self.owner, name='Documentation test',
git_repository_url='https://git.jorendegraaf.eu/owner/project.git')
Membership.objects.create(project=self.project, user=self.viewer, role='viewer')
def snapshot(self, path='docs/guide.md', markdown='# Guide\n\n[Next](next.md)\n', commit='a' * 40):
item = ProjectDocument(project=self.project, path=path, title='Guide', headings=['Guide'],
markdown=markdown, content_hash=hashlib.sha256(markdown.encode()).hexdigest(), source_commit=commit)
return (self.project.git_repository_url, 'refs/heads/main', commit, timezone.now(), [item], {})
def import_snapshot(self):
with patch('projects.documentation._fetch_snapshot', return_value=self.snapshot()):
return sync_project(self.owner, self.project)
def test_sync_replaces_deleted_documents_and_retains_last_good_on_error(self):
state = self.import_snapshot()
self.assertEqual(ProjectDocument.objects.filter(project=self.project).count(), 1)
self.assertEqual(state.indexed_commit, 'a' * 40)
DocumentationSync.objects.filter(project=self.project).update(last_attempt_at=timezone.now() - timedelta(minutes=2))
with patch('projects.documentation._fetch_snapshot', return_value=self.snapshot('README.md', '# New\n', 'b' * 40)):
sync_project(self.owner, self.project, force=True)
self.assertFalse(ProjectDocument.objects.filter(project=self.project, path='docs/guide.md').exists())
self.assertTrue(ProjectDocument.objects.filter(project=self.project, path='README.md').exists())
DocumentationSync.objects.filter(project=self.project).update(last_attempt_at=timezone.now() - timedelta(minutes=2))
from .documentation import DocumentationError
with patch('projects.documentation._fetch_snapshot', side_effect=DocumentationError('fetch unavailable')):
state = sync_project(self.owner, self.project, force=True)
self.assertEqual(state.indexed_commit, 'b' * 40)
self.assertIn('fetch unavailable', state.last_error)
self.assertTrue(ProjectDocument.objects.filter(project=self.project, path='README.md').exists())
def test_permissions_views_and_refresh(self):
self.import_snapshot()
home = reverse('projects:docs_home', args=[self.project.pk])
detail = reverse('projects:docs_document', args=[self.project.pk, 'docs/guide.md'])
refresh = reverse('projects:docs_refresh', args=[self.project.pk])
self.assertEqual(self.client.get(home).status_code, 302)
self.client.force_login(self.outsider)
self.assertEqual(self.client.get(home).status_code, 404)
self.assertEqual(self.client.get(detail).status_code, 404)
self.client.force_login(self.viewer)
self.assertContains(self.client.get(home), 'docs/guide.md')
self.assertContains(self.client.get(detail), 'Guide')
self.assertEqual(self.client.post(refresh).status_code, 403)
self.client.force_login(self.owner)
self.assertEqual(self.client.get(reverse('projects:docs_document', args=[self.project.pk, 'docs/../secret.md'])).status_code, 404)
csrf_client = Client(enforce_csrf_checks=True, HTTP_HOST='localhost')
csrf_client.force_login(self.owner)
self.assertEqual(csrf_client.post(refresh).status_code, 403)
csrf_client.get(home)
self.assertEqual(csrf_client.post(refresh, HTTP_X_CSRFTOKEN=csrf_client.cookies['csrftoken'].value).status_code, 302)
def test_markdown_links_and_untrusted_html(self):
html = str(render_document_markdown('# Guide\n\n[Next](next.md) [Missing](absent.md)\n\n<script>bad()</script>',
self.project.pk, 'docs/guide.md', {'docs/guide.md', 'docs/next.md'}))
self.assertIn(f'/projects/{self.project.pk}/docs/docs/next.md/', html)
self.assertIn('<h1 id="guide">Guide</h1>', html)
self.assertNotIn('href="absent.md"', html)
self.assertNotIn('<script>', html)
for path in ('../secret.md', '/etc/passwd', 'docs//file.md', 'docs\\file.md'):
with self.assertRaises(ValidationError):
normal_path(path)
def test_mcp_list_search_read_and_isolation(self):
self.import_snapshot()
ProjectDocument.objects.create(project=self.project, path='README.md', title='Read me',
markdown='# Read me\n', headings=['Read me'], content_hash=hashlib.sha256(b'# Read me\n').hexdigest(),
source_commit='a' * 40)
server = create_server()
token = AccessToken(token='test', client_id='test', scopes=['projects:read'], subject=str(self.viewer.pk))
with patch('mcp_server.server.get_access_token', return_value=token):
listed = async_to_sync(server.call_tool)('docs_list', {'project': str(self.project.pk), 'limit': 1}).structured_content
self.assertEqual(listed['items'][0]['path'], 'README.md')
following = async_to_sync(server.call_tool)('docs_list', {'project': str(self.project.pk), 'limit': 1, 'cursor': listed['next_cursor']}).structured_content
self.assertEqual(following['items'][0]['path'], 'docs/guide.md')
found = async_to_sync(server.call_tool)('docs_search', {'query': 'Guide'}).structured_content
self.assertEqual(found['total'], 1)
self.assertNotIn('markdown', found['items'][0])
read = async_to_sync(server.call_tool)('docs_read', {'project': str(self.project.pk), 'path': 'docs/guide.md', 'start_line': 1, 'end_line': 1}).structured_content
self.assertEqual(read['markdown'], '# Guide\n')
self.assertTrue(read['truncated'])
with self.assertRaises(ToolError):
async_to_sync(server.call_tool)('docs_list', {'project': str(self.project.pk), 'cursor': 'stale:1'})
with self.assertRaises(ToolError):
async_to_sync(server.call_tool)('docs_read', {'project': str(self.project.pk), 'path': '../secret.md'})
denied = AccessToken(token='test', client_id='test', scopes=['projects:read'], subject=str(self.outsider.pk))
with patch('mcp_server.server.get_access_token', return_value=denied):
self.assertEqual(async_to_sync(server.call_tool)('docs_search', {'query': 'Guide'}).structured_content['total'], 0)
with self.assertRaises(ToolError):
async_to_sync(server.call_tool)('docs_read', {'project': str(self.project.pk), 'path': 'docs/guide.md'})
def test_url_allowlist_and_per_project_credentials(self):
valid_url = self.project.git_repository_url
self.assertEqual(_repository_url(self.project), valid_url)
for url in ('http://git.jorendegraaf.eu/x', 'https://evil.example/x',
'https://user:secret@git.jorendegraaf.eu/x', 'https://git.jorendegraaf.eu/a/../b'):
self.project.git_repository_url = url
from .documentation import DocumentationError
with self.assertRaises(DocumentationError):
_repository_url(self.project)
self.project.git_repository_url = 'https://user:secret@git.jorendegraaf.eu/x'
with self.assertRaises(ValidationError):
self.project.full_clean()
self.project.git_repository_url = valid_url
credentials = {str(self.project.pk): {'repository_url': valid_url,
'username': 'readonly', 'token': 'secret-token'}}
with override_settings(PROJECT_DOCS_CREDENTIALS_JSON=json.dumps(credentials)):
env = _git_env(self.project)
self.assertIn('Authorization: Basic ', env['GIT_CONFIG_VALUE_0'])
other = create_project(self.owner, name='Other')
self.assertNotIn('GIT_CONFIG_VALUE_0', _git_env(other))
def test_invalid_audit_state_and_concurrent_claim(self):
state = self.import_snapshot()
self.assertEqual(audit_status(state), 'Unknown / due')
state.audit_state = {'last_successful_audit': 'broken', 'last_audited_commit': 'a' * 40}
self.assertEqual(audit_status(state), 'Unknown / due')
DocumentationSync.objects.filter(pk=state.pk).update(
sync_token='held', sync_started_at=timezone.now(), last_attempt_at=timezone.now() - timedelta(hours=1))
with patch('projects.documentation._fetch_snapshot') as fetch:
sync_project(self.owner, self.project)
fetch.assert_not_called()
def test_symlink_document_is_not_indexed(self):
blob = 'b' * 40
listing = f'120000 blob {blob}\tdocs/link.md\0'.encode() + f'100644 blob {blob}\tdocs/real.md\0'.encode()
outputs = [f'ref: refs/heads/main\tHEAD\n{"a" * 40}\tHEAD\n'.encode(), b'',
('a' * 40 + '\n').encode(), b'2026-10-20T12:00:00+00:00\n', listing, b'# Real\n']
with tempfile.TemporaryDirectory() as root, override_settings(PROJECT_DOCS_MIRROR_ROOT=root):
(Path(root) / str(self.project.pk)).mkdir()
with patch('projects.documentation._git', side_effect=outputs) as git_call:
_, _, _, _, docs, _ = _fetch_snapshot(self.project)
self.assertEqual([item.path for item in docs], ['docs/real.md'])
self.assertEqual(sum(1 for call in git_call.call_args_list if 'cat-file' in call.args[0]), 1)
def test_retained_git_history_and_diff_are_scoped(self):
def git(*args, cwd=None):
return subprocess.check_output(['git', *args], cwd=cwd, stderr=subprocess.DEVNULL).decode().strip()
with tempfile.TemporaryDirectory() as root, override_settings(PROJECT_DOCS_MIRROR_ROOT=Path(root) / 'mirrors'):
work = Path(root) / 'work'
work.mkdir()
git('init', '-q', cwd=work)
git('config', 'user.name', 'Test', cwd=work)
git('config', 'user.email', 'test@example.com', cwd=work)
(work / 'docs').mkdir()
path = work / 'docs' / 'guide.md'
path.write_text('# Old\n')
git('add', '.', cwd=work)
git('commit', '-qm', 'first', cwd=work)
first = git('rev-parse', 'HEAD', cwd=work)
path.write_text('# New\n')
git('commit', '-qam', 'second', cwd=work)
second = git('rev-parse', 'HEAD', cwd=work)
mirror = Path(root) / 'mirrors' / str(self.project.pk)
mirror.parent.mkdir()
git('clone', '--bare', '-q', str(work), str(mirror))
DocumentationSync.objects.create(project=self.project, indexed_commit=second,
indexed_repository_url=self.project.git_repository_url)
ProjectDocument.objects.create(project=self.project, path='docs/guide.md', title='New',
markdown='# New\n', headings=['New'], content_hash=hashlib.sha256(b'# New\n').hexdigest(),
source_commit=second)
history = document_history(self.viewer, self.project, 'docs/guide.md')
self.assertEqual([item['commit'] for item in history], [second, first])
old = historical_document(self.viewer, self.project, 'docs/guide.md', first)
self.assertEqual(old.markdown, '# Old\n')
diff, truncated = document_diff(self.viewer, self.project, 'docs/guide.md', first, second)
self.assertIn('-# Old', diff)
self.assertIn('+# New', diff)
self.assertFalse(truncated)
from django.http import Http404
with self.assertRaises(Http404):
document_history(self.outsider, self.project, 'docs/guide.md')
def test_project_delete_cleans_private_mirror_after_commit(self):
with tempfile.TemporaryDirectory() as root, override_settings(PROJECT_DOCS_MIRROR_ROOT=root):
mirror = Path(root) / str(self.project.pk)
mirror.mkdir()
(mirror / 'marker').write_text('private')
with self.captureOnCommitCallbacks(execute=True):
self.project.delete()
self.assertFalse(mirror.exists())
+3
View File
@@ -9,6 +9,9 @@ urlpatterns = [
path('profile/', views.profile, name='profile'),
path('profile/connections/<int:grant_id>/revoke/', views.revoke_connection, name='revoke_connection'),
path('<uuid:project_id>/', views.board, name='board'),
path('<uuid:project_id>/docs/', views.docs_home, name='docs_home'),
path('<uuid:project_id>/docs/refresh/', views.docs_refresh, name='docs_refresh'),
path('<uuid:project_id>/docs/<path:path>/', views.docs_document, name='docs_document'),
path('<uuid:project_id>/edit/', views.project_edit, name='edit'),
path('<uuid:project_id>/archive/', views.project_archive, name='archive'),
path('<uuid:project_id>/restore/', views.project_restore, name='restore'),
+82
View File
@@ -4,6 +4,7 @@ from django.contrib.auth.views import LoginView
from django.core.cache import cache
from django.core import signing
from django.http import HttpResponse, FileResponse
from django.http import Http404
import hashlib
import json
import uuid
@@ -22,7 +23,11 @@ from .forms import CardForm, ColumnForm, ProfileForm, ProjectForm, TagForm
from .models import Activity, Card, CardComment, CardAttachment, Column, Profile, Membership, Tag
from . import services
from . import attachments
from . import documentation
from .templatetags.project_markdown import safe_markdown
from .templatetags.project_markdown import render_document_markdown
from .models import DocumentationSync, ProjectDocument
from urllib.parse import quote
class ThrottledLoginView(LoginView):
@@ -303,6 +308,83 @@ def markdown_preview(request):
return HttpResponse(safe_markdown(request.POST.get('description_markdown', '')))
def _documentation_context(request, project):
state = documentation.maybe_sync(request.user, project)
can_edit = project.owner_id == request.user.pk or Membership.objects.filter(project=project, user=request.user, role='editor').exists()
return {'project': project, 'sync': state, 'sync_status': documentation.sync_status(project, state),
'can_edit': can_edit, 'audit_label': documentation.audit_status(state),
'section': 'projects', 'title': f'{project.name} documentation'}
@login_required
def docs_home(request, project_id):
project = services.project_for(request.user, project_id)
context = _documentation_context(request, project)
query = request.GET.get('q', '').strip()[:200]
prefix = request.GET.get('path', '').strip()
if prefix:
try:
documentation.normal_path(prefix)
except ValidationError as exc:
raise Http404('Folder unavailable') from exc
page = max(1, int(request.GET.get('page', '1'))) if request.GET.get('page', '1').isdigit() else 1
if query:
documents, total = documentation.search_documents(request.user, query, project=project, limit=30, offset=(page - 1) * 30)
else:
documents, total = documentation.list_documents(request.user, project, path=prefix, limit=30, offset=(page - 1) * 30)
paths = ProjectDocument.objects.filter(project=project).values_list('path', flat=True)
folders = sorted({part for path in paths for part in ('/'.join(path.split('/')[:i]) for i in range(1, len(path.split('/'))))})
folder_tree = []
folder_nodes = {}
for folder in folders:
node = {'name': folder.rsplit('/', 1)[-1], 'path': folder, 'children': []}
folder_nodes[folder] = node
parent = folder.rpartition('/')[0]
(folder_nodes[parent]['children'] if parent else folder_tree).append(node)
context.update({'documents': documents, 'query': query, 'page': page, 'has_next': page * 30 < total,
'has_previous': page > 1, 'total': total, 'prefix': prefix, 'folder_tree': folder_tree,
'results': [{'document': item, 'snippet': documentation.excerpt(item, query)} for item in documents] if query else []})
return render(request, 'projects/docs_home.html', context)
@login_required
def docs_document(request, project_id, path):
project = services.project_for(request.user, project_id)
context = _documentation_context(request, project)
try:
current = documentation.read_document(request.user, project, path)
revision = request.GET.get('revision', '')
document = documentation.historical_document(request.user, project, path, revision) if revision else current
except (ProjectDocument.DoesNotExist, ValidationError, documentation.DocumentationError) as exc:
raise Http404('Document unavailable') from exc
paths = set(ProjectDocument.objects.filter(project=project).values_list('path', flat=True))
context['document'] = document
context['file_name'] = path.rsplit('/', 1)[-1]
context['breadcrumbs'] = [{'name': part, 'path': '/'.join(path.split('/')[:index + 1])}
for index, part in enumerate(path.split('/')[:-1])]
context['is_historical'] = document.source_commit != current.source_commit
try:
context['history'] = documentation.document_history(request.user, project, path)
except documentation.DocumentationError:
context['history'] = []
context['document_commit_time'] = next((item['committed_at'] for item in context['history']
if item['commit'] == document.source_commit),
context['sync'].commit_time if context['sync'] and not context['is_historical'] else None)
context['rendered'] = render_document_markdown(document.markdown, project.pk, document.path, paths)
if context['sync'] and context['sync'].indexed_repository_url:
base = context['sync'].indexed_repository_url.rstrip('/').removesuffix('.git')
context['source_url'] = f'{base}/src/commit/{document.source_commit}/{quote(document.path)}'
return render(request, 'projects/docs_document.html', context)
@login_required
@require_POST
def docs_refresh(request, project_id):
project = services.project_for(request.user, project_id, write=True)
documentation.sync_project(request.user, project, force=True)
return redirect('projects:docs_home', project_id=project.pk)
@login_required
@require_POST
def card_move(request, card_id):