464 lines
22 KiB
Python
464 lines
22 KiB
Python
"""Read-only, project-scoped Git documentation index."""
|
||
|
||
import base64
|
||
import difflib
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import posixpath
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import tempfile
|
||
from time import monotonic
|
||
import uuid
|
||
from datetime import datetime, timedelta
|
||
from pathlib import Path
|
||
from urllib.parse import urlsplit
|
||
|
||
from django.conf import settings
|
||
from django.core.exceptions import ValidationError
|
||
from django.db import transaction
|
||
from django.db.models import Q
|
||
from django.utils import timezone
|
||
|
||
from . import services
|
||
from .models import DocumentationSync, ProjectDocument
|
||
|
||
|
||
MAX_RESULTS = 100
|
||
MAX_READ_LINES = 250
|
||
MAX_READ_CHARS = 30000
|
||
REF_RE = re.compile(r'^refs/heads/[A-Za-z0-9][A-Za-z0-9._/-]*$')
|
||
HASH_RE = re.compile(r'^[0-9a-f]{40}([0-9a-f]{24})?$')
|
||
HEADING_RE = re.compile(r'^#{1,6}\s+(.+?)\s*#*\s*$')
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class DocumentationError(Exception):
|
||
pass
|
||
|
||
|
||
def normal_path(value):
|
||
if not isinstance(value, str) or not value or len(value) > 500 or '\x00' in value:
|
||
raise ValidationError('Invalid document path.')
|
||
if value.startswith('/') or '\\' in value or any(part in ('', '.', '..') for part in value.split('/')):
|
||
raise ValidationError('Invalid document path.')
|
||
if any(ord(c) < 32 for c in value):
|
||
raise ValidationError('Invalid document path.')
|
||
return value
|
||
|
||
|
||
def indexed_path(value):
|
||
try:
|
||
normal_path(value)
|
||
except ValidationError:
|
||
return False
|
||
return value in ('README.md', 'AGENTS.md') or (value.startswith('docs/') and value.lower().endswith('.md'))
|
||
|
||
|
||
def _repository_url(project):
|
||
raw = project.git_repository_url
|
||
try:
|
||
parsed = urlsplit(raw)
|
||
hostname = parsed.hostname
|
||
port = parsed.port
|
||
except ValueError as exc:
|
||
raise DocumentationError('Repository URL is malformed.') from exc
|
||
allowed = {item.strip().lower() for item in settings.PROJECT_DOCS_ALLOWED_GIT_HOSTS.split(',') if item.strip()}
|
||
authority = hostname.lower() if hostname else ''
|
||
if port:
|
||
authority += f':{port}'
|
||
if (parsed.scheme != 'https' or not authority or authority not in allowed or
|
||
parsed.username or parsed.password or parsed.query or parsed.fragment or
|
||
not parsed.path.startswith('/') or '..' in parsed.path.split('/') or
|
||
'%' in raw or '\\' in raw or any(c.isspace() for c in raw)):
|
||
raise DocumentationError('Repository URL is not an approved HTTPS Git endpoint.')
|
||
return raw
|
||
|
||
|
||
def _git_env(project):
|
||
env = {key: value for key, value in os.environ.items()
|
||
if not key.startswith('GIT_') and key not in ('SSH_ASKPASS', 'SSH_ASKPASS_REQUIRE')}
|
||
env.update({'GIT_TERMINAL_PROMPT': '0', 'GIT_CONFIG_NOSYSTEM': '1',
|
||
'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_SYSTEM': os.devnull,
|
||
'GIT_NO_REPLACE_OBJECTS': '1'})
|
||
try:
|
||
secrets = json.loads(settings.PROJECT_DOCS_CREDENTIALS_JSON or '{}')
|
||
except (TypeError, ValueError) as exc:
|
||
raise DocumentationError('Git credentials are not configured correctly.') from exc
|
||
if not isinstance(secrets, dict):
|
||
raise DocumentationError('Git credentials are not configured correctly.')
|
||
entry = secrets.get(str(project.pk), {})
|
||
if not isinstance(entry, dict):
|
||
raise DocumentationError('Project Git credentials are not configured correctly.')
|
||
if entry:
|
||
username, token = entry.get('username'), entry.get('token')
|
||
if entry.get('repository_url') != _repository_url(project):
|
||
raise DocumentationError('Project Git credential is not bound to this repository URL.')
|
||
if not isinstance(username, str) or not isinstance(token, str) or not username or not token or '\n' in username + token or '\r' in username + token:
|
||
raise DocumentationError('Project Git credentials are incomplete.')
|
||
encoded = base64.b64encode(f'{username}:{token}'.encode()).decode()
|
||
env['GIT_CONFIG_COUNT'] = '1'
|
||
env['GIT_CONFIG_KEY_0'] = f'http.{_repository_url(project)}.extraHeader'
|
||
env['GIT_CONFIG_VALUE_0'] = f'Authorization: Basic {encoded}'
|
||
return env
|
||
|
||
|
||
def _git(args, *, env, timeout=30, max_output=20 * 1024 * 1024):
|
||
limiter = shutil.which('prlimit')
|
||
if not limiter:
|
||
raise DocumentationError('Git resource limiter is unavailable on this server.')
|
||
command = [limiter, f'--fsize={settings.PROJECT_DOCS_MAX_MIRROR_BYTES}', '--cpu=65', '--',
|
||
'git', '-c', 'credential.helper=', '-c', 'core.hooksPath=/dev/null',
|
||
'-c', 'protocol.file.allow=never', '-c', 'protocol.ext.allow=never',
|
||
'-c', 'http.followRedirects=false', '-c', 'transfer.fsckObjects=true',
|
||
'-c', 'gc.auto=0', *args]
|
||
try:
|
||
with tempfile.TemporaryFile() as output:
|
||
result = subprocess.run(command, env=env, stdout=output,
|
||
stderr=subprocess.DEVNULL, timeout=timeout, check=False)
|
||
if result.returncode or output.tell() > max_output:
|
||
raise DocumentationError('Git fetch or index failed; check repository access and size limits.')
|
||
output.seek(0)
|
||
return output.read(max_output + 1)
|
||
except (OSError, subprocess.TimeoutExpired, subprocess.SubprocessError) as exc:
|
||
raise DocumentationError('Git fetch or index timed out or could not start.') from exc
|
||
|
||
|
||
def _mirror_path(project, *, create=True):
|
||
root = Path(settings.PROJECT_DOCS_MIRROR_ROOT)
|
||
if create:
|
||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||
path = root / str(project.pk)
|
||
if path.is_symlink() or (path.exists() and not path.is_dir()):
|
||
raise DocumentationError('Repository storage is unavailable.')
|
||
return path
|
||
|
||
|
||
def _mirror_size(path):
|
||
return sum(item.stat().st_size for item in path.rglob('*') if item.is_file() and not item.is_symlink())
|
||
|
||
|
||
def _fetch_snapshot(project, observed_head=None):
|
||
url = _repository_url(project)
|
||
env = _git_env(project)
|
||
deadline = monotonic() + 90
|
||
|
||
def run_git(args, *, timeout=30, max_output=20 * 1024 * 1024):
|
||
remaining = deadline - monotonic()
|
||
if remaining <= 0:
|
||
raise DocumentationError('Git documentation refresh exceeded its time limit.')
|
||
return _git(args, env=env, timeout=min(timeout, remaining), max_output=max_output)
|
||
|
||
path = _mirror_path(project)
|
||
if not path.exists():
|
||
run_git(['init', '--bare', str(path)])
|
||
path.chmod(0o700)
|
||
if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES:
|
||
raise DocumentationError('Repository mirror exceeds the configured size limit.')
|
||
remote = run_git(['ls-remote', '--symref', url, 'HEAD'], max_output=4096).decode('ascii', 'replace')
|
||
match = re.search(r'^ref: (refs/heads/\S+)\s+HEAD$', remote, re.M)
|
||
if not match or not REF_RE.fullmatch(match.group(1)) or '..' in match.group(1) or '@{' in match.group(1):
|
||
raise DocumentationError('Repository does not advertise a safe default branch.')
|
||
branch = match.group(1)
|
||
head_match = re.search(r'^([0-9a-f]{40}(?:[0-9a-f]{24})?)\s+HEAD$', remote, re.M)
|
||
if not head_match:
|
||
raise DocumentationError('Repository did not provide a valid HEAD commit.')
|
||
if observed_head:
|
||
observed_head(branch, head_match.group(1))
|
||
run_git(['--git-dir', str(path), 'fetch', '--no-tags', '--depth=50', url,
|
||
f'+{branch}:refs/remotes/origin/{branch.removeprefix("refs/heads/")}'], timeout=60, max_output=1024)
|
||
if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES:
|
||
raise DocumentationError('Repository mirror exceeds the configured size limit.')
|
||
commit = run_git(['--git-dir', str(path), 'rev-parse', f'refs/remotes/origin/{branch.removeprefix("refs/heads/")}'],
|
||
max_output=128).decode().strip()
|
||
if not HASH_RE.fullmatch(commit):
|
||
raise DocumentationError('Repository returned an invalid commit.')
|
||
committed_at = run_git(['--git-dir', str(path), 'show', '-s', '--format=%cI', commit], max_output=128).decode().strip()
|
||
try:
|
||
commit_time = datetime.fromisoformat(committed_at)
|
||
if commit_time.tzinfo is None:
|
||
raise ValueError
|
||
except ValueError as exc:
|
||
raise DocumentationError('Repository returned an invalid commit time.') from exc
|
||
listing = run_git(['--git-dir', str(path), 'ls-tree', '-r', '-z', commit],
|
||
max_output=settings.PROJECT_DOCS_MAX_TREE_BYTES)
|
||
rows = []
|
||
audit_blob = None
|
||
total = 0
|
||
for raw in listing.split(b'\x00'):
|
||
if not raw:
|
||
continue
|
||
try:
|
||
metadata, name = raw.split(b'\t', 1)
|
||
mode, kind, blob = metadata.decode('ascii').split(' ')
|
||
name = name.decode('utf-8')
|
||
except (ValueError, UnicodeDecodeError) as exc:
|
||
if b'\tdocs/' in raw or raw.endswith((b'\tREADME.md', b'\tAGENTS.md')):
|
||
raise DocumentationError('Repository tree contains an unsupported documentation path.') from exc
|
||
continue
|
||
if name == 'docs/.audit-state.json' and mode == '100644' and kind == 'blob':
|
||
audit_blob = blob
|
||
if not indexed_path(name):
|
||
continue
|
||
if mode != '100644' or kind != 'blob' or not HASH_RE.fullmatch(blob):
|
||
continue
|
||
rows.append((name, blob))
|
||
if len(rows) > settings.PROJECT_DOCS_MAX_FILES:
|
||
raise DocumentationError('Repository has too many documentation files.')
|
||
documents = []
|
||
for name, blob in rows:
|
||
content = run_git(['--git-dir', str(path), 'cat-file', 'blob', blob],
|
||
max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES)
|
||
total += len(content)
|
||
if total > settings.PROJECT_DOCS_MAX_CONTENT_BYTES:
|
||
raise DocumentationError('Documentation exceeds the configured content limit.')
|
||
try:
|
||
markdown = content.decode('utf-8')
|
||
except UnicodeDecodeError as exc:
|
||
raise DocumentationError(f'Document {name} is not UTF-8.') from exc
|
||
headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))]
|
||
documents.append(ProjectDocument(project=project, path=name,
|
||
title=(headings[0] if headings else posixpath.basename(name))[:255], headings=headings[:100],
|
||
markdown=markdown, content_hash=hashlib.sha256(content).hexdigest(), source_commit=commit))
|
||
audit = {}
|
||
if audit_blob and HASH_RE.fullmatch(audit_blob):
|
||
raw = run_git(['--git-dir', str(path), 'cat-file', 'blob', audit_blob], max_output=65536)
|
||
try:
|
||
candidate = json.loads(raw)
|
||
if isinstance(candidate, dict) and candidate.get('schema_version') == 1:
|
||
limits = {'last_successful_audit': 40, 'last_audited_commit': 64,
|
||
'last_report_path': 300, 'last_attempt': 40, 'last_status': 40}
|
||
audit = {key: candidate[key] for key, size in limits.items()
|
||
if isinstance(candidate.get(key), str) and len(candidate[key]) <= size}
|
||
if isinstance(candidate.get('audit_interval_days'), int):
|
||
audit['audit_interval_days'] = candidate['audit_interval_days']
|
||
except (ValueError, UnicodeDecodeError):
|
||
pass
|
||
return url, branch, commit, commit_time, documents, audit
|
||
|
||
|
||
def sync_project(user, project, *, force=False):
|
||
services.project_for(user, project.pk, write=force)
|
||
if not project.git_repository_url:
|
||
raise DocumentationError('Configure a Git repository URL before refreshing documentation.')
|
||
state, _ = DocumentationSync.objects.get_or_create(project=project)
|
||
now = timezone.now()
|
||
interval = timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS))
|
||
if not force and state.last_attempt_at and now - state.last_attempt_at < interval:
|
||
return state
|
||
if force and state.last_attempt_at and now - state.last_attempt_at < timedelta(seconds=30):
|
||
return state
|
||
token = str(uuid.uuid4())
|
||
claimed = DocumentationSync.objects.filter(pk=state.pk).filter(
|
||
Q(sync_token='') | Q(sync_started_at__lt=now - timedelta(minutes=3))).update(
|
||
sync_token=token, sync_started_at=now, last_attempt_at=now)
|
||
if not claimed:
|
||
return DocumentationSync.objects.get(pk=state.pk)
|
||
try:
|
||
def record_head(branch, head):
|
||
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(branch=branch, remote_head=head)
|
||
|
||
url, branch, commit, commit_time, documents, audit = _fetch_snapshot(project, observed_head=record_head)
|
||
with transaction.atomic():
|
||
state = DocumentationSync.objects.select_for_update().get(pk=state.pk)
|
||
if state.sync_token != token:
|
||
return state
|
||
if state.indexed_commit != commit or state.indexed_repository_url != url:
|
||
ProjectDocument.objects.filter(project=project).delete()
|
||
ProjectDocument.objects.bulk_create(documents, batch_size=100)
|
||
state.indexed_commit = commit
|
||
state.remote_head = commit
|
||
state.indexed_repository_url = url
|
||
state.branch = branch
|
||
state.commit_time = commit_time
|
||
state.last_success_at = timezone.now()
|
||
state.last_error = ''
|
||
state.sync_token = ''
|
||
state.sync_started_at = None
|
||
state.audit_state = audit
|
||
state.save()
|
||
except DocumentationError as exc:
|
||
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(
|
||
last_error=str(exc)[:300], sync_token='', sync_started_at=None)
|
||
except Exception as exc:
|
||
logger.error('Documentation indexing failed for project %s (%s).', project.pk, type(exc).__name__)
|
||
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(
|
||
last_error='Documentation indexing failed; retry after checking server logs.',
|
||
sync_token='', sync_started_at=None)
|
||
return DocumentationSync.objects.get(pk=state.pk)
|
||
|
||
|
||
def maybe_sync(user, project):
|
||
"""On-demand polling: one bounded refresh per interval for authorized readers."""
|
||
services.project_for(user, project.pk)
|
||
if project.git_repository_url:
|
||
return sync_project(user, project)
|
||
return DocumentationSync.objects.filter(project=project).first()
|
||
|
||
|
||
def sync_status(project, state):
|
||
if not project.git_repository_url:
|
||
return 'No repository configured'
|
||
if not state or not state.indexed_commit:
|
||
if state and state.sync_token:
|
||
return 'Refreshing from Git; no indexed documentation yet'
|
||
return 'No indexed documentation' if not state or not state.last_error else f'Sync error: {state.last_error}'
|
||
if state.sync_token:
|
||
return 'Refreshing from Git; showing the last indexed version'
|
||
if state.last_error:
|
||
return f'Stale after sync error: {state.last_error}'
|
||
if state.indexed_repository_url != project.git_repository_url:
|
||
return 'Stale: repository URL changed'
|
||
if state.remote_head and state.remote_head != state.indexed_commit:
|
||
return 'Stale: remote HEAD differs from indexed commit'
|
||
if state.last_success_at and timezone.now() - state.last_success_at >= timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS)):
|
||
return 'Refresh due; indexed commit may lag remote'
|
||
return 'Indexed from Git; document accuracy is not guaranteed'
|
||
|
||
|
||
def audit_status(state):
|
||
audit = state.audit_state if state else {}
|
||
date_value = audit.get('last_successful_audit')
|
||
commit = audit.get('last_audited_commit')
|
||
partial = audit.get('last_status') in ('partial', 'error')
|
||
try:
|
||
last = datetime.fromisoformat(date_value.replace('Z', '+00:00'))
|
||
days = int(audit.get('audit_interval_days', 14))
|
||
if last.tzinfo is None or not HASH_RE.fullmatch(commit) or not 1 <= days <= 365:
|
||
raise ValueError
|
||
except (AttributeError, TypeError, ValueError):
|
||
return 'Partial or failed / due' if partial else 'Unknown / due'
|
||
if last > timezone.now() + timedelta(minutes=5):
|
||
return 'Unknown / due (audit timestamp is in the future)'
|
||
if timezone.now() >= last + timedelta(days=days):
|
||
return f'Overdue (last successful audit {last.date()})' + ('; latest attempt partial or failed' if partial else '')
|
||
if state.indexed_commit != commit:
|
||
return f'Last audited {last.date()}; indexed commit differs' + ('; latest attempt partial or failed' if partial else '')
|
||
return f'Last audited {last.date()} at this commit' + ('; latest attempt partial or failed' if partial else '')
|
||
|
||
|
||
def list_documents(user, project, *, path='', limit=50, offset=0):
|
||
services.project_for(user, project.pk)
|
||
if path:
|
||
normal_path(path)
|
||
size = min(max(int(limit), 1), MAX_RESULTS)
|
||
offset = max(int(offset), 0)
|
||
qs = ProjectDocument.objects.filter(project=project)
|
||
if path:
|
||
qs = qs.filter(path__startswith=path.rstrip('/') + '/')
|
||
qs = qs.order_by('path')
|
||
return list(qs[offset:offset + size]), qs.count()
|
||
|
||
|
||
def search_documents(user, query, *, project=None, limit=30, offset=0):
|
||
query = query.strip() if isinstance(query, str) else ''
|
||
if not query or len(query) > 200:
|
||
raise ValidationError('Search query must be 1–200 characters.')
|
||
qs = ProjectDocument.objects.select_related('project').filter(project__in=services.accessible(user))
|
||
if project:
|
||
services.project_for(user, project.pk)
|
||
qs = qs.filter(project=project)
|
||
qs = qs.filter(Q(path__icontains=query) | Q(title__icontains=query) | Q(markdown__icontains=query)).order_by('project_id', 'path')
|
||
size = min(max(int(limit), 1), MAX_RESULTS)
|
||
offset = max(int(offset), 0)
|
||
return list(qs[offset:offset + size]), qs.count()
|
||
|
||
|
||
def read_document(user, project, path):
|
||
services.project_for(user, project.pk)
|
||
return ProjectDocument.objects.get(project=project, path=normal_path(path))
|
||
|
||
|
||
def excerpt(document, query, radius=120):
|
||
content = document.markdown
|
||
at = content.lower().find(query.lower())
|
||
if at < 0:
|
||
at = 0
|
||
start = max(0, at - radius)
|
||
end = min(len(content), at + len(query) + radius)
|
||
return ('…' if start else '') + content[start:end].replace('\n', ' ') + ('…' if end < len(content) else '')
|
||
|
||
|
||
def line_window(document, start_line=1, end_line=None):
|
||
lines = document.markdown.splitlines(keepends=True)
|
||
start = max(int(start_line), 1)
|
||
if start > len(lines) + 1:
|
||
raise ValidationError('Start line is beyond the document.')
|
||
requested_end = int(end_line) if end_line is not None else start + MAX_READ_LINES - 1
|
||
if requested_end < start:
|
||
raise ValidationError('Invalid line range.')
|
||
end = min(requested_end, start + MAX_READ_LINES - 1, len(lines))
|
||
output = ''
|
||
partial_line = False
|
||
actual_end = start - 1
|
||
for number in range(start, end + 1):
|
||
line = lines[number - 1]
|
||
remaining = MAX_READ_CHARS - len(output)
|
||
if remaining <= 0:
|
||
break
|
||
if len(line) > remaining:
|
||
output += line[:remaining]
|
||
partial_line = True
|
||
actual_end = number
|
||
break
|
||
output += line
|
||
actual_end = number
|
||
return {'markdown': output, 'start_line': start, 'end_line': actual_end,
|
||
'total_lines': len(lines), 'truncated': actual_end < len(lines) or partial_line,
|
||
'partial_line': partial_line}
|
||
|
||
|
||
def document_history(user, project, path, limit=20):
|
||
services.project_for(user, project.pk)
|
||
normal_path(path)
|
||
state = DocumentationSync.objects.filter(project=project).first()
|
||
if not state or not state.indexed_commit:
|
||
return []
|
||
mirror = _mirror_path(project, create=False)
|
||
if not mirror.exists():
|
||
return []
|
||
output = _git(['--git-dir', str(mirror), 'log', f'--max-count={min(max(int(limit), 1), 50)}',
|
||
'--format=%H%x09%cI', state.indexed_commit, '--', path],
|
||
env=_git_env(project), max_output=8192)
|
||
rows = []
|
||
for line in output.decode('ascii', 'replace').splitlines():
|
||
try:
|
||
commit, committed_at = line.split('\t', 1)
|
||
except ValueError:
|
||
continue
|
||
if HASH_RE.fullmatch(commit):
|
||
rows.append({'commit': commit, 'committed_at': committed_at})
|
||
return rows
|
||
|
||
|
||
def historical_document(user, project, path, revision):
|
||
current = read_document(user, project, path)
|
||
if revision == current.source_commit:
|
||
return current
|
||
if not isinstance(revision, str) or not HASH_RE.fullmatch(revision):
|
||
raise ValidationError('Invalid Git revision.')
|
||
if revision not in {row['commit'] for row in document_history(user, project, path, limit=50)}:
|
||
raise ValidationError('Revision is unavailable in the retained Git history.')
|
||
mirror = _mirror_path(project, create=False)
|
||
content = _git(['--git-dir', str(mirror), 'show', f'{revision}:{path}'],
|
||
env=_git_env(project), max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES)
|
||
try:
|
||
markdown = content.decode('utf-8')
|
||
except UnicodeDecodeError as exc:
|
||
raise DocumentationError('Historical document is not UTF-8.') from exc
|
||
headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))]
|
||
return ProjectDocument(project=project, path=path, title=(headings[0] if headings else posixpath.basename(path))[:255],
|
||
headings=headings[:100], markdown=markdown,
|
||
content_hash=hashlib.sha256(content).hexdigest(), source_commit=revision)
|
||
|
||
|
||
def document_diff(user, project, path, before, after):
|
||
first = historical_document(user, project, path, before)
|
||
second = historical_document(user, project, path, after)
|
||
diff = ''.join(difflib.unified_diff(first.markdown.splitlines(keepends=True), second.markdown.splitlines(keepends=True),
|
||
fromfile=before, tofile=after))
|
||
return diff[:MAX_READ_CHARS], len(diff) > MAX_READ_CHARS
|