Files

464 lines
22 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Read-only, project-scoped Git documentation index."""
import base64
import difflib
import hashlib
import json
import logging
import os
import posixpath
import re
import shutil
import subprocess
import tempfile
from time import monotonic
import uuid
from datetime import datetime, timedelta
from pathlib import Path
from urllib.parse import urlsplit
from django.conf import settings
from django.core.exceptions import ValidationError
from django.db import transaction
from django.db.models import Q
from django.utils import timezone
from . import services
from .models import DocumentationSync, ProjectDocument
MAX_RESULTS = 100
MAX_READ_LINES = 250
MAX_READ_CHARS = 30000
REF_RE = re.compile(r'^refs/heads/[A-Za-z0-9][A-Za-z0-9._/-]*$')
HASH_RE = re.compile(r'^[0-9a-f]{40}([0-9a-f]{24})?$')
HEADING_RE = re.compile(r'^#{1,6}\s+(.+?)\s*#*\s*$')
logger = logging.getLogger(__name__)
class DocumentationError(Exception):
pass
def normal_path(value):
if not isinstance(value, str) or not value or len(value) > 500 or '\x00' in value:
raise ValidationError('Invalid document path.')
if value.startswith('/') or '\\' in value or any(part in ('', '.', '..') for part in value.split('/')):
raise ValidationError('Invalid document path.')
if any(ord(c) < 32 for c in value):
raise ValidationError('Invalid document path.')
return value
def indexed_path(value):
try:
normal_path(value)
except ValidationError:
return False
return value in ('README.md', 'AGENTS.md') or (value.startswith('docs/') and value.lower().endswith('.md'))
def _repository_url(project):
raw = project.git_repository_url
try:
parsed = urlsplit(raw)
hostname = parsed.hostname
port = parsed.port
except ValueError as exc:
raise DocumentationError('Repository URL is malformed.') from exc
allowed = {item.strip().lower() for item in settings.PROJECT_DOCS_ALLOWED_GIT_HOSTS.split(',') if item.strip()}
authority = hostname.lower() if hostname else ''
if port:
authority += f':{port}'
if (parsed.scheme != 'https' or not authority or authority not in allowed or
parsed.username or parsed.password or parsed.query or parsed.fragment or
not parsed.path.startswith('/') or '..' in parsed.path.split('/') or
'%' in raw or '\\' in raw or any(c.isspace() for c in raw)):
raise DocumentationError('Repository URL is not an approved HTTPS Git endpoint.')
return raw
def _git_env(project):
env = {key: value for key, value in os.environ.items()
if not key.startswith('GIT_') and key not in ('SSH_ASKPASS', 'SSH_ASKPASS_REQUIRE')}
env.update({'GIT_TERMINAL_PROMPT': '0', 'GIT_CONFIG_NOSYSTEM': '1',
'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_SYSTEM': os.devnull,
'GIT_NO_REPLACE_OBJECTS': '1'})
try:
secrets = json.loads(settings.PROJECT_DOCS_CREDENTIALS_JSON or '{}')
except (TypeError, ValueError) as exc:
raise DocumentationError('Git credentials are not configured correctly.') from exc
if not isinstance(secrets, dict):
raise DocumentationError('Git credentials are not configured correctly.')
entry = secrets.get(str(project.pk), {})
if not isinstance(entry, dict):
raise DocumentationError('Project Git credentials are not configured correctly.')
if entry:
username, token = entry.get('username'), entry.get('token')
if entry.get('repository_url') != _repository_url(project):
raise DocumentationError('Project Git credential is not bound to this repository URL.')
if not isinstance(username, str) or not isinstance(token, str) or not username or not token or '\n' in username + token or '\r' in username + token:
raise DocumentationError('Project Git credentials are incomplete.')
encoded = base64.b64encode(f'{username}:{token}'.encode()).decode()
env['GIT_CONFIG_COUNT'] = '1'
env['GIT_CONFIG_KEY_0'] = f'http.{_repository_url(project)}.extraHeader'
env['GIT_CONFIG_VALUE_0'] = f'Authorization: Basic {encoded}'
return env
def _git(args, *, env, timeout=30, max_output=20 * 1024 * 1024):
limiter = shutil.which('prlimit')
if not limiter:
raise DocumentationError('Git resource limiter is unavailable on this server.')
command = [limiter, f'--fsize={settings.PROJECT_DOCS_MAX_MIRROR_BYTES}', '--cpu=65', '--',
'git', '-c', 'credential.helper=', '-c', 'core.hooksPath=/dev/null',
'-c', 'protocol.file.allow=never', '-c', 'protocol.ext.allow=never',
'-c', 'http.followRedirects=false', '-c', 'transfer.fsckObjects=true',
'-c', 'gc.auto=0', *args]
try:
with tempfile.TemporaryFile() as output:
result = subprocess.run(command, env=env, stdout=output,
stderr=subprocess.DEVNULL, timeout=timeout, check=False)
if result.returncode or output.tell() > max_output:
raise DocumentationError('Git fetch or index failed; check repository access and size limits.')
output.seek(0)
return output.read(max_output + 1)
except (OSError, subprocess.TimeoutExpired, subprocess.SubprocessError) as exc:
raise DocumentationError('Git fetch or index timed out or could not start.') from exc
def _mirror_path(project, *, create=True):
root = Path(settings.PROJECT_DOCS_MIRROR_ROOT)
if create:
root.mkdir(mode=0o700, parents=True, exist_ok=True)
path = root / str(project.pk)
if path.is_symlink() or (path.exists() and not path.is_dir()):
raise DocumentationError('Repository storage is unavailable.')
return path
def _mirror_size(path):
return sum(item.stat().st_size for item in path.rglob('*') if item.is_file() and not item.is_symlink())
def _fetch_snapshot(project, observed_head=None):
url = _repository_url(project)
env = _git_env(project)
deadline = monotonic() + 90
def run_git(args, *, timeout=30, max_output=20 * 1024 * 1024):
remaining = deadline - monotonic()
if remaining <= 0:
raise DocumentationError('Git documentation refresh exceeded its time limit.')
return _git(args, env=env, timeout=min(timeout, remaining), max_output=max_output)
path = _mirror_path(project)
if not path.exists():
run_git(['init', '--bare', str(path)])
path.chmod(0o700)
if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES:
raise DocumentationError('Repository mirror exceeds the configured size limit.')
remote = run_git(['ls-remote', '--symref', url, 'HEAD'], max_output=4096).decode('ascii', 'replace')
match = re.search(r'^ref: (refs/heads/\S+)\s+HEAD$', remote, re.M)
if not match or not REF_RE.fullmatch(match.group(1)) or '..' in match.group(1) or '@{' in match.group(1):
raise DocumentationError('Repository does not advertise a safe default branch.')
branch = match.group(1)
head_match = re.search(r'^([0-9a-f]{40}(?:[0-9a-f]{24})?)\s+HEAD$', remote, re.M)
if not head_match:
raise DocumentationError('Repository did not provide a valid HEAD commit.')
if observed_head:
observed_head(branch, head_match.group(1))
run_git(['--git-dir', str(path), 'fetch', '--no-tags', '--depth=50', url,
f'+{branch}:refs/remotes/origin/{branch.removeprefix("refs/heads/")}'], timeout=60, max_output=1024)
if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES:
raise DocumentationError('Repository mirror exceeds the configured size limit.')
commit = run_git(['--git-dir', str(path), 'rev-parse', f'refs/remotes/origin/{branch.removeprefix("refs/heads/")}'],
max_output=128).decode().strip()
if not HASH_RE.fullmatch(commit):
raise DocumentationError('Repository returned an invalid commit.')
committed_at = run_git(['--git-dir', str(path), 'show', '-s', '--format=%cI', commit], max_output=128).decode().strip()
try:
commit_time = datetime.fromisoformat(committed_at)
if commit_time.tzinfo is None:
raise ValueError
except ValueError as exc:
raise DocumentationError('Repository returned an invalid commit time.') from exc
listing = run_git(['--git-dir', str(path), 'ls-tree', '-r', '-z', commit],
max_output=settings.PROJECT_DOCS_MAX_TREE_BYTES)
rows = []
audit_blob = None
total = 0
for raw in listing.split(b'\x00'):
if not raw:
continue
try:
metadata, name = raw.split(b'\t', 1)
mode, kind, blob = metadata.decode('ascii').split(' ')
name = name.decode('utf-8')
except (ValueError, UnicodeDecodeError) as exc:
if b'\tdocs/' in raw or raw.endswith((b'\tREADME.md', b'\tAGENTS.md')):
raise DocumentationError('Repository tree contains an unsupported documentation path.') from exc
continue
if name == 'docs/.audit-state.json' and mode == '100644' and kind == 'blob':
audit_blob = blob
if not indexed_path(name):
continue
if mode != '100644' or kind != 'blob' or not HASH_RE.fullmatch(blob):
continue
rows.append((name, blob))
if len(rows) > settings.PROJECT_DOCS_MAX_FILES:
raise DocumentationError('Repository has too many documentation files.')
documents = []
for name, blob in rows:
content = run_git(['--git-dir', str(path), 'cat-file', 'blob', blob],
max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES)
total += len(content)
if total > settings.PROJECT_DOCS_MAX_CONTENT_BYTES:
raise DocumentationError('Documentation exceeds the configured content limit.')
try:
markdown = content.decode('utf-8')
except UnicodeDecodeError as exc:
raise DocumentationError(f'Document {name} is not UTF-8.') from exc
headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))]
documents.append(ProjectDocument(project=project, path=name,
title=(headings[0] if headings else posixpath.basename(name))[:255], headings=headings[:100],
markdown=markdown, content_hash=hashlib.sha256(content).hexdigest(), source_commit=commit))
audit = {}
if audit_blob and HASH_RE.fullmatch(audit_blob):
raw = run_git(['--git-dir', str(path), 'cat-file', 'blob', audit_blob], max_output=65536)
try:
candidate = json.loads(raw)
if isinstance(candidate, dict) and candidate.get('schema_version') == 1:
limits = {'last_successful_audit': 40, 'last_audited_commit': 64,
'last_report_path': 300, 'last_attempt': 40, 'last_status': 40}
audit = {key: candidate[key] for key, size in limits.items()
if isinstance(candidate.get(key), str) and len(candidate[key]) <= size}
if isinstance(candidate.get('audit_interval_days'), int):
audit['audit_interval_days'] = candidate['audit_interval_days']
except (ValueError, UnicodeDecodeError):
pass
return url, branch, commit, commit_time, documents, audit
def sync_project(user, project, *, force=False):
services.project_for(user, project.pk, write=force)
if not project.git_repository_url:
raise DocumentationError('Configure a Git repository URL before refreshing documentation.')
state, _ = DocumentationSync.objects.get_or_create(project=project)
now = timezone.now()
interval = timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS))
if not force and state.last_attempt_at and now - state.last_attempt_at < interval:
return state
if force and state.last_attempt_at and now - state.last_attempt_at < timedelta(seconds=30):
return state
token = str(uuid.uuid4())
claimed = DocumentationSync.objects.filter(pk=state.pk).filter(
Q(sync_token='') | Q(sync_started_at__lt=now - timedelta(minutes=3))).update(
sync_token=token, sync_started_at=now, last_attempt_at=now)
if not claimed:
return DocumentationSync.objects.get(pk=state.pk)
try:
def record_head(branch, head):
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(branch=branch, remote_head=head)
url, branch, commit, commit_time, documents, audit = _fetch_snapshot(project, observed_head=record_head)
with transaction.atomic():
state = DocumentationSync.objects.select_for_update().get(pk=state.pk)
if state.sync_token != token:
return state
if state.indexed_commit != commit or state.indexed_repository_url != url:
ProjectDocument.objects.filter(project=project).delete()
ProjectDocument.objects.bulk_create(documents, batch_size=100)
state.indexed_commit = commit
state.remote_head = commit
state.indexed_repository_url = url
state.branch = branch
state.commit_time = commit_time
state.last_success_at = timezone.now()
state.last_error = ''
state.sync_token = ''
state.sync_started_at = None
state.audit_state = audit
state.save()
except DocumentationError as exc:
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(
last_error=str(exc)[:300], sync_token='', sync_started_at=None)
except Exception as exc:
logger.error('Documentation indexing failed for project %s (%s).', project.pk, type(exc).__name__)
DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(
last_error='Documentation indexing failed; retry after checking server logs.',
sync_token='', sync_started_at=None)
return DocumentationSync.objects.get(pk=state.pk)
def maybe_sync(user, project):
"""On-demand polling: one bounded refresh per interval for authorized readers."""
services.project_for(user, project.pk)
if project.git_repository_url:
return sync_project(user, project)
return DocumentationSync.objects.filter(project=project).first()
def sync_status(project, state):
if not project.git_repository_url:
return 'No repository configured'
if not state or not state.indexed_commit:
if state and state.sync_token:
return 'Refreshing from Git; no indexed documentation yet'
return 'No indexed documentation' if not state or not state.last_error else f'Sync error: {state.last_error}'
if state.sync_token:
return 'Refreshing from Git; showing the last indexed version'
if state.last_error:
return f'Stale after sync error: {state.last_error}'
if state.indexed_repository_url != project.git_repository_url:
return 'Stale: repository URL changed'
if state.remote_head and state.remote_head != state.indexed_commit:
return 'Stale: remote HEAD differs from indexed commit'
if state.last_success_at and timezone.now() - state.last_success_at >= timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS)):
return 'Refresh due; indexed commit may lag remote'
return 'Indexed from Git; document accuracy is not guaranteed'
def audit_status(state):
audit = state.audit_state if state else {}
date_value = audit.get('last_successful_audit')
commit = audit.get('last_audited_commit')
partial = audit.get('last_status') in ('partial', 'error')
try:
last = datetime.fromisoformat(date_value.replace('Z', '+00:00'))
days = int(audit.get('audit_interval_days', 14))
if last.tzinfo is None or not HASH_RE.fullmatch(commit) or not 1 <= days <= 365:
raise ValueError
except (AttributeError, TypeError, ValueError):
return 'Partial or failed / due' if partial else 'Unknown / due'
if last > timezone.now() + timedelta(minutes=5):
return 'Unknown / due (audit timestamp is in the future)'
if timezone.now() >= last + timedelta(days=days):
return f'Overdue (last successful audit {last.date()})' + ('; latest attempt partial or failed' if partial else '')
if state.indexed_commit != commit:
return f'Last audited {last.date()}; indexed commit differs' + ('; latest attempt partial or failed' if partial else '')
return f'Last audited {last.date()} at this commit' + ('; latest attempt partial or failed' if partial else '')
def list_documents(user, project, *, path='', limit=50, offset=0):
services.project_for(user, project.pk)
if path:
normal_path(path)
size = min(max(int(limit), 1), MAX_RESULTS)
offset = max(int(offset), 0)
qs = ProjectDocument.objects.filter(project=project)
if path:
qs = qs.filter(path__startswith=path.rstrip('/') + '/')
qs = qs.order_by('path')
return list(qs[offset:offset + size]), qs.count()
def search_documents(user, query, *, project=None, limit=30, offset=0):
query = query.strip() if isinstance(query, str) else ''
if not query or len(query) > 200:
raise ValidationError('Search query must be 1–200 characters.')
qs = ProjectDocument.objects.select_related('project').filter(project__in=services.accessible(user))
if project:
services.project_for(user, project.pk)
qs = qs.filter(project=project)
qs = qs.filter(Q(path__icontains=query) | Q(title__icontains=query) | Q(markdown__icontains=query)).order_by('project_id', 'path')
size = min(max(int(limit), 1), MAX_RESULTS)
offset = max(int(offset), 0)
return list(qs[offset:offset + size]), qs.count()
def read_document(user, project, path):
services.project_for(user, project.pk)
return ProjectDocument.objects.get(project=project, path=normal_path(path))
def excerpt(document, query, radius=120):
content = document.markdown
at = content.lower().find(query.lower())
if at < 0:
at = 0
start = max(0, at - radius)
end = min(len(content), at + len(query) + radius)
return ('…' if start else '') + content[start:end].replace('\n', ' ') + ('…' if end < len(content) else '')
def line_window(document, start_line=1, end_line=None):
lines = document.markdown.splitlines(keepends=True)
start = max(int(start_line), 1)
if start > len(lines) + 1:
raise ValidationError('Start line is beyond the document.')
requested_end = int(end_line) if end_line is not None else start + MAX_READ_LINES - 1
if requested_end < start:
raise ValidationError('Invalid line range.')
end = min(requested_end, start + MAX_READ_LINES - 1, len(lines))
output = ''
partial_line = False
actual_end = start - 1
for number in range(start, end + 1):
line = lines[number - 1]
remaining = MAX_READ_CHARS - len(output)
if remaining <= 0:
break
if len(line) > remaining:
output += line[:remaining]
partial_line = True
actual_end = number
break
output += line
actual_end = number
return {'markdown': output, 'start_line': start, 'end_line': actual_end,
'total_lines': len(lines), 'truncated': actual_end < len(lines) or partial_line,
'partial_line': partial_line}
def document_history(user, project, path, limit=20):
services.project_for(user, project.pk)
normal_path(path)
state = DocumentationSync.objects.filter(project=project).first()
if not state or not state.indexed_commit:
return []
mirror = _mirror_path(project, create=False)
if not mirror.exists():
return []
output = _git(['--git-dir', str(mirror), 'log', f'--max-count={min(max(int(limit), 1), 50)}',
'--format=%H%x09%cI', state.indexed_commit, '--', path],
env=_git_env(project), max_output=8192)
rows = []
for line in output.decode('ascii', 'replace').splitlines():
try:
commit, committed_at = line.split('\t', 1)
except ValueError:
continue
if HASH_RE.fullmatch(commit):
rows.append({'commit': commit, 'committed_at': committed_at})
return rows
def historical_document(user, project, path, revision):
current = read_document(user, project, path)
if revision == current.source_commit:
return current
if not isinstance(revision, str) or not HASH_RE.fullmatch(revision):
raise ValidationError('Invalid Git revision.')
if revision not in {row['commit'] for row in document_history(user, project, path, limit=50)}:
raise ValidationError('Revision is unavailable in the retained Git history.')
mirror = _mirror_path(project, create=False)
content = _git(['--git-dir', str(mirror), 'show', f'{revision}:{path}'],
env=_git_env(project), max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES)
try:
markdown = content.decode('utf-8')
except UnicodeDecodeError as exc:
raise DocumentationError('Historical document is not UTF-8.') from exc
headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))]
return ProjectDocument(project=project, path=path, title=(headings[0] if headings else posixpath.basename(path))[:255],
headings=headings[:100], markdown=markdown,
content_hash=hashlib.sha256(content).hexdigest(), source_commit=revision)
def document_diff(user, project, path, before, after):
first = historical_document(user, project, path, before)
second = historical_document(user, project, path, after)
diff = ''.join(difflib.unified_diff(first.markdown.splitlines(keepends=True), second.markdown.splitlines(keepends=True),
fromfile=before, tofile=after))
return diff[:MAX_READ_CHARS], len(diff) > MAX_READ_CHARS