"""Read-only, project-scoped Git documentation index.""" import base64 import difflib import hashlib import json import logging import os import posixpath import re import shutil import subprocess import tempfile from time import monotonic import uuid from datetime import datetime, timedelta from pathlib import Path from urllib.parse import urlsplit from django.conf import settings from django.core.exceptions import ValidationError from django.db import transaction from django.db.models import Q from django.utils import timezone from . import services from .models import DocumentationSync, ProjectDocument MAX_RESULTS = 100 MAX_READ_LINES = 250 MAX_READ_CHARS = 30000 REF_RE = re.compile(r'^refs/heads/[A-Za-z0-9][A-Za-z0-9._/-]*$') HASH_RE = re.compile(r'^[0-9a-f]{40}([0-9a-f]{24})?$') HEADING_RE = re.compile(r'^#{1,6}\s+(.+?)\s*#*\s*$') logger = logging.getLogger(__name__) class DocumentationError(Exception): pass def normal_path(value): if not isinstance(value, str) or not value or len(value) > 500 or '\x00' in value: raise ValidationError('Invalid document path.') if value.startswith('/') or '\\' in value or any(part in ('', '.', '..') for part in value.split('/')): raise ValidationError('Invalid document path.') if any(ord(c) < 32 for c in value): raise ValidationError('Invalid document path.') return value def indexed_path(value): try: normal_path(value) except ValidationError: return False return value in ('README.md', 'AGENTS.md') or (value.startswith('docs/') and value.lower().endswith('.md')) def _repository_url(project): raw = project.git_repository_url try: parsed = urlsplit(raw) hostname = parsed.hostname port = parsed.port except ValueError as exc: raise DocumentationError('Repository URL is malformed.') from exc allowed = {item.strip().lower() for item in settings.PROJECT_DOCS_ALLOWED_GIT_HOSTS.split(',') if item.strip()} authority = hostname.lower() if hostname else '' if port: authority += f':{port}' if (parsed.scheme != 'https' or not authority or authority not in allowed or parsed.username or parsed.password or parsed.query or parsed.fragment or not parsed.path.startswith('/') or '..' in parsed.path.split('/') or '%' in raw or '\\' in raw or any(c.isspace() for c in raw)): raise DocumentationError('Repository URL is not an approved HTTPS Git endpoint.') return raw def _git_env(project): env = {key: value for key, value in os.environ.items() if not key.startswith('GIT_') and key not in ('SSH_ASKPASS', 'SSH_ASKPASS_REQUIRE')} env.update({'GIT_TERMINAL_PROMPT': '0', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_SYSTEM': os.devnull, 'GIT_NO_REPLACE_OBJECTS': '1'}) try: secrets = json.loads(settings.PROJECT_DOCS_CREDENTIALS_JSON or '{}') except (TypeError, ValueError) as exc: raise DocumentationError('Git credentials are not configured correctly.') from exc if not isinstance(secrets, dict): raise DocumentationError('Git credentials are not configured correctly.') entry = secrets.get(str(project.pk), {}) if not isinstance(entry, dict): raise DocumentationError('Project Git credentials are not configured correctly.') if entry: username, token = entry.get('username'), entry.get('token') if entry.get('repository_url') != _repository_url(project): raise DocumentationError('Project Git credential is not bound to this repository URL.') if not isinstance(username, str) or not isinstance(token, str) or not username or not token or '\n' in username + token or '\r' in username + token: raise DocumentationError('Project Git credentials are incomplete.') encoded = base64.b64encode(f'{username}:{token}'.encode()).decode() env['GIT_CONFIG_COUNT'] = '1' env['GIT_CONFIG_KEY_0'] = f'http.{_repository_url(project)}.extraHeader' env['GIT_CONFIG_VALUE_0'] = f'Authorization: Basic {encoded}' return env def _git(args, *, env, timeout=30, max_output=20 * 1024 * 1024): limiter = shutil.which('prlimit') if not limiter: raise DocumentationError('Git resource limiter is unavailable on this server.') command = [limiter, f'--fsize={settings.PROJECT_DOCS_MAX_MIRROR_BYTES}', '--cpu=65', '--', 'git', '-c', 'credential.helper=', '-c', 'core.hooksPath=/dev/null', '-c', 'protocol.file.allow=never', '-c', 'protocol.ext.allow=never', '-c', 'http.followRedirects=false', '-c', 'transfer.fsckObjects=true', '-c', 'gc.auto=0', *args] try: with tempfile.TemporaryFile() as output: result = subprocess.run(command, env=env, stdout=output, stderr=subprocess.DEVNULL, timeout=timeout, check=False) if result.returncode or output.tell() > max_output: raise DocumentationError('Git fetch or index failed; check repository access and size limits.') output.seek(0) return output.read(max_output + 1) except (OSError, subprocess.TimeoutExpired, subprocess.SubprocessError) as exc: raise DocumentationError('Git fetch or index timed out or could not start.') from exc def _mirror_path(project, *, create=True): root = Path(settings.PROJECT_DOCS_MIRROR_ROOT) if create: root.mkdir(mode=0o700, parents=True, exist_ok=True) path = root / str(project.pk) if path.is_symlink() or (path.exists() and not path.is_dir()): raise DocumentationError('Repository storage is unavailable.') return path def _mirror_size(path): return sum(item.stat().st_size for item in path.rglob('*') if item.is_file() and not item.is_symlink()) def _fetch_snapshot(project, observed_head=None): url = _repository_url(project) env = _git_env(project) deadline = monotonic() + 90 def run_git(args, *, timeout=30, max_output=20 * 1024 * 1024): remaining = deadline - monotonic() if remaining <= 0: raise DocumentationError('Git documentation refresh exceeded its time limit.') return _git(args, env=env, timeout=min(timeout, remaining), max_output=max_output) path = _mirror_path(project) if not path.exists(): run_git(['init', '--bare', str(path)]) path.chmod(0o700) if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES: raise DocumentationError('Repository mirror exceeds the configured size limit.') remote = run_git(['ls-remote', '--symref', url, 'HEAD'], max_output=4096).decode('ascii', 'replace') match = re.search(r'^ref: (refs/heads/\S+)\s+HEAD$', remote, re.M) if not match or not REF_RE.fullmatch(match.group(1)) or '..' in match.group(1) or '@{' in match.group(1): raise DocumentationError('Repository does not advertise a safe default branch.') branch = match.group(1) head_match = re.search(r'^([0-9a-f]{40}(?:[0-9a-f]{24})?)\s+HEAD$', remote, re.M) if not head_match: raise DocumentationError('Repository did not provide a valid HEAD commit.') if observed_head: observed_head(branch, head_match.group(1)) run_git(['--git-dir', str(path), 'fetch', '--no-tags', '--depth=50', url, f'+{branch}:refs/remotes/origin/{branch.removeprefix("refs/heads/")}'], timeout=60, max_output=1024) if _mirror_size(path) > settings.PROJECT_DOCS_MAX_MIRROR_BYTES: raise DocumentationError('Repository mirror exceeds the configured size limit.') commit = run_git(['--git-dir', str(path), 'rev-parse', f'refs/remotes/origin/{branch.removeprefix("refs/heads/")}'], max_output=128).decode().strip() if not HASH_RE.fullmatch(commit): raise DocumentationError('Repository returned an invalid commit.') committed_at = run_git(['--git-dir', str(path), 'show', '-s', '--format=%cI', commit], max_output=128).decode().strip() try: commit_time = datetime.fromisoformat(committed_at) if commit_time.tzinfo is None: raise ValueError except ValueError as exc: raise DocumentationError('Repository returned an invalid commit time.') from exc listing = run_git(['--git-dir', str(path), 'ls-tree', '-r', '-z', commit], max_output=settings.PROJECT_DOCS_MAX_TREE_BYTES) rows = [] audit_blob = None total = 0 for raw in listing.split(b'\x00'): if not raw: continue try: metadata, name = raw.split(b'\t', 1) mode, kind, blob = metadata.decode('ascii').split(' ') name = name.decode('utf-8') except (ValueError, UnicodeDecodeError) as exc: if b'\tdocs/' in raw or raw.endswith((b'\tREADME.md', b'\tAGENTS.md')): raise DocumentationError('Repository tree contains an unsupported documentation path.') from exc continue if name == 'docs/.audit-state.json' and mode == '100644' and kind == 'blob': audit_blob = blob if not indexed_path(name): continue if mode != '100644' or kind != 'blob' or not HASH_RE.fullmatch(blob): continue rows.append((name, blob)) if len(rows) > settings.PROJECT_DOCS_MAX_FILES: raise DocumentationError('Repository has too many documentation files.') documents = [] for name, blob in rows: content = run_git(['--git-dir', str(path), 'cat-file', 'blob', blob], max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES) total += len(content) if total > settings.PROJECT_DOCS_MAX_CONTENT_BYTES: raise DocumentationError('Documentation exceeds the configured content limit.') try: markdown = content.decode('utf-8') except UnicodeDecodeError as exc: raise DocumentationError(f'Document {name} is not UTF-8.') from exc headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))] documents.append(ProjectDocument(project=project, path=name, title=(headings[0] if headings else posixpath.basename(name))[:255], headings=headings[:100], markdown=markdown, content_hash=hashlib.sha256(content).hexdigest(), source_commit=commit)) audit = {} if audit_blob and HASH_RE.fullmatch(audit_blob): raw = run_git(['--git-dir', str(path), 'cat-file', 'blob', audit_blob], max_output=65536) try: candidate = json.loads(raw) if isinstance(candidate, dict) and candidate.get('schema_version') == 1: limits = {'last_successful_audit': 40, 'last_audited_commit': 64, 'last_report_path': 300, 'last_attempt': 40, 'last_status': 40} audit = {key: candidate[key] for key, size in limits.items() if isinstance(candidate.get(key), str) and len(candidate[key]) <= size} if isinstance(candidate.get('audit_interval_days'), int): audit['audit_interval_days'] = candidate['audit_interval_days'] except (ValueError, UnicodeDecodeError): pass return url, branch, commit, commit_time, documents, audit def sync_project(user, project, *, force=False): services.project_for(user, project.pk, write=force) if not project.git_repository_url: raise DocumentationError('Configure a Git repository URL before refreshing documentation.') state, _ = DocumentationSync.objects.get_or_create(project=project) now = timezone.now() interval = timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS)) if not force and state.last_attempt_at and now - state.last_attempt_at < interval: return state if force and state.last_attempt_at and now - state.last_attempt_at < timedelta(seconds=30): return state token = str(uuid.uuid4()) claimed = DocumentationSync.objects.filter(pk=state.pk).filter( Q(sync_token='') | Q(sync_started_at__lt=now - timedelta(minutes=3))).update( sync_token=token, sync_started_at=now, last_attempt_at=now) if not claimed: return DocumentationSync.objects.get(pk=state.pk) try: def record_head(branch, head): DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update(branch=branch, remote_head=head) url, branch, commit, commit_time, documents, audit = _fetch_snapshot(project, observed_head=record_head) with transaction.atomic(): state = DocumentationSync.objects.select_for_update().get(pk=state.pk) if state.sync_token != token: return state if state.indexed_commit != commit or state.indexed_repository_url != url: ProjectDocument.objects.filter(project=project).delete() ProjectDocument.objects.bulk_create(documents, batch_size=100) state.indexed_commit = commit state.remote_head = commit state.indexed_repository_url = url state.branch = branch state.commit_time = commit_time state.last_success_at = timezone.now() state.last_error = '' state.sync_token = '' state.sync_started_at = None state.audit_state = audit state.save() except DocumentationError as exc: DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update( last_error=str(exc)[:300], sync_token='', sync_started_at=None) except Exception as exc: logger.error('Documentation indexing failed for project %s (%s).', project.pk, type(exc).__name__) DocumentationSync.objects.filter(pk=state.pk, sync_token=token).update( last_error='Documentation indexing failed; retry after checking server logs.', sync_token='', sync_started_at=None) return DocumentationSync.objects.get(pk=state.pk) def maybe_sync(user, project): """On-demand polling: one bounded refresh per interval for authorized readers.""" services.project_for(user, project.pk) if project.git_repository_url: return sync_project(user, project) return DocumentationSync.objects.filter(project=project).first() def sync_status(project, state): if not project.git_repository_url: return 'No repository configured' if not state or not state.indexed_commit: if state and state.sync_token: return 'Refreshing from Git; no indexed documentation yet' return 'No indexed documentation' if not state or not state.last_error else f'Sync error: {state.last_error}' if state.sync_token: return 'Refreshing from Git; showing the last indexed version' if state.last_error: return f'Stale after sync error: {state.last_error}' if state.indexed_repository_url != project.git_repository_url: return 'Stale: repository URL changed' if state.remote_head and state.remote_head != state.indexed_commit: return 'Stale: remote HEAD differs from indexed commit' if state.last_success_at and timezone.now() - state.last_success_at >= timedelta(seconds=max(60, settings.PROJECT_DOCS_SYNC_INTERVAL_SECONDS)): return 'Refresh due; indexed commit may lag remote' return 'Indexed from Git; document accuracy is not guaranteed' def audit_status(state): audit = state.audit_state if state else {} date_value = audit.get('last_successful_audit') commit = audit.get('last_audited_commit') partial = audit.get('last_status') in ('partial', 'error') try: last = datetime.fromisoformat(date_value.replace('Z', '+00:00')) days = int(audit.get('audit_interval_days', 14)) if last.tzinfo is None or not HASH_RE.fullmatch(commit) or not 1 <= days <= 365: raise ValueError except (AttributeError, TypeError, ValueError): return 'Partial or failed / due' if partial else 'Unknown / due' if last > timezone.now() + timedelta(minutes=5): return 'Unknown / due (audit timestamp is in the future)' if timezone.now() >= last + timedelta(days=days): return f'Overdue (last successful audit {last.date()})' + ('; latest attempt partial or failed' if partial else '') if state.indexed_commit != commit: return f'Last audited {last.date()}; indexed commit differs' + ('; latest attempt partial or failed' if partial else '') return f'Last audited {last.date()} at this commit' + ('; latest attempt partial or failed' if partial else '') def list_documents(user, project, *, path='', limit=50, offset=0): services.project_for(user, project.pk) if path: normal_path(path) size = min(max(int(limit), 1), MAX_RESULTS) offset = max(int(offset), 0) qs = ProjectDocument.objects.filter(project=project) if path: qs = qs.filter(path__startswith=path.rstrip('/') + '/') qs = qs.order_by('path') return list(qs[offset:offset + size]), qs.count() def search_documents(user, query, *, project=None, limit=30, offset=0): query = query.strip() if isinstance(query, str) else '' if not query or len(query) > 200: raise ValidationError('Search query must be 1–200 characters.') qs = ProjectDocument.objects.select_related('project').filter(project__in=services.accessible(user)) if project: services.project_for(user, project.pk) qs = qs.filter(project=project) qs = qs.filter(Q(path__icontains=query) | Q(title__icontains=query) | Q(markdown__icontains=query)).order_by('project_id', 'path') size = min(max(int(limit), 1), MAX_RESULTS) offset = max(int(offset), 0) return list(qs[offset:offset + size]), qs.count() def read_document(user, project, path): services.project_for(user, project.pk) return ProjectDocument.objects.get(project=project, path=normal_path(path)) def excerpt(document, query, radius=120): content = document.markdown at = content.lower().find(query.lower()) if at < 0: at = 0 start = max(0, at - radius) end = min(len(content), at + len(query) + radius) return ('…' if start else '') + content[start:end].replace('\n', ' ') + ('…' if end < len(content) else '') def line_window(document, start_line=1, end_line=None): lines = document.markdown.splitlines(keepends=True) start = max(int(start_line), 1) if start > len(lines) + 1: raise ValidationError('Start line is beyond the document.') requested_end = int(end_line) if end_line is not None else start + MAX_READ_LINES - 1 if requested_end < start: raise ValidationError('Invalid line range.') end = min(requested_end, start + MAX_READ_LINES - 1, len(lines)) output = '' partial_line = False actual_end = start - 1 for number in range(start, end + 1): line = lines[number - 1] remaining = MAX_READ_CHARS - len(output) if remaining <= 0: break if len(line) > remaining: output += line[:remaining] partial_line = True actual_end = number break output += line actual_end = number return {'markdown': output, 'start_line': start, 'end_line': actual_end, 'total_lines': len(lines), 'truncated': actual_end < len(lines) or partial_line, 'partial_line': partial_line} def document_history(user, project, path, limit=20): services.project_for(user, project.pk) normal_path(path) state = DocumentationSync.objects.filter(project=project).first() if not state or not state.indexed_commit: return [] mirror = _mirror_path(project, create=False) if not mirror.exists(): return [] output = _git(['--git-dir', str(mirror), 'log', f'--max-count={min(max(int(limit), 1), 50)}', '--format=%H%x09%cI', state.indexed_commit, '--', path], env=_git_env(project), max_output=8192) rows = [] for line in output.decode('ascii', 'replace').splitlines(): try: commit, committed_at = line.split('\t', 1) except ValueError: continue if HASH_RE.fullmatch(commit): rows.append({'commit': commit, 'committed_at': committed_at}) return rows def historical_document(user, project, path, revision): current = read_document(user, project, path) if revision == current.source_commit: return current if not isinstance(revision, str) or not HASH_RE.fullmatch(revision): raise ValidationError('Invalid Git revision.') if revision not in {row['commit'] for row in document_history(user, project, path, limit=50)}: raise ValidationError('Revision is unavailable in the retained Git history.') mirror = _mirror_path(project, create=False) content = _git(['--git-dir', str(mirror), 'show', f'{revision}:{path}'], env=_git_env(project), max_output=settings.PROJECT_DOCS_MAX_FILE_BYTES) try: markdown = content.decode('utf-8') except UnicodeDecodeError as exc: raise DocumentationError('Historical document is not UTF-8.') from exc headings = [m.group(1).strip()[:255] for line in markdown.splitlines() if (m := HEADING_RE.match(line))] return ProjectDocument(project=project, path=path, title=(headings[0] if headings else posixpath.basename(path))[:255], headings=headings[:100], markdown=markdown, content_hash=hashlib.sha256(content).hexdigest(), source_commit=revision) def document_diff(user, project, path, before, after): first = historical_document(user, project, path, before) second = historical_document(user, project, path, after) diff = ''.join(difflib.unified_diff(first.markdown.splitlines(keepends=True), second.markdown.splitlines(keepends=True), fromfile=before, tofile=after)) return diff[:MAX_READ_CHARS], len(diff) > MAX_READ_CHARS