"""Project discovery, root confinement, canonical loading, and graph validation.""" from __future__ import annotations import hashlib import json import re import subprocess import tomllib from collections import Counter from dataclasses import replace from pathlib import Path from typing import Any from .errors import DocForgeError from .models import ( ContextProfile, Edge, Limits, Node, ProjectDescriptor, ProjectSnapshot, ) _ID_PATTERN = re.compile(r"[a-z0-9][a-z0-9._-]{1,127}") _AUTHORITIES = frozenset({"authoritative", "approved_plan", "derived", "proposal", "historical"}) _SECRET_PARTS = frozenset({".git", ".ssh", ".gnupg", "secrets", "credentials"}) _CORE_METADATA = frozenset( { "schema_version", "id", "title", "family", "authority", "status", "tags", "summary", "source_anchor", "content", } ) _DESCRIPTOR_KEYS = frozenset( { "schema_version", "project_id", "title", "adapter", "sources", "derived", "graph", "limits", "profiles", } ) _SOURCE_KEYS = frozenset({"content_roots", "authority_files"}) _DERIVED_KEYS = frozenset({"cache_root", "index"}) _GRAPH_KEYS = frozenset({"allowed_relations"}) _PROFILE_KEYS = frozenset( {"id", "families", "statuses", "required_nodes", "token_budget", "dependency_depth"} ) def project_root_fingerprint(root: Path) -> str: return hashlib.sha256(str(root).encode()).hexdigest()[:16] def _require_string(document: dict[str, Any], key: str, source: Path) -> str: value = document.get(key) if not isinstance(value, str) or not value.strip(): raise DocForgeError("invalid_source", f"{source.name}: {key} must be a non-empty string") return value.strip() def _string_list(value: object, *, key: str, source: Path) -> tuple[str, ...]: if not isinstance(value, list) or any(not isinstance(item, str) or not item for item in value): raise DocForgeError("invalid_source", f"{source.name}: {key} must be a string list") if len(value) != len(set(value)): raise DocForgeError("invalid_source", f"{source.name}: {key} contains duplicates") return tuple(value) def _confined_path( root: Path, raw: object, *, field: str, must_exist: bool, expected: str | None = None, ) -> Path: if not isinstance(raw, str) or not raw: raise DocForgeError("invalid_config", f"{field} must be a non-empty relative path") relative = Path(raw) if relative.is_absolute() or ".." in relative.parts: raise DocForgeError("path_escape", f"{field} must stay inside the project root", path=raw) if any(part.lower() in _SECRET_PARTS for part in relative.parts): raise DocForgeError("secret_path", f"{field} may not reference a protected path", path=raw) resolved = (root / relative).resolve(strict=False) if not resolved.is_relative_to(root): raise DocForgeError("path_escape", f"{field} resolves outside the project root", path=raw) if must_exist and not resolved.exists(): raise DocForgeError("missing_path", f"{field} does not exist", path=raw) if expected == "file" and must_exist and not resolved.is_file(): raise DocForgeError("invalid_path", f"{field} must identify a file", path=raw) if expected == "directory" and must_exist and not resolved.is_dir(): raise DocForgeError("invalid_path", f"{field} must identify a directory", path=raw) return resolved def _load_descriptor(root: Path) -> ProjectDescriptor: descriptor_path = root / ".docforge" / "project.toml" if not descriptor_path.is_file(): raise DocForgeError("missing_config", "Missing .docforge/project.toml") try: descriptor_bytes = descriptor_path.read_bytes() document = tomllib.loads(descriptor_bytes.decode("utf-8")) except UnicodeDecodeError as error: raise DocForgeError("invalid_config", "Project descriptor is not UTF-8") from error except tomllib.TOMLDecodeError as error: raise DocForgeError("invalid_config", f"Invalid project descriptor: {error}") from error unknown_descriptor = sorted(set(document) - _DESCRIPTOR_KEYS) if unknown_descriptor: raise DocForgeError( "invalid_config", "Project descriptor has unknown fields", fields=unknown_descriptor ) if document.get("schema_version") != 1: raise DocForgeError("invalid_config", "Project descriptor schema_version must be 1") project_id = _require_string(document, "project_id", descriptor_path) if _ID_PATTERN.fullmatch(project_id) is None: raise DocForgeError( "invalid_config", "project_id is not a stable ID", project_id=project_id ) title = _require_string(document, "title", descriptor_path) adapter = _require_string(document, "adapter", descriptor_path) if adapter != "generic": raise DocForgeError("unsupported_adapter", "DFG-1 supports only the generic adapter") sources = document.get("sources") derived = document.get("derived") graph = document.get("graph") if ( not isinstance(sources, dict) or not isinstance(derived, dict) or not isinstance(graph, dict) ): raise DocForgeError("invalid_config", "sources, derived, and graph tables are required") for table, allowed, name in ( (sources, _SOURCE_KEYS, "sources"), (derived, _DERIVED_KEYS, "derived"), (graph, _GRAPH_KEYS, "graph"), ): unknown = sorted(set(table) - allowed) if unknown: raise DocForgeError("invalid_config", f"{name} has unknown fields", fields=unknown) content_roots = tuple( _confined_path( root, item, field="sources.content_roots", must_exist=True, expected="directory", ) for item in _string_list( sources.get("content_roots"), key="sources.content_roots", source=descriptor_path ) ) if len(content_roots) != len(set(content_roots)): raise DocForgeError("invalid_config", "sources.content_roots resolve to duplicates") authority_files = tuple( _confined_path( root, item, field="sources.authority_files", must_exist=True, expected="file", ) for item in _string_list( sources.get("authority_files", []), key="sources.authority_files", source=descriptor_path, ) ) cache_root = _confined_path( root, derived.get("cache_root"), field="derived.cache_root", must_exist=False ) index_path = _confined_path(root, derived.get("index"), field="derived.index", must_exist=False) if not index_path.is_relative_to(cache_root): raise DocForgeError("invalid_config", "derived.index must be inside derived.cache_root") for content_root in content_roots: if ( content_root == cache_root or content_root.is_relative_to(cache_root) or cache_root.is_relative_to(content_root) ): raise DocForgeError("invalid_config", "Canonical content and cache must not overlap") allowed_relations = _string_list( graph.get("allowed_relations"), key="graph.allowed_relations", source=descriptor_path ) if not allowed_relations: raise DocForgeError("invalid_config", "At least one relationship type is required") for relation in allowed_relations: if _ID_PATTERN.fullmatch(relation) is None: raise DocForgeError("invalid_config", "Relationship type is invalid", relation=relation) limit_values = document.get("limits", {}) if not isinstance(limit_values, dict): raise DocForgeError("invalid_config", "limits must be a table") defaults = Limits() unknown_limits = sorted(set(limit_values) - set(defaults.__dataclass_fields__)) if unknown_limits: raise DocForgeError("invalid_config", "limits has unknown fields", fields=unknown_limits) limits = Limits( **{ field: _positive_int(limit_values.get(field, getattr(defaults, field)), field) for field in defaults.__dataclass_fields__ } ) profile_documents = document.get("profiles", []) if not isinstance(profile_documents, list): raise DocForgeError("invalid_config", "profiles must be an array of tables") profiles: list[ContextProfile] = [] profile_ids: set[str] = set() for profile in profile_documents: if not isinstance(profile, dict): raise DocForgeError("invalid_config", "Each profile must be a table") unknown_profile = sorted(set(profile) - _PROFILE_KEYS) if unknown_profile: raise DocForgeError( "invalid_config", "Profile has unknown fields", fields=unknown_profile ) profile_id = _require_string(profile, "id", descriptor_path) if _ID_PATTERN.fullmatch(profile_id) is None or profile_id in profile_ids: raise DocForgeError( "invalid_config", "Profile ID is invalid or duplicated", id=profile_id ) profile_ids.add(profile_id) token_budget = _positive_int(profile.get("token_budget", 8_000), "profile.token_budget") dependency_depth = _positive_int( profile.get("dependency_depth", 1), "profile.dependency_depth", allow_zero=True ) if token_budget > limits.max_context_tokens: raise DocForgeError("invalid_config", "Profile token budget exceeds project limit") if dependency_depth > limits.max_traversal_depth: raise DocForgeError("invalid_config", "Profile dependency depth exceeds project limit") profiles.append( ContextProfile( profile_id=profile_id, families=_string_list( profile.get("families", []), key="profile.families", source=descriptor_path ), statuses=_string_list( profile.get("statuses", []), key="profile.statuses", source=descriptor_path ), required_nodes=_string_list( profile.get("required_nodes", []), key="profile.required_nodes", source=descriptor_path, ), token_budget=token_budget, dependency_depth=dependency_depth, ) ) return ProjectDescriptor( schema_version=1, project_id=project_id, title=title, adapter=adapter, root=root, descriptor_path=descriptor_path, descriptor_hash=hashlib.sha256(descriptor_bytes).hexdigest(), content_roots=content_roots, authority_files=authority_files, cache_root=cache_root, index_path=index_path, allowed_relations=allowed_relations, profiles=tuple(profiles), limits=limits, ) def _positive_int(value: object, field: str, *, allow_zero: bool = False) -> int: minimum = 0 if allow_zero else 1 if not isinstance(value, int) or isinstance(value, bool) or value < minimum: raise DocForgeError("invalid_config", f"{field} must be an integer >= {minimum}") return value def _markdown_record(path: Path, text: str) -> tuple[dict[str, Any], str]: lines = text.splitlines() if not lines or lines[0] != "+++": raise DocForgeError( "invalid_source", f"{path.name}: Markdown must start with TOML metadata" ) try: close = lines.index("+++", 1) except ValueError as error: raise DocForgeError( "invalid_source", f"{path.name}: metadata block is not closed" ) from error try: metadata = tomllib.loads("\n".join(lines[1:close])) except tomllib.TOMLDecodeError as error: raise DocForgeError("invalid_source", f"{path.name}: invalid metadata: {error}") from error return metadata, "\n".join(lines[close + 1 :]).strip() def _node_from_record( record: dict[str, Any], *, content: str, source: Path, relative_source: str, relations: tuple[str, ...], hash_bytes: bytes, ) -> tuple[Node, tuple[Edge, ...]]: if record.get("schema_version") != 1: raise DocForgeError("invalid_source", f"{source.name}: node schema_version must be 1") unknown = set(record) - _CORE_METADATA - set(relations) if unknown: raise DocForgeError( "invalid_source", f"{source.name}: unknown metadata", keys=sorted(unknown) ) node_id = _require_string(record, "id", source) if _ID_PATTERN.fullmatch(node_id) is None: raise DocForgeError("invalid_source", f"{source.name}: node ID is invalid", id=node_id) authority = _require_string(record, "authority", source) if authority not in _AUTHORITIES: raise DocForgeError( "invalid_source", f"{source.name}: authority is invalid", authority=authority ) tags = _string_list(record.get("tags", []), key="tags", source=source) anchor = record.get("source_anchor") if anchor is not None and (not isinstance(anchor, str) or not anchor): raise DocForgeError("invalid_source", f"{source.name}: source_anchor must be a string") summary = _require_string(record, "summary", source) if not content: raise DocForgeError("invalid_source", f"{source.name}: node content is empty", id=node_id) node = Node( node_id=node_id, title=_require_string(record, "title", source), family=_require_string(record, "family", source), authority=authority, status=_require_string(record, "status", source), tags=tags, summary=summary, content=content, source_path=relative_source, source_anchor=anchor, content_hash=hashlib.sha256(hash_bytes).hexdigest(), ) edges = tuple( Edge(node_id, relation, target) for relation in relations for target in _string_list(record.get(relation, []), key=relation, source=source) ) return node, edges def _load_source_file( descriptor: ProjectDescriptor, path: Path, raw: bytes ) -> tuple[tuple[Node, ...], tuple[Edge, ...]]: if len(raw) > descriptor.limits.max_source_bytes: raise DocForgeError( "source_too_large", "Canonical source exceeds configured limit", source=path.name ) try: text = raw.decode("utf-8") except UnicodeDecodeError as error: raise DocForgeError("invalid_source", f"{path.name}: source is not UTF-8") from error relative = path.relative_to(descriptor.root).as_posix() if path.suffix == ".md": record, content = _markdown_record(path, text) node, edges = _node_from_record( record, content=content, source=path, relative_source=relative, relations=descriptor.allowed_relations, hash_bytes=raw, ) return (node,), edges if path.suffix == ".toml": try: document = tomllib.loads(text) except tomllib.TOMLDecodeError as error: raise DocForgeError("invalid_source", f"{path.name}: invalid TOML: {error}") from error records = document.get("nodes") if set(document) != {"nodes"} or not isinstance(records, list) or not records: raise DocForgeError("invalid_source", f"{path.name}: TOML sources require [[nodes]]") nodes: list[Node] = [] edges: list[Edge] = [] for index, record in enumerate(records): if not isinstance(record, dict): raise DocForgeError("invalid_source", f"{path.name}: nodes must be tables") content = record.get("content") if not isinstance(content, str): raise DocForgeError( "invalid_source", f"{path.name}: TOML node content must be text" ) canonical = json.dumps(record, sort_keys=True, separators=(",", ":")).encode() node, node_edges = _node_from_record( record, content=content.strip(), source=path, relative_source=relative, relations=descriptor.allowed_relations, hash_bytes=canonical, ) nodes.append(replace(node, source_anchor=node.source_anchor or f"node-{index + 1}")) edges.extend(node_edges) return tuple(nodes), tuple(edges) raise DocForgeError("invalid_source", "Unsupported canonical source type", source=relative) def _validate_graph(nodes: tuple[Node, ...], edges: tuple[Edge, ...]) -> None: node_ids = {node.node_id for node in nodes} if len(node_ids) != len(nodes): counts = Counter(node.node_id for node in nodes) duplicates = sorted(node_id for node_id, count in counts.items() if count > 1) raise DocForgeError("duplicate_node", "Stable node IDs must be unique", ids=duplicates) edge_keys = {(edge.source_id, edge.relation, edge.target_id) for edge in edges} if len(edge_keys) != len(edges): raise DocForgeError("duplicate_edge", "Relationships must be unique") missing = sorted({edge.target_id for edge in edges if edge.target_id not in node_ids}) if missing: raise DocForgeError("broken_edge", "Relationships target missing nodes", targets=missing) dependencies = { node_id: sorted( edge.target_id for edge in edges if edge.source_id == node_id and edge.relation == "depends_on" ) for node_id in sorted(node_ids) } visiting: set[str] = set() visited: set[str] = set() def visit(node_id: str, trail: tuple[str, ...]) -> None: if node_id in visiting: raise DocForgeError( "dependency_cycle", "depends_on relationships contain a cycle", path=(*trail, node_id), ) if node_id in visited: return visiting.add(node_id) for target in dependencies[node_id]: visit(target, (*trail, node_id)) visiting.remove(node_id) visited.add(node_id) for node_id in sorted(node_ids): visit(node_id, ()) def _revision(root: Path) -> str: try: result = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=root, check=False, capture_output=True, text=True, timeout=2, ) except (OSError, subprocess.TimeoutExpired): return "unversioned" return ( result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unversioned" ) class Project: """One immutable project binding for loading and querying canonical documentation.""" def __init__(self, descriptor: ProjectDescriptor) -> None: self.descriptor = descriptor @classmethod def open(cls, project_root: str | Path) -> Project: try: root = Path(project_root).expanduser().resolve(strict=True) except OSError as error: raise DocForgeError("invalid_root", "Project root does not exist") from error if not root.is_dir(): raise DocForgeError("invalid_root", "Project root must be a directory") return cls(_load_descriptor(root)) def load(self) -> ProjectSnapshot: descriptor_bytes = self.descriptor.descriptor_path.read_bytes() if hashlib.sha256(descriptor_bytes).hexdigest() != self.descriptor.descriptor_hash: raise DocForgeError( "source_changed", "Project descriptor changed after the project was opened" ) ordered_sources = self._canonical_source_paths() captured = { path: path.read_bytes() for path in ( self.descriptor.descriptor_path, *self.descriptor.authority_files, *ordered_sources, ) } nodes: list[Node] = [] edges: list[Edge] = [] for path in ordered_sources: source_nodes, source_edges = _load_source_file(self.descriptor, path, captured[path]) nodes.extend(source_nodes) edges.extend(source_edges) if len(nodes) > self.descriptor.limits.max_nodes: raise DocForgeError("node_limit", "Project exceeds configured node limit") ordered_nodes = tuple(sorted(nodes, key=lambda node: node.node_id)) ordered_edges = tuple( sorted(edges, key=lambda edge: (edge.source_id, edge.relation, edge.target_id)) ) _validate_graph(ordered_nodes, ordered_edges) node_ids = {node.node_id for node in ordered_nodes} for profile in self.descriptor.profiles: missing = sorted(set(profile.required_nodes) - node_ids) if missing: raise DocForgeError( "invalid_config", "Context profile requires missing nodes", nodes=missing ) if self._canonical_source_paths() != ordered_sources: raise DocForgeError("source_changed", "Canonical source set changed during loading") for path, raw in captured.items(): if not path.is_file() or path.read_bytes() != raw: raise DocForgeError( "source_changed", "Canonical source changed during loading", source=path.relative_to(self.descriptor.root).as_posix(), ) digest = hashlib.sha256() for path in sorted( captured, key=lambda item: item.relative_to(self.descriptor.root).as_posix() ): relative = path.relative_to(self.descriptor.root).as_posix() digest.update(relative.encode()) digest.update(b"\0") digest.update(hashlib.sha256(captured[path]).digest()) digest.update(b"docforge-core:0.1.0:index:1") return ProjectSnapshot( descriptor=self.descriptor, nodes=ordered_nodes, edges=ordered_edges, source_hash=digest.hexdigest(), revision=_revision(self.descriptor.root), ) def _canonical_source_paths(self) -> tuple[Path, ...]: source_paths: set[Path] = set() for content_root in self.descriptor.content_roots: for path in content_root.rglob("*"): if path.suffix not in {".md", ".toml"} or not path.is_file(): continue resolved = path.resolve() if not resolved.is_relative_to(self.descriptor.root): raise DocForgeError( "path_escape", "Canonical source resolves outside project root" ) source_paths.add(resolved) ordered_sources = sorted( source_paths, key=lambda path: path.relative_to(self.descriptor.root).as_posix() ) if not ordered_sources: raise DocForgeError("empty_project", "No canonical Markdown or TOML sources were found") return tuple(ordered_sources)