"""Project discovery, root confinement, canonical loading, and graph validation.""" from __future__ import annotations import hashlib import json import subprocess import tomllib from collections import Counter from dataclasses import replace from pathlib import Path from typing import Any from .config_validation import ID_PATTERN, confined_path, positive_int, require_string, string_list from .errors import DocForgeError from .models import ( ContextProfile, Edge, Limits, Node, ProjectDescriptor, ProjectSnapshot, ProposalWriter, ) from .render_config import load_render_config _AUTHORITIES = frozenset({"authoritative", "approved_plan", "derived", "proposal", "historical"}) _CORE_METADATA = frozenset( { "schema_version", "id", "title", "family", "authority", "status", "tags", "summary", "source_anchor", "content", } ) _DESCRIPTOR_KEYS = frozenset( { "schema_version", "project_id", "title", "adapter", "sources", "derived", "changesets", "render", "graph", "limits", "profiles", } ) _SOURCE_KEYS = frozenset({"content_roots", "authority_files"}) _DERIVED_KEYS = frozenset({"cache_root", "index"}) _CHANGESET_KEYS = frozenset({"root", "writers"}) _WRITER_KEYS = frozenset({"id", "families", "operations"}) _GRAPH_KEYS = frozenset({"allowed_relations"}) _PROFILE_KEYS = frozenset( {"id", "families", "statuses", "required_nodes", "token_budget", "dependency_depth"} ) _OPERATIONS = frozenset({"create", "update", "move", "delete"}) def project_root_fingerprint(root: Path) -> str: return hashlib.sha256(str(root).encode()).hexdigest()[:16] def _load_descriptor(root: Path) -> ProjectDescriptor: descriptor_path = root / ".docforge" / "project.toml" if not descriptor_path.is_file(): raise DocForgeError("missing_config", "Missing .docforge/project.toml") try: descriptor_bytes = descriptor_path.read_bytes() document = tomllib.loads(descriptor_bytes.decode("utf-8")) except UnicodeDecodeError as error: raise DocForgeError("invalid_config", "Project descriptor is not UTF-8") from error except tomllib.TOMLDecodeError as error: raise DocForgeError("invalid_config", f"Invalid project descriptor: {error}") from error unknown_descriptor = sorted(set(document) - _DESCRIPTOR_KEYS) if unknown_descriptor: raise DocForgeError( "invalid_config", "Project descriptor has unknown fields", fields=unknown_descriptor ) if document.get("schema_version") != 1: raise DocForgeError("invalid_config", "Project descriptor schema_version must be 1") project_id = require_string(document, "project_id", descriptor_path) if ID_PATTERN.fullmatch(project_id) is None: raise DocForgeError( "invalid_config", "project_id is not a stable ID", project_id=project_id ) title = require_string(document, "title", descriptor_path) adapter = require_string(document, "adapter", descriptor_path) if adapter != "generic": raise DocForgeError("unsupported_adapter", "DFG-1 supports only the generic adapter") sources = document.get("sources") derived = document.get("derived") changesets = document.get("changesets") graph = document.get("graph") if ( not isinstance(sources, dict) or not isinstance(derived, dict) or not isinstance(changesets, dict) or not isinstance(graph, dict) ): raise DocForgeError( "invalid_config", "sources, derived, changesets, and graph tables are required" ) for table, allowed, name in ( (sources, _SOURCE_KEYS, "sources"), (derived, _DERIVED_KEYS, "derived"), (changesets, _CHANGESET_KEYS, "changesets"), (graph, _GRAPH_KEYS, "graph"), ): unknown = sorted(set(table) - allowed) if unknown: raise DocForgeError("invalid_config", f"{name} has unknown fields", fields=unknown) content_roots = tuple( confined_path( root, item, field="sources.content_roots", must_exist=True, expected="directory", ) for item in string_list( sources.get("content_roots"), key="sources.content_roots", source=descriptor_path ) ) if len(content_roots) != len(set(content_roots)): raise DocForgeError("invalid_config", "sources.content_roots resolve to duplicates") authority_files = tuple( confined_path( root, item, field="sources.authority_files", must_exist=True, expected="file", ) for item in string_list( sources.get("authority_files", []), key="sources.authority_files", source=descriptor_path, ) ) cache_root = confined_path( root, derived.get("cache_root"), field="derived.cache_root", must_exist=False ) index_path = confined_path(root, derived.get("index"), field="derived.index", must_exist=False) changeset_root = confined_path( root, changesets.get("root"), field="changesets.root", must_exist=False ) if not index_path.is_relative_to(cache_root): raise DocForgeError("invalid_config", "derived.index must be inside derived.cache_root") for content_root in content_roots: if ( content_root == cache_root or content_root.is_relative_to(cache_root) or cache_root.is_relative_to(content_root) ): raise DocForgeError("invalid_config", "Canonical content and cache must not overlap") if ( content_root == changeset_root or content_root.is_relative_to(changeset_root) or changeset_root.is_relative_to(content_root) ): raise DocForgeError( "invalid_config", "Canonical content and changesets must not overlap" ) if ( cache_root == changeset_root or cache_root.is_relative_to(changeset_root) or changeset_root.is_relative_to(cache_root) ): raise DocForgeError("invalid_config", "Cache and changesets must not overlap") writer_documents = changesets.get("writers") if not isinstance(writer_documents, list): raise DocForgeError("invalid_config", "changesets.writers must be an array of tables") proposal_writers: list[ProposalWriter] = [] writer_ids: set[str] = set() for writer in writer_documents: if not isinstance(writer, dict): raise DocForgeError("invalid_config", "Each changeset writer must be a table") unknown_writer = sorted(set(writer) - _WRITER_KEYS) if unknown_writer: raise DocForgeError( "invalid_config", "Changeset writer has unknown fields", fields=unknown_writer ) writer_id = require_string(writer, "id", descriptor_path) if ID_PATTERN.fullmatch(writer_id) is None or writer_id in writer_ids: raise DocForgeError( "invalid_config", "Changeset writer ID is invalid or duplicated", id=writer_id ) writer_ids.add(writer_id) families = string_list( writer.get("families"), key="changesets.writer.families", source=descriptor_path ) operations = string_list( writer.get("operations"), key="changesets.writer.operations", source=descriptor_path ) if not families: raise DocForgeError("invalid_config", "Changeset writer needs at least one family") invalid_operations = sorted(set(operations) - _OPERATIONS) if not operations or invalid_operations: raise DocForgeError( "invalid_config", "Changeset writer operations are empty or invalid", operations=invalid_operations, ) proposal_writers.append( ProposalWriter( writer_id=writer_id, families=tuple(sorted(families)), operations=tuple(sorted(operations)), ) ) allowed_relations = string_list( graph.get("allowed_relations"), key="graph.allowed_relations", source=descriptor_path ) if not allowed_relations: raise DocForgeError("invalid_config", "At least one relationship type is required") for relation in allowed_relations: if ID_PATTERN.fullmatch(relation) is None: raise DocForgeError("invalid_config", "Relationship type is invalid", relation=relation) limit_values = document.get("limits", {}) if not isinstance(limit_values, dict): raise DocForgeError("invalid_config", "limits must be a table") defaults = Limits() unknown_limits = sorted(set(limit_values) - set(defaults.__dataclass_fields__)) if unknown_limits: raise DocForgeError("invalid_config", "limits has unknown fields", fields=unknown_limits) limits = Limits( **{ field: positive_int(limit_values.get(field, getattr(defaults, field)), field) for field in defaults.__dataclass_fields__ } ) render = load_render_config( root, document.get("render"), descriptor_path=descriptor_path, content_roots=content_roots, authority_files=authority_files, cache_root=cache_root, index_path=index_path, changeset_root=changeset_root, limits=limits, ) profile_documents = document.get("profiles", []) if not isinstance(profile_documents, list): raise DocForgeError("invalid_config", "profiles must be an array of tables") profiles: list[ContextProfile] = [] profile_ids: set[str] = set() for profile in profile_documents: if not isinstance(profile, dict): raise DocForgeError("invalid_config", "Each profile must be a table") unknown_profile = sorted(set(profile) - _PROFILE_KEYS) if unknown_profile: raise DocForgeError( "invalid_config", "Profile has unknown fields", fields=unknown_profile ) profile_id = require_string(profile, "id", descriptor_path) if ID_PATTERN.fullmatch(profile_id) is None or profile_id in profile_ids: raise DocForgeError( "invalid_config", "Profile ID is invalid or duplicated", id=profile_id ) profile_ids.add(profile_id) token_budget = positive_int(profile.get("token_budget", 8_000), "profile.token_budget") dependency_depth = positive_int( profile.get("dependency_depth", 1), "profile.dependency_depth", allow_zero=True ) if token_budget > limits.max_context_tokens: raise DocForgeError("invalid_config", "Profile token budget exceeds project limit") if dependency_depth > limits.max_traversal_depth: raise DocForgeError("invalid_config", "Profile dependency depth exceeds project limit") profiles.append( ContextProfile( profile_id=profile_id, families=string_list( profile.get("families", []), key="profile.families", source=descriptor_path ), statuses=string_list( profile.get("statuses", []), key="profile.statuses", source=descriptor_path ), required_nodes=string_list( profile.get("required_nodes", []), key="profile.required_nodes", source=descriptor_path, ), token_budget=token_budget, dependency_depth=dependency_depth, ) ) return ProjectDescriptor( schema_version=1, project_id=project_id, title=title, adapter=adapter, root=root, descriptor_path=descriptor_path, descriptor_hash=hashlib.sha256(descriptor_bytes).hexdigest(), content_roots=content_roots, authority_files=authority_files, cache_root=cache_root, index_path=index_path, changeset_root=changeset_root, proposal_writers=tuple(sorted(proposal_writers, key=lambda writer: writer.writer_id)), render=render, allowed_relations=allowed_relations, profiles=tuple(profiles), limits=limits, ) def _markdown_record(path: Path, text: str) -> tuple[dict[str, Any], str]: lines = text.splitlines() if not lines or lines[0] != "+++": raise DocForgeError( "invalid_source", f"{path.name}: Markdown must start with TOML metadata" ) try: close = lines.index("+++", 1) except ValueError as error: raise DocForgeError( "invalid_source", f"{path.name}: metadata block is not closed" ) from error try: metadata = tomllib.loads("\n".join(lines[1:close])) except tomllib.TOMLDecodeError as error: raise DocForgeError("invalid_source", f"{path.name}: invalid metadata: {error}") from error return metadata, "\n".join(lines[close + 1 :]).strip() def validated_node_from_record( record: dict[str, Any], *, content: str, source: Path, relative_source: str, relations: tuple[str, ...], hash_bytes: bytes, ) -> tuple[Node, tuple[Edge, ...]]: if record.get("schema_version") != 1: raise DocForgeError("invalid_source", f"{source.name}: node schema_version must be 1") unknown = set(record) - _CORE_METADATA - set(relations) if unknown: raise DocForgeError( "invalid_source", f"{source.name}: unknown metadata", keys=sorted(unknown) ) node_id = require_string(record, "id", source) if ID_PATTERN.fullmatch(node_id) is None: raise DocForgeError("invalid_source", f"{source.name}: node ID is invalid", id=node_id) authority = require_string(record, "authority", source) if authority not in _AUTHORITIES: raise DocForgeError( "invalid_source", f"{source.name}: authority is invalid", authority=authority ) tags = string_list(record.get("tags", []), key="tags", source=source) anchor = record.get("source_anchor") if anchor is not None and (not isinstance(anchor, str) or not anchor): raise DocForgeError("invalid_source", f"{source.name}: source_anchor must be a string") summary = require_string(record, "summary", source) if not content: raise DocForgeError("invalid_source", f"{source.name}: node content is empty", id=node_id) node = Node( node_id=node_id, title=require_string(record, "title", source), family=require_string(record, "family", source), authority=authority, status=require_string(record, "status", source), tags=tags, summary=summary, content=content, source_path=relative_source, source_anchor=anchor, content_hash=hashlib.sha256(hash_bytes).hexdigest(), ) edges = tuple( Edge(node_id, relation, target) for relation in relations for target in string_list(record.get(relation, []), key=relation, source=source) ) return node, edges def _load_source_file( descriptor: ProjectDescriptor, path: Path, raw: bytes ) -> tuple[tuple[Node, ...], tuple[Edge, ...]]: if len(raw) > descriptor.limits.max_source_bytes: raise DocForgeError( "source_too_large", "Canonical source exceeds configured limit", source=path.name ) try: text = raw.decode("utf-8") except UnicodeDecodeError as error: raise DocForgeError("invalid_source", f"{path.name}: source is not UTF-8") from error relative = path.relative_to(descriptor.root).as_posix() if path.suffix == ".md": record, content = _markdown_record(path, text) node, edges = validated_node_from_record( record, content=content, source=path, relative_source=relative, relations=descriptor.allowed_relations, hash_bytes=raw, ) return (node,), edges if path.suffix == ".toml": try: document = tomllib.loads(text) except tomllib.TOMLDecodeError as error: raise DocForgeError("invalid_source", f"{path.name}: invalid TOML: {error}") from error records = document.get("nodes") if set(document) != {"nodes"} or not isinstance(records, list) or not records: raise DocForgeError("invalid_source", f"{path.name}: TOML sources require [[nodes]]") nodes: list[Node] = [] edges: list[Edge] = [] for index, record in enumerate(records): if not isinstance(record, dict): raise DocForgeError("invalid_source", f"{path.name}: nodes must be tables") content = record.get("content") if not isinstance(content, str): raise DocForgeError( "invalid_source", f"{path.name}: TOML node content must be text" ) canonical = json.dumps(record, sort_keys=True, separators=(",", ":")).encode() node, node_edges = validated_node_from_record( record, content=content.strip(), source=path, relative_source=relative, relations=descriptor.allowed_relations, hash_bytes=canonical, ) nodes.append(replace(node, source_anchor=node.source_anchor or f"node-{index + 1}")) edges.extend(node_edges) return tuple(nodes), tuple(edges) raise DocForgeError("invalid_source", "Unsupported canonical source type", source=relative) def validate_graph(nodes: tuple[Node, ...], edges: tuple[Edge, ...]) -> None: node_ids = {node.node_id for node in nodes} if len(node_ids) != len(nodes): counts = Counter(node.node_id for node in nodes) duplicates = sorted(node_id for node_id, count in counts.items() if count > 1) raise DocForgeError("duplicate_node", "Stable node IDs must be unique", ids=duplicates) edge_keys = {(edge.source_id, edge.relation, edge.target_id) for edge in edges} if len(edge_keys) != len(edges): raise DocForgeError("duplicate_edge", "Relationships must be unique") missing = sorted({edge.target_id for edge in edges if edge.target_id not in node_ids}) if missing: raise DocForgeError("broken_edge", "Relationships target missing nodes", targets=missing) dependencies = { node_id: sorted( edge.target_id for edge in edges if edge.source_id == node_id and edge.relation == "depends_on" ) for node_id in sorted(node_ids) } visiting: set[str] = set() visited: set[str] = set() def visit(node_id: str, trail: tuple[str, ...]) -> None: if node_id in visiting: raise DocForgeError( "dependency_cycle", "depends_on relationships contain a cycle", path=(*trail, node_id), ) if node_id in visited: return visiting.add(node_id) for target in dependencies[node_id]: visit(target, (*trail, node_id)) visiting.remove(node_id) visited.add(node_id) for node_id in sorted(node_ids): visit(node_id, ()) def _revision(root: Path) -> str: try: result = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=root, check=False, capture_output=True, text=True, timeout=2, ) except (OSError, subprocess.TimeoutExpired): return "unversioned" return ( result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unversioned" ) class Project: """One immutable project binding for loading and querying canonical documentation.""" def __init__(self, descriptor: ProjectDescriptor) -> None: self.descriptor = descriptor @classmethod def open(cls, project_root: str | Path) -> Project: try: root = Path(project_root).expanduser().resolve(strict=True) except OSError as error: raise DocForgeError("invalid_root", "Project root does not exist") from error if not root.is_dir(): raise DocForgeError("invalid_root", "Project root must be a directory") return cls(_load_descriptor(root)) def load(self) -> ProjectSnapshot: descriptor_bytes = self.descriptor.descriptor_path.read_bytes() if hashlib.sha256(descriptor_bytes).hexdigest() != self.descriptor.descriptor_hash: raise DocForgeError( "source_changed", "Project descriptor changed after the project was opened" ) ordered_sources = self.canonical_source_paths() captured = { path: path.read_bytes() for path in ( self.descriptor.descriptor_path, *self.descriptor.authority_files, *ordered_sources, ) } nodes: list[Node] = [] edges: list[Edge] = [] for path in ordered_sources: source_nodes, source_edges = _load_source_file(self.descriptor, path, captured[path]) nodes.extend(source_nodes) edges.extend(source_edges) if len(nodes) > self.descriptor.limits.max_nodes: raise DocForgeError("node_limit", "Project exceeds configured node limit") ordered_nodes = tuple(sorted(nodes, key=lambda node: node.node_id)) ordered_edges = tuple( sorted(edges, key=lambda edge: (edge.source_id, edge.relation, edge.target_id)) ) validate_graph(ordered_nodes, ordered_edges) node_ids = {node.node_id for node in ordered_nodes} for profile in self.descriptor.profiles: missing = sorted(set(profile.required_nodes) - node_ids) if missing: raise DocForgeError( "invalid_config", "Context profile requires missing nodes", nodes=missing ) if self.canonical_source_paths() != ordered_sources: raise DocForgeError("source_changed", "Canonical source set changed during loading") for path, raw in captured.items(): if not path.is_file() or path.read_bytes() != raw: raise DocForgeError( "source_changed", "Canonical source changed during loading", source=path.relative_to(self.descriptor.root).as_posix(), ) digest = hashlib.sha256() for path in sorted( captured, key=lambda item: item.relative_to(self.descriptor.root).as_posix() ): relative = path.relative_to(self.descriptor.root).as_posix() digest.update(relative.encode()) digest.update(b"\0") digest.update(hashlib.sha256(captured[path]).digest()) digest.update(b"docforge-core:0.3.0:index:1") return ProjectSnapshot( descriptor=self.descriptor, nodes=ordered_nodes, edges=ordered_edges, source_hash=digest.hexdigest(), revision=_revision(self.descriptor.root), ) def canonical_source_paths(self) -> tuple[Path, ...]: """Return the deterministic confined canonical source set.""" source_paths: set[Path] = set() for content_root in self.descriptor.content_roots: for path in content_root.rglob("*"): if path.suffix not in {".md", ".toml"} or not path.is_file(): continue resolved = path.resolve() if not resolved.is_relative_to(self.descriptor.root): raise DocForgeError( "path_escape", "Canonical source resolves outside project root" ) source_paths.add(resolved) ordered_sources = sorted( source_paths, key=lambda path: path.relative_to(self.descriptor.root).as_posix() ) if not ordered_sources: raise DocForgeError("empty_project", "No canonical Markdown or TOML sources were found") return tuple(ordered_sources)