feat: establish read-only DocForge MCP foundation

src/docforge/project.py

@@ -0,0 +1,576 @@
+"""Project discovery, root confinement, canonical loading, and graph validation."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import subprocess
+import tomllib
+from collections import Counter
+from dataclasses import replace
+from pathlib import Path
+from typing import Any
+
+from .errors import DocForgeError
+from .models import (
+    ContextProfile,
+    Edge,
+    Limits,
+    Node,
+    ProjectDescriptor,
+    ProjectSnapshot,
+)
+
+_ID_PATTERN = re.compile(r"[a-z0-9][a-z0-9._-]{1,127}")
+_AUTHORITIES = frozenset({"authoritative", "approved_plan", "derived", "proposal", "historical"})
+_SECRET_PARTS = frozenset({".git", ".ssh", ".gnupg", "secrets", "credentials"})
+_CORE_METADATA = frozenset(
+    {
+        "schema_version",
+        "id",
+        "title",
+        "family",
+        "authority",
+        "status",
+        "tags",
+        "summary",
+        "source_anchor",
+        "content",
+    }
+)
+_DESCRIPTOR_KEYS = frozenset(
+    {
+        "schema_version",
+        "project_id",
+        "title",
+        "adapter",
+        "sources",
+        "derived",
+        "graph",
+        "limits",
+        "profiles",
+    }
+)
+_SOURCE_KEYS = frozenset({"content_roots", "authority_files"})
+_DERIVED_KEYS = frozenset({"cache_root", "index"})
+_GRAPH_KEYS = frozenset({"allowed_relations"})
+_PROFILE_KEYS = frozenset(
+    {"id", "families", "statuses", "required_nodes", "token_budget", "dependency_depth"}
+)
+
+
+def project_root_fingerprint(root: Path) -> str:
+    return hashlib.sha256(str(root).encode()).hexdigest()[:16]
+
+
+def _require_string(document: dict[str, Any], key: str, source: Path) -> str:
+    value = document.get(key)
+    if not isinstance(value, str) or not value.strip():
+        raise DocForgeError("invalid_source", f"{source.name}: {key} must be a non-empty string")
+    return value.strip()
+
+
+def _string_list(value: object, *, key: str, source: Path) -> tuple[str, ...]:
+    if not isinstance(value, list) or any(not isinstance(item, str) or not item for item in value):
+        raise DocForgeError("invalid_source", f"{source.name}: {key} must be a string list")
+    if len(value) != len(set(value)):
+        raise DocForgeError("invalid_source", f"{source.name}: {key} contains duplicates")
+    return tuple(value)
+
+
+def _confined_path(
+    root: Path,
+    raw: object,
+    *,
+    field: str,
+    must_exist: bool,
+    expected: str | None = None,
+) -> Path:
+    if not isinstance(raw, str) or not raw:
+        raise DocForgeError("invalid_config", f"{field} must be a non-empty relative path")
+    relative = Path(raw)
+    if relative.is_absolute() or ".." in relative.parts:
+        raise DocForgeError("path_escape", f"{field} must stay inside the project root", path=raw)
+    if any(part.lower() in _SECRET_PARTS for part in relative.parts):
+        raise DocForgeError("secret_path", f"{field} may not reference a protected path", path=raw)
+    resolved = (root / relative).resolve(strict=False)
+    if not resolved.is_relative_to(root):
+        raise DocForgeError("path_escape", f"{field} resolves outside the project root", path=raw)
+    if must_exist and not resolved.exists():
+        raise DocForgeError("missing_path", f"{field} does not exist", path=raw)
+    if expected == "file" and must_exist and not resolved.is_file():
+        raise DocForgeError("invalid_path", f"{field} must identify a file", path=raw)
+    if expected == "directory" and must_exist and not resolved.is_dir():
+        raise DocForgeError("invalid_path", f"{field} must identify a directory", path=raw)
+    return resolved
+
+
+def _load_descriptor(root: Path) -> ProjectDescriptor:
+    descriptor_path = root / ".docforge" / "project.toml"
+    if not descriptor_path.is_file():
+        raise DocForgeError("missing_config", "Missing .docforge/project.toml")
+    try:
+        descriptor_bytes = descriptor_path.read_bytes()
+        document = tomllib.loads(descriptor_bytes.decode("utf-8"))
+    except UnicodeDecodeError as error:
+        raise DocForgeError("invalid_config", "Project descriptor is not UTF-8") from error
+    except tomllib.TOMLDecodeError as error:
+        raise DocForgeError("invalid_config", f"Invalid project descriptor: {error}") from error
+
+    unknown_descriptor = sorted(set(document) - _DESCRIPTOR_KEYS)
+    if unknown_descriptor:
+        raise DocForgeError(
+            "invalid_config", "Project descriptor has unknown fields", fields=unknown_descriptor
+        )
+
+    if document.get("schema_version") != 1:
+        raise DocForgeError("invalid_config", "Project descriptor schema_version must be 1")
+    project_id = _require_string(document, "project_id", descriptor_path)
+    if _ID_PATTERN.fullmatch(project_id) is None:
+        raise DocForgeError(
+            "invalid_config", "project_id is not a stable ID", project_id=project_id
+        )
+    title = _require_string(document, "title", descriptor_path)
+    adapter = _require_string(document, "adapter", descriptor_path)
+    if adapter != "generic":
+        raise DocForgeError("unsupported_adapter", "DFG-1 supports only the generic adapter")
+
+    sources = document.get("sources")
+    derived = document.get("derived")
+    graph = document.get("graph")
+    if (
+        not isinstance(sources, dict)
+        or not isinstance(derived, dict)
+        or not isinstance(graph, dict)
+    ):
+        raise DocForgeError("invalid_config", "sources, derived, and graph tables are required")
+    for table, allowed, name in (
+        (sources, _SOURCE_KEYS, "sources"),
+        (derived, _DERIVED_KEYS, "derived"),
+        (graph, _GRAPH_KEYS, "graph"),
+    ):
+        unknown = sorted(set(table) - allowed)
+        if unknown:
+            raise DocForgeError("invalid_config", f"{name} has unknown fields", fields=unknown)
+    content_roots = tuple(
+        _confined_path(
+            root,
+            item,
+            field="sources.content_roots",
+            must_exist=True,
+            expected="directory",
+        )
+        for item in _string_list(
+            sources.get("content_roots"), key="sources.content_roots", source=descriptor_path
+        )
+    )
+    if len(content_roots) != len(set(content_roots)):
+        raise DocForgeError("invalid_config", "sources.content_roots resolve to duplicates")
+    authority_files = tuple(
+        _confined_path(
+            root,
+            item,
+            field="sources.authority_files",
+            must_exist=True,
+            expected="file",
+        )
+        for item in _string_list(
+            sources.get("authority_files", []),
+            key="sources.authority_files",
+            source=descriptor_path,
+        )
+    )
+    cache_root = _confined_path(
+        root, derived.get("cache_root"), field="derived.cache_root", must_exist=False
+    )
+    index_path = _confined_path(root, derived.get("index"), field="derived.index", must_exist=False)
+    if not index_path.is_relative_to(cache_root):
+        raise DocForgeError("invalid_config", "derived.index must be inside derived.cache_root")
+    for content_root in content_roots:
+        if (
+            content_root == cache_root
+            or content_root.is_relative_to(cache_root)
+            or cache_root.is_relative_to(content_root)
+        ):
+            raise DocForgeError("invalid_config", "Canonical content and cache must not overlap")
+
+    allowed_relations = _string_list(
+        graph.get("allowed_relations"), key="graph.allowed_relations", source=descriptor_path
+    )
+    if not allowed_relations:
+        raise DocForgeError("invalid_config", "At least one relationship type is required")
+    for relation in allowed_relations:
+        if _ID_PATTERN.fullmatch(relation) is None:
+            raise DocForgeError("invalid_config", "Relationship type is invalid", relation=relation)
+
+    limit_values = document.get("limits", {})
+    if not isinstance(limit_values, dict):
+        raise DocForgeError("invalid_config", "limits must be a table")
+    defaults = Limits()
+    unknown_limits = sorted(set(limit_values) - set(defaults.__dataclass_fields__))
+    if unknown_limits:
+        raise DocForgeError("invalid_config", "limits has unknown fields", fields=unknown_limits)
+    limits = Limits(
+        **{
+            field: _positive_int(limit_values.get(field, getattr(defaults, field)), field)
+            for field in defaults.__dataclass_fields__
+        }
+    )
+
+    profile_documents = document.get("profiles", [])
+    if not isinstance(profile_documents, list):
+        raise DocForgeError("invalid_config", "profiles must be an array of tables")
+    profiles: list[ContextProfile] = []
+    profile_ids: set[str] = set()
+    for profile in profile_documents:
+        if not isinstance(profile, dict):
+            raise DocForgeError("invalid_config", "Each profile must be a table")
+        unknown_profile = sorted(set(profile) - _PROFILE_KEYS)
+        if unknown_profile:
+            raise DocForgeError(
+                "invalid_config", "Profile has unknown fields", fields=unknown_profile
+            )
+        profile_id = _require_string(profile, "id", descriptor_path)
+        if _ID_PATTERN.fullmatch(profile_id) is None or profile_id in profile_ids:
+            raise DocForgeError(
+                "invalid_config", "Profile ID is invalid or duplicated", id=profile_id
+            )
+        profile_ids.add(profile_id)
+        token_budget = _positive_int(profile.get("token_budget", 8_000), "profile.token_budget")
+        dependency_depth = _positive_int(
+            profile.get("dependency_depth", 1), "profile.dependency_depth", allow_zero=True
+        )
+        if token_budget > limits.max_context_tokens:
+            raise DocForgeError("invalid_config", "Profile token budget exceeds project limit")
+        if dependency_depth > limits.max_traversal_depth:
+            raise DocForgeError("invalid_config", "Profile dependency depth exceeds project limit")
+        profiles.append(
+            ContextProfile(
+                profile_id=profile_id,
+                families=_string_list(
+                    profile.get("families", []), key="profile.families", source=descriptor_path
+                ),
+                statuses=_string_list(
+                    profile.get("statuses", []), key="profile.statuses", source=descriptor_path
+                ),
+                required_nodes=_string_list(
+                    profile.get("required_nodes", []),
+                    key="profile.required_nodes",
+                    source=descriptor_path,
+                ),
+                token_budget=token_budget,
+                dependency_depth=dependency_depth,
+            )
+        )
+
+    return ProjectDescriptor(
+        schema_version=1,
+        project_id=project_id,
+        title=title,
+        adapter=adapter,
+        root=root,
+        descriptor_path=descriptor_path,
+        descriptor_hash=hashlib.sha256(descriptor_bytes).hexdigest(),
+        content_roots=content_roots,
+        authority_files=authority_files,
+        cache_root=cache_root,
+        index_path=index_path,
+        allowed_relations=allowed_relations,
+        profiles=tuple(profiles),
+        limits=limits,
+    )
+
+
+def _positive_int(value: object, field: str, *, allow_zero: bool = False) -> int:
+    minimum = 0 if allow_zero else 1
+    if not isinstance(value, int) or isinstance(value, bool) or value < minimum:
+        raise DocForgeError("invalid_config", f"{field} must be an integer >= {minimum}")
+    return value
+
+
+def _markdown_record(path: Path, text: str) -> tuple[dict[str, Any], str]:
+    lines = text.splitlines()
+    if not lines or lines[0] != "+++":
+        raise DocForgeError(
+            "invalid_source", f"{path.name}: Markdown must start with TOML metadata"
+        )
+    try:
+        close = lines.index("+++", 1)
+    except ValueError as error:
+        raise DocForgeError(
+            "invalid_source", f"{path.name}: metadata block is not closed"
+        ) from error
+    try:
+        metadata = tomllib.loads("\n".join(lines[1:close]))
+    except tomllib.TOMLDecodeError as error:
+        raise DocForgeError("invalid_source", f"{path.name}: invalid metadata: {error}") from error
+    return metadata, "\n".join(lines[close + 1 :]).strip()
+
+
+def _node_from_record(
+    record: dict[str, Any],
+    *,
+    content: str,
+    source: Path,
+    relative_source: str,
+    relations: tuple[str, ...],
+    hash_bytes: bytes,
+) -> tuple[Node, tuple[Edge, ...]]:
+    if record.get("schema_version") != 1:
+        raise DocForgeError("invalid_source", f"{source.name}: node schema_version must be 1")
+    unknown = set(record) - _CORE_METADATA - set(relations)
+    if unknown:
+        raise DocForgeError(
+            "invalid_source", f"{source.name}: unknown metadata", keys=sorted(unknown)
+        )
+    node_id = _require_string(record, "id", source)
+    if _ID_PATTERN.fullmatch(node_id) is None:
+        raise DocForgeError("invalid_source", f"{source.name}: node ID is invalid", id=node_id)
+    authority = _require_string(record, "authority", source)
+    if authority not in _AUTHORITIES:
+        raise DocForgeError(
+            "invalid_source", f"{source.name}: authority is invalid", authority=authority
+        )
+    tags = _string_list(record.get("tags", []), key="tags", source=source)
+    anchor = record.get("source_anchor")
+    if anchor is not None and (not isinstance(anchor, str) or not anchor):
+        raise DocForgeError("invalid_source", f"{source.name}: source_anchor must be a string")
+    summary = _require_string(record, "summary", source)
+    if not content:
+        raise DocForgeError("invalid_source", f"{source.name}: node content is empty", id=node_id)
+    node = Node(
+        node_id=node_id,
+        title=_require_string(record, "title", source),
+        family=_require_string(record, "family", source),
+        authority=authority,
+        status=_require_string(record, "status", source),
+        tags=tags,
+        summary=summary,
+        content=content,
+        source_path=relative_source,
+        source_anchor=anchor,
+        content_hash=hashlib.sha256(hash_bytes).hexdigest(),
+    )
+    edges = tuple(
+        Edge(node_id, relation, target)
+        for relation in relations
+        for target in _string_list(record.get(relation, []), key=relation, source=source)
+    )
+    return node, edges
+
+
+def _load_source_file(
+    descriptor: ProjectDescriptor, path: Path, raw: bytes
+) -> tuple[tuple[Node, ...], tuple[Edge, ...]]:
+    if len(raw) > descriptor.limits.max_source_bytes:
+        raise DocForgeError(
+            "source_too_large", "Canonical source exceeds configured limit", source=path.name
+        )
+    try:
+        text = raw.decode("utf-8")
+    except UnicodeDecodeError as error:
+        raise DocForgeError("invalid_source", f"{path.name}: source is not UTF-8") from error
+    relative = path.relative_to(descriptor.root).as_posix()
+    if path.suffix == ".md":
+        record, content = _markdown_record(path, text)
+        node, edges = _node_from_record(
+            record,
+            content=content,
+            source=path,
+            relative_source=relative,
+            relations=descriptor.allowed_relations,
+            hash_bytes=raw,
+        )
+        return (node,), edges
+    if path.suffix == ".toml":
+        try:
+            document = tomllib.loads(text)
+        except tomllib.TOMLDecodeError as error:
+            raise DocForgeError("invalid_source", f"{path.name}: invalid TOML: {error}") from error
+        records = document.get("nodes")
+        if set(document) != {"nodes"} or not isinstance(records, list) or not records:
+            raise DocForgeError("invalid_source", f"{path.name}: TOML sources require [[nodes]]")
+        nodes: list[Node] = []
+        edges: list[Edge] = []
+        for index, record in enumerate(records):
+            if not isinstance(record, dict):
+                raise DocForgeError("invalid_source", f"{path.name}: nodes must be tables")
+            content = record.get("content")
+            if not isinstance(content, str):
+                raise DocForgeError(
+                    "invalid_source", f"{path.name}: TOML node content must be text"
+                )
+            canonical = json.dumps(record, sort_keys=True, separators=(",", ":")).encode()
+            node, node_edges = _node_from_record(
+                record,
+                content=content.strip(),
+                source=path,
+                relative_source=relative,
+                relations=descriptor.allowed_relations,
+                hash_bytes=canonical,
+            )
+            nodes.append(replace(node, source_anchor=node.source_anchor or f"node-{index + 1}"))
+            edges.extend(node_edges)
+        return tuple(nodes), tuple(edges)
+    raise DocForgeError("invalid_source", "Unsupported canonical source type", source=relative)
+
+
+def _validate_graph(nodes: tuple[Node, ...], edges: tuple[Edge, ...]) -> None:
+    node_ids = {node.node_id for node in nodes}
+    if len(node_ids) != len(nodes):
+        counts = Counter(node.node_id for node in nodes)
+        duplicates = sorted(node_id for node_id, count in counts.items() if count > 1)
+        raise DocForgeError("duplicate_node", "Stable node IDs must be unique", ids=duplicates)
+    edge_keys = {(edge.source_id, edge.relation, edge.target_id) for edge in edges}
+    if len(edge_keys) != len(edges):
+        raise DocForgeError("duplicate_edge", "Relationships must be unique")
+    missing = sorted({edge.target_id for edge in edges if edge.target_id not in node_ids})
+    if missing:
+        raise DocForgeError("broken_edge", "Relationships target missing nodes", targets=missing)
+
+    dependencies = {
+        node_id: sorted(
+            edge.target_id
+            for edge in edges
+            if edge.source_id == node_id and edge.relation == "depends_on"
+        )
+        for node_id in sorted(node_ids)
+    }
+    visiting: set[str] = set()
+    visited: set[str] = set()
+
+    def visit(node_id: str, trail: tuple[str, ...]) -> None:
+        if node_id in visiting:
+            raise DocForgeError(
+                "dependency_cycle",
+                "depends_on relationships contain a cycle",
+                path=(*trail, node_id),
+            )
+        if node_id in visited:
+            return
+        visiting.add(node_id)
+        for target in dependencies[node_id]:
+            visit(target, (*trail, node_id))
+        visiting.remove(node_id)
+        visited.add(node_id)
+
+    for node_id in sorted(node_ids):
+        visit(node_id, ())
+
+
+def _revision(root: Path) -> str:
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            cwd=root,
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=2,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return "unversioned"
+    return (
+        result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unversioned"
+    )
+
+
+class Project:
+    """One immutable project binding for loading and querying canonical documentation."""
+
+    def __init__(self, descriptor: ProjectDescriptor) -> None:
+        self.descriptor = descriptor
+
+    @classmethod
+    def open(cls, project_root: str | Path) -> Project:
+        try:
+            root = Path(project_root).expanduser().resolve(strict=True)
+        except OSError as error:
+            raise DocForgeError("invalid_root", "Project root does not exist") from error
+        if not root.is_dir():
+            raise DocForgeError("invalid_root", "Project root must be a directory")
+        return cls(_load_descriptor(root))
+
+    def load(self) -> ProjectSnapshot:
+        descriptor_bytes = self.descriptor.descriptor_path.read_bytes()
+        if hashlib.sha256(descriptor_bytes).hexdigest() != self.descriptor.descriptor_hash:
+            raise DocForgeError(
+                "source_changed", "Project descriptor changed after the project was opened"
+            )
+        ordered_sources = self._canonical_source_paths()
+        captured = {
+            path: path.read_bytes()
+            for path in (
+                self.descriptor.descriptor_path,
+                *self.descriptor.authority_files,
+                *ordered_sources,
+            )
+        }
+
+        nodes: list[Node] = []
+        edges: list[Edge] = []
+        for path in ordered_sources:
+            source_nodes, source_edges = _load_source_file(self.descriptor, path, captured[path])
+            nodes.extend(source_nodes)
+            edges.extend(source_edges)
+            if len(nodes) > self.descriptor.limits.max_nodes:
+                raise DocForgeError("node_limit", "Project exceeds configured node limit")
+        ordered_nodes = tuple(sorted(nodes, key=lambda node: node.node_id))
+        ordered_edges = tuple(
+            sorted(edges, key=lambda edge: (edge.source_id, edge.relation, edge.target_id))
+        )
+        _validate_graph(ordered_nodes, ordered_edges)
+        node_ids = {node.node_id for node in ordered_nodes}
+        for profile in self.descriptor.profiles:
+            missing = sorted(set(profile.required_nodes) - node_ids)
+            if missing:
+                raise DocForgeError(
+                    "invalid_config", "Context profile requires missing nodes", nodes=missing
+                )
+
+        if self._canonical_source_paths() != ordered_sources:
+            raise DocForgeError("source_changed", "Canonical source set changed during loading")
+        for path, raw in captured.items():
+            if not path.is_file() or path.read_bytes() != raw:
+                raise DocForgeError(
+                    "source_changed",
+                    "Canonical source changed during loading",
+                    source=path.relative_to(self.descriptor.root).as_posix(),
+                )
+
+        digest = hashlib.sha256()
+        for path in sorted(
+            captured, key=lambda item: item.relative_to(self.descriptor.root).as_posix()
+        ):
+            relative = path.relative_to(self.descriptor.root).as_posix()
+            digest.update(relative.encode())
+            digest.update(b"\0")
+            digest.update(hashlib.sha256(captured[path]).digest())
+        digest.update(b"docforge-core:0.1.0:index:1")
+        return ProjectSnapshot(
+            descriptor=self.descriptor,
+            nodes=ordered_nodes,
+            edges=ordered_edges,
+            source_hash=digest.hexdigest(),
+            revision=_revision(self.descriptor.root),
+        )
+
+    def _canonical_source_paths(self) -> tuple[Path, ...]:
+        source_paths: set[Path] = set()
+        for content_root in self.descriptor.content_roots:
+            for path in content_root.rglob("*"):
+                if path.suffix not in {".md", ".toml"} or not path.is_file():
+                    continue
+                resolved = path.resolve()
+                if not resolved.is_relative_to(self.descriptor.root):
+                    raise DocForgeError(
+                        "path_escape", "Canonical source resolves outside project root"
+                    )
+                source_paths.add(resolved)
+        ordered_sources = sorted(
+            source_paths, key=lambda path: path.relative_to(self.descriptor.root).as_posix()
+        )
+        if not ordered_sources:
+            raise DocForgeError("empty_project", "No canonical Markdown or TOML sources were found")
+        return tuple(ordered_sources)