DocForge2/src/docforge/project.py

"""Project discovery, root confinement, canonical loading, and graph validation."""

from __future__ import annotations

import hashlib
import json
import subprocess
import tomllib
from collections import Counter
from dataclasses import replace
from pathlib import Path
from typing import Any

from .config_validation import ID_PATTERN, confined_path, positive_int, require_string, string_list
from .errors import DocForgeError
from .models import (
    ContextProfile,
    Edge,
    Limits,
    Node,
    ProjectDescriptor,
    ProjectSnapshot,
    ProposalWriter,
)
from .render_config import load_render_config

_AUTHORITIES = frozenset({"authoritative", "approved_plan", "derived", "proposal", "historical"})
_CORE_METADATA = frozenset(
    {
        "schema_version",
        "id",
        "title",
        "family",
        "authority",
        "status",
        "tags",
        "summary",
        "source_anchor",
        "content",
    }
)
_DESCRIPTOR_KEYS = frozenset(
    {
        "schema_version",
        "project_id",
        "title",
        "adapter",
        "sources",
        "derived",
        "changesets",
        "render",
        "graph",
        "limits",
        "profiles",
    }
)
_SOURCE_KEYS = frozenset({"content_roots", "authority_files"})
_DERIVED_KEYS = frozenset({"cache_root", "index"})
_CHANGESET_KEYS = frozenset({"root", "writers"})
_WRITER_KEYS = frozenset({"id", "families", "operations"})
_GRAPH_KEYS = frozenset({"allowed_relations"})
_PROFILE_KEYS = frozenset(
    {"id", "families", "statuses", "required_nodes", "token_budget", "dependency_depth"}
)
_OPERATIONS = frozenset({"create", "update", "move", "delete"})


def project_root_fingerprint(root: Path) -> str:
    return hashlib.sha256(str(root).encode()).hexdigest()[:16]


def _load_descriptor(root: Path) -> ProjectDescriptor:
    descriptor_path = root / ".docforge" / "project.toml"
    if not descriptor_path.is_file():
        raise DocForgeError("missing_config", "Missing .docforge/project.toml")
    try:
        descriptor_bytes = descriptor_path.read_bytes()
        document = tomllib.loads(descriptor_bytes.decode("utf-8"))
    except UnicodeDecodeError as error:
        raise DocForgeError("invalid_config", "Project descriptor is not UTF-8") from error
    except tomllib.TOMLDecodeError as error:
        raise DocForgeError("invalid_config", f"Invalid project descriptor: {error}") from error

    unknown_descriptor = sorted(set(document) - _DESCRIPTOR_KEYS)
    if unknown_descriptor:
        raise DocForgeError(
            "invalid_config", "Project descriptor has unknown fields", fields=unknown_descriptor
        )

    if document.get("schema_version") != 1:
        raise DocForgeError("invalid_config", "Project descriptor schema_version must be 1")
    project_id = require_string(document, "project_id", descriptor_path)
    if ID_PATTERN.fullmatch(project_id) is None:
        raise DocForgeError(
            "invalid_config", "project_id is not a stable ID", project_id=project_id
        )
    title = require_string(document, "title", descriptor_path)
    adapter = require_string(document, "adapter", descriptor_path)
    if adapter != "generic":
        raise DocForgeError("unsupported_adapter", "DFG-1 supports only the generic adapter")

    sources = document.get("sources")
    derived = document.get("derived")
    changesets = document.get("changesets")
    graph = document.get("graph")
    if (
        not isinstance(sources, dict)
        or not isinstance(derived, dict)
        or not isinstance(changesets, dict)
        or not isinstance(graph, dict)
    ):
        raise DocForgeError(
            "invalid_config", "sources, derived, changesets, and graph tables are required"
        )
    for table, allowed, name in (
        (sources, _SOURCE_KEYS, "sources"),
        (derived, _DERIVED_KEYS, "derived"),
        (changesets, _CHANGESET_KEYS, "changesets"),
        (graph, _GRAPH_KEYS, "graph"),
    ):
        unknown = sorted(set(table) - allowed)
        if unknown:
            raise DocForgeError("invalid_config", f"{name} has unknown fields", fields=unknown)
    content_roots = tuple(
        confined_path(
            root,
            item,
            field="sources.content_roots",
            must_exist=True,
            expected="directory",
        )
        for item in string_list(
            sources.get("content_roots"), key="sources.content_roots", source=descriptor_path
        )
    )
    if len(content_roots) != len(set(content_roots)):
        raise DocForgeError("invalid_config", "sources.content_roots resolve to duplicates")
    authority_files = tuple(
        confined_path(
            root,
            item,
            field="sources.authority_files",
            must_exist=True,
            expected="file",
        )
        for item in string_list(
            sources.get("authority_files", []),
            key="sources.authority_files",
            source=descriptor_path,
        )
    )
    cache_root = confined_path(
        root, derived.get("cache_root"), field="derived.cache_root", must_exist=False
    )
    index_path = confined_path(root, derived.get("index"), field="derived.index", must_exist=False)
    changeset_root = confined_path(
        root, changesets.get("root"), field="changesets.root", must_exist=False
    )
    if not index_path.is_relative_to(cache_root):
        raise DocForgeError("invalid_config", "derived.index must be inside derived.cache_root")
    for content_root in content_roots:
        if (
            content_root == cache_root
            or content_root.is_relative_to(cache_root)
            or cache_root.is_relative_to(content_root)
        ):
            raise DocForgeError("invalid_config", "Canonical content and cache must not overlap")
        if (
            content_root == changeset_root
            or content_root.is_relative_to(changeset_root)
            or changeset_root.is_relative_to(content_root)
        ):
            raise DocForgeError(
                "invalid_config", "Canonical content and changesets must not overlap"
            )
    if (
        cache_root == changeset_root
        or cache_root.is_relative_to(changeset_root)
        or changeset_root.is_relative_to(cache_root)
    ):
        raise DocForgeError("invalid_config", "Cache and changesets must not overlap")

    writer_documents = changesets.get("writers")
    if not isinstance(writer_documents, list):
        raise DocForgeError("invalid_config", "changesets.writers must be an array of tables")
    proposal_writers: list[ProposalWriter] = []
    writer_ids: set[str] = set()
    for writer in writer_documents:
        if not isinstance(writer, dict):
            raise DocForgeError("invalid_config", "Each changeset writer must be a table")
        unknown_writer = sorted(set(writer) - _WRITER_KEYS)
        if unknown_writer:
            raise DocForgeError(
                "invalid_config", "Changeset writer has unknown fields", fields=unknown_writer
            )
        writer_id = require_string(writer, "id", descriptor_path)
        if ID_PATTERN.fullmatch(writer_id) is None or writer_id in writer_ids:
            raise DocForgeError(
                "invalid_config", "Changeset writer ID is invalid or duplicated", id=writer_id
            )
        writer_ids.add(writer_id)
        families = string_list(
            writer.get("families"), key="changesets.writer.families", source=descriptor_path
        )
        operations = string_list(
            writer.get("operations"), key="changesets.writer.operations", source=descriptor_path
        )
        if not families:
            raise DocForgeError("invalid_config", "Changeset writer needs at least one family")
        invalid_operations = sorted(set(operations) - _OPERATIONS)
        if not operations or invalid_operations:
            raise DocForgeError(
                "invalid_config",
                "Changeset writer operations are empty or invalid",
                operations=invalid_operations,
            )
        proposal_writers.append(
            ProposalWriter(
                writer_id=writer_id,
                families=tuple(sorted(families)),
                operations=tuple(sorted(operations)),
            )
        )

    allowed_relations = string_list(
        graph.get("allowed_relations"), key="graph.allowed_relations", source=descriptor_path
    )
    if not allowed_relations:
        raise DocForgeError("invalid_config", "At least one relationship type is required")
    for relation in allowed_relations:
        if ID_PATTERN.fullmatch(relation) is None:
            raise DocForgeError("invalid_config", "Relationship type is invalid", relation=relation)

    limit_values = document.get("limits", {})
    if not isinstance(limit_values, dict):
        raise DocForgeError("invalid_config", "limits must be a table")
    defaults = Limits()
    unknown_limits = sorted(set(limit_values) - set(defaults.__dataclass_fields__))
    if unknown_limits:
        raise DocForgeError("invalid_config", "limits has unknown fields", fields=unknown_limits)
    limits = Limits(
        **{
            field: positive_int(limit_values.get(field, getattr(defaults, field)), field)
            for field in defaults.__dataclass_fields__
        }
    )

    render = load_render_config(
        root,
        document.get("render"),
        descriptor_path=descriptor_path,
        content_roots=content_roots,
        authority_files=authority_files,
        cache_root=cache_root,
        index_path=index_path,
        changeset_root=changeset_root,
        limits=limits,
    )

    profile_documents = document.get("profiles", [])
    if not isinstance(profile_documents, list):
        raise DocForgeError("invalid_config", "profiles must be an array of tables")
    profiles: list[ContextProfile] = []
    profile_ids: set[str] = set()
    for profile in profile_documents:
        if not isinstance(profile, dict):
            raise DocForgeError("invalid_config", "Each profile must be a table")
        unknown_profile = sorted(set(profile) - _PROFILE_KEYS)
        if unknown_profile:
            raise DocForgeError(
                "invalid_config", "Profile has unknown fields", fields=unknown_profile
            )
        profile_id = require_string(profile, "id", descriptor_path)
        if ID_PATTERN.fullmatch(profile_id) is None or profile_id in profile_ids:
            raise DocForgeError(
                "invalid_config", "Profile ID is invalid or duplicated", id=profile_id
            )
        profile_ids.add(profile_id)
        token_budget = positive_int(profile.get("token_budget", 8_000), "profile.token_budget")
        dependency_depth = positive_int(
            profile.get("dependency_depth", 1), "profile.dependency_depth", allow_zero=True
        )
        if token_budget > limits.max_context_tokens:
            raise DocForgeError("invalid_config", "Profile token budget exceeds project limit")
        if dependency_depth > limits.max_traversal_depth:
            raise DocForgeError("invalid_config", "Profile dependency depth exceeds project limit")
        profiles.append(
            ContextProfile(
                profile_id=profile_id,
                families=string_list(
                    profile.get("families", []), key="profile.families", source=descriptor_path
                ),
                statuses=string_list(
                    profile.get("statuses", []), key="profile.statuses", source=descriptor_path
                ),
                required_nodes=string_list(
                    profile.get("required_nodes", []),
                    key="profile.required_nodes",
                    source=descriptor_path,
                ),
                token_budget=token_budget,
                dependency_depth=dependency_depth,
            )
        )

    return ProjectDescriptor(
        schema_version=1,
        project_id=project_id,
        title=title,
        adapter=adapter,
        root=root,
        descriptor_path=descriptor_path,
        descriptor_hash=hashlib.sha256(descriptor_bytes).hexdigest(),
        content_roots=content_roots,
        authority_files=authority_files,
        cache_root=cache_root,
        index_path=index_path,
        changeset_root=changeset_root,
        proposal_writers=tuple(sorted(proposal_writers, key=lambda writer: writer.writer_id)),
        render=render,
        allowed_relations=allowed_relations,
        profiles=tuple(profiles),
        limits=limits,
    )


def _markdown_record(path: Path, text: str) -> tuple[dict[str, Any], str]:
    lines = text.splitlines()
    if not lines or lines[0] != "+++":
        raise DocForgeError(
            "invalid_source", f"{path.name}: Markdown must start with TOML metadata"
        )
    try:
        close = lines.index("+++", 1)
    except ValueError as error:
        raise DocForgeError(
            "invalid_source", f"{path.name}: metadata block is not closed"
        ) from error
    try:
        metadata = tomllib.loads("\n".join(lines[1:close]))
    except tomllib.TOMLDecodeError as error:
        raise DocForgeError("invalid_source", f"{path.name}: invalid metadata: {error}") from error
    return metadata, "\n".join(lines[close + 1 :]).strip()


def validated_node_from_record(
    record: dict[str, Any],
    *,
    content: str,
    source: Path,
    relative_source: str,
    relations: tuple[str, ...],
    hash_bytes: bytes,
) -> tuple[Node, tuple[Edge, ...]]:
    if record.get("schema_version") != 1:
        raise DocForgeError("invalid_source", f"{source.name}: node schema_version must be 1")
    unknown = set(record) - _CORE_METADATA - set(relations)
    if unknown:
        raise DocForgeError(
            "invalid_source", f"{source.name}: unknown metadata", keys=sorted(unknown)
        )
    node_id = require_string(record, "id", source)
    if ID_PATTERN.fullmatch(node_id) is None:
        raise DocForgeError("invalid_source", f"{source.name}: node ID is invalid", id=node_id)
    authority = require_string(record, "authority", source)
    if authority not in _AUTHORITIES:
        raise DocForgeError(
            "invalid_source", f"{source.name}: authority is invalid", authority=authority
        )
    tags = string_list(record.get("tags", []), key="tags", source=source)
    anchor = record.get("source_anchor")
    if anchor is not None and (not isinstance(anchor, str) or not anchor):
        raise DocForgeError("invalid_source", f"{source.name}: source_anchor must be a string")
    summary = require_string(record, "summary", source)
    if not content:
        raise DocForgeError("invalid_source", f"{source.name}: node content is empty", id=node_id)
    node = Node(
        node_id=node_id,
        title=require_string(record, "title", source),
        family=require_string(record, "family", source),
        authority=authority,
        status=require_string(record, "status", source),
        tags=tags,
        summary=summary,
        content=content,
        source_path=relative_source,
        source_anchor=anchor,
        content_hash=hashlib.sha256(hash_bytes).hexdigest(),
    )
    edges = tuple(
        Edge(node_id, relation, target)
        for relation in relations
        for target in string_list(record.get(relation, []), key=relation, source=source)
    )
    return node, edges


def _load_source_file(
    descriptor: ProjectDescriptor, path: Path, raw: bytes
) -> tuple[tuple[Node, ...], tuple[Edge, ...]]:
    if len(raw) > descriptor.limits.max_source_bytes:
        raise DocForgeError(
            "source_too_large", "Canonical source exceeds configured limit", source=path.name
        )
    try:
        text = raw.decode("utf-8")
    except UnicodeDecodeError as error:
        raise DocForgeError("invalid_source", f"{path.name}: source is not UTF-8") from error
    relative = path.relative_to(descriptor.root).as_posix()
    if path.suffix == ".md":
        record, content = _markdown_record(path, text)
        node, edges = validated_node_from_record(
            record,
            content=content,
            source=path,
            relative_source=relative,
            relations=descriptor.allowed_relations,
            hash_bytes=raw,
        )
        return (node,), edges
    if path.suffix == ".toml":
        try:
            document = tomllib.loads(text)
        except tomllib.TOMLDecodeError as error:
            raise DocForgeError("invalid_source", f"{path.name}: invalid TOML: {error}") from error
        records = document.get("nodes")
        if set(document) != {"nodes"} or not isinstance(records, list) or not records:
            raise DocForgeError("invalid_source", f"{path.name}: TOML sources require [[nodes]]")
        nodes: list[Node] = []
        edges: list[Edge] = []
        for index, record in enumerate(records):
            if not isinstance(record, dict):
                raise DocForgeError("invalid_source", f"{path.name}: nodes must be tables")
            content = record.get("content")
            if not isinstance(content, str):
                raise DocForgeError(
                    "invalid_source", f"{path.name}: TOML node content must be text"
                )
            canonical = json.dumps(record, sort_keys=True, separators=(",", ":")).encode()
            node, node_edges = validated_node_from_record(
                record,
                content=content.strip(),
                source=path,
                relative_source=relative,
                relations=descriptor.allowed_relations,
                hash_bytes=canonical,
            )
            nodes.append(replace(node, source_anchor=node.source_anchor or f"node-{index + 1}"))
            edges.extend(node_edges)
        return tuple(nodes), tuple(edges)
    raise DocForgeError("invalid_source", "Unsupported canonical source type", source=relative)


def validate_graph(nodes: tuple[Node, ...], edges: tuple[Edge, ...]) -> None:
    node_ids = {node.node_id for node in nodes}
    if len(node_ids) != len(nodes):
        counts = Counter(node.node_id for node in nodes)
        duplicates = sorted(node_id for node_id, count in counts.items() if count > 1)
        raise DocForgeError("duplicate_node", "Stable node IDs must be unique", ids=duplicates)
    edge_keys = {(edge.source_id, edge.relation, edge.target_id) for edge in edges}
    if len(edge_keys) != len(edges):
        raise DocForgeError("duplicate_edge", "Relationships must be unique")
    missing = sorted({edge.target_id for edge in edges if edge.target_id not in node_ids})
    if missing:
        raise DocForgeError("broken_edge", "Relationships target missing nodes", targets=missing)

    dependencies = {
        node_id: sorted(
            edge.target_id
            for edge in edges
            if edge.source_id == node_id and edge.relation == "depends_on"
        )
        for node_id in sorted(node_ids)
    }
    visiting: set[str] = set()
    visited: set[str] = set()

    def visit(node_id: str, trail: tuple[str, ...]) -> None:
        if node_id in visiting:
            raise DocForgeError(
                "dependency_cycle",
                "depends_on relationships contain a cycle",
                path=(*trail, node_id),
            )
        if node_id in visited:
            return
        visiting.add(node_id)
        for target in dependencies[node_id]:
            visit(target, (*trail, node_id))
        visiting.remove(node_id)
        visited.add(node_id)

    for node_id in sorted(node_ids):
        visit(node_id, ())


def _revision(root: Path) -> str:
    try:
        result = subprocess.run(
            ["git", "rev-parse", "HEAD"],
            cwd=root,
            check=False,
            capture_output=True,
            text=True,
            timeout=2,
        )
    except (OSError, subprocess.TimeoutExpired):
        return "unversioned"
    return (
        result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unversioned"
    )


class Project:
    """One immutable project binding for loading and querying canonical documentation."""

    def __init__(self, descriptor: ProjectDescriptor) -> None:
        self.descriptor = descriptor

    @classmethod
    def open(cls, project_root: str | Path) -> Project:
        try:
            root = Path(project_root).expanduser().resolve(strict=True)
        except OSError as error:
            raise DocForgeError("invalid_root", "Project root does not exist") from error
        if not root.is_dir():
            raise DocForgeError("invalid_root", "Project root must be a directory")
        return cls(_load_descriptor(root))

    def load(self) -> ProjectSnapshot:
        descriptor_bytes = self.descriptor.descriptor_path.read_bytes()
        if hashlib.sha256(descriptor_bytes).hexdigest() != self.descriptor.descriptor_hash:
            raise DocForgeError(
                "source_changed", "Project descriptor changed after the project was opened"
            )
        ordered_sources = self.canonical_source_paths()
        captured = {
            path: path.read_bytes()
            for path in (
                self.descriptor.descriptor_path,
                *self.descriptor.authority_files,
                *ordered_sources,
            )
        }

        nodes: list[Node] = []
        edges: list[Edge] = []
        for path in ordered_sources:
            source_nodes, source_edges = _load_source_file(self.descriptor, path, captured[path])
            nodes.extend(source_nodes)
            edges.extend(source_edges)
            if len(nodes) > self.descriptor.limits.max_nodes:
                raise DocForgeError("node_limit", "Project exceeds configured node limit")
        ordered_nodes = tuple(sorted(nodes, key=lambda node: node.node_id))
        ordered_edges = tuple(
            sorted(edges, key=lambda edge: (edge.source_id, edge.relation, edge.target_id))
        )
        validate_graph(ordered_nodes, ordered_edges)
        node_ids = {node.node_id for node in ordered_nodes}
        for profile in self.descriptor.profiles:
            missing = sorted(set(profile.required_nodes) - node_ids)
            if missing:
                raise DocForgeError(
                    "invalid_config", "Context profile requires missing nodes", nodes=missing
                )

        if self.canonical_source_paths() != ordered_sources:
            raise DocForgeError("source_changed", "Canonical source set changed during loading")
        for path, raw in captured.items():
            if not path.is_file() or path.read_bytes() != raw:
                raise DocForgeError(
                    "source_changed",
                    "Canonical source changed during loading",
                    source=path.relative_to(self.descriptor.root).as_posix(),
                )

        digest = hashlib.sha256()
        for path in sorted(
            captured, key=lambda item: item.relative_to(self.descriptor.root).as_posix()
        ):
            relative = path.relative_to(self.descriptor.root).as_posix()
            digest.update(relative.encode())
            digest.update(b"\0")
            digest.update(hashlib.sha256(captured[path]).digest())
        digest.update(b"docforge-core:0.3.0:index:1")
        return ProjectSnapshot(
            descriptor=self.descriptor,
            nodes=ordered_nodes,
            edges=ordered_edges,
            source_hash=digest.hexdigest(),
            revision=_revision(self.descriptor.root),
        )

    def canonical_source_paths(self) -> tuple[Path, ...]:
        """Return the deterministic confined canonical source set."""

        source_paths: set[Path] = set()
        for content_root in self.descriptor.content_roots:
            for path in content_root.rglob("*"):
                if path.suffix not in {".md", ".toml"} or not path.is_file():
                    continue
                resolved = path.resolve()
                if not resolved.is_relative_to(self.descriptor.root):
                    raise DocForgeError(
                        "path_escape", "Canonical source resolves outside project root"
                    )
                source_paths.add(resolved)
        ordered_sources = sorted(
            source_paths, key=lambda path: path.relative_to(self.descriptor.root).as_posix()
        )
        if not ordered_sources:
            raise DocForgeError("empty_project", "No canonical Markdown or TOML sources were found")
        return tuple(ordered_sources)