DocForge2/src/docforge/project.py

"""Project discovery, root confinement, canonical loading, and graph validation."""

from __future__ import annotations

import hashlib
import json
import subprocess
import tomllib
from collections import Counter
from collections.abc import Mapping
from dataclasses import replace
from pathlib import Path
from typing import Any, cast

from .config_validation import (
    AUTHORITIES,
    ID_PATTERN,
    confined_path,
    positive_int,
    require_string,
    string_list,
)
from .errors import DocForgeError
from .models import (
    ContextProfile,
    Edge,
    Limits,
    Node,
    ProjectDescriptor,
    ProjectSnapshot,
    ProposalWriter,
)
from .render_config import load_render_config

_CORE_METADATA = frozenset(
    {
        "schema_version",
        "id",
        "title",
        "family",
        "authority",
        "status",
        "tags",
        "summary",
        "source_anchor",
        "content",
    }
)
_DESCRIPTOR_KEYS = frozenset(
    {
        "schema_version",
        "project_id",
        "title",
        "adapter",
        "sources",
        "derived",
        "changesets",
        "render",
        "graph",
        "limits",
        "profiles",
    }
)
_SOURCE_KEYS = frozenset({"content_roots", "authority_files"})
_DERIVED_KEYS = frozenset({"cache_root", "index"})
_CHANGESET_KEYS = frozenset({"root", "writers"})
_WRITER_KEYS = frozenset({"id", "families", "operations"})
_GRAPH_KEYS = frozenset({"allowed_relations"})
_PROFILE_KEYS = frozenset(
    {"id", "families", "statuses", "required_nodes", "token_budget", "dependency_depth"}
)
_OPERATIONS = frozenset({"create", "update", "move", "delete"})


def project_root_fingerprint(root: Path) -> str:
    return hashlib.sha256(str(root).encode()).hexdigest()[:16]


def _load_descriptor(root: Path) -> ProjectDescriptor:
    descriptor_path = root / ".docforge" / "project.toml"
    if not descriptor_path.is_file():
        raise DocForgeError("missing_config", "Missing .docforge/project.toml")
    try:
        descriptor_bytes = descriptor_path.read_bytes()
        document = cast(dict[str, object], tomllib.loads(descriptor_bytes.decode("utf-8")))
    except UnicodeDecodeError as error:
        raise DocForgeError("invalid_config", "Project descriptor is not UTF-8") from error
    except tomllib.TOMLDecodeError as error:
        raise DocForgeError("invalid_config", f"Invalid project descriptor: {error}") from error

    unknown_descriptor = sorted(set(document) - _DESCRIPTOR_KEYS)
    if unknown_descriptor:
        raise DocForgeError(
            "invalid_config", "Project descriptor has unknown fields", fields=unknown_descriptor
        )

    if document.get("schema_version") != 1:
        raise DocForgeError("invalid_config", "Project descriptor schema_version must be 1")
    project_id = require_string(document, "project_id", descriptor_path)
    if ID_PATTERN.fullmatch(project_id) is None:
        raise DocForgeError(
            "invalid_config", "project_id is not a stable ID", project_id=project_id
        )
    title = require_string(document, "title", descriptor_path)
    adapter = require_string(document, "adapter", descriptor_path)
    if adapter != "generic":
        raise DocForgeError("unsupported_adapter", "DFG-1 supports only the generic adapter")

    sources = document.get("sources")
    derived = document.get("derived")
    changesets = document.get("changesets")
    graph = document.get("graph")
    if (
        not isinstance(sources, dict)
        or not isinstance(derived, dict)
        or not isinstance(changesets, dict)
        or not isinstance(graph, dict)
    ):
        raise DocForgeError(
            "invalid_config", "sources, derived, changesets, and graph tables are required"
        )
    sources = cast(dict[str, object], sources)
    derived = cast(dict[str, object], derived)
    changesets = cast(dict[str, object], changesets)
    graph = cast(dict[str, object], graph)
    for table, allowed, name in (
        (sources, _SOURCE_KEYS, "sources"),
        (derived, _DERIVED_KEYS, "derived"),
        (changesets, _CHANGESET_KEYS, "changesets"),
        (graph, _GRAPH_KEYS, "graph"),
    ):
        unknown = sorted(set(table) - allowed)
        if unknown:
            raise DocForgeError("invalid_config", f"{name} has unknown fields", fields=unknown)
    content_roots = tuple(
        confined_path(
            root,
            item,
            field="sources.content_roots",
            must_exist=True,
            expected="directory",
        )
        for item in string_list(
            sources.get("content_roots"), key="sources.content_roots", source=descriptor_path
        )
    )
    if len(content_roots) != len(set(content_roots)):
        raise DocForgeError("invalid_config", "sources.content_roots resolve to duplicates")
    authority_files = tuple(
        confined_path(
            root,
            item,
            field="sources.authority_files",
            must_exist=True,
            expected="file",
        )
        for item in string_list(
            sources.get("authority_files", []),
            key="sources.authority_files",
            source=descriptor_path,
        )
    )
    cache_root = confined_path(
        root, derived.get("cache_root"), field="derived.cache_root", must_exist=False
    )
    index_path = confined_path(root, derived.get("index"), field="derived.index", must_exist=False)
    changeset_root = confined_path(
        root, changesets.get("root"), field="changesets.root", must_exist=False
    )
    if not index_path.is_relative_to(cache_root):
        raise DocForgeError("invalid_config", "derived.index must be inside derived.cache_root")
    for content_root in content_roots:
        if (
            content_root == cache_root
            or content_root.is_relative_to(cache_root)
            or cache_root.is_relative_to(content_root)
        ):
            raise DocForgeError("invalid_config", "Canonical content and cache must not overlap")
        if (
            content_root == changeset_root
            or content_root.is_relative_to(changeset_root)
            or changeset_root.is_relative_to(content_root)
        ):
            raise DocForgeError(
                "invalid_config", "Canonical content and changesets must not overlap"
            )
    if (
        cache_root == changeset_root
        or cache_root.is_relative_to(changeset_root)
        or changeset_root.is_relative_to(cache_root)
    ):
        raise DocForgeError("invalid_config", "Cache and changesets must not overlap")

    writer_documents = changesets.get("writers")
    if not isinstance(writer_documents, list):
        raise DocForgeError("invalid_config", "changesets.writers must be an array of tables")
    proposal_writers: list[ProposalWriter] = []
    writer_ids: set[str] = set()
    for writer_value in cast(list[object], writer_documents):
        if not isinstance(writer_value, dict):
            raise DocForgeError("invalid_config", "Each changeset writer must be a table")
        writer = cast(dict[str, object], writer_value)
        unknown_writer = sorted(set(writer) - _WRITER_KEYS)
        if unknown_writer:
            raise DocForgeError(
                "invalid_config", "Changeset writer has unknown fields", fields=unknown_writer
            )
        writer_id = require_string(writer, "id", descriptor_path)
        if ID_PATTERN.fullmatch(writer_id) is None or writer_id in writer_ids:
            raise DocForgeError(
                "invalid_config", "Changeset writer ID is invalid or duplicated", id=writer_id
            )
        writer_ids.add(writer_id)
        families = string_list(
            writer.get("families"), key="changesets.writer.families", source=descriptor_path
        )
        operations = string_list(
            writer.get("operations"), key="changesets.writer.operations", source=descriptor_path
        )
        if not families:
            raise DocForgeError("invalid_config", "Changeset writer needs at least one family")
        invalid_operations = sorted(set(operations) - _OPERATIONS)
        if not operations or invalid_operations:
            raise DocForgeError(
                "invalid_config",
                "Changeset writer operations are empty or invalid",
                operations=invalid_operations,
            )
        proposal_writers.append(
            ProposalWriter(
                writer_id=writer_id,
                families=tuple(sorted(families)),
                operations=tuple(sorted(operations)),
            )
        )

    allowed_relations = string_list(
        graph.get("allowed_relations"), key="graph.allowed_relations", source=descriptor_path
    )
    if not allowed_relations:
        raise DocForgeError("invalid_config", "At least one relationship type is required")
    for relation in allowed_relations:
        if ID_PATTERN.fullmatch(relation) is None:
            raise DocForgeError("invalid_config", "Relationship type is invalid", relation=relation)

    limit_values = document.get("limits", {})
    if not isinstance(limit_values, dict):
        raise DocForgeError("invalid_config", "limits must be a table")
    limit_values = cast(dict[str, object], limit_values)
    defaults = Limits()
    unknown_limits = sorted(set(limit_values) - set(defaults.__dataclass_fields__))
    if unknown_limits:
        raise DocForgeError("invalid_config", "limits has unknown fields", fields=unknown_limits)
    limits = Limits(
        **{
            field: positive_int(limit_values.get(field, getattr(defaults, field)), field)
            for field in defaults.__dataclass_fields__
        }
    )

    render = load_render_config(
        root,
        document.get("render"),
        descriptor_path=descriptor_path,
        content_roots=content_roots,
        authority_files=authority_files,
        cache_root=cache_root,
        index_path=index_path,
        changeset_root=changeset_root,
        limits=limits,
    )

    profile_documents = document.get("profiles", [])
    if not isinstance(profile_documents, list):
        raise DocForgeError("invalid_config", "profiles must be an array of tables")
    profiles: list[ContextProfile] = []
    profile_ids: set[str] = set()
    for profile_value in cast(list[object], profile_documents):
        if not isinstance(profile_value, dict):
            raise DocForgeError("invalid_config", "Each profile must be a table")
        profile = cast(dict[str, object], profile_value)
        unknown_profile = sorted(set(profile) - _PROFILE_KEYS)
        if unknown_profile:
            raise DocForgeError(
                "invalid_config", "Profile has unknown fields", fields=unknown_profile
            )
        profile_id = require_string(profile, "id", descriptor_path)
        if ID_PATTERN.fullmatch(profile_id) is None or profile_id in profile_ids:
            raise DocForgeError(
                "invalid_config", "Profile ID is invalid or duplicated", id=profile_id
            )
        profile_ids.add(profile_id)
        token_budget = positive_int(profile.get("token_budget", 8_000), "profile.token_budget")
        dependency_depth = positive_int(
            profile.get("dependency_depth", 1), "profile.dependency_depth", allow_zero=True
        )
        if token_budget > limits.max_context_tokens:
            raise DocForgeError("invalid_config", "Profile token budget exceeds project limit")
        if dependency_depth > limits.max_traversal_depth:
            raise DocForgeError("invalid_config", "Profile dependency depth exceeds project limit")
        profiles.append(
            ContextProfile(
                profile_id=profile_id,
                families=string_list(
                    profile.get("families", []), key="profile.families", source=descriptor_path
                ),
                statuses=string_list(
                    profile.get("statuses", []), key="profile.statuses", source=descriptor_path
                ),
                required_nodes=string_list(
                    profile.get("required_nodes", []),
                    key="profile.required_nodes",
                    source=descriptor_path,
                ),
                token_budget=token_budget,
                dependency_depth=dependency_depth,
            )
        )

    return ProjectDescriptor(
        schema_version=1,
        project_id=project_id,
        title=title,
        adapter=adapter,
        root=root,
        descriptor_path=descriptor_path,
        descriptor_hash=hashlib.sha256(descriptor_bytes).hexdigest(),
        content_roots=content_roots,
        authority_files=authority_files,
        cache_root=cache_root,
        index_path=index_path,
        changeset_root=changeset_root,
        proposal_writers=tuple(sorted(proposal_writers, key=lambda writer: writer.writer_id)),
        render=render,
        allowed_relations=allowed_relations,
        profiles=tuple(profiles),
        limits=limits,
    )


def _markdown_record(path: Path, text: str) -> tuple[dict[str, Any], str]:
    lines = text.splitlines()
    if not lines or lines[0] != "+++":
        raise DocForgeError(
            "invalid_source", f"{path.name}: Markdown must start with TOML metadata"
        )
    try:
        close = lines.index("+++", 1)
    except ValueError as error:
        raise DocForgeError(
            "invalid_source", f"{path.name}: metadata block is not closed"
        ) from error
    try:
        metadata = tomllib.loads("\n".join(lines[1:close]))
    except tomllib.TOMLDecodeError as error:
        raise DocForgeError("invalid_source", f"{path.name}: invalid metadata: {error}") from error
    return metadata, "\n".join(lines[close + 1 :]).strip()


def validated_node_from_record(
    record: dict[str, Any],
    *,
    content: str,
    source: Path,
    relative_source: str,
    relations: tuple[str, ...],
    hash_bytes: bytes,
) -> tuple[Node, tuple[Edge, ...]]:
    if record.get("schema_version") != 1:
        raise DocForgeError("invalid_source", f"{source.name}: node schema_version must be 1")
    unknown = set(record) - _CORE_METADATA - set(relations)
    if unknown:
        raise DocForgeError(
            "invalid_source", f"{source.name}: unknown metadata", keys=sorted(unknown)
        )
    node_id = require_string(record, "id", source)
    if ID_PATTERN.fullmatch(node_id) is None:
        raise DocForgeError("invalid_source", f"{source.name}: node ID is invalid", id=node_id)
    authority = require_string(record, "authority", source)
    if authority not in AUTHORITIES:
        raise DocForgeError(
            "invalid_source", f"{source.name}: authority is invalid", authority=authority
        )
    tags = string_list(record.get("tags", []), key="tags", source=source)
    anchor = record.get("source_anchor")
    if anchor is not None and (not isinstance(anchor, str) or not anchor):
        raise DocForgeError("invalid_source", f"{source.name}: source_anchor must be a string")
    summary = require_string(record, "summary", source)
    if not content:
        raise DocForgeError("invalid_source", f"{source.name}: node content is empty", id=node_id)
    node = Node(
        node_id=node_id,
        title=require_string(record, "title", source),
        family=require_string(record, "family", source),
        authority=authority,
        status=require_string(record, "status", source),
        tags=tags,
        summary=summary,
        content=content,
        source_path=relative_source,
        source_anchor=anchor,
        content_hash=hashlib.sha256(hash_bytes).hexdigest(),
    )
    edges = tuple(
        Edge(node_id, relation, target)
        for relation in relations
        for target in string_list(record.get(relation, []), key=relation, source=source)
    )
    return node, edges


def _load_source_file(
    descriptor: ProjectDescriptor, path: Path, raw: bytes
) -> tuple[tuple[Node, ...], tuple[Edge, ...]]:
    if len(raw) > descriptor.limits.max_source_bytes:
        raise DocForgeError(
            "source_too_large", "Canonical source exceeds configured limit", source=path.name
        )
    try:
        text = raw.decode("utf-8")
    except UnicodeDecodeError as error:
        raise DocForgeError("invalid_source", f"{path.name}: source is not UTF-8") from error
    relative = path.relative_to(descriptor.root).as_posix()
    if path.suffix == ".md":
        record, content = _markdown_record(path, text)
        node, node_edges = validated_node_from_record(
            record,
            content=content,
            source=path,
            relative_source=relative,
            relations=descriptor.allowed_relations,
            hash_bytes=raw,
        )
        return (node,), node_edges
    if path.suffix == ".toml":
        try:
            document = cast(dict[str, object], tomllib.loads(text))
        except tomllib.TOMLDecodeError as error:
            raise DocForgeError("invalid_source", f"{path.name}: invalid TOML: {error}") from error
        records = document.get("nodes")
        if set(document) != {"nodes"} or not isinstance(records, list) or not records:
            raise DocForgeError("invalid_source", f"{path.name}: TOML sources require [[nodes]]")
        nodes: list[Node] = []
        edges: list[Edge] = []
        for index, record_value in enumerate(cast(list[object], records)):
            if not isinstance(record_value, dict):
                raise DocForgeError("invalid_source", f"{path.name}: nodes must be tables")
            record = cast(dict[str, Any], record_value)
            content = record.get("content")
            if not isinstance(content, str):
                raise DocForgeError(
                    "invalid_source", f"{path.name}: TOML node content must be text"
                )
            canonical = json.dumps(record, sort_keys=True, separators=(",", ":")).encode()
            node, node_edges = validated_node_from_record(
                record,
                content=content.strip(),
                source=path,
                relative_source=relative,
                relations=descriptor.allowed_relations,
                hash_bytes=canonical,
            )
            nodes.append(replace(node, source_anchor=node.source_anchor or f"node-{index + 1}"))
            edges.extend(node_edges)
        return tuple(nodes), tuple(edges)
    raise DocForgeError("invalid_source", "Unsupported canonical source type", source=relative)


def validate_graph(nodes: tuple[Node, ...], edges: tuple[Edge, ...]) -> None:
    node_ids = {node.node_id for node in nodes}
    if len(node_ids) != len(nodes):
        counts = Counter(node.node_id for node in nodes)
        duplicates = sorted(node_id for node_id, count in counts.items() if count > 1)
        raise DocForgeError("duplicate_node", "Stable node IDs must be unique", ids=duplicates)
    edge_keys = {(edge.source_id, edge.relation, edge.target_id) for edge in edges}
    if len(edge_keys) != len(edges):
        raise DocForgeError("duplicate_edge", "Relationships must be unique")
    missing = sorted({edge.target_id for edge in edges if edge.target_id not in node_ids})
    if missing:
        raise DocForgeError("broken_edge", "Relationships target missing nodes", targets=missing)

    dependencies: dict[str, list[str]] = {node_id: [] for node_id in node_ids}
    for edge in edges:
        if edge.relation == "depends_on":
            dependencies[edge.source_id].append(edge.target_id)
    for targets in dependencies.values():
        targets.sort()
    visiting: set[str] = set()
    visited: set[str] = set()

    def visit(node_id: str, trail: tuple[str, ...]) -> None:
        if node_id in visiting:
            raise DocForgeError(
                "dependency_cycle",
                "depends_on relationships contain a cycle",
                path=(*trail, node_id),
            )
        if node_id in visited:
            return
        visiting.add(node_id)
        for target in dependencies[node_id]:
            visit(target, (*trail, node_id))
        visiting.remove(node_id)
        visited.add(node_id)

    for node_id in sorted(node_ids):
        visit(node_id, ())


def validate_source_layout(nodes: tuple[Node, ...]) -> None:
    """Validate the generic Markdown and TOML source-layout contract."""

    markdown_sources: dict[str, list[str]] = {}
    anchors: dict[tuple[str, str], list[str]] = {}
    for node in nodes:
        if Path(node.source_path).suffix == ".md":
            markdown_sources.setdefault(node.source_path, []).append(node.node_id)
            continue
        if not node.source_anchor:
            raise DocForgeError(
                "source_anchor_required",
                "TOML nodes require a stable source anchor",
                node_id=node.node_id,
            )
        anchors.setdefault((node.source_path, node.source_anchor), []).append(node.node_id)
    markdown_conflicts = {
        source: sorted(node_ids)
        for source, node_ids in markdown_sources.items()
        if len(node_ids) > 1
    }
    if markdown_conflicts:
        raise DocForgeError(
            "source_conflict",
            "Markdown sources may contain only one node",
            sources=markdown_conflicts,
        )
    anchor_conflicts = [
        {"source": source, "source_anchor": anchor, "nodes": sorted(node_ids)}
        for (source, anchor), node_ids in sorted(anchors.items())
        if len(node_ids) > 1
    ]
    if anchor_conflicts:
        raise DocForgeError(
            "source_anchor_conflict",
            "TOML source anchors must be unique within their source",
            conflicts=anchor_conflicts,
        )


def _revision(root: Path) -> str:
    try:
        result = subprocess.run(
            ["git", "rev-parse", "HEAD"],
            cwd=root,
            check=False,
            capture_output=True,
            text=True,
            timeout=2,
        )
    except (OSError, subprocess.TimeoutExpired):
        return "unversioned"
    return (
        result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unversioned"
    )


class Project:
    """One immutable project binding for loading and querying canonical documentation."""

    def __init__(self, descriptor: ProjectDescriptor) -> None:
        self.descriptor = descriptor

    @classmethod
    def open(cls, project_root: str | Path) -> Project:
        try:
            root = Path(project_root).expanduser().resolve(strict=True)
        except OSError as error:
            raise DocForgeError("invalid_root", "Project root does not exist") from error
        if not root.is_dir():
            raise DocForgeError("invalid_root", "Project root must be a directory")
        return cls(_load_descriptor(root))

    def load(self) -> ProjectSnapshot:
        descriptor_bytes = self.descriptor.descriptor_path.read_bytes()
        if hashlib.sha256(descriptor_bytes).hexdigest() != self.descriptor.descriptor_hash:
            raise DocForgeError(
                "source_changed", "Project descriptor changed after the project was opened"
            )
        ordered_sources = self.canonical_source_paths()
        captured = {
            path: path.read_bytes()
            for path in (
                self.descriptor.descriptor_path,
                *self.descriptor.authority_files,
                *ordered_sources,
            )
        }

        nodes: list[Node] = []
        edges: list[Edge] = []
        for path in ordered_sources:
            source_nodes, source_edges = _load_source_file(self.descriptor, path, captured[path])
            nodes.extend(source_nodes)
            edges.extend(source_edges)
            if len(nodes) > self.descriptor.limits.max_nodes:
                raise DocForgeError("node_limit", "Project exceeds configured node limit")
        ordered_nodes = tuple(sorted(nodes, key=lambda node: node.node_id))
        ordered_edges = tuple(
            sorted(edges, key=lambda edge: (edge.source_id, edge.relation, edge.target_id))
        )
        validate_graph(ordered_nodes, ordered_edges)
        node_ids = {node.node_id for node in ordered_nodes}
        for profile in self.descriptor.profiles:
            missing = sorted(set(profile.required_nodes) - node_ids)
            if missing:
                raise DocForgeError(
                    "invalid_config", "Context profile requires missing nodes", nodes=missing
                )

        if self.canonical_source_paths() != ordered_sources:
            raise DocForgeError("source_changed", "Canonical source set changed during loading")
        for path, raw in captured.items():
            if not path.is_file() or path.read_bytes() != raw:
                raise DocForgeError(
                    "source_changed",
                    "Canonical source changed during loading",
                    source=path.relative_to(self.descriptor.root).as_posix(),
                )

        digest = hashlib.sha256()
        for path in sorted(
            captured, key=lambda item: item.relative_to(self.descriptor.root).as_posix()
        ):
            relative = path.relative_to(self.descriptor.root).as_posix()
            digest.update(relative.encode())
            digest.update(b"\0")
            digest.update(hashlib.sha256(captured[path]).digest())
        digest.update(b"docforge-core:0.7.1:index:1")
        return ProjectSnapshot(
            descriptor=self.descriptor,
            nodes=ordered_nodes,
            edges=ordered_edges,
            source_hash=digest.hexdigest(),
            revision=_revision(self.descriptor.root),
        )

    def canonical_source_paths(self) -> tuple[Path, ...]:
        """Return the deterministic confined canonical source set."""

        source_paths: set[Path] = set()
        for content_root in self.descriptor.content_roots:
            for path in content_root.rglob("*"):
                if path.suffix not in {".md", ".toml"} or not path.is_file():
                    continue
                resolved = path.resolve()
                if not resolved.is_relative_to(self.descriptor.root):
                    raise DocForgeError(
                        "path_escape", "Canonical source resolves outside project root"
                    )
                source_paths.add(resolved)
        ordered_sources = sorted(
            source_paths, key=lambda path: path.relative_to(self.descriptor.root).as_posix()
        )
        if not ordered_sources:
            raise DocForgeError("empty_project", "No canonical Markdown or TOML sources were found")
        return tuple(ordered_sources)

    def validate_proposal(
        self,
        base: ProjectSnapshot,
        projected: ProjectSnapshot,
        operations: tuple[Mapping[str, object], ...],
    ) -> None:
        """Generic sources require no policy beyond the core proposal validation."""

        del base, operations
        validate_source_layout(projected.nodes)