Add persistent source generations

src/docforge/project.py

@@ -4,12 +4,15 @@ from __future__ import annotations
 
 import hashlib
 import json
+import os
+import stat
 import subprocess
+import tempfile
 import tomllib
 from collections import Counter
 from collections.abc import Mapping
-from dataclasses import replace
-from pathlib import Path
+from dataclasses import dataclass, replace
+from pathlib import Path, PurePosixPath
 from typing import Any, cast
 
 from .config_validation import (
@@ -28,10 +31,14 @@ from .models import (
     Node,
     ProjectDescriptor,
     ProjectSnapshot,
+    ProjectState,
     ProposalWriter,
 )
 from .render_config import load_render_config
 
+SOURCE_GENERATION_SCHEMA_VERSION = 1
+GENERIC_SOURCE_CONTRACT = "docforge-core:0.7.1:index:1"
+
 _CORE_METADATA = frozenset(
     {
         "schema_version",
@@ -72,10 +79,109 @@ _PROFILE_KEYS = frozenset(
 _OPERATIONS = frozenset({"create", "update", "move", "delete"})
 
 
+@dataclass(frozen=True)
+class _CapturedGeneration:
+    source_hash: str
+    revision: str
+    files: tuple[tuple[str, int, int, int, int, int, int], ...]
+    directories: tuple[tuple[str, int, int, int, int, int], ...]
+
+
 def project_root_fingerprint(root: Path) -> str:
     return hashlib.sha256(str(root).encode()).hexdigest()[:16]
 
 
+def _file_generation(
+    root: Path,
+    paths: tuple[Path, ...],
+) -> tuple[tuple[str, int, int, int, int, int, int], ...]:
+    """Capture cheap identities that change on ordinary source or metadata mutation."""
+
+    identities: list[tuple[str, int, int, int, int, int, int]] = []
+    for path in paths:
+        try:
+            status = path.lstat()
+        except OSError as error:
+            raise DocForgeError(
+                "source_changed",
+                "Canonical source disappeared during generation capture",
+                source=path.relative_to(root).as_posix(),
+            ) from error
+        if not stat.S_ISREG(status.st_mode):
+            raise DocForgeError(
+                "source_changed",
+                "Canonical generation inputs must remain regular files",
+                source=path.relative_to(root).as_posix(),
+            )
+        identities.append(
+            (
+                path.relative_to(root).as_posix(),
+                status.st_dev,
+                status.st_ino,
+                status.st_mode,
+                status.st_size,
+                status.st_mtime_ns,
+                status.st_ctime_ns,
+            )
+        )
+    return tuple(identities)
+
+
+def _directory_generation(
+    root: Path,
+    paths: tuple[Path, ...],
+) -> tuple[tuple[str, int, int, int, int, int], ...]:
+    """Capture directory identities so source membership changes invalidate a receipt."""
+
+    identities: list[tuple[str, int, int, int, int, int]] = []
+    for path in paths:
+        try:
+            status = path.lstat()
+        except OSError as error:
+            raise DocForgeError(
+                "source_changed",
+                "Canonical source directory disappeared during generation capture",
+                source=path.relative_to(root).as_posix(),
+            ) from error
+        if not stat.S_ISDIR(status.st_mode):
+            raise DocForgeError(
+                "source_changed",
+                "Canonical source directories must remain directories",
+                source=path.relative_to(root).as_posix(),
+            )
+        identities.append(
+            (
+                path.relative_to(root).as_posix(),
+                status.st_dev,
+                status.st_ino,
+                status.st_mode,
+                status.st_mtime_ns,
+                status.st_ctime_ns,
+            )
+        )
+    return tuple(identities)
+
+
+def _receipt_paths(root: Path, value: object, *, width: int) -> tuple[Path, ...] | None:
+    if not isinstance(value, list):
+        return None
+    paths: list[Path] = []
+    for raw_item in cast(list[object], value):
+        if not isinstance(raw_item, list):
+            return None
+        item = cast(list[object], raw_item)
+        if len(item) != width or not isinstance(item[0], str):
+            return None
+        relative = PurePosixPath(item[0])
+        if relative.is_absolute() or not relative.parts or ".." in relative.parts:
+            return None
+        path = root.joinpath(*relative.parts)
+        if not path.is_relative_to(root):
+            return None
+        paths.append(path)
+    return tuple(paths)
+
+
 def _load_descriptor(root: Path) -> ProjectDescriptor:
     descriptor_path = root / ".docforge" / "project.toml"
     if not descriptor_path.is_file():
@@ -472,39 +578,60 @@ def validate_graph(nodes: tuple[Node, ...], edges: tuple[Edge, ...]) -> None:
         counts = Counter(node.node_id for node in nodes)
         duplicates = sorted(node_id for node_id, count in counts.items() if count > 1)
         raise DocForgeError("duplicate_node", "Stable node IDs must be unique", ids=duplicates)
-    edge_keys = {(edge.source_id, edge.relation, edge.target_id) for edge in edges}
-    if len(edge_keys) != len(edges):
-        raise DocForgeError("duplicate_edge", "Relationships must be unique")
-    missing = sorted({edge.target_id for edge in edges if edge.target_id not in node_ids})
-    if missing:
-        raise DocForgeError("broken_edge", "Relationships target missing nodes", targets=missing)
 
     dependencies: dict[str, list[str]] = {node_id: [] for node_id in node_ids}
+    edge_keys: set[tuple[str, str, str]] = set()
+    missing_sources: set[str] = set()
+    missing_targets: set[str] = set()
     for edge in edges:
-        if edge.relation == "depends_on":
+        key = (edge.source_id, edge.relation, edge.target_id)
+        if key in edge_keys:
+            raise DocForgeError("duplicate_edge", "Relationships must be unique")
+        edge_keys.add(key)
+        if edge.source_id not in node_ids:
+            missing_sources.add(edge.source_id)
+        if edge.target_id not in node_ids:
+            missing_targets.add(edge.target_id)
+        if edge.relation == "depends_on" and edge.source_id in dependencies:
             dependencies[edge.source_id].append(edge.target_id)
+    if missing_sources or missing_targets:
+        raise DocForgeError(
+            "broken_edge",
+            "Relationships reference missing nodes",
+            sources=sorted(missing_sources),
+            targets=sorted(missing_targets),
+        )
     for targets in dependencies.values():
         targets.sort()
-    visiting: set[str] = set()
-    visited: set[str] = set()
 
-    def visit(node_id: str, trail: tuple[str, ...]) -> None:
-        if node_id in visiting:
-            raise DocForgeError(
-                "dependency_cycle",
-                "depends_on relationships contain a cycle",
-                path=(*trail, node_id),
-            )
-        if node_id in visited:
-            return
-        visiting.add(node_id)
-        for target in dependencies[node_id]:
-            visit(target, (*trail, node_id))
-        visiting.remove(node_id)
-        visited.add(node_id)
-
-    for node_id in sorted(node_ids):
-        visit(node_id, ())
+    states: dict[str, int] = {}
+    for root in sorted(node_ids):
+        if states.get(root) == 2:
+            continue
+        path: list[str] = []
+        stack: list[tuple[str, int]] = [(root, 0)]
+        while stack:
+            node_id, child_index = stack[-1]
+            if states.get(node_id, 0) == 0:
+                states[node_id] = 1
+                path.append(node_id)
+            targets = dependencies[node_id]
+            if child_index < len(targets):
+                target = targets[child_index]
+                stack[-1] = (node_id, child_index + 1)
+                state = states.get(target, 0)
+                if state == 1:
+                    raise DocForgeError(
+                        "dependency_cycle",
+                        "depends_on relationships contain a cycle",
+                        path=(*path, target),
+                    )
+                if state == 0:
+                    stack.append((target, 0))
+                continue
+            stack.pop()
+            path.pop()
+            states[node_id] = 2
 
 
 def validate_source_layout(nodes: tuple[Node, ...]) -> None:
@@ -569,6 +696,7 @@ class Project:
 
     def __init__(self, descriptor: ProjectDescriptor) -> None:
         self.descriptor = descriptor
+        self._captured_generation: _CapturedGeneration | None = None
 
     @classmethod
     def open(cls, project_root: str | Path) -> Project:
@@ -586,15 +714,18 @@ class Project:
             raise DocForgeError(
                 "source_changed", "Project descriptor changed after the project was opened"
             )
-        ordered_sources = self.canonical_source_paths()
-        captured = {
-            path: path.read_bytes()
-            for path in (
-                self.descriptor.descriptor_path,
-                *self.descriptor.authority_files,
-                *ordered_sources,
-            )
-        }
+        ordered_sources, ordered_directories = self._canonical_inventory()
+        generation_paths = (
+            self.descriptor.descriptor_path,
+            *self.descriptor.authority_files,
+            *ordered_sources,
+        )
+        before_generation = _file_generation(self.descriptor.root, generation_paths)
+        before_directories = _directory_generation(
+            self.descriptor.root,
+            ordered_directories,
+        )
+        captured = {path: path.read_bytes() for path in generation_paths}
 
         nodes: list[Node] = []
         edges: list[Edge] = []
@@ -617,7 +748,8 @@ class Project:
                     "invalid_config", "Context profile requires missing nodes", nodes=missing
                 )
 
-        if self.canonical_source_paths() != ordered_sources:
+        current_sources, current_directories = self._canonical_inventory()
+        if current_sources != ordered_sources or current_directories != ordered_directories:
             raise DocForgeError("source_changed", "Canonical source set changed during loading")
         for path, raw in captured.items():
             if not path.is_file() or path.read_bytes() != raw:
@@ -626,6 +758,16 @@ class Project:
                     "Canonical source changed during loading",
                     source=path.relative_to(self.descriptor.root).as_posix(),
                 )
+        after_generation = _file_generation(self.descriptor.root, generation_paths)
+        after_directories = _directory_generation(
+            self.descriptor.root,
+            ordered_directories,
+        )
+        if after_generation != before_generation or after_directories != before_directories:
+            raise DocForgeError(
+                "source_changed",
+                "Canonical source metadata changed during loading",
+            )
 
         digest = hashlib.sha256()
         for path in sorted(
@@ -635,21 +777,157 @@ class Project:
             digest.update(relative.encode())
             digest.update(b"\0")
             digest.update(hashlib.sha256(captured[path]).digest())
-        digest.update(b"docforge-core:0.7.1:index:1")
-        return ProjectSnapshot(
+        digest.update(GENERIC_SOURCE_CONTRACT.encode("ascii"))
+        source_hash = digest.hexdigest()
+        revision = _revision(self.descriptor.root)
+        snapshot = ProjectSnapshot(
             descriptor=self.descriptor,
             nodes=ordered_nodes,
             edges=ordered_edges,
-            source_hash=digest.hexdigest(),
-            revision=_revision(self.descriptor.root),
+            source_hash=source_hash,
+            revision=revision,
         )
+        self._captured_generation = _CapturedGeneration(
+            source_hash=source_hash,
+            revision=revision,
+            files=after_generation,
+            directories=after_directories,
+        )
+        return snapshot
+
+    @property
+    def generation_path(self) -> Path:
+        """Return the confined disposable receipt for one verified source generation."""
+
+        return self.descriptor.cache_root / "source-generation.json"
+
+    def incremental_state(self) -> ProjectState | None:
+        """Return current source identity without reading or parsing canonical source bytes."""
+
+        path = self.generation_path
+        if not path.is_file() or path.is_symlink():
+            return None
+        try:
+            parsed: object = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+            return None
+        if not isinstance(parsed, dict):
+            return None
+        payload = cast(dict[str, object], parsed)
+        source_hash = payload.get("source_hash")
+        revision = payload.get("revision")
+        if (
+            payload.get("schema_version") != SOURCE_GENERATION_SCHEMA_VERSION
+            or payload.get("source_contract") != GENERIC_SOURCE_CONTRACT
+            or payload.get("project_id") != self.descriptor.project_id
+            or payload.get("project_root_fingerprint")
+            != project_root_fingerprint(self.descriptor.root)
+            or payload.get("adapter") != self.descriptor.adapter
+            or not isinstance(source_hash, str)
+            or len(source_hash) != 64
+            or not isinstance(revision, str)
+        ):
+            return None
+        directory_paths = _receipt_paths(
+            self.descriptor.root,
+            payload.get("directories"),
+            width=6,
+        )
+        file_paths = _receipt_paths(
+            self.descriptor.root,
+            payload.get("files"),
+            width=7,
+        )
+        if directory_paths is None or file_paths is None:
+            return None
+        try:
+            current_directories = _directory_generation(self.descriptor.root, directory_paths)
+        except DocForgeError:
+            return None
+        if payload.get("directories") != [list(identity) for identity in current_directories]:
+            return None
+        try:
+            current_files = _file_generation(self.descriptor.root, file_paths)
+        except DocForgeError:
+            return None
+        if payload.get("files") != [list(identity) for identity in current_files]:
+            return None
+        if _revision(self.descriptor.root) != revision:
+            return None
+        return ProjectState(source_hash=source_hash, revision=revision)
+
+    def record_generation(self, snapshot: ProjectSnapshot) -> None:
+        """Persist a generation only after its complete derived index was verified."""
+
+        captured = self._captured_generation
+        if (
+            captured is None
+            or captured.source_hash != snapshot.source_hash
+            or captured.revision != snapshot.revision
+        ):
+            raise DocForgeError(
+                "source_changed",
+                "Cannot record a source generation without a matching complete load",
+            )
+        root = self.descriptor.cache_root
+        path = self.generation_path
+        if path.parent != root or path.is_symlink() or root.resolve(strict=False) != root:
+            raise DocForgeError("path_escape", "Source generation receipt path is not safe")
+        root.mkdir(parents=True, exist_ok=True)
+        if not root.is_dir() or root.resolve(strict=False) != root:
+            raise DocForgeError("path_escape", "Source generation receipt directory is not safe")
+        payload = {
+            "schema_version": SOURCE_GENERATION_SCHEMA_VERSION,
+            "source_contract": GENERIC_SOURCE_CONTRACT,
+            "project_id": self.descriptor.project_id,
+            "project_root_fingerprint": project_root_fingerprint(self.descriptor.root),
+            "adapter": self.descriptor.adapter,
+            "source_hash": captured.source_hash,
+            "revision": captured.revision,
+            "files": [list(identity) for identity in captured.files],
+            "directories": [list(identity) for identity in captured.directories],
+        }
+        raw = json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n"
+        descriptor, temporary_name = tempfile.mkstemp(prefix=".source-generation-", dir=root)
+        temporary = Path(temporary_name)
+        try:
+            with os.fdopen(descriptor, "wb") as handle:
+                handle.write(raw)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temporary, path)
+            directory_descriptor = os.open(root, os.O_RDONLY)
+            try:
+                os.fsync(directory_descriptor)
+            finally:
+                os.close(directory_descriptor)
+        except Exception:
+            temporary.unlink(missing_ok=True)
+            raise
 
     def canonical_source_paths(self) -> tuple[Path, ...]:
         """Return the deterministic confined canonical source set."""
 
+        sources, _ = self._canonical_inventory()
+        return sources
+
+    def _canonical_inventory(self) -> tuple[tuple[Path, ...], tuple[Path, ...]]:
+        """Return deterministic canonical files and membership-bearing directories."""
+
         source_paths: set[Path] = set()
+        directories: set[Path] = set()
         for content_root in self.descriptor.content_roots:
+            directories.add(content_root)
             for path in content_root.rglob("*"):
+                if path.is_dir():
+                    resolved_directory = path.resolve()
+                    if not resolved_directory.is_relative_to(self.descriptor.root):
+                        raise DocForgeError(
+                            "path_escape",
+                            "Canonical source directory resolves outside project root",
+                        )
+                    directories.add(resolved_directory)
+                    continue
                 if path.suffix not in {".md", ".toml"} or not path.is_file():
                     continue
                 resolved = path.resolve()
@@ -663,7 +941,11 @@ class Project:
         )
         if not ordered_sources:
             raise DocForgeError("empty_project", "No canonical Markdown or TOML sources were found")
-        return tuple(ordered_sources)
+        ordered_directories = sorted(
+            directories,
+            key=lambda path: path.relative_to(self.descriptor.root).as_posix(),
+        )
+        return tuple(ordered_sources), tuple(ordered_directories)
 
     def validate_proposal(
         self,