Add persistent source generations

src/docforge/index.py

@@ -10,8 +10,9 @@ import sqlite3
 import tempfile
 import time
 from collections import deque
-from collections.abc import Generator
+from collections.abc import Callable, Generator
 from contextlib import contextmanager
+from dataclasses import dataclass
 from pathlib import Path
 from typing import cast
 
@@ -19,12 +20,14 @@ from .errors import DocForgeError
 from .models import (
     BuildReportingProject,
     Edge,
+    GenerationRecordingProject,
     IncrementalStateProject,
     LogicEdge,
     LogicNode,
     LogicProject,
     LogicProjection,
     Node,
+    ProjectDescriptor,
     ProjectService,
     ProjectSnapshot,
     ProjectState,
@@ -104,6 +107,45 @@ def _status(
     }
 
 
+@dataclass(frozen=True)
+class _IndexReadSnapshot:
+    """One request-scoped read transaction over a verified immutable generation."""
+
+    connection: sqlite3.Connection
+    checked: dict[str, object]
+
+    def project_snapshot(self, descriptor: ProjectDescriptor) -> ProjectSnapshot:
+        nodes = tuple(
+            _row_to_node(row)
+            for row in self.connection.execute("SELECT * FROM nodes ORDER BY node_id")
+        )
+        edges = tuple(
+            Edge(*row)
+            for row in self.connection.execute(
+                "SELECT source_id, relation, target_id FROM edges "
+                "ORDER BY source_id, relation, target_id"
+            )
+        )
+        return ProjectSnapshot(
+            descriptor=descriptor,
+            nodes=nodes,
+            edges=edges,
+            source_hash=cast(str, self.checked["source_hash"]),
+            revision=cast(str, self.checked["revision"]),
+        )
+
+    def result(self, **payload: object) -> dict[str, object]:
+        return {
+            "status": "ok",
+            "project_id": self.checked["project_id"],
+            "project_root_fingerprint": self.checked["project_root_fingerprint"],
+            "revision": self.checked["revision"],
+            "source_hash": self.checked["source_hash"],
+            "adapter": self.checked["adapter"],
+            **payload,
+        }
+
+
 class ProjectIndex:
     """A disposable index that always checks current canonical source before queries."""
 
@@ -346,6 +388,8 @@ class ProjectIndex:
             os.replace(temporary, self.path)
             self._verified_index_signature = self._index_signature()
             self._write_attestation()
+            if isinstance(self.project, GenerationRecordingProject):
+                self.project.record_generation(current)
         except sqlite3.Error as error:
             temporary.unlink(missing_ok=True)
             raise DocForgeError("index_failure", "Could not build the derived index") from error
@@ -408,6 +452,81 @@ class ProjectIndex:
                 logic_projection_count=projection_count,
             )
 
+    @contextmanager
+    def _read_snapshot(self) -> Generator[_IndexReadSnapshot, None, None]:
+        """Pin one verified index and source generation for a complete read request."""
+
+        checked = self.check(verify_rows=False)
+        signature = self._verified_index_signature
+        if signature is None or self._index_signature() != signature:
+            raise DocForgeError(
+                "invalid_index",
+                "Derived index changed after validation",
+            )
+        with _read_connection(self.path) as connection:
+            connection.execute("PRAGMA query_only=ON")
+            connection.execute("BEGIN")
+            application_id = connection.execute("PRAGMA application_id").fetchone()[0]
+            schema_version = connection.execute("PRAGMA user_version").fetchone()[0]
+            if application_id != APPLICATION_ID or schema_version != INDEX_SCHEMA_VERSION:
+                raise DocForgeError("invalid_index", "Derived index has an unsupported schema")
+            metadata = dict(connection.execute("SELECT key, value FROM metadata"))
+            for key in (
+                "project_id",
+                "project_root_fingerprint",
+                "revision",
+                "source_hash",
+                "index_schema_version",
+                "adapter",
+            ):
+                if metadata.get(key) != str(checked[key]):
+                    raise DocForgeError(
+                        "invalid_index",
+                        "Derived index changed after validation",
+                        field=key,
+                    )
+            try:
+                logic_projection_count = int(metadata["logic_projection_count"])
+            except (KeyError, ValueError) as error:
+                raise DocForgeError(
+                    "invalid_index",
+                    "Derived index has invalid Logic metadata",
+                ) from error
+            self._require_logic_allowed(logic_projection_count)
+            snapshot = _IndexReadSnapshot(connection=connection, checked=checked)
+            try:
+                yield snapshot
+            except Exception:
+                raise
+            else:
+                self._confirm_read(snapshot.checked, signature)
+
+    def _confirm_read(
+        self,
+        checked: dict[str, object],
+        signature: tuple[int, int, int, int, int],
+    ) -> None:
+        if self._index_signature() != signature:
+            raise DocForgeError("invalid_index", "Derived index changed during the query")
+        state: ProjectState | None = None
+        if isinstance(self.project, IncrementalStateProject):
+            state = self.project.incremental_state()
+        if state is None:
+            current = self.project.load()
+            state = ProjectState(source_hash=current.source_hash, revision=current.revision)
+        if state.source_hash != checked["source_hash"] or state.revision != checked["revision"]:
+            raise DocForgeError("source_changed", "Canonical source changed during the query")
+
+    def read_project_snapshot(
+        self,
+        reader: Callable[[ProjectSnapshot], dict[str, object]],
+    ) -> dict[str, object]:
+        """Run one bounded reader against an immutable derived project snapshot."""
+
+        with self._read_snapshot() as snapshot:
+            payload = reader(snapshot.project_snapshot(self.project.descriptor))
+            return snapshot.result(**payload)
+
     def check(self, *, verify_rows: bool = True) -> dict[str, object]:
         if isinstance(self.project, IncrementalStateProject):
             state = self.project.incremental_state()
@@ -465,6 +584,9 @@ class ProjectIndex:
                 or fts_count != len(snapshot.nodes)
             ):
                 raise DocForgeError("invalid_index", "Derived index rows do not match source")
+        self._verified_index_signature = self._index_signature()
+        if isinstance(self.project, GenerationRecordingProject):
+            self.project.record_generation(snapshot)
         return {**expected, "database": str(self.path)}
 
     def _check_incremental_state(
@@ -654,39 +776,38 @@ class ProjectIndex:
         )
 
     def get_node(self, node_id: str) -> dict[str, object]:
-        checked = self.check(verify_rows=False)
-        with _read_connection(self.path) as connection:
-            row = connection.execute("SELECT * FROM nodes WHERE node_id = ?", (node_id,)).fetchone()
-        if row is None:
-            raise DocForgeError(
-                "missing_node", "No node has the requested stable ID", node_id=node_id
-            )
-        return self._result(checked, node=_row_to_node(row).as_dict())
+        with self._read_snapshot() as snapshot:
+            row = snapshot.connection.execute(
+                "SELECT * FROM nodes WHERE node_id = ?",
+                (node_id,),
+            ).fetchone()
+            if row is None:
+                raise DocForgeError(
+                    "missing_node", "No node has the requested stable ID", node_id=node_id
+                )
+            return snapshot.result(node=_row_to_node(row).as_dict())
 
     def get_logic(self, owner_node_id: str) -> dict[str, object]:
         """Return one function-scoped control-flow projection without expanding the graph."""
 
-        checked = self.check(verify_rows=False)
-        with _read_connection(self.path) as connection:
-            owner = connection.execute(
+        with self._read_snapshot() as snapshot:
+            owner = snapshot.connection.execute(
                 "SELECT * FROM nodes WHERE node_id = ?", (owner_node_id,)
             ).fetchone()
-            projection = _logic_projection_from_connection(connection, owner_node_id)
-        if owner is None:
-            raise DocForgeError(
-                "missing_node",
-                "No node has the requested stable ID",
-                node_id=owner_node_id,
+            projection = _logic_projection_from_connection(snapshot.connection, owner_node_id)
+            if owner is None:
+                raise DocForgeError(
+                    "missing_node",
+                    "No node has the requested stable ID",
+                    node_id=owner_node_id,
+                )
+            return snapshot.result(
+                owner=_row_to_node(owner).as_dict(include_content=False),
+                available=projection is not None,
+                projection=projection.as_dict() if projection is not None else None,
             )
-        return self._result(
-            checked,
-            owner=_row_to_node(owner).as_dict(include_content=False),
-            available=projection is not None,
-            projection=projection.as_dict() if projection is not None else None,
-        )
 
     def search(self, query: str, *, limit: int | None = None) -> dict[str, object]:
-        checked = self.check(verify_rows=False)
         limits = self.project.descriptor.limits
         if not query.strip() or len(query) > limits.max_query_chars:
             raise DocForgeError("invalid_query", "Search query is empty or exceeds its limit")
@@ -695,8 +816,8 @@ class ProjectIndex:
         if not terms:
             raise DocForgeError("invalid_query", "Search query contains no searchable text")
         expression = " AND ".join(f'"{term.replace(chr(34), chr(34) * 2)}"' for term in terms)
-        with _read_connection(self.path) as connection:
-            rows = connection.execute(
+        with self._read_snapshot() as snapshot:
+            rows = snapshot.connection.execute(
                 """
                 SELECT nodes.*, bm25(node_fts) AS rank,
                        snippet(node_fts, 3, '[', ']', ' … ', 18) AS snippet
@@ -707,12 +828,12 @@ class ProjectIndex:
                 """,
                 (expression, bounded),
             ).fetchall()
-        results: list[dict[str, object]] = []
-        for row in rows:
-            payload = _row_to_node(row).as_dict(include_content=False)
-            payload.update({"rank": row["rank"], "snippet": row["snippet"]})
-            results.append(payload)
-        return self._result(checked, query=query, count=len(results), results=results)
+            results: list[dict[str, object]] = []
+            for row in rows:
+                payload = _row_to_node(row).as_dict(include_content=False)
+                payload.update({"rank": row["rank"], "snippet": row["snippet"]})
+                results.append(payload)
+            return snapshot.result(query=query, count=len(results), results=results)
 
     def filter_nodes(
         self,
@@ -723,7 +844,6 @@ class ProjectIndex:
         tag: str | None = None,
         limit: int | None = None,
     ) -> dict[str, object]:
-        checked = self.check(verify_rows=False)
         bounded = _bounded_limit(limit, self.project.descriptor.limits.max_results, default=100)
         clauses: list[str] = []
         values: list[object] = []
@@ -735,12 +855,12 @@ class ProjectIndex:
             clauses.append("EXISTS (SELECT 1 FROM json_each(tags_json) WHERE value = ?)")
             values.append(tag)
         where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
-        with _read_connection(self.path) as connection:
-            rows = connection.execute(
+        with self._read_snapshot() as snapshot:
+            rows = snapshot.connection.execute(
                 f"SELECT * FROM nodes {where} ORDER BY node_id LIMIT ?", (*values, bounded)
             ).fetchall()
-        results = [_row_to_node(row).as_dict(include_content=False) for row in rows]
-        return self._result(checked, count=len(results), results=results)
+            results = [_row_to_node(row).as_dict(include_content=False) for row in rows]
+            return snapshot.result(count=len(results), results=results)
 
     def backlinks(self, node_id: str, *, relation: str | None = None) -> dict[str, object]:
         return self._edges(node_id, incoming=True, relation=relation)
@@ -752,93 +872,80 @@ class ProjectIndex:
         return self._traverse(node_id, incoming=True, depth=depth, relation=None)
 
     def _edges(self, node_id: str, *, incoming: bool, relation: str | None) -> dict[str, object]:
-        checked = self.check(verify_rows=False)
-        self._require_node(node_id)
         source_column = "target_id" if incoming else "source_id"
         relation_clause = " AND relation = ?" if relation is not None else ""
         values: tuple[object, ...] = (node_id, relation) if relation is not None else (node_id,)
-        with _read_connection(self.path) as connection:
-            rows = connection.execute(
+        with self._read_snapshot() as snapshot:
+            self._require_node(snapshot.connection, node_id)
+            rows = snapshot.connection.execute(
                 f"SELECT source_id, relation, target_id FROM edges "
                 f"WHERE {source_column} = ?{relation_clause} "
                 "ORDER BY source_id, relation, target_id",
                 values,
             ).fetchall()
-        return self._result(checked, edges=[Edge(*row).as_dict() for row in rows])
+            return snapshot.result(edges=[Edge(*row).as_dict() for row in rows])
 
     def _traverse(
         self, node_id: str, *, incoming: bool, depth: int, relation: str | None
     ) -> dict[str, object]:
-        checked = self.check(verify_rows=False)
-        self._require_node(node_id)
         maximum = self.project.descriptor.limits.max_traversal_depth
         if type(depth) is not int or depth < 0 or depth > maximum:
             raise DocForgeError("invalid_depth", "Traversal depth is outside the configured limit")
-        with _read_connection(self.path) as connection:
+        with self._read_snapshot() as snapshot:
+            self._require_node(snapshot.connection, node_id)
             edges = tuple(
                 Edge(*row)
-                for row in connection.execute(
+                for row in snapshot.connection.execute(
                     "SELECT source_id, relation, target_id FROM edges "
                     "ORDER BY source_id, relation, target_id"
                 )
             )
-        queue: deque[tuple[str, int, tuple[str, ...]]] = deque([(node_id, 0, (node_id,))])
-        seen = {node_id}
-        results: list[dict[str, object]] = []
-        while queue:
-            current, current_depth, path = queue.popleft()
-            if current_depth >= depth:
-                continue
-            candidates = [
-                edge
-                for edge in edges
-                if (relation is None or edge.relation == relation)
-                and ((edge.target_id if incoming else edge.source_id) == current)
-            ]
-            for edge in candidates:
-                target = edge.source_id if incoming else edge.target_id
-                if target in seen:
+            queue: deque[tuple[str, int, tuple[str, ...]]] = deque([(node_id, 0, (node_id,))])
+            seen = {node_id}
+            results: list[dict[str, object]] = []
+            while queue:
+                current, current_depth, path = queue.popleft()
+                if current_depth >= depth:
                     continue
-                seen.add(target)
-                target_path = (*path, target)
-                results.append(
-                    {
-                        "node_id": target,
-                        "depth": current_depth + 1,
-                        "relation": edge.relation,
-                        "path": target_path,
-                    }
-                )
-                queue.append((target, current_depth + 1, target_path))
-        return self._result(checked, root=node_id, depth=depth, count=len(results), results=results)
+                candidates = [
+                    edge
+                    for edge in edges
+                    if (relation is None or edge.relation == relation)
+                    and ((edge.target_id if incoming else edge.source_id) == current)
+                ]
+                for edge in candidates:
+                    target = edge.source_id if incoming else edge.target_id
+                    if target in seen:
+                        continue
+                    seen.add(target)
+                    target_path = (*path, target)
+                    results.append(
+                        {
+                            "node_id": target,
+                            "depth": current_depth + 1,
+                            "relation": edge.relation,
+                            "path": target_path,
+                        }
+                    )
+                    queue.append((target, current_depth + 1, target_path))
+            return snapshot.result(
+                root=node_id,
+                depth=depth,
+                count=len(results),
+                results=results,
+            )
 
-    def _require_node(self, node_id: str) -> None:
-        with _read_connection(self.path) as connection:
-            exists = connection.execute(
-                "SELECT 1 FROM nodes WHERE node_id = ?", (node_id,)
-            ).fetchone()
+    @staticmethod
+    def _require_node(connection: sqlite3.Connection, node_id: str) -> None:
+        exists = connection.execute(
+            "SELECT 1 FROM nodes WHERE node_id = ?",
+            (node_id,),
+        ).fetchone()
         if exists is None:
             raise DocForgeError(
                 "missing_node", "No node has the requested stable ID", node_id=node_id
             )
 
-    def _result(self, checked: dict[str, object], **payload: object) -> dict[str, object]:
-        after = self.check(verify_rows=False)
-        if (
-            after["source_hash"] != checked["source_hash"]
-            or after["revision"] != checked["revision"]
-        ):
-            raise DocForgeError("source_changed", "Canonical source changed during the query")
-        return {
-            "status": "ok",
-            "project_id": checked["project_id"],
-            "project_root_fingerprint": checked["project_root_fingerprint"],
-            "revision": checked["revision"],
-            "source_hash": checked["source_hash"],
-            "adapter": checked["adapter"],
-            **payload,
-        }
-
 
 def _row_to_node(row: sqlite3.Row) -> Node:
     return Node(