Make project MCP workflows self-synchronizing

src/docforge/index.py

@@ -2,15 +2,18 @@
 
 from __future__ import annotations
 
+import fcntl
 import hashlib
 import json
 import os
 import sqlite3
 import tempfile
+import time
 from collections import deque
 from collections.abc import Generator
 from contextlib import contextmanager
 from pathlib import Path
+from typing import cast
 
 from .errors import DocForgeError
 from .models import (
@@ -106,12 +109,77 @@ class ProjectIndex:
 
     def __init__(self, project: ProjectService) -> None:
         self.project = project
+        self._verified_index_signature: tuple[int, int, int, int, int] | None = None
 
     @property
     def path(self) -> Path:
         return self.project.descriptor.index_path
 
+    @property
+    def attestation_path(self) -> Path:
+        """Return the project-confined receipt for one fully verified index file."""
+
+        return self.path.with_suffix(f"{self.path.suffix}.attestation.json")
+
     def build(self) -> dict[str, object]:
+        """Build one complete index while excluding concurrent publishers."""
+
+        with self._build_lock():
+            return self._build_locked()
+
+    def synchronize(self) -> dict[str, object]:
+        """Return a current index, rebuilding disposable state when necessary."""
+
+        started = time.perf_counter()
+        try:
+            checked = self.check(verify_rows=False)
+        except DocForgeError as error:
+            if error.code not in {"missing_index", "stale_index", "invalid_index"}:
+                raise
+            initial_error: dict[str, object] | None = error.as_dict()
+        else:
+            temporary_indexes = tuple(self.project.descriptor.cache_root.glob("index-*.sqlite3"))
+            if temporary_indexes:
+                with self._build_lock():
+                    removed = self._remove_temporary_indexes()
+            else:
+                removed = []
+            return {
+                **checked,
+                "synchronization": {
+                    "action": "current",
+                    "elapsed_seconds": round(time.perf_counter() - started, 6),
+                    "initial_error": None,
+                    "removed_temporary_indexes": removed,
+                },
+            }
+
+        with self._build_lock():
+            try:
+                checked = self.check(verify_rows=False)
+            except DocForgeError as error:
+                if error.code not in {"missing_index", "stale_index", "invalid_index"}:
+                    raise
+                removed = self._remove_temporary_indexes()
+                built = self._build_locked()
+                checked = self.check(verify_rows=False)
+                action = "rebuilt"
+                build = built.get("build")
+            else:
+                removed = self._remove_temporary_indexes()
+                action = "current_after_wait"
+                build = None
+        synchronization: dict[str, object] = {
+            "action": action,
+            "elapsed_seconds": round(time.perf_counter() - started, 6),
+            "initial_error": initial_error,
+            "removed_temporary_indexes": removed,
+        }
+        if build is not None:
+            synchronization["build"] = build
+        return {**checked, "synchronization": synchronization}
+
+    def _build_locked(self) -> dict[str, object]:
         snapshot = self.project.load()
         logic = self._logic_projections()
         status = _status(snapshot, logic)
@@ -275,6 +343,8 @@ class ProjectIndex:
             ):
                 raise DocForgeError("source_changed", "Canonical source changed during index build")
             os.replace(temporary, self.path)
+            self._verified_index_signature = self._index_signature()
+            self._write_attestation()
         except sqlite3.Error as error:
             temporary.unlink(missing_ok=True)
             raise DocForgeError("index_failure", "Could not build the derived index") from error
@@ -286,16 +356,49 @@ class ProjectIndex:
             result["build"] = build_report
         return result
 
+    @contextmanager
+    def _build_lock(self) -> Generator[None, None, None]:
+        cache_root = self.project.descriptor.cache_root
+        cache_root.mkdir(parents=True, exist_ok=True)
+        lock_path = cache_root / ".index.lock"
+        try:
+            descriptor = os.open(
+                lock_path,
+                os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW,
+                0o600,
+            )
+        except OSError as error:
+            raise DocForgeError("path_escape", "Index lock path is not safe") from error
+        with os.fdopen(descriptor, "a+b") as handle:
+            fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
+            try:
+                yield
+            finally:
+                fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+
+    def _remove_temporary_indexes(self) -> list[str]:
+        removed: list[str] = []
+        candidates = (
+            *self.project.descriptor.cache_root.glob("index-*.sqlite3"),
+            *self.project.descriptor.cache_root.glob(".index-attestation-*"),
+        )
+        for path in sorted(candidates):
+            if path == self.path or path.is_symlink() or not path.is_file():
+                continue
+            path.unlink()
+            removed.append(path.name)
+        return removed
+
     def _logic_projections(self) -> tuple[LogicProjection, ...]:
         if isinstance(self.project, LogicProject):
             return self.project.logic_projections()
         return ()
 
-    def check(self) -> dict[str, object]:
+    def check(self, *, verify_rows: bool = True) -> dict[str, object]:
         if isinstance(self.project, IncrementalStateProject):
             state = self.project.incremental_state()
             if state is not None:
-                return self._check_incremental_state(state)
+                return self._check_incremental_state(state, verify_rows=verify_rows)
         snapshot = self.project.load()
         logic = self._logic_projections()
         expected = _status(snapshot, logic)
@@ -350,7 +453,12 @@ class ProjectIndex:
                 raise DocForgeError("invalid_index", "Derived index rows do not match source")
         return {**expected, "database": str(self.path)}
 
-    def _check_incremental_state(self, state: ProjectState) -> dict[str, object]:
+    def _check_incremental_state(
+        self,
+        state: ProjectState,
+        *,
+        verify_rows: bool,
+    ) -> dict[str, object]:
         """Validate a published index against cheap current source identity."""
 
         descriptor = self.project.descriptor
@@ -373,6 +481,24 @@ class ProjectIndex:
                     raise DocForgeError(
                         "stale_index", "Derived index does not match canonical source", field=key
                     )
+            current_signature = self._index_signature()
+            if not verify_rows and (
+                current_signature == self._verified_index_signature or self._attestation_matches()
+            ):
+                self._verified_index_signature = current_signature
+                return {
+                    **identity,
+                    "node_hash": metadata["node_hash"],
+                    "node_count": int(metadata["node_count"]),
+                    "edge_hash": metadata["edge_hash"],
+                    "edge_count": int(metadata["edge_count"]),
+                    "logic_hash": metadata["logic_hash"],
+                    "logic_projection_count": int(metadata["logic_projection_count"]),
+                    "logic_node_count": int(metadata["logic_node_count"]),
+                    "logic_edge_count": int(metadata["logic_edge_count"]),
+                    "status": "ok",
+                    "database": str(self.path),
+                }
             integrity = connection.execute("PRAGMA integrity_check").fetchone()
             if integrity is None or integrity[0] != "ok":
                 raise DocForgeError("invalid_index", "Derived index failed SQLite integrity check")
@@ -406,6 +532,8 @@ class ProjectIndex:
                 or fts_count != len(indexed_nodes)
             ):
                 raise DocForgeError("invalid_index", "Derived index rows do not match metadata")
+        self._verified_index_signature = current_signature
+        self._write_attestation()
         return {
             **identity,
             "node_hash": node_hash,
@@ -420,8 +548,91 @@ class ProjectIndex:
             "database": str(self.path),
         }
 
+    def _attestation_matches(self) -> bool:
+        """Verify a persisted whole-file digest before trusting a warm derived index."""
+
+        path = self.attestation_path
+        if not path.is_file() or path.is_symlink():
+            return False
+        try:
+            parsed: object = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+            return False
+        if not isinstance(parsed, dict):
+            return False
+        payload = cast(dict[str, object], parsed)
+        expected_size = payload.get("index_size")
+        expected_hash = payload.get("index_sha256")
+        if (
+            payload.get("schema_version") != 1
+            or type(expected_size) is not int
+            or not isinstance(expected_hash, str)
+            or len(expected_hash) != 64
+        ):
+            return False
+        try:
+            if self.path.stat().st_size != expected_size:
+                return False
+            with self.path.open("rb") as handle:
+                actual_hash = hashlib.file_digest(handle, "sha256").hexdigest()
+        except OSError:
+            return False
+        return actual_hash == expected_hash
+
+    def _write_attestation(self) -> None:
+        """Atomically persist the digest of an index that passed complete verification."""
+
+        root = self.project.descriptor.cache_root
+        path = self.attestation_path
+        if path.parent != root or path.is_symlink():
+            raise DocForgeError("path_escape", "Index attestation path is not safe")
+        try:
+            size = self.path.stat().st_size
+            with self.path.open("rb") as handle:
+                index_hash = hashlib.file_digest(handle, "sha256").hexdigest()
+        except OSError as error:
+            raise DocForgeError("missing_index", "Derived index cannot be attested") from error
+        payload = {
+            "schema_version": 1,
+            "index_size": size,
+            "index_sha256": index_hash,
+        }
+        raw = json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n"
+        descriptor, temporary_name = tempfile.mkstemp(prefix=".index-attestation-", dir=root)
+        temporary = Path(temporary_name)
+        try:
+            with os.fdopen(descriptor, "wb") as handle:
+                handle.write(raw)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temporary, path)
+            directory_descriptor = os.open(root, os.O_RDONLY)
+            try:
+                os.fsync(directory_descriptor)
+            finally:
+                os.close(directory_descriptor)
+        except Exception:
+            temporary.unlink(missing_ok=True)
+            raise
+
+    def _index_signature(self) -> tuple[int, int, int, int, int]:
+        try:
+            status = self.path.stat()
+        except OSError as error:
+            raise DocForgeError(
+                "missing_index",
+                "Derived index does not exist; run build first",
+            ) from error
+        return (
+            status.st_dev,
+            status.st_ino,
+            status.st_size,
+            status.st_mtime_ns,
+            status.st_ctime_ns,
+        )
+
     def get_node(self, node_id: str) -> dict[str, object]:
-        checked = self.check()
+        checked = self.check(verify_rows=False)
         with _read_connection(self.path) as connection:
             row = connection.execute("SELECT * FROM nodes WHERE node_id = ?", (node_id,)).fetchone()
         if row is None:
@@ -433,7 +644,7 @@ class ProjectIndex:
     def get_logic(self, owner_node_id: str) -> dict[str, object]:
         """Return one function-scoped control-flow projection without expanding the graph."""
 
-        checked = self.check()
+        checked = self.check(verify_rows=False)
         with _read_connection(self.path) as connection:
             owner = connection.execute(
                 "SELECT * FROM nodes WHERE node_id = ?", (owner_node_id,)
@@ -453,7 +664,7 @@ class ProjectIndex:
         )
 
     def search(self, query: str, *, limit: int | None = None) -> dict[str, object]:
-        checked = self.check()
+        checked = self.check(verify_rows=False)
         limits = self.project.descriptor.limits
         if not query.strip() or len(query) > limits.max_query_chars:
             raise DocForgeError("invalid_query", "Search query is empty or exceeds its limit")
@@ -490,7 +701,7 @@ class ProjectIndex:
         tag: str | None = None,
         limit: int | None = None,
     ) -> dict[str, object]:
-        checked = self.check()
+        checked = self.check(verify_rows=False)
         bounded = _bounded_limit(limit, self.project.descriptor.limits.max_results, default=100)
         clauses: list[str] = []
         values: list[object] = []
@@ -519,7 +730,7 @@ class ProjectIndex:
         return self._traverse(node_id, incoming=True, depth=depth, relation=None)
 
     def _edges(self, node_id: str, *, incoming: bool, relation: str | None) -> dict[str, object]:
-        checked = self.check()
+        checked = self.check(verify_rows=False)
         self._require_node(node_id)
         source_column = "target_id" if incoming else "source_id"
         relation_clause = " AND relation = ?" if relation is not None else ""
@@ -536,7 +747,7 @@ class ProjectIndex:
     def _traverse(
         self, node_id: str, *, incoming: bool, depth: int, relation: str | None
     ) -> dict[str, object]:
-        checked = self.check()
+        checked = self.check(verify_rows=False)
         self._require_node(node_id)
         maximum = self.project.descriptor.limits.max_traversal_depth
         if type(depth) is not int or depth < 0 or depth > maximum:
@@ -590,7 +801,7 @@ class ProjectIndex:
             )
 
     def _result(self, checked: dict[str, object], **payload: object) -> dict[str, object]:
-        after = self.check()
+        after = self.check(verify_rows=False)
         if (
             after["source_hash"] != checked["source_hash"]
             or after["revision"] != checked["revision"]