1144 lines
46 KiB
Python
1144 lines
46 KiB
Python
"""Atomic derived SQLite index and deterministic read operations."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import tempfile
|
|
import time
|
|
from collections import deque
|
|
from collections.abc import Callable, Generator
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import cast
|
|
|
|
from .errors import DocForgeError
|
|
from .models import (
|
|
BuildReportingProject,
|
|
Edge,
|
|
GenerationRecordingProject,
|
|
IncrementalStateProject,
|
|
LogicEdge,
|
|
LogicNode,
|
|
LogicProject,
|
|
LogicProjection,
|
|
Node,
|
|
ProjectDescriptor,
|
|
ProjectService,
|
|
ProjectSnapshot,
|
|
ProjectState,
|
|
)
|
|
from .project import project_root_fingerprint
|
|
|
|
INDEX_SCHEMA_VERSION = 3
|
|
APPLICATION_ID = 1_146_683_778
|
|
|
|
|
|
def _node_hash(nodes: tuple[Node, ...]) -> str:
|
|
payload = json.dumps(
|
|
[node.as_dict() for node in nodes], sort_keys=True, separators=(",", ":")
|
|
).encode()
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def _edge_hash(edges: tuple[Edge, ...]) -> str:
|
|
payload = json.dumps(
|
|
[edge.as_dict() for edge in edges], sort_keys=True, separators=(",", ":")
|
|
).encode()
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def _logic_hash(projections: tuple[LogicProjection, ...]) -> str:
|
|
payload = json.dumps(
|
|
[projection.as_dict() for projection in projections],
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode()
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def _connect_read_only(path: Path) -> sqlite3.Connection:
|
|
if not path.is_file():
|
|
raise DocForgeError("missing_index", "Derived index does not exist; run build first")
|
|
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
connection.row_factory = sqlite3.Row
|
|
return connection
|
|
|
|
|
|
@contextmanager
|
|
def _read_connection(path: Path) -> Generator[sqlite3.Connection, None, None]:
|
|
connection: sqlite3.Connection | None = None
|
|
try:
|
|
connection = _connect_read_only(path)
|
|
yield connection
|
|
except DocForgeError:
|
|
raise
|
|
except (sqlite3.Error, json.JSONDecodeError, KeyError, TypeError) as error:
|
|
raise DocForgeError("invalid_index", "Derived index is corrupt or unreadable") from error
|
|
finally:
|
|
if connection is not None:
|
|
connection.close()
|
|
|
|
|
|
def _status(
|
|
snapshot: ProjectSnapshot,
|
|
logic: tuple[LogicProjection, ...],
|
|
) -> dict[str, object]:
|
|
return {
|
|
"project_id": snapshot.descriptor.project_id,
|
|
"project_root_fingerprint": project_root_fingerprint(snapshot.descriptor.root),
|
|
"revision": snapshot.revision,
|
|
"source_hash": snapshot.source_hash,
|
|
"node_hash": _node_hash(snapshot.nodes),
|
|
"node_count": len(snapshot.nodes),
|
|
"edge_hash": _edge_hash(snapshot.edges),
|
|
"edge_count": len(snapshot.edges),
|
|
"logic_hash": _logic_hash(logic),
|
|
"logic_projection_count": len(logic),
|
|
"logic_node_count": sum(len(projection.nodes) for projection in logic),
|
|
"logic_edge_count": sum(len(projection.edges) for projection in logic),
|
|
"index_schema_version": INDEX_SCHEMA_VERSION,
|
|
"adapter": snapshot.descriptor.adapter,
|
|
"status": "ok",
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _IndexReadSnapshot:
|
|
"""One request-scoped read transaction over a verified immutable generation."""
|
|
|
|
connection: sqlite3.Connection
|
|
checked: dict[str, object]
|
|
|
|
def project_snapshot(self, descriptor: ProjectDescriptor) -> ProjectSnapshot:
|
|
nodes = tuple(
|
|
_row_to_node(row)
|
|
for row in self.connection.execute("SELECT * FROM nodes ORDER BY node_id")
|
|
)
|
|
edges = tuple(
|
|
Edge(*row)
|
|
for row in self.connection.execute(
|
|
"SELECT source_id, relation, target_id FROM edges "
|
|
"ORDER BY source_id, relation, target_id"
|
|
)
|
|
)
|
|
return ProjectSnapshot(
|
|
descriptor=descriptor,
|
|
nodes=nodes,
|
|
edges=edges,
|
|
source_hash=cast(str, self.checked["source_hash"]),
|
|
revision=cast(str, self.checked["revision"]),
|
|
)
|
|
|
|
def result(self, **payload: object) -> dict[str, object]:
|
|
return {
|
|
"status": "ok",
|
|
"project_id": self.checked["project_id"],
|
|
"project_root_fingerprint": self.checked["project_root_fingerprint"],
|
|
"revision": self.checked["revision"],
|
|
"source_hash": self.checked["source_hash"],
|
|
"adapter": self.checked["adapter"],
|
|
**payload,
|
|
}
|
|
|
|
|
|
class ProjectIndex:
|
|
"""A disposable index that always checks current canonical source before queries."""
|
|
|
|
def __init__(self, project: ProjectService, *, allow_logic: bool = True) -> None:
|
|
self.project = project
|
|
self.allow_logic = allow_logic
|
|
self._verified_index_signature: tuple[int, int, int, int, int] | None = None
|
|
|
|
@property
|
|
def path(self) -> Path:
|
|
return self.project.descriptor.index_path
|
|
|
|
@property
|
|
def attestation_path(self) -> Path:
|
|
"""Return the project-confined receipt for one fully verified index file."""
|
|
|
|
return self.path.with_suffix(f"{self.path.suffix}.attestation.json")
|
|
|
|
def build(self) -> dict[str, object]:
|
|
"""Build one complete index while excluding concurrent publishers."""
|
|
|
|
with self._build_lock():
|
|
return self._build_locked()
|
|
|
|
def synchronize(self) -> dict[str, object]:
|
|
"""Return a current index, rebuilding disposable state when necessary."""
|
|
|
|
started = time.perf_counter()
|
|
try:
|
|
checked = self.check(verify_rows=False)
|
|
except DocForgeError as error:
|
|
if error.code not in {"missing_index", "stale_index", "invalid_index"}:
|
|
raise
|
|
initial_error: dict[str, object] | None = error.as_dict()
|
|
else:
|
|
temporary_indexes = tuple(self.project.descriptor.cache_root.glob("index-*.sqlite3"))
|
|
if temporary_indexes:
|
|
with self._build_lock():
|
|
removed = self._remove_temporary_indexes()
|
|
else:
|
|
removed = []
|
|
return {
|
|
**checked,
|
|
"synchronization": {
|
|
"action": "current",
|
|
"elapsed_seconds": round(time.perf_counter() - started, 6),
|
|
"initial_error": None,
|
|
"removed_temporary_indexes": removed,
|
|
},
|
|
}
|
|
|
|
with self._build_lock():
|
|
try:
|
|
checked = self.check(verify_rows=False)
|
|
except DocForgeError as error:
|
|
if error.code not in {"missing_index", "stale_index", "invalid_index"}:
|
|
raise
|
|
removed = self._remove_temporary_indexes()
|
|
built = self._build_locked()
|
|
checked = self.check(verify_rows=False)
|
|
action = "rebuilt"
|
|
build = built.get("build")
|
|
else:
|
|
removed = self._remove_temporary_indexes()
|
|
action = "current_after_wait"
|
|
build = None
|
|
synchronization: dict[str, object] = {
|
|
"action": action,
|
|
"elapsed_seconds": round(time.perf_counter() - started, 6),
|
|
"initial_error": initial_error,
|
|
"removed_temporary_indexes": removed,
|
|
}
|
|
if build is not None:
|
|
synchronization["build"] = build
|
|
return {**checked, "synchronization": synchronization}
|
|
|
|
def _build_locked(self) -> dict[str, object]:
|
|
snapshot = self.project.load()
|
|
logic = self._logic_projections()
|
|
status = _status(snapshot, logic)
|
|
build_report = (
|
|
self.project.build_report() if isinstance(self.project, BuildReportingProject) else None
|
|
)
|
|
if build_report is not None and build_report.get("mode") != "incremental":
|
|
build_report = None
|
|
cache_root = snapshot.descriptor.cache_root
|
|
cache_root.mkdir(parents=True, exist_ok=True)
|
|
with tempfile.NamedTemporaryFile(
|
|
prefix="index-", suffix=".sqlite3", dir=cache_root, delete=False
|
|
) as descriptor:
|
|
temporary = Path(descriptor.name)
|
|
try:
|
|
connection = sqlite3.connect(temporary)
|
|
try:
|
|
connection.execute(f"PRAGMA application_id={APPLICATION_ID}")
|
|
connection.execute(f"PRAGMA user_version={INDEX_SCHEMA_VERSION}")
|
|
connection.executescript(
|
|
"""
|
|
CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
CREATE TABLE nodes (
|
|
node_id TEXT PRIMARY KEY,
|
|
title TEXT NOT NULL,
|
|
family TEXT NOT NULL,
|
|
authority TEXT NOT NULL,
|
|
status TEXT NOT NULL,
|
|
tags_json TEXT NOT NULL,
|
|
summary TEXT NOT NULL,
|
|
content TEXT NOT NULL,
|
|
source_path TEXT NOT NULL,
|
|
source_anchor TEXT,
|
|
content_hash TEXT NOT NULL
|
|
);
|
|
CREATE TABLE edges (
|
|
source_id TEXT NOT NULL,
|
|
relation TEXT NOT NULL,
|
|
target_id TEXT NOT NULL,
|
|
PRIMARY KEY (source_id, relation, target_id)
|
|
);
|
|
CREATE INDEX edges_target ON edges(target_id, relation, source_id);
|
|
CREATE INDEX edges_target_source
|
|
ON edges(target_id, source_id, relation);
|
|
CREATE TABLE logic_owners (
|
|
owner_node_id TEXT PRIMARY KEY,
|
|
source_id TEXT NOT NULL
|
|
);
|
|
CREATE TABLE logic_nodes (
|
|
owner_node_id TEXT NOT NULL,
|
|
logic_id TEXT NOT NULL,
|
|
kind TEXT NOT NULL,
|
|
label TEXT NOT NULL,
|
|
source_anchor TEXT,
|
|
PRIMARY KEY (owner_node_id, logic_id)
|
|
);
|
|
CREATE INDEX logic_nodes_id ON logic_nodes(logic_id, owner_node_id);
|
|
CREATE TABLE logic_edges (
|
|
owner_node_id TEXT NOT NULL,
|
|
source_id TEXT NOT NULL,
|
|
relation TEXT NOT NULL,
|
|
target_id TEXT NOT NULL,
|
|
label TEXT,
|
|
ordinal INTEGER NOT NULL,
|
|
PRIMARY KEY (
|
|
owner_node_id, source_id, ordinal, relation, target_id
|
|
)
|
|
);
|
|
CREATE INDEX logic_edges_target
|
|
ON logic_edges(owner_node_id, target_id, source_id);
|
|
CREATE VIRTUAL TABLE node_fts USING fts5(
|
|
node_id UNINDEXED, title, summary, content, tags
|
|
);
|
|
"""
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO metadata(key, value) VALUES (?, ?)",
|
|
sorted((key, str(value)) for key, value in status.items()),
|
|
)
|
|
connection.executemany(
|
|
"""
|
|
INSERT INTO nodes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
[
|
|
(
|
|
node.node_id,
|
|
node.title,
|
|
node.family,
|
|
node.authority,
|
|
node.status,
|
|
json.dumps(node.tags, separators=(",", ":")),
|
|
node.summary,
|
|
node.content,
|
|
node.source_path,
|
|
node.source_anchor,
|
|
node.content_hash,
|
|
)
|
|
for node in snapshot.nodes
|
|
],
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO edges VALUES (?, ?, ?)",
|
|
[(edge.source_id, edge.relation, edge.target_id) for edge in snapshot.edges],
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO logic_owners VALUES (?, ?)",
|
|
[(projection.owner_node_id, projection.source_id) for projection in logic],
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO logic_nodes VALUES (?, ?, ?, ?, ?)",
|
|
[
|
|
(
|
|
projection.owner_node_id,
|
|
node.logic_id,
|
|
node.kind,
|
|
node.label,
|
|
node.source_anchor,
|
|
)
|
|
for projection in logic
|
|
for node in projection.nodes
|
|
],
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO logic_edges VALUES (?, ?, ?, ?, ?, ?)",
|
|
[
|
|
(
|
|
projection.owner_node_id,
|
|
edge.source_id,
|
|
edge.relation,
|
|
edge.target_id,
|
|
edge.label,
|
|
edge.ordinal,
|
|
)
|
|
for projection in logic
|
|
for edge in projection.edges
|
|
],
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO node_fts VALUES (?, ?, ?, ?, ?)",
|
|
[
|
|
(
|
|
node.node_id,
|
|
node.title,
|
|
node.summary,
|
|
node.content,
|
|
" ".join(node.tags),
|
|
)
|
|
for node in snapshot.nodes
|
|
],
|
|
)
|
|
connection.commit()
|
|
integrity = connection.execute("PRAGMA integrity_check").fetchone()
|
|
if integrity is None or integrity[0] != "ok":
|
|
raise DocForgeError("index_failure", "New index failed SQLite integrity check")
|
|
finally:
|
|
connection.close()
|
|
current = self.project.load()
|
|
current_logic = self._logic_projections()
|
|
if (
|
|
current.source_hash != snapshot.source_hash
|
|
or current.revision != snapshot.revision
|
|
or current_logic != logic
|
|
):
|
|
raise DocForgeError("source_changed", "Canonical source changed during index build")
|
|
os.replace(temporary, self.path)
|
|
self._verified_index_signature = self._index_signature()
|
|
self._write_attestation()
|
|
if isinstance(self.project, GenerationRecordingProject):
|
|
self.project.record_generation(current)
|
|
except sqlite3.Error as error:
|
|
temporary.unlink(missing_ok=True)
|
|
raise DocForgeError("index_failure", "Could not build the derived index") from error
|
|
except Exception:
|
|
temporary.unlink(missing_ok=True)
|
|
raise
|
|
result: dict[str, object] = {**status, "database": str(self.path)}
|
|
if build_report is not None:
|
|
result["build"] = build_report
|
|
return result
|
|
|
|
@contextmanager
|
|
def _build_lock(self) -> Generator[None, None, None]:
|
|
cache_root = self.project.descriptor.cache_root
|
|
cache_root.mkdir(parents=True, exist_ok=True)
|
|
lock_path = cache_root / ".index.lock"
|
|
try:
|
|
descriptor = os.open(
|
|
lock_path,
|
|
os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW,
|
|
0o600,
|
|
)
|
|
except OSError as error:
|
|
raise DocForgeError("path_escape", "Index lock path is not safe") from error
|
|
with os.fdopen(descriptor, "a+b") as handle:
|
|
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
|
try:
|
|
yield
|
|
finally:
|
|
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
|
|
|
def _remove_temporary_indexes(self) -> list[str]:
|
|
removed: list[str] = []
|
|
candidates = (
|
|
*self.project.descriptor.cache_root.glob("index-*.sqlite3"),
|
|
*self.project.descriptor.cache_root.glob(".index-attestation-*"),
|
|
)
|
|
for path in sorted(candidates):
|
|
if path == self.path or path.is_symlink() or not path.is_file():
|
|
continue
|
|
path.unlink()
|
|
removed.append(path.name)
|
|
return removed
|
|
|
|
def _logic_projections(self) -> tuple[LogicProjection, ...]:
|
|
if isinstance(self.project, LogicProject):
|
|
projections = self.project.logic_projections()
|
|
self._require_logic_allowed(len(projections))
|
|
return projections
|
|
return ()
|
|
|
|
def _require_logic_allowed(self, projection_count: int) -> None:
|
|
if projection_count and not self.allow_logic:
|
|
raise DocForgeError(
|
|
"adapter_policy_forbids_logic",
|
|
(
|
|
"This index preserves a no-AST adapter and refuses function-Logic "
|
|
"publication or retrieval"
|
|
),
|
|
logic_projection_count=projection_count,
|
|
)
|
|
|
|
@contextmanager
|
|
def _read_snapshot(self) -> Generator[_IndexReadSnapshot, None, None]:
|
|
"""Pin one verified index and source generation for a complete read request."""
|
|
|
|
checked = self.check(verify_rows=False)
|
|
signature = self._verified_index_signature
|
|
if signature is None or self._index_signature() != signature:
|
|
raise DocForgeError(
|
|
"invalid_index",
|
|
"Derived index changed after validation",
|
|
)
|
|
with _read_connection(self.path) as connection:
|
|
connection.execute("PRAGMA query_only=ON")
|
|
connection.execute("BEGIN")
|
|
application_id = connection.execute("PRAGMA application_id").fetchone()[0]
|
|
schema_version = connection.execute("PRAGMA user_version").fetchone()[0]
|
|
if application_id != APPLICATION_ID or schema_version != INDEX_SCHEMA_VERSION:
|
|
raise DocForgeError("invalid_index", "Derived index has an unsupported schema")
|
|
metadata = dict(connection.execute("SELECT key, value FROM metadata"))
|
|
for key in (
|
|
"project_id",
|
|
"project_root_fingerprint",
|
|
"revision",
|
|
"source_hash",
|
|
"index_schema_version",
|
|
"adapter",
|
|
):
|
|
if metadata.get(key) != str(checked[key]):
|
|
raise DocForgeError(
|
|
"invalid_index",
|
|
"Derived index changed after validation",
|
|
field=key,
|
|
)
|
|
try:
|
|
logic_projection_count = int(metadata["logic_projection_count"])
|
|
except (KeyError, ValueError) as error:
|
|
raise DocForgeError(
|
|
"invalid_index",
|
|
"Derived index has invalid Logic metadata",
|
|
) from error
|
|
self._require_logic_allowed(logic_projection_count)
|
|
snapshot = _IndexReadSnapshot(connection=connection, checked=checked)
|
|
try:
|
|
yield snapshot
|
|
except Exception:
|
|
raise
|
|
else:
|
|
self._confirm_read(snapshot.checked, signature)
|
|
|
|
def _confirm_read(
|
|
self,
|
|
checked: dict[str, object],
|
|
signature: tuple[int, int, int, int, int],
|
|
) -> None:
|
|
if self._index_signature() != signature:
|
|
raise DocForgeError("invalid_index", "Derived index changed during the query")
|
|
state: ProjectState | None = None
|
|
if isinstance(self.project, IncrementalStateProject):
|
|
state = self.project.incremental_state()
|
|
if state is None:
|
|
current = self.project.load()
|
|
state = ProjectState(source_hash=current.source_hash, revision=current.revision)
|
|
if state.source_hash != checked["source_hash"] or state.revision != checked["revision"]:
|
|
raise DocForgeError("source_changed", "Canonical source changed during the query")
|
|
|
|
def read_project_snapshot(
|
|
self,
|
|
reader: Callable[[ProjectSnapshot], dict[str, object]],
|
|
) -> dict[str, object]:
|
|
"""Run one bounded reader against an immutable derived project snapshot."""
|
|
|
|
with self._read_snapshot() as snapshot:
|
|
payload = reader(snapshot.project_snapshot(self.project.descriptor))
|
|
return snapshot.result(**payload)
|
|
|
|
def check(self, *, verify_rows: bool = True) -> dict[str, object]:
|
|
if isinstance(self.project, IncrementalStateProject):
|
|
state = self.project.incremental_state()
|
|
if state is not None:
|
|
return self._check_incremental_state(state, verify_rows=verify_rows)
|
|
snapshot = self.project.load()
|
|
logic = self._logic_projections()
|
|
expected = _status(snapshot, logic)
|
|
with _read_connection(self.path) as connection:
|
|
application_id = connection.execute("PRAGMA application_id").fetchone()[0]
|
|
schema_version = connection.execute("PRAGMA user_version").fetchone()[0]
|
|
if application_id != APPLICATION_ID or schema_version != INDEX_SCHEMA_VERSION:
|
|
raise DocForgeError("invalid_index", "Derived index has an unsupported schema")
|
|
metadata = dict(connection.execute("SELECT key, value FROM metadata"))
|
|
for key in (
|
|
"project_id",
|
|
"project_root_fingerprint",
|
|
"revision",
|
|
"source_hash",
|
|
"node_hash",
|
|
"node_count",
|
|
"edge_hash",
|
|
"edge_count",
|
|
"logic_hash",
|
|
"logic_projection_count",
|
|
"logic_node_count",
|
|
"logic_edge_count",
|
|
"index_schema_version",
|
|
"adapter",
|
|
):
|
|
if metadata.get(key) != str(expected[key]):
|
|
raise DocForgeError(
|
|
"stale_index", "Derived index does not match canonical source", field=key
|
|
)
|
|
integrity = connection.execute("PRAGMA integrity_check").fetchone()
|
|
if integrity is None or integrity[0] != "ok":
|
|
raise DocForgeError("invalid_index", "Derived index failed SQLite integrity check")
|
|
indexed_nodes = tuple(
|
|
_row_to_node(row)
|
|
for row in connection.execute("SELECT * FROM nodes ORDER BY node_id")
|
|
)
|
|
indexed_edges = tuple(
|
|
Edge(*row)
|
|
for row in connection.execute(
|
|
"SELECT source_id, relation, target_id FROM edges "
|
|
"ORDER BY source_id, relation, target_id"
|
|
)
|
|
)
|
|
indexed_logic = _logic_from_connection(connection)
|
|
fts_count = connection.execute("SELECT COUNT(*) FROM node_fts").fetchone()[0]
|
|
if (
|
|
indexed_nodes != snapshot.nodes
|
|
or indexed_edges != snapshot.edges
|
|
or indexed_logic != logic
|
|
or fts_count != len(snapshot.nodes)
|
|
):
|
|
raise DocForgeError("invalid_index", "Derived index rows do not match source")
|
|
self._verified_index_signature = self._index_signature()
|
|
if isinstance(self.project, GenerationRecordingProject):
|
|
self.project.record_generation(snapshot)
|
|
return {**expected, "database": str(self.path)}
|
|
|
|
def _check_incremental_state(
|
|
self,
|
|
state: ProjectState,
|
|
*,
|
|
verify_rows: bool,
|
|
) -> dict[str, object]:
|
|
"""Validate a published index against cheap current source identity."""
|
|
|
|
descriptor = self.project.descriptor
|
|
identity = {
|
|
"project_id": descriptor.project_id,
|
|
"project_root_fingerprint": project_root_fingerprint(descriptor.root),
|
|
"revision": state.revision,
|
|
"source_hash": state.source_hash,
|
|
"index_schema_version": INDEX_SCHEMA_VERSION,
|
|
"adapter": descriptor.adapter,
|
|
}
|
|
with _read_connection(self.path) as connection:
|
|
application_id = connection.execute("PRAGMA application_id").fetchone()[0]
|
|
schema_version = connection.execute("PRAGMA user_version").fetchone()[0]
|
|
if application_id != APPLICATION_ID or schema_version != INDEX_SCHEMA_VERSION:
|
|
raise DocForgeError("invalid_index", "Derived index has an unsupported schema")
|
|
metadata = dict(connection.execute("SELECT key, value FROM metadata"))
|
|
for key, expected in identity.items():
|
|
if metadata.get(key) != str(expected):
|
|
raise DocForgeError(
|
|
"stale_index", "Derived index does not match canonical source", field=key
|
|
)
|
|
try:
|
|
logic_projection_count = int(metadata["logic_projection_count"])
|
|
except (KeyError, ValueError) as error:
|
|
raise DocForgeError(
|
|
"invalid_index",
|
|
"Derived index has invalid Logic metadata",
|
|
) from error
|
|
self._require_logic_allowed(logic_projection_count)
|
|
current_signature = self._index_signature()
|
|
if not verify_rows and (
|
|
current_signature == self._verified_index_signature or self._attestation_matches()
|
|
):
|
|
self._verified_index_signature = current_signature
|
|
return {
|
|
**identity,
|
|
"node_hash": metadata["node_hash"],
|
|
"node_count": int(metadata["node_count"]),
|
|
"edge_hash": metadata["edge_hash"],
|
|
"edge_count": int(metadata["edge_count"]),
|
|
"logic_hash": metadata["logic_hash"],
|
|
"logic_projection_count": logic_projection_count,
|
|
"logic_node_count": int(metadata["logic_node_count"]),
|
|
"logic_edge_count": int(metadata["logic_edge_count"]),
|
|
"status": "ok",
|
|
"database": str(self.path),
|
|
}
|
|
integrity = connection.execute("PRAGMA integrity_check").fetchone()
|
|
if integrity is None or integrity[0] != "ok":
|
|
raise DocForgeError("invalid_index", "Derived index failed SQLite integrity check")
|
|
indexed_nodes = tuple(
|
|
_row_to_node(row)
|
|
for row in connection.execute("SELECT * FROM nodes ORDER BY node_id")
|
|
)
|
|
indexed_edges = tuple(
|
|
Edge(*row)
|
|
for row in connection.execute(
|
|
"SELECT source_id, relation, target_id FROM edges "
|
|
"ORDER BY source_id, relation, target_id"
|
|
)
|
|
)
|
|
indexed_logic = _logic_from_connection(connection)
|
|
node_hash = _node_hash(indexed_nodes)
|
|
edge_hash = _edge_hash(indexed_edges)
|
|
logic_hash = _logic_hash(indexed_logic)
|
|
fts_count = connection.execute("SELECT COUNT(*) FROM node_fts").fetchone()[0]
|
|
if (
|
|
metadata.get("node_hash") != node_hash
|
|
or metadata.get("edge_hash") != edge_hash
|
|
or metadata.get("logic_hash") != logic_hash
|
|
or metadata.get("node_count") != str(len(indexed_nodes))
|
|
or metadata.get("edge_count") != str(len(indexed_edges))
|
|
or metadata.get("logic_projection_count") != str(len(indexed_logic))
|
|
or metadata.get("logic_node_count")
|
|
!= str(sum(len(projection.nodes) for projection in indexed_logic))
|
|
or metadata.get("logic_edge_count")
|
|
!= str(sum(len(projection.edges) for projection in indexed_logic))
|
|
or fts_count != len(indexed_nodes)
|
|
):
|
|
raise DocForgeError("invalid_index", "Derived index rows do not match metadata")
|
|
self._verified_index_signature = current_signature
|
|
self._write_attestation()
|
|
return {
|
|
**identity,
|
|
"node_hash": node_hash,
|
|
"node_count": len(indexed_nodes),
|
|
"edge_hash": edge_hash,
|
|
"edge_count": len(indexed_edges),
|
|
"logic_hash": logic_hash,
|
|
"logic_projection_count": len(indexed_logic),
|
|
"logic_node_count": sum(len(projection.nodes) for projection in indexed_logic),
|
|
"logic_edge_count": sum(len(projection.edges) for projection in indexed_logic),
|
|
"status": "ok",
|
|
"database": str(self.path),
|
|
}
|
|
|
|
def _attestation_matches(self) -> bool:
|
|
"""Verify a persisted whole-file digest before trusting a warm derived index."""
|
|
|
|
path = self.attestation_path
|
|
if not path.is_file() or path.is_symlink():
|
|
return False
|
|
try:
|
|
parsed: object = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
return False
|
|
if not isinstance(parsed, dict):
|
|
return False
|
|
payload = cast(dict[str, object], parsed)
|
|
expected_size = payload.get("index_size")
|
|
expected_hash = payload.get("index_sha256")
|
|
if (
|
|
payload.get("schema_version") != 1
|
|
or type(expected_size) is not int
|
|
or not isinstance(expected_hash, str)
|
|
or len(expected_hash) != 64
|
|
):
|
|
return False
|
|
try:
|
|
if self.path.stat().st_size != expected_size:
|
|
return False
|
|
with self.path.open("rb") as handle:
|
|
actual_hash = hashlib.file_digest(handle, "sha256").hexdigest()
|
|
except OSError:
|
|
return False
|
|
return actual_hash == expected_hash
|
|
|
|
def _write_attestation(self) -> None:
|
|
"""Atomically persist the digest of an index that passed complete verification."""
|
|
|
|
root = self.project.descriptor.cache_root
|
|
path = self.attestation_path
|
|
if path.parent != root or path.is_symlink():
|
|
raise DocForgeError("path_escape", "Index attestation path is not safe")
|
|
try:
|
|
size = self.path.stat().st_size
|
|
with self.path.open("rb") as handle:
|
|
index_hash = hashlib.file_digest(handle, "sha256").hexdigest()
|
|
except OSError as error:
|
|
raise DocForgeError("missing_index", "Derived index cannot be attested") from error
|
|
payload = {
|
|
"schema_version": 1,
|
|
"index_size": size,
|
|
"index_sha256": index_hash,
|
|
}
|
|
raw = json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n"
|
|
descriptor, temporary_name = tempfile.mkstemp(prefix=".index-attestation-", dir=root)
|
|
temporary = Path(temporary_name)
|
|
try:
|
|
with os.fdopen(descriptor, "wb") as handle:
|
|
handle.write(raw)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, path)
|
|
directory_descriptor = os.open(root, os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory_descriptor)
|
|
finally:
|
|
os.close(directory_descriptor)
|
|
except Exception:
|
|
temporary.unlink(missing_ok=True)
|
|
raise
|
|
|
|
def _index_signature(self) -> tuple[int, int, int, int, int]:
|
|
try:
|
|
status = self.path.stat()
|
|
except OSError as error:
|
|
raise DocForgeError(
|
|
"missing_index",
|
|
"Derived index does not exist; run build first",
|
|
) from error
|
|
return (
|
|
status.st_dev,
|
|
status.st_ino,
|
|
status.st_size,
|
|
status.st_mtime_ns,
|
|
status.st_ctime_ns,
|
|
)
|
|
|
|
def get_node(self, node_id: str) -> dict[str, object]:
|
|
with self._read_snapshot() as snapshot:
|
|
row = snapshot.connection.execute(
|
|
"SELECT * FROM nodes WHERE node_id = ?",
|
|
(node_id,),
|
|
).fetchone()
|
|
if row is None:
|
|
raise DocForgeError(
|
|
"missing_node", "No node has the requested stable ID", node_id=node_id
|
|
)
|
|
return snapshot.result(node=_row_to_node(row).as_dict())
|
|
|
|
def get_logic(self, owner_node_id: str) -> dict[str, object]:
|
|
"""Return one function-scoped control-flow projection without expanding the graph."""
|
|
|
|
with self._read_snapshot() as snapshot:
|
|
owner = snapshot.connection.execute(
|
|
"SELECT * FROM nodes WHERE node_id = ?", (owner_node_id,)
|
|
).fetchone()
|
|
projection = _logic_projection_from_connection(snapshot.connection, owner_node_id)
|
|
if owner is None:
|
|
raise DocForgeError(
|
|
"missing_node",
|
|
"No node has the requested stable ID",
|
|
node_id=owner_node_id,
|
|
)
|
|
return snapshot.result(
|
|
owner=_row_to_node(owner).as_dict(include_content=False),
|
|
available=projection is not None,
|
|
projection=projection.as_dict() if projection is not None else None,
|
|
)
|
|
|
|
def search(self, query: str, *, limit: int | None = None) -> dict[str, object]:
|
|
limits = self.project.descriptor.limits
|
|
if not query.strip() or len(query) > limits.max_query_chars:
|
|
raise DocForgeError("invalid_query", "Search query is empty or exceeds its limit")
|
|
bounded = _bounded_limit(limit, limits.max_results, default=20)
|
|
terms = re_tokenize(query)
|
|
if not terms:
|
|
raise DocForgeError("invalid_query", "Search query contains no searchable text")
|
|
expression = " AND ".join(f'"{term.replace(chr(34), chr(34) * 2)}"' for term in terms)
|
|
with self._read_snapshot() as snapshot:
|
|
rows = snapshot.connection.execute(
|
|
"""
|
|
SELECT nodes.*, bm25(node_fts) AS rank,
|
|
snippet(node_fts, 3, '[', ']', ' … ', 18) AS snippet
|
|
FROM node_fts JOIN nodes USING(node_id)
|
|
WHERE node_fts MATCH ?
|
|
ORDER BY rank, nodes.node_id
|
|
LIMIT ?
|
|
""",
|
|
(expression, bounded + 1),
|
|
).fetchall()
|
|
results: list[dict[str, object]] = []
|
|
for row in rows[:bounded]:
|
|
payload = _row_to_node(row).as_dict(include_content=False)
|
|
payload.update({"rank": row["rank"], "snippet": row["snippet"]})
|
|
results.append(payload)
|
|
return snapshot.result(
|
|
query=query,
|
|
count=len(results),
|
|
limit=bounded,
|
|
truncated=len(rows) > bounded,
|
|
truncation_reason="result_limit" if len(rows) > bounded else None,
|
|
results=results,
|
|
)
|
|
|
|
def filter_nodes(
|
|
self,
|
|
*,
|
|
family: str | None = None,
|
|
authority: str | None = None,
|
|
status: str | None = None,
|
|
tag: str | None = None,
|
|
limit: int | None = None,
|
|
) -> dict[str, object]:
|
|
bounded = _bounded_limit(limit, self.project.descriptor.limits.max_results, default=100)
|
|
clauses: list[str] = []
|
|
values: list[object] = []
|
|
for field, value in (("family", family), ("authority", authority), ("status", status)):
|
|
if value is not None:
|
|
clauses.append(f"{field} = ?")
|
|
values.append(value)
|
|
if tag is not None:
|
|
clauses.append("EXISTS (SELECT 1 FROM json_each(tags_json) WHERE value = ?)")
|
|
values.append(tag)
|
|
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
|
with self._read_snapshot() as snapshot:
|
|
rows = snapshot.connection.execute(
|
|
f"SELECT * FROM nodes {where} ORDER BY node_id LIMIT ?",
|
|
(*values, bounded + 1),
|
|
).fetchall()
|
|
results = [_row_to_node(row).as_dict(include_content=False) for row in rows[:bounded]]
|
|
return snapshot.result(
|
|
count=len(results),
|
|
limit=bounded,
|
|
truncated=len(rows) > bounded,
|
|
truncation_reason="result_limit" if len(rows) > bounded else None,
|
|
results=results,
|
|
)
|
|
|
|
def backlinks(
|
|
self,
|
|
node_id: str,
|
|
*,
|
|
relation: str | None = None,
|
|
limit: int | None = None,
|
|
) -> dict[str, object]:
|
|
return self._edges(node_id, incoming=True, relation=relation, limit=limit)
|
|
|
|
def dependencies(
|
|
self,
|
|
node_id: str,
|
|
*,
|
|
depth: int = 2,
|
|
limit: int | None = None,
|
|
) -> dict[str, object]:
|
|
return self._traverse(
|
|
node_id,
|
|
incoming=False,
|
|
depth=depth,
|
|
relation="depends_on",
|
|
limit=limit,
|
|
)
|
|
|
|
def impact(
|
|
self,
|
|
node_id: str,
|
|
*,
|
|
depth: int = 2,
|
|
limit: int | None = None,
|
|
) -> dict[str, object]:
|
|
return self._traverse(
|
|
node_id,
|
|
incoming=True,
|
|
depth=depth,
|
|
relation=None,
|
|
limit=limit,
|
|
)
|
|
|
|
def _edges(
|
|
self,
|
|
node_id: str,
|
|
*,
|
|
incoming: bool,
|
|
relation: str | None,
|
|
limit: int | None,
|
|
) -> dict[str, object]:
|
|
bounded = _bounded_limit(
|
|
limit,
|
|
self.project.descriptor.limits.max_results,
|
|
default=self.project.descriptor.limits.max_results,
|
|
)
|
|
source_column = "target_id" if incoming else "source_id"
|
|
relation_clause = " AND relation = ?" if relation is not None else ""
|
|
values: tuple[object, ...] = (
|
|
(node_id, relation, bounded + 1) if relation is not None else (node_id, bounded + 1)
|
|
)
|
|
with self._read_snapshot() as snapshot:
|
|
self._require_node(snapshot.connection, node_id)
|
|
rows = snapshot.connection.execute(
|
|
f"SELECT source_id, relation, target_id FROM edges "
|
|
f"WHERE {source_column} = ?{relation_clause} "
|
|
"ORDER BY source_id, relation, target_id LIMIT ?",
|
|
values,
|
|
).fetchall()
|
|
truncated = len(rows) > bounded
|
|
edges = [Edge(*row).as_dict() for row in rows[:bounded]]
|
|
return snapshot.result(
|
|
root=node_id,
|
|
relation=relation,
|
|
count=len(edges),
|
|
limit=bounded,
|
|
truncated=truncated,
|
|
truncation_reason="result_limit" if truncated else None,
|
|
edges=edges,
|
|
)
|
|
|
|
def _traverse(
|
|
self,
|
|
node_id: str,
|
|
*,
|
|
incoming: bool,
|
|
depth: int,
|
|
relation: str | None,
|
|
limit: int | None,
|
|
) -> dict[str, object]:
|
|
maximum = self.project.descriptor.limits.max_traversal_depth
|
|
if type(depth) is not int or depth < 0 or depth > maximum:
|
|
raise DocForgeError("invalid_depth", "Traversal depth is outside the configured limit")
|
|
bounded = _bounded_limit(
|
|
limit,
|
|
self.project.descriptor.limits.max_results,
|
|
default=self.project.descriptor.limits.max_results,
|
|
)
|
|
examined_limit = (bounded + 1) ** 2
|
|
source_column = "target_id" if incoming else "source_id"
|
|
relation_clause = " AND relation = ?" if relation is not None else ""
|
|
with self._read_snapshot() as snapshot:
|
|
self._require_node(snapshot.connection, node_id)
|
|
queue: deque[tuple[str, int, tuple[str, ...]]] = deque([(node_id, 0, (node_id,))])
|
|
seen = {node_id}
|
|
results: list[dict[str, object]] = []
|
|
truncation_reason: str | None = None
|
|
candidate_edges_consumed = 0
|
|
while queue and truncation_reason is None:
|
|
current, current_depth, path = queue.popleft()
|
|
if current_depth >= depth:
|
|
continue
|
|
remaining = examined_limit - candidate_edges_consumed
|
|
if remaining <= 0:
|
|
truncation_reason = "edge_examination_limit"
|
|
break
|
|
values: tuple[object, ...] = (
|
|
(current, relation, remaining + 1)
|
|
if relation is not None
|
|
else (current, remaining + 1)
|
|
)
|
|
candidates = snapshot.connection.execute(
|
|
"SELECT source_id, relation, target_id FROM edges "
|
|
f"WHERE {source_column} = ?{relation_clause} "
|
|
"ORDER BY source_id, relation, target_id LIMIT ?",
|
|
values,
|
|
)
|
|
for row in candidates:
|
|
if candidate_edges_consumed >= examined_limit:
|
|
truncation_reason = "edge_examination_limit"
|
|
break
|
|
candidate_edges_consumed += 1
|
|
edge = Edge(*row)
|
|
target = edge.source_id if incoming else edge.target_id
|
|
if target in seen:
|
|
continue
|
|
if len(results) >= bounded:
|
|
truncation_reason = "result_limit"
|
|
break
|
|
seen.add(target)
|
|
target_path = (*path, target)
|
|
results.append(
|
|
{
|
|
"node_id": target,
|
|
"depth": current_depth + 1,
|
|
"relation": edge.relation,
|
|
"path": target_path,
|
|
}
|
|
)
|
|
queue.append((target, current_depth + 1, target_path))
|
|
return snapshot.result(
|
|
root=node_id,
|
|
depth=depth,
|
|
count=len(results),
|
|
limit=bounded,
|
|
truncated=truncation_reason is not None,
|
|
truncation_reason=truncation_reason,
|
|
candidate_edges_consumed=candidate_edges_consumed,
|
|
candidate_edges_limit=examined_limit,
|
|
results=results,
|
|
)
|
|
|
|
@staticmethod
|
|
def _require_node(connection: sqlite3.Connection, node_id: str) -> None:
|
|
exists = connection.execute(
|
|
"SELECT 1 FROM nodes WHERE node_id = ?",
|
|
(node_id,),
|
|
).fetchone()
|
|
if exists is None:
|
|
raise DocForgeError(
|
|
"missing_node", "No node has the requested stable ID", node_id=node_id
|
|
)
|
|
|
|
|
|
def _row_to_node(row: sqlite3.Row) -> Node:
|
|
return Node(
|
|
node_id=row["node_id"],
|
|
title=row["title"],
|
|
family=row["family"],
|
|
authority=row["authority"],
|
|
status=row["status"],
|
|
tags=tuple(json.loads(row["tags_json"])),
|
|
summary=row["summary"],
|
|
content=row["content"],
|
|
source_path=row["source_path"],
|
|
source_anchor=row["source_anchor"],
|
|
content_hash=row["content_hash"],
|
|
)
|
|
|
|
|
|
def _logic_projection_from_connection(
|
|
connection: sqlite3.Connection,
|
|
owner_node_id: str,
|
|
) -> LogicProjection | None:
|
|
owner = connection.execute(
|
|
"SELECT owner_node_id, source_id FROM logic_owners WHERE owner_node_id = ?",
|
|
(owner_node_id,),
|
|
).fetchone()
|
|
if owner is None:
|
|
return None
|
|
nodes = tuple(
|
|
LogicNode(
|
|
logic_id=row["logic_id"],
|
|
kind=row["kind"],
|
|
label=row["label"],
|
|
source_anchor=row["source_anchor"],
|
|
)
|
|
for row in connection.execute(
|
|
"SELECT logic_id, kind, label, source_anchor FROM logic_nodes "
|
|
"WHERE owner_node_id = ? ORDER BY logic_id",
|
|
(owner_node_id,),
|
|
)
|
|
)
|
|
edges = tuple(
|
|
LogicEdge(
|
|
source_id=row["source_id"],
|
|
relation=row["relation"],
|
|
target_id=row["target_id"],
|
|
label=row["label"],
|
|
ordinal=row["ordinal"],
|
|
)
|
|
for row in connection.execute(
|
|
"SELECT source_id, relation, target_id, label, ordinal FROM logic_edges "
|
|
"WHERE owner_node_id = ? "
|
|
"ORDER BY source_id, ordinal, relation, target_id",
|
|
(owner_node_id,),
|
|
)
|
|
)
|
|
return LogicProjection(
|
|
owner_node_id=owner["owner_node_id"],
|
|
source_id=owner["source_id"],
|
|
nodes=nodes,
|
|
edges=edges,
|
|
)
|
|
|
|
|
|
def _logic_from_connection(
|
|
connection: sqlite3.Connection,
|
|
) -> tuple[LogicProjection, ...]:
|
|
owners = connection.execute(
|
|
"SELECT owner_node_id FROM logic_owners ORDER BY owner_node_id"
|
|
).fetchall()
|
|
projections = [
|
|
_logic_projection_from_connection(connection, row["owner_node_id"]) for row in owners
|
|
]
|
|
return tuple(projection for projection in projections if projection is not None)
|
|
|
|
|
|
def _bounded_limit(value: int | None, maximum: int, *, default: int) -> int:
|
|
if value is None:
|
|
return min(default, maximum)
|
|
if type(value) is not int or value < 1 or value > maximum:
|
|
raise DocForgeError("invalid_limit", "Result limit is outside the configured range")
|
|
return value
|
|
|
|
|
|
def re_tokenize(query: str) -> tuple[str, ...]:
|
|
token = ""
|
|
tokens: list[str] = []
|
|
for character in query.casefold():
|
|
if character.isalnum() or character in {"_", "-"}:
|
|
token += character
|
|
elif token:
|
|
tokens.append(token)
|
|
token = ""
|
|
if token:
|
|
tokens.append(token)
|
|
return tuple(tokens)
|