administrator/DocForge2
1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions Pages

Add bounded generation transition receipts

Browse source
This commit is contained in:
Andraxion 2026-07-29 08:23:04 -04:00
parent 4cc6277054
commit 9a48233983
21 changed files with 3822 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

71
src/docforge/_fs_safety.py Normal file
View file

@ -0,0 +1,71 @@
"""Internal directory binding helpers for disposable publication paths."""
from __future__ import annotations
import os
import stat
from pathlib import Path
from .errors import DocForgeError
def open_bound_directory(path: Path) -> int:
"""Open one real directory and bind its current inode for later operations."""
try:
path_status = path.lstat()
if (
stat.S_ISLNK(path_status.st_mode)
or not stat.S_ISDIR(path_status.st_mode)
or path.resolve(strict=True) != path
):
raise DocForgeError(
"path_escape",
"Derived cache root is not a safe real directory",
)
directory_fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
except FileNotFoundError as error:
raise DocForgeError(
"missing_index",
"Derived cache root does not exist",
) from error
except OSError as error:
raise DocForgeError(
"path_escape",
"Derived cache root cannot be opened safely",
) from error
try:
opened_status = os.fstat(directory_fd)
if opened_status.st_dev != path_status.st_dev or opened_status.st_ino != path_status.st_ino:
raise DocForgeError(
"path_escape",
"Derived cache root changed while opening",
)
except Exception:
os.close(directory_fd)
raise
return directory_fd
def require_bound_directory(path: Path, directory_fd: int) -> None:
"""Require a path to still name the exact opened real directory."""
try:
path_status = path.lstat()
opened_status = os.fstat(directory_fd)
if (
stat.S_ISLNK(path_status.st_mode)
or not stat.S_ISDIR(path_status.st_mode)
or path.resolve(strict=True) != path
or opened_status.st_dev != path_status.st_dev
or opened_status.st_ino != path_status.st_ino
):
raise DocForgeError(
"path_escape",
"Derived cache root changed during publication",
)
except FileNotFoundError as error:
raise DocForgeError(
"path_escape",
"Derived cache root disappeared during publication",
) from error

13
src/docforge/cli.py
View file

@ -65,6 +65,9 @@ def _parser() -> argparse.ArgumentParser:
context.add_argument("--budget", type=int)
context.add_argument("--limit", type=int)
context.add_argument("--cursor")
generation_diff = commands.add_parser("generation-diff")
generation_diff.add_argument("--limit", type=int)
generation_diff.add_argument("--cursor")
render = commands.add_parser("render")
render.add_argument("view_id")
render_status = commands.add_parser("render-status")
@ -187,6 +190,16 @@ def _run(arguments: argparse.Namespace) -> dict[str, object]:
cursor=arguments.cursor,
)
return compile_context(index, arguments.profile, arguments.budget)
if arguments.command == "generation-diff":
from .mcp_server import DocForgeService
return DocForgeService(
project,
capability_mode_name="read",
).generation_diff(
limit=arguments.limit,
cursor=arguments.cursor,
)
if arguments.command == "render":
return RenderService(project).render(arguments.view_id)
if arguments.command == "render-status":

911
src/docforge/generation_diff.py Normal file
View file

@ -0,0 +1,911 @@
"""Bounded latest-generation transition receipts for disposable graph indexes."""
from __future__ import annotations
import json
import os
import secrets
import stat
from collections.abc import Mapping
from contextlib import suppress
from dataclasses import dataclass
from pathlib import Path
from typing import Literal, cast
from ._fs_safety import open_bound_directory, require_bound_directory
from .errors import DocForgeError
from .models import Edge, Node, ProjectDescriptor, ProjectSnapshot
from .pagination import canonical_hash
GENERATION_DIFF_SCHEMA_VERSION = 1
GENERATION_DIFF_SEMANTICS_VERSION = 1
MAX_GENERATION_DIFF_ITEMS = 1_000
MAX_GENERATION_DIFF_BYTES = 1_048_576
GENERATION_DIFF_FILENAME = "generation-diff.json"
IndexSignature = tuple[int, int, int, int, int]
PredecessorReason = Literal[
"no_predecessor",
"predecessor_unsafe",
"predecessor_unsupported_schema",
"predecessor_foreign",
"predecessor_policy_incompatible",
"predecessor_corrupt",
"predecessor_unattested",
"predecessor_changed",
"no_meaningful_transition",
]
_GENERATION_KEYS = frozenset(
{
"revision",
"source_hash",
"node_count",
"node_hash",
"edge_count",
"edge_hash",
"index_schema_version",
}
)
_SUMMARY_KEYS = frozenset(
{
"nodes_added",
"nodes_removed",
"nodes_changed",
"edges_added",
"edges_removed",
"total_changes",
}
)
_SIGNATURE_KEYS = frozenset({"device", "inode", "size", "mtime_ns", "ctime_ns"})
_RECEIPT_KEYS = frozenset(
{
"schema_version",
"diff_semantics_version",
"project_id",
"project_root_fingerprint",
"adapter",
"kind",
"reason",
"from_generation",
"to_generation",
"summary",
"items",
"full_item_count",
"retained_item_count",
"details_truncated",
"truncation_reason",
"full_collection_hash",
"retained_collection_hash",
"index_signature",
"receipt_hash",
}
)
_BASELINE_REASONS = frozenset(
{
"no_predecessor",
"predecessor_unsafe",
"predecessor_unsupported_schema",
"predecessor_foreign",
"predecessor_policy_incompatible",
"predecessor_corrupt",
"predecessor_unattested",
"predecessor_changed",
"no_meaningful_transition",
}
)
_NODE_ITEM_KEYS = frozenset(
{
"entity",
"change",
"node_id",
"before_content_hash",
"after_content_hash",
"before_source_path",
"after_source_path",
"before_node_hash",
"after_node_hash",
"changed_fields",
"item_hash",
}
)
_NODE_CHANGED_FIELDS = frozenset(
{
"title",
"family",
"authority",
"status",
"tags",
"summary",
"content",
"source_path",
"source_anchor",
"content_hash",
}
)
_EDGE_ITEM_KEYS = frozenset(
{
"entity",
"change",
"source_id",
"relation",
"target_id",
"item_hash",
}
)
@dataclass(frozen=True)
class PublishedGraph:
"""One completely verified predecessor publication."""
generation: dict[str, object]
nodes: tuple[Node, ...]
edges: tuple[Edge, ...]
logic_hash: str
signature: IndexSignature
@dataclass(frozen=True)
class GenerationDiffDraft:
"""A receipt body awaiting the committed index file identity."""
fields: dict[str, object]
items: tuple[dict[str, object], ...]
def generation_diff_path(descriptor: ProjectDescriptor) -> Path:
"""Return the fixed project-confined latest-transition receipt path."""
return descriptor.cache_root / GENERATION_DIFF_FILENAME
def index_signature(path: Path) -> IndexSignature:
"""Return the exact identity of one safe regular index publication."""
try:
status = path.lstat()
except OSError as error:
raise DocForgeError("missing_index", "Derived index does not exist") from error
if stat.S_ISLNK(status.st_mode) or not stat.S_ISREG(status.st_mode):
raise DocForgeError("path_escape", "Derived index path is not a safe regular file")
return (
status.st_dev,
status.st_ino,
status.st_size,
status.st_mtime_ns,
status.st_ctime_ns,
)
def signature_payload(signature: IndexSignature) -> dict[str, int]:
"""Convert one stat identity to its versioned JSON representation."""
return {
"device": signature[0],
"inode": signature[1],
"size": signature[2],
"mtime_ns": signature[3],
"ctime_ns": signature[4],
}
def generation_identity(status: Mapping[str, object]) -> dict[str, object]:
"""Select the primary-graph identity stored in a public diff receipt."""
return {
"revision": status["revision"],
"source_hash": status["source_hash"],
"node_count": status["node_count"],
"node_hash": status["node_hash"],
"edge_count": status["edge_count"],
"edge_hash": status["edge_hash"],
"index_schema_version": status["index_schema_version"],
}
def prepare_generation_diff(
descriptor: ProjectDescriptor,
*,
predecessor: PublishedGraph | None,
predecessor_reason: PredecessorReason | None,
current_snapshot: ProjectSnapshot,
current_status: Mapping[str, object],
preserved_receipt: Mapping[str, object] | None = None,
) -> GenerationDiffDraft:
"""Prepare one deterministic latest transition without publishing it."""
current_generation = generation_identity(current_status)
if predecessor is None:
return _baseline_draft(
descriptor,
current_generation,
predecessor_reason or "no_predecessor",
)
if predecessor.generation == current_generation:
if preserved_receipt is not None and _receipt_targets(
preserved_receipt,
descriptor,
current_generation,
predecessor.signature,
):
fields = {
key: value
for key, value in preserved_receipt.items()
if key not in {"index_signature", "receipt_hash", "items"}
}
preserved_items = preserved_receipt.get("items")
if not isinstance(preserved_items, list):
raise DocForgeError(
"invalid_generation_diff",
"Preserved generation diff has no item collection",
)
items = tuple(
cast(dict[str, object], item)
for item in cast(list[object], preserved_items)
if isinstance(item, dict)
)
return GenerationDiffDraft(fields=fields, items=items)
return _baseline_draft(
descriptor,
current_generation,
"no_meaningful_transition",
)
items = _change_items(
predecessor.nodes,
predecessor.edges,
current_snapshot.nodes,
current_snapshot.edges,
)
summary = _summary(items)
return GenerationDiffDraft(
fields={
"schema_version": GENERATION_DIFF_SCHEMA_VERSION,
"diff_semantics_version": GENERATION_DIFF_SEMANTICS_VERSION,
"project_id": descriptor.project_id,
"project_root_fingerprint": _root_fingerprint(descriptor),
"adapter": descriptor.adapter,
"kind": "transition",
"reason": None,
"from_generation": predecessor.generation,
"to_generation": current_generation,
"summary": summary,
"full_item_count": len(items),
"full_collection_hash": canonical_hash([item["item_hash"] for item in items]),
},
items=items,
)
def finalize_generation_diff(
draft: GenerationDiffDraft,
*,
signature: IndexSignature,
) -> dict[str, object]:
"""Bind a draft to the committed index and enforce fixed receipt ceilings."""
full_count = cast(int, draft.fields["full_item_count"])
retained = list(draft.items[:MAX_GENERATION_DIFF_ITEMS])
item_limited = full_count > len(retained)
byte_limited = draft.fields.get("truncation_reason") == "receipt_byte_limit"
while True:
receipt = _final_receipt(
draft.fields,
retained,
signature=signature,
item_limited=item_limited,
byte_limited=byte_limited,
)
if len(_receipt_bytes(receipt)) <= MAX_GENERATION_DIFF_BYTES:
return receipt
if not retained:
raise DocForgeError(
"generation_diff_failure",
"Generation diff identity exceeds the fixed receipt byte limit",
)
retained.pop()
byte_limited = True
def publish_generation_diff(
descriptor: ProjectDescriptor,
receipt: Mapping[str, object],
) -> None:
"""Atomically replace the single latest-generation receipt."""
root = descriptor.cache_root
if generation_diff_path(descriptor).parent != root:
raise DocForgeError("path_escape", "Generation diff path is not confined")
raw = _receipt_bytes(receipt)
if len(raw) > MAX_GENERATION_DIFF_BYTES:
raise DocForgeError(
"generation_diff_failure",
"Generation diff receipt exceeds the fixed byte limit",
)
root_fd = open_bound_directory(root)
temporary_name = f".generation-diff-{secrets.token_hex(12)}"
temporary_created = False
try:
try:
status = os.stat(
GENERATION_DIFF_FILENAME,
dir_fd=root_fd,
follow_symlinks=False,
)
except FileNotFoundError:
status = None
if status is not None and (
stat.S_ISLNK(status.st_mode) or not stat.S_ISREG(status.st_mode)
):
raise DocForgeError(
"path_escape",
"Generation diff receipt path is not a safe regular file",
)
descriptor_fd = os.open(
temporary_name,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
0o600,
dir_fd=root_fd,
)
temporary_created = True
with os.fdopen(descriptor_fd, "wb") as handle:
handle.write(raw)
handle.flush()
os.fsync(handle.fileno())
require_bound_directory(root, root_fd)
os.replace(
temporary_name,
GENERATION_DIFF_FILENAME,
src_dir_fd=root_fd,
dst_dir_fd=root_fd,
)
temporary_created = False
os.fsync(root_fd)
require_bound_directory(root, root_fd)
except Exception:
if temporary_created:
with suppress(OSError):
os.unlink(temporary_name, dir_fd=root_fd)
raise
finally:
os.close(root_fd)
def load_generation_diff(
descriptor: ProjectDescriptor,
) -> tuple[dict[str, object] | None, str | None, IndexSignature | None]:
"""Read and validate one receipt without repairing any derived state."""
try:
root_fd = open_bound_directory(descriptor.cache_root)
except DocForgeError as error:
reason = "missing_receipt" if error.code == "missing_index" else "unsafe_receipt"
return None, reason, None
try:
try:
descriptor_fd = os.open(
GENERATION_DIFF_FILENAME,
os.O_RDONLY | os.O_NOFOLLOW,
dir_fd=root_fd,
)
except FileNotFoundError:
return None, "missing_receipt", None
except OSError:
return None, "unsafe_receipt", None
with os.fdopen(descriptor_fd, "rb") as handle:
status_before = os.fstat(handle.fileno())
if stat.S_ISLNK(status_before.st_mode) or not stat.S_ISREG(status_before.st_mode):
return None, "unsafe_receipt", None
before: IndexSignature = (
status_before.st_dev,
status_before.st_ino,
status_before.st_size,
status_before.st_mtime_ns,
status_before.st_ctime_ns,
)
if before[2] > MAX_GENERATION_DIFF_BYTES:
return None, "oversized_receipt", before
raw = handle.read(MAX_GENERATION_DIFF_BYTES + 1)
status_after = os.fstat(handle.fileno())
after: IndexSignature = (
status_after.st_dev,
status_after.st_ino,
status_after.st_size,
status_after.st_mtime_ns,
status_after.st_ctime_ns,
)
if len(raw) > MAX_GENERATION_DIFF_BYTES:
return None, "oversized_receipt", before
try:
path_status = os.stat(
GENERATION_DIFF_FILENAME,
dir_fd=root_fd,
follow_symlinks=False,
)
except OSError:
return None, "receipt_changed", before
path_signature: IndexSignature = (
path_status.st_dev,
path_status.st_ino,
path_status.st_size,
path_status.st_mtime_ns,
path_status.st_ctime_ns,
)
try:
require_bound_directory(descriptor.cache_root, root_fd)
except DocForgeError:
return None, "unsafe_receipt", before
finally:
os.close(root_fd)
if before != after or before != path_signature or len(raw) != before[2]:
return None, "receipt_changed", before
try:
parsed: object = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
return None, "corrupt_receipt", before
if not isinstance(parsed, dict):
return None, "corrupt_receipt", before
receipt = cast(dict[str, object], parsed)
if not validate_generation_diff_receipt(receipt):
return None, "corrupt_receipt", before
if (
receipt.get("project_id") != descriptor.project_id
or receipt.get("project_root_fingerprint") != _root_fingerprint(descriptor)
or receipt.get("adapter") != descriptor.adapter
):
return None, "foreign_receipt", before
return receipt, None, before
def valid_generation_identity(value: object) -> bool:
"""Return whether a primary-graph generation has the exact version-1 shape."""
return _valid_generation(value)
def validate_generation_diff_receipt(
receipt: Mapping[str, object],
*,
descriptor: ProjectDescriptor | None = None,
) -> bool:
"""Strictly validate the complete version-1 receipt and its hashes."""
if frozenset(receipt) != _RECEIPT_KEYS:
return False
if (
receipt.get("schema_version") != GENERATION_DIFF_SCHEMA_VERSION
or receipt.get("diff_semantics_version") != GENERATION_DIFF_SEMANTICS_VERSION
or not _nonempty(receipt.get("project_id"))
or not _fingerprint(receipt.get("project_root_fingerprint"))
or not _nonempty(receipt.get("adapter"))
):
return False
if descriptor is not None and (
receipt["project_id"] != descriptor.project_id
or receipt["project_root_fingerprint"] != _root_fingerprint(descriptor)
or receipt["adapter"] != descriptor.adapter
):
return False
kind = receipt.get("kind")
reason = receipt.get("reason")
from_generation = receipt.get("from_generation")
if kind == "baseline":
if reason not in _BASELINE_REASONS or from_generation is not None:
return False
elif kind == "transition":
if (
reason is not None
or not _valid_generation(from_generation)
or from_generation == receipt.get("to_generation")
):
return False
else:
return False
if not _valid_generation(receipt.get("to_generation")):
return False
summary_value = receipt.get("summary")
items_value = receipt.get("items")
if not isinstance(summary_value, dict) or not isinstance(items_value, list):
return False
summary = cast(dict[str, object], summary_value)
items = cast(list[object], items_value)
if frozenset(summary) != _SUMMARY_KEYS or any(
type(value) is not int or value < 0 for value in summary.values()
):
return False
total = sum(
cast(int, summary[key])
for key in (
"nodes_added",
"nodes_removed",
"nodes_changed",
"edges_added",
"edges_removed",
)
)
if summary.get("total_changes") != total:
return False
if kind == "baseline" and (total != 0 or items):
return False
if len(items) > MAX_GENERATION_DIFF_ITEMS or not all(_valid_item(item) for item in items):
return False
typed_items = tuple(cast(dict[str, object], item) for item in items)
if list(typed_items) != sorted(typed_items, key=_item_sort_key):
return False
identities = tuple(_item_identity(item) for item in typed_items)
if len(identities) != len(set(identities)):
return False
retained_summary = _summary(typed_items)
if any(
retained_summary[key] > cast(int, summary[key])
for key in (
"nodes_added",
"nodes_removed",
"nodes_changed",
"edges_added",
"edges_removed",
)
):
return False
full_count = receipt.get("full_item_count")
retained_count = receipt.get("retained_item_count")
truncated = receipt.get("details_truncated")
truncation_reason = receipt.get("truncation_reason")
if (
type(full_count) is not int
or type(retained_count) is not int
or full_count != total
or retained_count != len(items)
or retained_count > full_count
or type(truncated) is not bool
or truncated != (retained_count < full_count)
):
return False
if truncation_reason is None:
if truncated or full_count > MAX_GENERATION_DIFF_ITEMS:
return False
elif truncation_reason == "receipt_item_limit":
if (
not truncated
or full_count <= MAX_GENERATION_DIFF_ITEMS
or retained_count != MAX_GENERATION_DIFF_ITEMS
):
return False
elif truncation_reason == "receipt_byte_limit":
if not truncated or retained_count >= min(full_count, MAX_GENERATION_DIFF_ITEMS):
return False
else:
return False
item_hashes = [cast(dict[str, object], item)["item_hash"] for item in items]
retained_hash = canonical_hash(item_hashes)
if receipt.get("retained_collection_hash") != retained_hash:
return False
full_hash = receipt.get("full_collection_hash")
if not _sha256(full_hash):
return False
if not truncated and full_hash != retained_hash:
return False
signature_value = receipt.get("index_signature")
if not isinstance(signature_value, dict):
return False
signature = cast(dict[str, object], signature_value)
if frozenset(signature) != _SIGNATURE_KEYS:
return False
if any(type(value) is not int or value < 0 for value in signature.values()):
return False
receipt_hash = receipt.get("receipt_hash")
if not _sha256(receipt_hash):
return False
unhashed = {key: value for key, value in receipt.items() if key != "receipt_hash"}
return receipt_hash == canonical_hash(unhashed)
def _baseline_draft(
descriptor: ProjectDescriptor,
current_generation: Mapping[str, object],
reason: PredecessorReason,
) -> GenerationDiffDraft:
empty_hash = canonical_hash([])
return GenerationDiffDraft(
fields={
"schema_version": GENERATION_DIFF_SCHEMA_VERSION,
"diff_semantics_version": GENERATION_DIFF_SEMANTICS_VERSION,
"project_id": descriptor.project_id,
"project_root_fingerprint": _root_fingerprint(descriptor),
"adapter": descriptor.adapter,
"kind": "baseline",
"reason": reason,
"from_generation": None,
"to_generation": dict(current_generation),
"summary": {
"nodes_added": 0,
"nodes_removed": 0,
"nodes_changed": 0,
"edges_added": 0,
"edges_removed": 0,
"total_changes": 0,
},
"full_item_count": 0,
"full_collection_hash": empty_hash,
},
items=(),
)
def _change_items(
before_nodes: tuple[Node, ...],
before_edges: tuple[Edge, ...],
after_nodes: tuple[Node, ...],
after_edges: tuple[Edge, ...],
) -> tuple[dict[str, object], ...]:
before_by_id = {node.node_id: node for node in before_nodes}
after_by_id = {node.node_id: node for node in after_nodes}
items: list[dict[str, object]] = []
for node_id in sorted(before_by_id.keys() | after_by_id.keys()):
before = before_by_id.get(node_id)
after = after_by_id.get(node_id)
if before == after:
continue
if before is None:
change = "added"
elif after is None:
change = "removed"
else:
change = "changed"
before_payload = before.as_dict() if before is not None else None
after_payload = after.as_dict() if after is not None else None
changed_fields = (
[]
if before_payload is None or after_payload is None
else sorted(key for key in before_payload if before_payload[key] != after_payload[key])
)
payload: dict[str, object] = {
"entity": "node",
"change": change,
"node_id": node_id,
"before_content_hash": None if before is None else before.content_hash,
"after_content_hash": None if after is None else after.content_hash,
"before_source_path": None if before is None else before.source_path,
"after_source_path": None if after is None else after.source_path,
"before_node_hash": (
None if before_payload is None else canonical_hash(before_payload)
),
"after_node_hash": (None if after_payload is None else canonical_hash(after_payload)),
"changed_fields": changed_fields,
}
payload["item_hash"] = canonical_hash(payload)
items.append(payload)
before_edge_set = {(edge.source_id, edge.relation, edge.target_id) for edge in before_edges}
after_edge_set = {(edge.source_id, edge.relation, edge.target_id) for edge in after_edges}
for change, values in (
("removed", sorted(before_edge_set - after_edge_set)),
("added", sorted(after_edge_set - before_edge_set)),
):
for source_id, relation, target_id in values:
payload = {
"entity": "edge",
"change": change,
"source_id": source_id,
"relation": relation,
"target_id": target_id,
}
payload["item_hash"] = canonical_hash(payload)
items.append(payload)
return tuple(sorted(items, key=_item_sort_key))
def _item_sort_key(item: Mapping[str, object]) -> tuple[str, str, str, str, str]:
return (
cast(str, item["entity"]),
cast(str, item.get("node_id", item.get("source_id", ""))),
cast(str, item.get("relation", "")),
cast(str, item.get("target_id", "")),
cast(str, item["change"]),
)
def _item_identity(item: Mapping[str, object]) -> tuple[str, ...]:
if item["entity"] == "node":
return ("node", cast(str, item["node_id"]))
return (
"edge",
cast(str, item["source_id"]),
cast(str, item["relation"]),
cast(str, item["target_id"]),
)
def _summary(items: tuple[dict[str, object], ...]) -> dict[str, int]:
result = {
"nodes_added": 0,
"nodes_removed": 0,
"nodes_changed": 0,
"edges_added": 0,
"edges_removed": 0,
"total_changes": len(items),
}
for item in items:
entity = cast(str, item["entity"])
change = cast(str, item["change"])
key = f"{entity}s_{change}"
result[key] += 1
return result
def _final_receipt(
fields: Mapping[str, object],
retained: list[dict[str, object]],
*,
signature: IndexSignature,
item_limited: bool,
byte_limited: bool,
) -> dict[str, object]:
full_count = cast(int, fields["full_item_count"])
truncated = len(retained) < full_count
if byte_limited:
reason: str | None = "receipt_byte_limit"
elif item_limited:
reason = "receipt_item_limit"
else:
reason = None
receipt = {
**fields,
"items": retained,
"retained_item_count": len(retained),
"details_truncated": truncated,
"truncation_reason": reason,
"retained_collection_hash": canonical_hash([item["item_hash"] for item in retained]),
"index_signature": signature_payload(signature),
}
receipt["receipt_hash"] = canonical_hash(receipt)
return receipt
def _receipt_targets(
receipt: Mapping[str, object],
descriptor: ProjectDescriptor,
generation: Mapping[str, object],
signature: IndexSignature,
) -> bool:
return (
validate_generation_diff_receipt(receipt, descriptor=descriptor)
and receipt.get("to_generation") == dict(generation)
and receipt.get("index_signature") == signature_payload(signature)
)
def _valid_generation(value: object) -> bool:
if not isinstance(value, dict):
return False
generation = cast(dict[str, object], value)
if frozenset(generation) != _GENERATION_KEYS:
return False
source_hash = generation.get("source_hash")
node_hash = generation.get("node_hash")
edge_hash = generation.get("edge_hash")
return (
_nonempty(generation.get("revision"))
and _sha256(source_hash)
and _sha256(node_hash)
and _sha256(edge_hash)
and type(generation.get("node_count")) is int
and cast(int, generation["node_count"]) >= 0
and type(generation.get("edge_count")) is int
and cast(int, generation["edge_count"]) >= 0
and type(generation.get("index_schema_version")) is int
and cast(int, generation["index_schema_version"]) >= 1
)
def _valid_item(value: object) -> bool:
if not isinstance(value, dict):
return False
payload = cast(dict[str, object], value)
entity = payload.get("entity")
if entity == "node":
if frozenset(payload) != _NODE_ITEM_KEYS:
return False
changed_fields_value = payload.get("changed_fields")
if not isinstance(changed_fields_value, list):
return False
changed_fields = cast(list[object], changed_fields_value)
if (
payload.get("change") not in {"added", "removed", "changed"}
or not _nonempty(payload.get("node_id"))
or any(
not _nonempty(field) or field not in _NODE_CHANGED_FIELDS
for field in changed_fields
)
or changed_fields != sorted(set(cast(list[str], changed_fields)))
):
return False
for key in (
"before_content_hash",
"after_content_hash",
"before_node_hash",
"after_node_hash",
):
candidate = payload.get(key)
if candidate is not None and not _sha256(candidate):
return False
for key in ("before_source_path", "after_source_path"):
candidate = payload.get(key)
if candidate is not None and not _nonempty(candidate):
return False
change = payload["change"]
before_values = (
payload["before_content_hash"],
payload["before_source_path"],
payload["before_node_hash"],
)
after_values = (
payload["after_content_hash"],
payload["after_source_path"],
payload["after_node_hash"],
)
if change == "added":
if (
any(value is not None for value in before_values)
or not all(value is not None for value in after_values)
or changed_fields
):
return False
elif change == "removed":
if (
not all(value is not None for value in before_values)
or any(value is not None for value in after_values)
or changed_fields
):
return False
elif (
not all(value is not None for value in (*before_values, *after_values))
or not changed_fields
or payload["before_node_hash"] == payload["after_node_hash"]
):
return False
elif entity == "edge":
if (
frozenset(payload) != _EDGE_ITEM_KEYS
or payload.get("change") not in {"added", "removed"}
or not _nonempty(payload.get("source_id"))
or not _nonempty(payload.get("relation"))
or not _nonempty(payload.get("target_id"))
):
return False
else:
return False
item_hash = payload.get("item_hash")
unhashed = {key: item for key, item in payload.items() if key != "item_hash"}
return _sha256(item_hash) and item_hash == canonical_hash(unhashed)
def _receipt_bytes(receipt: Mapping[str, object]) -> bytes:
return json.dumps(receipt, sort_keys=True, indent=2, ensure_ascii=True).encode("utf-8") + b"\n"
def _root_fingerprint(descriptor: ProjectDescriptor) -> str:
from .project import project_root_fingerprint
return project_root_fingerprint(descriptor.root)
def _nonempty(value: object) -> bool:
return isinstance(value, str) and bool(value)
def _sha256(value: object) -> bool:
return (
isinstance(value, str)
and len(value) == 64
and all(character in "0123456789abcdef" for character in value)
)
def _fingerprint(value: object) -> bool:
return (
isinstance(value, str)
and len(value) == 16
and all(character in "0123456789abcdef" for character in value)
)

717
src/docforge/index.py
View file

@ -6,17 +6,33 @@ import fcntl
import hashlib
import json
import os
import secrets
import sqlite3
import tempfile
import stat
import time
from collections import deque
from collections.abc import Callable, Generator
from contextlib import contextmanager
from contextlib import contextmanager, suppress
from dataclasses import dataclass
from pathlib import Path
from typing import Literal, cast
from ._fs_safety import open_bound_directory, require_bound_directory
from .errors import DocForgeError
from .generation_diff import (
GenerationDiffDraft,
PredecessorReason,
PublishedGraph,
finalize_generation_diff,
generation_diff_path,
generation_identity,
index_signature,
load_generation_diff,
prepare_generation_diff,
publish_generation_diff,
valid_generation_identity,
validate_generation_diff_receipt,
)
from .models import (
BuildReportingProject,
Edge,
@ -76,19 +92,27 @@ def _logic_hash(projections: tuple[LogicProjection, ...]) -> str:
return hashlib.sha256(payload).hexdigest()
def _connect_read_only(path: Path) -> sqlite3.Connection:
def _connect_read_only(path: Path, *, immutable: bool = False) -> sqlite3.Connection:
if not path.is_file():
raise DocForgeError("missing_index", "Derived index does not exist; run build first")
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
immutable_parameter = "&immutable=1" if immutable else ""
connection = sqlite3.connect(
f"file:{path}?mode=ro{immutable_parameter}",
uri=True,
)
connection.row_factory = sqlite3.Row
return connection
@contextmanager
def _read_connection(path: Path) -> Generator[sqlite3.Connection, None, None]:
def _read_connection(
path: Path,
*,
immutable: bool = False,
) -> Generator[sqlite3.Connection, None, None]:
connection: sqlite3.Connection | None = None
try:
connection = _connect_read_only(path)
connection = _connect_read_only(path, immutable=immutable)
yield connection
except DocForgeError:
raise
@ -259,9 +283,24 @@ class ProjectIndex:
return self._build_locked_core()
def _build_locked_core(self) -> dict[str, object]:
predecessor, predecessor_reason, predecessor_signature = self._capture_predecessor()
if predecessor_reason == "predecessor_unsafe":
raise DocForgeError(
"path_escape",
"Derived predecessor is not a safe regular file",
)
preserved_receipt, _, _ = load_generation_diff(self.project.descriptor)
snapshot = self.project.load()
logic = self._logic_projections()
status = _status(snapshot, logic)
draft = prepare_generation_diff(
snapshot.descriptor,
predecessor=predecessor,
predecessor_reason=predecessor_reason,
current_snapshot=snapshot,
current_status=status,
preserved_receipt=preserved_receipt,
)
build_report = (
self.project.build_report() if isinstance(self.project, BuildReportingProject) else None
)
@ -269,10 +308,18 @@ class ProjectIndex:
build_report = None
cache_root = snapshot.descriptor.cache_root
cache_root.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(
prefix="index-", suffix=".sqlite3", dir=cache_root, delete=False
) as descriptor:
temporary = Path(descriptor.name)
if self.path.parent != cache_root:
raise DocForgeError("path_escape", "Derived index path is not confined")
cache_root_fd = open_bound_directory(cache_root)
temporary_name = f"index-{secrets.token_hex(12)}.sqlite3"
temporary_descriptor = os.open(
temporary_name,
os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
0o600,
dir_fd=cache_root_fd,
)
os.close(temporary_descriptor)
temporary = Path(f"/proc/self/fd/{cache_root_fd}/{temporary_name}")
try:
connection = sqlite3.connect(temporary)
try:
@ -420,25 +467,367 @@ class ProjectIndex:
if (
current.source_hash != snapshot.source_hash
or current.revision != snapshot.revision
or current.nodes != snapshot.nodes
or current.edges != snapshot.edges
or current_logic != logic
):
raise DocForgeError("source_changed", "Canonical source changed during index build")
os.replace(temporary, self.path)
self._verified_index_signature = self._index_signature()
self._write_attestation()
if isinstance(self.project, GenerationRecordingProject):
self.project.record_generation(current)
if (
predecessor is not None
and predecessor.generation["source_hash"] == status["source_hash"]
and predecessor.generation["revision"] == status["revision"]
and (
predecessor.generation != generation_identity(status)
or predecessor.logic_hash != status["logic_hash"]
)
):
raise DocForgeError(
"generation_collision",
"One canonical generation produced different graph content",
)
if predecessor is None:
if predecessor_signature is None:
if self.path.exists() or self.path.is_symlink():
raise DocForgeError(
"source_changed",
"Derived index appeared during index build",
)
elif index_signature(self.path) != predecessor_signature:
raise DocForgeError(
"source_changed",
"Derived predecessor changed during index build",
)
elif index_signature(self.path) != predecessor.signature:
raise DocForgeError(
"source_changed",
"Derived predecessor changed during index build",
)
self._require_no_index_sidecars()
require_bound_directory(cache_root, cache_root_fd)
try:
os.replace(
temporary_name,
self.path.name,
src_dir_fd=cache_root_fd,
dst_dir_fd=cache_root_fd,
)
except OSError as error:
raise DocForgeError(
"index_failure",
"Could not publish the derived index",
) from error
publication_errors: list[dict[str, object]] = []
try:
self._fsync_cache_directory(cache_root_fd)
except Exception as error:
publication_errors.append(
{
"stage": "index_directory_sync",
"error_type": type(error).__name__,
}
)
try:
published_signature = self._published_index_signature(cache_root_fd)
except Exception as error:
publication_errors.append(
{
"stage": "index_identity",
"error_type": type(error).__name__,
}
)
self._verified_index_signature = None
publication = self._unavailable_post_commit_publication(publication_errors)
else:
self._verified_index_signature = published_signature
try:
publication = self._publish_post_commit_receipts(
current,
draft,
signature=published_signature,
initial_errors=publication_errors,
)
except Exception as error:
publication_errors.append(
{
"stage": "receipt_pipeline",
"error_type": type(error).__name__,
}
)
publication = self._unavailable_post_commit_publication(publication_errors)
except sqlite3.Error as error:
temporary.unlink(missing_ok=True)
with suppress(OSError):
os.unlink(temporary_name, dir_fd=cache_root_fd)
raise DocForgeError("index_failure", "Could not build the derived index") from error
except Exception:
temporary.unlink(missing_ok=True)
with suppress(OSError):
os.unlink(temporary_name, dir_fd=cache_root_fd)
raise
finally:
os.close(cache_root_fd)
result: dict[str, object] = {**status, "database": str(self.path)}
if build_report is not None:
result["build"] = build_report
if publication["state"] == "degraded":
result["publication"] = publication
return result
def _capture_predecessor(
self,
) -> tuple[PublishedGraph | None, PredecessorReason, tuple[int, int, int, int, int] | None]:
"""Read one predecessor completely without repairing any derived receipt."""
if not self.path.exists() and not self.path.is_symlink():
return None, "no_predecessor", None
self._require_no_index_sidecars()
try:
signature = index_signature(self.path)
except DocForgeError:
return None, "predecessor_unsafe", None
if not self._attestation_matches(expected_signature=signature):
return None, "predecessor_unattested", signature
try:
with _read_connection(self.path, immutable=True) as connection:
application_id = connection.execute("PRAGMA application_id").fetchone()[0]
schema_version = connection.execute("PRAGMA user_version").fetchone()[0]
if application_id != APPLICATION_ID or schema_version != INDEX_SCHEMA_VERSION:
return None, "predecessor_unsupported_schema", signature
metadata = dict(connection.execute("SELECT key, value FROM metadata"))
descriptor = self.project.descriptor
identity = {
"project_id": descriptor.project_id,
"project_root_fingerprint": project_root_fingerprint(descriptor.root),
"adapter": descriptor.adapter,
"index_schema_version": str(INDEX_SCHEMA_VERSION),
}
if (
any(metadata.get(key) != value for key, value in identity.items())
or metadata.get("status") != "ok"
):
return None, "predecessor_foreign", signature
integrity = connection.execute("PRAGMA integrity_check").fetchone()
if integrity is None or integrity[0] != "ok":
return None, "predecessor_corrupt", signature
nodes = tuple(
_row_to_node(row)
for row in connection.execute("SELECT * FROM nodes ORDER BY node_id")
)
edges = tuple(
Edge(*row)
for row in connection.execute(
"SELECT source_id, relation, target_id FROM edges "
"ORDER BY source_id, relation, target_id"
)
)
logic = _logic_from_connection(connection)
fts_count = connection.execute("SELECT COUNT(*) FROM node_fts").fetchone()[0]
node_hash = _node_hash(nodes)
edge_hash = _edge_hash(edges)
logic_hash = _logic_hash(logic)
logic_node_count = sum(len(projection.nodes) for projection in logic)
logic_edge_count = sum(len(projection.edges) for projection in logic)
if (
metadata.get("node_hash") != node_hash
or metadata.get("node_count") != str(len(nodes))
or metadata.get("edge_hash") != edge_hash
or metadata.get("edge_count") != str(len(edges))
or metadata.get("logic_hash") != logic_hash
or metadata.get("logic_projection_count") != str(len(logic))
or metadata.get("logic_node_count") != str(logic_node_count)
or metadata.get("logic_edge_count") != str(logic_edge_count)
or fts_count != len(nodes)
):
return None, "predecessor_corrupt", signature
if logic and not self.allow_logic:
return None, "predecessor_policy_incompatible", signature
generation: dict[str, object] = {
"revision": metadata["revision"],
"source_hash": metadata["source_hash"],
"node_count": len(nodes),
"node_hash": node_hash,
"edge_count": len(edges),
"edge_hash": edge_hash,
"index_schema_version": INDEX_SCHEMA_VERSION,
}
if not valid_generation_identity(generation):
return None, "predecessor_corrupt", signature
except (
DocForgeError,
sqlite3.Error,
json.JSONDecodeError,
KeyError,
TypeError,
ValueError,
):
return None, "predecessor_corrupt", signature
self._require_no_index_sidecars()
try:
if index_signature(self.path) != signature:
return None, "predecessor_changed", signature
except DocForgeError:
return None, "predecessor_changed", signature
return (
PublishedGraph(
generation=generation,
nodes=nodes,
edges=edges,
logic_hash=logic_hash,
signature=signature,
),
"no_predecessor",
signature,
)
def _publish_post_commit_receipts(
self,
snapshot: ProjectSnapshot,
draft: GenerationDiffDraft,
*,
signature: tuple[int, int, int, int, int],
initial_errors: list[dict[str, object]] | None = None,
) -> dict[str, object]:
"""Publish independent evidence without misreporting a committed index."""
receipts: dict[str, dict[str, object]] = {}
errors = list(initial_errors or ())
def attempt(name: str, operation: Callable[[], None]) -> None:
try:
if self._index_signature() != signature:
raise DocForgeError(
"invalid_index",
"Committed index changed before receipt publication",
)
operation()
if self._index_signature() != signature:
raise DocForgeError(
"invalid_index",
"Committed index changed during receipt publication",
)
except Exception as error:
receipts[name] = {"state": "unavailable", "reason": "publication_failed"}
errors.append(
{
"stage": name,
"error_type": type(error).__name__,
}
)
else:
receipts[name] = {"state": "published"}
attempt("attestation", lambda: self._write_attestation(expected_signature=signature))
if isinstance(self.project, GenerationRecordingProject):
record_generation = self.project.record_generation
attempt("source_generation", lambda: record_generation(snapshot))
else:
receipts["source_generation"] = {
"state": "unavailable",
"reason": "project_does_not_record_generation",
}
finalized: dict[str, object] | None = None
def publish_diff() -> None:
nonlocal finalized
finalized = finalize_generation_diff(draft, signature=signature)
if not validate_generation_diff_receipt(
finalized,
descriptor=self.project.descriptor,
):
raise DocForgeError(
"invalid_generation_diff",
"Finalized generation diff failed its publication contract",
)
publish_generation_diff(self.project.descriptor, finalized)
attempt("generation_diff", publish_diff)
if finalized is not None and receipts["generation_diff"]["state"] == "published":
receipts["generation_diff"].update(
{
"kind": finalized["kind"],
"receipt_hash": finalized["receipt_hash"],
"full_item_count": finalized["full_item_count"],
"retained_item_count": finalized["retained_item_count"],
"details_truncated": finalized["details_truncated"],
}
)
return {
"state": "degraded" if errors else "published",
"index": "published",
"receipts": receipts,
"errors": errors,
}
@staticmethod
def _unavailable_post_commit_publication(
errors: list[dict[str, object]],
) -> dict[str, object]:
return {
"state": "degraded",
"index": "published",
"receipts": {
name: {
"state": "unavailable",
"reason": "index_identity_unavailable",
}
for name in ("attestation", "source_generation", "generation_diff")
},
"errors": errors,
}
def _require_no_index_sidecars(self) -> None:
"""Refuse SQLite state that is not contained in the main index inode."""
for suffix in ("-wal", "-journal", "-shm"):
sidecar = Path(f"{self.path}{suffix}")
try:
sidecar.lstat()
except FileNotFoundError:
continue
raise DocForgeError(
"index_busy",
"Derived index has active or unproven SQLite sidecar state",
sidecar=suffix,
)
def _published_index_signature(
self,
cache_root_fd: int,
) -> tuple[int, int, int, int, int]:
require_bound_directory(self.project.descriptor.cache_root, cache_root_fd)
try:
status = os.stat(
self.path.name,
dir_fd=cache_root_fd,
follow_symlinks=False,
)
except OSError as error:
raise DocForgeError(
"missing_index",
"Committed index identity is unavailable",
) from error
if not stat.S_ISREG(status.st_mode) or stat.S_ISLNK(status.st_mode):
raise DocForgeError(
"path_escape",
"Committed index is not a safe regular file",
)
signature = (
status.st_dev,
status.st_ino,
status.st_size,
status.st_mtime_ns,
status.st_ctime_ns,
)
if index_signature(self.path) != signature:
raise DocForgeError(
"invalid_index",
"Committed index path is not bound to its publication inode",
)
return signature
@staticmethod
def _fsync_cache_directory(cache_root_fd: int) -> None:
os.fsync(cache_root_fd)
@contextmanager
def _build_lock(self) -> Generator[None, None, None]:
cache_root = self.project.descriptor.cache_root
@ -464,6 +853,7 @@ class ProjectIndex:
candidates = (
*self.project.descriptor.cache_root.glob("index-*.sqlite3"),
*self.project.descriptor.cache_root.glob(".index-attestation-*"),
*self.project.descriptor.cache_root.glob(".generation-diff-*"),
)
for path in sorted(candidates):
if path == self.path or path.is_symlink() or not path.is_file():
@ -740,15 +1130,49 @@ class ProjectIndex:
"database": str(self.path),
}
def _attestation_matches(self) -> bool:
def _attestation_matches(
self,
*,
expected_signature: tuple[int, int, int, int, int] | None = None,
) -> bool:
"""Verify a persisted whole-file digest before trusting a warm derived index."""
path = self.attestation_path
if not path.is_file() or path.is_symlink():
try:
before_path = index_signature(path)
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(descriptor, "rb") as handle:
before = os.fstat(handle.fileno())
raw = handle.read(4_097)
after = os.fstat(handle.fileno())
after_path = index_signature(path)
except (DocForgeError, OSError):
return False
before_signature = (
before.st_dev,
before.st_ino,
before.st_size,
before.st_mtime_ns,
before.st_ctime_ns,
)
after_signature = (
after.st_dev,
after.st_ino,
after.st_size,
after.st_mtime_ns,
after.st_ctime_ns,
)
if (
before_path != before_signature
or after_path != before_signature
or after_signature != before_signature
or len(raw) > 4_096
or len(raw) != before.st_size
):
return False
try:
parsed: object = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
parsed: object = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
return False
if not isinstance(parsed, dict):
return False
@ -756,72 +1180,133 @@ class ProjectIndex:
expected_size = payload.get("index_size")
expected_hash = payload.get("index_sha256")
if (
payload.get("schema_version") != 1
set(payload) != {"schema_version", "index_size", "index_sha256"}
or payload.get("schema_version") != 1
or type(expected_size) is not int
or not isinstance(expected_hash, str)
or len(expected_hash) != 64
or any(character not in "0123456789abcdef" for character in expected_hash)
):
return False
try:
if self.path.stat().st_size != expected_size:
return False
with self.path.open("rb") as handle:
actual_hash = hashlib.file_digest(handle, "sha256").hexdigest()
except OSError:
size, actual_hash, _ = self._index_digest(expected_signature=expected_signature)
except DocForgeError:
return False
return actual_hash == expected_hash
return size == expected_size and actual_hash == expected_hash
def _write_attestation(self) -> None:
def _index_digest(
self,
*,
expected_signature: tuple[int, int, int, int, int] | None = None,
) -> tuple[int, str, tuple[int, int, int, int, int]]:
"""Hash one exact safe index inode and prove it stayed path-bound."""
try:
descriptor = os.open(self.path, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(descriptor, "rb") as handle:
before = os.fstat(handle.fileno())
signature = (
before.st_dev,
before.st_ino,
before.st_size,
before.st_mtime_ns,
before.st_ctime_ns,
)
if expected_signature is not None and signature != expected_signature:
raise DocForgeError(
"invalid_index",
"Derived index changed before hashing",
)
index_hash = hashlib.file_digest(handle, "sha256").hexdigest()
after = os.fstat(handle.fileno())
after_signature = (
after.st_dev,
after.st_ino,
after.st_size,
after.st_mtime_ns,
after.st_ctime_ns,
)
if after_signature != signature or index_signature(self.path) != signature:
raise DocForgeError(
"invalid_index",
"Derived index changed during hashing",
)
except OSError as error:
raise DocForgeError("missing_index", "Derived index cannot be hashed") from error
return before.st_size, index_hash, signature
def _write_attestation(
self,
*,
expected_signature: tuple[int, int, int, int, int] | None = None,
) -> None:
"""Atomically persist the digest of an index that passed complete verification."""
root = self.project.descriptor.cache_root
path = self.attestation_path
if path.parent != root or path.is_symlink():
raise DocForgeError("path_escape", "Index attestation path is not safe")
try:
size = self.path.stat().st_size
with self.path.open("rb") as handle:
index_hash = hashlib.file_digest(handle, "sha256").hexdigest()
except OSError as error:
raise DocForgeError("missing_index", "Derived index cannot be attested") from error
size, index_hash, before = self._index_digest(
expected_signature=expected_signature,
)
payload = {
"schema_version": 1,
"index_size": size,
"index_sha256": index_hash,
}
raw = json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n"
descriptor, temporary_name = tempfile.mkstemp(prefix=".index-attestation-", dir=root)
temporary = Path(temporary_name)
root_fd = open_bound_directory(root)
temporary_name = f".index-attestation-{secrets.token_hex(12)}"
temporary_created = False
try:
try:
existing = os.stat(
path.name,
dir_fd=root_fd,
follow_symlinks=False,
)
except FileNotFoundError:
existing = None
if existing is not None and (
stat.S_ISLNK(existing.st_mode) or not stat.S_ISREG(existing.st_mode)
):
raise DocForgeError("path_escape", "Index attestation path is not safe")
descriptor = os.open(
temporary_name,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
0o600,
dir_fd=root_fd,
)
temporary_created = True
with os.fdopen(descriptor, "wb") as handle:
handle.write(raw)
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
directory_descriptor = os.open(root, os.O_RDONLY)
try:
os.fsync(directory_descriptor)
finally:
os.close(directory_descriptor)
require_bound_directory(root, root_fd)
if self._index_signature() != before:
raise DocForgeError(
"invalid_index",
"Derived index changed before attestation publication",
)
os.replace(
temporary_name,
path.name,
src_dir_fd=root_fd,
dst_dir_fd=root_fd,
)
temporary_created = False
os.fsync(root_fd)
require_bound_directory(root, root_fd)
except Exception:
temporary.unlink(missing_ok=True)
if temporary_created:
with suppress(OSError):
os.unlink(temporary_name, dir_fd=root_fd)
raise
finally:
os.close(root_fd)
def _index_signature(self) -> tuple[int, int, int, int, int]:
try:
status = self.path.stat()
except OSError as error:
raise DocForgeError(
"missing_index",
"Derived index does not exist; run build first",
) from error
return (
status.st_dev,
status.st_ino,
status.st_size,
status.st_mtime_ns,
status.st_ctime_ns,
)
return index_signature(self.path)
def task_context(self, plan: RetrievalPlanV1) -> dict[str, object]:
"""Execute one fixed task plan inside one immutable index generation."""
@ -831,6 +1316,122 @@ class ProjectIndex:
capsule = self._task_context_capsule(snapshot, plan)
return snapshot.result(capsule=capsule.as_dict())
def generation_diff(self) -> dict[str, object]:
"""Return the latest bounded transition without loading or repairing source."""
descriptor = self.project.descriptor
identity = {
"status": "ok",
"project_id": descriptor.project_id,
"project_root_fingerprint": project_root_fingerprint(descriptor.root),
"adapter": descriptor.adapter,
}
if not isinstance(self.project, IncrementalStateProject):
return {
**identity,
"revision": "unknown",
"source_hash": None,
"receipt_state": "unknown",
"receipt_reason": "source_identity_unavailable",
"generation_diff": None,
"staleness": "unknown",
}
before_state = self.project.incremental_state()
if before_state is None:
return {
**identity,
"revision": "unknown",
"source_hash": None,
"receipt_state": "unknown",
"receipt_reason": "source_identity_unavailable",
"generation_diff": None,
"staleness": "unknown",
}
receipt, reason, receipt_signature = load_generation_diff(descriptor)
if receipt is None:
receipt_state = (
"unsafe"
if reason == "unsafe_receipt"
else ("missing" if reason == "missing_receipt" else "unverified")
)
return {
**identity,
"revision": before_state.revision,
"source_hash": before_state.source_hash,
"receipt_state": receipt_state,
"receipt_reason": reason,
"generation_diff": None,
"staleness": "unknown",
}
try:
current_index_signature = index_signature(self.path)
except DocForgeError as error:
return {
**identity,
"revision": before_state.revision,
"source_hash": before_state.source_hash,
"receipt_state": "unsafe" if error.code == "path_escape" else "stale",
"receipt_reason": (
"unsafe_index" if error.code == "path_escape" else "missing_index"
),
"generation_diff": None,
"staleness": "stale",
}
target = cast(dict[str, object], receipt["to_generation"])
expected_index_signature = cast(dict[str, object], receipt["index_signature"])
observed_index_signature = {
"device": current_index_signature[0],
"inode": current_index_signature[1],
"size": current_index_signature[2],
"mtime_ns": current_index_signature[3],
"ctime_ns": current_index_signature[4],
}
if (
target["revision"] != before_state.revision
or target["source_hash"] != before_state.source_hash
or expected_index_signature != observed_index_signature
):
return {
**identity,
"revision": before_state.revision,
"source_hash": before_state.source_hash,
"receipt_state": "stale",
"receipt_reason": "generation_mismatch",
"generation_diff": None,
"staleness": "stale",
}
after_state = self.project.incremental_state()
try:
after_index_signature = index_signature(self.path)
after_receipt_signature = index_signature(generation_diff_path(descriptor))
except DocForgeError:
after_state = None
after_index_signature = ()
after_receipt_signature = ()
if (
after_state != before_state
or after_index_signature != current_index_signature
or after_receipt_signature != receipt_signature
):
return {
**identity,
"revision": before_state.revision,
"source_hash": before_state.source_hash,
"receipt_state": "unverified",
"receipt_reason": "concurrent_change",
"generation_diff": None,
"staleness": "unknown",
}
return {
**identity,
"revision": before_state.revision,
"source_hash": before_state.source_hash,
"receipt_state": "current",
"receipt_reason": None,
"generation_diff": receipt,
"staleness": "current",
}
def _task_context_capsule(
self,
snapshot: _IndexReadSnapshot,

158
src/docforge/mcp_server.py
View file

@ -50,6 +50,7 @@ READ_TOOLS = (
"docforge_visualize",
"docforge_stop_visualization",
"docforge_visualization_status",
"docforge_get_generation_diff",
)
PROPOSAL_TOOLS = (
"docforge_create_changeset",
@ -913,6 +914,151 @@ class DocForgeService:
return self.invoke(operation, operation_name="mcp.task_context")
def generation_diff(
self,
*,
limit: int | None = None,
cursor: str | None = None,
) -> dict[str, Any]:
"""Return one bounded page from the latest verified graph transition."""
def operation() -> dict[str, object]:
maximum_items = min(
self.project.descriptor.limits.max_results,
1_000,
)
selected_limit = page_limit(
limit,
default=min(20, maximum_items),
maximum=maximum_items,
)
result = self.index.generation_diff()
receipt_value = result.get("generation_diff")
if receipt_value is None:
if cursor is not None:
decode_cursor(
cursor,
kind="generation-diff.items",
binding={
"project_id": result.get("project_id"),
"receipt_state": result.get("receipt_state"),
"effective_policy_hash": canonical_hash(self.policy.as_dict()),
},
total_count=0,
)
return result
if not isinstance(receipt_value, Mapping):
raise DocForgeError(
"invalid_generation_diff",
"Generation diff receipt is malformed",
)
receipt = dict(cast(Mapping[str, object], receipt_value))
items_value = receipt.pop("items", None)
if not isinstance(items_value, list):
raise DocForgeError(
"invalid_generation_diff",
"Generation diff receipt has no bounded item collection",
)
items = cast(list[object], items_value)
stored_receipt_hash = receipt.pop("receipt_hash", None)
if not isinstance(stored_receipt_hash, str):
raise DocForgeError(
"invalid_generation_diff",
"Generation diff receipt has no stable identity",
)
receipt_header = {
**receipt,
"stored_receipt_hash": stored_receipt_hash,
}
binding = {
"stored_receipt_hash": stored_receipt_hash,
"effective_policy_hash": canonical_hash(self.policy.as_dict()),
}
position = decode_cursor(
cursor,
kind="generation-diff.items",
binding=binding,
total_count=len(items),
)
page_items: list[object] = []
page_omissions: list[dict[str, object]] = []
consumed = 0
maximum_chars = self.project.descriptor.limits.max_tool_output_chars
def page_result() -> dict[str, object]:
pagination = page_receipt(
kind="generation-diff.items",
binding=binding,
position=position,
count=consumed,
limit=selected_limit,
total_count=len(items),
)
page_hash = canonical_hash(
{
"page_schema_version": 1,
"receipt_state": result["receipt_state"],
"receipt_header": receipt_header,
"pagination": pagination,
"items": page_items,
"omissions": page_omissions,
}
)
return {
**result,
"generation_diff": {
"page_schema_version": 1,
"receipt_header": receipt_header,
"items": page_items,
"omissions": page_omissions,
"page_hash": page_hash,
},
"pagination": pagination,
}
candidates = items[position : position + selected_limit]
def fits(candidate_count: int) -> bool:
nonlocal consumed
page_items[:] = candidates[:candidate_count]
consumed = candidate_count
decorated = {
**page_result(),
"server_version": SERVER_VERSION,
"content_warning": CONTENT_WARNING,
}
return self._encoded_length(decorated) <= maximum_chars
lower = 0
upper = len(candidates)
while lower < upper:
midpoint = (lower + upper + 1) // 2
if fits(midpoint):
lower = midpoint
else:
upper = midpoint - 1
fits(lower)
if lower == 0 and candidates:
item = candidates[0]
item_payload: Mapping[str, object] = (
cast(Mapping[str, object], item) if isinstance(item, Mapping) else {}
)
page_omissions.append(
{
"code": "response_limit",
"item_hash": item_payload.get("item_hash"),
}
)
consumed = 1
return page_result()
return self.invoke(
operation,
synchronize=False,
load_error_identity=False,
operation_name="mcp.generation_diff",
)
def _page_task_context_result(
self,
result: dict[str, object],
@ -1436,6 +1582,15 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC
return service.visualization_status()
@server.tool(name="docforge_get_generation_diff")
def get_generation_diff(
limit: int | None = None,
cursor: str | None = None,
) -> dict[str, Any]:
"""Return the latest bounded primary-graph generation transition."""
return service.generation_diff(limit=limit, cursor=cursor)
_registered_read_tools = (
bootstrap,
synchronize,
@ -1453,8 +1608,9 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC
validate_project,
render_status,
visualize,
visualization_status,
stop_visualization,
visualization_status,
get_generation_diff,
)
if read_only:
return server

3
src/docforge/pagination.py
View file

@ -103,7 +103,6 @@ def decode_cursor(
or not isinstance(stored_binding, dict)
or type(position) is not int
or position < 0
or position >= total_count
or not isinstance(checksum, str)
or len(checksum) != 64
):
@ -117,6 +116,8 @@ def decode_cursor(
"stale_cursor",
"Pagination cursor does not match the current result generation",
)
if position >= total_count:
raise _invalid_cursor()
return position

2
src/docforge/telemetry.py
View file

@ -92,6 +92,7 @@ OPERATION_NAMES = frozenset(
"mcp.impact",
"mcp.context",
"mcp.task_context",
"mcp.generation_diff",
"mcp.validate_project",
"mcp.render_status",
"mcp.visualize",
@ -114,6 +115,7 @@ OPERATION_NAMES = frozenset(
"cli.dependencies",
"cli.impact",
"cli.context",
"cli.generation-diff",
"cli.render",
"cli.render-status",
"cli.preview",