"""Bounded latest-generation transition receipts for disposable graph indexes.""" from __future__ import annotations import json import os import secrets import stat from collections.abc import Mapping from contextlib import suppress from dataclasses import dataclass from pathlib import Path from typing import Literal, cast from ._fs_safety import open_bound_directory, require_bound_directory from .errors import DocForgeError from .models import Edge, Node, ProjectDescriptor, ProjectSnapshot from .pagination import canonical_hash GENERATION_DIFF_SCHEMA_VERSION = 1 GENERATION_DIFF_SEMANTICS_VERSION = 1 MAX_GENERATION_DIFF_ITEMS = 1_000 MAX_GENERATION_DIFF_BYTES = 1_048_576 GENERATION_DIFF_FILENAME = "generation-diff.json" IndexSignature = tuple[int, int, int, int, int] PredecessorReason = Literal[ "no_predecessor", "predecessor_unsafe", "predecessor_unsupported_schema", "predecessor_foreign", "predecessor_policy_incompatible", "predecessor_corrupt", "predecessor_unattested", "predecessor_changed", "no_meaningful_transition", ] _GENERATION_KEYS = frozenset( { "revision", "source_hash", "node_count", "node_hash", "edge_count", "edge_hash", "index_schema_version", } ) _SUMMARY_KEYS = frozenset( { "nodes_added", "nodes_removed", "nodes_changed", "edges_added", "edges_removed", "total_changes", } ) _SIGNATURE_KEYS = frozenset({"device", "inode", "size", "mtime_ns", "ctime_ns"}) _RECEIPT_KEYS = frozenset( { "schema_version", "diff_semantics_version", "project_id", "project_root_fingerprint", "adapter", "kind", "reason", "from_generation", "to_generation", "summary", "items", "full_item_count", "retained_item_count", "details_truncated", "truncation_reason", "full_collection_hash", "retained_collection_hash", "index_signature", "receipt_hash", } ) _BASELINE_REASONS = frozenset( { "no_predecessor", "predecessor_unsafe", "predecessor_unsupported_schema", "predecessor_foreign", "predecessor_policy_incompatible", "predecessor_corrupt", "predecessor_unattested", "predecessor_changed", "no_meaningful_transition", } ) _NODE_ITEM_KEYS = frozenset( { "entity", "change", "node_id", "before_content_hash", "after_content_hash", "before_source_path", "after_source_path", "before_node_hash", "after_node_hash", "changed_fields", "item_hash", } ) _NODE_CHANGED_FIELDS = frozenset( { "title", "family", "authority", "status", "tags", "summary", "content", "source_path", "source_anchor", "content_hash", } ) _EDGE_ITEM_KEYS = frozenset( { "entity", "change", "source_id", "relation", "target_id", "item_hash", } ) @dataclass(frozen=True) class PublishedGraph: """One completely verified predecessor publication.""" generation: dict[str, object] nodes: tuple[Node, ...] edges: tuple[Edge, ...] logic_hash: str signature: IndexSignature @dataclass(frozen=True) class GenerationDiffDraft: """A receipt body awaiting the committed index file identity.""" fields: dict[str, object] items: tuple[dict[str, object], ...] def generation_diff_path(descriptor: ProjectDescriptor) -> Path: """Return the fixed project-confined latest-transition receipt path.""" return descriptor.cache_root / GENERATION_DIFF_FILENAME def index_signature(path: Path) -> IndexSignature: """Return the exact identity of one safe regular index publication.""" try: status = path.lstat() except OSError as error: raise DocForgeError("missing_index", "Derived index does not exist") from error if stat.S_ISLNK(status.st_mode) or not stat.S_ISREG(status.st_mode): raise DocForgeError("path_escape", "Derived index path is not a safe regular file") return ( status.st_dev, status.st_ino, status.st_size, status.st_mtime_ns, status.st_ctime_ns, ) def signature_payload(signature: IndexSignature) -> dict[str, int]: """Convert one stat identity to its versioned JSON representation.""" return { "device": signature[0], "inode": signature[1], "size": signature[2], "mtime_ns": signature[3], "ctime_ns": signature[4], } def generation_identity(status: Mapping[str, object]) -> dict[str, object]: """Select the primary-graph identity stored in a public diff receipt.""" return { "revision": status["revision"], "source_hash": status["source_hash"], "node_count": status["node_count"], "node_hash": status["node_hash"], "edge_count": status["edge_count"], "edge_hash": status["edge_hash"], "index_schema_version": status["index_schema_version"], } def prepare_generation_diff( descriptor: ProjectDescriptor, *, predecessor: PublishedGraph | None, predecessor_reason: PredecessorReason | None, current_snapshot: ProjectSnapshot, current_status: Mapping[str, object], preserved_receipt: Mapping[str, object] | None = None, ) -> GenerationDiffDraft: """Prepare one deterministic latest transition without publishing it.""" current_generation = generation_identity(current_status) if predecessor is None: return _baseline_draft( descriptor, current_generation, predecessor_reason or "no_predecessor", ) if predecessor.generation == current_generation: if preserved_receipt is not None and _receipt_targets( preserved_receipt, descriptor, current_generation, predecessor.signature, ): fields = { key: value for key, value in preserved_receipt.items() if key not in {"index_signature", "receipt_hash", "items"} } preserved_items = preserved_receipt.get("items") if not isinstance(preserved_items, list): raise DocForgeError( "invalid_generation_diff", "Preserved generation diff has no item collection", ) items = tuple( cast(dict[str, object], item) for item in cast(list[object], preserved_items) if isinstance(item, dict) ) return GenerationDiffDraft(fields=fields, items=items) return _baseline_draft( descriptor, current_generation, "no_meaningful_transition", ) items = _change_items( predecessor.nodes, predecessor.edges, current_snapshot.nodes, current_snapshot.edges, ) summary = _summary(items) return GenerationDiffDraft( fields={ "schema_version": GENERATION_DIFF_SCHEMA_VERSION, "diff_semantics_version": GENERATION_DIFF_SEMANTICS_VERSION, "project_id": descriptor.project_id, "project_root_fingerprint": _root_fingerprint(descriptor), "adapter": descriptor.adapter, "kind": "transition", "reason": None, "from_generation": predecessor.generation, "to_generation": current_generation, "summary": summary, "full_item_count": len(items), "full_collection_hash": canonical_hash([item["item_hash"] for item in items]), }, items=items, ) def finalize_generation_diff( draft: GenerationDiffDraft, *, signature: IndexSignature, ) -> dict[str, object]: """Bind a draft to the committed index and enforce fixed receipt ceilings.""" full_count = cast(int, draft.fields["full_item_count"]) retained = list(draft.items[:MAX_GENERATION_DIFF_ITEMS]) item_limited = full_count > len(retained) byte_limited = draft.fields.get("truncation_reason") == "receipt_byte_limit" while True: receipt = _final_receipt( draft.fields, retained, signature=signature, item_limited=item_limited, byte_limited=byte_limited, ) if len(_receipt_bytes(receipt)) <= MAX_GENERATION_DIFF_BYTES: return receipt if not retained: raise DocForgeError( "generation_diff_failure", "Generation diff identity exceeds the fixed receipt byte limit", ) retained.pop() byte_limited = True def publish_generation_diff( descriptor: ProjectDescriptor, receipt: Mapping[str, object], ) -> None: """Atomically replace the single latest-generation receipt.""" root = descriptor.cache_root if generation_diff_path(descriptor).parent != root: raise DocForgeError("path_escape", "Generation diff path is not confined") raw = _receipt_bytes(receipt) if len(raw) > MAX_GENERATION_DIFF_BYTES: raise DocForgeError( "generation_diff_failure", "Generation diff receipt exceeds the fixed byte limit", ) root_fd = open_bound_directory(root) temporary_name = f".generation-diff-{secrets.token_hex(12)}" temporary_created = False try: try: status = os.stat( GENERATION_DIFF_FILENAME, dir_fd=root_fd, follow_symlinks=False, ) except FileNotFoundError: status = None if status is not None and ( stat.S_ISLNK(status.st_mode) or not stat.S_ISREG(status.st_mode) ): raise DocForgeError( "path_escape", "Generation diff receipt path is not a safe regular file", ) descriptor_fd = os.open( temporary_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=root_fd, ) temporary_created = True with os.fdopen(descriptor_fd, "wb") as handle: handle.write(raw) handle.flush() os.fsync(handle.fileno()) require_bound_directory(root, root_fd) os.replace( temporary_name, GENERATION_DIFF_FILENAME, src_dir_fd=root_fd, dst_dir_fd=root_fd, ) temporary_created = False os.fsync(root_fd) require_bound_directory(root, root_fd) except Exception: if temporary_created: with suppress(OSError): os.unlink(temporary_name, dir_fd=root_fd) raise finally: os.close(root_fd) def load_generation_diff( descriptor: ProjectDescriptor, ) -> tuple[dict[str, object] | None, str | None, IndexSignature | None]: """Read and validate one receipt without repairing any derived state.""" try: root_fd = open_bound_directory(descriptor.cache_root) except DocForgeError as error: reason = "missing_receipt" if error.code == "missing_index" else "unsafe_receipt" return None, reason, None try: try: descriptor_fd = os.open( GENERATION_DIFF_FILENAME, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=root_fd, ) except FileNotFoundError: return None, "missing_receipt", None except OSError: return None, "unsafe_receipt", None with os.fdopen(descriptor_fd, "rb") as handle: status_before = os.fstat(handle.fileno()) if stat.S_ISLNK(status_before.st_mode) or not stat.S_ISREG(status_before.st_mode): return None, "unsafe_receipt", None before: IndexSignature = ( status_before.st_dev, status_before.st_ino, status_before.st_size, status_before.st_mtime_ns, status_before.st_ctime_ns, ) if before[2] > MAX_GENERATION_DIFF_BYTES: return None, "oversized_receipt", before raw = handle.read(MAX_GENERATION_DIFF_BYTES + 1) status_after = os.fstat(handle.fileno()) after: IndexSignature = ( status_after.st_dev, status_after.st_ino, status_after.st_size, status_after.st_mtime_ns, status_after.st_ctime_ns, ) if len(raw) > MAX_GENERATION_DIFF_BYTES: return None, "oversized_receipt", before try: path_status = os.stat( GENERATION_DIFF_FILENAME, dir_fd=root_fd, follow_symlinks=False, ) except OSError: return None, "receipt_changed", before path_signature: IndexSignature = ( path_status.st_dev, path_status.st_ino, path_status.st_size, path_status.st_mtime_ns, path_status.st_ctime_ns, ) try: require_bound_directory(descriptor.cache_root, root_fd) except DocForgeError: return None, "unsafe_receipt", before finally: os.close(root_fd) if before != after or before != path_signature or len(raw) != before[2]: return None, "receipt_changed", before try: parsed: object = json.loads(raw.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError): return None, "corrupt_receipt", before if not isinstance(parsed, dict): return None, "corrupt_receipt", before receipt = cast(dict[str, object], parsed) if not validate_generation_diff_receipt(receipt): return None, "corrupt_receipt", before if ( receipt.get("project_id") != descriptor.project_id or receipt.get("project_root_fingerprint") != _root_fingerprint(descriptor) or receipt.get("adapter") != descriptor.adapter ): return None, "foreign_receipt", before return receipt, None, before def valid_generation_identity(value: object) -> bool: """Return whether a primary-graph generation has the exact version-1 shape.""" return _valid_generation(value) def validate_generation_diff_receipt( receipt: Mapping[str, object], *, descriptor: ProjectDescriptor | None = None, ) -> bool: """Strictly validate the complete version-1 receipt and its hashes.""" if frozenset(receipt) != _RECEIPT_KEYS: return False if ( receipt.get("schema_version") != GENERATION_DIFF_SCHEMA_VERSION or receipt.get("diff_semantics_version") != GENERATION_DIFF_SEMANTICS_VERSION or not _nonempty(receipt.get("project_id")) or not _fingerprint(receipt.get("project_root_fingerprint")) or not _nonempty(receipt.get("adapter")) ): return False if descriptor is not None and ( receipt["project_id"] != descriptor.project_id or receipt["project_root_fingerprint"] != _root_fingerprint(descriptor) or receipt["adapter"] != descriptor.adapter ): return False kind = receipt.get("kind") reason = receipt.get("reason") from_generation = receipt.get("from_generation") if kind == "baseline": if reason not in _BASELINE_REASONS or from_generation is not None: return False elif kind == "transition": if ( reason is not None or not _valid_generation(from_generation) or from_generation == receipt.get("to_generation") ): return False else: return False if not _valid_generation(receipt.get("to_generation")): return False summary_value = receipt.get("summary") items_value = receipt.get("items") if not isinstance(summary_value, dict) or not isinstance(items_value, list): return False summary = cast(dict[str, object], summary_value) items = cast(list[object], items_value) if frozenset(summary) != _SUMMARY_KEYS or any( type(value) is not int or value < 0 for value in summary.values() ): return False total = sum( cast(int, summary[key]) for key in ( "nodes_added", "nodes_removed", "nodes_changed", "edges_added", "edges_removed", ) ) if summary.get("total_changes") != total: return False if kind == "baseline" and (total != 0 or items): return False if len(items) > MAX_GENERATION_DIFF_ITEMS or not all(_valid_item(item) for item in items): return False typed_items = tuple(cast(dict[str, object], item) for item in items) if list(typed_items) != sorted(typed_items, key=_item_sort_key): return False identities = tuple(_item_identity(item) for item in typed_items) if len(identities) != len(set(identities)): return False retained_summary = _summary(typed_items) if any( retained_summary[key] > cast(int, summary[key]) for key in ( "nodes_added", "nodes_removed", "nodes_changed", "edges_added", "edges_removed", ) ): return False full_count = receipt.get("full_item_count") retained_count = receipt.get("retained_item_count") truncated = receipt.get("details_truncated") truncation_reason = receipt.get("truncation_reason") if ( type(full_count) is not int or type(retained_count) is not int or full_count != total or retained_count != len(items) or retained_count > full_count or type(truncated) is not bool or truncated != (retained_count < full_count) ): return False if truncation_reason is None: if truncated or full_count > MAX_GENERATION_DIFF_ITEMS: return False elif truncation_reason == "receipt_item_limit": if ( not truncated or full_count <= MAX_GENERATION_DIFF_ITEMS or retained_count != MAX_GENERATION_DIFF_ITEMS ): return False elif truncation_reason == "receipt_byte_limit": if not truncated or retained_count >= min(full_count, MAX_GENERATION_DIFF_ITEMS): return False else: return False item_hashes = [cast(dict[str, object], item)["item_hash"] for item in items] retained_hash = canonical_hash(item_hashes) if receipt.get("retained_collection_hash") != retained_hash: return False full_hash = receipt.get("full_collection_hash") if not _sha256(full_hash): return False if not truncated and full_hash != retained_hash: return False signature_value = receipt.get("index_signature") if not isinstance(signature_value, dict): return False signature = cast(dict[str, object], signature_value) if frozenset(signature) != _SIGNATURE_KEYS: return False if any(type(value) is not int or value < 0 for value in signature.values()): return False receipt_hash = receipt.get("receipt_hash") if not _sha256(receipt_hash): return False unhashed = {key: value for key, value in receipt.items() if key != "receipt_hash"} return receipt_hash == canonical_hash(unhashed) def _baseline_draft( descriptor: ProjectDescriptor, current_generation: Mapping[str, object], reason: PredecessorReason, ) -> GenerationDiffDraft: empty_hash = canonical_hash([]) return GenerationDiffDraft( fields={ "schema_version": GENERATION_DIFF_SCHEMA_VERSION, "diff_semantics_version": GENERATION_DIFF_SEMANTICS_VERSION, "project_id": descriptor.project_id, "project_root_fingerprint": _root_fingerprint(descriptor), "adapter": descriptor.adapter, "kind": "baseline", "reason": reason, "from_generation": None, "to_generation": dict(current_generation), "summary": { "nodes_added": 0, "nodes_removed": 0, "nodes_changed": 0, "edges_added": 0, "edges_removed": 0, "total_changes": 0, }, "full_item_count": 0, "full_collection_hash": empty_hash, }, items=(), ) def _change_items( before_nodes: tuple[Node, ...], before_edges: tuple[Edge, ...], after_nodes: tuple[Node, ...], after_edges: tuple[Edge, ...], ) -> tuple[dict[str, object], ...]: before_by_id = {node.node_id: node for node in before_nodes} after_by_id = {node.node_id: node for node in after_nodes} items: list[dict[str, object]] = [] for node_id in sorted(before_by_id.keys() | after_by_id.keys()): before = before_by_id.get(node_id) after = after_by_id.get(node_id) if before == after: continue if before is None: change = "added" elif after is None: change = "removed" else: change = "changed" before_payload = before.as_dict() if before is not None else None after_payload = after.as_dict() if after is not None else None changed_fields = ( [] if before_payload is None or after_payload is None else sorted(key for key in before_payload if before_payload[key] != after_payload[key]) ) payload: dict[str, object] = { "entity": "node", "change": change, "node_id": node_id, "before_content_hash": None if before is None else before.content_hash, "after_content_hash": None if after is None else after.content_hash, "before_source_path": None if before is None else before.source_path, "after_source_path": None if after is None else after.source_path, "before_node_hash": ( None if before_payload is None else canonical_hash(before_payload) ), "after_node_hash": (None if after_payload is None else canonical_hash(after_payload)), "changed_fields": changed_fields, } payload["item_hash"] = canonical_hash(payload) items.append(payload) before_edge_set = {(edge.source_id, edge.relation, edge.target_id) for edge in before_edges} after_edge_set = {(edge.source_id, edge.relation, edge.target_id) for edge in after_edges} for change, values in ( ("removed", sorted(before_edge_set - after_edge_set)), ("added", sorted(after_edge_set - before_edge_set)), ): for source_id, relation, target_id in values: payload = { "entity": "edge", "change": change, "source_id": source_id, "relation": relation, "target_id": target_id, } payload["item_hash"] = canonical_hash(payload) items.append(payload) return tuple(sorted(items, key=_item_sort_key)) def _item_sort_key(item: Mapping[str, object]) -> tuple[str, str, str, str, str]: return ( cast(str, item["entity"]), cast(str, item.get("node_id", item.get("source_id", ""))), cast(str, item.get("relation", "")), cast(str, item.get("target_id", "")), cast(str, item["change"]), ) def _item_identity(item: Mapping[str, object]) -> tuple[str, ...]: if item["entity"] == "node": return ("node", cast(str, item["node_id"])) return ( "edge", cast(str, item["source_id"]), cast(str, item["relation"]), cast(str, item["target_id"]), ) def _summary(items: tuple[dict[str, object], ...]) -> dict[str, int]: result = { "nodes_added": 0, "nodes_removed": 0, "nodes_changed": 0, "edges_added": 0, "edges_removed": 0, "total_changes": len(items), } for item in items: entity = cast(str, item["entity"]) change = cast(str, item["change"]) key = f"{entity}s_{change}" result[key] += 1 return result def _final_receipt( fields: Mapping[str, object], retained: list[dict[str, object]], *, signature: IndexSignature, item_limited: bool, byte_limited: bool, ) -> dict[str, object]: full_count = cast(int, fields["full_item_count"]) truncated = len(retained) < full_count if byte_limited: reason: str | None = "receipt_byte_limit" elif item_limited: reason = "receipt_item_limit" else: reason = None receipt = { **fields, "items": retained, "retained_item_count": len(retained), "details_truncated": truncated, "truncation_reason": reason, "retained_collection_hash": canonical_hash([item["item_hash"] for item in retained]), "index_signature": signature_payload(signature), } receipt["receipt_hash"] = canonical_hash(receipt) return receipt def _receipt_targets( receipt: Mapping[str, object], descriptor: ProjectDescriptor, generation: Mapping[str, object], signature: IndexSignature, ) -> bool: return ( validate_generation_diff_receipt(receipt, descriptor=descriptor) and receipt.get("to_generation") == dict(generation) and receipt.get("index_signature") == signature_payload(signature) ) def _valid_generation(value: object) -> bool: if not isinstance(value, dict): return False generation = cast(dict[str, object], value) if frozenset(generation) != _GENERATION_KEYS: return False source_hash = generation.get("source_hash") node_hash = generation.get("node_hash") edge_hash = generation.get("edge_hash") return ( _nonempty(generation.get("revision")) and _sha256(source_hash) and _sha256(node_hash) and _sha256(edge_hash) and type(generation.get("node_count")) is int and cast(int, generation["node_count"]) >= 0 and type(generation.get("edge_count")) is int and cast(int, generation["edge_count"]) >= 0 and type(generation.get("index_schema_version")) is int and cast(int, generation["index_schema_version"]) >= 1 ) def _valid_item(value: object) -> bool: if not isinstance(value, dict): return False payload = cast(dict[str, object], value) entity = payload.get("entity") if entity == "node": if frozenset(payload) != _NODE_ITEM_KEYS: return False changed_fields_value = payload.get("changed_fields") if not isinstance(changed_fields_value, list): return False changed_fields = cast(list[object], changed_fields_value) if ( payload.get("change") not in {"added", "removed", "changed"} or not _nonempty(payload.get("node_id")) or any( not _nonempty(field) or field not in _NODE_CHANGED_FIELDS for field in changed_fields ) or changed_fields != sorted(set(cast(list[str], changed_fields))) ): return False for key in ( "before_content_hash", "after_content_hash", "before_node_hash", "after_node_hash", ): candidate = payload.get(key) if candidate is not None and not _sha256(candidate): return False for key in ("before_source_path", "after_source_path"): candidate = payload.get(key) if candidate is not None and not _nonempty(candidate): return False change = payload["change"] before_values = ( payload["before_content_hash"], payload["before_source_path"], payload["before_node_hash"], ) after_values = ( payload["after_content_hash"], payload["after_source_path"], payload["after_node_hash"], ) if change == "added": if ( any(value is not None for value in before_values) or not all(value is not None for value in after_values) or changed_fields ): return False elif change == "removed": if ( not all(value is not None for value in before_values) or any(value is not None for value in after_values) or changed_fields ): return False elif ( not all(value is not None for value in (*before_values, *after_values)) or not changed_fields or payload["before_node_hash"] == payload["after_node_hash"] ): return False elif entity == "edge": if ( frozenset(payload) != _EDGE_ITEM_KEYS or payload.get("change") not in {"added", "removed"} or not _nonempty(payload.get("source_id")) or not _nonempty(payload.get("relation")) or not _nonempty(payload.get("target_id")) ): return False else: return False item_hash = payload.get("item_hash") unhashed = {key: item for key, item in payload.items() if key != "item_hash"} return _sha256(item_hash) and item_hash == canonical_hash(unhashed) def _receipt_bytes(receipt: Mapping[str, object]) -> bytes: return json.dumps(receipt, sort_keys=True, indent=2, ensure_ascii=True).encode("utf-8") + b"\n" def _root_fingerprint(descriptor: ProjectDescriptor) -> str: from .project import project_root_fingerprint return project_root_fingerprint(descriptor.root) def _nonempty(value: object) -> bool: return isinstance(value, str) and bool(value) def _sha256(value: object) -> bool: return ( isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value) ) def _fingerprint(value: object) -> bool: return ( isinstance(value, str) and len(value) == 16 and all(character in "0123456789abcdef" for character in value) )