From 9a48233983b7dd760f3f2c39f47eb37056ea3c63 Mon Sep 17 00:00:00 2001 From: Andraxion Date: Wed, 29 Jul 2026 08:23:04 -0400 Subject: [PATCH] Add bounded generation transition receipts --- ACTIVE_SLICE.md | 8 +- DEVELOPMENT_NOTES.md | 78 ++ README.md | 8 + docs/COMPATIBILITY.md | 10 + docs/CONTRACT.md | 18 + docs/MCP_CONTRACT.md | 32 + docs/USER_MANUAL.md | 31 + schemas/generation-diff-page.schema.json | 430 ++++++++++ schemas/generation-diff.schema.json | 363 +++++++++ schemas/result.schema.json | 16 +- src/docforge/_fs_safety.py | 71 ++ src/docforge/cli.py | 13 + src/docforge/generation_diff.py | 911 +++++++++++++++++++++ src/docforge/index.py | 717 ++++++++++++++-- src/docforge/mcp_server.py | 158 +++- src/docforge/pagination.py | 3 +- src/docforge/telemetry.py | 2 + tests/test_adapter_contract.py | 14 +- tests/test_generation_diff.py | 993 +++++++++++++++++++++++ tests/test_mcp_server.py | 1 + tests/test_public_contract.py | 10 + 21 files changed, 3822 insertions(+), 65 deletions(-) create mode 100644 schemas/generation-diff-page.schema.json create mode 100644 schemas/generation-diff.schema.json create mode 100644 src/docforge/_fs_safety.py create mode 100644 src/docforge/generation_diff.py create mode 100644 tests/test_generation_diff.py diff --git a/ACTIVE_SLICE.md b/ACTIVE_SLICE.md index c53b3f8..01a3810 100644 --- a/ACTIVE_SLICE.md +++ b/ACTIVE_SLICE.md @@ -6,9 +6,11 @@ Goal: Let one project-bound server return compact, task-shaped, explainable cont In scope: Capability modes; capability-aware bootstrap; versioned retrieval plans and context capsules; task-shaped context; generation diffs; evidence-gap diagnostics; generated client configuration; doctor checks. Out of scope: Independent render-plan packages; adapter SDK expansion; self-hosting; storage replacement; embeddings; WorldForge or ScrapeStation changes; production MCP repointing; tags and releases. Done when: Policy and capabilities are explicit; bootstrap recommends only available actions; task context is compact, deterministic, provenance-bearing, and bounded; generation and evidence gaps are explainable; generated configuration and doctor checks are safe and tested; the complete repository gate and Milestone 2 benchmark pass. -Status: Active. Effective policy is committed. Versioned task retrieval and context-capsule -transport pass independent audit and the complete repository gate. The latest-generation diff -receipt is the next slice. +Status: Active. Effective policy and versioned task retrieval are committed. The latest-generation +diff receipt is implemented with focused contract, failure, CLI, MCP, legacy-adapter, no-AST, and +zero-work tests. The full repository gate passes with 176 tests and 113 subtests. Independent +publication, contract, and performance audits approve the hardened tree for commit. Generated +client configuration and doctor checks follow. ``` Milestones 3–5 remain directional context and are not active. diff --git a/DEVELOPMENT_NOTES.md b/DEVELOPMENT_NOTES.md index 2427380..7eb8d0e 100644 --- a/DEVELOPMENT_NOTES.md +++ b/DEVELOPMENT_NOTES.md @@ -445,3 +445,81 @@ adapters. The complete repository gate passes with 158 tests and 101 subtests, zero Pyright diagnostics, warning-strict execution, package builds, public-contract validation, and the maintained Milestone 0 and Milestone 1 smoke benchmarks. Gitleaks 8.30.1 reports no secret findings in the working tree. + +### Latest-generation diff receipt + +Three read-only audits reconciled the index publication, public transport, compatibility, and +no-AST boundaries before implementation. The selected design stores one disposable +`generation-diff.json` receipt. It does not add a history database, arbitrary generation +selectors, source text, rendered content, or Logic details. + +Before a build loads current source, it accepts an existing index only when its exact main-file +inode has a matching stable whole-file attestation and no WAL, journal, or shared-memory sidecar. +It then captures that predecessor through an immutable main-file transaction. The capture validates +the SQLite application and schema IDs, project/root/adapter binding, integrity, complete node and +edge rows, Logic aggregate identity, FTS count, metadata hashes and counts, and final file +signature. It never calls normal check or synchronization and never repairs predecessor evidence. + +The final source revalidation now compares exact nodes and edges in addition to source hash, +revision, and Logic. A verified predecessor that maps the same source identity to different graph +content fails before publication as `generation_collision`. This closes a pre-existing adapter +determinism gap found during the generation-diff audit. + +SQLite replacement is now the explicit derived mutation commit point. Whole-file attestation, +cheap source-generation, and generation-diff receipts publish independently afterward. Any +post-commit receipt failure returns `status = ok`, `index = published`, a bounded degraded +publication record, and receipt-stage names. It never rolls back the new index or reports a false failed +mutation. Attestation hashing checks the exact index signature before, during, and immediately +before receipt publication. + +Version-1 diff semantics compare every core `Node` field by stable node ID and exact edge triples. +Node renames are removal plus addition. Edge changes are removal plus addition. Exact summary +counts and a full ordered item-hash collection cover every change. Retained details are +deterministically ordered and independently capped at 1,000 items and 1 MiB with explicit item- or +byte-limit evidence. A first build or untrusted predecessor is a baseline with no fabricated +all-added result. A same-generation reindex republishes the existing meaningful transition against +the new index file identity instead of erasing it with an empty diff. + +The additive public surfaces are: + +- CLI `generation-diff [--limit N] [--cursor OPAQUE]`. +- MCP `docforge_get_generation_diff(limit=None, cursor=None)`. +- Telemetry operations `cli.generation-diff` and `mcp.generation_diff`. + +Public reads do not open SQLite, call `project.load()`, extract an adapter projection, parse source, +check, synchronize, build, or repair. They strictly validate the bounded receipt, compare stable +receipt and index file identities, require two matching cheap source-generation checks, and report +unknown for legacy adapters without that capability. Missing, corrupt, foreign, oversized, +symlinked, stale, or concurrently changed evidence remains a read-only status outcome. + +Generation-diff pagination binds the complete stored receipt hash and effective policy. That hash +already covers project, generation, graph, collection, and committed-index identity. Page size may +change. A replaced receipt returns `stale_cursor`. One top-level pagination object owns the only +cursor. The nested version-1 page uses `receipt_header.stored_receipt_hash` so it never +misrepresents the complete receipt hash as the hash of a partial header. The summary distinguishes +additional retained pages from details permanently omitted by the fixed publication limits. + +Adversarial coverage now includes strict runtime/schema rejection, predecessor attestation and +generation identity, live and synthetic SQLite sidecars, cache-root symlink substitution, +source/sidecar changes during diff preparation, independent receipt failures, and degraded +post-commit identity and durability failures. Focused verification passes the direct, CLI, MCP, +schema, pagination, legacy, incremental no-AST, and zero-work suites. The complete repository gate +passes with 176 tests and 113 subtests, zero Pyright diagnostics, package builds, web checks, and +the maintained Milestone 0 and Milestone 1 smoke benchmarks. Final independent re-audit is in +progress before this slice is committed. + +The final dense benchmark uses a 1,000-node transition with 1,000 changed details. Its receipt is +775,663 bytes. Direct status is 37.659 ms median and 39.108 ms p95. A maximum-size MCP request +returns 307 items in 199,754 bytes at 54.037 ms median and 56.617 ms p95. Four pages reconstruct +all 1,000 retained details in 652,798 bytes at 201.55 ms median. Peak RSS is 79,096 KiB. +Every hidden-work counter remains zero; the read performs exactly two cheap source-generation +checks. + +Measurement found and removed two avoidable costs before commit. Receipt loading had repeated the +complete 1,000-item validator solely to check project identity; it now validates once and compares +the three binding fields directly. Page fitting had encoded every growing prefix; it now uses an +exact logarithmic search and retains the hash-only oversized-item omission path. The maximum page +fell from 272.06 ms p95 to 56.617 ms p95, while full traversal fell from roughly 859 ms to +203.55 ms p95. Regression tests require one receipt validation and at most 15 response encodes for +1,000 page candidates. Final independent publication, contract, and performance audits approve +the slice for commit. diff --git a/README.md b/README.md index a207622..d08b2ab 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ declared manuals, visualizes project structure, and manages reviewable documenta - Exposes project-bound CLI and MCP query surfaces. - Compiles versioned, generation-bound task context with cited evidence, explicit gaps, and bounded continuation. +- Records one bounded, versioned latest-generation graph transition without creating a history + database. - Automatically synchronizes disposable indexes before MCP work. - Creates, validates, diffs, and previews isolated changesets. - Registers complete proposals atomically without caller-managed hash chaining. @@ -61,6 +63,12 @@ executes it against one immutable index generation, and returns a hash-bound con Project relation names remain authoritative. DocForge classifies only its versioned alias set and preserves every unknown relation as `unclassified` instead of guessing semantics. +`docforge_get_generation_diff` reports the latest verified primary-graph transition through one +bounded disposable receipt. It includes exact node and edge change counts, hash-bound retained +details, and explicit truncation. Paged results use one top-level cursor and a versioned +`receipt_header`; `stored_receipt_hash` identifies the complete persisted receipt. The read never +exposes Logic details, loads canonical source, repairs derived state, or invents history. + ## Graph views The browser presents the primary architecture graph through three complementary views and loads a diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 14cf2f2..99053b1 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -79,6 +79,12 @@ Milestone 0 preserves: Context and changeset MCP reads accept optional limits and opaque generation-bound cursors. Direct Python changeset methods and the ordinary CLI context command retain full legacy results when pagination is not requested. +- Latest-generation-diff receipt schema version 1. The additive `generation-diff` CLI command and + `docforge_get_generation_diff` MCP read accept only optional pagination fields. They record one + primary-graph transition and do not create a history store or expose Logic details. +- Latest-generation-diff page schema version 1. Pages use one top-level pagination object and a + nested `receipt_header`. `stored_receipt_hash` names the complete stored receipt. Opaque cursors + may be restarted after a server or receipt change and are not durable public identifiers. Indexes, attestations, extraction caches, previews, and rendered artifacts are disposable. A schema change may rebuild them. Canonical project content and stored proposals may not be silently @@ -99,6 +105,10 @@ must retain `load_projection()` as their independent clean-build and equivalence Project adapters remain explicitly composed. Generic DocForge does not discover arbitrary adapter modules or choose a project globally. +The supported generation-diff Python boundary is `ProjectIndex.generation_diff()`. Helpers in the +`docforge.generation_diff` module implement the disposable publication contract and are internal; +they are not frozen as adapter-authoring imports. + ## Preserved no-AST binding `docforge-mcp --project-root /project --no-ast` is a stable shorthand for the diff --git a/docs/CONTRACT.md b/docs/CONTRACT.md index 39e2b8e..832a5af 100644 --- a/docs/CONTRACT.md +++ b/docs/CONTRACT.md @@ -20,6 +20,8 @@ commit when Git is available; it cannot change repository state. - Changeset schema: `schemas/changeset.schema.json`, version 1. - Effective policy: `schemas/policy.schema.json`, version 1. - Task context capsule: `schemas/context-capsule.schema.json`, version 1. +- Latest generation diff: `schemas/generation-diff.schema.json`, version 1. +- Latest generation-diff page: `schemas/generation-diff-page.schema.json`, version 1. - Index schema: version 3, disposable and reproducible. - Index attestation: schema version 1, disposable and reproducible. - Core, CLI, and MCP server: version 1.3.0.dev0. @@ -57,6 +59,22 @@ SQLite integrity verification. A fresh process may use that receipt to verify an without reconstructing all graph rows. A missing, malformed, or mismatched receipt falls back to complete verification and is repaired only after that verification succeeds. +Index replacement is the derived publication commit point. Attestation, cheap source-generation, +and latest-generation-diff receipts are independent post-commit evidence. Their failure produces +bounded degraded success and never falsely reports that a committed index mutation failed. + +Before replacement, a build accepts a predecessor only when its exact main-file inode has a +matching whole-file attestation, has no WAL, journal, or shared-memory sidecar, and passes the +published SQLite identity, row, hash, FTS, integrity, and policy checks. It uses an immutable +main-file read and never repairs predecessor evidence. The build then revalidates the new source +snapshot including exact node and edge equality and rejects a stable source identity that produces +different graph content as `generation_collision`. + +The version-1 generation-diff receipt stores one bounded latest primary-graph transition. It is +not history and contains no Logic details or source text. Public pages carry one +`receipt_header`; its `stored_receipt_hash` identifies the complete persisted receipt rather than +the header alone. One top-level pagination object carries the only continuation cursor. + ## Isolated proposal model Create, update, move, and delete are ordered node operations inside an isolated changeset. Every diff --git a/docs/MCP_CONTRACT.md b/docs/MCP_CONTRACT.md index 3dff2ec..b710bb8 100644 --- a/docs/MCP_CONTRACT.md +++ b/docs/MCP_CONTRACT.md @@ -48,6 +48,7 @@ schema version 1 and does not silently acquire machine-specific process policy. - `docforge_visualize` - `docforge_stop_visualization` - `docforge_visualization_status` +- `docforge_get_generation_diff` Each response states that document text is project content, not higher-priority instructions. Each response includes project identity, revision, source hash, adapter version, and staleness state. @@ -108,6 +109,37 @@ the effective policy and task request. One evidence item that cannot fit advance a hash-identified `response_limit` omission. A changed generation, policy, plan, or collection fails as `stale_cursor`. +`docforge_get_generation_diff` accepts only optional `limit` and `cursor` fields. It reads the one +latest version-1 primary-graph transition receipt; it does not accept arbitrary generations, +paths, or history selectors. Exact summary counts and the full item-collection hash cover the +complete transition. Pagination covers only the deterministically ordered retained details and +states separately when the fixed 1,000-item or 1 MiB publication limit permanently omitted +details. + +The receipt binds project, root, adapter, index schema, from/to source identity, node and edge +hashes and counts, the committed index file identity, retained and full collection hashes, and its +own canonical hash. Node changes compare every core `Node` field. Edge identity is the exact +`(source_id, relation, target_id)` triple. Logic is excluded from public diff details. + +Current pages use `page_schema_version = 1`. The nested `receipt_header` contains every stored +receipt field except `items`; its `stored_receipt_hash` is the hash of the complete stored receipt, +not of the header. Page items and hash-identified response-limit omissions are siblings of that +header. The only pagination object is at the top level, and its `next_cursor` is the only cursor +copy. The page hash covers the complete header, page items, omissions, receipt state, and +pagination receipt. + +Receipt states are fail-closed: `current` is proven against cheap source identity and exact index +and receipt inodes; `stale` is a proven generation mismatch; `missing` means no receipt; +`unsafe` means confinement or file-type checks failed; `unverified` covers corrupt, foreign, +oversized, or concurrently changed evidence; and `unknown` means the project cannot provide a +cheap generation identity. Only `current` returns a page. + +This status boundary is non-repairing. It never opens SQLite, loads or extracts an adapter +projection, parses source, synchronizes, builds, or writes a receipt. Cheap source identity and +stable receipt/index file identities can establish `current`; legacy projects without cheap +identity report `unknown`. Invalid or unavailable disposable evidence remains an explicit status +instead of triggering hidden recovery. + The legacy `docforge_get_context` tool and its custom three-argument provider contract remain unchanged. A server with a custom context provider does not silently inherit the core task planner; version 1 exposes no custom task-planner extension point. `docforge_get_task_context` returns diff --git a/docs/USER_MANUAL.md b/docs/USER_MANUAL.md index beeacbe..48e26b5 100644 --- a/docs/USER_MANUAL.md +++ b/docs/USER_MANUAL.md @@ -478,8 +478,15 @@ backlinks NODE_ID [--relation RELATION] [--limit N] dependencies NODE_ID [--depth N] [--limit N] impact NODE_ID [--depth N] [--limit N] context PROFILE [--budget N] [--limit N] [--cursor OPAQUE] +generation-diff [--limit N] [--cursor OPAQUE] ``` +`generation-diff` returns the latest verified primary-graph transition. It is not a history query. +Current results carry a version-1 page, a `receipt_header` bound to the complete stored receipt by +`stored_receipt_hash`, and one top-level pagination cursor. Missing, unsafe, stale, corrupt, or +unprovable disposable evidence is reported as a non-repairing receipt status. The command never +builds or repairs the index. + ### Render and proposal commands ```text @@ -588,6 +595,7 @@ Example MCP client configuration: - `docforge_visualize` - `docforge_visualization_status` - `docforge_stop_visualization` +- `docforge_get_generation_diff` ### Proposal tools @@ -644,6 +652,24 @@ Task context never exceeds 1,000 evidence items, 100,000 examined candidate edge query characters, even when a project configures broader general limits. An edge-work or unclassified-relation ceiling appears as an explicit omission rather than an unbounded response. +Use `docforge_get_generation_diff` after synchronization or a completed implementation slice to +inspect the one latest verified primary-graph transition. The version-1 receipt reports exact +added, removed, and changed node counts plus added and removed edge counts. Retained node details +identify changed fields and before/after hashes and source paths. Edge details retain the exact raw +relation triple. The receipt stores no source text, rendered content, Logic identities, or +historical sequence. + +The first successful publication is an explicit baseline and does not claim every current node was +added. A corrupt, foreign, unsafe, or unavailable predecessor produces an unavailable comparison +rather than fabricated removals. A same-generation reindex preserves the latest meaningful +transition. Each later real transition atomically replaces the single disposable receipt. + +Generation-diff reads use only the bounded receipt, stable file identities, and an adapter's cheap +source-generation proof. They do not open SQLite, load a complete adapter projection, parse source, +synchronize, build, or repair. Legacy adapters without cheap identity report `unknown`. Missing, +corrupt, foreign, oversized, or concurrently changed receipts report an explicit receipt state and +do not trigger hidden recovery. + Recommended release-candidate sequence: 1. Call `docforge_bootstrap`. It synchronizes derived state and reports the exact fixed binding. @@ -711,6 +737,11 @@ followed by capsule omissions. Keep the semantic task arguments unchanged while may change. Every page retains the same plan, collection, and capsule hashes. A `stale_cursor` means that the generation, policy, plan, or collection changed; discard earlier pages and restart. +`docforge_get_generation_diff` paginates only the details retained in the latest bounded receipt. +Its summary counts and full collection hash still cover permanently truncated details. The cursor +binds the exact receipt, target generation, retained and full collection hashes, receipt state, and +effective policy. A replacement receipt returns `stale_cursor`; restart from its first page. + Canonical application records its terminal receipt immediately after the project-owned serializer verifies the new canonical state. A later index or render refresh failure is reported as degraded derived state with remediation, not as permission to apply the same canonical change again. diff --git a/schemas/generation-diff-page.schema.json b/schemas/generation-diff-page.schema.json new file mode 100644 index 0000000..f2894ec --- /dev/null +++ b/schemas/generation-diff-page.schema.json @@ -0,0 +1,430 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://docforge.local/schema/generation-diff-page-v1.json", + "title": "DocForge bounded latest-generation diff page", + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "generation": { + "type": "object", + "required": [ + "revision", + "source_hash", + "node_count", + "node_hash", + "edge_count", + "edge_hash", + "index_schema_version" + ], + "properties": { + "revision": { "type": "string", "minLength": 1 }, + "source_hash": { "$ref": "#/$defs/sha256" }, + "node_count": { "type": "integer", "minimum": 0 }, + "node_hash": { "$ref": "#/$defs/sha256" }, + "edge_count": { "type": "integer", "minimum": 0 }, + "edge_hash": { "$ref": "#/$defs/sha256" }, + "index_schema_version": { "type": "integer", "minimum": 1 } + }, + "additionalProperties": false + }, + "summary": { + "type": "object", + "required": [ + "nodes_added", + "nodes_removed", + "nodes_changed", + "edges_added", + "edges_removed", + "total_changes" + ], + "properties": { + "nodes_added": { "type": "integer", "minimum": 0 }, + "nodes_removed": { "type": "integer", "minimum": 0 }, + "nodes_changed": { "type": "integer", "minimum": 0 }, + "edges_added": { "type": "integer", "minimum": 0 }, + "edges_removed": { "type": "integer", "minimum": 0 }, + "total_changes": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": false + }, + "node_item": { + "type": "object", + "required": [ + "entity", + "change", + "node_id", + "before_content_hash", + "after_content_hash", + "before_source_path", + "after_source_path", + "before_node_hash", + "after_node_hash", + "changed_fields", + "item_hash" + ], + "properties": { + "entity": { "const": "node" }, + "change": { "enum": ["added", "removed", "changed"] }, + "node_id": { "type": "string", "minLength": 1 }, + "before_content_hash": { + "oneOf": [{ "$ref": "#/$defs/sha256" }, { "type": "null" }] + }, + "after_content_hash": { + "oneOf": [{ "$ref": "#/$defs/sha256" }, { "type": "null" }] + }, + "before_source_path": { "type": ["string", "null"] }, + "after_source_path": { "type": ["string", "null"] }, + "before_node_hash": { + "oneOf": [{ "$ref": "#/$defs/sha256" }, { "type": "null" }] + }, + "after_node_hash": { + "oneOf": [{ "$ref": "#/$defs/sha256" }, { "type": "null" }] + }, + "changed_fields": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "title", + "family", + "authority", + "status", + "tags", + "summary", + "content", + "source_path", + "source_anchor", + "content_hash" + ] + } + }, + "item_hash": { "$ref": "#/$defs/sha256" } + }, + "allOf": [ + { + "if": { + "properties": { "change": { "const": "added" } }, + "required": ["change"] + }, + "then": { + "properties": { + "before_content_hash": { "type": "null" }, + "before_source_path": { "type": "null" }, + "before_node_hash": { "type": "null" }, + "after_content_hash": { "$ref": "#/$defs/sha256" }, + "after_source_path": { "type": "string", "minLength": 1 }, + "after_node_hash": { "$ref": "#/$defs/sha256" }, + "changed_fields": { "maxItems": 0 } + } + } + }, + { + "if": { + "properties": { "change": { "const": "removed" } }, + "required": ["change"] + }, + "then": { + "properties": { + "before_content_hash": { "$ref": "#/$defs/sha256" }, + "before_source_path": { "type": "string", "minLength": 1 }, + "before_node_hash": { "$ref": "#/$defs/sha256" }, + "after_content_hash": { "type": "null" }, + "after_source_path": { "type": "null" }, + "after_node_hash": { "type": "null" }, + "changed_fields": { "maxItems": 0 } + } + } + }, + { + "if": { + "properties": { "change": { "const": "changed" } }, + "required": ["change"] + }, + "then": { + "properties": { + "before_content_hash": { "$ref": "#/$defs/sha256" }, + "before_source_path": { "type": "string", "minLength": 1 }, + "before_node_hash": { "$ref": "#/$defs/sha256" }, + "after_content_hash": { "$ref": "#/$defs/sha256" }, + "after_source_path": { "type": "string", "minLength": 1 }, + "after_node_hash": { "$ref": "#/$defs/sha256" }, + "changed_fields": { "minItems": 1 } + } + } + } + ], + "additionalProperties": false + }, + "edge_item": { + "type": "object", + "required": [ + "entity", + "change", + "source_id", + "relation", + "target_id", + "item_hash" + ], + "properties": { + "entity": { "const": "edge" }, + "change": { "enum": ["added", "removed"] }, + "source_id": { "type": "string", "minLength": 1 }, + "relation": { "type": "string", "minLength": 1 }, + "target_id": { "type": "string", "minLength": 1 }, + "item_hash": { "$ref": "#/$defs/sha256" } + }, + "additionalProperties": false + }, + "receipt_header": { + "type": "object", + "required": [ + "schema_version", + "diff_semantics_version", + "project_id", + "project_root_fingerprint", + "adapter", + "kind", + "reason", + "from_generation", + "to_generation", + "summary", + "full_item_count", + "retained_item_count", + "details_truncated", + "truncation_reason", + "full_collection_hash", + "retained_collection_hash", + "index_signature", + "stored_receipt_hash" + ], + "properties": { + "schema_version": { "const": 1 }, + "diff_semantics_version": { "const": 1 }, + "project_id": { "type": "string", "minLength": 1 }, + "project_root_fingerprint": { + "type": "string", + "pattern": "^[0-9a-f]{16}$" + }, + "adapter": { "type": "string", "minLength": 1 }, + "kind": { "enum": ["baseline", "transition"] }, + "reason": { + "enum": [ + null, + "no_predecessor", + "predecessor_unsafe", + "predecessor_unsupported_schema", + "predecessor_foreign", + "predecessor_policy_incompatible", + "predecessor_corrupt", + "predecessor_unattested", + "predecessor_changed", + "no_meaningful_transition" + ] + }, + "from_generation": { + "oneOf": [{ "$ref": "#/$defs/generation" }, { "type": "null" }] + }, + "to_generation": { "$ref": "#/$defs/generation" }, + "summary": { "$ref": "#/$defs/summary" }, + "full_item_count": { "type": "integer", "minimum": 0 }, + "retained_item_count": { + "type": "integer", + "minimum": 0, + "maximum": 1000 + }, + "details_truncated": { "type": "boolean" }, + "truncation_reason": { + "enum": [null, "receipt_item_limit", "receipt_byte_limit"] + }, + "full_collection_hash": { "$ref": "#/$defs/sha256" }, + "retained_collection_hash": { "$ref": "#/$defs/sha256" }, + "index_signature": { + "type": "object", + "required": ["device", "inode", "size", "mtime_ns", "ctime_ns"], + "properties": { + "device": { "type": "integer", "minimum": 0 }, + "inode": { "type": "integer", "minimum": 0 }, + "size": { "type": "integer", "minimum": 0 }, + "mtime_ns": { "type": "integer", "minimum": 0 }, + "ctime_ns": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": false + }, + "stored_receipt_hash": { "$ref": "#/$defs/sha256" } + }, + "allOf": [ + { + "if": { + "properties": { "kind": { "const": "baseline" } }, + "required": ["kind"] + }, + "then": { + "properties": { + "from_generation": { "type": "null" }, + "reason": { "not": { "type": "null" } }, + "summary": { + "properties": { + "nodes_added": { "const": 0 }, + "nodes_removed": { "const": 0 }, + "nodes_changed": { "const": 0 }, + "edges_added": { "const": 0 }, + "edges_removed": { "const": 0 }, + "total_changes": { "const": 0 } + } + }, + "full_item_count": { "const": 0 }, + "retained_item_count": { "const": 0 }, + "details_truncated": { "const": false }, + "truncation_reason": { "const": null }, + "full_collection_hash": { + "const": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "retained_collection_hash": { + "const": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + } + }, + "else": { + "properties": { + "from_generation": { "$ref": "#/$defs/generation" }, + "reason": { "type": "null" } + } + } + }, + { + "if": { + "properties": { "truncation_reason": { "const": null } }, + "required": ["truncation_reason"] + }, + "then": { + "properties": { + "details_truncated": { "const": false }, + "full_item_count": { "maximum": 1000 } + } + } + }, + { + "if": { + "properties": { + "truncation_reason": { "const": "receipt_item_limit" } + }, + "required": ["truncation_reason"] + }, + "then": { + "properties": { + "details_truncated": { "const": true }, + "full_item_count": { "minimum": 1001 }, + "retained_item_count": { "const": 1000 } + } + } + }, + { + "if": { + "properties": { + "truncation_reason": { "const": "receipt_byte_limit" } + }, + "required": ["truncation_reason"] + }, + "then": { + "properties": { + "details_truncated": { "const": true } + } + } + } + ], + "additionalProperties": false + }, + "page_body": { + "type": "object", + "required": [ + "page_schema_version", + "receipt_header", + "items", + "omissions", + "page_hash" + ], + "properties": { + "page_schema_version": { "const": 1 }, + "receipt_header": { "$ref": "#/$defs/receipt_header" }, + "items": { + "type": "array", + "maxItems": 1000, + "items": { + "oneOf": [ + { "$ref": "#/$defs/node_item" }, + { "$ref": "#/$defs/edge_item" } + ] + } + }, + "omissions": { + "type": "array", + "maxItems": 1, + "items": { + "type": "object", + "required": ["code", "item_hash"], + "properties": { + "code": { "const": "response_limit" }, + "item_hash": { "$ref": "#/$defs/sha256" } + }, + "additionalProperties": false + } + }, + "page_hash": { "$ref": "#/$defs/sha256" } + }, + "additionalProperties": false + }, + "pagination": { + "type": "object", + "required": [ + "schema_version", + "kind", + "returned_count", + "limit", + "total_count", + "has_more", + "next_cursor" + ], + "properties": { + "schema_version": { "const": 1 }, + "kind": { "const": "generation-diff.items" }, + "returned_count": { "type": "integer", "minimum": 0 }, + "limit": { "type": "integer", "minimum": 1, "maximum": 1000 }, + "total_count": { "type": "integer", "minimum": 0, "maximum": 1000 }, + "has_more": { "type": "boolean" }, + "next_cursor": { + "type": ["string", "null"], + "minLength": 1, + "maxLength": 8192 + } + }, + "allOf": [ + { + "if": { + "properties": { "has_more": { "const": true } }, + "required": ["has_more"] + }, + "then": { + "properties": { + "next_cursor": { "type": "string", "minLength": 1 } + } + }, + "else": { + "properties": { + "next_cursor": { "type": "null" } + } + } + } + ], + "additionalProperties": false + } + }, + "type": "object", + "required": ["generation_diff", "pagination"], + "properties": { + "generation_diff": { "$ref": "#/$defs/page_body" }, + "pagination": { "$ref": "#/$defs/pagination" } + }, + "additionalProperties": false +} diff --git a/schemas/generation-diff.schema.json b/schemas/generation-diff.schema.json new file mode 100644 index 0000000..d4c05fc --- /dev/null +++ b/schemas/generation-diff.schema.json @@ -0,0 +1,363 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://docforge.local/schema/generation-diff-v1.json", + "title": "DocForge latest primary-graph generation diff receipt", + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "generation": { + "type": "object", + "required": [ + "revision", + "source_hash", + "node_count", + "node_hash", + "edge_count", + "edge_hash", + "index_schema_version" + ], + "properties": { + "revision": { "type": "string", "minLength": 1 }, + "source_hash": { "$ref": "#/$defs/sha256" }, + "node_count": { "type": "integer", "minimum": 0 }, + "node_hash": { "$ref": "#/$defs/sha256" }, + "edge_count": { "type": "integer", "minimum": 0 }, + "edge_hash": { "$ref": "#/$defs/sha256" }, + "index_schema_version": { "type": "integer", "minimum": 1 } + }, + "additionalProperties": false + }, + "node_item": { + "type": "object", + "required": [ + "entity", + "change", + "node_id", + "before_content_hash", + "after_content_hash", + "before_source_path", + "after_source_path", + "before_node_hash", + "after_node_hash", + "changed_fields", + "item_hash" + ], + "properties": { + "entity": { "const": "node" }, + "change": { "enum": ["added", "removed", "changed"] }, + "node_id": { "type": "string", "minLength": 1 }, + "before_content_hash": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "after_content_hash": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "before_source_path": { "type": ["string", "null"] }, + "after_source_path": { "type": ["string", "null"] }, + "before_node_hash": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "after_node_hash": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "changed_fields": { + "type": "array", + "items": { + "enum": [ + "title", + "family", + "authority", + "status", + "tags", + "summary", + "content", + "source_path", + "source_anchor", + "content_hash" + ] + }, + "uniqueItems": true + }, + "item_hash": { "$ref": "#/$defs/sha256" } + }, + "allOf": [ + { + "if": { + "properties": { "change": { "const": "added" } }, + "required": ["change"] + }, + "then": { + "properties": { + "before_content_hash": { "type": "null" }, + "before_source_path": { "type": "null" }, + "before_node_hash": { "type": "null" }, + "after_content_hash": { "$ref": "#/$defs/sha256" }, + "after_source_path": { "type": "string", "minLength": 1 }, + "after_node_hash": { "$ref": "#/$defs/sha256" }, + "changed_fields": { "maxItems": 0 } + } + } + }, + { + "if": { + "properties": { "change": { "const": "removed" } }, + "required": ["change"] + }, + "then": { + "properties": { + "before_content_hash": { "$ref": "#/$defs/sha256" }, + "before_source_path": { "type": "string", "minLength": 1 }, + "before_node_hash": { "$ref": "#/$defs/sha256" }, + "after_content_hash": { "type": "null" }, + "after_source_path": { "type": "null" }, + "after_node_hash": { "type": "null" }, + "changed_fields": { "maxItems": 0 } + } + } + }, + { + "if": { + "properties": { "change": { "const": "changed" } }, + "required": ["change"] + }, + "then": { + "properties": { + "before_content_hash": { "$ref": "#/$defs/sha256" }, + "before_source_path": { "type": "string", "minLength": 1 }, + "before_node_hash": { "$ref": "#/$defs/sha256" }, + "after_content_hash": { "$ref": "#/$defs/sha256" }, + "after_source_path": { "type": "string", "minLength": 1 }, + "after_node_hash": { "$ref": "#/$defs/sha256" }, + "changed_fields": { "minItems": 1 } + } + } + } + ], + "additionalProperties": false + }, + "edge_item": { + "type": "object", + "required": [ + "entity", + "change", + "source_id", + "relation", + "target_id", + "item_hash" + ], + "properties": { + "entity": { "const": "edge" }, + "change": { "enum": ["added", "removed"] }, + "source_id": { "type": "string", "minLength": 1 }, + "relation": { "type": "string", "minLength": 1 }, + "target_id": { "type": "string", "minLength": 1 }, + "item_hash": { "$ref": "#/$defs/sha256" } + }, + "additionalProperties": false + } + }, + "type": "object", + "required": [ + "schema_version", + "diff_semantics_version", + "project_id", + "project_root_fingerprint", + "adapter", + "kind", + "reason", + "from_generation", + "to_generation", + "summary", + "items", + "full_item_count", + "retained_item_count", + "details_truncated", + "truncation_reason", + "full_collection_hash", + "retained_collection_hash", + "index_signature", + "receipt_hash" + ], + "properties": { + "schema_version": { "const": 1 }, + "diff_semantics_version": { "const": 1 }, + "project_id": { "type": "string", "minLength": 1 }, + "project_root_fingerprint": { + "type": "string", + "pattern": "^[0-9a-f]{16}$" + }, + "adapter": { "type": "string", "minLength": 1 }, + "kind": { "enum": ["baseline", "transition"] }, + "reason": { + "enum": [ + null, + "no_predecessor", + "predecessor_unsafe", + "predecessor_unsupported_schema", + "predecessor_foreign", + "predecessor_policy_incompatible", + "predecessor_corrupt", + "predecessor_unattested", + "predecessor_changed", + "no_meaningful_transition" + ] + }, + "from_generation": { + "oneOf": [ + { "$ref": "#/$defs/generation" }, + { "type": "null" } + ] + }, + "to_generation": { "$ref": "#/$defs/generation" }, + "summary": { + "type": "object", + "required": [ + "nodes_added", + "nodes_removed", + "nodes_changed", + "edges_added", + "edges_removed", + "total_changes" + ], + "properties": { + "nodes_added": { "type": "integer", "minimum": 0 }, + "nodes_removed": { "type": "integer", "minimum": 0 }, + "nodes_changed": { "type": "integer", "minimum": 0 }, + "edges_added": { "type": "integer", "minimum": 0 }, + "edges_removed": { "type": "integer", "minimum": 0 }, + "total_changes": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": false + }, + "items": { + "type": "array", + "maxItems": 1000, + "items": { + "oneOf": [ + { "$ref": "#/$defs/node_item" }, + { "$ref": "#/$defs/edge_item" } + ] + } + }, + "full_item_count": { "type": "integer", "minimum": 0 }, + "retained_item_count": { + "type": "integer", + "minimum": 0, + "maximum": 1000 + }, + "details_truncated": { "type": "boolean" }, + "truncation_reason": { + "enum": [null, "receipt_item_limit", "receipt_byte_limit"] + }, + "full_collection_hash": { "$ref": "#/$defs/sha256" }, + "retained_collection_hash": { "$ref": "#/$defs/sha256" }, + "index_signature": { + "type": "object", + "required": ["device", "inode", "size", "mtime_ns", "ctime_ns"], + "properties": { + "device": { "type": "integer", "minimum": 0 }, + "inode": { "type": "integer", "minimum": 0 }, + "size": { "type": "integer", "minimum": 0 }, + "mtime_ns": { "type": "integer", "minimum": 0 }, + "ctime_ns": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": false + }, + "receipt_hash": { "$ref": "#/$defs/sha256" } + }, + "allOf": [ + { + "if": { + "properties": { "kind": { "const": "baseline" } }, + "required": ["kind"] + }, + "then": { + "properties": { + "from_generation": { "type": "null" }, + "reason": { "not": { "type": "null" } }, + "summary": { + "properties": { + "nodes_added": { "const": 0 }, + "nodes_removed": { "const": 0 }, + "nodes_changed": { "const": 0 }, + "edges_added": { "const": 0 }, + "edges_removed": { "const": 0 }, + "total_changes": { "const": 0 } + } + }, + "full_item_count": { "const": 0 }, + "retained_item_count": { "const": 0 }, + "details_truncated": { "const": false }, + "truncation_reason": { "const": null }, + "full_collection_hash": { + "const": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "retained_collection_hash": { + "const": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + } + }, + "else": { + "properties": { + "from_generation": { "$ref": "#/$defs/generation" }, + "reason": { "type": "null" } + } + } + }, + { + "if": { + "properties": { "truncation_reason": { "const": null } }, + "required": ["truncation_reason"] + }, + "then": { + "properties": { + "details_truncated": { "const": false }, + "full_item_count": { "maximum": 1000 } + } + } + }, + { + "if": { + "properties": { + "truncation_reason": { "const": "receipt_item_limit" } + }, + "required": ["truncation_reason"] + }, + "then": { + "properties": { + "details_truncated": { "const": true }, + "full_item_count": { "minimum": 1001 }, + "retained_item_count": { "const": 1000 } + } + } + }, + { + "if": { + "properties": { + "truncation_reason": { "const": "receipt_byte_limit" } + }, + "required": ["truncation_reason"] + }, + "then": { + "properties": { + "details_truncated": { "const": true } + } + } + } + ], + "additionalProperties": false +} diff --git a/schemas/result.schema.json b/schemas/result.schema.json index 2ea58f2..0a8057e 100644 --- a/schemas/result.schema.json +++ b/schemas/result.schema.json @@ -33,6 +33,7 @@ "mcp.impact", "mcp.context", "mcp.task_context", + "mcp.generation_diff", "mcp.validate_project", "mcp.render_status", "mcp.visualize", @@ -55,6 +56,7 @@ "cli.dependencies", "cli.impact", "cli.context", + "cli.generation-diff", "cli.render", "cli.render-status", "cli.preview", @@ -151,6 +153,7 @@ "enum": [ "context.items", "task-context.items", + "generation-diff.items", "changeset.list", "changeset.inspect", "changeset.validate", @@ -216,7 +219,18 @@ "properties": { "code": { "type": "string", "minLength": 1 }, "message": { "type": "string", "minLength": 1 }, - "details": { "type": "object" } + "details": { "type": "object" }, + "remediation": { + "type": "object", + "required": ["retryable"], + "properties": { + "retryable": { "type": "boolean" }, + "action": { "type": "string", "minLength": 1 }, + "tool": { "type": "string", "minLength": 1 }, + "arguments": { "type": "object" } + }, + "additionalProperties": false + } }, "additionalProperties": false } diff --git a/src/docforge/_fs_safety.py b/src/docforge/_fs_safety.py new file mode 100644 index 0000000..e2c8c94 --- /dev/null +++ b/src/docforge/_fs_safety.py @@ -0,0 +1,71 @@ +"""Internal directory binding helpers for disposable publication paths.""" + +from __future__ import annotations + +import os +import stat +from pathlib import Path + +from .errors import DocForgeError + + +def open_bound_directory(path: Path) -> int: + """Open one real directory and bind its current inode for later operations.""" + + try: + path_status = path.lstat() + if ( + stat.S_ISLNK(path_status.st_mode) + or not stat.S_ISDIR(path_status.st_mode) + or path.resolve(strict=True) != path + ): + raise DocForgeError( + "path_escape", + "Derived cache root is not a safe real directory", + ) + directory_fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + except FileNotFoundError as error: + raise DocForgeError( + "missing_index", + "Derived cache root does not exist", + ) from error + except OSError as error: + raise DocForgeError( + "path_escape", + "Derived cache root cannot be opened safely", + ) from error + try: + opened_status = os.fstat(directory_fd) + if opened_status.st_dev != path_status.st_dev or opened_status.st_ino != path_status.st_ino: + raise DocForgeError( + "path_escape", + "Derived cache root changed while opening", + ) + except Exception: + os.close(directory_fd) + raise + return directory_fd + + +def require_bound_directory(path: Path, directory_fd: int) -> None: + """Require a path to still name the exact opened real directory.""" + + try: + path_status = path.lstat() + opened_status = os.fstat(directory_fd) + if ( + stat.S_ISLNK(path_status.st_mode) + or not stat.S_ISDIR(path_status.st_mode) + or path.resolve(strict=True) != path + or opened_status.st_dev != path_status.st_dev + or opened_status.st_ino != path_status.st_ino + ): + raise DocForgeError( + "path_escape", + "Derived cache root changed during publication", + ) + except FileNotFoundError as error: + raise DocForgeError( + "path_escape", + "Derived cache root disappeared during publication", + ) from error diff --git a/src/docforge/cli.py b/src/docforge/cli.py index ef4d749..03e1f62 100644 --- a/src/docforge/cli.py +++ b/src/docforge/cli.py @@ -65,6 +65,9 @@ def _parser() -> argparse.ArgumentParser: context.add_argument("--budget", type=int) context.add_argument("--limit", type=int) context.add_argument("--cursor") + generation_diff = commands.add_parser("generation-diff") + generation_diff.add_argument("--limit", type=int) + generation_diff.add_argument("--cursor") render = commands.add_parser("render") render.add_argument("view_id") render_status = commands.add_parser("render-status") @@ -187,6 +190,16 @@ def _run(arguments: argparse.Namespace) -> dict[str, object]: cursor=arguments.cursor, ) return compile_context(index, arguments.profile, arguments.budget) + if arguments.command == "generation-diff": + from .mcp_server import DocForgeService + + return DocForgeService( + project, + capability_mode_name="read", + ).generation_diff( + limit=arguments.limit, + cursor=arguments.cursor, + ) if arguments.command == "render": return RenderService(project).render(arguments.view_id) if arguments.command == "render-status": diff --git a/src/docforge/generation_diff.py b/src/docforge/generation_diff.py new file mode 100644 index 0000000..4a76a8d --- /dev/null +++ b/src/docforge/generation_diff.py @@ -0,0 +1,911 @@ +"""Bounded latest-generation transition receipts for disposable graph indexes.""" + +from __future__ import annotations + +import json +import os +import secrets +import stat +from collections.abc import Mapping +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, cast + +from ._fs_safety import open_bound_directory, require_bound_directory +from .errors import DocForgeError +from .models import Edge, Node, ProjectDescriptor, ProjectSnapshot +from .pagination import canonical_hash + +GENERATION_DIFF_SCHEMA_VERSION = 1 +GENERATION_DIFF_SEMANTICS_VERSION = 1 +MAX_GENERATION_DIFF_ITEMS = 1_000 +MAX_GENERATION_DIFF_BYTES = 1_048_576 +GENERATION_DIFF_FILENAME = "generation-diff.json" + +IndexSignature = tuple[int, int, int, int, int] +PredecessorReason = Literal[ + "no_predecessor", + "predecessor_unsafe", + "predecessor_unsupported_schema", + "predecessor_foreign", + "predecessor_policy_incompatible", + "predecessor_corrupt", + "predecessor_unattested", + "predecessor_changed", + "no_meaningful_transition", +] + +_GENERATION_KEYS = frozenset( + { + "revision", + "source_hash", + "node_count", + "node_hash", + "edge_count", + "edge_hash", + "index_schema_version", + } +) +_SUMMARY_KEYS = frozenset( + { + "nodes_added", + "nodes_removed", + "nodes_changed", + "edges_added", + "edges_removed", + "total_changes", + } +) +_SIGNATURE_KEYS = frozenset({"device", "inode", "size", "mtime_ns", "ctime_ns"}) +_RECEIPT_KEYS = frozenset( + { + "schema_version", + "diff_semantics_version", + "project_id", + "project_root_fingerprint", + "adapter", + "kind", + "reason", + "from_generation", + "to_generation", + "summary", + "items", + "full_item_count", + "retained_item_count", + "details_truncated", + "truncation_reason", + "full_collection_hash", + "retained_collection_hash", + "index_signature", + "receipt_hash", + } +) +_BASELINE_REASONS = frozenset( + { + "no_predecessor", + "predecessor_unsafe", + "predecessor_unsupported_schema", + "predecessor_foreign", + "predecessor_policy_incompatible", + "predecessor_corrupt", + "predecessor_unattested", + "predecessor_changed", + "no_meaningful_transition", + } +) +_NODE_ITEM_KEYS = frozenset( + { + "entity", + "change", + "node_id", + "before_content_hash", + "after_content_hash", + "before_source_path", + "after_source_path", + "before_node_hash", + "after_node_hash", + "changed_fields", + "item_hash", + } +) +_NODE_CHANGED_FIELDS = frozenset( + { + "title", + "family", + "authority", + "status", + "tags", + "summary", + "content", + "source_path", + "source_anchor", + "content_hash", + } +) +_EDGE_ITEM_KEYS = frozenset( + { + "entity", + "change", + "source_id", + "relation", + "target_id", + "item_hash", + } +) + + +@dataclass(frozen=True) +class PublishedGraph: + """One completely verified predecessor publication.""" + + generation: dict[str, object] + nodes: tuple[Node, ...] + edges: tuple[Edge, ...] + logic_hash: str + signature: IndexSignature + + +@dataclass(frozen=True) +class GenerationDiffDraft: + """A receipt body awaiting the committed index file identity.""" + + fields: dict[str, object] + items: tuple[dict[str, object], ...] + + +def generation_diff_path(descriptor: ProjectDescriptor) -> Path: + """Return the fixed project-confined latest-transition receipt path.""" + + return descriptor.cache_root / GENERATION_DIFF_FILENAME + + +def index_signature(path: Path) -> IndexSignature: + """Return the exact identity of one safe regular index publication.""" + + try: + status = path.lstat() + except OSError as error: + raise DocForgeError("missing_index", "Derived index does not exist") from error + if stat.S_ISLNK(status.st_mode) or not stat.S_ISREG(status.st_mode): + raise DocForgeError("path_escape", "Derived index path is not a safe regular file") + return ( + status.st_dev, + status.st_ino, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + + +def signature_payload(signature: IndexSignature) -> dict[str, int]: + """Convert one stat identity to its versioned JSON representation.""" + + return { + "device": signature[0], + "inode": signature[1], + "size": signature[2], + "mtime_ns": signature[3], + "ctime_ns": signature[4], + } + + +def generation_identity(status: Mapping[str, object]) -> dict[str, object]: + """Select the primary-graph identity stored in a public diff receipt.""" + + return { + "revision": status["revision"], + "source_hash": status["source_hash"], + "node_count": status["node_count"], + "node_hash": status["node_hash"], + "edge_count": status["edge_count"], + "edge_hash": status["edge_hash"], + "index_schema_version": status["index_schema_version"], + } + + +def prepare_generation_diff( + descriptor: ProjectDescriptor, + *, + predecessor: PublishedGraph | None, + predecessor_reason: PredecessorReason | None, + current_snapshot: ProjectSnapshot, + current_status: Mapping[str, object], + preserved_receipt: Mapping[str, object] | None = None, +) -> GenerationDiffDraft: + """Prepare one deterministic latest transition without publishing it.""" + + current_generation = generation_identity(current_status) + if predecessor is None: + return _baseline_draft( + descriptor, + current_generation, + predecessor_reason or "no_predecessor", + ) + + if predecessor.generation == current_generation: + if preserved_receipt is not None and _receipt_targets( + preserved_receipt, + descriptor, + current_generation, + predecessor.signature, + ): + fields = { + key: value + for key, value in preserved_receipt.items() + if key not in {"index_signature", "receipt_hash", "items"} + } + preserved_items = preserved_receipt.get("items") + if not isinstance(preserved_items, list): + raise DocForgeError( + "invalid_generation_diff", + "Preserved generation diff has no item collection", + ) + items = tuple( + cast(dict[str, object], item) + for item in cast(list[object], preserved_items) + if isinstance(item, dict) + ) + return GenerationDiffDraft(fields=fields, items=items) + return _baseline_draft( + descriptor, + current_generation, + "no_meaningful_transition", + ) + + items = _change_items( + predecessor.nodes, + predecessor.edges, + current_snapshot.nodes, + current_snapshot.edges, + ) + summary = _summary(items) + return GenerationDiffDraft( + fields={ + "schema_version": GENERATION_DIFF_SCHEMA_VERSION, + "diff_semantics_version": GENERATION_DIFF_SEMANTICS_VERSION, + "project_id": descriptor.project_id, + "project_root_fingerprint": _root_fingerprint(descriptor), + "adapter": descriptor.adapter, + "kind": "transition", + "reason": None, + "from_generation": predecessor.generation, + "to_generation": current_generation, + "summary": summary, + "full_item_count": len(items), + "full_collection_hash": canonical_hash([item["item_hash"] for item in items]), + }, + items=items, + ) + + +def finalize_generation_diff( + draft: GenerationDiffDraft, + *, + signature: IndexSignature, +) -> dict[str, object]: + """Bind a draft to the committed index and enforce fixed receipt ceilings.""" + + full_count = cast(int, draft.fields["full_item_count"]) + retained = list(draft.items[:MAX_GENERATION_DIFF_ITEMS]) + item_limited = full_count > len(retained) + byte_limited = draft.fields.get("truncation_reason") == "receipt_byte_limit" + while True: + receipt = _final_receipt( + draft.fields, + retained, + signature=signature, + item_limited=item_limited, + byte_limited=byte_limited, + ) + if len(_receipt_bytes(receipt)) <= MAX_GENERATION_DIFF_BYTES: + return receipt + if not retained: + raise DocForgeError( + "generation_diff_failure", + "Generation diff identity exceeds the fixed receipt byte limit", + ) + retained.pop() + byte_limited = True + + +def publish_generation_diff( + descriptor: ProjectDescriptor, + receipt: Mapping[str, object], +) -> None: + """Atomically replace the single latest-generation receipt.""" + + root = descriptor.cache_root + if generation_diff_path(descriptor).parent != root: + raise DocForgeError("path_escape", "Generation diff path is not confined") + raw = _receipt_bytes(receipt) + if len(raw) > MAX_GENERATION_DIFF_BYTES: + raise DocForgeError( + "generation_diff_failure", + "Generation diff receipt exceeds the fixed byte limit", + ) + root_fd = open_bound_directory(root) + temporary_name = f".generation-diff-{secrets.token_hex(12)}" + temporary_created = False + try: + try: + status = os.stat( + GENERATION_DIFF_FILENAME, + dir_fd=root_fd, + follow_symlinks=False, + ) + except FileNotFoundError: + status = None + if status is not None and ( + stat.S_ISLNK(status.st_mode) or not stat.S_ISREG(status.st_mode) + ): + raise DocForgeError( + "path_escape", + "Generation diff receipt path is not a safe regular file", + ) + descriptor_fd = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=root_fd, + ) + temporary_created = True + with os.fdopen(descriptor_fd, "wb") as handle: + handle.write(raw) + handle.flush() + os.fsync(handle.fileno()) + require_bound_directory(root, root_fd) + os.replace( + temporary_name, + GENERATION_DIFF_FILENAME, + src_dir_fd=root_fd, + dst_dir_fd=root_fd, + ) + temporary_created = False + os.fsync(root_fd) + require_bound_directory(root, root_fd) + except Exception: + if temporary_created: + with suppress(OSError): + os.unlink(temporary_name, dir_fd=root_fd) + raise + finally: + os.close(root_fd) + + +def load_generation_diff( + descriptor: ProjectDescriptor, +) -> tuple[dict[str, object] | None, str | None, IndexSignature | None]: + """Read and validate one receipt without repairing any derived state.""" + + try: + root_fd = open_bound_directory(descriptor.cache_root) + except DocForgeError as error: + reason = "missing_receipt" if error.code == "missing_index" else "unsafe_receipt" + return None, reason, None + try: + try: + descriptor_fd = os.open( + GENERATION_DIFF_FILENAME, + os.O_RDONLY | os.O_NOFOLLOW, + dir_fd=root_fd, + ) + except FileNotFoundError: + return None, "missing_receipt", None + except OSError: + return None, "unsafe_receipt", None + with os.fdopen(descriptor_fd, "rb") as handle: + status_before = os.fstat(handle.fileno()) + if stat.S_ISLNK(status_before.st_mode) or not stat.S_ISREG(status_before.st_mode): + return None, "unsafe_receipt", None + before: IndexSignature = ( + status_before.st_dev, + status_before.st_ino, + status_before.st_size, + status_before.st_mtime_ns, + status_before.st_ctime_ns, + ) + if before[2] > MAX_GENERATION_DIFF_BYTES: + return None, "oversized_receipt", before + raw = handle.read(MAX_GENERATION_DIFF_BYTES + 1) + status_after = os.fstat(handle.fileno()) + after: IndexSignature = ( + status_after.st_dev, + status_after.st_ino, + status_after.st_size, + status_after.st_mtime_ns, + status_after.st_ctime_ns, + ) + if len(raw) > MAX_GENERATION_DIFF_BYTES: + return None, "oversized_receipt", before + try: + path_status = os.stat( + GENERATION_DIFF_FILENAME, + dir_fd=root_fd, + follow_symlinks=False, + ) + except OSError: + return None, "receipt_changed", before + path_signature: IndexSignature = ( + path_status.st_dev, + path_status.st_ino, + path_status.st_size, + path_status.st_mtime_ns, + path_status.st_ctime_ns, + ) + try: + require_bound_directory(descriptor.cache_root, root_fd) + except DocForgeError: + return None, "unsafe_receipt", before + finally: + os.close(root_fd) + if before != after or before != path_signature or len(raw) != before[2]: + return None, "receipt_changed", before + try: + parsed: object = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None, "corrupt_receipt", before + if not isinstance(parsed, dict): + return None, "corrupt_receipt", before + receipt = cast(dict[str, object], parsed) + if not validate_generation_diff_receipt(receipt): + return None, "corrupt_receipt", before + if ( + receipt.get("project_id") != descriptor.project_id + or receipt.get("project_root_fingerprint") != _root_fingerprint(descriptor) + or receipt.get("adapter") != descriptor.adapter + ): + return None, "foreign_receipt", before + return receipt, None, before + + +def valid_generation_identity(value: object) -> bool: + """Return whether a primary-graph generation has the exact version-1 shape.""" + + return _valid_generation(value) + + +def validate_generation_diff_receipt( + receipt: Mapping[str, object], + *, + descriptor: ProjectDescriptor | None = None, +) -> bool: + """Strictly validate the complete version-1 receipt and its hashes.""" + + if frozenset(receipt) != _RECEIPT_KEYS: + return False + if ( + receipt.get("schema_version") != GENERATION_DIFF_SCHEMA_VERSION + or receipt.get("diff_semantics_version") != GENERATION_DIFF_SEMANTICS_VERSION + or not _nonempty(receipt.get("project_id")) + or not _fingerprint(receipt.get("project_root_fingerprint")) + or not _nonempty(receipt.get("adapter")) + ): + return False + if descriptor is not None and ( + receipt["project_id"] != descriptor.project_id + or receipt["project_root_fingerprint"] != _root_fingerprint(descriptor) + or receipt["adapter"] != descriptor.adapter + ): + return False + kind = receipt.get("kind") + reason = receipt.get("reason") + from_generation = receipt.get("from_generation") + if kind == "baseline": + if reason not in _BASELINE_REASONS or from_generation is not None: + return False + elif kind == "transition": + if ( + reason is not None + or not _valid_generation(from_generation) + or from_generation == receipt.get("to_generation") + ): + return False + else: + return False + if not _valid_generation(receipt.get("to_generation")): + return False + summary_value = receipt.get("summary") + items_value = receipt.get("items") + if not isinstance(summary_value, dict) or not isinstance(items_value, list): + return False + summary = cast(dict[str, object], summary_value) + items = cast(list[object], items_value) + if frozenset(summary) != _SUMMARY_KEYS or any( + type(value) is not int or value < 0 for value in summary.values() + ): + return False + total = sum( + cast(int, summary[key]) + for key in ( + "nodes_added", + "nodes_removed", + "nodes_changed", + "edges_added", + "edges_removed", + ) + ) + if summary.get("total_changes") != total: + return False + if kind == "baseline" and (total != 0 or items): + return False + if len(items) > MAX_GENERATION_DIFF_ITEMS or not all(_valid_item(item) for item in items): + return False + typed_items = tuple(cast(dict[str, object], item) for item in items) + if list(typed_items) != sorted(typed_items, key=_item_sort_key): + return False + identities = tuple(_item_identity(item) for item in typed_items) + if len(identities) != len(set(identities)): + return False + retained_summary = _summary(typed_items) + if any( + retained_summary[key] > cast(int, summary[key]) + for key in ( + "nodes_added", + "nodes_removed", + "nodes_changed", + "edges_added", + "edges_removed", + ) + ): + return False + full_count = receipt.get("full_item_count") + retained_count = receipt.get("retained_item_count") + truncated = receipt.get("details_truncated") + truncation_reason = receipt.get("truncation_reason") + if ( + type(full_count) is not int + or type(retained_count) is not int + or full_count != total + or retained_count != len(items) + or retained_count > full_count + or type(truncated) is not bool + or truncated != (retained_count < full_count) + ): + return False + if truncation_reason is None: + if truncated or full_count > MAX_GENERATION_DIFF_ITEMS: + return False + elif truncation_reason == "receipt_item_limit": + if ( + not truncated + or full_count <= MAX_GENERATION_DIFF_ITEMS + or retained_count != MAX_GENERATION_DIFF_ITEMS + ): + return False + elif truncation_reason == "receipt_byte_limit": + if not truncated or retained_count >= min(full_count, MAX_GENERATION_DIFF_ITEMS): + return False + else: + return False + item_hashes = [cast(dict[str, object], item)["item_hash"] for item in items] + retained_hash = canonical_hash(item_hashes) + if receipt.get("retained_collection_hash") != retained_hash: + return False + full_hash = receipt.get("full_collection_hash") + if not _sha256(full_hash): + return False + if not truncated and full_hash != retained_hash: + return False + signature_value = receipt.get("index_signature") + if not isinstance(signature_value, dict): + return False + signature = cast(dict[str, object], signature_value) + if frozenset(signature) != _SIGNATURE_KEYS: + return False + if any(type(value) is not int or value < 0 for value in signature.values()): + return False + receipt_hash = receipt.get("receipt_hash") + if not _sha256(receipt_hash): + return False + unhashed = {key: value for key, value in receipt.items() if key != "receipt_hash"} + return receipt_hash == canonical_hash(unhashed) + + +def _baseline_draft( + descriptor: ProjectDescriptor, + current_generation: Mapping[str, object], + reason: PredecessorReason, +) -> GenerationDiffDraft: + empty_hash = canonical_hash([]) + return GenerationDiffDraft( + fields={ + "schema_version": GENERATION_DIFF_SCHEMA_VERSION, + "diff_semantics_version": GENERATION_DIFF_SEMANTICS_VERSION, + "project_id": descriptor.project_id, + "project_root_fingerprint": _root_fingerprint(descriptor), + "adapter": descriptor.adapter, + "kind": "baseline", + "reason": reason, + "from_generation": None, + "to_generation": dict(current_generation), + "summary": { + "nodes_added": 0, + "nodes_removed": 0, + "nodes_changed": 0, + "edges_added": 0, + "edges_removed": 0, + "total_changes": 0, + }, + "full_item_count": 0, + "full_collection_hash": empty_hash, + }, + items=(), + ) + + +def _change_items( + before_nodes: tuple[Node, ...], + before_edges: tuple[Edge, ...], + after_nodes: tuple[Node, ...], + after_edges: tuple[Edge, ...], +) -> tuple[dict[str, object], ...]: + before_by_id = {node.node_id: node for node in before_nodes} + after_by_id = {node.node_id: node for node in after_nodes} + items: list[dict[str, object]] = [] + for node_id in sorted(before_by_id.keys() | after_by_id.keys()): + before = before_by_id.get(node_id) + after = after_by_id.get(node_id) + if before == after: + continue + if before is None: + change = "added" + elif after is None: + change = "removed" + else: + change = "changed" + before_payload = before.as_dict() if before is not None else None + after_payload = after.as_dict() if after is not None else None + changed_fields = ( + [] + if before_payload is None or after_payload is None + else sorted(key for key in before_payload if before_payload[key] != after_payload[key]) + ) + payload: dict[str, object] = { + "entity": "node", + "change": change, + "node_id": node_id, + "before_content_hash": None if before is None else before.content_hash, + "after_content_hash": None if after is None else after.content_hash, + "before_source_path": None if before is None else before.source_path, + "after_source_path": None if after is None else after.source_path, + "before_node_hash": ( + None if before_payload is None else canonical_hash(before_payload) + ), + "after_node_hash": (None if after_payload is None else canonical_hash(after_payload)), + "changed_fields": changed_fields, + } + payload["item_hash"] = canonical_hash(payload) + items.append(payload) + before_edge_set = {(edge.source_id, edge.relation, edge.target_id) for edge in before_edges} + after_edge_set = {(edge.source_id, edge.relation, edge.target_id) for edge in after_edges} + for change, values in ( + ("removed", sorted(before_edge_set - after_edge_set)), + ("added", sorted(after_edge_set - before_edge_set)), + ): + for source_id, relation, target_id in values: + payload = { + "entity": "edge", + "change": change, + "source_id": source_id, + "relation": relation, + "target_id": target_id, + } + payload["item_hash"] = canonical_hash(payload) + items.append(payload) + return tuple(sorted(items, key=_item_sort_key)) + + +def _item_sort_key(item: Mapping[str, object]) -> tuple[str, str, str, str, str]: + return ( + cast(str, item["entity"]), + cast(str, item.get("node_id", item.get("source_id", ""))), + cast(str, item.get("relation", "")), + cast(str, item.get("target_id", "")), + cast(str, item["change"]), + ) + + +def _item_identity(item: Mapping[str, object]) -> tuple[str, ...]: + if item["entity"] == "node": + return ("node", cast(str, item["node_id"])) + return ( + "edge", + cast(str, item["source_id"]), + cast(str, item["relation"]), + cast(str, item["target_id"]), + ) + + +def _summary(items: tuple[dict[str, object], ...]) -> dict[str, int]: + result = { + "nodes_added": 0, + "nodes_removed": 0, + "nodes_changed": 0, + "edges_added": 0, + "edges_removed": 0, + "total_changes": len(items), + } + for item in items: + entity = cast(str, item["entity"]) + change = cast(str, item["change"]) + key = f"{entity}s_{change}" + result[key] += 1 + return result + + +def _final_receipt( + fields: Mapping[str, object], + retained: list[dict[str, object]], + *, + signature: IndexSignature, + item_limited: bool, + byte_limited: bool, +) -> dict[str, object]: + full_count = cast(int, fields["full_item_count"]) + truncated = len(retained) < full_count + if byte_limited: + reason: str | None = "receipt_byte_limit" + elif item_limited: + reason = "receipt_item_limit" + else: + reason = None + receipt = { + **fields, + "items": retained, + "retained_item_count": len(retained), + "details_truncated": truncated, + "truncation_reason": reason, + "retained_collection_hash": canonical_hash([item["item_hash"] for item in retained]), + "index_signature": signature_payload(signature), + } + receipt["receipt_hash"] = canonical_hash(receipt) + return receipt + + +def _receipt_targets( + receipt: Mapping[str, object], + descriptor: ProjectDescriptor, + generation: Mapping[str, object], + signature: IndexSignature, +) -> bool: + return ( + validate_generation_diff_receipt(receipt, descriptor=descriptor) + and receipt.get("to_generation") == dict(generation) + and receipt.get("index_signature") == signature_payload(signature) + ) + + +def _valid_generation(value: object) -> bool: + if not isinstance(value, dict): + return False + generation = cast(dict[str, object], value) + if frozenset(generation) != _GENERATION_KEYS: + return False + source_hash = generation.get("source_hash") + node_hash = generation.get("node_hash") + edge_hash = generation.get("edge_hash") + return ( + _nonempty(generation.get("revision")) + and _sha256(source_hash) + and _sha256(node_hash) + and _sha256(edge_hash) + and type(generation.get("node_count")) is int + and cast(int, generation["node_count"]) >= 0 + and type(generation.get("edge_count")) is int + and cast(int, generation["edge_count"]) >= 0 + and type(generation.get("index_schema_version")) is int + and cast(int, generation["index_schema_version"]) >= 1 + ) + + +def _valid_item(value: object) -> bool: + if not isinstance(value, dict): + return False + payload = cast(dict[str, object], value) + entity = payload.get("entity") + if entity == "node": + if frozenset(payload) != _NODE_ITEM_KEYS: + return False + changed_fields_value = payload.get("changed_fields") + if not isinstance(changed_fields_value, list): + return False + changed_fields = cast(list[object], changed_fields_value) + if ( + payload.get("change") not in {"added", "removed", "changed"} + or not _nonempty(payload.get("node_id")) + or any( + not _nonempty(field) or field not in _NODE_CHANGED_FIELDS + for field in changed_fields + ) + or changed_fields != sorted(set(cast(list[str], changed_fields))) + ): + return False + for key in ( + "before_content_hash", + "after_content_hash", + "before_node_hash", + "after_node_hash", + ): + candidate = payload.get(key) + if candidate is not None and not _sha256(candidate): + return False + for key in ("before_source_path", "after_source_path"): + candidate = payload.get(key) + if candidate is not None and not _nonempty(candidate): + return False + change = payload["change"] + before_values = ( + payload["before_content_hash"], + payload["before_source_path"], + payload["before_node_hash"], + ) + after_values = ( + payload["after_content_hash"], + payload["after_source_path"], + payload["after_node_hash"], + ) + if change == "added": + if ( + any(value is not None for value in before_values) + or not all(value is not None for value in after_values) + or changed_fields + ): + return False + elif change == "removed": + if ( + not all(value is not None for value in before_values) + or any(value is not None for value in after_values) + or changed_fields + ): + return False + elif ( + not all(value is not None for value in (*before_values, *after_values)) + or not changed_fields + or payload["before_node_hash"] == payload["after_node_hash"] + ): + return False + elif entity == "edge": + if ( + frozenset(payload) != _EDGE_ITEM_KEYS + or payload.get("change") not in {"added", "removed"} + or not _nonempty(payload.get("source_id")) + or not _nonempty(payload.get("relation")) + or not _nonempty(payload.get("target_id")) + ): + return False + else: + return False + item_hash = payload.get("item_hash") + unhashed = {key: item for key, item in payload.items() if key != "item_hash"} + return _sha256(item_hash) and item_hash == canonical_hash(unhashed) + + +def _receipt_bytes(receipt: Mapping[str, object]) -> bytes: + return json.dumps(receipt, sort_keys=True, indent=2, ensure_ascii=True).encode("utf-8") + b"\n" + + +def _root_fingerprint(descriptor: ProjectDescriptor) -> str: + from .project import project_root_fingerprint + + return project_root_fingerprint(descriptor.root) + + +def _nonempty(value: object) -> bool: + return isinstance(value, str) and bool(value) + + +def _sha256(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _fingerprint(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 16 + and all(character in "0123456789abcdef" for character in value) + ) diff --git a/src/docforge/index.py b/src/docforge/index.py index c1f237b..60647df 100644 --- a/src/docforge/index.py +++ b/src/docforge/index.py @@ -6,17 +6,33 @@ import fcntl import hashlib import json import os +import secrets import sqlite3 -import tempfile +import stat import time from collections import deque from collections.abc import Callable, Generator -from contextlib import contextmanager +from contextlib import contextmanager, suppress from dataclasses import dataclass from pathlib import Path from typing import Literal, cast +from ._fs_safety import open_bound_directory, require_bound_directory from .errors import DocForgeError +from .generation_diff import ( + GenerationDiffDraft, + PredecessorReason, + PublishedGraph, + finalize_generation_diff, + generation_diff_path, + generation_identity, + index_signature, + load_generation_diff, + prepare_generation_diff, + publish_generation_diff, + valid_generation_identity, + validate_generation_diff_receipt, +) from .models import ( BuildReportingProject, Edge, @@ -76,19 +92,27 @@ def _logic_hash(projections: tuple[LogicProjection, ...]) -> str: return hashlib.sha256(payload).hexdigest() -def _connect_read_only(path: Path) -> sqlite3.Connection: +def _connect_read_only(path: Path, *, immutable: bool = False) -> sqlite3.Connection: if not path.is_file(): raise DocForgeError("missing_index", "Derived index does not exist; run build first") - connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + immutable_parameter = "&immutable=1" if immutable else "" + connection = sqlite3.connect( + f"file:{path}?mode=ro{immutable_parameter}", + uri=True, + ) connection.row_factory = sqlite3.Row return connection @contextmanager -def _read_connection(path: Path) -> Generator[sqlite3.Connection, None, None]: +def _read_connection( + path: Path, + *, + immutable: bool = False, +) -> Generator[sqlite3.Connection, None, None]: connection: sqlite3.Connection | None = None try: - connection = _connect_read_only(path) + connection = _connect_read_only(path, immutable=immutable) yield connection except DocForgeError: raise @@ -259,9 +283,24 @@ class ProjectIndex: return self._build_locked_core() def _build_locked_core(self) -> dict[str, object]: + predecessor, predecessor_reason, predecessor_signature = self._capture_predecessor() + if predecessor_reason == "predecessor_unsafe": + raise DocForgeError( + "path_escape", + "Derived predecessor is not a safe regular file", + ) + preserved_receipt, _, _ = load_generation_diff(self.project.descriptor) snapshot = self.project.load() logic = self._logic_projections() status = _status(snapshot, logic) + draft = prepare_generation_diff( + snapshot.descriptor, + predecessor=predecessor, + predecessor_reason=predecessor_reason, + current_snapshot=snapshot, + current_status=status, + preserved_receipt=preserved_receipt, + ) build_report = ( self.project.build_report() if isinstance(self.project, BuildReportingProject) else None ) @@ -269,10 +308,18 @@ class ProjectIndex: build_report = None cache_root = snapshot.descriptor.cache_root cache_root.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile( - prefix="index-", suffix=".sqlite3", dir=cache_root, delete=False - ) as descriptor: - temporary = Path(descriptor.name) + if self.path.parent != cache_root: + raise DocForgeError("path_escape", "Derived index path is not confined") + cache_root_fd = open_bound_directory(cache_root) + temporary_name = f"index-{secrets.token_hex(12)}.sqlite3" + temporary_descriptor = os.open( + temporary_name, + os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=cache_root_fd, + ) + os.close(temporary_descriptor) + temporary = Path(f"/proc/self/fd/{cache_root_fd}/{temporary_name}") try: connection = sqlite3.connect(temporary) try: @@ -420,25 +467,367 @@ class ProjectIndex: if ( current.source_hash != snapshot.source_hash or current.revision != snapshot.revision + or current.nodes != snapshot.nodes + or current.edges != snapshot.edges or current_logic != logic ): raise DocForgeError("source_changed", "Canonical source changed during index build") - os.replace(temporary, self.path) - self._verified_index_signature = self._index_signature() - self._write_attestation() - if isinstance(self.project, GenerationRecordingProject): - self.project.record_generation(current) + if ( + predecessor is not None + and predecessor.generation["source_hash"] == status["source_hash"] + and predecessor.generation["revision"] == status["revision"] + and ( + predecessor.generation != generation_identity(status) + or predecessor.logic_hash != status["logic_hash"] + ) + ): + raise DocForgeError( + "generation_collision", + "One canonical generation produced different graph content", + ) + if predecessor is None: + if predecessor_signature is None: + if self.path.exists() or self.path.is_symlink(): + raise DocForgeError( + "source_changed", + "Derived index appeared during index build", + ) + elif index_signature(self.path) != predecessor_signature: + raise DocForgeError( + "source_changed", + "Derived predecessor changed during index build", + ) + elif index_signature(self.path) != predecessor.signature: + raise DocForgeError( + "source_changed", + "Derived predecessor changed during index build", + ) + self._require_no_index_sidecars() + require_bound_directory(cache_root, cache_root_fd) + try: + os.replace( + temporary_name, + self.path.name, + src_dir_fd=cache_root_fd, + dst_dir_fd=cache_root_fd, + ) + except OSError as error: + raise DocForgeError( + "index_failure", + "Could not publish the derived index", + ) from error + publication_errors: list[dict[str, object]] = [] + try: + self._fsync_cache_directory(cache_root_fd) + except Exception as error: + publication_errors.append( + { + "stage": "index_directory_sync", + "error_type": type(error).__name__, + } + ) + try: + published_signature = self._published_index_signature(cache_root_fd) + except Exception as error: + publication_errors.append( + { + "stage": "index_identity", + "error_type": type(error).__name__, + } + ) + self._verified_index_signature = None + publication = self._unavailable_post_commit_publication(publication_errors) + else: + self._verified_index_signature = published_signature + try: + publication = self._publish_post_commit_receipts( + current, + draft, + signature=published_signature, + initial_errors=publication_errors, + ) + except Exception as error: + publication_errors.append( + { + "stage": "receipt_pipeline", + "error_type": type(error).__name__, + } + ) + publication = self._unavailable_post_commit_publication(publication_errors) except sqlite3.Error as error: - temporary.unlink(missing_ok=True) + with suppress(OSError): + os.unlink(temporary_name, dir_fd=cache_root_fd) raise DocForgeError("index_failure", "Could not build the derived index") from error except Exception: - temporary.unlink(missing_ok=True) + with suppress(OSError): + os.unlink(temporary_name, dir_fd=cache_root_fd) raise + finally: + os.close(cache_root_fd) result: dict[str, object] = {**status, "database": str(self.path)} if build_report is not None: result["build"] = build_report + if publication["state"] == "degraded": + result["publication"] = publication return result + def _capture_predecessor( + self, + ) -> tuple[PublishedGraph | None, PredecessorReason, tuple[int, int, int, int, int] | None]: + """Read one predecessor completely without repairing any derived receipt.""" + + if not self.path.exists() and not self.path.is_symlink(): + return None, "no_predecessor", None + self._require_no_index_sidecars() + try: + signature = index_signature(self.path) + except DocForgeError: + return None, "predecessor_unsafe", None + if not self._attestation_matches(expected_signature=signature): + return None, "predecessor_unattested", signature + try: + with _read_connection(self.path, immutable=True) as connection: + application_id = connection.execute("PRAGMA application_id").fetchone()[0] + schema_version = connection.execute("PRAGMA user_version").fetchone()[0] + if application_id != APPLICATION_ID or schema_version != INDEX_SCHEMA_VERSION: + return None, "predecessor_unsupported_schema", signature + metadata = dict(connection.execute("SELECT key, value FROM metadata")) + descriptor = self.project.descriptor + identity = { + "project_id": descriptor.project_id, + "project_root_fingerprint": project_root_fingerprint(descriptor.root), + "adapter": descriptor.adapter, + "index_schema_version": str(INDEX_SCHEMA_VERSION), + } + if ( + any(metadata.get(key) != value for key, value in identity.items()) + or metadata.get("status") != "ok" + ): + return None, "predecessor_foreign", signature + integrity = connection.execute("PRAGMA integrity_check").fetchone() + if integrity is None or integrity[0] != "ok": + return None, "predecessor_corrupt", signature + nodes = tuple( + _row_to_node(row) + for row in connection.execute("SELECT * FROM nodes ORDER BY node_id") + ) + edges = tuple( + Edge(*row) + for row in connection.execute( + "SELECT source_id, relation, target_id FROM edges " + "ORDER BY source_id, relation, target_id" + ) + ) + logic = _logic_from_connection(connection) + fts_count = connection.execute("SELECT COUNT(*) FROM node_fts").fetchone()[0] + node_hash = _node_hash(nodes) + edge_hash = _edge_hash(edges) + logic_hash = _logic_hash(logic) + logic_node_count = sum(len(projection.nodes) for projection in logic) + logic_edge_count = sum(len(projection.edges) for projection in logic) + if ( + metadata.get("node_hash") != node_hash + or metadata.get("node_count") != str(len(nodes)) + or metadata.get("edge_hash") != edge_hash + or metadata.get("edge_count") != str(len(edges)) + or metadata.get("logic_hash") != logic_hash + or metadata.get("logic_projection_count") != str(len(logic)) + or metadata.get("logic_node_count") != str(logic_node_count) + or metadata.get("logic_edge_count") != str(logic_edge_count) + or fts_count != len(nodes) + ): + return None, "predecessor_corrupt", signature + if logic and not self.allow_logic: + return None, "predecessor_policy_incompatible", signature + generation: dict[str, object] = { + "revision": metadata["revision"], + "source_hash": metadata["source_hash"], + "node_count": len(nodes), + "node_hash": node_hash, + "edge_count": len(edges), + "edge_hash": edge_hash, + "index_schema_version": INDEX_SCHEMA_VERSION, + } + if not valid_generation_identity(generation): + return None, "predecessor_corrupt", signature + except ( + DocForgeError, + sqlite3.Error, + json.JSONDecodeError, + KeyError, + TypeError, + ValueError, + ): + return None, "predecessor_corrupt", signature + self._require_no_index_sidecars() + try: + if index_signature(self.path) != signature: + return None, "predecessor_changed", signature + except DocForgeError: + return None, "predecessor_changed", signature + return ( + PublishedGraph( + generation=generation, + nodes=nodes, + edges=edges, + logic_hash=logic_hash, + signature=signature, + ), + "no_predecessor", + signature, + ) + + def _publish_post_commit_receipts( + self, + snapshot: ProjectSnapshot, + draft: GenerationDiffDraft, + *, + signature: tuple[int, int, int, int, int], + initial_errors: list[dict[str, object]] | None = None, + ) -> dict[str, object]: + """Publish independent evidence without misreporting a committed index.""" + + receipts: dict[str, dict[str, object]] = {} + errors = list(initial_errors or ()) + + def attempt(name: str, operation: Callable[[], None]) -> None: + try: + if self._index_signature() != signature: + raise DocForgeError( + "invalid_index", + "Committed index changed before receipt publication", + ) + operation() + if self._index_signature() != signature: + raise DocForgeError( + "invalid_index", + "Committed index changed during receipt publication", + ) + except Exception as error: + receipts[name] = {"state": "unavailable", "reason": "publication_failed"} + errors.append( + { + "stage": name, + "error_type": type(error).__name__, + } + ) + else: + receipts[name] = {"state": "published"} + + attempt("attestation", lambda: self._write_attestation(expected_signature=signature)) + if isinstance(self.project, GenerationRecordingProject): + record_generation = self.project.record_generation + attempt("source_generation", lambda: record_generation(snapshot)) + else: + receipts["source_generation"] = { + "state": "unavailable", + "reason": "project_does_not_record_generation", + } + + finalized: dict[str, object] | None = None + + def publish_diff() -> None: + nonlocal finalized + finalized = finalize_generation_diff(draft, signature=signature) + if not validate_generation_diff_receipt( + finalized, + descriptor=self.project.descriptor, + ): + raise DocForgeError( + "invalid_generation_diff", + "Finalized generation diff failed its publication contract", + ) + publish_generation_diff(self.project.descriptor, finalized) + + attempt("generation_diff", publish_diff) + if finalized is not None and receipts["generation_diff"]["state"] == "published": + receipts["generation_diff"].update( + { + "kind": finalized["kind"], + "receipt_hash": finalized["receipt_hash"], + "full_item_count": finalized["full_item_count"], + "retained_item_count": finalized["retained_item_count"], + "details_truncated": finalized["details_truncated"], + } + ) + return { + "state": "degraded" if errors else "published", + "index": "published", + "receipts": receipts, + "errors": errors, + } + + @staticmethod + def _unavailable_post_commit_publication( + errors: list[dict[str, object]], + ) -> dict[str, object]: + return { + "state": "degraded", + "index": "published", + "receipts": { + name: { + "state": "unavailable", + "reason": "index_identity_unavailable", + } + for name in ("attestation", "source_generation", "generation_diff") + }, + "errors": errors, + } + + def _require_no_index_sidecars(self) -> None: + """Refuse SQLite state that is not contained in the main index inode.""" + + for suffix in ("-wal", "-journal", "-shm"): + sidecar = Path(f"{self.path}{suffix}") + try: + sidecar.lstat() + except FileNotFoundError: + continue + raise DocForgeError( + "index_busy", + "Derived index has active or unproven SQLite sidecar state", + sidecar=suffix, + ) + + def _published_index_signature( + self, + cache_root_fd: int, + ) -> tuple[int, int, int, int, int]: + require_bound_directory(self.project.descriptor.cache_root, cache_root_fd) + try: + status = os.stat( + self.path.name, + dir_fd=cache_root_fd, + follow_symlinks=False, + ) + except OSError as error: + raise DocForgeError( + "missing_index", + "Committed index identity is unavailable", + ) from error + if not stat.S_ISREG(status.st_mode) or stat.S_ISLNK(status.st_mode): + raise DocForgeError( + "path_escape", + "Committed index is not a safe regular file", + ) + signature = ( + status.st_dev, + status.st_ino, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + if index_signature(self.path) != signature: + raise DocForgeError( + "invalid_index", + "Committed index path is not bound to its publication inode", + ) + return signature + + @staticmethod + def _fsync_cache_directory(cache_root_fd: int) -> None: + os.fsync(cache_root_fd) + @contextmanager def _build_lock(self) -> Generator[None, None, None]: cache_root = self.project.descriptor.cache_root @@ -464,6 +853,7 @@ class ProjectIndex: candidates = ( *self.project.descriptor.cache_root.glob("index-*.sqlite3"), *self.project.descriptor.cache_root.glob(".index-attestation-*"), + *self.project.descriptor.cache_root.glob(".generation-diff-*"), ) for path in sorted(candidates): if path == self.path or path.is_symlink() or not path.is_file(): @@ -740,15 +1130,49 @@ class ProjectIndex: "database": str(self.path), } - def _attestation_matches(self) -> bool: + def _attestation_matches( + self, + *, + expected_signature: tuple[int, int, int, int, int] | None = None, + ) -> bool: """Verify a persisted whole-file digest before trusting a warm derived index.""" path = self.attestation_path - if not path.is_file() or path.is_symlink(): + try: + before_path = index_signature(path) + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as handle: + before = os.fstat(handle.fileno()) + raw = handle.read(4_097) + after = os.fstat(handle.fileno()) + after_path = index_signature(path) + except (DocForgeError, OSError): + return False + before_signature = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + after_signature = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + if ( + before_path != before_signature + or after_path != before_signature + or after_signature != before_signature + or len(raw) > 4_096 + or len(raw) != before.st_size + ): return False try: - parsed: object = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): + parsed: object = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): return False if not isinstance(parsed, dict): return False @@ -756,72 +1180,133 @@ class ProjectIndex: expected_size = payload.get("index_size") expected_hash = payload.get("index_sha256") if ( - payload.get("schema_version") != 1 + set(payload) != {"schema_version", "index_size", "index_sha256"} + or payload.get("schema_version") != 1 or type(expected_size) is not int or not isinstance(expected_hash, str) or len(expected_hash) != 64 + or any(character not in "0123456789abcdef" for character in expected_hash) ): return False try: - if self.path.stat().st_size != expected_size: - return False - with self.path.open("rb") as handle: - actual_hash = hashlib.file_digest(handle, "sha256").hexdigest() - except OSError: + size, actual_hash, _ = self._index_digest(expected_signature=expected_signature) + except DocForgeError: return False - return actual_hash == expected_hash + return size == expected_size and actual_hash == expected_hash - def _write_attestation(self) -> None: + def _index_digest( + self, + *, + expected_signature: tuple[int, int, int, int, int] | None = None, + ) -> tuple[int, str, tuple[int, int, int, int, int]]: + """Hash one exact safe index inode and prove it stayed path-bound.""" + + try: + descriptor = os.open(self.path, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as handle: + before = os.fstat(handle.fileno()) + signature = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + if expected_signature is not None and signature != expected_signature: + raise DocForgeError( + "invalid_index", + "Derived index changed before hashing", + ) + index_hash = hashlib.file_digest(handle, "sha256").hexdigest() + after = os.fstat(handle.fileno()) + after_signature = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + if after_signature != signature or index_signature(self.path) != signature: + raise DocForgeError( + "invalid_index", + "Derived index changed during hashing", + ) + except OSError as error: + raise DocForgeError("missing_index", "Derived index cannot be hashed") from error + return before.st_size, index_hash, signature + + def _write_attestation( + self, + *, + expected_signature: tuple[int, int, int, int, int] | None = None, + ) -> None: """Atomically persist the digest of an index that passed complete verification.""" root = self.project.descriptor.cache_root path = self.attestation_path if path.parent != root or path.is_symlink(): raise DocForgeError("path_escape", "Index attestation path is not safe") - try: - size = self.path.stat().st_size - with self.path.open("rb") as handle: - index_hash = hashlib.file_digest(handle, "sha256").hexdigest() - except OSError as error: - raise DocForgeError("missing_index", "Derived index cannot be attested") from error + size, index_hash, before = self._index_digest( + expected_signature=expected_signature, + ) payload = { "schema_version": 1, "index_size": size, "index_sha256": index_hash, } raw = json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n" - descriptor, temporary_name = tempfile.mkstemp(prefix=".index-attestation-", dir=root) - temporary = Path(temporary_name) + root_fd = open_bound_directory(root) + temporary_name = f".index-attestation-{secrets.token_hex(12)}" + temporary_created = False try: + try: + existing = os.stat( + path.name, + dir_fd=root_fd, + follow_symlinks=False, + ) + except FileNotFoundError: + existing = None + if existing is not None and ( + stat.S_ISLNK(existing.st_mode) or not stat.S_ISREG(existing.st_mode) + ): + raise DocForgeError("path_escape", "Index attestation path is not safe") + descriptor = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=root_fd, + ) + temporary_created = True with os.fdopen(descriptor, "wb") as handle: handle.write(raw) handle.flush() os.fsync(handle.fileno()) - os.replace(temporary, path) - directory_descriptor = os.open(root, os.O_RDONLY) - try: - os.fsync(directory_descriptor) - finally: - os.close(directory_descriptor) + require_bound_directory(root, root_fd) + if self._index_signature() != before: + raise DocForgeError( + "invalid_index", + "Derived index changed before attestation publication", + ) + os.replace( + temporary_name, + path.name, + src_dir_fd=root_fd, + dst_dir_fd=root_fd, + ) + temporary_created = False + os.fsync(root_fd) + require_bound_directory(root, root_fd) except Exception: - temporary.unlink(missing_ok=True) + if temporary_created: + with suppress(OSError): + os.unlink(temporary_name, dir_fd=root_fd) raise + finally: + os.close(root_fd) def _index_signature(self) -> tuple[int, int, int, int, int]: - try: - status = self.path.stat() - except OSError as error: - raise DocForgeError( - "missing_index", - "Derived index does not exist; run build first", - ) from error - return ( - status.st_dev, - status.st_ino, - status.st_size, - status.st_mtime_ns, - status.st_ctime_ns, - ) + return index_signature(self.path) def task_context(self, plan: RetrievalPlanV1) -> dict[str, object]: """Execute one fixed task plan inside one immutable index generation.""" @@ -831,6 +1316,122 @@ class ProjectIndex: capsule = self._task_context_capsule(snapshot, plan) return snapshot.result(capsule=capsule.as_dict()) + def generation_diff(self) -> dict[str, object]: + """Return the latest bounded transition without loading or repairing source.""" + + descriptor = self.project.descriptor + identity = { + "status": "ok", + "project_id": descriptor.project_id, + "project_root_fingerprint": project_root_fingerprint(descriptor.root), + "adapter": descriptor.adapter, + } + if not isinstance(self.project, IncrementalStateProject): + return { + **identity, + "revision": "unknown", + "source_hash": None, + "receipt_state": "unknown", + "receipt_reason": "source_identity_unavailable", + "generation_diff": None, + "staleness": "unknown", + } + before_state = self.project.incremental_state() + if before_state is None: + return { + **identity, + "revision": "unknown", + "source_hash": None, + "receipt_state": "unknown", + "receipt_reason": "source_identity_unavailable", + "generation_diff": None, + "staleness": "unknown", + } + receipt, reason, receipt_signature = load_generation_diff(descriptor) + if receipt is None: + receipt_state = ( + "unsafe" + if reason == "unsafe_receipt" + else ("missing" if reason == "missing_receipt" else "unverified") + ) + return { + **identity, + "revision": before_state.revision, + "source_hash": before_state.source_hash, + "receipt_state": receipt_state, + "receipt_reason": reason, + "generation_diff": None, + "staleness": "unknown", + } + try: + current_index_signature = index_signature(self.path) + except DocForgeError as error: + return { + **identity, + "revision": before_state.revision, + "source_hash": before_state.source_hash, + "receipt_state": "unsafe" if error.code == "path_escape" else "stale", + "receipt_reason": ( + "unsafe_index" if error.code == "path_escape" else "missing_index" + ), + "generation_diff": None, + "staleness": "stale", + } + target = cast(dict[str, object], receipt["to_generation"]) + expected_index_signature = cast(dict[str, object], receipt["index_signature"]) + observed_index_signature = { + "device": current_index_signature[0], + "inode": current_index_signature[1], + "size": current_index_signature[2], + "mtime_ns": current_index_signature[3], + "ctime_ns": current_index_signature[4], + } + if ( + target["revision"] != before_state.revision + or target["source_hash"] != before_state.source_hash + or expected_index_signature != observed_index_signature + ): + return { + **identity, + "revision": before_state.revision, + "source_hash": before_state.source_hash, + "receipt_state": "stale", + "receipt_reason": "generation_mismatch", + "generation_diff": None, + "staleness": "stale", + } + after_state = self.project.incremental_state() + try: + after_index_signature = index_signature(self.path) + after_receipt_signature = index_signature(generation_diff_path(descriptor)) + except DocForgeError: + after_state = None + after_index_signature = () + after_receipt_signature = () + if ( + after_state != before_state + or after_index_signature != current_index_signature + or after_receipt_signature != receipt_signature + ): + return { + **identity, + "revision": before_state.revision, + "source_hash": before_state.source_hash, + "receipt_state": "unverified", + "receipt_reason": "concurrent_change", + "generation_diff": None, + "staleness": "unknown", + } + return { + **identity, + "revision": before_state.revision, + "source_hash": before_state.source_hash, + "receipt_state": "current", + "receipt_reason": None, + "generation_diff": receipt, + "staleness": "current", + } + def _task_context_capsule( self, snapshot: _IndexReadSnapshot, diff --git a/src/docforge/mcp_server.py b/src/docforge/mcp_server.py index 46c3f3a..833dc1d 100644 --- a/src/docforge/mcp_server.py +++ b/src/docforge/mcp_server.py @@ -50,6 +50,7 @@ READ_TOOLS = ( "docforge_visualize", "docforge_stop_visualization", "docforge_visualization_status", + "docforge_get_generation_diff", ) PROPOSAL_TOOLS = ( "docforge_create_changeset", @@ -913,6 +914,151 @@ class DocForgeService: return self.invoke(operation, operation_name="mcp.task_context") + def generation_diff( + self, + *, + limit: int | None = None, + cursor: str | None = None, + ) -> dict[str, Any]: + """Return one bounded page from the latest verified graph transition.""" + + def operation() -> dict[str, object]: + maximum_items = min( + self.project.descriptor.limits.max_results, + 1_000, + ) + selected_limit = page_limit( + limit, + default=min(20, maximum_items), + maximum=maximum_items, + ) + result = self.index.generation_diff() + receipt_value = result.get("generation_diff") + if receipt_value is None: + if cursor is not None: + decode_cursor( + cursor, + kind="generation-diff.items", + binding={ + "project_id": result.get("project_id"), + "receipt_state": result.get("receipt_state"), + "effective_policy_hash": canonical_hash(self.policy.as_dict()), + }, + total_count=0, + ) + return result + if not isinstance(receipt_value, Mapping): + raise DocForgeError( + "invalid_generation_diff", + "Generation diff receipt is malformed", + ) + receipt = dict(cast(Mapping[str, object], receipt_value)) + items_value = receipt.pop("items", None) + if not isinstance(items_value, list): + raise DocForgeError( + "invalid_generation_diff", + "Generation diff receipt has no bounded item collection", + ) + items = cast(list[object], items_value) + stored_receipt_hash = receipt.pop("receipt_hash", None) + if not isinstance(stored_receipt_hash, str): + raise DocForgeError( + "invalid_generation_diff", + "Generation diff receipt has no stable identity", + ) + receipt_header = { + **receipt, + "stored_receipt_hash": stored_receipt_hash, + } + binding = { + "stored_receipt_hash": stored_receipt_hash, + "effective_policy_hash": canonical_hash(self.policy.as_dict()), + } + position = decode_cursor( + cursor, + kind="generation-diff.items", + binding=binding, + total_count=len(items), + ) + page_items: list[object] = [] + page_omissions: list[dict[str, object]] = [] + consumed = 0 + maximum_chars = self.project.descriptor.limits.max_tool_output_chars + + def page_result() -> dict[str, object]: + pagination = page_receipt( + kind="generation-diff.items", + binding=binding, + position=position, + count=consumed, + limit=selected_limit, + total_count=len(items), + ) + page_hash = canonical_hash( + { + "page_schema_version": 1, + "receipt_state": result["receipt_state"], + "receipt_header": receipt_header, + "pagination": pagination, + "items": page_items, + "omissions": page_omissions, + } + ) + return { + **result, + "generation_diff": { + "page_schema_version": 1, + "receipt_header": receipt_header, + "items": page_items, + "omissions": page_omissions, + "page_hash": page_hash, + }, + "pagination": pagination, + } + + candidates = items[position : position + selected_limit] + + def fits(candidate_count: int) -> bool: + nonlocal consumed + page_items[:] = candidates[:candidate_count] + consumed = candidate_count + decorated = { + **page_result(), + "server_version": SERVER_VERSION, + "content_warning": CONTENT_WARNING, + } + return self._encoded_length(decorated) <= maximum_chars + + lower = 0 + upper = len(candidates) + while lower < upper: + midpoint = (lower + upper + 1) // 2 + if fits(midpoint): + lower = midpoint + else: + upper = midpoint - 1 + fits(lower) + if lower == 0 and candidates: + item = candidates[0] + item_payload: Mapping[str, object] = ( + cast(Mapping[str, object], item) if isinstance(item, Mapping) else {} + ) + page_omissions.append( + { + "code": "response_limit", + "item_hash": item_payload.get("item_hash"), + } + ) + consumed = 1 + return page_result() + + return self.invoke( + operation, + synchronize=False, + load_error_identity=False, + operation_name="mcp.generation_diff", + ) + def _page_task_context_result( self, result: dict[str, object], @@ -1436,6 +1582,15 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC return service.visualization_status() + @server.tool(name="docforge_get_generation_diff") + def get_generation_diff( + limit: int | None = None, + cursor: str | None = None, + ) -> dict[str, Any]: + """Return the latest bounded primary-graph generation transition.""" + + return service.generation_diff(limit=limit, cursor=cursor) + _registered_read_tools = ( bootstrap, synchronize, @@ -1453,8 +1608,9 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC validate_project, render_status, visualize, - visualization_status, stop_visualization, + visualization_status, + get_generation_diff, ) if read_only: return server diff --git a/src/docforge/pagination.py b/src/docforge/pagination.py index a90916c..12e4a68 100644 --- a/src/docforge/pagination.py +++ b/src/docforge/pagination.py @@ -103,7 +103,6 @@ def decode_cursor( or not isinstance(stored_binding, dict) or type(position) is not int or position < 0 - or position >= total_count or not isinstance(checksum, str) or len(checksum) != 64 ): @@ -117,6 +116,8 @@ def decode_cursor( "stale_cursor", "Pagination cursor does not match the current result generation", ) + if position >= total_count: + raise _invalid_cursor() return position diff --git a/src/docforge/telemetry.py b/src/docforge/telemetry.py index 946c4de..c40cb7b 100644 --- a/src/docforge/telemetry.py +++ b/src/docforge/telemetry.py @@ -92,6 +92,7 @@ OPERATION_NAMES = frozenset( "mcp.impact", "mcp.context", "mcp.task_context", + "mcp.generation_diff", "mcp.validate_project", "mcp.render_status", "mcp.visualize", @@ -114,6 +115,7 @@ OPERATION_NAMES = frozenset( "cli.dependencies", "cli.impact", "cli.context", + "cli.generation-diff", "cli.render", "cli.render-status", "cli.preview", diff --git a/tests/test_adapter_contract.py b/tests/test_adapter_contract.py index 60f3c29..e4bc3f5 100644 --- a/tests/test_adapter_contract.py +++ b/tests/test_adapter_contract.py @@ -2,6 +2,7 @@ from __future__ import annotations import hashlib import importlib +import json import sqlite3 import sys import tempfile @@ -645,12 +646,17 @@ class AdapterContractTests(unittest.TestCase): def test_no_ast_index_accepts_legacy_and_non_logic_incremental_adapters(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory).resolve() + legacy_loader = Loader(self.projection(root)) legacy = AdapterProject( - Loader(self.projection(root)), + legacy_loader, cache_root=root / ".cache" / "legacy-no-ast", ) legacy_index = ProjectIndex(legacy, allow_logic=False) self.assertEqual(2, legacy_index.build()["node_count"]) + calls_after_build = legacy_loader.load_calls + legacy_diff = legacy_index.generation_diff() + self.assertEqual("unknown", legacy_diff["receipt_state"]) + self.assertEqual(calls_after_build, legacy_loader.load_calls) self.assertEqual( "guide.workflow", legacy_index.get_node("guide.workflow")["node"]["node_id"], @@ -674,6 +680,12 @@ class AdapterContractTests(unittest.TestCase): self.assertEqual(2, second["build"]["cache_hits"]) self.assertEqual(0, second["build"]["reparsed_sources"]) self.assertEqual(0, second["logic_projection_count"]) + generation_diff = index.generation_diff() + self.assertEqual("current", generation_diff["receipt_state"]) + self.assertNotIn( + "logic", + json.dumps(generation_diff["generation_diff"]).casefold(), + ) def test_no_ast_rejects_preexisting_logic_index_and_viewer_snapshot(self) -> None: with tempfile.TemporaryDirectory() as directory: diff --git a/tests/test_generation_diff.py b/tests/test_generation_diff.py new file mode 100644 index 0000000..bb67129 --- /dev/null +++ b/tests/test_generation_diff.py @@ -0,0 +1,993 @@ +from __future__ import annotations + +import contextlib +import io +import json +import shutil +import sqlite3 +import tempfile +import unittest +from collections.abc import Callable +from dataclasses import replace +from pathlib import Path +from unittest import mock + +from jsonschema import Draft202012Validator +from mcp.shared.memory import create_connected_server_and_client_session + +import docforge.generation_diff as generation_diff_module +from docforge.cli import main +from docforge.errors import DocForgeError +from docforge.generation_diff import ( + GenerationDiffDraft, + finalize_generation_diff, + generation_diff_path, + publish_generation_diff, + validate_generation_diff_receipt, +) +from docforge.index import ProjectIndex +from docforge.mcp_server import DocForgeService, create_server +from docforge.models import ProjectDescriptor, ProjectSnapshot +from docforge.pagination import canonical_hash +from docforge.project import Project, project_root_fingerprint +from docforge.telemetry import request + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "tests" / "fixtures" +GENERATION_DIFF_SCHEMA = json.loads( + (ROOT / "schemas" / "generation-diff.schema.json").read_text(encoding="utf-8") +) +GENERATION_DIFF_PAGE_SCHEMA = json.loads( + (ROOT / "schemas" / "generation-diff-page.schema.json").read_text(encoding="utf-8") +) +RESULT_SCHEMA = json.loads((ROOT / "schemas" / "result.schema.json").read_text(encoding="utf-8")) + + +class StaticProject: + """Small legacy one-method project used to prove load-free status behavior.""" + + def __init__(self, snapshot: ProjectSnapshot) -> None: + self.descriptor: ProjectDescriptor = snapshot.descriptor + self.snapshot = snapshot + self.load_calls = 0 + + def load(self) -> ProjectSnapshot: + self.load_calls += 1 + return self.snapshot + + def canonical_source_paths(self) -> tuple[Path, ...]: + return () + + def validate_proposal( + self, + base: ProjectSnapshot, + projected: ProjectSnapshot, + operations: tuple[object, ...], + ) -> None: + del base, projected, operations + + +class GenerationDiffTests(unittest.TestCase): + def copy_fixture(self, destination: Path) -> Path: + root = destination / "alpha" + shutil.copytree(FIXTURES / "alpha", root) + shutil.rmtree(root / ".docforge" / "cache", ignore_errors=True) + return root + + @staticmethod + def change_graph(root: Path, suffix: str = "changed") -> None: + foundation = root / "docs" / "content" / "foundation.md" + foundation.write_text( + foundation.read_text(encoding="utf-8").replace( + "Defines which Alpha files own documentation facts.", + f"Defines which Alpha files own documentation facts. {suffix}", + ), + encoding="utf-8", + ) + proof = root / "docs" / "content" / "proof.toml" + proof.write_text( + proof.read_text(encoding="utf-8").replace( + 'proves = ["guide.workflow"]', + 'proves = ["guide.foundation"]', + ), + encoding="utf-8", + ) + + @staticmethod + def rehash_receipt(receipt: dict[str, object]) -> dict[str, object]: + items = receipt["items"] + if not isinstance(items, list): + raise AssertionError("receipt items are not a list") + for item in items: + if not isinstance(item, dict): + raise AssertionError("receipt item is not an object") + item["item_hash"] = canonical_hash( + {key: value for key, value in item.items() if key != "item_hash"} + ) + retained_hash = canonical_hash([item["item_hash"] for item in items]) + receipt["retained_collection_hash"] = retained_hash + if not receipt["details_truncated"]: + receipt["full_collection_hash"] = retained_hash + receipt["receipt_hash"] = canonical_hash( + {key: value for key, value in receipt.items() if key != "receipt_hash"} + ) + return receipt + + def transition_receipt(self, root: Path) -> tuple[Project, ProjectIndex, dict[str, object]]: + project = Project.open(root) + index = ProjectIndex(project) + index.build() + self.change_graph(root) + index.build() + receipt = json.loads(generation_diff_path(project.descriptor).read_text(encoding="utf-8")) + return project, index, receipt + + def test_first_build_and_exact_transition_are_schema_valid(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + + first = index.build() + first_receipt = json.loads( + generation_diff_path(project.descriptor).read_text(encoding="utf-8") + ) + Draft202012Validator(GENERATION_DIFF_SCHEMA).validate(first_receipt) + self.assertTrue( + validate_generation_diff_receipt( + first_receipt, + descriptor=project.descriptor, + ) + ) + self.assertEqual("baseline", first_receipt["kind"]) + self.assertEqual("no_predecessor", first_receipt["reason"]) + self.assertEqual(0, first_receipt["full_item_count"]) + self.assertEqual("ok", first["status"]) + + self.change_graph(root) + second = index.build() + receipt = json.loads( + generation_diff_path(project.descriptor).read_text(encoding="utf-8") + ) + Draft202012Validator(GENERATION_DIFF_SCHEMA).validate(receipt) + self.assertEqual("transition", receipt["kind"]) + self.assertEqual( + { + "nodes_added": 0, + "nodes_removed": 0, + "nodes_changed": 2, + "edges_added": 1, + "edges_removed": 1, + "total_changes": 4, + }, + receipt["summary"], + ) + self.assertEqual( + [ + ("edge", "added"), + ("edge", "removed"), + ("node", "changed"), + ("node", "changed"), + ], + [(item["entity"], item["change"]) for item in receipt["items"]], + ) + self.assertNotIn("logic_hash", json.dumps(receipt, sort_keys=True).casefold()) + self.assertEqual("ok", second["status"]) + + status = ProjectIndex(project).generation_diff() + self.assertEqual("current", status["receipt_state"]) + self.assertEqual(receipt["receipt_hash"], status["generation_diff"]["receipt_hash"]) + + def test_same_generation_reindex_preserves_latest_meaningful_transition(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + self.change_graph(root) + index.build() + before = json.loads( + generation_diff_path(project.descriptor).read_text(encoding="utf-8") + ) + + index.build() + after = json.loads(generation_diff_path(project.descriptor).read_text(encoding="utf-8")) + + self.assertEqual("transition", after["kind"]) + self.assertEqual(before["from_generation"], after["from_generation"]) + self.assertEqual(before["to_generation"], after["to_generation"]) + self.assertEqual(before["summary"], after["summary"]) + self.assertEqual(before["full_collection_hash"], after["full_collection_hash"]) + self.assertNotEqual(before["index_signature"], after["index_signature"]) + self.assertEqual( + ["generation-diff.json"], + [path.name for path in project.descriptor.cache_root.glob("*generation-diff*")], + ) + + def test_receipt_item_and_byte_limits_preserve_exact_summary_hashes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + descriptor = Project.open(root).descriptor + generation = { + "revision": "unversioned", + "source_hash": "1" * 64, + "node_count": 0, + "node_hash": "2" * 64, + "edge_count": 0, + "edge_hash": "3" * 64, + "index_schema_version": 3, + } + items: list[dict[str, object]] = [] + for index in range(1_002): + item: dict[str, object] = { + "entity": "edge", + "change": "added", + "source_id": f"source-{index:04d}", + "relation": "relates_to", + "target_id": f"target-{index:04d}", + } + item["item_hash"] = canonical_hash(item) + items.append(item) + fields: dict[str, object] = { + "schema_version": 1, + "diff_semantics_version": 1, + "project_id": descriptor.project_id, + "project_root_fingerprint": project_root_fingerprint(descriptor.root), + "adapter": descriptor.adapter, + "kind": "transition", + "reason": None, + "from_generation": generation, + "to_generation": {**generation, "source_hash": "4" * 64}, + "summary": { + "nodes_added": 0, + "nodes_removed": 0, + "nodes_changed": 0, + "edges_added": len(items), + "edges_removed": 0, + "total_changes": len(items), + }, + "full_item_count": len(items), + "full_collection_hash": canonical_hash([item["item_hash"] for item in items]), + } + receipt = finalize_generation_diff( + GenerationDiffDraft(fields=fields, items=tuple(items)), + signature=(1, 2, 3, 4, 5), + ) + Draft202012Validator(GENERATION_DIFF_SCHEMA).validate(receipt) + self.assertEqual(1_002, receipt["full_item_count"]) + self.assertEqual(1_000, receipt["retained_item_count"]) + self.assertEqual("receipt_item_limit", receipt["truncation_reason"]) + self.assertNotEqual( + receipt["full_collection_hash"], + receipt["retained_collection_hash"], + ) + + huge = dict(items[0]) + huge["source_id"] = "source-" + ("x" * 1_100_000) + huge["item_hash"] = canonical_hash( + {key: value for key, value in huge.items() if key != "item_hash"} + ) + huge_fields = { + **fields, + "summary": { + **fields["summary"], + "edges_added": 1, + "total_changes": 1, + }, + "full_item_count": 1, + "full_collection_hash": canonical_hash([huge["item_hash"]]), + } + byte_limited = finalize_generation_diff( + GenerationDiffDraft(fields=huge_fields, items=(huge,)), + signature=(1, 2, 3, 4, 5), + ) + self.assertEqual(0, byte_limited["retained_item_count"]) + self.assertEqual("receipt_byte_limit", byte_limited["truncation_reason"]) + + def test_receipt_runtime_and_schema_reject_malformed_semantics(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project, _, receipt = self.transition_receipt(root) + validator = Draft202012Validator(GENERATION_DIFF_SCHEMA) + + malformed_fields = json.loads(json.dumps(receipt)) + changed = next( + item + for item in malformed_fields["items"] + if item["entity"] == "node" and item["change"] == "changed" + ) + changed["changed_fields"] = 1 + self.rehash_receipt(malformed_fields) + self.assertFalse( + validate_generation_diff_receipt( + malformed_fields, + descriptor=project.descriptor, + ) + ) + self.assertFalse(validator.is_valid(malformed_fields)) + + duplicate_fields = json.loads(json.dumps(receipt)) + changed = next( + item + for item in duplicate_fields["items"] + if item["entity"] == "node" and item["change"] == "changed" + ) + changed["changed_fields"] = ["content", "content"] + self.rehash_receipt(duplicate_fields) + self.assertFalse(validate_generation_diff_receipt(duplicate_fields)) + self.assertFalse(validator.is_valid(duplicate_fields)) + + impossible_added = json.loads(json.dumps(receipt)) + changed = next( + item + for item in impossible_added["items"] + if item["entity"] == "node" and item["change"] == "changed" + ) + changed["change"] = "added" + changed["changed_fields"] = [] + self.rehash_receipt(impossible_added) + self.assertFalse(validate_generation_diff_receipt(impossible_added)) + self.assertFalse(validator.is_valid(impossible_added)) + + reordered = json.loads(json.dumps(receipt)) + reordered["items"].reverse() + self.rehash_receipt(reordered) + self.assertFalse(validate_generation_diff_receipt(reordered)) + + same_generation = json.loads(json.dumps(receipt)) + same_generation["from_generation"] = same_generation["to_generation"] + self.rehash_receipt(same_generation) + self.assertFalse(validate_generation_diff_receipt(same_generation)) + + impossible_truncation = json.loads(json.dumps(receipt)) + impossible_truncation["items"].pop() + impossible_truncation["retained_item_count"] = len(impossible_truncation["items"]) + impossible_truncation["details_truncated"] = True + impossible_truncation["truncation_reason"] = "receipt_item_limit" + self.rehash_receipt(impossible_truncation) + self.assertFalse(validate_generation_diff_receipt(impossible_truncation)) + self.assertFalse(validator.is_valid(impossible_truncation)) + + baseline_with_changes = json.loads(json.dumps(receipt)) + baseline_with_changes["kind"] = "baseline" + baseline_with_changes["reason"] = "no_meaningful_transition" + baseline_with_changes["from_generation"] = None + self.rehash_receipt(baseline_with_changes) + self.assertFalse(validate_generation_diff_receipt(baseline_with_changes)) + self.assertFalse(validator.is_valid(baseline_with_changes)) + + def test_generation_collision_and_nondeterministic_build_fail_precommit(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + base = Project.open(root).load() + static = StaticProject(base) + index = ProjectIndex(static) + index.build() + original_index = index.path.read_bytes() + original_receipt = generation_diff_path(static.descriptor).read_bytes() + + changed_node = replace( + base.nodes[0], + title="Different graph under the same generation", + ) + static.snapshot = replace( + base, + nodes=(changed_node, *base.nodes[1:]), + ) + with self.assertRaises(DocForgeError) as collision: + index.build() + self.assertEqual("generation_collision", collision.exception.code) + self.assertEqual(original_index, index.path.read_bytes()) + self.assertEqual( + original_receipt, + generation_diff_path(static.descriptor).read_bytes(), + ) + + calls = 0 + + def unstable() -> ProjectSnapshot: + nonlocal calls + calls += 1 + return ( + base + if calls == 1 + else replace( + base, + nodes=(changed_node, *base.nodes[1:]), + ) + ) + + static.snapshot = base + with ( + mock.patch.object(static, "load", side_effect=unstable), + self.assertRaises(DocForgeError) as changed, + ): + ProjectIndex(static).build() + self.assertEqual("source_changed", changed.exception.code) + + def test_post_commit_receipt_failures_report_degraded_success(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + self.change_graph(root) + + with ( + mock.patch.object( + ProjectIndex, + "_write_attestation", + side_effect=OSError("attestation failed"), + ), + mock.patch.object( + Project, + "record_generation", + side_effect=OSError("generation failed"), + ), + mock.patch( + "docforge.index.publish_generation_diff", + side_effect=OSError("diff failed"), + ), + ): + result = index.build() + + self.assertEqual("ok", result["status"]) + self.assertEqual("degraded", result["publication"]["state"]) + self.assertEqual("published", result["publication"]["index"]) + self.assertEqual( + {"attestation", "source_generation", "generation_diff"}, + {error["stage"] for error in result["publication"]["errors"]}, + ) + with contextlib.closing(__import__("sqlite3").connect(index.path)) as connection: + metadata = dict(connection.execute("SELECT key, value FROM metadata")) + self.assertEqual(project.load().source_hash, metadata["source_hash"]) + + def test_predecessor_requires_attestation_and_valid_generation_identity(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + index.attestation_path.unlink() + self.change_graph(root) + result = index.build() + receipt = json.loads( + generation_diff_path(project.descriptor).read_text(encoding="utf-8") + ) + self.assertEqual("ok", result["status"]) + self.assertEqual("baseline", receipt["kind"]) + self.assertEqual("predecessor_unattested", receipt["reason"]) + self.assertTrue(validate_generation_diff_receipt(receipt)) + + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + with contextlib.closing(sqlite3.connect(index.path)) as connection: + connection.execute("UPDATE metadata SET value = 'bad' WHERE key = 'source_hash'") + connection.commit() + index._write_attestation() + self.change_graph(root) + index.build() + receipt = json.loads( + generation_diff_path(project.descriptor).read_text(encoding="utf-8") + ) + self.assertEqual("baseline", receipt["kind"]) + self.assertEqual("predecessor_corrupt", receipt["reason"]) + self.assertTrue(validate_generation_diff_receipt(receipt)) + + def test_sqlite_sidecars_refuse_precommit_publication(self) -> None: + for suffix in ("-wal", "-journal", "-shm"): + with self.subTest(suffix=suffix), tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + original_index = index.path.read_bytes() + original_receipt = generation_diff_path(project.descriptor).read_bytes() + Path(f"{index.path}{suffix}").write_bytes(b"unproven-sidecar") + self.change_graph(root) + with self.assertRaises(DocForgeError) as blocked: + index.build() + self.assertEqual("index_busy", blocked.exception.code) + self.assertEqual(original_index, index.path.read_bytes()) + self.assertEqual( + original_receipt, + generation_diff_path(project.descriptor).read_bytes(), + ) + + def test_live_wal_state_cannot_bypass_main_index_identity(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + original_receipt = generation_diff_path(project.descriptor).read_bytes() + connection = sqlite3.connect(index.path) + try: + self.assertEqual( + "wal", + connection.execute("PRAGMA journal_mode=WAL").fetchone()[0], + ) + connection.execute( + "UPDATE metadata SET value = ? WHERE key = 'source_hash'", + ("f" * 64,), + ) + connection.commit() + self.assertTrue(Path(f"{index.path}-wal").exists()) + main_file_after_wal = index.path.read_bytes() + with self.assertRaises(DocForgeError) as blocked: + index.build() + self.assertEqual("index_busy", blocked.exception.code) + self.assertEqual(main_file_after_wal, index.path.read_bytes()) + self.assertEqual( + original_receipt, + generation_diff_path(project.descriptor).read_bytes(), + ) + finally: + connection.close() + + def test_sidecar_or_source_change_during_diff_preparation_aborts_precommit(self) -> None: + for mutation in ("source", "sidecar"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + original_index = index.path.read_bytes() + original_receipt = generation_diff_path(project.descriptor).read_bytes() + self.change_graph(root) + from docforge import index as index_module + + real_prepare = index_module.prepare_generation_diff + + def mutate_after_prepare( + *args: object, + _prepare: Callable[..., object] = real_prepare, + _mutation: str = mutation, + _root: Path = root, + _index: ProjectIndex = index, + **kwargs: object, + ) -> object: + draft = _prepare(*args, **kwargs) + if _mutation == "source": + source = _root / "docs" / "content" / "workflow.md" + source.write_text( + source.read_text(encoding="utf-8") + "\nConcurrent change.\n", + encoding="utf-8", + ) + else: + Path(f"{_index.path}-wal").write_bytes(b"appeared") + return draft + + with ( + mock.patch( + "docforge.index.prepare_generation_diff", + side_effect=mutate_after_prepare, + ), + self.assertRaises(DocForgeError) as blocked, + ): + index.build() + self.assertEqual( + "source_changed" if mutation == "source" else "index_busy", + blocked.exception.code, + ) + self.assertEqual(original_index, index.path.read_bytes()) + self.assertEqual( + original_receipt, + generation_diff_path(project.descriptor).read_bytes(), + ) + + def test_cache_root_symlink_cannot_redirect_receipt_publication(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + receipt = json.loads( + generation_diff_path(project.descriptor).read_text(encoding="utf-8") + ) + cache_root = project.descriptor.cache_root + preserved = cache_root.with_name("preserved-cache") + outside = root / "outside-cache" + outside.mkdir() + cache_root.rename(preserved) + cache_root.symlink_to(outside, target_is_directory=True) + try: + with self.assertRaises(DocForgeError) as blocked: + publish_generation_diff(project.descriptor, receipt) + self.assertEqual("path_escape", blocked.exception.code) + self.assertFalse((outside / "generation-diff.json").exists()) + finally: + cache_root.unlink() + preserved.rename(cache_root) + + def test_post_commit_identity_and_durability_failures_are_degraded(self) -> None: + cases = ( + ("_fsync_cache_directory", OSError("fsync failed"), "index_directory_sync"), + ( + "_published_index_signature", + OSError("signature failed"), + "index_identity", + ), + ) + for method, failure, stage in cases: + with self.subTest(method=method), tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + self.change_graph(root) + with mock.patch.object(ProjectIndex, method, side_effect=failure): + result = index.build() + self.assertEqual("ok", result["status"]) + self.assertEqual("degraded", result["publication"]["state"]) + self.assertIn( + stage, + {error["stage"] for error in result["publication"]["errors"]}, + ) + with contextlib.closing(sqlite3.connect(index.path)) as connection: + metadata = dict(connection.execute("SELECT key, value FROM metadata")) + self.assertEqual(project.load().source_hash, metadata["source_hash"]) + + def test_post_commit_receipts_fail_independently(self) -> None: + cases = ( + ("attestation", "docforge.index.ProjectIndex._write_attestation"), + ("source_generation", "docforge.project.Project.record_generation"), + ("generation_diff", "docforge.index.publish_generation_diff"), + ) + for failed_receipt, target in cases: + with self.subTest(receipt=failed_receipt), tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + self.change_graph(root) + with mock.patch(target, side_effect=OSError("failed independently")): + result = index.build() + receipts = result["publication"]["receipts"] + self.assertEqual("unavailable", receipts[failed_receipt]["state"]) + for name in {"attestation", "source_generation", "generation_diff"} - { + failed_receipt + }: + self.assertEqual("published", receipts[name]["state"]) + + def test_read_is_bounded_read_only_and_legacy_projects_do_not_load(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + receipt_path = generation_diff_path(project.descriptor) + original_receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + before = { + path: path.stat().st_mtime_ns + for path in project.descriptor.cache_root.iterdir() + if path.is_file() + } + with ( + request("test", enabled=True) as collector, + mock.patch.object(project, "load", side_effect=AssertionError("loaded")), + mock.patch.object(index, "check", side_effect=AssertionError("checked")), + mock.patch.object(index, "build", side_effect=AssertionError("built")), + mock.patch.object(index, "synchronize", side_effect=AssertionError("synced")), + ): + result = index.generation_diff() + self.assertEqual("current", result["receipt_state"]) + self.assertIsNotNone(collector) + diagnostics = collector.as_dict(outcome="ok") + for counter in ( + "project_loads", + "source_files_parsed", + "adapter_projection_loads", + "adapter_source_extractions", + "index_checks", + "index_synchronizations", + "index_builds", + ): + self.assertEqual(0, diagnostics["counters"][counter]) + self.assertGreaterEqual(diagnostics["counters"]["source_generation_checks"], 2) + self.assertEqual( + before, + { + path: path.stat().st_mtime_ns + for path in project.descriptor.cache_root.iterdir() + if path.is_file() + }, + ) + with mock.patch.object( + generation_diff_module, + "validate_generation_diff_receipt", + wraps=generation_diff_module.validate_generation_diff_receipt, + ) as validated: + self.assertEqual("current", index.generation_diff()["receipt_state"]) + self.assertEqual(1, validated.call_count) + + receipt_path.write_text("{broken", encoding="utf-8") + corrupt_before = receipt_path.read_bytes() + corrupt = index.generation_diff() + self.assertEqual("unverified", corrupt["receipt_state"]) + self.assertEqual("corrupt_receipt", corrupt["receipt_reason"]) + self.assertEqual(corrupt_before, receipt_path.read_bytes()) + + foreign = {**original_receipt, "project_id": "foreign-project"} + foreign["receipt_hash"] = canonical_hash( + {key: value for key, value in foreign.items() if key != "receipt_hash"} + ) + receipt_path.write_text( + json.dumps(foreign, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + foreign_result = index.generation_diff() + self.assertEqual("unverified", foreign_result["receipt_state"]) + self.assertEqual("foreign_receipt", foreign_result["receipt_reason"]) + + outside = root / "foreign-generation-diff.json" + outside.write_text( + json.dumps(original_receipt, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + receipt_path.unlink() + receipt_path.symlink_to(outside) + unsafe = index.generation_diff() + self.assertEqual("unsafe", unsafe["receipt_state"]) + self.assertEqual("unsafe_receipt", unsafe["receipt_reason"]) + + snapshot = project.load() + legacy = StaticProject(snapshot) + calls = legacy.load_calls + unknown = ProjectIndex(legacy).generation_diff() + self.assertEqual("unknown", unknown["receipt_state"]) + self.assertEqual(calls, legacy.load_calls) + + def test_cli_generation_diff_is_additive_and_paged(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + self.change_graph(root) + index.build() + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = main( + [ + "--project-root", + str(root), + "generation-diff", + "--limit", + "1", + ] + ) + result = json.loads(output.getvalue()) + self.assertEqual(0, code) + Draft202012Validator(RESULT_SCHEMA).validate(result) + Draft202012Validator(GENERATION_DIFF_PAGE_SCHEMA).validate( + { + "generation_diff": result["generation_diff"], + "pagination": result["pagination"], + } + ) + self.assertEqual("generation-diff.items", result["pagination"]["kind"]) + self.assertEqual(1, result["pagination"]["returned_count"]) + self.assertTrue(result["pagination"]["has_more"]) + self.assertNotIn("next_cursor", result) + self.assertNotIn("pagination", result["generation_diff"]) + self.assertEqual( + { + "page_schema_version", + "receipt_header", + "items", + "omissions", + "page_hash", + }, + set(result["generation_diff"]), + ) + page = result["generation_diff"] + self.assertIn("stored_receipt_hash", page["receipt_header"]) + self.assertNotIn("receipt_hash", page["receipt_header"]) + self.assertEqual( + page["page_hash"], + canonical_hash( + { + "page_schema_version": 1, + "receipt_state": result["receipt_state"], + "receipt_header": page["receipt_header"], + "pagination": result["pagination"], + "items": page["items"], + "omissions": page["omissions"], + } + ), + ) + self.assertEqual( + result["pagination"]["returned_count"], + len(page["items"]) + len(page["omissions"]), + ) + self.assertEqual( + result["pagination"]["total_count"], + page["receipt_header"]["retained_item_count"], + ) + + malformed_page = json.loads(json.dumps(result)) + stored_receipt = json.loads( + generation_diff_path(project.descriptor).read_text(encoding="utf-8") + ) + node = next(item for item in stored_receipt["items"] if item["entity"] == "node") + node["change"] = "added" + malformed_page["generation_diff"]["items"] = [node] + self.assertFalse( + Draft202012Validator(GENERATION_DIFF_PAGE_SCHEMA).is_valid( + { + "generation_diff": malformed_page["generation_diff"], + "pagination": malformed_page["pagination"], + } + ) + ) + + malformed_header = json.loads(json.dumps(result)) + header = malformed_header["generation_diff"]["receipt_header"] + header["kind"] = "baseline" + header["reason"] = "no_meaningful_transition" + header["from_generation"] = None + self.assertFalse( + Draft202012Validator(GENERATION_DIFF_PAGE_SCHEMA).is_valid( + { + "generation_diff": malformed_header["generation_diff"], + "pagination": malformed_header["pagination"], + } + ) + ) + + contradictory_pagination = json.loads(json.dumps(result)) + contradictory_pagination["pagination"]["has_more"] = False + self.assertFalse( + Draft202012Validator(GENERATION_DIFF_PAGE_SCHEMA).is_valid( + { + "generation_diff": contradictory_pagination["generation_diff"], + "pagination": contradictory_pagination["pagination"], + } + ) + ) + missing_cursor = json.loads(json.dumps(result)) + missing_cursor["pagination"]["next_cursor"] = None + self.assertFalse( + Draft202012Validator(GENERATION_DIFF_PAGE_SCHEMA).is_valid( + { + "generation_diff": missing_cursor["generation_diff"], + "pagination": missing_cursor["pagination"], + } + ) + ) + + +class GenerationDiffMcpTests(unittest.IsolatedAsyncioTestCase): + def copy_fixture(self, destination: Path) -> Path: + root = destination / "alpha" + shutil.copytree(FIXTURES / "alpha", root) + shutil.rmtree(root / ".docforge" / "cache", ignore_errors=True) + return root + + async def test_mcp_surface_paginates_and_cursors_bind_the_receipt(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + GenerationDiffTests.change_graph(root) + index.build() + + async with create_connected_server_and_client_session( + create_server(root), + raise_exceptions=True, + ) as session: + tools = {tool.name: tool for tool in (await session.list_tools()).tools} + schema = tools["docforge_get_generation_diff"].inputSchema + self.assertEqual({"limit", "cursor"}, set(schema["properties"])) + self.assertEqual([], schema.get("required", [])) + + first = await session.call_tool( + "docforge_get_generation_diff", + {"limit": 1}, + ) + first_result = first.structuredContent + Draft202012Validator(RESULT_SCHEMA).validate(first_result) + Draft202012Validator(GENERATION_DIFF_PAGE_SCHEMA).validate( + { + "generation_diff": first_result["generation_diff"], + "pagination": first_result["pagination"], + } + ) + self.assertEqual("current", first_result["receipt_state"]) + self.assertEqual(1, first_result["pagination"]["returned_count"]) + cursor = first_result["pagination"]["next_cursor"] + self.assertIsInstance(cursor, str) + self.assertLess(len(cursor), 1_000) + self.assertNotIn("next_cursor", first_result) + + second = await session.call_tool( + "docforge_get_generation_diff", + {"limit": 2, "cursor": cursor}, + ) + self.assertEqual(2, second.structuredContent["pagination"]["returned_count"]) + + foundation = root / "docs" / "content" / "foundation.md" + foundation.write_text( + foundation.read_text(encoding="utf-8") + "\nAnother transition.\n", + encoding="utf-8", + ) + ProjectIndex(Project.open(root)).build() + stale = await session.call_tool( + "docforge_get_generation_diff", + {"limit": 1, "cursor": cursor}, + ) + self.assertEqual("stale_cursor", stale.structuredContent["error"]["code"]) + Draft202012Validator(RESULT_SCHEMA).validate(stale.structuredContent) + + def test_service_diagnostics_drop_before_primary_page(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + descriptor_path = root / ".docforge" / "project.toml" + descriptor_path.write_text( + descriptor_path.read_text(encoding="utf-8").replace( + "max_results = 20", + "max_results = 20\nmax_tool_output_chars = 2100", + ), + encoding="utf-8", + ) + project = Project.open(root) + ProjectIndex(project).build() + service = DocForgeService( + project, + diagnostics=True, + capability_mode_name="read", + ) + result = service.generation_diff() + self.assertEqual("ok", result["status"]) + self.assertNotIn("diagnostics", result) + self.assertLessEqual(service._encoded_length(result), 2_100) + + def test_page_sizing_uses_logarithmic_response_encodes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + descriptor_path = root / ".docforge" / "project.toml" + descriptor_path.write_text( + descriptor_path.read_text(encoding="utf-8").replace( + "max_results = 20", + "max_results = 1000", + ), + encoding="utf-8", + ) + project = Project.open(root) + index = ProjectIndex(project) + index.build() + stored = index.generation_diff() + receipt = dict(stored["generation_diff"]) + receipt["items"] = [ + { + "item_hash": canonical_hash({"ordinal": ordinal}), + "payload": "x" * 500, + } + for ordinal in range(1_000) + ] + service = DocForgeService( + project, + capability_mode_name="read", + ) + with ( + mock.patch.object( + service.index, + "generation_diff", + return_value={**stored, "generation_diff": receipt}, + ), + mock.patch.object( + service, + "_encoded_length", + wraps=service._encoded_length, + ) as encoded, + ): + result = service.generation_diff(limit=1_000) + self.assertEqual("ok", result["status"]) + self.assertGreater(result["pagination"]["returned_count"], 0) + self.assertLess(result["pagination"]["returned_count"], 1_000) + self.assertLessEqual(encoded.call_count, 15) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index adb14ac..287af17 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -90,6 +90,7 @@ class DocForgeMcpTests(unittest.IsolatedAsyncioTestCase): for name in ( "docforge_get_context", "docforge_get_task_context", + "docforge_get_generation_diff", "docforge_list_changesets", "docforge_get_changeset", "docforge_validate_changeset", diff --git a/tests/test_public_contract.py b/tests/test_public_contract.py index deae203..b33bef6 100644 --- a/tests/test_public_contract.py +++ b/tests/test_public_contract.py @@ -111,6 +111,7 @@ EXPECTED_CLI_COMMANDS = { "context", "dependencies", "filter", + "generation-diff", "impact", "info", "onboard", @@ -139,6 +140,7 @@ EXPECTED_MCP_TOOLS = { "docforge_get_changeset", "docforge_get_changeset_diff", "docforge_get_context", + "docforge_get_generation_diff", "docforge_get_task_context", "docforge_get_contract", "docforge_get_logic", @@ -263,6 +265,14 @@ class PublicContractTests(unittest.TestCase): result_validator = Draft202012Validator(self.schema("result.schema.json")) result_validator.validate(success) result_validator.validate(error) + generation_diff = service.generation_diff(limit=1) + result_validator.validate(generation_diff) + Draft202012Validator(self.schema("generation-diff-page.schema.json")).validate( + { + "generation_diff": generation_diff["generation_diff"], + "pagination": generation_diff["pagination"], + } + ) def test_changeset_hash_is_exact_canonical_json_sha256(self) -> None: document = {