From 696b62f9f81251dfca395b502e411537c2aca673 Mon Sep 17 00:00:00 2001 From: Andraxion Date: Sat, 25 Jul 2026 19:08:39 -0400 Subject: [PATCH] Add incremental adapter compiler boundary --- README.md | 5 + SLICE_HISTORY.md | 24 ++ docs/CONTRACT.md | 21 +- docs/INCREMENTAL_INDEXING.md | 127 ++++++++ docs/MCP_CONTRACT.md | 3 + docs/USER_MANUAL.md | 29 ++ pyproject.toml | 2 +- src/docforge/__init__.py | 2 +- src/docforge/adapter_contract.py | 474 ++++++++++++++++++++++------- src/docforge/adapter_validation.py | 334 ++++++++++++++++++++ src/docforge/changesets.py | 27 ++ src/docforge/incremental.py | 176 +++++++++++ src/docforge/index.py | 82 ++++- src/docforge/mcp_server.py | 26 +- src/docforge/models.py | 69 ++++- tests/test_adapter_contract.py | 221 ++++++++++++++ tests/test_changesets.py | 40 +++ tests/test_mcp_server.py | 45 ++- toreview.md | 5 +- uv.lock | 2 +- 20 files changed, 1592 insertions(+), 122 deletions(-) create mode 100644 docs/INCREMENTAL_INDEXING.md create mode 100644 src/docforge/adapter_validation.py create mode 100644 src/docforge/incremental.py diff --git a/README.md b/README.md index 6042cdc..c72ec15 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,9 @@ declared manuals, visualizes project structure, and manages reviewable documenta - Exposes project-bound CLI and MCP query surfaces. - Creates, validates, diffs, and previews isolated changesets. - Applies one explicitly approved changeset hash through CLI or gated MCP. +- Supports opt-in incremental adapters with reverse-dependency invalidation and full-build + equivalence checks. +- Keeps function-scoped control-flow projections separate from the primary architecture graph. - Runs a managed loopback graph browser with neighborhood, semantic Flow, convergence Web, source inspection, and branch-aware node hiding. - Supports generic documentation projects and project-owned source adapters. @@ -88,6 +91,8 @@ hash-bound `docforge_apply_changeset` tool. - [Viewer manager](docs/VIEWER_MANAGER.md) — native service setup and lifecycle. - [Adapter decision](docs/APPLICATION_DECISION.md) — why custom adapters own canonical serialization. +- [Incremental adapter indexing](docs/INCREMENTAL_INDEXING.md) — source-scoped extraction, + invalidation, equivalence, relationship changes, and the lazy Logic boundary. ## Development diff --git a/SLICE_HISTORY.md b/SLICE_HISTORY.md index 0bde1fb..e0d5a1d 100644 --- a/SLICE_HISTORY.md +++ b/SLICE_HISTORY.md @@ -1,5 +1,29 @@ # Completed slices +## Dev-Rewrite incremental compiler boundary + +### Changed + +- Added an opt-in source-scoped adapter manifest and extraction contract while preserving Release + 1 complete projections. +- Added persistent extraction caching with fingerprint, path, extractor-version, dependency, and + project/adapter identity invalidation. +- Added reverse-dependency invalidation for added, changed, renamed, deleted, and dependency-changed + sources. +- Added manifest-only stale-state checks so normal MCP reads do not reconstruct the complete + projection. +- Added full/incremental equivalence verification and deterministic build metrics. +- Added first-class relationship-only proposal updates over the existing hash-bound projector. +- Added a lazy function-scoped logic-projection boundary outside the primary architecture graph. + +### Verification + +- Tests cover cache hits, reverse invalidation, renames, dependency changes, deletion, corrupt + caches, failed extraction, atomic preservation, manifest-only stale checks, relationship-only + proposals, lazy logic persistence, and full-build equivalence. +- Strict Pyright, Ruff, formatting, compilation, warning-strict tests, web checks, dependency + audits, source/wheel builds, and isolated wheel installation pass. + ## Release 1.0.0 stable product boundary ### Changed diff --git a/docs/CONTRACT.md b/docs/CONTRACT.md index 59600ef..6ebb037 100644 --- a/docs/CONTRACT.md +++ b/docs/CONTRACT.md @@ -1,4 +1,4 @@ -# DocForge 0.14 contract +# DocForge 1.1 development contract ## Authority boundary @@ -19,7 +19,8 @@ commit when Git is available; it cannot change repository state. - Result envelope: `schemas/result.schema.json`, version 1. - Changeset schema: `schemas/changeset.schema.json`, version 1. - Index schema: version 1, disposable and reproducible. -- Core, CLI, and MCP server: version 1.0.0. +- Core, CLI, and MCP server: version 1.1.0.dev0 on `Dev-Rewrite`. +- Incremental extraction cache: version 1, disposable and reproducible. Schema files describe the generic interchange contract. Runtime validation remains responsible for path confinement, source hashing, relationship resolution, dependency cycles, project limits, stale @@ -48,6 +49,10 @@ operation names its expected base hash. A move preserves the stable node ID. A d every incident relationship. Proposal validation and storage are atomic. Application requires the exact final changeset hash; prose is never auto-merged. +Relationship-only additions and removals use the validated update operation without changing node +metadata or content. They remain bound to the complete changeset base hash and the anchor node's +expected content hash. + The MCP process binds to one configured writer identity at startup. The project descriptor grants that writer explicit families and operation types. A changeset records its creator, project root fingerprint, base revision, canonical source hash, and ordered operations. Every append requires the @@ -200,3 +205,15 @@ An explicit integration may construct the full fixed MCP surface for a configure and one startup-bound writer. Canonical application is registered only when the integration also supplies a startup-bound applier identity and project-owned `CanonicalApplier`. An adapter without proposal settings or validation remains read-only. + +An adapter may additionally implement the opt-in incremental contract. Its manifest inventories +stable source IDs, fingerprints, extractor versions, and source dependencies without parsing the +complete project. Each extraction owns deterministic nodes, relationships, and optional +function-scoped logic. Added, changed, deleted, and reverse-dependent sources are invalidated. +Cached and refreshed facts are always assembled into a complete projection and pass normal graph +validation before publication. The full projection loader remains the fallback and equivalence +oracle. + +Logic projections are not primary graph nodes. They remain source-scoped, function-owned, +independently cached control-flow data so ordinary search, Nodes, Flow, and Web do not become +statement graphs. diff --git a/docs/INCREMENTAL_INDEXING.md b/docs/INCREMENTAL_INDEXING.md new file mode 100644 index 0000000..bfeca43 --- /dev/null +++ b/docs/INCREMENTAL_INDEXING.md @@ -0,0 +1,127 @@ +# Incremental Adapter Indexing + +DocForge Release 1 adapters return one complete immutable projection. That contract remains +supported. The `Dev-Rewrite` compiler adds an opt-in source-scoped contract that avoids reparsing +unchanged files while preserving the same validated, atomically published graph. + +## Safety model + +Incremental indexing is an extraction optimization. It does not weaken publication: + +1. The adapter returns a cheap, deterministic `AdapterManifest`. +2. DocForge compares every source fingerprint and extractor version with the last cache generation. +3. Added, changed, deleted, and reverse-dependent sources are invalidated. +4. The adapter reparses only invalidated sources. +5. DocForge assembles cached and refreshed contributions into a complete candidate projection. +6. The complete graph passes the same validation as a full adapter projection. +7. DocForge rereads the manifest to prove sources remained stable. +8. The extraction cache and SQLite graph are published with atomic file replacement. + +An interrupted extraction never replaces the last validated SQLite index. A malformed, +incompatible, or missing cache is a cache miss, not a partial graph. + +## Adapter contract + +An incremental loader implements all three methods: + +```python +class MyAdapter: + def load_manifest(self) -> AdapterManifest: ... + def extract_source(self, source: AdapterSource) -> AdapterSourceProjection: ... + def load_projection(self) -> AdapterProjection: ... +``` + +`load_projection()` remains the deterministic full-rebuild fallback and equivalence oracle. + +Each `AdapterSource` declares: + +- A stable source ID. +- A safe project-relative source path. +- A SHA-256 content fingerprint. +- An extractor version. +- Other source IDs whose changes can alter this source's extracted facts. + +Each `AdapterSourceProjection` owns: + +- Its primary graph nodes. +- Its primary graph relationships, including cross-source relationships owned by that source. +- Optional function-scoped logic projections. + +Ownership must be deterministic. Two sources may not produce the same primary node or the same +function logic projection. + +## Invalidation + +DocForge invalidates a source when: + +- It is new. +- Its fingerprint changed. +- Its extractor version changed. +- A declared dependency was added, changed, or deleted. +- Any source in its reverse-dependency chain was invalidated. + +Deleted sources are omitted from the candidate projection. Their cached dependency declarations +remain available long enough to invalidate surviving dependents. + +If an adapter cannot precisely describe the affected sources, it should declare broader +dependencies or change its adapter/extractor version. Incorrectly retaining a stale relationship +is never an acceptable optimization. + +## Build reporting + +`build` and `reindex` include an extraction report: + +```json +{ + "build": { + "mode": "incremental", + "cache_hits": 391, + "reparsed_sources": 3, + "invalidated_sources": 3, + "deleted_sources": 0, + "total_sources": 394, + "cache_hit_ids": ["..."], + "reparsed_source_ids": ["..."] + } +} +``` + +Adapters can call `AdapterProject.verify_incremental_equivalence()` in release and contract tests. +The check compares project identity, revision, source hash, nodes, and relationships against +`load_projection()`. + +## Manual changes and relationships + +Changesets remain an approval queue, not a compiler queue. Compilation never silently applies a +proposal. + +After explicit application: + +1. The project-owned applier updates canonical files. +2. Changed manual files receive new fingerprints. +3. Incremental extraction reparses those files and affected dependents. +4. The complete candidate graph is validated and published. +5. Declared renders are regenerated. + +`docforge_propose_relationship_update` queues relationship-only additions and removals without +rewriting node content. It is still bound to the changeset's complete base source hash and the +anchor node's expected content hash. + +## Lazy logic boundary + +`LogicProjection` stores control flow separately from the primary architecture graph. It is owned +by one function or method node and one source extraction. + +Logic nodes can represent entries, conditions, basic blocks, calls, merges, loops, returns, and +raises. Logic edges retain relation, display label, and deterministic ordinal. Adapters may leave +logic empty until they implement a language analyzer. + +This boundary prevents thousands of boolean expressions and basic blocks from polluting Nodes, +Flow, Web, ordinary search, or architectural traversal. A future Logic view can request one +function-scoped projection on demand. + +## Full rebuilds + +Full rebuilds remain mandatory as a fallback and equivalence oracle. Change the adapter version, +extractor version, or cache schema whenever old cached facts are no longer valid. Removing the +confined extraction cache also forces a clean reparse without affecting canonical files. diff --git a/docs/MCP_CONTRACT.md b/docs/MCP_CONTRACT.md index af10059..4674665 100644 --- a/docs/MCP_CONTRACT.md +++ b/docs/MCP_CONTRACT.md @@ -49,6 +49,7 @@ gate. - `docforge_propose_node_create` - `docforge_propose_node_update` - `docforge_propose_node_move` +- `docforge_propose_relationship_update` - `docforge_propose_node_delete` - `docforge_validate_changeset` - `docforge_get_changeset_diff` @@ -58,6 +59,8 @@ Proposal tools may write only below the configured changeset or isolated preview change canonical files or declared project output. Without `--proposal-writer`, changeset mutation tools return `proposal_access_disabled`. Validation, diff retrieval, and preview remain available for existing changesets. A preview accepts a declared view ID, not a renderer name or command. +The relationship-update tool queues additions and removals without rewriting node content and +rejects an empty relationship list. ## Canonical application tool diff --git a/docs/USER_MANUAL.md b/docs/USER_MANUAL.md index 47888fc..6ead37a 100644 --- a/docs/USER_MANUAL.md +++ b/docs/USER_MANUAL.md @@ -16,7 +16,10 @@ declared rendering, and the Nodes/Flow/Web visualization model documented below. - Disposable SQLite indexing with lexical search, filters, backlinks, dependencies, and impact. - Bounded context profiles for AI agents, including source paths and content hashes. - Isolated, optimistic changesets with create, update, move, delete, validation, diffs, and previews. +- Relationship-only changeset operations that do not rewrite node content. - Hash-bound canonical application through both CLI and an explicitly enabled MCP tool. +- Opt-in incremental adapter extraction with reverse-dependency invalidation. +- Lazy function-scoped logic projections that do not densify the primary graph. - Declared HTML render views. Arbitrary templates, render commands, and output paths are rejected. - A loopback-only graph browser with Nodes, semantic Flow, and convergence Web views, relationship keys, source inspection, branch-aware node hiding, panel resizing, zooming, and @@ -463,6 +466,7 @@ Example MCP client configuration: - `docforge_propose_node_create` - `docforge_propose_node_update` - `docforge_propose_node_move` +- `docforge_propose_relationship_update` - `docforge_propose_node_delete` - `docforge_validate_changeset` - `docforge_get_changeset_diff` @@ -487,9 +491,34 @@ Recommended agent sequence: 7. Call `docforge_apply_changeset` with that exact hash. 8. Report changed canonical files and derived refresh results. +Use `docforge_propose_relationship_update` when the intended change is only an edge addition or +removal. It uses the same underlying validated update contract, but rejects empty relationship +lists and makes it explicit that node content will remain unchanged. + Custom adapters may expose the application tool only when they supply a project-owned `CanonicalApplier`. Core DocForge will not guess how adapter nodes map back to canonical sources. +## Incremental adapter compilation + +Release 1 complete-projection adapters remain supported. Adapters with large source trees can +implement the optional source-scoped manifest and extraction contract. DocForge then fingerprints +sources, reuses unchanged facts, reparses changed sources and their reverse dependents, validates a +complete candidate graph, and publishes the index atomically. + +Build results report cache hits, reparsed sources, invalidated sources, deleted sources, and total +sources. A full projection remains the fallback and equivalence oracle. + +Manual proposals remain separate from compilation. Applying an approved changeset updates +canonical sources first. Incremental compilation then notices those changed source fingerprints; +it never treats an unapplied proposal as canonical. + +Function-scoped `LogicProjection` data is cached alongside its owning source but remains separate +from the primary Nodes, Flow, and Web graph. This is the storage boundary for a future boolean and +control-flow view without adding every condition and basic block to ordinary graph traversal. + +See [Incremental Adapter Indexing](INCREMENTAL_INDEXING.md) for the complete contract, cache +invalidation rules, manual-application lifecycle, and lazy Logic boundary. + ## Troubleshooting ### `stale_index` or `visualization_stale` diff --git a/pyproject.toml b/pyproject.toml index 96a6163..b48243d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "docforge" -version = "1.0.0" +version = "1.1.0.dev0" description = "Project-scoped documentation indexing and context service" readme = "README.md" requires-python = ">=3.12" diff --git a/src/docforge/__init__.py b/src/docforge/__init__.py index eb3a2e0..783f59a 100644 --- a/src/docforge/__init__.py +++ b/src/docforge/__init__.py @@ -11,4 +11,4 @@ __all__ = [ "GenericCanonicalApplier", "Project", ] -__version__ = "1.0.0" +__version__ = "1.1.0.dev0" diff --git a/src/docforge/adapter_contract.py b/src/docforge/adapter_contract.py index 23c1a59..0c24f35 100644 --- a/src/docforge/adapter_contract.py +++ b/src/docforge/adapter_contract.py @@ -7,20 +7,35 @@ import json from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path -from typing import Protocol +from typing import Protocol, runtime_checkable -from .config_validation import AUTHORITIES, ID_PATTERN +from .adapter_validation import ( + source_payload, + source_projection, + validate_manifest, + validate_projection, + validate_source_projection, +) +from .config_validation import ID_PATTERN from .errors import DocForgeError +from .incremental import ( + CachedSource, + ExtractionCache, + affected_sources, + load_extraction_cache, + write_extraction_cache, +) from .models import ( Edge, Limits, + LogicProjection, Node, ProjectDescriptor, ProjectSnapshot, + ProjectState, ProposalWriter, RenderConfig, ) -from .project import validate_graph @dataclass(frozen=True) @@ -79,12 +94,63 @@ class AdapterProjection: return hashlib.sha256(encoded).hexdigest() +@dataclass(frozen=True) +class AdapterSource: + """One fingerprinted extraction unit declared by an incremental adapter.""" + + source_id: str + source_path: str + fingerprint: str + extractor_version: str + dependencies: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AdapterManifest: + """Cheap project identity and source inventory for incremental compilation.""" + + project_id: str + title: str + adapter_id: str + adapter_version: str + root: Path + revision: str + source_hash: str + families: tuple[str, ...] + allowed_relations: tuple[str, ...] + sources: tuple[AdapterSource, ...] + estimated_nodes: int = 10_000 + + def identity(self) -> tuple[str, str, str, Path]: + return self.project_id, self.adapter_id, self.adapter_version, self.root + + +@dataclass(frozen=True) +class AdapterSourceProjection: + """Nodes, relationships, and optional logic owned by one source unit.""" + + source_id: str + fingerprint: str + nodes: tuple[AdapterNode, ...] + edges: tuple[AdapterEdge, ...] + logic: tuple[LogicProjection, ...] = () + + class AdapterLoader(Protocol): """Load one current, deterministic, project-confined adapter projection.""" def load_projection(self) -> AdapterProjection: ... +@runtime_checkable +class IncrementalAdapterLoader(AdapterLoader, Protocol): + """Opt-in adapter contract for cacheable, source-scoped extraction.""" + + def load_manifest(self) -> AdapterManifest: ... + + def extract_source(self, source: AdapterSource) -> AdapterSourceProjection: ... + + ProposalValidator = Callable[ [ ProjectSnapshot, @@ -110,86 +176,6 @@ class AdapterProjectSettings: proposal_validator: ProposalValidator | None = None -def validate_projection(projection: AdapterProjection) -> None: - """Validate generic invariants without interpreting adapter metadata.""" - - root = projection.root.resolve(strict=True) - if not root.is_dir() or projection.root != root: - raise DocForgeError("invalid_adapter", "Adapter root must be a resolved directory") - for label, value in ( - ("project_id", projection.project_id), - ("title", projection.title), - ("adapter_id", projection.adapter_id), - ("adapter_version", projection.adapter_version), - ("revision", projection.revision), - ("source_hash", projection.source_hash), - ): - if not value.strip(): - raise DocForgeError("invalid_adapter", f"Adapter {label} must not be empty") - if ID_PATTERN.fullmatch(projection.project_id) is None: - raise DocForgeError("invalid_adapter", "Adapter project ID is invalid") - if len(projection.source_hash) != 64 or any( - character not in "0123456789abcdef" for character in projection.source_hash - ): - raise DocForgeError("invalid_adapter", "Adapter source hash must be lowercase SHA-256") - ordered_nodes = tuple(sorted(projection.nodes, key=lambda item: item.node.node_id)) - ordered_edges = tuple( - sorted( - projection.edges, - key=lambda item: ( - item.edge.source_id, - item.edge.relation, - item.edge.target_id, - ), - ) - ) - if projection.nodes != ordered_nodes or projection.edges != ordered_edges: - raise DocForgeError( - "invalid_adapter", "Adapter projection must be deterministically ordered" - ) - for item in (*projection.nodes, *projection.edges): - keys = [key for key, _ in item.metadata] - if keys != sorted(keys) or len(keys) != len(set(keys)): - raise DocForgeError( - "invalid_adapter", "Adapter metadata keys must be unique and ordered" - ) - for item in projection.nodes: - node = item.node - source = Path(node.source_path) - if ID_PATTERN.fullmatch(node.node_id) is None: - raise DocForgeError("invalid_adapter", "Adapter node ID is invalid", id=node.node_id) - if node.authority not in AUTHORITIES: - raise DocForgeError( - "invalid_adapter", "Adapter node authority is invalid", id=node.node_id - ) - if ( - not node.title.strip() - or not node.family.strip() - or not node.status.strip() - or not node.summary.strip() - or not node.content.strip() - ): - raise DocForgeError( - "invalid_adapter", "Adapter node has empty required content", id=node.node_id - ) - if source.is_absolute() or ".." in source.parts or not node.source_path: - raise DocForgeError( - "invalid_adapter", "Adapter node source path is unsafe", id=node.node_id - ) - if len(node.tags) != len(set(node.tags)) or any(not tag for tag in node.tags): - raise DocForgeError("invalid_adapter", "Adapter node tags are invalid", id=node.node_id) - if len(node.content_hash) != 64 or any( - character not in "0123456789abcdef" for character in node.content_hash - ): - raise DocForgeError( - "invalid_adapter", "Adapter node content hash is invalid", id=node.node_id - ) - for item in projection.edges: - if ID_PATTERN.fullmatch(item.edge.relation) is None: - raise DocForgeError("invalid_adapter", "Adapter relationship type is invalid") - validate_graph(projection.core_nodes(), projection.core_edges()) - - class AdapterProject: """Expose a validated adapter projection through the standard index boundary.""" @@ -202,9 +188,45 @@ class AdapterProject: ) -> None: self.loader = loader self.settings = settings or AdapterProjectSettings() - initial = loader.load_projection() - validate_projection(initial) - root = initial.root + self._incremental_loader = loader if isinstance(loader, IncrementalAdapterLoader) else None + initial: AdapterProjection | None = None + manifest: AdapterManifest | None = None + if self._incremental_loader is not None: + manifest = self._incremental_loader.load_manifest() + validate_manifest(manifest) + root = manifest.root + project_id = manifest.project_id + title = manifest.title + adapter_id = manifest.adapter_id + adapter_version = manifest.adapter_version + descriptor_hash = hashlib.sha256( + json.dumps( + { + "project_id": manifest.project_id, + "adapter_id": manifest.adapter_id, + "adapter_version": manifest.adapter_version, + "families": manifest.families, + "allowed_relations": manifest.allowed_relations, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + families = set(manifest.families) + allowed_relations = manifest.allowed_relations + estimated_nodes = manifest.estimated_nodes + else: + initial = loader.load_projection() + validate_projection(initial) + root = initial.root + project_id = initial.project_id + title = initial.title + adapter_id = initial.adapter_id + adapter_version = initial.adapter_version + descriptor_hash = initial.identity() + families = {item.node.family for item in initial.nodes} + allowed_relations = tuple(sorted({item.edge.relation for item in initial.edges})) + estimated_nodes = len(initial.nodes) resolved_cache = cache_root.resolve(strict=False) if ( resolved_cache == root @@ -215,11 +237,21 @@ class AdapterProject: "path_escape", "Adapter cache must be a confined project subdirectory" ) self._identity = ( - initial.project_id, - initial.adapter_id, - initial.adapter_version, - initial.root, + project_id, + adapter_id, + adapter_version, + root, ) + self._cache_path = resolved_cache / "extractions.json" + self._last_build_report: dict[str, object] = { + "mode": "full", + "cache_hits": 0, + "reparsed_sources": 0, + "invalidated_sources": 0, + "deleted_sources": 0, + "total_sources": 0, + } + self._last_logic: tuple[LogicProjection, ...] = () content_roots = self._resolved_directories( root, self.settings.content_roots, label="content root" ) @@ -264,19 +296,19 @@ class AdapterProject: "Proposal-enabled adapters require a confined descriptor file", ) limits = self.settings.limits or Limits( - max_nodes=max(10_000, len(initial.nodes)), + max_nodes=max(10_000, estimated_nodes), max_context_tokens=64_000, ) - self._validate_proposal_writers(initial, self.settings.proposal_writers) + self._validate_proposal_writers(families, self.settings.proposal_writers) self._validate_render(root, self.settings.render, content_roots, changeset_root) self.descriptor = ProjectDescriptor( schema_version=1, - project_id=initial.project_id, - title=initial.title, - adapter=f"{initial.adapter_id}@{initial.adapter_version}", + project_id=project_id, + title=title, + adapter=f"{adapter_id}@{adapter_version}", root=root, descriptor_path=descriptor_path, - descriptor_hash=initial.identity(), + descriptor_hash=descriptor_hash, content_roots=content_roots, authority_files=authority_files, cache_root=resolved_cache, @@ -284,7 +316,7 @@ class AdapterProject: changeset_root=changeset_root, proposal_writers=self.settings.proposal_writers, render=self.settings.render, - allowed_relations=tuple(sorted({item.edge.relation for item in initial.edges})), + allowed_relations=allowed_relations, profiles=(), limits=limits, ) @@ -293,7 +325,11 @@ class AdapterProject: def load(self) -> ProjectSnapshot: canonical_sources = self.canonical_source_paths() captured = {path: path.read_bytes() for path in canonical_sources} - projection = self.loader.load_projection() + projection = ( + self._load_incremental() + if self._incremental_loader is not None + else self.loader.load_projection() + ) validate_projection(projection) identity = ( projection.project_id, @@ -309,15 +345,12 @@ class AdapterProject: raise DocForgeError( "source_changed", "Adapter canonical source changed during the operation" ) - source_hash = projection.source_hash - if captured: - digest = hashlib.sha256(projection.source_hash.encode("ascii")) - for path in canonical_sources: - relative = path.relative_to(projection.root).as_posix().encode() - digest.update(len(relative).to_bytes(8, "big")) - digest.update(relative) - digest.update(hashlib.sha256(captured[path]).digest()) - source_hash = digest.hexdigest() + source_hash = self._combined_source_hash( + projection.source_hash, + projection.root, + canonical_sources, + captured, + ) return ProjectSnapshot( descriptor=self.descriptor, nodes=projection.core_nodes(), @@ -326,6 +359,211 @@ class AdapterProject: revision=projection.revision, ) + def incremental_state(self) -> ProjectState | None: + """Return current source identity without reconstructing the complete projection.""" + + loader = self._incremental_loader + if loader is None: + return None + canonical_sources = self.canonical_source_paths() + captured = {path: path.read_bytes() for path in canonical_sources} + manifest = loader.load_manifest() + validate_manifest(manifest) + if manifest.identity() != self._identity: + raise DocForgeError("adapter_changed", "Adapter identity changed during the operation") + if self.canonical_source_paths() != canonical_sources or any( + not path.is_file() or path.read_bytes() != raw for path, raw in captured.items() + ): + raise DocForgeError( + "source_changed", "Adapter canonical source changed during the operation" + ) + return ProjectState( + source_hash=self._combined_source_hash( + manifest.source_hash, + manifest.root, + canonical_sources, + captured, + ), + revision=manifest.revision, + ) + + def build_report(self) -> dict[str, object]: + """Return deterministic extraction metrics from the most recent load.""" + + return dict(self._last_build_report) + + def logic_projection(self, owner_node_id: str) -> LogicProjection | None: + """Load one lazily stored function-scoped logic projection.""" + + self.load() + return next( + ( + projection + for projection in self._last_logic + if projection.owner_node_id == owner_node_id + ), + None, + ) + + def verify_incremental_equivalence(self) -> dict[str, object]: + """Prove the incremental and full loader contracts produce the same graph.""" + + if self._incremental_loader is None: + raise DocForgeError( + "incremental_disabled", "Adapter does not implement incremental extraction" + ) + incremental = self._load_incremental() + full = self.loader.load_projection() + validate_projection(full) + fields = { + "project_id": incremental.project_id == full.project_id, + "adapter_id": incremental.adapter_id == full.adapter_id, + "adapter_version": incremental.adapter_version == full.adapter_version, + "revision": incremental.revision == full.revision, + "source_hash": incremental.source_hash == full.source_hash, + "nodes": incremental.nodes == full.nodes, + "edges": incremental.edges == full.edges, + } + mismatches = [field for field, matches in fields.items() if not matches] + if mismatches: + raise DocForgeError( + "incremental_mismatch", + "Incremental extraction does not match a full adapter projection", + fields=mismatches, + ) + return { + "status": "ok", + "project_id": incremental.project_id, + "revision": incremental.revision, + "source_hash": incremental.source_hash, + "node_count": len(incremental.nodes), + "edge_count": len(incremental.edges), + } + + def _load_incremental(self) -> AdapterProjection: + loader = self._incremental_loader + if loader is None: + raise DocForgeError("incremental_disabled", "Incremental adapter is not configured") + manifest = loader.load_manifest() + validate_manifest(manifest) + if manifest.identity() != self._identity: + raise DocForgeError("adapter_changed", "Adapter identity changed during the operation") + cache = load_extraction_cache( + self._cache_path, + project_id=manifest.project_id, + adapter_id=manifest.adapter_id, + adapter_version=manifest.adapter_version, + ) + cached = {source.source_id: source for source in cache.sources} if cache else {} + current = {source.source_id: source for source in manifest.sources} + changed = { + source_id + for source_id, source in current.items() + if source_id not in cached + or cached[source_id].source_path != source.source_path + or cached[source_id].fingerprint != source.fingerprint + or cached[source_id].extractor_version != source.extractor_version + or cached[source_id].dependencies != source.dependencies + } + deleted = set(cached) - set(current) + invalidated = affected_sources( + current_dependencies={ + source.source_id: source.dependencies for source in manifest.sources + }, + cached_dependencies={ + source.source_id: source.dependencies for source in cached.values() + }, + changed=changed | deleted, + ) + contributions: list[AdapterSourceProjection] = [] + cache_records: list[CachedSource] = [] + reparsed: list[str] = [] + hits: list[str] = [] + for source in manifest.sources: + if source.source_id in invalidated: + contribution = loader.extract_source(source) + reparsed.append(source.source_id) + else: + record = cached[source.source_id] + contribution = source_projection(record.payload) + hits.append(source.source_id) + validate_source_projection(source, contribution) + contributions.append(contribution) + cache_records.append( + CachedSource( + source_id=source.source_id, + source_path=source.source_path, + fingerprint=source.fingerprint, + extractor_version=source.extractor_version, + dependencies=source.dependencies, + payload=source_payload(contribution), + ) + ) + projection = AdapterProjection( + project_id=manifest.project_id, + title=manifest.title, + adapter_id=manifest.adapter_id, + adapter_version=manifest.adapter_version, + root=manifest.root, + revision=manifest.revision, + source_hash=manifest.source_hash, + nodes=tuple( + sorted( + (node for contribution in contributions for node in contribution.nodes), + key=lambda item: item.node.node_id, + ) + ), + edges=tuple( + sorted( + (edge for contribution in contributions for edge in contribution.edges), + key=lambda item: ( + item.edge.source_id, + item.edge.relation, + item.edge.target_id, + ), + ) + ), + ) + validate_projection(projection) + logic_projections = tuple( + sorted( + (logic for contribution in contributions for logic in contribution.logic), + key=lambda projection: projection.owner_node_id, + ) + ) + owners = [projection.owner_node_id for projection in logic_projections] + if len(owners) != len(set(owners)): + raise DocForgeError( + "invalid_adapter", "A primary graph node may own only one logic projection" + ) + stable = loader.load_manifest() + validate_manifest(stable) + if stable != manifest: + raise DocForgeError( + "source_changed", "Adapter sources changed during incremental extraction" + ) + write_extraction_cache( + self._cache_path, + ExtractionCache( + project_id=manifest.project_id, + adapter_id=manifest.adapter_id, + adapter_version=manifest.adapter_version, + sources=tuple(cache_records), + ), + ) + self._last_logic = logic_projections + self._last_build_report = { + "mode": "incremental", + "cache_hits": len(hits), + "reparsed_sources": len(reparsed), + "invalidated_sources": len(invalidated & set(current)), + "deleted_sources": len(deleted), + "total_sources": len(current), + "cache_hit_ids": hits, + "reparsed_source_ids": reparsed, + } + return projection + def canonical_source_paths(self) -> tuple[Path, ...]: """Adapters validate their own source sets before producing a projection.""" @@ -335,6 +573,23 @@ class AdapterProject: label="canonical source", ) + @staticmethod + def _combined_source_hash( + projection_hash: str, + root: Path, + canonical_sources: tuple[Path, ...], + captured: dict[Path, bytes], + ) -> str: + if not captured: + return projection_hash + digest = hashlib.sha256(projection_hash.encode("ascii")) + for path in canonical_sources: + relative = path.relative_to(root).as_posix().encode() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(hashlib.sha256(captured[path]).digest()) + return digest.hexdigest() + def validate_proposal( self, base: ProjectSnapshot, @@ -391,11 +646,8 @@ class AdapterProject: return current @staticmethod - def _validate_proposal_writers( - projection: AdapterProjection, writers: tuple[ProposalWriter, ...] - ) -> None: + def _validate_proposal_writers(families: set[str], writers: tuple[ProposalWriter, ...]) -> None: writer_ids = [writer.writer_id for writer in writers] - families = {item.node.family for item in projection.nodes} if len(writer_ids) != len(set(writer_ids)): raise DocForgeError("invalid_adapter", "Adapter proposal writer IDs repeat") for writer in writers: diff --git a/src/docforge/adapter_validation.py b/src/docforge/adapter_validation.py new file mode 100644 index 0000000..f3e0468 --- /dev/null +++ b/src/docforge/adapter_validation.py @@ -0,0 +1,334 @@ +"""Validation and cache serialization for adapter-owned graph projections.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from .config_validation import AUTHORITIES, ID_PATTERN +from .errors import DocForgeError +from .models import Edge, LogicEdge, LogicNode, LogicProjection, Node +from .project import validate_graph + +if TYPE_CHECKING: + from .adapter_contract import ( + AdapterManifest, + AdapterProjection, + AdapterSource, + AdapterSourceProjection, + ) + + +def validate_projection(projection: AdapterProjection) -> None: + """Validate generic invariants without interpreting adapter metadata.""" + + root = projection.root.resolve(strict=True) + if not root.is_dir() or projection.root != root: + raise DocForgeError("invalid_adapter", "Adapter root must be a resolved directory") + for label, value in ( + ("project_id", projection.project_id), + ("title", projection.title), + ("adapter_id", projection.adapter_id), + ("adapter_version", projection.adapter_version), + ("revision", projection.revision), + ("source_hash", projection.source_hash), + ): + if not value.strip(): + raise DocForgeError("invalid_adapter", f"Adapter {label} must not be empty") + if ID_PATTERN.fullmatch(projection.project_id) is None: + raise DocForgeError("invalid_adapter", "Adapter project ID is invalid") + validate_sha256(projection.source_hash, label="source hash") + ordered_nodes = tuple(sorted(projection.nodes, key=lambda item: item.node.node_id)) + ordered_edges = tuple( + sorted( + projection.edges, + key=lambda item: ( + item.edge.source_id, + item.edge.relation, + item.edge.target_id, + ), + ) + ) + if projection.nodes != ordered_nodes or projection.edges != ordered_edges: + raise DocForgeError( + "invalid_adapter", "Adapter projection must be deterministically ordered" + ) + for item in (*projection.nodes, *projection.edges): + keys = [key for key, _ in item.metadata] + if keys != sorted(keys) or len(keys) != len(set(keys)): + raise DocForgeError( + "invalid_adapter", "Adapter metadata keys must be unique and ordered" + ) + for item in projection.nodes: + node = item.node + source = Path(node.source_path) + if ID_PATTERN.fullmatch(node.node_id) is None: + raise DocForgeError("invalid_adapter", "Adapter node ID is invalid", id=node.node_id) + if node.authority not in AUTHORITIES: + raise DocForgeError( + "invalid_adapter", "Adapter node authority is invalid", id=node.node_id + ) + if ( + not node.title.strip() + or not node.family.strip() + or not node.status.strip() + or not node.summary.strip() + or not node.content.strip() + ): + raise DocForgeError( + "invalid_adapter", "Adapter node has empty required content", id=node.node_id + ) + if source.is_absolute() or ".." in source.parts or not node.source_path: + raise DocForgeError( + "invalid_adapter", "Adapter node source path is unsafe", id=node.node_id + ) + if len(node.tags) != len(set(node.tags)) or any(not tag for tag in node.tags): + raise DocForgeError("invalid_adapter", "Adapter node tags are invalid", id=node.node_id) + validate_sha256(node.content_hash, label="node content hash") + for item in projection.edges: + if ID_PATTERN.fullmatch(item.edge.relation) is None: + raise DocForgeError("invalid_adapter", "Adapter relationship type is invalid") + validate_graph(projection.core_nodes(), projection.core_edges()) + + +def validate_manifest(manifest: AdapterManifest) -> None: + """Validate a cheap incremental manifest without parsing project sources.""" + + root = manifest.root.resolve(strict=True) + if manifest.root != root or not root.is_dir(): + raise DocForgeError("invalid_adapter", "Adapter root must be a resolved directory") + for label, value in ( + ("project_id", manifest.project_id), + ("title", manifest.title), + ("adapter_id", manifest.adapter_id), + ("adapter_version", manifest.adapter_version), + ("revision", manifest.revision), + ("source_hash", manifest.source_hash), + ): + if not value.strip(): + raise DocForgeError("invalid_adapter", f"Adapter {label} must not be empty") + if ID_PATTERN.fullmatch(manifest.project_id) is None: + raise DocForgeError("invalid_adapter", "Adapter project ID is invalid") + validate_sha256(manifest.source_hash, label="source hash") + if manifest.estimated_nodes < 1: + raise DocForgeError("invalid_adapter", "Adapter estimated node count must be positive") + if ( + manifest.families != tuple(sorted(set(manifest.families))) + or not manifest.families + or any(not family.strip() for family in manifest.families) + ): + raise DocForgeError("invalid_adapter", "Adapter families must be unique and ordered") + if manifest.allowed_relations != tuple(sorted(set(manifest.allowed_relations))) or any( + ID_PATTERN.fullmatch(relation) is None for relation in manifest.allowed_relations + ): + raise DocForgeError( + "invalid_adapter", "Adapter relationship types must be valid and ordered" + ) + ordered = tuple(sorted(manifest.sources, key=lambda source: source.source_id)) + if manifest.sources != ordered or len({source.source_id for source in ordered}) != len(ordered): + raise DocForgeError("invalid_adapter", "Adapter sources must be unique and ordered") + paths: set[str] = set() + source_ids = {source.source_id for source in ordered} + for source in ordered: + if ID_PATTERN.fullmatch(source.source_id) is None: + raise DocForgeError("invalid_adapter", "Adapter source ID is invalid") + path = Path(source.source_path) + if ( + not source.source_path + or path.is_absolute() + or ".." in path.parts + or source.source_path in paths + ): + raise DocForgeError("invalid_adapter", "Adapter source path is unsafe or repeated") + paths.add(source.source_path) + validate_sha256(source.fingerprint, label="source fingerprint") + if not source.extractor_version.strip(): + raise DocForgeError("invalid_adapter", "Source extractor version must not be empty") + if ( + source.dependencies != tuple(sorted(set(source.dependencies))) + or source.source_id in source.dependencies + or not set(source.dependencies).issubset(source_ids) + ): + raise DocForgeError( + "invalid_adapter", "Adapter source dependencies are invalid or unordered" + ) + + +def validate_source_projection( + source: AdapterSource, contribution: AdapterSourceProjection +) -> None: + """Validate one extraction unit before it enters the reusable cache.""" + + if contribution.source_id != source.source_id or contribution.fingerprint != source.fingerprint: + raise DocForgeError( + "invalid_adapter", "Source projection identity does not match its manifest entry" + ) + if contribution.nodes != tuple( + sorted(contribution.nodes, key=lambda item: item.node.node_id) + ) or contribution.edges != tuple( + sorted( + contribution.edges, + key=lambda item: ( + item.edge.source_id, + item.edge.relation, + item.edge.target_id, + ), + ) + ): + raise DocForgeError( + "invalid_adapter", "Source projection must be deterministically ordered" + ) + for item in (*contribution.nodes, *contribution.edges): + keys = [key for key, _ in item.metadata] + if keys != sorted(keys) or len(keys) != len(set(keys)): + raise DocForgeError( + "invalid_adapter", "Adapter metadata keys must be unique and ordered" + ) + node_ids = {item.node.node_id for item in contribution.nodes} + if len(node_ids) != len(contribution.nodes): + raise DocForgeError("invalid_adapter", "Source projection contains duplicate nodes") + for logic in contribution.logic: + if logic.source_id != source.source_id or logic.owner_node_id not in node_ids: + raise DocForgeError( + "invalid_adapter", "Logic projection must be owned by a node in its source" + ) + validate_logic_projection(logic) + + +def validate_logic_projection(projection: LogicProjection) -> None: + node_ids = [node.logic_id for node in projection.nodes] + if ( + projection.nodes != tuple(sorted(projection.nodes, key=lambda node: node.logic_id)) + or len(node_ids) != len(set(node_ids)) + or any(ID_PATTERN.fullmatch(node_id) is None for node_id in node_ids) + ): + raise DocForgeError("invalid_adapter", "Logic nodes must be valid, unique, and ordered") + ordered_edges = tuple( + sorted( + projection.edges, + key=lambda edge: ( + edge.source_id, + edge.ordinal, + edge.relation, + edge.target_id, + edge.label or "", + ), + ) + ) + if projection.edges != ordered_edges: + raise DocForgeError("invalid_adapter", "Logic edges must be deterministically ordered") + known = set(node_ids) + for edge in projection.edges: + if ( + edge.source_id not in known + or edge.target_id not in known + or ID_PATTERN.fullmatch(edge.relation) is None + or edge.ordinal < 0 + ): + raise DocForgeError("invalid_adapter", "Logic edge is invalid") + + +def validate_sha256(value: str, *, label: str) -> None: + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise DocForgeError("invalid_adapter", f"Adapter {label} must be lowercase SHA-256") + + +def source_payload(contribution: AdapterSourceProjection) -> dict[str, object]: + return { + "source_id": contribution.source_id, + "fingerprint": contribution.fingerprint, + "nodes": [item.as_dict() for item in contribution.nodes], + "edges": [item.as_dict() for item in contribution.edges], + "logic": [projection.as_dict() for projection in contribution.logic], + } + + +def source_projection(payload: dict[str, object]) -> AdapterSourceProjection: + from .adapter_contract import AdapterEdge, AdapterNode, AdapterSourceProjection + + try: + raw_nodes = cast(list[dict[str, object]], payload["nodes"]) + raw_edges = cast(list[dict[str, object]], payload["edges"]) + raw_logic = cast(list[dict[str, object]], payload["logic"]) + nodes = tuple( + AdapterNode( + node=node_from_dict(cast(dict[str, object], item["node"])), + metadata=tuple( + sorted( + (str(key), str(value)) + for key, value in cast(dict[str, object], item["metadata"]).items() + ) + ), + ) + for item in raw_nodes + ) + edges = tuple( + AdapterEdge( + edge=Edge(**cast(dict[str, str], item["edge"])), + metadata=tuple( + sorted( + (str(key), str(value)) + for key, value in cast(dict[str, object], item["metadata"]).items() + ) + ), + ) + for item in raw_edges + ) + logic = tuple(logic_from_dict(item) for item in raw_logic) + return AdapterSourceProjection( + source_id=str(payload["source_id"]), + fingerprint=str(payload["fingerprint"]), + nodes=nodes, + edges=edges, + logic=logic, + ) + except (AttributeError, KeyError, TypeError, ValueError) as error: + raise DocForgeError( + "invalid_cache", "Incremental extraction cache contains invalid adapter data" + ) from error + + +def node_from_dict(payload: dict[str, object]) -> Node: + return Node( + node_id=str(payload["node_id"]), + title=str(payload["title"]), + family=str(payload["family"]), + authority=str(payload["authority"]), + status=str(payload["status"]), + tags=tuple(str(value) for value in cast(list[object], payload["tags"])), + summary=str(payload["summary"]), + content=str(payload["content"]), + source_path=str(payload["source_path"]), + source_anchor=( + str(payload["source_anchor"]) if payload.get("source_anchor") is not None else None + ), + content_hash=str(payload["content_hash"]), + ) + + +def logic_from_dict(payload: dict[str, object]) -> LogicProjection: + return LogicProjection( + owner_node_id=str(payload["owner_node_id"]), + source_id=str(payload["source_id"]), + nodes=tuple( + LogicNode( + logic_id=str(item["logic_id"]), + kind=str(item["kind"]), + label=str(item["label"]), + source_anchor=( + str(item["source_anchor"]) if item.get("source_anchor") is not None else None + ), + ) + for item in cast(list[dict[str, object]], payload["nodes"]) + ), + edges=tuple( + LogicEdge( + source_id=str(item["source_id"]), + relation=str(item["relation"]), + target_id=str(item["target_id"]), + label=str(item["label"]) if item.get("label") is not None else None, + ordinal=int(cast(int, item["ordinal"])), + ) + for item in cast(list[dict[str, object]], payload["edges"]) + ), + ) diff --git a/src/docforge/changesets.py b/src/docforge/changesets.py index 91599c8..312b62f 100644 --- a/src/docforge/changesets.py +++ b/src/docforge/changesets.py @@ -158,6 +158,33 @@ class ChangesetStore: }, ) + def propose_relationship_update( + self, + *, + changeset_id: str, + expected_changeset_hash: str, + node_id: str, + expected_content_hash: str, + relationship_changes: list[dict[str, Any]], + rationale: str, + ) -> dict[str, object]: + """Queue relationship-only changes against one exact existing node.""" + + if not relationship_changes: + raise DocForgeError( + "invalid_operation", "Relationship-only updates require at least one change" + ) + return self.propose_update( + changeset_id=changeset_id, + expected_changeset_hash=expected_changeset_hash, + node_id=node_id, + expected_content_hash=expected_content_hash, + metadata=None, + content=None, + relationship_changes=relationship_changes, + rationale=rationale, + ) + def propose_delete( self, *, diff --git a/src/docforge/incremental.py b/src/docforge/incremental.py new file mode 100644 index 0000000..776111d --- /dev/null +++ b/src/docforge/incremental.py @@ -0,0 +1,176 @@ +"""Versioned, confined extraction-cache primitives for incremental adapters.""" + +from __future__ import annotations + +import json +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +from .errors import DocForgeError + +EXTRACTION_CACHE_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class CachedSource: + """One adapter-owned cached source contribution.""" + + source_id: str + source_path: str + fingerprint: str + extractor_version: str + dependencies: tuple[str, ...] + payload: dict[str, object] + + +@dataclass(frozen=True) +class ExtractionCache: + """A complete cache generation bound to one adapter identity.""" + + project_id: str + adapter_id: str + adapter_version: str + sources: tuple[CachedSource, ...] + + +def load_extraction_cache( + path: Path, + *, + project_id: str, + adapter_id: str, + adapter_version: str, +) -> ExtractionCache | None: + """Read a cache generation, treating malformed or incompatible data as a miss.""" + + if not path.is_file() or path.is_symlink(): + return None + try: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + return None + document = cast(dict[str, Any], raw) + if ( + document.get("schema_version") != EXTRACTION_CACHE_SCHEMA_VERSION + or document.get("project_id") != project_id + or document.get("adapter_id") != adapter_id + or document.get("adapter_version") != adapter_version + ): + return None + raw_sources = document.get("sources") + if not isinstance(raw_sources, list): + return None + sources: list[CachedSource] = [] + for raw_source in cast(list[object], raw_sources): + if not isinstance(raw_source, dict): + return None + item = cast(dict[str, object], raw_source) + if set(item) != { + "source_id", + "source_path", + "fingerprint", + "extractor_version", + "dependencies", + "payload", + }: + return None + source_id = item["source_id"] + source_path = item["source_path"] + fingerprint = item["fingerprint"] + extractor_version = item["extractor_version"] + dependencies = item["dependencies"] + payload = item["payload"] + if ( + not isinstance(source_id, str) + or not isinstance(source_path, str) + or not isinstance(fingerprint, str) + or not isinstance(extractor_version, str) + or not isinstance(dependencies, list) + or not all(isinstance(value, str) for value in cast(list[object], dependencies)) + or not isinstance(payload, dict) + ): + return None + sources.append( + CachedSource( + source_id=source_id, + source_path=source_path, + fingerprint=fingerprint, + extractor_version=extractor_version, + dependencies=tuple(cast(list[str], dependencies)), + payload=cast(dict[str, object], payload), + ) + ) + ordered = tuple(sorted(sources, key=lambda item: item.source_id)) + if tuple(sources) != ordered or len({item.source_id for item in ordered}) != len(ordered): + return None + return ExtractionCache(project_id, adapter_id, adapter_version, ordered) + except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError, ValueError): + return None + + +def write_extraction_cache(path: Path, cache: ExtractionCache) -> None: + """Atomically publish one validated extraction-cache generation.""" + + path.parent.mkdir(parents=True, exist_ok=True) + document = { + "schema_version": EXTRACTION_CACHE_SCHEMA_VERSION, + "project_id": cache.project_id, + "adapter_id": cache.adapter_id, + "adapter_version": cache.adapter_version, + "sources": [ + { + "source_id": source.source_id, + "source_path": source.source_path, + "fingerprint": source.fingerprint, + "extractor_version": source.extractor_version, + "dependencies": list(source.dependencies), + "payload": source.payload, + } + for source in cache.sources + ], + } + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix="extractions-", + suffix=".json", + dir=path.parent, + delete=False, + ) as descriptor: + temporary = Path(descriptor.name) + json.dump(document, descriptor, sort_keys=True, separators=(",", ":")) + descriptor.write("\n") + descriptor.flush() + os.fsync(descriptor.fileno()) + try: + os.replace(temporary, path) + except OSError as error: + temporary.unlink(missing_ok=True) + raise DocForgeError( + "cache_failure", "Could not publish the incremental extraction cache" + ) from error + + +def affected_sources( + *, + current_dependencies: dict[str, tuple[str, ...]], + cached_dependencies: dict[str, tuple[str, ...]], + changed: set[str], +) -> set[str]: + """Return the reverse dependency closure of changed, added, or deleted sources.""" + + reverse: dict[str, set[str]] = {} + for source_id, dependencies in (*cached_dependencies.items(), *current_dependencies.items()): + for dependency in dependencies: + reverse.setdefault(dependency, set()).add(source_id) + affected = set(changed) + pending = list(sorted(changed)) + while pending: + source_id = pending.pop() + for dependent in sorted(reverse.get(source_id, ())): + if dependent not in affected: + affected.add(dependent) + pending.append(dependent) + return affected diff --git a/src/docforge/index.py b/src/docforge/index.py index 781de8e..685f385 100644 --- a/src/docforge/index.py +++ b/src/docforge/index.py @@ -13,7 +13,15 @@ from contextlib import contextmanager from pathlib import Path from .errors import DocForgeError -from .models import Edge, Node, ProjectService, ProjectSnapshot +from .models import ( + BuildReportingProject, + Edge, + IncrementalStateProject, + Node, + ProjectService, + ProjectSnapshot, + ProjectState, +) from .project import project_root_fingerprint INDEX_SCHEMA_VERSION = 1 @@ -86,6 +94,11 @@ class ProjectIndex: def build(self) -> dict[str, object]: snapshot = self.project.load() status = _status(snapshot) + build_report = ( + self.project.build_report() if isinstance(self.project, BuildReportingProject) else None + ) + if build_report is not None and build_report.get("mode") != "incremental": + build_report = None cache_root = snapshot.descriptor.cache_root cache_root.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile( @@ -183,9 +196,16 @@ class ProjectIndex: except Exception: temporary.unlink(missing_ok=True) raise - return {**status, "database": str(self.path)} + result: dict[str, object] = {**status, "database": str(self.path)} + if build_report is not None: + result["build"] = build_report + return result def check(self) -> dict[str, object]: + if isinstance(self.project, IncrementalStateProject): + state = self.project.incremental_state() + if state is not None: + return self._check_incremental_state(state) snapshot = self.project.load() expected = _status(snapshot) with _read_connection(self.path) as connection: @@ -233,6 +253,64 @@ class ProjectIndex: raise DocForgeError("invalid_index", "Derived index rows do not match source") return {**expected, "database": str(self.path)} + def _check_incremental_state(self, state: ProjectState) -> dict[str, object]: + """Validate a published index against cheap current source identity.""" + + descriptor = self.project.descriptor + identity = { + "project_id": descriptor.project_id, + "project_root_fingerprint": project_root_fingerprint(descriptor.root), + "revision": state.revision, + "source_hash": state.source_hash, + "index_schema_version": INDEX_SCHEMA_VERSION, + "adapter": descriptor.adapter, + } + with _read_connection(self.path) as connection: + application_id = connection.execute("PRAGMA application_id").fetchone()[0] + schema_version = connection.execute("PRAGMA user_version").fetchone()[0] + if application_id != APPLICATION_ID or schema_version != INDEX_SCHEMA_VERSION: + raise DocForgeError("invalid_index", "Derived index has an unsupported schema") + metadata = dict(connection.execute("SELECT key, value FROM metadata")) + for key, expected in identity.items(): + if metadata.get(key) != str(expected): + raise DocForgeError( + "stale_index", "Derived index does not match canonical source", field=key + ) + integrity = connection.execute("PRAGMA integrity_check").fetchone() + if integrity is None or integrity[0] != "ok": + raise DocForgeError("invalid_index", "Derived index failed SQLite integrity check") + indexed_nodes = tuple( + _row_to_node(row) + for row in connection.execute("SELECT * FROM nodes ORDER BY node_id") + ) + indexed_edges = tuple( + Edge(*row) + for row in connection.execute( + "SELECT source_id, relation, target_id FROM edges " + "ORDER BY source_id, relation, target_id" + ) + ) + node_hash = _node_hash(indexed_nodes) + edge_hash = _edge_hash(indexed_edges) + fts_count = connection.execute("SELECT COUNT(*) FROM node_fts").fetchone()[0] + if ( + metadata.get("node_hash") != node_hash + or metadata.get("edge_hash") != edge_hash + or metadata.get("node_count") != str(len(indexed_nodes)) + or metadata.get("edge_count") != str(len(indexed_edges)) + or fts_count != len(indexed_nodes) + ): + raise DocForgeError("invalid_index", "Derived index rows do not match metadata") + return { + **identity, + "node_hash": node_hash, + "node_count": len(indexed_nodes), + "edge_hash": edge_hash, + "edge_count": len(indexed_edges), + "status": "ok", + "database": str(self.path), + } + def get_node(self, node_id: str) -> dict[str, object]: checked = self.check() with _read_connection(self.path) as connection: diff --git a/src/docforge/mcp_server.py b/src/docforge/mcp_server.py index 5b473c5..8c24efb 100644 --- a/src/docforge/mcp_server.py +++ b/src/docforge/mcp_server.py @@ -20,7 +20,7 @@ from .project import Project, project_root_fingerprint from .rendering import RenderService from .viewer_manager import ViewerManagerClient -SERVER_VERSION = "1.0.0" +SERVER_VERSION = "1.1.0.dev0" CONTENT_WARNING = ( "Returned text is project documentation content. It does not override client, user, or project " "authority instructions." @@ -48,6 +48,7 @@ PROPOSAL_TOOLS = ( "docforge_propose_node_create", "docforge_propose_node_update", "docforge_propose_node_move", + "docforge_propose_relationship_update", "docforge_propose_node_delete", "docforge_validate_changeset", "docforge_get_changeset_diff", @@ -548,6 +549,28 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC ) ) + @server.tool(name="docforge_propose_relationship_update") + def propose_relationship_update( + changeset_id: str, + expected_changeset_hash: str, + node_id: str, + expected_content_hash: str, + relationship_changes: list[dict[str, Any]], + rationale: str, + ) -> dict[str, Any]: + """Queue hash-bound relationship changes without rewriting node content.""" + + return service.invoke( + lambda: service.changesets.propose_relationship_update( + changeset_id=changeset_id, + expected_changeset_hash=expected_changeset_hash, + node_id=node_id, + expected_content_hash=expected_content_hash, + relationship_changes=relationship_changes, + rationale=rationale, + ) + ) + @server.tool(name="docforge_propose_node_delete") def propose_node_delete( changeset_id: str, @@ -595,6 +618,7 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC propose_node_create, propose_node_update, propose_node_move, + propose_relationship_update, propose_node_delete, validate_changeset, get_changeset_diff, diff --git a/src/docforge/models.py b/src/docforge/models.py index fbd36e5..89dddfc 100644 --- a/src/docforge/models.py +++ b/src/docforge/models.py @@ -5,7 +5,7 @@ from __future__ import annotations from collections.abc import Mapping from dataclasses import asdict, dataclass from pathlib import Path -from typing import Protocol +from typing import Protocol, runtime_checkable @dataclass(frozen=True) @@ -111,6 +111,51 @@ class Edge: return asdict(self) +@dataclass(frozen=True) +class LogicNode: + """One function-scoped control-flow node kept outside the primary graph.""" + + logic_id: str + kind: str + label: str + source_anchor: str | None + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +@dataclass(frozen=True) +class LogicEdge: + """One directed control-flow transition with an explicit branch label.""" + + source_id: str + relation: str + target_id: str + label: str | None = None + ordinal: int = 0 + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +@dataclass(frozen=True) +class LogicProjection: + """A lazy control-flow projection owned by one primary graph symbol.""" + + owner_node_id: str + source_id: str + nodes: tuple[LogicNode, ...] + edges: tuple[LogicEdge, ...] + + def as_dict(self) -> dict[str, object]: + return { + "owner_node_id": self.owner_node_id, + "source_id": self.source_id, + "nodes": [node.as_dict() for node in self.nodes], + "edges": [edge.as_dict() for edge in self.edges], + } + + @dataclass(frozen=True) class ProjectSnapshot: descriptor: ProjectDescriptor @@ -120,6 +165,14 @@ class ProjectSnapshot: revision: str +@dataclass(frozen=True) +class ProjectState: + """Cheap canonical identity used to prove a derived snapshot is current.""" + + source_hash: str + revision: str + + class ProjectService(Protocol): """Minimum immutable project boundary required by derived read services.""" @@ -137,6 +190,20 @@ class ProjectService(Protocol): ) -> None: ... +@runtime_checkable +class BuildReportingProject(ProjectService, Protocol): + """Optional project boundary exposing extraction metrics for builds.""" + + def build_report(self) -> dict[str, object]: ... + + +@runtime_checkable +class IncrementalStateProject(ProjectService, Protocol): + """Optional project boundary for manifest-only stale-state checks.""" + + def incremental_state(self) -> ProjectState | None: ... + + @dataclass(frozen=True) class ContextEntry: node_id: str diff --git a/tests/test_adapter_contract.py b/tests/test_adapter_contract.py index 3aa13af..99355a8 100644 --- a/tests/test_adapter_contract.py +++ b/tests/test_adapter_contract.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import tempfile import unittest from collections.abc import Mapping @@ -10,10 +11,13 @@ from mcp.shared.memory import create_connected_server_and_client_session from docforge.adapter_contract import ( AdapterEdge, + AdapterManifest, AdapterNode, AdapterProject, AdapterProjection, AdapterProjectSettings, + AdapterSource, + AdapterSourceProjection, ShadowArtifact, compare_artifacts, validate_projection, @@ -28,6 +32,9 @@ from docforge.mcp_server import ( ) from docforge.models import ( Edge, + LogicEdge, + LogicNode, + LogicProjection, Node, ProjectSnapshot, ProposalWriter, @@ -44,6 +51,134 @@ class Loader: return self.projection +class IncrementalLoader: + def __init__(self, root: Path) -> None: + self.root = root + self.sources = { + "guide.foundation": "Foundation content.", + "guide.workflow": "Workflow content.", + } + self.source_paths = { + "guide.foundation": "docs/foundation.md", + "guide.workflow": "docs/workflow.md", + } + self.workflow_dependencies = ("guide.foundation",) + self.extract_calls: list[str] = [] + self.fail_source: str | None = None + + @staticmethod + def _hash(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + def load_manifest(self) -> AdapterManifest: + source_items = tuple( + AdapterSource( + source_id=source_id, + source_path=self.source_paths[source_id], + fingerprint=self._hash(content), + extractor_version="python-ast@1", + dependencies=(self.workflow_dependencies if source_id == "guide.workflow" else ()), + ) + for source_id, content in sorted(self.sources.items()) + ) + digest = hashlib.sha256() + for source in source_items: + digest.update(source.source_id.encode()) + digest.update(source.source_path.encode()) + digest.update(source.fingerprint.encode()) + digest.update(source.extractor_version.encode()) + for dependency in source.dependencies: + digest.update(dependency.encode()) + return AdapterManifest( + project_id="incremental-fixture", + title="Incremental fixture", + adapter_id="fixture-incremental", + adapter_version="1", + root=self.root, + revision=digest.hexdigest()[:12], + source_hash=digest.hexdigest(), + families=("guide",), + allowed_relations=("depends_on",), + sources=source_items, + estimated_nodes=10, + ) + + def extract_source(self, source: AdapterSource) -> AdapterSourceProjection: + self.extract_calls.append(source.source_id) + if self.fail_source == source.source_id: + raise RuntimeError("intentional extraction failure") + content = self.sources[source.source_id] + node = Node( + node_id=source.source_id, + title=source.source_id.rsplit(".", 1)[-1].title(), + family="guide", + authority="authoritative", + status="active", + tags=("guide",), + summary=f"{source.source_id} summary.", + content=content, + source_path=source.source_path, + source_anchor=None, + content_hash=self._hash(content), + ) + edges = ( + (AdapterEdge(Edge("guide.workflow", "depends_on", "guide.foundation")),) + if source.source_id == "guide.workflow" + else () + ) + logic = ( + ( + LogicProjection( + owner_node_id="guide.workflow", + source_id="guide.workflow", + nodes=( + LogicNode("entry", "entry", "Entry", None), + LogicNode("return", "return", "Return", None), + ), + edges=(LogicEdge("entry", "return", "return", "RETURN", 0),), + ), + ) + if source.source_id == "guide.workflow" + else () + ) + return AdapterSourceProjection( + source_id=source.source_id, + fingerprint=source.fingerprint, + nodes=(AdapterNode(node),), + edges=edges, + logic=logic, + ) + + def load_projection(self) -> AdapterProjection: + manifest = self.load_manifest() + contributions = tuple(self.extract_source(source) for source in manifest.sources) + return AdapterProjection( + project_id=manifest.project_id, + title=manifest.title, + adapter_id=manifest.adapter_id, + adapter_version=manifest.adapter_version, + root=manifest.root, + revision=manifest.revision, + source_hash=manifest.source_hash, + nodes=tuple( + sorted( + (node for contribution in contributions for node in contribution.nodes), + key=lambda item: item.node.node_id, + ) + ), + edges=tuple( + sorted( + (edge for contribution in contributions for edge in contribution.edges), + key=lambda item: ( + item.edge.source_id, + item.edge.relation, + item.edge.target_id, + ), + ) + ), + ) + + class AdapterContractTests(unittest.TestCase): def projection(self, root: Path) -> AdapterProjection: foundation = Node( @@ -157,6 +292,92 @@ class AdapterContractTests(unittest.TestCase): with self.assertRaisesRegex(DocForgeError, "confined"): AdapterProject(Loader(self.projection(root)), cache_root=outside) + def test_incremental_adapter_reuses_sources_and_invalidates_reverse_dependencies(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + loader = IncrementalLoader(root) + project = AdapterProject(loader, cache_root=root / ".cache" / "incremental") + index = ProjectIndex(project) + + first = index.build() + self.assertEqual(2, first["build"]["reparsed_sources"]) + self.assertEqual(0, first["build"]["cache_hits"]) + loader.extract_calls.clear() + + second = index.build() + self.assertEqual([], loader.extract_calls) + self.assertEqual(0, second["build"]["reparsed_sources"]) + self.assertEqual(2, second["build"]["cache_hits"]) + + loader.sources["guide.foundation"] = "Changed foundation." + loader.extract_calls.clear() + with self.assertRaisesRegex(DocForgeError, "does not match"): + index.check() + self.assertEqual([], loader.extract_calls) + changed = index.build() + self.assertEqual( + ["guide.foundation", "guide.workflow"], + loader.extract_calls, + ) + self.assertEqual(2, changed["build"]["invalidated_sources"]) + self.assertEqual( + "Changed foundation.", + index.get_node("guide.foundation")["node"]["content"], + ) + + loader.source_paths["guide.workflow"] = "docs/workflow-renamed.md" + loader.workflow_dependencies = () + loader.extract_calls.clear() + renamed = index.build() + self.assertEqual(["guide.workflow"], loader.extract_calls) + self.assertEqual(1, renamed["build"]["invalidated_sources"]) + self.assertEqual( + "docs/workflow-renamed.md", + index.get_node("guide.workflow")["node"]["source_path"], + ) + + def test_incremental_delete_failure_and_equivalence_are_safe(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + loader = IncrementalLoader(root) + project = AdapterProject(loader, cache_root=root / ".cache" / "incremental") + index = ProjectIndex(project) + index.build() + (root / ".cache" / "incremental" / "extractions.json").write_text( + "{broken", encoding="utf-8" + ) + loader.extract_calls.clear() + cache_miss = index.build() + self.assertEqual( + ["guide.foundation", "guide.workflow"], + loader.extract_calls, + ) + self.assertEqual(2, cache_miss["build"]["reparsed_sources"]) + original_index = index.path.read_bytes() + cache_path = root / ".cache" / "incremental" / "extractions.json" + original_cache = cache_path.read_bytes() + + loader.sources["guide.foundation"] = "Broken extraction." + loader.fail_source = "guide.foundation" + with self.assertRaisesRegex(RuntimeError, "intentional"): + index.build() + self.assertEqual(original_index, index.path.read_bytes()) + self.assertEqual(original_cache, cache_path.read_bytes()) + + loader.fail_source = None + index.build() + loader.sources.pop("guide.workflow") + deleted = index.build() + self.assertEqual(1, deleted["build"]["deleted_sources"]) + self.assertEqual(1, deleted["node_count"]) + self.assertEqual(0, deleted["edge_count"]) + self.assertIsNone(project.logic_projection("guide.workflow")) + + loader.extract_calls.clear() + equivalent = project.verify_incremental_equivalence() + self.assertEqual("ok", equivalent["status"]) + self.assertEqual(1, equivalent["node_count"]) + def test_artifact_comparison_is_complete_and_byte_exact(self) -> None: reference = ( ShadowArtifact("manual", b"same"), diff --git a/tests/test_changesets.py b/tests/test_changesets.py index 139b57e..6a97351 100644 --- a/tests/test_changesets.py +++ b/tests/test_changesets.py @@ -268,6 +268,46 @@ class DocForgeChangesetTests(unittest.TestCase): with self.assertRaisesRegex(DocForgeError, "Canonical project changed"): service.apply("apply-all", str(final["changeset_hash"])) + def test_relationship_only_update_is_hash_bound_and_does_not_rewrite_node(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture(Path(directory)) + project = Project.open(root) + store = ChangesetStore(project, "alpha-editor") + workflow = next( + node for node in project.load().nodes if node.node_id == "guide.workflow" + ) + created = store.create("relationship-only") + proposed = store.propose_relationship_update( + changeset_id="relationship-only", + expected_changeset_hash=str(created["changeset_hash"]), + node_id="guide.workflow", + expected_content_hash=workflow.content_hash, + relationship_changes=[ + { + "action": "remove", + "source_id": "guide.workflow", + "relation": "depends_on", + "target_id": "guide.foundation", + } + ], + rationale="Queue one relationship correction without changing node content.", + ) + + change = store.diff("relationship-only")["changes"][0] + self.assertEqual("update", change["operation"]) + self.assertEqual("", change["content_diff"]) + self.assertEqual({}, change["metadata"]) + self.assertEqual(1, proposed["projected_edge_count"]) + with self.assertRaisesRegex(DocForgeError, "at least one"): + store.propose_relationship_update( + changeset_id="relationship-only", + expected_changeset_hash=str(proposed["changeset_hash"]), + node_id="guide.foundation", + expected_content_hash=self.node_hash(project, "guide.foundation"), + relationship_changes=[], + rationale="Reject an empty relationship operation.", + ) + def test_optimistic_and_cross_changeset_conflicts_preserve_both_proposals(self) -> None: with tempfile.TemporaryDirectory() as directory: root = self.copy_fixture(Path(directory)) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index bac1e0d..7f48b26 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -69,7 +69,7 @@ class DocForgeMcpTests(unittest.IsolatedAsyncioTestCase): names = tuple(tool.name for tool in response.tools) self.assertEqual(ALL_TOOLS, names) - self.assertEqual(10, len(PROPOSAL_TOOLS)) + self.assertEqual(11, len(PROPOSAL_TOOLS)) self.assertFalse( any( token in name @@ -328,6 +328,49 @@ class DocForgeMcpTests(unittest.IsolatedAsyncioTestCase): self.assertTrue((root / ".docforge/previews/mcp-update/manual.html").is_file()) self.assertFalse((root / ".docforge/rendered/manual.html").exists()) + async def test_relationship_only_mcp_tool_queues_no_content_rewrite(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture("alpha", Path(directory)) + project = Project.open(root) + ProjectIndex(project).build() + workflow = next( + node for node in project.load().nodes if node.node_id == "guide.workflow" + ) + async with create_connected_server_and_client_session( + create_server(root, "alpha-editor"), raise_exceptions=True + ) as session: + created = await session.call_tool( + "docforge_create_changeset", {"changeset_id": "mcp-relationship"} + ) + proposed = await session.call_tool( + "docforge_propose_relationship_update", + { + "changeset_id": "mcp-relationship", + "expected_changeset_hash": created.structuredContent["changeset_hash"], + "node_id": "guide.workflow", + "expected_content_hash": workflow.content_hash, + "relationship_changes": [ + { + "action": "remove", + "source_id": "guide.workflow", + "relation": "depends_on", + "target_id": "guide.foundation", + } + ], + "rationale": "Exercise the relationship-only MCP boundary.", + }, + ) + diff = await session.call_tool( + "docforge_get_changeset_diff", + {"changeset_id": "mcp-relationship"}, + ) + + self.assertEqual("ok", proposed.structuredContent["status"]) + self.assertEqual("", diff.structuredContent["changes"][0]["content_diff"]) + self.assertEqual({}, diff.structuredContent["changes"][0]["metadata"]) + self.assertTrue((root / ".docforge/changesets/mcp-relationship.json").is_file()) + self.assertFalse((root / ".docforge/rendered/manual.html").exists()) + async def test_server_without_writer_rejects_proposal_mutation_structurally(self) -> None: with tempfile.TemporaryDirectory() as directory: root = self.copy_fixture("alpha", Path(directory)) diff --git a/toreview.md b/toreview.md index 99b76e7..f6c74e4 100644 --- a/toreview.md +++ b/toreview.md @@ -1,4 +1,7 @@ -# Incremental Adapter Indexing — Design Review Notes +# Incremental Adapter Indexing — Implemented Design Record + +Status: implemented on `Dev-Rewrite`. The maintained contract and adapter guide now live in +`docs/INCREMENTAL_INDEXING.md`. ## Current behavior diff --git a/uv.lock b/uv.lock index 93e4a5a..396f46f 100644 --- a/uv.lock +++ b/uv.lock @@ -206,7 +206,7 @@ wheels = [ [[package]] name = "docforge" -version = "1.0.0" +version = "1.1.0.dev0" source = { editable = "." } dependencies = [ { name = "markdown-it-py" },