From cd54cae71da2fbcca61305383ea1e1a4e3a174c0 Mon Sep 17 00:00:00 2001 From: Andraxion Date: Mon, 27 Jul 2026 16:01:40 -0400 Subject: [PATCH] Add deterministic incremental adapter assembly --- ACTIVE_SLICE.md | 20 +++--- SLICE_HISTORY.md | 23 +++++++ docs/INCREMENTAL_INDEXING.md | 19 ++++++ docs/PROJECT_ONBOARDING.md | 6 ++ src/docforge/adapter_contract.py | 111 +++++++++++++++++++++++-------- tests/test_adapter_contract.py | 83 +++++++++++++++++++++++ 6 files changed, 224 insertions(+), 38 deletions(-) diff --git a/ACTIVE_SLICE.md b/ACTIVE_SLICE.md index 900d229..6a4db97 100644 --- a/ACTIVE_SLICE.md +++ b/ACTIVE_SLICE.md @@ -1,15 +1,15 @@ # Active slice ```text -Slice: DFG-21 language-neutral project onboarding -Goal: Let an unfamiliar codebase assess DocForge readiness and create a valid generic manual without implying that detected source languages already have semantic extraction. -In scope: Read-only repository assessment; deterministic language and build-evidence detection; explicit multi-language selection; safe generic manual scaffolding; immediate index and render; a detailed language-neutral onboarding and adapter checklist; focused and complete quality proof. -Out of scope: Bundled C++, Rust, Java, or other source frontends; dependency installation; project builds; Git mutation through DocForge; MCP self-installation; automatic canonical-document import; inferred source-to-manual relationships; deployment or publication. -Done when: Assessment writes nothing, excluded trees and symlinks are ignored, mixed-language evidence is reported deterministically, scaffolding refuses conflicts and unsafe paths, a new project validates, indexes, and renders immediately, source graph status remains adapter_required, documentation explains the complete frontend and proof path, and the full DocForge quality gate passes. -Owners: DocForge owns generic assessment, scaffolding, graph contracts, incremental compilation, rendering, visualization, and MCP boundaries. Each project or reusable frontend owns language-specific source discovery and semantic extraction. Canonical project files retain authority. -Proof: The focused onboarding, CLI, and core suite passed 19 tests and 2 subtests. Strict Pyright passed with no errors or warnings. Ruff lint and formatting, Python compilation, and the HTML/CSS/JavaScript quality gate passed. The complete warning-strict suite passed 81 tests and 2 subtests. The scaffold test creates a Rust project, writes a confined generic configuration without replacement, builds one authoritative node, renders the manual, reopens the project, and retains source_graph_status=adapter_required. +Slice: DFG-22 deterministic incremental adapter assembly +Goal: Let language frontends cache repeated raw source evidence while publishing one deterministic graph without maintaining a second project-owned extraction cache. +In scope: Optional generic assembly contract; cached contribution assembly; manifest-bound identity checks; final Logic ownership checks; overlapping-evidence tests; incremental documentation and onboarding guidance. +Out of scope: Language-specific merge rules; bundled C++, Rust, Java, or other frontends; changes to the default unique-ownership path; hidden source reads during assembly; project-owned cache formats; source mutation; deployment or publication. +Done when: An adapter can cache overlapping source contributions, deterministically assemble one valid node, reuse the warm cache, reparse only one changed source, preserve the selected published fact, pass full/incremental equivalence, and the complete DocForge quality gate passes. +Owners: DocForge owns cached contribution delivery, the optional assembly boundary, identity validation, and final graph validation. The adapter owns deterministic evidence selection and merge semantics. Canonical source files remain authoritative. +Proof: The focused adapter and onboarding suite passed 14 tests. Strict Pyright passed with no errors or warnings. Ruff lint and formatting, Python compilation, and the HTML/CSS/JavaScript quality gate passed. The complete warning-strict suite passed 82 tests and 2 subtests. The overlap fixture cached two source contributions, published one deterministic node, reused both warm entries, reparsed only one changed source, preserved the selected node, and passed full/incremental equivalence. ``` -**Next gate:** Prove Worldforge's C++ integration against the generic incremental contract. -Extract a reusable language frontend only after a second consumer demonstrates which behavior is -genuinely shared. +**Next gate:** Prove Worldforge's C++ integration against the generic incremental and assembly +contracts. Extract a reusable language frontend only after a second consumer demonstrates which +behavior is genuinely shared. diff --git a/SLICE_HISTORY.md b/SLICE_HISTORY.md index 2b7a087..7f0e421 100644 --- a/SLICE_HISTORY.md +++ b/SLICE_HISTORY.md @@ -1,5 +1,28 @@ # Completed slices +## DFG-22 deterministic incremental adapter assembly + +### Changed + +- Added an optional language-neutral assembly contract after incremental source extraction. +- Kept raw source contributions inside DocForge's existing fingerprint, dependency invalidation, + cache, and atomic publication path. +- Required assembled projections to retain the manifest-bound identity, revision, and source hash. +- Validated the assembled primary graph and function Logic owners before publication. +- Preserved the stricter unique-source ownership path for adapters that do not need assembly. +- Documented assembly for compilers and language tools that repeat shared declarations across + extraction units. + +### Verification + +- The overlap fixture caches two repeated source contributions, publishes one deterministic node, + reuses both warm cache entries, reparses one changed source, preserves the selected fact, and + passes full/incremental equivalence. +- Focused adapter and onboarding tests passed 14 tests. +- Strict Pyright, Ruff lint and formatting, Python compilation, and the HTML/CSS/JavaScript quality + gate passed. +- The complete warning-strict suite passed 82 tests and 2 subtests. + ## DFG-21 language-neutral project onboarding ### Changed diff --git a/docs/INCREMENTAL_INDEXING.md b/docs/INCREMENTAL_INDEXING.md index 0bfbfda..2e05ec9 100644 --- a/docs/INCREMENTAL_INDEXING.md +++ b/docs/INCREMENTAL_INDEXING.md @@ -64,6 +64,25 @@ Each `AdapterSourceProjection` owns: Ownership must be deterministic. Two sources may not produce the same primary node or the same function logic projection. +Some language tools emit overlapping raw evidence before ownership can be resolved. An incremental +loader may additionally implement: + +```python +def assemble_projection( + manifest: AdapterManifest, + contributions: tuple[AdapterSourceProjection, ...], +) -> AdapterAssembly: ... +``` + +DocForge caches and invalidates the source contributions normally, then passes the complete current +contribution set to this pure assembly step. The assembler must deterministically select or merge +overlapping evidence and return one valid final graph and Logic set. It may not read hidden source +state or create a second extraction cache. The final identity, revision, and source hash must match +the manifest exactly. + +Without an assembler, the stricter default remains in force: two contributions may not publish the +same primary node or Logic owner. + ## Invalidation DocForge invalidates a source when: diff --git a/docs/PROJECT_ONBOARDING.md b/docs/PROJECT_ONBOARDING.md index 505efe0..7a92efd 100644 --- a/docs/PROJECT_ONBOARDING.md +++ b/docs/PROJECT_ONBOARDING.md @@ -123,6 +123,7 @@ All frontends emit the same DocForge contracts: - `AdapterManifest` inventories fingerprinted extraction units and dependencies. - `AdapterSourceProjection` owns nodes, relationships, and optional function Logic for one unit. - `AdapterProjection` provides the deterministic complete rebuild. +- `AdapterAssembly` optionally resolves overlapping raw evidence into the single published graph. Language metadata may differ. Graph publication, indexing, querying, visualization, and MCP behavior do not. @@ -130,6 +131,11 @@ behavior do not. Done when repeated extraction produces the same stable identities without inferred or guessed facts. +When compiler or language tooling repeats shared declarations across extraction units, use the +optional assembly contract. Cache the raw source contributions through DocForge, then +deterministically select or merge ownership from the complete contribution set. Do not hide a +second extraction cache inside the project adapter. + ### 5. Build-system evidence #### C and C++ diff --git a/src/docforge/adapter_contract.py b/src/docforge/adapter_contract.py index e1794b1..f2d6f4d 100644 --- a/src/docforge/adapter_contract.py +++ b/src/docforge/adapter_contract.py @@ -12,6 +12,7 @@ from typing import Protocol, runtime_checkable from .adapter_validation import ( source_payload, source_projection, + validate_logic_projection, validate_manifest, validate_projection, validate_source_projection, @@ -136,6 +137,14 @@ class AdapterSourceProjection: logic: tuple[LogicProjection, ...] = () +@dataclass(frozen=True) +class AdapterAssembly: + """One finalized graph and Logic set assembled from cached source contributions.""" + + projection: AdapterProjection + logic: tuple[LogicProjection, ...] = () + + class AdapterLoader(Protocol): """Load one current, deterministic, project-confined adapter projection.""" @@ -151,6 +160,17 @@ class IncrementalAdapterLoader(AdapterLoader, Protocol): def extract_source(self, source: AdapterSource) -> AdapterSourceProjection: ... +@runtime_checkable +class IncrementalAdapterAssembler(Protocol): + """Optionally normalize overlapping source evidence into one final projection.""" + + def assemble_projection( + self, + manifest: AdapterManifest, + contributions: tuple[AdapterSourceProjection, ...], + ) -> AdapterAssembly: ... + + ProposalValidator = Callable[ [ ProjectSnapshot, @@ -503,43 +523,78 @@ class AdapterProject: validate_source_projection(source, contribution) contributions.append(contribution) cache_records.append(cache_record) - projection = AdapterProjection( - project_id=manifest.project_id, - title=manifest.title, - adapter_id=manifest.adapter_id, - adapter_version=manifest.adapter_version, - root=manifest.root, - revision=manifest.revision, - source_hash=manifest.source_hash, - nodes=tuple( + if isinstance(loader, IncrementalAdapterAssembler): + assembly = loader.assemble_projection(manifest, tuple(contributions)) + projection = assembly.projection + logic_projections = tuple( sorted( - (node for contribution in contributions for node in contribution.nodes), - key=lambda item: item.node.node_id, + assembly.logic, + key=lambda projection: projection.owner_node_id, ) - ), - edges=tuple( - sorted( - (edge for contribution in contributions for edge in contribution.edges), - key=lambda item: ( - item.edge.source_id, - item.edge.relation, - item.edge.target_id, - ), - ) - ), - ) - validate_projection(projection) - logic_projections = tuple( - sorted( - (logic for contribution in contributions for logic in contribution.logic), - key=lambda projection: projection.owner_node_id, ) + else: + projection = AdapterProjection( + project_id=manifest.project_id, + title=manifest.title, + adapter_id=manifest.adapter_id, + adapter_version=manifest.adapter_version, + root=manifest.root, + revision=manifest.revision, + source_hash=manifest.source_hash, + nodes=tuple( + sorted( + (node for contribution in contributions for node in contribution.nodes), + key=lambda item: item.node.node_id, + ) + ), + edges=tuple( + sorted( + (edge for contribution in contributions for edge in contribution.edges), + key=lambda item: ( + item.edge.source_id, + item.edge.relation, + item.edge.target_id, + ), + ) + ), + ) + logic_projections = tuple( + sorted( + (logic for contribution in contributions for logic in contribution.logic), + key=lambda projection: projection.owner_node_id, + ) + ) + identity = ( + projection.project_id, + projection.adapter_id, + projection.adapter_version, + projection.root, ) + if ( + identity != manifest.identity() + or projection.title != manifest.title + or projection.revision != manifest.revision + or projection.source_hash != manifest.source_hash + ): + raise DocForgeError( + "invalid_adapter", + "Incremental assembly changed the manifest-bound project identity", + ) + validate_projection(projection) owners = [projection.owner_node_id for projection in logic_projections] if len(owners) != len(set(owners)): raise DocForgeError( "invalid_adapter", "A primary graph node may own only one logic projection" ) + node_ids = {item.node.node_id for item in projection.nodes} + for logic_projection in logic_projections: + validate_logic_projection(logic_projection) + if logic_projection.owner_node_id not in node_ids: + raise DocForgeError( + "invalid_adapter", + "A Logic projection owner must exist in the assembled primary graph", + owner_node_id=logic_projection.owner_node_id, + ) stable = loader.load_manifest() validate_manifest(stable) if stable != manifest: diff --git a/tests/test_adapter_contract.py b/tests/test_adapter_contract.py index 5237645..5ac3ba1 100644 --- a/tests/test_adapter_contract.py +++ b/tests/test_adapter_contract.py @@ -12,6 +12,7 @@ from pathlib import Path from mcp.shared.memory import create_connected_server_and_client_session from docforge.adapter_contract import ( + AdapterAssembly, AdapterEdge, AdapterManifest, AdapterNode, @@ -182,6 +183,56 @@ class IncrementalLoader: ) +class OverlappingIncrementalLoader(IncrementalLoader): + def extract_source(self, source: AdapterSource) -> AdapterSourceProjection: + self.extract_calls.append(source.source_id) + content = self.sources[source.source_id] + shared = Node( + node_id="guide.shared", + title="Shared", + family="guide", + authority="derived", + status="active", + tags=("guide",), + summary=f"Evidence selected from {source.source_id}.", + content=content, + source_path=source.source_path, + source_anchor=None, + content_hash=self._hash(content), + ) + return AdapterSourceProjection( + source_id=source.source_id, + fingerprint=source.fingerprint, + nodes=(AdapterNode(shared),), + edges=(), + ) + + def assemble_projection( + self, + manifest: AdapterManifest, + contributions: tuple[AdapterSourceProjection, ...], + ) -> AdapterAssembly: + selected = min(contributions, key=lambda item: item.source_id) + return AdapterAssembly( + AdapterProjection( + project_id=manifest.project_id, + title=manifest.title, + adapter_id=manifest.adapter_id, + adapter_version=manifest.adapter_version, + root=manifest.root, + revision=manifest.revision, + source_hash=manifest.source_hash, + nodes=selected.nodes, + edges=(), + ) + ) + + def load_projection(self) -> AdapterProjection: + manifest = self.load_manifest() + contributions = tuple(self.extract_source(source) for source in manifest.sources) + return self.assemble_projection(manifest, contributions).projection + + class AdapterContractTests(unittest.TestCase): def projection(self, root: Path) -> AdapterProjection: foundation = Node( @@ -426,6 +477,38 @@ class AdapterContractTests(unittest.TestCase): self.assertEqual("ok", equivalent["status"]) self.assertEqual(1, equivalent["node_count"]) + def test_incremental_adapter_can_assemble_overlapping_source_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + loader = OverlappingIncrementalLoader(root) + project = AdapterProject(loader, cache_root=root / ".cache" / "overlap") + index = ProjectIndex(project) + + first = index.build() + self.assertEqual(2, first["build"]["reparsed_sources"]) + self.assertEqual(1, first["node_count"]) + self.assertEqual( + "Foundation content.", + index.get_node("guide.shared")["node"]["content"], + ) + + loader.extract_calls.clear() + warm = index.build() + self.assertEqual(2, warm["build"]["cache_hits"]) + self.assertEqual([], loader.extract_calls) + + loader.sources["guide.workflow"] = "Changed overlapping evidence." + loader.extract_calls.clear() + changed = index.build() + self.assertEqual(["guide.workflow"], loader.extract_calls) + self.assertEqual(1, changed["build"]["reparsed_sources"]) + self.assertEqual(1, changed["node_count"]) + self.assertEqual( + "Foundation content.", + index.get_node("guide.shared")["node"]["content"], + ) + self.assertEqual("ok", project.verify_incremental_equivalence()["status"]) + def test_artifact_comparison_is_complete_and_byte_exact(self) -> None: reference = ( ShadowArtifact("manual", b"same"),