1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions Pages

Add incremental adapter compiler boundary

This commit is contained in:
Andraxion 2026-07-25 19:08:39 -04:00
parent 82b3b90521
commit 696b62f9f8
20 changed files with 1592 additions and 122 deletions

View file

@ -7,20 +7,35 @@ import json
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Protocol
from typing import Protocol, runtime_checkable
from .config_validation import AUTHORITIES, ID_PATTERN
from .adapter_validation import (
source_payload,
source_projection,
validate_manifest,
validate_projection,
validate_source_projection,
)
from .config_validation import ID_PATTERN
from .errors import DocForgeError
from .incremental import (
CachedSource,
ExtractionCache,
affected_sources,
load_extraction_cache,
write_extraction_cache,
)
from .models import (
Edge,
Limits,
LogicProjection,
Node,
ProjectDescriptor,
ProjectSnapshot,
ProjectState,
ProposalWriter,
RenderConfig,
)
from .project import validate_graph
@dataclass(frozen=True)
@ -79,12 +94,63 @@ class AdapterProjection:
return hashlib.sha256(encoded).hexdigest()
@dataclass(frozen=True)
class AdapterSource:
"""One fingerprinted extraction unit declared by an incremental adapter."""
source_id: str
source_path: str
fingerprint: str
extractor_version: str
dependencies: tuple[str, ...] = ()
@dataclass(frozen=True)
class AdapterManifest:
"""Cheap project identity and source inventory for incremental compilation."""
project_id: str
title: str
adapter_id: str
adapter_version: str
root: Path
revision: str
source_hash: str
families: tuple[str, ...]
allowed_relations: tuple[str, ...]
sources: tuple[AdapterSource, ...]
estimated_nodes: int = 10_000
def identity(self) -> tuple[str, str, str, Path]:
return self.project_id, self.adapter_id, self.adapter_version, self.root
@dataclass(frozen=True)
class AdapterSourceProjection:
"""Nodes, relationships, and optional logic owned by one source unit."""
source_id: str
fingerprint: str
nodes: tuple[AdapterNode, ...]
edges: tuple[AdapterEdge, ...]
logic: tuple[LogicProjection, ...] = ()
class AdapterLoader(Protocol):
"""Load one current, deterministic, project-confined adapter projection."""
def load_projection(self) -> AdapterProjection: ...
@runtime_checkable
class IncrementalAdapterLoader(AdapterLoader, Protocol):
"""Opt-in adapter contract for cacheable, source-scoped extraction."""
def load_manifest(self) -> AdapterManifest: ...
def extract_source(self, source: AdapterSource) -> AdapterSourceProjection: ...
ProposalValidator = Callable[
[
ProjectSnapshot,
@ -110,86 +176,6 @@ class AdapterProjectSettings:
proposal_validator: ProposalValidator | None = None
def validate_projection(projection: AdapterProjection) -> None:
"""Validate generic invariants without interpreting adapter metadata."""
root = projection.root.resolve(strict=True)
if not root.is_dir() or projection.root != root:
raise DocForgeError("invalid_adapter", "Adapter root must be a resolved directory")
for label, value in (
("project_id", projection.project_id),
("title", projection.title),
("adapter_id", projection.adapter_id),
("adapter_version", projection.adapter_version),
("revision", projection.revision),
("source_hash", projection.source_hash),
):
if not value.strip():
raise DocForgeError("invalid_adapter", f"Adapter {label} must not be empty")
if ID_PATTERN.fullmatch(projection.project_id) is None:
raise DocForgeError("invalid_adapter", "Adapter project ID is invalid")
if len(projection.source_hash) != 64 or any(
character not in "0123456789abcdef" for character in projection.source_hash
):
raise DocForgeError("invalid_adapter", "Adapter source hash must be lowercase SHA-256")
ordered_nodes = tuple(sorted(projection.nodes, key=lambda item: item.node.node_id))
ordered_edges = tuple(
sorted(
projection.edges,
key=lambda item: (
item.edge.source_id,
item.edge.relation,
item.edge.target_id,
),
)
)
if projection.nodes != ordered_nodes or projection.edges != ordered_edges:
raise DocForgeError(
"invalid_adapter", "Adapter projection must be deterministically ordered"
)
for item in (*projection.nodes, *projection.edges):
keys = [key for key, _ in item.metadata]
if keys != sorted(keys) or len(keys) != len(set(keys)):
raise DocForgeError(
"invalid_adapter", "Adapter metadata keys must be unique and ordered"
)
for item in projection.nodes:
node = item.node
source = Path(node.source_path)
if ID_PATTERN.fullmatch(node.node_id) is None:
raise DocForgeError("invalid_adapter", "Adapter node ID is invalid", id=node.node_id)
if node.authority not in AUTHORITIES:
raise DocForgeError(
"invalid_adapter", "Adapter node authority is invalid", id=node.node_id
)
if (
not node.title.strip()
or not node.family.strip()
or not node.status.strip()
or not node.summary.strip()
or not node.content.strip()
):
raise DocForgeError(
"invalid_adapter", "Adapter node has empty required content", id=node.node_id
)
if source.is_absolute() or ".." in source.parts or not node.source_path:
raise DocForgeError(
"invalid_adapter", "Adapter node source path is unsafe", id=node.node_id
)
if len(node.tags) != len(set(node.tags)) or any(not tag for tag in node.tags):
raise DocForgeError("invalid_adapter", "Adapter node tags are invalid", id=node.node_id)
if len(node.content_hash) != 64 or any(
character not in "0123456789abcdef" for character in node.content_hash
):
raise DocForgeError(
"invalid_adapter", "Adapter node content hash is invalid", id=node.node_id
)
for item in projection.edges:
if ID_PATTERN.fullmatch(item.edge.relation) is None:
raise DocForgeError("invalid_adapter", "Adapter relationship type is invalid")
validate_graph(projection.core_nodes(), projection.core_edges())
class AdapterProject:
"""Expose a validated adapter projection through the standard index boundary."""
@ -202,9 +188,45 @@ class AdapterProject:
) -> None:
self.loader = loader
self.settings = settings or AdapterProjectSettings()
initial = loader.load_projection()
validate_projection(initial)
root = initial.root
self._incremental_loader = loader if isinstance(loader, IncrementalAdapterLoader) else None
initial: AdapterProjection | None = None
manifest: AdapterManifest | None = None
if self._incremental_loader is not None:
manifest = self._incremental_loader.load_manifest()
validate_manifest(manifest)
root = manifest.root
project_id = manifest.project_id
title = manifest.title
adapter_id = manifest.adapter_id
adapter_version = manifest.adapter_version
descriptor_hash = hashlib.sha256(
json.dumps(
{
"project_id": manifest.project_id,
"adapter_id": manifest.adapter_id,
"adapter_version": manifest.adapter_version,
"families": manifest.families,
"allowed_relations": manifest.allowed_relations,
},
sort_keys=True,
separators=(",", ":"),
).encode()
).hexdigest()
families = set(manifest.families)
allowed_relations = manifest.allowed_relations
estimated_nodes = manifest.estimated_nodes
else:
initial = loader.load_projection()
validate_projection(initial)
root = initial.root
project_id = initial.project_id
title = initial.title
adapter_id = initial.adapter_id
adapter_version = initial.adapter_version
descriptor_hash = initial.identity()
families = {item.node.family for item in initial.nodes}
allowed_relations = tuple(sorted({item.edge.relation for item in initial.edges}))
estimated_nodes = len(initial.nodes)
resolved_cache = cache_root.resolve(strict=False)
if (
resolved_cache == root
@ -215,11 +237,21 @@ class AdapterProject:
"path_escape", "Adapter cache must be a confined project subdirectory"
)
self._identity = (
initial.project_id,
initial.adapter_id,
initial.adapter_version,
initial.root,
project_id,
adapter_id,
adapter_version,
root,
)
self._cache_path = resolved_cache / "extractions.json"
self._last_build_report: dict[str, object] = {
"mode": "full",
"cache_hits": 0,
"reparsed_sources": 0,
"invalidated_sources": 0,
"deleted_sources": 0,
"total_sources": 0,
}
self._last_logic: tuple[LogicProjection, ...] = ()
content_roots = self._resolved_directories(
root, self.settings.content_roots, label="content root"
)
@ -264,19 +296,19 @@ class AdapterProject:
"Proposal-enabled adapters require a confined descriptor file",
)
limits = self.settings.limits or Limits(
max_nodes=max(10_000, len(initial.nodes)),
max_nodes=max(10_000, estimated_nodes),
max_context_tokens=64_000,
)
self._validate_proposal_writers(initial, self.settings.proposal_writers)
self._validate_proposal_writers(families, self.settings.proposal_writers)
self._validate_render(root, self.settings.render, content_roots, changeset_root)
self.descriptor = ProjectDescriptor(
schema_version=1,
project_id=initial.project_id,
title=initial.title,
adapter=f"{initial.adapter_id}@{initial.adapter_version}",
project_id=project_id,
title=title,
adapter=f"{adapter_id}@{adapter_version}",
root=root,
descriptor_path=descriptor_path,
descriptor_hash=initial.identity(),
descriptor_hash=descriptor_hash,
content_roots=content_roots,
authority_files=authority_files,
cache_root=resolved_cache,
@ -284,7 +316,7 @@ class AdapterProject:
changeset_root=changeset_root,
proposal_writers=self.settings.proposal_writers,
render=self.settings.render,
allowed_relations=tuple(sorted({item.edge.relation for item in initial.edges})),
allowed_relations=allowed_relations,
profiles=(),
limits=limits,
)
@ -293,7 +325,11 @@ class AdapterProject:
def load(self) -> ProjectSnapshot:
canonical_sources = self.canonical_source_paths()
captured = {path: path.read_bytes() for path in canonical_sources}
projection = self.loader.load_projection()
projection = (
self._load_incremental()
if self._incremental_loader is not None
else self.loader.load_projection()
)
validate_projection(projection)
identity = (
projection.project_id,
@ -309,15 +345,12 @@ class AdapterProject:
raise DocForgeError(
"source_changed", "Adapter canonical source changed during the operation"
)
source_hash = projection.source_hash
if captured:
digest = hashlib.sha256(projection.source_hash.encode("ascii"))
for path in canonical_sources:
relative = path.relative_to(projection.root).as_posix().encode()
digest.update(len(relative).to_bytes(8, "big"))
digest.update(relative)
digest.update(hashlib.sha256(captured[path]).digest())
source_hash = digest.hexdigest()
source_hash = self._combined_source_hash(
projection.source_hash,
projection.root,
canonical_sources,
captured,
)
return ProjectSnapshot(
descriptor=self.descriptor,
nodes=projection.core_nodes(),
@ -326,6 +359,211 @@ class AdapterProject:
revision=projection.revision,
)
def incremental_state(self) -> ProjectState | None:
"""Return current source identity without reconstructing the complete projection."""
loader = self._incremental_loader
if loader is None:
return None
canonical_sources = self.canonical_source_paths()
captured = {path: path.read_bytes() for path in canonical_sources}
manifest = loader.load_manifest()
validate_manifest(manifest)
if manifest.identity() != self._identity:
raise DocForgeError("adapter_changed", "Adapter identity changed during the operation")
if self.canonical_source_paths() != canonical_sources or any(
not path.is_file() or path.read_bytes() != raw for path, raw in captured.items()
):
raise DocForgeError(
"source_changed", "Adapter canonical source changed during the operation"
)
return ProjectState(
source_hash=self._combined_source_hash(
manifest.source_hash,
manifest.root,
canonical_sources,
captured,
),
revision=manifest.revision,
)
def build_report(self) -> dict[str, object]:
"""Return deterministic extraction metrics from the most recent load."""
return dict(self._last_build_report)
def logic_projection(self, owner_node_id: str) -> LogicProjection | None:
"""Load one lazily stored function-scoped logic projection."""
self.load()
return next(
(
projection
for projection in self._last_logic
if projection.owner_node_id == owner_node_id
),
None,
)
def verify_incremental_equivalence(self) -> dict[str, object]:
"""Prove the incremental and full loader contracts produce the same graph."""
if self._incremental_loader is None:
raise DocForgeError(
"incremental_disabled", "Adapter does not implement incremental extraction"
)
incremental = self._load_incremental()
full = self.loader.load_projection()
validate_projection(full)
fields = {
"project_id": incremental.project_id == full.project_id,
"adapter_id": incremental.adapter_id == full.adapter_id,
"adapter_version": incremental.adapter_version == full.adapter_version,
"revision": incremental.revision == full.revision,
"source_hash": incremental.source_hash == full.source_hash,
"nodes": incremental.nodes == full.nodes,
"edges": incremental.edges == full.edges,
}
mismatches = [field for field, matches in fields.items() if not matches]
if mismatches:
raise DocForgeError(
"incremental_mismatch",
"Incremental extraction does not match a full adapter projection",
fields=mismatches,
)
return {
"status": "ok",
"project_id": incremental.project_id,
"revision": incremental.revision,
"source_hash": incremental.source_hash,
"node_count": len(incremental.nodes),
"edge_count": len(incremental.edges),
}
def _load_incremental(self) -> AdapterProjection:
loader = self._incremental_loader
if loader is None:
raise DocForgeError("incremental_disabled", "Incremental adapter is not configured")
manifest = loader.load_manifest()
validate_manifest(manifest)
if manifest.identity() != self._identity:
raise DocForgeError("adapter_changed", "Adapter identity changed during the operation")
cache = load_extraction_cache(
self._cache_path,
project_id=manifest.project_id,
adapter_id=manifest.adapter_id,
adapter_version=manifest.adapter_version,
)
cached = {source.source_id: source for source in cache.sources} if cache else {}
current = {source.source_id: source for source in manifest.sources}
changed = {
source_id
for source_id, source in current.items()
if source_id not in cached
or cached[source_id].source_path != source.source_path
or cached[source_id].fingerprint != source.fingerprint
or cached[source_id].extractor_version != source.extractor_version
or cached[source_id].dependencies != source.dependencies
}
deleted = set(cached) - set(current)
invalidated = affected_sources(
current_dependencies={
source.source_id: source.dependencies for source in manifest.sources
},
cached_dependencies={
source.source_id: source.dependencies for source in cached.values()
},
changed=changed | deleted,
)
contributions: list[AdapterSourceProjection] = []
cache_records: list[CachedSource] = []
reparsed: list[str] = []
hits: list[str] = []
for source in manifest.sources:
if source.source_id in invalidated:
contribution = loader.extract_source(source)
reparsed.append(source.source_id)
else:
record = cached[source.source_id]
contribution = source_projection(record.payload)
hits.append(source.source_id)
validate_source_projection(source, contribution)
contributions.append(contribution)
cache_records.append(
CachedSource(
source_id=source.source_id,
source_path=source.source_path,
fingerprint=source.fingerprint,
extractor_version=source.extractor_version,
dependencies=source.dependencies,
payload=source_payload(contribution),
)
)
projection = AdapterProjection(
project_id=manifest.project_id,
title=manifest.title,
adapter_id=manifest.adapter_id,
adapter_version=manifest.adapter_version,
root=manifest.root,
revision=manifest.revision,
source_hash=manifest.source_hash,
nodes=tuple(
sorted(
(node for contribution in contributions for node in contribution.nodes),
key=lambda item: item.node.node_id,
)
),
edges=tuple(
sorted(
(edge for contribution in contributions for edge in contribution.edges),
key=lambda item: (
item.edge.source_id,
item.edge.relation,
item.edge.target_id,
),
)
),
)
validate_projection(projection)
logic_projections = tuple(
sorted(
(logic for contribution in contributions for logic in contribution.logic),
key=lambda projection: projection.owner_node_id,
)
)
owners = [projection.owner_node_id for projection in logic_projections]
if len(owners) != len(set(owners)):
raise DocForgeError(
"invalid_adapter", "A primary graph node may own only one logic projection"
)
stable = loader.load_manifest()
validate_manifest(stable)
if stable != manifest:
raise DocForgeError(
"source_changed", "Adapter sources changed during incremental extraction"
)
write_extraction_cache(
self._cache_path,
ExtractionCache(
project_id=manifest.project_id,
adapter_id=manifest.adapter_id,
adapter_version=manifest.adapter_version,
sources=tuple(cache_records),
),
)
self._last_logic = logic_projections
self._last_build_report = {
"mode": "incremental",
"cache_hits": len(hits),
"reparsed_sources": len(reparsed),
"invalidated_sources": len(invalidated & set(current)),
"deleted_sources": len(deleted),
"total_sources": len(current),
"cache_hit_ids": hits,
"reparsed_source_ids": reparsed,
}
return projection
def canonical_source_paths(self) -> tuple[Path, ...]:
"""Adapters validate their own source sets before producing a projection."""
@ -335,6 +573,23 @@ class AdapterProject:
label="canonical source",
)
@staticmethod
def _combined_source_hash(
projection_hash: str,
root: Path,
canonical_sources: tuple[Path, ...],
captured: dict[Path, bytes],
) -> str:
if not captured:
return projection_hash
digest = hashlib.sha256(projection_hash.encode("ascii"))
for path in canonical_sources:
relative = path.relative_to(root).as_posix().encode()
digest.update(len(relative).to_bytes(8, "big"))
digest.update(relative)
digest.update(hashlib.sha256(captured[path]).digest())
return digest.hexdigest()
def validate_proposal(
self,
base: ProjectSnapshot,
@ -391,11 +646,8 @@ class AdapterProject:
return current
@staticmethod
def _validate_proposal_writers(
projection: AdapterProjection, writers: tuple[ProposalWriter, ...]
) -> None:
def _validate_proposal_writers(families: set[str], writers: tuple[ProposalWriter, ...]) -> None:
writer_ids = [writer.writer_id for writer in writers]
families = {item.node.family for item in projection.nodes}
if len(writer_ids) != len(set(writer_ids)):
raise DocForgeError("invalid_adapter", "Adapter proposal writer IDs repeat")
for writer in writers: