"""Milestone 4 Python adapter SDK, incremental, and recovery benchmark gates.""" from __future__ import annotations import argparse import ast import gc import hashlib import json import math import platform import resource import statistics import subprocess import sys import tempfile import time import tracemalloc from collections.abc import Callable, Generator, Mapping from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import cast from unittest import mock from docforge.adapter_sdk import ( AdapterAssembly, AdapterProject, AdapterSource, AdapterSourceProjection, ) from docforge.adapters.python import PythonReferenceAdapter from docforge.index import ProjectIndex ROOT = Path(__file__).resolve().parents[1] FULL_SOURCE_COUNT = 334 SMOKE_SOURCE_COUNT = 12 NODES_PER_SOURCE = 3 DEFAULT_FULL_SAMPLES = 3 DEFAULT_SMOKE_SAMPLES = 1 # These are regression tripwires, not performance targets. They deliberately # leave several times the Milestone 3 1,000-node in-process allowances for the # additional AST, Logic, extraction-cache, and SQLite work in this gate. MAX_TRACED_PEAK_BYTES = 256 * 1024 * 1024 MAX_PROCESS_PEAK_BYTES = 512 * 1024 * 1024 MAX_RESPONSE_BYTES = 256 * 1024 COLD_LIMIT_MS = 30_000.0 WARM_LIMIT_MS = 20_000.0 EQUIVALENCE_LIMIT_MS = 30_000.0 RECOVERY_LIMIT_MS = 30_000.0 @dataclass(frozen=True) class _InstrumentedResult: value: Mapping[str, object] ast_parse_calls: int extraction_calls: int class _RecordingPythonAdapter(PythonReferenceAdapter): """Count extraction entry points without weakening the production adapter.""" def __init__(self, root: Path) -> None: super().__init__( root, source_roots=("src",), project_id="milestone4-python-benchmark", title="Milestone 4 Python adapter benchmark", ) self.extraction_calls = 0 def extract_source(self, source: AdapterSource) -> AdapterSourceProjection: self.extraction_calls += 1 return super().extract_source(source) def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=("Gate DocForge2 Milestone 4 adapter behavior on a disposable Python project.") ) parser.add_argument("--mode", choices=("smoke", "full"), default="full") parser.add_argument("--sources", type=int) parser.add_argument("--samples", type=int) parser.add_argument("--output", type=Path) return parser def encode_report(value: object) -> str: """Serialize benchmark evidence in one deterministic JSON representation.""" return json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False) + "\n" def _compact_json(value: object) -> bytes: return json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ).encode("utf-8") def _sha256(value: object) -> str: return hashlib.sha256(_compact_json(value)).hexdigest() def _git(arguments: list[str]) -> str: return subprocess.run( ["git", *arguments], cwd=ROOT, check=True, capture_output=True, text=True, ).stdout.strip() def write_python_fixture(root: Path, source_count: int) -> None: """Write a deterministic project with three primary nodes per source.""" if source_count < 2: raise ValueError("The Milestone 4 fixture requires at least two sources") package = root / "src" / "benchmark" package.mkdir(parents=True) for index in range(source_count): dependency = "" call = f"value = {index}" if index: dependency = f"from benchmark.module_{index - 1:04d} import compute_{index - 1:04d}\n\n" call = f"value = compute_{index - 1:04d}(value)" source = ( f'"""Synthetic adapter benchmark module {index:04d}."""\n\n' f"{dependency}" f"def compute_{index:04d}(value: int) -> int:\n" f' """Return one deterministic branch result."""\n' f" {call}\n" " if value % 2:\n" f" return value + {index + 1}\n" " return value\n" ) (package / f"module_{index:04d}.py").write_text(source, encoding="utf-8") @contextmanager def _count_ast_parses() -> Generator[mock.MagicMock]: with mock.patch.object(ast, "parse", wraps=ast.parse) as parsed: yield parsed def _instrument( adapter: _RecordingPythonAdapter, operation: Callable[[], Mapping[str, object]], ) -> _InstrumentedResult: adapter.extraction_calls = 0 with _count_ast_parses() as counter: value = operation() return _InstrumentedResult( value=value, ast_parse_calls=counter.call_count, extraction_calls=adapter.extraction_calls, ) def _build_summary(sample: _InstrumentedResult) -> dict[str, object]: value = sample.value build = cast(Mapping[str, object], value["build"]) return { "status": value["status"], "revision": value["revision"], "source_hash": value["source_hash"], "node_hash": value["node_hash"], "node_count": value["node_count"], "edge_hash": value["edge_hash"], "edge_count": value["edge_count"], "logic_hash": value["logic_hash"], "logic_projection_count": value["logic_projection_count"], "logic_node_count": value["logic_node_count"], "logic_edge_count": value["logic_edge_count"], "cache_hits": build["cache_hits"], "reparsed_sources": build["reparsed_sources"], "invalidated_sources": build["invalidated_sources"], "deleted_sources": build["deleted_sources"], "total_sources": build["total_sources"], "ast_parse_calls": sample.ast_parse_calls, "extraction_calls": sample.extraction_calls, } def _equivalence_summary(sample: _InstrumentedResult) -> dict[str, object]: return { **sample.value, "ast_parse_calls": sample.ast_parse_calls, "extraction_calls": sample.extraction_calls, } def _synchronization_summary(sample: _InstrumentedResult) -> dict[str, object]: synchronization = cast(Mapping[str, object], sample.value["synchronization"]) build = cast(Mapping[str, object], synchronization["build"]) return { "status": sample.value["status"], "revision": sample.value["revision"], "source_hash": sample.value["source_hash"], "node_hash": sample.value["node_hash"], "node_count": sample.value["node_count"], "edge_hash": sample.value["edge_hash"], "edge_count": sample.value["edge_count"], "logic_hash": sample.value["logic_hash"], "logic_projection_count": sample.value["logic_projection_count"], "action": synchronization["action"], "initial_error_code": cast(Mapping[str, object], synchronization["initial_error"])["code"], "cache_hits": build["cache_hits"], "reparsed_sources": build["reparsed_sources"], "ast_parse_calls": sample.ast_parse_calls, "extraction_calls": sample.extraction_calls, } def _measure( operation: Callable[[], _InstrumentedResult], *, samples: int, p95_limit_ms: float, summary: Callable[[_InstrumentedResult], Mapping[str, object]], ) -> tuple[dict[str, object], _InstrumentedResult]: durations: list[float] = [] traced_peaks: list[int] = [] response_sizes: list[int] = [] stable_summary: dict[str, object] | None = None last: _InstrumentedResult | None = None for _ in range(samples): gc.collect() tracemalloc.start() started = time.perf_counter_ns() try: current = operation() elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000 _, traced_peak = tracemalloc.get_traced_memory() finally: tracemalloc.stop() current_summary = dict(summary(current)) if stable_summary is None: stable_summary = current_summary elif current_summary != stable_summary: raise RuntimeError("Milestone 4 operation changed its deterministic result") response_bytes = len(_compact_json(current.value)) if response_bytes > MAX_RESPONSE_BYTES: raise RuntimeError( "Milestone 4 response exceeded its fixed byte boundary: " f"{response_bytes} > {MAX_RESPONSE_BYTES}" ) if traced_peak > MAX_TRACED_PEAK_BYTES: raise RuntimeError( "Milestone 4 operation exceeded its traced-memory boundary: " f"{traced_peak} > {MAX_TRACED_PEAK_BYTES}" ) durations.append(elapsed_ms) traced_peaks.append(traced_peak) response_sizes.append(response_bytes) last = current ordered = sorted(durations) p95_index = max(0, math.ceil(len(ordered) * 0.95) - 1) p95 = ordered[p95_index] if p95 > p95_limit_ms: raise RuntimeError(f"Milestone 4 operation p95 {p95:.3f} ms exceeds {p95_limit_ms:.3f} ms") assert stable_summary is not None assert last is not None return ( { "samples": samples, "median_ms": round(statistics.median(ordered), 3), "p95_ms": round(p95, 3), "min_ms": round(ordered[0], 3), "max_ms": round(ordered[-1], 3), "p95_limit_ms": p95_limit_ms, "maximum_response_bytes": max(response_sizes), "response_limit_bytes": MAX_RESPONSE_BYTES, "maximum_traced_peak_bytes": max(traced_peaks), "traced_peak_limit_bytes": MAX_TRACED_PEAK_BYTES, "stable_result": stable_summary, }, last, ) def _semantic_assembly_hash(assembly: AdapterAssembly) -> str: projection = assembly.projection return _sha256( { "project_id": projection.project_id, "title": projection.title, "adapter_id": projection.adapter_id, "adapter_version": projection.adapter_version, "revision": projection.revision, "source_hash": projection.source_hash, "nodes": [item.as_dict() for item in projection.nodes], "edges": [item.as_dict() for item in projection.edges], "logic": [item.as_dict() for item in assembly.logic], } ) def _benchmark(root: Path, source_count: int, samples: int) -> dict[str, object]: write_python_fixture(root, source_count) adapter = _RecordingPythonAdapter(root) cache_root = root / ".docforge" / "benchmark" project = AdapterProject(adapter, cache_root=cache_root) index = ProjectIndex(project) cold, cold_sample = _measure( lambda: _instrument(adapter, index.build), samples=1, p95_limit_ms=COLD_LIMIT_MS, summary=_build_summary, ) cold_summary = _build_summary(cold_sample) expected_nodes = source_count * NODES_PER_SOURCE if ( cold_summary["node_count"] != expected_nodes or cold_summary["logic_projection_count"] != source_count or cold_summary["reparsed_sources"] != source_count or cast(int, cold_summary["ast_parse_calls"]) < source_count or cold_summary["extraction_calls"] != source_count ): raise RuntimeError("Cold adapter build did not cover the complete synthetic fixture") warm, warm_sample = _measure( lambda: _instrument(adapter, index.build), samples=samples, p95_limit_ms=WARM_LIMIT_MS, summary=_build_summary, ) warm_summary = _build_summary(warm_sample) if ( warm_summary["cache_hits"] != source_count or warm_summary["reparsed_sources"] != 0 or warm_summary["ast_parse_calls"] != 0 or warm_summary["extraction_calls"] != 0 ): raise RuntimeError("Warm adapter build performed forbidden parsing or extraction") equivalence, equivalence_sample = _measure( lambda: _instrument(adapter, project.verify_incremental_equivalence), samples=1, p95_limit_ms=EQUIVALENCE_LIMIT_MS, summary=_equivalence_summary, ) equivalence_summary = _equivalence_summary(equivalence_sample) if ( equivalence_summary["status"] != "ok" or equivalence_summary["node_count"] != expected_nodes or equivalence_summary["logic_projection_count"] != source_count ): raise RuntimeError("Full and incremental graph plus Logic output was not equivalent") complete_assembly = adapter.load_complete_assembly() assembly_hash = _semantic_assembly_hash(complete_assembly) extraction_cache = cache_root / "extractions.json" extraction_cache.write_bytes(b"{broken") cache_recovery, cache_recovery_sample = _measure( lambda: _instrument(adapter, index.build), samples=1, p95_limit_ms=RECOVERY_LIMIT_MS, summary=_build_summary, ) cache_recovery_summary = _build_summary(cache_recovery_sample) if ( cache_recovery_summary["reparsed_sources"] != source_count or cast(int, cache_recovery_summary["ast_parse_calls"]) < source_count or cache_recovery_summary["node_hash"] != cold_summary["node_hash"] or cache_recovery_summary["logic_hash"] != cold_summary["logic_hash"] ): raise RuntimeError("Corrupt extraction-cache recovery changed adapter output") index.path.write_bytes(b"not-a-sqlite-index") index_recovery, index_recovery_sample = _measure( lambda: _instrument(adapter, index.synchronize), samples=1, p95_limit_ms=RECOVERY_LIMIT_MS, summary=_synchronization_summary, ) index_recovery_summary = _synchronization_summary(index_recovery_sample) if ( index_recovery_summary["action"] != "rebuilt" or index_recovery_summary["cache_hits"] != source_count or index_recovery_summary["reparsed_sources"] != 0 or index_recovery_summary["ast_parse_calls"] != 0 or index_recovery_summary["extraction_calls"] != 0 or index_recovery_summary["node_hash"] != cold_summary["node_hash"] or index_recovery_summary["logic_hash"] != cold_summary["logic_hash"] ): raise RuntimeError("Corrupt derived-index recovery changed adapter output") evidence = { "source_count": source_count, "node_count": expected_nodes, "edge_count": cold_summary["edge_count"], "logic_projection_count": source_count, "logic_node_count": cold_summary["logic_node_count"], "logic_edge_count": cold_summary["logic_edge_count"], "source_hash": cold_summary["source_hash"], "node_hash": cold_summary["node_hash"], "edge_hash": cold_summary["edge_hash"], "logic_hash": cold_summary["logic_hash"], "complete_assembly_hash": assembly_hash, "full_incremental_graph_and_logic_exact": True, "warm_zero_ast_parse": True, "warm_zero_source_extraction": True, "cache_recovery_exact": True, "index_recovery_exact": True, } return { "fixture": { "kind": "synthetic_python_reference_adapter", "source_count": source_count, "nodes_per_source": NODES_PER_SOURCE, "expected_node_count": expected_nodes, }, "operations": { "cold_incremental_build": cold, "warm_incremental_build": warm, "full_incremental_equivalence": equivalence, "corrupt_extraction_cache_recovery": cache_recovery, "corrupt_index_recovery": index_recovery, }, "evidence": evidence, "evidence_sha256": _sha256(evidence), } def main() -> int: arguments = _parser().parse_args() default_sources = FULL_SOURCE_COUNT if arguments.mode == "full" else SMOKE_SOURCE_COUNT default_samples = DEFAULT_FULL_SAMPLES if arguments.mode == "full" else DEFAULT_SMOKE_SAMPLES source_count = default_sources if arguments.sources is None else arguments.sources samples = default_samples if arguments.samples is None else arguments.samples if not 2 <= source_count <= FULL_SOURCE_COUNT: raise SystemExit(f"--sources must be between 2 and {FULL_SOURCE_COUNT}") if arguments.mode == "full" and source_count != FULL_SOURCE_COUNT: raise SystemExit(f"--mode full requires exactly {FULL_SOURCE_COUNT} sources") if samples < 1: raise SystemExit("--samples must be positive") with tempfile.TemporaryDirectory(prefix="docforge-milestone4-") as directory: measurement = _benchmark(Path(directory).resolve(), source_count, samples) process_peak_bytes = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) * 1024 if process_peak_bytes > MAX_PROCESS_PEAK_BYTES: raise RuntimeError( "Milestone 4 process peak exceeded its memory boundary: " f"{process_peak_bytes} > {MAX_PROCESS_PEAK_BYTES}" ) status = _git(["status", "--porcelain"]) report: dict[str, object] = { "schema_version": 1, "benchmark": "docforge2_milestone4", "mode": arguments.mode, "source": { "revision": _git(["rev-parse", "HEAD"]), "dirty": bool(status), }, "environment": { "platform": platform.platform(), "machine": platform.machine(), "python": platform.python_version(), "implementation": platform.python_implementation(), }, "method": { "clock": "time.perf_counter_ns", "in_process_peak_memory": "tracemalloc per measured invocation", "process_peak_memory": "resource.getrusage(RUSAGE_SELF).ru_maxrss", "response_size": "UTF-8 bytes of compact sorted JSON", "ast_instrumentation": "temporary counter around stdlib ast.parse", "extraction_instrumentation": "reference adapter extract_source entry counter", "samples": samples, "full_mode_source_requirement": FULL_SOURCE_COUNT, "threshold_basis": ( "Regression tripwires leave several times the Milestone 3 1,000-node " "allowances for AST, Logic, extraction-cache, and SQLite work." ), }, **measurement, "memory": { "process_peak_bytes": process_peak_bytes, "process_peak_limit_bytes": MAX_PROCESS_PEAK_BYTES, }, } encoded = encode_report(report) if arguments.output is not None: output = arguments.output.resolve() output.parent.mkdir(parents=True, exist_ok=True) output.write_text(encoded, encoding="utf-8") sys.stdout.write(encoded) return 0 if __name__ == "__main__": raise SystemExit(main())