From e525efd2fcb5823e75e4e7ddbe541ff418b4a702 Mon Sep 17 00:00:00 2001 From: Andraxion Date: Wed, 29 Jul 2026 15:03:31 -0400 Subject: [PATCH] Add Milestone 4 adapter benchmark gate --- Makefile | 13 +- tools/milestone4_benchmark.py | 498 ++++++++++++++++++++++++++++++++++ 2 files changed, 509 insertions(+), 2 deletions(-) create mode 100644 tools/milestone4_benchmark.py diff --git a/Makefile b/Makefile index 64d52c8..808435b 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ NPM := npm PYTHONPYCACHEPREFIX := /tmp/docforge-quality-pycache PYTEST_BASETEMP := /tmp/docforge-quality-pytest -.PHONY: accessibility benchmark benchmark-m1 benchmark-m1-smoke benchmark-m2 benchmark-m2-smoke benchmark-m3 benchmark-m3-full benchmark-m3-smoke benchmark-smoke build compile contract dependencies format-check gate lint lock test type +.PHONY: accessibility benchmark benchmark-m1 benchmark-m1-smoke benchmark-m2 benchmark-m2-smoke benchmark-m3 benchmark-m3-full benchmark-m3-smoke benchmark-m4 benchmark-m4-full benchmark-m4-smoke benchmark-smoke build compile contract dependencies format-check gate lint lock test type accessibility: $(NPM) run test:accessibility @@ -89,4 +89,13 @@ benchmark-m3: benchmark-m3-full: benchmark-m3 -gate: format-check lint type compile contract test accessibility lock dependencies build benchmark-smoke benchmark-m1-smoke benchmark-m2-smoke benchmark-m3-smoke +benchmark-m4-smoke: + $(PYTHON) tools/milestone4_benchmark.py --mode smoke \ + --output /tmp/docforge-milestone4-smoke.json > /dev/null + +benchmark-m4: + $(PYTHON) tools/milestone4_benchmark.py --mode full + +benchmark-m4-full: benchmark-m4 + +gate: format-check lint type compile contract test accessibility lock dependencies build benchmark-smoke benchmark-m1-smoke benchmark-m2-smoke benchmark-m3-smoke benchmark-m4-smoke diff --git a/tools/milestone4_benchmark.py b/tools/milestone4_benchmark.py new file mode 100644 index 0000000..c6a41c1 --- /dev/null +++ b/tools/milestone4_benchmark.py @@ -0,0 +1,498 @@ +"""Milestone 4 Python adapter SDK, incremental, and recovery benchmark gates.""" + +from __future__ import annotations + +import argparse +import ast +import gc +import hashlib +import json +import math +import platform +import resource +import statistics +import subprocess +import sys +import tempfile +import time +import tracemalloc +from collections.abc import Callable, Generator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import cast +from unittest import mock + +from docforge.adapter_sdk import ( + AdapterAssembly, + AdapterProject, + AdapterSource, + AdapterSourceProjection, +) +from docforge.adapters.python import PythonReferenceAdapter +from docforge.index import ProjectIndex + +ROOT = Path(__file__).resolve().parents[1] +FULL_SOURCE_COUNT = 334 +SMOKE_SOURCE_COUNT = 12 +NODES_PER_SOURCE = 3 +DEFAULT_FULL_SAMPLES = 3 +DEFAULT_SMOKE_SAMPLES = 1 + +# These are regression tripwires, not performance targets. They deliberately +# leave several times the Milestone 3 1,000-node in-process allowances for the +# additional AST, Logic, extraction-cache, and SQLite work in this gate. +MAX_TRACED_PEAK_BYTES = 256 * 1024 * 1024 +MAX_PROCESS_PEAK_BYTES = 512 * 1024 * 1024 +MAX_RESPONSE_BYTES = 256 * 1024 +COLD_LIMIT_MS = 30_000.0 +WARM_LIMIT_MS = 20_000.0 +EQUIVALENCE_LIMIT_MS = 30_000.0 +RECOVERY_LIMIT_MS = 30_000.0 + + +@dataclass(frozen=True) +class _InstrumentedResult: + value: Mapping[str, object] + ast_parse_calls: int + extraction_calls: int + + +class _RecordingPythonAdapter(PythonReferenceAdapter): + """Count extraction entry points without weakening the production adapter.""" + + def __init__(self, root: Path) -> None: + super().__init__( + root, + source_roots=("src",), + project_id="milestone4-python-benchmark", + title="Milestone 4 Python adapter benchmark", + ) + self.extraction_calls = 0 + + def extract_source(self, source: AdapterSource) -> AdapterSourceProjection: + self.extraction_calls += 1 + return super().extract_source(source) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=("Gate DocForge2 Milestone 4 adapter behavior on a disposable Python project.") + ) + parser.add_argument("--mode", choices=("smoke", "full"), default="full") + parser.add_argument("--sources", type=int) + parser.add_argument("--samples", type=int) + parser.add_argument("--output", type=Path) + return parser + + +def encode_report(value: object) -> str: + """Serialize benchmark evidence in one deterministic JSON representation.""" + + return json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False) + "\n" + + +def _compact_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + + +def _sha256(value: object) -> str: + return hashlib.sha256(_compact_json(value)).hexdigest() + + +def _git(arguments: list[str]) -> str: + return subprocess.run( + ["git", *arguments], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def write_python_fixture(root: Path, source_count: int) -> None: + """Write a deterministic project with three primary nodes per source.""" + + if source_count < 2: + raise ValueError("The Milestone 4 fixture requires at least two sources") + package = root / "src" / "benchmark" + package.mkdir(parents=True) + for index in range(source_count): + dependency = "" + call = f"value = {index}" + if index: + dependency = f"from benchmark.module_{index - 1:04d} import compute_{index - 1:04d}\n\n" + call = f"value = compute_{index - 1:04d}(value)" + source = ( + f'"""Synthetic adapter benchmark module {index:04d}."""\n\n' + f"{dependency}" + f"def compute_{index:04d}(value: int) -> int:\n" + f' """Return one deterministic branch result."""\n' + f" {call}\n" + " if value % 2:\n" + f" return value + {index + 1}\n" + " return value\n" + ) + (package / f"module_{index:04d}.py").write_text(source, encoding="utf-8") + + +@contextmanager +def _count_ast_parses() -> Generator[mock.MagicMock]: + with mock.patch.object(ast, "parse", wraps=ast.parse) as parsed: + yield parsed + + +def _instrument( + adapter: _RecordingPythonAdapter, + operation: Callable[[], Mapping[str, object]], +) -> _InstrumentedResult: + adapter.extraction_calls = 0 + with _count_ast_parses() as counter: + value = operation() + return _InstrumentedResult( + value=value, + ast_parse_calls=counter.call_count, + extraction_calls=adapter.extraction_calls, + ) + + +def _build_summary(sample: _InstrumentedResult) -> dict[str, object]: + value = sample.value + build = cast(Mapping[str, object], value["build"]) + return { + "status": value["status"], + "revision": value["revision"], + "source_hash": value["source_hash"], + "node_hash": value["node_hash"], + "node_count": value["node_count"], + "edge_hash": value["edge_hash"], + "edge_count": value["edge_count"], + "logic_hash": value["logic_hash"], + "logic_projection_count": value["logic_projection_count"], + "logic_node_count": value["logic_node_count"], + "logic_edge_count": value["logic_edge_count"], + "cache_hits": build["cache_hits"], + "reparsed_sources": build["reparsed_sources"], + "invalidated_sources": build["invalidated_sources"], + "deleted_sources": build["deleted_sources"], + "total_sources": build["total_sources"], + "ast_parse_calls": sample.ast_parse_calls, + "extraction_calls": sample.extraction_calls, + } + + +def _equivalence_summary(sample: _InstrumentedResult) -> dict[str, object]: + return { + **sample.value, + "ast_parse_calls": sample.ast_parse_calls, + "extraction_calls": sample.extraction_calls, + } + + +def _synchronization_summary(sample: _InstrumentedResult) -> dict[str, object]: + synchronization = cast(Mapping[str, object], sample.value["synchronization"]) + build = cast(Mapping[str, object], synchronization["build"]) + return { + "status": sample.value["status"], + "revision": sample.value["revision"], + "source_hash": sample.value["source_hash"], + "node_hash": sample.value["node_hash"], + "node_count": sample.value["node_count"], + "edge_hash": sample.value["edge_hash"], + "edge_count": sample.value["edge_count"], + "logic_hash": sample.value["logic_hash"], + "logic_projection_count": sample.value["logic_projection_count"], + "action": synchronization["action"], + "initial_error_code": cast(Mapping[str, object], synchronization["initial_error"])["code"], + "cache_hits": build["cache_hits"], + "reparsed_sources": build["reparsed_sources"], + "ast_parse_calls": sample.ast_parse_calls, + "extraction_calls": sample.extraction_calls, + } + + +def _measure( + operation: Callable[[], _InstrumentedResult], + *, + samples: int, + p95_limit_ms: float, + summary: Callable[[_InstrumentedResult], Mapping[str, object]], +) -> tuple[dict[str, object], _InstrumentedResult]: + durations: list[float] = [] + traced_peaks: list[int] = [] + response_sizes: list[int] = [] + stable_summary: dict[str, object] | None = None + last: _InstrumentedResult | None = None + for _ in range(samples): + gc.collect() + tracemalloc.start() + started = time.perf_counter_ns() + try: + current = operation() + elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000 + _, traced_peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + current_summary = dict(summary(current)) + if stable_summary is None: + stable_summary = current_summary + elif current_summary != stable_summary: + raise RuntimeError("Milestone 4 operation changed its deterministic result") + response_bytes = len(_compact_json(current.value)) + if response_bytes > MAX_RESPONSE_BYTES: + raise RuntimeError( + "Milestone 4 response exceeded its fixed byte boundary: " + f"{response_bytes} > {MAX_RESPONSE_BYTES}" + ) + if traced_peak > MAX_TRACED_PEAK_BYTES: + raise RuntimeError( + "Milestone 4 operation exceeded its traced-memory boundary: " + f"{traced_peak} > {MAX_TRACED_PEAK_BYTES}" + ) + durations.append(elapsed_ms) + traced_peaks.append(traced_peak) + response_sizes.append(response_bytes) + last = current + ordered = sorted(durations) + p95_index = max(0, math.ceil(len(ordered) * 0.95) - 1) + p95 = ordered[p95_index] + if p95 > p95_limit_ms: + raise RuntimeError(f"Milestone 4 operation p95 {p95:.3f} ms exceeds {p95_limit_ms:.3f} ms") + assert stable_summary is not None + assert last is not None + return ( + { + "samples": samples, + "median_ms": round(statistics.median(ordered), 3), + "p95_ms": round(p95, 3), + "min_ms": round(ordered[0], 3), + "max_ms": round(ordered[-1], 3), + "p95_limit_ms": p95_limit_ms, + "maximum_response_bytes": max(response_sizes), + "response_limit_bytes": MAX_RESPONSE_BYTES, + "maximum_traced_peak_bytes": max(traced_peaks), + "traced_peak_limit_bytes": MAX_TRACED_PEAK_BYTES, + "stable_result": stable_summary, + }, + last, + ) + + +def _semantic_assembly_hash(assembly: AdapterAssembly) -> str: + projection = assembly.projection + return _sha256( + { + "project_id": projection.project_id, + "title": projection.title, + "adapter_id": projection.adapter_id, + "adapter_version": projection.adapter_version, + "revision": projection.revision, + "source_hash": projection.source_hash, + "nodes": [item.as_dict() for item in projection.nodes], + "edges": [item.as_dict() for item in projection.edges], + "logic": [item.as_dict() for item in assembly.logic], + } + ) + + +def _benchmark(root: Path, source_count: int, samples: int) -> dict[str, object]: + write_python_fixture(root, source_count) + adapter = _RecordingPythonAdapter(root) + cache_root = root / ".docforge" / "benchmark" + project = AdapterProject(adapter, cache_root=cache_root) + index = ProjectIndex(project) + + cold, cold_sample = _measure( + lambda: _instrument(adapter, index.build), + samples=1, + p95_limit_ms=COLD_LIMIT_MS, + summary=_build_summary, + ) + cold_summary = _build_summary(cold_sample) + expected_nodes = source_count * NODES_PER_SOURCE + if ( + cold_summary["node_count"] != expected_nodes + or cold_summary["logic_projection_count"] != source_count + or cold_summary["reparsed_sources"] != source_count + or cast(int, cold_summary["ast_parse_calls"]) < source_count + or cold_summary["extraction_calls"] != source_count + ): + raise RuntimeError("Cold adapter build did not cover the complete synthetic fixture") + + warm, warm_sample = _measure( + lambda: _instrument(adapter, index.build), + samples=samples, + p95_limit_ms=WARM_LIMIT_MS, + summary=_build_summary, + ) + warm_summary = _build_summary(warm_sample) + if ( + warm_summary["cache_hits"] != source_count + or warm_summary["reparsed_sources"] != 0 + or warm_summary["ast_parse_calls"] != 0 + or warm_summary["extraction_calls"] != 0 + ): + raise RuntimeError("Warm adapter build performed forbidden parsing or extraction") + + equivalence, equivalence_sample = _measure( + lambda: _instrument(adapter, project.verify_incremental_equivalence), + samples=1, + p95_limit_ms=EQUIVALENCE_LIMIT_MS, + summary=_equivalence_summary, + ) + equivalence_summary = _equivalence_summary(equivalence_sample) + if ( + equivalence_summary["status"] != "ok" + or equivalence_summary["node_count"] != expected_nodes + or equivalence_summary["logic_projection_count"] != source_count + ): + raise RuntimeError("Full and incremental graph plus Logic output was not equivalent") + complete_assembly = adapter.load_complete_assembly() + assembly_hash = _semantic_assembly_hash(complete_assembly) + + extraction_cache = cache_root / "extractions.json" + extraction_cache.write_bytes(b"{broken") + cache_recovery, cache_recovery_sample = _measure( + lambda: _instrument(adapter, index.build), + samples=1, + p95_limit_ms=RECOVERY_LIMIT_MS, + summary=_build_summary, + ) + cache_recovery_summary = _build_summary(cache_recovery_sample) + if ( + cache_recovery_summary["reparsed_sources"] != source_count + or cast(int, cache_recovery_summary["ast_parse_calls"]) < source_count + or cache_recovery_summary["node_hash"] != cold_summary["node_hash"] + or cache_recovery_summary["logic_hash"] != cold_summary["logic_hash"] + ): + raise RuntimeError("Corrupt extraction-cache recovery changed adapter output") + + index.path.write_bytes(b"not-a-sqlite-index") + index_recovery, index_recovery_sample = _measure( + lambda: _instrument(adapter, index.synchronize), + samples=1, + p95_limit_ms=RECOVERY_LIMIT_MS, + summary=_synchronization_summary, + ) + index_recovery_summary = _synchronization_summary(index_recovery_sample) + if ( + index_recovery_summary["action"] != "rebuilt" + or index_recovery_summary["cache_hits"] != source_count + or index_recovery_summary["reparsed_sources"] != 0 + or index_recovery_summary["ast_parse_calls"] != 0 + or index_recovery_summary["extraction_calls"] != 0 + or index_recovery_summary["node_hash"] != cold_summary["node_hash"] + or index_recovery_summary["logic_hash"] != cold_summary["logic_hash"] + ): + raise RuntimeError("Corrupt derived-index recovery changed adapter output") + + evidence = { + "source_count": source_count, + "node_count": expected_nodes, + "edge_count": cold_summary["edge_count"], + "logic_projection_count": source_count, + "logic_node_count": cold_summary["logic_node_count"], + "logic_edge_count": cold_summary["logic_edge_count"], + "source_hash": cold_summary["source_hash"], + "node_hash": cold_summary["node_hash"], + "edge_hash": cold_summary["edge_hash"], + "logic_hash": cold_summary["logic_hash"], + "complete_assembly_hash": assembly_hash, + "full_incremental_graph_and_logic_exact": True, + "warm_zero_ast_parse": True, + "warm_zero_source_extraction": True, + "cache_recovery_exact": True, + "index_recovery_exact": True, + } + return { + "fixture": { + "kind": "synthetic_python_reference_adapter", + "source_count": source_count, + "nodes_per_source": NODES_PER_SOURCE, + "expected_node_count": expected_nodes, + }, + "operations": { + "cold_incremental_build": cold, + "warm_incremental_build": warm, + "full_incremental_equivalence": equivalence, + "corrupt_extraction_cache_recovery": cache_recovery, + "corrupt_index_recovery": index_recovery, + }, + "evidence": evidence, + "evidence_sha256": _sha256(evidence), + } + + +def main() -> int: + arguments = _parser().parse_args() + default_sources = FULL_SOURCE_COUNT if arguments.mode == "full" else SMOKE_SOURCE_COUNT + default_samples = DEFAULT_FULL_SAMPLES if arguments.mode == "full" else DEFAULT_SMOKE_SAMPLES + source_count = default_sources if arguments.sources is None else arguments.sources + samples = default_samples if arguments.samples is None else arguments.samples + if not 2 <= source_count <= FULL_SOURCE_COUNT: + raise SystemExit(f"--sources must be between 2 and {FULL_SOURCE_COUNT}") + if arguments.mode == "full" and source_count != FULL_SOURCE_COUNT: + raise SystemExit(f"--mode full requires exactly {FULL_SOURCE_COUNT} sources") + if samples < 1: + raise SystemExit("--samples must be positive") + + with tempfile.TemporaryDirectory(prefix="docforge-milestone4-") as directory: + measurement = _benchmark(Path(directory).resolve(), source_count, samples) + + process_peak_bytes = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) * 1024 + if process_peak_bytes > MAX_PROCESS_PEAK_BYTES: + raise RuntimeError( + "Milestone 4 process peak exceeded its memory boundary: " + f"{process_peak_bytes} > {MAX_PROCESS_PEAK_BYTES}" + ) + status = _git(["status", "--porcelain"]) + report: dict[str, object] = { + "schema_version": 1, + "benchmark": "docforge2_milestone4", + "mode": arguments.mode, + "source": { + "revision": _git(["rev-parse", "HEAD"]), + "dirty": bool(status), + }, + "environment": { + "platform": platform.platform(), + "machine": platform.machine(), + "python": platform.python_version(), + "implementation": platform.python_implementation(), + }, + "method": { + "clock": "time.perf_counter_ns", + "in_process_peak_memory": "tracemalloc per measured invocation", + "process_peak_memory": "resource.getrusage(RUSAGE_SELF).ru_maxrss", + "response_size": "UTF-8 bytes of compact sorted JSON", + "ast_instrumentation": "temporary counter around stdlib ast.parse", + "extraction_instrumentation": "reference adapter extract_source entry counter", + "samples": samples, + "full_mode_source_requirement": FULL_SOURCE_COUNT, + "threshold_basis": ( + "Regression tripwires leave several times the Milestone 3 1,000-node " + "allowances for AST, Logic, extraction-cache, and SQLite work." + ), + }, + **measurement, + "memory": { + "process_peak_bytes": process_peak_bytes, + "process_peak_limit_bytes": MAX_PROCESS_PEAK_BYTES, + }, + } + encoded = encode_report(report) + if arguments.output is not None: + output = arguments.output.resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(encoded, encoding="utf-8") + sys.stdout.write(encoded) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())