Add Milestone 4 adapter benchmark gate

Makefile

@@ -5,7 +5,7 @@ NPM := npm
 PYTHONPYCACHEPREFIX := /tmp/docforge-quality-pycache
 PYTEST_BASETEMP := /tmp/docforge-quality-pytest
 
-.PHONY: accessibility benchmark benchmark-m1 benchmark-m1-smoke benchmark-m2 benchmark-m2-smoke benchmark-m3 benchmark-m3-full benchmark-m3-smoke benchmark-smoke build compile contract dependencies format-check gate lint lock test type
+.PHONY: accessibility benchmark benchmark-m1 benchmark-m1-smoke benchmark-m2 benchmark-m2-smoke benchmark-m3 benchmark-m3-full benchmark-m3-smoke benchmark-m4 benchmark-m4-full benchmark-m4-smoke benchmark-smoke build compile contract dependencies format-check gate lint lock test type
 
 accessibility:
 	$(NPM) run test:accessibility
@@ -89,4 +89,13 @@ benchmark-m3:
 
 benchmark-m3-full: benchmark-m3
 
-gate: format-check lint type compile contract test accessibility lock dependencies build benchmark-smoke benchmark-m1-smoke benchmark-m2-smoke benchmark-m3-smoke
+benchmark-m4-smoke:
+	$(PYTHON) tools/milestone4_benchmark.py --mode smoke \
+		--output /tmp/docforge-milestone4-smoke.json > /dev/null
+
+benchmark-m4:
+	$(PYTHON) tools/milestone4_benchmark.py --mode full
+
+benchmark-m4-full: benchmark-m4
+
+gate: format-check lint type compile contract test accessibility lock dependencies build benchmark-smoke benchmark-m1-smoke benchmark-m2-smoke benchmark-m3-smoke benchmark-m4-smoke

tools/milestone4_benchmark.py

@@ -0,0 +1,498 @@
+"""Milestone 4 Python adapter SDK, incremental, and recovery benchmark gates."""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import gc
+import hashlib
+import json
+import math
+import platform
+import resource
+import statistics
+import subprocess
+import sys
+import tempfile
+import time
+import tracemalloc
+from collections.abc import Callable, Generator, Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+from unittest import mock
+
+from docforge.adapter_sdk import (
+    AdapterAssembly,
+    AdapterProject,
+    AdapterSource,
+    AdapterSourceProjection,
+)
+from docforge.adapters.python import PythonReferenceAdapter
+from docforge.index import ProjectIndex
+
+ROOT = Path(__file__).resolve().parents[1]
+FULL_SOURCE_COUNT = 334
+SMOKE_SOURCE_COUNT = 12
+NODES_PER_SOURCE = 3
+DEFAULT_FULL_SAMPLES = 3
+DEFAULT_SMOKE_SAMPLES = 1
+
+# These are regression tripwires, not performance targets. They deliberately
+# leave several times the Milestone 3 1,000-node in-process allowances for the
+# additional AST, Logic, extraction-cache, and SQLite work in this gate.
+MAX_TRACED_PEAK_BYTES = 256 * 1024 * 1024
+MAX_PROCESS_PEAK_BYTES = 512 * 1024 * 1024
+MAX_RESPONSE_BYTES = 256 * 1024
+COLD_LIMIT_MS = 30_000.0
+WARM_LIMIT_MS = 20_000.0
+EQUIVALENCE_LIMIT_MS = 30_000.0
+RECOVERY_LIMIT_MS = 30_000.0
+
+
+@dataclass(frozen=True)
+class _InstrumentedResult:
+    value: Mapping[str, object]
+    ast_parse_calls: int
+    extraction_calls: int
+
+
+class _RecordingPythonAdapter(PythonReferenceAdapter):
+    """Count extraction entry points without weakening the production adapter."""
+
+    def __init__(self, root: Path) -> None:
+        super().__init__(
+            root,
+            source_roots=("src",),
+            project_id="milestone4-python-benchmark",
+            title="Milestone 4 Python adapter benchmark",
+        )
+        self.extraction_calls = 0
+
+    def extract_source(self, source: AdapterSource) -> AdapterSourceProjection:
+        self.extraction_calls += 1
+        return super().extract_source(source)
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=("Gate DocForge2 Milestone 4 adapter behavior on a disposable Python project.")
+    )
+    parser.add_argument("--mode", choices=("smoke", "full"), default="full")
+    parser.add_argument("--sources", type=int)
+    parser.add_argument("--samples", type=int)
+    parser.add_argument("--output", type=Path)
+    return parser
+
+
+def encode_report(value: object) -> str:
+    """Serialize benchmark evidence in one deterministic JSON representation."""
+
+    return json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False) + "\n"
+
+
+def _compact_json(value: object) -> bytes:
+    return json.dumps(
+        value,
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+    ).encode("utf-8")
+
+
+def _sha256(value: object) -> str:
+    return hashlib.sha256(_compact_json(value)).hexdigest()
+
+
+def _git(arguments: list[str]) -> str:
+    return subprocess.run(
+        ["git", *arguments],
+        cwd=ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+
+
+def write_python_fixture(root: Path, source_count: int) -> None:
+    """Write a deterministic project with three primary nodes per source."""
+
+    if source_count < 2:
+        raise ValueError("The Milestone 4 fixture requires at least two sources")
+    package = root / "src" / "benchmark"
+    package.mkdir(parents=True)
+    for index in range(source_count):
+        dependency = ""
+        call = f"value = {index}"
+        if index:
+            dependency = f"from benchmark.module_{index - 1:04d} import compute_{index - 1:04d}\n\n"
+            call = f"value = compute_{index - 1:04d}(value)"
+        source = (
+            f'"""Synthetic adapter benchmark module {index:04d}."""\n\n'
+            f"{dependency}"
+            f"def compute_{index:04d}(value: int) -> int:\n"
+            f'    """Return one deterministic branch result."""\n'
+            f"    {call}\n"
+            "    if value % 2:\n"
+            f"        return value + {index + 1}\n"
+            "    return value\n"
+        )
+        (package / f"module_{index:04d}.py").write_text(source, encoding="utf-8")
+
+
+@contextmanager
+def _count_ast_parses() -> Generator[mock.MagicMock]:
+    with mock.patch.object(ast, "parse", wraps=ast.parse) as parsed:
+        yield parsed
+
+
+def _instrument(
+    adapter: _RecordingPythonAdapter,
+    operation: Callable[[], Mapping[str, object]],
+) -> _InstrumentedResult:
+    adapter.extraction_calls = 0
+    with _count_ast_parses() as counter:
+        value = operation()
+    return _InstrumentedResult(
+        value=value,
+        ast_parse_calls=counter.call_count,
+        extraction_calls=adapter.extraction_calls,
+    )
+
+
+def _build_summary(sample: _InstrumentedResult) -> dict[str, object]:
+    value = sample.value
+    build = cast(Mapping[str, object], value["build"])
+    return {
+        "status": value["status"],
+        "revision": value["revision"],
+        "source_hash": value["source_hash"],
+        "node_hash": value["node_hash"],
+        "node_count": value["node_count"],
+        "edge_hash": value["edge_hash"],
+        "edge_count": value["edge_count"],
+        "logic_hash": value["logic_hash"],
+        "logic_projection_count": value["logic_projection_count"],
+        "logic_node_count": value["logic_node_count"],
+        "logic_edge_count": value["logic_edge_count"],
+        "cache_hits": build["cache_hits"],
+        "reparsed_sources": build["reparsed_sources"],
+        "invalidated_sources": build["invalidated_sources"],
+        "deleted_sources": build["deleted_sources"],
+        "total_sources": build["total_sources"],
+        "ast_parse_calls": sample.ast_parse_calls,
+        "extraction_calls": sample.extraction_calls,
+    }
+
+
+def _equivalence_summary(sample: _InstrumentedResult) -> dict[str, object]:
+    return {
+        **sample.value,
+        "ast_parse_calls": sample.ast_parse_calls,
+        "extraction_calls": sample.extraction_calls,
+    }
+
+
+def _synchronization_summary(sample: _InstrumentedResult) -> dict[str, object]:
+    synchronization = cast(Mapping[str, object], sample.value["synchronization"])
+    build = cast(Mapping[str, object], synchronization["build"])
+    return {
+        "status": sample.value["status"],
+        "revision": sample.value["revision"],
+        "source_hash": sample.value["source_hash"],
+        "node_hash": sample.value["node_hash"],
+        "node_count": sample.value["node_count"],
+        "edge_hash": sample.value["edge_hash"],
+        "edge_count": sample.value["edge_count"],
+        "logic_hash": sample.value["logic_hash"],
+        "logic_projection_count": sample.value["logic_projection_count"],
+        "action": synchronization["action"],
+        "initial_error_code": cast(Mapping[str, object], synchronization["initial_error"])["code"],
+        "cache_hits": build["cache_hits"],
+        "reparsed_sources": build["reparsed_sources"],
+        "ast_parse_calls": sample.ast_parse_calls,
+        "extraction_calls": sample.extraction_calls,
+    }
+
+
+def _measure(
+    operation: Callable[[], _InstrumentedResult],
+    *,
+    samples: int,
+    p95_limit_ms: float,
+    summary: Callable[[_InstrumentedResult], Mapping[str, object]],
+) -> tuple[dict[str, object], _InstrumentedResult]:
+    durations: list[float] = []
+    traced_peaks: list[int] = []
+    response_sizes: list[int] = []
+    stable_summary: dict[str, object] | None = None
+    last: _InstrumentedResult | None = None
+    for _ in range(samples):
+        gc.collect()
+        tracemalloc.start()
+        started = time.perf_counter_ns()
+        try:
+            current = operation()
+            elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000
+            _, traced_peak = tracemalloc.get_traced_memory()
+        finally:
+            tracemalloc.stop()
+        current_summary = dict(summary(current))
+        if stable_summary is None:
+            stable_summary = current_summary
+        elif current_summary != stable_summary:
+            raise RuntimeError("Milestone 4 operation changed its deterministic result")
+        response_bytes = len(_compact_json(current.value))
+        if response_bytes > MAX_RESPONSE_BYTES:
+            raise RuntimeError(
+                "Milestone 4 response exceeded its fixed byte boundary: "
+                f"{response_bytes} > {MAX_RESPONSE_BYTES}"
+            )
+        if traced_peak > MAX_TRACED_PEAK_BYTES:
+            raise RuntimeError(
+                "Milestone 4 operation exceeded its traced-memory boundary: "
+                f"{traced_peak} > {MAX_TRACED_PEAK_BYTES}"
+            )
+        durations.append(elapsed_ms)
+        traced_peaks.append(traced_peak)
+        response_sizes.append(response_bytes)
+        last = current
+    ordered = sorted(durations)
+    p95_index = max(0, math.ceil(len(ordered) * 0.95) - 1)
+    p95 = ordered[p95_index]
+    if p95 > p95_limit_ms:
+        raise RuntimeError(f"Milestone 4 operation p95 {p95:.3f} ms exceeds {p95_limit_ms:.3f} ms")
+    assert stable_summary is not None
+    assert last is not None
+    return (
+        {
+            "samples": samples,
+            "median_ms": round(statistics.median(ordered), 3),
+            "p95_ms": round(p95, 3),
+            "min_ms": round(ordered[0], 3),
+            "max_ms": round(ordered[-1], 3),
+            "p95_limit_ms": p95_limit_ms,
+            "maximum_response_bytes": max(response_sizes),
+            "response_limit_bytes": MAX_RESPONSE_BYTES,
+            "maximum_traced_peak_bytes": max(traced_peaks),
+            "traced_peak_limit_bytes": MAX_TRACED_PEAK_BYTES,
+            "stable_result": stable_summary,
+        },
+        last,
+    )
+
+
+def _semantic_assembly_hash(assembly: AdapterAssembly) -> str:
+    projection = assembly.projection
+    return _sha256(
+        {
+            "project_id": projection.project_id,
+            "title": projection.title,
+            "adapter_id": projection.adapter_id,
+            "adapter_version": projection.adapter_version,
+            "revision": projection.revision,
+            "source_hash": projection.source_hash,
+            "nodes": [item.as_dict() for item in projection.nodes],
+            "edges": [item.as_dict() for item in projection.edges],
+            "logic": [item.as_dict() for item in assembly.logic],
+        }
+    )
+
+
+def _benchmark(root: Path, source_count: int, samples: int) -> dict[str, object]:
+    write_python_fixture(root, source_count)
+    adapter = _RecordingPythonAdapter(root)
+    cache_root = root / ".docforge" / "benchmark"
+    project = AdapterProject(adapter, cache_root=cache_root)
+    index = ProjectIndex(project)
+
+    cold, cold_sample = _measure(
+        lambda: _instrument(adapter, index.build),
+        samples=1,
+        p95_limit_ms=COLD_LIMIT_MS,
+        summary=_build_summary,
+    )
+    cold_summary = _build_summary(cold_sample)
+    expected_nodes = source_count * NODES_PER_SOURCE
+    if (
+        cold_summary["node_count"] != expected_nodes
+        or cold_summary["logic_projection_count"] != source_count
+        or cold_summary["reparsed_sources"] != source_count
+        or cast(int, cold_summary["ast_parse_calls"]) < source_count
+        or cold_summary["extraction_calls"] != source_count
+    ):
+        raise RuntimeError("Cold adapter build did not cover the complete synthetic fixture")
+
+    warm, warm_sample = _measure(
+        lambda: _instrument(adapter, index.build),
+        samples=samples,
+        p95_limit_ms=WARM_LIMIT_MS,
+        summary=_build_summary,
+    )
+    warm_summary = _build_summary(warm_sample)
+    if (
+        warm_summary["cache_hits"] != source_count
+        or warm_summary["reparsed_sources"] != 0
+        or warm_summary["ast_parse_calls"] != 0
+        or warm_summary["extraction_calls"] != 0
+    ):
+        raise RuntimeError("Warm adapter build performed forbidden parsing or extraction")
+
+    equivalence, equivalence_sample = _measure(
+        lambda: _instrument(adapter, project.verify_incremental_equivalence),
+        samples=1,
+        p95_limit_ms=EQUIVALENCE_LIMIT_MS,
+        summary=_equivalence_summary,
+    )
+    equivalence_summary = _equivalence_summary(equivalence_sample)
+    if (
+        equivalence_summary["status"] != "ok"
+        or equivalence_summary["node_count"] != expected_nodes
+        or equivalence_summary["logic_projection_count"] != source_count
+    ):
+        raise RuntimeError("Full and incremental graph plus Logic output was not equivalent")
+    complete_assembly = adapter.load_complete_assembly()
+    assembly_hash = _semantic_assembly_hash(complete_assembly)
+
+    extraction_cache = cache_root / "extractions.json"
+    extraction_cache.write_bytes(b"{broken")
+    cache_recovery, cache_recovery_sample = _measure(
+        lambda: _instrument(adapter, index.build),
+        samples=1,
+        p95_limit_ms=RECOVERY_LIMIT_MS,
+        summary=_build_summary,
+    )
+    cache_recovery_summary = _build_summary(cache_recovery_sample)
+    if (
+        cache_recovery_summary["reparsed_sources"] != source_count
+        or cast(int, cache_recovery_summary["ast_parse_calls"]) < source_count
+        or cache_recovery_summary["node_hash"] != cold_summary["node_hash"]
+        or cache_recovery_summary["logic_hash"] != cold_summary["logic_hash"]
+    ):
+        raise RuntimeError("Corrupt extraction-cache recovery changed adapter output")
+
+    index.path.write_bytes(b"not-a-sqlite-index")
+    index_recovery, index_recovery_sample = _measure(
+        lambda: _instrument(adapter, index.synchronize),
+        samples=1,
+        p95_limit_ms=RECOVERY_LIMIT_MS,
+        summary=_synchronization_summary,
+    )
+    index_recovery_summary = _synchronization_summary(index_recovery_sample)
+    if (
+        index_recovery_summary["action"] != "rebuilt"
+        or index_recovery_summary["cache_hits"] != source_count
+        or index_recovery_summary["reparsed_sources"] != 0
+        or index_recovery_summary["ast_parse_calls"] != 0
+        or index_recovery_summary["extraction_calls"] != 0
+        or index_recovery_summary["node_hash"] != cold_summary["node_hash"]
+        or index_recovery_summary["logic_hash"] != cold_summary["logic_hash"]
+    ):
+        raise RuntimeError("Corrupt derived-index recovery changed adapter output")
+
+    evidence = {
+        "source_count": source_count,
+        "node_count": expected_nodes,
+        "edge_count": cold_summary["edge_count"],
+        "logic_projection_count": source_count,
+        "logic_node_count": cold_summary["logic_node_count"],
+        "logic_edge_count": cold_summary["logic_edge_count"],
+        "source_hash": cold_summary["source_hash"],
+        "node_hash": cold_summary["node_hash"],
+        "edge_hash": cold_summary["edge_hash"],
+        "logic_hash": cold_summary["logic_hash"],
+        "complete_assembly_hash": assembly_hash,
+        "full_incremental_graph_and_logic_exact": True,
+        "warm_zero_ast_parse": True,
+        "warm_zero_source_extraction": True,
+        "cache_recovery_exact": True,
+        "index_recovery_exact": True,
+    }
+    return {
+        "fixture": {
+            "kind": "synthetic_python_reference_adapter",
+            "source_count": source_count,
+            "nodes_per_source": NODES_PER_SOURCE,
+            "expected_node_count": expected_nodes,
+        },
+        "operations": {
+            "cold_incremental_build": cold,
+            "warm_incremental_build": warm,
+            "full_incremental_equivalence": equivalence,
+            "corrupt_extraction_cache_recovery": cache_recovery,
+            "corrupt_index_recovery": index_recovery,
+        },
+        "evidence": evidence,
+        "evidence_sha256": _sha256(evidence),
+    }
+
+
+def main() -> int:
+    arguments = _parser().parse_args()
+    default_sources = FULL_SOURCE_COUNT if arguments.mode == "full" else SMOKE_SOURCE_COUNT
+    default_samples = DEFAULT_FULL_SAMPLES if arguments.mode == "full" else DEFAULT_SMOKE_SAMPLES
+    source_count = default_sources if arguments.sources is None else arguments.sources
+    samples = default_samples if arguments.samples is None else arguments.samples
+    if not 2 <= source_count <= FULL_SOURCE_COUNT:
+        raise SystemExit(f"--sources must be between 2 and {FULL_SOURCE_COUNT}")
+    if arguments.mode == "full" and source_count != FULL_SOURCE_COUNT:
+        raise SystemExit(f"--mode full requires exactly {FULL_SOURCE_COUNT} sources")
+    if samples < 1:
+        raise SystemExit("--samples must be positive")
+
+    with tempfile.TemporaryDirectory(prefix="docforge-milestone4-") as directory:
+        measurement = _benchmark(Path(directory).resolve(), source_count, samples)
+
+    process_peak_bytes = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) * 1024
+    if process_peak_bytes > MAX_PROCESS_PEAK_BYTES:
+        raise RuntimeError(
+            "Milestone 4 process peak exceeded its memory boundary: "
+            f"{process_peak_bytes} > {MAX_PROCESS_PEAK_BYTES}"
+        )
+    status = _git(["status", "--porcelain"])
+    report: dict[str, object] = {
+        "schema_version": 1,
+        "benchmark": "docforge2_milestone4",
+        "mode": arguments.mode,
+        "source": {
+            "revision": _git(["rev-parse", "HEAD"]),
+            "dirty": bool(status),
+        },
+        "environment": {
+            "platform": platform.platform(),
+            "machine": platform.machine(),
+            "python": platform.python_version(),
+            "implementation": platform.python_implementation(),
+        },
+        "method": {
+            "clock": "time.perf_counter_ns",
+            "in_process_peak_memory": "tracemalloc per measured invocation",
+            "process_peak_memory": "resource.getrusage(RUSAGE_SELF).ru_maxrss",
+            "response_size": "UTF-8 bytes of compact sorted JSON",
+            "ast_instrumentation": "temporary counter around stdlib ast.parse",
+            "extraction_instrumentation": "reference adapter extract_source entry counter",
+            "samples": samples,
+            "full_mode_source_requirement": FULL_SOURCE_COUNT,
+            "threshold_basis": (
+                "Regression tripwires leave several times the Milestone 3 1,000-node "
+                "allowances for AST, Logic, extraction-cache, and SQLite work."
+            ),
+        },
+        **measurement,
+        "memory": {
+            "process_peak_bytes": process_peak_bytes,
+            "process_peak_limit_bytes": MAX_PROCESS_PEAK_BYTES,
+        },
+    }
+    encoded = encode_report(report)
+    if arguments.output is not None:
+        output = arguments.output.resolve()
+        output.parent.mkdir(parents=True, exist_ok=True)
+        output.write_text(encoded, encoding="utf-8")
+    sys.stdout.write(encoded)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())