"""Reproducible Milestone 0 timing, memory, rendering, and response-size baseline.""" from __future__ import annotations import argparse import json import math import platform import resource import statistics import subprocess import sys import tempfile import time from collections.abc import Callable from pathlib import Path from typing import cast from docforge.application import CanonicalApplicationService, GenericCanonicalApplier from docforge.changesets import ChangesetStore from docforge.context import compile_context from docforge.index import ProjectIndex from docforge.mcp_server import DocForgeService from docforge.project import Project from docforge.rendering import RenderService from docforge.visualization import VisualizationIndexSnapshot ROOT = Path(__file__).resolve().parents[1] def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Measure DocForge against a disposable deterministic generic project." ) parser.add_argument("--nodes", type=int, default=1000) parser.add_argument("--samples", type=int, default=10) parser.add_argument("--cold-samples", type=int, default=3) parser.add_argument("--output", type=Path) return parser def _node_id(index: int) -> str: return f"guide.node-{index:04d}" def _write_project(root: Path, node_count: int) -> None: content_root = root / "docs" / "content" template_root = root / "docs" / "templates" descriptor_root = root / ".docforge" content_root.mkdir(parents=True) template_root.mkdir(parents=True) descriptor_root.mkdir(parents=True) (root / "POLICY.md").write_text( "# Synthetic benchmark policy\n\n" "This disposable project measures repository-native DocForge operations.\n", encoding="utf-8", ) (template_root / "manual.html").write_text( '' "{{ docforge_title }}" '

\n', encoding="utf-8", ) (descriptor_root / "project.toml").write_text( f"""schema_version = 1 project_id = "synthetic-{node_count}" title = "Synthetic {node_count} Node Baseline" adapter = "generic" [sources] content_roots = ["docs/content"] authority_files = ["POLICY.md"] [derived] cache_root = ".docforge/cache" index = ".docforge/cache/index.sqlite3" [changesets] root = ".docforge/changesets" [[changesets.writers]] id = "benchmark-editor" families = ["guide"] operations = ["create", "update", "move", "delete"] [render] template_root = "docs/templates" preview_root = ".docforge/previews" [[render.views]] id = "manual" renderer = "generic_html" template = "manual.html" output = ".docforge/rendered/manual.html" title = "Synthetic Manual" families = ["guide"] [graph] allowed_relations = ["depends_on", "relates_to"] [limits] max_source_bytes = 100000 max_nodes = {max(node_count * 2, 100)} max_query_chars = 200 max_results = 100 max_traversal_depth = 8 max_context_tokens = 32000 max_tool_output_chars = 5000000 max_changesets = 100 max_changeset_operations = 100 max_changeset_bytes = 1000000 max_render_views = 10 max_template_bytes = 1000000 max_render_bytes = 20000000 [[profiles]] id = "active" families = ["guide"] statuses = ["active"] required_nodes = ["{_node_id(node_count - 1)}"] token_budget = 32000 dependency_depth = 8 """, encoding="utf-8", ) for index in range(node_count): relationships = f'depends_on = ["{_node_id(index - 1)}"]\n' if index > 0 else "" (content_root / f"node-{index:04d}.md").write_text( f"""+++ schema_version = 1 id = "{_node_id(index)}" title = "Synthetic node {index:04d}" family = "guide" authority = "derived" status = "active" tags = ["synthetic", "batch-{index // 100:02d}"] summary = "Synthetic measurement node {index:04d} for the repository-native baseline." {relationships}+++ This deterministic benchmark content exists only in a disposable temporary directory. """, encoding="utf-8", ) def write_synthetic_project(root: Path, node_count: int) -> None: """Create the shared disposable generic benchmark fixture.""" _write_project(root, node_count) def _json_size(value: object) -> int | None: if value is None: return None if isinstance(value, str): return len(value.encode("utf-8")) return len( json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") ) def _measure( operation: Callable[[], object], *, samples: int, warmups: int = 1, response_size: bool = True, ) -> tuple[dict[str, object], object]: for _ in range(warmups): operation() durations: list[float] = [] last: object = None for _ in range(samples): started = time.perf_counter_ns() last = operation() durations.append((time.perf_counter_ns() - started) / 1_000_000) ordered = sorted(durations) p95_index = max(0, math.ceil(len(ordered) * 0.95) - 1) result: dict[str, object] = { "samples": samples, "median_ms": round(statistics.median(ordered), 3), "p95_ms": round(ordered[p95_index], 3), "min_ms": round(ordered[0], 3), "max_ms": round(ordered[-1], 3), } if response_size: result["response_bytes"] = _json_size(last) return result, last def measure_operation( operation: Callable[[], object], *, samples: int, warmups: int = 1, response_size: bool = True, ) -> tuple[dict[str, object], object]: """Measure one operation using the shared baseline method.""" return _measure( operation, samples=samples, warmups=warmups, response_size=response_size, ) def synthetic_node_id(index: int) -> str: """Return one deterministic node identifier from the shared fixture.""" return _node_id(index) def _run(command: list[str]) -> str: return subprocess.run( command, cwd=ROOT, check=True, capture_output=True, text=True, ).stdout def _git(command: list[str]) -> str: return subprocess.run( ["git", *command], cwd=ROOT, check=True, capture_output=True, text=True, ).stdout.strip() def _benchmark(root: Path, node_count: int, samples: int, cold_samples: int) -> dict[str, object]: target = _node_id(node_count - 1) project = Project.open(root) index = ProjectIndex(project) operations: dict[str, object] = {} operations["project_open"], _ = _measure( lambda: Project.open(root), samples=samples, response_size=False, ) operations["project_load"], _ = _measure( project.load, samples=samples, response_size=False, ) operations["full_index_build"], _ = _measure( index.build, samples=max(1, cold_samples), warmups=0, ) def cold_synchronize() -> dict[str, object]: index.path.unlink(missing_ok=True) index.attestation_path.unlink(missing_ok=True) return ProjectIndex(project).synchronize() operations["cold_synchronize"], _ = _measure( cold_synchronize, samples=cold_samples, warmups=0, ) index = ProjectIndex(project) index.synchronize() operations["full_index_check"], _ = _measure(index.check, samples=samples) operations["warm_no_change_synchronize"], _ = _measure( index.synchronize, samples=samples, ) operations["exact_node"], _ = _measure( lambda: index.get_node(target), samples=samples, ) operations["search_limit_20"], _ = _measure( lambda: index.search("Synthetic measurement", limit=20), samples=samples, ) operations["dependencies_depth_8"], _ = _measure( lambda: index.dependencies(target, depth=8), samples=samples, ) operations["impact_depth_8"], _ = _measure( lambda: index.impact(_node_id(0), depth=8), samples=samples, ) operations["context_32k"], context = _measure( lambda: compile_context(index, "active", 32000), samples=samples, ) renderer = RenderService(project) operations["manual_render"], _ = _measure( lambda: renderer.render("manual"), samples=max(1, cold_samples), warmups=0, ) operations["manual_render_status"], _ = _measure( lambda: renderer.status("manual"), samples=samples, ) operations["viewer_snapshot_pin"], _ = _measure( lambda: VisualizationIndexSnapshot(index, index.check()), samples=max(1, cold_samples), response_size=False, ) snapshot = VisualizationIndexSnapshot(index, index.check()) operations["viewer_overview"], _ = _measure(snapshot.overview, samples=samples) operations["viewer_search_limit_20"], _ = _measure( lambda: snapshot.search( query="Synthetic measurement", family=None, kind=None, language=None, capability=None, limit=20, ), samples=samples, ) operations["viewer_neighborhood_depth_8"], _ = _measure( lambda: snapshot.node(target, depth=8, limit=100), samples=samples, ) operations["viewer_web_depth_8"], _ = _measure( lambda: snapshot.web(target, depth=8, limit=100), samples=samples, ) service = DocForgeService(project) operations["mcp_bootstrap"], _ = _measure(service.bootstrap, samples=samples) operations["mcp_exact_node"], _ = _measure( lambda: service.invoke(lambda: service.index.get_node(target)), samples=samples, ) operations["mcp_search_limit_20"], _ = _measure( lambda: service.invoke(lambda: service.index.search("Synthetic measurement", limit=20)), samples=samples, ) operations["mcp_context_32k"], _ = _measure( lambda: service.invoke(lambda: compile_context(service.index, "active", 32000)), samples=samples, ) operations["mcp_render_status"], _ = _measure( lambda: service.render_status("manual"), samples=samples, ) store = ChangesetStore(project, "benchmark-editor") registration_counter = 0 def register() -> dict[str, object]: nonlocal registration_counter registration_counter += 1 proposal_node = _node_id(node_count - registration_counter) return store.register( f"benchmark-{registration_counter:03d}", [ { "operation": "update", "node_id": proposal_node, "metadata": { "summary": ( "Synthetic measurement node updated only inside a benchmark proposal." ) }, "rationale": "Measure atomic registration without changing canonical sources.", } ], ) registration_samples = min(samples, 10, node_count) operations["changeset_register"], registered = _measure( register, samples=registration_samples, warmups=0, ) changeset_id = f"benchmark-{registration_counter:03d}" operations["changeset_validate"], _ = _measure( lambda: store.validate(changeset_id), samples=samples, ) operations["changeset_diff"], _ = _measure( lambda: store.diff(changeset_id), samples=samples, ) if not isinstance(registered, dict): raise RuntimeError("Changeset registration returned an invalid result") registered_result = cast(dict[str, object], registered) application = CanonicalApplicationService( project, applier_id="benchmark-editor", applier=GenericCanonicalApplier(project), ) operations["exact_hash_apply_and_refresh"], _ = _measure( lambda: application.apply(changeset_id, str(registered_result["changeset_hash"])), samples=1, warmups=0, ) operations["cli_info_startup"], _ = _measure( lambda: _run( [ sys.executable, "-m", "docforge.cli", "--project-root", str(root), "info", ] ), samples=max(1, cold_samples), warmups=0, ) operations["cli_exact_startup"], _ = _measure( lambda: _run( [ sys.executable, "-m", "docforge.cli", "--project-root", str(root), "show", target, ] ), samples=max(1, cold_samples), warmups=0, ) operations["mcp_import_and_help"], _ = _measure( lambda: _run([sys.executable, "-m", "docforge.mcp_server", "--help"]), samples=max(1, cold_samples), warmups=0, ) manual_path = root / ".docforge" / "rendered" / "manual.html" static_asset_bytes = sum( path.stat().st_size for path in ( ROOT / "src" / "docforge" / "assets" / "graph.html", ROOT / "src" / "docforge" / "assets" / "graph.css", ROOT / "src" / "docforge" / "assets" / "graph.js", ) ) return { "fixture": { "kind": "synthetic_generic", "node_count": node_count, "edge_count": node_count - 1, "source_file_count": node_count, "context_budget_tokens": 32000, "traversal_depth": 8, }, "operations": operations, "sizes": { "context_compact_bytes": _json_size(context), "manual_artifact_bytes": manual_path.stat().st_size, "static_viewer_assets_bytes": static_asset_bytes, }, "process_peak_rss_kib": int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss), } def main() -> int: arguments = _parser().parse_args() if arguments.nodes < 2: raise SystemExit("--nodes must be at least 2") if arguments.samples < 1 or arguments.cold_samples < 1: raise SystemExit("sample counts must be positive") with tempfile.TemporaryDirectory(prefix="docforge-milestone0-") as directory: root = Path(directory).resolve() _write_project(root, arguments.nodes) measurement = _benchmark( root, arguments.nodes, arguments.samples, arguments.cold_samples, ) status = _git(["status", "--porcelain"]) result: dict[str, object] = { "schema_version": 1, "benchmark": "docforge2_milestone0", "source": { "revision": _git(["rev-parse", "HEAD"]), "dirty": bool(status), }, "environment": { "platform": platform.platform(), "machine": platform.machine(), "python": platform.python_version(), "implementation": platform.python_implementation(), }, "method": { "clock": "time.perf_counter_ns", "memory": "resource.getrusage(RUSAGE_SELF).ru_maxrss", "response_size": "UTF-8 bytes of compact sorted JSON", "samples": arguments.samples, "cold_samples": arguments.cold_samples, }, **measurement, "known_gaps": [ "Generic warm reads still parse canonical source files.", "Compiler stages are not separately instrumented.", "Scaled incremental extraction is not measured by this generic fixture.", "Manual planning is not separated from rendering.", "Portable graph planning and rendering do not exist in Milestone 0.", "Per-operation peak RSS requires an external process harness.", ], } encoded = json.dumps(result, sort_keys=True, indent=2) + "\n" if arguments.output is not None: output = arguments.output.resolve() output.parent.mkdir(parents=True, exist_ok=True) output.write_text(encoded, encoding="utf-8") sys.stdout.write(encoded) return 0 if __name__ == "__main__": raise SystemExit(main())