523 lines
16 KiB
Python
523 lines
16 KiB
Python
"""Reproducible Milestone 0 timing, memory, rendering, and response-size baseline."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import platform
|
|
import resource
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
from typing import cast
|
|
|
|
from docforge.application import CanonicalApplicationService, GenericCanonicalApplier
|
|
from docforge.changesets import ChangesetStore
|
|
from docforge.context import compile_context
|
|
from docforge.index import ProjectIndex
|
|
from docforge.mcp_server import DocForgeService
|
|
from docforge.project import Project
|
|
from docforge.rendering import RenderService
|
|
from docforge.visualization import VisualizationIndexSnapshot
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def _parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
description="Measure DocForge against a disposable deterministic generic project."
|
|
)
|
|
parser.add_argument("--nodes", type=int, default=1000)
|
|
parser.add_argument("--samples", type=int, default=10)
|
|
parser.add_argument("--cold-samples", type=int, default=3)
|
|
parser.add_argument("--output", type=Path)
|
|
return parser
|
|
|
|
|
|
def _node_id(index: int) -> str:
|
|
return f"guide.node-{index:04d}"
|
|
|
|
|
|
def _write_project(root: Path, node_count: int) -> None:
|
|
content_root = root / "docs" / "content"
|
|
template_root = root / "docs" / "templates"
|
|
descriptor_root = root / ".docforge"
|
|
content_root.mkdir(parents=True)
|
|
template_root.mkdir(parents=True)
|
|
descriptor_root.mkdir(parents=True)
|
|
(root / "POLICY.md").write_text(
|
|
"# Synthetic benchmark policy\n\n"
|
|
"This disposable project measures repository-native DocForge operations.\n",
|
|
encoding="utf-8",
|
|
)
|
|
(template_root / "manual.html").write_text(
|
|
'<!doctype html><html lang="en"><head><meta charset="utf-8">'
|
|
"<title>{{ docforge_title }}</title></head>"
|
|
'<body data-project="{{ docforge_project_id }}" '
|
|
'data-view="{{ docforge_view_id }}"><main>{{ docforge_content }}</main></body></html>\n',
|
|
encoding="utf-8",
|
|
)
|
|
(descriptor_root / "project.toml").write_text(
|
|
f"""schema_version = 1
|
|
project_id = "synthetic-{node_count}"
|
|
title = "Synthetic {node_count} Node Baseline"
|
|
adapter = "generic"
|
|
|
|
[sources]
|
|
content_roots = ["docs/content"]
|
|
authority_files = ["POLICY.md"]
|
|
|
|
[derived]
|
|
cache_root = ".docforge/cache"
|
|
index = ".docforge/cache/index.sqlite3"
|
|
|
|
[changesets]
|
|
root = ".docforge/changesets"
|
|
|
|
[[changesets.writers]]
|
|
id = "benchmark-editor"
|
|
families = ["guide"]
|
|
operations = ["create", "update", "move", "delete"]
|
|
|
|
[render]
|
|
template_root = "docs/templates"
|
|
preview_root = ".docforge/previews"
|
|
|
|
[[render.views]]
|
|
id = "manual"
|
|
renderer = "generic_html"
|
|
template = "manual.html"
|
|
output = ".docforge/rendered/manual.html"
|
|
title = "Synthetic Manual"
|
|
families = ["guide"]
|
|
|
|
[graph]
|
|
allowed_relations = ["depends_on", "relates_to"]
|
|
|
|
[limits]
|
|
max_source_bytes = 100000
|
|
max_nodes = {max(node_count * 2, 100)}
|
|
max_query_chars = 200
|
|
max_results = 100
|
|
max_traversal_depth = 8
|
|
max_context_tokens = 32000
|
|
max_tool_output_chars = 5000000
|
|
max_changesets = 100
|
|
max_changeset_operations = 100
|
|
max_changeset_bytes = 1000000
|
|
max_render_views = 10
|
|
max_template_bytes = 1000000
|
|
max_render_bytes = 20000000
|
|
|
|
[[profiles]]
|
|
id = "active"
|
|
families = ["guide"]
|
|
statuses = ["active"]
|
|
required_nodes = ["{_node_id(node_count - 1)}"]
|
|
token_budget = 32000
|
|
dependency_depth = 8
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
for index in range(node_count):
|
|
relationships = f'depends_on = ["{_node_id(index - 1)}"]\n' if index > 0 else ""
|
|
(content_root / f"node-{index:04d}.md").write_text(
|
|
f"""+++
|
|
schema_version = 1
|
|
id = "{_node_id(index)}"
|
|
title = "Synthetic node {index:04d}"
|
|
family = "guide"
|
|
authority = "derived"
|
|
status = "active"
|
|
tags = ["synthetic", "batch-{index // 100:02d}"]
|
|
summary = "Synthetic measurement node {index:04d} for the repository-native baseline."
|
|
{relationships}+++
|
|
|
|
This deterministic benchmark content exists only in a disposable temporary directory.
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def write_synthetic_project(root: Path, node_count: int) -> None:
|
|
"""Create the shared disposable generic benchmark fixture."""
|
|
|
|
_write_project(root, node_count)
|
|
|
|
|
|
def _json_size(value: object) -> int | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, str):
|
|
return len(value.encode("utf-8"))
|
|
return len(
|
|
json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
|
)
|
|
|
|
|
|
def _measure(
|
|
operation: Callable[[], object],
|
|
*,
|
|
samples: int,
|
|
warmups: int = 1,
|
|
response_size: bool = True,
|
|
) -> tuple[dict[str, object], object]:
|
|
for _ in range(warmups):
|
|
operation()
|
|
durations: list[float] = []
|
|
last: object = None
|
|
for _ in range(samples):
|
|
started = time.perf_counter_ns()
|
|
last = operation()
|
|
durations.append((time.perf_counter_ns() - started) / 1_000_000)
|
|
ordered = sorted(durations)
|
|
p95_index = max(0, math.ceil(len(ordered) * 0.95) - 1)
|
|
result: dict[str, object] = {
|
|
"samples": samples,
|
|
"median_ms": round(statistics.median(ordered), 3),
|
|
"p95_ms": round(ordered[p95_index], 3),
|
|
"min_ms": round(ordered[0], 3),
|
|
"max_ms": round(ordered[-1], 3),
|
|
}
|
|
if response_size:
|
|
result["response_bytes"] = _json_size(last)
|
|
return result, last
|
|
|
|
|
|
def measure_operation(
|
|
operation: Callable[[], object],
|
|
*,
|
|
samples: int,
|
|
warmups: int = 1,
|
|
response_size: bool = True,
|
|
) -> tuple[dict[str, object], object]:
|
|
"""Measure one operation using the shared baseline method."""
|
|
|
|
return _measure(
|
|
operation,
|
|
samples=samples,
|
|
warmups=warmups,
|
|
response_size=response_size,
|
|
)
|
|
|
|
|
|
def synthetic_node_id(index: int) -> str:
|
|
"""Return one deterministic node identifier from the shared fixture."""
|
|
|
|
return _node_id(index)
|
|
|
|
|
|
def _run(command: list[str]) -> str:
|
|
return subprocess.run(
|
|
command,
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout
|
|
|
|
|
|
def _git(command: list[str]) -> str:
|
|
return subprocess.run(
|
|
["git", *command],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
|
|
|
|
def _benchmark(root: Path, node_count: int, samples: int, cold_samples: int) -> dict[str, object]:
|
|
target = _node_id(node_count - 1)
|
|
project = Project.open(root)
|
|
index = ProjectIndex(project)
|
|
operations: dict[str, object] = {}
|
|
|
|
operations["project_open"], _ = _measure(
|
|
lambda: Project.open(root),
|
|
samples=samples,
|
|
response_size=False,
|
|
)
|
|
operations["project_load"], _ = _measure(
|
|
project.load,
|
|
samples=samples,
|
|
response_size=False,
|
|
)
|
|
operations["full_index_build"], _ = _measure(
|
|
index.build,
|
|
samples=max(1, cold_samples),
|
|
warmups=0,
|
|
)
|
|
|
|
def cold_synchronize() -> dict[str, object]:
|
|
index.path.unlink(missing_ok=True)
|
|
index.attestation_path.unlink(missing_ok=True)
|
|
return ProjectIndex(project).synchronize()
|
|
|
|
operations["cold_synchronize"], _ = _measure(
|
|
cold_synchronize,
|
|
samples=cold_samples,
|
|
warmups=0,
|
|
)
|
|
index = ProjectIndex(project)
|
|
index.synchronize()
|
|
operations["full_index_check"], _ = _measure(index.check, samples=samples)
|
|
operations["warm_no_change_synchronize"], _ = _measure(
|
|
index.synchronize,
|
|
samples=samples,
|
|
)
|
|
operations["exact_node"], _ = _measure(
|
|
lambda: index.get_node(target),
|
|
samples=samples,
|
|
)
|
|
operations["search_limit_20"], _ = _measure(
|
|
lambda: index.search("Synthetic measurement", limit=20),
|
|
samples=samples,
|
|
)
|
|
operations["dependencies_depth_8"], _ = _measure(
|
|
lambda: index.dependencies(target, depth=8),
|
|
samples=samples,
|
|
)
|
|
operations["impact_depth_8"], _ = _measure(
|
|
lambda: index.impact(_node_id(0), depth=8),
|
|
samples=samples,
|
|
)
|
|
operations["context_32k"], context = _measure(
|
|
lambda: compile_context(index, "active", 32000),
|
|
samples=samples,
|
|
)
|
|
|
|
renderer = RenderService(project)
|
|
operations["manual_render"], _ = _measure(
|
|
lambda: renderer.render("manual"),
|
|
samples=max(1, cold_samples),
|
|
warmups=0,
|
|
)
|
|
operations["manual_render_status"], _ = _measure(
|
|
lambda: renderer.status("manual"),
|
|
samples=samples,
|
|
)
|
|
|
|
operations["viewer_snapshot_pin"], _ = _measure(
|
|
lambda: VisualizationIndexSnapshot(index, index.check()),
|
|
samples=max(1, cold_samples),
|
|
response_size=False,
|
|
)
|
|
snapshot = VisualizationIndexSnapshot(index, index.check())
|
|
operations["viewer_overview"], _ = _measure(snapshot.overview, samples=samples)
|
|
operations["viewer_search_limit_20"], _ = _measure(
|
|
lambda: snapshot.search(
|
|
query="Synthetic measurement",
|
|
family=None,
|
|
kind=None,
|
|
language=None,
|
|
capability=None,
|
|
limit=20,
|
|
),
|
|
samples=samples,
|
|
)
|
|
operations["viewer_neighborhood_depth_8"], _ = _measure(
|
|
lambda: snapshot.node(target, depth=8, limit=100),
|
|
samples=samples,
|
|
)
|
|
operations["viewer_web_depth_8"], _ = _measure(
|
|
lambda: snapshot.web(target, depth=8, limit=100),
|
|
samples=samples,
|
|
)
|
|
|
|
service = DocForgeService(project)
|
|
operations["mcp_bootstrap"], _ = _measure(service.bootstrap, samples=samples)
|
|
operations["mcp_exact_node"], _ = _measure(
|
|
lambda: service.invoke(lambda: service.index.get_node(target)),
|
|
samples=samples,
|
|
)
|
|
operations["mcp_search_limit_20"], _ = _measure(
|
|
lambda: service.invoke(lambda: service.index.search("Synthetic measurement", limit=20)),
|
|
samples=samples,
|
|
)
|
|
operations["mcp_context_32k"], _ = _measure(
|
|
lambda: service.invoke(lambda: compile_context(service.index, "active", 32000)),
|
|
samples=samples,
|
|
)
|
|
operations["mcp_render_status"], _ = _measure(
|
|
lambda: service.render_status("manual"),
|
|
samples=samples,
|
|
)
|
|
|
|
store = ChangesetStore(project, "benchmark-editor")
|
|
registration_counter = 0
|
|
|
|
def register() -> dict[str, object]:
|
|
nonlocal registration_counter
|
|
registration_counter += 1
|
|
proposal_node = _node_id(node_count - registration_counter)
|
|
return store.register(
|
|
f"benchmark-{registration_counter:03d}",
|
|
[
|
|
{
|
|
"operation": "update",
|
|
"node_id": proposal_node,
|
|
"metadata": {
|
|
"summary": (
|
|
"Synthetic measurement node updated only inside a benchmark proposal."
|
|
)
|
|
},
|
|
"rationale": "Measure atomic registration without changing canonical sources.",
|
|
}
|
|
],
|
|
)
|
|
|
|
registration_samples = min(samples, 10, node_count)
|
|
operations["changeset_register"], registered = _measure(
|
|
register,
|
|
samples=registration_samples,
|
|
warmups=0,
|
|
)
|
|
changeset_id = f"benchmark-{registration_counter:03d}"
|
|
operations["changeset_validate"], _ = _measure(
|
|
lambda: store.validate(changeset_id),
|
|
samples=samples,
|
|
)
|
|
operations["changeset_diff"], _ = _measure(
|
|
lambda: store.diff(changeset_id),
|
|
samples=samples,
|
|
)
|
|
if not isinstance(registered, dict):
|
|
raise RuntimeError("Changeset registration returned an invalid result")
|
|
registered_result = cast(dict[str, object], registered)
|
|
application = CanonicalApplicationService(
|
|
project,
|
|
applier_id="benchmark-editor",
|
|
applier=GenericCanonicalApplier(project),
|
|
)
|
|
operations["exact_hash_apply_and_refresh"], _ = _measure(
|
|
lambda: application.apply(changeset_id, str(registered_result["changeset_hash"])),
|
|
samples=1,
|
|
warmups=0,
|
|
)
|
|
|
|
operations["cli_info_startup"], _ = _measure(
|
|
lambda: _run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"docforge.cli",
|
|
"--project-root",
|
|
str(root),
|
|
"info",
|
|
]
|
|
),
|
|
samples=max(1, cold_samples),
|
|
warmups=0,
|
|
)
|
|
operations["cli_exact_startup"], _ = _measure(
|
|
lambda: _run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"docforge.cli",
|
|
"--project-root",
|
|
str(root),
|
|
"show",
|
|
target,
|
|
]
|
|
),
|
|
samples=max(1, cold_samples),
|
|
warmups=0,
|
|
)
|
|
operations["mcp_import_and_help"], _ = _measure(
|
|
lambda: _run([sys.executable, "-m", "docforge.mcp_server", "--help"]),
|
|
samples=max(1, cold_samples),
|
|
warmups=0,
|
|
)
|
|
|
|
manual_path = root / ".docforge" / "rendered" / "manual.html"
|
|
static_asset_bytes = sum(
|
|
path.stat().st_size
|
|
for path in (
|
|
ROOT / "src" / "docforge" / "assets" / "graph.html",
|
|
ROOT / "src" / "docforge" / "assets" / "graph.css",
|
|
ROOT / "src" / "docforge" / "assets" / "graph.js",
|
|
)
|
|
)
|
|
return {
|
|
"fixture": {
|
|
"kind": "synthetic_generic",
|
|
"node_count": node_count,
|
|
"edge_count": node_count - 1,
|
|
"source_file_count": node_count,
|
|
"context_budget_tokens": 32000,
|
|
"traversal_depth": 8,
|
|
},
|
|
"operations": operations,
|
|
"sizes": {
|
|
"context_compact_bytes": _json_size(context),
|
|
"manual_artifact_bytes": manual_path.stat().st_size,
|
|
"static_viewer_assets_bytes": static_asset_bytes,
|
|
},
|
|
"process_peak_rss_kib": int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
arguments = _parser().parse_args()
|
|
if arguments.nodes < 2:
|
|
raise SystemExit("--nodes must be at least 2")
|
|
if arguments.samples < 1 or arguments.cold_samples < 1:
|
|
raise SystemExit("sample counts must be positive")
|
|
with tempfile.TemporaryDirectory(prefix="docforge-milestone0-") as directory:
|
|
root = Path(directory).resolve()
|
|
_write_project(root, arguments.nodes)
|
|
measurement = _benchmark(
|
|
root,
|
|
arguments.nodes,
|
|
arguments.samples,
|
|
arguments.cold_samples,
|
|
)
|
|
status = _git(["status", "--porcelain"])
|
|
result: dict[str, object] = {
|
|
"schema_version": 1,
|
|
"benchmark": "docforge2_milestone0",
|
|
"source": {
|
|
"revision": _git(["rev-parse", "HEAD"]),
|
|
"dirty": bool(status),
|
|
},
|
|
"environment": {
|
|
"platform": platform.platform(),
|
|
"machine": platform.machine(),
|
|
"python": platform.python_version(),
|
|
"implementation": platform.python_implementation(),
|
|
},
|
|
"method": {
|
|
"clock": "time.perf_counter_ns",
|
|
"memory": "resource.getrusage(RUSAGE_SELF).ru_maxrss",
|
|
"response_size": "UTF-8 bytes of compact sorted JSON",
|
|
"samples": arguments.samples,
|
|
"cold_samples": arguments.cold_samples,
|
|
},
|
|
**measurement,
|
|
"known_gaps": [
|
|
"Generic warm reads still parse canonical source files.",
|
|
"Compiler stages are not separately instrumented.",
|
|
"Scaled incremental extraction is not measured by this generic fixture.",
|
|
"Manual planning is not separated from rendering.",
|
|
"Portable graph planning and rendering do not exist in Milestone 0.",
|
|
"Per-operation peak RSS requires an external process harness.",
|
|
],
|
|
}
|
|
encoded = json.dumps(result, sort_keys=True, indent=2) + "\n"
|
|
if arguments.output is not None:
|
|
output = arguments.output.resolve()
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(encoded, encoding="utf-8")
|
|
sys.stdout.write(encoded)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|