Establish Milestone 0 compatibility and quality gates
This commit is contained in:
parent
15a913003c
commit
8ebb78a71d
15 changed files with 1114 additions and 27 deletions
493
tools/milestone0_baseline.py
Normal file
493
tools/milestone0_baseline.py
Normal file
|
|
@ -0,0 +1,493 @@
|
|||
"""Reproducible Milestone 0 timing, memory, rendering, and response-size baseline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import platform
|
||||
import resource
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
from docforge.application import CanonicalApplicationService, GenericCanonicalApplier
|
||||
from docforge.changesets import ChangesetStore
|
||||
from docforge.context import compile_context
|
||||
from docforge.index import ProjectIndex
|
||||
from docforge.mcp_server import DocForgeService
|
||||
from docforge.project import Project
|
||||
from docforge.rendering import RenderService
|
||||
from docforge.visualization import VisualizationIndexSnapshot
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Measure DocForge against a disposable deterministic generic project."
|
||||
)
|
||||
parser.add_argument("--nodes", type=int, default=1000)
|
||||
parser.add_argument("--samples", type=int, default=10)
|
||||
parser.add_argument("--cold-samples", type=int, default=3)
|
||||
parser.add_argument("--output", type=Path)
|
||||
return parser
|
||||
|
||||
|
||||
def _node_id(index: int) -> str:
|
||||
return f"guide.node-{index:04d}"
|
||||
|
||||
|
||||
def _write_project(root: Path, node_count: int) -> None:
|
||||
content_root = root / "docs" / "content"
|
||||
template_root = root / "docs" / "templates"
|
||||
descriptor_root = root / ".docforge"
|
||||
content_root.mkdir(parents=True)
|
||||
template_root.mkdir(parents=True)
|
||||
descriptor_root.mkdir(parents=True)
|
||||
(root / "POLICY.md").write_text(
|
||||
"# Synthetic benchmark policy\n\n"
|
||||
"This disposable project measures repository-native DocForge operations.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(template_root / "manual.html").write_text(
|
||||
'<!doctype html><html lang="en"><head><meta charset="utf-8">'
|
||||
"<title>{{ docforge_title }}</title></head>"
|
||||
'<body data-project="{{ docforge_project_id }}" '
|
||||
'data-view="{{ docforge_view_id }}"><main>{{ docforge_content }}</main></body></html>\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
(descriptor_root / "project.toml").write_text(
|
||||
f"""schema_version = 1
|
||||
project_id = "synthetic-{node_count}"
|
||||
title = "Synthetic {node_count} Node Baseline"
|
||||
adapter = "generic"
|
||||
|
||||
[sources]
|
||||
content_roots = ["docs/content"]
|
||||
authority_files = ["POLICY.md"]
|
||||
|
||||
[derived]
|
||||
cache_root = ".docforge/cache"
|
||||
index = ".docforge/cache/index.sqlite3"
|
||||
|
||||
[changesets]
|
||||
root = ".docforge/changesets"
|
||||
|
||||
[[changesets.writers]]
|
||||
id = "benchmark-editor"
|
||||
families = ["guide"]
|
||||
operations = ["create", "update", "move", "delete"]
|
||||
|
||||
[render]
|
||||
template_root = "docs/templates"
|
||||
preview_root = ".docforge/previews"
|
||||
|
||||
[[render.views]]
|
||||
id = "manual"
|
||||
renderer = "generic_html"
|
||||
template = "manual.html"
|
||||
output = ".docforge/rendered/manual.html"
|
||||
title = "Synthetic Manual"
|
||||
families = ["guide"]
|
||||
|
||||
[graph]
|
||||
allowed_relations = ["depends_on", "relates_to"]
|
||||
|
||||
[limits]
|
||||
max_source_bytes = 100000
|
||||
max_nodes = {max(node_count * 2, 100)}
|
||||
max_query_chars = 200
|
||||
max_results = 100
|
||||
max_traversal_depth = 8
|
||||
max_context_tokens = 32000
|
||||
max_tool_output_chars = 5000000
|
||||
max_changesets = 100
|
||||
max_changeset_operations = 100
|
||||
max_changeset_bytes = 1000000
|
||||
max_render_views = 10
|
||||
max_template_bytes = 1000000
|
||||
max_render_bytes = 20000000
|
||||
|
||||
[[profiles]]
|
||||
id = "active"
|
||||
families = ["guide"]
|
||||
statuses = ["active"]
|
||||
required_nodes = ["{_node_id(node_count - 1)}"]
|
||||
token_budget = 32000
|
||||
dependency_depth = 8
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
for index in range(node_count):
|
||||
relationships = f'depends_on = ["{_node_id(index - 1)}"]\n' if index > 0 else ""
|
||||
(content_root / f"node-{index:04d}.md").write_text(
|
||||
f"""+++
|
||||
schema_version = 1
|
||||
id = "{_node_id(index)}"
|
||||
title = "Synthetic node {index:04d}"
|
||||
family = "guide"
|
||||
authority = "derived"
|
||||
status = "active"
|
||||
tags = ["synthetic", "batch-{index // 100:02d}"]
|
||||
summary = "Synthetic measurement node {index:04d} for the repository-native baseline."
|
||||
{relationships}+++
|
||||
|
||||
This deterministic benchmark content exists only in a disposable temporary directory.
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _json_size(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
return len(value.encode("utf-8"))
|
||||
return len(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
||||
)
|
||||
|
||||
|
||||
def _measure(
|
||||
operation: Callable[[], object],
|
||||
*,
|
||||
samples: int,
|
||||
warmups: int = 1,
|
||||
response_size: bool = True,
|
||||
) -> tuple[dict[str, object], object]:
|
||||
for _ in range(warmups):
|
||||
operation()
|
||||
durations: list[float] = []
|
||||
last: object = None
|
||||
for _ in range(samples):
|
||||
started = time.perf_counter_ns()
|
||||
last = operation()
|
||||
durations.append((time.perf_counter_ns() - started) / 1_000_000)
|
||||
ordered = sorted(durations)
|
||||
p95_index = max(0, math.ceil(len(ordered) * 0.95) - 1)
|
||||
result: dict[str, object] = {
|
||||
"samples": samples,
|
||||
"median_ms": round(statistics.median(ordered), 3),
|
||||
"p95_ms": round(ordered[p95_index], 3),
|
||||
"min_ms": round(ordered[0], 3),
|
||||
"max_ms": round(ordered[-1], 3),
|
||||
}
|
||||
if response_size:
|
||||
result["response_bytes"] = _json_size(last)
|
||||
return result, last
|
||||
|
||||
|
||||
def _run(command: list[str]) -> str:
|
||||
return subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
|
||||
|
||||
def _git(command: list[str]) -> str:
|
||||
return subprocess.run(
|
||||
["git", *command],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
|
||||
|
||||
def _benchmark(root: Path, node_count: int, samples: int, cold_samples: int) -> dict[str, object]:
|
||||
target = _node_id(node_count - 1)
|
||||
project = Project.open(root)
|
||||
index = ProjectIndex(project)
|
||||
operations: dict[str, object] = {}
|
||||
|
||||
operations["project_open"], _ = _measure(
|
||||
lambda: Project.open(root),
|
||||
samples=samples,
|
||||
response_size=False,
|
||||
)
|
||||
operations["project_load"], _ = _measure(
|
||||
project.load,
|
||||
samples=samples,
|
||||
response_size=False,
|
||||
)
|
||||
operations["full_index_build"], _ = _measure(
|
||||
index.build,
|
||||
samples=max(1, cold_samples),
|
||||
warmups=0,
|
||||
)
|
||||
|
||||
def cold_synchronize() -> dict[str, object]:
|
||||
index.path.unlink(missing_ok=True)
|
||||
index.attestation_path.unlink(missing_ok=True)
|
||||
return ProjectIndex(project).synchronize()
|
||||
|
||||
operations["cold_synchronize"], _ = _measure(
|
||||
cold_synchronize,
|
||||
samples=cold_samples,
|
||||
warmups=0,
|
||||
)
|
||||
index = ProjectIndex(project)
|
||||
index.synchronize()
|
||||
operations["full_index_check"], _ = _measure(index.check, samples=samples)
|
||||
operations["warm_no_change_synchronize"], _ = _measure(
|
||||
index.synchronize,
|
||||
samples=samples,
|
||||
)
|
||||
operations["exact_node"], _ = _measure(
|
||||
lambda: index.get_node(target),
|
||||
samples=samples,
|
||||
)
|
||||
operations["search_limit_20"], _ = _measure(
|
||||
lambda: index.search("Synthetic measurement", limit=20),
|
||||
samples=samples,
|
||||
)
|
||||
operations["dependencies_depth_8"], _ = _measure(
|
||||
lambda: index.dependencies(target, depth=8),
|
||||
samples=samples,
|
||||
)
|
||||
operations["impact_depth_8"], _ = _measure(
|
||||
lambda: index.impact(_node_id(0), depth=8),
|
||||
samples=samples,
|
||||
)
|
||||
operations["context_32k"], context = _measure(
|
||||
lambda: compile_context(index, "active", 32000),
|
||||
samples=samples,
|
||||
)
|
||||
|
||||
renderer = RenderService(project)
|
||||
operations["manual_render"], _ = _measure(
|
||||
lambda: renderer.render("manual"),
|
||||
samples=max(1, cold_samples),
|
||||
warmups=0,
|
||||
)
|
||||
operations["manual_render_status"], _ = _measure(
|
||||
lambda: renderer.status("manual"),
|
||||
samples=samples,
|
||||
)
|
||||
|
||||
operations["viewer_snapshot_pin"], _ = _measure(
|
||||
lambda: VisualizationIndexSnapshot(index, index.check()),
|
||||
samples=max(1, cold_samples),
|
||||
response_size=False,
|
||||
)
|
||||
snapshot = VisualizationIndexSnapshot(index, index.check())
|
||||
operations["viewer_overview"], _ = _measure(snapshot.overview, samples=samples)
|
||||
operations["viewer_search_limit_20"], _ = _measure(
|
||||
lambda: snapshot.search(
|
||||
query="Synthetic measurement",
|
||||
family=None,
|
||||
kind=None,
|
||||
language=None,
|
||||
capability=None,
|
||||
limit=20,
|
||||
),
|
||||
samples=samples,
|
||||
)
|
||||
operations["viewer_neighborhood_depth_8"], _ = _measure(
|
||||
lambda: snapshot.node(target, depth=8, limit=100),
|
||||
samples=samples,
|
||||
)
|
||||
operations["viewer_web_depth_8"], _ = _measure(
|
||||
lambda: snapshot.web(target, depth=8, limit=100),
|
||||
samples=samples,
|
||||
)
|
||||
|
||||
service = DocForgeService(project)
|
||||
operations["mcp_bootstrap"], _ = _measure(service.bootstrap, samples=samples)
|
||||
operations["mcp_exact_node"], _ = _measure(
|
||||
lambda: service.invoke(lambda: service.index.get_node(target)),
|
||||
samples=samples,
|
||||
)
|
||||
operations["mcp_search_limit_20"], _ = _measure(
|
||||
lambda: service.invoke(lambda: service.index.search("Synthetic measurement", limit=20)),
|
||||
samples=samples,
|
||||
)
|
||||
operations["mcp_context_32k"], _ = _measure(
|
||||
lambda: service.invoke(lambda: compile_context(service.index, "active", 32000)),
|
||||
samples=samples,
|
||||
)
|
||||
operations["mcp_render_status"], _ = _measure(
|
||||
lambda: service.render_status("manual"),
|
||||
samples=samples,
|
||||
)
|
||||
|
||||
store = ChangesetStore(project, "benchmark-editor")
|
||||
registration_counter = 0
|
||||
|
||||
def register() -> dict[str, object]:
|
||||
nonlocal registration_counter
|
||||
registration_counter += 1
|
||||
return store.register(
|
||||
f"benchmark-{registration_counter:03d}",
|
||||
[
|
||||
{
|
||||
"operation": "update",
|
||||
"node_id": target,
|
||||
"metadata": {
|
||||
"summary": (
|
||||
"Synthetic measurement node updated only inside a benchmark proposal."
|
||||
)
|
||||
},
|
||||
"rationale": "Measure atomic registration without changing canonical sources.",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
registration_samples = min(samples, 10)
|
||||
operations["changeset_register"], registered = _measure(
|
||||
register,
|
||||
samples=registration_samples,
|
||||
warmups=0,
|
||||
)
|
||||
changeset_id = f"benchmark-{registration_counter:03d}"
|
||||
operations["changeset_validate"], _ = _measure(
|
||||
lambda: store.validate(changeset_id),
|
||||
samples=samples,
|
||||
)
|
||||
operations["changeset_diff"], _ = _measure(
|
||||
lambda: store.diff(changeset_id),
|
||||
samples=samples,
|
||||
)
|
||||
if not isinstance(registered, dict):
|
||||
raise RuntimeError("Changeset registration returned an invalid result")
|
||||
registered_result = cast(dict[str, object], registered)
|
||||
application = CanonicalApplicationService(
|
||||
project,
|
||||
applier_id="benchmark-editor",
|
||||
applier=GenericCanonicalApplier(project),
|
||||
)
|
||||
operations["exact_hash_apply_and_refresh"], _ = _measure(
|
||||
lambda: application.apply(changeset_id, str(registered_result["changeset_hash"])),
|
||||
samples=1,
|
||||
warmups=0,
|
||||
)
|
||||
|
||||
operations["cli_info_startup"], _ = _measure(
|
||||
lambda: _run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"docforge.cli",
|
||||
"--project-root",
|
||||
str(root),
|
||||
"info",
|
||||
]
|
||||
),
|
||||
samples=max(1, cold_samples),
|
||||
warmups=0,
|
||||
)
|
||||
operations["cli_exact_startup"], _ = _measure(
|
||||
lambda: _run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"docforge.cli",
|
||||
"--project-root",
|
||||
str(root),
|
||||
"show",
|
||||
target,
|
||||
]
|
||||
),
|
||||
samples=max(1, cold_samples),
|
||||
warmups=0,
|
||||
)
|
||||
operations["mcp_import_and_help"], _ = _measure(
|
||||
lambda: _run([sys.executable, "-m", "docforge.mcp_server", "--help"]),
|
||||
samples=max(1, cold_samples),
|
||||
warmups=0,
|
||||
)
|
||||
|
||||
manual_path = root / ".docforge" / "rendered" / "manual.html"
|
||||
static_asset_bytes = sum(
|
||||
path.stat().st_size
|
||||
for path in (
|
||||
ROOT / "src" / "docforge" / "assets" / "graph.html",
|
||||
ROOT / "src" / "docforge" / "assets" / "graph.css",
|
||||
ROOT / "src" / "docforge" / "assets" / "graph.js",
|
||||
)
|
||||
)
|
||||
return {
|
||||
"fixture": {
|
||||
"kind": "synthetic_generic",
|
||||
"node_count": node_count,
|
||||
"edge_count": node_count - 1,
|
||||
"source_file_count": node_count,
|
||||
"context_budget_tokens": 32000,
|
||||
"traversal_depth": 8,
|
||||
},
|
||||
"operations": operations,
|
||||
"sizes": {
|
||||
"context_compact_bytes": _json_size(context),
|
||||
"manual_artifact_bytes": manual_path.stat().st_size,
|
||||
"static_viewer_assets_bytes": static_asset_bytes,
|
||||
},
|
||||
"process_peak_rss_kib": int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
arguments = _parser().parse_args()
|
||||
if arguments.nodes < 2:
|
||||
raise SystemExit("--nodes must be at least 2")
|
||||
if arguments.samples < 1 or arguments.cold_samples < 1:
|
||||
raise SystemExit("sample counts must be positive")
|
||||
with tempfile.TemporaryDirectory(prefix="docforge-milestone0-") as directory:
|
||||
root = Path(directory).resolve()
|
||||
_write_project(root, arguments.nodes)
|
||||
measurement = _benchmark(
|
||||
root,
|
||||
arguments.nodes,
|
||||
arguments.samples,
|
||||
arguments.cold_samples,
|
||||
)
|
||||
status = _git(["status", "--porcelain"])
|
||||
result: dict[str, object] = {
|
||||
"schema_version": 1,
|
||||
"benchmark": "docforge2_milestone0",
|
||||
"source": {
|
||||
"revision": _git(["rev-parse", "HEAD"]),
|
||||
"dirty": bool(status),
|
||||
},
|
||||
"environment": {
|
||||
"platform": platform.platform(),
|
||||
"machine": platform.machine(),
|
||||
"python": platform.python_version(),
|
||||
"implementation": platform.python_implementation(),
|
||||
},
|
||||
"method": {
|
||||
"clock": "time.perf_counter_ns",
|
||||
"memory": "resource.getrusage(RUSAGE_SELF).ru_maxrss",
|
||||
"response_size": "UTF-8 bytes of compact sorted JSON",
|
||||
"samples": arguments.samples,
|
||||
"cold_samples": arguments.cold_samples,
|
||||
},
|
||||
**measurement,
|
||||
"known_gaps": [
|
||||
"Generic warm reads still parse canonical source files.",
|
||||
"Compiler stages are not separately instrumented.",
|
||||
"Scaled incremental extraction is not measured by this generic fixture.",
|
||||
"Manual planning is not separated from rendering.",
|
||||
"Portable graph planning and rendering do not exist in Milestone 0.",
|
||||
"Per-operation peak RSS requires an external process harness.",
|
||||
],
|
||||
}
|
||||
encoded = json.dumps(result, sort_keys=True, indent=2) + "\n"
|
||||
if arguments.output is not None:
|
||||
output = arguments.output.resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(encoded, encoding="utf-8")
|
||||
sys.stdout.write(encoded)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue