Bound paged retrieval responses
This commit is contained in:
parent
176b2d2784
commit
529accf858
15 changed files with 1567 additions and 30 deletions
|
|
@ -21,7 +21,6 @@ from milestone0_baseline import (
|
|||
write_synthetic_project,
|
||||
)
|
||||
|
||||
from docforge.context import compile_context
|
||||
from docforge.index import ProjectIndex
|
||||
from docforge.mcp_server import DocForgeService
|
||||
from docforge.project import Project
|
||||
|
|
@ -80,27 +79,145 @@ def _diagnostics(result: object) -> Mapping[str, object]:
|
|||
return diagnostics
|
||||
|
||||
|
||||
def _result_summary(result: Mapping[str, object]) -> dict[str, object]:
|
||||
"""Retain bounded semantic evidence without copying primary result payloads."""
|
||||
|
||||
summary: dict[str, object] = {}
|
||||
for key in (
|
||||
"status",
|
||||
"count",
|
||||
"limit",
|
||||
"truncated",
|
||||
"truncation_reason",
|
||||
"candidate_edges_consumed",
|
||||
"candidate_edges_limit",
|
||||
"budget",
|
||||
"estimated_tokens",
|
||||
"state",
|
||||
"verification",
|
||||
"configured",
|
||||
"snapshot_state",
|
||||
"staleness",
|
||||
):
|
||||
if key in result:
|
||||
summary[key] = result[key]
|
||||
error = result.get("error")
|
||||
if isinstance(error, Mapping):
|
||||
error_payload = cast(Mapping[str, object], error)
|
||||
if isinstance(error_payload.get("code"), str):
|
||||
summary["error_code"] = error_payload["code"]
|
||||
synchronization = result.get("synchronization")
|
||||
if isinstance(synchronization, Mapping):
|
||||
synchronization_payload = cast(Mapping[str, object], synchronization)
|
||||
if isinstance(synchronization_payload.get("action"), str):
|
||||
summary["synchronization_action"] = synchronization_payload["action"]
|
||||
freshness = result.get("freshness")
|
||||
if isinstance(freshness, Mapping):
|
||||
freshness_payload = cast(Mapping[str, object], freshness)
|
||||
summary["freshness"] = {
|
||||
key: freshness_payload[key]
|
||||
for key in ("index", "source")
|
||||
if isinstance(freshness_payload.get(key), str)
|
||||
}
|
||||
outputs = result.get("outputs")
|
||||
if isinstance(outputs, list):
|
||||
summarized_outputs: list[dict[str, object]] = []
|
||||
for item in cast(list[object], outputs)[:10]:
|
||||
if not isinstance(item, Mapping):
|
||||
continue
|
||||
item_payload = cast(Mapping[str, object], item)
|
||||
summarized_outputs.append(
|
||||
{
|
||||
key: item_payload[key]
|
||||
for key in ("view_id", "state", "reason")
|
||||
if key in item_payload
|
||||
}
|
||||
)
|
||||
summary["outputs"] = summarized_outputs
|
||||
entries = result.get("entries")
|
||||
if isinstance(entries, list):
|
||||
summary["entry_count"] = len(cast(list[object], entries))
|
||||
omissions = result.get("omissions")
|
||||
if isinstance(omissions, list):
|
||||
summary["omission_count"] = len(cast(list[object], omissions))
|
||||
pagination = result.get("pagination")
|
||||
if isinstance(pagination, Mapping):
|
||||
pagination_payload = cast(Mapping[str, object], pagination)
|
||||
summary["pagination"] = {
|
||||
key: pagination_payload[key]
|
||||
for key in ("kind", "returned_count", "limit", "total_count", "has_more")
|
||||
if key in pagination_payload
|
||||
}
|
||||
return summary
|
||||
|
||||
|
||||
def _operation(
|
||||
operation: Callable[[], dict[str, object]],
|
||||
*,
|
||||
samples: int,
|
||||
p95_limit_ms: float,
|
||||
expected_status: str = "ok",
|
||||
expected_counters: Mapping[str, int],
|
||||
) -> dict[str, object]:
|
||||
measurement, last = measure_operation(operation, samples=samples)
|
||||
diagnostics_records: list[Mapping[str, object]] = []
|
||||
result_summaries: list[dict[str, object]] = []
|
||||
|
||||
def validated_operation() -> dict[str, object]:
|
||||
result = operation()
|
||||
diagnostics = _diagnostics(result)
|
||||
counters_value = diagnostics["counters"]
|
||||
if not isinstance(counters_value, Mapping):
|
||||
raise RuntimeError("Measured diagnostics did not return counters")
|
||||
counters = cast(Mapping[str, object], counters_value)
|
||||
for counter, expected in expected_counters.items():
|
||||
if counters.get(counter) != expected:
|
||||
raise RuntimeError(
|
||||
f"Warm operation expected {counter}={expected}, "
|
||||
f"received {counters.get(counter)!r}"
|
||||
)
|
||||
diagnostics_records.append(diagnostics)
|
||||
result_summaries.append(_result_summary(result))
|
||||
return result
|
||||
|
||||
measurement, last = measure_operation(validated_operation, samples=samples)
|
||||
if not isinstance(last, Mapping):
|
||||
raise RuntimeError(f"Measured operation did not return status={expected_status}")
|
||||
last_payload = cast(Mapping[str, object], last)
|
||||
if last_payload.get("status") != expected_status:
|
||||
raise RuntimeError(f"Measured operation did not return status={expected_status}")
|
||||
diagnostics = _diagnostics(last_payload)
|
||||
for summary in result_summaries:
|
||||
if summary.get("status") != expected_status:
|
||||
raise RuntimeError(f"Measured operation did not return status={expected_status}")
|
||||
p95_ms = float(cast(float, measurement["p95_ms"]))
|
||||
if p95_ms > p95_limit_ms:
|
||||
raise RuntimeError(f"Warm operation p95 {p95_ms:.3f} ms exceeds {p95_limit_ms:.3f} ms")
|
||||
counter_names = sorted(
|
||||
{
|
||||
key
|
||||
for diagnostics in diagnostics_records
|
||||
for key in cast(Mapping[str, object], diagnostics["counters"])
|
||||
}
|
||||
)
|
||||
counter_ranges = {
|
||||
counter: {
|
||||
"minimum": min(
|
||||
cast(int, cast(Mapping[str, object], record["counters"])[counter])
|
||||
for record in diagnostics_records
|
||||
),
|
||||
"maximum": max(
|
||||
cast(int, cast(Mapping[str, object], record["counters"])[counter])
|
||||
for record in diagnostics_records
|
||||
),
|
||||
}
|
||||
for counter in counter_names
|
||||
}
|
||||
return {
|
||||
**measurement,
|
||||
"p95_limit_ms": p95_limit_ms,
|
||||
"diagnostics": diagnostics,
|
||||
"validated_invocations": len(diagnostics_records),
|
||||
"counter_expectations": dict(sorted(expected_counters.items())),
|
||||
"counter_ranges": counter_ranges,
|
||||
"result_summary": result_summaries[-1],
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -110,11 +227,33 @@ def _benchmark(root: Path, node_count: int, samples: int) -> dict[str, object]:
|
|||
RenderService(project).render("manual")
|
||||
service = DocForgeService(project, diagnostics=True)
|
||||
target = synthetic_node_id(node_count - 1)
|
||||
backlink_target = synthetic_node_id(node_count - 2)
|
||||
first = synthetic_node_id(0)
|
||||
read_counters = {
|
||||
"index_checks": 1,
|
||||
"index_synchronizations": 0,
|
||||
"viewer_manager_requests": 0,
|
||||
}
|
||||
status_counters = {
|
||||
"index_checks": 0,
|
||||
"index_synchronizations": 0,
|
||||
"viewer_manager_requests": 0,
|
||||
}
|
||||
visualization_counters = {
|
||||
"index_checks": 0,
|
||||
"index_synchronizations": 0,
|
||||
"viewer_manager_requests": 1,
|
||||
}
|
||||
operations = {
|
||||
"warm_no_change_synchronize": _operation(
|
||||
service.synchronize,
|
||||
samples=samples,
|
||||
p95_limit_ms=100,
|
||||
expected_counters={
|
||||
"index_checks": 1,
|
||||
"index_synchronizations": 1,
|
||||
"viewer_manager_requests": 0,
|
||||
},
|
||||
),
|
||||
"exact_node": _operation(
|
||||
lambda: service.invoke(
|
||||
|
|
@ -123,6 +262,7 @@ def _benchmark(root: Path, node_count: int, samples: int) -> dict[str, object]:
|
|||
),
|
||||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_counters=read_counters,
|
||||
),
|
||||
"missing_node_error": _operation(
|
||||
lambda: service.invoke(
|
||||
|
|
@ -132,6 +272,7 @@ def _benchmark(root: Path, node_count: int, samples: int) -> dict[str, object]:
|
|||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_status="error",
|
||||
expected_counters=read_counters,
|
||||
),
|
||||
"search_limit_20": _operation(
|
||||
lambda: service.invoke(
|
||||
|
|
@ -140,6 +281,25 @@ def _benchmark(root: Path, node_count: int, samples: int) -> dict[str, object]:
|
|||
),
|
||||
samples=samples,
|
||||
p95_limit_ms=100,
|
||||
expected_counters=read_counters,
|
||||
),
|
||||
"filter_limit_20": _operation(
|
||||
lambda: service.invoke(
|
||||
lambda: service.index.filter_nodes(family="guide", limit=20),
|
||||
operation_name="mcp.filter",
|
||||
),
|
||||
samples=samples,
|
||||
p95_limit_ms=100,
|
||||
expected_counters=read_counters,
|
||||
),
|
||||
"backlinks_limit_20": _operation(
|
||||
lambda: service.invoke(
|
||||
lambda: service.index.backlinks(backlink_target, limit=20),
|
||||
operation_name="mcp.backlinks",
|
||||
),
|
||||
samples=samples,
|
||||
p95_limit_ms=100,
|
||||
expected_counters=read_counters,
|
||||
),
|
||||
"dependencies_depth_8": _operation(
|
||||
lambda: service.invoke(
|
||||
|
|
@ -148,21 +308,58 @@ def _benchmark(root: Path, node_count: int, samples: int) -> dict[str, object]:
|
|||
),
|
||||
samples=samples,
|
||||
p95_limit_ms=100,
|
||||
expected_counters=read_counters,
|
||||
),
|
||||
"context_32k": _operation(
|
||||
"impact_depth_8": _operation(
|
||||
lambda: service.invoke(
|
||||
lambda: compile_context(service.index, "active", 32_000),
|
||||
operation_name="mcp.context",
|
||||
lambda: service.index.impact(first, depth=8, limit=100),
|
||||
operation_name="mcp.impact",
|
||||
),
|
||||
samples=samples,
|
||||
p95_limit_ms=100,
|
||||
expected_counters=read_counters,
|
||||
),
|
||||
"context_32k": _operation(
|
||||
lambda: service.context("active", 32_000, limit=20),
|
||||
samples=samples,
|
||||
p95_limit_ms=250,
|
||||
expected_counters=read_counters,
|
||||
),
|
||||
"render_receipt_status": _operation(
|
||||
lambda: service.render_status("manual"),
|
||||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_counters=status_counters,
|
||||
),
|
||||
}
|
||||
template = root / "docs" / "templates" / "manual.html"
|
||||
original_template = template.read_bytes()
|
||||
template.write_bytes(original_template + b"\n")
|
||||
operations["render_stale_status"] = _operation(
|
||||
lambda: service.render_status("manual"),
|
||||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_counters=status_counters,
|
||||
)
|
||||
template.write_bytes(original_template)
|
||||
RenderService(project).render("manual")
|
||||
receipt_path = root / ".docforge" / "cache" / "render-receipts" / "manual.json"
|
||||
receipt_path.unlink()
|
||||
operations["render_missing_receipt_status"] = _operation(
|
||||
lambda: service.render_status("manual"),
|
||||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_counters=status_counters,
|
||||
)
|
||||
RenderService(project).render("manual")
|
||||
receipt_path.write_text("{", encoding="utf-8")
|
||||
operations["render_corrupt_receipt_status"] = _operation(
|
||||
lambda: service.render_status("manual"),
|
||||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_counters=status_counters,
|
||||
)
|
||||
RenderService(project).render("manual")
|
||||
state_path = root / ".docforge" / "benchmark-viewer-manager.json"
|
||||
manager = ViewerManager(state_path, check_interval_seconds=0.02)
|
||||
manager_thread = threading.Thread(target=manager.serve_forever, daemon=True)
|
||||
|
|
@ -181,6 +378,7 @@ def _benchmark(root: Path, node_count: int, samples: int) -> dict[str, object]:
|
|||
service.visualization_status,
|
||||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_counters=visualization_counters,
|
||||
)
|
||||
with service.index.path.open("ab") as stream:
|
||||
stream.write(b"\n")
|
||||
|
|
@ -188,12 +386,14 @@ def _benchmark(root: Path, node_count: int, samples: int) -> dict[str, object]:
|
|||
service.visualization_status,
|
||||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_counters=visualization_counters,
|
||||
)
|
||||
service.stop_visualization()
|
||||
operations["visualization_not_running_status"] = _operation(
|
||||
service.visualization_status,
|
||||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_counters=visualization_counters,
|
||||
)
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
|
@ -204,6 +404,7 @@ def _benchmark(root: Path, node_count: int, samples: int) -> dict[str, object]:
|
|||
samples=samples,
|
||||
p95_limit_ms=50,
|
||||
expected_status="error",
|
||||
expected_counters=visualization_counters,
|
||||
)
|
||||
return {
|
||||
"fixture": {
|
||||
|
|
@ -243,8 +444,13 @@ def main() -> int:
|
|||
"method": {
|
||||
"clock": "time.perf_counter_ns",
|
||||
"memory": "resource.getrusage(RUSAGE_SELF).ru_maxrss",
|
||||
"memory_scope": (
|
||||
"cumulative main-process high-water mark; detached viewer-worker memory excluded"
|
||||
),
|
||||
"response_size": "UTF-8 bytes of compact sorted JSON",
|
||||
"samples": arguments.samples,
|
||||
"warmups": 1,
|
||||
"percentile": "nearest-rank",
|
||||
"zero_work_counters": list(ZERO_WORK_COUNTERS),
|
||||
},
|
||||
**measurement,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue