1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions Pages

Bound paged retrieval responses

This commit is contained in:
Andraxion 2026-07-29 06:02:07 -04:00
parent 176b2d2784
commit 529accf858
15 changed files with 1567 additions and 30 deletions

View file

@ -20,9 +20,12 @@ from .changeset_contract import (
)
from .errors import DocForgeError
from .models import Edge, Node, ProjectService, ProjectSnapshot, ProposalWriter
from .pagination import canonical_hash, decode_cursor, page_limit, page_receipt
from .project import project_root_fingerprint
from .proposal_projection import ProposalProjector
MAX_ABANDON_REASON_CHARS = 2_000
class ChangesetStore:
"""One project-bound proposal store with an optional immutable writer identity."""
@ -277,23 +280,37 @@ class ChangesetStore:
},
)
def validate(self, changeset_id: str) -> dict[str, object]:
def validate(
self,
changeset_id: str,
*,
limit: int | None = None,
cursor: str | None = None,
) -> dict[str, object]:
validate_id(changeset_id, "changeset_id")
with self._lock():
snapshot, document, nodes, edges = self._validate_locked(changeset_id)
return self._result(
result = self._result(
snapshot,
document,
valid=True,
projected_node_count=len(nodes),
projected_edge_count=len(edges),
)
return self._page_document_result(
result,
kind="changeset.validate",
limit=limit,
cursor=cursor,
)
def list_changesets(
self,
*,
include_history: bool = True,
status: str | None = None,
limit: int | None = None,
cursor: str | None = None,
) -> dict[str, object]:
with self._lock():
snapshot = self.project.load()
@ -322,14 +339,93 @@ class ChangesetStore:
"operation_count": len(document["operations"]),
}
)
return self._base_result(snapshot, count=len(records), changesets=records)
result = self._base_result(snapshot, count=len(records), changesets=records)
if limit is None and cursor is None:
return result
selected_limit = page_limit(
limit,
default=20,
maximum=self.project.descriptor.limits.max_results,
)
binding = {
"project_id": snapshot.descriptor.project_id,
"project_root_fingerprint": project_root_fingerprint(snapshot.descriptor.root),
"adapter": snapshot.descriptor.adapter,
"revision": snapshot.revision,
"source_hash": snapshot.source_hash,
"include_history": include_history,
"status": status,
"collection_hash": canonical_hash(records),
}
position = decode_cursor(
cursor,
kind="changeset.list",
binding=binding,
total_count=len(records),
)
page = records[position : position + selected_limit]
while page:
candidate = {
**result,
"count": len(page),
"total_count": len(records),
"changesets": page,
"pagination": page_receipt(
kind="changeset.list",
binding=binding,
position=position,
count=len(page),
limit=selected_limit,
total_count=len(records),
),
}
if self._encoded_length(candidate) <= self._safe_page_chars():
return candidate
page.pop()
if position < len(records):
compact_record = self._compact_list_record(records[position])
return {
**result,
"count": 1,
"total_count": len(records),
"changesets": [compact_record],
"result_mode": "changeset_summaries",
"pagination": page_receipt(
kind="changeset.list",
binding=binding,
position=position,
count=1,
limit=selected_limit,
total_count=len(records),
),
}
return {
**result,
"count": 0,
"total_count": len(records),
"changesets": [],
"pagination": page_receipt(
kind="changeset.list",
binding=binding,
position=position,
count=0,
limit=selected_limit,
total_count=len(records),
),
}
def inspect(self, changeset_id: str) -> dict[str, object]:
def inspect(
self,
changeset_id: str,
*,
limit: int | None = None,
cursor: str | None = None,
) -> dict[str, object]:
validate_id(changeset_id, "changeset_id")
with self._lock():
document = self._read(self._path(changeset_id))
snapshot = self.project.load()
return self._result(
result = self._result(
snapshot,
document,
base_state=self._base_state(document, snapshot),
@ -338,6 +434,12 @@ class ChangesetStore:
self._base_state(document, snapshot),
),
)
return self._page_document_result(
result,
kind="changeset.inspect",
limit=limit,
cursor=cursor,
)
def rebase(
self,
@ -398,8 +500,15 @@ class ChangesetStore:
validate_id(changeset_id, "changeset_id")
validate_hash(expected_changeset_hash, "expected_changeset_hash")
if not reason.strip():
normalized_reason = reason.strip()
if not normalized_reason:
raise DocForgeError("invalid_operation", "Abandon reason must be non-empty")
if len(normalized_reason) > MAX_ABANDON_REASON_CHARS:
raise DocForgeError(
"changeset_too_large",
"Abandon reason exceeds its character limit",
maximum=MAX_ABANDON_REASON_CHARS,
)
with self._lock():
document = self._read(self._path(changeset_id))
actual_hash = document_hash(document)
@ -418,7 +527,7 @@ class ChangesetStore:
{
"status": "abandoned",
"changeset_hash": actual_hash,
"reason": reason.strip(),
"reason": normalized_reason,
"revision": snapshot.revision,
"source_hash": snapshot.source_hash,
},
@ -430,7 +539,13 @@ class ChangesetStore:
lifecycle=receipt,
)
def diff(self, changeset_id: str) -> dict[str, object]:
def diff(
self,
changeset_id: str,
*,
limit: int | None = None,
cursor: str | None = None,
) -> dict[str, object]:
validate_id(changeset_id, "changeset_id")
with self._lock():
snapshot, document, _, _ = self._validate_locked(changeset_id)
@ -453,7 +568,14 @@ class ChangesetStore:
sorted(before_edges - edges),
)
)
return self._result(snapshot, document, valid=True, changes=changes)
result = self._result(snapshot, document, valid=True, changes=changes)
return self._page_document_result(
result,
kind="changeset.diff",
limit=limit,
cursor=cursor,
parallel_key="changes",
)
def projected_snapshot(self, changeset_id: str) -> tuple[ProjectSnapshot, str]:
"""Return a validated in-memory proposal projection for derived preview use."""
@ -790,6 +912,219 @@ class ChangesetStore:
**payload,
)
def _page_document_result(
self,
result: dict[str, object],
*,
kind: str,
limit: int | None,
cursor: str | None,
parallel_key: str | None = None,
) -> dict[str, object]:
"""Page bulky operation-aligned payloads while preserving direct full defaults."""
if limit is None and cursor is None:
return result
selected_limit = page_limit(
limit,
default=20,
maximum=self.project.descriptor.limits.max_results,
)
operations_value = result.get("operations")
if not isinstance(operations_value, list):
raise DocForgeError(
"invalid_pagination_source",
"Changeset result does not contain a deterministic operation list",
)
operations = cast(list[object], operations_value)
parallel: list[object] | None = None
if parallel_key is not None:
parallel_value = result.get(parallel_key)
if not isinstance(parallel_value, list):
raise DocForgeError(
"invalid_pagination_source",
"Changeset result does not contain an aligned detail list",
)
parallel = cast(list[object], parallel_value)
if len(parallel) != len(operations):
raise DocForgeError(
"invalid_pagination_source",
"Changeset detail list is not aligned with its operations",
)
binding = {
"project_id": result["project_id"],
"project_root_fingerprint": result["project_root_fingerprint"],
"revision": result["revision"],
"source_hash": result["source_hash"],
"changeset_id": result["changeset_id"],
"changeset_hash": result["changeset_hash"],
"adapter": result["adapter"],
"result_hash": canonical_hash(
{
"operations": operations,
**({parallel_key: parallel} if parallel_key is not None else {}),
}
),
}
if (
kind == "changeset.diff"
and parallel_key is not None
and parallel is not None
and self._encoded_length(result) > self._safe_page_chars()
):
return self._page_json_chunks(
result,
operations=operations,
changes=parallel,
binding=binding,
cursor=cursor,
)
position = decode_cursor(
cursor,
kind=kind,
binding=binding,
total_count=len(operations),
)
page = operations[position : position + selected_limit]
paged = {
**result,
"operations": page,
"returned_operation_count": len(page),
"pagination": page_receipt(
kind=kind,
binding=binding,
position=position,
count=len(page),
limit=selected_limit,
total_count=len(operations),
),
}
if parallel_key is not None and parallel is not None:
paged[parallel_key] = parallel[position : position + selected_limit]
if self._encoded_length(paged) > self._safe_page_chars():
paged["operations"] = [
self._operation_summary(item)
for item in operations[position : position + selected_limit]
]
paged["result_mode"] = "operation_summaries"
paged["detail_tool"] = "docforge_get_changeset_diff"
return paged
def _page_json_chunks(
self,
result: dict[str, object],
*,
operations: list[object],
changes: list[object],
binding: Mapping[str, object],
cursor: str | None,
) -> dict[str, object]:
payload = {"operations": operations, "changes": changes}
encoded = json.dumps(
payload,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
allow_nan=False,
)
chunk_chars = max(512, min(64_000, self._safe_page_chars() // 2))
chunks = [
encoded[offset : offset + chunk_chars] for offset in range(0, len(encoded), chunk_chars)
] or [""]
chunk_binding = {
**binding,
"payload_hash": canonical_hash(payload),
"chunk_chars": chunk_chars,
}
position = decode_cursor(
cursor,
kind="changeset.diff-chunks",
binding=chunk_binding,
total_count=len(chunks),
)
compact = {
key: value for key, value in result.items() if key not in {"operations", "changes"}
}
return {
**compact,
"result_mode": "canonical_json_chunk",
"payload": "changeset_diff",
"payload_hash": chunk_binding["payload_hash"],
"payload_characters": len(encoded),
"chunk": {
"index": position,
"characters": len(chunks[position]),
"content": chunks[position],
},
"pagination": page_receipt(
kind="changeset.diff-chunks",
binding=chunk_binding,
position=position,
count=1,
limit=1,
total_count=len(chunks),
),
}
@staticmethod
def _operation_summary(operation: object) -> dict[str, object]:
if not isinstance(operation, Mapping):
raise DocForgeError(
"invalid_pagination_source",
"Changeset operation is not a deterministic object",
)
payload = cast(Mapping[str, object], operation)
summary = {
key: payload.get(key)
for key in (
"sequence",
"operation",
"node_id",
"expected_content_hash",
"target_source",
)
}
for key in ("metadata", "content", "relationship_changes", "rationale"):
value = payload.get(key)
summary[f"{key}_hash"] = canonical_hash(value)
summary[f"{key}_characters"] = len(
json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
allow_nan=False,
)
)
return summary
@staticmethod
def _compact_list_record(record: dict[str, object]) -> dict[str, object]:
lifecycle = record.get("lifecycle")
if not isinstance(lifecycle, Mapping):
return record
lifecycle_payload = cast(Mapping[str, object], lifecycle)
reason = lifecycle_payload.get("reason")
if not isinstance(reason, str):
return record
return {
**record,
"lifecycle": {
**lifecycle_payload,
"reason": {
"characters": len(reason),
"sha256": canonical_hash(reason),
},
},
}
def _safe_page_chars(self) -> int:
return max(1_024, self.project.descriptor.limits.max_tool_output_chars - 2_048)
@staticmethod
def _encoded_length(value: Mapping[str, object]) -> int:
return len(json.dumps(value, sort_keys=True, separators=(",", ":")))
@staticmethod
def _base_result(snapshot: ProjectSnapshot, **payload: object) -> dict[str, object]:
return {

View file

@ -63,6 +63,8 @@ def _parser() -> argparse.ArgumentParser:
context = commands.add_parser("context")
context.add_argument("profile")
context.add_argument("--budget", type=int)
context.add_argument("--limit", type=int)
context.add_argument("--cursor")
render = commands.add_parser("render")
render.add_argument("view_id")
render_status = commands.add_parser("render-status")
@ -175,6 +177,15 @@ def _run(arguments: argparse.Namespace) -> dict[str, object]:
limit=arguments.limit,
)
if arguments.command == "context":
if arguments.limit is not None or arguments.cursor is not None:
from .mcp_server import DocForgeService
return DocForgeService(project).context(
arguments.profile,
arguments.budget,
limit=arguments.limit,
cursor=arguments.cursor,
)
return compile_context(index, arguments.profile, arguments.budget)
if arguments.command == "render":
return RenderService(project).render(arguments.view_id)

View file

@ -17,6 +17,7 @@ from .context import compile_context
from .errors import DocForgeError
from .index import ProjectIndex
from .models import IncrementalStateProject, ProjectService, RuntimeValidatedProject
from .pagination import canonical_hash, decode_cursor, page_limit, page_receipt
from .project import Project, project_root_fingerprint
from .rendering import RenderService
from .telemetry import request, stage
@ -87,6 +88,7 @@ STALE_ERROR_CODES = frozenset(
"content_conflict",
"source_changed",
"stale_adapter_source",
"stale_cursor",
"stale_index",
}
)
@ -488,6 +490,11 @@ class DocForgeService:
"tool": "docforge_get_changeset",
"arguments": {"changeset_id": "<same>"},
}
if error.code == "stale_cursor":
return {
"retryable": True,
"action": "restart_pagination",
}
return None
def synchronize(self) -> dict[str, object]:
@ -707,12 +714,158 @@ class DocForgeService:
operation_name="mcp.render_status",
)
def context(self, profile: str, budget: int | None = None) -> dict[str, Any]:
def context(
self,
profile: str,
budget: int | None = None,
*,
limit: int | None = None,
cursor: str | None = None,
) -> dict[str, Any]:
def operation() -> dict[str, object]:
selected_limit = page_limit(
limit,
default=20,
maximum=self.project.descriptor.limits.max_results,
)
result = self.context_provider(self.index, profile, budget)
return self._page_context_result(
result,
profile=profile,
selected_limit=selected_limit,
cursor=cursor,
)
return self.invoke(
lambda: self.context_provider(self.index, profile, budget),
operation,
operation_name="mcp.context",
)
def _page_context_result(
self,
result: dict[str, object],
*,
profile: str,
selected_limit: int,
cursor: str | None,
) -> dict[str, object]:
entries_value = result.get("entries")
omissions_value = result.get("omissions")
if not isinstance(entries_value, list) or not isinstance(omissions_value, list):
raise DocForgeError(
"invalid_context_result",
"Context provider did not return deterministic entries and omissions",
)
entries = cast(list[object], entries_value)
omissions = cast(list[object], omissions_value)
binding = {
"project_id": result.get("project_id"),
"project_root_fingerprint": result.get("project_root_fingerprint"),
"adapter": result.get("adapter"),
"revision": result.get("revision"),
"source_hash": result.get("source_hash"),
"profile": profile,
"budget": result.get("budget"),
"selection_hash": canonical_hash(
{
"entries": entries,
"omissions": omissions,
}
),
}
evidence = [
*(("entry", item) for item in entries),
*(("omission", item) for item in omissions),
]
position = decode_cursor(
cursor,
kind="context.items",
binding=binding,
total_count=len(evidence),
)
page_entries: list[object] = []
page_omissions: list[object] = []
consumed = 0
truncation_reason: str | None = None
maximum = self.project.descriptor.limits.max_tool_output_chars
def page_result() -> dict[str, object]:
pagination = page_receipt(
kind="context.items",
binding=binding,
position=position,
count=consumed,
limit=selected_limit,
total_count=len(evidence),
)
return {
**result,
"entries": page_entries,
"omissions": page_omissions,
"entry_count": len(page_entries),
"omission_count": len(page_omissions),
"page_count": consumed,
"page_estimated_tokens": sum(
cast(int, cast(Mapping[str, object], item).get("estimated_tokens", 0))
for item in page_entries
if isinstance(item, Mapping)
),
"summary": {
"entry_count": len(entries),
"omission_count": len(omissions),
"evidence_count": len(evidence),
"estimated_tokens": result.get("estimated_tokens"),
},
"truncation_reason": truncation_reason,
"next_cursor": pagination["next_cursor"],
"pagination": pagination,
}
for kind, item in evidence[position:]:
if consumed >= selected_limit:
truncation_reason = "result_limit"
break
destination = page_entries if kind == "entry" else page_omissions
destination.append(item)
consumed += 1
candidate = page_result()
decorated = {
**candidate,
"server_version": SERVER_VERSION,
"content_warning": CONTENT_WARNING,
"staleness": "current",
}
if self._encoded_length(decorated) <= maximum:
continue
destination.pop()
consumed -= 1
truncation_reason = "response_limit"
if consumed == 0:
compact = self._oversized_context_omission(kind, item)
page_omissions.append(compact)
consumed = 1
break
if position + consumed < len(evidence) and truncation_reason is None:
truncation_reason = "result_limit"
return page_result()
@staticmethod
def _oversized_context_omission(kind: str, item: object) -> dict[str, object]:
node_id = "unknown"
hash_source = item
if isinstance(item, Mapping):
item_payload = cast(Mapping[str, object], item)
candidate = item_payload.get("node_id")
if isinstance(candidate, str) and candidate:
node_id = candidate[:256]
hash_source = dict(item_payload)
return {
"node_id": node_id,
"reason": "response size limit",
"original_evidence": kind,
"detail_hash": canonical_hash(hash_source),
}
def visualize(
self,
node_id: str | None = None,
@ -897,10 +1050,15 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC
)
@server.tool(name="docforge_get_context")
def get_context(profile: str, budget: int | None = None) -> dict[str, Any]:
def get_context(
profile: str,
budget: int | None = None,
limit: int | None = None,
cursor: str | None = None,
) -> dict[str, Any]:
"""Compile bounded cited context from one configured profile with explicit omissions."""
return service.context(profile, budget)
return service.context(profile, budget, limit=limit, cursor=cursor)
@server.tool(name="docforge_validate_project")
def validate_project() -> dict[str, Any]:
@ -1000,6 +1158,8 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC
def list_changesets(
include_history: bool = False,
status: str | None = None,
limit: int | None = 20,
cursor: str | None = None,
) -> dict[str, Any]:
"""List active proposals by default, with optional lifecycle history."""
@ -1007,16 +1167,26 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC
lambda: service.changesets.list_changesets(
include_history=include_history,
status=status,
limit=20 if limit is None else limit,
cursor=cursor,
),
operation_name="mcp.changeset",
)
@server.tool(name="docforge_get_changeset")
def get_changeset(changeset_id: str) -> dict[str, Any]:
def get_changeset(
changeset_id: str,
limit: int | None = 20,
cursor: str | None = None,
) -> dict[str, Any]:
"""Inspect a stored proposal even when its canonical base has become stale."""
return service.invoke(
lambda: service.changesets.inspect(changeset_id),
lambda: service.changesets.inspect(
changeset_id,
limit=20 if limit is None else limit,
cursor=cursor,
),
operation_name="mcp.changeset",
)
@ -1225,20 +1395,36 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC
)
@server.tool(name="docforge_validate_changeset")
def validate_changeset(changeset_id: str) -> dict[str, Any]:
def validate_changeset(
changeset_id: str,
limit: int | None = 20,
cursor: str | None = None,
) -> dict[str, Any]:
"""Validate a proposal against its exact canonical base and other active proposals."""
return service.invoke(
lambda: service.changesets.validate(changeset_id),
lambda: service.changesets.validate(
changeset_id,
limit=20 if limit is None else limit,
cursor=cursor,
),
operation_name="mcp.changeset",
)
@server.tool(name="docforge_get_changeset_diff")
def get_changeset_diff(changeset_id: str) -> dict[str, Any]:
def get_changeset_diff(
changeset_id: str,
limit: int | None = 20,
cursor: str | None = None,
) -> dict[str, Any]:
"""Return a deterministic structured and textual diff without applying the proposal."""
return service.invoke(
lambda: service.changesets.diff(changeset_id),
lambda: service.changesets.diff(
changeset_id,
limit=20 if limit is None else limit,
cursor=cursor,
),
operation_name="mcp.changeset",
)

163
src/docforge/pagination.py Normal file
View file

@ -0,0 +1,163 @@
"""Deterministic, generation-bound pagination cursors for bounded public results."""
from __future__ import annotations
import base64
import binascii
import hashlib
import hmac
import json
from collections.abc import Mapping
from typing import cast
from .errors import DocForgeError
CURSOR_SCHEMA_VERSION = 1
MAX_CURSOR_CHARS = 8_192
_CURSOR_DOMAIN = b"docforge-page-cursor-v1\0"
_CURSOR_KEYS = frozenset({"schema_version", "kind", "binding", "position", "checksum"})
def canonical_hash(value: object) -> str:
"""Hash one JSON-compatible value using DocForge's deterministic JSON form."""
try:
encoded = _canonical_bytes(value)
except (TypeError, ValueError) as error:
raise DocForgeError(
"invalid_pagination_source",
"Pagination source data is not deterministic JSON",
) from error
return hashlib.sha256(encoded).hexdigest()
def page_limit(limit: int | None, *, default: int, maximum: int) -> int:
"""Validate one additive page size against the project result policy."""
selected = default if limit is None else limit
if type(selected) is not int or selected < 1 or selected > maximum:
raise DocForgeError(
"invalid_limit",
"Page limit is outside the configured result limit",
maximum=maximum,
)
return selected
def encode_cursor(
*,
kind: str,
binding: Mapping[str, object],
position: int,
) -> str:
"""Encode a corruption-detecting cursor bound to an immutable result identity."""
if not kind or type(position) is not int or position < 0:
raise ValueError("Cursor kind and position must be valid")
body: dict[str, object] = {
"schema_version": CURSOR_SCHEMA_VERSION,
"kind": kind,
"binding": dict(binding),
"position": position,
}
checksum = hashlib.sha256(_CURSOR_DOMAIN + _canonical_bytes(body)).hexdigest()
envelope = {**body, "checksum": checksum}
return base64.urlsafe_b64encode(_canonical_bytes(envelope)).decode("ascii").rstrip("=")
def decode_cursor(
cursor: str | None,
*,
kind: str,
binding: Mapping[str, object],
total_count: int,
) -> int:
"""Return a validated position, rejecting corrupt, foreign, or stale cursors."""
if cursor is None:
return 0
if not cursor or len(cursor) > MAX_CURSOR_CHARS or not cursor.isascii():
raise _invalid_cursor()
padding = "=" * (-len(cursor) % 4)
try:
raw = base64.b64decode(
(cursor + padding).encode("ascii"),
altchars=b"-_",
validate=True,
)
parsed: object = json.loads(raw.decode("utf-8"))
except (binascii.Error, UnicodeDecodeError, json.JSONDecodeError):
raise _invalid_cursor() from None
if base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") != cursor or not isinstance(
parsed, dict
):
raise _invalid_cursor()
payload = cast(dict[str, object], parsed)
checksum = payload.get("checksum")
position = payload.get("position")
stored_binding = payload.get("binding")
if (
frozenset(payload) != _CURSOR_KEYS
or payload.get("schema_version") != CURSOR_SCHEMA_VERSION
or payload.get("kind") != kind
or not isinstance(stored_binding, dict)
or type(position) is not int
or position < 0
or position >= total_count
or not isinstance(checksum, str)
or len(checksum) != 64
):
raise _invalid_cursor()
body = {key: payload[key] for key in payload if key != "checksum"}
expected = hashlib.sha256(_CURSOR_DOMAIN + _canonical_bytes(body)).hexdigest()
if not hmac.compare_digest(checksum, expected):
raise _invalid_cursor()
if stored_binding != dict(binding):
raise DocForgeError(
"stale_cursor",
"Pagination cursor does not match the current result generation",
)
return position
def page_receipt(
*,
kind: str,
binding: Mapping[str, object],
position: int,
count: int,
limit: int,
total_count: int,
) -> dict[str, object]:
"""Return one bounded page receipt and the next generation-bound cursor."""
next_position = position + count
has_more = next_position < total_count
return {
"schema_version": CURSOR_SCHEMA_VERSION,
"kind": kind,
"returned_count": count,
"limit": limit,
"total_count": total_count,
"has_more": has_more,
"next_cursor": (
encode_cursor(kind=kind, binding=binding, position=next_position) if has_more else None
),
}
def _canonical_bytes(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
allow_nan=False,
).encode("utf-8")
def _invalid_cursor() -> DocForgeError:
return DocForgeError(
"invalid_cursor",
"Pagination cursor is malformed or does not match its operation",
)