diff --git a/DEVELOPMENT_NOTES.md b/DEVELOPMENT_NOTES.md index 5b8da3c..d9d5151 100644 --- a/DEVELOPMENT_NOTES.md +++ b/DEVELOPMENT_NOTES.md @@ -149,10 +149,17 @@ Omitted limits are capped by the project `max_results` policy. Traversal no longer loads the complete edge table and repeatedly scans it. It performs deterministically ordered frontier queries through the existing source primary key or target index. Each request also has a deterministic edge-examination budget derived from its result limit. The -response includes `examined_edges` and `examined_edges_limit` counters so algorithmic work can be -asserted independently of machine timing. `truncated` is true when either another unique result -exists or the work budget prevents proving completeness. A focused core, CLI, MCP, Ruff, and -Pyright gate passes for this work-in-progress slice. +response includes `candidate_edges_consumed`, `candidate_edges_limit`, and `truncation_reason` +counters so algorithmic work can be asserted independently of machine timing. `truncated` is true +when either another unique result exists or the work budget prevents proving completeness. + +The read-only query-plan audit found that source-ordered unfiltered incoming traversal required a +temporary SQLite sort with the version-2 `(target_id, relation, source_id)` index. Direct +`EXPLAIN QUERY PLAN` evidence showed `USE TEMP B-TREE FOR ORDER BY`. A measured additive +`(target_id, source_id, relation)` index removes that sort. The disposable index schema is now +version 3, so existing version-2 indexes rebuild without changing canonical source or proposals. +Frontier cursors are streamed and stop immediately on the first omitted unique result. A focused +core, CLI, MCP, Ruff, and Pyright gate passes for this work-in-progress slice. ### Initial design constraints diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index d570ff4..be98b17 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -68,7 +68,8 @@ Milestone 0 preserves: - Edge schema version 1. - Changeset schema version 1. - Result-envelope schema version 1. -- SQLite index schema version 2. +- SQLite index schema version 3. Version 2 indexes remain disposable and automatically rebuild; + version 3 adds a source-ordered incoming-edge index for bounded impact traversal. - Index-attestation schema version 1. - Incremental extraction-cache schema version 1. diff --git a/docs/MCP_CONTRACT.md b/docs/MCP_CONTRACT.md index d930b60..f07b453 100644 --- a/docs/MCP_CONTRACT.md +++ b/docs/MCP_CONTRACT.md @@ -38,6 +38,11 @@ returns the complete fixed binding, active index path, proposal and application recommended workflow. `docforge_sync` exposes the same idempotent synchronization explicitly. Neither operation changes canonical sources. +Search, filter, backlinks, dependencies, and impact accept explicit result limits bounded by the +project `max_results` policy. Omitted limits are still capped. Collection responses report whether +they were truncated. Traversal also reports whether truncation came from the result limit or its +deterministic candidate-edge work budget; it does not scan or materialize the complete edge table. + Adapter-backed servers also validate their process-start implementation fingerprint before every tool. `adapter_restart_required` is stale but not synchronizable. Its remediation is `restart_project_server`; the current process does not reload project code, update Git staging, or diff --git a/docs/USER_MANUAL.md b/docs/USER_MANUAL.md index 9f92c01..27cedb4 100644 --- a/docs/USER_MANUAL.md +++ b/docs/USER_MANUAL.md @@ -463,9 +463,9 @@ validate-index show NODE_ID search QUERY [--limit N] filter [--family X] [--authority X] [--status X] [--tag X] [--limit N] -backlinks NODE_ID [--relation RELATION] -dependencies NODE_ID [--depth N] -impact NODE_ID [--depth N] +backlinks NODE_ID [--relation RELATION] [--limit N] +dependencies NODE_ID [--depth N] [--limit N] +impact NODE_ID [--depth N] [--limit N] context PROFILE [--budget N] ``` diff --git a/src/docforge/index.py b/src/docforge/index.py index 9c7a749..9afab07 100644 --- a/src/docforge/index.py +++ b/src/docforge/index.py @@ -34,7 +34,7 @@ from .models import ( ) from .project import project_root_fingerprint -INDEX_SCHEMA_VERSION = 2 +INDEX_SCHEMA_VERSION = 3 APPLICATION_ID = 1_146_683_778 @@ -265,6 +265,8 @@ class ProjectIndex: PRIMARY KEY (source_id, relation, target_id) ); CREATE INDEX edges_target ON edges(target_id, relation, source_id); + CREATE INDEX edges_target_source + ON edges(target_id, source_id, relation); CREATE TABLE logic_owners ( owner_node_id TEXT PRIMARY KEY, source_id TEXT NOT NULL @@ -838,6 +840,7 @@ class ProjectIndex: count=len(results), limit=bounded, truncated=len(rows) > bounded, + truncation_reason="result_limit" if len(rows) > bounded else None, results=results, ) @@ -871,6 +874,7 @@ class ProjectIndex: count=len(results), limit=bounded, truncated=len(rows) > bounded, + truncation_reason="result_limit" if len(rows) > bounded else None, results=results, ) @@ -942,9 +946,12 @@ class ProjectIndex: truncated = len(rows) > bounded edges = [Edge(*row).as_dict() for row in rows[:bounded]] return snapshot.result( + root=node_id, + relation=relation, count=len(edges), limit=bounded, truncated=truncated, + truncation_reason="result_limit" if truncated else None, edges=edges, ) @@ -973,13 +980,16 @@ class ProjectIndex: queue: deque[tuple[str, int, tuple[str, ...]]] = deque([(node_id, 0, (node_id,))]) seen = {node_id} results: list[dict[str, object]] = [] - truncated = False - examined_edges = 0 - while queue and len(results) <= bounded and examined_edges < examined_limit: + truncation_reason: str | None = None + candidate_edges_consumed = 0 + while queue and truncation_reason is None: current, current_depth, path = queue.popleft() if current_depth >= depth: continue - remaining = examined_limit - examined_edges + remaining = examined_limit - candidate_edges_consumed + if remaining <= 0: + truncation_reason = "edge_examination_limit" + break values: tuple[object, ...] = ( (current, relation, remaining + 1) if relation is not None @@ -990,18 +1000,18 @@ class ProjectIndex: f"WHERE {source_column} = ?{relation_clause} " "ORDER BY source_id, relation, target_id LIMIT ?", values, - ).fetchall() - if len(candidates) > remaining: - truncated = True - candidates = candidates[:remaining] - examined_edges += len(candidates) + ) for row in candidates: + if candidate_edges_consumed >= examined_limit: + truncation_reason = "edge_examination_limit" + break + candidate_edges_consumed += 1 edge = Edge(*row) target = edge.source_id if incoming else edge.target_id if target in seen: continue if len(results) >= bounded: - truncated = True + truncation_reason = "result_limit" break seen.add(target) target_path = (*path, target) @@ -1014,16 +1024,15 @@ class ProjectIndex: } ) queue.append((target, current_depth + 1, target_path)) - if queue and examined_edges >= examined_limit: - truncated = True return snapshot.result( root=node_id, depth=depth, count=len(results), limit=bounded, - truncated=truncated, - examined_edges=examined_edges, - examined_edges_limit=examined_limit, + truncated=truncation_reason is not None, + truncation_reason=truncation_reason, + candidate_edges_consumed=candidate_edges_consumed, + candidate_edges_limit=examined_limit, results=results, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9c9a2c6..2338435 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -24,6 +24,22 @@ class DocForgeCliTests(unittest.TestCase): shutil.copytree(FIXTURES / "alpha", root) return root + def test_traversal_commands_accept_explicit_result_limits(self) -> None: + parser = _parser() + for command in ("backlinks", "dependencies", "impact"): + with self.subTest(command=command): + arguments = parser.parse_args( + [ + "--project-root", + "/tmp/project", + command, + "guide.workflow", + "--limit", + "7", + ] + ) + self.assertEqual(7, arguments.limit) + def test_reindex_apply_and_visualization_commands_are_self_service(self) -> None: with tempfile.TemporaryDirectory() as directory: parent = Path(directory) diff --git a/tests/test_core.py b/tests/test_core.py index dd5ece8..5c61b23 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -353,20 +353,31 @@ class DocForgeCoreTests(unittest.TestCase): self.assertEqual(["guide.workflow"], [item["node_id"] for item in limited["results"]]) self.assertEqual(1, limited["limit"]) self.assertTrue(limited["truncated"]) + self.assertEqual("result_limit", limited["truncation_reason"]) self.assertLessEqual( - limited["examined_edges"], - limited["examined_edges_limit"], + limited["candidate_edges_consumed"], + limited["candidate_edges_limit"], ) complete_limit = index.dependencies("guide.workflow", depth=2, limit=1) self.assertEqual(1, complete_limit["count"]) self.assertFalse(complete_limit["truncated"]) + self.assertIsNone(complete_limit["truncation_reason"]) limited_backlinks = index.backlinks("guide.workflow", limit=1) self.assertEqual(1, limited_backlinks["count"]) self.assertEqual(1, limited_backlinks["limit"]) self.assertFalse(limited_backlinks["truncated"]) + for operation in ( + lambda: index.backlinks("guide.workflow", limit=0), + lambda: index.dependencies("guide.workflow", limit=True), + lambda: index.impact("guide.workflow", limit=101), + ): + with self.subTest(operation=operation), self.assertRaises(DocForgeError) as invalid: + operation() + self.assertEqual("invalid_limit", invalid.exception.code) + def test_query_rechecks_source_identity_before_returning(self) -> None: with tempfile.TemporaryDirectory() as directory: root = self.copy_fixture("alpha", Path(directory)) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 0fee356..6e14530 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -70,6 +70,14 @@ class DocForgeMcpTests(unittest.IsolatedAsyncioTestCase): names = tuple(tool.name for tool in response.tools) self.assertEqual(ALL_TOOLS, names) self.assertEqual(14, len(PROPOSAL_TOOLS)) + tools = {tool.name: tool for tool in response.tools} + for name in ( + "docforge_backlinks", + "docforge_dependencies", + "docforge_impact", + ): + self.assertIn("limit", tools[name].inputSchema["properties"]) + self.assertNotIn("limit", tools[name].inputSchema.get("required", [])) self.assertFalse( any( token in name @@ -193,6 +201,24 @@ class DocForgeMcpTests(unittest.IsolatedAsyncioTestCase): self.assertLessEqual(context["estimated_tokens"], 180) self.assertTrue(context["omissions"]) + async def test_invalid_traversal_limit_is_a_structured_domain_error(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.copy_fixture("alpha", Path(directory)) + ProjectIndex(Project.open(root)).build() + async with create_connected_server_and_client_session( + create_server(root), raise_exceptions=True + ) as session: + result = await session.call_tool( + "docforge_impact", + {"node_id": "guide.foundation", "limit": 0}, + ) + + self.assertEqual("error", result.structuredContent["status"]) + self.assertEqual( + "invalid_limit", + result.structuredContent["error"]["code"], + ) + async def test_sync_register_rebase_apply_and_lifecycle_are_one_bound_workflow( self, ) -> None: