Bound indexed retrieval operations
This commit is contained in:
parent
ad4f52b239
commit
c69cd16515
6 changed files with 208 additions and 46 deletions
|
|
@ -826,14 +826,20 @@ class ProjectIndex:
|
|||
ORDER BY rank, nodes.node_id
|
||||
LIMIT ?
|
||||
""",
|
||||
(expression, bounded),
|
||||
(expression, bounded + 1),
|
||||
).fetchall()
|
||||
results: list[dict[str, object]] = []
|
||||
for row in rows:
|
||||
for row in rows[:bounded]:
|
||||
payload = _row_to_node(row).as_dict(include_content=False)
|
||||
payload.update({"rank": row["rank"], "snippet": row["snippet"]})
|
||||
results.append(payload)
|
||||
return snapshot.result(query=query, count=len(results), results=results)
|
||||
return snapshot.result(
|
||||
query=query,
|
||||
count=len(results),
|
||||
limit=bounded,
|
||||
truncated=len(rows) > bounded,
|
||||
results=results,
|
||||
)
|
||||
|
||||
def filter_nodes(
|
||||
self,
|
||||
|
|
@ -857,66 +863,146 @@ class ProjectIndex:
|
|||
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
||||
with self._read_snapshot() as snapshot:
|
||||
rows = snapshot.connection.execute(
|
||||
f"SELECT * FROM nodes {where} ORDER BY node_id LIMIT ?", (*values, bounded)
|
||||
f"SELECT * FROM nodes {where} ORDER BY node_id LIMIT ?",
|
||||
(*values, bounded + 1),
|
||||
).fetchall()
|
||||
results = [_row_to_node(row).as_dict(include_content=False) for row in rows]
|
||||
return snapshot.result(count=len(results), results=results)
|
||||
results = [_row_to_node(row).as_dict(include_content=False) for row in rows[:bounded]]
|
||||
return snapshot.result(
|
||||
count=len(results),
|
||||
limit=bounded,
|
||||
truncated=len(rows) > bounded,
|
||||
results=results,
|
||||
)
|
||||
|
||||
def backlinks(self, node_id: str, *, relation: str | None = None) -> dict[str, object]:
|
||||
return self._edges(node_id, incoming=True, relation=relation)
|
||||
def backlinks(
|
||||
self,
|
||||
node_id: str,
|
||||
*,
|
||||
relation: str | None = None,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
return self._edges(node_id, incoming=True, relation=relation, limit=limit)
|
||||
|
||||
def dependencies(self, node_id: str, *, depth: int = 2) -> dict[str, object]:
|
||||
return self._traverse(node_id, incoming=False, depth=depth, relation="depends_on")
|
||||
def dependencies(
|
||||
self,
|
||||
node_id: str,
|
||||
*,
|
||||
depth: int = 2,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
return self._traverse(
|
||||
node_id,
|
||||
incoming=False,
|
||||
depth=depth,
|
||||
relation="depends_on",
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def impact(self, node_id: str, *, depth: int = 2) -> dict[str, object]:
|
||||
return self._traverse(node_id, incoming=True, depth=depth, relation=None)
|
||||
def impact(
|
||||
self,
|
||||
node_id: str,
|
||||
*,
|
||||
depth: int = 2,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
return self._traverse(
|
||||
node_id,
|
||||
incoming=True,
|
||||
depth=depth,
|
||||
relation=None,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def _edges(self, node_id: str, *, incoming: bool, relation: str | None) -> dict[str, object]:
|
||||
def _edges(
|
||||
self,
|
||||
node_id: str,
|
||||
*,
|
||||
incoming: bool,
|
||||
relation: str | None,
|
||||
limit: int | None,
|
||||
) -> dict[str, object]:
|
||||
bounded = _bounded_limit(
|
||||
limit,
|
||||
self.project.descriptor.limits.max_results,
|
||||
default=self.project.descriptor.limits.max_results,
|
||||
)
|
||||
source_column = "target_id" if incoming else "source_id"
|
||||
relation_clause = " AND relation = ?" if relation is not None else ""
|
||||
values: tuple[object, ...] = (node_id, relation) if relation is not None else (node_id,)
|
||||
values: tuple[object, ...] = (
|
||||
(node_id, relation, bounded + 1) if relation is not None else (node_id, bounded + 1)
|
||||
)
|
||||
with self._read_snapshot() as snapshot:
|
||||
self._require_node(snapshot.connection, node_id)
|
||||
rows = snapshot.connection.execute(
|
||||
f"SELECT source_id, relation, target_id FROM edges "
|
||||
f"WHERE {source_column} = ?{relation_clause} "
|
||||
"ORDER BY source_id, relation, target_id",
|
||||
"ORDER BY source_id, relation, target_id LIMIT ?",
|
||||
values,
|
||||
).fetchall()
|
||||
return snapshot.result(edges=[Edge(*row).as_dict() for row in rows])
|
||||
truncated = len(rows) > bounded
|
||||
edges = [Edge(*row).as_dict() for row in rows[:bounded]]
|
||||
return snapshot.result(
|
||||
count=len(edges),
|
||||
limit=bounded,
|
||||
truncated=truncated,
|
||||
edges=edges,
|
||||
)
|
||||
|
||||
def _traverse(
|
||||
self, node_id: str, *, incoming: bool, depth: int, relation: str | None
|
||||
self,
|
||||
node_id: str,
|
||||
*,
|
||||
incoming: bool,
|
||||
depth: int,
|
||||
relation: str | None,
|
||||
limit: int | None,
|
||||
) -> dict[str, object]:
|
||||
maximum = self.project.descriptor.limits.max_traversal_depth
|
||||
if type(depth) is not int or depth < 0 or depth > maximum:
|
||||
raise DocForgeError("invalid_depth", "Traversal depth is outside the configured limit")
|
||||
bounded = _bounded_limit(
|
||||
limit,
|
||||
self.project.descriptor.limits.max_results,
|
||||
default=self.project.descriptor.limits.max_results,
|
||||
)
|
||||
examined_limit = (bounded + 1) ** 2
|
||||
source_column = "target_id" if incoming else "source_id"
|
||||
relation_clause = " AND relation = ?" if relation is not None else ""
|
||||
with self._read_snapshot() as snapshot:
|
||||
self._require_node(snapshot.connection, node_id)
|
||||
edges = tuple(
|
||||
Edge(*row)
|
||||
for row in snapshot.connection.execute(
|
||||
"SELECT source_id, relation, target_id FROM edges "
|
||||
"ORDER BY source_id, relation, target_id"
|
||||
)
|
||||
)
|
||||
queue: deque[tuple[str, int, tuple[str, ...]]] = deque([(node_id, 0, (node_id,))])
|
||||
seen = {node_id}
|
||||
results: list[dict[str, object]] = []
|
||||
while queue:
|
||||
truncated = False
|
||||
examined_edges = 0
|
||||
while queue and len(results) <= bounded and examined_edges < examined_limit:
|
||||
current, current_depth, path = queue.popleft()
|
||||
if current_depth >= depth:
|
||||
continue
|
||||
candidates = [
|
||||
edge
|
||||
for edge in edges
|
||||
if (relation is None or edge.relation == relation)
|
||||
and ((edge.target_id if incoming else edge.source_id) == current)
|
||||
]
|
||||
for edge in candidates:
|
||||
remaining = examined_limit - examined_edges
|
||||
values: tuple[object, ...] = (
|
||||
(current, relation, remaining + 1)
|
||||
if relation is not None
|
||||
else (current, remaining + 1)
|
||||
)
|
||||
candidates = snapshot.connection.execute(
|
||||
"SELECT source_id, relation, target_id FROM edges "
|
||||
f"WHERE {source_column} = ?{relation_clause} "
|
||||
"ORDER BY source_id, relation, target_id LIMIT ?",
|
||||
values,
|
||||
).fetchall()
|
||||
if len(candidates) > remaining:
|
||||
truncated = True
|
||||
candidates = candidates[:remaining]
|
||||
examined_edges += len(candidates)
|
||||
for row in candidates:
|
||||
edge = Edge(*row)
|
||||
target = edge.source_id if incoming else edge.target_id
|
||||
if target in seen:
|
||||
continue
|
||||
if len(results) >= bounded:
|
||||
truncated = True
|
||||
break
|
||||
seen.add(target)
|
||||
target_path = (*path, target)
|
||||
results.append(
|
||||
|
|
@ -928,10 +1014,16 @@ class ProjectIndex:
|
|||
}
|
||||
)
|
||||
queue.append((target, current_depth + 1, target_path))
|
||||
if queue and examined_edges >= examined_limit:
|
||||
truncated = True
|
||||
return snapshot.result(
|
||||
root=node_id,
|
||||
depth=depth,
|
||||
count=len(results),
|
||||
limit=bounded,
|
||||
truncated=truncated,
|
||||
examined_edges=examined_edges,
|
||||
examined_edges_limit=examined_limit,
|
||||
results=results,
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue