953 lines
37 KiB
Python
953 lines
37 KiB
Python
|
"""Project-bound, read-only graph visualization over a validated DocForge index."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import atexit
|
||
|
|
import json
|
||
|
|
import secrets
|
||
|
|
import sqlite3
|
||
|
|
import threading
|
||
|
|
import urllib.parse
|
||
|
|
from collections.abc import Generator
|
||
|
|
from contextlib import contextmanager
|
||
|
|
from http import HTTPStatus
|
||
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
|
|
|
||
|
|
from .errors import DocForgeError
|
||
|
|
from .index import APPLICATION_ID, INDEX_SCHEMA_VERSION, ProjectIndex, re_tokenize
|
||
|
|
|
||
|
|
VISUALIZATION_TEMPLATE = "graph-browser@1"
|
||
|
|
DEFAULT_EDGE_LIMIT = 100
|
||
|
|
MAX_EDGE_LIMIT = 400
|
||
|
|
|
||
|
|
|
||
|
|
class _VisualizationHttpServer(ThreadingHTTPServer):
|
||
|
|
daemon_threads = True
|
||
|
|
allow_reuse_address = False
|
||
|
|
|
||
|
|
|
||
|
|
class VisualizationIndexSnapshot:
|
||
|
|
"""Fast read model pinned to one index file validated by ``ProjectIndex.check``."""
|
||
|
|
|
||
|
|
_IDENTITY_KEYS = (
|
||
|
|
"project_id",
|
||
|
|
"project_root_fingerprint",
|
||
|
|
"revision",
|
||
|
|
"source_hash",
|
||
|
|
"adapter",
|
||
|
|
"node_count",
|
||
|
|
"edge_count",
|
||
|
|
)
|
||
|
|
|
||
|
|
def __init__(self, index: ProjectIndex, checked: dict[str, object]) -> None:
|
||
|
|
self.path = index.path
|
||
|
|
self.title = index.project.descriptor.title
|
||
|
|
self.max_query_chars = index.project.descriptor.limits.max_query_chars
|
||
|
|
self.max_results = index.project.descriptor.limits.max_results
|
||
|
|
self.max_depth = index.project.descriptor.limits.max_traversal_depth
|
||
|
|
self.identity = {key: checked[key] for key in self._IDENTITY_KEYS}
|
||
|
|
self._stat = self._safe_stat()
|
||
|
|
|
||
|
|
def overview(self) -> dict[str, object]:
|
||
|
|
with self._connection() as connection:
|
||
|
|
return self._result(
|
||
|
|
title=self.title,
|
||
|
|
node_count=connection.execute("SELECT COUNT(*) FROM nodes").fetchone()[0],
|
||
|
|
edge_count=connection.execute("SELECT COUNT(*) FROM edges").fetchone()[0],
|
||
|
|
families=_facet_rows(connection, "nodes", "family"),
|
||
|
|
authorities=_facet_rows(connection, "nodes", "authority"),
|
||
|
|
statuses=_facet_rows(connection, "nodes", "status"),
|
||
|
|
relations=_facet_rows(connection, "edges", "relation"),
|
||
|
|
max_results=self.max_results,
|
||
|
|
snapshot=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
def search(
|
||
|
|
self,
|
||
|
|
*,
|
||
|
|
query: str,
|
||
|
|
family: str | None,
|
||
|
|
limit: int,
|
||
|
|
) -> dict[str, object]:
|
||
|
|
bounded = self._bounded_limit(limit)
|
||
|
|
with self._connection() as connection:
|
||
|
|
if query:
|
||
|
|
if len(query) > self.max_query_chars:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_query", "Search query exceeds the configured limit"
|
||
|
|
)
|
||
|
|
terms = re_tokenize(query)
|
||
|
|
if not terms:
|
||
|
|
raise DocForgeError("invalid_query", "Search query contains no searchable text")
|
||
|
|
expression = " AND ".join(
|
||
|
|
f'"{term.replace(chr(34), chr(34) * 2)}"' for term in terms
|
||
|
|
)
|
||
|
|
family_clause = "AND nodes.family = ?" if family else ""
|
||
|
|
values: tuple[object, ...] = (
|
||
|
|
(expression, family, bounded) if family else (expression, bounded)
|
||
|
|
)
|
||
|
|
rows = connection.execute(
|
||
|
|
"""
|
||
|
|
SELECT nodes.*, bm25(node_fts) AS rank,
|
||
|
|
snippet(node_fts, 3, '[', ']', ' … ', 18) AS snippet
|
||
|
|
FROM node_fts JOIN nodes USING(node_id)
|
||
|
|
WHERE node_fts MATCH ?
|
||
|
|
"""
|
||
|
|
+ family_clause
|
||
|
|
+ " ORDER BY rank, nodes.node_id LIMIT ?",
|
||
|
|
values,
|
||
|
|
).fetchall()
|
||
|
|
results = []
|
||
|
|
for row in rows:
|
||
|
|
item = _node_dict(row, include_content=False)
|
||
|
|
item.update({"rank": row["rank"], "snippet": row["snippet"]})
|
||
|
|
results.append(item)
|
||
|
|
else:
|
||
|
|
family_clause = "WHERE family = ?" if family else ""
|
||
|
|
values = (family, bounded) if family else (bounded,)
|
||
|
|
rows = connection.execute(
|
||
|
|
f"SELECT * FROM nodes {family_clause} ORDER BY node_id LIMIT ?",
|
||
|
|
values,
|
||
|
|
).fetchall()
|
||
|
|
results = [_node_dict(row, include_content=False) for row in rows]
|
||
|
|
return self._result(
|
||
|
|
query=query,
|
||
|
|
family=family,
|
||
|
|
count=len(results),
|
||
|
|
results=results,
|
||
|
|
snapshot=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
def node(self, node_id: str, *, depth: int, limit: int) -> dict[str, object]:
|
||
|
|
if type(depth) is not int or depth < 1 or depth > self.max_depth:
|
||
|
|
raise DocForgeError("invalid_depth", "Traversal depth is outside the configured limit")
|
||
|
|
if type(limit) is not int or limit < 1 or limit > MAX_EDGE_LIMIT:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_limit",
|
||
|
|
"Visualization edge limit exceeds the fixed safety boundary",
|
||
|
|
maximum=MAX_EDGE_LIMIT,
|
||
|
|
)
|
||
|
|
with self._connection() as connection:
|
||
|
|
root_row = connection.execute(
|
||
|
|
"SELECT * FROM nodes WHERE node_id = ?", (node_id,)
|
||
|
|
).fetchone()
|
||
|
|
if root_row is None:
|
||
|
|
raise DocForgeError(
|
||
|
|
"missing_node",
|
||
|
|
"No node has the requested stable ID",
|
||
|
|
node_id=node_id,
|
||
|
|
)
|
||
|
|
visited = {node_id}
|
||
|
|
frontier = {node_id}
|
||
|
|
selected: list[dict[str, str]] = []
|
||
|
|
selected_keys: set[tuple[str, str, str]] = set()
|
||
|
|
truncated = False
|
||
|
|
for _ in range(depth):
|
||
|
|
if not frontier or len(selected) >= limit:
|
||
|
|
break
|
||
|
|
placeholders = ",".join("?" for _ in frontier)
|
||
|
|
values = tuple(sorted(frontier))
|
||
|
|
rows = connection.execute(
|
||
|
|
"SELECT source_id, relation, target_id FROM edges "
|
||
|
|
f"WHERE source_id IN ({placeholders}) OR target_id IN ({placeholders}) "
|
||
|
|
"ORDER BY source_id, relation, target_id LIMIT ?",
|
||
|
|
(*values, *values, limit + 1),
|
||
|
|
).fetchall()
|
||
|
|
next_frontier: set[str] = set()
|
||
|
|
for row in rows:
|
||
|
|
key = (row["source_id"], row["relation"], row["target_id"])
|
||
|
|
if key in selected_keys:
|
||
|
|
continue
|
||
|
|
if len(selected) >= limit:
|
||
|
|
truncated = True
|
||
|
|
break
|
||
|
|
selected_keys.add(key)
|
||
|
|
edge = {
|
||
|
|
"source_id": row["source_id"],
|
||
|
|
"relation": row["relation"],
|
||
|
|
"target_id": row["target_id"],
|
||
|
|
}
|
||
|
|
selected.append(edge)
|
||
|
|
for candidate in (edge["source_id"], edge["target_id"]):
|
||
|
|
if candidate not in visited:
|
||
|
|
visited.add(candidate)
|
||
|
|
next_frontier.add(candidate)
|
||
|
|
frontier = next_frontier
|
||
|
|
placeholders = ",".join("?" for _ in visited)
|
||
|
|
node_rows = connection.execute(
|
||
|
|
f"SELECT * FROM nodes WHERE node_id IN ({placeholders}) ORDER BY node_id",
|
||
|
|
tuple(sorted(visited)),
|
||
|
|
).fetchall()
|
||
|
|
return self._result(
|
||
|
|
root=node_id,
|
||
|
|
depth=depth,
|
||
|
|
edge_limit=limit,
|
||
|
|
truncated=truncated,
|
||
|
|
node=_node_dict(root_row),
|
||
|
|
nodes=[_node_dict(row, include_content=False) for row in node_rows],
|
||
|
|
edges=selected,
|
||
|
|
snapshot=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
def require_node(self, node_id: str) -> None:
|
||
|
|
with self._connection() as connection:
|
||
|
|
row = connection.execute("SELECT 1 FROM nodes WHERE node_id = ?", (node_id,)).fetchone()
|
||
|
|
if row is None:
|
||
|
|
raise DocForgeError(
|
||
|
|
"missing_node",
|
||
|
|
"No node has the requested stable ID",
|
||
|
|
node_id=node_id,
|
||
|
|
)
|
||
|
|
|
||
|
|
def _bounded_limit(self, value: int) -> int:
|
||
|
|
if type(value) is not int or value < 1 or value > self.max_results:
|
||
|
|
raise DocForgeError("invalid_limit", "Result limit is outside the configured range")
|
||
|
|
return value
|
||
|
|
|
||
|
|
def _safe_stat(self) -> tuple[int, int, int, int]:
|
||
|
|
if (
|
||
|
|
self.path.is_symlink()
|
||
|
|
or not self.path.is_file()
|
||
|
|
or self.path.resolve(strict=True) != self.path
|
||
|
|
):
|
||
|
|
raise DocForgeError("missing_index", "Validated visualization index is unavailable")
|
||
|
|
stat = self.path.stat()
|
||
|
|
return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns)
|
||
|
|
|
||
|
|
@contextmanager
|
||
|
|
def _connection(self) -> Generator[sqlite3.Connection, None, None]:
|
||
|
|
if self._safe_stat() != self._stat:
|
||
|
|
raise DocForgeError(
|
||
|
|
"visualization_stale",
|
||
|
|
"The validated index changed; invoke docforge_visualize again",
|
||
|
|
)
|
||
|
|
connection: sqlite3.Connection | None = None
|
||
|
|
try:
|
||
|
|
connection = sqlite3.connect(f"file:{self.path}?mode=ro", uri=True)
|
||
|
|
connection.row_factory = sqlite3.Row
|
||
|
|
application_id = connection.execute("PRAGMA application_id").fetchone()[0]
|
||
|
|
schema_version = connection.execute("PRAGMA user_version").fetchone()[0]
|
||
|
|
if application_id != APPLICATION_ID or schema_version != INDEX_SCHEMA_VERSION:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_index", "Visualization index has an unsupported schema"
|
||
|
|
)
|
||
|
|
metadata = dict(connection.execute("SELECT key, value FROM metadata"))
|
||
|
|
if any(metadata.get(key) != str(value) for key, value in self.identity.items()):
|
||
|
|
raise DocForgeError(
|
||
|
|
"visualization_stale",
|
||
|
|
"The validated index identity changed; invoke docforge_visualize again",
|
||
|
|
)
|
||
|
|
yield connection
|
||
|
|
if self._safe_stat() != self._stat:
|
||
|
|
raise DocForgeError(
|
||
|
|
"visualization_stale",
|
||
|
|
"The validated index changed during the request",
|
||
|
|
)
|
||
|
|
except DocForgeError:
|
||
|
|
raise
|
||
|
|
except (json.JSONDecodeError, KeyError, sqlite3.Error, TypeError) as error:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_index", "Visualization index is corrupt or unreadable"
|
||
|
|
) from error
|
||
|
|
finally:
|
||
|
|
if connection is not None:
|
||
|
|
connection.close()
|
||
|
|
|
||
|
|
def _result(self, **payload: object) -> dict[str, object]:
|
||
|
|
return {
|
||
|
|
"status": "ok",
|
||
|
|
**self.identity,
|
||
|
|
**payload,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
class VisualizationRunner:
|
||
|
|
"""Start one token-protected loopback reader for one immutable project binding."""
|
||
|
|
|
||
|
|
def __init__(self, index: ProjectIndex) -> None:
|
||
|
|
self.index = index
|
||
|
|
self._lock = threading.Lock()
|
||
|
|
self._token = secrets.token_urlsafe(24)
|
||
|
|
self._server: _VisualizationHttpServer | None = None
|
||
|
|
self._thread: threading.Thread | None = None
|
||
|
|
self._atexit_registered = False
|
||
|
|
self._reader: VisualizationIndexSnapshot | None = None
|
||
|
|
|
||
|
|
def start(
|
||
|
|
self,
|
||
|
|
*,
|
||
|
|
node_id: str | None = None,
|
||
|
|
query: str | None = None,
|
||
|
|
depth: int = 1,
|
||
|
|
) -> dict[str, object]:
|
||
|
|
if node_id is not None and query is not None:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_visualization_target",
|
||
|
|
"Choose either one exact node ID or one search query",
|
||
|
|
)
|
||
|
|
maximum_depth = self.index.project.descriptor.limits.max_traversal_depth
|
||
|
|
if type(depth) is not int or depth < 1 or depth > maximum_depth:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_depth",
|
||
|
|
"Visualization depth is outside the configured traversal limit",
|
||
|
|
)
|
||
|
|
reader = VisualizationIndexSnapshot(self.index, self.index.check())
|
||
|
|
if node_id is not None:
|
||
|
|
reader.require_node(node_id)
|
||
|
|
elif query is not None:
|
||
|
|
reader.search(query=query, family=None, limit=1)
|
||
|
|
|
||
|
|
with self._lock:
|
||
|
|
self._reader = reader
|
||
|
|
if self._server is None:
|
||
|
|
runner = self
|
||
|
|
|
||
|
|
class Handler(BaseHTTPRequestHandler):
|
||
|
|
def do_GET(self) -> None: # noqa: N802
|
||
|
|
runner._handle_get(self)
|
||
|
|
|
||
|
|
def do_HEAD(self) -> None: # noqa: N802
|
||
|
|
runner._handle_get(self, include_body=False)
|
||
|
|
|
||
|
|
def do_POST(self) -> None: # noqa: N802
|
||
|
|
runner._respond_error(
|
||
|
|
self,
|
||
|
|
HTTPStatus.METHOD_NOT_ALLOWED,
|
||
|
|
"method_not_allowed",
|
||
|
|
"The visualization service is read-only",
|
||
|
|
)
|
||
|
|
|
||
|
|
def do_PUT(self) -> None: # noqa: N802
|
||
|
|
self.do_POST()
|
||
|
|
|
||
|
|
def do_PATCH(self) -> None: # noqa: N802
|
||
|
|
self.do_POST()
|
||
|
|
|
||
|
|
def do_DELETE(self) -> None: # noqa: N802
|
||
|
|
self.do_POST()
|
||
|
|
|
||
|
|
def log_message(self, _format: str, *_args: object) -> None:
|
||
|
|
return
|
||
|
|
|
||
|
|
self._server = _VisualizationHttpServer(("127.0.0.1", 0), Handler)
|
||
|
|
self._thread = threading.Thread(
|
||
|
|
target=self._server.serve_forever,
|
||
|
|
name="docforge-visualization",
|
||
|
|
daemon=True,
|
||
|
|
)
|
||
|
|
self._thread.start()
|
||
|
|
if not self._atexit_registered:
|
||
|
|
atexit.register(self.stop)
|
||
|
|
self._atexit_registered = True
|
||
|
|
|
||
|
|
server = self._server
|
||
|
|
assert server is not None
|
||
|
|
port = int(server.server_address[1])
|
||
|
|
|
||
|
|
parameters: dict[str, str] = {"depth": str(depth)}
|
||
|
|
if node_id is not None:
|
||
|
|
parameters["node"] = node_id
|
||
|
|
if query is not None:
|
||
|
|
parameters["q"] = query
|
||
|
|
query_string = urllib.parse.urlencode(parameters)
|
||
|
|
url = f"http://127.0.0.1:{port}/{self._token}/"
|
||
|
|
if query_string:
|
||
|
|
url = f"{url}?{query_string}"
|
||
|
|
return {
|
||
|
|
"state": "running",
|
||
|
|
"url": url,
|
||
|
|
"bind": "127.0.0.1",
|
||
|
|
"port": port,
|
||
|
|
"template": VISUALIZATION_TEMPLATE,
|
||
|
|
"read_only": True,
|
||
|
|
"project_bound": True,
|
||
|
|
"target": {
|
||
|
|
"node_id": node_id,
|
||
|
|
"query": query,
|
||
|
|
"depth": depth,
|
||
|
|
},
|
||
|
|
"snapshot": dict(reader.identity),
|
||
|
|
}
|
||
|
|
|
||
|
|
def stop(self) -> None:
|
||
|
|
with self._lock:
|
||
|
|
server = self._server
|
||
|
|
thread = self._thread
|
||
|
|
self._server = None
|
||
|
|
self._thread = None
|
||
|
|
self._reader = None
|
||
|
|
if server is None:
|
||
|
|
return
|
||
|
|
server.shutdown()
|
||
|
|
server.server_close()
|
||
|
|
if thread is not None and thread is not threading.current_thread():
|
||
|
|
thread.join(timeout=2)
|
||
|
|
|
||
|
|
def _handle_get(self, handler: BaseHTTPRequestHandler, *, include_body: bool = True) -> None:
|
||
|
|
parsed = urllib.parse.urlparse(handler.path)
|
||
|
|
prefix = f"/{self._token}"
|
||
|
|
if parsed.path not in {prefix, f"{prefix}/"} and not parsed.path.startswith(
|
||
|
|
f"{prefix}/api/"
|
||
|
|
):
|
||
|
|
self._respond_error(
|
||
|
|
handler,
|
||
|
|
HTTPStatus.NOT_FOUND,
|
||
|
|
"not_found",
|
||
|
|
"Not found",
|
||
|
|
include_body=include_body,
|
||
|
|
)
|
||
|
|
return
|
||
|
|
if parsed.path in {prefix, f"{prefix}/"}:
|
||
|
|
self._respond(
|
||
|
|
handler,
|
||
|
|
_GRAPH_BROWSER_HTML.encode("utf-8"),
|
||
|
|
"text/html; charset=utf-8",
|
||
|
|
include_body=include_body,
|
||
|
|
)
|
||
|
|
return
|
||
|
|
|
||
|
|
try:
|
||
|
|
params = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
|
||
|
|
reader = self._current_reader()
|
||
|
|
if parsed.path == f"{prefix}/api/overview":
|
||
|
|
payload = reader.overview()
|
||
|
|
elif parsed.path == f"{prefix}/api/search":
|
||
|
|
payload = self._search(reader, params)
|
||
|
|
elif parsed.path == f"{prefix}/api/node":
|
||
|
|
payload = self._node(reader, params)
|
||
|
|
else:
|
||
|
|
self._respond_error(
|
||
|
|
handler,
|
||
|
|
HTTPStatus.NOT_FOUND,
|
||
|
|
"not_found",
|
||
|
|
"Not found",
|
||
|
|
include_body=include_body,
|
||
|
|
)
|
||
|
|
return
|
||
|
|
except DocForgeError as error:
|
||
|
|
status = {
|
||
|
|
"missing_node": HTTPStatus.NOT_FOUND,
|
||
|
|
"stale_index": HTTPStatus.CONFLICT,
|
||
|
|
"source_changed": HTTPStatus.CONFLICT,
|
||
|
|
"stale_adapter_source": HTTPStatus.CONFLICT,
|
||
|
|
"visualization_stale": HTTPStatus.CONFLICT,
|
||
|
|
"invalid_query": HTTPStatus.BAD_REQUEST,
|
||
|
|
"invalid_depth": HTTPStatus.BAD_REQUEST,
|
||
|
|
"invalid_limit": HTTPStatus.BAD_REQUEST,
|
||
|
|
}.get(error.code, HTTPStatus.SERVICE_UNAVAILABLE)
|
||
|
|
self._respond_json(
|
||
|
|
handler,
|
||
|
|
{"status": "error", "error": error.as_dict()},
|
||
|
|
status=status,
|
||
|
|
include_body=include_body,
|
||
|
|
)
|
||
|
|
return
|
||
|
|
except (TypeError, ValueError):
|
||
|
|
self._respond_error(
|
||
|
|
handler,
|
||
|
|
HTTPStatus.BAD_REQUEST,
|
||
|
|
"invalid_request",
|
||
|
|
"Request parameters are invalid",
|
||
|
|
include_body=include_body,
|
||
|
|
)
|
||
|
|
return
|
||
|
|
self._respond_json(handler, payload, include_body=include_body)
|
||
|
|
|
||
|
|
def _search(
|
||
|
|
self,
|
||
|
|
reader: VisualizationIndexSnapshot,
|
||
|
|
params: dict[str, list[str]],
|
||
|
|
) -> dict[str, object]:
|
||
|
|
query = _one(params, "q").strip()
|
||
|
|
family = _one(params, "family").strip() or None
|
||
|
|
limit = _integer(_one(params, "limit") or "50")
|
||
|
|
return reader.search(query=query, family=family, limit=limit)
|
||
|
|
|
||
|
|
def _node(
|
||
|
|
self,
|
||
|
|
reader: VisualizationIndexSnapshot,
|
||
|
|
params: dict[str, list[str]],
|
||
|
|
) -> dict[str, object]:
|
||
|
|
node_id = _one(params, "id").strip()
|
||
|
|
if not node_id:
|
||
|
|
raise DocForgeError("missing_node", "One exact node ID is required")
|
||
|
|
depth = _integer(_one(params, "depth") or "1")
|
||
|
|
limit = _integer(_one(params, "limit") or str(DEFAULT_EDGE_LIMIT))
|
||
|
|
if limit > MAX_EDGE_LIMIT:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_limit",
|
||
|
|
"Visualization edge limit exceeds the fixed safety boundary",
|
||
|
|
maximum=MAX_EDGE_LIMIT,
|
||
|
|
)
|
||
|
|
return reader.node(node_id, depth=depth, limit=limit)
|
||
|
|
|
||
|
|
def _current_reader(self) -> VisualizationIndexSnapshot:
|
||
|
|
with self._lock:
|
||
|
|
reader = self._reader
|
||
|
|
if reader is None:
|
||
|
|
raise DocForgeError(
|
||
|
|
"visualization_unavailable",
|
||
|
|
"The visualization snapshot is unavailable",
|
||
|
|
)
|
||
|
|
return reader
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _respond(
|
||
|
|
handler: BaseHTTPRequestHandler,
|
||
|
|
payload: bytes,
|
||
|
|
content_type: str,
|
||
|
|
*,
|
||
|
|
status: HTTPStatus = HTTPStatus.OK,
|
||
|
|
include_body: bool = True,
|
||
|
|
) -> None:
|
||
|
|
handler.send_response(status)
|
||
|
|
handler.send_header("Content-Type", content_type)
|
||
|
|
handler.send_header("Content-Length", str(len(payload)))
|
||
|
|
handler.send_header("Cache-Control", "no-store")
|
||
|
|
handler.send_header(
|
||
|
|
"Content-Security-Policy",
|
||
|
|
"default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; "
|
||
|
|
"connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; "
|
||
|
|
"frame-ancestors 'none'",
|
||
|
|
)
|
||
|
|
handler.send_header("X-Content-Type-Options", "nosniff")
|
||
|
|
handler.send_header("X-Frame-Options", "DENY")
|
||
|
|
handler.send_header("Referrer-Policy", "no-referrer")
|
||
|
|
handler.end_headers()
|
||
|
|
if include_body:
|
||
|
|
handler.wfile.write(payload)
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def _respond_json(
|
||
|
|
cls,
|
||
|
|
handler: BaseHTTPRequestHandler,
|
||
|
|
body: dict[str, object],
|
||
|
|
*,
|
||
|
|
status: HTTPStatus = HTTPStatus.OK,
|
||
|
|
include_body: bool = True,
|
||
|
|
) -> None:
|
||
|
|
cls._respond(
|
||
|
|
handler,
|
||
|
|
json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8"),
|
||
|
|
"application/json; charset=utf-8",
|
||
|
|
status=status,
|
||
|
|
include_body=include_body,
|
||
|
|
)
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def _respond_error(
|
||
|
|
cls,
|
||
|
|
handler: BaseHTTPRequestHandler,
|
||
|
|
status: HTTPStatus,
|
||
|
|
code: str,
|
||
|
|
message: str,
|
||
|
|
*,
|
||
|
|
include_body: bool = True,
|
||
|
|
) -> None:
|
||
|
|
cls._respond_json(
|
||
|
|
handler,
|
||
|
|
{
|
||
|
|
"status": "error",
|
||
|
|
"error": {"code": code, "message": message, "details": {}},
|
||
|
|
},
|
||
|
|
status=status,
|
||
|
|
include_body=include_body,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _one(params: dict[str, list[str]], name: str) -> str:
|
||
|
|
values = params.get(name) or [""]
|
||
|
|
return values[0]
|
||
|
|
|
||
|
|
|
||
|
|
def _integer(value: str) -> int:
|
||
|
|
return int(value)
|
||
|
|
|
||
|
|
|
||
|
|
def _node_dict(row: sqlite3.Row, *, include_content: bool = True) -> dict[str, object]:
|
||
|
|
result = {
|
||
|
|
"node_id": row["node_id"],
|
||
|
|
"title": row["title"],
|
||
|
|
"family": row["family"],
|
||
|
|
"authority": row["authority"],
|
||
|
|
"status": row["status"],
|
||
|
|
"tags": json.loads(row["tags_json"]),
|
||
|
|
"summary": row["summary"],
|
||
|
|
"source_path": row["source_path"],
|
||
|
|
"source_anchor": row["source_anchor"],
|
||
|
|
"content_hash": row["content_hash"],
|
||
|
|
}
|
||
|
|
if include_content:
|
||
|
|
result["content"] = row["content"]
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _facet_rows(connection: sqlite3.Connection, table: str, column: str) -> list[dict[str, object]]:
|
||
|
|
allowed = {
|
||
|
|
("nodes", "family"),
|
||
|
|
("nodes", "authority"),
|
||
|
|
("nodes", "status"),
|
||
|
|
("edges", "relation"),
|
||
|
|
}
|
||
|
|
if (table, column) not in allowed:
|
||
|
|
raise DocForgeError("invalid_index", "Unsupported visualization facet")
|
||
|
|
rows = connection.execute(
|
||
|
|
f"SELECT {column}, COUNT(*) AS count FROM {table} "
|
||
|
|
f"GROUP BY {column} ORDER BY count DESC, {column}"
|
||
|
|
).fetchall()
|
||
|
|
return [{"value": row[0], "count": row[1]} for row in rows]
|
||
|
|
|
||
|
|
|
||
|
|
_GRAPH_BROWSER_HTML = r"""<!doctype html>
|
||
|
|
<html lang="en">
|
||
|
|
<head>
|
||
|
|
<meta charset="utf-8">
|
||
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
|
|
<title>DocForge graph</title>
|
||
|
|
<style>
|
||
|
|
:root {
|
||
|
|
color-scheme: dark;
|
||
|
|
--bg: #07101a;
|
||
|
|
--panel: #0c1825;
|
||
|
|
--panel-2: #111f2f;
|
||
|
|
--line: #263b51;
|
||
|
|
--text: #ecf5ff;
|
||
|
|
--muted: #93abc3;
|
||
|
|
--accent: #51d7ff;
|
||
|
|
--accent-2: #8fffc5;
|
||
|
|
--warn: #ffd27a;
|
||
|
|
font: 14px/1.45 Inter, ui-sans-serif, system-ui, sans-serif;
|
||
|
|
}
|
||
|
|
* { box-sizing: border-box; }
|
||
|
|
body { margin: 0; min-height: 100vh; background: var(--bg); color: var(--text); }
|
||
|
|
button, input, select { font: inherit; }
|
||
|
|
button { cursor: pointer; }
|
||
|
|
code, pre { font-family: ui-monospace, SFMono-Regular, Consolas, monospace; }
|
||
|
|
.app { display: grid; grid-template-rows: auto 1fr; min-height: 100vh; }
|
||
|
|
header {
|
||
|
|
display: flex; gap: 18px; align-items: center; padding: 14px 18px;
|
||
|
|
border-bottom: 1px solid var(--line); background: rgba(8, 18, 30, .96);
|
||
|
|
}
|
||
|
|
header h1 { margin: 0; font-size: 17px; }
|
||
|
|
.stats { display: flex; gap: 14px; color: var(--muted); }
|
||
|
|
.status { margin-left: auto; color: var(--muted); }
|
||
|
|
.layout { min-height: 0; display: grid; grid-template-columns: 300px minmax(360px, 1fr) 340px; }
|
||
|
|
aside { min-height: 0; overflow: auto; padding: 16px; background: var(--panel); }
|
||
|
|
.left { border-right: 1px solid var(--line); }
|
||
|
|
.right { border-left: 1px solid var(--line); }
|
||
|
|
form { display: grid; gap: 8px; }
|
||
|
|
input, select {
|
||
|
|
width: 100%; border: 1px solid var(--line); border-radius: 8px;
|
||
|
|
padding: 9px 10px; background: var(--panel-2); color: var(--text);
|
||
|
|
}
|
||
|
|
.search-row { display: grid; grid-template-columns: 1fr auto; gap: 8px; }
|
||
|
|
.button {
|
||
|
|
border: 1px solid #277fa0; border-radius: 8px; padding: 8px 12px;
|
||
|
|
background: #12384a; color: var(--text);
|
||
|
|
}
|
||
|
|
.results { display: grid; gap: 7px; margin-top: 14px; }
|
||
|
|
.result {
|
||
|
|
width: 100%; text-align: left; border: 1px solid var(--line); border-radius: 9px;
|
||
|
|
padding: 9px; background: var(--panel-2); color: var(--text);
|
||
|
|
}
|
||
|
|
.result:hover, .result:focus { border-color: var(--accent); }
|
||
|
|
.result strong, .result span { display: block; overflow: hidden; text-overflow: ellipsis; }
|
||
|
|
.result span { color: var(--muted); font-size: 12px; white-space: nowrap; }
|
||
|
|
.canvas { position: relative; min-height: 0; overflow: hidden; }
|
||
|
|
svg { width: 100%; height: 100%; min-height: 620px; background:
|
||
|
|
radial-gradient(circle at center, #10243a 0, #07101a 64%); }
|
||
|
|
.edge { stroke: #476177; stroke-opacity: .56; stroke-width: 1.2; }
|
||
|
|
.edge-label { fill: #8198ae; font-size: 9px; pointer-events: none; }
|
||
|
|
.node circle { fill: #17344a; stroke: #70b9d4; stroke-width: 1.5; }
|
||
|
|
.node.root circle { fill: #19566a; stroke: var(--accent-2); stroke-width: 3; }
|
||
|
|
.node:hover circle { stroke: #fff; stroke-width: 3; }
|
||
|
|
.node text { fill: var(--text); font-size: 10px; pointer-events: none; }
|
||
|
|
.node .family { fill: var(--muted); font-size: 8px; }
|
||
|
|
.empty {
|
||
|
|
position: absolute; inset: 0; display: grid; place-items: center;
|
||
|
|
color: var(--muted); pointer-events: none;
|
||
|
|
}
|
||
|
|
.detail-head { display: flex; align-items: start; gap: 10px; }
|
||
|
|
.detail-head h2 { margin: 0; font-size: 18px; overflow-wrap: anywhere; }
|
||
|
|
.badge {
|
||
|
|
display: inline-block; margin: 4px 5px 0 0; border: 1px solid var(--line);
|
||
|
|
border-radius: 999px; padding: 3px 7px; color: var(--muted); font-size: 11px;
|
||
|
|
}
|
||
|
|
.meta { display: grid; gap: 8px; margin: 14px 0; }
|
||
|
|
.meta div { display: grid; grid-template-columns: 82px 1fr; gap: 8px; }
|
||
|
|
.meta dt { color: var(--muted); }
|
||
|
|
.meta dd { margin: 0; overflow-wrap: anywhere; }
|
||
|
|
.summary { color: #c9d9e8; }
|
||
|
|
pre {
|
||
|
|
white-space: pre-wrap; overflow-wrap: anywhere; max-height: 44vh; overflow: auto;
|
||
|
|
padding: 12px; border: 1px solid var(--line); border-radius: 9px;
|
||
|
|
background: #07111c; color: #d6e6f5;
|
||
|
|
}
|
||
|
|
.error { color: #ff9aac; }
|
||
|
|
@media (max-width: 980px) {
|
||
|
|
.layout { grid-template-columns: 240px 1fr; }
|
||
|
|
.right { grid-column: 1 / -1; border-left: 0; border-top: 1px solid var(--line); }
|
||
|
|
}
|
||
|
|
@media (max-width: 700px) {
|
||
|
|
.layout { display: block; }
|
||
|
|
aside { max-height: none; }
|
||
|
|
svg { min-height: 520px; }
|
||
|
|
header { flex-wrap: wrap; }
|
||
|
|
.status { width: 100%; margin-left: 0; }
|
||
|
|
}
|
||
|
|
</style>
|
||
|
|
</head>
|
||
|
|
<body>
|
||
|
|
<div class="app">
|
||
|
|
<header>
|
||
|
|
<h1 id="project-title">DocForge graph</h1>
|
||
|
|
<div class="stats">
|
||
|
|
<span><strong id="node-count">—</strong> nodes</span>
|
||
|
|
<span><strong id="edge-count">—</strong> edges</span>
|
||
|
|
</div>
|
||
|
|
<div class="status" id="status">Loading validated index snapshot…</div>
|
||
|
|
</header>
|
||
|
|
<div class="layout">
|
||
|
|
<aside class="left">
|
||
|
|
<form id="search-form">
|
||
|
|
<label for="search">Find nodes</label>
|
||
|
|
<div class="search-row">
|
||
|
|
<input id="search" name="q" autocomplete="off" placeholder="title, symbol, path…">
|
||
|
|
<button class="button" type="submit">Find</button>
|
||
|
|
</div>
|
||
|
|
<label for="family">Family</label>
|
||
|
|
<select id="family" name="family"><option value="">All families</option></select>
|
||
|
|
</form>
|
||
|
|
<div class="results" id="results"></div>
|
||
|
|
</aside>
|
||
|
|
<main class="canvas">
|
||
|
|
<svg id="graph" viewBox="-600 -410 1200 820"
|
||
|
|
role="img" aria-label="Node neighborhood"></svg>
|
||
|
|
<div class="empty" id="empty">Search for a node to inspect its neighborhood.</div>
|
||
|
|
</main>
|
||
|
|
<aside class="right">
|
||
|
|
<div id="details"><p class="summary">Choose a search result or graph node.</p></div>
|
||
|
|
</aside>
|
||
|
|
</div>
|
||
|
|
</div>
|
||
|
|
<script>
|
||
|
|
const base = location.pathname.replace(/\/?$/, "/");
|
||
|
|
const state = { overview: null, graph: null, root: null, depth: 1, searchLimit: 1 };
|
||
|
|
const $ = (id) => document.getElementById(id);
|
||
|
|
const api = async (path) => {
|
||
|
|
const response = await fetch(`${base}api/${path}`, {cache: "no-store"});
|
||
|
|
const body = await response.json();
|
||
|
|
if (!response.ok || body.status === "error") {
|
||
|
|
throw new Error(body.error?.message || "DocForge request failed");
|
||
|
|
}
|
||
|
|
return body;
|
||
|
|
};
|
||
|
|
const escapeText = (value) => String(value ?? "");
|
||
|
|
const short = (value, length = 34) => {
|
||
|
|
const text = escapeText(value);
|
||
|
|
return text.length > length ? `${text.slice(0, length - 1)}…` : text;
|
||
|
|
};
|
||
|
|
function setStatus(message, error = false) {
|
||
|
|
$("status").textContent = message;
|
||
|
|
$("status").classList.toggle("error", error);
|
||
|
|
}
|
||
|
|
function renderOverview(data) {
|
||
|
|
state.overview = data;
|
||
|
|
state.searchLimit = Math.max(1, Number(data.max_results) || 1);
|
||
|
|
$("project-title").textContent = data.title || data.project_id;
|
||
|
|
$("node-count").textContent = Number(data.node_count).toLocaleString();
|
||
|
|
$("edge-count").textContent = Number(data.edge_count).toLocaleString();
|
||
|
|
const family = $("family");
|
||
|
|
for (const item of data.families) {
|
||
|
|
const option = document.createElement("option");
|
||
|
|
option.value = item.value;
|
||
|
|
option.textContent = `${item.value} (${item.count})`;
|
||
|
|
family.append(option);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
function renderResults(items) {
|
||
|
|
const results = $("results");
|
||
|
|
results.replaceChildren();
|
||
|
|
if (!items.length) {
|
||
|
|
const note = document.createElement("p");
|
||
|
|
note.className = "summary";
|
||
|
|
note.textContent = "No matching nodes.";
|
||
|
|
results.append(note);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
for (const item of items) {
|
||
|
|
const button = document.createElement("button");
|
||
|
|
button.type = "button";
|
||
|
|
button.className = "result";
|
||
|
|
const title = document.createElement("strong");
|
||
|
|
title.textContent = item.title;
|
||
|
|
const id = document.createElement("span");
|
||
|
|
id.textContent = item.node_id;
|
||
|
|
const family = document.createElement("span");
|
||
|
|
family.textContent = `${item.family} · ${item.source_path}`;
|
||
|
|
button.append(title, id, family);
|
||
|
|
button.addEventListener("click", () => loadNode(item.node_id));
|
||
|
|
results.append(button);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
function layoutNodes(nodes, rootId) {
|
||
|
|
const ordered = [...nodes].sort((a, b) => a.node_id.localeCompare(b.node_id));
|
||
|
|
const root = ordered.find((node) => node.node_id === rootId);
|
||
|
|
const others = ordered.filter((node) => node.node_id !== rootId);
|
||
|
|
const positions = new Map();
|
||
|
|
if (root) positions.set(root.node_id, {x: 0, y: 0});
|
||
|
|
others.forEach((node, index) => {
|
||
|
|
const ring = Math.floor(index / 18) + 1;
|
||
|
|
const ringStart = (ring - 1) * 18;
|
||
|
|
const ringCount = Math.min(18, others.length - ringStart);
|
||
|
|
const angle = ((index - ringStart) / Math.max(1, ringCount)) * Math.PI * 2 - Math.PI / 2;
|
||
|
|
const radius = 165 + (ring - 1) * 145;
|
||
|
|
positions.set(node.node_id, {x: Math.cos(angle) * radius, y: Math.sin(angle) * radius});
|
||
|
|
});
|
||
|
|
return positions;
|
||
|
|
}
|
||
|
|
function svgElement(name, attributes = {}) {
|
||
|
|
const element = document.createElementNS("http://www.w3.org/2000/svg", name);
|
||
|
|
for (const [key, value] of Object.entries(attributes)) element.setAttribute(key, value);
|
||
|
|
return element;
|
||
|
|
}
|
||
|
|
function renderGraph(data) {
|
||
|
|
state.graph = data;
|
||
|
|
state.root = data.root;
|
||
|
|
const svg = $("graph");
|
||
|
|
svg.replaceChildren();
|
||
|
|
$("empty").hidden = data.nodes.length > 0;
|
||
|
|
const positions = layoutNodes(data.nodes, data.root);
|
||
|
|
const edgeLayer = svgElement("g");
|
||
|
|
const nodeLayer = svgElement("g");
|
||
|
|
for (const edge of data.edges) {
|
||
|
|
const source = positions.get(edge.source_id);
|
||
|
|
const target = positions.get(edge.target_id);
|
||
|
|
if (!source || !target) continue;
|
||
|
|
edgeLayer.append(svgElement("line", {
|
||
|
|
x1: source.x, y1: source.y, x2: target.x, y2: target.y, class: "edge"
|
||
|
|
}));
|
||
|
|
const label = svgElement("text", {
|
||
|
|
x: (source.x + target.x) / 2,
|
||
|
|
y: (source.y + target.y) / 2,
|
||
|
|
class: "edge-label",
|
||
|
|
"text-anchor": "middle"
|
||
|
|
});
|
||
|
|
label.textContent = edge.relation;
|
||
|
|
edgeLayer.append(label);
|
||
|
|
}
|
||
|
|
for (const node of data.nodes) {
|
||
|
|
const point = positions.get(node.node_id);
|
||
|
|
if (!point) continue;
|
||
|
|
const group = svgElement("g", {
|
||
|
|
class: `node${node.node_id === data.root ? " root" : ""}`,
|
||
|
|
transform: `translate(${point.x} ${point.y})`,
|
||
|
|
tabindex: "0",
|
||
|
|
role: "button",
|
||
|
|
"aria-label": `${node.title}, ${node.family}`
|
||
|
|
});
|
||
|
|
group.append(svgElement("circle", {r: node.node_id === data.root ? 25 : 18}));
|
||
|
|
const title = svgElement("text", {y: 35, "text-anchor": "middle"});
|
||
|
|
title.textContent = short(node.title, 26);
|
||
|
|
const family = svgElement("text", {y: 48, "text-anchor": "middle", class: "family"});
|
||
|
|
family.textContent = short(node.family, 22);
|
||
|
|
group.append(title, family);
|
||
|
|
group.addEventListener("click", () => loadNode(node.node_id));
|
||
|
|
group.addEventListener("keydown", (event) => {
|
||
|
|
if (event.key === "Enter" || event.key === " ") loadNode(node.node_id);
|
||
|
|
});
|
||
|
|
nodeLayer.append(group);
|
||
|
|
}
|
||
|
|
svg.append(edgeLayer, nodeLayer);
|
||
|
|
}
|
||
|
|
function renderDetails(node, data) {
|
||
|
|
const details = $("details");
|
||
|
|
details.replaceChildren();
|
||
|
|
const heading = document.createElement("div");
|
||
|
|
heading.className = "detail-head";
|
||
|
|
const title = document.createElement("h2");
|
||
|
|
title.textContent = node.title;
|
||
|
|
heading.append(title);
|
||
|
|
const badges = document.createElement("div");
|
||
|
|
for (const value of [node.family, node.authority, node.status, ...node.tags]) {
|
||
|
|
const badge = document.createElement("span");
|
||
|
|
badge.className = "badge";
|
||
|
|
badge.textContent = value;
|
||
|
|
badges.append(badge);
|
||
|
|
}
|
||
|
|
const summary = document.createElement("p");
|
||
|
|
summary.className = "summary";
|
||
|
|
summary.textContent = node.summary;
|
||
|
|
const dl = document.createElement("dl");
|
||
|
|
dl.className = "meta";
|
||
|
|
for (const [label, value] of [
|
||
|
|
["ID", node.node_id],
|
||
|
|
["Source", node.source_path],
|
||
|
|
["Anchor", node.source_anchor || "—"],
|
||
|
|
["Neighbors", `${data.nodes.length - 1} nodes · ${data.edges.length} edges`],
|
||
|
|
]) {
|
||
|
|
const row = document.createElement("div");
|
||
|
|
const dt = document.createElement("dt");
|
||
|
|
const dd = document.createElement("dd");
|
||
|
|
dt.textContent = label;
|
||
|
|
dd.textContent = escapeText(value);
|
||
|
|
row.append(dt, dd);
|
||
|
|
dl.append(row);
|
||
|
|
}
|
||
|
|
const content = document.createElement("pre");
|
||
|
|
content.textContent = node.content;
|
||
|
|
details.append(heading, badges, summary, dl, content);
|
||
|
|
}
|
||
|
|
async function search() {
|
||
|
|
const params = new URLSearchParams({
|
||
|
|
q: $("search").value.trim(),
|
||
|
|
family: $("family").value,
|
||
|
|
limit: String(state.searchLimit),
|
||
|
|
});
|
||
|
|
try {
|
||
|
|
setStatus("Searching validated index…");
|
||
|
|
const data = await api(`search?${params}`);
|
||
|
|
renderResults(data.results || []);
|
||
|
|
setStatus(`${data.count} matching node${data.count === 1 ? "" : "s"}`);
|
||
|
|
} catch (error) {
|
||
|
|
setStatus(error.message, true);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
async function loadNode(nodeId) {
|
||
|
|
try {
|
||
|
|
setStatus(`Loading ${nodeId}…`);
|
||
|
|
const params = new URLSearchParams({id: nodeId, depth: String(state.depth), limit: "100"});
|
||
|
|
const data = await api(`node?${params}`);
|
||
|
|
renderGraph(data);
|
||
|
|
renderDetails(data.node, data);
|
||
|
|
setStatus(`${data.nodes.length} nodes · ${data.edges.length} edges in neighborhood`);
|
||
|
|
history.replaceState(null, "", `?node=${encodeURIComponent(nodeId)}&depth=${state.depth}`);
|
||
|
|
} catch (error) {
|
||
|
|
setStatus(error.message, true);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
$("search-form").addEventListener("submit", (event) => { event.preventDefault(); search(); });
|
||
|
|
$("family").addEventListener("change", search);
|
||
|
|
(async () => {
|
||
|
|
try {
|
||
|
|
const params = new URLSearchParams(location.search);
|
||
|
|
state.depth = Math.max(1, Number(params.get("depth")) || 1);
|
||
|
|
const overview = await api("overview");
|
||
|
|
renderOverview(overview);
|
||
|
|
const nodeId = params.get("node");
|
||
|
|
const query = params.get("q");
|
||
|
|
if (nodeId) {
|
||
|
|
await loadNode(nodeId);
|
||
|
|
} else {
|
||
|
|
if (query) $("search").value = query;
|
||
|
|
await search();
|
||
|
|
}
|
||
|
|
} catch (error) {
|
||
|
|
setStatus(error.message, true);
|
||
|
|
}
|
||
|
|
})();
|
||
|
|
</script>
|
||
|
|
</body>
|
||
|
|
</html>
|
||
|
|
"""
|