1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions Pages

Add durable portable graph publication

This commit is contained in:
Andraxion 2026-07-29 11:37:33 -04:00
parent 96e3965855
commit 1134c2d375
19 changed files with 2542 additions and 13 deletions

View file

@ -0,0 +1,793 @@
"""Declared portable graph planning, publication, and receipt-only status."""
from __future__ import annotations
import fcntl
import json
import os
from collections.abc import Callable, Generator
from contextlib import contextmanager
from pathlib import Path
from typing import cast
from ._fs_safety import (
atomic_replace_bytes_at,
open_confined_directory,
read_bounded_file_at,
require_bound_directory,
safe_file_identity_at,
)
from .errors import DocForgeError
from .graph_projection import (
GraphViewRequestV1,
build_graph_projection_package,
build_graph_view_plan,
)
from .models import (
GenerationRecordingProject,
GraphRenderConfig,
GraphRenderView,
IncrementalStateProject,
ProjectService,
ProjectSnapshot,
ProjectState,
)
from .project import project_root_fingerprint
from .projection_contract import GraphViewPlanV1, ProjectionReceiptV1, projection_hash
GRAPH_RENDERER_ID = "portable_graph_html"
GRAPH_RENDERER_VERSION = "1"
GRAPH_PUBLICATION_MANIFEST_VERSION = 1
GRAPH_PUBLICATION_CONTRACT = "docforge.graph-publication"
MAX_GRAPH_PUBLICATION_BYTES = 256_000
class GraphRenderService:
"""Publish one declared artifact while keeping planning and rendering independent."""
def __init__(self, project: ProjectService, *, allow_logic: bool = False) -> None:
self.project = project
self.allow_logic = allow_logic
def plan(self, view_id: str) -> dict[str, object]:
snapshot = self.project.load()
view = self._view(self._config(snapshot), view_id)
plan = self._plan(snapshot, view)
return {
"status": "ok",
**self._identity(snapshot),
"view_id": view.view_id,
"plan": plan.as_dict(),
}
def status(self, view_id: str | None = None) -> dict[str, object]:
config = self.project.descriptor.graph_render
current = self._current_state()
if config is None:
return self._status_result(
current,
configured=False,
state="not_configured",
outputs=[],
)
views = config.views if view_id is None else (self._view(config, view_id),)
first_outputs = [self._manifest_status(view, current) for view in views]
outputs = [self._manifest_status(view, current) for view in views]
if outputs != first_outputs:
for output in outputs:
if output["state"] == "current":
output["state"] = "stale"
output["reason"] = "publication_changed_during_status"
final = self._current_state()
if final != current:
for output in outputs:
if output["state"] == "current":
output["state"] = "stale"
output["reason"] = "source_changed_during_status"
identity = final if final is not None else current
return self._status_result(
identity,
configured=True,
state="current" if all(item["state"] == "current" for item in outputs) else "stale",
outputs=outputs,
)
def render(self, view_id: str) -> dict[str, object]:
with self._lock():
current_status = self.status(view_id)
current_outputs = cast(list[dict[str, object]], current_status["outputs"])
if current_status["state"] == "current" and current_outputs:
return {
**current_status,
"publication": "unchanged",
"output": current_outputs[0],
}
snapshot = self.project.load()
view = self._view(self._config(snapshot), view_id)
plan = self._plan(snapshot, view)
package = build_graph_projection_package(
plan,
renderer_id=GRAPH_RENDERER_ID,
renderer_version=GRAPH_RENDERER_VERSION,
max_output_bytes=snapshot.descriptor.limits.max_render_bytes,
)
from docforge_renderers.graph import PortableGraphHtmlRenderer
result = PortableGraphHtmlRenderer().render(package)
if len(result.artifacts) != 1:
raise DocForgeError(
"invalid_projection",
"Portable graph renderer returned an unsupported artifact set",
)
artifact = result.artifacts[0]
def verify() -> None:
current = self.project.load()
if (
current.revision != snapshot.revision
or current.source_hash != snapshot.source_hash
):
raise DocForgeError(
"render_input_changed",
"Canonical input changed during portable graph rendering",
)
verify()
if isinstance(self.project, GenerationRecordingProject):
self.project.record_generation(snapshot)
artifact_evidence = artifact.evidence()
try:
store_identity = self._publish_artifact(
snapshot,
artifact_evidence["sha256"],
artifact.content,
verify=verify,
)
except DocForgeError as error:
if self._mutation_committed(error):
return self._degraded_publication(
snapshot,
view,
plan,
package.package_id,
result.receipt.as_dict(),
artifact_evidence,
stage="artifact_store",
error=error,
output_published=False,
)
raise
try:
output_identity = self._publish_output(
snapshot,
view,
artifact.content,
verify=verify,
)
except DocForgeError as error:
if self._mutation_committed(error):
return self._degraded_publication(
snapshot,
view,
plan,
package.package_id,
result.receipt.as_dict(),
artifact_evidence,
stage="output",
error=error,
output_published=True,
)
raise
manifest = self._manifest(
snapshot,
view,
plan,
package.package_id,
result.receipt.as_dict(),
artifact_evidence,
store_identity,
output_identity,
)
try:
self._publish_manifest(snapshot, view, manifest, verify=verify)
except DocForgeError as error:
return self._degraded_publication(
snapshot,
view,
plan,
package.package_id,
result.receipt.as_dict(),
artifact_evidence,
stage="manifest",
error=error,
output_published=True,
)
return {
"status": "ok",
**self._identity(snapshot),
"view_id": view.view_id,
"state": "current",
"publication": "published",
"plan_id": plan.plan_id,
"package_id": package.package_id,
"output": {
**artifact.evidence(),
"path": view.output_path.relative_to(snapshot.descriptor.root).as_posix(),
},
"receipt": result.receipt.as_dict(),
"manifest": {
"state": "current",
"publication_id": manifest["publication_id"],
},
}
@staticmethod
def _mutation_committed(error: DocForgeError) -> bool:
return error.details.get("mutation_committed") is True
def _degraded_publication(
self,
snapshot: ProjectSnapshot,
view: GraphRenderView,
plan: GraphViewPlanV1,
package_id: str,
receipt: dict[str, object],
artifact: dict[str, object],
*,
stage: str,
error: DocForgeError,
output_published: bool,
) -> dict[str, object]:
return {
"status": "ok",
**self._identity(snapshot),
"view_id": view.view_id,
"state": "degraded",
"publication": "published" if output_published else "partial",
"committed_stage": stage,
"plan_id": plan.plan_id,
"package_id": package_id,
"artifact": artifact,
"output": {
**artifact,
"path": view.output_path.relative_to(snapshot.descriptor.root).as_posix(),
"state": "unverified" if output_published else "not_published",
},
"receipt": receipt,
"manifest": {
"state": "failed",
"error": error.as_dict(),
},
}
def _manifest_status(
self,
view: GraphRenderView,
current: ProjectState | None,
) -> dict[str, object]:
manifest = self._read_manifest(view)
base = {
"view_id": view.view_id,
"renderer": GRAPH_RENDERER_ID,
"renderer_version": GRAPH_RENDERER_VERSION,
"path": view.output_path.relative_to(self.project.descriptor.root).as_posix(),
"verification": "manifest",
}
if manifest is None:
return {**base, "state": "missing", "reason": "manifest_missing"}
if not self._valid_manifest(view, manifest):
return {**base, "state": "unverified", "reason": "manifest_invalid"}
if current is None:
reason = (
"source_generation_changed"
if isinstance(self.project, GenerationRecordingProject)
else "source_generation_unavailable"
)
return {
**base,
"state": (
"stale"
if isinstance(self.project, GenerationRecordingProject)
else "unverified"
),
"reason": reason,
"plan_id": manifest.get("plan_id"),
"package_id": manifest.get("package_id"),
}
project = cast(dict[str, object], manifest["project"])
if project["revision"] != current.revision or project["source_hash"] != current.source_hash:
return {
**base,
"state": "stale",
"reason": "source_generation_changed",
"plan_id": manifest["plan_id"],
"package_id": manifest["package_id"],
}
artifact = cast(dict[str, object], manifest["artifact"])
store = cast(dict[str, object], manifest["store"])
artifact_root = self.project.descriptor.cache_root / "projection-artifacts"
if not artifact_root.exists():
return {
**base,
"state": "stale",
"reason": "artifact_store_missing",
"plan_id": manifest["plan_id"],
"package_id": manifest["package_id"],
}
if artifact_root.is_symlink() or not artifact_root.is_dir():
return {
**base,
"state": "unsafe",
"reason": "artifact_store_unsafe",
"plan_id": manifest["plan_id"],
"package_id": manifest["package_id"],
}
artifact_directory: int | None = None
try:
artifact_directory = open_confined_directory(
self.project.descriptor.root,
artifact_root,
create=False,
)
artifact_identity = safe_file_identity_at(
artifact_root,
artifact_directory,
f"{artifact['sha256']}.html",
)
except DocForgeError:
return {
**base,
"state": "unsafe",
"reason": "artifact_store_unsafe",
"plan_id": manifest["plan_id"],
"package_id": manifest["package_id"],
}
finally:
if artifact_directory is not None:
os.close(artifact_directory)
if artifact_identity is None:
return {
**base,
"state": "stale",
"reason": "artifact_store_missing",
"plan_id": manifest["plan_id"],
"package_id": manifest["package_id"],
}
if artifact_identity != store:
return {
**base,
"state": "stale",
"reason": "artifact_store_changed",
"plan_id": manifest["plan_id"],
"package_id": manifest["package_id"],
}
try:
directory = open_confined_directory(
self.project.descriptor.root,
view.output_path.parent,
create=False,
)
except DocForgeError:
return {**base, "state": "unsafe", "reason": "output_root_unsafe"}
try:
identity = safe_file_identity_at(
view.output_path.parent, directory, view.output_path.name
)
except DocForgeError:
return {**base, "state": "unsafe", "reason": "output_unsafe"}
finally:
os.close(directory)
expected = cast(dict[str, object], manifest["output"])
if identity != expected:
return {
**base,
"state": "stale",
"reason": "output_changed",
"plan_id": manifest["plan_id"],
"package_id": manifest["package_id"],
}
return {
**base,
"state": "current",
"reason": None,
"plan_id": manifest["plan_id"],
"package_id": manifest["package_id"],
"publication_id": manifest["publication_id"],
"artifact": manifest["artifact"],
}
def _read_manifest(self, view: GraphRenderView) -> dict[str, object] | None:
root = self._manifest_root()
if not root.is_dir() or root.is_symlink():
return None
try:
descriptor = open_confined_directory(
self.project.descriptor.root,
root,
create=False,
)
except DocForgeError:
return None
try:
raw = read_bounded_file_at(
descriptor,
f"{view.view_id}.json",
MAX_GRAPH_PUBLICATION_BYTES,
)
except DocForgeError:
return None
finally:
os.close(descriptor)
if raw is None:
return None
try:
value: object = json.loads(raw)
except (UnicodeDecodeError, json.JSONDecodeError):
return None
return cast(dict[str, object], value) if isinstance(value, dict) else None
def _valid_manifest(self, view: GraphRenderView, manifest: dict[str, object]) -> bool:
required = {
"schema_version",
"contract",
"publication_id",
"project",
"view_id",
"view_config_hash",
"plan_id",
"package_id",
"renderer",
"receipt",
"artifact",
"store",
"output",
}
try:
if (
set(manifest) != required
or manifest.get("schema_version") != GRAPH_PUBLICATION_MANIFEST_VERSION
or manifest.get("contract") != GRAPH_PUBLICATION_CONTRACT
or manifest.get("view_id") != view.view_id
or manifest.get("view_config_hash") != self._view_hash(view)
or not self._hash(manifest.get("plan_id"))
or not self._hash(manifest.get("package_id"))
):
return False
project = manifest.get("project")
descriptor = self.project.descriptor
if not isinstance(project, dict):
return False
project_document = cast(dict[str, object], project)
if (
set(project_document)
!= {
"project_id",
"project_root_fingerprint",
"adapter",
"revision",
"source_hash",
}
or project_document.get("project_id") != descriptor.project_id
or project_document.get("project_root_fingerprint")
!= project_root_fingerprint(descriptor.root)
or project_document.get("adapter") != descriptor.adapter
or not isinstance(project_document.get("revision"), str)
or not project_document["revision"]
or not self._hash(project_document.get("source_hash"))
):
return False
renderer = manifest.get("renderer")
if renderer != {
"renderer_id": GRAPH_RENDERER_ID,
"renderer_version": GRAPH_RENDERER_VERSION,
}:
return False
receipt_value = manifest.get("receipt")
if not isinstance(receipt_value, dict):
return False
receipt = ProjectionReceiptV1.from_dict(
dict(cast(dict[str, object], receipt_value))
).as_dict()
artifacts = receipt.get("artifacts")
if not isinstance(artifacts, list):
return False
artifact_values = cast(list[object], artifacts)
if (
receipt.get("kind") != "graph"
or receipt.get("plan_id") != manifest["plan_id"]
or receipt.get("package_id") != manifest["package_id"]
or receipt.get("renderer") != renderer
or len(artifact_values) != 1
or manifest.get("artifact") != artifact_values[0]
):
return False
artifact = artifact_values[0]
if not isinstance(artifact, dict):
return False
artifact_document = cast(dict[str, object], artifact)
if (
artifact_document.get("artifact_id") != "portable-graph.html"
or artifact_document.get("media_type") != "text/html; charset=utf-8"
):
return False
artifact_hash = artifact_document.get("sha256")
artifact_bytes = artifact_document.get("bytes")
store = manifest.get("store")
output = manifest.get("output")
if (
not self._file_identity(store, expected_name=f"{artifact_hash}.html")
or not self._file_identity(output, expected_name=view.output_path.name)
or type(artifact_bytes) is not int
or cast(dict[str, object], store)["size"] != artifact_bytes
or cast(dict[str, object], output)["size"] != artifact_bytes
):
return False
body = dict(manifest)
publication_id = body.pop("publication_id", None)
return self._hash(publication_id) and publication_id == projection_hash(body)
except (DocForgeError, KeyError, TypeError, ValueError):
return False
@staticmethod
def _hash(value: object) -> bool:
return (
isinstance(value, str)
and len(value) == 64
and all(character in "0123456789abcdef" for character in value)
)
@staticmethod
def _file_identity(value: object, *, expected_name: str) -> bool:
if not isinstance(value, dict):
return False
document = cast(dict[str, object], value)
required = {"path", "device", "inode", "mode", "size", "mtime_ns", "ctime_ns"}
return (
set(document) == required
and document.get("path") == expected_name
and all(
type(document.get(field)) is int and cast(int, document[field]) >= 0
for field in required - {"path"}
)
)
def _manifest(
self,
snapshot: ProjectSnapshot,
view: GraphRenderView,
plan: GraphViewPlanV1,
package_id: str,
receipt: dict[str, object],
artifact: dict[str, object],
store: dict[str, object],
output: dict[str, object],
) -> dict[str, object]:
body: dict[str, object] = {
"schema_version": GRAPH_PUBLICATION_MANIFEST_VERSION,
"contract": GRAPH_PUBLICATION_CONTRACT,
"project": self._identity(snapshot),
"view_id": view.view_id,
"view_config_hash": self._view_hash(view),
"plan_id": plan.plan_id,
"package_id": package_id,
"renderer": {
"renderer_id": GRAPH_RENDERER_ID,
"renderer_version": GRAPH_RENDERER_VERSION,
},
"receipt": receipt,
"artifact": artifact,
"store": store,
"output": output,
}
return {**body, "publication_id": projection_hash(body)}
def _publish_artifact(
self,
snapshot: ProjectSnapshot,
artifact_hash: object,
content: bytes,
*,
verify: Callable[[], None],
) -> dict[str, object]:
if not isinstance(artifact_hash, str):
raise DocForgeError("invalid_projection", "Artifact hash is invalid")
root = snapshot.descriptor.cache_root / "projection-artifacts"
descriptor = open_confined_directory(snapshot.descriptor.root, root, create=True)
name = f"{artifact_hash}.html"
try:
try:
existing = read_bounded_file_at(descriptor, name, len(content))
except DocForgeError as error:
if error.code != "invalid_projection":
raise
existing = None
if existing == content:
identity = safe_file_identity_at(root, descriptor, name)
assert identity is not None
return identity
return atomic_replace_bytes_at(root, descriptor, name, content, verify=verify)
finally:
os.close(descriptor)
def _publish_output(
self,
snapshot: ProjectSnapshot,
view: GraphRenderView,
content: bytes,
*,
verify: Callable[[], None],
) -> dict[str, object]:
root = view.output_path.parent
descriptor = open_confined_directory(snapshot.descriptor.root, root, create=True)
try:
try:
existing = read_bounded_file_at(
descriptor,
view.output_path.name,
len(content),
)
except DocForgeError as error:
if error.code != "invalid_projection":
raise
existing = None
if existing == content:
identity = safe_file_identity_at(root, descriptor, view.output_path.name)
assert identity is not None
return identity
return atomic_replace_bytes_at(
root,
descriptor,
view.output_path.name,
content,
verify=verify,
)
finally:
os.close(descriptor)
def _publish_manifest(
self,
snapshot: ProjectSnapshot,
view: GraphRenderView,
manifest: dict[str, object],
*,
verify: Callable[[], None],
) -> None:
raw = json.dumps(manifest, sort_keys=True, indent=2).encode() + b"\n"
if len(raw) > MAX_GRAPH_PUBLICATION_BYTES:
raise DocForgeError(
"projection_too_large",
"Portable graph publication manifest exceeds its fixed limit",
)
root = self._manifest_root()
descriptor = open_confined_directory(snapshot.descriptor.root, root, create=True)
try:
atomic_replace_bytes_at(
root,
descriptor,
f"{view.view_id}.json",
raw,
verify=verify,
)
finally:
os.close(descriptor)
@contextmanager
def _lock(self) -> Generator[None]:
root = self.project.descriptor.cache_root
descriptor = open_confined_directory(self.project.descriptor.root, root, create=True)
lock_descriptor: int | None = None
try:
lock_descriptor = os.open(
"graph-render.lock",
os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW,
0o600,
dir_fd=descriptor,
)
fcntl.flock(lock_descriptor, fcntl.LOCK_EX)
require_bound_directory(root, descriptor)
yield
except OSError as error:
raise DocForgeError(
"publication_failure",
"Portable graph render lock is unavailable",
) from error
finally:
if lock_descriptor is not None:
os.close(lock_descriptor)
os.close(descriptor)
def _plan(self, snapshot: ProjectSnapshot, view: GraphRenderView) -> GraphViewPlanV1:
return build_graph_view_plan(
snapshot,
GraphViewRequestV1(
view_id=view.view_id,
title=view.title,
root_node_id=view.root_node_id,
query=view.query,
initial_mode=view.initial_mode,
depth=view.depth,
max_nodes=view.max_nodes,
max_edges=view.max_edges,
max_work=view.max_work,
families=view.families,
relations=view.relations,
authorities=view.authorities,
statuses=view.statuses,
tags=view.tags,
include_logic=view.include_logic,
),
self.allow_logic,
)
def _current_state(self) -> ProjectState | None:
if isinstance(self.project, IncrementalStateProject):
return self.project.incremental_state()
return None
def _config(self, snapshot: ProjectSnapshot) -> GraphRenderConfig:
config = snapshot.descriptor.graph_render
if config is None:
raise DocForgeError(
"graph_render_not_configured",
"Project has no portable graph render configuration",
)
return config
@staticmethod
def _view(config: GraphRenderConfig, view_id: str) -> GraphRenderView:
for view in config.views:
if view.view_id == view_id:
return view
raise DocForgeError(
"unknown_graph_render_view",
"Portable graph view is not declared",
view_id=view_id,
)
def _manifest_root(self) -> Path:
return self.project.descriptor.cache_root / "projection-publications" / "graph"
@staticmethod
def _view_hash(view: GraphRenderView) -> str:
return projection_hash(
{
"view_id": view.view_id,
"renderer": view.renderer,
"title": view.title,
"root_node_id": view.root_node_id,
"query": view.query,
"initial_mode": view.initial_mode,
"depth": view.depth,
"max_nodes": view.max_nodes,
"max_edges": view.max_edges,
"max_work": view.max_work,
"families": list(view.families),
"relations": list(view.relations),
"authorities": list(view.authorities),
"statuses": list(view.statuses),
"tags": list(view.tags),
"include_logic": view.include_logic,
}
)
def _status_result(self, current: ProjectState | None, **payload: object) -> dict[str, object]:
descriptor = self.project.descriptor
return {
"status": "ok",
"project_id": descriptor.project_id,
"project_root_fingerprint": project_root_fingerprint(descriptor.root),
"adapter": descriptor.adapter,
"revision": current.revision if current is not None else "unknown",
"source_hash": current.source_hash if current is not None else None,
**payload,
}
@staticmethod
def _identity(snapshot: ProjectSnapshot) -> dict[str, object]:
return {
"project_id": snapshot.descriptor.project_id,
"project_root_fingerprint": project_root_fingerprint(snapshot.descriptor.root),
"adapter": snapshot.descriptor.adapter,
"revision": snapshot.revision,
"source_hash": snapshot.source_hash,
}