Complete independent projection runtime
This commit is contained in:
parent
1134c2d375
commit
f1fabaf0ca
38 changed files with 4907 additions and 87 deletions
436
src/docforge/projection_worker.py
Normal file
436
src/docforge/projection_worker.py
Normal file
|
|
@ -0,0 +1,436 @@
|
|||
"""One-shot detached execution for the fixed built-in projection renderers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from importlib.metadata import version
|
||||
from typing import cast
|
||||
|
||||
from .errors import DocForgeError
|
||||
from .projection_contract import (
|
||||
MAX_PACKAGE_BYTES,
|
||||
MAX_PROJECTION_ARTIFACTS,
|
||||
MAX_RECEIPT_BYTES,
|
||||
ProjectionArtifact,
|
||||
ProjectionPackageV1,
|
||||
ProjectionReceiptV1,
|
||||
ProjectionRenderResult,
|
||||
canonical_projection_bytes,
|
||||
)
|
||||
|
||||
WORKER_PROTOCOL_VERSION = 1
|
||||
MAX_WORKER_ARTIFACT_BYTES = 20_000_000
|
||||
MAX_WORKER_REQUEST_BYTES = MAX_PACKAGE_BYTES + 1
|
||||
MAX_WORKER_RESPONSE_BYTES = 4 * ((MAX_WORKER_ARTIFACT_BYTES + 2) // 3) + MAX_RECEIPT_BYTES + 256_000
|
||||
WORKER_TIMEOUT_SECONDS = 30
|
||||
|
||||
_GENERIC_HTML_RENDERER_ID = "generic_html"
|
||||
_PORTABLE_GRAPH_RENDERER_ID = "portable_graph_html"
|
||||
_PORTABLE_GRAPH_RENDERER_VERSION = "1"
|
||||
|
||||
|
||||
def _generic_html_renderer_version() -> str:
|
||||
return f"1+markdown-it-py-{version('markdown-it-py')}"
|
||||
|
||||
|
||||
def _worker_failure(message: str, **details: object) -> DocForgeError:
|
||||
return DocForgeError("projection_worker_failure", message, **details)
|
||||
|
||||
|
||||
def _renderer_identity(package: ProjectionPackageV1) -> dict[str, object]:
|
||||
renderer_value = package.document.get("renderer")
|
||||
if not isinstance(renderer_value, dict):
|
||||
raise DocForgeError("unsupported_renderer", "Projection renderer identity is invalid")
|
||||
renderer = cast(dict[str, object], renderer_value)
|
||||
if set(renderer) != {"renderer_id", "renderer_version"}:
|
||||
raise DocForgeError("unsupported_renderer", "Projection renderer identity is invalid")
|
||||
renderer_id = renderer.get("renderer_id")
|
||||
renderer_version = renderer.get("renderer_version")
|
||||
if not isinstance(renderer_id, str) or not isinstance(renderer_version, str):
|
||||
raise DocForgeError("unsupported_renderer", "Projection renderer identity is invalid")
|
||||
supported = (
|
||||
package.kind == "manual"
|
||||
and renderer_id == _GENERIC_HTML_RENDERER_ID
|
||||
and renderer_version == _generic_html_renderer_version()
|
||||
) or (
|
||||
package.kind == "graph"
|
||||
and renderer_id == _PORTABLE_GRAPH_RENDERER_ID
|
||||
and renderer_version == _PORTABLE_GRAPH_RENDERER_VERSION
|
||||
)
|
||||
if not supported:
|
||||
raise DocForgeError(
|
||||
"unsupported_renderer",
|
||||
"Projection worker supports only the fixed built-in renderer versions",
|
||||
)
|
||||
return dict(renderer)
|
||||
|
||||
|
||||
def _output_policy(package: ProjectionPackageV1) -> tuple[tuple[str, ...], int]:
|
||||
policy_value = package.document.get("output_policy")
|
||||
if not isinstance(policy_value, dict):
|
||||
raise DocForgeError("invalid_projection", "Projection output policy is invalid")
|
||||
policy = cast(dict[str, object], policy_value)
|
||||
if set(policy) != {"artifact_ids", "max_total_bytes"}:
|
||||
raise DocForgeError("invalid_projection", "Projection output policy is invalid")
|
||||
artifact_ids_value = policy.get("artifact_ids")
|
||||
maximum = policy.get("max_total_bytes")
|
||||
if not isinstance(artifact_ids_value, list):
|
||||
raise DocForgeError("invalid_projection", "Projection artifact inventory is invalid")
|
||||
artifact_ids_objects = cast(list[object], artifact_ids_value)
|
||||
if (
|
||||
not artifact_ids_objects
|
||||
or len(artifact_ids_objects) > MAX_PROJECTION_ARTIFACTS
|
||||
or not all(
|
||||
isinstance(artifact_id, str)
|
||||
and bool(artifact_id)
|
||||
and "/" not in artifact_id
|
||||
and artifact_id not in {".", ".."}
|
||||
for artifact_id in artifact_ids_objects
|
||||
)
|
||||
):
|
||||
raise DocForgeError("invalid_projection", "Projection artifact inventory is invalid")
|
||||
artifact_ids = tuple(cast(list[str], artifact_ids_objects))
|
||||
if len(set(artifact_ids)) != len(artifact_ids):
|
||||
raise DocForgeError("invalid_projection", "Projection artifact inventory is duplicated")
|
||||
if type(maximum) is not int or maximum < 1:
|
||||
raise DocForgeError(
|
||||
"invalid_projection",
|
||||
"Projection output byte allowance is invalid",
|
||||
)
|
||||
return artifact_ids, maximum
|
||||
|
||||
|
||||
def _validated_package(package: object) -> ProjectionPackageV1:
|
||||
if not isinstance(package, ProjectionPackageV1):
|
||||
raise TypeError("Projection worker requires ProjectionPackageV1")
|
||||
validated = ProjectionPackageV1.from_dict(package.as_dict())
|
||||
_renderer_identity(validated)
|
||||
_output_policy(validated)
|
||||
return validated
|
||||
|
||||
|
||||
def _validate_result(
|
||||
package: ProjectionPackageV1,
|
||||
result: ProjectionRenderResult,
|
||||
*,
|
||||
require_peak_memory: bool,
|
||||
) -> ProjectionRenderResult:
|
||||
artifact_ids, maximum = _output_policy(package)
|
||||
if (
|
||||
len(result.artifacts) != len(artifact_ids)
|
||||
or tuple(artifact.artifact_id for artifact in result.artifacts) != artifact_ids
|
||||
):
|
||||
raise _worker_failure("Projection worker returned an invalid artifact inventory")
|
||||
total_bytes = 0
|
||||
evidence: list[dict[str, object]] = []
|
||||
for artifact in result.artifacts:
|
||||
if not artifact.media_type or type(artifact.content) is not bytes:
|
||||
raise _worker_failure("Projection worker returned an invalid artifact")
|
||||
total_bytes += len(artifact.content)
|
||||
if total_bytes > maximum or total_bytes > MAX_WORKER_ARTIFACT_BYTES:
|
||||
raise _worker_failure("Projection worker artifact transfer exceeded its fixed boundary")
|
||||
evidence.append(artifact.evidence())
|
||||
|
||||
try:
|
||||
receipt = ProjectionReceiptV1.from_dict(result.receipt.as_dict())
|
||||
except DocForgeError as error:
|
||||
raise _worker_failure("Projection worker receipt is invalid") from error
|
||||
receipt_document = receipt.document
|
||||
renderer = _renderer_identity(package)
|
||||
timing_value = receipt_document.get("timing")
|
||||
if not isinstance(timing_value, dict):
|
||||
raise _worker_failure("Projection worker receipt timing is invalid")
|
||||
timing = cast(dict[str, object], timing_value)
|
||||
elapsed = timing.get("elapsed_ns")
|
||||
peak_memory = receipt_document.get("peak_memory_bytes")
|
||||
if (
|
||||
receipt_document.get("kind") != package.kind
|
||||
or receipt_document.get("package_id") != package.package_id
|
||||
or receipt_document.get("plan_id") != package.document.get("plan_id")
|
||||
or receipt_document.get("renderer") != renderer
|
||||
or receipt_document.get("artifacts") != evidence
|
||||
or type(elapsed) is not int
|
||||
or elapsed < 0
|
||||
or (require_peak_memory and (type(peak_memory) is not int or peak_memory <= 0))
|
||||
):
|
||||
raise _worker_failure("Projection worker receipt does not attest the requested package")
|
||||
return ProjectionRenderResult(tuple(result.artifacts), receipt)
|
||||
|
||||
|
||||
def _decode_canonical_line(raw: bytes, *, maximum: int, label: str) -> dict[str, object]:
|
||||
if type(raw) is not bytes or len(raw) > maximum:
|
||||
raise _worker_failure(f"{label} exceeded its fixed boundary", maximum_bytes=maximum)
|
||||
if not raw or not raw.endswith(b"\n") or raw.count(b"\n") != 1:
|
||||
raise _worker_failure(f"{label} framing is invalid")
|
||||
payload = raw[:-1]
|
||||
try:
|
||||
value: object = json.loads(payload)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
||||
raise _worker_failure(f"{label} is not valid JSON") from error
|
||||
if not isinstance(value, dict):
|
||||
raise _worker_failure(f"{label} must be one JSON object")
|
||||
document = cast(dict[str, object], value)
|
||||
if canonical_projection_bytes(document) != payload:
|
||||
raise _worker_failure(f"{label} is not canonical JSON")
|
||||
return document
|
||||
|
||||
|
||||
def _encode_request(package: ProjectionPackageV1) -> bytes:
|
||||
encoded = canonical_projection_bytes(package.as_dict()) + b"\n"
|
||||
if len(encoded) > MAX_WORKER_REQUEST_BYTES:
|
||||
raise DocForgeError(
|
||||
"projection_too_large",
|
||||
"Projection worker request exceeds its fixed boundary",
|
||||
maximum_bytes=MAX_WORKER_REQUEST_BYTES,
|
||||
)
|
||||
return encoded
|
||||
|
||||
|
||||
def _invoke_worker(request: bytes) -> subprocess.CompletedProcess[bytes]:
|
||||
environment = {key: os.environ[key] for key in ("SYSTEMROOT", "WINDIR") if key in os.environ}
|
||||
environment.update(
|
||||
{
|
||||
"PYTHONIOENCODING": "utf-8",
|
||||
"PYTHONUTF8": "1",
|
||||
}
|
||||
)
|
||||
command = [sys.executable, "-I", "-m", "docforge.projection_worker"]
|
||||
with tempfile.TemporaryFile() as output:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
input=request,
|
||||
stdout=output,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
timeout=WORKER_TIMEOUT_SECONDS,
|
||||
shell=False,
|
||||
cwd=sys.prefix,
|
||||
env=environment,
|
||||
)
|
||||
output.seek(0)
|
||||
stdout = output.read(MAX_WORKER_RESPONSE_BYTES + 1)
|
||||
return subprocess.CompletedProcess(
|
||||
command,
|
||||
completed.returncode,
|
||||
stdout=stdout,
|
||||
)
|
||||
|
||||
|
||||
def _decode_response(package: ProjectionPackageV1, raw: bytes) -> ProjectionRenderResult:
|
||||
document = _decode_canonical_line(
|
||||
raw,
|
||||
maximum=MAX_WORKER_RESPONSE_BYTES,
|
||||
label="Projection worker response",
|
||||
)
|
||||
if (
|
||||
set(document) != {"schema_version", "artifacts", "receipt"}
|
||||
or document.get("schema_version") != WORKER_PROTOCOL_VERSION
|
||||
):
|
||||
raise _worker_failure("Projection worker response contract is invalid")
|
||||
artifact_values = document.get("artifacts")
|
||||
receipt_value = document.get("receipt")
|
||||
if not isinstance(artifact_values, list) or not isinstance(receipt_value, dict):
|
||||
raise _worker_failure("Projection worker response structure is invalid")
|
||||
artifacts: list[ProjectionArtifact] = []
|
||||
total_bytes = 0
|
||||
for value in cast(list[object], artifact_values):
|
||||
if not isinstance(value, dict):
|
||||
raise _worker_failure("Projection worker artifact envelope is invalid")
|
||||
artifact = cast(dict[str, object], value)
|
||||
if set(artifact) != {"artifact_id", "media_type", "content_base64"}:
|
||||
raise _worker_failure("Projection worker artifact envelope is invalid")
|
||||
artifact_id = artifact.get("artifact_id")
|
||||
media_type = artifact.get("media_type")
|
||||
encoded = artifact.get("content_base64")
|
||||
if (
|
||||
not isinstance(artifact_id, str)
|
||||
or not isinstance(media_type, str)
|
||||
or not isinstance(encoded, str)
|
||||
):
|
||||
raise _worker_failure("Projection worker artifact envelope is invalid")
|
||||
try:
|
||||
content = base64.b64decode(encoded.encode("ascii"), validate=True)
|
||||
except (UnicodeEncodeError, binascii.Error, ValueError) as error:
|
||||
raise _worker_failure("Projection worker artifact encoding is invalid") from error
|
||||
total_bytes += len(content)
|
||||
if total_bytes > MAX_WORKER_ARTIFACT_BYTES:
|
||||
raise _worker_failure("Projection worker artifact transfer exceeded its fixed boundary")
|
||||
artifacts.append(ProjectionArtifact(artifact_id, media_type, content))
|
||||
try:
|
||||
receipt = ProjectionReceiptV1.from_dict(cast(dict[str, object], receipt_value))
|
||||
except DocForgeError as error:
|
||||
raise _worker_failure("Projection worker receipt is invalid") from error
|
||||
return _validate_result(
|
||||
package,
|
||||
ProjectionRenderResult(tuple(artifacts), receipt),
|
||||
require_peak_memory=True,
|
||||
)
|
||||
|
||||
|
||||
def render_projection_in_worker(package: ProjectionPackageV1) -> ProjectionRenderResult:
|
||||
"""Render one validated path-free package in a fixed one-shot child process."""
|
||||
|
||||
validated = _validated_package(package)
|
||||
request = _encode_request(validated)
|
||||
try:
|
||||
completed = _invoke_worker(request)
|
||||
except subprocess.TimeoutExpired as error:
|
||||
raise DocForgeError(
|
||||
"projection_worker_timeout",
|
||||
"Detached projection worker exceeded its fixed timeout",
|
||||
timeout_seconds=WORKER_TIMEOUT_SECONDS,
|
||||
) from error
|
||||
except OSError as error:
|
||||
raise _worker_failure("Detached projection worker could not be launched") from error
|
||||
if completed.returncode != 0:
|
||||
if completed.returncode == 3 and completed.stdout:
|
||||
try:
|
||||
failure = _decode_canonical_line(
|
||||
completed.stdout,
|
||||
maximum=MAX_RECEIPT_BYTES,
|
||||
label="Projection worker error response",
|
||||
)
|
||||
error = failure.get("error")
|
||||
error_document = cast(dict[str, object], error) if isinstance(error, dict) else None
|
||||
if (
|
||||
set(failure) == {"schema_version", "error"}
|
||||
and failure.get("schema_version") == WORKER_PROTOCOL_VERSION
|
||||
and error_document is not None
|
||||
and set(error_document) == {"code", "message", "details"}
|
||||
and isinstance(error_document.get("code"), str)
|
||||
and bool(error_document["code"])
|
||||
and isinstance(error_document.get("message"), str)
|
||||
and bool(error_document["message"])
|
||||
and isinstance(error_document.get("details"), dict)
|
||||
):
|
||||
raise DocForgeError(
|
||||
cast(str, error_document["code"]),
|
||||
cast(str, error_document["message"]),
|
||||
**cast(dict[str, object], error_document["details"]),
|
||||
)
|
||||
except DocForgeError as error:
|
||||
if error.code != "projection_worker_failure":
|
||||
raise
|
||||
if completed.returncode < 0:
|
||||
raise _worker_failure(
|
||||
"Detached projection worker terminated by signal",
|
||||
signal=-completed.returncode,
|
||||
)
|
||||
raise _worker_failure(
|
||||
"Detached projection worker exited unsuccessfully",
|
||||
exit_code=completed.returncode,
|
||||
)
|
||||
if type(completed.stdout) is not bytes:
|
||||
raise _worker_failure("Detached projection worker returned invalid output")
|
||||
return _decode_response(validated, completed.stdout)
|
||||
|
||||
|
||||
def _render_package(package: ProjectionPackageV1) -> ProjectionRenderResult:
|
||||
renderer = _renderer_identity(package)
|
||||
if renderer["renderer_id"] == _GENERIC_HTML_RENDERER_ID:
|
||||
from docforge_renderers.manual import ManualHtmlRenderer
|
||||
|
||||
result = ManualHtmlRenderer(cast(str, renderer["renderer_version"])).render(package)
|
||||
else:
|
||||
from docforge_renderers.graph import PortableGraphHtmlRenderer
|
||||
|
||||
result = PortableGraphHtmlRenderer().render(package)
|
||||
return _validate_result(package, result, require_peak_memory=False)
|
||||
|
||||
|
||||
def _peak_memory_bytes() -> int:
|
||||
peak = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
||||
return max(1, peak if sys.platform == "darwin" else peak * 1024)
|
||||
|
||||
|
||||
def _child_response(package: ProjectionPackageV1) -> bytes:
|
||||
result = _render_package(package)
|
||||
original = result.receipt.document
|
||||
receipt = ProjectionReceiptV1.create(
|
||||
kind=package.kind,
|
||||
package_id=package.package_id,
|
||||
plan_id=cast(str, package.document["plan_id"]),
|
||||
renderer=cast(dict[str, object], original["renderer"]),
|
||||
artifacts=[artifact.evidence() for artifact in result.artifacts],
|
||||
diagnostics=cast(dict[str, object], original["diagnostics"]),
|
||||
timing=cast(dict[str, object], original["timing"]),
|
||||
peak_memory_bytes=_peak_memory_bytes(),
|
||||
)
|
||||
validated = _validate_result(
|
||||
package,
|
||||
ProjectionRenderResult(result.artifacts, receipt),
|
||||
require_peak_memory=True,
|
||||
)
|
||||
document: dict[str, object] = {
|
||||
"schema_version": WORKER_PROTOCOL_VERSION,
|
||||
"artifacts": [
|
||||
{
|
||||
"artifact_id": artifact.artifact_id,
|
||||
"media_type": artifact.media_type,
|
||||
"content_base64": base64.b64encode(artifact.content).decode("ascii"),
|
||||
}
|
||||
for artifact in validated.artifacts
|
||||
],
|
||||
"receipt": validated.receipt.as_dict(),
|
||||
}
|
||||
encoded = canonical_projection_bytes(document) + b"\n"
|
||||
if len(encoded) > MAX_WORKER_RESPONSE_BYTES:
|
||||
raise _worker_failure(
|
||||
"Projection worker response exceeded its fixed boundary",
|
||||
maximum_bytes=MAX_WORKER_RESPONSE_BYTES,
|
||||
)
|
||||
return encoded
|
||||
|
||||
|
||||
def _read_child_request() -> ProjectionPackageV1:
|
||||
raw = sys.stdin.buffer.read(MAX_WORKER_REQUEST_BYTES + 1)
|
||||
document = _decode_canonical_line(
|
||||
raw,
|
||||
maximum=MAX_WORKER_REQUEST_BYTES,
|
||||
label="Projection worker request",
|
||||
)
|
||||
return _validated_package(ProjectionPackageV1.from_dict(document))
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Run the closed one-request child protocol."""
|
||||
|
||||
arguments = sys.argv[1:] if argv is None else argv
|
||||
if arguments:
|
||||
return 2
|
||||
try:
|
||||
package = _read_child_request()
|
||||
except Exception:
|
||||
return 2
|
||||
try:
|
||||
response = _child_response(package)
|
||||
sys.stdout.buffer.write(response)
|
||||
sys.stdout.buffer.flush()
|
||||
except DocForgeError as error:
|
||||
response = (
|
||||
canonical_projection_bytes(
|
||||
{
|
||||
"schema_version": WORKER_PROTOCOL_VERSION,
|
||||
"error": error.as_dict(),
|
||||
}
|
||||
)
|
||||
+ b"\n"
|
||||
)
|
||||
if len(response) <= MAX_RECEIPT_BYTES:
|
||||
sys.stdout.buffer.write(response)
|
||||
sys.stdout.buffer.flush()
|
||||
return 3
|
||||
except Exception:
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue