436 lines
17 KiB
Python
436 lines
17 KiB
Python
"""One-shot detached execution for the fixed built-in projection renderers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import binascii
|
|
import json
|
|
import os
|
|
import resource
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from importlib.metadata import version
|
|
from typing import cast
|
|
|
|
from .errors import DocForgeError
|
|
from .projection_contract import (
|
|
MAX_PACKAGE_BYTES,
|
|
MAX_PROJECTION_ARTIFACTS,
|
|
MAX_RECEIPT_BYTES,
|
|
ProjectionArtifact,
|
|
ProjectionPackageV1,
|
|
ProjectionReceiptV1,
|
|
ProjectionRenderResult,
|
|
canonical_projection_bytes,
|
|
)
|
|
|
|
WORKER_PROTOCOL_VERSION = 1
|
|
MAX_WORKER_ARTIFACT_BYTES = 20_000_000
|
|
MAX_WORKER_REQUEST_BYTES = MAX_PACKAGE_BYTES + 1
|
|
MAX_WORKER_RESPONSE_BYTES = 4 * ((MAX_WORKER_ARTIFACT_BYTES + 2) // 3) + MAX_RECEIPT_BYTES + 256_000
|
|
WORKER_TIMEOUT_SECONDS = 30
|
|
|
|
_GENERIC_HTML_RENDERER_ID = "generic_html"
|
|
_PORTABLE_GRAPH_RENDERER_ID = "portable_graph_html"
|
|
_PORTABLE_GRAPH_RENDERER_VERSION = "1"
|
|
|
|
|
|
def _generic_html_renderer_version() -> str:
|
|
return f"1+markdown-it-py-{version('markdown-it-py')}"
|
|
|
|
|
|
def _worker_failure(message: str, **details: object) -> DocForgeError:
|
|
return DocForgeError("projection_worker_failure", message, **details)
|
|
|
|
|
|
def _renderer_identity(package: ProjectionPackageV1) -> dict[str, object]:
|
|
renderer_value = package.document.get("renderer")
|
|
if not isinstance(renderer_value, dict):
|
|
raise DocForgeError("unsupported_renderer", "Projection renderer identity is invalid")
|
|
renderer = cast(dict[str, object], renderer_value)
|
|
if set(renderer) != {"renderer_id", "renderer_version"}:
|
|
raise DocForgeError("unsupported_renderer", "Projection renderer identity is invalid")
|
|
renderer_id = renderer.get("renderer_id")
|
|
renderer_version = renderer.get("renderer_version")
|
|
if not isinstance(renderer_id, str) or not isinstance(renderer_version, str):
|
|
raise DocForgeError("unsupported_renderer", "Projection renderer identity is invalid")
|
|
supported = (
|
|
package.kind == "manual"
|
|
and renderer_id == _GENERIC_HTML_RENDERER_ID
|
|
and renderer_version == _generic_html_renderer_version()
|
|
) or (
|
|
package.kind == "graph"
|
|
and renderer_id == _PORTABLE_GRAPH_RENDERER_ID
|
|
and renderer_version == _PORTABLE_GRAPH_RENDERER_VERSION
|
|
)
|
|
if not supported:
|
|
raise DocForgeError(
|
|
"unsupported_renderer",
|
|
"Projection worker supports only the fixed built-in renderer versions",
|
|
)
|
|
return dict(renderer)
|
|
|
|
|
|
def _output_policy(package: ProjectionPackageV1) -> tuple[tuple[str, ...], int]:
|
|
policy_value = package.document.get("output_policy")
|
|
if not isinstance(policy_value, dict):
|
|
raise DocForgeError("invalid_projection", "Projection output policy is invalid")
|
|
policy = cast(dict[str, object], policy_value)
|
|
if set(policy) != {"artifact_ids", "max_total_bytes"}:
|
|
raise DocForgeError("invalid_projection", "Projection output policy is invalid")
|
|
artifact_ids_value = policy.get("artifact_ids")
|
|
maximum = policy.get("max_total_bytes")
|
|
if not isinstance(artifact_ids_value, list):
|
|
raise DocForgeError("invalid_projection", "Projection artifact inventory is invalid")
|
|
artifact_ids_objects = cast(list[object], artifact_ids_value)
|
|
if (
|
|
not artifact_ids_objects
|
|
or len(artifact_ids_objects) > MAX_PROJECTION_ARTIFACTS
|
|
or not all(
|
|
isinstance(artifact_id, str)
|
|
and bool(artifact_id)
|
|
and "/" not in artifact_id
|
|
and artifact_id not in {".", ".."}
|
|
for artifact_id in artifact_ids_objects
|
|
)
|
|
):
|
|
raise DocForgeError("invalid_projection", "Projection artifact inventory is invalid")
|
|
artifact_ids = tuple(cast(list[str], artifact_ids_objects))
|
|
if len(set(artifact_ids)) != len(artifact_ids):
|
|
raise DocForgeError("invalid_projection", "Projection artifact inventory is duplicated")
|
|
if type(maximum) is not int or maximum < 1:
|
|
raise DocForgeError(
|
|
"invalid_projection",
|
|
"Projection output byte allowance is invalid",
|
|
)
|
|
return artifact_ids, maximum
|
|
|
|
|
|
def _validated_package(package: object) -> ProjectionPackageV1:
|
|
if not isinstance(package, ProjectionPackageV1):
|
|
raise TypeError("Projection worker requires ProjectionPackageV1")
|
|
validated = ProjectionPackageV1.from_dict(package.as_dict())
|
|
_renderer_identity(validated)
|
|
_output_policy(validated)
|
|
return validated
|
|
|
|
|
|
def _validate_result(
|
|
package: ProjectionPackageV1,
|
|
result: ProjectionRenderResult,
|
|
*,
|
|
require_peak_memory: bool,
|
|
) -> ProjectionRenderResult:
|
|
artifact_ids, maximum = _output_policy(package)
|
|
if (
|
|
len(result.artifacts) != len(artifact_ids)
|
|
or tuple(artifact.artifact_id for artifact in result.artifacts) != artifact_ids
|
|
):
|
|
raise _worker_failure("Projection worker returned an invalid artifact inventory")
|
|
total_bytes = 0
|
|
evidence: list[dict[str, object]] = []
|
|
for artifact in result.artifacts:
|
|
if not artifact.media_type or type(artifact.content) is not bytes:
|
|
raise _worker_failure("Projection worker returned an invalid artifact")
|
|
total_bytes += len(artifact.content)
|
|
if total_bytes > maximum or total_bytes > MAX_WORKER_ARTIFACT_BYTES:
|
|
raise _worker_failure("Projection worker artifact transfer exceeded its fixed boundary")
|
|
evidence.append(artifact.evidence())
|
|
|
|
try:
|
|
receipt = ProjectionReceiptV1.from_dict(result.receipt.as_dict())
|
|
except DocForgeError as error:
|
|
raise _worker_failure("Projection worker receipt is invalid") from error
|
|
receipt_document = receipt.document
|
|
renderer = _renderer_identity(package)
|
|
timing_value = receipt_document.get("timing")
|
|
if not isinstance(timing_value, dict):
|
|
raise _worker_failure("Projection worker receipt timing is invalid")
|
|
timing = cast(dict[str, object], timing_value)
|
|
elapsed = timing.get("elapsed_ns")
|
|
peak_memory = receipt_document.get("peak_memory_bytes")
|
|
if (
|
|
receipt_document.get("kind") != package.kind
|
|
or receipt_document.get("package_id") != package.package_id
|
|
or receipt_document.get("plan_id") != package.document.get("plan_id")
|
|
or receipt_document.get("renderer") != renderer
|
|
or receipt_document.get("artifacts") != evidence
|
|
or type(elapsed) is not int
|
|
or elapsed < 0
|
|
or (require_peak_memory and (type(peak_memory) is not int or peak_memory <= 0))
|
|
):
|
|
raise _worker_failure("Projection worker receipt does not attest the requested package")
|
|
return ProjectionRenderResult(tuple(result.artifacts), receipt)
|
|
|
|
|
|
def _decode_canonical_line(raw: bytes, *, maximum: int, label: str) -> dict[str, object]:
|
|
if type(raw) is not bytes or len(raw) > maximum:
|
|
raise _worker_failure(f"{label} exceeded its fixed boundary", maximum_bytes=maximum)
|
|
if not raw or not raw.endswith(b"\n") or raw.count(b"\n") != 1:
|
|
raise _worker_failure(f"{label} framing is invalid")
|
|
payload = raw[:-1]
|
|
try:
|
|
value: object = json.loads(payload)
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
|
raise _worker_failure(f"{label} is not valid JSON") from error
|
|
if not isinstance(value, dict):
|
|
raise _worker_failure(f"{label} must be one JSON object")
|
|
document = cast(dict[str, object], value)
|
|
if canonical_projection_bytes(document) != payload:
|
|
raise _worker_failure(f"{label} is not canonical JSON")
|
|
return document
|
|
|
|
|
|
def _encode_request(package: ProjectionPackageV1) -> bytes:
|
|
encoded = canonical_projection_bytes(package.as_dict()) + b"\n"
|
|
if len(encoded) > MAX_WORKER_REQUEST_BYTES:
|
|
raise DocForgeError(
|
|
"projection_too_large",
|
|
"Projection worker request exceeds its fixed boundary",
|
|
maximum_bytes=MAX_WORKER_REQUEST_BYTES,
|
|
)
|
|
return encoded
|
|
|
|
|
|
def _invoke_worker(request: bytes) -> subprocess.CompletedProcess[bytes]:
|
|
environment = {key: os.environ[key] for key in ("SYSTEMROOT", "WINDIR") if key in os.environ}
|
|
environment.update(
|
|
{
|
|
"PYTHONIOENCODING": "utf-8",
|
|
"PYTHONUTF8": "1",
|
|
}
|
|
)
|
|
command = [sys.executable, "-I", "-m", "docforge._projection_worker_main"]
|
|
with tempfile.TemporaryFile() as output:
|
|
completed = subprocess.run(
|
|
command,
|
|
input=request,
|
|
stdout=output,
|
|
stderr=subprocess.DEVNULL,
|
|
check=False,
|
|
timeout=WORKER_TIMEOUT_SECONDS,
|
|
shell=False,
|
|
cwd=sys.prefix,
|
|
env=environment,
|
|
)
|
|
output.seek(0)
|
|
stdout = output.read(MAX_WORKER_RESPONSE_BYTES + 1)
|
|
return subprocess.CompletedProcess(
|
|
command,
|
|
completed.returncode,
|
|
stdout=stdout,
|
|
)
|
|
|
|
|
|
def _decode_response(package: ProjectionPackageV1, raw: bytes) -> ProjectionRenderResult:
|
|
document = _decode_canonical_line(
|
|
raw,
|
|
maximum=MAX_WORKER_RESPONSE_BYTES,
|
|
label="Projection worker response",
|
|
)
|
|
if (
|
|
set(document) != {"schema_version", "artifacts", "receipt"}
|
|
or document.get("schema_version") != WORKER_PROTOCOL_VERSION
|
|
):
|
|
raise _worker_failure("Projection worker response contract is invalid")
|
|
artifact_values = document.get("artifacts")
|
|
receipt_value = document.get("receipt")
|
|
if not isinstance(artifact_values, list) or not isinstance(receipt_value, dict):
|
|
raise _worker_failure("Projection worker response structure is invalid")
|
|
artifacts: list[ProjectionArtifact] = []
|
|
total_bytes = 0
|
|
for value in cast(list[object], artifact_values):
|
|
if not isinstance(value, dict):
|
|
raise _worker_failure("Projection worker artifact envelope is invalid")
|
|
artifact = cast(dict[str, object], value)
|
|
if set(artifact) != {"artifact_id", "media_type", "content_base64"}:
|
|
raise _worker_failure("Projection worker artifact envelope is invalid")
|
|
artifact_id = artifact.get("artifact_id")
|
|
media_type = artifact.get("media_type")
|
|
encoded = artifact.get("content_base64")
|
|
if (
|
|
not isinstance(artifact_id, str)
|
|
or not isinstance(media_type, str)
|
|
or not isinstance(encoded, str)
|
|
):
|
|
raise _worker_failure("Projection worker artifact envelope is invalid")
|
|
try:
|
|
content = base64.b64decode(encoded.encode("ascii"), validate=True)
|
|
except (UnicodeEncodeError, binascii.Error, ValueError) as error:
|
|
raise _worker_failure("Projection worker artifact encoding is invalid") from error
|
|
total_bytes += len(content)
|
|
if total_bytes > MAX_WORKER_ARTIFACT_BYTES:
|
|
raise _worker_failure("Projection worker artifact transfer exceeded its fixed boundary")
|
|
artifacts.append(ProjectionArtifact(artifact_id, media_type, content))
|
|
try:
|
|
receipt = ProjectionReceiptV1.from_dict(cast(dict[str, object], receipt_value))
|
|
except DocForgeError as error:
|
|
raise _worker_failure("Projection worker receipt is invalid") from error
|
|
return _validate_result(
|
|
package,
|
|
ProjectionRenderResult(tuple(artifacts), receipt),
|
|
require_peak_memory=True,
|
|
)
|
|
|
|
|
|
def render_projection_in_worker(package: ProjectionPackageV1) -> ProjectionRenderResult:
|
|
"""Render one validated path-free package in a fixed one-shot child process."""
|
|
|
|
validated = _validated_package(package)
|
|
request = _encode_request(validated)
|
|
try:
|
|
completed = _invoke_worker(request)
|
|
except subprocess.TimeoutExpired as error:
|
|
raise DocForgeError(
|
|
"projection_worker_timeout",
|
|
"Detached projection worker exceeded its fixed timeout",
|
|
timeout_seconds=WORKER_TIMEOUT_SECONDS,
|
|
) from error
|
|
except OSError as error:
|
|
raise _worker_failure("Detached projection worker could not be launched") from error
|
|
if completed.returncode != 0:
|
|
if completed.returncode == 3 and completed.stdout:
|
|
try:
|
|
failure = _decode_canonical_line(
|
|
completed.stdout,
|
|
maximum=MAX_RECEIPT_BYTES,
|
|
label="Projection worker error response",
|
|
)
|
|
error = failure.get("error")
|
|
error_document = cast(dict[str, object], error) if isinstance(error, dict) else None
|
|
if (
|
|
set(failure) == {"schema_version", "error"}
|
|
and failure.get("schema_version") == WORKER_PROTOCOL_VERSION
|
|
and error_document is not None
|
|
and set(error_document) == {"code", "message", "details"}
|
|
and isinstance(error_document.get("code"), str)
|
|
and bool(error_document["code"])
|
|
and isinstance(error_document.get("message"), str)
|
|
and bool(error_document["message"])
|
|
and isinstance(error_document.get("details"), dict)
|
|
):
|
|
raise DocForgeError(
|
|
cast(str, error_document["code"]),
|
|
cast(str, error_document["message"]),
|
|
**cast(dict[str, object], error_document["details"]),
|
|
)
|
|
except DocForgeError as error:
|
|
if error.code != "projection_worker_failure":
|
|
raise
|
|
if completed.returncode < 0:
|
|
raise _worker_failure(
|
|
"Detached projection worker terminated by signal",
|
|
signal=-completed.returncode,
|
|
)
|
|
raise _worker_failure(
|
|
"Detached projection worker exited unsuccessfully",
|
|
exit_code=completed.returncode,
|
|
)
|
|
if type(completed.stdout) is not bytes:
|
|
raise _worker_failure("Detached projection worker returned invalid output")
|
|
return _decode_response(validated, completed.stdout)
|
|
|
|
|
|
def _render_package(package: ProjectionPackageV1) -> ProjectionRenderResult:
|
|
renderer = _renderer_identity(package)
|
|
if renderer["renderer_id"] == _GENERIC_HTML_RENDERER_ID:
|
|
from docforge_renderers.manual import ManualHtmlRenderer
|
|
|
|
result = ManualHtmlRenderer(cast(str, renderer["renderer_version"])).render(package)
|
|
else:
|
|
from docforge_renderers.graph import PortableGraphHtmlRenderer
|
|
|
|
result = PortableGraphHtmlRenderer().render(package)
|
|
return _validate_result(package, result, require_peak_memory=False)
|
|
|
|
|
|
def _peak_memory_bytes() -> int:
|
|
peak = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
|
return max(1, peak if sys.platform == "darwin" else peak * 1024)
|
|
|
|
|
|
def _child_response(package: ProjectionPackageV1) -> bytes:
|
|
result = _render_package(package)
|
|
original = result.receipt.document
|
|
receipt = ProjectionReceiptV1.create(
|
|
kind=package.kind,
|
|
package_id=package.package_id,
|
|
plan_id=cast(str, package.document["plan_id"]),
|
|
renderer=cast(dict[str, object], original["renderer"]),
|
|
artifacts=[artifact.evidence() for artifact in result.artifacts],
|
|
diagnostics=cast(dict[str, object], original["diagnostics"]),
|
|
timing=cast(dict[str, object], original["timing"]),
|
|
peak_memory_bytes=_peak_memory_bytes(),
|
|
)
|
|
validated = _validate_result(
|
|
package,
|
|
ProjectionRenderResult(result.artifacts, receipt),
|
|
require_peak_memory=True,
|
|
)
|
|
document: dict[str, object] = {
|
|
"schema_version": WORKER_PROTOCOL_VERSION,
|
|
"artifacts": [
|
|
{
|
|
"artifact_id": artifact.artifact_id,
|
|
"media_type": artifact.media_type,
|
|
"content_base64": base64.b64encode(artifact.content).decode("ascii"),
|
|
}
|
|
for artifact in validated.artifacts
|
|
],
|
|
"receipt": validated.receipt.as_dict(),
|
|
}
|
|
encoded = canonical_projection_bytes(document) + b"\n"
|
|
if len(encoded) > MAX_WORKER_RESPONSE_BYTES:
|
|
raise _worker_failure(
|
|
"Projection worker response exceeded its fixed boundary",
|
|
maximum_bytes=MAX_WORKER_RESPONSE_BYTES,
|
|
)
|
|
return encoded
|
|
|
|
|
|
def _read_child_request() -> ProjectionPackageV1:
|
|
raw = sys.stdin.buffer.read(MAX_WORKER_REQUEST_BYTES + 1)
|
|
document = _decode_canonical_line(
|
|
raw,
|
|
maximum=MAX_WORKER_REQUEST_BYTES,
|
|
label="Projection worker request",
|
|
)
|
|
return _validated_package(ProjectionPackageV1.from_dict(document))
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
"""Run the closed one-request child protocol."""
|
|
|
|
arguments = sys.argv[1:] if argv is None else argv
|
|
if arguments:
|
|
return 2
|
|
try:
|
|
package = _read_child_request()
|
|
except Exception:
|
|
return 2
|
|
try:
|
|
response = _child_response(package)
|
|
sys.stdout.buffer.write(response)
|
|
sys.stdout.buffer.flush()
|
|
except DocForgeError as error:
|
|
response = (
|
|
canonical_projection_bytes(
|
|
{
|
|
"schema_version": WORKER_PROTOCOL_VERSION,
|
|
"error": error.as_dict(),
|
|
}
|
|
)
|
|
+ b"\n"
|
|
)
|
|
if len(response) <= MAX_RECEIPT_BYTES:
|
|
sys.stdout.buffer.write(response)
|
|
sys.stdout.buffer.flush()
|
|
return 3
|
|
except Exception:
|
|
return 2
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|