administrator/DocForge2
1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions Pages

Complete independent projection runtime

Browse source
This commit is contained in:
Andraxion 2026-07-29 12:38:25 -04:00
parent 1134c2d375
commit f1fabaf0ca
38 changed files with 4907 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

20
Makefile
View file

@ -5,7 +5,10 @@ NPM := npm
PYTHONPYCACHEPREFIX := /tmp/docforge-quality-pycache
PYTEST_BASETEMP := /tmp/docforge-quality-pytest
.PHONY: benchmark benchmark-m1 benchmark-m1-smoke benchmark-m2 benchmark-m2-smoke benchmark-smoke build compile contract dependencies format-check gate lint lock test type
.PHONY: accessibility benchmark benchmark-m1 benchmark-m1-smoke benchmark-m2 benchmark-m2-smoke benchmark-m3 benchmark-m3-full benchmark-m3-smoke benchmark-smoke build compile contract dependencies format-check gate lint lock test type
accessibility:
$(NPM) run test:accessibility
format-check:
$(PYTHON) -m ruff format --check src tests tools
@ -25,6 +28,10 @@ contract:
-p no:cacheprovider --basetemp=$(PYTEST_BASETEMP) \
tests/test_public_contract.py \
tests/test_policy.py \
tests/test_projection_policy.py \
tests/test_projection_policy_integration.py \
tests/test_projection_worker.py \
tests/test_projection_fragments.py \
tests/test_retrieval.py \
tests/test_generation_diff.py \
tests/test_client_integration.py \
@ -73,4 +80,13 @@ benchmark-m2-smoke:
benchmark-m2:
$(PYTHON) tools/milestone2_benchmark.py --nodes 1000 --samples 10
gate: format-check lint type compile contract test lock dependencies build benchmark-smoke benchmark-m1-smoke benchmark-m2-smoke
benchmark-m3-smoke:
$(PYTHON) tools/milestone3_benchmark.py --mode smoke \
--output /tmp/docforge-milestone3-smoke.json > /dev/null
benchmark-m3:
$(PYTHON) tools/milestone3_benchmark.py --mode full
benchmark-m3-full: benchmark-m3
gate: format-check lint type compile contract test accessibility lock dependencies build benchmark-smoke benchmark-m1-smoke benchmark-m2-smoke benchmark-m3-smoke

22
eslint.config.mjs
View file

@ -21,4 +21,26 @@ export default [
"prefer-const": "error",
},
},
{
files: ["playwright.accessibility.config.mjs", "tests/accessibility.spec.mjs"],
...js.configs.recommended,
languageOptions: {
ecmaVersion: 2024,
sourceType: "module",
globals: {
...globals.browser,
...globals.node,
},
},
linterOptions: {
reportUnusedDisableDirectives: "error",
},
rules: {
...js.configs.recommended.rules,
eqeqeq: "error",
"no-implicit-coercion": "error",
"no-var": "error",
"prefer-const": "error",
},
},
];

88
package-lock.json generated
View file

@ -8,7 +8,9 @@
"name": "docforge-web-quality",
"version": "0.0.0",
"devDependencies": {
"@axe-core/playwright": "4.12.1",
"@eslint/js": "10.0.1",
"@playwright/test": "1.62.0",
"eslint": "10.8.0",
"globals": "17.7.0",
"html-validate": "11.5.6",
@ -18,6 +20,19 @@
"stylelint-csstree-validator": "4.0.0"
}
},
"node_modules/@axe-core/playwright": {
"version": "4.12.1",
"resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.12.1.tgz",
"integrity": "sha512-rMd7xriptqKpP+w5265i4Hdkv2X5kbu6uiBi/B2I7uf3hieRBM3qDCfaKPtxfiYb2mKXfF+yLODJwIx+Jv1GDw==",
"dev": true,
"license": "MPL-2.0",
"dependencies": {
"axe-core": "~4.12.1"
},
"peerDependencies": {
"playwright-core": ">= 1.0.0"
}
},
"node_modules/@babel/code-frame": {
"version": "7.29.7",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz",
@ -515,6 +530,22 @@
"node": ">= 8"
}
},
"node_modules/@playwright/test": {
"version": "1.62.0",
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.0.tgz",
"integrity": "sha512-9zOJ6ZQRAena31MpOH9VSzIz8Ou3YJ/wtY/eQm5T2uhfhG7/U3COrMS8xOtUrZrp9OgdmzEnIYODye3nY1VqzA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"playwright": "1.62.0"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=20"
}
},
"node_modules/@sindresorhus/merge-streams": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz",
@ -635,6 +666,16 @@
"node": ">=8"
}
},
"node_modules/axe-core": {
"version": "4.12.1",
"resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.12.1.tgz",
"integrity": "sha512-s7iGf5GaVMxEG0ENN9x+xTr7GFZCb1ZP/1uATUpCEK2X78nDB3RwbtFCo9pGAf9ru+VwoQ464DkaLEeRM08wJA==",
"dev": true,
"license": "MPL-2.0",
"engines": {
"node": ">=4"
}
},
"node_modules/balanced-match": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
@ -1943,6 +1984,53 @@
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/playwright": {
"version": "1.62.0",
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.0.tgz",
"integrity": "sha512-Z14dG305dgaLu6foB1TXQagFiW8JfSUIUaUuPaKQ6NtBPKF1P/qXcqfh6c6K/icPqdy37JmjbiBXf6JNg6Sylw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"playwright-core": "1.62.0"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=20"
},
"optionalDependencies": {
"fsevents": "2.3.2"
}
},
"node_modules/playwright-core": {
"version": "1.62.0",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.0.tgz",
"integrity": "sha512-nsNRyq0r2zsG8AcRHWknc9QRA5XCueC7gWMrs+Gx2tlZn9hcl8zudfh00lhJPY1DE7NmZ6bDsT9g2yey8mXljA==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"playwright-core": "cli.js"
},
"engines": {
"node": ">=20"
}
},
"node_modules/playwright/node_modules/fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/postcss": {
"version": "8.5.23",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz",

6
package.json
View file

@ -4,10 +4,14 @@
"private": true,
"packageManager": "npm@10.9.7",
"scripts": {
"lint:web": "uv run python tools/check_web_assets.py"
"install:accessibility-browser": "playwright install chromium",
"lint:web": "uv run python tools/check_web_assets.py && eslint --max-warnings=0 playwright.accessibility.config.mjs tests/accessibility.spec.mjs",
"test:accessibility": "npm run install:accessibility-browser && playwright test --config=playwright.accessibility.config.mjs"
},
"devDependencies": {
"@axe-core/playwright": "4.12.1",
"@eslint/js": "10.0.1",
"@playwright/test": "1.62.0",
"eslint": "10.8.0",
"globals": "17.7.0",
"html-validate": "11.5.6",

24
playwright.accessibility.config.mjs Normal file
View file

@ -0,0 +1,24 @@
import { defineConfig } from "@playwright/test";
export default defineConfig({
testDir: "./tests",
testMatch: "accessibility.spec.mjs",
fullyParallel: false,
workers: 1,
retries: 0,
reporter: "line",
outputDir: "/tmp/docforge-playwright-accessibility",
timeout: 30_000,
expect: {
timeout: 5_000,
},
use: {
browserName: "chromium",
bypassCSP: true,
headless: true,
viewport: {
width: 1440,
height: 1000,
},
},
});

38
schemas/client-configuration.schema.json
View file

@ -127,6 +127,17 @@
},
"additionalProperties": false
},
"projection_policy": {
"type": "object",
"required": ["schema_version", "manual", "portable_graph", "live_viewer"],
"properties": {
"schema_version": { "const": 2 },
"manual": { "enum": ["auto", "explicit", "disabled"] },
"portable_graph": { "enum": ["explicit", "disabled"] },
"live_viewer": { "enum": ["on-demand", "disabled"] }
},
"additionalProperties": false
},
"diagnostics": {
"type": "object",
"required": [
@ -211,6 +222,9 @@
"project",
"binding",
"effective_policy",
"projection_policy",
"projection_policy_hash",
"projection_availability",
"artifact",
"configuration_hash",
"warnings"
@ -231,7 +245,8 @@
"project_id",
"project_root",
"project_root_fingerprint",
"adapter"
"adapter",
"descriptor_hash"
],
"properties": {
"project_id": { "type": "string", "minLength": 1 },
@ -240,7 +255,8 @@
"type": "string",
"pattern": "^[0-9a-f]{16}$"
},
"adapter": { "type": "string", "minLength": 1 }
"adapter": { "type": "string", "minLength": 1 },
"descriptor_hash": { "$ref": "#/$defs/sha256" }
},
"additionalProperties": false
},
@ -317,6 +333,24 @@
"additionalProperties": false
},
"effective_policy": { "$ref": "#/$defs/effective_policy" },
"projection_policy": { "$ref": "#/$defs/projection_policy" },
"projection_policy_hash": { "$ref": "#/$defs/sha256" },
"projection_availability": {
"type": "object",
"required": [
"manual_configured",
"portable_graph_configured",
"application_enabled",
"live_viewer_available"
],
"properties": {
"manual_configured": { "type": "boolean" },
"portable_graph_configured": { "type": "boolean" },
"application_enabled": { "type": "boolean" },
"live_viewer_available": { "const": true }
},
"additionalProperties": false
},
"artifact": {
"type": "object",
"required": [

19
schemas/projection-policy.schema.json Normal file
View file

@ -0,0 +1,19 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://docforge.local/schema/projection-policy-v2.json",
"title": "DocForge independent projection policy",
"type": "object",
"required": [
"schema_version",
"manual",
"portable_graph",
"live_viewer"
],
"properties": {
"schema_version": { "const": 2 },
"manual": { "enum": ["auto", "explicit", "disabled"] },
"portable_graph": { "enum": ["explicit", "disabled"] },
"live_viewer": { "enum": ["on-demand", "disabled"] }
},
"additionalProperties": false
}

3
schemas/result.schema.json
View file

@ -20,6 +20,7 @@
"test",
"benchmark.m1",
"benchmark.m2",
"benchmark.m3",
"mcp.invoke",
"mcp.bootstrap",
"mcp.sync",
@ -37,6 +38,8 @@
"mcp.generation_diff",
"mcp.validate_project",
"mcp.render_status",
"mcp.graph_plan",
"mcp.graph_render_status",
"mcp.visualize",
"mcp.visualization_status",
"mcp.stop_visualization",

21
src/docforge/application.py
View file

@ -15,6 +15,7 @@ from .changesets import ChangesetStore
from .errors import DocForgeError
from .index import ProjectIndex
from .models import Node, ProjectService, ProjectSnapshot
from .projection_policy import ManualProjectionMode, validate_manual_projection_mode
from .rendering import RenderService
@ -353,13 +354,19 @@ class CanonicalApplicationService:
applier_id: str | None,
applier: CanonicalApplier | None,
index: ProjectIndex | None = None,
manual_policy: ManualProjectionMode = "auto",
) -> None:
self.project = project
self.applier_id = applier_id
self.applier = applier
self.changesets = ChangesetStore(project, applier_id)
self.index = index or ProjectIndex(project)
self.rendering = RenderService(project, self.changesets)
self.manual_policy = validate_manual_projection_mode(manual_policy)
self.rendering = RenderService(
project,
self.changesets,
manual_policy=self.manual_policy,
)
@property
def enabled(self) -> bool:
@ -402,7 +409,9 @@ class CanonicalApplicationService:
)
renders: list[dict[str, object]] = []
config = self.project.descriptor.render
if config is not None:
render_action = "not_configured"
if config is not None and self.manual_policy == "auto":
render_action = "rendered"
for view in config.views:
try:
rendered = self.rendering.render(view.view_id)
@ -435,6 +444,10 @@ class CanonicalApplicationService:
"error": error.as_dict(),
}
)
elif config is not None:
render_action = (
"skipped_explicit" if self.manual_policy == "explicit" else "skipped_disabled"
)
return {
**applied,
"derived_refresh": {
@ -442,6 +455,10 @@ class CanonicalApplicationService:
"index": index_result,
"check": index_check,
"renders": renders,
"render_policy": {
"mode": self.manual_policy,
"action": render_action,
},
"errors": refresh_errors,
},
}

2
src/docforge/assets/graph.html
View file

@ -95,7 +95,7 @@
aria-label="Visible relationship color and symbol key"></ul>
</details>
<svg id="graph" viewBox="-600 -410 1200 820"
role="img" aria-label="Node neighborhood"></svg>
role="group" aria-label="Interactive node neighborhood"></svg>
<div class="empty" id="empty">Search for a node to inspect its neighborhood.</div>
<div class="connection-state" id="connection-state" role="alert" hidden>
<strong>Visualization disconnected</strong>

106
src/docforge/cli.py
View file

@ -17,6 +17,7 @@ from .graph_rendering import GraphRenderService
from .index import ProjectIndex
from .onboarding import assess_project, scaffold_project
from .project import Project, project_root_fingerprint
from .projection_policy import compose_projection_policy
from .rendering import RenderService
from .telemetry import request
from .viewer_manager import ViewerManagerClient
@ -30,6 +31,18 @@ def _parser() -> argparse.ArgumentParser:
action="store_true",
help="Attach bounded request-local stage timings and counters",
)
parser.add_argument(
"--manual-render-policy",
choices=("auto", "explicit", "disabled"),
)
parser.add_argument(
"--portable-graph-policy",
choices=("explicit", "disabled"),
)
parser.add_argument(
"--live-viewer-policy",
choices=("on-demand", "disabled"),
)
commands = parser.add_subparsers(dest="command", required=True)
configure = commands.add_parser("configure")
configure.add_argument("client", choices=CLIENT_NAMES)
@ -43,6 +56,21 @@ def _parser() -> argparse.ArgumentParser:
configure.add_argument("--proposal-writer")
configure.add_argument("--canonical-applier")
configure.add_argument("--no-ast", action="store_true")
configure.add_argument(
"--manual-render-policy",
choices=("auto", "explicit", "disabled"),
default=argparse.SUPPRESS,
)
configure.add_argument(
"--portable-graph-policy",
choices=("explicit", "disabled"),
default=argparse.SUPPRESS,
)
configure.add_argument(
"--live-viewer-policy",
choices=("on-demand", "disabled"),
default=argparse.SUPPRESS,
)
configure.add_argument("--startup-timeout", type=int, default=30)
configure.add_argument("--tool-timeout", type=int, default=300)
configure.add_argument("--output", type=Path)
@ -131,6 +159,9 @@ def _run(arguments: argparse.Namespace) -> dict[str, object]:
proposal_writer=arguments.proposal_writer,
canonical_applier=arguments.canonical_applier,
no_ast=arguments.no_ast,
manual_render_policy=arguments.manual_render_policy,
portable_graph_policy=arguments.portable_graph_policy,
live_viewer_policy=arguments.live_viewer_policy,
startup_timeout=arguments.startup_timeout,
tool_timeout=arguments.tool_timeout,
output=arguments.output,
@ -159,12 +190,39 @@ def _run(arguments: argparse.Namespace) -> dict[str, object]:
content_root=arguments.content_root,
)
project = Project.open(arguments.project_root)
projection_policy = compose_projection_policy(
manual=arguments.manual_render_policy,
portable_graph=arguments.portable_graph_policy,
live_viewer=arguments.live_viewer_policy,
manual_configured=project.descriptor.render is not None,
portable_graph_configured=project.descriptor.graph_render is not None,
application_enabled=False,
)
build = ProjectIndex(project).build()
render = RenderService(project).render("manual")
render = (
{
"status": "ok",
"state": "skipped",
"reason": "projection_policy_disabled",
}
if projection_policy.manual == "disabled"
else RenderService(
project,
manual_policy=projection_policy.manual,
).render("manual")
)
return {**scaffold, "build": build, "render": render}
return assess_project(arguments.project_root, requested_languages=languages)
project = Project.open(arguments.project_root)
index = ProjectIndex(project)
projection_policy = compose_projection_policy(
manual=arguments.manual_render_policy,
portable_graph=arguments.portable_graph_policy,
live_viewer=arguments.live_viewer_policy,
manual_configured=project.descriptor.render is not None,
portable_graph_configured=project.descriptor.graph_render is not None,
application_enabled=arguments.command == "apply",
)
if arguments.command == "info":
snapshot = project.load()
return {
@ -257,30 +315,52 @@ def _run(arguments: argparse.Namespace) -> dict[str, object]:
cursor=arguments.cursor,
)
if arguments.command == "render":
return RenderService(project).render(arguments.view_id)
return RenderService(
project,
manual_policy=projection_policy.manual,
).render(arguments.view_id)
if arguments.command == "render-status":
rendering = RenderService(project)
rendering = RenderService(
project,
manual_policy=projection_policy.manual,
)
return (
rendering.deep_status(arguments.view_id)
if arguments.deep
else rendering.status(arguments.view_id)
)
if arguments.command == "graph-plan":
return GraphRenderService(project).plan(arguments.view_id)
return GraphRenderService(
project,
portable_graph_policy=projection_policy.portable_graph,
).plan(arguments.view_id)
if arguments.command == "graph-render":
return GraphRenderService(project).render(arguments.view_id)
return GraphRenderService(
project,
portable_graph_policy=projection_policy.portable_graph,
).render(arguments.view_id)
if arguments.command == "graph-render-status":
return GraphRenderService(project).status(arguments.view_id)
return GraphRenderService(
project,
portable_graph_policy=projection_policy.portable_graph,
).status(arguments.view_id)
if arguments.command == "preview":
return RenderService(project).preview(arguments.changeset_id, arguments.view_id)
return RenderService(
project,
manual_policy=projection_policy.manual,
).preview(arguments.changeset_id, arguments.view_id)
if arguments.command == "apply":
return CanonicalApplicationService(
project,
applier_id=arguments.applier,
applier=GenericCanonicalApplier(project),
manual_policy=projection_policy.manual,
).apply(arguments.changeset_id, arguments.changeset_hash)
if arguments.command == "visualize":
visualization = ViewerManagerClient(index).start(
visualization = ViewerManagerClient(
index,
live_viewer_policy=projection_policy.live_viewer,
).start(
node_id=arguments.node,
query=arguments.query,
depth=arguments.depth,
@ -297,9 +377,15 @@ def _run(arguments: argparse.Namespace) -> dict[str, object]:
"visualization": visualization,
}
if arguments.command == "visualization-status":
return ViewerManagerClient(index).status()
return ViewerManagerClient(
index,
live_viewer_policy=projection_policy.live_viewer,
).status()
if arguments.command == "visualization-stop":
return ViewerManagerClient(index).stop()
return ViewerManagerClient(
index,
live_viewer_policy=projection_policy.live_viewer,
).stop()
raise DocForgeError("invalid_command", "Unknown command")

158
src/docforge/client_config.py
View file

@ -18,9 +18,10 @@ from typing import Literal, cast
from .changeset_contract import document_hash
from .errors import DocForgeError
from .models import ProjectService
from .models import ProjectDescriptor, ProjectService
from .policy import CapabilityMode, compose_effective_policy
from .project import project_root_fingerprint, validate_descriptor_binding
from .project import Project, project_root_fingerprint, validate_descriptor_binding
from .projection_policy import compose_projection_policy
ClientName = Literal["codex", "claude", "openclaw"]
CLIENT_NAMES: tuple[ClientName, ...] = ("codex", "claude", "openclaw")
@ -605,11 +606,37 @@ def _atomic_write(
os.close(directory_fd)
def _validate_configuration_result(result: dict[str, object]) -> None:
def _validate_configuration_result(
result: dict[str, object],
*,
trusted_descriptor: ProjectDescriptor | None = None,
) -> None:
artifact = cast(dict[str, object], result["artifact"])
binding = cast(dict[str, object], result["binding"])
policy = cast(dict[str, object], result["effective_policy"])
projection_policy = cast(dict[str, object], result["projection_policy"])
projection_availability = cast(
dict[str, object],
result["projection_availability"],
)
project = cast(dict[str, object], result["project"])
if trusted_descriptor is None:
try:
bound_descriptor = Project.open(cast(str, project["project_root"])).descriptor
except (DocForgeError, KeyError, TypeError) as error:
raise AssertionError(
"Generated client project binding cannot be independently validated"
) from error
else:
bound_descriptor = trusted_descriptor
if (
project["project_id"] != bound_descriptor.project_id
or project["project_root"] != str(bound_descriptor.root)
or project["project_root_fingerprint"] != project_root_fingerprint(bound_descriptor.root)
or project["adapter"] != bound_descriptor.adapter
or project["descriptor_hash"] != bound_descriptor.descriptor_hash
):
raise AssertionError("Generated client project binding drifted")
content = cast(str, artifact["content"])
if artifact["content_sha256"] != hashlib.sha256(content.encode("utf-8")).hexdigest():
raise AssertionError("Generated client content hash drifted")
@ -644,6 +671,29 @@ def _validate_configuration_result(result: dict[str, object]) -> None:
if remaining[-1:] != ["--no-ast"] or arguments.count("--no-ast") != 1:
raise AssertionError("Generated no-AST argument layout drifted")
remaining = remaining[:-1]
projection_arguments: dict[str, str] = {}
authority_arguments: list[str] = []
position = 0
projection_options = {
"--manual-render-policy": "manual",
"--portable-graph-policy": "portable_graph",
"--live-viewer-policy": "live_viewer",
}
while position < len(remaining):
option = remaining[position]
field = projection_options.get(option)
if field is None:
authority_arguments.append(option)
position += 1
continue
if position + 1 >= len(remaining) or option in projection_arguments:
raise AssertionError("Generated projection policy argument layout drifted")
value = remaining[position + 1]
projection_arguments[option] = value
if projection_policy[field] != value:
raise AssertionError("Generated projection policy argument drifted")
position += 2
remaining = authority_arguments
mode = binding["capability_mode"]
if (
(mode == "read" and remaining)
@ -663,6 +713,34 @@ def _validate_configuration_result(result: dict[str, object]) -> None:
)
):
raise AssertionError("Generated authority argument layout drifted")
expected_projection_policy = compose_projection_policy(
manual=projection_arguments.get("--manual-render-policy"),
portable_graph=projection_arguments.get("--portable-graph-policy"),
live_viewer=projection_arguments.get("--live-viewer-policy"),
manual_configured=cast(bool, projection_availability["manual_configured"]),
portable_graph_configured=cast(
bool,
projection_availability["portable_graph_configured"],
),
application_enabled=cast(
bool,
projection_availability["application_enabled"],
),
live_viewer_available=cast(
bool,
projection_availability["live_viewer_available"],
),
)
if (
projection_policy != expected_projection_policy.as_dict()
or projection_availability["manual_configured"] != (render_policy["manual"] != "disabled")
or projection_availability["manual_configured"] != (bound_descriptor.render is not None)
or projection_availability["portable_graph_configured"]
!= (bound_descriptor.graph_render is not None)
or projection_availability["application_enabled"] != (mode == "application")
or projection_availability["live_viewer_available"] is not True
):
raise AssertionError("Generated projection policy drifted from its availability")
composed_policy = compose_effective_policy(
selected_mode=cast(CapabilityMode, mode),
capability_source="explicit",
@ -698,6 +776,18 @@ def _validate_configuration_result(result: dict[str, object]) -> None:
)
):
raise AssertionError("Generated client policy drifted from its binding")
if (
result["projection_policy_hash"]
!= hashlib.sha256(
json.dumps(
projection_policy,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
).hexdigest()
):
raise AssertionError("Generated projection policy hash drifted")
expected_hash = document_hash(
{
"schema_version": 1,
@ -706,6 +796,9 @@ def _validate_configuration_result(result: dict[str, object]) -> None:
"project": project,
"binding": binding,
"effective_policy": policy,
"projection_policy": projection_policy,
"projection_policy_hash": result["projection_policy_hash"],
"projection_availability": projection_availability,
"artifact_format": artifact["format"],
"artifact_content_sha256": artifact["content_sha256"],
}
@ -723,6 +816,9 @@ def generate_client_configuration(
proposal_writer: str | None = None,
canonical_applier: str | None = None,
no_ast: bool = False,
manual_render_policy: str | None = None,
portable_graph_policy: str | None = None,
live_viewer_policy: str | None = None,
startup_timeout: int = 30,
tool_timeout: int = 300,
output: Path | None = None,
@ -839,9 +935,6 @@ def generate_client_configuration(
arguments.extend(("--proposal-writer", proposal_writer))
if canonical_applier is not None:
arguments.extend(("--canonical-applier", canonical_applier))
if no_ast:
arguments.append("--no-ast")
policy = compose_effective_policy(
selected_mode=selected_mode,
capability_source="explicit",
@ -850,6 +943,40 @@ def generate_client_configuration(
render_configured=descriptor.render is not None,
application_enabled=canonical_applier is not None,
)
projection_policy = compose_projection_policy(
manual=manual_render_policy,
portable_graph=portable_graph_policy,
live_viewer=live_viewer_policy,
manual_configured=descriptor.render is not None,
portable_graph_configured=descriptor.graph_render is not None,
application_enabled=canonical_applier is not None,
)
default_projection_policy = compose_projection_policy(
manual_configured=descriptor.render is not None,
portable_graph_configured=descriptor.graph_render is not None,
application_enabled=canonical_applier is not None,
)
for option, selected, default in (
(
"--manual-render-policy",
projection_policy.manual,
default_projection_policy.manual,
),
(
"--portable-graph-policy",
projection_policy.portable_graph,
default_projection_policy.portable_graph,
),
(
"--live-viewer-policy",
projection_policy.live_viewer,
default_projection_policy.live_viewer,
),
):
if selected != default:
arguments.extend((option, selected))
if no_ast:
arguments.append("--no-ast")
artifact_format, content, warning = _artifact(
selected_client,
server_name=selected_name,
@ -902,6 +1029,7 @@ def generate_client_configuration(
"project_root": str(descriptor.root),
"project_root_fingerprint": fingerprint,
"adapter": descriptor.adapter,
"descriptor_hash": descriptor.descriptor_hash,
}
policy_payload = policy.as_dict()
plan_hash = document_hash(
@ -912,6 +1040,14 @@ def generate_client_configuration(
"project": project_binding,
"binding": binding,
"effective_policy": policy_payload,
"projection_policy": projection_policy.as_dict(),
"projection_policy_hash": projection_policy.policy_hash,
"projection_availability": {
"manual_configured": descriptor.render is not None,
"portable_graph_configured": descriptor.graph_render is not None,
"application_enabled": canonical_applier is not None,
"live_viewer_available": True,
},
"artifact_format": artifact_format,
"artifact_content_sha256": artifact["content_sha256"],
}
@ -926,6 +1062,14 @@ def generate_client_configuration(
"project": project_binding,
"binding": binding,
"effective_policy": policy_payload,
"projection_policy": projection_policy.as_dict(),
"projection_policy_hash": projection_policy.policy_hash,
"projection_availability": {
"manual_configured": descriptor.render is not None,
"portable_graph_configured": descriptor.graph_render is not None,
"application_enabled": canonical_applier is not None,
"live_viewer_available": True,
},
"artifact": artifact,
"configuration_hash": plan_hash,
"warnings": [
@ -933,5 +1077,5 @@ def generate_client_configuration(
*([] if publication_warning is None else [{"code": publication_warning}]),
],
}
_validate_configuration_result(result)
_validate_configuration_result(result, trusted_descriptor=descriptor)
return result

17
src/docforge/doctor.py
View file

@ -20,6 +20,7 @@ from .project import (
project_root_fingerprint,
validate_descriptor_binding,
)
from .projection_policy import compose_projection_policy
MAX_CLIENT_CONFIG_BYTES = 1_000_000
MAX_CLIENT_SERVERS = 256
@ -614,6 +615,9 @@ def _parse_binding(arguments: list[str]) -> dict[str, object]:
"--proposal-writer",
"--canonical-applier",
"--capability-mode",
"--manual-render-policy",
"--portable-graph-policy",
"--live-viewer-policy",
}
flag_options = {"--no-ast", "--diagnostics"}
position = 0
@ -662,6 +666,9 @@ def _parse_binding(arguments: list[str]) -> dict[str, object]:
"capability_mode_implicit": implicit,
"no_ast": "--no-ast" in flags,
"diagnostics": "--diagnostics" in flags,
"manual_render_policy": values.get("--manual-render-policy"),
"portable_graph_policy": values.get("--portable-graph-policy"),
"live_viewer_policy": values.get("--live-viewer-policy"),
}
@ -749,6 +756,16 @@ def _runtime_policy_check(
canonical_applier is not None and selected_mode in {"application", "operator"}
),
)
compose_projection_policy(
manual=cast(str | None, binding["manual_render_policy"]),
portable_graph=cast(str | None, binding["portable_graph_policy"]),
live_viewer=cast(str | None, binding["live_viewer_policy"]),
manual_configured=project.descriptor.render is not None,
portable_graph_configured=project.descriptor.graph_render is not None,
application_enabled=(
canonical_applier is not None and selected_mode in {"application", "operator"}
),
)
if cast(bool, binding["capability_mode_implicit"]):
return (
"warning",

30
src/docforge/graph_rendering.py
View file

@ -34,6 +34,11 @@ from .models import (
)
from .project import project_root_fingerprint
from .projection_contract import GraphViewPlanV1, ProjectionReceiptV1, projection_hash
from .projection_policy import (
PortableGraphProjectionMode,
validate_portable_graph_projection_mode,
)
from .projection_worker import render_projection_in_worker
GRAPH_RENDERER_ID = "portable_graph_html"
GRAPH_RENDERER_VERSION = "1"
@ -45,11 +50,29 @@ MAX_GRAPH_PUBLICATION_BYTES = 256_000
class GraphRenderService:
"""Publish one declared artifact while keeping planning and rendering independent."""
def __init__(self, project: ProjectService, *, allow_logic: bool = False) -> None:
def __init__(
self,
project: ProjectService,
*,
allow_logic: bool = False,
portable_graph_policy: PortableGraphProjectionMode = "explicit",
) -> None:
self.project = project
self.allow_logic = allow_logic
self.portable_graph_policy = validate_portable_graph_projection_mode(portable_graph_policy)
def _require_rendering(self, operation: str) -> None:
if self.portable_graph_policy == "disabled":
raise DocForgeError(
"projection_policy_forbids_operation",
"Portable graph projection policy disables rendering work",
projection="portable_graph",
mode=self.portable_graph_policy,
operation=operation,
)
def plan(self, view_id: str) -> dict[str, object]:
self._require_rendering("plan")
snapshot = self.project.load()
view = self._view(self._config(snapshot), view_id)
plan = self._plan(snapshot, view)
@ -93,6 +116,7 @@ class GraphRenderService:
)
def render(self, view_id: str) -> dict[str, object]:
self._require_rendering("render")
with self._lock():
current_status = self.status(view_id)
current_outputs = cast(list[dict[str, object]], current_status["outputs"])
@ -111,9 +135,7 @@ class GraphRenderService:
renderer_version=GRAPH_RENDERER_VERSION,
max_output_bytes=snapshot.descriptor.limits.max_render_bytes,
)
from docforge_renderers.graph import PortableGraphHtmlRenderer
result = PortableGraphHtmlRenderer().render(package)
result = render_projection_in_worker(package)
if len(result.artifacts) != 1:
raise DocForgeError(
"invalid_projection",

29
src/docforge/manual_projection.py
View file

@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
from collections import defaultdict
from collections.abc import Sequence
from .errors import DocForgeError
from .models import Edge, ProjectSnapshot, RenderView
@ -161,6 +162,8 @@ def build_manual_projection_package(
renderer_id: str,
renderer_version: str,
max_output_bytes: int,
render_identity: str | None = None,
fragment_records: Sequence[dict[str, object]] = (),
) -> ProjectionPackageV1:
"""Bind one plan and inert template asset for a path-free manual renderer."""
@ -168,6 +171,23 @@ def build_manual_projection_package(
template = template_bytes.decode("utf-8")
except UnicodeDecodeError as error:
raise DocForgeError("invalid_template", "Render template is not valid UTF-8") from error
template_asset: dict[str, object] = {
"asset_id": "manual.template",
"media_type": "text/html; charset=utf-8",
"sha256": hashlib.sha256(template_bytes).hexdigest(),
"text": template,
}
if render_identity is not None:
template_asset["render_identity"] = render_identity
assets = [template_asset]
if fragment_records:
assets.append(
{
"asset_id": "manual.fragments",
"media_type": "application/vnd.docforge.projection-fragments.v1+json",
"records": list(fragment_records),
}
)
return ProjectionPackageV1.create(
kind="manual",
plan=plan,
@ -176,14 +196,7 @@ def build_manual_projection_package(
{"component_id": "manual.document@1"},
{"component_id": "manual.commonmark@1"},
],
assets=[
{
"asset_id": "manual.template",
"media_type": "text/html; charset=utf-8",
"sha256": hashlib.sha256(template_bytes).hexdigest(),
"text": template,
}
],
assets=assets,
output_policy={
"artifact_ids": ["manual.html"],
"max_total_bytes": max_output_bytes,

109
src/docforge/mcp_server.py
View file

@ -15,11 +15,13 @@ from .application import CanonicalApplicationService, CanonicalApplier, GenericC
from .changesets import ChangesetStore
from .context import compile_context
from .errors import DocForgeError
from .graph_rendering import GraphRenderService
from .index import ProjectIndex
from .models import IncrementalStateProject, ProjectService, RuntimeValidatedProject
from .pagination import canonical_hash, decode_cursor, page_limit, page_receipt
from .policy import CapabilityMode, capability_mode, compose_effective_policy
from .project import Project, project_root_fingerprint
from .projection_policy import compose_projection_policy
from .rendering import RenderService
from .retrieval import MAX_TASK_EVIDENCE, TaskKind, build_retrieval_plan
from .telemetry import request, stage
@ -47,6 +49,8 @@ READ_TOOLS = (
"docforge_get_task_context",
"docforge_validate_project",
"docforge_render_status",
"docforge_graph_plan",
"docforge_graph_render_status",
"docforge_visualize",
"docforge_stop_visualization",
"docforge_visualization_status",
@ -134,6 +138,9 @@ class DocForgeService:
no_ast: bool = False,
diagnostics: bool = False,
capability_mode_name: str | None = None,
manual_projection_policy: str | None = None,
portable_graph_policy: str | None = None,
live_viewer_policy: str | None = None,
) -> None:
self.project = project
default_mode: CapabilityMode = (
@ -153,19 +160,40 @@ class DocForgeService:
render_configured=project.descriptor.render is not None,
application_enabled=application_enabled,
)
self.projection_policy = compose_projection_policy(
manual=manual_projection_policy,
portable_graph=portable_graph_policy,
live_viewer=live_viewer_policy,
manual_configured=project.descriptor.render is not None,
portable_graph_configured=project.descriptor.graph_render is not None,
application_enabled=application_enabled,
)
self.index = ProjectIndex(self.project, allow_logic=not self.policy.no_ast)
self.changesets = ChangesetStore(
self.project,
proposal_writer if selected_mode != "read" else None,
)
self.rendering = RenderService(self.project, self.changesets)
self.rendering = RenderService(
self.project,
self.changesets,
manual_policy=self.projection_policy.manual,
)
self.graph_rendering = GraphRenderService(
self.project,
allow_logic=not self.policy.no_ast,
portable_graph_policy=self.projection_policy.portable_graph,
)
self.application = CanonicalApplicationService(
self.project,
applier_id=canonical_applier_id if application_enabled else None,
applier=canonical_applier if application_enabled else None,
index=self.index,
manual_policy=self.projection_policy.manual,
)
self.visualization = ViewerManagerClient(
self.index,
live_viewer_policy=self.projection_policy.live_viewer,
)
self.visualization = ViewerManagerClient(self.index)
self.context_provider = context_provider
self.task_context_available = context_provider is compile_context
self.binding_metadata = dict(binding_metadata or {})
@ -621,6 +649,7 @@ class DocForgeService:
}
capabilities = self.capabilities()
effective_policy = self.policy.as_dict()
projection_policy = self.projection_policy.as_dict()
session_contract: dict[str, object] = {
"schema_version": 1,
"binding": binding,
@ -630,6 +659,8 @@ class DocForgeService:
"freshness": "current",
},
"effective_policy": effective_policy,
"projection_policy": projection_policy,
"projection_policy_hash": self.projection_policy.policy_hash,
"capabilities": capabilities,
"render_policies": {
"manual": effective_policy["manual_render"],
@ -652,6 +683,8 @@ class DocForgeService:
"canonical_paths": [str(path) for path in descriptor.content_roots],
"adapter_policy": self.adapter_policy(),
"effective_policy": effective_policy,
"projection_policy": projection_policy,
"projection_policy_hash": self.projection_policy.policy_hash,
"capabilities": capabilities,
"session_contract": session_contract,
"proposal_access": proposal_access,
@ -714,6 +747,8 @@ class DocForgeService:
),
"adapter_policy": self.adapter_policy(),
"effective_policy": self.policy.as_dict(),
"projection_policy": self.projection_policy.as_dict(),
"projection_policy_hash": self.projection_policy.policy_hash,
"capabilities": self.capabilities(),
"canonical_paths": [
*(relative(path) for path in snapshot.descriptor.content_roots),
@ -829,6 +864,29 @@ class DocForgeService:
operation_name="mcp.render_status",
)
def graph_plan(self, view_id: str) -> dict[str, object]:
"""Plan one declared portable graph without publishing derived output."""
return self.invoke(
lambda: self.graph_rendering.plan(view_id),
synchronize=False,
load_error_identity=False,
operation_name="mcp.graph_plan",
)
def graph_render_status(
self,
view_id: str | None = None,
) -> dict[str, object]:
"""Report portable-graph publication state without planning or rendering."""
return self.invoke(
lambda: self.graph_rendering.status(view_id),
synchronize=False,
load_error_identity=False,
operation_name="mcp.graph_render_status",
)
def context(
self,
profile: str,
@ -1575,6 +1633,18 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC
return service.render_status(view_id, deep=deep)
@server.tool(name="docforge_graph_plan")
def graph_plan(view_id: str) -> dict[str, Any]:
"""Plan one declared portable graph without publishing derived output."""
return service.graph_plan(view_id)
@server.tool(name="docforge_graph_render_status")
def graph_render_status(view_id: str | None = None) -> dict[str, Any]:
"""Report portable-graph publication state without rendering."""
return service.graph_render_status(view_id)
@server.tool(name="docforge_visualize")
def visualize(
node_id: str | None = None,
@ -1622,6 +1692,8 @@ def _create_bound_server(service: DocForgeService, *, read_only: bool) -> FastMC
get_task_context,
validate_project,
render_status,
graph_plan,
graph_render_status,
visualize,
stop_visualization,
visualization_status,
@ -2004,6 +2076,9 @@ def create_server(
no_ast: bool = False,
diagnostics: bool = False,
capability_mode: str | None = None,
manual_projection_policy: str | None = None,
portable_graph_policy: str | None = None,
live_viewer_policy: str | None = None,
) -> FastMCP:
project = Project.open(project_root)
return create_project_server(
@ -2020,6 +2095,9 @@ def create_server(
no_ast=no_ast,
diagnostics=diagnostics,
capability_mode=capability_mode,
manual_projection_policy=manual_projection_policy,
portable_graph_policy=portable_graph_policy,
live_viewer_policy=live_viewer_policy,
)
@ -2034,6 +2112,9 @@ def create_project_server(
no_ast: bool = False,
diagnostics: bool = False,
capability_mode: str | None = None,
manual_projection_policy: str | None = None,
portable_graph_policy: str | None = None,
live_viewer_policy: str | None = None,
) -> FastMCP:
"""Create the full fixed MCP surface for one explicitly configured project service."""
@ -2047,6 +2128,9 @@ def create_project_server(
no_ast=no_ast,
diagnostics=diagnostics,
capability_mode_name=capability_mode,
manual_projection_policy=manual_projection_policy,
portable_graph_policy=portable_graph_policy,
live_viewer_policy=live_viewer_policy,
)
return _create_bound_server(
service,
@ -2062,6 +2146,9 @@ def create_read_only_server(
no_ast: bool = False,
diagnostics: bool = False,
capability_mode: str | None = None,
manual_projection_policy: str | None = None,
portable_graph_policy: str | None = None,
live_viewer_policy: str | None = None,
) -> FastMCP:
"""Create an adapter-capable MCP server exposing only the fixed read tool surface."""
@ -2073,6 +2160,9 @@ def create_read_only_server(
no_ast=no_ast,
diagnostics=diagnostics,
capability_mode_name="read" if capability_mode is None else capability_mode,
manual_projection_policy=manual_projection_policy,
portable_graph_policy=portable_graph_policy,
live_viewer_policy=live_viewer_policy,
)
if service.policy.capability_mode != "read":
raise DocForgeError(
@ -2106,6 +2196,18 @@ def main() -> None:
choices=("read", "proposal", "application", "operator"),
help="Expose the versioned project-bound capability surface",
)
parser.add_argument(
"--manual-render-policy",
choices=("auto", "explicit", "disabled"),
)
parser.add_argument(
"--portable-graph-policy",
choices=("explicit", "disabled"),
)
parser.add_argument(
"--live-viewer-policy",
choices=("on-demand", "disabled"),
)
arguments = parser.parse_args()
create_server(
arguments.project_root,
@ -2114,6 +2216,9 @@ def main() -> None:
no_ast=arguments.no_ast,
diagnostics=arguments.diagnostics,
capability_mode=arguments.capability_mode,
manual_projection_policy=arguments.manual_render_policy,
portable_graph_policy=arguments.portable_graph_policy,
live_viewer_policy=arguments.live_viewer_policy,
).run(transport="stdio")

1
src/docforge/project.py
View file

@ -550,7 +550,6 @@ def _load_descriptor(root: Path) -> ProjectDescriptor:
for field in defaults.__dataclass_fields__
}
)
render = load_render_config(
root,
document.get("render"),

489
src/docforge/projection_fragments.py Normal file
View file

@ -0,0 +1,489 @@
"""Bounded, path-free, disposable projection fragment caching."""
from __future__ import annotations
import base64
import hashlib
import json
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Literal, cast
from ._fs_safety import (
atomic_replace_bytes_at,
open_confined_directory,
read_bounded_file_at,
require_bound_directory,
)
from .errors import DocForgeError
from .projection_contract import canonical_projection_bytes, projection_hash
FRAGMENT_SCHEMA_VERSION = 1
FRAGMENT_KEY_CONTRACT = "docforge.projection-fragment-key"
FRAGMENT_RECORD_CONTRACT = "docforge.projection-fragment-record"
FRAGMENT_CACHE_DIRECTORY = "projection-fragments-v1"
MAX_FRAGMENT_CONTENT_BYTES = 4_000_000
MAX_FRAGMENT_ID_CHARS = 256
MAX_FRAGMENT_CACHE_ENTRIES = 10_000
MAX_FRAGMENT_CACHE_BYTES = 64_000_000
_MAX_RECORD_OVERHEAD_BYTES = 8_192
ProjectionKind = Literal["manual", "graph"]
def _is_hash(value: object) -> bool:
return (
isinstance(value, str)
and len(value) == 64
and all(character in "0123456789abcdef" for character in value)
)
def _version_string(value: object, *, field: str) -> str:
if (
not isinstance(value, str)
or not value
or value != value.strip()
or len(value) > MAX_FRAGMENT_ID_CHARS
or any(ord(character) < 32 for character in value)
):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment identity is invalid",
field=field,
)
return value
def fragment_semantic_hash(value: object) -> str:
"""Hash one complete semantic input using the projection canonical JSON form."""
try:
return hashlib.sha256(canonical_projection_bytes(value)).hexdigest()
except (TypeError, ValueError) as error:
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment semantic input is not canonical JSON",
) from error
@dataclass(frozen=True)
class FragmentKey:
"""Versioned identity for one renderer component's complete semantics."""
projection_kind: ProjectionKind
renderer_id: str
renderer_version: str
component_version: str
semantic_input_hash: str
key_id: str
@classmethod
def create(
cls,
*,
projection_kind: ProjectionKind,
renderer_id: str,
renderer_version: str,
component_version: str,
semantic_input_hash: str,
) -> FragmentKey:
body = cls._body(
projection_kind=projection_kind,
renderer_id=renderer_id,
renderer_version=renderer_version,
component_version=component_version,
semantic_input_hash=semantic_input_hash,
)
return cls._from_validated({**body, "key_id": projection_hash(body)})
@classmethod
def from_dict(cls, value: object) -> FragmentKey:
if not isinstance(value, dict):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment key is invalid",
)
return cls._from_validated(dict(cast(dict[str, object], value)))
@staticmethod
def _body(
*,
projection_kind: object,
renderer_id: object,
renderer_version: object,
component_version: object,
semantic_input_hash: object,
) -> dict[str, object]:
if projection_kind not in {"manual", "graph"}:
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment kind is invalid",
)
if not _is_hash(semantic_input_hash):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment semantic input hash is invalid",
)
return {
"schema_version": FRAGMENT_SCHEMA_VERSION,
"contract": FRAGMENT_KEY_CONTRACT,
"projection_kind": projection_kind,
"renderer_id": _version_string(renderer_id, field="renderer_id"),
"renderer_version": _version_string(
renderer_version,
field="renderer_version",
),
"component_version": _version_string(
component_version,
field="component_version",
),
"semantic_input_hash": semantic_input_hash,
}
@classmethod
def _from_validated(cls, value: dict[str, object]) -> FragmentKey:
required = {
"schema_version",
"contract",
"projection_kind",
"renderer_id",
"renderer_version",
"component_version",
"semantic_input_hash",
"key_id",
}
if (
set(value) != required
or value.get("schema_version") != FRAGMENT_SCHEMA_VERSION
or value.get("contract") != FRAGMENT_KEY_CONTRACT
):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment key contract is incompatible",
)
body = cls._body(
projection_kind=value.get("projection_kind"),
renderer_id=value.get("renderer_id"),
renderer_version=value.get("renderer_version"),
component_version=value.get("component_version"),
semantic_input_hash=value.get("semantic_input_hash"),
)
key_id = value.get("key_id")
if not _is_hash(key_id) or key_id != projection_hash(body):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment key does not match its semantics",
)
return cls(
projection_kind=cast(ProjectionKind, body["projection_kind"]),
renderer_id=cast(str, body["renderer_id"]),
renderer_version=cast(str, body["renderer_version"]),
component_version=cast(str, body["component_version"]),
semantic_input_hash=cast(str, body["semantic_input_hash"]),
key_id=cast(str, key_id),
)
def as_dict(self) -> dict[str, object]:
return {
"schema_version": FRAGMENT_SCHEMA_VERSION,
"contract": FRAGMENT_KEY_CONTRACT,
"projection_kind": self.projection_kind,
"renderer_id": self.renderer_id,
"renderer_version": self.renderer_version,
"component_version": self.component_version,
"semantic_input_hash": self.semantic_input_hash,
"key_id": self.key_id,
}
@dataclass(frozen=True)
class FragmentRecord:
"""One path-free fragment payload with complete byte evidence."""
key: FragmentKey
content: bytes
byte_count: int
content_sha256: str
record_id: str
@classmethod
def create(cls, key: FragmentKey, content: bytes) -> FragmentRecord:
if len(content) > MAX_FRAGMENT_CONTENT_BYTES:
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment content is invalid or oversized",
maximum_bytes=MAX_FRAGMENT_CONTENT_BYTES,
)
body = cls._body(key, content)
return cls(
key=key,
content=content,
byte_count=len(content),
content_sha256=hashlib.sha256(content).hexdigest(),
record_id=projection_hash(body),
)
@classmethod
def from_dict(cls, value: object) -> FragmentRecord:
if not isinstance(value, dict):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment record is invalid",
)
document = dict(cast(dict[str, object], value))
required = {
"schema_version",
"contract",
"record_id",
"key",
"content_encoding",
"content",
"byte_count",
"content_sha256",
}
if (
set(document) != required
or document.get("schema_version") != FRAGMENT_SCHEMA_VERSION
or document.get("contract") != FRAGMENT_RECORD_CONTRACT
or document.get("content_encoding") != "base64"
):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment record contract is incompatible",
)
encoded = document.get("content")
if not isinstance(encoded, str):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment content encoding is invalid",
)
try:
content = base64.b64decode(encoded.encode("ascii"), validate=True)
except (UnicodeEncodeError, ValueError) as error:
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment content encoding is invalid",
) from error
if len(content) > MAX_FRAGMENT_CONTENT_BYTES:
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment content is oversized",
maximum_bytes=MAX_FRAGMENT_CONTENT_BYTES,
)
key = FragmentKey.from_dict(document.get("key"))
body = cls._body(key, content)
record_id = document.get("record_id")
if (
type(document.get("byte_count")) is not int
or document.get("byte_count") != len(content)
or document.get("content_sha256") != hashlib.sha256(content).hexdigest()
or not _is_hash(record_id)
or record_id != projection_hash(body)
):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment byte evidence is invalid",
)
return cls(
key=key,
content=content,
byte_count=len(content),
content_sha256=hashlib.sha256(content).hexdigest(),
record_id=cast(str, record_id),
)
@classmethod
def from_bytes(cls, raw: bytes) -> FragmentRecord:
try:
value: object = json.loads(raw)
except (UnicodeDecodeError, json.JSONDecodeError) as error:
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment record is not valid JSON",
) from error
record = cls.from_dict(value)
if record.to_bytes() != raw:
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment record is not canonically serialized",
)
return record
@staticmethod
def _body(key: FragmentKey, content: bytes) -> dict[str, object]:
return {
"schema_version": FRAGMENT_SCHEMA_VERSION,
"contract": FRAGMENT_RECORD_CONTRACT,
"key": key.as_dict(),
"content_encoding": "base64",
"content": base64.b64encode(content).decode("ascii"),
"byte_count": len(content),
"content_sha256": hashlib.sha256(content).hexdigest(),
}
def as_dict(self) -> dict[str, object]:
return {
**self._body(self.key, self.content),
"record_id": self.record_id,
}
def to_bytes(self) -> bytes:
return canonical_projection_bytes(self.as_dict())
class ProjectionFragmentCache:
"""Confined best-effort storage for immutable projection fragments."""
def __init__(
self,
project_root: Path,
cache_root: Path,
*,
maximum_content_bytes: int = MAX_FRAGMENT_CONTENT_BYTES,
) -> None:
if (
type(maximum_content_bytes) is not int
or not 1 <= maximum_content_bytes <= MAX_FRAGMENT_CONTENT_BYTES
):
raise DocForgeError(
"invalid_projection_fragment",
"Projection fragment cache byte limit is invalid",
maximum_bytes=MAX_FRAGMENT_CONTENT_BYTES,
)
self.project_root = project_root
self.cache_root = cache_root
self.fragment_root = cache_root / FRAGMENT_CACHE_DIRECTORY
self.maximum_content_bytes = maximum_content_bytes
@property
def maximum_record_bytes(self) -> int:
encoded = ((self.maximum_content_bytes + 2) // 3) * 4
return encoded + _MAX_RECORD_OVERHEAD_BYTES
def get(self, key: FragmentKey) -> FragmentRecord | None:
"""Return one exact compatible fragment, treating every cache defect as a miss."""
descriptor: int | None = None
try:
descriptor = self._open(create=False)
raw = read_bounded_file_at(
descriptor,
f"{key.key_id}.json",
self.maximum_record_bytes,
)
if raw is None:
return None
record = FragmentRecord.from_bytes(raw)
if record.key != key or record.byte_count > self.maximum_content_bytes:
return None
require_bound_directory(self.fragment_root, descriptor)
return record
except (DocForgeError, OSError):
return None
finally:
if descriptor is not None:
os.close(descriptor)
def put(self, key: FragmentKey, content: bytes) -> FragmentRecord | None:
"""Durably publish one fragment, returning ``None`` on disposable cache failure."""
if len(content) > self.maximum_content_bytes:
return None
try:
record = FragmentRecord.create(key, content)
except DocForgeError:
return None
raw = record.to_bytes()
if len(raw) > self.maximum_record_bytes:
return None
descriptor: int | None = None
try:
descriptor = self._open(create=True)
name = f"{key.key_id}.json"
try:
existing = read_bounded_file_at(
descriptor,
name,
self.maximum_record_bytes,
)
except DocForgeError as error:
if error.code == "path_escape":
return None
existing = None
if existing == raw:
require_bound_directory(self.fragment_root, descriptor)
return record
atomic_replace_bytes_at(
self.fragment_root,
descriptor,
name,
raw,
verify=lambda: require_bound_directory(self.fragment_root, descriptor),
)
return record
except (DocForgeError, OSError):
return None
finally:
if descriptor is not None:
os.close(descriptor)
def prune(self, keep: tuple[FragmentKey, ...]) -> bool:
"""Remove every stale entry and prove the retained inventory is bounded."""
keep_ids = {key.key_id for key in keep}
if len(keep_ids) > MAX_FRAGMENT_CACHE_ENTRIES:
return False
descriptor: int | None = None
try:
descriptor = self._open(create=False)
retained_entries = 0
retained_bytes = 0
with os.scandir(descriptor) as entries:
for entry in entries:
name = entry.name
retained = (
len(name) == 69
and name.endswith(".json")
and name[:-5] in keep_ids
and all(character in "0123456789abcdef" for character in name[:-5])
)
if retained:
identity = entry.stat(follow_symlinks=False)
if not entry.is_file(follow_symlinks=False):
return False
retained_entries += 1
retained_bytes += identity.st_size
continue
try:
os.unlink(name, dir_fd=descriptor)
except OSError:
return False
if (
retained_entries > MAX_FRAGMENT_CACHE_ENTRIES
or retained_bytes > MAX_FRAGMENT_CACHE_BYTES
):
return False
require_bound_directory(self.fragment_root, descriptor)
os.fsync(descriptor)
return True
except (DocForgeError, OSError):
return False
finally:
if descriptor is not None:
os.close(descriptor)
def _open(self, *, create: bool) -> int:
if self.cache_root == self.project_root or not self.cache_root.is_relative_to(
self.project_root
):
raise DocForgeError(
"path_escape",
"Projection fragment cache is not confined to a derived project root",
)
return open_confined_directory(
self.project_root,
self.fragment_root,
create=create,
)

234
src/docforge/projection_policy.py Normal file
View file

@ -0,0 +1,234 @@
"""Independent version-2 policy for manual, portable graph, and live projections."""
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass
from typing import Literal, cast
from .errors import DocForgeError
ManualProjectionMode = Literal["auto", "explicit", "disabled"]
PortableGraphProjectionMode = Literal["explicit", "disabled"]
LiveViewerProjectionMode = Literal["on-demand", "disabled"]
MANUAL_PROJECTION_MODES: tuple[ManualProjectionMode, ...] = (
"auto",
"explicit",
"disabled",
)
PORTABLE_GRAPH_PROJECTION_MODES: tuple[PortableGraphProjectionMode, ...] = (
"explicit",
"disabled",
)
LIVE_VIEWER_PROJECTION_MODES: tuple[LiveViewerProjectionMode, ...] = (
"on-demand",
"disabled",
)
def _invalid_mode(field: str, value: object, allowed: tuple[str, ...]) -> DocForgeError:
return DocForgeError(
"invalid_projection_policy",
"Projection policy mode is unsupported",
projection=field,
mode=value,
allowed=list(allowed),
)
def _select_mode(
value: object | None,
*,
field: str,
default: str,
allowed: tuple[str, ...],
) -> str:
selected: object = default if value is None else value
if not isinstance(selected, str) or selected not in allowed:
raise _invalid_mode(field, selected, allowed)
return selected
def _require_boolean(field: str, value: object) -> bool:
if type(value) is not bool:
raise DocForgeError(
"invalid_projection_policy",
"Projection policy availability must be Boolean",
field=field,
)
return value
def _unavailable(field: str, mode: str, required: str) -> DocForgeError:
return DocForgeError(
"projection_policy_unavailable",
"Projection policy mode is unavailable",
projection=field,
mode=mode,
required=required,
)
def validate_manual_projection_mode(value: object) -> ManualProjectionMode:
"""Validate one direct manual-service policy selection."""
return cast(
ManualProjectionMode,
_select_mode(
value,
field="manual",
default="explicit",
allowed=MANUAL_PROJECTION_MODES,
),
)
def validate_portable_graph_projection_mode(
value: object,
) -> PortableGraphProjectionMode:
"""Validate one direct portable-graph service policy selection."""
return cast(
PortableGraphProjectionMode,
_select_mode(
value,
field="portable_graph",
default="explicit",
allowed=PORTABLE_GRAPH_PROJECTION_MODES,
),
)
def validate_live_viewer_projection_mode(value: object) -> LiveViewerProjectionMode:
"""Validate one direct live-viewer service policy selection."""
return cast(
LiveViewerProjectionMode,
_select_mode(
value,
field="live_viewer",
default="on-demand",
allowed=LIVE_VIEWER_PROJECTION_MODES,
),
)
@dataclass(frozen=True)
class ProjectionPolicyV2:
"""One immutable policy for three independent projection consumers."""
manual: ManualProjectionMode
portable_graph: PortableGraphProjectionMode
live_viewer: LiveViewerProjectionMode
def __post_init__(self) -> None:
if self.manual not in MANUAL_PROJECTION_MODES:
raise _invalid_mode("manual", self.manual, MANUAL_PROJECTION_MODES)
if self.portable_graph not in PORTABLE_GRAPH_PROJECTION_MODES:
raise _invalid_mode(
"portable_graph",
self.portable_graph,
PORTABLE_GRAPH_PROJECTION_MODES,
)
if self.live_viewer not in LIVE_VIEWER_PROJECTION_MODES:
raise _invalid_mode(
"live_viewer",
self.live_viewer,
LIVE_VIEWER_PROJECTION_MODES,
)
def as_dict(self) -> dict[str, object]:
return {
"schema_version": 2,
"manual": self.manual,
"portable_graph": self.portable_graph,
"live_viewer": self.live_viewer,
}
@property
def policy_hash(self) -> str:
raw = json.dumps(
self.as_dict(),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
return hashlib.sha256(raw).hexdigest()
def compose_projection_policy(
*,
manual: str | None = None,
portable_graph: str | None = None,
live_viewer: str | None = None,
manual_configured: bool,
portable_graph_configured: bool,
application_enabled: bool,
live_viewer_available: bool = True,
) -> ProjectionPolicyV2:
"""Compose compatible defaults with explicit availability-checked selections."""
manual_available = _require_boolean("manual_configured", manual_configured)
graph_available = _require_boolean(
"portable_graph_configured",
portable_graph_configured,
)
application_available = _require_boolean("application_enabled", application_enabled)
viewer_available = _require_boolean("live_viewer_available", live_viewer_available)
default_manual = (
"auto"
if manual_available and application_available
else ("explicit" if manual_available else "disabled")
)
default_graph = "explicit" if graph_available else "disabled"
default_viewer = "on-demand" if viewer_available else "disabled"
manual_mode = cast(
ManualProjectionMode,
_select_mode(
manual,
field="manual",
default=default_manual,
allowed=MANUAL_PROJECTION_MODES,
),
)
graph_mode = cast(
PortableGraphProjectionMode,
_select_mode(
portable_graph,
field="portable_graph",
default=default_graph,
allowed=PORTABLE_GRAPH_PROJECTION_MODES,
),
)
viewer_mode = cast(
LiveViewerProjectionMode,
_select_mode(
live_viewer,
field="live_viewer",
default=default_viewer,
allowed=LIVE_VIEWER_PROJECTION_MODES,
),
)
if manual_mode != "disabled" and not manual_available:
raise _unavailable("manual", manual_mode, "manual_render_config")
if manual_mode == "auto" and not application_available:
raise _unavailable("manual", manual_mode, "canonical_application")
if graph_mode == "explicit" and not graph_available:
raise _unavailable(
"portable_graph",
graph_mode,
"portable_graph_render_config",
)
if viewer_mode == "on-demand" and not viewer_available:
raise _unavailable("live_viewer", viewer_mode, "live_viewer_runtime")
return ProjectionPolicyV2(
manual=manual_mode,
portable_graph=graph_mode,
live_viewer=viewer_mode,
)

436
src/docforge/projection_worker.py Normal file
View file

@ -0,0 +1,436 @@
"""One-shot detached execution for the fixed built-in projection renderers."""
from __future__ import annotations
import base64
import binascii
import json
import os
import resource
import subprocess
import sys
import tempfile
from importlib.metadata import version
from typing import cast
from .errors import DocForgeError
from .projection_contract import (
MAX_PACKAGE_BYTES,
MAX_PROJECTION_ARTIFACTS,
MAX_RECEIPT_BYTES,
ProjectionArtifact,
ProjectionPackageV1,
ProjectionReceiptV1,
ProjectionRenderResult,
canonical_projection_bytes,
)
WORKER_PROTOCOL_VERSION = 1
MAX_WORKER_ARTIFACT_BYTES = 20_000_000
MAX_WORKER_REQUEST_BYTES = MAX_PACKAGE_BYTES + 1
MAX_WORKER_RESPONSE_BYTES = 4 * ((MAX_WORKER_ARTIFACT_BYTES + 2) // 3) + MAX_RECEIPT_BYTES + 256_000
WORKER_TIMEOUT_SECONDS = 30
_GENERIC_HTML_RENDERER_ID = "generic_html"
_PORTABLE_GRAPH_RENDERER_ID = "portable_graph_html"
_PORTABLE_GRAPH_RENDERER_VERSION = "1"
def _generic_html_renderer_version() -> str:
return f"1+markdown-it-py-{version('markdown-it-py')}"
def _worker_failure(message: str, **details: object) -> DocForgeError:
return DocForgeError("projection_worker_failure", message, **details)
def _renderer_identity(package: ProjectionPackageV1) -> dict[str, object]:
renderer_value = package.document.get("renderer")
if not isinstance(renderer_value, dict):
raise DocForgeError("unsupported_renderer", "Projection renderer identity is invalid")
renderer = cast(dict[str, object], renderer_value)
if set(renderer) != {"renderer_id", "renderer_version"}:
raise DocForgeError("unsupported_renderer", "Projection renderer identity is invalid")
renderer_id = renderer.get("renderer_id")
renderer_version = renderer.get("renderer_version")
if not isinstance(renderer_id, str) or not isinstance(renderer_version, str):
raise DocForgeError("unsupported_renderer", "Projection renderer identity is invalid")
supported = (
package.kind == "manual"
and renderer_id == _GENERIC_HTML_RENDERER_ID
and renderer_version == _generic_html_renderer_version()
) or (
package.kind == "graph"
and renderer_id == _PORTABLE_GRAPH_RENDERER_ID
and renderer_version == _PORTABLE_GRAPH_RENDERER_VERSION
)
if not supported:
raise DocForgeError(
"unsupported_renderer",
"Projection worker supports only the fixed built-in renderer versions",
)
return dict(renderer)
def _output_policy(package: ProjectionPackageV1) -> tuple[tuple[str, ...], int]:
policy_value = package.document.get("output_policy")
if not isinstance(policy_value, dict):
raise DocForgeError("invalid_projection", "Projection output policy is invalid")
policy = cast(dict[str, object], policy_value)
if set(policy) != {"artifact_ids", "max_total_bytes"}:
raise DocForgeError("invalid_projection", "Projection output policy is invalid")
artifact_ids_value = policy.get("artifact_ids")
maximum = policy.get("max_total_bytes")
if not isinstance(artifact_ids_value, list):
raise DocForgeError("invalid_projection", "Projection artifact inventory is invalid")
artifact_ids_objects = cast(list[object], artifact_ids_value)
if (
not artifact_ids_objects
or len(artifact_ids_objects) > MAX_PROJECTION_ARTIFACTS
or not all(
isinstance(artifact_id, str)
and bool(artifact_id)
and "/" not in artifact_id
and artifact_id not in {".", ".."}
for artifact_id in artifact_ids_objects
)
):
raise DocForgeError("invalid_projection", "Projection artifact inventory is invalid")
artifact_ids = tuple(cast(list[str], artifact_ids_objects))
if len(set(artifact_ids)) != len(artifact_ids):
raise DocForgeError("invalid_projection", "Projection artifact inventory is duplicated")
if type(maximum) is not int or maximum < 1:
raise DocForgeError(
"invalid_projection",
"Projection output byte allowance is invalid",
)
return artifact_ids, maximum
def _validated_package(package: object) -> ProjectionPackageV1:
if not isinstance(package, ProjectionPackageV1):
raise TypeError("Projection worker requires ProjectionPackageV1")
validated = ProjectionPackageV1.from_dict(package.as_dict())
_renderer_identity(validated)
_output_policy(validated)
return validated
def _validate_result(
package: ProjectionPackageV1,
result: ProjectionRenderResult,
*,
require_peak_memory: bool,
) -> ProjectionRenderResult:
artifact_ids, maximum = _output_policy(package)
if (
len(result.artifacts) != len(artifact_ids)
or tuple(artifact.artifact_id for artifact in result.artifacts) != artifact_ids
):
raise _worker_failure("Projection worker returned an invalid artifact inventory")
total_bytes = 0
evidence: list[dict[str, object]] = []
for artifact in result.artifacts:
if not artifact.media_type or type(artifact.content) is not bytes:
raise _worker_failure("Projection worker returned an invalid artifact")
total_bytes += len(artifact.content)
if total_bytes > maximum or total_bytes > MAX_WORKER_ARTIFACT_BYTES:
raise _worker_failure("Projection worker artifact transfer exceeded its fixed boundary")
evidence.append(artifact.evidence())
try:
receipt = ProjectionReceiptV1.from_dict(result.receipt.as_dict())
except DocForgeError as error:
raise _worker_failure("Projection worker receipt is invalid") from error
receipt_document = receipt.document
renderer = _renderer_identity(package)
timing_value = receipt_document.get("timing")
if not isinstance(timing_value, dict):
raise _worker_failure("Projection worker receipt timing is invalid")
timing = cast(dict[str, object], timing_value)
elapsed = timing.get("elapsed_ns")
peak_memory = receipt_document.get("peak_memory_bytes")
if (
receipt_document.get("kind") != package.kind
or receipt_document.get("package_id") != package.package_id
or receipt_document.get("plan_id") != package.document.get("plan_id")
or receipt_document.get("renderer") != renderer
or receipt_document.get("artifacts") != evidence
or type(elapsed) is not int
or elapsed < 0
or (require_peak_memory and (type(peak_memory) is not int or peak_memory <= 0))
):
raise _worker_failure("Projection worker receipt does not attest the requested package")
return ProjectionRenderResult(tuple(result.artifacts), receipt)
def _decode_canonical_line(raw: bytes, *, maximum: int, label: str) -> dict[str, object]:
if type(raw) is not bytes or len(raw) > maximum:
raise _worker_failure(f"{label} exceeded its fixed boundary", maximum_bytes=maximum)
if not raw or not raw.endswith(b"\n") or raw.count(b"\n") != 1:
raise _worker_failure(f"{label} framing is invalid")
payload = raw[:-1]
try:
value: object = json.loads(payload)
except (UnicodeDecodeError, json.JSONDecodeError) as error:
raise _worker_failure(f"{label} is not valid JSON") from error
if not isinstance(value, dict):
raise _worker_failure(f"{label} must be one JSON object")
document = cast(dict[str, object], value)
if canonical_projection_bytes(document) != payload:
raise _worker_failure(f"{label} is not canonical JSON")
return document
def _encode_request(package: ProjectionPackageV1) -> bytes:
encoded = canonical_projection_bytes(package.as_dict()) + b"\n"
if len(encoded) > MAX_WORKER_REQUEST_BYTES:
raise DocForgeError(
"projection_too_large",
"Projection worker request exceeds its fixed boundary",
maximum_bytes=MAX_WORKER_REQUEST_BYTES,
)
return encoded
def _invoke_worker(request: bytes) -> subprocess.CompletedProcess[bytes]:
environment = {key: os.environ[key] for key in ("SYSTEMROOT", "WINDIR") if key in os.environ}
environment.update(
{
"PYTHONIOENCODING": "utf-8",
"PYTHONUTF8": "1",
}
)
command = [sys.executable, "-I", "-m", "docforge.projection_worker"]
with tempfile.TemporaryFile() as output:
completed = subprocess.run(
command,
input=request,
stdout=output,
stderr=subprocess.DEVNULL,
check=False,
timeout=WORKER_TIMEOUT_SECONDS,
shell=False,
cwd=sys.prefix,
env=environment,
)
output.seek(0)
stdout = output.read(MAX_WORKER_RESPONSE_BYTES + 1)
return subprocess.CompletedProcess(
command,
completed.returncode,
stdout=stdout,
)
def _decode_response(package: ProjectionPackageV1, raw: bytes) -> ProjectionRenderResult:
document = _decode_canonical_line(
raw,
maximum=MAX_WORKER_RESPONSE_BYTES,
label="Projection worker response",
)
if (
set(document) != {"schema_version", "artifacts", "receipt"}
or document.get("schema_version") != WORKER_PROTOCOL_VERSION
):
raise _worker_failure("Projection worker response contract is invalid")
artifact_values = document.get("artifacts")
receipt_value = document.get("receipt")
if not isinstance(artifact_values, list) or not isinstance(receipt_value, dict):
raise _worker_failure("Projection worker response structure is invalid")
artifacts: list[ProjectionArtifact] = []
total_bytes = 0
for value in cast(list[object], artifact_values):
if not isinstance(value, dict):
raise _worker_failure("Projection worker artifact envelope is invalid")
artifact = cast(dict[str, object], value)
if set(artifact) != {"artifact_id", "media_type", "content_base64"}:
raise _worker_failure("Projection worker artifact envelope is invalid")
artifact_id = artifact.get("artifact_id")
media_type = artifact.get("media_type")
encoded = artifact.get("content_base64")
if (
not isinstance(artifact_id, str)
or not isinstance(media_type, str)
or not isinstance(encoded, str)
):
raise _worker_failure("Projection worker artifact envelope is invalid")
try:
content = base64.b64decode(encoded.encode("ascii"), validate=True)
except (UnicodeEncodeError, binascii.Error, ValueError) as error:
raise _worker_failure("Projection worker artifact encoding is invalid") from error
total_bytes += len(content)
if total_bytes > MAX_WORKER_ARTIFACT_BYTES:
raise _worker_failure("Projection worker artifact transfer exceeded its fixed boundary")
artifacts.append(ProjectionArtifact(artifact_id, media_type, content))
try:
receipt = ProjectionReceiptV1.from_dict(cast(dict[str, object], receipt_value))
except DocForgeError as error:
raise _worker_failure("Projection worker receipt is invalid") from error
return _validate_result(
package,
ProjectionRenderResult(tuple(artifacts), receipt),
require_peak_memory=True,
)
def render_projection_in_worker(package: ProjectionPackageV1) -> ProjectionRenderResult:
"""Render one validated path-free package in a fixed one-shot child process."""
validated = _validated_package(package)
request = _encode_request(validated)
try:
completed = _invoke_worker(request)
except subprocess.TimeoutExpired as error:
raise DocForgeError(
"projection_worker_timeout",
"Detached projection worker exceeded its fixed timeout",
timeout_seconds=WORKER_TIMEOUT_SECONDS,
) from error
except OSError as error:
raise _worker_failure("Detached projection worker could not be launched") from error
if completed.returncode != 0:
if completed.returncode == 3 and completed.stdout:
try:
failure = _decode_canonical_line(
completed.stdout,
maximum=MAX_RECEIPT_BYTES,
label="Projection worker error response",
)
error = failure.get("error")
error_document = cast(dict[str, object], error) if isinstance(error, dict) else None
if (
set(failure) == {"schema_version", "error"}
and failure.get("schema_version") == WORKER_PROTOCOL_VERSION
and error_document is not None
and set(error_document) == {"code", "message", "details"}
and isinstance(error_document.get("code"), str)
and bool(error_document["code"])
and isinstance(error_document.get("message"), str)
and bool(error_document["message"])
and isinstance(error_document.get("details"), dict)
):
raise DocForgeError(
cast(str, error_document["code"]),
cast(str, error_document["message"]),
**cast(dict[str, object], error_document["details"]),
)
except DocForgeError as error:
if error.code != "projection_worker_failure":
raise
if completed.returncode < 0:
raise _worker_failure(
"Detached projection worker terminated by signal",
signal=-completed.returncode,
)
raise _worker_failure(
"Detached projection worker exited unsuccessfully",
exit_code=completed.returncode,
)
if type(completed.stdout) is not bytes:
raise _worker_failure("Detached projection worker returned invalid output")
return _decode_response(validated, completed.stdout)
def _render_package(package: ProjectionPackageV1) -> ProjectionRenderResult:
renderer = _renderer_identity(package)
if renderer["renderer_id"] == _GENERIC_HTML_RENDERER_ID:
from docforge_renderers.manual import ManualHtmlRenderer
result = ManualHtmlRenderer(cast(str, renderer["renderer_version"])).render(package)
else:
from docforge_renderers.graph import PortableGraphHtmlRenderer
result = PortableGraphHtmlRenderer().render(package)
return _validate_result(package, result, require_peak_memory=False)
def _peak_memory_bytes() -> int:
peak = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
return max(1, peak if sys.platform == "darwin" else peak * 1024)
def _child_response(package: ProjectionPackageV1) -> bytes:
result = _render_package(package)
original = result.receipt.document
receipt = ProjectionReceiptV1.create(
kind=package.kind,
package_id=package.package_id,
plan_id=cast(str, package.document["plan_id"]),
renderer=cast(dict[str, object], original["renderer"]),
artifacts=[artifact.evidence() for artifact in result.artifacts],
diagnostics=cast(dict[str, object], original["diagnostics"]),
timing=cast(dict[str, object], original["timing"]),
peak_memory_bytes=_peak_memory_bytes(),
)
validated = _validate_result(
package,
ProjectionRenderResult(result.artifacts, receipt),
require_peak_memory=True,
)
document: dict[str, object] = {
"schema_version": WORKER_PROTOCOL_VERSION,
"artifacts": [
{
"artifact_id": artifact.artifact_id,
"media_type": artifact.media_type,
"content_base64": base64.b64encode(artifact.content).decode("ascii"),
}
for artifact in validated.artifacts
],
"receipt": validated.receipt.as_dict(),
}
encoded = canonical_projection_bytes(document) + b"\n"
if len(encoded) > MAX_WORKER_RESPONSE_BYTES:
raise _worker_failure(
"Projection worker response exceeded its fixed boundary",
maximum_bytes=MAX_WORKER_RESPONSE_BYTES,
)
return encoded
def _read_child_request() -> ProjectionPackageV1:
raw = sys.stdin.buffer.read(MAX_WORKER_REQUEST_BYTES + 1)
document = _decode_canonical_line(
raw,
maximum=MAX_WORKER_REQUEST_BYTES,
label="Projection worker request",
)
return _validated_package(ProjectionPackageV1.from_dict(document))
def main(argv: list[str] | None = None) -> int:
"""Run the closed one-request child protocol."""
arguments = sys.argv[1:] if argv is None else argv
if arguments:
return 2
try:
package = _read_child_request()
except Exception:
return 2
try:
response = _child_response(package)
sys.stdout.buffer.write(response)
sys.stdout.buffer.flush()
except DocForgeError as error:
response = (
canonical_projection_bytes(
{
"schema_version": WORKER_PROTOCOL_VERSION,
"error": error.as_dict(),
}
)
+ b"\n"
)
if len(response) <= MAX_RECEIPT_BYTES:
sys.stdout.buffer.write(response)
sys.stdout.buffer.flush()
return 3
except Exception:
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())

149
src/docforge/render_contract.py
View file

@ -3,15 +3,28 @@
from __future__ import annotations
import hashlib
import html
import json
from dataclasses import dataclass
from importlib.metadata import version
from pathlib import Path
from typing import Protocol
from typing import Protocol, cast
from .errors import DocForgeError
from .manual_projection import build_manual_projection_package, build_manual_render_plan
from .models import ProjectSnapshot, RenderView
from .projection_contract import (
ManualRenderPlanV1,
ProjectionPackageV1,
ProjectionRenderResult,
)
from .projection_fragments import (
FragmentKey,
FragmentRecord,
ProjectionFragmentCache,
fragment_semantic_hash,
)
from .projection_worker import render_projection_in_worker
@dataclass(frozen=True)
@ -22,6 +35,7 @@ class PreparedRender:
renderer: str
renderer_version: str
template_hash: str
projection_receipt: dict[str, object] | None = None
class Renderer(Protocol):
@ -46,10 +60,13 @@ class GenericHtmlRenderer:
renderer_id = "generic_html"
contract_version = "1"
def __init__(self) -> None:
page_component_version = "manual.page@1"
def __init__(self, *, incremental: bool = True) -> None:
self.renderer_version = (
f"{self.contract_version}+markdown-it-py-{version('markdown-it-py')}"
)
self.incremental = incremental
def prepare(
self,
@ -98,19 +115,24 @@ class GenericHtmlRenderer:
json.dumps(identity_payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
plan = build_manual_render_plan(snapshot, view, changeset_hash=changeset_hash)
package = build_manual_projection_package(
full_package = build_manual_projection_package(
plan,
template_bytes,
renderer_id=self.renderer_id,
renderer_version=self.renderer_version,
max_output_bytes=snapshot.descriptor.limits.max_render_bytes,
)
from docforge_renderers.manual import ManualHtmlRenderer
result = ManualHtmlRenderer(self.renderer_version).render(
package,
render_identity=render_identity,
)
if self.incremental:
result = self._incremental_result(
snapshot,
plan,
template_bytes,
full_package=full_package,
render_identity=render_identity,
)
else:
result = render_projection_in_worker(full_package)
if len(result.artifacts) != 1:
raise DocForgeError(
"invalid_projection",
@ -124,21 +146,128 @@ class GenericHtmlRenderer:
renderer=self.renderer_id,
renderer_version=self.renderer_version,
template_hash=template_hash,
projection_receipt=result.receipt.as_dict(),
)
def _incremental_result(
self,
snapshot: ProjectSnapshot,
plan: ManualRenderPlanV1,
template_bytes: bytes,
*,
full_package: ProjectionPackageV1,
render_identity: str,
) -> ProjectionRenderResult:
cache = ProjectionFragmentCache(
snapshot.descriptor.root,
snapshot.descriptor.cache_root,
)
pages = cast(list[object], plan.document["pages"])
records: list[FragmentRecord | None] = []
missing: list[tuple[int, FragmentKey]] = []
for value in pages:
page = cast(dict[str, object], value)
key = FragmentKey.create(
projection_kind="manual",
renderer_id=self.renderer_id,
renderer_version=self.renderer_version,
component_version=self.page_component_version,
semantic_input_hash=fragment_semantic_hash(page),
)
record = cache.get(key)
if record is None:
missing.append((len(records), key))
records.append(record)
if not records:
return render_projection_in_worker(full_package)
full_result: ProjectionRenderResult | None = None
new_records: list[FragmentRecord] = []
if missing:
full_result = render_projection_in_worker(full_package)
fragments = self._extract_page_fragments(
pages,
full_result.artifacts[0].content,
)
try:
for position, key in missing:
record = FragmentRecord.create(key, fragments[position])
records[position] = record
new_records.append(record)
except DocForgeError:
return full_result
complete_records = [record for record in records if record is not None]
if len(complete_records) != len(records):
return full_result or render_projection_in_worker(full_package)
try:
incremental_package = build_manual_projection_package(
plan,
template_bytes,
fragment_records=[record.as_dict() for record in complete_records],
renderer_id=self.renderer_id,
renderer_version=self.renderer_version,
max_output_bytes=snapshot.descriptor.limits.max_render_bytes,
render_identity=render_identity,
)
incremental_result = render_projection_in_worker(incremental_package)
except DocForgeError:
cache.prune(())
return full_result or render_projection_in_worker(full_package)
if full_result is None:
cache.prune(tuple(record.key for record in complete_records))
return incremental_result
if tuple(
(artifact.artifact_id, artifact.media_type, artifact.content)
for artifact in incremental_result.artifacts
) != tuple(
(artifact.artifact_id, artifact.media_type, artifact.content)
for artifact in full_result.artifacts
):
cache.prune(())
return full_result
for record in new_records:
cache.put(record.key, record.content)
cache.prune(tuple(record.key for record in complete_records))
return incremental_result
@staticmethod
def _extract_page_fragments(
pages: list[object],
output: bytes,
) -> list[bytes]:
"""Extract exact deterministic page sections from one trusted full artifact."""
fragments: list[bytes] = []
cursor = 0
closing = b"</section>"
for value in pages:
page = cast(dict[str, object], value)
node_id = cast(str, page["node_id"])
marker = f'<section id="node-{html.escape(node_id, quote=True)}">'.encode()
start = output.find(marker, cursor)
end = output.find(closing, start + len(marker)) if start >= 0 else -1
if start < 0 or end < 0:
raise DocForgeError(
"invalid_projection",
"Full manual artifact does not contain its planned page fragments",
)
end += len(closing)
fragments.append(output[start:end])
cursor = end
return fragments
_RENDERERS: dict[str, type[GenericHtmlRenderer]] = {
GenericHtmlRenderer.renderer_id: GenericHtmlRenderer
}
def renderer_for(view: RenderView) -> Renderer:
def renderer_for(view: RenderView, *, incremental: bool = True) -> Renderer:
factory = _RENDERERS.get(view.renderer)
if factory is None:
raise DocForgeError(
"unsupported_renderer", "View does not name a supported built-in renderer"
)
return factory()
return factory(incremental=incremental)
def relative_output(snapshot: ProjectSnapshot, path: Path) -> str:

86
src/docforge/rendering.py
View file

@ -26,6 +26,8 @@ from .models import (
RenderView,
)
from .project import project_root_fingerprint
from .projection_contract import ProjectionReceiptV1
from .projection_policy import ManualProjectionMode, validate_manual_projection_mode
from .render_contract import PreparedRender, relative_output, renderer_for
from .telemetry import increment, stage
@ -36,9 +38,26 @@ MAX_RENDER_RECEIPT_BYTES = 64_000
class RenderService:
"""Render only declared views through fixed built-in renderer implementations."""
def __init__(self, project: ProjectService, changesets: ChangesetStore | None = None) -> None:
def __init__(
self,
project: ProjectService,
changesets: ChangesetStore | None = None,
*,
manual_policy: ManualProjectionMode = "explicit",
) -> None:
self.project = project
self.changesets = changesets or ChangesetStore(project)
self.manual_policy = validate_manual_projection_mode(manual_policy)
def _require_rendering(self, operation: str) -> None:
if self.manual_policy == "disabled":
raise DocForgeError(
"projection_policy_forbids_operation",
"Manual projection policy disables rendering work",
projection="manual",
mode=self.manual_policy,
operation=operation,
)
def status(self, view_id: str | None = None) -> dict[str, object]:
"""Report publication state from bounded receipts without rendering canonical content."""
@ -86,6 +105,7 @@ class RenderService:
def deep_status(self, view_id: str | None = None) -> dict[str, object]:
"""Recompute render output as the explicit side-effect-free equivalence oracle."""
self._require_rendering("deep_status")
snapshot = self.project.load()
config = snapshot.descriptor.render
if config is None:
@ -107,7 +127,12 @@ class RenderService:
snapshot.descriptor.root,
view.output_path,
)
prepared, _ = self._prepare(snapshot, view, changeset_hash=None)
prepared, _ = self._prepare(
snapshot,
view,
changeset_hash=None,
incremental=False,
)
state = "missing"
actual_hash: str | None = None
output = view.output_path
@ -151,6 +176,7 @@ class RenderService:
)
def render(self, view_id: str) -> dict[str, object]:
self._require_rendering("render")
with self._lock():
snapshot = self.project.load()
config = self._config(snapshot)
@ -373,6 +399,8 @@ class RenderService:
"template_file": final_template_file,
"output_file": final_output_file,
}
if prepared.projection_receipt is not None:
payload["projection_receipt"] = prepared.projection_receipt
raw = json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n"
if len(raw) > MAX_RENDER_RECEIPT_BYTES:
raise DocForgeError(
@ -572,8 +600,10 @@ class RenderService:
renderer = renderer_for(view)
template_file = receipt.get("template_file")
output_file = receipt.get("output_file")
fields = set(receipt)
projection_receipt = receipt.get("projection_receipt")
return (
set(receipt) == required
fields in (required, required | {"projection_receipt"})
and receipt.get("schema_version") == RENDER_RECEIPT_SCHEMA_VERSION
and receipt.get("project_id") == descriptor.project_id
and receipt.get("project_root_fingerprint") == project_root_fingerprint(descriptor.root)
@ -602,6 +632,50 @@ class RenderService:
descriptor.limits.max_render_bytes,
)
and cast(dict[str, object], output_file)["size"] == receipt.get("output_bytes")
and (
projection_receipt is None
or self._valid_projection_receipt(
projection_receipt,
renderer_id=renderer.renderer_id,
renderer_version=renderer.renderer_version,
output_hash=cast(str, receipt["output_hash"]),
output_bytes=cast(int, receipt["output_bytes"]),
)
)
)
@staticmethod
def _valid_projection_receipt(
value: object,
*,
renderer_id: str,
renderer_version: str,
output_hash: str,
output_bytes: int,
) -> bool:
if not isinstance(value, dict):
return False
try:
receipt = ProjectionReceiptV1.from_dict(cast(dict[str, object], value))
except DocForgeError:
return False
document = receipt.document
return (
document.get("kind") == "manual"
and document.get("renderer")
== {
"renderer_id": renderer_id,
"renderer_version": renderer_version,
}
and document.get("artifacts")
== [
{
"artifact_id": "manual.html",
"media_type": "text/html; charset=utf-8",
"sha256": output_hash,
"bytes": output_bytes,
}
]
)
@staticmethod
@ -663,6 +737,7 @@ class RenderService:
"reason": reason,
"verification": "receipt",
"receipt_schema_version": payload.get("schema_version"),
"projection_receipt": payload.get("projection_receipt"),
}
def _current_state(self) -> ProjectState | None:
@ -687,6 +762,7 @@ class RenderService:
}
def preview(self, changeset_id: str, view_id: str) -> dict[str, object]:
self._require_rendering("preview")
with self._lock():
snapshot, changeset_hash = self.changesets.projected_snapshot(changeset_id)
config = self._config(snapshot)
@ -731,11 +807,12 @@ class RenderService:
view: RenderView,
*,
changeset_hash: str | None,
incremental: bool = True,
) -> tuple[PreparedRender, bytes]:
increment("render_prepare_calls")
template = self._template_bytes(snapshot, view)
with stage("render.prepare"):
prepared = renderer_for(view).prepare(
prepared = renderer_for(view, incremental=incremental).prepare(
snapshot,
view,
template,
@ -850,6 +927,7 @@ class RenderService:
"expected_output_hash": prepared.output_hash,
"actual_output_hash": actual_hash,
"template_hash": prepared.template_hash,
"projection_receipt": prepared.projection_receipt,
"path": relative_output(snapshot, view.output_path),
"state": state,
}

3
src/docforge/telemetry.py
View file

@ -79,6 +79,7 @@ OPERATION_NAMES = frozenset(
"test",
"benchmark.m1",
"benchmark.m2",
"benchmark.m3",
"mcp.invoke",
"mcp.bootstrap",
"mcp.sync",
@ -96,6 +97,8 @@ OPERATION_NAMES = frozenset(
"mcp.generation_diff",
"mcp.validate_project",
"mcp.render_status",
"mcp.graph_plan",
"mcp.graph_render_status",
"mcp.visualize",
"mcp.visualization_status",
"mcp.stop_visualization",

21
src/docforge/viewer_manager.py
View file

@ -28,6 +28,10 @@ from .errors import DocForgeError
from .index import ProjectIndex
from .models import IncrementalStateProject
from .project import project_root_fingerprint
from .projection_policy import (
LiveViewerProjectionMode,
validate_live_viewer_projection_mode,
)
from .telemetry import increment, stage
from .visualization import VISUALIZATION_TEMPLATE, VisualizationIndexSnapshot
@ -623,9 +627,16 @@ class ViewerManager:
class ViewerManagerClient:
"""Project-bound MCP-side client for the separately supervised manager service."""
def __init__(self, index: ProjectIndex, *, state_path: Path | None = None) -> None:
def __init__(
self,
index: ProjectIndex,
*,
state_path: Path | None = None,
live_viewer_policy: LiveViewerProjectionMode = "on-demand",
) -> None:
self.index = index
self.state_path = state_path or default_state_path()
self.live_viewer_policy = validate_live_viewer_projection_mode(live_viewer_policy)
def start(
self,
@ -634,6 +645,14 @@ class ViewerManagerClient:
query: str | None = None,
depth: int = 1,
) -> dict[str, object]:
if self.live_viewer_policy == "disabled":
raise DocForgeError(
"projection_policy_forbids_operation",
"Live viewer projection policy disables viewer startup",
projection="live_viewer",
mode=self.live_viewer_policy,
operation="start",
)
if node_id is not None and query is not None:
raise DocForgeError(
"invalid_visualization_target",

2
src/docforge_renderers/graph.py
View file

@ -45,7 +45,7 @@ html[data-enhanced="true"] main[data-mode="flow"] [data-panel="nodes"] { display
table { border-collapse: collapse; width: 100%; }
th, td { text-align: left; border-bottom: 1px solid GrayText; padding: .5rem; vertical-align: top; }
caption { text-align: left; font-weight: 700; margin-bottom: .5rem; }
.muted { color: GrayText; }
.muted { color: CanvasText; }
dialog {
max-width: min(42rem, calc(100% - 2rem));
border: 1px solid GrayText; border-radius: .5rem;

128
src/docforge_renderers/manual.py
View file

@ -17,6 +17,10 @@ from docforge.projection_contract import (
ProjectionReceiptV1,
ProjectionRenderResult,
)
from docforge.projection_fragments import (
FragmentRecord,
fragment_semantic_hash,
)
_TEMPLATE_TOKEN = re.compile(r"{{\s*([a-z_][a-z0-9_]*)\s*}}")
_ALLOWED_TOKENS = frozenset(
@ -35,6 +39,7 @@ _ACTIVE_TEMPLATE_CONTENT = re.compile(
r"|<\s*meta\b[^>]*\bhttp-equiv\s*=\s*[\"']?\s*refresh\b",
re.IGNORECASE,
)
_MANUAL_PAGE_COMPONENT = "manual.page@1"
class ManualHtmlRenderer:
@ -65,11 +70,15 @@ class ManualHtmlRenderer:
raise DocForgeError("unsupported_renderer", "Manual renderer identity is incompatible")
plan = cast(dict[str, object], document["plan"])
assets = cast(list[object], document["assets"])
if len(assets) != 1 or not isinstance(assets[0], dict):
if len(assets) not in {1, 2} or not isinstance(assets[0], dict):
raise DocForgeError("invalid_projection", "Manual template asset is invalid")
asset = cast(dict[str, object], assets[0])
if (
set(asset) != {"asset_id", "media_type", "sha256", "text"}
set(asset)
not in (
{"asset_id", "media_type", "sha256", "text"},
{"asset_id", "media_type", "sha256", "text", "render_identity"},
)
or asset.get("asset_id") != "manual.template"
or asset.get("media_type") != "text/html; charset=utf-8"
or not isinstance(asset.get("text"), str)
@ -98,11 +107,27 @@ class ManualHtmlRenderer:
"invalid_template",
"Render template must contain docforge_content exactly once",
)
identity = render_identity or cast(str, plan["plan_id"])
packaged_identity = asset.get("render_identity")
if packaged_identity is not None and (
not isinstance(packaged_identity, str)
or len(packaged_identity) != 64
or any(character not in "0123456789abcdef" for character in packaged_identity)
):
raise DocForgeError(
"invalid_projection",
"Manual render identity is invalid",
)
if render_identity is not None and packaged_identity not in {None, render_identity}:
raise DocForgeError(
"invalid_projection",
"Manual render identity is inconsistent",
)
identity = render_identity or packaged_identity or cast(str, plan["plan_id"])
fragments = self._fragments(plan, assets[1:] if len(assets) == 2 else [])
project = cast(dict[str, object], plan["project"])
view = cast(dict[str, object], plan["view"])
replacements = {
"docforge_content": self._content(plan),
"docforge_content": self._content(plan, fragments),
"docforge_project_id": html.escape(cast(str, project["project_id"]), quote=True),
"docforge_render_identity": identity,
"docforge_title": html.escape(cast(str, view["title"]), quote=True),
@ -131,21 +156,69 @@ class ManualHtmlRenderer:
)
return ProjectionRenderResult((artifact,), receipt)
def _content(self, plan: dict[str, object]) -> str:
navigation = ['<nav aria-label="Documentation"><ul>']
for value in cast(list[object], plan["navigation"]):
item = cast(dict[str, object], value)
navigation.append(
f'<li><a href="#node-{html.escape(cast(str, item["node_id"]), quote=True)}">'
f"{html.escape(cast(str, item['title']))}</a></li>"
def _fragments(
self,
plan: dict[str, object],
assets: list[object],
) -> dict[str, str]:
if not assets:
return {}
asset = assets[0]
if not isinstance(asset, dict):
raise DocForgeError("invalid_projection", "Manual fragment asset is invalid")
document = cast(dict[str, object], asset)
if (
set(document) != {"asset_id", "media_type", "records"}
or document.get("asset_id") != "manual.fragments"
or document.get("media_type") != "application/vnd.docforge.projection-fragments.v1+json"
or not isinstance(document.get("records"), list)
):
raise DocForgeError("invalid_projection", "Manual fragment asset is invalid")
pages = cast(list[object], plan["pages"])
records = cast(list[object], document["records"])
if len(records) != len(pages):
raise DocForgeError(
"invalid_projection",
"Manual fragment inventory is incomplete",
)
navigation.append("</ul></nav>")
sections = [*navigation]
for value in cast(list[object], plan["pages"]):
page = cast(dict[str, object], value)
fragments: dict[str, str] = {}
for page_value, record_value in zip(pages, records, strict=True):
if not isinstance(page_value, dict):
raise DocForgeError("invalid_projection", "Manual page is invalid")
page = cast(dict[str, object], page_value)
record = FragmentRecord.from_dict(record_value)
if (
record.key.projection_kind != "manual"
or record.key.renderer_id != self.renderer_id
or record.key.renderer_version != self.renderer_version
or record.key.component_version != _MANUAL_PAGE_COMPONENT
or record.key.semantic_input_hash != fragment_semantic_hash(page)
):
raise DocForgeError(
"invalid_projection",
"Manual fragment identity does not match its page semantics",
)
try:
fragment = record.content.decode("utf-8")
except UnicodeDecodeError as error:
raise DocForgeError(
"invalid_projection",
"Manual fragment content is not valid UTF-8",
) from error
node_id = cast(str, page["node_id"])
sections.extend(
[
if node_id in fragments or fragment != self.render_page_fragment(page):
raise DocForgeError(
"invalid_projection",
"Manual fragment content does not match its page semantics",
)
fragments[node_id] = fragment
return fragments
def render_page_fragment(self, page: dict[str, object]) -> str:
"""Render one page from complete plan semantics without project authority."""
node_id = cast(str, page["node_id"])
sections = [
f'<section id="node-{html.escape(node_id, quote=True)}">',
f"<h2>{html.escape(cast(str, page['title']))}</h2>",
'<dl class="docforge-node-meta">',
@ -157,7 +230,6 @@ class ManualHtmlRenderer:
f'<p class="docforge-summary">{html.escape(cast(str, page["summary"]))}</p>',
self.markdown.render(cast(str, page["content"])).rstrip(),
]
)
relationships = cast(list[object], page["cross_references"])
if relationships:
sections.append('<ul class="docforge-relationships">')
@ -170,3 +242,23 @@ class ManualHtmlRenderer:
sections.append("</ul>")
sections.append("</section>")
return "\n".join(sections)
def _content(
self,
plan: dict[str, object],
fragments: dict[str, str],
) -> str:
navigation = ['<nav aria-label="Documentation"><ul>']
for value in cast(list[object], plan["navigation"]):
item = cast(dict[str, object], value)
navigation.append(
f'<li><a href="#node-{html.escape(cast(str, item["node_id"]), quote=True)}">'
f"{html.escape(cast(str, item['title']))}</a></li>"
)
navigation.append("</ul></nav>")
sections = [*navigation]
for value in cast(list[object], plan["pages"]):
page = cast(dict[str, object], value)
node_id = cast(str, page["node_id"])
sections.append(fragments.get(node_id, self.render_page_fragment(page)))
return "\n".join(sections)

187
tests/accessibility.spec.mjs Normal file
View file

@ -0,0 +1,187 @@
import AxeBuilder from "@axe-core/playwright";
import { expect, test } from "@playwright/test";
import { spawn } from "node:child_process";
import { createInterface } from "node:readline";
const AXE_TAGS = [
"wcag2a",
"wcag2aa",
"wcag21a",
"wcag21aa",
"wcag22a",
"wcag22aa",
];
const MANUAL_AXE_TAGS = AXE_TAGS.filter((tag) => !tag.startsWith("wcag22"));
let fixtureProcess;
let surfaces;
function startFixture() {
const python = process.env.DOCFORGE_PYTHON || ".venv/bin/python";
const child = spawn(python, ["tools/accessibility_fixture.py"], {
cwd: process.cwd(),
stdio: ["pipe", "pipe", "pipe"],
});
let stderr = "";
child.stderr.setEncoding("utf8");
child.stderr.on("data", (chunk) => {
stderr += chunk;
});
const lines = createInterface({ input: child.stdout });
const ready = new Promise((resolve, reject) => {
let settled = false;
lines.once("line", (line) => {
settled = true;
try {
const payload = JSON.parse(line);
if (
payload.schema_version !== 1
|| typeof payload.manual_html !== "string"
|| typeof payload.portable_html !== "string"
|| typeof payload.live_url !== "string"
) {
throw new Error("Accessibility fixture returned an invalid payload");
}
resolve(payload);
} catch (error) {
reject(error);
} finally {
lines.close();
}
});
child.once("exit", (code, signal) => {
if (!settled) {
reject(
new Error(
`Accessibility fixture exited before readiness `
+ `(code=${code}, signal=${signal}):\n${stderr}`,
),
);
}
});
});
return { child, ready };
}
async function stopFixture(child) {
if (child.exitCode !== null || child.signalCode !== null) {
return;
}
const exited = new Promise((resolve) => child.once("exit", resolve));
child.stdin.end();
await Promise.race([
exited,
new Promise((_, reject) => {
setTimeout(() => reject(new Error("Accessibility fixture did not stop")), 5_000);
}),
]);
}
function violationReport(violations) {
return violations.map((violation) => {
const targets = violation.nodes
.flatMap((node) => node.target)
.join(", ");
return `${violation.id} (${violation.impact}): ${violation.help}\n ${targets}`;
}).join("\n");
}
async function expectNoAxeViolations(page, tags = AXE_TAGS) {
const results = await new AxeBuilder({ page }).withTags(tags).analyze();
expect(results.violations, violationReport(results.violations)).toEqual([]);
}
async function tabTo(page, selector, maximumTabs = 40) {
for (let count = 0; count < maximumTabs; count += 1) {
await page.keyboard.press("Tab");
if (await page.evaluate((target) => document.activeElement?.matches(target), selector)) {
return page.locator(selector).filter({ visible: true }).first();
}
}
throw new Error(`Keyboard focus did not reach ${selector}`);
}
test.beforeAll(async () => {
const fixture = startFixture();
fixtureProcess = fixture.child;
surfaces = await fixture.ready;
});
test.afterAll(async () => {
await stopFixture(fixtureProcess);
});
test("generated manual has no axe violations and its navigation works by keyboard", async ({
page,
}) => {
await page.setContent(surfaces.manual_html, { waitUntil: "load" });
await expect(page.locator("main section")).not.toHaveCount(0);
// The generic renderer owns structure, while this frozen project template owns target sizing.
await expectNoAxeViolations(page, MANUAL_AXE_TAGS);
await page.keyboard.press("Tab");
const firstNavigationLink = page.locator("nav[aria-label='Documentation'] a").first();
await expect(firstNavigationLink).toBeFocused();
const target = await firstNavigationLink.getAttribute("href");
expect(target).toMatch(/^#[A-Za-z0-9_.-]+$/);
await page.keyboard.press("Enter");
await expect.poll(() => page.evaluate(() => window.location.hash)).toBe(target);
});
test("portable graph supports skip, filter, view, and dialog keyboard flows", async ({ page }) => {
await page.goto("about:blank");
await page.setContent(surfaces.portable_html, { waitUntil: "load" });
await expect(page.locator("#status")).toContainText("nodes and");
await expectNoAxeViolations(page);
await page.keyboard.press("Tab");
await expect(page.locator("a.skip")).toBeFocused();
await page.keyboard.press("Enter");
await expect(page.locator("main#main")).toBeFocused();
await page.goto("about:blank");
await page.setContent(surfaces.portable_html, { waitUntil: "load" });
await page.keyboard.press("Tab");
await page.keyboard.press("Tab");
await expect(page.locator("#filter")).toBeFocused();
await page.keyboard.type("guide.workflow");
await expect(page.locator("#status")).toContainText("1 nodes and");
await page.keyboard.press("Tab");
await expect(page.locator("#mode")).toBeFocused();
await page.keyboard.press("ArrowDown");
await expect(page.locator("main#main")).toHaveAttribute("data-mode", "flow");
await page.keyboard.press("ArrowUp");
await expect(page.locator("main#main")).toHaveAttribute("data-mode", "nodes");
await page.keyboard.press("Tab");
const nodeButton = page.locator("#node-list button").first();
await expect(nodeButton).toBeFocused();
await page.keyboard.press("Enter");
await expect(page.locator("#node-dialog")).toHaveAttribute("open", "");
await expect(page.locator("#close-dialog")).toBeFocused();
await expectNoAxeViolations(page);
await page.keyboard.press("Escape");
await expect(page.locator("#node-dialog")).not.toHaveAttribute("open", "");
await expect(nodeButton).toBeFocused();
});
test("live viewer passes axe and exposes keyboard graph and resize controls", async ({ page }) => {
await page.goto(surfaces.live_url);
await expect(page.locator("#status")).toContainText("nodes ·");
await expect(page.locator("#graph g.node[role='button']").first()).toBeVisible();
await expectNoAxeViolations(page);
const resizer = await tabTo(page, "#left-resizer");
const originalWidth = Number(await resizer.getAttribute("aria-valuenow"));
await page.keyboard.press("ArrowRight");
await expect(resizer).toHaveAttribute("aria-valuenow", String(originalWidth + 16));
await tabTo(page, "#graph g.node[role='button']");
await page.keyboard.press("Shift+Enter");
await expect(page.locator("#node-dialog")).toHaveAttribute("open", "");
await expect(page.locator("#close-node-dialog")).toBeFocused();
await expectNoAxeViolations(page);
await page.keyboard.press("Escape");
await expect(page.locator("#node-dialog")).not.toHaveAttribute("open", "");
});

265
tests/test_client_integration.py
View file

@ -1,6 +1,7 @@
from __future__ import annotations
import contextlib
import hashlib
import io
import json
import os
@ -18,6 +19,7 @@ from jsonschema import Draft202012Validator
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
from docforge.changeset_contract import document_hash
from docforge.cli import _parser, _run, main
from docforge.client_config import (
_read_existing,
@ -39,6 +41,19 @@ CONFIGURATION_SCHEMA = json.loads(
DOCTOR_SCHEMA = json.loads((SCHEMAS / "doctor-result.schema.json").read_text(encoding="utf-8"))
POLICY_SCHEMA = json.loads((SCHEMAS / "policy.schema.json").read_text(encoding="utf-8"))
GRAPH_RENDER_CONFIG = """
[graph_render]
output_root = ".docforge/portable-graph"
[[graph_render.views]]
id = "architecture"
renderer = "portable_graph_html"
output = "architecture.html"
title = "Alpha architecture"
root = "guide.workflow"
"""
class ClientIntegrationTests(unittest.TestCase):
def copy_fixture(self, destination: Path) -> Path:
@ -156,6 +171,256 @@ class ClientIntegrationTests(unittest.TestCase):
)
self.assertEqual("invalid_capability_binding", escalated.exception.code)
def test_nondefault_projection_policy_selectors_serialize_and_validate(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = self.copy_fixture(Path(directory))
descriptor = root / ".docforge" / "project.toml"
descriptor.write_text(
descriptor.read_text(encoding="utf-8") + GRAPH_RENDER_CONFIG,
encoding="utf-8",
)
project = Project.open(root)
expected_arguments = [
"-I",
"-m",
"docforge.mcp_server",
"--project-root",
str(root),
"--capability-mode",
"read",
"--manual-render-policy",
"disabled",
"--portable-graph-policy",
"disabled",
"--live-viewer-policy",
"disabled",
]
for client in ("codex", "claude", "openclaw"):
with self.subTest(client=client):
result = generate_client_configuration(
project,
client,
manual_render_policy="disabled",
portable_graph_policy="disabled",
live_viewer_policy="disabled",
)
Draft202012Validator(CONFIGURATION_SCHEMA).validate(result)
_validate_configuration_result(result)
self.assertEqual(
{
"schema_version": 2,
"manual": "disabled",
"portable_graph": "disabled",
"live_viewer": "disabled",
},
result["projection_policy"],
)
self.assertEqual(expected_arguments, result["binding"]["args"])
content = result["artifact"]["content"]
if client == "codex":
document = tomllib.loads(content)
serialized = document["mcp_servers"][result["server_name"]]["args"]
elif client == "claude":
document = json.loads(content)
serialized = document["mcpServers"][result["server_name"]]["args"]
else:
document = json.loads(content)
serialized = document["mcp"]["servers"][result["server_name"]]["args"]
self.assertEqual(expected_arguments, serialized)
canonical = json.dumps(
result["projection_policy"],
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
self.assertEqual(
hashlib.sha256(canonical).hexdigest(),
result["projection_policy_hash"],
)
def test_projection_policy_schema_and_configuration_validation_reject_drift(self) -> None:
with tempfile.TemporaryDirectory() as directory:
project = Project.open(self.copy_fixture(Path(directory)))
result = generate_client_configuration(
project,
"codex",
manual_render_policy="disabled",
live_viewer_policy="disabled",
)
validator = Draft202012Validator(CONFIGURATION_SCHEMA)
validator.validate(result)
_validate_configuration_result(result)
for field, value in (
("schema_version", 1),
("manual", "on-demand"),
("portable_graph", "auto"),
("live_viewer", "explicit"),
):
with self.subTest(field=field):
drifted = json.loads(json.dumps(result))
drifted["projection_policy"][field] = value
self.assertTrue(list(validator.iter_errors(drifted)))
missing = json.loads(json.dumps(result))
missing.pop("projection_policy")
self.assertTrue(list(validator.iter_errors(missing)))
extra = json.loads(json.dumps(result))
extra["projection_policy"]["project_path"] = "/private/project"
self.assertTrue(list(validator.iter_errors(extra)))
mismatched = json.loads(json.dumps(result))
mismatched["projection_policy"]["manual"] = "explicit"
canonical = json.dumps(
mismatched["projection_policy"],
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
mismatched["projection_policy_hash"] = hashlib.sha256(canonical).hexdigest()
with self.assertRaises(AssertionError):
_validate_configuration_result(mismatched)
bad_hash = json.loads(json.dumps(result))
bad_hash["projection_policy_hash"] = "0" * 64
with self.assertRaises(AssertionError):
_validate_configuration_result(bad_hash)
defaulted = generate_client_configuration(project, "codex")
self.assertNotIn(
"--manual-render-policy",
defaulted["binding"]["args"],
)
defaulted["projection_policy"]["manual"] = "disabled"
canonical = json.dumps(
defaulted["projection_policy"],
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
defaulted["projection_policy_hash"] = hashlib.sha256(canonical).hexdigest()
with self.assertRaises(AssertionError):
_validate_configuration_result(defaulted)
unavailable = json.loads(json.dumps(result))
unavailable["projection_availability"]["manual_configured"] = False
with self.assertRaises(AssertionError):
_validate_configuration_result(unavailable)
descriptor = project.descriptor.descriptor_path
descriptor.write_text(
descriptor.read_text(encoding="utf-8") + GRAPH_RENDER_CONFIG,
encoding="utf-8",
)
graph_project = Project.open(project.descriptor.root)
graph_defaulted = generate_client_configuration(graph_project, "codex")
self.assertNotIn(
"--portable-graph-policy",
graph_defaulted["binding"]["args"],
)
coordinated_graph_drift = json.loads(json.dumps(graph_defaulted))
coordinated_graph_drift["projection_policy"]["portable_graph"] = "disabled"
coordinated_graph_drift["projection_availability"]["portable_graph_configured"] = False
canonical = json.dumps(
coordinated_graph_drift["projection_policy"],
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
coordinated_graph_drift["projection_policy_hash"] = hashlib.sha256(
canonical
).hexdigest()
coordinated_graph_drift["configuration_hash"] = document_hash(
{
"schema_version": 1,
"client": coordinated_graph_drift["client"],
"server_name": coordinated_graph_drift["server_name"],
"project": coordinated_graph_drift["project"],
"binding": coordinated_graph_drift["binding"],
"effective_policy": coordinated_graph_drift["effective_policy"],
"projection_policy": coordinated_graph_drift["projection_policy"],
"projection_policy_hash": coordinated_graph_drift["projection_policy_hash"],
"projection_availability": coordinated_graph_drift["projection_availability"],
"artifact_format": coordinated_graph_drift["artifact"]["format"],
"artifact_content_sha256": coordinated_graph_drift["artifact"][
"content_sha256"
],
}
)
with self.assertRaises(AssertionError):
_validate_configuration_result(coordinated_graph_drift)
invalid_selections = (
{"manual_render_policy": "sometimes"},
{"portable_graph_policy": "auto"},
{"live_viewer_policy": "always"},
)
for selection in invalid_selections:
with self.subTest(selection=selection):
with self.assertRaises(DocForgeError) as raised:
generate_client_configuration(graph_project, "codex", **selection)
self.assertEqual("invalid_projection_policy", raised.exception.code)
def test_doctor_round_trips_projection_selectors_and_rejects_invalid_modes(self) -> None:
with tempfile.TemporaryDirectory() as directory:
parent = Path(directory)
project = Project.open(self.copy_fixture(parent))
ProjectIndex(project).build()
config = parent / "openclaw.json"
generated = generate_client_configuration(
project,
"openclaw",
manual_render_policy="disabled",
live_viewer_policy="disabled",
output=config,
)
healthy = run_doctor(
project,
"openclaw",
config_path=config,
server_name=generated["server_name"],
)
Draft202012Validator(DOCTOR_SCHEMA).validate(healthy)
self.assertEqual("healthy", healthy["doctor_state"])
self.assertIn(
"effective_policy_valid",
[check["code"] for check in healthy["checks"]],
)
document = json.loads(config.read_text(encoding="utf-8"))
entry = document["mcp"]["servers"][generated["server_name"]]
arguments = entry["args"]
manual_position = arguments.index("--manual-render-policy") + 1
arguments[manual_position] = "sometimes"
config.write_text(json.dumps(document, sort_keys=True), encoding="utf-8")
invalid = run_doctor(
project,
"openclaw",
config_path=config,
server_name=generated["server_name"],
)
Draft202012Validator(DOCTOR_SCHEMA).validate(invalid)
self.assertEqual("unhealthy", invalid["doctor_state"])
policy_check = next(
check for check in invalid["checks"] if check["check_id"] == "policy.effective"
)
self.assertEqual("invalid_projection_policy", policy_check["code"])
arguments[manual_position] = "auto"
config.write_text(json.dumps(document, sort_keys=True), encoding="utf-8")
unavailable = run_doctor(
project,
"openclaw",
config_path=config,
server_name=generated["server_name"],
)
Draft202012Validator(DOCTOR_SCHEMA).validate(unavailable)
self.assertEqual("unhealthy", unavailable["doctor_state"])
policy_check = next(
check for check in unavailable["checks"] if check["check_id"] == "policy.effective"
)
self.assertEqual("projection_policy_unavailable", policy_check["code"])
def test_explicit_fragment_write_is_atomic_conflict_aware_and_private(self) -> None:
with tempfile.TemporaryDirectory() as directory:
parent = Path(directory)

23
tests/test_onboarding.py
View file

@ -3,6 +3,7 @@ from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from docforge.cli import _parser, _run
from docforge.errors import DocForgeError
@ -10,6 +11,28 @@ from docforge.project import Project
class DocForgeOnboardingTests(unittest.TestCase):
def test_scaffold_honors_disabled_manual_projection_policy(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory) / "disabled-render"
root.mkdir()
arguments = _parser().parse_args(
[
"--project-root",
str(root),
"--manual-render-policy",
"disabled",
"onboard",
"--scaffold",
]
)
with mock.patch(
"docforge.cli.RenderService.render",
side_effect=AssertionError("disabled onboarding must not render"),
):
result = _run(arguments)
self.assertEqual("skipped", result["render"]["state"])
self.assertFalse((root / ".docforge/rendered/manual.html").exists())
def test_assessment_detects_multiple_languages_without_writing(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory) / "polyglot"

167
tests/test_projection_contract.py
View file

@ -30,6 +30,8 @@ from docforge.projection_contract import (
canonical_projection_bytes,
projection_hash,
)
from docforge.projection_fragments import FRAGMENT_CACHE_DIRECTORY, FragmentRecord
from docforge.projection_worker import render_projection_in_worker
from docforge.render_contract import GenericHtmlRenderer
from docforge_renderers.manual import ManualHtmlRenderer
@ -412,6 +414,171 @@ class ProjectionContractTests(unittest.TestCase):
prepared.output,
)
def test_manual_fragments_are_incremental_and_full_worker_output_is_the_oracle(
self,
) -> None:
renderer = GenericHtmlRenderer()
with mock.patch(
"docforge.render_contract.render_projection_in_worker",
wraps=render_projection_in_worker,
) as detached:
cold = renderer.prepare(
self.snapshot,
self.view,
self.template,
changeset_hash=None,
)
self.assertEqual(2, detached.call_count)
fragment_root = self.snapshot.descriptor.cache_root / FRAGMENT_CACHE_DIRECTORY
self.assertEqual(
len(self.plan.document["pages"]),
len(list(fragment_root.glob("*.json"))),
)
with mock.patch(
"docforge.render_contract.render_projection_in_worker",
wraps=render_projection_in_worker,
) as detached:
warm = renderer.prepare(
self.snapshot,
self.view,
self.template,
changeset_hash=None,
)
self.assertEqual(1, detached.call_count)
self.assertEqual(cold.output, warm.output)
with mock.patch(
"docforge.render_contract.render_projection_in_worker",
wraps=render_projection_in_worker,
) as detached:
full = GenericHtmlRenderer(incremental=False).prepare(
self.snapshot,
self.view,
self.template,
changeset_hash=None,
)
self.assertEqual(1, detached.call_count)
self.assertEqual(full.output, warm.output)
first_fragment = sorted(fragment_root.glob("*.json"))[0]
first_fragment.write_bytes(b"{corrupt")
with mock.patch(
"docforge.render_contract.render_projection_in_worker",
wraps=render_projection_in_worker,
) as detached:
recovered = renderer.prepare(
self.snapshot,
self.view,
self.template,
changeset_hash=None,
)
self.assertEqual(2, detached.call_count)
self.assertEqual(full.output, recovered.output)
original_record = FragmentRecord.from_bytes(first_fragment.read_bytes())
first_fragment.write_bytes(
FragmentRecord.create(
original_record.key,
b"<script>forged fragment</script>",
).to_bytes()
)
with mock.patch(
"docforge.render_contract.render_projection_in_worker",
wraps=render_projection_in_worker,
) as detached:
forged = renderer.prepare(
self.snapshot,
self.view,
self.template,
changeset_hash=None,
)
self.assertEqual(2, detached.call_count)
self.assertEqual(full.output, forged.output)
self.assertNotIn(b"forged fragment", forged.output)
def test_oversized_manual_fragment_bypasses_cache_and_uses_full_worker(self) -> None:
content = "x" * 4_100_000
node = replace(
self.snapshot.nodes[0],
content=content,
content_hash=hashlib.sha256(content.encode()).hexdigest(),
)
snapshot = replace(
self.snapshot,
descriptor=replace(
self.snapshot.descriptor,
limits=replace(
self.snapshot.descriptor.limits,
max_render_bytes=20_000_000,
),
),
nodes=(node,),
edges=(),
source_hash=hashlib.sha256(b"oversized-fragment").hexdigest(),
)
with mock.patch(
"docforge.render_contract.render_projection_in_worker",
wraps=render_projection_in_worker,
) as detached:
prepared = GenericHtmlRenderer().prepare(
snapshot,
self.view,
self.template,
changeset_hash=None,
)
self.assertEqual(1, detached.call_count)
full = GenericHtmlRenderer(incremental=False).prepare(
snapshot,
self.view,
self.template,
changeset_hash=None,
)
self.assertEqual(full.output, prepared.output)
def test_fragment_package_overflow_falls_back_to_full_worker(self) -> None:
content = "x" * 2_700_000
base = self.snapshot.nodes[0]
nodes = tuple(
replace(
base,
node_id=f"guide.large-{index}",
title=f"Large {index}",
content=content,
source_path=f"docs/content/large-{index}.md",
content_hash=hashlib.sha256(f"{index}:{content}".encode()).hexdigest(),
)
for index in range(4)
)
descriptor = replace(
self.snapshot.descriptor,
limits=replace(
self.snapshot.descriptor.limits,
max_render_bytes=20_000_000,
),
)
snapshot = replace(
self.snapshot,
descriptor=descriptor,
nodes=nodes,
edges=(),
source_hash=hashlib.sha256(b"large-fragment-fixture").hexdigest(),
)
incremental = GenericHtmlRenderer().prepare(
snapshot,
self.view,
self.template,
changeset_hash=None,
)
full = GenericHtmlRenderer(incremental=False).prepare(
snapshot,
self.view,
self.template,
changeset_hash=None,
)
self.assertGreater(len(full.output), 10_000_000)
self.assertEqual(full.output, incremental.output)
def test_manual_renderer_rejects_project_provided_active_content(self) -> None:
for active in (
"<script>alert(1)</script>{{ docforge_content }}",

205
tests/test_projection_fragments.py Normal file
View file

@ -0,0 +1,205 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from docforge.projection_fragments import (
FRAGMENT_CACHE_DIRECTORY,
FRAGMENT_KEY_CONTRACT,
FRAGMENT_RECORD_CONTRACT,
FRAGMENT_SCHEMA_VERSION,
FragmentKey,
FragmentRecord,
ProjectionFragmentCache,
fragment_semantic_hash,
)
class ProjectionFragmentCacheTests(unittest.TestCase):
def setUp(self) -> None:
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name) / "project"
self.root.mkdir()
self.cache_root = self.root / ".docforge" / "cache"
self.cache = ProjectionFragmentCache(
self.root,
self.cache_root,
maximum_content_bytes=1_000,
)
self.key = self.make_key({"node": "guide.alpha", "summary": "Alpha"})
@staticmethod
def make_key(
semantics: object,
*,
renderer_version: str = "1",
component_version: str = "graph.node@1",
) -> FragmentKey:
return FragmentKey.create(
projection_kind="graph",
renderer_id="portable_graph_html",
renderer_version=renderer_version,
component_version=component_version,
semantic_input_hash=fragment_semantic_hash(semantics),
)
@property
def entry_path(self) -> Path:
return self.cache_root / FRAGMENT_CACHE_DIRECTORY / f"{self.key.key_id}.json"
def test_miss_hit_and_unchanged_reuse_are_exact(self) -> None:
self.assertIsNone(self.cache.get(self.key))
with mock.patch(
"docforge.projection_fragments.atomic_replace_bytes_at",
wraps=__import__(
"docforge.projection_fragments",
fromlist=["atomic_replace_bytes_at"],
).atomic_replace_bytes_at,
) as atomic:
stored = self.cache.put(self.key, b"<li>Alpha</li>")
self.assertIsNotNone(stored)
self.assertEqual(1, atomic.call_count)
before = self.entry_path.stat()
repeated = self.cache.put(self.key, b"<li>Alpha</li>")
self.assertEqual(1, atomic.call_count)
self.assertEqual(stored, repeated)
self.assertEqual(stored, self.cache.get(self.key))
after = self.entry_path.stat()
self.assertEqual((before.st_dev, before.st_ino), (after.st_dev, after.st_ino))
def test_key_versions_and_complete_semantics_invalidate_independently(self) -> None:
self.assertIsNotNone(self.cache.put(self.key, b"alpha"))
variants = (
self.make_key(
{"node": "guide.alpha", "summary": "Alpha"},
renderer_version="2",
),
self.make_key(
{"node": "guide.alpha", "summary": "Alpha"},
component_version="graph.node@2",
),
self.make_key({"node": "guide.alpha", "summary": "Changed"}),
)
for variant in variants:
with self.subTest(key=variant):
self.assertNotEqual(self.key.key_id, variant.key_id)
self.assertIsNone(self.cache.get(variant))
def test_add_delete_and_reorder_change_sequence_semantics(self) -> None:
base = ["a", "b"]
added = ["a", "b", "c"]
deleted = ["a"]
reordered = ["b", "a"]
keys = [self.make_key(value) for value in (base, added, deleted, reordered)]
self.assertEqual(4, len({key.key_id for key in keys}))
self.assertEqual(
fragment_semantic_hash({"a": 1, "b": 2}),
fragment_semantic_hash({"b": 2, "a": 1}),
)
def test_corrupt_oversized_incompatible_and_foreign_entries_are_misses(self) -> None:
self.assertIsNotNone(self.cache.put(self.key, b"alpha"))
self.entry_path.write_bytes(b"{bad-json")
self.assertIsNone(self.cache.get(self.key))
self.assertIsNotNone(self.cache.put(self.key, b"repaired"))
repaired = self.cache.get(self.key)
self.assertIsNotNone(repaired)
assert repaired is not None
self.assertEqual(b"repaired", repaired.content)
self.entry_path.write_bytes(b"x" * (self.cache.maximum_record_bytes + 1))
self.assertIsNone(self.cache.get(self.key))
incompatible = FragmentRecord.create(self.key, b"alpha").as_dict()
incompatible["schema_version"] = FRAGMENT_SCHEMA_VERSION + 1
self.entry_path.write_bytes(
json.dumps(incompatible, sort_keys=True, separators=(",", ":")).encode()
)
self.assertIsNone(self.cache.get(self.key))
foreign_key = self.make_key({"node": "foreign"})
foreign = FragmentRecord.create(foreign_key, b"foreign").to_bytes()
self.entry_path.write_bytes(foreign)
self.assertIsNone(self.cache.get(self.key))
def test_symlinked_entry_and_cache_root_fail_closed_without_outside_writes(self) -> None:
self.entry_path.parent.mkdir(parents=True)
outside = Path(self.temporary.name) / "outside.json"
outside.write_bytes(FragmentRecord.create(self.key, b"outside").to_bytes())
self.entry_path.symlink_to(outside)
self.assertIsNone(self.cache.get(self.key))
self.assertIsNone(self.cache.put(self.key, b"replacement"))
self.assertEqual(b"outside", FragmentRecord.from_bytes(outside.read_bytes()).content)
outside_cache = Path(self.temporary.name) / "outside-cache"
escaped = ProjectionFragmentCache(self.root, outside_cache)
self.assertIsNone(escaped.put(self.key, b"escaped"))
self.assertFalse(outside_cache.exists())
def test_keys_and_records_have_deterministic_path_free_serialization(self) -> None:
same_key = self.make_key({"summary": "Alpha", "node": "guide.alpha"})
self.assertEqual(self.key, same_key)
first = FragmentRecord.create(self.key, b"\x00fragment\xff")
second = FragmentRecord.create(same_key, b"\x00fragment\xff")
self.assertEqual(first, second)
self.assertEqual(first.to_bytes(), second.to_bytes())
self.assertEqual(first, FragmentRecord.from_bytes(first.to_bytes()))
document = json.loads(first.to_bytes())
self.assertEqual(FRAGMENT_SCHEMA_VERSION, document["schema_version"])
self.assertEqual(FRAGMENT_RECORD_CONTRACT, document["contract"])
self.assertEqual(FRAGMENT_KEY_CONTRACT, document["key"]["contract"])
self.assertNotIn(str(self.root), first.to_bytes().decode("utf-8"))
self.assertEqual(first.byte_count, len(first.content))
self.assertEqual(64, len(first.content_sha256))
def test_cache_write_failure_is_a_miss_and_does_not_mutate_canonical_files(self) -> None:
canonical = self.root / "canonical.md"
canonical.write_text("canonical", encoding="utf-8")
with mock.patch(
"docforge.projection_fragments.atomic_replace_bytes_at",
side_effect=OSError("synthetic cache failure"),
):
self.assertIsNone(self.cache.put(self.key, b"fragment"))
self.assertIsNone(self.cache.get(self.key))
self.assertEqual("canonical", canonical.read_text(encoding="utf-8"))
def test_record_rejects_noncanonical_serialization_and_invalid_evidence(self) -> None:
record = FragmentRecord.create(self.key, b"alpha")
pretty = json.dumps(record.as_dict(), sort_keys=True, indent=2).encode()
with self.assertRaisesRegex(Exception, "canonically serialized"):
FragmentRecord.from_bytes(pretty)
tampered = record.as_dict()
tampered["byte_count"] = record.byte_count + 1
canonical = json.dumps(tampered, sort_keys=True, separators=(",", ":")).encode()
with self.assertRaisesRegex(Exception, "byte evidence"):
FragmentRecord.from_bytes(canonical)
def test_prune_retains_only_the_exact_bounded_active_inventory(self) -> None:
retained = self.key
stale = self.make_key({"node": "stale"})
self.assertIsNotNone(self.cache.put(retained, b"retained"))
self.assertIsNotNone(self.cache.put(stale, b"stale"))
foreign = self.entry_path.parent / "foreign.tmp"
foreign.write_bytes(b"foreign")
self.assertTrue(self.cache.prune((retained,)))
retained_record = self.cache.get(retained)
self.assertIsNotNone(retained_record)
assert retained_record is not None
self.assertEqual(b"retained", retained_record.content)
self.assertIsNone(self.cache.get(stale))
self.assertFalse(foreign.exists())
with mock.patch(
"docforge.projection_fragments.MAX_FRAGMENT_CACHE_BYTES",
1,
):
self.assertFalse(self.cache.prune((retained,)))
if __name__ == "__main__":
unittest.main()

266
tests/test_projection_policy.py Normal file
View file

@ -0,0 +1,266 @@
from __future__ import annotations
import copy
import hashlib
import json
import unittest
from dataclasses import FrozenInstanceError
from pathlib import Path
from jsonschema import Draft202012Validator
from docforge.errors import DocForgeError
from docforge.projection_policy import (
ProjectionPolicyV2,
compose_projection_policy,
)
ROOT = Path(__file__).resolve().parents[1]
POLICY_SCHEMA = json.loads(
(ROOT / "schemas" / "projection-policy.schema.json").read_text(encoding="utf-8")
)
class ProjectionPolicyV2Tests(unittest.TestCase):
def test_compatible_defaults_are_independent_and_schema_valid(self) -> None:
cases = (
(
{
"manual_configured": False,
"portable_graph_configured": False,
"application_enabled": False,
},
("disabled", "disabled", "on-demand"),
),
(
{
"manual_configured": True,
"portable_graph_configured": False,
"application_enabled": False,
},
("explicit", "disabled", "on-demand"),
),
(
{
"manual_configured": True,
"portable_graph_configured": True,
"application_enabled": True,
},
("auto", "explicit", "on-demand"),
),
(
{
"manual_configured": True,
"portable_graph_configured": True,
"application_enabled": False,
"live_viewer_available": False,
},
("explicit", "explicit", "disabled"),
),
)
validator = Draft202012Validator(POLICY_SCHEMA)
for arguments, expected in cases:
with self.subTest(arguments=arguments):
policy = compose_projection_policy(**arguments)
self.assertEqual(
expected, (policy.manual, policy.portable_graph, policy.live_viewer)
)
validator.validate(policy.as_dict())
def test_explicit_selections_can_narrow_each_consumer_independently(self) -> None:
policy = compose_projection_policy(
manual="explicit",
portable_graph="disabled",
live_viewer="disabled",
manual_configured=True,
portable_graph_configured=True,
application_enabled=True,
)
self.assertEqual(
{
"schema_version": 2,
"manual": "explicit",
"portable_graph": "disabled",
"live_viewer": "disabled",
},
policy.as_dict(),
)
fully_disabled = compose_projection_policy(
manual="disabled",
portable_graph="disabled",
live_viewer="disabled",
manual_configured=False,
portable_graph_configured=False,
application_enabled=False,
live_viewer_available=False,
)
self.assertEqual("disabled", fully_disabled.manual)
self.assertEqual("disabled", fully_disabled.portable_graph)
self.assertEqual("disabled", fully_disabled.live_viewer)
def test_policy_is_frozen_and_hashes_exact_canonical_payload(self) -> None:
first = compose_projection_policy(
manual_configured=True,
portable_graph_configured=True,
application_enabled=False,
)
second = compose_projection_policy(
manual_configured=True,
portable_graph_configured=True,
application_enabled=False,
)
canonical = json.dumps(
first.as_dict(),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
self.assertEqual(first.as_dict(), second.as_dict())
self.assertEqual(hashlib.sha256(canonical).hexdigest(), first.policy_hash)
self.assertEqual(first.policy_hash, second.policy_hash)
changed = compose_projection_policy(
manual="disabled",
manual_configured=True,
portable_graph_configured=True,
application_enabled=False,
)
self.assertNotEqual(first.policy_hash, changed.policy_hash)
with self.assertRaises(FrozenInstanceError):
first.manual = "disabled" # type: ignore[misc]
def test_invalid_modes_and_direct_construction_fail_closed(self) -> None:
cases = (
("manual", {"manual": "sometimes"}),
("portable_graph", {"portable_graph": "auto"}),
("live_viewer", {"live_viewer": "always"}),
("manual", {"manual": 1}),
)
for expected_projection, selection in cases:
with self.subTest(selection=selection):
with self.assertRaises(DocForgeError) as raised:
compose_projection_policy(
**selection,
manual_configured=True,
portable_graph_configured=True,
application_enabled=True,
)
self.assertEqual("invalid_projection_policy", raised.exception.code)
self.assertEqual(
expected_projection,
raised.exception.details["projection"],
)
with self.assertRaises(DocForgeError) as direct:
ProjectionPolicyV2(
manual="automatic", # type: ignore[arg-type]
portable_graph="explicit",
live_viewer="on-demand",
)
self.assertEqual("invalid_projection_policy", direct.exception.code)
def test_resource_and_capability_unavailability_are_distinct(self) -> None:
cases = (
(
{"manual": "explicit"},
{
"manual_configured": False,
"portable_graph_configured": False,
"application_enabled": False,
},
"manual",
"manual_render_config",
),
(
{"manual": "auto"},
{
"manual_configured": True,
"portable_graph_configured": False,
"application_enabled": False,
},
"manual",
"canonical_application",
),
(
{"portable_graph": "explicit"},
{
"manual_configured": False,
"portable_graph_configured": False,
"application_enabled": False,
},
"portable_graph",
"portable_graph_render_config",
),
(
{"live_viewer": "on-demand"},
{
"manual_configured": False,
"portable_graph_configured": False,
"application_enabled": False,
"live_viewer_available": False,
},
"live_viewer",
"live_viewer_runtime",
),
)
for selection, availability, projection, required in cases:
with self.subTest(selection=selection):
with self.assertRaises(DocForgeError) as raised:
compose_projection_policy(**selection, **availability)
self.assertEqual("projection_policy_unavailable", raised.exception.code)
self.assertEqual(projection, raised.exception.details["projection"])
self.assertEqual(required, raised.exception.details["required"])
def test_availability_inputs_must_be_real_booleans(self) -> None:
cases = (
{"manual_configured": 1},
{"portable_graph_configured": 0},
{"application_enabled": "yes"},
{"live_viewer_available": None},
)
for replacement in cases:
arguments = {
"manual_configured": False,
"portable_graph_configured": False,
"application_enabled": False,
"live_viewer_available": True,
**replacement,
}
with self.subTest(replacement=replacement):
with self.assertRaises(DocForgeError) as raised:
compose_projection_policy(**arguments)
self.assertEqual("invalid_projection_policy", raised.exception.code)
def test_schema_rejects_every_runtime_contract_drift(self) -> None:
validator = Draft202012Validator(POLICY_SCHEMA)
valid = compose_projection_policy(
manual_configured=True,
portable_graph_configured=True,
application_enabled=True,
).as_dict()
invalid_documents: list[dict[str, object]] = []
for field in ("schema_version", "manual", "portable_graph", "live_viewer"):
missing = copy.deepcopy(valid)
missing.pop(field)
invalid_documents.append(missing)
for field, value in (
("schema_version", 1),
("manual", "on-demand"),
("portable_graph", "auto"),
("live_viewer", "explicit"),
):
changed = copy.deepcopy(valid)
changed[field] = value
invalid_documents.append(changed)
extra = copy.deepcopy(valid)
extra["project_path"] = "/private/project"
invalid_documents.append(extra)
for document in invalid_documents:
with self.subTest(document=document):
self.assertFalse(validator.is_valid(document))
if __name__ == "__main__":
unittest.main()

239
tests/test_projection_policy_integration.py Normal file
View file

@ -0,0 +1,239 @@
from __future__ import annotations
import shutil
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from docforge.application import CanonicalApplicationService
from docforge.errors import DocForgeError
from docforge.graph_rendering import GraphRenderService
from docforge.index import ProjectIndex
from docforge.mcp_server import DocForgeService
from docforge.project import Project
from docforge.rendering import RenderService
from docforge.viewer_manager import ViewerManagerClient
ROOT = Path(__file__).resolve().parents[1]
FIXTURES = ROOT / "tests" / "fixtures"
GRAPH_CONFIG = """
[graph_render]
output_root = ".docforge/portable-graph"
[[graph_render.views]]
id = "architecture"
renderer = "portable_graph_html"
output = "architecture.html"
title = "Alpha architecture"
root = "guide.workflow"
initial_mode = "nodes"
depth = 2
max_nodes = 20
max_edges = 40
max_work = 1000
include_logic = false
"""
class ProjectionPolicyIntegrationTests(unittest.TestCase):
def setUp(self) -> None:
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name) / "alpha"
shutil.copytree(FIXTURES / "alpha", self.root)
def project(self, *, graph: bool = False) -> Project:
if graph:
descriptor = self.root / ".docforge/project.toml"
descriptor.write_text(
descriptor.read_text(encoding="utf-8") + GRAPH_CONFIG,
encoding="utf-8",
)
return Project.open(self.root)
def test_disabled_manual_blocks_work_before_load_but_status_remains_receipt_only(self) -> None:
project = self.project()
service = RenderService(project, manual_policy="disabled")
with mock.patch.object(
project,
"load",
side_effect=AssertionError("disabled manual policy must fail before loading"),
):
status = service.status("manual")
self.assertEqual("stale", status["state"])
for operation in (
lambda: service.render("manual"),
lambda: service.deep_status("manual"),
lambda: service.preview("proposal", "manual"),
):
with self.subTest(operation=operation), self.assertRaises(DocForgeError) as raised:
operation()
self.assertEqual(
"projection_policy_forbids_operation",
raised.exception.code,
)
def test_disabled_graph_blocks_plan_and_render_but_status_remains_available(self) -> None:
project = self.project(graph=True)
service = GraphRenderService(project, portable_graph_policy="disabled")
with mock.patch.object(
project,
"load",
side_effect=AssertionError("disabled graph policy must fail before loading"),
):
status = service.status("architecture")
self.assertEqual("stale", status["state"])
for operation in (
lambda: service.plan("architecture"),
lambda: service.render("architecture"),
):
with self.subTest(operation=operation), self.assertRaises(DocForgeError) as raised:
operation()
self.assertEqual(
"projection_policy_forbids_operation",
raised.exception.code,
)
def test_live_viewer_disabled_blocks_start_before_index_work_but_allows_lifecycle(self) -> None:
project = self.project()
index = ProjectIndex(project)
client = ViewerManagerClient(index, live_viewer_policy="disabled")
with (
mock.patch.object(
index,
"check",
side_effect=AssertionError("disabled viewer must fail before index work"),
),
self.assertRaises(DocForgeError) as raised,
):
client.start(node_id="guide.workflow")
self.assertEqual("projection_policy_forbids_operation", raised.exception.code)
with mock.patch.object(
client,
"_lifecycle_request",
side_effect=(
{"status": "ok", "state": "not_running"},
{"status": "ok", "state": "stopped"},
),
):
self.assertEqual("not_running", client.status()["state"])
self.assertEqual("stopped", client.stop()["state"])
def test_application_auto_renders_and_explicit_or_disabled_skip_cleanly(self) -> None:
project = self.project()
for mode, expected_action, expected_calls in (
("auto", "rendered", 1),
("explicit", "skipped_explicit", 0),
("disabled", "skipped_disabled", 0),
):
with self.subTest(mode=mode):
service = CanonicalApplicationService(
project,
applier_id="alpha-editor",
applier=mock.Mock(),
manual_policy=mode, # type: ignore[arg-type]
)
with (
mock.patch.object(
service.changesets,
"apply",
return_value={"status": "ok", "applied": True},
),
mock.patch.object(
service.index,
"build",
return_value={"status": "ok"},
),
mock.patch.object(
service.index,
"check",
return_value={"status": "ok"},
),
mock.patch.object(
service.rendering,
"render",
return_value={"status": "ok", "state": "current"},
) as rendered,
):
result = service.apply("policy-application", "a" * 64)
refresh = result["derived_refresh"]
assert isinstance(refresh, dict)
self.assertEqual(
{"mode": mode, "action": expected_action},
refresh["render_policy"],
)
self.assertEqual(expected_calls, rendered.call_count)
self.assertEqual("ok", refresh["status"])
def test_mcp_exposes_v2_without_changing_v1_policy_or_legacy_render_projection(self) -> None:
project = self.project(graph=True)
baseline = DocForgeService(project, capability_mode_name="read")
narrowed = DocForgeService(
project,
capability_mode_name="read",
manual_projection_policy="disabled",
portable_graph_policy="disabled",
live_viewer_policy="disabled",
)
self.assertEqual(baseline.policy.as_dict(), narrowed.policy.as_dict())
result = narrowed.bootstrap()
self.assertEqual(narrowed.policy.as_dict(), result["effective_policy"])
self.assertEqual(
narrowed.projection_policy.as_dict(),
result["projection_policy"],
)
self.assertEqual(
narrowed.projection_policy.policy_hash,
result["projection_policy_hash"],
)
session = result["session_contract"]
assert isinstance(session, dict)
self.assertEqual(
{
"manual": narrowed.policy.manual_render,
"graph": narrowed.policy.graph_render,
"live_viewer": narrowed.policy.live_viewer,
},
session["render_policies"],
)
self.assertEqual(
"stale",
narrowed.graph_render_status("architecture")["state"],
)
blocked_plan = narrowed.graph_plan("architecture")
self.assertEqual(
"projection_policy_forbids_operation",
blocked_plan["error"]["code"],
)
def test_direct_service_policy_values_are_runtime_validated(self) -> None:
project = self.project(graph=True)
factories = (
lambda: RenderService(project, manual_policy="bogus"), # type: ignore[arg-type]
lambda: GraphRenderService(
project,
portable_graph_policy="bogus", # type: ignore[arg-type]
),
lambda: ViewerManagerClient(
ProjectIndex(project),
live_viewer_policy="bogus", # type: ignore[arg-type]
),
lambda: CanonicalApplicationService(
project,
applier_id=None,
applier=None,
manual_policy="bogus", # type: ignore[arg-type]
),
)
for factory in factories:
with self.subTest(factory=factory), self.assertRaises(DocForgeError) as raised:
factory()
self.assertEqual("invalid_projection_policy", raised.exception.code)
if __name__ == "__main__":
unittest.main()

387
tests/test_projection_worker.py Normal file
View file

@ -0,0 +1,387 @@
from __future__ import annotations
import base64
import contextlib
import copy
import json
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from docforge.errors import DocForgeError
from docforge.graph_projection import (
GraphViewRequestV1,
build_graph_projection_package,
build_graph_view_plan,
)
from docforge.manual_projection import (
build_manual_projection_package,
build_manual_render_plan,
)
from docforge.project import Project
from docforge.projection_contract import (
ProjectionPackageV1,
ProjectionReceiptV1,
canonical_projection_bytes,
)
from docforge.projection_worker import (
MAX_WORKER_ARTIFACT_BYTES,
MAX_WORKER_RESPONSE_BYTES,
_child_response,
render_projection_in_worker,
)
from docforge.render_contract import GenericHtmlRenderer
from docforge_renderers.graph import PortableGraphHtmlRenderer
from docforge_renderers.manual import ManualHtmlRenderer
ROOT = Path(__file__).resolve().parents[1]
FIXTURES = ROOT / "tests" / "fixtures"
class ProjectionWorkerTests(unittest.TestCase):
def setUp(self) -> None:
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name) / "alpha"
shutil.copytree(FIXTURES / "alpha", self.root)
self.project = Project.open(self.root)
self.snapshot = self.project.load()
render = self.snapshot.descriptor.render
assert render is not None
self.manual_view = render.views[0]
self.manual_version = GenericHtmlRenderer().renderer_version
self.manual_plan = build_manual_render_plan(
self.snapshot,
self.manual_view,
changeset_hash=None,
)
self.manual_package = build_manual_projection_package(
self.manual_plan,
self.manual_view.template_path.read_bytes(),
renderer_id="generic_html",
renderer_version=self.manual_version,
max_output_bytes=self.snapshot.descriptor.limits.max_render_bytes,
)
self.graph_plan = build_graph_view_plan(
self.snapshot,
GraphViewRequestV1(
view_id="architecture",
title="Alpha architecture",
root_node_id="guide.workflow",
depth=2,
max_nodes=20,
max_edges=40,
max_work=1_000,
),
False,
)
self.graph_package = build_graph_projection_package(
self.graph_plan,
renderer_id=PortableGraphHtmlRenderer.renderer_id,
renderer_version=PortableGraphHtmlRenderer.renderer_version,
max_output_bytes=self.snapshot.descriptor.limits.max_render_bytes,
)
def test_manual_and_graph_workers_match_the_in_process_renderers(self) -> None:
cases = (
(
self.manual_package,
ManualHtmlRenderer(self.manual_version).render(self.manual_package),
),
(
self.graph_package,
PortableGraphHtmlRenderer().render(self.graph_package),
),
)
for package, expected in cases:
with self.subTest(kind=package.kind):
result = render_projection_in_worker(package)
self.assertEqual(expected.artifacts, result.artifacts)
receipt = result.receipt.as_dict()
self.assertEqual(package.package_id, receipt["package_id"])
self.assertEqual(package.document["plan_id"], receipt["plan_id"])
self.assertIs(type(receipt["peak_memory_bytes"]), int)
self.assertGreater(receipt["peak_memory_bytes"], 0)
def test_parent_launch_is_fixed_and_request_contains_no_runtime_authority(self) -> None:
original = subprocess.run
with mock.patch(
"docforge.projection_worker.subprocess.run",
wraps=original,
) as launched:
render_projection_in_worker(self.manual_package)
command = launched.call_args.args[0]
options = launched.call_args.kwargs
self.assertEqual(
[sys.executable, "-I", "-m", "docforge.projection_worker"],
command,
)
self.assertIs(options["shell"], False)
self.assertEqual(sys.prefix, options["cwd"])
self.assertEqual(
{"PYTHONIOENCODING": "utf-8", "PYTHONUTF8": "1"},
options["env"],
)
request = options["input"]
self.assertEqual(
canonical_projection_bytes(self.manual_package.as_dict()) + b"\n",
request,
)
for forbidden in (
str(self.root).encode(),
b"project_path",
b"index_path",
b"database_path",
b'"command"',
b'"module"',
b'"shell"',
b'"sql"',
):
self.assertNotIn(forbidden, request)
def test_worker_ignores_hostile_cwd_and_pythonpath_import_shadows(self) -> None:
shadow_root = Path(self.temporary.name) / "shadow"
shadow_package = shadow_root / "docforge"
shadow_package.mkdir(parents=True)
sentinel = shadow_root / "executed"
shadow_package.joinpath("__init__.py").write_text(
"from pathlib import Path\n"
f"Path({str(sentinel)!r}).write_text('executed', encoding='utf-8')\n",
encoding="utf-8",
)
with (
contextlib.chdir(shadow_root),
mock.patch.dict(
os.environ,
{"PYTHONPATH": str(shadow_root)},
clear=False,
),
):
result = render_projection_in_worker(self.manual_package)
self.assertEqual("manual", result.receipt.document["kind"])
self.assertFalse(sentinel.exists())
def test_parent_accepts_only_valid_packages_and_closed_renderer_versions(self) -> None:
with self.assertRaises(TypeError):
render_projection_in_worker({}) # type: ignore[arg-type]
unsupported = ProjectionPackageV1.create(
kind="manual",
plan=self.manual_plan,
renderer={"renderer_id": "generic_html", "renderer_version": "other"},
components=[{"component_id": "manual.document@1"}],
assets=copy.deepcopy(self.manual_package.document["assets"]),
output_policy=copy.deepcopy(self.manual_package.document["output_policy"]),
)
with (
mock.patch("docforge.projection_worker._invoke_worker") as invoke,
self.assertRaises(DocForgeError) as raised,
):
render_projection_in_worker(unsupported)
self.assertEqual("unsupported_renderer", raised.exception.code)
invoke.assert_not_called()
permissive_allowance = ProjectionPackageV1.create(
kind="manual",
plan=self.manual_plan,
renderer={
"renderer_id": "generic_html",
"renderer_version": self.manual_version,
},
components=[{"component_id": "manual.document@1"}],
assets=copy.deepcopy(self.manual_package.document["assets"]),
output_policy={
"artifact_ids": ["manual.html"],
"max_total_bytes": MAX_WORKER_ARTIFACT_BYTES + 1,
},
)
rendered = render_projection_in_worker(permissive_allowance)
self.assertEqual(
ManualHtmlRenderer(self.manual_version)
.render(permissive_allowance)
.artifacts[0]
.content,
rendered.artifacts[0].content,
)
def test_process_timeout_exit_and_signal_fail_closed(self) -> None:
failures = (
(
subprocess.TimeoutExpired(["worker"], 30),
"projection_worker_timeout",
),
(
subprocess.CompletedProcess(["worker"], 2, stdout=b""),
"projection_worker_failure",
),
(
subprocess.CompletedProcess(["worker"], -9, stdout=b""),
"projection_worker_failure",
),
)
for outcome, code in failures:
with self.subTest(outcome=type(outcome).__name__):
patch = (
mock.patch(
"docforge.projection_worker._invoke_worker",
side_effect=outcome,
)
if isinstance(outcome, BaseException)
else mock.patch(
"docforge.projection_worker._invoke_worker",
return_value=outcome,
)
)
with patch, self.assertRaises(DocForgeError) as raised:
render_projection_in_worker(self.manual_package)
self.assertEqual(code, raised.exception.code)
def test_malformed_trailing_and_oversized_responses_fail_closed(self) -> None:
valid = _child_response(self.manual_package)
malformed = (
b"",
b"{}",
b"not-json\n",
b'{ "schema_version": 1 }\n',
valid + b"{}\n",
b"x" * (MAX_WORKER_RESPONSE_BYTES + 1),
)
for response in malformed:
with (
self.subTest(length=len(response)),
mock.patch(
"docforge.projection_worker._invoke_worker",
return_value=subprocess.CompletedProcess(
["worker"],
0,
stdout=response,
),
),
self.assertRaises(DocForgeError) as raised,
):
render_projection_in_worker(self.manual_package)
self.assertEqual("projection_worker_failure", raised.exception.code)
def test_wrong_artifact_and_receipt_evidence_fail_closed(self) -> None:
valid = json.loads(_child_response(self.manual_package))
cases: dict[str, dict[str, object]] = {}
wrong_content = copy.deepcopy(valid)
wrong_content["artifacts"][0]["content_base64"] = base64.b64encode(b"changed").decode()
cases["content_hash"] = wrong_content
wrong_id = copy.deepcopy(valid)
wrong_id["artifacts"][0]["artifact_id"] = "other.html"
cases["artifact_id"] = wrong_id
missing_artifact = copy.deepcopy(valid)
missing_artifact["artifacts"] = []
cases["artifact_count"] = missing_artifact
wrong_media_type = copy.deepcopy(valid)
wrong_media_type["artifacts"][0]["media_type"] = "application/octet-stream"
cases["media_type"] = wrong_media_type
bad_base64 = copy.deepcopy(valid)
bad_base64["artifacts"][0]["content_base64"] = "***"
cases["base64"] = bad_base64
wrong_package = copy.deepcopy(valid)
receipt = wrong_package["receipt"]
replacement = ProjectionReceiptV1.create(
kind="manual",
package_id="0" * 64,
plan_id=receipt["plan_id"],
renderer=receipt["renderer"],
artifacts=receipt["artifacts"],
diagnostics=receipt["diagnostics"],
timing=receipt["timing"],
peak_memory_bytes=receipt["peak_memory_bytes"],
)
wrong_package["receipt"] = replacement.as_dict()
cases["package_id"] = wrong_package
zero_peak = copy.deepcopy(valid)
receipt = zero_peak["receipt"]
replacement = ProjectionReceiptV1.create(
kind="manual",
package_id=receipt["package_id"],
plan_id=receipt["plan_id"],
renderer=receipt["renderer"],
artifacts=receipt["artifacts"],
diagnostics=receipt["diagnostics"],
timing=receipt["timing"],
peak_memory_bytes=0,
)
zero_peak["receipt"] = replacement.as_dict()
cases["peak_memory"] = zero_peak
wrong_size = copy.deepcopy(valid)
receipt = wrong_size["receipt"]
artifacts = copy.deepcopy(receipt["artifacts"])
artifacts[0]["bytes"] += 1
replacement = ProjectionReceiptV1.create(
kind="manual",
package_id=receipt["package_id"],
plan_id=receipt["plan_id"],
renderer=receipt["renderer"],
artifacts=artifacts,
diagnostics=receipt["diagnostics"],
timing=receipt["timing"],
peak_memory_bytes=receipt["peak_memory_bytes"],
)
wrong_size["receipt"] = replacement.as_dict()
cases["artifact_size"] = wrong_size
configured_oversize = copy.deepcopy(valid)
maximum = self.manual_package.document["output_policy"]["max_total_bytes"]
configured_oversize["artifacts"][0]["content_base64"] = base64.b64encode(
b"x" * (maximum + 1)
).decode()
cases["configured_aggregate"] = configured_oversize
for name, document in cases.items():
response = canonical_projection_bytes(document) + b"\n"
with (
self.subTest(name=name),
mock.patch(
"docforge.projection_worker._invoke_worker",
return_value=subprocess.CompletedProcess(
["worker"],
0,
stdout=response,
),
),
self.assertRaises(DocForgeError) as raised,
):
render_projection_in_worker(self.manual_package)
self.assertEqual("projection_worker_failure", raised.exception.code)
def test_child_rejects_noncanonical_invalid_and_trailing_requests(self) -> None:
valid = canonical_projection_bytes(self.manual_package.as_dict()) + b"\n"
requests = (
b"{}\n",
b'{ "schema_version": 1 }\n',
valid + b"{}\n",
)
for request in requests:
with self.subTest(length=len(request)):
completed = subprocess.run(
[sys.executable, "-m", "docforge.projection_worker"],
cwd=ROOT,
input=request,
capture_output=True,
check=False,
timeout=10,
)
self.assertEqual(2, completed.returncode)
self.assertEqual(b"", completed.stdout)
if __name__ == "__main__":
unittest.main()

19
tests/test_public_contract.py
View file

@ -94,6 +94,19 @@ PUBLIC_IMPORTS = {
"capability_mode",
"compose_effective_policy",
),
"docforge.projection_policy": (
"ProjectionPolicyV2",
"compose_projection_policy",
"validate_live_viewer_projection_mode",
"validate_manual_projection_mode",
"validate_portable_graph_projection_mode",
),
"docforge.projection_fragments": (
"FragmentKey",
"FragmentRecord",
"ProjectionFragmentCache",
"fragment_semantic_hash",
),
"docforge.projection_contract": (
"GraphViewPlanV1",
"ManualRenderPlanV1",
@ -104,6 +117,7 @@ PUBLIC_IMPORTS = {
"canonical_projection_bytes",
"projection_hash",
),
"docforge.projection_worker": ("render_projection_in_worker",),
"docforge.retrieval": (
"ContextCapsuleV1",
"RetrievalPlanV1",
@ -186,6 +200,8 @@ EXPECTED_MCP_TOOLS = {
"docforge_rebase_changeset",
"docforge_register_changes",
"docforge_render_status",
"docforge_graph_plan",
"docforge_graph_render_status",
"docforge_search",
"docforge_stop_visualization",
"docforge_sync",
@ -242,6 +258,9 @@ class PublicContractTests(unittest.TestCase):
)
self.assertIn("--project-root", completed.stdout)
self.assertIn("--no-ast", completed.stdout)
self.assertIn("--manual-render-policy", completed.stdout)
self.assertIn("--portable-graph-policy", completed.stdout)
self.assertIn("--live-viewer-policy", completed.stdout)
def test_published_schemas_validate_their_current_contract_examples(self) -> None:
for path in sorted(SCHEMAS.glob("*.json")):

30
tests/test_rendering.py
View file

@ -55,6 +55,14 @@ class DocForgeRenderingTests(unittest.TestCase):
self.assertEqual("missing", missing["outputs"][0]["state"])
first = service.render("manual")
projection_receipt = first["output"]["projection_receipt"]
self.assertIsInstance(projection_receipt, dict)
assert isinstance(projection_receipt, dict)
self.assertEqual("manual", projection_receipt["kind"])
self.assertEqual(
first["output"]["actual_output_hash"],
projection_receipt["artifacts"][0]["sha256"],
)
output = root / ".docforge/rendered/manual.html"
first_bytes = output.read_bytes()
second = service.render("manual")
@ -191,7 +199,12 @@ class DocForgeRenderingTests(unittest.TestCase):
self.assertEqual("receipt_corrupt", corrupt["outputs"][0]["reason"])
def test_render_receipt_schema_and_renderer_version_fail_closed(self) -> None:
for mutation in ("missing_hash", "renderer_version", "file_identity"):
for mutation in (
"missing_hash",
"renderer_version",
"file_identity",
"projection_artifact",
):
with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
root = self.copy_fixture("alpha", Path(directory))
service = RenderService(Project.open(root))
@ -202,6 +215,8 @@ class DocForgeRenderingTests(unittest.TestCase):
receipt.pop("output_hash")
elif mutation == "renderer_version":
receipt["renderer_version"] = "obsolete"
elif mutation == "projection_artifact":
receipt["projection_receipt"]["artifacts"][0]["sha256"] = "0" * 64
else:
receipt["output_file"].pop("ctime_ns")
receipt_path.write_text(
@ -401,6 +416,19 @@ class DocForgeRenderingTests(unittest.TestCase):
with self.assertRaisesRegex(DocForgeError, "configured limit") as limit_error:
limit_service.render("manual")
self.assertEqual("render_too_large", limit_error.exception.code)
excessive = self.copy_fixture("alpha", Path(directory) / "excessive")
excessive_descriptor = excessive / ".docforge/project.toml"
excessive_descriptor.write_text(
excessive_descriptor.read_text(encoding="utf-8").replace(
"max_changeset_bytes = 100000",
"max_changeset_bytes = 100000\nmax_render_bytes = 20000001",
),
encoding="utf-8",
)
permissive_limit = RenderService(Project.open(excessive)).render("manual")
self.assertEqual("current", permissive_limit["receipt"]["state"])
self.assertTrue((excessive / ".docforge/rendered/manual.html").is_file())
self.assertFalse((limit_root / ".docforge/rendered/manual.html").exists())
template_limit_root = self.copy_fixture("alpha", parent / "template-limit")

99
tools/accessibility_fixture.py Normal file
View file

@ -0,0 +1,99 @@
"""Serve exact DocForge accessibility fixtures to the Playwright gate."""
from __future__ import annotations
import json
import shutil
import sys
import tempfile
from pathlib import Path
from typing import cast
from docforge.graph_projection import (
GraphViewRequestV1,
build_graph_projection_package,
build_graph_view_plan,
)
from docforge.index import ProjectIndex
from docforge.project import Project
from docforge.rendering import RenderService
from docforge.visualization import VisualizationRunner
from docforge_renderers.graph import PortableGraphHtmlRenderer
ROOT = Path(__file__).resolve().parents[1]
def _manual_html(project: Project) -> str:
result = RenderService(project).render("manual")
output = result.get("output")
if not isinstance(output, dict):
raise RuntimeError("Manual render did not return output identity")
relative_path = cast(dict[str, object], output).get("path")
if not isinstance(relative_path, str):
raise RuntimeError("Manual render did not return an output path")
return (project.descriptor.root / relative_path).read_text(encoding="utf-8")
def _portable_graph_html(project: Project) -> str:
plan = build_graph_view_plan(
project.load(),
GraphViewRequestV1(
view_id="accessibility-gate",
title="Portable graph accessibility fixture",
root_node_id="guide.workflow",
depth=2,
max_nodes=20,
max_edges=40,
max_work=1_000,
),
False,
)
package = build_graph_projection_package(
plan,
renderer_id=PortableGraphHtmlRenderer.renderer_id,
renderer_version=PortableGraphHtmlRenderer.renderer_version,
max_output_bytes=1_000_000,
)
artifact = PortableGraphHtmlRenderer().render(package).artifacts[0]
return artifact.content.decode("utf-8")
def main() -> int:
with tempfile.TemporaryDirectory(prefix="docforge-accessibility-") as directory:
project_root = Path(directory) / "alpha"
shutil.copytree(ROOT / "tests" / "fixtures" / "alpha", project_root)
project = Project.open(project_root)
index = ProjectIndex(project)
index.build()
portable_html = _portable_graph_html(project)
manual_html = _manual_html(project)
runner = VisualizationRunner(
index,
register_atexit=False,
persistent=True,
)
try:
visualization = runner.start(node_id="guide.workflow", depth=2)
live_url = visualization.get("url")
if not isinstance(live_url, str):
raise RuntimeError("Live viewer did not return a URL")
print(
json.dumps(
{
"schema_version": 1,
"manual_html": manual_html,
"portable_html": portable_html,
"live_url": live_url,
},
separators=(",", ":"),
),
flush=True,
)
sys.stdin.buffer.read()
finally:
runner.stop()
return 0
if __name__ == "__main__":
sys.exit(main())

842
tools/milestone3_benchmark.py Normal file
View file

@ -0,0 +1,842 @@
"""Milestone 3 projection, worker, fragment, and publication benchmark gates."""
from __future__ import annotations
import argparse
import gc
import hashlib
import json
import math
import platform
import resource
import statistics
import subprocess
import sys
import tempfile
import time
import tracemalloc
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, replace
from pathlib import Path
from typing import cast
from milestone0_baseline import write_synthetic_project
from docforge.graph_projection import (
GraphViewRequestV1,
build_graph_projection_package,
build_graph_view_plan,
)
from docforge.graph_rendering import GraphRenderService
from docforge.manual_projection import (
build_manual_projection_package,
build_manual_render_plan,
)
from docforge.models import ProjectSnapshot
from docforge.project import Project
from docforge.projection_contract import (
MAX_PACKAGE_BYTES,
MAX_PLAN_BYTES,
MAX_RECEIPT_BYTES,
GraphViewPlanV1,
ManualRenderPlanV1,
ProjectionPackageV1,
ProjectionRenderResult,
canonical_projection_bytes,
projection_hash,
)
from docforge.projection_fragments import (
FragmentKey,
FragmentRecord,
ProjectionFragmentCache,
fragment_semantic_hash,
)
from docforge.projection_worker import (
MAX_WORKER_ARTIFACT_BYTES,
render_projection_in_worker,
)
from docforge.render_contract import GenericHtmlRenderer, PreparedRender
from docforge.rendering import RenderService
from docforge.telemetry import COUNTER_NAMES, request
from docforge_renderers.graph import PortableGraphHtmlRenderer
from docforge_renderers.manual import ManualHtmlRenderer
ROOT = Path(__file__).resolve().parents[1]
FULL_NODE_COUNT = 1_000
SMOKE_NODE_COUNT = 25
DEFAULT_FULL_SAMPLES = 3
DEFAULT_SMOKE_SAMPLES = 1
MAX_STATUS_RESPONSE_BYTES = 256_000
MAX_TRACED_PEAK_BYTES = 256 * 1024 * 1024
MAX_CHILD_PEAK_BYTES = 256 * 1024 * 1024
ZERO_WORK_COUNTERS = tuple(
counter for counter in COUNTER_NAMES if counter != "source_generation_checks"
)
@dataclass(frozen=True)
class _ProjectionSample:
plan_id: str
package_id: str
artifact: bytes
receipt: Mapping[str, object]
plan_bytes: int
package_bytes: int
child_peak_memory_bytes: int | None
@dataclass(frozen=True)
class _StatusSample:
response: Mapping[str, object]
counters: Mapping[str, object]
@dataclass(frozen=True)
class _FragmentSweep:
fragment_count: int
aggregate_content_bytes: int
ordered_record_hash: str
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Gate DocForge2 Milestone 3 projection behavior on a disposable project."
)
parser.add_argument("--mode", choices=("smoke", "full"), default="full")
parser.add_argument("--nodes", type=int)
parser.add_argument("--samples", type=int)
parser.add_argument("--output", type=Path)
return parser
def encode_report(value: object) -> str:
"""Serialize one report deterministically for files, CI logs, and comparisons."""
return json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False) + "\n"
def _git(arguments: list[str]) -> str:
return subprocess.run(
["git", *arguments],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
).stdout.strip()
def _compact_size(value: object) -> int:
return len(canonical_projection_bytes(value))
def _sha256(content: bytes) -> str:
return hashlib.sha256(content).hexdigest()
def _prepare_fixture(root: Path, node_count: int) -> None:
write_synthetic_project(root, node_count)
descriptor = root / ".docforge" / "project.toml"
maximum_edges = node_count - 1
maximum_work = max(100, node_count * 4)
original = descriptor.read_text(encoding="utf-8")
original = original.replace(
"max_render_bytes = 20000000",
"max_render_bytes = 4000000",
)
descriptor.write_text(
original
+ f"""
[graph_render]
output_root = ".docforge/portable-graph"
[[graph_render.views]]
id = "architecture"
renderer = "portable_graph_html"
output = "architecture.html"
title = "Synthetic architecture"
query = "synthetic measurement"
initial_mode = "web"
depth = 1
max_nodes = {node_count}
max_edges = {maximum_edges}
max_work = {maximum_work}
families = ["guide"]
relations = ["depends_on"]
authorities = []
statuses = ["active"]
tags = []
include_logic = false
""",
encoding="utf-8",
)
def _manual_package(
snapshot: ProjectSnapshot,
) -> tuple[ManualRenderPlanV1, ProjectionPackageV1, str]:
descriptor = snapshot.descriptor
render = descriptor.render
if render is None:
raise RuntimeError("Milestone 3 fixture has no manual render configuration")
view = render.views[0]
renderer_version = GenericHtmlRenderer(incremental=False).renderer_version
plan = build_manual_render_plan(snapshot, view, changeset_hash=None)
package = build_manual_projection_package(
plan,
view.template_path.read_bytes(),
renderer_id=ManualHtmlRenderer.renderer_id,
renderer_version=renderer_version,
max_output_bytes=descriptor.limits.max_render_bytes,
)
return plan, package, renderer_version
def _graph_package(
snapshot: ProjectSnapshot,
node_count: int,
) -> tuple[GraphViewPlanV1, ProjectionPackageV1]:
descriptor = snapshot.descriptor
plan = build_graph_view_plan(
snapshot,
GraphViewRequestV1(
view_id="architecture",
title="Synthetic architecture",
query="synthetic measurement",
initial_mode="web",
depth=1,
max_nodes=node_count,
max_edges=node_count - 1,
max_work=max(100, node_count * 4),
families=("guide",),
relations=("depends_on",),
statuses=("active",),
),
False,
)
package = build_graph_projection_package(
plan,
renderer_id=PortableGraphHtmlRenderer.renderer_id,
renderer_version=PortableGraphHtmlRenderer.renderer_version,
max_output_bytes=descriptor.limits.max_render_bytes,
)
return plan, package
def _projection_sample(
plan: ManualRenderPlanV1 | GraphViewPlanV1,
package: ProjectionPackageV1,
result: ProjectionRenderResult,
) -> _ProjectionSample:
if len(result.artifacts) != 1:
raise RuntimeError("Projection benchmark expected exactly one artifact")
artifact = result.artifacts[0].content
peak = result.receipt.document.get("peak_memory_bytes")
if peak is not None and (type(peak) is not int or peak < 1):
raise RuntimeError("Detached worker did not report valid peak memory")
return _ProjectionSample(
plan_id=plan.plan_id,
package_id=package.package_id,
artifact=artifact,
receipt=result.receipt.as_dict(),
plan_bytes=_compact_size(plan.as_dict()),
package_bytes=_compact_size(package.as_dict()),
child_peak_memory_bytes=peak,
)
def _projection_summary(sample: _ProjectionSample) -> dict[str, object]:
return {
"plan_id": sample.plan_id,
"package_id": sample.package_id,
"artifact_sha256": _sha256(sample.artifact),
"artifact_bytes": len(sample.artifact),
"plan_bytes": sample.plan_bytes,
"package_bytes": sample.package_bytes,
}
def _measure(
operation: Callable[[], object],
*,
samples: int,
p95_limit_ms: float,
response_limit_bytes: int,
summary: Callable[[object], Mapping[str, object]],
response_size: Callable[[object], int],
warmups: int = 0,
) -> tuple[dict[str, object], object]:
for _ in range(warmups):
operation()
durations: list[float] = []
traced_peaks: list[int] = []
response_sizes: list[int] = []
stable_summary: Mapping[str, object] | None = None
last: object = None
for _ in range(samples):
gc.collect()
tracemalloc.start()
started = time.perf_counter_ns()
try:
value = operation()
elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000
_, traced_peak = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()
current_summary = dict(summary(value))
if stable_summary is None:
stable_summary = current_summary
elif current_summary != stable_summary:
raise RuntimeError("Milestone 3 operation changed deterministic result across samples")
current_response_size = response_size(value)
if current_response_size > response_limit_bytes:
raise RuntimeError(
"Milestone 3 response exceeded its fixed benchmark boundary: "
f"{current_response_size} > {response_limit_bytes}"
)
if traced_peak > MAX_TRACED_PEAK_BYTES:
raise RuntimeError(
"Milestone 3 operation exceeded its traced-memory boundary: "
f"{traced_peak} > {MAX_TRACED_PEAK_BYTES}"
)
durations.append(elapsed_ms)
traced_peaks.append(traced_peak)
response_sizes.append(current_response_size)
last = value
ordered = sorted(durations)
p95_index = max(0, math.ceil(len(ordered) * 0.95) - 1)
p95 = ordered[p95_index]
if p95 > p95_limit_ms:
raise RuntimeError(f"Milestone 3 operation p95 {p95:.3f} ms exceeds {p95_limit_ms:.3f} ms")
assert stable_summary is not None
return (
{
"samples": samples,
"median_ms": round(statistics.median(ordered), 3),
"p95_ms": round(p95, 3),
"min_ms": round(ordered[0], 3),
"max_ms": round(ordered[-1], 3),
"p95_limit_ms": p95_limit_ms,
"maximum_response_bytes": max(response_sizes),
"response_limit_bytes": response_limit_bytes,
"maximum_traced_peak_bytes": max(traced_peaks),
"traced_peak_limit_bytes": MAX_TRACED_PEAK_BYTES,
"stable_result": dict(stable_summary),
},
last,
)
def _projection_measurement(
operation: Callable[[], _ProjectionSample],
*,
samples: int,
p95_limit_ms: float,
) -> tuple[dict[str, object], _ProjectionSample]:
child_peaks: list[int] = []
def observed_operation() -> _ProjectionSample:
sample = operation()
if sample.child_peak_memory_bytes is not None:
if sample.child_peak_memory_bytes > MAX_CHILD_PEAK_BYTES:
raise RuntimeError(
"Detached worker peak memory "
f"{sample.child_peak_memory_bytes} exceeds "
f"{MAX_CHILD_PEAK_BYTES} bytes"
)
child_peaks.append(sample.child_peak_memory_bytes)
return sample
measurement, value = _measure(
observed_operation,
samples=samples,
p95_limit_ms=p95_limit_ms,
response_limit_bytes=MAX_RECEIPT_BYTES,
summary=lambda item: _projection_summary(cast(_ProjectionSample, item)),
response_size=lambda item: _compact_size(cast(_ProjectionSample, item).receipt),
)
sample = cast(_ProjectionSample, value)
if sample.plan_bytes > MAX_PLAN_BYTES or sample.package_bytes > MAX_PACKAGE_BYTES:
raise RuntimeError("Projection plan or package exceeded its protocol boundary")
if child_peaks:
measurement["maximum_child_peak_bytes"] = max(child_peaks)
measurement["child_peak_limit_bytes"] = MAX_CHILD_PEAK_BYTES
return measurement, sample
def _profiled_status(operation: Callable[[], Mapping[str, object]]) -> _StatusSample:
with request("benchmark.m3", enabled=True) as collector:
response = operation()
if collector is None:
raise RuntimeError("Milestone 3 telemetry collector was not created")
counters = collector.as_dict(outcome="ok")["counters"]
typed_counters = cast(Mapping[str, object], counters)
for counter in ZERO_WORK_COUNTERS:
if typed_counters[counter] != 0:
raise RuntimeError(f"Receipt-only status performed forbidden work: {counter}")
if typed_counters["source_generation_checks"] != 2:
raise RuntimeError("Receipt-only status did not perform its two race-safe source checks")
if response.get("status") != "ok" or response.get("state") != "current":
raise RuntimeError("Receipt-only status did not report a current publication")
return _StatusSample(response=response, counters=typed_counters)
def _status_summary(value: object) -> Mapping[str, object]:
sample = cast(_StatusSample, value)
return {
"response": dict(sample.response),
"counters": dict(sample.counters),
}
def _fragment_summary(value: object) -> Mapping[str, object]:
sample = cast(_FragmentSweep, value)
return {
"fragment_count": sample.fragment_count,
"aggregate_content_bytes": sample.aggregate_content_bytes,
"ordered_record_hash": sample.ordered_record_hash,
}
def _fragment_sweep(records: Sequence[FragmentRecord]) -> _FragmentSweep:
return _FragmentSweep(
fragment_count=len(records),
aggregate_content_bytes=sum(record.byte_count for record in records),
ordered_record_hash=projection_hash([record.record_id for record in records]),
)
def _prepared_summary(value: object) -> Mapping[str, object]:
prepared = cast(PreparedRender, value)
return {
"render_identity": prepared.render_identity,
"output_sha256": prepared.output_hash,
"output_bytes": len(prepared.output),
}
def _prepared_response_size(value: object) -> int:
prepared = cast(PreparedRender, value)
return _compact_size(prepared.projection_receipt)
def _benchmark(root: Path, node_count: int, samples: int) -> dict[str, object]:
project = Project.open(root)
snapshot = project.load()
if len(snapshot.nodes) != node_count or len(snapshot.edges) != node_count - 1:
raise RuntimeError("Milestone 3 fixture does not have full synthetic coverage")
operations: dict[str, object] = {}
def manual_full() -> _ProjectionSample:
plan, package, renderer_version = _manual_package(snapshot)
result = ManualHtmlRenderer(renderer_version).render(package)
return _projection_sample(plan, package, result)
operations["manual_full_render"], manual_result = _projection_measurement(
manual_full,
samples=samples,
p95_limit_ms=15_000,
)
def graph_full() -> _ProjectionSample:
plan, package = _graph_package(snapshot, node_count)
result = PortableGraphHtmlRenderer().render(package)
return _projection_sample(plan, package, result)
operations["portable_graph_full_render"], graph_result = _projection_measurement(
graph_full,
samples=samples,
p95_limit_ms=10_000,
)
manual_plan, manual_package, manual_renderer_version = _manual_package(snapshot)
graph_plan, graph_package = _graph_package(snapshot, node_count)
graph_diagnostics = cast(Mapping[str, object], graph_plan.document["diagnostics"])
manual_pages = cast(list[dict[str, object]], manual_plan.document["pages"])
if (
len(manual_pages) != node_count
or graph_diagnostics["returned_nodes"] != node_count
or graph_diagnostics["returned_edges"] != node_count - 1
):
raise RuntimeError("Projection plans did not retain every synthetic node and edge")
manual_worker_measurement, manual_worker = _projection_measurement(
lambda: _projection_sample(
manual_plan,
manual_package,
render_projection_in_worker(manual_package),
),
samples=samples,
p95_limit_ms=20_000,
)
operations["manual_detached_worker"] = manual_worker_measurement
manual_worker_peak = cast(
int,
manual_worker_measurement["maximum_child_peak_bytes"],
)
graph_worker_measurement, graph_worker = _projection_measurement(
lambda: _projection_sample(
graph_plan,
graph_package,
render_projection_in_worker(graph_package),
),
samples=samples,
p95_limit_ms=20_000,
)
operations["portable_graph_detached_worker"] = graph_worker_measurement
graph_worker_peak = cast(
int,
graph_worker_measurement["maximum_child_peak_bytes"],
)
if (
manual_worker.artifact != manual_result.artifact
or graph_worker.artifact != graph_result.artifact
):
raise RuntimeError("Detached worker output is not byte-equivalent to in-process output")
manual_renderer = ManualHtmlRenderer(manual_renderer_version)
fragment_records = [
FragmentRecord.create(
FragmentKey.create(
projection_kind="manual",
renderer_id=ManualHtmlRenderer.renderer_id,
renderer_version=manual_renderer_version,
component_version=GenericHtmlRenderer.page_component_version,
semantic_input_hash=fragment_semantic_hash(page),
),
manual_renderer.render_page_fragment(page).encode("utf-8"),
)
for page in manual_pages
]
fragment_cache = ProjectionFragmentCache(
root,
root / ".docforge" / "milestone3-fragment-cache",
)
def fragment_misses() -> _FragmentSweep:
if any(fragment_cache.get(record.key) is not None for record in fragment_records):
raise RuntimeError("Cold fragment-cache lookup unexpectedly hit")
return _fragment_sweep(fragment_records)
operations["fragment_cache_miss_sweep"], _ = _measure(
fragment_misses,
samples=samples,
p95_limit_ms=5_000,
response_limit_bytes=32_768,
summary=_fragment_summary,
response_size=lambda item: _compact_size(_fragment_summary(item)),
)
def fragment_puts() -> _FragmentSweep:
published: list[FragmentRecord] = []
for record in fragment_records:
stored = fragment_cache.put(record.key, record.content)
if stored != record:
raise RuntimeError("Fragment cache did not publish an exact record")
assert stored is not None
published.append(stored)
return _fragment_sweep(published)
operations["fragment_cache_put_sweep"], _ = _measure(
fragment_puts,
samples=1,
p95_limit_ms=10_000,
response_limit_bytes=32_768,
summary=_fragment_summary,
response_size=lambda item: _compact_size(_fragment_summary(item)),
)
def fragment_hits() -> _FragmentSweep:
loaded: list[FragmentRecord] = []
for expected in fragment_records:
record = fragment_cache.get(expected.key)
if record != expected:
raise RuntimeError("Fragment cache hit was not byte-exact")
assert record is not None
loaded.append(record)
return _fragment_sweep(loaded)
operations["fragment_cache_hit_sweep"], _ = _measure(
fragment_hits,
samples=samples,
p95_limit_ms=5_000,
response_limit_bytes=32_768,
summary=_fragment_summary,
response_size=lambda item: _compact_size(_fragment_summary(item)),
)
render_config = snapshot.descriptor.render
assert render_config is not None
incremental_package = build_manual_projection_package(
manual_plan,
render_config.views[0].template_path.read_bytes(),
renderer_id=ManualHtmlRenderer.renderer_id,
renderer_version=manual_renderer_version,
max_output_bytes=snapshot.descriptor.limits.max_render_bytes,
fragment_records=[record.as_dict() for record in fragment_records],
)
def fragment_equivalence() -> _ProjectionSample:
sample = _projection_sample(
manual_plan,
incremental_package,
ManualHtmlRenderer(manual_renderer_version).render(incremental_package),
)
if sample.artifact != manual_result.artifact:
raise RuntimeError("Fragment-assisted manual output is not byte-equivalent")
return sample
operations["fragment_assisted_equivalence"], fragment_result = _projection_measurement(
fragment_equivalence,
samples=samples,
p95_limit_ms=15_000,
)
render_config = snapshot.descriptor.render
assert render_config is not None
manual_view = render_config.views[0]
template_bytes = manual_view.template_path.read_bytes()
production_renderer = GenericHtmlRenderer()
operations["manual_incremental_cold"], cold_prepared = _measure(
lambda: production_renderer.prepare(
snapshot,
manual_view,
template_bytes,
changeset_hash=None,
),
samples=1,
p95_limit_ms=20_000,
response_limit_bytes=MAX_RECEIPT_BYTES,
summary=_prepared_summary,
response_size=_prepared_response_size,
)
operations["manual_incremental_warm"], warm_prepared = _measure(
lambda: production_renderer.prepare(
snapshot,
manual_view,
template_bytes,
changeset_hash=None,
),
samples=samples,
p95_limit_ms=20_000,
response_limit_bytes=MAX_RECEIPT_BYTES,
summary=_prepared_summary,
response_size=_prepared_response_size,
)
operations["manual_forced_full"], forced_prepared = _measure(
lambda: GenericHtmlRenderer(incremental=False).prepare(
snapshot,
manual_view,
template_bytes,
changeset_hash=None,
),
samples=samples,
p95_limit_ms=20_000,
response_limit_bytes=MAX_RECEIPT_BYTES,
summary=_prepared_summary,
response_size=_prepared_response_size,
)
production_values = tuple(
cast(PreparedRender, value) for value in (cold_prepared, warm_prepared, forced_prepared)
)
if len({value.output for value in production_values}) != 1:
raise RuntimeError("Production cold, warm, and forced-full manual output differs")
changed_node = replace(
snapshot.nodes[0],
content=snapshot.nodes[0].content + "\nChanged projection content.\n",
content_hash=_sha256(
(snapshot.nodes[0].content + "\nChanged projection content.\n").encode()
),
)
added_node = replace(
snapshot.nodes[-1],
node_id="guide.synthetic-added",
title="Synthetic added",
source_path="docs/content/synthetic-added.md",
content_hash=_sha256(b"synthetic-added"),
)
variants = {
"change": replace(
snapshot,
nodes=(changed_node, *snapshot.nodes[1:]),
source_hash=_sha256(b"manual-change-variant"),
),
"add": replace(
snapshot,
nodes=(*snapshot.nodes, added_node),
source_hash=_sha256(b"manual-add-variant"),
),
"delete": replace(
snapshot,
nodes=snapshot.nodes[:-1],
edges=tuple(
edge
for edge in snapshot.edges
if edge.source_id != snapshot.nodes[-1].node_id
and edge.target_id != snapshot.nodes[-1].node_id
),
source_hash=_sha256(b"manual-delete-variant"),
),
"reorder": replace(
snapshot,
nodes=tuple(reversed(snapshot.nodes)),
source_hash=_sha256(b"manual-reorder-variant"),
),
}
variant_equivalence: dict[str, bool] = {}
for name, variant in variants.items():
incremental = GenericHtmlRenderer().prepare(
variant,
manual_view,
template_bytes,
changeset_hash=None,
)
full = GenericHtmlRenderer(incremental=False).prepare(
variant,
manual_view,
template_bytes,
changeset_hash=None,
)
variant_equivalence[name] = incremental.output == full.output
if not all(variant_equivalence.values()):
raise RuntimeError("Production incremental mutation output differs from forced full")
manual_service = RenderService(project)
graph_service = GraphRenderService(project)
manual_service.render("manual")
graph_service.render("architecture")
operations["manual_status_no_work"], manual_status = _measure(
lambda: _profiled_status(lambda: manual_service.status("manual")),
samples=samples,
p95_limit_ms=500,
response_limit_bytes=MAX_STATUS_RESPONSE_BYTES,
summary=_status_summary,
response_size=lambda item: _compact_size(cast(_StatusSample, item).response),
)
operations["portable_graph_status_no_work"], graph_status = _measure(
lambda: _profiled_status(lambda: graph_service.status("architecture")),
samples=samples,
p95_limit_ms=500,
response_limit_bytes=MAX_STATUS_RESPONSE_BYTES,
summary=_status_summary,
response_size=lambda item: _compact_size(cast(_StatusSample, item).response),
)
return {
"fixture": {
"kind": "synthetic_generic_projection",
"node_count": node_count,
"edge_count": node_count - 1,
"manual_page_count": len(manual_pages),
"portable_graph_node_count": graph_diagnostics["returned_nodes"],
"portable_graph_edge_count": graph_diagnostics["returned_edges"],
"full_coverage": (
len(manual_pages) == node_count
and graph_diagnostics["returned_nodes"] == node_count
and graph_diagnostics["returned_edges"] == node_count - 1
),
},
"operations": operations,
"equivalence": {
"manual_in_process_vs_detached": manual_worker.artifact == manual_result.artifact,
"portable_graph_in_process_vs_detached": graph_worker.artifact == graph_result.artifact,
"manual_full_vs_fragment_assisted": fragment_result.artifact == manual_result.artifact,
"manual_production_cold_warm_full": len({value.output for value in production_values})
== 1,
"manual_production_variants": variant_equivalence,
},
"sizes": {
"manual": {
**_projection_summary(manual_result),
"receipt_bytes": _compact_size(manual_result.receipt),
},
"portable_graph": {
**_projection_summary(graph_result),
"receipt_bytes": _compact_size(graph_result.receipt),
},
"fragment_assisted_manual": {
**_projection_summary(fragment_result),
"receipt_bytes": _compact_size(fragment_result.receipt),
"fragment_count": len(fragment_records),
"aggregate_fragment_content_bytes": sum(
record.byte_count for record in fragment_records
),
},
"manual_status_response_bytes": _compact_size(
cast(_StatusSample, manual_status).response
),
"portable_graph_status_response_bytes": _compact_size(
cast(_StatusSample, graph_status).response
),
},
"memory": {
"process_peak_rss_kib": int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss),
"manual_worker_peak_bytes": manual_worker_peak,
"portable_graph_worker_peak_bytes": graph_worker_peak,
},
}
def main() -> int:
arguments = _parser().parse_args()
default_nodes = FULL_NODE_COUNT if arguments.mode == "full" else SMOKE_NODE_COUNT
default_samples = DEFAULT_FULL_SAMPLES if arguments.mode == "full" else DEFAULT_SMOKE_SAMPLES
node_count = default_nodes if arguments.nodes is None else arguments.nodes
samples = default_samples if arguments.samples is None else arguments.samples
if not 2 <= node_count <= FULL_NODE_COUNT:
raise SystemExit("--nodes must be between 2 and 1000")
if arguments.mode == "full" and node_count != FULL_NODE_COUNT:
raise SystemExit("--mode full requires exactly 1000 nodes")
if samples < 1:
raise SystemExit("--samples must be positive")
with tempfile.TemporaryDirectory(prefix="docforge-milestone3-") as directory:
benchmark_root = Path(directory).resolve()
_prepare_fixture(benchmark_root, node_count)
measurement = _benchmark(benchmark_root, node_count, samples)
status = _git(["status", "--porcelain"])
report: dict[str, object] = {
"schema_version": 1,
"benchmark": "docforge2_milestone3",
"mode": arguments.mode,
"source": {
"revision": _git(["rev-parse", "HEAD"]),
"dirty": bool(status),
},
"environment": {
"platform": platform.platform(),
"machine": platform.machine(),
"python": platform.python_version(),
"implementation": platform.python_implementation(),
},
"method": {
"clock": "time.perf_counter_ns",
"in_process_peak_memory": "tracemalloc per measured invocation",
"detached_peak_memory": "worker receipt resource peak RSS",
"process_peak_memory": "resource.getrusage(RUSAGE_SELF).ru_maxrss",
"response_size": "UTF-8 bytes of canonical compact sorted JSON",
"samples": samples,
"full_mode_node_requirement": FULL_NODE_COUNT,
"determinism": (
"stable semantic summaries must match across samples; report JSON uses sorted keys"
),
"maximum_worker_artifact_bytes": MAX_WORKER_ARTIFACT_BYTES,
},
**measurement,
}
encoded = encode_report(report)
if arguments.output is not None:
output = arguments.output.resolve()
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(encoded, encoding="utf-8")
sys.stdout.write(encoded)
return 0
if __name__ == "__main__":
raise SystemExit(main())