1223 lines
46 KiB
Python
1223 lines
46 KiB
Python
"""Project discovery, root confinement, canonical loading, and graph validation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
import tomllib
|
|
from collections import Counter
|
|
from collections.abc import Mapping
|
|
from contextlib import suppress
|
|
from dataclasses import dataclass, replace
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any, cast
|
|
|
|
from .config_validation import (
|
|
AUTHORITIES,
|
|
ID_PATTERN,
|
|
confined_path,
|
|
positive_int,
|
|
require_string,
|
|
string_list,
|
|
)
|
|
from .errors import DocForgeError
|
|
from .graph_render_config import load_graph_render_config
|
|
from .models import (
|
|
ContextProfile,
|
|
Edge,
|
|
Limits,
|
|
Node,
|
|
ProjectDescriptor,
|
|
ProjectSnapshot,
|
|
ProjectState,
|
|
ProposalWriter,
|
|
)
|
|
from .render_config import load_render_config
|
|
from .telemetry import increment, stage
|
|
|
|
SOURCE_GENERATION_SCHEMA_VERSION = 1
|
|
GENERIC_SOURCE_CONTRACT = "docforge-core:0.7.1:index:1"
|
|
MAX_PROJECT_DESCRIPTOR_BYTES = 1_000_000
|
|
|
|
_CORE_METADATA = frozenset(
|
|
{
|
|
"schema_version",
|
|
"id",
|
|
"title",
|
|
"family",
|
|
"authority",
|
|
"status",
|
|
"tags",
|
|
"summary",
|
|
"source_anchor",
|
|
"content",
|
|
}
|
|
)
|
|
_DESCRIPTOR_KEYS = frozenset(
|
|
{
|
|
"schema_version",
|
|
"project_id",
|
|
"title",
|
|
"adapter",
|
|
"sources",
|
|
"derived",
|
|
"changesets",
|
|
"render",
|
|
"graph_render",
|
|
"graph",
|
|
"limits",
|
|
"profiles",
|
|
}
|
|
)
|
|
_SOURCE_KEYS = frozenset({"content_roots", "authority_files"})
|
|
_DERIVED_KEYS = frozenset({"cache_root", "index"})
|
|
_CHANGESET_KEYS = frozenset({"root", "writers"})
|
|
_WRITER_KEYS = frozenset({"id", "families", "operations"})
|
|
_GRAPH_KEYS = frozenset({"allowed_relations"})
|
|
_PROFILE_KEYS = frozenset(
|
|
{"id", "families", "statuses", "required_nodes", "token_budget", "dependency_depth"}
|
|
)
|
|
_OPERATIONS = frozenset({"create", "update", "move", "delete"})
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _CapturedGeneration:
|
|
source_hash: str
|
|
revision: str
|
|
files: tuple[tuple[str, int, int, int, int, int, int], ...]
|
|
directories: tuple[tuple[str, int, int, int, int, int], ...]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _ParsedGenerationReceipt:
|
|
signature: tuple[int, int, int, int, int]
|
|
source_hash: str
|
|
revision: str
|
|
files: tuple[tuple[object, ...], ...]
|
|
directories: tuple[tuple[object, ...], ...]
|
|
file_paths: tuple[Path, ...]
|
|
directory_paths: tuple[Path, ...]
|
|
|
|
|
|
def project_root_fingerprint(root: Path) -> str:
|
|
return hashlib.sha256(str(root).encode()).hexdigest()[:16]
|
|
|
|
|
|
def _file_generation(
|
|
root: Path,
|
|
paths: tuple[Path, ...],
|
|
) -> tuple[tuple[str, int, int, int, int, int, int], ...]:
|
|
"""Capture cheap identities that change on ordinary source or metadata mutation."""
|
|
|
|
identities: list[tuple[str, int, int, int, int, int, int]] = []
|
|
for path in paths:
|
|
try:
|
|
status = path.lstat()
|
|
except OSError as error:
|
|
raise DocForgeError(
|
|
"source_changed",
|
|
"Canonical source disappeared during generation capture",
|
|
source=path.relative_to(root).as_posix(),
|
|
) from error
|
|
if not stat.S_ISREG(status.st_mode):
|
|
raise DocForgeError(
|
|
"source_changed",
|
|
"Canonical generation inputs must remain regular files",
|
|
source=path.relative_to(root).as_posix(),
|
|
)
|
|
identities.append(
|
|
(
|
|
path.relative_to(root).as_posix(),
|
|
status.st_dev,
|
|
status.st_ino,
|
|
status.st_mode,
|
|
status.st_size,
|
|
status.st_mtime_ns,
|
|
status.st_ctime_ns,
|
|
)
|
|
)
|
|
return tuple(identities)
|
|
|
|
|
|
def _directory_generation(
|
|
root: Path,
|
|
paths: tuple[Path, ...],
|
|
) -> tuple[tuple[str, int, int, int, int, int], ...]:
|
|
"""Capture directory identities so source membership changes invalidate a receipt."""
|
|
|
|
identities: list[tuple[str, int, int, int, int, int]] = []
|
|
for path in paths:
|
|
try:
|
|
status = path.lstat()
|
|
except OSError as error:
|
|
raise DocForgeError(
|
|
"source_changed",
|
|
"Canonical source directory disappeared during generation capture",
|
|
source=path.relative_to(root).as_posix(),
|
|
) from error
|
|
if not stat.S_ISDIR(status.st_mode):
|
|
raise DocForgeError(
|
|
"source_changed",
|
|
"Canonical source directories must remain directories",
|
|
source=path.relative_to(root).as_posix(),
|
|
)
|
|
identities.append(
|
|
(
|
|
path.relative_to(root).as_posix(),
|
|
status.st_dev,
|
|
status.st_ino,
|
|
status.st_mode,
|
|
status.st_mtime_ns,
|
|
status.st_ctime_ns,
|
|
)
|
|
)
|
|
return tuple(identities)
|
|
|
|
|
|
def _receipt_paths(root: Path, value: object, *, width: int) -> tuple[Path, ...] | None:
|
|
if not isinstance(value, list):
|
|
return None
|
|
paths: list[Path] = []
|
|
for raw_item in cast(list[object], value):
|
|
if not isinstance(raw_item, list):
|
|
return None
|
|
item = cast(list[object], raw_item)
|
|
if len(item) != width or not isinstance(item[0], str):
|
|
return None
|
|
relative = PurePosixPath(item[0])
|
|
if relative.is_absolute() or not relative.parts or ".." in relative.parts:
|
|
return None
|
|
path = root.joinpath(*relative.parts)
|
|
if not path.is_relative_to(root):
|
|
return None
|
|
paths.append(path)
|
|
return tuple(paths)
|
|
|
|
|
|
def _receipt_signature(path: Path) -> tuple[int, int, int, int, int] | None:
|
|
try:
|
|
status = path.lstat()
|
|
except OSError:
|
|
return None
|
|
if not stat.S_ISREG(status.st_mode):
|
|
return None
|
|
return (
|
|
status.st_dev,
|
|
status.st_ino,
|
|
status.st_size,
|
|
status.st_mtime_ns,
|
|
status.st_ctime_ns,
|
|
)
|
|
|
|
|
|
def _read_descriptor(descriptor_path: Path) -> bytes:
|
|
try:
|
|
parent = descriptor_path.parent
|
|
parent_status = parent.lstat()
|
|
if (
|
|
stat.S_ISLNK(parent_status.st_mode)
|
|
or not stat.S_ISDIR(parent_status.st_mode)
|
|
or parent.resolve(strict=True) != parent
|
|
):
|
|
raise DocForgeError(
|
|
"project_descriptor_unsafe",
|
|
"Project descriptor parent must be one real confined directory",
|
|
)
|
|
directory_fd = os.open(
|
|
parent,
|
|
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
|
)
|
|
except FileNotFoundError as error:
|
|
raise DocForgeError("missing_config", "Missing .docforge/project.toml") from error
|
|
except DocForgeError:
|
|
raise
|
|
except OSError as error:
|
|
raise DocForgeError(
|
|
"project_descriptor_unsafe",
|
|
"Project descriptor cannot be inspected safely",
|
|
) from error
|
|
opened_parent = os.fstat(directory_fd)
|
|
if opened_parent.st_dev != parent_status.st_dev or opened_parent.st_ino != parent_status.st_ino:
|
|
with suppress(OSError):
|
|
os.close(directory_fd)
|
|
raise DocForgeError(
|
|
"project_descriptor_changed",
|
|
"Project descriptor parent changed while it was opened",
|
|
)
|
|
try:
|
|
|
|
def parent_current() -> bool:
|
|
try:
|
|
before = parent.lstat()
|
|
resolved = parent.resolve(strict=True)
|
|
after = parent.lstat()
|
|
opened = os.fstat(directory_fd)
|
|
return (
|
|
not stat.S_ISLNK(before.st_mode)
|
|
and stat.S_ISDIR(before.st_mode)
|
|
and resolved == parent
|
|
and (before.st_dev, before.st_ino, before.st_mode)
|
|
== (after.st_dev, after.st_ino, after.st_mode)
|
|
== (opened.st_dev, opened.st_ino, opened.st_mode)
|
|
)
|
|
except OSError:
|
|
return False
|
|
|
|
if not parent_current():
|
|
raise DocForgeError(
|
|
"project_descriptor_changed",
|
|
"Project descriptor parent changed before it was read",
|
|
)
|
|
try:
|
|
before = os.stat(
|
|
descriptor_path.name,
|
|
dir_fd=directory_fd,
|
|
follow_symlinks=False,
|
|
)
|
|
except FileNotFoundError as error:
|
|
raise DocForgeError("missing_config", "Missing .docforge/project.toml") from error
|
|
except OSError as error:
|
|
raise DocForgeError(
|
|
"project_descriptor_unsafe",
|
|
"Project descriptor cannot be inspected safely",
|
|
) from error
|
|
if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode):
|
|
raise DocForgeError(
|
|
"project_descriptor_unsafe",
|
|
"Project descriptor must be a regular file and not a symbolic link",
|
|
)
|
|
if before.st_size > MAX_PROJECT_DESCRIPTOR_BYTES:
|
|
raise DocForgeError(
|
|
"project_descriptor_oversized",
|
|
"Project descriptor exceeds the bounded configuration limit",
|
|
maximum_bytes=MAX_PROJECT_DESCRIPTOR_BYTES,
|
|
)
|
|
try:
|
|
descriptor = os.open(
|
|
descriptor_path.name,
|
|
os.O_RDONLY | os.O_NOFOLLOW,
|
|
dir_fd=directory_fd,
|
|
)
|
|
except OSError as error:
|
|
raise DocForgeError(
|
|
"project_descriptor_unsafe",
|
|
"Project descriptor cannot be opened safely",
|
|
) from error
|
|
try:
|
|
opened = os.fstat(descriptor)
|
|
if opened.st_dev != before.st_dev or opened.st_ino != before.st_ino:
|
|
raise DocForgeError(
|
|
"project_descriptor_changed",
|
|
"Project descriptor changed while it was opened",
|
|
)
|
|
chunks: list[bytes] = []
|
|
remaining = MAX_PROJECT_DESCRIPTOR_BYTES + 1
|
|
while remaining:
|
|
chunk = os.read(descriptor, min(65_536, remaining))
|
|
if not chunk:
|
|
break
|
|
chunks.append(chunk)
|
|
remaining -= len(chunk)
|
|
raw = b"".join(chunks)
|
|
finally:
|
|
with suppress(OSError):
|
|
os.close(descriptor)
|
|
if len(raw) > MAX_PROJECT_DESCRIPTOR_BYTES:
|
|
raise DocForgeError(
|
|
"project_descriptor_oversized",
|
|
"Project descriptor exceeds the bounded configuration limit",
|
|
maximum_bytes=MAX_PROJECT_DESCRIPTOR_BYTES,
|
|
)
|
|
try:
|
|
after = os.stat(
|
|
descriptor_path.name,
|
|
dir_fd=directory_fd,
|
|
follow_symlinks=False,
|
|
)
|
|
except OSError as error:
|
|
raise DocForgeError(
|
|
"project_descriptor_changed",
|
|
"Project descriptor changed while it was read",
|
|
) from error
|
|
if (
|
|
before.st_dev,
|
|
before.st_ino,
|
|
before.st_size,
|
|
before.st_mtime_ns,
|
|
before.st_ctime_ns,
|
|
) != (
|
|
after.st_dev,
|
|
after.st_ino,
|
|
after.st_size,
|
|
after.st_mtime_ns,
|
|
after.st_ctime_ns,
|
|
):
|
|
raise DocForgeError(
|
|
"project_descriptor_changed",
|
|
"Project descriptor changed while it was read",
|
|
)
|
|
if not parent_current():
|
|
raise DocForgeError(
|
|
"project_descriptor_changed",
|
|
"Project descriptor parent changed while it was read",
|
|
)
|
|
return raw
|
|
finally:
|
|
with suppress(OSError):
|
|
os.close(directory_fd)
|
|
|
|
|
|
def _load_descriptor(root: Path) -> ProjectDescriptor:
|
|
descriptor_path = root / ".docforge" / "project.toml"
|
|
descriptor_bytes = _read_descriptor(descriptor_path)
|
|
try:
|
|
document = cast(dict[str, object], tomllib.loads(descriptor_bytes.decode("utf-8")))
|
|
except UnicodeDecodeError as error:
|
|
raise DocForgeError("invalid_config", "Project descriptor is not UTF-8") from error
|
|
except tomllib.TOMLDecodeError as error:
|
|
raise DocForgeError("invalid_config", f"Invalid project descriptor: {error}") from error
|
|
|
|
unknown_descriptor = sorted(set(document) - _DESCRIPTOR_KEYS)
|
|
if unknown_descriptor:
|
|
raise DocForgeError(
|
|
"invalid_config", "Project descriptor has unknown fields", fields=unknown_descriptor
|
|
)
|
|
|
|
if document.get("schema_version") != 1:
|
|
raise DocForgeError("invalid_config", "Project descriptor schema_version must be 1")
|
|
project_id = require_string(document, "project_id", descriptor_path)
|
|
if ID_PATTERN.fullmatch(project_id) is None:
|
|
raise DocForgeError(
|
|
"invalid_config", "project_id is not a stable ID", project_id=project_id
|
|
)
|
|
title = require_string(document, "title", descriptor_path)
|
|
adapter = require_string(document, "adapter", descriptor_path)
|
|
if adapter != "generic":
|
|
raise DocForgeError(
|
|
"unsupported_adapter", "This project loader supports only the generic adapter"
|
|
)
|
|
|
|
sources = document.get("sources")
|
|
derived = document.get("derived")
|
|
changesets = document.get("changesets")
|
|
graph = document.get("graph")
|
|
if (
|
|
not isinstance(sources, dict)
|
|
or not isinstance(derived, dict)
|
|
or not isinstance(changesets, dict)
|
|
or not isinstance(graph, dict)
|
|
):
|
|
raise DocForgeError(
|
|
"invalid_config", "sources, derived, changesets, and graph tables are required"
|
|
)
|
|
sources = cast(dict[str, object], sources)
|
|
derived = cast(dict[str, object], derived)
|
|
changesets = cast(dict[str, object], changesets)
|
|
graph = cast(dict[str, object], graph)
|
|
for table, allowed, name in (
|
|
(sources, _SOURCE_KEYS, "sources"),
|
|
(derived, _DERIVED_KEYS, "derived"),
|
|
(changesets, _CHANGESET_KEYS, "changesets"),
|
|
(graph, _GRAPH_KEYS, "graph"),
|
|
):
|
|
unknown = sorted(set(table) - allowed)
|
|
if unknown:
|
|
raise DocForgeError("invalid_config", f"{name} has unknown fields", fields=unknown)
|
|
content_roots = tuple(
|
|
confined_path(
|
|
root,
|
|
item,
|
|
field="sources.content_roots",
|
|
must_exist=True,
|
|
expected="directory",
|
|
)
|
|
for item in string_list(
|
|
sources.get("content_roots"), key="sources.content_roots", source=descriptor_path
|
|
)
|
|
)
|
|
if len(content_roots) != len(set(content_roots)):
|
|
raise DocForgeError("invalid_config", "sources.content_roots resolve to duplicates")
|
|
authority_files = tuple(
|
|
confined_path(
|
|
root,
|
|
item,
|
|
field="sources.authority_files",
|
|
must_exist=True,
|
|
expected="file",
|
|
)
|
|
for item in string_list(
|
|
sources.get("authority_files", []),
|
|
key="sources.authority_files",
|
|
source=descriptor_path,
|
|
)
|
|
)
|
|
cache_root = confined_path(
|
|
root, derived.get("cache_root"), field="derived.cache_root", must_exist=False
|
|
)
|
|
index_path = confined_path(root, derived.get("index"), field="derived.index", must_exist=False)
|
|
changeset_root = confined_path(
|
|
root, changesets.get("root"), field="changesets.root", must_exist=False
|
|
)
|
|
if not index_path.is_relative_to(cache_root):
|
|
raise DocForgeError("invalid_config", "derived.index must be inside derived.cache_root")
|
|
for content_root in content_roots:
|
|
if (
|
|
content_root == cache_root
|
|
or content_root.is_relative_to(cache_root)
|
|
or cache_root.is_relative_to(content_root)
|
|
):
|
|
raise DocForgeError("invalid_config", "Canonical content and cache must not overlap")
|
|
if (
|
|
content_root == changeset_root
|
|
or content_root.is_relative_to(changeset_root)
|
|
or changeset_root.is_relative_to(content_root)
|
|
):
|
|
raise DocForgeError(
|
|
"invalid_config", "Canonical content and changesets must not overlap"
|
|
)
|
|
if (
|
|
cache_root == changeset_root
|
|
or cache_root.is_relative_to(changeset_root)
|
|
or changeset_root.is_relative_to(cache_root)
|
|
):
|
|
raise DocForgeError("invalid_config", "Cache and changesets must not overlap")
|
|
|
|
writer_documents = changesets.get("writers")
|
|
if not isinstance(writer_documents, list):
|
|
raise DocForgeError("invalid_config", "changesets.writers must be an array of tables")
|
|
proposal_writers: list[ProposalWriter] = []
|
|
writer_ids: set[str] = set()
|
|
for writer_value in cast(list[object], writer_documents):
|
|
if not isinstance(writer_value, dict):
|
|
raise DocForgeError("invalid_config", "Each changeset writer must be a table")
|
|
writer = cast(dict[str, object], writer_value)
|
|
unknown_writer = sorted(set(writer) - _WRITER_KEYS)
|
|
if unknown_writer:
|
|
raise DocForgeError(
|
|
"invalid_config", "Changeset writer has unknown fields", fields=unknown_writer
|
|
)
|
|
writer_id = require_string(writer, "id", descriptor_path)
|
|
if ID_PATTERN.fullmatch(writer_id) is None or writer_id in writer_ids:
|
|
raise DocForgeError(
|
|
"invalid_config", "Changeset writer ID is invalid or duplicated", id=writer_id
|
|
)
|
|
writer_ids.add(writer_id)
|
|
families = string_list(
|
|
writer.get("families"), key="changesets.writer.families", source=descriptor_path
|
|
)
|
|
operations = string_list(
|
|
writer.get("operations"), key="changesets.writer.operations", source=descriptor_path
|
|
)
|
|
if not families:
|
|
raise DocForgeError("invalid_config", "Changeset writer needs at least one family")
|
|
invalid_operations = sorted(set(operations) - _OPERATIONS)
|
|
if not operations or invalid_operations:
|
|
raise DocForgeError(
|
|
"invalid_config",
|
|
"Changeset writer operations are empty or invalid",
|
|
operations=invalid_operations,
|
|
)
|
|
proposal_writers.append(
|
|
ProposalWriter(
|
|
writer_id=writer_id,
|
|
families=tuple(sorted(families)),
|
|
operations=tuple(sorted(operations)),
|
|
)
|
|
)
|
|
|
|
allowed_relations = string_list(
|
|
graph.get("allowed_relations"), key="graph.allowed_relations", source=descriptor_path
|
|
)
|
|
if not allowed_relations:
|
|
raise DocForgeError("invalid_config", "At least one relationship type is required")
|
|
for relation in allowed_relations:
|
|
if ID_PATTERN.fullmatch(relation) is None:
|
|
raise DocForgeError("invalid_config", "Relationship type is invalid", relation=relation)
|
|
|
|
limit_values = document.get("limits", {})
|
|
if not isinstance(limit_values, dict):
|
|
raise DocForgeError("invalid_config", "limits must be a table")
|
|
limit_values = cast(dict[str, object], limit_values)
|
|
defaults = Limits()
|
|
unknown_limits = sorted(set(limit_values) - set(defaults.__dataclass_fields__))
|
|
if unknown_limits:
|
|
raise DocForgeError("invalid_config", "limits has unknown fields", fields=unknown_limits)
|
|
limits = Limits(
|
|
**{
|
|
field: positive_int(limit_values.get(field, getattr(defaults, field)), field)
|
|
for field in defaults.__dataclass_fields__
|
|
}
|
|
)
|
|
render = load_render_config(
|
|
root,
|
|
document.get("render"),
|
|
descriptor_path=descriptor_path,
|
|
content_roots=content_roots,
|
|
authority_files=authority_files,
|
|
cache_root=cache_root,
|
|
index_path=index_path,
|
|
changeset_root=changeset_root,
|
|
limits=limits,
|
|
)
|
|
graph_render = load_graph_render_config(
|
|
root,
|
|
document.get("graph_render"),
|
|
descriptor_path=descriptor_path,
|
|
content_roots=content_roots,
|
|
authority_files=authority_files,
|
|
cache_root=cache_root,
|
|
index_path=index_path,
|
|
changeset_root=changeset_root,
|
|
manual_render=render,
|
|
limits=limits,
|
|
)
|
|
|
|
profile_documents = document.get("profiles", [])
|
|
if not isinstance(profile_documents, list):
|
|
raise DocForgeError("invalid_config", "profiles must be an array of tables")
|
|
profiles: list[ContextProfile] = []
|
|
profile_ids: set[str] = set()
|
|
for profile_value in cast(list[object], profile_documents):
|
|
if not isinstance(profile_value, dict):
|
|
raise DocForgeError("invalid_config", "Each profile must be a table")
|
|
profile = cast(dict[str, object], profile_value)
|
|
unknown_profile = sorted(set(profile) - _PROFILE_KEYS)
|
|
if unknown_profile:
|
|
raise DocForgeError(
|
|
"invalid_config", "Profile has unknown fields", fields=unknown_profile
|
|
)
|
|
profile_id = require_string(profile, "id", descriptor_path)
|
|
if ID_PATTERN.fullmatch(profile_id) is None or profile_id in profile_ids:
|
|
raise DocForgeError(
|
|
"invalid_config", "Profile ID is invalid or duplicated", id=profile_id
|
|
)
|
|
profile_ids.add(profile_id)
|
|
token_budget = positive_int(profile.get("token_budget", 8_000), "profile.token_budget")
|
|
dependency_depth = positive_int(
|
|
profile.get("dependency_depth", 1), "profile.dependency_depth", allow_zero=True
|
|
)
|
|
if token_budget > limits.max_context_tokens:
|
|
raise DocForgeError("invalid_config", "Profile token budget exceeds project limit")
|
|
if dependency_depth > limits.max_traversal_depth:
|
|
raise DocForgeError("invalid_config", "Profile dependency depth exceeds project limit")
|
|
profiles.append(
|
|
ContextProfile(
|
|
profile_id=profile_id,
|
|
families=string_list(
|
|
profile.get("families", []), key="profile.families", source=descriptor_path
|
|
),
|
|
statuses=string_list(
|
|
profile.get("statuses", []), key="profile.statuses", source=descriptor_path
|
|
),
|
|
required_nodes=string_list(
|
|
profile.get("required_nodes", []),
|
|
key="profile.required_nodes",
|
|
source=descriptor_path,
|
|
),
|
|
token_budget=token_budget,
|
|
dependency_depth=dependency_depth,
|
|
)
|
|
)
|
|
|
|
return ProjectDescriptor(
|
|
schema_version=1,
|
|
project_id=project_id,
|
|
title=title,
|
|
adapter=adapter,
|
|
root=root,
|
|
descriptor_path=descriptor_path,
|
|
descriptor_hash=hashlib.sha256(descriptor_bytes).hexdigest(),
|
|
content_roots=content_roots,
|
|
authority_files=authority_files,
|
|
cache_root=cache_root,
|
|
index_path=index_path,
|
|
changeset_root=changeset_root,
|
|
proposal_writers=tuple(sorted(proposal_writers, key=lambda writer: writer.writer_id)),
|
|
render=render,
|
|
graph_render=graph_render,
|
|
allowed_relations=allowed_relations,
|
|
profiles=tuple(profiles),
|
|
limits=limits,
|
|
)
|
|
|
|
|
|
def validate_descriptor_binding(descriptor: ProjectDescriptor) -> None:
|
|
"""Require the bounded descriptor bytes to match one opened project binding."""
|
|
|
|
descriptor_bytes = _read_descriptor(descriptor.descriptor_path)
|
|
if hashlib.sha256(descriptor_bytes).hexdigest() != descriptor.descriptor_hash:
|
|
raise DocForgeError(
|
|
"source_changed",
|
|
"Project descriptor changed after the project was opened",
|
|
)
|
|
|
|
|
|
def _markdown_record(path: Path, text: str) -> tuple[dict[str, Any], str]:
|
|
lines = text.splitlines()
|
|
if not lines or lines[0] != "+++":
|
|
raise DocForgeError(
|
|
"invalid_source", f"{path.name}: Markdown must start with TOML metadata"
|
|
)
|
|
try:
|
|
close = lines.index("+++", 1)
|
|
except ValueError as error:
|
|
raise DocForgeError(
|
|
"invalid_source", f"{path.name}: metadata block is not closed"
|
|
) from error
|
|
try:
|
|
metadata = tomllib.loads("\n".join(lines[1:close]))
|
|
except tomllib.TOMLDecodeError as error:
|
|
raise DocForgeError("invalid_source", f"{path.name}: invalid metadata: {error}") from error
|
|
return metadata, "\n".join(lines[close + 1 :]).strip()
|
|
|
|
|
|
def validated_node_from_record(
|
|
record: dict[str, Any],
|
|
*,
|
|
content: str,
|
|
source: Path,
|
|
relative_source: str,
|
|
relations: tuple[str, ...],
|
|
hash_bytes: bytes,
|
|
) -> tuple[Node, tuple[Edge, ...]]:
|
|
if record.get("schema_version") != 1:
|
|
raise DocForgeError("invalid_source", f"{source.name}: node schema_version must be 1")
|
|
unknown = set(record) - _CORE_METADATA - set(relations)
|
|
if unknown:
|
|
raise DocForgeError(
|
|
"invalid_source", f"{source.name}: unknown metadata", keys=sorted(unknown)
|
|
)
|
|
node_id = require_string(record, "id", source)
|
|
if ID_PATTERN.fullmatch(node_id) is None:
|
|
raise DocForgeError("invalid_source", f"{source.name}: node ID is invalid", id=node_id)
|
|
authority = require_string(record, "authority", source)
|
|
if authority not in AUTHORITIES:
|
|
raise DocForgeError(
|
|
"invalid_source", f"{source.name}: authority is invalid", authority=authority
|
|
)
|
|
tags = string_list(record.get("tags", []), key="tags", source=source)
|
|
anchor = record.get("source_anchor")
|
|
if anchor is not None and (not isinstance(anchor, str) or not anchor):
|
|
raise DocForgeError("invalid_source", f"{source.name}: source_anchor must be a string")
|
|
summary = require_string(record, "summary", source)
|
|
if not content:
|
|
raise DocForgeError("invalid_source", f"{source.name}: node content is empty", id=node_id)
|
|
node = Node(
|
|
node_id=node_id,
|
|
title=require_string(record, "title", source),
|
|
family=require_string(record, "family", source),
|
|
authority=authority,
|
|
status=require_string(record, "status", source),
|
|
tags=tags,
|
|
summary=summary,
|
|
content=content,
|
|
source_path=relative_source,
|
|
source_anchor=anchor,
|
|
content_hash=hashlib.sha256(hash_bytes).hexdigest(),
|
|
)
|
|
edges = tuple(
|
|
Edge(node_id, relation, target)
|
|
for relation in relations
|
|
for target in string_list(record.get(relation, []), key=relation, source=source)
|
|
)
|
|
return node, edges
|
|
|
|
|
|
def _load_source_file(
|
|
descriptor: ProjectDescriptor, path: Path, raw: bytes
|
|
) -> tuple[tuple[Node, ...], tuple[Edge, ...]]:
|
|
if len(raw) > descriptor.limits.max_source_bytes:
|
|
raise DocForgeError(
|
|
"source_too_large", "Canonical source exceeds configured limit", source=path.name
|
|
)
|
|
try:
|
|
text = raw.decode("utf-8")
|
|
except UnicodeDecodeError as error:
|
|
raise DocForgeError("invalid_source", f"{path.name}: source is not UTF-8") from error
|
|
relative = path.relative_to(descriptor.root).as_posix()
|
|
if path.suffix == ".md":
|
|
record, content = _markdown_record(path, text)
|
|
node, node_edges = validated_node_from_record(
|
|
record,
|
|
content=content,
|
|
source=path,
|
|
relative_source=relative,
|
|
relations=descriptor.allowed_relations,
|
|
hash_bytes=raw,
|
|
)
|
|
return (node,), node_edges
|
|
if path.suffix == ".toml":
|
|
try:
|
|
document = cast(dict[str, object], tomllib.loads(text))
|
|
except tomllib.TOMLDecodeError as error:
|
|
raise DocForgeError("invalid_source", f"{path.name}: invalid TOML: {error}") from error
|
|
records = document.get("nodes")
|
|
if set(document) != {"nodes"} or not isinstance(records, list) or not records:
|
|
raise DocForgeError("invalid_source", f"{path.name}: TOML sources require [[nodes]]")
|
|
nodes: list[Node] = []
|
|
edges: list[Edge] = []
|
|
for index, record_value in enumerate(cast(list[object], records)):
|
|
if not isinstance(record_value, dict):
|
|
raise DocForgeError("invalid_source", f"{path.name}: nodes must be tables")
|
|
record = cast(dict[str, Any], record_value)
|
|
content = record.get("content")
|
|
if not isinstance(content, str):
|
|
raise DocForgeError(
|
|
"invalid_source", f"{path.name}: TOML node content must be text"
|
|
)
|
|
canonical = json.dumps(record, sort_keys=True, separators=(",", ":")).encode()
|
|
node, node_edges = validated_node_from_record(
|
|
record,
|
|
content=content.strip(),
|
|
source=path,
|
|
relative_source=relative,
|
|
relations=descriptor.allowed_relations,
|
|
hash_bytes=canonical,
|
|
)
|
|
nodes.append(replace(node, source_anchor=node.source_anchor or f"node-{index + 1}"))
|
|
edges.extend(node_edges)
|
|
return tuple(nodes), tuple(edges)
|
|
raise DocForgeError("invalid_source", "Unsupported canonical source type", source=relative)
|
|
|
|
|
|
def validate_graph(nodes: tuple[Node, ...], edges: tuple[Edge, ...]) -> None:
|
|
node_ids = {node.node_id for node in nodes}
|
|
if len(node_ids) != len(nodes):
|
|
counts = Counter(node.node_id for node in nodes)
|
|
duplicates = sorted(node_id for node_id, count in counts.items() if count > 1)
|
|
raise DocForgeError("duplicate_node", "Stable node IDs must be unique", ids=duplicates)
|
|
|
|
dependencies: dict[str, list[str]] = {node_id: [] for node_id in node_ids}
|
|
edge_keys: set[tuple[str, str, str]] = set()
|
|
missing_sources: set[str] = set()
|
|
missing_targets: set[str] = set()
|
|
for edge in edges:
|
|
key = (edge.source_id, edge.relation, edge.target_id)
|
|
if key in edge_keys:
|
|
raise DocForgeError("duplicate_edge", "Relationships must be unique")
|
|
edge_keys.add(key)
|
|
if edge.source_id not in node_ids:
|
|
missing_sources.add(edge.source_id)
|
|
if edge.target_id not in node_ids:
|
|
missing_targets.add(edge.target_id)
|
|
if edge.relation == "depends_on" and edge.source_id in dependencies:
|
|
dependencies[edge.source_id].append(edge.target_id)
|
|
if missing_sources or missing_targets:
|
|
raise DocForgeError(
|
|
"broken_edge",
|
|
"Relationships reference missing nodes",
|
|
sources=sorted(missing_sources),
|
|
targets=sorted(missing_targets),
|
|
)
|
|
for targets in dependencies.values():
|
|
targets.sort()
|
|
|
|
states: dict[str, int] = {}
|
|
for root in sorted(node_ids):
|
|
if states.get(root) == 2:
|
|
continue
|
|
path: list[str] = []
|
|
stack: list[tuple[str, int]] = [(root, 0)]
|
|
while stack:
|
|
node_id, child_index = stack[-1]
|
|
if states.get(node_id, 0) == 0:
|
|
states[node_id] = 1
|
|
path.append(node_id)
|
|
targets = dependencies[node_id]
|
|
if child_index < len(targets):
|
|
target = targets[child_index]
|
|
stack[-1] = (node_id, child_index + 1)
|
|
state = states.get(target, 0)
|
|
if state == 1:
|
|
raise DocForgeError(
|
|
"dependency_cycle",
|
|
"depends_on relationships contain a cycle",
|
|
path=(*path, target),
|
|
)
|
|
if state == 0:
|
|
stack.append((target, 0))
|
|
continue
|
|
stack.pop()
|
|
path.pop()
|
|
states[node_id] = 2
|
|
|
|
|
|
def validate_source_layout(nodes: tuple[Node, ...]) -> None:
|
|
"""Validate the generic Markdown and TOML source-layout contract."""
|
|
|
|
markdown_sources: dict[str, list[str]] = {}
|
|
anchors: dict[tuple[str, str], list[str]] = {}
|
|
for node in nodes:
|
|
if Path(node.source_path).suffix == ".md":
|
|
markdown_sources.setdefault(node.source_path, []).append(node.node_id)
|
|
continue
|
|
if not node.source_anchor:
|
|
raise DocForgeError(
|
|
"source_anchor_required",
|
|
"TOML nodes require a stable source anchor",
|
|
node_id=node.node_id,
|
|
)
|
|
anchors.setdefault((node.source_path, node.source_anchor), []).append(node.node_id)
|
|
markdown_conflicts = {
|
|
source: sorted(node_ids)
|
|
for source, node_ids in markdown_sources.items()
|
|
if len(node_ids) > 1
|
|
}
|
|
if markdown_conflicts:
|
|
raise DocForgeError(
|
|
"source_conflict",
|
|
"Markdown sources may contain only one node",
|
|
sources=markdown_conflicts,
|
|
)
|
|
anchor_conflicts = [
|
|
{"source": source, "source_anchor": anchor, "nodes": sorted(node_ids)}
|
|
for (source, anchor), node_ids in sorted(anchors.items())
|
|
if len(node_ids) > 1
|
|
]
|
|
if anchor_conflicts:
|
|
raise DocForgeError(
|
|
"source_anchor_conflict",
|
|
"TOML source anchors must be unique within their source",
|
|
conflicts=anchor_conflicts,
|
|
)
|
|
|
|
|
|
def _revision(root: Path) -> str:
|
|
try:
|
|
result = subprocess.run(
|
|
["git", "rev-parse", "HEAD"],
|
|
cwd=root,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=2,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return "unversioned"
|
|
return (
|
|
result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unversioned"
|
|
)
|
|
|
|
|
|
class Project:
|
|
"""One immutable project binding for loading and querying canonical documentation."""
|
|
|
|
def __init__(self, descriptor: ProjectDescriptor) -> None:
|
|
self.descriptor = descriptor
|
|
self._captured_generation: _CapturedGeneration | None = None
|
|
self._generation_receipt_cache: _ParsedGenerationReceipt | None = None
|
|
|
|
@classmethod
|
|
def open(cls, project_root: str | Path) -> Project:
|
|
try:
|
|
root = Path(project_root).expanduser().resolve(strict=True)
|
|
except OSError as error:
|
|
raise DocForgeError("invalid_root", "Project root does not exist") from error
|
|
if not root.is_dir():
|
|
raise DocForgeError("invalid_root", "Project root must be a directory")
|
|
return cls(_load_descriptor(root))
|
|
|
|
def load(self) -> ProjectSnapshot:
|
|
increment("project_loads")
|
|
validate_descriptor_binding(self.descriptor)
|
|
ordered_sources, ordered_directories = self._canonical_inventory()
|
|
generation_paths = (
|
|
self.descriptor.descriptor_path,
|
|
*self.descriptor.authority_files,
|
|
*ordered_sources,
|
|
)
|
|
before_generation = _file_generation(self.descriptor.root, generation_paths)
|
|
before_directories = _directory_generation(
|
|
self.descriptor.root,
|
|
ordered_directories,
|
|
)
|
|
captured = {path: path.read_bytes() for path in generation_paths}
|
|
|
|
nodes: list[Node] = []
|
|
edges: list[Edge] = []
|
|
for path in ordered_sources:
|
|
raw = captured[path]
|
|
increment("source_files_parsed")
|
|
increment("source_bytes_parsed", len(raw))
|
|
with stage("source.parse"):
|
|
source_nodes, source_edges = _load_source_file(
|
|
self.descriptor,
|
|
path,
|
|
raw,
|
|
)
|
|
nodes.extend(source_nodes)
|
|
edges.extend(source_edges)
|
|
if len(nodes) > self.descriptor.limits.max_nodes:
|
|
raise DocForgeError("node_limit", "Project exceeds configured node limit")
|
|
ordered_nodes = tuple(sorted(nodes, key=lambda node: node.node_id))
|
|
ordered_edges = tuple(
|
|
sorted(edges, key=lambda edge: (edge.source_id, edge.relation, edge.target_id))
|
|
)
|
|
validate_graph(ordered_nodes, ordered_edges)
|
|
node_ids = {node.node_id for node in ordered_nodes}
|
|
for profile in self.descriptor.profiles:
|
|
missing = sorted(set(profile.required_nodes) - node_ids)
|
|
if missing:
|
|
raise DocForgeError(
|
|
"invalid_config", "Context profile requires missing nodes", nodes=missing
|
|
)
|
|
|
|
current_sources, current_directories = self._canonical_inventory()
|
|
if current_sources != ordered_sources or current_directories != ordered_directories:
|
|
raise DocForgeError("source_changed", "Canonical source set changed during loading")
|
|
for path, raw in captured.items():
|
|
if not path.is_file() or path.read_bytes() != raw:
|
|
raise DocForgeError(
|
|
"source_changed",
|
|
"Canonical source changed during loading",
|
|
source=path.relative_to(self.descriptor.root).as_posix(),
|
|
)
|
|
after_generation = _file_generation(self.descriptor.root, generation_paths)
|
|
after_directories = _directory_generation(
|
|
self.descriptor.root,
|
|
ordered_directories,
|
|
)
|
|
if after_generation != before_generation or after_directories != before_directories:
|
|
raise DocForgeError(
|
|
"source_changed",
|
|
"Canonical source metadata changed during loading",
|
|
)
|
|
|
|
digest = hashlib.sha256()
|
|
for path in sorted(
|
|
captured, key=lambda item: item.relative_to(self.descriptor.root).as_posix()
|
|
):
|
|
relative = path.relative_to(self.descriptor.root).as_posix()
|
|
digest.update(relative.encode())
|
|
digest.update(b"\0")
|
|
digest.update(hashlib.sha256(captured[path]).digest())
|
|
digest.update(GENERIC_SOURCE_CONTRACT.encode("ascii"))
|
|
source_hash = digest.hexdigest()
|
|
revision = _revision(self.descriptor.root)
|
|
snapshot = ProjectSnapshot(
|
|
descriptor=self.descriptor,
|
|
nodes=ordered_nodes,
|
|
edges=ordered_edges,
|
|
source_hash=source_hash,
|
|
revision=revision,
|
|
)
|
|
self._captured_generation = _CapturedGeneration(
|
|
source_hash=source_hash,
|
|
revision=revision,
|
|
files=after_generation,
|
|
directories=after_directories,
|
|
)
|
|
return snapshot
|
|
|
|
@property
|
|
def generation_path(self) -> Path:
|
|
"""Return the confined disposable receipt for one verified source generation."""
|
|
|
|
return self.descriptor.cache_root / "source-generation.json"
|
|
|
|
def incremental_state(self) -> ProjectState | None:
|
|
"""Return current source identity without reading or parsing canonical source bytes."""
|
|
|
|
increment("source_generation_checks")
|
|
with stage("source.generation"):
|
|
return self._incremental_state()
|
|
|
|
def _incremental_state(self) -> ProjectState | None:
|
|
path = self.generation_path
|
|
signature = _receipt_signature(path)
|
|
if signature is None:
|
|
self._generation_receipt_cache = None
|
|
return None
|
|
receipt = self._generation_receipt_cache
|
|
if receipt is None or receipt.signature != signature:
|
|
receipt = self._parse_generation_receipt(path, signature)
|
|
self._generation_receipt_cache = receipt
|
|
if receipt is None:
|
|
return None
|
|
try:
|
|
current_directories = _directory_generation(
|
|
self.descriptor.root,
|
|
receipt.directory_paths,
|
|
)
|
|
except DocForgeError:
|
|
return None
|
|
if receipt.directories != cast(tuple[tuple[object, ...], ...], current_directories):
|
|
return None
|
|
try:
|
|
current_files = _file_generation(self.descriptor.root, receipt.file_paths)
|
|
except DocForgeError:
|
|
return None
|
|
if receipt.files != cast(tuple[tuple[object, ...], ...], current_files):
|
|
return None
|
|
if _revision(self.descriptor.root) != receipt.revision:
|
|
return None
|
|
return ProjectState(
|
|
source_hash=receipt.source_hash,
|
|
revision=receipt.revision,
|
|
)
|
|
|
|
def _parse_generation_receipt(
|
|
self,
|
|
path: Path,
|
|
signature: tuple[int, int, int, int, int],
|
|
) -> _ParsedGenerationReceipt | None:
|
|
try:
|
|
parsed: object = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
return None
|
|
if _receipt_signature(path) != signature or not isinstance(parsed, dict):
|
|
return None
|
|
payload = cast(dict[str, object], parsed)
|
|
source_hash = payload.get("source_hash")
|
|
revision = payload.get("revision")
|
|
files_value = payload.get("files")
|
|
directories_value = payload.get("directories")
|
|
if (
|
|
payload.get("schema_version") != SOURCE_GENERATION_SCHEMA_VERSION
|
|
or payload.get("source_contract") != GENERIC_SOURCE_CONTRACT
|
|
or payload.get("project_id") != self.descriptor.project_id
|
|
or payload.get("project_root_fingerprint")
|
|
!= project_root_fingerprint(self.descriptor.root)
|
|
or payload.get("adapter") != self.descriptor.adapter
|
|
or not isinstance(source_hash, str)
|
|
or len(source_hash) != 64
|
|
or not isinstance(revision, str)
|
|
or not isinstance(files_value, list)
|
|
or not isinstance(directories_value, list)
|
|
):
|
|
return None
|
|
directory_paths = _receipt_paths(
|
|
self.descriptor.root,
|
|
cast(list[object], directories_value),
|
|
width=6,
|
|
)
|
|
file_paths = _receipt_paths(
|
|
self.descriptor.root,
|
|
cast(list[object], files_value),
|
|
width=7,
|
|
)
|
|
if directory_paths is None or file_paths is None:
|
|
return None
|
|
return _ParsedGenerationReceipt(
|
|
signature=signature,
|
|
source_hash=source_hash,
|
|
revision=revision,
|
|
files=tuple(
|
|
tuple(cast(list[object], item))
|
|
for item in cast(list[object], files_value)
|
|
if isinstance(item, list)
|
|
),
|
|
directories=tuple(
|
|
tuple(cast(list[object], item))
|
|
for item in cast(list[object], directories_value)
|
|
if isinstance(item, list)
|
|
),
|
|
file_paths=file_paths,
|
|
directory_paths=directory_paths,
|
|
)
|
|
|
|
def record_generation(self, snapshot: ProjectSnapshot) -> None:
|
|
"""Persist a generation only after its complete derived index was verified."""
|
|
|
|
captured = self._captured_generation
|
|
if (
|
|
captured is None
|
|
or captured.source_hash != snapshot.source_hash
|
|
or captured.revision != snapshot.revision
|
|
):
|
|
raise DocForgeError(
|
|
"source_changed",
|
|
"Cannot record a source generation without a matching complete load",
|
|
)
|
|
root = self.descriptor.cache_root
|
|
path = self.generation_path
|
|
if path.parent != root or path.is_symlink() or root.resolve(strict=False) != root:
|
|
raise DocForgeError("path_escape", "Source generation receipt path is not safe")
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
if not root.is_dir() or root.resolve(strict=False) != root:
|
|
raise DocForgeError("path_escape", "Source generation receipt directory is not safe")
|
|
payload = {
|
|
"schema_version": SOURCE_GENERATION_SCHEMA_VERSION,
|
|
"source_contract": GENERIC_SOURCE_CONTRACT,
|
|
"project_id": self.descriptor.project_id,
|
|
"project_root_fingerprint": project_root_fingerprint(self.descriptor.root),
|
|
"adapter": self.descriptor.adapter,
|
|
"source_hash": captured.source_hash,
|
|
"revision": captured.revision,
|
|
"files": [list(identity) for identity in captured.files],
|
|
"directories": [list(identity) for identity in captured.directories],
|
|
}
|
|
raw = json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n"
|
|
descriptor, temporary_name = tempfile.mkstemp(prefix=".source-generation-", dir=root)
|
|
temporary = Path(temporary_name)
|
|
try:
|
|
with os.fdopen(descriptor, "wb") as handle:
|
|
handle.write(raw)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, path)
|
|
directory_descriptor = os.open(root, os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory_descriptor)
|
|
finally:
|
|
os.close(directory_descriptor)
|
|
except Exception:
|
|
temporary.unlink(missing_ok=True)
|
|
raise
|
|
|
|
def canonical_source_paths(self) -> tuple[Path, ...]:
|
|
"""Return the deterministic confined canonical source set."""
|
|
|
|
sources, _ = self._canonical_inventory()
|
|
return sources
|
|
|
|
def _canonical_inventory(self) -> tuple[tuple[Path, ...], tuple[Path, ...]]:
|
|
"""Return deterministic canonical files and membership-bearing directories."""
|
|
|
|
source_paths: set[Path] = set()
|
|
directories: set[Path] = set()
|
|
for content_root in self.descriptor.content_roots:
|
|
directories.add(content_root)
|
|
for path in content_root.rglob("*"):
|
|
if path.is_dir():
|
|
resolved_directory = path.resolve()
|
|
if not resolved_directory.is_relative_to(self.descriptor.root):
|
|
raise DocForgeError(
|
|
"path_escape",
|
|
"Canonical source directory resolves outside project root",
|
|
)
|
|
directories.add(resolved_directory)
|
|
continue
|
|
if path.suffix not in {".md", ".toml"} or not path.is_file():
|
|
continue
|
|
resolved = path.resolve()
|
|
if not resolved.is_relative_to(self.descriptor.root):
|
|
raise DocForgeError(
|
|
"path_escape", "Canonical source resolves outside project root"
|
|
)
|
|
source_paths.add(resolved)
|
|
ordered_sources = sorted(
|
|
source_paths, key=lambda path: path.relative_to(self.descriptor.root).as_posix()
|
|
)
|
|
if not ordered_sources:
|
|
raise DocForgeError("empty_project", "No canonical Markdown or TOML sources were found")
|
|
ordered_directories = sorted(
|
|
directories,
|
|
key=lambda path: path.relative_to(self.descriptor.root).as_posix(),
|
|
)
|
|
return tuple(ordered_sources), tuple(ordered_directories)
|
|
|
|
def validate_proposal(
|
|
self,
|
|
base: ProjectSnapshot,
|
|
projected: ProjectSnapshot,
|
|
operations: tuple[Mapping[str, object], ...],
|
|
) -> None:
|
|
"""Generic sources require no policy beyond the core proposal validation."""
|
|
|
|
del base, operations
|
|
validate_source_layout(projected.nodes)
|