"""Fail-closed canonical changeset application and derived-state refresh.""" from __future__ import annotations import json import os import stat import tempfile from collections import defaultdict from collections.abc import Mapping, Sequence from contextlib import suppress from dataclasses import dataclass from pathlib import Path from typing import Literal, Protocol, cast from ._fs_safety import open_bound_directory, rename_exchange_at, require_bound_directory from .changesets import ChangesetStore from .errors import DocForgeError from .index import ProjectIndex from .models import Node, ProjectService, ProjectSnapshot from .projection_policy import ManualProjectionMode, validate_manual_projection_mode from .rendering import RenderService class CanonicalApplier(Protocol): """Project-owned serializer for one already validated proposal projection.""" def apply( self, base: ProjectSnapshot, projected: ProjectSnapshot, operations: tuple[Mapping[str, object], ...], ) -> dict[str, object]: ... @dataclass(frozen=True) class _CanonicalFile: device: int inode: int mode: int owner: int group: int size: int modified_ns: int changed_ns: int content: bytes def unchanged(self, other: _CanonicalFile | None) -> bool: return self == other def renamed_to(self, other: _CanonicalFile | None) -> bool: if other is None: return False return ( self.device, self.inode, self.mode, self.owner, self.group, self.size, self.modified_ns, self.content, ) == ( other.device, other.inode, other.mode, other.owner, other.group, other.size, other.modified_ns, other.content, ) @dataclass class _CanonicalPublication: relative: str target: Path action: Literal["create", "update", "delete"] expected: _CanonicalFile | None staged: Path staged_snapshot: _CanonicalFile backup_snapshot: _CanonicalFile | None = None published_snapshot: _CanonicalFile | None = None committed: bool = False class GenericCanonicalApplier: """Apply generic Markdown/TOML projections inside declared content roots.""" def __init__(self, project: ProjectService) -> None: self.project = project def apply( self, base: ProjectSnapshot, projected: ProjectSnapshot, operations: tuple[Mapping[str, object], ...], ) -> dict[str, object]: del operations changed_sources = self._changed_sources(base, projected) if not changed_sources: raise DocForgeError("empty_changeset", "Changeset produces no canonical changes") base_by_source = self._nodes_by_source(base) projected_by_source = self._nodes_by_source(projected) publications: list[_CanonicalPublication] = [] created_directories: list[Path] = [] targets = {relative: self._target(relative) for relative in sorted(changed_sources)} try: for relative, target in targets.items(): self._prepare_parent(target.parent, created_directories) expected = self._capture(target, base.descriptor.limits.max_source_bytes) existed_in_base = relative in base_by_source if existed_in_base != (expected is not None): raise DocForgeError( "base_conflict", "Canonical target changed before application staging", source=relative, ) nodes = projected_by_source.get(relative, ()) raw = self._serialize_source(projected, relative, nodes) if nodes else b"" if nodes and len(raw) > base.descriptor.limits.max_source_bytes: raise DocForgeError( "source_too_large", "Applied canonical source exceeds the configured limit", source=relative, ) staged, staged_snapshot = self._stage( target, raw, mode=(stat.S_IMODE(expected.mode) if expected is not None else 0o600), ) publications.append( _CanonicalPublication( relative=relative, target=target, action=( "delete" if not nodes else ("update" if expected is not None else "create") ), expected=expected, staged=staged, staged_snapshot=staged_snapshot, ) ) current = self.project.load() if current.source_hash != base.source_hash or current.revision != base.revision: raise DocForgeError( "base_conflict", "Canonical project changed while the changeset was being staged", expected_revision=base.revision, actual_revision=current.revision, expected_source_hash=base.source_hash, actual_source_hash=current.source_hash, ) for target in targets.values(): if target.is_symlink(): raise DocForgeError("path_escape", "Canonical target became a symbolic link") for publication in publications: self._publish(publication) applied = self.project.load() if self._semantic_snapshot(applied) != self._semantic_snapshot(projected): raise DocForgeError( "application_mismatch", "Applied canonical files do not reproduce the validated proposal", ) except Exception as error: recovery = self._rollback(publications) self._discard_unowned_staging(publications) self._remove_empty_directories(created_directories) if recovery: cause = error.code if isinstance(error, DocForgeError) else type(error).__name__ raise DocForgeError( "application_recovery_required", ( "Canonical application raced external writes and could not be " "rolled back without overwriting foreign data" ), cause=cause, conflicts=recovery, remediation=( "Preserve the retained files, compare them with the named canonical " "targets, and resolve the project before retrying a new changeset." ), ) from error if isinstance(error, DocForgeError) and error.code == "application_recovery_required": raise DocForgeError( "base_conflict", "Canonical target raced publication but was restored without data loss", ) from error raise retained = self._discard_backups(publications) return { "applied_sources": sorted(changed_sources), "removed_sources": sorted( source for source in changed_sources if source not in projected_by_source ), "retained_recovery_files": retained, } def _publish(self, publication: _CanonicalPublication) -> None: target = publication.target parent_fd = open_bound_directory(target.parent) try: current = self._capture_at( target.parent, parent_fd, target.name, self.project.descriptor.limits.max_source_bytes, ) if publication.expected is None: if current is not None: raise DocForgeError( "base_conflict", "Canonical create target appeared during application", source=publication.relative, ) elif not publication.expected.unchanged(current): raise DocForgeError( "base_conflict", "Canonical target changed during application", source=publication.relative, ) staged = self._capture_at( target.parent, parent_fd, publication.staged.name, self.project.descriptor.limits.max_source_bytes, ) if staged is None or not publication.staged_snapshot.unchanged(staged): raise DocForgeError( "application_mismatch", "Canonical staging file changed before publication", source=publication.relative, ) if publication.action == "create": self._publish_create(publication, parent_fd) else: self._publish_exchange(publication, parent_fd) finally: os.close(parent_fd) def _publish_create( self, publication: _CanonicalPublication, parent_fd: int, ) -> None: try: os.link( publication.staged.name, publication.target.name, src_dir_fd=parent_fd, dst_dir_fd=parent_fd, follow_symlinks=False, ) except FileExistsError as error: raise DocForgeError( "base_conflict", "Canonical create target appeared during application", source=publication.relative, ) from error except OSError as error: raise DocForgeError( "application_failure", "Canonical create target could not be published", source=publication.relative, ) from error publication.committed = True published = self._capture_at( publication.target.parent, parent_fd, publication.target.name, self.project.descriptor.limits.max_source_bytes, ) if not publication.staged_snapshot.renamed_to(published): raise DocForgeError( "application_mismatch", "Published canonical create target does not match its staging file", source=publication.relative, ) publication.published_snapshot = published os.unlink(publication.staged.name, dir_fd=parent_fd) os.fsync(parent_fd) require_bound_directory(publication.target.parent, parent_fd) def _publish_exchange( self, publication: _CanonicalPublication, parent_fd: int, ) -> None: rename_exchange_at(parent_fd, publication.staged.name, publication.target.name) publication.committed = True displaced = self._capture_at( publication.target.parent, parent_fd, publication.staged.name, self.project.descriptor.limits.max_source_bytes, ) published = self._capture_at( publication.target.parent, parent_fd, publication.target.name, self.project.descriptor.limits.max_source_bytes, ) publication.backup_snapshot = displaced publication.published_snapshot = published if ( publication.expected is None or not publication.expected.renamed_to(displaced) or not publication.staged_snapshot.renamed_to(published) ): if not self._exchange_back(publication, parent_fd): raise DocForgeError( "application_recovery_required", "Canonical target raced atomic publication and displaced data was retained", source=publication.relative, retained=self._relative(publication.staged), ) raise DocForgeError( "base_conflict", "Canonical target changed at the atomic publication boundary", source=publication.relative, ) if publication.action == "delete": os.unlink(publication.target.name, dir_fd=parent_fd) publication.published_snapshot = None os.fsync(parent_fd) require_bound_directory(publication.target.parent, parent_fd) def _exchange_back( self, publication: _CanonicalPublication, parent_fd: int, ) -> bool: backup = publication.backup_snapshot published = publication.published_snapshot if backup is None or published is None: return False try: rename_exchange_at(parent_fd, publication.staged.name, publication.target.name) restored = self._capture_at( publication.target.parent, parent_fd, publication.target.name, self.project.descriptor.limits.max_source_bytes, ) staged = self._capture_at( publication.target.parent, parent_fd, publication.staged.name, self.project.descriptor.limits.max_source_bytes, ) if not backup.renamed_to(restored) or not published.renamed_to(staged): return False publication.backup_snapshot = None publication.published_snapshot = None publication.committed = False os.fsync(parent_fd) return True except (DocForgeError, OSError): return False def _rollback( self, publications: Sequence[_CanonicalPublication], ) -> list[dict[str, object]]: conflicts: list[dict[str, object]] = [] for publication in reversed(publications): if not publication.committed: continue try: if publication.action == "create": conflict = self._rollback_create(publication) elif publication.action == "delete" and publication.published_snapshot is None: conflict = self._rollback_delete(publication) else: conflict = self._rollback_update(publication) except Exception as error: conflict = self._recovery_conflict(publication, "rollback_failed") conflict["error"] = ( error.code if isinstance(error, DocForgeError) else type(error).__name__ ) if conflict is not None: conflicts.append(conflict) return conflicts def _rollback_update( self, publication: _CanonicalPublication, ) -> dict[str, object] | None: parent_fd = open_bound_directory(publication.target.parent) try: current = self._capture_at( publication.target.parent, parent_fd, publication.target.name, self.project.descriptor.limits.max_source_bytes, ) backup = self._capture_at( publication.target.parent, parent_fd, publication.staged.name, self.project.descriptor.limits.max_source_bytes, ) if ( publication.published_snapshot is None or publication.backup_snapshot is None or not publication.published_snapshot.unchanged(current) or not publication.backup_snapshot.unchanged(backup) ): return self._recovery_conflict(publication, "target_or_backup_changed") if not self._exchange_back(publication, parent_fd): return self._recovery_conflict(publication, "atomic_restore_unconfirmed") os.unlink(publication.staged.name, dir_fd=parent_fd) os.fsync(parent_fd) return None finally: os.close(parent_fd) def _rollback_delete( self, publication: _CanonicalPublication, ) -> dict[str, object] | None: parent_fd = open_bound_directory(publication.target.parent) try: current = self._capture_at( publication.target.parent, parent_fd, publication.target.name, self.project.descriptor.limits.max_source_bytes, ) backup = self._capture_at( publication.target.parent, parent_fd, publication.staged.name, self.project.descriptor.limits.max_source_bytes, ) if current is not None: return self._recovery_conflict(publication, "deleted_target_reappeared") if publication.backup_snapshot is None or not publication.backup_snapshot.unchanged( backup ): return self._recovery_conflict(publication, "backup_changed") try: os.link( publication.staged.name, publication.target.name, src_dir_fd=parent_fd, dst_dir_fd=parent_fd, follow_symlinks=False, ) except FileExistsError: return self._recovery_conflict(publication, "deleted_target_reappeared") restored = self._capture_at( publication.target.parent, parent_fd, publication.target.name, self.project.descriptor.limits.max_source_bytes, ) if not publication.backup_snapshot.renamed_to(restored): return self._recovery_conflict(publication, "restore_unconfirmed") os.unlink(publication.staged.name, dir_fd=parent_fd) publication.backup_snapshot = None publication.committed = False os.fsync(parent_fd) return None finally: os.close(parent_fd) def _rollback_create( self, publication: _CanonicalPublication, ) -> dict[str, object] | None: parent_fd = open_bound_directory(publication.target.parent) tombstone: Path | None = None try: current = self._capture_at( publication.target.parent, parent_fd, publication.target.name, self.project.descriptor.limits.max_source_bytes, ) if current is None: publication.committed = False return None if ( publication.published_snapshot is None or not publication.published_snapshot.unchanged(current) ): return self._recovery_conflict(publication, "created_target_changed") tombstone, tombstone_snapshot = self._stage(publication.target, b"", mode=0o600) rename_exchange_at(parent_fd, tombstone.name, publication.target.name) displaced = self._capture_at( publication.target.parent, parent_fd, tombstone.name, self.project.descriptor.limits.max_source_bytes, ) published_tombstone = self._capture_at( publication.target.parent, parent_fd, publication.target.name, self.project.descriptor.limits.max_source_bytes, ) if not publication.published_snapshot.renamed_to( displaced ) or not tombstone_snapshot.renamed_to(published_tombstone): with suppress(DocForgeError): rename_exchange_at(parent_fd, tombstone.name, publication.target.name) return self._recovery_conflict(publication, "create_rollback_raced") os.unlink(publication.target.name, dir_fd=parent_fd) os.unlink(tombstone.name, dir_fd=parent_fd) tombstone = None publication.committed = False publication.published_snapshot = None os.fsync(parent_fd) return None finally: if tombstone is not None: tombstone.unlink(missing_ok=True) os.close(parent_fd) def _discard_backups( self, publications: Sequence[_CanonicalPublication], ) -> list[dict[str, object]]: retained: list[dict[str, object]] = [] for publication in publications: if publication.action != "create" and publication.staged.exists(): parent_fd = open_bound_directory(publication.target.parent) try: backup = self._capture_at( publication.target.parent, parent_fd, publication.staged.name, self.project.descriptor.limits.max_source_bytes, ) if ( publication.backup_snapshot is None or not publication.backup_snapshot.unchanged(backup) ): retained.append( self._recovery_conflict(publication, "backup_cleanup_raced") ) continue os.unlink(publication.staged.name, dir_fd=parent_fd) os.fsync(parent_fd) finally: os.close(parent_fd) publication.committed = False publication.backup_snapshot = None return retained @staticmethod def _discard_unowned_staging(publications: Sequence[_CanonicalPublication]) -> None: for publication in publications: if not publication.committed and publication.staged.exists(): publication.staged.unlink() def _stage(self, target: Path, content: bytes, *, mode: int) -> tuple[Path, _CanonicalFile]: descriptor, temporary_name = tempfile.mkstemp( prefix=".docforge-apply-", dir=target.parent, ) temporary = Path(temporary_name) try: os.fchmod(descriptor, mode) with os.fdopen(descriptor, "wb") as handle: handle.write(content) handle.flush() os.fsync(handle.fileno()) snapshot = self._capture(temporary, max(1, len(content))) if snapshot is None: raise DocForgeError( "application_mismatch", "Canonical staging file disappeared", source=self._relative(target), ) return temporary, snapshot except Exception: with suppress(OSError): os.close(descriptor) temporary.unlink(missing_ok=True) raise def _capture(self, path: Path, maximum: int) -> _CanonicalFile | None: parent_fd = open_bound_directory(path.parent) try: return self._capture_at(path.parent, parent_fd, path.name, maximum) finally: os.close(parent_fd) @staticmethod def _capture_at( parent: Path, parent_fd: int, name: str, maximum: int, ) -> _CanonicalFile | None: del parent try: descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=parent_fd) except FileNotFoundError: return None except OSError as error: raise DocForgeError( "path_escape", "Canonical target cannot be opened safely", target=name, ) from error with os.fdopen(descriptor, "rb") as handle: before = os.fstat(handle.fileno()) if not stat.S_ISREG(before.st_mode) or before.st_size > maximum: raise DocForgeError( "source_too_large", "Canonical target is invalid or exceeds its configured limit", target=name, ) content = handle.read(maximum + 1) after = os.fstat(handle.fileno()) if len(content) > maximum: raise DocForgeError( "source_too_large", "Canonical target exceeds its configured limit", target=name, ) current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) before_identity = ( before.st_dev, before.st_ino, before.st_mode, before.st_uid, before.st_gid, before.st_size, before.st_mtime_ns, before.st_ctime_ns, ) after_identity = ( after.st_dev, after.st_ino, after.st_mode, after.st_uid, after.st_gid, after.st_size, after.st_mtime_ns, after.st_ctime_ns, ) current_identity = ( current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_gid, current.st_size, current.st_mtime_ns, current.st_ctime_ns, ) if before_identity != after_identity or before_identity != current_identity: raise DocForgeError( "base_conflict", "Canonical target changed while it was captured", target=name, ) return _CanonicalFile(*before_identity, content) def _recovery_conflict( self, publication: _CanonicalPublication, reason: str, ) -> dict[str, object]: return { "source": publication.relative, "reason": reason, "target": self._relative(publication.target), "retained": ( self._relative(publication.staged) if publication.staged.exists() else None ), } def _relative(self, path: Path) -> str: return path.relative_to(self.project.descriptor.root).as_posix() def _target(self, relative: str) -> Path: candidate = Path(relative) if candidate.is_absolute() or ".." in candidate.parts: raise DocForgeError("path_escape", "Canonical target path is unsafe", source=relative) root = self.project.descriptor.root target = (root / candidate).resolve(strict=False) if ( not target.is_relative_to(root) or not any( target.is_relative_to(item) for item in self.project.descriptor.content_roots ) or target.suffix not in {".md", ".toml"} ): raise DocForgeError( "path_escape", "Canonical target is outside a declared content root", source=relative, ) if target.exists() and (target.is_symlink() or not target.is_file()): raise DocForgeError("path_escape", "Canonical target is not a regular file") return target def _prepare_parent(self, parent: Path, created: list[Path]) -> None: root = self.project.descriptor.root missing: list[Path] = [] cursor = parent while not cursor.exists(): missing.append(cursor) cursor = cursor.parent if cursor.is_symlink() or cursor.resolve(strict=True) != cursor or not cursor.is_dir(): raise DocForgeError("path_escape", "Canonical target parent is unsafe") if not cursor.is_relative_to(root): raise DocForgeError("path_escape", "Canonical target parent escaped the project root") parent.mkdir(parents=True, exist_ok=True) if parent.resolve(strict=True) != parent: raise DocForgeError("path_escape", "Canonical target parent resolves unexpectedly") created.extend(reversed(missing)) @staticmethod def _fsync_directory(path: Path) -> None: descriptor = os.open(path, os.O_RDONLY) try: os.fsync(descriptor) finally: os.close(descriptor) @staticmethod def _remove_empty_directories(paths: Sequence[Path]) -> None: for path in reversed(paths): with suppress(OSError): path.rmdir() @staticmethod def _nodes_by_source(snapshot: ProjectSnapshot) -> dict[str, tuple[Node, ...]]: grouped: dict[str, list[Node]] = defaultdict(list) for node in snapshot.nodes: grouped[node.source_path].append(node) return { source: tuple(sorted(nodes, key=lambda node: node.node_id)) for source, nodes in grouped.items() } def _changed_sources( self, base: ProjectSnapshot, projected: ProjectSnapshot, ) -> set[str]: base_sources = self._source_signatures(base) projected_sources = self._source_signatures(projected) return { source for source in set(base_sources) | set(projected_sources) if base_sources.get(source) != projected_sources.get(source) } @staticmethod def _source_signatures(snapshot: ProjectSnapshot) -> dict[str, object]: outgoing: dict[str, list[tuple[str, str]]] = defaultdict(list) for edge in snapshot.edges: outgoing[edge.source_id].append((edge.relation, edge.target_id)) signatures: dict[str, list[object]] = defaultdict(list) for node in snapshot.nodes: signatures[node.source_path].append( ( node.node_id, node.title, node.family, node.authority, node.status, node.tags, node.summary, node.content, node.source_anchor, tuple(sorted(outgoing[node.node_id])), ) ) return {source: tuple(items) for source, items in signatures.items()} @classmethod def _semantic_snapshot(cls, snapshot: ProjectSnapshot) -> tuple[object, object]: nodes = tuple( ( node.node_id, node.title, node.family, node.authority, node.status, node.tags, node.summary, node.content, node.source_path, node.source_anchor, ) for node in snapshot.nodes ) edges = tuple((edge.source_id, edge.relation, edge.target_id) for edge in snapshot.edges) return nodes, edges def _serialize_source( self, snapshot: ProjectSnapshot, relative: str, nodes: tuple[Node, ...], ) -> bytes: if Path(relative).suffix == ".md": if len(nodes) != 1: raise DocForgeError( "source_conflict", "Markdown canonical sources may contain only one node", source=relative, ) node = nodes[0] metadata = self._record(snapshot, node, include_content=False) lines = ["+++", *self._toml_record(metadata), "+++", "", node.content.strip(), ""] return "\n".join(lines).encode("utf-8") lines: list[str] = [] for index, node in enumerate(nodes): if index: lines.append("") lines.append("[[nodes]]") lines.extend(self._toml_record(self._record(snapshot, node, include_content=True))) lines.append("") return "\n".join(lines).encode("utf-8") @staticmethod def _record( snapshot: ProjectSnapshot, node: Node, *, include_content: bool, ) -> dict[str, object]: record: dict[str, object] = { "schema_version": 1, "id": node.node_id, "title": node.title, "family": node.family, "authority": node.authority, "status": node.status, "tags": list(node.tags), "summary": node.summary, } if node.source_anchor is not None: record["source_anchor"] = node.source_anchor for relation in snapshot.descriptor.allowed_relations: targets = sorted( edge.target_id for edge in snapshot.edges if edge.source_id == node.node_id and edge.relation == relation ) if targets: record[relation] = targets if include_content: record["content"] = node.content return record @staticmethod def _toml_record(record: dict[str, object]) -> list[str]: lines: list[str] = [] for key, value in record.items(): if isinstance(value, int): encoded = str(value) elif isinstance(value, str): encoded = json.dumps(value, ensure_ascii=False) elif isinstance(value, list): string_items: list[str] = [] for item in cast(list[object], value): if not isinstance(item, str): raise DocForgeError( "application_mismatch", "Generic canonical list values must contain only strings", field=key, ) string_items.append(item) items = ", ".join(json.dumps(item, ensure_ascii=False) for item in string_items) encoded = f"[{items}]" else: raise DocForgeError( "application_mismatch", "Generic canonical serialization encountered an unsupported value", field=key, ) lines.append(f"{key} = {encoded}") return lines class CanonicalApplicationService: """Apply one hash-bound changeset, then refresh all declared derived state.""" def __init__( self, project: ProjectService, *, applier_id: str | None, applier: CanonicalApplier | None, index: ProjectIndex | None = None, manual_policy: ManualProjectionMode = "auto", ) -> None: self.project = project self.applier_id = applier_id self.applier = applier self.changesets = ChangesetStore(project, applier_id) self.index = index or ProjectIndex(project) self.manual_policy = validate_manual_projection_mode(manual_policy) self.rendering = RenderService( project, self.changesets, manual_policy=self.manual_policy, ) @property def enabled(self) -> bool: return self.applier_id is not None and self.applier is not None def access(self) -> dict[str, object]: return { "enabled": self.enabled, "applier": self.applier_id if self.enabled else None, } def apply(self, changeset_id: str, expected_changeset_hash: str) -> dict[str, object]: if not self.enabled or self.applier is None or self.applier_id is None: raise DocForgeError( "canonical_application_disabled", "Server has no configured canonical applier", ) applied = self.changesets.apply( changeset_id=changeset_id, expected_changeset_hash=expected_changeset_hash, applier_id=self.applier_id, application=self.applier.apply, ) refresh_errors: list[dict[str, object]] = [] index_result: dict[str, object] | None = None index_check: dict[str, object] | None = None try: index_result = self.index.build() index_check = self.index.check() except DocForgeError as error: refresh_errors.append( { "component": "index", "error": error.as_dict(), "remediation": { "tool": "docforge_sync", "arguments": {}, }, } ) renders: list[dict[str, object]] = [] config = self.project.descriptor.render render_action = "not_configured" if config is not None and self.manual_policy == "auto": render_action = "rendered" for view in config.views: try: rendered = self.rendering.render(view.view_id) renders.append(rendered) if rendered.get("state") == "degraded": receipt = rendered.get("receipt") refresh_errors.append( { "component": "render_receipt", "view_id": view.view_id, "error": ( cast(Mapping[str, object], receipt).get("error") if isinstance(receipt, dict) else { "code": "render_receipt_failure", "message": ( "Rendered output was published without a " "verification receipt" ), "details": {}, } ), } ) except DocForgeError as error: refresh_errors.append( { "component": "render", "view_id": view.view_id, "error": error.as_dict(), } ) elif config is not None: render_action = ( "skipped_explicit" if self.manual_policy == "explicit" else "skipped_disabled" ) return { **applied, "derived_refresh": { "status": "degraded" if refresh_errors else "ok", "index": index_result, "check": index_check, "renders": renders, "render_policy": { "mode": self.manual_policy, "action": render_action, }, "errors": refresh_errors, }, }