1315 lines
49 KiB
Python
1315 lines
49 KiB
Python
"""Fail-closed canonical changeset application and derived-state refresh."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import secrets
|
|
import stat
|
|
import tempfile
|
|
from collections import defaultdict
|
|
from collections.abc import Mapping, Sequence
|
|
from contextlib import suppress
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Literal, Protocol, cast
|
|
|
|
from ._fs_safety import (
|
|
open_bound_directory,
|
|
rename_exchange_between_at,
|
|
rename_noreplace_between_at,
|
|
require_bound_directory,
|
|
)
|
|
from .changesets import ChangesetStore
|
|
from .errors import DocForgeError
|
|
from .index import ProjectIndex
|
|
from .models import Node, ProjectService, ProjectSnapshot
|
|
from .projection_policy import ManualProjectionMode, validate_manual_projection_mode
|
|
from .rendering import RenderService
|
|
|
|
|
|
class CanonicalApplier(Protocol):
|
|
"""Project-owned serializer for one already validated proposal projection."""
|
|
|
|
def apply(
|
|
self,
|
|
base: ProjectSnapshot,
|
|
projected: ProjectSnapshot,
|
|
operations: tuple[Mapping[str, object], ...],
|
|
) -> dict[str, object]: ...
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _CanonicalFile:
|
|
device: int
|
|
inode: int
|
|
mode: int
|
|
owner: int
|
|
group: int
|
|
size: int
|
|
modified_ns: int
|
|
changed_ns: int
|
|
content: bytes
|
|
|
|
def unchanged(self, other: _CanonicalFile | None) -> bool:
|
|
return self == other
|
|
|
|
def renamed_to(self, other: _CanonicalFile | None) -> bool:
|
|
if other is None:
|
|
return False
|
|
return (
|
|
self.device,
|
|
self.inode,
|
|
self.mode,
|
|
self.owner,
|
|
self.group,
|
|
self.size,
|
|
self.modified_ns,
|
|
self.content,
|
|
) == (
|
|
other.device,
|
|
other.inode,
|
|
other.mode,
|
|
other.owner,
|
|
other.group,
|
|
other.size,
|
|
other.modified_ns,
|
|
other.content,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class _CanonicalPublication:
|
|
relative: str
|
|
target: Path
|
|
action: Literal["create", "update", "delete"]
|
|
expected: _CanonicalFile | None
|
|
staged: Path
|
|
staged_snapshot: _CanonicalFile
|
|
backup_snapshot: _CanonicalFile | None = None
|
|
published_snapshot: _CanonicalFile | None = None
|
|
committed: bool = False
|
|
cleanup_conflicts: list[dict[str, object]] = field(
|
|
default_factory=lambda: list[dict[str, object]]()
|
|
)
|
|
|
|
|
|
class GenericCanonicalApplier:
|
|
"""Apply generic Markdown/TOML projections inside declared content roots."""
|
|
|
|
def __init__(self, project: ProjectService) -> None:
|
|
self.project = project
|
|
|
|
def apply(
|
|
self,
|
|
base: ProjectSnapshot,
|
|
projected: ProjectSnapshot,
|
|
operations: tuple[Mapping[str, object], ...],
|
|
) -> dict[str, object]:
|
|
del operations
|
|
changed_sources = self._changed_sources(base, projected)
|
|
if not changed_sources:
|
|
raise DocForgeError("empty_changeset", "Changeset produces no canonical changes")
|
|
base_by_source = self._nodes_by_source(base)
|
|
projected_by_source = self._nodes_by_source(projected)
|
|
publications: list[_CanonicalPublication] = []
|
|
created_directories: list[Path] = []
|
|
transaction_root: Path | None = None
|
|
targets = {relative: self._target(relative) for relative in sorted(changed_sources)}
|
|
try:
|
|
transaction_root = self._prepare_transaction_root()
|
|
for relative, target in targets.items():
|
|
self._prepare_parent(target.parent, created_directories)
|
|
expected = self._capture(target, base.descriptor.limits.max_source_bytes)
|
|
existed_in_base = relative in base_by_source
|
|
if existed_in_base != (expected is not None):
|
|
raise DocForgeError(
|
|
"base_conflict",
|
|
"Canonical target changed before application staging",
|
|
source=relative,
|
|
)
|
|
nodes = projected_by_source.get(relative, ())
|
|
raw = self._serialize_source(projected, relative, nodes) if nodes else b""
|
|
if nodes and len(raw) > base.descriptor.limits.max_source_bytes:
|
|
raise DocForgeError(
|
|
"source_too_large",
|
|
"Applied canonical source exceeds the configured limit",
|
|
source=relative,
|
|
)
|
|
staged, staged_snapshot = self._stage(
|
|
transaction_root,
|
|
raw,
|
|
mode=(stat.S_IMODE(expected.mode) if expected is not None else 0o600),
|
|
owner=(expected.owner if expected is not None else None),
|
|
group=(expected.group if expected is not None else None),
|
|
source=relative,
|
|
)
|
|
publications.append(
|
|
_CanonicalPublication(
|
|
relative=relative,
|
|
target=target,
|
|
action=(
|
|
"delete"
|
|
if not nodes
|
|
else ("update" if expected is not None else "create")
|
|
),
|
|
expected=expected,
|
|
staged=staged,
|
|
staged_snapshot=staged_snapshot,
|
|
)
|
|
)
|
|
|
|
current = self.project.load()
|
|
if current.source_hash != base.source_hash or current.revision != base.revision:
|
|
raise DocForgeError(
|
|
"base_conflict",
|
|
"Canonical project changed while the changeset was being staged",
|
|
expected_revision=base.revision,
|
|
actual_revision=current.revision,
|
|
expected_source_hash=base.source_hash,
|
|
actual_source_hash=current.source_hash,
|
|
)
|
|
for target in targets.values():
|
|
if target.is_symlink():
|
|
raise DocForgeError("path_escape", "Canonical target became a symbolic link")
|
|
for publication in publications:
|
|
self._publish(publication)
|
|
|
|
applied = self.project.load()
|
|
if self._semantic_snapshot(applied) != self._semantic_snapshot(projected):
|
|
raise DocForgeError(
|
|
"application_mismatch",
|
|
"Applied canonical files do not reproduce the validated proposal",
|
|
)
|
|
except Exception as error:
|
|
recovery = self._rollback(publications)
|
|
recovery.extend(self._discard_unowned_staging(publications))
|
|
if transaction_root is not None:
|
|
recovery.extend(self._finish_transaction(transaction_root))
|
|
self._remove_empty_directories(created_directories)
|
|
if recovery:
|
|
cause = error.code if isinstance(error, DocForgeError) else type(error).__name__
|
|
raise DocForgeError(
|
|
"application_recovery_required",
|
|
(
|
|
"Canonical application raced external writes and could not be "
|
|
"rolled back without overwriting foreign data"
|
|
),
|
|
cause=cause,
|
|
conflicts=recovery,
|
|
remediation=(
|
|
"Preserve the retained files, compare them with the named canonical "
|
|
"targets, and resolve the project before retrying a new changeset."
|
|
),
|
|
) from error
|
|
if isinstance(error, DocForgeError) and error.code == "application_recovery_required":
|
|
raise DocForgeError(
|
|
"base_conflict",
|
|
"Canonical target raced publication but was restored without data loss",
|
|
) from error
|
|
raise
|
|
retained = [
|
|
conflict for publication in publications for conflict in publication.cleanup_conflicts
|
|
]
|
|
retained.extend(self._discard_backups(publications))
|
|
assert transaction_root is not None
|
|
retained.extend(self._finish_transaction(transaction_root))
|
|
recovery_status = "cleanup_required" if retained else "clean"
|
|
return {
|
|
"applied_sources": sorted(changed_sources),
|
|
"removed_sources": sorted(
|
|
source for source in changed_sources if source not in projected_by_source
|
|
),
|
|
"retained_recovery_files": retained,
|
|
"application_recovery": {
|
|
"status": recovery_status,
|
|
"retained": retained,
|
|
"remediation": (
|
|
(
|
|
"Canonical content is committed. Preserve and inspect the retained "
|
|
"transaction files, then remove only confirmed DocForge-owned artifacts."
|
|
)
|
|
if retained
|
|
else None
|
|
),
|
|
},
|
|
}
|
|
|
|
def _publish(self, publication: _CanonicalPublication) -> None:
|
|
target = publication.target
|
|
parent_fd = open_bound_directory(target.parent)
|
|
staging_fd = open_bound_directory(publication.staged.parent)
|
|
try:
|
|
current = self._capture_at(
|
|
target.parent,
|
|
parent_fd,
|
|
target.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if publication.expected is None:
|
|
if current is not None:
|
|
raise DocForgeError(
|
|
"base_conflict",
|
|
"Canonical create target appeared during application",
|
|
source=publication.relative,
|
|
)
|
|
elif not publication.expected.unchanged(current):
|
|
raise DocForgeError(
|
|
"base_conflict",
|
|
"Canonical target changed during application",
|
|
source=publication.relative,
|
|
)
|
|
staged = self._capture_at(
|
|
publication.staged.parent,
|
|
staging_fd,
|
|
publication.staged.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if staged is None or not publication.staged_snapshot.unchanged(staged):
|
|
raise DocForgeError(
|
|
"application_mismatch",
|
|
"Canonical staging file changed before publication",
|
|
source=publication.relative,
|
|
)
|
|
if publication.action == "create":
|
|
self._publish_create(publication, parent_fd, staging_fd)
|
|
else:
|
|
self._publish_exchange(publication, parent_fd, staging_fd)
|
|
finally:
|
|
os.close(staging_fd)
|
|
os.close(parent_fd)
|
|
|
|
def _publish_create(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
parent_fd: int,
|
|
staging_fd: int,
|
|
) -> None:
|
|
try:
|
|
os.link(
|
|
publication.staged.name,
|
|
publication.target.name,
|
|
src_dir_fd=staging_fd,
|
|
dst_dir_fd=parent_fd,
|
|
follow_symlinks=False,
|
|
)
|
|
except FileExistsError as error:
|
|
raise DocForgeError(
|
|
"base_conflict",
|
|
"Canonical create target appeared during application",
|
|
source=publication.relative,
|
|
) from error
|
|
except OSError as error:
|
|
raise DocForgeError(
|
|
"application_failure",
|
|
"Canonical create target could not be published",
|
|
source=publication.relative,
|
|
) from error
|
|
publication.committed = True
|
|
published = self._capture_at(
|
|
publication.target.parent,
|
|
parent_fd,
|
|
publication.target.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if not publication.staged_snapshot.renamed_to(published):
|
|
raise DocForgeError(
|
|
"application_mismatch",
|
|
"Published canonical create target does not match its staging file",
|
|
source=publication.relative,
|
|
)
|
|
assert published is not None
|
|
publication.published_snapshot = published
|
|
private_copy = self._capture_at(
|
|
publication.staged.parent,
|
|
staging_fd,
|
|
publication.staged.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if private_copy is None or not publication.staged_snapshot.renamed_to(private_copy):
|
|
publication.cleanup_conflicts.append(
|
|
self._recovery_conflict(publication, "private_create_link_changed")
|
|
)
|
|
else:
|
|
conflict = self._remove_private(
|
|
publication,
|
|
publication.staged,
|
|
private_copy,
|
|
"private_create_link_cleanup_failed",
|
|
)
|
|
if conflict is not None:
|
|
publication.cleanup_conflicts.append(conflict)
|
|
refreshed = self._capture_at(
|
|
publication.target.parent,
|
|
parent_fd,
|
|
publication.target.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if not published.renamed_to(refreshed):
|
|
raise DocForgeError(
|
|
"application_mismatch",
|
|
"Published canonical create target changed during private cleanup",
|
|
source=publication.relative,
|
|
)
|
|
publication.published_snapshot = refreshed
|
|
os.fsync(parent_fd)
|
|
require_bound_directory(publication.target.parent, parent_fd)
|
|
|
|
def _publish_exchange(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
parent_fd: int,
|
|
staging_fd: int,
|
|
) -> None:
|
|
rename_exchange_between_at(
|
|
staging_fd,
|
|
publication.staged.name,
|
|
parent_fd,
|
|
publication.target.name,
|
|
)
|
|
publication.committed = True
|
|
displaced = self._capture_at(
|
|
publication.staged.parent,
|
|
staging_fd,
|
|
publication.staged.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
published = self._capture_at(
|
|
publication.target.parent,
|
|
parent_fd,
|
|
publication.target.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
publication.backup_snapshot = displaced
|
|
publication.published_snapshot = published
|
|
if (
|
|
publication.expected is None
|
|
or not publication.expected.renamed_to(displaced)
|
|
or not publication.staged_snapshot.renamed_to(published)
|
|
):
|
|
if not self._exchange_back(publication, parent_fd, staging_fd):
|
|
raise DocForgeError(
|
|
"application_recovery_required",
|
|
"Canonical target raced atomic publication and displaced data was retained",
|
|
source=publication.relative,
|
|
retained=self._relative(publication.staged),
|
|
)
|
|
raise DocForgeError(
|
|
"base_conflict",
|
|
"Canonical target changed at the atomic publication boundary",
|
|
source=publication.relative,
|
|
)
|
|
if publication.action == "delete":
|
|
assert published is not None
|
|
conflict = self._detach_canonical(
|
|
publication,
|
|
parent_fd,
|
|
staging_fd,
|
|
published,
|
|
)
|
|
if conflict is not None:
|
|
if publication.published_snapshot is None:
|
|
publication.cleanup_conflicts.append(conflict)
|
|
else:
|
|
raise DocForgeError(
|
|
"application_recovery_required",
|
|
"Canonical delete target raced final detachment and was retained",
|
|
source=publication.relative,
|
|
conflict=conflict,
|
|
)
|
|
os.fsync(parent_fd)
|
|
os.fsync(staging_fd)
|
|
require_bound_directory(publication.target.parent, parent_fd)
|
|
require_bound_directory(publication.staged.parent, staging_fd)
|
|
|
|
def _detach_canonical(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
parent_fd: int,
|
|
staging_fd: int,
|
|
expected: _CanonicalFile,
|
|
) -> dict[str, object] | None:
|
|
detached_name: str | None = None
|
|
for _ in range(8):
|
|
candidate = f".detached-{secrets.token_hex(12)}"
|
|
if rename_noreplace_between_at(
|
|
parent_fd,
|
|
publication.target.name,
|
|
staging_fd,
|
|
candidate,
|
|
):
|
|
detached_name = candidate
|
|
break
|
|
if detached_name is None:
|
|
return self._recovery_conflict(publication, "private_name_collisions")
|
|
detached = publication.staged.parent / detached_name
|
|
moved: _CanonicalFile | None = None
|
|
if self._same_inode_at(staging_fd, detached_name, expected):
|
|
with suppress(DocForgeError, OSError):
|
|
moved = self._capture_at(
|
|
detached.parent,
|
|
staging_fd,
|
|
detached.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if moved is None or not expected.renamed_to(moved):
|
|
restored = rename_noreplace_between_at(
|
|
staging_fd,
|
|
detached_name,
|
|
parent_fd,
|
|
publication.target.name,
|
|
)
|
|
os.fsync(staging_fd)
|
|
os.fsync(parent_fd)
|
|
conflict = self._recovery_conflict(publication, "canonical_detach_raced")
|
|
conflict["raced_data"] = None if restored else self._relative(detached)
|
|
conflict["foreign_target_restored"] = restored
|
|
return conflict
|
|
publication.published_snapshot = None
|
|
conflict = self._remove_private(
|
|
publication,
|
|
detached,
|
|
moved,
|
|
"detached_cleanup_failed",
|
|
)
|
|
os.fsync(parent_fd)
|
|
return conflict
|
|
|
|
def _exchange_back(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
parent_fd: int,
|
|
staging_fd: int,
|
|
) -> bool:
|
|
backup = publication.backup_snapshot
|
|
published = publication.published_snapshot
|
|
if backup is None or published is None:
|
|
return False
|
|
try:
|
|
rename_exchange_between_at(
|
|
staging_fd,
|
|
publication.staged.name,
|
|
parent_fd,
|
|
publication.target.name,
|
|
)
|
|
restored = self._capture_at(
|
|
publication.target.parent,
|
|
parent_fd,
|
|
publication.target.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
staged = self._capture_at(
|
|
publication.staged.parent,
|
|
staging_fd,
|
|
publication.staged.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if not backup.renamed_to(restored) or not published.renamed_to(staged):
|
|
return False
|
|
assert staged is not None
|
|
publication.staged_snapshot = staged
|
|
publication.backup_snapshot = None
|
|
publication.published_snapshot = None
|
|
publication.committed = False
|
|
os.fsync(parent_fd)
|
|
os.fsync(staging_fd)
|
|
return True
|
|
except (DocForgeError, OSError):
|
|
return False
|
|
|
|
@staticmethod
|
|
def _same_inode_at(
|
|
directory_fd: int,
|
|
name: str,
|
|
expected: _CanonicalFile,
|
|
) -> bool:
|
|
try:
|
|
current = os.stat(name, dir_fd=directory_fd, follow_symlinks=False)
|
|
except OSError:
|
|
return False
|
|
return current.st_dev == expected.device and current.st_ino == expected.inode
|
|
|
|
def _rollback(
|
|
self,
|
|
publications: Sequence[_CanonicalPublication],
|
|
) -> list[dict[str, object]]:
|
|
conflicts: list[dict[str, object]] = []
|
|
for publication in reversed(publications):
|
|
if not publication.committed:
|
|
continue
|
|
try:
|
|
if publication.action == "create":
|
|
conflict = self._rollback_create(publication)
|
|
elif publication.action == "delete" and publication.published_snapshot is None:
|
|
conflict = self._rollback_delete(publication)
|
|
else:
|
|
conflict = self._rollback_update(publication)
|
|
except Exception as error:
|
|
conflict = self._recovery_conflict(publication, "rollback_failed")
|
|
conflict["error"] = (
|
|
error.code if isinstance(error, DocForgeError) else type(error).__name__
|
|
)
|
|
if conflict is not None:
|
|
conflicts.append(conflict)
|
|
return conflicts
|
|
|
|
def _rollback_update(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
) -> dict[str, object] | None:
|
|
parent_fd = open_bound_directory(publication.target.parent)
|
|
staging_fd = open_bound_directory(publication.staged.parent)
|
|
try:
|
|
current = self._capture_at(
|
|
publication.target.parent,
|
|
parent_fd,
|
|
publication.target.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
backup = self._capture_at(
|
|
publication.staged.parent,
|
|
staging_fd,
|
|
publication.staged.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if (
|
|
publication.published_snapshot is None
|
|
or publication.backup_snapshot is None
|
|
or not publication.published_snapshot.unchanged(current)
|
|
or not publication.backup_snapshot.unchanged(backup)
|
|
):
|
|
return self._recovery_conflict(publication, "target_or_backup_changed")
|
|
if not self._exchange_back(publication, parent_fd, staging_fd):
|
|
return self._recovery_conflict(publication, "atomic_restore_unconfirmed")
|
|
return self._remove_private(
|
|
publication,
|
|
publication.staged,
|
|
publication.staged_snapshot,
|
|
"rollback_staging_cleanup_failed",
|
|
)
|
|
finally:
|
|
os.close(staging_fd)
|
|
os.close(parent_fd)
|
|
|
|
def _rollback_delete(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
) -> dict[str, object] | None:
|
|
parent_fd = open_bound_directory(publication.target.parent)
|
|
staging_fd = open_bound_directory(publication.staged.parent)
|
|
try:
|
|
current = self._capture_at(
|
|
publication.target.parent,
|
|
parent_fd,
|
|
publication.target.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
backup = self._capture_at(
|
|
publication.staged.parent,
|
|
staging_fd,
|
|
publication.staged.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if current is not None:
|
|
return self._recovery_conflict(publication, "deleted_target_reappeared")
|
|
if publication.backup_snapshot is None or not publication.backup_snapshot.unchanged(
|
|
backup
|
|
):
|
|
return self._recovery_conflict(publication, "backup_changed")
|
|
if not rename_noreplace_between_at(
|
|
staging_fd,
|
|
publication.staged.name,
|
|
parent_fd,
|
|
publication.target.name,
|
|
):
|
|
return self._recovery_conflict(publication, "deleted_target_reappeared")
|
|
restored = self._capture_at(
|
|
publication.target.parent,
|
|
parent_fd,
|
|
publication.target.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if not publication.backup_snapshot.renamed_to(restored):
|
|
return self._recovery_conflict(publication, "restore_unconfirmed")
|
|
publication.backup_snapshot = None
|
|
publication.committed = False
|
|
os.fsync(parent_fd)
|
|
os.fsync(staging_fd)
|
|
return None
|
|
finally:
|
|
os.close(staging_fd)
|
|
os.close(parent_fd)
|
|
|
|
def _rollback_create(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
) -> dict[str, object] | None:
|
|
parent_fd = open_bound_directory(publication.target.parent)
|
|
staging_fd = open_bound_directory(publication.staged.parent)
|
|
try:
|
|
current = self._capture_at(
|
|
publication.target.parent,
|
|
parent_fd,
|
|
publication.target.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if current is None:
|
|
publication.committed = False
|
|
return None
|
|
if (
|
|
publication.published_snapshot is None
|
|
or not publication.published_snapshot.unchanged(current)
|
|
):
|
|
return self._recovery_conflict(publication, "created_target_changed")
|
|
conflict = self._detach_canonical(
|
|
publication,
|
|
parent_fd,
|
|
staging_fd,
|
|
current,
|
|
)
|
|
if conflict is not None:
|
|
return conflict
|
|
publication.committed = False
|
|
publication.published_snapshot = None
|
|
return None
|
|
finally:
|
|
os.close(staging_fd)
|
|
os.close(parent_fd)
|
|
|
|
def _discard_backups(
|
|
self,
|
|
publications: Sequence[_CanonicalPublication],
|
|
) -> list[dict[str, object]]:
|
|
retained: list[dict[str, object]] = []
|
|
for publication in publications:
|
|
if publication.action != "create" and publication.backup_snapshot is not None:
|
|
try:
|
|
staging_fd = open_bound_directory(publication.staged.parent)
|
|
try:
|
|
backup = self._capture_at(
|
|
publication.staged.parent,
|
|
staging_fd,
|
|
publication.staged.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
finally:
|
|
os.close(staging_fd)
|
|
except Exception as error:
|
|
conflict = self._recovery_conflict(
|
|
publication,
|
|
"backup_cleanup_inspection_failed",
|
|
)
|
|
conflict["error"] = (
|
|
error.code if isinstance(error, DocForgeError) else type(error).__name__
|
|
)
|
|
retained.append(conflict)
|
|
continue
|
|
if not publication.backup_snapshot.unchanged(backup):
|
|
retained.append(self._recovery_conflict(publication, "backup_cleanup_raced"))
|
|
continue
|
|
conflict = self._remove_private(
|
|
publication,
|
|
publication.staged,
|
|
publication.backup_snapshot,
|
|
"backup_cleanup_failed",
|
|
)
|
|
if conflict is not None:
|
|
retained.append(conflict)
|
|
continue
|
|
publication.committed = False
|
|
publication.backup_snapshot = None
|
|
return retained
|
|
|
|
def _discard_unowned_staging(
|
|
self,
|
|
publications: Sequence[_CanonicalPublication],
|
|
) -> list[dict[str, object]]:
|
|
retained: list[dict[str, object]] = []
|
|
for publication in publications:
|
|
if publication.committed:
|
|
continue
|
|
conflict = self._remove_private(
|
|
publication,
|
|
publication.staged,
|
|
publication.staged_snapshot,
|
|
"staging_cleanup_failed",
|
|
)
|
|
if conflict is not None:
|
|
retained.append(conflict)
|
|
return retained
|
|
|
|
def _remove_private(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
path: Path,
|
|
expected: _CanonicalFile,
|
|
reason: str,
|
|
) -> dict[str, object] | None:
|
|
try:
|
|
directory_fd = open_bound_directory(path.parent)
|
|
try:
|
|
backup = self._capture_at(
|
|
path.parent,
|
|
directory_fd,
|
|
path.name,
|
|
self.project.descriptor.limits.max_source_bytes,
|
|
)
|
|
if backup is None:
|
|
return None
|
|
if not expected.unchanged(backup):
|
|
return self._private_recovery_conflict(publication, path, reason)
|
|
os.unlink(path.name, dir_fd=directory_fd)
|
|
os.fsync(directory_fd)
|
|
return None
|
|
finally:
|
|
os.close(directory_fd)
|
|
except Exception as error:
|
|
conflict = self._private_recovery_conflict(publication, path, reason)
|
|
conflict["error"] = (
|
|
error.code if isinstance(error, DocForgeError) else type(error).__name__
|
|
)
|
|
return conflict
|
|
|
|
def _stage(
|
|
self,
|
|
staging_root: Path,
|
|
content: bytes,
|
|
*,
|
|
mode: int,
|
|
owner: int | None,
|
|
group: int | None,
|
|
source: str,
|
|
) -> tuple[Path, _CanonicalFile]:
|
|
descriptor, temporary_name = tempfile.mkstemp(
|
|
prefix="staged-",
|
|
dir=staging_root,
|
|
)
|
|
temporary = Path(temporary_name)
|
|
try:
|
|
with os.fdopen(descriptor, "wb") as handle:
|
|
handle.write(content)
|
|
handle.flush()
|
|
if owner is not None and group is not None:
|
|
os.fchown(handle.fileno(), owner, group)
|
|
os.fchmod(handle.fileno(), mode)
|
|
os.fsync(handle.fileno())
|
|
snapshot = self._capture(temporary, max(1, len(content)))
|
|
if snapshot is None:
|
|
raise DocForgeError(
|
|
"application_mismatch",
|
|
"Canonical staging file disappeared",
|
|
source=source,
|
|
)
|
|
return temporary, snapshot
|
|
except Exception:
|
|
with suppress(OSError):
|
|
os.close(descriptor)
|
|
temporary.unlink(missing_ok=True)
|
|
raise
|
|
|
|
def _capture(self, path: Path, maximum: int) -> _CanonicalFile | None:
|
|
parent_fd = open_bound_directory(path.parent)
|
|
try:
|
|
return self._capture_at(path.parent, parent_fd, path.name, maximum)
|
|
finally:
|
|
os.close(parent_fd)
|
|
|
|
@staticmethod
|
|
def _capture_at(
|
|
parent: Path,
|
|
parent_fd: int,
|
|
name: str,
|
|
maximum: int,
|
|
) -> _CanonicalFile | None:
|
|
del parent
|
|
try:
|
|
descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=parent_fd)
|
|
except FileNotFoundError:
|
|
return None
|
|
except OSError as error:
|
|
raise DocForgeError(
|
|
"path_escape",
|
|
"Canonical target cannot be opened safely",
|
|
target=name,
|
|
) from error
|
|
with os.fdopen(descriptor, "rb") as handle:
|
|
before = os.fstat(handle.fileno())
|
|
if not stat.S_ISREG(before.st_mode) or before.st_size > maximum:
|
|
raise DocForgeError(
|
|
"source_too_large",
|
|
"Canonical target is invalid or exceeds its configured limit",
|
|
target=name,
|
|
)
|
|
content = handle.read(maximum + 1)
|
|
after = os.fstat(handle.fileno())
|
|
if len(content) > maximum:
|
|
raise DocForgeError(
|
|
"source_too_large",
|
|
"Canonical target exceeds its configured limit",
|
|
target=name,
|
|
)
|
|
current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
before_identity = (
|
|
before.st_dev,
|
|
before.st_ino,
|
|
before.st_mode,
|
|
before.st_uid,
|
|
before.st_gid,
|
|
before.st_size,
|
|
before.st_mtime_ns,
|
|
before.st_ctime_ns,
|
|
)
|
|
after_identity = (
|
|
after.st_dev,
|
|
after.st_ino,
|
|
after.st_mode,
|
|
after.st_uid,
|
|
after.st_gid,
|
|
after.st_size,
|
|
after.st_mtime_ns,
|
|
after.st_ctime_ns,
|
|
)
|
|
current_identity = (
|
|
current.st_dev,
|
|
current.st_ino,
|
|
current.st_mode,
|
|
current.st_uid,
|
|
current.st_gid,
|
|
current.st_size,
|
|
current.st_mtime_ns,
|
|
current.st_ctime_ns,
|
|
)
|
|
if before_identity != after_identity or before_identity != current_identity:
|
|
raise DocForgeError(
|
|
"base_conflict",
|
|
"Canonical target changed while it was captured",
|
|
target=name,
|
|
)
|
|
return _CanonicalFile(*before_identity, content)
|
|
|
|
def _recovery_conflict(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
reason: str,
|
|
) -> dict[str, object]:
|
|
return {
|
|
"source": publication.relative,
|
|
"reason": reason,
|
|
"target": self._relative(publication.target),
|
|
"retained": (
|
|
self._relative(publication.staged) if publication.staged.exists() else None
|
|
),
|
|
}
|
|
|
|
def _private_recovery_conflict(
|
|
self,
|
|
publication: _CanonicalPublication,
|
|
path: Path,
|
|
reason: str,
|
|
) -> dict[str, object]:
|
|
return {
|
|
"source": publication.relative,
|
|
"reason": reason,
|
|
"target": self._relative(publication.target),
|
|
"retained": self._relative(path) if path.exists() else None,
|
|
}
|
|
|
|
def _relative(self, path: Path) -> str:
|
|
return path.relative_to(self.project.descriptor.root).as_posix()
|
|
|
|
def _target(self, relative: str) -> Path:
|
|
candidate = Path(relative)
|
|
if candidate.is_absolute() or ".." in candidate.parts:
|
|
raise DocForgeError("path_escape", "Canonical target path is unsafe", source=relative)
|
|
root = self.project.descriptor.root
|
|
target = (root / candidate).resolve(strict=False)
|
|
if (
|
|
not target.is_relative_to(root)
|
|
or not any(
|
|
target.is_relative_to(item) for item in self.project.descriptor.content_roots
|
|
)
|
|
or target.suffix not in {".md", ".toml"}
|
|
):
|
|
raise DocForgeError(
|
|
"path_escape",
|
|
"Canonical target is outside a declared content root",
|
|
source=relative,
|
|
)
|
|
if target.exists() and (target.is_symlink() or not target.is_file()):
|
|
raise DocForgeError("path_escape", "Canonical target is not a regular file")
|
|
return target
|
|
|
|
def _prepare_parent(self, parent: Path, created: list[Path]) -> None:
|
|
root = self.project.descriptor.root
|
|
created.extend(self._create_directories_durable(parent, root=root, mode=0o755))
|
|
|
|
def _prepare_transaction_root(self) -> Path:
|
|
root = self.project.descriptor.root
|
|
namespace = root / ".docforge/application"
|
|
self._create_directories_durable(namespace, root=root, mode=0o700)
|
|
transaction = Path(tempfile.mkdtemp(prefix="transaction-", dir=namespace))
|
|
transaction.chmod(0o700)
|
|
self._fsync_directory(transaction)
|
|
self._fsync_directory(namespace)
|
|
return transaction
|
|
|
|
def _create_directories_durable(
|
|
self,
|
|
path: Path,
|
|
*,
|
|
root: Path,
|
|
mode: int,
|
|
) -> list[Path]:
|
|
missing: list[Path] = []
|
|
cursor = path
|
|
while not cursor.exists():
|
|
missing.append(cursor)
|
|
cursor = cursor.parent
|
|
if cursor.is_symlink() or cursor.resolve(strict=True) != cursor or not cursor.is_dir():
|
|
raise DocForgeError("path_escape", "Canonical target parent is unsafe")
|
|
if not cursor.is_relative_to(root) or not path.is_relative_to(root):
|
|
raise DocForgeError("path_escape", "Canonical target parent escaped the project root")
|
|
created: list[Path] = []
|
|
for directory in reversed(missing):
|
|
try:
|
|
directory.mkdir(mode=mode)
|
|
except FileExistsError as error:
|
|
if (
|
|
directory.is_symlink()
|
|
or not directory.is_dir()
|
|
or directory.resolve(strict=True) != directory
|
|
):
|
|
raise DocForgeError(
|
|
"path_escape",
|
|
"Canonical target parent became unsafe during creation",
|
|
) from error
|
|
continue
|
|
self._fsync_directory(directory)
|
|
self._fsync_directory(directory.parent)
|
|
created.append(directory)
|
|
if path.resolve(strict=True) != path:
|
|
raise DocForgeError("path_escape", "Canonical target parent resolves unexpectedly")
|
|
return created
|
|
|
|
def _finish_transaction(self, transaction: Path) -> list[dict[str, object]]:
|
|
try:
|
|
retained = sorted(
|
|
(self._relative(path) for path in transaction.iterdir()),
|
|
)
|
|
except Exception as error:
|
|
return [
|
|
{
|
|
"reason": "transaction_inspection_failed",
|
|
"retained": self._relative(transaction),
|
|
"error": (
|
|
error.code if isinstance(error, DocForgeError) else type(error).__name__
|
|
),
|
|
}
|
|
]
|
|
if retained:
|
|
return [
|
|
{
|
|
"reason": "transaction_files_retained",
|
|
"retained": retained,
|
|
}
|
|
]
|
|
try:
|
|
transaction.rmdir()
|
|
self._fsync_directory(transaction.parent)
|
|
except OSError as error:
|
|
return [
|
|
{
|
|
"reason": "transaction_cleanup_failed",
|
|
"retained": self._relative(transaction),
|
|
"error": type(error).__name__,
|
|
}
|
|
]
|
|
return []
|
|
|
|
@staticmethod
|
|
def _fsync_directory(path: Path) -> None:
|
|
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
def _remove_empty_directories(self, paths: Sequence[Path]) -> None:
|
|
for path in reversed(paths):
|
|
try:
|
|
path.rmdir()
|
|
self._fsync_directory(path.parent)
|
|
except OSError:
|
|
pass
|
|
|
|
@staticmethod
|
|
def _nodes_by_source(snapshot: ProjectSnapshot) -> dict[str, tuple[Node, ...]]:
|
|
grouped: dict[str, list[Node]] = defaultdict(list)
|
|
for node in snapshot.nodes:
|
|
grouped[node.source_path].append(node)
|
|
return {
|
|
source: tuple(sorted(nodes, key=lambda node: node.node_id))
|
|
for source, nodes in grouped.items()
|
|
}
|
|
|
|
def _changed_sources(
|
|
self,
|
|
base: ProjectSnapshot,
|
|
projected: ProjectSnapshot,
|
|
) -> set[str]:
|
|
base_sources = self._source_signatures(base)
|
|
projected_sources = self._source_signatures(projected)
|
|
return {
|
|
source
|
|
for source in set(base_sources) | set(projected_sources)
|
|
if base_sources.get(source) != projected_sources.get(source)
|
|
}
|
|
|
|
@staticmethod
|
|
def _source_signatures(snapshot: ProjectSnapshot) -> dict[str, object]:
|
|
outgoing: dict[str, list[tuple[str, str]]] = defaultdict(list)
|
|
for edge in snapshot.edges:
|
|
outgoing[edge.source_id].append((edge.relation, edge.target_id))
|
|
signatures: dict[str, list[object]] = defaultdict(list)
|
|
for node in snapshot.nodes:
|
|
signatures[node.source_path].append(
|
|
(
|
|
node.node_id,
|
|
node.title,
|
|
node.family,
|
|
node.authority,
|
|
node.status,
|
|
node.tags,
|
|
node.summary,
|
|
node.content,
|
|
node.source_anchor,
|
|
tuple(sorted(outgoing[node.node_id])),
|
|
)
|
|
)
|
|
return {source: tuple(items) for source, items in signatures.items()}
|
|
|
|
@classmethod
|
|
def _semantic_snapshot(cls, snapshot: ProjectSnapshot) -> tuple[object, object]:
|
|
nodes = tuple(
|
|
(
|
|
node.node_id,
|
|
node.title,
|
|
node.family,
|
|
node.authority,
|
|
node.status,
|
|
node.tags,
|
|
node.summary,
|
|
node.content,
|
|
node.source_path,
|
|
node.source_anchor,
|
|
)
|
|
for node in snapshot.nodes
|
|
)
|
|
edges = tuple((edge.source_id, edge.relation, edge.target_id) for edge in snapshot.edges)
|
|
return nodes, edges
|
|
|
|
def _serialize_source(
|
|
self,
|
|
snapshot: ProjectSnapshot,
|
|
relative: str,
|
|
nodes: tuple[Node, ...],
|
|
) -> bytes:
|
|
if Path(relative).suffix == ".md":
|
|
if len(nodes) != 1:
|
|
raise DocForgeError(
|
|
"source_conflict",
|
|
"Markdown canonical sources may contain only one node",
|
|
source=relative,
|
|
)
|
|
node = nodes[0]
|
|
metadata = self._record(snapshot, node, include_content=False)
|
|
lines = ["+++", *self._toml_record(metadata), "+++", "", node.content.strip(), ""]
|
|
return "\n".join(lines).encode("utf-8")
|
|
lines: list[str] = []
|
|
for index, node in enumerate(nodes):
|
|
if index:
|
|
lines.append("")
|
|
lines.append("[[nodes]]")
|
|
lines.extend(self._toml_record(self._record(snapshot, node, include_content=True)))
|
|
lines.append("")
|
|
return "\n".join(lines).encode("utf-8")
|
|
|
|
@staticmethod
|
|
def _record(
|
|
snapshot: ProjectSnapshot,
|
|
node: Node,
|
|
*,
|
|
include_content: bool,
|
|
) -> dict[str, object]:
|
|
record: dict[str, object] = {
|
|
"schema_version": 1,
|
|
"id": node.node_id,
|
|
"title": node.title,
|
|
"family": node.family,
|
|
"authority": node.authority,
|
|
"status": node.status,
|
|
"tags": list(node.tags),
|
|
"summary": node.summary,
|
|
}
|
|
if node.source_anchor is not None:
|
|
record["source_anchor"] = node.source_anchor
|
|
for relation in snapshot.descriptor.allowed_relations:
|
|
targets = sorted(
|
|
edge.target_id
|
|
for edge in snapshot.edges
|
|
if edge.source_id == node.node_id and edge.relation == relation
|
|
)
|
|
if targets:
|
|
record[relation] = targets
|
|
if include_content:
|
|
record["content"] = node.content
|
|
return record
|
|
|
|
@staticmethod
|
|
def _toml_record(record: dict[str, object]) -> list[str]:
|
|
lines: list[str] = []
|
|
for key, value in record.items():
|
|
if isinstance(value, int):
|
|
encoded = str(value)
|
|
elif isinstance(value, str):
|
|
encoded = json.dumps(value, ensure_ascii=False)
|
|
elif isinstance(value, list):
|
|
string_items: list[str] = []
|
|
for item in cast(list[object], value):
|
|
if not isinstance(item, str):
|
|
raise DocForgeError(
|
|
"application_mismatch",
|
|
"Generic canonical list values must contain only strings",
|
|
field=key,
|
|
)
|
|
string_items.append(item)
|
|
items = ", ".join(json.dumps(item, ensure_ascii=False) for item in string_items)
|
|
encoded = f"[{items}]"
|
|
else:
|
|
raise DocForgeError(
|
|
"application_mismatch",
|
|
"Generic canonical serialization encountered an unsupported value",
|
|
field=key,
|
|
)
|
|
lines.append(f"{key} = {encoded}")
|
|
return lines
|
|
|
|
|
|
class CanonicalApplicationService:
|
|
"""Apply one hash-bound changeset, then refresh all declared derived state."""
|
|
|
|
def __init__(
|
|
self,
|
|
project: ProjectService,
|
|
*,
|
|
applier_id: str | None,
|
|
applier: CanonicalApplier | None,
|
|
index: ProjectIndex | None = None,
|
|
manual_policy: ManualProjectionMode = "auto",
|
|
) -> None:
|
|
self.project = project
|
|
self.applier_id = applier_id
|
|
self.applier = applier
|
|
self.changesets = ChangesetStore(project, applier_id)
|
|
self.index = index or ProjectIndex(project)
|
|
self.manual_policy = validate_manual_projection_mode(manual_policy)
|
|
self.rendering = RenderService(
|
|
project,
|
|
self.changesets,
|
|
manual_policy=self.manual_policy,
|
|
)
|
|
|
|
@property
|
|
def enabled(self) -> bool:
|
|
return self.applier_id is not None and self.applier is not None
|
|
|
|
def access(self) -> dict[str, object]:
|
|
return {
|
|
"enabled": self.enabled,
|
|
"applier": self.applier_id if self.enabled else None,
|
|
}
|
|
|
|
def apply(self, changeset_id: str, expected_changeset_hash: str) -> dict[str, object]:
|
|
if not self.enabled or self.applier is None or self.applier_id is None:
|
|
raise DocForgeError(
|
|
"canonical_application_disabled",
|
|
"Server has no configured canonical applier",
|
|
)
|
|
applied = self.changesets.apply(
|
|
changeset_id=changeset_id,
|
|
expected_changeset_hash=expected_changeset_hash,
|
|
applier_id=self.applier_id,
|
|
application=self.applier.apply,
|
|
)
|
|
refresh_errors: list[dict[str, object]] = []
|
|
index_result: dict[str, object] | None = None
|
|
index_check: dict[str, object] | None = None
|
|
try:
|
|
index_result = self.index.build()
|
|
index_check = self.index.check()
|
|
except DocForgeError as error:
|
|
refresh_errors.append(
|
|
{
|
|
"component": "index",
|
|
"error": error.as_dict(),
|
|
"remediation": {
|
|
"tool": "docforge_sync",
|
|
"arguments": {},
|
|
},
|
|
}
|
|
)
|
|
renders: list[dict[str, object]] = []
|
|
config = self.project.descriptor.render
|
|
render_action = "not_configured"
|
|
if config is not None and self.manual_policy == "auto":
|
|
render_action = "rendered"
|
|
for view in config.views:
|
|
try:
|
|
rendered = self.rendering.render(view.view_id)
|
|
renders.append(rendered)
|
|
if rendered.get("state") == "degraded":
|
|
receipt = rendered.get("receipt")
|
|
refresh_errors.append(
|
|
{
|
|
"component": "render_receipt",
|
|
"view_id": view.view_id,
|
|
"error": (
|
|
cast(Mapping[str, object], receipt).get("error")
|
|
if isinstance(receipt, dict)
|
|
else {
|
|
"code": "render_receipt_failure",
|
|
"message": (
|
|
"Rendered output was published without a "
|
|
"verification receipt"
|
|
),
|
|
"details": {},
|
|
}
|
|
),
|
|
}
|
|
)
|
|
except DocForgeError as error:
|
|
refresh_errors.append(
|
|
{
|
|
"component": "render",
|
|
"view_id": view.view_id,
|
|
"error": error.as_dict(),
|
|
}
|
|
)
|
|
elif config is not None:
|
|
render_action = (
|
|
"skipped_explicit" if self.manual_policy == "explicit" else "skipped_disabled"
|
|
)
|
|
return {
|
|
**applied,
|
|
"derived_refresh": {
|
|
"status": "degraded" if refresh_errors else "ok",
|
|
"index": index_result,
|
|
"check": index_check,
|
|
"renders": renders,
|
|
"render_policy": {
|
|
"mode": self.manual_policy,
|
|
"action": render_action,
|
|
},
|
|
"errors": refresh_errors,
|
|
},
|
|
}
|