Harden canonical publication against races

src/docforge/application.py

@@ -4,13 +4,16 @@ from __future__ import annotations
 
 import json
 import os
+import stat
 import tempfile
 from collections import defaultdict
 from collections.abc import Mapping, Sequence
 from contextlib import suppress
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Protocol, cast
+from typing import Literal, Protocol, cast
 
+from ._fs_safety import open_bound_directory, rename_exchange_at, require_bound_directory
 from .changesets import ChangesetStore
 from .errors import DocForgeError
 from .index import ProjectIndex
@@ -30,6 +33,58 @@ class CanonicalApplier(Protocol):
     ) -> dict[str, object]: ...
 
 
+@dataclass(frozen=True)
+class _CanonicalFile:
+    device: int
+    inode: int
+    mode: int
+    owner: int
+    group: int
+    size: int
+    modified_ns: int
+    changed_ns: int
+    content: bytes
+
+    def unchanged(self, other: _CanonicalFile | None) -> bool:
+        return self == other
+
+    def renamed_to(self, other: _CanonicalFile | None) -> bool:
+        if other is None:
+            return False
+        return (
+            self.device,
+            self.inode,
+            self.mode,
+            self.owner,
+            self.group,
+            self.size,
+            self.modified_ns,
+            self.content,
+        ) == (
+            other.device,
+            other.inode,
+            other.mode,
+            other.owner,
+            other.group,
+            other.size,
+            other.modified_ns,
+            other.content,
+        )
+
+
+@dataclass
+class _CanonicalPublication:
+    relative: str
+    target: Path
+    action: Literal["create", "update", "delete"]
+    expected: _CanonicalFile | None
+    staged: Path
+    staged_snapshot: _CanonicalFile
+    backup_snapshot: _CanonicalFile | None = None
+    published_snapshot: _CanonicalFile | None = None
+    committed: bool = False
+
+
 class GenericCanonicalApplier:
     """Apply generic Markdown/TOML projections inside declared content roots."""
 
@@ -46,35 +101,49 @@ class GenericCanonicalApplier:
         changed_sources = self._changed_sources(base, projected)
         if not changed_sources:
             raise DocForgeError("empty_changeset", "Changeset produces no canonical changes")
+        base_by_source = self._nodes_by_source(base)
         projected_by_source = self._nodes_by_source(projected)
-        staged: dict[Path, Path] = {}
-        previous: dict[Path, bytes | None] = {}
+        publications: list[_CanonicalPublication] = []
         created_directories: list[Path] = []
         targets = {relative: self._target(relative) for relative in sorted(changed_sources)}
         try:
             for relative, target in targets.items():
-                previous[target] = target.read_bytes() if target.is_file() else None
-                nodes = projected_by_source.get(relative, ())
-                if not nodes:
-                    continue
                 self._prepare_parent(target.parent, created_directories)
-                raw = self._serialize_source(projected, relative, nodes)
-                if len(raw) > base.descriptor.limits.max_source_bytes:
+                expected = self._capture(target, base.descriptor.limits.max_source_bytes)
+                existed_in_base = relative in base_by_source
+                if existed_in_base != (expected is not None):
+                    raise DocForgeError(
+                        "base_conflict",
+                        "Canonical target changed before application staging",
+                        source=relative,
+                    )
+                nodes = projected_by_source.get(relative, ())
+                raw = self._serialize_source(projected, relative, nodes) if nodes else b""
+                if nodes and len(raw) > base.descriptor.limits.max_source_bytes:
                     raise DocForgeError(
                         "source_too_large",
                         "Applied canonical source exceeds the configured limit",
                         source=relative,
                     )
-                descriptor, temporary_name = tempfile.mkstemp(
-                    prefix=".docforge-apply-",
-                    dir=target.parent,
+                staged, staged_snapshot = self._stage(
+                    target,
+                    raw,
+                    mode=(stat.S_IMODE(expected.mode) if expected is not None else 0o600),
+                )
+                publications.append(
+                    _CanonicalPublication(
+                        relative=relative,
+                        target=target,
+                        action=(
+                            "delete"
+                            if not nodes
+                            else ("update" if expected is not None else "create")
+                        ),
+                        expected=expected,
+                        staged=staged,
+                        staged_snapshot=staged_snapshot,
+                    )
                 )
-                temporary = Path(temporary_name)
-                with os.fdopen(descriptor, "wb") as handle:
-                    handle.write(raw)
-                    handle.flush()
-                    os.fsync(handle.fileno())
-                staged[target] = temporary
 
             current = self.project.load()
             if current.source_hash != base.source_hash or current.revision != base.revision:
@@ -89,13 +158,8 @@ class GenericCanonicalApplier:
             for target in targets.values():
                 if target.is_symlink():
                     raise DocForgeError("path_escape", "Canonical target became a symbolic link")
-            for target in sorted(targets.values(), key=str):
-                temporary = staged.get(target)
-                if temporary is None:
-                    target.unlink(missing_ok=True)
-                else:
-                    os.replace(temporary, target)
-                self._fsync_directory(target.parent)
+            for publication in publications:
+                self._publish(publication)
 
             applied = self.project.load()
             if self._semantic_snapshot(applied) != self._semantic_snapshot(projected):
@@ -103,19 +167,522 @@ class GenericCanonicalApplier:
                     "application_mismatch",
                     "Applied canonical files do not reproduce the validated proposal",
                 )
-        except Exception:
-            for temporary in staged.values():
-                temporary.unlink(missing_ok=True)
-            self._restore(previous)
+        except Exception as error:
+            recovery = self._rollback(publications)
+            self._discard_unowned_staging(publications)
             self._remove_empty_directories(created_directories)
+            if recovery:
+                cause = error.code if isinstance(error, DocForgeError) else type(error).__name__
+                raise DocForgeError(
+                    "application_recovery_required",
+                    (
+                        "Canonical application raced external writes and could not be "
+                        "rolled back without overwriting foreign data"
+                    ),
+                    cause=cause,
+                    conflicts=recovery,
+                    remediation=(
+                        "Preserve the retained files, compare them with the named canonical "
+                        "targets, and resolve the project before retrying a new changeset."
+                    ),
+                ) from error
+            if isinstance(error, DocForgeError) and error.code == "application_recovery_required":
+                raise DocForgeError(
+                    "base_conflict",
+                    "Canonical target raced publication but was restored without data loss",
+                ) from error
             raise
+        retained = self._discard_backups(publications)
         return {
             "applied_sources": sorted(changed_sources),
             "removed_sources": sorted(
                 source for source in changed_sources if source not in projected_by_source
             ),
+            "retained_recovery_files": retained,
         }
 
+    def _publish(self, publication: _CanonicalPublication) -> None:
+        target = publication.target
+        parent_fd = open_bound_directory(target.parent)
+        try:
+            current = self._capture_at(
+                target.parent,
+                parent_fd,
+                target.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            if publication.expected is None:
+                if current is not None:
+                    raise DocForgeError(
+                        "base_conflict",
+                        "Canonical create target appeared during application",
+                        source=publication.relative,
+                    )
+            elif not publication.expected.unchanged(current):
+                raise DocForgeError(
+                    "base_conflict",
+                    "Canonical target changed during application",
+                    source=publication.relative,
+                )
+            staged = self._capture_at(
+                target.parent,
+                parent_fd,
+                publication.staged.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            if staged is None or not publication.staged_snapshot.unchanged(staged):
+                raise DocForgeError(
+                    "application_mismatch",
+                    "Canonical staging file changed before publication",
+                    source=publication.relative,
+                )
+            if publication.action == "create":
+                self._publish_create(publication, parent_fd)
+            else:
+                self._publish_exchange(publication, parent_fd)
+        finally:
+            os.close(parent_fd)
+
+    def _publish_create(
+        self,
+        publication: _CanonicalPublication,
+        parent_fd: int,
+    ) -> None:
+        try:
+            os.link(
+                publication.staged.name,
+                publication.target.name,
+                src_dir_fd=parent_fd,
+                dst_dir_fd=parent_fd,
+                follow_symlinks=False,
+            )
+        except FileExistsError as error:
+            raise DocForgeError(
+                "base_conflict",
+                "Canonical create target appeared during application",
+                source=publication.relative,
+            ) from error
+        except OSError as error:
+            raise DocForgeError(
+                "application_failure",
+                "Canonical create target could not be published",
+                source=publication.relative,
+            ) from error
+        publication.committed = True
+        published = self._capture_at(
+            publication.target.parent,
+            parent_fd,
+            publication.target.name,
+            self.project.descriptor.limits.max_source_bytes,
+        )
+        if not publication.staged_snapshot.renamed_to(published):
+            raise DocForgeError(
+                "application_mismatch",
+                "Published canonical create target does not match its staging file",
+                source=publication.relative,
+            )
+        publication.published_snapshot = published
+        os.unlink(publication.staged.name, dir_fd=parent_fd)
+        os.fsync(parent_fd)
+        require_bound_directory(publication.target.parent, parent_fd)
+
+    def _publish_exchange(
+        self,
+        publication: _CanonicalPublication,
+        parent_fd: int,
+    ) -> None:
+        rename_exchange_at(parent_fd, publication.staged.name, publication.target.name)
+        publication.committed = True
+        displaced = self._capture_at(
+            publication.target.parent,
+            parent_fd,
+            publication.staged.name,
+            self.project.descriptor.limits.max_source_bytes,
+        )
+        published = self._capture_at(
+            publication.target.parent,
+            parent_fd,
+            publication.target.name,
+            self.project.descriptor.limits.max_source_bytes,
+        )
+        publication.backup_snapshot = displaced
+        publication.published_snapshot = published
+        if (
+            publication.expected is None
+            or not publication.expected.renamed_to(displaced)
+            or not publication.staged_snapshot.renamed_to(published)
+        ):
+            if not self._exchange_back(publication, parent_fd):
+                raise DocForgeError(
+                    "application_recovery_required",
+                    "Canonical target raced atomic publication and displaced data was retained",
+                    source=publication.relative,
+                    retained=self._relative(publication.staged),
+                )
+            raise DocForgeError(
+                "base_conflict",
+                "Canonical target changed at the atomic publication boundary",
+                source=publication.relative,
+            )
+        if publication.action == "delete":
+            os.unlink(publication.target.name, dir_fd=parent_fd)
+            publication.published_snapshot = None
+        os.fsync(parent_fd)
+        require_bound_directory(publication.target.parent, parent_fd)
+
+    def _exchange_back(
+        self,
+        publication: _CanonicalPublication,
+        parent_fd: int,
+    ) -> bool:
+        backup = publication.backup_snapshot
+        published = publication.published_snapshot
+        if backup is None or published is None:
+            return False
+        try:
+            rename_exchange_at(parent_fd, publication.staged.name, publication.target.name)
+            restored = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                publication.target.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            staged = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                publication.staged.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            if not backup.renamed_to(restored) or not published.renamed_to(staged):
+                return False
+            publication.backup_snapshot = None
+            publication.published_snapshot = None
+            publication.committed = False
+            os.fsync(parent_fd)
+            return True
+        except (DocForgeError, OSError):
+            return False
+
+    def _rollback(
+        self,
+        publications: Sequence[_CanonicalPublication],
+    ) -> list[dict[str, object]]:
+        conflicts: list[dict[str, object]] = []
+        for publication in reversed(publications):
+            if not publication.committed:
+                continue
+            try:
+                if publication.action == "create":
+                    conflict = self._rollback_create(publication)
+                elif publication.action == "delete" and publication.published_snapshot is None:
+                    conflict = self._rollback_delete(publication)
+                else:
+                    conflict = self._rollback_update(publication)
+            except Exception as error:
+                conflict = self._recovery_conflict(publication, "rollback_failed")
+                conflict["error"] = (
+                    error.code if isinstance(error, DocForgeError) else type(error).__name__
+                )
+            if conflict is not None:
+                conflicts.append(conflict)
+        return conflicts
+
+    def _rollback_update(
+        self,
+        publication: _CanonicalPublication,
+    ) -> dict[str, object] | None:
+        parent_fd = open_bound_directory(publication.target.parent)
+        try:
+            current = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                publication.target.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            backup = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                publication.staged.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            if (
+                publication.published_snapshot is None
+                or publication.backup_snapshot is None
+                or not publication.published_snapshot.unchanged(current)
+                or not publication.backup_snapshot.unchanged(backup)
+            ):
+                return self._recovery_conflict(publication, "target_or_backup_changed")
+            if not self._exchange_back(publication, parent_fd):
+                return self._recovery_conflict(publication, "atomic_restore_unconfirmed")
+            os.unlink(publication.staged.name, dir_fd=parent_fd)
+            os.fsync(parent_fd)
+            return None
+        finally:
+            os.close(parent_fd)
+
+    def _rollback_delete(
+        self,
+        publication: _CanonicalPublication,
+    ) -> dict[str, object] | None:
+        parent_fd = open_bound_directory(publication.target.parent)
+        try:
+            current = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                publication.target.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            backup = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                publication.staged.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            if current is not None:
+                return self._recovery_conflict(publication, "deleted_target_reappeared")
+            if publication.backup_snapshot is None or not publication.backup_snapshot.unchanged(
+                backup
+            ):
+                return self._recovery_conflict(publication, "backup_changed")
+            try:
+                os.link(
+                    publication.staged.name,
+                    publication.target.name,
+                    src_dir_fd=parent_fd,
+                    dst_dir_fd=parent_fd,
+                    follow_symlinks=False,
+                )
+            except FileExistsError:
+                return self._recovery_conflict(publication, "deleted_target_reappeared")
+            restored = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                publication.target.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            if not publication.backup_snapshot.renamed_to(restored):
+                return self._recovery_conflict(publication, "restore_unconfirmed")
+            os.unlink(publication.staged.name, dir_fd=parent_fd)
+            publication.backup_snapshot = None
+            publication.committed = False
+            os.fsync(parent_fd)
+            return None
+        finally:
+            os.close(parent_fd)
+
+    def _rollback_create(
+        self,
+        publication: _CanonicalPublication,
+    ) -> dict[str, object] | None:
+        parent_fd = open_bound_directory(publication.target.parent)
+        tombstone: Path | None = None
+        try:
+            current = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                publication.target.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            if current is None:
+                publication.committed = False
+                return None
+            if (
+                publication.published_snapshot is None
+                or not publication.published_snapshot.unchanged(current)
+            ):
+                return self._recovery_conflict(publication, "created_target_changed")
+            tombstone, tombstone_snapshot = self._stage(publication.target, b"", mode=0o600)
+            rename_exchange_at(parent_fd, tombstone.name, publication.target.name)
+            displaced = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                tombstone.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            published_tombstone = self._capture_at(
+                publication.target.parent,
+                parent_fd,
+                publication.target.name,
+                self.project.descriptor.limits.max_source_bytes,
+            )
+            if not publication.published_snapshot.renamed_to(
+                displaced
+            ) or not tombstone_snapshot.renamed_to(published_tombstone):
+                with suppress(DocForgeError):
+                    rename_exchange_at(parent_fd, tombstone.name, publication.target.name)
+                return self._recovery_conflict(publication, "create_rollback_raced")
+            os.unlink(publication.target.name, dir_fd=parent_fd)
+            os.unlink(tombstone.name, dir_fd=parent_fd)
+            tombstone = None
+            publication.committed = False
+            publication.published_snapshot = None
+            os.fsync(parent_fd)
+            return None
+        finally:
+            if tombstone is not None:
+                tombstone.unlink(missing_ok=True)
+            os.close(parent_fd)
+
+    def _discard_backups(
+        self,
+        publications: Sequence[_CanonicalPublication],
+    ) -> list[dict[str, object]]:
+        retained: list[dict[str, object]] = []
+        for publication in publications:
+            if publication.action != "create" and publication.staged.exists():
+                parent_fd = open_bound_directory(publication.target.parent)
+                try:
+                    backup = self._capture_at(
+                        publication.target.parent,
+                        parent_fd,
+                        publication.staged.name,
+                        self.project.descriptor.limits.max_source_bytes,
+                    )
+                    if (
+                        publication.backup_snapshot is None
+                        or not publication.backup_snapshot.unchanged(backup)
+                    ):
+                        retained.append(
+                            self._recovery_conflict(publication, "backup_cleanup_raced")
+                        )
+                        continue
+                    os.unlink(publication.staged.name, dir_fd=parent_fd)
+                    os.fsync(parent_fd)
+                finally:
+                    os.close(parent_fd)
+            publication.committed = False
+            publication.backup_snapshot = None
+        return retained
+
+    @staticmethod
+    def _discard_unowned_staging(publications: Sequence[_CanonicalPublication]) -> None:
+        for publication in publications:
+            if not publication.committed and publication.staged.exists():
+                publication.staged.unlink()
+
+    def _stage(self, target: Path, content: bytes, *, mode: int) -> tuple[Path, _CanonicalFile]:
+        descriptor, temporary_name = tempfile.mkstemp(
+            prefix=".docforge-apply-",
+            dir=target.parent,
+        )
+        temporary = Path(temporary_name)
+        try:
+            os.fchmod(descriptor, mode)
+            with os.fdopen(descriptor, "wb") as handle:
+                handle.write(content)
+                handle.flush()
+                os.fsync(handle.fileno())
+            snapshot = self._capture(temporary, max(1, len(content)))
+            if snapshot is None:
+                raise DocForgeError(
+                    "application_mismatch",
+                    "Canonical staging file disappeared",
+                    source=self._relative(target),
+                )
+            return temporary, snapshot
+        except Exception:
+            with suppress(OSError):
+                os.close(descriptor)
+            temporary.unlink(missing_ok=True)
+            raise
+
+    def _capture(self, path: Path, maximum: int) -> _CanonicalFile | None:
+        parent_fd = open_bound_directory(path.parent)
+        try:
+            return self._capture_at(path.parent, parent_fd, path.name, maximum)
+        finally:
+            os.close(parent_fd)
+
+    @staticmethod
+    def _capture_at(
+        parent: Path,
+        parent_fd: int,
+        name: str,
+        maximum: int,
+    ) -> _CanonicalFile | None:
+        del parent
+        try:
+            descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=parent_fd)
+        except FileNotFoundError:
+            return None
+        except OSError as error:
+            raise DocForgeError(
+                "path_escape",
+                "Canonical target cannot be opened safely",
+                target=name,
+            ) from error
+        with os.fdopen(descriptor, "rb") as handle:
+            before = os.fstat(handle.fileno())
+            if not stat.S_ISREG(before.st_mode) or before.st_size > maximum:
+                raise DocForgeError(
+                    "source_too_large",
+                    "Canonical target is invalid or exceeds its configured limit",
+                    target=name,
+                )
+            content = handle.read(maximum + 1)
+            after = os.fstat(handle.fileno())
+        if len(content) > maximum:
+            raise DocForgeError(
+                "source_too_large",
+                "Canonical target exceeds its configured limit",
+                target=name,
+            )
+        current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+        before_identity = (
+            before.st_dev,
+            before.st_ino,
+            before.st_mode,
+            before.st_uid,
+            before.st_gid,
+            before.st_size,
+            before.st_mtime_ns,
+            before.st_ctime_ns,
+        )
+        after_identity = (
+            after.st_dev,
+            after.st_ino,
+            after.st_mode,
+            after.st_uid,
+            after.st_gid,
+            after.st_size,
+            after.st_mtime_ns,
+            after.st_ctime_ns,
+        )
+        current_identity = (
+            current.st_dev,
+            current.st_ino,
+            current.st_mode,
+            current.st_uid,
+            current.st_gid,
+            current.st_size,
+            current.st_mtime_ns,
+            current.st_ctime_ns,
+        )
+        if before_identity != after_identity or before_identity != current_identity:
+            raise DocForgeError(
+                "base_conflict",
+                "Canonical target changed while it was captured",
+                target=name,
+            )
+        return _CanonicalFile(*before_identity, content)
+
+    def _recovery_conflict(
+        self,
+        publication: _CanonicalPublication,
+        reason: str,
+    ) -> dict[str, object]:
+        return {
+            "source": publication.relative,
+            "reason": reason,
+            "target": self._relative(publication.target),
+            "retained": (
+                self._relative(publication.staged) if publication.staged.exists() else None
+            ),
+        }
+
+    def _relative(self, path: Path) -> str:
+        return path.relative_to(self.project.descriptor.root).as_posix()
+
     def _target(self, relative: str) -> Path:
         candidate = Path(relative)
         if candidate.is_absolute() or ".." in candidate.parts:
@@ -162,29 +729,6 @@ class GenericCanonicalApplier:
         finally:
             os.close(descriptor)
 
-    def _restore(self, previous: dict[Path, bytes | None]) -> None:
-        for target in sorted(previous, key=str):
-            raw = previous[target]
-            if raw is None:
-                target.unlink(missing_ok=True)
-                continue
-            target.parent.mkdir(parents=True, exist_ok=True)
-            descriptor, temporary_name = tempfile.mkstemp(
-                prefix=".docforge-rollback-",
-                dir=target.parent,
-            )
-            temporary = Path(temporary_name)
-            try:
-                with os.fdopen(descriptor, "wb") as handle:
-                    handle.write(raw)
-                    handle.flush()
-                    os.fsync(handle.fileno())
-                os.replace(temporary, target)
-                self._fsync_directory(target.parent)
-            except Exception:
-                temporary.unlink(missing_ok=True)
-                raise
-
     @staticmethod
     def _remove_empty_directories(paths: Sequence[Path]) -> None:
         for path in reversed(paths):