490 lines
17 KiB
Python
490 lines
17 KiB
Python
|
"""Bounded, path-free, disposable projection fragment caching."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import base64
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
from dataclasses import dataclass
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Literal, cast
|
||
|
|
|
||
|
|
from ._fs_safety import (
|
||
|
|
atomic_replace_bytes_at,
|
||
|
|
open_confined_directory,
|
||
|
|
read_bounded_file_at,
|
||
|
|
require_bound_directory,
|
||
|
|
)
|
||
|
|
from .errors import DocForgeError
|
||
|
|
from .projection_contract import canonical_projection_bytes, projection_hash
|
||
|
|
|
||
|
|
FRAGMENT_SCHEMA_VERSION = 1
|
||
|
|
FRAGMENT_KEY_CONTRACT = "docforge.projection-fragment-key"
|
||
|
|
FRAGMENT_RECORD_CONTRACT = "docforge.projection-fragment-record"
|
||
|
|
FRAGMENT_CACHE_DIRECTORY = "projection-fragments-v1"
|
||
|
|
MAX_FRAGMENT_CONTENT_BYTES = 4_000_000
|
||
|
|
MAX_FRAGMENT_ID_CHARS = 256
|
||
|
|
MAX_FRAGMENT_CACHE_ENTRIES = 10_000
|
||
|
|
MAX_FRAGMENT_CACHE_BYTES = 64_000_000
|
||
|
|
_MAX_RECORD_OVERHEAD_BYTES = 8_192
|
||
|
|
|
||
|
|
ProjectionKind = Literal["manual", "graph"]
|
||
|
|
|
||
|
|
|
||
|
|
def _is_hash(value: object) -> bool:
|
||
|
|
return (
|
||
|
|
isinstance(value, str)
|
||
|
|
and len(value) == 64
|
||
|
|
and all(character in "0123456789abcdef" for character in value)
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _version_string(value: object, *, field: str) -> str:
|
||
|
|
if (
|
||
|
|
not isinstance(value, str)
|
||
|
|
or not value
|
||
|
|
or value != value.strip()
|
||
|
|
or len(value) > MAX_FRAGMENT_ID_CHARS
|
||
|
|
or any(ord(character) < 32 for character in value)
|
||
|
|
):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment identity is invalid",
|
||
|
|
field=field,
|
||
|
|
)
|
||
|
|
return value
|
||
|
|
|
||
|
|
|
||
|
|
def fragment_semantic_hash(value: object) -> str:
|
||
|
|
"""Hash one complete semantic input using the projection canonical JSON form."""
|
||
|
|
|
||
|
|
try:
|
||
|
|
return hashlib.sha256(canonical_projection_bytes(value)).hexdigest()
|
||
|
|
except (TypeError, ValueError) as error:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment semantic input is not canonical JSON",
|
||
|
|
) from error
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass(frozen=True)
|
||
|
|
class FragmentKey:
|
||
|
|
"""Versioned identity for one renderer component's complete semantics."""
|
||
|
|
|
||
|
|
projection_kind: ProjectionKind
|
||
|
|
renderer_id: str
|
||
|
|
renderer_version: str
|
||
|
|
component_version: str
|
||
|
|
semantic_input_hash: str
|
||
|
|
key_id: str
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def create(
|
||
|
|
cls,
|
||
|
|
*,
|
||
|
|
projection_kind: ProjectionKind,
|
||
|
|
renderer_id: str,
|
||
|
|
renderer_version: str,
|
||
|
|
component_version: str,
|
||
|
|
semantic_input_hash: str,
|
||
|
|
) -> FragmentKey:
|
||
|
|
body = cls._body(
|
||
|
|
projection_kind=projection_kind,
|
||
|
|
renderer_id=renderer_id,
|
||
|
|
renderer_version=renderer_version,
|
||
|
|
component_version=component_version,
|
||
|
|
semantic_input_hash=semantic_input_hash,
|
||
|
|
)
|
||
|
|
return cls._from_validated({**body, "key_id": projection_hash(body)})
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def from_dict(cls, value: object) -> FragmentKey:
|
||
|
|
if not isinstance(value, dict):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment key is invalid",
|
||
|
|
)
|
||
|
|
return cls._from_validated(dict(cast(dict[str, object], value)))
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _body(
|
||
|
|
*,
|
||
|
|
projection_kind: object,
|
||
|
|
renderer_id: object,
|
||
|
|
renderer_version: object,
|
||
|
|
component_version: object,
|
||
|
|
semantic_input_hash: object,
|
||
|
|
) -> dict[str, object]:
|
||
|
|
if projection_kind not in {"manual", "graph"}:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment kind is invalid",
|
||
|
|
)
|
||
|
|
if not _is_hash(semantic_input_hash):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment semantic input hash is invalid",
|
||
|
|
)
|
||
|
|
return {
|
||
|
|
"schema_version": FRAGMENT_SCHEMA_VERSION,
|
||
|
|
"contract": FRAGMENT_KEY_CONTRACT,
|
||
|
|
"projection_kind": projection_kind,
|
||
|
|
"renderer_id": _version_string(renderer_id, field="renderer_id"),
|
||
|
|
"renderer_version": _version_string(
|
||
|
|
renderer_version,
|
||
|
|
field="renderer_version",
|
||
|
|
),
|
||
|
|
"component_version": _version_string(
|
||
|
|
component_version,
|
||
|
|
field="component_version",
|
||
|
|
),
|
||
|
|
"semantic_input_hash": semantic_input_hash,
|
||
|
|
}
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def _from_validated(cls, value: dict[str, object]) -> FragmentKey:
|
||
|
|
required = {
|
||
|
|
"schema_version",
|
||
|
|
"contract",
|
||
|
|
"projection_kind",
|
||
|
|
"renderer_id",
|
||
|
|
"renderer_version",
|
||
|
|
"component_version",
|
||
|
|
"semantic_input_hash",
|
||
|
|
"key_id",
|
||
|
|
}
|
||
|
|
if (
|
||
|
|
set(value) != required
|
||
|
|
or value.get("schema_version") != FRAGMENT_SCHEMA_VERSION
|
||
|
|
or value.get("contract") != FRAGMENT_KEY_CONTRACT
|
||
|
|
):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment key contract is incompatible",
|
||
|
|
)
|
||
|
|
body = cls._body(
|
||
|
|
projection_kind=value.get("projection_kind"),
|
||
|
|
renderer_id=value.get("renderer_id"),
|
||
|
|
renderer_version=value.get("renderer_version"),
|
||
|
|
component_version=value.get("component_version"),
|
||
|
|
semantic_input_hash=value.get("semantic_input_hash"),
|
||
|
|
)
|
||
|
|
key_id = value.get("key_id")
|
||
|
|
if not _is_hash(key_id) or key_id != projection_hash(body):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment key does not match its semantics",
|
||
|
|
)
|
||
|
|
return cls(
|
||
|
|
projection_kind=cast(ProjectionKind, body["projection_kind"]),
|
||
|
|
renderer_id=cast(str, body["renderer_id"]),
|
||
|
|
renderer_version=cast(str, body["renderer_version"]),
|
||
|
|
component_version=cast(str, body["component_version"]),
|
||
|
|
semantic_input_hash=cast(str, body["semantic_input_hash"]),
|
||
|
|
key_id=cast(str, key_id),
|
||
|
|
)
|
||
|
|
|
||
|
|
def as_dict(self) -> dict[str, object]:
|
||
|
|
return {
|
||
|
|
"schema_version": FRAGMENT_SCHEMA_VERSION,
|
||
|
|
"contract": FRAGMENT_KEY_CONTRACT,
|
||
|
|
"projection_kind": self.projection_kind,
|
||
|
|
"renderer_id": self.renderer_id,
|
||
|
|
"renderer_version": self.renderer_version,
|
||
|
|
"component_version": self.component_version,
|
||
|
|
"semantic_input_hash": self.semantic_input_hash,
|
||
|
|
"key_id": self.key_id,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass(frozen=True)
|
||
|
|
class FragmentRecord:
|
||
|
|
"""One path-free fragment payload with complete byte evidence."""
|
||
|
|
|
||
|
|
key: FragmentKey
|
||
|
|
content: bytes
|
||
|
|
byte_count: int
|
||
|
|
content_sha256: str
|
||
|
|
record_id: str
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def create(cls, key: FragmentKey, content: bytes) -> FragmentRecord:
|
||
|
|
if len(content) > MAX_FRAGMENT_CONTENT_BYTES:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment content is invalid or oversized",
|
||
|
|
maximum_bytes=MAX_FRAGMENT_CONTENT_BYTES,
|
||
|
|
)
|
||
|
|
body = cls._body(key, content)
|
||
|
|
return cls(
|
||
|
|
key=key,
|
||
|
|
content=content,
|
||
|
|
byte_count=len(content),
|
||
|
|
content_sha256=hashlib.sha256(content).hexdigest(),
|
||
|
|
record_id=projection_hash(body),
|
||
|
|
)
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def from_dict(cls, value: object) -> FragmentRecord:
|
||
|
|
if not isinstance(value, dict):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment record is invalid",
|
||
|
|
)
|
||
|
|
document = dict(cast(dict[str, object], value))
|
||
|
|
required = {
|
||
|
|
"schema_version",
|
||
|
|
"contract",
|
||
|
|
"record_id",
|
||
|
|
"key",
|
||
|
|
"content_encoding",
|
||
|
|
"content",
|
||
|
|
"byte_count",
|
||
|
|
"content_sha256",
|
||
|
|
}
|
||
|
|
if (
|
||
|
|
set(document) != required
|
||
|
|
or document.get("schema_version") != FRAGMENT_SCHEMA_VERSION
|
||
|
|
or document.get("contract") != FRAGMENT_RECORD_CONTRACT
|
||
|
|
or document.get("content_encoding") != "base64"
|
||
|
|
):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment record contract is incompatible",
|
||
|
|
)
|
||
|
|
encoded = document.get("content")
|
||
|
|
if not isinstance(encoded, str):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment content encoding is invalid",
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
content = base64.b64decode(encoded.encode("ascii"), validate=True)
|
||
|
|
except (UnicodeEncodeError, ValueError) as error:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment content encoding is invalid",
|
||
|
|
) from error
|
||
|
|
if len(content) > MAX_FRAGMENT_CONTENT_BYTES:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment content is oversized",
|
||
|
|
maximum_bytes=MAX_FRAGMENT_CONTENT_BYTES,
|
||
|
|
)
|
||
|
|
key = FragmentKey.from_dict(document.get("key"))
|
||
|
|
body = cls._body(key, content)
|
||
|
|
record_id = document.get("record_id")
|
||
|
|
if (
|
||
|
|
type(document.get("byte_count")) is not int
|
||
|
|
or document.get("byte_count") != len(content)
|
||
|
|
or document.get("content_sha256") != hashlib.sha256(content).hexdigest()
|
||
|
|
or not _is_hash(record_id)
|
||
|
|
or record_id != projection_hash(body)
|
||
|
|
):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment byte evidence is invalid",
|
||
|
|
)
|
||
|
|
return cls(
|
||
|
|
key=key,
|
||
|
|
content=content,
|
||
|
|
byte_count=len(content),
|
||
|
|
content_sha256=hashlib.sha256(content).hexdigest(),
|
||
|
|
record_id=cast(str, record_id),
|
||
|
|
)
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def from_bytes(cls, raw: bytes) -> FragmentRecord:
|
||
|
|
try:
|
||
|
|
value: object = json.loads(raw)
|
||
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment record is not valid JSON",
|
||
|
|
) from error
|
||
|
|
record = cls.from_dict(value)
|
||
|
|
if record.to_bytes() != raw:
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment record is not canonically serialized",
|
||
|
|
)
|
||
|
|
return record
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _body(key: FragmentKey, content: bytes) -> dict[str, object]:
|
||
|
|
return {
|
||
|
|
"schema_version": FRAGMENT_SCHEMA_VERSION,
|
||
|
|
"contract": FRAGMENT_RECORD_CONTRACT,
|
||
|
|
"key": key.as_dict(),
|
||
|
|
"content_encoding": "base64",
|
||
|
|
"content": base64.b64encode(content).decode("ascii"),
|
||
|
|
"byte_count": len(content),
|
||
|
|
"content_sha256": hashlib.sha256(content).hexdigest(),
|
||
|
|
}
|
||
|
|
|
||
|
|
def as_dict(self) -> dict[str, object]:
|
||
|
|
return {
|
||
|
|
**self._body(self.key, self.content),
|
||
|
|
"record_id": self.record_id,
|
||
|
|
}
|
||
|
|
|
||
|
|
def to_bytes(self) -> bytes:
|
||
|
|
return canonical_projection_bytes(self.as_dict())
|
||
|
|
|
||
|
|
|
||
|
|
class ProjectionFragmentCache:
|
||
|
|
"""Confined best-effort storage for immutable projection fragments."""
|
||
|
|
|
||
|
|
def __init__(
|
||
|
|
self,
|
||
|
|
project_root: Path,
|
||
|
|
cache_root: Path,
|
||
|
|
*,
|
||
|
|
maximum_content_bytes: int = MAX_FRAGMENT_CONTENT_BYTES,
|
||
|
|
) -> None:
|
||
|
|
if (
|
||
|
|
type(maximum_content_bytes) is not int
|
||
|
|
or not 1 <= maximum_content_bytes <= MAX_FRAGMENT_CONTENT_BYTES
|
||
|
|
):
|
||
|
|
raise DocForgeError(
|
||
|
|
"invalid_projection_fragment",
|
||
|
|
"Projection fragment cache byte limit is invalid",
|
||
|
|
maximum_bytes=MAX_FRAGMENT_CONTENT_BYTES,
|
||
|
|
)
|
||
|
|
self.project_root = project_root
|
||
|
|
self.cache_root = cache_root
|
||
|
|
self.fragment_root = cache_root / FRAGMENT_CACHE_DIRECTORY
|
||
|
|
self.maximum_content_bytes = maximum_content_bytes
|
||
|
|
|
||
|
|
@property
|
||
|
|
def maximum_record_bytes(self) -> int:
|
||
|
|
encoded = ((self.maximum_content_bytes + 2) // 3) * 4
|
||
|
|
return encoded + _MAX_RECORD_OVERHEAD_BYTES
|
||
|
|
|
||
|
|
def get(self, key: FragmentKey) -> FragmentRecord | None:
|
||
|
|
"""Return one exact compatible fragment, treating every cache defect as a miss."""
|
||
|
|
|
||
|
|
descriptor: int | None = None
|
||
|
|
try:
|
||
|
|
descriptor = self._open(create=False)
|
||
|
|
raw = read_bounded_file_at(
|
||
|
|
descriptor,
|
||
|
|
f"{key.key_id}.json",
|
||
|
|
self.maximum_record_bytes,
|
||
|
|
)
|
||
|
|
if raw is None:
|
||
|
|
return None
|
||
|
|
record = FragmentRecord.from_bytes(raw)
|
||
|
|
if record.key != key or record.byte_count > self.maximum_content_bytes:
|
||
|
|
return None
|
||
|
|
require_bound_directory(self.fragment_root, descriptor)
|
||
|
|
return record
|
||
|
|
except (DocForgeError, OSError):
|
||
|
|
return None
|
||
|
|
finally:
|
||
|
|
if descriptor is not None:
|
||
|
|
os.close(descriptor)
|
||
|
|
|
||
|
|
def put(self, key: FragmentKey, content: bytes) -> FragmentRecord | None:
|
||
|
|
"""Durably publish one fragment, returning ``None`` on disposable cache failure."""
|
||
|
|
|
||
|
|
if len(content) > self.maximum_content_bytes:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
record = FragmentRecord.create(key, content)
|
||
|
|
except DocForgeError:
|
||
|
|
return None
|
||
|
|
raw = record.to_bytes()
|
||
|
|
if len(raw) > self.maximum_record_bytes:
|
||
|
|
return None
|
||
|
|
descriptor: int | None = None
|
||
|
|
try:
|
||
|
|
descriptor = self._open(create=True)
|
||
|
|
name = f"{key.key_id}.json"
|
||
|
|
try:
|
||
|
|
existing = read_bounded_file_at(
|
||
|
|
descriptor,
|
||
|
|
name,
|
||
|
|
self.maximum_record_bytes,
|
||
|
|
)
|
||
|
|
except DocForgeError as error:
|
||
|
|
if error.code == "path_escape":
|
||
|
|
return None
|
||
|
|
existing = None
|
||
|
|
if existing == raw:
|
||
|
|
require_bound_directory(self.fragment_root, descriptor)
|
||
|
|
return record
|
||
|
|
atomic_replace_bytes_at(
|
||
|
|
self.fragment_root,
|
||
|
|
descriptor,
|
||
|
|
name,
|
||
|
|
raw,
|
||
|
|
verify=lambda: require_bound_directory(self.fragment_root, descriptor),
|
||
|
|
)
|
||
|
|
return record
|
||
|
|
except (DocForgeError, OSError):
|
||
|
|
return None
|
||
|
|
finally:
|
||
|
|
if descriptor is not None:
|
||
|
|
os.close(descriptor)
|
||
|
|
|
||
|
|
def prune(self, keep: tuple[FragmentKey, ...]) -> bool:
|
||
|
|
"""Remove every stale entry and prove the retained inventory is bounded."""
|
||
|
|
|
||
|
|
keep_ids = {key.key_id for key in keep}
|
||
|
|
if len(keep_ids) > MAX_FRAGMENT_CACHE_ENTRIES:
|
||
|
|
return False
|
||
|
|
descriptor: int | None = None
|
||
|
|
try:
|
||
|
|
descriptor = self._open(create=False)
|
||
|
|
retained_entries = 0
|
||
|
|
retained_bytes = 0
|
||
|
|
with os.scandir(descriptor) as entries:
|
||
|
|
for entry in entries:
|
||
|
|
name = entry.name
|
||
|
|
retained = (
|
||
|
|
len(name) == 69
|
||
|
|
and name.endswith(".json")
|
||
|
|
and name[:-5] in keep_ids
|
||
|
|
and all(character in "0123456789abcdef" for character in name[:-5])
|
||
|
|
)
|
||
|
|
if retained:
|
||
|
|
identity = entry.stat(follow_symlinks=False)
|
||
|
|
if not entry.is_file(follow_symlinks=False):
|
||
|
|
return False
|
||
|
|
retained_entries += 1
|
||
|
|
retained_bytes += identity.st_size
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
os.unlink(name, dir_fd=descriptor)
|
||
|
|
except OSError:
|
||
|
|
return False
|
||
|
|
if (
|
||
|
|
retained_entries > MAX_FRAGMENT_CACHE_ENTRIES
|
||
|
|
or retained_bytes > MAX_FRAGMENT_CACHE_BYTES
|
||
|
|
):
|
||
|
|
return False
|
||
|
|
require_bound_directory(self.fragment_root, descriptor)
|
||
|
|
os.fsync(descriptor)
|
||
|
|
return True
|
||
|
|
except (DocForgeError, OSError):
|
||
|
|
return False
|
||
|
|
finally:
|
||
|
|
if descriptor is not None:
|
||
|
|
os.close(descriptor)
|
||
|
|
|
||
|
|
def _open(self, *, create: bool) -> int:
|
||
|
|
if self.cache_root == self.project_root or not self.cache_root.is_relative_to(
|
||
|
|
self.project_root
|
||
|
|
):
|
||
|
|
raise DocForgeError(
|
||
|
|
"path_escape",
|
||
|
|
"Projection fragment cache is not confined to a derived project root",
|
||
|
|
)
|
||
|
|
return open_confined_directory(
|
||
|
|
self.project_root,
|
||
|
|
self.fragment_root,
|
||
|
|
create=create,
|
||
|
|
)
|