feat: complete production workflow migration
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
"""Safe synchronization of independently evolving source projects."""
|
||||
|
||||
from .model import (
|
||||
ApplyResult,
|
||||
FileComparison,
|
||||
ManifestEntry,
|
||||
SourceProject,
|
||||
SyncState,
|
||||
)
|
||||
from .service import SourceSyncService
|
||||
|
||||
__all__ = [
|
||||
"ApplyResult",
|
||||
"FileComparison",
|
||||
"ManifestEntry",
|
||||
"SourceProject",
|
||||
"SourceSyncService",
|
||||
"SyncState",
|
||||
]
|
||||
@@ -0,0 +1,136 @@
|
||||
"""CLI boundary for safe, manifest-driven source synchronization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import TextIO
|
||||
|
||||
from .service import SourceSyncService
|
||||
|
||||
|
||||
def add_sources_parser(commands: argparse._SubParsersAction) -> None:
|
||||
"""Register source status and explicit safe-apply commands."""
|
||||
|
||||
sources_parser = commands.add_parser(
|
||||
"sources",
|
||||
help="compare and safely synchronize upstream source projects",
|
||||
)
|
||||
source_commands = sources_parser.add_subparsers(
|
||||
dest="sources_command",
|
||||
required=True,
|
||||
)
|
||||
for command, help_text in (
|
||||
("status", "show three-way source and target differences"),
|
||||
("apply", "copy safe source-only changes"),
|
||||
):
|
||||
parser = source_commands.add_parser(command, help=help_text)
|
||||
parser.add_argument("--project")
|
||||
parser.add_argument(
|
||||
"--source-root",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="MODULE=PATH",
|
||||
help="override one source root; repeat for multiple modules",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
if command == "apply":
|
||||
parser.add_argument(
|
||||
"--execute",
|
||||
action="store_true",
|
||||
help="perform safe copies; required for any write",
|
||||
)
|
||||
|
||||
|
||||
def run_sources_command(
|
||||
arguments: argparse.Namespace,
|
||||
*,
|
||||
project_root: Path,
|
||||
output: TextIO,
|
||||
) -> int:
|
||||
"""Execute one sources subcommand without constructing workflow services."""
|
||||
|
||||
source_roots = parse_source_roots(arguments.source_root)
|
||||
service = SourceSyncService(project_root, source_roots=source_roots)
|
||||
try:
|
||||
if arguments.sources_command == "status":
|
||||
comparisons = service.status(arguments.project)
|
||||
payload = _status_payload(comparisons)
|
||||
elif arguments.sources_command == "apply":
|
||||
if not arguments.execute:
|
||||
raise ValueError("sources apply requires explicit --execute")
|
||||
result = service.apply_safe(arguments.project)
|
||||
payload = {
|
||||
"applied": list(result.applied),
|
||||
"skipped": list(result.skipped),
|
||||
}
|
||||
else:
|
||||
raise ValueError(
|
||||
f"unsupported sources command: {arguments.sources_command}"
|
||||
)
|
||||
except KeyError as exc:
|
||||
raise ValueError(str(exc)) from exc
|
||||
|
||||
if arguments.json:
|
||||
output.write(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||
output.write("\n")
|
||||
elif arguments.sources_command == "status":
|
||||
for item in payload["files"]:
|
||||
output.write(
|
||||
f"{item['project']}\t{item['state']}\t"
|
||||
f"{item['source_relative_path']}\n"
|
||||
)
|
||||
else:
|
||||
output.write(
|
||||
f"sources: applied={len(payload['applied'])}; "
|
||||
f"skipped={len(payload['skipped'])}\n"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def parse_source_roots(values: Sequence[str]) -> dict[str, Path]:
|
||||
"""Parse repeatable ``MODULE=PATH`` arguments without silent overrides."""
|
||||
|
||||
roots: dict[str, Path] = {}
|
||||
for value in values:
|
||||
module, separator, raw_path = value.partition("=")
|
||||
if (
|
||||
not separator
|
||||
or not module
|
||||
or not raw_path
|
||||
or module in roots
|
||||
or "\x00" in value
|
||||
):
|
||||
raise ValueError(
|
||||
"source roots must be unique non-empty MODULE=PATH values"
|
||||
)
|
||||
roots[module] = Path(raw_path)
|
||||
return roots
|
||||
|
||||
|
||||
def _status_payload(comparisons) -> dict[str, object]: # type: ignore[no-untyped-def]
|
||||
summary = Counter(comparison.state.value for comparison in comparisons)
|
||||
return {
|
||||
"summary": dict(sorted(summary.items())),
|
||||
"files": [
|
||||
{
|
||||
"project": comparison.project,
|
||||
"source_relative_path": comparison.source_relative_path,
|
||||
"target_relative_path": comparison.target_relative_path,
|
||||
"transformed": comparison.transformed,
|
||||
"state": comparison.state.value,
|
||||
"safe_to_apply": comparison.safe_to_apply,
|
||||
}
|
||||
for comparison in comparisons
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"add_sources_parser",
|
||||
"parse_source_roots",
|
||||
"run_sources_command",
|
||||
]
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Value objects for safe, manifest-driven source synchronization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class SyncState(str, Enum):
|
||||
"""Three-way comparison states for one mapped source file."""
|
||||
|
||||
IN_SYNC = "in_sync"
|
||||
SOURCE_CHANGED = "source_changed"
|
||||
TARGET_CHANGED = "target_changed"
|
||||
CONFLICT = "conflict"
|
||||
SOURCE_MISSING = "source_missing"
|
||||
TARGET_MISSING = "target_missing"
|
||||
NEW_SOURCE = "new_source"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourceProject:
|
||||
"""Declarative connection between a module, manifest, and source root."""
|
||||
|
||||
module: str
|
||||
manifest: str
|
||||
root_env: str
|
||||
target_root: str
|
||||
source_project: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ManifestEntry:
|
||||
"""The synchronization fields read from a source manifest entry."""
|
||||
|
||||
source_relative_path: str
|
||||
target_relative_path: str
|
||||
source_sha256: str
|
||||
target_sha256: str
|
||||
transformed: bool
|
||||
category: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FileComparison:
|
||||
"""Current hashes and three-way state for one mapped file."""
|
||||
|
||||
project: str
|
||||
source_relative_path: str
|
||||
target_relative_path: str
|
||||
transformed: bool
|
||||
state: SyncState
|
||||
baseline_source_sha256: str
|
||||
baseline_target_sha256: str
|
||||
source_sha256: str | None
|
||||
target_sha256: str | None
|
||||
|
||||
@property
|
||||
def safe_to_apply(self) -> bool:
|
||||
"""Whether the entry can be copied without overwriting local work."""
|
||||
|
||||
return (
|
||||
self.state is SyncState.SOURCE_CHANGED
|
||||
and not self.transformed
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ApplyResult:
|
||||
"""Stable summary of an explicit safe-apply operation."""
|
||||
|
||||
applied: tuple[str, ...]
|
||||
skipped: tuple[str, ...]
|
||||
@@ -0,0 +1,561 @@
|
||||
"""Read-only-by-default three-way source synchronization service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from collections.abc import Mapping, Sequence
|
||||
from fnmatch import fnmatchcase
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from gyxx_flow.core.artifacts import sha256_file
|
||||
|
||||
from .model import (
|
||||
ApplyResult,
|
||||
FileComparison,
|
||||
ManifestEntry,
|
||||
SourceProject,
|
||||
SyncState,
|
||||
)
|
||||
|
||||
_SENSITIVE_NAMES = {
|
||||
".env",
|
||||
"cookie.json",
|
||||
"cookies.json",
|
||||
"credential.json",
|
||||
"credentials.json",
|
||||
"id_ed25519",
|
||||
"id_rsa",
|
||||
"storage_state.json",
|
||||
}
|
||||
_SENSITIVE_SUFFIXES = {".key", ".p12", ".pem", ".pfx"}
|
||||
_FORBIDDEN_PARTS = {".git", ".venv", "__pycache__"}
|
||||
_DEFAULT_EXCLUDED_PATTERNS = (
|
||||
".git/**",
|
||||
".venv/**",
|
||||
"venv/**",
|
||||
"**/__pycache__/**",
|
||||
"**/*.pyc",
|
||||
".pytest_cache/**",
|
||||
".mypy_cache/**",
|
||||
".ruff_cache/**",
|
||||
"**/logs/**",
|
||||
"**/*cookie*",
|
||||
"**/*profile*/**",
|
||||
)
|
||||
|
||||
|
||||
class SourceSyncService:
|
||||
"""Compare source projects with migrated targets and apply safe updates."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repository_root: Path | str,
|
||||
*,
|
||||
config_path: Path | str | None = None,
|
||||
source_roots: Mapping[str, Path | str] | None = None,
|
||||
environ: Mapping[str, str] | None = None,
|
||||
) -> None:
|
||||
self.repository_root = Path(repository_root).resolve()
|
||||
configured_path = (
|
||||
Path(config_path)
|
||||
if config_path is not None
|
||||
else self.repository_root / "config" / "source-projects.json"
|
||||
)
|
||||
if not configured_path.is_absolute():
|
||||
configured_path = self.repository_root / configured_path
|
||||
self.config_path = _path_within(
|
||||
self.repository_root,
|
||||
configured_path,
|
||||
label="source-projects config",
|
||||
)
|
||||
self._source_roots = {
|
||||
key: Path(value).resolve()
|
||||
for key, value in (source_roots or {}).items()
|
||||
}
|
||||
self._environ = dict(os.environ if environ is None else environ)
|
||||
self._projects = self._load_projects()
|
||||
|
||||
def status(self, project: str | None = None) -> tuple[FileComparison, ...]:
|
||||
"""Return current three-way state without modifying either side."""
|
||||
|
||||
comparisons: list[FileComparison] = []
|
||||
for specification in self._select_projects(project):
|
||||
manifest, entries, _ = self._load_manifest(specification)
|
||||
source_root = self._source_root(specification)
|
||||
for entry in entries:
|
||||
source_path = self._mapped_path(
|
||||
source_root,
|
||||
entry.source_relative_path,
|
||||
label="source",
|
||||
reject_sensitive=True,
|
||||
)
|
||||
target_path = self._mapped_path(
|
||||
self.repository_root,
|
||||
entry.target_relative_path,
|
||||
label="target",
|
||||
reject_sensitive=True,
|
||||
)
|
||||
source_digest = _optional_sha256(source_path)
|
||||
target_digest = _optional_sha256(target_path)
|
||||
comparisons.append(
|
||||
FileComparison(
|
||||
project=specification.module,
|
||||
source_relative_path=entry.source_relative_path,
|
||||
target_relative_path=entry.target_relative_path,
|
||||
transformed=entry.transformed,
|
||||
state=_classify(
|
||||
source_digest=source_digest,
|
||||
target_digest=target_digest,
|
||||
baseline_source=entry.source_sha256,
|
||||
baseline_target=entry.target_sha256,
|
||||
),
|
||||
baseline_source_sha256=entry.source_sha256,
|
||||
baseline_target_sha256=entry.target_sha256,
|
||||
source_sha256=source_digest,
|
||||
target_sha256=target_digest,
|
||||
)
|
||||
)
|
||||
mapped_sources = {entry.source_relative_path for entry in entries}
|
||||
exclusions = manifest.get("intentionally_excluded", [])
|
||||
if not isinstance(exclusions, list):
|
||||
raise ValueError("intentionally_excluded must be a list")
|
||||
for source_path in _iter_source_files(source_root, exclusions):
|
||||
relative_path = source_path.relative_to(source_root)
|
||||
source_relative_path = relative_path.as_posix()
|
||||
if (
|
||||
source_relative_path in mapped_sources
|
||||
or _is_sensitive(relative_path)
|
||||
or _is_excluded(source_relative_path, exclusions)
|
||||
):
|
||||
continue
|
||||
target_relative_path = (
|
||||
PurePosixPath(specification.target_root)
|
||||
/ PurePosixPath(source_relative_path)
|
||||
).as_posix()
|
||||
self._mapped_path(
|
||||
self.repository_root,
|
||||
target_relative_path,
|
||||
label="target",
|
||||
reject_sensitive=True,
|
||||
)
|
||||
comparisons.append(
|
||||
FileComparison(
|
||||
project=specification.module,
|
||||
source_relative_path=source_relative_path,
|
||||
target_relative_path=target_relative_path,
|
||||
transformed=True,
|
||||
state=SyncState.NEW_SOURCE,
|
||||
baseline_source_sha256="",
|
||||
baseline_target_sha256="",
|
||||
source_sha256=sha256_file(source_path),
|
||||
target_sha256=None,
|
||||
)
|
||||
)
|
||||
return tuple(comparisons)
|
||||
|
||||
def apply_safe(self, project: str | None = None) -> ApplyResult:
|
||||
"""Copy only untransformed source-only changes and refresh manifests."""
|
||||
|
||||
applied: list[str] = []
|
||||
skipped: list[str] = []
|
||||
target_backups: dict[Path, bytes | None] = {}
|
||||
manifest_backups: dict[Path, bytes] = {}
|
||||
by_project: dict[str, list[FileComparison]] = {}
|
||||
for comparison in self.status(project):
|
||||
by_project.setdefault(comparison.project, []).append(comparison)
|
||||
|
||||
try:
|
||||
for specification in self._select_projects(project):
|
||||
manifest, _, manifest_path = self._load_manifest(specification)
|
||||
source_root = self._source_root(specification)
|
||||
project_applied: list[str] = []
|
||||
entries_by_target = {
|
||||
_required_string(entry, "target_relative_path"): entry
|
||||
for entry in _required_list(manifest, "files")
|
||||
}
|
||||
for comparison in by_project.get(specification.module, []):
|
||||
if not comparison.safe_to_apply:
|
||||
skipped.append(comparison.target_relative_path)
|
||||
continue
|
||||
|
||||
source_path = self._mapped_path(
|
||||
source_root,
|
||||
comparison.source_relative_path,
|
||||
label="source",
|
||||
reject_sensitive=True,
|
||||
)
|
||||
target_path = self._mapped_path(
|
||||
self.repository_root,
|
||||
comparison.target_relative_path,
|
||||
label="target",
|
||||
reject_sensitive=True,
|
||||
)
|
||||
self._verify_unchanged(comparison, source_path, target_path)
|
||||
target_backups.setdefault(
|
||||
target_path,
|
||||
target_path.read_bytes() if target_path.exists() else None,
|
||||
)
|
||||
_atomic_copy(source_path, target_path, comparison.source_sha256)
|
||||
|
||||
entry = entries_by_target[comparison.target_relative_path]
|
||||
entry["source_sha256"] = comparison.source_sha256
|
||||
entry["target_sha256"] = comparison.source_sha256
|
||||
project_applied.append(comparison.target_relative_path)
|
||||
|
||||
if project_applied:
|
||||
manifest_backups.setdefault(
|
||||
manifest_path,
|
||||
manifest_path.read_bytes(),
|
||||
)
|
||||
_atomic_write_json(manifest_path, manifest)
|
||||
applied.extend(project_applied)
|
||||
except BaseException:
|
||||
for manifest_path, content in reversed(manifest_backups.items()):
|
||||
_atomic_write_bytes(manifest_path, content)
|
||||
for target_path, content in reversed(target_backups.items()):
|
||||
if content is None:
|
||||
target_path.unlink(missing_ok=True)
|
||||
else:
|
||||
_atomic_write_bytes(target_path, content)
|
||||
raise
|
||||
|
||||
return ApplyResult(applied=tuple(applied), skipped=tuple(skipped))
|
||||
|
||||
def _load_projects(self) -> tuple[SourceProject, ...]:
|
||||
payload = _read_json(self.config_path)
|
||||
if payload.get("schema_version") != 1:
|
||||
raise ValueError("source-projects config schema_version must be 1")
|
||||
projects: list[SourceProject] = []
|
||||
seen: set[str] = set()
|
||||
for raw_project in _required_list(payload, "projects"):
|
||||
module = _required_string(raw_project, "module")
|
||||
if module in seen:
|
||||
raise ValueError(f"duplicate source project module: {module}")
|
||||
seen.add(module)
|
||||
source_project = raw_project.get("source_project")
|
||||
if source_project is not None and not isinstance(source_project, str):
|
||||
raise ValueError("source_project must be a string when provided")
|
||||
projects.append(
|
||||
SourceProject(
|
||||
module=module,
|
||||
manifest=_required_string(raw_project, "manifest"),
|
||||
root_env=_required_string(raw_project, "root_env"),
|
||||
target_root=_required_string(raw_project, "target_root"),
|
||||
source_project=source_project,
|
||||
)
|
||||
)
|
||||
return tuple(projects)
|
||||
|
||||
def _select_projects(self, project: str | None) -> tuple[SourceProject, ...]:
|
||||
if project is None:
|
||||
return self._projects
|
||||
selected = tuple(item for item in self._projects if item.module == project)
|
||||
if not selected:
|
||||
raise KeyError(f"unknown source project: {project}")
|
||||
return selected
|
||||
|
||||
def _source_root(self, specification: SourceProject) -> Path:
|
||||
configured = self._source_roots.get(specification.module)
|
||||
if configured is None and specification.source_project is not None:
|
||||
configured = self._source_roots.get(specification.source_project)
|
||||
if configured is None:
|
||||
raw_root = self._environ.get(specification.root_env)
|
||||
if not raw_root:
|
||||
raise ValueError(
|
||||
f"source root for {specification.module!r} is not configured; "
|
||||
f"pass source_roots or set {specification.root_env}"
|
||||
)
|
||||
configured = Path(raw_root).resolve()
|
||||
if not configured.is_dir():
|
||||
raise ValueError(
|
||||
f"source root for {specification.module!r} is not a directory: "
|
||||
f"{configured}"
|
||||
)
|
||||
return configured
|
||||
|
||||
def _load_manifest(
|
||||
self,
|
||||
specification: SourceProject,
|
||||
) -> tuple[dict[str, Any], tuple[ManifestEntry, ...], Path]:
|
||||
manifest_path = self._mapped_path(
|
||||
self.repository_root,
|
||||
specification.manifest,
|
||||
label="manifest",
|
||||
reject_sensitive=False,
|
||||
)
|
||||
payload = _read_json(manifest_path)
|
||||
if payload.get("schema_version") != 1:
|
||||
raise ValueError(
|
||||
f"source manifest schema_version must be 1: {manifest_path}"
|
||||
)
|
||||
if payload.get("module") != specification.module:
|
||||
raise ValueError(
|
||||
f"manifest module does not match {specification.module!r}: "
|
||||
f"{manifest_path}"
|
||||
)
|
||||
entries: list[ManifestEntry] = []
|
||||
for raw_entry in _required_list(payload, "files"):
|
||||
transformed = raw_entry.get("transformed")
|
||||
if not isinstance(transformed, bool):
|
||||
raise ValueError("manifest transformed must be a boolean")
|
||||
entries.append(
|
||||
ManifestEntry(
|
||||
source_relative_path=_required_string(
|
||||
raw_entry,
|
||||
"source_relative_path",
|
||||
),
|
||||
target_relative_path=_required_string(
|
||||
raw_entry,
|
||||
"target_relative_path",
|
||||
),
|
||||
source_sha256=_required_digest(raw_entry, "source_sha256"),
|
||||
target_sha256=_required_digest(raw_entry, "target_sha256"),
|
||||
transformed=transformed,
|
||||
category=_required_string(raw_entry, "category"),
|
||||
)
|
||||
)
|
||||
return payload, tuple(entries), manifest_path
|
||||
|
||||
@staticmethod
|
||||
def _mapped_path(
|
||||
root: Path,
|
||||
relative_path: str,
|
||||
*,
|
||||
label: str,
|
||||
reject_sensitive: bool,
|
||||
) -> Path:
|
||||
relative = Path(relative_path)
|
||||
if (
|
||||
not relative_path
|
||||
or relative.is_absolute()
|
||||
or ".." in relative.parts
|
||||
):
|
||||
raise ValueError(f"unsafe relative path for {label}: {relative_path!r}")
|
||||
if reject_sensitive and _is_sensitive(relative):
|
||||
raise ValueError(f"sensitive {label} file is not allowed: {relative_path!r}")
|
||||
return _path_within(root, root / relative, label=label)
|
||||
|
||||
@staticmethod
|
||||
def _verify_unchanged(
|
||||
comparison: FileComparison,
|
||||
source_path: Path,
|
||||
target_path: Path,
|
||||
) -> None:
|
||||
current_source = _optional_sha256(source_path)
|
||||
current_target = _optional_sha256(target_path)
|
||||
if (
|
||||
current_source != comparison.source_sha256
|
||||
or current_target != comparison.target_sha256
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"source or target changed during synchronization: "
|
||||
f"{comparison.target_relative_path}"
|
||||
)
|
||||
|
||||
|
||||
def _classify(
|
||||
*,
|
||||
source_digest: str | None,
|
||||
target_digest: str | None,
|
||||
baseline_source: str,
|
||||
baseline_target: str,
|
||||
) -> SyncState:
|
||||
if source_digest is None:
|
||||
return SyncState.SOURCE_MISSING
|
||||
if target_digest is None:
|
||||
return SyncState.TARGET_MISSING
|
||||
source_changed = source_digest != baseline_source
|
||||
target_changed = target_digest != baseline_target
|
||||
if source_changed and target_changed:
|
||||
return SyncState.CONFLICT
|
||||
if source_changed:
|
||||
return SyncState.SOURCE_CHANGED
|
||||
if target_changed:
|
||||
return SyncState.TARGET_CHANGED
|
||||
return SyncState.IN_SYNC
|
||||
|
||||
|
||||
def _optional_sha256(path: Path) -> str | None:
|
||||
if not path.exists():
|
||||
return None
|
||||
if not path.is_file():
|
||||
raise ValueError(f"mapped path is not a regular file: {path}")
|
||||
return sha256_file(path)
|
||||
|
||||
|
||||
def _path_within(root: Path, path: Path, *, label: str) -> Path:
|
||||
resolved_root = root.resolve()
|
||||
resolved_path = path.resolve()
|
||||
try:
|
||||
resolved_path.relative_to(resolved_root)
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{label} path escapes its root: {path}") from error
|
||||
return resolved_path
|
||||
|
||||
|
||||
def _is_sensitive(path: Path) -> bool:
|
||||
lowered_parts = tuple(part.lower() for part in path.parts)
|
||||
name = path.name.lower()
|
||||
return (
|
||||
name in _SENSITIVE_NAMES
|
||||
or path.suffix.lower() in _SENSITIVE_SUFFIXES
|
||||
or any(part in _FORBIDDEN_PARTS for part in lowered_parts)
|
||||
)
|
||||
|
||||
|
||||
def _is_excluded(
|
||||
source_relative_path: str,
|
||||
exclusions: Sequence[object],
|
||||
) -> bool:
|
||||
candidate = source_relative_path.casefold()
|
||||
patterns = list(_DEFAULT_EXCLUDED_PATTERNS)
|
||||
explicit_paths: set[str] = set()
|
||||
for exclusion in exclusions:
|
||||
if not isinstance(exclusion, dict):
|
||||
raise ValueError("intentionally_excluded entries must be objects")
|
||||
pattern = exclusion.get("pattern")
|
||||
explicit_path = exclusion.get("source_relative_path")
|
||||
if pattern is not None:
|
||||
if not isinstance(pattern, str) or not pattern:
|
||||
raise ValueError("exclusion pattern must be a non-empty string")
|
||||
patterns.append(pattern)
|
||||
if explicit_path is not None:
|
||||
if not isinstance(explicit_path, str) or not explicit_path:
|
||||
raise ValueError(
|
||||
"excluded source_relative_path must be a non-empty string"
|
||||
)
|
||||
explicit_paths.add(PurePosixPath(explicit_path).as_posix().casefold())
|
||||
normalized_patterns: list[str] = []
|
||||
for pattern in patterns:
|
||||
normalized = pattern.casefold()
|
||||
normalized_patterns.append(normalized)
|
||||
if normalized.startswith("**/"):
|
||||
normalized_patterns.append(normalized[3:])
|
||||
return candidate in explicit_paths or any(
|
||||
fnmatchcase(candidate, pattern) for pattern in normalized_patterns
|
||||
)
|
||||
|
||||
|
||||
def _iter_source_files(
|
||||
source_root: Path,
|
||||
exclusions: Sequence[object],
|
||||
) -> tuple[Path, ...]:
|
||||
"""Enumerate source candidates without descending into excluded data trees."""
|
||||
|
||||
candidates: list[Path] = []
|
||||
for raw_root, directory_names, file_names in os.walk(source_root, topdown=True):
|
||||
current_root = Path(raw_root)
|
||||
retained_directories: list[str] = []
|
||||
for directory_name in directory_names:
|
||||
directory_path = current_root / directory_name
|
||||
relative = directory_path.relative_to(source_root)
|
||||
relative_posix = relative.as_posix()
|
||||
if _is_sensitive(relative):
|
||||
continue
|
||||
if _is_excluded(f"{relative_posix}/__gyxx_probe__", exclusions):
|
||||
continue
|
||||
retained_directories.append(directory_name)
|
||||
directory_names[:] = retained_directories
|
||||
|
||||
for file_name in file_names:
|
||||
source_path = current_root / file_name
|
||||
relative = source_path.relative_to(source_root)
|
||||
relative_posix = relative.as_posix()
|
||||
if _is_sensitive(relative) or _is_excluded(relative_posix, exclusions):
|
||||
continue
|
||||
candidates.append(source_path)
|
||||
return tuple(sorted(candidates))
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8-sig") as handle:
|
||||
payload = json.load(handle)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"expected a JSON object: {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def _required_list(
|
||||
payload: Mapping[str, Any],
|
||||
field: str,
|
||||
) -> Sequence[dict[str, Any]]:
|
||||
value = payload.get(field)
|
||||
if not isinstance(value, list) or any(
|
||||
not isinstance(item, dict) for item in value
|
||||
):
|
||||
raise ValueError(f"{field} must be a list of objects")
|
||||
return value
|
||||
|
||||
|
||||
def _required_string(payload: Mapping[str, Any], field: str) -> str:
|
||||
value = payload.get(field)
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ValueError(f"{field} must be a non-empty string")
|
||||
return value
|
||||
|
||||
|
||||
def _required_digest(payload: Mapping[str, Any], field: str) -> str:
|
||||
value = _required_string(payload, field)
|
||||
if len(value) != 64 or any(character not in "0123456789abcdef" for character in value):
|
||||
raise ValueError(f"{field} must be a lowercase SHA-256 digest")
|
||||
return value
|
||||
|
||||
|
||||
def _atomic_copy(source: Path, target: Path, expected_sha256: str | None) -> None:
|
||||
if expected_sha256 is None:
|
||||
raise RuntimeError(f"source disappeared during synchronization: {source}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{target.name}.",
|
||||
suffix=".tmp",
|
||||
dir=target.parent,
|
||||
)
|
||||
os.close(descriptor)
|
||||
temporary = Path(temporary_name)
|
||||
try:
|
||||
shutil.copy2(source, temporary)
|
||||
if sha256_file(temporary) != expected_sha256:
|
||||
raise RuntimeError(f"source changed while it was being copied: {source}")
|
||||
os.replace(temporary, target)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None:
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
dir=path.parent,
|
||||
)
|
||||
temporary = Path(temporary_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _atomic_write_bytes(path: Path, payload: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
dir=path.parent,
|
||||
)
|
||||
temporary = Path(temporary_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
Reference in New Issue
Block a user