562 lines
21 KiB
Python
562 lines
21 KiB
Python
"""Read-only-by-default three-way source synchronization service."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
from collections.abc import Mapping, Sequence
|
|
from fnmatch import fnmatchcase
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any
|
|
|
|
from gyxx_flow.core.artifacts import sha256_file
|
|
|
|
from .model import (
|
|
ApplyResult,
|
|
FileComparison,
|
|
ManifestEntry,
|
|
SourceProject,
|
|
SyncState,
|
|
)
|
|
|
|
_SENSITIVE_NAMES = {
|
|
".env",
|
|
"cookie.json",
|
|
"cookies.json",
|
|
"credential.json",
|
|
"credentials.json",
|
|
"id_ed25519",
|
|
"id_rsa",
|
|
"storage_state.json",
|
|
}
|
|
_SENSITIVE_SUFFIXES = {".key", ".p12", ".pem", ".pfx"}
|
|
_FORBIDDEN_PARTS = {".git", ".venv", "__pycache__"}
|
|
_DEFAULT_EXCLUDED_PATTERNS = (
|
|
".git/**",
|
|
".venv/**",
|
|
"venv/**",
|
|
"**/__pycache__/**",
|
|
"**/*.pyc",
|
|
".pytest_cache/**",
|
|
".mypy_cache/**",
|
|
".ruff_cache/**",
|
|
"**/logs/**",
|
|
"**/*cookie*",
|
|
"**/*profile*/**",
|
|
)
|
|
|
|
|
|
class SourceSyncService:
|
|
"""Compare source projects with migrated targets and apply safe updates."""
|
|
|
|
def __init__(
|
|
self,
|
|
repository_root: Path | str,
|
|
*,
|
|
config_path: Path | str | None = None,
|
|
source_roots: Mapping[str, Path | str] | None = None,
|
|
environ: Mapping[str, str] | None = None,
|
|
) -> None:
|
|
self.repository_root = Path(repository_root).resolve()
|
|
configured_path = (
|
|
Path(config_path)
|
|
if config_path is not None
|
|
else self.repository_root / "config" / "source-projects.json"
|
|
)
|
|
if not configured_path.is_absolute():
|
|
configured_path = self.repository_root / configured_path
|
|
self.config_path = _path_within(
|
|
self.repository_root,
|
|
configured_path,
|
|
label="source-projects config",
|
|
)
|
|
self._source_roots = {
|
|
key: Path(value).resolve()
|
|
for key, value in (source_roots or {}).items()
|
|
}
|
|
self._environ = dict(os.environ if environ is None else environ)
|
|
self._projects = self._load_projects()
|
|
|
|
def status(self, project: str | None = None) -> tuple[FileComparison, ...]:
|
|
"""Return current three-way state without modifying either side."""
|
|
|
|
comparisons: list[FileComparison] = []
|
|
for specification in self._select_projects(project):
|
|
manifest, entries, _ = self._load_manifest(specification)
|
|
source_root = self._source_root(specification)
|
|
for entry in entries:
|
|
source_path = self._mapped_path(
|
|
source_root,
|
|
entry.source_relative_path,
|
|
label="source",
|
|
reject_sensitive=True,
|
|
)
|
|
target_path = self._mapped_path(
|
|
self.repository_root,
|
|
entry.target_relative_path,
|
|
label="target",
|
|
reject_sensitive=True,
|
|
)
|
|
source_digest = _optional_sha256(source_path)
|
|
target_digest = _optional_sha256(target_path)
|
|
comparisons.append(
|
|
FileComparison(
|
|
project=specification.module,
|
|
source_relative_path=entry.source_relative_path,
|
|
target_relative_path=entry.target_relative_path,
|
|
transformed=entry.transformed,
|
|
state=_classify(
|
|
source_digest=source_digest,
|
|
target_digest=target_digest,
|
|
baseline_source=entry.source_sha256,
|
|
baseline_target=entry.target_sha256,
|
|
),
|
|
baseline_source_sha256=entry.source_sha256,
|
|
baseline_target_sha256=entry.target_sha256,
|
|
source_sha256=source_digest,
|
|
target_sha256=target_digest,
|
|
)
|
|
)
|
|
mapped_sources = {entry.source_relative_path for entry in entries}
|
|
exclusions = manifest.get("intentionally_excluded", [])
|
|
if not isinstance(exclusions, list):
|
|
raise ValueError("intentionally_excluded must be a list")
|
|
for source_path in _iter_source_files(source_root, exclusions):
|
|
relative_path = source_path.relative_to(source_root)
|
|
source_relative_path = relative_path.as_posix()
|
|
if (
|
|
source_relative_path in mapped_sources
|
|
or _is_sensitive(relative_path)
|
|
or _is_excluded(source_relative_path, exclusions)
|
|
):
|
|
continue
|
|
target_relative_path = (
|
|
PurePosixPath(specification.target_root)
|
|
/ PurePosixPath(source_relative_path)
|
|
).as_posix()
|
|
self._mapped_path(
|
|
self.repository_root,
|
|
target_relative_path,
|
|
label="target",
|
|
reject_sensitive=True,
|
|
)
|
|
comparisons.append(
|
|
FileComparison(
|
|
project=specification.module,
|
|
source_relative_path=source_relative_path,
|
|
target_relative_path=target_relative_path,
|
|
transformed=True,
|
|
state=SyncState.NEW_SOURCE,
|
|
baseline_source_sha256="",
|
|
baseline_target_sha256="",
|
|
source_sha256=sha256_file(source_path),
|
|
target_sha256=None,
|
|
)
|
|
)
|
|
return tuple(comparisons)
|
|
|
|
def apply_safe(self, project: str | None = None) -> ApplyResult:
|
|
"""Copy only untransformed source-only changes and refresh manifests."""
|
|
|
|
applied: list[str] = []
|
|
skipped: list[str] = []
|
|
target_backups: dict[Path, bytes | None] = {}
|
|
manifest_backups: dict[Path, bytes] = {}
|
|
by_project: dict[str, list[FileComparison]] = {}
|
|
for comparison in self.status(project):
|
|
by_project.setdefault(comparison.project, []).append(comparison)
|
|
|
|
try:
|
|
for specification in self._select_projects(project):
|
|
manifest, _, manifest_path = self._load_manifest(specification)
|
|
source_root = self._source_root(specification)
|
|
project_applied: list[str] = []
|
|
entries_by_target = {
|
|
_required_string(entry, "target_relative_path"): entry
|
|
for entry in _required_list(manifest, "files")
|
|
}
|
|
for comparison in by_project.get(specification.module, []):
|
|
if not comparison.safe_to_apply:
|
|
skipped.append(comparison.target_relative_path)
|
|
continue
|
|
|
|
source_path = self._mapped_path(
|
|
source_root,
|
|
comparison.source_relative_path,
|
|
label="source",
|
|
reject_sensitive=True,
|
|
)
|
|
target_path = self._mapped_path(
|
|
self.repository_root,
|
|
comparison.target_relative_path,
|
|
label="target",
|
|
reject_sensitive=True,
|
|
)
|
|
self._verify_unchanged(comparison, source_path, target_path)
|
|
target_backups.setdefault(
|
|
target_path,
|
|
target_path.read_bytes() if target_path.exists() else None,
|
|
)
|
|
_atomic_copy(source_path, target_path, comparison.source_sha256)
|
|
|
|
entry = entries_by_target[comparison.target_relative_path]
|
|
entry["source_sha256"] = comparison.source_sha256
|
|
entry["target_sha256"] = comparison.source_sha256
|
|
project_applied.append(comparison.target_relative_path)
|
|
|
|
if project_applied:
|
|
manifest_backups.setdefault(
|
|
manifest_path,
|
|
manifest_path.read_bytes(),
|
|
)
|
|
_atomic_write_json(manifest_path, manifest)
|
|
applied.extend(project_applied)
|
|
except BaseException:
|
|
for manifest_path, content in reversed(manifest_backups.items()):
|
|
_atomic_write_bytes(manifest_path, content)
|
|
for target_path, content in reversed(target_backups.items()):
|
|
if content is None:
|
|
target_path.unlink(missing_ok=True)
|
|
else:
|
|
_atomic_write_bytes(target_path, content)
|
|
raise
|
|
|
|
return ApplyResult(applied=tuple(applied), skipped=tuple(skipped))
|
|
|
|
def _load_projects(self) -> tuple[SourceProject, ...]:
|
|
payload = _read_json(self.config_path)
|
|
if payload.get("schema_version") != 1:
|
|
raise ValueError("source-projects config schema_version must be 1")
|
|
projects: list[SourceProject] = []
|
|
seen: set[str] = set()
|
|
for raw_project in _required_list(payload, "projects"):
|
|
module = _required_string(raw_project, "module")
|
|
if module in seen:
|
|
raise ValueError(f"duplicate source project module: {module}")
|
|
seen.add(module)
|
|
source_project = raw_project.get("source_project")
|
|
if source_project is not None and not isinstance(source_project, str):
|
|
raise ValueError("source_project must be a string when provided")
|
|
projects.append(
|
|
SourceProject(
|
|
module=module,
|
|
manifest=_required_string(raw_project, "manifest"),
|
|
root_env=_required_string(raw_project, "root_env"),
|
|
target_root=_required_string(raw_project, "target_root"),
|
|
source_project=source_project,
|
|
)
|
|
)
|
|
return tuple(projects)
|
|
|
|
def _select_projects(self, project: str | None) -> tuple[SourceProject, ...]:
|
|
if project is None:
|
|
return self._projects
|
|
selected = tuple(item for item in self._projects if item.module == project)
|
|
if not selected:
|
|
raise KeyError(f"unknown source project: {project}")
|
|
return selected
|
|
|
|
def _source_root(self, specification: SourceProject) -> Path:
|
|
configured = self._source_roots.get(specification.module)
|
|
if configured is None and specification.source_project is not None:
|
|
configured = self._source_roots.get(specification.source_project)
|
|
if configured is None:
|
|
raw_root = self._environ.get(specification.root_env)
|
|
if not raw_root:
|
|
raise ValueError(
|
|
f"source root for {specification.module!r} is not configured; "
|
|
f"pass source_roots or set {specification.root_env}"
|
|
)
|
|
configured = Path(raw_root).resolve()
|
|
if not configured.is_dir():
|
|
raise ValueError(
|
|
f"source root for {specification.module!r} is not a directory: "
|
|
f"{configured}"
|
|
)
|
|
return configured
|
|
|
|
def _load_manifest(
|
|
self,
|
|
specification: SourceProject,
|
|
) -> tuple[dict[str, Any], tuple[ManifestEntry, ...], Path]:
|
|
manifest_path = self._mapped_path(
|
|
self.repository_root,
|
|
specification.manifest,
|
|
label="manifest",
|
|
reject_sensitive=False,
|
|
)
|
|
payload = _read_json(manifest_path)
|
|
if payload.get("schema_version") != 1:
|
|
raise ValueError(
|
|
f"source manifest schema_version must be 1: {manifest_path}"
|
|
)
|
|
if payload.get("module") != specification.module:
|
|
raise ValueError(
|
|
f"manifest module does not match {specification.module!r}: "
|
|
f"{manifest_path}"
|
|
)
|
|
entries: list[ManifestEntry] = []
|
|
for raw_entry in _required_list(payload, "files"):
|
|
transformed = raw_entry.get("transformed")
|
|
if not isinstance(transformed, bool):
|
|
raise ValueError("manifest transformed must be a boolean")
|
|
entries.append(
|
|
ManifestEntry(
|
|
source_relative_path=_required_string(
|
|
raw_entry,
|
|
"source_relative_path",
|
|
),
|
|
target_relative_path=_required_string(
|
|
raw_entry,
|
|
"target_relative_path",
|
|
),
|
|
source_sha256=_required_digest(raw_entry, "source_sha256"),
|
|
target_sha256=_required_digest(raw_entry, "target_sha256"),
|
|
transformed=transformed,
|
|
category=_required_string(raw_entry, "category"),
|
|
)
|
|
)
|
|
return payload, tuple(entries), manifest_path
|
|
|
|
@staticmethod
|
|
def _mapped_path(
|
|
root: Path,
|
|
relative_path: str,
|
|
*,
|
|
label: str,
|
|
reject_sensitive: bool,
|
|
) -> Path:
|
|
relative = Path(relative_path)
|
|
if (
|
|
not relative_path
|
|
or relative.is_absolute()
|
|
or ".." in relative.parts
|
|
):
|
|
raise ValueError(f"unsafe relative path for {label}: {relative_path!r}")
|
|
if reject_sensitive and _is_sensitive(relative):
|
|
raise ValueError(f"sensitive {label} file is not allowed: {relative_path!r}")
|
|
return _path_within(root, root / relative, label=label)
|
|
|
|
@staticmethod
|
|
def _verify_unchanged(
|
|
comparison: FileComparison,
|
|
source_path: Path,
|
|
target_path: Path,
|
|
) -> None:
|
|
current_source = _optional_sha256(source_path)
|
|
current_target = _optional_sha256(target_path)
|
|
if (
|
|
current_source != comparison.source_sha256
|
|
or current_target != comparison.target_sha256
|
|
):
|
|
raise RuntimeError(
|
|
f"source or target changed during synchronization: "
|
|
f"{comparison.target_relative_path}"
|
|
)
|
|
|
|
|
|
def _classify(
|
|
*,
|
|
source_digest: str | None,
|
|
target_digest: str | None,
|
|
baseline_source: str,
|
|
baseline_target: str,
|
|
) -> SyncState:
|
|
if source_digest is None:
|
|
return SyncState.SOURCE_MISSING
|
|
if target_digest is None:
|
|
return SyncState.TARGET_MISSING
|
|
source_changed = source_digest != baseline_source
|
|
target_changed = target_digest != baseline_target
|
|
if source_changed and target_changed:
|
|
return SyncState.CONFLICT
|
|
if source_changed:
|
|
return SyncState.SOURCE_CHANGED
|
|
if target_changed:
|
|
return SyncState.TARGET_CHANGED
|
|
return SyncState.IN_SYNC
|
|
|
|
|
|
def _optional_sha256(path: Path) -> str | None:
|
|
if not path.exists():
|
|
return None
|
|
if not path.is_file():
|
|
raise ValueError(f"mapped path is not a regular file: {path}")
|
|
return sha256_file(path)
|
|
|
|
|
|
def _path_within(root: Path, path: Path, *, label: str) -> Path:
|
|
resolved_root = root.resolve()
|
|
resolved_path = path.resolve()
|
|
try:
|
|
resolved_path.relative_to(resolved_root)
|
|
except ValueError as error:
|
|
raise ValueError(f"{label} path escapes its root: {path}") from error
|
|
return resolved_path
|
|
|
|
|
|
def _is_sensitive(path: Path) -> bool:
|
|
lowered_parts = tuple(part.lower() for part in path.parts)
|
|
name = path.name.lower()
|
|
return (
|
|
name in _SENSITIVE_NAMES
|
|
or path.suffix.lower() in _SENSITIVE_SUFFIXES
|
|
or any(part in _FORBIDDEN_PARTS for part in lowered_parts)
|
|
)
|
|
|
|
|
|
def _is_excluded(
|
|
source_relative_path: str,
|
|
exclusions: Sequence[object],
|
|
) -> bool:
|
|
candidate = source_relative_path.casefold()
|
|
patterns = list(_DEFAULT_EXCLUDED_PATTERNS)
|
|
explicit_paths: set[str] = set()
|
|
for exclusion in exclusions:
|
|
if not isinstance(exclusion, dict):
|
|
raise ValueError("intentionally_excluded entries must be objects")
|
|
pattern = exclusion.get("pattern")
|
|
explicit_path = exclusion.get("source_relative_path")
|
|
if pattern is not None:
|
|
if not isinstance(pattern, str) or not pattern:
|
|
raise ValueError("exclusion pattern must be a non-empty string")
|
|
patterns.append(pattern)
|
|
if explicit_path is not None:
|
|
if not isinstance(explicit_path, str) or not explicit_path:
|
|
raise ValueError(
|
|
"excluded source_relative_path must be a non-empty string"
|
|
)
|
|
explicit_paths.add(PurePosixPath(explicit_path).as_posix().casefold())
|
|
normalized_patterns: list[str] = []
|
|
for pattern in patterns:
|
|
normalized = pattern.casefold()
|
|
normalized_patterns.append(normalized)
|
|
if normalized.startswith("**/"):
|
|
normalized_patterns.append(normalized[3:])
|
|
return candidate in explicit_paths or any(
|
|
fnmatchcase(candidate, pattern) for pattern in normalized_patterns
|
|
)
|
|
|
|
|
|
def _iter_source_files(
|
|
source_root: Path,
|
|
exclusions: Sequence[object],
|
|
) -> tuple[Path, ...]:
|
|
"""Enumerate source candidates without descending into excluded data trees."""
|
|
|
|
candidates: list[Path] = []
|
|
for raw_root, directory_names, file_names in os.walk(source_root, topdown=True):
|
|
current_root = Path(raw_root)
|
|
retained_directories: list[str] = []
|
|
for directory_name in directory_names:
|
|
directory_path = current_root / directory_name
|
|
relative = directory_path.relative_to(source_root)
|
|
relative_posix = relative.as_posix()
|
|
if _is_sensitive(relative):
|
|
continue
|
|
if _is_excluded(f"{relative_posix}/__gyxx_probe__", exclusions):
|
|
continue
|
|
retained_directories.append(directory_name)
|
|
directory_names[:] = retained_directories
|
|
|
|
for file_name in file_names:
|
|
source_path = current_root / file_name
|
|
relative = source_path.relative_to(source_root)
|
|
relative_posix = relative.as_posix()
|
|
if _is_sensitive(relative) or _is_excluded(relative_posix, exclusions):
|
|
continue
|
|
candidates.append(source_path)
|
|
return tuple(sorted(candidates))
|
|
|
|
|
|
def _read_json(path: Path) -> dict[str, Any]:
|
|
with path.open("r", encoding="utf-8-sig") as handle:
|
|
payload = json.load(handle)
|
|
if not isinstance(payload, dict):
|
|
raise ValueError(f"expected a JSON object: {path}")
|
|
return payload
|
|
|
|
|
|
def _required_list(
|
|
payload: Mapping[str, Any],
|
|
field: str,
|
|
) -> Sequence[dict[str, Any]]:
|
|
value = payload.get(field)
|
|
if not isinstance(value, list) or any(
|
|
not isinstance(item, dict) for item in value
|
|
):
|
|
raise ValueError(f"{field} must be a list of objects")
|
|
return value
|
|
|
|
|
|
def _required_string(payload: Mapping[str, Any], field: str) -> str:
|
|
value = payload.get(field)
|
|
if not isinstance(value, str) or not value:
|
|
raise ValueError(f"{field} must be a non-empty string")
|
|
return value
|
|
|
|
|
|
def _required_digest(payload: Mapping[str, Any], field: str) -> str:
|
|
value = _required_string(payload, field)
|
|
if len(value) != 64 or any(character not in "0123456789abcdef" for character in value):
|
|
raise ValueError(f"{field} must be a lowercase SHA-256 digest")
|
|
return value
|
|
|
|
|
|
def _atomic_copy(source: Path, target: Path, expected_sha256: str | None) -> None:
|
|
if expected_sha256 is None:
|
|
raise RuntimeError(f"source disappeared during synchronization: {source}")
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
descriptor, temporary_name = tempfile.mkstemp(
|
|
prefix=f".{target.name}.",
|
|
suffix=".tmp",
|
|
dir=target.parent,
|
|
)
|
|
os.close(descriptor)
|
|
temporary = Path(temporary_name)
|
|
try:
|
|
shutil.copy2(source, temporary)
|
|
if sha256_file(temporary) != expected_sha256:
|
|
raise RuntimeError(f"source changed while it was being copied: {source}")
|
|
os.replace(temporary, target)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None:
|
|
descriptor, temporary_name = tempfile.mkstemp(
|
|
prefix=f".{path.name}.",
|
|
suffix=".tmp",
|
|
dir=path.parent,
|
|
)
|
|
temporary = Path(temporary_name)
|
|
try:
|
|
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle:
|
|
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
|
handle.write("\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, path)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def _atomic_write_bytes(path: Path, payload: bytes) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
descriptor, temporary_name = tempfile.mkstemp(
|
|
prefix=f".{path.name}.",
|
|
suffix=".tmp",
|
|
dir=path.parent,
|
|
)
|
|
temporary = Path(temporary_name)
|
|
try:
|
|
with os.fdopen(descriptor, "wb") as handle:
|
|
handle.write(payload)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, path)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|