"""Read-only-by-default three-way source synchronization service.""" from __future__ import annotations import json import os import shutil import tempfile from collections.abc import Mapping, Sequence from fnmatch import fnmatchcase from pathlib import Path, PurePosixPath from typing import Any from gyxx_flow.core.artifacts import sha256_file from .model import ( ApplyResult, FileComparison, ManifestEntry, SourceProject, SyncState, ) _SENSITIVE_NAMES = { ".env", "cookie.json", "cookies.json", "credential.json", "credentials.json", "id_ed25519", "id_rsa", "storage_state.json", } _SENSITIVE_SUFFIXES = {".key", ".p12", ".pem", ".pfx"} _FORBIDDEN_PARTS = {".git", ".venv", "__pycache__"} _DEFAULT_EXCLUDED_PATTERNS = ( ".git/**", ".venv/**", "venv/**", "**/__pycache__/**", "**/*.pyc", ".pytest_cache/**", ".mypy_cache/**", ".ruff_cache/**", "**/logs/**", "**/*cookie*", "**/*profile*/**", ) class SourceSyncService: """Compare source projects with migrated targets and apply safe updates.""" def __init__( self, repository_root: Path | str, *, config_path: Path | str | None = None, source_roots: Mapping[str, Path | str] | None = None, environ: Mapping[str, str] | None = None, ) -> None: self.repository_root = Path(repository_root).resolve() configured_path = ( Path(config_path) if config_path is not None else self.repository_root / "config" / "source-projects.json" ) if not configured_path.is_absolute(): configured_path = self.repository_root / configured_path self.config_path = _path_within( self.repository_root, configured_path, label="source-projects config", ) self._source_roots = { key: Path(value).resolve() for key, value in (source_roots or {}).items() } self._environ = dict(os.environ if environ is None else environ) self._projects = self._load_projects() def status(self, project: str | None = None) -> tuple[FileComparison, ...]: """Return current three-way state without modifying either side.""" comparisons: list[FileComparison] = [] for specification in self._select_projects(project): manifest, entries, _ = self._load_manifest(specification) source_root = self._source_root(specification) for entry in entries: source_path = self._mapped_path( source_root, entry.source_relative_path, label="source", reject_sensitive=True, ) target_path = self._mapped_path( self.repository_root, entry.target_relative_path, label="target", reject_sensitive=True, ) source_digest = _optional_sha256(source_path) target_digest = _optional_sha256(target_path) comparisons.append( FileComparison( project=specification.module, source_relative_path=entry.source_relative_path, target_relative_path=entry.target_relative_path, transformed=entry.transformed, state=_classify( source_digest=source_digest, target_digest=target_digest, baseline_source=entry.source_sha256, baseline_target=entry.target_sha256, ), baseline_source_sha256=entry.source_sha256, baseline_target_sha256=entry.target_sha256, source_sha256=source_digest, target_sha256=target_digest, ) ) mapped_sources = {entry.source_relative_path for entry in entries} exclusions = manifest.get("intentionally_excluded", []) if not isinstance(exclusions, list): raise ValueError("intentionally_excluded must be a list") for source_path in _iter_source_files(source_root, exclusions): relative_path = source_path.relative_to(source_root) source_relative_path = relative_path.as_posix() if ( source_relative_path in mapped_sources or _is_sensitive(relative_path) or _is_excluded(source_relative_path, exclusions) ): continue target_relative_path = ( PurePosixPath(specification.target_root) / PurePosixPath(source_relative_path) ).as_posix() self._mapped_path( self.repository_root, target_relative_path, label="target", reject_sensitive=True, ) comparisons.append( FileComparison( project=specification.module, source_relative_path=source_relative_path, target_relative_path=target_relative_path, transformed=True, state=SyncState.NEW_SOURCE, baseline_source_sha256="", baseline_target_sha256="", source_sha256=sha256_file(source_path), target_sha256=None, ) ) return tuple(comparisons) def apply_safe(self, project: str | None = None) -> ApplyResult: """Copy only untransformed source-only changes and refresh manifests.""" applied: list[str] = [] skipped: list[str] = [] target_backups: dict[Path, bytes | None] = {} manifest_backups: dict[Path, bytes] = {} by_project: dict[str, list[FileComparison]] = {} for comparison in self.status(project): by_project.setdefault(comparison.project, []).append(comparison) try: for specification in self._select_projects(project): manifest, _, manifest_path = self._load_manifest(specification) source_root = self._source_root(specification) project_applied: list[str] = [] entries_by_target = { _required_string(entry, "target_relative_path"): entry for entry in _required_list(manifest, "files") } for comparison in by_project.get(specification.module, []): if not comparison.safe_to_apply: skipped.append(comparison.target_relative_path) continue source_path = self._mapped_path( source_root, comparison.source_relative_path, label="source", reject_sensitive=True, ) target_path = self._mapped_path( self.repository_root, comparison.target_relative_path, label="target", reject_sensitive=True, ) self._verify_unchanged(comparison, source_path, target_path) target_backups.setdefault( target_path, target_path.read_bytes() if target_path.exists() else None, ) _atomic_copy(source_path, target_path, comparison.source_sha256) entry = entries_by_target[comparison.target_relative_path] entry["source_sha256"] = comparison.source_sha256 entry["target_sha256"] = comparison.source_sha256 project_applied.append(comparison.target_relative_path) if project_applied: manifest_backups.setdefault( manifest_path, manifest_path.read_bytes(), ) _atomic_write_json(manifest_path, manifest) applied.extend(project_applied) except BaseException: for manifest_path, content in reversed(manifest_backups.items()): _atomic_write_bytes(manifest_path, content) for target_path, content in reversed(target_backups.items()): if content is None: target_path.unlink(missing_ok=True) else: _atomic_write_bytes(target_path, content) raise return ApplyResult(applied=tuple(applied), skipped=tuple(skipped)) def _load_projects(self) -> tuple[SourceProject, ...]: payload = _read_json(self.config_path) if payload.get("schema_version") != 1: raise ValueError("source-projects config schema_version must be 1") projects: list[SourceProject] = [] seen: set[str] = set() for raw_project in _required_list(payload, "projects"): module = _required_string(raw_project, "module") if module in seen: raise ValueError(f"duplicate source project module: {module}") seen.add(module) source_project = raw_project.get("source_project") if source_project is not None and not isinstance(source_project, str): raise ValueError("source_project must be a string when provided") projects.append( SourceProject( module=module, manifest=_required_string(raw_project, "manifest"), root_env=_required_string(raw_project, "root_env"), target_root=_required_string(raw_project, "target_root"), source_project=source_project, ) ) return tuple(projects) def _select_projects(self, project: str | None) -> tuple[SourceProject, ...]: if project is None: return self._projects selected = tuple(item for item in self._projects if item.module == project) if not selected: raise KeyError(f"unknown source project: {project}") return selected def _source_root(self, specification: SourceProject) -> Path: configured = self._source_roots.get(specification.module) if configured is None and specification.source_project is not None: configured = self._source_roots.get(specification.source_project) if configured is None: raw_root = self._environ.get(specification.root_env) if not raw_root: raise ValueError( f"source root for {specification.module!r} is not configured; " f"pass source_roots or set {specification.root_env}" ) configured = Path(raw_root).resolve() if not configured.is_dir(): raise ValueError( f"source root for {specification.module!r} is not a directory: " f"{configured}" ) return configured def _load_manifest( self, specification: SourceProject, ) -> tuple[dict[str, Any], tuple[ManifestEntry, ...], Path]: manifest_path = self._mapped_path( self.repository_root, specification.manifest, label="manifest", reject_sensitive=False, ) payload = _read_json(manifest_path) if payload.get("schema_version") != 1: raise ValueError( f"source manifest schema_version must be 1: {manifest_path}" ) if payload.get("module") != specification.module: raise ValueError( f"manifest module does not match {specification.module!r}: " f"{manifest_path}" ) entries: list[ManifestEntry] = [] for raw_entry in _required_list(payload, "files"): transformed = raw_entry.get("transformed") if not isinstance(transformed, bool): raise ValueError("manifest transformed must be a boolean") entries.append( ManifestEntry( source_relative_path=_required_string( raw_entry, "source_relative_path", ), target_relative_path=_required_string( raw_entry, "target_relative_path", ), source_sha256=_required_digest(raw_entry, "source_sha256"), target_sha256=_required_digest(raw_entry, "target_sha256"), transformed=transformed, category=_required_string(raw_entry, "category"), ) ) return payload, tuple(entries), manifest_path @staticmethod def _mapped_path( root: Path, relative_path: str, *, label: str, reject_sensitive: bool, ) -> Path: relative = Path(relative_path) if ( not relative_path or relative.is_absolute() or ".." in relative.parts ): raise ValueError(f"unsafe relative path for {label}: {relative_path!r}") if reject_sensitive and _is_sensitive(relative): raise ValueError(f"sensitive {label} file is not allowed: {relative_path!r}") return _path_within(root, root / relative, label=label) @staticmethod def _verify_unchanged( comparison: FileComparison, source_path: Path, target_path: Path, ) -> None: current_source = _optional_sha256(source_path) current_target = _optional_sha256(target_path) if ( current_source != comparison.source_sha256 or current_target != comparison.target_sha256 ): raise RuntimeError( f"source or target changed during synchronization: " f"{comparison.target_relative_path}" ) def _classify( *, source_digest: str | None, target_digest: str | None, baseline_source: str, baseline_target: str, ) -> SyncState: if source_digest is None: return SyncState.SOURCE_MISSING if target_digest is None: return SyncState.TARGET_MISSING source_changed = source_digest != baseline_source target_changed = target_digest != baseline_target if source_changed and target_changed: return SyncState.CONFLICT if source_changed: return SyncState.SOURCE_CHANGED if target_changed: return SyncState.TARGET_CHANGED return SyncState.IN_SYNC def _optional_sha256(path: Path) -> str | None: if not path.exists(): return None if not path.is_file(): raise ValueError(f"mapped path is not a regular file: {path}") return sha256_file(path) def _path_within(root: Path, path: Path, *, label: str) -> Path: resolved_root = root.resolve() resolved_path = path.resolve() try: resolved_path.relative_to(resolved_root) except ValueError as error: raise ValueError(f"{label} path escapes its root: {path}") from error return resolved_path def _is_sensitive(path: Path) -> bool: lowered_parts = tuple(part.lower() for part in path.parts) name = path.name.lower() return ( name in _SENSITIVE_NAMES or path.suffix.lower() in _SENSITIVE_SUFFIXES or any(part in _FORBIDDEN_PARTS for part in lowered_parts) ) def _is_excluded( source_relative_path: str, exclusions: Sequence[object], ) -> bool: candidate = source_relative_path.casefold() patterns = list(_DEFAULT_EXCLUDED_PATTERNS) explicit_paths: set[str] = set() for exclusion in exclusions: if not isinstance(exclusion, dict): raise ValueError("intentionally_excluded entries must be objects") pattern = exclusion.get("pattern") explicit_path = exclusion.get("source_relative_path") if pattern is not None: if not isinstance(pattern, str) or not pattern: raise ValueError("exclusion pattern must be a non-empty string") patterns.append(pattern) if explicit_path is not None: if not isinstance(explicit_path, str) or not explicit_path: raise ValueError( "excluded source_relative_path must be a non-empty string" ) explicit_paths.add(PurePosixPath(explicit_path).as_posix().casefold()) normalized_patterns: list[str] = [] for pattern in patterns: normalized = pattern.casefold() normalized_patterns.append(normalized) if normalized.startswith("**/"): normalized_patterns.append(normalized[3:]) return candidate in explicit_paths or any( fnmatchcase(candidate, pattern) for pattern in normalized_patterns ) def _iter_source_files( source_root: Path, exclusions: Sequence[object], ) -> tuple[Path, ...]: """Enumerate source candidates without descending into excluded data trees.""" candidates: list[Path] = [] for raw_root, directory_names, file_names in os.walk(source_root, topdown=True): current_root = Path(raw_root) retained_directories: list[str] = [] for directory_name in directory_names: directory_path = current_root / directory_name relative = directory_path.relative_to(source_root) relative_posix = relative.as_posix() if _is_sensitive(relative): continue if _is_excluded(f"{relative_posix}/__gyxx_probe__", exclusions): continue retained_directories.append(directory_name) directory_names[:] = retained_directories for file_name in file_names: source_path = current_root / file_name relative = source_path.relative_to(source_root) relative_posix = relative.as_posix() if _is_sensitive(relative) or _is_excluded(relative_posix, exclusions): continue candidates.append(source_path) return tuple(sorted(candidates)) def _read_json(path: Path) -> dict[str, Any]: with path.open("r", encoding="utf-8-sig") as handle: payload = json.load(handle) if not isinstance(payload, dict): raise ValueError(f"expected a JSON object: {path}") return payload def _required_list( payload: Mapping[str, Any], field: str, ) -> Sequence[dict[str, Any]]: value = payload.get(field) if not isinstance(value, list) or any( not isinstance(item, dict) for item in value ): raise ValueError(f"{field} must be a list of objects") return value def _required_string(payload: Mapping[str, Any], field: str) -> str: value = payload.get(field) if not isinstance(value, str) or not value: raise ValueError(f"{field} must be a non-empty string") return value def _required_digest(payload: Mapping[str, Any], field: str) -> str: value = _required_string(payload, field) if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): raise ValueError(f"{field} must be a lowercase SHA-256 digest") return value def _atomic_copy(source: Path, target: Path, expected_sha256: str | None) -> None: if expected_sha256 is None: raise RuntimeError(f"source disappeared during synchronization: {source}") target.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary_name = tempfile.mkstemp( prefix=f".{target.name}.", suffix=".tmp", dir=target.parent, ) os.close(descriptor) temporary = Path(temporary_name) try: shutil.copy2(source, temporary) if sha256_file(temporary) != expected_sha256: raise RuntimeError(f"source changed while it was being copied: {source}") os.replace(temporary, target) finally: temporary.unlink(missing_ok=True) def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: descriptor, temporary_name = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp", dir=path.parent, ) temporary = Path(temporary_name) try: with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle: json.dump(payload, handle, ensure_ascii=False, indent=2) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) finally: temporary.unlink(missing_ok=True) def _atomic_write_bytes(path: Path, payload: bytes) -> None: path.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary_name = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp", dir=path.parent, ) temporary = Path(temporary_name) try: with os.fdopen(descriptor, "wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) finally: temporary.unlink(missing_ok=True)