feat: consolidate legacy workflows into gyxx-flow
This commit is contained in:
wangyunlong committed
commit
c23b62a8c8
374 files changed
+132990
No files matched your search
@@ -0,0 +1,33 @@
|
||||
"""Safe, read-only migration boundary for legacy workflows."""
|
||||
|
||||
from gyxx_flow.migration.data import (
|
||||
DataLayer,
|
||||
HistoricalDataMigrationExecutor,
|
||||
HistoricalDataMigrationPlan,
|
||||
HistoricalDataMigrationPlanner,
|
||||
HistoricalDataMigrationReport,
|
||||
MigrationConflictError,
|
||||
MigrationExecutionError,
|
||||
MigrationSource,
|
||||
MigrationValidationError,
|
||||
)
|
||||
from gyxx_flow.migration.legacy import (
|
||||
LegacyCommandAdapter,
|
||||
LegacyConfigurationError,
|
||||
LegacyProjectRoots,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"LegacyCommandAdapter",
|
||||
"LegacyConfigurationError",
|
||||
"LegacyProjectRoots",
|
||||
"DataLayer",
|
||||
"HistoricalDataMigrationExecutor",
|
||||
"HistoricalDataMigrationPlan",
|
||||
"HistoricalDataMigrationPlanner",
|
||||
"HistoricalDataMigrationReport",
|
||||
"MigrationConflictError",
|
||||
"MigrationExecutionError",
|
||||
"MigrationSource",
|
||||
"MigrationValidationError",
|
||||
]
|
||||
@@ -0,0 +1,556 @@
|
||||
"""Plan and execute read-only-source historical file migrations.
|
||||
|
||||
The planner records an immutable source snapshot. The executor is deliberately
|
||||
plan-only unless ``apply=True`` is supplied and only ever copies files into the
|
||||
configured data root.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from gyxx_flow.core.artifacts import atomic_write_json, sha256_file
|
||||
|
||||
_SAFE_ID = re.compile(r"^[a-z][a-z0-9_\-]*$")
|
||||
_PROGRESS_INTERVAL = 1000
|
||||
|
||||
|
||||
class MigrationValidationError(ValueError):
|
||||
"""Raised when a migration cannot be planned safely."""
|
||||
|
||||
|
||||
class MigrationConflictError(MigrationValidationError):
|
||||
"""Raised when a target exists with different content."""
|
||||
|
||||
|
||||
class MigrationExecutionError(RuntimeError):
|
||||
"""Raised when an approved migration cannot be completed or reconciled."""
|
||||
|
||||
|
||||
class DataLayer(str, Enum):
|
||||
RAW = "raw"
|
||||
NORMALIZED = "normalized"
|
||||
CURATED = "curated"
|
||||
EXPORT = "exports"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MigrationSource:
|
||||
source_id: str
|
||||
module: str
|
||||
layer: DataLayer
|
||||
root: Path
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_id(self.source_id, "source_id")
|
||||
_validate_id(self.module, "module")
|
||||
if not isinstance(self.layer, DataLayer):
|
||||
raise MigrationValidationError("layer must be a DataLayer")
|
||||
object.__setattr__(self, "root", Path(self.root).expanduser())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MigrationFile:
|
||||
source_id: str
|
||||
source_path: str
|
||||
relative_path: str
|
||||
target_relative_path: str
|
||||
byte_size: int
|
||||
sha256: str
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MigrationSummary:
|
||||
file_count: int
|
||||
total_bytes: int
|
||||
aggregate_sha256: str
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MigrationPrecheck:
|
||||
status: str
|
||||
required_bytes: int
|
||||
free_bytes: int
|
||||
existing_matching_files: int
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalDataMigrationPlan:
|
||||
plan_id: str
|
||||
created_at: str
|
||||
data_root: str
|
||||
files: tuple[MigrationFile, ...]
|
||||
source_summary: MigrationSummary
|
||||
precheck: MigrationPrecheck
|
||||
schema_version: int = 1
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": self.schema_version,
|
||||
"plan_id": self.plan_id,
|
||||
"created_at": self.created_at,
|
||||
"data_root": self.data_root,
|
||||
"source_summary": self.source_summary.to_dict(),
|
||||
"precheck": self.precheck.to_dict(),
|
||||
"files": [entry.to_dict() for entry in self.files],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MigrationFileResult:
|
||||
target_relative_path: str
|
||||
status: str
|
||||
byte_size: int
|
||||
sha256: str
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistoricalDataMigrationReport:
|
||||
plan_id: str
|
||||
status: str
|
||||
applied: bool
|
||||
copied_count: int
|
||||
skipped_count: int
|
||||
source_summary: MigrationSummary
|
||||
destination_summary: MigrationSummary
|
||||
mismatches: tuple[str, ...]
|
||||
files: tuple[MigrationFileResult, ...]
|
||||
updated_at: str
|
||||
schema_version: int = 1
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": self.schema_version,
|
||||
"plan_id": self.plan_id,
|
||||
"status": self.status,
|
||||
"applied": self.applied,
|
||||
"copied_count": self.copied_count,
|
||||
"skipped_count": self.skipped_count,
|
||||
"source_summary": self.source_summary.to_dict(),
|
||||
"destination_summary": self.destination_summary.to_dict(),
|
||||
"mismatches": list(self.mismatches),
|
||||
"files": [entry.to_dict() for entry in self.files],
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
|
||||
|
||||
class HistoricalDataMigrationPlanner:
|
||||
"""Build a verified, machine-readable copy plan without mutating sources."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
data_root: Path | str,
|
||||
free_space_provider: Callable[[Path], int] | None = None,
|
||||
) -> None:
|
||||
self._data_root = Path(data_root).expanduser().resolve()
|
||||
self._free_space_provider = free_space_provider or _available_bytes
|
||||
|
||||
def create_plan(
|
||||
self,
|
||||
sources: Iterable[MigrationSource],
|
||||
*,
|
||||
manifest_path: Path | str | None = None,
|
||||
) -> HistoricalDataMigrationPlan:
|
||||
entries: list[MigrationFile] = []
|
||||
target_paths: set[str] = set()
|
||||
for source in sources:
|
||||
source_root = _resolve_source_root(source.root)
|
||||
_reject_overlap(source_root, self._data_root)
|
||||
for entry in _inventory_source(source, source_root):
|
||||
if entry.target_relative_path in target_paths:
|
||||
raise MigrationValidationError(
|
||||
f"duplicate migration target: {entry.target_relative_path}"
|
||||
)
|
||||
target_paths.add(entry.target_relative_path)
|
||||
entries.append(entry)
|
||||
entries.sort(key=lambda item: item.target_relative_path)
|
||||
|
||||
pending_bytes, existing = self._inspect_destinations(entries)
|
||||
free_bytes = self._free_space_provider(_existing_parent(self._data_root))
|
||||
if free_bytes < pending_bytes:
|
||||
raise MigrationValidationError(
|
||||
f"insufficient disk space: need {pending_bytes} bytes, have {free_bytes}"
|
||||
)
|
||||
plan = HistoricalDataMigrationPlan(
|
||||
plan_id=uuid.uuid4().hex,
|
||||
created_at=_now(),
|
||||
data_root=str(self._data_root),
|
||||
files=tuple(entries),
|
||||
source_summary=_summarize_expected(entries),
|
||||
precheck=MigrationPrecheck(
|
||||
status="ready",
|
||||
required_bytes=pending_bytes,
|
||||
free_bytes=free_bytes,
|
||||
existing_matching_files=existing,
|
||||
),
|
||||
)
|
||||
if manifest_path is not None:
|
||||
atomic_write_json(manifest_path, plan.to_dict())
|
||||
return plan
|
||||
|
||||
def _inspect_destinations(self, entries: list[MigrationFile]) -> tuple[int, int]:
|
||||
pending_bytes = 0
|
||||
existing_matching = 0
|
||||
planned_targets = {
|
||||
_safe_destination(self._data_root, entry.target_relative_path)
|
||||
for entry in entries
|
||||
}
|
||||
for unplanned in _find_unplanned_targets(entries, self._data_root, planned_targets):
|
||||
raise MigrationConflictError(f"unplanned target conflict: {unplanned}")
|
||||
for entry in entries:
|
||||
destination = _safe_destination(self._data_root, entry.target_relative_path)
|
||||
if not destination.exists():
|
||||
pending_bytes += entry.byte_size
|
||||
continue
|
||||
if not destination.is_file() or destination.is_symlink():
|
||||
raise MigrationConflictError(f"target conflict: {destination}")
|
||||
if destination.stat().st_size != entry.byte_size or sha256_file(destination) != entry.sha256:
|
||||
raise MigrationConflictError(f"target conflict: {destination}")
|
||||
existing_matching += 1
|
||||
return pending_bytes, existing_matching
|
||||
|
||||
|
||||
class HistoricalDataMigrationExecutor:
|
||||
"""Execute a plan with resumable atomic copies and full reconciliation."""
|
||||
|
||||
def execute(
|
||||
self,
|
||||
plan: HistoricalDataMigrationPlan,
|
||||
*,
|
||||
apply: bool = False,
|
||||
report_path: Path | str | None = None,
|
||||
) -> HistoricalDataMigrationReport:
|
||||
data_root = Path(plan.data_root).resolve()
|
||||
if not apply:
|
||||
report = self._build_report(plan, "plan_only", False, 0, 0, (), ())
|
||||
_write_report(report_path, report)
|
||||
return report
|
||||
|
||||
copied = 0
|
||||
skipped = 0
|
||||
results: list[MigrationFileResult] = []
|
||||
try:
|
||||
self._execution_precheck(plan, data_root)
|
||||
for entry in plan.files:
|
||||
source = Path(entry.source_path)
|
||||
destination = _safe_destination(data_root, entry.target_relative_path)
|
||||
if destination.exists():
|
||||
skipped += 1
|
||||
file_status = "skipped_matching"
|
||||
else:
|
||||
self._copy_file_atomic(
|
||||
source, destination, entry.byte_size, entry.sha256
|
||||
)
|
||||
if destination.stat().st_size != entry.byte_size or sha256_file(destination) != entry.sha256:
|
||||
raise MigrationExecutionError(
|
||||
f"destination verification failed: {entry.target_relative_path}"
|
||||
)
|
||||
copied += 1
|
||||
file_status = "copied"
|
||||
results.append(
|
||||
MigrationFileResult(
|
||||
entry.target_relative_path,
|
||||
file_status,
|
||||
entry.byte_size,
|
||||
entry.sha256,
|
||||
)
|
||||
)
|
||||
if report_path is not None and len(results) % _PROGRESS_INTERVAL == 0:
|
||||
progress = self._build_report(
|
||||
plan, "in_progress", True, copied, skipped, tuple(results), ()
|
||||
)
|
||||
_write_report(report_path, progress)
|
||||
except Exception as exc:
|
||||
interrupted = self._build_report(
|
||||
plan,
|
||||
"interrupted",
|
||||
True,
|
||||
copied,
|
||||
skipped,
|
||||
tuple(results),
|
||||
(str(exc),),
|
||||
)
|
||||
_write_report(report_path, interrupted)
|
||||
if isinstance(exc, MigrationExecutionError):
|
||||
raise
|
||||
raise MigrationExecutionError(str(exc)) from exc
|
||||
|
||||
destination_summary, mismatches = _summarize_destinations(plan.files, data_root)
|
||||
status = "reconciled" if not mismatches and destination_summary == plan.source_summary else "mismatch"
|
||||
report = HistoricalDataMigrationReport(
|
||||
plan_id=plan.plan_id,
|
||||
status=status,
|
||||
applied=True,
|
||||
copied_count=copied,
|
||||
skipped_count=skipped,
|
||||
source_summary=plan.source_summary,
|
||||
destination_summary=destination_summary,
|
||||
mismatches=tuple(mismatches),
|
||||
files=tuple(results),
|
||||
updated_at=_now(),
|
||||
)
|
||||
_write_report(report_path, report)
|
||||
if status != "reconciled":
|
||||
raise MigrationExecutionError("destination reconciliation failed")
|
||||
return report
|
||||
|
||||
def _execution_precheck(
|
||||
self, plan: HistoricalDataMigrationPlan, data_root: Path
|
||||
) -> None:
|
||||
pending_bytes = 0
|
||||
planned_targets = {
|
||||
_safe_destination(data_root, entry.target_relative_path)
|
||||
for entry in plan.files
|
||||
}
|
||||
unplanned = _find_unplanned_targets(plan.files, data_root, planned_targets)
|
||||
if unplanned:
|
||||
raise MigrationExecutionError(
|
||||
f"unplanned target conflict: {unplanned[0]}"
|
||||
)
|
||||
for entry in plan.files:
|
||||
source = Path(entry.source_path)
|
||||
if not source.is_file() or source.is_symlink():
|
||||
raise MigrationExecutionError(f"source changed or missing: {source}")
|
||||
if source.stat().st_size != entry.byte_size or sha256_file(source) != entry.sha256:
|
||||
raise MigrationExecutionError(f"source changed after planning: {source}")
|
||||
destination = _safe_destination(data_root, entry.target_relative_path)
|
||||
if destination.exists():
|
||||
if (
|
||||
not destination.is_file()
|
||||
or destination.is_symlink()
|
||||
or destination.stat().st_size != entry.byte_size
|
||||
or sha256_file(destination) != entry.sha256
|
||||
):
|
||||
raise MigrationExecutionError(f"target conflict: {destination}")
|
||||
else:
|
||||
pending_bytes += entry.byte_size
|
||||
free_bytes = _available_bytes(_existing_parent(data_root))
|
||||
if free_bytes < pending_bytes:
|
||||
raise MigrationExecutionError(
|
||||
f"insufficient disk space: need {pending_bytes} bytes, have {free_bytes}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _copy_file_atomic(
|
||||
source: Path,
|
||||
destination: Path,
|
||||
expected_size: int,
|
||||
expected_sha256: str,
|
||||
) -> None:
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp")
|
||||
try:
|
||||
shutil.copy2(source, temporary)
|
||||
if (
|
||||
temporary.stat().st_size != expected_size
|
||||
or sha256_file(temporary) != expected_sha256
|
||||
):
|
||||
raise MigrationExecutionError(
|
||||
f"temporary copy verification failed: {destination}"
|
||||
)
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
@staticmethod
|
||||
def _build_report(
|
||||
plan: HistoricalDataMigrationPlan,
|
||||
status: str,
|
||||
applied: bool,
|
||||
copied: int,
|
||||
skipped: int,
|
||||
results: tuple[MigrationFileResult, ...],
|
||||
mismatches: tuple[str, ...],
|
||||
) -> HistoricalDataMigrationReport:
|
||||
if applied:
|
||||
summary = _summary(
|
||||
(
|
||||
result.target_relative_path,
|
||||
result.byte_size,
|
||||
result.sha256,
|
||||
)
|
||||
for result in results
|
||||
)
|
||||
discovered_mismatches = []
|
||||
else:
|
||||
summary = MigrationSummary(0, 0, _empty_digest())
|
||||
discovered_mismatches = []
|
||||
return HistoricalDataMigrationReport(
|
||||
plan_id=plan.plan_id,
|
||||
status=status,
|
||||
applied=applied,
|
||||
copied_count=copied,
|
||||
skipped_count=skipped,
|
||||
source_summary=plan.source_summary,
|
||||
destination_summary=summary,
|
||||
mismatches=tuple(mismatches or tuple(discovered_mismatches)),
|
||||
files=results,
|
||||
updated_at=_now(),
|
||||
)
|
||||
|
||||
|
||||
def _inventory_source(source: MigrationSource, root: Path) -> list[MigrationFile]:
|
||||
entries: list[MigrationFile] = []
|
||||
for path in sorted(root.rglob("*")):
|
||||
if path.is_symlink():
|
||||
raise MigrationValidationError(f"symbolic link is not allowed: {path}")
|
||||
if not path.is_file():
|
||||
continue
|
||||
relative = path.relative_to(root).as_posix()
|
||||
before = path.stat()
|
||||
digest = sha256_file(path)
|
||||
after = path.stat()
|
||||
if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
|
||||
raise MigrationValidationError(f"source changed while planning: {path}")
|
||||
target = "/".join(
|
||||
("data", source.layer.value, source.module, "legacy", source.source_id, relative)
|
||||
)
|
||||
entries.append(
|
||||
MigrationFile(
|
||||
source_id=source.source_id,
|
||||
source_path=str(path),
|
||||
relative_path=relative,
|
||||
target_relative_path=target,
|
||||
byte_size=after.st_size,
|
||||
sha256=digest,
|
||||
)
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
def _summarize_expected(entries: Iterable[MigrationFile]) -> MigrationSummary:
|
||||
rows = [
|
||||
(entry.target_relative_path, entry.byte_size, entry.sha256)
|
||||
for entry in entries
|
||||
]
|
||||
return _summary(rows)
|
||||
|
||||
|
||||
def _summarize_destinations(
|
||||
entries: Iterable[MigrationFile], data_root: Path
|
||||
) -> tuple[MigrationSummary, list[str]]:
|
||||
rows: list[tuple[str, int, str]] = []
|
||||
mismatches: list[str] = []
|
||||
for entry in entries:
|
||||
destination = _safe_destination(data_root, entry.target_relative_path)
|
||||
if not destination.is_file() or destination.is_symlink():
|
||||
mismatches.append(f"missing: {entry.target_relative_path}")
|
||||
continue
|
||||
size = destination.stat().st_size
|
||||
digest = sha256_file(destination)
|
||||
rows.append((entry.target_relative_path, size, digest))
|
||||
if size != entry.byte_size or digest != entry.sha256:
|
||||
mismatches.append(f"content mismatch: {entry.target_relative_path}")
|
||||
return _summary(rows), mismatches
|
||||
|
||||
|
||||
def _find_unplanned_targets(
|
||||
entries: Iterable[MigrationFile],
|
||||
data_root: Path,
|
||||
planned_targets: set[Path],
|
||||
) -> list[Path]:
|
||||
unplanned: list[Path] = []
|
||||
scopes = {
|
||||
data_root.joinpath(*Path(entry.target_relative_path).parts[:5])
|
||||
for entry in entries
|
||||
}
|
||||
for scope in sorted(scopes):
|
||||
if not scope.exists():
|
||||
continue
|
||||
for candidate in sorted(scope.rglob("*")):
|
||||
if candidate.is_symlink():
|
||||
unplanned.append(candidate)
|
||||
continue
|
||||
if candidate.is_file() and candidate.resolve() not in planned_targets:
|
||||
unplanned.append(candidate)
|
||||
return unplanned
|
||||
|
||||
|
||||
def _summary(rows: Iterable[tuple[str, int, str]]) -> MigrationSummary:
|
||||
ordered = sorted(rows)
|
||||
aggregate = hashlib.sha256()
|
||||
total_bytes = 0
|
||||
for relative, byte_size, digest in ordered:
|
||||
aggregate.update(f"{relative}\0{byte_size}\0{digest}\n".encode("utf-8"))
|
||||
total_bytes += byte_size
|
||||
return MigrationSummary(len(ordered), total_bytes, aggregate.hexdigest())
|
||||
|
||||
|
||||
def _empty_digest() -> str:
|
||||
return hashlib.sha256().hexdigest()
|
||||
|
||||
|
||||
def _resolve_source_root(root: Path) -> Path:
|
||||
try:
|
||||
resolved = root.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise MigrationValidationError(f"source root cannot be resolved: {root}") from exc
|
||||
if not resolved.is_dir():
|
||||
raise MigrationValidationError(f"source root is not a directory: {resolved}")
|
||||
return resolved
|
||||
|
||||
|
||||
def _reject_overlap(source_root: Path, data_root: Path) -> None:
|
||||
if source_root == data_root or source_root.is_relative_to(data_root) or data_root.is_relative_to(source_root):
|
||||
raise MigrationValidationError(
|
||||
f"source and destination overlap: {source_root} / {data_root}"
|
||||
)
|
||||
|
||||
|
||||
def _safe_destination(data_root: Path, relative: str) -> Path:
|
||||
candidate = (data_root / Path(relative)).resolve()
|
||||
if not candidate.is_relative_to(data_root):
|
||||
raise MigrationValidationError(f"target escapes data root: {relative}")
|
||||
return candidate
|
||||
|
||||
|
||||
def _existing_parent(path: Path) -> Path:
|
||||
candidate = path
|
||||
while not candidate.exists():
|
||||
parent = candidate.parent
|
||||
if parent == candidate:
|
||||
raise MigrationValidationError(f"cannot locate filesystem for {path}")
|
||||
candidate = parent
|
||||
return candidate
|
||||
|
||||
|
||||
def _available_bytes(path: Path) -> int:
|
||||
return shutil.disk_usage(path).free
|
||||
|
||||
|
||||
def _validate_id(value: str, field: str) -> None:
|
||||
if not isinstance(value, str) or not _SAFE_ID.fullmatch(value):
|
||||
raise MigrationValidationError(f"unsafe {field}: {value!r}")
|
||||
|
||||
|
||||
def _write_report(
|
||||
report_path: Path | str | None, report: HistoricalDataMigrationReport
|
||||
) -> None:
|
||||
if report_path is not None:
|
||||
atomic_write_json(report_path, report.to_dict())
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Build legacy command steps without executing or hard-coding old projects."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path, PurePosixPath
|
||||
from types import MappingProxyType
|
||||
from typing import Mapping
|
||||
|
||||
from gyxx_flow.catalog import WorkflowEntry
|
||||
from gyxx_flow.core.context import RunContext
|
||||
from gyxx_flow.workflow.steps import CommandStep
|
||||
|
||||
DEFAULT_PROJECT_IDS = ("content", "shop", "product", "supply")
|
||||
_PROJECT_ID = re.compile(r"^[a-z][a-z0-9_]*$")
|
||||
|
||||
|
||||
class LegacyConfigurationError(ValueError):
|
||||
"""Raised when a legacy project or command cannot be resolved safely."""
|
||||
|
||||
|
||||
class LegacyProjectRoots:
|
||||
"""Validated legacy roots supplied explicitly or through the environment."""
|
||||
|
||||
def __init__(self, roots: Mapping[str, Path | str]) -> None:
|
||||
resolved: dict[str, Path] = {}
|
||||
for project_id, configured_root in roots.items():
|
||||
if not isinstance(project_id, str) or not _PROJECT_ID.fullmatch(project_id):
|
||||
raise LegacyConfigurationError(f"invalid legacy project id: {project_id!r}")
|
||||
resolved[project_id] = _resolve_project_root(project_id, configured_root)
|
||||
self._roots = MappingProxyType(resolved)
|
||||
|
||||
@classmethod
|
||||
def from_env(
|
||||
cls,
|
||||
*,
|
||||
env: Mapping[str, str] | None = None,
|
||||
project_ids: tuple[str, ...] = DEFAULT_PROJECT_IDS,
|
||||
) -> "LegacyProjectRoots":
|
||||
values = os.environ if env is None else env
|
||||
missing: list[str] = []
|
||||
roots: dict[str, str] = {}
|
||||
for project_id in project_ids:
|
||||
if not _PROJECT_ID.fullmatch(project_id):
|
||||
raise LegacyConfigurationError(f"invalid legacy project id: {project_id!r}")
|
||||
key = f"GYXX_LEGACY_{project_id.upper()}_ROOT"
|
||||
value = values.get(key, "").strip()
|
||||
if not value:
|
||||
missing.append(key)
|
||||
else:
|
||||
roots[project_id] = value
|
||||
if missing:
|
||||
raise LegacyConfigurationError(
|
||||
"missing legacy root environment variables: " + ", ".join(missing)
|
||||
)
|
||||
return cls(roots)
|
||||
|
||||
def get(self, project_id: str) -> Path:
|
||||
try:
|
||||
return self._roots[project_id]
|
||||
except KeyError as exc:
|
||||
raise LegacyConfigurationError(
|
||||
f"unknown legacy project: {project_id!r}"
|
||||
) from exc
|
||||
|
||||
def as_dict(self) -> dict[str, Path]:
|
||||
return dict(self._roots)
|
||||
|
||||
|
||||
class LegacyCommandAdapter:
|
||||
"""Translate catalog entries to inert ``CommandStep`` definitions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
roots: LegacyProjectRoots,
|
||||
*,
|
||||
base_env: Mapping[str, str] | None = None,
|
||||
python_executable: Path | str | None = None,
|
||||
) -> None:
|
||||
self._roots = roots
|
||||
self._base_env = dict(os.environ if base_env is None else base_env)
|
||||
configured_python = sys.executable if python_executable is None else python_executable
|
||||
self._python_executable = str(configured_python)
|
||||
_validate_text(self._python_executable, field="python_executable")
|
||||
|
||||
def build(self, entry: WorkflowEntry, *, context: RunContext) -> CommandStep:
|
||||
if not entry.source_project:
|
||||
raise LegacyConfigurationError(
|
||||
f"workflow has no source-project provenance: {entry.workflow_id}"
|
||||
)
|
||||
root = self._roots.get(entry.source_project)
|
||||
if entry.workflow_id != context.workflow_id:
|
||||
raise LegacyConfigurationError(
|
||||
f"workflow entry {entry.workflow_id!r} does not match "
|
||||
f"run context {context.workflow_id!r}"
|
||||
)
|
||||
if entry.trigger == "unavailable":
|
||||
raise LegacyConfigurationError(
|
||||
f"legacy workflow is unavailable: {entry.workflow_id}"
|
||||
)
|
||||
|
||||
executable = _resolve_entry(root, entry.entry)
|
||||
arguments = tuple(entry.args)
|
||||
for argument in arguments:
|
||||
_validate_text(argument, field="legacy argument")
|
||||
argv = (
|
||||
(self._python_executable, str(executable), *arguments)
|
||||
if executable.suffix.casefold() == ".py"
|
||||
else (str(executable), *arguments)
|
||||
)
|
||||
env = dict(self._base_env)
|
||||
env.update(
|
||||
{
|
||||
"GYXX_WORKFLOW_ID": context.workflow_id,
|
||||
"GYXX_RUN_ID": context.run_id,
|
||||
"GYXX_BUSINESS_DATE": context.business_date.isoformat(),
|
||||
"GYXX_SHADOW": str(context.shadow).lower(),
|
||||
"GYXX_LEGACY_PROJECT_ROOT": str(root),
|
||||
}
|
||||
)
|
||||
try:
|
||||
return CommandStep(argv=argv, cwd=root, env=env)
|
||||
except ValueError as exc:
|
||||
raise LegacyConfigurationError(str(exc)) from exc
|
||||
|
||||
|
||||
def _resolve_project_root(project_id: str, configured_root: Path | str) -> Path:
|
||||
try:
|
||||
raw_root = Path(configured_root).expanduser()
|
||||
if not raw_root.exists():
|
||||
raise LegacyConfigurationError(
|
||||
f"legacy project root does not exist for {project_id!r}: {raw_root}"
|
||||
)
|
||||
root = raw_root.resolve(strict=True)
|
||||
except (OSError, TypeError, ValueError) as exc:
|
||||
if isinstance(exc, LegacyConfigurationError):
|
||||
raise
|
||||
raise LegacyConfigurationError(
|
||||
f"cannot resolve legacy project root for {project_id!r}: {exc}"
|
||||
) from exc
|
||||
if not root.is_dir():
|
||||
raise LegacyConfigurationError(
|
||||
f"legacy project root is not a directory for {project_id!r}: {root}"
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
def _resolve_entry(root: Path, configured_entry: str) -> Path:
|
||||
if not isinstance(configured_entry, str) or not configured_entry:
|
||||
raise LegacyConfigurationError("legacy entry must be a non-empty relative path")
|
||||
if "\\" in configured_entry or ":" in configured_entry or "\x00" in configured_entry:
|
||||
raise LegacyConfigurationError("legacy entry must use a safe relative path")
|
||||
relative = PurePosixPath(configured_entry)
|
||||
if relative.is_absolute() or ".." in relative.parts:
|
||||
raise LegacyConfigurationError("legacy entry cannot escape its project root")
|
||||
try:
|
||||
candidate = (root / Path(*relative.parts)).resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise LegacyConfigurationError(
|
||||
f"legacy entry does not exist or cannot be resolved: {configured_entry}"
|
||||
) from exc
|
||||
if not candidate.is_relative_to(root):
|
||||
raise LegacyConfigurationError("legacy entry cannot escape its project root")
|
||||
if not candidate.is_file():
|
||||
raise LegacyConfigurationError(f"legacy entry is not a file: {configured_entry}")
|
||||
return candidate
|
||||
|
||||
|
||||
def _validate_text(value: object, *, field: str) -> None:
|
||||
if not isinstance(value, str) or not value or "\x00" in value:
|
||||
raise LegacyConfigurationError(f"{field} must be a non-empty safe string")
|
||||
@@ -0,0 +1,489 @@
|
||||
"""Privacy-conscious comparisons for shadow workflow outputs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import csv
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from gyxx_flow.core.artifacts import atomic_write_json
|
||||
|
||||
_DEFAULT_HASH_SECRET = b"gyxx-flow-public-comparison-v1"
|
||||
_STRUCTURED_SUFFIXES = {
|
||||
".csv": "csv",
|
||||
".json": "json",
|
||||
".jsonl": "jsonl",
|
||||
".ndjson": "jsonl",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MetricTolerance:
|
||||
"""Allowed absolute and relative drift for one numeric metric."""
|
||||
|
||||
absolute: float = 0.0
|
||||
relative: float = 0.0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_tolerance(self.absolute, "absolute")
|
||||
_validate_tolerance(self.relative, "relative")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ComparisonSpec:
|
||||
"""Declarative schema for comparing two structured result sets."""
|
||||
|
||||
primary_key: tuple[str, ...]
|
||||
metrics: Mapping[str, MetricTolerance] = field(default_factory=dict)
|
||||
error_fields: tuple[str, ...] = ()
|
||||
safe_fields: tuple[str, ...] = ()
|
||||
difference_limit: int = 1000
|
||||
hash_secret: str | bytes | None = field(default=None, repr=False, compare=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
primary_key = _validate_fields(self.primary_key, "primary_key", required=True)
|
||||
error_fields = _validate_fields(self.error_fields, "error_fields")
|
||||
safe_fields = _validate_fields(self.safe_fields, "safe_fields")
|
||||
if not isinstance(self.difference_limit, int) or isinstance(
|
||||
self.difference_limit, bool
|
||||
) or self.difference_limit < 1:
|
||||
raise ValueError("difference_limit must be a positive integer")
|
||||
if not isinstance(self.metrics, Mapping):
|
||||
raise ValueError("metrics must be a mapping")
|
||||
metrics: dict[str, MetricTolerance] = {}
|
||||
for name, tolerance in self.metrics.items():
|
||||
_validate_field_name(name, "metric")
|
||||
if not isinstance(tolerance, MetricTolerance):
|
||||
raise ValueError(f"metric {name!r} must declare MetricTolerance")
|
||||
metrics[name] = tolerance
|
||||
if self.hash_secret is not None and not isinstance(
|
||||
self.hash_secret, (str, bytes)
|
||||
):
|
||||
raise ValueError("hash_secret must be text or bytes")
|
||||
if self.hash_secret == "" or self.hash_secret == b"":
|
||||
raise ValueError("hash_secret cannot be empty")
|
||||
object.__setattr__(self, "primary_key", primary_key)
|
||||
object.__setattr__(self, "error_fields", error_fields)
|
||||
object.__setattr__(self, "safe_fields", safe_fields)
|
||||
object.__setattr__(self, "metrics", MappingProxyType(metrics))
|
||||
|
||||
def hash_key(self) -> bytes:
|
||||
if self.hash_secret is None:
|
||||
return _DEFAULT_HASH_SECRET
|
||||
if isinstance(self.hash_secret, str):
|
||||
return self.hash_secret.encode("utf-8")
|
||||
return self.hash_secret
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ComparisonReport:
|
||||
"""Immutable facade over a JSON-serializable comparison report."""
|
||||
|
||||
_payload: Mapping[str, Any] = field(repr=False)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return copy.deepcopy(dict(self._payload))
|
||||
|
||||
|
||||
def compare_structured_files(
|
||||
baseline_path: Path | str,
|
||||
candidate_path: Path | str,
|
||||
*,
|
||||
spec: ComparisonSpec,
|
||||
generated_at: datetime | None = None,
|
||||
) -> ComparisonReport:
|
||||
"""Compare JSON, JSONL, or CSV results without emitting source rows."""
|
||||
|
||||
if not isinstance(spec, ComparisonSpec):
|
||||
raise TypeError("spec must be ComparisonSpec")
|
||||
baseline_format, baseline_rows = _load_records(Path(baseline_path))
|
||||
candidate_format, candidate_rows = _load_records(Path(candidate_path))
|
||||
baseline = _index_rows(baseline_rows, spec.primary_key, side="baseline")
|
||||
candidate = _index_rows(candidate_rows, spec.primary_key, side="candidate")
|
||||
|
||||
baseline_keys = set(baseline)
|
||||
candidate_keys = set(candidate)
|
||||
common_keys = baseline_keys & candidate_keys
|
||||
missing_keys = baseline_keys - candidate_keys
|
||||
extra_keys = candidate_keys - baseline_keys
|
||||
secret = spec.hash_key()
|
||||
|
||||
metric_reports = _compare_metrics(
|
||||
baseline, candidate, common_keys, spec=spec, secret=secret
|
||||
)
|
||||
baseline_errors = _collect_errors(baseline_rows, spec.error_fields)
|
||||
candidate_errors = _collect_errors(candidate_rows, spec.error_fields)
|
||||
error_report = _compare_errors(
|
||||
baseline_errors,
|
||||
candidate_errors,
|
||||
secret=secret,
|
||||
limit=spec.difference_limit,
|
||||
)
|
||||
metric_mismatches = sum(
|
||||
metric["mismatch_count"] for metric in metric_reports.values()
|
||||
)
|
||||
error_differences = (
|
||||
error_report["only_baseline_total"] + error_report["only_candidate_total"]
|
||||
)
|
||||
has_differences = bool(
|
||||
missing_keys or extra_keys or metric_mismatches or error_differences
|
||||
)
|
||||
timestamp = generated_at or datetime.now(timezone.utc)
|
||||
if timestamp.tzinfo is None or timestamp.utcoffset() is None:
|
||||
raise ValueError("generated_at must be timezone-aware")
|
||||
|
||||
missing_identifiers = _sample_identifiers(
|
||||
missing_keys, baseline, spec=spec, secret=secret
|
||||
)
|
||||
extra_identifiers = _sample_identifiers(
|
||||
extra_keys, candidate, spec=spec, secret=secret
|
||||
)
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": 1,
|
||||
"status": "mismatch" if has_differences else "match",
|
||||
"generated_at": timestamp.isoformat(timespec="seconds"),
|
||||
"sources": {
|
||||
"baseline_format": baseline_format,
|
||||
"candidate_format": candidate_format,
|
||||
},
|
||||
"spec": {
|
||||
"primary_key": list(spec.primary_key),
|
||||
"metrics": {
|
||||
name: {
|
||||
"absolute_tolerance": tolerance.absolute,
|
||||
"relative_tolerance": tolerance.relative,
|
||||
}
|
||||
for name, tolerance in spec.metrics.items()
|
||||
},
|
||||
"error_fields": list(spec.error_fields),
|
||||
"safe_fields": list(spec.safe_fields),
|
||||
"difference_limit": spec.difference_limit,
|
||||
},
|
||||
"summary": {
|
||||
"baseline_rows": len(baseline_rows),
|
||||
"candidate_rows": len(candidate_rows),
|
||||
"common_keys": len(common_keys),
|
||||
"missing_keys": len(missing_keys),
|
||||
"extra_keys": len(extra_keys),
|
||||
"metric_mismatches": metric_mismatches,
|
||||
"error_set_differences": error_differences,
|
||||
},
|
||||
"keys": {
|
||||
"missing_total": len(missing_keys),
|
||||
"missing_truncated": len(missing_keys) > spec.difference_limit,
|
||||
"missing": missing_identifiers,
|
||||
"extra_total": len(extra_keys),
|
||||
"extra_truncated": len(extra_keys) > spec.difference_limit,
|
||||
"extra": extra_identifiers,
|
||||
},
|
||||
"metrics": metric_reports,
|
||||
"errors": error_report,
|
||||
}
|
||||
return ComparisonReport(payload)
|
||||
|
||||
|
||||
def write_comparison_report(
|
||||
path: Path | str, report: ComparisonReport
|
||||
) -> Path:
|
||||
"""Atomically persist a machine-readable comparison report."""
|
||||
|
||||
if not isinstance(report, ComparisonReport):
|
||||
raise TypeError("report must be ComparisonReport")
|
||||
return atomic_write_json(path, report.to_dict())
|
||||
|
||||
|
||||
def _load_records(path: Path) -> tuple[str, list[dict[str, Any]]]:
|
||||
data_format = _STRUCTURED_SUFFIXES.get(path.suffix.casefold())
|
||||
if data_format is None:
|
||||
raise ValueError(f"unsupported structured data format: {path.suffix or '<none>'}")
|
||||
if data_format == "csv":
|
||||
return data_format, _load_csv(path)
|
||||
if data_format == "jsonl":
|
||||
return data_format, _load_jsonl(path)
|
||||
return data_format, _load_json(path)
|
||||
|
||||
|
||||
def _load_csv(path: Path) -> list[dict[str, Any]]:
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for line_number, row in enumerate(reader, start=2):
|
||||
if None in row:
|
||||
raise ValueError(f"CSV record has extra columns at line {line_number}")
|
||||
rows.append(dict(row))
|
||||
return rows
|
||||
|
||||
|
||||
def _load_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
with path.open("r", encoding="utf-8-sig") as handle:
|
||||
for line_number, line in enumerate(handle, start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"invalid JSONL at line {line_number}") from exc
|
||||
rows.append(_require_record(row, location=f"JSONL line {line_number}"))
|
||||
return rows
|
||||
|
||||
|
||||
def _load_json(path: Path) -> list[dict[str, Any]]:
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError("invalid JSON document") from exc
|
||||
if isinstance(payload, dict) and set(payload) == {"records"}:
|
||||
payload = payload["records"]
|
||||
if not isinstance(payload, list):
|
||||
raise ValueError("JSON document must be an array or a records envelope")
|
||||
return [
|
||||
_require_record(row, location=f"JSON record {index}")
|
||||
for index, row in enumerate(payload, start=1)
|
||||
]
|
||||
|
||||
|
||||
def _require_record(value: object, *, location: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise ValueError(f"record must be a JSON object at {location}")
|
||||
return dict(value)
|
||||
|
||||
|
||||
def _index_rows(
|
||||
rows: Sequence[Mapping[str, Any]], primary_key: tuple[str, ...], *, side: str
|
||||
) -> dict[tuple[str, ...], Mapping[str, Any]]:
|
||||
indexed: dict[tuple[str, ...], Mapping[str, Any]] = {}
|
||||
for row in rows:
|
||||
key = _primary_key(row, primary_key, side=side)
|
||||
if key in indexed:
|
||||
raise ValueError(f"duplicate primary key in {side} data")
|
||||
indexed[key] = row
|
||||
return indexed
|
||||
|
||||
|
||||
def _primary_key(
|
||||
row: Mapping[str, Any], fields: tuple[str, ...], *, side: str
|
||||
) -> tuple[str, ...]:
|
||||
values: list[str] = []
|
||||
for field_name in fields:
|
||||
if field_name not in row or row[field_name] is None or row[field_name] == "":
|
||||
raise ValueError(f"missing primary key field {field_name!r} in {side} data")
|
||||
value = row[field_name]
|
||||
if isinstance(value, (dict, list, tuple, set)):
|
||||
raise ValueError(f"primary key field {field_name!r} must be scalar")
|
||||
if isinstance(value, bool):
|
||||
values.append("true" if value else "false")
|
||||
else:
|
||||
values.append(str(value))
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def _compare_metrics(
|
||||
baseline: Mapping[tuple[str, ...], Mapping[str, Any]],
|
||||
candidate: Mapping[tuple[str, ...], Mapping[str, Any]],
|
||||
common_keys: set[tuple[str, ...]],
|
||||
*,
|
||||
spec: ComparisonSpec,
|
||||
secret: bytes,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
reports: dict[str, dict[str, Any]] = {}
|
||||
for field_name, tolerance in spec.metrics.items():
|
||||
mismatches: list[tuple[str, ...]] = []
|
||||
max_absolute = Decimal(0)
|
||||
max_relative = Decimal(0)
|
||||
absolute_tolerance = Decimal(str(tolerance.absolute))
|
||||
relative_tolerance = Decimal(str(tolerance.relative))
|
||||
for key in common_keys:
|
||||
baseline_value = _numeric_value(baseline[key], field_name, "baseline")
|
||||
candidate_value = _numeric_value(candidate[key], field_name, "candidate")
|
||||
absolute_delta = abs(candidate_value - baseline_value)
|
||||
scale = max(abs(candidate_value), abs(baseline_value))
|
||||
relative_delta = absolute_delta / scale if scale else Decimal(0)
|
||||
max_absolute = max(max_absolute, absolute_delta)
|
||||
max_relative = max(max_relative, relative_delta)
|
||||
allowed_delta = max(absolute_tolerance, relative_tolerance * scale)
|
||||
if absolute_delta > allowed_delta:
|
||||
mismatches.append(key)
|
||||
reports[field_name] = {
|
||||
"compared": len(common_keys),
|
||||
"mismatch_count": len(mismatches),
|
||||
"max_absolute_delta": float(max_absolute),
|
||||
"max_relative_delta": float(max_relative),
|
||||
"mismatches_truncated": len(mismatches) > spec.difference_limit,
|
||||
"mismatches": _sample_identifiers(
|
||||
mismatches, baseline, spec=spec, secret=secret
|
||||
),
|
||||
}
|
||||
return reports
|
||||
|
||||
|
||||
def _numeric_value(
|
||||
row: Mapping[str, Any], field_name: str, side: str
|
||||
) -> Decimal:
|
||||
if field_name not in row:
|
||||
raise ValueError(f"missing metric field {field_name!r} in {side} record")
|
||||
value = row[field_name]
|
||||
if isinstance(value, bool) or value is None:
|
||||
raise ValueError(f"metric field {field_name!r} must be finite numeric")
|
||||
try:
|
||||
numeric = Decimal(str(value).strip())
|
||||
except (InvalidOperation, ValueError) as exc:
|
||||
raise ValueError(f"metric field {field_name!r} must be finite numeric") from exc
|
||||
if not numeric.is_finite():
|
||||
raise ValueError(f"metric field {field_name!r} must be finite numeric")
|
||||
try:
|
||||
machine_value = float(numeric)
|
||||
except (OverflowError, ValueError) as exc:
|
||||
raise ValueError(f"metric field {field_name!r} exceeds report range") from exc
|
||||
if not math.isfinite(machine_value):
|
||||
raise ValueError(f"metric field {field_name!r} exceeds report range")
|
||||
return numeric
|
||||
|
||||
|
||||
def _sample_identifiers(
|
||||
keys: Sequence[tuple[str, ...]] | set[tuple[str, ...]],
|
||||
rows: Mapping[tuple[str, ...], Mapping[str, Any]],
|
||||
*,
|
||||
spec: ComparisonSpec,
|
||||
secret: bytes,
|
||||
) -> list[dict[str, Any]]:
|
||||
identifiers = [
|
||||
_identifier(key, rows[key], spec=spec, secret=secret) for key in keys
|
||||
]
|
||||
identifiers.sort(key=lambda item: item["key_hash"])
|
||||
return identifiers[: spec.difference_limit]
|
||||
|
||||
|
||||
def _identifier(
|
||||
key: tuple[str, ...],
|
||||
row: Mapping[str, Any],
|
||||
*,
|
||||
spec: ComparisonSpec,
|
||||
secret: bytes,
|
||||
) -> dict[str, Any]:
|
||||
canonical = json.dumps(key, ensure_ascii=False, separators=(",", ":"))
|
||||
identifier: dict[str, Any] = {
|
||||
"key_hash": _secure_hash(canonical, secret=secret, domain="primary-key")
|
||||
}
|
||||
if spec.safe_fields:
|
||||
safe_values = {
|
||||
name: _safe_value(row[name]) for name in spec.safe_fields if name in row
|
||||
}
|
||||
if safe_values:
|
||||
identifier["safe_fields"] = safe_values
|
||||
return identifier
|
||||
|
||||
|
||||
def _safe_value(value: Any) -> Any:
|
||||
if value is None or isinstance(value, (str, int, float, bool)):
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
raise ValueError("safe field must contain a finite JSON scalar")
|
||||
return value
|
||||
raise ValueError("safe field must contain a JSON scalar")
|
||||
|
||||
|
||||
def _collect_errors(
|
||||
rows: Sequence[Mapping[str, Any]], fields: tuple[str, ...]
|
||||
) -> set[str]:
|
||||
errors: set[str] = set()
|
||||
for row in rows:
|
||||
for field_name in fields:
|
||||
if field_name in row:
|
||||
errors.update(_error_tokens(row[field_name]))
|
||||
return errors
|
||||
|
||||
|
||||
def _error_tokens(value: Any) -> set[str]:
|
||||
if value is None or value == "":
|
||||
return set()
|
||||
if isinstance(value, str):
|
||||
stripped = value.strip()
|
||||
if not stripped:
|
||||
return set()
|
||||
if stripped[:1] in {"[", "{"}:
|
||||
try:
|
||||
return _error_tokens(json.loads(stripped))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return {_canonical_json(stripped)}
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
tokens: set[str] = set()
|
||||
for item in value:
|
||||
tokens.update(_error_tokens(item))
|
||||
return tokens
|
||||
return {_canonical_json(value)}
|
||||
|
||||
|
||||
def _compare_errors(
|
||||
baseline: set[str], candidate: set[str], *, secret: bytes, limit: int
|
||||
) -> dict[str, Any]:
|
||||
only_baseline = sorted(
|
||||
_secure_hash(value, secret=secret, domain="error")
|
||||
for value in baseline - candidate
|
||||
)
|
||||
only_candidate = sorted(
|
||||
_secure_hash(value, secret=secret, domain="error")
|
||||
for value in candidate - baseline
|
||||
)
|
||||
return {
|
||||
"baseline_unique_count": len(baseline),
|
||||
"candidate_unique_count": len(candidate),
|
||||
"only_baseline_total": len(only_baseline),
|
||||
"only_baseline_truncated": len(only_baseline) > limit,
|
||||
"only_baseline_hashes": only_baseline[:limit],
|
||||
"only_candidate_total": len(only_candidate),
|
||||
"only_candidate_truncated": len(only_candidate) > limit,
|
||||
"only_candidate_hashes": only_candidate[:limit],
|
||||
}
|
||||
|
||||
|
||||
def _secure_hash(value: str, *, secret: bytes, domain: str) -> str:
|
||||
message = f"gyxx-shadow-{domain}-v1\0{value}".encode("utf-8")
|
||||
return hmac.new(secret, message, hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def _canonical_json(value: Any) -> str:
|
||||
try:
|
||||
return json.dumps(
|
||||
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError("error fields must contain JSON-compatible values") from exc
|
||||
|
||||
|
||||
def _validate_fields(
|
||||
values: object, label: str, *, required: bool = False
|
||||
) -> tuple[str, ...]:
|
||||
if not isinstance(values, tuple):
|
||||
raise ValueError(f"{label} must be a tuple of field names")
|
||||
if required and not values:
|
||||
raise ValueError(f"{label} must contain at least one field")
|
||||
for value in values:
|
||||
_validate_field_name(value, label)
|
||||
if len(set(values)) != len(values):
|
||||
raise ValueError(f"{label} cannot contain duplicate fields")
|
||||
return values
|
||||
|
||||
|
||||
def _validate_field_name(value: object, label: str) -> None:
|
||||
if not isinstance(value, str) or not value.strip() or "\x00" in value:
|
||||
raise ValueError(f"{label} contains an invalid field name")
|
||||
|
||||
|
||||
def _validate_tolerance(value: object, label: str) -> None:
|
||||
if (
|
||||
isinstance(value, bool)
|
||||
or not isinstance(value, (int, float))
|
||||
or not math.isfinite(value)
|
||||
or value < 0
|
||||
):
|
||||
raise ValueError(f"{label} tolerance must be a finite non-negative number")
|
||||
Reference in new issue
Block a user