feat: consolidate legacy workflows into gyxx-flow
This commit is contained in:
@@ -0,0 +1,339 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import gyxx_flow.migration.data as migration_data
|
||||
from gyxx_flow.core.artifacts import sha256_file
|
||||
from gyxx_flow.migration.data import (
|
||||
DataLayer,
|
||||
HistoricalDataMigrationExecutor,
|
||||
HistoricalDataMigrationPlanner,
|
||||
MigrationConflictError,
|
||||
MigrationExecutionError,
|
||||
MigrationSource,
|
||||
MigrationValidationError,
|
||||
)
|
||||
|
||||
|
||||
def _source_tree(root: Path) -> dict[str, bytes]:
|
||||
files = {
|
||||
"2025/01/orders.csv": b"order_id,total\n1,19.90\n",
|
||||
"2025/01/detail.json": b'{"id": 1, "items": 2}\n',
|
||||
}
|
||||
for relative, content in files.items():
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(content)
|
||||
return files
|
||||
|
||||
|
||||
def _planner(data_root: Path, *, free_bytes: int = 10_000_000) -> HistoricalDataMigrationPlanner:
|
||||
return HistoricalDataMigrationPlanner(
|
||||
data_root=data_root,
|
||||
free_space_provider=lambda _path: free_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _source(root: Path, *, layer: DataLayer = DataLayer.RAW) -> MigrationSource:
|
||||
return MigrationSource(
|
||||
source_id="legacy-orders",
|
||||
module="product_commerce",
|
||||
layer=layer,
|
||||
root=root,
|
||||
)
|
||||
|
||||
|
||||
def test_plan_is_machine_readable_and_routes_files_by_module_and_layer(tmp_path: Path) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
contents = _source_tree(source_root)
|
||||
manifest_path = tmp_path / "new" / "plans" / "migration.json"
|
||||
|
||||
plan = _planner(tmp_path / "new").create_plan(
|
||||
[_source(source_root)], manifest_path=manifest_path
|
||||
)
|
||||
|
||||
assert plan.source_summary.file_count == 2
|
||||
assert plan.source_summary.total_bytes == sum(map(len, contents.values()))
|
||||
assert {entry.relative_path for entry in plan.files} == set(contents)
|
||||
assert {
|
||||
entry.target_relative_path for entry in plan.files
|
||||
} == {
|
||||
f"data/raw/product_commerce/legacy/legacy-orders/{relative}"
|
||||
for relative in contents
|
||||
}
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
assert payload["schema_version"] == 1
|
||||
assert payload["source_summary"]["file_count"] == 2
|
||||
assert all(len(entry["sha256"]) == 64 for entry in payload["files"])
|
||||
assert payload["precheck"]["status"] == "ready"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layer", list(DataLayer))
|
||||
def test_each_supported_layer_has_an_explicit_target_directory(
|
||||
tmp_path: Path, layer: DataLayer
|
||||
) -> None:
|
||||
source_root = tmp_path / f"legacy-{layer.value}"
|
||||
(source_root / "one.txt").parent.mkdir(parents=True)
|
||||
(source_root / "one.txt").write_text("one", encoding="utf-8")
|
||||
|
||||
plan = _planner(tmp_path / "new").create_plan([_source(source_root, layer=layer)])
|
||||
|
||||
assert plan.files[0].target_relative_path.startswith(
|
||||
f"data/{layer.value}/product_commerce/legacy/legacy-orders/"
|
||||
)
|
||||
|
||||
|
||||
def test_default_execution_is_plan_only_and_never_creates_data_files(tmp_path: Path) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
data_root = tmp_path / "new"
|
||||
plan = _planner(data_root).create_plan([_source(source_root)])
|
||||
|
||||
report = HistoricalDataMigrationExecutor().execute(plan)
|
||||
|
||||
assert report.status == "plan_only"
|
||||
assert report.applied is False
|
||||
assert report.destination_summary.file_count == 0
|
||||
assert not (data_root / "data").exists()
|
||||
|
||||
|
||||
def test_apply_copies_without_changing_or_removing_source_and_reconciles(tmp_path: Path) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
contents = _source_tree(source_root)
|
||||
data_root = tmp_path / "new"
|
||||
report_path = data_root / "evidence" / "migration-report.json"
|
||||
before = {
|
||||
relative: (
|
||||
sha256_file(source_root / relative),
|
||||
(source_root / relative).stat().st_mtime_ns,
|
||||
)
|
||||
for relative in contents
|
||||
}
|
||||
plan = _planner(data_root).create_plan([_source(source_root)])
|
||||
|
||||
report = HistoricalDataMigrationExecutor().execute(
|
||||
plan, apply=True, report_path=report_path
|
||||
)
|
||||
|
||||
assert report.status == "reconciled"
|
||||
assert report.applied is True
|
||||
assert report.source_summary == report.destination_summary
|
||||
assert report.copied_count == 2
|
||||
assert report.mismatches == ()
|
||||
for relative, expected_content in contents.items():
|
||||
source_path = source_root / relative
|
||||
destination = (
|
||||
data_root
|
||||
/ "data/raw/product_commerce/legacy/legacy-orders"
|
||||
/ relative
|
||||
)
|
||||
assert source_path.exists()
|
||||
assert source_path.read_bytes() == expected_content
|
||||
assert (sha256_file(source_path), source_path.stat().st_mtime_ns) == before[relative]
|
||||
assert destination.read_bytes() == expected_content
|
||||
assert sha256_file(destination) == before[relative][0]
|
||||
payload = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
assert payload["status"] == "reconciled"
|
||||
assert payload["destination_summary"] == payload["source_summary"]
|
||||
|
||||
|
||||
def test_progress_reporting_performs_full_destination_reconciliation_only_once(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
plan = _planner(tmp_path / "new").create_plan([_source(source_root)])
|
||||
original = migration_data._summarize_destinations
|
||||
original_write = migration_data._write_report
|
||||
calls = 0
|
||||
writes = 0
|
||||
|
||||
def counted(entries, data_root):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return original(entries, data_root)
|
||||
|
||||
monkeypatch.setattr(migration_data, "_summarize_destinations", counted)
|
||||
|
||||
def counted_write(path, report):
|
||||
nonlocal writes
|
||||
writes += 1
|
||||
return original_write(path, report)
|
||||
|
||||
monkeypatch.setattr(migration_data, "_write_report", counted_write)
|
||||
|
||||
report = HistoricalDataMigrationExecutor().execute(plan, apply=True)
|
||||
|
||||
assert report.status == "reconciled"
|
||||
assert calls == 1
|
||||
assert writes == 1
|
||||
|
||||
|
||||
def test_repeated_apply_is_idempotent_and_skips_matching_files(tmp_path: Path) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
data_root = tmp_path / "new"
|
||||
plan = _planner(data_root).create_plan([_source(source_root)])
|
||||
executor = HistoricalDataMigrationExecutor()
|
||||
first = executor.execute(plan, apply=True)
|
||||
destination = data_root / plan.files[0].target_relative_path
|
||||
first_mtime = destination.stat().st_mtime_ns
|
||||
|
||||
second = executor.execute(plan, apply=True)
|
||||
|
||||
assert first.copied_count == 2
|
||||
assert second.copied_count == 0
|
||||
assert second.skipped_count == 2
|
||||
assert destination.stat().st_mtime_ns == first_mtime
|
||||
assert second.status == "reconciled"
|
||||
|
||||
|
||||
def test_precheck_rejects_conflicting_target_before_copying_anything(tmp_path: Path) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
data_root = tmp_path / "new"
|
||||
conflict = (
|
||||
data_root
|
||||
/ "data/raw/product_commerce/legacy/legacy-orders/2025/01/orders.csv"
|
||||
)
|
||||
conflict.parent.mkdir(parents=True)
|
||||
conflict.write_text("different", encoding="utf-8")
|
||||
|
||||
with pytest.raises(MigrationConflictError, match="conflict"):
|
||||
_planner(data_root).create_plan([_source(source_root)])
|
||||
|
||||
assert not (
|
||||
data_root
|
||||
/ "data/raw/product_commerce/legacy/legacy-orders/2025/01/detail.json"
|
||||
).exists()
|
||||
|
||||
|
||||
def test_precheck_rejects_unplanned_files_in_dedicated_target_scope(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
data_root = tmp_path / "new"
|
||||
unexpected = (
|
||||
data_root
|
||||
/ "data/raw/product_commerce/legacy/legacy-orders/unexpected.txt"
|
||||
)
|
||||
unexpected.parent.mkdir(parents=True)
|
||||
unexpected.write_text("not in source", encoding="utf-8")
|
||||
|
||||
with pytest.raises(MigrationConflictError, match="unplanned target"):
|
||||
_planner(data_root).create_plan([_source(source_root)])
|
||||
|
||||
|
||||
def test_precheck_rejects_insufficient_disk_space(tmp_path: Path) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
|
||||
with pytest.raises(MigrationValidationError, match="disk space"):
|
||||
_planner(tmp_path / "new", free_bytes=1).create_plan([_source(source_root)])
|
||||
|
||||
|
||||
def test_source_and_destination_must_not_overlap(tmp_path: Path) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
|
||||
with pytest.raises(MigrationValidationError, match="overlap"):
|
||||
_planner(source_root).create_plan([_source(source_root)])
|
||||
|
||||
|
||||
def test_source_change_after_planning_stops_before_overwriting_destination(tmp_path: Path) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
data_root = tmp_path / "new"
|
||||
plan = _planner(data_root).create_plan([_source(source_root)])
|
||||
changed = source_root / "2025/01/orders.csv"
|
||||
changed.write_text("changed after planning", encoding="utf-8")
|
||||
|
||||
with pytest.raises(MigrationExecutionError, match="source changed"):
|
||||
HistoricalDataMigrationExecutor().execute(plan, apply=True)
|
||||
|
||||
target = data_root / next(
|
||||
entry.target_relative_path
|
||||
for entry in plan.files
|
||||
if entry.relative_path == "2025/01/orders.csv"
|
||||
)
|
||||
assert not target.exists()
|
||||
|
||||
|
||||
def test_apply_persists_progress_and_can_resume_after_interruption(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
data_root = tmp_path / "new"
|
||||
report_path = data_root / "evidence" / "progress.json"
|
||||
plan = _planner(data_root).create_plan([_source(source_root)])
|
||||
executor = HistoricalDataMigrationExecutor()
|
||||
original_copy = executor._copy_file_atomic
|
||||
calls = 0
|
||||
|
||||
def fail_on_second(
|
||||
source: Path, destination: Path, expected_size: int, expected_sha256: str
|
||||
) -> None:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 2:
|
||||
raise OSError("simulated interruption")
|
||||
original_copy(source, destination, expected_size, expected_sha256)
|
||||
|
||||
monkeypatch.setattr(executor, "_copy_file_atomic", fail_on_second)
|
||||
with pytest.raises(MigrationExecutionError, match="simulated interruption"):
|
||||
executor.execute(plan, apply=True, report_path=report_path)
|
||||
|
||||
interrupted = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
assert interrupted["status"] == "interrupted"
|
||||
assert interrupted["copied_count"] == 1
|
||||
|
||||
resumed = HistoricalDataMigrationExecutor().execute(
|
||||
plan, apply=True, report_path=report_path
|
||||
)
|
||||
assert resumed.status == "reconciled"
|
||||
assert resumed.copied_count == 1
|
||||
assert resumed.skipped_count == 1
|
||||
|
||||
|
||||
def test_corrupt_temporary_copy_is_never_committed_to_destination(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
(source_root / "one.txt").parent.mkdir(parents=True)
|
||||
(source_root / "one.txt").write_text("original", encoding="utf-8")
|
||||
data_root = tmp_path / "new"
|
||||
plan = _planner(data_root).create_plan([_source(source_root)])
|
||||
|
||||
def corrupt_copy(_source: Path, temporary: Path) -> None:
|
||||
temporary.write_text("corrupt", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr("gyxx_flow.migration.data.shutil.copy2", corrupt_copy)
|
||||
with pytest.raises(MigrationExecutionError, match="temporary copy verification"):
|
||||
HistoricalDataMigrationExecutor().execute(plan, apply=True)
|
||||
|
||||
destination = data_root / plan.files[0].target_relative_path
|
||||
assert not destination.exists()
|
||||
|
||||
|
||||
def test_unsafe_ids_and_symlink_escape_are_rejected(tmp_path: Path) -> None:
|
||||
source_root = tmp_path / "legacy"
|
||||
_source_tree(source_root)
|
||||
|
||||
with pytest.raises(MigrationValidationError, match="module"):
|
||||
MigrationSource("source", "../escape", DataLayer.RAW, source_root)
|
||||
|
||||
outside = tmp_path / "outside.txt"
|
||||
outside.write_text("outside", encoding="utf-8")
|
||||
link = source_root / "escape.txt"
|
||||
try:
|
||||
link.symlink_to(outside)
|
||||
except OSError as exc:
|
||||
pytest.skip(f"symlinks are unavailable: {exc}")
|
||||
|
||||
with pytest.raises(MigrationValidationError, match="symbolic link"):
|
||||
_planner(tmp_path / "new").create_plan([_source(source_root)])
|
||||
Reference in New Issue
Block a user