feat: consolidate legacy workflows into gyxx-flow
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.core.artifacts import ArtifactManifest, atomic_write_json, sha256_file
|
||||
from gyxx_flow.core.config import Settings
|
||||
from gyxx_flow.core.context import RunContext
|
||||
from gyxx_flow.core.layout import DataLayout
|
||||
|
||||
|
||||
def test_settings_default_data_root_is_project_local_var(tmp_path: Path) -> None:
|
||||
settings = Settings.from_env(project_root=tmp_path, env={})
|
||||
|
||||
assert settings.project_root == tmp_path.resolve()
|
||||
assert settings.data_root == (tmp_path / "var").resolve()
|
||||
|
||||
|
||||
def test_settings_accepts_portable_data_root_override(tmp_path: Path) -> None:
|
||||
external = tmp_path / "runtime"
|
||||
|
||||
settings = Settings.from_env(
|
||||
project_root=tmp_path / "repo",
|
||||
env={"GYXX_DATA_ROOT": str(external)},
|
||||
)
|
||||
|
||||
assert settings.data_root == external.resolve()
|
||||
|
||||
|
||||
def test_run_context_validates_date_and_builds_traceable_id() -> None:
|
||||
fixed_now = datetime(2026, 7, 27, 6, 30, 45, tzinfo=timezone.utc)
|
||||
|
||||
context = RunContext.create(
|
||||
"product.daily",
|
||||
"2026-07-26",
|
||||
shadow=True,
|
||||
now=fixed_now,
|
||||
random_suffix="abc123",
|
||||
)
|
||||
|
||||
assert context.business_date.isoformat() == "2026-07-26"
|
||||
assert context.run_id == "product.daily__20260726__20260727T063045Z__abc123"
|
||||
assert context.shadow is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["2026-02-30", "27-07-2026", ""])
|
||||
def test_run_context_rejects_invalid_business_date(value: str) -> None:
|
||||
with pytest.raises(ValueError, match="business_date"):
|
||||
RunContext.create("product.daily", value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("workflow_id", ["../escape", "Bad Workflow", "D:\\legacy"])
|
||||
def test_run_context_rejects_unsafe_workflow_id(workflow_id: str) -> None:
|
||||
with pytest.raises(ValueError, match="workflow_id"):
|
||||
RunContext.create(workflow_id, "2026-07-26")
|
||||
|
||||
|
||||
def test_data_layout_separates_raw_runs_and_runtime_state(tmp_path: Path) -> None:
|
||||
layout = DataLayout(tmp_path)
|
||||
|
||||
raw = layout.raw(
|
||||
domain="commerce",
|
||||
source="jd",
|
||||
dataset="product_daily",
|
||||
business_date="2026-07-26",
|
||||
run_id="run-001",
|
||||
)
|
||||
|
||||
assert raw == (
|
||||
tmp_path
|
||||
/ "data"
|
||||
/ "raw"
|
||||
/ "commerce"
|
||||
/ "jd"
|
||||
/ "product_daily"
|
||||
/ "business_date=2026-07-26"
|
||||
/ "run_id=run-001"
|
||||
)
|
||||
assert layout.state("cookies", "xingtu") == tmp_path / "state" / "cookies" / "xingtu"
|
||||
assert layout.tmp("run-001") == tmp_path / "tmp" / "run-001"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("segment", ["..", "a/b", "a\\b", "C:\\data", ""])
|
||||
def test_data_layout_rejects_unsafe_segments(tmp_path: Path, segment: str) -> None:
|
||||
layout = DataLayout(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="path segment"):
|
||||
layout.state("cookies", segment)
|
||||
|
||||
|
||||
def test_atomic_json_and_artifact_manifest_are_verifiable(tmp_path: Path) -> None:
|
||||
payload_path = tmp_path / "part-000.json"
|
||||
atomic_write_json(payload_path, {"rows": [{"sku": "A-1", "qty": 3}]})
|
||||
|
||||
payload = json.loads(payload_path.read_text(encoding="utf-8"))
|
||||
digest = sha256_file(payload_path)
|
||||
manifest = ArtifactManifest.from_file(
|
||||
payload_path,
|
||||
artifact_id="artifact-001",
|
||||
dataset="product_daily",
|
||||
workflow_id="product.daily",
|
||||
run_id="run-001",
|
||||
business_date="2026-07-26",
|
||||
schema_version="1",
|
||||
row_count=1,
|
||||
upstream_artifact_ids=("source-001",),
|
||||
)
|
||||
|
||||
assert payload["rows"][0]["sku"] == "A-1"
|
||||
assert len(digest) == 64
|
||||
assert manifest.sha256 == digest
|
||||
assert manifest.byte_size == payload_path.stat().st_size
|
||||
assert manifest.to_dict()["upstream_artifact_ids"] == ["source-001"]
|
||||
assert list(tmp_path.glob("*.tmp")) == []
|
||||
Reference in New Issue
Block a user