feat: consolidate legacy workflows into gyxx-flow

This commit is contained in:
2026-07-28 14:51:15 +08:00
commit c23b62a8c8
374 changed files with 132990 additions and 0 deletions
+117
View File
@@ -0,0 +1,117 @@
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
import pytest
from gyxx_flow.core.artifacts import ArtifactManifest, atomic_write_json, sha256_file
from gyxx_flow.core.config import Settings
from gyxx_flow.core.context import RunContext
from gyxx_flow.core.layout import DataLayout
def test_settings_default_data_root_is_project_local_var(tmp_path: Path) -> None:
settings = Settings.from_env(project_root=tmp_path, env={})
assert settings.project_root == tmp_path.resolve()
assert settings.data_root == (tmp_path / "var").resolve()
def test_settings_accepts_portable_data_root_override(tmp_path: Path) -> None:
external = tmp_path / "runtime"
settings = Settings.from_env(
project_root=tmp_path / "repo",
env={"GYXX_DATA_ROOT": str(external)},
)
assert settings.data_root == external.resolve()
def test_run_context_validates_date_and_builds_traceable_id() -> None:
fixed_now = datetime(2026, 7, 27, 6, 30, 45, tzinfo=timezone.utc)
context = RunContext.create(
"product.daily",
"2026-07-26",
shadow=True,
now=fixed_now,
random_suffix="abc123",
)
assert context.business_date.isoformat() == "2026-07-26"
assert context.run_id == "product.daily__20260726__20260727T063045Z__abc123"
assert context.shadow is True
@pytest.mark.parametrize("value", ["2026-02-30", "27-07-2026", ""])
def test_run_context_rejects_invalid_business_date(value: str) -> None:
with pytest.raises(ValueError, match="business_date"):
RunContext.create("product.daily", value)
@pytest.mark.parametrize("workflow_id", ["../escape", "Bad Workflow", "D:\\legacy"])
def test_run_context_rejects_unsafe_workflow_id(workflow_id: str) -> None:
with pytest.raises(ValueError, match="workflow_id"):
RunContext.create(workflow_id, "2026-07-26")
def test_data_layout_separates_raw_runs_and_runtime_state(tmp_path: Path) -> None:
layout = DataLayout(tmp_path)
raw = layout.raw(
domain="commerce",
source="jd",
dataset="product_daily",
business_date="2026-07-26",
run_id="run-001",
)
assert raw == (
tmp_path
/ "data"
/ "raw"
/ "commerce"
/ "jd"
/ "product_daily"
/ "business_date=2026-07-26"
/ "run_id=run-001"
)
assert layout.state("cookies", "xingtu") == tmp_path / "state" / "cookies" / "xingtu"
assert layout.tmp("run-001") == tmp_path / "tmp" / "run-001"
@pytest.mark.parametrize("segment", ["..", "a/b", "a\\b", "C:\\data", ""])
def test_data_layout_rejects_unsafe_segments(tmp_path: Path, segment: str) -> None:
layout = DataLayout(tmp_path)
with pytest.raises(ValueError, match="path segment"):
layout.state("cookies", segment)
def test_atomic_json_and_artifact_manifest_are_verifiable(tmp_path: Path) -> None:
payload_path = tmp_path / "part-000.json"
atomic_write_json(payload_path, {"rows": [{"sku": "A-1", "qty": 3}]})
payload = json.loads(payload_path.read_text(encoding="utf-8"))
digest = sha256_file(payload_path)
manifest = ArtifactManifest.from_file(
payload_path,
artifact_id="artifact-001",
dataset="product_daily",
workflow_id="product.daily",
run_id="run-001",
business_date="2026-07-26",
schema_version="1",
row_count=1,
upstream_artifact_ids=("source-001",),
)
assert payload["rows"][0]["sku"] == "A-1"
assert len(digest) == 64
assert manifest.sha256 == digest
assert manifest.byte_size == payload_path.stat().st_size
assert manifest.to_dict()["upstream_artifact_ids"] == ["source-001"]
assert list(tmp_path.glob("*.tmp")) == []