118 lines
3.7 KiB
Python
118 lines
3.7 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from gyxx_flow.core.artifacts import ArtifactManifest, atomic_write_json, sha256_file
|
|
from gyxx_flow.core.config import Settings
|
|
from gyxx_flow.core.context import RunContext
|
|
from gyxx_flow.core.layout import DataLayout
|
|
|
|
|
|
def test_settings_default_data_root_is_project_local_var(tmp_path: Path) -> None:
|
|
settings = Settings.from_env(project_root=tmp_path, env={})
|
|
|
|
assert settings.project_root == tmp_path.resolve()
|
|
assert settings.data_root == (tmp_path / "var").resolve()
|
|
|
|
|
|
def test_settings_accepts_portable_data_root_override(tmp_path: Path) -> None:
|
|
external = tmp_path / "runtime"
|
|
|
|
settings = Settings.from_env(
|
|
project_root=tmp_path / "repo",
|
|
env={"GYXX_DATA_ROOT": str(external)},
|
|
)
|
|
|
|
assert settings.data_root == external.resolve()
|
|
|
|
|
|
def test_run_context_validates_date_and_builds_traceable_id() -> None:
|
|
fixed_now = datetime(2026, 7, 27, 6, 30, 45, tzinfo=timezone.utc)
|
|
|
|
context = RunContext.create(
|
|
"product.daily",
|
|
"2026-07-26",
|
|
shadow=True,
|
|
now=fixed_now,
|
|
random_suffix="abc123",
|
|
)
|
|
|
|
assert context.business_date.isoformat() == "2026-07-26"
|
|
assert context.run_id == "product.daily__20260726__20260727T063045Z__abc123"
|
|
assert context.shadow is True
|
|
|
|
|
|
@pytest.mark.parametrize("value", ["2026-02-30", "27-07-2026", ""])
|
|
def test_run_context_rejects_invalid_business_date(value: str) -> None:
|
|
with pytest.raises(ValueError, match="business_date"):
|
|
RunContext.create("product.daily", value)
|
|
|
|
|
|
@pytest.mark.parametrize("workflow_id", ["../escape", "Bad Workflow", "D:\\legacy"])
|
|
def test_run_context_rejects_unsafe_workflow_id(workflow_id: str) -> None:
|
|
with pytest.raises(ValueError, match="workflow_id"):
|
|
RunContext.create(workflow_id, "2026-07-26")
|
|
|
|
|
|
def test_data_layout_separates_raw_runs_and_runtime_state(tmp_path: Path) -> None:
|
|
layout = DataLayout(tmp_path)
|
|
|
|
raw = layout.raw(
|
|
domain="commerce",
|
|
source="jd",
|
|
dataset="product_daily",
|
|
business_date="2026-07-26",
|
|
run_id="run-001",
|
|
)
|
|
|
|
assert raw == (
|
|
tmp_path
|
|
/ "data"
|
|
/ "raw"
|
|
/ "commerce"
|
|
/ "jd"
|
|
/ "product_daily"
|
|
/ "business_date=2026-07-26"
|
|
/ "run_id=run-001"
|
|
)
|
|
assert layout.state("cookies", "xingtu") == tmp_path / "state" / "cookies" / "xingtu"
|
|
assert layout.tmp("run-001") == tmp_path / "tmp" / "run-001"
|
|
|
|
|
|
@pytest.mark.parametrize("segment", ["..", "a/b", "a\\b", "C:\\data", ""])
|
|
def test_data_layout_rejects_unsafe_segments(tmp_path: Path, segment: str) -> None:
|
|
layout = DataLayout(tmp_path)
|
|
|
|
with pytest.raises(ValueError, match="path segment"):
|
|
layout.state("cookies", segment)
|
|
|
|
|
|
def test_atomic_json_and_artifact_manifest_are_verifiable(tmp_path: Path) -> None:
|
|
payload_path = tmp_path / "part-000.json"
|
|
atomic_write_json(payload_path, {"rows": [{"sku": "A-1", "qty": 3}]})
|
|
|
|
payload = json.loads(payload_path.read_text(encoding="utf-8"))
|
|
digest = sha256_file(payload_path)
|
|
manifest = ArtifactManifest.from_file(
|
|
payload_path,
|
|
artifact_id="artifact-001",
|
|
dataset="product_daily",
|
|
workflow_id="product.daily",
|
|
run_id="run-001",
|
|
business_date="2026-07-26",
|
|
schema_version="1",
|
|
row_count=1,
|
|
upstream_artifact_ids=("source-001",),
|
|
)
|
|
|
|
assert payload["rows"][0]["sku"] == "A-1"
|
|
assert len(digest) == 64
|
|
assert manifest.sha256 == digest
|
|
assert manifest.byte_size == payload_path.stat().st_size
|
|
assert manifest.to_dict()["upstream_artifact_ids"] == ["source-001"]
|
|
assert list(tmp_path.glob("*.tmp")) == []
|