feat: consolidate legacy workflows into gyxx-flow
This commit is contained in:
@@ -0,0 +1,240 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters.native import ModuleCommandAdapter, ModuleSourceRoots
|
||||
from gyxx_flow.catalog import WorkflowEntry
|
||||
from gyxx_flow.core.context import RunContext
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULES_ROOT = PROJECT_ROOT / "src" / "gyxx_flow" / "modules"
|
||||
|
||||
|
||||
def _read(relative: str) -> str:
|
||||
return (PROJECT_ROOT / relative).read_text(encoding="utf-8-sig")
|
||||
|
||||
|
||||
def _load_shop_paths(monkeypatch: pytest.MonkeyPatch, data_root: Path):
|
||||
monkeypatch.setenv("GYXX_DATA_ROOT", str(data_root))
|
||||
path = MODULES_ROOT / "shop_intelligence" / "runtime" / "paths.py"
|
||||
name = "_gyxx_test_shop_routing_paths"
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
try:
|
||||
spec.loader.exec_module(module)
|
||||
finally:
|
||||
sys.modules.pop(name, None)
|
||||
return module
|
||||
|
||||
|
||||
def test_shop_output_resolver_confines_collectors_to_raw_root(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
paths = _load_shop_paths(monkeypatch, tmp_path / "portable")
|
||||
default = paths.RAW_ROOT / "京东" / "店铺数据"
|
||||
|
||||
assert paths.resolve_raw_output(None, default=default) == default
|
||||
assert paths.resolve_raw_output("京东/竞店数据", default=default) == (
|
||||
paths.RAW_ROOT / "京东" / "竞店数据"
|
||||
)
|
||||
assert paths.resolve_raw_output(
|
||||
str(paths.RAW_ROOT / "京东" / "店铺数据"), default=default
|
||||
) == (paths.RAW_ROOT / "京东" / "店铺数据")
|
||||
|
||||
with pytest.raises(ValueError, match="RAW_ROOT"):
|
||||
paths.resolve_raw_output(tmp_path / "outside", default=default)
|
||||
with pytest.raises(ValueError, match="RAW_ROOT"):
|
||||
paths.resolve_raw_output("../outside", default=default)
|
||||
|
||||
|
||||
def test_shop_debug_and_cli_outputs_use_canonical_roots() -> None:
|
||||
jd_shop = _read(
|
||||
"src/gyxx_flow/modules/shop_intelligence/runtime/collectors/jd_data_collector.py"
|
||||
)
|
||||
jd_peer = _read(
|
||||
"src/gyxx_flow/modules/shop_intelligence/runtime/collectors/"
|
||||
"jd_peer_store_data_collector.py"
|
||||
)
|
||||
|
||||
assert "os.getcwd()" not in jd_shop
|
||||
assert 'TMP_ROOT / "debug" / "jd_date_picker"' in jd_shop
|
||||
assert "resolve_raw_output(args.output" in jd_shop
|
||||
assert "resolve_raw_output(args.output" in jd_peer
|
||||
|
||||
|
||||
def test_supply_python_outputs_use_state_raw_tmp_and_export_layers() -> None:
|
||||
feishu = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/feishu_sheets.py"
|
||||
)
|
||||
workflow = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/mcp_workflow.py"
|
||||
)
|
||||
monitor = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/monitor.py"
|
||||
)
|
||||
replenishment = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/"
|
||||
"ProductReplenishment.py"
|
||||
)
|
||||
confirmation = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/"
|
||||
"PurchaseConfirmation.py"
|
||||
)
|
||||
batch = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/"
|
||||
"batch_process.py"
|
||||
)
|
||||
bitable = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/"
|
||||
"insert_replenishment_bitable.py"
|
||||
)
|
||||
|
||||
assert 'STATE_ROOT / "cache" / "discontinued_skus_feishu.json"' in feishu
|
||||
assert 'STATE_ROOT / "locks"' in workflow
|
||||
assert "STATUS_FILE = STATE_ROOT / \"status.json\"" in monitor
|
||||
assert "PROCESSING_FILE = STATE_ROOT / \"processing_status.json\"" in monitor
|
||||
|
||||
assert "REPLENISHMENT_RAW_ROOT" in replenishment
|
||||
assert "REPLENISHMENT_WORK_ROOT" in replenishment
|
||||
assert "REPLENISHMENT_EXPORT_ROOT" in replenishment
|
||||
assert 'output_dir="data"' not in replenishment
|
||||
assert "os.listdir(raw_dir)" not in replenishment
|
||||
assert "PURCHASE_CONFIRMATION_RAW_ROOT" in confirmation
|
||||
assert "PURCHASE_CONFIRMATION_EXPORT_ROOT" in confirmation
|
||||
assert "Downloads" not in replenishment
|
||||
assert "Downloads" not in batch
|
||||
assert 'glob("run_id=*/pending_insert.json")' in bitable
|
||||
|
||||
|
||||
def test_supply_raw_outputs_are_run_scoped_and_never_cleaned_in_place() -> None:
|
||||
workflow = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/mcp_workflow.py"
|
||||
)
|
||||
for script in (
|
||||
"collect_confirmation.ps1",
|
||||
"collect_purchase_order_update.ps1",
|
||||
"collect_replenishment.ps1",
|
||||
):
|
||||
source = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + script
|
||||
)
|
||||
assert '"run_id=$workflowId"' in source
|
||||
assert (
|
||||
"Get-ChildItem -Path $sharedDir -File -ErrorAction SilentlyContinue | Remove-Item"
|
||||
not in source
|
||||
)
|
||||
assert (
|
||||
"Get-ChildItem -Path $sharedReplenishmentDir -File -ErrorAction SilentlyContinue | Remove-Item"
|
||||
not in source
|
||||
)
|
||||
assert (
|
||||
"Get-ChildItem -Path $sharedAlertDir -File -ErrorAction SilentlyContinue | Remove-Item"
|
||||
not in source
|
||||
)
|
||||
assert "Remove-Item -LiteralPath $chromeProfileDir" not in source
|
||||
|
||||
clear_body = workflow.split("def _clear_shared_outputs", 1)[1].split(
|
||||
"def _matching_shared_outputs", 1
|
||||
)[0]
|
||||
cleanup_body = workflow.split("def _cleanup_workflow_artifacts", 1)[1].split(
|
||||
"def _verify_bitable_insert_in_pg", 1
|
||||
)[0]
|
||||
assert ".unlink(" not in clear_body
|
||||
assert "SHARED_DIR" not in cleanup_body
|
||||
assert ".rglob(pattern)" in workflow
|
||||
assert 'collector_dir / "data"' not in workflow
|
||||
assert "def _publish_recent_script_outputs" not in workflow
|
||||
|
||||
|
||||
def test_purchase_order_trigger_uses_state_task_and_never_cleans_raw_or_source() -> None:
|
||||
trigger = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/"
|
||||
"trigger_purchase_order_update.py"
|
||||
)
|
||||
collector = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/"
|
||||
"collect_purchase_order_update.ps1"
|
||||
)
|
||||
|
||||
assert "TASK_FILE = STATE_ROOT" in trigger
|
||||
assert "cleanup_old_artifacts" not in trigger
|
||||
assert ".unlink(" not in trigger
|
||||
assert '$taskFile = Join-Path $stateRoot "tasks\\purchase-order-update\\task.json"' in collector
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"script",
|
||||
[
|
||||
"collect_confirmation.ps1",
|
||||
"collect_purchase_order_update.ps1",
|
||||
"collect_replenishment.ps1",
|
||||
],
|
||||
)
|
||||
def test_supply_powershell_prefers_injected_module_roots(script: str) -> None:
|
||||
source = _read(
|
||||
"src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + script
|
||||
)
|
||||
|
||||
for variable in (
|
||||
"GYXX_SUPPLY_RAW_ROOT",
|
||||
"GYXX_SUPPLY_STATE_ROOT",
|
||||
"GYXX_SUPPLY_EXPORT_ROOT",
|
||||
"GYXX_SUPPLY_WORK_ROOT",
|
||||
):
|
||||
assert f"$env:{variable}" in source
|
||||
assert 'Join-Path $projectRoot "var"' in source
|
||||
assert 'Join-Path $dataRoot "raw\\supply_chain"' not in source
|
||||
assert 'Join-Path $dataRoot "exports\\supply_chain"' not in source
|
||||
|
||||
if script == "collect_confirmation.ps1":
|
||||
assert "$collectionStartedAt = Get-Date" in source
|
||||
assert 'Get-ChildItem -Path $dataDir -Include "*.csv", "*.xlsx"' not in source
|
||||
|
||||
|
||||
def test_native_adapter_injects_supply_module_roots(tmp_path: Path) -> None:
|
||||
runtime = tmp_path / "runtime"
|
||||
runtime.mkdir()
|
||||
(runtime / "collect.ps1").write_text("Write-Output ok", encoding="utf-8")
|
||||
adapter = ModuleCommandAdapter(
|
||||
ModuleSourceRoots({"supply_chain": runtime}),
|
||||
base_env={},
|
||||
project_root=tmp_path,
|
||||
data_root=tmp_path / "portable-data",
|
||||
)
|
||||
entry = WorkflowEntry(
|
||||
workflow_id="supply.collect",
|
||||
module="supply_chain",
|
||||
trigger="manual",
|
||||
entry="collect.ps1",
|
||||
args=(),
|
||||
source_project="supply",
|
||||
)
|
||||
context = RunContext.create(
|
||||
"supply.collect",
|
||||
"2026-07-27",
|
||||
now=datetime(2026, 7, 27, tzinfo=timezone.utc),
|
||||
random_suffix="routing1",
|
||||
)
|
||||
|
||||
command = adapter.build(entry, context=context)
|
||||
|
||||
module_root = tmp_path / "portable-data"
|
||||
assert command.env["GYXX_SUPPLY_RAW_ROOT"] == str(
|
||||
(module_root / "data" / "raw" / "supply_chain").resolve()
|
||||
)
|
||||
assert command.env["GYXX_SUPPLY_STATE_ROOT"] == str(
|
||||
(module_root / "state" / "supply_chain").resolve()
|
||||
)
|
||||
assert command.env["GYXX_SUPPLY_EXPORT_ROOT"] == str(
|
||||
(module_root / "data" / "exports" / "supply_chain").resolve()
|
||||
)
|
||||
assert command.env["GYXX_SUPPLY_WORK_ROOT"] == str(
|
||||
(module_root / "tmp" / "supply_chain").resolve()
|
||||
)
|
||||
Reference in New Issue
Block a user