477 lines
15 KiB
Python
477 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from gyxx_flow.catalog import CatalogError, WorkflowCatalog
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def test_catalog_maps_23_project_scheduled_tasks() -> None:
|
|
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
|
scheduled = catalog.scheduled_workflows()
|
|
|
|
assert len(scheduled) == 23
|
|
assert len({item.source_task_name for item in scheduled}) == 23
|
|
assert Counter(item.module for item in scheduled) == {
|
|
"content_marketing": 8,
|
|
"product_commerce": 8,
|
|
"shop_intelligence": 4,
|
|
"supply_chain": 3,
|
|
}
|
|
|
|
|
|
def test_catalog_schedules_douyin_price_appeal() -> None:
|
|
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
|
schedule = catalog.schedule_for("shop.douyin_price_appeal")
|
|
|
|
assert len(catalog.workflows) == 23
|
|
assert catalog.manual_workflows() == ()
|
|
assert "shop.douyin_price_appeal" in {
|
|
schedule.workflow_id for schedule in catalog.schedules
|
|
}
|
|
assert catalog.unavailable_workflows() == ()
|
|
assert schedule.kind == "daily"
|
|
assert schedule.effective_times == ("08:00", "16:00", "22:00")
|
|
|
|
|
|
def test_product_daily_passes_the_previous_business_day_explicitly() -> None:
|
|
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
|
workflow = next(
|
|
item for item in catalog.workflows if item.workflow_id == "product.daily"
|
|
)
|
|
|
|
assert workflow.steps[0].args == (
|
|
"--target-date",
|
|
"{business_date}",
|
|
"--stages",
|
|
"collect,analyze,export,insert",
|
|
)
|
|
assert catalog.schedule_for(workflow.workflow_id).business_date_offset_days == -1
|
|
|
|
|
|
def test_every_scheduled_workflow_has_one_valid_asia_shanghai_schedule() -> None:
|
|
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
|
|
|
assert catalog.timezone == "Asia/Shanghai"
|
|
assert len(catalog.schedules) == 23
|
|
for workflow in catalog.scheduled_workflows():
|
|
schedule = catalog.schedule_for(workflow.workflow_id)
|
|
assert schedule.workflow_id == workflow.workflow_id
|
|
assert schedule.kind in {"daily", "weekly", "monthly", "interval_days"}
|
|
assert schedule.at.count(":") == 1
|
|
assert sum(schedule.enabled for schedule in catalog.schedules) == 23
|
|
|
|
|
|
def test_scheduled_workflows_use_python_entries_and_content_tasks_are_graphs() -> None:
|
|
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
|
|
|
for workflow in catalog.scheduled_workflows():
|
|
assert workflow.steps, f"scheduled workflow must declare LangGraph steps: {workflow.workflow_id}"
|
|
assert all(step.name and step.description for step in workflow.steps)
|
|
assert all(
|
|
step.data_flow
|
|
and step.data_flow.sources
|
|
and step.data_flow.processing
|
|
and step.data_flow.destinations
|
|
for step in workflow.steps
|
|
)
|
|
entries = [step.entry for step in workflow.steps]
|
|
assert all(
|
|
not entry.casefold().endswith((".bat", ".cmd", ".ps1"))
|
|
for entry in entries
|
|
)
|
|
|
|
daily = next(
|
|
workflow
|
|
for workflow in catalog.workflows
|
|
if workflow.workflow_id == "content.metrics.daily"
|
|
)
|
|
assert [step.step_id for step in daily.steps] == [
|
|
"collect_collaborators",
|
|
"refresh_self_mapping",
|
|
"collect_self_bilibili",
|
|
"collect_self_douyin",
|
|
"sync",
|
|
]
|
|
assert daily.steps[1].depends_on == ("collect_collaborators",)
|
|
assert daily.steps[2].entry == "self_bilibili_scraper.py"
|
|
assert daily.steps[3].entry == "chanmama_scraper.py"
|
|
assert "--include-self-operated" not in daily.steps[0].args
|
|
assert all(step.run_after_failure for step in daily.steps[1:])
|
|
assert catalog.schedule_for(daily.workflow_id).at == "22:00"
|
|
|
|
marketing_report = next(
|
|
workflow
|
|
for workflow in catalog.workflows
|
|
if workflow.workflow_id == "content.marketing_report.daily"
|
|
)
|
|
report_flow = marketing_report.steps[0].data_flow
|
|
assert report_flow is not None
|
|
assert {source.system for source in report_flow.sources} == {
|
|
"PostgreSQL",
|
|
"飞书",
|
|
}
|
|
assert "调用 Hermes 生成营销分析" in report_flow.processing
|
|
assert any(
|
|
destination.system == "飞书" and destination.condition
|
|
for destination in report_flow.destinations
|
|
)
|
|
|
|
comments = next(
|
|
workflow
|
|
for workflow in catalog.workflows
|
|
if workflow.workflow_id == "content.comments.weekly"
|
|
)
|
|
assert [step.step_id for step in comments.steps] == [
|
|
"bilibili",
|
|
"xiaohongshu",
|
|
"douyin",
|
|
]
|
|
assert all(step.depends_on == () for step in comments.steps)
|
|
assert "三平台并行启动" in comments.note
|
|
|
|
main_image = next(
|
|
workflow
|
|
for workflow in catalog.workflows
|
|
if workflow.workflow_id == "product.main_image.weekly"
|
|
)
|
|
assert [step.step_id for step in main_image.steps] == ["jd", "tmall"]
|
|
assert [step.entry for step in main_image.steps] == [
|
|
"run_weekly_jd_main_image.py",
|
|
"run_weekly_main_image.py",
|
|
]
|
|
assert all(step.depends_on == () for step in main_image.steps)
|
|
assert all(step.run_after_failure is False for step in main_image.steps)
|
|
assert "并行执行京东与天猫" in main_image.note
|
|
assert catalog.schedule_for(main_image.workflow_id).at == "08:30"
|
|
|
|
|
|
def test_shop_steps_preserve_declared_replay_policy_at_module_boundary() -> None:
|
|
from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule
|
|
|
|
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
|
definitions = ShopIntelligenceModule.from_catalog(catalog).workflow_definitions()
|
|
|
|
policies = {
|
|
workflow.workflow_id: {step.replay_policy for step in workflow.steps}
|
|
for workflow in definitions
|
|
}
|
|
|
|
assert policies["shop.metrics.weekly"] == {"idempotent"}
|
|
assert policies["shop.competitor.weekly"] == {"idempotent"}
|
|
assert policies["shop.jd_self_operated.daily"] == {"idempotent"}
|
|
assert policies["shop.douyin_price_appeal"] == {"idempotent"}
|
|
|
|
|
|
def test_product_daily_import_is_declared_idempotent() -> None:
|
|
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
|
workflow = next(
|
|
item
|
|
for item in catalog.workflows
|
|
if item.workflow_id == "product.import.daily"
|
|
)
|
|
|
|
assert len(workflow.steps) == 1
|
|
assert workflow.steps[0].replay_policy == "idempotent"
|
|
|
|
|
|
def test_catalog_loads_optional_step_timeout_seconds(tmp_path: Path) -> None:
|
|
(tmp_path / "workflows.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 3,
|
|
"workflows": [
|
|
{
|
|
"id": "content.timeout_probe",
|
|
"module": "content_marketing",
|
|
"trigger": "manual",
|
|
"execution": {
|
|
"steps": [
|
|
{
|
|
"id": "run",
|
|
"entry": "run.py",
|
|
"timeout_seconds": 12.5,
|
|
}
|
|
]
|
|
},
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "schedules.json").write_text(
|
|
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
catalog = WorkflowCatalog.load(tmp_path)
|
|
|
|
assert catalog.workflows[0].steps[0].timeout_seconds == 12.5
|
|
|
|
|
|
@pytest.mark.parametrize("timeout_seconds", [0, -1, True, "60", float("inf")])
|
|
def test_catalog_rejects_invalid_step_timeout_seconds(
|
|
tmp_path: Path,
|
|
timeout_seconds: object,
|
|
) -> None:
|
|
(tmp_path / "workflows.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 3,
|
|
"workflows": [
|
|
{
|
|
"id": "content.timeout_probe",
|
|
"module": "content_marketing",
|
|
"trigger": "manual",
|
|
"execution": {
|
|
"steps": [
|
|
{
|
|
"id": "run",
|
|
"entry": "run.py",
|
|
"timeout_seconds": timeout_seconds,
|
|
}
|
|
]
|
|
},
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "schedules.json").write_text(
|
|
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with pytest.raises(CatalogError, match="workflow step timeout_seconds is invalid"):
|
|
WorkflowCatalog.load(tmp_path)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("field", "value", "message"),
|
|
[
|
|
("name", "", "workflow step name is invalid"),
|
|
("description", 42, "workflow step description is invalid"),
|
|
],
|
|
)
|
|
def test_catalog_rejects_invalid_step_explanation(
|
|
tmp_path: Path,
|
|
field: str,
|
|
value: object,
|
|
message: str,
|
|
) -> None:
|
|
step = {
|
|
"id": "run",
|
|
"entry": "run.py",
|
|
"name": "运行任务",
|
|
"description": "执行测试任务。",
|
|
field: value,
|
|
}
|
|
(tmp_path / "workflows.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 3,
|
|
"workflows": [
|
|
{
|
|
"id": "a.one",
|
|
"module": "content_marketing",
|
|
"trigger": "manual",
|
|
"execution": {"steps": [step]},
|
|
}
|
|
],
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "schedules.json").write_text(
|
|
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with pytest.raises(CatalogError, match=message):
|
|
WorkflowCatalog.load(tmp_path)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("data_flow", "message"),
|
|
[
|
|
([], "data_flow must be an object"),
|
|
(
|
|
{"sources": [], "processing": ["处理"], "destinations": []},
|
|
"sources is invalid",
|
|
),
|
|
(
|
|
{
|
|
"sources": [{"label": "输入"}],
|
|
"processing": [],
|
|
"destinations": [{"label": "输出"}],
|
|
},
|
|
"processing is invalid",
|
|
),
|
|
(
|
|
{
|
|
"sources": [{"label": "输入"}],
|
|
"processing": ["处理"],
|
|
"destinations": [{"label": ""}],
|
|
},
|
|
"destinations is invalid",
|
|
),
|
|
],
|
|
)
|
|
def test_catalog_rejects_invalid_step_data_flow(
|
|
tmp_path: Path,
|
|
data_flow: object,
|
|
message: str,
|
|
) -> None:
|
|
(tmp_path / "workflows.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 3,
|
|
"workflows": [
|
|
{
|
|
"id": "a.one",
|
|
"module": "content_marketing",
|
|
"trigger": "manual",
|
|
"execution": {
|
|
"steps": [
|
|
{
|
|
"id": "run",
|
|
"entry": "run.py",
|
|
"name": "运行任务",
|
|
"description": "执行测试任务。",
|
|
"data_flow": data_flow,
|
|
}
|
|
]
|
|
},
|
|
}
|
|
],
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "schedules.json").write_text(
|
|
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with pytest.raises(CatalogError, match=message):
|
|
WorkflowCatalog.load(tmp_path)
|
|
|
|
|
|
def test_catalog_rejects_duplicate_workflow_ids(tmp_path: Path) -> None:
|
|
(tmp_path / "workflows.json").write_text(
|
|
'{"schema_version":3,"workflows":['
|
|
'{"id":"a.one","module":"content_marketing","trigger":"manual",'
|
|
'"execution":{"entry":"run.py"}},'
|
|
'{"id":"a.one","module":"content_marketing","trigger":"manual",'
|
|
'"execution":{"entry":"other.py"}}]}',
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "schedules.json").write_text(
|
|
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with pytest.raises(CatalogError, match="duplicate workflow id"):
|
|
WorkflowCatalog.load(tmp_path)
|
|
|
|
|
|
def test_catalog_rejects_absolute_execution_entry(tmp_path: Path) -> None:
|
|
(tmp_path / "workflows.json").write_text(
|
|
'{"schema_version":3,"workflows":['
|
|
'{"id":"a.one","module":"content_marketing","trigger":"manual",'
|
|
'"execution":{"entry":"D:/old/run.py"}}]}',
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "schedules.json").write_text(
|
|
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with pytest.raises(CatalogError, match="relative"):
|
|
WorkflowCatalog.load(tmp_path)
|
|
|
|
|
|
def _write_single_scheduled_catalog(tmp_path: Path, at: object) -> None:
|
|
(tmp_path / "workflows.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 3,
|
|
"workflows": [
|
|
{
|
|
"id": "shop.price_appeal",
|
|
"module": "shop_intelligence",
|
|
"trigger": "scheduled",
|
|
"execution": {"entry": "run.py"},
|
|
"provenance": {"task_name": "price appeal"},
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "schedules.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 1,
|
|
"timezone": "Asia/Shanghai",
|
|
"schedules": [
|
|
{
|
|
"workflow_id": "shop.price_appeal",
|
|
"kind": "daily",
|
|
"at": at,
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def test_catalog_loads_multiple_daily_schedule_times(tmp_path: Path) -> None:
|
|
_write_single_scheduled_catalog(tmp_path, ["08:00", "16:00", "22:00"])
|
|
|
|
schedule = WorkflowCatalog.load(tmp_path).schedules[0]
|
|
|
|
assert schedule.at == "08:00"
|
|
assert schedule.at_times == ("08:00", "16:00", "22:00")
|
|
assert schedule.effective_times == ("08:00", "16:00", "22:00")
|
|
|
|
|
|
def test_catalog_preserves_legacy_single_schedule_time(tmp_path: Path) -> None:
|
|
_write_single_scheduled_catalog(tmp_path, "08:00")
|
|
|
|
schedule = WorkflowCatalog.load(tmp_path).schedules[0]
|
|
|
|
assert schedule.at == "08:00"
|
|
assert schedule.at_times == ()
|
|
assert schedule.effective_times == ("08:00",)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"at",
|
|
[
|
|
[],
|
|
["08:00", "08:00"],
|
|
["08:00", "24:00"],
|
|
["08:00", 16],
|
|
{"morning": "08:00"},
|
|
],
|
|
)
|
|
def test_catalog_rejects_invalid_multiple_schedule_times(
|
|
tmp_path: Path,
|
|
at: object,
|
|
) -> None:
|
|
_write_single_scheduled_catalog(tmp_path, at)
|
|
|
|
with pytest.raises(CatalogError, match="invalid schedule time"):
|
|
WorkflowCatalog.load(tmp_path)
|