wangyunlong/gyxx-flow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete production workflow migration

Browse Source
This commit is contained in:
wangyunlong
2026-08-06 14:29:57 +08:00
parent 7f215e79c4
commit 8df5266abb
448 changed files with 56937 additions and 14619 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_catalog.py
+410 -20
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import json
from collections import Counter
from pathlib import Path
@@ -10,51 +11,364 @@ from gyxx_flow.catalog import CatalogError, WorkflowCatalog
PROJECT_ROOT = Path(__file__).resolve().parents[1]
def test_catalog_maps_exactly_21_current_scheduled_tasks() -> None:
def test_catalog_maps_23_project_scheduled_tasks() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
scheduled = catalog.scheduled_workflows()
assert len(scheduled) == 21
assert len({item.source_task_name for item in scheduled}) == 21
assert len(scheduled) == 23
assert len({item.source_task_name for item in scheduled}) == 23
assert Counter(item.module for item in scheduled) == {
"content_marketing": 9,
"product_commerce": 7,
"shop_intelligence": 2,
"content_marketing": 8,
"product_commerce": 8,
"shop_intelligence": 4,
"supply_chain": 3,
}
def test_catalog_keeps_known_manual_or_unregistered_workflows_unscheduled() -> None:
def test_catalog_schedules_douyin_price_appeal() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
schedule = catalog.schedule_for("shop.douyin_price_appeal")
assert {
"content.mapping.refresh",
"content.retry_failed",
"product.backfill",
"product.market_rank",
"product.review_collection",
"supply.purchase_order_update",
}.issubset({item.workflow_id for item in catalog.manual_workflows()})
assert "product.weekly_aggregate.documented_missing" in {
item.workflow_id for item in catalog.unavailable_workflows()
assert len(catalog.workflows) == 23
assert catalog.manual_workflows() == ()
assert "shop.douyin_price_appeal" in {
schedule.workflow_id for schedule in catalog.schedules
}
assert catalog.unavailable_workflows() == ()
assert schedule.kind == "daily"
assert schedule.effective_times == ("08:00", "16:00", "22:00")
def test_product_daily_passes_the_previous_business_day_explicitly() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
workflow = next(
item for item in catalog.workflows if item.workflow_id == "product.daily"
)
assert workflow.steps[0].args == (
"--target-date",
"{business_date}",
"--stages",
"collect,analyze,export,insert",
)
assert catalog.schedule_for(workflow.workflow_id).business_date_offset_days == -1
def test_every_scheduled_workflow_has_one_valid_asia_shanghai_schedule() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
assert catalog.timezone == "Asia/Shanghai"
assert len(catalog.schedules) == 21
assert len(catalog.schedules) == 23
for workflow in catalog.scheduled_workflows():
schedule = catalog.schedule_for(workflow.workflow_id)
assert schedule.workflow_id == workflow.workflow_id
assert schedule.kind in {"daily", "weekly", "monthly", "interval_days"}
assert schedule.at.count(":") == 1
assert sum(schedule.enabled for schedule in catalog.schedules) == 23
def test_scheduled_workflows_use_python_entries_and_content_tasks_are_graphs() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
for workflow in catalog.scheduled_workflows():
assert workflow.steps, f"scheduled workflow must declare LangGraph steps: {workflow.workflow_id}"
assert all(step.name and step.description for step in workflow.steps)
assert all(
step.data_flow
and step.data_flow.sources
and step.data_flow.processing
and step.data_flow.destinations
for step in workflow.steps
)
entries = [step.entry for step in workflow.steps]
assert all(
not entry.casefold().endswith((".bat", ".cmd", ".ps1"))
for entry in entries
)
daily = next(
workflow
for workflow in catalog.workflows
if workflow.workflow_id == "content.metrics.daily"
)
assert [step.step_id for step in daily.steps] == [
"collect_collaborators",
"refresh_self_mapping",
"collect_self_bilibili",
"collect_self_douyin",
"sync",
]
assert daily.steps[1].depends_on == ("collect_collaborators",)
assert daily.steps[2].entry == "self_bilibili_scraper.py"
assert daily.steps[3].entry == "chanmama_scraper.py"
assert "--include-self-operated" not in daily.steps[0].args
assert all(step.run_after_failure for step in daily.steps[1:])
assert catalog.schedule_for(daily.workflow_id).at == "22:00"
marketing_report = next(
workflow
for workflow in catalog.workflows
if workflow.workflow_id == "content.marketing_report.daily"
)
report_flow = marketing_report.steps[0].data_flow
assert report_flow is not None
assert {source.system for source in report_flow.sources} == {
"PostgreSQL",
"飞书",
}
assert "调用 Hermes 生成营销分析" in report_flow.processing
assert any(
destination.system == "飞书" and destination.condition
for destination in report_flow.destinations
)
comments = next(
workflow
for workflow in catalog.workflows
if workflow.workflow_id == "content.comments.weekly"
)
assert [step.step_id for step in comments.steps] == [
"bilibili",
"xiaohongshu",
"douyin",
]
assert all(step.depends_on == () for step in comments.steps)
assert "三平台并行启动" in comments.note
main_image = next(
workflow
for workflow in catalog.workflows
if workflow.workflow_id == "product.main_image.weekly"
)
assert [step.step_id for step in main_image.steps] == ["jd", "tmall"]
assert [step.entry for step in main_image.steps] == [
"run_weekly_jd_main_image.py",
"run_weekly_main_image.py",
]
assert all(step.depends_on == () for step in main_image.steps)
assert all(step.run_after_failure is False for step in main_image.steps)
assert "并行执行京东与天猫" in main_image.note
assert catalog.schedule_for(main_image.workflow_id).at == "08:30"
def test_shop_steps_preserve_declared_replay_policy_at_module_boundary() -> None:
from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
definitions = ShopIntelligenceModule.from_catalog(catalog).workflow_definitions()
policies = {
workflow.workflow_id: {step.replay_policy for step in workflow.steps}
for workflow in definitions
}
assert policies["shop.metrics.weekly"] == {"idempotent"}
assert policies["shop.competitor.weekly"] == {"idempotent"}
assert policies["shop.jd_self_operated.daily"] == {"idempotent"}
assert policies["shop.douyin_price_appeal"] == {"idempotent"}
def test_product_daily_import_is_declared_idempotent() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
workflow = next(
item
for item in catalog.workflows
if item.workflow_id == "product.import.daily"
)
assert len(workflow.steps) == 1
assert workflow.steps[0].replay_policy == "idempotent"
def test_catalog_loads_optional_step_timeout_seconds(tmp_path: Path) -> None:
(tmp_path / "workflows.json").write_text(
json.dumps(
{
"schema_version": 3,
"workflows": [
{
"id": "content.timeout_probe",
"module": "content_marketing",
"trigger": "manual",
"execution": {
"steps": [
{
"id": "run",
"entry": "run.py",
"timeout_seconds": 12.5,
}
]
},
}
],
}
),
encoding="utf-8",
)
(tmp_path / "schedules.json").write_text(
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
encoding="utf-8",
)
catalog = WorkflowCatalog.load(tmp_path)
assert catalog.workflows[0].steps[0].timeout_seconds == 12.5
@pytest.mark.parametrize("timeout_seconds", [0, -1, True, "60", float("inf")])
def test_catalog_rejects_invalid_step_timeout_seconds(
tmp_path: Path,
timeout_seconds: object,
) -> None:
(tmp_path / "workflows.json").write_text(
json.dumps(
{
"schema_version": 3,
"workflows": [
{
"id": "content.timeout_probe",
"module": "content_marketing",
"trigger": "manual",
"execution": {
"steps": [
{
"id": "run",
"entry": "run.py",
"timeout_seconds": timeout_seconds,
}
]
},
}
],
}
),
encoding="utf-8",
)
(tmp_path / "schedules.json").write_text(
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
encoding="utf-8",
)
with pytest.raises(CatalogError, match="workflow step timeout_seconds is invalid"):
WorkflowCatalog.load(tmp_path)
@pytest.mark.parametrize(
("field", "value", "message"),
[
("name", "", "workflow step name is invalid"),
("description", 42, "workflow step description is invalid"),
],
)
def test_catalog_rejects_invalid_step_explanation(
tmp_path: Path,
field: str,
value: object,
message: str,
) -> None:
step = {
"id": "run",
"entry": "run.py",
"name": "运行任务",
"description": "执行测试任务。",
field: value,
}
(tmp_path / "workflows.json").write_text(
json.dumps(
{
"schema_version": 3,
"workflows": [
{
"id": "a.one",
"module": "content_marketing",
"trigger": "manual",
"execution": {"steps": [step]},
}
],
},
ensure_ascii=False,
),
encoding="utf-8",
)
(tmp_path / "schedules.json").write_text(
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
encoding="utf-8",
)
with pytest.raises(CatalogError, match=message):
WorkflowCatalog.load(tmp_path)
@pytest.mark.parametrize(
("data_flow", "message"),
[
([], "data_flow must be an object"),
(
{"sources": [], "processing": ["处理"], "destinations": []},
"sources is invalid",
),
(
{
"sources": [{"label": "输入"}],
"processing": [],
"destinations": [{"label": "输出"}],
},
"processing is invalid",
),
(
{
"sources": [{"label": "输入"}],
"processing": ["处理"],
"destinations": [{"label": ""}],
},
"destinations is invalid",
),
],
)
def test_catalog_rejects_invalid_step_data_flow(
tmp_path: Path,
data_flow: object,
message: str,
) -> None:
(tmp_path / "workflows.json").write_text(
json.dumps(
{
"schema_version": 3,
"workflows": [
{
"id": "a.one",
"module": "content_marketing",
"trigger": "manual",
"execution": {
"steps": [
{
"id": "run",
"entry": "run.py",
"name": "运行任务",
"description": "执行测试任务。",
"data_flow": data_flow,
}
]
},
}
],
},
ensure_ascii=False,
),
encoding="utf-8",
)
(tmp_path / "schedules.json").write_text(
'{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}',
encoding="utf-8",
)
with pytest.raises(CatalogError, match=message):
WorkflowCatalog.load(tmp_path)
def test_catalog_rejects_duplicate_workflow_ids(tmp_path: Path) -> None:
(tmp_path / "workflows.json").write_text(
'{"schema_version":2,"workflows":['
'{"schema_version":3,"workflows":['
'{"id":"a.one","module":"content_marketing","trigger":"manual",'
'"execution":{"entry":"run.py"}},'
'{"id":"a.one","module":"content_marketing","trigger":"manual",'
@@ -72,7 +386,7 @@ def test_catalog_rejects_duplicate_workflow_ids(tmp_path: Path) -> None:
def test_catalog_rejects_absolute_execution_entry(tmp_path: Path) -> None:
(tmp_path / "workflows.json").write_text(
'{"schema_version":2,"workflows":['
'{"schema_version":3,"workflows":['
'{"id":"a.one","module":"content_marketing","trigger":"manual",'
'"execution":{"entry":"D:/old/run.py"}}]}',
encoding="utf-8",
@@ -84,3 +398,79 @@ def test_catalog_rejects_absolute_execution_entry(tmp_path: Path) -> None:
with pytest.raises(CatalogError, match="relative"):
WorkflowCatalog.load(tmp_path)
def _write_single_scheduled_catalog(tmp_path: Path, at: object) -> None:
(tmp_path / "workflows.json").write_text(
json.dumps(
{
"schema_version": 3,
"workflows": [
{
"id": "shop.price_appeal",
"module": "shop_intelligence",
"trigger": "scheduled",
"execution": {"entry": "run.py"},
"provenance": {"task_name": "price appeal"},
}
],
}
),
encoding="utf-8",
)
(tmp_path / "schedules.json").write_text(
json.dumps(
{
"schema_version": 1,
"timezone": "Asia/Shanghai",
"schedules": [
{
"workflow_id": "shop.price_appeal",
"kind": "daily",
"at": at,
}
],
}
),
encoding="utf-8",
)
def test_catalog_loads_multiple_daily_schedule_times(tmp_path: Path) -> None:
_write_single_scheduled_catalog(tmp_path, ["08:00", "16:00", "22:00"])
schedule = WorkflowCatalog.load(tmp_path).schedules[0]
assert schedule.at == "08:00"
assert schedule.at_times == ("08:00", "16:00", "22:00")
assert schedule.effective_times == ("08:00", "16:00", "22:00")
def test_catalog_preserves_legacy_single_schedule_time(tmp_path: Path) -> None:
_write_single_scheduled_catalog(tmp_path, "08:00")
schedule = WorkflowCatalog.load(tmp_path).schedules[0]
assert schedule.at == "08:00"
assert schedule.at_times == ()
assert schedule.effective_times == ("08:00",)
@pytest.mark.parametrize(
"at",
[
[],
["08:00", "08:00"],
["08:00", "24:00"],
["08:00", 16],
{"morning": "08:00"},
],
)
def test_catalog_rejects_invalid_multiple_schedule_times(
tmp_path: Path,
at: object,
) -> None:
_write_single_scheduled_catalog(tmp_path, at)
with pytest.raises(CatalogError, match="invalid schedule time"):
WorkflowCatalog.load(tmp_path)