from __future__ import annotations import json from collections import Counter from pathlib import Path import pytest from gyxx_flow.catalog import CatalogError, WorkflowCatalog PROJECT_ROOT = Path(__file__).resolve().parents[1] def test_catalog_maps_23_project_scheduled_tasks() -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") scheduled = catalog.scheduled_workflows() assert len(scheduled) == 23 assert len({item.source_task_name for item in scheduled}) == 23 assert Counter(item.module for item in scheduled) == { "content_marketing": 8, "product_commerce": 8, "shop_intelligence": 4, "supply_chain": 3, } def test_catalog_schedules_douyin_price_appeal() -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") schedule = catalog.schedule_for("shop.douyin_price_appeal") assert len(catalog.workflows) == 23 assert catalog.manual_workflows() == () assert "shop.douyin_price_appeal" in { schedule.workflow_id for schedule in catalog.schedules } assert catalog.unavailable_workflows() == () assert schedule.kind == "daily" assert schedule.effective_times == ("08:00", "16:00", "22:00") def test_product_daily_passes_the_previous_business_day_explicitly() -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") workflow = next( item for item in catalog.workflows if item.workflow_id == "product.daily" ) assert workflow.steps[0].args == ( "--target-date", "{business_date}", "--stages", "collect,analyze,export,insert", ) assert catalog.schedule_for(workflow.workflow_id).business_date_offset_days == -1 def test_every_scheduled_workflow_has_one_valid_asia_shanghai_schedule() -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") assert catalog.timezone == "Asia/Shanghai" assert len(catalog.schedules) == 23 for workflow in catalog.scheduled_workflows(): schedule = catalog.schedule_for(workflow.workflow_id) assert schedule.workflow_id == workflow.workflow_id assert schedule.kind in {"daily", "weekly", "monthly", "interval_days"} assert schedule.at.count(":") == 1 assert sum(schedule.enabled for schedule in catalog.schedules) == 23 def test_scheduled_workflows_use_python_entries_and_content_tasks_are_graphs() -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") for workflow in catalog.scheduled_workflows(): assert workflow.steps, f"scheduled workflow must declare LangGraph steps: {workflow.workflow_id}" assert all(step.name and step.description for step in workflow.steps) assert all( step.data_flow and step.data_flow.sources and step.data_flow.processing and step.data_flow.destinations for step in workflow.steps ) entries = [step.entry for step in workflow.steps] assert all( not entry.casefold().endswith((".bat", ".cmd", ".ps1")) for entry in entries ) daily = next( workflow for workflow in catalog.workflows if workflow.workflow_id == "content.metrics.daily" ) assert [step.step_id for step in daily.steps] == [ "collect_collaborators", "refresh_self_mapping", "collect_self_bilibili", "collect_self_douyin", "sync", ] assert daily.steps[1].depends_on == ("collect_collaborators",) assert daily.steps[2].entry == "self_bilibili_scraper.py" assert daily.steps[3].entry == "chanmama_scraper.py" assert "--include-self-operated" not in daily.steps[0].args assert all(step.run_after_failure for step in daily.steps[1:]) assert catalog.schedule_for(daily.workflow_id).at == "22:00" marketing_report = next( workflow for workflow in catalog.workflows if workflow.workflow_id == "content.marketing_report.daily" ) report_flow = marketing_report.steps[0].data_flow assert report_flow is not None assert {source.system for source in report_flow.sources} == { "PostgreSQL", "飞书", } assert "调用 Hermes 生成营销分析" in report_flow.processing assert any( destination.system == "飞书" and destination.condition for destination in report_flow.destinations ) comments = next( workflow for workflow in catalog.workflows if workflow.workflow_id == "content.comments.weekly" ) assert [step.step_id for step in comments.steps] == [ "bilibili", "xiaohongshu", "douyin", ] assert all(step.depends_on == () for step in comments.steps) assert "三平台并行启动" in comments.note main_image = next( workflow for workflow in catalog.workflows if workflow.workflow_id == "product.main_image.weekly" ) assert [step.step_id for step in main_image.steps] == ["jd", "tmall"] assert [step.entry for step in main_image.steps] == [ "run_weekly_jd_main_image.py", "run_weekly_main_image.py", ] assert all(step.depends_on == () for step in main_image.steps) assert all(step.run_after_failure is False for step in main_image.steps) assert "并行执行京东与天猫" in main_image.note assert catalog.schedule_for(main_image.workflow_id).at == "08:30" def test_shop_steps_preserve_declared_replay_policy_at_module_boundary() -> None: from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") definitions = ShopIntelligenceModule.from_catalog(catalog).workflow_definitions() policies = { workflow.workflow_id: {step.replay_policy for step in workflow.steps} for workflow in definitions } assert policies["shop.metrics.weekly"] == {"idempotent"} assert policies["shop.competitor.weekly"] == {"idempotent"} assert policies["shop.jd_self_operated.daily"] == {"idempotent"} assert policies["shop.douyin_price_appeal"] == {"idempotent"} def test_product_daily_import_is_declared_idempotent() -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") workflow = next( item for item in catalog.workflows if item.workflow_id == "product.import.daily" ) assert len(workflow.steps) == 1 assert workflow.steps[0].replay_policy == "idempotent" def test_catalog_loads_optional_step_timeout_seconds(tmp_path: Path) -> None: (tmp_path / "workflows.json").write_text( json.dumps( { "schema_version": 3, "workflows": [ { "id": "content.timeout_probe", "module": "content_marketing", "trigger": "manual", "execution": { "steps": [ { "id": "run", "entry": "run.py", "timeout_seconds": 12.5, } ] }, } ], } ), encoding="utf-8", ) (tmp_path / "schedules.json").write_text( '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', encoding="utf-8", ) catalog = WorkflowCatalog.load(tmp_path) assert catalog.workflows[0].steps[0].timeout_seconds == 12.5 @pytest.mark.parametrize("timeout_seconds", [0, -1, True, "60", float("inf")]) def test_catalog_rejects_invalid_step_timeout_seconds( tmp_path: Path, timeout_seconds: object, ) -> None: (tmp_path / "workflows.json").write_text( json.dumps( { "schema_version": 3, "workflows": [ { "id": "content.timeout_probe", "module": "content_marketing", "trigger": "manual", "execution": { "steps": [ { "id": "run", "entry": "run.py", "timeout_seconds": timeout_seconds, } ] }, } ], } ), encoding="utf-8", ) (tmp_path / "schedules.json").write_text( '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', encoding="utf-8", ) with pytest.raises(CatalogError, match="workflow step timeout_seconds is invalid"): WorkflowCatalog.load(tmp_path) @pytest.mark.parametrize( ("field", "value", "message"), [ ("name", "", "workflow step name is invalid"), ("description", 42, "workflow step description is invalid"), ], ) def test_catalog_rejects_invalid_step_explanation( tmp_path: Path, field: str, value: object, message: str, ) -> None: step = { "id": "run", "entry": "run.py", "name": "运行任务", "description": "执行测试任务。", field: value, } (tmp_path / "workflows.json").write_text( json.dumps( { "schema_version": 3, "workflows": [ { "id": "a.one", "module": "content_marketing", "trigger": "manual", "execution": {"steps": [step]}, } ], }, ensure_ascii=False, ), encoding="utf-8", ) (tmp_path / "schedules.json").write_text( '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', encoding="utf-8", ) with pytest.raises(CatalogError, match=message): WorkflowCatalog.load(tmp_path) @pytest.mark.parametrize( ("data_flow", "message"), [ ([], "data_flow must be an object"), ( {"sources": [], "processing": ["处理"], "destinations": []}, "sources is invalid", ), ( { "sources": [{"label": "输入"}], "processing": [], "destinations": [{"label": "输出"}], }, "processing is invalid", ), ( { "sources": [{"label": "输入"}], "processing": ["处理"], "destinations": [{"label": ""}], }, "destinations is invalid", ), ], ) def test_catalog_rejects_invalid_step_data_flow( tmp_path: Path, data_flow: object, message: str, ) -> None: (tmp_path / "workflows.json").write_text( json.dumps( { "schema_version": 3, "workflows": [ { "id": "a.one", "module": "content_marketing", "trigger": "manual", "execution": { "steps": [ { "id": "run", "entry": "run.py", "name": "运行任务", "description": "执行测试任务。", "data_flow": data_flow, } ] }, } ], }, ensure_ascii=False, ), encoding="utf-8", ) (tmp_path / "schedules.json").write_text( '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', encoding="utf-8", ) with pytest.raises(CatalogError, match=message): WorkflowCatalog.load(tmp_path) def test_catalog_rejects_duplicate_workflow_ids(tmp_path: Path) -> None: (tmp_path / "workflows.json").write_text( '{"schema_version":3,"workflows":[' '{"id":"a.one","module":"content_marketing","trigger":"manual",' '"execution":{"entry":"run.py"}},' '{"id":"a.one","module":"content_marketing","trigger":"manual",' '"execution":{"entry":"other.py"}}]}', encoding="utf-8", ) (tmp_path / "schedules.json").write_text( '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', encoding="utf-8", ) with pytest.raises(CatalogError, match="duplicate workflow id"): WorkflowCatalog.load(tmp_path) def test_catalog_rejects_absolute_execution_entry(tmp_path: Path) -> None: (tmp_path / "workflows.json").write_text( '{"schema_version":3,"workflows":[' '{"id":"a.one","module":"content_marketing","trigger":"manual",' '"execution":{"entry":"D:/old/run.py"}}]}', encoding="utf-8", ) (tmp_path / "schedules.json").write_text( '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', encoding="utf-8", ) with pytest.raises(CatalogError, match="relative"): WorkflowCatalog.load(tmp_path) def _write_single_scheduled_catalog(tmp_path: Path, at: object) -> None: (tmp_path / "workflows.json").write_text( json.dumps( { "schema_version": 3, "workflows": [ { "id": "shop.price_appeal", "module": "shop_intelligence", "trigger": "scheduled", "execution": {"entry": "run.py"}, "provenance": {"task_name": "price appeal"}, } ], } ), encoding="utf-8", ) (tmp_path / "schedules.json").write_text( json.dumps( { "schema_version": 1, "timezone": "Asia/Shanghai", "schedules": [ { "workflow_id": "shop.price_appeal", "kind": "daily", "at": at, } ], } ), encoding="utf-8", ) def test_catalog_loads_multiple_daily_schedule_times(tmp_path: Path) -> None: _write_single_scheduled_catalog(tmp_path, ["08:00", "16:00", "22:00"]) schedule = WorkflowCatalog.load(tmp_path).schedules[0] assert schedule.at == "08:00" assert schedule.at_times == ("08:00", "16:00", "22:00") assert schedule.effective_times == ("08:00", "16:00", "22:00") def test_catalog_preserves_legacy_single_schedule_time(tmp_path: Path) -> None: _write_single_scheduled_catalog(tmp_path, "08:00") schedule = WorkflowCatalog.load(tmp_path).schedules[0] assert schedule.at == "08:00" assert schedule.at_times == () assert schedule.effective_times == ("08:00",) @pytest.mark.parametrize( "at", [ [], ["08:00", "08:00"], ["08:00", "24:00"], ["08:00", 16], {"morning": "08:00"}, ], ) def test_catalog_rejects_invalid_multiple_schedule_times( tmp_path: Path, at: object, ) -> None: _write_single_scheduled_catalog(tmp_path, at) with pytest.raises(CatalogError, match="invalid schedule time"): WorkflowCatalog.load(tmp_path)