feat: expand module workflows, dynamic config, notifications and console API

tests/test_catalog.py

@@ -11,17 +11,17 @@ from gyxx_flow.catalog import CatalogError, WorkflowCatalog
 PROJECT_ROOT = Path(__file__).resolve().parents[1]
 
 
-def test_catalog_maps_23_project_scheduled_tasks() -> None:
+def test_catalog_maps_27_project_scheduled_tasks() -> None:
     catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
     scheduled = catalog.scheduled_workflows()
 
-    assert len(scheduled) == 23
-    assert len({item.source_task_name for item in scheduled}) == 23
+    assert len(scheduled) == 27
+    assert len({item.source_task_name for item in scheduled}) == 27
     assert Counter(item.module for item in scheduled) == {
-        "content_marketing": 8,
-        "product_commerce": 8,
+        "content_marketing": 9,
+        "product_commerce": 10,
         "shop_intelligence": 4,
-        "supply_chain": 3,
+        "supply_chain": 4,
     }
 
 
@@ -29,8 +29,11 @@ def test_catalog_schedules_douyin_price_appeal() -> None:
     catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
     schedule = catalog.schedule_for("shop.douyin_price_appeal")
 
-    assert len(catalog.workflows) == 23
-    assert catalog.manual_workflows() == ()
+    assert len(catalog.workflows) == 29
+    assert tuple(item.workflow_id for item in catalog.manual_workflows()) == (
+        "product.video_upload",
+        "product.jd_video_upload",
+    )
     assert "shop.douyin_price_appeal" in {
         schedule.workflow_id for schedule in catalog.schedules
     }
@@ -39,6 +42,96 @@ def test_catalog_schedules_douyin_price_appeal() -> None:
     assert schedule.effective_times == ("08:00", "16:00", "22:00")
 
 
+def test_tmall_video_upload_is_an_idempotent_store_selected_manual_workflow() -> None:
+    catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
+    workflow = next(
+        item for item in catalog.workflows if item.workflow_id == "product.video_upload"
+    )
+
+    assert workflow.trigger == "manual"
+    assert [step.step_id for step in workflow.steps] == ["upload"]
+    step = workflow.steps[0]
+    assert step.entry == "upload_video_to_guanghe.py"
+    assert step.args == (
+        "--execute",
+        "--keep-browser-open-on-failure",
+    )
+    assert step.replay_policy == "idempotent"
+    assert step.timeout_seconds == 21600
+    assert workflow.topic_config is not None
+    assert workflow.topic_config.default_keyword == "我的夏日焕新清单"
+    with pytest.raises(KeyError):
+        catalog.schedule_for(workflow.workflow_id)
+
+
+def test_jd_video_upload_is_an_idempotent_manual_workflow() -> None:
+    catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
+    workflow = next(
+        item
+        for item in catalog.workflows
+        if item.workflow_id == "product.jd_video_upload"
+    )
+
+    assert workflow.trigger == "manual"
+    assert [step.step_id for step in workflow.steps] == ["upload"]
+    step = workflow.steps[0]
+    assert step.entry == "upload_video_to_jd.py"
+    assert step.args == (
+        "--anchor-record-id",
+        "recvrXTRWWFXeJ",
+        "--manual-login",
+        "--execute",
+    )
+    assert step.replay_policy == "idempotent"
+    assert step.timeout_seconds == 21600
+    with pytest.raises(KeyError):
+        catalog.schedule_for(workflow.workflow_id)
+
+
+def test_tmall_baibu_apply_is_a_six_step_hourly_workflow() -> None:
+    catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
+    workflow = next(
+        item
+        for item in catalog.workflows
+        if item.workflow_id == "product.tmall_baibu_apply"
+    )
+
+    assert workflow.trigger == "scheduled"
+    assert [step.step_id for step in workflow.steps] == [
+        "old_chain_bag",
+        "old_all_bag",
+        "old_all_3c",
+        "old_chain_3c",
+        "new_chain_bag",
+        "new_all_bag",
+    ]
+    assert [step.entry for step in workflow.steps] == [
+        "tmall_baibu_apply_old_chain_bag.py",
+        "tmall_baibu_apply_old_all_bag.py",
+        "tmall_baibu_apply_old_all_3c.py",
+        "tmall_baibu_apply_old_chain_3c.py",
+        "tmall_baibu_apply_new_chain_bag.py",
+        "tmall_baibu_apply_new_all_bag.py",
+    ]
+    assert workflow.steps[0].depends_on == ()
+    assert all(
+        step.depends_on == (previous.step_id,)
+        for previous, step in zip(workflow.steps, workflow.steps[1:])
+    )
+    assert all(step.run_after_failure for step in workflow.steps[1:])
+    assert all(step.replay_policy == "repeatable" for step in workflow.steps)
+    assert [step.step_id for step in workflow.steps if step.has_hyperlink] == [
+        "old_chain_bag",
+        "old_chain_3c",
+        "new_chain_bag",
+    ]
+    schedule = catalog.schedule_for(workflow.workflow_id)
+    assert schedule.kind == "daily"
+    assert schedule.enabled is True
+    assert schedule.business_date_offset_days == 0
+    assert schedule.effective_times == tuple(f"{hour:02d}:00" for hour in range(24))
+
+
 def test_product_daily_passes_the_previous_business_day_explicitly() -> None:
     catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
     workflow = next(
@@ -49,16 +142,46 @@ def test_product_daily_passes_the_previous_business_day_explicitly() -> None:
         "--target-date",
         "{business_date}",
         "--stages",
-        "collect,analyze,export,insert",
+        "collect,import,analyze,export,insert",
     )
     assert catalog.schedule_for(workflow.workflow_id).business_date_offset_days == -1
 
 
+def test_style_analysis_uses_the_previous_complete_business_day() -> None:
+    catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
+    schedule = catalog.schedule_for("product.style_analysis.interval")
+
+    assert schedule.kind == "interval_days"
+    assert schedule.every_days == 3
+    assert schedule.business_date_offset_days == -1
+
+
+def test_erp_all_shop_daily_runs_at_nine_for_the_previous_day() -> None:
+    catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
+    workflow = next(
+        item
+        for item in catalog.workflows
+        if item.workflow_id == "product.erp_all_shop_daily"
+    )
+
+    assert workflow.steps[0].args == (
+        "--from",
+        "{business_date}",
+        "--to",
+        "{business_date}",
+        "--execute",
+        "--force",
+    )
+    schedule = catalog.schedule_for(workflow.workflow_id)
+    assert schedule.at == "09:00"
+    assert schedule.business_date_offset_days == -1
+
+
 def test_every_scheduled_workflow_has_one_valid_asia_shanghai_schedule() -> None:
     catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
 
     assert catalog.timezone == "Asia/Shanghai"
-    assert len(catalog.schedules) == 23
+    assert len(catalog.schedules) == 27
     for workflow in catalog.scheduled_workflows():
         schedule = catalog.schedule_for(workflow.workflow_id)
         assert schedule.workflow_id == workflow.workflow_id
@@ -98,12 +221,36 @@ def test_scheduled_workflows_use_python_entries_and_content_tasks_are_graphs() -
         "collect_self_douyin",
         "sync",
     ]
+    assert daily.steps[0].depends_on == ()
     assert daily.steps[1].depends_on == ("collect_collaborators",)
-    assert daily.steps[2].entry == "self_bilibili_scraper.py"
-    assert daily.steps[3].entry == "chanmama_scraper.py"
+    assert daily.steps[2].depends_on == ("refresh_self_mapping",)
+    assert daily.steps[3].depends_on == ("collect_self_bilibili",)
+    assert daily.steps[4].depends_on == ("collect_self_douyin",)
     assert "--include-self-operated" not in daily.steps[0].args
-    assert all(step.run_after_failure for step in daily.steps[1:])
-    assert catalog.schedule_for(daily.workflow_id).at == "22:00"
+    assert all(step.replay_policy == "repeatable" for step in daily.steps)
+    assert catalog.schedule_for(daily.workflow_id).effective_times == (
+        "01:00",
+        "05:00",
+    )
+
+    backfill = next(
+        workflow
+        for workflow in catalog.workflows
+        if workflow.workflow_id == "content.metrics.backfill"
+    )
+    assert [step.step_id for step in backfill.steps] == [
+        "collect_collaborators",
+        "sync",
+    ]
+    assert backfill.steps[0].entry == "run_all.py"
+    assert backfill.steps[0].args == ("--daily-scope",)
+    assert backfill.steps[1].entry == "data/tools/sync_metrics_to_cmt_notes.py"
+    assert backfill.steps[1].depends_on == ("collect_collaborators",)
+    assert backfill.steps[1].run_after_failure is True
+    backfill_schedule = catalog.schedule_for(backfill.workflow_id)
+    assert backfill_schedule.kind == "weekly"
+    assert backfill_schedule.days == ("Monday",)
+    assert backfill_schedule.at == "14:00"
 
     marketing_report = next(
         workflow
@@ -122,6 +269,15 @@ def test_scheduled_workflows_use_python_entries_and_content_tasks_are_graphs() -
         for destination in report_flow.destinations
     )
 
+    notes_master = next(
+        workflow
+        for workflow in catalog.workflows
+        if workflow.workflow_id == "content.notes_master.daily"
+    )
+    assert notes_master.steps[0].entry == "data/tools/sync_notes_master.py"
+    assert notes_master.steps[0].args == ("--execute",)
+    assert catalog.schedule_for(notes_master.workflow_id).at == "09:00"
+
     comments = next(
         workflow
         for workflow in catalog.workflows
@@ -150,6 +306,21 @@ def test_scheduled_workflows_use_python_entries_and_content_tasks_are_graphs() -
     assert "并行执行京东与天猫" in main_image.note
     assert catalog.schedule_for(main_image.workflow_id).at == "08:30"
 
+    shop_metrics = next(
+        workflow
+        for workflow in catalog.workflows
+        if workflow.workflow_id == "shop.metrics.weekly"
+    )
+    assert [step.step_id for step in shop_metrics.steps] == ["jd", "dy", "tm"]
+    assert shop_metrics.steps[0].args == ("--platform", "jd", "--skip-feishu")
+    assert shop_metrics.steps[1].args == ("--platform", "dy", "--skip-feishu")
+    assert all(
+        destination.system != "飞书 Base"
+        for step in shop_metrics.steps
+        for destination in step.data_flow.destinations
+    )
+    assert "店铺经营指标周采集不写飞书 Base" in shop_metrics.note
+
 
 def test_shop_steps_preserve_declared_replay_policy_at_module_boundary() -> None:
     from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule
@@ -168,16 +339,18 @@ def test_shop_steps_preserve_declared_replay_policy_at_module_boundary() -> None
     assert policies["shop.douyin_price_appeal"] == {"idempotent"}
 
 
-def test_product_daily_import_is_declared_idempotent() -> None:
+def test_product_daily_includes_the_idempotent_import_stage() -> None:
     catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
     workflow = next(
         item
         for item in catalog.workflows
-        if item.workflow_id == "product.import.daily"
+        if item.workflow_id == "product.daily"
     )
 
-    assert len(workflow.steps) == 1
-    assert workflow.steps[0].replay_policy == "idempotent"
+    assert "collect,import,analyze,export,insert" in workflow.steps[0].args
+    assert not any(
+        item.workflow_id == "product.import.daily" for item in catalog.workflows
+    )
 
 
 def test_catalog_loads_optional_step_timeout_seconds(tmp_path: Path) -> None: