feat: complete production workflow migration

2026-08-06 14:29:57 +08:00
parent 7f215e79c4
commit 8df5266abb
448 changed files with 56937 additions and 14619 deletions
@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from gyxx_flow.adapters import environment_for_child_script
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+MODULE_ROOT = PROJECT_ROOT / "src" / "gyxx_flow" / "modules"
+
+NESTED_TARGETS = {
+    "content_marketing": (
+        "bilibili_scraper.py",
+        "pgy_xhs_scraper_v2.py",
+        "xingtu_scraper_v2.py",
+        "self_douyin_scraper.py",
+    ),
+    "shop_intelligence": (
+        "collectors/jd_data_collector.py",
+        "collectors/jd_peer_store_data_collector.py",
+        "collectors/dy_store_competitor_store_scraping.py",
+        "collectors/taobao_sycm.py",
+        "collectors/jd_self_operated_brand_daily.py",
+        "collectors/jd_self_operated_product_daily.py",
+    ),
+    "product_commerce": (
+        "orchestrate_daily_collection.py",
+        "collect_persona_to_bitable.py",
+        "collect_dy_persona_to_bitable.py",
+        "collect_jd_persona_to_bitable.py",
+        "taobao_dmp_item_crowd_insight_screenshots.py",
+        "dy_audience_profile_collect.py",
+        "collect_erp_yesterday_metrics.py",
+        "jd_product_data_collector.py",
+        "dy_product_scraping.py",
+        "taobao_sycm_collect.py",
+        "taobao_sycm_collect_backfill.py",
+        "import_product_reviews.py",
+        "aggregate_daily_final.py",
+        "export_bitable_records.py",
+        "insert_bitable_records.py",
+        "analyze_style_with_hermes.py",
+        "check_nine_day_decline.py",
+        "collect_sycm_market_rank.py",
+        "collect_jd_market_rank.py",
+        "collect_dy_market_rank.py",
+        "taobao_wanxiang_ai_creative_report.py",
+        "scripts/insert_main_image_records.py",
+        "jd_main_image_collector.py",
+        "scripts/insert_jd_main_image_records.py",
+    ),
+}
+
+
+def test_nested_workflow_targets_have_complete_unique_browser_bindings(tmp_path) -> None:
+    bindings: list[dict[str, str]] = []
+    for module, entries in NESTED_TARGETS.items():
+        module_root = MODULE_ROOT / module
+        base_environment = {
+            "GYXX_MODULE_ID": module,
+            "GYXX_MODULE_ROOT": str(module_root),
+            "GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
+            "GYXX_DATA_ROOT": str(tmp_path / "portable-data"),
+        }
+        for entry in entries:
+            environment = environment_for_child_script(
+                module_root / entry,
+                base_environment,
+            )
+            assert environment["GYXX_SCRIPT_ID"] == f"{module}:{entry}"
+            for key in (
+                "GYXX_BROWSER_CDP_PORT",
+                "GYXX_BROWSER_CDP_URL",
+                "GYXX_BROWSER_PROFILE_DIR",
+                "GYXX_BROWSER_COOKIE_FILE",
+                "GYXX_BROWSER_STORAGE_STATE_FILE",
+            ):
+                assert environment[key]
+            bindings.append(environment)
+
+    assert len({item["GYXX_BROWSER_CDP_PORT"] for item in bindings}) == len(bindings)
+    assert len({item["GYXX_BROWSER_CDP_URL"] for item in bindings}) == len(bindings)
+    assert len({item["GYXX_BROWSER_PROFILE_DIR"] for item in bindings}) == len(bindings)
+    assert len({item["GYXX_BROWSER_COOKIE_FILE"] for item in bindings}) == len(bindings)
+    assert len(
+        {item["GYXX_BROWSER_STORAGE_STATE_FILE"] for item in bindings}
+    ) == len(bindings)