87 lines
3.2 KiB
Python
87 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from gyxx_flow.adapters import environment_for_child_script
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
MODULE_ROOT = PROJECT_ROOT / "src" / "gyxx_flow" / "modules"
|
|
|
|
NESTED_TARGETS = {
|
|
"content_marketing": (
|
|
"bilibili_scraper.py",
|
|
"pgy_xhs_scraper_v2.py",
|
|
"xingtu_scraper_v2.py",
|
|
"self_douyin_scraper.py",
|
|
),
|
|
"shop_intelligence": (
|
|
"collectors/jd_data_collector.py",
|
|
"collectors/jd_peer_store_data_collector.py",
|
|
"collectors/dy_store_competitor_store_scraping.py",
|
|
"collectors/taobao_sycm.py",
|
|
"collectors/jd_self_operated_brand_daily.py",
|
|
"collectors/jd_self_operated_product_daily.py",
|
|
),
|
|
"product_commerce": (
|
|
"orchestrate_daily_collection.py",
|
|
"collect_persona_to_bitable.py",
|
|
"collect_dy_persona_to_bitable.py",
|
|
"collect_jd_persona_to_bitable.py",
|
|
"taobao_dmp_item_crowd_insight_screenshots.py",
|
|
"dy_audience_profile_collect.py",
|
|
"collect_erp_yesterday_metrics.py",
|
|
"jd_product_data_collector.py",
|
|
"dy_product_scraping.py",
|
|
"taobao_sycm_collect.py",
|
|
"taobao_sycm_collect_backfill.py",
|
|
"import_product_reviews.py",
|
|
"aggregate_daily_final.py",
|
|
"export_bitable_records.py",
|
|
"insert_bitable_records.py",
|
|
"analyze_style_with_hermes.py",
|
|
"check_nine_day_decline.py",
|
|
"collect_sycm_market_rank.py",
|
|
"collect_jd_market_rank.py",
|
|
"collect_dy_market_rank.py",
|
|
"taobao_wanxiang_ai_creative_report.py",
|
|
"scripts/insert_main_image_records.py",
|
|
"jd_main_image_collector.py",
|
|
"scripts/insert_jd_main_image_records.py",
|
|
),
|
|
}
|
|
|
|
|
|
def test_nested_workflow_targets_have_complete_unique_browser_bindings(tmp_path) -> None:
|
|
bindings: list[dict[str, str]] = []
|
|
for module, entries in NESTED_TARGETS.items():
|
|
module_root = MODULE_ROOT / module
|
|
base_environment = {
|
|
"GYXX_MODULE_ID": module,
|
|
"GYXX_MODULE_ROOT": str(module_root),
|
|
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
|
|
"GYXX_DATA_ROOT": str(tmp_path / "portable-data"),
|
|
}
|
|
for entry in entries:
|
|
environment = environment_for_child_script(
|
|
module_root / entry,
|
|
base_environment,
|
|
)
|
|
assert environment["GYXX_SCRIPT_ID"] == f"{module}:{entry}"
|
|
for key in (
|
|
"GYXX_BROWSER_CDP_PORT",
|
|
"GYXX_BROWSER_CDP_URL",
|
|
"GYXX_BROWSER_PROFILE_DIR",
|
|
"GYXX_BROWSER_COOKIE_FILE",
|
|
"GYXX_BROWSER_STORAGE_STATE_FILE",
|
|
):
|
|
assert environment[key]
|
|
bindings.append(environment)
|
|
|
|
assert len({item["GYXX_BROWSER_CDP_PORT"] for item in bindings}) == len(bindings)
|
|
assert len({item["GYXX_BROWSER_CDP_URL"] for item in bindings}) == len(bindings)
|
|
assert len({item["GYXX_BROWSER_PROFILE_DIR"] for item in bindings}) == len(bindings)
|
|
assert len({item["GYXX_BROWSER_COOKIE_FILE"] for item in bindings}) == len(bindings)
|
|
assert len(
|
|
{item["GYXX_BROWSER_STORAGE_STATE_FILE"] for item in bindings}
|
|
) == len(bindings)
|