wangyunlong
6f1c193f4b
采集端与 data-hub 统一用 style_id (= dim_style.id) 关联款式，style_name 降级为显示冗余。
款式主键：
- 14 张采集表加 style_id BIGINT + style_match_method + *_style_id_idx 索引
- 一律逻辑外键：不建 FOREIGN KEY，早期 10 个 *_style_id_fk 在 schema.sql 里幂等 DROP
- loader 下发 dim_style.id；db 层 attach_style_ids / resolve_style_ids_by_product 兜底
- 新增 backfill_style_id.py 三段回填（款名 / 日报按商品 ID / 广告表按 spu_id）
- 新增视图 v_style_id_coverage、v_style_unmatched_product、v_style_unmatched_source
- 实库补录 199 个商品 ID 到 67 款 spus；日报 style_id 覆盖 49.1% -> 79.0%，
tm 广告 99.4% / jd 92.4% / dy 83.2%，其余 11 张 100%
评价采集下线：
- product_reviews 归档为 product_reviews_archived_20261101
- 差评改由 shop.{tmall,jd,dy}_negative_reviews.daily 采集，消费方读 v_negative_reviews
- 删除 import_product_reviews.py / orchestrate_review_collection.py 及相关命令与绑定
- dy_product_scraping 默认流程摘除评论采集，保留 --reviews-only
天猫广告：
- 迁移到独立表 tm_product_ad_daily_metrics，不再写 product_daily_metrics.raw_data
同批附带：
- 光合上传支持天猫新店 new_bag（仅黑曜石系列）
- 京东商智双月日历排除 -diff-month 补位格
- 万相主体搜索词覆盖 STYLE_SEARCH_KEYWORDS
- ERP 部分编码失败时用已成功编码销量兜底，避免整款归零
86 lines
3.2 KiB
Python
86 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from gyxx_flow.adapters import environment_for_child_script
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
MODULE_ROOT = PROJECT_ROOT / "src" / "gyxx_flow" / "modules"
|
|
|
|
NESTED_TARGETS = {
|
|
"content_marketing": (
|
|
"bilibili_scraper.py",
|
|
"pgy_xhs_scraper_v2.py",
|
|
"xingtu_scraper_v2.py",
|
|
"self_douyin_scraper.py",
|
|
),
|
|
"shop_intelligence": (
|
|
"collectors/jd_data_collector.py",
|
|
"collectors/jd_peer_store_data_collector.py",
|
|
"collectors/dy_store_competitor_store_scraping.py",
|
|
"collectors/taobao_sycm.py",
|
|
"collectors/jd_self_operated_brand_daily.py",
|
|
"collectors/jd_self_operated_product_daily.py",
|
|
),
|
|
"product_commerce": (
|
|
"orchestrate_daily_collection.py",
|
|
"collect_persona_to_bitable.py",
|
|
"collect_dy_persona_to_bitable.py",
|
|
"collect_jd_persona_to_bitable.py",
|
|
"taobao_dmp_item_crowd_insight_screenshots.py",
|
|
"dy_audience_profile_collect.py",
|
|
"collect_erp_yesterday_metrics.py",
|
|
"jd_product_data_collector.py",
|
|
"dy_product_scraping.py",
|
|
"taobao_sycm_collect.py",
|
|
"taobao_sycm_collect_backfill.py",
|
|
"aggregate_daily_final.py",
|
|
"export_bitable_records.py",
|
|
"insert_bitable_records.py",
|
|
"analyze_style.py",
|
|
"check_nine_day_decline.py",
|
|
"collect_sycm_market_rank.py",
|
|
"collect_jd_market_rank.py",
|
|
"collect_dy_market_rank.py",
|
|
"taobao_wanxiang_ai_creative_report.py",
|
|
"scripts/insert_main_image_records.py",
|
|
"jd_main_image_collector.py",
|
|
"scripts/insert_jd_main_image_records.py",
|
|
),
|
|
}
|
|
|
|
|
|
def test_nested_workflow_targets_have_complete_unique_browser_bindings(tmp_path) -> None:
|
|
bindings: list[dict[str, str]] = []
|
|
for module, entries in NESTED_TARGETS.items():
|
|
module_root = MODULE_ROOT / module
|
|
base_environment = {
|
|
"GYXX_MODULE_ID": module,
|
|
"GYXX_MODULE_ROOT": str(module_root),
|
|
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
|
|
"GYXX_DATA_ROOT": str(tmp_path / "portable-data"),
|
|
}
|
|
for entry in entries:
|
|
environment = environment_for_child_script(
|
|
module_root / entry,
|
|
base_environment,
|
|
)
|
|
assert environment["GYXX_SCRIPT_ID"] == f"{module}:{entry}"
|
|
for key in (
|
|
"GYXX_BROWSER_CDP_PORT",
|
|
"GYXX_BROWSER_CDP_URL",
|
|
"GYXX_BROWSER_PROFILE_DIR",
|
|
"GYXX_BROWSER_COOKIE_FILE",
|
|
"GYXX_BROWSER_STORAGE_STATE_FILE",
|
|
):
|
|
assert environment[key]
|
|
bindings.append(environment)
|
|
|
|
assert len({item["GYXX_BROWSER_CDP_PORT"] for item in bindings}) == len(bindings)
|
|
assert len({item["GYXX_BROWSER_CDP_URL"] for item in bindings}) == len(bindings)
|
|
assert len({item["GYXX_BROWSER_PROFILE_DIR"] for item in bindings}) == len(bindings)
|
|
assert len({item["GYXX_BROWSER_COOKIE_FILE"] for item in bindings}) == len(bindings)
|
|
assert len(
|
|
{item["GYXX_BROWSER_STORAGE_STATE_FILE"] for item in bindings}
|
|
) == len(bindings)
|