wangyunlong
6f1c193f4b
采集端与 data-hub 统一用 style_id (= dim_style.id) 关联款式，style_name 降级为显示冗余。
款式主键：
- 14 张采集表加 style_id BIGINT + style_match_method + *_style_id_idx 索引
- 一律逻辑外键：不建 FOREIGN KEY，早期 10 个 *_style_id_fk 在 schema.sql 里幂等 DROP
- loader 下发 dim_style.id；db 层 attach_style_ids / resolve_style_ids_by_product 兜底
- 新增 backfill_style_id.py 三段回填（款名 / 日报按商品 ID / 广告表按 spu_id）
- 新增视图 v_style_id_coverage、v_style_unmatched_product、v_style_unmatched_source
- 实库补录 199 个商品 ID 到 67 款 spus；日报 style_id 覆盖 49.1% -> 79.0%，
tm 广告 99.4% / jd 92.4% / dy 83.2%，其余 11 张 100%
评价采集下线：
- product_reviews 归档为 product_reviews_archived_20261101
- 差评改由 shop.{tmall,jd,dy}_negative_reviews.daily 采集，消费方读 v_negative_reviews
- 删除 import_product_reviews.py / orchestrate_review_collection.py 及相关命令与绑定
- dy_product_scraping 默认流程摘除评论采集，保留 --reviews-only
天猫广告：
- 迁移到独立表 tm_product_ad_daily_metrics，不再写 product_daily_metrics.raw_data
同批附带：
- 光合上传支持天猫新店 new_bag（仅黑曜石系列）
- 京东商智双月日历排除 -diff-month 补位格
- 万相主体搜索词覆盖 STYLE_SEARCH_KEYWORDS
- ERP 部分编码失败时用已成功编码销量兜底，避免整款归零
85 lines
2.3 KiB
Python
85 lines
2.3 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def _run_relocated(
|
|
relocated: Path, data_root: Path, *args: str
|
|
) -> subprocess.CompletedProcess[str]:
|
|
env = os.environ.copy()
|
|
for name in tuple(env):
|
|
if name.startswith("GYXX_LEGACY_"):
|
|
env.pop(name)
|
|
env["PYTHONPATH"] = str(relocated / "src")
|
|
env["GYXX_DATA_ROOT"] = str(data_root)
|
|
return subprocess.run(
|
|
[sys.executable, "-m", "gyxx_flow", *args],
|
|
cwd=relocated,
|
|
env=env,
|
|
text=True,
|
|
encoding="utf-8",
|
|
capture_output=True,
|
|
check=False,
|
|
timeout=60,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def relocated_project(tmp_path: Path) -> Path:
|
|
relocated = tmp_path / "moved-gyxx-flow"
|
|
shutil.copytree(PROJECT_ROOT / "src", relocated / "src")
|
|
shutil.copytree(PROJECT_ROOT / "config", relocated / "config")
|
|
return relocated
|
|
|
|
|
|
def test_catalog_and_scripts_work_after_project_and_data_roots_move(
|
|
relocated_project: Path, tmp_path: Path
|
|
) -> None:
|
|
data_root = tmp_path / "independent-data"
|
|
|
|
workflows = _run_relocated(relocated_project, data_root, "list", "--json")
|
|
scripts = _run_relocated(
|
|
relocated_project, data_root, "scripts", "list", "--json"
|
|
)
|
|
|
|
assert workflows.returncode == 0, workflows.stderr
|
|
assert scripts.returncode == 0, scripts.stderr
|
|
assert len(json.loads(workflows.stdout)) == 40
|
|
commands = json.loads(scripts.stdout)
|
|
assert len(commands) == 61
|
|
assert len({row["command_id"] for row in commands}) == 61
|
|
assert all(row["legacy_script_id"].startswith(f"{row['module']}:") for row in commands)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"workflow_id",
|
|
(
|
|
"content.metrics.daily",
|
|
"product.daily",
|
|
"shop.metrics.weekly",
|
|
"supply.purchase_confirmation.daily",
|
|
),
|
|
)
|
|
def test_each_relocated_module_dry_runs_without_old_projects(
|
|
relocated_project: Path, tmp_path: Path, workflow_id: str
|
|
) -> None:
|
|
result = _run_relocated(
|
|
relocated_project,
|
|
tmp_path / "independent-data",
|
|
"run",
|
|
workflow_id,
|
|
"--date",
|
|
"2026-07-27",
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|