wangyunlong
6f1c193f4b
采集端与 data-hub 统一用 style_id (= dim_style.id) 关联款式，style_name 降级为显示冗余。
款式主键：
- 14 张采集表加 style_id BIGINT + style_match_method + *_style_id_idx 索引
- 一律逻辑外键：不建 FOREIGN KEY，早期 10 个 *_style_id_fk 在 schema.sql 里幂等 DROP
- loader 下发 dim_style.id；db 层 attach_style_ids / resolve_style_ids_by_product 兜底
- 新增 backfill_style_id.py 三段回填（款名 / 日报按商品 ID / 广告表按 spu_id）
- 新增视图 v_style_id_coverage、v_style_unmatched_product、v_style_unmatched_source
- 实库补录 199 个商品 ID 到 67 款 spus；日报 style_id 覆盖 49.1% -> 79.0%，
tm 广告 99.4% / jd 92.4% / dy 83.2%，其余 11 张 100%
评价采集下线：
- product_reviews 归档为 product_reviews_archived_20261101
- 差评改由 shop.{tmall,jd,dy}_negative_reviews.daily 采集，消费方读 v_negative_reviews
- 删除 import_product_reviews.py / orchestrate_review_collection.py 及相关命令与绑定
- dy_product_scraping 默认流程摘除评论采集，保留 --reviews-only
天猫广告：
- 迁移到独立表 tm_product_ad_daily_metrics，不再写 product_daily_metrics.raw_data
同批附带：
- 光合上传支持天猫新店 new_bag（仅黑曜石系列）
- 京东商智双月日历排除 -diff-month 补位格
- 万相主体搜索词覆盖 STYLE_SEARCH_KEYWORDS
- ERP 部分编码失败时用已成功编码销量兜底，避免整款归零
63 lines
2.1 KiB
Python
63 lines
2.1 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
from gyxx_flow.catalog import WorkflowCatalog
|
|
from gyxx_flow.script_catalog import ScriptCatalog
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
MODULES = (
|
|
"content_marketing",
|
|
"product_commerce",
|
|
"shop_intelligence",
|
|
"supply_chain",
|
|
)
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def test_all_source_manifests_use_one_schema_and_verify_target_hashes() -> None:
|
|
for module in MODULES:
|
|
path = PROJECT_ROOT / "config" / "source-manifests" / f"{module}.json"
|
|
manifest = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
assert manifest["schema_version"] == 1
|
|
assert manifest["module"] == module
|
|
assert isinstance(manifest["snapshot"], str) and manifest["snapshot"]
|
|
assert manifest["files"]
|
|
assert isinstance(manifest["intentionally_excluded"], list)
|
|
for item in manifest["files"]:
|
|
target = PROJECT_ROOT.joinpath(
|
|
*PurePosixPath(item["target_relative_path"]).parts
|
|
)
|
|
assert target.is_file(), target
|
|
assert _sha256(target) == item["target_sha256"], target
|
|
|
|
|
|
def test_every_workflow_and_manual_script_resolves_inside_new_project() -> None:
|
|
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
|
for workflow in catalog.workflows:
|
|
if workflow.trigger == "unavailable":
|
|
continue
|
|
target = (
|
|
PROJECT_ROOT
|
|
/ "src"
|
|
/ "gyxx_flow"
|
|
/ "modules"
|
|
/ workflow.module
|
|
/ workflow.entry
|
|
).resolve(strict=True)
|
|
assert target.is_relative_to(PROJECT_ROOT)
|
|
|
|
commands = ScriptCatalog.load_default()
|
|
assert len(commands.scripts) == 61
|
|
assert len(commands.command_ids) == len(set(commands.command_ids))
|
|
assert all(command.path.is_relative_to(PROJECT_ROOT) for command in commands.scripts)
|