feat(product-commerce): 款式主键统一 style_id 并下线评价采集

采集端与 data-hub 统一用 style_id (= dim_style.id) 关联款式,style_name 降级为显示冗余。

款式主键:
- 14 张采集表加 style_id BIGINT + style_match_method + *_style_id_idx 索引
- 一律逻辑外键:不建 FOREIGN KEY,早期 10 个 *_style_id_fk 在 schema.sql 里幂等 DROP
- loader 下发 dim_style.id;db 层 attach_style_ids / resolve_style_ids_by_product 兜底
- 新增 backfill_style_id.py 三段回填(款名 / 日报按商品 ID / 广告表按 spu_id)
- 新增视图 v_style_id_coverage、v_style_unmatched_product、v_style_unmatched_source
- 实库补录 199 个商品 ID 到 67 款 spus;日报 style_id 覆盖 49.1% -> 79.0%,
  tm 广告 99.4% / jd 92.4% / dy 83.2%,其余 11 张 100%

评价采集下线:
- product_reviews 归档为 product_reviews_archived_20261101
- 差评改由 shop.{tmall,jd,dy}_negative_reviews.daily 采集,消费方读 v_negative_reviews
- 删除 import_product_reviews.py / orchestrate_review_collection.py 及相关命令与绑定
- dy_product_scraping 默认流程摘除评论采集,保留 --reviews-only

天猫广告:
- 迁移到独立表 tm_product_ad_daily_metrics,不再写 product_daily_metrics.raw_data

同批附带:
- 光合上传支持天猫新店 new_bag(仅黑曜石系列)
- 京东商智双月日历排除 -diff-month 补位格
- 万相主体搜索词覆盖 STYLE_SEARCH_KEYWORDS
- ERP 部分编码失败时用已成功编码销量兜底,避免整款归零
This commit is contained in:
wangyunlong committed 2026-11-01 16:43:27 +08:00
1 parent 622a609800
commit 6f1c193f4b
50 files changed
+3301 -1150

No files matched your search

@@ -426,27 +426,31 @@ def test_platform_artifact_verifier_rejects_an_export_with_no_product_matches(
daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns)
def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
def test_daily_collect_no_longer_runs_review_import(monkeypatch) -> None:
"""product_reviews 已下线 (2026-11-01)。
collect 阶段不得再调评论入库——差评由 shop.jd/dy/tmall_negative_reviews.daily
三个独立工作流采集,消费方读 v_negative_reviews。这条断言守住下线状态,
防止有人顺手把旧链路加回来。
"""
args = SimpleNamespace(
skip_erp=True,
skip_platforms=False,
serial_platforms=False,
dry_run=False,
skip_reviews_import=False,
)
monkeypatch.setattr(daily, "run_platforms", lambda **_kwargs: [])
calls = []
monkeypatch.setattr(
daily,
"run_step_with_retry",
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 1},
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0},
)
daily.run_collect(args, "2026-08-03", ["jd"], failed=None)
assert len(calls) == 1
assert calls[0][0][0] == "import_product_reviews"
assert calls[0][1]["critical"] is False
assert calls == []
def test_daily_import_stage_uses_the_collected_business_date(monkeypatch) -> None:
@@ -310,6 +310,14 @@ def test_database_upsert_uses_date_and_style_conflict_key(monkeypatch) -> None:
def __exit__(self, *_args):
return None
# db.attach_style_ids 会按款名查 dim_style;这里返回空映射,
# 断言的是 SQL 形状与业务列,不依赖真实款式表。
def execute(self, *_args, **_kwargs):
return None
def fetchall(self):
return []
class Connection:
def cursor(self):
return Cursor()
@@ -15,8 +15,8 @@ class ErpMetricOverridesTests(unittest.TestCase):
def tearDown(self):
self.temp_dir.cleanup()
def _write_erp(self, code_results):
data_dir = self.data_root / "dy" / "apollo_2026-07-22"
def _write_erp(self, code_results, platform="dy"):
data_dir = self.data_root / platform / "apollo_2026-07-22"
data_dir.mkdir(parents=True)
(data_dir / "data.json").write_text(
json.dumps(
@@ -33,12 +33,12 @@ class ErpMetricOverridesTests(unittest.TestCase):
encoding="utf-8",
)
def _apply(self, platform_sales, platform_refunds):
def _apply(self, platform_sales, platform_refunds, platform="dy"):
with patch.object(overrides, "DATA_ROOT", self.data_root), patch.dict(
overrides.PLATFORM_DIRS, {"dy": "dy"}
overrides.PLATFORM_DIRS, {platform: platform}
):
return overrides.apply_erp_override(
"dy", "阿波罗x1", 394, 46,
platform, "阿波罗x1", 394, 46,
target_date="2026-07-22",
platform_sales=platform_sales,
platform_refunds=platform_refunds,
@@ -87,6 +87,21 @@ class ErpMetricOverridesTests(unittest.TestCase):
self.assertEqual(result["销量"], 0)
self.assertEqual(result["退款订单数"], 0)
def test_partial_codes_used_when_platform_has_no_sales(self):
"""TM 没有平台销量字段:部分编码失败时不能整款归零,用成功编码合计。"""
self._write_erp([
{"erp_style_code": "1", "ok": True, "yesterday_sales": 3, "yesterday_returns": 1},
{"erp_style_code": "2", "ok": False, "error": "timeout", "yesterday_sales": 0, "yesterday_returns": 0},
], platform="tm")
result = self._apply(platform_sales=None, platform_refunds=None, platform="tm")
self.assertTrue(result["erp_unavailable"])
self.assertTrue(result["erp_partial"])
self.assertEqual(result["销量"], 3)
self.assertEqual(result["退款订单数"], 1)
self.assertEqual(result["failed_codes"], ["2"])
if __name__ == "__main__":
unittest.main()
@@ -3,6 +3,7 @@ import unittest
from upload_video_to_guanghe import (
STORE_FLAGSHIP,
STORE_LUGGAGE,
STORE_NEW_BAG,
TM_UPLOAD_FIELD,
select_pending_records,
)
@@ -113,6 +114,53 @@ class GuangHeStoreRoutingTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "未知光合店铺"):
select_pending_records([record()], "unknown")
def test_new_bag_uploads_only_obsidian_series_regardless_of_tm_status(self):
unchecked = record(gender="女", uploaded=False)
unchecked["代拍产品"] = "黑曜石托特包"
checked = record(gender="女", uploaded=True)
checked["代拍产品"] = "黑曜石吐司包"
records = [
unchecked,
checked,
record(gender="男"),
]
selected = select_pending_records(records, STORE_NEW_BAG)
self.assertEqual(selected, [unchecked, checked])
def test_new_bag_skips_mixed_and_non_obsidian_styles(self):
obsidian = record()
obsidian["代拍产品"] = " 黑曜石电脑包 "
mixed = record()
mixed["代拍产品"] = "黑曜石饺子 / 极星双肩"
other = record()
other["代拍产品"] = "极星双肩"
skipped = record()
skipped["代拍产品"] = "黑曜石饺子 / 未到货"
no_product = record()
other_brand = record(brand="OZKO")
other_brand["代拍产品"] = "黑曜石托特包"
no_attachment = record(attachments=False)
no_attachment["代拍产品"] = "黑曜石托特包"
selected = select_pending_records(
[obsidian, mixed, other, skipped, no_product, other_brand, no_attachment],
STORE_NEW_BAG,
)
self.assertEqual(selected, [obsidian])
def test_new_bag_rejects_republish_from_date(self):
rec = record(date="2026-06-03 00:00:00")
rec["代拍产品"] = "黑曜石托特包"
with self.assertRaisesRegex(ValueError, "旗舰店"):
select_pending_records(
[rec],
STORE_NEW_BAG,
republish_from_date="2026-06-03",
)
if __name__ == "__main__":
unittest.main()
@@ -119,6 +119,9 @@ def test_upsert_uses_advertiser_date_product_business_key(monkeypatch) -> None:
executed.update(sql=sql, rows=rows)
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
# resolve_style_ids 走独立查询;本测试只验证 SQL 形状与款式匹配列,
# 用空映射避免和上面的 style catalog 假数据串台。
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
record = {
"stat_date": date(2026, 8, 23),
"advertiser_id": importer.ADVERTISER_ID,
@@ -162,6 +165,7 @@ def test_upsert_falls_back_to_exact_product_name_style_match(monkeypatch) -> Non
executed.update(sql=sql, rows=rows)
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
record = {
"stat_date": date(2026, 8, 23),
"advertiser_id": importer.ADVERTISER_ID,
@@ -99,7 +99,7 @@ def test_find_report_files_uses_current_manifest_csv(monkeypatch, tmp_path: Path
assert importer.find_report_files(target_date) == [current_csv.resolve()]
def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
def test_upsert_writes_dedicated_table_with_style_and_metrics(monkeypatch) -> None:
captured: dict[str, object] = {}
def fake_execute_values(_cursor, sql, rows, *, template=None):
@@ -108,7 +108,9 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
captured["template"] = template
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
written = product_db.upsert_tmall_product_ad_raw_metrics(
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
written = product_db.upsert_tmall_product_ad_daily_metrics(
_Connection(),
[
{
@@ -119,6 +121,7 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
"spend": "90.25",
"attributed_deal_amount": "730.02",
"roi": "8.09",
"impressions": 1000,
},
"source_file": "商品报表.csv",
}
@@ -127,16 +130,22 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
assert written == 1
sql = str(captured["sql"])
assert "INSERT INTO product_daily_metrics" in sql
assert "jsonb_set" in sql
assert "tm_product_ad_daily_metrics" not in sql
assert captured["template"] == "('tm', %s, %s, %s, %s::jsonb, %s, NOW())"
assert "INSERT INTO tm_product_ad_daily_metrics" in sql
assert "product_daily_metrics" not in sql
assert "jsonb_set" not in sql
row = captured["rows"][0]
payload = json.loads(row[3])
assert payload["tm_ad_metrics"]["spend"] == "90.25"
assert payload["tm_ad_metrics"]["style_name"] == "极星"
assert payload["tm_ad_metrics"]["style_match_method"] == "product_id"
assert row[1] == "1001"
assert row[3] == "极星"
assert row[4] == "product_id"
assert str(row[5]) == "90.25"
assert str(row[8]) == "730.02"
assert str(row[9]) == "8.09"
# 15 个业务列 + style_id(款式主键统一 2026-11-01)
assert captured["template"].count("%s") == 16
assert row[-1] is None # resolve_style_ids 被 mock 成空映射
metrics = json.loads(row[11])
assert metrics["impressions"] == 1000
assert "style_name" not in metrics
def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> None:
@@ -154,7 +163,9 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) ->
captured["rows"] = rows
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
written = product_db.upsert_tmall_product_ad_raw_metrics(
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
written = product_db.upsert_tmall_product_ad_daily_metrics(
_NameFallbackConnection(),
[
{
@@ -169,9 +180,41 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) ->
assert written == 1
row = captured["rows"][0]
payload = json.loads(row[3])
assert payload["tm_ad_metrics"]["style_name"] == "极星"
assert payload["tm_ad_metrics"]["style_match_method"] == "product_name"
assert row[3] == "极星"
assert row[4] == "product_name"
def test_upsert_dedupes_repeated_product_day(monkeypatch) -> None:
captured: dict[str, object] = {}
def fake_execute_values(_cursor, _sql, rows, *, template=None):
captured["rows"] = rows
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
written = product_db.upsert_tmall_product_ad_daily_metrics(
_Connection(),
[
{
"stat_date": importer.date(2026, 8, 17),
"product_id": "1001",
"product_name": "older",
"ad_metrics": {"spend": "1.00"},
},
{
"stat_date": importer.date(2026, 8, 17),
"product_id": "1001",
"product_name": "latest",
"ad_metrics": {"spend": "2.00"},
},
],
)
assert written == 1
row = captured["rows"][0]
assert row[2] == "latest"
assert str(row[5]) == "2.00"
def test_import_date_filter_and_dry_run(tmp_path: Path, capsys) -> None:
@@ -312,7 +312,9 @@ class MainImageDatabaseTests(unittest.TestCase):
}
for platform in ("jd", "tm")
]
with patch.object(product_db, "execute_values") as execute_values:
with patch.object(product_db, "execute_values") as execute_values, patch.object(
product_db, "resolve_style_ids", return_value={}
):
count = product_db.upsert_main_image_creatives(Connection(), records)
self.assertEqual(count, 2)
@@ -12,6 +12,13 @@ class _Cursor:
def __exit__(self, *_args):
return False
# db 层按商品 ID 反查 dim_style 时用;空映射表示这些商品都没建档。
def execute(self, *_args, **_kwargs):
return None
def fetchall(self):
return []
class _Connection:
def cursor(self):
@@ -11,7 +11,6 @@ import collect_persona_to_bitable as tm_persona_worker
import dy_product_scraping as dy_product
import orchestrate_daily_collection as daily
import orchestrate_market_rank_collection as market_rank
import orchestrate_review_collection as reviews
import run_alerts_with_retry as alerts
import run_daily_persona as persona
import run_weekly_jd_main_image as jd_main_image
@@ -191,24 +190,6 @@ def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None:
_assert_child_binding(environment)
def test_review_run_step_rebinds_to_command_target(monkeypatch) -> None:
observed: dict[str, object] = {}
monkeypatch.setattr(reviews, "environment_for_child_script", _fake_rebind(observed))
def fake_run(command, **kwargs):
observed["environment"] = kwargs["env"]
return subprocess.CompletedProcess(command, 0)
monkeypatch.setattr(reviews.subprocess, "run", fake_run)
target = reviews.PROJECT_ROOT / "dy_product_scraping.py"
result = reviews.run_step("dy", ["python", str(target)], critical=True)
assert result["code"] == 0
assert observed["target"] == target
_assert_child_binding(observed["environment"])
def test_main_image_launchers_rebind_to_command_target(monkeypatch) -> None:
for launcher in (tm_main_image, jd_main_image):
observed: dict[str, object] = {}
@@ -116,7 +116,8 @@ class StyleAnalysisWindowTests(unittest.TestCase):
self.assertNotIn("近 60d", prompt)
self.assertIn("逐日经营数据", prompt)
self.assertIn("人物画像", prompt)
self.assertIn("商品评价", prompt)
# product_reviews 下线后数据源改为差评视图,标题同步改成「商品差评」
self.assertIn("商品差评", prompt)
self.assertIn("主图表现", prompt)
self.assertIn("营销笔记、曝光互动与评论", prompt)
self.assertIn("肩带偏硬", prompt)
@@ -73,6 +73,9 @@ def test_database_row_uses_explicit_style_name_not_style_content() -> None:
"series_name": "",
"is_series_anchor": False,
"collect_enabled": True,
# 款式主键统一 (2026-11-01): loader 把 dim_style.id 作为正式字段下发
"style_id": None,
"style_match_method": "",
}
]
@@ -0,0 +1,470 @@
"""款式主键统一 (style_id = dim_style.id) 的契约测试。
守住四件事:
1. 写入端必须把 style_id 落库(13 张采集表);
2. 调用方已带 style_id 时不覆盖,缺省时由 db 层按款名 / 平台商品 ID 兜底;
3. 匹配不上要留痕 ``unmatched``,不能静默写成 NULL 了事;
4. schema.sql 里必须有列、索引、观测视图,且**不得**建物理外键(style_id 只做逻辑外键),
同时不再建已下线的 product_reviews。
全部测试离线运行(Fake connection),不依赖 PostgreSQL。
"""
from __future__ import annotations
from pathlib import Path
import db
import pytest
MODULE_ROOT = (
Path(__file__).resolve().parents[3]
/ "src"
/ "gyxx_flow"
/ "modules"
/ "product_commerce"
)
SCHEMA_PATH = MODULE_ROOT / "db" / "schema.sql"
class RecordingCursor:
def __init__(self, conn: "RecordingConnection") -> None:
self._conn = conn
self._rows: list[tuple] = []
def __enter__(self) -> "RecordingCursor":
return self
def __exit__(self, *_args) -> bool:
return False
def execute(self, sql: str, params=None) -> None:
self._conn.sqls.append((sql, params))
if "unnest" in sql and params:
self._rows = [
(item, self._conn.product_lookup[item])
for item in params[0]
if item in self._conn.product_lookup
]
elif "FROM dim_style" in sql and "ANY" in sql and params:
self._rows = [
(name, self._conn.name_lookup[name])
for name in params[0]
if name in self._conn.name_lookup
]
else:
self._rows = []
def fetchall(self) -> list[tuple]:
return self._rows
def fetchone(self):
return None
@property
def rowcount(self) -> int:
return 0
class RecordingConnection:
def __init__(self, name_lookup=None, product_lookup=None) -> None:
self.name_lookup = dict(name_lookup or {})
self.product_lookup = dict(product_lookup or {})
self.sqls: list[tuple] = []
def cursor(self) -> RecordingCursor:
return RecordingCursor(self)
@pytest.fixture()
def captured(monkeypatch):
"""拦截 execute_values,暴露最终写入的 SQL 与 rows。"""
box: dict[str, object] = {}
def fake_execute_values(_cursor, sql, rows):
box["sql"] = sql
box["rows"] = list(rows)
monkeypatch.setattr(db, "execute_values", fake_execute_values)
return box
def test_attach_style_ids_keeps_supplied_identity() -> None:
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
records = db.attach_style_ids(
conn,
[{"style_name": "盖亚斜挎", "style_id": 42, "style_match_method": "dim_style"}],
)
assert records[0]["style_id"] == 42
assert records[0]["style_match_method"] == "dim_style"
# 已有 style_id 时不应再查库
assert conn.sqls == []
def test_attach_style_ids_fills_missing_and_marks_unmatched() -> None:
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
records = db.attach_style_ids(
conn,
[{"style_name": "盖亚斜挎"}, {"style_name": "未建档款"}],
)
assert records[0]["style_id"] == 7
assert records[0]["style_match_method"] == db.STYLE_MATCH_BY_NAME
assert records[1]["style_id"] is None
assert records[1]["style_match_method"] == db.STYLE_MATCH_UNMATCHED
def test_attach_style_ids_does_not_mutate_caller_records() -> None:
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
original = [{"style_name": "盖亚斜挎"}]
db.attach_style_ids(conn, original)
assert "style_id" not in original[0]
def test_daily_style_metrics_upsert_writes_style_id(captured) -> None:
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
written = db.upsert_daily_metrics(
conn,
[
{
"metric_date": "2026-10-31",
"style_name": "盖亚斜挎",
"platform": "tm",
"sales": 3,
}
],
)
assert written == 1
sql = captured["sql"]
assert "style_id" in sql
assert "style_match_method" in sql
# 冲突时不能把已填的 id 抹成 NULL
assert "COALESCE(EXCLUDED.style_id" in sql
row = captured["rows"][0]
assert row[-2] == 7
assert row[-1] == db.STYLE_MATCH_BY_NAME
def test_erp_all_shop_upsert_writes_style_id(captured) -> None:
conn = RecordingConnection(name_lookup={"极星双肩": 11})
db.upsert_erp_all_shop_style_daily_metrics(
conn,
[
{
"metric_date": "2026-10-31",
"style_name": "极星双肩",
"sales": 10,
"selected_shop_count": 2,
"source": "jushuitan",
}
],
)
row = captured["rows"][0]
assert row[-2] == 11
assert row[-1] == db.STYLE_MATCH_BY_NAME
def test_weekly_metrics_upsert_writes_style_id(captured) -> None:
conn = RecordingConnection(name_lookup={"极星双肩": 11})
db.upsert_weekly_metrics(
conn,
[
{
"week_end": "2026-10-31",
"week_start": "2026-10-25",
"style_name": "极星双肩",
"platform": "tm",
"sales": 5,
}
],
)
assert "style_id" in captured["sql"]
assert captured["rows"][0][-2] == 11
def test_product_daily_metrics_resolves_by_platform_product_id(captured, monkeypatch) -> None:
conn = RecordingConnection(product_lookup={"700001": 33})
written = db.upsert_product_daily_metrics(
conn,
[
{
"platform": "tm",
"stat_date": "2026-10-31",
"product_id": "700001",
"product_name": "GYXX 极星双肩包",
"paid_amount": 100,
},
{
"platform": "tm",
"stat_date": "2026-10-31",
"product_id": "999999",
"product_name": "未建档商品",
"paid_amount": 5,
},
],
)
assert written == 2
rows = captured["rows"]
assert rows[0][-2] == 33
assert rows[0][-1] == db.STYLE_MATCH_BY_PRODUCT
assert rows[1][-2] is None
assert rows[1][-1] == db.STYLE_MATCH_UNMATCHED
def test_product_daily_metrics_keeps_existing_identity_on_conflict(captured) -> None:
conn = RecordingConnection(product_lookup={})
db.upsert_product_daily_metrics(
conn,
[
{
"platform": "jd",
"stat_date": "2026-10-31",
"product_id": "abc",
"paid_amount": 1,
}
],
)
sql = captured["sql"]
assert "COALESCE(EXCLUDED.style_id, product_daily_metrics.style_id)" in sql
def test_ad_tables_write_style_id(captured) -> None:
conn = RecordingConnection(
name_lookup={"盖亚斜挎": 7},
product_lookup={"5001": 7},
)
db.upsert_jd_product_ad_daily_metrics(
conn,
[
{
"stat_date": "2026-10-31",
"advertiser_pin": "pin",
"shop_account": "shop",
"report_type": "spu",
"row_key": "r1",
"style_name": "盖亚斜挎",
"spend": 12,
}
],
)
assert "style_id" in captured["sql"]
assert captured["rows"][0][-2] == 7
db.upsert_douyin_product_ad_daily_metrics(
conn,
[
{
"stat_date": "2026-10-31",
"advertiser_id": "adv",
"product_id": "5001",
"product_name": "盖亚斜挎包",
"style_name": "盖亚斜挎",
"style_match_method": "product_id",
"spend": 8,
}
],
)
assert "style_id" in captured["sql"]
assert captured["rows"][0][-1] == 7
def test_persona_and_main_image_write_style_id(captured) -> None:
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
db.upsert_persona_metrics(
conn,
[
{
"collect_date": "2026-10-31",
"style_name": "盖亚斜挎",
"platform": "tm",
"payload": {},
}
],
)
assert captured["rows"][0][-2] == 7
db.upsert_main_image_creatives(
conn,
[
{
"collect_date": "2026-10-31",
"style_name": "盖亚斜挎",
"image_key": "k1",
}
],
)
assert captured["rows"][0][-2] == 7
@pytest.mark.parametrize(
"table",
[
"daily_style_metrics",
"weekly_style_metrics",
"erp_all_shop_style_daily_metrics",
"daily_persona_metrics",
"main_image_creatives",
"fact_style_analysis_reports",
"decline_events",
"fact_alert_history",
"jd_self_weekly_sales",
"persona_screenshots",
"product_daily_metrics",
"jd_product_ad_daily_metrics",
"dy_product_ad_daily_metrics",
],
)
def test_schema_adds_style_id_column(table: str) -> None:
text = SCHEMA_PATH.read_text(encoding="utf-8")
assert f"ALTER TABLE {table}" in text
assert f"ADD COLUMN IF NOT EXISTS style_id BIGINT" in text
def test_schema_defines_observation_views_and_no_physical_foreign_keys() -> None:
text = SCHEMA_PATH.read_text(encoding="utf-8")
assert "CREATE OR REPLACE VIEW v_style_unmatched_product" in text
assert "CREATE OR REPLACE VIEW v_style_id_coverage" in text
assert "CREATE OR REPLACE VIEW v_negative_reviews" in text
# style_id 只做逻辑外键:库里不得出现指向 dim_style 的物理外键。
# 采集语义是「宁可留 NULL 待回填,也不要整批写入失败」,硬约束与之冲突。
assert "FOREIGN KEY (style_id) REFERENCES dim_style(id)" not in text
assert "REFERENCES dim_style(id)" not in text
# 老库重跑 schema 时要能自己清掉早期建的约束(幂等 DROP)
assert "DROP CONSTRAINT IF EXISTS" in text
assert "_style_id_fk" in text
def test_schema_no_longer_creates_product_reviews() -> None:
text = SCHEMA_PATH.read_text(encoding="utf-8")
assert "CREATE TABLE IF NOT EXISTS product_reviews " not in text
assert "ALTER TABLE product_reviews " not in text
assert "product_reviews_archived_20261101" in text
# ---------------------------------------------------------------- 回填脚本
def test_backfill_covers_three_ad_tables_with_right_id_column() -> None:
"""广告表的 style_name 大量为空,必须按商品 ID 再回填一遍。"""
import backfill_style_id as bf
assert set(bf.AD_PRODUCT_TABLES) == {
"tm_product_ad_daily_metrics",
"jd_product_ad_daily_metrics",
"dy_product_ad_daily_metrics",
}
assert bf.AD_PRODUCT_TABLES["tm_product_ad_daily_metrics"] == ("product_id", "tm")
# 京东广告表的商品列叫 spu_id,不是 product_id
assert bf.AD_PRODUCT_TABLES["jd_product_ad_daily_metrics"] == ("spu_id", "jd")
assert bf.AD_PRODUCT_TABLES["dy_product_ad_daily_metrics"] == ("product_id", "dy")
# 款名表里的广告表项不能删,否则 style_name 有值的那部分就回填不了
for table in bf.AD_PRODUCT_TABLES:
assert table in bf.NAME_TABLES
class _BackfillCursor:
def __init__(self, log: list) -> None:
self._log = log
self._rows: list[tuple] = []
self.rowcount = 0
def __enter__(self) -> "_BackfillCursor":
return self
def __exit__(self, *_args) -> bool:
return False
def execute(self, sql: str, params=None) -> None:
self._log.append((" ".join(sql.split()), params))
if "SELECT DISTINCT" in sql:
self._rows = [("1001",), ("9999",)]
elif "SELECT id, style_name FROM dim_style" in sql:
self._rows = [(7, "赤潮")]
elif sql.lstrip().startswith("UPDATE"):
self.rowcount = 5 if "style_id IS NULL AND style_match_method IS NULL" not in sql else 0
elif "COUNT(*)" in sql:
self._rows = [(10, 2)]
else:
self._rows = []
def fetchall(self) -> list[tuple]:
return self._rows
def fetchone(self) -> tuple:
return self._rows[0] if self._rows else (0, 0)
class _BackfillConn:
def __init__(self) -> None:
self.log: list = []
def cursor(self) -> _BackfillCursor:
return _BackfillCursor(self.log)
def test_backfill_ad_tables_fills_style_id_and_keeps_platform_style_name(monkeypatch) -> None:
"""回填广告表时:style_id 照写,style_name 只在原值为空时补。"""
import backfill_style_id as bf
monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7})
conn = _BackfillConn()
result = bf._backfill_ad_tables_by_product(
conn, ["jd_product_ad_daily_metrics"], dry_run=False
)[0]
updates = [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")]
payload = next(sql for sql in updates if "NULLIF(BTRIM(style_name)" in sql)
assert "COALESCE(NULLIF(BTRIM(style_name), ''), %s)" in payload, (
"style_name 只能补空,不能覆盖平台已给出的值"
)
assert "AND style_id IS NULL" in payload, "幂等:已归属的行不能被改写"
assert "spu_id = %s" in payload, "京东广告表要按 spu_id 反查"
# 匹配上的 1 个 ID 写 style_id,1 个未建档
assert result["updated"] == 5
assert result["unmatched_product_ids"] == 1
# 未建档的显式标 unmatched,供观测视图统计
assert any(
params and params[0] == "unmatched"
for sql, params in conn.log
if sql.lstrip().startswith("UPDATE")
)
def test_backfill_ad_tables_dry_run_does_not_update(monkeypatch) -> None:
import backfill_style_id as bf
monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7})
conn = _BackfillConn()
result = bf._backfill_ad_tables_by_product(
conn, ["dy_product_ad_daily_metrics"], dry_run=True
)[0]
assert result["dry_run"] is True
assert result["planned_keys"] == 1
assert not [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")]
@@ -176,13 +176,22 @@ def test_discovery_creates_one_candidate_per_exact_video_attachment() -> None:
def test_flagship_and_luggage_use_distinct_account_scopes() -> None:
flagship = _one_candidate(store=tmall.STORE_FLAGSHIP)
luggage = _one_candidate(store=tmall.STORE_LUGGAGE)
new_bag = _one_candidate(store=tmall.STORE_NEW_BAG)
assert flagship.target_account_key == "guanghe-flagship"
assert luggage.target_account_key == "guanghe-luggage"
assert flagship.target_account_key != luggage.target_account_key
assert new_bag.target_account_key == "guanghe-new-bag"
assert len(
{
flagship.target_account_key,
luggage.target_account_key,
new_bag.target_account_key,
}
) == 3
assert tmall.STORE_ACCOUNT_KEYS == {
tmall.STORE_FLAGSHIP: flagship.target_account_key,
tmall.STORE_LUGGAGE: luggage.target_account_key,
tmall.STORE_NEW_BAG: new_bag.target_account_key,
}
@@ -199,6 +208,25 @@ def test_luggage_runtime_uses_new_store_account_label_without_changing_password_
assert password == "same-runtime-secret"
def test_new_bag_runtime_uses_its_own_credentials_and_profile(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("GUANGHE_NEW_BAG_USERNAME", "gyxx箱包旗舰店:龙虾仔")
monkeypatch.setenv("GUANGHE_NEW_BAG_PASSWORD", "new-bag-runtime-secret")
store_name, username, password, profile = tmall._store_runtime(tmall.STORE_NEW_BAG)
assert store_name == "gyxx箱包旗舰店:龙虾仔"
assert username == "gyxx箱包旗舰店:龙虾仔"
assert password == "new-bag-runtime-secret"
assert str(profile).endswith("guanghe-new-bag")
assert profile != tmall._store_runtime(tmall.STORE_LUGGAGE)[3]
assert profile != tmall._store_runtime(tmall.STORE_FLAGSHIP)[3]
assert tmall._expected_creator_identity_markers(
tmall.STORE_NEW_BAG, store_name
) == ("GYXX箱包天猫官方",)
def test_account_identity_markers_accept_store_name_with_login_nickname() -> None:
assert tmall.GuangHeUploader._account_identity_markers(
"光影行星鑫华达专卖店:龙虾仔"
@@ -148,5 +148,80 @@ class WanxiangReportTemplateTests(unittest.TestCase):
session.close.assert_called_once()
class WanxiangSearchKeywordTests(unittest.TestCase):
"""搜索词可与款名不同,但款式身份始终是 style_name。"""
def test_obsidian_toast_style_searches_shorter_keyword_first(self):
self.assertEqual(
wanxiang.search_keywords_for("黑曜石吐司包"),
("黑曜石吐司", "黑曜石吐司包"),
)
def test_styles_without_override_keep_their_own_name(self):
self.assertEqual(wanxiang.search_keywords_for("星云2"), ("星云2",))
self.assertEqual(
wanxiang.search_keywords_for("黑曜石托特包"), ("黑曜石托特包",)
)
def test_collect_style_searches_keyword_but_keeps_style_identity(self):
searched: list[str] = []
row = {
"creative_id": "c1",
"impressions": 10,
"clicks": 2,
"cost": 0.5,
"creative_image_url": "https://img/1.jpg",
}
with (
patch.object(
wanxiang,
"search_style_by_subject",
side_effect=lambda _page, keyword: searched.append(keyword),
),
patch.object(wanxiang, "extract_current_page_rows", return_value=[row]),
patch.object(wanxiang, "collect_current_page_image_map", return_value={}),
patch.object(wanxiang, "click_next_page", return_value=False),
patch.object(wanxiang, "aggregate_rows", return_value={}),
):
result = wanxiang.collect_style(
Mock(), "黑曜石吐司包", 1, item_ids=["1022486093557"]
)
self.assertEqual(searched, ["黑曜石吐司"])
self.assertEqual(result["style_name"], "黑曜石吐司包")
self.assertEqual(result["search_keyword"], "黑曜石吐司")
def test_empty_keyword_falls_back_to_original_style_name(self):
state = {"searched": [], "checks": 0}
row = {
"creative_id": "c1",
"impressions": 10,
"clicks": 2,
"cost": 0.5,
"creative_image_url": "https://img/1.jpg",
}
def fake_extract(_page):
state["checks"] += 1
return [] if state["checks"] == 1 else [row]
with (
patch.object(
wanxiang,
"search_style_by_subject",
side_effect=lambda _page, keyword: state["searched"].append(keyword),
),
patch.object(wanxiang, "extract_current_page_rows", side_effect=fake_extract),
patch.object(wanxiang, "collect_current_page_image_map", return_value={}),
patch.object(wanxiang, "click_next_page", return_value=False),
patch.object(wanxiang, "aggregate_rows", return_value={}),
):
result = wanxiang.collect_style(Mock(), "黑曜石吐司包", 1, item_ids=["1022"])
self.assertEqual(state["searched"], ["黑曜石吐司", "黑曜石吐司包"])
self.assertEqual(result["style_name"], "黑曜石吐司包")
self.assertEqual(result["search_keyword"], "黑曜石吐司包")
if __name__ == "__main__":
unittest.main()
+26
View File
@@ -1709,6 +1709,28 @@ def test_tmall_video_trigger_routes_only_the_selected_store(
assert flagship["store"] == "flagship"
assert flagship["topic_keyword"] == "我的夏日焕新清单"
assert launcher.calls[-1]["store"] == "flagship"
launcher.complete("product.video_upload")
new_bag = service.trigger(
"product.video_upload",
{"business_date": "2026-08-11", "store": "new_bag"},
)
assert new_bag["store"] == "new_bag"
assert launcher.calls[-1]["store"] == "new_bag"
def test_tmall_video_trigger_rejects_unknown_store(console_settings: Settings) -> None:
service = WorkflowConsoleService(
console_settings,
launcher=FakeConsoleLauncher(),
)
with pytest.raises(ConsoleRequestError, match="店铺参数无效"):
service.trigger(
"product.video_upload",
{"business_date": "2026-08-11", "store": "obsidian"},
)
@pytest.mark.parametrize("value", [123, True, "x" * 121])
@@ -2824,6 +2846,8 @@ def test_http_home_assets_and_overview_have_security_headers(
assert "强制重跑:忽略幂等跳过与已有结果,重新采集并正式执行" in page
assert '<option value="flagship">光影行星旗舰店</option>' in page
assert '<option value="luggage">光影行星鑫华达专卖店:龙虾仔</option>' in page
assert '<option value="new_bag">' in page
assert "天猫新店 · gyxx箱包旗舰店:龙虾仔" in page
assert 'id="run-tmall-topic-field"' in page
assert 'id="run-tmall-topic-keyword"' in page
assert "存在未上传附件时才登录该店铺账号" in page
@@ -2915,6 +2939,8 @@ def test_http_home_assets_and_overview_have_security_headers(
assert "tmall_topic_keyword" in decoded_asset
assert "syncTmallTopicField" in decoded_asset
assert "光影行星鑫华达专卖店:龙虾仔" in decoded_asset
assert "天猫新店" in decoded_asset
assert 'value: "new_bag"' in decoded_asset
assert "renderCardLiveState" in decoded_asset
assert 'data-live-progress="${completed}/${steps.length}"' in decoded_asset
assert 'data-current-step="${esc(currentNames.join("、"))}"' in decoded_asset
+1
View File
@@ -36,6 +36,7 @@ def test_repository_keeps_the_simplified_public_structure() -> None:
"note-inventory.md",
"plan.md",
"runbook.md",
"style-identity-refactor-plan.md",
"workflow-acceptance-test-report.md",
"workflow-acceptance-test-requirements.md",
"workflow-console.md",
@@ -34,7 +34,6 @@ NESTED_TARGETS = {
"dy_product_scraping.py",
"taobao_sycm_collect.py",
"taobao_sycm_collect_backfill.py",
"import_product_reviews.py",
"aggregate_daily_final.py",
"export_bitable_records.py",
"insert_bitable_records.py",
@@ -88,7 +88,6 @@ def test_runtime_writers_do_not_fall_back_to_cwd_or_source_tree(
[
"collect_jd_market_rank.py",
"collect_sycm_market_rank.py",
"import_product_reviews.py",
"scripts/insert_jd_main_image_records.py",
"scripts/insert_main_image_records.py",
],
+3 -4
View File
@@ -29,6 +29,7 @@ PRODUCTION_PYTHON = {
"backfill_poseidon_sales.py",
"backfill_collect.py",
"backfill_one_day.py",
"backfill_style_id.py",
"check_nine_day_decline.py",
"collect_dy_market_rank.py",
"collect_dy_persona_to_bitable.py",
@@ -48,7 +49,6 @@ PRODUCTION_PYTHON = {
"export_bitable_records.py",
"feishu_doc_native.py",
"import_product_daily.py",
"import_product_reviews.py",
"insert_bitable_records.py",
"jd_main_image_collector.py",
"jd_product_data_collector.py",
@@ -62,7 +62,6 @@ PRODUCTION_PYTHON = {
"market_rank_report_archive.py",
"orchestrate_daily_collection.py",
"orchestrate_market_rank_collection.py",
"orchestrate_review_collection.py",
"reapply_erp_override.py",
"rebuild_market_rank_documents.py",
"run_alerts_with_retry.py",
@@ -121,6 +120,7 @@ LEGACY_TESTS = {
"test_persona_launcher.py",
"test_style_analysis_orchestration.py",
"test_style_config_loader_erp_source.py",
"test_style_id_identity.py",
"test_sycm_market_rank.py",
"test_tm_persona_recovery.py",
"test_wanxiang_report_template.py",
@@ -162,7 +162,6 @@ SUBPROCESS_TARGETS = {
"dy_audience_profile_collect.py",
"dy_product_scraping.py",
"export_bitable_records.py",
"import_product_reviews.py",
"insert_bitable_records.py",
"jd_main_image_collector.py",
"jd_product_data_collector.py",
@@ -195,7 +194,7 @@ def _load_runtime_paths():
def test_product_runtime_contains_the_complete_reviewed_source_snapshot() -> None:
assert len(PRODUCTION_PYTHON) == 63
assert len(PRODUCTION_PYTHON) == 62
assert all((MODULE_ROOT / relative).is_file() for relative in PRODUCTION_PYTHON)
assert all((MODULE_ROOT / relative).is_file() for relative in RESOURCES)
assert all((MIGRATED_TEST_ROOT / name).is_file() for name in LEGACY_TESTS)
+2 -2
View File
@@ -55,8 +55,8 @@ def test_catalog_and_scripts_work_after_project_and_data_roots_move(
assert scripts.returncode == 0, scripts.stderr
assert len(json.loads(workflows.stdout)) == 40
commands = json.loads(scripts.stdout)
assert len(commands) == 62
assert len({row["command_id"] for row in commands}) == 62
assert len(commands) == 61
assert len({row["command_id"] for row in commands}) == 61
assert all(row["legacy_script_id"].startswith(f"{row['module']}:") for row in commands)
+11 -3
View File
@@ -102,10 +102,10 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
data_root=project_root / "var",
)
assert len(scripts.command_ids) == 62
assert len(first.command_ids) == 157
assert len(scripts.command_ids) == 61
assert len(first.command_ids) == 155
bindings = [first.binding_for(command_id) for command_id in first.command_ids]
assert len({binding.cdp_port for binding in bindings}) == 157
assert len({binding.cdp_port for binding in bindings}) == 155
assert all(22000 <= binding.cdp_port <= 22999 for binding in bindings)
assert all(binding.cdp_url.startswith("http://127.0.0.1:") for binding in bindings)
assert bindings == [second.binding_for(item.script_id) for item in bindings]
@@ -131,6 +131,8 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
"GUANGHE_PASSWORD",
"GUANGHE_LUGGAGE_USERNAME",
"GUANGHE_LUGGAGE_PASSWORD",
"GUANGHE_NEW_BAG_USERNAME",
"GUANGHE_NEW_BAG_PASSWORD",
)
video_environment = first.environment_for("product.video_upload.run", {})
assert video_environment["GUANGHE_USER_DATA_DIR"] == str(
@@ -142,6 +144,12 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
assert video_environment["GUANGHE_LUGGAGE_USER_DATA_DIR"].endswith(
"-luggage"
)
assert video_environment["GUANGHE_NEW_BAG_USER_DATA_DIR"].endswith(
"-new-bag"
)
assert video_environment["GUANGHE_NEW_BAG_USER_DATA_DIR"] != (
video_environment["GUANGHE_LUGGAGE_USER_DATA_DIR"]
)
jd_video_upload = first.binding_for("product.jd_video_upload.run")
assert jd_video_upload is first.binding_for(
"product_commerce:upload_video_to_jd.py"
+2 -7
View File
@@ -83,7 +83,7 @@ def test_default_registry_covers_every_executable_workflow_entry() -> None:
assert scheduled_workflows == 40
assert manual_workflows == 0
assert len(workflow_entries) == 56
assert len(catalog.scripts) == 62
assert len(catalog.scripts) == 61
assert workflow_entries < set(catalog.script_ids)
assert {
"content.mapping.rebuild",
@@ -97,7 +97,6 @@ def test_default_registry_covers_every_executable_workflow_entry() -> None:
"shop.douyin_after_sales_quality.collect",
"product.backfill.run",
"product.erp_all_shop_daily",
"product.review.orchestrate",
"product.video_upload.run",
"product.jd_video_upload.run",
"supply.workflow.run",
@@ -147,10 +146,6 @@ def test_default_registry_covers_every_executable_workflow_entry() -> None:
"--to",
"{business_date}",
)
assert catalog.get("product.review.orchestrate").default_args == (
"--target-date",
"{business_date}",
)
assert catalog.get("product.import.tmall_ads").default_args == (
"--date",
"{business_date}",
@@ -280,7 +275,7 @@ def test_cli_lists_and_dry_runs_registered_commands(tmp_path: Path) -> None:
assert main(["scripts", "list", "--json"], settings=settings, stdout=output) == 0
rows = json.loads(output.getvalue())
assert len(rows) == 62
assert len(rows) == 61
assert {row["module"] for row in rows} == {
"content_marketing",
"product_commerce",
+1 -1
View File
@@ -57,6 +57,6 @@ def test_every_workflow_and_manual_script_resolves_inside_new_project() -> None:
assert target.is_relative_to(PROJECT_ROOT)
commands = ScriptCatalog.load_default()
assert len(commands.scripts) == 62
assert len(commands.scripts) == 61
assert len(commands.command_ids) == len(set(commands.command_ids))
assert all(command.path.is_relative_to(PROJECT_ROOT) for command in commands.scripts)