feat(product-commerce): 款式主键统一 style_id 并下线评价采集
采集端与 data-hub 统一用 style_id (= dim_style.id) 关联款式,style_name 降级为显示冗余。
款式主键:
- 14 张采集表加 style_id BIGINT + style_match_method + *_style_id_idx 索引
- 一律逻辑外键:不建 FOREIGN KEY,早期 10 个 *_style_id_fk 在 schema.sql 里幂等 DROP
- loader 下发 dim_style.id;db 层 attach_style_ids / resolve_style_ids_by_product 兜底
- 新增 backfill_style_id.py 三段回填(款名 / 日报按商品 ID / 广告表按 spu_id)
- 新增视图 v_style_id_coverage、v_style_unmatched_product、v_style_unmatched_source
- 实库补录 199 个商品 ID 到 67 款 spus;日报 style_id 覆盖 49.1% -> 79.0%,
tm 广告 99.4% / jd 92.4% / dy 83.2%,其余 11 张 100%
评价采集下线:
- product_reviews 归档为 product_reviews_archived_20261101
- 差评改由 shop.{tmall,jd,dy}_negative_reviews.daily 采集,消费方读 v_negative_reviews
- 删除 import_product_reviews.py / orchestrate_review_collection.py 及相关命令与绑定
- dy_product_scraping 默认流程摘除评论采集,保留 --reviews-only
天猫广告:
- 迁移到独立表 tm_product_ad_daily_metrics,不再写 product_daily_metrics.raw_data
同批附带:
- 光合上传支持天猫新店 new_bag(仅黑曜石系列)
- 京东商智双月日历排除 -diff-month 补位格
- 万相主体搜索词覆盖 STYLE_SEARCH_KEYWORDS
- ERP 部分编码失败时用已成功编码销量兜底,避免整款归零
This commit is contained in:
1 parent
622a609800
commit
6f1c193f4b
50 files changed
+3301
-1150
No files matched your search
@@ -426,27 +426,31 @@ def test_platform_artifact_verifier_rejects_an_export_with_no_product_matches(
|
||||
daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns)
|
||||
|
||||
|
||||
def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
|
||||
def test_daily_collect_no_longer_runs_review_import(monkeypatch) -> None:
|
||||
"""product_reviews 已下线 (2026-11-01)。
|
||||
|
||||
collect 阶段不得再调评论入库——差评由 shop.jd/dy/tmall_negative_reviews.daily
|
||||
三个独立工作流采集,消费方读 v_negative_reviews。这条断言守住下线状态,
|
||||
防止有人顺手把旧链路加回来。
|
||||
"""
|
||||
|
||||
args = SimpleNamespace(
|
||||
skip_erp=True,
|
||||
skip_platforms=False,
|
||||
serial_platforms=False,
|
||||
dry_run=False,
|
||||
skip_reviews_import=False,
|
||||
)
|
||||
monkeypatch.setattr(daily, "run_platforms", lambda **_kwargs: [])
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 1},
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0},
|
||||
)
|
||||
|
||||
daily.run_collect(args, "2026-08-03", ["jd"], failed=None)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0][0] == "import_product_reviews"
|
||||
assert calls[0][1]["critical"] is False
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_daily_import_stage_uses_the_collected_business_date(monkeypatch) -> None:
|
||||
|
||||
@@ -310,6 +310,14 @@ def test_database_upsert_uses_date_and_style_conflict_key(monkeypatch) -> None:
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
# db.attach_style_ids 会按款名查 dim_style;这里返回空映射,
|
||||
# 断言的是 SQL 形状与业务列,不依赖真实款式表。
|
||||
def execute(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
return []
|
||||
|
||||
class Connection:
|
||||
def cursor(self):
|
||||
return Cursor()
|
||||
|
||||
@@ -15,8 +15,8 @@ class ErpMetricOverridesTests(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
self.temp_dir.cleanup()
|
||||
|
||||
def _write_erp(self, code_results):
|
||||
data_dir = self.data_root / "dy" / "apollo_2026-07-22"
|
||||
def _write_erp(self, code_results, platform="dy"):
|
||||
data_dir = self.data_root / platform / "apollo_2026-07-22"
|
||||
data_dir.mkdir(parents=True)
|
||||
(data_dir / "data.json").write_text(
|
||||
json.dumps(
|
||||
@@ -33,12 +33,12 @@ class ErpMetricOverridesTests(unittest.TestCase):
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _apply(self, platform_sales, platform_refunds):
|
||||
def _apply(self, platform_sales, platform_refunds, platform="dy"):
|
||||
with patch.object(overrides, "DATA_ROOT", self.data_root), patch.dict(
|
||||
overrides.PLATFORM_DIRS, {"dy": "dy"}
|
||||
overrides.PLATFORM_DIRS, {platform: platform}
|
||||
):
|
||||
return overrides.apply_erp_override(
|
||||
"dy", "阿波罗x1", 394, 46,
|
||||
platform, "阿波罗x1", 394, 46,
|
||||
target_date="2026-07-22",
|
||||
platform_sales=platform_sales,
|
||||
platform_refunds=platform_refunds,
|
||||
@@ -87,6 +87,21 @@ class ErpMetricOverridesTests(unittest.TestCase):
|
||||
self.assertEqual(result["销量"], 0)
|
||||
self.assertEqual(result["退款订单数"], 0)
|
||||
|
||||
def test_partial_codes_used_when_platform_has_no_sales(self):
|
||||
"""TM 没有平台销量字段:部分编码失败时不能整款归零,用成功编码合计。"""
|
||||
self._write_erp([
|
||||
{"erp_style_code": "1", "ok": True, "yesterday_sales": 3, "yesterday_returns": 1},
|
||||
{"erp_style_code": "2", "ok": False, "error": "timeout", "yesterday_sales": 0, "yesterday_returns": 0},
|
||||
], platform="tm")
|
||||
|
||||
result = self._apply(platform_sales=None, platform_refunds=None, platform="tm")
|
||||
|
||||
self.assertTrue(result["erp_unavailable"])
|
||||
self.assertTrue(result["erp_partial"])
|
||||
self.assertEqual(result["销量"], 3)
|
||||
self.assertEqual(result["退款订单数"], 1)
|
||||
self.assertEqual(result["failed_codes"], ["2"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -3,6 +3,7 @@ import unittest
|
||||
from upload_video_to_guanghe import (
|
||||
STORE_FLAGSHIP,
|
||||
STORE_LUGGAGE,
|
||||
STORE_NEW_BAG,
|
||||
TM_UPLOAD_FIELD,
|
||||
select_pending_records,
|
||||
)
|
||||
@@ -113,6 +114,53 @@ class GuangHeStoreRoutingTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "未知光合店铺"):
|
||||
select_pending_records([record()], "unknown")
|
||||
|
||||
def test_new_bag_uploads_only_obsidian_series_regardless_of_tm_status(self):
|
||||
unchecked = record(gender="女", uploaded=False)
|
||||
unchecked["代拍产品"] = "黑曜石托特包"
|
||||
checked = record(gender="女", uploaded=True)
|
||||
checked["代拍产品"] = "黑曜石吐司包"
|
||||
records = [
|
||||
unchecked,
|
||||
checked,
|
||||
record(gender="男"),
|
||||
]
|
||||
|
||||
selected = select_pending_records(records, STORE_NEW_BAG)
|
||||
|
||||
self.assertEqual(selected, [unchecked, checked])
|
||||
|
||||
def test_new_bag_skips_mixed_and_non_obsidian_styles(self):
|
||||
obsidian = record()
|
||||
obsidian["代拍产品"] = " 黑曜石电脑包 "
|
||||
mixed = record()
|
||||
mixed["代拍产品"] = "黑曜石饺子 / 极星双肩"
|
||||
other = record()
|
||||
other["代拍产品"] = "极星双肩"
|
||||
skipped = record()
|
||||
skipped["代拍产品"] = "黑曜石饺子 / 未到货"
|
||||
no_product = record()
|
||||
other_brand = record(brand="OZKO")
|
||||
other_brand["代拍产品"] = "黑曜石托特包"
|
||||
no_attachment = record(attachments=False)
|
||||
no_attachment["代拍产品"] = "黑曜石托特包"
|
||||
|
||||
selected = select_pending_records(
|
||||
[obsidian, mixed, other, skipped, no_product, other_brand, no_attachment],
|
||||
STORE_NEW_BAG,
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [obsidian])
|
||||
|
||||
def test_new_bag_rejects_republish_from_date(self):
|
||||
rec = record(date="2026-06-03 00:00:00")
|
||||
rec["代拍产品"] = "黑曜石托特包"
|
||||
with self.assertRaisesRegex(ValueError, "旗舰店"):
|
||||
select_pending_records(
|
||||
[rec],
|
||||
STORE_NEW_BAG,
|
||||
republish_from_date="2026-06-03",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -119,6 +119,9 @@ def test_upsert_uses_advertiser_date_product_business_key(monkeypatch) -> None:
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
# resolve_style_ids 走独立查询;本测试只验证 SQL 形状与款式匹配列,
|
||||
# 用空映射避免和上面的 style catalog 假数据串台。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
@@ -162,6 +165,7 @@ def test_upsert_falls_back_to_exact_product_name_style_match(monkeypatch) -> Non
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
|
||||
@@ -99,7 +99,7 @@ def test_find_report_files_uses_current_manifest_csv(monkeypatch, tmp_path: Path
|
||||
assert importer.find_report_files(target_date) == [current_csv.resolve()]
|
||||
|
||||
|
||||
def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
def test_upsert_writes_dedicated_table_with_style_and_metrics(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows, *, template=None):
|
||||
@@ -108,7 +108,9 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
captured["template"] = template
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
written = product_db.upsert_tmall_product_ad_daily_metrics(
|
||||
_Connection(),
|
||||
[
|
||||
{
|
||||
@@ -119,6 +121,7 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
"spend": "90.25",
|
||||
"attributed_deal_amount": "730.02",
|
||||
"roi": "8.09",
|
||||
"impressions": 1000,
|
||||
},
|
||||
"source_file": "商品报表.csv",
|
||||
}
|
||||
@@ -127,16 +130,22 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
|
||||
assert written == 1
|
||||
sql = str(captured["sql"])
|
||||
assert "INSERT INTO product_daily_metrics" in sql
|
||||
assert "jsonb_set" in sql
|
||||
assert "tm_product_ad_daily_metrics" not in sql
|
||||
assert captured["template"] == "('tm', %s, %s, %s, %s::jsonb, %s, NOW())"
|
||||
assert "INSERT INTO tm_product_ad_daily_metrics" in sql
|
||||
assert "product_daily_metrics" not in sql
|
||||
assert "jsonb_set" not in sql
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["spend"] == "90.25"
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_id"
|
||||
assert row[1] == "1001"
|
||||
assert row[3] == "极星"
|
||||
assert row[4] == "product_id"
|
||||
assert str(row[5]) == "90.25"
|
||||
assert str(row[8]) == "730.02"
|
||||
assert str(row[9]) == "8.09"
|
||||
# 15 个业务列 + style_id(款式主键统一 2026-11-01)
|
||||
assert captured["template"].count("%s") == 16
|
||||
assert row[-1] is None # resolve_style_ids 被 mock 成空映射
|
||||
metrics = json.loads(row[11])
|
||||
assert metrics["impressions"] == 1000
|
||||
assert "style_name" not in metrics
|
||||
|
||||
|
||||
def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> None:
|
||||
@@ -154,7 +163,9 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) ->
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
written = product_db.upsert_tmall_product_ad_daily_metrics(
|
||||
_NameFallbackConnection(),
|
||||
[
|
||||
{
|
||||
@@ -169,9 +180,41 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) ->
|
||||
|
||||
assert written == 1
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_name"
|
||||
assert row[3] == "极星"
|
||||
assert row[4] == "product_name"
|
||||
|
||||
|
||||
def test_upsert_dedupes_repeated_product_day(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, _sql, rows, *, template=None):
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
written = product_db.upsert_tmall_product_ad_daily_metrics(
|
||||
_Connection(),
|
||||
[
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "older",
|
||||
"ad_metrics": {"spend": "1.00"},
|
||||
},
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "latest",
|
||||
"ad_metrics": {"spend": "2.00"},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
row = captured["rows"][0]
|
||||
assert row[2] == "latest"
|
||||
assert str(row[5]) == "2.00"
|
||||
|
||||
|
||||
def test_import_date_filter_and_dry_run(tmp_path: Path, capsys) -> None:
|
||||
|
||||
@@ -312,7 +312,9 @@ class MainImageDatabaseTests(unittest.TestCase):
|
||||
}
|
||||
for platform in ("jd", "tm")
|
||||
]
|
||||
with patch.object(product_db, "execute_values") as execute_values:
|
||||
with patch.object(product_db, "execute_values") as execute_values, patch.object(
|
||||
product_db, "resolve_style_ids", return_value={}
|
||||
):
|
||||
count = product_db.upsert_main_image_creatives(Connection(), records)
|
||||
|
||||
self.assertEqual(count, 2)
|
||||
|
||||
@@ -12,6 +12,13 @@ class _Cursor:
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
# db 层按商品 ID 反查 dim_style 时用;空映射表示这些商品都没建档。
|
||||
def execute(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
return []
|
||||
|
||||
|
||||
class _Connection:
|
||||
def cursor(self):
|
||||
|
||||
@@ -11,7 +11,6 @@ import collect_persona_to_bitable as tm_persona_worker
|
||||
import dy_product_scraping as dy_product
|
||||
import orchestrate_daily_collection as daily
|
||||
import orchestrate_market_rank_collection as market_rank
|
||||
import orchestrate_review_collection as reviews
|
||||
import run_alerts_with_retry as alerts
|
||||
import run_daily_persona as persona
|
||||
import run_weekly_jd_main_image as jd_main_image
|
||||
@@ -191,24 +190,6 @@ def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None:
|
||||
_assert_child_binding(environment)
|
||||
|
||||
|
||||
def test_review_run_step_rebinds_to_command_target(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(reviews, "environment_for_child_script", _fake_rebind(observed))
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
observed["environment"] = kwargs["env"]
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(reviews.subprocess, "run", fake_run)
|
||||
target = reviews.PROJECT_ROOT / "dy_product_scraping.py"
|
||||
|
||||
result = reviews.run_step("dy", ["python", str(target)], critical=True)
|
||||
|
||||
assert result["code"] == 0
|
||||
assert observed["target"] == target
|
||||
_assert_child_binding(observed["environment"])
|
||||
|
||||
|
||||
def test_main_image_launchers_rebind_to_command_target(monkeypatch) -> None:
|
||||
for launcher in (tm_main_image, jd_main_image):
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
@@ -116,7 +116,8 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
self.assertNotIn("近 60d", prompt)
|
||||
self.assertIn("逐日经营数据", prompt)
|
||||
self.assertIn("人物画像", prompt)
|
||||
self.assertIn("商品评价", prompt)
|
||||
# product_reviews 下线后数据源改为差评视图,标题同步改成「商品差评」
|
||||
self.assertIn("商品差评", prompt)
|
||||
self.assertIn("主图表现", prompt)
|
||||
self.assertIn("营销笔记、曝光互动与评论", prompt)
|
||||
self.assertIn("肩带偏硬", prompt)
|
||||
|
||||
@@ -73,6 +73,9 @@ def test_database_row_uses_explicit_style_name_not_style_content() -> None:
|
||||
"series_name": "",
|
||||
"is_series_anchor": False,
|
||||
"collect_enabled": True,
|
||||
# 款式主键统一 (2026-11-01): loader 把 dim_style.id 作为正式字段下发
|
||||
"style_id": None,
|
||||
"style_match_method": "",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
"""款式主键统一 (style_id = dim_style.id) 的契约测试。
|
||||
|
||||
守住四件事:
|
||||
1. 写入端必须把 style_id 落库(13 张采集表);
|
||||
2. 调用方已带 style_id 时不覆盖,缺省时由 db 层按款名 / 平台商品 ID 兜底;
|
||||
3. 匹配不上要留痕 ``unmatched``,不能静默写成 NULL 了事;
|
||||
4. schema.sql 里必须有列、索引、观测视图,且**不得**建物理外键(style_id 只做逻辑外键),
|
||||
同时不再建已下线的 product_reviews。
|
||||
|
||||
全部测试离线运行(Fake connection),不依赖 PostgreSQL。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import db
|
||||
import pytest
|
||||
|
||||
|
||||
MODULE_ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
)
|
||||
SCHEMA_PATH = MODULE_ROOT / "db" / "schema.sql"
|
||||
|
||||
|
||||
class RecordingCursor:
|
||||
def __init__(self, conn: "RecordingConnection") -> None:
|
||||
self._conn = conn
|
||||
self._rows: list[tuple] = []
|
||||
|
||||
def __enter__(self) -> "RecordingCursor":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> bool:
|
||||
return False
|
||||
|
||||
def execute(self, sql: str, params=None) -> None:
|
||||
self._conn.sqls.append((sql, params))
|
||||
if "unnest" in sql and params:
|
||||
self._rows = [
|
||||
(item, self._conn.product_lookup[item])
|
||||
for item in params[0]
|
||||
if item in self._conn.product_lookup
|
||||
]
|
||||
elif "FROM dim_style" in sql and "ANY" in sql and params:
|
||||
self._rows = [
|
||||
(name, self._conn.name_lookup[name])
|
||||
for name in params[0]
|
||||
if name in self._conn.name_lookup
|
||||
]
|
||||
else:
|
||||
self._rows = []
|
||||
|
||||
def fetchall(self) -> list[tuple]:
|
||||
return self._rows
|
||||
|
||||
def fetchone(self):
|
||||
return None
|
||||
|
||||
@property
|
||||
def rowcount(self) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
class RecordingConnection:
|
||||
def __init__(self, name_lookup=None, product_lookup=None) -> None:
|
||||
self.name_lookup = dict(name_lookup or {})
|
||||
self.product_lookup = dict(product_lookup or {})
|
||||
self.sqls: list[tuple] = []
|
||||
|
||||
def cursor(self) -> RecordingCursor:
|
||||
return RecordingCursor(self)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def captured(monkeypatch):
|
||||
"""拦截 execute_values,暴露最终写入的 SQL 与 rows。"""
|
||||
|
||||
box: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows):
|
||||
box["sql"] = sql
|
||||
box["rows"] = list(rows)
|
||||
|
||||
monkeypatch.setattr(db, "execute_values", fake_execute_values)
|
||||
return box
|
||||
|
||||
|
||||
def test_attach_style_ids_keeps_supplied_identity() -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
records = db.attach_style_ids(
|
||||
conn,
|
||||
[{"style_name": "盖亚斜挎", "style_id": 42, "style_match_method": "dim_style"}],
|
||||
)
|
||||
|
||||
assert records[0]["style_id"] == 42
|
||||
assert records[0]["style_match_method"] == "dim_style"
|
||||
# 已有 style_id 时不应再查库
|
||||
assert conn.sqls == []
|
||||
|
||||
|
||||
def test_attach_style_ids_fills_missing_and_marks_unmatched() -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
records = db.attach_style_ids(
|
||||
conn,
|
||||
[{"style_name": "盖亚斜挎"}, {"style_name": "未建档款"}],
|
||||
)
|
||||
|
||||
assert records[0]["style_id"] == 7
|
||||
assert records[0]["style_match_method"] == db.STYLE_MATCH_BY_NAME
|
||||
assert records[1]["style_id"] is None
|
||||
assert records[1]["style_match_method"] == db.STYLE_MATCH_UNMATCHED
|
||||
|
||||
|
||||
def test_attach_style_ids_does_not_mutate_caller_records() -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
original = [{"style_name": "盖亚斜挎"}]
|
||||
|
||||
db.attach_style_ids(conn, original)
|
||||
|
||||
assert "style_id" not in original[0]
|
||||
|
||||
|
||||
def test_daily_style_metrics_upsert_writes_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
written = db.upsert_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"metric_date": "2026-10-31",
|
||||
"style_name": "盖亚斜挎",
|
||||
"platform": "tm",
|
||||
"sales": 3,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
sql = captured["sql"]
|
||||
assert "style_id" in sql
|
||||
assert "style_match_method" in sql
|
||||
# 冲突时不能把已填的 id 抹成 NULL
|
||||
assert "COALESCE(EXCLUDED.style_id" in sql
|
||||
row = captured["rows"][0]
|
||||
assert row[-2] == 7
|
||||
assert row[-1] == db.STYLE_MATCH_BY_NAME
|
||||
|
||||
|
||||
def test_erp_all_shop_upsert_writes_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"极星双肩": 11})
|
||||
|
||||
db.upsert_erp_all_shop_style_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"metric_date": "2026-10-31",
|
||||
"style_name": "极星双肩",
|
||||
"sales": 10,
|
||||
"selected_shop_count": 2,
|
||||
"source": "jushuitan",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
row = captured["rows"][0]
|
||||
assert row[-2] == 11
|
||||
assert row[-1] == db.STYLE_MATCH_BY_NAME
|
||||
|
||||
|
||||
def test_weekly_metrics_upsert_writes_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"极星双肩": 11})
|
||||
|
||||
db.upsert_weekly_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"week_end": "2026-10-31",
|
||||
"week_start": "2026-10-25",
|
||||
"style_name": "极星双肩",
|
||||
"platform": "tm",
|
||||
"sales": 5,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert "style_id" in captured["sql"]
|
||||
assert captured["rows"][0][-2] == 11
|
||||
|
||||
|
||||
def test_product_daily_metrics_resolves_by_platform_product_id(captured, monkeypatch) -> None:
|
||||
conn = RecordingConnection(product_lookup={"700001": 33})
|
||||
|
||||
written = db.upsert_product_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"platform": "tm",
|
||||
"stat_date": "2026-10-31",
|
||||
"product_id": "700001",
|
||||
"product_name": "GYXX 极星双肩包",
|
||||
"paid_amount": 100,
|
||||
},
|
||||
{
|
||||
"platform": "tm",
|
||||
"stat_date": "2026-10-31",
|
||||
"product_id": "999999",
|
||||
"product_name": "未建档商品",
|
||||
"paid_amount": 5,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 2
|
||||
rows = captured["rows"]
|
||||
assert rows[0][-2] == 33
|
||||
assert rows[0][-1] == db.STYLE_MATCH_BY_PRODUCT
|
||||
assert rows[1][-2] is None
|
||||
assert rows[1][-1] == db.STYLE_MATCH_UNMATCHED
|
||||
|
||||
|
||||
def test_product_daily_metrics_keeps_existing_identity_on_conflict(captured) -> None:
|
||||
conn = RecordingConnection(product_lookup={})
|
||||
|
||||
db.upsert_product_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"platform": "jd",
|
||||
"stat_date": "2026-10-31",
|
||||
"product_id": "abc",
|
||||
"paid_amount": 1,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
sql = captured["sql"]
|
||||
assert "COALESCE(EXCLUDED.style_id, product_daily_metrics.style_id)" in sql
|
||||
|
||||
|
||||
def test_ad_tables_write_style_id(captured) -> None:
|
||||
conn = RecordingConnection(
|
||||
name_lookup={"盖亚斜挎": 7},
|
||||
product_lookup={"5001": 7},
|
||||
)
|
||||
|
||||
db.upsert_jd_product_ad_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"stat_date": "2026-10-31",
|
||||
"advertiser_pin": "pin",
|
||||
"shop_account": "shop",
|
||||
"report_type": "spu",
|
||||
"row_key": "r1",
|
||||
"style_name": "盖亚斜挎",
|
||||
"spend": 12,
|
||||
}
|
||||
],
|
||||
)
|
||||
assert "style_id" in captured["sql"]
|
||||
assert captured["rows"][0][-2] == 7
|
||||
|
||||
db.upsert_douyin_product_ad_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"stat_date": "2026-10-31",
|
||||
"advertiser_id": "adv",
|
||||
"product_id": "5001",
|
||||
"product_name": "盖亚斜挎包",
|
||||
"style_name": "盖亚斜挎",
|
||||
"style_match_method": "product_id",
|
||||
"spend": 8,
|
||||
}
|
||||
],
|
||||
)
|
||||
assert "style_id" in captured["sql"]
|
||||
assert captured["rows"][0][-1] == 7
|
||||
|
||||
|
||||
def test_persona_and_main_image_write_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
db.upsert_persona_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"collect_date": "2026-10-31",
|
||||
"style_name": "盖亚斜挎",
|
||||
"platform": "tm",
|
||||
"payload": {},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert captured["rows"][0][-2] == 7
|
||||
|
||||
db.upsert_main_image_creatives(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"collect_date": "2026-10-31",
|
||||
"style_name": "盖亚斜挎",
|
||||
"image_key": "k1",
|
||||
}
|
||||
],
|
||||
)
|
||||
assert captured["rows"][0][-2] == 7
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"table",
|
||||
[
|
||||
"daily_style_metrics",
|
||||
"weekly_style_metrics",
|
||||
"erp_all_shop_style_daily_metrics",
|
||||
"daily_persona_metrics",
|
||||
"main_image_creatives",
|
||||
"fact_style_analysis_reports",
|
||||
"decline_events",
|
||||
"fact_alert_history",
|
||||
"jd_self_weekly_sales",
|
||||
"persona_screenshots",
|
||||
"product_daily_metrics",
|
||||
"jd_product_ad_daily_metrics",
|
||||
"dy_product_ad_daily_metrics",
|
||||
],
|
||||
)
|
||||
def test_schema_adds_style_id_column(table: str) -> None:
|
||||
text = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
assert f"ALTER TABLE {table}" in text
|
||||
assert f"ADD COLUMN IF NOT EXISTS style_id BIGINT" in text
|
||||
|
||||
|
||||
def test_schema_defines_observation_views_and_no_physical_foreign_keys() -> None:
|
||||
text = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
|
||||
assert "CREATE OR REPLACE VIEW v_style_unmatched_product" in text
|
||||
assert "CREATE OR REPLACE VIEW v_style_id_coverage" in text
|
||||
assert "CREATE OR REPLACE VIEW v_negative_reviews" in text
|
||||
|
||||
# style_id 只做逻辑外键:库里不得出现指向 dim_style 的物理外键。
|
||||
# 采集语义是「宁可留 NULL 待回填,也不要整批写入失败」,硬约束与之冲突。
|
||||
assert "FOREIGN KEY (style_id) REFERENCES dim_style(id)" not in text
|
||||
assert "REFERENCES dim_style(id)" not in text
|
||||
# 老库重跑 schema 时要能自己清掉早期建的约束(幂等 DROP)
|
||||
assert "DROP CONSTRAINT IF EXISTS" in text
|
||||
assert "_style_id_fk" in text
|
||||
|
||||
|
||||
def test_schema_no_longer_creates_product_reviews() -> None:
|
||||
text = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
|
||||
assert "CREATE TABLE IF NOT EXISTS product_reviews " not in text
|
||||
assert "ALTER TABLE product_reviews " not in text
|
||||
assert "product_reviews_archived_20261101" in text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 回填脚本
|
||||
|
||||
def test_backfill_covers_three_ad_tables_with_right_id_column() -> None:
|
||||
"""广告表的 style_name 大量为空,必须按商品 ID 再回填一遍。"""
|
||||
|
||||
import backfill_style_id as bf
|
||||
|
||||
assert set(bf.AD_PRODUCT_TABLES) == {
|
||||
"tm_product_ad_daily_metrics",
|
||||
"jd_product_ad_daily_metrics",
|
||||
"dy_product_ad_daily_metrics",
|
||||
}
|
||||
assert bf.AD_PRODUCT_TABLES["tm_product_ad_daily_metrics"] == ("product_id", "tm")
|
||||
# 京东广告表的商品列叫 spu_id,不是 product_id
|
||||
assert bf.AD_PRODUCT_TABLES["jd_product_ad_daily_metrics"] == ("spu_id", "jd")
|
||||
assert bf.AD_PRODUCT_TABLES["dy_product_ad_daily_metrics"] == ("product_id", "dy")
|
||||
|
||||
# 款名表里的广告表项不能删,否则 style_name 有值的那部分就回填不了
|
||||
for table in bf.AD_PRODUCT_TABLES:
|
||||
assert table in bf.NAME_TABLES
|
||||
|
||||
|
||||
class _BackfillCursor:
|
||||
def __init__(self, log: list) -> None:
|
||||
self._log = log
|
||||
self._rows: list[tuple] = []
|
||||
self.rowcount = 0
|
||||
|
||||
def __enter__(self) -> "_BackfillCursor":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> bool:
|
||||
return False
|
||||
|
||||
def execute(self, sql: str, params=None) -> None:
|
||||
self._log.append((" ".join(sql.split()), params))
|
||||
if "SELECT DISTINCT" in sql:
|
||||
self._rows = [("1001",), ("9999",)]
|
||||
elif "SELECT id, style_name FROM dim_style" in sql:
|
||||
self._rows = [(7, "赤潮")]
|
||||
elif sql.lstrip().startswith("UPDATE"):
|
||||
self.rowcount = 5 if "style_id IS NULL AND style_match_method IS NULL" not in sql else 0
|
||||
elif "COUNT(*)" in sql:
|
||||
self._rows = [(10, 2)]
|
||||
else:
|
||||
self._rows = []
|
||||
|
||||
def fetchall(self) -> list[tuple]:
|
||||
return self._rows
|
||||
|
||||
def fetchone(self) -> tuple:
|
||||
return self._rows[0] if self._rows else (0, 0)
|
||||
|
||||
|
||||
class _BackfillConn:
|
||||
def __init__(self) -> None:
|
||||
self.log: list = []
|
||||
|
||||
def cursor(self) -> _BackfillCursor:
|
||||
return _BackfillCursor(self.log)
|
||||
|
||||
|
||||
def test_backfill_ad_tables_fills_style_id_and_keeps_platform_style_name(monkeypatch) -> None:
|
||||
"""回填广告表时:style_id 照写,style_name 只在原值为空时补。"""
|
||||
|
||||
import backfill_style_id as bf
|
||||
|
||||
monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7})
|
||||
conn = _BackfillConn()
|
||||
result = bf._backfill_ad_tables_by_product(
|
||||
conn, ["jd_product_ad_daily_metrics"], dry_run=False
|
||||
)[0]
|
||||
|
||||
updates = [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")]
|
||||
payload = next(sql for sql in updates if "NULLIF(BTRIM(style_name)" in sql)
|
||||
assert "COALESCE(NULLIF(BTRIM(style_name), ''), %s)" in payload, (
|
||||
"style_name 只能补空,不能覆盖平台已给出的值"
|
||||
)
|
||||
assert "AND style_id IS NULL" in payload, "幂等:已归属的行不能被改写"
|
||||
assert "spu_id = %s" in payload, "京东广告表要按 spu_id 反查"
|
||||
|
||||
# 匹配上的 1 个 ID 写 style_id,1 个未建档
|
||||
assert result["updated"] == 5
|
||||
assert result["unmatched_product_ids"] == 1
|
||||
|
||||
# 未建档的显式标 unmatched,供观测视图统计
|
||||
assert any(
|
||||
params and params[0] == "unmatched"
|
||||
for sql, params in conn.log
|
||||
if sql.lstrip().startswith("UPDATE")
|
||||
)
|
||||
|
||||
|
||||
def test_backfill_ad_tables_dry_run_does_not_update(monkeypatch) -> None:
|
||||
import backfill_style_id as bf
|
||||
|
||||
monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7})
|
||||
conn = _BackfillConn()
|
||||
result = bf._backfill_ad_tables_by_product(
|
||||
conn, ["dy_product_ad_daily_metrics"], dry_run=True
|
||||
)[0]
|
||||
|
||||
assert result["dry_run"] is True
|
||||
assert result["planned_keys"] == 1
|
||||
assert not [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")]
|
||||
@@ -176,13 +176,22 @@ def test_discovery_creates_one_candidate_per_exact_video_attachment() -> None:
|
||||
def test_flagship_and_luggage_use_distinct_account_scopes() -> None:
|
||||
flagship = _one_candidate(store=tmall.STORE_FLAGSHIP)
|
||||
luggage = _one_candidate(store=tmall.STORE_LUGGAGE)
|
||||
new_bag = _one_candidate(store=tmall.STORE_NEW_BAG)
|
||||
|
||||
assert flagship.target_account_key == "guanghe-flagship"
|
||||
assert luggage.target_account_key == "guanghe-luggage"
|
||||
assert flagship.target_account_key != luggage.target_account_key
|
||||
assert new_bag.target_account_key == "guanghe-new-bag"
|
||||
assert len(
|
||||
{
|
||||
flagship.target_account_key,
|
||||
luggage.target_account_key,
|
||||
new_bag.target_account_key,
|
||||
}
|
||||
) == 3
|
||||
assert tmall.STORE_ACCOUNT_KEYS == {
|
||||
tmall.STORE_FLAGSHIP: flagship.target_account_key,
|
||||
tmall.STORE_LUGGAGE: luggage.target_account_key,
|
||||
tmall.STORE_NEW_BAG: new_bag.target_account_key,
|
||||
}
|
||||
|
||||
|
||||
@@ -199,6 +208,25 @@ def test_luggage_runtime_uses_new_store_account_label_without_changing_password_
|
||||
assert password == "same-runtime-secret"
|
||||
|
||||
|
||||
def test_new_bag_runtime_uses_its_own_credentials_and_profile(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GUANGHE_NEW_BAG_USERNAME", "gyxx箱包旗舰店:龙虾仔")
|
||||
monkeypatch.setenv("GUANGHE_NEW_BAG_PASSWORD", "new-bag-runtime-secret")
|
||||
|
||||
store_name, username, password, profile = tmall._store_runtime(tmall.STORE_NEW_BAG)
|
||||
|
||||
assert store_name == "gyxx箱包旗舰店:龙虾仔"
|
||||
assert username == "gyxx箱包旗舰店:龙虾仔"
|
||||
assert password == "new-bag-runtime-secret"
|
||||
assert str(profile).endswith("guanghe-new-bag")
|
||||
assert profile != tmall._store_runtime(tmall.STORE_LUGGAGE)[3]
|
||||
assert profile != tmall._store_runtime(tmall.STORE_FLAGSHIP)[3]
|
||||
assert tmall._expected_creator_identity_markers(
|
||||
tmall.STORE_NEW_BAG, store_name
|
||||
) == ("GYXX箱包天猫官方",)
|
||||
|
||||
|
||||
def test_account_identity_markers_accept_store_name_with_login_nickname() -> None:
|
||||
assert tmall.GuangHeUploader._account_identity_markers(
|
||||
"光影行星鑫华达专卖店:龙虾仔"
|
||||
|
||||
@@ -148,5 +148,80 @@ class WanxiangReportTemplateTests(unittest.TestCase):
|
||||
session.close.assert_called_once()
|
||||
|
||||
|
||||
class WanxiangSearchKeywordTests(unittest.TestCase):
|
||||
"""搜索词可与款名不同,但款式身份始终是 style_name。"""
|
||||
|
||||
def test_obsidian_toast_style_searches_shorter_keyword_first(self):
|
||||
self.assertEqual(
|
||||
wanxiang.search_keywords_for("黑曜石吐司包"),
|
||||
("黑曜石吐司", "黑曜石吐司包"),
|
||||
)
|
||||
|
||||
def test_styles_without_override_keep_their_own_name(self):
|
||||
self.assertEqual(wanxiang.search_keywords_for("星云2"), ("星云2",))
|
||||
self.assertEqual(
|
||||
wanxiang.search_keywords_for("黑曜石托特包"), ("黑曜石托特包",)
|
||||
)
|
||||
|
||||
def test_collect_style_searches_keyword_but_keeps_style_identity(self):
|
||||
searched: list[str] = []
|
||||
row = {
|
||||
"creative_id": "c1",
|
||||
"impressions": 10,
|
||||
"clicks": 2,
|
||||
"cost": 0.5,
|
||||
"creative_image_url": "https://img/1.jpg",
|
||||
}
|
||||
with (
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"search_style_by_subject",
|
||||
side_effect=lambda _page, keyword: searched.append(keyword),
|
||||
),
|
||||
patch.object(wanxiang, "extract_current_page_rows", return_value=[row]),
|
||||
patch.object(wanxiang, "collect_current_page_image_map", return_value={}),
|
||||
patch.object(wanxiang, "click_next_page", return_value=False),
|
||||
patch.object(wanxiang, "aggregate_rows", return_value={}),
|
||||
):
|
||||
result = wanxiang.collect_style(
|
||||
Mock(), "黑曜石吐司包", 1, item_ids=["1022486093557"]
|
||||
)
|
||||
|
||||
self.assertEqual(searched, ["黑曜石吐司"])
|
||||
self.assertEqual(result["style_name"], "黑曜石吐司包")
|
||||
self.assertEqual(result["search_keyword"], "黑曜石吐司")
|
||||
|
||||
def test_empty_keyword_falls_back_to_original_style_name(self):
|
||||
state = {"searched": [], "checks": 0}
|
||||
row = {
|
||||
"creative_id": "c1",
|
||||
"impressions": 10,
|
||||
"clicks": 2,
|
||||
"cost": 0.5,
|
||||
"creative_image_url": "https://img/1.jpg",
|
||||
}
|
||||
|
||||
def fake_extract(_page):
|
||||
state["checks"] += 1
|
||||
return [] if state["checks"] == 1 else [row]
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"search_style_by_subject",
|
||||
side_effect=lambda _page, keyword: state["searched"].append(keyword),
|
||||
),
|
||||
patch.object(wanxiang, "extract_current_page_rows", side_effect=fake_extract),
|
||||
patch.object(wanxiang, "collect_current_page_image_map", return_value={}),
|
||||
patch.object(wanxiang, "click_next_page", return_value=False),
|
||||
patch.object(wanxiang, "aggregate_rows", return_value={}),
|
||||
):
|
||||
result = wanxiang.collect_style(Mock(), "黑曜石吐司包", 1, item_ids=["1022"])
|
||||
|
||||
self.assertEqual(state["searched"], ["黑曜石吐司", "黑曜石吐司包"])
|
||||
self.assertEqual(result["style_name"], "黑曜石吐司包")
|
||||
self.assertEqual(result["search_keyword"], "黑曜石吐司包")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user