feat(product-commerce): 款式主键统一 style_id 并下线评价采集
采集端与 data-hub 统一用 style_id (= dim_style.id) 关联款式,style_name 降级为显示冗余。
款式主键:
- 14 张采集表加 style_id BIGINT + style_match_method + *_style_id_idx 索引
- 一律逻辑外键:不建 FOREIGN KEY,早期 10 个 *_style_id_fk 在 schema.sql 里幂等 DROP
- loader 下发 dim_style.id;db 层 attach_style_ids / resolve_style_ids_by_product 兜底
- 新增 backfill_style_id.py 三段回填(款名 / 日报按商品 ID / 广告表按 spu_id)
- 新增视图 v_style_id_coverage、v_style_unmatched_product、v_style_unmatched_source
- 实库补录 199 个商品 ID 到 67 款 spus;日报 style_id 覆盖 49.1% -> 79.0%,
tm 广告 99.4% / jd 92.4% / dy 83.2%,其余 11 张 100%
评价采集下线:
- product_reviews 归档为 product_reviews_archived_20261101
- 差评改由 shop.{tmall,jd,dy}_negative_reviews.daily 采集,消费方读 v_negative_reviews
- 删除 import_product_reviews.py / orchestrate_review_collection.py 及相关命令与绑定
- dy_product_scraping 默认流程摘除评论采集,保留 --reviews-only
天猫广告:
- 迁移到独立表 tm_product_ad_daily_metrics,不再写 product_daily_metrics.raw_data
同批附带:
- 光合上传支持天猫新店 new_bag(仅黑曜石系列)
- 京东商智双月日历排除 -diff-month 补位格
- 万相主体搜索词覆盖 STYLE_SEARCH_KEYWORDS
- ERP 部分编码失败时用已成功编码销量兜底,避免整款归零
This commit is contained in:
1 parent
622a609800
commit
6f1c193f4b
50 files changed
+3293
-1142
No files matched your search
@@ -512,16 +512,6 @@
|
||||
"kind": "python",
|
||||
"notification_workflow_id": "product.market_rank"
|
||||
},
|
||||
{
|
||||
"id": "product.review.orchestrate",
|
||||
"module": "product_commerce",
|
||||
"entry": "orchestrate_review_collection.py",
|
||||
"kind": "python",
|
||||
"default_args": [
|
||||
"--target-date",
|
||||
"{business_date}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "supply.replenishment.arrival_sync",
|
||||
"module": "supply_chain",
|
||||
|
||||
@@ -1178,14 +1178,6 @@
|
||||
],
|
||||
"cdp_port": 22076
|
||||
},
|
||||
"product_commerce:import_product_reviews.py": {
|
||||
"script_id": "product_commerce:import_product_reviews.py",
|
||||
"state_key": "product_commerce:import_product_reviews.py",
|
||||
"aliases": [
|
||||
"product_commerce:import_product_reviews.py"
|
||||
],
|
||||
"cdp_port": 22077
|
||||
},
|
||||
"product_commerce:insert_bitable_records.py": {
|
||||
"script_id": "product_commerce:insert_bitable_records.py",
|
||||
"state_key": "product_commerce:insert_bitable_records.py",
|
||||
@@ -1256,14 +1248,6 @@
|
||||
],
|
||||
"cdp_port": 22083
|
||||
},
|
||||
"product.review.orchestrate": {
|
||||
"script_id": "product_commerce:orchestrate_review_collection.py",
|
||||
"state_key": "product_commerce:orchestrate_review_collection.py",
|
||||
"aliases": [
|
||||
"product_commerce:orchestrate_review_collection.py"
|
||||
],
|
||||
"cdp_port": 22084
|
||||
},
|
||||
"product_commerce:reapply_erp_override.py": {
|
||||
"script_id": "product_commerce:reapply_erp_override.py",
|
||||
"state_key": "product_commerce:reapply_erp_override.py",
|
||||
@@ -1468,7 +1452,9 @@
|
||||
"GUANGHE_USERNAME",
|
||||
"GUANGHE_PASSWORD",
|
||||
"GUANGHE_LUGGAGE_USERNAME",
|
||||
"GUANGHE_LUGGAGE_PASSWORD"
|
||||
"GUANGHE_LUGGAGE_PASSWORD",
|
||||
"GUANGHE_NEW_BAG_USERNAME",
|
||||
"GUANGHE_NEW_BAG_PASSWORD"
|
||||
]
|
||||
},
|
||||
"product.jd_video_upload.run": {
|
||||
|
||||
@@ -3,6 +3,86 @@
|
||||
"module": "product_commerce",
|
||||
"snapshot": "current-filesystem",
|
||||
"files": [
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "c7ad2a359730d8c4d449b0e37e5a4fb2940b4e8b5a4293a1b0a3767c185ee2cc",
|
||||
"target_sha256": "106a6a15ac2226e4db4d4bcdeda09ba8eca9facbd6e80f2ade6eea8821be5c5f",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_alerts.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_alerts.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "a96d18ccf79ea43d2152f2ab7de8de5e77c5a45e1cbbb2ec4495eb92e69bbe1b",
|
||||
"target_sha256": "1f286ed25cdebb4315aa64b878a7529ae74514b2def33d021834645116d3b48e",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_collect.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_collect.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "a7c8f6a777515f53160996b61804a93aedd4f3b37fa6374dc3eb64ff1588046a",
|
||||
"target_sha256": "4211341edcf8be34feef7b0be1e7779cb28707c4af5cc8c77d36d85835c28037",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_import.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "3df6a5bdda8448c342f2aa37992f471f572df7d5e51d5b0ad50911b4805cbb67",
|
||||
"target_sha256": "aa44824a204ee6b05a97fd58dd1964ea6d1b6f477823f10adc5c681384caeced",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_import.ps1",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.ps1"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "0c64ed81bed07ecdcd522549d0dcaccc97ed966cbb21a2ae60b37c634234feef",
|
||||
"target_sha256": "8a4af678695c849d5f49b295edb656155e967796229a8bb580a5d8955c6ba6a3",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_persona.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_persona.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "e461e25912cb3dafe3d6b0a092e599d121a756e11e564f1d33706d0709611cf1",
|
||||
"target_sha256": "c05295617b0361337192b21a76557c99dae14aad0d549c5980535f0f0090a41b",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_style_analysis_3d.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_style_analysis_3d.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "7edfb8cdbfdb88bbaca364b1ad213caef1818eb654b21a73131d67044ef356c0",
|
||||
"target_sha256": "42613ed10f083c803657dd6609c3b388fca88f7e0ee9dc105807ad8cab288689",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_jd_main_image.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_jd_main_image.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "1beadfdb5a0183b6e4a59307827ee4891eab985b1a1b17d0765bd02414bbd12c",
|
||||
"target_sha256": "c853990ca8857ef10b46ba71ba2177d4b6897dab4e420bce23bb7f2b22a23fae",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_main_image.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_main_image.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "656e72dc9a2999960b96d1921125e6649f0a204066664622cb2d6b1d45b290b5",
|
||||
"target_sha256": "be206938f49d9d5343dfb49e04514d6cdd9c7850a02e43f8d8c790b08b24c6fa",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_market_rank.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_market_rank.bat"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "589679874b75643d69ea44522e2fe40914a396506ff8e3232150d7e2a96ae2f9",
|
||||
"target_sha256": "dbe5760e9cbcaedadf82a2cfd84733a56a12c21e77d00327e95a8a013a5779ad",
|
||||
"transformed": true,
|
||||
"source_relative_path": ".env.example",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/.env.example"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "071286d2fb8b0516f0f22f0d088eda9e4a85ae4227619eb1217087ad0dbe3db6",
|
||||
@@ -14,19 +94,11 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "cff768f79672cfc7db7f6866caf5ab59936b8c51eb2a7d729b81441940124512",
|
||||
"target_sha256": "c1c0167b96c818a4e7d44aa5c860b5510c9925cb096b7fb4510b8319b91592fc",
|
||||
"target_sha256": "01ed46fbbf089f2c46bd67853d64774831ff2c63e2b3792e397bd01df7d9e847",
|
||||
"transformed": true,
|
||||
"source_relative_path": "analyze_style_with_hermes.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/analyze_style.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "524bff755d6921d64857b7ca6adfe1cf3a64730ddb66ae318751e028f5cf6f10",
|
||||
"target_sha256": "dd230b802fc88e16c67abcc8fd08a64c4dbfb68a58b74ea90ee45bad2d4d7870",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tmp_backfill_poseidon_sales.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_poseidon_sales.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "4d24f0e0cc65be9c2f6ce4047817794a520801f4e6563fbae2c3174c3d411d72",
|
||||
@@ -43,6 +115,38 @@
|
||||
"source_relative_path": "backfill_one_day.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_one_day.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "524bff755d6921d64857b7ca6adfe1cf3a64730ddb66ae318751e028f5cf6f10",
|
||||
"target_sha256": "dd230b802fc88e16c67abcc8fd08a64c4dbfb68a58b74ea90ee45bad2d4d7870",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tmp_backfill_poseidon_sales.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_poseidon_sales.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "8a2b09f8e5d4d49239c99a28fbc70bb5e44c8f510b6a1eca86b928eb45f21564",
|
||||
"target_sha256": "8a2b09f8e5d4d49239c99a28fbc70bb5e44c8f510b6a1eca86b928eb45f21564",
|
||||
"transformed": false,
|
||||
"source_relative_path": "backfill_style_id.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_style_id.py"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a",
|
||||
"target_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a",
|
||||
"transformed": false,
|
||||
"source_relative_path": "bitable_main_image_map.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_main_image_map.json"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2",
|
||||
"target_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2",
|
||||
"transformed": false,
|
||||
"source_relative_path": "bitable_style_map.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_style_map.json"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "6fb7cd48665c8cb7cf40dd69693408047a85a6abba3cbd410b4e5e9a9a55e9ba",
|
||||
@@ -123,10 +227,18 @@
|
||||
"source_relative_path": "config/__init__.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/__init__.py"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "277f80393a7ead72b260b56173471e924e139db0ba60e1ae534d85cc1ef5f9cc",
|
||||
"target_sha256": "bd0f4087fc061555cb8d2bab485bc0d98417dc1816d210bd58bf7ff6801cff39",
|
||||
"transformed": true,
|
||||
"source_relative_path": "config/auto-flow-config.example.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/auto-flow-config.example.json"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "22dcc9d8b2f1cf598a62e4c71df765a952250694cb26b9a73f477c5ca1a29884",
|
||||
"target_sha256": "ce328f87c883257c91025708209338ba6ee34d3c284a149c1969aeae85ffbdb7",
|
||||
"target_sha256": "5af46cd70a75930ca34142775d145416c073699ecec265dab6a34f27a59794fe",
|
||||
"transformed": true,
|
||||
"source_relative_path": "config/style_config_loader.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/style_config_loader.py"
|
||||
@@ -134,11 +246,19 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "a8900b9f25db6cf2bef3117a1d2cd86283054fea6f5a8590fd91b5cfbbcaf87e",
|
||||
"target_sha256": "a34efe965b312fe02509a4955cba835f7cdc081daa8a1624d571a8f46bf9380e",
|
||||
"target_sha256": "61cb3cc7e5ecbe76dea8f7557327c371971e9b301a62f99727708ed342caeaf4",
|
||||
"transformed": true,
|
||||
"source_relative_path": "db/__init__.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/__init__.py"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746",
|
||||
"target_sha256": "51893cc8b9cb241820fcdccc84729bc2303e5c705d47cef7f76db911fb6984cb",
|
||||
"transformed": true,
|
||||
"source_relative_path": "db/schema.sql",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "78c74c0e52e3651b3b042a2095e596ab250e60309c7ae2de905850822aad4922",
|
||||
@@ -150,7 +270,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "492916088fa5e8c170461cc9eba97a88ed1cc1e50c7b99e5f6a5e49167fea157",
|
||||
"target_sha256": "855feba2cfd51bbbb70b2fc912a7b5b517f91c3fa47ff5129ba05e2f7ffa329c",
|
||||
"target_sha256": "1ab238bf134a4e639c81d60f3e2cff63ec07040b3330c90289fb2e661f082573",
|
||||
"transformed": true,
|
||||
"source_relative_path": "dy_product_scraping.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/dy_product_scraping.py"
|
||||
@@ -166,7 +286,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "fcd8db904a14cd5239a5e075e1a888af18c5b24c928608576f6decf4c48c1d8f",
|
||||
"target_sha256": "282e9eaee483fde3394d330a62887fb4832cbd876b67a002e27651cd49b8730a",
|
||||
"target_sha256": "18cfd51b7f6df00a4a4834ccce59acc83035177b39a40e1cb4b0f3639545a2fc",
|
||||
"transformed": true,
|
||||
"source_relative_path": "erp_metric_overrides.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/erp_metric_overrides.py"
|
||||
@@ -195,14 +315,6 @@
|
||||
"source_relative_path": "import_product_daily.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/import_product_daily.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "003a8969da4c50bd500f48fd6b3dcd6378322d87893cfcaf755b3890579e7a52",
|
||||
"target_sha256": "50d757be4bfdf8849a43471f419ee460cb844921cf567454de35fcaadbae737f",
|
||||
"transformed": true,
|
||||
"source_relative_path": "import_product_reviews.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/import_product_reviews.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "057a01e8a22cdd2d1806d290da207209954af89f7da6339e9cdcef3bc0297e63",
|
||||
@@ -222,7 +334,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "d99ce71a008128549ac83fc3da52d88540b74538e3fe5c85d47c3b985e414353",
|
||||
"target_sha256": "a597578741795926844b93fa6b74bd8316e1c72cac5196f9dbc2e492e76d88e3",
|
||||
"target_sha256": "a24cc909d319c801fa63372dfcb6e3a0bd1cb6326e28008db7ada7293802a743",
|
||||
"transformed": true,
|
||||
"source_relative_path": "jd_product_data_collector.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py"
|
||||
@@ -269,11 +381,11 @@
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba",
|
||||
"target_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba",
|
||||
"transformed": false,
|
||||
"source_relative_path": "market_rank_report_archive.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_report_archive.py"
|
||||
"source_sha256": "f200d890e62bb9e387949264f5b518ed22c2bb620a60fcd28b673ff3e513c398",
|
||||
"target_sha256": "b6a3ad04efbe9b42a598caeb67f7408171ef1862f146fce4dc3eeb57fb2ce24c",
|
||||
"transformed": true,
|
||||
"source_relative_path": "market_rank_limits.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_limits.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
@@ -285,16 +397,16 @@
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "f200d890e62bb9e387949264f5b518ed22c2bb620a60fcd28b673ff3e513c398",
|
||||
"target_sha256": "b6a3ad04efbe9b42a598caeb67f7408171ef1862f146fce4dc3eeb57fb2ce24c",
|
||||
"transformed": true,
|
||||
"source_relative_path": "market_rank_limits.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_limits.py"
|
||||
"source_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba",
|
||||
"target_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba",
|
||||
"transformed": false,
|
||||
"source_relative_path": "market_rank_report_archive.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_report_archive.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "37edbfa24dfb126d52992b08fd6aeccae3ee18c83db56426a0fb02934e4541ae",
|
||||
"target_sha256": "47489fd55cbd38a39a8cf3ac0673549e8eee9a39448c187bb34c5cd57b244b97",
|
||||
"target_sha256": "b52b75e188906de1cabf55eb8791976f740c4bc9f1192971a5a4a5e7a082498c",
|
||||
"transformed": true,
|
||||
"source_relative_path": "orchestrate_daily_collection.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py"
|
||||
@@ -307,14 +419,6 @@
|
||||
"source_relative_path": "orchestrate_market_rank_collection.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_market_rank_collection.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "0d0f2c0a1533a09e698cd23aca42662b309c865659b86f9b9522de8207c3fb8b",
|
||||
"target_sha256": "47b2943790486ae6a1ca36a7e972467b89547f0ef6c37aa673a3733316277ec8",
|
||||
"transformed": true,
|
||||
"source_relative_path": "orchestrate_review_collection.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_review_collection.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "69f376f2c209d8e2df8f154aceb501eab93d7e077ab0278f5ff35c97356ba296",
|
||||
@@ -379,6 +483,14 @@
|
||||
"source_relative_path": "scripts/insert_main_image_records.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/scripts/insert_main_image_records.py"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3",
|
||||
"target_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3",
|
||||
"transformed": false,
|
||||
"source_relative_path": "styles_input.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/styles_input.json"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "b3e9addd72d8e31292ed948b67018aa9e212d628d7ba39141a81116515ca6721",
|
||||
@@ -414,7 +526,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "3f0c32c443c87417c43881da69aa7776828ba973465f8d32f50b3dc698b0a4fe",
|
||||
"target_sha256": "6363ec2a70a22b078af23a70fcdda6e3cce9f95eb4aa7ebc54f4cb85a293cb6d",
|
||||
"target_sha256": "ed86f1b7d4fe6d8e68f4d17abc6bb299df8d3083a0f54c005470491e753c1755",
|
||||
"transformed": true,
|
||||
"source_relative_path": "taobao_wanxiang_ai_creative_report.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py"
|
||||
@@ -422,7 +534,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "e4d99956d9649d108baeb3e3a2d1969c227622d1037062bd277af66375556d82",
|
||||
"target_sha256": "2ccb359ab943c185b17854f0f00cfb76b8a1235d7b031209a1c07e9386f8d93f",
|
||||
"target_sha256": "c5ed343c698efa35b76f35d95e6deeb19be113f0107d170bbd91596bab2f8e3b",
|
||||
"transformed": true,
|
||||
"source_relative_path": "upload_video_to_guanghe.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py"
|
||||
@@ -510,52 +622,12 @@
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/weekly_aggregate.py"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "589679874b75643d69ea44522e2fe40914a396506ff8e3232150d7e2a96ae2f9",
|
||||
"target_sha256": "dbe5760e9cbcaedadf82a2cfd84733a56a12c21e77d00327e95a8a013a5779ad",
|
||||
"category": "regression_test",
|
||||
"source_sha256": "55f55a9658453e6751ed055c6b2d1a43538012ce715835bbef59d88203211696",
|
||||
"target_sha256": "5fc5e15e0356b1438ea158ed1d64b769bfaf8c94bfd9f6b3a8d828a15d53c231",
|
||||
"transformed": true,
|
||||
"source_relative_path": ".env.example",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/.env.example"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a",
|
||||
"target_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a",
|
||||
"transformed": false,
|
||||
"source_relative_path": "bitable_main_image_map.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_main_image_map.json"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2",
|
||||
"target_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2",
|
||||
"transformed": false,
|
||||
"source_relative_path": "bitable_style_map.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_style_map.json"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "277f80393a7ead72b260b56173471e924e139db0ba60e1ae534d85cc1ef5f9cc",
|
||||
"target_sha256": "bd0f4087fc061555cb8d2bab485bc0d98417dc1816d210bd58bf7ff6801cff39",
|
||||
"transformed": true,
|
||||
"source_relative_path": "config/auto-flow-config.example.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/auto-flow-config.example.json"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746",
|
||||
"target_sha256": "0ea747eb7e40875338bebe7fc5b8d2524869e55ba1b503a322b37c857b274d8e",
|
||||
"transformed": true,
|
||||
"source_relative_path": "db/schema.sql",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3",
|
||||
"target_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3",
|
||||
"transformed": false,
|
||||
"source_relative_path": "styles_input.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/styles_input.json"
|
||||
"source_relative_path": "tests/test_tmp_backfill_poseidon_sales.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_backfill_poseidon_sales.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
@@ -583,8 +655,8 @@
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "4041e1104d5adbb92703b88e457795ac221bc7bc4fe4aa2cb72dff08f0c4ab6b",
|
||||
"target_sha256": "4041e1104d5adbb92703b88e457795ac221bc7bc4fe4aa2cb72dff08f0c4ab6b",
|
||||
"source_sha256": "a9b397cbc0a04cf3ba0caec54c0194ab97a96a4e8bacc3207cd0a0b7b8cbee8b",
|
||||
"target_sha256": "a9b397cbc0a04cf3ba0caec54c0194ab97a96a4e8bacc3207cd0a0b7b8cbee8b",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_erp_metric_overrides.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_erp_metric_overrides.py"
|
||||
@@ -623,8 +695,8 @@
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "e8848b95c28a822b9fc92b5cd8ad75716b80d00fbac1844bfad9b7f877ea0173",
|
||||
"target_sha256": "e8848b95c28a822b9fc92b5cd8ad75716b80d00fbac1844bfad9b7f877ea0173",
|
||||
"source_sha256": "6ba0ead03f6889cb66ea77799d4416bd4d9ccc81ccaa1df6ee79c1ff4c5dd1bc",
|
||||
"target_sha256": "6ba0ead03f6889cb66ea77799d4416bd4d9ccc81ccaa1df6ee79c1ff4c5dd1bc",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_guanghe_store_routing.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_guanghe_store_routing.py"
|
||||
@@ -656,7 +728,7 @@
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "820e17b910b62a213f0bebafcff72998c2c978307c97f976c074276150ce7c6a",
|
||||
"target_sha256": "77221bd2bfb9e2314585ba9bbaea770093a86cfe5918ffe9c67fc1fadba3b04a",
|
||||
"target_sha256": "88ff084edc73a011a4c29ed61fbd88e12d9771c3e7aed38805f152add09a64d1",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_main_image_concurrency.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_main_image_concurrency.py"
|
||||
@@ -671,11 +743,11 @@
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69",
|
||||
"target_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_market_rank_report_archive.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_market_rank_report_archive.py"
|
||||
"source_sha256": "1e0f8a519ad3ce0079240fa2b6b1d2f2df1b2c0091da89049f7df9aeec660196",
|
||||
"target_sha256": "72b85f901d43f836cdc5e8eabe9d83f67a89f8bbd49ef8c5267f92e3114d28b1",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_market_rank_limits.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_market_rank_limits.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
@@ -687,11 +759,11 @@
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "1e0f8a519ad3ce0079240fa2b6b1d2f2df1b2c0091da89049f7df9aeec660196",
|
||||
"target_sha256": "72b85f901d43f836cdc5e8eabe9d83f67a89f8bbd49ef8c5267f92e3114d28b1",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_market_rank_limits.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_market_rank_limits.py"
|
||||
"source_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69",
|
||||
"target_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_market_rank_report_archive.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_market_rank_report_archive.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
@@ -712,7 +784,7 @@
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "f9cfbb49fd313444790ac194fbf980dae21912081b9f26ad8e3850912126f09e",
|
||||
"target_sha256": "043b2dd69172bf5ea1b324bf07f8bba4c43f4bca9817c30a2c805ffe8ce70525",
|
||||
"target_sha256": "857b87a507807d522136199ad4acc54a2bbb4c7667514cf264f136e2cc6a28c9",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_style_analysis_orchestration.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_style_analysis_orchestration.py"
|
||||
@@ -720,18 +792,18 @@
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "17761a5ba93a5387ffa67662424955178058aeec708bbcd3eb0341a859d8a1f1",
|
||||
"target_sha256": "edc906fdf2ab2fab731fdcf56dd8ad7cf272949a8b4f66fb71c1693de5822f1f",
|
||||
"target_sha256": "b1725d401b59a2a19ef4824438d8362d2c68e4ccc0c06d75a2143dbc3d62d1bf",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_style_config_loader_erp_source.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_style_config_loader_erp_source.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "55f55a9658453e6751ed055c6b2d1a43538012ce715835bbef59d88203211696",
|
||||
"target_sha256": "5fc5e15e0356b1438ea158ed1d64b769bfaf8c94bfd9f6b3a8d828a15d53c231",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_tmp_backfill_poseidon_sales.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_backfill_poseidon_sales.py"
|
||||
"source_sha256": "895e9e2f149d7ef01a712246f0a5c3c641c9103908b6216571cf5764f6bff0dd",
|
||||
"target_sha256": "895e9e2f149d7ef01a712246f0a5c3c641c9103908b6216571cf5764f6bff0dd",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_style_id_identity.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_style_id_identity.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
@@ -752,82 +824,10 @@
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "5598dcbf6d9a30fb3dc4133e983ed78c2860d90162c6f7750509d4533466917a",
|
||||
"target_sha256": "198cb3979b355a81863683dc357f4152afaf331cca963259d4010ba63778b40b",
|
||||
"target_sha256": "09981f868a2f1b2b26ce14cc6bd96f1bb4cb2b8f53e2d668ae157b81966b9b36",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_wanxiang_report_template.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_wanxiang_report_template.py"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "c7ad2a359730d8c4d449b0e37e5a4fb2940b4e8b5a4293a1b0a3767c185ee2cc",
|
||||
"target_sha256": "106a6a15ac2226e4db4d4bcdeda09ba8eca9facbd6e80f2ade6eea8821be5c5f",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_alerts.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_alerts.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "a96d18ccf79ea43d2152f2ab7de8de5e77c5a45e1cbbb2ec4495eb92e69bbe1b",
|
||||
"target_sha256": "1f286ed25cdebb4315aa64b878a7529ae74514b2def33d021834645116d3b48e",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_collect.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_collect.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "a7c8f6a777515f53160996b61804a93aedd4f3b37fa6374dc3eb64ff1588046a",
|
||||
"target_sha256": "4211341edcf8be34feef7b0be1e7779cb28707c4af5cc8c77d36d85835c28037",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_import.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "3df6a5bdda8448c342f2aa37992f471f572df7d5e51d5b0ad50911b4805cbb67",
|
||||
"target_sha256": "aa44824a204ee6b05a97fd58dd1964ea6d1b6f477823f10adc5c681384caeced",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_import.ps1",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.ps1"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "0c64ed81bed07ecdcd522549d0dcaccc97ed966cbb21a2ae60b37c634234feef",
|
||||
"target_sha256": "8a4af678695c849d5f49b295edb656155e967796229a8bb580a5d8955c6ba6a3",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_persona.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_persona.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "e461e25912cb3dafe3d6b0a092e599d121a756e11e564f1d33706d0709611cf1",
|
||||
"target_sha256": "c05295617b0361337192b21a76557c99dae14aad0d549c5980535f0f0090a41b",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_style_analysis_3d.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_style_analysis_3d.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "7edfb8cdbfdb88bbaca364b1ad213caef1818eb654b21a73131d67044ef356c0",
|
||||
"target_sha256": "42613ed10f083c803657dd6609c3b388fca88f7e0ee9dc105807ad8cab288689",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_jd_main_image.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_jd_main_image.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "1beadfdb5a0183b6e4a59307827ee4891eab985b1a1b17d0765bd02414bbd12c",
|
||||
"target_sha256": "c853990ca8857ef10b46ba71ba2177d4b6897dab4e420bce23bb7f2b22a23fae",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_main_image.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_main_image.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "656e72dc9a2999960b96d1921125e6649f0a204066664622cb2d6b1d45b290b5",
|
||||
"target_sha256": "be206938f49d9d5343dfb49e04514d6cdd9c7850a02e43f8d8c790b08b24c6fa",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_market_rank.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_market_rank.bat"
|
||||
}
|
||||
],
|
||||
"intentionally_excluded": [
|
||||
|
||||
@@ -1796,7 +1796,7 @@
|
||||
{
|
||||
"id": "tmall_import",
|
||||
"name": "导入万相台商品广告指标",
|
||||
"description": "解析解压后的商品 CSV,将花费、广告归因成交金额和投产比写入 product_daily_metrics.raw_data.tm_ad_metrics;款式按商品 ID 优先、商品名称精确包含兜底,并记录匹配方式,不覆盖经营日报字段。",
|
||||
"description": "解析解压后的商品 CSV,将花费、广告归因成交金额和投产比写入独立的 tm_product_ad_daily_metrics(每日每商品一行),报表其余指标存 metrics JSON;款式按商品 ID 优先、商品名称精确包含兜底,并记录匹配方式。不再写 product_daily_metrics.raw_data。",
|
||||
"entry": "import_tmall_product_ads.py",
|
||||
"depends_on": [
|
||||
"tmall_download"
|
||||
@@ -1818,13 +1818,13 @@
|
||||
"processing": [
|
||||
"校验 CSV 编码、日期、商品主体和广告指标",
|
||||
"合并同商品同日多计划行",
|
||||
"按平台/日期/商品幂等更新广告 JSON 节点"
|
||||
"按日期/商品幂等写入天猫商品投放日报表"
|
||||
],
|
||||
"destinations": [
|
||||
{
|
||||
"system": "PostgreSQL",
|
||||
"label": "product_daily_metrics.raw_data.tm_ad_metrics",
|
||||
"detail": "由 data-hub 产品生命进程读取并计算广告成交、投产比、电商费比;节点同时保存 style_name/style_match_method"
|
||||
"label": "tm_product_ad_daily_metrics",
|
||||
"detail": "由 data-hub 产品生命进程读取 spend/attributed_deal_amount 计算广告成交、投产比、电商费比;style_name/style_match_method 记录款式匹配,metrics 保留报表全部指标"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -2305,7 +2305,7 @@
|
||||
{
|
||||
"id": "upload",
|
||||
"name": "上传所选天猫店铺短视频",
|
||||
"description": "先按前端所选店铺扫描飞书并筛选符合路由的视频附件,再按视频 file_token、目标账号和飞书记录写入 PostgreSQL 附件级发布账本;仅在存在可领取附件时登录对应光合账号上传,已发布附件自动跳过。",
|
||||
"description": "先按前端所选店铺扫描飞书并筛选符合路由的视频附件,再按视频 file_token、目标账号和飞书记录写入 PostgreSQL 附件级发布账本;仅在存在可领取附件时登录对应光合账号上传,已发布附件自动跳过。支持旗舰店、鑫华达专卖店和天猫新店(天猫新店只发黑曜石系列,视频源与其他店铺为同一张飞书表)。",
|
||||
"replay_policy": "idempotent",
|
||||
"data_flow": {
|
||||
"sources": [
|
||||
@@ -2319,11 +2319,11 @@
|
||||
},
|
||||
{
|
||||
"system": "淘宝光合浏览器状态",
|
||||
"label": "旗舰店与箱包店独立账号和 Profile"
|
||||
"label": "旗舰店、箱包店与天猫新店独立账号和 Profile"
|
||||
}
|
||||
],
|
||||
"processing": [
|
||||
"按前端所选旗舰店或箱包店规则筛选飞书记录并拆分视频附件",
|
||||
"按前端所选旗舰店、箱包店或天猫新店规则筛选飞书记录并拆分视频附件;天猫新店仅保留全部款式都属于黑曜石系列的记录",
|
||||
"按视频 file_token、目标账号和飞书记录原子领取未上传附件",
|
||||
"确认存在可上传附件后登录所选店铺账号",
|
||||
"生成标题、关联天猫商品并发布;明确成功后写 published,提交后结果不明写 ambiguous"
|
||||
@@ -2364,7 +2364,7 @@
|
||||
"label": "天猫话题关键词",
|
||||
"hint": "输入完整话题或关键词;例如输入“秋上新”也会匹配并选中“秋上新开拍了”。"
|
||||
},
|
||||
"note": "默认由常驻调度器按 config/schedules.json 每 3 天 14:00 执行;前端仍可手动选择旗舰店或箱包店后执行。脚本先扫描飞书并按所选店铺规则筛选,只有数据库存在可领取附件时才登录对应账号上传。允许同一业务日期重复扫描新增附件,PostgreSQL 附件级账本阻止已发布或结果不明的附件自动重发;框架安全预演不会启动上传脚本。"
|
||||
"note": "默认由常驻调度器按 config/schedules.json 每 3 天 14:00 执行;前端仍可手动选择旗舰店、箱包店或天猫新店后执行。天猫新店账号为gyxx箱包旗舰店:龙虾仔(凭据 GUANGHE_NEW_BAG_USERNAME/GUANGHE_NEW_BAG_PASSWORD),只上传黑曜石系列,视频来源与其他店铺相同。脚本先扫描飞书并按所选店铺规则筛选,只有数据库存在可领取附件时才登录对应账号上传。允许同一业务日期重复扫描新增附件,PostgreSQL 附件级账本阻止已发布或结果不明的附件自动重发;框架安全预演不会启动上传脚本。"
|
||||
},
|
||||
{
|
||||
"id": "product.jd_video_upload",
|
||||
|
||||
+128
-11
@@ -43,7 +43,7 @@ uv run gyxx scripts run <command_id> --date 2026-08-01 --execute
|
||||
| 京东自营品牌单日回采 | `shop.jd_self_operated.collect_brand` | `--date` 自动成为起止日期 |
|
||||
| 商品历史补采 | `product.backfill.run` | `--date` 自动成为 `--from/--to` |
|
||||
| 商品评价补采 | `product.review.orchestrate` | `--date` 自动成为目标日期 |
|
||||
| 天猫旗舰店和箱包店视频上传 | `product.video_upload.run` | 业务日期用于运行审计;默认顺序扫描两店,由 PostgreSQL 附件级账本防重 |
|
||||
| 天猫旗舰店、箱包店与天猫新店视频上传 | `product.video_upload.run` | 业务日期用于运行审计;`--store all` 顺序扫描三家,由 PostgreSQL 附件级账本防重 |
|
||||
| 京东旗舰店视频上传 | `product.jd_video_upload.run` | 业务日期只用于运行审计;从固定凌云air记录锚点按飞书视图顺序扫描,日期可为空 |
|
||||
| 采购单更新 | `supply.workflow.run` | 自动选择 `mcp-run purchase-order-update` |
|
||||
|
||||
@@ -51,16 +51,16 @@ uv run gyxx scripts run <command_id> --date 2026-08-01 --execute
|
||||
|
||||
### 天猫视频上传工作流
|
||||
|
||||
天猫视频上传注册为幂等定时工作流 `product.video_upload`,按 `config/schedules.json` 以 2026-09-04 为起始日每 3 天 14:00(Asia/Shanghai)执行;前端仍可手动选择“光影行星旗舰店 / 光影行星鑫华达专卖店:龙虾仔”并立即运行,也可填写天猫话题完整名称或关键词。正式执行只处理所选店铺,话题搜索结果按包含关系选中。默认 dry-run 只生成运行记录并跳过浏览器节点;确认飞书待传记录、对应淘宝光合店铺的登录态和 PostgreSQL 后,再显式正式执行:
|
||||
天猫视频上传注册为幂等定时工作流 `product.video_upload`,按 `config/schedules.json` 以 2026-09-04 为起始日每 3 天 14:00(Asia/Shanghai)执行;前端仍可手动选择“光影行星旗舰店 / 光影行星鑫华达专卖店:龙虾仔 / 天猫新店(gyxx箱包旗舰店:龙虾仔)”并立即运行,也可填写天猫话题完整名称或关键词。正式执行只处理所选店铺,话题搜索结果按包含关系选中。默认 dry-run 只生成运行记录并跳过浏览器节点;确认飞书待传记录、对应淘宝光合店铺的登录态和 PostgreSQL 后,再显式正式执行:
|
||||
|
||||
```bash
|
||||
uv run gyxx run product.video_upload --date 2026-08-07
|
||||
uv run gyxx run product.video_upload --date 2026-08-07 --execute
|
||||
```
|
||||
|
||||
前端把店铺选择作为受限运行参数传给脚本,只允许 `flagship` 或 `luggage`。脚本先读取飞书并应用所选店铺路由,再写入/读取附件级账本;只有存在可领取的未上传附件时才启动有头 Chrome、登录对应账号并上传。旗舰店只选择“天猫上传”未勾选且有附件的光影行星男/女记录;箱包店只选择女性、全部款式命中箱包店白名单且有附件的记录。两店分别使用 `GUANGHE_USERNAME` / `GUANGHE_PASSWORD` 和 `GUANGHE_LUGGAGE_USERNAME` / `GUANGHE_LUGGAGE_PASSWORD`,箱包店当前账号为 `光影行星鑫华达专卖店:龙虾仔`,密码配置保持不变,并保留独立浏览器 Profile。发生登录或页面错误时保留窗口供人工定位,排查完成后手动关闭。
|
||||
前端把店铺选择作为受限运行参数传给脚本,只允许 `flagship`、`luggage` 或 `new_bag`。脚本先读取飞书并应用所选店铺路由,再写入/读取附件级账本;只有存在可领取的未上传附件时才启动有头 Chrome、登录对应账号并上传。旗舰店只选择“天猫上传”未勾选且有附件的光影行星男/女记录;箱包店只选择女性、全部款式命中箱包店白名单且有附件的记录;天猫新店不区分性别,只要求一条记录的**全部**「代拍产品」款式都属黑曜石系列(关键词 `黑曜石`,如黑曜石托特、黑曜石吐司、黑曜石电脑包、黑曜石饺子),组合款里掺了其它系列整条记录都不进新店,视频源与其它店铺为同一张飞书表。三店分别使用 `GUANGHE_USERNAME` / `GUANGHE_PASSWORD`、`GUANGHE_LUGGAGE_USERNAME` / `GUANGHE_LUGGAGE_PASSWORD` 和 `GUANGHE_NEW_BAG_USERNAME` / `GUANGHE_NEW_BAG_PASSWORD`,箱包店当前账号为 `光影行星鑫华达专卖店:龙虾仔`,天猫新店当前账号为 `gyxx箱包旗舰店:龙虾仔`,密码配置保持不变,各自保留独立浏览器 Profile(`-luggage` / `-new-bag`)。发生登录或页面错误时保留窗口供人工定位,排查完成后手动关闭。
|
||||
|
||||
每次正式扫描都会把视频附件写入 PostgreSQL 附件级发布账本,防重身份由视频 `file_token + 目标账号 + 飞书记录` 构成。账本状态为 `published` 的附件在下次执行时自动跳过,因此同一业务日期可以安全重跑并发现新增附件。旗舰店仍只在一条飞书记录的全部附件发布成功后回写“天猫上传”;箱包店不借用该勾选状态,是否已上传完全以自己的目标账号账本为准。
|
||||
每次正式扫描都会把视频附件写入 PostgreSQL 附件级发布账本,防重身份由视频 `file_token + 目标账号 + 飞书记录` 构成。账本状态为 `published` 的附件在下次执行时自动跳过,因此同一业务日期可以安全重跑并发现新增附件。旗舰店仍只在一条飞书记录的全部附件发布成功后回写“天猫上传”;箱包店和天猫新店不借用该勾选状态,是否已上传完全以自己的目标账号账本(`guanghe-luggage` / `guanghe-new-bag`)为准,同一视频可以分别发到多家店铺。
|
||||
|
||||
首次正式执行会读取 `config/tmall-video-publication-backfill.json`,将 2026-07-26 箱包店旧日志中可按 `record_id + file_token + 文件名` 再次核验的 25 个明确成功附件回填为 `published`。配置与当前飞书任一身份不一致都会停止,不会静默猜测。该回填不依赖部署机保留旧 `var/` 日志文件;证据路径与成功行号已固化在配置中供审计。
|
||||
|
||||
@@ -103,11 +103,17 @@ uv run gyxx scripts run product.jd_video_upload.run --date 2026-08-07 --arg=--re
|
||||
|
||||
唯一工作流 ID 是 `product.ecommerce_costs.daily`,每天 10:00 由项目内 Python 调度器运行,
|
||||
业务日期为前一天;天猫、京东、抖音三个下载分支并行启动,各自完成下载后再导入。天猫流程先进入万相台商品报表选择昨日,提交下载任务;任务在下载列表显示“生成成功”
|
||||
后下载 ZIP、做路径安全校验并解压 CSV,随后将商品级广告指标幂等写入
|
||||
`product_daily_metrics.raw_data.tm_ad_metrics`。商品款式先按 `dim_style.tm_spus` 的商品 ID 匹配,
|
||||
匹配不到再按商品名称做精确包含匹配,并在节点内保留 `style_name` 与
|
||||
`style_match_method`(`product_id`/`product_name`/`ambiguous`/`unmatched`)。它不会覆盖同一商品日期
|
||||
已有的访客、成交等经营日报字段;data-hub 产品生命进程从该节点读取广告成交金额和花费并计算投产比、电商费比。
|
||||
后下载 ZIP、做路径安全校验并解压 CSV,随后将商品级广告指标幂等写入独立表
|
||||
`tm_product_ad_daily_metrics`(每日每商品一行,主键 `(stat_date, product_id)`)。花费、直接/间接/归因
|
||||
成交金额和投产比走标量列,报表其余指标(展现、点击、点击率、转化率、预售成交等)原样存在 `metrics`
|
||||
JSONB。商品款式先按 `dim_style.tm_spus` 的商品 ID 匹配,匹配不到再按商品名称做精确包含匹配,结果
|
||||
写进 `style_name` 与 `style_match_method`(`product_id`/`product_name`/`ambiguous`/`unmatched`)。
|
||||
天猫广告数据**不再写入** `product_daily_metrics.raw_data.tm_ad_metrics`,投放与经营日报分表,经营数据
|
||||
重新同步不会覆盖投放数据;data-hub 产品生命进程从新表读取花费和广告归因成交金额并计算投产比、电商费比。
|
||||
|
||||
历史数据迁移(2026-11-01 一次性执行):`src/gyxx_flow/modules/product_commerce/scripts/
|
||||
migrate_tm_ad_metrics_to_table.py`,默认 dry-run,`--execute` 先备份旧节点到运行时 TMP_ROOT 再回填,
|
||||
校验行数与花费合计一致后才允许 `--clear-source` 清除 `raw_data` 里的旧节点。
|
||||
|
||||
正式服务必须注入 `WANXIANG_ACCOUNT`、`WANXIANG_PASSWORD`、`GYXX_DATA_ROOT` 和
|
||||
`GYXX_POSTGRES_DSN`。默认浏览器 Profile 是
|
||||
@@ -139,8 +145,9 @@ uv run gyxx scripts run product.import.tmall_ads --date 2026-08-17 --execute
|
||||
1. 检查 `gyxx schedule status`、服务日志和对应 `run_id`,确认失败发生在申请、下载、解压还是入库。
|
||||
2. 检查同一 Profile 是否仍能打开万相台商品报表;登录失效时在有头命令中人工完成验证。
|
||||
3. 检查日期目录中的 `manifest.json`、CSV 表头和 `日期/主体ID/主体名称`;ZIP 存在不代表数据库已导入。
|
||||
4. 检查 PostgreSQL 的 `product_daily_metrics` 是否存在 `platform='tm'`、目标日期和商品 ID,且
|
||||
`raw_data->'tm_ad_metrics'` 已更新;确认经营日报其他 JSON 节点仍保留。
|
||||
4. 检查 PostgreSQL 的 `tm_product_ad_daily_metrics` 是否存在目标日期与商品 ID 的行,`spend`、
|
||||
`attributed_deal_amount` 是否为目标值,`style_name` 是否已匹配;确认 `product_daily_metrics`
|
||||
同日期经营数据(访客、成交)仍完整。
|
||||
5. 只有在外部状态核对清楚后才按同一业务日期重跑,不能用换日期或重复申请掩盖下载结果不明。
|
||||
|
||||
## 调度操作
|
||||
@@ -385,6 +392,116 @@ ORDER BY style_name;
|
||||
必须把它拆成独立系列(单独的 `series_name` + 锚点标记)并把地址写在自己那行;只写地址
|
||||
而不拆系列的话,聚合会把它并入原系列,只在日志里留一条告警,不会真的生效。
|
||||
|
||||
## 款式主键统一(style_id = dim_style.id,2026-11-01)
|
||||
|
||||
采集端不再只用 `style_name` 关联款式。**唯一关联键是 `style_id`**(= `dim_style.id`,
|
||||
与 data-hub 的 22 张表同一套主键),`style_name` 降级为显示冗余。这样在 dim_style
|
||||
改款名时,历史指标表不会变成孤儿。
|
||||
|
||||
### 数据从哪里来
|
||||
|
||||
| 环节 | 机制 |
|
||||
|---|---|
|
||||
| loader 下发 | `merge_lifecycle_style_records()` 输出 `style_id` + `style_match_method`,经 `style_config_loader` 进入每条款式配置 |
|
||||
| db 层兜底 | `db.attach_style_ids()`(按款名)/ `db.resolve_style_ids_by_product()`(按平台商品 ID)在 upsert 内补齐 |
|
||||
| 历史回填 | `backfill_style_id.py` |
|
||||
|
||||
### 回填与验收
|
||||
|
||||
```powershell
|
||||
# 只报数,不写库
|
||||
.venv\Scripts\python.exe backfill_style_id.py --dry-run
|
||||
# 执行(幂等,只补 style_id IS NULL 的行)
|
||||
.venv\Scripts\python.exe backfill_style_id.py
|
||||
# 只补款名表,跳过按商品 ID 的两步
|
||||
.venv\Scripts\python.exe backfill_style_id.py --skip-product-table --skip-ad-tables
|
||||
```
|
||||
|
||||
回填分三段,缺一段覆盖率就上不去:
|
||||
|
||||
| 段 | 表 | 依据 |
|
||||
|---|---|---|
|
||||
| 按款名 | 13 张(`daily_style_metrics` / `weekly_style_metrics` / `erp_all_shop_style_daily_metrics` 等) | `t.style_name = dim_style.style_name` |
|
||||
| 按商品 ID | `product_daily_metrics` | `platform + product_id` 反查 `*_spus` |
|
||||
| 按商品 ID(二次) | 三张广告表 | 同上;**顺带把空的 `style_name` 补成 dim_style 的权威款名** |
|
||||
|
||||
广告表必须走第三段:平台的报表不给款式名,`jd_product_ad_daily_metrics` 有 27%、
|
||||
`dy_product_ad_daily_metrics` 有 40% 的行 `style_name` 为空,只按款名回填完全无效。
|
||||
补完后 data-hub 现有按款名的 join 立刻多认回一批行,不必等主键切换。
|
||||
|
||||
```sql
|
||||
-- 各表回填进度;除 product_daily_metrics 与三张广告表外都应 100%
|
||||
SELECT * FROM v_style_id_coverage ORDER BY tbl;
|
||||
-- 未建档的平台商品 ID,仅统计经营日报口径
|
||||
SELECT * FROM v_style_unmatched_product ORDER BY platform, rows DESC;
|
||||
-- 同上,但把三张广告表也算进来(纯投流商品只在这里出现)
|
||||
SELECT * FROM v_style_unmatched_source ORDER BY platform, rows DESC;
|
||||
```
|
||||
|
||||
`style_match_method` 取值:`dim_style`(按款名精确)、`product_id`(按平台商品 ID)、
|
||||
`product_name`(按商品名包含)、`unmatched`(确实没建档)。
|
||||
|
||||
### 补 spus 的正确做法
|
||||
|
||||
`product_daily_metrics.product_no` 存的是**款名**(不是 ERP 编码),天猫填充率 100%、
|
||||
京东 97.8%、抖音 0%。补 spus 时按这个优先级,命中率远高于猜商品标题:
|
||||
|
||||
1. `product_no` 精确等于 `dim_style.style_name`;
|
||||
2. `product_no` 与款名互为前后缀(长度差 ≤ 3,如「蓝鹊pro」→「蓝鹊」、「知更大号」→「知更」);
|
||||
3. 商品标题包含完整款名(抖音只能用这一条),并用「命中位置后的版本后缀」排除串款
|
||||
—— 例如标题「盖亚斜挎包**3**」不能归到「盖亚斜挎」;
|
||||
4. 编码与标题互相矛盾时**不要自动写**,列出来人工定(实测有天猫单品编码写「理岩」、
|
||||
标题写「阿波罗 air」,支付额 50 万,猜错代价极大)。
|
||||
|
||||
物料类(数据线、内胆包、补邮费、配件包、礼品袋、定制服务)不该进 `spus`,应留
|
||||
`unmatched`。`dim_style` 里本身建成款的物料(`配件包`、`飞机盒`、`电泵` 等)也不写。
|
||||
|
||||
### 必须知道的边界
|
||||
|
||||
- `product_daily_metrics` 表里**没有款名列**,只能拿 `platform + product_id` 去
|
||||
`tm_spus / jd_spus / dy_spus` 反查。未建档的商品 ID 回填不了,要靠建档页补 `spus`
|
||||
才能涨,不是脚本 bug。
|
||||
- 2026-11-01 补过一轮:`product_daily_metrics` 49.1% → **86.0%(jd)/ 77.3%(tm)/
|
||||
76.7%(dy)**,整体 79.0%;广告表 tm 99.4% / jd 92.4% / dy 83.2%。
|
||||
剩余 259 个商品 ID(dy 107 / jd 52 / tm 100)未建档,清单见 `v_style_unmatched_source`。
|
||||
- **不建物理外键**:14 张表的 `style_id` 都只做**逻辑外键**(与 data-hub 侧一致)。
|
||||
库里没有任何指向 `dim_style(id)` 的 FK 约束,完整性靠写入端(loader 下发 +
|
||||
`attach_style_ids` 兜底)、`backfill_style_id.py --dry-run`、`v_style_id_coverage`
|
||||
三道保证。理由:采集语义是「宁可留 NULL 待回填,也不要整批写入失败」,
|
||||
物理外键会在传错 id 时直接中断整天采集;此外拆系列/合并同款要改数据时,
|
||||
还得先删约束。早期 P4 建过的 10 个 `<表名>_style_id_fk` 已在 schema.sql 里
|
||||
幂等 `DROP CONSTRAINT IF EXISTS` 清掉,老库重跑 schema 即自动回到无 FK 状态。
|
||||
- 主键一律没动:`daily_style_metrics` 仍是 `(metric_date, style_name, platform)`。
|
||||
data-hub 有 6 处按 `style_name` join,改主键收益小、回归面大。
|
||||
|
||||
## 商品评价采集下线(product_reviews,2026-11-01)
|
||||
|
||||
`product_reviews` 已**停止写入并归档**为 `product_reviews_archived_20261101`。
|
||||
原因是 JD 侧 `style_name` 全靠 `sku_id` 反查,而映射表不存在,214 行全部为 NULL,
|
||||
两个消费方都是 `WHERE style_name = ?`,等于入库了读不到。
|
||||
|
||||
差评统一由三个独立工作流采集:`shop.tmall_negative_reviews.daily`、
|
||||
`shop.jd_negative_reviews.daily`、`shop.dy_negative_reviews.daily`。
|
||||
|
||||
### 消费方改读视图
|
||||
|
||||
```sql
|
||||
SELECT platform, comment_date, rating, content, style_id, style_name, style_match_method
|
||||
FROM v_negative_reviews; -- tm + jd + dy 三张差评表 UNION,款式按 ID 优先、商品名兜底
|
||||
```
|
||||
|
||||
改动点:`analyze_style.py:pull_reviews`、`content_marketing/daily_marketing_report.py:_review_context`。
|
||||
|
||||
### 口径变化(必须知道)
|
||||
|
||||
- 数据源**只有差评一档**,不再有好评/中评。风格分析的「评价等级分布」因此只剩差评。
|
||||
- 营销日报的 `sample_size` = 差评条数,`negative_rate` 恒为 1.0;门槛从「至少 5 条评价」
|
||||
改成「至少 1 条差评」(旧门槛在纯差评源上会导致永不产出)。
|
||||
- 覆盖面反而更全:天猫差评 97 条(旧表天猫为 0)、京东 32 条(旧表 8 月断更)。
|
||||
- 若产品侧确实需要好评样本,应把差评采集器扩成「按等级采集」,**不要恢复 product_reviews**。
|
||||
- `dy_product_scraping.py` 默认流程里的评论采集已摘除(只产出无人读取的本地 JSON);
|
||||
需要历史样本时用 `--reviews-only` 显式触发。
|
||||
|
||||
## 业务源码同步
|
||||
|
||||
外部业务源码目录只用于发现更新,GYXX Flow 运行时不导入或挂载这些目录。只读检查:
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
# 款式主键统一 + 评论采集下线 方案
|
||||
|
||||
> 状态:**已实施(2026-11-01)**,见文末「八、实施记录」
|
||||
> 涉及项目:`gyxx-flow`(采集端)、`data-hub`(消费端,共用同一个 PostgreSQL)
|
||||
> 编写日期:2026-11-01
|
||||
|
||||
## 一、三条结论
|
||||
|
||||
1. **`product_reviews` 可以下线**,但不能直接删——它有两个消费方(风格分析、营销日报)要改成读三张差评表。
|
||||
2. **`style_id` 就是 `dim_style.id`**,与 data-hub 现有 `style_id` 同一体系,不会产生第二套 ID。已实测:`erp_product.style_id` 的 135 个值 **135/135** 命中 `dim_style.id`,`own_brand_product.style_id` 31/34 命中(3 条为历史脏数据)。
|
||||
3. **改造入口只有一处**:`DIM_STYLE_RUNTIME_SQL` 早已 `SELECT s.id`,`merge_lifecycle_style_records()` 也已经拿到这个 id——只是被编码进 `source_record_id = "lifecycle_style:<id>"` 字符串里,没有作为独立字段下发给写入脚本。把它提成正式字段,全链路就都有了。
|
||||
|
||||
## 二、现状盘点
|
||||
|
||||
### 2.1 两套引用方式并存
|
||||
|
||||
| 侧 | 关联方式 | 表 |
|
||||
|---|---|---|
|
||||
| data-hub(产品生命、营销中台、客服、内容) | `style_id BIGINT` → `dim_style.id` | `erp_product`、`own_brand_product`、`erp_marketing_*`、`erp_cs_*`、`cmt_*`、`market_product_classification` 等 22 张 |
|
||||
| gyxx-flow(采集端) | `style_name TEXT` | 15 张,见 2.3 |
|
||||
|
||||
### 2.2 实测匹配率(2026-11-01)
|
||||
|
||||
| 表 | 总行数 | 按 style_name 命中 dim_style |
|
||||
|---|---|---|
|
||||
| `daily_style_metrics` | 17,352 | 17,352(100%) |
|
||||
| `erp_all_shop_style_daily_metrics` | 9,399 | 9,399(100%) |
|
||||
| `product_reviews` | 1,936 | 1,434(74%,全部来自抖音;京东 214 行 style_name 全 NULL) |
|
||||
| `product_daily_metrics` | 75,568 | **无款式列**,只能靠 `tm_spus/jd_spus/dy_spus @> to_jsonb(product_id)` 现算 |
|
||||
|
||||
`product_daily_metrics` 按商品 ID 反查的现状:
|
||||
|
||||
| 平台 | 行数 | 可匹配 | 去重商品 ID | 可匹配 ID |
|
||||
|---|---|---|---|---|
|
||||
| dy | 28,972 | 13,615(47%) | 253 | 83 |
|
||||
| jd | 17,044 | 9,394(55%) | 160 | 61 |
|
||||
| tm | 29,552 | 14,089(48%) | 281 | 92 |
|
||||
|
||||
一对多歧义 **0 条**。约一半行匹配不上,原因是 **458 个商品 ID 从未写进 dim_style 的 spus 数组**(dy 170 / jd 99 / tm 189),不是字段设计问题。
|
||||
|
||||
### 2.3 采集端 15 张只存 style_name 的表
|
||||
|
||||
`daily_style_metrics`、`weekly_style_metrics`、`erp_all_shop_style_daily_metrics`、`daily_persona_metrics`、`main_image_creatives`、`fact_style_analysis_reports`、`decline_events`、`fact_alert_history`、`jd_self_weekly_sales`、`persona_screenshots`、`product_reviews`、`jd_product_ad_daily_metrics`、`dy_product_ad_daily_metrics`、`workflow_dynamic_styles`、`product_daily_metrics`(无列,靠 ID 反查)。
|
||||
|
||||
## 三、目标模型
|
||||
|
||||
- **唯一关联键 = `style_id`(`dim_style.id`)**,所有采集表落库时必须带上。
|
||||
- `style_name` 保留为**显示冗余**,不再作为 join 键;等下游切完 join 后可以只在视图里出现。
|
||||
- 匹配不上的行(未建档商品、广告未匹配 SKU、差评)**允许 `style_id IS NULL`**,但必须同时写 `style_match_method`(`product_id` / `product_name` / `anchor` / `unmatched`),便于统计漏维护。
|
||||
- **不动现有主键**。`daily_style_metrics` 的 PK 仍是 `(metric_date, style_name, platform)`——data-hub 有 6 处 `metric.style_name = style.name` 的 join,改 PK 收益小、回归面大。加列、加索引、切 join 即可。
|
||||
|
||||
## 四、实施分期
|
||||
|
||||
### P0 观测层(只读,不动数据)
|
||||
|
||||
新增两个视图,先把问题量化出来,方案落地后靠它们验收:
|
||||
|
||||
```sql
|
||||
-- 未建档的平台商品 ID(推动在 dim_style 里补 spus)
|
||||
CREATE OR REPLACE VIEW v_style_unmatched_product AS
|
||||
SELECT m.platform, m.product_id,
|
||||
MAX(m.product_name) AS product_name,
|
||||
MIN(m.stat_date) AS first_date, MAX(m.stat_date) AS last_date,
|
||||
COUNT(*) AS rows
|
||||
FROM product_daily_metrics m
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM dim_style d
|
||||
WHERE (m.platform = 'tm' AND d.tm_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'jd' AND d.jd_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'dy' AND d.dy_spus @> to_jsonb(m.product_id)))
|
||||
GROUP BY m.platform, m.product_id;
|
||||
|
||||
-- 各表 style_id 回填进度
|
||||
CREATE OR REPLACE VIEW v_style_id_coverage AS
|
||||
SELECT 'daily_style_metrics' AS tbl, COUNT(*) AS rows,
|
||||
COUNT(style_id) AS with_style_id FROM daily_style_metrics
|
||||
UNION ALL SELECT 'erp_all_shop_style_daily_metrics', COUNT(*), COUNT(style_id)
|
||||
FROM erp_all_shop_style_daily_metrics
|
||||
UNION ALL SELECT 'product_daily_metrics', COUNT(*), COUNT(style_id)
|
||||
FROM product_daily_metrics;
|
||||
```
|
||||
|
||||
验收锚点:`v_style_id_coverage` 里前三张表 `with_style_id` 开始上涨到预期比例;`v_style_unmatched_product` 的条数随建档推进下降。
|
||||
|
||||
### P1 评论采集链路下线
|
||||
|
||||
**要删的东西**
|
||||
|
||||
| 位置 | 动作 |
|
||||
|---|---|
|
||||
| `orchestrate_daily_collection.py:437-452` | 移除 `import_product_reviews` 步骤 |
|
||||
| `import_product_reviews.py` | 删除 |
|
||||
| `orchestrate_review_collection.py` | 删除 |
|
||||
| `config/commands.json:516` `product.review.orchestrate` | 移除命令 |
|
||||
| `db/__init__.py` `upsert_product_reviews` | 删除 |
|
||||
| `db/schema.sql:813-870` `product_reviews` 建表段 | 移除(库里的表归档不删) |
|
||||
|
||||
**要补的东西**
|
||||
|
||||
1. 建统一差评视图,字段对齐两个消费方现在的用法(`platform` / `rating` / `content` / `style_id` / `comment_date`):
|
||||
|
||||
```sql
|
||||
CREATE OR REPLACE VIEW v_negative_reviews AS
|
||||
SELECT 'tm' AS platform, n.business_date AS comment_date, n.review_level AS rating,
|
||||
n.content, NULL::text AS order_id, NULL::text AS product_id,
|
||||
n.product_name, n.review_id, n.review_at, n.raw_review
|
||||
FROM tmall_negative_reviews n
|
||||
UNION ALL
|
||||
SELECT 'jd', n.business_date, n.review_level, n.content, n.order_id, NULL,
|
||||
n.product_name, n.review_id, n.review_at, n.raw_review
|
||||
FROM jd_negative_reviews n
|
||||
UNION ALL
|
||||
SELECT 'dy', n.business_date, n.rating_level, n.content, n.order_id, n.product_id,
|
||||
n.product_name, n.review_id, n.review_at, n.raw_review
|
||||
FROM dy_negative_reviews n;
|
||||
```
|
||||
|
||||
2. 改两个消费方:
|
||||
- `analyze_style.py:112 pull_reviews()` → 改查 `v_negative_reviews`(按 `style_id` 或 `product_id` 关联)
|
||||
- `content_marketing/daily_marketing_report.py:1020 _review_context()` → 同上
|
||||
|
||||
3. `product_reviews` 归档而不是删除:
|
||||
|
||||
```sql
|
||||
ALTER TABLE product_reviews RENAME TO product_reviews_archived_20261101;
|
||||
```
|
||||
|
||||
**必须明确的功能退化**:`product_reviews` 存的是好/中/差全量,差评三表只有差评。下线后:
|
||||
- 风格分析里的「评价等级分布」从三档变成只有差评一档(差评样本反而更完整:天猫 97 + 京东 32 + 抖音 14,而 `product_reviews` 里天猫是 0、京东 8 月就断了)。
|
||||
- 营销日报的「近 7 天差评归因」不受影响,反而多了天猫来源。
|
||||
- 若产品侧确实需要好评样本沉淀,需要把三个差评采集器扩成「按等级采集」,不要恢复 `product_reviews`。
|
||||
|
||||
另外 `dy_product_scraping.py` 主流程里的评论采集动作只产出本地 JSON,`product_reviews` 停写后就没有消费方,建议同步关掉(或保留但不再进 `product.daily`)。
|
||||
|
||||
### P2 loader 下发 style_id(1 处改动,收益最大)
|
||||
|
||||
`src/gyxx_flow/adapters/workflow_config.py:322 merge_lifecycle_style_records()`:
|
||||
|
||||
```python
|
||||
output["style_id"] = lifecycle.get("id") # dim_style.id
|
||||
output["style_match_method"] = "dim_style"
|
||||
```
|
||||
|
||||
同时 `source_record_id` 保留(控制台/账本还在用),只是不再承担传 ID 的职责。
|
||||
|
||||
影响面:所有通过 `runtime_workflow_configs()` 取款的脚本自动获得 `style_id`,包括日报聚合、周报、ERP 全店铺、人像、主图、声明式分析。这一步不改表、不改写入逻辑,可以先合入验证 loader 输出。
|
||||
|
||||
### P3 采集表加列 + 回填 + 写入端填充
|
||||
|
||||
**统一 DDL 模板**(以 `daily_style_metrics` 为例,其余同构):
|
||||
|
||||
```sql
|
||||
ALTER TABLE daily_style_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
UPDATE daily_style_metrics d SET style_id = s.id
|
||||
FROM dim_style s WHERE s.style_name = d.style_name AND d.style_id IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS daily_style_metrics_style_date_idx
|
||||
ON daily_style_metrics (style_id, metric_date DESC);
|
||||
```
|
||||
|
||||
按表处理(**必须匹配**的加 `NOT NULL`,**允许未匹配**的保持可空):
|
||||
|
||||
| 表 | 款式来源 | style_id | 写入端改动 |
|
||||
|---|---|---|---|
|
||||
| `daily_style_metrics` | loader 款式清单 | NOT NULL | `aggregate_daily_final.py` + `db.upsert_daily_metrics` |
|
||||
| `weekly_style_metrics` | 从 daily 聚合 | NOT NULL | `weekly_aggregate.py` |
|
||||
| `erp_all_shop_style_daily_metrics` | loader ERP 款 | NOT NULL | `backfill_erp_all_shop_daily.py` |
|
||||
| `daily_persona_metrics` | loader | NOT NULL | `collect_*_persona_to_bitable.py` |
|
||||
| `main_image_creatives` | loader | NOT NULL | `insert_main_image_records.py` / `insert_jd_main_image_records.py` |
|
||||
| `fact_style_analysis_reports` | loader | NOT NULL | `analyze_style.py` |
|
||||
| `decline_events` / `fact_alert_history` | daily 派生 | NOT NULL | `check_nine_day_decline.py` |
|
||||
| `jd_self_weekly_sales` | loader | NOT NULL | `jd_self_inventory_sales_collector.py` |
|
||||
| `persona_screenshots` | loader | NOT NULL | `taobao_dmp_item_crowd_insight_screenshots.py` |
|
||||
| `product_daily_metrics` | 平台商品 ID 反查 | **可空** | `import_product_daily.py` |
|
||||
| `jd_product_ad_daily_metrics` / `dy_product_ad_daily_metrics` | 名称/ID 反查 | **可空**(已有 `match_status`) | 两个 `import_*_ad_*.py` |
|
||||
| 差评三表 | 反查 | **可空** | `collectors/*_negative_reviews.py` 入库侧 |
|
||||
|
||||
**`product_daily_metrics` 的回填要处理系列聚合带来的歧义**(当前 0 条歧义,但要按锚点优先写死优先级):
|
||||
|
||||
```sql
|
||||
UPDATE product_daily_metrics m
|
||||
SET style_id = pick.style_id
|
||||
FROM LATERAL (
|
||||
SELECT d.id AS style_id
|
||||
FROM dim_style d
|
||||
WHERE (m.platform = 'tm' AND d.tm_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'jd' AND d.jd_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'dy' AND d.dy_spus @> to_jsonb(m.product_id))
|
||||
ORDER BY d.is_series_anchor DESC, LENGTH(d.style_name), d.id
|
||||
LIMIT 1
|
||||
) pick
|
||||
WHERE m.style_id IS NULL;
|
||||
```
|
||||
|
||||
(真实脚本里用 Python 侧 `_resolve_style_match()` 复用同一套优先级,SQL 只用于一次性回填。)
|
||||
|
||||
回填后预期的非空比例 = 2.2 节的实测值:**dy 47% / jd 55% / tm 48%**。这不是 bug——剩下的是 458 个未建档商品 ID,要靠 P5 的人补 spus 才能涨。
|
||||
|
||||
### P4 约束 + 下游切 join
|
||||
|
||||
1. ~~对「必须匹配」的表加外键~~ → **改为不建物理外键(2026-11-01 定案)**。
|
||||
|
||||
原始设想是给 10 张「必须匹配」的表加 `FOREIGN KEY (style_id) REFERENCES dim_style(id) ON UPDATE CASCADE`,
|
||||
实施后又按业主意见拆掉了。理由是采集链路的语义与硬约束冲突:
|
||||
|
||||
- 采集要求 **fail-soft**:一个商品 ID 没建档,正确行为是把该行 `style_id` 留 NULL 并标
|
||||
`unmatched`,继续写其它行。物理外键会让整批 INSERT/UPDATE 直接报错,等于「一个商品
|
||||
未建档 → 当天数据全丢」,比留 NULL 更糟。
|
||||
- 款式会动:拆系列、合并同款、重建 dim_style 行都要先卸约束再动数据,运维成本不小。
|
||||
- data-hub 侧 22 张表本来就用 `style_id` 关联、**不带物理外键**,两边保持一致才好维护。
|
||||
|
||||
改为**逻辑外键**,完整性由三处保证:
|
||||
|
||||
| 环节 | 手段 |
|
||||
|---|---|
|
||||
| 写入 | loader 下发 `style_id`(源自 dim_style 本身);缺失时 `attach_style_ids` / `resolve_style_ids_by_product` 批量兜底 |
|
||||
| 校验 | `backfill_style_id.py --dry-run` 预演;`v_style_id_coverage` 报每张表覆盖率 |
|
||||
| 观测 | `v_style_unmatched_product`(日报口径)/ `v_style_unmatched_source`(全源口径)列未建档清单 |
|
||||
|
||||
老库里早期建过的 10 个 `<表名>_style_id_fk` 由 `schema.sql` 里的 `DROP CONSTRAINT IF EXISTS`
|
||||
幂等清理;只保留 `*_style_id_idx` 索引(join 性能靠索引,不靠约束)。
|
||||
|
||||
2. data-hub 侧把 `metric.style_name = style.name` 换成 `metric.style_id = style.id`。涉及 `ProductLifecycleMapper.xml` 第 120、155 行(`daily_style_metrics`),以及 `JOIN dim_style dim ON dim.style_name = style.name`(第 15 行)。改完后 text join 变 bigint join,是纯性能收益;`style_name` 列仍在,改错了可以立即回切。
|
||||
|
||||
3. 加测试:
|
||||
- `tests/.../test_daily_style_metrics_style_id.py`:写入必须带 `style_id`,且等于 loader 输出的 id
|
||||
- `tests/.../test_style_id_coverage.py`:`v_style_id_coverage` 里「必须匹配」的表 `with_style_id == rows`
|
||||
- 现有 `test_import_tmall_product_ads.py` 的风格(断言 SQL 形状)继续用在新增字段上
|
||||
|
||||
### P5 数据治理(不是代码活,但决定 P3 的收益)
|
||||
|
||||
`v_style_unmatched_product` 出来的 458 个商品 ID 需要在 dim_style 建档页补 `tm_spus/jd_spus/dy_spus`。分批推进:先补最近 90 天有销量的,历史行允许永久 NULL。
|
||||
|
||||
## 五、风险与取舍
|
||||
|
||||
| 风险 | 影响 | 应对 |
|
||||
|---|---|---|
|
||||
| 款名在 dim_style 被改,历史表 `style_name` 变孤儿 | 现在 join 直接丢数据,这正是要 style_id 的原因 | P3 之后 join 走 id,款名改动不再影响 |
|
||||
| 加 `NOT NULL` 时某天有未建档款写入 | 写入失败,工作流中断 | 先在写入端做「未匹配记 NULL + 打 WARN」,P5 补齐后再收紧为 NOT NULL |
|
||||
| `product_reviews` 下线后没有好评样本 | 风格分析评价分布退化 | 明确接受;需要时扩差评采集器等级,不恢复旧表 |
|
||||
| 改 PK(有人会想顺手做) | data-hub 6 处 join + 飞书脚本全回归 | **本次明确不做**,PK 保持 style_name |
|
||||
| 差评表天猫侧没有 product_id | 只能靠商品名反查款式 | 采集器补 product_id(页面有商品链接可解析),或退化为名称匹配并标记 `unmatched` |
|
||||
|
||||
## 六、验收标准
|
||||
|
||||
1. `v_style_id_coverage`:`daily_style_metrics`、`erp_all_shop_style_daily_metrics`、`weekly_style_metrics` 的 `with_style_id = rows`(100%)。
|
||||
2. `product_daily_metrics` 的 `style_id` 非空比例不低于 2.2 节的实测基线(47%/55%/48%),且逐月上升。
|
||||
3. `product_reviews` 无新增行;两个消费方在 `v_negative_reviews` 上跑通,风格分析与营销日报输出结构不变。
|
||||
4. data-hub 产品生命页、营销中台预算页的款式指标与改造前同月数值一致(对账 2026-10 全月)。
|
||||
|
||||
## 七、建议执行顺序
|
||||
|
||||
P0 → P1(下线评论链路,独立可交付)→ P2(loader 一行改动,验证输出)→ P3(分表灰度,先做 `daily_style_metrics`)→ P4(约束 + 下游切换)→ P5(补建档)。
|
||||
|
||||
P1 与 P2/P3 没有依赖关系,可以先做 P1 拿到收益(少一条采集链路、少一张表),再从容做款式主键统一。
|
||||
|
||||
## 八、实施记录(2026-11-01)
|
||||
|
||||
### 已落地
|
||||
|
||||
| 阶段 | 成果 |
|
||||
|---|---|
|
||||
| P0 | `v_style_unmatched_product`、`v_style_id_coverage` 两个观测视图已建并验证 |
|
||||
| P1 | 删 `import_product_reviews.py` / `orchestrate_review_collection.py`;移除 `product.review.orchestrate` 命令与 runtime binding;`run_collect` 不再调评论入库;建 `v_negative_reviews`;两个消费方改读视图;`product_reviews` 归档为 `product_reviews_archived_20261101`(1936 行已备份到 `var/tmp/product_reviews_archive_20261101.json`) |
|
||||
| P2 | `merge_lifecycle_style_records` 输出 `style_id` + `style_match_method`;loader 透传;实库验证 **204/204 款全部拿到 style_id** |
|
||||
| P3 | 14 张采集表加 `style_id`(+ `style_match_method`)+ 索引;db 层 `attach_style_ids` / `resolve_style_ids_by_product` 统一兜底;`backfill_style_id.py` 回填历史 |
|
||||
| P4 | ~~10 张「必须匹配」表加 `style_id → dim_style(id)` 物理外键~~ → 已按要求**全部拆除**,改逻辑外键(库里 0 个 FK,只留 14 个 `style_id` 索引);新增 `tests/modules/product_commerce/test_style_id_identity.py`(28 条契约测试) |
|
||||
| P5 | 提供 `v_style_unmatched_product` 清单(dy 170 / jd 99 / tm 189),需业务侧补 spus |
|
||||
|
||||
### 回填结果(实库)
|
||||
|
||||
| 表 | 行数 | 有 style_id | 比例 |
|
||||
|---|---|---|---|
|
||||
| daily_style_metrics | 17,352 | 17,352 | 100% |
|
||||
| weekly_style_metrics | 532 | 532 | 100% |
|
||||
| erp_all_shop_style_daily_metrics | 9,399 | 9,399 | 100% |
|
||||
| daily_persona_metrics | 2,357 | 2,357 | 100% |
|
||||
| main_image_creatives | 9,094 | 9,094 | 100% |
|
||||
| fact_style_analysis_reports | 465 | 465 | 100% |
|
||||
| decline_events / fact_alert_history | 14 / 14 | 14 / 14 | 100% |
|
||||
| persona_screenshots | 25 | 25 | 100% |
|
||||
| jd_self_weekly_sales | 117 | 111 | 94.9%(6 款已不在 dim_style) |
|
||||
| tm_product_ad_daily_metrics | 11,727 | 11,108 | 94.7% |
|
||||
| jd_product_ad_daily_metrics | 33,808 | 24,707 | 73.1% |
|
||||
| dy_product_ad_daily_metrics | 18,202 | 10,875 | 59.7% |
|
||||
| product_daily_metrics | 75,568 | 37,098 | 49.1%(未建档商品 ID 所致) |
|
||||
|
||||
### 与方案不同之处
|
||||
|
||||
1. **天猫广告表已先行迁到独立表**:方案编写时天猫指标还写在
|
||||
`product_daily_metrics.raw_data.tm_ad_metrics`,实施期间该迁移已完成,因此改为给
|
||||
`tm_product_ad_daily_metrics` 加 `style_id`(并纳入 `v_style_id_coverage`),
|
||||
并未改动 `raw_data` 节点。
|
||||
2. **写入端没有逐个脚本改**:方案原计划在 13 个写入脚本里各自填充,实际收敛到 db 层
|
||||
的 `attach_style_ids` / `resolve_style_ids_by_product`(一次批量查询),再加 loader
|
||||
下发。少改 13 处、不会漏,代价是 upsert 多一次按款名/商品 ID 的查询。
|
||||
3. **`dy_product_scraping.py` 默认流程的评论采集直接摘除**(仍保留 `--reviews-only`
|
||||
显式入口),比方案建议的「保留但不再进 product.daily」更彻底。
|
||||
4. **物理外键最终没有保留**:P4 建过的 10 个 `<表名>_style_id_fk` 已全部 DROP,
|
||||
`schema.sql` 改为「幂等清理 + 只建索引」。`style_id` 与 `dim_style.id` 是纯逻辑
|
||||
外键关系,理由见 P4 段(fail-soft 采集语义、款式可重建、与 data-hub 对齐)。
|
||||
|
||||
### 仍待处理
|
||||
|
||||
- **data-hub 侧 join 未切**(`ProductLifecycleMapper.xml` 6 处仍是 `metric.style_name = style.name`)。
|
||||
现在两边都有 id,可以随时切;切换后是 bigint join 替代 text join。
|
||||
- P5 补建档:458 个商品 ID(dy 170 / jd 99 / tm 189)写进 `dim_style.spus`。
|
||||
- 既有红灯(与本次无关):`test_tm_brand_profiles` 的 backfill 用例、`shop_intelligence`
|
||||
的 12 个 jd 竞店用例,改动前即失败。
|
||||
|
||||
## 九、P5 补 spus 实施记录(2026-11-01)
|
||||
|
||||
### 依据的权威字段
|
||||
|
||||
补 spus 的匹配依据不是商品标题,而是 **`product_daily_metrics.product_no` —— 它存的就是款名**:
|
||||
|
||||
| 平台 | `product_no` 填充率 | 可用性 |
|
||||
|---|---:|---|
|
||||
| 天猫 | 100%(29552/29552) | 权威 |
|
||||
| 京东 | 97.8%(16664/17044) | 权威 |
|
||||
| 抖音 | 0%(0/28972) | 只能靠标题 |
|
||||
|
||||
这一条此前被忽略,导致「458 个未建档」被误判为只能猜标题。实际天猫/京东 136 个商品
|
||||
靠 `product_no` 就精确命中,另有 31 个靠前后缀近似命中(「蓝鹊pro」→「蓝鹊」、
|
||||
「知更大号」→「知更」、「星迹」→「星迹2」)。
|
||||
|
||||
注意 `product_no` **不是** ERP 编码:`dim_style.erp_style_codes` 存的是 `["10510"]`
|
||||
这种数字码,两者用 `product_no = erp_style_codes` 只能对上 2 个,别混淆。
|
||||
|
||||
### 匹配算法(四路,按优先级)
|
||||
|
||||
1. `product_no` 精确 = `style_name` → 直接写;
|
||||
2. `product_no` 与款名互为前后缀且长度差 ≤ 3,且候选唯一 → 直接写;
|
||||
3. 标题包含完整款名(抖音只能走这条)→ 用「命中位置后的版本后缀」防串款:
|
||||
标题「盖亚斜挎包**3**」、`dim_style` 有「盖亚斜挎3」时必须指后者而非「盖亚斜挎」;
|
||||
若该版本款没建档,判「版本款未建档」而不是硬归到基础款;
|
||||
4. 编码与标题互相矛盾 → 进人工清单,**不自动写**。
|
||||
|
||||
实测第 4 类里最贵的一条:天猫 `649545919334` 商品编码写「理岩」、标题写「阿波罗air」,
|
||||
支付额 **50.2 万**。猜错就是 50 万挂错款,只能人工定。
|
||||
|
||||
### 写入结果
|
||||
|
||||
| 项 | 数值 |
|
||||
|---|---:|
|
||||
| 写入商品 ID | 199 个 |
|
||||
| 涉及款式 | 67 个 |
|
||||
| 覆盖 `product_daily_metrics` 行 | 22,636 |
|
||||
| 未写(物料 / 冲突待定) | 87 个 |
|
||||
|
||||
合并策略:`tm_spus / jd_spus / dy_spus` 取**并集追加**,不覆盖已有值;写前备份到
|
||||
`var/tmp/dim_style_spus_backup_20261101.json`(204 行)。
|
||||
|
||||
### 覆盖率变化
|
||||
|
||||
| 表 | 补前 | 补后 |
|
||||
|---|---:|---:|
|
||||
| product_daily_metrics | 49.1% | **79.0%** |
|
||||
| tm_product_ad_daily_metrics | 94.7% | **99.4%** |
|
||||
| jd_product_ad_daily_metrics | 73.1% | **92.4%** |
|
||||
| dy_product_ad_daily_metrics | 59.7% | **83.2%** |
|
||||
|
||||
广告表的提升来自新增的第三段回填(按商品 ID + 补空 `style_name`):京东广告里
|
||||
32 个商品 ID 就占了 9,101 行空值,补完 `style_name` 空白从 9,095 降到 2,554,
|
||||
抖音从 7,327 降到 3,062。这意味着 **data-hub 侧不改任何 SQL,按款名的 join 已经多认回
|
||||
11,359 行**。
|
||||
|
||||
### 剩余 259 个未建档 ID 的构成
|
||||
|
||||
| 分类 | 商品数 | 支付额 | 处置 |
|
||||
|---|---:|---:|---|
|
||||
| 待建档-真款 | 128 | 6.9 万 | 需在 dim_style 新建款(星云系列链接、拾影单肩包、幻月、纪元、极地冰冠、阿瑞斯、盖亚S1、漫游者、光/暗系列等) |
|
||||
| 待建档-ozko 线 | 20 | 0.2 万 | ozko 子品牌款(逐云、微笑系列、甜酷系列、银鸥、独角兽、HB、cdbb、761X…) |
|
||||
| 白名单-物料配件 | 30 | 5.6 万 | 不该进 spus(内胆包、数据线、风扇、防丢器…) |
|
||||
| 白名单-服务/券 | 4 | 0 | 补邮费、兑换券、补贴 |
|
||||
| 冲突待定 | 10 | 62.9 万 | 编码与标题矛盾,需人工定 |
|
||||
|
||||
清单:`var/tmp/spus_todo.csv`(按分档 + 金额排序)。
|
||||
|
||||
### 新增的长期机制
|
||||
|
||||
- `backfill_style_id.py` 增加 `AD_PRODUCT_TABLES` 与 `_backfill_ad_tables_by_product`,
|
||||
广告表按 `spu_id / product_id` 二次回填,并只在 `style_name` 为空时补权威款名
|
||||
(`COALESCE(NULLIF(BTRIM(style_name), ''), %s)`,不覆盖平台给出的值)。
|
||||
- 新增视图 `v_style_unmatched_source`:`product_daily_metrics` + 三张广告表统一口径,
|
||||
补建档只看这一个视图就够(实测与原 `v_style_unmatched_product` 数值一致,
|
||||
说明没有纯投流商品只出现在广告表)。
|
||||
@@ -766,6 +766,11 @@ class RuntimeIntegrationCatalog:
|
||||
f"{binding.profile_dir.name}-luggage"
|
||||
)
|
||||
),
|
||||
"GUANGHE_NEW_BAG_USER_DATA_DIR": str(
|
||||
binding.profile_dir.with_name(
|
||||
f"{binding.profile_dir.name}-new-bag"
|
||||
)
|
||||
),
|
||||
"DY_COOKIES_FILE": str(binding.cookie_file),
|
||||
"DY_STORAGE_STATE_FILE": str(binding.storage_state_file),
|
||||
}
|
||||
|
||||
@@ -360,6 +360,10 @@ def merge_lifecycle_style_records(
|
||||
record = dict(row)
|
||||
for field in URL_FIELDS:
|
||||
record[field] = ""
|
||||
# 已不在 dim_style 的残留款:没有主键可下发,显式标记未匹配,
|
||||
# 避免下游把 None 当成「还没回填」而不是「确实没建档」。
|
||||
record["style_id"] = None
|
||||
record["style_match_method"] = "unmatched"
|
||||
records.append(record)
|
||||
continue
|
||||
|
||||
@@ -423,6 +427,17 @@ def merge_lifecycle_style_records(
|
||||
template["sales_bitable_url"] = sales_bitable_url
|
||||
template["series_name"] = series_name
|
||||
template["is_series_anchor"] = is_series_anchor
|
||||
# 款式主键统一 (2026-11-01): dim_style.id 作为正式字段下发。
|
||||
# 原先它只被编码进 source_record_id="lifecycle_style:<id>" 字符串,写入脚本
|
||||
# 拿不到,只能靠 style_name 关联——改款名历史行就成孤儿。这里提成字段后,
|
||||
# 所有经 runtime_workflow_configs() 取款的脚本自动获得 style_id。
|
||||
style_id_value = lifecycle.get("id")
|
||||
template["style_id"] = (
|
||||
int(style_id_value) if style_id_value is not None else None
|
||||
)
|
||||
template["style_match_method"] = (
|
||||
"dim_style" if style_id_value is not None else "unmatched"
|
||||
)
|
||||
for field in URL_FIELDS:
|
||||
template[field] = str(lifecycle.get(field) or "").strip()
|
||||
# 采集开关由产品生命进程在 dim_style 上维护(系列级,读锚点那行)。
|
||||
|
||||
@@ -191,7 +191,6 @@ _WORKFLOW_NAMES = {
|
||||
"product.backfill": "商品数据补采",
|
||||
"product.market_rank": "市场排行周采集",
|
||||
"product.tmall_baibu_apply": "天猫百亿补贴批量报名",
|
||||
"product.review_collection": "商品评价采集",
|
||||
"product.weekly_aggregate.documented_missing": "商品周聚合(待接入)",
|
||||
"shop.metrics.weekly": "店铺经营指标周采集",
|
||||
"shop.competitor.weekly": "竞店指标周采集",
|
||||
@@ -2247,7 +2246,7 @@ class WorkflowConsoleService:
|
||||
jd_customer_service_store: str | None = None
|
||||
if workflow_id == "product.video_upload":
|
||||
raw_store = payload.get("store", "flagship")
|
||||
if raw_store not in {"flagship", "luggage"}:
|
||||
if raw_store not in {"flagship", "luggage", "new_bag"}:
|
||||
raise ConsoleRequestError("天猫视频上传店铺参数无效")
|
||||
video_upload_store = str(raw_store)
|
||||
tmall_topic_keyword = _normalize_tmall_topic_keyword(
|
||||
|
||||
@@ -1019,18 +1019,28 @@ def _creative_context(metric_date: date, style_names: list[str]) -> list[dict[st
|
||||
|
||||
|
||||
def _review_context(metric_date: date, style_names: list[str]) -> list[dict[str, Any]]:
|
||||
"""近 7 天差评归因(读 v_negative_reviews)。
|
||||
|
||||
2026-11-01 起 ``product_reviews`` 下线,改读三平台差评归一视图。口径随之变化:
|
||||
|
||||
* 数据源只剩差评,因此 ``sample_size`` = 差评条数、``negative_rate`` 恒为 1.0,
|
||||
不再表示「差评/全部评价」;真正要看比例的场合请取平台评价总数另算。
|
||||
* 输出门槛从「窗口内至少 5 条评价」改为「至少 1 条差评」——旧门槛在只有差评
|
||||
的数据源上会导致几乎永不产出。
|
||||
* 覆盖天猫(旧表天猫为 0 行)、京东(旧表 8 月断更),负向样本反而更全。
|
||||
"""
|
||||
|
||||
if not style_names:
|
||||
return []
|
||||
negative_terms = ("差", "坏", "退", "重", "小", "掉色", "线头", "不值", "失望")
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT style_name, platform, content, score, tags
|
||||
FROM product_reviews
|
||||
SELECT style_name, platform, content
|
||||
FROM v_negative_reviews
|
||||
WHERE style_name = ANY(%s) AND comment_date BETWEEN %s AND %s
|
||||
AND content IS NOT NULL AND BTRIM(content) <> ''
|
||||
ORDER BY style_name, comment_date DESC, id DESC
|
||||
ORDER BY style_name, comment_date DESC, review_at DESC
|
||||
""",
|
||||
(style_names, metric_date - timedelta(days=6), metric_date),
|
||||
)
|
||||
@@ -1040,17 +1050,15 @@ def _review_context(metric_date: date, style_names: list[str]) -> list[dict[str,
|
||||
grouped.setdefault(row[0], []).append(row)
|
||||
output = []
|
||||
for style_name, items in grouped.items():
|
||||
if len(items) < 5:
|
||||
continue
|
||||
negative = [item for item in items if (item[3] is not None and item[3] <= 2) or any(term in item[2] for term in negative_terms)]
|
||||
output.append({
|
||||
"style_name": style_name,
|
||||
"window_start": metric_date - timedelta(days=6),
|
||||
"window_end": metric_date,
|
||||
"sample_size": len(items),
|
||||
"negative_count": len(negative),
|
||||
"negative_rate": round(len(negative) / len(items), 4),
|
||||
"negative_examples": [item[2][:80] for item in negative[:2]],
|
||||
"negative_count": len(items),
|
||||
"negative_rate": 1.0,
|
||||
"negative_examples": [item[2][:80] for item in items[:2]],
|
||||
"source": "v_negative_reviews",
|
||||
})
|
||||
return output
|
||||
|
||||
|
||||
@@ -109,12 +109,26 @@ def missing_window_dates(rows: list[dict], start: date, end: date) -> list[date]
|
||||
|
||||
|
||||
def pull_reviews(style: str, start: date, end: date) -> dict:
|
||||
"""仅返回当前三日窗口内的评论分布与样本。"""
|
||||
"""仅返回当前三日窗口内的差评分布与样本。
|
||||
|
||||
2026-11-01 起 ``product_reviews`` 下线,改读三平台差评归一视图
|
||||
``v_negative_reviews``(tm + jd + dy 三张差评表 UNION)。
|
||||
|
||||
口径变化(必须知道):旧表存好/中/差全量,视图只有差评一档,所以这里
|
||||
返回的 key 会集中在 ``<平台>|负面评价``。换来的是覆盖率更高——天猫差评
|
||||
旧表根本没有(0 行),京东旧表 8 月就断更。
|
||||
|
||||
款式归因走 ``style_id``(dim_style.id),款名只作显示冗余。
|
||||
"""
|
||||
|
||||
with get_conn() as c, c.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT platform, rating, COUNT(*) n, array_agg(content ORDER BY comment_date DESC) samples
|
||||
FROM product_reviews
|
||||
WHERE style_name=%s AND comment_date BETWEEN %s AND %s
|
||||
"""SELECT platform, rating, COUNT(*) n,
|
||||
array_agg(content ORDER BY comment_date DESC) samples
|
||||
FROM v_negative_reviews
|
||||
WHERE style_id = (SELECT id FROM dim_style WHERE style_name = %s)
|
||||
AND comment_date BETWEEN %s AND %s
|
||||
AND content IS NOT NULL AND BTRIM(content) <> ''
|
||||
GROUP BY platform, rating""",
|
||||
(style, start, end),
|
||||
)
|
||||
@@ -380,7 +394,7 @@ def aggregate_style_total(platforms: dict[str, dict]) -> dict:
|
||||
LEGACY_SYSTEM = """你是光影行星 (GYXX) 的资深电商数据分析师, 擅长把数据现象挖到根因并给出可执行方案.
|
||||
用户会给定一个款式近 3 天的真实业务数据:
|
||||
- 站内 (店铺自身): 天猫/京东/抖音三平台当前 3 天的 daily_style_metrics、
|
||||
product_reviews、daily_persona_metrics、main_image_creatives
|
||||
v_negative_reviews(三平台差评)、daily_persona_metrics、main_image_creatives
|
||||
- 站外 (种草营销): cmt_notes_master (笔记+合作投入), cmt_comments (笔记评论),
|
||||
cmt_creators (创作者静态属性)
|
||||
|
||||
@@ -787,7 +801,7 @@ def build_user_prompt(style: str, start: date, end: date, payload: dict) -> str:
|
||||
# 人物画像
|
||||
{json.dumps(payload['persona'], ensure_ascii=False, indent=2)}
|
||||
|
||||
# 商品评价
|
||||
# 商品差评 (仅负向评价, 覆盖天猫/京东/抖音; 无好评与中评样本)
|
||||
{json.dumps(payload['product_reviews'], ensure_ascii=False, indent=2)}
|
||||
|
||||
# 主图表现
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
#!/usr/bin/env python3
|
||||
"""回填采集表的 style_id (= dim_style.id),完成款式主键统一。
|
||||
|
||||
背景
|
||||
----
|
||||
2026-11-01 之前,采集端所有表只用 ``style_name`` 关联款式,而 data-hub 侧 22 张
|
||||
表用的是 ``style_id``(= dim_style.id)。同一库里两种引用方式的后果是:款名在
|
||||
dim_style 被改一次,历史指标表就整体变孤儿,join 静默丢数据。
|
||||
|
||||
本脚本负责把**历史数据**补上 style_id;**新数据**由写入端负责:
|
||||
* loader 下发(``merge_lifecycle_style_records`` → ``style_config_loader``)
|
||||
* db 层兜底(``db.attach_style_ids`` / ``db.resolve_style_ids_by_product``)
|
||||
|
||||
两类表,两种回填口径
|
||||
--------------------
|
||||
1. **按款名**(13 张):款式清单本来就来自 dim_style loader,理论上 100% 命中。
|
||||
2. **按平台商品 ID**(``product_daily_metrics`` + 三张广告表):表里没有可用的
|
||||
款名列(``product_daily_metrics`` 完全没有;广告表的 ``style_name`` 由平台
|
||||
报表决定,京东 27%、抖音 40% 是空的),只能拿 ``platform + product_id`` 去
|
||||
``tm_spus / jd_spus / dy_spus`` 里反查。未建档的商品 ID 回填不了 —— 这不是
|
||||
脚本的问题,要用 ``v_style_unmatched_product`` 推动补建档。
|
||||
广告表回填时会顺带把空的 ``style_name`` 补成 dim_style 的权威款名,让下游
|
||||
现有按款名的 join 立即生效。
|
||||
|
||||
用法
|
||||
----
|
||||
.venv/Scripts/python.exe backfill_style_id.py --dry-run # 只报数
|
||||
.venv/Scripts/python.exe backfill_style_id.py # 执行
|
||||
.venv/Scripts/python.exe backfill_style_id.py --tables daily_style_metrics
|
||||
.venv/Scripts/python.exe backfill_style_id.py --skip-schema # 不跑建表
|
||||
.venv/Scripts/python.exe backfill_style_id.py --skip-ad-tables # 只补款名表
|
||||
|
||||
幂等:所有 UPDATE 都带 ``style_id IS NULL`` 条件,重复执行不会覆盖已有归属。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from typing import Any, Sequence
|
||||
|
||||
try:
|
||||
from .db import (
|
||||
STYLE_MATCH_BY_NAME,
|
||||
STYLE_MATCH_BY_PRODUCT,
|
||||
STYLE_MATCH_UNMATCHED,
|
||||
get_conn,
|
||||
init_schema,
|
||||
resolve_style_ids_by_product,
|
||||
)
|
||||
except ImportError: # Direct script execution from the module root.
|
||||
from db import ( # type: ignore[no-redef]
|
||||
STYLE_MATCH_BY_NAME,
|
||||
STYLE_MATCH_BY_PRODUCT,
|
||||
STYLE_MATCH_UNMATCHED,
|
||||
get_conn,
|
||||
init_schema,
|
||||
resolve_style_ids_by_product,
|
||||
)
|
||||
|
||||
|
||||
# 按款名回填:这些表的 style_name 来自 dim_style 款式清单,属于「必须匹配」。
|
||||
NAME_TABLES: tuple[str, ...] = (
|
||||
"daily_style_metrics",
|
||||
"weekly_style_metrics",
|
||||
"erp_all_shop_style_daily_metrics",
|
||||
"daily_persona_metrics",
|
||||
"main_image_creatives",
|
||||
"fact_style_analysis_reports",
|
||||
"decline_events",
|
||||
"fact_alert_history",
|
||||
"jd_self_weekly_sales",
|
||||
"persona_screenshots",
|
||||
"jd_product_ad_daily_metrics",
|
||||
"dy_product_ad_daily_metrics",
|
||||
"tm_product_ad_daily_metrics",
|
||||
)
|
||||
|
||||
# 按平台商品 ID 回填:允许未匹配(商品 ID 还没建档到 dim_style.spus)。
|
||||
PRODUCT_TABLE = "product_daily_metrics"
|
||||
PLATFORM_COLUMNS = ("tm", "jd", "dy")
|
||||
|
||||
# 广告表:表里带平台商品 ID,但 style_name 常为空(平台报表不给款式名)。
|
||||
# 只按款名的回填对它们无效,必须再走一遍商品 ID 反查。
|
||||
# {表名: (商品 ID 列, 平台编码)}
|
||||
AD_PRODUCT_TABLES: dict[str, tuple[str, str]] = {
|
||||
"tm_product_ad_daily_metrics": ("product_id", "tm"),
|
||||
"jd_product_ad_daily_metrics": ("spu_id", "jd"),
|
||||
"dy_product_ad_daily_metrics": ("product_id", "dy"),
|
||||
}
|
||||
|
||||
|
||||
def _coverage(conn, table: str) -> tuple[int, int]:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"SELECT COUNT(*), COUNT(style_id) FROM {table}"
|
||||
)
|
||||
rows, with_id = cur.fetchone()
|
||||
return int(rows or 0), int(with_id or 0)
|
||||
|
||||
|
||||
def _backfill_by_name(conn, table: str, *, dry_run: bool) -> dict[str, Any]:
|
||||
"""按款名精确回填。返回 {before, after, updated}。"""
|
||||
|
||||
before_rows, before_with_id = _coverage(conn, table)
|
||||
if dry_run:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""SELECT COUNT(*)
|
||||
FROM {table} t
|
||||
JOIN dim_style s ON s.style_name = t.style_name
|
||||
WHERE t.style_id IS NULL"""
|
||||
)
|
||||
pending = int(cur.fetchone()[0] or 0)
|
||||
return {
|
||||
"table": table,
|
||||
"rows": before_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"updated": pending,
|
||||
"dry_run": True,
|
||||
}
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""UPDATE {table} t
|
||||
SET style_id = s.id,
|
||||
style_match_method = COALESCE(t.style_match_method, %s)
|
||||
FROM dim_style s
|
||||
WHERE t.style_id IS NULL
|
||||
AND t.style_name = s.style_name""",
|
||||
(STYLE_MATCH_BY_NAME,),
|
||||
)
|
||||
updated = cur.rowcount or 0
|
||||
after_rows, after_with_id = _coverage(conn, table)
|
||||
return {
|
||||
"table": table,
|
||||
"rows": after_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"with_style_id_after": after_with_id,
|
||||
"updated": updated,
|
||||
}
|
||||
|
||||
|
||||
def _backfill_product_table(conn, *, dry_run: bool) -> dict[str, Any]:
|
||||
"""按 platform + product_id 反查回填 product_daily_metrics。
|
||||
|
||||
逐平台取去重商品 ID → 用 db.resolve_style_ids_by_product 解析(锚点优先、
|
||||
款名最短优先,与写入端同一套优先级)→ 按 (platform, product_id) 批量更新。
|
||||
"""
|
||||
|
||||
before_rows, before_with_id = _coverage(conn, PRODUCT_TABLE)
|
||||
planned: list[tuple[int, str, str]] = []
|
||||
unmatched_total = 0
|
||||
with conn.cursor() as cur:
|
||||
for platform in PLATFORM_COLUMNS:
|
||||
cur.execute(
|
||||
f"""SELECT DISTINCT product_id FROM {PRODUCT_TABLE}
|
||||
WHERE platform = %s AND style_id IS NULL AND product_id IS NOT NULL""",
|
||||
(platform,),
|
||||
)
|
||||
product_ids = [str(row[0]) for row in cur.fetchall() if row[0] is not None]
|
||||
if not product_ids:
|
||||
continue
|
||||
matched = resolve_style_ids_by_product(conn, platform, product_ids)
|
||||
unmatched_total += len(product_ids) - len(matched)
|
||||
planned.extend(
|
||||
(style_id, platform, product_id)
|
||||
for product_id, style_id in matched.items()
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
return {
|
||||
"table": PRODUCT_TABLE,
|
||||
"rows": before_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"planned_keys": len(planned),
|
||||
"unmatched_product_ids": unmatched_total,
|
||||
"dry_run": True,
|
||||
}
|
||||
|
||||
updated = 0
|
||||
with conn.cursor() as cur:
|
||||
for style_id, platform, product_id in planned:
|
||||
cur.execute(
|
||||
f"""UPDATE {PRODUCT_TABLE}
|
||||
SET style_id = %s, style_match_method = %s
|
||||
WHERE platform = %s AND product_id = %s AND style_id IS NULL""",
|
||||
(style_id, STYLE_MATCH_BY_PRODUCT, platform, product_id),
|
||||
)
|
||||
updated += cur.rowcount or 0
|
||||
# 回填不了的显式标记 unmatched,供 v_style_unmatched_product 统计
|
||||
cur.execute(
|
||||
f"""UPDATE {PRODUCT_TABLE}
|
||||
SET style_match_method = %s
|
||||
WHERE style_id IS NULL AND style_match_method IS NULL""",
|
||||
(STYLE_MATCH_UNMATCHED,),
|
||||
)
|
||||
|
||||
after_rows, after_with_id = _coverage(conn, PRODUCT_TABLE)
|
||||
return {
|
||||
"table": PRODUCT_TABLE,
|
||||
"rows": after_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"with_style_id_after": after_with_id,
|
||||
"updated": updated,
|
||||
"unmatched_product_ids": unmatched_total,
|
||||
}
|
||||
|
||||
|
||||
def _backfill_ad_tables_by_product(
|
||||
conn, tables: Sequence[str], *, dry_run: bool
|
||||
) -> list[dict[str, Any]]:
|
||||
"""按平台商品 ID 反查回填广告表的 style_id(顺带补空的 style_name)。
|
||||
|
||||
广告表的 ``style_name`` 由平台报表决定,经常为空(京东 27%、抖音 40%),
|
||||
只按款名回填对它们完全无效。这里改为用 ``spu_id`` / ``product_id`` 去
|
||||
``dim_style.<platform>_spus`` 反查,与写入端 ``resolve_style_ids_by_product``
|
||||
共用同一套优先级。
|
||||
|
||||
``style_name`` 只在原来为空时补写(``NULLIF(BTRIM(...), '')``),不覆盖
|
||||
平台已经给出的值 —— 这样 data-hub 侧现有按款名 join 的口径立刻能多认回
|
||||
一批行,不必等主键切换。
|
||||
"""
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for table in tables:
|
||||
id_column, platform = AD_PRODUCT_TABLES[table]
|
||||
before_rows, before_with_id = _coverage(conn, table)
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""SELECT DISTINCT {id_column} FROM {table}
|
||||
WHERE style_id IS NULL AND {id_column} IS NOT NULL
|
||||
AND BTRIM({id_column}) <> ''"""
|
||||
)
|
||||
product_ids = [str(row[0]) for row in cur.fetchall() if row[0] is not None]
|
||||
|
||||
matched: dict[str, int] = {}
|
||||
if product_ids:
|
||||
matched = resolve_style_ids_by_product(conn, platform, product_ids)
|
||||
|
||||
if dry_run:
|
||||
results.append(
|
||||
{
|
||||
"table": table,
|
||||
"rows": before_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"planned_keys": len(matched),
|
||||
"unmatched_product_ids": len(product_ids) - len(matched),
|
||||
"dry_run": True,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
name_by_id: dict[int, str] = {}
|
||||
if matched:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, style_name FROM dim_style WHERE id = ANY(%s)",
|
||||
(list({v for v in matched.values()}),),
|
||||
)
|
||||
name_by_id = {int(i): n for i, n in cur.fetchall()}
|
||||
|
||||
updated = 0
|
||||
with conn.cursor() as cur:
|
||||
for product_id, style_id in matched.items():
|
||||
cur.execute(
|
||||
f"""UPDATE {table}
|
||||
SET style_id = %s,
|
||||
style_match_method = %s,
|
||||
style_name = COALESCE(NULLIF(BTRIM(style_name), ''), %s)
|
||||
WHERE {id_column} = %s AND style_id IS NULL""",
|
||||
(style_id, STYLE_MATCH_BY_PRODUCT, name_by_id.get(style_id), product_id),
|
||||
)
|
||||
updated += cur.rowcount or 0
|
||||
cur.execute(
|
||||
f"""UPDATE {table} SET style_match_method = %s
|
||||
WHERE style_id IS NULL AND style_match_method IS NULL""",
|
||||
(STYLE_MATCH_UNMATCHED,),
|
||||
)
|
||||
|
||||
after_rows, after_with_id = _coverage(conn, table)
|
||||
results.append(
|
||||
{
|
||||
"table": table,
|
||||
"rows": after_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"with_style_id_after": after_with_id,
|
||||
"updated": updated,
|
||||
"unmatched_product_ids": len(product_ids) - len(matched),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def _print_coverage(conn) -> None:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT tbl, rows, with_style_id FROM v_style_id_coverage ORDER BY tbl")
|
||||
rows = cur.fetchall()
|
||||
cur.execute("SELECT platform, COUNT(*) FROM v_style_unmatched_product GROUP BY platform ORDER BY platform")
|
||||
unmatched = cur.fetchall()
|
||||
|
||||
print("\n[覆盖率] v_style_id_coverage")
|
||||
print(f" {'表':<38}{'行数':>9}{'有 style_id':>12}{'比例':>9}")
|
||||
for table, total, with_id in rows:
|
||||
ratio = (with_id / total * 100) if total else 0.0
|
||||
print(f" {table:<38}{total:>9}{with_id:>12}{ratio:>8.1f}%")
|
||||
|
||||
if unmatched:
|
||||
print("\n[未建档] v_style_unmatched_product")
|
||||
for platform, count in unmatched:
|
||||
print(f" {platform}: {count} 个商品 ID 未写进 dim_style.spus")
|
||||
print(" -> 这些行的 style_id 只能保持 NULL,需要在建档页补 spus")
|
||||
else:
|
||||
print("\n[未建档] v_style_unmatched_product: 无,全部商品 ID 已建档")
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="回填采集表 style_id (= dim_style.id),完成款式主键统一"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tables",
|
||||
default="",
|
||||
help="逗号分隔,只回填指定表;留空表示全部",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="只统计不写库")
|
||||
parser.add_argument(
|
||||
"--skip-schema",
|
||||
action="store_true",
|
||||
help="跳过 init_schema(列已存在时可用,节省时间)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-product-table",
|
||||
action="store_true",
|
||||
help="跳过 product_daily_metrics(按商品 ID 反查,最慢的一步)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-ad-tables",
|
||||
action="store_true",
|
||||
help="跳过三张广告表的按商品 ID 二次回填",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not args.skip_schema and not args.dry_run:
|
||||
init_schema()
|
||||
|
||||
requested = {
|
||||
item.strip() for item in str(args.tables or "").split(",") if item.strip()
|
||||
}
|
||||
name_tables = [t for t in NAME_TABLES if not requested or t in requested]
|
||||
run_product = (
|
||||
not args.skip_product_table
|
||||
and (not requested or PRODUCT_TABLE in requested)
|
||||
)
|
||||
ad_tables = (
|
||||
[]
|
||||
if args.skip_ad_tables
|
||||
else [t for t in AD_PRODUCT_TABLES if not requested or t in requested]
|
||||
)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
with get_conn() as conn:
|
||||
try:
|
||||
for table in name_tables:
|
||||
result = _backfill_by_name(conn, table, dry_run=args.dry_run)
|
||||
results.append(result)
|
||||
print(
|
||||
f"[{'DRY' if args.dry_run else 'OK '}] {table:<38} "
|
||||
f"{result['with_style_id_before']}/{result['rows']} -> "
|
||||
f"{result.get('updated', 0)} 行可填"
|
||||
)
|
||||
if run_product:
|
||||
result = _backfill_product_table(conn, dry_run=args.dry_run)
|
||||
results.append(result)
|
||||
if args.dry_run:
|
||||
detail = (
|
||||
f"{result['planned_keys']} 个商品 ID 可解析, "
|
||||
f"{result['unmatched_product_ids']} 个未建档"
|
||||
)
|
||||
else:
|
||||
detail = f"{result.get('updated', 0)} 行已填"
|
||||
print(
|
||||
f"[{'DRY' if args.dry_run else 'OK '}] {PRODUCT_TABLE:<38} "
|
||||
f"{result['with_style_id_before']}/{result['rows']} -> {detail}"
|
||||
)
|
||||
if args.dry_run:
|
||||
conn.rollback()
|
||||
else:
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
if ad_tables:
|
||||
# 广告表的 style_name 大量为空,按款名回填无效,必须再按商品 ID 走一遍。
|
||||
# 单独开事务,避免和上面的款名回填混在一个长事务里。
|
||||
with get_conn() as conn:
|
||||
try:
|
||||
ad_results = _backfill_ad_tables_by_product(
|
||||
conn, ad_tables, dry_run=args.dry_run
|
||||
)
|
||||
results.extend(ad_results)
|
||||
for result in ad_results:
|
||||
if args.dry_run:
|
||||
detail = (
|
||||
f"{result['planned_keys']} 个商品 ID 可解析, "
|
||||
f"{result['unmatched_product_ids']} 个未建档"
|
||||
)
|
||||
else:
|
||||
detail = f"{result.get('updated', 0)} 行已填"
|
||||
print(
|
||||
f"[{'DRY' if args.dry_run else 'OK '}] {result['table']:<38} "
|
||||
f"{result['with_style_id_before']}/{result['rows']} -> {detail} (按商品ID)"
|
||||
)
|
||||
if args.dry_run:
|
||||
conn.rollback()
|
||||
else:
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
total_updated = sum(int(r.get("updated", 0)) for r in results)
|
||||
print(f"\n{'[DRY]' if args.dry_run else '[DONE]'} 合计更新 {total_updated} 行")
|
||||
|
||||
if not args.dry_run:
|
||||
with get_conn() as conn:
|
||||
_print_coverage(conn)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -32,7 +32,7 @@ except ImportError: # Direct script execution from the module root.
|
||||
from runtime_paths import NORMALIZED_DATA_ROOT, STATE_ROOT, managed_data_path
|
||||
|
||||
|
||||
CACHE_SCHEMA_VERSION = 8
|
||||
CACHE_SCHEMA_VERSION = 9 # 9: 款式主键统一, 每条 style 增加 style_id / style_match_method
|
||||
|
||||
# 系列级飞书地址:约定只维护在锚点行,子款误填也会被整系列聚合去重后统一下发。
|
||||
# 顺序即写入目标表的优先级说明顺序,不含 bitable_sku_master(人工维护,不参与采集)。
|
||||
@@ -307,6 +307,10 @@ class StyleConfigLoader:
|
||||
"destinations": row.get("destinations", []),
|
||||
"series_name": row.get("series_name", ""),
|
||||
"is_series_anchor": bool(row.get("is_series_anchor", False)),
|
||||
# 款式主键统一 (2026-11-01): dim_style.id 正式下发, 写入脚本
|
||||
# 直接用它落库, style_name 只作显示冗余。
|
||||
"style_id": row.get("style_id"),
|
||||
"style_match_method": row.get("style_match_method", ""),
|
||||
# 采集开关随记录下发;缺字段(旧 cmt_styles 回退)按启用处理
|
||||
"collect_enabled": True
|
||||
if row.get("collect_enabled") is None
|
||||
@@ -347,6 +351,8 @@ class StyleConfigLoader:
|
||||
def _empty(style_name: str) -> dict:
|
||||
return {
|
||||
"name": style_name,
|
||||
"style_id": None,
|
||||
"style_match_method": "",
|
||||
"brand": "",
|
||||
"erp_codes": [],
|
||||
"tm": {"item_ids": []},
|
||||
@@ -379,6 +385,13 @@ class StyleConfigLoader:
|
||||
|
||||
style = styles.setdefault(style_name, _empty(style_name))
|
||||
|
||||
# 款式主键: dim_style.id, 与 data-hub 的 22 张表同一套主键。
|
||||
# merge 已把同款展开成多条平台记录, 取值一致, 取第一个非空即可。
|
||||
if style.get("style_id") is None and record.get("style_id") is not None:
|
||||
style["style_id"] = int(record["style_id"])
|
||||
if not style.get("style_match_method") and record.get("style_match_method"):
|
||||
style["style_match_method"] = str(record["style_match_method"])
|
||||
|
||||
# 品牌(款式级别,取第一个非空)
|
||||
brand = str(record.get("brand") or "").strip()
|
||||
if brand and not style["brand"]:
|
||||
|
||||
@@ -38,10 +38,12 @@
|
||||
写: import_product_daily.py (扫 data/<plat>单品数据/_downloads/* 入库)
|
||||
读: 三平台单品日维度查询 / 跨平台对账
|
||||
product_daily_metrics.raw_data.tm_ad_metrics
|
||||
写: import_tmall_product_ads.py (万相台商品投放 CSV)
|
||||
读: data-hub 产品生命进程广告成交金额 / 投产比 / 电商费比;节点内保存款式匹配结果
|
||||
已废弃(2026-11-01):天猫广告指标改为写 tm_product_ad_daily_metrics;
|
||||
历史节点由 scripts/migrate_tm_ad_metrics_to_table.py 迁移后清除
|
||||
tm_product_ad_daily_metrics
|
||||
兼容旧版独立广告表;新流程不再写入该表
|
||||
写: import_tmall_product_ads.py (万相台商品投放 CSV)
|
||||
读: data-hub 产品生命进程广告花费 / 广告归因成交 / 投产比 / 电商费比
|
||||
注: 标量列供直接聚合,metrics 保留报表全部指标;款式按 tm_spus 反查
|
||||
jd_product_ad_daily_metrics
|
||||
写: import_jd_ad_costs.py (京东营销概况全站 SPU 与非全站 SKU)
|
||||
读: 京东电商费用 / 款式广告成本分析
|
||||
@@ -49,8 +51,9 @@
|
||||
写: import_douyin_qianchuan_ads.py (巨量千川商品推广日报)
|
||||
读: 抖音电商费用 / 款式广告成本分析;款式按 ID 优先、名称兜底反查
|
||||
product_reviews
|
||||
写: import_product_reviews.py (扫 data/{jd,dy}单品数据/评价*.json 入库)
|
||||
读: 评价分析 / 差评归因 / 标签挖掘
|
||||
已下线 (2026-11-01): 差评统一由 shop_intelligence 三张差评表采集,
|
||||
消费方改读视图 v_negative_reviews; 存量表归档为
|
||||
product_reviews_archived_20261101, 不再写入。
|
||||
persona_screenshots
|
||||
写: taobao_dmp_item_crowd_insight_screenshots.py (v2 改造后只入库, 不推飞书; insert_persona_to_bitable_v2.py 已删)
|
||||
读: 人物画像审计 / 飞书附件溯源
|
||||
@@ -84,6 +87,7 @@ import os
|
||||
import time
|
||||
import unicodedata
|
||||
from contextlib import contextmanager
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Mapping, Sequence
|
||||
|
||||
@@ -216,21 +220,36 @@ def ensure_tmall_product_ad_daily_table(conn) -> None:
|
||||
stat_date DATE NOT NULL,
|
||||
product_id TEXT NOT NULL,
|
||||
product_name TEXT,
|
||||
style_name TEXT,
|
||||
style_match_method TEXT NOT NULL DEFAULT 'unmatched',
|
||||
spend NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
direct_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
indirect_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
attributed_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
reported_roas NUMERIC(18, 6),
|
||||
calculated_roas NUMERIC(18, 6),
|
||||
metrics JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
duplicate_row_count INTEGER NOT NULL DEFAULT 1,
|
||||
source_file TEXT,
|
||||
source_row_number INTEGER,
|
||||
imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (stat_date, product_id)
|
||||
);
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS style_name TEXT;
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS style_match_method TEXT
|
||||
NOT NULL DEFAULT 'unmatched';
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS metrics JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS source_row_number INTEGER;
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_date_idx
|
||||
ON tm_product_ad_daily_metrics (stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_product_date_idx
|
||||
ON tm_product_ad_daily_metrics (product_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_style_date_idx
|
||||
ON tm_product_ad_daily_metrics (style_name, stat_date DESC);
|
||||
COMMENT ON TABLE tm_product_ad_daily_metrics IS
|
||||
'天猫商品级投放日报,每日每商品一行;广告归因成交额仅用于计算投产比。';
|
||||
"""
|
||||
@@ -332,6 +351,103 @@ def ensure_douyin_product_ad_daily_table(conn) -> None:
|
||||
cur.execute(sql)
|
||||
|
||||
|
||||
# ── 款式主键 (style_id = dim_style.id) 解析 ──────────────────────────────────
|
||||
#
|
||||
# 采集端 2026-11-01 起统一用 style_id 关联款式, style_name 降级为显示冗余。
|
||||
# 好处: 与 data-hub 的 22 张表同一套主键; 改款名不再让历史行变孤儿。
|
||||
#
|
||||
# 解析优先复用调用方传入的 style_id (loader 已下发), 缺失时在 db 层批量兜底,
|
||||
# 避免为 13 张表的每个写入脚本单独加查询。
|
||||
|
||||
STYLE_MATCH_BY_NAME = "dim_style"
|
||||
STYLE_MATCH_BY_PRODUCT = "product_id"
|
||||
STYLE_MATCH_BY_ANCHOR = "anchor"
|
||||
STYLE_MATCH_UNMATCHED = "unmatched"
|
||||
|
||||
_PLATFORM_SPU_COLUMN = {"tm": "tm_spus", "jd": "jd_spus", "dy": "dy_spus"}
|
||||
|
||||
|
||||
def resolve_style_ids(conn, style_names: Sequence[Any]) -> dict[str, int]:
|
||||
"""按款名批量解析 dim_style.id。查不到的款名不出现在返回值里。"""
|
||||
|
||||
names = sorted(
|
||||
{" ".join(str(name or "").split()) for name in style_names if str(name or "").strip()}
|
||||
)
|
||||
if not names:
|
||||
return {}
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT style_name, id FROM dim_style WHERE style_name = ANY(%s)",
|
||||
(names,),
|
||||
)
|
||||
return {str(name): int(style_id) for name, style_id in cur.fetchall()}
|
||||
|
||||
|
||||
def resolve_style_ids_by_product(
|
||||
conn, platform: str, product_ids: Sequence[Any]
|
||||
) -> dict[str, int]:
|
||||
"""按平台商品 ID 反查 dim_style.id。
|
||||
|
||||
优先级: 系列锚点 > 款名最短 > id 最小。锚点优先是因为平台编码在 loader 里
|
||||
已经整系列聚合到锚点, 同一商品 ID 归属锚点才与采集口径一致; 款名最短是为了
|
||||
在「极星」与「极星双肩」同时命中时选更具体的那个。
|
||||
"""
|
||||
|
||||
column = _PLATFORM_SPU_COLUMN.get(str(platform or "").strip())
|
||||
ids = sorted({str(item).strip() for item in product_ids if str(item or "").strip()})
|
||||
if not column or not ids:
|
||||
return {}
|
||||
sql = f"""
|
||||
SELECT p.product_id, pick.id
|
||||
FROM unnest(%s::text[]) AS p(product_id)
|
||||
JOIN LATERAL (
|
||||
SELECT d.id
|
||||
FROM dim_style d
|
||||
WHERE d.{column} @> to_jsonb(p.product_id)
|
||||
ORDER BY COALESCE(d.is_series_anchor, FALSE) DESC,
|
||||
LENGTH(d.style_name),
|
||||
d.id
|
||||
LIMIT 1
|
||||
) pick ON TRUE
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (ids,))
|
||||
return {str(pid): int(style_id) for pid, style_id in cur.fetchall()}
|
||||
|
||||
|
||||
def attach_style_ids(
|
||||
conn,
|
||||
records: Sequence[dict],
|
||||
*,
|
||||
name_key: str = "style_name",
|
||||
) -> list[dict]:
|
||||
"""就地补齐 records 的 style_id / style_match_method, 返回同一批 dict 副本。
|
||||
|
||||
调用方已带 style_id 的(loader 下发)不覆盖, 只补缺失的。
|
||||
"""
|
||||
|
||||
copied = [dict(record) for record in records]
|
||||
pending = [
|
||||
str(record.get(name_key) or "").strip()
|
||||
for record in copied
|
||||
if record.get("style_id") is None
|
||||
]
|
||||
mapping = resolve_style_ids(conn, pending) if pending else {}
|
||||
for record in copied:
|
||||
if record.get("style_id") is not None:
|
||||
if not record.get("style_match_method"):
|
||||
record["style_match_method"] = STYLE_MATCH_BY_NAME
|
||||
continue
|
||||
name = str(record.get(name_key) or "").strip()
|
||||
style_id = mapping.get(name)
|
||||
record["style_id"] = style_id
|
||||
if not record.get("style_match_method"):
|
||||
record["style_match_method"] = (
|
||||
STYLE_MATCH_BY_NAME if style_id is not None else STYLE_MATCH_UNMATCHED
|
||||
)
|
||||
return copied
|
||||
|
||||
|
||||
# ── daily_style_metrics ──────────────────────────────────────────────────────
|
||||
|
||||
def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
@@ -351,6 +467,7 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"metric_date": r["metric_date"],
|
||||
"style_name": r["style_name"],
|
||||
"platform": r["platform"],
|
||||
"style_id": r.get("style_id"),
|
||||
"sales": 0,
|
||||
"refund_orders": 0,
|
||||
"visitors": 0,
|
||||
@@ -359,6 +476,8 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"source": r.get("source"),
|
||||
}
|
||||
cur = dedup[key]
|
||||
if cur.get("style_id") is None and r.get("style_id") is not None:
|
||||
cur["style_id"] = r["style_id"]
|
||||
for fld in ("sales", "refund_orders", "visitors", "cart_users"):
|
||||
cur[fld] = int(cur.get(fld, 0) or 0) + int(r.get(fld, 0) or 0)
|
||||
existing_codes = set(cur.get("erp_style_codes") or [])
|
||||
@@ -367,7 +486,7 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
cur["erp_style_codes"] = sorted(existing_codes)
|
||||
if r.get("source"):
|
||||
cur["source"] = r["source"]
|
||||
records = list(dedup.values())
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
|
||||
rows = [
|
||||
(
|
||||
@@ -380,12 +499,15 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
int(r.get("cart_users", 0) or 0),
|
||||
r.get("erp_style_codes") or [],
|
||||
r.get("source"),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO daily_style_metrics
|
||||
(metric_date, style_name, platform, sales, refund_orders, visitors, cart_users, erp_style_codes, source)
|
||||
(metric_date, style_name, platform, sales, refund_orders, visitors, cart_users,
|
||||
erp_style_codes, source, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (metric_date, style_name, platform) DO UPDATE SET
|
||||
sales = EXCLUDED.sales,
|
||||
@@ -394,6 +516,9 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
cart_users = EXCLUDED.cart_users,
|
||||
erp_style_codes = EXCLUDED.erp_style_codes,
|
||||
source = EXCLUDED.source,
|
||||
style_id = COALESCE(EXCLUDED.style_id, daily_style_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, daily_style_metrics.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -414,6 +539,7 @@ def upsert_erp_all_shop_style_daily_metrics(conn, records: Sequence[dict]) -> in
|
||||
dedup[key] = dict(record)
|
||||
if not dedup:
|
||||
return 0
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
|
||||
rows = [
|
||||
(
|
||||
@@ -431,13 +557,16 @@ def upsert_erp_all_shop_style_daily_metrics(conn, records: Sequence[dict]) -> in
|
||||
int(record["selected_shop_count"]),
|
||||
int(record.get("matched_product_rows", 0) or 0),
|
||||
str(record["source"]),
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
for record in dedup.values()
|
||||
for record in records
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO erp_all_shop_style_daily_metrics
|
||||
(metric_date, style_name, sales, return_qty, erp_style_codes,
|
||||
selected_shop_count, matched_product_rows, source)
|
||||
selected_shop_count, matched_product_rows, source,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (metric_date, style_name) DO UPDATE SET
|
||||
sales = EXCLUDED.sales,
|
||||
@@ -446,6 +575,9 @@ def upsert_erp_all_shop_style_daily_metrics(conn, records: Sequence[dict]) -> in
|
||||
selected_shop_count = EXCLUDED.selected_shop_count,
|
||||
matched_product_rows = EXCLUDED.matched_product_rows,
|
||||
source = EXCLUDED.source,
|
||||
style_id = COALESCE(EXCLUDED.style_id, erp_all_shop_style_daily_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, erp_all_shop_style_daily_metrics.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -491,8 +623,11 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"present_days": 0,
|
||||
"erp_style_codes": [],
|
||||
"erp_source": r.get("erp_source"),
|
||||
"style_id": r.get("style_id"),
|
||||
}
|
||||
cur = dedup[key]
|
||||
if cur.get("style_id") is None and r.get("style_id") is not None:
|
||||
cur["style_id"] = r["style_id"]
|
||||
# 数字字段累加
|
||||
for fld in ("sales", "pop_sales", "jd_self_sales", "refund_orders",
|
||||
"visitors", "cart_users", "present_days"):
|
||||
@@ -505,7 +640,7 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
# 保留最后一行的 erp_source 标记
|
||||
if r.get("erp_source"):
|
||||
cur["erp_source"] = r["erp_source"]
|
||||
records = list(dedup.values())
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
# ── dedup 完毕 ──────────────────────────────────────────────────────
|
||||
|
||||
rows = [
|
||||
@@ -527,6 +662,8 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
int(r.get("present_days", 0) or 0),
|
||||
r.get("erp_style_codes") or [],
|
||||
r.get("erp_source"),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
@@ -534,7 +671,8 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
INSERT INTO weekly_style_metrics
|
||||
(week_end, week_start, time_period, style_name, platform, sales, pop_sales,
|
||||
jd_self_sales, refund_orders, visitors, cart_users, conversion_rate,
|
||||
cart_rate, refund_rate, present_days, erp_style_codes, erp_source)
|
||||
cart_rate, refund_rate, present_days, erp_style_codes, erp_source,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (week_end, style_name, platform) DO UPDATE SET
|
||||
week_start = EXCLUDED.week_start,
|
||||
@@ -551,6 +689,9 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
present_days = EXCLUDED.present_days,
|
||||
erp_style_codes = EXCLUDED.erp_style_codes,
|
||||
erp_source = EXCLUDED.erp_source,
|
||||
style_id = COALESCE(EXCLUDED.style_id, weekly_style_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, weekly_style_metrics.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -583,6 +724,7 @@ def upsert_jd_self_sales(conn, rows: Sequence[dict]) -> int:
|
||||
"""rows: [{'week_end', 'week_start', 'style_name', 'sku', 'outbound_qty', 'found', 'collected_at'}, ...]"""
|
||||
if not rows:
|
||||
return 0
|
||||
rows = attach_style_ids(conn, rows)
|
||||
payload = [
|
||||
(
|
||||
r["week_end"],
|
||||
@@ -592,17 +734,23 @@ def upsert_jd_self_sales(conn, rows: Sequence[dict]) -> int:
|
||||
int(r.get("outbound_qty", 0) or 0),
|
||||
bool(r.get("found", False)),
|
||||
r.get("collected_at"),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO jd_self_weekly_sales
|
||||
(week_end, week_start, style_name, sku, outbound_qty, found, collected_at)
|
||||
(week_end, week_start, style_name, sku, outbound_qty, found, collected_at,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (week_end, style_name, sku) DO UPDATE SET
|
||||
outbound_qty = EXCLUDED.outbound_qty,
|
||||
found = EXCLUDED.found,
|
||||
collected_at = EXCLUDED.collected_at,
|
||||
style_id = COALESCE(EXCLUDED.style_id, jd_self_weekly_sales.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, jd_self_weekly_sales.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -696,11 +844,13 @@ def fetch_daily_for_range(conn, start_date: str, end_date: str) -> dict[tuple[st
|
||||
# ── decline_events ──────────────────────────────────────────────────────────
|
||||
|
||||
def upsert_decline_event(conn, event: dict, hermes_response: Any = None) -> None:
|
||||
event = attach_style_ids(conn, [event])[0]
|
||||
sql = """
|
||||
INSERT INTO decline_events
|
||||
(end_date, window_days, style_name, platform, window_start,
|
||||
seg1_sales, seg2_sales, seg3_sales, drop_pct_1to2, drop_pct_2to3,
|
||||
drop_pct_1to3, erp_style_codes, hermes_notified, hermes_response)
|
||||
drop_pct_1to3, erp_style_codes, hermes_notified, hermes_response,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (end_date, style_name, platform) DO UPDATE SET
|
||||
window_days = EXCLUDED.window_days,
|
||||
@@ -712,6 +862,9 @@ def upsert_decline_event(conn, event: dict, hermes_response: Any = None) -> None
|
||||
drop_pct_2to3 = EXCLUDED.drop_pct_2to3,
|
||||
drop_pct_1to3 = EXCLUDED.drop_pct_1to3,
|
||||
erp_style_codes = EXCLUDED.erp_style_codes,
|
||||
style_id = COALESCE(EXCLUDED.style_id, decline_events.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, decline_events.style_match_method),
|
||||
hermes_notified = decline_events.hermes_notified
|
||||
OR EXCLUDED.hermes_notified,
|
||||
hermes_response = COALESCE(
|
||||
@@ -735,6 +888,8 @@ def upsert_decline_event(conn, event: dict, hermes_response: Any = None) -> None
|
||||
event.get("erp_style_codes") or [],
|
||||
bool(hermes_response is not None and not (isinstance(hermes_response, dict) and "error" in hermes_response)),
|
||||
json.dumps(hermes_response, ensure_ascii=False) if hermes_response is not None else None,
|
||||
event.get("style_id"),
|
||||
event.get("style_match_method"),
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, [payload])
|
||||
@@ -863,11 +1018,13 @@ def insert_fact_alert_history(conn, record: dict) -> int:
|
||||
sent_at, recipient, send_channel, hermes_message_id, hermes_response, decline_payload}
|
||||
唯一键: (style_name, trigger_date, recipient) — 同款同日每位收件人只发一次
|
||||
"""
|
||||
record = attach_style_ids(conn, [record])[0]
|
||||
sql = """
|
||||
INSERT INTO fact_alert_history
|
||||
(decline_event_key, style_name, trigger_date, consecutive_days, sales_drop_pct,
|
||||
decline_window_start, decline_window_end, baseline_window_start, baseline_window_end,
|
||||
sent_at, recipient, send_channel, hermes_message_id, hermes_response, decline_payload)
|
||||
sent_at, recipient, send_channel, hermes_message_id, hermes_response, decline_payload,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (style_name, trigger_date, recipient) DO UPDATE SET
|
||||
decline_event_key = EXCLUDED.decline_event_key,
|
||||
@@ -881,7 +1038,10 @@ def insert_fact_alert_history(conn, record: dict) -> int:
|
||||
send_channel = EXCLUDED.send_channel,
|
||||
hermes_message_id = EXCLUDED.hermes_message_id,
|
||||
hermes_response = EXCLUDED.hermes_response,
|
||||
decline_payload = EXCLUDED.decline_payload
|
||||
decline_payload = EXCLUDED.decline_payload,
|
||||
style_id = COALESCE(EXCLUDED.style_id, fact_alert_history.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, fact_alert_history.style_match_method)
|
||||
RETURNING id
|
||||
"""
|
||||
payload = (
|
||||
@@ -900,6 +1060,8 @@ def insert_fact_alert_history(conn, record: dict) -> int:
|
||||
record.get("hermes_message_id"),
|
||||
json.dumps(record["hermes_response"], ensure_ascii=False) if record.get("hermes_response") is not None else None,
|
||||
json.dumps(record["decline_payload"], ensure_ascii=False, default=str) if record.get("decline_payload") is not None else None,
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, [payload])
|
||||
@@ -924,8 +1086,31 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
if not records:
|
||||
return 0
|
||||
|
||||
# 款式归因: 本表只有平台商品 ID, 没有款名列, 按 spus 数组反查 dim_style.id。
|
||||
# 未建档 (product_id 没写进 dim_style) 的行 style_id 留 NULL 并标 unmatched,
|
||||
# 规模用 v_style_unmatched_product 监控。
|
||||
pending: dict[str, list[str]] = {}
|
||||
for r in records:
|
||||
if r.get("style_id") is None:
|
||||
pending.setdefault(str(r.get("platform") or ""), []).append(r.get("product_id"))
|
||||
resolved: dict[tuple[str, str], int] = {}
|
||||
for platform, product_ids in pending.items():
|
||||
for product_id, style_id in resolve_style_ids_by_product(
|
||||
conn, platform, product_ids
|
||||
).items():
|
||||
resolved[(platform, product_id)] = style_id
|
||||
|
||||
rows = []
|
||||
for r in records:
|
||||
platform = str(r.get("platform") or "")
|
||||
product_id = str(r.get("product_id") or "")
|
||||
style_id = r.get("style_id")
|
||||
match_method = r.get("style_match_method")
|
||||
if style_id is None:
|
||||
style_id = resolved.get((platform, product_id))
|
||||
match_method = match_method or (
|
||||
STYLE_MATCH_BY_PRODUCT if style_id is not None else STYLE_MATCH_UNMATCHED
|
||||
)
|
||||
rows.append((
|
||||
r["platform"],
|
||||
r["stat_date"],
|
||||
@@ -950,6 +1135,8 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
r.get("detail_page_bounce_rate"),
|
||||
json.dumps(r.get("raw_data") or {}, ensure_ascii=False, default=str),
|
||||
r.get("source_file"),
|
||||
style_id,
|
||||
match_method or STYLE_MATCH_UNMATCHED,
|
||||
))
|
||||
|
||||
sql = """
|
||||
@@ -960,7 +1147,7 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
paid_amount, paid_qty, paid_buyers,
|
||||
refund_amount, refund_orders,
|
||||
conversion_rate, avg_order_value, uv_value, detail_page_bounce_rate,
|
||||
raw_data, source_file)
|
||||
raw_data, source_file, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (platform, stat_date, product_id) DO UPDATE SET
|
||||
shop_id = EXCLUDED.shop_id,
|
||||
@@ -985,6 +1172,14 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
raw_data = COALESCE(product_daily_metrics.raw_data, '{}'::jsonb)
|
||||
|| EXCLUDED.raw_data,
|
||||
source_file = EXCLUDED.source_file,
|
||||
-- 款式归因只在新值命中时覆盖,避免 spus 临时缺维护时把历史归属抹掉
|
||||
style_id = COALESCE(EXCLUDED.style_id, product_daily_metrics.style_id),
|
||||
style_match_method = CASE
|
||||
WHEN EXCLUDED.style_id IS NOT NULL
|
||||
THEN EXCLUDED.style_match_method
|
||||
ELSE COALESCE(product_daily_metrics.style_match_method,
|
||||
EXCLUDED.style_match_method)
|
||||
END,
|
||||
imported_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -992,74 +1187,25 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
return len(rows)
|
||||
|
||||
|
||||
def upsert_tmall_product_ad_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""兼容旧调用:写入历史独立广告表。
|
||||
|
||||
新的万相台工作流使用 :func:`upsert_tmall_product_ad_raw_metrics`,因为
|
||||
data-hub 产品生命进程读取的是 ``product_daily_metrics.raw_data``。
|
||||
"""
|
||||
if not records:
|
||||
return 0
|
||||
|
||||
rows = [
|
||||
(
|
||||
record["stat_date"],
|
||||
record["product_id"],
|
||||
record.get("product_name"),
|
||||
record.get("spend") or 0,
|
||||
record.get("direct_deal_amount") or 0,
|
||||
record.get("indirect_deal_amount") or 0,
|
||||
record.get("attributed_deal_amount") or 0,
|
||||
record.get("reported_roas"),
|
||||
record.get("calculated_roas"),
|
||||
int(record.get("duplicate_row_count") or 1),
|
||||
record.get("source_file"),
|
||||
)
|
||||
for record in records
|
||||
]
|
||||
|
||||
sql = """
|
||||
INSERT INTO tm_product_ad_daily_metrics
|
||||
(stat_date, product_id, product_name, spend,
|
||||
direct_deal_amount, indirect_deal_amount, attributed_deal_amount,
|
||||
reported_roas, calculated_roas, duplicate_row_count, source_file)
|
||||
VALUES %s
|
||||
ON CONFLICT (stat_date, product_id) DO UPDATE SET
|
||||
product_name = EXCLUDED.product_name,
|
||||
spend = EXCLUDED.spend,
|
||||
direct_deal_amount = EXCLUDED.direct_deal_amount,
|
||||
indirect_deal_amount = EXCLUDED.indirect_deal_amount,
|
||||
attributed_deal_amount = EXCLUDED.attributed_deal_amount,
|
||||
reported_roas = EXCLUDED.reported_roas,
|
||||
calculated_roas = EXCLUDED.calculated_roas,
|
||||
duplicate_row_count = EXCLUDED.duplicate_row_count,
|
||||
source_file = EXCLUDED.source_file,
|
||||
imported_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, rows)
|
||||
return len(rows)
|
||||
def _tm_ad_number(value: Any) -> Decimal | None:
|
||||
"""把报表里的金额/比率单元格转成 Decimal;缺失值返回 None。"""
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip().replace(",", "")
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return Decimal(text)
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def upsert_tmall_product_ad_raw_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""只更新 ``product_daily_metrics.raw_data.tm_ad_metrics``。
|
||||
def _tm_ad_metrics_payload(record: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""取报表指标字典,兼容调用方传 ``ad_metrics`` 或扁平标量键两种写法。"""
|
||||
|
||||
``product_daily_metrics`` 可能已经由三平台经营日报导入过;这里不能用
|
||||
``upsert_product_daily_metrics``,否则会把 visitors/paid_amount 等经营
|
||||
字段清成 NULL。JSONB 节点级更新同时满足重复导入幂等和字段保留;广告
|
||||
节点内的款式按 ``dim_style.tm_spus`` 精确匹配,未命中再按商品名称兜底。
|
||||
"""
|
||||
|
||||
if not records:
|
||||
return 0
|
||||
|
||||
style_by_product_id, style_names = _load_platform_style_catalog(conn, "tm_spus")
|
||||
rows = []
|
||||
for record in records:
|
||||
metrics = dict(record.get("ad_metrics") or {})
|
||||
if not metrics:
|
||||
# Keep callers written against the first draft of this importer
|
||||
# compatible while making the canonical storage shape explicit.
|
||||
if metrics:
|
||||
return metrics
|
||||
for key in (
|
||||
"spend",
|
||||
"attributed_deal_amount",
|
||||
@@ -1070,46 +1216,127 @@ def upsert_tmall_product_ad_raw_metrics(conn, records: Sequence[dict]) -> int:
|
||||
):
|
||||
if record.get(key) is not None:
|
||||
metrics[key] = record[key]
|
||||
return metrics
|
||||
|
||||
|
||||
def upsert_tmall_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""把万相台商品广告指标幂等写入 ``tm_product_ad_daily_metrics``。
|
||||
|
||||
天猫广告数据不再写入 ``product_daily_metrics.raw_data.tm_ad_metrics``:广告
|
||||
与经营日报分表后,经营数据重新同步不会覆盖投放数据。花费/直接/间接/归因成交
|
||||
额和投产比走标量列(下游可直接 SUM),报表其余指标(展现、点击、点击率、
|
||||
转化率、预售成交等)原样存在 ``metrics`` JSONB。款式按 ``dim_style.tm_spus``
|
||||
的商品 ID 精确匹配,未命中再按商品名称兜底,结果写进 ``style_name`` /
|
||||
``style_match_method``。
|
||||
"""
|
||||
|
||||
if not records:
|
||||
return 0
|
||||
|
||||
# 同一 (stat_date, product_id) 在一次批量里出现两次会让 ON CONFLICT 报
|
||||
# "cannot affect row a second time",先按输入顺序去重(后者覆盖前者)。
|
||||
dedup: dict[tuple[Any, str], dict] = {}
|
||||
for record in records:
|
||||
dedup[(record["stat_date"], str(record["product_id"]))] = dict(record)
|
||||
|
||||
style_by_product_id, style_names = _load_platform_style_catalog(conn, "tm_spus")
|
||||
# 先解析款式名与匹配方式,再统一补 style_id(款式主键统一 2026-11-01)。
|
||||
for record in dedup.values():
|
||||
style_name, match_method = _resolve_style_match(
|
||||
record, style_by_product_id, style_names
|
||||
)
|
||||
metrics["style_name"] = style_name
|
||||
metrics["style_match_method"] = match_method
|
||||
record["style_name"] = style_name
|
||||
record["style_match_method"] = match_method
|
||||
enriched = iter(attach_style_ids(conn, list(dedup.values())))
|
||||
|
||||
rows = []
|
||||
for (stat_date, product_id), record in dedup.items():
|
||||
style_id_record = next(enriched)
|
||||
metrics = _tm_ad_metrics_payload(record)
|
||||
metrics.pop("style_name", None)
|
||||
metrics.pop("style_match_method", None)
|
||||
style_name = record["style_name"]
|
||||
match_method = record["style_match_method"]
|
||||
spend = _tm_ad_number(metrics.get("spend")) or Decimal("0")
|
||||
attributed = _tm_ad_number(metrics.get("attributed_deal_amount")) or Decimal("0")
|
||||
calculated_roas = (
|
||||
(attributed / spend).quantize(Decimal("0.000001")) if spend else None
|
||||
)
|
||||
rows.append(
|
||||
(
|
||||
record["stat_date"],
|
||||
str(record["product_id"]),
|
||||
stat_date,
|
||||
product_id,
|
||||
record.get("product_name"),
|
||||
json.dumps({"tm_ad_metrics": metrics}, ensure_ascii=False, default=str),
|
||||
style_name,
|
||||
match_method,
|
||||
spend,
|
||||
_tm_ad_number(metrics.get("direct_deal_amount")) or Decimal("0"),
|
||||
_tm_ad_number(metrics.get("indirect_deal_amount")) or Decimal("0"),
|
||||
attributed,
|
||||
_tm_ad_number(metrics.get("roi"))
|
||||
or _tm_ad_number(record.get("reported_roas")),
|
||||
calculated_roas,
|
||||
json.dumps(metrics, ensure_ascii=False, default=str),
|
||||
int(record.get("duplicate_row_count") or 1),
|
||||
record.get("source_file"),
|
||||
record.get("source_row_number"),
|
||||
style_id_record.get("style_id"),
|
||||
)
|
||||
)
|
||||
|
||||
sql = """
|
||||
INSERT INTO product_daily_metrics
|
||||
(platform, stat_date, product_id, product_name, raw_data, source_file, imported_at)
|
||||
INSERT INTO tm_product_ad_daily_metrics
|
||||
(stat_date, product_id, product_name, style_name, style_match_method,
|
||||
spend, direct_deal_amount, indirect_deal_amount, attributed_deal_amount,
|
||||
reported_roas, calculated_roas, metrics, duplicate_row_count,
|
||||
source_file, source_row_number, style_id)
|
||||
VALUES %s
|
||||
ON CONFLICT (platform, stat_date, product_id) DO UPDATE SET
|
||||
product_name = CASE
|
||||
WHEN COALESCE(BTRIM(product_daily_metrics.product_name), '') = ''
|
||||
THEN EXCLUDED.product_name
|
||||
ELSE product_daily_metrics.product_name
|
||||
END,
|
||||
raw_data = jsonb_set(
|
||||
COALESCE(product_daily_metrics.raw_data, '{}'::jsonb),
|
||||
'{tm_ad_metrics}',
|
||||
EXCLUDED.raw_data -> 'tm_ad_metrics',
|
||||
true
|
||||
ON CONFLICT (stat_date, product_id) DO UPDATE SET
|
||||
product_name = COALESCE(
|
||||
NULLIF(BTRIM(EXCLUDED.product_name), ''),
|
||||
tm_product_ad_daily_metrics.product_name
|
||||
),
|
||||
-- 款式只有本次能匹配到才覆盖,避免后续导入用 NULL 清掉已维护的映射。
|
||||
style_name = COALESCE(
|
||||
EXCLUDED.style_name, tm_product_ad_daily_metrics.style_name
|
||||
),
|
||||
style_match_method = CASE
|
||||
WHEN EXCLUDED.style_name IS NULL
|
||||
THEN tm_product_ad_daily_metrics.style_match_method
|
||||
ELSE EXCLUDED.style_match_method
|
||||
END,
|
||||
style_id = COALESCE(EXCLUDED.style_id,
|
||||
tm_product_ad_daily_metrics.style_id),
|
||||
spend = EXCLUDED.spend,
|
||||
direct_deal_amount = EXCLUDED.direct_deal_amount,
|
||||
indirect_deal_amount = EXCLUDED.indirect_deal_amount,
|
||||
attributed_deal_amount = EXCLUDED.attributed_deal_amount,
|
||||
reported_roas = EXCLUDED.reported_roas,
|
||||
calculated_roas = EXCLUDED.calculated_roas,
|
||||
metrics = EXCLUDED.metrics,
|
||||
duplicate_row_count = EXCLUDED.duplicate_row_count,
|
||||
source_file = EXCLUDED.source_file,
|
||||
source_row_number = EXCLUDED.source_row_number,
|
||||
imported_at = NOW()
|
||||
"""
|
||||
template = "('tm', %s, %s, %s, %s::jsonb, %s, NOW())"
|
||||
template = "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s, %s)"
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, rows, template=template)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def upsert_tmall_product_ad_raw_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""兼容历史调用名;实际写入 ``tm_product_ad_daily_metrics``。"""
|
||||
|
||||
return upsert_tmall_product_ad_daily_metrics(conn, records)
|
||||
|
||||
|
||||
def upsert_tmall_product_ad_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""兼容历史调用名;实际写入 ``tm_product_ad_daily_metrics``。"""
|
||||
|
||||
return upsert_tmall_product_ad_daily_metrics(conn, records)
|
||||
|
||||
|
||||
def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""幂等写入京东全站 SPU 与非全站 SKU 广告指标。"""
|
||||
|
||||
@@ -1129,6 +1356,7 @@ def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
)
|
||||
dedup[key] = dict(record)
|
||||
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
rows = [
|
||||
(
|
||||
record["stat_date"],
|
||||
@@ -1147,14 +1375,17 @@ def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
json.dumps(record.get("metrics") or {}, ensure_ascii=False, default=str),
|
||||
record.get("source_file"),
|
||||
record.get("source_row_number"),
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
for record in dedup.values()
|
||||
for record in records
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO jd_product_ad_daily_metrics
|
||||
(stat_date, advertiser_pin, shop_account, report_type, row_key,
|
||||
spu_id, spu_name, sku_id, sku_name, style_name, match_status,
|
||||
spend, reported_roi, metrics, source_file, source_row_number)
|
||||
spend, reported_roi, metrics, source_file, source_row_number,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (stat_date, advertiser_pin, report_type, row_key) DO UPDATE SET
|
||||
shop_account = EXCLUDED.shop_account,
|
||||
@@ -1169,6 +1400,9 @@ def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
metrics = EXCLUDED.metrics,
|
||||
source_file = EXCLUDED.source_file,
|
||||
source_row_number = EXCLUDED.source_row_number,
|
||||
style_id = COALESCE(EXCLUDED.style_id, jd_product_ad_daily_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, jd_product_ad_daily_metrics.style_match_method),
|
||||
imported_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -1289,6 +1523,7 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int
|
||||
)
|
||||
record["style_name"] = style_name
|
||||
record["style_match_method"] = match_method
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
fields = (
|
||||
"stat_date",
|
||||
"advertiser_id",
|
||||
@@ -1313,8 +1548,9 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int
|
||||
"refund_rate_1h",
|
||||
"source_file",
|
||||
"source_row_number",
|
||||
"style_id",
|
||||
)
|
||||
rows = [tuple(record.get(field) for field in fields) for record in dedup.values()]
|
||||
rows = [tuple(record.get(field) for field in fields) for record in records]
|
||||
sql = """
|
||||
INSERT INTO dy_product_ad_daily_metrics
|
||||
(stat_date, advertiser_id, product_id, product_name,
|
||||
@@ -1322,7 +1558,8 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int
|
||||
overall_conversion_rate, spend, gross_deal_amount,
|
||||
gross_payment_roi, gross_order_cost, user_actual_paid_amount,
|
||||
platform_subsidy_amount, net_roi, net_deal_amount, net_order_cost,
|
||||
net_settlement_rate, refund_rate_1h, source_file, source_row_number)
|
||||
net_settlement_rate, refund_rate_1h, source_file, source_row_number,
|
||||
style_id)
|
||||
VALUES %s
|
||||
ON CONFLICT (stat_date, advertiser_id, product_id) DO UPDATE SET
|
||||
product_name = EXCLUDED.product_name,
|
||||
@@ -1345,6 +1582,7 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int
|
||||
refund_rate_1h = EXCLUDED.refund_rate_1h,
|
||||
source_file = EXCLUDED.source_file,
|
||||
source_row_number = EXCLUDED.source_row_number,
|
||||
style_id = COALESCE(EXCLUDED.style_id, dy_product_ad_daily_metrics.style_id),
|
||||
imported_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -1364,11 +1602,12 @@ def upsert_persona_screenshot(conn, record: dict) -> int:
|
||||
}
|
||||
主键 = (collect_date, style_name, platform)。同款同采集日期幂等覆盖。
|
||||
"""
|
||||
record = attach_style_ids(conn, [record])[0]
|
||||
sql = """
|
||||
INSERT INTO persona_screenshots
|
||||
(collect_date, style_name, platform, item_id, screenshot_path, image_data, file_size,
|
||||
bitable_base, bitable_table, bitable_record_id, bitable_file_token,
|
||||
uploaded_to_bitable)
|
||||
uploaded_to_bitable, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (collect_date, style_name, platform) DO UPDATE SET
|
||||
item_id = EXCLUDED.item_id,
|
||||
@@ -1380,6 +1619,9 @@ def upsert_persona_screenshot(conn, record: dict) -> int:
|
||||
bitable_record_id = EXCLUDED.bitable_record_id,
|
||||
bitable_file_token = EXCLUDED.bitable_file_token,
|
||||
uploaded_to_bitable = EXCLUDED.uploaded_to_bitable,
|
||||
style_id = COALESCE(EXCLUDED.style_id, persona_screenshots.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, persona_screenshots.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
payload = (
|
||||
@@ -1395,6 +1637,8 @@ def upsert_persona_screenshot(conn, record: dict) -> int:
|
||||
record.get("bitable_record_id"),
|
||||
record.get("bitable_file_token"),
|
||||
bool(record.get("uploaded_to_bitable", False)),
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, [payload])
|
||||
@@ -1417,13 +1661,14 @@ def upsert_main_image_creatives(conn, records: Sequence[dict]) -> int:
|
||||
"""
|
||||
if not records:
|
||||
return 0
|
||||
records = attach_style_ids(conn, records)
|
||||
sql = """
|
||||
INSERT INTO main_image_creatives
|
||||
(collect_date, style_name, platform, image_key, creative_ids, creative_image_url,
|
||||
impressions, clicks, cost, click_rate, row_count,
|
||||
local_image_path, image_downloaded, image_fingerprint,
|
||||
bitable_base, bitable_table, bitable_record_id, bitable_file_token,
|
||||
uploaded_to_bitable)
|
||||
uploaded_to_bitable, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (collect_date, style_name, platform, image_key) DO UPDATE SET
|
||||
creative_ids = EXCLUDED.creative_ids,
|
||||
@@ -1441,6 +1686,9 @@ def upsert_main_image_creatives(conn, records: Sequence[dict]) -> int:
|
||||
bitable_record_id = EXCLUDED.bitable_record_id,
|
||||
bitable_file_token = EXCLUDED.bitable_file_token,
|
||||
uploaded_to_bitable = EXCLUDED.uploaded_to_bitable,
|
||||
style_id = COALESCE(EXCLUDED.style_id, main_image_creatives.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, main_image_creatives.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
rows = [
|
||||
@@ -1464,6 +1712,8 @@ def upsert_main_image_creatives(conn, records: Sequence[dict]) -> int:
|
||||
r.get("bitable_record_id"),
|
||||
r.get("bitable_file_token"),
|
||||
bool(r.get("uploaded_to_bitable", False)),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
@@ -1575,10 +1825,12 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""
|
||||
if not records:
|
||||
return 0
|
||||
records = attach_style_ids(conn, records)
|
||||
sql = """
|
||||
INSERT INTO daily_persona_metrics
|
||||
(collect_date, style_name, platform, item_id,
|
||||
payload, raw_payload, source_file, collected_at)
|
||||
payload, raw_payload, source_file, collected_at,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (collect_date, style_name, platform) DO UPDATE SET
|
||||
item_id = EXCLUDED.item_id,
|
||||
@@ -1586,6 +1838,9 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int:
|
||||
raw_payload = EXCLUDED.raw_payload,
|
||||
source_file = EXCLUDED.source_file,
|
||||
collected_at = EXCLUDED.collected_at,
|
||||
style_id = COALESCE(EXCLUDED.style_id, daily_persona_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, daily_persona_metrics.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
rows = [
|
||||
@@ -1598,6 +1853,8 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int:
|
||||
json.dumps(r.get("raw_payload"), ensure_ascii=False) if r.get("raw_payload") is not None else None,
|
||||
r.get("source_file"),
|
||||
r.get("collected_at"),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
@@ -1606,72 +1863,6 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int:
|
||||
return len(rows)
|
||||
|
||||
|
||||
# ── product_reviews ───────────────────────────────────────────────────────────
|
||||
|
||||
def upsert_product_reviews(conn, records: Sequence[dict]) -> int:
|
||||
"""records: [{
|
||||
'platform' ('jd'/'dy'), 'collect_date', 'comment_date',
|
||||
'style_name' (可空, DY 通过 product_id 反查 dim_style.dy_spus 填充),
|
||||
'rating' ('好评'/'中评'/'差评'), 'score', 'content', 'tags' (list),
|
||||
'user_name', 'order_id', 'product_id', 'product_name',
|
||||
'sku_id', 'sku_name', 'sku_size',
|
||||
'raw_data' (dict), 'source_file'
|
||||
}, ...]
|
||||
UNIQUE (platform, order_id) -> 同订单重抓覆盖。
|
||||
"""
|
||||
if not records:
|
||||
return 0
|
||||
rows = [
|
||||
(
|
||||
r["platform"],
|
||||
r["collect_date"],
|
||||
r.get("comment_date"),
|
||||
r.get("style_name"),
|
||||
r["rating"],
|
||||
r.get("score"),
|
||||
r.get("content"),
|
||||
r.get("tags") or None,
|
||||
r.get("user_name"),
|
||||
r["order_id"],
|
||||
r.get("product_id"),
|
||||
r.get("product_name"),
|
||||
r.get("sku_id"),
|
||||
r.get("sku_name"),
|
||||
r.get("sku_size"),
|
||||
json.dumps(r.get("raw_data") or {}, ensure_ascii=False, default=str),
|
||||
r.get("source_file"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO product_reviews
|
||||
(platform, collect_date, comment_date, style_name, rating, score,
|
||||
content, tags, user_name, order_id, product_id, product_name,
|
||||
sku_id, sku_name, sku_size, raw_data, source_file)
|
||||
VALUES %s
|
||||
ON CONFLICT (platform, order_id) DO UPDATE SET
|
||||
collect_date = EXCLUDED.collect_date,
|
||||
comment_date = EXCLUDED.comment_date,
|
||||
style_name = EXCLUDED.style_name,
|
||||
rating = EXCLUDED.rating,
|
||||
score = EXCLUDED.score,
|
||||
content = EXCLUDED.content,
|
||||
tags = EXCLUDED.tags,
|
||||
user_name = EXCLUDED.user_name,
|
||||
product_id = EXCLUDED.product_id,
|
||||
product_name = EXCLUDED.product_name,
|
||||
sku_id = EXCLUDED.sku_id,
|
||||
sku_name = EXCLUDED.sku_name,
|
||||
sku_size = EXCLUDED.sku_size,
|
||||
raw_data = EXCLUDED.raw_data,
|
||||
source_file = EXCLUDED.source_file,
|
||||
imported_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, rows)
|
||||
return len(rows)
|
||||
|
||||
|
||||
# ── fact_style_analysis_reports ──────────────────────────────────────────────
|
||||
|
||||
def upsert_style_analysis_report(conn, record: dict) -> int:
|
||||
@@ -1687,12 +1878,13 @@ def upsert_style_analysis_report(conn, record: dict) -> int:
|
||||
业务 UNIQUE = (style_name, analysis_date, window_days). 同款同日同窗口重跑覆盖.
|
||||
返回写入行的 id.
|
||||
"""
|
||||
record = attach_style_ids(conn, [record])[0]
|
||||
sql = """
|
||||
INSERT INTO fact_style_analysis_reports
|
||||
(style_name, analysis_date, window_days, window_start, window_end,
|
||||
platforms_included, feishu_doc_id, feishu_doc_url, report_title,
|
||||
report_markdown, hermes_model, prompt_tokens, completion_tokens,
|
||||
source_payload)
|
||||
source_payload, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (style_name, analysis_date, window_days) DO UPDATE SET
|
||||
window_start = EXCLUDED.window_start,
|
||||
@@ -1706,6 +1898,10 @@ def upsert_style_analysis_report(conn, record: dict) -> int:
|
||||
prompt_tokens = EXCLUDED.prompt_tokens,
|
||||
completion_tokens = EXCLUDED.completion_tokens,
|
||||
source_payload = EXCLUDED.source_payload,
|
||||
style_id = COALESCE(EXCLUDED.style_id,
|
||||
fact_style_analysis_reports.style_id),
|
||||
style_match_method = COALESCE(EXCLUDED.style_match_method,
|
||||
fact_style_analysis_reports.style_match_method),
|
||||
updated_at = NOW()
|
||||
RETURNING id
|
||||
"""
|
||||
@@ -1728,6 +1924,8 @@ def upsert_style_analysis_report(conn, record: dict) -> int:
|
||||
record.get("completion_tokens"),
|
||||
json.dumps(record["source_payload"], ensure_ascii=False, default=str)
|
||||
if record.get("source_payload") is not None else None,
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, [payload])
|
||||
|
||||
@@ -490,7 +490,8 @@ COMMENT ON COLUMN product_daily_metrics.detail_page_bounce_rate IS '详情页跳
|
||||
|
||||
-- ============================================================================
|
||||
-- tm_product_ad_daily_metrics: 天猫商品投放日报
|
||||
-- 数据源: 天猫投放平台导出的商品报表 CSV。
|
||||
-- 数据源: 万相台商品报表 CSV (import_tmall_product_ads.py 解析后直接写本表)。
|
||||
-- 每日每商品一行;花费/成交/投产比走标量列供下游直接聚合,报表其余指标留在 metrics。
|
||||
-- 与 product_daily_metrics 分表,避免经营数据重新同步时覆盖投放数据。
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS tm_product_ad_daily_metrics (
|
||||
@@ -498,27 +499,47 @@ CREATE TABLE IF NOT EXISTS tm_product_ad_daily_metrics (
|
||||
stat_date DATE NOT NULL,
|
||||
product_id TEXT NOT NULL,
|
||||
product_name TEXT,
|
||||
style_name TEXT,
|
||||
style_match_method TEXT NOT NULL DEFAULT 'unmatched',
|
||||
spend NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
direct_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
indirect_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
attributed_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
reported_roas NUMERIC(18, 6),
|
||||
calculated_roas NUMERIC(18, 6),
|
||||
metrics JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
duplicate_row_count INTEGER NOT NULL DEFAULT 1,
|
||||
source_file TEXT,
|
||||
source_row_number INTEGER,
|
||||
imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (stat_date, product_id)
|
||||
);
|
||||
-- 兼容已创建的窄表:补齐款式匹配结果与完整指标快照列。
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS style_name TEXT;
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS style_match_method TEXT
|
||||
NOT NULL DEFAULT 'unmatched';
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS metrics JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS source_row_number INTEGER;
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_date_idx
|
||||
ON tm_product_ad_daily_metrics (stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_product_date_idx
|
||||
ON tm_product_ad_daily_metrics (product_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_style_date_idx
|
||||
ON tm_product_ad_daily_metrics (style_name, stat_date DESC);
|
||||
|
||||
COMMENT ON TABLE tm_product_ad_daily_metrics IS '天猫商品级投放日报,每日每商品一行;广告归因成交额仅用于计算投产比。';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.product_id IS '万相台主体ID(商品),按文本保存';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.style_name IS '款式名;先按商品 ID 反查 dim_style.tm_spus,未命中再按商品名称精确包含匹配,无法确认时为 NULL';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.style_match_method IS '款式匹配方式:product_id=商品 ID,product_name=商品名称,ambiguous=名称歧义,unmatched=未匹配';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.spend IS '天猫广告花费(页面电商支付金额)';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.attributed_deal_amount IS '广告归因总成交金额,仅用于计算投产比';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.reported_roas IS '源报表投入产出比';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.calculated_roas IS '归因成交金额 / 花费';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.metrics IS '万相台报表指标原样快照 (展现/点击/点击率/转化率/预售成交等),键名与源报表中文列一一对应';
|
||||
|
||||
-- ============================================================================
|
||||
-- jd_product_ad_daily_metrics: 京东营销概况广告日报
|
||||
@@ -615,7 +636,7 @@ COMMENT ON COLUMN dy_product_ad_daily_metrics.overall_ctr IS '报表整体点击
|
||||
COMMENT ON COLUMN dy_product_ad_daily_metrics.overall_conversion_rate IS '报表整体转化率,单位为百分点';
|
||||
COMMENT ON COLUMN dy_product_ad_daily_metrics.net_settlement_rate IS '净成交金额结算率,单位为百分点';
|
||||
COMMENT ON COLUMN dy_product_ad_daily_metrics.refund_rate_1h IS '1 小时内退款率,单位为百分点';
|
||||
COMMENT ON COLUMN product_daily_metrics.raw_data IS '平台特有原始列 (下单金额/件数/买家数、聚划算、竞争力评分、月累计、搜索引导...) 原样保留;天猫 tm_ad_metrics 同时含 style_name/style_match_method';
|
||||
COMMENT ON COLUMN product_daily_metrics.raw_data IS '平台特有原始列 (下单金额/件数/买家数、聚划算、竞争力评分、月累计、搜索引导...) 原样保留;天猫广告指标已迁至 tm_product_ad_daily_metrics,本列不再写 tm_ad_metrics';
|
||||
COMMENT ON COLUMN product_daily_metrics.source_file IS '原始文件名 (溯源用)';
|
||||
COMMENT ON COLUMN product_daily_metrics.imported_at IS '入库时间';
|
||||
|
||||
@@ -810,69 +831,19 @@ COMMENT ON COLUMN main_image_creatives.collected_at IS '采集时间 (脚本写
|
||||
COMMENT ON COLUMN main_image_creatives.updated_at IS '最近更新时间';
|
||||
|
||||
-- ============================================================================
|
||||
-- product_reviews: 三平台商品评价 (JD + DY, TM 暂未接入)
|
||||
-- 数据源:
|
||||
-- JD: jd_product_data_collector.py --reviews-only -> data/jd单品数据/评价汇总_<date>.json
|
||||
-- DY: dy_product_scraping.py (主流程已挂载) -> data/dy单品数据/评论汇总_<date>.json
|
||||
-- 设计: 三平台字段归一到独立列, 平台特有字段进 raw_data JSONB。
|
||||
-- 写入策略: ON CONFLICT (platform, order_id) DO UPDATE 全量覆盖 (order_id 是评价业务键)。
|
||||
-- product_reviews: 已下线 (2026-11-01)
|
||||
--
|
||||
-- 原设计是 JD + DY 商品评价合一表, 由 import_product_reviews.py 从平台 JSON 入库。
|
||||
-- 下线原因:
|
||||
-- 1. JD 侧 style_name 全靠 sku_id 反查, 而 dim_jd_sku_spu 映射表不存在,
|
||||
-- 214 行 style_name 全为 NULL, 两个消费方 (analyze_style / 营销日报) 都是
|
||||
-- WHERE style_name = ?, 等于入库了但没人能读。
|
||||
-- 2. 差评全量已由 shop_intelligence 的三张差评表 (tmall/jd/dy_negative_reviews)
|
||||
-- 独立采集, 覆盖面更全 (天猫 97 条, 而本表天猫为 0)。
|
||||
--
|
||||
-- 现消费方统一读视图 v_negative_reviews (见文件末尾「款式主键统一」段)。
|
||||
-- 存量表已归档为 product_reviews_archived_20261101, 不再建表、不再写入。
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS product_reviews (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
platform TEXT NOT NULL CHECK (platform IN ('jd','dy')),
|
||||
collect_date DATE NOT NULL, -- 采集日期 (脚本运行日)
|
||||
comment_date DATE, -- 评价时间 (JD 取评价时间日期部分, DY 取 comment_time 日期部分)
|
||||
style_name TEXT, -- 款式名 (DY: 通过 product_id 反查 dim_style.dy_spus 填充; 未匹配时为 NULL)
|
||||
rating TEXT NOT NULL, -- '好评' / '中评' / '差评'
|
||||
score SMALLINT, -- JD 评分 (1-5 星), DY 留空
|
||||
content TEXT,
|
||||
tags TEXT[], -- JD 评价标签 (做工精细/容量大 等), DY 留空
|
||||
user_name TEXT,
|
||||
order_id TEXT NOT NULL, -- 订单 ID (业务键, 用于去重)
|
||||
product_id TEXT, -- 平台商品 ID (DY product_id / JD sku_name 对应的 SPU)
|
||||
product_name TEXT,
|
||||
sku_id TEXT, -- JD SKU 编码, DY 留空
|
||||
sku_name TEXT, -- JD SKU 名称, DY 留空
|
||||
sku_size TEXT, -- JD 规格, DY 留空
|
||||
raw_data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
source_file TEXT, -- 原始 JSON 文件路径
|
||||
imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (platform, order_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS product_reviews_platform_date_idx
|
||||
ON product_reviews (platform, comment_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS product_reviews_rating_idx
|
||||
ON product_reviews (platform, rating, comment_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS product_reviews_product_idx
|
||||
ON product_reviews (platform, product_id, comment_date DESC);
|
||||
-- product_reviews_style_idx 在文件末尾 ALTER TABLE 加 style_name 列之后建 (老表兼容)
|
||||
|
||||
COMMENT ON TABLE product_reviews IS '商品评价 - JD + DY 合一表。来源: jd_product_data_collector.py --reviews-only / dy_product_scraping.py。UNIQUE (platform, order_id) -> 同订单评价重抓覆盖。';
|
||||
COMMENT ON COLUMN product_reviews.platform IS '平台代码 (''jd'' / ''dy'')';
|
||||
COMMENT ON COLUMN product_reviews.collect_date IS '采集日期 (脚本运行日, 不等于评价发生日)';
|
||||
COMMENT ON COLUMN product_reviews.comment_date IS '评价时间 (从评价发生时间戳抽日期部分)';
|
||||
-- style_name 列的 COMMENT 在文件末尾 ALTER TABLE 加列之后 (老表兼容)
|
||||
COMMENT ON COLUMN product_reviews.rating IS '评价等级: 好评 / 中评 / 差评';
|
||||
COMMENT ON COLUMN product_reviews.score IS 'JD 评分 (1-5 星, aria-valuenow), DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.content IS '评价正文';
|
||||
COMMENT ON COLUMN product_reviews.tags IS 'JD 评价标签数组 (如 [做工精细, 容量大]), DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.user_name IS '用户昵称';
|
||||
COMMENT ON COLUMN product_reviews.order_id IS '订单 ID (业务键, 同订单只保留一条, 重抓覆盖)';
|
||||
COMMENT ON COLUMN product_reviews.product_id IS '平台商品 ID (DY product_id / JD SPU)';
|
||||
COMMENT ON COLUMN product_reviews.product_name IS '商品名称';
|
||||
COMMENT ON COLUMN product_reviews.sku_id IS 'JD SKU 编码, DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.sku_name IS 'JD SKU 名称, DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.sku_size IS 'JD 规格文本, DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.raw_data IS '平台特有原始字段 (JD 完整评价对象 / DY comment_rows 行)';
|
||||
COMMENT ON COLUMN product_reviews.source_file IS '源 JSON 文件相对路径 (溯源用)';
|
||||
COMMENT ON COLUMN product_reviews.imported_at IS '入库时间';
|
||||
|
||||
-- 兼容已有库: product_reviews 加 style_name 列 (CREATE TABLE IF NOT EXISTS 不会改已有表)
|
||||
ALTER TABLE product_reviews ADD COLUMN IF NOT EXISTS style_name TEXT;
|
||||
CREATE INDEX IF NOT EXISTS product_reviews_style_idx
|
||||
ON product_reviews (style_name, comment_date DESC)
|
||||
WHERE style_name IS NOT NULL;
|
||||
COMMENT ON COLUMN product_reviews.style_name IS '款式名 (DY: 通过 product_id 反查 dim_style.dy_spus 填充; 未匹配时为 NULL, 表示 dim_style 缺该商品 ID 维护)';
|
||||
|
||||
-- ============================================================================
|
||||
-- fact_style_analysis_reports: 单款式多维度分析报告 (直连 MiniMax 产出)
|
||||
@@ -1364,3 +1335,276 @@ CREATE TABLE IF NOT EXISTS erp_sales_theme_daily_metrics (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS erp_sales_theme_daily_metrics_date_idx
|
||||
ON erp_sales_theme_daily_metrics (stat_date DESC);
|
||||
|
||||
-- ============================================================================
|
||||
-- 款式主键统一 (2026-11-01)
|
||||
--
|
||||
-- 目标: 采集端所有带款式的表统一用 style_id (= dim_style.id) 关联, style_name
|
||||
-- 只作为显示冗余。data-hub 侧 22 张表本来就用 style_id, 两端从此同一套主键,
|
||||
-- 不再出现「同一库两种引用方式」。已实测 erp_product.style_id 135/135 命中。
|
||||
--
|
||||
-- 约定:
|
||||
-- * style_id 可空的表 = 允许「未建档」的行 (平台商品 ID 未写进 dim_style 的
|
||||
-- spus 数组), 这类行必须同时写 style_match_method='unmatched', 便于统计
|
||||
-- 漏维护的规模。
|
||||
-- * style_id 非空的表 = 款式清单来自 dim_style loader, 理论上必然命中;
|
||||
-- 但**不加 NOT NULL**, 由 backfill_style_id.py 与写入端负责填充,
|
||||
-- 避免历史脏款名在 ALTER 阶段中断建表。
|
||||
-- * 主键一律不动 (daily_style_metrics 仍是 metric_date+style_name+platform),
|
||||
-- data-hub 有 6 处按名字 join, 改 PK 收益小、回归面大。
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE daily_style_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE weekly_style_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE erp_all_shop_style_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE daily_persona_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE main_image_creatives ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE fact_style_analysis_reports ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE decline_events ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE fact_alert_history ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE jd_self_weekly_sales ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE persona_screenshots ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE product_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE jd_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE dy_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE tm_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
|
||||
-- 匹配方式留痕: dim_style / product_id / product_name / anchor / unmatched
|
||||
ALTER TABLE daily_style_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE weekly_style_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE erp_all_shop_style_daily_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE daily_persona_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE main_image_creatives ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE fact_style_analysis_reports ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE decline_events ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE fact_alert_history ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE jd_self_weekly_sales ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE persona_screenshots ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE product_daily_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE jd_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
|
||||
COMMENT ON COLUMN daily_style_metrics.style_id IS 'dim_style.id; 采集端统一款式主键, style_name 仅作显示冗余';
|
||||
COMMENT ON COLUMN product_daily_metrics.style_id IS 'dim_style.id; 按 platform+product_id 反查 spus 得到, 未建档为 NULL 且 style_match_method=unmatched';
|
||||
|
||||
-- 查询索引: 按款式跨日期取数 (产品生命页、风格分析、预警)
|
||||
CREATE INDEX IF NOT EXISTS daily_style_metrics_style_id_idx
|
||||
ON daily_style_metrics (style_id, metric_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS weekly_style_metrics_style_id_idx
|
||||
ON weekly_style_metrics (style_id, week_end DESC);
|
||||
CREATE INDEX IF NOT EXISTS erp_all_shop_style_daily_metrics_style_id_idx
|
||||
ON erp_all_shop_style_daily_metrics (style_id, metric_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS daily_persona_metrics_style_id_idx
|
||||
ON daily_persona_metrics (style_id, collect_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS main_image_creatives_style_id_idx
|
||||
ON main_image_creatives (style_id, collect_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS fact_style_analysis_reports_style_id_idx
|
||||
ON fact_style_analysis_reports (style_id, analysis_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS decline_events_style_id_idx
|
||||
ON decline_events (style_id, end_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS fact_alert_history_style_id_idx
|
||||
ON fact_alert_history (style_id, trigger_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS jd_self_weekly_sales_style_id_idx
|
||||
ON jd_self_weekly_sales (style_id, week_end DESC);
|
||||
CREATE INDEX IF NOT EXISTS persona_screenshots_style_id_idx
|
||||
ON persona_screenshots (style_id, collect_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS product_daily_metrics_style_id_idx
|
||||
ON product_daily_metrics (style_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS jd_product_ad_daily_metrics_style_id_idx
|
||||
ON jd_product_ad_daily_metrics (style_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS dy_product_ad_daily_metrics_style_id_idx
|
||||
ON dy_product_ad_daily_metrics (style_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_style_id_idx
|
||||
ON tm_product_ad_daily_metrics (style_id, stat_date DESC);
|
||||
|
||||
-- ── P0 观测视图 ─────────────────────────────────────────────────────────────
|
||||
|
||||
-- 未建档的平台商品 ID: 推动在 dim_style 里补 spus, 是 style_id 覆盖率的天花板。
|
||||
CREATE OR REPLACE VIEW v_style_unmatched_product AS
|
||||
SELECT m.platform,
|
||||
m.product_id,
|
||||
MAX(m.product_name) AS product_name,
|
||||
MIN(m.stat_date) AS first_date,
|
||||
MAX(m.stat_date) AS last_date,
|
||||
COUNT(*) AS rows
|
||||
FROM product_daily_metrics m
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM dim_style d
|
||||
WHERE (m.platform = 'tm' AND d.tm_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'jd' AND d.jd_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'dy' AND d.dy_spus @> to_jsonb(m.product_id)))
|
||||
GROUP BY m.platform, m.product_id;
|
||||
|
||||
COMMENT ON VIEW v_style_unmatched_product IS
|
||||
'未写进 dim_style.spus 的平台商品 ID 清单; 这些行的 product_daily_metrics.style_id 必然为 NULL。';
|
||||
|
||||
-- 同上, 但把三张广告表也算进来。广告投的商品不一定有经营日报 (纯投流款), 只看
|
||||
-- v_style_unmatched_product 会漏掉这部分待建档 ID。列结构保持一致, 便于并排检索。
|
||||
CREATE OR REPLACE VIEW v_style_unmatched_source AS
|
||||
WITH all_rows AS (
|
||||
SELECT platform, product_id, product_name, stat_date FROM product_daily_metrics
|
||||
UNION ALL
|
||||
SELECT 'tm', product_id, product_name, stat_date FROM tm_product_ad_daily_metrics
|
||||
UNION ALL
|
||||
SELECT 'jd', spu_id, spu_name, stat_date FROM jd_product_ad_daily_metrics
|
||||
UNION ALL
|
||||
SELECT 'dy', product_id, product_name, stat_date FROM dy_product_ad_daily_metrics
|
||||
)
|
||||
SELECT s.platform,
|
||||
s.product_id,
|
||||
MAX(s.product_name) AS product_name,
|
||||
MIN(s.stat_date) AS first_date,
|
||||
MAX(s.stat_date) AS last_date,
|
||||
COUNT(*) AS rows
|
||||
FROM all_rows s
|
||||
WHERE s.product_id IS NOT NULL
|
||||
AND BTRIM(s.product_id) <> ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM dim_style d
|
||||
WHERE (s.platform = 'tm' AND d.tm_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id)))
|
||||
GROUP BY s.platform, s.product_id;
|
||||
|
||||
COMMENT ON VIEW v_style_unmatched_source IS
|
||||
'全部采集表 (经营日报 + 三张广告表) 里未写进 dim_style.spus 的商品 ID; 补建档的完整待办清单。';
|
||||
|
||||
-- 各表 style_id 回填进度 (验收用)。
|
||||
CREATE OR REPLACE VIEW v_style_id_coverage AS
|
||||
SELECT 'daily_style_metrics' AS tbl, COUNT(*) AS rows, COUNT(style_id) AS with_style_id FROM daily_style_metrics
|
||||
UNION ALL SELECT 'weekly_style_metrics', COUNT(*), COUNT(style_id) FROM weekly_style_metrics
|
||||
UNION ALL SELECT 'erp_all_shop_style_daily_metrics', COUNT(*), COUNT(style_id) FROM erp_all_shop_style_daily_metrics
|
||||
UNION ALL SELECT 'daily_persona_metrics', COUNT(*), COUNT(style_id) FROM daily_persona_metrics
|
||||
UNION ALL SELECT 'main_image_creatives', COUNT(*), COUNT(style_id) FROM main_image_creatives
|
||||
UNION ALL SELECT 'fact_style_analysis_reports', COUNT(*), COUNT(style_id) FROM fact_style_analysis_reports
|
||||
UNION ALL SELECT 'decline_events', COUNT(*), COUNT(style_id) FROM decline_events
|
||||
UNION ALL SELECT 'fact_alert_history', COUNT(*), COUNT(style_id) FROM fact_alert_history
|
||||
UNION ALL SELECT 'jd_self_weekly_sales', COUNT(*), COUNT(style_id) FROM jd_self_weekly_sales
|
||||
UNION ALL SELECT 'persona_screenshots', COUNT(*), COUNT(style_id) FROM persona_screenshots
|
||||
UNION ALL SELECT 'product_daily_metrics', COUNT(*), COUNT(style_id) FROM product_daily_metrics
|
||||
UNION ALL SELECT 'jd_product_ad_daily_metrics', COUNT(*), COUNT(style_id) FROM jd_product_ad_daily_metrics
|
||||
UNION ALL SELECT 'dy_product_ad_daily_metrics', COUNT(*), COUNT(style_id) FROM dy_product_ad_daily_metrics
|
||||
UNION ALL SELECT 'tm_product_ad_daily_metrics', COUNT(*), COUNT(style_id) FROM tm_product_ad_daily_metrics;
|
||||
|
||||
COMMENT ON VIEW v_style_id_coverage IS
|
||||
'各采集表 style_id 非空比例; 除 product_daily_metrics / 广告表外应为 100%。';
|
||||
|
||||
-- ── P1 统一差评视图 ────────────────────────────────────────────────────────
|
||||
|
||||
-- product_reviews 下线后, 风格分析与营销日报统一读这张视图。
|
||||
-- 与旧表的口径差异: 只有差评一档 (旧表含好/中/差), 但覆盖三个平台。
|
||||
-- 款式归因优先级: 平台商品 ID 精确命中 > 商品名称包含款名 (取最长款名, 更具体)。
|
||||
-- 差评三表由 shop_intelligence 模块建表, 这里用守卫避免全新库初始化时视图先于源表。
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('public.tmall_negative_reviews') IS NOT NULL
|
||||
AND to_regclass('public.jd_negative_reviews') IS NOT NULL
|
||||
AND to_regclass('public.dy_negative_reviews') IS NOT NULL THEN
|
||||
EXECUTE $view$
|
||||
CREATE OR REPLACE VIEW v_negative_reviews AS
|
||||
WITH src AS (
|
||||
SELECT 'tm'::text AS platform,
|
||||
n.business_date AS comment_date,
|
||||
COALESCE(n.review_level, '负面评价') AS rating,
|
||||
n.content,
|
||||
NULL::text AS order_id,
|
||||
NULL::text AS product_id,
|
||||
n.product_name,
|
||||
n.review_id,
|
||||
n.review_at,
|
||||
n.raw_review
|
||||
FROM tmall_negative_reviews n
|
||||
UNION ALL
|
||||
SELECT 'jd', n.business_date, COALESCE(n.review_level, '负面评价'),
|
||||
n.content, n.order_id, NULL::text, n.product_name,
|
||||
n.review_id, n.review_at, n.raw_review
|
||||
FROM jd_negative_reviews n
|
||||
UNION ALL
|
||||
SELECT 'dy', n.business_date, COALESCE(n.rating_level, '负面评价'),
|
||||
n.content, n.order_id, n.product_id, n.product_name,
|
||||
n.review_id, n.review_at, n.raw_review
|
||||
FROM dy_negative_reviews n
|
||||
)
|
||||
SELECT s.platform, s.comment_date, s.rating, s.content,
|
||||
s.order_id, s.product_id, s.product_name,
|
||||
s.review_id, s.review_at, s.raw_review,
|
||||
m.style_id, m.style_name, m.style_match_method
|
||||
FROM src s
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT d.id AS style_id,
|
||||
d.style_name,
|
||||
CASE
|
||||
WHEN (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id))
|
||||
THEN 'product_id'
|
||||
ELSE 'product_name'
|
||||
END AS style_match_method
|
||||
FROM dim_style d
|
||||
WHERE COALESCE(d.style_name, '') <> ''
|
||||
AND (
|
||||
(s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id))
|
||||
OR (s.product_name IS NOT NULL AND s.product_name LIKE '%' || d.style_name || '%')
|
||||
)
|
||||
ORDER BY (
|
||||
CASE
|
||||
WHEN (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id))
|
||||
THEN 0 ELSE 1
|
||||
END
|
||||
),
|
||||
COALESCE(d.is_series_anchor, FALSE) DESC,
|
||||
LENGTH(d.style_name) DESC,
|
||||
d.id
|
||||
LIMIT 1
|
||||
) m ON TRUE
|
||||
$view$;
|
||||
EXECUTE $c$COMMENT ON VIEW v_negative_reviews IS
|
||||
'三平台差评归一视图 (tm/jd/dy_negative_reviews); 取代已下线的 product_reviews。style_id 为 dim_style.id, 未匹配为 NULL。'$c$;
|
||||
ELSE
|
||||
RAISE NOTICE 'skip v_negative_reviews: 差评表尚未创建 (先运行 shop_intelligence init_schema)';
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
-- ── style_id 一律走「逻辑外键」, 不建物理外键 ───────────────────────────────
|
||||
--
|
||||
-- 2026-11-01 定案: style_id 与 dim_style.id 的关联由应用层保证, 库里**不加**
|
||||
-- FOREIGN KEY 约束。原因:
|
||||
-- * 物理外键会在写入端传错/传旧 id 时直接报错中断整批采集, 而采集是「宁可
|
||||
-- 留 NULL 待回填, 也不要整日落空」的语义, 硬约束与这个语义冲突;
|
||||
-- * 款式被清理/重建 (拆系列、合并同款) 时要先删约束才能动数据, 运维成本高;
|
||||
-- * data-hub 侧 22 张表同样只用 style_id 做关联, 不带物理外键, 两边保持一致。
|
||||
--
|
||||
-- 完整性靠三件事保证, 不需要数据库兜底:
|
||||
-- 1. 写入端: loader 下发的 style_id 来自 dim_style 本身;
|
||||
-- 按商品 ID / 款名兜底的路径由 attach_style_ids 统一查询后填充;
|
||||
-- 2. 校验: backfill_style_id.py --dry-run 与 v_style_id_coverage 报覆盖率;
|
||||
-- 3. 观火: v_style_unmatched_product / v_style_unmatched_source 列未建档清单。
|
||||
--
|
||||
-- 下面这段只做**历史清理**: 早期 P4 曾给 10 张表建过 <表名>_style_id_fk,
|
||||
-- 这里幂等删除, 保证老库重跑 schema 后回到「只有索引、没有物理外键」的状态。
|
||||
DO $$
|
||||
DECLARE
|
||||
target_table TEXT;
|
||||
BEGIN
|
||||
FOREACH target_table IN ARRAY ARRAY[
|
||||
'daily_style_metrics',
|
||||
'weekly_style_metrics',
|
||||
'erp_all_shop_style_daily_metrics',
|
||||
'daily_persona_metrics',
|
||||
'main_image_creatives',
|
||||
'fact_style_analysis_reports',
|
||||
'decline_events',
|
||||
'fact_alert_history',
|
||||
'jd_self_weekly_sales',
|
||||
'persona_screenshots'
|
||||
] LOOP
|
||||
IF to_regclass('public.' || target_table) IS NOT NULL THEN
|
||||
EXECUTE format(
|
||||
'ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I',
|
||||
target_table, target_table || '_style_id_fk'
|
||||
);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
@@ -3258,14 +3258,12 @@ def _page_automation(page) -> None:
|
||||
for product in summary.get("products", [])
|
||||
]
|
||||
|
||||
# 评论采集挂在默认流程末尾, 让 daily --target-date 也能采到评价.
|
||||
# _collect_comment_summary 内部会 _open_comment_detail_page 跳转到评论页,
|
||||
# 采完好/中/差后覆盖 评论汇总_<YYYYMMDD>.json. 幂等: 同 order_id 重采覆盖.
|
||||
try:
|
||||
COMMENTS_SUMMARY = _collect_comment_summary(page)
|
||||
_save_comment_summary(COMMENTS_SUMMARY)
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 评论采集失败 (不影响单品数据已落盘): {exc}")
|
||||
# 评论采集已于 2026-11-01 从默认流程摘除:
|
||||
# 它只产出本地 评论汇总_<YYYYMMDD>.json,而唯一消费方 product_reviews 已下线
|
||||
# (差评改由 shop.dy_negative_reviews.daily 独立采集,消费方读 v_negative_reviews)。
|
||||
# 继续跑等于每天多一次浏览器跳转 + 一份无人读取的文件。
|
||||
# 需要历史评价样本时用 --reviews-only 显式触发,不要恢复默认流程。
|
||||
print("[SKIP] 评论采集已在默认流程下线(如需补采请用 --reviews-only)")
|
||||
COMMENTS_SUMMARY = {}
|
||||
|
||||
_results_saved = True
|
||||
|
||||
@@ -75,9 +75,20 @@ def load_erp_metrics(platform, style_name, target_date=None):
|
||||
failed = [c for c in code_results if not c.get("ok") or c.get("error")]
|
||||
if code_results and failed:
|
||||
# 任一编码失败都会造成 ERP 合计不完整,整款回退到平台原值。
|
||||
# 但 TM/DY 没有平台销量字段,整款回退会被兜底成 0,把整款销量抹掉
|
||||
# (如 10244 allsku iframe 偶发 timeout → 宙斯天猫销量变 0)。
|
||||
# 所以同时给出"已成功编码合计",供无平台原值时使用。
|
||||
return {
|
||||
"sales": None,
|
||||
"refund_orders": None,
|
||||
"partial_sales": (
|
||||
sum(int(parse_number(c.get("yesterday_sales"))) for c in ok_results)
|
||||
if ok_results else None
|
||||
),
|
||||
"partial_refunds": (
|
||||
sum(int(parse_number(c.get("yesterday_returns"))) for c in ok_results)
|
||||
if ok_results else None
|
||||
),
|
||||
"source": str(json_path),
|
||||
"erp_style_codes": payload.get("erp_style_codes", []),
|
||||
"all_failed": True,
|
||||
@@ -114,9 +125,17 @@ def apply_erp_override(platform, style_name, visitors, cart_users, target_date=N
|
||||
c = int(parse_number(cart_users))
|
||||
|
||||
if not erp or erp.get("all_failed"):
|
||||
# ERP 拿不到数据 → 用平台原值(如果有),否则 0
|
||||
sales = int(parse_number(platform_sales)) if platform_sales is not None else 0
|
||||
if platform_sales is not None:
|
||||
# 平台自己有销量字段(JD/DY)→ 用平台原值,最接近真相
|
||||
sales = int(parse_number(platform_sales))
|
||||
refunds = int(parse_number(platform_refunds)) if platform_refunds is not None else 0
|
||||
elif erp and erp.get("partial_sales") is not None:
|
||||
# 平台没有销量字段(TM)→ 用已成功编码的合计,避免整款归零
|
||||
sales = int(erp["partial_sales"])
|
||||
refunds = int(erp.get("partial_refunds") or 0)
|
||||
else:
|
||||
sales = 0
|
||||
refunds = 0
|
||||
return {
|
||||
"销量": sales,
|
||||
"访客数": v,
|
||||
@@ -127,6 +146,7 @@ def apply_erp_override(platform, style_name, visitors, cart_users, target_date=N
|
||||
"退款率": format_percent(refunds, sales),
|
||||
"erp_override": erp,
|
||||
"erp_unavailable": True,
|
||||
"erp_partial": bool(erp and erp.get("partial_sales") is not None and platform_sales is None),
|
||||
"failed_codes": (erp or {}).get("failed_codes", []),
|
||||
}
|
||||
|
||||
|
||||
@@ -1,331 +0,0 @@
|
||||
"""
|
||||
扫描 data/{jd,dy}单品数据/评价*.json (或 评论汇总_*.json),
|
||||
归一化后写入 product_reviews 表 (ON CONFLICT 全量覆盖)。
|
||||
|
||||
DY 评价入库时会反查 dim_style.dy_spus 填 style_name;
|
||||
JD 评价缺 SPU->款式映射, style_name 留 NULL (待 dim_jd_sku_spu 映射表补齐后再做)。
|
||||
|
||||
用法:
|
||||
.venv\\Scripts\\python.exe import_product_reviews.py # 全量 (默认只 dy)
|
||||
.venv/Scripts/python.exe import_product_reviews.py --date 2026-07-13 # 单日
|
||||
.venv/Scripts/python.exe import_product_reviews.py --from 2026-06-01 --to 2026-07-13
|
||||
.venv/Scripts/python.exe import_product_reviews.py --platforms dy
|
||||
.venv/Scripts/python.exe import_product_reviews.py --dry-run # 只解析不入库
|
||||
|
||||
源 JSON 文件:
|
||||
JD: data/jd单品数据/评价汇总_<YYYY-MM-DD>.json
|
||||
结构: {platform, shop, collect_date, comment_date_range, captured_at,
|
||||
total_count, ratings: {好评/中评/差评: {collected_count, comments:[{
|
||||
user_name, score, tags, content, time, order_id,
|
||||
sku_name, sku_id, sku_size, rating}]}}}
|
||||
DY: data/dy单品数据/评论汇总_<YYYYMMDD>.json
|
||||
结构: {shop, period, captured_at, url, total_collected_count,
|
||||
ratings: {好评/中评/差评: {expected_count, collected_count, comments:[{
|
||||
rating, order_id, comment_time, comment_text, product_name,
|
||||
product_id, spec_name, spec_id, raw_text}]}}}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from runtime_paths import RAW_DATA_ROOT, artifact_relative_path
|
||||
from typing import Any, Iterable
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from db import get_conn, upsert_product_reviews # noqa: E402
|
||||
|
||||
|
||||
DATA_ROOT = RAW_DATA_ROOT
|
||||
|
||||
PLATFORM_DIRS = {
|
||||
"jd": "jd单品数据",
|
||||
"dy": "dy单品数据",
|
||||
}
|
||||
|
||||
# 文件名模式: JD 用 评价汇总_2026-07-14.json, DY 用 评论汇总_20260604.json
|
||||
FILE_PATTERNS = {
|
||||
"jd": re.compile(r"^评价汇总_(\d{4}-\d{2}-\d{2})\.json$"),
|
||||
"dy": re.compile(r"^评论汇总_(\d{8})\.json$"),
|
||||
}
|
||||
|
||||
VALID_PLATFORMS = tuple(PLATFORM_DIRS.keys())
|
||||
|
||||
RATING_VALUES = ("好评", "中评", "差评")
|
||||
|
||||
|
||||
def _parse_jd_date(s: str) -> date:
|
||||
return datetime.strptime(s, "%Y-%m-%d").date()
|
||||
|
||||
|
||||
def _parse_dy_date(s: str) -> date:
|
||||
return datetime.strptime(s, "%Y%m%d").date()
|
||||
|
||||
|
||||
def _parse_comment_date(s: str) -> date | None:
|
||||
"""从评价时间字符串抽日期部分。
|
||||
|
||||
JD time 格式: '2026-07-13 12:34:56' 或 '2026-07-13'
|
||||
DY comment_time 格式: '2026/06/05 16:39:18' 或 '2026/06/05'
|
||||
"""
|
||||
if not s:
|
||||
return None
|
||||
s = s.strip()
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d %H:%M:%S", "%Y/%m/%d"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
m = re.match(r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})", s)
|
||||
if m:
|
||||
try:
|
||||
return date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _to_smallint(v: Any) -> int | None:
|
||||
if v is None or v == "":
|
||||
return None
|
||||
try:
|
||||
return int(float(str(v)))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_jd(payload: dict, source_file: str, collect_date: date,
|
||||
style_map: dict[str, str] | None = None) -> list[dict]:
|
||||
"""JD JSON -> product_reviews 记录列表
|
||||
|
||||
JD 评价表只有 sku_id (无 SPU), 暂不反查 style_name, 留 NULL。
|
||||
"""
|
||||
records = []
|
||||
for rating in RATING_VALUES:
|
||||
bucket = (payload.get("ratings") or {}).get(rating) or {}
|
||||
for c in bucket.get("comments") or []:
|
||||
comment_date = _parse_comment_date(c.get("time", ""))
|
||||
records.append({
|
||||
"platform": "jd",
|
||||
"collect_date": collect_date,
|
||||
"comment_date": comment_date,
|
||||
"style_name": None, # JD sku_id -> SPU 映射暂缺, 待 dim_jd_sku_spu 补齐
|
||||
"rating": rating,
|
||||
"score": _to_smallint(c.get("score")),
|
||||
"content": (c.get("content") or "").strip() or None,
|
||||
"tags": [t for t in (c.get("tags") or []) if t] or None,
|
||||
"user_name": c.get("user_name") or None,
|
||||
"order_id": str(c.get("order_id") or "").strip(),
|
||||
"product_id": None, # JD 评价表只有 sku_id, SPU 留空
|
||||
"product_name": c.get("sku_name") or None,
|
||||
"sku_id": c.get("sku_id") or None,
|
||||
"sku_name": c.get("sku_name") or None,
|
||||
"sku_size": c.get("sku_size") or None,
|
||||
"raw_data": c,
|
||||
"source_file": source_file,
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
def _normalize_dy(payload: dict, source_file: str, collect_date: date,
|
||||
style_map: dict[str, str] | None = None) -> list[dict]:
|
||||
"""DY JSON -> product_reviews 记录列表
|
||||
|
||||
style_map: {product_id: style_name}, 从 dim_style.dy_spus 构建。
|
||||
匹配不上时 style_name = None (表示 dim_style 缺该商品 ID 维护)。
|
||||
"""
|
||||
style_map = style_map or {}
|
||||
records = []
|
||||
for rating in RATING_VALUES:
|
||||
bucket = (payload.get("ratings") or {}).get(rating) or {}
|
||||
for c in bucket.get("comments") or []:
|
||||
comment_date = _parse_comment_date(c.get("comment_time", ""))
|
||||
product_id = str(c.get("product_id") or "").strip()
|
||||
records.append({
|
||||
"platform": "dy",
|
||||
"collect_date": collect_date,
|
||||
"comment_date": comment_date,
|
||||
"style_name": style_map.get(product_id),
|
||||
"rating": rating,
|
||||
"score": None, # DY 无星级
|
||||
"content": (c.get("comment_text") or "").strip() or None,
|
||||
"tags": None,
|
||||
"user_name": None, # DY 评论无独立用户名字段
|
||||
"order_id": str(c.get("order_id") or "").strip(),
|
||||
"product_id": product_id or None,
|
||||
"product_name": c.get("product_name") or None,
|
||||
"sku_id": c.get("spec_id") or None,
|
||||
"sku_name": c.get("spec_name") or None,
|
||||
"sku_size": None,
|
||||
"raw_data": c,
|
||||
"source_file": source_file,
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
NORMALIZERS = {
|
||||
"jd": _normalize_jd,
|
||||
"dy": _normalize_dy,
|
||||
}
|
||||
|
||||
|
||||
def _load_dy_style_map() -> dict[str, str]:
|
||||
"""从 dim_style.dy_spus 构建 {product_id: style_name} 映射。"""
|
||||
mapping: dict[str, str] = {}
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT style_name, dy_spus FROM dim_style WHERE dy_spus IS NOT NULL")
|
||||
for style_name, spus in cur.fetchall():
|
||||
if not spus:
|
||||
continue
|
||||
for pid in spus:
|
||||
mapping[str(pid)] = style_name
|
||||
return mapping
|
||||
|
||||
|
||||
def _discover_files(platforms: list[str], date_from: date | None, date_to: date | None) -> list[tuple[str, date, Path]]:
|
||||
"""返回 [(platform, collect_date, file_path), ...]"""
|
||||
out = []
|
||||
for plat in platforms:
|
||||
plat_dir = DATA_ROOT / PLATFORM_DIRS[plat]
|
||||
if not plat_dir.exists():
|
||||
continue
|
||||
pattern = FILE_PATTERNS[plat]
|
||||
for path in sorted(plat_dir.iterdir()):
|
||||
m = pattern.match(path.name)
|
||||
if not m:
|
||||
continue
|
||||
try:
|
||||
collect_date = (_parse_jd_date if plat == "jd" else _parse_dy_date)(m.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if date_from and collect_date < date_from:
|
||||
continue
|
||||
if date_to and collect_date > date_to:
|
||||
continue
|
||||
out.append((plat, collect_date, path))
|
||||
return out
|
||||
|
||||
|
||||
def _load_and_normalize(plat: str, collect_date: date, path: Path,
|
||||
style_map: dict[str, str] | None = None) -> list[dict]:
|
||||
with path.open(encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
rel = artifact_relative_path(path)
|
||||
return NORMALIZERS[plat](payload, rel, collect_date, style_map=style_map)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="扫描 JD/DY 评价 JSON 入库 product_reviews")
|
||||
parser.add_argument("--date", help="只导入指定采集日期 (YYYY-MM-DD); JD 用 - 分隔, DY 无分隔也可")
|
||||
parser.add_argument("--from", dest="date_from", help="起始采集日期 (YYYY-MM-DD)")
|
||||
parser.add_argument("--to", dest="date_to", help="结束采集日期 (YYYY-MM-DD)")
|
||||
parser.add_argument("--platforms", default="dy", help="平台, 逗号分隔 (默认 dy; JD 待 SPU 映射补齐)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="只解析不入库")
|
||||
args = parser.parse_args()
|
||||
|
||||
platforms = [p.strip() for p in args.platforms.split(",") if p.strip()]
|
||||
for p in platforms:
|
||||
if p not in VALID_PLATFORMS:
|
||||
print(f"[ERR] 未知平台: {p} (允许: {VALID_PLATFORMS})")
|
||||
return 2
|
||||
|
||||
date_from = _parse_single(args.date_from) if args.date_from else None
|
||||
date_to = _parse_single(args.date_to) if args.date_to else None
|
||||
if args.date:
|
||||
d = _parse_single(args.date)
|
||||
date_from = d
|
||||
date_to = d
|
||||
|
||||
files = _discover_files(platforms, date_from, date_to)
|
||||
if not files:
|
||||
print(f"[WARN] 未找到匹配的 JSON 文件 (platforms={platforms}, from={date_from}, to={date_to})")
|
||||
return 0
|
||||
|
||||
# 预加载 DY 款式映射 (dim_style.dy_spus -> {product_id: style_name})
|
||||
style_map: dict[str, str] = {}
|
||||
if "dy" in platforms and not args.dry_run:
|
||||
style_map = _load_dy_style_map()
|
||||
print(f"[INFO] dim_style.dy_spus 映射: {len(style_map)} 个 product_id -> style_name")
|
||||
elif "dy" in platforms and args.dry_run:
|
||||
# dry-run 也加载, 便于预览匹配情况
|
||||
style_map = _load_dy_style_map()
|
||||
print(f"[INFO] (dry-run) dim_style.dy_spus 映射: {len(style_map)} 个 product_id -> style_name")
|
||||
|
||||
total = 0
|
||||
skipped = 0
|
||||
by_plat: dict[str, int] = {p: 0 for p in platforms}
|
||||
by_rating: dict[str, int] = {"好评": 0, "中评": 0, "差评": 0}
|
||||
matched_styles: set[str] = set()
|
||||
|
||||
for plat, collect_date, path in files:
|
||||
try:
|
||||
records = _load_and_normalize(plat, collect_date, path, style_map=style_map)
|
||||
except Exception as e:
|
||||
print(f"[ERR] 解析 {path}: {e}")
|
||||
continue
|
||||
# 去重: 同 order_id 多条 (理论上 UNIQUE 会处理, 这里先在内存去重避免执行 batch fail)
|
||||
seen = set()
|
||||
deduped = []
|
||||
for r in records:
|
||||
if not r["order_id"]:
|
||||
skipped += 1
|
||||
continue
|
||||
if r["order_id"] in seen:
|
||||
skipped += 1
|
||||
continue
|
||||
seen.add(r["order_id"])
|
||||
deduped.append(r)
|
||||
by_rating[r["rating"]] = by_rating.get(r["rating"], 0) + 1
|
||||
if r.get("style_name"):
|
||||
matched_styles.add(r["style_name"])
|
||||
|
||||
plat_style_matched = sum(1 for r in deduped if r.get("style_name"))
|
||||
print(f"[{plat}] {collect_date} {path.name} -> {len(deduped)} 条 (款式匹配 {plat_style_matched}/{len(deduped)})")
|
||||
|
||||
if args.dry_run:
|
||||
if deduped:
|
||||
sample = deduped[0].copy()
|
||||
sample["raw_data"] = "(省略)"
|
||||
print(f" (dry-run, 不入库) 样本: {sample}")
|
||||
else:
|
||||
with get_conn() as conn:
|
||||
n = upsert_product_reviews(conn, deduped)
|
||||
print(f" -> upsert {n} rows")
|
||||
|
||||
total += len(deduped)
|
||||
by_plat[plat] = by_plat.get(plat, 0) + len(deduped)
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print(f" 总计: {total} 条 (跳过 {skipped} 条无 order_id)")
|
||||
for p, n in by_plat.items():
|
||||
print(f" {p}: {n} 条")
|
||||
print(" 按评级:")
|
||||
for r, n in by_rating.items():
|
||||
print(f" {r}: {n} 条")
|
||||
if matched_styles:
|
||||
print(f" 匹配到款式 ({len(matched_styles)} 款): {', '.join(sorted(matched_styles))}")
|
||||
print("=" * 60)
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_single(s: str) -> date:
|
||||
s = s.strip()
|
||||
for fmt in ("%Y-%m-%d", "%Y%m%d"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
raise SystemExit(f"无法解析日期: {s}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,9 +1,11 @@
|
||||
"""将万相台商品广告 CSV 幂等写入产品生命进程读取的字段。
|
||||
"""将万相台商品广告 CSV 幂等写入天猫商品投放日报表。
|
||||
|
||||
真实数据契约是 ``product_daily_metrics.raw_data.tm_ad_metrics``,不是一张
|
||||
独立的广告表。导入时只替换该 JSON 节点,保留同一商品日期上已经存在的
|
||||
经营日报字段;同时把款式匹配结果写入该节点的 ``style_name`` /
|
||||
``style_match_method``,数据-hub 后端据此计算广告成交金额、投产比和电商费比。
|
||||
数据契约是独立表 ``tm_product_ad_daily_metrics``,每日每商品一行:花费、直接/
|
||||
间接/归因成交金额与投产比走标量列,报表其余指标原样存 ``metrics`` JSONB。
|
||||
款式按 ``dim_style.tm_spus`` 的商品 ID 精确匹配,未命中再按商品名称精确包含
|
||||
兜底,结果写进 ``style_name`` / ``style_match_method``,data-hub 后端据此计算
|
||||
广告成交金额、投产比和电商费比。广告与经营日报分表存储,经营数据重新同步不会
|
||||
覆盖投放数据。
|
||||
|
||||
默认只解析校验;显式传入 ``--execute`` 才写 PostgreSQL。
|
||||
"""
|
||||
@@ -25,8 +27,9 @@ PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from db import ( # noqa: E402
|
||||
ensure_tmall_product_ad_daily_table,
|
||||
get_conn,
|
||||
upsert_tmall_product_ad_raw_metrics,
|
||||
upsert_tmall_product_ad_daily_metrics,
|
||||
)
|
||||
|
||||
AMOUNT_COLUMNS = {
|
||||
@@ -327,8 +330,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return 0
|
||||
|
||||
with get_conn() as conn:
|
||||
written = upsert_tmall_product_ad_raw_metrics(conn, records)
|
||||
print(f"[OK] 已写入 product_daily_metrics.raw_data.tm_ad_metrics:{written} 行")
|
||||
ensure_tmall_product_ad_daily_table(conn)
|
||||
written = upsert_tmall_product_ad_daily_metrics(conn, records)
|
||||
print(f"[OK] 已写入 tm_product_ad_daily_metrics:{written} 行")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ from collection_progress import SpuProgress
|
||||
|
||||
from jd_data_collector import (
|
||||
DEFAULT_CONFIG,
|
||||
close_ai_helper_drawer_if_present as close_jd_ai_helper_if_present,
|
||||
step_login,
|
||||
step_enter_shangzhi,
|
||||
extract,
|
||||
@@ -932,19 +933,40 @@ _JMT_CUSTOM_REACT_CLICK = r"""(args) => {
|
||||
const [ty, tm, targetDay] = args;
|
||||
const pad = (n) => String(n).padStart(2, '0');
|
||||
const ymRe = new RegExp('^' + ty + '\\u5e74' + pad(tm) + '\\u6708$');
|
||||
// 商智新版「自定义」是双月日历: 每个月的面板里除了本月日期, 还会渲染
|
||||
// 上下月的补位日期 (class 带 -diff-month)。同一个月面板里因此可能出现
|
||||
// 两个同号日期 (如 10 月面板里的 9/30 补位格与 10/30 真身), 之前按
|
||||
// 「第一个匹配」点击会落到补位格, 于是 echo 变成 2026-09-30。
|
||||
// 这里先定位目标月份自己的面板, 再排除补位格与禁用格。
|
||||
let panel = null;
|
||||
for (const el of document.querySelectorAll('*')) {
|
||||
const t = (el.textContent || '').trim();
|
||||
if (!ymRe.test(t)) continue;
|
||||
panel = el.closest('[class*="calendar"]') || el.parentElement.parentElement || el.parentElement;
|
||||
let cur = el.parentElement;
|
||||
let depth = 0;
|
||||
while (cur && depth < 6) {
|
||||
if (cur.querySelector && cur.querySelector('td')) break;
|
||||
cur = cur.parentElement;
|
||||
depth++;
|
||||
}
|
||||
if (cur && cur.querySelector('td')) {
|
||||
panel = cur;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!panel) return {status: 'no_panel'};
|
||||
for (const cell of panel.querySelectorAll('td, [class*="cell"]')) {
|
||||
if ((cell.textContent || '').trim() !== targetDay) continue;
|
||||
if (/disabled|off/i.test(String(cell.className))) continue;
|
||||
const cells = Array.from(panel.querySelectorAll('td, [class*="cell"]')).filter((cell) => {
|
||||
if ((cell.textContent || '').trim() !== targetDay) return false;
|
||||
const cls = String(cell.className || '');
|
||||
if (/diff-month|disabled|-off\b|other-month|prev-month|next-month/i.test(cls)) return false;
|
||||
const rect = cell.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
});
|
||||
if (!cells.length) return {status: 'no_cell'};
|
||||
for (const cell of cells) {
|
||||
const nodes = [cell, ...cell.querySelectorAll('*'), cell.parentElement];
|
||||
for (const node of nodes) {
|
||||
if (!node) continue;
|
||||
const propsKey = Object.keys(node).find((k) => k.startsWith('__reactProps$'));
|
||||
if (!propsKey) continue;
|
||||
const props = node[propsKey];
|
||||
@@ -955,9 +977,8 @@ _JMT_CUSTOM_REACT_CLICK = r"""(args) => {
|
||||
return {status: 'ok'};
|
||||
}
|
||||
}
|
||||
return {status: 'no_handler'};
|
||||
}
|
||||
return {status: 'no_cell'};
|
||||
return {status: 'no_handler'};
|
||||
}"""
|
||||
|
||||
|
||||
@@ -1006,12 +1027,32 @@ def _select_custom_date_new_version(page, target) -> bool:
|
||||
点工具条「自定义」→ 在双月日历里对目标日连续触发两次 React onClick
|
||||
(起止同日 = 单日区间) → echo 显示目标日期即成功.
|
||||
"""
|
||||
# 商智会随机弹出 AI 助手抽屉 / 引导层, 全屏 jd-modal-wrap 会拦截
|
||||
# 日期工具条的点击; 先尝试收起再点「自定义」。
|
||||
try:
|
||||
page.keyboard.press("Escape")
|
||||
page.wait_for_timeout(300)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
close_jd_ai_helper_if_present(page)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
tag = page.get_by_text("自定义", exact=True).first
|
||||
tag.wait_for(state="visible", timeout=8000)
|
||||
tag.click(timeout=8000)
|
||||
except Exception as exc:
|
||||
print(f" [WARN] 新版「自定义」入口点击失败: {exc}")
|
||||
print(f" [WARN] 新版「自定义」入口点击失败({str(exc)[:120]}), 改用 DOM 事件点击")
|
||||
clicked = page.evaluate(
|
||||
r"""() => {
|
||||
const el = document.querySelector('li[data-event-content="当前时间_自定义"]');
|
||||
if (!el) return false;
|
||||
el.dispatchEvent(new MouseEvent('click', {bubbles: true, cancelable: true}));
|
||||
return true;
|
||||
}"""
|
||||
)
|
||||
if not clicked:
|
||||
return False
|
||||
page.wait_for_timeout(1200)
|
||||
|
||||
|
||||
@@ -433,20 +433,9 @@ def run_collect(args, target_date, platforms, failed):
|
||||
failed=failed,
|
||||
historical=is_historical_target(target_date),
|
||||
)
|
||||
# 评论入库 (DY 默认分支已采评论 -> 评论汇总_<YYYYMMDD>.json; JD 单独 --reviews-only 采)
|
||||
# 幂等: product_reviews UNIQUE (platform, order_id) ON CONFLICT DO UPDATE, 重跑覆盖不双写.
|
||||
# critical=False: 评论入库失败不影响后续 analyze/export/insert.
|
||||
if not args.skip_platforms and not args.skip_reviews_import:
|
||||
review_plats = [p for p in platforms if p in ("jd", "dy")]
|
||||
if review_plats:
|
||||
run_step_with_retry(
|
||||
"import_product_reviews",
|
||||
[sys.executable, str(runtime_script("import_product_reviews.py")),
|
||||
"--platforms", ",".join(review_plats)],
|
||||
dry_run=args.dry_run,
|
||||
critical=False,
|
||||
failed=failed,
|
||||
)
|
||||
# 评论入库已下线 (2026-11-01): 差评由 shop.jd/dy/tmall_negative_reviews.daily
|
||||
# 三个独立工作流采集, 消费方统一读视图 v_negative_reviews。
|
||||
# product_reviews 停写并归档为 product_reviews_archived_20261101。
|
||||
|
||||
|
||||
def collect_daily_skip_report(date_str: str) -> Path | None:
|
||||
@@ -675,8 +664,6 @@ def main() -> int:
|
||||
parser.add_argument("--platforms", default="jd,dy,tm", help="逗号分隔: jd,dy,tm")
|
||||
parser.add_argument("--serial-platforms", action="store_true", help="三平台串行跑")
|
||||
parser.add_argument("--skip-insert", action="store_true", help="只导出,不插入飞书")
|
||||
parser.add_argument("--skip-reviews-import", action="store_true",
|
||||
help="跳过评论入库 (默认 daily collect 后跑 import_product_reviews)")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument(
|
||||
"--erp-mode",
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
"""评论采集编排器: 默认只跑 DY 采集 + 入库 PG。
|
||||
|
||||
JD 评价采集代码已就绪 (jd_product_data_collector.py --reviews-only),
|
||||
但 JD 评价表只有 sku_id 缺 SPU, 暂时无法反查到款式, 所以编排器默认不跑 JD。
|
||||
等 dim_jd_sku_spu 映射表补齐后可加 --platforms jd,dy。
|
||||
|
||||
DY 主流程 (dy_product_scraping.py 默认分支) 已挂载 _collect_comment_summary,
|
||||
daily --target-date 会顺带采评论; 本编排器用 --reviews-only 只跑评论 (跳过单品列表/
|
||||
下载/人群画像), 用于历史补采或重跑。
|
||||
入库调 import_product_reviews.py, ON CONFLICT (platform, order_id) 幂等覆盖。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py --target-date 2026-07-13
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py --platforms jd,dy # JD 待 SPU 映射补齐
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py --skip-collect # 只入库
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py --dry-run
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from collect_retry_utils import FailedCollector
|
||||
from runtime_paths import runtime_script
|
||||
|
||||
from gyxx_flow.adapters import environment_for_child_script
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
MAX_RUN_RETRIES = 2
|
||||
RUN_SLEEP_SECONDS = 5
|
||||
|
||||
|
||||
def default_target_date() -> str:
|
||||
return (datetime.now().date() - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def validate_date(value: str) -> str:
|
||||
try:
|
||||
datetime.strptime(value, "%Y-%m-%d")
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError("date must be YYYY-MM-DD") from exc
|
||||
return value
|
||||
|
||||
|
||||
def run_step(name, command, dry_run=False, critical=True):
|
||||
"""执行子命令。critical=True 失败抛 RuntimeError, False 仅 [WARN] 继续。"""
|
||||
printable = " ".join(str(c) for c in command)
|
||||
print(f"[RUN] {name}: {printable}")
|
||||
if dry_run:
|
||||
return {"name": name, "code": 0, "dry_run": True}
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
command,
|
||||
cwd=PROJECT_ROOT,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
env=environment_for_child_script(command[1], os.environ),
|
||||
)
|
||||
except (FileNotFoundError, OSError) as exc:
|
||||
msg = f"{name} {type(exc).__name__}: {exc}"
|
||||
if critical:
|
||||
raise RuntimeError(msg) from exc
|
||||
print(f"[WARN] {msg}")
|
||||
return {"name": name, "code": 126, "error": msg}
|
||||
|
||||
if proc.returncode:
|
||||
msg = f"{name} failed with exit code {proc.returncode}"
|
||||
if critical:
|
||||
raise RuntimeError(msg)
|
||||
print(f"[WARN] {msg}")
|
||||
return {"name": name, "code": proc.returncode, "error": msg}
|
||||
print(f"[OK] {name}")
|
||||
return {"name": name, "code": proc.returncode}
|
||||
|
||||
|
||||
def run_step_with_retry(name, command, dry_run=False, critical=True,
|
||||
failed: FailedCollector | None = None,
|
||||
max_retries: int = MAX_RUN_RETRIES,
|
||||
sleep_seconds: float = RUN_SLEEP_SECONDS) -> dict:
|
||||
if dry_run:
|
||||
return run_step(name, command, dry_run=True, critical=critical)
|
||||
last_result: dict = {"name": name, "code": 0}
|
||||
total = max_retries + 1
|
||||
for attempt in range(1, total + 1):
|
||||
result = run_step(name, command, dry_run=False, critical=False)
|
||||
last_result = result
|
||||
if result.get("code") == 0:
|
||||
if attempt > 1:
|
||||
print(f"[OK] {name} 第 {attempt}/{total} 次重试成功", flush=True)
|
||||
return result
|
||||
err = result.get("error", "unknown")
|
||||
if attempt >= total:
|
||||
if failed is not None:
|
||||
failed.add_page(name, f"{err} (attempts={attempt})", attempts=attempt)
|
||||
if critical:
|
||||
raise RuntimeError(f"{name} failed after {attempt} attempts: {err}")
|
||||
print(f"[WARN] {name} 失败 (已重试 {max_retries} 次, {err}), 继续", flush=True)
|
||||
return result
|
||||
print(f"[RETRY] {name} 第 {attempt}/{total} 次失败, {sleep_seconds}s 后重试...", flush=True)
|
||||
import time
|
||||
time.sleep(sleep_seconds)
|
||||
return last_result
|
||||
|
||||
|
||||
def build_collect_command(plat: str, target_date: str) -> list[str]:
|
||||
if plat == "jd":
|
||||
# JD --reviews-only: 只跑评价采集 (好/中/差全采), 跳过商智单品明细
|
||||
return [sys.executable, str(runtime_script("jd_product_data_collector.py")),
|
||||
"--reviews-only", "--target-date", target_date]
|
||||
if plat == "dy":
|
||||
# DY --reviews-only: 登录后直接跳评论页, 跳过单品列表/下载/人群画像
|
||||
return [sys.executable, str(runtime_script("dy_product_scraping.py")),
|
||||
"--reviews-only", "--target-date", target_date]
|
||||
raise ValueError(f"unknown platform: {plat}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="评论采集编排器 (JD + DY)")
|
||||
parser.add_argument("--target-date", type=validate_date, default=None,
|
||||
help="采集指定日期 YYYY-MM-DD (默认昨天)")
|
||||
parser.add_argument("--platforms", default="dy",
|
||||
help="平台, 逗号分隔 (默认 dy; JD 待 SPU 映射补齐, 可手动 jd,dy)")
|
||||
parser.add_argument("--skip-collect", action="store_true",
|
||||
help="跳过采集, 只跑 import_product_reviews.py")
|
||||
parser.add_argument("--skip-import", action="store_true",
|
||||
help="跳过入库, 只跑采集")
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="只打印命令不执行")
|
||||
args = parser.parse_args()
|
||||
|
||||
target_date = args.target_date or default_target_date()
|
||||
platforms = [p.strip() for p in args.platforms.split(",") if p.strip()]
|
||||
for p in platforms:
|
||||
if p not in ("jd", "dy"):
|
||||
print(f"[ERR] 未知平台: {p} (当前评论采集仅支持 jd, dy)")
|
||||
return 2
|
||||
|
||||
today_str = datetime.now().strftime("%Y-%m-%d")
|
||||
failed = FailedCollector("reviews", today_str)
|
||||
|
||||
print("=" * 60)
|
||||
print(" 评论采集编排器 (默认 DY)")
|
||||
print("=" * 60)
|
||||
print(f" Target date: {target_date}")
|
||||
print(f" Platforms: {platforms}")
|
||||
print(f" Skip collect: {args.skip_collect}")
|
||||
print(f" Skip import: {args.skip_import}")
|
||||
print(f" Dry run: {args.dry_run}")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
# Stage 1: 采集 (串行, Chrome profile 互锁)
|
||||
if not args.skip_collect:
|
||||
for plat in platforms:
|
||||
cmd = build_collect_command(plat, target_date)
|
||||
run_step_with_retry(
|
||||
f"collect_{plat}", cmd,
|
||||
dry_run=args.dry_run, critical=False, failed=failed,
|
||||
)
|
||||
else:
|
||||
print("[INFO] --skip-collect: 跳过采集阶段\n")
|
||||
|
||||
# Stage 2: 入库
|
||||
if not args.skip_import:
|
||||
import_cmd = [sys.executable, str(runtime_script("import_product_reviews.py")),
|
||||
"--platforms", ",".join(platforms)]
|
||||
if args.dry_run:
|
||||
import_cmd.append("--dry-run")
|
||||
run_step_with_retry(
|
||||
"import_reviews", import_cmd,
|
||||
dry_run=args.dry_run, critical=False, failed=failed,
|
||||
)
|
||||
else:
|
||||
print("[INFO] --skip-import: 跳过入库阶段\n")
|
||||
|
||||
# 失败清单
|
||||
failed_path = failed.write()
|
||||
if failed_path:
|
||||
print(f"\n[FAIL_LOG] {failed.summary_line()} -> {failed_path}")
|
||||
return 1
|
||||
print("\n[OK] 评论采集编排完成, 无失败")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,349 @@
|
||||
"""一次性迁移:把天猫广告指标从 ``product_daily_metrics.raw_data`` 搬到独立表。
|
||||
|
||||
背景:天猫广告指标原先写在 ``product_daily_metrics.raw_data.tm_ad_metrics``
|
||||
JSONB 节点里,与经营日报共用一行;经营数据重新同步时需要靠 JSONB 节点级合并
|
||||
才能保住投放数据。2026-11-01 起改为写 ``tm_product_ad_daily_metrics``(与京东、
|
||||
抖音同构),本脚本负责把历史节点回填进新表,并在校验通过后清除旧节点。
|
||||
|
||||
行为:
|
||||
- 默认 dry-run:只统计源数据规模、金额合计、款式匹配情况,不写库。
|
||||
- ``--execute``:先备份源节点到 JSON(``--backup-dir``,默认运行时 TMP_ROOT),
|
||||
再按 ``(stat_date, product_id)`` 幂等 upsert 到新表。
|
||||
- ``--clear-source``:仅在行数/金额校验一致、备份写盘成功后,才执行
|
||||
``raw_data - 'tm_ad_metrics'`` 清除节点;任何一项不通过都不清理并返回非零。
|
||||
|
||||
用法::
|
||||
|
||||
python scripts/migrate_tm_ad_metrics_to_table.py
|
||||
python scripts/migrate_tm_ad_metrics_to_table.py --execute
|
||||
python scripts/migrate_tm_ad_metrics_to_table.py --execute --clear-source
|
||||
|
||||
连接信息来自 PG_* 环境变量(与 db/__init__.py 同一套)。
|
||||
"""
|
||||
# ruff: noqa: E402
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
import sys
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
import db # noqa: E402
|
||||
from runtime_paths import TMP_ROOT # noqa: E402
|
||||
|
||||
SOURCE_PREDICATE = (
|
||||
"platform = 'tm' AND jsonb_typeof(raw_data -> 'tm_ad_metrics') = 'object'"
|
||||
)
|
||||
BATCH_SIZE = 500
|
||||
|
||||
|
||||
def _decimal(value: Any) -> Decimal:
|
||||
if value is None:
|
||||
return Decimal("0")
|
||||
text = str(value).strip().replace(",", "")
|
||||
if not text:
|
||||
return Decimal("0")
|
||||
try:
|
||||
return Decimal(text)
|
||||
except InvalidOperation:
|
||||
return Decimal("0")
|
||||
|
||||
|
||||
def _iter_source_rows(
|
||||
conn, date_from: date | None, date_to: date | None, batch_size: int
|
||||
) -> Iterator[list[dict[str, Any]]]:
|
||||
"""按 (stat_date, product_id) keyset 分页读源节点,避免一次性载入全部行。"""
|
||||
|
||||
clauses = [SOURCE_PREDICATE]
|
||||
params: list[Any] = []
|
||||
if date_from:
|
||||
clauses.append("stat_date >= %s")
|
||||
params.append(date_from)
|
||||
if date_to:
|
||||
clauses.append("stat_date <= %s")
|
||||
params.append(date_to)
|
||||
where = " AND ".join(clauses)
|
||||
|
||||
last: tuple[date, str] | None = None
|
||||
while True:
|
||||
page_where = where
|
||||
page_params = list(params)
|
||||
if last is not None:
|
||||
page_where += " AND (stat_date, product_id) > (%s, %s)"
|
||||
page_params.extend([last[0], last[1]])
|
||||
sql = f"""
|
||||
SELECT stat_date, product_id, product_name,
|
||||
raw_data -> 'tm_ad_metrics' AS node,
|
||||
source_file
|
||||
FROM product_daily_metrics
|
||||
WHERE {page_where}
|
||||
ORDER BY stat_date, product_id
|
||||
LIMIT {batch_size}
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, page_params)
|
||||
page = cur.fetchall()
|
||||
if not page:
|
||||
return
|
||||
yield [
|
||||
{
|
||||
"stat_date": row[0],
|
||||
"product_id": str(row[1]),
|
||||
"product_name": row[2],
|
||||
"node": row[3] or {},
|
||||
"source_file": row[4],
|
||||
}
|
||||
for row in page
|
||||
]
|
||||
last = (page[-1][0], str(page[-1][1]))
|
||||
if len(page) < batch_size:
|
||||
return
|
||||
|
||||
|
||||
def _to_records(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""把 JSONB 节点转成 upsert 函数接受的记录形态。"""
|
||||
|
||||
records = []
|
||||
for row in rows:
|
||||
node = dict(row["node"])
|
||||
metrics = {
|
||||
key: value
|
||||
for key, value in node.items()
|
||||
if key not in {"style_name", "style_match_method"}
|
||||
}
|
||||
records.append(
|
||||
{
|
||||
"stat_date": row["stat_date"],
|
||||
"product_id": row["product_id"],
|
||||
"product_name": row["product_name"],
|
||||
"ad_metrics": metrics,
|
||||
# 节点里已有的款式映射作为兜底;能按 tm_spus/名称重新匹配的以重算为准。
|
||||
"style_name": node.get("style_name"),
|
||||
"style_match_method": node.get("style_match_method"),
|
||||
"source_file": row["source_file"],
|
||||
}
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def _write_backup(rows: list[dict[str, Any]], backup_dir: Path) -> Path:
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
path = backup_dir / f"tm_ad_metrics_backup_{stamp}.json.gz"
|
||||
payload = [
|
||||
{
|
||||
"stat_date": row["stat_date"].isoformat(),
|
||||
"product_id": row["product_id"],
|
||||
"product_name": row["product_name"],
|
||||
"source_file": row["source_file"],
|
||||
"tm_ad_metrics": row["node"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
with gzip.open(path, "wt", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, default=str)
|
||||
return path
|
||||
|
||||
|
||||
def _target_totals(conn, date_from: date | None, date_to: date | None) -> tuple[int, Decimal]:
|
||||
clauses = ["TRUE"]
|
||||
params: list[Any] = []
|
||||
if date_from:
|
||||
clauses.append("stat_date >= %s")
|
||||
params.append(date_from)
|
||||
if date_to:
|
||||
clauses.append("stat_date <= %s")
|
||||
params.append(date_to)
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
SELECT COUNT(*), COALESCE(SUM(spend), 0)
|
||||
FROM tm_product_ad_daily_metrics
|
||||
WHERE {' AND '.join(clauses)}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
count, spend = cur.fetchone()
|
||||
return int(count), Decimal(str(spend))
|
||||
|
||||
|
||||
def _source_count(conn, date_from: date | None, date_to: date | None) -> int:
|
||||
clauses = [SOURCE_PREDICATE]
|
||||
params: list[Any] = []
|
||||
if date_from:
|
||||
clauses.append("stat_date >= %s")
|
||||
params.append(date_from)
|
||||
if date_to:
|
||||
clauses.append("stat_date <= %s")
|
||||
params.append(date_to)
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"SELECT COUNT(*) FROM product_daily_metrics WHERE {' AND '.join(clauses)}",
|
||||
params,
|
||||
)
|
||||
return int(cur.fetchone()[0])
|
||||
|
||||
|
||||
def _parse_date(value: str | None) -> date | None:
|
||||
return date.fromisoformat(value) if value else None
|
||||
|
||||
|
||||
def _missing_target_columns(conn) -> list[str]:
|
||||
"""dry-run 只读检查目标表是否已具备款式/指标列。"""
|
||||
|
||||
required = {"style_name", "style_match_method", "metrics", "source_row_number"}
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'tm_product_ad_daily_metrics'
|
||||
"""
|
||||
)
|
||||
existing = {row[0] for row in cur.fetchall()}
|
||||
if not existing:
|
||||
return sorted(required | {"<表不存在>"})
|
||||
return sorted(required - existing)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="迁移天猫广告指标到独立表")
|
||||
parser.add_argument("--date-from", help="只迁移该日期起 YYYY-MM-DD")
|
||||
parser.add_argument("--date-to", help="只迁移到该日期 YYYY-MM-DD")
|
||||
parser.add_argument("--execute", action="store_true", help="确认写入 PostgreSQL")
|
||||
parser.add_argument(
|
||||
"--clear-source",
|
||||
action="store_true",
|
||||
help="校验通过后清除 product_daily_metrics 里的旧 JSONB 节点",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backup-dir", type=Path, help="旧节点备份目录(默认运行时 TMP_ROOT)"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
date_from = _parse_date(args.date_from)
|
||||
date_to = _parse_date(args.date_to)
|
||||
backup_dir = args.backup_dir or TMP_ROOT
|
||||
|
||||
with db.get_conn() as conn:
|
||||
total_rows = _source_count(conn, date_from, date_to)
|
||||
if total_rows == 0:
|
||||
print("[INFO] 没有可迁移的天猫广告节点,退出")
|
||||
return 0
|
||||
|
||||
tally = {
|
||||
"rows": 0,
|
||||
"spend": Decimal("0"),
|
||||
"deal": Decimal("0"),
|
||||
"missing_style": 0,
|
||||
"unknown_style": 0,
|
||||
"min_date": None,
|
||||
"max_date": None,
|
||||
}
|
||||
all_rows: list[dict[str, Any]] = []
|
||||
resolved_style: dict[tuple[str, str], str | None] = {}
|
||||
|
||||
style_by_product_id, style_names = db._load_platform_style_catalog(conn, "tm_spus")
|
||||
for batch in _iter_source_rows(conn, date_from, date_to, BATCH_SIZE):
|
||||
all_rows.extend(batch)
|
||||
for record in _to_records(batch):
|
||||
node_style = str(record.get("style_name") or "").strip()
|
||||
if not node_style:
|
||||
tally["missing_style"] += 1
|
||||
key = (record["product_id"], str(record.get("product_name") or ""))
|
||||
if key not in resolved_style:
|
||||
resolved_style[key] = db._resolve_style_match(
|
||||
record, style_by_product_id, style_names
|
||||
)[0]
|
||||
if not (resolved_style[key] or node_style):
|
||||
tally["unknown_style"] += 1
|
||||
tally["rows"] += 1
|
||||
tally["spend"] += _decimal(record["ad_metrics"].get("spend"))
|
||||
tally["deal"] += _decimal(
|
||||
record["ad_metrics"].get("attributed_deal_amount")
|
||||
)
|
||||
stat_date = record["stat_date"]
|
||||
tally["min_date"] = (
|
||||
stat_date if tally["min_date"] is None else min(tally["min_date"], stat_date)
|
||||
)
|
||||
tally["max_date"] = (
|
||||
stat_date if tally["max_date"] is None else max(tally["max_date"], stat_date)
|
||||
)
|
||||
|
||||
print(
|
||||
f"[INFO] 源节点 {tally['rows']} 行({tally['min_date']} ~ {tally['max_date']}),"
|
||||
f"花费 {tally['spend']:.2f},广告归因成交 {tally['deal']:.2f}"
|
||||
)
|
||||
print(
|
||||
f"[INFO] 款式:源节点缺 style_name {tally['missing_style']} 行;"
|
||||
f"按 tm_spus/名称仍无法确定 {tally['unknown_style']} 行"
|
||||
)
|
||||
|
||||
if not args.execute:
|
||||
existing_count, _ = _target_totals(conn, date_from, date_to)
|
||||
missing = _missing_target_columns(conn)
|
||||
print(
|
||||
f"[INFO] 目标表现有 {existing_count} 行(同日期区间);"
|
||||
"DRY-RUN 未写库,确认后添加 --execute"
|
||||
)
|
||||
print(
|
||||
"[INFO] 目标表待补列:"
|
||||
+ (", ".join(missing) if missing else "无(结构已就绪)")
|
||||
)
|
||||
return 0
|
||||
|
||||
db.ensure_tmall_product_ad_daily_table(conn)
|
||||
|
||||
backup_path = _write_backup(all_rows, backup_dir)
|
||||
print(f"[OK] 旧节点已备份:{backup_path}")
|
||||
|
||||
written = 0
|
||||
for offset in range(0, len(all_rows), BATCH_SIZE):
|
||||
records = _to_records(all_rows[offset : offset + BATCH_SIZE])
|
||||
written += db.upsert_tmall_product_ad_daily_metrics(conn, records)
|
||||
print(f"[OK] 已写入 tm_product_ad_daily_metrics:{written} 行")
|
||||
|
||||
target_count, target_spend = _target_totals(conn, date_from, date_to)
|
||||
ok = target_count == tally["rows"] and target_spend == tally["spend"]
|
||||
print(
|
||||
f"[CHECK] 行数 {target_count}/{tally['rows']},"
|
||||
f"花费 {target_spend:.2f}/{tally['spend']:.2f} -> "
|
||||
f"{'一致' if ok else '不一致'}"
|
||||
)
|
||||
if not ok:
|
||||
print("[FAIL] 校验不一致,保留旧 JSONB 节点,请人工核对后重跑", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if not args.clear_source:
|
||||
print("[INFO] 未指定 --clear-source,旧节点保留")
|
||||
return 0
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE product_daily_metrics
|
||||
SET raw_data = COALESCE(raw_data, '{{}}'::jsonb) - 'tm_ad_metrics'
|
||||
WHERE {SOURCE_PREDICATE}
|
||||
{"AND stat_date >= %s" if date_from else ""}
|
||||
{"AND stat_date <= %s" if date_to else ""}
|
||||
""",
|
||||
[value for value in (date_from, date_to) if value is not None],
|
||||
)
|
||||
cleared = cur.rowcount
|
||||
remaining = _source_count(conn, date_from, date_to)
|
||||
print(f"[OK] 已清除旧节点:{cleared} 行,剩余 {remaining} 行")
|
||||
if remaining:
|
||||
print("[FAIL] 仍有残留节点,请重跑 --clear-source", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -86,6 +86,23 @@ _STATIC_STYLE_NAMES = [
|
||||
]
|
||||
|
||||
|
||||
# 万相「主体名称/ID」搜索词覆盖(2026-10-30):
|
||||
# 款式身份仍然只用 style_name —— 输出目录、飞书主图表、PG 记录都以款名为键;
|
||||
# 只有搜索框里填的词可以换,用于投放主体名与内部款名不一致的款。
|
||||
# 主搜索词无结果时按「原始款名 → 天猫商品ID」逐级兜底,不会因为改名少采集。
|
||||
STYLE_SEARCH_KEYWORDS = {
|
||||
"黑曜石吐司包": "黑曜石吐司",
|
||||
}
|
||||
|
||||
|
||||
def search_keywords_for(style_name: str) -> tuple[str, ...]:
|
||||
"""返回按优先级排列的万相搜索词;首个元素是主搜索词。"""
|
||||
keyword = STYLE_SEARCH_KEYWORDS.get(style_name, "").strip()
|
||||
if not keyword or keyword == style_name:
|
||||
return (style_name,)
|
||||
return (keyword, style_name)
|
||||
|
||||
|
||||
def _get_style_names() -> list[str]:
|
||||
"""采集目标=身份齐全的天猫款式;主图飞书表地址只影响写表,不影响采集。"""
|
||||
try:
|
||||
@@ -1656,17 +1673,37 @@ def aggregate_rows(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
return aggregated
|
||||
|
||||
|
||||
def collect_style(page, style_name: str, max_pages: int, item_ids: list[str] | None = None) -> dict[str, Any]:
|
||||
search_style_by_subject(page, style_name)
|
||||
def collect_style(
|
||||
page,
|
||||
style_name: str,
|
||||
max_pages: int,
|
||||
item_ids: list[str] | None = None,
|
||||
search_keywords: tuple[str, ...] | list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
# 搜索词可与款名不同(见 STYLE_SEARCH_KEYWORDS),但 style_name 始终是结果身份,
|
||||
# 输出目录/飞书/PG 都用它,不受搜索词影响。
|
||||
keywords = tuple(search_keywords) if search_keywords else search_keywords_for(style_name)
|
||||
used_keyword = keywords[0]
|
||||
if used_keyword != style_name:
|
||||
print(f" [SEARCH] 搜索词覆盖: {style_name} -> {used_keyword}")
|
||||
search_style_by_subject(page, used_keyword)
|
||||
all_rows: list[dict[str, Any]] = []
|
||||
visited_signatures: set[str] = set()
|
||||
|
||||
# 先检查款式名搜索是否有结果,无结果时用商品ID回退
|
||||
# 先检查主搜索词是否有结果,无结果时按「原始款名 → 商品ID」逐级回退
|
||||
first_check = extract_current_page_rows(page)
|
||||
for fallback in keywords[1:]:
|
||||
if first_check:
|
||||
break
|
||||
print(f" [FALLBACK] 搜索词“{used_keyword}”无结果,改用“{fallback}”重搜")
|
||||
search_style_by_subject(page, fallback)
|
||||
used_keyword = fallback
|
||||
first_check = extract_current_page_rows(page)
|
||||
if not first_check and item_ids:
|
||||
first_id = item_ids[0]
|
||||
print(f" [FALLBACK] 款式名搜不到结果,用商品ID {first_id} 重搜")
|
||||
search_style_by_subject(page, first_id)
|
||||
used_keyword = first_id
|
||||
|
||||
for page_no in range(1, max_pages + 1):
|
||||
image_map = collect_current_page_image_map(page)
|
||||
@@ -1692,6 +1729,7 @@ def collect_style(page, style_name: str, max_pages: int, item_ids: list[str] | N
|
||||
aggregated = aggregate_rows(all_rows)
|
||||
return {
|
||||
"style_name": style_name,
|
||||
"search_keyword": used_keyword,
|
||||
"collected_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"page_count": len(visited_signatures),
|
||||
"raw_row_count": len(all_rows),
|
||||
@@ -1731,6 +1769,7 @@ def write_style_outputs(result: dict[str, Any], output_dir: Path) -> tuple[Path,
|
||||
f"# {style_name} 万相创意主图汇总",
|
||||
"",
|
||||
f"- 采集时间: {result.get('collected_at', '')}",
|
||||
f"- 搜索词: {result.get('search_keyword') or style_name}",
|
||||
f"- 翻页数: {result.get('page_count', 0)}",
|
||||
f"- 原始行数: {result.get('raw_row_count', 0)}",
|
||||
f"- 聚合主图数: {result.get('creative_count', 0)}",
|
||||
|
||||
@@ -114,6 +114,10 @@ LUGGAGE_USER_DATA_DIR = os.getenv(
|
||||
"GUANGHE_LUGGAGE_USER_DATA_DIR",
|
||||
str(profile_path("guanghe-luggage")),
|
||||
)
|
||||
NEW_BAG_USER_DATA_DIR = os.getenv(
|
||||
"GUANGHE_NEW_BAG_USER_DATA_DIR",
|
||||
str(profile_path("guanghe-new-bag")),
|
||||
)
|
||||
|
||||
# ─────────────────────────── 配置 ───────────────────────────
|
||||
BASE_TOKEN = "TH1JbxhfUaxetis6F4pcoqvWncd"
|
||||
@@ -134,10 +138,24 @@ TABLE_FIELDS = [
|
||||
|
||||
STORE_FLAGSHIP = "flagship"
|
||||
STORE_LUGGAGE = "luggage"
|
||||
# 「天猫新店」(淘宝账号 gyxx箱包旗舰店:龙虾仔)只接黑曜石系列,视频源与其他店铺同一张飞书表。
|
||||
STORE_NEW_BAG = "new_bag"
|
||||
STORE_ALL = "all"
|
||||
STORE_ACCOUNT_KEYS = {
|
||||
STORE_FLAGSHIP: "guanghe-flagship",
|
||||
STORE_LUGGAGE: "guanghe-luggage",
|
||||
STORE_NEW_BAG: "guanghe-new-bag",
|
||||
}
|
||||
# 每个店铺独立的运行时凭据前缀与浏览器 Profile 目录环境变量;新增店铺只需在此登记。
|
||||
STORE_CREDENTIAL_PREFIXES = {
|
||||
STORE_FLAGSHIP: "GUANGHE",
|
||||
STORE_LUGGAGE: "GUANGHE_LUGGAGE",
|
||||
STORE_NEW_BAG: "GUANGHE_NEW_BAG",
|
||||
}
|
||||
# 天猫新店没有独立的款式白名单表,按「代拍产品」里的系列关键词过滤:
|
||||
# 一条记录的全部款式都必须落在该系列内才会上传,避免把非黑曜石视频发到新店。
|
||||
STORE_STYLE_KEYWORDS = {
|
||||
STORE_NEW_BAG: "黑曜石",
|
||||
}
|
||||
# Guanghe exposes a creator account, not the merchant-store name used by the
|
||||
# source routing and publication ledger. The flagship creator card currently
|
||||
@@ -147,6 +165,9 @@ STORE_ACCOUNT_KEYS = {
|
||||
# the platform no longer renders the merchant-store name on its home page.
|
||||
STORE_CREATOR_IDENTITY_MARKERS: dict[str, tuple[str, ...]] = {
|
||||
STORE_FLAGSHIP: ("光影行星天猫官方", "逛逛号:3301283916"),
|
||||
# 天猫新店的逛逛内容号由店铺资料自动生成,创作者后台只显示这个名字,
|
||||
# 淘宝登录名 gyxx箱包旗舰店:龙虾仔 不会出现在光合页面上。
|
||||
STORE_NEW_BAG: ("GYXX箱包天猫官方",),
|
||||
}
|
||||
|
||||
EXIT_OK = 0
|
||||
@@ -2630,24 +2651,13 @@ class GuangHeUploader:
|
||||
# 一次性打开弹窗,复用同一搜索框
|
||||
self._click_by_text("添加商品", timeout=10_000)
|
||||
print("[OK] 已点开关联商品弹窗")
|
||||
time.sleep(1.5)
|
||||
|
||||
# 搜索框(占位符含「搜索」)
|
||||
search = None
|
||||
for sel in [
|
||||
"input[placeholder*='搜索']",
|
||||
"input[type='search']",
|
||||
"input[placeholder*='商品']",
|
||||
]:
|
||||
try:
|
||||
loc = self._loc(sel).first
|
||||
if loc.count() > 0:
|
||||
search = loc
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
# 天猫新店的商品选择弹窗渲染比旗舰店慢,且占位文案不固定(有的店铺是
|
||||
# 「请输入商品ID/名称」),因此轮询多个候选选择器,超时后再打诊断。
|
||||
search = self._find_dialog_search_input(timeout_seconds=12)
|
||||
if search is None:
|
||||
print("[ERR] 关联商品弹窗内找不到搜索框 → 跳过本次发布")
|
||||
self._dump_dialog_diagnostics()
|
||||
for label in ["取消", "关 闭", "关闭"]:
|
||||
try:
|
||||
self._click_by_text(label, timeout=2000)
|
||||
@@ -3183,6 +3193,75 @@ class GuangHeUploader:
|
||||
print("[WARN] 未明确检测到发布成功提示,请人工确认")
|
||||
return published
|
||||
|
||||
DIALOG_SEARCH_SELECTORS = (
|
||||
"input[placeholder*='搜索']",
|
||||
"input[placeholder*='请输入']",
|
||||
"input[placeholder*='商品']",
|
||||
"input[type='search']",
|
||||
".next-dialog input[type='text']",
|
||||
".next-dialog input:not([type])",
|
||||
)
|
||||
|
||||
def _dialog_contexts(self) -> list:
|
||||
"""发布页 iframe、主页面以及其中所有子 frame,弹窗可能落在任意一层。"""
|
||||
contexts: list = []
|
||||
if self.pub_frame is not None:
|
||||
contexts.append(self.pub_frame)
|
||||
contexts.append(self.page)
|
||||
try:
|
||||
for frame in self.page.frames:
|
||||
if frame not in contexts:
|
||||
contexts.append(frame)
|
||||
except Exception:
|
||||
pass
|
||||
return contexts
|
||||
|
||||
def _find_dialog_search_input(self, timeout_seconds: float = 12):
|
||||
"""轮询各 frame 找关联商品弹窗的搜索框,找不到返回 None。"""
|
||||
deadline = time.time() + timeout_seconds
|
||||
while True:
|
||||
for ctx in self._dialog_contexts():
|
||||
for selector in self.DIALOG_SEARCH_SELECTORS:
|
||||
try:
|
||||
loc = ctx.locator(selector).first
|
||||
if loc.count() > 0:
|
||||
return loc
|
||||
except Exception:
|
||||
continue
|
||||
if time.time() >= deadline:
|
||||
return None
|
||||
time.sleep(0.8)
|
||||
|
||||
def _dump_dialog_diagnostics(self) -> None:
|
||||
"""找不到搜索框时把弹窗真实结构写进日志,避免只能靠复现定位。"""
|
||||
for idx, ctx in enumerate(self._dialog_contexts()):
|
||||
try:
|
||||
url = str(getattr(ctx, "url", "") or "")
|
||||
except Exception:
|
||||
url = ""
|
||||
print(f" [诊断 ctx#{idx}] url={url[:120]}")
|
||||
try:
|
||||
text = ctx.locator("body").inner_text(timeout=2000)
|
||||
print(f" text={re.sub(r'\\s+', ' ', text)[:300]}")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
handles = ctx.locator("input").all()
|
||||
for handle in handles[:12]:
|
||||
try:
|
||||
placeholder = handle.get_attribute("placeholder", timeout=1000)
|
||||
input_type = handle.get_attribute("type", timeout=1000)
|
||||
class_name = handle.get_attribute("class", timeout=1000)
|
||||
print(
|
||||
f" input type={input_type} placeholder={placeholder} "
|
||||
f"class={class_name}"
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
self._snapshot("07_link_product_no_search")
|
||||
|
||||
def _click_by_text(self, text: str, timeout: int = 10_000):
|
||||
"""根据文本点击元素,支持逗号分隔多个候选文本。优先在 pub_frame(iframe)里找,fallback 到主页面。"""
|
||||
candidates = [t.strip() for t in text.split(",")]
|
||||
@@ -3255,8 +3334,10 @@ class GuangHeUploader:
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="将飞书 Base 视频上传到淘宝光合平台")
|
||||
runtime_store = os.getenv("GYXX_VIDEO_UPLOAD_STORE", "").strip() or STORE_FLAGSHIP
|
||||
if runtime_store not in {STORE_FLAGSHIP, STORE_LUGGAGE, STORE_ALL}:
|
||||
raise ValueError("GYXX_VIDEO_UPLOAD_STORE 仅支持 flagship、luggage 或 all")
|
||||
if runtime_store not in {STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG, STORE_ALL}:
|
||||
raise ValueError(
|
||||
"GYXX_VIDEO_UPLOAD_STORE 仅支持 flagship、luggage、new_bag 或 all"
|
||||
)
|
||||
p.add_argument("--limit", type=int, default=None, help="最多处理多少条记录")
|
||||
p.add_argument("--debug", action="store_true", help="启用调试截图")
|
||||
mode = p.add_mutually_exclusive_group()
|
||||
@@ -3305,9 +3386,12 @@ def parse_args() -> argparse.Namespace:
|
||||
)
|
||||
p.add_argument(
|
||||
"--store",
|
||||
choices=[STORE_FLAGSHIP, STORE_LUGGAGE, STORE_ALL],
|
||||
choices=[STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG, STORE_ALL],
|
||||
default=runtime_store,
|
||||
help="上传店铺:flagship=旗舰店,luggage=箱包店,all=顺序运行两家",
|
||||
help=(
|
||||
"上传店铺:flagship=旗舰店,luggage=箱包店,"
|
||||
"new_bag=天猫新店(只发黑曜石系列),all=顺序运行三家"
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--topic-keyword",
|
||||
@@ -3354,7 +3438,7 @@ def select_pending_records(
|
||||
republish_from_date: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply the explicit brand/gender/status route for one Guanghe store."""
|
||||
if store not in {STORE_FLAGSHIP, STORE_LUGGAGE}:
|
||||
if store not in {STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG}:
|
||||
raise ValueError(f"未知光合店铺: {store}")
|
||||
if store == STORE_LUGGAGE and not luggage_style_whitelist:
|
||||
raise ValueError("箱包店款式白名单未加载或为空")
|
||||
@@ -3363,6 +3447,7 @@ def select_pending_records(
|
||||
normalized_whitelist = {
|
||||
_normalize_style_name(name) for name in (luggage_style_whitelist or set())
|
||||
}
|
||||
style_keyword = _normalize_style_name(STORE_STYLE_KEYWORDS.get(store, ""))
|
||||
|
||||
result = []
|
||||
for rec in records:
|
||||
@@ -3390,6 +3475,23 @@ def select_pending_records(
|
||||
or not normalized_products.issubset(normalized_whitelist)
|
||||
):
|
||||
continue
|
||||
if store == STORE_NEW_BAG:
|
||||
# 天猫新店按系列关键词过滤:全部款式都必须属于黑曜石系列,
|
||||
# 组合款里只要掺了别的系列整条记录都不进新店。
|
||||
product_names = GuangHeUploader._parse_product_names(
|
||||
rec.get("代拍产品")
|
||||
)
|
||||
if "__SKIP__" in product_names:
|
||||
continue
|
||||
normalized_products = [
|
||||
_normalize_style_name(name)
|
||||
for name in product_names
|
||||
if name and name != "__SKIP__"
|
||||
]
|
||||
if not normalized_products or not all(
|
||||
style_keyword in name for name in normalized_products
|
||||
):
|
||||
continue
|
||||
if store == STORE_FLAGSHIP:
|
||||
if republish_from_date:
|
||||
record_date = str(rec.get("日期") or "")[:10]
|
||||
@@ -3680,6 +3782,13 @@ def _store_runtime(store: str) -> Tuple[str, str, str, str]:
|
||||
os.getenv("GUANGHE_LUGGAGE_PASSWORD", ""),
|
||||
LUGGAGE_USER_DATA_DIR,
|
||||
)
|
||||
if store == STORE_NEW_BAG:
|
||||
return (
|
||||
"gyxx箱包旗舰店:龙虾仔",
|
||||
os.getenv("GUANGHE_NEW_BAG_USERNAME", ""),
|
||||
os.getenv("GUANGHE_NEW_BAG_PASSWORD", ""),
|
||||
NEW_BAG_USER_DATA_DIR,
|
||||
)
|
||||
raise ValueError(f"未知光合店铺: {store}")
|
||||
|
||||
|
||||
@@ -3870,7 +3979,7 @@ def _run_store(
|
||||
print(f"[OK] {store_name} 没有可入账的视频附件")
|
||||
return EXIT_OK
|
||||
if not username or not password:
|
||||
prefix = "GUANGHE" if store == STORE_FLAGSHIP else "GUANGHE_LUGGAGE"
|
||||
prefix = STORE_CREDENTIAL_PREFIXES[store]
|
||||
raise RuntimeError(f"请在运行时配置 {prefix}_USERNAME / {prefix}_PASSWORD")
|
||||
if not direct_llm_configured():
|
||||
raise RuntimeError(
|
||||
@@ -4186,7 +4295,7 @@ def main() -> int:
|
||||
print(f"[OK] 读取到 {len(records)} 条记录")
|
||||
|
||||
stores = (
|
||||
[STORE_FLAGSHIP, STORE_LUGGAGE]
|
||||
[STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG]
|
||||
if args.store == STORE_ALL
|
||||
else [args.store]
|
||||
)
|
||||
|
||||
@@ -143,6 +143,7 @@
|
||||
options: [
|
||||
{ value: "flagship", label: "光影行星旗舰店" },
|
||||
{ value: "luggage", label: "光影行星鑫华达专卖店:龙虾仔" },
|
||||
{ value: "new_bag", label: "天猫新店 · gyxx箱包旗舰店:龙虾仔(只发黑曜石系列)" },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<meta name="theme-color" content="#f5f7fb">
|
||||
<title>GYXX Flow · 工作流控制台</title>
|
||||
<meta name="description" content="按业务模块查看、配置和运行 GYXX Flow 工作流。">
|
||||
<link rel="stylesheet" href="/assets/app.css?v=scheduler-control-v1-20260831">
|
||||
<link rel="stylesheet" href="/assets/app.css?v=scheduler-control-v1-20261030">
|
||||
</head>
|
||||
<body>
|
||||
<a class="skip-link" href="#main-content">跳到主要内容</a>
|
||||
@@ -475,6 +475,7 @@
|
||||
<select id="run-store" name="store">
|
||||
<option value="flagship">光影行星旗舰店</option>
|
||||
<option value="luggage">光影行星鑫华达专卖店:龙虾仔</option>
|
||||
<option value="new_bag">天猫新店 · gyxx箱包旗舰店:龙虾仔(只发黑曜石系列)</option>
|
||||
</select>
|
||||
<small id="run-store-hint">先扫描飞书并按所选店铺规则筛选;存在未上传附件时才登录该店铺账号。</small>
|
||||
</label>
|
||||
@@ -636,6 +637,6 @@
|
||||
|
||||
<div class="toast-region" id="toast-region" aria-live="polite" aria-atomic="false"></div>
|
||||
|
||||
<script src="/assets/app.js?v=scheduler-control-v1-20260831" defer></script>
|
||||
<script src="/assets/app.js?v=scheduler-control-v1-20261030" defer></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -426,27 +426,31 @@ def test_platform_artifact_verifier_rejects_an_export_with_no_product_matches(
|
||||
daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns)
|
||||
|
||||
|
||||
def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
|
||||
def test_daily_collect_no_longer_runs_review_import(monkeypatch) -> None:
|
||||
"""product_reviews 已下线 (2026-11-01)。
|
||||
|
||||
collect 阶段不得再调评论入库——差评由 shop.jd/dy/tmall_negative_reviews.daily
|
||||
三个独立工作流采集,消费方读 v_negative_reviews。这条断言守住下线状态,
|
||||
防止有人顺手把旧链路加回来。
|
||||
"""
|
||||
|
||||
args = SimpleNamespace(
|
||||
skip_erp=True,
|
||||
skip_platforms=False,
|
||||
serial_platforms=False,
|
||||
dry_run=False,
|
||||
skip_reviews_import=False,
|
||||
)
|
||||
monkeypatch.setattr(daily, "run_platforms", lambda **_kwargs: [])
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 1},
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0},
|
||||
)
|
||||
|
||||
daily.run_collect(args, "2026-08-03", ["jd"], failed=None)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0][0] == "import_product_reviews"
|
||||
assert calls[0][1]["critical"] is False
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_daily_import_stage_uses_the_collected_business_date(monkeypatch) -> None:
|
||||
|
||||
@@ -310,6 +310,14 @@ def test_database_upsert_uses_date_and_style_conflict_key(monkeypatch) -> None:
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
# db.attach_style_ids 会按款名查 dim_style;这里返回空映射,
|
||||
# 断言的是 SQL 形状与业务列,不依赖真实款式表。
|
||||
def execute(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
return []
|
||||
|
||||
class Connection:
|
||||
def cursor(self):
|
||||
return Cursor()
|
||||
|
||||
@@ -15,8 +15,8 @@ class ErpMetricOverridesTests(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
self.temp_dir.cleanup()
|
||||
|
||||
def _write_erp(self, code_results):
|
||||
data_dir = self.data_root / "dy" / "apollo_2026-07-22"
|
||||
def _write_erp(self, code_results, platform="dy"):
|
||||
data_dir = self.data_root / platform / "apollo_2026-07-22"
|
||||
data_dir.mkdir(parents=True)
|
||||
(data_dir / "data.json").write_text(
|
||||
json.dumps(
|
||||
@@ -33,12 +33,12 @@ class ErpMetricOverridesTests(unittest.TestCase):
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _apply(self, platform_sales, platform_refunds):
|
||||
def _apply(self, platform_sales, platform_refunds, platform="dy"):
|
||||
with patch.object(overrides, "DATA_ROOT", self.data_root), patch.dict(
|
||||
overrides.PLATFORM_DIRS, {"dy": "dy"}
|
||||
overrides.PLATFORM_DIRS, {platform: platform}
|
||||
):
|
||||
return overrides.apply_erp_override(
|
||||
"dy", "阿波罗x1", 394, 46,
|
||||
platform, "阿波罗x1", 394, 46,
|
||||
target_date="2026-07-22",
|
||||
platform_sales=platform_sales,
|
||||
platform_refunds=platform_refunds,
|
||||
@@ -87,6 +87,21 @@ class ErpMetricOverridesTests(unittest.TestCase):
|
||||
self.assertEqual(result["销量"], 0)
|
||||
self.assertEqual(result["退款订单数"], 0)
|
||||
|
||||
def test_partial_codes_used_when_platform_has_no_sales(self):
|
||||
"""TM 没有平台销量字段:部分编码失败时不能整款归零,用成功编码合计。"""
|
||||
self._write_erp([
|
||||
{"erp_style_code": "1", "ok": True, "yesterday_sales": 3, "yesterday_returns": 1},
|
||||
{"erp_style_code": "2", "ok": False, "error": "timeout", "yesterday_sales": 0, "yesterday_returns": 0},
|
||||
], platform="tm")
|
||||
|
||||
result = self._apply(platform_sales=None, platform_refunds=None, platform="tm")
|
||||
|
||||
self.assertTrue(result["erp_unavailable"])
|
||||
self.assertTrue(result["erp_partial"])
|
||||
self.assertEqual(result["销量"], 3)
|
||||
self.assertEqual(result["退款订单数"], 1)
|
||||
self.assertEqual(result["failed_codes"], ["2"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -3,6 +3,7 @@ import unittest
|
||||
from upload_video_to_guanghe import (
|
||||
STORE_FLAGSHIP,
|
||||
STORE_LUGGAGE,
|
||||
STORE_NEW_BAG,
|
||||
TM_UPLOAD_FIELD,
|
||||
select_pending_records,
|
||||
)
|
||||
@@ -113,6 +114,53 @@ class GuangHeStoreRoutingTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "未知光合店铺"):
|
||||
select_pending_records([record()], "unknown")
|
||||
|
||||
def test_new_bag_uploads_only_obsidian_series_regardless_of_tm_status(self):
|
||||
unchecked = record(gender="女", uploaded=False)
|
||||
unchecked["代拍产品"] = "黑曜石托特包"
|
||||
checked = record(gender="女", uploaded=True)
|
||||
checked["代拍产品"] = "黑曜石吐司包"
|
||||
records = [
|
||||
unchecked,
|
||||
checked,
|
||||
record(gender="男"),
|
||||
]
|
||||
|
||||
selected = select_pending_records(records, STORE_NEW_BAG)
|
||||
|
||||
self.assertEqual(selected, [unchecked, checked])
|
||||
|
||||
def test_new_bag_skips_mixed_and_non_obsidian_styles(self):
|
||||
obsidian = record()
|
||||
obsidian["代拍产品"] = " 黑曜石电脑包 "
|
||||
mixed = record()
|
||||
mixed["代拍产品"] = "黑曜石饺子 / 极星双肩"
|
||||
other = record()
|
||||
other["代拍产品"] = "极星双肩"
|
||||
skipped = record()
|
||||
skipped["代拍产品"] = "黑曜石饺子 / 未到货"
|
||||
no_product = record()
|
||||
other_brand = record(brand="OZKO")
|
||||
other_brand["代拍产品"] = "黑曜石托特包"
|
||||
no_attachment = record(attachments=False)
|
||||
no_attachment["代拍产品"] = "黑曜石托特包"
|
||||
|
||||
selected = select_pending_records(
|
||||
[obsidian, mixed, other, skipped, no_product, other_brand, no_attachment],
|
||||
STORE_NEW_BAG,
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [obsidian])
|
||||
|
||||
def test_new_bag_rejects_republish_from_date(self):
|
||||
rec = record(date="2026-06-03 00:00:00")
|
||||
rec["代拍产品"] = "黑曜石托特包"
|
||||
with self.assertRaisesRegex(ValueError, "旗舰店"):
|
||||
select_pending_records(
|
||||
[rec],
|
||||
STORE_NEW_BAG,
|
||||
republish_from_date="2026-06-03",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -119,6 +119,9 @@ def test_upsert_uses_advertiser_date_product_business_key(monkeypatch) -> None:
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
# resolve_style_ids 走独立查询;本测试只验证 SQL 形状与款式匹配列,
|
||||
# 用空映射避免和上面的 style catalog 假数据串台。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
@@ -162,6 +165,7 @@ def test_upsert_falls_back_to_exact_product_name_style_match(monkeypatch) -> Non
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
|
||||
@@ -99,7 +99,7 @@ def test_find_report_files_uses_current_manifest_csv(monkeypatch, tmp_path: Path
|
||||
assert importer.find_report_files(target_date) == [current_csv.resolve()]
|
||||
|
||||
|
||||
def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
def test_upsert_writes_dedicated_table_with_style_and_metrics(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows, *, template=None):
|
||||
@@ -108,7 +108,9 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
captured["template"] = template
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
written = product_db.upsert_tmall_product_ad_daily_metrics(
|
||||
_Connection(),
|
||||
[
|
||||
{
|
||||
@@ -119,6 +121,7 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
"spend": "90.25",
|
||||
"attributed_deal_amount": "730.02",
|
||||
"roi": "8.09",
|
||||
"impressions": 1000,
|
||||
},
|
||||
"source_file": "商品报表.csv",
|
||||
}
|
||||
@@ -127,16 +130,22 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
|
||||
assert written == 1
|
||||
sql = str(captured["sql"])
|
||||
assert "INSERT INTO product_daily_metrics" in sql
|
||||
assert "jsonb_set" in sql
|
||||
assert "tm_product_ad_daily_metrics" not in sql
|
||||
assert captured["template"] == "('tm', %s, %s, %s, %s::jsonb, %s, NOW())"
|
||||
assert "INSERT INTO tm_product_ad_daily_metrics" in sql
|
||||
assert "product_daily_metrics" not in sql
|
||||
assert "jsonb_set" not in sql
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["spend"] == "90.25"
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_id"
|
||||
assert row[1] == "1001"
|
||||
assert row[3] == "极星"
|
||||
assert row[4] == "product_id"
|
||||
assert str(row[5]) == "90.25"
|
||||
assert str(row[8]) == "730.02"
|
||||
assert str(row[9]) == "8.09"
|
||||
# 15 个业务列 + style_id(款式主键统一 2026-11-01)
|
||||
assert captured["template"].count("%s") == 16
|
||||
assert row[-1] is None # resolve_style_ids 被 mock 成空映射
|
||||
metrics = json.loads(row[11])
|
||||
assert metrics["impressions"] == 1000
|
||||
assert "style_name" not in metrics
|
||||
|
||||
|
||||
def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> None:
|
||||
@@ -154,7 +163,9 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) ->
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
written = product_db.upsert_tmall_product_ad_daily_metrics(
|
||||
_NameFallbackConnection(),
|
||||
[
|
||||
{
|
||||
@@ -169,9 +180,41 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) ->
|
||||
|
||||
assert written == 1
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_name"
|
||||
assert row[3] == "极星"
|
||||
assert row[4] == "product_name"
|
||||
|
||||
|
||||
def test_upsert_dedupes_repeated_product_day(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, _sql, rows, *, template=None):
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
written = product_db.upsert_tmall_product_ad_daily_metrics(
|
||||
_Connection(),
|
||||
[
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "older",
|
||||
"ad_metrics": {"spend": "1.00"},
|
||||
},
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "latest",
|
||||
"ad_metrics": {"spend": "2.00"},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
row = captured["rows"][0]
|
||||
assert row[2] == "latest"
|
||||
assert str(row[5]) == "2.00"
|
||||
|
||||
|
||||
def test_import_date_filter_and_dry_run(tmp_path: Path, capsys) -> None:
|
||||
|
||||
@@ -312,7 +312,9 @@ class MainImageDatabaseTests(unittest.TestCase):
|
||||
}
|
||||
for platform in ("jd", "tm")
|
||||
]
|
||||
with patch.object(product_db, "execute_values") as execute_values:
|
||||
with patch.object(product_db, "execute_values") as execute_values, patch.object(
|
||||
product_db, "resolve_style_ids", return_value={}
|
||||
):
|
||||
count = product_db.upsert_main_image_creatives(Connection(), records)
|
||||
|
||||
self.assertEqual(count, 2)
|
||||
|
||||
@@ -12,6 +12,13 @@ class _Cursor:
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
# db 层按商品 ID 反查 dim_style 时用;空映射表示这些商品都没建档。
|
||||
def execute(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
return []
|
||||
|
||||
|
||||
class _Connection:
|
||||
def cursor(self):
|
||||
|
||||
@@ -11,7 +11,6 @@ import collect_persona_to_bitable as tm_persona_worker
|
||||
import dy_product_scraping as dy_product
|
||||
import orchestrate_daily_collection as daily
|
||||
import orchestrate_market_rank_collection as market_rank
|
||||
import orchestrate_review_collection as reviews
|
||||
import run_alerts_with_retry as alerts
|
||||
import run_daily_persona as persona
|
||||
import run_weekly_jd_main_image as jd_main_image
|
||||
@@ -191,24 +190,6 @@ def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None:
|
||||
_assert_child_binding(environment)
|
||||
|
||||
|
||||
def test_review_run_step_rebinds_to_command_target(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(reviews, "environment_for_child_script", _fake_rebind(observed))
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
observed["environment"] = kwargs["env"]
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(reviews.subprocess, "run", fake_run)
|
||||
target = reviews.PROJECT_ROOT / "dy_product_scraping.py"
|
||||
|
||||
result = reviews.run_step("dy", ["python", str(target)], critical=True)
|
||||
|
||||
assert result["code"] == 0
|
||||
assert observed["target"] == target
|
||||
_assert_child_binding(observed["environment"])
|
||||
|
||||
|
||||
def test_main_image_launchers_rebind_to_command_target(monkeypatch) -> None:
|
||||
for launcher in (tm_main_image, jd_main_image):
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
@@ -116,7 +116,8 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
self.assertNotIn("近 60d", prompt)
|
||||
self.assertIn("逐日经营数据", prompt)
|
||||
self.assertIn("人物画像", prompt)
|
||||
self.assertIn("商品评价", prompt)
|
||||
# product_reviews 下线后数据源改为差评视图,标题同步改成「商品差评」
|
||||
self.assertIn("商品差评", prompt)
|
||||
self.assertIn("主图表现", prompt)
|
||||
self.assertIn("营销笔记、曝光互动与评论", prompt)
|
||||
self.assertIn("肩带偏硬", prompt)
|
||||
|
||||
@@ -73,6 +73,9 @@ def test_database_row_uses_explicit_style_name_not_style_content() -> None:
|
||||
"series_name": "",
|
||||
"is_series_anchor": False,
|
||||
"collect_enabled": True,
|
||||
# 款式主键统一 (2026-11-01): loader 把 dim_style.id 作为正式字段下发
|
||||
"style_id": None,
|
||||
"style_match_method": "",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
"""款式主键统一 (style_id = dim_style.id) 的契约测试。
|
||||
|
||||
守住四件事:
|
||||
1. 写入端必须把 style_id 落库(13 张采集表);
|
||||
2. 调用方已带 style_id 时不覆盖,缺省时由 db 层按款名 / 平台商品 ID 兜底;
|
||||
3. 匹配不上要留痕 ``unmatched``,不能静默写成 NULL 了事;
|
||||
4. schema.sql 里必须有列、索引、观测视图,且**不得**建物理外键(style_id 只做逻辑外键),
|
||||
同时不再建已下线的 product_reviews。
|
||||
|
||||
全部测试离线运行(Fake connection),不依赖 PostgreSQL。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import db
|
||||
import pytest
|
||||
|
||||
|
||||
MODULE_ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
)
|
||||
SCHEMA_PATH = MODULE_ROOT / "db" / "schema.sql"
|
||||
|
||||
|
||||
class RecordingCursor:
|
||||
def __init__(self, conn: "RecordingConnection") -> None:
|
||||
self._conn = conn
|
||||
self._rows: list[tuple] = []
|
||||
|
||||
def __enter__(self) -> "RecordingCursor":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> bool:
|
||||
return False
|
||||
|
||||
def execute(self, sql: str, params=None) -> None:
|
||||
self._conn.sqls.append((sql, params))
|
||||
if "unnest" in sql and params:
|
||||
self._rows = [
|
||||
(item, self._conn.product_lookup[item])
|
||||
for item in params[0]
|
||||
if item in self._conn.product_lookup
|
||||
]
|
||||
elif "FROM dim_style" in sql and "ANY" in sql and params:
|
||||
self._rows = [
|
||||
(name, self._conn.name_lookup[name])
|
||||
for name in params[0]
|
||||
if name in self._conn.name_lookup
|
||||
]
|
||||
else:
|
||||
self._rows = []
|
||||
|
||||
def fetchall(self) -> list[tuple]:
|
||||
return self._rows
|
||||
|
||||
def fetchone(self):
|
||||
return None
|
||||
|
||||
@property
|
||||
def rowcount(self) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
class RecordingConnection:
|
||||
def __init__(self, name_lookup=None, product_lookup=None) -> None:
|
||||
self.name_lookup = dict(name_lookup or {})
|
||||
self.product_lookup = dict(product_lookup or {})
|
||||
self.sqls: list[tuple] = []
|
||||
|
||||
def cursor(self) -> RecordingCursor:
|
||||
return RecordingCursor(self)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def captured(monkeypatch):
|
||||
"""拦截 execute_values,暴露最终写入的 SQL 与 rows。"""
|
||||
|
||||
box: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows):
|
||||
box["sql"] = sql
|
||||
box["rows"] = list(rows)
|
||||
|
||||
monkeypatch.setattr(db, "execute_values", fake_execute_values)
|
||||
return box
|
||||
|
||||
|
||||
def test_attach_style_ids_keeps_supplied_identity() -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
records = db.attach_style_ids(
|
||||
conn,
|
||||
[{"style_name": "盖亚斜挎", "style_id": 42, "style_match_method": "dim_style"}],
|
||||
)
|
||||
|
||||
assert records[0]["style_id"] == 42
|
||||
assert records[0]["style_match_method"] == "dim_style"
|
||||
# 已有 style_id 时不应再查库
|
||||
assert conn.sqls == []
|
||||
|
||||
|
||||
def test_attach_style_ids_fills_missing_and_marks_unmatched() -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
records = db.attach_style_ids(
|
||||
conn,
|
||||
[{"style_name": "盖亚斜挎"}, {"style_name": "未建档款"}],
|
||||
)
|
||||
|
||||
assert records[0]["style_id"] == 7
|
||||
assert records[0]["style_match_method"] == db.STYLE_MATCH_BY_NAME
|
||||
assert records[1]["style_id"] is None
|
||||
assert records[1]["style_match_method"] == db.STYLE_MATCH_UNMATCHED
|
||||
|
||||
|
||||
def test_attach_style_ids_does_not_mutate_caller_records() -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
original = [{"style_name": "盖亚斜挎"}]
|
||||
|
||||
db.attach_style_ids(conn, original)
|
||||
|
||||
assert "style_id" not in original[0]
|
||||
|
||||
|
||||
def test_daily_style_metrics_upsert_writes_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
written = db.upsert_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"metric_date": "2026-10-31",
|
||||
"style_name": "盖亚斜挎",
|
||||
"platform": "tm",
|
||||
"sales": 3,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
sql = captured["sql"]
|
||||
assert "style_id" in sql
|
||||
assert "style_match_method" in sql
|
||||
# 冲突时不能把已填的 id 抹成 NULL
|
||||
assert "COALESCE(EXCLUDED.style_id" in sql
|
||||
row = captured["rows"][0]
|
||||
assert row[-2] == 7
|
||||
assert row[-1] == db.STYLE_MATCH_BY_NAME
|
||||
|
||||
|
||||
def test_erp_all_shop_upsert_writes_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"极星双肩": 11})
|
||||
|
||||
db.upsert_erp_all_shop_style_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"metric_date": "2026-10-31",
|
||||
"style_name": "极星双肩",
|
||||
"sales": 10,
|
||||
"selected_shop_count": 2,
|
||||
"source": "jushuitan",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
row = captured["rows"][0]
|
||||
assert row[-2] == 11
|
||||
assert row[-1] == db.STYLE_MATCH_BY_NAME
|
||||
|
||||
|
||||
def test_weekly_metrics_upsert_writes_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"极星双肩": 11})
|
||||
|
||||
db.upsert_weekly_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"week_end": "2026-10-31",
|
||||
"week_start": "2026-10-25",
|
||||
"style_name": "极星双肩",
|
||||
"platform": "tm",
|
||||
"sales": 5,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert "style_id" in captured["sql"]
|
||||
assert captured["rows"][0][-2] == 11
|
||||
|
||||
|
||||
def test_product_daily_metrics_resolves_by_platform_product_id(captured, monkeypatch) -> None:
|
||||
conn = RecordingConnection(product_lookup={"700001": 33})
|
||||
|
||||
written = db.upsert_product_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"platform": "tm",
|
||||
"stat_date": "2026-10-31",
|
||||
"product_id": "700001",
|
||||
"product_name": "GYXX 极星双肩包",
|
||||
"paid_amount": 100,
|
||||
},
|
||||
{
|
||||
"platform": "tm",
|
||||
"stat_date": "2026-10-31",
|
||||
"product_id": "999999",
|
||||
"product_name": "未建档商品",
|
||||
"paid_amount": 5,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 2
|
||||
rows = captured["rows"]
|
||||
assert rows[0][-2] == 33
|
||||
assert rows[0][-1] == db.STYLE_MATCH_BY_PRODUCT
|
||||
assert rows[1][-2] is None
|
||||
assert rows[1][-1] == db.STYLE_MATCH_UNMATCHED
|
||||
|
||||
|
||||
def test_product_daily_metrics_keeps_existing_identity_on_conflict(captured) -> None:
|
||||
conn = RecordingConnection(product_lookup={})
|
||||
|
||||
db.upsert_product_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"platform": "jd",
|
||||
"stat_date": "2026-10-31",
|
||||
"product_id": "abc",
|
||||
"paid_amount": 1,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
sql = captured["sql"]
|
||||
assert "COALESCE(EXCLUDED.style_id, product_daily_metrics.style_id)" in sql
|
||||
|
||||
|
||||
def test_ad_tables_write_style_id(captured) -> None:
|
||||
conn = RecordingConnection(
|
||||
name_lookup={"盖亚斜挎": 7},
|
||||
product_lookup={"5001": 7},
|
||||
)
|
||||
|
||||
db.upsert_jd_product_ad_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"stat_date": "2026-10-31",
|
||||
"advertiser_pin": "pin",
|
||||
"shop_account": "shop",
|
||||
"report_type": "spu",
|
||||
"row_key": "r1",
|
||||
"style_name": "盖亚斜挎",
|
||||
"spend": 12,
|
||||
}
|
||||
],
|
||||
)
|
||||
assert "style_id" in captured["sql"]
|
||||
assert captured["rows"][0][-2] == 7
|
||||
|
||||
db.upsert_douyin_product_ad_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"stat_date": "2026-10-31",
|
||||
"advertiser_id": "adv",
|
||||
"product_id": "5001",
|
||||
"product_name": "盖亚斜挎包",
|
||||
"style_name": "盖亚斜挎",
|
||||
"style_match_method": "product_id",
|
||||
"spend": 8,
|
||||
}
|
||||
],
|
||||
)
|
||||
assert "style_id" in captured["sql"]
|
||||
assert captured["rows"][0][-1] == 7
|
||||
|
||||
|
||||
def test_persona_and_main_image_write_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
db.upsert_persona_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"collect_date": "2026-10-31",
|
||||
"style_name": "盖亚斜挎",
|
||||
"platform": "tm",
|
||||
"payload": {},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert captured["rows"][0][-2] == 7
|
||||
|
||||
db.upsert_main_image_creatives(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"collect_date": "2026-10-31",
|
||||
"style_name": "盖亚斜挎",
|
||||
"image_key": "k1",
|
||||
}
|
||||
],
|
||||
)
|
||||
assert captured["rows"][0][-2] == 7
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"table",
|
||||
[
|
||||
"daily_style_metrics",
|
||||
"weekly_style_metrics",
|
||||
"erp_all_shop_style_daily_metrics",
|
||||
"daily_persona_metrics",
|
||||
"main_image_creatives",
|
||||
"fact_style_analysis_reports",
|
||||
"decline_events",
|
||||
"fact_alert_history",
|
||||
"jd_self_weekly_sales",
|
||||
"persona_screenshots",
|
||||
"product_daily_metrics",
|
||||
"jd_product_ad_daily_metrics",
|
||||
"dy_product_ad_daily_metrics",
|
||||
],
|
||||
)
|
||||
def test_schema_adds_style_id_column(table: str) -> None:
|
||||
text = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
assert f"ALTER TABLE {table}" in text
|
||||
assert f"ADD COLUMN IF NOT EXISTS style_id BIGINT" in text
|
||||
|
||||
|
||||
def test_schema_defines_observation_views_and_no_physical_foreign_keys() -> None:
|
||||
text = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
|
||||
assert "CREATE OR REPLACE VIEW v_style_unmatched_product" in text
|
||||
assert "CREATE OR REPLACE VIEW v_style_id_coverage" in text
|
||||
assert "CREATE OR REPLACE VIEW v_negative_reviews" in text
|
||||
|
||||
# style_id 只做逻辑外键:库里不得出现指向 dim_style 的物理外键。
|
||||
# 采集语义是「宁可留 NULL 待回填,也不要整批写入失败」,硬约束与之冲突。
|
||||
assert "FOREIGN KEY (style_id) REFERENCES dim_style(id)" not in text
|
||||
assert "REFERENCES dim_style(id)" not in text
|
||||
# 老库重跑 schema 时要能自己清掉早期建的约束(幂等 DROP)
|
||||
assert "DROP CONSTRAINT IF EXISTS" in text
|
||||
assert "_style_id_fk" in text
|
||||
|
||||
|
||||
def test_schema_no_longer_creates_product_reviews() -> None:
|
||||
text = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
|
||||
assert "CREATE TABLE IF NOT EXISTS product_reviews " not in text
|
||||
assert "ALTER TABLE product_reviews " not in text
|
||||
assert "product_reviews_archived_20261101" in text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 回填脚本
|
||||
|
||||
def test_backfill_covers_three_ad_tables_with_right_id_column() -> None:
|
||||
"""广告表的 style_name 大量为空,必须按商品 ID 再回填一遍。"""
|
||||
|
||||
import backfill_style_id as bf
|
||||
|
||||
assert set(bf.AD_PRODUCT_TABLES) == {
|
||||
"tm_product_ad_daily_metrics",
|
||||
"jd_product_ad_daily_metrics",
|
||||
"dy_product_ad_daily_metrics",
|
||||
}
|
||||
assert bf.AD_PRODUCT_TABLES["tm_product_ad_daily_metrics"] == ("product_id", "tm")
|
||||
# 京东广告表的商品列叫 spu_id,不是 product_id
|
||||
assert bf.AD_PRODUCT_TABLES["jd_product_ad_daily_metrics"] == ("spu_id", "jd")
|
||||
assert bf.AD_PRODUCT_TABLES["dy_product_ad_daily_metrics"] == ("product_id", "dy")
|
||||
|
||||
# 款名表里的广告表项不能删,否则 style_name 有值的那部分就回填不了
|
||||
for table in bf.AD_PRODUCT_TABLES:
|
||||
assert table in bf.NAME_TABLES
|
||||
|
||||
|
||||
class _BackfillCursor:
|
||||
def __init__(self, log: list) -> None:
|
||||
self._log = log
|
||||
self._rows: list[tuple] = []
|
||||
self.rowcount = 0
|
||||
|
||||
def __enter__(self) -> "_BackfillCursor":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> bool:
|
||||
return False
|
||||
|
||||
def execute(self, sql: str, params=None) -> None:
|
||||
self._log.append((" ".join(sql.split()), params))
|
||||
if "SELECT DISTINCT" in sql:
|
||||
self._rows = [("1001",), ("9999",)]
|
||||
elif "SELECT id, style_name FROM dim_style" in sql:
|
||||
self._rows = [(7, "赤潮")]
|
||||
elif sql.lstrip().startswith("UPDATE"):
|
||||
self.rowcount = 5 if "style_id IS NULL AND style_match_method IS NULL" not in sql else 0
|
||||
elif "COUNT(*)" in sql:
|
||||
self._rows = [(10, 2)]
|
||||
else:
|
||||
self._rows = []
|
||||
|
||||
def fetchall(self) -> list[tuple]:
|
||||
return self._rows
|
||||
|
||||
def fetchone(self) -> tuple:
|
||||
return self._rows[0] if self._rows else (0, 0)
|
||||
|
||||
|
||||
class _BackfillConn:
|
||||
def __init__(self) -> None:
|
||||
self.log: list = []
|
||||
|
||||
def cursor(self) -> _BackfillCursor:
|
||||
return _BackfillCursor(self.log)
|
||||
|
||||
|
||||
def test_backfill_ad_tables_fills_style_id_and_keeps_platform_style_name(monkeypatch) -> None:
|
||||
"""回填广告表时:style_id 照写,style_name 只在原值为空时补。"""
|
||||
|
||||
import backfill_style_id as bf
|
||||
|
||||
monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7})
|
||||
conn = _BackfillConn()
|
||||
result = bf._backfill_ad_tables_by_product(
|
||||
conn, ["jd_product_ad_daily_metrics"], dry_run=False
|
||||
)[0]
|
||||
|
||||
updates = [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")]
|
||||
payload = next(sql for sql in updates if "NULLIF(BTRIM(style_name)" in sql)
|
||||
assert "COALESCE(NULLIF(BTRIM(style_name), ''), %s)" in payload, (
|
||||
"style_name 只能补空,不能覆盖平台已给出的值"
|
||||
)
|
||||
assert "AND style_id IS NULL" in payload, "幂等:已归属的行不能被改写"
|
||||
assert "spu_id = %s" in payload, "京东广告表要按 spu_id 反查"
|
||||
|
||||
# 匹配上的 1 个 ID 写 style_id,1 个未建档
|
||||
assert result["updated"] == 5
|
||||
assert result["unmatched_product_ids"] == 1
|
||||
|
||||
# 未建档的显式标 unmatched,供观测视图统计
|
||||
assert any(
|
||||
params and params[0] == "unmatched"
|
||||
for sql, params in conn.log
|
||||
if sql.lstrip().startswith("UPDATE")
|
||||
)
|
||||
|
||||
|
||||
def test_backfill_ad_tables_dry_run_does_not_update(monkeypatch) -> None:
|
||||
import backfill_style_id as bf
|
||||
|
||||
monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7})
|
||||
conn = _BackfillConn()
|
||||
result = bf._backfill_ad_tables_by_product(
|
||||
conn, ["dy_product_ad_daily_metrics"], dry_run=True
|
||||
)[0]
|
||||
|
||||
assert result["dry_run"] is True
|
||||
assert result["planned_keys"] == 1
|
||||
assert not [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")]
|
||||
@@ -176,13 +176,22 @@ def test_discovery_creates_one_candidate_per_exact_video_attachment() -> None:
|
||||
def test_flagship_and_luggage_use_distinct_account_scopes() -> None:
|
||||
flagship = _one_candidate(store=tmall.STORE_FLAGSHIP)
|
||||
luggage = _one_candidate(store=tmall.STORE_LUGGAGE)
|
||||
new_bag = _one_candidate(store=tmall.STORE_NEW_BAG)
|
||||
|
||||
assert flagship.target_account_key == "guanghe-flagship"
|
||||
assert luggage.target_account_key == "guanghe-luggage"
|
||||
assert flagship.target_account_key != luggage.target_account_key
|
||||
assert new_bag.target_account_key == "guanghe-new-bag"
|
||||
assert len(
|
||||
{
|
||||
flagship.target_account_key,
|
||||
luggage.target_account_key,
|
||||
new_bag.target_account_key,
|
||||
}
|
||||
) == 3
|
||||
assert tmall.STORE_ACCOUNT_KEYS == {
|
||||
tmall.STORE_FLAGSHIP: flagship.target_account_key,
|
||||
tmall.STORE_LUGGAGE: luggage.target_account_key,
|
||||
tmall.STORE_NEW_BAG: new_bag.target_account_key,
|
||||
}
|
||||
|
||||
|
||||
@@ -199,6 +208,25 @@ def test_luggage_runtime_uses_new_store_account_label_without_changing_password_
|
||||
assert password == "same-runtime-secret"
|
||||
|
||||
|
||||
def test_new_bag_runtime_uses_its_own_credentials_and_profile(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GUANGHE_NEW_BAG_USERNAME", "gyxx箱包旗舰店:龙虾仔")
|
||||
monkeypatch.setenv("GUANGHE_NEW_BAG_PASSWORD", "new-bag-runtime-secret")
|
||||
|
||||
store_name, username, password, profile = tmall._store_runtime(tmall.STORE_NEW_BAG)
|
||||
|
||||
assert store_name == "gyxx箱包旗舰店:龙虾仔"
|
||||
assert username == "gyxx箱包旗舰店:龙虾仔"
|
||||
assert password == "new-bag-runtime-secret"
|
||||
assert str(profile).endswith("guanghe-new-bag")
|
||||
assert profile != tmall._store_runtime(tmall.STORE_LUGGAGE)[3]
|
||||
assert profile != tmall._store_runtime(tmall.STORE_FLAGSHIP)[3]
|
||||
assert tmall._expected_creator_identity_markers(
|
||||
tmall.STORE_NEW_BAG, store_name
|
||||
) == ("GYXX箱包天猫官方",)
|
||||
|
||||
|
||||
def test_account_identity_markers_accept_store_name_with_login_nickname() -> None:
|
||||
assert tmall.GuangHeUploader._account_identity_markers(
|
||||
"光影行星鑫华达专卖店:龙虾仔"
|
||||
|
||||
@@ -148,5 +148,80 @@ class WanxiangReportTemplateTests(unittest.TestCase):
|
||||
session.close.assert_called_once()
|
||||
|
||||
|
||||
class WanxiangSearchKeywordTests(unittest.TestCase):
|
||||
"""搜索词可与款名不同,但款式身份始终是 style_name。"""
|
||||
|
||||
def test_obsidian_toast_style_searches_shorter_keyword_first(self):
|
||||
self.assertEqual(
|
||||
wanxiang.search_keywords_for("黑曜石吐司包"),
|
||||
("黑曜石吐司", "黑曜石吐司包"),
|
||||
)
|
||||
|
||||
def test_styles_without_override_keep_their_own_name(self):
|
||||
self.assertEqual(wanxiang.search_keywords_for("星云2"), ("星云2",))
|
||||
self.assertEqual(
|
||||
wanxiang.search_keywords_for("黑曜石托特包"), ("黑曜石托特包",)
|
||||
)
|
||||
|
||||
def test_collect_style_searches_keyword_but_keeps_style_identity(self):
|
||||
searched: list[str] = []
|
||||
row = {
|
||||
"creative_id": "c1",
|
||||
"impressions": 10,
|
||||
"clicks": 2,
|
||||
"cost": 0.5,
|
||||
"creative_image_url": "https://img/1.jpg",
|
||||
}
|
||||
with (
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"search_style_by_subject",
|
||||
side_effect=lambda _page, keyword: searched.append(keyword),
|
||||
),
|
||||
patch.object(wanxiang, "extract_current_page_rows", return_value=[row]),
|
||||
patch.object(wanxiang, "collect_current_page_image_map", return_value={}),
|
||||
patch.object(wanxiang, "click_next_page", return_value=False),
|
||||
patch.object(wanxiang, "aggregate_rows", return_value={}),
|
||||
):
|
||||
result = wanxiang.collect_style(
|
||||
Mock(), "黑曜石吐司包", 1, item_ids=["1022486093557"]
|
||||
)
|
||||
|
||||
self.assertEqual(searched, ["黑曜石吐司"])
|
||||
self.assertEqual(result["style_name"], "黑曜石吐司包")
|
||||
self.assertEqual(result["search_keyword"], "黑曜石吐司")
|
||||
|
||||
def test_empty_keyword_falls_back_to_original_style_name(self):
|
||||
state = {"searched": [], "checks": 0}
|
||||
row = {
|
||||
"creative_id": "c1",
|
||||
"impressions": 10,
|
||||
"clicks": 2,
|
||||
"cost": 0.5,
|
||||
"creative_image_url": "https://img/1.jpg",
|
||||
}
|
||||
|
||||
def fake_extract(_page):
|
||||
state["checks"] += 1
|
||||
return [] if state["checks"] == 1 else [row]
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"search_style_by_subject",
|
||||
side_effect=lambda _page, keyword: state["searched"].append(keyword),
|
||||
),
|
||||
patch.object(wanxiang, "extract_current_page_rows", side_effect=fake_extract),
|
||||
patch.object(wanxiang, "collect_current_page_image_map", return_value={}),
|
||||
patch.object(wanxiang, "click_next_page", return_value=False),
|
||||
patch.object(wanxiang, "aggregate_rows", return_value={}),
|
||||
):
|
||||
result = wanxiang.collect_style(Mock(), "黑曜石吐司包", 1, item_ids=["1022"])
|
||||
|
||||
self.assertEqual(state["searched"], ["黑曜石吐司", "黑曜石吐司包"])
|
||||
self.assertEqual(result["style_name"], "黑曜石吐司包")
|
||||
self.assertEqual(result["search_keyword"], "黑曜石吐司包")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1709,6 +1709,28 @@ def test_tmall_video_trigger_routes_only_the_selected_store(
|
||||
assert flagship["store"] == "flagship"
|
||||
assert flagship["topic_keyword"] == "我的夏日焕新清单"
|
||||
assert launcher.calls[-1]["store"] == "flagship"
|
||||
launcher.complete("product.video_upload")
|
||||
|
||||
new_bag = service.trigger(
|
||||
"product.video_upload",
|
||||
{"business_date": "2026-08-11", "store": "new_bag"},
|
||||
)
|
||||
|
||||
assert new_bag["store"] == "new_bag"
|
||||
assert launcher.calls[-1]["store"] == "new_bag"
|
||||
|
||||
|
||||
def test_tmall_video_trigger_rejects_unknown_store(console_settings: Settings) -> None:
|
||||
service = WorkflowConsoleService(
|
||||
console_settings,
|
||||
launcher=FakeConsoleLauncher(),
|
||||
)
|
||||
|
||||
with pytest.raises(ConsoleRequestError, match="店铺参数无效"):
|
||||
service.trigger(
|
||||
"product.video_upload",
|
||||
{"business_date": "2026-08-11", "store": "obsidian"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [123, True, "x" * 121])
|
||||
@@ -2824,6 +2846,8 @@ def test_http_home_assets_and_overview_have_security_headers(
|
||||
assert "强制重跑:忽略幂等跳过与已有结果,重新采集并正式执行" in page
|
||||
assert '<option value="flagship">光影行星旗舰店</option>' in page
|
||||
assert '<option value="luggage">光影行星鑫华达专卖店:龙虾仔</option>' in page
|
||||
assert '<option value="new_bag">' in page
|
||||
assert "天猫新店 · gyxx箱包旗舰店:龙虾仔" in page
|
||||
assert 'id="run-tmall-topic-field"' in page
|
||||
assert 'id="run-tmall-topic-keyword"' in page
|
||||
assert "存在未上传附件时才登录该店铺账号" in page
|
||||
@@ -2915,6 +2939,8 @@ def test_http_home_assets_and_overview_have_security_headers(
|
||||
assert "tmall_topic_keyword" in decoded_asset
|
||||
assert "syncTmallTopicField" in decoded_asset
|
||||
assert "光影行星鑫华达专卖店:龙虾仔" in decoded_asset
|
||||
assert "天猫新店" in decoded_asset
|
||||
assert 'value: "new_bag"' in decoded_asset
|
||||
assert "renderCardLiveState" in decoded_asset
|
||||
assert 'data-live-progress="${completed}/${steps.length}"' in decoded_asset
|
||||
assert 'data-current-step="${esc(currentNames.join("、"))}"' in decoded_asset
|
||||
|
||||
@@ -36,6 +36,7 @@ def test_repository_keeps_the_simplified_public_structure() -> None:
|
||||
"note-inventory.md",
|
||||
"plan.md",
|
||||
"runbook.md",
|
||||
"style-identity-refactor-plan.md",
|
||||
"workflow-acceptance-test-report.md",
|
||||
"workflow-acceptance-test-requirements.md",
|
||||
"workflow-console.md",
|
||||
|
||||
@@ -34,7 +34,6 @@ NESTED_TARGETS = {
|
||||
"dy_product_scraping.py",
|
||||
"taobao_sycm_collect.py",
|
||||
"taobao_sycm_collect_backfill.py",
|
||||
"import_product_reviews.py",
|
||||
"aggregate_daily_final.py",
|
||||
"export_bitable_records.py",
|
||||
"insert_bitable_records.py",
|
||||
|
||||
@@ -88,7 +88,6 @@ def test_runtime_writers_do_not_fall_back_to_cwd_or_source_tree(
|
||||
[
|
||||
"collect_jd_market_rank.py",
|
||||
"collect_sycm_market_rank.py",
|
||||
"import_product_reviews.py",
|
||||
"scripts/insert_jd_main_image_records.py",
|
||||
"scripts/insert_main_image_records.py",
|
||||
],
|
||||
|
||||
@@ -29,6 +29,7 @@ PRODUCTION_PYTHON = {
|
||||
"backfill_poseidon_sales.py",
|
||||
"backfill_collect.py",
|
||||
"backfill_one_day.py",
|
||||
"backfill_style_id.py",
|
||||
"check_nine_day_decline.py",
|
||||
"collect_dy_market_rank.py",
|
||||
"collect_dy_persona_to_bitable.py",
|
||||
@@ -48,7 +49,6 @@ PRODUCTION_PYTHON = {
|
||||
"export_bitable_records.py",
|
||||
"feishu_doc_native.py",
|
||||
"import_product_daily.py",
|
||||
"import_product_reviews.py",
|
||||
"insert_bitable_records.py",
|
||||
"jd_main_image_collector.py",
|
||||
"jd_product_data_collector.py",
|
||||
@@ -62,7 +62,6 @@ PRODUCTION_PYTHON = {
|
||||
"market_rank_report_archive.py",
|
||||
"orchestrate_daily_collection.py",
|
||||
"orchestrate_market_rank_collection.py",
|
||||
"orchestrate_review_collection.py",
|
||||
"reapply_erp_override.py",
|
||||
"rebuild_market_rank_documents.py",
|
||||
"run_alerts_with_retry.py",
|
||||
@@ -121,6 +120,7 @@ LEGACY_TESTS = {
|
||||
"test_persona_launcher.py",
|
||||
"test_style_analysis_orchestration.py",
|
||||
"test_style_config_loader_erp_source.py",
|
||||
"test_style_id_identity.py",
|
||||
"test_sycm_market_rank.py",
|
||||
"test_tm_persona_recovery.py",
|
||||
"test_wanxiang_report_template.py",
|
||||
@@ -162,7 +162,6 @@ SUBPROCESS_TARGETS = {
|
||||
"dy_audience_profile_collect.py",
|
||||
"dy_product_scraping.py",
|
||||
"export_bitable_records.py",
|
||||
"import_product_reviews.py",
|
||||
"insert_bitable_records.py",
|
||||
"jd_main_image_collector.py",
|
||||
"jd_product_data_collector.py",
|
||||
@@ -195,7 +194,7 @@ def _load_runtime_paths():
|
||||
|
||||
|
||||
def test_product_runtime_contains_the_complete_reviewed_source_snapshot() -> None:
|
||||
assert len(PRODUCTION_PYTHON) == 63
|
||||
assert len(PRODUCTION_PYTHON) == 62
|
||||
assert all((MODULE_ROOT / relative).is_file() for relative in PRODUCTION_PYTHON)
|
||||
assert all((MODULE_ROOT / relative).is_file() for relative in RESOURCES)
|
||||
assert all((MIGRATED_TEST_ROOT / name).is_file() for name in LEGACY_TESTS)
|
||||
|
||||
@@ -55,8 +55,8 @@ def test_catalog_and_scripts_work_after_project_and_data_roots_move(
|
||||
assert scripts.returncode == 0, scripts.stderr
|
||||
assert len(json.loads(workflows.stdout)) == 40
|
||||
commands = json.loads(scripts.stdout)
|
||||
assert len(commands) == 62
|
||||
assert len({row["command_id"] for row in commands}) == 62
|
||||
assert len(commands) == 61
|
||||
assert len({row["command_id"] for row in commands}) == 61
|
||||
assert all(row["legacy_script_id"].startswith(f"{row['module']}:") for row in commands)
|
||||
|
||||
|
||||
|
||||
@@ -102,10 +102,10 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
|
||||
data_root=project_root / "var",
|
||||
)
|
||||
|
||||
assert len(scripts.command_ids) == 62
|
||||
assert len(first.command_ids) == 157
|
||||
assert len(scripts.command_ids) == 61
|
||||
assert len(first.command_ids) == 155
|
||||
bindings = [first.binding_for(command_id) for command_id in first.command_ids]
|
||||
assert len({binding.cdp_port for binding in bindings}) == 157
|
||||
assert len({binding.cdp_port for binding in bindings}) == 155
|
||||
assert all(22000 <= binding.cdp_port <= 22999 for binding in bindings)
|
||||
assert all(binding.cdp_url.startswith("http://127.0.0.1:") for binding in bindings)
|
||||
assert bindings == [second.binding_for(item.script_id) for item in bindings]
|
||||
@@ -131,6 +131,8 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
|
||||
"GUANGHE_PASSWORD",
|
||||
"GUANGHE_LUGGAGE_USERNAME",
|
||||
"GUANGHE_LUGGAGE_PASSWORD",
|
||||
"GUANGHE_NEW_BAG_USERNAME",
|
||||
"GUANGHE_NEW_BAG_PASSWORD",
|
||||
)
|
||||
video_environment = first.environment_for("product.video_upload.run", {})
|
||||
assert video_environment["GUANGHE_USER_DATA_DIR"] == str(
|
||||
@@ -142,6 +144,12 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
|
||||
assert video_environment["GUANGHE_LUGGAGE_USER_DATA_DIR"].endswith(
|
||||
"-luggage"
|
||||
)
|
||||
assert video_environment["GUANGHE_NEW_BAG_USER_DATA_DIR"].endswith(
|
||||
"-new-bag"
|
||||
)
|
||||
assert video_environment["GUANGHE_NEW_BAG_USER_DATA_DIR"] != (
|
||||
video_environment["GUANGHE_LUGGAGE_USER_DATA_DIR"]
|
||||
)
|
||||
jd_video_upload = first.binding_for("product.jd_video_upload.run")
|
||||
assert jd_video_upload is first.binding_for(
|
||||
"product_commerce:upload_video_to_jd.py"
|
||||
|
||||
@@ -83,7 +83,7 @@ def test_default_registry_covers_every_executable_workflow_entry() -> None:
|
||||
assert scheduled_workflows == 40
|
||||
assert manual_workflows == 0
|
||||
assert len(workflow_entries) == 56
|
||||
assert len(catalog.scripts) == 62
|
||||
assert len(catalog.scripts) == 61
|
||||
assert workflow_entries < set(catalog.script_ids)
|
||||
assert {
|
||||
"content.mapping.rebuild",
|
||||
@@ -97,7 +97,6 @@ def test_default_registry_covers_every_executable_workflow_entry() -> None:
|
||||
"shop.douyin_after_sales_quality.collect",
|
||||
"product.backfill.run",
|
||||
"product.erp_all_shop_daily",
|
||||
"product.review.orchestrate",
|
||||
"product.video_upload.run",
|
||||
"product.jd_video_upload.run",
|
||||
"supply.workflow.run",
|
||||
@@ -147,10 +146,6 @@ def test_default_registry_covers_every_executable_workflow_entry() -> None:
|
||||
"--to",
|
||||
"{business_date}",
|
||||
)
|
||||
assert catalog.get("product.review.orchestrate").default_args == (
|
||||
"--target-date",
|
||||
"{business_date}",
|
||||
)
|
||||
assert catalog.get("product.import.tmall_ads").default_args == (
|
||||
"--date",
|
||||
"{business_date}",
|
||||
@@ -280,7 +275,7 @@ def test_cli_lists_and_dry_runs_registered_commands(tmp_path: Path) -> None:
|
||||
|
||||
assert main(["scripts", "list", "--json"], settings=settings, stdout=output) == 0
|
||||
rows = json.loads(output.getvalue())
|
||||
assert len(rows) == 62
|
||||
assert len(rows) == 61
|
||||
assert {row["module"] for row in rows} == {
|
||||
"content_marketing",
|
||||
"product_commerce",
|
||||
|
||||
@@ -57,6 +57,6 @@ def test_every_workflow_and_manual_script_resolves_inside_new_project() -> None:
|
||||
assert target.is_relative_to(PROJECT_ROOT)
|
||||
|
||||
commands = ScriptCatalog.load_default()
|
||||
assert len(commands.scripts) == 62
|
||||
assert len(commands.scripts) == 61
|
||||
assert len(commands.command_ids) == len(set(commands.command_ids))
|
||||
assert all(command.path.is_relative_to(PROJECT_ROOT) for command in commands.scripts)
|
||||
Reference in new issue
Block a user