Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6f1c193f4b | ||
|
|
622a609800 | ||
|
|
2f31d5872f | ||
|
|
423f8d1f75 | ||
|
|
ee6fd4170e | ||
|
|
3cf7048140 | ||
|
|
47ed6ec3f6 | ||
|
|
b1861f69cd | ||
|
|
2ef635be0e |
No files matched your search
+3
-4
@@ -40,9 +40,8 @@ Thumbs.db
|
||||
# Local tool artifacts
|
||||
/.playwright-cli/
|
||||
/output/
|
||||
|
||||
# Local dsh workbench scaffold (generated at runtime, bundles node_modules)
|
||||
/workbench/
|
||||
/nul
|
||||
/new_all_bag.xlsx
|
||||
|
||||
# Workbench (dsh plugin) local artifacts
|
||||
/workbench/plugin/node_modules/
|
||||
/workbench/cordis.local.yml
|
||||
@@ -40,17 +40,6 @@ var/ 默认运行数据;不属于源码
|
||||
- `shop_intelligence`:店铺、竞店和京东自营业绩。
|
||||
- `supply_chain`:采购确认、补货、库存预警和采购单更新。
|
||||
|
||||
## 智能工作台(dsh-refact 分支)
|
||||
|
||||
`workbench/` 以 [deepseek-harness](https://github.com/deepseek-ai/deepseek-harness) 为
|
||||
agent 底座,把现有工作流以 dsh 插件形式接入 Web UI:侧边栏选中任意工作流即可提问、
|
||||
诊断、修复,支持监控告警与受控启动。使用与设计见 [workbench/README.md](workbench/README.md)
|
||||
与 [docs/workbench.md](docs/workbench.md)。
|
||||
|
||||
```powershell
|
||||
powershell -File workbench\bin\start-workbench.ps1
|
||||
```
|
||||
|
||||
## 开发环境
|
||||
|
||||
```powershell
|
||||
|
||||
+11
-10
@@ -274,6 +274,17 @@
|
||||
"kind": "python",
|
||||
"notification_workflow_id": "product.online_stock_alert.daily"
|
||||
},
|
||||
{
|
||||
"id": "product.points_exchange_weekly.run",
|
||||
"module": "product_commerce",
|
||||
"entry": "points_exchange_weekly.py",
|
||||
"kind": "python",
|
||||
"default_args": [
|
||||
"--from",
|
||||
"{business_date}"
|
||||
],
|
||||
"notification_workflow_id": "product.points_exchange.weekly"
|
||||
},
|
||||
{
|
||||
"id": "product.import.daily",
|
||||
"module": "product_commerce",
|
||||
@@ -501,16 +512,6 @@
|
||||
"kind": "python",
|
||||
"notification_workflow_id": "product.market_rank"
|
||||
},
|
||||
{
|
||||
"id": "product.review.orchestrate",
|
||||
"module": "product_commerce",
|
||||
"entry": "orchestrate_review_collection.py",
|
||||
"kind": "python",
|
||||
"default_args": [
|
||||
"--target-date",
|
||||
"{business_date}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "supply.replenishment.arrival_sync",
|
||||
"module": "supply_chain",
|
||||
|
||||
@@ -231,6 +231,9 @@
|
||||
"product.online_stock_alert.daily": {
|
||||
"condition": "线上商品当前库存低于 3 时向配置接收人发送库存预警。"
|
||||
},
|
||||
"product.points_exchange.weekly": {
|
||||
"condition": "每周汇总上一周积分兑换商品排行后向配置接收人发送周报。"
|
||||
},
|
||||
"supply.purchase_confirmation.daily": {
|
||||
"condition": "采购确认结果生成完成后投递最终业务通知。"
|
||||
},
|
||||
@@ -250,6 +253,15 @@
|
||||
"wang_yunlong",
|
||||
"stock_alert_user_19832771651"
|
||||
]
|
||||
},
|
||||
"product.points_exchange.weekly": {
|
||||
"enabled": true,
|
||||
"app_profile": "hermes-analyzer",
|
||||
"person_ids": [
|
||||
"online_stock_alert_recipient",
|
||||
"wang_yunlong",
|
||||
"stock_alert_user_19832771651"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1119,21 +1119,14 @@
|
||||
],
|
||||
"cdp_port": 22163
|
||||
},
|
||||
"product_commerce:db/sync_dim_style.py": {
|
||||
"script_id": "product_commerce:db/sync_dim_style.py",
|
||||
"state_key": "product_commerce:db/sync_dim_style.py",
|
||||
"product.points_exchange_weekly.run": {
|
||||
"script_id": "product_commerce:points_exchange_weekly.py",
|
||||
"state_key": "product_commerce:points_exchange_weekly.py",
|
||||
"aliases": [
|
||||
"product_commerce:db/sync_dim_style.py"
|
||||
"product_commerce:points_exchange_weekly.py",
|
||||
"product.points_exchange_weekly.run"
|
||||
],
|
||||
"cdp_port": 22070
|
||||
},
|
||||
"product_commerce:db/sync_sku_master.py": {
|
||||
"script_id": "product_commerce:db/sync_sku_master.py",
|
||||
"state_key": "product_commerce:db/sync_sku_master.py",
|
||||
"aliases": [
|
||||
"product_commerce:db/sync_sku_master.py"
|
||||
],
|
||||
"cdp_port": 22071
|
||||
"cdp_port": 22168
|
||||
},
|
||||
"product_commerce:dy_audience_profile_collect.py": {
|
||||
"script_id": "product_commerce:dy_audience_profile_collect.py",
|
||||
@@ -1185,14 +1178,6 @@
|
||||
],
|
||||
"cdp_port": 22076
|
||||
},
|
||||
"product_commerce:import_product_reviews.py": {
|
||||
"script_id": "product_commerce:import_product_reviews.py",
|
||||
"state_key": "product_commerce:import_product_reviews.py",
|
||||
"aliases": [
|
||||
"product_commerce:import_product_reviews.py"
|
||||
],
|
||||
"cdp_port": 22077
|
||||
},
|
||||
"product_commerce:insert_bitable_records.py": {
|
||||
"script_id": "product_commerce:insert_bitable_records.py",
|
||||
"state_key": "product_commerce:insert_bitable_records.py",
|
||||
@@ -1263,14 +1248,6 @@
|
||||
],
|
||||
"cdp_port": 22083
|
||||
},
|
||||
"product.review.orchestrate": {
|
||||
"script_id": "product_commerce:orchestrate_review_collection.py",
|
||||
"state_key": "product_commerce:orchestrate_review_collection.py",
|
||||
"aliases": [
|
||||
"product_commerce:orchestrate_review_collection.py"
|
||||
],
|
||||
"cdp_port": 22084
|
||||
},
|
||||
"product_commerce:reapply_erp_override.py": {
|
||||
"script_id": "product_commerce:reapply_erp_override.py",
|
||||
"state_key": "product_commerce:reapply_erp_override.py",
|
||||
@@ -1475,7 +1452,9 @@
|
||||
"GUANGHE_USERNAME",
|
||||
"GUANGHE_PASSWORD",
|
||||
"GUANGHE_LUGGAGE_USERNAME",
|
||||
"GUANGHE_LUGGAGE_PASSWORD"
|
||||
"GUANGHE_LUGGAGE_PASSWORD",
|
||||
"GUANGHE_NEW_BAG_USERNAME",
|
||||
"GUANGHE_NEW_BAG_PASSWORD"
|
||||
]
|
||||
},
|
||||
"product.jd_video_upload.run": {
|
||||
|
||||
+13
-1
@@ -39,7 +39,7 @@
|
||||
"kind": "daily",
|
||||
"workflow_id": "content.marketing_report.daily"
|
||||
},
|
||||
{
|
||||
{
|
||||
"at": "12:00",
|
||||
"days": [
|
||||
"Sunday"
|
||||
@@ -197,6 +197,16 @@
|
||||
"kind": "daily",
|
||||
"workflow_id": "product.online_stock_alert.daily"
|
||||
},
|
||||
{
|
||||
"at": "09:45",
|
||||
"business_date_offset_days": -7,
|
||||
"days": [
|
||||
"Monday"
|
||||
],
|
||||
"enabled": true,
|
||||
"kind": "weekly",
|
||||
"workflow_id": "product.points_exchange.weekly"
|
||||
},
|
||||
{
|
||||
"at": "08:40",
|
||||
"business_date_offset_days": -1,
|
||||
@@ -325,6 +335,8 @@
|
||||
},
|
||||
{
|
||||
"at": "07:00",
|
||||
"business_date_offset_days": 0,
|
||||
"enabled": false,
|
||||
"kind": "daily",
|
||||
"workflow_id": "supply.replenishment_alert.daily"
|
||||
},
|
||||
|
||||
@@ -3,6 +3,86 @@
|
||||
"module": "product_commerce",
|
||||
"snapshot": "current-filesystem",
|
||||
"files": [
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "c7ad2a359730d8c4d449b0e37e5a4fb2940b4e8b5a4293a1b0a3767c185ee2cc",
|
||||
"target_sha256": "106a6a15ac2226e4db4d4bcdeda09ba8eca9facbd6e80f2ade6eea8821be5c5f",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_alerts.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_alerts.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "a96d18ccf79ea43d2152f2ab7de8de5e77c5a45e1cbbb2ec4495eb92e69bbe1b",
|
||||
"target_sha256": "1f286ed25cdebb4315aa64b878a7529ae74514b2def33d021834645116d3b48e",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_collect.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_collect.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "a7c8f6a777515f53160996b61804a93aedd4f3b37fa6374dc3eb64ff1588046a",
|
||||
"target_sha256": "4211341edcf8be34feef7b0be1e7779cb28707c4af5cc8c77d36d85835c28037",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_import.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "3df6a5bdda8448c342f2aa37992f471f572df7d5e51d5b0ad50911b4805cbb67",
|
||||
"target_sha256": "aa44824a204ee6b05a97fd58dd1964ea6d1b6f477823f10adc5c681384caeced",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_import.ps1",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.ps1"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "0c64ed81bed07ecdcd522549d0dcaccc97ed966cbb21a2ae60b37c634234feef",
|
||||
"target_sha256": "8a4af678695c849d5f49b295edb656155e967796229a8bb580a5d8955c6ba6a3",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_persona.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_persona.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "e461e25912cb3dafe3d6b0a092e599d121a756e11e564f1d33706d0709611cf1",
|
||||
"target_sha256": "c05295617b0361337192b21a76557c99dae14aad0d549c5980535f0f0090a41b",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_style_analysis_3d.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_style_analysis_3d.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "7edfb8cdbfdb88bbaca364b1ad213caef1818eb654b21a73131d67044ef356c0",
|
||||
"target_sha256": "42613ed10f083c803657dd6609c3b388fca88f7e0ee9dc105807ad8cab288689",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_jd_main_image.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_jd_main_image.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "1beadfdb5a0183b6e4a59307827ee4891eab985b1a1b17d0765bd02414bbd12c",
|
||||
"target_sha256": "c853990ca8857ef10b46ba71ba2177d4b6897dab4e420bce23bb7f2b22a23fae",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_main_image.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_main_image.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "656e72dc9a2999960b96d1921125e6649f0a204066664622cb2d6b1d45b290b5",
|
||||
"target_sha256": "be206938f49d9d5343dfb49e04514d6cdd9c7850a02e43f8d8c790b08b24c6fa",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_market_rank.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_market_rank.bat"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "589679874b75643d69ea44522e2fe40914a396506ff8e3232150d7e2a96ae2f9",
|
||||
"target_sha256": "dbe5760e9cbcaedadf82a2cfd84733a56a12c21e77d00327e95a8a013a5779ad",
|
||||
"transformed": true,
|
||||
"source_relative_path": ".env.example",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/.env.example"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "071286d2fb8b0516f0f22f0d088eda9e4a85ae4227619eb1217087ad0dbe3db6",
|
||||
@@ -14,19 +94,11 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "cff768f79672cfc7db7f6866caf5ab59936b8c51eb2a7d729b81441940124512",
|
||||
"target_sha256": "c1c0167b96c818a4e7d44aa5c860b5510c9925cb096b7fb4510b8319b91592fc",
|
||||
"target_sha256": "01ed46fbbf089f2c46bd67853d64774831ff2c63e2b3792e397bd01df7d9e847",
|
||||
"transformed": true,
|
||||
"source_relative_path": "analyze_style_with_hermes.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/analyze_style.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "524bff755d6921d64857b7ca6adfe1cf3a64730ddb66ae318751e028f5cf6f10",
|
||||
"target_sha256": "dd230b802fc88e16c67abcc8fd08a64c4dbfb68a58b74ea90ee45bad2d4d7870",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tmp_backfill_poseidon_sales.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_poseidon_sales.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "4d24f0e0cc65be9c2f6ce4047817794a520801f4e6563fbae2c3174c3d411d72",
|
||||
@@ -43,6 +115,38 @@
|
||||
"source_relative_path": "backfill_one_day.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_one_day.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "524bff755d6921d64857b7ca6adfe1cf3a64730ddb66ae318751e028f5cf6f10",
|
||||
"target_sha256": "dd230b802fc88e16c67abcc8fd08a64c4dbfb68a58b74ea90ee45bad2d4d7870",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tmp_backfill_poseidon_sales.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_poseidon_sales.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "8a2b09f8e5d4d49239c99a28fbc70bb5e44c8f510b6a1eca86b928eb45f21564",
|
||||
"target_sha256": "8a2b09f8e5d4d49239c99a28fbc70bb5e44c8f510b6a1eca86b928eb45f21564",
|
||||
"transformed": false,
|
||||
"source_relative_path": "backfill_style_id.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_style_id.py"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a",
|
||||
"target_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a",
|
||||
"transformed": false,
|
||||
"source_relative_path": "bitable_main_image_map.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_main_image_map.json"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2",
|
||||
"target_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2",
|
||||
"transformed": false,
|
||||
"source_relative_path": "bitable_style_map.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_style_map.json"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "6fb7cd48665c8cb7cf40dd69693408047a85a6abba3cbd410b4e5e9a9a55e9ba",
|
||||
@@ -70,7 +174,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "c7b54d941ab7f056e2d0ea4355543ff4a5c34ad986c8ed32145e988ac9501ea2",
|
||||
"target_sha256": "8576678688d7d90f6f7285110de005982bc1ba82bc7e9977224ba1bdb4101283",
|
||||
"target_sha256": "b306195f9fd90d6fba3c8fd9335677fe8dcfa1653604c8ace9f3cc340f43a5dc",
|
||||
"transformed": true,
|
||||
"source_relative_path": "collect_erp_yesterday_metrics.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py"
|
||||
@@ -123,10 +227,18 @@
|
||||
"source_relative_path": "config/__init__.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/__init__.py"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "277f80393a7ead72b260b56173471e924e139db0ba60e1ae534d85cc1ef5f9cc",
|
||||
"target_sha256": "bd0f4087fc061555cb8d2bab485bc0d98417dc1816d210bd58bf7ff6801cff39",
|
||||
"transformed": true,
|
||||
"source_relative_path": "config/auto-flow-config.example.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/auto-flow-config.example.json"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "22dcc9d8b2f1cf598a62e4c71df765a952250694cb26b9a73f477c5ca1a29884",
|
||||
"target_sha256": "ffeaf697a1a58b21520fda3f3b0936c591d05b328537c41f2048366a9ba1b8d5",
|
||||
"target_sha256": "5af46cd70a75930ca34142775d145416c073699ecec265dab6a34f27a59794fe",
|
||||
"transformed": true,
|
||||
"source_relative_path": "config/style_config_loader.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/style_config_loader.py"
|
||||
@@ -134,26 +246,18 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "a8900b9f25db6cf2bef3117a1d2cd86283054fea6f5a8590fd91b5cfbbcaf87e",
|
||||
"target_sha256": "47cdfd9ebf27e31be06497b2f74c80da4ffced259a873730eb024b0db5eba740",
|
||||
"target_sha256": "61cb3cc7e5ecbe76dea8f7557327c371971e9b301a62f99727708ed342caeaf4",
|
||||
"transformed": true,
|
||||
"source_relative_path": "db/__init__.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/__init__.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "fc19a57bcbc023ab20156820a708d64c5526fad33ef5eb5fdea56c0bb04b84e3",
|
||||
"target_sha256": "2cbd79e4d984d556bac35a0aeb5b8f803309fd3fac2399da459cd61a3aa2484c",
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746",
|
||||
"target_sha256": "51893cc8b9cb241820fcdccc84729bc2303e5c705d47cef7f76db911fb6984cb",
|
||||
"transformed": true,
|
||||
"source_relative_path": "db/sync_dim_style.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/sync_dim_style.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "2dc00b88fb5f28c2db619074e97c4a7c446aec29926af9a6890065409c8fac0e",
|
||||
"target_sha256": "840f98d9fb79557016a59a2e3a1194a1e328a1574b4670f419c99ac85731d623",
|
||||
"transformed": true,
|
||||
"source_relative_path": "db/sync_sku_master.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/sync_sku_master.py"
|
||||
"source_relative_path": "db/schema.sql",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
@@ -166,7 +270,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "492916088fa5e8c170461cc9eba97a88ed1cc1e50c7b99e5f6a5e49167fea157",
|
||||
"target_sha256": "7b0ae5d648245e89f4aacf262250009de812d3dbb8ee55386f3704cabca48799",
|
||||
"target_sha256": "1ab238bf134a4e639c81d60f3e2cff63ec07040b3330c90289fb2e661f082573",
|
||||
"transformed": true,
|
||||
"source_relative_path": "dy_product_scraping.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/dy_product_scraping.py"
|
||||
@@ -182,7 +286,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "fcd8db904a14cd5239a5e075e1a888af18c5b24c928608576f6decf4c48c1d8f",
|
||||
"target_sha256": "282e9eaee483fde3394d330a62887fb4832cbd876b67a002e27651cd49b8730a",
|
||||
"target_sha256": "18cfd51b7f6df00a4a4834ccce59acc83035177b39a40e1cb4b0f3639545a2fc",
|
||||
"transformed": true,
|
||||
"source_relative_path": "erp_metric_overrides.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/erp_metric_overrides.py"
|
||||
@@ -211,18 +315,10 @@
|
||||
"source_relative_path": "import_product_daily.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/import_product_daily.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "003a8969da4c50bd500f48fd6b3dcd6378322d87893cfcaf755b3890579e7a52",
|
||||
"target_sha256": "50d757be4bfdf8849a43471f419ee460cb844921cf567454de35fcaadbae737f",
|
||||
"transformed": true,
|
||||
"source_relative_path": "import_product_reviews.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/import_product_reviews.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "057a01e8a22cdd2d1806d290da207209954af89f7da6339e9cdcef3bc0297e63",
|
||||
"target_sha256": "0360bd218db1625088663b8ba509b5f53edd09563b71823006f2d46cdd5367ae",
|
||||
"target_sha256": "815c00d039e4f81e4f9deae95891fab288c6584e72d21b73f4b6f348e158e126",
|
||||
"transformed": true,
|
||||
"source_relative_path": "insert_bitable_records.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/insert_bitable_records.py"
|
||||
@@ -230,7 +326,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "e0f016bafb3f527f1cae46eaf3ef31510184128aaaa7f52024a7c3f978eec42b",
|
||||
"target_sha256": "b157f24430170d5a991bb2283af0a747b26dd11c0f0dec37289774f115132fee",
|
||||
"target_sha256": "cef210f479d45e4b92cc597ddde0d80e62e37b1546c3ee27099ad7ad9c71d108",
|
||||
"transformed": true,
|
||||
"source_relative_path": "jd_main_image_collector.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_main_image_collector.py"
|
||||
@@ -238,7 +334,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "d99ce71a008128549ac83fc3da52d88540b74538e3fe5c85d47c3b985e414353",
|
||||
"target_sha256": "cfe98b322e4ef7bd875325c2a9ffedeabb364e98b2983773dc98f9e0e75a6581",
|
||||
"target_sha256": "a24cc909d319c801fa63372dfcb6e3a0bd1cb6326e28008db7ada7293802a743",
|
||||
"transformed": true,
|
||||
"source_relative_path": "jd_product_data_collector.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py"
|
||||
@@ -285,11 +381,11 @@
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba",
|
||||
"target_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba",
|
||||
"transformed": false,
|
||||
"source_relative_path": "market_rank_report_archive.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_report_archive.py"
|
||||
"source_sha256": "f200d890e62bb9e387949264f5b518ed22c2bb620a60fcd28b673ff3e513c398",
|
||||
"target_sha256": "b6a3ad04efbe9b42a598caeb67f7408171ef1862f146fce4dc3eeb57fb2ce24c",
|
||||
"transformed": true,
|
||||
"source_relative_path": "market_rank_limits.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_limits.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
@@ -301,16 +397,16 @@
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "f200d890e62bb9e387949264f5b518ed22c2bb620a60fcd28b673ff3e513c398",
|
||||
"target_sha256": "b6a3ad04efbe9b42a598caeb67f7408171ef1862f146fce4dc3eeb57fb2ce24c",
|
||||
"transformed": true,
|
||||
"source_relative_path": "market_rank_limits.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_limits.py"
|
||||
"source_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba",
|
||||
"target_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba",
|
||||
"transformed": false,
|
||||
"source_relative_path": "market_rank_report_archive.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_report_archive.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "37edbfa24dfb126d52992b08fd6aeccae3ee18c83db56426a0fb02934e4541ae",
|
||||
"target_sha256": "47489fd55cbd38a39a8cf3ac0673549e8eee9a39448c187bb34c5cd57b244b97",
|
||||
"target_sha256": "b52b75e188906de1cabf55eb8791976f740c4bc9f1192971a5a4a5e7a082498c",
|
||||
"transformed": true,
|
||||
"source_relative_path": "orchestrate_daily_collection.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py"
|
||||
@@ -323,14 +419,6 @@
|
||||
"source_relative_path": "orchestrate_market_rank_collection.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_market_rank_collection.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "0d0f2c0a1533a09e698cd23aca42662b309c865659b86f9b9522de8207c3fb8b",
|
||||
"target_sha256": "47b2943790486ae6a1ca36a7e972467b89547f0ef6c37aa673a3733316277ec8",
|
||||
"transformed": true,
|
||||
"source_relative_path": "orchestrate_review_collection.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_review_collection.py"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "69f376f2c209d8e2df8f154aceb501eab93d7e077ab0278f5ff35c97356ba296",
|
||||
@@ -374,7 +462,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "af9b436a31c3454dc6e0b34b77876962dab0b477e8a561db49911fdfc24eade3",
|
||||
"target_sha256": "e3b91d3c4b529291f7a6ceee5ff99326351d78104a66babc531f90faec6b7e1a",
|
||||
"target_sha256": "096834b64e4b8e5120e38a116efb4c6eb02440d7c4567bf4d6783c18aee0050d",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_main_image.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/run_weekly_main_image.py"
|
||||
@@ -382,7 +470,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "34d935efb8dff20afd3ab89ef150c5f8db9cee80c6256c7fe4ef0bae2f72ed63",
|
||||
"target_sha256": "787a207a2c8174518a6465587644937ad6d11b1d86b07488e1c86b07e1410171",
|
||||
"target_sha256": "3fdaa7e23e96897c2039dcd1cdc6c6fc4ee1fb9cff828262e2360d453d778f47",
|
||||
"transformed": true,
|
||||
"source_relative_path": "scripts/insert_jd_main_image_records.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/scripts/insert_jd_main_image_records.py"
|
||||
@@ -390,11 +478,19 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "a1b097ead2a3f7e20ca63d7c15828a8d04030b0d6722934c3bf65d9e975fba76",
|
||||
"target_sha256": "7d8587f77615040087d2d475a7b440ad16a39c5874e143418dd700ecbc5db0f8",
|
||||
"target_sha256": "6c2ef5acbfff574a9f7460ebaccc8c4e46a3ccbd8b72944791e0fded69038e77",
|
||||
"transformed": true,
|
||||
"source_relative_path": "scripts/insert_main_image_records.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/scripts/insert_main_image_records.py"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3",
|
||||
"target_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3",
|
||||
"transformed": false,
|
||||
"source_relative_path": "styles_input.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/styles_input.json"
|
||||
},
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "b3e9addd72d8e31292ed948b67018aa9e212d628d7ba39141a81116515ca6721",
|
||||
@@ -406,7 +502,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "4474dd7d68117247af87e20bbeb6b7c072368ef0f606de5d3e186f459d34a0f5",
|
||||
"target_sha256": "862ae9392227113ad4266c2170769595c36b2f98fdecc54e54a648ba834baebf",
|
||||
"target_sha256": "9e87faad21459b8ac79c335734e11b1da636d34747f05e80c7073b66f01e35f1",
|
||||
"transformed": true,
|
||||
"source_relative_path": "taobao_sycm_collect.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_sycm_collect.py"
|
||||
@@ -430,7 +526,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "3f0c32c443c87417c43881da69aa7776828ba973465f8d32f50b3dc698b0a4fe",
|
||||
"target_sha256": "548df6ddf3af3b3c066d18687114f1ce672afa054535f6da0c254d60f8306b86",
|
||||
"target_sha256": "ed86f1b7d4fe6d8e68f4d17abc6bb299df8d3083a0f54c005470491e753c1755",
|
||||
"transformed": true,
|
||||
"source_relative_path": "taobao_wanxiang_ai_creative_report.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py"
|
||||
@@ -438,7 +534,7 @@
|
||||
{
|
||||
"category": "production_source",
|
||||
"source_sha256": "e4d99956d9649d108baeb3e3a2d1969c227622d1037062bd277af66375556d82",
|
||||
"target_sha256": "2ccb359ab943c185b17854f0f00cfb76b8a1235d7b031209a1c07e9386f8d93f",
|
||||
"target_sha256": "c5ed343c698efa35b76f35d95e6deeb19be113f0107d170bbd91596bab2f8e3b",
|
||||
"transformed": true,
|
||||
"source_relative_path": "upload_video_to_guanghe.py",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py"
|
||||
@@ -526,52 +622,12 @@
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/weekly_aggregate.py"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "589679874b75643d69ea44522e2fe40914a396506ff8e3232150d7e2a96ae2f9",
|
||||
"target_sha256": "dbe5760e9cbcaedadf82a2cfd84733a56a12c21e77d00327e95a8a013a5779ad",
|
||||
"category": "regression_test",
|
||||
"source_sha256": "55f55a9658453e6751ed055c6b2d1a43538012ce715835bbef59d88203211696",
|
||||
"target_sha256": "5fc5e15e0356b1438ea158ed1d64b769bfaf8c94bfd9f6b3a8d828a15d53c231",
|
||||
"transformed": true,
|
||||
"source_relative_path": ".env.example",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/.env.example"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a",
|
||||
"target_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a",
|
||||
"transformed": false,
|
||||
"source_relative_path": "bitable_main_image_map.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_main_image_map.json"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2",
|
||||
"target_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2",
|
||||
"transformed": false,
|
||||
"source_relative_path": "bitable_style_map.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_style_map.json"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "277f80393a7ead72b260b56173471e924e139db0ba60e1ae534d85cc1ef5f9cc",
|
||||
"target_sha256": "bd0f4087fc061555cb8d2bab485bc0d98417dc1816d210bd58bf7ff6801cff39",
|
||||
"transformed": true,
|
||||
"source_relative_path": "config/auto-flow-config.example.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/auto-flow-config.example.json"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746",
|
||||
"target_sha256": "6bbd36048836ef7ed0807223fc028ac17349b20314b1ea353ccedb451957ba55",
|
||||
"transformed": true,
|
||||
"source_relative_path": "db/schema.sql",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql"
|
||||
},
|
||||
{
|
||||
"category": "runtime_resource",
|
||||
"source_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3",
|
||||
"target_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3",
|
||||
"transformed": false,
|
||||
"source_relative_path": "styles_input.json",
|
||||
"target_relative_path": "src/gyxx_flow/modules/product_commerce/styles_input.json"
|
||||
"source_relative_path": "tests/test_tmp_backfill_poseidon_sales.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_backfill_poseidon_sales.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
@@ -599,8 +655,8 @@
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "4041e1104d5adbb92703b88e457795ac221bc7bc4fe4aa2cb72dff08f0c4ab6b",
|
||||
"target_sha256": "4041e1104d5adbb92703b88e457795ac221bc7bc4fe4aa2cb72dff08f0c4ab6b",
|
||||
"source_sha256": "a9b397cbc0a04cf3ba0caec54c0194ab97a96a4e8bacc3207cd0a0b7b8cbee8b",
|
||||
"target_sha256": "a9b397cbc0a04cf3ba0caec54c0194ab97a96a4e8bacc3207cd0a0b7b8cbee8b",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_erp_metric_overrides.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_erp_metric_overrides.py"
|
||||
@@ -639,8 +695,8 @@
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "e8848b95c28a822b9fc92b5cd8ad75716b80d00fbac1844bfad9b7f877ea0173",
|
||||
"target_sha256": "e8848b95c28a822b9fc92b5cd8ad75716b80d00fbac1844bfad9b7f877ea0173",
|
||||
"source_sha256": "6ba0ead03f6889cb66ea77799d4416bd4d9ccc81ccaa1df6ee79c1ff4c5dd1bc",
|
||||
"target_sha256": "6ba0ead03f6889cb66ea77799d4416bd4d9ccc81ccaa1df6ee79c1ff4c5dd1bc",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_guanghe_store_routing.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_guanghe_store_routing.py"
|
||||
@@ -672,7 +728,7 @@
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "820e17b910b62a213f0bebafcff72998c2c978307c97f976c074276150ce7c6a",
|
||||
"target_sha256": "cdedf8c9a4871ad763e0a9098fc18777f0c3206991e8d440a45ccdfbda089a07",
|
||||
"target_sha256": "88ff084edc73a011a4c29ed61fbd88e12d9771c3e7aed38805f152add09a64d1",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_main_image_concurrency.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_main_image_concurrency.py"
|
||||
@@ -687,11 +743,11 @@
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69",
|
||||
"target_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_market_rank_report_archive.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_market_rank_report_archive.py"
|
||||
"source_sha256": "1e0f8a519ad3ce0079240fa2b6b1d2f2df1b2c0091da89049f7df9aeec660196",
|
||||
"target_sha256": "72b85f901d43f836cdc5e8eabe9d83f67a89f8bbd49ef8c5267f92e3114d28b1",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_market_rank_limits.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_market_rank_limits.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
@@ -703,11 +759,11 @@
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "1e0f8a519ad3ce0079240fa2b6b1d2f2df1b2c0091da89049f7df9aeec660196",
|
||||
"target_sha256": "72b85f901d43f836cdc5e8eabe9d83f67a89f8bbd49ef8c5267f92e3114d28b1",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_market_rank_limits.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_market_rank_limits.py"
|
||||
"source_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69",
|
||||
"target_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_market_rank_report_archive.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_market_rank_report_archive.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
@@ -728,7 +784,7 @@
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "f9cfbb49fd313444790ac194fbf980dae21912081b9f26ad8e3850912126f09e",
|
||||
"target_sha256": "043b2dd69172bf5ea1b324bf07f8bba4c43f4bca9817c30a2c805ffe8ce70525",
|
||||
"target_sha256": "857b87a507807d522136199ad4acc54a2bbb4c7667514cf264f136e2cc6a28c9",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_style_analysis_orchestration.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_style_analysis_orchestration.py"
|
||||
@@ -736,18 +792,18 @@
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "17761a5ba93a5387ffa67662424955178058aeec708bbcd3eb0341a859d8a1f1",
|
||||
"target_sha256": "549cb13dd32c9bed76d78eafa7542ea6e5c638d9f29c857eff4dde6d3b8e9c40",
|
||||
"target_sha256": "b1725d401b59a2a19ef4824438d8362d2c68e4ccc0c06d75a2143dbc3d62d1bf",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_style_config_loader_erp_source.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_style_config_loader_erp_source.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "55f55a9658453e6751ed055c6b2d1a43538012ce715835bbef59d88203211696",
|
||||
"target_sha256": "5fc5e15e0356b1438ea158ed1d64b769bfaf8c94bfd9f6b3a8d828a15d53c231",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_tmp_backfill_poseidon_sales.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_backfill_poseidon_sales.py"
|
||||
"source_sha256": "895e9e2f149d7ef01a712246f0a5c3c641c9103908b6216571cf5764f6bff0dd",
|
||||
"target_sha256": "895e9e2f149d7ef01a712246f0a5c3c641c9103908b6216571cf5764f6bff0dd",
|
||||
"transformed": false,
|
||||
"source_relative_path": "tests/test_style_id_identity.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_style_id_identity.py"
|
||||
},
|
||||
{
|
||||
"category": "regression_test",
|
||||
@@ -768,82 +824,10 @@
|
||||
{
|
||||
"category": "regression_test",
|
||||
"source_sha256": "5598dcbf6d9a30fb3dc4133e983ed78c2860d90162c6f7750509d4533466917a",
|
||||
"target_sha256": "198cb3979b355a81863683dc357f4152afaf331cca963259d4010ba63778b40b",
|
||||
"target_sha256": "09981f868a2f1b2b26ce14cc6bd96f1bb4cb2b8f53e2d668ae157b81966b9b36",
|
||||
"transformed": true,
|
||||
"source_relative_path": "tests/test_wanxiang_report_template.py",
|
||||
"target_relative_path": "tests/modules/product_commerce/test_wanxiang_report_template.py"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "c7ad2a359730d8c4d449b0e37e5a4fb2940b4e8b5a4293a1b0a3767c185ee2cc",
|
||||
"target_sha256": "106a6a15ac2226e4db4d4bcdeda09ba8eca9facbd6e80f2ade6eea8821be5c5f",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_alerts.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_alerts.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "a96d18ccf79ea43d2152f2ab7de8de5e77c5a45e1cbbb2ec4495eb92e69bbe1b",
|
||||
"target_sha256": "1f286ed25cdebb4315aa64b878a7529ae74514b2def33d021834645116d3b48e",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_collect.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_collect.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "a7c8f6a777515f53160996b61804a93aedd4f3b37fa6374dc3eb64ff1588046a",
|
||||
"target_sha256": "4211341edcf8be34feef7b0be1e7779cb28707c4af5cc8c77d36d85835c28037",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_import.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "3df6a5bdda8448c342f2aa37992f471f572df7d5e51d5b0ad50911b4805cbb67",
|
||||
"target_sha256": "aa44824a204ee6b05a97fd58dd1964ea6d1b6f477823f10adc5c681384caeced",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_import.ps1",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.ps1"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "0c64ed81bed07ecdcd522549d0dcaccc97ed966cbb21a2ae60b37c634234feef",
|
||||
"target_sha256": "8a4af678695c849d5f49b295edb656155e967796229a8bb580a5d8955c6ba6a3",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_daily_persona.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_persona.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "e461e25912cb3dafe3d6b0a092e599d121a756e11e564f1d33706d0709611cf1",
|
||||
"target_sha256": "c05295617b0361337192b21a76557c99dae14aad0d549c5980535f0f0090a41b",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_style_analysis_3d.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_style_analysis_3d.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "7edfb8cdbfdb88bbaca364b1ad213caef1818eb654b21a73131d67044ef356c0",
|
||||
"target_sha256": "42613ed10f083c803657dd6609c3b388fca88f7e0ee9dc105807ad8cab288689",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_jd_main_image.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_jd_main_image.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "1beadfdb5a0183b6e4a59307827ee4891eab985b1a1b17d0765bd02414bbd12c",
|
||||
"target_sha256": "c853990ca8857ef10b46ba71ba2177d4b6897dab4e420bce23bb7f2b22a23fae",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_main_image.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_main_image.bat"
|
||||
},
|
||||
{
|
||||
"category": "launcher_provenance",
|
||||
"source_sha256": "656e72dc9a2999960b96d1921125e6649f0a204066664622cb2d6b1d45b290b5",
|
||||
"target_sha256": "be206938f49d9d5343dfb49e04514d6cdd9c7850a02e43f8d8c790b08b24c6fa",
|
||||
"transformed": true,
|
||||
"source_relative_path": "run_weekly_market_rank.bat",
|
||||
"target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_market_rank.bat"
|
||||
}
|
||||
],
|
||||
"intentionally_excluded": [
|
||||
|
||||
+56
-11
@@ -1126,7 +1126,7 @@
|
||||
"30",
|
||||
"--headless"
|
||||
],
|
||||
"timeout_seconds": 14400,
|
||||
"timeout_seconds": 28800,
|
||||
"replay_policy": "idempotent"
|
||||
}
|
||||
]
|
||||
@@ -1180,7 +1180,7 @@
|
||||
"{business_date}",
|
||||
"--end-date",
|
||||
"{business_date}",
|
||||
"--headless"
|
||||
"--headed"
|
||||
],
|
||||
"timeout_seconds": 14400,
|
||||
"replay_policy": "idempotent"
|
||||
@@ -1705,6 +1705,51 @@
|
||||
"task_name": "OnlineItemStockAlert"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "product.points_exchange.weekly",
|
||||
"module": "product_commerce",
|
||||
"trigger": "scheduled",
|
||||
"execution": {
|
||||
"steps": [
|
||||
{
|
||||
"id": "query_rank_notify",
|
||||
"name": "查询积分兑换明细并按商品生成兑换排行周报",
|
||||
"description": "调用企业数据交互积分兑换查询接口(startTime/endTime 覆盖整周自然日),按商品汇总 quantity 兑换数量并从高到低排名;默认统计上一个完整自然周(周一至周日),将排行榜以飞书卡片发送给私域库存预警的同一批接收人。",
|
||||
"replay_policy": "idempotent",
|
||||
"data_flow": {
|
||||
"sources": [
|
||||
{
|
||||
"system": "企业数据交互接口",
|
||||
"label": "积分兑换明细",
|
||||
"detail": "/open/data/v1/points-exchange/query,startTime 为窗口首日 00:00:00、endTime 为窗口末日 23:59:59"
|
||||
}
|
||||
],
|
||||
"processing": [
|
||||
"以业务日期作为窗口起点,缺省统计 7 个自然日",
|
||||
"按 itemId 汇总 quantity,并保留商品名称与 SKU 数",
|
||||
"按兑换数量从高到低排序并分段组织飞书消息"
|
||||
],
|
||||
"destinations": [
|
||||
{
|
||||
"system": "飞书",
|
||||
"label": "积分兑换商品排行周报",
|
||||
"condition": "窗口内存在兑换数量大于 0 的商品"
|
||||
}
|
||||
]
|
||||
},
|
||||
"entry": "points_exchange_weekly.py",
|
||||
"args": [
|
||||
"--from",
|
||||
"{business_date}"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"provenance": {
|
||||
"source_project": "data-hub",
|
||||
"task_name": "PointsExchangeWeeklyRanking"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "product.ecommerce_costs.daily",
|
||||
"module": "product_commerce",
|
||||
@@ -1751,7 +1796,7 @@
|
||||
{
|
||||
"id": "tmall_import",
|
||||
"name": "导入万相台商品广告指标",
|
||||
"description": "解析解压后的商品 CSV,将花费、广告归因成交金额和投产比写入 product_daily_metrics.raw_data.tm_ad_metrics;款式按商品 ID 优先、商品名称精确包含兜底,并记录匹配方式,不覆盖经营日报字段。",
|
||||
"description": "解析解压后的商品 CSV,将花费、广告归因成交金额和投产比写入独立的 tm_product_ad_daily_metrics(每日每商品一行),报表其余指标存 metrics JSON;款式按商品 ID 优先、商品名称精确包含兜底,并记录匹配方式。不再写 product_daily_metrics.raw_data。",
|
||||
"entry": "import_tmall_product_ads.py",
|
||||
"depends_on": [
|
||||
"tmall_download"
|
||||
@@ -1773,13 +1818,13 @@
|
||||
"processing": [
|
||||
"校验 CSV 编码、日期、商品主体和广告指标",
|
||||
"合并同商品同日多计划行",
|
||||
"按平台/日期/商品幂等更新广告 JSON 节点"
|
||||
"按日期/商品幂等写入天猫商品投放日报表"
|
||||
],
|
||||
"destinations": [
|
||||
{
|
||||
"system": "PostgreSQL",
|
||||
"label": "product_daily_metrics.raw_data.tm_ad_metrics",
|
||||
"detail": "由 data-hub 产品生命进程读取并计算广告成交、投产比、电商费比;节点同时保存 style_name/style_match_method"
|
||||
"label": "tm_product_ad_daily_metrics",
|
||||
"detail": "由 data-hub 产品生命进程读取 spend/attributed_deal_amount 计算广告成交、投产比、电商费比;style_name/style_match_method 记录款式匹配,metrics 保留报表全部指标"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -2260,7 +2305,7 @@
|
||||
{
|
||||
"id": "upload",
|
||||
"name": "上传所选天猫店铺短视频",
|
||||
"description": "先按前端所选店铺扫描飞书并筛选符合路由的视频附件,再按视频 file_token、目标账号和飞书记录写入 PostgreSQL 附件级发布账本;仅在存在可领取附件时登录对应光合账号上传,已发布附件自动跳过。",
|
||||
"description": "先按前端所选店铺扫描飞书并筛选符合路由的视频附件,再按视频 file_token、目标账号和飞书记录写入 PostgreSQL 附件级发布账本;仅在存在可领取附件时登录对应光合账号上传,已发布附件自动跳过。支持旗舰店、鑫华达专卖店和天猫新店(天猫新店只发黑曜石系列,视频源与其他店铺为同一张飞书表)。",
|
||||
"replay_policy": "idempotent",
|
||||
"data_flow": {
|
||||
"sources": [
|
||||
@@ -2274,11 +2319,11 @@
|
||||
},
|
||||
{
|
||||
"system": "淘宝光合浏览器状态",
|
||||
"label": "旗舰店与箱包店独立账号和 Profile"
|
||||
"label": "旗舰店、箱包店与天猫新店独立账号和 Profile"
|
||||
}
|
||||
],
|
||||
"processing": [
|
||||
"按前端所选旗舰店或箱包店规则筛选飞书记录并拆分视频附件",
|
||||
"按前端所选旗舰店、箱包店或天猫新店规则筛选飞书记录并拆分视频附件;天猫新店仅保留全部款式都属于黑曜石系列的记录",
|
||||
"按视频 file_token、目标账号和飞书记录原子领取未上传附件",
|
||||
"确认存在可上传附件后登录所选店铺账号",
|
||||
"生成标题、关联天猫商品并发布;明确成功后写 published,提交后结果不明写 ambiguous"
|
||||
@@ -2319,7 +2364,7 @@
|
||||
"label": "天猫话题关键词",
|
||||
"hint": "输入完整话题或关键词;例如输入“秋上新”也会匹配并选中“秋上新开拍了”。"
|
||||
},
|
||||
"note": "默认由常驻调度器按 config/schedules.json 每 3 天 14:00 执行;前端仍可手动选择旗舰店或箱包店后执行。脚本先扫描飞书并按所选店铺规则筛选,只有数据库存在可领取附件时才登录对应账号上传。允许同一业务日期重复扫描新增附件,PostgreSQL 附件级账本阻止已发布或结果不明的附件自动重发;框架安全预演不会启动上传脚本。"
|
||||
"note": "默认由常驻调度器按 config/schedules.json 每 3 天 14:00 执行;前端仍可手动选择旗舰店、箱包店或天猫新店后执行。天猫新店账号为gyxx箱包旗舰店:龙虾仔(凭据 GUANGHE_NEW_BAG_USERNAME/GUANGHE_NEW_BAG_PASSWORD),只上传黑曜石系列,视频来源与其他店铺相同。脚本先扫描飞书并按所选店铺规则筛选,只有数据库存在可领取附件时才登录对应账号上传。允许同一业务日期重复扫描新增附件,PostgreSQL 附件级账本阻止已发布或结果不明的附件自动重发;框架安全预演不会启动上传脚本。"
|
||||
},
|
||||
{
|
||||
"id": "product.jd_video_upload",
|
||||
@@ -2835,7 +2880,7 @@
|
||||
"source_project": "supply",
|
||||
"task_name": "auto-flow-replenishment-arrival-sync"
|
||||
},
|
||||
"note": "队列表为空时成功结束;失败行保留 failed 状态,下一次调度自动重试。默认租户为 1,可用 GYXX_ERP_TENANT_ID 覆盖。"
|
||||
"note": "队列表为空时成功结束;失败行保留 failed 状态,下一次调度自动重试。data-hub 单公司化后队列表已无 tenant_id:脚本会先探测实际表结构,有该列仍按 GYXX_ERP_TENANT_ID(默认 1)过滤,没有则按单公司口径直接认领。"
|
||||
},
|
||||
{
|
||||
"id": "product.market_rank",
|
||||
|
||||
@@ -33,8 +33,10 @@ PostgreSQL 的动态配置表。生命进程负责款式名、品牌、ERP 款
|
||||
| `product.sales_sheet.daily` | 款式与 ERP 编码 | `sync_monthly_sales_sheet.py` |
|
||||
| `product.erp_all_shop_daily` | 全部款式与 ERP 编码 | `backfill_erp_all_shop_daily.py` |
|
||||
|
||||
共 12 个定时工作流依赖这套动态配置。维护脚本 `feishu_comment_batch.py` 和
|
||||
`db/sync_sku_master.py` 也已改为读取同一项目数据库,避免从非定时入口绕回旧主表。
|
||||
共 12 个定时工作流依赖这套动态配置。维护脚本 `feishu_comment_batch.py` 也已改为读取同一项目数据库,
|
||||
避免从非定时入口绕回旧主表。原先承担「动态配置 → 款式身份」投影的 `db/sync_dim_style.py` 与
|
||||
`db/sync_sku_master.py` 均已废弃:运行时身份已直读 `dim_style`,这两个脚本跑一次等于把自己写回自己,
|
||||
款式新增与改款统一走 data-hub「产品生命进程」建档页。
|
||||
|
||||
`product.alert.daily`、商品导入、万相台广告、视频发布和市场排行使用各自数据库或专用配置,
|
||||
不依赖旧配置主表,因此不在本次切换范围内。
|
||||
@@ -56,14 +58,15 @@ PostgreSQL 的动态配置表。生命进程负责款式名、品牌、ERP 款
|
||||
|
||||
数据库使用父子表:
|
||||
|
||||
- `cmt_styles`:产品生命进程主表,一个款式一行,保存 ERP 编码、四个平台商品编码和 NFC 数量。
|
||||
- `dim_style`:产品生命进程主表,一个款式一行,保存 ERP 款式编码、四个平台商品编码、系列归属和销量飞书表;采集端运行时身份的唯一权威来源。
|
||||
- `cmt_styles`:笔记分析用的款式挂载表(`cmt_notes_master.style_id` 引用其 id),已退出 product 采集身份,仅在 `dim_style` 不可用时作兜底。
|
||||
- `workflow_dynamic_styles`:一个款式一行,保存共享飞书目标及内容工作流参数。
|
||||
- `workflow_dynamic_style_platforms`:一个款式可有多个平台子行,保留平台画像表、启停状态和旧商品 ID 兼容快照。
|
||||
- 旧 `workflow_dynamic_configs` 保留为原始迁移审计,不再参与运行时读取。
|
||||
- `erp_codes`、`item_ids` 使用 PostgreSQL 数组,页面接受中英文逗号或换行并自动去重。
|
||||
- `destinations` 使用通用 JSONB 目标列表,每项包含 `key`、`label`、`url`、`description`、`enabled`;六个旧目标字段仍同步保留,保证已有工作流兼容。后续新增分析逻辑只需增加一个稳定的 `key` 和对应飞书表地址。
|
||||
- 款式带递增 `revision`;整页保存和删除必须提交 `If-Match`,避免并发覆盖。
|
||||
- `cmt_styles.platform_product_ids` 的四个键固定为 `tm`(天猫)、`jd`(京东旗舰店)、`jd_self`(京东自营)、`dy`(抖音);空数组表示该平台尚未补齐。
|
||||
- `dim_style` 的四个平台列固定为 `tm_spus`(天猫)、`jd_spus`(京东旗舰店)、`jd_self_skus`(京东自营)、`dy_spus`(抖音);空数组表示该平台尚未补齐。同系列只需主款填写,子款靠系列归属继承。
|
||||
|
||||
## 首次迁移
|
||||
|
||||
@@ -98,31 +101,78 @@ uv run --env-file 'D:/product-collector-analyze-flow/.env' `
|
||||
uv run --env-file 'D:/product-collector-analyze-flow/.env' `
|
||||
--project 'D:/gyxx-flow' `
|
||||
python -m gyxx_flow.migration.sync_feishu_style_identity --apply
|
||||
|
||||
uv run --env-file 'D:/product-collector-analyze-flow/.env' `
|
||||
--project 'D:/gyxx-flow' `
|
||||
python 'src/gyxx_flow/modules/product_commerce/db/sync_dim_style.py'
|
||||
```
|
||||
|
||||
脚本按款式合并旧表的多平台行,只写入 `cmt_styles` 的品牌、ERP 编码和四个平台商品编码,
|
||||
不会覆盖 NFC、图片、类目或市场标签;空源字段不会清空生命进程已有身份。拼多多等当前四键
|
||||
不会覆盖 NFC、图片、类目或市场标签;空源字段不会清空已有身份。拼多多等当前四键
|
||||
模型之外的平台只保留在动态平台配置/审计中,不会伪装成天猫、京东或抖音编码。默认是预览,
|
||||
必须显式传 `--apply` 才会写数据库。
|
||||
|
||||
> **`db/sync_dim_style.py` 已废弃并删除(2026-10-28)。**
|
||||
> 采集端运行时身份改为直接读 `dim_style`,该脚本再跑就是把 `dim_style` 投影回它自己,
|
||||
> 既补不了旧台账数据,也可能把空值写回。款式新增与改款统一走 data-hub「产品生命进程」
|
||||
> 建档页,下一轮调度自动生效,不需要再手工同步。
|
||||
|
||||
服务器必须注入 `GYXX_POSTGRES_DSN`,或完整的 `PG_HOST/PG_PORT/PG_DB/PG_USER/PG_PASSWORD`。
|
||||
没有数据库且没有数据库生成的运行缓存时,相关工作流会明确失败,不会静默回读旧飞书主表。
|
||||
|
||||
## 运行时路径
|
||||
|
||||
```text
|
||||
产品生命进程 cmt_styles (商品身份)
|
||||
产品生命进程 dim_style (商品身份:ERP 编码 / 四平台编码 / 系列归属 / 销量表)
|
||||
+
|
||||
workflow_dynamic_styles + workflow_dynamic_style_platforms (目标/画像/启停)
|
||||
↓
|
||||
统一运行时聚合
|
||||
统一运行时聚合(dim_style 优先,cmt_styles 仅在 dim_style 缺失时兜底)
|
||||
├─ 完整身份门 → StyleConfigLoader → 商品经营工作流
|
||||
└─ 目标兼容输出 → feishu_mapping → 内容营销工作流 → 各业务飞书目标表
|
||||
```
|
||||
|
||||
系列共享规则:平台商品编码、ERP 编码与 9 类飞书地址都以「系列」为一个采集单元,
|
||||
子款留空即可;子款误填也不会丢——loader 会把整条系列聚合并去重后挂到锚点。
|
||||
|
||||
- **平台编码聚合去重到锚点**:整条系列各平台的商品 ID 先并集去重,再挂到锚点,由锚点
|
||||
一个采集单元去抓。子款的原始编码保留在 `series.platform_ids` 供诊断,采集口判定让出
|
||||
(「缺 tm商品编码(同系列「X」已在采)」),同一个商品 ID 不会被采两次、访客与加购
|
||||
不会重复入账。锚点不在采集范围内时子款仍可回退自采。
|
||||
- **ERP 编码先聚合再采集**:正常建档形态是锚点填一条款式编码 + 各平台商品编码,子款只填
|
||||
自己的款式编码;也可能出现锚点 `erp_style_codes` 为空、编码全散落在子款上的形态。
|
||||
loader 一律把整条系列的 ERP 编码聚合成并集挂到锚点(`series.erp_owner`),锚点作为唯一
|
||||
ERP 采集单元去聚水潭取数,子款让出并记录「ERP 已由同系列 X 归集采集」,但仍保留自己的
|
||||
原始编码(`series.own_erp_codes`)供按款查询与月度销量表使用。
|
||||
**例外(2026-10-30)**:子款自身 `collect_enabled = FALSE` 时,它的 ERP 编码不并入聚合
|
||||
(`series.erp_excluded` 可见),锚点不代采;锚点自身 `FALSE` 仍是整系列停采。
|
||||
平台商品编码不受子款开关影响,仍按整系列聚合。
|
||||
- **飞书地址聚合去重下发**:`workflow_config._propagate_series_targets()` 与
|
||||
`_apply_series_sharing()` 两处都按「锚点优先、其余按款名」收集整系列非空地址,去重后
|
||||
下发到每一行。一个表位只能落一张表,同字段出现多个不同地址时保留优先级最高的一条并打
|
||||
`[WARN] 系列「X」<字段> 有 N 个不同表`。子款想用独立一套表,必须拆成独立系列。
|
||||
- **月度销量表按款明细**:`product.sales_sheet.daily` 的报表口径**不用**锚点上的聚合并集,
|
||||
而是每个款式只查自己建档时填的编码。采集端的并集是给聚水潭当唯一取数单元用的,报表
|
||||
照搬会让同系列每个成员行各查一遍并集、销量重复计入。锚点自身没编码时该行按 0 写入
|
||||
(`--allow-missing-erp-as-zero`),同系列销量落在各子款行上,合计与采集总量一致。
|
||||
|
||||
采集门只校验身份:ERP 日采要求 ERP 款式编码,平台日采再叠加对应平台商品编码。
|
||||
**销量飞书表地址不再是采集前置条件**——缺表地址的款式照常采集并落库,只是写表环节按
|
||||
地址缺失跳过(`get_bitable_style_map` 不生成该款目标),不影响数据入库与后续分析。
|
||||
主图采集同理:`get_main_image_styles` 只要求身份齐全,**主图飞书表地址不参与采集门**,
|
||||
缺地址的款照常抓图并写 `main_image_creatives`,`get_bitable_main_image_map` 只返回有地址
|
||||
的款,写表脚本对没有地址的款打 `[NO-TABLE]` 跳过飞书、仍入库。
|
||||
|
||||
**写表脚本对目标地址 fail closed(2026-10-30)**:`insert_bitable_records.py`、
|
||||
`scripts/insert_main_image_records.py`、`scripts/insert_jd_main_image_records.py` 读取
|
||||
`get_bitable_style_map()` / `get_bitable_main_image_map()` 失败或返回空时,直接打
|
||||
`[FAIL] …拒绝回退本地 JSON` 并退出码 1,**不再回退** `bitable_style_map.json` /
|
||||
`bitable_main_image_map.json`。库与本地文件不一致时,回退会让数据静默写到错误的表,
|
||||
比直接失败更难发现。`styles_input.json` 同理由 `collect_erp_yesterday_metrics.load_erp_styles`
|
||||
与 `export_bitable_records.load_style_meta` 兜住:身份源不可用以 `lifecycle_unavailable` 失败。
|
||||
|
||||
**旧表只保留「行集合 + 启停」职责**:`workflow_dynamic_styles` / `_platforms` 仍决定有哪些
|
||||
款式、`enabled` 启停、`note`、`style_content`、`destinations`,但 ERP 编码、平台商品编码与
|
||||
9 类地址全部由 `dim_style` 覆盖(`merge_lifecycle_style_records` 中 `managed` 由
|
||||
`jsonb_build_object` 恒生成四个键而恒为真)。旧表里**不在 dim_style 的残留行**(孤儿款)
|
||||
地址会被清空、不再采集,属于建档改名/合并后的历史垃圾,应直接删除
|
||||
(2026-10-30 已清理 `拓界mode`、`逐星evo`,备份 `var/tmp/workflow_dynamic_orphans_20261030.json`)。
|
||||
|
||||
`StyleConfigLoader` 只在数据库短暂不可用时使用最近一次数据库成功读取后生成的本地缓存;
|
||||
`feishu_mapping` 的目标表字段缓存仍保留,用来减少对各业务飞书表的字段查询。
|
||||
+227
-11
@@ -43,7 +43,7 @@ uv run gyxx scripts run <command_id> --date 2026-08-01 --execute
|
||||
| 京东自营品牌单日回采 | `shop.jd_self_operated.collect_brand` | `--date` 自动成为起止日期 |
|
||||
| 商品历史补采 | `product.backfill.run` | `--date` 自动成为 `--from/--to` |
|
||||
| 商品评价补采 | `product.review.orchestrate` | `--date` 自动成为目标日期 |
|
||||
| 天猫旗舰店和箱包店视频上传 | `product.video_upload.run` | 业务日期用于运行审计;默认顺序扫描两店,由 PostgreSQL 附件级账本防重 |
|
||||
| 天猫旗舰店、箱包店与天猫新店视频上传 | `product.video_upload.run` | 业务日期用于运行审计;`--store all` 顺序扫描三家,由 PostgreSQL 附件级账本防重 |
|
||||
| 京东旗舰店视频上传 | `product.jd_video_upload.run` | 业务日期只用于运行审计;从固定凌云air记录锚点按飞书视图顺序扫描,日期可为空 |
|
||||
| 采购单更新 | `supply.workflow.run` | 自动选择 `mcp-run purchase-order-update` |
|
||||
|
||||
@@ -51,16 +51,16 @@ uv run gyxx scripts run <command_id> --date 2026-08-01 --execute
|
||||
|
||||
### 天猫视频上传工作流
|
||||
|
||||
天猫视频上传注册为幂等定时工作流 `product.video_upload`,按 `config/schedules.json` 以 2026-09-04 为起始日每 3 天 14:00(Asia/Shanghai)执行;前端仍可手动选择“光影行星旗舰店 / 光影行星鑫华达专卖店:龙虾仔”并立即运行,也可填写天猫话题完整名称或关键词。正式执行只处理所选店铺,话题搜索结果按包含关系选中。默认 dry-run 只生成运行记录并跳过浏览器节点;确认飞书待传记录、对应淘宝光合店铺的登录态和 PostgreSQL 后,再显式正式执行:
|
||||
天猫视频上传注册为幂等定时工作流 `product.video_upload`,按 `config/schedules.json` 以 2026-09-04 为起始日每 3 天 14:00(Asia/Shanghai)执行;前端仍可手动选择“光影行星旗舰店 / 光影行星鑫华达专卖店:龙虾仔 / 天猫新店(gyxx箱包旗舰店:龙虾仔)”并立即运行,也可填写天猫话题完整名称或关键词。正式执行只处理所选店铺,话题搜索结果按包含关系选中。默认 dry-run 只生成运行记录并跳过浏览器节点;确认飞书待传记录、对应淘宝光合店铺的登录态和 PostgreSQL 后,再显式正式执行:
|
||||
|
||||
```bash
|
||||
uv run gyxx run product.video_upload --date 2026-08-07
|
||||
uv run gyxx run product.video_upload --date 2026-08-07 --execute
|
||||
```
|
||||
|
||||
前端把店铺选择作为受限运行参数传给脚本,只允许 `flagship` 或 `luggage`。脚本先读取飞书并应用所选店铺路由,再写入/读取附件级账本;只有存在可领取的未上传附件时才启动有头 Chrome、登录对应账号并上传。旗舰店只选择“天猫上传”未勾选且有附件的光影行星男/女记录;箱包店只选择女性、全部款式命中箱包店白名单且有附件的记录。两店分别使用 `GUANGHE_USERNAME` / `GUANGHE_PASSWORD` 和 `GUANGHE_LUGGAGE_USERNAME` / `GUANGHE_LUGGAGE_PASSWORD`,箱包店当前账号为 `光影行星鑫华达专卖店:龙虾仔`,密码配置保持不变,并保留独立浏览器 Profile。发生登录或页面错误时保留窗口供人工定位,排查完成后手动关闭。
|
||||
前端把店铺选择作为受限运行参数传给脚本,只允许 `flagship`、`luggage` 或 `new_bag`。脚本先读取飞书并应用所选店铺路由,再写入/读取附件级账本;只有存在可领取的未上传附件时才启动有头 Chrome、登录对应账号并上传。旗舰店只选择“天猫上传”未勾选且有附件的光影行星男/女记录;箱包店只选择女性、全部款式命中箱包店白名单且有附件的记录;天猫新店不区分性别,只要求一条记录的**全部**「代拍产品」款式都属黑曜石系列(关键词 `黑曜石`,如黑曜石托特、黑曜石吐司、黑曜石电脑包、黑曜石饺子),组合款里掺了其它系列整条记录都不进新店,视频源与其它店铺为同一张飞书表。三店分别使用 `GUANGHE_USERNAME` / `GUANGHE_PASSWORD`、`GUANGHE_LUGGAGE_USERNAME` / `GUANGHE_LUGGAGE_PASSWORD` 和 `GUANGHE_NEW_BAG_USERNAME` / `GUANGHE_NEW_BAG_PASSWORD`,箱包店当前账号为 `光影行星鑫华达专卖店:龙虾仔`,天猫新店当前账号为 `gyxx箱包旗舰店:龙虾仔`,密码配置保持不变,各自保留独立浏览器 Profile(`-luggage` / `-new-bag`)。发生登录或页面错误时保留窗口供人工定位,排查完成后手动关闭。
|
||||
|
||||
每次正式扫描都会把视频附件写入 PostgreSQL 附件级发布账本,防重身份由视频 `file_token + 目标账号 + 飞书记录` 构成。账本状态为 `published` 的附件在下次执行时自动跳过,因此同一业务日期可以安全重跑并发现新增附件。旗舰店仍只在一条飞书记录的全部附件发布成功后回写“天猫上传”;箱包店不借用该勾选状态,是否已上传完全以自己的目标账号账本为准。
|
||||
每次正式扫描都会把视频附件写入 PostgreSQL 附件级发布账本,防重身份由视频 `file_token + 目标账号 + 飞书记录` 构成。账本状态为 `published` 的附件在下次执行时自动跳过,因此同一业务日期可以安全重跑并发现新增附件。旗舰店仍只在一条飞书记录的全部附件发布成功后回写“天猫上传”;箱包店和天猫新店不借用该勾选状态,是否已上传完全以自己的目标账号账本(`guanghe-luggage` / `guanghe-new-bag`)为准,同一视频可以分别发到多家店铺。
|
||||
|
||||
首次正式执行会读取 `config/tmall-video-publication-backfill.json`,将 2026-07-26 箱包店旧日志中可按 `record_id + file_token + 文件名` 再次核验的 25 个明确成功附件回填为 `published`。配置与当前飞书任一身份不一致都会停止,不会静默猜测。该回填不依赖部署机保留旧 `var/` 日志文件;证据路径与成功行号已固化在配置中供审计。
|
||||
|
||||
@@ -103,11 +103,17 @@ uv run gyxx scripts run product.jd_video_upload.run --date 2026-08-07 --arg=--re
|
||||
|
||||
唯一工作流 ID 是 `product.ecommerce_costs.daily`,每天 10:00 由项目内 Python 调度器运行,
|
||||
业务日期为前一天;天猫、京东、抖音三个下载分支并行启动,各自完成下载后再导入。天猫流程先进入万相台商品报表选择昨日,提交下载任务;任务在下载列表显示“生成成功”
|
||||
后下载 ZIP、做路径安全校验并解压 CSV,随后将商品级广告指标幂等写入
|
||||
`product_daily_metrics.raw_data.tm_ad_metrics`。商品款式先按 `dim_style.tm_spus` 的商品 ID 匹配,
|
||||
匹配不到再按商品名称做精确包含匹配,并在节点内保留 `style_name` 与
|
||||
`style_match_method`(`product_id`/`product_name`/`ambiguous`/`unmatched`)。它不会覆盖同一商品日期
|
||||
已有的访客、成交等经营日报字段;data-hub 产品生命进程从该节点读取广告成交金额和花费并计算投产比、电商费比。
|
||||
后下载 ZIP、做路径安全校验并解压 CSV,随后将商品级广告指标幂等写入独立表
|
||||
`tm_product_ad_daily_metrics`(每日每商品一行,主键 `(stat_date, product_id)`)。花费、直接/间接/归因
|
||||
成交金额和投产比走标量列,报表其余指标(展现、点击、点击率、转化率、预售成交等)原样存在 `metrics`
|
||||
JSONB。商品款式先按 `dim_style.tm_spus` 的商品 ID 匹配,匹配不到再按商品名称做精确包含匹配,结果
|
||||
写进 `style_name` 与 `style_match_method`(`product_id`/`product_name`/`ambiguous`/`unmatched`)。
|
||||
天猫广告数据**不再写入** `product_daily_metrics.raw_data.tm_ad_metrics`,投放与经营日报分表,经营数据
|
||||
重新同步不会覆盖投放数据;data-hub 产品生命进程从新表读取花费和广告归因成交金额并计算投产比、电商费比。
|
||||
|
||||
历史数据迁移(2026-11-01 一次性执行):`src/gyxx_flow/modules/product_commerce/scripts/
|
||||
migrate_tm_ad_metrics_to_table.py`,默认 dry-run,`--execute` 先备份旧节点到运行时 TMP_ROOT 再回填,
|
||||
校验行数与花费合计一致后才允许 `--clear-source` 清除 `raw_data` 里的旧节点。
|
||||
|
||||
正式服务必须注入 `WANXIANG_ACCOUNT`、`WANXIANG_PASSWORD`、`GYXX_DATA_ROOT` 和
|
||||
`GYXX_POSTGRES_DSN`。默认浏览器 Profile 是
|
||||
@@ -139,8 +145,9 @@ uv run gyxx scripts run product.import.tmall_ads --date 2026-08-17 --execute
|
||||
1. 检查 `gyxx schedule status`、服务日志和对应 `run_id`,确认失败发生在申请、下载、解压还是入库。
|
||||
2. 检查同一 Profile 是否仍能打开万相台商品报表;登录失效时在有头命令中人工完成验证。
|
||||
3. 检查日期目录中的 `manifest.json`、CSV 表头和 `日期/主体ID/主体名称`;ZIP 存在不代表数据库已导入。
|
||||
4. 检查 PostgreSQL 的 `product_daily_metrics` 是否存在 `platform='tm'`、目标日期和商品 ID,且
|
||||
`raw_data->'tm_ad_metrics'` 已更新;确认经营日报其他 JSON 节点仍保留。
|
||||
4. 检查 PostgreSQL 的 `tm_product_ad_daily_metrics` 是否存在目标日期与商品 ID 的行,`spend`、
|
||||
`attributed_deal_amount` 是否为目标值,`style_name` 是否已匹配;确认 `product_daily_metrics`
|
||||
同日期经营数据(访客、成交)仍完整。
|
||||
5. 只有在外部状态核对清楚后才按同一业务日期重跑,不能用换日期或重复申请掩盖下载结果不明。
|
||||
|
||||
## 调度操作
|
||||
@@ -286,6 +293,215 @@ uv run gyxx effects reconcile <workflow_id> \
|
||||
|
||||
浏览器状态异常时,只备份并处理目标脚本的 `state/browser/<module>/<script>`,不要清理其他脚本状态。调度状态异常时先停止唯一调度进程并备份 `state/scheduler`,不要同时启动第二实例。
|
||||
|
||||
## 商品日报采集开关(product.daily)
|
||||
|
||||
采集开关是 `dim_style.collect_enabled`(2026-10-29 起),**两层语义**(2026-10-30 起):
|
||||
|
||||
1. **锚点是系列总开关**:以该系列 `is_series_anchor = TRUE` 那一行为准;系列没有锚点时按款名
|
||||
排序第一个。锚点 `FALSE` → **整条系列一款都不采**,同系列其余款式跟随锚点。
|
||||
2. **锚点 `TRUE` 时,子款开关只决定自己的 ERP 编码是否参与聚合**:子款 `TRUE` 的 ERP 款式编码
|
||||
并入锚点一起去聚水潭取数;子款 `FALSE` 的编码不并入(既不由它自己采,也不让锚点代采)。
|
||||
被排除的子款列在 `series.erp_excluded`,运行时打
|
||||
`[INFO] 系列「X」子款 Y 已关闭采集,ERP 编码不纳入「Z」聚合`。平台商品编码仍按整系列聚合,
|
||||
不受子款开关影响。
|
||||
|
||||
旧的独立表 `product_daily_collect_scope` 已迁移并删除,备份在
|
||||
`var/tmp/collect_scope_before_drop_20261029.json`。
|
||||
|
||||
默认值是 `TRUE`(采集),因此产品生命进程在 `dim_style` 新增的款式**无需任何操作**就会进入日报;
|
||||
配件、包装、耗材、停售款才需要显式关掉。
|
||||
|
||||
关掉整条系列(改锚点那一行即可,子款自动跟随):
|
||||
|
||||
```sql
|
||||
-- 单系列
|
||||
UPDATE dim_style SET collect_enabled = FALSE WHERE style_name = '黑曜石托特包2';
|
||||
|
||||
-- 批量:把若干系列的所有行一起关(无锚点的系列也覆盖到)
|
||||
UPDATE dim_style SET collect_enabled = FALSE
|
||||
WHERE COALESCE(NULLIF(btrim(series_name), ''), style_name)
|
||||
IN ('飞机盒', '香片', '雨衣', '鞋袋');
|
||||
```
|
||||
|
||||
恢复采集把 `FALSE` 改成 `TRUE`。改完无需重启控制台或调度器:`StyleConfigLoader.load()`
|
||||
每次运行都直接读库,下一轮 `product.daily`(每天 08:40,业务日 T-1)即生效。
|
||||
|
||||
两点注意:
|
||||
|
||||
- **开关落在锚点行上才控制整条系列。** ERP 编码与三平台商品编码都由
|
||||
`_apply_series_sharing()` 按整系列聚合并去重后挂到锚点(子款误填的编码也会并入,但
|
||||
`collect_enabled = FALSE` 的子款 ERP 编码除外),锚点是系列唯一的采集单元;子款自己的
|
||||
编码保留在 `series.own_erp_codes` 与 `series.platform_ids` 备查。
|
||||
- 关掉只影响**商品日报**(tm/jd/dy 平台段 + ERP 段)。天猫/京东主图周采集、聚水潭全店铺日报、
|
||||
三平台电商费用**不受此开关影响**。
|
||||
|
||||
放行不等于一定会采:采集门仍要求「ERP 款式编码 + 各平台商品编码」齐备;飞书目标表地址
|
||||
(下一节的 9 类 `bitable_*` 列)只决定能否写飞书,缺地址的款照常采集入库。缺口写在
|
||||
`var/logs/product_commerce/<date>_skipped_styles.json` 的 `skipped.reason`。
|
||||
|
||||
改款式名时要同步改 `series_name`/锚点标记,否则旧开关会变成孤儿行、新名字按默认 TRUE 采集。
|
||||
|
||||
## 飞书表地址维护(dim_style)
|
||||
|
||||
2026-10-29 起,9 类飞书多维表格地址**全部维护在 `dim_style`**,而且只写在系列锚点行上。
|
||||
`workflow_dynamic_styles` / `workflow_dynamic_style_platforms` 里的同名字段已废弃,
|
||||
代码不再读取;旧值备份在 `var/tmp/bitable_urls_before_move_20261029.json`。
|
||||
|
||||
| 用途 | `dim_style` 列 | 原动态配置字段 |
|
||||
|---|---|---|
|
||||
| 销量表 | `bitable_weekly_url` | `sales_bitable_url` |
|
||||
| 主图表 | `bitable_main_image_url` | `main_image_bitable_url` |
|
||||
| 合作达人表 | `bitable_creator_url` | `creator_bitable_url` |
|
||||
| 自营合作达人表 | `bitable_self_creator_url` | `self_creator_bitable_url` |
|
||||
| 每周/每月笔记分析 | `bitable_weekly_note_url` | `weekly_note_analysis_url` |
|
||||
| 平台单品分析 | `bitable_style_analysis_url` | `style_analysis_bitable_url` |
|
||||
| 人群画像(天猫) | `bitable_persona_tm_url` | `tm_persona_bitable_url` |
|
||||
| 人群画像(京东) | `bitable_persona_jd_url` | `jd_persona_bitable_url` |
|
||||
| 人群画像(抖音) | `bitable_persona_dy_url` | `dy_persona_bitable_url` |
|
||||
|
||||
子款留空即可:两处代码会把整条系列的地址**聚合去重**后统一下发——
|
||||
`workflow_config._propagate_series_targets()` 在合并建档数据时按「锚点优先、其余按款名」
|
||||
收集整系列的非空地址,去重后写回每一行;`style_config_loader._apply_series_sharing()`
|
||||
在聚合配置时再做一次同样的兜底。因此子款误填的地址不会丢、也不会让同系列出现两套表。
|
||||
一个表位只能落一张表:同一字段填了多个不同地址时,保留优先级最高的那条,并在日志里打
|
||||
`[WARN] 系列「X」<字段> 有 N 个不同表`,提示到锚点手动裁决。
|
||||
|
||||
```sql
|
||||
-- 改某个系列的主图表(只改锚点一行,子款自动跟随)
|
||||
UPDATE dim_style
|
||||
SET bitable_main_image_url = 'https://xxx.feishu.cn/base/<base>?table=<tbl>'
|
||||
WHERE style_name = '逐星'; -- 必须是该系列的锚点行
|
||||
|
||||
-- 查看哪些锚点还缺地址
|
||||
SELECT style_name, series_name,
|
||||
bitable_weekly_url, bitable_main_image_url, bitable_creator_url
|
||||
FROM dim_style
|
||||
WHERE is_series_anchor AND COALESCE(bitable_weekly_url, '') = ''
|
||||
ORDER BY style_name;
|
||||
```
|
||||
|
||||
改完不需要重启:`StyleConfigLoader.load()` 每次运行都读库,下一轮工作流即生效。
|
||||
若某类地址为空,对应环节会跳过写表但照常采集入库(例如主图缺表地址打 `[NO-TABLE]`)。
|
||||
|
||||
**地址读取是 fail-closed 的**(2026-10-30):写表脚本拿不到 `dim_style` 地址时会打
|
||||
`[FAIL] … 拒绝回退本地 JSON` 并以退出码 1 结束,**不会**改用 `bitable_style_map.json` /
|
||||
`bitable_main_image_map.json` 里的旧地址。所以看到这类 `[FAIL]` 先去查库里的地址,
|
||||
不要去改本地 JSON(这两个文件已无读取方)。
|
||||
|
||||
新增系列时,地址写在锚点行。若某款业务上确实是独立一套表(历史上 `逐星GT` 这种情形),
|
||||
必须把它拆成独立系列(单独的 `series_name` + 锚点标记)并把地址写在自己那行;只写地址
|
||||
而不拆系列的话,聚合会把它并入原系列,只在日志里留一条告警,不会真的生效。
|
||||
|
||||
## 款式主键统一(style_id = dim_style.id,2026-11-01)
|
||||
|
||||
采集端不再只用 `style_name` 关联款式。**唯一关联键是 `style_id`**(= `dim_style.id`,
|
||||
与 data-hub 的 22 张表同一套主键),`style_name` 降级为显示冗余。这样在 dim_style
|
||||
改款名时,历史指标表不会变成孤儿。
|
||||
|
||||
### 数据从哪里来
|
||||
|
||||
| 环节 | 机制 |
|
||||
|---|---|
|
||||
| loader 下发 | `merge_lifecycle_style_records()` 输出 `style_id` + `style_match_method`,经 `style_config_loader` 进入每条款式配置 |
|
||||
| db 层兜底 | `db.attach_style_ids()`(按款名)/ `db.resolve_style_ids_by_product()`(按平台商品 ID)在 upsert 内补齐 |
|
||||
| 历史回填 | `backfill_style_id.py` |
|
||||
|
||||
### 回填与验收
|
||||
|
||||
```powershell
|
||||
# 只报数,不写库
|
||||
.venv\Scripts\python.exe backfill_style_id.py --dry-run
|
||||
# 执行(幂等,只补 style_id IS NULL 的行)
|
||||
.venv\Scripts\python.exe backfill_style_id.py
|
||||
# 只补款名表,跳过按商品 ID 的两步
|
||||
.venv\Scripts\python.exe backfill_style_id.py --skip-product-table --skip-ad-tables
|
||||
```
|
||||
|
||||
回填分三段,缺一段覆盖率就上不去:
|
||||
|
||||
| 段 | 表 | 依据 |
|
||||
|---|---|---|
|
||||
| 按款名 | 13 张(`daily_style_metrics` / `weekly_style_metrics` / `erp_all_shop_style_daily_metrics` 等) | `t.style_name = dim_style.style_name` |
|
||||
| 按商品 ID | `product_daily_metrics` | `platform + product_id` 反查 `*_spus` |
|
||||
| 按商品 ID(二次) | 三张广告表 | 同上;**顺带把空的 `style_name` 补成 dim_style 的权威款名** |
|
||||
|
||||
广告表必须走第三段:平台的报表不给款式名,`jd_product_ad_daily_metrics` 有 27%、
|
||||
`dy_product_ad_daily_metrics` 有 40% 的行 `style_name` 为空,只按款名回填完全无效。
|
||||
补完后 data-hub 现有按款名的 join 立刻多认回一批行,不必等主键切换。
|
||||
|
||||
```sql
|
||||
-- 各表回填进度;除 product_daily_metrics 与三张广告表外都应 100%
|
||||
SELECT * FROM v_style_id_coverage ORDER BY tbl;
|
||||
-- 未建档的平台商品 ID,仅统计经营日报口径
|
||||
SELECT * FROM v_style_unmatched_product ORDER BY platform, rows DESC;
|
||||
-- 同上,但把三张广告表也算进来(纯投流商品只在这里出现)
|
||||
SELECT * FROM v_style_unmatched_source ORDER BY platform, rows DESC;
|
||||
```
|
||||
|
||||
`style_match_method` 取值:`dim_style`(按款名精确)、`product_id`(按平台商品 ID)、
|
||||
`product_name`(按商品名包含)、`unmatched`(确实没建档)。
|
||||
|
||||
### 补 spus 的正确做法
|
||||
|
||||
`product_daily_metrics.product_no` 存的是**款名**(不是 ERP 编码),天猫填充率 100%、
|
||||
京东 97.8%、抖音 0%。补 spus 时按这个优先级,命中率远高于猜商品标题:
|
||||
|
||||
1. `product_no` 精确等于 `dim_style.style_name`;
|
||||
2. `product_no` 与款名互为前后缀(长度差 ≤ 3,如「蓝鹊pro」→「蓝鹊」、「知更大号」→「知更」);
|
||||
3. 商品标题包含完整款名(抖音只能用这一条),并用「命中位置后的版本后缀」排除串款
|
||||
—— 例如标题「盖亚斜挎包**3**」不能归到「盖亚斜挎」;
|
||||
4. 编码与标题互相矛盾时**不要自动写**,列出来人工定(实测有天猫单品编码写「理岩」、
|
||||
标题写「阿波罗 air」,支付额 50 万,猜错代价极大)。
|
||||
|
||||
物料类(数据线、内胆包、补邮费、配件包、礼品袋、定制服务)不该进 `spus`,应留
|
||||
`unmatched`。`dim_style` 里本身建成款的物料(`配件包`、`飞机盒`、`电泵` 等)也不写。
|
||||
|
||||
### 必须知道的边界
|
||||
|
||||
- `product_daily_metrics` 表里**没有款名列**,只能拿 `platform + product_id` 去
|
||||
`tm_spus / jd_spus / dy_spus` 反查。未建档的商品 ID 回填不了,要靠建档页补 `spus`
|
||||
才能涨,不是脚本 bug。
|
||||
- 2026-11-01 补过一轮:`product_daily_metrics` 49.1% → **86.0%(jd)/ 77.3%(tm)/
|
||||
76.7%(dy)**,整体 79.0%;广告表 tm 99.4% / jd 92.4% / dy 83.2%。
|
||||
剩余 259 个商品 ID(dy 107 / jd 52 / tm 100)未建档,清单见 `v_style_unmatched_source`。
|
||||
- **不建物理外键**:14 张表的 `style_id` 都只做**逻辑外键**(与 data-hub 侧一致)。
|
||||
库里没有任何指向 `dim_style(id)` 的 FK 约束,完整性靠写入端(loader 下发 +
|
||||
`attach_style_ids` 兜底)、`backfill_style_id.py --dry-run`、`v_style_id_coverage`
|
||||
三道保证。理由:采集语义是「宁可留 NULL 待回填,也不要整批写入失败」,
|
||||
物理外键会在传错 id 时直接中断整天采集;此外拆系列/合并同款要改数据时,
|
||||
还得先删约束。早期 P4 建过的 10 个 `<表名>_style_id_fk` 已在 schema.sql 里
|
||||
幂等 `DROP CONSTRAINT IF EXISTS` 清掉,老库重跑 schema 即自动回到无 FK 状态。
|
||||
- 主键一律没动:`daily_style_metrics` 仍是 `(metric_date, style_name, platform)`。
|
||||
data-hub 有 6 处按 `style_name` join,改主键收益小、回归面大。
|
||||
|
||||
## 商品评价采集下线(product_reviews,2026-11-01)
|
||||
|
||||
`product_reviews` 已**停止写入并归档**为 `product_reviews_archived_20261101`。
|
||||
原因是 JD 侧 `style_name` 全靠 `sku_id` 反查,而映射表不存在,214 行全部为 NULL,
|
||||
两个消费方都是 `WHERE style_name = ?`,等于入库了读不到。
|
||||
|
||||
差评统一由三个独立工作流采集:`shop.tmall_negative_reviews.daily`、
|
||||
`shop.jd_negative_reviews.daily`、`shop.dy_negative_reviews.daily`。
|
||||
|
||||
### 消费方改读视图
|
||||
|
||||
```sql
|
||||
SELECT platform, comment_date, rating, content, style_id, style_name, style_match_method
|
||||
FROM v_negative_reviews; -- tm + jd + dy 三张差评表 UNION,款式按 ID 优先、商品名兜底
|
||||
```
|
||||
|
||||
改动点:`analyze_style.py:pull_reviews`、`content_marketing/daily_marketing_report.py:_review_context`。
|
||||
|
||||
### 口径变化(必须知道)
|
||||
|
||||
- 数据源**只有差评一档**,不再有好评/中评。风格分析的「评价等级分布」因此只剩差评。
|
||||
- 营销日报的 `sample_size` = 差评条数,`negative_rate` 恒为 1.0;门槛从「至少 5 条评价」
|
||||
改成「至少 1 条差评」(旧门槛在纯差评源上会导致永不产出)。
|
||||
- 覆盖面反而更全:天猫差评 97 条(旧表天猫为 0)、京东 32 条(旧表 8 月断更)。
|
||||
- 若产品侧确实需要好评样本,应把差评采集器扩成「按等级采集」,**不要恢复 product_reviews**。
|
||||
- `dy_product_scraping.py` 默认流程里的评论采集已摘除(只产出无人读取的本地 JSON);
|
||||
需要历史样本时用 `--reviews-only` 显式触发。
|
||||
|
||||
## 业务源码同步
|
||||
|
||||
外部业务源码目录只用于发现更新,GYXX Flow 运行时不导入或挂载这些目录。只读检查:
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
# 款式主键统一 + 评论采集下线 方案
|
||||
|
||||
> 状态:**已实施(2026-11-01)**,见文末「八、实施记录」
|
||||
> 涉及项目:`gyxx-flow`(采集端)、`data-hub`(消费端,共用同一个 PostgreSQL)
|
||||
> 编写日期:2026-11-01
|
||||
|
||||
## 一、三条结论
|
||||
|
||||
1. **`product_reviews` 可以下线**,但不能直接删——它有两个消费方(风格分析、营销日报)要改成读三张差评表。
|
||||
2. **`style_id` 就是 `dim_style.id`**,与 data-hub 现有 `style_id` 同一体系,不会产生第二套 ID。已实测:`erp_product.style_id` 的 135 个值 **135/135** 命中 `dim_style.id`,`own_brand_product.style_id` 31/34 命中(3 条为历史脏数据)。
|
||||
3. **改造入口只有一处**:`DIM_STYLE_RUNTIME_SQL` 早已 `SELECT s.id`,`merge_lifecycle_style_records()` 也已经拿到这个 id——只是被编码进 `source_record_id = "lifecycle_style:<id>"` 字符串里,没有作为独立字段下发给写入脚本。把它提成正式字段,全链路就都有了。
|
||||
|
||||
## 二、现状盘点
|
||||
|
||||
### 2.1 两套引用方式并存
|
||||
|
||||
| 侧 | 关联方式 | 表 |
|
||||
|---|---|---|
|
||||
| data-hub(产品生命、营销中台、客服、内容) | `style_id BIGINT` → `dim_style.id` | `erp_product`、`own_brand_product`、`erp_marketing_*`、`erp_cs_*`、`cmt_*`、`market_product_classification` 等 22 张 |
|
||||
| gyxx-flow(采集端) | `style_name TEXT` | 15 张,见 2.3 |
|
||||
|
||||
### 2.2 实测匹配率(2026-11-01)
|
||||
|
||||
| 表 | 总行数 | 按 style_name 命中 dim_style |
|
||||
|---|---|---|
|
||||
| `daily_style_metrics` | 17,352 | 17,352(100%) |
|
||||
| `erp_all_shop_style_daily_metrics` | 9,399 | 9,399(100%) |
|
||||
| `product_reviews` | 1,936 | 1,434(74%,全部来自抖音;京东 214 行 style_name 全 NULL) |
|
||||
| `product_daily_metrics` | 75,568 | **无款式列**,只能靠 `tm_spus/jd_spus/dy_spus @> to_jsonb(product_id)` 现算 |
|
||||
|
||||
`product_daily_metrics` 按商品 ID 反查的现状:
|
||||
|
||||
| 平台 | 行数 | 可匹配 | 去重商品 ID | 可匹配 ID |
|
||||
|---|---|---|---|---|
|
||||
| dy | 28,972 | 13,615(47%) | 253 | 83 |
|
||||
| jd | 17,044 | 9,394(55%) | 160 | 61 |
|
||||
| tm | 29,552 | 14,089(48%) | 281 | 92 |
|
||||
|
||||
一对多歧义 **0 条**。约一半行匹配不上,原因是 **458 个商品 ID 从未写进 dim_style 的 spus 数组**(dy 170 / jd 99 / tm 189),不是字段设计问题。
|
||||
|
||||
### 2.3 采集端 15 张只存 style_name 的表
|
||||
|
||||
`daily_style_metrics`、`weekly_style_metrics`、`erp_all_shop_style_daily_metrics`、`daily_persona_metrics`、`main_image_creatives`、`fact_style_analysis_reports`、`decline_events`、`fact_alert_history`、`jd_self_weekly_sales`、`persona_screenshots`、`product_reviews`、`jd_product_ad_daily_metrics`、`dy_product_ad_daily_metrics`、`workflow_dynamic_styles`、`product_daily_metrics`(无列,靠 ID 反查)。
|
||||
|
||||
## 三、目标模型
|
||||
|
||||
- **唯一关联键 = `style_id`(`dim_style.id`)**,所有采集表落库时必须带上。
|
||||
- `style_name` 保留为**显示冗余**,不再作为 join 键;等下游切完 join 后可以只在视图里出现。
|
||||
- 匹配不上的行(未建档商品、广告未匹配 SKU、差评)**允许 `style_id IS NULL`**,但必须同时写 `style_match_method`(`product_id` / `product_name` / `anchor` / `unmatched`),便于统计漏维护。
|
||||
- **不动现有主键**。`daily_style_metrics` 的 PK 仍是 `(metric_date, style_name, platform)`——data-hub 有 6 处 `metric.style_name = style.name` 的 join,改 PK 收益小、回归面大。加列、加索引、切 join 即可。
|
||||
|
||||
## 四、实施分期
|
||||
|
||||
### P0 观测层(只读,不动数据)
|
||||
|
||||
新增两个视图,先把问题量化出来,方案落地后靠它们验收:
|
||||
|
||||
```sql
|
||||
-- 未建档的平台商品 ID(推动在 dim_style 里补 spus)
|
||||
CREATE OR REPLACE VIEW v_style_unmatched_product AS
|
||||
SELECT m.platform, m.product_id,
|
||||
MAX(m.product_name) AS product_name,
|
||||
MIN(m.stat_date) AS first_date, MAX(m.stat_date) AS last_date,
|
||||
COUNT(*) AS rows
|
||||
FROM product_daily_metrics m
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM dim_style d
|
||||
WHERE (m.platform = 'tm' AND d.tm_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'jd' AND d.jd_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'dy' AND d.dy_spus @> to_jsonb(m.product_id)))
|
||||
GROUP BY m.platform, m.product_id;
|
||||
|
||||
-- 各表 style_id 回填进度
|
||||
CREATE OR REPLACE VIEW v_style_id_coverage AS
|
||||
SELECT 'daily_style_metrics' AS tbl, COUNT(*) AS rows,
|
||||
COUNT(style_id) AS with_style_id FROM daily_style_metrics
|
||||
UNION ALL SELECT 'erp_all_shop_style_daily_metrics', COUNT(*), COUNT(style_id)
|
||||
FROM erp_all_shop_style_daily_metrics
|
||||
UNION ALL SELECT 'product_daily_metrics', COUNT(*), COUNT(style_id)
|
||||
FROM product_daily_metrics;
|
||||
```
|
||||
|
||||
验收锚点:`v_style_id_coverage` 里前三张表 `with_style_id` 开始上涨到预期比例;`v_style_unmatched_product` 的条数随建档推进下降。
|
||||
|
||||
### P1 评论采集链路下线
|
||||
|
||||
**要删的东西**
|
||||
|
||||
| 位置 | 动作 |
|
||||
|---|---|
|
||||
| `orchestrate_daily_collection.py:437-452` | 移除 `import_product_reviews` 步骤 |
|
||||
| `import_product_reviews.py` | 删除 |
|
||||
| `orchestrate_review_collection.py` | 删除 |
|
||||
| `config/commands.json:516` `product.review.orchestrate` | 移除命令 |
|
||||
| `db/__init__.py` `upsert_product_reviews` | 删除 |
|
||||
| `db/schema.sql:813-870` `product_reviews` 建表段 | 移除(库里的表归档不删) |
|
||||
|
||||
**要补的东西**
|
||||
|
||||
1. 建统一差评视图,字段对齐两个消费方现在的用法(`platform` / `rating` / `content` / `style_id` / `comment_date`):
|
||||
|
||||
```sql
|
||||
CREATE OR REPLACE VIEW v_negative_reviews AS
|
||||
SELECT 'tm' AS platform, n.business_date AS comment_date, n.review_level AS rating,
|
||||
n.content, NULL::text AS order_id, NULL::text AS product_id,
|
||||
n.product_name, n.review_id, n.review_at, n.raw_review
|
||||
FROM tmall_negative_reviews n
|
||||
UNION ALL
|
||||
SELECT 'jd', n.business_date, n.review_level, n.content, n.order_id, NULL,
|
||||
n.product_name, n.review_id, n.review_at, n.raw_review
|
||||
FROM jd_negative_reviews n
|
||||
UNION ALL
|
||||
SELECT 'dy', n.business_date, n.rating_level, n.content, n.order_id, n.product_id,
|
||||
n.product_name, n.review_id, n.review_at, n.raw_review
|
||||
FROM dy_negative_reviews n;
|
||||
```
|
||||
|
||||
2. 改两个消费方:
|
||||
- `analyze_style.py:112 pull_reviews()` → 改查 `v_negative_reviews`(按 `style_id` 或 `product_id` 关联)
|
||||
- `content_marketing/daily_marketing_report.py:1020 _review_context()` → 同上
|
||||
|
||||
3. `product_reviews` 归档而不是删除:
|
||||
|
||||
```sql
|
||||
ALTER TABLE product_reviews RENAME TO product_reviews_archived_20261101;
|
||||
```
|
||||
|
||||
**必须明确的功能退化**:`product_reviews` 存的是好/中/差全量,差评三表只有差评。下线后:
|
||||
- 风格分析里的「评价等级分布」从三档变成只有差评一档(差评样本反而更完整:天猫 97 + 京东 32 + 抖音 14,而 `product_reviews` 里天猫是 0、京东 8 月就断了)。
|
||||
- 营销日报的「近 7 天差评归因」不受影响,反而多了天猫来源。
|
||||
- 若产品侧确实需要好评样本沉淀,需要把三个差评采集器扩成「按等级采集」,不要恢复 `product_reviews`。
|
||||
|
||||
另外 `dy_product_scraping.py` 主流程里的评论采集动作只产出本地 JSON,`product_reviews` 停写后就没有消费方,建议同步关掉(或保留但不再进 `product.daily`)。
|
||||
|
||||
### P2 loader 下发 style_id(1 处改动,收益最大)
|
||||
|
||||
`src/gyxx_flow/adapters/workflow_config.py:322 merge_lifecycle_style_records()`:
|
||||
|
||||
```python
|
||||
output["style_id"] = lifecycle.get("id") # dim_style.id
|
||||
output["style_match_method"] = "dim_style"
|
||||
```
|
||||
|
||||
同时 `source_record_id` 保留(控制台/账本还在用),只是不再承担传 ID 的职责。
|
||||
|
||||
影响面:所有通过 `runtime_workflow_configs()` 取款的脚本自动获得 `style_id`,包括日报聚合、周报、ERP 全店铺、人像、主图、声明式分析。这一步不改表、不改写入逻辑,可以先合入验证 loader 输出。
|
||||
|
||||
### P3 采集表加列 + 回填 + 写入端填充
|
||||
|
||||
**统一 DDL 模板**(以 `daily_style_metrics` 为例,其余同构):
|
||||
|
||||
```sql
|
||||
ALTER TABLE daily_style_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
UPDATE daily_style_metrics d SET style_id = s.id
|
||||
FROM dim_style s WHERE s.style_name = d.style_name AND d.style_id IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS daily_style_metrics_style_date_idx
|
||||
ON daily_style_metrics (style_id, metric_date DESC);
|
||||
```
|
||||
|
||||
按表处理(**必须匹配**的加 `NOT NULL`,**允许未匹配**的保持可空):
|
||||
|
||||
| 表 | 款式来源 | style_id | 写入端改动 |
|
||||
|---|---|---|---|
|
||||
| `daily_style_metrics` | loader 款式清单 | NOT NULL | `aggregate_daily_final.py` + `db.upsert_daily_metrics` |
|
||||
| `weekly_style_metrics` | 从 daily 聚合 | NOT NULL | `weekly_aggregate.py` |
|
||||
| `erp_all_shop_style_daily_metrics` | loader ERP 款 | NOT NULL | `backfill_erp_all_shop_daily.py` |
|
||||
| `daily_persona_metrics` | loader | NOT NULL | `collect_*_persona_to_bitable.py` |
|
||||
| `main_image_creatives` | loader | NOT NULL | `insert_main_image_records.py` / `insert_jd_main_image_records.py` |
|
||||
| `fact_style_analysis_reports` | loader | NOT NULL | `analyze_style.py` |
|
||||
| `decline_events` / `fact_alert_history` | daily 派生 | NOT NULL | `check_nine_day_decline.py` |
|
||||
| `jd_self_weekly_sales` | loader | NOT NULL | `jd_self_inventory_sales_collector.py` |
|
||||
| `persona_screenshots` | loader | NOT NULL | `taobao_dmp_item_crowd_insight_screenshots.py` |
|
||||
| `product_daily_metrics` | 平台商品 ID 反查 | **可空** | `import_product_daily.py` |
|
||||
| `jd_product_ad_daily_metrics` / `dy_product_ad_daily_metrics` | 名称/ID 反查 | **可空**(已有 `match_status`) | 两个 `import_*_ad_*.py` |
|
||||
| 差评三表 | 反查 | **可空** | `collectors/*_negative_reviews.py` 入库侧 |
|
||||
|
||||
**`product_daily_metrics` 的回填要处理系列聚合带来的歧义**(当前 0 条歧义,但要按锚点优先写死优先级):
|
||||
|
||||
```sql
|
||||
UPDATE product_daily_metrics m
|
||||
SET style_id = pick.style_id
|
||||
FROM LATERAL (
|
||||
SELECT d.id AS style_id
|
||||
FROM dim_style d
|
||||
WHERE (m.platform = 'tm' AND d.tm_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'jd' AND d.jd_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'dy' AND d.dy_spus @> to_jsonb(m.product_id))
|
||||
ORDER BY d.is_series_anchor DESC, LENGTH(d.style_name), d.id
|
||||
LIMIT 1
|
||||
) pick
|
||||
WHERE m.style_id IS NULL;
|
||||
```
|
||||
|
||||
(真实脚本里用 Python 侧 `_resolve_style_match()` 复用同一套优先级,SQL 只用于一次性回填。)
|
||||
|
||||
回填后预期的非空比例 = 2.2 节的实测值:**dy 47% / jd 55% / tm 48%**。这不是 bug——剩下的是 458 个未建档商品 ID,要靠 P5 的人补 spus 才能涨。
|
||||
|
||||
### P4 约束 + 下游切 join
|
||||
|
||||
1. ~~对「必须匹配」的表加外键~~ → **改为不建物理外键(2026-11-01 定案)**。
|
||||
|
||||
原始设想是给 10 张「必须匹配」的表加 `FOREIGN KEY (style_id) REFERENCES dim_style(id) ON UPDATE CASCADE`,
|
||||
实施后又按业主意见拆掉了。理由是采集链路的语义与硬约束冲突:
|
||||
|
||||
- 采集要求 **fail-soft**:一个商品 ID 没建档,正确行为是把该行 `style_id` 留 NULL 并标
|
||||
`unmatched`,继续写其它行。物理外键会让整批 INSERT/UPDATE 直接报错,等于「一个商品
|
||||
未建档 → 当天数据全丢」,比留 NULL 更糟。
|
||||
- 款式会动:拆系列、合并同款、重建 dim_style 行都要先卸约束再动数据,运维成本不小。
|
||||
- data-hub 侧 22 张表本来就用 `style_id` 关联、**不带物理外键**,两边保持一致才好维护。
|
||||
|
||||
改为**逻辑外键**,完整性由三处保证:
|
||||
|
||||
| 环节 | 手段 |
|
||||
|---|---|
|
||||
| 写入 | loader 下发 `style_id`(源自 dim_style 本身);缺失时 `attach_style_ids` / `resolve_style_ids_by_product` 批量兜底 |
|
||||
| 校验 | `backfill_style_id.py --dry-run` 预演;`v_style_id_coverage` 报每张表覆盖率 |
|
||||
| 观测 | `v_style_unmatched_product`(日报口径)/ `v_style_unmatched_source`(全源口径)列未建档清单 |
|
||||
|
||||
老库里早期建过的 10 个 `<表名>_style_id_fk` 由 `schema.sql` 里的 `DROP CONSTRAINT IF EXISTS`
|
||||
幂等清理;只保留 `*_style_id_idx` 索引(join 性能靠索引,不靠约束)。
|
||||
|
||||
2. data-hub 侧把 `metric.style_name = style.name` 换成 `metric.style_id = style.id`。涉及 `ProductLifecycleMapper.xml` 第 120、155 行(`daily_style_metrics`),以及 `JOIN dim_style dim ON dim.style_name = style.name`(第 15 行)。改完后 text join 变 bigint join,是纯性能收益;`style_name` 列仍在,改错了可以立即回切。
|
||||
|
||||
3. 加测试:
|
||||
- `tests/.../test_daily_style_metrics_style_id.py`:写入必须带 `style_id`,且等于 loader 输出的 id
|
||||
- `tests/.../test_style_id_coverage.py`:`v_style_id_coverage` 里「必须匹配」的表 `with_style_id == rows`
|
||||
- 现有 `test_import_tmall_product_ads.py` 的风格(断言 SQL 形状)继续用在新增字段上
|
||||
|
||||
### P5 数据治理(不是代码活,但决定 P3 的收益)
|
||||
|
||||
`v_style_unmatched_product` 出来的 458 个商品 ID 需要在 dim_style 建档页补 `tm_spus/jd_spus/dy_spus`。分批推进:先补最近 90 天有销量的,历史行允许永久 NULL。
|
||||
|
||||
## 五、风险与取舍
|
||||
|
||||
| 风险 | 影响 | 应对 |
|
||||
|---|---|---|
|
||||
| 款名在 dim_style 被改,历史表 `style_name` 变孤儿 | 现在 join 直接丢数据,这正是要 style_id 的原因 | P3 之后 join 走 id,款名改动不再影响 |
|
||||
| 加 `NOT NULL` 时某天有未建档款写入 | 写入失败,工作流中断 | 先在写入端做「未匹配记 NULL + 打 WARN」,P5 补齐后再收紧为 NOT NULL |
|
||||
| `product_reviews` 下线后没有好评样本 | 风格分析评价分布退化 | 明确接受;需要时扩差评采集器等级,不恢复旧表 |
|
||||
| 改 PK(有人会想顺手做) | data-hub 6 处 join + 飞书脚本全回归 | **本次明确不做**,PK 保持 style_name |
|
||||
| 差评表天猫侧没有 product_id | 只能靠商品名反查款式 | 采集器补 product_id(页面有商品链接可解析),或退化为名称匹配并标记 `unmatched` |
|
||||
|
||||
## 六、验收标准
|
||||
|
||||
1. `v_style_id_coverage`:`daily_style_metrics`、`erp_all_shop_style_daily_metrics`、`weekly_style_metrics` 的 `with_style_id = rows`(100%)。
|
||||
2. `product_daily_metrics` 的 `style_id` 非空比例不低于 2.2 节的实测基线(47%/55%/48%),且逐月上升。
|
||||
3. `product_reviews` 无新增行;两个消费方在 `v_negative_reviews` 上跑通,风格分析与营销日报输出结构不变。
|
||||
4. data-hub 产品生命页、营销中台预算页的款式指标与改造前同月数值一致(对账 2026-10 全月)。
|
||||
|
||||
## 七、建议执行顺序
|
||||
|
||||
P0 → P1(下线评论链路,独立可交付)→ P2(loader 一行改动,验证输出)→ P3(分表灰度,先做 `daily_style_metrics`)→ P4(约束 + 下游切换)→ P5(补建档)。
|
||||
|
||||
P1 与 P2/P3 没有依赖关系,可以先做 P1 拿到收益(少一条采集链路、少一张表),再从容做款式主键统一。
|
||||
|
||||
## 八、实施记录(2026-11-01)
|
||||
|
||||
### 已落地
|
||||
|
||||
| 阶段 | 成果 |
|
||||
|---|---|
|
||||
| P0 | `v_style_unmatched_product`、`v_style_id_coverage` 两个观测视图已建并验证 |
|
||||
| P1 | 删 `import_product_reviews.py` / `orchestrate_review_collection.py`;移除 `product.review.orchestrate` 命令与 runtime binding;`run_collect` 不再调评论入库;建 `v_negative_reviews`;两个消费方改读视图;`product_reviews` 归档为 `product_reviews_archived_20261101`(1936 行已备份到 `var/tmp/product_reviews_archive_20261101.json`) |
|
||||
| P2 | `merge_lifecycle_style_records` 输出 `style_id` + `style_match_method`;loader 透传;实库验证 **204/204 款全部拿到 style_id** |
|
||||
| P3 | 14 张采集表加 `style_id`(+ `style_match_method`)+ 索引;db 层 `attach_style_ids` / `resolve_style_ids_by_product` 统一兜底;`backfill_style_id.py` 回填历史 |
|
||||
| P4 | ~~10 张「必须匹配」表加 `style_id → dim_style(id)` 物理外键~~ → 已按要求**全部拆除**,改逻辑外键(库里 0 个 FK,只留 14 个 `style_id` 索引);新增 `tests/modules/product_commerce/test_style_id_identity.py`(28 条契约测试) |
|
||||
| P5 | 提供 `v_style_unmatched_product` 清单(dy 170 / jd 99 / tm 189),需业务侧补 spus |
|
||||
|
||||
### 回填结果(实库)
|
||||
|
||||
| 表 | 行数 | 有 style_id | 比例 |
|
||||
|---|---|---|---|
|
||||
| daily_style_metrics | 17,352 | 17,352 | 100% |
|
||||
| weekly_style_metrics | 532 | 532 | 100% |
|
||||
| erp_all_shop_style_daily_metrics | 9,399 | 9,399 | 100% |
|
||||
| daily_persona_metrics | 2,357 | 2,357 | 100% |
|
||||
| main_image_creatives | 9,094 | 9,094 | 100% |
|
||||
| fact_style_analysis_reports | 465 | 465 | 100% |
|
||||
| decline_events / fact_alert_history | 14 / 14 | 14 / 14 | 100% |
|
||||
| persona_screenshots | 25 | 25 | 100% |
|
||||
| jd_self_weekly_sales | 117 | 111 | 94.9%(6 款已不在 dim_style) |
|
||||
| tm_product_ad_daily_metrics | 11,727 | 11,108 | 94.7% |
|
||||
| jd_product_ad_daily_metrics | 33,808 | 24,707 | 73.1% |
|
||||
| dy_product_ad_daily_metrics | 18,202 | 10,875 | 59.7% |
|
||||
| product_daily_metrics | 75,568 | 37,098 | 49.1%(未建档商品 ID 所致) |
|
||||
|
||||
### 与方案不同之处
|
||||
|
||||
1. **天猫广告表已先行迁到独立表**:方案编写时天猫指标还写在
|
||||
`product_daily_metrics.raw_data.tm_ad_metrics`,实施期间该迁移已完成,因此改为给
|
||||
`tm_product_ad_daily_metrics` 加 `style_id`(并纳入 `v_style_id_coverage`),
|
||||
并未改动 `raw_data` 节点。
|
||||
2. **写入端没有逐个脚本改**:方案原计划在 13 个写入脚本里各自填充,实际收敛到 db 层
|
||||
的 `attach_style_ids` / `resolve_style_ids_by_product`(一次批量查询),再加 loader
|
||||
下发。少改 13 处、不会漏,代价是 upsert 多一次按款名/商品 ID 的查询。
|
||||
3. **`dy_product_scraping.py` 默认流程的评论采集直接摘除**(仍保留 `--reviews-only`
|
||||
显式入口),比方案建议的「保留但不再进 product.daily」更彻底。
|
||||
4. **物理外键最终没有保留**:P4 建过的 10 个 `<表名>_style_id_fk` 已全部 DROP,
|
||||
`schema.sql` 改为「幂等清理 + 只建索引」。`style_id` 与 `dim_style.id` 是纯逻辑
|
||||
外键关系,理由见 P4 段(fail-soft 采集语义、款式可重建、与 data-hub 对齐)。
|
||||
|
||||
### 仍待处理
|
||||
|
||||
- **data-hub 侧 join 未切**(`ProductLifecycleMapper.xml` 6 处仍是 `metric.style_name = style.name`)。
|
||||
现在两边都有 id,可以随时切;切换后是 bigint join 替代 text join。
|
||||
- P5 补建档:458 个商品 ID(dy 170 / jd 99 / tm 189)写进 `dim_style.spus`。
|
||||
- 既有红灯(与本次无关):`test_tm_brand_profiles` 的 backfill 用例、`shop_intelligence`
|
||||
的 12 个 jd 竞店用例,改动前即失败。
|
||||
|
||||
## 九、P5 补 spus 实施记录(2026-11-01)
|
||||
|
||||
### 依据的权威字段
|
||||
|
||||
补 spus 的匹配依据不是商品标题,而是 **`product_daily_metrics.product_no` —— 它存的就是款名**:
|
||||
|
||||
| 平台 | `product_no` 填充率 | 可用性 |
|
||||
|---|---:|---|
|
||||
| 天猫 | 100%(29552/29552) | 权威 |
|
||||
| 京东 | 97.8%(16664/17044) | 权威 |
|
||||
| 抖音 | 0%(0/28972) | 只能靠标题 |
|
||||
|
||||
这一条此前被忽略,导致「458 个未建档」被误判为只能猜标题。实际天猫/京东 136 个商品
|
||||
靠 `product_no` 就精确命中,另有 31 个靠前后缀近似命中(「蓝鹊pro」→「蓝鹊」、
|
||||
「知更大号」→「知更」、「星迹」→「星迹2」)。
|
||||
|
||||
注意 `product_no` **不是** ERP 编码:`dim_style.erp_style_codes` 存的是 `["10510"]`
|
||||
这种数字码,两者用 `product_no = erp_style_codes` 只能对上 2 个,别混淆。
|
||||
|
||||
### 匹配算法(四路,按优先级)
|
||||
|
||||
1. `product_no` 精确 = `style_name` → 直接写;
|
||||
2. `product_no` 与款名互为前后缀且长度差 ≤ 3,且候选唯一 → 直接写;
|
||||
3. 标题包含完整款名(抖音只能走这条)→ 用「命中位置后的版本后缀」防串款:
|
||||
标题「盖亚斜挎包**3**」、`dim_style` 有「盖亚斜挎3」时必须指后者而非「盖亚斜挎」;
|
||||
若该版本款没建档,判「版本款未建档」而不是硬归到基础款;
|
||||
4. 编码与标题互相矛盾 → 进人工清单,**不自动写**。
|
||||
|
||||
实测第 4 类里最贵的一条:天猫 `649545919334` 商品编码写「理岩」、标题写「阿波罗air」,
|
||||
支付额 **50.2 万**。猜错就是 50 万挂错款,只能人工定。
|
||||
|
||||
### 写入结果
|
||||
|
||||
| 项 | 数值 |
|
||||
|---|---:|
|
||||
| 写入商品 ID | 199 个 |
|
||||
| 涉及款式 | 67 个 |
|
||||
| 覆盖 `product_daily_metrics` 行 | 22,636 |
|
||||
| 未写(物料 / 冲突待定) | 87 个 |
|
||||
|
||||
合并策略:`tm_spus / jd_spus / dy_spus` 取**并集追加**,不覆盖已有值;写前备份到
|
||||
`var/tmp/dim_style_spus_backup_20261101.json`(204 行)。
|
||||
|
||||
### 覆盖率变化
|
||||
|
||||
| 表 | 补前 | 补后 |
|
||||
|---|---:|---:|
|
||||
| product_daily_metrics | 49.1% | **79.0%** |
|
||||
| tm_product_ad_daily_metrics | 94.7% | **99.4%** |
|
||||
| jd_product_ad_daily_metrics | 73.1% | **92.4%** |
|
||||
| dy_product_ad_daily_metrics | 59.7% | **83.2%** |
|
||||
|
||||
广告表的提升来自新增的第三段回填(按商品 ID + 补空 `style_name`):京东广告里
|
||||
32 个商品 ID 就占了 9,101 行空值,补完 `style_name` 空白从 9,095 降到 2,554,
|
||||
抖音从 7,327 降到 3,062。这意味着 **data-hub 侧不改任何 SQL,按款名的 join 已经多认回
|
||||
11,359 行**。
|
||||
|
||||
### 剩余 259 个未建档 ID 的构成
|
||||
|
||||
| 分类 | 商品数 | 支付额 | 处置 |
|
||||
|---|---:|---:|---|
|
||||
| 待建档-真款 | 128 | 6.9 万 | 需在 dim_style 新建款(星云系列链接、拾影单肩包、幻月、纪元、极地冰冠、阿瑞斯、盖亚S1、漫游者、光/暗系列等) |
|
||||
| 待建档-ozko 线 | 20 | 0.2 万 | ozko 子品牌款(逐云、微笑系列、甜酷系列、银鸥、独角兽、HB、cdbb、761X…) |
|
||||
| 白名单-物料配件 | 30 | 5.6 万 | 不该进 spus(内胆包、数据线、风扇、防丢器…) |
|
||||
| 白名单-服务/券 | 4 | 0 | 补邮费、兑换券、补贴 |
|
||||
| 冲突待定 | 10 | 62.9 万 | 编码与标题矛盾,需人工定 |
|
||||
|
||||
清单:`var/tmp/spus_todo.csv`(按分档 + 金额排序)。
|
||||
|
||||
### 新增的长期机制
|
||||
|
||||
- `backfill_style_id.py` 增加 `AD_PRODUCT_TABLES` 与 `_backfill_ad_tables_by_product`,
|
||||
广告表按 `spu_id / product_id` 二次回填,并只在 `style_name` 为空时补权威款名
|
||||
(`COALESCE(NULLIF(BTRIM(style_name), ''), %s)`,不覆盖平台给出的值)。
|
||||
- 新增视图 `v_style_unmatched_source`:`product_daily_metrics` + 三张广告表统一口径,
|
||||
补建档只看这一个视图就够(实测与原 `v_style_unmatched_product` 数值一致,
|
||||
说明没有纯投流商品只出现在广告表)。
|
||||
@@ -1,89 +0,0 @@
|
||||
# 智能工作台设计(dsh-refact 分支)
|
||||
|
||||
> 状态:v1 已实现。本文档记录「以 deepseek-harness 为 agent 底座、gyxx-flow 工作流插件化」
|
||||
> 的架构决策与边界。使用说明见 [../workbench/README.md](../workbench/README.md)。
|
||||
|
||||
## 目标与约束
|
||||
|
||||
- **底座**:deepseek-harness(`dsh`)Web UI + DeepSeek 模型 = 智能体运行时;
|
||||
gyxx-flow 保持唯一的工作流执行/调度事实来源(LangGraph 引擎、`RunJournal`、
|
||||
`LockManager`、`EffectLedger` 契约不变)。
|
||||
- **插件化**:不重写工作流;通过 dsh「everything-is-a-plugin」扩展点把现有
|
||||
控制台能力投影为智能体工具与 UI 面板。
|
||||
- **UI**:在原 dsh Web UI 侧边栏追加工作流入口(`sidebar.footer.action` +
|
||||
`shell.overlay` 两个**追加型**插槽,不替换任何内置区域),选中工作流即可
|
||||
提问 / 诊断 / 修复。
|
||||
- **生产约束**(AGENTS.md):`gyxx schedule run` 仍是唯一生产调度路径;
|
||||
正式执行保持 `execute + confirmed` 双确认;控制台仍以 `--env-file` 注入云端凭据
|
||||
(启动脚本默认 `D:\product-collector-analyze-flow\.env`)。
|
||||
|
||||
## 为什么不把 gyxx-flow 改写进 dsh monorepo
|
||||
|
||||
dsh 是 pnpm monorepo(host/client 双聚合、Typert 远程契约、自有构建链)。
|
||||
把 Python 工作流引擎迁进去既不可能也无必要。dsh 的外部插件机制
|
||||
(`--patch` cordis.yml + 绝对路径插件 + `dsh.client` 包声明)就是为这种
|
||||
「外部系统接入」设计的。我们选择**进程外集成**:
|
||||
|
||||
- gyxx-flow 控制台 HTTP API 是唯一集成面(已含脱敏、并发守卫、写操作令牌);
|
||||
- 工作台插件只是控制台的客户端 + 智能体能力注册器;
|
||||
- 控制台挂了,dsh 照常可用;dsh 挂了,调度器照常跑。
|
||||
|
||||
## 组件
|
||||
|
||||
### 1. 控制台诊断端点(Python,`src/gyxx_flow/console.py`)
|
||||
|
||||
| 端点 | 说明 |
|
||||
| --- | --- |
|
||||
| `GET /api/workflows/{id}` | 单工作流详情(overview 投影 + `schedule_revision`) |
|
||||
| `GET /api/workflows/{id}/runs/{run_id}` | 单次运行 + 步骤明细 |
|
||||
| `GET /api/workflows/{id}/runs/{run_id}/diagnosis` | 诊断包:运行 + journal trace + 脱敏日志尾部 |
|
||||
|
||||
诊断包日志来源:`run.json` 的 `trace.paths.log` 目录 + 运行时间窗内的控制台日志
|
||||
(`logs/console/{workflow_id}-*.log`),每个文件限读尾部 256KB,经 `_sanitize_error`
|
||||
脱敏并截断,单响应最多 5 个文件。`run_id` 与 `workflow_id` 强制配对(404 不泄露
|
||||
跨工作流记录)。
|
||||
|
||||
### 2. 宿主插件(`workbench/plugin/gyxx-workbench.mjs`,零运行时依赖)
|
||||
|
||||
- **7 个工具**(原始 JSON-Schema `ToolDefinition`,不 import 任何 dsh 包,
|
||||
保证以绝对路径加载时无解析风险);
|
||||
- **系统提示词段**(`gyxx-workbench`,order 700):中文运维契约与安全规则;
|
||||
- **失败监控器**:`ctx.effect` 轮询,`workflow_id+run_id` 去重,自定义事件
|
||||
`gyxx-workbench/alert` + 可选自动诊断会话;
|
||||
- **桥接服务**(127.0.0.1:8790):为浏览器面板代理控制台 API(控制台禁 CORS,
|
||||
面板无法直连),并承载 `/bridge/ask` 会话创建与降级页面。
|
||||
写操作校验 `x-gyxx-workbench: 1` + JSON content-type;CORS 仅回环来源。
|
||||
|
||||
### 3. 侧边栏面板(`workbench/plugin/client/`)
|
||||
|
||||
- React 源码经 esbuild 打包为 CJS,平台模块(react 等)按 dsh 模块表协议外置,
|
||||
包装为 `window.__ModuleLoader__.load({id, factory})`——与官方 tsdown 产物同协议;
|
||||
- `package.json` 的 `dsh.client` 声明让 `client-modules` 扫描器(支持路径型
|
||||
Loader 条目,向上找最近 package.json)发现并提供该浏览器包;
|
||||
- 插槽:`sidebar.footer.action`(「工作流」开关 + 聚合状态点)、`shell.overlay`
|
||||
(左抽屉面板 + 失败 toast)。均为 list 型追加插槽,不替换内置区域;
|
||||
- 面板动作:提问/诊断/修复(创建带工作流上下文的智能体会话)、试运行/正式运行
|
||||
(浏览器端 confirm)、停止、运行历史查看、单运行诊断视图。
|
||||
|
||||
### 4. 会话创建(`/bridge/ask`)
|
||||
|
||||
`ctx.agents.create({ sessionId, meta: { cwd, origin } })` → `agent.followup()` 注入
|
||||
带工作流上下文的首条用户消息。新会话自动出现在 dsh 会话列表(host 侧创建即入册)。
|
||||
文案模板区分 ask/diagnose/repair 三种动作。
|
||||
|
||||
## 已知限制(v1)
|
||||
|
||||
- 会话创建后不能自动聚焦(dsh 客户端无公开的「选中会话」运行时 API);面板以
|
||||
toast 提示用户在会话列表中点开。
|
||||
- `sidebar.workspaces` 为 single 型插槽(替换即失去会话树),故面板走 overlay
|
||||
抽屉而非内嵌会话树;若未来 dsh 提供追加型侧栏区块插槽,可平移。
|
||||
- 客户端包协议依赖 dsh 未冻结的 developer-preview 契约;升级 dsh 需回归验证
|
||||
(README「依赖的 dsh 扩展点」一节列出了核对清单)。
|
||||
- `autoDiagnose` 默认关闭:自动开会话会产生模型调用成本。
|
||||
|
||||
## 测试
|
||||
|
||||
- Python:`tests/test_console.py` 新增 5 用例(详情/运行详情/诊断包/越权配对/HTTP 路由)。
|
||||
- 插件:`workbench/plugin/tests/`(node:test)——工具注册、离线错误结构化、
|
||||
confirmed 守卫、桥接代理。
|
||||
- 端到端手工验证:`start-workbench.ps1` → 面板选中工作流 → 提问/诊断/修复/试运行。
|
||||
@@ -1,7 +1,7 @@
|
||||
# 工作流总览(未隐藏)
|
||||
|
||||
> 生成日期:2026-10-08。数据来源:`config/workflows.json`、`config/schedules.json`、控制台显示名(`var/state/console/workflow-display-names.json`);数据库表提取自各入口脚本及共享 DB 辅助模块的内联 SQL。
|
||||
> 范围:控制台中未被隐藏的工作流共 **30 个**(全部 39 个中剔除浏览器端隐藏的 10 个,其中 `content.relogin.weekly` 已不在当前配置中)。
|
||||
> 范围:控制台中未被隐藏的工作流共 **31 个**(全部 40 个中剔除浏览器端隐藏的 10 个,其中 `content.relogin.weekly` 已不在当前配置中)。
|
||||
> 定时时间均为 **Asia/Shanghai**,由常驻调度器 `gyxx schedule run` 按 `config/schedules.json` 触发;标注「未启用」的仅支持控制台手动运行。
|
||||
> 启动脚本为仓库根目录相对路径,由工作流引擎相对 `src/gyxx_flow/modules/<模块>/` 解析执行;`×N` 表示同一脚本被 N 个步骤以不同参数复用。
|
||||
>
|
||||
@@ -36,13 +36,14 @@
|
||||
| 天猫差评采集 | `shop.tmall_negative_reviews.daily` | 固定筛选负面评价并按评价时间范围采集后入库 | 每月 25 日 03:50(未启用,仅手动) | `src/gyxx_flow/modules/shop_intelligence/collectors/tmall_negative_reviews.py` | `tmall_negative_reviews`(写) |
|
||||
| 抖音待改价申诉 | `shop.douyin_price_appeal` | 每个时点重新扫描可申诉并发起价格申诉;已申诉商品不重复出现 | 每日每小时整点(00:00–23:00) | `src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py` | 无(浏览器采集 + 本地 JSON) |
|
||||
|
||||
## 商品电商(product_commerce,12 个)
|
||||
## 商品电商(product_commerce,13 个)
|
||||
|
||||
| 工作流名称 | ID | 功能 | 定时执行 | 启动脚本 | 关联数据库表 |
|
||||
|---|---|---|---|---|---|
|
||||
| 商品人群画像周采集 | `product.persona.daily` | 天猫、抖音、京东人群画像并行采集并写入对应飞书多维表 | 每周二 12:00 | `src/gyxx_flow/modules/product_commerce/run_daily_persona.py` | `daily_persona_metrics`(写) |
|
||||
| 商品经营日报采集 | `product.daily` | 按业务日期(运行日 −1)采集 ERP 与三平台数据,幂等写 PostgreSQL,汇总分析后导出并写飞书日报 | 每日 08:40 | `src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py` | 写:`product_reviews`、`product_daily_metrics`、`daily_style_metrics`、`fact_style_analysis_reports`、`decline_events`、`fact_alert_history`<br>读:`weekly_style_metrics`、`daily_persona_metrics`、`main_image_creatives`、`cmt_styles`、`cmt_notes_master`、`cmt_comments`、`cmt_creators`(及上述写表) |
|
||||
| 私域库存预警 | `product.online_stock_alert.daily` | 查询线上渠道全部商品,库存低于 3 时汇总 SKU 与库存发飞书预警 | 每日 09:30 | `src/gyxx_flow/modules/product_commerce/online_stock_alert.py` | 无(企业数据交互 HTTP API + 飞书) |
|
||||
| 私域商品兑换排行 | `product.points_exchange.weekly` | 按自然周查询私域积分兑换明细,按商品汇总 `quantity` 兑换数量并从高到低排名,以飞书卡片发送周报 | 每周一 09:45(统计上一个完整自然周) | `src/gyxx_flow/modules/product_commerce/points_exchange_weekly.py` | 无(企业数据交互 HTTP API + 飞书) |
|
||||
| 三平台电商费用采集 | `product.ecommerce_costs.daily` | 并行采集天猫万相台、京东、抖音千川业务日前一天的费用,各平台先下载再导入,单平台失败不阻断其他 | 每日 10:00 | `…/product_commerce/collect_tmall_wanxiang_ads.py`<br>`…/product_commerce/import_tmall_product_ads.py`<br>`…/product_commerce/collect_jd_ad_costs.py`<br>`…/product_commerce/import_jd_ad_costs.py`<br>`…/product_commerce/collect_douyin_qianchuan_ads.py`<br>`…/product_commerce/import_douyin_qianchuan_ads.py` | `product_daily_metrics`(写,天猫原始费用并入 raw_data JSONB)<br>`jd_product_ad_daily_metrics`、`dy_product_ad_daily_metrics`(建+写)<br>`dim_style`(读) |
|
||||
| 京东与天猫主图周采集 | `product.main_image.weekly` | 京东与天猫万相台主图并行采集,各自完成 PG 入库和飞书插入,互不影响 | 每周日 08:30 | `…/product_commerce/run_weekly_jd_main_image.py`<br>`…/product_commerce/run_weekly_main_image.py` | `main_image_creatives`(写) |
|
||||
| 飞书商品月度销量表填写 | `product.sales_sheet.daily` | 抓取并校验全部 ERP 报表后填写飞书月度销量表,同名月份页幂等覆盖,无 ERP 编码产品按 0 写入 | 每月 1 日 10:00 | `src/gyxx_flow/modules/product_commerce/sync_monthly_sales_sheet.py` | 无(聚水潭浏览器采集 + 飞书写入) |
|
||||
|
||||
@@ -223,6 +223,8 @@ class AccountKeepaliveManager:
|
||||
self._next_due: dict[str, float] = {}
|
||||
self._locks = LockManager(settings.data_root / "state" / "locks")
|
||||
self._login_runner = login_runner or self._run_configured_login
|
||||
self._audit_store: Any | None = None
|
||||
self._audit_seen: dict[str, tuple[str, float]] = {}
|
||||
|
||||
def tick(self) -> list[dict[str, object]]:
|
||||
"""Run due account refreshes once; never starts work in dry-run mode."""
|
||||
@@ -586,8 +588,63 @@ class AccountKeepaliveManager:
|
||||
if result.get("status") == "success":
|
||||
payload["last_success_at"] = timestamp
|
||||
atomic_write_json(path, payload)
|
||||
self._audit_keepalive(account, result)
|
||||
return dict(result)
|
||||
|
||||
_AUDIT_STATE_BY_PROBE = {
|
||||
"success": "healthy",
|
||||
"expired": "expired",
|
||||
"not_logged_in": "not_logged_in",
|
||||
"busy": "busy",
|
||||
"disabled": "disabled",
|
||||
}
|
||||
|
||||
def _audit_keepalive(self, account: Any, result: dict[str, object]) -> None:
|
||||
"""Record probe outcomes so the real cookie lifetime becomes measurable.
|
||||
|
||||
The resident scheduler probes every ``keepalive_interval_seconds``
|
||||
(10 minutes by default), which is what turns "logins seem to last about
|
||||
a week" into a measured distribution. Repeated identical outcomes
|
||||
within an hour collapse into one row, so the event table tracks state
|
||||
changes rather than polling noise.
|
||||
"""
|
||||
|
||||
try:
|
||||
from gyxx_flow.ops.audit import snapshot_for_account
|
||||
|
||||
probe = str(result.get("status", "") or "error")
|
||||
state = self._AUDIT_STATE_BY_PROBE.get(probe, "probe_error")
|
||||
key = str(account.account_id)
|
||||
previous = self._audit_seen.get(key)
|
||||
if previous is not None:
|
||||
previous_state, previous_at = previous
|
||||
if previous_state == state and (self._clock() - previous_at) < 3600:
|
||||
return
|
||||
store = self._audit()
|
||||
if store is None or not store.enabled:
|
||||
return
|
||||
if store.record_cookie_event(
|
||||
account_id=key,
|
||||
state=state,
|
||||
snapshot=snapshot_for_account(account),
|
||||
probe_status=probe,
|
||||
probe_source="keepalive",
|
||||
detail=str(result.get("error_type", "") or ""),
|
||||
):
|
||||
self._audit_seen[key] = (state, self._clock())
|
||||
except Exception: # noqa: BLE001 - telemetry must never break keepalive
|
||||
return
|
||||
|
||||
def _audit(self) -> Any | None:
|
||||
if self._audit_store is None:
|
||||
try:
|
||||
from gyxx_flow.ops.audit import AuditStore
|
||||
|
||||
self._audit_store = AuditStore(self.settings.data_root)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return self._audit_store
|
||||
|
||||
|
||||
def _vault_cookie_valid(account: Any, store: BrowserCookieStore) -> bool:
|
||||
if not account.cookie_file.exists():
|
||||
|
||||
@@ -6,16 +6,20 @@ import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ITEMS_QUERY_URI = "/open/data/v1/items/query"
|
||||
POINTS_EXCHANGE_QUERY_URI = "/open/data/v1/points-exchange/query"
|
||||
DEFAULT_DATA_INTERACTION_BASE_URL = "https://test.gyxx.cn/api"
|
||||
_MAX_RESPONSE_BYTES = 16 * 1024 * 1024
|
||||
_API_DATETIME = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$")
|
||||
|
||||
|
||||
class DataInteractionError(RuntimeError):
|
||||
@@ -33,6 +37,71 @@ def query_items(
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Query the item list and return every item row from the API response."""
|
||||
|
||||
return _signed_query(
|
||||
ITEMS_QUERY_URI,
|
||||
{"channel": channel, "keyword": keyword},
|
||||
label="items query",
|
||||
base_url=base_url,
|
||||
app_key=app_key,
|
||||
app_secret=app_secret,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def query_points_exchange(
|
||||
*,
|
||||
start_time: str | datetime,
|
||||
end_time: str | datetime,
|
||||
base_url: str | None = None,
|
||||
app_key: str | None = None,
|
||||
app_secret: str | None = None,
|
||||
timeout: float = 60,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Query points-exchange consumption rows for one closed time window."""
|
||||
|
||||
start = _api_datetime(start_time, field="startTime")
|
||||
end = _api_datetime(end_time, field="endTime")
|
||||
if start > end:
|
||||
raise DataInteractionError("points exchange window start is after end")
|
||||
return _signed_query(
|
||||
POINTS_EXCHANGE_QUERY_URI,
|
||||
{"startTime": start, "endTime": end},
|
||||
label="points exchange query",
|
||||
base_url=base_url,
|
||||
app_key=app_key,
|
||||
app_secret=app_secret,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def _api_datetime(value: str | datetime, *, field: str) -> str:
|
||||
if isinstance(value, datetime):
|
||||
if value.tzinfo is not None:
|
||||
raise DataInteractionError(
|
||||
f"{field} must be a naive Asia/Shanghai datetime"
|
||||
)
|
||||
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||||
if not isinstance(value, str) or not _API_DATETIME.fullmatch(value.strip()):
|
||||
raise DataInteractionError(
|
||||
f"{field} must use the 'YYYY-MM-DD HH:MM:SS' format"
|
||||
)
|
||||
try:
|
||||
datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S")
|
||||
except ValueError as exc:
|
||||
raise DataInteractionError(f"{field} is not a valid datetime") from exc
|
||||
return value.strip()
|
||||
|
||||
|
||||
def _signed_query(
|
||||
uri: str,
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
label: str,
|
||||
base_url: str | None,
|
||||
app_key: str | None,
|
||||
app_secret: str | None,
|
||||
timeout: float,
|
||||
) -> list[dict[str, Any]]:
|
||||
secret = app_secret or os.environ.get("GYXX_DATA_INTERACTION_APP_SECRET", "")
|
||||
if not secret.strip():
|
||||
raise DataInteractionError(
|
||||
@@ -50,19 +119,19 @@ def query_items(
|
||||
raise DataInteractionError("GYXX_DATA_INTERACTION_BASE_URL must be an HTTP(S) URL")
|
||||
|
||||
body = json.dumps(
|
||||
{"channel": channel, "keyword": keyword},
|
||||
dict(payload),
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
timestamp = str(int(time.time() * 1000))
|
||||
body_sha = hashlib.sha256(body).hexdigest()
|
||||
canonical = f"POST\n{ITEMS_QUERY_URI}\n{timestamp}\n{body_sha}"
|
||||
canonical = f"POST\n{uri}\n{timestamp}\n{body_sha}"
|
||||
signature = hmac.new(
|
||||
secret.encode("utf-8"), canonical.encode("utf-8"), hashlib.sha256
|
||||
).hexdigest()
|
||||
|
||||
request = Request(
|
||||
base.rstrip("/") + ITEMS_QUERY_URI,
|
||||
base.rstrip("/") + uri,
|
||||
data=body,
|
||||
headers={
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
@@ -78,31 +147,31 @@ def query_items(
|
||||
except HTTPError as exc:
|
||||
detail = exc.read(2_000).decode("utf-8", errors="replace").strip()
|
||||
suffix = f": {detail}" if detail else ""
|
||||
raise DataInteractionError(f"items query returned HTTP {exc.code}{suffix}") from exc
|
||||
raise DataInteractionError(f"{label} returned HTTP {exc.code}{suffix}") from exc
|
||||
except URLError as exc:
|
||||
raise DataInteractionError(f"items query request failed: {exc.reason}") from exc
|
||||
raise DataInteractionError(f"{label} request failed: {exc.reason}") from exc
|
||||
|
||||
if len(raw) > _MAX_RESPONSE_BYTES:
|
||||
raise DataInteractionError("items query response exceeds the size limit")
|
||||
raise DataInteractionError(f"{label} response exceeds the size limit")
|
||||
try:
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise DataInteractionError("items query returned invalid JSON") from exc
|
||||
raise DataInteractionError(f"{label} returned invalid JSON") from exc
|
||||
|
||||
return _extract_item_rows(payload)
|
||||
return _extract_rows(payload, label=label)
|
||||
|
||||
|
||||
def _extract_item_rows(payload: object) -> list[dict[str, Any]]:
|
||||
def _extract_rows(payload: object, *, label: str) -> list[dict[str, Any]]:
|
||||
if isinstance(payload, Mapping):
|
||||
code = payload.get("code")
|
||||
if code not in (None, 0, 200, "0", "200") and payload.get("success") is not True:
|
||||
message = str(payload.get("message") or payload.get("msg") or "")[:300]
|
||||
suffix = f": {message}" if message else ""
|
||||
raise DataInteractionError(f"items query API reported code {code}{suffix}")
|
||||
raise DataInteractionError(f"{label} API reported code {code}{suffix}")
|
||||
if payload.get("success") is False:
|
||||
message = str(payload.get("message") or payload.get("msg") or "")[:300]
|
||||
suffix = f": {message}" if message else ""
|
||||
raise DataInteractionError(f"items query API reported failure{suffix}")
|
||||
raise DataInteractionError(f"{label} API reported failure{suffix}")
|
||||
data: object = payload.get("data", payload.get("result"))
|
||||
else:
|
||||
data = payload
|
||||
@@ -114,9 +183,9 @@ def _extract_item_rows(payload: object) -> list[dict[str, Any]]:
|
||||
data = candidate
|
||||
break
|
||||
if not isinstance(data, list):
|
||||
raise DataInteractionError("items query response does not contain an item array")
|
||||
raise DataInteractionError(f"{label} response does not contain an item array")
|
||||
if not all(isinstance(row, Mapping) for row in data):
|
||||
raise DataInteractionError("items query response contains a non-object item")
|
||||
raise DataInteractionError(f"{label} response contains a non-object item")
|
||||
return [dict(row) for row in data]
|
||||
|
||||
|
||||
@@ -124,5 +193,7 @@ __all__ = [
|
||||
"DEFAULT_DATA_INTERACTION_BASE_URL",
|
||||
"DataInteractionError",
|
||||
"ITEMS_QUERY_URI",
|
||||
"POINTS_EXCHANGE_QUERY_URI",
|
||||
"query_items",
|
||||
"query_points_exchange",
|
||||
]
|
||||
@@ -766,6 +766,11 @@ class RuntimeIntegrationCatalog:
|
||||
f"{binding.profile_dir.name}-luggage"
|
||||
)
|
||||
),
|
||||
"GUANGHE_NEW_BAG_USER_DATA_DIR": str(
|
||||
binding.profile_dir.with_name(
|
||||
f"{binding.profile_dir.name}-new-bag"
|
||||
)
|
||||
),
|
||||
"DY_COOKIES_FILE": str(binding.cookie_file),
|
||||
"DY_STORAGE_STATE_FILE": str(binding.storage_state_file),
|
||||
}
|
||||
|
||||
@@ -33,6 +33,34 @@ DEFAULT_SEED_PATH = (
|
||||
/ "workflow-dynamic-config-seed.json"
|
||||
)
|
||||
|
||||
DIM_STYLE_RUNTIME_SQL = """
|
||||
SELECT s.id,
|
||||
s.style_name AS name,
|
||||
COALESCE(s.brand, '') AS brand,
|
||||
COALESCE(s.erp_style_codes, '[]'::jsonb) AS erp_style_codes,
|
||||
jsonb_build_object(
|
||||
'tm', COALESCE(s.tm_spus, '[]'::jsonb),
|
||||
'jd', COALESCE(s.jd_spus, '[]'::jsonb),
|
||||
'dy', COALESCE(s.dy_spus, '[]'::jsonb),
|
||||
'jd_self', COALESCE(s.jd_self_skus, '[]'::jsonb)
|
||||
) AS platform_product_ids,
|
||||
COALESCE(s.bitable_weekly_url, '') AS sales_bitable_url,
|
||||
COALESCE(s.bitable_main_image_url, '') AS main_image_bitable_url,
|
||||
COALESCE(s.bitable_creator_url, '') AS creator_bitable_url,
|
||||
COALESCE(s.bitable_self_creator_url, '') AS self_creator_bitable_url,
|
||||
COALESCE(s.bitable_weekly_note_url, '') AS weekly_note_analysis_url,
|
||||
COALESCE(s.bitable_style_analysis_url, '') AS style_analysis_bitable_url,
|
||||
COALESCE(s.bitable_persona_tm_url, '') AS tm_persona_bitable_url,
|
||||
COALESCE(s.bitable_persona_jd_url, '') AS jd_persona_bitable_url,
|
||||
COALESCE(s.bitable_persona_dy_url, '') AS dy_persona_bitable_url,
|
||||
COALESCE(s.series_name, '') AS series_name,
|
||||
COALESCE(s.is_series_anchor, FALSE) AS is_series_anchor,
|
||||
COALESCE(s.collect_enabled, TRUE) AS collect_enabled
|
||||
FROM dim_style s
|
||||
WHERE COALESCE(s.style_name, '') <> ''
|
||||
ORDER BY s.style_name, s.id
|
||||
"""
|
||||
|
||||
CONFIG_FIELDS = (
|
||||
"platform",
|
||||
"erp_codes",
|
||||
@@ -298,8 +326,9 @@ def merge_lifecycle_style_records(
|
||||
"""Overlay lifecycle product identity on legacy target configuration rows.
|
||||
|
||||
``workflow_dynamic_*`` still owns Feishu target URLs and persona targets, while
|
||||
``cmt_styles`` owns ERP codes and product IDs. The result intentionally keeps
|
||||
the old flat runtime contract so existing collectors need no per-workflow fork.
|
||||
``dim_style`` (产品生命进程建档) owns ERP codes, product IDs and the sales sheet.
|
||||
The result intentionally keeps the old flat runtime contract so existing
|
||||
collectors need no per-workflow fork.
|
||||
"""
|
||||
|
||||
dynamic_by_style: dict[str, list[Mapping[str, Any]]] = {}
|
||||
@@ -311,7 +340,7 @@ def merge_lifecycle_style_records(
|
||||
lifecycle_by_style: dict[str, Mapping[str, Any]] = {}
|
||||
lifecycle_order: list[str] = []
|
||||
for row in lifecycle_rows:
|
||||
style_name = " ".join(str(row.get("name") or "").split())
|
||||
style_name = " ".join(str(row.get("name") or row.get("style_name") or "").split())
|
||||
if not style_name or style_name in lifecycle_by_style:
|
||||
continue
|
||||
lifecycle_by_style[style_name] = row
|
||||
@@ -325,7 +354,17 @@ def merge_lifecycle_style_records(
|
||||
legacy_rows = dynamic_by_style.get(style_name, [])
|
||||
lifecycle = lifecycle_by_style.get(style_name)
|
||||
if lifecycle is None:
|
||||
records.extend(dict(row) for row in legacy_rows)
|
||||
# 不在 dim_style 的款式(改名/删除后的残留行)不再继承旧表地址:
|
||||
# 飞书地址已统一由 dim_style 锚点持有,旧表列只是历史数据。
|
||||
for row in legacy_rows:
|
||||
record = dict(row)
|
||||
for field in URL_FIELDS:
|
||||
record[field] = ""
|
||||
# 已不在 dim_style 的残留款:没有主键可下发,显式标记未匹配,
|
||||
# 避免下游把 None 当成「还没回填」而不是「确实没建档」。
|
||||
record["style_id"] = None
|
||||
record["style_match_method"] = "unmatched"
|
||||
records.append(record)
|
||||
continue
|
||||
|
||||
managed, lifecycle_ids = _decode_lifecycle_product_ids(
|
||||
@@ -379,6 +418,33 @@ def merge_lifecycle_style_records(
|
||||
if item_id not in dynamic_unknown_ids.setdefault(label, []):
|
||||
dynamic_unknown_ids[label].append(item_id)
|
||||
|
||||
# 飞书地址统一以 dim_style 为准(2026-10-29):9 类地址全部维护在系列
|
||||
# 锚点行,旧表同名字段彻底不参与,缺失就是缺失、不再回退。
|
||||
sales_bitable_url = str(lifecycle.get("sales_bitable_url") or "").strip()
|
||||
series_name = str(lifecycle.get("series_name") or "").strip()
|
||||
is_series_anchor = bool(lifecycle.get("is_series_anchor"))
|
||||
|
||||
template["sales_bitable_url"] = sales_bitable_url
|
||||
template["series_name"] = series_name
|
||||
template["is_series_anchor"] = is_series_anchor
|
||||
# 款式主键统一 (2026-11-01): dim_style.id 作为正式字段下发。
|
||||
# 原先它只被编码进 source_record_id="lifecycle_style:<id>" 字符串,写入脚本
|
||||
# 拿不到,只能靠 style_name 关联——改款名历史行就成孤儿。这里提成字段后,
|
||||
# 所有经 runtime_workflow_configs() 取款的脚本自动获得 style_id。
|
||||
style_id_value = lifecycle.get("id")
|
||||
template["style_id"] = (
|
||||
int(style_id_value) if style_id_value is not None else None
|
||||
)
|
||||
template["style_match_method"] = (
|
||||
"dim_style" if style_id_value is not None else "unmatched"
|
||||
)
|
||||
for field in URL_FIELDS:
|
||||
template[field] = str(lifecycle.get(field) or "").strip()
|
||||
# 采集开关由产品生命进程在 dim_style 上维护(系列级,读锚点那行)。
|
||||
# 旧 cmt_styles 回退路径没有该列,按默认 TRUE 处理,保持历史口径。
|
||||
enabled_flag = lifecycle.get("collect_enabled")
|
||||
template["collect_enabled"] = True if enabled_flag is None else bool(enabled_flag)
|
||||
|
||||
if managed:
|
||||
labels = []
|
||||
for label in LIFECYCLE_PLATFORM_LABELS.values():
|
||||
@@ -413,13 +479,70 @@ def merge_lifecycle_style_records(
|
||||
else list(dynamic_ids.get(key, dynamic_unknown_ids.get(label, [])))
|
||||
)
|
||||
output["source_record_id"] = (
|
||||
f"cmt_style:{lifecycle.get('id')}" if key and managed else output.get("source_record_id")
|
||||
f"lifecycle_style:{lifecycle.get('id')}" if key and managed else output.get("source_record_id")
|
||||
)
|
||||
records.append(output)
|
||||
|
||||
_propagate_series_targets(records)
|
||||
return records
|
||||
|
||||
|
||||
def _propagate_series_targets(records: list[dict[str, Any]]) -> None:
|
||||
"""把系列内 9 类飞书地址聚合去重后统一下发到整条系列(就地修改)。
|
||||
|
||||
约定是地址只维护在 ``dim_style`` 的系列锚点行,子款留空。但建档时子款误填
|
||||
无法从表结构上禁止,所以这里不「只认锚点」,而是把整条系列的非空地址按
|
||||
「锚点优先、其余按款名」收集起来,按 URL 去重后取优先级最高的一条下发到
|
||||
该系列每一行——子款写的地址不会因为不在锚点上而被静默丢弃。
|
||||
|
||||
一个表位只能落一张表:同一字段出现多个不同地址时保留优先级最高的那条并
|
||||
打印告警,提示到锚点手动裁决,避免同系列数据被拆到两个表。
|
||||
|
||||
没有锚点时按款名排序取第一个,与采集开关、ERP 归集的判定口径保持一致。
|
||||
"""
|
||||
|
||||
by_series: dict[str, list[dict[str, Any]]] = {}
|
||||
for record in records:
|
||||
series = " ".join(str(record.get("series_name") or "").split())
|
||||
if series:
|
||||
by_series.setdefault(series, []).append(record)
|
||||
|
||||
for series, rows in by_series.items():
|
||||
# merge 会把同一款展开成多条平台记录,聚合前先按款归并,避免重复计数
|
||||
by_style: dict[str, list[dict[str, Any]]] = {}
|
||||
for row in rows:
|
||||
by_style.setdefault(str(row.get("style_name") or ""), []).append(row)
|
||||
ordered = sorted(
|
||||
by_style,
|
||||
key=lambda name: (
|
||||
not any(row.get("is_series_anchor") for row in by_style[name]),
|
||||
name,
|
||||
),
|
||||
)
|
||||
|
||||
chosen: dict[str, str] = {}
|
||||
for field in URL_FIELDS:
|
||||
candidates: list[str] = []
|
||||
for name in ordered:
|
||||
for row in by_style[name]:
|
||||
value = _plain_url(row.get(field))
|
||||
if value and value not in candidates:
|
||||
candidates.append(value)
|
||||
if not candidates:
|
||||
continue
|
||||
if len(candidates) > 1:
|
||||
print(
|
||||
f"[WARN] 系列「{series}」{field} 有 {len(candidates)} 个不同地址,"
|
||||
f"按「{ordered[0]}」下发并忽略其余: {candidates[1:]}"
|
||||
)
|
||||
chosen[field] = candidates[0]
|
||||
|
||||
if not chosen:
|
||||
continue
|
||||
for row in rows:
|
||||
row.update(chosen)
|
||||
|
||||
|
||||
def _plain_url(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
match = re.search(r"\]\((https?://[^)]+)\)", text)
|
||||
@@ -1158,16 +1281,24 @@ class PostgresWorkflowConfigStore:
|
||||
"WHERE s.enabled ORDER BY s.style_name, p.id"
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
cursor.execute("SELECT to_regclass('public.cmt_styles') AS table_name")
|
||||
cmt_table = cursor.fetchone() or {}
|
||||
if cmt_table.get("table_name"):
|
||||
cursor.execute(
|
||||
"SELECT id, name, brand, erp_style_codes, platform_product_ids "
|
||||
"FROM cmt_styles ORDER BY name, id"
|
||||
)
|
||||
cursor.execute("SELECT to_regclass('public.dim_style') AS table_name")
|
||||
dim_table = cursor.fetchone() or {}
|
||||
if dim_table.get("table_name"):
|
||||
# 产品生命进程(dim_style)是款式身份的唯一权威来源:
|
||||
# ERP 款式编码逐款填写,平台商品编码与销量表允许整系列只填主款。
|
||||
cursor.execute(DIM_STYLE_RUNTIME_SQL)
|
||||
lifecycle_rows = cursor.fetchall()
|
||||
else:
|
||||
lifecycle_rows = []
|
||||
cursor.execute("SELECT to_regclass('public.cmt_styles') AS table_name")
|
||||
cmt_table = cursor.fetchone() or {}
|
||||
if cmt_table.get("table_name"):
|
||||
cursor.execute(
|
||||
"SELECT id, name, brand, erp_style_codes, platform_product_ids "
|
||||
"FROM cmt_styles ORDER BY name, id"
|
||||
)
|
||||
lifecycle_rows = cursor.fetchall()
|
||||
else:
|
||||
lifecycle_rows = []
|
||||
personas: dict[int, dict[str, str]] = {}
|
||||
for row in rows:
|
||||
if row.get("platform_enabled") is False:
|
||||
|
||||
@@ -229,6 +229,36 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
status_parser = acceptance_commands.add_parser("status")
|
||||
status_parser.add_argument("--json", action="store_true")
|
||||
status_parser.add_argument("--output", type=Path)
|
||||
|
||||
audit_parser = commands.add_parser(
|
||||
"audit", help="query the PostgreSQL operational audit tables"
|
||||
)
|
||||
audit_commands = audit_parser.add_subparsers(dest="audit_command", required=True)
|
||||
|
||||
audit_runs = audit_commands.add_parser("runs", help="recent workflow runs")
|
||||
audit_runs.add_argument("--workflow")
|
||||
audit_runs.add_argument("--status")
|
||||
audit_runs.add_argument("--date", dest="business_date")
|
||||
audit_runs.add_argument("--limit", type=int, default=30)
|
||||
audit_runs.add_argument("--json", action="store_true")
|
||||
|
||||
audit_cookies = audit_commands.add_parser(
|
||||
"cookies", help="recent cookie observations"
|
||||
)
|
||||
audit_cookies.add_argument("--account")
|
||||
audit_cookies.add_argument("--state")
|
||||
audit_cookies.add_argument("--limit", type=int, default=30)
|
||||
audit_cookies.add_argument("--json", action="store_true")
|
||||
|
||||
audit_stats = audit_commands.add_parser(
|
||||
"cookie-stats", help="measured cookie lifetime per account"
|
||||
)
|
||||
audit_stats.add_argument("--account")
|
||||
audit_stats.add_argument("--json", action="store_true")
|
||||
|
||||
audit_commands.add_parser(
|
||||
"schema", help="print the DDL that creates the audit tables"
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
@@ -304,6 +334,8 @@ def main(
|
||||
and arguments.acceptance_command == "status"
|
||||
):
|
||||
return _acceptance_status_command(arguments, resolved_settings, output)
|
||||
if arguments.command == "audit":
|
||||
return _audit_command(arguments, resolved_settings, output)
|
||||
if arguments.command == "scripts" and arguments.scripts_command == "list":
|
||||
return _list_scripts(arguments, output)
|
||||
if arguments.command == "scripts" and arguments.scripts_command == "run":
|
||||
@@ -549,6 +581,152 @@ def _acceptance_status_command(
|
||||
return EXIT_SUCCESS if report.is_complete else EXIT_ACCEPTANCE_INCOMPLETE
|
||||
|
||||
|
||||
def _format_seconds(value: object) -> str:
|
||||
"""Human-readable duration for the audit tables (None -> '-')."""
|
||||
|
||||
if value is None:
|
||||
return "-"
|
||||
try:
|
||||
seconds = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return "-"
|
||||
if seconds >= 86400:
|
||||
return f"{seconds / 86400:.1f}d"
|
||||
if seconds >= 3600:
|
||||
return f"{seconds / 3600:.1f}h"
|
||||
if seconds >= 60:
|
||||
return f"{seconds / 60:.0f}m"
|
||||
return f"{seconds}s"
|
||||
|
||||
|
||||
def _format_timestamp(value: object) -> str:
|
||||
if value is None:
|
||||
return "-"
|
||||
text = value.isoformat() if hasattr(value, "isoformat") else str(value)
|
||||
return text.replace("T", " ")[:19]
|
||||
|
||||
|
||||
def _audit_command(
|
||||
arguments: argparse.Namespace, settings: Settings, output: TextIO
|
||||
) -> int:
|
||||
from gyxx_flow.ops.audit import SCHEMA_SQL, AuditStore
|
||||
|
||||
command = arguments.audit_command
|
||||
if command == "schema":
|
||||
output.write(SCHEMA_SQL.strip() + "\n")
|
||||
return EXIT_SUCCESS
|
||||
|
||||
store = AuditStore(settings.data_root)
|
||||
if not store.enabled:
|
||||
output.write("审计已关闭(GYXX_OPS_AUDIT=off)\n")
|
||||
return EXIT_CONFIGURATION
|
||||
|
||||
if command == "runs":
|
||||
rows = store.query_runs(
|
||||
workflow_id=arguments.workflow,
|
||||
status=arguments.status,
|
||||
business_date=arguments.business_date,
|
||||
limit=arguments.limit,
|
||||
)
|
||||
elif command == "cookies":
|
||||
rows = store.query_cookie_events(
|
||||
account_id=arguments.account,
|
||||
state=arguments.state,
|
||||
limit=arguments.limit,
|
||||
)
|
||||
elif command == "cookie-stats":
|
||||
rows = (
|
||||
store.query_cookie_lifetimes(account_id=arguments.account)
|
||||
if arguments.account
|
||||
else store.query_cookie_stats()
|
||||
)
|
||||
else: # pragma: no cover - argparse 已限制取值
|
||||
raise CliConfigurationError(f"unsupported audit command: {command}")
|
||||
|
||||
if store.last_error and not rows:
|
||||
output.write(f"审计库不可用:{store.last_error}\n")
|
||||
return EXIT_CONFIGURATION
|
||||
|
||||
if getattr(arguments, "json", False):
|
||||
_write_json(output, {"rows": rows})
|
||||
return EXIT_SUCCESS
|
||||
|
||||
if not rows:
|
||||
output.write("暂无审计记录\n")
|
||||
return EXIT_SUCCESS
|
||||
|
||||
if command == "runs":
|
||||
output.write(
|
||||
f"{'started':19s} {'workflow':40s} {'date':10s} {'mode':7s} "
|
||||
f"{'status':8s} {'dur':>6s} steps\n"
|
||||
)
|
||||
for row in rows:
|
||||
steps = (
|
||||
f"{row.get('step_success', 0)}/{row.get('step_total', 0)}"
|
||||
f" failed={row.get('step_failed', 0)}"
|
||||
)
|
||||
output.write(
|
||||
f"{_format_timestamp(row.get('started_at')):19s} "
|
||||
f"{str(row.get('workflow_id', ''))[:40]:40s} "
|
||||
f"{_format_timestamp(row.get('business_date'))[:10]:10s} "
|
||||
f"{str(row.get('mode', '')):7s} "
|
||||
f"{str(row.get('status', '')):8s} "
|
||||
f"{_format_seconds(row.get('duration_seconds')):>6s} {steps}\n"
|
||||
)
|
||||
if row.get("error"):
|
||||
output.write(f"{'':21s}error: {str(row['error'])[:160]}\n")
|
||||
return EXIT_SUCCESS
|
||||
|
||||
if command == "cookies":
|
||||
output.write(
|
||||
f"{'observed':19s} {'account':24s} {'state':14s} {'probe':14s} "
|
||||
f"{'fingerprint':12s} expires_at\n"
|
||||
)
|
||||
for row in rows:
|
||||
output.write(
|
||||
f"{_format_timestamp(row.get('observed_at')):19s} "
|
||||
f"{str(row.get('account_id', ''))[:24]:24s} "
|
||||
f"{str(row.get('state', '')):14s} "
|
||||
f"{str(row.get('probe_status', '')):14s} "
|
||||
f"{str(row.get('fingerprint', ''))[:12]:12s} "
|
||||
f"{_format_timestamp(row.get('declared_expires_at'))}\n"
|
||||
)
|
||||
return EXIT_SUCCESS
|
||||
|
||||
# cookie-stats / 单账号 lifetime
|
||||
if arguments.account:
|
||||
output.write(
|
||||
f"{'fingerprint':12s} {'first_seen':19s} {'last_healthy':19s} "
|
||||
f"{'failed_at':19s} {'lifetime':>9s} {'declared':19s}\n"
|
||||
)
|
||||
for row in rows:
|
||||
output.write(
|
||||
f"{str(row.get('fingerprint', ''))[:12]:12s} "
|
||||
f"{_format_timestamp(row.get('first_seen_at')):19s} "
|
||||
f"{_format_timestamp(row.get('last_healthy_at')):19s} "
|
||||
f"{_format_timestamp(row.get('first_failed_at')):19s} "
|
||||
f"{_format_seconds(row.get('lifetime_seconds')):>9s} "
|
||||
f"{_format_timestamp(row.get('declared_expires_at')):19s}\n"
|
||||
)
|
||||
return EXIT_SUCCESS
|
||||
|
||||
output.write(
|
||||
f"{'account':24s} {'sessions':>8s} {'expired':>7s} {'avg':>7s} "
|
||||
f"{'median':>7s} {'min':>7s} {'max':>7s} last_failed\n"
|
||||
)
|
||||
for row in rows:
|
||||
output.write(
|
||||
f"{str(row.get('account_id', ''))[:24]:24s} "
|
||||
f"{row.get('sessions', 0):>8d} {row.get('expired_sessions', 0):>7d} "
|
||||
f"{_format_seconds(row.get('avg_lifetime_seconds')):>7s} "
|
||||
f"{_format_seconds(row.get('median_lifetime_seconds')):>7s} "
|
||||
f"{_format_seconds(row.get('min_lifetime_seconds')):>7s} "
|
||||
f"{_format_seconds(row.get('max_lifetime_seconds')):>7s} "
|
||||
f"{_format_timestamp(row.get('last_failed_at'))}\n"
|
||||
)
|
||||
return EXIT_SUCCESS
|
||||
|
||||
|
||||
def _list_workflows(
|
||||
arguments: argparse.Namespace,
|
||||
registry: WorkflowRegistry,
|
||||
|
||||
@@ -191,7 +191,6 @@ _WORKFLOW_NAMES = {
|
||||
"product.backfill": "商品数据补采",
|
||||
"product.market_rank": "市场排行周采集",
|
||||
"product.tmall_baibu_apply": "天猫百亿补贴批量报名",
|
||||
"product.review_collection": "商品评价采集",
|
||||
"product.weekly_aggregate.documented_missing": "商品周聚合(待接入)",
|
||||
"shop.metrics.weekly": "店铺经营指标周采集",
|
||||
"shop.competitor.weekly": "竞店指标周采集",
|
||||
@@ -2247,7 +2246,7 @@ class WorkflowConsoleService:
|
||||
jd_customer_service_store: str | None = None
|
||||
if workflow_id == "product.video_upload":
|
||||
raw_store = payload.get("store", "flagship")
|
||||
if raw_store not in {"flagship", "luggage"}:
|
||||
if raw_store not in {"flagship", "luggage", "new_bag"}:
|
||||
raise ConsoleRequestError("天猫视频上传店铺参数无效")
|
||||
video_upload_store = str(raw_store)
|
||||
tmall_topic_keyword = _normalize_tmall_topic_keyword(
|
||||
|
||||
@@ -147,6 +147,7 @@ class CookieGate:
|
||||
self._now = now or (lambda: datetime.now(timezone.utc))
|
||||
self._sender = sender or send_lark_bot_message
|
||||
self._workflow_catalog: WorkflowCatalog | None = None
|
||||
self._audit_store: Any | None = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Account resolution
|
||||
@@ -236,7 +237,7 @@ class CookieGate:
|
||||
static_valid = _vault_cookie_valid(account, store)
|
||||
status = _read_keepalive_status(account)
|
||||
if static_valid and self._keepalive_success_is_fresh(status):
|
||||
self._write_state(account, STATE_HEALTHY, probe="fresh-keepalive")
|
||||
self._write_state(account, STATE_HEALTHY, probe="fresh-keepalive", context=context)
|
||||
return GateVerdict(
|
||||
account_id=account_id,
|
||||
mode=mode,
|
||||
@@ -269,6 +270,7 @@ class CookieGate:
|
||||
STATE_HEALTHY,
|
||||
probe="success",
|
||||
recovered=recovered,
|
||||
context=context,
|
||||
)
|
||||
message = (
|
||||
f"账号 {account_id} 登录态有效(自动账密重登成功)"
|
||||
@@ -367,6 +369,20 @@ class CookieGate:
|
||||
state.get("pending_reruns"), context, _now_iso(self._now)
|
||||
)
|
||||
self._save_state(account, state)
|
||||
try:
|
||||
keepalive_status = _read_keepalive_status(account) or {}
|
||||
except Exception: # noqa: BLE001 - state file is advisory
|
||||
keepalive_status = {}
|
||||
self._audit_cookie(
|
||||
account,
|
||||
"not_logged_in"
|
||||
if str(keepalive_status.get("status", "")) == "not_logged_in"
|
||||
else "expired",
|
||||
probe_status=str(keepalive_status.get("status", "expired")),
|
||||
probe_source="gate",
|
||||
detail=detail,
|
||||
context=context,
|
||||
)
|
||||
message = _expiry_message(account_id, mode, context, detail)
|
||||
if notified:
|
||||
message += " 已通过飞书通知登录负责人。"
|
||||
@@ -387,7 +403,17 @@ class CookieGate:
|
||||
*,
|
||||
probe: str,
|
||||
recovered: bool = False,
|
||||
context: RunContext | None = None,
|
||||
) -> None:
|
||||
if state_name == STATE_HEALTHY:
|
||||
# 健康心跳要落在每一次判定上,才能算准「最后一次可用」的时间。
|
||||
self._audit_cookie(
|
||||
account,
|
||||
"healthy",
|
||||
probe_status=probe,
|
||||
probe_source="gate",
|
||||
context=context,
|
||||
)
|
||||
state = self._read_state(account)
|
||||
if (
|
||||
state_name == STATE_HEALTHY
|
||||
@@ -431,6 +457,64 @@ class CookieGate:
|
||||
except OSError: # pragma: no cover - state file is advisory only
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cookie telemetry (best-effort; never blocks or fails the gate)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _audit(self) -> Any | None:
|
||||
if self._audit_store is None:
|
||||
try:
|
||||
from gyxx_flow.ops.audit import AuditStore
|
||||
|
||||
self._audit_store = AuditStore(
|
||||
self.settings.data_root,
|
||||
environment=self._environment,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - telemetry is optional
|
||||
return None
|
||||
return self._audit_store
|
||||
|
||||
def _audit_cookie(
|
||||
self,
|
||||
account: Any,
|
||||
state: str,
|
||||
*,
|
||||
probe_status: str,
|
||||
probe_source: str,
|
||||
detail: str = "",
|
||||
context: RunContext | None = None,
|
||||
) -> None:
|
||||
"""Append one cookie observation.
|
||||
|
||||
Stores a value-free fingerprint plus the earliest declared expiry of the
|
||||
required cookies, so failure patterns (how long a login really lasts,
|
||||
and whether it dies before or after the declared expiry) can be derived
|
||||
from history instead of guessed. Errors are swallowed: gating must
|
||||
never depend on the audit database.
|
||||
"""
|
||||
|
||||
try:
|
||||
store = self._audit()
|
||||
if store is None or not store.enabled:
|
||||
return
|
||||
store.record_cookie_event(
|
||||
account_id=str(getattr(account, "account_id", "") or ""),
|
||||
state=state,
|
||||
snapshot=self._cookie_snapshot(account),
|
||||
probe_status=probe_status,
|
||||
probe_source=probe_source,
|
||||
detail=detail,
|
||||
workflow_id=str(getattr(context, "workflow_id", "") or ""),
|
||||
run_id=str(getattr(context, "run_id", "") or ""),
|
||||
)
|
||||
except Exception: # noqa: BLE001 - telemetry is advisory only
|
||||
return
|
||||
|
||||
def _cookie_snapshot(self, account: Any) -> Any | None:
|
||||
from gyxx_flow.ops.audit import snapshot_for_account
|
||||
|
||||
return snapshot_for_account(account)
|
||||
|
||||
def _within_notify_interval(self, state: Mapping[str, object]) -> bool:
|
||||
raw = state.get("last_notified_at")
|
||||
if not isinstance(raw, str):
|
||||
|
||||
@@ -1019,18 +1019,28 @@ def _creative_context(metric_date: date, style_names: list[str]) -> list[dict[st
|
||||
|
||||
|
||||
def _review_context(metric_date: date, style_names: list[str]) -> list[dict[str, Any]]:
|
||||
"""近 7 天差评归因(读 v_negative_reviews)。
|
||||
|
||||
2026-11-01 起 ``product_reviews`` 下线,改读三平台差评归一视图。口径随之变化:
|
||||
|
||||
* 数据源只剩差评,因此 ``sample_size`` = 差评条数、``negative_rate`` 恒为 1.0,
|
||||
不再表示「差评/全部评价」;真正要看比例的场合请取平台评价总数另算。
|
||||
* 输出门槛从「窗口内至少 5 条评价」改为「至少 1 条差评」——旧门槛在只有差评
|
||||
的数据源上会导致几乎永不产出。
|
||||
* 覆盖天猫(旧表天猫为 0 行)、京东(旧表 8 月断更),负向样本反而更全。
|
||||
"""
|
||||
|
||||
if not style_names:
|
||||
return []
|
||||
negative_terms = ("差", "坏", "退", "重", "小", "掉色", "线头", "不值", "失望")
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT style_name, platform, content, score, tags
|
||||
FROM product_reviews
|
||||
SELECT style_name, platform, content
|
||||
FROM v_negative_reviews
|
||||
WHERE style_name = ANY(%s) AND comment_date BETWEEN %s AND %s
|
||||
AND content IS NOT NULL AND BTRIM(content) <> ''
|
||||
ORDER BY style_name, comment_date DESC, id DESC
|
||||
ORDER BY style_name, comment_date DESC, review_at DESC
|
||||
""",
|
||||
(style_names, metric_date - timedelta(days=6), metric_date),
|
||||
)
|
||||
@@ -1040,17 +1050,15 @@ def _review_context(metric_date: date, style_names: list[str]) -> list[dict[str,
|
||||
grouped.setdefault(row[0], []).append(row)
|
||||
output = []
|
||||
for style_name, items in grouped.items():
|
||||
if len(items) < 5:
|
||||
continue
|
||||
negative = [item for item in items if (item[3] is not None and item[3] <= 2) or any(term in item[2] for term in negative_terms)]
|
||||
output.append({
|
||||
"style_name": style_name,
|
||||
"window_start": metric_date - timedelta(days=6),
|
||||
"window_end": metric_date,
|
||||
"sample_size": len(items),
|
||||
"negative_count": len(negative),
|
||||
"negative_rate": round(len(negative) / len(items), 4),
|
||||
"negative_examples": [item[2][:80] for item in negative[:2]],
|
||||
"negative_count": len(items),
|
||||
"negative_rate": 1.0,
|
||||
"negative_examples": [item[2][:80] for item in items[:2]],
|
||||
"source": "v_negative_reviews",
|
||||
})
|
||||
return output
|
||||
|
||||
|
||||
@@ -166,6 +166,23 @@ def get_style_id(name: str) -> int | None:
|
||||
return row["id"] if row else None
|
||||
|
||||
|
||||
def get_dim_style_id(name: str) -> int | None:
|
||||
"""按款式名解析 dim_style 主档 id(2026-10-27 新增)。
|
||||
|
||||
data-hub 消费 cmt_notes_master.style_id 时按 dim_style(系列锚点款)口径,
|
||||
cmt_styles.id 在款式主档重排后会变成悬挂 id;写笔记前用本函数归一,
|
||||
解析不到返回 None(新款尚未收录款式库,由调用方决定回退策略)。
|
||||
"""
|
||||
with get_dict_conn() as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(
|
||||
"SELECT id FROM dim_style WHERE style_name = %s ORDER BY id LIMIT 1",
|
||||
(name,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return row["id"] if row else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Creators
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -362,9 +362,14 @@ def sync_notes_to_db(note_tasks: list[NoteTask], style_ids: dict[str, int]) -> d
|
||||
creator_id = creator_ids[creator_name]
|
||||
db.link_style_creator(style_id, creator_id)
|
||||
|
||||
# 2026-10-27 起笔记归属改挂 dim_style 主档 id(data-hub 消费口径),
|
||||
# cmt_styles.id 仅供 cmt_style_creators 等采集端内部关系表使用;
|
||||
# dim_style 缺同名款时回退 cmt_styles.id,待款式收录后归一
|
||||
dim_style_id = db.get_dim_style_id(task.style_name) or style_id
|
||||
|
||||
platform = detect_platform(task.note_url, task.platform_hint)
|
||||
note_id = db.upsert_note(
|
||||
style_id=style_id,
|
||||
style_id=dim_style_id,
|
||||
creator_id=creator_id,
|
||||
platform=platform,
|
||||
url=task.note_url,
|
||||
|
||||
@@ -426,9 +426,21 @@ def _backfill_record(mapping: dict, style_name: str, rid: str) -> dict | None:
|
||||
|
||||
|
||||
def _style_id_lookup(cur) -> dict[str, int]:
|
||||
"""name -> cmt_styles.id 映射(一次查全表)"""
|
||||
"""name -> 笔记归属款 id 映射(一次查全表)。
|
||||
|
||||
2026-10-27 起优先 dim_style 主档 id(data-hub 按 dim_style 消费;直接用
|
||||
cmt_styles.id 在款式主档重排后会变成悬挂 id,笔记在系统中"消失")。
|
||||
dim_style 缺同名款时回退 cmt_styles.id,待款式收录 + 存量重映射迁移归一。
|
||||
同名多行取最小 id。
|
||||
"""
|
||||
mapping: dict[str, int] = {}
|
||||
cur.execute("SELECT style_name, id FROM dim_style ORDER BY id")
|
||||
for name, sid in cur.fetchall():
|
||||
mapping.setdefault(str(name), sid)
|
||||
cur.execute("SELECT id, name FROM cmt_styles")
|
||||
return {name: id for id, name in cur.fetchall()}
|
||||
for sid, name in cur.fetchall():
|
||||
mapping.setdefault(str(name), sid)
|
||||
return mapping
|
||||
|
||||
|
||||
def _ensure_creator(cur, name: str | None) -> int | None:
|
||||
|
||||
@@ -219,6 +219,36 @@ def parse_wan_count(text: str) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
# 抖音"复制分享文案"格式:标题 + 话题链接段 [话题](url)(个别登记写成了 {url} 花括号包裹),
|
||||
# 或裸 URL。登记人常把整段分享文案粘进「发布笔记标题」列,入库前剥掉链接只留标题文本。
|
||||
_TITLE_LINK_RE = re.compile(
|
||||
r"\[[^\]]*\]\([^)]*\)" # [text](url) 圆括号链接
|
||||
r"|\[[^\]]*\]\{[^}]*\}" # [text]{url} 花括号链接
|
||||
r"|\{https?://[^}]*\}" # {url} 裸花括号链接
|
||||
r"|https?://[^\s))\]]+" # 裸 URL
|
||||
)
|
||||
|
||||
|
||||
def clean_note_title(raw: str) -> str:
|
||||
"""剥掉标题里混入的话题链接段与裸 URL,只留标题文本。
|
||||
|
||||
兜底顺序:剥链接后非空 → 用它;整段都是链接(标题只剩话题/搜索词)→
|
||||
取方括号内的文本去重拼接;再不行 → 保留原文,避免落空。
|
||||
"""
|
||||
text = raw or ""
|
||||
cleaned = _TITLE_LINK_RE.sub("", text)
|
||||
cleaned = cleaned.replace("%20", " ") # 链接段之间残留的 URL 编码空格
|
||||
cleaned = re.sub(r"\s{2,}", " ", cleaned).strip()
|
||||
if cleaned:
|
||||
return cleaned
|
||||
inner = [t.strip() for t in re.findall(r"\[([^\]]*)\]", text)]
|
||||
seen: list[str] = []
|
||||
for item in inner:
|
||||
if item and item not in seen:
|
||||
seen.append(item)
|
||||
return " ".join(seen) if seen else text.strip()
|
||||
|
||||
|
||||
def normalize_publish_time(cell: Any) -> str | None:
|
||||
text = cell_text(cell)
|
||||
if not text:
|
||||
@@ -340,7 +370,7 @@ def parse_records(
|
||||
record_id=record_id,
|
||||
platform=resolve_platform(value(record, "platform"), url),
|
||||
creator_name=creator_name,
|
||||
title=cell_text(value(record, "note_title"))[:512],
|
||||
title=clean_note_title(cell_text(value(record, "note_title")))[:512],
|
||||
publish_time=normalize_publish_time(value(record, "publish_time")),
|
||||
url=url,
|
||||
creator_attrs=creator_attrs,
|
||||
@@ -479,6 +509,22 @@ def sync_table(
|
||||
raise ValueError("mapping 缺少款式/base_token/table_id")
|
||||
|
||||
style_id = _ensure_style(cur, style_name)
|
||||
# 2026-10-27 起笔记归属改挂 dim_style 主档 id(data-hub 按 dim_style 系列锚点款消费,
|
||||
# 历史上直接写 cmt_styles.id 在款式主档重排后变成悬挂 id,导致笔记在系统里"消失")。
|
||||
# 按款名解析 dim_style.id;解析不到(新款尚未收录款式库)时回退 cmt_styles.id 并告警,
|
||||
# 后续由款式收录 + 存量重映射迁移归一。
|
||||
cur.execute(
|
||||
"SELECT id FROM dim_style WHERE style_name = %s ORDER BY id LIMIT 1",
|
||||
(style_name,),
|
||||
)
|
||||
dim_row = cur.fetchone()
|
||||
if dim_row:
|
||||
style_id = dim_row[0]
|
||||
else:
|
||||
_log(
|
||||
f"[{style_name}] dim_style 未找到同名款式,笔记暂挂 cmt_styles.id={style_id},"
|
||||
"待款式收录后归一"
|
||||
)
|
||||
creators = 0
|
||||
for row in rows:
|
||||
creator_id: int | None = None
|
||||
|
||||
@@ -15,6 +15,7 @@ PRODUCT_SCHEDULED_WORKFLOW_IDS = (
|
||||
"product.daily",
|
||||
"product.alert.daily",
|
||||
"product.online_stock_alert.daily",
|
||||
"product.points_exchange.weekly",
|
||||
"product.ecommerce_costs.daily",
|
||||
"product.style_analysis.interval",
|
||||
"product.main_image.weekly",
|
||||
|
||||
@@ -109,12 +109,26 @@ def missing_window_dates(rows: list[dict], start: date, end: date) -> list[date]
|
||||
|
||||
|
||||
def pull_reviews(style: str, start: date, end: date) -> dict:
|
||||
"""仅返回当前三日窗口内的评论分布与样本。"""
|
||||
"""仅返回当前三日窗口内的差评分布与样本。
|
||||
|
||||
2026-11-01 起 ``product_reviews`` 下线,改读三平台差评归一视图
|
||||
``v_negative_reviews``(tm + jd + dy 三张差评表 UNION)。
|
||||
|
||||
口径变化(必须知道):旧表存好/中/差全量,视图只有差评一档,所以这里
|
||||
返回的 key 会集中在 ``<平台>|负面评价``。换来的是覆盖率更高——天猫差评
|
||||
旧表根本没有(0 行),京东旧表 8 月就断更。
|
||||
|
||||
款式归因走 ``style_id``(dim_style.id),款名只作显示冗余。
|
||||
"""
|
||||
|
||||
with get_conn() as c, c.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT platform, rating, COUNT(*) n, array_agg(content ORDER BY comment_date DESC) samples
|
||||
FROM product_reviews
|
||||
WHERE style_name=%s AND comment_date BETWEEN %s AND %s
|
||||
"""SELECT platform, rating, COUNT(*) n,
|
||||
array_agg(content ORDER BY comment_date DESC) samples
|
||||
FROM v_negative_reviews
|
||||
WHERE style_id = (SELECT id FROM dim_style WHERE style_name = %s)
|
||||
AND comment_date BETWEEN %s AND %s
|
||||
AND content IS NOT NULL AND BTRIM(content) <> ''
|
||||
GROUP BY platform, rating""",
|
||||
(style, start, end),
|
||||
)
|
||||
@@ -380,7 +394,7 @@ def aggregate_style_total(platforms: dict[str, dict]) -> dict:
|
||||
LEGACY_SYSTEM = """你是光影行星 (GYXX) 的资深电商数据分析师, 擅长把数据现象挖到根因并给出可执行方案.
|
||||
用户会给定一个款式近 3 天的真实业务数据:
|
||||
- 站内 (店铺自身): 天猫/京东/抖音三平台当前 3 天的 daily_style_metrics、
|
||||
product_reviews、daily_persona_metrics、main_image_creatives
|
||||
v_negative_reviews(三平台差评)、daily_persona_metrics、main_image_creatives
|
||||
- 站外 (种草营销): cmt_notes_master (笔记+合作投入), cmt_comments (笔记评论),
|
||||
cmt_creators (创作者静态属性)
|
||||
|
||||
@@ -787,7 +801,7 @@ def build_user_prompt(style: str, start: date, end: date, payload: dict) -> str:
|
||||
# 人物画像
|
||||
{json.dumps(payload['persona'], ensure_ascii=False, indent=2)}
|
||||
|
||||
# 商品评价
|
||||
# 商品差评 (仅负向评价, 覆盖天猫/京东/抖音; 无好评与中评样本)
|
||||
{json.dumps(payload['product_reviews'], ensure_ascii=False, indent=2)}
|
||||
|
||||
# 主图表现
|
||||
|
||||
@@ -18,7 +18,7 @@ from scrapling.fetchers import DynamicFetcher
|
||||
try:
|
||||
from . import collect_erp_yesterday_metrics as erp
|
||||
from .collect_retry_utils import retry_step
|
||||
from .config.style_config_loader import StyleConfigLoader, has_complete_style_identity
|
||||
from .config.style_config_loader import StyleConfigLoader
|
||||
from .db import (
|
||||
get_conn,
|
||||
init_schema,
|
||||
@@ -35,7 +35,7 @@ try:
|
||||
except ImportError: # Direct script execution from the module root.
|
||||
import collect_erp_yesterday_metrics as erp
|
||||
from collect_retry_utils import retry_step
|
||||
from config.style_config_loader import StyleConfigLoader, has_complete_style_identity
|
||||
from config.style_config_loader import StyleConfigLoader
|
||||
from db import get_conn, init_schema, upsert_erp_all_shop_style_daily_metrics
|
||||
from erp_login_product_analysis import (
|
||||
DEFAULT_CONFIG,
|
||||
@@ -51,6 +51,11 @@ DEFAULT_FROM_DATE = date.today() - timedelta(days=1)
|
||||
DEFAULT_TO_DATE = DEFAULT_FROM_DATE
|
||||
MAX_RANGE_DAYS = 366
|
||||
EVIDENCE_DIR_NAME = "erp全店铺款式日报"
|
||||
# 聚水潭商品主题分析报表固定每页 200 行明细,且没有可用的页大小设置;
|
||||
# 款式扩到 198 个 ERP 编码后必须翻页,否则只拿到第一页的 62 个编码。
|
||||
PAGE_ROW_LIMIT = 200
|
||||
MAX_REPORT_PAGES = 12
|
||||
PAGE_TURN_TIMEOUT = 45
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -76,10 +81,19 @@ def inclusive_dates(start: date, end: date) -> tuple[date, ...]:
|
||||
|
||||
|
||||
def build_style_plans(payload: Mapping[str, Any]) -> tuple[StylePlan, ...]:
|
||||
plans: list[StylePlan] = []
|
||||
code_owners: dict[str, str] = {}
|
||||
"""Build per-style ERP plans, folding series codes into their anchor.
|
||||
|
||||
同系列多款时 loader 会把子款编码聚合并集到锚点(series.erp_owner),
|
||||
子款仍保留自己的编码供按款查询。采集计划按归属只保留一次:编码归锚点,
|
||||
子款剔除已被锚点承接的重复项;真正跨系列的重复仍然报错。
|
||||
"""
|
||||
|
||||
codes_by_style: dict[str, tuple[str, ...]] = {}
|
||||
owner_of: dict[str, str] = {}
|
||||
for raw_name, raw_config in (payload.get("styles") or {}).items():
|
||||
style_name = str(raw_name).strip()
|
||||
if not style_name:
|
||||
continue
|
||||
config = raw_config or {}
|
||||
codes = tuple(
|
||||
dict.fromkeys(
|
||||
@@ -88,15 +102,42 @@ def build_style_plans(payload: Mapping[str, Any]) -> tuple[StylePlan, ...]:
|
||||
if str(code).strip()
|
||||
)
|
||||
)
|
||||
if not style_name or not codes or not has_complete_style_identity(config):
|
||||
if not codes:
|
||||
continue
|
||||
codes_by_style[style_name] = codes
|
||||
owner = str((config.get("series") or {}).get("erp_owner") or "").strip()
|
||||
if owner:
|
||||
owner_of[style_name] = owner
|
||||
|
||||
holders: dict[str, list[str]] = {}
|
||||
for style_name, codes in codes_by_style.items():
|
||||
for code in codes:
|
||||
previous = code_owners.setdefault(code, style_name)
|
||||
if previous != style_name:
|
||||
raise ValueError(
|
||||
f"ERP style code {code} belongs to both {previous!r} and {style_name!r}"
|
||||
)
|
||||
plans.append(StylePlan(style_name=style_name, erp_style_codes=codes))
|
||||
holders.setdefault(code, []).append(style_name)
|
||||
|
||||
resolved: dict[str, str] = {}
|
||||
for code, names in holders.items():
|
||||
if len(names) == 1:
|
||||
resolved[code] = names[0]
|
||||
continue
|
||||
# 同系列归集:持有者里被其余持有者指认为 ERP 锚点的那一款承接该编码
|
||||
candidates = [
|
||||
name
|
||||
for name in names
|
||||
if any(owner_of.get(other) == name for other in names if other != name)
|
||||
]
|
||||
if len(candidates) == 1:
|
||||
resolved[code] = candidates[0]
|
||||
continue
|
||||
raise ValueError(
|
||||
f"ERP style code {code} belongs to both {names[0]!r} and {names[1]!r}"
|
||||
)
|
||||
|
||||
plans: list[StylePlan] = []
|
||||
for style_name, codes in codes_by_style.items():
|
||||
own_codes = tuple(code for code in codes if resolved.get(code) == style_name)
|
||||
if not own_codes:
|
||||
continue
|
||||
plans.append(StylePlan(style_name=style_name, erp_style_codes=own_codes))
|
||||
if not plans:
|
||||
raise RuntimeError("style configuration contains no ERP style codes")
|
||||
return tuple(plans)
|
||||
@@ -206,7 +247,9 @@ def set_all_shop_daily_filters(
|
||||
|
||||
|
||||
ENSURE_CUSTOM_COLUMNS_JS = r"""
|
||||
() => {
|
||||
(options) => {
|
||||
const redrawPage = (options && options.redrawPage) || 1;
|
||||
const forceRedraw = Boolean(options && options.forceRedraw);
|
||||
const cols = window._Cols || [];
|
||||
let changed = 0;
|
||||
for (const col of cols) {
|
||||
@@ -216,14 +259,58 @@ ENSURE_CUSTOM_COLUMNS_JS = r"""
|
||||
changed += 1;
|
||||
}
|
||||
}
|
||||
if (changed > 0 && typeof window._RptRefresh === 'function') {
|
||||
try { window._RptRefresh(); } catch (e) {}
|
||||
// 只改 visible 不会补出数据行缺失的 td:聚水潭按旧列定义渲染数据行,
|
||||
// 行内仍是旧的单元格数,按新列位置取值就会读到 undefined(历史全 0 成因)。
|
||||
// 换页会强制按当前列定义重绘,所以顺带跳到目标页。
|
||||
let redrawn = false;
|
||||
if (changed > 0 || forceRedraw) {
|
||||
// _RptRefresh 会走一次完整重算(公式列随之补齐),GotoPage 负责
|
||||
// 让数据行按新列定义重绘;两者都试,缺一都可能停在旧行结构上。
|
||||
if (forceRedraw && typeof window._RptRefresh === 'function') {
|
||||
try { window._RptRefresh(); } catch (e) {}
|
||||
}
|
||||
if (typeof GotoPage === 'function') {
|
||||
try { GotoPage(redrawPage || 1); redrawn = true; } catch (e) {}
|
||||
}
|
||||
}
|
||||
return changed;
|
||||
return { changed: changed, redrawn: redrawn };
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def ensure_custom_columns_rendered(
|
||||
page: Any, *, redraw_page: int = 1, force_redraw: bool = False
|
||||
) -> dict[str, Any]:
|
||||
"""确保自定义销量/退货量列可见,并触发一次重绘补齐数据行单元格。
|
||||
|
||||
2026-10-11 起聚水潭报表模板把自定义公式列重置为隐藏,隐藏列不渲染
|
||||
数据单元格;而 ``generate_report`` 每次查询都会把列可见性重置回隐藏。
|
||||
因此本函数必须在查询之后调用,并在 visible 变化时换页重绘,否则
|
||||
``自定义销量`` 整列读不到值。
|
||||
|
||||
``force_redraw`` 用于「列已 visible、但数据行仍按旧列数渲染」的中间态:
|
||||
此时列状态无需修改,仍要换页才能让每行补齐到新的单元格数。
|
||||
"""
|
||||
|
||||
summary = {"changed": 0, "redrawn": 0}
|
||||
for frame in page.frames:
|
||||
if "allsku.aspx" not in (frame.url or "").lower():
|
||||
continue
|
||||
try:
|
||||
result = frame.evaluate(
|
||||
ENSURE_CUSTOM_COLUMNS_JS,
|
||||
{"redrawPage": int(redraw_page), "forceRedraw": bool(force_redraw)},
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 自定义列补设失败: {type(exc).__name__}: {exc}", flush=True)
|
||||
continue
|
||||
if isinstance(result, dict):
|
||||
summary["changed"] += int(result.get("changed") or 0)
|
||||
if result.get("redrawn"):
|
||||
summary["redrawn"] += 1
|
||||
return summary
|
||||
|
||||
|
||||
def ensure_custom_columns_visible(page: Any) -> int:
|
||||
"""Keep 自定义销量/自定义退货量 visible before the report renders.
|
||||
|
||||
@@ -232,15 +319,47 @@ def ensure_custom_columns_visible(page: Any) -> int:
|
||||
(其尾部浮头重建可能抛错,但不影响数据行渲染)。
|
||||
"""
|
||||
|
||||
changed_total = 0
|
||||
return int(ensure_custom_columns_rendered(page, redraw_page=1)["changed"])
|
||||
|
||||
|
||||
def _allsku_frame(page: Any) -> Any:
|
||||
for frame in page.frames:
|
||||
if "allsku.aspx" not in (frame.url or "").lower():
|
||||
continue
|
||||
try:
|
||||
changed_total += int(frame.evaluate(ENSURE_CUSTOM_COLUMNS_JS))
|
||||
except Exception:
|
||||
continue
|
||||
return changed_total
|
||||
if "allsku.aspx" in (frame.url or "").lower():
|
||||
return frame
|
||||
return None
|
||||
|
||||
|
||||
def report_page_count(page: Any) -> int:
|
||||
frame = _allsku_frame(page)
|
||||
if frame is None:
|
||||
return 1
|
||||
try:
|
||||
return max(1, int(frame.evaluate("() => Number(window._RptMaxPage) || 1")))
|
||||
except Exception:
|
||||
return 1
|
||||
|
||||
|
||||
GOTO_REPORT_PAGE_JS = r"""
|
||||
(options) => {
|
||||
if (typeof GotoPage !== 'function') return false;
|
||||
try {
|
||||
GotoPage((options && options.page) || 1);
|
||||
return true;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def goto_report_page(page: Any, page_no: int) -> bool:
|
||||
frame = _allsku_frame(page)
|
||||
if frame is None:
|
||||
return False
|
||||
try:
|
||||
return bool(frame.evaluate(GOTO_REPORT_PAGE_JS, {"page": int(page_no)}))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def extract_style_rows(
|
||||
@@ -296,44 +415,117 @@ def extract_style_rows(
|
||||
}
|
||||
|
||||
// 隐藏列的 th 仍在表头,但数据行不渲染对应 td:数据行的第 j 格
|
||||
// 对应第 j 个可见列。用 window._Cols 把表头列索引换算成行内位
|
||||
// 置,避免隐藏列造成索引错位;拿不到 _Cols 时按原索引直取。
|
||||
let stylePos = styleCol;
|
||||
let productPos = productCol;
|
||||
let namePos = productNameCol;
|
||||
let salesPos = salesCol;
|
||||
let returnsPos = returnsCol;
|
||||
if (Array.isArray(window._Cols) && window._Cols.length === headerTexts.length) {
|
||||
const visibleIdx = [];
|
||||
for (let i = 0; i < window._Cols.length; i++) {
|
||||
if (window._Cols[i].visible !== false) visibleIdx.push(i);
|
||||
}
|
||||
stylePos = visibleIdx.indexOf(styleCol);
|
||||
productPos = productCol >= 0 ? visibleIdx.indexOf(productCol) : -1;
|
||||
namePos = productNameCol >= 0 ? visibleIdx.indexOf(productNameCol) : -1;
|
||||
salesPos = visibleIdx.indexOf(salesCol);
|
||||
returnsPos = visibleIdx.indexOf(returnsCol);
|
||||
// 对应第 j 个可见列。必须用 window._Cols 把表头列索引换算成行内
|
||||
// 位置——不做换算而按表头索引直取,会整体偏移到相邻列(历史表现
|
||||
// 为销量读到月累计/金额列,被放大上百倍)。换算所需的 _Cols 一旦
|
||||
// 缺失或与表头列数不一致,就无法确定位置,此时一律 fail closed,
|
||||
// 绝不退回"按原索引直取"。
|
||||
if (!Array.isArray(window._Cols) || window._Cols.length !== headerTexts.length) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'column_layout_unknown',
|
||||
error: `column layout unresolved: _Cols=${Array.isArray(window._Cols) ? window._Cols.length : 'none'} header=${headerTexts.length}`,
|
||||
header_texts: headerTexts,
|
||||
};
|
||||
}
|
||||
const colLabel = index => {
|
||||
const col = window._Cols[index];
|
||||
return norm(col && (col.text || col.defaultText));
|
||||
};
|
||||
// 表头 th 文本与 _Cols 必须是同一列,否则说明表头已重建而列定义
|
||||
// 尚未同步(中间态),按任何一个索引取都不可信。
|
||||
const labelMismatch = [
|
||||
[styleCol, '款式编码'],
|
||||
[salesCol, '自定义销量'],
|
||||
[returnsCol, '自定义退货量'],
|
||||
].filter(([index, expected]) => colLabel(index) !== norm(expected));
|
||||
if (labelMismatch.length) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'column_layout_unknown',
|
||||
error: `header/_Cols label mismatch: ${labelMismatch.map(([i]) => i).join(',')}`,
|
||||
header_texts: headerTexts,
|
||||
};
|
||||
}
|
||||
const visibleIdx = [];
|
||||
for (let i = 0; i < window._Cols.length; i++) {
|
||||
if (window._Cols[i].visible !== false) visibleIdx.push(i);
|
||||
}
|
||||
const toPos = index => (index >= 0 ? visibleIdx.indexOf(index) : -1);
|
||||
const stylePos = toPos(styleCol);
|
||||
const productPos = toPos(productCol);
|
||||
const namePos = toPos(productNameCol);
|
||||
const salesPos = toPos(salesCol);
|
||||
const returnsPos = toPos(returnsCol);
|
||||
// 值域交叉校验用的参考列:自定义销量是"当期销量"口径,正常不可能
|
||||
// 超过同行的月销量一个量级。读到月累计/金额列时这里会炸。
|
||||
const monthSalesPos = toPos(findCol('月销量'));
|
||||
const monthReturnsPos = toPos(findCol('月退货量'));
|
||||
// 每行应有几个 td:等于可见列数。数量不符=该行仍按旧列定义渲染
|
||||
// (重绘未完成),位置不可信。
|
||||
const expectedCells = visibleIdx.length;
|
||||
if (stylePos < 0 || salesPos < 0 || returnsPos < 0) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'custom_columns_hidden',
|
||||
error: `custom sales/return columns hidden: style=${stylePos}, sales=${salesPos}, returns=${returnsPos}`,
|
||||
header_texts: headerTexts,
|
||||
};
|
||||
}
|
||||
|
||||
const rows = [];
|
||||
let renderedSalesCells = 0;
|
||||
let blankSalesCells = 0;
|
||||
let zeroSalesRows = 0;
|
||||
let layoutMismatchRows = 0;
|
||||
let outOfRangeRows = 0;
|
||||
let monthRefRows = 0;
|
||||
for (const tr of Array.from(document.querySelectorAll('tr'))) {
|
||||
const cells = Array.from(tr.querySelectorAll('td, th'));
|
||||
if (cells.length <= Math.max(stylePos, salesPos, returnsPos)) continue;
|
||||
// 只用 td:表头行的 th 会让列位置整体右移,
|
||||
// 而分组行/浮头行里混入的 th 同样会污染样式编码列的取值。
|
||||
const cells = Array.from(tr.querySelectorAll('td'));
|
||||
if (!cells.length) continue;
|
||||
const styleCode = norm(cells[stylePos]?.textContent);
|
||||
// 只有命中本次请求编码的行才是数据行:报表里的小计/分组行 td
|
||||
// 数与列定义天然不同,不能拿它们判定列错位。
|
||||
if (!requested.has(styleCode)) continue;
|
||||
// 数据行 td 数必须与可见列数完全相等。少一格就说明该行仍按旧
|
||||
// 列定义渲染,此时任何位置取值都可能落到相邻列(读到月累计或
|
||||
// 金额),必须整页拒绝重绘,不能按"位置够长就往里取"处理。
|
||||
if (cells.length !== expectedCells) {
|
||||
layoutMismatchRows += 1;
|
||||
continue;
|
||||
}
|
||||
const salesText = String(cells[salesPos]?.textContent ?? '').trim();
|
||||
const returnText = String(cells[returnsPos]?.textContent ?? '').trim();
|
||||
const salesValue = parseInteger(salesText);
|
||||
const returnValue = parseInteger(returnText);
|
||||
// 值域交叉校验:自定义销量是"当期"口径,正常不会超过同行月销量
|
||||
// 一个量级。列错位读到月累计/金额时这里的比例会失真,用于兜住
|
||||
// 「结构看起来正常但语义错列」的情况(2026-10-30 事故)。
|
||||
if (monthSalesPos >= 0) {
|
||||
const monthSalesText = String(cells[monthSalesPos]?.textContent ?? '').trim();
|
||||
if (monthSalesText) {
|
||||
monthRefRows += 1;
|
||||
if (salesValue > parseInteger(monthSalesText) * 2 + 50) outOfRangeRows += 1;
|
||||
}
|
||||
}
|
||||
if (monthReturnsPos >= 0) {
|
||||
const monthReturnsText = String(cells[monthReturnsPos]?.textContent ?? '').trim();
|
||||
if (monthReturnsText) {
|
||||
monthRefRows += 1;
|
||||
if (returnValue > parseInteger(monthReturnsText) * 2 + 50) outOfRangeRows += 1;
|
||||
}
|
||||
}
|
||||
if (salesText) renderedSalesCells += 1;
|
||||
else blankSalesCells += 1;
|
||||
if (salesValue === 0) zeroSalesRows += 1;
|
||||
rows.push({
|
||||
erp_style_code: styleCode,
|
||||
product_code: productPos >= 0 ? norm(cells[productPos]?.textContent) : '',
|
||||
product_name: namePos >= 0 ? norm(cells[namePos]?.textContent) : '',
|
||||
sales: parseInteger(cells[salesPos]?.textContent),
|
||||
return_qty: parseInteger(cells[returnsPos]?.textContent),
|
||||
sales: salesValue,
|
||||
return_qty: returnValue,
|
||||
});
|
||||
}
|
||||
const foundCodes = Array.from(new Set(rows.map(row => row.erp_style_code))).sort();
|
||||
@@ -344,7 +536,58 @@ def extract_style_rows(
|
||||
if (!rows.length) {
|
||||
return {
|
||||
ok: false,
|
||||
error: 'report contains none of the requested ERP style codes',
|
||||
reason: layoutMismatchRows ? 'row_layout_mismatch' : 'no_requested_rows',
|
||||
error: `report contains none of the requested ERP style codes (mismatched rows: ${layoutMismatchRows})`,
|
||||
layout_mismatch_rows: layoutMismatchRows,
|
||||
header_texts: headerTexts,
|
||||
};
|
||||
}
|
||||
if (renderedSalesCells === 0) {
|
||||
// 自定义销量是聚水潭的公式列:列被隐藏或公式尚未算完时
|
||||
// 单元格是空的。空 ≠ 0,此时返回未就绪让调用方继续等待,
|
||||
// 绝不能静默写 0(历史上 10-13~10-19、10-22 全表为 0 就是这样来的)。
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'custom_sales_not_rendered',
|
||||
error: `custom sales cells blank: ${blankSalesCells}/${rows.length}`,
|
||||
row_count: rows.length,
|
||||
sales_col: salesCol,
|
||||
sales_pos: salesPos,
|
||||
header_texts: headerTexts,
|
||||
};
|
||||
}
|
||||
if (layoutMismatchRows > 0) {
|
||||
// 有行的 td 数与可见列数不符:整页列位置不可信,重绘后重读。
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'row_layout_mismatch',
|
||||
error: `row cells mismatch: ${layoutMismatchRows} rows`,
|
||||
row_count: rows.length,
|
||||
expected_cells: expectedCells,
|
||||
header_texts: headerTexts,
|
||||
};
|
||||
}
|
||||
if (outOfRangeRows > 0) {
|
||||
// 结构与列名都对上了,但值本身不可信(销量超过同行月销量一个
|
||||
// 量级)。典型成因是日期筛选未真正生效、报表返回了月/年累计值。
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'sales_out_of_range',
|
||||
error: `sales out of monthly range: ${outOfRangeRows} rows (reference rows ${monthRefRows})`,
|
||||
row_count: rows.length,
|
||||
out_of_range_rows: outOfRangeRows,
|
||||
header_texts: headerTexts,
|
||||
};
|
||||
}
|
||||
if (rows.length >= 100 && zeroSalesRows / rows.length >= 0.8) {
|
||||
// 公式列在补值过程中会先把整列渲染成 0 再逐个算出来;正常日
|
||||
// 零销量行占比约 50%~60%(2026-10 实测),超过 80% 说明整页
|
||||
// 还没算完。宁可继续等也不要写一批 0 覆盖真实销量。
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'custom_sales_zero_placeholder',
|
||||
error: `custom sales zero rows: ${zeroSalesRows}/${rows.length}`,
|
||||
row_count: rows.length,
|
||||
header_texts: headerTexts,
|
||||
};
|
||||
}
|
||||
@@ -355,6 +598,25 @@ def extract_style_rows(
|
||||
missing_codes: missingCodes,
|
||||
signature,
|
||||
header_texts: headerTexts,
|
||||
rendered_sales_cells: renderedSalesCells,
|
||||
blank_sales_cells: blankSalesCells,
|
||||
zero_sales_rows: zeroSalesRows,
|
||||
layout_mismatch_rows: layoutMismatchRows,
|
||||
// 列布局快照:写进 evidence,下次出问题可直接比对,不用复现。
|
||||
layout: {
|
||||
header_cells: headerTexts.length,
|
||||
visible_cells: expectedCells,
|
||||
style_col: styleCol,
|
||||
style_pos: stylePos,
|
||||
sales_col: salesCol,
|
||||
sales_pos: salesPos,
|
||||
returns_col: returnsCol,
|
||||
returns_pos: returnsPos,
|
||||
month_sales_pos: monthSalesPos,
|
||||
month_returns_pos: monthReturnsPos,
|
||||
row_cells: expectedCells,
|
||||
month_ref_rows: monthRefRows,
|
||||
},
|
||||
};
|
||||
}
|
||||
""",
|
||||
@@ -476,6 +738,56 @@ def aggregate_style_records(
|
||||
]
|
||||
|
||||
|
||||
def _recent_sales_baseline(report_date: date, *, days: int = 14) -> float | None:
|
||||
"""最近若干天单日销量合计的中位数,用于识别量级异常。"""
|
||||
|
||||
with get_conn() as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY day_total)
|
||||
FROM (
|
||||
SELECT metric_date, sum(sales) AS day_total
|
||||
FROM erp_all_shop_style_daily_metrics
|
||||
WHERE metric_date < %s
|
||||
AND metric_date >= %s::date - (%s || ' days')::interval
|
||||
GROUP BY metric_date
|
||||
) recent
|
||||
""",
|
||||
(report_date, report_date, days),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
value = row[0] if row else None
|
||||
return float(value) if value else None
|
||||
|
||||
|
||||
def _guard_daily_total(
|
||||
report_date: date,
|
||||
records: Sequence[Mapping[str, Any]],
|
||||
*,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
"""单日合计与近 14 天中位数差 8 倍以上就拒绝落库。
|
||||
|
||||
2026-10-30 事故里,日期筛选未真正生效使报表返回月累计值,单日合计
|
||||
704,140 而正常约 2,000(350 倍)。列级校验挡住列错位,这道闸挡住
|
||||
「列对但统计口径错」这一层,宁可失败也不写脏数据。
|
||||
"""
|
||||
|
||||
if force:
|
||||
return
|
||||
total = sum(int(record.get("sales") or 0) for record in records)
|
||||
baseline = _recent_sales_baseline(report_date)
|
||||
if not baseline or baseline <= 0:
|
||||
return
|
||||
if total <= baseline * 8:
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"all-shop daily total looks wrong for {report_date.isoformat()}: "
|
||||
f"sales={total} vs recent median={baseline:.0f} "
|
||||
f"({total / baseline:.1f}x). Refusing to persist; rerun with --force if business volume really changed."
|
||||
)
|
||||
|
||||
|
||||
def write_evidence(
|
||||
report_date: date,
|
||||
*,
|
||||
@@ -493,8 +805,12 @@ def write_evidence(
|
||||
"sales_header": "自定义销量",
|
||||
"return_header": "自定义退货量",
|
||||
"selection": dict(selection),
|
||||
"pages": int(report.get("pages") or 1),
|
||||
"max_page": int(report.get("max_page") or 1),
|
||||
"found_codes": list(report.get("found_codes") or ()),
|
||||
"missing_codes": list(report.get("missing_codes") or ()),
|
||||
# 列布局快照:出问题时可直接判断「列错位」还是「数据本身」,无需复现。
|
||||
"layout": dict(report.get("layout") or {}),
|
||||
"rows": list(report.get("rows") or ()),
|
||||
"styles": [
|
||||
{
|
||||
@@ -510,6 +826,206 @@ def write_evidence(
|
||||
return output_path
|
||||
|
||||
|
||||
def _wait_rendered_page(
|
||||
page: Any,
|
||||
requested_codes: Sequence[str],
|
||||
*,
|
||||
page_no: int,
|
||||
timeout_seconds: int,
|
||||
previous_signature: str | None = None,
|
||||
expect_full_page: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Wait for one report page whose custom sales cells are actually rendered.
|
||||
|
||||
``previous_signature`` 用于翻页:``GotoPage`` 是异步的,调用后 DOM 仍是
|
||||
上一页,若只看「签名稳定」会立刻把上一页数据当成新页收下,
|
||||
结果多页合并后去重回同一批行(10-28 实测 3 页只有 200 行)。
|
||||
|
||||
``expect_full_page``:报表每页固定 200 行,除最后一页外不该出现不满页。
|
||||
不满页说明数据行还在陆续插入,此时签名字段虽已稳定但行集不全,
|
||||
直接收下会漏采编码(2026-10-30 实测第 1 页只有 174 行、只命中 57/199
|
||||
个编码,后续页面也补不回第一页缺的行)。
|
||||
"""
|
||||
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
last_signature: str | None = None
|
||||
stable = 0
|
||||
redraws = 0
|
||||
reached = previous_signature is None
|
||||
last_error = "report page did not render custom sales column"
|
||||
|
||||
def _declared_pages() -> int:
|
||||
"""报表自己声明的总页数;不可信时返回 1(只做加法兜底)。"""
|
||||
|
||||
frame = _allsku_frame(page)
|
||||
if frame is None:
|
||||
return 1
|
||||
try:
|
||||
return max(1, int(frame.evaluate("() => Number(window._RptMaxPage) || 1")))
|
||||
except Exception:
|
||||
return 1
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
frame = _allsku_frame(page)
|
||||
if frame is None:
|
||||
last_error = "allsku iframe not found"
|
||||
time.sleep(0.5)
|
||||
continue
|
||||
try:
|
||||
result = extract_style_rows(frame, requested_codes)
|
||||
except Exception as exc:
|
||||
result = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
|
||||
if result.get("ok") and len(result.get("rows") or ()) < PAGE_ROW_LIMIT:
|
||||
# 不满 200 行只有两种可能:这是最后一页,或数据行还在陆续插入。
|
||||
# 报表声明的总页数大于当前页时必然是后者,继续等。
|
||||
short = len(result.get("rows") or ())
|
||||
if expect_full_page or _declared_pages() > page_no:
|
||||
result = {
|
||||
"ok": False,
|
||||
"reason": "partial_page",
|
||||
"error": f"page {page_no} rendered {short}/{PAGE_ROW_LIMIT} rows",
|
||||
}
|
||||
if result.get("ok"):
|
||||
signature = str(result.get("signature") or "")
|
||||
if not reached:
|
||||
# 翻页后内容尚未变化:继续等新页渲染出来。
|
||||
if signature == previous_signature:
|
||||
last_signature = None
|
||||
stable = 0
|
||||
time.sleep(0.5)
|
||||
continue
|
||||
reached = True
|
||||
if signature == last_signature:
|
||||
stable += 1
|
||||
else:
|
||||
last_signature = signature
|
||||
stable = 1
|
||||
# 自定义销量是公式列,翻页后先出 0、再逐个单元格补齐;
|
||||
# 稳定阈值取 6 次(≈2.4 秒不变)才能跨过「补值中」的中间态。
|
||||
required = 5 if result.get("reason") == "no_data" else 6
|
||||
if stable >= required:
|
||||
result["page_no"] = page_no
|
||||
result["frame_url"] = frame.url
|
||||
return result
|
||||
else:
|
||||
last_error = str(result.get("error") or result.get("reason") or last_error)
|
||||
last_signature = None
|
||||
stable = 0
|
||||
# 列为空/被隐藏/行内单元格错位/整列占位 0 都是「未渲染完」,
|
||||
# 不是「销量为 0」:补设列并强制重绘后重读。
|
||||
reason = result.get("reason")
|
||||
if reason in (
|
||||
"custom_sales_not_rendered",
|
||||
"custom_columns_hidden",
|
||||
"column_layout_unknown",
|
||||
"row_layout_mismatch",
|
||||
"custom_sales_zero_placeholder",
|
||||
"sales_out_of_range",
|
||||
"partial_page",
|
||||
) and redraws < 6:
|
||||
summary = ensure_custom_columns_rendered(
|
||||
page,
|
||||
redraw_page=page_no,
|
||||
# 这几种原因都说明「列已可见但数据行/公式没算完」,
|
||||
# 必须强制重算,否则只会原地等同一份旧 DOM。
|
||||
force_redraw=True,
|
||||
)
|
||||
if summary.get("changed") or summary.get("redrawn"):
|
||||
redraws += 1
|
||||
last_error = f"{last_error}; redraw #{redraws} on page {page_no}"
|
||||
time.sleep(3.0)
|
||||
continue
|
||||
time.sleep(0.4)
|
||||
raise TimeoutError(
|
||||
f"timeout waiting for rendered page {page_no}: {last_error}"
|
||||
)
|
||||
|
||||
|
||||
def collect_report_pages(
|
||||
page: Any,
|
||||
requested_codes: Sequence[str],
|
||||
*,
|
||||
report_timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Collect every page of the allsku report and merge the product rows.
|
||||
|
||||
报表固定每页 200 行;款式扩到 198 个 ERP 编码后单页放不下,
|
||||
只读第一页会漏掉大半编码(10-28 就是这样:198 编码只覆盖 62 个)。
|
||||
|
||||
翻页条件按「上一页是否装满一页」判断,不依赖 ``window._RptMaxPage``:
|
||||
该变量在增量查询后会被重置为 1,实测连续补采 8 天只有第一天翻页成功。
|
||||
"""
|
||||
|
||||
first = _wait_rendered_page(
|
||||
page,
|
||||
requested_codes,
|
||||
page_no=1,
|
||||
timeout_seconds=report_timeout,
|
||||
)
|
||||
# 等待结束后再读总页数:生成报表的瞬间 _RptMaxPage 还是旧值 1。
|
||||
max_page = report_page_count(page)
|
||||
rows: list[dict[str, Any]] = list(first.get("rows") or ())
|
||||
previous_signature = str(first.get("signature") or "")
|
||||
current_count = len(rows)
|
||||
visited = 1
|
||||
page_no = 1
|
||||
# 翻页条件:上一页装满 200 行,或 window._RptMaxPage 指出还有后续页。
|
||||
# 后者用于兜住「第 1 页渲染不全、行数不足 200」的情况——只看行数会直接
|
||||
# 判成最后一页而漏采。该变量在增量查询后可能被重置为 1,因此只做加法
|
||||
# 兜底,翻到空页时由 _wait_rendered_page 超时退出。
|
||||
while current_count >= PAGE_ROW_LIMIT or page_no < max_page:
|
||||
page_no += 1
|
||||
if page_no > MAX_REPORT_PAGES:
|
||||
print(f"[WARN] 全店铺报表页数超过 {MAX_REPORT_PAGES},按上限停止", flush=True)
|
||||
break
|
||||
if not goto_report_page(page, page_no):
|
||||
print(f"[WARN] 全店铺报表翻页失败,停在页 {visited}", flush=True)
|
||||
break
|
||||
try:
|
||||
last = _wait_rendered_page(
|
||||
page,
|
||||
requested_codes,
|
||||
page_no=page_no,
|
||||
timeout_seconds=min(report_timeout, PAGE_TURN_TIMEOUT),
|
||||
previous_signature=previous_signature,
|
||||
expect_full_page=page_no < max_page,
|
||||
)
|
||||
except TimeoutError:
|
||||
# 翻页无新内容=已经到最后一页(GotoPage 越界不报错,只是不刷新)。
|
||||
break
|
||||
previous_signature = str(last.get("signature") or "")
|
||||
page_rows = list(last.get("rows") or ())
|
||||
print(f"[ERP ALL] 第 {page_no} 页 {len(page_rows)} 行", flush=True)
|
||||
rows.extend(page_rows)
|
||||
current_count = len(page_rows)
|
||||
visited = page_no
|
||||
|
||||
deduped: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for row in rows:
|
||||
key = (str(row.get("erp_style_code") or ""), str(row.get("product_code") or ""))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped.append(row)
|
||||
found_codes = sorted({str(row.get("erp_style_code") or "") for row in deduped})
|
||||
missing_codes = [code for code in requested_codes if str(code) not in found_codes]
|
||||
return {
|
||||
"rows": deduped,
|
||||
"found_codes": found_codes,
|
||||
"missing_codes": missing_codes,
|
||||
"signature": json.dumps(
|
||||
[[row.get("erp_style_code"), row.get("product_code"), row.get("sales"), row.get("return_qty")]
|
||||
for row in deduped],
|
||||
ensure_ascii=False,
|
||||
),
|
||||
"pages": visited,
|
||||
"max_page": visited,
|
||||
"header_texts": first.get("header_texts"),
|
||||
"layout": first.get("layout"),
|
||||
}
|
||||
|
||||
|
||||
def collect_one_date(
|
||||
page: Any,
|
||||
filter_frame: Any,
|
||||
@@ -520,20 +1036,23 @@ def collect_one_date(
|
||||
previous_signature: str | None,
|
||||
report_timeout: int,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
del previous_url, previous_signature # 多页合并后按页内稳定签名判断新鲜度
|
||||
codes = all_erp_codes(plans)
|
||||
erp.clear_code(filter_frame)
|
||||
selection = set_all_shop_daily_filters(
|
||||
filter_frame, report_date=report_date, erp_codes=codes
|
||||
)
|
||||
ensure_custom_columns_visible(page)
|
||||
erp.generate_report(filter_frame)
|
||||
report = wait_for_style_rows(
|
||||
page,
|
||||
requested_codes=codes,
|
||||
previous_url=previous_url,
|
||||
previous_signature=previous_signature,
|
||||
filter_frame=filter_frame,
|
||||
timeout_seconds=report_timeout,
|
||||
# 查询会把自定义列重置回隐藏;补列交给 collect_report_pages 的等待循环,
|
||||
# 那里能区分「列隐藏」与「列可见但数据行未重绘」两种状态再重绘。
|
||||
report = collect_report_pages(
|
||||
page, codes, report_timeout=report_timeout
|
||||
)
|
||||
print(
|
||||
f"[ERP ALL] {report_date.isoformat()} 采集 {report['pages']} 页 "
|
||||
f"行 {len(report['rows'])} 命中编码 {len(report['found_codes'])}/{len(codes)} "
|
||||
f"缺 {len(report['missing_codes'])}",
|
||||
flush=True,
|
||||
)
|
||||
return selection, report
|
||||
|
||||
@@ -609,6 +1128,7 @@ def collect_and_persist(
|
||||
timeout_ms: int,
|
||||
report_timeout: int,
|
||||
report_attempts: int,
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
config = load_config(config_path)
|
||||
login_config = config.get("login") or {}
|
||||
@@ -641,6 +1161,7 @@ def collect_and_persist(
|
||||
selected_shop_count=selected_shop_count,
|
||||
source=provisional_source,
|
||||
)
|
||||
_guard_daily_total(report_date, records, force=force)
|
||||
evidence_path = write_evidence(
|
||||
report_date, selection=selection, report=report, records=records
|
||||
)
|
||||
@@ -766,6 +1287,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
timeout_ms=args.timeout_ms,
|
||||
report_timeout=args.report_timeout,
|
||||
report_attempts=args.report_attempts,
|
||||
force=args.force,
|
||||
)
|
||||
print(f"[DONE] Collected {len(pending_dates)} all-shop daily reports.")
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
#!/usr/bin/env python3
|
||||
"""回填采集表的 style_id (= dim_style.id),完成款式主键统一。
|
||||
|
||||
背景
|
||||
----
|
||||
2026-11-01 之前,采集端所有表只用 ``style_name`` 关联款式,而 data-hub 侧 22 张
|
||||
表用的是 ``style_id``(= dim_style.id)。同一库里两种引用方式的后果是:款名在
|
||||
dim_style 被改一次,历史指标表就整体变孤儿,join 静默丢数据。
|
||||
|
||||
本脚本负责把**历史数据**补上 style_id;**新数据**由写入端负责:
|
||||
* loader 下发(``merge_lifecycle_style_records`` → ``style_config_loader``)
|
||||
* db 层兜底(``db.attach_style_ids`` / ``db.resolve_style_ids_by_product``)
|
||||
|
||||
两类表,两种回填口径
|
||||
--------------------
|
||||
1. **按款名**(13 张):款式清单本来就来自 dim_style loader,理论上 100% 命中。
|
||||
2. **按平台商品 ID**(``product_daily_metrics`` + 三张广告表):表里没有可用的
|
||||
款名列(``product_daily_metrics`` 完全没有;广告表的 ``style_name`` 由平台
|
||||
报表决定,京东 27%、抖音 40% 是空的),只能拿 ``platform + product_id`` 去
|
||||
``tm_spus / jd_spus / dy_spus`` 里反查。未建档的商品 ID 回填不了 —— 这不是
|
||||
脚本的问题,要用 ``v_style_unmatched_product`` 推动补建档。
|
||||
广告表回填时会顺带把空的 ``style_name`` 补成 dim_style 的权威款名,让下游
|
||||
现有按款名的 join 立即生效。
|
||||
|
||||
用法
|
||||
----
|
||||
.venv/Scripts/python.exe backfill_style_id.py --dry-run # 只报数
|
||||
.venv/Scripts/python.exe backfill_style_id.py # 执行
|
||||
.venv/Scripts/python.exe backfill_style_id.py --tables daily_style_metrics
|
||||
.venv/Scripts/python.exe backfill_style_id.py --skip-schema # 不跑建表
|
||||
.venv/Scripts/python.exe backfill_style_id.py --skip-ad-tables # 只补款名表
|
||||
|
||||
幂等:所有 UPDATE 都带 ``style_id IS NULL`` 条件,重复执行不会覆盖已有归属。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from typing import Any, Sequence
|
||||
|
||||
try:
|
||||
from .db import (
|
||||
STYLE_MATCH_BY_NAME,
|
||||
STYLE_MATCH_BY_PRODUCT,
|
||||
STYLE_MATCH_UNMATCHED,
|
||||
get_conn,
|
||||
init_schema,
|
||||
resolve_style_ids_by_product,
|
||||
)
|
||||
except ImportError: # Direct script execution from the module root.
|
||||
from db import ( # type: ignore[no-redef]
|
||||
STYLE_MATCH_BY_NAME,
|
||||
STYLE_MATCH_BY_PRODUCT,
|
||||
STYLE_MATCH_UNMATCHED,
|
||||
get_conn,
|
||||
init_schema,
|
||||
resolve_style_ids_by_product,
|
||||
)
|
||||
|
||||
|
||||
# 按款名回填:这些表的 style_name 来自 dim_style 款式清单,属于「必须匹配」。
|
||||
NAME_TABLES: tuple[str, ...] = (
|
||||
"daily_style_metrics",
|
||||
"weekly_style_metrics",
|
||||
"erp_all_shop_style_daily_metrics",
|
||||
"daily_persona_metrics",
|
||||
"main_image_creatives",
|
||||
"fact_style_analysis_reports",
|
||||
"decline_events",
|
||||
"fact_alert_history",
|
||||
"jd_self_weekly_sales",
|
||||
"persona_screenshots",
|
||||
"jd_product_ad_daily_metrics",
|
||||
"dy_product_ad_daily_metrics",
|
||||
"tm_product_ad_daily_metrics",
|
||||
)
|
||||
|
||||
# 按平台商品 ID 回填:允许未匹配(商品 ID 还没建档到 dim_style.spus)。
|
||||
PRODUCT_TABLE = "product_daily_metrics"
|
||||
PLATFORM_COLUMNS = ("tm", "jd", "dy")
|
||||
|
||||
# 广告表:表里带平台商品 ID,但 style_name 常为空(平台报表不给款式名)。
|
||||
# 只按款名的回填对它们无效,必须再走一遍商品 ID 反查。
|
||||
# {表名: (商品 ID 列, 平台编码)}
|
||||
AD_PRODUCT_TABLES: dict[str, tuple[str, str]] = {
|
||||
"tm_product_ad_daily_metrics": ("product_id", "tm"),
|
||||
"jd_product_ad_daily_metrics": ("spu_id", "jd"),
|
||||
"dy_product_ad_daily_metrics": ("product_id", "dy"),
|
||||
}
|
||||
|
||||
|
||||
def _coverage(conn, table: str) -> tuple[int, int]:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"SELECT COUNT(*), COUNT(style_id) FROM {table}"
|
||||
)
|
||||
rows, with_id = cur.fetchone()
|
||||
return int(rows or 0), int(with_id or 0)
|
||||
|
||||
|
||||
def _backfill_by_name(conn, table: str, *, dry_run: bool) -> dict[str, Any]:
|
||||
"""按款名精确回填。返回 {before, after, updated}。"""
|
||||
|
||||
before_rows, before_with_id = _coverage(conn, table)
|
||||
if dry_run:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""SELECT COUNT(*)
|
||||
FROM {table} t
|
||||
JOIN dim_style s ON s.style_name = t.style_name
|
||||
WHERE t.style_id IS NULL"""
|
||||
)
|
||||
pending = int(cur.fetchone()[0] or 0)
|
||||
return {
|
||||
"table": table,
|
||||
"rows": before_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"updated": pending,
|
||||
"dry_run": True,
|
||||
}
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""UPDATE {table} t
|
||||
SET style_id = s.id,
|
||||
style_match_method = COALESCE(t.style_match_method, %s)
|
||||
FROM dim_style s
|
||||
WHERE t.style_id IS NULL
|
||||
AND t.style_name = s.style_name""",
|
||||
(STYLE_MATCH_BY_NAME,),
|
||||
)
|
||||
updated = cur.rowcount or 0
|
||||
after_rows, after_with_id = _coverage(conn, table)
|
||||
return {
|
||||
"table": table,
|
||||
"rows": after_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"with_style_id_after": after_with_id,
|
||||
"updated": updated,
|
||||
}
|
||||
|
||||
|
||||
def _backfill_product_table(conn, *, dry_run: bool) -> dict[str, Any]:
|
||||
"""按 platform + product_id 反查回填 product_daily_metrics。
|
||||
|
||||
逐平台取去重商品 ID → 用 db.resolve_style_ids_by_product 解析(锚点优先、
|
||||
款名最短优先,与写入端同一套优先级)→ 按 (platform, product_id) 批量更新。
|
||||
"""
|
||||
|
||||
before_rows, before_with_id = _coverage(conn, PRODUCT_TABLE)
|
||||
planned: list[tuple[int, str, str]] = []
|
||||
unmatched_total = 0
|
||||
with conn.cursor() as cur:
|
||||
for platform in PLATFORM_COLUMNS:
|
||||
cur.execute(
|
||||
f"""SELECT DISTINCT product_id FROM {PRODUCT_TABLE}
|
||||
WHERE platform = %s AND style_id IS NULL AND product_id IS NOT NULL""",
|
||||
(platform,),
|
||||
)
|
||||
product_ids = [str(row[0]) for row in cur.fetchall() if row[0] is not None]
|
||||
if not product_ids:
|
||||
continue
|
||||
matched = resolve_style_ids_by_product(conn, platform, product_ids)
|
||||
unmatched_total += len(product_ids) - len(matched)
|
||||
planned.extend(
|
||||
(style_id, platform, product_id)
|
||||
for product_id, style_id in matched.items()
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
return {
|
||||
"table": PRODUCT_TABLE,
|
||||
"rows": before_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"planned_keys": len(planned),
|
||||
"unmatched_product_ids": unmatched_total,
|
||||
"dry_run": True,
|
||||
}
|
||||
|
||||
updated = 0
|
||||
with conn.cursor() as cur:
|
||||
for style_id, platform, product_id in planned:
|
||||
cur.execute(
|
||||
f"""UPDATE {PRODUCT_TABLE}
|
||||
SET style_id = %s, style_match_method = %s
|
||||
WHERE platform = %s AND product_id = %s AND style_id IS NULL""",
|
||||
(style_id, STYLE_MATCH_BY_PRODUCT, platform, product_id),
|
||||
)
|
||||
updated += cur.rowcount or 0
|
||||
# 回填不了的显式标记 unmatched,供 v_style_unmatched_product 统计
|
||||
cur.execute(
|
||||
f"""UPDATE {PRODUCT_TABLE}
|
||||
SET style_match_method = %s
|
||||
WHERE style_id IS NULL AND style_match_method IS NULL""",
|
||||
(STYLE_MATCH_UNMATCHED,),
|
||||
)
|
||||
|
||||
after_rows, after_with_id = _coverage(conn, PRODUCT_TABLE)
|
||||
return {
|
||||
"table": PRODUCT_TABLE,
|
||||
"rows": after_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"with_style_id_after": after_with_id,
|
||||
"updated": updated,
|
||||
"unmatched_product_ids": unmatched_total,
|
||||
}
|
||||
|
||||
|
||||
def _backfill_ad_tables_by_product(
|
||||
conn, tables: Sequence[str], *, dry_run: bool
|
||||
) -> list[dict[str, Any]]:
|
||||
"""按平台商品 ID 反查回填广告表的 style_id(顺带补空的 style_name)。
|
||||
|
||||
广告表的 ``style_name`` 由平台报表决定,经常为空(京东 27%、抖音 40%),
|
||||
只按款名回填对它们完全无效。这里改为用 ``spu_id`` / ``product_id`` 去
|
||||
``dim_style.<platform>_spus`` 反查,与写入端 ``resolve_style_ids_by_product``
|
||||
共用同一套优先级。
|
||||
|
||||
``style_name`` 只在原来为空时补写(``NULLIF(BTRIM(...), '')``),不覆盖
|
||||
平台已经给出的值 —— 这样 data-hub 侧现有按款名 join 的口径立刻能多认回
|
||||
一批行,不必等主键切换。
|
||||
"""
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for table in tables:
|
||||
id_column, platform = AD_PRODUCT_TABLES[table]
|
||||
before_rows, before_with_id = _coverage(conn, table)
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""SELECT DISTINCT {id_column} FROM {table}
|
||||
WHERE style_id IS NULL AND {id_column} IS NOT NULL
|
||||
AND BTRIM({id_column}) <> ''"""
|
||||
)
|
||||
product_ids = [str(row[0]) for row in cur.fetchall() if row[0] is not None]
|
||||
|
||||
matched: dict[str, int] = {}
|
||||
if product_ids:
|
||||
matched = resolve_style_ids_by_product(conn, platform, product_ids)
|
||||
|
||||
if dry_run:
|
||||
results.append(
|
||||
{
|
||||
"table": table,
|
||||
"rows": before_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"planned_keys": len(matched),
|
||||
"unmatched_product_ids": len(product_ids) - len(matched),
|
||||
"dry_run": True,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
name_by_id: dict[int, str] = {}
|
||||
if matched:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, style_name FROM dim_style WHERE id = ANY(%s)",
|
||||
(list({v for v in matched.values()}),),
|
||||
)
|
||||
name_by_id = {int(i): n for i, n in cur.fetchall()}
|
||||
|
||||
updated = 0
|
||||
with conn.cursor() as cur:
|
||||
for product_id, style_id in matched.items():
|
||||
cur.execute(
|
||||
f"""UPDATE {table}
|
||||
SET style_id = %s,
|
||||
style_match_method = %s,
|
||||
style_name = COALESCE(NULLIF(BTRIM(style_name), ''), %s)
|
||||
WHERE {id_column} = %s AND style_id IS NULL""",
|
||||
(style_id, STYLE_MATCH_BY_PRODUCT, name_by_id.get(style_id), product_id),
|
||||
)
|
||||
updated += cur.rowcount or 0
|
||||
cur.execute(
|
||||
f"""UPDATE {table} SET style_match_method = %s
|
||||
WHERE style_id IS NULL AND style_match_method IS NULL""",
|
||||
(STYLE_MATCH_UNMATCHED,),
|
||||
)
|
||||
|
||||
after_rows, after_with_id = _coverage(conn, table)
|
||||
results.append(
|
||||
{
|
||||
"table": table,
|
||||
"rows": after_rows,
|
||||
"with_style_id_before": before_with_id,
|
||||
"with_style_id_after": after_with_id,
|
||||
"updated": updated,
|
||||
"unmatched_product_ids": len(product_ids) - len(matched),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def _print_coverage(conn) -> None:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT tbl, rows, with_style_id FROM v_style_id_coverage ORDER BY tbl")
|
||||
rows = cur.fetchall()
|
||||
cur.execute("SELECT platform, COUNT(*) FROM v_style_unmatched_product GROUP BY platform ORDER BY platform")
|
||||
unmatched = cur.fetchall()
|
||||
|
||||
print("\n[覆盖率] v_style_id_coverage")
|
||||
print(f" {'表':<38}{'行数':>9}{'有 style_id':>12}{'比例':>9}")
|
||||
for table, total, with_id in rows:
|
||||
ratio = (with_id / total * 100) if total else 0.0
|
||||
print(f" {table:<38}{total:>9}{with_id:>12}{ratio:>8.1f}%")
|
||||
|
||||
if unmatched:
|
||||
print("\n[未建档] v_style_unmatched_product")
|
||||
for platform, count in unmatched:
|
||||
print(f" {platform}: {count} 个商品 ID 未写进 dim_style.spus")
|
||||
print(" -> 这些行的 style_id 只能保持 NULL,需要在建档页补 spus")
|
||||
else:
|
||||
print("\n[未建档] v_style_unmatched_product: 无,全部商品 ID 已建档")
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="回填采集表 style_id (= dim_style.id),完成款式主键统一"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tables",
|
||||
default="",
|
||||
help="逗号分隔,只回填指定表;留空表示全部",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="只统计不写库")
|
||||
parser.add_argument(
|
||||
"--skip-schema",
|
||||
action="store_true",
|
||||
help="跳过 init_schema(列已存在时可用,节省时间)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-product-table",
|
||||
action="store_true",
|
||||
help="跳过 product_daily_metrics(按商品 ID 反查,最慢的一步)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-ad-tables",
|
||||
action="store_true",
|
||||
help="跳过三张广告表的按商品 ID 二次回填",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not args.skip_schema and not args.dry_run:
|
||||
init_schema()
|
||||
|
||||
requested = {
|
||||
item.strip() for item in str(args.tables or "").split(",") if item.strip()
|
||||
}
|
||||
name_tables = [t for t in NAME_TABLES if not requested or t in requested]
|
||||
run_product = (
|
||||
not args.skip_product_table
|
||||
and (not requested or PRODUCT_TABLE in requested)
|
||||
)
|
||||
ad_tables = (
|
||||
[]
|
||||
if args.skip_ad_tables
|
||||
else [t for t in AD_PRODUCT_TABLES if not requested or t in requested]
|
||||
)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
with get_conn() as conn:
|
||||
try:
|
||||
for table in name_tables:
|
||||
result = _backfill_by_name(conn, table, dry_run=args.dry_run)
|
||||
results.append(result)
|
||||
print(
|
||||
f"[{'DRY' if args.dry_run else 'OK '}] {table:<38} "
|
||||
f"{result['with_style_id_before']}/{result['rows']} -> "
|
||||
f"{result.get('updated', 0)} 行可填"
|
||||
)
|
||||
if run_product:
|
||||
result = _backfill_product_table(conn, dry_run=args.dry_run)
|
||||
results.append(result)
|
||||
if args.dry_run:
|
||||
detail = (
|
||||
f"{result['planned_keys']} 个商品 ID 可解析, "
|
||||
f"{result['unmatched_product_ids']} 个未建档"
|
||||
)
|
||||
else:
|
||||
detail = f"{result.get('updated', 0)} 行已填"
|
||||
print(
|
||||
f"[{'DRY' if args.dry_run else 'OK '}] {PRODUCT_TABLE:<38} "
|
||||
f"{result['with_style_id_before']}/{result['rows']} -> {detail}"
|
||||
)
|
||||
if args.dry_run:
|
||||
conn.rollback()
|
||||
else:
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
if ad_tables:
|
||||
# 广告表的 style_name 大量为空,按款名回填无效,必须再按商品 ID 走一遍。
|
||||
# 单独开事务,避免和上面的款名回填混在一个长事务里。
|
||||
with get_conn() as conn:
|
||||
try:
|
||||
ad_results = _backfill_ad_tables_by_product(
|
||||
conn, ad_tables, dry_run=args.dry_run
|
||||
)
|
||||
results.extend(ad_results)
|
||||
for result in ad_results:
|
||||
if args.dry_run:
|
||||
detail = (
|
||||
f"{result['planned_keys']} 个商品 ID 可解析, "
|
||||
f"{result['unmatched_product_ids']} 个未建档"
|
||||
)
|
||||
else:
|
||||
detail = f"{result.get('updated', 0)} 行已填"
|
||||
print(
|
||||
f"[{'DRY' if args.dry_run else 'OK '}] {result['table']:<38} "
|
||||
f"{result['with_style_id_before']}/{result['rows']} -> {detail} (按商品ID)"
|
||||
)
|
||||
if args.dry_run:
|
||||
conn.rollback()
|
||||
else:
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
total_updated = sum(int(r.get("updated", 0)) for r in results)
|
||||
print(f"\n{'[DRY]' if args.dry_run else '[DONE]'} 合计更新 {total_updated} 行")
|
||||
|
||||
if not args.dry_run:
|
||||
with get_conn() as conn:
|
||||
_print_coverage(conn)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -63,8 +63,9 @@ LOGIN_URL_DEFAULT = "https://www.erp321.com/login.aspx"
|
||||
EVIDENCE_DIR_NAME = "erp_sales_theme_daily"
|
||||
|
||||
# ── 业务口径常量(与页面筛选一致) ─────────────────────────────────────────────
|
||||
# 商品品牌: 光影行星 / 光影 / gyxx (2026-10-12 截图勾选项)
|
||||
BRANDS: tuple[str, ...] = ("光影行星", "光影", "gyxx")
|
||||
# 商品品牌: 全部不勾 = 不限品牌(2026-10-19 用户确认)。SET_FILTERS_JS 对空列表
|
||||
# 会把品牌区全部取消勾选后回读校验。
|
||||
BRANDS: tuple[str, ...] = ()
|
||||
|
||||
# 销售渠道(店铺): 光影系 22 家渠道(2026-10-12 用户确认补回 JD光影行星旗舰店
|
||||
# 与 得物(直发)-光影行星)。名称必须与 filter.aspx 复选框标签逐字一致;
|
||||
|
||||
@@ -69,6 +69,14 @@ DEFAULT_BRAND = "光影行星"
|
||||
# 保留机制供临时排障使用,但不要把业务款式永久写成成功的 0/0。
|
||||
ERP_SLOW_SKIP_CODES: set[str] = set()
|
||||
|
||||
# 重试后仍失败的「编码级」条目容忍上限(2026-10-29)。
|
||||
# 背景:锚点款按系列聚合 ERP 编码后,会把个别只存在于聚水潭、三平台都没有商品编码
|
||||
# 的占位编码(例如「宙斯-10244」)一起采。这类编码本身查不到数据,某些店铺的
|
||||
# allsku 报表不渲染 iframe 而一直超时,导致整份商品经营日报当天完全不入库。
|
||||
# 这里的容忍只针对 failed_codes:页面级 / 款式级失败、或编码失败数超过上限,
|
||||
# 仍然 exit 1 阻断日报。失败编码照常写入 failed_<date>_erp.json 供补采。
|
||||
ERP_TOLERATED_FAILED_CODES = 3
|
||||
|
||||
# 兼容 PLATFORMS 旧名 (其它模块可能 import 它, 兜底给光影行星)
|
||||
PLATFORMS: dict[str, dict[str, Any]] = PLATFORMS_BY_BRAND[DEFAULT_BRAND]
|
||||
|
||||
@@ -245,7 +253,7 @@ def print_skip_summary(skip_report: dict[str, Any]) -> None:
|
||||
return
|
||||
print(
|
||||
f"[ERP] 来源={src}, 采集 {len(skip_report['collectable'])} 款, "
|
||||
f"跳过 {len(skip_report['skipped'])} 款 (缺 ERP 编码 / 平台商品编码 / 飞书表地址)"
|
||||
f"跳过 {len(skip_report['skipped'])} 款 (未标记采集 / 缺 ERP 编码 / 已由同系列归集 / 缺销量表)"
|
||||
)
|
||||
for s in skip_report["skipped"]:
|
||||
print(f" - {s['style']}: {s['reason']}")
|
||||
@@ -405,7 +413,9 @@ def _extract_by_idx(table_frame: Any, sales_idx: int, returns_idx: int) -> dict[
|
||||
|
||||
|
||||
ENSURE_CUSTOM_COLUMNS_JS = r"""
|
||||
() => {
|
||||
(options) => {
|
||||
const redrawPage = (options && options.redrawPage) || 1;
|
||||
const forceRedraw = Boolean(options && options.forceRedraw);
|
||||
const cols = window._Cols || [];
|
||||
let changed = 0;
|
||||
for (const col of cols) {
|
||||
@@ -415,31 +425,43 @@ ENSURE_CUSTOM_COLUMNS_JS = r"""
|
||||
changed += 1;
|
||||
}
|
||||
}
|
||||
if (changed > 0 && typeof window._RptRefresh === 'function') {
|
||||
try { window._RptRefresh(); } catch (e) {}
|
||||
// 只改 visible 不会补出数据行缺失的 td:聚水潭按旧列定义渲染数据行,
|
||||
// 行内仍是旧的单元格数,按新列位置取值就会读到 undefined(历史全 0 成因)。
|
||||
// 换页会强制按当前列定义重绘,所以顺带跳到目标页。
|
||||
let redrawn = false;
|
||||
if ((changed > 0 || forceRedraw) && typeof GotoPage === 'function') {
|
||||
try { GotoPage(redrawPage || 1); redrawn = true; } catch (e) {}
|
||||
}
|
||||
return changed;
|
||||
return { changed: changed, redrawn: redrawn };
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def ensure_custom_columns_visible(page: Any) -> int:
|
||||
"""backfill 模式: 把模板重置隐藏的自定义销量/退货量列恢复可见。
|
||||
def ensure_custom_columns_visible(page: Any, *, force_redraw: bool = False) -> dict[str, Any]:
|
||||
"""backfill 模式: 恢复自定义销量/退货量列可见,并触发重绘补齐行内单元格。
|
||||
|
||||
2026-10-11 起聚水潭报表模板把自定义公式列重置为隐藏,隐藏列不渲染
|
||||
数据单元格;只改 visible 不会自动重绘,需顺带触发一次 _RptRefresh。
|
||||
与 backfill_erp_all_shop_daily.py 同源(该脚本 10-13 实测有效)。
|
||||
数据单元格;只改 ``visible`` 不会重绘,``_RptRefresh()`` 在未渲染时直接
|
||||
抛错,因此改用 ``GotoPage`` 换页强制按当前列定义重绘。与
|
||||
``backfill_erp_all_shop_daily.py`` 同一套实现(该脚本 10-13 起实测有效)。
|
||||
"""
|
||||
|
||||
changed_total = 0
|
||||
summary = {"changed": 0, "redrawn": 0}
|
||||
for frame in page.frames:
|
||||
if "allsku.aspx" not in (frame.url or "").lower():
|
||||
continue
|
||||
try:
|
||||
changed_total += int(frame.evaluate(ENSURE_CUSTOM_COLUMNS_JS))
|
||||
result = frame.evaluate(
|
||||
ENSURE_CUSTOM_COLUMNS_JS,
|
||||
{"redrawPage": 1, "forceRedraw": bool(force_redraw)},
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
return changed_total
|
||||
if isinstance(result, dict):
|
||||
summary["changed"] += int(result.get("changed") or 0)
|
||||
if result.get("redrawn"):
|
||||
summary["redrawn"] += 1
|
||||
return summary
|
||||
|
||||
|
||||
def _extract_by_header(table_frame: Any, sales_header: str, returns_header: str) -> dict[str, Any]:
|
||||
@@ -1049,9 +1071,17 @@ def wait_for_metrics(page: Any, erp_code: str, timeout_seconds: int = 45,
|
||||
# previous_url 可能在 _reload_allsku_iframe 后被重置为 None
|
||||
|
||||
while True:
|
||||
# backfill 模式: 报表刷新会把自定义列的可见性重置回隐藏, 每轮恢复一次
|
||||
# backfill 模式: 报表刷新会把自定义列的可见性重置回隐藏, 每轮恢复一次;
|
||||
# 首轮额外强制换页重绘, 否则数据行仍按旧列数渲染, 取到的是空单元格(会被读成 0)
|
||||
if MODE == "backfill":
|
||||
ensure_custom_columns_visible(page)
|
||||
redraw = ensure_custom_columns_visible(page, force_redraw=poll_count == 0)
|
||||
if redraw["changed"] or redraw["redrawn"]:
|
||||
print(
|
||||
f"[ERP] {erp_code}: 自定义列重绘 changed={redraw['changed']} "
|
||||
f"redrawn={redraw['redrawn']}",
|
||||
flush=True,
|
||||
)
|
||||
page.wait_for_timeout(1200)
|
||||
# 先检测"操作频繁"弹窗 (在主页 DOM, 不在 allsku iframe)
|
||||
if filter_frame is not None and freq_dialog_handled < MAX_FREQ_DIALOG:
|
||||
if _detect_and_close_freq_dialog(page):
|
||||
@@ -1253,7 +1283,7 @@ def _run_one_code(page: Any, filter_frame: Any, platform: dict[str, Any], erp_co
|
||||
set_platform_and_code(filter_frame, platform["shops"], erp_code)
|
||||
if MODE == "backfill":
|
||||
# 自定义销量/退货量列被模板重置隐藏时先恢复可见, 否则数据行不渲染对应格
|
||||
ensure_custom_columns_visible(page)
|
||||
ensure_custom_columns_visible(page, force_redraw=True)
|
||||
prev_state = _get_allsku_frame_state(page)
|
||||
generate_report(filter_frame)
|
||||
return wait_for_metrics(
|
||||
@@ -1515,6 +1545,15 @@ def collect_in_page(page: Any, styles: list[dict[str, Any]], data_dir: Path,
|
||||
"reports": len(all_results),
|
||||
"valid_codes": valid_codes,
|
||||
"has_failures": failed.has_failures(),
|
||||
# 失败明细计数:供 main 判断「个别编码单平台超时」能否降级为警告,
|
||||
# 页面级/款式级失败仍然一律阻断(见 ERP_TOLERATED_FAILED_CODES)。
|
||||
"failed_codes": len(failed.failed_codes),
|
||||
"failed_styles": len(failed.failed_styles),
|
||||
"failed_pages": len(failed.failed_pages),
|
||||
"failed_code_details": [
|
||||
f"{item.get('style')} {item.get('code')}: {item.get('error')}"
|
||||
for item in failed.failed_codes
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -1652,8 +1691,25 @@ def main() -> int:
|
||||
return 1
|
||||
summary = state.get("collector_summary") or {}
|
||||
if summary.get("has_failures"):
|
||||
print("[FAIL] ERP collector retained failed codes after retry")
|
||||
return 1
|
||||
# 2026-10-29: 页/款级失败仍然阻断;只把「个别编码在单平台重试后仍超时」
|
||||
# 降级为警告。这类编码(如 10244 在抖音查不到数据导致 allsku iframe 不渲染)
|
||||
# 本身贡献 0 销量,款级汇总值已正确,不该让整份日报当天不入库。
|
||||
tolerated = (
|
||||
0 < int(summary.get("failed_codes") or 0) <= ERP_TOLERATED_FAILED_CODES
|
||||
and int(summary.get("failed_styles") or 0) == 0
|
||||
and int(summary.get("failed_pages") or 0) == 0
|
||||
)
|
||||
if tolerated:
|
||||
print(
|
||||
"[WARN] ERP 采集完成,"
|
||||
f"{summary.get('failed_codes')} 个编码单平台重试后仍失败,"
|
||||
"已记 0 并保留失败清单,按部分成功继续日报:"
|
||||
)
|
||||
for detail in summary.get("failed_code_details") or []:
|
||||
print(f" - {detail}", flush=True)
|
||||
else:
|
||||
print("[FAIL] ERP collector retained failed codes after retry")
|
||||
return 1
|
||||
if int(summary.get("reports") or 0) <= 0 or int(summary.get("valid_codes") or 0) <= 0:
|
||||
print("[FAIL] ERP collector produced no valid report output")
|
||||
return 1
|
||||
|
||||
@@ -1423,6 +1423,38 @@ def ensure_logged_in(page, account: str, password: str) -> None:
|
||||
raise JDAdCostCollectionError("京东登录态未能进入商家首页")
|
||||
|
||||
|
||||
_JZT_ONBOARDING_URL_MARKER = "/settled/"
|
||||
_JZT_ONBOARDING_TEXT_MARKERS = ("确认联系人手机号", "京准通欢迎您的加入")
|
||||
|
||||
|
||||
def _jzt_onboarding_present(page) -> bool:
|
||||
"""京准通会把未完成开通引导的会话重定向到 /settled 确认页。"""
|
||||
|
||||
if _JZT_ONBOARDING_URL_MARKER in str(getattr(page, "url", "")):
|
||||
return True
|
||||
body = _page_text(page)
|
||||
return any(marker in body for marker in _JZT_ONBOARDING_TEXT_MARKERS)
|
||||
|
||||
|
||||
def ensure_jzt_ready(page) -> None:
|
||||
"""Block early when 京准通 still needs the account owner's own confirmation.
|
||||
|
||||
``/settled/#/information/bonus`` (code=-3012) asks the logged-in merchant to
|
||||
confirm a contact mobile number and accept 《京准通平台相关协议》 before the
|
||||
custom-report routes render. The script must not accept that on the
|
||||
operator's behalf, so surface it as an actionable blocker instead of the
|
||||
generic "页面出现「下载报表」" timeout.
|
||||
"""
|
||||
|
||||
if not _jzt_onboarding_present(page):
|
||||
return
|
||||
raise JDAdCostCollectionError(
|
||||
"京东京准通需要人工完成「确认联系人手机号」开通引导"
|
||||
"(/settled/#/information/bonus,code=-3012):"
|
||||
"请在可见浏览器中由账号本人确认后重跑"
|
||||
)
|
||||
|
||||
|
||||
def _request_report(
|
||||
page, report_type: ReportType, target_date: date, advertiser_pin: str
|
||||
) -> None:
|
||||
@@ -1570,7 +1602,13 @@ def run(args: argparse.Namespace) -> list[Path]:
|
||||
pass
|
||||
ensure_logged_in(page, login_account, PASSWORD)
|
||||
safe_goto(page, DOWNLOAD_URL)
|
||||
_wait_for_text(page, "下载报表", timeout_seconds=60)
|
||||
# 京准通对未完成开通引导的会话会重定向到 /settled 确认页。先按正常
|
||||
# 超时等待,超时后再判定是否为引导页拦截,避免刚跳转时的残留文案误报。
|
||||
try:
|
||||
_wait_for_text(page, "下载报表", timeout_seconds=60)
|
||||
except JDAdCostCollectionError:
|
||||
ensure_jzt_ready(page)
|
||||
raise
|
||||
_clear_download_center(page)
|
||||
reports = {"non_full": "absent", "full": "absent"}
|
||||
safe_goto(page, OVERVIEW_URL)
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"note": "电商费用按父款式汇总:key 是平台报表里匹配到的子款式,value 是展示口径的父款式。只在对应平台的导入脚本里生效,不影响 dim_style 原始映射。",
|
||||
"jd": {
|
||||
"宙斯3": "宙斯",
|
||||
"宙斯3 Air": "宙斯",
|
||||
"宙斯Z Air": "宙斯"
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,21 @@ except ImportError: # Direct script execution from the module root.
|
||||
from runtime_paths import NORMALIZED_DATA_ROOT, STATE_ROOT, managed_data_path
|
||||
|
||||
|
||||
CACHE_SCHEMA_VERSION = 5
|
||||
CACHE_SCHEMA_VERSION = 9 # 9: 款式主键统一, 每条 style 增加 style_id / style_match_method
|
||||
|
||||
# 系列级飞书地址:约定只维护在锚点行,子款误填也会被整系列聚合去重后统一下发。
|
||||
# 顺序即写入目标表的优先级说明顺序,不含 bitable_sku_master(人工维护,不参与采集)。
|
||||
SERIES_TARGET_FIELDS = (
|
||||
"sales_bitable",
|
||||
"main_image_bitable",
|
||||
"creator_bitable",
|
||||
"self_creator_bitable",
|
||||
"weekly_note_analysis",
|
||||
"persona_bitable",
|
||||
"jd_persona_bitable",
|
||||
"dy_persona_bitable",
|
||||
"style_analysis_bitable",
|
||||
)
|
||||
|
||||
# 表格字段名(按实际表格设置,支持别名)
|
||||
FIELD_ALIASES = {
|
||||
@@ -111,6 +125,10 @@ class StyleConfigLoader:
|
||||
del base_token, table_id, view_id
|
||||
self.identity = identity
|
||||
self.records_provider = records_provider or runtime_workflow_configs
|
||||
# 注入 records_provider 的调用方(单测 / 离线脚本)用的是假数据,绝不能
|
||||
# 写进 STATE_ROOT 的生产快照:那份快照只在数据库读取失败时被当作真配置
|
||||
# 使用,用它采集会按假款式去取数。因此注入 provider 时整套缓存读写都关掉。
|
||||
self.cache_enabled = records_provider is None
|
||||
|
||||
project_root = Path(__file__).resolve().parent.parent
|
||||
self.cache_path = (
|
||||
@@ -229,6 +247,24 @@ class StyleConfigLoader:
|
||||
parsed = self._parse_bitable_url(url)
|
||||
return bool(parsed["base_token"] and parsed["table_id"])
|
||||
|
||||
def _has_creator_bitable(self, record: dict) -> bool:
|
||||
"""合作达人表 Base/Table 同时存在。"""
|
||||
url = self._parse_markdown_link(record.get("creator_bitable_url"))
|
||||
parsed = self._parse_bitable_url(url)
|
||||
return bool(parsed["base_token"] and parsed["table_id"])
|
||||
|
||||
def _has_self_creator_bitable(self, record: dict) -> bool:
|
||||
"""自营合作达人表 Base/Table 同时存在。"""
|
||||
url = self._parse_markdown_link(record.get("self_creator_bitable_url"))
|
||||
parsed = self._parse_bitable_url(url)
|
||||
return bool(parsed["base_token"] and parsed["table_id"])
|
||||
|
||||
def _has_weekly_note_analysis(self, record: dict) -> bool:
|
||||
"""每周/每月笔记分析表 Base/Table 同时存在。"""
|
||||
url = self._parse_markdown_link(record.get("weekly_note_analysis_url"))
|
||||
parsed = self._parse_bitable_url(url)
|
||||
return bool(parsed["base_token"] and parsed["table_id"])
|
||||
|
||||
def _has_daily_required_fields(self, record: dict) -> bool:
|
||||
"""每日采集四条件:erp_codes、style、item_ids、sales_bitable。"""
|
||||
style_name = str(record.get("style") or "").strip()
|
||||
@@ -259,6 +295,9 @@ class StyleConfigLoader:
|
||||
"erp_codes": row.get("erp_codes", []),
|
||||
"sales_bitable_url": row.get("sales_bitable_url", ""),
|
||||
"main_image_bitable_url": row.get("main_image_bitable_url", ""),
|
||||
"creator_bitable_url": row.get("creator_bitable_url", ""),
|
||||
"self_creator_bitable_url": row.get("self_creator_bitable_url", ""),
|
||||
"weekly_note_analysis_url": row.get("weekly_note_analysis_url", ""),
|
||||
"persona_bitable_url": row.get("tm_persona_bitable_url", ""),
|
||||
"dy_persona_bitable_url": row.get("dy_persona_bitable_url", ""),
|
||||
"jd_persona_bitable_url": row.get("jd_persona_bitable_url", ""),
|
||||
@@ -266,6 +305,16 @@ class StyleConfigLoader:
|
||||
"style_analysis_bitable_url", ""
|
||||
),
|
||||
"destinations": row.get("destinations", []),
|
||||
"series_name": row.get("series_name", ""),
|
||||
"is_series_anchor": bool(row.get("is_series_anchor", False)),
|
||||
# 款式主键统一 (2026-11-01): dim_style.id 正式下发, 写入脚本
|
||||
# 直接用它落库, style_name 只作显示冗余。
|
||||
"style_id": row.get("style_id"),
|
||||
"style_match_method": row.get("style_match_method", ""),
|
||||
# 采集开关随记录下发;缺字段(旧 cmt_styles 回退)按启用处理
|
||||
"collect_enabled": True
|
||||
if row.get("collect_enabled") is None
|
||||
else bool(row.get("collect_enabled")),
|
||||
}
|
||||
)
|
||||
return records
|
||||
@@ -302,6 +351,8 @@ class StyleConfigLoader:
|
||||
def _empty(style_name: str) -> dict:
|
||||
return {
|
||||
"name": style_name,
|
||||
"style_id": None,
|
||||
"style_match_method": "",
|
||||
"brand": "",
|
||||
"erp_codes": [],
|
||||
"tm": {"item_ids": []},
|
||||
@@ -310,11 +361,18 @@ class StyleConfigLoader:
|
||||
"dy": {"item_ids": []},
|
||||
"sales_bitable": {},
|
||||
"main_image_bitable": {},
|
||||
"creator_bitable": {},
|
||||
"self_creator_bitable": {},
|
||||
"weekly_note_analysis": {},
|
||||
"persona_bitable": {},
|
||||
"dy_persona_bitable": {},
|
||||
"jd_persona_bitable": {},
|
||||
"style_analysis_bitable": {},
|
||||
"destinations": [],
|
||||
"series_name": "",
|
||||
"is_series_anchor": False,
|
||||
"collect_enabled": True,
|
||||
"series": {},
|
||||
}
|
||||
|
||||
# ── 第一遍:款式级字段 ──────────────────────────────────────
|
||||
@@ -327,11 +385,30 @@ class StyleConfigLoader:
|
||||
|
||||
style = styles.setdefault(style_name, _empty(style_name))
|
||||
|
||||
# 款式主键: dim_style.id, 与 data-hub 的 22 张表同一套主键。
|
||||
# merge 已把同款展开成多条平台记录, 取值一致, 取第一个非空即可。
|
||||
if style.get("style_id") is None and record.get("style_id") is not None:
|
||||
style["style_id"] = int(record["style_id"])
|
||||
if not style.get("style_match_method") and record.get("style_match_method"):
|
||||
style["style_match_method"] = str(record["style_match_method"])
|
||||
|
||||
# 品牌(款式级别,取第一个非空)
|
||||
brand = str(record.get("brand") or "").strip()
|
||||
if brand and not style["brand"]:
|
||||
style["brand"] = brand
|
||||
|
||||
# 系列归属(产品生命进程建档维护):同系列可共享平台编码与销量表
|
||||
series_name = str(record.get("series_name") or "").strip()
|
||||
if series_name and not style["series_name"]:
|
||||
style["series_name"] = series_name
|
||||
if record.get("is_series_anchor"):
|
||||
style["is_series_anchor"] = True
|
||||
|
||||
# 采集开关(产品生命进程在 dim_style.collect_enabled 上按系列维护):
|
||||
# merge 已把同一款展开成多条平台记录且取值一致;任一条显式 False 即关闭。
|
||||
if record.get("collect_enabled") is False:
|
||||
style["collect_enabled"] = False
|
||||
|
||||
destinations = record.get("destinations")
|
||||
if isinstance(destinations, list) and destinations and not style["destinations"]:
|
||||
style["destinations"] = [
|
||||
@@ -352,41 +429,38 @@ class StyleConfigLoader:
|
||||
f"{style['erp_codes']} vs {erp_codes}; 保留第一条天猫编码"
|
||||
)
|
||||
|
||||
# 销量飞书表地址(款式级别,只要 URL 完整就收)
|
||||
# 飞书地址:2026-10-29 起统一由 dim_style 系列锚点持有,并在
|
||||
# workflow_config 里下发到该款的每条平台记录,所以任意分组都能
|
||||
# 读到同样的值,不再像以前那样限定只能从天猫分组取。
|
||||
if self._has_sales_bitable(record) and not style["sales_bitable"]:
|
||||
sales_url = self._parse_markdown_link(record.get("sales_bitable_url"))
|
||||
if sales_url:
|
||||
style["sales_bitable"] = self._parse_bitable_url(sales_url)
|
||||
|
||||
# 主图表地址:只读天猫分组记录,只要求主图 URL 非空
|
||||
if platform == "tm" and self._has_main_image_bitable(record):
|
||||
if self._has_main_image_bitable(record) and not style["main_image_bitable"]:
|
||||
main_url = self._parse_markdown_link(record.get("main_image_bitable_url"))
|
||||
if main_url and not style["main_image_bitable"]:
|
||||
if main_url:
|
||||
style["main_image_bitable"] = self._parse_bitable_url(main_url)
|
||||
style["main_image_bitable"]["platform"] = "天猫"
|
||||
|
||||
# 人群画像(天猫)表地址:只读天猫分组记录
|
||||
if platform == "tm" and self._has_persona_bitable(record):
|
||||
persona_url = self._parse_markdown_link(record.get("persona_bitable_url"))
|
||||
if persona_url and not style["persona_bitable"]:
|
||||
style["persona_bitable"] = self._parse_bitable_url(persona_url)
|
||||
|
||||
# 人群画像(抖音)表地址:款式级别,任意平台记录有值就收
|
||||
if self._has_dy_persona_bitable(record):
|
||||
dy_persona_url = self._parse_markdown_link(record.get("dy_persona_bitable_url"))
|
||||
if dy_persona_url and not style["dy_persona_bitable"]:
|
||||
style["dy_persona_bitable"] = self._parse_bitable_url(dy_persona_url)
|
||||
|
||||
# 人群画像(京东)表地址:款式级别,任意平台记录有值就收
|
||||
if self._has_jd_persona_bitable(record):
|
||||
jd_persona_url = self._parse_markdown_link(record.get("jd_persona_bitable_url"))
|
||||
if jd_persona_url and not style["jd_persona_bitable"]:
|
||||
style["jd_persona_bitable"] = self._parse_bitable_url(jd_persona_url)
|
||||
|
||||
if self._has_style_analysis_bitable(record) and not style["style_analysis_bitable"]:
|
||||
analysis_url = self._parse_markdown_link(record.get("style_analysis_bitable_url"))
|
||||
if analysis_url:
|
||||
style["style_analysis_bitable"] = self._parse_bitable_url(analysis_url)
|
||||
for key, record_key, checker in (
|
||||
("creator_bitable", "creator_bitable_url", self._has_creator_bitable),
|
||||
(
|
||||
"self_creator_bitable",
|
||||
"self_creator_bitable_url",
|
||||
self._has_self_creator_bitable,
|
||||
),
|
||||
("weekly_note_analysis", "weekly_note_analysis_url", self._has_weekly_note_analysis),
|
||||
("persona_bitable", "persona_bitable_url", self._has_persona_bitable),
|
||||
("dy_persona_bitable", "dy_persona_bitable_url", self._has_dy_persona_bitable),
|
||||
("jd_persona_bitable", "jd_persona_bitable_url", self._has_jd_persona_bitable),
|
||||
("style_analysis_bitable", "style_analysis_bitable_url", self._has_style_analysis_bitable),
|
||||
):
|
||||
if style[key] or not checker(record):
|
||||
continue
|
||||
url = self._parse_markdown_link(record.get(record_key))
|
||||
if url:
|
||||
style[key] = self._parse_bitable_url(url)
|
||||
|
||||
# 非天猫分组的 ERP 编码不会参与聚合;只在冲突或天猫缺失时提示,
|
||||
# 避免重复填写相同编码造成无意义日志。
|
||||
@@ -427,6 +501,9 @@ class StyleConfigLoader:
|
||||
elif platform == "jd_self" and ids:
|
||||
style["jd_self"]["skus"] = ids
|
||||
|
||||
# ── 第三遍: 系列内共享(平台编码 / 销量表只要求主款填写)──────────
|
||||
self._apply_series_sharing(styles)
|
||||
|
||||
return {
|
||||
"styles": styles,
|
||||
"source": "database",
|
||||
@@ -434,6 +511,153 @@ class StyleConfigLoader:
|
||||
"schema_version": CACHE_SCHEMA_VERSION,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _apply_series_sharing(styles: dict[str, dict]) -> None:
|
||||
"""把整条系列的平台商品编码、ERP 编码与飞书地址聚合去重到锚点。
|
||||
|
||||
录入约定是「锚点建档、子款留空」,但子款误填无法从表结构上禁止,所以
|
||||
这里不依赖人工守规矩,统一做聚合兜底:
|
||||
|
||||
- 平台商品编码:整系列并集去重后挂锚点,由锚点一个采集单元去抓;
|
||||
子款保留自己的原始编码(series.platform_ids 里可见)供诊断,采集口
|
||||
按归属判定让出,避免同一商品 ID 被采两次、访客与加购重复入账。
|
||||
- ERP 编码:整系列并集去重后挂锚点(series.erp_owner),子款保留自己
|
||||
的原始编码(own_erp_codes)供按款查询。**子款自身 collect_enabled=false
|
||||
时不并入**:锚点是系列总开关,子款开关只决定自己的编码是否参与聚合。
|
||||
- 飞书地址:逐字段聚合并去重,锚点优先、其余按款名,取到的那套表下发
|
||||
全系列。一个表位只能落一张表,出现多个不同表时告警并保留优先级最高
|
||||
的一条。
|
||||
|
||||
聚合的目标是「子款误填也不丢、不重复入账、同系列只有一套表」,而不是
|
||||
让子款变成独立采集单元。
|
||||
"""
|
||||
|
||||
by_series: dict[str, list[str]] = {}
|
||||
for name, style in styles.items():
|
||||
series = str(style.get("series_name") or "").strip()
|
||||
if series:
|
||||
by_series.setdefault(series, []).append(name)
|
||||
|
||||
for series, members in by_series.items():
|
||||
# 锚点优先,其余按名称稳定排序,保证每次运行归属一致
|
||||
members.sort(key=lambda n: (not styles[n].get("is_series_anchor"), n))
|
||||
anchor = members[0]
|
||||
erp_owner = anchor
|
||||
multi = len(members) > 1
|
||||
platform_owner: dict[str, str] = {}
|
||||
platform_ids: dict[str, dict[str, list[str]]] = {}
|
||||
merged_platform: dict[str, list[str]] = {}
|
||||
for platform, ids_key in PRODUCT_ID_FIELDS.items():
|
||||
holders: dict[str, list[str]] = {}
|
||||
merged: list[str] = []
|
||||
for member in members:
|
||||
ids = [
|
||||
str(item).strip()
|
||||
for item in styles[member].get(platform, {}).get(ids_key) or ()
|
||||
if str(item).strip()
|
||||
]
|
||||
if not ids:
|
||||
continue
|
||||
holders[member] = ids
|
||||
platform_owner.setdefault(platform, member)
|
||||
for item in ids:
|
||||
if item not in merged:
|
||||
merged.append(item)
|
||||
if not holders:
|
||||
continue
|
||||
merged_platform[platform] = merged
|
||||
if multi:
|
||||
# 子款写的编码也并入锚点:整系列先去重聚合,锚点作为唯一
|
||||
# 采集单元持有全部商品 ID。holders 必须重建以保证锚点排在
|
||||
# 第一位——_platform_collection_owner 依赖插入顺序判断
|
||||
# 「谁的编码覆盖了谁」。
|
||||
ordered_holders: dict[str, list[str]] = {anchor: list(merged)}
|
||||
for member in members:
|
||||
if member != anchor and member in holders:
|
||||
ordered_holders[member] = holders[member]
|
||||
holders = ordered_holders
|
||||
platform_ids[platform] = holders
|
||||
merged_erp: list[str] = []
|
||||
erp_excluded: list[str] = []
|
||||
for member in members:
|
||||
# 锚点是系列总开关(见下面 series_collect);子款自身的 collect_enabled
|
||||
# 决定它的 ERP 编码是否并入系列。子款关掉采集时,它的编码既不由自己
|
||||
# 去聚水潭取数,也不允许锚点代采——直接排除在聚合之外。
|
||||
if member != anchor and not bool(
|
||||
styles[member].get("collect_enabled", True)
|
||||
):
|
||||
erp_excluded.append(member)
|
||||
continue
|
||||
for code in styles[member].get("erp_codes") or ():
|
||||
if code not in merged_erp:
|
||||
merged_erp.append(code)
|
||||
if multi and erp_excluded and bool(styles[anchor].get("collect_enabled", True)):
|
||||
print(
|
||||
f"[INFO] 系列「{series}」子款 {', '.join(erp_excluded)} "
|
||||
f"已关闭采集,ERP 编码不纳入「{erp_owner}」聚合"
|
||||
)
|
||||
# 系列级采集开关:以锚点为准(members[0] 已按「锚点优先、其次款名」
|
||||
# 排序,无锚点时即款名第一个)。产品生命进程只改锚点那一行即可控制
|
||||
# 整条系列,其余子款跟随。
|
||||
series_collect = bool(styles[anchor].get("collect_enabled", True))
|
||||
# 系列级飞书地址:整系列逐字段聚合去重,锚点优先、其余按款名。子款
|
||||
# 误填的地址同样参与聚合而不会被丢弃;一套表下发全系列,写表口径
|
||||
# 统一。同字段出现多个不同表时保留优先级最高的一条并告警。
|
||||
series_targets: dict[str, dict] = {}
|
||||
for field in SERIES_TARGET_FIELDS:
|
||||
seen: set[tuple] = set()
|
||||
candidates: list[dict] = []
|
||||
for member in members:
|
||||
target = styles[member].get(field) or {}
|
||||
if not target.get("base_token"):
|
||||
continue
|
||||
key = (
|
||||
target.get("base_token"),
|
||||
target.get("table_id"),
|
||||
target.get("view_id"),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
candidates.append(dict(target))
|
||||
if not candidates:
|
||||
continue
|
||||
if len(candidates) > 1:
|
||||
print(
|
||||
f"[WARN] 系列「{series}」{field} 有 {len(candidates)} 个不同表,"
|
||||
f"按「{anchor}」下发并忽略其余"
|
||||
)
|
||||
series_targets[field] = candidates[0]
|
||||
|
||||
for member in members:
|
||||
style = styles[member]
|
||||
style["series"] = {
|
||||
"name": series,
|
||||
"is_anchor": bool(style.get("is_series_anchor")),
|
||||
"collect_enabled": series_collect,
|
||||
"platform_owner": dict(platform_owner),
|
||||
}
|
||||
if multi:
|
||||
# 单款即可自成系列,无需归属标注;多款才需要指明 ERP 由谁承接
|
||||
style["series"]["erp_owner"] = erp_owner
|
||||
style["series"]["own_erp_codes"] = list(style.get("erp_codes") or ())
|
||||
# 自身 collect_enabled=false 而被排除出 ERP 聚合的子款,供诊断
|
||||
style["series"]["erp_excluded"] = list(erp_excluded)
|
||||
style["series"]["platform_ids"] = {
|
||||
platform: {holder: list(ids) for holder, ids in holders.items()}
|
||||
for platform, holders in platform_ids.items()
|
||||
}
|
||||
for field, target in series_targets.items():
|
||||
style[field] = dict(target)
|
||||
if multi:
|
||||
# 锚点持有整条系列的平台商品编码与 ERP 编码
|
||||
for platform, ids_key in PRODUCT_ID_FIELDS.items():
|
||||
merged = merged_platform.get(platform)
|
||||
if merged:
|
||||
styles[anchor][platform][ids_key] = list(merged)
|
||||
if merged_erp:
|
||||
styles[erp_owner]["erp_codes"] = merged_erp
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 缓存与回退
|
||||
# ------------------------------------------------------------------
|
||||
@@ -478,13 +702,14 @@ class StyleConfigLoader:
|
||||
try:
|
||||
records = self._load_records_from_database()
|
||||
payload = self._aggregate(records)
|
||||
self._write_cache(payload)
|
||||
if self.cache_enabled:
|
||||
self._write_cache(payload)
|
||||
if sync_fallback:
|
||||
self._write_style_map_fallback(payload)
|
||||
return payload
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 云端动态配置读取失败: {exc},尝试数据库快照缓存")
|
||||
cached = self._read_cache()
|
||||
cached = self._read_cache() if self.cache_enabled else None
|
||||
if cached and cached.get("source") == "database":
|
||||
return cached
|
||||
raise StyleConfigError("云端动态配置不可用且没有数据库快照缓存") from exc
|
||||
@@ -656,10 +881,11 @@ class StyleConfigLoader:
|
||||
]
|
||||
|
||||
def get_erp_styles_input(self) -> list[dict]:
|
||||
"""ERP 编码清单:只要求款式编码,平台编码/销量表由系列主款承担。"""
|
||||
return [
|
||||
{"style_name": name, "erp_style_codes": cfg["erp_codes"]}
|
||||
for name, cfg in self.load()["styles"].items()
|
||||
if self._has_complete_identity(cfg)
|
||||
if cfg.get("erp_codes")
|
||||
]
|
||||
|
||||
def get_bitable_style_map(self) -> dict:
|
||||
@@ -712,47 +938,89 @@ class StyleConfigLoader:
|
||||
# ------------------------------------------------------------------
|
||||
# 启动期「待采 / 跳过」门 — 显式返回带 reason 的跳过清单
|
||||
# ------------------------------------------------------------------
|
||||
def _cfg_has_sales_bitable(self, cfg: dict) -> bool:
|
||||
return bool(cfg.get("sales_bitable", {}).get("base_token")) and bool(
|
||||
cfg.get("sales_bitable", {}).get("table_id")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _has_any_product_id(cfg: dict) -> bool:
|
||||
return any(
|
||||
bool(cfg.get(platform, {}).get(ids_key))
|
||||
for platform, ids_key in PRODUCT_ID_FIELDS.items()
|
||||
)
|
||||
|
||||
def _has_complete_identity(self, cfg: dict, platform: str | None = None) -> bool:
|
||||
"""商品采集统一门:ERP 款式编码 + 至少一个平台商品编码。"""
|
||||
return has_complete_style_identity(cfg, platform)
|
||||
|
||||
def _identity_missing(self, cfg: dict) -> list[str]:
|
||||
missing: list[str] = []
|
||||
if not cfg.get("erp_codes"):
|
||||
missing.append("ERP款式编码")
|
||||
if not self._has_any_product_id(cfg):
|
||||
missing.append("平台商品编码")
|
||||
return missing
|
||||
def _erp_owner_for_collection(self, name: str, cfg: dict, styles: dict) -> str:
|
||||
"""本款是否需要把 ERP 采集让给同系列锚点。
|
||||
|
||||
返回空串表示本款自己采;返回款名表示编码已由该款归集,本款不重复采。
|
||||
锚点自身被关闭采集时不再承接(返回空串)——但系列级总开关此时已让整条
|
||||
系列停采,这里只是避免把编码归属标注到一个根本不采的款上。
|
||||
"""
|
||||
|
||||
series = cfg.get("series") or {}
|
||||
owner = str(series.get("erp_owner") or "").strip()
|
||||
if not owner or owner == name:
|
||||
return ""
|
||||
if not self._is_collect_enabled(styles.get(owner) or {}):
|
||||
return ""
|
||||
return owner
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 采集开关 (dim_style.collect_enabled)
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def _is_collect_enabled(cfg: dict) -> bool:
|
||||
"""本款是否整体纳入采集(系列级总开关)。
|
||||
|
||||
``dim_style.collect_enabled`` 是两层语义:
|
||||
|
||||
- **系列级总开关**:由锚点那一行决定,``_apply_series_sharing`` 已把锚点的
|
||||
值摊到 ``series.collect_enabled``,所以同系列任意一款得到相同判定。
|
||||
锚点 false → 整条系列一款都不采。产品侧只需改锚点那一行。
|
||||
- **子款级 ERP 参与权**:锚点为 true 时,子款自身的 ``collect_enabled``
|
||||
只决定「它的 ERP 款式编码是否并入系列聚合」(见 ``_apply_series_sharing``
|
||||
的 ``erp_excluded``);子款 false 不参与聚合,但不会让其他子款停采。
|
||||
|
||||
没有 series_name 的款直接用自身取值;字段缺失时默认采集。
|
||||
"""
|
||||
series = cfg.get("series") or {}
|
||||
if "collect_enabled" in series:
|
||||
return bool(series["collect_enabled"])
|
||||
return bool(cfg.get("collect_enabled", True))
|
||||
|
||||
def get_daily_styles(self) -> dict:
|
||||
"""每日商品采集门:ERP 编码、销量目标和平台商品编码必须完整。
|
||||
"""每日商品采集门:ERP 编码与平台商品编码必须完整。
|
||||
|
||||
销量飞书表地址只决定结果能否写表,不再作为采集前置条件——缺表地址
|
||||
的款式照常采集并落库,写表环节自己按地址缺失跳过。
|
||||
|
||||
返回 {tm/jd/dy: {collectable: [...], skipped: [{style, reason}]}}。
|
||||
incomplete style 永远不会进入 collectable,并记录具体缺失字段。
|
||||
采集开关取 ``dim_style.collect_enabled``:锚点是系列总开关(false 则整系列
|
||||
不采),子款自身的开关只决定它的 ERP 编码是否参与系列聚合;缺字段默认采集。
|
||||
|
||||
ERP 与平台编码按两个维度分开判定:
|
||||
- ERP:整条系列聚合到锚点(series.erp_owner)由一个单元去聚水潭取数;
|
||||
子款即使让出 ERP,只要自己持有某平台的商品 ID,仍会继续采该平台。
|
||||
- 平台:整系列只要有一款填了就算齐,谁填谁采;本款没填时指向承担款,
|
||||
本款编码被更高优先级的款式完整覆盖时让出,避免重复入账。
|
||||
"""
|
||||
payload = self.load()
|
||||
styles = payload["styles"]
|
||||
result = {p: {"collectable": [], "skipped": []} for p in ("tm", "jd", "dy")}
|
||||
for name, cfg in payload["styles"].items():
|
||||
identity_missing = self._identity_missing(cfg)
|
||||
for name, cfg in styles.items():
|
||||
if not self._is_collect_enabled(cfg):
|
||||
for plat in ("tm", "jd", "dy"):
|
||||
result[plat]["skipped"].append(
|
||||
{"style": name, "reason": "已标记不采集"}
|
||||
)
|
||||
continue
|
||||
# 自有编码,或同系列锚点已在采(本款不必重复去聚水潭)
|
||||
has_erp = bool(cfg.get("erp_codes")) or bool(
|
||||
self._erp_owner_for_collection(name, cfg, styles)
|
||||
)
|
||||
for plat in ("tm", "jd", "dy"):
|
||||
ids_key = "spus" if plat == "jd" else "item_ids"
|
||||
missing = list(identity_missing)
|
||||
if not self._cfg_has_sales_bitable(cfg):
|
||||
missing.append("销量目标地址")
|
||||
if not cfg.get(plat, {}).get(ids_key):
|
||||
missing.append(f"{plat}商品编码")
|
||||
owner = self._platform_collection_owner(name, cfg, plat, styles)
|
||||
missing: list[str] = []
|
||||
if not has_erp:
|
||||
missing.append("ERP款式编码")
|
||||
if owner != name:
|
||||
if owner:
|
||||
missing.append(f"{plat}商品编码(同系列「{owner}」已在采)")
|
||||
else:
|
||||
missing.append(f"{plat}商品编码")
|
||||
if missing:
|
||||
result[plat]["skipped"].append(
|
||||
{"style": name, "reason": "缺 " + "/".join(dict.fromkeys(missing))}
|
||||
@@ -761,27 +1029,82 @@ class StyleConfigLoader:
|
||||
result[plat]["collectable"].append(name)
|
||||
return result
|
||||
|
||||
def _platform_collection_owner(
|
||||
self, name: str, cfg: dict, platform: str, styles: dict
|
||||
) -> str:
|
||||
"""本款是否承担该平台的流量采集;空串表示整条系列都没填编码。
|
||||
|
||||
平台商品编码整系列只要有一款填了就算齐:谁填谁采。本款的编码若被
|
||||
同系列优先级更高的款式完整覆盖(锚点常见做法是把全系列商品 ID 都
|
||||
挂在锚点上),本款让出,避免同一个商品 ID 被采两次、访客与加购
|
||||
重复入账。本款的编码与别人的编码互不包含时各自采各自的商品。
|
||||
"""
|
||||
|
||||
ids_key = PRODUCT_ID_FIELDS[platform]
|
||||
own_ids = {
|
||||
str(item).strip()
|
||||
for item in cfg.get(platform, {}).get(ids_key) or ()
|
||||
if str(item).strip()
|
||||
}
|
||||
series = cfg.get("series") or {}
|
||||
holders: dict[str, list[str]] = (
|
||||
(series.get("platform_ids") or {}).get(platform) or {}
|
||||
)
|
||||
if own_ids:
|
||||
# holders 的插入顺序即系列内的优先级(锚点优先,其次按款名)
|
||||
for member in holders:
|
||||
if member == name:
|
||||
return name
|
||||
if not self._is_collect_enabled(styles.get(member) or {}):
|
||||
continue
|
||||
covered = {
|
||||
str(item).strip() for item in holders.get(member) or () if str(item).strip()
|
||||
}
|
||||
if own_ids <= covered:
|
||||
return member
|
||||
return name
|
||||
# 本款没填:由系列内已填且启用的款式承担(锚点优先)
|
||||
for member in holders:
|
||||
if not self._is_collect_enabled(styles.get(member) or {}):
|
||||
continue
|
||||
return member
|
||||
return ""
|
||||
|
||||
def get_erp_daily_styles(self) -> dict:
|
||||
"""ERP daily 门:ERP 编码、至少一个平台商品编码和销量目标都要有。
|
||||
"""ERP daily 门:只要求 ERP 款式编码。
|
||||
|
||||
销量飞书表地址只决定结果能否写表,不再作为采集前置条件——缺表地址
|
||||
的款式照常去聚水潭取数并落库,写表环节自己按地址缺失跳过。
|
||||
|
||||
返回 {collectable, skipped, brand_map: {style: brand}}。
|
||||
ERP 一次跑覆盖 jd/dy/tm 三个平台,平台商品编码只用于确认款式已完成商品身份配置。
|
||||
ERP 销量按款式编码逐款查;同系列多款时编码已由锚点聚合并集
|
||||
(series.erp_owner),子款让出采集,避免同一编码被采两次。
|
||||
brand_map 给 collect_erp_yesterday_metrics.py 按品牌选店铺
|
||||
(光影行星旗下店铺 vs ozko旗舰店)。
|
||||
采集开关取 dim_style.collect_enabled:锚点为 false 时整系列不采;锚点为
|
||||
true 时,自身 collect_enabled=false 的子款其 ERP 编码不并入锚点聚合。
|
||||
缺字段默认采集。
|
||||
"""
|
||||
payload = self.load()
|
||||
styles = payload["styles"]
|
||||
collectable: list[str] = []
|
||||
skipped: list[dict] = []
|
||||
brand_map: dict[str, str] = {}
|
||||
for name, cfg in payload["styles"].items():
|
||||
missing = self._identity_missing(cfg)
|
||||
if not self._cfg_has_sales_bitable(cfg):
|
||||
missing.append("销量目标地址")
|
||||
if not missing:
|
||||
collectable.append(name)
|
||||
brand_map[name] = cfg.get("brand") or "光影行星"
|
||||
else:
|
||||
skipped.append({"style": name, "reason": "缺 " + "/".join(dict.fromkeys(missing))})
|
||||
for name, cfg in styles.items():
|
||||
if not self._is_collect_enabled(cfg):
|
||||
skipped.append({"style": name, "reason": "已标记不采集"})
|
||||
continue
|
||||
erp_owner = self._erp_owner_for_collection(name, cfg, styles)
|
||||
if erp_owner:
|
||||
skipped.append(
|
||||
{"style": name, "reason": f"ERP 已由同系列「{erp_owner}」归集采集"}
|
||||
)
|
||||
continue
|
||||
if not cfg.get("erp_codes"):
|
||||
skipped.append({"style": name, "reason": "缺 ERP款式编码"})
|
||||
continue
|
||||
collectable.append(name)
|
||||
brand_map[name] = cfg.get("brand") or "光影行星"
|
||||
return {"collectable": collectable, "skipped": skipped, "brand_map": brand_map}
|
||||
|
||||
def get_jd_self_styles(self) -> list[str]:
|
||||
@@ -793,14 +1116,18 @@ class StyleConfigLoader:
|
||||
]
|
||||
|
||||
def get_main_image_styles(self) -> list[str]:
|
||||
"""天猫主图门:完整身份、天猫商品编码和主图目标地址同时存在。"""
|
||||
"""天猫主图采集目标:身份齐全(ERP 款式编码 + 天猫商品编码)即可。
|
||||
|
||||
主图飞书表地址只决定结果能否写表,不再作为采集前置条件——缺地址的
|
||||
款式照常抓图并落库,写表环节按地址缺失跳过(见 get_bitable_main_image_map),
|
||||
与商品经营日报「缺销量表不挡采集」同一口径。
|
||||
"""
|
||||
payload = self.load()
|
||||
result: list[str] = []
|
||||
for name, cfg in payload["styles"].items():
|
||||
main = cfg.get("main_image_bitable") or {}
|
||||
if self._has_complete_identity(cfg, "tm") and main.get("base_token") and main.get("table_id"):
|
||||
result.append(name)
|
||||
return result
|
||||
return [
|
||||
name
|
||||
for name, cfg in payload["styles"].items()
|
||||
if self._has_complete_identity(cfg, "tm")
|
||||
]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
@@ -27,8 +27,10 @@
|
||||
写: check_nine_day_decline.py (去重)
|
||||
读: check_nine_day_decline.py._already_notified
|
||||
dim_style
|
||||
写: db/sync_dim_style.py (人工触发,投影产品生命进程身份)
|
||||
读: bitable 同步 / 业务查询
|
||||
写: data-hub 建档页 (产品生命进程,采集端身份的唯一权威来源)
|
||||
读: 采集端运行时取款 / bitable 同步 / 业务查询
|
||||
注: 原 db/sync_dim_style.py 已废弃并删除(改造后它读的就是 dim_style,
|
||||
投影等于自己写自己)
|
||||
fact_platform_xlsx
|
||||
写: 暂无 caller (db/__init__.py.upsert_platform_xlsx_rows 已写好,import 脚本待补)
|
||||
读: 平台 XLSX 明细查询
|
||||
@@ -36,10 +38,12 @@
|
||||
写: import_product_daily.py (扫 data/<plat>单品数据/_downloads/* 入库)
|
||||
读: 三平台单品日维度查询 / 跨平台对账
|
||||
product_daily_metrics.raw_data.tm_ad_metrics
|
||||
写: import_tmall_product_ads.py (万相台商品投放 CSV)
|
||||
读: data-hub 产品生命进程广告成交金额 / 投产比 / 电商费比;节点内保存款式匹配结果
|
||||
已废弃(2026-11-01):天猫广告指标改为写 tm_product_ad_daily_metrics;
|
||||
历史节点由 scripts/migrate_tm_ad_metrics_to_table.py 迁移后清除
|
||||
tm_product_ad_daily_metrics
|
||||
兼容旧版独立广告表;新流程不再写入该表
|
||||
写: import_tmall_product_ads.py (万相台商品投放 CSV)
|
||||
读: data-hub 产品生命进程广告花费 / 广告归因成交 / 投产比 / 电商费比
|
||||
注: 标量列供直接聚合,metrics 保留报表全部指标;款式按 tm_spus 反查
|
||||
jd_product_ad_daily_metrics
|
||||
写: import_jd_ad_costs.py (京东营销概况全站 SPU 与非全站 SKU)
|
||||
读: 京东电商费用 / 款式广告成本分析
|
||||
@@ -47,8 +51,9 @@
|
||||
写: import_douyin_qianchuan_ads.py (巨量千川商品推广日报)
|
||||
读: 抖音电商费用 / 款式广告成本分析;款式按 ID 优先、名称兜底反查
|
||||
product_reviews
|
||||
写: import_product_reviews.py (扫 data/{jd,dy}单品数据/评价*.json 入库)
|
||||
读: 评价分析 / 差评归因 / 标签挖掘
|
||||
已下线 (2026-11-01): 差评统一由 shop_intelligence 三张差评表采集,
|
||||
消费方改读视图 v_negative_reviews; 存量表归档为
|
||||
product_reviews_archived_20261101, 不再写入。
|
||||
persona_screenshots
|
||||
写: taobao_dmp_item_crowd_insight_screenshots.py (v2 改造后只入库, 不推飞书; insert_persona_to_bitable_v2.py 已删)
|
||||
读: 人物画像审计 / 飞书附件溯源
|
||||
@@ -82,6 +87,7 @@ import os
|
||||
import time
|
||||
import unicodedata
|
||||
from contextlib import contextmanager
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Mapping, Sequence
|
||||
|
||||
@@ -214,21 +220,36 @@ def ensure_tmall_product_ad_daily_table(conn) -> None:
|
||||
stat_date DATE NOT NULL,
|
||||
product_id TEXT NOT NULL,
|
||||
product_name TEXT,
|
||||
style_name TEXT,
|
||||
style_match_method TEXT NOT NULL DEFAULT 'unmatched',
|
||||
spend NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
direct_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
indirect_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
attributed_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
reported_roas NUMERIC(18, 6),
|
||||
calculated_roas NUMERIC(18, 6),
|
||||
metrics JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
duplicate_row_count INTEGER NOT NULL DEFAULT 1,
|
||||
source_file TEXT,
|
||||
source_row_number INTEGER,
|
||||
imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (stat_date, product_id)
|
||||
);
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS style_name TEXT;
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS style_match_method TEXT
|
||||
NOT NULL DEFAULT 'unmatched';
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS metrics JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS source_row_number INTEGER;
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_date_idx
|
||||
ON tm_product_ad_daily_metrics (stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_product_date_idx
|
||||
ON tm_product_ad_daily_metrics (product_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_style_date_idx
|
||||
ON tm_product_ad_daily_metrics (style_name, stat_date DESC);
|
||||
COMMENT ON TABLE tm_product_ad_daily_metrics IS
|
||||
'天猫商品级投放日报,每日每商品一行;广告归因成交额仅用于计算投产比。';
|
||||
"""
|
||||
@@ -330,6 +351,103 @@ def ensure_douyin_product_ad_daily_table(conn) -> None:
|
||||
cur.execute(sql)
|
||||
|
||||
|
||||
# ── 款式主键 (style_id = dim_style.id) 解析 ──────────────────────────────────
|
||||
#
|
||||
# 采集端 2026-11-01 起统一用 style_id 关联款式, style_name 降级为显示冗余。
|
||||
# 好处: 与 data-hub 的 22 张表同一套主键; 改款名不再让历史行变孤儿。
|
||||
#
|
||||
# 解析优先复用调用方传入的 style_id (loader 已下发), 缺失时在 db 层批量兜底,
|
||||
# 避免为 13 张表的每个写入脚本单独加查询。
|
||||
|
||||
STYLE_MATCH_BY_NAME = "dim_style"
|
||||
STYLE_MATCH_BY_PRODUCT = "product_id"
|
||||
STYLE_MATCH_BY_ANCHOR = "anchor"
|
||||
STYLE_MATCH_UNMATCHED = "unmatched"
|
||||
|
||||
_PLATFORM_SPU_COLUMN = {"tm": "tm_spus", "jd": "jd_spus", "dy": "dy_spus"}
|
||||
|
||||
|
||||
def resolve_style_ids(conn, style_names: Sequence[Any]) -> dict[str, int]:
|
||||
"""按款名批量解析 dim_style.id。查不到的款名不出现在返回值里。"""
|
||||
|
||||
names = sorted(
|
||||
{" ".join(str(name or "").split()) for name in style_names if str(name or "").strip()}
|
||||
)
|
||||
if not names:
|
||||
return {}
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT style_name, id FROM dim_style WHERE style_name = ANY(%s)",
|
||||
(names,),
|
||||
)
|
||||
return {str(name): int(style_id) for name, style_id in cur.fetchall()}
|
||||
|
||||
|
||||
def resolve_style_ids_by_product(
|
||||
conn, platform: str, product_ids: Sequence[Any]
|
||||
) -> dict[str, int]:
|
||||
"""按平台商品 ID 反查 dim_style.id。
|
||||
|
||||
优先级: 系列锚点 > 款名最短 > id 最小。锚点优先是因为平台编码在 loader 里
|
||||
已经整系列聚合到锚点, 同一商品 ID 归属锚点才与采集口径一致; 款名最短是为了
|
||||
在「极星」与「极星双肩」同时命中时选更具体的那个。
|
||||
"""
|
||||
|
||||
column = _PLATFORM_SPU_COLUMN.get(str(platform or "").strip())
|
||||
ids = sorted({str(item).strip() for item in product_ids if str(item or "").strip()})
|
||||
if not column or not ids:
|
||||
return {}
|
||||
sql = f"""
|
||||
SELECT p.product_id, pick.id
|
||||
FROM unnest(%s::text[]) AS p(product_id)
|
||||
JOIN LATERAL (
|
||||
SELECT d.id
|
||||
FROM dim_style d
|
||||
WHERE d.{column} @> to_jsonb(p.product_id)
|
||||
ORDER BY COALESCE(d.is_series_anchor, FALSE) DESC,
|
||||
LENGTH(d.style_name),
|
||||
d.id
|
||||
LIMIT 1
|
||||
) pick ON TRUE
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (ids,))
|
||||
return {str(pid): int(style_id) for pid, style_id in cur.fetchall()}
|
||||
|
||||
|
||||
def attach_style_ids(
|
||||
conn,
|
||||
records: Sequence[dict],
|
||||
*,
|
||||
name_key: str = "style_name",
|
||||
) -> list[dict]:
|
||||
"""就地补齐 records 的 style_id / style_match_method, 返回同一批 dict 副本。
|
||||
|
||||
调用方已带 style_id 的(loader 下发)不覆盖, 只补缺失的。
|
||||
"""
|
||||
|
||||
copied = [dict(record) for record in records]
|
||||
pending = [
|
||||
str(record.get(name_key) or "").strip()
|
||||
for record in copied
|
||||
if record.get("style_id") is None
|
||||
]
|
||||
mapping = resolve_style_ids(conn, pending) if pending else {}
|
||||
for record in copied:
|
||||
if record.get("style_id") is not None:
|
||||
if not record.get("style_match_method"):
|
||||
record["style_match_method"] = STYLE_MATCH_BY_NAME
|
||||
continue
|
||||
name = str(record.get(name_key) or "").strip()
|
||||
style_id = mapping.get(name)
|
||||
record["style_id"] = style_id
|
||||
if not record.get("style_match_method"):
|
||||
record["style_match_method"] = (
|
||||
STYLE_MATCH_BY_NAME if style_id is not None else STYLE_MATCH_UNMATCHED
|
||||
)
|
||||
return copied
|
||||
|
||||
|
||||
# ── daily_style_metrics ──────────────────────────────────────────────────────
|
||||
|
||||
def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
@@ -349,6 +467,7 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"metric_date": r["metric_date"],
|
||||
"style_name": r["style_name"],
|
||||
"platform": r["platform"],
|
||||
"style_id": r.get("style_id"),
|
||||
"sales": 0,
|
||||
"refund_orders": 0,
|
||||
"visitors": 0,
|
||||
@@ -357,6 +476,8 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"source": r.get("source"),
|
||||
}
|
||||
cur = dedup[key]
|
||||
if cur.get("style_id") is None and r.get("style_id") is not None:
|
||||
cur["style_id"] = r["style_id"]
|
||||
for fld in ("sales", "refund_orders", "visitors", "cart_users"):
|
||||
cur[fld] = int(cur.get(fld, 0) or 0) + int(r.get(fld, 0) or 0)
|
||||
existing_codes = set(cur.get("erp_style_codes") or [])
|
||||
@@ -365,7 +486,7 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
cur["erp_style_codes"] = sorted(existing_codes)
|
||||
if r.get("source"):
|
||||
cur["source"] = r["source"]
|
||||
records = list(dedup.values())
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
|
||||
rows = [
|
||||
(
|
||||
@@ -378,12 +499,15 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
int(r.get("cart_users", 0) or 0),
|
||||
r.get("erp_style_codes") or [],
|
||||
r.get("source"),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO daily_style_metrics
|
||||
(metric_date, style_name, platform, sales, refund_orders, visitors, cart_users, erp_style_codes, source)
|
||||
(metric_date, style_name, platform, sales, refund_orders, visitors, cart_users,
|
||||
erp_style_codes, source, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (metric_date, style_name, platform) DO UPDATE SET
|
||||
sales = EXCLUDED.sales,
|
||||
@@ -392,6 +516,9 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
cart_users = EXCLUDED.cart_users,
|
||||
erp_style_codes = EXCLUDED.erp_style_codes,
|
||||
source = EXCLUDED.source,
|
||||
style_id = COALESCE(EXCLUDED.style_id, daily_style_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, daily_style_metrics.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -412,6 +539,7 @@ def upsert_erp_all_shop_style_daily_metrics(conn, records: Sequence[dict]) -> in
|
||||
dedup[key] = dict(record)
|
||||
if not dedup:
|
||||
return 0
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
|
||||
rows = [
|
||||
(
|
||||
@@ -429,13 +557,16 @@ def upsert_erp_all_shop_style_daily_metrics(conn, records: Sequence[dict]) -> in
|
||||
int(record["selected_shop_count"]),
|
||||
int(record.get("matched_product_rows", 0) or 0),
|
||||
str(record["source"]),
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
for record in dedup.values()
|
||||
for record in records
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO erp_all_shop_style_daily_metrics
|
||||
(metric_date, style_name, sales, return_qty, erp_style_codes,
|
||||
selected_shop_count, matched_product_rows, source)
|
||||
selected_shop_count, matched_product_rows, source,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (metric_date, style_name) DO UPDATE SET
|
||||
sales = EXCLUDED.sales,
|
||||
@@ -444,6 +575,9 @@ def upsert_erp_all_shop_style_daily_metrics(conn, records: Sequence[dict]) -> in
|
||||
selected_shop_count = EXCLUDED.selected_shop_count,
|
||||
matched_product_rows = EXCLUDED.matched_product_rows,
|
||||
source = EXCLUDED.source,
|
||||
style_id = COALESCE(EXCLUDED.style_id, erp_all_shop_style_daily_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, erp_all_shop_style_daily_metrics.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -489,8 +623,11 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"present_days": 0,
|
||||
"erp_style_codes": [],
|
||||
"erp_source": r.get("erp_source"),
|
||||
"style_id": r.get("style_id"),
|
||||
}
|
||||
cur = dedup[key]
|
||||
if cur.get("style_id") is None and r.get("style_id") is not None:
|
||||
cur["style_id"] = r["style_id"]
|
||||
# 数字字段累加
|
||||
for fld in ("sales", "pop_sales", "jd_self_sales", "refund_orders",
|
||||
"visitors", "cart_users", "present_days"):
|
||||
@@ -503,7 +640,7 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
# 保留最后一行的 erp_source 标记
|
||||
if r.get("erp_source"):
|
||||
cur["erp_source"] = r["erp_source"]
|
||||
records = list(dedup.values())
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
# ── dedup 完毕 ──────────────────────────────────────────────────────
|
||||
|
||||
rows = [
|
||||
@@ -525,6 +662,8 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
int(r.get("present_days", 0) or 0),
|
||||
r.get("erp_style_codes") or [],
|
||||
r.get("erp_source"),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
@@ -532,7 +671,8 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
INSERT INTO weekly_style_metrics
|
||||
(week_end, week_start, time_period, style_name, platform, sales, pop_sales,
|
||||
jd_self_sales, refund_orders, visitors, cart_users, conversion_rate,
|
||||
cart_rate, refund_rate, present_days, erp_style_codes, erp_source)
|
||||
cart_rate, refund_rate, present_days, erp_style_codes, erp_source,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (week_end, style_name, platform) DO UPDATE SET
|
||||
week_start = EXCLUDED.week_start,
|
||||
@@ -549,6 +689,9 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int:
|
||||
present_days = EXCLUDED.present_days,
|
||||
erp_style_codes = EXCLUDED.erp_style_codes,
|
||||
erp_source = EXCLUDED.erp_source,
|
||||
style_id = COALESCE(EXCLUDED.style_id, weekly_style_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, weekly_style_metrics.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -581,6 +724,7 @@ def upsert_jd_self_sales(conn, rows: Sequence[dict]) -> int:
|
||||
"""rows: [{'week_end', 'week_start', 'style_name', 'sku', 'outbound_qty', 'found', 'collected_at'}, ...]"""
|
||||
if not rows:
|
||||
return 0
|
||||
rows = attach_style_ids(conn, rows)
|
||||
payload = [
|
||||
(
|
||||
r["week_end"],
|
||||
@@ -590,17 +734,23 @@ def upsert_jd_self_sales(conn, rows: Sequence[dict]) -> int:
|
||||
int(r.get("outbound_qty", 0) or 0),
|
||||
bool(r.get("found", False)),
|
||||
r.get("collected_at"),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO jd_self_weekly_sales
|
||||
(week_end, week_start, style_name, sku, outbound_qty, found, collected_at)
|
||||
(week_end, week_start, style_name, sku, outbound_qty, found, collected_at,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (week_end, style_name, sku) DO UPDATE SET
|
||||
outbound_qty = EXCLUDED.outbound_qty,
|
||||
found = EXCLUDED.found,
|
||||
collected_at = EXCLUDED.collected_at,
|
||||
style_id = COALESCE(EXCLUDED.style_id, jd_self_weekly_sales.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, jd_self_weekly_sales.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -694,11 +844,13 @@ def fetch_daily_for_range(conn, start_date: str, end_date: str) -> dict[tuple[st
|
||||
# ── decline_events ──────────────────────────────────────────────────────────
|
||||
|
||||
def upsert_decline_event(conn, event: dict, hermes_response: Any = None) -> None:
|
||||
event = attach_style_ids(conn, [event])[0]
|
||||
sql = """
|
||||
INSERT INTO decline_events
|
||||
(end_date, window_days, style_name, platform, window_start,
|
||||
seg1_sales, seg2_sales, seg3_sales, drop_pct_1to2, drop_pct_2to3,
|
||||
drop_pct_1to3, erp_style_codes, hermes_notified, hermes_response)
|
||||
drop_pct_1to3, erp_style_codes, hermes_notified, hermes_response,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (end_date, style_name, platform) DO UPDATE SET
|
||||
window_days = EXCLUDED.window_days,
|
||||
@@ -710,6 +862,9 @@ def upsert_decline_event(conn, event: dict, hermes_response: Any = None) -> None
|
||||
drop_pct_2to3 = EXCLUDED.drop_pct_2to3,
|
||||
drop_pct_1to3 = EXCLUDED.drop_pct_1to3,
|
||||
erp_style_codes = EXCLUDED.erp_style_codes,
|
||||
style_id = COALESCE(EXCLUDED.style_id, decline_events.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, decline_events.style_match_method),
|
||||
hermes_notified = decline_events.hermes_notified
|
||||
OR EXCLUDED.hermes_notified,
|
||||
hermes_response = COALESCE(
|
||||
@@ -733,6 +888,8 @@ def upsert_decline_event(conn, event: dict, hermes_response: Any = None) -> None
|
||||
event.get("erp_style_codes") or [],
|
||||
bool(hermes_response is not None and not (isinstance(hermes_response, dict) and "error" in hermes_response)),
|
||||
json.dumps(hermes_response, ensure_ascii=False) if hermes_response is not None else None,
|
||||
event.get("style_id"),
|
||||
event.get("style_match_method"),
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, [payload])
|
||||
@@ -861,11 +1018,13 @@ def insert_fact_alert_history(conn, record: dict) -> int:
|
||||
sent_at, recipient, send_channel, hermes_message_id, hermes_response, decline_payload}
|
||||
唯一键: (style_name, trigger_date, recipient) — 同款同日每位收件人只发一次
|
||||
"""
|
||||
record = attach_style_ids(conn, [record])[0]
|
||||
sql = """
|
||||
INSERT INTO fact_alert_history
|
||||
(decline_event_key, style_name, trigger_date, consecutive_days, sales_drop_pct,
|
||||
decline_window_start, decline_window_end, baseline_window_start, baseline_window_end,
|
||||
sent_at, recipient, send_channel, hermes_message_id, hermes_response, decline_payload)
|
||||
sent_at, recipient, send_channel, hermes_message_id, hermes_response, decline_payload,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (style_name, trigger_date, recipient) DO UPDATE SET
|
||||
decline_event_key = EXCLUDED.decline_event_key,
|
||||
@@ -879,7 +1038,10 @@ def insert_fact_alert_history(conn, record: dict) -> int:
|
||||
send_channel = EXCLUDED.send_channel,
|
||||
hermes_message_id = EXCLUDED.hermes_message_id,
|
||||
hermes_response = EXCLUDED.hermes_response,
|
||||
decline_payload = EXCLUDED.decline_payload
|
||||
decline_payload = EXCLUDED.decline_payload,
|
||||
style_id = COALESCE(EXCLUDED.style_id, fact_alert_history.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, fact_alert_history.style_match_method)
|
||||
RETURNING id
|
||||
"""
|
||||
payload = (
|
||||
@@ -898,6 +1060,8 @@ def insert_fact_alert_history(conn, record: dict) -> int:
|
||||
record.get("hermes_message_id"),
|
||||
json.dumps(record["hermes_response"], ensure_ascii=False) if record.get("hermes_response") is not None else None,
|
||||
json.dumps(record["decline_payload"], ensure_ascii=False, default=str) if record.get("decline_payload") is not None else None,
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, [payload])
|
||||
@@ -922,8 +1086,31 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
if not records:
|
||||
return 0
|
||||
|
||||
# 款式归因: 本表只有平台商品 ID, 没有款名列, 按 spus 数组反查 dim_style.id。
|
||||
# 未建档 (product_id 没写进 dim_style) 的行 style_id 留 NULL 并标 unmatched,
|
||||
# 规模用 v_style_unmatched_product 监控。
|
||||
pending: dict[str, list[str]] = {}
|
||||
for r in records:
|
||||
if r.get("style_id") is None:
|
||||
pending.setdefault(str(r.get("platform") or ""), []).append(r.get("product_id"))
|
||||
resolved: dict[tuple[str, str], int] = {}
|
||||
for platform, product_ids in pending.items():
|
||||
for product_id, style_id in resolve_style_ids_by_product(
|
||||
conn, platform, product_ids
|
||||
).items():
|
||||
resolved[(platform, product_id)] = style_id
|
||||
|
||||
rows = []
|
||||
for r in records:
|
||||
platform = str(r.get("platform") or "")
|
||||
product_id = str(r.get("product_id") or "")
|
||||
style_id = r.get("style_id")
|
||||
match_method = r.get("style_match_method")
|
||||
if style_id is None:
|
||||
style_id = resolved.get((platform, product_id))
|
||||
match_method = match_method or (
|
||||
STYLE_MATCH_BY_PRODUCT if style_id is not None else STYLE_MATCH_UNMATCHED
|
||||
)
|
||||
rows.append((
|
||||
r["platform"],
|
||||
r["stat_date"],
|
||||
@@ -948,6 +1135,8 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
r.get("detail_page_bounce_rate"),
|
||||
json.dumps(r.get("raw_data") or {}, ensure_ascii=False, default=str),
|
||||
r.get("source_file"),
|
||||
style_id,
|
||||
match_method or STYLE_MATCH_UNMATCHED,
|
||||
))
|
||||
|
||||
sql = """
|
||||
@@ -958,7 +1147,7 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
paid_amount, paid_qty, paid_buyers,
|
||||
refund_amount, refund_orders,
|
||||
conversion_rate, avg_order_value, uv_value, detail_page_bounce_rate,
|
||||
raw_data, source_file)
|
||||
raw_data, source_file, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (platform, stat_date, product_id) DO UPDATE SET
|
||||
shop_id = EXCLUDED.shop_id,
|
||||
@@ -983,6 +1172,14 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
raw_data = COALESCE(product_daily_metrics.raw_data, '{}'::jsonb)
|
||||
|| EXCLUDED.raw_data,
|
||||
source_file = EXCLUDED.source_file,
|
||||
-- 款式归因只在新值命中时覆盖,避免 spus 临时缺维护时把历史归属抹掉
|
||||
style_id = COALESCE(EXCLUDED.style_id, product_daily_metrics.style_id),
|
||||
style_match_method = CASE
|
||||
WHEN EXCLUDED.style_id IS NOT NULL
|
||||
THEN EXCLUDED.style_match_method
|
||||
ELSE COALESCE(product_daily_metrics.style_match_method,
|
||||
EXCLUDED.style_match_method)
|
||||
END,
|
||||
imported_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -990,122 +1187,154 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
return len(rows)
|
||||
|
||||
|
||||
def upsert_tmall_product_ad_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""兼容旧调用:写入历史独立广告表。
|
||||
def _tm_ad_number(value: Any) -> Decimal | None:
|
||||
"""把报表里的金额/比率单元格转成 Decimal;缺失值返回 None。"""
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip().replace(",", "")
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return Decimal(text)
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
新的万相台工作流使用 :func:`upsert_tmall_product_ad_raw_metrics`,因为
|
||||
data-hub 产品生命进程读取的是 ``product_daily_metrics.raw_data``。
|
||||
|
||||
def _tm_ad_metrics_payload(record: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""取报表指标字典,兼容调用方传 ``ad_metrics`` 或扁平标量键两种写法。"""
|
||||
|
||||
metrics = dict(record.get("ad_metrics") or {})
|
||||
if metrics:
|
||||
return metrics
|
||||
for key in (
|
||||
"spend",
|
||||
"attributed_deal_amount",
|
||||
"direct_deal_amount",
|
||||
"indirect_deal_amount",
|
||||
"reported_roas",
|
||||
"calculated_roas",
|
||||
):
|
||||
if record.get(key) is not None:
|
||||
metrics[key] = record[key]
|
||||
return metrics
|
||||
|
||||
|
||||
def upsert_tmall_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""把万相台商品广告指标幂等写入 ``tm_product_ad_daily_metrics``。
|
||||
|
||||
天猫广告数据不再写入 ``product_daily_metrics.raw_data.tm_ad_metrics``:广告
|
||||
与经营日报分表后,经营数据重新同步不会覆盖投放数据。花费/直接/间接/归因成交
|
||||
额和投产比走标量列(下游可直接 SUM),报表其余指标(展现、点击、点击率、
|
||||
转化率、预售成交等)原样存在 ``metrics`` JSONB。款式按 ``dim_style.tm_spus``
|
||||
的商品 ID 精确匹配,未命中再按商品名称兜底,结果写进 ``style_name`` /
|
||||
``style_match_method``。
|
||||
"""
|
||||
|
||||
if not records:
|
||||
return 0
|
||||
|
||||
rows = [
|
||||
(
|
||||
record["stat_date"],
|
||||
record["product_id"],
|
||||
record.get("product_name"),
|
||||
record.get("spend") or 0,
|
||||
record.get("direct_deal_amount") or 0,
|
||||
record.get("indirect_deal_amount") or 0,
|
||||
record.get("attributed_deal_amount") or 0,
|
||||
record.get("reported_roas"),
|
||||
record.get("calculated_roas"),
|
||||
int(record.get("duplicate_row_count") or 1),
|
||||
record.get("source_file"),
|
||||
# 同一 (stat_date, product_id) 在一次批量里出现两次会让 ON CONFLICT 报
|
||||
# "cannot affect row a second time",先按输入顺序去重(后者覆盖前者)。
|
||||
dedup: dict[tuple[Any, str], dict] = {}
|
||||
for record in records:
|
||||
dedup[(record["stat_date"], str(record["product_id"]))] = dict(record)
|
||||
|
||||
style_by_product_id, style_names = _load_platform_style_catalog(conn, "tm_spus")
|
||||
# 先解析款式名与匹配方式,再统一补 style_id(款式主键统一 2026-11-01)。
|
||||
for record in dedup.values():
|
||||
style_name, match_method = _resolve_style_match(
|
||||
record, style_by_product_id, style_names
|
||||
)
|
||||
record["style_name"] = style_name
|
||||
record["style_match_method"] = match_method
|
||||
enriched = iter(attach_style_ids(conn, list(dedup.values())))
|
||||
|
||||
rows = []
|
||||
for (stat_date, product_id), record in dedup.items():
|
||||
style_id_record = next(enriched)
|
||||
metrics = _tm_ad_metrics_payload(record)
|
||||
metrics.pop("style_name", None)
|
||||
metrics.pop("style_match_method", None)
|
||||
style_name = record["style_name"]
|
||||
match_method = record["style_match_method"]
|
||||
spend = _tm_ad_number(metrics.get("spend")) or Decimal("0")
|
||||
attributed = _tm_ad_number(metrics.get("attributed_deal_amount")) or Decimal("0")
|
||||
calculated_roas = (
|
||||
(attributed / spend).quantize(Decimal("0.000001")) if spend else None
|
||||
)
|
||||
rows.append(
|
||||
(
|
||||
stat_date,
|
||||
product_id,
|
||||
record.get("product_name"),
|
||||
style_name,
|
||||
match_method,
|
||||
spend,
|
||||
_tm_ad_number(metrics.get("direct_deal_amount")) or Decimal("0"),
|
||||
_tm_ad_number(metrics.get("indirect_deal_amount")) or Decimal("0"),
|
||||
attributed,
|
||||
_tm_ad_number(metrics.get("roi"))
|
||||
or _tm_ad_number(record.get("reported_roas")),
|
||||
calculated_roas,
|
||||
json.dumps(metrics, ensure_ascii=False, default=str),
|
||||
int(record.get("duplicate_row_count") or 1),
|
||||
record.get("source_file"),
|
||||
record.get("source_row_number"),
|
||||
style_id_record.get("style_id"),
|
||||
)
|
||||
)
|
||||
for record in records
|
||||
]
|
||||
|
||||
sql = """
|
||||
INSERT INTO tm_product_ad_daily_metrics
|
||||
(stat_date, product_id, product_name, spend,
|
||||
direct_deal_amount, indirect_deal_amount, attributed_deal_amount,
|
||||
reported_roas, calculated_roas, duplicate_row_count, source_file)
|
||||
(stat_date, product_id, product_name, style_name, style_match_method,
|
||||
spend, direct_deal_amount, indirect_deal_amount, attributed_deal_amount,
|
||||
reported_roas, calculated_roas, metrics, duplicate_row_count,
|
||||
source_file, source_row_number, style_id)
|
||||
VALUES %s
|
||||
ON CONFLICT (stat_date, product_id) DO UPDATE SET
|
||||
product_name = EXCLUDED.product_name,
|
||||
product_name = COALESCE(
|
||||
NULLIF(BTRIM(EXCLUDED.product_name), ''),
|
||||
tm_product_ad_daily_metrics.product_name
|
||||
),
|
||||
-- 款式只有本次能匹配到才覆盖,避免后续导入用 NULL 清掉已维护的映射。
|
||||
style_name = COALESCE(
|
||||
EXCLUDED.style_name, tm_product_ad_daily_metrics.style_name
|
||||
),
|
||||
style_match_method = CASE
|
||||
WHEN EXCLUDED.style_name IS NULL
|
||||
THEN tm_product_ad_daily_metrics.style_match_method
|
||||
ELSE EXCLUDED.style_match_method
|
||||
END,
|
||||
style_id = COALESCE(EXCLUDED.style_id,
|
||||
tm_product_ad_daily_metrics.style_id),
|
||||
spend = EXCLUDED.spend,
|
||||
direct_deal_amount = EXCLUDED.direct_deal_amount,
|
||||
indirect_deal_amount = EXCLUDED.indirect_deal_amount,
|
||||
attributed_deal_amount = EXCLUDED.attributed_deal_amount,
|
||||
reported_roas = EXCLUDED.reported_roas,
|
||||
calculated_roas = EXCLUDED.calculated_roas,
|
||||
metrics = EXCLUDED.metrics,
|
||||
duplicate_row_count = EXCLUDED.duplicate_row_count,
|
||||
source_file = EXCLUDED.source_file,
|
||||
imported_at = NOW()
|
||||
source_row_number = EXCLUDED.source_row_number,
|
||||
imported_at = NOW()
|
||||
"""
|
||||
template = "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s, %s)"
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, rows)
|
||||
execute_values(cur, sql, rows, template=template)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def upsert_tmall_product_ad_raw_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""只更新 ``product_daily_metrics.raw_data.tm_ad_metrics``。
|
||||
"""兼容历史调用名;实际写入 ``tm_product_ad_daily_metrics``。"""
|
||||
|
||||
``product_daily_metrics`` 可能已经由三平台经营日报导入过;这里不能用
|
||||
``upsert_product_daily_metrics``,否则会把 visitors/paid_amount 等经营
|
||||
字段清成 NULL。JSONB 节点级更新同时满足重复导入幂等和字段保留;广告
|
||||
节点内的款式按 ``dim_style.tm_spus`` 精确匹配,未命中再按商品名称兜底。
|
||||
"""
|
||||
return upsert_tmall_product_ad_daily_metrics(conn, records)
|
||||
|
||||
if not records:
|
||||
return 0
|
||||
|
||||
style_by_product_id, style_names = _load_platform_style_catalog(conn, "tm_spus")
|
||||
rows = []
|
||||
for record in records:
|
||||
metrics = dict(record.get("ad_metrics") or {})
|
||||
if not metrics:
|
||||
# Keep callers written against the first draft of this importer
|
||||
# compatible while making the canonical storage shape explicit.
|
||||
for key in (
|
||||
"spend",
|
||||
"attributed_deal_amount",
|
||||
"direct_deal_amount",
|
||||
"indirect_deal_amount",
|
||||
"reported_roas",
|
||||
"calculated_roas",
|
||||
):
|
||||
if record.get(key) is not None:
|
||||
metrics[key] = record[key]
|
||||
style_name, match_method = _resolve_style_match(
|
||||
record, style_by_product_id, style_names
|
||||
)
|
||||
metrics["style_name"] = style_name
|
||||
metrics["style_match_method"] = match_method
|
||||
rows.append(
|
||||
(
|
||||
record["stat_date"],
|
||||
str(record["product_id"]),
|
||||
record.get("product_name"),
|
||||
json.dumps({"tm_ad_metrics": metrics}, ensure_ascii=False, default=str),
|
||||
record.get("source_file"),
|
||||
)
|
||||
)
|
||||
def upsert_tmall_product_ad_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""兼容历史调用名;实际写入 ``tm_product_ad_daily_metrics``。"""
|
||||
|
||||
sql = """
|
||||
INSERT INTO product_daily_metrics
|
||||
(platform, stat_date, product_id, product_name, raw_data, source_file, imported_at)
|
||||
VALUES %s
|
||||
ON CONFLICT (platform, stat_date, product_id) DO UPDATE SET
|
||||
product_name = CASE
|
||||
WHEN COALESCE(BTRIM(product_daily_metrics.product_name), '') = ''
|
||||
THEN EXCLUDED.product_name
|
||||
ELSE product_daily_metrics.product_name
|
||||
END,
|
||||
raw_data = jsonb_set(
|
||||
COALESCE(product_daily_metrics.raw_data, '{}'::jsonb),
|
||||
'{tm_ad_metrics}',
|
||||
EXCLUDED.raw_data -> 'tm_ad_metrics',
|
||||
true
|
||||
),
|
||||
source_file = EXCLUDED.source_file,
|
||||
imported_at = NOW()
|
||||
"""
|
||||
template = "('tm', %s, %s, %s, %s::jsonb, %s, NOW())"
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, rows, template=template)
|
||||
return len(rows)
|
||||
return upsert_tmall_product_ad_daily_metrics(conn, records)
|
||||
|
||||
|
||||
def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
@@ -1127,6 +1356,7 @@ def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
)
|
||||
dedup[key] = dict(record)
|
||||
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
rows = [
|
||||
(
|
||||
record["stat_date"],
|
||||
@@ -1145,14 +1375,17 @@ def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
json.dumps(record.get("metrics") or {}, ensure_ascii=False, default=str),
|
||||
record.get("source_file"),
|
||||
record.get("source_row_number"),
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
for record in dedup.values()
|
||||
for record in records
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO jd_product_ad_daily_metrics
|
||||
(stat_date, advertiser_pin, shop_account, report_type, row_key,
|
||||
spu_id, spu_name, sku_id, sku_name, style_name, match_status,
|
||||
spend, reported_roi, metrics, source_file, source_row_number)
|
||||
spend, reported_roi, metrics, source_file, source_row_number,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (stat_date, advertiser_pin, report_type, row_key) DO UPDATE SET
|
||||
shop_account = EXCLUDED.shop_account,
|
||||
@@ -1167,6 +1400,9 @@ def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int:
|
||||
metrics = EXCLUDED.metrics,
|
||||
source_file = EXCLUDED.source_file,
|
||||
source_row_number = EXCLUDED.source_row_number,
|
||||
style_id = COALESCE(EXCLUDED.style_id, jd_product_ad_daily_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, jd_product_ad_daily_metrics.style_match_method),
|
||||
imported_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -1287,6 +1523,7 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int
|
||||
)
|
||||
record["style_name"] = style_name
|
||||
record["style_match_method"] = match_method
|
||||
records = attach_style_ids(conn, list(dedup.values()))
|
||||
fields = (
|
||||
"stat_date",
|
||||
"advertiser_id",
|
||||
@@ -1311,8 +1548,9 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int
|
||||
"refund_rate_1h",
|
||||
"source_file",
|
||||
"source_row_number",
|
||||
"style_id",
|
||||
)
|
||||
rows = [tuple(record.get(field) for field in fields) for record in dedup.values()]
|
||||
rows = [tuple(record.get(field) for field in fields) for record in records]
|
||||
sql = """
|
||||
INSERT INTO dy_product_ad_daily_metrics
|
||||
(stat_date, advertiser_id, product_id, product_name,
|
||||
@@ -1320,7 +1558,8 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int
|
||||
overall_conversion_rate, spend, gross_deal_amount,
|
||||
gross_payment_roi, gross_order_cost, user_actual_paid_amount,
|
||||
platform_subsidy_amount, net_roi, net_deal_amount, net_order_cost,
|
||||
net_settlement_rate, refund_rate_1h, source_file, source_row_number)
|
||||
net_settlement_rate, refund_rate_1h, source_file, source_row_number,
|
||||
style_id)
|
||||
VALUES %s
|
||||
ON CONFLICT (stat_date, advertiser_id, product_id) DO UPDATE SET
|
||||
product_name = EXCLUDED.product_name,
|
||||
@@ -1343,6 +1582,7 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int
|
||||
refund_rate_1h = EXCLUDED.refund_rate_1h,
|
||||
source_file = EXCLUDED.source_file,
|
||||
source_row_number = EXCLUDED.source_row_number,
|
||||
style_id = COALESCE(EXCLUDED.style_id, dy_product_ad_daily_metrics.style_id),
|
||||
imported_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
@@ -1362,11 +1602,12 @@ def upsert_persona_screenshot(conn, record: dict) -> int:
|
||||
}
|
||||
主键 = (collect_date, style_name, platform)。同款同采集日期幂等覆盖。
|
||||
"""
|
||||
record = attach_style_ids(conn, [record])[0]
|
||||
sql = """
|
||||
INSERT INTO persona_screenshots
|
||||
(collect_date, style_name, platform, item_id, screenshot_path, image_data, file_size,
|
||||
bitable_base, bitable_table, bitable_record_id, bitable_file_token,
|
||||
uploaded_to_bitable)
|
||||
uploaded_to_bitable, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (collect_date, style_name, platform) DO UPDATE SET
|
||||
item_id = EXCLUDED.item_id,
|
||||
@@ -1378,6 +1619,9 @@ def upsert_persona_screenshot(conn, record: dict) -> int:
|
||||
bitable_record_id = EXCLUDED.bitable_record_id,
|
||||
bitable_file_token = EXCLUDED.bitable_file_token,
|
||||
uploaded_to_bitable = EXCLUDED.uploaded_to_bitable,
|
||||
style_id = COALESCE(EXCLUDED.style_id, persona_screenshots.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, persona_screenshots.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
payload = (
|
||||
@@ -1393,6 +1637,8 @@ def upsert_persona_screenshot(conn, record: dict) -> int:
|
||||
record.get("bitable_record_id"),
|
||||
record.get("bitable_file_token"),
|
||||
bool(record.get("uploaded_to_bitable", False)),
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, [payload])
|
||||
@@ -1415,13 +1661,14 @@ def upsert_main_image_creatives(conn, records: Sequence[dict]) -> int:
|
||||
"""
|
||||
if not records:
|
||||
return 0
|
||||
records = attach_style_ids(conn, records)
|
||||
sql = """
|
||||
INSERT INTO main_image_creatives
|
||||
(collect_date, style_name, platform, image_key, creative_ids, creative_image_url,
|
||||
impressions, clicks, cost, click_rate, row_count,
|
||||
local_image_path, image_downloaded, image_fingerprint,
|
||||
bitable_base, bitable_table, bitable_record_id, bitable_file_token,
|
||||
uploaded_to_bitable)
|
||||
uploaded_to_bitable, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (collect_date, style_name, platform, image_key) DO UPDATE SET
|
||||
creative_ids = EXCLUDED.creative_ids,
|
||||
@@ -1439,6 +1686,9 @@ def upsert_main_image_creatives(conn, records: Sequence[dict]) -> int:
|
||||
bitable_record_id = EXCLUDED.bitable_record_id,
|
||||
bitable_file_token = EXCLUDED.bitable_file_token,
|
||||
uploaded_to_bitable = EXCLUDED.uploaded_to_bitable,
|
||||
style_id = COALESCE(EXCLUDED.style_id, main_image_creatives.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, main_image_creatives.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
rows = [
|
||||
@@ -1462,6 +1712,8 @@ def upsert_main_image_creatives(conn, records: Sequence[dict]) -> int:
|
||||
r.get("bitable_record_id"),
|
||||
r.get("bitable_file_token"),
|
||||
bool(r.get("uploaded_to_bitable", False)),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
@@ -1573,10 +1825,12 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int:
|
||||
"""
|
||||
if not records:
|
||||
return 0
|
||||
records = attach_style_ids(conn, records)
|
||||
sql = """
|
||||
INSERT INTO daily_persona_metrics
|
||||
(collect_date, style_name, platform, item_id,
|
||||
payload, raw_payload, source_file, collected_at)
|
||||
payload, raw_payload, source_file, collected_at,
|
||||
style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (collect_date, style_name, platform) DO UPDATE SET
|
||||
item_id = EXCLUDED.item_id,
|
||||
@@ -1584,6 +1838,9 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int:
|
||||
raw_payload = EXCLUDED.raw_payload,
|
||||
source_file = EXCLUDED.source_file,
|
||||
collected_at = EXCLUDED.collected_at,
|
||||
style_id = COALESCE(EXCLUDED.style_id, daily_persona_metrics.style_id),
|
||||
style_match_method = COALESCE(
|
||||
EXCLUDED.style_match_method, daily_persona_metrics.style_match_method),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
rows = [
|
||||
@@ -1596,6 +1853,8 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int:
|
||||
json.dumps(r.get("raw_payload"), ensure_ascii=False) if r.get("raw_payload") is not None else None,
|
||||
r.get("source_file"),
|
||||
r.get("collected_at"),
|
||||
r.get("style_id"),
|
||||
r.get("style_match_method"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
@@ -1604,72 +1863,6 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int:
|
||||
return len(rows)
|
||||
|
||||
|
||||
# ── product_reviews ───────────────────────────────────────────────────────────
|
||||
|
||||
def upsert_product_reviews(conn, records: Sequence[dict]) -> int:
|
||||
"""records: [{
|
||||
'platform' ('jd'/'dy'), 'collect_date', 'comment_date',
|
||||
'style_name' (可空, DY 通过 product_id 反查 dim_style.dy_spus 填充),
|
||||
'rating' ('好评'/'中评'/'差评'), 'score', 'content', 'tags' (list),
|
||||
'user_name', 'order_id', 'product_id', 'product_name',
|
||||
'sku_id', 'sku_name', 'sku_size',
|
||||
'raw_data' (dict), 'source_file'
|
||||
}, ...]
|
||||
UNIQUE (platform, order_id) -> 同订单重抓覆盖。
|
||||
"""
|
||||
if not records:
|
||||
return 0
|
||||
rows = [
|
||||
(
|
||||
r["platform"],
|
||||
r["collect_date"],
|
||||
r.get("comment_date"),
|
||||
r.get("style_name"),
|
||||
r["rating"],
|
||||
r.get("score"),
|
||||
r.get("content"),
|
||||
r.get("tags") or None,
|
||||
r.get("user_name"),
|
||||
r["order_id"],
|
||||
r.get("product_id"),
|
||||
r.get("product_name"),
|
||||
r.get("sku_id"),
|
||||
r.get("sku_name"),
|
||||
r.get("sku_size"),
|
||||
json.dumps(r.get("raw_data") or {}, ensure_ascii=False, default=str),
|
||||
r.get("source_file"),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO product_reviews
|
||||
(platform, collect_date, comment_date, style_name, rating, score,
|
||||
content, tags, user_name, order_id, product_id, product_name,
|
||||
sku_id, sku_name, sku_size, raw_data, source_file)
|
||||
VALUES %s
|
||||
ON CONFLICT (platform, order_id) DO UPDATE SET
|
||||
collect_date = EXCLUDED.collect_date,
|
||||
comment_date = EXCLUDED.comment_date,
|
||||
style_name = EXCLUDED.style_name,
|
||||
rating = EXCLUDED.rating,
|
||||
score = EXCLUDED.score,
|
||||
content = EXCLUDED.content,
|
||||
tags = EXCLUDED.tags,
|
||||
user_name = EXCLUDED.user_name,
|
||||
product_id = EXCLUDED.product_id,
|
||||
product_name = EXCLUDED.product_name,
|
||||
sku_id = EXCLUDED.sku_id,
|
||||
sku_name = EXCLUDED.sku_name,
|
||||
sku_size = EXCLUDED.sku_size,
|
||||
raw_data = EXCLUDED.raw_data,
|
||||
source_file = EXCLUDED.source_file,
|
||||
imported_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, rows)
|
||||
return len(rows)
|
||||
|
||||
|
||||
# ── fact_style_analysis_reports ──────────────────────────────────────────────
|
||||
|
||||
def upsert_style_analysis_report(conn, record: dict) -> int:
|
||||
@@ -1685,12 +1878,13 @@ def upsert_style_analysis_report(conn, record: dict) -> int:
|
||||
业务 UNIQUE = (style_name, analysis_date, window_days). 同款同日同窗口重跑覆盖.
|
||||
返回写入行的 id.
|
||||
"""
|
||||
record = attach_style_ids(conn, [record])[0]
|
||||
sql = """
|
||||
INSERT INTO fact_style_analysis_reports
|
||||
(style_name, analysis_date, window_days, window_start, window_end,
|
||||
platforms_included, feishu_doc_id, feishu_doc_url, report_title,
|
||||
report_markdown, hermes_model, prompt_tokens, completion_tokens,
|
||||
source_payload)
|
||||
source_payload, style_id, style_match_method)
|
||||
VALUES %s
|
||||
ON CONFLICT (style_name, analysis_date, window_days) DO UPDATE SET
|
||||
window_start = EXCLUDED.window_start,
|
||||
@@ -1704,6 +1898,10 @@ def upsert_style_analysis_report(conn, record: dict) -> int:
|
||||
prompt_tokens = EXCLUDED.prompt_tokens,
|
||||
completion_tokens = EXCLUDED.completion_tokens,
|
||||
source_payload = EXCLUDED.source_payload,
|
||||
style_id = COALESCE(EXCLUDED.style_id,
|
||||
fact_style_analysis_reports.style_id),
|
||||
style_match_method = COALESCE(EXCLUDED.style_match_method,
|
||||
fact_style_analysis_reports.style_match_method),
|
||||
updated_at = NOW()
|
||||
RETURNING id
|
||||
"""
|
||||
@@ -1726,6 +1924,8 @@ def upsert_style_analysis_report(conn, record: dict) -> int:
|
||||
record.get("completion_tokens"),
|
||||
json.dumps(record["source_payload"], ensure_ascii=False, default=str)
|
||||
if record.get("source_payload") is not None else None,
|
||||
record.get("style_id"),
|
||||
record.get("style_match_method"),
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, [payload])
|
||||
|
||||
@@ -4,29 +4,63 @@
|
||||
-- 设计原则: schema 与 data/ 目录下的 JSON 产物 1:1 映射;upsert 幂等
|
||||
-- ============================================================================
|
||||
|
||||
-- 款式维度表:维护每款的 (各平台商品 ID + 京东自营 SKU + ERP 编码 + 飞书表地址)
|
||||
-- 身份主来源:产品生命进程 cmt_styles.platform_product_ids / erp_style_codes,
|
||||
-- 由 workflow_config.runtime_records() 统一投影;运行时不再从各采集器、JSON 产物反向拼接身份。
|
||||
-- 款式维度表:维护每款的 (各平台商品 ID + 京东自营 SKU + ERP 款式编码 + 飞书表地址)
|
||||
-- 身份主来源:本表自身(2026-10-27 起由产品生命进程 cmt_styles 切换过来)。
|
||||
-- * ERP 编码用 erp_style_codes (JSONB 数组,可多编码),旧列 erp_codes 已废弃,勿再写入/读取。
|
||||
-- * 平台商品 ID 用 tm_spus / jd_spus / dy_spus / jd_self_skus (均为 JSONB 数组)。
|
||||
-- 读取入口:style_config_loader / db.sync_daily_collect_scope 直接查本表;
|
||||
-- cmt_styles 仅在 dim_style 缺同名款时作为回退。
|
||||
-- bitable_weekly_* 仍由 bitable_style_map.json 派生;bitable_sku_master_* 仍由飞书 SKU 主表人工维护。
|
||||
CREATE TABLE IF NOT EXISTS dim_style (
|
||||
style_name TEXT PRIMARY KEY,
|
||||
jd_spus TEXT[], -- 京东 POP 商品 SPU
|
||||
dy_spus TEXT[], -- 抖音商品 SPU
|
||||
tm_spus TEXT[], -- 天猫商品 SPU
|
||||
jd_self_skus TEXT[], -- 京东自营供应链 SKU 集合
|
||||
erp_codes TEXT[], -- 聚水潭 ERP 编码
|
||||
bitable_weekly_url TEXT, -- 款式周报表 URL (bitable_style_map 派生)
|
||||
bitable_weekly_base TEXT, -- 款式周报表 base_token
|
||||
bitable_weekly_table TEXT, -- 款式周报表 table_id
|
||||
bitable_sku_master_url TEXT, -- 飞书 SKU 主表 URL (人工维护)
|
||||
bitable_sku_master_base TEXT, -- 飞书 SKU 主表 base_token (人工维护)
|
||||
bitable_sku_master_table TEXT, -- 飞书 SKU 主表 table_id (人工维护)
|
||||
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
style_name TEXT NOT NULL UNIQUE,
|
||||
brand TEXT NOT NULL DEFAULT '',
|
||||
category TEXT,
|
||||
launch_date DATE,
|
||||
notes TEXT,
|
||||
jd_self_skus JSONB NOT NULL DEFAULT '[]'::jsonb, -- 京东自营供应链 SKU 集合
|
||||
jd_spus JSONB NOT NULL DEFAULT '[]'::jsonb, -- 京东 POP 商品 SPU
|
||||
dy_spus JSONB NOT NULL DEFAULT '[]'::jsonb, -- 抖音商品 SPU
|
||||
tm_spus JSONB NOT NULL DEFAULT '[]'::jsonb, -- 天猫商品 SPU
|
||||
erp_style_codes JSONB NOT NULL DEFAULT '[]'::jsonb, -- 聚水潭 ERP 款式编码(正源)
|
||||
erp_codes BIGINT, -- 已废弃:历史单值列,仅供回溯
|
||||
product_tags JSONB NOT NULL DEFAULT '[]'::jsonb, -- 产品标签
|
||||
lifecycle_stage VARCHAR, -- 生命周期阶段
|
||||
material TEXT,
|
||||
selling_points TEXT,
|
||||
features TEXT,
|
||||
compatible_devices TEXT,
|
||||
image_url TEXT NOT NULL DEFAULT '',
|
||||
big_nfc_count INTEGER NOT NULL DEFAULT 1,
|
||||
small_nfc_count INTEGER NOT NULL DEFAULT 1,
|
||||
jushuitan_category VARCHAR, -- 聚水潭分类
|
||||
series_name VARCHAR, -- 所属系列
|
||||
is_series_anchor BOOLEAN NOT NULL DEFAULT FALSE, -- 系列锚点款
|
||||
series_sort_order INTEGER,
|
||||
collect_enabled BOOLEAN NOT NULL DEFAULT TRUE, -- 商品日报采集开关(系列级:以锚点为准)
|
||||
-- 飞书多维表格地址(9 类,全部以「系列锚点行」为准,子款留空即可)
|
||||
bitable_weekly_url TEXT, -- 销量表 URL (bitable_style_map 派生)
|
||||
bitable_weekly_base TEXT, -- 销量表 base_token
|
||||
bitable_weekly_table TEXT, -- 销量表 table_id
|
||||
bitable_main_image_url TEXT, -- 主图表 URL
|
||||
bitable_creator_url TEXT, -- 合作达人表 URL
|
||||
bitable_self_creator_url TEXT, -- 自营合作达人表 URL
|
||||
bitable_weekly_note_url TEXT, -- 每周/每月笔记分析 URL
|
||||
bitable_style_analysis_url TEXT, -- 平台单品分析 URL
|
||||
bitable_persona_tm_url TEXT, -- 人群画像(天猫)URL
|
||||
bitable_persona_jd_url TEXT, -- 人群画像(京东)URL
|
||||
bitable_persona_dy_url TEXT, -- 人群画像(抖音)URL
|
||||
bitable_sku_master_url TEXT, -- 飞书 SKU 主表 URL (人工维护)
|
||||
bitable_sku_master_base TEXT, -- 飞书 SKU 主表 base_token (人工维护)
|
||||
bitable_sku_master_table TEXT, -- 飞书 SKU 主表 table_id (人工维护)
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- 商品日报采集开关已上移到 dim_style.collect_enabled(系列级,以锚点为准)。
|
||||
-- 旧表 product_daily_collect_scope 已于 2026-10-29 迁移并删除,
|
||||
-- 备份在 var/tmp/collect_scope_before_drop_20261029.json。
|
||||
DROP TABLE IF EXISTS product_daily_collect_scope;
|
||||
|
||||
-- 每日款式级指标 (来自 aggregate_daily_final.py → data/<date>/<款>/final_summary.json)
|
||||
CREATE TABLE IF NOT EXISTS daily_style_metrics (
|
||||
metric_date DATE NOT NULL,
|
||||
@@ -206,13 +240,35 @@ CREATE INDEX IF NOT EXISTS plat_xlsx_item_idx
|
||||
ON fact_platform_xlsx (platform, item_id);
|
||||
|
||||
-- ── 兼容旧版 dim_style:init_schema 在已有库上跑时给老表加列 ─────────────
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS jd_spus TEXT[];
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS dy_spus TEXT[];
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS tm_spus TEXT[];
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS erp_codes TEXT[];
|
||||
-- 平台 ID / ERP 编码统一为 JSONB 数组;ERP 正源是 erp_style_codes,erp_codes 已废弃。
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS jd_spus JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS dy_spus JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS tm_spus JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS erp_style_codes JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS brand TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS lifecycle_stage VARCHAR;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS product_tags JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS jushuitan_category VARCHAR;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS series_name VARCHAR;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS is_series_anchor BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS series_sort_order INTEGER;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS collect_enabled BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS image_url TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS big_nfc_count INTEGER NOT NULL DEFAULT 1;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS small_nfc_count INTEGER NOT NULL DEFAULT 1;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_weekly_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_weekly_base TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_weekly_table TEXT;
|
||||
-- 飞书地址统一上收到 dim_style(2026-10-29):锚点行唯一来源,子款留空自动继承。
|
||||
-- workflow_dynamic_styles / _style_platforms 的同名字段已废弃,代码不再读取。
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_main_image_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_creator_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_self_creator_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_weekly_note_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_style_analysis_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_persona_tm_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_persona_jd_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_persona_dy_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_url TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_base TEXT;
|
||||
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_table TEXT;
|
||||
@@ -223,13 +279,14 @@ ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_table TEXT;
|
||||
-- ============================================================================
|
||||
|
||||
-- dim_style
|
||||
COMMENT ON TABLE dim_style IS '款式维度表 — 维护每款 (各平台商品 ID + 京东自营 SKU + ERP 编码 + 飞书表地址 + 品类/上架日期/备注)。身份投影自产品生命进程,周报地址与 SKU 主表仍按各自来源维护。';
|
||||
COMMENT ON COLUMN dim_style.style_name IS '款式名 (主键),与 PRODUCT_STYLE_GROUPS 的 key 对齐';
|
||||
COMMENT ON COLUMN dim_style.jd_spus IS '京东 POP 商品 SPU 列表';
|
||||
COMMENT ON COLUMN dim_style.dy_spus IS '抖音商品 SPU 列表';
|
||||
COMMENT ON COLUMN dim_style.tm_spus IS '天猫商品 SPU 列表';
|
||||
COMMENT ON COLUMN dim_style.jd_self_skus IS '京东自营供应链 SKU 集合';
|
||||
COMMENT ON COLUMN dim_style.erp_codes IS '聚水潭 ERP 编码列表';
|
||||
COMMENT ON TABLE dim_style IS '款式维度表 — 款式主档与身份正源:各平台商品 ID + 京东自营 SKU + ERP 款式编码 + 飞书表地址 + 品类/上架日期/备注。';
|
||||
COMMENT ON COLUMN dim_style.style_name IS '款式名 (业务唯一键),与产品生命进程款名对齐';
|
||||
COMMENT ON COLUMN dim_style.jd_spus IS '京东 POP 商品 SPU 列表 (JSONB 数组)';
|
||||
COMMENT ON COLUMN dim_style.dy_spus IS '抖音商品 SPU 列表 (JSONB 数组)';
|
||||
COMMENT ON COLUMN dim_style.tm_spus IS '天猫商品 SPU 列表 (JSONB 数组)';
|
||||
COMMENT ON COLUMN dim_style.jd_self_skus IS '京东自营供应链 SKU 集合 (JSONB 数组)';
|
||||
COMMENT ON COLUMN dim_style.erp_style_codes IS '聚水潭 ERP 款式编码列表 (JSONB 数组,ERP 身份正源)';
|
||||
COMMENT ON COLUMN dim_style.erp_codes IS '已废弃:历史单值 ERP 编码列,新代码禁止写入/读取,请改用 erp_style_codes';
|
||||
COMMENT ON COLUMN dim_style.bitable_weekly_url IS '款式周报表 URL (从 bitable_style_map.json 派生)';
|
||||
COMMENT ON COLUMN dim_style.bitable_weekly_base IS '款式周报表 base_token';
|
||||
COMMENT ON COLUMN dim_style.bitable_weekly_table IS '款式周报表 table_id';
|
||||
@@ -433,7 +490,8 @@ COMMENT ON COLUMN product_daily_metrics.detail_page_bounce_rate IS '详情页跳
|
||||
|
||||
-- ============================================================================
|
||||
-- tm_product_ad_daily_metrics: 天猫商品投放日报
|
||||
-- 数据源: 天猫投放平台导出的商品报表 CSV。
|
||||
-- 数据源: 万相台商品报表 CSV (import_tmall_product_ads.py 解析后直接写本表)。
|
||||
-- 每日每商品一行;花费/成交/投产比走标量列供下游直接聚合,报表其余指标留在 metrics。
|
||||
-- 与 product_daily_metrics 分表,避免经营数据重新同步时覆盖投放数据。
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS tm_product_ad_daily_metrics (
|
||||
@@ -441,27 +499,47 @@ CREATE TABLE IF NOT EXISTS tm_product_ad_daily_metrics (
|
||||
stat_date DATE NOT NULL,
|
||||
product_id TEXT NOT NULL,
|
||||
product_name TEXT,
|
||||
style_name TEXT,
|
||||
style_match_method TEXT NOT NULL DEFAULT 'unmatched',
|
||||
spend NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
direct_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
indirect_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
attributed_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0,
|
||||
reported_roas NUMERIC(18, 6),
|
||||
calculated_roas NUMERIC(18, 6),
|
||||
metrics JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
duplicate_row_count INTEGER NOT NULL DEFAULT 1,
|
||||
source_file TEXT,
|
||||
source_row_number INTEGER,
|
||||
imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (stat_date, product_id)
|
||||
);
|
||||
-- 兼容已创建的窄表:补齐款式匹配结果与完整指标快照列。
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS style_name TEXT;
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS style_match_method TEXT
|
||||
NOT NULL DEFAULT 'unmatched';
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS metrics JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
ALTER TABLE tm_product_ad_daily_metrics
|
||||
ADD COLUMN IF NOT EXISTS source_row_number INTEGER;
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_date_idx
|
||||
ON tm_product_ad_daily_metrics (stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_product_date_idx
|
||||
ON tm_product_ad_daily_metrics (product_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_style_date_idx
|
||||
ON tm_product_ad_daily_metrics (style_name, stat_date DESC);
|
||||
|
||||
COMMENT ON TABLE tm_product_ad_daily_metrics IS '天猫商品级投放日报,每日每商品一行;广告归因成交额仅用于计算投产比。';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.product_id IS '万相台主体ID(商品),按文本保存';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.style_name IS '款式名;先按商品 ID 反查 dim_style.tm_spus,未命中再按商品名称精确包含匹配,无法确认时为 NULL';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.style_match_method IS '款式匹配方式:product_id=商品 ID,product_name=商品名称,ambiguous=名称歧义,unmatched=未匹配';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.spend IS '天猫广告花费(页面电商支付金额)';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.attributed_deal_amount IS '广告归因总成交金额,仅用于计算投产比';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.reported_roas IS '源报表投入产出比';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.calculated_roas IS '归因成交金额 / 花费';
|
||||
COMMENT ON COLUMN tm_product_ad_daily_metrics.metrics IS '万相台报表指标原样快照 (展现/点击/点击率/转化率/预售成交等),键名与源报表中文列一一对应';
|
||||
|
||||
-- ============================================================================
|
||||
-- jd_product_ad_daily_metrics: 京东营销概况广告日报
|
||||
@@ -558,7 +636,7 @@ COMMENT ON COLUMN dy_product_ad_daily_metrics.overall_ctr IS '报表整体点击
|
||||
COMMENT ON COLUMN dy_product_ad_daily_metrics.overall_conversion_rate IS '报表整体转化率,单位为百分点';
|
||||
COMMENT ON COLUMN dy_product_ad_daily_metrics.net_settlement_rate IS '净成交金额结算率,单位为百分点';
|
||||
COMMENT ON COLUMN dy_product_ad_daily_metrics.refund_rate_1h IS '1 小时内退款率,单位为百分点';
|
||||
COMMENT ON COLUMN product_daily_metrics.raw_data IS '平台特有原始列 (下单金额/件数/买家数、聚划算、竞争力评分、月累计、搜索引导...) 原样保留;天猫 tm_ad_metrics 同时含 style_name/style_match_method';
|
||||
COMMENT ON COLUMN product_daily_metrics.raw_data IS '平台特有原始列 (下单金额/件数/买家数、聚划算、竞争力评分、月累计、搜索引导...) 原样保留;天猫广告指标已迁至 tm_product_ad_daily_metrics,本列不再写 tm_ad_metrics';
|
||||
COMMENT ON COLUMN product_daily_metrics.source_file IS '原始文件名 (溯源用)';
|
||||
COMMENT ON COLUMN product_daily_metrics.imported_at IS '入库时间';
|
||||
|
||||
@@ -753,69 +831,19 @@ COMMENT ON COLUMN main_image_creatives.collected_at IS '采集时间 (脚本写
|
||||
COMMENT ON COLUMN main_image_creatives.updated_at IS '最近更新时间';
|
||||
|
||||
-- ============================================================================
|
||||
-- product_reviews: 三平台商品评价 (JD + DY, TM 暂未接入)
|
||||
-- 数据源:
|
||||
-- JD: jd_product_data_collector.py --reviews-only -> data/jd单品数据/评价汇总_<date>.json
|
||||
-- DY: dy_product_scraping.py (主流程已挂载) -> data/dy单品数据/评论汇总_<date>.json
|
||||
-- 设计: 三平台字段归一到独立列, 平台特有字段进 raw_data JSONB。
|
||||
-- 写入策略: ON CONFLICT (platform, order_id) DO UPDATE 全量覆盖 (order_id 是评价业务键)。
|
||||
-- product_reviews: 已下线 (2026-11-01)
|
||||
--
|
||||
-- 原设计是 JD + DY 商品评价合一表, 由 import_product_reviews.py 从平台 JSON 入库。
|
||||
-- 下线原因:
|
||||
-- 1. JD 侧 style_name 全靠 sku_id 反查, 而 dim_jd_sku_spu 映射表不存在,
|
||||
-- 214 行 style_name 全为 NULL, 两个消费方 (analyze_style / 营销日报) 都是
|
||||
-- WHERE style_name = ?, 等于入库了但没人能读。
|
||||
-- 2. 差评全量已由 shop_intelligence 的三张差评表 (tmall/jd/dy_negative_reviews)
|
||||
-- 独立采集, 覆盖面更全 (天猫 97 条, 而本表天猫为 0)。
|
||||
--
|
||||
-- 现消费方统一读视图 v_negative_reviews (见文件末尾「款式主键统一」段)。
|
||||
-- 存量表已归档为 product_reviews_archived_20261101, 不再建表、不再写入。
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS product_reviews (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
platform TEXT NOT NULL CHECK (platform IN ('jd','dy')),
|
||||
collect_date DATE NOT NULL, -- 采集日期 (脚本运行日)
|
||||
comment_date DATE, -- 评价时间 (JD 取评价时间日期部分, DY 取 comment_time 日期部分)
|
||||
style_name TEXT, -- 款式名 (DY: 通过 product_id 反查 dim_style.dy_spus 填充; 未匹配时为 NULL)
|
||||
rating TEXT NOT NULL, -- '好评' / '中评' / '差评'
|
||||
score SMALLINT, -- JD 评分 (1-5 星), DY 留空
|
||||
content TEXT,
|
||||
tags TEXT[], -- JD 评价标签 (做工精细/容量大 等), DY 留空
|
||||
user_name TEXT,
|
||||
order_id TEXT NOT NULL, -- 订单 ID (业务键, 用于去重)
|
||||
product_id TEXT, -- 平台商品 ID (DY product_id / JD sku_name 对应的 SPU)
|
||||
product_name TEXT,
|
||||
sku_id TEXT, -- JD SKU 编码, DY 留空
|
||||
sku_name TEXT, -- JD SKU 名称, DY 留空
|
||||
sku_size TEXT, -- JD 规格, DY 留空
|
||||
raw_data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
source_file TEXT, -- 原始 JSON 文件路径
|
||||
imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (platform, order_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS product_reviews_platform_date_idx
|
||||
ON product_reviews (platform, comment_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS product_reviews_rating_idx
|
||||
ON product_reviews (platform, rating, comment_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS product_reviews_product_idx
|
||||
ON product_reviews (platform, product_id, comment_date DESC);
|
||||
-- product_reviews_style_idx 在文件末尾 ALTER TABLE 加 style_name 列之后建 (老表兼容)
|
||||
|
||||
COMMENT ON TABLE product_reviews IS '商品评价 - JD + DY 合一表。来源: jd_product_data_collector.py --reviews-only / dy_product_scraping.py。UNIQUE (platform, order_id) -> 同订单评价重抓覆盖。';
|
||||
COMMENT ON COLUMN product_reviews.platform IS '平台代码 (''jd'' / ''dy'')';
|
||||
COMMENT ON COLUMN product_reviews.collect_date IS '采集日期 (脚本运行日, 不等于评价发生日)';
|
||||
COMMENT ON COLUMN product_reviews.comment_date IS '评价时间 (从评价发生时间戳抽日期部分)';
|
||||
-- style_name 列的 COMMENT 在文件末尾 ALTER TABLE 加列之后 (老表兼容)
|
||||
COMMENT ON COLUMN product_reviews.rating IS '评价等级: 好评 / 中评 / 差评';
|
||||
COMMENT ON COLUMN product_reviews.score IS 'JD 评分 (1-5 星, aria-valuenow), DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.content IS '评价正文';
|
||||
COMMENT ON COLUMN product_reviews.tags IS 'JD 评价标签数组 (如 [做工精细, 容量大]), DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.user_name IS '用户昵称';
|
||||
COMMENT ON COLUMN product_reviews.order_id IS '订单 ID (业务键, 同订单只保留一条, 重抓覆盖)';
|
||||
COMMENT ON COLUMN product_reviews.product_id IS '平台商品 ID (DY product_id / JD SPU)';
|
||||
COMMENT ON COLUMN product_reviews.product_name IS '商品名称';
|
||||
COMMENT ON COLUMN product_reviews.sku_id IS 'JD SKU 编码, DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.sku_name IS 'JD SKU 名称, DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.sku_size IS 'JD 规格文本, DY 留空';
|
||||
COMMENT ON COLUMN product_reviews.raw_data IS '平台特有原始字段 (JD 完整评价对象 / DY comment_rows 行)';
|
||||
COMMENT ON COLUMN product_reviews.source_file IS '源 JSON 文件相对路径 (溯源用)';
|
||||
COMMENT ON COLUMN product_reviews.imported_at IS '入库时间';
|
||||
|
||||
-- 兼容已有库: product_reviews 加 style_name 列 (CREATE TABLE IF NOT EXISTS 不会改已有表)
|
||||
ALTER TABLE product_reviews ADD COLUMN IF NOT EXISTS style_name TEXT;
|
||||
CREATE INDEX IF NOT EXISTS product_reviews_style_idx
|
||||
ON product_reviews (style_name, comment_date DESC)
|
||||
WHERE style_name IS NOT NULL;
|
||||
COMMENT ON COLUMN product_reviews.style_name IS '款式名 (DY: 通过 product_id 反查 dim_style.dy_spus 填充; 未匹配时为 NULL, 表示 dim_style 缺该商品 ID 维护)';
|
||||
|
||||
-- ============================================================================
|
||||
-- fact_style_analysis_reports: 单款式多维度分析报告 (直连 MiniMax 产出)
|
||||
@@ -1307,3 +1335,276 @@ CREATE TABLE IF NOT EXISTS erp_sales_theme_daily_metrics (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS erp_sales_theme_daily_metrics_date_idx
|
||||
ON erp_sales_theme_daily_metrics (stat_date DESC);
|
||||
|
||||
-- ============================================================================
|
||||
-- 款式主键统一 (2026-11-01)
|
||||
--
|
||||
-- 目标: 采集端所有带款式的表统一用 style_id (= dim_style.id) 关联, style_name
|
||||
-- 只作为显示冗余。data-hub 侧 22 张表本来就用 style_id, 两端从此同一套主键,
|
||||
-- 不再出现「同一库两种引用方式」。已实测 erp_product.style_id 135/135 命中。
|
||||
--
|
||||
-- 约定:
|
||||
-- * style_id 可空的表 = 允许「未建档」的行 (平台商品 ID 未写进 dim_style 的
|
||||
-- spus 数组), 这类行必须同时写 style_match_method='unmatched', 便于统计
|
||||
-- 漏维护的规模。
|
||||
-- * style_id 非空的表 = 款式清单来自 dim_style loader, 理论上必然命中;
|
||||
-- 但**不加 NOT NULL**, 由 backfill_style_id.py 与写入端负责填充,
|
||||
-- 避免历史脏款名在 ALTER 阶段中断建表。
|
||||
-- * 主键一律不动 (daily_style_metrics 仍是 metric_date+style_name+platform),
|
||||
-- data-hub 有 6 处按名字 join, 改 PK 收益小、回归面大。
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE daily_style_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE weekly_style_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE erp_all_shop_style_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE daily_persona_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE main_image_creatives ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE fact_style_analysis_reports ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE decline_events ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE fact_alert_history ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE jd_self_weekly_sales ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE persona_screenshots ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE product_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE jd_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE dy_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
ALTER TABLE tm_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT;
|
||||
|
||||
-- 匹配方式留痕: dim_style / product_id / product_name / anchor / unmatched
|
||||
ALTER TABLE daily_style_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE weekly_style_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE erp_all_shop_style_daily_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE daily_persona_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE main_image_creatives ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE fact_style_analysis_reports ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE decline_events ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE fact_alert_history ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE jd_self_weekly_sales ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE persona_screenshots ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE product_daily_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
ALTER TABLE jd_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT;
|
||||
|
||||
COMMENT ON COLUMN daily_style_metrics.style_id IS 'dim_style.id; 采集端统一款式主键, style_name 仅作显示冗余';
|
||||
COMMENT ON COLUMN product_daily_metrics.style_id IS 'dim_style.id; 按 platform+product_id 反查 spus 得到, 未建档为 NULL 且 style_match_method=unmatched';
|
||||
|
||||
-- 查询索引: 按款式跨日期取数 (产品生命页、风格分析、预警)
|
||||
CREATE INDEX IF NOT EXISTS daily_style_metrics_style_id_idx
|
||||
ON daily_style_metrics (style_id, metric_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS weekly_style_metrics_style_id_idx
|
||||
ON weekly_style_metrics (style_id, week_end DESC);
|
||||
CREATE INDEX IF NOT EXISTS erp_all_shop_style_daily_metrics_style_id_idx
|
||||
ON erp_all_shop_style_daily_metrics (style_id, metric_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS daily_persona_metrics_style_id_idx
|
||||
ON daily_persona_metrics (style_id, collect_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS main_image_creatives_style_id_idx
|
||||
ON main_image_creatives (style_id, collect_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS fact_style_analysis_reports_style_id_idx
|
||||
ON fact_style_analysis_reports (style_id, analysis_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS decline_events_style_id_idx
|
||||
ON decline_events (style_id, end_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS fact_alert_history_style_id_idx
|
||||
ON fact_alert_history (style_id, trigger_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS jd_self_weekly_sales_style_id_idx
|
||||
ON jd_self_weekly_sales (style_id, week_end DESC);
|
||||
CREATE INDEX IF NOT EXISTS persona_screenshots_style_id_idx
|
||||
ON persona_screenshots (style_id, collect_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS product_daily_metrics_style_id_idx
|
||||
ON product_daily_metrics (style_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS jd_product_ad_daily_metrics_style_id_idx
|
||||
ON jd_product_ad_daily_metrics (style_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS dy_product_ad_daily_metrics_style_id_idx
|
||||
ON dy_product_ad_daily_metrics (style_id, stat_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_style_id_idx
|
||||
ON tm_product_ad_daily_metrics (style_id, stat_date DESC);
|
||||
|
||||
-- ── P0 观测视图 ─────────────────────────────────────────────────────────────
|
||||
|
||||
-- 未建档的平台商品 ID: 推动在 dim_style 里补 spus, 是 style_id 覆盖率的天花板。
|
||||
CREATE OR REPLACE VIEW v_style_unmatched_product AS
|
||||
SELECT m.platform,
|
||||
m.product_id,
|
||||
MAX(m.product_name) AS product_name,
|
||||
MIN(m.stat_date) AS first_date,
|
||||
MAX(m.stat_date) AS last_date,
|
||||
COUNT(*) AS rows
|
||||
FROM product_daily_metrics m
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM dim_style d
|
||||
WHERE (m.platform = 'tm' AND d.tm_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'jd' AND d.jd_spus @> to_jsonb(m.product_id))
|
||||
OR (m.platform = 'dy' AND d.dy_spus @> to_jsonb(m.product_id)))
|
||||
GROUP BY m.platform, m.product_id;
|
||||
|
||||
COMMENT ON VIEW v_style_unmatched_product IS
|
||||
'未写进 dim_style.spus 的平台商品 ID 清单; 这些行的 product_daily_metrics.style_id 必然为 NULL。';
|
||||
|
||||
-- 同上, 但把三张广告表也算进来。广告投的商品不一定有经营日报 (纯投流款), 只看
|
||||
-- v_style_unmatched_product 会漏掉这部分待建档 ID。列结构保持一致, 便于并排检索。
|
||||
CREATE OR REPLACE VIEW v_style_unmatched_source AS
|
||||
WITH all_rows AS (
|
||||
SELECT platform, product_id, product_name, stat_date FROM product_daily_metrics
|
||||
UNION ALL
|
||||
SELECT 'tm', product_id, product_name, stat_date FROM tm_product_ad_daily_metrics
|
||||
UNION ALL
|
||||
SELECT 'jd', spu_id, spu_name, stat_date FROM jd_product_ad_daily_metrics
|
||||
UNION ALL
|
||||
SELECT 'dy', product_id, product_name, stat_date FROM dy_product_ad_daily_metrics
|
||||
)
|
||||
SELECT s.platform,
|
||||
s.product_id,
|
||||
MAX(s.product_name) AS product_name,
|
||||
MIN(s.stat_date) AS first_date,
|
||||
MAX(s.stat_date) AS last_date,
|
||||
COUNT(*) AS rows
|
||||
FROM all_rows s
|
||||
WHERE s.product_id IS NOT NULL
|
||||
AND BTRIM(s.product_id) <> ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM dim_style d
|
||||
WHERE (s.platform = 'tm' AND d.tm_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id)))
|
||||
GROUP BY s.platform, s.product_id;
|
||||
|
||||
COMMENT ON VIEW v_style_unmatched_source IS
|
||||
'全部采集表 (经营日报 + 三张广告表) 里未写进 dim_style.spus 的商品 ID; 补建档的完整待办清单。';
|
||||
|
||||
-- 各表 style_id 回填进度 (验收用)。
|
||||
CREATE OR REPLACE VIEW v_style_id_coverage AS
|
||||
SELECT 'daily_style_metrics' AS tbl, COUNT(*) AS rows, COUNT(style_id) AS with_style_id FROM daily_style_metrics
|
||||
UNION ALL SELECT 'weekly_style_metrics', COUNT(*), COUNT(style_id) FROM weekly_style_metrics
|
||||
UNION ALL SELECT 'erp_all_shop_style_daily_metrics', COUNT(*), COUNT(style_id) FROM erp_all_shop_style_daily_metrics
|
||||
UNION ALL SELECT 'daily_persona_metrics', COUNT(*), COUNT(style_id) FROM daily_persona_metrics
|
||||
UNION ALL SELECT 'main_image_creatives', COUNT(*), COUNT(style_id) FROM main_image_creatives
|
||||
UNION ALL SELECT 'fact_style_analysis_reports', COUNT(*), COUNT(style_id) FROM fact_style_analysis_reports
|
||||
UNION ALL SELECT 'decline_events', COUNT(*), COUNT(style_id) FROM decline_events
|
||||
UNION ALL SELECT 'fact_alert_history', COUNT(*), COUNT(style_id) FROM fact_alert_history
|
||||
UNION ALL SELECT 'jd_self_weekly_sales', COUNT(*), COUNT(style_id) FROM jd_self_weekly_sales
|
||||
UNION ALL SELECT 'persona_screenshots', COUNT(*), COUNT(style_id) FROM persona_screenshots
|
||||
UNION ALL SELECT 'product_daily_metrics', COUNT(*), COUNT(style_id) FROM product_daily_metrics
|
||||
UNION ALL SELECT 'jd_product_ad_daily_metrics', COUNT(*), COUNT(style_id) FROM jd_product_ad_daily_metrics
|
||||
UNION ALL SELECT 'dy_product_ad_daily_metrics', COUNT(*), COUNT(style_id) FROM dy_product_ad_daily_metrics
|
||||
UNION ALL SELECT 'tm_product_ad_daily_metrics', COUNT(*), COUNT(style_id) FROM tm_product_ad_daily_metrics;
|
||||
|
||||
COMMENT ON VIEW v_style_id_coverage IS
|
||||
'各采集表 style_id 非空比例; 除 product_daily_metrics / 广告表外应为 100%。';
|
||||
|
||||
-- ── P1 统一差评视图 ────────────────────────────────────────────────────────
|
||||
|
||||
-- product_reviews 下线后, 风格分析与营销日报统一读这张视图。
|
||||
-- 与旧表的口径差异: 只有差评一档 (旧表含好/中/差), 但覆盖三个平台。
|
||||
-- 款式归因优先级: 平台商品 ID 精确命中 > 商品名称包含款名 (取最长款名, 更具体)。
|
||||
-- 差评三表由 shop_intelligence 模块建表, 这里用守卫避免全新库初始化时视图先于源表。
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('public.tmall_negative_reviews') IS NOT NULL
|
||||
AND to_regclass('public.jd_negative_reviews') IS NOT NULL
|
||||
AND to_regclass('public.dy_negative_reviews') IS NOT NULL THEN
|
||||
EXECUTE $view$
|
||||
CREATE OR REPLACE VIEW v_negative_reviews AS
|
||||
WITH src AS (
|
||||
SELECT 'tm'::text AS platform,
|
||||
n.business_date AS comment_date,
|
||||
COALESCE(n.review_level, '负面评价') AS rating,
|
||||
n.content,
|
||||
NULL::text AS order_id,
|
||||
NULL::text AS product_id,
|
||||
n.product_name,
|
||||
n.review_id,
|
||||
n.review_at,
|
||||
n.raw_review
|
||||
FROM tmall_negative_reviews n
|
||||
UNION ALL
|
||||
SELECT 'jd', n.business_date, COALESCE(n.review_level, '负面评价'),
|
||||
n.content, n.order_id, NULL::text, n.product_name,
|
||||
n.review_id, n.review_at, n.raw_review
|
||||
FROM jd_negative_reviews n
|
||||
UNION ALL
|
||||
SELECT 'dy', n.business_date, COALESCE(n.rating_level, '负面评价'),
|
||||
n.content, n.order_id, n.product_id, n.product_name,
|
||||
n.review_id, n.review_at, n.raw_review
|
||||
FROM dy_negative_reviews n
|
||||
)
|
||||
SELECT s.platform, s.comment_date, s.rating, s.content,
|
||||
s.order_id, s.product_id, s.product_name,
|
||||
s.review_id, s.review_at, s.raw_review,
|
||||
m.style_id, m.style_name, m.style_match_method
|
||||
FROM src s
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT d.id AS style_id,
|
||||
d.style_name,
|
||||
CASE
|
||||
WHEN (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id))
|
||||
THEN 'product_id'
|
||||
ELSE 'product_name'
|
||||
END AS style_match_method
|
||||
FROM dim_style d
|
||||
WHERE COALESCE(d.style_name, '') <> ''
|
||||
AND (
|
||||
(s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id))
|
||||
OR (s.product_name IS NOT NULL AND s.product_name LIKE '%' || d.style_name || '%')
|
||||
)
|
||||
ORDER BY (
|
||||
CASE
|
||||
WHEN (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id))
|
||||
OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id))
|
||||
THEN 0 ELSE 1
|
||||
END
|
||||
),
|
||||
COALESCE(d.is_series_anchor, FALSE) DESC,
|
||||
LENGTH(d.style_name) DESC,
|
||||
d.id
|
||||
LIMIT 1
|
||||
) m ON TRUE
|
||||
$view$;
|
||||
EXECUTE $c$COMMENT ON VIEW v_negative_reviews IS
|
||||
'三平台差评归一视图 (tm/jd/dy_negative_reviews); 取代已下线的 product_reviews。style_id 为 dim_style.id, 未匹配为 NULL。'$c$;
|
||||
ELSE
|
||||
RAISE NOTICE 'skip v_negative_reviews: 差评表尚未创建 (先运行 shop_intelligence init_schema)';
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
-- ── style_id 一律走「逻辑外键」, 不建物理外键 ───────────────────────────────
|
||||
--
|
||||
-- 2026-11-01 定案: style_id 与 dim_style.id 的关联由应用层保证, 库里**不加**
|
||||
-- FOREIGN KEY 约束。原因:
|
||||
-- * 物理外键会在写入端传错/传旧 id 时直接报错中断整批采集, 而采集是「宁可
|
||||
-- 留 NULL 待回填, 也不要整日落空」的语义, 硬约束与这个语义冲突;
|
||||
-- * 款式被清理/重建 (拆系列、合并同款) 时要先删约束才能动数据, 运维成本高;
|
||||
-- * data-hub 侧 22 张表同样只用 style_id 做关联, 不带物理外键, 两边保持一致。
|
||||
--
|
||||
-- 完整性靠三件事保证, 不需要数据库兜底:
|
||||
-- 1. 写入端: loader 下发的 style_id 来自 dim_style 本身;
|
||||
-- 按商品 ID / 款名兜底的路径由 attach_style_ids 统一查询后填充;
|
||||
-- 2. 校验: backfill_style_id.py --dry-run 与 v_style_id_coverage 报覆盖率;
|
||||
-- 3. 观火: v_style_unmatched_product / v_style_unmatched_source 列未建档清单。
|
||||
--
|
||||
-- 下面这段只做**历史清理**: 早期 P4 曾给 10 张表建过 <表名>_style_id_fk,
|
||||
-- 这里幂等删除, 保证老库重跑 schema 后回到「只有索引、没有物理外键」的状态。
|
||||
DO $$
|
||||
DECLARE
|
||||
target_table TEXT;
|
||||
BEGIN
|
||||
FOREACH target_table IN ARRAY ARRAY[
|
||||
'daily_style_metrics',
|
||||
'weekly_style_metrics',
|
||||
'erp_all_shop_style_daily_metrics',
|
||||
'daily_persona_metrics',
|
||||
'main_image_creatives',
|
||||
'fact_style_analysis_reports',
|
||||
'decline_events',
|
||||
'fact_alert_history',
|
||||
'jd_self_weekly_sales',
|
||||
'persona_screenshots'
|
||||
] LOOP
|
||||
IF to_regclass('public.' || target_table) IS NOT NULL THEN
|
||||
EXECUTE format(
|
||||
'ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I',
|
||||
target_table, target_table || '_style_id_fk'
|
||||
);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
@@ -1,390 +0,0 @@
|
||||
"""同步 dim_style:4 个数据源自动 upsert。
|
||||
|
||||
数据源:
|
||||
产品生命进程 + 动态配置运行时聚合 -> 商品编码、ERP 编码和飞书目标地址
|
||||
|
||||
历史文件、采集产物和硬编码仅保留在本文件的兼容函数中,不再作为同步入口,
|
||||
避免人工修改生命进程后被旧脚本反向覆盖。
|
||||
|
||||
SKU 主表的飞书地址 (bitable_sku_master_*) 由人工维护,不在自动同步范围。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe db/sync_dim_style.py # 全量
|
||||
.venv/Scripts/python.exe db/sync_dim_style.py --dry-run # 扫描不入库
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
sys.path.insert(0, str(PROJECT_ROOT.parents[2]))
|
||||
from runtime_paths import RAW_DATA_ROOT, runtime_script
|
||||
from gyxx_flow.adapters.workflow_config import runtime_workflow_configs # noqa: E402
|
||||
|
||||
from db import get_conn # noqa: E402
|
||||
|
||||
|
||||
PLATFORM_DIRS = {
|
||||
"jd": "jd单品数据",
|
||||
"dy": "dy单品数据",
|
||||
"tm": "tm单品数据",
|
||||
}
|
||||
|
||||
|
||||
# ── 数据源 1: bitable_style_map.json ────────────────────────────────────────
|
||||
|
||||
def load_bitable_weekly() -> dict[str, dict[str, str]]:
|
||||
"""从 bitable_style_map.json 抓每个款式的飞书周报表 base/table。"""
|
||||
path = PROJECT_ROOT / "bitable_style_map.json"
|
||||
if not path.exists():
|
||||
return {}
|
||||
raw = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
out: dict[str, dict[str, str]] = {}
|
||||
for style_name, info in raw.items():
|
||||
if not isinstance(info, dict) or not info.get("enabled", True):
|
||||
continue
|
||||
base = info.get("base_token", "")
|
||||
table = info.get("table_id", "")
|
||||
if not base or not table:
|
||||
continue
|
||||
url = f"https://feishu.cn/base/{base}?table={table}"
|
||||
out[style_name] = {
|
||||
"bitable_weekly_base": base,
|
||||
"bitable_weekly_table": table,
|
||||
"bitable_weekly_url": url,
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
# ── 数据源 2: jd_self SKU 集合 ──────────────────────────────────────────────
|
||||
|
||||
def load_jd_self_skus() -> dict[str, list[str]]:
|
||||
"""从 jd_self_inventory_sales_collector.STYLE_SKU_GROUPS 抓款→SKU 集合。
|
||||
|
||||
直接解析源文件里 STYLE_SKU_GROUPS = [...] 的字面量,避免 import 拉起 playwright。
|
||||
"""
|
||||
path = runtime_script("jd_self_inventory_sales_collector.py")
|
||||
text = path.read_text(encoding="utf-8")
|
||||
needle = "STYLE_SKU_GROUPS = "
|
||||
i = text.find(needle)
|
||||
if i < 0:
|
||||
return {}
|
||||
i += len(needle)
|
||||
# 平衡括号找匹配的 ]
|
||||
depth = 0
|
||||
in_str: str | None = None
|
||||
escape = False
|
||||
end = None
|
||||
for j in range(i, len(text)):
|
||||
ch = text[j]
|
||||
if in_str:
|
||||
if escape:
|
||||
escape = False
|
||||
elif ch == "\\":
|
||||
escape = True
|
||||
elif ch == in_str:
|
||||
in_str = None
|
||||
continue
|
||||
if ch in ("'", '"'):
|
||||
in_str = ch
|
||||
elif ch == "[":
|
||||
depth += 1
|
||||
elif ch == "]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
end = j + 1
|
||||
break
|
||||
if end is None:
|
||||
return {}
|
||||
chunk = text[i:end]
|
||||
try:
|
||||
# 用 ast.literal_eval 解析 Python 字面量(兼容末尾逗号 / 单引号)
|
||||
import ast
|
||||
groups = ast.literal_eval(chunk)
|
||||
except Exception as exc:
|
||||
print(f" [WARN] 解析 STYLE_SKU_GROUPS 失败: {exc}")
|
||||
return {}
|
||||
return {g["style_name"]: list(g.get("skus") or []) for g in groups if "style_name" in g}
|
||||
|
||||
|
||||
# ── 数据源 3: 各平台 SPU 集合 ───────────────────────────────────────────────
|
||||
|
||||
def _latest_style_json(plat_dir: Path, style_name: str) -> Path | None:
|
||||
"""取某款式在 plat_dir 下的最新日期 JSON。
|
||||
|
||||
目录名约定: <款名>_<YYYY-MM-DD> (jd/dy/tm 主流) 或 业务款式_<YYYY-MM-DD> (tm 部分)
|
||||
|
||||
优先级:
|
||||
1) <safe>_<date>.json (jd / dy / tm 主流)
|
||||
2) 业务款式_<date>.json (tm 习惯命名)
|
||||
3) data.json (兜底,但 data.json 通常是 ERP 抓数,可能没有 SPU 字段)
|
||||
"""
|
||||
safe = "".join("_" if ch in '\\/:*?"<>|' else ch for ch in style_name).strip(" ._")
|
||||
# 只匹配以 "<safe>_" 开头的目录,避免拿到别的款式的目录
|
||||
cands: list[tuple[str, Path]] = []
|
||||
for d in plat_dir.iterdir():
|
||||
if not d.is_dir() or not d.name.startswith(safe + "_"):
|
||||
continue
|
||||
date_part = d.name[len(safe) + 1:]
|
||||
try:
|
||||
datetime.strptime(date_part, "%Y-%m-%d")
|
||||
except ValueError:
|
||||
continue
|
||||
cands.append((date_part, d))
|
||||
if not cands:
|
||||
return None
|
||||
cands.sort(key=lambda x: x[0], reverse=True)
|
||||
latest_date, latest_dir = cands[0]
|
||||
|
||||
# 优先级 1: <safe>_<date>.json
|
||||
p1 = latest_dir / f"{safe}_{latest_date}.json"
|
||||
if p1.exists():
|
||||
return p1
|
||||
# 优先级 2: 业务款式_<date>.json (tm)
|
||||
p2 = latest_dir / f"业务款式_{latest_date}.json"
|
||||
if p2.exists():
|
||||
return p2
|
||||
# 优先级 3: data.json
|
||||
p3 = latest_dir / "data.json"
|
||||
if p3.exists():
|
||||
return p3
|
||||
return None
|
||||
|
||||
|
||||
def _extract_spus(payload: dict, platform: str) -> list[str]:
|
||||
"""从款式 JSON payload 提取 SPU/商品 ID 列表。
|
||||
|
||||
各平台字段名:
|
||||
jd: payload['匹配SPU'] 或 payload['spu列表']
|
||||
dy: payload['matched_product_ids'] 或 payload['product_ids']
|
||||
tm: payload['style']['匹配商品ID'] 或 payload['style']['SPU']
|
||||
"""
|
||||
if platform == "jd":
|
||||
return list(payload.get("匹配SPU") or payload.get("spu列表") or [])
|
||||
if platform == "dy":
|
||||
return list(payload.get("matched_product_ids") or payload.get("product_ids") or [])
|
||||
if platform == "tm":
|
||||
style = payload.get("style") or {}
|
||||
return list(style.get("匹配商品ID") or style.get("SPU") or [])
|
||||
return []
|
||||
|
||||
|
||||
def load_platform_spus() -> dict[str, dict[str, list[str]]]:
|
||||
"""返回 {style_name: {platform: [spu, ...]}},跨日期取最近一次抓数。"""
|
||||
out: dict[str, dict[str, list[str]]] = {}
|
||||
# 先枚举所有款式名(从 weekly_style_metrics 拿权威款式列表)
|
||||
data_root = RAW_DATA_ROOT
|
||||
# 用 daily_style_metrics 拿所有款式名
|
||||
try:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT DISTINCT style_name FROM daily_style_metrics")
|
||||
style_names = [r[0] for r in cur.fetchall()]
|
||||
except Exception:
|
||||
style_names = []
|
||||
|
||||
for style_name in style_names:
|
||||
out.setdefault(style_name, {})
|
||||
for platform, dir_name in PLATFORM_DIRS.items():
|
||||
plat_dir = data_root / dir_name
|
||||
if not plat_dir.exists():
|
||||
continue
|
||||
json_path = _latest_style_json(plat_dir, style_name)
|
||||
if not json_path:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(json_path.read_text(encoding="utf-8-sig"))
|
||||
except Exception:
|
||||
continue
|
||||
spus = _extract_spus(payload, platform)
|
||||
if spus:
|
||||
out[style_name][f"{platform}_spus"] = spus
|
||||
return out
|
||||
|
||||
|
||||
# ── 数据源 4: ERP 编码(PG 聚合)──────────────────────────────────────────
|
||||
|
||||
def load_erp_codes() -> dict[str, list[str]]:
|
||||
"""从 daily_style_metrics 聚每个款式的 ERP 编码。"""
|
||||
out: dict[str, list[str]] = {}
|
||||
try:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT style_name, ARRAY(
|
||||
SELECT DISTINCT unnest(erp_style_codes)
|
||||
FROM daily_style_metrics s
|
||||
WHERE s.style_name = dm.style_name
|
||||
AND s.erp_style_codes IS NOT NULL
|
||||
) AS codes
|
||||
FROM (SELECT DISTINCT style_name FROM daily_style_metrics) dm
|
||||
""")
|
||||
for style_name, codes in cur.fetchall():
|
||||
out[style_name] = [c for c in (codes or []) if c]
|
||||
except Exception as exc:
|
||||
print(f" [WARN] 读 daily_style_metrics 失败: {exc}")
|
||||
return out
|
||||
|
||||
|
||||
# ── 合并 + upsert ──────────────────────────────────────────────────────────
|
||||
|
||||
def _merge_records(
|
||||
style_names: Iterable[str],
|
||||
weekly: dict[str, dict],
|
||||
self_skus: dict[str, list[str]],
|
||||
spus_map: dict[str, dict],
|
||||
erp_map: dict[str, list[str]],
|
||||
) -> list[dict]:
|
||||
rows: list[dict] = []
|
||||
for name in style_names:
|
||||
rows.append({
|
||||
"style_name": name,
|
||||
"jd_spus": spus_map.get(name, {}).get("jd_spus", []) or [],
|
||||
"dy_spus": spus_map.get(name, {}).get("dy_spus", []) or [],
|
||||
"tm_spus": spus_map.get(name, {}).get("tm_spus", []) or [],
|
||||
"jd_self_skus": self_skus.get(name, []) or [],
|
||||
"erp_codes": erp_map.get(name, []) or [],
|
||||
"bitable_weekly_url": weekly.get(name, {}).get("bitable_weekly_url", ""),
|
||||
"bitable_weekly_base": weekly.get(name, {}).get("bitable_weekly_base", ""),
|
||||
"bitable_weekly_table": weekly.get(name, {}).get("bitable_weekly_table", ""),
|
||||
# bitable_sku_master_* 由人工维护,不动
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def load_lifecycle_records() -> list[dict[str, Any]]:
|
||||
"""读取生命进程身份与动态目标的统一运行时视图。"""
|
||||
|
||||
return runtime_workflow_configs()
|
||||
|
||||
|
||||
def _lifecycle_to_dim_rows(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
grouped: dict[str, dict[str, Any]] = {}
|
||||
platform_fields = {
|
||||
"天猫": "tm_spus",
|
||||
"京东旗舰店": "jd_spus",
|
||||
"京东": "jd_spus",
|
||||
"京东自营": "jd_self_skus",
|
||||
"抖音": "dy_spus",
|
||||
}
|
||||
for record in records:
|
||||
name = str(record.get("style_name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
row = grouped.setdefault(
|
||||
name,
|
||||
{
|
||||
"style_name": name,
|
||||
"jd_spus": [],
|
||||
"dy_spus": [],
|
||||
"tm_spus": [],
|
||||
"jd_self_skus": [],
|
||||
"erp_codes": [],
|
||||
"bitable_weekly_url": "",
|
||||
"bitable_weekly_base": "",
|
||||
"bitable_weekly_table": "",
|
||||
},
|
||||
)
|
||||
for code in record.get("erp_codes") or []:
|
||||
code = str(code).strip()
|
||||
if code and code not in row["erp_codes"]:
|
||||
row["erp_codes"].append(code)
|
||||
target_url = str(record.get("sales_bitable_url") or "").strip()
|
||||
if target_url and not row["bitable_weekly_url"]:
|
||||
row["bitable_weekly_url"] = target_url
|
||||
match = re.search(
|
||||
r"/base/([A-Za-z0-9]+)\?[^\s]*table=([A-Za-z0-9]+)",
|
||||
target_url,
|
||||
)
|
||||
if match:
|
||||
row["bitable_weekly_base"] = match.group(1)
|
||||
row["bitable_weekly_table"] = match.group(2)
|
||||
field = platform_fields.get(str(record.get("platform") or "").strip())
|
||||
if not field:
|
||||
continue
|
||||
for product_id in record.get("item_ids") or []:
|
||||
product_id = str(product_id).strip()
|
||||
if product_id and product_id not in row[field]:
|
||||
row[field].append(product_id)
|
||||
return list(grouped.values())
|
||||
|
||||
|
||||
def upsert_dim_style(conn, records: list[dict]) -> int:
|
||||
if not records:
|
||||
return 0
|
||||
def arr(v):
|
||||
# 空 list / None 都写空数组 {}(PG 数组维度不空),非空 list 原样写
|
||||
if v is None:
|
||||
return []
|
||||
return list(v)
|
||||
payload = [
|
||||
(
|
||||
r["style_name"],
|
||||
arr(r.get("jd_spus")),
|
||||
arr(r.get("dy_spus")),
|
||||
arr(r.get("tm_spus")),
|
||||
arr(r.get("jd_self_skus")),
|
||||
arr(r.get("erp_codes")),
|
||||
r.get("bitable_weekly_url") or None,
|
||||
r.get("bitable_weekly_base") or None,
|
||||
r.get("bitable_weekly_table") or None,
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO dim_style
|
||||
(style_name, jd_spus, dy_spus, tm_spus, jd_self_skus, erp_codes,
|
||||
bitable_weekly_url, bitable_weekly_base, bitable_weekly_table)
|
||||
VALUES %s
|
||||
ON CONFLICT (style_name) DO UPDATE SET
|
||||
jd_spus = EXCLUDED.jd_spus,
|
||||
dy_spus = EXCLUDED.dy_spus,
|
||||
tm_spus = EXCLUDED.tm_spus,
|
||||
jd_self_skus = EXCLUDED.jd_self_skus,
|
||||
erp_codes = EXCLUDED.erp_codes,
|
||||
bitable_weekly_url = EXCLUDED.bitable_weekly_url,
|
||||
bitable_weekly_base = EXCLUDED.bitable_weekly_base,
|
||||
bitable_weekly_table = EXCLUDED.bitable_weekly_table,
|
||||
-- bitable_sku_master_* 字段:保留人工已填值
|
||||
bitable_sku_master_url = COALESCE(dim_style.bitable_sku_master_url, EXCLUDED.bitable_sku_master_url),
|
||||
bitable_sku_master_base = COALESCE(dim_style.bitable_sku_master_base, EXCLUDED.bitable_sku_master_base),
|
||||
bitable_sku_master_table = COALESCE(dim_style.bitable_sku_master_table, EXCLUDED.bitable_sku_master_table),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
from psycopg2.extras import execute_values
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, payload)
|
||||
return len(payload)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="同步 dim_style (款式 SPU/SKU + 飞书表地址)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="扫描不入库")
|
||||
args = ap.parse_args()
|
||||
|
||||
runtime_records = load_lifecycle_records()
|
||||
records = _lifecycle_to_dim_rows(runtime_records)
|
||||
|
||||
print(f"[INFO] 数据源: product_lifecycle_runtime={len(runtime_records)} 条")
|
||||
print(f"[INFO] 合并款式: {len(records)}")
|
||||
if args.dry_run:
|
||||
for r in records[:5]:
|
||||
print(f" - {r['style_name']}: jd_spus={r['jd_spus']} jd_self_skus={r['jd_self_skus']} weekly_url={r['bitable_weekly_url'][:60]}")
|
||||
return 0
|
||||
|
||||
with get_conn() as conn:
|
||||
n = upsert_dim_style(conn, records)
|
||||
print(f"[OK] dim_style upsert: {n} 行")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,170 +0,0 @@
|
||||
"""同步项目动态配置 → dim_style。
|
||||
|
||||
从项目数据库读取配置,按款式聚合 4 个平台 ID 集合 + ERP 编码 + 飞书链接,
|
||||
upsert 到 dim_style 的 jd_spus / dy_spus / tm_spus / jd_self_skus / erp_codes / bitable_sku_master_* 字段。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe db/sync_sku_master.py # 全量拉 + 入库
|
||||
.venv/Scripts/python.exe db/sync_sku_master.py --dry-run # 拉数据 + 显示不入库
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.adapters.workflow_config import runtime_workflow_configs
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from db import get_conn # noqa: E402
|
||||
|
||||
|
||||
def fetch_sku_master_records() -> list[dict]:
|
||||
"""从项目数据库读取全部启用的动态配置。"""
|
||||
return runtime_workflow_configs()
|
||||
|
||||
|
||||
def _split_ids(value) -> list[str]:
|
||||
"""兼容数据库数组和旧的逗号、换行分隔值。"""
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
s = str(value)
|
||||
# 先按换行 split(京东自营风格),再按逗号 split(其它平台风格)
|
||||
parts: list[str] = []
|
||||
for line in s.splitlines():
|
||||
for piece in line.split(","):
|
||||
piece = piece.strip()
|
||||
if piece:
|
||||
parts.append(piece)
|
||||
return parts
|
||||
|
||||
|
||||
def aggregate(records: list[dict]) -> dict[str, dict]:
|
||||
"""按款式聚合: jd_spus / dy_spus / tm_spus / jd_self_skus / erp_codes / sku_master_link。"""
|
||||
by_style: dict[str, dict] = defaultdict(lambda: {
|
||||
"jd_spus": [], "dy_spus": [], "tm_spus": [], "jd_self_skus": [],
|
||||
"erp_codes": [], "sku_master_url": "", "sku_master_base": "",
|
||||
"sku_master_table": "",
|
||||
})
|
||||
for r in records:
|
||||
plat_field = r.get("platform")
|
||||
plat_list = plat_field if isinstance(plat_field, list) else ([plat_field] if plat_field else [])
|
||||
plat_s = ",".join(str(p) for p in plat_list)
|
||||
name = str(r.get("style_name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
ids_field = r.get("item_ids")
|
||||
id_list = _split_ids(ids_field)
|
||||
erp_field = r.get("erp_codes")
|
||||
erp_list = _split_ids(erp_field)
|
||||
link_field = str(r.get("sales_bitable_url") or "")
|
||||
|
||||
if "京东自营" in plat_s:
|
||||
by_style[name]["jd_self_skus"].extend(id_list)
|
||||
elif "京东" in plat_s:
|
||||
by_style[name]["jd_spus"].extend(id_list)
|
||||
if "抖音" in plat_s:
|
||||
by_style[name]["dy_spus"].extend(id_list)
|
||||
if "天猫" in plat_s:
|
||||
by_style[name]["tm_spus"].extend(id_list)
|
||||
for ec in erp_list:
|
||||
if ec not in by_style[name]["erp_codes"]:
|
||||
by_style[name]["erp_codes"].append(ec)
|
||||
|
||||
if link_field and not by_style[name]["sku_master_url"]:
|
||||
# 解析 https://.../base/XXXXX?table=YYYYY[&view=ZZZZ]
|
||||
m = re.search(
|
||||
r"(https?://[^/]+/base/([A-Za-z0-9]+)\?[^\s]*table=([A-Za-z0-9]+)[^\s]*)",
|
||||
link_field,
|
||||
)
|
||||
if m:
|
||||
by_style[name]["sku_master_url"] = m.group(1)
|
||||
by_style[name]["sku_master_base"] = m.group(2)
|
||||
by_style[name]["sku_master_table"] = m.group(3)
|
||||
else:
|
||||
# fallback: 整段存
|
||||
by_style[name]["sku_master_url"] = link_field
|
||||
return by_style
|
||||
|
||||
|
||||
def upsert_from_sku_master(conn, by_style: dict[str, dict]) -> int:
|
||||
"""upsert SKU 主表数据到 dim_style (只更新 SPU/SKU/ERP/sku_master_* 字段,其它保留)。"""
|
||||
if not by_style:
|
||||
return 0
|
||||
from psycopg2.extras import execute_values
|
||||
|
||||
def arr(v):
|
||||
if v is None:
|
||||
return []
|
||||
return list(v)
|
||||
|
||||
payload = [
|
||||
(
|
||||
name,
|
||||
arr(s.get("jd_spus")),
|
||||
arr(s.get("dy_spus")),
|
||||
arr(s.get("tm_spus")),
|
||||
arr(s.get("jd_self_skus")),
|
||||
arr(s.get("erp_codes")),
|
||||
s.get("sku_master_url") or None,
|
||||
s.get("sku_master_base") or None,
|
||||
s.get("sku_master_table") or None,
|
||||
)
|
||||
for name, s in by_style.items()
|
||||
]
|
||||
sql = """
|
||||
INSERT INTO dim_style
|
||||
(style_name, jd_spus, dy_spus, tm_spus, jd_self_skus, erp_codes,
|
||||
bitable_sku_master_url, bitable_sku_master_base, bitable_sku_master_table)
|
||||
VALUES %s
|
||||
ON CONFLICT (style_name) DO UPDATE SET
|
||||
jd_spus = EXCLUDED.jd_spus,
|
||||
dy_spus = EXCLUDED.dy_spus,
|
||||
tm_spus = EXCLUDED.tm_spus,
|
||||
jd_self_skus = EXCLUDED.jd_self_skus,
|
||||
erp_codes = EXCLUDED.erp_codes,
|
||||
bitable_sku_master_url = EXCLUDED.bitable_sku_master_url,
|
||||
bitable_sku_master_base = EXCLUDED.bitable_sku_master_base,
|
||||
bitable_sku_master_table = EXCLUDED.bitable_sku_master_table,
|
||||
updated_at = NOW()
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, sql, payload)
|
||||
return len(payload)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="同步项目动态配置 → dim_style")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
print("[STEP] 读取项目动态配置 ...")
|
||||
records = fetch_sku_master_records()
|
||||
print(f"[INFO] 拉取 {len(records)} 条记录")
|
||||
by_style = aggregate(records)
|
||||
print(f"[INFO] 聚合 {len(by_style)} 个款式")
|
||||
|
||||
for name in sorted(by_style.keys())[:5]:
|
||||
s = by_style[name]
|
||||
print(f" - {name}: jd={s['jd_spus']} dy={s['dy_spus']} tm={s['tm_spus']} jd_self={s['jd_self_skus']} erp={s['erp_codes']}")
|
||||
|
||||
if args.dry_run:
|
||||
print("[DRY] 不入库")
|
||||
return 0
|
||||
|
||||
with get_conn() as conn:
|
||||
n = upsert_from_sku_master(conn, by_style)
|
||||
print(f"[OK] dim_style upsert: {n} 行")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -249,7 +249,7 @@ def _print_preflight() -> None:
|
||||
return
|
||||
collectable = daily["dy"]["collectable"]
|
||||
skipped = daily["dy"]["skipped"]
|
||||
print(f"[DY] 飞书表格 待采 {len(collectable)} 款, 跳过 {len(skipped)} 款 (款式级缺 erp/sales,或 DY 缺商品链接ID)")
|
||||
print(f"[DY] 飞书表格 待采 {len(collectable)} 款, 跳过 {len(skipped)} 款 (款式级缺 erp 或已由同系列归集,或 DY 缺商品链接ID)")
|
||||
for s in skipped:
|
||||
print(f" - {s['style']}: {s['reason']}")
|
||||
|
||||
@@ -3258,15 +3258,13 @@ def _page_automation(page) -> None:
|
||||
for product in summary.get("products", [])
|
||||
]
|
||||
|
||||
# 评论采集挂在默认流程末尾, 让 daily --target-date 也能采到评价.
|
||||
# _collect_comment_summary 内部会 _open_comment_detail_page 跳转到评论页,
|
||||
# 采完好/中/差后覆盖 评论汇总_<YYYYMMDD>.json. 幂等: 同 order_id 重采覆盖.
|
||||
try:
|
||||
COMMENTS_SUMMARY = _collect_comment_summary(page)
|
||||
_save_comment_summary(COMMENTS_SUMMARY)
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 评论采集失败 (不影响单品数据已落盘): {exc}")
|
||||
COMMENTS_SUMMARY = {}
|
||||
# 评论采集已于 2026-11-01 从默认流程摘除:
|
||||
# 它只产出本地 评论汇总_<YYYYMMDD>.json,而唯一消费方 product_reviews 已下线
|
||||
# (差评改由 shop.dy_negative_reviews.daily 独立采集,消费方读 v_negative_reviews)。
|
||||
# 继续跑等于每天多一次浏览器跳转 + 一份无人读取的文件。
|
||||
# 需要历史评价样本时用 --reviews-only 显式触发,不要恢复默认流程。
|
||||
print("[SKIP] 评论采集已在默认流程下线(如需补采请用 --reviews-only)")
|
||||
COMMENTS_SUMMARY = {}
|
||||
|
||||
_results_saved = True
|
||||
|
||||
|
||||
@@ -75,9 +75,20 @@ def load_erp_metrics(platform, style_name, target_date=None):
|
||||
failed = [c for c in code_results if not c.get("ok") or c.get("error")]
|
||||
if code_results and failed:
|
||||
# 任一编码失败都会造成 ERP 合计不完整,整款回退到平台原值。
|
||||
# 但 TM/DY 没有平台销量字段,整款回退会被兜底成 0,把整款销量抹掉
|
||||
# (如 10244 allsku iframe 偶发 timeout → 宙斯天猫销量变 0)。
|
||||
# 所以同时给出"已成功编码合计",供无平台原值时使用。
|
||||
return {
|
||||
"sales": None,
|
||||
"refund_orders": None,
|
||||
"partial_sales": (
|
||||
sum(int(parse_number(c.get("yesterday_sales"))) for c in ok_results)
|
||||
if ok_results else None
|
||||
),
|
||||
"partial_refunds": (
|
||||
sum(int(parse_number(c.get("yesterday_returns"))) for c in ok_results)
|
||||
if ok_results else None
|
||||
),
|
||||
"source": str(json_path),
|
||||
"erp_style_codes": payload.get("erp_style_codes", []),
|
||||
"all_failed": True,
|
||||
@@ -114,9 +125,17 @@ def apply_erp_override(platform, style_name, visitors, cart_users, target_date=N
|
||||
c = int(parse_number(cart_users))
|
||||
|
||||
if not erp or erp.get("all_failed"):
|
||||
# ERP 拿不到数据 → 用平台原值(如果有),否则 0
|
||||
sales = int(parse_number(platform_sales)) if platform_sales is not None else 0
|
||||
refunds = int(parse_number(platform_refunds)) if platform_refunds is not None else 0
|
||||
if platform_sales is not None:
|
||||
# 平台自己有销量字段(JD/DY)→ 用平台原值,最接近真相
|
||||
sales = int(parse_number(platform_sales))
|
||||
refunds = int(parse_number(platform_refunds)) if platform_refunds is not None else 0
|
||||
elif erp and erp.get("partial_sales") is not None:
|
||||
# 平台没有销量字段(TM)→ 用已成功编码的合计,避免整款归零
|
||||
sales = int(erp["partial_sales"])
|
||||
refunds = int(erp.get("partial_refunds") or 0)
|
||||
else:
|
||||
sales = 0
|
||||
refunds = 0
|
||||
return {
|
||||
"销量": sales,
|
||||
"访客数": v,
|
||||
@@ -127,6 +146,7 @@ def apply_erp_override(platform, style_name, visitors, cart_users, target_date=N
|
||||
"退款率": format_percent(refunds, sales),
|
||||
"erp_override": erp,
|
||||
"erp_unavailable": True,
|
||||
"erp_partial": bool(erp and erp.get("partial_sales") is not None and platform_sales is None),
|
||||
"failed_codes": (erp or {}).get("failed_codes", []),
|
||||
}
|
||||
|
||||
|
||||
@@ -402,11 +402,106 @@ def _unique(values: Iterable[str]) -> list[str]:
|
||||
return list(dict.fromkeys(value for value in values if value))
|
||||
|
||||
|
||||
ROLLUP_CONFIG_PATH = Path(__file__).resolve().parent / "config" / "ad_cost_rollup.json"
|
||||
_rollup_cache: dict[str, dict[str, str]] | None = None
|
||||
|
||||
|
||||
def load_rollup_map(platform: str = "jd") -> dict[str, str]:
|
||||
"""读取父款式汇总口径(子款式 -> 父款式),缺失或非法时返回空映射。
|
||||
|
||||
``宙斯3`` / ``宙斯Z Air`` 这类子款式在京准通报表里名称更长,会先命中
|
||||
名称匹配;父款式 ``宙斯`` 的电商费比需要把整个商品族的消耗算进来,
|
||||
因此汇总口径放在 ``config/ad_cost_rollup.json`` 里显式声明。
|
||||
"""
|
||||
|
||||
global _rollup_cache
|
||||
if _rollup_cache is None:
|
||||
try:
|
||||
payload = json.loads(ROLLUP_CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
payload = {}
|
||||
_rollup_cache = {
|
||||
key: {str(child): str(parent) for child, parent in value.items()}
|
||||
for key, value in payload.items()
|
||||
if isinstance(value, dict)
|
||||
}
|
||||
return dict(_rollup_cache.get(platform, {}))
|
||||
|
||||
|
||||
def _apply_rollup(result: dict[str, Any], rollup: Mapping[str, str]) -> dict[str, Any]:
|
||||
style_name = result.get("style_name")
|
||||
parent = rollup.get(style_name) if style_name else None
|
||||
if not parent or parent == style_name:
|
||||
return result
|
||||
return {
|
||||
**result,
|
||||
"style_name": parent,
|
||||
"reason": f"{result.get('reason', '')} -> 按父款式汇总「{parent}」",
|
||||
}
|
||||
|
||||
|
||||
def _spu_owner_map(
|
||||
style_catalog: Sequence[Mapping[str, Any]],
|
||||
) -> dict[str, str]:
|
||||
"""显式 jd_spus 归属:SPU 挂在哪个款式上,就优先算哪个款式。
|
||||
|
||||
``dim_style.jd_spus`` 是人工维护的商品→款式权威映射。SKU 名称匹配只是
|
||||
fallback:同一商品族的子款式(如「宙斯3」「宙斯Z Air」)名称更长、更容易
|
||||
命中最长匹配,但父款式已经在 jd_spus 里声明了这些 SPU 时,消耗必须归到
|
||||
jd_spus 声明的款式,否则父款式的广告费比恒为空。
|
||||
"""
|
||||
|
||||
owners: dict[str, str] = {}
|
||||
conflicts: set[str] = set()
|
||||
for item in style_catalog:
|
||||
style_name = _text(item.get("style_name"))
|
||||
if not style_name:
|
||||
continue
|
||||
for spu in _spu_values(item.get("jd_spus")):
|
||||
existing = owners.get(spu)
|
||||
if existing is not None and existing != style_name:
|
||||
conflicts.add(spu)
|
||||
continue
|
||||
owners[spu] = style_name
|
||||
for spu in conflicts:
|
||||
owners.pop(spu, None)
|
||||
return owners
|
||||
|
||||
|
||||
def _apply_spu_owner(
|
||||
result: dict[str, Any], spu_owner: Mapping[str, str]
|
||||
) -> dict[str, Any]:
|
||||
spu_id = _text(result.get("spu_id"))
|
||||
if not spu_id:
|
||||
return result
|
||||
owner = spu_owner.get(spu_id)
|
||||
if not owner or owner == result.get("style_name"):
|
||||
return result
|
||||
return {
|
||||
**result,
|
||||
"style_name": owner,
|
||||
"reason": f"{result.get('reason', '')} -> dim_style.jd_spus 归属「{owner}」",
|
||||
}
|
||||
|
||||
|
||||
def resolve_sku_match(
|
||||
sku_name: str,
|
||||
style_catalog: Sequence[Mapping[str, Any]],
|
||||
full_records: Sequence[Mapping[str, Any]],
|
||||
spu_catalog: Sequence[Mapping[str, Any]] = (),
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve one SKU to a style, then honour explicit ``dim_style.jd_spus``."""
|
||||
|
||||
result = _resolve_sku_match(sku_name, style_catalog, full_records, spu_catalog)
|
||||
result = _apply_spu_owner(result, _spu_owner_map(style_catalog))
|
||||
return _apply_rollup(result, load_rollup_map("jd"))
|
||||
|
||||
|
||||
def _resolve_sku_match(
|
||||
sku_name: str,
|
||||
style_catalog: Sequence[Mapping[str, Any]],
|
||||
full_records: Sequence[Mapping[str, Any]],
|
||||
spu_catalog: Sequence[Mapping[str, Any]] = (),
|
||||
) -> dict[str, Any]:
|
||||
"""Return a conservative style/SPU match for one SKU name.
|
||||
|
||||
@@ -671,6 +766,15 @@ def apply_full_style_matches(
|
||||
"full-site SPU has no dim_style.jd_spus or name match"
|
||||
)
|
||||
|
||||
rollup = load_rollup_map("jd")
|
||||
resolved_style = item.get("style_name")
|
||||
parent_style = rollup.get(resolved_style) if resolved_style else None
|
||||
if parent_style and parent_style != resolved_style:
|
||||
item["style_name"] = parent_style
|
||||
item["match_reason"] = (
|
||||
f"{item['match_reason']} -> 按父款式汇总「{parent_style}」"
|
||||
)
|
||||
|
||||
metrics = dict(item.get("metrics") or {})
|
||||
metrics["match"] = {
|
||||
"status": item.get("match_status", "direct"),
|
||||
|
||||
@@ -1,331 +0,0 @@
|
||||
"""
|
||||
扫描 data/{jd,dy}单品数据/评价*.json (或 评论汇总_*.json),
|
||||
归一化后写入 product_reviews 表 (ON CONFLICT 全量覆盖)。
|
||||
|
||||
DY 评价入库时会反查 dim_style.dy_spus 填 style_name;
|
||||
JD 评价缺 SPU->款式映射, style_name 留 NULL (待 dim_jd_sku_spu 映射表补齐后再做)。
|
||||
|
||||
用法:
|
||||
.venv\\Scripts\\python.exe import_product_reviews.py # 全量 (默认只 dy)
|
||||
.venv/Scripts/python.exe import_product_reviews.py --date 2026-07-13 # 单日
|
||||
.venv/Scripts/python.exe import_product_reviews.py --from 2026-06-01 --to 2026-07-13
|
||||
.venv/Scripts/python.exe import_product_reviews.py --platforms dy
|
||||
.venv/Scripts/python.exe import_product_reviews.py --dry-run # 只解析不入库
|
||||
|
||||
源 JSON 文件:
|
||||
JD: data/jd单品数据/评价汇总_<YYYY-MM-DD>.json
|
||||
结构: {platform, shop, collect_date, comment_date_range, captured_at,
|
||||
total_count, ratings: {好评/中评/差评: {collected_count, comments:[{
|
||||
user_name, score, tags, content, time, order_id,
|
||||
sku_name, sku_id, sku_size, rating}]}}}
|
||||
DY: data/dy单品数据/评论汇总_<YYYYMMDD>.json
|
||||
结构: {shop, period, captured_at, url, total_collected_count,
|
||||
ratings: {好评/中评/差评: {expected_count, collected_count, comments:[{
|
||||
rating, order_id, comment_time, comment_text, product_name,
|
||||
product_id, spec_name, spec_id, raw_text}]}}}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from runtime_paths import RAW_DATA_ROOT, artifact_relative_path
|
||||
from typing import Any, Iterable
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from db import get_conn, upsert_product_reviews # noqa: E402
|
||||
|
||||
|
||||
DATA_ROOT = RAW_DATA_ROOT
|
||||
|
||||
PLATFORM_DIRS = {
|
||||
"jd": "jd单品数据",
|
||||
"dy": "dy单品数据",
|
||||
}
|
||||
|
||||
# 文件名模式: JD 用 评价汇总_2026-07-14.json, DY 用 评论汇总_20260604.json
|
||||
FILE_PATTERNS = {
|
||||
"jd": re.compile(r"^评价汇总_(\d{4}-\d{2}-\d{2})\.json$"),
|
||||
"dy": re.compile(r"^评论汇总_(\d{8})\.json$"),
|
||||
}
|
||||
|
||||
VALID_PLATFORMS = tuple(PLATFORM_DIRS.keys())
|
||||
|
||||
RATING_VALUES = ("好评", "中评", "差评")
|
||||
|
||||
|
||||
def _parse_jd_date(s: str) -> date:
|
||||
return datetime.strptime(s, "%Y-%m-%d").date()
|
||||
|
||||
|
||||
def _parse_dy_date(s: str) -> date:
|
||||
return datetime.strptime(s, "%Y%m%d").date()
|
||||
|
||||
|
||||
def _parse_comment_date(s: str) -> date | None:
|
||||
"""从评价时间字符串抽日期部分。
|
||||
|
||||
JD time 格式: '2026-07-13 12:34:56' 或 '2026-07-13'
|
||||
DY comment_time 格式: '2026/06/05 16:39:18' 或 '2026/06/05'
|
||||
"""
|
||||
if not s:
|
||||
return None
|
||||
s = s.strip()
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d %H:%M:%S", "%Y/%m/%d"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
m = re.match(r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})", s)
|
||||
if m:
|
||||
try:
|
||||
return date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _to_smallint(v: Any) -> int | None:
|
||||
if v is None or v == "":
|
||||
return None
|
||||
try:
|
||||
return int(float(str(v)))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_jd(payload: dict, source_file: str, collect_date: date,
|
||||
style_map: dict[str, str] | None = None) -> list[dict]:
|
||||
"""JD JSON -> product_reviews 记录列表
|
||||
|
||||
JD 评价表只有 sku_id (无 SPU), 暂不反查 style_name, 留 NULL。
|
||||
"""
|
||||
records = []
|
||||
for rating in RATING_VALUES:
|
||||
bucket = (payload.get("ratings") or {}).get(rating) or {}
|
||||
for c in bucket.get("comments") or []:
|
||||
comment_date = _parse_comment_date(c.get("time", ""))
|
||||
records.append({
|
||||
"platform": "jd",
|
||||
"collect_date": collect_date,
|
||||
"comment_date": comment_date,
|
||||
"style_name": None, # JD sku_id -> SPU 映射暂缺, 待 dim_jd_sku_spu 补齐
|
||||
"rating": rating,
|
||||
"score": _to_smallint(c.get("score")),
|
||||
"content": (c.get("content") or "").strip() or None,
|
||||
"tags": [t for t in (c.get("tags") or []) if t] or None,
|
||||
"user_name": c.get("user_name") or None,
|
||||
"order_id": str(c.get("order_id") or "").strip(),
|
||||
"product_id": None, # JD 评价表只有 sku_id, SPU 留空
|
||||
"product_name": c.get("sku_name") or None,
|
||||
"sku_id": c.get("sku_id") or None,
|
||||
"sku_name": c.get("sku_name") or None,
|
||||
"sku_size": c.get("sku_size") or None,
|
||||
"raw_data": c,
|
||||
"source_file": source_file,
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
def _normalize_dy(payload: dict, source_file: str, collect_date: date,
|
||||
style_map: dict[str, str] | None = None) -> list[dict]:
|
||||
"""DY JSON -> product_reviews 记录列表
|
||||
|
||||
style_map: {product_id: style_name}, 从 dim_style.dy_spus 构建。
|
||||
匹配不上时 style_name = None (表示 dim_style 缺该商品 ID 维护)。
|
||||
"""
|
||||
style_map = style_map or {}
|
||||
records = []
|
||||
for rating in RATING_VALUES:
|
||||
bucket = (payload.get("ratings") or {}).get(rating) or {}
|
||||
for c in bucket.get("comments") or []:
|
||||
comment_date = _parse_comment_date(c.get("comment_time", ""))
|
||||
product_id = str(c.get("product_id") or "").strip()
|
||||
records.append({
|
||||
"platform": "dy",
|
||||
"collect_date": collect_date,
|
||||
"comment_date": comment_date,
|
||||
"style_name": style_map.get(product_id),
|
||||
"rating": rating,
|
||||
"score": None, # DY 无星级
|
||||
"content": (c.get("comment_text") or "").strip() or None,
|
||||
"tags": None,
|
||||
"user_name": None, # DY 评论无独立用户名字段
|
||||
"order_id": str(c.get("order_id") or "").strip(),
|
||||
"product_id": product_id or None,
|
||||
"product_name": c.get("product_name") or None,
|
||||
"sku_id": c.get("spec_id") or None,
|
||||
"sku_name": c.get("spec_name") or None,
|
||||
"sku_size": None,
|
||||
"raw_data": c,
|
||||
"source_file": source_file,
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
NORMALIZERS = {
|
||||
"jd": _normalize_jd,
|
||||
"dy": _normalize_dy,
|
||||
}
|
||||
|
||||
|
||||
def _load_dy_style_map() -> dict[str, str]:
|
||||
"""从 dim_style.dy_spus 构建 {product_id: style_name} 映射。"""
|
||||
mapping: dict[str, str] = {}
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT style_name, dy_spus FROM dim_style WHERE dy_spus IS NOT NULL")
|
||||
for style_name, spus in cur.fetchall():
|
||||
if not spus:
|
||||
continue
|
||||
for pid in spus:
|
||||
mapping[str(pid)] = style_name
|
||||
return mapping
|
||||
|
||||
|
||||
def _discover_files(platforms: list[str], date_from: date | None, date_to: date | None) -> list[tuple[str, date, Path]]:
|
||||
"""返回 [(platform, collect_date, file_path), ...]"""
|
||||
out = []
|
||||
for plat in platforms:
|
||||
plat_dir = DATA_ROOT / PLATFORM_DIRS[plat]
|
||||
if not plat_dir.exists():
|
||||
continue
|
||||
pattern = FILE_PATTERNS[plat]
|
||||
for path in sorted(plat_dir.iterdir()):
|
||||
m = pattern.match(path.name)
|
||||
if not m:
|
||||
continue
|
||||
try:
|
||||
collect_date = (_parse_jd_date if plat == "jd" else _parse_dy_date)(m.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if date_from and collect_date < date_from:
|
||||
continue
|
||||
if date_to and collect_date > date_to:
|
||||
continue
|
||||
out.append((plat, collect_date, path))
|
||||
return out
|
||||
|
||||
|
||||
def _load_and_normalize(plat: str, collect_date: date, path: Path,
|
||||
style_map: dict[str, str] | None = None) -> list[dict]:
|
||||
with path.open(encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
rel = artifact_relative_path(path)
|
||||
return NORMALIZERS[plat](payload, rel, collect_date, style_map=style_map)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="扫描 JD/DY 评价 JSON 入库 product_reviews")
|
||||
parser.add_argument("--date", help="只导入指定采集日期 (YYYY-MM-DD); JD 用 - 分隔, DY 无分隔也可")
|
||||
parser.add_argument("--from", dest="date_from", help="起始采集日期 (YYYY-MM-DD)")
|
||||
parser.add_argument("--to", dest="date_to", help="结束采集日期 (YYYY-MM-DD)")
|
||||
parser.add_argument("--platforms", default="dy", help="平台, 逗号分隔 (默认 dy; JD 待 SPU 映射补齐)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="只解析不入库")
|
||||
args = parser.parse_args()
|
||||
|
||||
platforms = [p.strip() for p in args.platforms.split(",") if p.strip()]
|
||||
for p in platforms:
|
||||
if p not in VALID_PLATFORMS:
|
||||
print(f"[ERR] 未知平台: {p} (允许: {VALID_PLATFORMS})")
|
||||
return 2
|
||||
|
||||
date_from = _parse_single(args.date_from) if args.date_from else None
|
||||
date_to = _parse_single(args.date_to) if args.date_to else None
|
||||
if args.date:
|
||||
d = _parse_single(args.date)
|
||||
date_from = d
|
||||
date_to = d
|
||||
|
||||
files = _discover_files(platforms, date_from, date_to)
|
||||
if not files:
|
||||
print(f"[WARN] 未找到匹配的 JSON 文件 (platforms={platforms}, from={date_from}, to={date_to})")
|
||||
return 0
|
||||
|
||||
# 预加载 DY 款式映射 (dim_style.dy_spus -> {product_id: style_name})
|
||||
style_map: dict[str, str] = {}
|
||||
if "dy" in platforms and not args.dry_run:
|
||||
style_map = _load_dy_style_map()
|
||||
print(f"[INFO] dim_style.dy_spus 映射: {len(style_map)} 个 product_id -> style_name")
|
||||
elif "dy" in platforms and args.dry_run:
|
||||
# dry-run 也加载, 便于预览匹配情况
|
||||
style_map = _load_dy_style_map()
|
||||
print(f"[INFO] (dry-run) dim_style.dy_spus 映射: {len(style_map)} 个 product_id -> style_name")
|
||||
|
||||
total = 0
|
||||
skipped = 0
|
||||
by_plat: dict[str, int] = {p: 0 for p in platforms}
|
||||
by_rating: dict[str, int] = {"好评": 0, "中评": 0, "差评": 0}
|
||||
matched_styles: set[str] = set()
|
||||
|
||||
for plat, collect_date, path in files:
|
||||
try:
|
||||
records = _load_and_normalize(plat, collect_date, path, style_map=style_map)
|
||||
except Exception as e:
|
||||
print(f"[ERR] 解析 {path}: {e}")
|
||||
continue
|
||||
# 去重: 同 order_id 多条 (理论上 UNIQUE 会处理, 这里先在内存去重避免执行 batch fail)
|
||||
seen = set()
|
||||
deduped = []
|
||||
for r in records:
|
||||
if not r["order_id"]:
|
||||
skipped += 1
|
||||
continue
|
||||
if r["order_id"] in seen:
|
||||
skipped += 1
|
||||
continue
|
||||
seen.add(r["order_id"])
|
||||
deduped.append(r)
|
||||
by_rating[r["rating"]] = by_rating.get(r["rating"], 0) + 1
|
||||
if r.get("style_name"):
|
||||
matched_styles.add(r["style_name"])
|
||||
|
||||
plat_style_matched = sum(1 for r in deduped if r.get("style_name"))
|
||||
print(f"[{plat}] {collect_date} {path.name} -> {len(deduped)} 条 (款式匹配 {plat_style_matched}/{len(deduped)})")
|
||||
|
||||
if args.dry_run:
|
||||
if deduped:
|
||||
sample = deduped[0].copy()
|
||||
sample["raw_data"] = "(省略)"
|
||||
print(f" (dry-run, 不入库) 样本: {sample}")
|
||||
else:
|
||||
with get_conn() as conn:
|
||||
n = upsert_product_reviews(conn, deduped)
|
||||
print(f" -> upsert {n} rows")
|
||||
|
||||
total += len(deduped)
|
||||
by_plat[plat] = by_plat.get(plat, 0) + len(deduped)
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print(f" 总计: {total} 条 (跳过 {skipped} 条无 order_id)")
|
||||
for p, n in by_plat.items():
|
||||
print(f" {p}: {n} 条")
|
||||
print(" 按评级:")
|
||||
for r, n in by_rating.items():
|
||||
print(f" {r}: {n} 条")
|
||||
if matched_styles:
|
||||
print(f" 匹配到款式 ({len(matched_styles)} 款): {', '.join(sorted(matched_styles))}")
|
||||
print("=" * 60)
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_single(s: str) -> date:
|
||||
s = s.strip()
|
||||
for fmt in ("%Y-%m-%d", "%Y%m%d"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
raise SystemExit(f"无法解析日期: {s}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,9 +1,11 @@
|
||||
"""将万相台商品广告 CSV 幂等写入产品生命进程读取的字段。
|
||||
"""将万相台商品广告 CSV 幂等写入天猫商品投放日报表。
|
||||
|
||||
真实数据契约是 ``product_daily_metrics.raw_data.tm_ad_metrics``,不是一张
|
||||
独立的广告表。导入时只替换该 JSON 节点,保留同一商品日期上已经存在的
|
||||
经营日报字段;同时把款式匹配结果写入该节点的 ``style_name`` /
|
||||
``style_match_method``,数据-hub 后端据此计算广告成交金额、投产比和电商费比。
|
||||
数据契约是独立表 ``tm_product_ad_daily_metrics``,每日每商品一行:花费、直接/
|
||||
间接/归因成交金额与投产比走标量列,报表其余指标原样存 ``metrics`` JSONB。
|
||||
款式按 ``dim_style.tm_spus`` 的商品 ID 精确匹配,未命中再按商品名称精确包含
|
||||
兜底,结果写进 ``style_name`` / ``style_match_method``,data-hub 后端据此计算
|
||||
广告成交金额、投产比和电商费比。广告与经营日报分表存储,经营数据重新同步不会
|
||||
覆盖投放数据。
|
||||
|
||||
默认只解析校验;显式传入 ``--execute`` 才写 PostgreSQL。
|
||||
"""
|
||||
@@ -25,8 +27,9 @@ PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from db import ( # noqa: E402
|
||||
ensure_tmall_product_ad_daily_table,
|
||||
get_conn,
|
||||
upsert_tmall_product_ad_raw_metrics,
|
||||
upsert_tmall_product_ad_daily_metrics,
|
||||
)
|
||||
|
||||
AMOUNT_COLUMNS = {
|
||||
@@ -327,8 +330,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return 0
|
||||
|
||||
with get_conn() as conn:
|
||||
written = upsert_tmall_product_ad_raw_metrics(conn, records)
|
||||
print(f"[OK] 已写入 product_daily_metrics.raw_data.tm_ad_metrics:{written} 行")
|
||||
ensure_tmall_product_ad_daily_table(conn)
|
||||
written = upsert_tmall_product_ad_daily_metrics(conn, records)
|
||||
print(f"[OK] 已写入 tm_product_ad_daily_metrics:{written} 行")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from runtime_paths import EXPORTS_DATA_ROOT, LOG_ROOT, RAW_DATA_ROOT
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
DATA_ROOT = RAW_DATA_ROOT
|
||||
STYLE_MAP_PATH = PROJECT_ROOT / "bitable_style_map.json"
|
||||
|
||||
|
||||
def read_json(path):
|
||||
@@ -283,10 +282,15 @@ def main(argv: list[str] | None = None) -> int:
|
||||
try:
|
||||
style_map = get_bitable_style_map()
|
||||
if not style_map:
|
||||
raise RuntimeError("飞书动态配置为空")
|
||||
raise RuntimeError("dim_style 未返回任何带销量表地址的款式")
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 动态读取销量表配置失败,使用本地文件回退: {exc}")
|
||||
style_map = read_json(STYLE_MAP_PATH)
|
||||
# 飞书地址唯一来源是 dim_style:读不到就失败退出,不再回退本地 JSON,
|
||||
# 否则库与本地文件不一致时会静默写到错误的表。
|
||||
print(
|
||||
f"[FAIL] 销量表配置读取失败,拒绝回退本地 JSON: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
if args.source == "db":
|
||||
print(f"[INFO] source=db: 查 weekly_style_metrics week_end={args.date}")
|
||||
records = load_records(args.date, args.record_file, source=args.source)
|
||||
|
||||
@@ -72,6 +72,59 @@ OUTPUT_DIR = RAW_DATA_ROOT / "主图信息"
|
||||
DEBUG_DIR = TMP_ROOT / "debug" / "jd_main_image_snapshot"
|
||||
DEBUG = os.getenv("JD_MAIN_IMAGE_DEBUG", "0") == "1"
|
||||
|
||||
# 京东商家后台门户(登录成功后应停留在这些主机上)。
|
||||
JD_PORTAL_HOSTS = frozenset({"shop.jd.com", "wares-jdm.jd.com"})
|
||||
# 京东风控/账号安全认证页:出现即代表登录态被判定异常,必须人工完成短信/人脸验证。
|
||||
JD_SECURITY_CHALLENGE_HOSTS = frozenset(
|
||||
{"aq.jd.com", "safe.jd.com", "verify.jd.com", "risk.jd.com"}
|
||||
)
|
||||
# 密码登录后等待后台就绪的上限(京东会先加载登录页再异步跳转)。
|
||||
LOGIN_SETTLE_TIMEOUT_MS = 120_000
|
||||
# 无头模式下登录页长时间不跳转即视为登录失败,避免白等到超时。
|
||||
HEADLESS_LOGIN_STALL_SECONDS = 25.0
|
||||
|
||||
|
||||
def _url_summary(url: str) -> str:
|
||||
"""只保留协议/主机/路径。
|
||||
|
||||
京东认证页的查询串里带一次性 token(签名、trackId、returnUrl),
|
||||
不能原样写进日志、运行记录和飞书告警。
|
||||
"""
|
||||
|
||||
parsed = urlparse(str(url))
|
||||
if not parsed.hostname:
|
||||
return "<unknown>"
|
||||
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
|
||||
|
||||
|
||||
def _security_challenge_host(url: str) -> str:
|
||||
"""返回命中的京东安全认证主机名,未命中返回空串。"""
|
||||
|
||||
hostname = (urlparse(str(url)).hostname or "").casefold()
|
||||
return hostname if hostname in JD_SECURITY_CHALLENGE_HOSTS else ""
|
||||
|
||||
|
||||
def _security_challenge_error(url: str) -> RuntimeError:
|
||||
"""登录态被京东风控拦截时的统一可执行错误。"""
|
||||
|
||||
host = _security_challenge_host(url) or "京东安全页"
|
||||
return RuntimeError(
|
||||
f"京东触发安全认证({host}):登录态被判定需要人工验证,"
|
||||
"账号密码登录无法自动完成短信/图形验证。请在能打开浏览器的机器上执行 "
|
||||
"`uv run gyxx accounts login jd-shop --force` 完成验证,再重跑本步骤。"
|
||||
f" 当前页面: {_url_summary(url)}"
|
||||
)
|
||||
|
||||
|
||||
def _is_authenticated_portal(url: str) -> bool:
|
||||
"""当前 URL 是否已是可用的京东商家后台页面。"""
|
||||
|
||||
parsed = urlparse(str(url))
|
||||
hostname = (parsed.hostname or "").casefold()
|
||||
if hostname not in JD_PORTAL_HOSTS:
|
||||
return False
|
||||
return not _looks_like_login_url(url) and not _security_challenge_host(url)
|
||||
|
||||
# 直接使用新版商品明细页,避免旧版别名在批量采集过程中偶发 ERR_CONNECTION_CLOSED。
|
||||
DETAIL_URL = "https://jdsz.jd.com/szweb/view/product/productDetail.html"
|
||||
SEARCH_OVERVIEW_URL = "https://jdsz.jd.com/squidweb/view/flow/searchOverview.html"
|
||||
@@ -240,10 +293,13 @@ def _wait_for_product_detail(page, timeout_ms: int = 30_000) -> None:
|
||||
except Exception:
|
||||
last_url = ""
|
||||
|
||||
if _security_challenge_host(last_url):
|
||||
raise _security_challenge_error(last_url)
|
||||
|
||||
if _looks_like_login_url(last_url):
|
||||
raise RuntimeError(
|
||||
"京东商智登录态已失效:商品明细页跳转到登录页 "
|
||||
f"{last_url};请先刷新 jd-shop 账号登录态后再运行"
|
||||
f"{_url_summary(last_url)};请先刷新 jd-shop 账号登录态后再运行"
|
||||
)
|
||||
|
||||
# A transient non-JD URL can be observed while the new tab settles;
|
||||
@@ -261,14 +317,16 @@ def _wait_for_product_detail(page, timeout_ms: int = 30_000) -> None:
|
||||
except Exception:
|
||||
time.sleep(0.5)
|
||||
|
||||
if _security_challenge_host(last_url):
|
||||
raise _security_challenge_error(last_url)
|
||||
if _looks_like_login_url(last_url):
|
||||
raise RuntimeError(
|
||||
"京东商智登录态已失效:商品明细页跳转到登录页 "
|
||||
f"{last_url};请先刷新 jd-shop 账号登录态后再运行"
|
||||
f"{_url_summary(last_url)};请先刷新 jd-shop 账号登录态后再运行"
|
||||
)
|
||||
raise RuntimeError(
|
||||
"京东商品明细页未加载完成,未找到商品ID搜索框;"
|
||||
f"当前 URL: {last_url or '<unknown>'}"
|
||||
f"当前 URL: {_url_summary(last_url)}"
|
||||
)
|
||||
|
||||
|
||||
@@ -287,10 +345,12 @@ def _wait_for_search_overview(page, timeout_ms: int = 30_000) -> None:
|
||||
last_url = str(page.url)
|
||||
except Exception:
|
||||
last_url = ""
|
||||
if _security_challenge_host(last_url):
|
||||
raise _security_challenge_error(last_url)
|
||||
if _looks_like_login_url(last_url):
|
||||
raise RuntimeError(
|
||||
"京东商智登录态已失效:搜索数据概览跳转到登录页 "
|
||||
f"{last_url};请先刷新 jd-shop 账号登录态后再运行"
|
||||
f"{_url_summary(last_url)};请先刷新 jd-shop 账号登录态后再运行"
|
||||
)
|
||||
try:
|
||||
if page.locator("input.jmt-core-input").first.is_visible(timeout=800):
|
||||
@@ -303,7 +363,7 @@ def _wait_for_search_overview(page, timeout_ms: int = 30_000) -> None:
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(
|
||||
"京东搜索数据概览页未加载完成,未找到 SKU 筛选输入框;"
|
||||
f"当前 URL: {last_url or '<unknown>'}"
|
||||
f"当前 URL: {_url_summary(last_url)}"
|
||||
)
|
||||
|
||||
|
||||
@@ -698,6 +758,30 @@ def _ensure_page(context, page):
|
||||
return context.new_page()
|
||||
|
||||
|
||||
def _page_reports_no_data(page) -> bool:
|
||||
"""商品明细表是否明确渲染了「暂无数据」空态。
|
||||
|
||||
商智商品明细只展示窗口内有曝光/成交的商品;某款当日零曝光时查询结果
|
||||
为空,这是正常业务空值。老逻辑把它当成「解析失败」,一款无数据就会让
|
||||
整周工作流失败、其余款式也写不进飞书和 PG(2026-10-18 即此故障)。
|
||||
"""
|
||||
|
||||
for selector in (
|
||||
".jmt-base-table-empty .jmt-empty-description-color",
|
||||
".jmt-base-table-empty",
|
||||
):
|
||||
try:
|
||||
locator = page.locator(selector).first
|
||||
if not locator.is_visible(timeout=800):
|
||||
continue
|
||||
text = locator.inner_text(timeout=800) or ""
|
||||
except Exception:
|
||||
continue
|
||||
if "暂无数据" in text:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def collect_style(context, page, style_name: str, spus: list[str], output_dir: Path) -> dict[str, Any]:
|
||||
"""采集单款: 明细页取 SPU 树与主图 -> 搜索概览取曝光/点击 -> 聚合下载写 JSON。"""
|
||||
if not spus:
|
||||
@@ -722,6 +806,20 @@ def collect_style(context, page, style_name: str, spus: list[str], output_dir: P
|
||||
snapshot_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
snapshot_path.write_text(page.content(), encoding="utf-8")
|
||||
print(f" [DEBUG] snapshot: {snapshot_path}")
|
||||
if _page_reports_no_data(page):
|
||||
empty = {
|
||||
"style_name": style_name,
|
||||
"platform": "jd",
|
||||
"collected_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"spu_count": 0,
|
||||
"raw_sku_count": 0,
|
||||
"skipped": True,
|
||||
"skip_reason": "京东商智当日无曝光/成交数据(查询结果为空)",
|
||||
"records": [],
|
||||
}
|
||||
write_style_outputs(empty, output_dir)
|
||||
print(f" [SKIP] {style_name}: 京东商智当日无数据(暂无数据),记录空结果并继续")
|
||||
return empty
|
||||
raise RuntimeError(f"款 {style_name} 明细页未解析到 SKU 行")
|
||||
|
||||
metrics = collect_sku_search_metrics(context, [r["sku_id"] for r in sku_rows])
|
||||
@@ -757,6 +855,65 @@ def parse_styles(value: str | None) -> list[dict]:
|
||||
return [g for g in groups if g["style"] in wanted]
|
||||
|
||||
|
||||
def wait_for_portal_session(page, *, headless: bool) -> None:
|
||||
"""等密码登录后回到商家后台,识别登录页与京东风控认证页。
|
||||
|
||||
2026-10-18 的实际故障就是停在 ``aq.jd.com/certified`` 认证页:
|
||||
老逻辑只判断 ``shop.jd.com in url``,报错时把带一次性 token 的完整 URL
|
||||
打进运行记录,运维无法判断该做什么。这里把三种终态分开处理:
|
||||
|
||||
- 回到后台:继续采集;
|
||||
- 登录页:无头模式下短暂等待异步跳转,仍不跳转即判定登录失败;
|
||||
- 安全认证页:无头模式直接抛可执行错误(无法自动完成短信/图形验证),
|
||||
有界面模式给人工留出完成验证的时间。
|
||||
"""
|
||||
|
||||
deadline = time.monotonic() + LOGIN_SETTLE_TIMEOUT_MS / 1000
|
||||
login_stuck_since: float | None = None
|
||||
notice_printed = False
|
||||
last_url = ""
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
last_url = str(page.url)
|
||||
except Exception:
|
||||
last_url = ""
|
||||
|
||||
if _is_authenticated_portal(last_url):
|
||||
page.wait_for_timeout(1500)
|
||||
return
|
||||
|
||||
challenge = _security_challenge_host(last_url)
|
||||
if challenge:
|
||||
if headless:
|
||||
raise _security_challenge_error(last_url)
|
||||
if not notice_printed:
|
||||
print(
|
||||
f" [需要操作] 京东安全认证页({challenge})已打开,"
|
||||
"请在浏览器内完成短信/图形验证"
|
||||
)
|
||||
notice_printed = True
|
||||
elif _looks_like_login_url(last_url):
|
||||
login_stuck_since = login_stuck_since or time.monotonic()
|
||||
if headless and time.monotonic() - login_stuck_since >= HEADLESS_LOGIN_STALL_SECONDS:
|
||||
break
|
||||
else:
|
||||
login_stuck_since = None
|
||||
|
||||
try:
|
||||
page.wait_for_timeout(500)
|
||||
except Exception:
|
||||
time.sleep(0.5)
|
||||
|
||||
if _security_challenge_host(last_url):
|
||||
raise _security_challenge_error(last_url)
|
||||
raise RuntimeError(
|
||||
"京东登录未完成:密码登录后没有回到商家后台,请执行 "
|
||||
"`uv run gyxx accounts login jd-shop --force` 人工确认登录态后重跑;"
|
||||
f"当前页面: {_url_summary(last_url)}"
|
||||
)
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> None:
|
||||
global _DATE_LABEL
|
||||
output_dir = managed_data_path(args.output)
|
||||
@@ -781,12 +938,15 @@ def run(args: argparse.Namespace) -> None:
|
||||
# Step 1: 登录
|
||||
page.goto("https://shop.jd.com/jdm/home", wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_timeout(2000)
|
||||
if "shop.jd.com" in page.url and not _looks_like_login_url(page.url):
|
||||
if _is_authenticated_portal(page.url):
|
||||
print("[OK] 检测到京东已登录态")
|
||||
else:
|
||||
if _security_challenge_host(page.url):
|
||||
# 认证页上再提交一次账密只会重复触发风控,直接给出人工处置路径。
|
||||
raise _security_challenge_error(page.url)
|
||||
step_login(page, args.shop, args.password)
|
||||
if "shop.jd.com" not in page.url or _looks_like_login_url(page.url):
|
||||
raise RuntimeError(f"京东登录未完成,当前 URL: {page.url}")
|
||||
wait_for_portal_session(page, headless=args.headless)
|
||||
print("[OK] 京东登录完成")
|
||||
|
||||
# Step 2-3: 商智 -> 新版页面
|
||||
sz_page = step_enter_shangzhi(context, page)
|
||||
|
||||
@@ -45,6 +45,7 @@ from collection_progress import SpuProgress
|
||||
|
||||
from jd_data_collector import (
|
||||
DEFAULT_CONFIG,
|
||||
close_ai_helper_drawer_if_present as close_jd_ai_helper_if_present,
|
||||
step_login,
|
||||
step_enter_shangzhi,
|
||||
extract,
|
||||
@@ -128,7 +129,7 @@ def _print_preflight() -> None:
|
||||
return
|
||||
collectable = daily["jd"]["collectable"]
|
||||
skipped = daily["jd"]["skipped"]
|
||||
print(f"[JD] 飞书表格 待采 {len(collectable)} 款, 跳过 {len(skipped)} 款 (款式级缺 erp/sales,或 JD 缺商品链接ID)")
|
||||
print(f"[JD] 飞书表格 待采 {len(collectable)} 款, 跳过 {len(skipped)} 款 (款式级缺 erp 或已由同系列归集,或 JD 缺商品链接ID)")
|
||||
for s in skipped:
|
||||
print(f" - {s['style']}: {s['reason']}")
|
||||
|
||||
@@ -932,19 +933,40 @@ _JMT_CUSTOM_REACT_CLICK = r"""(args) => {
|
||||
const [ty, tm, targetDay] = args;
|
||||
const pad = (n) => String(n).padStart(2, '0');
|
||||
const ymRe = new RegExp('^' + ty + '\\u5e74' + pad(tm) + '\\u6708$');
|
||||
// 商智新版「自定义」是双月日历: 每个月的面板里除了本月日期, 还会渲染
|
||||
// 上下月的补位日期 (class 带 -diff-month)。同一个月面板里因此可能出现
|
||||
// 两个同号日期 (如 10 月面板里的 9/30 补位格与 10/30 真身), 之前按
|
||||
// 「第一个匹配」点击会落到补位格, 于是 echo 变成 2026-09-30。
|
||||
// 这里先定位目标月份自己的面板, 再排除补位格与禁用格。
|
||||
let panel = null;
|
||||
for (const el of document.querySelectorAll('*')) {
|
||||
const t = (el.textContent || '').trim();
|
||||
if (!ymRe.test(t)) continue;
|
||||
panel = el.closest('[class*="calendar"]') || el.parentElement.parentElement || el.parentElement;
|
||||
break;
|
||||
let cur = el.parentElement;
|
||||
let depth = 0;
|
||||
while (cur && depth < 6) {
|
||||
if (cur.querySelector && cur.querySelector('td')) break;
|
||||
cur = cur.parentElement;
|
||||
depth++;
|
||||
}
|
||||
if (cur && cur.querySelector('td')) {
|
||||
panel = cur;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!panel) return {status: 'no_panel'};
|
||||
for (const cell of panel.querySelectorAll('td, [class*="cell"]')) {
|
||||
if ((cell.textContent || '').trim() !== targetDay) continue;
|
||||
if (/disabled|off/i.test(String(cell.className))) continue;
|
||||
const cells = Array.from(panel.querySelectorAll('td, [class*="cell"]')).filter((cell) => {
|
||||
if ((cell.textContent || '').trim() !== targetDay) return false;
|
||||
const cls = String(cell.className || '');
|
||||
if (/diff-month|disabled|-off\b|other-month|prev-month|next-month/i.test(cls)) return false;
|
||||
const rect = cell.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
});
|
||||
if (!cells.length) return {status: 'no_cell'};
|
||||
for (const cell of cells) {
|
||||
const nodes = [cell, ...cell.querySelectorAll('*'), cell.parentElement];
|
||||
for (const node of nodes) {
|
||||
if (!node) continue;
|
||||
const propsKey = Object.keys(node).find((k) => k.startsWith('__reactProps$'));
|
||||
if (!propsKey) continue;
|
||||
const props = node[propsKey];
|
||||
@@ -955,9 +977,8 @@ _JMT_CUSTOM_REACT_CLICK = r"""(args) => {
|
||||
return {status: 'ok'};
|
||||
}
|
||||
}
|
||||
return {status: 'no_handler'};
|
||||
}
|
||||
return {status: 'no_cell'};
|
||||
return {status: 'no_handler'};
|
||||
}"""
|
||||
|
||||
|
||||
@@ -1006,13 +1027,33 @@ def _select_custom_date_new_version(page, target) -> bool:
|
||||
点工具条「自定义」→ 在双月日历里对目标日连续触发两次 React onClick
|
||||
(起止同日 = 单日区间) → echo 显示目标日期即成功.
|
||||
"""
|
||||
# 商智会随机弹出 AI 助手抽屉 / 引导层, 全屏 jd-modal-wrap 会拦截
|
||||
# 日期工具条的点击; 先尝试收起再点「自定义」。
|
||||
try:
|
||||
page.keyboard.press("Escape")
|
||||
page.wait_for_timeout(300)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
close_jd_ai_helper_if_present(page)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
tag = page.get_by_text("自定义", exact=True).first
|
||||
tag.wait_for(state="visible", timeout=8000)
|
||||
tag.click(timeout=8000)
|
||||
except Exception as exc:
|
||||
print(f" [WARN] 新版「自定义」入口点击失败: {exc}")
|
||||
return False
|
||||
print(f" [WARN] 新版「自定义」入口点击失败({str(exc)[:120]}), 改用 DOM 事件点击")
|
||||
clicked = page.evaluate(
|
||||
r"""() => {
|
||||
const el = document.querySelector('li[data-event-content="当前时间_自定义"]');
|
||||
if (!el) return false;
|
||||
el.dispatchEvent(new MouseEvent('click', {bubbles: true, cancelable: true}));
|
||||
return true;
|
||||
}"""
|
||||
)
|
||||
if not clicked:
|
||||
return False
|
||||
page.wait_for_timeout(1200)
|
||||
|
||||
for _ in range(2):
|
||||
|
||||
@@ -433,20 +433,9 @@ def run_collect(args, target_date, platforms, failed):
|
||||
failed=failed,
|
||||
historical=is_historical_target(target_date),
|
||||
)
|
||||
# 评论入库 (DY 默认分支已采评论 -> 评论汇总_<YYYYMMDD>.json; JD 单独 --reviews-only 采)
|
||||
# 幂等: product_reviews UNIQUE (platform, order_id) ON CONFLICT DO UPDATE, 重跑覆盖不双写.
|
||||
# critical=False: 评论入库失败不影响后续 analyze/export/insert.
|
||||
if not args.skip_platforms and not args.skip_reviews_import:
|
||||
review_plats = [p for p in platforms if p in ("jd", "dy")]
|
||||
if review_plats:
|
||||
run_step_with_retry(
|
||||
"import_product_reviews",
|
||||
[sys.executable, str(runtime_script("import_product_reviews.py")),
|
||||
"--platforms", ",".join(review_plats)],
|
||||
dry_run=args.dry_run,
|
||||
critical=False,
|
||||
failed=failed,
|
||||
)
|
||||
# 评论入库已下线 (2026-11-01): 差评由 shop.jd/dy/tmall_negative_reviews.daily
|
||||
# 三个独立工作流采集, 消费方统一读视图 v_negative_reviews。
|
||||
# product_reviews 停写并归档为 product_reviews_archived_20261101。
|
||||
|
||||
|
||||
def collect_daily_skip_report(date_str: str) -> Path | None:
|
||||
@@ -675,8 +664,6 @@ def main() -> int:
|
||||
parser.add_argument("--platforms", default="jd,dy,tm", help="逗号分隔: jd,dy,tm")
|
||||
parser.add_argument("--serial-platforms", action="store_true", help="三平台串行跑")
|
||||
parser.add_argument("--skip-insert", action="store_true", help="只导出,不插入飞书")
|
||||
parser.add_argument("--skip-reviews-import", action="store_true",
|
||||
help="跳过评论入库 (默认 daily collect 后跑 import_product_reviews)")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument(
|
||||
"--erp-mode",
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
"""评论采集编排器: 默认只跑 DY 采集 + 入库 PG。
|
||||
|
||||
JD 评价采集代码已就绪 (jd_product_data_collector.py --reviews-only),
|
||||
但 JD 评价表只有 sku_id 缺 SPU, 暂时无法反查到款式, 所以编排器默认不跑 JD。
|
||||
等 dim_jd_sku_spu 映射表补齐后可加 --platforms jd,dy。
|
||||
|
||||
DY 主流程 (dy_product_scraping.py 默认分支) 已挂载 _collect_comment_summary,
|
||||
daily --target-date 会顺带采评论; 本编排器用 --reviews-only 只跑评论 (跳过单品列表/
|
||||
下载/人群画像), 用于历史补采或重跑。
|
||||
入库调 import_product_reviews.py, ON CONFLICT (platform, order_id) 幂等覆盖。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py --target-date 2026-07-13
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py --platforms jd,dy # JD 待 SPU 映射补齐
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py --skip-collect # 只入库
|
||||
.venv/Scripts/python.exe orchestrate_review_collection.py --dry-run
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from collect_retry_utils import FailedCollector
|
||||
from runtime_paths import runtime_script
|
||||
|
||||
from gyxx_flow.adapters import environment_for_child_script
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
MAX_RUN_RETRIES = 2
|
||||
RUN_SLEEP_SECONDS = 5
|
||||
|
||||
|
||||
def default_target_date() -> str:
|
||||
return (datetime.now().date() - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def validate_date(value: str) -> str:
|
||||
try:
|
||||
datetime.strptime(value, "%Y-%m-%d")
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError("date must be YYYY-MM-DD") from exc
|
||||
return value
|
||||
|
||||
|
||||
def run_step(name, command, dry_run=False, critical=True):
|
||||
"""执行子命令。critical=True 失败抛 RuntimeError, False 仅 [WARN] 继续。"""
|
||||
printable = " ".join(str(c) for c in command)
|
||||
print(f"[RUN] {name}: {printable}")
|
||||
if dry_run:
|
||||
return {"name": name, "code": 0, "dry_run": True}
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
command,
|
||||
cwd=PROJECT_ROOT,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
env=environment_for_child_script(command[1], os.environ),
|
||||
)
|
||||
except (FileNotFoundError, OSError) as exc:
|
||||
msg = f"{name} {type(exc).__name__}: {exc}"
|
||||
if critical:
|
||||
raise RuntimeError(msg) from exc
|
||||
print(f"[WARN] {msg}")
|
||||
return {"name": name, "code": 126, "error": msg}
|
||||
|
||||
if proc.returncode:
|
||||
msg = f"{name} failed with exit code {proc.returncode}"
|
||||
if critical:
|
||||
raise RuntimeError(msg)
|
||||
print(f"[WARN] {msg}")
|
||||
return {"name": name, "code": proc.returncode, "error": msg}
|
||||
print(f"[OK] {name}")
|
||||
return {"name": name, "code": proc.returncode}
|
||||
|
||||
|
||||
def run_step_with_retry(name, command, dry_run=False, critical=True,
|
||||
failed: FailedCollector | None = None,
|
||||
max_retries: int = MAX_RUN_RETRIES,
|
||||
sleep_seconds: float = RUN_SLEEP_SECONDS) -> dict:
|
||||
if dry_run:
|
||||
return run_step(name, command, dry_run=True, critical=critical)
|
||||
last_result: dict = {"name": name, "code": 0}
|
||||
total = max_retries + 1
|
||||
for attempt in range(1, total + 1):
|
||||
result = run_step(name, command, dry_run=False, critical=False)
|
||||
last_result = result
|
||||
if result.get("code") == 0:
|
||||
if attempt > 1:
|
||||
print(f"[OK] {name} 第 {attempt}/{total} 次重试成功", flush=True)
|
||||
return result
|
||||
err = result.get("error", "unknown")
|
||||
if attempt >= total:
|
||||
if failed is not None:
|
||||
failed.add_page(name, f"{err} (attempts={attempt})", attempts=attempt)
|
||||
if critical:
|
||||
raise RuntimeError(f"{name} failed after {attempt} attempts: {err}")
|
||||
print(f"[WARN] {name} 失败 (已重试 {max_retries} 次, {err}), 继续", flush=True)
|
||||
return result
|
||||
print(f"[RETRY] {name} 第 {attempt}/{total} 次失败, {sleep_seconds}s 后重试...", flush=True)
|
||||
import time
|
||||
time.sleep(sleep_seconds)
|
||||
return last_result
|
||||
|
||||
|
||||
def build_collect_command(plat: str, target_date: str) -> list[str]:
|
||||
if plat == "jd":
|
||||
# JD --reviews-only: 只跑评价采集 (好/中/差全采), 跳过商智单品明细
|
||||
return [sys.executable, str(runtime_script("jd_product_data_collector.py")),
|
||||
"--reviews-only", "--target-date", target_date]
|
||||
if plat == "dy":
|
||||
# DY --reviews-only: 登录后直接跳评论页, 跳过单品列表/下载/人群画像
|
||||
return [sys.executable, str(runtime_script("dy_product_scraping.py")),
|
||||
"--reviews-only", "--target-date", target_date]
|
||||
raise ValueError(f"unknown platform: {plat}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="评论采集编排器 (JD + DY)")
|
||||
parser.add_argument("--target-date", type=validate_date, default=None,
|
||||
help="采集指定日期 YYYY-MM-DD (默认昨天)")
|
||||
parser.add_argument("--platforms", default="dy",
|
||||
help="平台, 逗号分隔 (默认 dy; JD 待 SPU 映射补齐, 可手动 jd,dy)")
|
||||
parser.add_argument("--skip-collect", action="store_true",
|
||||
help="跳过采集, 只跑 import_product_reviews.py")
|
||||
parser.add_argument("--skip-import", action="store_true",
|
||||
help="跳过入库, 只跑采集")
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="只打印命令不执行")
|
||||
args = parser.parse_args()
|
||||
|
||||
target_date = args.target_date or default_target_date()
|
||||
platforms = [p.strip() for p in args.platforms.split(",") if p.strip()]
|
||||
for p in platforms:
|
||||
if p not in ("jd", "dy"):
|
||||
print(f"[ERR] 未知平台: {p} (当前评论采集仅支持 jd, dy)")
|
||||
return 2
|
||||
|
||||
today_str = datetime.now().strftime("%Y-%m-%d")
|
||||
failed = FailedCollector("reviews", today_str)
|
||||
|
||||
print("=" * 60)
|
||||
print(" 评论采集编排器 (默认 DY)")
|
||||
print("=" * 60)
|
||||
print(f" Target date: {target_date}")
|
||||
print(f" Platforms: {platforms}")
|
||||
print(f" Skip collect: {args.skip_collect}")
|
||||
print(f" Skip import: {args.skip_import}")
|
||||
print(f" Dry run: {args.dry_run}")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
# Stage 1: 采集 (串行, Chrome profile 互锁)
|
||||
if not args.skip_collect:
|
||||
for plat in platforms:
|
||||
cmd = build_collect_command(plat, target_date)
|
||||
run_step_with_retry(
|
||||
f"collect_{plat}", cmd,
|
||||
dry_run=args.dry_run, critical=False, failed=failed,
|
||||
)
|
||||
else:
|
||||
print("[INFO] --skip-collect: 跳过采集阶段\n")
|
||||
|
||||
# Stage 2: 入库
|
||||
if not args.skip_import:
|
||||
import_cmd = [sys.executable, str(runtime_script("import_product_reviews.py")),
|
||||
"--platforms", ",".join(platforms)]
|
||||
if args.dry_run:
|
||||
import_cmd.append("--dry-run")
|
||||
run_step_with_retry(
|
||||
"import_reviews", import_cmd,
|
||||
dry_run=args.dry_run, critical=False, failed=failed,
|
||||
)
|
||||
else:
|
||||
print("[INFO] --skip-import: 跳过入库阶段\n")
|
||||
|
||||
# 失败清单
|
||||
failed_path = failed.write()
|
||||
if failed_path:
|
||||
print(f"\n[FAIL_LOG] {failed.summary_line()} -> {failed_path}")
|
||||
return 1
|
||||
print("\n[OK] 评论采集编排完成, 无失败")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,344 @@
|
||||
"""Weekly points-exchange ranking for private-domain material exchange.
|
||||
|
||||
The data-hub points-exchange endpoint returns one row per exchanged SKU with a
|
||||
``quantity`` field. This workflow aggregates those rows into one ranking of
|
||||
items (商品) for a closed time window and posts the ranking to Feishu.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from gyxx_flow.adapters import (
|
||||
ANALYZER_NOTIFICATION_APP_PROFILE,
|
||||
resolve_notification_route,
|
||||
send_lark_bot_message,
|
||||
)
|
||||
from gyxx_flow.adapters.data_interaction import query_points_exchange
|
||||
|
||||
WORKFLOW_ID = "product.points_exchange.weekly"
|
||||
DEFAULT_WINDOW_DAYS = 7
|
||||
MAX_WINDOW_DAYS = 92
|
||||
MESSAGE_CHUNK_SIZE = 40
|
||||
STDOUT_TOP_ROWS = 20
|
||||
SHANGHAI = ZoneInfo("Asia/Shanghai")
|
||||
_SAFE_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExchangeItem:
|
||||
"""One exchanged item aggregated across every SKU of that item."""
|
||||
|
||||
item_id: str
|
||||
title: str
|
||||
quantity: Decimal
|
||||
sku_count: int
|
||||
|
||||
|
||||
def _parse_date(value: str, field: str) -> date:
|
||||
try:
|
||||
parsed = date.fromisoformat(value.strip())
|
||||
except (AttributeError, TypeError, ValueError) as exc:
|
||||
raise ValueError(f"{field} must use the YYYY-MM-DD format: {value!r}") from exc
|
||||
if not _SAFE_DATE.fullmatch(value.strip()):
|
||||
raise ValueError(f"{field} must use the YYYY-MM-DD format: {value!r}")
|
||||
return parsed
|
||||
|
||||
|
||||
def _business_date() -> date:
|
||||
value = os.environ.get("GYXX_BUSINESS_DATE", "").strip()
|
||||
if value and _SAFE_DATE.fullmatch(value):
|
||||
return _parse_date(value, "GYXX_BUSINESS_DATE")
|
||||
return datetime.now(SHANGHAI).date()
|
||||
|
||||
|
||||
def _previous_full_week(today: date) -> tuple[date, date]:
|
||||
"""Return the Monday/Sunday pair of the last completed natural week."""
|
||||
|
||||
start = today - timedelta(days=today.weekday() + DEFAULT_WINDOW_DAYS)
|
||||
return start, start + timedelta(days=DEFAULT_WINDOW_DAYS - 1)
|
||||
|
||||
|
||||
def resolve_window(
|
||||
start_value: str | None,
|
||||
end_value: str | None,
|
||||
) -> tuple[date, date]:
|
||||
"""Resolve one closed date window, defaulting to last week's Mon-Sun."""
|
||||
|
||||
if start_value and end_value:
|
||||
start = _parse_date(start_value, "--from")
|
||||
end = _parse_date(end_value, "--to")
|
||||
elif start_value:
|
||||
start = _parse_date(start_value, "--from")
|
||||
end = start + timedelta(days=DEFAULT_WINDOW_DAYS - 1)
|
||||
elif end_value:
|
||||
end = _parse_date(end_value, "--to")
|
||||
start = end - timedelta(days=DEFAULT_WINDOW_DAYS - 1)
|
||||
else:
|
||||
start, end = _previous_full_week(_business_date())
|
||||
|
||||
if end < start:
|
||||
raise ValueError(f"window end {end} is before start {start}")
|
||||
span = (end - start).days + 1
|
||||
if span > MAX_WINDOW_DAYS:
|
||||
raise ValueError(
|
||||
f"window spans {span} days; the supported maximum is {MAX_WINDOW_DAYS}"
|
||||
)
|
||||
return start, end
|
||||
|
||||
|
||||
def _quantity(row: dict[str, object], index: int) -> Decimal:
|
||||
value = row.get("quantity")
|
||||
if value is None or isinstance(value, bool):
|
||||
raise ValueError(f"points-exchange row {index} has no quantity")
|
||||
try:
|
||||
quantity = Decimal(str(value).strip())
|
||||
except (InvalidOperation, ValueError) as exc:
|
||||
raise ValueError(f"points-exchange row {index} has invalid quantity") from exc
|
||||
if not quantity.is_finite():
|
||||
raise ValueError(f"points-exchange row {index} has non-finite quantity")
|
||||
return quantity
|
||||
|
||||
|
||||
def _text_field(row: dict[str, object], *names: str) -> str:
|
||||
for name in names:
|
||||
value = row.get(name)
|
||||
if value is not None and str(value).strip():
|
||||
return " ".join(str(value).split())
|
||||
return ""
|
||||
|
||||
|
||||
def aggregate_items(rows: list[dict[str, object]]) -> list[ExchangeItem]:
|
||||
"""Aggregate SKU rows into an item ranking ordered by quantity desc."""
|
||||
|
||||
titles: dict[str, dict[str, Decimal]] = {}
|
||||
sku_ids: dict[str, set[str]] = {}
|
||||
totals: dict[str, Decimal] = {}
|
||||
for index, row in enumerate(rows, start=1):
|
||||
quantity = _quantity(row, index)
|
||||
item_id = _text_field(row, "itemId", "item_id")
|
||||
title = _text_field(row, "title", "item_name", "name") or "未命名商品"
|
||||
sku = _text_field(row, "skuId", "sku_id", "skuCode", "sku_code")
|
||||
if not item_id:
|
||||
item_id = f"sku:{sku}" if sku else f"row:{index}"
|
||||
if quantity <= 0:
|
||||
continue
|
||||
totals[item_id] = totals.get(item_id, Decimal("0")) + quantity
|
||||
titles.setdefault(item_id, {})
|
||||
titles[item_id][title] = titles[item_id].get(title, Decimal("0")) + quantity
|
||||
sku_ids.setdefault(item_id, set())
|
||||
if sku:
|
||||
sku_ids[item_id].add(sku)
|
||||
|
||||
items = [
|
||||
ExchangeItem(
|
||||
item_id=item_id,
|
||||
title=max(titles[item_id].items(), key=lambda pair: (pair[1], pair[0]))[0],
|
||||
quantity=quantity,
|
||||
sku_count=len(sku_ids.get(item_id, ())),
|
||||
)
|
||||
for item_id, quantity in totals.items()
|
||||
]
|
||||
return sorted(items, key=lambda item: (-item.quantity, item.title, item.item_id))
|
||||
|
||||
|
||||
def _quantity_text(quantity: Decimal) -> str:
|
||||
return format(quantity.normalize(), "f")
|
||||
|
||||
|
||||
def _card(
|
||||
items: list[ExchangeItem],
|
||||
*,
|
||||
start: date,
|
||||
end: date,
|
||||
total_quantity: Decimal,
|
||||
item_count: int,
|
||||
page: int,
|
||||
pages: int,
|
||||
) -> dict[str, object]:
|
||||
summary = (
|
||||
f"**周期**:{start.isoformat()} ~ {end.isoformat()}\n"
|
||||
f"**兑换商品**:{item_count} 个 · **兑换总量**:{_quantity_text(total_quantity)}\n"
|
||||
)
|
||||
if items:
|
||||
top = items[0]
|
||||
summary += f"**TOP1**:{top.title} · {_quantity_text(top.quantity)}"
|
||||
if pages > 1:
|
||||
summary += f"\n第 {page}/{pages} 条,本条 {len(items)} 个商品"
|
||||
return {
|
||||
"schema": "2.0",
|
||||
"config": {"update_multi": True, "width_mode": "fill"},
|
||||
"header": {
|
||||
"title": {"tag": "plain_text", "content": "私域积分兑换周报"},
|
||||
"subtitle": {
|
||||
"tag": "plain_text",
|
||||
"content": "积分兑换物资消耗排行 · 按兑换数量从高到低",
|
||||
},
|
||||
"template": "blue",
|
||||
},
|
||||
"body": {
|
||||
"direction": "vertical",
|
||||
"padding": "12px 12px 20px 12px",
|
||||
"vertical_spacing": "8px",
|
||||
"elements": [
|
||||
{
|
||||
"tag": "markdown",
|
||||
"element_id": "summary",
|
||||
"content": summary,
|
||||
},
|
||||
{
|
||||
"tag": "table",
|
||||
"element_id": "exchange_ranking",
|
||||
"page_size": 10,
|
||||
"row_height": "auto",
|
||||
"row_max_height": "124px",
|
||||
"freeze_first_column": True,
|
||||
"columns": [
|
||||
{
|
||||
"name": "rank",
|
||||
"display_name": "排名",
|
||||
"data_type": "text",
|
||||
# Feishu rejects any column narrower than 80px.
|
||||
"width": "80px",
|
||||
"horizontal_align": "center",
|
||||
},
|
||||
{
|
||||
"name": "title",
|
||||
"display_name": "商品",
|
||||
"data_type": "text",
|
||||
"width": "auto",
|
||||
},
|
||||
{
|
||||
"name": "sku_count",
|
||||
"display_name": "SKU 数",
|
||||
"data_type": "text",
|
||||
"width": "80px",
|
||||
"horizontal_align": "right",
|
||||
},
|
||||
{
|
||||
"name": "quantity",
|
||||
"display_name": "兑换数量",
|
||||
"data_type": "text",
|
||||
"width": "88px",
|
||||
"horizontal_align": "right",
|
||||
},
|
||||
],
|
||||
"rows": [
|
||||
{
|
||||
"rank": str(rank),
|
||||
"title": item.title,
|
||||
"sku_count": str(item.sku_count),
|
||||
"quantity": _quantity_text(item.quantity),
|
||||
}
|
||||
for rank, item in enumerate(items, start=1)
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="汇总私域积分兑换商品的消耗数量排行并发送飞书周报。",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--from",
|
||||
dest="start_date",
|
||||
help="窗口起始日(YYYY-MM-DD),缺省取上一个完整自然周的周一",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--to",
|
||||
dest="end_date",
|
||||
help="窗口结束日(YYYY-MM-DD),缺省为起始日 +6 天",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="只打印排行,不发送飞书消息",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def run(argv: list[str] | None = None) -> int:
|
||||
arguments = _parse_args(argv)
|
||||
start, end = resolve_window(arguments.start_date, arguments.end_date)
|
||||
rows = query_points_exchange(
|
||||
start_time=f"{start.isoformat()} 00:00:00",
|
||||
end_time=f"{end.isoformat()} 23:59:59",
|
||||
)
|
||||
items = aggregate_items(rows)
|
||||
total_quantity = sum((item.quantity for item in items), Decimal("0"))
|
||||
print(
|
||||
f"积分兑换查询完成:{start.isoformat()} ~ {end.isoformat()},"
|
||||
f"明细 {len(rows)} 条,商品 {len(items)} 个,兑换总量 {_quantity_text(total_quantity)}"
|
||||
)
|
||||
for rank, item in enumerate(items[:STDOUT_TOP_ROWS], start=1):
|
||||
print(
|
||||
f"{rank:>3}. {item.title}\tSKU {item.sku_count}\t"
|
||||
f"{_quantity_text(item.quantity)}"
|
||||
)
|
||||
|
||||
if not items:
|
||||
print("该周期没有积分兑换记录,不发送飞书周报。")
|
||||
return 0
|
||||
if arguments.dry_run:
|
||||
print("dry-run:跳过飞书发送。")
|
||||
return 0
|
||||
|
||||
route = resolve_notification_route(WORKFLOW_ID, ())
|
||||
if not route.enabled:
|
||||
print("该工作流的飞书通知路由已停用。")
|
||||
return 0
|
||||
if not route.open_ids:
|
||||
raise RuntimeError("积分兑换周报通知路由没有配置接收人")
|
||||
|
||||
pages = (len(items) + MESSAGE_CHUNK_SIZE - 1) // MESSAGE_CHUNK_SIZE
|
||||
window_key = f"{start.isoformat()}~{end.isoformat()}"
|
||||
for page in range(pages):
|
||||
chunk = items[page * MESSAGE_CHUNK_SIZE : (page + 1) * MESSAGE_CHUNK_SIZE]
|
||||
card = _card(
|
||||
chunk,
|
||||
start=start,
|
||||
end=end,
|
||||
total_quantity=total_quantity,
|
||||
item_count=len(items),
|
||||
page=page + 1,
|
||||
pages=pages,
|
||||
)
|
||||
card_bytes = json.dumps(
|
||||
card, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||
).encode("utf-8")
|
||||
chunk_hash = hashlib.sha256(card_bytes).hexdigest()[:20]
|
||||
idempotency_key = f"{WORKFLOW_ID}:{window_key}:{page + 1}:{chunk_hash}"
|
||||
for open_id in route.open_ids:
|
||||
send_lark_bot_message(
|
||||
user_id=open_id,
|
||||
content=card,
|
||||
msg_type="interactive",
|
||||
idempotency_key=idempotency_key,
|
||||
profile=route.app_profile or ANALYZER_NOTIFICATION_APP_PROFILE,
|
||||
)
|
||||
print(f"积分兑换周报已发送:{len(items)} 个商品,{pages} 条飞书消息。")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
return run()
|
||||
except Exception as exc:
|
||||
print(f"积分兑换周报工作流失败:{exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -8,7 +8,8 @@
|
||||
|
||||
款式来源 (2026-06-27 改造):
|
||||
- --styles 显式传: 用 CLI 参数指定的款式列表
|
||||
- --styles 留空: 动态从项目数据库读 (get_main_image_styles: 天猫分组 + 主图飞书多维表格地址 同时存在)
|
||||
- --styles 留空: 动态从项目数据库读 (get_main_image_styles: 身份齐全的天猫款式)
|
||||
· 主图飞书表地址不参与采集门:缺地址的款照常抓图并入库,写表环节跳过
|
||||
|
||||
数据落点: data/主图信息/<date>/<款式>/wanxiang_creative_main_images.json
|
||||
插入幂等: 飞书按 (时间, 平台, 图片) 更新,PG 按日期/款式/平台/图片覆盖。
|
||||
@@ -112,7 +113,7 @@ def main() -> int:
|
||||
parser.add_argument("--skip-collect", action="store_true", help="跳过采集,只跑 insert")
|
||||
parser.add_argument("--skip-insert", action="store_true", help="只采集,不推飞书")
|
||||
parser.add_argument("--styles", default="",
|
||||
help="只采集/插入指定款式,逗号分隔 (传空=动态从项目数据库读天猫 + 主图飞书多维表格地址)")
|
||||
help="只采集/插入指定款式,逗号分隔 (传空=动态从项目数据库读身份齐全的天猫款式)")
|
||||
parser.add_argument("--headless", action="store_true",
|
||||
help="采集时用无头浏览器 (服务器/无人值守场景)")
|
||||
parser.add_argument("--max-pages", type=int, default=100,
|
||||
@@ -128,11 +129,11 @@ def main() -> int:
|
||||
target_date = last_sunday.strftime("%Y-%m-%d")
|
||||
py = sys.executable
|
||||
|
||||
# --styles 为空时动态从项目数据库读 (天猫分组 + 主图飞书多维表格地址 同时存在)
|
||||
# --styles 为空时动态从项目数据库读(身份齐全即可,主图地址不参与采集门)
|
||||
if not args.styles:
|
||||
bitable_styles = load_dynamic_styles()
|
||||
args.styles = ",".join(bitable_styles)
|
||||
print(f"[PREFLIGHT] 飞书多维表格动态读主图款式 {len(bitable_styles)} 款")
|
||||
print(f"[PREFLIGHT] 动态读主图款式 {len(bitable_styles)} 款(缺主图地址的款只入库不写表)")
|
||||
else:
|
||||
print(f"[PREFLIGHT] --styles 显式指定: {args.styles}")
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ from runtime_paths import LOG_ROOT, artifact_relative_path, managed_data_path
|
||||
# 复用 TM insert 脚本的工具函数
|
||||
from scripts.insert_main_image_records import (
|
||||
DEFAULT_DATA_ROOT,
|
||||
MAIN_IMAGE_MAP,
|
||||
batch_create_records,
|
||||
date_to_month_week,
|
||||
discover_fields,
|
||||
@@ -121,6 +120,11 @@ def process_style(style_key: str, cfg: dict, date_str: str, data_root: Path, dry
|
||||
row["_pg_record"] = pg_record
|
||||
pg_records.append(pg_record)
|
||||
|
||||
# 缺主图飞书表地址:照常入库,只是不写表(与商品经营日报缺销量表同口径)
|
||||
if not str(cfg.get("base_token") or "").strip() or not str(cfg.get("table_id") or "").strip():
|
||||
print(f" [SKIP-TABLE] {style_key}: 未配置主图飞书表地址,仅入库")
|
||||
return result
|
||||
|
||||
try:
|
||||
fields = discover_fields(cfg["base_token"], cfg["table_id"])
|
||||
except Exception as e:
|
||||
@@ -223,11 +227,14 @@ def main():
|
||||
try:
|
||||
config = get_bitable_main_image_map()
|
||||
if not config:
|
||||
raise RuntimeError("飞书动态配置为空")
|
||||
raise RuntimeError("dim_style 未返回任何带主图表地址的款式")
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 动态读取主图配置失败,使用本地文件回退: {exc}", file=sys.stderr)
|
||||
config = read_json(MAIN_IMAGE_MAP)
|
||||
config.pop("_comment", None)
|
||||
# 主图表地址唯一来源是 dim_style;读不到即失败,不再回退本地 JSON。
|
||||
print(
|
||||
f"[FAIL] 主图配置读取失败,拒绝回退本地 JSON: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
data_root = managed_data_path(args.data_root)
|
||||
date_dir = data_root / args.date
|
||||
@@ -262,9 +269,12 @@ def main():
|
||||
for style_dir_name in target_styles:
|
||||
key = normalize_key(style_dir_name)
|
||||
if key not in norm_config:
|
||||
print(f"[SKIP] {style_dir_name}: config 里找不到 (lowercase 匹配)")
|
||||
continue
|
||||
orig_key, cfg = norm_config[key]
|
||||
# 缺主图飞书表地址:照常采图入库,只是不写表
|
||||
orig_key = style_dir_name
|
||||
cfg = {"display_name": style_dir_name, "platform": "京东", "enabled": True}
|
||||
print(f"[NO-TABLE] {style_dir_name}: 未配置主图飞书表地址,仅入库")
|
||||
else:
|
||||
orig_key, cfg = norm_config[key]
|
||||
if not cfg.get("enabled", True):
|
||||
print(f"[SKIP] {orig_key}: enabled=false")
|
||||
continue
|
||||
|
||||
@@ -30,7 +30,6 @@ from runtime_paths import (
|
||||
managed_data_path,
|
||||
)
|
||||
|
||||
MAIN_IMAGE_MAP = PROJECT_ROOT / "bitable_main_image_map.json"
|
||||
DEFAULT_DATA_ROOT = RAW_DATA_ROOT / "主图信息"
|
||||
|
||||
LARK_CLI = (
|
||||
@@ -41,10 +40,14 @@ LARK_CLI = (
|
||||
|
||||
|
||||
def _print_preflight() -> None:
|
||||
"""启动期打印: 飞书表格里本周要插主图记录的款式数 (仅要求 主图飞书多维表格地址 非空)。"""
|
||||
"""启动期打印: 采集目标款数,以及其中能写飞书表的款数。"""
|
||||
try:
|
||||
styles = get_main_image_styles()
|
||||
print(f"[主图-insert] 飞书表格 周汇总目标 {len(styles)} 款 (天猫 + 主图飞书多维表格地址)")
|
||||
write_targets = get_bitable_main_image_map()
|
||||
print(
|
||||
f"[主图-insert] 采集目标 {len(styles)} 款;"
|
||||
f"可写飞书表 {len(write_targets)} 款(其余仅入库)"
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 飞书读取失败, 跳过 preflight: {exc}", file=sys.stderr)
|
||||
|
||||
@@ -492,6 +495,11 @@ def process_product(style_key: str, cfg: dict, date_str: str, data_root: Path, d
|
||||
row["_pg_record"] = pg_record
|
||||
pg_records.append(pg_record)
|
||||
|
||||
# 缺主图飞书表地址:照常入库,只是不写表(与商品经营日报缺销量表同口径)
|
||||
if not str(cfg.get("base_token") or "").strip() or not str(cfg.get("table_id") or "").strip():
|
||||
print(f" [SKIP-TABLE] {style_key}: 未配置主图飞书表地址,仅入库")
|
||||
return result
|
||||
|
||||
# discover field ids
|
||||
try:
|
||||
fields = discover_fields(cfg["base_token"], cfg["table_id"])
|
||||
@@ -598,11 +606,14 @@ def main():
|
||||
try:
|
||||
config = get_bitable_main_image_map()
|
||||
if not config:
|
||||
raise RuntimeError("飞书动态配置为空")
|
||||
raise RuntimeError("dim_style 未返回任何带主图表地址的款式")
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 动态读取主图表配置失败,使用本地文件回退: {exc}", file=sys.stderr)
|
||||
config = read_json(MAIN_IMAGE_MAP)
|
||||
config.pop("_comment", None)
|
||||
# 主图表地址唯一来源是 dim_style;读不到即失败,不再回退本地 JSON。
|
||||
print(
|
||||
f"[FAIL] 主图表配置读取失败,拒绝回退本地 JSON: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
data_root = managed_data_path(args.data_root)
|
||||
date_dir = data_root / args.date
|
||||
if not date_dir.exists():
|
||||
@@ -640,9 +651,12 @@ def main():
|
||||
for style_dir_name in target_styles:
|
||||
key = normalize_key(style_dir_name)
|
||||
if key not in norm_config:
|
||||
print(f"[SKIP] {style_dir_name}: config 里找不到 (lowercase 匹配)")
|
||||
continue
|
||||
orig_key, cfg = norm_config[key]
|
||||
# 缺主图飞书表地址:照常采图入库,只是不写表
|
||||
orig_key = style_dir_name
|
||||
cfg = {"display_name": style_dir_name, "platform": "天猫", "enabled": True}
|
||||
print(f"[NO-TABLE] {style_dir_name}: 未配置主图飞书表地址,仅入库")
|
||||
else:
|
||||
orig_key, cfg = norm_config[key]
|
||||
if not cfg.get("enabled", True):
|
||||
print(f"[SKIP] {orig_key}: enabled=false")
|
||||
continue
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
"""一次性迁移:把天猫广告指标从 ``product_daily_metrics.raw_data`` 搬到独立表。
|
||||
|
||||
背景:天猫广告指标原先写在 ``product_daily_metrics.raw_data.tm_ad_metrics``
|
||||
JSONB 节点里,与经营日报共用一行;经营数据重新同步时需要靠 JSONB 节点级合并
|
||||
才能保住投放数据。2026-11-01 起改为写 ``tm_product_ad_daily_metrics``(与京东、
|
||||
抖音同构),本脚本负责把历史节点回填进新表,并在校验通过后清除旧节点。
|
||||
|
||||
行为:
|
||||
- 默认 dry-run:只统计源数据规模、金额合计、款式匹配情况,不写库。
|
||||
- ``--execute``:先备份源节点到 JSON(``--backup-dir``,默认运行时 TMP_ROOT),
|
||||
再按 ``(stat_date, product_id)`` 幂等 upsert 到新表。
|
||||
- ``--clear-source``:仅在行数/金额校验一致、备份写盘成功后,才执行
|
||||
``raw_data - 'tm_ad_metrics'`` 清除节点;任何一项不通过都不清理并返回非零。
|
||||
|
||||
用法::
|
||||
|
||||
python scripts/migrate_tm_ad_metrics_to_table.py
|
||||
python scripts/migrate_tm_ad_metrics_to_table.py --execute
|
||||
python scripts/migrate_tm_ad_metrics_to_table.py --execute --clear-source
|
||||
|
||||
连接信息来自 PG_* 环境变量(与 db/__init__.py 同一套)。
|
||||
"""
|
||||
# ruff: noqa: E402
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
import sys
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
import db # noqa: E402
|
||||
from runtime_paths import TMP_ROOT # noqa: E402
|
||||
|
||||
SOURCE_PREDICATE = (
|
||||
"platform = 'tm' AND jsonb_typeof(raw_data -> 'tm_ad_metrics') = 'object'"
|
||||
)
|
||||
BATCH_SIZE = 500
|
||||
|
||||
|
||||
def _decimal(value: Any) -> Decimal:
|
||||
if value is None:
|
||||
return Decimal("0")
|
||||
text = str(value).strip().replace(",", "")
|
||||
if not text:
|
||||
return Decimal("0")
|
||||
try:
|
||||
return Decimal(text)
|
||||
except InvalidOperation:
|
||||
return Decimal("0")
|
||||
|
||||
|
||||
def _iter_source_rows(
|
||||
conn, date_from: date | None, date_to: date | None, batch_size: int
|
||||
) -> Iterator[list[dict[str, Any]]]:
|
||||
"""按 (stat_date, product_id) keyset 分页读源节点,避免一次性载入全部行。"""
|
||||
|
||||
clauses = [SOURCE_PREDICATE]
|
||||
params: list[Any] = []
|
||||
if date_from:
|
||||
clauses.append("stat_date >= %s")
|
||||
params.append(date_from)
|
||||
if date_to:
|
||||
clauses.append("stat_date <= %s")
|
||||
params.append(date_to)
|
||||
where = " AND ".join(clauses)
|
||||
|
||||
last: tuple[date, str] | None = None
|
||||
while True:
|
||||
page_where = where
|
||||
page_params = list(params)
|
||||
if last is not None:
|
||||
page_where += " AND (stat_date, product_id) > (%s, %s)"
|
||||
page_params.extend([last[0], last[1]])
|
||||
sql = f"""
|
||||
SELECT stat_date, product_id, product_name,
|
||||
raw_data -> 'tm_ad_metrics' AS node,
|
||||
source_file
|
||||
FROM product_daily_metrics
|
||||
WHERE {page_where}
|
||||
ORDER BY stat_date, product_id
|
||||
LIMIT {batch_size}
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, page_params)
|
||||
page = cur.fetchall()
|
||||
if not page:
|
||||
return
|
||||
yield [
|
||||
{
|
||||
"stat_date": row[0],
|
||||
"product_id": str(row[1]),
|
||||
"product_name": row[2],
|
||||
"node": row[3] or {},
|
||||
"source_file": row[4],
|
||||
}
|
||||
for row in page
|
||||
]
|
||||
last = (page[-1][0], str(page[-1][1]))
|
||||
if len(page) < batch_size:
|
||||
return
|
||||
|
||||
|
||||
def _to_records(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""把 JSONB 节点转成 upsert 函数接受的记录形态。"""
|
||||
|
||||
records = []
|
||||
for row in rows:
|
||||
node = dict(row["node"])
|
||||
metrics = {
|
||||
key: value
|
||||
for key, value in node.items()
|
||||
if key not in {"style_name", "style_match_method"}
|
||||
}
|
||||
records.append(
|
||||
{
|
||||
"stat_date": row["stat_date"],
|
||||
"product_id": row["product_id"],
|
||||
"product_name": row["product_name"],
|
||||
"ad_metrics": metrics,
|
||||
# 节点里已有的款式映射作为兜底;能按 tm_spus/名称重新匹配的以重算为准。
|
||||
"style_name": node.get("style_name"),
|
||||
"style_match_method": node.get("style_match_method"),
|
||||
"source_file": row["source_file"],
|
||||
}
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def _write_backup(rows: list[dict[str, Any]], backup_dir: Path) -> Path:
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
path = backup_dir / f"tm_ad_metrics_backup_{stamp}.json.gz"
|
||||
payload = [
|
||||
{
|
||||
"stat_date": row["stat_date"].isoformat(),
|
||||
"product_id": row["product_id"],
|
||||
"product_name": row["product_name"],
|
||||
"source_file": row["source_file"],
|
||||
"tm_ad_metrics": row["node"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
with gzip.open(path, "wt", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, default=str)
|
||||
return path
|
||||
|
||||
|
||||
def _target_totals(conn, date_from: date | None, date_to: date | None) -> tuple[int, Decimal]:
|
||||
clauses = ["TRUE"]
|
||||
params: list[Any] = []
|
||||
if date_from:
|
||||
clauses.append("stat_date >= %s")
|
||||
params.append(date_from)
|
||||
if date_to:
|
||||
clauses.append("stat_date <= %s")
|
||||
params.append(date_to)
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
SELECT COUNT(*), COALESCE(SUM(spend), 0)
|
||||
FROM tm_product_ad_daily_metrics
|
||||
WHERE {' AND '.join(clauses)}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
count, spend = cur.fetchone()
|
||||
return int(count), Decimal(str(spend))
|
||||
|
||||
|
||||
def _source_count(conn, date_from: date | None, date_to: date | None) -> int:
|
||||
clauses = [SOURCE_PREDICATE]
|
||||
params: list[Any] = []
|
||||
if date_from:
|
||||
clauses.append("stat_date >= %s")
|
||||
params.append(date_from)
|
||||
if date_to:
|
||||
clauses.append("stat_date <= %s")
|
||||
params.append(date_to)
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"SELECT COUNT(*) FROM product_daily_metrics WHERE {' AND '.join(clauses)}",
|
||||
params,
|
||||
)
|
||||
return int(cur.fetchone()[0])
|
||||
|
||||
|
||||
def _parse_date(value: str | None) -> date | None:
|
||||
return date.fromisoformat(value) if value else None
|
||||
|
||||
|
||||
def _missing_target_columns(conn) -> list[str]:
|
||||
"""dry-run 只读检查目标表是否已具备款式/指标列。"""
|
||||
|
||||
required = {"style_name", "style_match_method", "metrics", "source_row_number"}
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'tm_product_ad_daily_metrics'
|
||||
"""
|
||||
)
|
||||
existing = {row[0] for row in cur.fetchall()}
|
||||
if not existing:
|
||||
return sorted(required | {"<表不存在>"})
|
||||
return sorted(required - existing)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="迁移天猫广告指标到独立表")
|
||||
parser.add_argument("--date-from", help="只迁移该日期起 YYYY-MM-DD")
|
||||
parser.add_argument("--date-to", help="只迁移到该日期 YYYY-MM-DD")
|
||||
parser.add_argument("--execute", action="store_true", help="确认写入 PostgreSQL")
|
||||
parser.add_argument(
|
||||
"--clear-source",
|
||||
action="store_true",
|
||||
help="校验通过后清除 product_daily_metrics 里的旧 JSONB 节点",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backup-dir", type=Path, help="旧节点备份目录(默认运行时 TMP_ROOT)"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
date_from = _parse_date(args.date_from)
|
||||
date_to = _parse_date(args.date_to)
|
||||
backup_dir = args.backup_dir or TMP_ROOT
|
||||
|
||||
with db.get_conn() as conn:
|
||||
total_rows = _source_count(conn, date_from, date_to)
|
||||
if total_rows == 0:
|
||||
print("[INFO] 没有可迁移的天猫广告节点,退出")
|
||||
return 0
|
||||
|
||||
tally = {
|
||||
"rows": 0,
|
||||
"spend": Decimal("0"),
|
||||
"deal": Decimal("0"),
|
||||
"missing_style": 0,
|
||||
"unknown_style": 0,
|
||||
"min_date": None,
|
||||
"max_date": None,
|
||||
}
|
||||
all_rows: list[dict[str, Any]] = []
|
||||
resolved_style: dict[tuple[str, str], str | None] = {}
|
||||
|
||||
style_by_product_id, style_names = db._load_platform_style_catalog(conn, "tm_spus")
|
||||
for batch in _iter_source_rows(conn, date_from, date_to, BATCH_SIZE):
|
||||
all_rows.extend(batch)
|
||||
for record in _to_records(batch):
|
||||
node_style = str(record.get("style_name") or "").strip()
|
||||
if not node_style:
|
||||
tally["missing_style"] += 1
|
||||
key = (record["product_id"], str(record.get("product_name") or ""))
|
||||
if key not in resolved_style:
|
||||
resolved_style[key] = db._resolve_style_match(
|
||||
record, style_by_product_id, style_names
|
||||
)[0]
|
||||
if not (resolved_style[key] or node_style):
|
||||
tally["unknown_style"] += 1
|
||||
tally["rows"] += 1
|
||||
tally["spend"] += _decimal(record["ad_metrics"].get("spend"))
|
||||
tally["deal"] += _decimal(
|
||||
record["ad_metrics"].get("attributed_deal_amount")
|
||||
)
|
||||
stat_date = record["stat_date"]
|
||||
tally["min_date"] = (
|
||||
stat_date if tally["min_date"] is None else min(tally["min_date"], stat_date)
|
||||
)
|
||||
tally["max_date"] = (
|
||||
stat_date if tally["max_date"] is None else max(tally["max_date"], stat_date)
|
||||
)
|
||||
|
||||
print(
|
||||
f"[INFO] 源节点 {tally['rows']} 行({tally['min_date']} ~ {tally['max_date']}),"
|
||||
f"花费 {tally['spend']:.2f},广告归因成交 {tally['deal']:.2f}"
|
||||
)
|
||||
print(
|
||||
f"[INFO] 款式:源节点缺 style_name {tally['missing_style']} 行;"
|
||||
f"按 tm_spus/名称仍无法确定 {tally['unknown_style']} 行"
|
||||
)
|
||||
|
||||
if not args.execute:
|
||||
existing_count, _ = _target_totals(conn, date_from, date_to)
|
||||
missing = _missing_target_columns(conn)
|
||||
print(
|
||||
f"[INFO] 目标表现有 {existing_count} 行(同日期区间);"
|
||||
"DRY-RUN 未写库,确认后添加 --execute"
|
||||
)
|
||||
print(
|
||||
"[INFO] 目标表待补列:"
|
||||
+ (", ".join(missing) if missing else "无(结构已就绪)")
|
||||
)
|
||||
return 0
|
||||
|
||||
db.ensure_tmall_product_ad_daily_table(conn)
|
||||
|
||||
backup_path = _write_backup(all_rows, backup_dir)
|
||||
print(f"[OK] 旧节点已备份:{backup_path}")
|
||||
|
||||
written = 0
|
||||
for offset in range(0, len(all_rows), BATCH_SIZE):
|
||||
records = _to_records(all_rows[offset : offset + BATCH_SIZE])
|
||||
written += db.upsert_tmall_product_ad_daily_metrics(conn, records)
|
||||
print(f"[OK] 已写入 tm_product_ad_daily_metrics:{written} 行")
|
||||
|
||||
target_count, target_spend = _target_totals(conn, date_from, date_to)
|
||||
ok = target_count == tally["rows"] and target_spend == tally["spend"]
|
||||
print(
|
||||
f"[CHECK] 行数 {target_count}/{tally['rows']},"
|
||||
f"花费 {target_spend:.2f}/{tally['spend']:.2f} -> "
|
||||
f"{'一致' if ok else '不一致'}"
|
||||
)
|
||||
if not ok:
|
||||
print("[FAIL] 校验不一致,保留旧 JSONB 节点,请人工核对后重跑", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if not args.clear_source:
|
||||
print("[INFO] 未指定 --clear-source,旧节点保留")
|
||||
return 0
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE product_daily_metrics
|
||||
SET raw_data = COALESCE(raw_data, '{{}}'::jsonb) - 'tm_ad_metrics'
|
||||
WHERE {SOURCE_PREDICATE}
|
||||
{"AND stat_date >= %s" if date_from else ""}
|
||||
{"AND stat_date <= %s" if date_to else ""}
|
||||
""",
|
||||
[value for value in (date_from, date_to) if value is not None],
|
||||
)
|
||||
cleared = cur.rowcount
|
||||
remaining = _source_count(conn, date_from, date_to)
|
||||
print(f"[OK] 已清除旧节点:{cleared} 行,剩余 {remaining} 行")
|
||||
if remaining:
|
||||
print("[FAIL] 仍有残留节点,请重跑 --clear-source", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -23,10 +23,7 @@ try:
|
||||
collect_monthly_erp_metrics,
|
||||
monthly_report_interval,
|
||||
)
|
||||
from .config.style_config_loader import (
|
||||
StyleConfigLoader,
|
||||
has_complete_style_identity,
|
||||
)
|
||||
from .config.style_config_loader import StyleConfigLoader
|
||||
from .lark_cli_runtime import run_lark_cli
|
||||
except ImportError: # Direct script execution from the module root.
|
||||
from collect_erp_monthly_metrics import (
|
||||
@@ -35,10 +32,7 @@ except ImportError: # Direct script execution from the module root.
|
||||
collect_monthly_erp_metrics,
|
||||
monthly_report_interval,
|
||||
)
|
||||
from config.style_config_loader import (
|
||||
StyleConfigLoader,
|
||||
has_complete_style_identity,
|
||||
)
|
||||
from config.style_config_loader import StyleConfigLoader
|
||||
from lark_cli_runtime import run_lark_cli
|
||||
|
||||
_MARKDOWN_LINK = re.compile(r"^\s*\[([^]]+)]\([^)]*\)\s*$")
|
||||
@@ -324,13 +318,20 @@ def plan_erp_styles(
|
||||
missing.append(target.product_name)
|
||||
continue
|
||||
style_name, style_config = resolved
|
||||
raw_codes = style_config.get("erp_codes") or ()
|
||||
# 月度表按款明细:取款式建档时自己填的编码(series.own_erp_codes)。
|
||||
# loader 把整条系列的编码并集挂到了锚点的 erp_codes 上,那是给采集端
|
||||
# 当唯一取数单元用的;报表若照搬,同系列每个成员行都会把并集查一遍,
|
||||
# 销量被重复计入。锚点自身没填编码时这里为空,按 0 写入即可。
|
||||
series = style_config.get("series") or {}
|
||||
raw_codes = series.get("own_erp_codes")
|
||||
if raw_codes is None:
|
||||
raw_codes = style_config.get("erp_codes") or ()
|
||||
codes = tuple(
|
||||
dict.fromkeys(
|
||||
str(code).strip() for code in raw_codes if str(code).strip()
|
||||
)
|
||||
)
|
||||
if not codes or not has_complete_style_identity(style_config):
|
||||
if not codes:
|
||||
missing.append(target.product_name)
|
||||
continue
|
||||
key = normalized_name(style_name)
|
||||
|
||||
@@ -174,7 +174,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
daily = get_daily_styles()
|
||||
tm_collectable = daily["tm"]["collectable"]
|
||||
tm_skipped = daily["tm"]["skipped"]
|
||||
print(f"[TM] 飞书表格 待采 {len(tm_collectable)} 款, 跳过 {len(tm_skipped)} 款 (款式级缺 erp/sales,或 TM 缺商品链接ID)")
|
||||
print(f"[TM] 飞书表格 待采 {len(tm_collectable)} 款, 跳过 {len(tm_skipped)} 款 (款式级缺 erp 或已由同系列归集,或 TM 缺商品链接ID)")
|
||||
for s in tm_skipped:
|
||||
print(f" - {s['style']}: {s['reason']}")
|
||||
brands: dict[str, list[str]] = defaultdict(list)
|
||||
|
||||
@@ -26,7 +26,6 @@ from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from config.style_config_loader import (
|
||||
get_bitable_main_image_map,
|
||||
get_main_image_styles,
|
||||
get_tm_style_id_map,
|
||||
)
|
||||
@@ -87,12 +86,29 @@ _STATIC_STYLE_NAMES = [
|
||||
]
|
||||
|
||||
|
||||
# 万相「主体名称/ID」搜索词覆盖(2026-10-30):
|
||||
# 款式身份仍然只用 style_name —— 输出目录、飞书主图表、PG 记录都以款名为键;
|
||||
# 只有搜索框里填的词可以换,用于投放主体名与内部款名不一致的款。
|
||||
# 主搜索词无结果时按「原始款名 → 天猫商品ID」逐级兜底,不会因为改名少采集。
|
||||
STYLE_SEARCH_KEYWORDS = {
|
||||
"黑曜石吐司包": "黑曜石吐司",
|
||||
}
|
||||
|
||||
|
||||
def search_keywords_for(style_name: str) -> tuple[str, ...]:
|
||||
"""返回按优先级排列的万相搜索词;首个元素是主搜索词。"""
|
||||
keyword = STYLE_SEARCH_KEYWORDS.get(style_name, "").strip()
|
||||
if not keyword or keyword == style_name:
|
||||
return (style_name,)
|
||||
return (keyword, style_name)
|
||||
|
||||
|
||||
def _get_style_names() -> list[str]:
|
||||
"""优先从飞书读取有主图表配置的款式列表,失败则回退硬编码。"""
|
||||
"""采集目标=身份齐全的天猫款式;主图飞书表地址只影响写表,不影响采集。"""
|
||||
try:
|
||||
cfg = get_bitable_main_image_map()
|
||||
cfg = get_main_image_styles()
|
||||
if cfg:
|
||||
return sorted(cfg.keys())
|
||||
return sorted(cfg)
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 动态读取主图款式列表失败,使用硬编码回退: {exc}")
|
||||
return list(_STATIC_STYLE_NAMES)
|
||||
@@ -1657,17 +1673,37 @@ def aggregate_rows(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
return aggregated
|
||||
|
||||
|
||||
def collect_style(page, style_name: str, max_pages: int, item_ids: list[str] | None = None) -> dict[str, Any]:
|
||||
search_style_by_subject(page, style_name)
|
||||
def collect_style(
|
||||
page,
|
||||
style_name: str,
|
||||
max_pages: int,
|
||||
item_ids: list[str] | None = None,
|
||||
search_keywords: tuple[str, ...] | list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
# 搜索词可与款名不同(见 STYLE_SEARCH_KEYWORDS),但 style_name 始终是结果身份,
|
||||
# 输出目录/飞书/PG 都用它,不受搜索词影响。
|
||||
keywords = tuple(search_keywords) if search_keywords else search_keywords_for(style_name)
|
||||
used_keyword = keywords[0]
|
||||
if used_keyword != style_name:
|
||||
print(f" [SEARCH] 搜索词覆盖: {style_name} -> {used_keyword}")
|
||||
search_style_by_subject(page, used_keyword)
|
||||
all_rows: list[dict[str, Any]] = []
|
||||
visited_signatures: set[str] = set()
|
||||
|
||||
# 先检查款式名搜索是否有结果,无结果时用商品ID回退
|
||||
# 先检查主搜索词是否有结果,无结果时按「原始款名 → 商品ID」逐级回退
|
||||
first_check = extract_current_page_rows(page)
|
||||
for fallback in keywords[1:]:
|
||||
if first_check:
|
||||
break
|
||||
print(f" [FALLBACK] 搜索词“{used_keyword}”无结果,改用“{fallback}”重搜")
|
||||
search_style_by_subject(page, fallback)
|
||||
used_keyword = fallback
|
||||
first_check = extract_current_page_rows(page)
|
||||
if not first_check and item_ids:
|
||||
first_id = item_ids[0]
|
||||
print(f" [FALLBACK] 款式名搜不到结果,用商品ID {first_id} 重搜")
|
||||
search_style_by_subject(page, first_id)
|
||||
used_keyword = first_id
|
||||
|
||||
for page_no in range(1, max_pages + 1):
|
||||
image_map = collect_current_page_image_map(page)
|
||||
@@ -1693,6 +1729,7 @@ def collect_style(page, style_name: str, max_pages: int, item_ids: list[str] | N
|
||||
aggregated = aggregate_rows(all_rows)
|
||||
return {
|
||||
"style_name": style_name,
|
||||
"search_keyword": used_keyword,
|
||||
"collected_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"page_count": len(visited_signatures),
|
||||
"raw_row_count": len(all_rows),
|
||||
@@ -1732,6 +1769,7 @@ def write_style_outputs(result: dict[str, Any], output_dir: Path) -> tuple[Path,
|
||||
f"# {style_name} 万相创意主图汇总",
|
||||
"",
|
||||
f"- 采集时间: {result.get('collected_at', '')}",
|
||||
f"- 搜索词: {result.get('search_keyword') or style_name}",
|
||||
f"- 翻页数: {result.get('page_count', 0)}",
|
||||
f"- 原始行数: {result.get('raw_row_count', 0)}",
|
||||
f"- 聚合主图数: {result.get('creative_count', 0)}",
|
||||
|
||||
@@ -114,6 +114,10 @@ LUGGAGE_USER_DATA_DIR = os.getenv(
|
||||
"GUANGHE_LUGGAGE_USER_DATA_DIR",
|
||||
str(profile_path("guanghe-luggage")),
|
||||
)
|
||||
NEW_BAG_USER_DATA_DIR = os.getenv(
|
||||
"GUANGHE_NEW_BAG_USER_DATA_DIR",
|
||||
str(profile_path("guanghe-new-bag")),
|
||||
)
|
||||
|
||||
# ─────────────────────────── 配置 ───────────────────────────
|
||||
BASE_TOKEN = "TH1JbxhfUaxetis6F4pcoqvWncd"
|
||||
@@ -134,10 +138,24 @@ TABLE_FIELDS = [
|
||||
|
||||
STORE_FLAGSHIP = "flagship"
|
||||
STORE_LUGGAGE = "luggage"
|
||||
# 「天猫新店」(淘宝账号 gyxx箱包旗舰店:龙虾仔)只接黑曜石系列,视频源与其他店铺同一张飞书表。
|
||||
STORE_NEW_BAG = "new_bag"
|
||||
STORE_ALL = "all"
|
||||
STORE_ACCOUNT_KEYS = {
|
||||
STORE_FLAGSHIP: "guanghe-flagship",
|
||||
STORE_LUGGAGE: "guanghe-luggage",
|
||||
STORE_NEW_BAG: "guanghe-new-bag",
|
||||
}
|
||||
# 每个店铺独立的运行时凭据前缀与浏览器 Profile 目录环境变量;新增店铺只需在此登记。
|
||||
STORE_CREDENTIAL_PREFIXES = {
|
||||
STORE_FLAGSHIP: "GUANGHE",
|
||||
STORE_LUGGAGE: "GUANGHE_LUGGAGE",
|
||||
STORE_NEW_BAG: "GUANGHE_NEW_BAG",
|
||||
}
|
||||
# 天猫新店没有独立的款式白名单表,按「代拍产品」里的系列关键词过滤:
|
||||
# 一条记录的全部款式都必须落在该系列内才会上传,避免把非黑曜石视频发到新店。
|
||||
STORE_STYLE_KEYWORDS = {
|
||||
STORE_NEW_BAG: "黑曜石",
|
||||
}
|
||||
# Guanghe exposes a creator account, not the merchant-store name used by the
|
||||
# source routing and publication ledger. The flagship creator card currently
|
||||
@@ -147,6 +165,9 @@ STORE_ACCOUNT_KEYS = {
|
||||
# the platform no longer renders the merchant-store name on its home page.
|
||||
STORE_CREATOR_IDENTITY_MARKERS: dict[str, tuple[str, ...]] = {
|
||||
STORE_FLAGSHIP: ("光影行星天猫官方", "逛逛号:3301283916"),
|
||||
# 天猫新店的逛逛内容号由店铺资料自动生成,创作者后台只显示这个名字,
|
||||
# 淘宝登录名 gyxx箱包旗舰店:龙虾仔 不会出现在光合页面上。
|
||||
STORE_NEW_BAG: ("GYXX箱包天猫官方",),
|
||||
}
|
||||
|
||||
EXIT_OK = 0
|
||||
@@ -2630,24 +2651,13 @@ class GuangHeUploader:
|
||||
# 一次性打开弹窗,复用同一搜索框
|
||||
self._click_by_text("添加商品", timeout=10_000)
|
||||
print("[OK] 已点开关联商品弹窗")
|
||||
time.sleep(1.5)
|
||||
|
||||
# 搜索框(占位符含「搜索」)
|
||||
search = None
|
||||
for sel in [
|
||||
"input[placeholder*='搜索']",
|
||||
"input[type='search']",
|
||||
"input[placeholder*='商品']",
|
||||
]:
|
||||
try:
|
||||
loc = self._loc(sel).first
|
||||
if loc.count() > 0:
|
||||
search = loc
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
# 天猫新店的商品选择弹窗渲染比旗舰店慢,且占位文案不固定(有的店铺是
|
||||
# 「请输入商品ID/名称」),因此轮询多个候选选择器,超时后再打诊断。
|
||||
search = self._find_dialog_search_input(timeout_seconds=12)
|
||||
if search is None:
|
||||
print("[ERR] 关联商品弹窗内找不到搜索框 → 跳过本次发布")
|
||||
self._dump_dialog_diagnostics()
|
||||
for label in ["取消", "关 闭", "关闭"]:
|
||||
try:
|
||||
self._click_by_text(label, timeout=2000)
|
||||
@@ -3183,6 +3193,75 @@ class GuangHeUploader:
|
||||
print("[WARN] 未明确检测到发布成功提示,请人工确认")
|
||||
return published
|
||||
|
||||
DIALOG_SEARCH_SELECTORS = (
|
||||
"input[placeholder*='搜索']",
|
||||
"input[placeholder*='请输入']",
|
||||
"input[placeholder*='商品']",
|
||||
"input[type='search']",
|
||||
".next-dialog input[type='text']",
|
||||
".next-dialog input:not([type])",
|
||||
)
|
||||
|
||||
def _dialog_contexts(self) -> list:
|
||||
"""发布页 iframe、主页面以及其中所有子 frame,弹窗可能落在任意一层。"""
|
||||
contexts: list = []
|
||||
if self.pub_frame is not None:
|
||||
contexts.append(self.pub_frame)
|
||||
contexts.append(self.page)
|
||||
try:
|
||||
for frame in self.page.frames:
|
||||
if frame not in contexts:
|
||||
contexts.append(frame)
|
||||
except Exception:
|
||||
pass
|
||||
return contexts
|
||||
|
||||
def _find_dialog_search_input(self, timeout_seconds: float = 12):
|
||||
"""轮询各 frame 找关联商品弹窗的搜索框,找不到返回 None。"""
|
||||
deadline = time.time() + timeout_seconds
|
||||
while True:
|
||||
for ctx in self._dialog_contexts():
|
||||
for selector in self.DIALOG_SEARCH_SELECTORS:
|
||||
try:
|
||||
loc = ctx.locator(selector).first
|
||||
if loc.count() > 0:
|
||||
return loc
|
||||
except Exception:
|
||||
continue
|
||||
if time.time() >= deadline:
|
||||
return None
|
||||
time.sleep(0.8)
|
||||
|
||||
def _dump_dialog_diagnostics(self) -> None:
|
||||
"""找不到搜索框时把弹窗真实结构写进日志,避免只能靠复现定位。"""
|
||||
for idx, ctx in enumerate(self._dialog_contexts()):
|
||||
try:
|
||||
url = str(getattr(ctx, "url", "") or "")
|
||||
except Exception:
|
||||
url = ""
|
||||
print(f" [诊断 ctx#{idx}] url={url[:120]}")
|
||||
try:
|
||||
text = ctx.locator("body").inner_text(timeout=2000)
|
||||
print(f" text={re.sub(r'\\s+', ' ', text)[:300]}")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
handles = ctx.locator("input").all()
|
||||
for handle in handles[:12]:
|
||||
try:
|
||||
placeholder = handle.get_attribute("placeholder", timeout=1000)
|
||||
input_type = handle.get_attribute("type", timeout=1000)
|
||||
class_name = handle.get_attribute("class", timeout=1000)
|
||||
print(
|
||||
f" input type={input_type} placeholder={placeholder} "
|
||||
f"class={class_name}"
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
self._snapshot("07_link_product_no_search")
|
||||
|
||||
def _click_by_text(self, text: str, timeout: int = 10_000):
|
||||
"""根据文本点击元素,支持逗号分隔多个候选文本。优先在 pub_frame(iframe)里找,fallback 到主页面。"""
|
||||
candidates = [t.strip() for t in text.split(",")]
|
||||
@@ -3255,8 +3334,10 @@ class GuangHeUploader:
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="将飞书 Base 视频上传到淘宝光合平台")
|
||||
runtime_store = os.getenv("GYXX_VIDEO_UPLOAD_STORE", "").strip() or STORE_FLAGSHIP
|
||||
if runtime_store not in {STORE_FLAGSHIP, STORE_LUGGAGE, STORE_ALL}:
|
||||
raise ValueError("GYXX_VIDEO_UPLOAD_STORE 仅支持 flagship、luggage 或 all")
|
||||
if runtime_store not in {STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG, STORE_ALL}:
|
||||
raise ValueError(
|
||||
"GYXX_VIDEO_UPLOAD_STORE 仅支持 flagship、luggage、new_bag 或 all"
|
||||
)
|
||||
p.add_argument("--limit", type=int, default=None, help="最多处理多少条记录")
|
||||
p.add_argument("--debug", action="store_true", help="启用调试截图")
|
||||
mode = p.add_mutually_exclusive_group()
|
||||
@@ -3305,9 +3386,12 @@ def parse_args() -> argparse.Namespace:
|
||||
)
|
||||
p.add_argument(
|
||||
"--store",
|
||||
choices=[STORE_FLAGSHIP, STORE_LUGGAGE, STORE_ALL],
|
||||
choices=[STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG, STORE_ALL],
|
||||
default=runtime_store,
|
||||
help="上传店铺:flagship=旗舰店,luggage=箱包店,all=顺序运行两家",
|
||||
help=(
|
||||
"上传店铺:flagship=旗舰店,luggage=箱包店,"
|
||||
"new_bag=天猫新店(只发黑曜石系列),all=顺序运行三家"
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--topic-keyword",
|
||||
@@ -3354,7 +3438,7 @@ def select_pending_records(
|
||||
republish_from_date: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply the explicit brand/gender/status route for one Guanghe store."""
|
||||
if store not in {STORE_FLAGSHIP, STORE_LUGGAGE}:
|
||||
if store not in {STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG}:
|
||||
raise ValueError(f"未知光合店铺: {store}")
|
||||
if store == STORE_LUGGAGE and not luggage_style_whitelist:
|
||||
raise ValueError("箱包店款式白名单未加载或为空")
|
||||
@@ -3363,6 +3447,7 @@ def select_pending_records(
|
||||
normalized_whitelist = {
|
||||
_normalize_style_name(name) for name in (luggage_style_whitelist or set())
|
||||
}
|
||||
style_keyword = _normalize_style_name(STORE_STYLE_KEYWORDS.get(store, ""))
|
||||
|
||||
result = []
|
||||
for rec in records:
|
||||
@@ -3390,6 +3475,23 @@ def select_pending_records(
|
||||
or not normalized_products.issubset(normalized_whitelist)
|
||||
):
|
||||
continue
|
||||
if store == STORE_NEW_BAG:
|
||||
# 天猫新店按系列关键词过滤:全部款式都必须属于黑曜石系列,
|
||||
# 组合款里只要掺了别的系列整条记录都不进新店。
|
||||
product_names = GuangHeUploader._parse_product_names(
|
||||
rec.get("代拍产品")
|
||||
)
|
||||
if "__SKIP__" in product_names:
|
||||
continue
|
||||
normalized_products = [
|
||||
_normalize_style_name(name)
|
||||
for name in product_names
|
||||
if name and name != "__SKIP__"
|
||||
]
|
||||
if not normalized_products or not all(
|
||||
style_keyword in name for name in normalized_products
|
||||
):
|
||||
continue
|
||||
if store == STORE_FLAGSHIP:
|
||||
if republish_from_date:
|
||||
record_date = str(rec.get("日期") or "")[:10]
|
||||
@@ -3680,6 +3782,13 @@ def _store_runtime(store: str) -> Tuple[str, str, str, str]:
|
||||
os.getenv("GUANGHE_LUGGAGE_PASSWORD", ""),
|
||||
LUGGAGE_USER_DATA_DIR,
|
||||
)
|
||||
if store == STORE_NEW_BAG:
|
||||
return (
|
||||
"gyxx箱包旗舰店:龙虾仔",
|
||||
os.getenv("GUANGHE_NEW_BAG_USERNAME", ""),
|
||||
os.getenv("GUANGHE_NEW_BAG_PASSWORD", ""),
|
||||
NEW_BAG_USER_DATA_DIR,
|
||||
)
|
||||
raise ValueError(f"未知光合店铺: {store}")
|
||||
|
||||
|
||||
@@ -3870,7 +3979,7 @@ def _run_store(
|
||||
print(f"[OK] {store_name} 没有可入账的视频附件")
|
||||
return EXIT_OK
|
||||
if not username or not password:
|
||||
prefix = "GUANGHE" if store == STORE_FLAGSHIP else "GUANGHE_LUGGAGE"
|
||||
prefix = STORE_CREDENTIAL_PREFIXES[store]
|
||||
raise RuntimeError(f"请在运行时配置 {prefix}_USERNAME / {prefix}_PASSWORD")
|
||||
if not direct_llm_configured():
|
||||
raise RuntimeError(
|
||||
@@ -4186,7 +4295,7 @@ def main() -> int:
|
||||
print(f"[OK] 读取到 {len(records)} 条记录")
|
||||
|
||||
stores = (
|
||||
[STORE_FLAGSHIP, STORE_LUGGAGE]
|
||||
[STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG]
|
||||
if args.store == STORE_ALL
|
||||
else [args.store]
|
||||
)
|
||||
|
||||
@@ -1882,12 +1882,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
default=None,
|
||||
help="单日业务日期 YYYY-MM-DD(申请时间筛选的开始日期);控制台会自动传入",
|
||||
)
|
||||
date_arguments.add_argument(
|
||||
parser.add_argument(
|
||||
"--range-days",
|
||||
type=int,
|
||||
default=0,
|
||||
help="采集 business_date 往前 N 天(含当天)的区间;定时任务使用,"
|
||||
"0 表示仅按 --date/--from/--to 采集",
|
||||
help="采集 business_date 往前 N 天(含当天)的区间;定时任务与控制台使用,"
|
||||
"需配合 --date 指定区间终点,0 表示仅按 --date/--from/--to 采集",
|
||||
)
|
||||
date_arguments.add_argument(
|
||||
"--from",
|
||||
|
||||
@@ -330,10 +330,13 @@ def _date_picker_input(page: Page) -> Any:
|
||||
def _open_date_picker(page: Page) -> Any:
|
||||
date_input = _date_picker_input(page)
|
||||
trigger = date_input.locator(".jmtd-base-input-top").first
|
||||
if trigger.count():
|
||||
trigger.click()
|
||||
else:
|
||||
date_input.click()
|
||||
target = trigger if trigger.count() else date_input
|
||||
try:
|
||||
target.click(timeout=5_000)
|
||||
except Exception:
|
||||
# 商智新版页面在时间控件上叠了透明遮罩,Playwright 的可点击性
|
||||
# 检查会一直等待;直接派发 DOM 点击,弹层同样正常展开。
|
||||
target.evaluate("node => node.click()")
|
||||
return _visible_date_picker(page)
|
||||
|
||||
|
||||
@@ -1342,13 +1345,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
default=None,
|
||||
help="单日业务日期 YYYY-MM-DD;定时任务使用",
|
||||
)
|
||||
date_arguments.add_argument(
|
||||
"--range-days",
|
||||
type=int,
|
||||
default=0,
|
||||
help="采集 business_date 往前 N 天(含当天)的区间;定时任务使用,"
|
||||
"0 表示仅按 --date/--from/--to 采集",
|
||||
)
|
||||
date_arguments.add_argument(
|
||||
"--from",
|
||||
dest="from_date",
|
||||
@@ -1356,6 +1352,13 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
default=None,
|
||||
help="自定义查询范围开始日期 YYYY-MM-DD",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--range-days",
|
||||
type=int,
|
||||
default=0,
|
||||
help="采集 business_date 往前 N 天(含当天)的区间;定时任务使用,"
|
||||
"0 表示仅按 --date/--from/--to 采集",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--to",
|
||||
dest="to_date",
|
||||
|
||||
@@ -334,6 +334,7 @@ def open_chat_log(page: Page) -> None:
|
||||
page.get_by_text("聊天记录", exact=True).first.wait_for(
|
||||
state="visible", timeout=30_000
|
||||
)
|
||||
_dismiss_notice_modal(page)
|
||||
|
||||
|
||||
def _visible_date_dropdown(page: Page) -> Any:
|
||||
@@ -379,17 +380,67 @@ def _select_date_cell(page: Page, target_date: date) -> Any:
|
||||
raise RuntimeError(f"日期面板无法导航到 {date_text}")
|
||||
|
||||
|
||||
def _dismiss_notice_modal(page: Page) -> None:
|
||||
"""关闭客服后台的公告弹窗,避免它遮挡日期与查询控件。"""
|
||||
|
||||
for _ in range(3):
|
||||
wrap = page.locator(
|
||||
".kf-manage-lite-modal-wrap.notice-modal-container:visible"
|
||||
).first
|
||||
try:
|
||||
if not wrap.count():
|
||||
return
|
||||
except Exception:
|
||||
return
|
||||
closed = False
|
||||
for target in (
|
||||
wrap.get_by_role("button", name="我已知晓"),
|
||||
wrap.locator("button.kf-manage-lite-modal-close").first,
|
||||
):
|
||||
try:
|
||||
if not target.count():
|
||||
continue
|
||||
target.click(timeout=5_000, force=True)
|
||||
page.wait_for_timeout(300)
|
||||
closed = True
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
if not closed:
|
||||
# 兜底:公告只是遮挡层,直接摘掉节点后继续。
|
||||
try:
|
||||
wrap.evaluate("node => node.remove()")
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(200)
|
||||
if not wrap.count():
|
||||
return
|
||||
|
||||
|
||||
def _pick_date(page: Page, target_date: date) -> None:
|
||||
"""选中目标自然日;双月面板重叠时退回 DOM 点击。"""
|
||||
|
||||
cell = _select_date_cell(page, target_date)
|
||||
try:
|
||||
cell.click(timeout=5_000)
|
||||
except Exception:
|
||||
# 京东双月面板的表格会互相遮挡,Playwright 始终判定被拦截;
|
||||
# 单元格本身可用,直接派发 DOM 点击即可选中。
|
||||
cell.evaluate("node => node.click()")
|
||||
|
||||
|
||||
def set_single_day(page: Page, target_date: date) -> None:
|
||||
"""将客服聊天记录的开始、结束时间设置为同一自然日。"""
|
||||
|
||||
_dismiss_notice_modal(page)
|
||||
start = page.locator('input[placeholder="开始日期"]').first
|
||||
end = page.locator('input[placeholder="结束日期"]').first
|
||||
start.click()
|
||||
_select_date_cell(page, target_date).click()
|
||||
_pick_date(page, target_date)
|
||||
page.wait_for_timeout(120)
|
||||
# 京东 RangePicker 需要显式切到结束输入框;同一天再选一次才会得到 23:59:59。
|
||||
end.click()
|
||||
_select_date_cell(page, target_date).click()
|
||||
_pick_date(page, target_date)
|
||||
page.wait_for_timeout(150)
|
||||
|
||||
dropdown = page.locator(".kf-manage-lite-picker-dropdown:visible").first
|
||||
@@ -1061,7 +1112,7 @@ def run(
|
||||
conversation_count, message_count = _persist_records(
|
||||
normalized, no_db=args.no_db
|
||||
)
|
||||
collected[customer_id] = {
|
||||
day_collected[customer_id] = {
|
||||
"session_count": normalized["session_count"],
|
||||
"message_count": normalized["message_count"],
|
||||
"filtered_message_count": normalized[
|
||||
@@ -1104,7 +1155,9 @@ def run(
|
||||
f"顾客序号 {index + 1}(共 {len(expected_ids)})连续 {args.retries} 次采集失败"
|
||||
) from last_error
|
||||
|
||||
unresolved = [item for item in expected_ids if item not in collected]
|
||||
unresolved = [
|
||||
item for item in expected_ids if item not in day_collected
|
||||
]
|
||||
if unresolved:
|
||||
save_progress(
|
||||
output_path,
|
||||
|
||||
@@ -173,27 +173,9 @@ def _identity_markers(
|
||||
return ("自营",) if store_key == "self-operated" else ("光影行星",)
|
||||
|
||||
|
||||
def open_and_verify_reviews(
|
||||
page: Page,
|
||||
*,
|
||||
store_key: str,
|
||||
environment: dict[str, str],
|
||||
) -> None:
|
||||
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000)
|
||||
_wait_for_page(page)
|
||||
page.wait_for_timeout(2_000)
|
||||
if "login" in page.url.lower():
|
||||
raise RuntimeError(f"登录未成功,评价页面跳转到了登录页: {page.url}")
|
||||
page.get_by_text("评价管理", exact=True).first.wait_for(
|
||||
state="visible", timeout=30_000
|
||||
)
|
||||
page.get_by_text("评价等级", exact=True).first.wait_for(
|
||||
state="visible", timeout=30_000
|
||||
)
|
||||
_hide_feedback_widgets(page)
|
||||
markers = _identity_markers(environment, store_key)
|
||||
if store_key == "self-operated":
|
||||
markers = (*markers, "光影自营", "自营")
|
||||
def _verify_shop_markers(page: Page, markers: tuple[str, ...]) -> None:
|
||||
"""确认当前登录店铺身份,避免把共享京东 Cookie 用到错误店铺。"""
|
||||
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
body_text = page.locator("body").inner_text(timeout=15_000)
|
||||
@@ -213,6 +195,36 @@ def open_and_verify_reviews(
|
||||
)
|
||||
|
||||
|
||||
def open_and_verify_reviews(
|
||||
page: Page,
|
||||
*,
|
||||
store_key: str,
|
||||
environment: dict[str, str],
|
||||
) -> None:
|
||||
markers = _identity_markers(environment, store_key)
|
||||
if store_key == "self-operated":
|
||||
markers = (*markers, "光影自营", "自营")
|
||||
# 评价后台独立页已不再展示店铺名,先在京麦首页确认店铺身份,
|
||||
# 再进入评价页校验列表锚点。
|
||||
page.goto(JD_SHOP_HOME_URL, wait_until="domcontentloaded", timeout=60_000)
|
||||
_wait_for_page(page)
|
||||
if "login" in page.url.lower():
|
||||
raise RuntimeError(f"登录未成功,京东商家后台跳转到了登录页: {page.url}")
|
||||
_verify_shop_markers(page, markers)
|
||||
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000)
|
||||
_wait_for_page(page)
|
||||
page.wait_for_timeout(2_000)
|
||||
if "login" in page.url.lower():
|
||||
raise RuntimeError(f"登录未成功,评价页面跳转到了登录页: {page.url}")
|
||||
page.get_by_text("评价列表", exact=True).first.wait_for(
|
||||
state="visible", timeout=30_000
|
||||
)
|
||||
page.get_by_text("评价等级", exact=True).first.wait_for(
|
||||
state="visible", timeout=30_000
|
||||
)
|
||||
_hide_feedback_widgets(page)
|
||||
|
||||
|
||||
def _visible_date_dialog(page: Page) -> Any:
|
||||
dialog = page.locator('[role="dialog"]:visible').last
|
||||
dialog.wait_for(state="visible", timeout=10_000)
|
||||
|
||||
+114
-40
@@ -117,6 +117,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
default=True,
|
||||
help="使用隐藏指纹浏览器(默认)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--headed",
|
||||
action="store_true",
|
||||
help="显示浏览器窗口;淘宝弹滑块验证时便于人工拖动后继续采集",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
@@ -254,64 +259,130 @@ def open_and_verify(page: Any) -> Any:
|
||||
return frame
|
||||
|
||||
|
||||
def _calendar_cell(picker: Any, target: date) -> Any | None:
|
||||
date_selector = f'td[title="{target.isoformat()}"]'
|
||||
current_month_cell = picker.locator(
|
||||
date_selector
|
||||
+ ":not(.next-calendar-cell-prev-month):not(.next-calendar-cell-next-month)"
|
||||
).first
|
||||
if current_month_cell.count() and current_month_cell.is_visible():
|
||||
return current_month_cell
|
||||
cell = picker.locator(date_selector).first
|
||||
if cell.count() and cell.is_visible():
|
||||
return cell
|
||||
return None
|
||||
CAPTCHA_HINTS = ("请拖动下方滑块", "请按住滑块")
|
||||
|
||||
|
||||
def _select_calendar_day(picker: Any, target: date) -> None:
|
||||
def _captcha_visible(page: Any) -> bool:
|
||||
try:
|
||||
return bool(
|
||||
page.evaluate(
|
||||
"""() => {
|
||||
const text = document.body ? (document.body.innerText || '') : '';
|
||||
return text.includes('请拖动下方滑块') || text.includes('请按住滑块');
|
||||
}"""
|
||||
)
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
|
||||
def _wait_for_captcha(page: Any, timeout_seconds: int = 300) -> None:
|
||||
"""淘宝在点开会话详情时会弹滑块验证;出现后等待人工拖动再继续。
|
||||
|
||||
没有这道检测时,验证弹窗只会让后续点击超时重试,整轮运行会长时间空转。
|
||||
"""
|
||||
|
||||
if not _captcha_visible(page):
|
||||
return
|
||||
print(
|
||||
" [WARN] 检测到淘宝滑块验证,请在浏览器窗口拖动滑块(最多等待 5 分钟)…",
|
||||
flush=True,
|
||||
)
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
while time.monotonic() < deadline:
|
||||
page.wait_for_timeout(2_000)
|
||||
if not _captcha_visible(page):
|
||||
print(" [OK] 滑块验证已通过,继续采集", flush=True)
|
||||
return
|
||||
raise RuntimeError("淘宝滑块验证未在 5 分钟内通过,请人工处理后重跑")
|
||||
|
||||
|
||||
def _date_input_values(frame: Any) -> list[str]:
|
||||
return frame.locator(
|
||||
f'{DATE_INPUTS["start"]}, {DATE_INPUTS["end"]}'
|
||||
).evaluate_all("elements => elements.map(element => element.value)")
|
||||
|
||||
|
||||
def _picker(frame: Any) -> Any:
|
||||
picker = frame.locator(".next-range-picker-body:visible").first
|
||||
picker.wait_for(state="visible", timeout=10_000)
|
||||
return picker
|
||||
|
||||
|
||||
def _pick_day(frame: Any, target: date) -> None:
|
||||
"""点选一个日期。
|
||||
|
||||
每次都重新定位面板并直接用 `td[title]` 取格:曾用
|
||||
`:not(.next-calendar-cell-prev-month):not(.next-calendar-cell-next-month)`
|
||||
过滤“当前月”版本,但该面板在双月视图下会取到非预期节点,导致第二次点击
|
||||
被当成新区间起点(区间被重置)。
|
||||
"""
|
||||
|
||||
iso = target.isoformat()
|
||||
wait = frame.page.wait_for_timeout
|
||||
for _ in range(MAX_CALENDAR_NAV_STEPS):
|
||||
cell = _calendar_cell(picker, target)
|
||||
if cell is not None:
|
||||
picker = _picker(frame)
|
||||
cell = picker.locator(f'td[title="{iso}"]').first
|
||||
if cell.count():
|
||||
classes = cell.get_attribute("class") or ""
|
||||
if "disabled" in classes.casefold():
|
||||
raise RuntimeError(f"天猫客服后台尚未开放日期 {target.isoformat()}")
|
||||
raise RuntimeError(f"天猫客服后台尚未开放日期 {iso}")
|
||||
cell.click()
|
||||
wait(700)
|
||||
return
|
||||
|
||||
visible_dates = picker.locator("td[title]").evaluate_all(
|
||||
titles = picker.locator("td[title]").evaluate_all(
|
||||
"elements => elements.map(element => element.title).filter(Boolean)"
|
||||
)
|
||||
parsed_dates: list[date] = []
|
||||
for value in visible_dates:
|
||||
parsed: list[date] = []
|
||||
for value in titles:
|
||||
try:
|
||||
parsed_dates.append(date.fromisoformat(value))
|
||||
parsed.append(date.fromisoformat(value))
|
||||
except ValueError:
|
||||
continue
|
||||
if not parsed_dates:
|
||||
if not parsed:
|
||||
raise RuntimeError("天猫客服日期选择器没有可用日期")
|
||||
if target < min(parsed_dates):
|
||||
if target < min(parsed):
|
||||
picker.locator(".next-calendar-btn-prev-month").first.click()
|
||||
elif target > max(parsed_dates):
|
||||
elif target > max(parsed):
|
||||
picker.locator(".next-calendar-btn-next-month").first.click()
|
||||
else:
|
||||
raise RuntimeError(f"日期选择器无法选择 {target.isoformat()}")
|
||||
picker.locator("td[title]").first.wait_for(state="visible", timeout=5_000)
|
||||
raise RuntimeError(f"日期选择器翻页次数超限: {target.isoformat()}")
|
||||
raise RuntimeError(f"日期选择器无法选择 {iso}")
|
||||
wait(400)
|
||||
raise RuntimeError(f"日期选择器翻页次数超限: {iso}")
|
||||
|
||||
|
||||
def set_date_range(frame: Any, start: date, end: date) -> None:
|
||||
frame.locator(DATE_INPUTS["start"]).click()
|
||||
picker = frame.locator(".next-range-picker-body:visible").first
|
||||
picker.wait_for(state="visible", timeout=10_000)
|
||||
_select_calendar_day(picker, start)
|
||||
_select_calendar_day(picker, end)
|
||||
picker.get_by_role("button", name="确定").click()
|
||||
actual = frame.locator(
|
||||
f'{DATE_INPUTS["start"]}, {DATE_INPUTS["end"]}'
|
||||
).evaluate_all("elements => elements.map(element => element.value)")
|
||||
"""设置聊天记录查询区间。
|
||||
|
||||
该 RangePicker 在选完开始日时会把结束日补成当前值:若结束日恰好已是目标
|
||||
日期,再点一次该日期等于重新开区间(开始被重置),所以只在结束日不符时
|
||||
才补点。面板点选后 DOM 会重渲染,因此每次点选都重新定位面板。带一次重试,
|
||||
避免面板状态残留导致的偶发不生效。
|
||||
"""
|
||||
|
||||
expected = [start.isoformat(), end.isoformat()]
|
||||
if actual != expected:
|
||||
raise RuntimeError("天猫客服日期范围未按预期设置")
|
||||
wait = frame.page.wait_for_timeout
|
||||
for _ in range(2):
|
||||
frame.locator(DATE_INPUTS["start"]).click()
|
||||
_picker(frame)
|
||||
# 面板展开有动画;动画期间点选会让组件把区间当成“重新开始”。
|
||||
wait(1_200)
|
||||
_pick_day(frame, start)
|
||||
wait(600)
|
||||
if _date_input_values(frame) != expected:
|
||||
_pick_day(frame, end)
|
||||
wait(600)
|
||||
picker = frame.locator(".next-range-picker-body:visible").first
|
||||
if picker.count():
|
||||
confirm = picker.get_by_role("button", name="确定")
|
||||
if confirm.count():
|
||||
confirm.first.click()
|
||||
wait(600)
|
||||
if _date_input_values(frame) == expected:
|
||||
return
|
||||
raise RuntimeError(
|
||||
f"天猫客服日期范围未按预期设置,期望 {expected},实际 {_date_input_values(frame)}"
|
||||
)
|
||||
|
||||
|
||||
def _query_window(frame: Any, start: date, end: date) -> list[Any]:
|
||||
@@ -866,7 +937,9 @@ def run(
|
||||
init_schema()
|
||||
|
||||
browser = ScraplingBrowser(
|
||||
**browser_runtime_options(environment, headless=args.headless)
|
||||
**browser_runtime_options(
|
||||
environment, headless=args.headless and not args.headed
|
||||
)
|
||||
)
|
||||
try:
|
||||
browser.start()
|
||||
@@ -893,6 +966,7 @@ def run(
|
||||
filtered_counts: Counter[str] = Counter()
|
||||
failures: list[dict[str, Any]] = []
|
||||
for row_index, row in enumerate(rows, start=1):
|
||||
_wait_for_captcha(page)
|
||||
try:
|
||||
conversation, conversation_messages = _capture_customer(
|
||||
frame,
|
||||
@@ -1029,7 +1103,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return 130
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"[ERROR] 天猫客服聊天记录采集失败: {type(exc).__name__}",
|
||||
f"[ERROR] 天猫客服聊天记录采集失败: {type(exc).__name__}: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
@@ -235,14 +235,83 @@ def _select_calendar_day(picker: Any, target: date) -> None:
|
||||
if not visible_dates:
|
||||
raise RuntimeError("天猫评价日期选择器没有可用日期")
|
||||
if target < min(visible_dates):
|
||||
picker.locator(".next-calendar-btn-prev-month").first.click()
|
||||
_shift_calendar_month(picker, "prev")
|
||||
elif target > max(visible_dates):
|
||||
picker.locator(".next-calendar-btn-next-month").first.click()
|
||||
_shift_calendar_month(picker, "next")
|
||||
else:
|
||||
raise RuntimeError(f"评价日期选择器无法选择 {target.isoformat()}")
|
||||
raise RuntimeError(f"评价日期选择器翻页次数超限: {target.isoformat()}")
|
||||
|
||||
|
||||
PICKER_SELECTORS = (
|
||||
".next-range-picker2-panel:visible",
|
||||
".next-range-picker-body:visible",
|
||||
".next-range-picker-panel:visible",
|
||||
)
|
||||
|
||||
PREV_MONTH_SELECTORS = (
|
||||
".next-calendar2-header-left-btn",
|
||||
".next-calendar-btn-prev-month",
|
||||
)
|
||||
|
||||
NEXT_MONTH_SELECTORS = (
|
||||
".next-calendar2-header-right-btn",
|
||||
".next-calendar-btn-next-month",
|
||||
)
|
||||
|
||||
|
||||
def _visible_picker(page: Any) -> Any | None:
|
||||
"""返回当前展开的日期面板;新版为 next-range-picker2-panel。"""
|
||||
|
||||
for selector in PICKER_SELECTORS:
|
||||
locator = page.locator(selector).first
|
||||
try:
|
||||
if locator.count() and locator.is_visible():
|
||||
return locator
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _shift_calendar_month(picker: Any, direction: str) -> None:
|
||||
selectors = PREV_MONTH_SELECTORS if direction == "prev" else NEXT_MONTH_SELECTORS
|
||||
for selector in selectors:
|
||||
button = picker.locator(selector).first
|
||||
try:
|
||||
if button.count() and button.is_visible():
|
||||
button.click()
|
||||
return
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
raise RuntimeError(f"天猫评价日期面板找不到{direction}翻月按钮")
|
||||
|
||||
|
||||
def _pick_range_on_calendar(page: Any, start: date, end: date) -> bool:
|
||||
"""优先用日历面板点选区间。
|
||||
|
||||
天猫新版 RangePicker 是受控组件:直接 `fill` 只改了 DOM 值,组件状态没变,
|
||||
查询仍按“全部日期”执行(会捞出区间外的评价)。因此必须点选单元格。
|
||||
"""
|
||||
|
||||
start_input = _date_input(page, "起始日期")
|
||||
start_input.click()
|
||||
picker = _visible_picker(page)
|
||||
if picker is None:
|
||||
return False
|
||||
_select_calendar_day(picker, start)
|
||||
page.wait_for_timeout(400)
|
||||
picker = _visible_picker(page)
|
||||
if picker is not None:
|
||||
_select_calendar_day(picker, end)
|
||||
page.wait_for_timeout(600)
|
||||
picker = _visible_picker(page)
|
||||
if picker is not None:
|
||||
confirm = picker.get_by_role("button", name="确定", exact=True)
|
||||
if confirm.count() and confirm.is_visible():
|
||||
confirm.click()
|
||||
return True
|
||||
|
||||
|
||||
def set_review_date_range(page: Any, start: date, end: date) -> None:
|
||||
"""显式设置评价时间;不得在空日期条件下点击搜索。"""
|
||||
|
||||
@@ -250,37 +319,73 @@ def set_review_date_range(page: Any, start: date, end: date) -> None:
|
||||
raise ValueError("评价结束日期不能早于开始日期")
|
||||
start_input = _date_input(page, "起始日期")
|
||||
end_input = _date_input(page, "结束日期")
|
||||
readonly = any(
|
||||
locator.get_attribute("readonly") is not None
|
||||
or locator.get_attribute("aria-readonly") == "true"
|
||||
for locator in (start_input, end_input)
|
||||
)
|
||||
if not readonly:
|
||||
|
||||
actual: list[str] = []
|
||||
picked = False
|
||||
try:
|
||||
picked = _pick_range_on_calendar(page, start, end)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if not is_browser_timeout_error(exc):
|
||||
raise
|
||||
if picked:
|
||||
actual = [start_input.input_value(), end_input.input_value()]
|
||||
|
||||
if actual != [start.isoformat(), end.isoformat()]:
|
||||
# 面板点选不可用时退回直接写入;随后的日期范围校验会把不一致暴露出来。
|
||||
try:
|
||||
start_input.fill(start.isoformat())
|
||||
end_input.fill(end.isoformat())
|
||||
end_input.press("Tab")
|
||||
except Exception as exc:
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if not is_browser_timeout_error(exc):
|
||||
# A date input can reject fill even without a readonly attribute;
|
||||
# use the platform calendar as the supported fallback.
|
||||
readonly = True
|
||||
if readonly:
|
||||
start_input.click()
|
||||
picker = page.locator(".next-range-picker-body:visible").first
|
||||
picker.wait_for(state="visible", timeout=10_000)
|
||||
_select_calendar_day(picker, start)
|
||||
_select_calendar_day(picker, end)
|
||||
confirm = picker.get_by_role("button", name="确定", exact=True)
|
||||
if confirm.count() and confirm.is_visible():
|
||||
confirm.click()
|
||||
raise
|
||||
actual = [start_input.input_value(), end_input.input_value()]
|
||||
|
||||
actual = [start_input.input_value(), end_input.input_value()]
|
||||
expected = [start.isoformat(), end.isoformat()]
|
||||
if actual != expected:
|
||||
raise RuntimeError(f"评价日期范围未生效,期望 {expected},实际 {actual}")
|
||||
|
||||
|
||||
def _dismiss_open_overlays(page: Any) -> None:
|
||||
"""关闭残留的 next-overlay 弹层。
|
||||
|
||||
日期选择器/下拉在部分分支会留下 `.next-overlay-backdrop`,它会拦截
|
||||
「搜索」按钮的指针事件,导致点击一直重试到超时。
|
||||
"""
|
||||
|
||||
for _ in range(3):
|
||||
try:
|
||||
if not page.locator(".next-overlay-wrapper.opened:visible").count():
|
||||
return
|
||||
except Exception: # noqa: BLE001
|
||||
return
|
||||
try:
|
||||
page.keyboard.press("Escape")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
page.wait_for_timeout(200)
|
||||
try:
|
||||
page.locator("body").click(position={"x": 4, "y": 4}, timeout=2_000)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
page.wait_for_timeout(200)
|
||||
|
||||
|
||||
def _click_search(page: Any) -> None:
|
||||
"""点击「搜索」;被弹层遮挡时退回 DOM 点击。"""
|
||||
|
||||
button = page.get_by_role("button", name="搜索", exact=True).first
|
||||
_dismiss_open_overlays(page)
|
||||
try:
|
||||
button.click(timeout=10_000)
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if not is_browser_timeout_error(exc):
|
||||
raise
|
||||
_dismiss_open_overlays(page)
|
||||
button.evaluate("element => element.click()")
|
||||
|
||||
|
||||
def clear_review_date_range(page: Any) -> None:
|
||||
"""只在显式 --all-time 时清空页面时间筛选,并确认两个日期为空。"""
|
||||
|
||||
@@ -687,7 +792,7 @@ def run(
|
||||
assert start_date is not None and end_date is not None
|
||||
set_review_date_range(page, start_date, end_date)
|
||||
_ensure_negative_filter(page)
|
||||
page.get_by_role("button", name="搜索", exact=True).first.click()
|
||||
_click_search(page)
|
||||
total, _initial_visible_count = _wait_for_results(page)
|
||||
expected_pages = math.ceil(total / PAGE_SIZE) if total else 0
|
||||
artifact["total_count"] = total
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
"""PostgreSQL queue access for replenishment arrival-date synchronization.
|
||||
|
||||
The ERP application writes one row per tenant/SKU to
|
||||
The ERP application writes one row per SKU to
|
||||
``erp_replenishment_arrival_sync``. This module owns only the queue boundary:
|
||||
claiming work and recording the result after the browser worker has finished.
|
||||
Browser actions stay in the workflow script so that a claimed row can be
|
||||
completed one at a time and a changed target date cannot be overwritten by an
|
||||
older worker.
|
||||
|
||||
data-hub 单公司化之后该表已没有 ``tenant_id``,但旧部署仍可能保留多租户结构,
|
||||
因此读写 SQL 会先探测实际列再决定是否带租户条件。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -45,18 +48,42 @@ ConnectionFactory = Callable[[], AbstractContextManager[Any]]
|
||||
|
||||
|
||||
class ArrivalSyncQueue:
|
||||
"""Claim and update arrival synchronization rows with optimistic fencing."""
|
||||
"""Claim and update arrival synchronization rows with optimistic fencing.
|
||||
|
||||
data-hub 在「单公司化」迁移(``sql/上线待执行/20261012_remove_tenant_id.sql``)
|
||||
之后删除了队列表的 ``tenant_id`` 列,而旧部署仍然是多租户表。这里按实际
|
||||
表结构自适应:有该列时继续按租户过滤,没有时退化为单公司口径。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
*,
|
||||
connect_factory: ConnectionFactory | None = None,
|
||||
tenant_column: bool | None = None,
|
||||
) -> None:
|
||||
self._environment = dict(os.environ if environment is None else environment)
|
||||
self._connect_factory = connect_factory or (
|
||||
lambda: _connect_from_environment(self._environment)
|
||||
)
|
||||
self._tenant_column = tenant_column
|
||||
|
||||
def _has_tenant_column(self) -> bool:
|
||||
"""Detect the queue schema once; tolerate the pre-migration layout."""
|
||||
|
||||
if self._tenant_column is None:
|
||||
sql = (
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_name = %s AND column_name = 'tenant_id'"
|
||||
)
|
||||
try:
|
||||
with self._connect_factory() as connection:
|
||||
with connection.cursor(row_factory=dict_row) as cursor:
|
||||
cursor.execute(sql, (QUEUE_TABLE,))
|
||||
self._tenant_column = cursor.fetchone() is not None
|
||||
except Exception: # noqa: BLE001 - fall back to the legacy layout
|
||||
self._tenant_column = True
|
||||
return bool(self._tenant_column)
|
||||
|
||||
def claim(
|
||||
self,
|
||||
@@ -68,12 +95,14 @@ class ArrivalSyncQueue:
|
||||
|
||||
_validate_positive("tenant_id", tenant_id)
|
||||
_validate_positive("stale_after_seconds", stale_after_seconds)
|
||||
tenant_scope = self._has_tenant_column()
|
||||
tenant_predicate = "tenant_id = %s AND " if tenant_scope else ""
|
||||
tenant_column = "queue.tenant_id, " if tenant_scope else ""
|
||||
sql = f"""
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM {QUEUE_TABLE}
|
||||
WHERE tenant_id = %s
|
||||
AND deleted = 0
|
||||
WHERE {tenant_predicate}deleted = 0
|
||||
AND jushuitan_updated = FALSE
|
||||
AND (
|
||||
sync_status IN ('pending', 'failed')
|
||||
@@ -95,25 +124,26 @@ class ArrivalSyncQueue:
|
||||
updater = %s
|
||||
FROM candidates
|
||||
WHERE queue.id = candidates.id
|
||||
RETURNING queue.id, queue.tenant_id, queue.sku_code,
|
||||
RETURNING queue.id, {tenant_column}queue.sku_code,
|
||||
queue.expected_arrival_time
|
||||
"""
|
||||
params: tuple[Any, ...] = (
|
||||
(tenant_id, stale_after_seconds, self._operator())
|
||||
if tenant_scope
|
||||
else (stale_after_seconds, self._operator())
|
||||
)
|
||||
with self._connect_factory() as connection:
|
||||
with connection.cursor(row_factory=dict_row) as cursor:
|
||||
cursor.execute(
|
||||
sql,
|
||||
(
|
||||
tenant_id,
|
||||
stale_after_seconds,
|
||||
self._operator(),
|
||||
),
|
||||
)
|
||||
cursor.execute(sql, params)
|
||||
rows = cursor.fetchall()
|
||||
return [_task_from_row(row) for row in rows]
|
||||
|
||||
def mark_success(self, task: ArrivalSyncTask) -> bool:
|
||||
"""Mark a task successful only if its target date is still current."""
|
||||
|
||||
tenant_predicate = (
|
||||
"\n AND tenant_id = %s" if self._has_tenant_column() else ""
|
||||
)
|
||||
sql = f"""
|
||||
UPDATE {QUEUE_TABLE}
|
||||
SET sync_status = 'success',
|
||||
@@ -122,8 +152,7 @@ class ArrivalSyncQueue:
|
||||
last_error = '',
|
||||
update_time = CURRENT_TIMESTAMP,
|
||||
updater = %s
|
||||
WHERE id = %s
|
||||
AND tenant_id = %s
|
||||
WHERE id = %s{tenant_predicate}
|
||||
AND sync_status = 'processing'
|
||||
AND expected_arrival_time IS NOT DISTINCT FROM %s
|
||||
RETURNING id
|
||||
@@ -134,6 +163,9 @@ class ArrivalSyncQueue:
|
||||
"""Record a retryable failure without clobbering a newer ERP target."""
|
||||
|
||||
normalized_error = " ".join(str(error or "同步失败").split())[:MAX_ERROR_LENGTH]
|
||||
tenant_predicate = (
|
||||
"\n AND tenant_id = %s" if self._has_tenant_column() else ""
|
||||
)
|
||||
sql = f"""
|
||||
UPDATE {QUEUE_TABLE}
|
||||
SET sync_status = 'failed',
|
||||
@@ -141,8 +173,7 @@ class ArrivalSyncQueue:
|
||||
last_error = %s,
|
||||
update_time = CURRENT_TIMESTAMP,
|
||||
updater = %s
|
||||
WHERE id = %s
|
||||
AND tenant_id = %s
|
||||
WHERE id = %s{tenant_predicate}
|
||||
AND sync_status = 'processing'
|
||||
AND expected_arrival_time IS NOT DISTINCT FROM %s
|
||||
RETURNING id
|
||||
@@ -159,12 +190,14 @@ class ArrivalSyncQueue:
|
||||
task: ArrivalSyncTask,
|
||||
prefix_params: tuple[Any, ...],
|
||||
) -> bool:
|
||||
fence_params: tuple[Any, ...] = (
|
||||
(task.id, task.tenant_id, task.expected_arrival_time)
|
||||
if self._has_tenant_column()
|
||||
else (task.id, task.expected_arrival_time)
|
||||
)
|
||||
with self._connect_factory() as connection:
|
||||
with connection.cursor(row_factory=dict_row) as cursor:
|
||||
cursor.execute(
|
||||
sql,
|
||||
(*prefix_params, task.id, task.tenant_id, task.expected_arrival_time),
|
||||
)
|
||||
cursor.execute(sql, (*prefix_params, *fence_params))
|
||||
return cursor.fetchone() is not None
|
||||
|
||||
def _operator(self) -> str:
|
||||
@@ -178,7 +211,8 @@ def _task_from_row(row: Mapping[str, Any]) -> ArrivalSyncTask:
|
||||
try:
|
||||
return ArrivalSyncTask(
|
||||
id=int(row["id"]),
|
||||
tenant_id=int(row["tenant_id"]),
|
||||
# 单公司化后队列表没有 tenant_id 列;任务里保留默认租户用于日志。
|
||||
tenant_id=int(row.get("tenant_id") or DEFAULT_TENANT_ID),
|
||||
sku_code=str(row["sku_code"]).strip(),
|
||||
expected_arrival_time=row["expected_arrival_time"],
|
||||
)
|
||||
|
||||
@@ -55,6 +55,7 @@ NOTIFICATION_CAPABILITIES: Mapping[str, str] = MappingProxyType(
|
||||
"product.alert.daily": "命中销量下滑规则且尚未重复通知时投递告警。",
|
||||
"product.market_rank": "存在本次新生成的平台报告链接且尚未重复通知时投递。",
|
||||
"product.online_stock_alert.daily": "线上商品当前库存低于 3 时向配置接收人发送库存预警。",
|
||||
"product.points_exchange.weekly": "每周汇总上一周积分兑换商品排行后向配置接收人发送周报。",
|
||||
"supply.purchase_confirmation.daily": "采购确认结果生成完成后投递最终业务通知。",
|
||||
"supply.replenishment.weekly": "补货处理完成后投递最终结果,包括无新增补货项的结果。",
|
||||
"supply.replenishment_alert.daily": "存在命中库存阈值的 SKU 时投递预警。",
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# 运维审计库(ops_* 表)
|
||||
|
||||
工作流执行历史与登录 cookie 遥测的 PostgreSQL 落库。本地 `var/runs/**/run.json`
|
||||
仍是权威源,这里做的是**跨机器可查、可长期留存的关系型镜像**。
|
||||
|
||||
## 为什么需要
|
||||
|
||||
- 本地 journal 只在部署机磁盘上:换机、扩容、清理后历史就断了。
|
||||
- `var/logs` 是文本,做不了「近 30 天哪些工作流失败最多」这类查询。
|
||||
- cookie 状态原先只存在 `keepalive_status.json`(只有最新值),无法回答
|
||||
「登录态到底能活几天」。
|
||||
|
||||
## 表与视图
|
||||
|
||||
| 对象 | 粒度 | 用途 |
|
||||
|---|---|---|
|
||||
| `ops_workflow_runs` | 一次运行一行 | 状态、模式、耗时、步骤计数、错误、journal 路径、主机 |
|
||||
| `ops_workflow_run_steps` | 一次尝试一行 | 含重试;`exit_code`、错误、耗时 |
|
||||
| `ops_account_cookie_events` | 一次观测一行 | 事件流;只存指纹,不存 cookie 明文 |
|
||||
| `ops_account_cookie_lifetimes`(视图) | 一份 cookie 一行 | 从首次出现到失效的寿命 |
|
||||
| `ops_account_cookie_stats`(视图) | 一个账号一行 | 平均 / 中位 / 最短 / 最长寿命 |
|
||||
|
||||
## 自动建表
|
||||
|
||||
首次写入时执行 `SCHEMA_SQL`(幂等 DDL),无需手工迁移。也可以用:
|
||||
|
||||
```bash
|
||||
uv run gyxx audit schema > 001_ops_audit.sql # 导出 DDL 交 DBA 预建
|
||||
psql "$GYXX_POSTGRES_DSN" -f 001_ops_audit.sql
|
||||
```
|
||||
|
||||
## 查询
|
||||
|
||||
```bash
|
||||
uv run gyxx audit runs --workflow content.metrics.self_operated.daily --limit 20
|
||||
uv run gyxx audit runs --status failed --limit 50
|
||||
uv run gyxx audit cookies --account douyin-shop --limit 50
|
||||
uv run gyxx audit cookie-stats # 各账号失效规律
|
||||
uv run gyxx audit cookie-stats --account douyin-shop # 单账号每次的寿命明细
|
||||
```
|
||||
|
||||
都支持 `--json`,方便接到面板或告警脚本。
|
||||
|
||||
直接 SQL 也一样:
|
||||
|
||||
```sql
|
||||
-- 最近 7 天失败最多的 10 个工作流
|
||||
SELECT workflow_id, COUNT(*) AS failures
|
||||
FROM ops_workflow_runs
|
||||
WHERE status = 'failed' AND started_at > NOW() - INTERVAL '7 days'
|
||||
GROUP BY workflow_id ORDER BY failures DESC LIMIT 10;
|
||||
|
||||
-- 每次失败卡在哪个步骤
|
||||
SELECT r.workflow_id, s.step_id, COUNT(*) AS n
|
||||
FROM ops_workflow_run_steps s
|
||||
JOIN ops_workflow_runs r USING (run_id)
|
||||
WHERE s.status = 'failed' AND r.started_at > NOW() - INTERVAL '7 days'
|
||||
GROUP BY 1, 2 ORDER BY n DESC;
|
||||
|
||||
-- 哪个账号的登录态最短命(预警阈值参考)
|
||||
SELECT account_id, sessions, expired_sessions,
|
||||
avg_lifetime_seconds / 86400.0 AS avg_days,
|
||||
min_lifetime_seconds / 86400.0 AS min_days,
|
||||
last_failed_at
|
||||
FROM ops_account_cookie_stats ORDER BY min_lifetime_seconds NULLS LAST;
|
||||
```
|
||||
|
||||
## 怎么读「失效规律」
|
||||
|
||||
一次 cookie 代由 `fingerprint` 标识(需求域名/名称/值哈希排序后取摘要,与文件顺序无关):
|
||||
|
||||
- `first_healthy_at` → `first_failed_at` = `lifetime_seconds`:**实际可用时长**
|
||||
- `healthy_to_failure_seconds`:最后一次探活成功到发现失效,反映**探测精度**
|
||||
- `declared_expires_at`:cookie 自带的最早过期时间
|
||||
|
||||
把 `lifetime_seconds` 和 `declared_expires_at` 对比,能区分三种情况:
|
||||
|
||||
1. 实际寿命接近声明寿命 → 平台按声明失效,可按 `declared_expires_at` 提前预警;
|
||||
2. 实际远短于声明 → 服务端提前吊销(常见于多端登录、风控);
|
||||
3. `declared_expires_at` 为空(session cookie)→ 只能靠观测,`min_lifetime_seconds`
|
||||
就是安全刷新周期的上界。
|
||||
|
||||
`fingerprint` 变化表示换了一份 cookie(重新登录),不算失效,也不会把两段寿命混在一起。
|
||||
|
||||
## 采集点
|
||||
|
||||
| 来源 | 触发 | `probe_source` |
|
||||
|---|---|---|
|
||||
| `CookieGate.check` | 每次工作流运行前的门禁判定 | `gate` |
|
||||
| `AccountKeepaliveManager` | 常驻调度器每 `keepalive_interval_seconds`(默认 600s) | `keepalive` |
|
||||
| `RunIndex.index_journal` | 每次运行开始/结束/异常 | — |
|
||||
|
||||
keepalive 是主要的精度来源:默认 10 分钟一次,同类状态一小时内只落一行,
|
||||
状态翻转立即落行,所以「首次失效时间」误差不超过一个探活周期。
|
||||
|
||||
## 容错
|
||||
|
||||
审计**永不阻断业务**:
|
||||
|
||||
- 数据库不可达 / 缺 DSN / 建表失败 → 写入返回 `False`,调用方继续执行;
|
||||
- 连续失败 3 次后熔断冷却(默认 300s),避免每个工作流都卡在连接超时;
|
||||
- 失败记录写到 `<GYXX_DATA_ROOT>/logs/ops-audit/audit-failures.log`。
|
||||
|
||||
## 开关
|
||||
|
||||
| 变量 | 默认 | 说明 |
|
||||
|---|---|---|
|
||||
| `GYXX_OPS_AUDIT` | 开 | 设 `off` 关闭全部审计写入 |
|
||||
| `GYXX_OPS_AUDIT_HOST` | 主机名 | 多机部署时写入手工标识 |
|
||||
| `GYXX_OPS_AUDIT_FAILURE_COOLDOWN` | 300 | 熔断冷却秒数 |
|
||||
|
||||
## 不存什么
|
||||
|
||||
- **cookie 值**:只存哈希指纹;排查具体 cookie 请走 vault 文件。
|
||||
- **日志正文**:`ops_workflow_runs` 只存 `journal_path` 指向本地文件;
|
||||
完整堆栈仍在 `var/logs/console-restarts/`。
|
||||
- **凭据**:DSN 只从 `GYXX_POSTGRES_DSN`(或 `PG_*`)读,不落表。
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Operational run indexing and durable outbox services."""
|
||||
"""Operational run indexing, audit telemetry and durable outbox services."""
|
||||
|
||||
from .audit import AuditStore, CookieSnapshot, summarize_cookies
|
||||
from .effects import (
|
||||
EffectAlreadyApplied,
|
||||
EffectClaim,
|
||||
@@ -10,6 +11,8 @@ from .index import RunIndex, RunRecord
|
||||
from .outbox import IdempotencyConflict, Outbox, OutboxMessage
|
||||
|
||||
__all__ = [
|
||||
"AuditStore",
|
||||
"CookieSnapshot",
|
||||
"IdempotencyConflict",
|
||||
"EffectAlreadyApplied",
|
||||
"EffectClaim",
|
||||
@@ -19,4 +22,5 @@ __all__ = [
|
||||
"OutboxMessage",
|
||||
"RunIndex",
|
||||
"RunRecord",
|
||||
"summarize_cookies",
|
||||
]
|
||||
@@ -0,0 +1,901 @@
|
||||
"""PostgreSQL-backed operational audit: workflow run history and cookie telemetry.
|
||||
|
||||
设计原则
|
||||
--------
|
||||
1. **可观测性不拖累业务**:审计写入永远不向调用方抛异常。数据库不可达时
|
||||
只记录一条 failure 日志并熔断冷却,工作流照常执行。
|
||||
2. **本地 journal 仍是权威源**:本模块是镜像/汇总,供跨机器查询与长期留存;
|
||||
`var/runs/**/run.json` 不会被替代。
|
||||
3. **不落明文凭据**:cookie 只存指纹(域名+名称+值哈希),值本身永不入库。
|
||||
|
||||
环境开关
|
||||
--------
|
||||
``GYXX_OPS_AUDIT=off`` 关闭审计(默认开启)
|
||||
``GYXX_OPS_AUDIT_HOST`` 覆盖主机标识(默认 socket.gethostname())
|
||||
``GYXX_OPS_AUDIT_FAILURE_COOLDOWN`` 熔断冷却秒数(默认 300)
|
||||
``GYXX_POSTGRES_DSN`` 与动态配置共用同一 DSN
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator, Mapping, Sequence
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
OPS_AUDIT_ENV = "GYXX_OPS_AUDIT"
|
||||
OPS_AUDIT_HOST_ENV = "GYXX_OPS_AUDIT_HOST"
|
||||
OPS_AUDIT_COOLDOWN_ENV = "GYXX_OPS_AUDIT_FAILURE_COOLDOWN"
|
||||
|
||||
_DEFAULT_COOLDOWN_SECONDS = 300
|
||||
_MAX_CONSECUTIVE_FAILURES = 3
|
||||
_MAX_TEXT_CHARS = 4000
|
||||
_SCHEMA_LOCK_KEY = "gyxx:ops-audit:migration"
|
||||
|
||||
HEALTHY_STATES = frozenset({"healthy"})
|
||||
FAILED_STATES = frozenset({"expired", "not_logged_in"})
|
||||
_COOKIE_STATES = frozenset(
|
||||
{"healthy", "expired", "not_logged_in", "probe_error", "busy", "disabled"}
|
||||
)
|
||||
|
||||
SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS ops_workflow_runs (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
workflow_id TEXT NOT NULL,
|
||||
business_date DATE NOT NULL,
|
||||
mode TEXT NOT NULL DEFAULT 'unknown',
|
||||
shadow BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
status TEXT NOT NULL,
|
||||
started_at TIMESTAMPTZ,
|
||||
ended_at TIMESTAMPTZ,
|
||||
duration_seconds INTEGER,
|
||||
error TEXT,
|
||||
step_total INTEGER NOT NULL DEFAULT 0,
|
||||
step_success INTEGER NOT NULL DEFAULT 0,
|
||||
step_failed INTEGER NOT NULL DEFAULT 0,
|
||||
step_skipped INTEGER NOT NULL DEFAULT 0,
|
||||
step_running INTEGER NOT NULL DEFAULT 0,
|
||||
journal_path TEXT NOT NULL DEFAULT '',
|
||||
host TEXT NOT NULL DEFAULT '',
|
||||
indexed_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ops_workflow_runs_workflow_idx
|
||||
ON ops_workflow_runs (workflow_id, started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS ops_workflow_runs_status_idx
|
||||
ON ops_workflow_runs (status, started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS ops_workflow_runs_business_date_idx
|
||||
ON ops_workflow_runs (business_date, workflow_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops_workflow_run_steps (
|
||||
run_id TEXT NOT NULL
|
||||
REFERENCES ops_workflow_runs (run_id) ON DELETE CASCADE,
|
||||
step_id TEXT NOT NULL,
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
status TEXT NOT NULL DEFAULT '',
|
||||
exit_code INTEGER,
|
||||
started_at TIMESTAMPTZ,
|
||||
ended_at TIMESTAMPTZ,
|
||||
duration_seconds INTEGER,
|
||||
error TEXT,
|
||||
PRIMARY KEY (run_id, step_id, attempt)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ops_workflow_run_steps_step_idx
|
||||
ON ops_workflow_run_steps (step_id, status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops_account_cookie_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
account_id TEXT NOT NULL,
|
||||
observed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
state TEXT NOT NULL,
|
||||
probe_status TEXT NOT NULL DEFAULT '',
|
||||
probe_source TEXT NOT NULL DEFAULT '',
|
||||
cookie_count INTEGER,
|
||||
required_count INTEGER,
|
||||
fingerprint TEXT NOT NULL DEFAULT '',
|
||||
declared_expires_at TIMESTAMPTZ,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
workflow_id TEXT NOT NULL DEFAULT '',
|
||||
run_id TEXT NOT NULL DEFAULT '',
|
||||
host TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ops_account_cookie_events_account_idx
|
||||
ON ops_account_cookie_events (account_id, observed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS ops_account_cookie_events_state_idx
|
||||
ON ops_account_cookie_events (state, observed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS ops_account_cookie_events_fingerprint_idx
|
||||
ON ops_account_cookie_events (account_id, fingerprint);
|
||||
|
||||
-- 一段「cookie 代」的生命周期:同一 fingerprint 从首次出现到失效。
|
||||
-- first_failed_at 只取最后一次 healthy 之后的失败,避免恢复后再次失败时口径前移。
|
||||
CREATE OR REPLACE VIEW ops_account_cookie_lifetimes AS
|
||||
WITH per_fingerprint AS (
|
||||
SELECT
|
||||
account_id,
|
||||
fingerprint,
|
||||
MIN(observed_at) AS first_seen_at,
|
||||
MIN(observed_at) FILTER (WHERE state = 'healthy') AS first_healthy_at,
|
||||
MAX(observed_at) FILTER (WHERE state = 'healthy') AS last_healthy_at,
|
||||
MAX(declared_expires_at) AS declared_expires_at
|
||||
FROM ops_account_cookie_events
|
||||
WHERE fingerprint <> ''
|
||||
GROUP BY account_id, fingerprint
|
||||
)
|
||||
SELECT
|
||||
p.account_id,
|
||||
p.fingerprint,
|
||||
p.first_seen_at,
|
||||
p.first_healthy_at,
|
||||
p.last_healthy_at,
|
||||
p.declared_expires_at,
|
||||
f.first_failed_at,
|
||||
(f.first_failed_at IS NOT NULL) AS expired,
|
||||
CASE
|
||||
WHEN p.first_healthy_at IS NOT NULL AND p.last_healthy_at IS NOT NULL
|
||||
THEN EXTRACT(EPOCH FROM (p.last_healthy_at - p.first_healthy_at))::BIGINT
|
||||
END AS healthy_span_seconds,
|
||||
CASE
|
||||
WHEN p.first_healthy_at IS NOT NULL AND f.first_failed_at IS NOT NULL
|
||||
THEN EXTRACT(EPOCH FROM (f.first_failed_at - p.first_healthy_at))::BIGINT
|
||||
END AS lifetime_seconds,
|
||||
CASE
|
||||
WHEN p.last_healthy_at IS NOT NULL AND f.first_failed_at IS NOT NULL
|
||||
THEN EXTRACT(EPOCH FROM (f.first_failed_at - p.last_healthy_at))::BIGINT
|
||||
END AS healthy_to_failure_seconds
|
||||
FROM per_fingerprint p
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT MIN(e.observed_at) AS first_failed_at
|
||||
FROM ops_account_cookie_events e
|
||||
WHERE e.account_id = p.account_id
|
||||
AND e.fingerprint = p.fingerprint
|
||||
AND e.state IN ('expired', 'not_logged_in')
|
||||
AND (p.last_healthy_at IS NULL OR e.observed_at > p.last_healthy_at)
|
||||
) f ON TRUE;
|
||||
|
||||
-- 按账号汇总的失效规律:平均/中位/最短/最长寿命,直接支撑「多久失效一次」。
|
||||
CREATE OR REPLACE VIEW ops_account_cookie_stats AS
|
||||
SELECT
|
||||
account_id,
|
||||
COUNT(*) AS sessions,
|
||||
COUNT(*) FILTER (WHERE expired) AS expired_sessions,
|
||||
AVG(lifetime_seconds)::BIGINT AS avg_lifetime_seconds,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY lifetime_seconds)::BIGINT
|
||||
AS median_lifetime_seconds,
|
||||
MIN(lifetime_seconds) AS min_lifetime_seconds,
|
||||
MAX(lifetime_seconds) AS max_lifetime_seconds,
|
||||
MAX(first_failed_at) AS last_failed_at,
|
||||
MAX(last_healthy_at) AS last_healthy_at
|
||||
FROM ops_account_cookie_lifetimes
|
||||
WHERE lifetime_seconds IS NOT NULL
|
||||
GROUP BY account_id;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops_audit_schema (
|
||||
schema_version INTEGER PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 纯函数:cookie 快照
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CookieSnapshot:
|
||||
"""一次静态检查得到的 cookie 画像,不含任何明文值。"""
|
||||
|
||||
cookie_count: int
|
||||
required_count: int
|
||||
fingerprint: str
|
||||
declared_expires_at: datetime | None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"cookie_count": self.cookie_count,
|
||||
"required_count": self.required_count,
|
||||
"fingerprint": self.fingerprint,
|
||||
"declared_expires_at": (
|
||||
self.declared_expires_at.isoformat()
|
||||
if self.declared_expires_at is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _domain_matches(domain: str, required_domains: Sequence[str]) -> bool:
|
||||
normalized = str(domain or "").lstrip(".").casefold()
|
||||
if not normalized:
|
||||
return False
|
||||
return any(
|
||||
normalized == required.casefold() or normalized.endswith("." + required.casefold())
|
||||
for required in required_domains
|
||||
)
|
||||
|
||||
|
||||
def _expires_at(cookie: Mapping[str, Any]) -> datetime | None:
|
||||
raw = cookie.get("expires")
|
||||
if raw is None or isinstance(raw, bool):
|
||||
return None
|
||||
try:
|
||||
value = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
# Playwright 用 -1 表示 session cookie;0 与负数都视为「不声明」。
|
||||
if value <= 0:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromtimestamp(value, tz=timezone.utc)
|
||||
except (OverflowError, OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def summarize_cookies(
|
||||
cookies: Sequence[Mapping[str, Any]],
|
||||
*,
|
||||
required_domains: Sequence[str] = (),
|
||||
required_names: Sequence[str] = (),
|
||||
) -> CookieSnapshot:
|
||||
"""Build a value-free snapshot: counts, fingerprint and earliest declared expiry.
|
||||
|
||||
指纹只看「必需 cookie」的域名/名称/值哈希,因此登录态真正换了一份 cookie
|
||||
时指纹会变,而无关 cookie 的增删不会造成抖动。
|
||||
"""
|
||||
|
||||
required_domains = tuple(item for item in required_domains if item)
|
||||
required_names = tuple(item for item in required_names if item)
|
||||
components: list[str] = []
|
||||
required_count = 0
|
||||
earliest: datetime | None = None
|
||||
|
||||
for cookie in cookies:
|
||||
if not isinstance(cookie, Mapping):
|
||||
continue
|
||||
name = str(cookie.get("name", ""))
|
||||
domain = str(cookie.get("domain", ""))
|
||||
is_required = bool(required_domains) and _domain_matches(domain, required_domains)
|
||||
if required_names:
|
||||
is_required = is_required and name in required_names
|
||||
elif not required_domains:
|
||||
is_required = True
|
||||
if not is_required:
|
||||
continue
|
||||
required_count += 1
|
||||
value = str(cookie.get("value", ""))
|
||||
value_hash = hashlib.sha256(value.encode("utf-8", errors="replace")).hexdigest()[:16]
|
||||
components.append(f"{domain.lstrip('.')}|{name}|{value_hash}")
|
||||
expiry = _expires_at(cookie)
|
||||
if expiry is not None and (earliest is None or expiry < earliest):
|
||||
earliest = expiry
|
||||
|
||||
# 浏览器写入 cookie 的顺序并不稳定;排序后哈希才能保证同一份登录态得到同一指纹。
|
||||
digest = hashlib.sha256()
|
||||
for component in sorted(components):
|
||||
digest.update(f"{component}\n".encode("utf-8"))
|
||||
|
||||
return CookieSnapshot(
|
||||
cookie_count=len(cookies),
|
||||
required_count=required_count,
|
||||
fingerprint=digest.hexdigest()[:32] if required_count else "",
|
||||
declared_expires_at=earliest,
|
||||
)
|
||||
|
||||
|
||||
def snapshot_for_account(account: Any) -> CookieSnapshot | None:
|
||||
"""Read an account's cookie vault and summarise it, or ``None`` when unreadable.
|
||||
|
||||
Shared by the gate and the keepalive manager so both record the exact same
|
||||
fingerprint for the same vault contents.
|
||||
"""
|
||||
|
||||
cookie_file = getattr(account, "cookie_file", None)
|
||||
if cookie_file is None:
|
||||
return None
|
||||
try:
|
||||
from gyxx_flow.adapters.browser import BrowserCookieStore
|
||||
|
||||
store = BrowserCookieStore(
|
||||
cookie_file,
|
||||
getattr(account, "storage_state_file", cookie_file),
|
||||
)
|
||||
cookies = store.load_cookies()
|
||||
except Exception: # noqa: BLE001 - a missing/corrupt vault is itself a signal
|
||||
return None
|
||||
return summarize_cookies(
|
||||
cookies,
|
||||
required_domains=tuple(getattr(account, "required_cookie_domains", ()) or ()),
|
||||
required_names=tuple(getattr(account, "required_cookie_names", ()) or ()),
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 时间与文本工具
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
text = value.strip().replace("Z", "+00:00")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _parse_date(value: Any) -> date | None:
|
||||
if isinstance(value, date) and not isinstance(value, datetime):
|
||||
return value
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _seconds_between(start: datetime | None, end: datetime | None) -> int | None:
|
||||
if start is None or end is None:
|
||||
return None
|
||||
delta = (end - start).total_seconds()
|
||||
return max(0, int(delta))
|
||||
|
||||
|
||||
def _clip(value: Any, limit: int = _MAX_TEXT_CHARS) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
text = value if isinstance(value, str) else str(value)
|
||||
return text[:limit]
|
||||
|
||||
|
||||
def split_step_key(key: str) -> tuple[str, int]:
|
||||
"""``sync.attempt-2`` -> ``("sync", 2)``; 无法解析时原样当作 step_id。"""
|
||||
|
||||
if ".attempt-" in key:
|
||||
name, _, raw = key.rpartition(".attempt-")
|
||||
try:
|
||||
attempt = int(raw)
|
||||
except ValueError:
|
||||
return key, 1
|
||||
if name and attempt > 0:
|
||||
return name, attempt
|
||||
return key, 1
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 审计存储
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class AuditStore:
|
||||
"""Idempotent writer for the ``ops_*`` tables.
|
||||
|
||||
所有公开写入方法都返回 bool,并且**永不抛出**异常;数据库问题只会导致本次
|
||||
审计丢失,不会影响工作流执行。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
data_root: Path | str,
|
||||
*,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
connect: Any | None = None,
|
||||
clock: Any | None = None,
|
||||
) -> None:
|
||||
self._data_root = Path(data_root).expanduser().resolve()
|
||||
self._environment = dict(os.environ if environment is None else environment)
|
||||
self._connect_impl = connect
|
||||
self._clock = clock or (lambda: datetime.now(timezone.utc))
|
||||
self._schema_ready = False
|
||||
self._consecutive_failures = 0
|
||||
self._cooldown_until = 0.0
|
||||
self._last_error = ""
|
||||
self._failure_log = self._data_root / "logs" / "ops-audit" / "audit-failures.log"
|
||||
|
||||
# -- 状态 ---------------------------------------------------------
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
raw = str(self._environment.get(OPS_AUDIT_ENV, "")).strip().lower()
|
||||
return raw not in {"0", "off", "false", "no", "disabled"}
|
||||
|
||||
@property
|
||||
def last_error(self) -> str:
|
||||
return self._last_error
|
||||
|
||||
def _host(self) -> str:
|
||||
override = str(self._environment.get(OPS_AUDIT_HOST_ENV, "")).strip()
|
||||
if override:
|
||||
return override
|
||||
try:
|
||||
return socket.gethostname()[:128]
|
||||
except OSError: # pragma: no cover - 极端环境
|
||||
return ""
|
||||
|
||||
def _cooldown_seconds(self) -> int:
|
||||
raw = str(self._environment.get(OPS_AUDIT_COOLDOWN_ENV, "")).strip()
|
||||
try:
|
||||
return max(0, int(raw))
|
||||
except ValueError:
|
||||
return _DEFAULT_COOLDOWN_SECONDS
|
||||
|
||||
def _in_cooldown(self) -> bool:
|
||||
import time
|
||||
|
||||
return time.monotonic() < self._cooldown_until
|
||||
|
||||
# -- 连接 ---------------------------------------------------------
|
||||
|
||||
@contextmanager
|
||||
def _connection(self) -> Iterator[Any]:
|
||||
if self._connect_impl is not None:
|
||||
conn = self._connect_impl()
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
close = getattr(conn, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
return
|
||||
|
||||
import psycopg # 延迟导入:未安装 psycopg 时整个模块仍可导入
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
dsn, kwargs = database_kwargs(self._environment)
|
||||
if dsn:
|
||||
conn = psycopg.connect(dsn, row_factory=dict_row)
|
||||
else:
|
||||
conn = psycopg.connect(**kwargs, row_factory=dict_row)
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _ensure_schema(self, conn: Any) -> None:
|
||||
"""Create the audit objects once, serialising the first DDL pass.
|
||||
|
||||
Several workflows can start at the same moment; running CREATE/COMMENT
|
||||
concurrently lets sessions take DDL locks in different orders and can
|
||||
deadlock. The advisory lock is only taken when the schema is missing,
|
||||
so steady-state processes skip the round trip entirely.
|
||||
"""
|
||||
|
||||
if self._schema_ready:
|
||||
return
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM pg_class "
|
||||
"WHERE relname = 'ops_audit_schema' AND relkind = 'r'"
|
||||
)
|
||||
initialised = cursor.fetchone() is not None
|
||||
if not initialised:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtext(%s))",
|
||||
(_SCHEMA_LOCK_KEY,),
|
||||
)
|
||||
cursor.execute(SCHEMA_SQL)
|
||||
cursor.execute(
|
||||
"INSERT INTO ops_audit_schema (schema_version) VALUES (%s) "
|
||||
"ON CONFLICT (schema_version) DO NOTHING",
|
||||
(SCHEMA_VERSION,),
|
||||
)
|
||||
conn.commit()
|
||||
self._schema_ready = True
|
||||
|
||||
# -- 失败处理 -----------------------------------------------------
|
||||
|
||||
def _record_failure(self, operation: str, exc: BaseException) -> None:
|
||||
import time
|
||||
|
||||
self._consecutive_failures += 1
|
||||
self._last_error = f"{type(exc).__name__}: {exc}"[:300]
|
||||
if self._consecutive_failures >= _MAX_CONSECUTIVE_FAILURES:
|
||||
self._cooldown_until = time.monotonic() + self._cooldown_seconds()
|
||||
try:
|
||||
self._failure_log.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"at": self._clock().isoformat(timespec="seconds"),
|
||||
"operation": operation,
|
||||
"error": self._last_error,
|
||||
"consecutive_failures": self._consecutive_failures,
|
||||
}
|
||||
with self._failure_log.open("a", encoding="utf-8") as stream:
|
||||
stream.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _run_guarded(self, operation: str, action: Any) -> bool:
|
||||
if not self.enabled or self._in_cooldown():
|
||||
return False
|
||||
try:
|
||||
with self._connection() as conn:
|
||||
self._ensure_schema(conn)
|
||||
action(conn)
|
||||
self._consecutive_failures = 0
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001 - 审计绝不影响业务
|
||||
self._record_failure(operation, exc)
|
||||
return False
|
||||
|
||||
# -- 运行记录 -----------------------------------------------------
|
||||
|
||||
def record_run(
|
||||
self,
|
||||
record: Mapping[str, Any],
|
||||
steps: Mapping[str, Any] | None = None,
|
||||
) -> bool:
|
||||
"""Upsert one run and its steps from a journal payload or RunRecord dict."""
|
||||
|
||||
workflow_id = _clip(record.get("workflow_id"), 200)
|
||||
run_id = _clip(record.get("run_id"), 200)
|
||||
business_date = _parse_date(record.get("business_date"))
|
||||
if not workflow_id or not run_id or business_date is None:
|
||||
return False
|
||||
status = _clip(record.get("status"), 32) or "unknown"
|
||||
mode = _clip(record.get("mode"), 32) or "unknown"
|
||||
started_at = _parse_datetime(record.get("started_at"))
|
||||
ended_at = _parse_datetime(record.get("ended_at"))
|
||||
step_rows = self._step_rows(run_id, steps)
|
||||
counts = self._step_counts(steps, record)
|
||||
|
||||
def action(conn: Any) -> None:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO ops_workflow_runs (
|
||||
run_id, workflow_id, business_date, mode, shadow, status,
|
||||
started_at, ended_at, duration_seconds, error,
|
||||
step_total, step_success, step_failed, step_skipped,
|
||||
step_running, journal_path, host, indexed_at
|
||||
) VALUES (
|
||||
%(run_id)s, %(workflow_id)s, %(business_date)s, %(mode)s,
|
||||
%(shadow)s, %(status)s, %(started_at)s, %(ended_at)s,
|
||||
%(duration_seconds)s, %(error)s, %(step_total)s,
|
||||
%(step_success)s, %(step_failed)s, %(step_skipped)s,
|
||||
%(step_running)s, %(journal_path)s, %(host)s, NOW()
|
||||
)
|
||||
ON CONFLICT (run_id) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
ended_at = EXCLUDED.ended_at,
|
||||
duration_seconds = EXCLUDED.duration_seconds,
|
||||
error = EXCLUDED.error,
|
||||
step_total = EXCLUDED.step_total,
|
||||
step_success = EXCLUDED.step_success,
|
||||
step_failed = EXCLUDED.step_failed,
|
||||
step_skipped = EXCLUDED.step_skipped,
|
||||
step_running = EXCLUDED.step_running,
|
||||
journal_path = EXCLUDED.journal_path,
|
||||
indexed_at = NOW()
|
||||
""",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"workflow_id": workflow_id,
|
||||
"business_date": business_date,
|
||||
"mode": mode,
|
||||
"shadow": bool(record.get("shadow", False)),
|
||||
"status": status,
|
||||
"started_at": started_at,
|
||||
"ended_at": ended_at,
|
||||
"duration_seconds": _seconds_between(started_at, ended_at),
|
||||
"error": _clip(record.get("error")),
|
||||
"step_total": counts["total"],
|
||||
"step_success": counts["success"],
|
||||
"step_failed": counts["failed"],
|
||||
"step_skipped": counts["skipped"],
|
||||
"step_running": counts["running"],
|
||||
"journal_path": _clip(record.get("journal_path"), 1000),
|
||||
"host": self._host(),
|
||||
},
|
||||
)
|
||||
if step_rows:
|
||||
cursor.executemany(
|
||||
"""
|
||||
INSERT INTO ops_workflow_run_steps (
|
||||
run_id, step_id, attempt, status, exit_code,
|
||||
started_at, ended_at, duration_seconds, error
|
||||
) VALUES (
|
||||
%(run_id)s, %(step_id)s, %(attempt)s, %(status)s,
|
||||
%(exit_code)s, %(started_at)s, %(ended_at)s,
|
||||
%(duration_seconds)s, %(error)s
|
||||
)
|
||||
ON CONFLICT (run_id, step_id, attempt) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
exit_code = EXCLUDED.exit_code,
|
||||
started_at = COALESCE(
|
||||
ops_workflow_run_steps.started_at,
|
||||
EXCLUDED.started_at
|
||||
),
|
||||
ended_at = EXCLUDED.ended_at,
|
||||
duration_seconds = EXCLUDED.duration_seconds,
|
||||
error = EXCLUDED.error
|
||||
""",
|
||||
step_rows,
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
return self._run_guarded("record_run", action)
|
||||
|
||||
@staticmethod
|
||||
def _step_rows(
|
||||
run_id: str, steps: Mapping[str, Any] | None
|
||||
) -> list[dict[str, Any]]:
|
||||
if not isinstance(steps, Mapping):
|
||||
return []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for key, raw in steps.items():
|
||||
if not isinstance(raw, Mapping):
|
||||
continue
|
||||
step_id, attempt = split_step_key(str(key))
|
||||
attempt_value = raw.get("attempt")
|
||||
if isinstance(attempt_value, int) and not isinstance(attempt_value, bool):
|
||||
attempt = max(1, attempt_value)
|
||||
started_at = _parse_datetime(raw.get("started_at"))
|
||||
ended_at = _parse_datetime(raw.get("ended_at"))
|
||||
exit_code = raw.get("exit_code")
|
||||
rows.append(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"step_id": _clip(step_id, 200),
|
||||
"attempt": attempt,
|
||||
"status": _clip(raw.get("status"), 32),
|
||||
"exit_code": exit_code if isinstance(exit_code, int) else None,
|
||||
"started_at": started_at,
|
||||
"ended_at": ended_at,
|
||||
"duration_seconds": _seconds_between(started_at, ended_at),
|
||||
"error": _clip(raw.get("error")),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
@staticmethod
|
||||
def _step_counts(
|
||||
steps: Mapping[str, Any] | None, record: Mapping[str, Any]
|
||||
) -> dict[str, int]:
|
||||
counts = {"total": 0, "success": 0, "failed": 0, "skipped": 0, "running": 0}
|
||||
if isinstance(steps, Mapping) and steps:
|
||||
for raw in steps.values():
|
||||
if not isinstance(raw, Mapping):
|
||||
continue
|
||||
status = str(raw.get("status", ""))
|
||||
counts["total"] += 1
|
||||
if status in counts:
|
||||
counts[status] += 1
|
||||
return counts
|
||||
# 回退到 RunRecord 的 step_counts 投影
|
||||
projection = record.get("step_counts")
|
||||
if isinstance(projection, Mapping):
|
||||
for name in ("success", "failed", "skipped", "running"):
|
||||
value = projection.get(name)
|
||||
if isinstance(value, int) and not isinstance(value, bool):
|
||||
counts[name] = max(0, value)
|
||||
counts["total"] = counts["success"] + counts["failed"] + counts["skipped"] + counts["running"]
|
||||
return counts
|
||||
|
||||
# -- cookie 事件 ---------------------------------------------------
|
||||
|
||||
def record_cookie_event(
|
||||
self,
|
||||
*,
|
||||
account_id: str,
|
||||
state: str,
|
||||
snapshot: CookieSnapshot | None = None,
|
||||
probe_status: str = "",
|
||||
probe_source: str = "",
|
||||
detail: str = "",
|
||||
workflow_id: str = "",
|
||||
run_id: str = "",
|
||||
observed_at: datetime | None = None,
|
||||
) -> bool:
|
||||
"""Append one observation. 状态必须是白名单值,避免脏数据污染统计。"""
|
||||
|
||||
account_id = _clip(account_id, 200)
|
||||
if not account_id:
|
||||
return False
|
||||
state = str(state or "").strip()
|
||||
if state not in _COOKIE_STATES:
|
||||
raise ValueError(f"invalid cookie state: {state!r}")
|
||||
|
||||
def action(conn: Any) -> None:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO ops_account_cookie_events (
|
||||
account_id, observed_at, state, probe_status, probe_source,
|
||||
cookie_count, required_count, fingerprint, declared_expires_at,
|
||||
detail, workflow_id, run_id, host
|
||||
) VALUES (
|
||||
%(account_id)s, %(observed_at)s, %(state)s, %(probe_status)s,
|
||||
%(probe_source)s, %(cookie_count)s, %(required_count)s,
|
||||
%(fingerprint)s, %(declared_expires_at)s, %(detail)s,
|
||||
%(workflow_id)s, %(run_id)s, %(host)s
|
||||
)
|
||||
""",
|
||||
{
|
||||
"account_id": account_id,
|
||||
"observed_at": observed_at or self._clock(),
|
||||
"state": state,
|
||||
"probe_status": _clip(probe_status, 64),
|
||||
"probe_source": _clip(probe_source, 64),
|
||||
"cookie_count": snapshot.cookie_count if snapshot else None,
|
||||
"required_count": snapshot.required_count if snapshot else None,
|
||||
"fingerprint": snapshot.fingerprint if snapshot else "",
|
||||
"declared_expires_at": (
|
||||
snapshot.declared_expires_at if snapshot else None
|
||||
),
|
||||
"detail": _clip(detail),
|
||||
"workflow_id": _clip(workflow_id, 200),
|
||||
"run_id": _clip(run_id, 200),
|
||||
"host": self._host(),
|
||||
},
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
return self._run_guarded("record_cookie_event", action)
|
||||
|
||||
# -- 查询(供 CLI) ------------------------------------------------
|
||||
|
||||
def query_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
status: str | None = None,
|
||||
business_date: str | None = None,
|
||||
limit: int = 50,
|
||||
) -> list[dict[str, Any]]:
|
||||
conditions: list[str] = []
|
||||
params: dict[str, Any] = {"limit": max(1, min(int(limit), 500))}
|
||||
if workflow_id:
|
||||
conditions.append("workflow_id = %(workflow_id)s")
|
||||
params["workflow_id"] = workflow_id
|
||||
if status:
|
||||
conditions.append("status = %(status)s")
|
||||
params["status"] = status
|
||||
if business_date:
|
||||
parsed = _parse_date(business_date)
|
||||
if parsed is not None:
|
||||
conditions.append("business_date = %(business_date)s")
|
||||
params["business_date"] = parsed
|
||||
where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
sql = f"""
|
||||
SELECT * FROM ops_workflow_runs
|
||||
{where}
|
||||
ORDER BY started_at DESC NULLS LAST, run_id DESC
|
||||
LIMIT %(limit)s
|
||||
"""
|
||||
return self._fetch(sql, params)
|
||||
|
||||
def query_steps(self, run_id: str) -> list[dict[str, Any]]:
|
||||
return self._fetch(
|
||||
"""
|
||||
SELECT * FROM ops_workflow_run_steps
|
||||
WHERE run_id = %(run_id)s
|
||||
ORDER BY step_id, attempt
|
||||
""",
|
||||
{"run_id": run_id},
|
||||
)
|
||||
|
||||
def query_cookie_events(
|
||||
self,
|
||||
*,
|
||||
account_id: str | None = None,
|
||||
state: str | None = None,
|
||||
limit: int = 50,
|
||||
) -> list[dict[str, Any]]:
|
||||
conditions: list[str] = []
|
||||
params: dict[str, Any] = {"limit": max(1, min(int(limit), 500))}
|
||||
if account_id:
|
||||
conditions.append("account_id = %(account_id)s")
|
||||
params["account_id"] = account_id
|
||||
if state:
|
||||
conditions.append("state = %(state)s")
|
||||
params["state"] = state
|
||||
where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
sql = f"""
|
||||
SELECT * FROM ops_account_cookie_events
|
||||
{where}
|
||||
ORDER BY observed_at DESC
|
||||
LIMIT %(limit)s
|
||||
"""
|
||||
return self._fetch(sql, params)
|
||||
|
||||
def query_cookie_lifetimes(self, *, account_id: str | None = None) -> list[dict[str, Any]]:
|
||||
if account_id:
|
||||
return self._fetch(
|
||||
"""
|
||||
SELECT * FROM ops_account_cookie_lifetimes
|
||||
WHERE account_id = %(account_id)s
|
||||
ORDER BY first_seen_at DESC
|
||||
""",
|
||||
{"account_id": account_id},
|
||||
)
|
||||
return self._fetch(
|
||||
"SELECT * FROM ops_account_cookie_lifetimes ORDER BY first_seen_at DESC",
|
||||
{},
|
||||
)
|
||||
|
||||
def query_cookie_stats(self) -> list[dict[str, Any]]:
|
||||
return self._fetch(
|
||||
"SELECT * FROM ops_account_cookie_stats ORDER BY account_id",
|
||||
{},
|
||||
)
|
||||
|
||||
def _fetch(self, sql: str, params: Mapping[str, Any]) -> list[dict[str, Any]]:
|
||||
if not self.enabled or self._in_cooldown():
|
||||
return []
|
||||
try:
|
||||
with self._connection() as conn:
|
||||
self._ensure_schema(conn)
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(sql, params)
|
||||
rows = cursor.fetchall()
|
||||
self._consecutive_failures = 0
|
||||
return [dict(row) for row in rows]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._record_failure("query", exc)
|
||||
return []
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 连接参数(与 PostgresWorkflowConfigStore 同口径)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def database_kwargs(environment: Mapping[str, str]) -> tuple[str | None, dict[str, Any]]:
|
||||
values = _apply_runtime_policy(environment)
|
||||
dsn = str(values.get("GYXX_POSTGRES_DSN", "")).strip()
|
||||
if dsn:
|
||||
return dsn, {}
|
||||
required = ("PG_HOST", "PG_PORT", "PG_DB", "PG_USER", "PG_PASSWORD")
|
||||
missing = [name for name in required if not str(values.get(name, "")).strip()]
|
||||
if missing:
|
||||
raise RuntimeError("审计数据库配置缺失: " + ", ".join(missing))
|
||||
try:
|
||||
port = int(values["PG_PORT"])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RuntimeError("PG_PORT 必须是整数") from exc
|
||||
return None, {
|
||||
"host": str(values["PG_HOST"]).strip(),
|
||||
"port": port,
|
||||
"dbname": str(values["PG_DB"]).strip(),
|
||||
"user": str(values["PG_USER"]).strip(),
|
||||
"password": str(values["PG_PASSWORD"]),
|
||||
}
|
||||
|
||||
|
||||
def _apply_runtime_policy(environment: Mapping[str, str]) -> Mapping[str, str]:
|
||||
"""复用 RuntimeServicePolicy,保证与控制台/工作流看到同一个数据库地址。"""
|
||||
|
||||
try:
|
||||
from gyxx_flow.adapters import RuntimeServicePolicy
|
||||
except Exception: # pragma: no cover - 循环导入兜底
|
||||
return environment
|
||||
try:
|
||||
return RuntimeServicePolicy().apply(dict(environment))
|
||||
except Exception: # pragma: no cover
|
||||
return environment
|
||||
|
||||
|
||||
def default_store(data_root: Path | str | None = None) -> AuditStore:
|
||||
"""Best-effort store resolved from the standard data root."""
|
||||
|
||||
if data_root is None:
|
||||
try:
|
||||
from gyxx_flow.core.config import Settings
|
||||
|
||||
data_root = Settings.from_env().data_root
|
||||
except Exception: # pragma: no cover
|
||||
data_root = Path.cwd() / "var"
|
||||
return AuditStore(data_root)
|
||||
@@ -86,6 +86,7 @@ class RunIndex:
|
||||
resource = f"ops:run-index:{_digest(record.run_id)}"
|
||||
with self._locks.acquire(resource, owner=_owner(), timeout_seconds=30):
|
||||
atomic_write_json(destination, record.to_dict())
|
||||
_mirror_run_to_audit(self._data_root, record, raw)
|
||||
return record
|
||||
|
||||
def get(self, run_id: str) -> RunRecord | None:
|
||||
@@ -255,3 +256,42 @@ def _digest(value: str) -> str:
|
||||
import hashlib
|
||||
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()[:32]
|
||||
|
||||
|
||||
_AUDIT_STORES: dict[str, Any] = {}
|
||||
|
||||
|
||||
def _audit_store_for(data_root: Path) -> Any:
|
||||
"""Reuse one store per data root so schema bootstrap and the circuit breaker
|
||||
survive the short-lived ``RunIndex`` instances created per run."""
|
||||
|
||||
key = str(data_root)
|
||||
store = _AUDIT_STORES.get(key)
|
||||
if store is None:
|
||||
from gyxx_flow.ops.audit import AuditStore
|
||||
|
||||
store = AuditStore(data_root)
|
||||
_AUDIT_STORES[key] = store
|
||||
return store
|
||||
|
||||
|
||||
def _mirror_run_to_audit(
|
||||
data_root: Path,
|
||||
record: RunRecord,
|
||||
journal_payload: Any,
|
||||
) -> None:
|
||||
"""Best-effort mirror of a run into the PostgreSQL audit tables.
|
||||
|
||||
Auditing is strictly additive: any failure here (missing DSN, unreachable
|
||||
database, unexpected schema state) is swallowed so that journal indexing and
|
||||
the workflow itself are never affected.
|
||||
"""
|
||||
|
||||
try:
|
||||
store = _audit_store_for(data_root)
|
||||
if not store.enabled:
|
||||
return
|
||||
steps = journal_payload.get("steps") if isinstance(journal_payload, dict) else None
|
||||
store.record_run(record.to_dict(), steps)
|
||||
except Exception: # noqa: BLE001 - audit mirroring must never raise
|
||||
return
|
||||
@@ -0,0 +1,152 @@
|
||||
-- Migration 001: operational audit tables (workflow runs + login cookie telemetry).
|
||||
--
|
||||
-- 本文件由 src/gyxx_flow/ops/audit.py 的 SCHEMA_SQL 导出,请勿手工编辑;
|
||||
-- 运行时首次写入会自动执行同一份 DDL(幂等),此处仅供 DBA 审阅或预建。
|
||||
--
|
||||
-- 用法: psql "$GYXX_POSTGRES_DSN" -f 001_ops_audit.sql
|
||||
--
|
||||
-- 说明:
|
||||
-- ops_workflow_runs 一次工作流运行一行,run.json 的关系型投影
|
||||
-- ops_workflow_run_steps 每次尝试一行(含重试),保留 exit_code 与错误
|
||||
-- ops_account_cookie_events cookie 观测事件流,只存指纹不存明文值
|
||||
-- ops_account_cookie_lifetimes 一处登录态从首次出现到失效(视图)
|
||||
-- ops_account_cookie_stats 每账号失效规律汇总(视图)
|
||||
--
|
||||
-- 日志正文仍留在 var/logs 下,数据库只存结构化元数据与指向路径。
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops_workflow_runs (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
workflow_id TEXT NOT NULL,
|
||||
business_date DATE NOT NULL,
|
||||
mode TEXT NOT NULL DEFAULT 'unknown',
|
||||
shadow BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
status TEXT NOT NULL,
|
||||
started_at TIMESTAMPTZ,
|
||||
ended_at TIMESTAMPTZ,
|
||||
duration_seconds INTEGER,
|
||||
error TEXT,
|
||||
step_total INTEGER NOT NULL DEFAULT 0,
|
||||
step_success INTEGER NOT NULL DEFAULT 0,
|
||||
step_failed INTEGER NOT NULL DEFAULT 0,
|
||||
step_skipped INTEGER NOT NULL DEFAULT 0,
|
||||
step_running INTEGER NOT NULL DEFAULT 0,
|
||||
journal_path TEXT NOT NULL DEFAULT '',
|
||||
host TEXT NOT NULL DEFAULT '',
|
||||
indexed_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ops_workflow_runs_workflow_idx
|
||||
ON ops_workflow_runs (workflow_id, started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS ops_workflow_runs_status_idx
|
||||
ON ops_workflow_runs (status, started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS ops_workflow_runs_business_date_idx
|
||||
ON ops_workflow_runs (business_date, workflow_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops_workflow_run_steps (
|
||||
run_id TEXT NOT NULL
|
||||
REFERENCES ops_workflow_runs (run_id) ON DELETE CASCADE,
|
||||
step_id TEXT NOT NULL,
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
status TEXT NOT NULL DEFAULT '',
|
||||
exit_code INTEGER,
|
||||
started_at TIMESTAMPTZ,
|
||||
ended_at TIMESTAMPTZ,
|
||||
duration_seconds INTEGER,
|
||||
error TEXT,
|
||||
PRIMARY KEY (run_id, step_id, attempt)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ops_workflow_run_steps_step_idx
|
||||
ON ops_workflow_run_steps (step_id, status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops_account_cookie_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
account_id TEXT NOT NULL,
|
||||
observed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
state TEXT NOT NULL,
|
||||
probe_status TEXT NOT NULL DEFAULT '',
|
||||
probe_source TEXT NOT NULL DEFAULT '',
|
||||
cookie_count INTEGER,
|
||||
required_count INTEGER,
|
||||
fingerprint TEXT NOT NULL DEFAULT '',
|
||||
declared_expires_at TIMESTAMPTZ,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
workflow_id TEXT NOT NULL DEFAULT '',
|
||||
run_id TEXT NOT NULL DEFAULT '',
|
||||
host TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ops_account_cookie_events_account_idx
|
||||
ON ops_account_cookie_events (account_id, observed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS ops_account_cookie_events_state_idx
|
||||
ON ops_account_cookie_events (state, observed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS ops_account_cookie_events_fingerprint_idx
|
||||
ON ops_account_cookie_events (account_id, fingerprint);
|
||||
|
||||
-- 一段「cookie 代」的生命周期:同一 fingerprint 从首次出现到失效。
|
||||
-- first_failed_at 只取最后一次 healthy 之后的失败,避免恢复后再次失败时口径前移。
|
||||
CREATE OR REPLACE VIEW ops_account_cookie_lifetimes AS
|
||||
WITH per_fingerprint AS (
|
||||
SELECT
|
||||
account_id,
|
||||
fingerprint,
|
||||
MIN(observed_at) AS first_seen_at,
|
||||
MIN(observed_at) FILTER (WHERE state = 'healthy') AS first_healthy_at,
|
||||
MAX(observed_at) FILTER (WHERE state = 'healthy') AS last_healthy_at,
|
||||
MAX(declared_expires_at) AS declared_expires_at
|
||||
FROM ops_account_cookie_events
|
||||
WHERE fingerprint <> ''
|
||||
GROUP BY account_id, fingerprint
|
||||
)
|
||||
SELECT
|
||||
p.account_id,
|
||||
p.fingerprint,
|
||||
p.first_seen_at,
|
||||
p.first_healthy_at,
|
||||
p.last_healthy_at,
|
||||
p.declared_expires_at,
|
||||
f.first_failed_at,
|
||||
(f.first_failed_at IS NOT NULL) AS expired,
|
||||
CASE
|
||||
WHEN p.first_healthy_at IS NOT NULL AND p.last_healthy_at IS NOT NULL
|
||||
THEN EXTRACT(EPOCH FROM (p.last_healthy_at - p.first_healthy_at))::BIGINT
|
||||
END AS healthy_span_seconds,
|
||||
CASE
|
||||
WHEN p.first_healthy_at IS NOT NULL AND f.first_failed_at IS NOT NULL
|
||||
THEN EXTRACT(EPOCH FROM (f.first_failed_at - p.first_healthy_at))::BIGINT
|
||||
END AS lifetime_seconds,
|
||||
CASE
|
||||
WHEN p.last_healthy_at IS NOT NULL AND f.first_failed_at IS NOT NULL
|
||||
THEN EXTRACT(EPOCH FROM (f.first_failed_at - p.last_healthy_at))::BIGINT
|
||||
END AS healthy_to_failure_seconds
|
||||
FROM per_fingerprint p
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT MIN(e.observed_at) AS first_failed_at
|
||||
FROM ops_account_cookie_events e
|
||||
WHERE e.account_id = p.account_id
|
||||
AND e.fingerprint = p.fingerprint
|
||||
AND e.state IN ('expired', 'not_logged_in')
|
||||
AND (p.last_healthy_at IS NULL OR e.observed_at > p.last_healthy_at)
|
||||
) f ON TRUE;
|
||||
|
||||
-- 按账号汇总的失效规律:平均/中位/最短/最长寿命,直接支撑「多久失效一次」。
|
||||
CREATE OR REPLACE VIEW ops_account_cookie_stats AS
|
||||
SELECT
|
||||
account_id,
|
||||
COUNT(*) AS sessions,
|
||||
COUNT(*) FILTER (WHERE expired) AS expired_sessions,
|
||||
AVG(lifetime_seconds)::BIGINT AS avg_lifetime_seconds,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY lifetime_seconds)::BIGINT
|
||||
AS median_lifetime_seconds,
|
||||
MIN(lifetime_seconds) AS min_lifetime_seconds,
|
||||
MAX(lifetime_seconds) AS max_lifetime_seconds,
|
||||
MAX(first_failed_at) AS last_failed_at,
|
||||
MAX(last_healthy_at) AS last_healthy_at
|
||||
FROM ops_account_cookie_lifetimes
|
||||
WHERE lifetime_seconds IS NOT NULL
|
||||
GROUP BY account_id;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ops_audit_schema (
|
||||
schema_version INTEGER PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
@@ -143,6 +143,7 @@
|
||||
options: [
|
||||
{ value: "flagship", label: "光影行星旗舰店" },
|
||||
{ value: "luggage", label: "光影行星鑫华达专卖店:龙虾仔" },
|
||||
{ value: "new_bag", label: "天猫新店 · gyxx箱包旗舰店:龙虾仔(只发黑曜石系列)" },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<meta name="theme-color" content="#f5f7fb">
|
||||
<title>GYXX Flow · 工作流控制台</title>
|
||||
<meta name="description" content="按业务模块查看、配置和运行 GYXX Flow 工作流。">
|
||||
<link rel="stylesheet" href="/assets/app.css?v=scheduler-control-v1-20260831">
|
||||
<link rel="stylesheet" href="/assets/app.css?v=scheduler-control-v1-20261030">
|
||||
</head>
|
||||
<body>
|
||||
<a class="skip-link" href="#main-content">跳到主要内容</a>
|
||||
@@ -475,6 +475,7 @@
|
||||
<select id="run-store" name="store">
|
||||
<option value="flagship">光影行星旗舰店</option>
|
||||
<option value="luggage">光影行星鑫华达专卖店:龙虾仔</option>
|
||||
<option value="new_bag">天猫新店 · gyxx箱包旗舰店:龙虾仔(只发黑曜石系列)</option>
|
||||
</select>
|
||||
<small id="run-store-hint">先扫描飞书并按所选店铺规则筛选;存在未上传附件时才登录该店铺账号。</small>
|
||||
</label>
|
||||
@@ -636,6 +637,6 @@
|
||||
|
||||
<div class="toast-region" id="toast-region" aria-live="polite" aria-atomic="false"></div>
|
||||
|
||||
<script src="/assets/app.js?v=scheduler-control-v1-20260831" defer></script>
|
||||
<script src="/assets/app.js?v=scheduler-control-v1-20261030" defer></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,74 @@
|
||||
"""飞书目标表地址唯一来源:dim_style,失败即闭环保,不回退本地 JSON。
|
||||
|
||||
2026-10-30 起三个写表脚本(日报 / 天猫主图 / 京东主图)都去掉了
|
||||
``read_json(bitable_*_map.json)`` 回退——库与本地文件不一致时,回退会让
|
||||
数据静默写到错误的表,比直接失败更难发现。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
import insert_bitable_records as daily_insert
|
||||
import pytest
|
||||
from scripts import insert_jd_main_image_records as jd_insert
|
||||
from scripts import insert_main_image_records as tmall_insert
|
||||
|
||||
|
||||
def _raise_config_error() -> dict:
|
||||
raise RuntimeError("云端动态配置读取失败")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"module, attr",
|
||||
[
|
||||
(daily_insert, "STYLE_MAP_PATH"),
|
||||
(tmall_insert, "MAIN_IMAGE_MAP"),
|
||||
],
|
||||
)
|
||||
def test_local_json_target_maps_are_gone(module, attr) -> None:
|
||||
"""本地 JSON 回退入口必须彻底移除,防止以后被重新接回链路。"""
|
||||
|
||||
assert not hasattr(module, attr)
|
||||
|
||||
|
||||
def test_daily_insert_fails_closed_without_style_map(monkeypatch) -> None:
|
||||
monkeypatch.setattr(daily_insert, "get_bitable_style_map", _raise_config_error)
|
||||
|
||||
assert daily_insert.main(["--date", "2026-08-03"]) == 1
|
||||
|
||||
|
||||
def test_daily_insert_fails_closed_on_empty_style_map(monkeypatch) -> None:
|
||||
monkeypatch.setattr(daily_insert, "get_bitable_style_map", lambda: {})
|
||||
|
||||
assert daily_insert.main(["--date", "2026-08-03"]) == 1
|
||||
|
||||
|
||||
def test_tmall_main_image_insert_fails_closed(monkeypatch) -> None:
|
||||
monkeypatch.setattr(sys, "argv", ["insert_main_image_records.py", "--date", "2026-10-25"])
|
||||
monkeypatch.setattr(tmall_insert, "get_main_image_styles", lambda: ["某款"])
|
||||
monkeypatch.setattr(tmall_insert, "get_bitable_main_image_map", _raise_config_error)
|
||||
|
||||
assert tmall_insert.main() == 1
|
||||
|
||||
|
||||
def test_tmall_main_image_insert_fails_closed_on_empty_map(monkeypatch) -> None:
|
||||
monkeypatch.setattr(sys, "argv", ["insert_main_image_records.py", "--date", "2026-10-25"])
|
||||
monkeypatch.setattr(tmall_insert, "get_main_image_styles", lambda: ["某款"])
|
||||
monkeypatch.setattr(tmall_insert, "get_bitable_main_image_map", lambda: {})
|
||||
|
||||
assert tmall_insert.main() == 1
|
||||
|
||||
|
||||
def test_jd_main_image_insert_fails_closed(monkeypatch) -> None:
|
||||
monkeypatch.setattr(sys, "argv", ["insert_jd_main_image_records.py", "--date", "2026-10-25"])
|
||||
monkeypatch.setattr(jd_insert, "get_bitable_main_image_map", _raise_config_error)
|
||||
|
||||
assert jd_insert.main() == 1
|
||||
|
||||
|
||||
def test_jd_main_image_insert_fails_closed_on_empty_map(monkeypatch) -> None:
|
||||
monkeypatch.setattr(sys, "argv", ["insert_jd_main_image_records.py", "--date", "2026-10-25"])
|
||||
monkeypatch.setattr(jd_insert, "get_bitable_main_image_map", lambda: {})
|
||||
|
||||
assert jd_insert.main() == 1
|
||||
@@ -0,0 +1,151 @@
|
||||
"""商品日报采集开关(dim_style.collect_enabled,系列级)判定行为。
|
||||
|
||||
开关已从独立表 product_daily_collect_scope 迁到 dim_style.collect_enabled:
|
||||
- 锚点是系列总开关,为 False 时整条系列不采;
|
||||
- 锚点为 True 时,子款自身的开关只决定它的 ERP 编码是否并入聚合;
|
||||
- 缺字段默认采集;
|
||||
- 产品生命进程(另一个项目)只改锚点行即可控制整条系列。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from gyxx_flow.modules.product_commerce.config.style_config_loader import (
|
||||
StyleConfigLoader,
|
||||
)
|
||||
|
||||
SALES_URL = "https://example.feishu.cn/base/sales?table=tblSales"
|
||||
|
||||
|
||||
def _row(
|
||||
style: str,
|
||||
item_id: str,
|
||||
*,
|
||||
platform: str = "天猫",
|
||||
collect_enabled: bool | None = True,
|
||||
series_name: str = "",
|
||||
is_series_anchor: bool = False,
|
||||
erp_codes: list[str] | None = None,
|
||||
) -> dict:
|
||||
row = {
|
||||
"platform": platform,
|
||||
"style_name": style,
|
||||
"erp_codes": erp_codes if erp_codes is not None else [f"ERP-{style}"],
|
||||
"item_ids": [item_id],
|
||||
"sales_bitable_url": SALES_URL,
|
||||
"series_name": series_name,
|
||||
"is_series_anchor": is_series_anchor,
|
||||
}
|
||||
if collect_enabled is not None:
|
||||
row["collect_enabled"] = collect_enabled
|
||||
return row
|
||||
|
||||
|
||||
def _loader(rows: list[dict]) -> StyleConfigLoader:
|
||||
return StyleConfigLoader(records_provider=lambda: rows)
|
||||
|
||||
|
||||
def _two_plain_styles(flag_b: bool | None = True) -> list[dict]:
|
||||
return [
|
||||
_row("款A", "TM-1"),
|
||||
_row("款B", "TM-2", collect_enabled=flag_b),
|
||||
]
|
||||
|
||||
|
||||
def test_style_flagged_disabled_is_not_collected() -> None:
|
||||
loader = _loader(_two_plain_styles(flag_b=False))
|
||||
|
||||
daily = loader.get_daily_styles()
|
||||
|
||||
assert daily["tm"]["collectable"] == ["款A"]
|
||||
assert {"style": "款B", "reason": "已标记不采集"} in daily["tm"]["skipped"]
|
||||
assert loader.get_erp_daily_styles()["collectable"] == ["款A"]
|
||||
|
||||
|
||||
def test_missing_flag_defaults_to_enabled() -> None:
|
||||
"""字段缺失(旧 cmt_styles 回退 / 动态配置独有款)默认采集。"""
|
||||
|
||||
loader = _loader(_two_plain_styles(flag_b=None))
|
||||
|
||||
assert loader.get_daily_styles()["tm"]["collectable"] == ["款A", "款B"]
|
||||
assert loader.get_erp_daily_styles()["collectable"] == ["款A", "款B"]
|
||||
|
||||
|
||||
def test_series_anchor_disable_closes_whole_series() -> None:
|
||||
"""系列级开关:锚点关掉,即使子款自己的 flag 为 True 也不采。"""
|
||||
|
||||
rows = [
|
||||
_row("宙斯", "TM-9", series_name="宙斯", is_series_anchor=True,
|
||||
collect_enabled=False, erp_codes=[]),
|
||||
_row("宙斯3", "TM-10", series_name="宙斯", is_series_anchor=False,
|
||||
collect_enabled=True, erp_codes=["10244"]),
|
||||
]
|
||||
loader = _loader(rows)
|
||||
|
||||
daily = loader.get_daily_styles()
|
||||
|
||||
assert daily["tm"]["collectable"] == []
|
||||
assert {"style": "宙斯", "reason": "已标记不采集"} in daily["tm"]["skipped"]
|
||||
assert {"style": "宙斯3", "reason": "已标记不采集"} in daily["tm"]["skipped"]
|
||||
assert loader.get_erp_daily_styles()["collectable"] == []
|
||||
|
||||
|
||||
def test_series_anchor_enable_wins_over_child_flag() -> None:
|
||||
"""锚点是总开关:子款 False 不会让整系列停采,但它的 ERP 编码不并入聚合。"""
|
||||
|
||||
rows = [
|
||||
_row("逐星", "TM-11", series_name="逐星", is_series_anchor=True,
|
||||
collect_enabled=True, erp_codes=[]),
|
||||
_row("逐星GT", "TM-12", series_name="逐星", is_series_anchor=False,
|
||||
collect_enabled=False, erp_codes=["10494"]),
|
||||
]
|
||||
loader = _loader(rows)
|
||||
|
||||
erp_daily = loader.get_erp_daily_styles()
|
||||
|
||||
# 子款关闭后编码不进聚合,锚点自己没有编码 → 该系列不出现在采集清单
|
||||
assert erp_daily["collectable"] == []
|
||||
assert {"style": "逐星", "reason": "缺 ERP款式编码"} in erp_daily["skipped"]
|
||||
# 子款仍然让出,不会被自己单独采一次
|
||||
assert {"style": "逐星GT", "reason": "ERP 已由同系列「逐星」归集采集"} in erp_daily["skipped"]
|
||||
|
||||
|
||||
def test_anchor_keeps_own_codes_when_child_disabled() -> None:
|
||||
"""锚点 true + 子款 false:只把子款编码排除,锚点自己的编码照常采集。"""
|
||||
|
||||
rows = [
|
||||
_row("逐星", "TM-11", series_name="逐星", is_series_anchor=True,
|
||||
collect_enabled=True, erp_codes=["10493"]),
|
||||
_row("逐星GT", "TM-12", series_name="逐星", is_series_anchor=False,
|
||||
collect_enabled=False, erp_codes=["10494"]),
|
||||
]
|
||||
loader = _loader(rows)
|
||||
|
||||
payload = loader.load()
|
||||
assert payload["styles"]["逐星"]["erp_codes"] == ["10493"]
|
||||
assert payload["styles"]["逐星"]["series"]["erp_excluded"] == ["逐星GT"]
|
||||
# 子款自己的原始编码保留,供月度销量表按款查询
|
||||
assert payload["styles"]["逐星GT"]["series"]["own_erp_codes"] == ["10494"]
|
||||
assert loader.get_erp_daily_styles()["collectable"] == ["逐星"]
|
||||
|
||||
|
||||
def test_series_without_anchor_uses_sorted_first_style() -> None:
|
||||
"""没有锚点标记时,系列内按款名排序第一个的值作为系列开关。"""
|
||||
|
||||
rows = [
|
||||
_row("B款", "TM-1", series_name="无锚系列", collect_enabled=True),
|
||||
_row("A款", "TM-2", series_name="无锚系列", collect_enabled=False),
|
||||
]
|
||||
loader = _loader(rows)
|
||||
|
||||
daily = loader.get_daily_styles()
|
||||
|
||||
# A款 排在前,其 False 成为系列值
|
||||
assert daily["tm"]["collectable"] == []
|
||||
|
||||
|
||||
def test_injected_records_provider_keeps_legacy_behavior() -> None:
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [_row("款A", "TM-1"), _row("款B", "TM-2")]
|
||||
)
|
||||
|
||||
assert loader.get_daily_styles()["tm"]["collectable"] == ["款A", "款B"]
|
||||
@@ -426,27 +426,31 @@ def test_platform_artifact_verifier_rejects_an_export_with_no_product_matches(
|
||||
daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns)
|
||||
|
||||
|
||||
def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
|
||||
def test_daily_collect_no_longer_runs_review_import(monkeypatch) -> None:
|
||||
"""product_reviews 已下线 (2026-11-01)。
|
||||
|
||||
collect 阶段不得再调评论入库——差评由 shop.jd/dy/tmall_negative_reviews.daily
|
||||
三个独立工作流采集,消费方读 v_negative_reviews。这条断言守住下线状态,
|
||||
防止有人顺手把旧链路加回来。
|
||||
"""
|
||||
|
||||
args = SimpleNamespace(
|
||||
skip_erp=True,
|
||||
skip_platforms=False,
|
||||
serial_platforms=False,
|
||||
dry_run=False,
|
||||
skip_reviews_import=False,
|
||||
)
|
||||
monkeypatch.setattr(daily, "run_platforms", lambda **_kwargs: [])
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 1},
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0},
|
||||
)
|
||||
|
||||
daily.run_collect(args, "2026-08-03", ["jd"], failed=None)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0][0] == "import_product_reviews"
|
||||
assert calls[0][1]["critical"] is False
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_daily_import_stage_uses_the_collected_business_date(monkeypatch) -> None:
|
||||
|
||||
@@ -43,7 +43,9 @@ def test_style_plan_rejects_one_erp_code_owned_by_two_styles() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_style_plan_skips_erp_only_style_until_product_id_is_added() -> None:
|
||||
def test_style_plan_keeps_erp_only_style() -> None:
|
||||
"""全店铺日报按 ERP 编码取数,不需要平台商品编码。"""
|
||||
|
||||
plans = backfill.build_style_plans(
|
||||
{
|
||||
"styles": {
|
||||
@@ -56,7 +58,44 @@ def test_style_plan_skips_erp_only_style_until_product_id_is_added() -> None:
|
||||
}
|
||||
)
|
||||
|
||||
assert plans == (backfill.StylePlan("已上架款", ("ERP-READY",)),)
|
||||
assert plans == (
|
||||
backfill.StylePlan("研发款", ("ERP-ONLY",)),
|
||||
backfill.StylePlan("已上架款", ("ERP-READY",)),
|
||||
)
|
||||
|
||||
|
||||
def test_style_plan_folds_series_codes_into_anchor() -> None:
|
||||
"""同系列多款:锚点承接并集编码,子款让出重复编码,不报冲突。"""
|
||||
|
||||
plans = backfill.build_style_plans(
|
||||
{
|
||||
"styles": {
|
||||
"宙斯": {
|
||||
"erp_codes": ["10244", "10387"],
|
||||
"tm": {"item_ids": ["TM-1"]},
|
||||
"series": {"name": "宙斯", "erp_owner": "宙斯"},
|
||||
},
|
||||
"宙斯-10244": {
|
||||
"erp_codes": ["10244"],
|
||||
"series": {
|
||||
"name": "宙斯",
|
||||
"erp_owner": "宙斯",
|
||||
"own_erp_codes": ["10244"],
|
||||
},
|
||||
},
|
||||
"宙斯Z": {
|
||||
"erp_codes": ["10387"],
|
||||
"series": {
|
||||
"name": "宙斯",
|
||||
"erp_owner": "宙斯",
|
||||
"own_erp_codes": ["10387"],
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
assert plans == (backfill.StylePlan("宙斯", ("10244", "10387")),)
|
||||
|
||||
|
||||
def test_all_shop_filter_selects_every_shop_and_batches_all_codes() -> None:
|
||||
@@ -140,9 +179,15 @@ def test_style_extraction_maps_hidden_columns_via_visible_positions() -> None:
|
||||
backfill.extract_style_rows(frame, requested_codes=("10416",))
|
||||
|
||||
assert "window._Cols" in frame.script
|
||||
assert "visibleIdx.indexOf" in frame.script
|
||||
assert "const toPos = index" in frame.script
|
||||
assert "cells[stylePos]" in frame.script
|
||||
assert "cells.length <= Math.max(stylePos, salesPos, returnsPos)" in frame.script
|
||||
# 列布局未知时必须 fail closed,不能退回「按表头索引直取」
|
||||
assert "column_layout_unknown" in frame.script
|
||||
# 行 td 数必须与可见列数严格相等
|
||||
assert "cells.length !== expectedCells" in frame.script
|
||||
# 值域交叉校验:销量不得超过同行月销量一个量级
|
||||
assert "sales_out_of_range" in frame.script
|
||||
assert "month_sales_pos" in frame.script
|
||||
|
||||
|
||||
def test_ensure_custom_columns_visible_only_touches_allsku_frames() -> None:
|
||||
@@ -150,16 +195,16 @@ def test_ensure_custom_columns_visible_only_touches_allsku_frames() -> None:
|
||||
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx?r=1"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.scripts: list[str] = []
|
||||
self.calls: list[tuple] = []
|
||||
|
||||
def evaluate(self, script):
|
||||
self.scripts.append(script)
|
||||
return 2
|
||||
def evaluate(self, script, *args):
|
||||
self.calls.append((script, *args))
|
||||
return {"changed": 2, "redrawn": True}
|
||||
|
||||
class OtherFrame:
|
||||
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/filter.aspx"
|
||||
|
||||
def evaluate(self, script):
|
||||
def evaluate(self, script, *args):
|
||||
raise AssertionError("must not touch non-allsku frames")
|
||||
|
||||
class Page:
|
||||
@@ -170,10 +215,59 @@ def test_ensure_custom_columns_visible_only_touches_allsku_frames() -> None:
|
||||
|
||||
assert changed == 2
|
||||
allsku = page.frames[1]
|
||||
assert len(allsku.scripts) == 1
|
||||
assert "'自定义销量'" in allsku.scripts[0]
|
||||
assert "'自定义退货量'" in allsku.scripts[0]
|
||||
assert "_RptRefresh" in allsku.scripts[0]
|
||||
assert len(allsku.calls) == 1
|
||||
script, options = allsku.calls[0]
|
||||
assert "'自定义销量'" in script
|
||||
assert "'自定义退货量'" in script
|
||||
# 换页重绘才是补齐数据行单元格的手段;_RptRefresh 在未渲染时会抛错。
|
||||
assert "GotoPage" in script
|
||||
assert options["redrawPage"] == 1
|
||||
|
||||
|
||||
def test_ensure_custom_columns_rendered_redraws_requested_page() -> None:
|
||||
class AllskuFrame:
|
||||
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx?r=1"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple] = []
|
||||
|
||||
def evaluate(self, script, *args):
|
||||
self.calls.append((script, *args))
|
||||
return {"changed": 2, "redrawn": True}
|
||||
|
||||
class Page:
|
||||
frames = [AllskuFrame()]
|
||||
|
||||
page = Page()
|
||||
summary = backfill.ensure_custom_columns_rendered(page, redraw_page=3)
|
||||
|
||||
assert summary == {"changed": 2, "redrawn": 1}
|
||||
assert page.frames[0].calls[0][1]["redrawPage"] == 3
|
||||
assert page.frames[0].calls[0][1]["forceRedraw"] is False
|
||||
|
||||
|
||||
def test_ensure_custom_columns_rendered_force_redraw_without_visibility_change() -> None:
|
||||
class AllskuFrame:
|
||||
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx?r=1"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple] = []
|
||||
|
||||
def evaluate(self, script, *args):
|
||||
self.calls.append((script, *args))
|
||||
return {"changed": 0, "redrawn": True}
|
||||
|
||||
class Page:
|
||||
frames = [AllskuFrame()]
|
||||
|
||||
page = Page()
|
||||
summary = backfill.ensure_custom_columns_rendered(
|
||||
page, redraw_page=2, force_redraw=True
|
||||
)
|
||||
|
||||
assert summary == {"changed": 0, "redrawn": 1}
|
||||
assert page.frames[0].calls[0][1]["redrawPage"] == 2
|
||||
assert page.frames[0].calls[0][1]["forceRedraw"] is True
|
||||
|
||||
|
||||
def test_range_collector_queries_once_per_date_not_once_per_style(monkeypatch) -> None:
|
||||
@@ -216,6 +310,14 @@ def test_database_upsert_uses_date_and_style_conflict_key(monkeypatch) -> None:
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
# db.attach_style_ids 会按款名查 dim_style;这里返回空映射,
|
||||
# 断言的是 SQL 形状与业务列,不依赖真实款式表。
|
||||
def execute(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
return []
|
||||
|
||||
class Connection:
|
||||
def cursor(self):
|
||||
return Cursor()
|
||||
@@ -256,3 +358,73 @@ def test_schema_keeps_all_shop_metrics_separate_from_platform_metrics() -> None:
|
||||
assert "CREATE TABLE IF NOT EXISTS erp_all_shop_style_daily_metrics" in schema
|
||||
assert "PRIMARY KEY (metric_date, style_name)" in schema
|
||||
assert "return_qty" in schema
|
||||
|
||||
|
||||
def test_daily_total_guard_rejects_month_scale_values(monkeypatch) -> None:
|
||||
"""单日合计比近 14 天中位数高 8 倍以上必须拒绝落库。"""
|
||||
|
||||
monkeypatch.setattr(backfill, "_recent_sales_baseline", lambda *a, **k: 2000.0)
|
||||
|
||||
with pytest.raises(RuntimeError, match="looks wrong"):
|
||||
backfill._guard_daily_total(
|
||||
date(2026, 10, 29),
|
||||
[{"sales": 700000, "return_qty": 120000}],
|
||||
)
|
||||
|
||||
# 正常量级放行
|
||||
backfill._guard_daily_total(
|
||||
date(2026, 10, 29), [{"sales": 3656, "return_qty": 534}]
|
||||
)
|
||||
# --force 时不做量级判断(业务量确实变化的场景)
|
||||
backfill._guard_daily_total(
|
||||
date(2026, 10, 29),
|
||||
[{"sales": 700000, "return_qty": 120000}],
|
||||
force=True,
|
||||
)
|
||||
|
||||
|
||||
def test_daily_total_guard_skips_when_no_baseline(monkeypatch) -> None:
|
||||
monkeypatch.setattr(backfill, "_recent_sales_baseline", lambda *a, **k: None)
|
||||
backfill._guard_daily_total(
|
||||
date(2026, 10, 29), [{"sales": 999999, "return_qty": 0}]
|
||||
)
|
||||
|
||||
|
||||
def test_partial_page_is_treated_as_not_ready(monkeypatch) -> None:
|
||||
"""非最后一页渲染不满 200 行必须重试,不能当成已完成收下。"""
|
||||
|
||||
class Frame:
|
||||
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx"
|
||||
|
||||
def evaluate(self, script, args=None):
|
||||
return {"ok": True, "rows": [{"sales": 1}] * 174, "signature": "s"}
|
||||
|
||||
monkeypatch.setattr(backfill, "_allsku_frame", lambda page: Frame())
|
||||
monkeypatch.setattr(
|
||||
backfill, "ensure_custom_columns_rendered", lambda *a, **k: {"changed": 1}
|
||||
)
|
||||
|
||||
with pytest.raises(TimeoutError, match="174/200 rows"):
|
||||
backfill._wait_rendered_page(
|
||||
object(), ("10416",), page_no=1, timeout_seconds=1, expect_full_page=True
|
||||
)
|
||||
|
||||
|
||||
def test_last_page_is_allowed_to_be_partial(monkeypatch) -> None:
|
||||
class Frame:
|
||||
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx"
|
||||
|
||||
def evaluate(self, script, args=None):
|
||||
return {"ok": True, "rows": [{"sales": 1}] * 59, "signature": "s"}
|
||||
|
||||
monkeypatch.setattr(backfill, "_allsku_frame", lambda page: Frame())
|
||||
monkeypatch.setattr(
|
||||
backfill, "ensure_custom_columns_rendered", lambda *a, **k: {"changed": 0}
|
||||
)
|
||||
|
||||
result = backfill._wait_rendered_page(
|
||||
object(), ("10416",), page_no=3, timeout_seconds=5, expect_full_page=False
|
||||
)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert len(result["rows"]) == 59
|
||||
@@ -15,8 +15,8 @@ class ErpMetricOverridesTests(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
self.temp_dir.cleanup()
|
||||
|
||||
def _write_erp(self, code_results):
|
||||
data_dir = self.data_root / "dy" / "apollo_2026-07-22"
|
||||
def _write_erp(self, code_results, platform="dy"):
|
||||
data_dir = self.data_root / platform / "apollo_2026-07-22"
|
||||
data_dir.mkdir(parents=True)
|
||||
(data_dir / "data.json").write_text(
|
||||
json.dumps(
|
||||
@@ -33,12 +33,12 @@ class ErpMetricOverridesTests(unittest.TestCase):
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _apply(self, platform_sales, platform_refunds):
|
||||
def _apply(self, platform_sales, platform_refunds, platform="dy"):
|
||||
with patch.object(overrides, "DATA_ROOT", self.data_root), patch.dict(
|
||||
overrides.PLATFORM_DIRS, {"dy": "dy"}
|
||||
overrides.PLATFORM_DIRS, {platform: platform}
|
||||
):
|
||||
return overrides.apply_erp_override(
|
||||
"dy", "阿波罗x1", 394, 46,
|
||||
platform, "阿波罗x1", 394, 46,
|
||||
target_date="2026-07-22",
|
||||
platform_sales=platform_sales,
|
||||
platform_refunds=platform_refunds,
|
||||
@@ -87,6 +87,21 @@ class ErpMetricOverridesTests(unittest.TestCase):
|
||||
self.assertEqual(result["销量"], 0)
|
||||
self.assertEqual(result["退款订单数"], 0)
|
||||
|
||||
def test_partial_codes_used_when_platform_has_no_sales(self):
|
||||
"""TM 没有平台销量字段:部分编码失败时不能整款归零,用成功编码合计。"""
|
||||
self._write_erp([
|
||||
{"erp_style_code": "1", "ok": True, "yesterday_sales": 3, "yesterday_returns": 1},
|
||||
{"erp_style_code": "2", "ok": False, "error": "timeout", "yesterday_sales": 0, "yesterday_returns": 0},
|
||||
], platform="tm")
|
||||
|
||||
result = self._apply(platform_sales=None, platform_refunds=None, platform="tm")
|
||||
|
||||
self.assertTrue(result["erp_unavailable"])
|
||||
self.assertTrue(result["erp_partial"])
|
||||
self.assertEqual(result["销量"], 3)
|
||||
self.assertEqual(result["退款订单数"], 1)
|
||||
self.assertEqual(result["failed_codes"], ["2"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -156,9 +156,9 @@ def test_upsert_sales_theme_dedups_and_normalizes(monkeypatch: pytest.MonkeyPatc
|
||||
|
||||
|
||||
def test_business_scope_constants_are_consistent() -> None:
|
||||
assert len(collector.BRANDS) == 3
|
||||
assert sorted(collector.BRANDS) == ["gyxx", "光影", "光影行星"]
|
||||
assert len(collector.SHOP_NAMES) == len(set(collector.SHOP_NAMES)) > 0
|
||||
# 2026-10-19 起品牌不限(全不勾); 店铺固定 22 家光影系渠道
|
||||
assert collector.BRANDS == ()
|
||||
assert len(collector.SHOP_NAMES) == 22 == len(set(collector.SHOP_NAMES))
|
||||
for name in collector.SHOP_NAMES:
|
||||
assert name == name.strip()
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import unittest
|
||||
from upload_video_to_guanghe import (
|
||||
STORE_FLAGSHIP,
|
||||
STORE_LUGGAGE,
|
||||
STORE_NEW_BAG,
|
||||
TM_UPLOAD_FIELD,
|
||||
select_pending_records,
|
||||
)
|
||||
@@ -113,6 +114,53 @@ class GuangHeStoreRoutingTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "未知光合店铺"):
|
||||
select_pending_records([record()], "unknown")
|
||||
|
||||
def test_new_bag_uploads_only_obsidian_series_regardless_of_tm_status(self):
|
||||
unchecked = record(gender="女", uploaded=False)
|
||||
unchecked["代拍产品"] = "黑曜石托特包"
|
||||
checked = record(gender="女", uploaded=True)
|
||||
checked["代拍产品"] = "黑曜石吐司包"
|
||||
records = [
|
||||
unchecked,
|
||||
checked,
|
||||
record(gender="男"),
|
||||
]
|
||||
|
||||
selected = select_pending_records(records, STORE_NEW_BAG)
|
||||
|
||||
self.assertEqual(selected, [unchecked, checked])
|
||||
|
||||
def test_new_bag_skips_mixed_and_non_obsidian_styles(self):
|
||||
obsidian = record()
|
||||
obsidian["代拍产品"] = " 黑曜石电脑包 "
|
||||
mixed = record()
|
||||
mixed["代拍产品"] = "黑曜石饺子 / 极星双肩"
|
||||
other = record()
|
||||
other["代拍产品"] = "极星双肩"
|
||||
skipped = record()
|
||||
skipped["代拍产品"] = "黑曜石饺子 / 未到货"
|
||||
no_product = record()
|
||||
other_brand = record(brand="OZKO")
|
||||
other_brand["代拍产品"] = "黑曜石托特包"
|
||||
no_attachment = record(attachments=False)
|
||||
no_attachment["代拍产品"] = "黑曜石托特包"
|
||||
|
||||
selected = select_pending_records(
|
||||
[obsidian, mixed, other, skipped, no_product, other_brand, no_attachment],
|
||||
STORE_NEW_BAG,
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [obsidian])
|
||||
|
||||
def test_new_bag_rejects_republish_from_date(self):
|
||||
rec = record(date="2026-06-03 00:00:00")
|
||||
rec["代拍产品"] = "黑曜石托特包"
|
||||
with self.assertRaisesRegex(ValueError, "旗舰店"):
|
||||
select_pending_records(
|
||||
[rec],
|
||||
STORE_NEW_BAG,
|
||||
republish_from_date="2026-06-03",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -119,6 +119,9 @@ def test_upsert_uses_advertiser_date_product_business_key(monkeypatch) -> None:
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
# resolve_style_ids 走独立查询;本测试只验证 SQL 形状与款式匹配列,
|
||||
# 用空映射避免和上面的 style catalog 假数据串台。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
@@ -162,6 +165,7 @@ def test_upsert_falls_back_to_exact_product_name_style_match(monkeypatch) -> Non
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
|
||||
@@ -274,3 +274,51 @@ def test_parse_files_still_fails_when_both_reports_are_empty(tmp_path: Path) ->
|
||||
advertiser_pin="光影行星旗舰店霖",
|
||||
shop_account="光影行星研发",
|
||||
)
|
||||
|
||||
|
||||
def test_sku_match_prefers_explicit_jd_spus_owner_over_sub_style_name() -> None:
|
||||
"""子款式名称更长时,仍按 dim_style.jd_spus 归属归到父款式(宙斯系)。"""
|
||||
|
||||
result = importer.resolve_sku_match(
|
||||
"光影行星(GYXX)宙斯3双肩背包男士电脑包2026新款 【宙斯3】曜夜黑/大",
|
||||
style_catalog=[
|
||||
{"style_name": "宙斯", "jd_spus": ["spu-zeus3", "spu-zair"]},
|
||||
{"style_name": "宙斯3", "jd_spus": []},
|
||||
{"style_name": "宙斯Z Air", "jd_spus": []},
|
||||
],
|
||||
full_records=[
|
||||
{"spu_id": "spu-zeus3", "spu_name": "光影行星(GYXX)宙斯3双肩背包男士电脑包"},
|
||||
],
|
||||
)
|
||||
|
||||
assert result["status"] == "matched"
|
||||
assert result["spu_id"] == "spu-zeus3"
|
||||
assert result["style_name"] == "宙斯"
|
||||
assert "dim_style.jd_spus" in result["reason"]
|
||||
|
||||
|
||||
def test_sku_match_keeps_style_when_spu_owner_is_itself() -> None:
|
||||
result = importer.resolve_sku_match(
|
||||
"光影行星(GYXX)盖亚C1斜挎包 曜夜黑",
|
||||
style_catalog=[{"style_name": "盖亚斜挎", "jd_spus": ["spu-gy"]}],
|
||||
full_records=[{"spu_id": "spu-gy", "spu_name": "盖亚C1斜挎包"}],
|
||||
)
|
||||
|
||||
assert result["style_name"] == "盖亚斜挎"
|
||||
|
||||
|
||||
def test_rollup_map_moves_sub_style_to_parent() -> None:
|
||||
rolled = importer._apply_rollup(
|
||||
{"status": "matched", "style_name": "宙斯3", "reason": "unique SPU name containment"},
|
||||
{"宙斯3": "宙斯"},
|
||||
)
|
||||
|
||||
assert rolled["style_name"] == "宙斯"
|
||||
assert "父款式" in rolled["reason"]
|
||||
|
||||
|
||||
def test_default_rollup_config_declares_zeus_family() -> None:
|
||||
rollup = importer.load_rollup_map("jd")
|
||||
|
||||
assert rollup.get("宙斯3") == "宙斯"
|
||||
assert rollup.get("宙斯Z Air") == "宙斯"
|
||||
@@ -99,7 +99,7 @@ def test_find_report_files_uses_current_manifest_csv(monkeypatch, tmp_path: Path
|
||||
assert importer.find_report_files(target_date) == [current_csv.resolve()]
|
||||
|
||||
|
||||
def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
def test_upsert_writes_dedicated_table_with_style_and_metrics(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows, *, template=None):
|
||||
@@ -108,7 +108,9 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
captured["template"] = template
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
written = product_db.upsert_tmall_product_ad_daily_metrics(
|
||||
_Connection(),
|
||||
[
|
||||
{
|
||||
@@ -119,6 +121,7 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
"spend": "90.25",
|
||||
"attributed_deal_amount": "730.02",
|
||||
"roi": "8.09",
|
||||
"impressions": 1000,
|
||||
},
|
||||
"source_file": "商品报表.csv",
|
||||
}
|
||||
@@ -127,16 +130,22 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
|
||||
assert written == 1
|
||||
sql = str(captured["sql"])
|
||||
assert "INSERT INTO product_daily_metrics" in sql
|
||||
assert "jsonb_set" in sql
|
||||
assert "tm_product_ad_daily_metrics" not in sql
|
||||
assert captured["template"] == "('tm', %s, %s, %s, %s::jsonb, %s, NOW())"
|
||||
assert "INSERT INTO tm_product_ad_daily_metrics" in sql
|
||||
assert "product_daily_metrics" not in sql
|
||||
assert "jsonb_set" not in sql
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["spend"] == "90.25"
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_id"
|
||||
assert row[1] == "1001"
|
||||
assert row[3] == "极星"
|
||||
assert row[4] == "product_id"
|
||||
assert str(row[5]) == "90.25"
|
||||
assert str(row[8]) == "730.02"
|
||||
assert str(row[9]) == "8.09"
|
||||
# 15 个业务列 + style_id(款式主键统一 2026-11-01)
|
||||
assert captured["template"].count("%s") == 16
|
||||
assert row[-1] is None # resolve_style_ids 被 mock 成空映射
|
||||
metrics = json.loads(row[11])
|
||||
assert metrics["impressions"] == 1000
|
||||
assert "style_name" not in metrics
|
||||
|
||||
|
||||
def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> None:
|
||||
@@ -154,7 +163,9 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) ->
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
written = product_db.upsert_tmall_product_ad_daily_metrics(
|
||||
_NameFallbackConnection(),
|
||||
[
|
||||
{
|
||||
@@ -169,9 +180,41 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) ->
|
||||
|
||||
assert written == 1
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_name"
|
||||
assert row[3] == "极星"
|
||||
assert row[4] == "product_name"
|
||||
|
||||
|
||||
def test_upsert_dedupes_repeated_product_day(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, _sql, rows, *, template=None):
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
# 款式主键由 resolve_style_ids 单独查询;本测试只验证列顺序与款式匹配结果。
|
||||
monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {})
|
||||
written = product_db.upsert_tmall_product_ad_daily_metrics(
|
||||
_Connection(),
|
||||
[
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "older",
|
||||
"ad_metrics": {"spend": "1.00"},
|
||||
},
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "latest",
|
||||
"ad_metrics": {"spend": "2.00"},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
row = captured["rows"][0]
|
||||
assert row[2] == "latest"
|
||||
assert str(row[5]) == "2.00"
|
||||
|
||||
|
||||
def test_import_date_filter_and_dry_run(tmp_path: Path, capsys) -> None:
|
||||
|
||||
@@ -96,6 +96,84 @@ class MainImageWorkflowPreflightTests(unittest.TestCase):
|
||||
jd_runner.load_dynamic_styles()
|
||||
|
||||
|
||||
class MainImageMissingTableAddressTests(unittest.TestCase):
|
||||
"""缺主图飞书表地址:照常采图入库,只是不写表(不报错、不阻断)。"""
|
||||
|
||||
def _write_tmall_payload(self, root: Path, style: str) -> None:
|
||||
style_dir = root / "2026-08-02" / style
|
||||
images = style_dir / "tm_images"
|
||||
images.mkdir(parents=True)
|
||||
(images / "a.png").write_bytes(b"img")
|
||||
(style_dir / "wanxiang_creative_main_images.json").write_text(
|
||||
json.dumps({"creatives": [{"image_key": "a.png", "impressions": 10, "clicks": 2}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _write_jd_payload(self, root: Path, style: str) -> None:
|
||||
style_dir = root / "2026-08-02" / style
|
||||
images = style_dir / "jd_images"
|
||||
images.mkdir(parents=True)
|
||||
(images / "a.jpg").write_bytes(b"img")
|
||||
(style_dir / "jd_main_images.json").write_text(
|
||||
json.dumps({"records": [{"image_key": "a.jpg", "spu": "JD-1", "impressions": 5, "clicks": 1}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def test_tmall_style_without_address_only_lands_in_postgres(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._write_tmall_payload(root, "缺地址款")
|
||||
pg_records: list = []
|
||||
with (
|
||||
patch.object(tmall_insert, "artifact_relative_path", return_value="a.png"),
|
||||
patch.object(tmall_insert, "discover_fields") as discover,
|
||||
patch.object(tmall_insert, "run_lark") as run_lark,
|
||||
):
|
||||
result = tmall_insert.process_product(
|
||||
"缺地址款",
|
||||
{"display_name": "缺地址款", "platform": "天猫", "enabled": True},
|
||||
"2026-08-02",
|
||||
root,
|
||||
dry_run=False,
|
||||
pg_records=pg_records,
|
||||
)
|
||||
|
||||
discover.assert_not_called()
|
||||
run_lark.assert_not_called()
|
||||
self.assertEqual(result["found"], 1)
|
||||
self.assertEqual(result["inserted"], 0)
|
||||
self.assertEqual(result["errors"], [])
|
||||
self.assertEqual([r["style_name"] for r in pg_records], ["缺地址款"])
|
||||
self.assertIsNone(pg_records[0]["bitable_base"])
|
||||
self.assertFalse(pg_records[0]["uploaded_to_bitable"])
|
||||
|
||||
def test_jd_style_without_address_only_lands_in_postgres(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._write_jd_payload(root, "缺地址款")
|
||||
pg_records: list = []
|
||||
with (
|
||||
patch.object(jd_insert, "artifact_relative_path", return_value="a.jpg"),
|
||||
patch.object(jd_insert, "discover_fields") as discover,
|
||||
patch.object(jd_insert, "batch_create_records") as create,
|
||||
):
|
||||
result = jd_insert.process_style(
|
||||
"缺地址款",
|
||||
{"display_name": "缺地址款", "platform": "京东", "enabled": True},
|
||||
"2026-08-02",
|
||||
root,
|
||||
dry_run=False,
|
||||
pg_records=pg_records,
|
||||
)
|
||||
|
||||
discover.assert_not_called()
|
||||
create.assert_not_called()
|
||||
self.assertEqual(result["found"], 1)
|
||||
self.assertEqual(result["inserted"], 0)
|
||||
self.assertEqual([r["style_name"] for r in pg_records], ["缺地址款"])
|
||||
self.assertIsNone(pg_records[0]["bitable_base"])
|
||||
|
||||
|
||||
class TmallCleanupTests(unittest.TestCase):
|
||||
def test_locked_duplicate_is_retried_without_console_encoding_failure(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
@@ -234,7 +312,9 @@ class MainImageDatabaseTests(unittest.TestCase):
|
||||
}
|
||||
for platform in ("jd", "tm")
|
||||
]
|
||||
with patch.object(product_db, "execute_values") as execute_values:
|
||||
with patch.object(product_db, "execute_values") as execute_values, patch.object(
|
||||
product_db, "resolve_style_ids", return_value={}
|
||||
):
|
||||
count = product_db.upsert_main_image_creatives(Connection(), records)
|
||||
|
||||
self.assertEqual(count, 2)
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Tests for the weekly points-exchange ranking workflow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
MODULE_PATH = (
|
||||
PROJECT_ROOT
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "points_exchange_weekly.py"
|
||||
)
|
||||
|
||||
ROWS: list[dict[str, object]] = [
|
||||
{"itemId": "A", "title": "物资【电源包】", "skuId": "s1", "skuCode": "c1", "quantity": 491},
|
||||
{"itemId": "A", "title": "物资【电源包】", "skuId": "s2", "skuCode": "c2", "quantity": 472},
|
||||
{"itemId": "B", "title": "物资【耳机包】", "skuId": "s3", "skuCode": "c3", "quantity": 133},
|
||||
{
|
||||
"itemId": "B",
|
||||
"title": "GYXX-物资【耳机包】",
|
||||
"skuId": "s4",
|
||||
"skuCode": "c4",
|
||||
"quantity": 191,
|
||||
},
|
||||
{"itemId": "C", "title": "物资【零兑换】", "skuId": "s5", "skuCode": "c5", "quantity": 0},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(name="module")
|
||||
def module_fixture() -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"points_exchange_weekly_under_test", MODULE_PATH
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_default_window_is_last_completed_natural_week(
|
||||
module: ModuleType, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-10-26")
|
||||
|
||||
start, end = module.resolve_window(None, None)
|
||||
|
||||
assert (start, end) == (date(2026, 10, 19), date(2026, 10, 25))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("start_value", "end_value", "expected"),
|
||||
[
|
||||
("2026-10-19", None, (date(2026, 10, 19), date(2026, 10, 25))),
|
||||
(None, "2026-10-27", (date(2026, 10, 21), date(2026, 10, 27))),
|
||||
("2026-10-01", "2026-10-27", (date(2026, 10, 1), date(2026, 10, 27))),
|
||||
],
|
||||
)
|
||||
def test_explicit_window_overrides_the_default(
|
||||
module: ModuleType,
|
||||
start_value: str | None,
|
||||
end_value: str | None,
|
||||
expected: tuple[date, date],
|
||||
) -> None:
|
||||
assert module.resolve_window(start_value, end_value) == expected
|
||||
|
||||
|
||||
def test_window_rejects_reversed_or_oversized_ranges(module: ModuleType) -> None:
|
||||
with pytest.raises(ValueError, match="before start"):
|
||||
module.resolve_window("2026-10-27", "2026-10-01")
|
||||
with pytest.raises(ValueError, match="supported maximum"):
|
||||
module.resolve_window("2026-01-01", "2026-12-31")
|
||||
with pytest.raises(ValueError, match="YYYY-MM-DD"):
|
||||
module.resolve_window("2026/10/01", None)
|
||||
|
||||
|
||||
def test_items_aggregate_every_sku_and_rank_by_quantity(module: ModuleType) -> None:
|
||||
items = module.aggregate_items(ROWS)
|
||||
|
||||
assert [item.item_id for item in items] == ["A", "B"]
|
||||
assert items[0].quantity == 963
|
||||
assert items[0].sku_count == 2
|
||||
assert items[0].title == "物资【电源包】"
|
||||
assert items[1].quantity == 324
|
||||
assert items[1].sku_count == 2
|
||||
assert items[1].title == "GYXX-物资【耳机包】"
|
||||
assert module._quantity_text(items[0].quantity) == "963"
|
||||
|
||||
|
||||
def test_item_without_identifier_falls_back_to_sku(module: ModuleType) -> None:
|
||||
items = module.aggregate_items(
|
||||
[{"title": "物资【无 ID】", "skuCode": "c9", "quantity": 3}]
|
||||
)
|
||||
|
||||
assert items[0].item_id == "sku:c9"
|
||||
assert items[0].sku_count == 1
|
||||
|
||||
|
||||
def test_non_numeric_quantity_is_rejected(module: ModuleType) -> None:
|
||||
with pytest.raises(ValueError, match="invalid quantity"):
|
||||
module.aggregate_items([{"itemId": "A", "quantity": "abc"}])
|
||||
|
||||
|
||||
def test_card_contains_ranked_rows_and_window_summary(module: ModuleType) -> None:
|
||||
items = module.aggregate_items(ROWS)
|
||||
card = module._card(
|
||||
items,
|
||||
start=date(2026, 10, 19),
|
||||
end=date(2026, 10, 25),
|
||||
total_quantity=sum((item.quantity for item in items), Decimal("0")),
|
||||
item_count=len(items),
|
||||
page=1,
|
||||
pages=1,
|
||||
)
|
||||
|
||||
assert card["header"]["title"]["content"] == "私域积分兑换周报"
|
||||
assert card["header"]["template"] == "blue"
|
||||
summary = card["body"]["elements"][0]["content"]
|
||||
assert "2026-10-19 ~ 2026-10-25" in summary
|
||||
assert "**兑换商品**:2 个" in summary
|
||||
assert "**兑换总量**:1287" in summary
|
||||
table = card["body"]["elements"][1]
|
||||
assert [column["display_name"] for column in table["columns"]] == [
|
||||
"排名",
|
||||
"商品",
|
||||
"SKU 数",
|
||||
"兑换数量",
|
||||
]
|
||||
assert table["rows"][0] == {
|
||||
"rank": "1",
|
||||
"title": "物资【电源包】",
|
||||
"sku_count": "2",
|
||||
"quantity": "963",
|
||||
}
|
||||
assert table["rows"][1]["rank"] == "2"
|
||||
|
||||
# Feishu rejects the whole card (ErrCode 200912) when any table column is
|
||||
# narrower than 80px, so keep every fixed-width column at or above it.
|
||||
for column in table["columns"]:
|
||||
width = str(column["width"])
|
||||
if width.endswith("px"):
|
||||
assert int(width.removesuffix("px")) >= 80, (
|
||||
f"Feishu table column {column['name']} is narrower than 80px"
|
||||
)
|
||||
|
||||
|
||||
def test_run_sends_one_card_to_every_stock_alert_recipient(
|
||||
module: ModuleType, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
sent: list[dict[str, object]] = []
|
||||
queries: list[dict[str, str]] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"query_points_exchange",
|
||||
lambda **kwargs: queries.append(kwargs) or ROWS,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"resolve_notification_route",
|
||||
lambda workflow_id, defaults: SimpleNamespace(
|
||||
enabled=True,
|
||||
open_ids=("ou_aaaaaaaa", "ou_bbbbbbbb"),
|
||||
app_profile="hermes-analyzer",
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"send_lark_bot_message",
|
||||
lambda **kwargs: sent.append(kwargs) or {"message_id": "om_1"},
|
||||
)
|
||||
|
||||
assert module.run(["--from", "2026-10-19", "--to", "2026-10-25"]) == 0
|
||||
|
||||
assert queries == [
|
||||
{"start_time": "2026-10-19 00:00:00", "end_time": "2026-10-25 23:59:59"}
|
||||
]
|
||||
assert [call["user_id"] for call in sent] == ["ou_aaaaaaaa", "ou_bbbbbbbb"]
|
||||
assert all(call["msg_type"] == "interactive" for call in sent)
|
||||
idempotency_keys = {call["idempotency_key"] for call in sent}
|
||||
assert len(idempotency_keys) == 1
|
||||
key = idempotency_keys.pop()
|
||||
assert key.startswith("product.points_exchange.weekly:2026-10-19~2026-10-25:1:")
|
||||
card = sent[0]["content"]
|
||||
assert card["header"]["title"]["content"] == "私域积分兑换周报"
|
||||
assert len(card["body"]["elements"][1]["rows"]) == 2
|
||||
|
||||
|
||||
def test_dry_run_skips_feishu_delivery(
|
||||
module: ModuleType, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
sent: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(module, "query_points_exchange", lambda **kwargs: ROWS)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"send_lark_bot_message",
|
||||
lambda **kwargs: sent.append(kwargs) or {"message_id": "om_1"},
|
||||
)
|
||||
|
||||
assert module.run(["--from", "2026-10-19", "--dry-run"]) == 0
|
||||
assert sent == []
|
||||
|
||||
|
||||
def test_empty_window_does_not_notify(
|
||||
module: ModuleType, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
sent: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(module, "query_points_exchange", lambda **kwargs: [])
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"send_lark_bot_message",
|
||||
lambda **kwargs: sent.append(kwargs) or {"message_id": "om_1"},
|
||||
)
|
||||
|
||||
assert module.run(["--from", "2026-10-19"]) == 0
|
||||
assert sent == []
|
||||
|
||||
|
||||
def test_missing_recipients_is_a_hard_failure(
|
||||
module: ModuleType, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(module, "query_points_exchange", lambda **kwargs: ROWS)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"resolve_notification_route",
|
||||
lambda workflow_id, defaults: SimpleNamespace(
|
||||
enabled=True, open_ids=(), app_profile="hermes-analyzer"
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="没有配置接收人"):
|
||||
module.run(["--from", "2026-10-19"])
|
||||
@@ -12,6 +12,13 @@ class _Cursor:
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
# db 层按商品 ID 反查 dim_style 时用;空映射表示这些商品都没建档。
|
||||
def execute(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
return []
|
||||
|
||||
|
||||
class _Connection:
|
||||
def cursor(self):
|
||||
|
||||
@@ -11,7 +11,6 @@ import collect_persona_to_bitable as tm_persona_worker
|
||||
import dy_product_scraping as dy_product
|
||||
import orchestrate_daily_collection as daily
|
||||
import orchestrate_market_rank_collection as market_rank
|
||||
import orchestrate_review_collection as reviews
|
||||
import run_alerts_with_retry as alerts
|
||||
import run_daily_persona as persona
|
||||
import run_weekly_jd_main_image as jd_main_image
|
||||
@@ -191,24 +190,6 @@ def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None:
|
||||
_assert_child_binding(environment)
|
||||
|
||||
|
||||
def test_review_run_step_rebinds_to_command_target(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(reviews, "environment_for_child_script", _fake_rebind(observed))
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
observed["environment"] = kwargs["env"]
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(reviews.subprocess, "run", fake_run)
|
||||
target = reviews.PROJECT_ROOT / "dy_product_scraping.py"
|
||||
|
||||
result = reviews.run_step("dy", ["python", str(target)], critical=True)
|
||||
|
||||
assert result["code"] == 0
|
||||
assert observed["target"] == target
|
||||
_assert_child_binding(observed["environment"])
|
||||
|
||||
|
||||
def test_main_image_launchers_rebind_to_command_target(monkeypatch) -> None:
|
||||
for launcher in (tm_main_image, jd_main_image):
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
"""同系列 ERP 编码归集到锚点 + 销量表地址不再挡采集。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from gyxx_flow.modules.product_commerce.config.style_config_loader import (
|
||||
StyleConfigLoader,
|
||||
)
|
||||
|
||||
SALES_URL = "https://example.feishu.cn/base/sales?table=tblSales"
|
||||
|
||||
|
||||
def _series_rows() -> list[dict]:
|
||||
"""宙斯系列:锚点只做占位(ERP 空),编码散落在两个子款上。"""
|
||||
|
||||
return [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "宙斯",
|
||||
"erp_codes": [],
|
||||
"item_ids": ["TM-ZS"],
|
||||
"sales_bitable_url": SALES_URL,
|
||||
"series_name": "宙斯",
|
||||
"is_series_anchor": True,
|
||||
},
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "宙斯-10244",
|
||||
"erp_codes": ["10244"],
|
||||
"item_ids": [],
|
||||
"series_name": "宙斯",
|
||||
},
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "宙斯3",
|
||||
"erp_codes": ["10484"],
|
||||
"item_ids": [],
|
||||
"series_name": "宙斯",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _loader(flags: dict[str, bool] | None = None) -> StyleConfigLoader:
|
||||
def rows() -> list[dict]:
|
||||
data = _series_rows()
|
||||
if flags:
|
||||
for row in data:
|
||||
row["collect_enabled"] = flags.get(row["style_name"], True)
|
||||
return data
|
||||
|
||||
return StyleConfigLoader(records_provider=rows)
|
||||
|
||||
|
||||
def test_anchor_takes_union_of_series_erp_codes() -> None:
|
||||
styles = _loader().load()["styles"]
|
||||
|
||||
assert styles["宙斯"]["erp_codes"] == ["10244", "10484"]
|
||||
assert styles["宙斯"]["series"]["erp_owner"] == "宙斯"
|
||||
|
||||
|
||||
def test_child_keeps_own_codes_for_lookup() -> None:
|
||||
styles = _loader().load()["styles"]
|
||||
|
||||
assert styles["宙斯3"]["erp_codes"] == ["10484"]
|
||||
assert styles["宙斯3"]["series"]["own_erp_codes"] == ["10484"]
|
||||
assert styles["宙斯3"]["series"]["erp_owner"] == "宙斯"
|
||||
|
||||
|
||||
def test_anchor_is_the_only_erp_collection_unit() -> None:
|
||||
loader = _loader()
|
||||
|
||||
erp_daily = loader.get_erp_daily_styles()
|
||||
|
||||
assert erp_daily["collectable"] == ["宙斯"]
|
||||
assert {"style": "宙斯-10244", "reason": "ERP 已由同系列「宙斯」归集采集"} in erp_daily["skipped"]
|
||||
assert loader.get_daily_styles()["tm"]["collectable"] == ["宙斯"]
|
||||
|
||||
|
||||
def test_anchor_disabled_closes_whole_series_erp() -> None:
|
||||
# 系列级开关语义:开关以锚点为准,锚点关掉后子款不会因为自己还有编码
|
||||
# 就绕过系列开关自行采集(旧的白名单语义下子款会自采)。
|
||||
loader = _loader({"宙斯": False})
|
||||
|
||||
erp_daily = loader.get_erp_daily_styles()
|
||||
|
||||
assert erp_daily["collectable"] == []
|
||||
for name in ("宙斯", "宙斯-10244", "宙斯3"):
|
||||
assert {"style": name, "reason": "已标记不采集"} in erp_daily["skipped"]
|
||||
|
||||
|
||||
def test_disabled_child_erp_codes_are_excluded_from_anchor() -> None:
|
||||
"""锚点为 true 时,自身 collect_enabled=false 的子款不并入 ERP 聚合。"""
|
||||
|
||||
styles = _loader({"宙斯3": False}).load()["styles"]
|
||||
|
||||
assert styles["宙斯"]["erp_codes"] == ["10244"]
|
||||
assert styles["宙斯"]["series"]["erp_excluded"] == ["宙斯3"]
|
||||
# 子款保留自己的原始编码,只是不再参与聚合
|
||||
assert styles["宙斯3"]["series"]["own_erp_codes"] == ["10484"]
|
||||
|
||||
|
||||
def test_enabled_child_keeps_contributing_erp_codes() -> None:
|
||||
styles = _loader({"宙斯-10244": False}).load()["styles"]
|
||||
|
||||
assert styles["宙斯"]["erp_codes"] == ["10484"]
|
||||
assert styles["宙斯"]["series"]["erp_excluded"] == ["宙斯-10244"]
|
||||
|
||||
|
||||
def test_anchor_without_any_enabled_child_code_stops_erp_collection() -> None:
|
||||
"""子款全部关闭时锚点没有可采编码,整条系列不采 ERP。"""
|
||||
|
||||
erp_daily = _loader({"宙斯-10244": False, "宙斯3": False}).get_erp_daily_styles()
|
||||
|
||||
assert erp_daily["collectable"] == []
|
||||
assert {"style": "宙斯", "reason": "缺 ERP款式编码"} in erp_daily["skipped"]
|
||||
|
||||
|
||||
def test_injected_provider_never_writes_production_cache(tmp_path) -> None:
|
||||
"""注入假数据的 loader 不能写生产快照,否则数据库抖动时会按假款式采集。"""
|
||||
|
||||
cache = tmp_path / ".style_config_cache.json"
|
||||
loader = StyleConfigLoader(records_provider=_series_rows, cache_path=cache)
|
||||
|
||||
loader.load()
|
||||
|
||||
assert loader.cache_enabled is False
|
||||
assert not cache.exists()
|
||||
|
||||
|
||||
def test_default_provider_enables_cache(tmp_path) -> None:
|
||||
loader = StyleConfigLoader(cache_path=tmp_path / "cache.json")
|
||||
|
||||
assert loader.cache_enabled is True
|
||||
|
||||
|
||||
def test_single_style_series_has_no_erp_owner() -> None:
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "独苗",
|
||||
"erp_codes": ["10500"],
|
||||
"item_ids": ["TM-1"],
|
||||
"series_name": "独苗",
|
||||
"is_series_anchor": True,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert "erp_owner" not in loader.load()["styles"]["独苗"]["series"]
|
||||
|
||||
|
||||
def test_missing_sales_bitable_still_collects_but_skips_table_write() -> None:
|
||||
"""简行三款的场景:平台编码与 ERP 都齐,只缺销量飞书表。"""
|
||||
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "简行手提",
|
||||
"erp_codes": ["10523"],
|
||||
"item_ids": ["TM-1"],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert loader.get_erp_daily_styles()["collectable"] == ["简行手提"]
|
||||
assert loader.get_daily_styles()["tm"]["collectable"] == ["简行手提"]
|
||||
# 写表配置仍然跳过该款:没有销量表地址就不写飞书
|
||||
assert loader.get_bitable_style_map() == {}
|
||||
|
||||
|
||||
def test_child_platform_ids_are_merged_into_anchor() -> None:
|
||||
"""逐星GT 的场景:子款自己填的商品 ID 并入锚点,整系列由一个单元采集。"""
|
||||
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "逐星",
|
||||
"erp_codes": [],
|
||||
"item_ids": ["TM-A", "TM-B"],
|
||||
"series_name": "逐星",
|
||||
"is_series_anchor": True,
|
||||
},
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "逐星GT",
|
||||
"erp_codes": ["10494"],
|
||||
"item_ids": ["TM-GT"],
|
||||
"series_name": "逐星",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
styles = loader.load()["styles"]
|
||||
erp_daily = loader.get_erp_daily_styles()
|
||||
daily = loader.get_daily_styles()
|
||||
|
||||
# 子款编码聚合去重后挂锚点,顺序为「锚点原有 → 子款」
|
||||
assert styles["逐星"]["tm"]["item_ids"] == ["TM-A", "TM-B", "TM-GT"]
|
||||
# 子款保留原始编码供诊断,但不再单独采集
|
||||
assert styles["逐星GT"]["series"]["platform_ids"]["tm"] == {
|
||||
"逐星": ["TM-A", "TM-B", "TM-GT"],
|
||||
"逐星GT": ["TM-GT"],
|
||||
}
|
||||
assert erp_daily["collectable"] == ["逐星"]
|
||||
assert {"style": "逐星GT", "reason": "ERP 已由同系列「逐星」归集采集"} in erp_daily["skipped"]
|
||||
assert daily["tm"]["collectable"] == ["逐星"]
|
||||
assert {
|
||||
"style": "逐星GT",
|
||||
"reason": "缺 tm商品编码(同系列「逐星」已在采)",
|
||||
} in daily["tm"]["skipped"]
|
||||
|
||||
|
||||
def test_platform_ids_filled_on_child_are_lifted_to_anchor() -> None:
|
||||
"""锚点平台编码为空、子款填了:聚合后由锚点承接,采集口径不变。"""
|
||||
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "某系列",
|
||||
"erp_codes": ["1001"],
|
||||
"item_ids": [],
|
||||
"series_name": "某系列",
|
||||
"is_series_anchor": True,
|
||||
},
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "某系列-2",
|
||||
"erp_codes": ["1002"],
|
||||
"item_ids": ["TM-2"],
|
||||
"series_name": "某系列",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
styles = loader.load()["styles"]
|
||||
daily = loader.get_daily_styles()
|
||||
|
||||
assert styles["某系列"]["tm"]["item_ids"] == ["TM-2"]
|
||||
assert daily["tm"]["collectable"] == ["某系列"]
|
||||
assert daily["tm"]["skipped"][0]["style"] == "某系列-2"
|
||||
|
||||
|
||||
def test_series_addresses_aggregate_from_child_when_anchor_empty() -> None:
|
||||
"""子款误填飞书地址、锚点为空时,地址上浮到锚点并下发全系列。"""
|
||||
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "地址系列",
|
||||
"erp_codes": [],
|
||||
"item_ids": ["TM-1"],
|
||||
"series_name": "地址系列",
|
||||
"is_series_anchor": True,
|
||||
},
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "地址系列-2",
|
||||
"erp_codes": ["1002"],
|
||||
"item_ids": [],
|
||||
"sales_bitable_url": SALES_URL,
|
||||
"series_name": "地址系列",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
styles = loader.load()["styles"]
|
||||
|
||||
assert styles["地址系列"]["sales_bitable"]["base_token"] == "sales"
|
||||
assert styles["地址系列-2"]["sales_bitable"]["base_token"] == "sales"
|
||||
# 地址上浮后锚点成为唯一采集单元,写表清单里只有锚点
|
||||
assert set(loader.get_bitable_style_map()) == {"地址系列"}
|
||||
|
||||
|
||||
def test_series_address_conflict_keeps_anchor_target(capsys) -> None:
|
||||
"""同字段多套表:保留锚点那套并告警,避免同系列拆到两张表。"""
|
||||
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "冲突系列",
|
||||
"erp_codes": ["1001"],
|
||||
"item_ids": ["TM-1"],
|
||||
"sales_bitable_url": SALES_URL,
|
||||
"series_name": "冲突系列",
|
||||
"is_series_anchor": True,
|
||||
},
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "冲突系列-2",
|
||||
"erp_codes": ["1002"],
|
||||
"item_ids": [],
|
||||
"sales_bitable_url": "https://example.feishu.cn/base/child?table=tblChild",
|
||||
"series_name": "冲突系列",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
styles = loader.load()["styles"]
|
||||
|
||||
assert styles["冲突系列"]["sales_bitable"]["base_token"] == "sales"
|
||||
assert styles["冲突系列-2"]["sales_bitable"]["base_token"] == "sales"
|
||||
assert "个不同表" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_child_ids_covered_by_anchor_are_left_to_the_anchor() -> None:
|
||||
"""锚点聚合了全系列商品 ID 时,子款不重复采同一批编码。"""
|
||||
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "聚合系列",
|
||||
"erp_codes": ["2001"],
|
||||
"item_ids": ["TM-1", "TM-2"],
|
||||
"series_name": "聚合系列",
|
||||
"is_series_anchor": True,
|
||||
},
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "聚合系列-2",
|
||||
"erp_codes": ["2002"],
|
||||
"item_ids": ["TM-2"],
|
||||
"series_name": "聚合系列",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
daily = loader.get_daily_styles()
|
||||
|
||||
assert daily["tm"]["collectable"] == ["聚合系列"]
|
||||
assert {
|
||||
"style": "聚合系列-2",
|
||||
"reason": "缺 tm商品编码(同系列「聚合系列」已在采)",
|
||||
} in daily["tm"]["skipped"]
|
||||
|
||||
|
||||
def test_anchor_owning_erp_and_all_platform_ids_owns_every_gate() -> None:
|
||||
"""正常建档形态:锚点填 ERP + 全平台编码,子款只填 ERP。"""
|
||||
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "正常系列",
|
||||
"erp_codes": ["3001"],
|
||||
"item_ids": ["TM-1"],
|
||||
"series_name": "正常系列",
|
||||
"is_series_anchor": True,
|
||||
},
|
||||
{
|
||||
"platform": "京东旗舰店",
|
||||
"style_name": "正常系列",
|
||||
"erp_codes": [],
|
||||
"item_ids": ["JD-1"],
|
||||
"series_name": "正常系列",
|
||||
"is_series_anchor": True,
|
||||
},
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "正常系列-2",
|
||||
"erp_codes": ["3002"],
|
||||
"item_ids": [],
|
||||
"series_name": "正常系列",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
styles = loader.load()["styles"]
|
||||
daily = loader.get_daily_styles()
|
||||
|
||||
assert styles["正常系列"]["erp_codes"] == ["3001", "3002"]
|
||||
assert loader.get_erp_daily_styles()["collectable"] == ["正常系列"]
|
||||
assert daily["tm"]["collectable"] == ["正常系列"]
|
||||
assert daily["jd"]["collectable"] == ["正常系列"]
|
||||
assert {
|
||||
"style": "正常系列-2",
|
||||
"reason": "缺 tm商品编码(同系列「正常系列」已在采)",
|
||||
} in daily["tm"]["skipped"]
|
||||
|
||||
@@ -116,7 +116,8 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
self.assertNotIn("近 60d", prompt)
|
||||
self.assertIn("逐日经营数据", prompt)
|
||||
self.assertIn("人物画像", prompt)
|
||||
self.assertIn("商品评价", prompt)
|
||||
# product_reviews 下线后数据源改为差评视图,标题同步改成「商品差评」
|
||||
self.assertIn("商品差评", prompt)
|
||||
self.assertIn("主图表现", prompt)
|
||||
self.assertIn("营销笔记、曝光互动与评论", prompt)
|
||||
self.assertIn("肩带偏硬", prompt)
|
||||
|
||||
@@ -62,16 +62,31 @@ def test_database_row_uses_explicit_style_name_not_style_content() -> None:
|
||||
"brand": "",
|
||||
"sales_bitable_url": "",
|
||||
"main_image_bitable_url": "",
|
||||
"creator_bitable_url": "",
|
||||
"self_creator_bitable_url": "",
|
||||
"weekly_note_analysis_url": "",
|
||||
"persona_bitable_url": "",
|
||||
"dy_persona_bitable_url": "",
|
||||
"jd_persona_bitable_url": "",
|
||||
"style_analysis_bitable_url": "",
|
||||
"destinations": [],
|
||||
}
|
||||
"jd_persona_bitable_url": "",
|
||||
"style_analysis_bitable_url": "",
|
||||
"destinations": [],
|
||||
"series_name": "",
|
||||
"is_series_anchor": False,
|
||||
"collect_enabled": True,
|
||||
# 款式主键统一 (2026-11-01): loader 把 dim_style.id 作为正式字段下发
|
||||
"style_id": None,
|
||||
"style_match_method": "",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_erp_only_style_is_skipped_by_every_product_collection_gate() -> None:
|
||||
def test_erp_only_style_collects_erp_but_skips_platform_gates() -> None:
|
||||
"""只有 ERP 编码、没有平台编码的款式:ERP 销量照采,平台流量跳过。
|
||||
|
||||
销量表地址与平台编码都不再挡 ERP 采集——缺表地址只影响写飞书,缺平台
|
||||
编码只影响平台流量。
|
||||
"""
|
||||
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
@@ -89,14 +104,12 @@ def test_erp_only_style_is_skipped_by_every_product_collection_gate() -> None:
|
||||
erp_daily = loader.get_erp_daily_styles()
|
||||
|
||||
assert daily["tm"]["collectable"] == []
|
||||
assert daily["tm"]["skipped"] == [
|
||||
{"style": "研发款", "reason": "缺 平台商品编码/tm商品编码"}
|
||||
assert daily["tm"]["skipped"] == [{"style": "研发款", "reason": "缺 tm商品编码"}]
|
||||
assert erp_daily["collectable"] == ["研发款"]
|
||||
assert erp_daily["skipped"] == []
|
||||
assert loader.get_erp_styles_input() == [
|
||||
{"style_name": "研发款", "erp_style_codes": ["ERP-ONLY"]}
|
||||
]
|
||||
assert erp_daily["collectable"] == []
|
||||
assert erp_daily["skipped"] == [
|
||||
{"style": "研发款", "reason": "缺 平台商品编码"}
|
||||
]
|
||||
assert loader.get_erp_styles_input() == []
|
||||
assert loader.get_main_image_styles() == []
|
||||
|
||||
|
||||
@@ -127,3 +140,31 @@ def test_complete_identity_allows_only_configured_platforms() -> None:
|
||||
assert daily["dy"]["collectable"] == []
|
||||
assert loader.get_erp_daily_styles()["collectable"] == ["完整款"]
|
||||
assert loader.get_main_image_styles() == ["完整款"]
|
||||
|
||||
|
||||
def test_main_image_gate_collects_without_table_address() -> None:
|
||||
"""主图采集只要求身份:缺主图飞书表地址照样采图入库,只是不写表。"""
|
||||
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "有地址款",
|
||||
"erp_codes": ["ERP-1"],
|
||||
"item_ids": ["TM-1"],
|
||||
"main_image_bitable_url": "https://example.feishu.cn/base/image?table=tblImage",
|
||||
},
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style_name": "缺地址款",
|
||||
"erp_codes": ["ERP-2"],
|
||||
"item_ids": ["TM-2"],
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
# 采集目标:两款都要采(身份齐全即可)
|
||||
assert sorted(loader.get_main_image_styles()) == ["有地址款", "缺地址款"]
|
||||
assert loader.get_tm_style_id_map() == {"有地址款": ["TM-1"], "缺地址款": ["TM-2"]}
|
||||
# 写表目标:只有配了地址的那一款
|
||||
assert list(loader.get_bitable_main_image_map()) == ["有地址款"]
|
||||
@@ -0,0 +1,470 @@
|
||||
"""款式主键统一 (style_id = dim_style.id) 的契约测试。
|
||||
|
||||
守住四件事:
|
||||
1. 写入端必须把 style_id 落库(13 张采集表);
|
||||
2. 调用方已带 style_id 时不覆盖,缺省时由 db 层按款名 / 平台商品 ID 兜底;
|
||||
3. 匹配不上要留痕 ``unmatched``,不能静默写成 NULL 了事;
|
||||
4. schema.sql 里必须有列、索引、观测视图,且**不得**建物理外键(style_id 只做逻辑外键),
|
||||
同时不再建已下线的 product_reviews。
|
||||
|
||||
全部测试离线运行(Fake connection),不依赖 PostgreSQL。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import db
|
||||
import pytest
|
||||
|
||||
|
||||
MODULE_ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
)
|
||||
SCHEMA_PATH = MODULE_ROOT / "db" / "schema.sql"
|
||||
|
||||
|
||||
class RecordingCursor:
|
||||
def __init__(self, conn: "RecordingConnection") -> None:
|
||||
self._conn = conn
|
||||
self._rows: list[tuple] = []
|
||||
|
||||
def __enter__(self) -> "RecordingCursor":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> bool:
|
||||
return False
|
||||
|
||||
def execute(self, sql: str, params=None) -> None:
|
||||
self._conn.sqls.append((sql, params))
|
||||
if "unnest" in sql and params:
|
||||
self._rows = [
|
||||
(item, self._conn.product_lookup[item])
|
||||
for item in params[0]
|
||||
if item in self._conn.product_lookup
|
||||
]
|
||||
elif "FROM dim_style" in sql and "ANY" in sql and params:
|
||||
self._rows = [
|
||||
(name, self._conn.name_lookup[name])
|
||||
for name in params[0]
|
||||
if name in self._conn.name_lookup
|
||||
]
|
||||
else:
|
||||
self._rows = []
|
||||
|
||||
def fetchall(self) -> list[tuple]:
|
||||
return self._rows
|
||||
|
||||
def fetchone(self):
|
||||
return None
|
||||
|
||||
@property
|
||||
def rowcount(self) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
class RecordingConnection:
|
||||
def __init__(self, name_lookup=None, product_lookup=None) -> None:
|
||||
self.name_lookup = dict(name_lookup or {})
|
||||
self.product_lookup = dict(product_lookup or {})
|
||||
self.sqls: list[tuple] = []
|
||||
|
||||
def cursor(self) -> RecordingCursor:
|
||||
return RecordingCursor(self)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def captured(monkeypatch):
|
||||
"""拦截 execute_values,暴露最终写入的 SQL 与 rows。"""
|
||||
|
||||
box: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows):
|
||||
box["sql"] = sql
|
||||
box["rows"] = list(rows)
|
||||
|
||||
monkeypatch.setattr(db, "execute_values", fake_execute_values)
|
||||
return box
|
||||
|
||||
|
||||
def test_attach_style_ids_keeps_supplied_identity() -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
records = db.attach_style_ids(
|
||||
conn,
|
||||
[{"style_name": "盖亚斜挎", "style_id": 42, "style_match_method": "dim_style"}],
|
||||
)
|
||||
|
||||
assert records[0]["style_id"] == 42
|
||||
assert records[0]["style_match_method"] == "dim_style"
|
||||
# 已有 style_id 时不应再查库
|
||||
assert conn.sqls == []
|
||||
|
||||
|
||||
def test_attach_style_ids_fills_missing_and_marks_unmatched() -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
records = db.attach_style_ids(
|
||||
conn,
|
||||
[{"style_name": "盖亚斜挎"}, {"style_name": "未建档款"}],
|
||||
)
|
||||
|
||||
assert records[0]["style_id"] == 7
|
||||
assert records[0]["style_match_method"] == db.STYLE_MATCH_BY_NAME
|
||||
assert records[1]["style_id"] is None
|
||||
assert records[1]["style_match_method"] == db.STYLE_MATCH_UNMATCHED
|
||||
|
||||
|
||||
def test_attach_style_ids_does_not_mutate_caller_records() -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
original = [{"style_name": "盖亚斜挎"}]
|
||||
|
||||
db.attach_style_ids(conn, original)
|
||||
|
||||
assert "style_id" not in original[0]
|
||||
|
||||
|
||||
def test_daily_style_metrics_upsert_writes_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
written = db.upsert_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"metric_date": "2026-10-31",
|
||||
"style_name": "盖亚斜挎",
|
||||
"platform": "tm",
|
||||
"sales": 3,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
sql = captured["sql"]
|
||||
assert "style_id" in sql
|
||||
assert "style_match_method" in sql
|
||||
# 冲突时不能把已填的 id 抹成 NULL
|
||||
assert "COALESCE(EXCLUDED.style_id" in sql
|
||||
row = captured["rows"][0]
|
||||
assert row[-2] == 7
|
||||
assert row[-1] == db.STYLE_MATCH_BY_NAME
|
||||
|
||||
|
||||
def test_erp_all_shop_upsert_writes_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"极星双肩": 11})
|
||||
|
||||
db.upsert_erp_all_shop_style_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"metric_date": "2026-10-31",
|
||||
"style_name": "极星双肩",
|
||||
"sales": 10,
|
||||
"selected_shop_count": 2,
|
||||
"source": "jushuitan",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
row = captured["rows"][0]
|
||||
assert row[-2] == 11
|
||||
assert row[-1] == db.STYLE_MATCH_BY_NAME
|
||||
|
||||
|
||||
def test_weekly_metrics_upsert_writes_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"极星双肩": 11})
|
||||
|
||||
db.upsert_weekly_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"week_end": "2026-10-31",
|
||||
"week_start": "2026-10-25",
|
||||
"style_name": "极星双肩",
|
||||
"platform": "tm",
|
||||
"sales": 5,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert "style_id" in captured["sql"]
|
||||
assert captured["rows"][0][-2] == 11
|
||||
|
||||
|
||||
def test_product_daily_metrics_resolves_by_platform_product_id(captured, monkeypatch) -> None:
|
||||
conn = RecordingConnection(product_lookup={"700001": 33})
|
||||
|
||||
written = db.upsert_product_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"platform": "tm",
|
||||
"stat_date": "2026-10-31",
|
||||
"product_id": "700001",
|
||||
"product_name": "GYXX 极星双肩包",
|
||||
"paid_amount": 100,
|
||||
},
|
||||
{
|
||||
"platform": "tm",
|
||||
"stat_date": "2026-10-31",
|
||||
"product_id": "999999",
|
||||
"product_name": "未建档商品",
|
||||
"paid_amount": 5,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 2
|
||||
rows = captured["rows"]
|
||||
assert rows[0][-2] == 33
|
||||
assert rows[0][-1] == db.STYLE_MATCH_BY_PRODUCT
|
||||
assert rows[1][-2] is None
|
||||
assert rows[1][-1] == db.STYLE_MATCH_UNMATCHED
|
||||
|
||||
|
||||
def test_product_daily_metrics_keeps_existing_identity_on_conflict(captured) -> None:
|
||||
conn = RecordingConnection(product_lookup={})
|
||||
|
||||
db.upsert_product_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"platform": "jd",
|
||||
"stat_date": "2026-10-31",
|
||||
"product_id": "abc",
|
||||
"paid_amount": 1,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
sql = captured["sql"]
|
||||
assert "COALESCE(EXCLUDED.style_id, product_daily_metrics.style_id)" in sql
|
||||
|
||||
|
||||
def test_ad_tables_write_style_id(captured) -> None:
|
||||
conn = RecordingConnection(
|
||||
name_lookup={"盖亚斜挎": 7},
|
||||
product_lookup={"5001": 7},
|
||||
)
|
||||
|
||||
db.upsert_jd_product_ad_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"stat_date": "2026-10-31",
|
||||
"advertiser_pin": "pin",
|
||||
"shop_account": "shop",
|
||||
"report_type": "spu",
|
||||
"row_key": "r1",
|
||||
"style_name": "盖亚斜挎",
|
||||
"spend": 12,
|
||||
}
|
||||
],
|
||||
)
|
||||
assert "style_id" in captured["sql"]
|
||||
assert captured["rows"][0][-2] == 7
|
||||
|
||||
db.upsert_douyin_product_ad_daily_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"stat_date": "2026-10-31",
|
||||
"advertiser_id": "adv",
|
||||
"product_id": "5001",
|
||||
"product_name": "盖亚斜挎包",
|
||||
"style_name": "盖亚斜挎",
|
||||
"style_match_method": "product_id",
|
||||
"spend": 8,
|
||||
}
|
||||
],
|
||||
)
|
||||
assert "style_id" in captured["sql"]
|
||||
assert captured["rows"][0][-1] == 7
|
||||
|
||||
|
||||
def test_persona_and_main_image_write_style_id(captured) -> None:
|
||||
conn = RecordingConnection(name_lookup={"盖亚斜挎": 7})
|
||||
|
||||
db.upsert_persona_metrics(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"collect_date": "2026-10-31",
|
||||
"style_name": "盖亚斜挎",
|
||||
"platform": "tm",
|
||||
"payload": {},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert captured["rows"][0][-2] == 7
|
||||
|
||||
db.upsert_main_image_creatives(
|
||||
conn,
|
||||
[
|
||||
{
|
||||
"collect_date": "2026-10-31",
|
||||
"style_name": "盖亚斜挎",
|
||||
"image_key": "k1",
|
||||
}
|
||||
],
|
||||
)
|
||||
assert captured["rows"][0][-2] == 7
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"table",
|
||||
[
|
||||
"daily_style_metrics",
|
||||
"weekly_style_metrics",
|
||||
"erp_all_shop_style_daily_metrics",
|
||||
"daily_persona_metrics",
|
||||
"main_image_creatives",
|
||||
"fact_style_analysis_reports",
|
||||
"decline_events",
|
||||
"fact_alert_history",
|
||||
"jd_self_weekly_sales",
|
||||
"persona_screenshots",
|
||||
"product_daily_metrics",
|
||||
"jd_product_ad_daily_metrics",
|
||||
"dy_product_ad_daily_metrics",
|
||||
],
|
||||
)
|
||||
def test_schema_adds_style_id_column(table: str) -> None:
|
||||
text = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
assert f"ALTER TABLE {table}" in text
|
||||
assert f"ADD COLUMN IF NOT EXISTS style_id BIGINT" in text
|
||||
|
||||
|
||||
def test_schema_defines_observation_views_and_no_physical_foreign_keys() -> None:
|
||||
text = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
|
||||
assert "CREATE OR REPLACE VIEW v_style_unmatched_product" in text
|
||||
assert "CREATE OR REPLACE VIEW v_style_id_coverage" in text
|
||||
assert "CREATE OR REPLACE VIEW v_negative_reviews" in text
|
||||
|
||||
# style_id 只做逻辑外键:库里不得出现指向 dim_style 的物理外键。
|
||||
# 采集语义是「宁可留 NULL 待回填,也不要整批写入失败」,硬约束与之冲突。
|
||||
assert "FOREIGN KEY (style_id) REFERENCES dim_style(id)" not in text
|
||||
assert "REFERENCES dim_style(id)" not in text
|
||||
# 老库重跑 schema 时要能自己清掉早期建的约束(幂等 DROP)
|
||||
assert "DROP CONSTRAINT IF EXISTS" in text
|
||||
assert "_style_id_fk" in text
|
||||
|
||||
|
||||
def test_schema_no_longer_creates_product_reviews() -> None:
|
||||
text = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
|
||||
assert "CREATE TABLE IF NOT EXISTS product_reviews " not in text
|
||||
assert "ALTER TABLE product_reviews " not in text
|
||||
assert "product_reviews_archived_20261101" in text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 回填脚本
|
||||
|
||||
def test_backfill_covers_three_ad_tables_with_right_id_column() -> None:
|
||||
"""广告表的 style_name 大量为空,必须按商品 ID 再回填一遍。"""
|
||||
|
||||
import backfill_style_id as bf
|
||||
|
||||
assert set(bf.AD_PRODUCT_TABLES) == {
|
||||
"tm_product_ad_daily_metrics",
|
||||
"jd_product_ad_daily_metrics",
|
||||
"dy_product_ad_daily_metrics",
|
||||
}
|
||||
assert bf.AD_PRODUCT_TABLES["tm_product_ad_daily_metrics"] == ("product_id", "tm")
|
||||
# 京东广告表的商品列叫 spu_id,不是 product_id
|
||||
assert bf.AD_PRODUCT_TABLES["jd_product_ad_daily_metrics"] == ("spu_id", "jd")
|
||||
assert bf.AD_PRODUCT_TABLES["dy_product_ad_daily_metrics"] == ("product_id", "dy")
|
||||
|
||||
# 款名表里的广告表项不能删,否则 style_name 有值的那部分就回填不了
|
||||
for table in bf.AD_PRODUCT_TABLES:
|
||||
assert table in bf.NAME_TABLES
|
||||
|
||||
|
||||
class _BackfillCursor:
|
||||
def __init__(self, log: list) -> None:
|
||||
self._log = log
|
||||
self._rows: list[tuple] = []
|
||||
self.rowcount = 0
|
||||
|
||||
def __enter__(self) -> "_BackfillCursor":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> bool:
|
||||
return False
|
||||
|
||||
def execute(self, sql: str, params=None) -> None:
|
||||
self._log.append((" ".join(sql.split()), params))
|
||||
if "SELECT DISTINCT" in sql:
|
||||
self._rows = [("1001",), ("9999",)]
|
||||
elif "SELECT id, style_name FROM dim_style" in sql:
|
||||
self._rows = [(7, "赤潮")]
|
||||
elif sql.lstrip().startswith("UPDATE"):
|
||||
self.rowcount = 5 if "style_id IS NULL AND style_match_method IS NULL" not in sql else 0
|
||||
elif "COUNT(*)" in sql:
|
||||
self._rows = [(10, 2)]
|
||||
else:
|
||||
self._rows = []
|
||||
|
||||
def fetchall(self) -> list[tuple]:
|
||||
return self._rows
|
||||
|
||||
def fetchone(self) -> tuple:
|
||||
return self._rows[0] if self._rows else (0, 0)
|
||||
|
||||
|
||||
class _BackfillConn:
|
||||
def __init__(self) -> None:
|
||||
self.log: list = []
|
||||
|
||||
def cursor(self) -> _BackfillCursor:
|
||||
return _BackfillCursor(self.log)
|
||||
|
||||
|
||||
def test_backfill_ad_tables_fills_style_id_and_keeps_platform_style_name(monkeypatch) -> None:
|
||||
"""回填广告表时:style_id 照写,style_name 只在原值为空时补。"""
|
||||
|
||||
import backfill_style_id as bf
|
||||
|
||||
monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7})
|
||||
conn = _BackfillConn()
|
||||
result = bf._backfill_ad_tables_by_product(
|
||||
conn, ["jd_product_ad_daily_metrics"], dry_run=False
|
||||
)[0]
|
||||
|
||||
updates = [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")]
|
||||
payload = next(sql for sql in updates if "NULLIF(BTRIM(style_name)" in sql)
|
||||
assert "COALESCE(NULLIF(BTRIM(style_name), ''), %s)" in payload, (
|
||||
"style_name 只能补空,不能覆盖平台已给出的值"
|
||||
)
|
||||
assert "AND style_id IS NULL" in payload, "幂等:已归属的行不能被改写"
|
||||
assert "spu_id = %s" in payload, "京东广告表要按 spu_id 反查"
|
||||
|
||||
# 匹配上的 1 个 ID 写 style_id,1 个未建档
|
||||
assert result["updated"] == 5
|
||||
assert result["unmatched_product_ids"] == 1
|
||||
|
||||
# 未建档的显式标 unmatched,供观测视图统计
|
||||
assert any(
|
||||
params and params[0] == "unmatched"
|
||||
for sql, params in conn.log
|
||||
if sql.lstrip().startswith("UPDATE")
|
||||
)
|
||||
|
||||
|
||||
def test_backfill_ad_tables_dry_run_does_not_update(monkeypatch) -> None:
|
||||
import backfill_style_id as bf
|
||||
|
||||
monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7})
|
||||
conn = _BackfillConn()
|
||||
result = bf._backfill_ad_tables_by_product(
|
||||
conn, ["dy_product_ad_daily_metrics"], dry_run=True
|
||||
)[0]
|
||||
|
||||
assert result["dry_run"] is True
|
||||
assert result["planned_keys"] == 1
|
||||
assert not [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")]
|
||||
@@ -234,6 +234,62 @@ def test_erp_style_plan_uses_style_base_codes_and_aliases() -> None:
|
||||
assert missing == ["云栖相机双肩包"]
|
||||
|
||||
|
||||
def test_erp_style_plan_uses_own_codes_not_merged_series_union() -> None:
|
||||
"""月度表按款明细:不能照搬 loader 挂在锚点上的系列并集。
|
||||
|
||||
采集端把同系列编码聚合成并集交给锚点当唯一取数单元;报表若也用这份
|
||||
并集,锚点和子款会把同一批编码各查一遍,销量重复计入。
|
||||
"""
|
||||
|
||||
targets = [
|
||||
ProductTarget("宙斯", 100),
|
||||
ProductTarget("宙斯pro", 200),
|
||||
ProductTarget("星迹2", 300),
|
||||
ProductTarget("星迹2lite", 400),
|
||||
]
|
||||
series = {"name": "宙斯", "erp_owner": "宙斯"}
|
||||
styles = {
|
||||
# 锚点只做系列占位、自身没填编码:loader 聚合后 erp_codes 是系列并集
|
||||
"宙斯": {
|
||||
"erp_codes": ["10244", "10276", "10387"],
|
||||
"series": {**series, "own_erp_codes": []},
|
||||
},
|
||||
"宙斯pro": {
|
||||
"erp_codes": ["10276"],
|
||||
"series": {**series, "own_erp_codes": ["10276"]},
|
||||
},
|
||||
# 正常建档形态:锚点一条编码,子款各一条
|
||||
"星迹2": {
|
||||
"erp_codes": ["10394", "10504"],
|
||||
"series": {
|
||||
"name": "星迹2",
|
||||
"erp_owner": "星迹2",
|
||||
"own_erp_codes": ["10394"],
|
||||
},
|
||||
},
|
||||
"星迹2lite": {
|
||||
"erp_codes": ["10504"],
|
||||
"series": {
|
||||
"name": "星迹2",
|
||||
"erp_owner": "星迹2",
|
||||
"own_erp_codes": ["10504"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
plans, missing = plan_erp_styles(targets, styles, {})
|
||||
|
||||
assert plans == [
|
||||
ErpStylePlan("宙斯pro", ("10276",)),
|
||||
ErpStylePlan("星迹2", ("10394",)),
|
||||
ErpStylePlan("星迹2lite", ("10504",)),
|
||||
]
|
||||
# 锚点自身没编码:按 0 写入(--allow-missing-erp-as-zero),不算配置缺失
|
||||
assert missing == ["宙斯"]
|
||||
planned = [code for plan in plans for code in plan.erp_style_codes]
|
||||
assert len(planned) == len(set(planned))
|
||||
|
||||
|
||||
def test_execute_guard_rejects_incomplete_erp_mapping_before_sheet_write() -> None:
|
||||
with pytest.raises(RuntimeError, match="目标表未清空"):
|
||||
require_complete_erp_mappings(["云栖相机双肩包", "轻风双肩包"])
|
||||
|
||||
@@ -176,13 +176,22 @@ def test_discovery_creates_one_candidate_per_exact_video_attachment() -> None:
|
||||
def test_flagship_and_luggage_use_distinct_account_scopes() -> None:
|
||||
flagship = _one_candidate(store=tmall.STORE_FLAGSHIP)
|
||||
luggage = _one_candidate(store=tmall.STORE_LUGGAGE)
|
||||
new_bag = _one_candidate(store=tmall.STORE_NEW_BAG)
|
||||
|
||||
assert flagship.target_account_key == "guanghe-flagship"
|
||||
assert luggage.target_account_key == "guanghe-luggage"
|
||||
assert flagship.target_account_key != luggage.target_account_key
|
||||
assert new_bag.target_account_key == "guanghe-new-bag"
|
||||
assert len(
|
||||
{
|
||||
flagship.target_account_key,
|
||||
luggage.target_account_key,
|
||||
new_bag.target_account_key,
|
||||
}
|
||||
) == 3
|
||||
assert tmall.STORE_ACCOUNT_KEYS == {
|
||||
tmall.STORE_FLAGSHIP: flagship.target_account_key,
|
||||
tmall.STORE_LUGGAGE: luggage.target_account_key,
|
||||
tmall.STORE_NEW_BAG: new_bag.target_account_key,
|
||||
}
|
||||
|
||||
|
||||
@@ -199,6 +208,25 @@ def test_luggage_runtime_uses_new_store_account_label_without_changing_password_
|
||||
assert password == "same-runtime-secret"
|
||||
|
||||
|
||||
def test_new_bag_runtime_uses_its_own_credentials_and_profile(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GUANGHE_NEW_BAG_USERNAME", "gyxx箱包旗舰店:龙虾仔")
|
||||
monkeypatch.setenv("GUANGHE_NEW_BAG_PASSWORD", "new-bag-runtime-secret")
|
||||
|
||||
store_name, username, password, profile = tmall._store_runtime(tmall.STORE_NEW_BAG)
|
||||
|
||||
assert store_name == "gyxx箱包旗舰店:龙虾仔"
|
||||
assert username == "gyxx箱包旗舰店:龙虾仔"
|
||||
assert password == "new-bag-runtime-secret"
|
||||
assert str(profile).endswith("guanghe-new-bag")
|
||||
assert profile != tmall._store_runtime(tmall.STORE_LUGGAGE)[3]
|
||||
assert profile != tmall._store_runtime(tmall.STORE_FLAGSHIP)[3]
|
||||
assert tmall._expected_creator_identity_markers(
|
||||
tmall.STORE_NEW_BAG, store_name
|
||||
) == ("GYXX箱包天猫官方",)
|
||||
|
||||
|
||||
def test_account_identity_markers_accept_store_name_with_login_nickname() -> None:
|
||||
assert tmall.GuangHeUploader._account_identity_markers(
|
||||
"光影行星鑫华达专卖店:龙虾仔"
|
||||
|
||||
@@ -148,5 +148,80 @@ class WanxiangReportTemplateTests(unittest.TestCase):
|
||||
session.close.assert_called_once()
|
||||
|
||||
|
||||
class WanxiangSearchKeywordTests(unittest.TestCase):
|
||||
"""搜索词可与款名不同,但款式身份始终是 style_name。"""
|
||||
|
||||
def test_obsidian_toast_style_searches_shorter_keyword_first(self):
|
||||
self.assertEqual(
|
||||
wanxiang.search_keywords_for("黑曜石吐司包"),
|
||||
("黑曜石吐司", "黑曜石吐司包"),
|
||||
)
|
||||
|
||||
def test_styles_without_override_keep_their_own_name(self):
|
||||
self.assertEqual(wanxiang.search_keywords_for("星云2"), ("星云2",))
|
||||
self.assertEqual(
|
||||
wanxiang.search_keywords_for("黑曜石托特包"), ("黑曜石托特包",)
|
||||
)
|
||||
|
||||
def test_collect_style_searches_keyword_but_keeps_style_identity(self):
|
||||
searched: list[str] = []
|
||||
row = {
|
||||
"creative_id": "c1",
|
||||
"impressions": 10,
|
||||
"clicks": 2,
|
||||
"cost": 0.5,
|
||||
"creative_image_url": "https://img/1.jpg",
|
||||
}
|
||||
with (
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"search_style_by_subject",
|
||||
side_effect=lambda _page, keyword: searched.append(keyword),
|
||||
),
|
||||
patch.object(wanxiang, "extract_current_page_rows", return_value=[row]),
|
||||
patch.object(wanxiang, "collect_current_page_image_map", return_value={}),
|
||||
patch.object(wanxiang, "click_next_page", return_value=False),
|
||||
patch.object(wanxiang, "aggregate_rows", return_value={}),
|
||||
):
|
||||
result = wanxiang.collect_style(
|
||||
Mock(), "黑曜石吐司包", 1, item_ids=["1022486093557"]
|
||||
)
|
||||
|
||||
self.assertEqual(searched, ["黑曜石吐司"])
|
||||
self.assertEqual(result["style_name"], "黑曜石吐司包")
|
||||
self.assertEqual(result["search_keyword"], "黑曜石吐司")
|
||||
|
||||
def test_empty_keyword_falls_back_to_original_style_name(self):
|
||||
state = {"searched": [], "checks": 0}
|
||||
row = {
|
||||
"creative_id": "c1",
|
||||
"impressions": 10,
|
||||
"clicks": 2,
|
||||
"cost": 0.5,
|
||||
"creative_image_url": "https://img/1.jpg",
|
||||
}
|
||||
|
||||
def fake_extract(_page):
|
||||
state["checks"] += 1
|
||||
return [] if state["checks"] == 1 else [row]
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"search_style_by_subject",
|
||||
side_effect=lambda _page, keyword: state["searched"].append(keyword),
|
||||
),
|
||||
patch.object(wanxiang, "extract_current_page_rows", side_effect=fake_extract),
|
||||
patch.object(wanxiang, "collect_current_page_image_map", return_value={}),
|
||||
patch.object(wanxiang, "click_next_page", return_value=False),
|
||||
patch.object(wanxiang, "aggregate_rows", return_value={}),
|
||||
):
|
||||
result = wanxiang.collect_style(Mock(), "黑曜石吐司包", 1, item_ids=["1022"])
|
||||
|
||||
self.assertEqual(state["searched"], ["黑曜石吐司", "黑曜石吐司包"])
|
||||
self.assertEqual(result["style_name"], "黑曜石吐司包")
|
||||
self.assertEqual(result["search_keyword"], "黑曜石吐司包")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -34,7 +34,7 @@ def test_catalog_composition_registers_all_workflows() -> None:
|
||||
|
||||
registry = create_default_registry(catalog=catalog)
|
||||
|
||||
assert len(registry.workflow_ids) == 39
|
||||
assert len(registry.workflow_ids) == 40
|
||||
assert set(registry.workflow_ids) == {
|
||||
"shop.metrics.weekly",
|
||||
"shop.competitor.weekly",
|
||||
@@ -64,6 +64,7 @@ def test_catalog_composition_registers_all_workflows() -> None:
|
||||
"product.persona.daily",
|
||||
"product.daily",
|
||||
"product.online_stock_alert.daily",
|
||||
"product.points_exchange.weekly",
|
||||
"product.hk_anpei_public_inventory.daily",
|
||||
"product.alert.daily",
|
||||
"product.ecommerce_costs.daily",
|
||||
|
||||
@@ -11,15 +11,15 @@ from gyxx_flow.catalog import CatalogError, WorkflowCatalog
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_catalog_maps_39_project_scheduled_tasks() -> None:
|
||||
def test_catalog_maps_40_project_scheduled_tasks() -> None:
|
||||
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
scheduled = catalog.scheduled_workflows()
|
||||
|
||||
assert len(scheduled) == 39
|
||||
assert len({item.source_task_name for item in scheduled}) == 39
|
||||
assert len(scheduled) == 40
|
||||
assert len({item.source_task_name for item in scheduled}) == 40
|
||||
assert Counter(item.module for item in scheduled) == {
|
||||
"content_marketing": 9,
|
||||
"product_commerce": 14,
|
||||
"product_commerce": 15,
|
||||
"shop_intelligence": 12,
|
||||
"supply_chain": 4,
|
||||
}
|
||||
@@ -29,7 +29,7 @@ def test_catalog_schedules_douyin_price_appeal() -> None:
|
||||
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
schedule = catalog.schedule_for("shop.douyin_price_appeal")
|
||||
|
||||
assert len(catalog.workflows) == 39
|
||||
assert len(catalog.workflows) == 40
|
||||
assert catalog.manual_workflows() == ()
|
||||
assert "shop.douyin_price_appeal" in {
|
||||
schedule.workflow_id for schedule in catalog.schedules
|
||||
@@ -185,7 +185,7 @@ def test_every_scheduled_workflow_has_one_valid_asia_shanghai_schedule() -> None
|
||||
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
|
||||
assert catalog.timezone == "Asia/Shanghai"
|
||||
assert len(catalog.schedules) == 39
|
||||
assert len(catalog.schedules) == 40
|
||||
for workflow in catalog.scheduled_workflows():
|
||||
schedule = catalog.schedule_for(workflow.workflow_id)
|
||||
assert schedule.workflow_id == workflow.workflow_id
|
||||
|
||||
@@ -54,7 +54,7 @@ def test_default_cli_registry_exposes_all_project_scheduled_workflows(
|
||||
scheduled_ids = {
|
||||
entry.workflow_id for entry in registry.catalog.scheduled_workflows()
|
||||
}
|
||||
assert len(scheduled_ids) == 39
|
||||
assert len(scheduled_ids) == 40
|
||||
assert scheduled_ids == {
|
||||
schedule.workflow_id for schedule in registry.catalog.schedules
|
||||
}
|
||||
|
||||
@@ -1709,6 +1709,28 @@ def test_tmall_video_trigger_routes_only_the_selected_store(
|
||||
assert flagship["store"] == "flagship"
|
||||
assert flagship["topic_keyword"] == "我的夏日焕新清单"
|
||||
assert launcher.calls[-1]["store"] == "flagship"
|
||||
launcher.complete("product.video_upload")
|
||||
|
||||
new_bag = service.trigger(
|
||||
"product.video_upload",
|
||||
{"business_date": "2026-08-11", "store": "new_bag"},
|
||||
)
|
||||
|
||||
assert new_bag["store"] == "new_bag"
|
||||
assert launcher.calls[-1]["store"] == "new_bag"
|
||||
|
||||
|
||||
def test_tmall_video_trigger_rejects_unknown_store(console_settings: Settings) -> None:
|
||||
service = WorkflowConsoleService(
|
||||
console_settings,
|
||||
launcher=FakeConsoleLauncher(),
|
||||
)
|
||||
|
||||
with pytest.raises(ConsoleRequestError, match="店铺参数无效"):
|
||||
service.trigger(
|
||||
"product.video_upload",
|
||||
{"business_date": "2026-08-11", "store": "obsidian"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [123, True, "x" * 121])
|
||||
@@ -2824,6 +2846,8 @@ def test_http_home_assets_and_overview_have_security_headers(
|
||||
assert "强制重跑:忽略幂等跳过与已有结果,重新采集并正式执行" in page
|
||||
assert '<option value="flagship">光影行星旗舰店</option>' in page
|
||||
assert '<option value="luggage">光影行星鑫华达专卖店:龙虾仔</option>' in page
|
||||
assert '<option value="new_bag">' in page
|
||||
assert "天猫新店 · gyxx箱包旗舰店:龙虾仔" in page
|
||||
assert 'id="run-tmall-topic-field"' in page
|
||||
assert 'id="run-tmall-topic-keyword"' in page
|
||||
assert "存在未上传附件时才登录该店铺账号" in page
|
||||
@@ -2915,6 +2939,8 @@ def test_http_home_assets_and_overview_have_security_headers(
|
||||
assert "tmall_topic_keyword" in decoded_asset
|
||||
assert "syncTmallTopicField" in decoded_asset
|
||||
assert "光影行星鑫华达专卖店:龙虾仔" in decoded_asset
|
||||
assert "天猫新店" in decoded_asset
|
||||
assert 'value: "new_bag"' in decoded_asset
|
||||
assert "renderCardLiveState" in decoded_asset
|
||||
assert 'data-live-progress="${completed}/${steps.length}"' in decoded_asset
|
||||
assert 'data-current-step="${esc(currentNames.join("、"))}"' in decoded_asset
|
||||
|
||||
@@ -36,10 +36,11 @@ def test_repository_keeps_the_simplified_public_structure() -> None:
|
||||
"note-inventory.md",
|
||||
"plan.md",
|
||||
"runbook.md",
|
||||
"workbench.md",
|
||||
"style-identity-refactor-plan.md",
|
||||
"workflow-acceptance-test-report.md",
|
||||
"workflow-acceptance-test-requirements.md",
|
||||
"workflow-console.md",
|
||||
"workflows-overview.md",
|
||||
"workflow-time-distribution.md",
|
||||
}
|
||||
assert not {
|
||||
|
||||
@@ -34,7 +34,6 @@ NESTED_TARGETS = {
|
||||
"dy_product_scraping.py",
|
||||
"taobao_sycm_collect.py",
|
||||
"taobao_sycm_collect_backfill.py",
|
||||
"import_product_reviews.py",
|
||||
"aggregate_daily_final.py",
|
||||
"export_bitable_records.py",
|
||||
"insert_bitable_records.py",
|
||||
|
||||
@@ -80,8 +80,19 @@ def test_baseline_contains_all_provided_people_and_routable_capabilities(
|
||||
assert mobiles["wang_yunlong"] == "+8613663785145"
|
||||
assert mobiles["stock_alert_user_19832771651"] == "+8619832771651"
|
||||
assert set(payload["capabilities"]) == set(NOTIFICATION_CAPABILITIES)
|
||||
assert set(payload["routes"]) == {"product.online_stock_alert.daily"}
|
||||
assert set(payload["routes"]) == {
|
||||
"product.online_stock_alert.daily",
|
||||
"product.points_exchange.weekly",
|
||||
}
|
||||
assert payload["routes"]["product.online_stock_alert.daily"]["enabled"] is True
|
||||
exchange_route = payload["routes"]["product.points_exchange.weekly"]
|
||||
assert exchange_route["enabled"] is True
|
||||
assert exchange_route["person_ids"] == payload["routes"][
|
||||
"product.online_stock_alert.daily"
|
||||
]["person_ids"]
|
||||
assert exchange_route["app_profile"] == payload["routes"][
|
||||
"product.online_stock_alert.daily"
|
||||
]["app_profile"]
|
||||
assert len(revision) == 64
|
||||
|
||||
wang = payload["people"]["wang_yunlong"]
|
||||
|
||||
Loaded 100 of 131 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user