feat(product-commerce): 系列聚合去重与采集开关两层语义

配置来源收敛
- 系列平台编码 tm/jd/dy/jd_self 与 9 类飞书地址改为整系列并集去重后挂锚点,
  子款编码保留在 series.platform_ids 备查;多表冲突保留最高优先级并告警
- 写表脚本(当日/天猫主图/京东主图)去掉本地 JSON 回退,地址缺失或读取失败
  直接 fail closed,避免库与表地址不一致被静默掩盖
- 测试注入 records_provider 时禁用配置缓存读写,防止假数据写入生产快照

采集开关(dim_style.collect_enabled)
- 锚点为系列总开关:false 则整系列不采
- 锚点为 true 时,子款自身 false 仅把自己的 ERP 编码排除出聚合,其余子款照常
- 排除结果记录在 series.erp_excluded 并在运行时提示

聚水潭全店铺日报防错
- 列布局 fail closed、行 td 数校验、满页判定 + 总页数兜底、值域交叉校验
- 新增单日总量闸:超近 14 天中位数 8 倍拒绝落库

其他
- 新增电商费用归集配置 ad_cost_rollup.json 与积分兑换周报采集
- 店铺情报、供应链到货同步、通知路由与相关配置/文档同步更新
- 忽略本地 dsh workbench 脚手架(含 node_modules)
This commit is contained in:
wangyunlong committed 2026-10-30 11:49:31 +08:00
1 parent 2f31d5872f
commit 622a609800
66 files changed
+4156 -469

No files matched your search

+3
View File
@@ -40,5 +40,8 @@ Thumbs.db
# Local tool artifacts
/.playwright-cli/
/output/
# Local dsh workbench scaffold (generated at runtime, bundles node_modules)
/workbench/
/nul
/new_all_bag.xlsx
+11
View File
@@ -274,6 +274,17 @@
"kind": "python",
"notification_workflow_id": "product.online_stock_alert.daily"
},
{
"id": "product.points_exchange_weekly.run",
"module": "product_commerce",
"entry": "points_exchange_weekly.py",
"kind": "python",
"default_args": [
"--from",
"{business_date}"
],
"notification_workflow_id": "product.points_exchange.weekly"
},
{
"id": "product.import.daily",
"module": "product_commerce",
+12
View File
@@ -231,6 +231,9 @@
"product.online_stock_alert.daily": {
"condition": "线上商品当前库存低于 3 时向配置接收人发送库存预警。"
},
"product.points_exchange.weekly": {
"condition": "每周汇总上一周积分兑换商品排行后向配置接收人发送周报。"
},
"supply.purchase_confirmation.daily": {
"condition": "采购确认结果生成完成后投递最终业务通知。"
},
@@ -250,6 +253,15 @@
"wang_yunlong",
"stock_alert_user_19832771651"
]
},
"product.points_exchange.weekly": {
"enabled": true,
"app_profile": "hermes-analyzer",
"person_ids": [
"online_stock_alert_recipient",
"wang_yunlong",
"stock_alert_user_19832771651"
]
}
}
}
+9
View File
@@ -1119,6 +1119,15 @@
],
"cdp_port": 22163
},
"product.points_exchange_weekly.run": {
"script_id": "product_commerce:points_exchange_weekly.py",
"state_key": "product_commerce:points_exchange_weekly.py",
"aliases": [
"product_commerce:points_exchange_weekly.py",
"product.points_exchange_weekly.run"
],
"cdp_port": 22168
},
"product_commerce:dy_audience_profile_collect.py": {
"script_id": "product_commerce:dy_audience_profile_collect.py",
"state_key": "product_commerce:dy_audience_profile_collect.py",
+13 -1
View File
@@ -39,7 +39,7 @@
"kind": "daily",
"workflow_id": "content.marketing_report.daily"
},
{
{
"at": "12:00",
"days": [
"Sunday"
@@ -197,6 +197,16 @@
"kind": "daily",
"workflow_id": "product.online_stock_alert.daily"
},
{
"at": "09:45",
"business_date_offset_days": -7,
"days": [
"Monday"
],
"enabled": true,
"kind": "weekly",
"workflow_id": "product.points_exchange.weekly"
},
{
"at": "08:40",
"business_date_offset_days": -1,
@@ -325,6 +335,8 @@
},
{
"at": "07:00",
"business_date_offset_days": 0,
"enabled": false,
"kind": "daily",
"workflow_id": "supply.replenishment_alert.daily"
},
+15 -15
View File
@@ -70,7 +70,7 @@
{
"category": "production_source",
"source_sha256": "c7b54d941ab7f056e2d0ea4355543ff4a5c34ad986c8ed32145e988ac9501ea2",
"target_sha256": "8576678688d7d90f6f7285110de005982bc1ba82bc7e9977224ba1bdb4101283",
"target_sha256": "b306195f9fd90d6fba3c8fd9335677fe8dcfa1653604c8ace9f3cc340f43a5dc",
"transformed": true,
"source_relative_path": "collect_erp_yesterday_metrics.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py"
@@ -126,7 +126,7 @@
{
"category": "production_source",
"source_sha256": "22dcc9d8b2f1cf598a62e4c71df765a952250694cb26b9a73f477c5ca1a29884",
"target_sha256": "ffeaf697a1a58b21520fda3f3b0936c591d05b328537c41f2048366a9ba1b8d5",
"target_sha256": "ce328f87c883257c91025708209338ba6ee34d3c284a149c1969aeae85ffbdb7",
"transformed": true,
"source_relative_path": "config/style_config_loader.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/style_config_loader.py"
@@ -134,7 +134,7 @@
{
"category": "production_source",
"source_sha256": "a8900b9f25db6cf2bef3117a1d2cd86283054fea6f5a8590fd91b5cfbbcaf87e",
"target_sha256": "47cdfd9ebf27e31be06497b2f74c80da4ffced259a873730eb024b0db5eba740",
"target_sha256": "a34efe965b312fe02509a4955cba835f7cdc081daa8a1624d571a8f46bf9380e",
"transformed": true,
"source_relative_path": "db/__init__.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/__init__.py"
@@ -150,7 +150,7 @@
{
"category": "production_source",
"source_sha256": "492916088fa5e8c170461cc9eba97a88ed1cc1e50c7b99e5f6a5e49167fea157",
"target_sha256": "7b0ae5d648245e89f4aacf262250009de812d3dbb8ee55386f3704cabca48799",
"target_sha256": "855feba2cfd51bbbb70b2fc912a7b5b517f91c3fa47ff5129ba05e2f7ffa329c",
"transformed": true,
"source_relative_path": "dy_product_scraping.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/dy_product_scraping.py"
@@ -206,7 +206,7 @@
{
"category": "production_source",
"source_sha256": "057a01e8a22cdd2d1806d290da207209954af89f7da6339e9cdcef3bc0297e63",
"target_sha256": "0360bd218db1625088663b8ba509b5f53edd09563b71823006f2d46cdd5367ae",
"target_sha256": "815c00d039e4f81e4f9deae95891fab288c6584e72d21b73f4b6f348e158e126",
"transformed": true,
"source_relative_path": "insert_bitable_records.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/insert_bitable_records.py"
@@ -214,7 +214,7 @@
{
"category": "production_source",
"source_sha256": "e0f016bafb3f527f1cae46eaf3ef31510184128aaaa7f52024a7c3f978eec42b",
"target_sha256": "b157f24430170d5a991bb2283af0a747b26dd11c0f0dec37289774f115132fee",
"target_sha256": "cef210f479d45e4b92cc597ddde0d80e62e37b1546c3ee27099ad7ad9c71d108",
"transformed": true,
"source_relative_path": "jd_main_image_collector.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_main_image_collector.py"
@@ -222,7 +222,7 @@
{
"category": "production_source",
"source_sha256": "d99ce71a008128549ac83fc3da52d88540b74538e3fe5c85d47c3b985e414353",
"target_sha256": "cfe98b322e4ef7bd875325c2a9ffedeabb364e98b2983773dc98f9e0e75a6581",
"target_sha256": "a597578741795926844b93fa6b74bd8316e1c72cac5196f9dbc2e492e76d88e3",
"transformed": true,
"source_relative_path": "jd_product_data_collector.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py"
@@ -358,7 +358,7 @@
{
"category": "production_source",
"source_sha256": "af9b436a31c3454dc6e0b34b77876962dab0b477e8a561db49911fdfc24eade3",
"target_sha256": "e3b91d3c4b529291f7a6ceee5ff99326351d78104a66babc531f90faec6b7e1a",
"target_sha256": "096834b64e4b8e5120e38a116efb4c6eb02440d7c4567bf4d6783c18aee0050d",
"transformed": true,
"source_relative_path": "run_weekly_main_image.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/run_weekly_main_image.py"
@@ -366,7 +366,7 @@
{
"category": "production_source",
"source_sha256": "34d935efb8dff20afd3ab89ef150c5f8db9cee80c6256c7fe4ef0bae2f72ed63",
"target_sha256": "787a207a2c8174518a6465587644937ad6d11b1d86b07488e1c86b07e1410171",
"target_sha256": "3fdaa7e23e96897c2039dcd1cdc6c6fc4ee1fb9cff828262e2360d453d778f47",
"transformed": true,
"source_relative_path": "scripts/insert_jd_main_image_records.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/scripts/insert_jd_main_image_records.py"
@@ -374,7 +374,7 @@
{
"category": "production_source",
"source_sha256": "a1b097ead2a3f7e20ca63d7c15828a8d04030b0d6722934c3bf65d9e975fba76",
"target_sha256": "7d8587f77615040087d2d475a7b440ad16a39c5874e143418dd700ecbc5db0f8",
"target_sha256": "6c2ef5acbfff574a9f7460ebaccc8c4e46a3ccbd8b72944791e0fded69038e77",
"transformed": true,
"source_relative_path": "scripts/insert_main_image_records.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/scripts/insert_main_image_records.py"
@@ -390,7 +390,7 @@
{
"category": "production_source",
"source_sha256": "4474dd7d68117247af87e20bbeb6b7c072368ef0f606de5d3e186f459d34a0f5",
"target_sha256": "862ae9392227113ad4266c2170769595c36b2f98fdecc54e54a648ba834baebf",
"target_sha256": "9e87faad21459b8ac79c335734e11b1da636d34747f05e80c7073b66f01e35f1",
"transformed": true,
"source_relative_path": "taobao_sycm_collect.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_sycm_collect.py"
@@ -414,7 +414,7 @@
{
"category": "production_source",
"source_sha256": "3f0c32c443c87417c43881da69aa7776828ba973465f8d32f50b3dc698b0a4fe",
"target_sha256": "548df6ddf3af3b3c066d18687114f1ce672afa054535f6da0c254d60f8306b86",
"target_sha256": "6363ec2a70a22b078af23a70fcdda6e3cce9f95eb4aa7ebc54f4cb85a293cb6d",
"transformed": true,
"source_relative_path": "taobao_wanxiang_ai_creative_report.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py"
@@ -544,7 +544,7 @@
{
"category": "runtime_resource",
"source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746",
"target_sha256": "6bbd36048836ef7ed0807223fc028ac17349b20314b1ea353ccedb451957ba55",
"target_sha256": "0ea747eb7e40875338bebe7fc5b8d2524869e55ba1b503a322b37c857b274d8e",
"transformed": true,
"source_relative_path": "db/schema.sql",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql"
@@ -656,7 +656,7 @@
{
"category": "regression_test",
"source_sha256": "820e17b910b62a213f0bebafcff72998c2c978307c97f976c074276150ce7c6a",
"target_sha256": "cdedf8c9a4871ad763e0a9098fc18777f0c3206991e8d440a45ccdfbda089a07",
"target_sha256": "77221bd2bfb9e2314585ba9bbaea770093a86cfe5918ffe9c67fc1fadba3b04a",
"transformed": true,
"source_relative_path": "tests/test_main_image_concurrency.py",
"target_relative_path": "tests/modules/product_commerce/test_main_image_concurrency.py"
@@ -720,7 +720,7 @@
{
"category": "regression_test",
"source_sha256": "17761a5ba93a5387ffa67662424955178058aeec708bbcd3eb0341a859d8a1f1",
"target_sha256": "549cb13dd32c9bed76d78eafa7542ea6e5c638d9f29c857eff4dde6d3b8e9c40",
"target_sha256": "edc906fdf2ab2fab731fdcf56dd8ad7cf272949a8b4f66fb71c1693de5822f1f",
"transformed": true,
"source_relative_path": "tests/test_style_config_loader_erp_source.py",
"target_relative_path": "tests/modules/product_commerce/test_style_config_loader_erp_source.py"
+48 -3
View File
@@ -1126,7 +1126,7 @@
"30",
"--headless"
],
"timeout_seconds": 14400,
"timeout_seconds": 28800,
"replay_policy": "idempotent"
}
]
@@ -1180,7 +1180,7 @@
"{business_date}",
"--end-date",
"{business_date}",
"--headless"
"--headed"
],
"timeout_seconds": 14400,
"replay_policy": "idempotent"
@@ -1705,6 +1705,51 @@
"task_name": "OnlineItemStockAlert"
}
},
{
"id": "product.points_exchange.weekly",
"module": "product_commerce",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "query_rank_notify",
"name": "查询积分兑换明细并按商品生成兑换排行周报",
"description": "调用企业数据交互积分兑换查询接口(startTime/endTime 覆盖整周自然日),按商品汇总 quantity 兑换数量并从高到低排名;默认统计上一个完整自然周(周一至周日),将排行榜以飞书卡片发送给私域库存预警的同一批接收人。",
"replay_policy": "idempotent",
"data_flow": {
"sources": [
{
"system": "企业数据交互接口",
"label": "积分兑换明细",
"detail": "/open/data/v1/points-exchange/query,startTime 为窗口首日 00:00:00、endTime 为窗口末日 23:59:59"
}
],
"processing": [
"以业务日期作为窗口起点,缺省统计 7 个自然日",
"按 itemId 汇总 quantity,并保留商品名称与 SKU 数",
"按兑换数量从高到低排序并分段组织飞书消息"
],
"destinations": [
{
"system": "飞书",
"label": "积分兑换商品排行周报",
"condition": "窗口内存在兑换数量大于 0 的商品"
}
]
},
"entry": "points_exchange_weekly.py",
"args": [
"--from",
"{business_date}"
]
}
]
},
"provenance": {
"source_project": "data-hub",
"task_name": "PointsExchangeWeeklyRanking"
}
},
{
"id": "product.ecommerce_costs.daily",
"module": "product_commerce",
@@ -2835,7 +2880,7 @@
"source_project": "supply",
"task_name": "auto-flow-replenishment-arrival-sync"
},
"note": "队列表为空时成功结束;失败行保留 failed 状态,下一次调度自动重试。默认租户为 1,可用 GYXX_ERP_TENANT_ID 覆盖。"
"note": "队列表为空时成功结束;失败行保留 failed 状态,下一次调度自动重试。data-hub 单公司化后队列表已无 tenant_id:脚本会先探测实际表结构,有该列仍按 GYXX_ERP_TENANT_ID(默认 1)过滤,没有则按单公司口径直接认领。"
},
{
"id": "product.market_rank",
+45 -3
View File
@@ -128,9 +128,51 @@ workflow_dynamic_styles + workflow_dynamic_style_platforms (目标/画像/启停
└─ 目标兼容输出 → feishu_mapping → 内容营销工作流 → 各业务飞书目标表
```
系列共享规则:ERP 款式编码逐款必填;平台商品编码与销量飞书表整系列只需主款填写,
子款通过系列归属继承销量表,但**不复制平台编码**——同一个商品 ID 被多款重复采集会把
访客/加购重复入账。
系列共享规则:平台商品编码、ERP 编码与 9 类飞书地址都以「系列」为一个采集单元,
子款留空即可;子款误填也不会丢——loader 会把整条系列聚合并去重后挂到锚点。
- **平台编码聚合去重到锚点**:整条系列各平台的商品 ID 先并集去重,再挂到锚点,由锚点
一个采集单元去抓。子款的原始编码保留在 `series.platform_ids` 供诊断,采集口判定让出
(「缺 tm商品编码(同系列「X」已在采)」),同一个商品 ID 不会被采两次、访客与加购
不会重复入账。锚点不在采集范围内时子款仍可回退自采。
- **ERP 编码先聚合再采集**:正常建档形态是锚点填一条款式编码 + 各平台商品编码,子款只填
自己的款式编码;也可能出现锚点 `erp_style_codes` 为空、编码全散落在子款上的形态。
loader 一律把整条系列的 ERP 编码聚合成并集挂到锚点(`series.erp_owner`),锚点作为唯一
ERP 采集单元去聚水潭取数,子款让出并记录「ERP 已由同系列 X 归集采集」,但仍保留自己的
原始编码(`series.own_erp_codes`)供按款查询与月度销量表使用。
**例外(2026-10-30)**:子款自身 `collect_enabled = FALSE` 时,它的 ERP 编码不并入聚合
(`series.erp_excluded` 可见),锚点不代采;锚点自身 `FALSE` 仍是整系列停采。
平台商品编码不受子款开关影响,仍按整系列聚合。
- **飞书地址聚合去重下发**:`workflow_config._propagate_series_targets()` 与
`_apply_series_sharing()` 两处都按「锚点优先、其余按款名」收集整系列非空地址,去重后
下发到每一行。一个表位只能落一张表,同字段出现多个不同地址时保留优先级最高的一条并打
`[WARN] 系列「X」<字段> 有 N 个不同表`。子款想用独立一套表,必须拆成独立系列。
- **月度销量表按款明细**:`product.sales_sheet.daily` 的报表口径**不用**锚点上的聚合并集,
而是每个款式只查自己建档时填的编码。采集端的并集是给聚水潭当唯一取数单元用的,报表
照搬会让同系列每个成员行各查一遍并集、销量重复计入。锚点自身没编码时该行按 0 写入
(`--allow-missing-erp-as-zero`),同系列销量落在各子款行上,合计与采集总量一致。
采集门只校验身份:ERP 日采要求 ERP 款式编码,平台日采再叠加对应平台商品编码。
**销量飞书表地址不再是采集前置条件**——缺表地址的款式照常采集并落库,只是写表环节按
地址缺失跳过(`get_bitable_style_map` 不生成该款目标),不影响数据入库与后续分析。
主图采集同理:`get_main_image_styles` 只要求身份齐全,**主图飞书表地址不参与采集门**,
缺地址的款照常抓图并写 `main_image_creatives`,`get_bitable_main_image_map` 只返回有地址
的款,写表脚本对没有地址的款打 `[NO-TABLE]` 跳过飞书、仍入库。
**写表脚本对目标地址 fail closed(2026-10-30)**:`insert_bitable_records.py`、
`scripts/insert_main_image_records.py`、`scripts/insert_jd_main_image_records.py` 读取
`get_bitable_style_map()` / `get_bitable_main_image_map()` 失败或返回空时,直接打
`[FAIL] …拒绝回退本地 JSON` 并退出码 1,**不再回退** `bitable_style_map.json` /
`bitable_main_image_map.json`。库与本地文件不一致时,回退会让数据静默写到错误的表,
比直接失败更难发现。`styles_input.json` 同理由 `collect_erp_yesterday_metrics.load_erp_styles`
与 `export_bitable_records.load_style_meta` 兜住:身份源不可用以 `lifecycle_unavailable` 失败。
**旧表只保留「行集合 + 启停」职责**:`workflow_dynamic_styles` / `_platforms` 仍决定有哪些
款式、`enabled` 启停、`note`、`style_content`、`destinations`,但 ERP 编码、平台商品编码与
9 类地址全部由 `dim_style` 覆盖(`merge_lifecycle_style_records` 中 `managed` 由
`jsonb_build_object` 恒生成四个键而恒为真)。旧表里**不在 dim_style 的残留行**(孤儿款)
地址会被清空、不再采集,属于建档改名/合并后的历史垃圾,应直接删除
(2026-10-30 已清理 `拓界mode`、`逐星evo`,备份 `var/tmp/workflow_dynamic_orphans_20261030.json`)。
`StyleConfigLoader` 只在数据库短暂不可用时使用最近一次数据库成功读取后生成的本地缓存;
`feishu_mapping` 的目标表字段缓存仍保留,用来减少对各业务飞书表的字段查询。
+99
View File
@@ -286,6 +286,105 @@ uv run gyxx effects reconcile <workflow_id> \
浏览器状态异常时,只备份并处理目标脚本的 `state/browser/<module>/<script>`,不要清理其他脚本状态。调度状态异常时先停止唯一调度进程并备份 `state/scheduler`,不要同时启动第二实例。
## 商品日报采集开关(product.daily)
采集开关是 `dim_style.collect_enabled`(2026-10-29 起),**两层语义**(2026-10-30 起):
1. **锚点是系列总开关**:以该系列 `is_series_anchor = TRUE` 那一行为准;系列没有锚点时按款名
排序第一个。锚点 `FALSE` → **整条系列一款都不采**,同系列其余款式跟随锚点。
2. **锚点 `TRUE` 时,子款开关只决定自己的 ERP 编码是否参与聚合**:子款 `TRUE` 的 ERP 款式编码
并入锚点一起去聚水潭取数;子款 `FALSE` 的编码不并入(既不由它自己采,也不让锚点代采)。
被排除的子款列在 `series.erp_excluded`,运行时打
`[INFO] 系列「X」子款 Y 已关闭采集,ERP 编码不纳入「Z」聚合`。平台商品编码仍按整系列聚合,
不受子款开关影响。
旧的独立表 `product_daily_collect_scope` 已迁移并删除,备份在
`var/tmp/collect_scope_before_drop_20261029.json`。
默认值是 `TRUE`(采集),因此产品生命进程在 `dim_style` 新增的款式**无需任何操作**就会进入日报;
配件、包装、耗材、停售款才需要显式关掉。
关掉整条系列(改锚点那一行即可,子款自动跟随):
```sql
-- 单系列
UPDATE dim_style SET collect_enabled = FALSE WHERE style_name = '黑曜石托特包2';
-- 批量:把若干系列的所有行一起关(无锚点的系列也覆盖到)
UPDATE dim_style SET collect_enabled = FALSE
WHERE COALESCE(NULLIF(btrim(series_name), ''), style_name)
IN ('飞机盒', '香片', '雨衣', '鞋袋');
```
恢复采集把 `FALSE` 改成 `TRUE`。改完无需重启控制台或调度器:`StyleConfigLoader.load()`
每次运行都直接读库,下一轮 `product.daily`(每天 08:40,业务日 T-1)即生效。
两点注意:
- **开关落在锚点行上才控制整条系列。** ERP 编码与三平台商品编码都由
`_apply_series_sharing()` 按整系列聚合并去重后挂到锚点(子款误填的编码也会并入,但
`collect_enabled = FALSE` 的子款 ERP 编码除外),锚点是系列唯一的采集单元;子款自己的
编码保留在 `series.own_erp_codes` 与 `series.platform_ids` 备查。
- 关掉只影响**商品日报**(tm/jd/dy 平台段 + ERP 段)。天猫/京东主图周采集、聚水潭全店铺日报、
三平台电商费用**不受此开关影响**。
放行不等于一定会采:采集门仍要求「ERP 款式编码 + 各平台商品编码」齐备;飞书目标表地址
(下一节的 9 类 `bitable_*` 列)只决定能否写飞书,缺地址的款照常采集入库。缺口写在
`var/logs/product_commerce/<date>_skipped_styles.json` 的 `skipped.reason`。
改款式名时要同步改 `series_name`/锚点标记,否则旧开关会变成孤儿行、新名字按默认 TRUE 采集。
## 飞书表地址维护(dim_style)
2026-10-29 起,9 类飞书多维表格地址**全部维护在 `dim_style`**,而且只写在系列锚点行上。
`workflow_dynamic_styles` / `workflow_dynamic_style_platforms` 里的同名字段已废弃,
代码不再读取;旧值备份在 `var/tmp/bitable_urls_before_move_20261029.json`。
| 用途 | `dim_style` 列 | 原动态配置字段 |
|---|---|---|
| 销量表 | `bitable_weekly_url` | `sales_bitable_url` |
| 主图表 | `bitable_main_image_url` | `main_image_bitable_url` |
| 合作达人表 | `bitable_creator_url` | `creator_bitable_url` |
| 自营合作达人表 | `bitable_self_creator_url` | `self_creator_bitable_url` |
| 每周/每月笔记分析 | `bitable_weekly_note_url` | `weekly_note_analysis_url` |
| 平台单品分析 | `bitable_style_analysis_url` | `style_analysis_bitable_url` |
| 人群画像(天猫) | `bitable_persona_tm_url` | `tm_persona_bitable_url` |
| 人群画像(京东) | `bitable_persona_jd_url` | `jd_persona_bitable_url` |
| 人群画像(抖音) | `bitable_persona_dy_url` | `dy_persona_bitable_url` |
子款留空即可:两处代码会把整条系列的地址**聚合去重**后统一下发——
`workflow_config._propagate_series_targets()` 在合并建档数据时按「锚点优先、其余按款名」
收集整系列的非空地址,去重后写回每一行;`style_config_loader._apply_series_sharing()`
在聚合配置时再做一次同样的兜底。因此子款误填的地址不会丢、也不会让同系列出现两套表。
一个表位只能落一张表:同一字段填了多个不同地址时,保留优先级最高的那条,并在日志里打
`[WARN] 系列「X」<字段> 有 N 个不同表`,提示到锚点手动裁决。
```sql
-- 改某个系列的主图表(只改锚点一行,子款自动跟随)
UPDATE dim_style
SET bitable_main_image_url = 'https://xxx.feishu.cn/base/<base>?table=<tbl>'
WHERE style_name = '逐星'; -- 必须是该系列的锚点行
-- 查看哪些锚点还缺地址
SELECT style_name, series_name,
bitable_weekly_url, bitable_main_image_url, bitable_creator_url
FROM dim_style
WHERE is_series_anchor AND COALESCE(bitable_weekly_url, '') = ''
ORDER BY style_name;
```
改完不需要重启:`StyleConfigLoader.load()` 每次运行都读库,下一轮工作流即生效。
若某类地址为空,对应环节会跳过写表但照常采集入库(例如主图缺表地址打 `[NO-TABLE]`)。
**地址读取是 fail-closed 的**(2026-10-30):写表脚本拿不到 `dim_style` 地址时会打
`[FAIL] … 拒绝回退本地 JSON` 并以退出码 1 结束,**不会**改用 `bitable_style_map.json` /
`bitable_main_image_map.json` 里的旧地址。所以看到这类 `[FAIL]` 先去查库里的地址,
不要去改本地 JSON(这两个文件已无读取方)。
新增系列时,地址写在锚点行。若某款业务上确实是独立一套表(历史上 `逐星GT` 这种情形),
必须把它拆成独立系列(单独的 `series_name` + 锚点标记)并把地址写在自己那行;只写地址
而不拆系列的话,聚合会把它并入原系列,只在日志里留一条告警,不会真的生效。
## 业务源码同步
外部业务源码目录只用于发现更新,GYXX Flow 运行时不导入或挂载这些目录。只读检查:
+3 -2
View File
@@ -1,7 +1,7 @@
# 工作流总览(未隐藏)
> 生成日期:2026-10-08。数据来源:`config/workflows.json`、`config/schedules.json`、控制台显示名(`var/state/console/workflow-display-names.json`);数据库表提取自各入口脚本及共享 DB 辅助模块的内联 SQL。
> 范围:控制台中未被隐藏的工作流共 **30 个**(全部 39 个中剔除浏览器端隐藏的 10 个,其中 `content.relogin.weekly` 已不在当前配置中)。
> 范围:控制台中未被隐藏的工作流共 **31 个**(全部 40 个中剔除浏览器端隐藏的 10 个,其中 `content.relogin.weekly` 已不在当前配置中)。
> 定时时间均为 **Asia/Shanghai**,由常驻调度器 `gyxx schedule run` 按 `config/schedules.json` 触发;标注「未启用」的仅支持控制台手动运行。
> 启动脚本为仓库根目录相对路径,由工作流引擎相对 `src/gyxx_flow/modules/<模块>/` 解析执行;`×N` 表示同一脚本被 N 个步骤以不同参数复用。
>
@@ -36,13 +36,14 @@
| 天猫差评采集 | `shop.tmall_negative_reviews.daily` | 固定筛选负面评价并按评价时间范围采集后入库 | 每月 25 日 03:50(未启用,仅手动) | `src/gyxx_flow/modules/shop_intelligence/collectors/tmall_negative_reviews.py` | `tmall_negative_reviews`(写) |
| 抖音待改价申诉 | `shop.douyin_price_appeal` | 每个时点重新扫描可申诉并发起价格申诉;已申诉商品不重复出现 | 每日每小时整点(00:00–23:00) | `src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py` | 无(浏览器采集 + 本地 JSON) |
## 商品电商(product_commerce,12 个)
## 商品电商(product_commerce,13 个)
| 工作流名称 | ID | 功能 | 定时执行 | 启动脚本 | 关联数据库表 |
|---|---|---|---|---|---|
| 商品人群画像周采集 | `product.persona.daily` | 天猫、抖音、京东人群画像并行采集并写入对应飞书多维表 | 每周二 12:00 | `src/gyxx_flow/modules/product_commerce/run_daily_persona.py` | `daily_persona_metrics`(写) |
| 商品经营日报采集 | `product.daily` | 按业务日期(运行日 −1)采集 ERP 与三平台数据,幂等写 PostgreSQL,汇总分析后导出并写飞书日报 | 每日 08:40 | `src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py` | 写:`product_reviews`、`product_daily_metrics`、`daily_style_metrics`、`fact_style_analysis_reports`、`decline_events`、`fact_alert_history`<br>读:`weekly_style_metrics`、`daily_persona_metrics`、`main_image_creatives`、`cmt_styles`、`cmt_notes_master`、`cmt_comments`、`cmt_creators`(及上述写表) |
| 私域库存预警 | `product.online_stock_alert.daily` | 查询线上渠道全部商品,库存低于 3 时汇总 SKU 与库存发飞书预警 | 每日 09:30 | `src/gyxx_flow/modules/product_commerce/online_stock_alert.py` | 无(企业数据交互 HTTP API + 飞书) |
| 私域商品兑换排行 | `product.points_exchange.weekly` | 按自然周查询私域积分兑换明细,按商品汇总 `quantity` 兑换数量并从高到低排名,以飞书卡片发送周报 | 每周一 09:45(统计上一个完整自然周) | `src/gyxx_flow/modules/product_commerce/points_exchange_weekly.py` | 无(企业数据交互 HTTP API + 飞书) |
| 三平台电商费用采集 | `product.ecommerce_costs.daily` | 并行采集天猫万相台、京东、抖音千川业务日前一天的费用,各平台先下载再导入,单平台失败不阻断其他 | 每日 10:00 | `…/product_commerce/collect_tmall_wanxiang_ads.py`<br>`…/product_commerce/import_tmall_product_ads.py`<br>`…/product_commerce/collect_jd_ad_costs.py`<br>`…/product_commerce/import_jd_ad_costs.py`<br>`…/product_commerce/collect_douyin_qianchuan_ads.py`<br>`…/product_commerce/import_douyin_qianchuan_ads.py` | `product_daily_metrics`(写,天猫原始费用并入 raw_data JSONB)<br>`jd_product_ad_daily_metrics`、`dy_product_ad_daily_metrics`(建+写)<br>`dim_style`(读) |
| 京东与天猫主图周采集 | `product.main_image.weekly` | 京东与天猫万相台主图并行采集,各自完成 PG 入库和飞书插入,互不影响 | 每周日 08:30 | `…/product_commerce/run_weekly_jd_main_image.py`<br>`…/product_commerce/run_weekly_main_image.py` | `main_image_creatives`(写) |
| 飞书商品月度销量表填写 | `product.sales_sheet.daily` | 抓取并校验全部 ERP 报表后填写飞书月度销量表,同名月份页幂等覆盖,无 ERP 编码产品按 0 写入 | 每月 1 日 10:00 | `src/gyxx_flow/modules/product_commerce/sync_monthly_sales_sheet.py` | 无(聚水潭浏览器采集 + 飞书写入) |
+84 -13
View File
@@ -6,16 +6,20 @@ import hashlib
import hmac
import json
import os
import re
import time
from collections.abc import Mapping
from datetime import datetime
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit
from urllib.request import Request, urlopen
ITEMS_QUERY_URI = "/open/data/v1/items/query"
POINTS_EXCHANGE_QUERY_URI = "/open/data/v1/points-exchange/query"
DEFAULT_DATA_INTERACTION_BASE_URL = "https://test.gyxx.cn/api"
_MAX_RESPONSE_BYTES = 16 * 1024 * 1024
_API_DATETIME = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$")
class DataInteractionError(RuntimeError):
@@ -33,6 +37,71 @@ def query_items(
) -> list[dict[str, Any]]:
"""Query the item list and return every item row from the API response."""
return _signed_query(
ITEMS_QUERY_URI,
{"channel": channel, "keyword": keyword},
label="items query",
base_url=base_url,
app_key=app_key,
app_secret=app_secret,
timeout=timeout,
)
def query_points_exchange(
*,
start_time: str | datetime,
end_time: str | datetime,
base_url: str | None = None,
app_key: str | None = None,
app_secret: str | None = None,
timeout: float = 60,
) -> list[dict[str, Any]]:
"""Query points-exchange consumption rows for one closed time window."""
start = _api_datetime(start_time, field="startTime")
end = _api_datetime(end_time, field="endTime")
if start > end:
raise DataInteractionError("points exchange window start is after end")
return _signed_query(
POINTS_EXCHANGE_QUERY_URI,
{"startTime": start, "endTime": end},
label="points exchange query",
base_url=base_url,
app_key=app_key,
app_secret=app_secret,
timeout=timeout,
)
def _api_datetime(value: str | datetime, *, field: str) -> str:
if isinstance(value, datetime):
if value.tzinfo is not None:
raise DataInteractionError(
f"{field} must be a naive Asia/Shanghai datetime"
)
return value.strftime("%Y-%m-%d %H:%M:%S")
if not isinstance(value, str) or not _API_DATETIME.fullmatch(value.strip()):
raise DataInteractionError(
f"{field} must use the 'YYYY-MM-DD HH:MM:SS' format"
)
try:
datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S")
except ValueError as exc:
raise DataInteractionError(f"{field} is not a valid datetime") from exc
return value.strip()
def _signed_query(
uri: str,
payload: Mapping[str, Any],
*,
label: str,
base_url: str | None,
app_key: str | None,
app_secret: str | None,
timeout: float,
) -> list[dict[str, Any]]:
secret = app_secret or os.environ.get("GYXX_DATA_INTERACTION_APP_SECRET", "")
if not secret.strip():
raise DataInteractionError(
@@ -50,19 +119,19 @@ def query_items(
raise DataInteractionError("GYXX_DATA_INTERACTION_BASE_URL must be an HTTP(S) URL")
body = json.dumps(
{"channel": channel, "keyword": keyword},
dict(payload),
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
timestamp = str(int(time.time() * 1000))
body_sha = hashlib.sha256(body).hexdigest()
canonical = f"POST\n{ITEMS_QUERY_URI}\n{timestamp}\n{body_sha}"
canonical = f"POST\n{uri}\n{timestamp}\n{body_sha}"
signature = hmac.new(
secret.encode("utf-8"), canonical.encode("utf-8"), hashlib.sha256
).hexdigest()
request = Request(
base.rstrip("/") + ITEMS_QUERY_URI,
base.rstrip("/") + uri,
data=body,
headers={
"Content-Type": "application/json; charset=utf-8",
@@ -78,31 +147,31 @@ def query_items(
except HTTPError as exc:
detail = exc.read(2_000).decode("utf-8", errors="replace").strip()
suffix = f": {detail}" if detail else ""
raise DataInteractionError(f"items query returned HTTP {exc.code}{suffix}") from exc
raise DataInteractionError(f"{label} returned HTTP {exc.code}{suffix}") from exc
except URLError as exc:
raise DataInteractionError(f"items query request failed: {exc.reason}") from exc
raise DataInteractionError(f"{label} request failed: {exc.reason}") from exc
if len(raw) > _MAX_RESPONSE_BYTES:
raise DataInteractionError("items query response exceeds the size limit")
raise DataInteractionError(f"{label} response exceeds the size limit")
try:
payload = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise DataInteractionError("items query returned invalid JSON") from exc
raise DataInteractionError(f"{label} returned invalid JSON") from exc
return _extract_item_rows(payload)
return _extract_rows(payload, label=label)
def _extract_item_rows(payload: object) -> list[dict[str, Any]]:
def _extract_rows(payload: object, *, label: str) -> list[dict[str, Any]]:
if isinstance(payload, Mapping):
code = payload.get("code")
if code not in (None, 0, 200, "0", "200") and payload.get("success") is not True:
message = str(payload.get("message") or payload.get("msg") or "")[:300]
suffix = f": {message}" if message else ""
raise DataInteractionError(f"items query API reported code {code}{suffix}")
raise DataInteractionError(f"{label} API reported code {code}{suffix}")
if payload.get("success") is False:
message = str(payload.get("message") or payload.get("msg") or "")[:300]
suffix = f": {message}" if message else ""
raise DataInteractionError(f"items query API reported failure{suffix}")
raise DataInteractionError(f"{label} API reported failure{suffix}")
data: object = payload.get("data", payload.get("result"))
else:
data = payload
@@ -114,9 +183,9 @@ def _extract_item_rows(payload: object) -> list[dict[str, Any]]:
data = candidate
break
if not isinstance(data, list):
raise DataInteractionError("items query response does not contain an item array")
raise DataInteractionError(f"{label} response does not contain an item array")
if not all(isinstance(row, Mapping) for row in data):
raise DataInteractionError("items query response contains a non-object item")
raise DataInteractionError(f"{label} response contains a non-object item")
return [dict(row) for row in data]
@@ -124,5 +193,7 @@ __all__ = [
"DEFAULT_DATA_INTERACTION_BASE_URL",
"DataInteractionError",
"ITEMS_QUERY_URI",
"POINTS_EXCHANGE_QUERY_URI",
"query_items",
"query_points_exchange",
]
+82 -7
View File
@@ -45,8 +45,17 @@ DIM_STYLE_RUNTIME_SQL = """
'jd_self', COALESCE(s.jd_self_skus, '[]'::jsonb)
) AS platform_product_ids,
COALESCE(s.bitable_weekly_url, '') AS sales_bitable_url,
COALESCE(s.bitable_main_image_url, '') AS main_image_bitable_url,
COALESCE(s.bitable_creator_url, '') AS creator_bitable_url,
COALESCE(s.bitable_self_creator_url, '') AS self_creator_bitable_url,
COALESCE(s.bitable_weekly_note_url, '') AS weekly_note_analysis_url,
COALESCE(s.bitable_style_analysis_url, '') AS style_analysis_bitable_url,
COALESCE(s.bitable_persona_tm_url, '') AS tm_persona_bitable_url,
COALESCE(s.bitable_persona_jd_url, '') AS jd_persona_bitable_url,
COALESCE(s.bitable_persona_dy_url, '') AS dy_persona_bitable_url,
COALESCE(s.series_name, '') AS series_name,
COALESCE(s.is_series_anchor, FALSE) AS is_series_anchor
COALESCE(s.is_series_anchor, FALSE) AS is_series_anchor,
COALESCE(s.collect_enabled, TRUE) AS collect_enabled
FROM dim_style s
WHERE COALESCE(s.style_name, '') <> ''
ORDER BY s.style_name, s.id
@@ -345,7 +354,13 @@ def merge_lifecycle_style_records(
legacy_rows = dynamic_by_style.get(style_name, [])
lifecycle = lifecycle_by_style.get(style_name)
if lifecycle is None:
records.extend(dict(row) for row in legacy_rows)
# 不在 dim_style 的款式(改名/删除后的残留行)不再继承旧表地址:
# 飞书地址已统一由 dim_style 锚点持有,旧表列只是历史数据。
for row in legacy_rows:
record = dict(row)
for field in URL_FIELDS:
record[field] = ""
records.append(record)
continue
managed, lifecycle_ids = _decode_lifecycle_product_ids(
@@ -399,18 +414,21 @@ def merge_lifecycle_style_records(
if item_id not in dynamic_unknown_ids.setdefault(label, []):
dynamic_unknown_ids[label].append(item_id)
# 销量飞书表:生命进程建档的地址优先,动态配置只在建档为空时兜底。
# 飞书地址统一以 dim_style 为准(2026-10-29):9 类地址全部维护在系列
# 锚点行,旧表同名字段彻底不参与,缺失就是缺失、不再回退。
sales_bitable_url = str(lifecycle.get("sales_bitable_url") or "").strip()
if not sales_bitable_url and legacy_rows:
sales_bitable_url = str(legacy_rows[0].get("sales_bitable_url") or "").strip()
series_name = str(lifecycle.get("series_name") or "").strip()
is_series_anchor = bool(lifecycle.get("is_series_anchor"))
# 生命进程建档是唯一权威:销量表与系列归属直接写进运行时记录,
# 子款即使没填平台编码/销量表,也能被下游按系列主款补齐。
template["sales_bitable_url"] = sales_bitable_url
template["series_name"] = series_name
template["is_series_anchor"] = is_series_anchor
for field in URL_FIELDS:
template[field] = str(lifecycle.get(field) or "").strip()
# 采集开关由产品生命进程在 dim_style 上维护(系列级,读锚点那行)。
# 旧 cmt_styles 回退路径没有该列,按默认 TRUE 处理,保持历史口径。
enabled_flag = lifecycle.get("collect_enabled")
template["collect_enabled"] = True if enabled_flag is None else bool(enabled_flag)
if managed:
labels = []
@@ -450,9 +468,66 @@ def merge_lifecycle_style_records(
)
records.append(output)
_propagate_series_targets(records)
return records
def _propagate_series_targets(records: list[dict[str, Any]]) -> None:
"""把系列内 9 类飞书地址聚合去重后统一下发到整条系列(就地修改)。
约定是地址只维护在 ``dim_style`` 的系列锚点行,子款留空。但建档时子款误填
无法从表结构上禁止,所以这里不「只认锚点」,而是把整条系列的非空地址按
「锚点优先、其余按款名」收集起来,按 URL 去重后取优先级最高的一条下发到
该系列每一行——子款写的地址不会因为不在锚点上而被静默丢弃。
一个表位只能落一张表:同一字段出现多个不同地址时保留优先级最高的那条并
打印告警,提示到锚点手动裁决,避免同系列数据被拆到两个表。
没有锚点时按款名排序取第一个,与采集开关、ERP 归集的判定口径保持一致。
"""
by_series: dict[str, list[dict[str, Any]]] = {}
for record in records:
series = " ".join(str(record.get("series_name") or "").split())
if series:
by_series.setdefault(series, []).append(record)
for series, rows in by_series.items():
# merge 会把同一款展开成多条平台记录,聚合前先按款归并,避免重复计数
by_style: dict[str, list[dict[str, Any]]] = {}
for row in rows:
by_style.setdefault(str(row.get("style_name") or ""), []).append(row)
ordered = sorted(
by_style,
key=lambda name: (
not any(row.get("is_series_anchor") for row in by_style[name]),
name,
),
)
chosen: dict[str, str] = {}
for field in URL_FIELDS:
candidates: list[str] = []
for name in ordered:
for row in by_style[name]:
value = _plain_url(row.get(field))
if value and value not in candidates:
candidates.append(value)
if not candidates:
continue
if len(candidates) > 1:
print(
f"[WARN] 系列「{series}」{field} 有 {len(candidates)} 个不同地址,"
f"按「{ordered[0]}」下发并忽略其余: {candidates[1:]}"
)
chosen[field] = candidates[0]
if not chosen:
continue
for row in rows:
row.update(chosen)
def _plain_url(value: Any) -> str:
text = str(value or "").strip()
match = re.search(r"\]\((https?://[^)]+)\)", text)
@@ -15,6 +15,7 @@ PRODUCT_SCHEDULED_WORKFLOW_IDS = (
"product.daily",
"product.alert.daily",
"product.online_stock_alert.daily",
"product.points_exchange.weekly",
"product.ecommerce_costs.daily",
"product.style_analysis.interval",
"product.main_image.weekly",
@@ -51,6 +51,11 @@ DEFAULT_FROM_DATE = date.today() - timedelta(days=1)
DEFAULT_TO_DATE = DEFAULT_FROM_DATE
MAX_RANGE_DAYS = 366
EVIDENCE_DIR_NAME = "erp全店铺款式日报"
# 聚水潭商品主题分析报表固定每页 200 行明细,且没有可用的页大小设置;
# 款式扩到 198 个 ERP 编码后必须翻页,否则只拿到第一页的 62 个编码。
PAGE_ROW_LIMIT = 200
MAX_REPORT_PAGES = 12
PAGE_TURN_TIMEOUT = 45
@dataclass(frozen=True, slots=True)
@@ -76,10 +81,19 @@ def inclusive_dates(start: date, end: date) -> tuple[date, ...]:
def build_style_plans(payload: Mapping[str, Any]) -> tuple[StylePlan, ...]:
plans: list[StylePlan] = []
code_owners: dict[str, str] = {}
"""Build per-style ERP plans, folding series codes into their anchor.
同系列多款时 loader 会把子款编码聚合并集到锚点(series.erp_owner),
子款仍保留自己的编码供按款查询。采集计划按归属只保留一次:编码归锚点,
子款剔除已被锚点承接的重复项;真正跨系列的重复仍然报错。
"""
codes_by_style: dict[str, tuple[str, ...]] = {}
owner_of: dict[str, str] = {}
for raw_name, raw_config in (payload.get("styles") or {}).items():
style_name = str(raw_name).strip()
if not style_name:
continue
config = raw_config or {}
codes = tuple(
dict.fromkeys(
@@ -88,15 +102,42 @@ def build_style_plans(payload: Mapping[str, Any]) -> tuple[StylePlan, ...]:
if str(code).strip()
)
)
if not style_name or not codes:
if not codes:
continue
codes_by_style[style_name] = codes
owner = str((config.get("series") or {}).get("erp_owner") or "").strip()
if owner:
owner_of[style_name] = owner
holders: dict[str, list[str]] = {}
for style_name, codes in codes_by_style.items():
for code in codes:
previous = code_owners.setdefault(code, style_name)
if previous != style_name:
raise ValueError(
f"ERP style code {code} belongs to both {previous!r} and {style_name!r}"
)
plans.append(StylePlan(style_name=style_name, erp_style_codes=codes))
holders.setdefault(code, []).append(style_name)
resolved: dict[str, str] = {}
for code, names in holders.items():
if len(names) == 1:
resolved[code] = names[0]
continue
# 同系列归集:持有者里被其余持有者指认为 ERP 锚点的那一款承接该编码
candidates = [
name
for name in names
if any(owner_of.get(other) == name for other in names if other != name)
]
if len(candidates) == 1:
resolved[code] = candidates[0]
continue
raise ValueError(
f"ERP style code {code} belongs to both {names[0]!r} and {names[1]!r}"
)
plans: list[StylePlan] = []
for style_name, codes in codes_by_style.items():
own_codes = tuple(code for code in codes if resolved.get(code) == style_name)
if not own_codes:
continue
plans.append(StylePlan(style_name=style_name, erp_style_codes=own_codes))
if not plans:
raise RuntimeError("style configuration contains no ERP style codes")
return tuple(plans)
@@ -206,7 +247,9 @@ def set_all_shop_daily_filters(
ENSURE_CUSTOM_COLUMNS_JS = r"""
() => {
(options) => {
const redrawPage = (options && options.redrawPage) || 1;
const forceRedraw = Boolean(options && options.forceRedraw);
const cols = window._Cols || [];
let changed = 0;
for (const col of cols) {
@@ -216,14 +259,58 @@ ENSURE_CUSTOM_COLUMNS_JS = r"""
changed += 1;
}
}
if (changed > 0 && typeof window._RptRefresh === 'function') {
try { window._RptRefresh(); } catch (e) {}
// 只改 visible 不会补出数据行缺失的 td:聚水潭按旧列定义渲染数据行,
// 行内仍是旧的单元格数,按新列位置取值就会读到 undefined(历史全 0 成因)。
// 换页会强制按当前列定义重绘,所以顺带跳到目标页。
let redrawn = false;
if (changed > 0 || forceRedraw) {
// _RptRefresh 会走一次完整重算(公式列随之补齐),GotoPage 负责
// 让数据行按新列定义重绘;两者都试,缺一都可能停在旧行结构上。
if (forceRedraw && typeof window._RptRefresh === 'function') {
try { window._RptRefresh(); } catch (e) {}
}
if (typeof GotoPage === 'function') {
try { GotoPage(redrawPage || 1); redrawn = true; } catch (e) {}
}
}
return changed;
return { changed: changed, redrawn: redrawn };
}
"""
def ensure_custom_columns_rendered(
page: Any, *, redraw_page: int = 1, force_redraw: bool = False
) -> dict[str, Any]:
"""确保自定义销量/退货量列可见,并触发一次重绘补齐数据行单元格。
2026-10-11 起聚水潭报表模板把自定义公式列重置为隐藏,隐藏列不渲染
数据单元格;而 ``generate_report`` 每次查询都会把列可见性重置回隐藏。
因此本函数必须在查询之后调用,并在 visible 变化时换页重绘,否则
``自定义销量`` 整列读不到值。
``force_redraw`` 用于「列已 visible、但数据行仍按旧列数渲染」的中间态:
此时列状态无需修改,仍要换页才能让每行补齐到新的单元格数。
"""
summary = {"changed": 0, "redrawn": 0}
for frame in page.frames:
if "allsku.aspx" not in (frame.url or "").lower():
continue
try:
result = frame.evaluate(
ENSURE_CUSTOM_COLUMNS_JS,
{"redrawPage": int(redraw_page), "forceRedraw": bool(force_redraw)},
)
except Exception as exc:
print(f"[WARN] 自定义列补设失败: {type(exc).__name__}: {exc}", flush=True)
continue
if isinstance(result, dict):
summary["changed"] += int(result.get("changed") or 0)
if result.get("redrawn"):
summary["redrawn"] += 1
return summary
def ensure_custom_columns_visible(page: Any) -> int:
"""Keep 自定义销量/自定义退货量 visible before the report renders.
@@ -232,15 +319,47 @@ def ensure_custom_columns_visible(page: Any) -> int:
(其尾部浮头重建可能抛错,但不影响数据行渲染)。
"""
changed_total = 0
return int(ensure_custom_columns_rendered(page, redraw_page=1)["changed"])
def _allsku_frame(page: Any) -> Any:
for frame in page.frames:
if "allsku.aspx" not in (frame.url or "").lower():
continue
try:
changed_total += int(frame.evaluate(ENSURE_CUSTOM_COLUMNS_JS))
except Exception:
continue
return changed_total
if "allsku.aspx" in (frame.url or "").lower():
return frame
return None
def report_page_count(page: Any) -> int:
frame = _allsku_frame(page)
if frame is None:
return 1
try:
return max(1, int(frame.evaluate("() => Number(window._RptMaxPage) || 1")))
except Exception:
return 1
GOTO_REPORT_PAGE_JS = r"""
(options) => {
if (typeof GotoPage !== 'function') return false;
try {
GotoPage((options && options.page) || 1);
return true;
} catch (e) {
return false;
}
}
"""
def goto_report_page(page: Any, page_no: int) -> bool:
frame = _allsku_frame(page)
if frame is None:
return False
try:
return bool(frame.evaluate(GOTO_REPORT_PAGE_JS, {"page": int(page_no)}))
except Exception:
return False
def extract_style_rows(
@@ -296,44 +415,117 @@ def extract_style_rows(
}
// 隐藏列的 th 仍在表头,但数据行不渲染对应 td:数据行的第 j 格
// 对应第 j 个可见列。用 window._Cols 把表头列索引换算成行内位
// 置,避免隐藏列造成索引错位;拿不到 _Cols 时按原索引直取。
let stylePos = styleCol;
let productPos = productCol;
let namePos = productNameCol;
let salesPos = salesCol;
let returnsPos = returnsCol;
if (Array.isArray(window._Cols) && window._Cols.length === headerTexts.length) {
const visibleIdx = [];
for (let i = 0; i < window._Cols.length; i++) {
if (window._Cols[i].visible !== false) visibleIdx.push(i);
}
stylePos = visibleIdx.indexOf(styleCol);
productPos = productCol >= 0 ? visibleIdx.indexOf(productCol) : -1;
namePos = productNameCol >= 0 ? visibleIdx.indexOf(productNameCol) : -1;
salesPos = visibleIdx.indexOf(salesCol);
returnsPos = visibleIdx.indexOf(returnsCol);
// 对应第 j 个可见列。必须用 window._Cols 把表头列索引换算成行内
// 位置——不做换算而按表头索引直取,会整体偏移到相邻列(历史表现
// 为销量读到月累计/金额列,被放大上百倍)。换算所需的 _Cols 一旦
// 缺失或与表头列数不一致,就无法确定位置,此时一律 fail closed,
// 绝不退回"按原索引直取"。
if (!Array.isArray(window._Cols) || window._Cols.length !== headerTexts.length) {
return {
ok: false,
reason: 'column_layout_unknown',
error: `column layout unresolved: _Cols=${Array.isArray(window._Cols) ? window._Cols.length : 'none'} header=${headerTexts.length}`,
header_texts: headerTexts,
};
}
const colLabel = index => {
const col = window._Cols[index];
return norm(col && (col.text || col.defaultText));
};
// 表头 th 文本与 _Cols 必须是同一列,否则说明表头已重建而列定义
// 尚未同步(中间态),按任何一个索引取都不可信。
const labelMismatch = [
[styleCol, '款式编码'],
[salesCol, '自定义销量'],
[returnsCol, '自定义退货量'],
].filter(([index, expected]) => colLabel(index) !== norm(expected));
if (labelMismatch.length) {
return {
ok: false,
reason: 'column_layout_unknown',
error: `header/_Cols label mismatch: ${labelMismatch.map(([i]) => i).join(',')}`,
header_texts: headerTexts,
};
}
const visibleIdx = [];
for (let i = 0; i < window._Cols.length; i++) {
if (window._Cols[i].visible !== false) visibleIdx.push(i);
}
const toPos = index => (index >= 0 ? visibleIdx.indexOf(index) : -1);
const stylePos = toPos(styleCol);
const productPos = toPos(productCol);
const namePos = toPos(productNameCol);
const salesPos = toPos(salesCol);
const returnsPos = toPos(returnsCol);
// 值域交叉校验用的参考列:自定义销量是"当期销量"口径,正常不可能
// 超过同行的月销量一个量级。读到月累计/金额列时这里会炸。
const monthSalesPos = toPos(findCol('月销量'));
const monthReturnsPos = toPos(findCol('月退货量'));
// 每行应有几个 td:等于可见列数。数量不符=该行仍按旧列定义渲染
// (重绘未完成),位置不可信。
const expectedCells = visibleIdx.length;
if (stylePos < 0 || salesPos < 0 || returnsPos < 0) {
return {
ok: false,
reason: 'custom_columns_hidden',
error: `custom sales/return columns hidden: style=${stylePos}, sales=${salesPos}, returns=${returnsPos}`,
header_texts: headerTexts,
};
}
const rows = [];
let renderedSalesCells = 0;
let blankSalesCells = 0;
let zeroSalesRows = 0;
let layoutMismatchRows = 0;
let outOfRangeRows = 0;
let monthRefRows = 0;
for (const tr of Array.from(document.querySelectorAll('tr'))) {
const cells = Array.from(tr.querySelectorAll('td, th'));
if (cells.length <= Math.max(stylePos, salesPos, returnsPos)) continue;
// 只用 td:表头行的 th 会让列位置整体右移,
// 而分组行/浮头行里混入的 th 同样会污染样式编码列的取值。
const cells = Array.from(tr.querySelectorAll('td'));
if (!cells.length) continue;
const styleCode = norm(cells[stylePos]?.textContent);
// 只有命中本次请求编码的行才是数据行:报表里的小计/分组行 td
// 数与列定义天然不同,不能拿它们判定列错位。
if (!requested.has(styleCode)) continue;
// 数据行 td 数必须与可见列数完全相等。少一格就说明该行仍按旧
// 列定义渲染,此时任何位置取值都可能落到相邻列(读到月累计或
// 金额),必须整页拒绝重绘,不能按"位置够长就往里取"处理。
if (cells.length !== expectedCells) {
layoutMismatchRows += 1;
continue;
}
const salesText = String(cells[salesPos]?.textContent ?? '').trim();
const returnText = String(cells[returnsPos]?.textContent ?? '').trim();
const salesValue = parseInteger(salesText);
const returnValue = parseInteger(returnText);
// 值域交叉校验:自定义销量是"当期"口径,正常不会超过同行月销量
// 一个量级。列错位读到月累计/金额时这里的比例会失真,用于兜住
// 「结构看起来正常但语义错列」的情况(2026-10-30 事故)。
if (monthSalesPos >= 0) {
const monthSalesText = String(cells[monthSalesPos]?.textContent ?? '').trim();
if (monthSalesText) {
monthRefRows += 1;
if (salesValue > parseInteger(monthSalesText) * 2 + 50) outOfRangeRows += 1;
}
}
if (monthReturnsPos >= 0) {
const monthReturnsText = String(cells[monthReturnsPos]?.textContent ?? '').trim();
if (monthReturnsText) {
monthRefRows += 1;
if (returnValue > parseInteger(monthReturnsText) * 2 + 50) outOfRangeRows += 1;
}
}
if (salesText) renderedSalesCells += 1;
else blankSalesCells += 1;
if (salesValue === 0) zeroSalesRows += 1;
rows.push({
erp_style_code: styleCode,
product_code: productPos >= 0 ? norm(cells[productPos]?.textContent) : '',
product_name: namePos >= 0 ? norm(cells[namePos]?.textContent) : '',
sales: parseInteger(cells[salesPos]?.textContent),
return_qty: parseInteger(cells[returnsPos]?.textContent),
sales: salesValue,
return_qty: returnValue,
});
}
const foundCodes = Array.from(new Set(rows.map(row => row.erp_style_code))).sort();
@@ -344,7 +536,58 @@ def extract_style_rows(
if (!rows.length) {
return {
ok: false,
error: 'report contains none of the requested ERP style codes',
reason: layoutMismatchRows ? 'row_layout_mismatch' : 'no_requested_rows',
error: `report contains none of the requested ERP style codes (mismatched rows: ${layoutMismatchRows})`,
layout_mismatch_rows: layoutMismatchRows,
header_texts: headerTexts,
};
}
if (renderedSalesCells === 0) {
// 自定义销量是聚水潭的公式列:列被隐藏或公式尚未算完时
// 单元格是空的。空 ≠ 0,此时返回未就绪让调用方继续等待,
// 绝不能静默写 0(历史上 10-13~10-19、10-22 全表为 0 就是这样来的)。
return {
ok: false,
reason: 'custom_sales_not_rendered',
error: `custom sales cells blank: ${blankSalesCells}/${rows.length}`,
row_count: rows.length,
sales_col: salesCol,
sales_pos: salesPos,
header_texts: headerTexts,
};
}
if (layoutMismatchRows > 0) {
// 有行的 td 数与可见列数不符:整页列位置不可信,重绘后重读。
return {
ok: false,
reason: 'row_layout_mismatch',
error: `row cells mismatch: ${layoutMismatchRows} rows`,
row_count: rows.length,
expected_cells: expectedCells,
header_texts: headerTexts,
};
}
if (outOfRangeRows > 0) {
// 结构与列名都对上了,但值本身不可信(销量超过同行月销量一个
// 量级)。典型成因是日期筛选未真正生效、报表返回了月/年累计值。
return {
ok: false,
reason: 'sales_out_of_range',
error: `sales out of monthly range: ${outOfRangeRows} rows (reference rows ${monthRefRows})`,
row_count: rows.length,
out_of_range_rows: outOfRangeRows,
header_texts: headerTexts,
};
}
if (rows.length >= 100 && zeroSalesRows / rows.length >= 0.8) {
// 公式列在补值过程中会先把整列渲染成 0 再逐个算出来;正常日
// 零销量行占比约 50%~60%(2026-10 实测),超过 80% 说明整页
// 还没算完。宁可继续等也不要写一批 0 覆盖真实销量。
return {
ok: false,
reason: 'custom_sales_zero_placeholder',
error: `custom sales zero rows: ${zeroSalesRows}/${rows.length}`,
row_count: rows.length,
header_texts: headerTexts,
};
}
@@ -355,6 +598,25 @@ def extract_style_rows(
missing_codes: missingCodes,
signature,
header_texts: headerTexts,
rendered_sales_cells: renderedSalesCells,
blank_sales_cells: blankSalesCells,
zero_sales_rows: zeroSalesRows,
layout_mismatch_rows: layoutMismatchRows,
// 列布局快照:写进 evidence,下次出问题可直接比对,不用复现。
layout: {
header_cells: headerTexts.length,
visible_cells: expectedCells,
style_col: styleCol,
style_pos: stylePos,
sales_col: salesCol,
sales_pos: salesPos,
returns_col: returnsCol,
returns_pos: returnsPos,
month_sales_pos: monthSalesPos,
month_returns_pos: monthReturnsPos,
row_cells: expectedCells,
month_ref_rows: monthRefRows,
},
};
}
""",
@@ -476,6 +738,56 @@ def aggregate_style_records(
]
def _recent_sales_baseline(report_date: date, *, days: int = 14) -> float | None:
"""最近若干天单日销量合计的中位数,用于识别量级异常。"""
with get_conn() as conn, conn.cursor() as cur:
cur.execute(
"""
SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY day_total)
FROM (
SELECT metric_date, sum(sales) AS day_total
FROM erp_all_shop_style_daily_metrics
WHERE metric_date < %s
AND metric_date >= %s::date - (%s || ' days')::interval
GROUP BY metric_date
) recent
""",
(report_date, report_date, days),
)
row = cur.fetchone()
value = row[0] if row else None
return float(value) if value else None
def _guard_daily_total(
report_date: date,
records: Sequence[Mapping[str, Any]],
*,
force: bool = False,
) -> None:
"""单日合计与近 14 天中位数差 8 倍以上就拒绝落库。
2026-10-30 事故里,日期筛选未真正生效使报表返回月累计值,单日合计
704,140 而正常约 2,000(350 倍)。列级校验挡住列错位,这道闸挡住
「列对但统计口径错」这一层,宁可失败也不写脏数据。
"""
if force:
return
total = sum(int(record.get("sales") or 0) for record in records)
baseline = _recent_sales_baseline(report_date)
if not baseline or baseline <= 0:
return
if total <= baseline * 8:
return
raise RuntimeError(
f"all-shop daily total looks wrong for {report_date.isoformat()}: "
f"sales={total} vs recent median={baseline:.0f} "
f"({total / baseline:.1f}x). Refusing to persist; rerun with --force if business volume really changed."
)
def write_evidence(
report_date: date,
*,
@@ -493,8 +805,12 @@ def write_evidence(
"sales_header": "自定义销量",
"return_header": "自定义退货量",
"selection": dict(selection),
"pages": int(report.get("pages") or 1),
"max_page": int(report.get("max_page") or 1),
"found_codes": list(report.get("found_codes") or ()),
"missing_codes": list(report.get("missing_codes") or ()),
# 列布局快照:出问题时可直接判断「列错位」还是「数据本身」,无需复现。
"layout": dict(report.get("layout") or {}),
"rows": list(report.get("rows") or ()),
"styles": [
{
@@ -510,6 +826,206 @@ def write_evidence(
return output_path
def _wait_rendered_page(
page: Any,
requested_codes: Sequence[str],
*,
page_no: int,
timeout_seconds: int,
previous_signature: str | None = None,
expect_full_page: bool = False,
) -> dict[str, Any]:
"""Wait for one report page whose custom sales cells are actually rendered.
``previous_signature`` 用于翻页:``GotoPage`` 是异步的,调用后 DOM 仍是
上一页,若只看「签名稳定」会立刻把上一页数据当成新页收下,
结果多页合并后去重回同一批行(10-28 实测 3 页只有 200 行)。
``expect_full_page``:报表每页固定 200 行,除最后一页外不该出现不满页。
不满页说明数据行还在陆续插入,此时签名字段虽已稳定但行集不全,
直接收下会漏采编码(2026-10-30 实测第 1 页只有 174 行、只命中 57/199
个编码,后续页面也补不回第一页缺的行)。
"""
deadline = time.monotonic() + timeout_seconds
last_signature: str | None = None
stable = 0
redraws = 0
reached = previous_signature is None
last_error = "report page did not render custom sales column"
def _declared_pages() -> int:
"""报表自己声明的总页数;不可信时返回 1(只做加法兜底)。"""
frame = _allsku_frame(page)
if frame is None:
return 1
try:
return max(1, int(frame.evaluate("() => Number(window._RptMaxPage) || 1")))
except Exception:
return 1
while time.monotonic() < deadline:
frame = _allsku_frame(page)
if frame is None:
last_error = "allsku iframe not found"
time.sleep(0.5)
continue
try:
result = extract_style_rows(frame, requested_codes)
except Exception as exc:
result = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
if result.get("ok") and len(result.get("rows") or ()) < PAGE_ROW_LIMIT:
# 不满 200 行只有两种可能:这是最后一页,或数据行还在陆续插入。
# 报表声明的总页数大于当前页时必然是后者,继续等。
short = len(result.get("rows") or ())
if expect_full_page or _declared_pages() > page_no:
result = {
"ok": False,
"reason": "partial_page",
"error": f"page {page_no} rendered {short}/{PAGE_ROW_LIMIT} rows",
}
if result.get("ok"):
signature = str(result.get("signature") or "")
if not reached:
# 翻页后内容尚未变化:继续等新页渲染出来。
if signature == previous_signature:
last_signature = None
stable = 0
time.sleep(0.5)
continue
reached = True
if signature == last_signature:
stable += 1
else:
last_signature = signature
stable = 1
# 自定义销量是公式列,翻页后先出 0、再逐个单元格补齐;
# 稳定阈值取 6 次(≈2.4 秒不变)才能跨过「补值中」的中间态。
required = 5 if result.get("reason") == "no_data" else 6
if stable >= required:
result["page_no"] = page_no
result["frame_url"] = frame.url
return result
else:
last_error = str(result.get("error") or result.get("reason") or last_error)
last_signature = None
stable = 0
# 列为空/被隐藏/行内单元格错位/整列占位 0 都是「未渲染完」,
# 不是「销量为 0」:补设列并强制重绘后重读。
reason = result.get("reason")
if reason in (
"custom_sales_not_rendered",
"custom_columns_hidden",
"column_layout_unknown",
"row_layout_mismatch",
"custom_sales_zero_placeholder",
"sales_out_of_range",
"partial_page",
) and redraws < 6:
summary = ensure_custom_columns_rendered(
page,
redraw_page=page_no,
# 这几种原因都说明「列已可见但数据行/公式没算完」,
# 必须强制重算,否则只会原地等同一份旧 DOM。
force_redraw=True,
)
if summary.get("changed") or summary.get("redrawn"):
redraws += 1
last_error = f"{last_error}; redraw #{redraws} on page {page_no}"
time.sleep(3.0)
continue
time.sleep(0.4)
raise TimeoutError(
f"timeout waiting for rendered page {page_no}: {last_error}"
)
def collect_report_pages(
page: Any,
requested_codes: Sequence[str],
*,
report_timeout: int,
) -> dict[str, Any]:
"""Collect every page of the allsku report and merge the product rows.
报表固定每页 200 行;款式扩到 198 个 ERP 编码后单页放不下,
只读第一页会漏掉大半编码(10-28 就是这样:198 编码只覆盖 62 个)。
翻页条件按「上一页是否装满一页」判断,不依赖 ``window._RptMaxPage``:
该变量在增量查询后会被重置为 1,实测连续补采 8 天只有第一天翻页成功。
"""
first = _wait_rendered_page(
page,
requested_codes,
page_no=1,
timeout_seconds=report_timeout,
)
# 等待结束后再读总页数:生成报表的瞬间 _RptMaxPage 还是旧值 1。
max_page = report_page_count(page)
rows: list[dict[str, Any]] = list(first.get("rows") or ())
previous_signature = str(first.get("signature") or "")
current_count = len(rows)
visited = 1
page_no = 1
# 翻页条件:上一页装满 200 行,或 window._RptMaxPage 指出还有后续页。
# 后者用于兜住「第 1 页渲染不全、行数不足 200」的情况——只看行数会直接
# 判成最后一页而漏采。该变量在增量查询后可能被重置为 1,因此只做加法
# 兜底,翻到空页时由 _wait_rendered_page 超时退出。
while current_count >= PAGE_ROW_LIMIT or page_no < max_page:
page_no += 1
if page_no > MAX_REPORT_PAGES:
print(f"[WARN] 全店铺报表页数超过 {MAX_REPORT_PAGES},按上限停止", flush=True)
break
if not goto_report_page(page, page_no):
print(f"[WARN] 全店铺报表翻页失败,停在页 {visited}", flush=True)
break
try:
last = _wait_rendered_page(
page,
requested_codes,
page_no=page_no,
timeout_seconds=min(report_timeout, PAGE_TURN_TIMEOUT),
previous_signature=previous_signature,
expect_full_page=page_no < max_page,
)
except TimeoutError:
# 翻页无新内容=已经到最后一页(GotoPage 越界不报错,只是不刷新)。
break
previous_signature = str(last.get("signature") or "")
page_rows = list(last.get("rows") or ())
print(f"[ERP ALL] 第 {page_no} 页 {len(page_rows)} 行", flush=True)
rows.extend(page_rows)
current_count = len(page_rows)
visited = page_no
deduped: list[dict[str, Any]] = []
seen: set[tuple[str, str]] = set()
for row in rows:
key = (str(row.get("erp_style_code") or ""), str(row.get("product_code") or ""))
if key in seen:
continue
seen.add(key)
deduped.append(row)
found_codes = sorted({str(row.get("erp_style_code") or "") for row in deduped})
missing_codes = [code for code in requested_codes if str(code) not in found_codes]
return {
"rows": deduped,
"found_codes": found_codes,
"missing_codes": missing_codes,
"signature": json.dumps(
[[row.get("erp_style_code"), row.get("product_code"), row.get("sales"), row.get("return_qty")]
for row in deduped],
ensure_ascii=False,
),
"pages": visited,
"max_page": visited,
"header_texts": first.get("header_texts"),
"layout": first.get("layout"),
}
def collect_one_date(
page: Any,
filter_frame: Any,
@@ -520,20 +1036,23 @@ def collect_one_date(
previous_signature: str | None,
report_timeout: int,
) -> tuple[dict[str, Any], dict[str, Any]]:
del previous_url, previous_signature # 多页合并后按页内稳定签名判断新鲜度
codes = all_erp_codes(plans)
erp.clear_code(filter_frame)
selection = set_all_shop_daily_filters(
filter_frame, report_date=report_date, erp_codes=codes
)
ensure_custom_columns_visible(page)
erp.generate_report(filter_frame)
report = wait_for_style_rows(
page,
requested_codes=codes,
previous_url=previous_url,
previous_signature=previous_signature,
filter_frame=filter_frame,
timeout_seconds=report_timeout,
# 查询会把自定义列重置回隐藏;补列交给 collect_report_pages 的等待循环,
# 那里能区分「列隐藏」与「列可见但数据行未重绘」两种状态再重绘。
report = collect_report_pages(
page, codes, report_timeout=report_timeout
)
print(
f"[ERP ALL] {report_date.isoformat()} 采集 {report['pages']} 页 "
f"行 {len(report['rows'])} 命中编码 {len(report['found_codes'])}/{len(codes)} "
f"缺 {len(report['missing_codes'])}",
flush=True,
)
return selection, report
@@ -609,6 +1128,7 @@ def collect_and_persist(
timeout_ms: int,
report_timeout: int,
report_attempts: int,
force: bool = False,
) -> None:
config = load_config(config_path)
login_config = config.get("login") or {}
@@ -641,6 +1161,7 @@ def collect_and_persist(
selected_shop_count=selected_shop_count,
source=provisional_source,
)
_guard_daily_total(report_date, records, force=force)
evidence_path = write_evidence(
report_date, selection=selection, report=report, records=records
)
@@ -766,6 +1287,7 @@ def main(argv: Sequence[str] | None = None) -> int:
timeout_ms=args.timeout_ms,
report_timeout=args.report_timeout,
report_attempts=args.report_attempts,
force=args.force,
)
print(f"[DONE] Collected {len(pending_dates)} all-shop daily reports.")
return 0
@@ -63,8 +63,9 @@ LOGIN_URL_DEFAULT = "https://www.erp321.com/login.aspx"
EVIDENCE_DIR_NAME = "erp_sales_theme_daily"
# ── 业务口径常量(与页面筛选一致) ─────────────────────────────────────────────
# 商品品牌: 光影行星 / 光影 / gyxx (2026-10-12 截图勾选项)
BRANDS: tuple[str, ...] = ("光影行星", "光影", "gyxx")
# 商品品牌: 全部不勾 = 不限品牌(2026-10-19 用户确认)。SET_FILTERS_JS 对空列表
# 会把品牌区全部取消勾选后回读校验。
BRANDS: tuple[str, ...] = ()
# 销售渠道(店铺): 光影系 22 家渠道(2026-10-12 用户确认补回 JD光影行星旗舰店
# 与 得物(直发)-光影行星)。名称必须与 filter.aspx 复选框标签逐字一致;
@@ -69,6 +69,14 @@ DEFAULT_BRAND = "光影行星"
# 保留机制供临时排障使用,但不要把业务款式永久写成成功的 0/0。
ERP_SLOW_SKIP_CODES: set[str] = set()
# 重试后仍失败的「编码级」条目容忍上限(2026-10-29)。
# 背景:锚点款按系列聚合 ERP 编码后,会把个别只存在于聚水潭、三平台都没有商品编码
# 的占位编码(例如「宙斯-10244」)一起采。这类编码本身查不到数据,某些店铺的
# allsku 报表不渲染 iframe 而一直超时,导致整份商品经营日报当天完全不入库。
# 这里的容忍只针对 failed_codes:页面级 / 款式级失败、或编码失败数超过上限,
# 仍然 exit 1 阻断日报。失败编码照常写入 failed_<date>_erp.json 供补采。
ERP_TOLERATED_FAILED_CODES = 3
# 兼容 PLATFORMS 旧名 (其它模块可能 import 它, 兜底给光影行星)
PLATFORMS: dict[str, dict[str, Any]] = PLATFORMS_BY_BRAND[DEFAULT_BRAND]
@@ -245,7 +253,7 @@ def print_skip_summary(skip_report: dict[str, Any]) -> None:
return
print(
f"[ERP] 来源={src}, 采集 {len(skip_report['collectable'])} 款, "
f"跳过 {len(skip_report['skipped'])} 款 (缺 ERP 编码 / 平台商品编码 / 飞书表地址)"
f"跳过 {len(skip_report['skipped'])} 款 (未标记采集 / 缺 ERP 编码 / 已由同系列归集 / 缺销量表)"
)
for s in skip_report["skipped"]:
print(f" - {s['style']}: {s['reason']}")
@@ -405,7 +413,9 @@ def _extract_by_idx(table_frame: Any, sales_idx: int, returns_idx: int) -> dict[
ENSURE_CUSTOM_COLUMNS_JS = r"""
() => {
(options) => {
const redrawPage = (options && options.redrawPage) || 1;
const forceRedraw = Boolean(options && options.forceRedraw);
const cols = window._Cols || [];
let changed = 0;
for (const col of cols) {
@@ -415,31 +425,43 @@ ENSURE_CUSTOM_COLUMNS_JS = r"""
changed += 1;
}
}
if (changed > 0 && typeof window._RptRefresh === 'function') {
try { window._RptRefresh(); } catch (e) {}
// 只改 visible 不会补出数据行缺失的 td:聚水潭按旧列定义渲染数据行,
// 行内仍是旧的单元格数,按新列位置取值就会读到 undefined(历史全 0 成因)。
// 换页会强制按当前列定义重绘,所以顺带跳到目标页。
let redrawn = false;
if ((changed > 0 || forceRedraw) && typeof GotoPage === 'function') {
try { GotoPage(redrawPage || 1); redrawn = true; } catch (e) {}
}
return changed;
return { changed: changed, redrawn: redrawn };
}
"""
def ensure_custom_columns_visible(page: Any) -> int:
"""backfill 模式: 把模板重置隐藏的自定义销量/退货量列恢复可见。
def ensure_custom_columns_visible(page: Any, *, force_redraw: bool = False) -> dict[str, Any]:
"""backfill 模式: 恢复自定义销量/退货量列可见,并触发重绘补齐行内单元格。
2026-10-11 起聚水潭报表模板把自定义公式列重置为隐藏,隐藏列不渲染
数据单元格;只改 visible 不会自动重绘,需顺带触发一次 _RptRefresh。
与 backfill_erp_all_shop_daily.py 同源(该脚本 10-13 实测有效)。
数据单元格;只改 ``visible`` 不会重绘,``_RptRefresh()`` 在未渲染时直接
抛错,因此改用 ``GotoPage`` 换页强制按当前列定义重绘。与
``backfill_erp_all_shop_daily.py`` 同一套实现(该脚本 10-13 起实测有效)。
"""
changed_total = 0
summary = {"changed": 0, "redrawn": 0}
for frame in page.frames:
if "allsku.aspx" not in (frame.url or "").lower():
continue
try:
changed_total += int(frame.evaluate(ENSURE_CUSTOM_COLUMNS_JS))
result = frame.evaluate(
ENSURE_CUSTOM_COLUMNS_JS,
{"redrawPage": 1, "forceRedraw": bool(force_redraw)},
)
except Exception:
continue
return changed_total
if isinstance(result, dict):
summary["changed"] += int(result.get("changed") or 0)
if result.get("redrawn"):
summary["redrawn"] += 1
return summary
def _extract_by_header(table_frame: Any, sales_header: str, returns_header: str) -> dict[str, Any]:
@@ -1049,9 +1071,17 @@ def wait_for_metrics(page: Any, erp_code: str, timeout_seconds: int = 45,
# previous_url 可能在 _reload_allsku_iframe 后被重置为 None
while True:
# backfill 模式: 报表刷新会把自定义列的可见性重置回隐藏, 每轮恢复一次
# backfill 模式: 报表刷新会把自定义列的可见性重置回隐藏, 每轮恢复一次;
# 首轮额外强制换页重绘, 否则数据行仍按旧列数渲染, 取到的是空单元格(会被读成 0)
if MODE == "backfill":
ensure_custom_columns_visible(page)
redraw = ensure_custom_columns_visible(page, force_redraw=poll_count == 0)
if redraw["changed"] or redraw["redrawn"]:
print(
f"[ERP] {erp_code}: 自定义列重绘 changed={redraw['changed']} "
f"redrawn={redraw['redrawn']}",
flush=True,
)
page.wait_for_timeout(1200)
# 先检测"操作频繁"弹窗 (在主页 DOM, 不在 allsku iframe)
if filter_frame is not None and freq_dialog_handled < MAX_FREQ_DIALOG:
if _detect_and_close_freq_dialog(page):
@@ -1253,7 +1283,7 @@ def _run_one_code(page: Any, filter_frame: Any, platform: dict[str, Any], erp_co
set_platform_and_code(filter_frame, platform["shops"], erp_code)
if MODE == "backfill":
# 自定义销量/退货量列被模板重置隐藏时先恢复可见, 否则数据行不渲染对应格
ensure_custom_columns_visible(page)
ensure_custom_columns_visible(page, force_redraw=True)
prev_state = _get_allsku_frame_state(page)
generate_report(filter_frame)
return wait_for_metrics(
@@ -1515,6 +1545,15 @@ def collect_in_page(page: Any, styles: list[dict[str, Any]], data_dir: Path,
"reports": len(all_results),
"valid_codes": valid_codes,
"has_failures": failed.has_failures(),
# 失败明细计数:供 main 判断「个别编码单平台超时」能否降级为警告,
# 页面级/款式级失败仍然一律阻断(见 ERP_TOLERATED_FAILED_CODES)。
"failed_codes": len(failed.failed_codes),
"failed_styles": len(failed.failed_styles),
"failed_pages": len(failed.failed_pages),
"failed_code_details": [
f"{item.get('style')} {item.get('code')}: {item.get('error')}"
for item in failed.failed_codes
],
}
@@ -1652,8 +1691,25 @@ def main() -> int:
return 1
summary = state.get("collector_summary") or {}
if summary.get("has_failures"):
print("[FAIL] ERP collector retained failed codes after retry")
return 1
# 2026-10-29: 页/款级失败仍然阻断;只把「个别编码在单平台重试后仍超时」
# 降级为警告。这类编码(如 10244 在抖音查不到数据导致 allsku iframe 不渲染)
# 本身贡献 0 销量,款级汇总值已正确,不该让整份日报当天不入库。
tolerated = (
0 < int(summary.get("failed_codes") or 0) <= ERP_TOLERATED_FAILED_CODES
and int(summary.get("failed_styles") or 0) == 0
and int(summary.get("failed_pages") or 0) == 0
)
if tolerated:
print(
"[WARN] ERP 采集完成,"
f"{summary.get('failed_codes')} 个编码单平台重试后仍失败,"
"已记 0 并保留失败清单,按部分成功继续日报:"
)
for detail in summary.get("failed_code_details") or []:
print(f" - {detail}", flush=True)
else:
print("[FAIL] ERP collector retained failed codes after retry")
return 1
if int(summary.get("reports") or 0) <= 0 or int(summary.get("valid_codes") or 0) <= 0:
print("[FAIL] ERP collector produced no valid report output")
return 1
@@ -1423,6 +1423,38 @@ def ensure_logged_in(page, account: str, password: str) -> None:
raise JDAdCostCollectionError("京东登录态未能进入商家首页")
_JZT_ONBOARDING_URL_MARKER = "/settled/"
_JZT_ONBOARDING_TEXT_MARKERS = ("确认联系人手机号", "京准通欢迎您的加入")
def _jzt_onboarding_present(page) -> bool:
"""京准通会把未完成开通引导的会话重定向到 /settled 确认页。"""
if _JZT_ONBOARDING_URL_MARKER in str(getattr(page, "url", "")):
return True
body = _page_text(page)
return any(marker in body for marker in _JZT_ONBOARDING_TEXT_MARKERS)
def ensure_jzt_ready(page) -> None:
"""Block early when 京准通 still needs the account owner's own confirmation.
``/settled/#/information/bonus`` (code=-3012) asks the logged-in merchant to
confirm a contact mobile number and accept 《京准通平台相关协议》 before the
custom-report routes render. The script must not accept that on the
operator's behalf, so surface it as an actionable blocker instead of the
generic "页面出现「下载报表」" timeout.
"""
if not _jzt_onboarding_present(page):
return
raise JDAdCostCollectionError(
"京东京准通需要人工完成「确认联系人手机号」开通引导"
"(/settled/#/information/bonus,code=-3012):"
"请在可见浏览器中由账号本人确认后重跑"
)
def _request_report(
page, report_type: ReportType, target_date: date, advertiser_pin: str
) -> None:
@@ -1570,7 +1602,13 @@ def run(args: argparse.Namespace) -> list[Path]:
pass
ensure_logged_in(page, login_account, PASSWORD)
safe_goto(page, DOWNLOAD_URL)
_wait_for_text(page, "下载报表", timeout_seconds=60)
# 京准通对未完成开通引导的会话会重定向到 /settled 确认页。先按正常
# 超时等待,超时后再判定是否为引导页拦截,避免刚跳转时的残留文案误报。
try:
_wait_for_text(page, "下载报表", timeout_seconds=60)
except JDAdCostCollectionError:
ensure_jzt_ready(page)
raise
_clear_download_center(page)
reports = {"non_full": "absent", "full": "absent"}
safe_goto(page, OVERVIEW_URL)
@@ -0,0 +1,9 @@
{
"schema_version": 1,
"note": "电商费用按父款式汇总:key 是平台报表里匹配到的子款式,value 是展示口径的父款式。只在对应平台的导入脚本里生效,不影响 dim_style 原始映射。",
"jd": {
"宙斯3": "宙斯",
"宙斯3 Air": "宙斯",
"宙斯Z Air": "宙斯"
}
}
@@ -32,7 +32,21 @@ except ImportError: # Direct script execution from the module root.
from runtime_paths import NORMALIZED_DATA_ROOT, STATE_ROOT, managed_data_path
CACHE_SCHEMA_VERSION = 6
CACHE_SCHEMA_VERSION = 8
# 系列级飞书地址:约定只维护在锚点行,子款误填也会被整系列聚合去重后统一下发。
# 顺序即写入目标表的优先级说明顺序,不含 bitable_sku_master(人工维护,不参与采集)。
SERIES_TARGET_FIELDS = (
"sales_bitable",
"main_image_bitable",
"creator_bitable",
"self_creator_bitable",
"weekly_note_analysis",
"persona_bitable",
"jd_persona_bitable",
"dy_persona_bitable",
"style_analysis_bitable",
)
# 表格字段名(按实际表格设置,支持别名)
FIELD_ALIASES = {
@@ -111,6 +125,10 @@ class StyleConfigLoader:
del base_token, table_id, view_id
self.identity = identity
self.records_provider = records_provider or runtime_workflow_configs
# 注入 records_provider 的调用方(单测 / 离线脚本)用的是假数据,绝不能
# 写进 STATE_ROOT 的生产快照:那份快照只在数据库读取失败时被当作真配置
# 使用,用它采集会按假款式去取数。因此注入 provider 时整套缓存读写都关掉。
self.cache_enabled = records_provider is None
project_root = Path(__file__).resolve().parent.parent
self.cache_path = (
@@ -229,6 +247,24 @@ class StyleConfigLoader:
parsed = self._parse_bitable_url(url)
return bool(parsed["base_token"] and parsed["table_id"])
def _has_creator_bitable(self, record: dict) -> bool:
"""合作达人表 Base/Table 同时存在。"""
url = self._parse_markdown_link(record.get("creator_bitable_url"))
parsed = self._parse_bitable_url(url)
return bool(parsed["base_token"] and parsed["table_id"])
def _has_self_creator_bitable(self, record: dict) -> bool:
"""自营合作达人表 Base/Table 同时存在。"""
url = self._parse_markdown_link(record.get("self_creator_bitable_url"))
parsed = self._parse_bitable_url(url)
return bool(parsed["base_token"] and parsed["table_id"])
def _has_weekly_note_analysis(self, record: dict) -> bool:
"""每周/每月笔记分析表 Base/Table 同时存在。"""
url = self._parse_markdown_link(record.get("weekly_note_analysis_url"))
parsed = self._parse_bitable_url(url)
return bool(parsed["base_token"] and parsed["table_id"])
def _has_daily_required_fields(self, record: dict) -> bool:
"""每日采集四条件:erp_codes、style、item_ids、sales_bitable。"""
style_name = str(record.get("style") or "").strip()
@@ -259,6 +295,9 @@ class StyleConfigLoader:
"erp_codes": row.get("erp_codes", []),
"sales_bitable_url": row.get("sales_bitable_url", ""),
"main_image_bitable_url": row.get("main_image_bitable_url", ""),
"creator_bitable_url": row.get("creator_bitable_url", ""),
"self_creator_bitable_url": row.get("self_creator_bitable_url", ""),
"weekly_note_analysis_url": row.get("weekly_note_analysis_url", ""),
"persona_bitable_url": row.get("tm_persona_bitable_url", ""),
"dy_persona_bitable_url": row.get("dy_persona_bitable_url", ""),
"jd_persona_bitable_url": row.get("jd_persona_bitable_url", ""),
@@ -268,6 +307,10 @@ class StyleConfigLoader:
"destinations": row.get("destinations", []),
"series_name": row.get("series_name", ""),
"is_series_anchor": bool(row.get("is_series_anchor", False)),
# 采集开关随记录下发;缺字段(旧 cmt_styles 回退)按启用处理
"collect_enabled": True
if row.get("collect_enabled") is None
else bool(row.get("collect_enabled")),
}
)
return records
@@ -312,6 +355,9 @@ class StyleConfigLoader:
"dy": {"item_ids": []},
"sales_bitable": {},
"main_image_bitable": {},
"creator_bitable": {},
"self_creator_bitable": {},
"weekly_note_analysis": {},
"persona_bitable": {},
"dy_persona_bitable": {},
"jd_persona_bitable": {},
@@ -319,6 +365,7 @@ class StyleConfigLoader:
"destinations": [],
"series_name": "",
"is_series_anchor": False,
"collect_enabled": True,
"series": {},
}
@@ -344,6 +391,11 @@ class StyleConfigLoader:
if record.get("is_series_anchor"):
style["is_series_anchor"] = True
# 采集开关(产品生命进程在 dim_style.collect_enabled 上按系列维护):
# merge 已把同一款展开成多条平台记录且取值一致;任一条显式 False 即关闭。
if record.get("collect_enabled") is False:
style["collect_enabled"] = False
destinations = record.get("destinations")
if isinstance(destinations, list) and destinations and not style["destinations"]:
style["destinations"] = [
@@ -364,41 +416,38 @@ class StyleConfigLoader:
f"{style['erp_codes']} vs {erp_codes}; 保留第一条天猫编码"
)
# 销量飞书表地址(款式级别,只要 URL 完整就收)
# 飞书地址:2026-10-29 起统一由 dim_style 系列锚点持有,并在
# workflow_config 里下发到该款的每条平台记录,所以任意分组都能
# 读到同样的值,不再像以前那样限定只能从天猫分组取。
if self._has_sales_bitable(record) and not style["sales_bitable"]:
sales_url = self._parse_markdown_link(record.get("sales_bitable_url"))
if sales_url:
style["sales_bitable"] = self._parse_bitable_url(sales_url)
# 主图表地址:只读天猫分组记录,只要求主图 URL 非空
if platform == "tm" and self._has_main_image_bitable(record):
if self._has_main_image_bitable(record) and not style["main_image_bitable"]:
main_url = self._parse_markdown_link(record.get("main_image_bitable_url"))
if main_url and not style["main_image_bitable"]:
if main_url:
style["main_image_bitable"] = self._parse_bitable_url(main_url)
style["main_image_bitable"]["platform"] = "天猫"
# 人群画像(天猫)表地址:只读天猫分组记录
if platform == "tm" and self._has_persona_bitable(record):
persona_url = self._parse_markdown_link(record.get("persona_bitable_url"))
if persona_url and not style["persona_bitable"]:
style["persona_bitable"] = self._parse_bitable_url(persona_url)
# 人群画像(抖音)表地址:款式级别,任意平台记录有值就收
if self._has_dy_persona_bitable(record):
dy_persona_url = self._parse_markdown_link(record.get("dy_persona_bitable_url"))
if dy_persona_url and not style["dy_persona_bitable"]:
style["dy_persona_bitable"] = self._parse_bitable_url(dy_persona_url)
# 人群画像(京东)表地址:款式级别,任意平台记录有值就收
if self._has_jd_persona_bitable(record):
jd_persona_url = self._parse_markdown_link(record.get("jd_persona_bitable_url"))
if jd_persona_url and not style["jd_persona_bitable"]:
style["jd_persona_bitable"] = self._parse_bitable_url(jd_persona_url)
if self._has_style_analysis_bitable(record) and not style["style_analysis_bitable"]:
analysis_url = self._parse_markdown_link(record.get("style_analysis_bitable_url"))
if analysis_url:
style["style_analysis_bitable"] = self._parse_bitable_url(analysis_url)
for key, record_key, checker in (
("creator_bitable", "creator_bitable_url", self._has_creator_bitable),
(
"self_creator_bitable",
"self_creator_bitable_url",
self._has_self_creator_bitable,
),
("weekly_note_analysis", "weekly_note_analysis_url", self._has_weekly_note_analysis),
("persona_bitable", "persona_bitable_url", self._has_persona_bitable),
("dy_persona_bitable", "dy_persona_bitable_url", self._has_dy_persona_bitable),
("jd_persona_bitable", "jd_persona_bitable_url", self._has_jd_persona_bitable),
("style_analysis_bitable", "style_analysis_bitable_url", self._has_style_analysis_bitable),
):
if style[key] or not checker(record):
continue
url = self._parse_markdown_link(record.get(record_key))
if url:
style[key] = self._parse_bitable_url(url)
# 非天猫分组的 ERP 编码不会参与聚合;只在冲突或天猫缺失时提示,
# 避免重复填写相同编码造成无意义日志。
@@ -451,12 +500,23 @@ class StyleConfigLoader:
@staticmethod
def _apply_series_sharing(styles: dict[str, dict]) -> None:
"""把同系列的平台编码归属与销量表摊到每个子款。
"""把整条系列的平台商品编码、ERP 编码与飞书地址聚合去重到锚点。
产品生命进程的录入约定:ERP 款式编码逐款必填,平台商品编码和
销量飞书表整系列只需要主款(锚点)填一次。这里只做「归属」标注
和销量表继承,不复制平台商品编码——同一个商品 ID 若被同系列多款
分别采集,会把同一份访客/加购重复写进每日指标。
录入约定是「锚点建档、子款留空」,但子款误填无法从表结构上禁止,所以
这里不依赖人工守规矩,统一做聚合兜底:
- 平台商品编码:整系列并集去重后挂锚点,由锚点一个采集单元去抓;
子款保留自己的原始编码(series.platform_ids 里可见)供诊断,采集口
按归属判定让出,避免同一商品 ID 被采两次、访客与加购重复入账。
- ERP 编码:整系列并集去重后挂锚点(series.erp_owner),子款保留自己
的原始编码(own_erp_codes)供按款查询。**子款自身 collect_enabled=false
时不并入**:锚点是系列总开关,子款开关只决定自己的编码是否参与聚合。
- 飞书地址:逐字段聚合并去重,锚点优先、其余按款名,取到的那套表下发
全系列。一个表位只能落一张表,出现多个不同表时告警并保留优先级最高
的一条。
聚合的目标是「子款误填也不丢、不重复入账、同系列只有一套表」,而不是
让子款变成独立采集单元。
"""
by_series: dict[str, list[str]] = {}
@@ -468,26 +528,122 @@ class StyleConfigLoader:
for series, members in by_series.items():
# 锚点优先,其余按名称稳定排序,保证每次运行归属一致
members.sort(key=lambda n: (not styles[n].get("is_series_anchor"), n))
anchor = members[0]
erp_owner = anchor
multi = len(members) > 1
platform_owner: dict[str, str] = {}
platform_ids: dict[str, dict[str, list[str]]] = {}
merged_platform: dict[str, list[str]] = {}
for platform, ids_key in PRODUCT_ID_FIELDS.items():
holders: dict[str, list[str]] = {}
merged: list[str] = []
for member in members:
if styles[member].get(platform, {}).get(ids_key):
platform_owner[platform] = member
break
sales_owner = next(
(m for m in members if (styles[m].get("sales_bitable") or {}).get("base_token")),
"",
)
ids = [
str(item).strip()
for item in styles[member].get(platform, {}).get(ids_key) or ()
if str(item).strip()
]
if not ids:
continue
holders[member] = ids
platform_owner.setdefault(platform, member)
for item in ids:
if item not in merged:
merged.append(item)
if not holders:
continue
merged_platform[platform] = merged
if multi:
# 子款写的编码也并入锚点:整系列先去重聚合,锚点作为唯一
# 采集单元持有全部商品 ID。holders 必须重建以保证锚点排在
# 第一位——_platform_collection_owner 依赖插入顺序判断
# 「谁的编码覆盖了谁」。
ordered_holders: dict[str, list[str]] = {anchor: list(merged)}
for member in members:
if member != anchor and member in holders:
ordered_holders[member] = holders[member]
holders = ordered_holders
platform_ids[platform] = holders
merged_erp: list[str] = []
erp_excluded: list[str] = []
for member in members:
# 锚点是系列总开关(见下面 series_collect);子款自身的 collect_enabled
# 决定它的 ERP 编码是否并入系列。子款关掉采集时,它的编码既不由自己
# 去聚水潭取数,也不允许锚点代采——直接排除在聚合之外。
if member != anchor and not bool(
styles[member].get("collect_enabled", True)
):
erp_excluded.append(member)
continue
for code in styles[member].get("erp_codes") or ():
if code not in merged_erp:
merged_erp.append(code)
if multi and erp_excluded and bool(styles[anchor].get("collect_enabled", True)):
print(
f"[INFO] 系列「{series}」子款 {', '.join(erp_excluded)} "
f"已关闭采集,ERP 编码不纳入「{erp_owner}」聚合"
)
# 系列级采集开关:以锚点为准(members[0] 已按「锚点优先、其次款名」
# 排序,无锚点时即款名第一个)。产品生命进程只改锚点那一行即可控制
# 整条系列,其余子款跟随。
series_collect = bool(styles[anchor].get("collect_enabled", True))
# 系列级飞书地址:整系列逐字段聚合去重,锚点优先、其余按款名。子款
# 误填的地址同样参与聚合而不会被丢弃;一套表下发全系列,写表口径
# 统一。同字段出现多个不同表时保留优先级最高的一条并告警。
series_targets: dict[str, dict] = {}
for field in SERIES_TARGET_FIELDS:
seen: set[tuple] = set()
candidates: list[dict] = []
for member in members:
target = styles[member].get(field) or {}
if not target.get("base_token"):
continue
key = (
target.get("base_token"),
target.get("table_id"),
target.get("view_id"),
)
if key in seen:
continue
seen.add(key)
candidates.append(dict(target))
if not candidates:
continue
if len(candidates) > 1:
print(
f"[WARN] 系列「{series}」{field} 有 {len(candidates)} 个不同表,"
f"按「{anchor}」下发并忽略其余"
)
series_targets[field] = candidates[0]
for member in members:
style = styles[member]
style["series"] = {
"name": series,
"is_anchor": bool(style.get("is_series_anchor")),
"collect_enabled": series_collect,
"platform_owner": dict(platform_owner),
"sales_owner": sales_owner,
}
if not style["sales_bitable"] and sales_owner:
style["sales_bitable"] = dict(styles[sales_owner]["sales_bitable"])
if multi:
# 单款即可自成系列,无需归属标注;多款才需要指明 ERP 由谁承接
style["series"]["erp_owner"] = erp_owner
style["series"]["own_erp_codes"] = list(style.get("erp_codes") or ())
# 自身 collect_enabled=false 而被排除出 ERP 聚合的子款,供诊断
style["series"]["erp_excluded"] = list(erp_excluded)
style["series"]["platform_ids"] = {
platform: {holder: list(ids) for holder, ids in holders.items()}
for platform, holders in platform_ids.items()
}
for field, target in series_targets.items():
style[field] = dict(target)
if multi:
# 锚点持有整条系列的平台商品编码与 ERP 编码
for platform, ids_key in PRODUCT_ID_FIELDS.items():
merged = merged_platform.get(platform)
if merged:
styles[anchor][platform][ids_key] = list(merged)
if merged_erp:
styles[erp_owner]["erp_codes"] = merged_erp
# ------------------------------------------------------------------
# 缓存与回退
@@ -533,13 +689,14 @@ class StyleConfigLoader:
try:
records = self._load_records_from_database()
payload = self._aggregate(records)
self._write_cache(payload)
if self.cache_enabled:
self._write_cache(payload)
if sync_fallback:
self._write_style_map_fallback(payload)
return payload
except Exception as exc:
print(f"[WARN] 云端动态配置读取失败: {exc},尝试数据库快照缓存")
cached = self._read_cache()
cached = self._read_cache() if self.cache_enabled else None
if cached and cached.get("source") == "database":
return cached
raise StyleConfigError("云端动态配置不可用且没有数据库快照缓存") from exc
@@ -768,50 +925,89 @@ class StyleConfigLoader:
# ------------------------------------------------------------------
# 启动期「待采 / 跳过」门 — 显式返回带 reason 的跳过清单
# ------------------------------------------------------------------
def _cfg_has_sales_bitable(self, cfg: dict) -> bool:
return bool(cfg.get("sales_bitable", {}).get("base_token")) and bool(
cfg.get("sales_bitable", {}).get("table_id")
)
@staticmethod
def _has_any_product_id(cfg: dict) -> bool:
return any(
bool(cfg.get(platform, {}).get(ids_key))
for platform, ids_key in PRODUCT_ID_FIELDS.items()
)
def _has_complete_identity(self, cfg: dict, platform: str | None = None) -> bool:
"""商品采集统一门:ERP 款式编码 + 至少一个平台商品编码。"""
return has_complete_style_identity(cfg, platform)
def _identity_missing(self, cfg: dict) -> list[str]:
missing: list[str] = []
if not cfg.get("erp_codes"):
missing.append("ERP款式编码")
if not self._has_any_product_id(cfg):
missing.append("平台商品编码")
return missing
def _erp_owner_for_collection(self, name: str, cfg: dict, styles: dict) -> str:
"""本款是否需要把 ERP 采集让给同系列锚点。
返回空串表示本款自己采;返回款名表示编码已由该款归集,本款不重复采。
锚点自身被关闭采集时不再承接(返回空串)——但系列级总开关此时已让整条
系列停采,这里只是避免把编码归属标注到一个根本不采的款上。
"""
series = cfg.get("series") or {}
owner = str(series.get("erp_owner") or "").strip()
if not owner or owner == name:
return ""
if not self._is_collect_enabled(styles.get(owner) or {}):
return ""
return owner
# ------------------------------------------------------------------
# 采集开关 (dim_style.collect_enabled)
# ------------------------------------------------------------------
@staticmethod
def _is_collect_enabled(cfg: dict) -> bool:
"""本款是否整体纳入采集(系列级总开关)。
``dim_style.collect_enabled`` 是两层语义:
- **系列级总开关**:由锚点那一行决定,``_apply_series_sharing`` 已把锚点的
值摊到 ``series.collect_enabled``,所以同系列任意一款得到相同判定。
锚点 false → 整条系列一款都不采。产品侧只需改锚点那一行。
- **子款级 ERP 参与权**:锚点为 true 时,子款自身的 ``collect_enabled``
只决定「它的 ERP 款式编码是否并入系列聚合」(见 ``_apply_series_sharing``
的 ``erp_excluded``);子款 false 不参与聚合,但不会让其他子款停采。
没有 series_name 的款直接用自身取值;字段缺失时默认采集。
"""
series = cfg.get("series") or {}
if "collect_enabled" in series:
return bool(series["collect_enabled"])
return bool(cfg.get("collect_enabled", True))
def get_daily_styles(self) -> dict:
"""每日商品采集门:ERP 编码、销量目标和平台商品编码必须完整。
"""每日商品采集门:ERP 编码与平台商品编码必须完整。
销量飞书表地址只决定结果能否写表,不再作为采集前置条件——缺表地址
的款式照常采集并落库,写表环节自己按地址缺失跳过。
返回 {tm/jd/dy: {collectable: [...], skipped: [{style, reason}]}}。
incomplete style 永远不会进入 collectable,并记录具体缺失字段。
采集开关取 ``dim_style.collect_enabled``:锚点是系列总开关(false 则整系列
不采),子款自身的开关只决定它的 ERP 编码是否参与系列聚合;缺字段默认采集。
ERP 与平台编码按两个维度分开判定:
- ERP:整条系列聚合到锚点(series.erp_owner)由一个单元去聚水潭取数;
子款即使让出 ERP,只要自己持有某平台的商品 ID,仍会继续采该平台。
- 平台:整系列只要有一款填了就算齐,谁填谁采;本款没填时指向承担款,
本款编码被更高优先级的款式完整覆盖时让出,避免重复入账。
"""
payload = self.load()
styles = payload["styles"]
result = {p: {"collectable": [], "skipped": []} for p in ("tm", "jd", "dy")}
for name, cfg in payload["styles"].items():
# 平台编码在下面按平台逐项判定,这里不重复报「平台商品编码」
identity_missing = [
item for item in self._identity_missing(cfg) if item != "平台商品编码"
]
for name, cfg in styles.items():
if not self._is_collect_enabled(cfg):
for plat in ("tm", "jd", "dy"):
result[plat]["skipped"].append(
{"style": name, "reason": "已标记不采集"}
)
continue
# 自有编码,或同系列锚点已在采(本款不必重复去聚水潭)
has_erp = bool(cfg.get("erp_codes")) or bool(
self._erp_owner_for_collection(name, cfg, styles)
)
for plat in ("tm", "jd", "dy"):
ids_key = "spus" if plat == "jd" else "item_ids"
missing = list(identity_missing)
if not self._cfg_has_sales_bitable(cfg):
missing.append("销量目标地址")
if not cfg.get(plat, {}).get(ids_key):
missing.append(self._platform_missing_label(cfg, plat))
owner = self._platform_collection_owner(name, cfg, plat, styles)
missing: list[str] = []
if not has_erp:
missing.append("ERP款式编码")
if owner != name:
if owner:
missing.append(f"{plat}商品编码(同系列「{owner}」已在采)")
else:
missing.append(f"{plat}商品编码")
if missing:
result[plat]["skipped"].append(
{"style": name, "reason": "缺 " + "/".join(dict.fromkeys(missing))}
@@ -820,43 +1016,82 @@ class StyleConfigLoader:
result[plat]["collectable"].append(name)
return result
def _platform_missing_label(self, cfg: dict, platform: str) -> str:
"""平台商品编码缺失的提示文案。
def _platform_collection_owner(
self, name: str, cfg: dict, platform: str, styles: dict
) -> str:
"""本款是否承担该平台的流量采集;空串表示整条系列都没填编码。
同系列其它款已填编码时,本款不采平台流量是预期行为(平台数据是
商品/系列级,重复采集会把同一份访客数写进多个款),因此文案指向
承担款而不是报了「缺配置」。
平台商品编码整系列只要有一款填了就算齐:谁填谁采。本款的编码若被
同系列优先级更高的款式完整覆盖(锚点常见做法是把全系列商品 ID 都
挂在锚点上),本款让出,避免同一个商品 ID 被采两次、访客与加购
重复入账。本款的编码与别人的编码互不包含时各自采各自的商品。
"""
owner = (cfg.get("series") or {}).get("platform_owner", {}).get(platform, "")
if owner and owner != cfg.get("name"):
return f"{platform}商品编码(同系列「{owner}」已在采,本款只采 ERP 销量)"
return f"{platform}商品编码"
ids_key = PRODUCT_ID_FIELDS[platform]
own_ids = {
str(item).strip()
for item in cfg.get(platform, {}).get(ids_key) or ()
if str(item).strip()
}
series = cfg.get("series") or {}
holders: dict[str, list[str]] = (
(series.get("platform_ids") or {}).get(platform) or {}
)
if own_ids:
# holders 的插入顺序即系列内的优先级(锚点优先,其次按款名)
for member in holders:
if member == name:
return name
if not self._is_collect_enabled(styles.get(member) or {}):
continue
covered = {
str(item).strip() for item in holders.get(member) or () if str(item).strip()
}
if own_ids <= covered:
return member
return name
# 本款没填:由系列内已填且启用的款式承担(锚点优先)
for member in holders:
if not self._is_collect_enabled(styles.get(member) or {}):
continue
return member
return ""
def get_erp_daily_styles(self) -> dict:
"""ERP daily 门:只要求 ERP 款式编码(有销量表写入需求时再校验表地址)。
"""ERP daily 门:只要求 ERP 款式编码。
销量飞书表地址只决定结果能否写表,不再作为采集前置条件——缺表地址
的款式照常去聚水潭取数并落库,写表环节自己按地址缺失跳过。
返回 {collectable, skipped, brand_map: {style: brand}}。
ERP 销量按款式编码逐款查,平台商品编码与销量表都是系列级配置,
由同系列主款承担,子款不需要重复填写。
ERP 销量按款式编码逐款查;同系列多款时编码已由锚点聚合并集
(series.erp_owner),子款让出采集,避免同一编码被采两次。
brand_map 给 collect_erp_yesterday_metrics.py 按品牌选店铺
(光影行星旗下店铺 vs ozko旗舰店)。
采集开关取 dim_style.collect_enabled:锚点为 false 时整系列不采;锚点为
true 时,自身 collect_enabled=false 的子款其 ERP 编码不并入锚点聚合。
缺字段默认采集。
"""
payload = self.load()
styles = payload["styles"]
collectable: list[str] = []
skipped: list[dict] = []
brand_map: dict[str, str] = {}
for name, cfg in payload["styles"].items():
missing: list[str] = []
for name, cfg in styles.items():
if not self._is_collect_enabled(cfg):
skipped.append({"style": name, "reason": "已标记不采集"})
continue
erp_owner = self._erp_owner_for_collection(name, cfg, styles)
if erp_owner:
skipped.append(
{"style": name, "reason": f"ERP 已由同系列「{erp_owner}」归集采集"}
)
continue
if not cfg.get("erp_codes"):
missing.append("ERP款式编码")
if not self._cfg_has_sales_bitable(cfg):
missing.append("销量目标地址")
if not missing:
collectable.append(name)
brand_map[name] = cfg.get("brand") or "光影行星"
else:
skipped.append({"style": name, "reason": "缺 " + "/".join(dict.fromkeys(missing))})
skipped.append({"style": name, "reason": "缺 ERP款式编码"})
continue
collectable.append(name)
brand_map[name] = cfg.get("brand") or "光影行星"
return {"collectable": collectable, "skipped": skipped, "brand_map": brand_map}
def get_jd_self_styles(self) -> list[str]:
@@ -868,14 +1103,18 @@ class StyleConfigLoader:
]
def get_main_image_styles(self) -> list[str]:
"""天猫主图门:完整身份、天猫商品编码和主图目标地址同时存在。"""
"""天猫主图采集目标:身份齐全(ERP 款式编码 + 天猫商品编码)即可。
主图飞书表地址只决定结果能否写表,不再作为采集前置条件——缺地址的
款式照常抓图并落库,写表环节按地址缺失跳过(见 get_bitable_main_image_map),
与商品经营日报「缺销量表不挡采集」同一口径。
"""
payload = self.load()
result: list[str] = []
for name, cfg in payload["styles"].items():
main = cfg.get("main_image_bitable") or {}
if self._has_complete_identity(cfg, "tm") and main.get("base_token") and main.get("table_id"):
result.append(name)
return result
return [
name
for name, cfg in payload["styles"].items()
if self._has_complete_identity(cfg, "tm")
]
# ------------------------------------------------------------------------------
@@ -4,29 +4,63 @@
-- 设计原则: schema 与 data/ 目录下的 JSON 产物 1:1 映射;upsert 幂等
-- ============================================================================
-- 款式维度表:维护每款的 (各平台商品 ID + 京东自营 SKU + ERP 编码 + 飞书表地址)
-- 身份主来源:产品生命进程 cmt_styles.platform_product_ids / erp_style_codes,
-- 由 workflow_config.runtime_records() 统一投影;运行时不再从各采集器、JSON 产物反向拼接身份。
-- 款式维度表:维护每款的 (各平台商品 ID + 京东自营 SKU + ERP 款式编码 + 飞书表地址)
-- 身份主来源:本表自身(2026-10-27 起由产品生命进程 cmt_styles 切换过来)。
-- * ERP 编码用 erp_style_codes (JSONB 数组,可多编码),旧列 erp_codes 已废弃,勿再写入/读取。
-- * 平台商品 ID 用 tm_spus / jd_spus / dy_spus / jd_self_skus (均为 JSONB 数组)。
-- 读取入口:style_config_loader / db.sync_daily_collect_scope 直接查本表;
-- cmt_styles 仅在 dim_style 缺同名款时作为回退。
-- bitable_weekly_* 仍由 bitable_style_map.json 派生;bitable_sku_master_* 仍由飞书 SKU 主表人工维护。
CREATE TABLE IF NOT EXISTS dim_style (
style_name TEXT PRIMARY KEY,
jd_spus TEXT[], -- 京东 POP 商品 SPU
dy_spus TEXT[], -- 抖音商品 SPU
tm_spus TEXT[], -- 天猫商品 SPU
jd_self_skus TEXT[], -- 京东自营供应链 SKU 集合
erp_codes TEXT[], -- 聚水潭 ERP 编码
bitable_weekly_url TEXT, -- 款式周报表 URL (bitable_style_map 派生)
bitable_weekly_base TEXT, -- 款式周报表 base_token
bitable_weekly_table TEXT, -- 款式周报表 table_id
bitable_sku_master_url TEXT, -- 飞书 SKU 主表 URL (人工维护)
bitable_sku_master_base TEXT, -- 飞书 SKU 主表 base_token (人工维护)
bitable_sku_master_table TEXT, -- 飞书 SKU 主表 table_id (人工维护)
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
style_name TEXT NOT NULL UNIQUE,
brand TEXT NOT NULL DEFAULT '',
category TEXT,
launch_date DATE,
notes TEXT,
jd_self_skus JSONB NOT NULL DEFAULT '[]'::jsonb, -- 京东自营供应链 SKU 集合
jd_spus JSONB NOT NULL DEFAULT '[]'::jsonb, -- 京东 POP 商品 SPU
dy_spus JSONB NOT NULL DEFAULT '[]'::jsonb, -- 抖音商品 SPU
tm_spus JSONB NOT NULL DEFAULT '[]'::jsonb, -- 天猫商品 SPU
erp_style_codes JSONB NOT NULL DEFAULT '[]'::jsonb, -- 聚水潭 ERP 款式编码(正源)
erp_codes BIGINT, -- 已废弃:历史单值列,仅供回溯
product_tags JSONB NOT NULL DEFAULT '[]'::jsonb, -- 产品标签
lifecycle_stage VARCHAR, -- 生命周期阶段
material TEXT,
selling_points TEXT,
features TEXT,
compatible_devices TEXT,
image_url TEXT NOT NULL DEFAULT '',
big_nfc_count INTEGER NOT NULL DEFAULT 1,
small_nfc_count INTEGER NOT NULL DEFAULT 1,
jushuitan_category VARCHAR, -- 聚水潭分类
series_name VARCHAR, -- 所属系列
is_series_anchor BOOLEAN NOT NULL DEFAULT FALSE, -- 系列锚点款
series_sort_order INTEGER,
collect_enabled BOOLEAN NOT NULL DEFAULT TRUE, -- 商品日报采集开关(系列级:以锚点为准)
-- 飞书多维表格地址(9 类,全部以「系列锚点行」为准,子款留空即可)
bitable_weekly_url TEXT, -- 销量表 URL (bitable_style_map 派生)
bitable_weekly_base TEXT, -- 销量表 base_token
bitable_weekly_table TEXT, -- 销量表 table_id
bitable_main_image_url TEXT, -- 主图表 URL
bitable_creator_url TEXT, -- 合作达人表 URL
bitable_self_creator_url TEXT, -- 自营合作达人表 URL
bitable_weekly_note_url TEXT, -- 每周/每月笔记分析 URL
bitable_style_analysis_url TEXT, -- 平台单品分析 URL
bitable_persona_tm_url TEXT, -- 人群画像(天猫)URL
bitable_persona_jd_url TEXT, -- 人群画像(京东)URL
bitable_persona_dy_url TEXT, -- 人群画像(抖音)URL
bitable_sku_master_url TEXT, -- 飞书 SKU 主表 URL (人工维护)
bitable_sku_master_base TEXT, -- 飞书 SKU 主表 base_token (人工维护)
bitable_sku_master_table TEXT, -- 飞书 SKU 主表 table_id (人工维护)
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
-- 商品日报采集开关已上移到 dim_style.collect_enabled(系列级,以锚点为准)。
-- 旧表 product_daily_collect_scope 已于 2026-10-29 迁移并删除,
-- 备份在 var/tmp/collect_scope_before_drop_20261029.json。
DROP TABLE IF EXISTS product_daily_collect_scope;
-- 每日款式级指标 (来自 aggregate_daily_final.py → data/<date>/<款>/final_summary.json)
CREATE TABLE IF NOT EXISTS daily_style_metrics (
metric_date DATE NOT NULL,
@@ -206,13 +240,35 @@ CREATE INDEX IF NOT EXISTS plat_xlsx_item_idx
ON fact_platform_xlsx (platform, item_id);
-- ── 兼容旧版 dim_style:init_schema 在已有库上跑时给老表加列 ─────────────
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS jd_spus TEXT[];
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS dy_spus TEXT[];
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS tm_spus TEXT[];
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS erp_codes TEXT[];
-- 平台 ID / ERP 编码统一为 JSONB 数组;ERP 正源是 erp_style_codes,erp_codes 已废弃。
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS jd_spus JSONB NOT NULL DEFAULT '[]'::jsonb;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS dy_spus JSONB NOT NULL DEFAULT '[]'::jsonb;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS tm_spus JSONB NOT NULL DEFAULT '[]'::jsonb;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS erp_style_codes JSONB NOT NULL DEFAULT '[]'::jsonb;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS brand TEXT NOT NULL DEFAULT '';
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS lifecycle_stage VARCHAR;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS product_tags JSONB NOT NULL DEFAULT '[]'::jsonb;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS jushuitan_category VARCHAR;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS series_name VARCHAR;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS is_series_anchor BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS series_sort_order INTEGER;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS collect_enabled BOOLEAN NOT NULL DEFAULT TRUE;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS image_url TEXT NOT NULL DEFAULT '';
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS big_nfc_count INTEGER NOT NULL DEFAULT 1;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS small_nfc_count INTEGER NOT NULL DEFAULT 1;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_weekly_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_weekly_base TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_weekly_table TEXT;
-- 飞书地址统一上收到 dim_style(2026-10-29):锚点行唯一来源,子款留空自动继承。
-- workflow_dynamic_styles / _style_platforms 的同名字段已废弃,代码不再读取。
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_main_image_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_creator_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_self_creator_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_weekly_note_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_style_analysis_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_persona_tm_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_persona_jd_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_persona_dy_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_url TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_base TEXT;
ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_table TEXT;
@@ -223,13 +279,14 @@ ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_table TEXT;
-- ============================================================================
-- dim_style
COMMENT ON TABLE dim_style IS '款式维度表 — 维护每款 (各平台商品 ID + 京东自营 SKU + ERP 编码 + 飞书表地址 + 品类/上架日期/备注)。身份投影自产品生命进程,周报地址与 SKU 主表仍按各自来源维护。';
COMMENT ON COLUMN dim_style.style_name IS '款式名 (主键),与 PRODUCT_STYLE_GROUPS 的 key 对齐';
COMMENT ON COLUMN dim_style.jd_spus IS '京东 POP 商品 SPU 列表';
COMMENT ON COLUMN dim_style.dy_spus IS '抖音商品 SPU 列表';
COMMENT ON COLUMN dim_style.tm_spus IS '天猫商品 SPU 列表';
COMMENT ON COLUMN dim_style.jd_self_skus IS '京东自营供应链 SKU 集合';
COMMENT ON COLUMN dim_style.erp_codes IS '聚水潭 ERP 编码列表';
COMMENT ON TABLE dim_style IS '款式维度表 — 款式主档与身份正源:各平台商品 ID + 京东自营 SKU + ERP 款式编码 + 飞书表地址 + 品类/上架日期/备注。';
COMMENT ON COLUMN dim_style.style_name IS '款式名 (业务唯一键),与产品生命进程款名对齐';
COMMENT ON COLUMN dim_style.jd_spus IS '京东 POP 商品 SPU 列表 (JSONB 数组)';
COMMENT ON COLUMN dim_style.dy_spus IS '抖音商品 SPU 列表 (JSONB 数组)';
COMMENT ON COLUMN dim_style.tm_spus IS '天猫商品 SPU 列表 (JSONB 数组)';
COMMENT ON COLUMN dim_style.jd_self_skus IS '京东自营供应链 SKU 集合 (JSONB 数组)';
COMMENT ON COLUMN dim_style.erp_style_codes IS '聚水潭 ERP 款式编码列表 (JSONB 数组,ERP 身份正源)';
COMMENT ON COLUMN dim_style.erp_codes IS '已废弃:历史单值 ERP 编码列,新代码禁止写入/读取,请改用 erp_style_codes';
COMMENT ON COLUMN dim_style.bitable_weekly_url IS '款式周报表 URL (从 bitable_style_map.json 派生)';
COMMENT ON COLUMN dim_style.bitable_weekly_base IS '款式周报表 base_token';
COMMENT ON COLUMN dim_style.bitable_weekly_table IS '款式周报表 table_id';
@@ -249,7 +249,7 @@ def _print_preflight() -> None:
return
collectable = daily["dy"]["collectable"]
skipped = daily["dy"]["skipped"]
print(f"[DY] 飞书表格 待采 {len(collectable)} 款, 跳过 {len(skipped)} 款 (款式级缺 erp/sales,或 DY 缺商品链接ID)")
print(f"[DY] 飞书表格 待采 {len(collectable)} 款, 跳过 {len(skipped)} 款 (款式级缺 erp 或已由同系列归集,或 DY 缺商品链接ID)")
for s in skipped:
print(f" - {s['style']}: {s['reason']}")
@@ -402,11 +402,106 @@ def _unique(values: Iterable[str]) -> list[str]:
return list(dict.fromkeys(value for value in values if value))
ROLLUP_CONFIG_PATH = Path(__file__).resolve().parent / "config" / "ad_cost_rollup.json"
_rollup_cache: dict[str, dict[str, str]] | None = None
def load_rollup_map(platform: str = "jd") -> dict[str, str]:
"""读取父款式汇总口径(子款式 -> 父款式),缺失或非法时返回空映射。
``宙斯3`` / ``宙斯Z Air`` 这类子款式在京准通报表里名称更长,会先命中
名称匹配;父款式 ``宙斯`` 的电商费比需要把整个商品族的消耗算进来,
因此汇总口径放在 ``config/ad_cost_rollup.json`` 里显式声明。
"""
global _rollup_cache
if _rollup_cache is None:
try:
payload = json.loads(ROLLUP_CONFIG_PATH.read_text(encoding="utf-8"))
except (OSError, ValueError):
payload = {}
_rollup_cache = {
key: {str(child): str(parent) for child, parent in value.items()}
for key, value in payload.items()
if isinstance(value, dict)
}
return dict(_rollup_cache.get(platform, {}))
def _apply_rollup(result: dict[str, Any], rollup: Mapping[str, str]) -> dict[str, Any]:
style_name = result.get("style_name")
parent = rollup.get(style_name) if style_name else None
if not parent or parent == style_name:
return result
return {
**result,
"style_name": parent,
"reason": f"{result.get('reason', '')} -> 按父款式汇总「{parent}」",
}
def _spu_owner_map(
style_catalog: Sequence[Mapping[str, Any]],
) -> dict[str, str]:
"""显式 jd_spus 归属:SPU 挂在哪个款式上,就优先算哪个款式。
``dim_style.jd_spus`` 是人工维护的商品→款式权威映射。SKU 名称匹配只是
fallback:同一商品族的子款式(如「宙斯3」「宙斯Z Air」)名称更长、更容易
命中最长匹配,但父款式已经在 jd_spus 里声明了这些 SPU 时,消耗必须归到
jd_spus 声明的款式,否则父款式的广告费比恒为空。
"""
owners: dict[str, str] = {}
conflicts: set[str] = set()
for item in style_catalog:
style_name = _text(item.get("style_name"))
if not style_name:
continue
for spu in _spu_values(item.get("jd_spus")):
existing = owners.get(spu)
if existing is not None and existing != style_name:
conflicts.add(spu)
continue
owners[spu] = style_name
for spu in conflicts:
owners.pop(spu, None)
return owners
def _apply_spu_owner(
result: dict[str, Any], spu_owner: Mapping[str, str]
) -> dict[str, Any]:
spu_id = _text(result.get("spu_id"))
if not spu_id:
return result
owner = spu_owner.get(spu_id)
if not owner or owner == result.get("style_name"):
return result
return {
**result,
"style_name": owner,
"reason": f"{result.get('reason', '')} -> dim_style.jd_spus 归属「{owner}」",
}
def resolve_sku_match(
sku_name: str,
style_catalog: Sequence[Mapping[str, Any]],
full_records: Sequence[Mapping[str, Any]],
spu_catalog: Sequence[Mapping[str, Any]] = (),
) -> dict[str, Any]:
"""Resolve one SKU to a style, then honour explicit ``dim_style.jd_spus``."""
result = _resolve_sku_match(sku_name, style_catalog, full_records, spu_catalog)
result = _apply_spu_owner(result, _spu_owner_map(style_catalog))
return _apply_rollup(result, load_rollup_map("jd"))
def _resolve_sku_match(
sku_name: str,
style_catalog: Sequence[Mapping[str, Any]],
full_records: Sequence[Mapping[str, Any]],
spu_catalog: Sequence[Mapping[str, Any]] = (),
) -> dict[str, Any]:
"""Return a conservative style/SPU match for one SKU name.
@@ -671,6 +766,15 @@ def apply_full_style_matches(
"full-site SPU has no dim_style.jd_spus or name match"
)
rollup = load_rollup_map("jd")
resolved_style = item.get("style_name")
parent_style = rollup.get(resolved_style) if resolved_style else None
if parent_style and parent_style != resolved_style:
item["style_name"] = parent_style
item["match_reason"] = (
f"{item['match_reason']} -> 按父款式汇总「{parent_style}」"
)
metrics = dict(item.get("metrics") or {})
metrics["match"] = {
"status": item.get("match_status", "direct"),
@@ -11,7 +11,6 @@ from runtime_paths import EXPORTS_DATA_ROOT, LOG_ROOT, RAW_DATA_ROOT
PROJECT_ROOT = Path(__file__).resolve().parent
DATA_ROOT = RAW_DATA_ROOT
STYLE_MAP_PATH = PROJECT_ROOT / "bitable_style_map.json"
def read_json(path):
@@ -283,10 +282,15 @@ def main(argv: list[str] | None = None) -> int:
try:
style_map = get_bitable_style_map()
if not style_map:
raise RuntimeError("飞书动态配置为空")
raise RuntimeError("dim_style 未返回任何带销量表地址的款式")
except Exception as exc:
print(f"[WARN] 动态读取销量表配置失败,使用本地文件回退: {exc}")
style_map = read_json(STYLE_MAP_PATH)
# 飞书地址唯一来源是 dim_style:读不到就失败退出,不再回退本地 JSON,
# 否则库与本地文件不一致时会静默写到错误的表。
print(
f"[FAIL] 销量表配置读取失败,拒绝回退本地 JSON: {exc}",
file=sys.stderr,
)
return 1
if args.source == "db":
print(f"[INFO] source=db: 查 weekly_style_metrics week_end={args.date}")
records = load_records(args.date, args.record_file, source=args.source)
@@ -72,6 +72,59 @@ OUTPUT_DIR = RAW_DATA_ROOT / "主图信息"
DEBUG_DIR = TMP_ROOT / "debug" / "jd_main_image_snapshot"
DEBUG = os.getenv("JD_MAIN_IMAGE_DEBUG", "0") == "1"
# 京东商家后台门户(登录成功后应停留在这些主机上)。
JD_PORTAL_HOSTS = frozenset({"shop.jd.com", "wares-jdm.jd.com"})
# 京东风控/账号安全认证页:出现即代表登录态被判定异常,必须人工完成短信/人脸验证。
JD_SECURITY_CHALLENGE_HOSTS = frozenset(
{"aq.jd.com", "safe.jd.com", "verify.jd.com", "risk.jd.com"}
)
# 密码登录后等待后台就绪的上限(京东会先加载登录页再异步跳转)。
LOGIN_SETTLE_TIMEOUT_MS = 120_000
# 无头模式下登录页长时间不跳转即视为登录失败,避免白等到超时。
HEADLESS_LOGIN_STALL_SECONDS = 25.0
def _url_summary(url: str) -> str:
"""只保留协议/主机/路径。
京东认证页的查询串里带一次性 token(签名、trackId、returnUrl),
不能原样写进日志、运行记录和飞书告警。
"""
parsed = urlparse(str(url))
if not parsed.hostname:
return "<unknown>"
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
def _security_challenge_host(url: str) -> str:
"""返回命中的京东安全认证主机名,未命中返回空串。"""
hostname = (urlparse(str(url)).hostname or "").casefold()
return hostname if hostname in JD_SECURITY_CHALLENGE_HOSTS else ""
def _security_challenge_error(url: str) -> RuntimeError:
"""登录态被京东风控拦截时的统一可执行错误。"""
host = _security_challenge_host(url) or "京东安全页"
return RuntimeError(
f"京东触发安全认证({host}):登录态被判定需要人工验证,"
"账号密码登录无法自动完成短信/图形验证。请在能打开浏览器的机器上执行 "
"`uv run gyxx accounts login jd-shop --force` 完成验证,再重跑本步骤。"
f" 当前页面: {_url_summary(url)}"
)
def _is_authenticated_portal(url: str) -> bool:
"""当前 URL 是否已是可用的京东商家后台页面。"""
parsed = urlparse(str(url))
hostname = (parsed.hostname or "").casefold()
if hostname not in JD_PORTAL_HOSTS:
return False
return not _looks_like_login_url(url) and not _security_challenge_host(url)
# 直接使用新版商品明细页,避免旧版别名在批量采集过程中偶发 ERR_CONNECTION_CLOSED。
DETAIL_URL = "https://jdsz.jd.com/szweb/view/product/productDetail.html"
SEARCH_OVERVIEW_URL = "https://jdsz.jd.com/squidweb/view/flow/searchOverview.html"
@@ -240,10 +293,13 @@ def _wait_for_product_detail(page, timeout_ms: int = 30_000) -> None:
except Exception:
last_url = ""
if _security_challenge_host(last_url):
raise _security_challenge_error(last_url)
if _looks_like_login_url(last_url):
raise RuntimeError(
"京东商智登录态已失效:商品明细页跳转到登录页 "
f"{last_url};请先刷新 jd-shop 账号登录态后再运行"
f"{_url_summary(last_url)};请先刷新 jd-shop 账号登录态后再运行"
)
# A transient non-JD URL can be observed while the new tab settles;
@@ -261,14 +317,16 @@ def _wait_for_product_detail(page, timeout_ms: int = 30_000) -> None:
except Exception:
time.sleep(0.5)
if _security_challenge_host(last_url):
raise _security_challenge_error(last_url)
if _looks_like_login_url(last_url):
raise RuntimeError(
"京东商智登录态已失效:商品明细页跳转到登录页 "
f"{last_url};请先刷新 jd-shop 账号登录态后再运行"
f"{_url_summary(last_url)};请先刷新 jd-shop 账号登录态后再运行"
)
raise RuntimeError(
"京东商品明细页未加载完成,未找到商品ID搜索框;"
f"当前 URL: {last_url or '<unknown>'}"
f"当前 URL: {_url_summary(last_url)}"
)
@@ -287,10 +345,12 @@ def _wait_for_search_overview(page, timeout_ms: int = 30_000) -> None:
last_url = str(page.url)
except Exception:
last_url = ""
if _security_challenge_host(last_url):
raise _security_challenge_error(last_url)
if _looks_like_login_url(last_url):
raise RuntimeError(
"京东商智登录态已失效:搜索数据概览跳转到登录页 "
f"{last_url};请先刷新 jd-shop 账号登录态后再运行"
f"{_url_summary(last_url)};请先刷新 jd-shop 账号登录态后再运行"
)
try:
if page.locator("input.jmt-core-input").first.is_visible(timeout=800):
@@ -303,7 +363,7 @@ def _wait_for_search_overview(page, timeout_ms: int = 30_000) -> None:
time.sleep(0.5)
raise RuntimeError(
"京东搜索数据概览页未加载完成,未找到 SKU 筛选输入框;"
f"当前 URL: {last_url or '<unknown>'}"
f"当前 URL: {_url_summary(last_url)}"
)
@@ -698,6 +758,30 @@ def _ensure_page(context, page):
return context.new_page()
def _page_reports_no_data(page) -> bool:
"""商品明细表是否明确渲染了「暂无数据」空态。
商智商品明细只展示窗口内有曝光/成交的商品;某款当日零曝光时查询结果
为空,这是正常业务空值。老逻辑把它当成「解析失败」,一款无数据就会让
整周工作流失败、其余款式也写不进飞书和 PG(2026-10-18 即此故障)。
"""
for selector in (
".jmt-base-table-empty .jmt-empty-description-color",
".jmt-base-table-empty",
):
try:
locator = page.locator(selector).first
if not locator.is_visible(timeout=800):
continue
text = locator.inner_text(timeout=800) or ""
except Exception:
continue
if "暂无数据" in text:
return True
return False
def collect_style(context, page, style_name: str, spus: list[str], output_dir: Path) -> dict[str, Any]:
"""采集单款: 明细页取 SPU 树与主图 -> 搜索概览取曝光/点击 -> 聚合下载写 JSON。"""
if not spus:
@@ -722,6 +806,20 @@ def collect_style(context, page, style_name: str, spus: list[str], output_dir: P
snapshot_path.parent.mkdir(parents=True, exist_ok=True)
snapshot_path.write_text(page.content(), encoding="utf-8")
print(f" [DEBUG] snapshot: {snapshot_path}")
if _page_reports_no_data(page):
empty = {
"style_name": style_name,
"platform": "jd",
"collected_at": datetime.now().isoformat(timespec="seconds"),
"spu_count": 0,
"raw_sku_count": 0,
"skipped": True,
"skip_reason": "京东商智当日无曝光/成交数据(查询结果为空)",
"records": [],
}
write_style_outputs(empty, output_dir)
print(f" [SKIP] {style_name}: 京东商智当日无数据(暂无数据),记录空结果并继续")
return empty
raise RuntimeError(f"款 {style_name} 明细页未解析到 SKU 行")
metrics = collect_sku_search_metrics(context, [r["sku_id"] for r in sku_rows])
@@ -757,6 +855,65 @@ def parse_styles(value: str | None) -> list[dict]:
return [g for g in groups if g["style"] in wanted]
def wait_for_portal_session(page, *, headless: bool) -> None:
"""等密码登录后回到商家后台,识别登录页与京东风控认证页。
2026-10-18 的实际故障就是停在 ``aq.jd.com/certified`` 认证页:
老逻辑只判断 ``shop.jd.com in url``,报错时把带一次性 token 的完整 URL
打进运行记录,运维无法判断该做什么。这里把三种终态分开处理:
- 回到后台:继续采集;
- 登录页:无头模式下短暂等待异步跳转,仍不跳转即判定登录失败;
- 安全认证页:无头模式直接抛可执行错误(无法自动完成短信/图形验证),
有界面模式给人工留出完成验证的时间。
"""
deadline = time.monotonic() + LOGIN_SETTLE_TIMEOUT_MS / 1000
login_stuck_since: float | None = None
notice_printed = False
last_url = ""
while time.monotonic() < deadline:
try:
last_url = str(page.url)
except Exception:
last_url = ""
if _is_authenticated_portal(last_url):
page.wait_for_timeout(1500)
return
challenge = _security_challenge_host(last_url)
if challenge:
if headless:
raise _security_challenge_error(last_url)
if not notice_printed:
print(
f" [需要操作] 京东安全认证页({challenge})已打开,"
"请在浏览器内完成短信/图形验证"
)
notice_printed = True
elif _looks_like_login_url(last_url):
login_stuck_since = login_stuck_since or time.monotonic()
if headless and time.monotonic() - login_stuck_since >= HEADLESS_LOGIN_STALL_SECONDS:
break
else:
login_stuck_since = None
try:
page.wait_for_timeout(500)
except Exception:
time.sleep(0.5)
if _security_challenge_host(last_url):
raise _security_challenge_error(last_url)
raise RuntimeError(
"京东登录未完成:密码登录后没有回到商家后台,请执行 "
"`uv run gyxx accounts login jd-shop --force` 人工确认登录态后重跑;"
f"当前页面: {_url_summary(last_url)}"
)
def run(args: argparse.Namespace) -> None:
global _DATE_LABEL
output_dir = managed_data_path(args.output)
@@ -781,12 +938,15 @@ def run(args: argparse.Namespace) -> None:
# Step 1: 登录
page.goto("https://shop.jd.com/jdm/home", wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(2000)
if "shop.jd.com" in page.url and not _looks_like_login_url(page.url):
if _is_authenticated_portal(page.url):
print("[OK] 检测到京东已登录态")
else:
if _security_challenge_host(page.url):
# 认证页上再提交一次账密只会重复触发风控,直接给出人工处置路径。
raise _security_challenge_error(page.url)
step_login(page, args.shop, args.password)
if "shop.jd.com" not in page.url or _looks_like_login_url(page.url):
raise RuntimeError(f"京东登录未完成,当前 URL: {page.url}")
wait_for_portal_session(page, headless=args.headless)
print("[OK] 京东登录完成")
# Step 2-3: 商智 -> 新版页面
sz_page = step_enter_shangzhi(context, page)
@@ -128,7 +128,7 @@ def _print_preflight() -> None:
return
collectable = daily["jd"]["collectable"]
skipped = daily["jd"]["skipped"]
print(f"[JD] 飞书表格 待采 {len(collectable)} 款, 跳过 {len(skipped)} 款 (款式级缺 erp/sales,或 JD 缺商品链接ID)")
print(f"[JD] 飞书表格 待采 {len(collectable)} 款, 跳过 {len(skipped)} 款 (款式级缺 erp 或已由同系列归集,或 JD 缺商品链接ID)")
for s in skipped:
print(f" - {s['style']}: {s['reason']}")
@@ -0,0 +1,344 @@
"""Weekly points-exchange ranking for private-domain material exchange.
The data-hub points-exchange endpoint returns one row per exchanged SKU with a
``quantity`` field. This workflow aggregates those rows into one ranking of
items (商品) for a closed time window and posts the ranking to Feishu.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import sys
from dataclasses import dataclass
from datetime import date, datetime, timedelta
from decimal import Decimal, InvalidOperation
from zoneinfo import ZoneInfo
from gyxx_flow.adapters import (
ANALYZER_NOTIFICATION_APP_PROFILE,
resolve_notification_route,
send_lark_bot_message,
)
from gyxx_flow.adapters.data_interaction import query_points_exchange
WORKFLOW_ID = "product.points_exchange.weekly"
DEFAULT_WINDOW_DAYS = 7
MAX_WINDOW_DAYS = 92
MESSAGE_CHUNK_SIZE = 40
STDOUT_TOP_ROWS = 20
SHANGHAI = ZoneInfo("Asia/Shanghai")
_SAFE_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
@dataclass(frozen=True, slots=True)
class ExchangeItem:
"""One exchanged item aggregated across every SKU of that item."""
item_id: str
title: str
quantity: Decimal
sku_count: int
def _parse_date(value: str, field: str) -> date:
try:
parsed = date.fromisoformat(value.strip())
except (AttributeError, TypeError, ValueError) as exc:
raise ValueError(f"{field} must use the YYYY-MM-DD format: {value!r}") from exc
if not _SAFE_DATE.fullmatch(value.strip()):
raise ValueError(f"{field} must use the YYYY-MM-DD format: {value!r}")
return parsed
def _business_date() -> date:
value = os.environ.get("GYXX_BUSINESS_DATE", "").strip()
if value and _SAFE_DATE.fullmatch(value):
return _parse_date(value, "GYXX_BUSINESS_DATE")
return datetime.now(SHANGHAI).date()
def _previous_full_week(today: date) -> tuple[date, date]:
"""Return the Monday/Sunday pair of the last completed natural week."""
start = today - timedelta(days=today.weekday() + DEFAULT_WINDOW_DAYS)
return start, start + timedelta(days=DEFAULT_WINDOW_DAYS - 1)
def resolve_window(
start_value: str | None,
end_value: str | None,
) -> tuple[date, date]:
"""Resolve one closed date window, defaulting to last week's Mon-Sun."""
if start_value and end_value:
start = _parse_date(start_value, "--from")
end = _parse_date(end_value, "--to")
elif start_value:
start = _parse_date(start_value, "--from")
end = start + timedelta(days=DEFAULT_WINDOW_DAYS - 1)
elif end_value:
end = _parse_date(end_value, "--to")
start = end - timedelta(days=DEFAULT_WINDOW_DAYS - 1)
else:
start, end = _previous_full_week(_business_date())
if end < start:
raise ValueError(f"window end {end} is before start {start}")
span = (end - start).days + 1
if span > MAX_WINDOW_DAYS:
raise ValueError(
f"window spans {span} days; the supported maximum is {MAX_WINDOW_DAYS}"
)
return start, end
def _quantity(row: dict[str, object], index: int) -> Decimal:
value = row.get("quantity")
if value is None or isinstance(value, bool):
raise ValueError(f"points-exchange row {index} has no quantity")
try:
quantity = Decimal(str(value).strip())
except (InvalidOperation, ValueError) as exc:
raise ValueError(f"points-exchange row {index} has invalid quantity") from exc
if not quantity.is_finite():
raise ValueError(f"points-exchange row {index} has non-finite quantity")
return quantity
def _text_field(row: dict[str, object], *names: str) -> str:
for name in names:
value = row.get(name)
if value is not None and str(value).strip():
return " ".join(str(value).split())
return ""
def aggregate_items(rows: list[dict[str, object]]) -> list[ExchangeItem]:
"""Aggregate SKU rows into an item ranking ordered by quantity desc."""
titles: dict[str, dict[str, Decimal]] = {}
sku_ids: dict[str, set[str]] = {}
totals: dict[str, Decimal] = {}
for index, row in enumerate(rows, start=1):
quantity = _quantity(row, index)
item_id = _text_field(row, "itemId", "item_id")
title = _text_field(row, "title", "item_name", "name") or "未命名商品"
sku = _text_field(row, "skuId", "sku_id", "skuCode", "sku_code")
if not item_id:
item_id = f"sku:{sku}" if sku else f"row:{index}"
if quantity <= 0:
continue
totals[item_id] = totals.get(item_id, Decimal("0")) + quantity
titles.setdefault(item_id, {})
titles[item_id][title] = titles[item_id].get(title, Decimal("0")) + quantity
sku_ids.setdefault(item_id, set())
if sku:
sku_ids[item_id].add(sku)
items = [
ExchangeItem(
item_id=item_id,
title=max(titles[item_id].items(), key=lambda pair: (pair[1], pair[0]))[0],
quantity=quantity,
sku_count=len(sku_ids.get(item_id, ())),
)
for item_id, quantity in totals.items()
]
return sorted(items, key=lambda item: (-item.quantity, item.title, item.item_id))
def _quantity_text(quantity: Decimal) -> str:
return format(quantity.normalize(), "f")
def _card(
items: list[ExchangeItem],
*,
start: date,
end: date,
total_quantity: Decimal,
item_count: int,
page: int,
pages: int,
) -> dict[str, object]:
summary = (
f"**周期**:{start.isoformat()} ~ {end.isoformat()}\n"
f"**兑换商品**:{item_count} 个 · **兑换总量**:{_quantity_text(total_quantity)}\n"
)
if items:
top = items[0]
summary += f"**TOP1**:{top.title} · {_quantity_text(top.quantity)}"
if pages > 1:
summary += f"\n第 {page}/{pages} 条,本条 {len(items)} 个商品"
return {
"schema": "2.0",
"config": {"update_multi": True, "width_mode": "fill"},
"header": {
"title": {"tag": "plain_text", "content": "私域积分兑换周报"},
"subtitle": {
"tag": "plain_text",
"content": "积分兑换物资消耗排行 · 按兑换数量从高到低",
},
"template": "blue",
},
"body": {
"direction": "vertical",
"padding": "12px 12px 20px 12px",
"vertical_spacing": "8px",
"elements": [
{
"tag": "markdown",
"element_id": "summary",
"content": summary,
},
{
"tag": "table",
"element_id": "exchange_ranking",
"page_size": 10,
"row_height": "auto",
"row_max_height": "124px",
"freeze_first_column": True,
"columns": [
{
"name": "rank",
"display_name": "排名",
"data_type": "text",
# Feishu rejects any column narrower than 80px.
"width": "80px",
"horizontal_align": "center",
},
{
"name": "title",
"display_name": "商品",
"data_type": "text",
"width": "auto",
},
{
"name": "sku_count",
"display_name": "SKU 数",
"data_type": "text",
"width": "80px",
"horizontal_align": "right",
},
{
"name": "quantity",
"display_name": "兑换数量",
"data_type": "text",
"width": "88px",
"horizontal_align": "right",
},
],
"rows": [
{
"rank": str(rank),
"title": item.title,
"sku_count": str(item.sku_count),
"quantity": _quantity_text(item.quantity),
}
for rank, item in enumerate(items, start=1)
],
},
],
},
}
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="汇总私域积分兑换商品的消耗数量排行并发送飞书周报。",
)
parser.add_argument(
"--from",
dest="start_date",
help="窗口起始日(YYYY-MM-DD),缺省取上一个完整自然周的周一",
)
parser.add_argument(
"--to",
dest="end_date",
help="窗口结束日(YYYY-MM-DD),缺省为起始日 +6 天",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="只打印排行,不发送飞书消息",
)
return parser.parse_args(argv)
def run(argv: list[str] | None = None) -> int:
arguments = _parse_args(argv)
start, end = resolve_window(arguments.start_date, arguments.end_date)
rows = query_points_exchange(
start_time=f"{start.isoformat()} 00:00:00",
end_time=f"{end.isoformat()} 23:59:59",
)
items = aggregate_items(rows)
total_quantity = sum((item.quantity for item in items), Decimal("0"))
print(
f"积分兑换查询完成:{start.isoformat()} ~ {end.isoformat()},"
f"明细 {len(rows)} 条,商品 {len(items)} 个,兑换总量 {_quantity_text(total_quantity)}"
)
for rank, item in enumerate(items[:STDOUT_TOP_ROWS], start=1):
print(
f"{rank:>3}. {item.title}\tSKU {item.sku_count}\t"
f"{_quantity_text(item.quantity)}"
)
if not items:
print("该周期没有积分兑换记录,不发送飞书周报。")
return 0
if arguments.dry_run:
print("dry-run:跳过飞书发送。")
return 0
route = resolve_notification_route(WORKFLOW_ID, ())
if not route.enabled:
print("该工作流的飞书通知路由已停用。")
return 0
if not route.open_ids:
raise RuntimeError("积分兑换周报通知路由没有配置接收人")
pages = (len(items) + MESSAGE_CHUNK_SIZE - 1) // MESSAGE_CHUNK_SIZE
window_key = f"{start.isoformat()}~{end.isoformat()}"
for page in range(pages):
chunk = items[page * MESSAGE_CHUNK_SIZE : (page + 1) * MESSAGE_CHUNK_SIZE]
card = _card(
chunk,
start=start,
end=end,
total_quantity=total_quantity,
item_count=len(items),
page=page + 1,
pages=pages,
)
card_bytes = json.dumps(
card, ensure_ascii=False, sort_keys=True, separators=(",", ":")
).encode("utf-8")
chunk_hash = hashlib.sha256(card_bytes).hexdigest()[:20]
idempotency_key = f"{WORKFLOW_ID}:{window_key}:{page + 1}:{chunk_hash}"
for open_id in route.open_ids:
send_lark_bot_message(
user_id=open_id,
content=card,
msg_type="interactive",
idempotency_key=idempotency_key,
profile=route.app_profile or ANALYZER_NOTIFICATION_APP_PROFILE,
)
print(f"积分兑换周报已发送:{len(items)} 个商品,{pages} 条飞书消息。")
return 0
def main() -> int:
try:
return run()
except Exception as exc:
print(f"积分兑换周报工作流失败:{exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
@@ -8,7 +8,8 @@
款式来源 (2026-06-27 改造):
- --styles 显式传: 用 CLI 参数指定的款式列表
- --styles 留空: 动态从项目数据库读 (get_main_image_styles: 天猫分组 + 主图飞书多维表格地址 同时存在)
- --styles 留空: 动态从项目数据库读 (get_main_image_styles: 身份齐全的天猫款式)
· 主图飞书表地址不参与采集门:缺地址的款照常抓图并入库,写表环节跳过
数据落点: data/主图信息/<date>/<款式>/wanxiang_creative_main_images.json
插入幂等: 飞书按 (时间, 平台, 图片) 更新,PG 按日期/款式/平台/图片覆盖。
@@ -112,7 +113,7 @@ def main() -> int:
parser.add_argument("--skip-collect", action="store_true", help="跳过采集,只跑 insert")
parser.add_argument("--skip-insert", action="store_true", help="只采集,不推飞书")
parser.add_argument("--styles", default="",
help="只采集/插入指定款式,逗号分隔 (传空=动态从项目数据库读天猫 + 主图飞书多维表格地址)")
help="只采集/插入指定款式,逗号分隔 (传空=动态从项目数据库读身份齐全的天猫款式)")
parser.add_argument("--headless", action="store_true",
help="采集时用无头浏览器 (服务器/无人值守场景)")
parser.add_argument("--max-pages", type=int, default=100,
@@ -128,11 +129,11 @@ def main() -> int:
target_date = last_sunday.strftime("%Y-%m-%d")
py = sys.executable
# --styles 为空时动态从项目数据库读 (天猫分组 + 主图飞书多维表格地址 同时存在)
# --styles 为空时动态从项目数据库读(身份齐全即可,主图地址不参与采集门)
if not args.styles:
bitable_styles = load_dynamic_styles()
args.styles = ",".join(bitable_styles)
print(f"[PREFLIGHT] 飞书多维表格动态读主图款式 {len(bitable_styles)} 款")
print(f"[PREFLIGHT] 动态读主图款式 {len(bitable_styles)} 款(缺主图地址的款只入库不写表)")
else:
print(f"[PREFLIGHT] --styles 显式指定: {args.styles}")
@@ -27,7 +27,6 @@ from runtime_paths import LOG_ROOT, artifact_relative_path, managed_data_path
# 复用 TM insert 脚本的工具函数
from scripts.insert_main_image_records import (
DEFAULT_DATA_ROOT,
MAIN_IMAGE_MAP,
batch_create_records,
date_to_month_week,
discover_fields,
@@ -121,6 +120,11 @@ def process_style(style_key: str, cfg: dict, date_str: str, data_root: Path, dry
row["_pg_record"] = pg_record
pg_records.append(pg_record)
# 缺主图飞书表地址:照常入库,只是不写表(与商品经营日报缺销量表同口径)
if not str(cfg.get("base_token") or "").strip() or not str(cfg.get("table_id") or "").strip():
print(f" [SKIP-TABLE] {style_key}: 未配置主图飞书表地址,仅入库")
return result
try:
fields = discover_fields(cfg["base_token"], cfg["table_id"])
except Exception as e:
@@ -223,11 +227,14 @@ def main():
try:
config = get_bitable_main_image_map()
if not config:
raise RuntimeError("飞书动态配置为空")
raise RuntimeError("dim_style 未返回任何带主图表地址的款式")
except Exception as exc:
print(f"[WARN] 动态读取主图配置失败,使用本地文件回退: {exc}", file=sys.stderr)
config = read_json(MAIN_IMAGE_MAP)
config.pop("_comment", None)
# 主图表地址唯一来源是 dim_style;读不到即失败,不再回退本地 JSON。
print(
f"[FAIL] 主图配置读取失败,拒绝回退本地 JSON: {exc}",
file=sys.stderr,
)
return 1
data_root = managed_data_path(args.data_root)
date_dir = data_root / args.date
@@ -262,9 +269,12 @@ def main():
for style_dir_name in target_styles:
key = normalize_key(style_dir_name)
if key not in norm_config:
print(f"[SKIP] {style_dir_name}: config 里找不到 (lowercase 匹配)")
continue
orig_key, cfg = norm_config[key]
# 缺主图飞书表地址:照常采图入库,只是不写表
orig_key = style_dir_name
cfg = {"display_name": style_dir_name, "platform": "京东", "enabled": True}
print(f"[NO-TABLE] {style_dir_name}: 未配置主图飞书表地址,仅入库")
else:
orig_key, cfg = norm_config[key]
if not cfg.get("enabled", True):
print(f"[SKIP] {orig_key}: enabled=false")
continue
@@ -30,7 +30,6 @@ from runtime_paths import (
managed_data_path,
)
MAIN_IMAGE_MAP = PROJECT_ROOT / "bitable_main_image_map.json"
DEFAULT_DATA_ROOT = RAW_DATA_ROOT / "主图信息"
LARK_CLI = (
@@ -41,10 +40,14 @@ LARK_CLI = (
def _print_preflight() -> None:
"""启动期打印: 飞书表格里本周要插主图记录的款式数 (仅要求 主图飞书多维表格地址 非空)。"""
"""启动期打印: 采集目标款数,以及其中能写飞书表的款数。"""
try:
styles = get_main_image_styles()
print(f"[主图-insert] 飞书表格 周汇总目标 {len(styles)} 款 (天猫 + 主图飞书多维表格地址)")
write_targets = get_bitable_main_image_map()
print(
f"[主图-insert] 采集目标 {len(styles)} 款;"
f"可写飞书表 {len(write_targets)} 款(其余仅入库)"
)
except Exception as exc:
print(f"[WARN] 飞书读取失败, 跳过 preflight: {exc}", file=sys.stderr)
@@ -492,6 +495,11 @@ def process_product(style_key: str, cfg: dict, date_str: str, data_root: Path, d
row["_pg_record"] = pg_record
pg_records.append(pg_record)
# 缺主图飞书表地址:照常入库,只是不写表(与商品经营日报缺销量表同口径)
if not str(cfg.get("base_token") or "").strip() or not str(cfg.get("table_id") or "").strip():
print(f" [SKIP-TABLE] {style_key}: 未配置主图飞书表地址,仅入库")
return result
# discover field ids
try:
fields = discover_fields(cfg["base_token"], cfg["table_id"])
@@ -598,11 +606,14 @@ def main():
try:
config = get_bitable_main_image_map()
if not config:
raise RuntimeError("飞书动态配置为空")
raise RuntimeError("dim_style 未返回任何带主图表地址的款式")
except Exception as exc:
print(f"[WARN] 动态读取主图表配置失败,使用本地文件回退: {exc}", file=sys.stderr)
config = read_json(MAIN_IMAGE_MAP)
config.pop("_comment", None)
# 主图表地址唯一来源是 dim_style;读不到即失败,不再回退本地 JSON。
print(
f"[FAIL] 主图表配置读取失败,拒绝回退本地 JSON: {exc}",
file=sys.stderr,
)
return 1
data_root = managed_data_path(args.data_root)
date_dir = data_root / args.date
if not date_dir.exists():
@@ -640,9 +651,12 @@ def main():
for style_dir_name in target_styles:
key = normalize_key(style_dir_name)
if key not in norm_config:
print(f"[SKIP] {style_dir_name}: config 里找不到 (lowercase 匹配)")
continue
orig_key, cfg = norm_config[key]
# 缺主图飞书表地址:照常采图入库,只是不写表
orig_key = style_dir_name
cfg = {"display_name": style_dir_name, "platform": "天猫", "enabled": True}
print(f"[NO-TABLE] {style_dir_name}: 未配置主图飞书表地址,仅入库")
else:
orig_key, cfg = norm_config[key]
if not cfg.get("enabled", True):
print(f"[SKIP] {orig_key}: enabled=false")
continue
@@ -318,7 +318,14 @@ def plan_erp_styles(
missing.append(target.product_name)
continue
style_name, style_config = resolved
raw_codes = style_config.get("erp_codes") or ()
# 月度表按款明细:取款式建档时自己填的编码(series.own_erp_codes)。
# loader 把整条系列的编码并集挂到了锚点的 erp_codes 上,那是给采集端
# 当唯一取数单元用的;报表若照搬,同系列每个成员行都会把并集查一遍,
# 销量被重复计入。锚点自身没填编码时这里为空,按 0 写入即可。
series = style_config.get("series") or {}
raw_codes = series.get("own_erp_codes")
if raw_codes is None:
raw_codes = style_config.get("erp_codes") or ()
codes = tuple(
dict.fromkeys(
str(code).strip() for code in raw_codes if str(code).strip()
@@ -174,7 +174,7 @@ def main(argv: list[str] | None = None) -> int:
daily = get_daily_styles()
tm_collectable = daily["tm"]["collectable"]
tm_skipped = daily["tm"]["skipped"]
print(f"[TM] 飞书表格 待采 {len(tm_collectable)} 款, 跳过 {len(tm_skipped)} 款 (款式级缺 erp/sales,或 TM 缺商品链接ID)")
print(f"[TM] 飞书表格 待采 {len(tm_collectable)} 款, 跳过 {len(tm_skipped)} 款 (款式级缺 erp 或已由同系列归集,或 TM 缺商品链接ID)")
for s in tm_skipped:
print(f" - {s['style']}: {s['reason']}")
brands: dict[str, list[str]] = defaultdict(list)
@@ -26,7 +26,6 @@ from urllib.parse import quote
from urllib.request import Request, urlopen
from config.style_config_loader import (
get_bitable_main_image_map,
get_main_image_styles,
get_tm_style_id_map,
)
@@ -88,11 +87,11 @@ _STATIC_STYLE_NAMES = [
def _get_style_names() -> list[str]:
"""优先从飞书读取有主图表配置的款式列表,失败则回退硬编码。"""
"""采集目标=身份齐全的天猫款式;主图飞书表地址只影响写表,不影响采集。"""
try:
cfg = get_bitable_main_image_map()
cfg = get_main_image_styles()
if cfg:
return sorted(cfg.keys())
return sorted(cfg)
except Exception as exc:
print(f"[WARN] 动态读取主图款式列表失败,使用硬编码回退: {exc}")
return list(_STATIC_STYLE_NAMES)
@@ -1882,12 +1882,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
default=None,
help="单日业务日期 YYYY-MM-DD(申请时间筛选的开始日期);控制台会自动传入",
)
date_arguments.add_argument(
parser.add_argument(
"--range-days",
type=int,
default=0,
help="采集 business_date 往前 N 天(含当天)的区间;定时任务使用,"
"0 表示仅按 --date/--from/--to 采集",
help="采集 business_date 往前 N 天(含当天)的区间;定时任务与控制台使用,"
"需配合 --date 指定区间终点,0 表示仅按 --date/--from/--to 采集",
)
date_arguments.add_argument(
"--from",
@@ -330,10 +330,13 @@ def _date_picker_input(page: Page) -> Any:
def _open_date_picker(page: Page) -> Any:
date_input = _date_picker_input(page)
trigger = date_input.locator(".jmtd-base-input-top").first
if trigger.count():
trigger.click()
else:
date_input.click()
target = trigger if trigger.count() else date_input
try:
target.click(timeout=5_000)
except Exception:
# 商智新版页面在时间控件上叠了透明遮罩,Playwright 的可点击性
# 检查会一直等待;直接派发 DOM 点击,弹层同样正常展开。
target.evaluate("node => node.click()")
return _visible_date_picker(page)
@@ -1342,13 +1345,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
default=None,
help="单日业务日期 YYYY-MM-DD;定时任务使用",
)
date_arguments.add_argument(
"--range-days",
type=int,
default=0,
help="采集 business_date 往前 N 天(含当天)的区间;定时任务使用,"
"0 表示仅按 --date/--from/--to 采集",
)
date_arguments.add_argument(
"--from",
dest="from_date",
@@ -1356,6 +1352,13 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
default=None,
help="自定义查询范围开始日期 YYYY-MM-DD",
)
parser.add_argument(
"--range-days",
type=int,
default=0,
help="采集 business_date 往前 N 天(含当天)的区间;定时任务使用,"
"0 表示仅按 --date/--from/--to 采集",
)
parser.add_argument(
"--to",
dest="to_date",
@@ -334,6 +334,7 @@ def open_chat_log(page: Page) -> None:
page.get_by_text("聊天记录", exact=True).first.wait_for(
state="visible", timeout=30_000
)
_dismiss_notice_modal(page)
def _visible_date_dropdown(page: Page) -> Any:
@@ -379,17 +380,67 @@ def _select_date_cell(page: Page, target_date: date) -> Any:
raise RuntimeError(f"日期面板无法导航到 {date_text}")
def _dismiss_notice_modal(page: Page) -> None:
"""关闭客服后台的公告弹窗,避免它遮挡日期与查询控件。"""
for _ in range(3):
wrap = page.locator(
".kf-manage-lite-modal-wrap.notice-modal-container:visible"
).first
try:
if not wrap.count():
return
except Exception:
return
closed = False
for target in (
wrap.get_by_role("button", name="我已知晓"),
wrap.locator("button.kf-manage-lite-modal-close").first,
):
try:
if not target.count():
continue
target.click(timeout=5_000, force=True)
page.wait_for_timeout(300)
closed = True
break
except Exception:
continue
if not closed:
# 兜底:公告只是遮挡层,直接摘掉节点后继续。
try:
wrap.evaluate("node => node.remove()")
except Exception:
pass
page.wait_for_timeout(200)
if not wrap.count():
return
def _pick_date(page: Page, target_date: date) -> None:
"""选中目标自然日;双月面板重叠时退回 DOM 点击。"""
cell = _select_date_cell(page, target_date)
try:
cell.click(timeout=5_000)
except Exception:
# 京东双月面板的表格会互相遮挡,Playwright 始终判定被拦截;
# 单元格本身可用,直接派发 DOM 点击即可选中。
cell.evaluate("node => node.click()")
def set_single_day(page: Page, target_date: date) -> None:
"""将客服聊天记录的开始、结束时间设置为同一自然日。"""
_dismiss_notice_modal(page)
start = page.locator('input[placeholder="开始日期"]').first
end = page.locator('input[placeholder="结束日期"]').first
start.click()
_select_date_cell(page, target_date).click()
_pick_date(page, target_date)
page.wait_for_timeout(120)
# 京东 RangePicker 需要显式切到结束输入框;同一天再选一次才会得到 23:59:59。
end.click()
_select_date_cell(page, target_date).click()
_pick_date(page, target_date)
page.wait_for_timeout(150)
dropdown = page.locator(".kf-manage-lite-picker-dropdown:visible").first
@@ -1061,7 +1112,7 @@ def run(
conversation_count, message_count = _persist_records(
normalized, no_db=args.no_db
)
collected[customer_id] = {
day_collected[customer_id] = {
"session_count": normalized["session_count"],
"message_count": normalized["message_count"],
"filtered_message_count": normalized[
@@ -1104,7 +1155,9 @@ def run(
f"顾客序号 {index + 1}(共 {len(expected_ids)})连续 {args.retries} 次采集失败"
) from last_error
unresolved = [item for item in expected_ids if item not in collected]
unresolved = [
item for item in expected_ids if item not in day_collected
]
if unresolved:
save_progress(
output_path,
@@ -173,27 +173,9 @@ def _identity_markers(
return ("自营",) if store_key == "self-operated" else ("光影行星",)
def open_and_verify_reviews(
page: Page,
*,
store_key: str,
environment: dict[str, str],
) -> None:
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000)
_wait_for_page(page)
page.wait_for_timeout(2_000)
if "login" in page.url.lower():
raise RuntimeError(f"登录未成功,评价页面跳转到了登录页: {page.url}")
page.get_by_text("评价管理", exact=True).first.wait_for(
state="visible", timeout=30_000
)
page.get_by_text("评价等级", exact=True).first.wait_for(
state="visible", timeout=30_000
)
_hide_feedback_widgets(page)
markers = _identity_markers(environment, store_key)
if store_key == "self-operated":
markers = (*markers, "光影自营", "自营")
def _verify_shop_markers(page: Page, markers: tuple[str, ...]) -> None:
"""确认当前登录店铺身份,避免把共享京东 Cookie 用到错误店铺。"""
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
body_text = page.locator("body").inner_text(timeout=15_000)
@@ -213,6 +195,36 @@ def open_and_verify_reviews(
)
def open_and_verify_reviews(
page: Page,
*,
store_key: str,
environment: dict[str, str],
) -> None:
markers = _identity_markers(environment, store_key)
if store_key == "self-operated":
markers = (*markers, "光影自营", "自营")
# 评价后台独立页已不再展示店铺名,先在京麦首页确认店铺身份,
# 再进入评价页校验列表锚点。
page.goto(JD_SHOP_HOME_URL, wait_until="domcontentloaded", timeout=60_000)
_wait_for_page(page)
if "login" in page.url.lower():
raise RuntimeError(f"登录未成功,京东商家后台跳转到了登录页: {page.url}")
_verify_shop_markers(page, markers)
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000)
_wait_for_page(page)
page.wait_for_timeout(2_000)
if "login" in page.url.lower():
raise RuntimeError(f"登录未成功,评价页面跳转到了登录页: {page.url}")
page.get_by_text("评价列表", exact=True).first.wait_for(
state="visible", timeout=30_000
)
page.get_by_text("评价等级", exact=True).first.wait_for(
state="visible", timeout=30_000
)
_hide_feedback_widgets(page)
def _visible_date_dialog(page: Page) -> Any:
dialog = page.locator('[role="dialog"]:visible').last
dialog.wait_for(state="visible", timeout=10_000)
@@ -117,6 +117,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
default=True,
help="使用隐藏指纹浏览器(默认)",
)
parser.add_argument(
"--headed",
action="store_true",
help="显示浏览器窗口;淘宝弹滑块验证时便于人工拖动后继续采集",
)
return parser.parse_args(argv)
@@ -254,64 +259,130 @@ def open_and_verify(page: Any) -> Any:
return frame
def _calendar_cell(picker: Any, target: date) -> Any | None:
date_selector = f'td[title="{target.isoformat()}"]'
current_month_cell = picker.locator(
date_selector
+ ":not(.next-calendar-cell-prev-month):not(.next-calendar-cell-next-month)"
).first
if current_month_cell.count() and current_month_cell.is_visible():
return current_month_cell
cell = picker.locator(date_selector).first
if cell.count() and cell.is_visible():
return cell
return None
CAPTCHA_HINTS = ("请拖动下方滑块", "请按住滑块")
def _select_calendar_day(picker: Any, target: date) -> None:
def _captcha_visible(page: Any) -> bool:
try:
return bool(
page.evaluate(
"""() => {
const text = document.body ? (document.body.innerText || '') : '';
return text.includes('请拖动下方滑块') || text.includes('请按住滑块');
}"""
)
)
except Exception: # noqa: BLE001
return False
def _wait_for_captcha(page: Any, timeout_seconds: int = 300) -> None:
"""淘宝在点开会话详情时会弹滑块验证;出现后等待人工拖动再继续。
没有这道检测时,验证弹窗只会让后续点击超时重试,整轮运行会长时间空转。
"""
if not _captcha_visible(page):
return
print(
" [WARN] 检测到淘宝滑块验证,请在浏览器窗口拖动滑块(最多等待 5 分钟)…",
flush=True,
)
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
page.wait_for_timeout(2_000)
if not _captcha_visible(page):
print(" [OK] 滑块验证已通过,继续采集", flush=True)
return
raise RuntimeError("淘宝滑块验证未在 5 分钟内通过,请人工处理后重跑")
def _date_input_values(frame: Any) -> list[str]:
return frame.locator(
f'{DATE_INPUTS["start"]}, {DATE_INPUTS["end"]}'
).evaluate_all("elements => elements.map(element => element.value)")
def _picker(frame: Any) -> Any:
picker = frame.locator(".next-range-picker-body:visible").first
picker.wait_for(state="visible", timeout=10_000)
return picker
def _pick_day(frame: Any, target: date) -> None:
"""点选一个日期。
每次都重新定位面板并直接用 `td[title]` 取格:曾用
`:not(.next-calendar-cell-prev-month):not(.next-calendar-cell-next-month)`
过滤“当前月”版本,但该面板在双月视图下会取到非预期节点,导致第二次点击
被当成新区间起点(区间被重置)。
"""
iso = target.isoformat()
wait = frame.page.wait_for_timeout
for _ in range(MAX_CALENDAR_NAV_STEPS):
cell = _calendar_cell(picker, target)
if cell is not None:
picker = _picker(frame)
cell = picker.locator(f'td[title="{iso}"]').first
if cell.count():
classes = cell.get_attribute("class") or ""
if "disabled" in classes.casefold():
raise RuntimeError(f"天猫客服后台尚未开放日期 {target.isoformat()}")
raise RuntimeError(f"天猫客服后台尚未开放日期 {iso}")
cell.click()
wait(700)
return
visible_dates = picker.locator("td[title]").evaluate_all(
titles = picker.locator("td[title]").evaluate_all(
"elements => elements.map(element => element.title).filter(Boolean)"
)
parsed_dates: list[date] = []
for value in visible_dates:
parsed: list[date] = []
for value in titles:
try:
parsed_dates.append(date.fromisoformat(value))
parsed.append(date.fromisoformat(value))
except ValueError:
continue
if not parsed_dates:
if not parsed:
raise RuntimeError("天猫客服日期选择器没有可用日期")
if target < min(parsed_dates):
if target < min(parsed):
picker.locator(".next-calendar-btn-prev-month").first.click()
elif target > max(parsed_dates):
elif target > max(parsed):
picker.locator(".next-calendar-btn-next-month").first.click()
else:
raise RuntimeError(f"日期选择器无法选择 {target.isoformat()}")
picker.locator("td[title]").first.wait_for(state="visible", timeout=5_000)
raise RuntimeError(f"日期选择器翻页次数超限: {target.isoformat()}")
raise RuntimeError(f"日期选择器无法选择 {iso}")
wait(400)
raise RuntimeError(f"日期选择器翻页次数超限: {iso}")
def set_date_range(frame: Any, start: date, end: date) -> None:
frame.locator(DATE_INPUTS["start"]).click()
picker = frame.locator(".next-range-picker-body:visible").first
picker.wait_for(state="visible", timeout=10_000)
_select_calendar_day(picker, start)
_select_calendar_day(picker, end)
picker.get_by_role("button", name="确定").click()
actual = frame.locator(
f'{DATE_INPUTS["start"]}, {DATE_INPUTS["end"]}'
).evaluate_all("elements => elements.map(element => element.value)")
"""设置聊天记录查询区间。
该 RangePicker 在选完开始日时会把结束日补成当前值:若结束日恰好已是目标
日期,再点一次该日期等于重新开区间(开始被重置),所以只在结束日不符时
才补点。面板点选后 DOM 会重渲染,因此每次点选都重新定位面板。带一次重试,
避免面板状态残留导致的偶发不生效。
"""
expected = [start.isoformat(), end.isoformat()]
if actual != expected:
raise RuntimeError("天猫客服日期范围未按预期设置")
wait = frame.page.wait_for_timeout
for _ in range(2):
frame.locator(DATE_INPUTS["start"]).click()
_picker(frame)
# 面板展开有动画;动画期间点选会让组件把区间当成“重新开始”。
wait(1_200)
_pick_day(frame, start)
wait(600)
if _date_input_values(frame) != expected:
_pick_day(frame, end)
wait(600)
picker = frame.locator(".next-range-picker-body:visible").first
if picker.count():
confirm = picker.get_by_role("button", name="确定")
if confirm.count():
confirm.first.click()
wait(600)
if _date_input_values(frame) == expected:
return
raise RuntimeError(
f"天猫客服日期范围未按预期设置,期望 {expected},实际 {_date_input_values(frame)}"
)
def _query_window(frame: Any, start: date, end: date) -> list[Any]:
@@ -866,7 +937,9 @@ def run(
init_schema()
browser = ScraplingBrowser(
**browser_runtime_options(environment, headless=args.headless)
**browser_runtime_options(
environment, headless=args.headless and not args.headed
)
)
try:
browser.start()
@@ -893,6 +966,7 @@ def run(
filtered_counts: Counter[str] = Counter()
failures: list[dict[str, Any]] = []
for row_index, row in enumerate(rows, start=1):
_wait_for_captcha(page)
try:
conversation, conversation_messages = _capture_customer(
frame,
@@ -1029,7 +1103,7 @@ def main(argv: list[str] | None = None) -> int:
return 130
except Exception as exc:
print(
f"[ERROR] 天猫客服聊天记录采集失败: {type(exc).__name__}",
f"[ERROR] 天猫客服聊天记录采集失败: {type(exc).__name__}: {exc}",
file=sys.stderr,
)
return 1
@@ -235,14 +235,83 @@ def _select_calendar_day(picker: Any, target: date) -> None:
if not visible_dates:
raise RuntimeError("天猫评价日期选择器没有可用日期")
if target < min(visible_dates):
picker.locator(".next-calendar-btn-prev-month").first.click()
_shift_calendar_month(picker, "prev")
elif target > max(visible_dates):
picker.locator(".next-calendar-btn-next-month").first.click()
_shift_calendar_month(picker, "next")
else:
raise RuntimeError(f"评价日期选择器无法选择 {target.isoformat()}")
raise RuntimeError(f"评价日期选择器翻页次数超限: {target.isoformat()}")
PICKER_SELECTORS = (
".next-range-picker2-panel:visible",
".next-range-picker-body:visible",
".next-range-picker-panel:visible",
)
PREV_MONTH_SELECTORS = (
".next-calendar2-header-left-btn",
".next-calendar-btn-prev-month",
)
NEXT_MONTH_SELECTORS = (
".next-calendar2-header-right-btn",
".next-calendar-btn-next-month",
)
def _visible_picker(page: Any) -> Any | None:
"""返回当前展开的日期面板;新版为 next-range-picker2-panel。"""
for selector in PICKER_SELECTORS:
locator = page.locator(selector).first
try:
if locator.count() and locator.is_visible():
return locator
except Exception: # noqa: BLE001
continue
return None
def _shift_calendar_month(picker: Any, direction: str) -> None:
selectors = PREV_MONTH_SELECTORS if direction == "prev" else NEXT_MONTH_SELECTORS
for selector in selectors:
button = picker.locator(selector).first
try:
if button.count() and button.is_visible():
button.click()
return
except Exception: # noqa: BLE001
continue
raise RuntimeError(f"天猫评价日期面板找不到{direction}翻月按钮")
def _pick_range_on_calendar(page: Any, start: date, end: date) -> bool:
"""优先用日历面板点选区间。
天猫新版 RangePicker 是受控组件:直接 `fill` 只改了 DOM 值,组件状态没变,
查询仍按“全部日期”执行(会捞出区间外的评价)。因此必须点选单元格。
"""
start_input = _date_input(page, "起始日期")
start_input.click()
picker = _visible_picker(page)
if picker is None:
return False
_select_calendar_day(picker, start)
page.wait_for_timeout(400)
picker = _visible_picker(page)
if picker is not None:
_select_calendar_day(picker, end)
page.wait_for_timeout(600)
picker = _visible_picker(page)
if picker is not None:
confirm = picker.get_by_role("button", name="确定", exact=True)
if confirm.count() and confirm.is_visible():
confirm.click()
return True
def set_review_date_range(page: Any, start: date, end: date) -> None:
"""显式设置评价时间;不得在空日期条件下点击搜索。"""
@@ -250,37 +319,73 @@ def set_review_date_range(page: Any, start: date, end: date) -> None:
raise ValueError("评价结束日期不能早于开始日期")
start_input = _date_input(page, "起始日期")
end_input = _date_input(page, "结束日期")
readonly = any(
locator.get_attribute("readonly") is not None
or locator.get_attribute("aria-readonly") == "true"
for locator in (start_input, end_input)
)
if not readonly:
actual: list[str] = []
picked = False
try:
picked = _pick_range_on_calendar(page, start, end)
except Exception as exc: # noqa: BLE001
if not is_browser_timeout_error(exc):
raise
if picked:
actual = [start_input.input_value(), end_input.input_value()]
if actual != [start.isoformat(), end.isoformat()]:
# 面板点选不可用时退回直接写入;随后的日期范围校验会把不一致暴露出来。
try:
start_input.fill(start.isoformat())
end_input.fill(end.isoformat())
end_input.press("Tab")
except Exception as exc:
except Exception as exc: # noqa: BLE001
if not is_browser_timeout_error(exc):
# A date input can reject fill even without a readonly attribute;
# use the platform calendar as the supported fallback.
readonly = True
if readonly:
start_input.click()
picker = page.locator(".next-range-picker-body:visible").first
picker.wait_for(state="visible", timeout=10_000)
_select_calendar_day(picker, start)
_select_calendar_day(picker, end)
confirm = picker.get_by_role("button", name="确定", exact=True)
if confirm.count() and confirm.is_visible():
confirm.click()
raise
actual = [start_input.input_value(), end_input.input_value()]
actual = [start_input.input_value(), end_input.input_value()]
expected = [start.isoformat(), end.isoformat()]
if actual != expected:
raise RuntimeError(f"评价日期范围未生效,期望 {expected},实际 {actual}")
def _dismiss_open_overlays(page: Any) -> None:
"""关闭残留的 next-overlay 弹层。
日期选择器/下拉在部分分支会留下 `.next-overlay-backdrop`,它会拦截
「搜索」按钮的指针事件,导致点击一直重试到超时。
"""
for _ in range(3):
try:
if not page.locator(".next-overlay-wrapper.opened:visible").count():
return
except Exception: # noqa: BLE001
return
try:
page.keyboard.press("Escape")
except Exception: # noqa: BLE001
pass
page.wait_for_timeout(200)
try:
page.locator("body").click(position={"x": 4, "y": 4}, timeout=2_000)
except Exception: # noqa: BLE001
pass
page.wait_for_timeout(200)
def _click_search(page: Any) -> None:
"""点击「搜索」;被弹层遮挡时退回 DOM 点击。"""
button = page.get_by_role("button", name="搜索", exact=True).first
_dismiss_open_overlays(page)
try:
button.click(timeout=10_000)
return
except Exception as exc: # noqa: BLE001
if not is_browser_timeout_error(exc):
raise
_dismiss_open_overlays(page)
button.evaluate("element => element.click()")
def clear_review_date_range(page: Any) -> None:
"""只在显式 --all-time 时清空页面时间筛选,并确认两个日期为空。"""
@@ -687,7 +792,7 @@ def run(
assert start_date is not None and end_date is not None
set_review_date_range(page, start_date, end_date)
_ensure_negative_filter(page)
page.get_by_role("button", name="搜索", exact=True).first.click()
_click_search(page)
total, _initial_visible_count = _wait_for_results(page)
expected_pages = math.ceil(total / PAGE_SIZE) if total else 0
artifact["total_count"] = total
@@ -1,11 +1,14 @@
"""PostgreSQL queue access for replenishment arrival-date synchronization.
The ERP application writes one row per tenant/SKU to
The ERP application writes one row per SKU to
``erp_replenishment_arrival_sync``. This module owns only the queue boundary:
claiming work and recording the result after the browser worker has finished.
Browser actions stay in the workflow script so that a claimed row can be
completed one at a time and a changed target date cannot be overwritten by an
older worker.
data-hub 单公司化之后该表已没有 ``tenant_id``,但旧部署仍可能保留多租户结构,
因此读写 SQL 会先探测实际列再决定是否带租户条件。
"""
from __future__ import annotations
@@ -45,18 +48,42 @@ ConnectionFactory = Callable[[], AbstractContextManager[Any]]
class ArrivalSyncQueue:
"""Claim and update arrival synchronization rows with optimistic fencing."""
"""Claim and update arrival synchronization rows with optimistic fencing.
data-hub 在「单公司化」迁移(``sql/上线待执行/20261012_remove_tenant_id.sql``)
之后删除了队列表的 ``tenant_id`` 列,而旧部署仍然是多租户表。这里按实际
表结构自适应:有该列时继续按租户过滤,没有时退化为单公司口径。
"""
def __init__(
self,
environment: Mapping[str, str] | None = None,
*,
connect_factory: ConnectionFactory | None = None,
tenant_column: bool | None = None,
) -> None:
self._environment = dict(os.environ if environment is None else environment)
self._connect_factory = connect_factory or (
lambda: _connect_from_environment(self._environment)
)
self._tenant_column = tenant_column
def _has_tenant_column(self) -> bool:
"""Detect the queue schema once; tolerate the pre-migration layout."""
if self._tenant_column is None:
sql = (
"SELECT 1 FROM information_schema.columns "
"WHERE table_name = %s AND column_name = 'tenant_id'"
)
try:
with self._connect_factory() as connection:
with connection.cursor(row_factory=dict_row) as cursor:
cursor.execute(sql, (QUEUE_TABLE,))
self._tenant_column = cursor.fetchone() is not None
except Exception: # noqa: BLE001 - fall back to the legacy layout
self._tenant_column = True
return bool(self._tenant_column)
def claim(
self,
@@ -68,12 +95,14 @@ class ArrivalSyncQueue:
_validate_positive("tenant_id", tenant_id)
_validate_positive("stale_after_seconds", stale_after_seconds)
tenant_scope = self._has_tenant_column()
tenant_predicate = "tenant_id = %s AND " if tenant_scope else ""
tenant_column = "queue.tenant_id, " if tenant_scope else ""
sql = f"""
WITH candidates AS (
SELECT id
FROM {QUEUE_TABLE}
WHERE tenant_id = %s
AND deleted = 0
WHERE {tenant_predicate}deleted = 0
AND jushuitan_updated = FALSE
AND (
sync_status IN ('pending', 'failed')
@@ -95,25 +124,26 @@ class ArrivalSyncQueue:
updater = %s
FROM candidates
WHERE queue.id = candidates.id
RETURNING queue.id, queue.tenant_id, queue.sku_code,
RETURNING queue.id, {tenant_column}queue.sku_code,
queue.expected_arrival_time
"""
params: tuple[Any, ...] = (
(tenant_id, stale_after_seconds, self._operator())
if tenant_scope
else (stale_after_seconds, self._operator())
)
with self._connect_factory() as connection:
with connection.cursor(row_factory=dict_row) as cursor:
cursor.execute(
sql,
(
tenant_id,
stale_after_seconds,
self._operator(),
),
)
cursor.execute(sql, params)
rows = cursor.fetchall()
return [_task_from_row(row) for row in rows]
def mark_success(self, task: ArrivalSyncTask) -> bool:
"""Mark a task successful only if its target date is still current."""
tenant_predicate = (
"\n AND tenant_id = %s" if self._has_tenant_column() else ""
)
sql = f"""
UPDATE {QUEUE_TABLE}
SET sync_status = 'success',
@@ -122,8 +152,7 @@ class ArrivalSyncQueue:
last_error = '',
update_time = CURRENT_TIMESTAMP,
updater = %s
WHERE id = %s
AND tenant_id = %s
WHERE id = %s{tenant_predicate}
AND sync_status = 'processing'
AND expected_arrival_time IS NOT DISTINCT FROM %s
RETURNING id
@@ -134,6 +163,9 @@ class ArrivalSyncQueue:
"""Record a retryable failure without clobbering a newer ERP target."""
normalized_error = " ".join(str(error or "同步失败").split())[:MAX_ERROR_LENGTH]
tenant_predicate = (
"\n AND tenant_id = %s" if self._has_tenant_column() else ""
)
sql = f"""
UPDATE {QUEUE_TABLE}
SET sync_status = 'failed',
@@ -141,8 +173,7 @@ class ArrivalSyncQueue:
last_error = %s,
update_time = CURRENT_TIMESTAMP,
updater = %s
WHERE id = %s
AND tenant_id = %s
WHERE id = %s{tenant_predicate}
AND sync_status = 'processing'
AND expected_arrival_time IS NOT DISTINCT FROM %s
RETURNING id
@@ -159,12 +190,14 @@ class ArrivalSyncQueue:
task: ArrivalSyncTask,
prefix_params: tuple[Any, ...],
) -> bool:
fence_params: tuple[Any, ...] = (
(task.id, task.tenant_id, task.expected_arrival_time)
if self._has_tenant_column()
else (task.id, task.expected_arrival_time)
)
with self._connect_factory() as connection:
with connection.cursor(row_factory=dict_row) as cursor:
cursor.execute(
sql,
(*prefix_params, task.id, task.tenant_id, task.expected_arrival_time),
)
cursor.execute(sql, (*prefix_params, *fence_params))
return cursor.fetchone() is not None
def _operator(self) -> str:
@@ -178,7 +211,8 @@ def _task_from_row(row: Mapping[str, Any]) -> ArrivalSyncTask:
try:
return ArrivalSyncTask(
id=int(row["id"]),
tenant_id=int(row["tenant_id"]),
# 单公司化后队列表没有 tenant_id 列;任务里保留默认租户用于日志。
tenant_id=int(row.get("tenant_id") or DEFAULT_TENANT_ID),
sku_code=str(row["sku_code"]).strip(),
expected_arrival_time=row["expected_arrival_time"],
)
+1
View File
@@ -55,6 +55,7 @@ NOTIFICATION_CAPABILITIES: Mapping[str, str] = MappingProxyType(
"product.alert.daily": "命中销量下滑规则且尚未重复通知时投递告警。",
"product.market_rank": "存在本次新生成的平台报告链接且尚未重复通知时投递。",
"product.online_stock_alert.daily": "线上商品当前库存低于 3 时向配置接收人发送库存预警。",
"product.points_exchange.weekly": "每周汇总上一周积分兑换商品排行后向配置接收人发送周报。",
"supply.purchase_confirmation.daily": "采购确认结果生成完成后投递最终业务通知。",
"supply.replenishment.weekly": "补货处理完成后投递最终结果,包括无新增补货项的结果。",
"supply.replenishment_alert.daily": "存在命中库存阈值的 SKU 时投递预警。",
@@ -0,0 +1,74 @@
"""飞书目标表地址唯一来源:dim_style,失败即闭环保,不回退本地 JSON。
2026-10-30 起三个写表脚本(日报 / 天猫主图 / 京东主图)都去掉了
``read_json(bitable_*_map.json)`` 回退——库与本地文件不一致时,回退会让
数据静默写到错误的表,比直接失败更难发现。
"""
from __future__ import annotations
import sys
import insert_bitable_records as daily_insert
import pytest
from scripts import insert_jd_main_image_records as jd_insert
from scripts import insert_main_image_records as tmall_insert
def _raise_config_error() -> dict:
raise RuntimeError("云端动态配置读取失败")
@pytest.mark.parametrize(
"module, attr",
[
(daily_insert, "STYLE_MAP_PATH"),
(tmall_insert, "MAIN_IMAGE_MAP"),
],
)
def test_local_json_target_maps_are_gone(module, attr) -> None:
"""本地 JSON 回退入口必须彻底移除,防止以后被重新接回链路。"""
assert not hasattr(module, attr)
def test_daily_insert_fails_closed_without_style_map(monkeypatch) -> None:
monkeypatch.setattr(daily_insert, "get_bitable_style_map", _raise_config_error)
assert daily_insert.main(["--date", "2026-08-03"]) == 1
def test_daily_insert_fails_closed_on_empty_style_map(monkeypatch) -> None:
monkeypatch.setattr(daily_insert, "get_bitable_style_map", lambda: {})
assert daily_insert.main(["--date", "2026-08-03"]) == 1
def test_tmall_main_image_insert_fails_closed(monkeypatch) -> None:
monkeypatch.setattr(sys, "argv", ["insert_main_image_records.py", "--date", "2026-10-25"])
monkeypatch.setattr(tmall_insert, "get_main_image_styles", lambda: ["某款"])
monkeypatch.setattr(tmall_insert, "get_bitable_main_image_map", _raise_config_error)
assert tmall_insert.main() == 1
def test_tmall_main_image_insert_fails_closed_on_empty_map(monkeypatch) -> None:
monkeypatch.setattr(sys, "argv", ["insert_main_image_records.py", "--date", "2026-10-25"])
monkeypatch.setattr(tmall_insert, "get_main_image_styles", lambda: ["某款"])
monkeypatch.setattr(tmall_insert, "get_bitable_main_image_map", lambda: {})
assert tmall_insert.main() == 1
def test_jd_main_image_insert_fails_closed(monkeypatch) -> None:
monkeypatch.setattr(sys, "argv", ["insert_jd_main_image_records.py", "--date", "2026-10-25"])
monkeypatch.setattr(jd_insert, "get_bitable_main_image_map", _raise_config_error)
assert jd_insert.main() == 1
def test_jd_main_image_insert_fails_closed_on_empty_map(monkeypatch) -> None:
monkeypatch.setattr(sys, "argv", ["insert_jd_main_image_records.py", "--date", "2026-10-25"])
monkeypatch.setattr(jd_insert, "get_bitable_main_image_map", lambda: {})
assert jd_insert.main() == 1
@@ -0,0 +1,151 @@
"""商品日报采集开关(dim_style.collect_enabled,系列级)判定行为。
开关已从独立表 product_daily_collect_scope 迁到 dim_style.collect_enabled:
- 锚点是系列总开关,为 False 时整条系列不采;
- 锚点为 True 时,子款自身的开关只决定它的 ERP 编码是否并入聚合;
- 缺字段默认采集;
- 产品生命进程(另一个项目)只改锚点行即可控制整条系列。
"""
from __future__ import annotations
from gyxx_flow.modules.product_commerce.config.style_config_loader import (
StyleConfigLoader,
)
SALES_URL = "https://example.feishu.cn/base/sales?table=tblSales"
def _row(
style: str,
item_id: str,
*,
platform: str = "天猫",
collect_enabled: bool | None = True,
series_name: str = "",
is_series_anchor: bool = False,
erp_codes: list[str] | None = None,
) -> dict:
row = {
"platform": platform,
"style_name": style,
"erp_codes": erp_codes if erp_codes is not None else [f"ERP-{style}"],
"item_ids": [item_id],
"sales_bitable_url": SALES_URL,
"series_name": series_name,
"is_series_anchor": is_series_anchor,
}
if collect_enabled is not None:
row["collect_enabled"] = collect_enabled
return row
def _loader(rows: list[dict]) -> StyleConfigLoader:
return StyleConfigLoader(records_provider=lambda: rows)
def _two_plain_styles(flag_b: bool | None = True) -> list[dict]:
return [
_row("款A", "TM-1"),
_row("款B", "TM-2", collect_enabled=flag_b),
]
def test_style_flagged_disabled_is_not_collected() -> None:
loader = _loader(_two_plain_styles(flag_b=False))
daily = loader.get_daily_styles()
assert daily["tm"]["collectable"] == ["款A"]
assert {"style": "款B", "reason": "已标记不采集"} in daily["tm"]["skipped"]
assert loader.get_erp_daily_styles()["collectable"] == ["款A"]
def test_missing_flag_defaults_to_enabled() -> None:
"""字段缺失(旧 cmt_styles 回退 / 动态配置独有款)默认采集。"""
loader = _loader(_two_plain_styles(flag_b=None))
assert loader.get_daily_styles()["tm"]["collectable"] == ["款A", "款B"]
assert loader.get_erp_daily_styles()["collectable"] == ["款A", "款B"]
def test_series_anchor_disable_closes_whole_series() -> None:
"""系列级开关:锚点关掉,即使子款自己的 flag 为 True 也不采。"""
rows = [
_row("宙斯", "TM-9", series_name="宙斯", is_series_anchor=True,
collect_enabled=False, erp_codes=[]),
_row("宙斯3", "TM-10", series_name="宙斯", is_series_anchor=False,
collect_enabled=True, erp_codes=["10244"]),
]
loader = _loader(rows)
daily = loader.get_daily_styles()
assert daily["tm"]["collectable"] == []
assert {"style": "宙斯", "reason": "已标记不采集"} in daily["tm"]["skipped"]
assert {"style": "宙斯3", "reason": "已标记不采集"} in daily["tm"]["skipped"]
assert loader.get_erp_daily_styles()["collectable"] == []
def test_series_anchor_enable_wins_over_child_flag() -> None:
"""锚点是总开关:子款 False 不会让整系列停采,但它的 ERP 编码不并入聚合。"""
rows = [
_row("逐星", "TM-11", series_name="逐星", is_series_anchor=True,
collect_enabled=True, erp_codes=[]),
_row("逐星GT", "TM-12", series_name="逐星", is_series_anchor=False,
collect_enabled=False, erp_codes=["10494"]),
]
loader = _loader(rows)
erp_daily = loader.get_erp_daily_styles()
# 子款关闭后编码不进聚合,锚点自己没有编码 → 该系列不出现在采集清单
assert erp_daily["collectable"] == []
assert {"style": "逐星", "reason": "缺 ERP款式编码"} in erp_daily["skipped"]
# 子款仍然让出,不会被自己单独采一次
assert {"style": "逐星GT", "reason": "ERP 已由同系列「逐星」归集采集"} in erp_daily["skipped"]
def test_anchor_keeps_own_codes_when_child_disabled() -> None:
"""锚点 true + 子款 false:只把子款编码排除,锚点自己的编码照常采集。"""
rows = [
_row("逐星", "TM-11", series_name="逐星", is_series_anchor=True,
collect_enabled=True, erp_codes=["10493"]),
_row("逐星GT", "TM-12", series_name="逐星", is_series_anchor=False,
collect_enabled=False, erp_codes=["10494"]),
]
loader = _loader(rows)
payload = loader.load()
assert payload["styles"]["逐星"]["erp_codes"] == ["10493"]
assert payload["styles"]["逐星"]["series"]["erp_excluded"] == ["逐星GT"]
# 子款自己的原始编码保留,供月度销量表按款查询
assert payload["styles"]["逐星GT"]["series"]["own_erp_codes"] == ["10494"]
assert loader.get_erp_daily_styles()["collectable"] == ["逐星"]
def test_series_without_anchor_uses_sorted_first_style() -> None:
"""没有锚点标记时,系列内按款名排序第一个的值作为系列开关。"""
rows = [
_row("B款", "TM-1", series_name="无锚系列", collect_enabled=True),
_row("A款", "TM-2", series_name="无锚系列", collect_enabled=False),
]
loader = _loader(rows)
daily = loader.get_daily_styles()
# A款 排在前,其 False 成为系列值
assert daily["tm"]["collectable"] == []
def test_injected_records_provider_keeps_legacy_behavior() -> None:
loader = StyleConfigLoader(
records_provider=lambda: [_row("款A", "TM-1"), _row("款B", "TM-2")]
)
assert loader.get_daily_styles()["tm"]["collectable"] == ["款A", "款B"]
@@ -43,7 +43,9 @@ def test_style_plan_rejects_one_erp_code_owned_by_two_styles() -> None:
)
def test_style_plan_skips_erp_only_style_until_product_id_is_added() -> None:
def test_style_plan_keeps_erp_only_style() -> None:
"""全店铺日报按 ERP 编码取数,不需要平台商品编码。"""
plans = backfill.build_style_plans(
{
"styles": {
@@ -56,7 +58,44 @@ def test_style_plan_skips_erp_only_style_until_product_id_is_added() -> None:
}
)
assert plans == (backfill.StylePlan("已上架款", ("ERP-READY",)),)
assert plans == (
backfill.StylePlan("研发款", ("ERP-ONLY",)),
backfill.StylePlan("已上架款", ("ERP-READY",)),
)
def test_style_plan_folds_series_codes_into_anchor() -> None:
"""同系列多款:锚点承接并集编码,子款让出重复编码,不报冲突。"""
plans = backfill.build_style_plans(
{
"styles": {
"宙斯": {
"erp_codes": ["10244", "10387"],
"tm": {"item_ids": ["TM-1"]},
"series": {"name": "宙斯", "erp_owner": "宙斯"},
},
"宙斯-10244": {
"erp_codes": ["10244"],
"series": {
"name": "宙斯",
"erp_owner": "宙斯",
"own_erp_codes": ["10244"],
},
},
"宙斯Z": {
"erp_codes": ["10387"],
"series": {
"name": "宙斯",
"erp_owner": "宙斯",
"own_erp_codes": ["10387"],
},
},
}
}
)
assert plans == (backfill.StylePlan("宙斯", ("10244", "10387")),)
def test_all_shop_filter_selects_every_shop_and_batches_all_codes() -> None:
@@ -140,9 +179,15 @@ def test_style_extraction_maps_hidden_columns_via_visible_positions() -> None:
backfill.extract_style_rows(frame, requested_codes=("10416",))
assert "window._Cols" in frame.script
assert "visibleIdx.indexOf" in frame.script
assert "const toPos = index" in frame.script
assert "cells[stylePos]" in frame.script
assert "cells.length <= Math.max(stylePos, salesPos, returnsPos)" in frame.script
# 列布局未知时必须 fail closed,不能退回「按表头索引直取」
assert "column_layout_unknown" in frame.script
# 行 td 数必须与可见列数严格相等
assert "cells.length !== expectedCells" in frame.script
# 值域交叉校验:销量不得超过同行月销量一个量级
assert "sales_out_of_range" in frame.script
assert "month_sales_pos" in frame.script
def test_ensure_custom_columns_visible_only_touches_allsku_frames() -> None:
@@ -150,16 +195,16 @@ def test_ensure_custom_columns_visible_only_touches_allsku_frames() -> None:
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx?r=1"
def __init__(self) -> None:
self.scripts: list[str] = []
self.calls: list[tuple] = []
def evaluate(self, script):
self.scripts.append(script)
return 2
def evaluate(self, script, *args):
self.calls.append((script, *args))
return {"changed": 2, "redrawn": True}
class OtherFrame:
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/filter.aspx"
def evaluate(self, script):
def evaluate(self, script, *args):
raise AssertionError("must not touch non-allsku frames")
class Page:
@@ -170,10 +215,59 @@ def test_ensure_custom_columns_visible_only_touches_allsku_frames() -> None:
assert changed == 2
allsku = page.frames[1]
assert len(allsku.scripts) == 1
assert "'自定义销量'" in allsku.scripts[0]
assert "'自定义退货量'" in allsku.scripts[0]
assert "_RptRefresh" in allsku.scripts[0]
assert len(allsku.calls) == 1
script, options = allsku.calls[0]
assert "'自定义销量'" in script
assert "'自定义退货量'" in script
# 换页重绘才是补齐数据行单元格的手段;_RptRefresh 在未渲染时会抛错。
assert "GotoPage" in script
assert options["redrawPage"] == 1
def test_ensure_custom_columns_rendered_redraws_requested_page() -> None:
class AllskuFrame:
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx?r=1"
def __init__(self) -> None:
self.calls: list[tuple] = []
def evaluate(self, script, *args):
self.calls.append((script, *args))
return {"changed": 2, "redrawn": True}
class Page:
frames = [AllskuFrame()]
page = Page()
summary = backfill.ensure_custom_columns_rendered(page, redraw_page=3)
assert summary == {"changed": 2, "redrawn": 1}
assert page.frames[0].calls[0][1]["redrawPage"] == 3
assert page.frames[0].calls[0][1]["forceRedraw"] is False
def test_ensure_custom_columns_rendered_force_redraw_without_visibility_change() -> None:
class AllskuFrame:
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx?r=1"
def __init__(self) -> None:
self.calls: list[tuple] = []
def evaluate(self, script, *args):
self.calls.append((script, *args))
return {"changed": 0, "redrawn": True}
class Page:
frames = [AllskuFrame()]
page = Page()
summary = backfill.ensure_custom_columns_rendered(
page, redraw_page=2, force_redraw=True
)
assert summary == {"changed": 0, "redrawn": 1}
assert page.frames[0].calls[0][1]["redrawPage"] == 2
assert page.frames[0].calls[0][1]["forceRedraw"] is True
def test_range_collector_queries_once_per_date_not_once_per_style(monkeypatch) -> None:
@@ -256,3 +350,73 @@ def test_schema_keeps_all_shop_metrics_separate_from_platform_metrics() -> None:
assert "CREATE TABLE IF NOT EXISTS erp_all_shop_style_daily_metrics" in schema
assert "PRIMARY KEY (metric_date, style_name)" in schema
assert "return_qty" in schema
def test_daily_total_guard_rejects_month_scale_values(monkeypatch) -> None:
"""单日合计比近 14 天中位数高 8 倍以上必须拒绝落库。"""
monkeypatch.setattr(backfill, "_recent_sales_baseline", lambda *a, **k: 2000.0)
with pytest.raises(RuntimeError, match="looks wrong"):
backfill._guard_daily_total(
date(2026, 10, 29),
[{"sales": 700000, "return_qty": 120000}],
)
# 正常量级放行
backfill._guard_daily_total(
date(2026, 10, 29), [{"sales": 3656, "return_qty": 534}]
)
# --force 时不做量级判断(业务量确实变化的场景)
backfill._guard_daily_total(
date(2026, 10, 29),
[{"sales": 700000, "return_qty": 120000}],
force=True,
)
def test_daily_total_guard_skips_when_no_baseline(monkeypatch) -> None:
monkeypatch.setattr(backfill, "_recent_sales_baseline", lambda *a, **k: None)
backfill._guard_daily_total(
date(2026, 10, 29), [{"sales": 999999, "return_qty": 0}]
)
def test_partial_page_is_treated_as_not_ready(monkeypatch) -> None:
"""非最后一页渲染不满 200 行必须重试,不能当成已完成收下。"""
class Frame:
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx"
def evaluate(self, script, args=None):
return {"ok": True, "rows": [{"sales": 1}] * 174, "signature": "s"}
monkeypatch.setattr(backfill, "_allsku_frame", lambda page: Frame())
monkeypatch.setattr(
backfill, "ensure_custom_columns_rendered", lambda *a, **k: {"changed": 1}
)
with pytest.raises(TimeoutError, match="174/200 rows"):
backfill._wait_rendered_page(
object(), ("10416",), page_no=1, timeout_seconds=1, expect_full_page=True
)
def test_last_page_is_allowed_to_be_partial(monkeypatch) -> None:
class Frame:
url = "https://bi.erp321.com/app/report/subject/itemskuanalysis/allsku.aspx"
def evaluate(self, script, args=None):
return {"ok": True, "rows": [{"sales": 1}] * 59, "signature": "s"}
monkeypatch.setattr(backfill, "_allsku_frame", lambda page: Frame())
monkeypatch.setattr(
backfill, "ensure_custom_columns_rendered", lambda *a, **k: {"changed": 0}
)
result = backfill._wait_rendered_page(
object(), ("10416",), page_no=3, timeout_seconds=5, expect_full_page=False
)
assert result["ok"] is True
assert len(result["rows"]) == 59
@@ -156,9 +156,9 @@ def test_upsert_sales_theme_dedups_and_normalizes(monkeypatch: pytest.MonkeyPatc
def test_business_scope_constants_are_consistent() -> None:
assert len(collector.BRANDS) == 3
assert sorted(collector.BRANDS) == ["gyxx", "光影", "光影行星"]
assert len(collector.SHOP_NAMES) == len(set(collector.SHOP_NAMES)) > 0
# 2026-10-19 起品牌不限(全不勾); 店铺固定 22 家光影系渠道
assert collector.BRANDS == ()
assert len(collector.SHOP_NAMES) == 22 == len(set(collector.SHOP_NAMES))
for name in collector.SHOP_NAMES:
assert name == name.strip()
@@ -274,3 +274,51 @@ def test_parse_files_still_fails_when_both_reports_are_empty(tmp_path: Path) ->
advertiser_pin="光影行星旗舰店霖",
shop_account="光影行星研发",
)
def test_sku_match_prefers_explicit_jd_spus_owner_over_sub_style_name() -> None:
"""子款式名称更长时,仍按 dim_style.jd_spus 归属归到父款式(宙斯系)。"""
result = importer.resolve_sku_match(
"光影行星(GYXX)宙斯3双肩背包男士电脑包2026新款 【宙斯3】曜夜黑/大",
style_catalog=[
{"style_name": "宙斯", "jd_spus": ["spu-zeus3", "spu-zair"]},
{"style_name": "宙斯3", "jd_spus": []},
{"style_name": "宙斯Z Air", "jd_spus": []},
],
full_records=[
{"spu_id": "spu-zeus3", "spu_name": "光影行星(GYXX)宙斯3双肩背包男士电脑包"},
],
)
assert result["status"] == "matched"
assert result["spu_id"] == "spu-zeus3"
assert result["style_name"] == "宙斯"
assert "dim_style.jd_spus" in result["reason"]
def test_sku_match_keeps_style_when_spu_owner_is_itself() -> None:
result = importer.resolve_sku_match(
"光影行星(GYXX)盖亚C1斜挎包 曜夜黑",
style_catalog=[{"style_name": "盖亚斜挎", "jd_spus": ["spu-gy"]}],
full_records=[{"spu_id": "spu-gy", "spu_name": "盖亚C1斜挎包"}],
)
assert result["style_name"] == "盖亚斜挎"
def test_rollup_map_moves_sub_style_to_parent() -> None:
rolled = importer._apply_rollup(
{"status": "matched", "style_name": "宙斯3", "reason": "unique SPU name containment"},
{"宙斯3": "宙斯"},
)
assert rolled["style_name"] == "宙斯"
assert "父款式" in rolled["reason"]
def test_default_rollup_config_declares_zeus_family() -> None:
rollup = importer.load_rollup_map("jd")
assert rollup.get("宙斯3") == "宙斯"
assert rollup.get("宙斯Z Air") == "宙斯"
@@ -96,6 +96,84 @@ class MainImageWorkflowPreflightTests(unittest.TestCase):
jd_runner.load_dynamic_styles()
class MainImageMissingTableAddressTests(unittest.TestCase):
"""缺主图飞书表地址:照常采图入库,只是不写表(不报错、不阻断)。"""
def _write_tmall_payload(self, root: Path, style: str) -> None:
style_dir = root / "2026-08-02" / style
images = style_dir / "tm_images"
images.mkdir(parents=True)
(images / "a.png").write_bytes(b"img")
(style_dir / "wanxiang_creative_main_images.json").write_text(
json.dumps({"creatives": [{"image_key": "a.png", "impressions": 10, "clicks": 2}]}),
encoding="utf-8",
)
def _write_jd_payload(self, root: Path, style: str) -> None:
style_dir = root / "2026-08-02" / style
images = style_dir / "jd_images"
images.mkdir(parents=True)
(images / "a.jpg").write_bytes(b"img")
(style_dir / "jd_main_images.json").write_text(
json.dumps({"records": [{"image_key": "a.jpg", "spu": "JD-1", "impressions": 5, "clicks": 1}]}),
encoding="utf-8",
)
def test_tmall_style_without_address_only_lands_in_postgres(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self._write_tmall_payload(root, "缺地址款")
pg_records: list = []
with (
patch.object(tmall_insert, "artifact_relative_path", return_value="a.png"),
patch.object(tmall_insert, "discover_fields") as discover,
patch.object(tmall_insert, "run_lark") as run_lark,
):
result = tmall_insert.process_product(
"缺地址款",
{"display_name": "缺地址款", "platform": "天猫", "enabled": True},
"2026-08-02",
root,
dry_run=False,
pg_records=pg_records,
)
discover.assert_not_called()
run_lark.assert_not_called()
self.assertEqual(result["found"], 1)
self.assertEqual(result["inserted"], 0)
self.assertEqual(result["errors"], [])
self.assertEqual([r["style_name"] for r in pg_records], ["缺地址款"])
self.assertIsNone(pg_records[0]["bitable_base"])
self.assertFalse(pg_records[0]["uploaded_to_bitable"])
def test_jd_style_without_address_only_lands_in_postgres(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self._write_jd_payload(root, "缺地址款")
pg_records: list = []
with (
patch.object(jd_insert, "artifact_relative_path", return_value="a.jpg"),
patch.object(jd_insert, "discover_fields") as discover,
patch.object(jd_insert, "batch_create_records") as create,
):
result = jd_insert.process_style(
"缺地址款",
{"display_name": "缺地址款", "platform": "京东", "enabled": True},
"2026-08-02",
root,
dry_run=False,
pg_records=pg_records,
)
discover.assert_not_called()
create.assert_not_called()
self.assertEqual(result["found"], 1)
self.assertEqual(result["inserted"], 0)
self.assertEqual([r["style_name"] for r in pg_records], ["缺地址款"])
self.assertIsNone(pg_records[0]["bitable_base"])
class TmallCleanupTests(unittest.TestCase):
def test_locked_duplicate_is_retried_without_console_encoding_failure(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -0,0 +1,242 @@
"""Tests for the weekly points-exchange ranking workflow."""
from __future__ import annotations
import importlib.util
import sys
from datetime import date
from decimal import Decimal
from pathlib import Path
from types import ModuleType, SimpleNamespace
import pytest
PROJECT_ROOT = Path(__file__).resolve().parents[3]
MODULE_PATH = (
PROJECT_ROOT
/ "src"
/ "gyxx_flow"
/ "modules"
/ "product_commerce"
/ "points_exchange_weekly.py"
)
ROWS: list[dict[str, object]] = [
{"itemId": "A", "title": "物资【电源包】", "skuId": "s1", "skuCode": "c1", "quantity": 491},
{"itemId": "A", "title": "物资【电源包】", "skuId": "s2", "skuCode": "c2", "quantity": 472},
{"itemId": "B", "title": "物资【耳机包】", "skuId": "s3", "skuCode": "c3", "quantity": 133},
{
"itemId": "B",
"title": "GYXX-物资【耳机包】",
"skuId": "s4",
"skuCode": "c4",
"quantity": 191,
},
{"itemId": "C", "title": "物资【零兑换】", "skuId": "s5", "skuCode": "c5", "quantity": 0},
]
@pytest.fixture(name="module")
def module_fixture() -> ModuleType:
spec = importlib.util.spec_from_file_location(
"points_exchange_weekly_under_test", MODULE_PATH
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_default_window_is_last_completed_natural_week(
module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-10-26")
start, end = module.resolve_window(None, None)
assert (start, end) == (date(2026, 10, 19), date(2026, 10, 25))
@pytest.mark.parametrize(
("start_value", "end_value", "expected"),
[
("2026-10-19", None, (date(2026, 10, 19), date(2026, 10, 25))),
(None, "2026-10-27", (date(2026, 10, 21), date(2026, 10, 27))),
("2026-10-01", "2026-10-27", (date(2026, 10, 1), date(2026, 10, 27))),
],
)
def test_explicit_window_overrides_the_default(
module: ModuleType,
start_value: str | None,
end_value: str | None,
expected: tuple[date, date],
) -> None:
assert module.resolve_window(start_value, end_value) == expected
def test_window_rejects_reversed_or_oversized_ranges(module: ModuleType) -> None:
with pytest.raises(ValueError, match="before start"):
module.resolve_window("2026-10-27", "2026-10-01")
with pytest.raises(ValueError, match="supported maximum"):
module.resolve_window("2026-01-01", "2026-12-31")
with pytest.raises(ValueError, match="YYYY-MM-DD"):
module.resolve_window("2026/10/01", None)
def test_items_aggregate_every_sku_and_rank_by_quantity(module: ModuleType) -> None:
items = module.aggregate_items(ROWS)
assert [item.item_id for item in items] == ["A", "B"]
assert items[0].quantity == 963
assert items[0].sku_count == 2
assert items[0].title == "物资【电源包】"
assert items[1].quantity == 324
assert items[1].sku_count == 2
assert items[1].title == "GYXX-物资【耳机包】"
assert module._quantity_text(items[0].quantity) == "963"
def test_item_without_identifier_falls_back_to_sku(module: ModuleType) -> None:
items = module.aggregate_items(
[{"title": "物资【无 ID】", "skuCode": "c9", "quantity": 3}]
)
assert items[0].item_id == "sku:c9"
assert items[0].sku_count == 1
def test_non_numeric_quantity_is_rejected(module: ModuleType) -> None:
with pytest.raises(ValueError, match="invalid quantity"):
module.aggregate_items([{"itemId": "A", "quantity": "abc"}])
def test_card_contains_ranked_rows_and_window_summary(module: ModuleType) -> None:
items = module.aggregate_items(ROWS)
card = module._card(
items,
start=date(2026, 10, 19),
end=date(2026, 10, 25),
total_quantity=sum((item.quantity for item in items), Decimal("0")),
item_count=len(items),
page=1,
pages=1,
)
assert card["header"]["title"]["content"] == "私域积分兑换周报"
assert card["header"]["template"] == "blue"
summary = card["body"]["elements"][0]["content"]
assert "2026-10-19 ~ 2026-10-25" in summary
assert "**兑换商品**:2 个" in summary
assert "**兑换总量**:1287" in summary
table = card["body"]["elements"][1]
assert [column["display_name"] for column in table["columns"]] == [
"排名",
"商品",
"SKU 数",
"兑换数量",
]
assert table["rows"][0] == {
"rank": "1",
"title": "物资【电源包】",
"sku_count": "2",
"quantity": "963",
}
assert table["rows"][1]["rank"] == "2"
# Feishu rejects the whole card (ErrCode 200912) when any table column is
# narrower than 80px, so keep every fixed-width column at or above it.
for column in table["columns"]:
width = str(column["width"])
if width.endswith("px"):
assert int(width.removesuffix("px")) >= 80, (
f"Feishu table column {column['name']} is narrower than 80px"
)
def test_run_sends_one_card_to_every_stock_alert_recipient(
module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
sent: list[dict[str, object]] = []
queries: list[dict[str, str]] = []
monkeypatch.setattr(
module,
"query_points_exchange",
lambda **kwargs: queries.append(kwargs) or ROWS,
)
monkeypatch.setattr(
module,
"resolve_notification_route",
lambda workflow_id, defaults: SimpleNamespace(
enabled=True,
open_ids=("ou_aaaaaaaa", "ou_bbbbbbbb"),
app_profile="hermes-analyzer",
),
)
monkeypatch.setattr(
module,
"send_lark_bot_message",
lambda **kwargs: sent.append(kwargs) or {"message_id": "om_1"},
)
assert module.run(["--from", "2026-10-19", "--to", "2026-10-25"]) == 0
assert queries == [
{"start_time": "2026-10-19 00:00:00", "end_time": "2026-10-25 23:59:59"}
]
assert [call["user_id"] for call in sent] == ["ou_aaaaaaaa", "ou_bbbbbbbb"]
assert all(call["msg_type"] == "interactive" for call in sent)
idempotency_keys = {call["idempotency_key"] for call in sent}
assert len(idempotency_keys) == 1
key = idempotency_keys.pop()
assert key.startswith("product.points_exchange.weekly:2026-10-19~2026-10-25:1:")
card = sent[0]["content"]
assert card["header"]["title"]["content"] == "私域积分兑换周报"
assert len(card["body"]["elements"][1]["rows"]) == 2
def test_dry_run_skips_feishu_delivery(
module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
sent: list[dict[str, object]] = []
monkeypatch.setattr(module, "query_points_exchange", lambda **kwargs: ROWS)
monkeypatch.setattr(
module,
"send_lark_bot_message",
lambda **kwargs: sent.append(kwargs) or {"message_id": "om_1"},
)
assert module.run(["--from", "2026-10-19", "--dry-run"]) == 0
assert sent == []
def test_empty_window_does_not_notify(
module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
sent: list[dict[str, object]] = []
monkeypatch.setattr(module, "query_points_exchange", lambda **kwargs: [])
monkeypatch.setattr(
module,
"send_lark_bot_message",
lambda **kwargs: sent.append(kwargs) or {"message_id": "om_1"},
)
assert module.run(["--from", "2026-10-19"]) == 0
assert sent == []
def test_missing_recipients_is_a_hard_failure(
module: ModuleType, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(module, "query_points_exchange", lambda **kwargs: ROWS)
monkeypatch.setattr(
module,
"resolve_notification_route",
lambda workflow_id, defaults: SimpleNamespace(
enabled=True, open_ids=(), app_profile="hermes-analyzer"
),
)
with pytest.raises(RuntimeError, match="没有配置接收人"):
module.run(["--from", "2026-10-19"])
@@ -0,0 +1,384 @@
"""同系列 ERP 编码归集到锚点 + 销量表地址不再挡采集。"""
from __future__ import annotations
from gyxx_flow.modules.product_commerce.config.style_config_loader import (
StyleConfigLoader,
)
SALES_URL = "https://example.feishu.cn/base/sales?table=tblSales"
def _series_rows() -> list[dict]:
"""宙斯系列:锚点只做占位(ERP 空),编码散落在两个子款上。"""
return [
{
"platform": "天猫",
"style_name": "宙斯",
"erp_codes": [],
"item_ids": ["TM-ZS"],
"sales_bitable_url": SALES_URL,
"series_name": "宙斯",
"is_series_anchor": True,
},
{
"platform": "天猫",
"style_name": "宙斯-10244",
"erp_codes": ["10244"],
"item_ids": [],
"series_name": "宙斯",
},
{
"platform": "天猫",
"style_name": "宙斯3",
"erp_codes": ["10484"],
"item_ids": [],
"series_name": "宙斯",
},
]
def _loader(flags: dict[str, bool] | None = None) -> StyleConfigLoader:
def rows() -> list[dict]:
data = _series_rows()
if flags:
for row in data:
row["collect_enabled"] = flags.get(row["style_name"], True)
return data
return StyleConfigLoader(records_provider=rows)
def test_anchor_takes_union_of_series_erp_codes() -> None:
styles = _loader().load()["styles"]
assert styles["宙斯"]["erp_codes"] == ["10244", "10484"]
assert styles["宙斯"]["series"]["erp_owner"] == "宙斯"
def test_child_keeps_own_codes_for_lookup() -> None:
styles = _loader().load()["styles"]
assert styles["宙斯3"]["erp_codes"] == ["10484"]
assert styles["宙斯3"]["series"]["own_erp_codes"] == ["10484"]
assert styles["宙斯3"]["series"]["erp_owner"] == "宙斯"
def test_anchor_is_the_only_erp_collection_unit() -> None:
loader = _loader()
erp_daily = loader.get_erp_daily_styles()
assert erp_daily["collectable"] == ["宙斯"]
assert {"style": "宙斯-10244", "reason": "ERP 已由同系列「宙斯」归集采集"} in erp_daily["skipped"]
assert loader.get_daily_styles()["tm"]["collectable"] == ["宙斯"]
def test_anchor_disabled_closes_whole_series_erp() -> None:
# 系列级开关语义:开关以锚点为准,锚点关掉后子款不会因为自己还有编码
# 就绕过系列开关自行采集(旧的白名单语义下子款会自采)。
loader = _loader({"宙斯": False})
erp_daily = loader.get_erp_daily_styles()
assert erp_daily["collectable"] == []
for name in ("宙斯", "宙斯-10244", "宙斯3"):
assert {"style": name, "reason": "已标记不采集"} in erp_daily["skipped"]
def test_disabled_child_erp_codes_are_excluded_from_anchor() -> None:
"""锚点为 true 时,自身 collect_enabled=false 的子款不并入 ERP 聚合。"""
styles = _loader({"宙斯3": False}).load()["styles"]
assert styles["宙斯"]["erp_codes"] == ["10244"]
assert styles["宙斯"]["series"]["erp_excluded"] == ["宙斯3"]
# 子款保留自己的原始编码,只是不再参与聚合
assert styles["宙斯3"]["series"]["own_erp_codes"] == ["10484"]
def test_enabled_child_keeps_contributing_erp_codes() -> None:
styles = _loader({"宙斯-10244": False}).load()["styles"]
assert styles["宙斯"]["erp_codes"] == ["10484"]
assert styles["宙斯"]["series"]["erp_excluded"] == ["宙斯-10244"]
def test_anchor_without_any_enabled_child_code_stops_erp_collection() -> None:
"""子款全部关闭时锚点没有可采编码,整条系列不采 ERP。"""
erp_daily = _loader({"宙斯-10244": False, "宙斯3": False}).get_erp_daily_styles()
assert erp_daily["collectable"] == []
assert {"style": "宙斯", "reason": "缺 ERP款式编码"} in erp_daily["skipped"]
def test_injected_provider_never_writes_production_cache(tmp_path) -> None:
"""注入假数据的 loader 不能写生产快照,否则数据库抖动时会按假款式采集。"""
cache = tmp_path / ".style_config_cache.json"
loader = StyleConfigLoader(records_provider=_series_rows, cache_path=cache)
loader.load()
assert loader.cache_enabled is False
assert not cache.exists()
def test_default_provider_enables_cache(tmp_path) -> None:
loader = StyleConfigLoader(cache_path=tmp_path / "cache.json")
assert loader.cache_enabled is True
def test_single_style_series_has_no_erp_owner() -> None:
loader = StyleConfigLoader(
records_provider=lambda: [
{
"platform": "天猫",
"style_name": "独苗",
"erp_codes": ["10500"],
"item_ids": ["TM-1"],
"series_name": "独苗",
"is_series_anchor": True,
}
]
)
assert "erp_owner" not in loader.load()["styles"]["独苗"]["series"]
def test_missing_sales_bitable_still_collects_but_skips_table_write() -> None:
"""简行三款的场景:平台编码与 ERP 都齐,只缺销量飞书表。"""
loader = StyleConfigLoader(
records_provider=lambda: [
{
"platform": "天猫",
"style_name": "简行手提",
"erp_codes": ["10523"],
"item_ids": ["TM-1"],
}
]
)
assert loader.get_erp_daily_styles()["collectable"] == ["简行手提"]
assert loader.get_daily_styles()["tm"]["collectable"] == ["简行手提"]
# 写表配置仍然跳过该款:没有销量表地址就不写飞书
assert loader.get_bitable_style_map() == {}
def test_child_platform_ids_are_merged_into_anchor() -> None:
"""逐星GT 的场景:子款自己填的商品 ID 并入锚点,整系列由一个单元采集。"""
loader = StyleConfigLoader(
records_provider=lambda: [
{
"platform": "天猫",
"style_name": "逐星",
"erp_codes": [],
"item_ids": ["TM-A", "TM-B"],
"series_name": "逐星",
"is_series_anchor": True,
},
{
"platform": "天猫",
"style_name": "逐星GT",
"erp_codes": ["10494"],
"item_ids": ["TM-GT"],
"series_name": "逐星",
},
]
)
styles = loader.load()["styles"]
erp_daily = loader.get_erp_daily_styles()
daily = loader.get_daily_styles()
# 子款编码聚合去重后挂锚点,顺序为「锚点原有 → 子款」
assert styles["逐星"]["tm"]["item_ids"] == ["TM-A", "TM-B", "TM-GT"]
# 子款保留原始编码供诊断,但不再单独采集
assert styles["逐星GT"]["series"]["platform_ids"]["tm"] == {
"逐星": ["TM-A", "TM-B", "TM-GT"],
"逐星GT": ["TM-GT"],
}
assert erp_daily["collectable"] == ["逐星"]
assert {"style": "逐星GT", "reason": "ERP 已由同系列「逐星」归集采集"} in erp_daily["skipped"]
assert daily["tm"]["collectable"] == ["逐星"]
assert {
"style": "逐星GT",
"reason": "缺 tm商品编码(同系列「逐星」已在采)",
} in daily["tm"]["skipped"]
def test_platform_ids_filled_on_child_are_lifted_to_anchor() -> None:
"""锚点平台编码为空、子款填了:聚合后由锚点承接,采集口径不变。"""
loader = StyleConfigLoader(
records_provider=lambda: [
{
"platform": "天猫",
"style_name": "某系列",
"erp_codes": ["1001"],
"item_ids": [],
"series_name": "某系列",
"is_series_anchor": True,
},
{
"platform": "天猫",
"style_name": "某系列-2",
"erp_codes": ["1002"],
"item_ids": ["TM-2"],
"series_name": "某系列",
},
]
)
styles = loader.load()["styles"]
daily = loader.get_daily_styles()
assert styles["某系列"]["tm"]["item_ids"] == ["TM-2"]
assert daily["tm"]["collectable"] == ["某系列"]
assert daily["tm"]["skipped"][0]["style"] == "某系列-2"
def test_series_addresses_aggregate_from_child_when_anchor_empty() -> None:
"""子款误填飞书地址、锚点为空时,地址上浮到锚点并下发全系列。"""
loader = StyleConfigLoader(
records_provider=lambda: [
{
"platform": "天猫",
"style_name": "地址系列",
"erp_codes": [],
"item_ids": ["TM-1"],
"series_name": "地址系列",
"is_series_anchor": True,
},
{
"platform": "天猫",
"style_name": "地址系列-2",
"erp_codes": ["1002"],
"item_ids": [],
"sales_bitable_url": SALES_URL,
"series_name": "地址系列",
},
]
)
styles = loader.load()["styles"]
assert styles["地址系列"]["sales_bitable"]["base_token"] == "sales"
assert styles["地址系列-2"]["sales_bitable"]["base_token"] == "sales"
# 地址上浮后锚点成为唯一采集单元,写表清单里只有锚点
assert set(loader.get_bitable_style_map()) == {"地址系列"}
def test_series_address_conflict_keeps_anchor_target(capsys) -> None:
"""同字段多套表:保留锚点那套并告警,避免同系列拆到两张表。"""
loader = StyleConfigLoader(
records_provider=lambda: [
{
"platform": "天猫",
"style_name": "冲突系列",
"erp_codes": ["1001"],
"item_ids": ["TM-1"],
"sales_bitable_url": SALES_URL,
"series_name": "冲突系列",
"is_series_anchor": True,
},
{
"platform": "天猫",
"style_name": "冲突系列-2",
"erp_codes": ["1002"],
"item_ids": [],
"sales_bitable_url": "https://example.feishu.cn/base/child?table=tblChild",
"series_name": "冲突系列",
},
]
)
styles = loader.load()["styles"]
assert styles["冲突系列"]["sales_bitable"]["base_token"] == "sales"
assert styles["冲突系列-2"]["sales_bitable"]["base_token"] == "sales"
assert "个不同表" in capsys.readouterr().out
def test_child_ids_covered_by_anchor_are_left_to_the_anchor() -> None:
"""锚点聚合了全系列商品 ID 时,子款不重复采同一批编码。"""
loader = StyleConfigLoader(
records_provider=lambda: [
{
"platform": "天猫",
"style_name": "聚合系列",
"erp_codes": ["2001"],
"item_ids": ["TM-1", "TM-2"],
"series_name": "聚合系列",
"is_series_anchor": True,
},
{
"platform": "天猫",
"style_name": "聚合系列-2",
"erp_codes": ["2002"],
"item_ids": ["TM-2"],
"series_name": "聚合系列",
},
]
)
daily = loader.get_daily_styles()
assert daily["tm"]["collectable"] == ["聚合系列"]
assert {
"style": "聚合系列-2",
"reason": "缺 tm商品编码(同系列「聚合系列」已在采)",
} in daily["tm"]["skipped"]
def test_anchor_owning_erp_and_all_platform_ids_owns_every_gate() -> None:
"""正常建档形态:锚点填 ERP + 全平台编码,子款只填 ERP。"""
loader = StyleConfigLoader(
records_provider=lambda: [
{
"platform": "天猫",
"style_name": "正常系列",
"erp_codes": ["3001"],
"item_ids": ["TM-1"],
"series_name": "正常系列",
"is_series_anchor": True,
},
{
"platform": "京东旗舰店",
"style_name": "正常系列",
"erp_codes": [],
"item_ids": ["JD-1"],
"series_name": "正常系列",
"is_series_anchor": True,
},
{
"platform": "天猫",
"style_name": "正常系列-2",
"erp_codes": ["3002"],
"item_ids": [],
"series_name": "正常系列",
},
]
)
styles = loader.load()["styles"]
daily = loader.get_daily_styles()
assert styles["正常系列"]["erp_codes"] == ["3001", "3002"]
assert loader.get_erp_daily_styles()["collectable"] == ["正常系列"]
assert daily["tm"]["collectable"] == ["正常系列"]
assert daily["jd"]["collectable"] == ["正常系列"]
assert {
"style": "正常系列-2",
"reason": "缺 tm商品编码(同系列「正常系列」已在采)",
} in daily["tm"]["skipped"]
@@ -62,16 +62,28 @@ def test_database_row_uses_explicit_style_name_not_style_content() -> None:
"brand": "",
"sales_bitable_url": "",
"main_image_bitable_url": "",
"creator_bitable_url": "",
"self_creator_bitable_url": "",
"weekly_note_analysis_url": "",
"persona_bitable_url": "",
"dy_persona_bitable_url": "",
"jd_persona_bitable_url": "",
"style_analysis_bitable_url": "",
"destinations": [],
}
"jd_persona_bitable_url": "",
"style_analysis_bitable_url": "",
"destinations": [],
"series_name": "",
"is_series_anchor": False,
"collect_enabled": True,
}
]
def test_erp_only_style_is_skipped_by_every_product_collection_gate() -> None:
def test_erp_only_style_collects_erp_but_skips_platform_gates() -> None:
"""只有 ERP 编码、没有平台编码的款式:ERP 销量照采,平台流量跳过。
销量表地址与平台编码都不再挡 ERP 采集——缺表地址只影响写飞书,缺平台
编码只影响平台流量。
"""
loader = StyleConfigLoader(
records_provider=lambda: [
{
@@ -89,14 +101,12 @@ def test_erp_only_style_is_skipped_by_every_product_collection_gate() -> None:
erp_daily = loader.get_erp_daily_styles()
assert daily["tm"]["collectable"] == []
assert daily["tm"]["skipped"] == [
{"style": "研发款", "reason": "缺 平台商品编码/tm商品编码"}
assert daily["tm"]["skipped"] == [{"style": "研发款", "reason": "缺 tm商品编码"}]
assert erp_daily["collectable"] == ["研发款"]
assert erp_daily["skipped"] == []
assert loader.get_erp_styles_input() == [
{"style_name": "研发款", "erp_style_codes": ["ERP-ONLY"]}
]
assert erp_daily["collectable"] == []
assert erp_daily["skipped"] == [
{"style": "研发款", "reason": "缺 平台商品编码"}
]
assert loader.get_erp_styles_input() == []
assert loader.get_main_image_styles() == []
@@ -127,3 +137,31 @@ def test_complete_identity_allows_only_configured_platforms() -> None:
assert daily["dy"]["collectable"] == []
assert loader.get_erp_daily_styles()["collectable"] == ["完整款"]
assert loader.get_main_image_styles() == ["完整款"]
def test_main_image_gate_collects_without_table_address() -> None:
"""主图采集只要求身份:缺主图飞书表地址照样采图入库,只是不写表。"""
loader = StyleConfigLoader(
records_provider=lambda: [
{
"platform": "天猫",
"style_name": "有地址款",
"erp_codes": ["ERP-1"],
"item_ids": ["TM-1"],
"main_image_bitable_url": "https://example.feishu.cn/base/image?table=tblImage",
},
{
"platform": "天猫",
"style_name": "缺地址款",
"erp_codes": ["ERP-2"],
"item_ids": ["TM-2"],
},
]
)
# 采集目标:两款都要采(身份齐全即可)
assert sorted(loader.get_main_image_styles()) == ["有地址款", "缺地址款"]
assert loader.get_tm_style_id_map() == {"有地址款": ["TM-1"], "缺地址款": ["TM-2"]}
# 写表目标:只有配了地址的那一款
assert list(loader.get_bitable_main_image_map()) == ["有地址款"]
@@ -234,6 +234,62 @@ def test_erp_style_plan_uses_style_base_codes_and_aliases() -> None:
assert missing == ["云栖相机双肩包"]
def test_erp_style_plan_uses_own_codes_not_merged_series_union() -> None:
"""月度表按款明细:不能照搬 loader 挂在锚点上的系列并集。
采集端把同系列编码聚合成并集交给锚点当唯一取数单元;报表若也用这份
并集,锚点和子款会把同一批编码各查一遍,销量重复计入。
"""
targets = [
ProductTarget("宙斯", 100),
ProductTarget("宙斯pro", 200),
ProductTarget("星迹2", 300),
ProductTarget("星迹2lite", 400),
]
series = {"name": "宙斯", "erp_owner": "宙斯"}
styles = {
# 锚点只做系列占位、自身没填编码:loader 聚合后 erp_codes 是系列并集
"宙斯": {
"erp_codes": ["10244", "10276", "10387"],
"series": {**series, "own_erp_codes": []},
},
"宙斯pro": {
"erp_codes": ["10276"],
"series": {**series, "own_erp_codes": ["10276"]},
},
# 正常建档形态:锚点一条编码,子款各一条
"星迹2": {
"erp_codes": ["10394", "10504"],
"series": {
"name": "星迹2",
"erp_owner": "星迹2",
"own_erp_codes": ["10394"],
},
},
"星迹2lite": {
"erp_codes": ["10504"],
"series": {
"name": "星迹2",
"erp_owner": "星迹2",
"own_erp_codes": ["10504"],
},
},
}
plans, missing = plan_erp_styles(targets, styles, {})
assert plans == [
ErpStylePlan("宙斯pro", ("10276",)),
ErpStylePlan("星迹2", ("10394",)),
ErpStylePlan("星迹2lite", ("10504",)),
]
# 锚点自身没编码:按 0 写入(--allow-missing-erp-as-zero),不算配置缺失
assert missing == ["宙斯"]
planned = [code for plan in plans for code in plan.erp_style_codes]
assert len(planned) == len(set(planned))
def test_execute_guard_rejects_incomplete_erp_mapping_before_sheet_write() -> None:
with pytest.raises(RuntimeError, match="目标表未清空"):
require_complete_erp_mappings(["云栖相机双肩包", "轻风双肩包"])
+2 -1
View File
@@ -34,7 +34,7 @@ def test_catalog_composition_registers_all_workflows() -> None:
registry = create_default_registry(catalog=catalog)
assert len(registry.workflow_ids) == 39
assert len(registry.workflow_ids) == 40
assert set(registry.workflow_ids) == {
"shop.metrics.weekly",
"shop.competitor.weekly",
@@ -64,6 +64,7 @@ def test_catalog_composition_registers_all_workflows() -> None:
"product.persona.daily",
"product.daily",
"product.online_stock_alert.daily",
"product.points_exchange.weekly",
"product.hk_anpei_public_inventory.daily",
"product.alert.daily",
"product.ecommerce_costs.daily",
+6 -6
View File
@@ -11,15 +11,15 @@ from gyxx_flow.catalog import CatalogError, WorkflowCatalog
PROJECT_ROOT = Path(__file__).resolve().parents[1]
def test_catalog_maps_39_project_scheduled_tasks() -> None:
def test_catalog_maps_40_project_scheduled_tasks() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
scheduled = catalog.scheduled_workflows()
assert len(scheduled) == 39
assert len({item.source_task_name for item in scheduled}) == 39
assert len(scheduled) == 40
assert len({item.source_task_name for item in scheduled}) == 40
assert Counter(item.module for item in scheduled) == {
"content_marketing": 9,
"product_commerce": 14,
"product_commerce": 15,
"shop_intelligence": 12,
"supply_chain": 4,
}
@@ -29,7 +29,7 @@ def test_catalog_schedules_douyin_price_appeal() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
schedule = catalog.schedule_for("shop.douyin_price_appeal")
assert len(catalog.workflows) == 39
assert len(catalog.workflows) == 40
assert catalog.manual_workflows() == ()
assert "shop.douyin_price_appeal" in {
schedule.workflow_id for schedule in catalog.schedules
@@ -185,7 +185,7 @@ def test_every_scheduled_workflow_has_one_valid_asia_shanghai_schedule() -> None
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
assert catalog.timezone == "Asia/Shanghai"
assert len(catalog.schedules) == 39
assert len(catalog.schedules) == 40
for workflow in catalog.scheduled_workflows():
schedule = catalog.schedule_for(workflow.workflow_id)
assert schedule.workflow_id == workflow.workflow_id
+1 -1
View File
@@ -54,7 +54,7 @@ def test_default_cli_registry_exposes_all_project_scheduled_workflows(
scheduled_ids = {
entry.workflow_id for entry in registry.catalog.scheduled_workflows()
}
assert len(scheduled_ids) == 39
assert len(scheduled_ids) == 40
assert scheduled_ids == {
schedule.workflow_id for schedule in registry.catalog.schedules
}
+1
View File
@@ -39,6 +39,7 @@ def test_repository_keeps_the_simplified_public_structure() -> None:
"workflow-acceptance-test-report.md",
"workflow-acceptance-test-requirements.md",
"workflow-console.md",
"workflows-overview.md",
"workflow-time-distribution.md",
}
assert not {
+12 -1
View File
@@ -80,8 +80,19 @@ def test_baseline_contains_all_provided_people_and_routable_capabilities(
assert mobiles["wang_yunlong"] == "+8613663785145"
assert mobiles["stock_alert_user_19832771651"] == "+8619832771651"
assert set(payload["capabilities"]) == set(NOTIFICATION_CAPABILITIES)
assert set(payload["routes"]) == {"product.online_stock_alert.daily"}
assert set(payload["routes"]) == {
"product.online_stock_alert.daily",
"product.points_exchange.weekly",
}
assert payload["routes"]["product.online_stock_alert.daily"]["enabled"] is True
exchange_route = payload["routes"]["product.points_exchange.weekly"]
assert exchange_route["enabled"] is True
assert exchange_route["person_ids"] == payload["routes"][
"product.online_stock_alert.daily"
]["person_ids"]
assert exchange_route["app_profile"] == payload["routes"][
"product.online_stock_alert.daily"
]["app_profile"]
assert len(revision) == 64
wang = payload["people"]["wang_yunlong"]
-1
View File
@@ -15,7 +15,6 @@ from gyxx_flow.ops.audit import (
)
from gyxx_flow.ops.index import RunIndex
DOUYIN_COOKIES: list[dict[str, Any]] = [
{
"name": "sessionid",
+1 -1
View File
@@ -24,7 +24,7 @@ def test_product_module_registers_scheduled_and_manual_entries() -> None:
WorkflowCatalog.load(PROJECT_ROOT / "config")
)
assert tuple(item.workflow_id for item in module.workflow_definitions()) == PRODUCT_WORKFLOW_IDS
assert len(PRODUCT_SCHEDULED_WORKFLOW_IDS) == 14
assert len(PRODUCT_SCHEDULED_WORKFLOW_IDS) == 15
assert PRODUCT_MANUAL_WORKFLOW_IDS == ()
+3 -3
View File
@@ -53,10 +53,10 @@ def test_catalog_and_scripts_work_after_project_and_data_roots_move(
assert workflows.returncode == 0, workflows.stderr
assert scripts.returncode == 0, scripts.stderr
assert len(json.loads(workflows.stdout)) == 39
assert len(json.loads(workflows.stdout)) == 40
commands = json.loads(scripts.stdout)
assert len(commands) == 60
assert len({row["command_id"] for row in commands}) == 60
assert len(commands) == 62
assert len({row["command_id"] for row in commands}) == 62
assert all(row["legacy_script_id"].startswith(f"{row['module']}:") for row in commands)
+3 -3
View File
@@ -102,10 +102,10 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
data_root=project_root / "var",
)
assert len(scripts.command_ids) == 61
assert len(first.command_ids) == 156
assert len(scripts.command_ids) == 62
assert len(first.command_ids) == 157
bindings = [first.binding_for(command_id) for command_id in first.command_ids]
assert len({binding.cdp_port for binding in bindings}) == 156
assert len({binding.cdp_port for binding in bindings}) == 157
assert all(22000 <= binding.cdp_port <= 22999 for binding in bindings)
assert all(binding.cdp_url.startswith("http://127.0.0.1:") for binding in bindings)
assert bindings == [second.binding_for(item.script_id) for item in bindings]
+1 -1
View File
@@ -54,7 +54,7 @@ def test_python_scheduler_status_is_project_owned_and_cross_platform(tmp_path: P
assert exit_code == 0
payload = json.loads(output.getvalue())
assert payload["timezone"] == "Asia/Shanghai"
assert payload["scheduled_count"] == 39
assert payload["scheduled_count"] == 40
assert payload["state"]["schema_version"] == 1
+2 -1
View File
@@ -206,7 +206,7 @@ def test_misfire_older_than_grace_is_not_launched(tmp_path: Path) -> None:
def test_configured_schedules_disable_cookie_refresh_job() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
assert len(catalog.schedules) == 39
assert len(catalog.schedules) == 40
assert [
schedule.workflow_id for schedule in catalog.schedules if not schedule.enabled
] == [
@@ -219,6 +219,7 @@ def test_configured_schedules_disable_cookie_refresh_job() -> None:
"shop.tmall_customer_service.daily",
"product.alert.daily",
"product.style_analysis.interval",
"supply.replenishment_alert.daily",
"supply.purchase_confirmation.daily",
"supply.replenishment.weekly",
]
+4 -4
View File
@@ -80,10 +80,10 @@ def test_default_registry_covers_every_executable_workflow_entry() -> None:
for entry in entries:
workflow_entries.add(f"{workflow['module']}:{entry}")
assert scheduled_workflows == 39
assert scheduled_workflows == 40
assert manual_workflows == 0
assert len(workflow_entries) == 55
assert len(catalog.scripts) == 61
assert len(workflow_entries) == 56
assert len(catalog.scripts) == 62
assert workflow_entries < set(catalog.script_ids)
assert {
"content.mapping.rebuild",
@@ -280,7 +280,7 @@ def test_cli_lists_and_dry_runs_registered_commands(tmp_path: Path) -> None:
assert main(["scripts", "list", "--json"], settings=settings, stdout=output) == 0
rows = json.loads(output.getvalue())
assert len(rows) == 61
assert len(rows) == 62
assert {row["module"] for row in rows} == {
"content_marketing",
"product_commerce",
+1 -1
View File
@@ -47,7 +47,7 @@ def test_deployment_documents_one_scheduler_and_external_data_root() -> None:
assert "GYXX_DATA_ROOT" in guide
assert "/var/lib/gyxx-flow" in guide
assert "Windows Task Scheduler" in guide
assert len(payload["schedules"]) == 39
assert len(payload["schedules"]) == 40
assert sum(item.get("enabled", True) for item in payload["schedules"]) == 28
+1 -1
View File
@@ -57,6 +57,6 @@ def test_every_workflow_and_manual_script_resolves_inside_new_project() -> None:
assert target.is_relative_to(PROJECT_ROOT)
commands = ScriptCatalog.load_default()
assert len(commands.scripts) == 60
assert len(commands.scripts) == 62
assert len(commands.command_ids) == len(set(commands.command_ids))
assert all(command.path.is_relative_to(PROJECT_ROOT) for command in commands.scripts)
@@ -286,6 +286,7 @@ def test_queue_claims_rows_with_skip_locked_and_maps_tasks() -> None:
queue = ArrivalSyncQueue(
{"GYXX_ARRIVAL_SYNC_OPERATOR": "worker"},
connect_factory=lambda: connection,
tenant_column=True,
)
tasks = queue.claim(tenant_id=1, stale_after_seconds=60)
@@ -307,7 +308,10 @@ def test_queue_claims_rows_with_skip_locked_and_maps_tasks() -> None:
def test_queue_completion_is_fenced_by_the_claimed_target_timestamp() -> None:
connection = _Connection()
queue = ArrivalSyncQueue(connect_factory=lambda: connection)
queue = ArrivalSyncQueue(
connect_factory=lambda: connection,
tenant_column=True,
)
task = ArrivalSyncTask(
id=8,
tenant_id=1,
@@ -358,3 +362,76 @@ def test_cdp_session_is_checked_before_optional_password_validation(
)
assert manager.login_with_browser() is True
def test_single_company_queue_omits_tenant_filters() -> None:
"""单公司化后队列表没有 tenant_id,认领与回写都不能再带租户条件。"""
connection = _Connection(
claim_rows=[
{
"id": 9,
"sku_code": "SKU-9",
"expected_arrival_time": datetime(2026, 10, 19, 9, 0),
}
]
)
queue = ArrivalSyncQueue(
{"GYXX_ARRIVAL_SYNC_OPERATOR": "worker"},
connect_factory=lambda: connection,
tenant_column=False,
)
tasks = queue.claim(tenant_id=1, stale_after_seconds=60)
claim_sql, claim_params = connection.calls[0]
assert "tenant_id" not in claim_sql
assert claim_params == (60, "worker")
assert tasks == [
ArrivalSyncTask(
id=9,
tenant_id=1,
sku_code="SKU-9",
expected_arrival_time=datetime(2026, 10, 19, 9, 0),
)
]
assert queue.mark_success(tasks[0]) is True
success_sql, success_params = connection.calls[-1]
assert "tenant_id" not in success_sql
assert success_params == ("worker", 9, datetime(2026, 10, 19, 9, 0))
def test_queue_schema_probe_falls_back_when_tenant_column_absent() -> None:
class _ProbeCursor:
def __enter__(self) -> "_ProbeCursor":
return self
def __exit__(self, *_args: object) -> None:
return None
@staticmethod
def execute(_sql: str, _params: tuple[object, ...]) -> None:
return None
@staticmethod
def fetchone() -> None:
return None
class _ProbeConnection:
def __enter__(self) -> "_ProbeConnection":
return self
def __exit__(self, *_args: object) -> None:
return None
@contextmanager
def cursor(self, **_kwargs: object):
yield _ProbeCursor()
queue = ArrivalSyncQueue(
connect_factory=lambda: _ProbeConnection(),
tenant_column=None,
)
assert queue._has_tenant_column() is False
+137 -3
View File
@@ -161,7 +161,9 @@ def test_normalize_style_config_owns_erp_once_and_platform_ids_per_child() -> No
]
def test_runtime_records_use_lifecycle_identity_and_keep_dynamic_targets() -> None:
def test_runtime_records_use_lifecycle_identity_and_lifecycle_targets() -> None:
"""地址以 dim_style 为准:动态配置里的旧地址不再覆盖建档值。"""
dynamic_records = [
{
"style_name": "款式甲",
@@ -185,6 +187,8 @@ def test_runtime_records_use_lifecycle_identity_and_keep_dynamic_targets() -> No
"jd_self": [],
"dy": [],
},
"sales_bitable_url": "https://example.feishu.cn/base/newsales?table=tblNewSales",
"tm_persona_bitable_url": "tm-persona-new",
}
]
@@ -196,8 +200,138 @@ def test_runtime_records_use_lifecycle_identity_and_keep_dynamic_targets() -> No
assert by_platform["京东自营"]["item_ids"] == []
assert by_platform["抖音"]["item_ids"] == []
assert all(row["erp_codes"] == ["NEW-ERP"] for row in records)
assert by_platform["天猫"]["sales_bitable_url"].endswith("table=tblSales")
assert by_platform["天猫"]["tm_persona_bitable_url"] == "tm-persona"
assert by_platform["天猫"]["sales_bitable_url"].endswith("table=tblNewSales")
assert by_platform["天猫"]["tm_persona_bitable_url"] == "tm-persona-new"
def test_dynamic_targets_are_ignored_when_lifecycle_has_none() -> None:
"""建档地址为空时不再回退旧表:飞书地址只有 dim_style 一个来源。"""
records = merge_lifecycle_style_records(
[
{
"style_name": "款式乙",
"erp_codes": ["ERP-B"],
"platform": "天猫",
"item_ids": ["TM-B"],
"sales_bitable_url": "https://example.feishu.cn/base/legacy?table=tblLegacy",
"main_image_bitable_url": "https://example.feishu.cn/base/legacy?table=tblLegacyMain",
"creator_bitable_url": "https://example.feishu.cn/base/legacy?table=tblLegacyCreator",
}
],
[
{
"id": 9,
"name": "款式乙",
"erp_style_codes": ["ERP-B"],
"platform_product_ids": {"tm": ["TM-B"], "jd": [], "jd_self": [], "dy": []},
}
],
)
row = next(r for r in records if r["platform"] == "天猫")
assert row["sales_bitable_url"] == ""
assert row["main_image_bitable_url"] == ""
assert row["creator_bitable_url"] == ""
def test_series_targets_propagate_from_anchor() -> None:
"""锚点的 9 类地址下发到同系列子款;子款自身的旧地址被覆盖。"""
lifecycle_rows = [
{
"id": 1,
"name": "逐星",
"erp_style_codes": ["E-ANCHOR"],
"series_name": "逐星",
"is_series_anchor": True,
"platform_product_ids": {"tm": ["TM-A"], "jd": [], "jd_self": [], "dy": []},
"sales_bitable_url": "https://example.feishu.cn/base/anchor?table=tblAnchorSales",
"main_image_bitable_url": "https://example.feishu.cn/base/anchor?table=tblAnchorMain",
"creator_bitable_url": "https://example.feishu.cn/base/anchor?table=tblAnchorCreator",
},
{
"id": 2,
"name": "逐星GT",
"erp_style_codes": ["E-CHILD"],
"series_name": "逐星",
"is_series_anchor": False,
"platform_product_ids": {"tm": ["TM-C"], "jd": [], "jd_self": [], "dy": []},
"sales_bitable_url": "https://example.feishu.cn/base/child?table=tblChildSales",
},
]
records = merge_lifecycle_style_records([], lifecycle_rows)
child = next(r for r in records if r["style_name"] == "逐星GT")
assert "anchor" in child["sales_bitable_url"]
assert "anchor" in child["main_image_bitable_url"]
assert "anchor" in child["creator_bitable_url"]
def test_series_targets_aggregate_from_child_when_anchor_empty() -> None:
"""子款误填地址、锚点为空:聚合上浮到锚点并下发整条系列,不丢配置。"""
lifecycle_rows = [
{
"id": 1,
"name": "云栖",
"erp_style_codes": ["E-ANCHOR"],
"series_name": "云栖",
"is_series_anchor": True,
"platform_product_ids": {"tm": ["TM-A"], "jd": [], "jd_self": [], "dy": []},
},
{
"id": 2,
"name": "云栖标准版",
"erp_style_codes": ["E-CHILD"],
"series_name": "云栖",
"is_series_anchor": False,
"platform_product_ids": {"tm": [], "jd": [], "jd_self": [], "dy": []},
"sales_bitable_url": "https://example.feishu.cn/base/child?table=tblChildSales",
"main_image_bitable_url": "https://example.feishu.cn/base/child?table=tblChildMain",
},
]
records = merge_lifecycle_style_records([], lifecycle_rows)
anchor_sales = {r["sales_bitable_url"] for r in records if r["style_name"] == "云栖"}
child_sales = {r["sales_bitable_url"] for r in records if r["style_name"] == "云栖标准版"}
# 子款地址上浮到锚点,并统一下发到整条系列
assert anchor_sales == {"https://example.feishu.cn/base/child?table=tblChildSales"}
assert child_sales == anchor_sales
def test_series_targets_conflict_keeps_anchor_address(capsys) -> None:
"""同一字段多张表:保留锚点那条并告警,避免同系列拆到两个表。"""
lifecycle_rows = [
{
"id": 1,
"name": "冲突款",
"erp_style_codes": ["E-ANCHOR"],
"series_name": "冲突系列",
"is_series_anchor": True,
"platform_product_ids": {"tm": ["TM-A"], "jd": [], "jd_self": [], "dy": []},
"sales_bitable_url": "https://example.feishu.cn/base/anchor?table=tblAnchorSales",
},
{
"id": 2,
"name": "冲突款-2",
"erp_style_codes": ["E-CHILD"],
"series_name": "冲突系列",
"is_series_anchor": False,
"platform_product_ids": {"tm": [], "jd": [], "jd_self": [], "dy": []},
"sales_bitable_url": "https://example.feishu.cn/base/child?table=tblChildSales",
},
]
records = merge_lifecycle_style_records([], lifecycle_rows)
assert {r["sales_bitable_url"] for r in records} == {
"https://example.feishu.cn/base/anchor?table=tblAnchorSales"
}
assert "个不同地址" in capsys.readouterr().out
def test_runtime_records_empty_lifecycle_ids_clear_legacy_platform_ids() -> None: