diff --git a/.gitignore b/.gitignore index 634e972..a0b501e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,5 +40,8 @@ Thumbs.db # Local tool artifacts /.playwright-cli/ /output/ + +# Local dsh workbench scaffold (generated at runtime, bundles node_modules) +/workbench/ /nul /new_all_bag.xlsx diff --git a/config/commands.json b/config/commands.json index c4ba062..f9ed91f 100644 --- a/config/commands.json +++ b/config/commands.json @@ -274,6 +274,17 @@ "kind": "python", "notification_workflow_id": "product.online_stock_alert.daily" }, + { + "id": "product.points_exchange_weekly.run", + "module": "product_commerce", + "entry": "points_exchange_weekly.py", + "kind": "python", + "default_args": [ + "--from", + "{business_date}" + ], + "notification_workflow_id": "product.points_exchange.weekly" + }, { "id": "product.import.daily", "module": "product_commerce", diff --git a/config/notification-routing.json b/config/notification-routing.json index 7cff130..4776379 100644 --- a/config/notification-routing.json +++ b/config/notification-routing.json @@ -231,6 +231,9 @@ "product.online_stock_alert.daily": { "condition": "线上商品当前库存低于 3 时向配置接收人发送库存预警。" }, + "product.points_exchange.weekly": { + "condition": "每周汇总上一周积分兑换商品排行后向配置接收人发送周报。" + }, "supply.purchase_confirmation.daily": { "condition": "采购确认结果生成完成后投递最终业务通知。" }, @@ -250,6 +253,15 @@ "wang_yunlong", "stock_alert_user_19832771651" ] + }, + "product.points_exchange.weekly": { + "enabled": true, + "app_profile": "hermes-analyzer", + "person_ids": [ + "online_stock_alert_recipient", + "wang_yunlong", + "stock_alert_user_19832771651" + ] } } } diff --git a/config/runtime-bindings.json b/config/runtime-bindings.json index b6b33e0..fca16b3 100644 --- a/config/runtime-bindings.json +++ b/config/runtime-bindings.json @@ -1119,6 +1119,15 @@ ], "cdp_port": 22163 }, + "product.points_exchange_weekly.run": { + "script_id": "product_commerce:points_exchange_weekly.py", + "state_key": "product_commerce:points_exchange_weekly.py", + "aliases": [ + "product_commerce:points_exchange_weekly.py", + "product.points_exchange_weekly.run" + ], + "cdp_port": 22168 + }, "product_commerce:dy_audience_profile_collect.py": { "script_id": "product_commerce:dy_audience_profile_collect.py", "state_key": "product_commerce:dy_audience_profile_collect.py", diff --git a/config/schedules.json b/config/schedules.json index 8352b81..152d75f 100644 --- a/config/schedules.json +++ b/config/schedules.json @@ -39,7 +39,7 @@ "kind": "daily", "workflow_id": "content.marketing_report.daily" }, - { + { "at": "12:00", "days": [ "Sunday" @@ -197,6 +197,16 @@ "kind": "daily", "workflow_id": "product.online_stock_alert.daily" }, + { + "at": "09:45", + "business_date_offset_days": -7, + "days": [ + "Monday" + ], + "enabled": true, + "kind": "weekly", + "workflow_id": "product.points_exchange.weekly" + }, { "at": "08:40", "business_date_offset_days": -1, @@ -325,6 +335,8 @@ }, { "at": "07:00", + "business_date_offset_days": 0, + "enabled": false, "kind": "daily", "workflow_id": "supply.replenishment_alert.daily" }, diff --git a/config/source-manifests/product_commerce.json b/config/source-manifests/product_commerce.json index 0f5c0e8..7bceb55 100644 --- a/config/source-manifests/product_commerce.json +++ b/config/source-manifests/product_commerce.json @@ -70,7 +70,7 @@ { "category": "production_source", "source_sha256": "c7b54d941ab7f056e2d0ea4355543ff4a5c34ad986c8ed32145e988ac9501ea2", - "target_sha256": "8576678688d7d90f6f7285110de005982bc1ba82bc7e9977224ba1bdb4101283", + "target_sha256": "b306195f9fd90d6fba3c8fd9335677fe8dcfa1653604c8ace9f3cc340f43a5dc", "transformed": true, "source_relative_path": "collect_erp_yesterday_metrics.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py" @@ -126,7 +126,7 @@ { "category": "production_source", "source_sha256": "22dcc9d8b2f1cf598a62e4c71df765a952250694cb26b9a73f477c5ca1a29884", - "target_sha256": "ffeaf697a1a58b21520fda3f3b0936c591d05b328537c41f2048366a9ba1b8d5", + "target_sha256": "ce328f87c883257c91025708209338ba6ee34d3c284a149c1969aeae85ffbdb7", "transformed": true, "source_relative_path": "config/style_config_loader.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/config/style_config_loader.py" @@ -134,7 +134,7 @@ { "category": "production_source", "source_sha256": "a8900b9f25db6cf2bef3117a1d2cd86283054fea6f5a8590fd91b5cfbbcaf87e", - "target_sha256": "47cdfd9ebf27e31be06497b2f74c80da4ffced259a873730eb024b0db5eba740", + "target_sha256": "a34efe965b312fe02509a4955cba835f7cdc081daa8a1624d571a8f46bf9380e", "transformed": true, "source_relative_path": "db/__init__.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/db/__init__.py" @@ -150,7 +150,7 @@ { "category": "production_source", "source_sha256": "492916088fa5e8c170461cc9eba97a88ed1cc1e50c7b99e5f6a5e49167fea157", - "target_sha256": "7b0ae5d648245e89f4aacf262250009de812d3dbb8ee55386f3704cabca48799", + "target_sha256": "855feba2cfd51bbbb70b2fc912a7b5b517f91c3fa47ff5129ba05e2f7ffa329c", "transformed": true, "source_relative_path": "dy_product_scraping.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/dy_product_scraping.py" @@ -206,7 +206,7 @@ { "category": "production_source", "source_sha256": "057a01e8a22cdd2d1806d290da207209954af89f7da6339e9cdcef3bc0297e63", - "target_sha256": "0360bd218db1625088663b8ba509b5f53edd09563b71823006f2d46cdd5367ae", + "target_sha256": "815c00d039e4f81e4f9deae95891fab288c6584e72d21b73f4b6f348e158e126", "transformed": true, "source_relative_path": "insert_bitable_records.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/insert_bitable_records.py" @@ -214,7 +214,7 @@ { "category": "production_source", "source_sha256": "e0f016bafb3f527f1cae46eaf3ef31510184128aaaa7f52024a7c3f978eec42b", - "target_sha256": "b157f24430170d5a991bb2283af0a747b26dd11c0f0dec37289774f115132fee", + "target_sha256": "cef210f479d45e4b92cc597ddde0d80e62e37b1546c3ee27099ad7ad9c71d108", "transformed": true, "source_relative_path": "jd_main_image_collector.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_main_image_collector.py" @@ -222,7 +222,7 @@ { "category": "production_source", "source_sha256": "d99ce71a008128549ac83fc3da52d88540b74538e3fe5c85d47c3b985e414353", - "target_sha256": "cfe98b322e4ef7bd875325c2a9ffedeabb364e98b2983773dc98f9e0e75a6581", + "target_sha256": "a597578741795926844b93fa6b74bd8316e1c72cac5196f9dbc2e492e76d88e3", "transformed": true, "source_relative_path": "jd_product_data_collector.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py" @@ -358,7 +358,7 @@ { "category": "production_source", "source_sha256": "af9b436a31c3454dc6e0b34b77876962dab0b477e8a561db49911fdfc24eade3", - "target_sha256": "e3b91d3c4b529291f7a6ceee5ff99326351d78104a66babc531f90faec6b7e1a", + "target_sha256": "096834b64e4b8e5120e38a116efb4c6eb02440d7c4567bf4d6783c18aee0050d", "transformed": true, "source_relative_path": "run_weekly_main_image.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/run_weekly_main_image.py" @@ -366,7 +366,7 @@ { "category": "production_source", "source_sha256": "34d935efb8dff20afd3ab89ef150c5f8db9cee80c6256c7fe4ef0bae2f72ed63", - "target_sha256": "787a207a2c8174518a6465587644937ad6d11b1d86b07488e1c86b07e1410171", + "target_sha256": "3fdaa7e23e96897c2039dcd1cdc6c6fc4ee1fb9cff828262e2360d453d778f47", "transformed": true, "source_relative_path": "scripts/insert_jd_main_image_records.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/scripts/insert_jd_main_image_records.py" @@ -374,7 +374,7 @@ { "category": "production_source", "source_sha256": "a1b097ead2a3f7e20ca63d7c15828a8d04030b0d6722934c3bf65d9e975fba76", - "target_sha256": "7d8587f77615040087d2d475a7b440ad16a39c5874e143418dd700ecbc5db0f8", + "target_sha256": "6c2ef5acbfff574a9f7460ebaccc8c4e46a3ccbd8b72944791e0fded69038e77", "transformed": true, "source_relative_path": "scripts/insert_main_image_records.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/scripts/insert_main_image_records.py" @@ -390,7 +390,7 @@ { "category": "production_source", "source_sha256": "4474dd7d68117247af87e20bbeb6b7c072368ef0f606de5d3e186f459d34a0f5", - "target_sha256": "862ae9392227113ad4266c2170769595c36b2f98fdecc54e54a648ba834baebf", + "target_sha256": "9e87faad21459b8ac79c335734e11b1da636d34747f05e80c7073b66f01e35f1", "transformed": true, "source_relative_path": "taobao_sycm_collect.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_sycm_collect.py" @@ -414,7 +414,7 @@ { "category": "production_source", "source_sha256": "3f0c32c443c87417c43881da69aa7776828ba973465f8d32f50b3dc698b0a4fe", - "target_sha256": "548df6ddf3af3b3c066d18687114f1ce672afa054535f6da0c254d60f8306b86", + "target_sha256": "6363ec2a70a22b078af23a70fcdda6e3cce9f95eb4aa7ebc54f4cb85a293cb6d", "transformed": true, "source_relative_path": "taobao_wanxiang_ai_creative_report.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py" @@ -544,7 +544,7 @@ { "category": "runtime_resource", "source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746", - "target_sha256": "6bbd36048836ef7ed0807223fc028ac17349b20314b1ea353ccedb451957ba55", + "target_sha256": "0ea747eb7e40875338bebe7fc5b8d2524869e55ba1b503a322b37c857b274d8e", "transformed": true, "source_relative_path": "db/schema.sql", "target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql" @@ -656,7 +656,7 @@ { "category": "regression_test", "source_sha256": "820e17b910b62a213f0bebafcff72998c2c978307c97f976c074276150ce7c6a", - "target_sha256": "cdedf8c9a4871ad763e0a9098fc18777f0c3206991e8d440a45ccdfbda089a07", + "target_sha256": "77221bd2bfb9e2314585ba9bbaea770093a86cfe5918ffe9c67fc1fadba3b04a", "transformed": true, "source_relative_path": "tests/test_main_image_concurrency.py", "target_relative_path": "tests/modules/product_commerce/test_main_image_concurrency.py" @@ -720,7 +720,7 @@ { "category": "regression_test", "source_sha256": "17761a5ba93a5387ffa67662424955178058aeec708bbcd3eb0341a859d8a1f1", - "target_sha256": "549cb13dd32c9bed76d78eafa7542ea6e5c638d9f29c857eff4dde6d3b8e9c40", + "target_sha256": "edc906fdf2ab2fab731fdcf56dd8ad7cf272949a8b4f66fb71c1693de5822f1f", "transformed": true, "source_relative_path": "tests/test_style_config_loader_erp_source.py", "target_relative_path": "tests/modules/product_commerce/test_style_config_loader_erp_source.py" diff --git a/config/workflows.json b/config/workflows.json index b08bdcc..b127681 100644 --- a/config/workflows.json +++ b/config/workflows.json @@ -1126,7 +1126,7 @@ "30", "--headless" ], - "timeout_seconds": 14400, + "timeout_seconds": 28800, "replay_policy": "idempotent" } ] @@ -1180,7 +1180,7 @@ "{business_date}", "--end-date", "{business_date}", - "--headless" + "--headed" ], "timeout_seconds": 14400, "replay_policy": "idempotent" @@ -1705,6 +1705,51 @@ "task_name": "OnlineItemStockAlert" } }, + { + "id": "product.points_exchange.weekly", + "module": "product_commerce", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "query_rank_notify", + "name": "查询积分兑换明细并按商品生成兑换排行周报", + "description": "调用企业数据交互积分兑换查询接口(startTime/endTime 覆盖整周自然日),按商品汇总 quantity 兑换数量并从高到低排名;默认统计上一个完整自然周(周一至周日),将排行榜以飞书卡片发送给私域库存预警的同一批接收人。", + "replay_policy": "idempotent", + "data_flow": { + "sources": [ + { + "system": "企业数据交互接口", + "label": "积分兑换明细", + "detail": "/open/data/v1/points-exchange/query,startTime 为窗口首日 00:00:00、endTime 为窗口末日 23:59:59" + } + ], + "processing": [ + "以业务日期作为窗口起点,缺省统计 7 个自然日", + "按 itemId 汇总 quantity,并保留商品名称与 SKU 数", + "按兑换数量从高到低排序并分段组织飞书消息" + ], + "destinations": [ + { + "system": "飞书", + "label": "积分兑换商品排行周报", + "condition": "窗口内存在兑换数量大于 0 的商品" + } + ] + }, + "entry": "points_exchange_weekly.py", + "args": [ + "--from", + "{business_date}" + ] + } + ] + }, + "provenance": { + "source_project": "data-hub", + "task_name": "PointsExchangeWeeklyRanking" + } + }, { "id": "product.ecommerce_costs.daily", "module": "product_commerce", @@ -2835,7 +2880,7 @@ "source_project": "supply", "task_name": "auto-flow-replenishment-arrival-sync" }, - "note": "队列表为空时成功结束;失败行保留 failed 状态,下一次调度自动重试。默认租户为 1,可用 GYXX_ERP_TENANT_ID 覆盖。" + "note": "队列表为空时成功结束;失败行保留 failed 状态,下一次调度自动重试。data-hub 单公司化后队列表已无 tenant_id:脚本会先探测实际表结构,有该列仍按 GYXX_ERP_TENANT_ID(默认 1)过滤,没有则按单公司口径直接认领。" }, { "id": "product.market_rank", diff --git a/docs/dynamic-workflow-config.md b/docs/dynamic-workflow-config.md index a4e75b0..bdac6c8 100644 --- a/docs/dynamic-workflow-config.md +++ b/docs/dynamic-workflow-config.md @@ -128,9 +128,51 @@ workflow_dynamic_styles + workflow_dynamic_style_platforms (目标/画像/启停 └─ 目标兼容输出 → feishu_mapping → 内容营销工作流 → 各业务飞书目标表 ``` -系列共享规则:ERP 款式编码逐款必填;平台商品编码与销量飞书表整系列只需主款填写, -子款通过系列归属继承销量表,但**不复制平台编码**——同一个商品 ID 被多款重复采集会把 -访客/加购重复入账。 +系列共享规则:平台商品编码、ERP 编码与 9 类飞书地址都以「系列」为一个采集单元, +子款留空即可;子款误填也不会丢——loader 会把整条系列聚合并去重后挂到锚点。 + +- **平台编码聚合去重到锚点**:整条系列各平台的商品 ID 先并集去重,再挂到锚点,由锚点 + 一个采集单元去抓。子款的原始编码保留在 `series.platform_ids` 供诊断,采集口判定让出 + (「缺 tm商品编码(同系列「X」已在采)」),同一个商品 ID 不会被采两次、访客与加购 + 不会重复入账。锚点不在采集范围内时子款仍可回退自采。 +- **ERP 编码先聚合再采集**:正常建档形态是锚点填一条款式编码 + 各平台商品编码,子款只填 + 自己的款式编码;也可能出现锚点 `erp_style_codes` 为空、编码全散落在子款上的形态。 + loader 一律把整条系列的 ERP 编码聚合成并集挂到锚点(`series.erp_owner`),锚点作为唯一 + ERP 采集单元去聚水潭取数,子款让出并记录「ERP 已由同系列 X 归集采集」,但仍保留自己的 + 原始编码(`series.own_erp_codes`)供按款查询与月度销量表使用。 + **例外(2026-10-30)**:子款自身 `collect_enabled = FALSE` 时,它的 ERP 编码不并入聚合 + (`series.erp_excluded` 可见),锚点不代采;锚点自身 `FALSE` 仍是整系列停采。 + 平台商品编码不受子款开关影响,仍按整系列聚合。 +- **飞书地址聚合去重下发**:`workflow_config._propagate_series_targets()` 与 + `_apply_series_sharing()` 两处都按「锚点优先、其余按款名」收集整系列非空地址,去重后 + 下发到每一行。一个表位只能落一张表,同字段出现多个不同地址时保留优先级最高的一条并打 + `[WARN] 系列「X」<字段> 有 N 个不同表`。子款想用独立一套表,必须拆成独立系列。 +- **月度销量表按款明细**:`product.sales_sheet.daily` 的报表口径**不用**锚点上的聚合并集, + 而是每个款式只查自己建档时填的编码。采集端的并集是给聚水潭当唯一取数单元用的,报表 + 照搬会让同系列每个成员行各查一遍并集、销量重复计入。锚点自身没编码时该行按 0 写入 + (`--allow-missing-erp-as-zero`),同系列销量落在各子款行上,合计与采集总量一致。 + +采集门只校验身份:ERP 日采要求 ERP 款式编码,平台日采再叠加对应平台商品编码。 +**销量飞书表地址不再是采集前置条件**——缺表地址的款式照常采集并落库,只是写表环节按 +地址缺失跳过(`get_bitable_style_map` 不生成该款目标),不影响数据入库与后续分析。 +主图采集同理:`get_main_image_styles` 只要求身份齐全,**主图飞书表地址不参与采集门**, +缺地址的款照常抓图并写 `main_image_creatives`,`get_bitable_main_image_map` 只返回有地址 +的款,写表脚本对没有地址的款打 `[NO-TABLE]` 跳过飞书、仍入库。 + +**写表脚本对目标地址 fail closed(2026-10-30)**:`insert_bitable_records.py`、 +`scripts/insert_main_image_records.py`、`scripts/insert_jd_main_image_records.py` 读取 +`get_bitable_style_map()` / `get_bitable_main_image_map()` 失败或返回空时,直接打 +`[FAIL] …拒绝回退本地 JSON` 并退出码 1,**不再回退** `bitable_style_map.json` / +`bitable_main_image_map.json`。库与本地文件不一致时,回退会让数据静默写到错误的表, +比直接失败更难发现。`styles_input.json` 同理由 `collect_erp_yesterday_metrics.load_erp_styles` +与 `export_bitable_records.load_style_meta` 兜住:身份源不可用以 `lifecycle_unavailable` 失败。 + +**旧表只保留「行集合 + 启停」职责**:`workflow_dynamic_styles` / `_platforms` 仍决定有哪些 +款式、`enabled` 启停、`note`、`style_content`、`destinations`,但 ERP 编码、平台商品编码与 +9 类地址全部由 `dim_style` 覆盖(`merge_lifecycle_style_records` 中 `managed` 由 +`jsonb_build_object` 恒生成四个键而恒为真)。旧表里**不在 dim_style 的残留行**(孤儿款) +地址会被清空、不再采集,属于建档改名/合并后的历史垃圾,应直接删除 +(2026-10-30 已清理 `拓界mode`、`逐星evo`,备份 `var/tmp/workflow_dynamic_orphans_20261030.json`)。 `StyleConfigLoader` 只在数据库短暂不可用时使用最近一次数据库成功读取后生成的本地缓存; `feishu_mapping` 的目标表字段缓存仍保留,用来减少对各业务飞书表的字段查询。 diff --git a/docs/runbook.md b/docs/runbook.md index 3879bbb..055bb2a 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -286,6 +286,105 @@ uv run gyxx effects reconcile \ 浏览器状态异常时,只备份并处理目标脚本的 `state/browser//