From 485e77ee7405fe255b991c94af377a34ad676b45 Mon Sep 17 00:00:00 2001 From: wangyunlong <3080433063@qq.com> Date: Mon, 7 Sep 2026 15:01:04 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20[workflow]=20=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E9=A3=9E=E4=B9=A6=E6=AC=BE=E5=BC=8F=E8=BA=AB=E4=BB=BD=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/dynamic-workflow-config.md | 73 +++- src/gyxx_flow/adapters/workflow_config.py | 269 +++++++++++++- .../migration/feishu_style_config_seed.py | 20 +- .../migration/sync_feishu_style_identity.py | 347 ++++++++++++++++++ .../backfill_erp_all_shop_daily.py | 9 +- .../collect_erp_yesterday_metrics.py | 77 ++-- .../config/style_config_loader.py | 125 ++++--- .../modules/product_commerce/db/__init__.py | 2 +- .../modules/product_commerce/db/schema.sql | 13 +- .../product_commerce/db/sync_dim_style.py | 84 ++++- .../test_style_config_loader_erp_source.py | 65 +++- tests/test_sync_feishu_style_identity.py | 102 +++++ tests/test_workflow_dynamic_config.py | 102 +++++ 13 files changed, 1153 insertions(+), 135 deletions(-) create mode 100644 src/gyxx_flow/migration/sync_feishu_style_identity.py create mode 100644 tests/test_sync_feishu_style_identity.py diff --git a/docs/dynamic-workflow-config.md b/docs/dynamic-workflow-config.md index 3db85fe..a0218ea 100644 --- a/docs/dynamic-workflow-config.md +++ b/docs/dynamic-workflow-config.md @@ -2,11 +2,17 @@ ## 配置源 -商品 ID、ERP 款式编码和各业务飞书目标表现在统一保存在云端 PostgreSQL。 -控制台“配置中心 → 款式平台配置”提供查询、新增、编辑、 -停用和删除。工作流运行时只读取这些项目表，不再读取旧配置主表 +商品身份现在统一由产品生命进程维护，目标地址和平台画像等运行参数仍保存在云端 +PostgreSQL 的动态配置表。生命进程负责款式名、品牌、ERP 款式编码、各平台商品编码 +以及大/小 NFC 数量；控制台“配置中心 → 款式平台配置”继续负责目标地址、画像表和启停状态。 +工作流运行时将两者合并，不再读取旧配置主表 `TtoCb1NuQaDy3NsZWTpc0GIvnph/tblKCjplVAFrRwMC`。 +产品采集的统一身份门是：ERP 款式编码 + 至少一个平台商品编码。只有 ERP 款式编码的 +研发新款会保留在生命进程中，但不会进入商品采集；如果指定平台没有自己的商品编码， +该平台也不会触发采集。内容汇总类工作流只需要款式名和对应目标，不因缺少商品编码而被 +这道商品采集门拦截。 + 目标飞书表仍是工作流的业务输入或输出，例如销量表、主图表、人群画像表和合作达人表； 本次下线的是集中维护这些地址和商品 ID 的旧飞书索引表，不是业务目标表本身。 @@ -38,21 +44,25 @@ | 层级 | 字段 | 使用目的 | | --- | --- | --- | -| 款式 | 款式名、品牌、ERP 款式编码 | 统一业务身份；ERP 日采、月度销量表和款式主数据使用 | -| 款式 | 销量表、主图表 | 商品日报和天猫/京东主图工作流的款式目标表 | -| 款式 | 合作达人表、自营合作达人表 | 内容采集、合作同步和笔记清单使用 | -| 款式 | 每周/每月笔记分析、平台单品分析 | 内容周月报和款式周期分析使用 | -| 平台 | 平台名、商品 ID、启用状态 | 天猫商品 ID、京东 SPU、京东自营 SKU、抖音/PDD 商品 ID | +| 产品生命进程 | 款式名、品牌、ERP 款式编码 | 统一业务身份；ERP 日采、月度销量表和款式主数据使用 | +| 产品生命进程 | `platform_product_ids` | JSONB 固定键 `tm`、`jd`、`jd_self`、`dy`，值为去重后的字符串编码数组 | +| 产品生命进程 | 大 NFC 数、小 NFC 数 | 产品规格记录，默认值为 1，允许填写 0 | +| 动态配置父表 | 销量表、主图表 | 商品日报和天猫/京东主图工作流的款式目标表 | +| 动态配置父表 | 合作达人表、自营合作达人表 | 内容采集、合作同步和笔记清单使用 | +| 动态配置父表 | 每周/每月笔记分析、平台单品分析 | 内容周月报和款式周期分析使用 | +| 动态配置子表 | 平台名、启用状态和画像目标 | 商品 ID 只保留兼容快照；运行时商品 ID 由生命进程覆盖，平台停用状态仍然生效 | | 平台 | 本平台人群画像表 | 天猫、京东、抖音画像采集分别写入自己的目标表 | 数据库使用父子表： -- `workflow_dynamic_styles`：一个款式一行，保存 ERP 和共享飞书目标。 -- `workflow_dynamic_style_platforms`：一个款式可有多个平台子行，只保存平台商品 ID、平台画像表和启停状态。 +- `cmt_styles`：产品生命进程主表，一个款式一行，保存 ERP 编码、四个平台商品编码和 NFC 数量。 +- `workflow_dynamic_styles`：一个款式一行，保存共享飞书目标及内容工作流参数。 +- `workflow_dynamic_style_platforms`：一个款式可有多个平台子行，保留平台画像表、启停状态和旧商品 ID 兼容快照。 - 旧 `workflow_dynamic_configs` 保留为原始迁移审计，不再参与运行时读取。 - `erp_codes`、`item_ids` 使用 PostgreSQL 数组，页面接受中英文逗号或换行并自动去重。 - `destinations` 使用通用 JSONB 目标列表，每项包含 `key`、`label`、`url`、`description`、`enabled`；六个旧目标字段仍同步保留，保证已有工作流兼容。后续新增分析逻辑只需增加一个稳定的 `key` 和对应飞书表地址。 - 款式带递增 `revision`；整页保存和删除必须提交 `If-Match`，避免并发覆盖。 +- `cmt_styles.platform_product_ids` 的四个键固定为 `tm`（天猫）、`jd`（京东旗舰店）、`jd_self`（京东自营）、`dy`（抖音）；空数组表示该平台尚未补齐。 ## 首次迁移 @@ -65,17 +75,52 @@ 4. 运行 v2 归一化迁移：按款式合并共享字段，按平台合并商品 ID；旧表中集中在天猫行的三平台画像地址分别迁入对应平台子行； 5. 写入迁移标记 `workflow-style-platform-config-20260819-v2`，此后不会重复迁移。 +产品生命进程身份迁移随后执行 `product-lifecycle-style-identity-20260907-v1`： + +1. 为 `cmt_styles` 补齐平台商品编码和大/小 NFC 字段； +2. 将已有 `dim_style` 的 ERP、天猫、京东、京东自营、抖音编码补入生命进程的空字段； +3. 对仍没有生命进程商品编码的旧动态配置做一次兼容提升； +4. 运行时以生命进程中的非空 JSONB 固定键为准，显式空数组会清除旧动态配置中的对应编码。 + +因此新增款式可先只填写 ERP 款式编码，等平台商品编码补齐后再进入对应采集；不会因为动态配置表 +中残留旧编码而提前采集。 + +### 从旧飞书表重新校准生命进程 + +如果需要用现有“各平台款式 ID 收集表”补齐或校准生命进程身份，使用一次性同步脚本： + +```powershell +uv run --env-file 'D:/product-collector-analyze-flow/.env' ` + --project 'D:/gyxx-flow' ` + python -m gyxx_flow.migration.sync_feishu_style_identity + +uv run --env-file 'D:/product-collector-analyze-flow/.env' ` + --project 'D:/gyxx-flow' ` + python -m gyxx_flow.migration.sync_feishu_style_identity --apply + +uv run --env-file 'D:/product-collector-analyze-flow/.env' ` + --project 'D:/gyxx-flow' ` + python 'src/gyxx_flow/modules/product_commerce/db/sync_dim_style.py' +``` + +脚本按款式合并旧表的多平台行，只写入 `cmt_styles` 的品牌、ERP 编码和四个平台商品编码， +不会覆盖 NFC、图片、类目或市场标签；空源字段不会清空生命进程已有身份。拼多多等当前四键 +模型之外的平台只保留在动态平台配置/审计中，不会伪装成天猫、京东或抖音编码。默认是预览， +必须显式传 `--apply` 才会写数据库。 + 服务器必须注入 `GYXX_POSTGRES_DSN`，或完整的 `PG_HOST/PG_PORT/PG_DB/PG_USER/PG_PASSWORD`。 没有数据库且没有数据库生成的运行缓存时，相关工作流会明确失败，不会静默回读旧飞书主表。 ## 运行时路径 ```text -控制台分组 CRUD +产品生命进程 cmt_styles (商品身份) + + +workflow_dynamic_styles + workflow_dynamic_style_platforms (目标/画像/启停) ↓ -workflow_dynamic_styles + workflow_dynamic_style_platforms (PostgreSQL) - ├─ 兼容聚合输出 → StyleConfigLoader → 商品经营工作流 - └─ 兼容聚合输出 → feishu_mapping → 内容营销工作流 → 各业务飞书目标表 +统一运行时聚合 + ├─ 完整身份门 → StyleConfigLoader → 商品经营工作流 + └─ 目标兼容输出 → feishu_mapping → 内容营销工作流 → 各业务飞书目标表 ``` `StyleConfigLoader` 只在数据库短暂不可用时使用最近一次数据库成功读取后生成的本地缓存； diff --git a/src/gyxx_flow/adapters/workflow_config.py b/src/gyxx_flow/adapters/workflow_config.py index 24978b4..9fa3c04 100644 --- a/src/gyxx_flow/adapters/workflow_config.py +++ b/src/gyxx_flow/adapters/workflow_config.py @@ -26,6 +26,7 @@ from gyxx_flow.adapters.integration import ( MIGRATION_KEY = "feishu-style-config-20260819-v1" NORMALIZED_MIGRATION_KEY = "workflow-style-platform-config-20260819-v2" PERSONA_PLATFORM_MIGRATION_KEY = "workflow-style-platform-persona-20260819-v3" +LIFECYCLE_IDENTITY_MIGRATION_KEY = "product-lifecycle-style-identity-20260907-v1" DEFAULT_SEED_PATH = ( Path(__file__).resolve().parents[3] / "config" @@ -77,6 +78,13 @@ KNOWN_PLATFORMS = ( "拼多多光影行星官方旗舰店", "光影行星GYXX箱包专卖店", ) +LIFECYCLE_PLATFORM_LABELS = { + "tm": "天猫", + "jd": "京东旗舰店", + "jd_self": "京东自营", + "dy": "抖音", +} +LIFECYCLE_PLATFORM_KEYS = tuple(LIFECYCLE_PLATFORM_LABELS) STYLE_FIELDS = ( "style_name", @@ -252,7 +260,7 @@ class WorkflowConfigRepository(Protocol): def _clean_list(value: Any) -> list[str]: if value is None: return [] - parts = value if isinstance(value, list) else re.split(r"[,，\r\n]", str(value)) + parts = value if isinstance(value, (list, tuple, set)) else re.split(r"[,，\r\n]", str(value)) result: list[str] = [] seen: set[str] = set() for item in parts: @@ -263,6 +271,155 @@ def _clean_list(value: Any) -> list[str]: return result +def _decode_lifecycle_product_ids(value: Any) -> tuple[bool, dict[str, list[str]]]: + """返回 (是否由生命进程接管, 各平台商品编码)。 + + 老数据的 JSONB 可能仍是空对象，此时允许动态配置作为迁移兼容来源； + 生命进程保存的对象始终带有四个固定键，即使某个平台为空数组也要视为空值生效。 + """ + + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + value = None + if not isinstance(value, Mapping): + return False, {key: [] for key in LIFECYCLE_PLATFORM_KEYS} + managed = any(key in value for key in LIFECYCLE_PLATFORM_KEYS) + return managed, { + key: _clean_list(value.get(key)) for key in LIFECYCLE_PLATFORM_KEYS + } + + +def merge_lifecycle_style_records( + dynamic_records: list[Mapping[str, Any]], + lifecycle_rows: list[Mapping[str, Any]], +) -> list[dict[str, Any]]: + """Overlay lifecycle product identity on legacy target configuration rows. + + ``workflow_dynamic_*`` still owns Feishu target URLs and persona targets, while + ``cmt_styles`` owns ERP codes and product IDs. The result intentionally keeps + the old flat runtime contract so existing collectors need no per-workflow fork. + """ + + dynamic_by_style: dict[str, list[Mapping[str, Any]]] = {} + for record in dynamic_records: + style_name = " ".join(str(record.get("style_name") or "").split()) + if style_name: + dynamic_by_style.setdefault(style_name, []).append(record) + + lifecycle_by_style: dict[str, Mapping[str, Any]] = {} + lifecycle_order: list[str] = [] + for row in lifecycle_rows: + style_name = " ".join(str(row.get("name") or "").split()) + if not style_name or style_name in lifecycle_by_style: + continue + lifecycle_by_style[style_name] = row + lifecycle_order.append(style_name) + + style_order = list(dynamic_by_style) + style_order.extend(name for name in lifecycle_order if name not in dynamic_by_style) + records: list[dict[str, Any]] = [] + + for style_name in style_order: + legacy_rows = dynamic_by_style.get(style_name, []) + lifecycle = lifecycle_by_style.get(style_name) + if lifecycle is None: + records.extend(dict(row) for row in legacy_rows) + continue + + managed, lifecycle_ids = _decode_lifecycle_product_ids( + lifecycle.get("platform_product_ids") + ) + lifecycle_erp = _clean_list(lifecycle.get("erp_style_codes")) + legacy_erp = _clean_list(legacy_rows[0].get("erp_codes")) if legacy_rows else [] + erp_codes = lifecycle_erp if lifecycle_erp or managed else legacy_erp + brand = str(lifecycle.get("brand") or "").strip() + if not brand and legacy_rows: + brand = str(legacy_rows[0].get("brand") or "").strip() + + template = dict(legacy_rows[0]) if legacy_rows else { + "style_name": str(lifecycle.get("name") or style_name), + "brand": brand, + "erp_codes": erp_codes, + "note": "", + "sales_bitable_url": "", + "main_image_bitable_url": "", + "creator_bitable_url": "", + "tm_persona_bitable_url": "", + "jd_persona_bitable_url": "", + "dy_persona_bitable_url": "", + "self_creator_bitable_url": "", + "weekly_note_analysis_url": "", + "style_analysis_bitable_url": "", + "style_content": "", + "destinations": [], + "enabled": True, + } + + dynamic_ids: dict[str, list[str]] = {key: [] for key in LIFECYCLE_PLATFORM_KEYS} + dynamic_unknown_ids: dict[str, list[str]] = {} + dynamic_labels: list[str] = [] + for row in legacy_rows: + if row.get("enabled", True) is False: + continue + label = str(row.get("platform") or "").strip() + if label and label not in dynamic_labels: + dynamic_labels.append(label) + key = next( + (key for key, known_label in LIFECYCLE_PLATFORM_LABELS.items() if known_label == label), + None, + ) + if key: + for item_id in _clean_list(row.get("item_ids")): + if item_id not in dynamic_ids[key]: + dynamic_ids[key].append(item_id) + elif label: + for item_id in _clean_list(row.get("item_ids")): + if item_id not in dynamic_unknown_ids.setdefault(label, []): + dynamic_unknown_ids[label].append(item_id) + + if managed: + labels = [] + for label in LIFECYCLE_PLATFORM_LABELS.values(): + platform_rows = [ + row for row in legacy_rows + if str(row.get("platform") or "").strip() == label + ] + # 没有动态平台行时保持生命进程身份可被读取；一旦存在平台行， + # 则尊重该行的启停状态，避免停用后又被固定平台列表重新打开。 + if not platform_rows or any( + row.get("enabled", True) is not False for row in platform_rows + ): + labels.append(label) + else: + labels = list(dynamic_labels) + labels.extend(label for label in dynamic_labels if label not in labels) + + for label in labels: + key = next( + (key for key, known_label in LIFECYCLE_PLATFORM_LABELS.items() if known_label == label), + None, + ) + output = dict(template) + output["enabled"] = True + output["style_name"] = str(lifecycle.get("name") or style_name).strip() + output["brand"] = brand + output["erp_codes"] = list(erp_codes) + output["platform"] = label + output["item_ids"] = ( + list(lifecycle_ids[key]) + if managed and key + else list(dynamic_ids.get(key, dynamic_unknown_ids.get(label, []))) + ) + output["source_record_id"] = ( + f"cmt_style:{lifecycle.get('id')}" if key and managed else output.get("source_record_id") + ) + records.append(output) + + return records + + def _plain_url(value: Any) -> str: text = str(value or "").strip() match = re.search(r"\]\((https?://[^)]+)\)", text) @@ -579,6 +736,13 @@ class PostgresWorkflowConfigStore: ) if cursor.fetchone() is None: self._preserve_persona_platforms(cursor) + cursor.execute( + "SELECT 1 FROM workflow_dynamic_config_migrations " + "WHERE migration_key = %s", + (LIFECYCLE_IDENTITY_MIGRATION_KEY,), + ) + if cursor.fetchone() is None: + self._bootstrap_lifecycle_identity(cursor) conn.commit() self._initialized = True except WorkflowConfigError: @@ -779,6 +943,88 @@ class PostgresWorkflowConfigStore: (PERSONA_PLATFORM_MIGRATION_KEY, NORMALIZED_MIGRATION_KEY, preserved), ) + def _bootstrap_lifecycle_identity(self, cursor: psycopg.Cursor[Any]) -> None: + """Promote the last dynamic-config identity snapshot to cmt_styles once.""" + + cursor.execute("SELECT to_regclass('public.cmt_styles') AS table_name") + table = cursor.fetchone() or {} + if not table.get("table_name"): + # Content schema is installed separately in some environments. Leave the + # marker absent so the promotion can retry after that schema is available. + return + + cursor.execute( + "ALTER TABLE cmt_styles " + "ADD COLUMN IF NOT EXISTS brand VARCHAR(128) DEFAULT '', " + "ADD COLUMN IF NOT EXISTS erp_style_codes TEXT[] DEFAULT '{}', " + "ADD COLUMN IF NOT EXISTS platform_product_ids JSONB DEFAULT '{}'::jsonb" + ) + cursor.execute( + "SELECT s.style_name, s.brand, s.erp_codes, p.platform, p.item_ids " + "FROM workflow_dynamic_styles s " + "JOIN workflow_dynamic_style_platforms p ON p.style_id = s.id " + "WHERE s.enabled AND p.enabled ORDER BY s.style_name, p.id" + ) + grouped: dict[str, dict[str, Any]] = {} + platform_keys = { + label: key for key, label in LIFECYCLE_PLATFORM_LABELS.items() + } + for row in cursor.fetchall(): + style_name = " ".join(str(row.get("style_name") or "").split()) + if not style_name: + continue + item = grouped.setdefault( + style_name, + { + "brand": str(row.get("brand") or "").strip(), + "erp_codes": _clean_list(row.get("erp_codes")), + "platform_product_ids": { + key: [] for key in LIFECYCLE_PLATFORM_KEYS + }, + }, + ) + key = platform_keys.get(str(row.get("platform") or "").strip()) + if not key: + continue + for product_id in _clean_list(row.get("item_ids")): + if product_id not in item["platform_product_ids"][key]: + item["platform_product_ids"][key].append(product_id) + + for style_name, values in grouped.items(): + cursor.execute( + "INSERT INTO cmt_styles " + "(name, brand, erp_style_codes, platform_product_ids) " + "VALUES (%s, %s, %s, %s::jsonb) " + "ON CONFLICT (name) DO UPDATE SET " + "brand = CASE WHEN COALESCE(BTRIM(cmt_styles.brand), '') = '' " + "THEN EXCLUDED.brand ELSE cmt_styles.brand END, " + "erp_style_codes = CASE " + "WHEN COALESCE(array_length(cmt_styles.erp_style_codes, 1), 0) = 0 " + "AND COALESCE(cmt_styles.platform_product_ids, '{}'::jsonb) = '{}'::jsonb " + "THEN EXCLUDED.erp_style_codes ELSE cmt_styles.erp_style_codes END, " + "platform_product_ids = CASE " + "WHEN COALESCE(cmt_styles.platform_product_ids, '{}'::jsonb) = '{}'::jsonb " + "THEN EXCLUDED.platform_product_ids " + "ELSE cmt_styles.platform_product_ids END, " + "updated_at = NOW()", + ( + style_name, + values["brand"], + values["erp_codes"], + json.dumps(values["platform_product_ids"], ensure_ascii=False), + ), + ) + cursor.execute( + "INSERT INTO workflow_dynamic_config_migrations " + "(migration_key, source, record_count) VALUES (%s, %s, %s) " + "ON CONFLICT (migration_key) DO NOTHING", + ( + LIFECYCLE_IDENTITY_MIGRATION_KEY, + "workflow_dynamic_styles + workflow_dynamic_style_platforms", + len(grouped), + ), + ) + @staticmethod def _public_style( row: Mapping[str, Any], platforms: list[Mapping[str, Any]] @@ -905,14 +1151,27 @@ class PostgresWorkflowConfigStore: with conn.cursor() as cursor: cursor.execute( "SELECT s.*, p.id AS platform_id, p.platform, p.item_ids, " - "p.persona_bitable_url, p.source_record_ids " + "p.persona_bitable_url, p.source_record_ids, " + "p.enabled AS platform_enabled " "FROM workflow_dynamic_styles s " "JOIN workflow_dynamic_style_platforms p ON p.style_id = s.id " - "WHERE s.enabled AND p.enabled ORDER BY s.style_name, p.id" + "WHERE s.enabled ORDER BY s.style_name, p.id" ) rows = cursor.fetchall() + cursor.execute("SELECT to_regclass('public.cmt_styles') AS table_name") + cmt_table = cursor.fetchone() or {} + if cmt_table.get("table_name"): + cursor.execute( + "SELECT id, name, brand, erp_style_codes, platform_product_ids " + "FROM cmt_styles ORDER BY name, id" + ) + lifecycle_rows = cursor.fetchall() + else: + lifecycle_rows = [] personas: dict[int, dict[str, str]] = {} for row in rows: + if row.get("platform_enabled") is False: + continue style_personas = personas.setdefault(int(row["id"]), {}) style_personas[str(row["platform"])] = str( row.get("persona_bitable_url") or "" @@ -943,10 +1202,10 @@ class PostgresWorkflowConfigStore: "style_analysis_bitable_url": row.get("style_analysis_bitable_url", ""), "style_content": row.get("style_content", ""), "destinations": _public_destinations(row), - "enabled": True, + "enabled": row.get("platform_enabled") is not False, } ) - return records + return merge_lifecycle_style_records(records, lifecycle_rows) def create(self, payload: Mapping[str, Any]) -> dict[str, Any]: values = normalize_style_config(payload) diff --git a/src/gyxx_flow/migration/feishu_style_config_seed.py b/src/gyxx_flow/migration/feishu_style_config_seed.py index 30b1bd3..8c2d308 100644 --- a/src/gyxx_flow/migration/feishu_style_config_seed.py +++ b/src/gyxx_flow/migration/feishu_style_config_seed.py @@ -79,7 +79,19 @@ def _lark_cli() -> str: return executable -def read_source_records() -> list[dict[str, Any]]: +def read_source_records( + *, + base_token: str = BASE_TOKEN, + table_id: str = TABLE_ID, + view_id: str = VIEW_ID, +) -> list[dict[str, Any]]: + """Read all rows from the legacy Feishu configuration table. + + The defaults preserve the original seed export behavior. Callers that + need to audit a particular shared view can pass the resolved IDs instead + of duplicating the pagination and field-mapping logic. + """ + records: list[dict[str, Any]] = [] offset = 0 while True: @@ -88,11 +100,11 @@ def read_source_records() -> list[dict[str, Any]]: "base", "+record-list", "--base-token", - BASE_TOKEN, + base_token, "--table-id", - TABLE_ID, + table_id, "--view-id", - VIEW_ID, + view_id, "--offset", str(offset), "--limit", diff --git a/src/gyxx_flow/migration/sync_feishu_style_identity.py b/src/gyxx_flow/migration/sync_feishu_style_identity.py new file mode 100644 index 0000000..696b6f9 --- /dev/null +++ b/src/gyxx_flow/migration/sync_feishu_style_identity.py @@ -0,0 +1,347 @@ +"""将飞书款式 ID 表一次性合并到产品生命进程身份表。 + +这个脚本只负责初始化/校准 ``cmt_styles`` 的身份字段： + +* 按款式聚合旧表的一行多平台记录； +* 将 ERP 款式编码和四个平台商品编码去重后写入生命进程； +* 保留生命进程已有的 NFC、图片、类目、市场标签等字段； +* 空的源字段不会清空已有身份，避免旧表不完整行破坏已维护数据； +* 默认只预览，只有显式传入 ``--apply`` 才写入数据库。 + +运行时采集仍从 PostgreSQL 的产品生命进程读取，不会把飞书旧表重新 +变成第二个运行时配置源。 +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import dataclass, field +from typing import Any, Mapping + +import psycopg +from psycopg.rows import dict_row + +from gyxx_flow.migration.feishu_style_config_seed import ( + BASE_TOKEN as SOURCE_BASE_TOKEN, +) +from gyxx_flow.migration.feishu_style_config_seed import ( + TABLE_ID as SOURCE_TABLE_ID, +) +from gyxx_flow.migration.feishu_style_config_seed import ( + read_source_records, +) + +SOURCE_URL = ( + "https://bu0zgpibak.feishu.cn/base/" + f"{SOURCE_BASE_TOKEN}?table={SOURCE_TABLE_ID}&view=vewT7L4JUr" +) +SOURCE_VIEW_ID = "vewT7L4JUr" + +PLATFORM_KEYS = { + "天猫": "tm", + "京东旗舰店": "jd", + "京东": "jd", + "京东自营": "jd_self", + "抖音": "dy", +} +KNOWN_PLATFORM_KEYS = ("tm", "jd", "jd_self", "dy") + + +class FeishuLifecycleSyncError(RuntimeError): + """Raised when the source or lifecycle target cannot be safely synchronized.""" + + +def _clean_list(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, (list, tuple, set)): + parts = value + else: + parts = str(value).replace("，", ",").replace("\r", ",").replace("\n", ",").split(",") + result: list[str] = [] + seen: set[str] = set() + for item in parts: + text = str(item or "").strip() + if text and text not in seen: + seen.add(text) + result.append(text) + return result + + +def _clean_style_name(value: Any) -> str: + return " ".join(str(value or "").split()) + + +@dataclass +class StyleIdentity: + name: str + brand: str = "" + erp_codes: list[str] = field(default_factory=list) + platform_product_ids: dict[str, list[str]] = field( + default_factory=lambda: {key: [] for key in KNOWN_PLATFORM_KEYS} + ) + source_record_ids: list[str] = field(default_factory=list) + unsupported_platform_ids: dict[str, list[str]] = field(default_factory=dict) + + def add_unique(self, field_name: str, values: list[str]) -> None: + target = getattr(self, field_name) + for value in values: + if value not in target: + target.append(value) + + +def build_sync_plan(source_records: list[Mapping[str, Any]]) -> tuple[list[StyleIdentity], dict[str, Any]]: + """Group the legacy per-platform rows into lifecycle style identities.""" + + grouped: dict[str, StyleIdentity] = {} + skipped_rows: list[dict[str, str]] = [] + unknown_platform_rows: list[dict[str, Any]] = [] + + for entry in source_records: + config = entry.get("config") or {} + if not isinstance(config, Mapping): + skipped_rows.append({"record_id": str(entry.get("source_record_id") or ""), "reason": "配置结构无效"}) + continue + source_record_id = str(entry.get("source_record_id") or "").strip() + name = _clean_style_name(config.get("style_name")) + if not name: + skipped_rows.append({"record_id": source_record_id, "reason": "款式名称为空"}) + continue + + identity = grouped.setdefault(name, StyleIdentity(name=name)) + brand = str(config.get("brand") or "").strip() + if brand and not identity.brand: + identity.brand = brand + identity.add_unique("erp_codes", _clean_list(config.get("erp_codes"))) + if source_record_id and source_record_id not in identity.source_record_ids: + identity.source_record_ids.append(source_record_id) + + platform = str(config.get("platform") or "").strip() + item_ids = _clean_list(config.get("item_ids")) + key = PLATFORM_KEYS.get(platform) + if key: + for item_id in item_ids: + if item_id not in identity.platform_product_ids[key]: + identity.platform_product_ids[key].append(item_id) + elif platform and item_ids: + unknown = identity.unsupported_platform_ids.setdefault(platform, []) + for item_id in item_ids: + if item_id not in unknown: + unknown.append(item_id) + unknown_platform_rows.append( + { + "style": name, + "platform": platform, + "item_count": len(item_ids), + "record_id": source_record_id, + } + ) + + styles = [grouped[name] for name in sorted(grouped)] + report = { + "source_url": SOURCE_URL, + "source_records": len(source_records), + "named_styles": len(styles), + "skipped_rows": skipped_rows, + "unknown_platform_rows": unknown_platform_rows, + "styles_without_erp": [style.name for style in styles if not style.erp_codes], + "styles_without_supported_platform_id": [ + style.name + for style in styles + if not any(style.platform_product_ids.values()) + ], + "collectable_styles": [ + style.name + for style in styles + if style.erp_codes and any(style.platform_product_ids.values()) + ], + } + return styles, report + + +def _connect(environment: Mapping[str, str] | None = None) -> psycopg.Connection[Any]: + env = environment if environment is not None else os.environ + dsn = str(env.get("GYXX_POSTGRES_DSN", "")).strip() + if dsn: + return psycopg.connect(dsn, row_factory=dict_row) + required = ("PG_HOST", "PG_PORT", "PG_DB", "PG_USER", "PG_PASSWORD") + missing = [name for name in required if not str(env.get(name, "")).strip()] + if missing: + raise FeishuLifecycleSyncError( + "缺少 PostgreSQL 配置: " + ", ".join(missing) + ) + try: + port = int(env["PG_PORT"]) + except ValueError as exc: + raise FeishuLifecycleSyncError("PG_PORT 必须是整数") from exc + return psycopg.connect( + host=env["PG_HOST"].strip(), + port=port, + dbname=env["PG_DB"].strip(), + user=env["PG_USER"].strip(), + password=env["PG_PASSWORD"], + row_factory=dict_row, + ) + + +def _require_target_schema(conn: psycopg.Connection[Any]) -> None: + with conn.cursor() as cursor: + cursor.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = 'cmt_styles'" + ) + columns = {str(row["column_name"]) for row in cursor.fetchall()} + required = {"name", "brand", "erp_style_codes", "platform_product_ids"} + missing = sorted(required - columns) + if missing: + raise FeishuLifecycleSyncError( + "cmt_styles 缺少身份字段: " + ", ".join(missing) + ) + + +def _existing_rows( + conn: psycopg.Connection[Any], names: list[str] +) -> dict[str, dict[str, Any]]: + if not names: + return {} + with conn.cursor() as cursor: + cursor.execute( + "SELECT name, brand, erp_style_codes, platform_product_ids " + "FROM cmt_styles WHERE name = ANY(%s)", + (names,), + ) + return {str(row["name"]): dict(row) for row in cursor.fetchall()} + + +def _json_platform_ids(value: Any) -> dict[str, list[str]]: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + value = {} + if not isinstance(value, Mapping): + value = {} + # ProductLifecycleServiceImpl strictly rejects unknown JSON keys. The + # legacy database may contain Chinese platform labels, so only carry the + # canonical four keys into the lifecycle record. + return { + key: _clean_list(value.get(key)) + for key in KNOWN_PLATFORM_KEYS + } + + +def _merge_target_row( + style: StyleIdentity, existing: Mapping[str, Any] | None +) -> dict[str, Any]: + existing = existing or {} + erp_codes = _clean_list(existing.get("erp_style_codes")) + if style.erp_codes: + erp_codes = list(style.erp_codes) + + platform_ids = _json_platform_ids(existing.get("platform_product_ids")) + for key in KNOWN_PLATFORM_KEYS: + if style.platform_product_ids[key]: + platform_ids[key] = list(style.platform_product_ids[key]) + + return { + "name": style.name, + "brand": style.brand or str(existing.get("brand") or "").strip(), + "erp_style_codes": erp_codes, + "platform_product_ids": platform_ids, + "existing": existing, + } + + +def build_target_rows( + conn: psycopg.Connection[Any], styles: list[StyleIdentity] +) -> list[dict[str, Any]]: + existing = _existing_rows(conn, [style.name for style in styles]) + return [_merge_target_row(style, existing.get(style.name)) for style in styles] + + +def apply_target_rows( + conn: psycopg.Connection[Any], rows: list[Mapping[str, Any]] +) -> int: + with conn.cursor() as cursor: + for row in rows: + cursor.execute( + "INSERT INTO cmt_styles " + "(name, brand, erp_style_codes, platform_product_ids) " + "VALUES (%s, %s, %s, %s::jsonb) " + "ON CONFLICT (name) DO UPDATE SET " + "brand = CASE WHEN EXCLUDED.brand <> '' " + "THEN EXCLUDED.brand ELSE cmt_styles.brand END, " + "erp_style_codes = CASE " + "WHEN COALESCE(array_length(EXCLUDED.erp_style_codes, 1), 0) > 0 " + "THEN EXCLUDED.erp_style_codes ELSE cmt_styles.erp_style_codes END, " + "platform_product_ids = EXCLUDED.platform_product_ids, " + "updated_at = CURRENT_TIMESTAMP", + ( + row["name"], + row["brand"], + row["erp_style_codes"], + json.dumps(row["platform_product_ids"], ensure_ascii=False), + ), + ) + return len(rows) + + +def _print_report(report: Mapping[str, Any], rows: list[Mapping[str, Any]]) -> None: + print(f"[INFO] 飞书源记录: {report['source_records']}") + print(f"[INFO] 有效款式: {report['named_styles']}") + print(f"[INFO] 将合并到 cmt_styles: {len(rows)}") + print(f"[INFO] 可采集款式: {len(report['collectable_styles'])}") + print(f"[INFO] ERP 缺失款式: {len(report['styles_without_erp'])}") + print( + "[INFO] 不支持的平台行: " + f"{len(report['unknown_platform_rows'])}（保留在旧动态平台配置，不写入四平台身份 JSON）" + ) + if report["skipped_rows"]: + print(f"[WARN] 跳过空款式行: {len(report['skipped_rows'])}") + for row in rows[:5]: + print( + f" - {row['name']}: erp={row['erp_style_codes']} " + f"platform_ids={row['platform_product_ids']}" + ) + + +def run(*, apply: bool, environment: Mapping[str, str] | None = None) -> int: + source_records = read_source_records( + base_token=SOURCE_BASE_TOKEN, + table_id=SOURCE_TABLE_ID, + view_id=SOURCE_VIEW_ID, + ) + styles, report = build_sync_plan(source_records) + with _connect(environment) as conn: + _require_target_schema(conn) + rows = build_target_rows(conn, styles) + _print_report(report, rows) + if not apply: + print("[DRY-RUN] 未写入数据库；使用 --apply 执行同步") + return 0 + applied = apply_target_rows(conn, rows) + conn.commit() + print(f"[OK] 产品生命进程 cmt_styles 已同步: {applied} 行") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="飞书款式配置同步到产品生命进程") + parser.add_argument( + "--apply", + action="store_true", + help="执行数据库写入；默认仅预览", + ) + args = parser.parse_args(argv) + try: + return run(apply=args.apply) + except (FeishuLifecycleSyncError, psycopg.Error) as exc: + print(f"[FAIL] 飞书款式同步失败: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/product_commerce/backfill_erp_all_shop_daily.py b/src/gyxx_flow/modules/product_commerce/backfill_erp_all_shop_daily.py index 482a893..cf93f1e 100644 --- a/src/gyxx_flow/modules/product_commerce/backfill_erp_all_shop_daily.py +++ b/src/gyxx_flow/modules/product_commerce/backfill_erp_all_shop_daily.py @@ -18,7 +18,7 @@ from scrapling.fetchers import DynamicFetcher try: from . import collect_erp_yesterday_metrics as erp from .collect_retry_utils import retry_step - from .config.style_config_loader import StyleConfigLoader + from .config.style_config_loader import StyleConfigLoader, has_complete_style_identity from .db import ( get_conn, init_schema, @@ -35,7 +35,7 @@ try: except ImportError: # Direct script execution from the module root. import collect_erp_yesterday_metrics as erp from collect_retry_utils import retry_step - from config.style_config_loader import StyleConfigLoader + from config.style_config_loader import StyleConfigLoader, has_complete_style_identity from db import get_conn, init_schema, upsert_erp_all_shop_style_daily_metrics from erp_login_product_analysis import ( DEFAULT_CONFIG, @@ -80,14 +80,15 @@ def build_style_plans(payload: Mapping[str, Any]) -> tuple[StylePlan, ...]: code_owners: dict[str, str] = {} for raw_name, raw_config in (payload.get("styles") or {}).items(): style_name = str(raw_name).strip() + config = raw_config or {} codes = tuple( dict.fromkeys( str(code).strip() - for code in (raw_config or {}).get("erp_codes") or () + for code in config.get("erp_codes") or () if str(code).strip() ) ) - if not style_name or not codes: + if not style_name or not codes or not has_complete_style_identity(config): continue for code in codes: previous = code_owners.setdefault(code, style_name) diff --git a/src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py b/src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py index 2fdbaaf..aefc867 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py +++ b/src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py @@ -174,7 +174,7 @@ def load_styles(path: Path) -> list[dict[str, Any]]: def merge_styles_with_dynamic_config(styles: list[dict[str, Any]]) -> list[dict[str, Any]]: - """[已废弃] 飞书为唯一来源,本地 styles_input.json 退为兜底。保留仅为兼容旧调用。""" + """[已废弃] 产品生命进程是 ERP 身份唯一来源；保留仅为兼容旧调用。""" return styles @@ -182,64 +182,67 @@ def load_erp_styles( loader: StyleConfigLoader | None, fallback_path: Path, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """飞书为唯一来源,本地 styles_input.json 仅在飞书读取失败时回退。 + """只从产品生命进程读取 ERP 采集款式，身份源不可用时失败闭环。 返回 (styles, skip_report): styles = [{"style_name", "erp_style_codes", "brand"}] - skip_report = {"source": "lark"|"fallback_json", "collectable": [...], "skipped": [{style, reason}], "brand_map": {...}} + skip_report = {"source": "product_lifecycle"|"lifecycle_unavailable", ...} brand 字段让 collect_in_page 按品牌选店铺 (光影行星旗下店铺 vs ozko旗舰店)。 """ - if loader is not None: - try: - data = loader.get_erp_daily_styles() - payload = loader.load() - brand_map = data.get("brand_map", {}) - styles: list[dict[str, Any]] = [] - for name in data["collectable"]: - codes = payload["styles"][name]["erp_codes"] - styles.append({ - "style_name": name, - "erp_style_codes": [str(c).strip() for c in codes if str(c).strip()], - "brand": brand_map.get(name) or DEFAULT_BRAND, - }) - return styles, { - "source": "lark", - "collectable": data["collectable"], - "skipped": data["skipped"], - "brand_map": brand_map, - } - except Exception as exc: - print(f"[WARN] 飞书读取失败,回退本地 {fallback_path}: {exc}") - - styles = load_styles(fallback_path) - for s in styles: - s.setdefault("brand", DEFAULT_BRAND) - return styles, { - "source": "fallback_json", - "collectable": [], - "skipped": [], - "brand_map": {s["style_name"]: s.get("brand", DEFAULT_BRAND) for s in styles}, - } + del fallback_path # 保留参数以兼容旧调用，商品身份不再从本地文件回退。 + if loader is None: + return [], { + "source": "lifecycle_unavailable", + "collectable": [], + "skipped": [{"style": "*", "reason": "产品生命进程身份源不可用"}], + "brand_map": {}, + } + try: + data = loader.get_erp_daily_styles() + payload = loader.load() + brand_map = data.get("brand_map", {}) + styles: list[dict[str, Any]] = [] + for name in data["collectable"]: + codes = payload["styles"][name]["erp_codes"] + styles.append({ + "style_name": name, + "erp_style_codes": [str(c).strip() for c in codes if str(c).strip()], + "brand": brand_map.get(name) or DEFAULT_BRAND, + }) + return styles, { + "source": "product_lifecycle", + "collectable": data["collectable"], + "skipped": data["skipped"], + "brand_map": brand_map, + } + except Exception as exc: + print(f"[FAIL] 产品生命进程身份源读取失败，不启动 ERP 采集: {exc}") + return [], { + "source": "lifecycle_unavailable", + "collectable": [], + "skipped": [{"style": "*", "reason": "产品生命进程身份源不可用"}], + "brand_map": {}, + } def print_skip_summary(skip_report: dict[str, Any]) -> None: src = skip_report.get("source", "?") if not skip_report.get("skipped"): - if src == "lark": + if src == "product_lifecycle": print(f"[ERP] 来源={src}, 采集 {len(skip_report['collectable'])} 款, 跳过 0 款") else: print(f"[ERP] 来源={src}, 跳过清单未生成 (回退路径不带 reason)") return print( f"[ERP] 来源={src}, 采集 {len(skip_report['collectable'])} 款, " - f"跳过 {len(skip_report['skipped'])} 款 (缺 ERP 编码 / TM 分组 ID / 飞书表地址)" + f"跳过 {len(skip_report['skipped'])} 款 (缺 ERP 编码 / 平台商品编码 / 飞书表地址)" ) for s in skip_report["skipped"]: print(f" - {s['style']}: {s['reason']}") def write_erp_skip_report(skip_report: dict[str, Any], date_str: str) -> Path | None: - if skip_report.get("source") != "lark": + if skip_report.get("source") not in {"product_lifecycle", "lifecycle_unavailable"}: return None SKIP_REPORT_DIR.mkdir(parents=True, exist_ok=True) out_path = SKIP_REPORT_DIR / f"erp_skipped_{date_str}.json" diff --git a/src/gyxx_flow/modules/product_commerce/config/style_config_loader.py b/src/gyxx_flow/modules/product_commerce/config/style_config_loader.py index 8a4040f..9a5d0a2 100644 --- a/src/gyxx_flow/modules/product_commerce/config/style_config_loader.py +++ b/src/gyxx_flow/modules/product_commerce/config/style_config_loader.py @@ -17,7 +17,7 @@ import json import re from datetime import datetime from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Mapping from urllib.parse import parse_qs, urlparse from gyxx_flow.adapters.workflow_config import runtime_workflow_configs @@ -56,6 +56,31 @@ PLATFORM_MAP = { "抖音": "dy", } +PRODUCT_ID_FIELDS = { + "tm": "item_ids", + "jd": "spus", + "jd_self": "skus", + "dy": "item_ids", +} + + +def has_complete_style_identity( + cfg: Mapping[str, Any], platform: str | None = None +) -> bool: + """判断款式是否具备商品采集所需的统一身份。""" + + if not cfg.get("erp_codes"): + return False + if not any( + bool(cfg.get(name, {}).get(ids_key)) + for name, ids_key in PRODUCT_ID_FIELDS.items() + ): + return False + if platform is None: + return True + ids_key = PRODUCT_ID_FIELDS.get(platform) + return bool(ids_key and cfg.get(platform, {}).get(ids_key)) + class StyleConfigError(Exception): pass @@ -528,11 +553,11 @@ class StyleConfigLoader: return sorted(self.load()["styles"].keys()) def get_tm_style_id_map(self) -> dict[str, list[str]]: - """每日天猫采集列表：_aggregate 已保证满足四条件（erp/款式/item_ids/销量表）。""" + """天猫商品 ID 映射；只有完整身份的款式才进入采集。""" return { name: cfg["tm"]["item_ids"] for name, cfg in self.load()["styles"].items() - if cfg["tm"]["item_ids"] + if self._has_complete_identity(cfg, "tm") } def get_tm_persona_bitable_map(self) -> dict[str, dict[str, str]]: @@ -540,7 +565,7 @@ class StyleConfigLoader: 只返回同时有 tm item_ids 和 persona_bitable 的款式（三者同时存在才采）。""" result: dict[str, dict[str, str]] = {} for name, cfg in self.load()["styles"].items(): - if not cfg["tm"]["item_ids"]: + if not self._has_complete_identity(cfg, "tm"): continue persona = cfg.get("persona_bitable") or {} if persona.get("base_token") and persona.get("table_id"): @@ -556,7 +581,7 @@ class StyleConfigLoader: return { name: cfg.get("brand", "") for name, cfg in self.load()["styles"].items() - if cfg["tm"]["item_ids"] + if self._has_complete_identity(cfg, "tm") } def get_jd_spu_groups(self) -> list[dict]: @@ -564,7 +589,7 @@ class StyleConfigLoader: return [ {"style": name, "spus": cfg["jd"]["spus"]} for name, cfg in self.load()["styles"].items() - if cfg["jd"]["spus"] + if self._has_complete_identity(cfg, "jd") ] def get_dy_product_groups(self) -> dict[str, list[str]]: @@ -572,7 +597,7 @@ class StyleConfigLoader: return { name: cfg["dy"]["item_ids"] for name, cfg in self.load()["styles"].items() - if cfg["dy"]["item_ids"] + if self._has_complete_identity(cfg, "dy") } def get_dy_persona_bitable_map(self) -> dict[str, dict[str, str]]: @@ -580,7 +605,7 @@ class StyleConfigLoader: 只返回同时有 dy item_ids 和 dy_persona_bitable 的款式。""" result: dict[str, dict[str, str]] = {} for name, cfg in self.load()["styles"].items(): - if not cfg["dy"]["item_ids"]: + if not self._has_complete_identity(cfg, "dy"): continue persona = cfg.get("dy_persona_bitable") or {} if persona.get("base_token") and persona.get("table_id"): @@ -596,7 +621,7 @@ class StyleConfigLoader: 只返回同时有 jd spus 和 jd_persona_bitable 的款式。""" result: dict[str, dict[str, str]] = {} for name, cfg in self.load()["styles"].items(): - if not cfg["jd"]["spus"]: + if not self._has_complete_identity(cfg, "jd"): continue persona = cfg.get("jd_persona_bitable") or {} if persona.get("base_token") and persona.get("table_id"): @@ -611,6 +636,8 @@ class StyleConfigLoader: """款式 -> 平台单品分析目标表。""" result: dict[str, dict[str, str]] = {} for name, cfg in self.load()["styles"].items(): + if not self._has_complete_identity(cfg): + continue target = cfg.get("style_analysis_bitable") or {} if target.get("base_token") and target.get("table_id"): result[name] = { @@ -625,20 +652,22 @@ class StyleConfigLoader: return [ {"style_name": name, "skus": cfg["jd_self"]["skus"]} for name, cfg in self.load()["styles"].items() - if cfg["jd_self"]["skus"] + if self._has_complete_identity(cfg, "jd_self") ] def get_erp_styles_input(self) -> list[dict]: return [ {"style_name": name, "erp_style_codes": cfg["erp_codes"]} for name, cfg in self.load()["styles"].items() - if cfg["erp_codes"] + if self._has_complete_identity(cfg) ] def get_bitable_style_map(self) -> dict: """生成与 bitable_style_map.json 兼容的结构。""" result: dict = {} for name, cfg in self.load()["styles"].items(): + if not self._has_complete_identity(cfg): + continue sales = cfg.get("sales_bitable") or {} if not sales.get("base_token") or not sales.get("table_id"): continue @@ -665,6 +694,8 @@ class StyleConfigLoader: """生成与 bitable_main_image_map.json 兼容的结构。""" result: dict = {} for name, cfg in self.load()["styles"].items(): + if not self._has_complete_identity(cfg, "tm"): + continue main = cfg.get("main_image_bitable") or {} if not main.get("base_token") or not main.get("table_id"): continue @@ -686,38 +717,55 @@ class StyleConfigLoader: cfg.get("sales_bitable", {}).get("table_id") ) + @staticmethod + def _has_any_product_id(cfg: dict) -> bool: + return any( + bool(cfg.get(platform, {}).get(ids_key)) + for platform, ids_key in PRODUCT_ID_FIELDS.items() + ) + + def _has_complete_identity(self, cfg: dict, platform: str | None = None) -> bool: + """商品采集统一门：ERP 款式编码 + 至少一个平台商品编码。""" + return has_complete_style_identity(cfg, platform) + + def _identity_missing(self, cfg: dict) -> list[str]: + missing: list[str] = [] + if not cfg.get("erp_codes"): + missing.append("ERP款式编码") + if not self._has_any_product_id(cfg): + missing.append("平台商品编码") + return missing + def get_daily_styles(self) -> dict: - """每日采集门 (v2.2): - - 款式级门: erp_codes ∩ sales_bitable 都非空 (从任意记录收集) - - 平台级门: 该平台分组 item_ids 非空 + """每日商品采集门：ERP 编码、销量目标和平台商品编码必须完整。 返回 {tm/jd/dy: {collectable: [...], skipped: [{style, reason}]}}。 - skipped 仅记录「款式级门通过、平台级缺 IDs」的款式;款式级失败不计入 skipped - (因为该款式没有任何平台可采)。 + incomplete style 永远不会进入 collectable，并记录具体缺失字段。 """ payload = self.load() result = {p: {"collectable": [], "skipped": []} for p in ("tm", "jd", "dy")} for name, cfg in payload["styles"].items(): - style_ok = bool(cfg.get("erp_codes")) and self._cfg_has_sales_bitable(cfg) - if not style_ok: - continue # 款式级失败,无任何平台可采 + identity_missing = self._identity_missing(cfg) for plat in ("tm", "jd", "dy"): ids_key = "spus" if plat == "jd" else "item_ids" - if cfg.get(plat, {}).get(ids_key): - result[plat]["collectable"].append(name) - else: + missing = list(identity_missing) + if not self._cfg_has_sales_bitable(cfg): + missing.append("销量目标地址") + if not cfg.get(plat, {}).get(ids_key): + missing.append(f"{plat}商品编码") + if missing: result[plat]["skipped"].append( - {"style": name, "reason": f"缺 {plat} 商品链接ID"} + {"style": name, "reason": "缺 " + "/".join(dict.fromkeys(missing))} ) + else: + result[plat]["collectable"].append(name) return result def get_erp_daily_styles(self) -> dict: - """ERP daily 门:款式 + ERP 编码 + TM 分组有 item_ids + 飞书销量地址 都要有。 + """ERP daily 门：ERP 编码、至少一个平台商品编码和销量目标都要有。 返回 {collectable, skipped, brand_map: {style: brand}}。 - ERP 一次跑覆盖 jd/dy/tm 三平台(用同一组 erp_codes 查三个店铺), - 所以 jd/dy 没商品 ID 也能跑,但 TM 分组必须有 item_ids - (飞书表格里 TM 分组是「基础分组」, 款式只要在 TM 上架就算款式存在)。 + ERP 一次跑覆盖 jd/dy/tm 三个平台，平台商品编码只用于确认款式已完成商品身份配置。 brand_map 给 collect_erp_yesterday_metrics.py 按品牌选店铺 (光影行星旗下店铺 vs ozko旗舰店)。 """ @@ -726,36 +774,31 @@ class StyleConfigLoader: skipped: list[dict] = [] brand_map: dict[str, str] = {} for name, cfg in payload["styles"].items(): - has_tm = bool(cfg.get("tm", {}).get("item_ids")) - if cfg.get("erp_codes") and has_tm and self._cfg_has_sales_bitable(cfg): + missing = self._identity_missing(cfg) + if not self._cfg_has_sales_bitable(cfg): + missing.append("销量目标地址") + if not missing: collectable.append(name) brand_map[name] = cfg.get("brand") or "光影行星" else: - miss = [] - if not cfg.get("erp_codes"): - miss.append("ERP款式编码") - if not has_tm: - miss.append("天猫分组商品ID") - if not self._cfg_has_sales_bitable(cfg): - miss.append("飞书多维表格地址") - skipped.append({"style": name, "reason": "缺 " + "/".join(miss)}) + skipped.append({"style": name, "reason": "缺 " + "/".join(dict.fromkeys(missing))}) return {"collectable": collectable, "skipped": skipped, "brand_map": brand_map} def get_jd_self_styles(self) -> list[str]: - """京东自营周汇总门:仅 SKU 非空即可 (用户确认保持现状)。""" + """京东自营周汇总门：ERP 编码和京东自营 SKU 都要有。""" return [ name for name, cfg in self.load()["styles"].items() - if cfg.get("jd_self", {}).get("skus") + if self._has_complete_identity(cfg, "jd_self") ] def get_main_image_styles(self) -> list[str]: - """主图门:天猫分组 + 主图飞书多维表格地址 同时存在。""" + """天猫主图门：完整身份、天猫商品编码和主图目标地址同时存在。""" payload = self.load() result: list[str] = [] for name, cfg in payload["styles"].items(): main = cfg.get("main_image_bitable") or {} - if main.get("base_token") and main.get("table_id"): + if self._has_complete_identity(cfg, "tm") and main.get("base_token") and main.get("table_id"): result.append(name) return result diff --git a/src/gyxx_flow/modules/product_commerce/db/__init__.py b/src/gyxx_flow/modules/product_commerce/db/__init__.py index b220264..76618a2 100644 --- a/src/gyxx_flow/modules/product_commerce/db/__init__.py +++ b/src/gyxx_flow/modules/product_commerce/db/__init__.py @@ -27,7 +27,7 @@ 写: check_nine_day_decline.py (去重) 读: check_nine_day_decline.py._already_notified dim_style - 写: db/sync_dim_style.py (人工触发，全量扫 4 个数据源) + 写: db/sync_dim_style.py (人工触发，投影产品生命进程身份) 读: bitable 同步 / 业务查询 fact_platform_xlsx 写: 暂无 caller (db/__init__.py.upsert_platform_xlsx_rows 已写好，import 脚本待补) diff --git a/src/gyxx_flow/modules/product_commerce/db/schema.sql b/src/gyxx_flow/modules/product_commerce/db/schema.sql index bfeb155..2cd211f 100644 --- a/src/gyxx_flow/modules/product_commerce/db/schema.sql +++ b/src/gyxx_flow/modules/product_commerce/db/schema.sql @@ -4,13 +4,10 @@ -- 设计原则: schema 与 data/ 目录下的 JSON 产物 1:1 映射；upsert 幂等 -- ============================================================================ --- 款式维度表：维护每款的 (各平台 SPU/ID + 京东自营 SKU + ERP 编码 + 飞书表地址) --- 数据源: --- jd_self_skus ← jd_self_inventory_sales_collector.STYLE_SKU_GROUPS --- jd_spus/dy_spus/tm_spus ← data//<款>_<日期>/<款>_<日期>.json 的"匹配SPU" --- erp_codes ← daily_style_metrics.erp_style_codes (自动聚) --- bitable_weekly_* ← bitable_style_map.json (每款周报表地址) --- bitable_sku_master_* ← 飞书 SKU 主表（人工维护；表 base_token + table_id 填这里） +-- 款式维度表：维护每款的 (各平台商品 ID + 京东自营 SKU + ERP 编码 + 飞书表地址) +-- 身份主来源：产品生命进程 cmt_styles.platform_product_ids / erp_style_codes， +-- 由 workflow_config.runtime_records() 统一投影；运行时不再从各采集器、JSON 产物反向拼接身份。 +-- bitable_weekly_* 仍由 bitable_style_map.json 派生；bitable_sku_master_* 仍由飞书 SKU 主表人工维护。 CREATE TABLE IF NOT EXISTS dim_style ( style_name TEXT PRIMARY KEY, jd_spus TEXT[], -- 京东 POP 商品 SPU @@ -226,7 +223,7 @@ ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_table TEXT; -- ============================================================================ -- dim_style -COMMENT ON TABLE dim_style IS '款式维度表 — 维护每款 (各平台 SPU/ID + 京东自营 SKU + ERP 编码 + 飞书表地址 + 品类/上架日期/备注)。数据源: jd_self_skus、jd/dy/tm_spus、erp_codes、bitable_style_map.json、飞书 SKU 主表 (人工维护)。'; +COMMENT ON TABLE dim_style IS '款式维度表 — 维护每款 (各平台商品 ID + 京东自营 SKU + ERP 编码 + 飞书表地址 + 品类/上架日期/备注)。身份投影自产品生命进程，周报地址与 SKU 主表仍按各自来源维护。'; COMMENT ON COLUMN dim_style.style_name IS '款式名 (主键)，与 PRODUCT_STYLE_GROUPS 的 key 对齐'; COMMENT ON COLUMN dim_style.jd_spus IS '京东 POP 商品 SPU 列表'; COMMENT ON COLUMN dim_style.dy_spus IS '抖音商品 SPU 列表'; diff --git a/src/gyxx_flow/modules/product_commerce/db/sync_dim_style.py b/src/gyxx_flow/modules/product_commerce/db/sync_dim_style.py index 6825670..8181d79 100644 --- a/src/gyxx_flow/modules/product_commerce/db/sync_dim_style.py +++ b/src/gyxx_flow/modules/product_commerce/db/sync_dim_style.py @@ -1,10 +1,10 @@ """同步 dim_style：4 个数据源自动 upsert。 数据源: - 1) bitable_style_map.json -> bitable_weekly_url/base/table - 2) jd_self_inventory_sales_collector.STYLE_SKU_GROUPS -> jd_self_skus - 3) data//<款>_<最近日期>/<款>_<最近日期>.json -> jd_spus / dy_spus / tm_spus - 4) daily_style_metrics.erp_style_codes -> erp_codes + 产品生命进程 + 动态配置运行时聚合 -> 商品编码、ERP 编码和飞书目标地址 + +历史文件、采集产物和硬编码仅保留在本文件的兼容函数中，不再作为同步入口， +避免人工修改生命进程后被旧脚本反向覆盖。 SKU 主表的飞书地址 (bitable_sku_master_*) 由人工维护，不在自动同步范围。 @@ -25,7 +25,9 @@ from typing import Any, Iterable PROJECT_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(PROJECT_ROOT)) +sys.path.insert(0, str(PROJECT_ROOT.parents[2])) from runtime_paths import RAW_DATA_ROOT, runtime_script +from gyxx_flow.adapters.workflow_config import runtime_workflow_configs # noqa: E402 from db import get_conn # noqa: E402 @@ -258,6 +260,63 @@ def _merge_records( return rows +def load_lifecycle_records() -> list[dict[str, Any]]: + """读取生命进程身份与动态目标的统一运行时视图。""" + + return runtime_workflow_configs() + + +def _lifecycle_to_dim_rows(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + grouped: dict[str, dict[str, Any]] = {} + platform_fields = { + "天猫": "tm_spus", + "京东旗舰店": "jd_spus", + "京东": "jd_spus", + "京东自营": "jd_self_skus", + "抖音": "dy_spus", + } + for record in records: + name = str(record.get("style_name") or "").strip() + if not name: + continue + row = grouped.setdefault( + name, + { + "style_name": name, + "jd_spus": [], + "dy_spus": [], + "tm_spus": [], + "jd_self_skus": [], + "erp_codes": [], + "bitable_weekly_url": "", + "bitable_weekly_base": "", + "bitable_weekly_table": "", + }, + ) + for code in record.get("erp_codes") or []: + code = str(code).strip() + if code and code not in row["erp_codes"]: + row["erp_codes"].append(code) + target_url = str(record.get("sales_bitable_url") or "").strip() + if target_url and not row["bitable_weekly_url"]: + row["bitable_weekly_url"] = target_url + match = re.search( + r"/base/([A-Za-z0-9]+)\?[^\s]*table=([A-Za-z0-9]+)", + target_url, + ) + if match: + row["bitable_weekly_base"] = match.group(1) + row["bitable_weekly_table"] = match.group(2) + field = platform_fields.get(str(record.get("platform") or "").strip()) + if not field: + continue + for product_id in record.get("item_ids") or []: + product_id = str(product_id).strip() + if product_id and product_id not in row[field]: + row[field].append(product_id) + return list(grouped.values()) + + def upsert_dim_style(conn, records: list[dict]) -> int: if not records: return 0 @@ -311,21 +370,10 @@ def main(): ap.add_argument("--dry-run", action="store_true", help="扫描不入库") args = ap.parse_args() - weekly = load_bitable_weekly() - self_skus = load_jd_self_skus() - spus_map = load_platform_spus() - erp_map = load_erp_codes() + runtime_records = load_lifecycle_records() + records = _lifecycle_to_dim_rows(runtime_records) - # 款式名全集：合并 4 个数据源 - style_names: set[str] = set() - style_names.update(weekly.keys()) - style_names.update(self_skus.keys()) - style_names.update(spus_map.keys()) - style_names.update(erp_map.keys()) - - records = _merge_records(sorted(style_names), weekly, self_skus, spus_map, erp_map) - - print(f"[INFO] 数据源: weekly={len(weekly)} self_skus={len(self_skus)} spus={len(spus_map)} erp={len(erp_map)}") + print(f"[INFO] 数据源: product_lifecycle_runtime={len(runtime_records)} 条") print(f"[INFO] 合并款式: {len(records)}") if args.dry_run: for r in records[:5]: diff --git a/tests/modules/product_commerce/test_style_config_loader_erp_source.py b/tests/modules/product_commerce/test_style_config_loader_erp_source.py index 28b2094..3f03ead 100644 --- a/tests/modules/product_commerce/test_style_config_loader_erp_source.py +++ b/tests/modules/product_commerce/test_style_config_loader_erp_source.py @@ -64,7 +64,66 @@ def test_database_row_uses_explicit_style_name_not_style_content() -> None: "main_image_bitable_url": "", "persona_bitable_url": "", "dy_persona_bitable_url": "", - "jd_persona_bitable_url": "", - "style_analysis_bitable_url": "", - } + "jd_persona_bitable_url": "", + "style_analysis_bitable_url": "", + "destinations": [], + } ] + + +def test_erp_only_style_is_skipped_by_every_product_collection_gate() -> None: + loader = StyleConfigLoader( + records_provider=lambda: [ + { + "platform": "天猫", + "style_name": "研发款", + "erp_codes": ["ERP-ONLY"], + "item_ids": [], + "sales_bitable_url": "https://example.feishu.cn/base/sales?table=tblSales", + "main_image_bitable_url": "https://example.feishu.cn/base/image?table=tblImage", + } + ] + ) + + daily = loader.get_daily_styles() + erp_daily = loader.get_erp_daily_styles() + + assert daily["tm"]["collectable"] == [] + assert daily["tm"]["skipped"] == [ + {"style": "研发款", "reason": "缺 平台商品编码/tm商品编码"} + ] + assert erp_daily["collectable"] == [] + assert erp_daily["skipped"] == [ + {"style": "研发款", "reason": "缺 平台商品编码"} + ] + assert loader.get_erp_styles_input() == [] + assert loader.get_main_image_styles() == [] + + +def test_complete_identity_allows_only_configured_platforms() -> None: + loader = StyleConfigLoader( + records_provider=lambda: [ + { + "platform": "天猫", + "style_name": "完整款", + "erp_codes": ["ERP-1"], + "item_ids": ["TM-1"], + "sales_bitable_url": "https://example.feishu.cn/base/sales?table=tblSales", + "main_image_bitable_url": "https://example.feishu.cn/base/image?table=tblImage", + }, + { + "platform": "京东旗舰店", + "style_name": "完整款", + "erp_codes": [], + "item_ids": ["JD-1"], + }, + ] + ) + + daily = loader.get_daily_styles() + + assert daily["tm"]["collectable"] == ["完整款"] + assert daily["jd"]["collectable"] == ["完整款"] + assert daily["dy"]["collectable"] == [] + assert loader.get_erp_daily_styles()["collectable"] == ["完整款"] + assert loader.get_main_image_styles() == ["完整款"] diff --git a/tests/test_sync_feishu_style_identity.py b/tests/test_sync_feishu_style_identity.py new file mode 100644 index 0000000..e947ffe --- /dev/null +++ b/tests/test_sync_feishu_style_identity.py @@ -0,0 +1,102 @@ +from gyxx_flow.migration.sync_feishu_style_identity import ( + StyleIdentity, + _merge_target_row, + build_sync_plan, +) + + +def _row(record_id: str, **config): + return {"source_record_id": record_id, "config": config} + + +def test_build_sync_plan_groups_platform_rows_and_deduplicates_ids(): + styles, report = build_sync_plan( + [ + _row( + "tm-1", + style_name=" 极星pro ", + brand="光影行星", + erp_codes="10398,10398", + platform="天猫", + item_ids="TM-1,TM-2", + ), + _row( + "jd-1", + style_name="极星pro", + brand="光影行星", + erp_codes=["10398"], + platform="京东旗舰店", + item_ids=["JD-1", "JD-1"], + ), + ] + ) + + assert [style.name for style in styles] == ["极星pro"] + assert styles[0].erp_codes == ["10398"] + assert styles[0].platform_product_ids == { + "tm": ["TM-1", "TM-2"], + "jd": ["JD-1"], + "jd_self": [], + "dy": [], + } + assert report["collectable_styles"] == ["极星pro"] + + +def test_build_sync_plan_reports_unknown_platform_and_blank_style(): + styles, report = build_sync_plan( + [ + _row( + "pdd-1", + style_name="宙斯", + brand="光影行星", + erp_codes="10489", + platform="拼多多", + item_ids="PDD-1", + ), + _row( + "blank-1", + style_name="", + brand="", + erp_codes="", + platform="天猫", + item_ids="", + ), + ] + ) + + assert [style.name for style in styles] == ["宙斯"] + assert report["unknown_platform_rows"] == [ + { + "style": "宙斯", + "platform": "拼多多", + "item_count": 1, + "record_id": "pdd-1", + } + ] + assert report["skipped_rows"] == [ + {"record_id": "blank-1", "reason": "款式名称为空"} + ] + + +def test_merge_target_row_does_not_erase_existing_identity_with_blank_source(): + style = StyleIdentity(name="宙斯") + merged = _merge_target_row( + style, + { + "name": "宙斯", + "brand": "光影行星", + "erp_style_codes": ["10489"], + "platform_product_ids": { + "天猫": ["OLD-TM"], + "tm": ["TM-1"], + "jd": ["JD-1"], + }, + }, + ) + + assert merged["brand"] == "光影行星" + assert merged["erp_style_codes"] == ["10489"] + assert merged["platform_product_ids"]["tm"] == ["TM-1"] + assert merged["platform_product_ids"]["jd"] == ["JD-1"] + assert merged["platform_product_ids"]["jd_self"] == [] + assert merged["platform_product_ids"]["dy"] == [] diff --git a/tests/test_workflow_dynamic_config.py b/tests/test_workflow_dynamic_config.py index 779e1f2..00270de 100644 --- a/tests/test_workflow_dynamic_config.py +++ b/tests/test_workflow_dynamic_config.py @@ -11,6 +11,7 @@ import pytest from gyxx_flow.adapters.workflow_config import ( PostgresWorkflowConfigStore, WorkflowConfigConflictError, + merge_lifecycle_style_records, normalize_config, normalize_style_config, ) @@ -160,6 +161,107 @@ def test_normalize_style_config_owns_erp_once_and_platform_ids_per_child() -> No ] +def test_runtime_records_use_lifecycle_identity_and_keep_dynamic_targets() -> None: + dynamic_records = [ + { + "style_name": "款式甲", + "brand": "旧品牌", + "erp_codes": ["OLD-ERP"], + "platform": "天猫", + "item_ids": ["OLD-TM"], + "sales_bitable_url": "https://example.feishu.cn/base/sales?table=tblSales", + "tm_persona_bitable_url": "tm-persona", + } + ] + lifecycle_rows = [ + { + "id": 8, + "name": "款式甲", + "brand": "光影行星", + "erp_style_codes": ["NEW-ERP"], + "platform_product_ids": { + "tm": ["NEW-TM"], + "jd": ["NEW-JD"], + "jd_self": [], + "dy": [], + }, + } + ] + + records = merge_lifecycle_style_records(dynamic_records, lifecycle_rows) + + by_platform = {row["platform"]: row for row in records} + assert by_platform["天猫"]["item_ids"] == ["NEW-TM"] + assert by_platform["京东旗舰店"]["item_ids"] == ["NEW-JD"] + assert by_platform["京东自营"]["item_ids"] == [] + assert by_platform["抖音"]["item_ids"] == [] + assert all(row["erp_codes"] == ["NEW-ERP"] for row in records) + assert by_platform["天猫"]["sales_bitable_url"].endswith("table=tblSales") + assert by_platform["天猫"]["tm_persona_bitable_url"] == "tm-persona" + + +def test_runtime_records_empty_lifecycle_ids_clear_legacy_platform_ids() -> None: + records = merge_lifecycle_style_records( + [ + { + "style_name": "研发款", + "erp_codes": ["ERP-1"], + "platform": "天猫", + "item_ids": ["LEGACY-TM"], + } + ], + [ + { + "id": 9, + "name": "研发款", + "erp_style_codes": ["ERP-1"], + "platform_product_ids": { + "tm": [], + "jd": [], + "jd_self": [], + "dy": [], + }, + } + ], + ) + + assert all(not row["item_ids"] for row in records) + + +def test_lifecycle_merge_respects_disabled_dynamic_platform() -> None: + records = merge_lifecycle_style_records( + [ + { + "style_name": "款式甲", + "platform": "天猫", + "item_ids": ["TM-1"], + "enabled": False, + }, + { + "style_name": "款式甲", + "platform": "京东旗舰店", + "item_ids": ["JD-1"], + "enabled": True, + }, + ], + [ + { + "id": 10, + "name": "款式甲", + "erp_style_codes": ["ERP-1"], + "platform_product_ids": { + "tm": ["TM-1"], + "jd": ["JD-1"], + "jd_self": [], + "dy": [], + }, + } + ], + ) + + assert {row["platform"] for row in records} == {"京东旗舰店", "京东自营", "抖音"} + + def test_normalize_style_config_supports_extensible_business_targets() -> None: payload = _payload() payload["destinations"] = [