feat: [workflow] 统一飞书款式身份配置
This commit is contained in:
@@ -26,6 +26,7 @@ from gyxx_flow.adapters.integration import (
|
||||
MIGRATION_KEY = "feishu-style-config-20260819-v1"
|
||||
NORMALIZED_MIGRATION_KEY = "workflow-style-platform-config-20260819-v2"
|
||||
PERSONA_PLATFORM_MIGRATION_KEY = "workflow-style-platform-persona-20260819-v3"
|
||||
LIFECYCLE_IDENTITY_MIGRATION_KEY = "product-lifecycle-style-identity-20260907-v1"
|
||||
DEFAULT_SEED_PATH = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "config"
|
||||
@@ -77,6 +78,13 @@ KNOWN_PLATFORMS = (
|
||||
"拼多多光影行星官方旗舰店",
|
||||
"光影行星GYXX箱包专卖店",
|
||||
)
|
||||
LIFECYCLE_PLATFORM_LABELS = {
|
||||
"tm": "天猫",
|
||||
"jd": "京东旗舰店",
|
||||
"jd_self": "京东自营",
|
||||
"dy": "抖音",
|
||||
}
|
||||
LIFECYCLE_PLATFORM_KEYS = tuple(LIFECYCLE_PLATFORM_LABELS)
|
||||
|
||||
STYLE_FIELDS = (
|
||||
"style_name",
|
||||
@@ -252,7 +260,7 @@ class WorkflowConfigRepository(Protocol):
|
||||
def _clean_list(value: Any) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
parts = value if isinstance(value, list) else re.split(r"[,,\r\n]", str(value))
|
||||
parts = value if isinstance(value, (list, tuple, set)) else re.split(r"[,,\r\n]", str(value))
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in parts:
|
||||
@@ -263,6 +271,155 @@ def _clean_list(value: Any) -> list[str]:
|
||||
return result
|
||||
|
||||
|
||||
def _decode_lifecycle_product_ids(value: Any) -> tuple[bool, dict[str, list[str]]]:
|
||||
"""返回 (是否由生命进程接管, 各平台商品编码)。
|
||||
|
||||
老数据的 JSONB 可能仍是空对象,此时允许动态配置作为迁移兼容来源;
|
||||
生命进程保存的对象始终带有四个固定键,即使某个平台为空数组也要视为空值生效。
|
||||
"""
|
||||
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
value = None
|
||||
if not isinstance(value, Mapping):
|
||||
return False, {key: [] for key in LIFECYCLE_PLATFORM_KEYS}
|
||||
managed = any(key in value for key in LIFECYCLE_PLATFORM_KEYS)
|
||||
return managed, {
|
||||
key: _clean_list(value.get(key)) for key in LIFECYCLE_PLATFORM_KEYS
|
||||
}
|
||||
|
||||
|
||||
def merge_lifecycle_style_records(
|
||||
dynamic_records: list[Mapping[str, Any]],
|
||||
lifecycle_rows: list[Mapping[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Overlay lifecycle product identity on legacy target configuration rows.
|
||||
|
||||
``workflow_dynamic_*`` still owns Feishu target URLs and persona targets, while
|
||||
``cmt_styles`` owns ERP codes and product IDs. The result intentionally keeps
|
||||
the old flat runtime contract so existing collectors need no per-workflow fork.
|
||||
"""
|
||||
|
||||
dynamic_by_style: dict[str, list[Mapping[str, Any]]] = {}
|
||||
for record in dynamic_records:
|
||||
style_name = " ".join(str(record.get("style_name") or "").split())
|
||||
if style_name:
|
||||
dynamic_by_style.setdefault(style_name, []).append(record)
|
||||
|
||||
lifecycle_by_style: dict[str, Mapping[str, Any]] = {}
|
||||
lifecycle_order: list[str] = []
|
||||
for row in lifecycle_rows:
|
||||
style_name = " ".join(str(row.get("name") or "").split())
|
||||
if not style_name or style_name in lifecycle_by_style:
|
||||
continue
|
||||
lifecycle_by_style[style_name] = row
|
||||
lifecycle_order.append(style_name)
|
||||
|
||||
style_order = list(dynamic_by_style)
|
||||
style_order.extend(name for name in lifecycle_order if name not in dynamic_by_style)
|
||||
records: list[dict[str, Any]] = []
|
||||
|
||||
for style_name in style_order:
|
||||
legacy_rows = dynamic_by_style.get(style_name, [])
|
||||
lifecycle = lifecycle_by_style.get(style_name)
|
||||
if lifecycle is None:
|
||||
records.extend(dict(row) for row in legacy_rows)
|
||||
continue
|
||||
|
||||
managed, lifecycle_ids = _decode_lifecycle_product_ids(
|
||||
lifecycle.get("platform_product_ids")
|
||||
)
|
||||
lifecycle_erp = _clean_list(lifecycle.get("erp_style_codes"))
|
||||
legacy_erp = _clean_list(legacy_rows[0].get("erp_codes")) if legacy_rows else []
|
||||
erp_codes = lifecycle_erp if lifecycle_erp or managed else legacy_erp
|
||||
brand = str(lifecycle.get("brand") or "").strip()
|
||||
if not brand and legacy_rows:
|
||||
brand = str(legacy_rows[0].get("brand") or "").strip()
|
||||
|
||||
template = dict(legacy_rows[0]) if legacy_rows else {
|
||||
"style_name": str(lifecycle.get("name") or style_name),
|
||||
"brand": brand,
|
||||
"erp_codes": erp_codes,
|
||||
"note": "",
|
||||
"sales_bitable_url": "",
|
||||
"main_image_bitable_url": "",
|
||||
"creator_bitable_url": "",
|
||||
"tm_persona_bitable_url": "",
|
||||
"jd_persona_bitable_url": "",
|
||||
"dy_persona_bitable_url": "",
|
||||
"self_creator_bitable_url": "",
|
||||
"weekly_note_analysis_url": "",
|
||||
"style_analysis_bitable_url": "",
|
||||
"style_content": "",
|
||||
"destinations": [],
|
||||
"enabled": True,
|
||||
}
|
||||
|
||||
dynamic_ids: dict[str, list[str]] = {key: [] for key in LIFECYCLE_PLATFORM_KEYS}
|
||||
dynamic_unknown_ids: dict[str, list[str]] = {}
|
||||
dynamic_labels: list[str] = []
|
||||
for row in legacy_rows:
|
||||
if row.get("enabled", True) is False:
|
||||
continue
|
||||
label = str(row.get("platform") or "").strip()
|
||||
if label and label not in dynamic_labels:
|
||||
dynamic_labels.append(label)
|
||||
key = next(
|
||||
(key for key, known_label in LIFECYCLE_PLATFORM_LABELS.items() if known_label == label),
|
||||
None,
|
||||
)
|
||||
if key:
|
||||
for item_id in _clean_list(row.get("item_ids")):
|
||||
if item_id not in dynamic_ids[key]:
|
||||
dynamic_ids[key].append(item_id)
|
||||
elif label:
|
||||
for item_id in _clean_list(row.get("item_ids")):
|
||||
if item_id not in dynamic_unknown_ids.setdefault(label, []):
|
||||
dynamic_unknown_ids[label].append(item_id)
|
||||
|
||||
if managed:
|
||||
labels = []
|
||||
for label in LIFECYCLE_PLATFORM_LABELS.values():
|
||||
platform_rows = [
|
||||
row for row in legacy_rows
|
||||
if str(row.get("platform") or "").strip() == label
|
||||
]
|
||||
# 没有动态平台行时保持生命进程身份可被读取;一旦存在平台行,
|
||||
# 则尊重该行的启停状态,避免停用后又被固定平台列表重新打开。
|
||||
if not platform_rows or any(
|
||||
row.get("enabled", True) is not False for row in platform_rows
|
||||
):
|
||||
labels.append(label)
|
||||
else:
|
||||
labels = list(dynamic_labels)
|
||||
labels.extend(label for label in dynamic_labels if label not in labels)
|
||||
|
||||
for label in labels:
|
||||
key = next(
|
||||
(key for key, known_label in LIFECYCLE_PLATFORM_LABELS.items() if known_label == label),
|
||||
None,
|
||||
)
|
||||
output = dict(template)
|
||||
output["enabled"] = True
|
||||
output["style_name"] = str(lifecycle.get("name") or style_name).strip()
|
||||
output["brand"] = brand
|
||||
output["erp_codes"] = list(erp_codes)
|
||||
output["platform"] = label
|
||||
output["item_ids"] = (
|
||||
list(lifecycle_ids[key])
|
||||
if managed and key
|
||||
else list(dynamic_ids.get(key, dynamic_unknown_ids.get(label, [])))
|
||||
)
|
||||
output["source_record_id"] = (
|
||||
f"cmt_style:{lifecycle.get('id')}" if key and managed else output.get("source_record_id")
|
||||
)
|
||||
records.append(output)
|
||||
|
||||
return records
|
||||
|
||||
|
||||
def _plain_url(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
match = re.search(r"\]\((https?://[^)]+)\)", text)
|
||||
@@ -579,6 +736,13 @@ class PostgresWorkflowConfigStore:
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
self._preserve_persona_platforms(cursor)
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM workflow_dynamic_config_migrations "
|
||||
"WHERE migration_key = %s",
|
||||
(LIFECYCLE_IDENTITY_MIGRATION_KEY,),
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
self._bootstrap_lifecycle_identity(cursor)
|
||||
conn.commit()
|
||||
self._initialized = True
|
||||
except WorkflowConfigError:
|
||||
@@ -779,6 +943,88 @@ class PostgresWorkflowConfigStore:
|
||||
(PERSONA_PLATFORM_MIGRATION_KEY, NORMALIZED_MIGRATION_KEY, preserved),
|
||||
)
|
||||
|
||||
def _bootstrap_lifecycle_identity(self, cursor: psycopg.Cursor[Any]) -> None:
|
||||
"""Promote the last dynamic-config identity snapshot to cmt_styles once."""
|
||||
|
||||
cursor.execute("SELECT to_regclass('public.cmt_styles') AS table_name")
|
||||
table = cursor.fetchone() or {}
|
||||
if not table.get("table_name"):
|
||||
# Content schema is installed separately in some environments. Leave the
|
||||
# marker absent so the promotion can retry after that schema is available.
|
||||
return
|
||||
|
||||
cursor.execute(
|
||||
"ALTER TABLE cmt_styles "
|
||||
"ADD COLUMN IF NOT EXISTS brand VARCHAR(128) DEFAULT '', "
|
||||
"ADD COLUMN IF NOT EXISTS erp_style_codes TEXT[] DEFAULT '{}', "
|
||||
"ADD COLUMN IF NOT EXISTS platform_product_ids JSONB DEFAULT '{}'::jsonb"
|
||||
)
|
||||
cursor.execute(
|
||||
"SELECT s.style_name, s.brand, s.erp_codes, p.platform, p.item_ids "
|
||||
"FROM workflow_dynamic_styles s "
|
||||
"JOIN workflow_dynamic_style_platforms p ON p.style_id = s.id "
|
||||
"WHERE s.enabled AND p.enabled ORDER BY s.style_name, p.id"
|
||||
)
|
||||
grouped: dict[str, dict[str, Any]] = {}
|
||||
platform_keys = {
|
||||
label: key for key, label in LIFECYCLE_PLATFORM_LABELS.items()
|
||||
}
|
||||
for row in cursor.fetchall():
|
||||
style_name = " ".join(str(row.get("style_name") or "").split())
|
||||
if not style_name:
|
||||
continue
|
||||
item = grouped.setdefault(
|
||||
style_name,
|
||||
{
|
||||
"brand": str(row.get("brand") or "").strip(),
|
||||
"erp_codes": _clean_list(row.get("erp_codes")),
|
||||
"platform_product_ids": {
|
||||
key: [] for key in LIFECYCLE_PLATFORM_KEYS
|
||||
},
|
||||
},
|
||||
)
|
||||
key = platform_keys.get(str(row.get("platform") or "").strip())
|
||||
if not key:
|
||||
continue
|
||||
for product_id in _clean_list(row.get("item_ids")):
|
||||
if product_id not in item["platform_product_ids"][key]:
|
||||
item["platform_product_ids"][key].append(product_id)
|
||||
|
||||
for style_name, values in grouped.items():
|
||||
cursor.execute(
|
||||
"INSERT INTO cmt_styles "
|
||||
"(name, brand, erp_style_codes, platform_product_ids) "
|
||||
"VALUES (%s, %s, %s, %s::jsonb) "
|
||||
"ON CONFLICT (name) DO UPDATE SET "
|
||||
"brand = CASE WHEN COALESCE(BTRIM(cmt_styles.brand), '') = '' "
|
||||
"THEN EXCLUDED.brand ELSE cmt_styles.brand END, "
|
||||
"erp_style_codes = CASE "
|
||||
"WHEN COALESCE(array_length(cmt_styles.erp_style_codes, 1), 0) = 0 "
|
||||
"AND COALESCE(cmt_styles.platform_product_ids, '{}'::jsonb) = '{}'::jsonb "
|
||||
"THEN EXCLUDED.erp_style_codes ELSE cmt_styles.erp_style_codes END, "
|
||||
"platform_product_ids = CASE "
|
||||
"WHEN COALESCE(cmt_styles.platform_product_ids, '{}'::jsonb) = '{}'::jsonb "
|
||||
"THEN EXCLUDED.platform_product_ids "
|
||||
"ELSE cmt_styles.platform_product_ids END, "
|
||||
"updated_at = NOW()",
|
||||
(
|
||||
style_name,
|
||||
values["brand"],
|
||||
values["erp_codes"],
|
||||
json.dumps(values["platform_product_ids"], ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
cursor.execute(
|
||||
"INSERT INTO workflow_dynamic_config_migrations "
|
||||
"(migration_key, source, record_count) VALUES (%s, %s, %s) "
|
||||
"ON CONFLICT (migration_key) DO NOTHING",
|
||||
(
|
||||
LIFECYCLE_IDENTITY_MIGRATION_KEY,
|
||||
"workflow_dynamic_styles + workflow_dynamic_style_platforms",
|
||||
len(grouped),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _public_style(
|
||||
row: Mapping[str, Any], platforms: list[Mapping[str, Any]]
|
||||
@@ -905,14 +1151,27 @@ class PostgresWorkflowConfigStore:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT s.*, p.id AS platform_id, p.platform, p.item_ids, "
|
||||
"p.persona_bitable_url, p.source_record_ids "
|
||||
"p.persona_bitable_url, p.source_record_ids, "
|
||||
"p.enabled AS platform_enabled "
|
||||
"FROM workflow_dynamic_styles s "
|
||||
"JOIN workflow_dynamic_style_platforms p ON p.style_id = s.id "
|
||||
"WHERE s.enabled AND p.enabled ORDER BY s.style_name, p.id"
|
||||
"WHERE s.enabled ORDER BY s.style_name, p.id"
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
cursor.execute("SELECT to_regclass('public.cmt_styles') AS table_name")
|
||||
cmt_table = cursor.fetchone() or {}
|
||||
if cmt_table.get("table_name"):
|
||||
cursor.execute(
|
||||
"SELECT id, name, brand, erp_style_codes, platform_product_ids "
|
||||
"FROM cmt_styles ORDER BY name, id"
|
||||
)
|
||||
lifecycle_rows = cursor.fetchall()
|
||||
else:
|
||||
lifecycle_rows = []
|
||||
personas: dict[int, dict[str, str]] = {}
|
||||
for row in rows:
|
||||
if row.get("platform_enabled") is False:
|
||||
continue
|
||||
style_personas = personas.setdefault(int(row["id"]), {})
|
||||
style_personas[str(row["platform"])] = str(
|
||||
row.get("persona_bitable_url") or ""
|
||||
@@ -943,10 +1202,10 @@ class PostgresWorkflowConfigStore:
|
||||
"style_analysis_bitable_url": row.get("style_analysis_bitable_url", ""),
|
||||
"style_content": row.get("style_content", ""),
|
||||
"destinations": _public_destinations(row),
|
||||
"enabled": True,
|
||||
"enabled": row.get("platform_enabled") is not False,
|
||||
}
|
||||
)
|
||||
return records
|
||||
return merge_lifecycle_style_records(records, lifecycle_rows)
|
||||
|
||||
def create(self, payload: Mapping[str, Any]) -> dict[str, Any]:
|
||||
values = normalize_style_config(payload)
|
||||
|
||||
@@ -79,7 +79,19 @@ def _lark_cli() -> str:
|
||||
return executable
|
||||
|
||||
|
||||
def read_source_records() -> list[dict[str, Any]]:
|
||||
def read_source_records(
|
||||
*,
|
||||
base_token: str = BASE_TOKEN,
|
||||
table_id: str = TABLE_ID,
|
||||
view_id: str = VIEW_ID,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Read all rows from the legacy Feishu configuration table.
|
||||
|
||||
The defaults preserve the original seed export behavior. Callers that
|
||||
need to audit a particular shared view can pass the resolved IDs instead
|
||||
of duplicating the pagination and field-mapping logic.
|
||||
"""
|
||||
|
||||
records: list[dict[str, Any]] = []
|
||||
offset = 0
|
||||
while True:
|
||||
@@ -88,11 +100,11 @@ def read_source_records() -> list[dict[str, Any]]:
|
||||
"base",
|
||||
"+record-list",
|
||||
"--base-token",
|
||||
BASE_TOKEN,
|
||||
base_token,
|
||||
"--table-id",
|
||||
TABLE_ID,
|
||||
table_id,
|
||||
"--view-id",
|
||||
VIEW_ID,
|
||||
view_id,
|
||||
"--offset",
|
||||
str(offset),
|
||||
"--limit",
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
"""将飞书款式 ID 表一次性合并到产品生命进程身份表。
|
||||
|
||||
这个脚本只负责初始化/校准 ``cmt_styles`` 的身份字段:
|
||||
|
||||
* 按款式聚合旧表的一行多平台记录;
|
||||
* 将 ERP 款式编码和四个平台商品编码去重后写入生命进程;
|
||||
* 保留生命进程已有的 NFC、图片、类目、市场标签等字段;
|
||||
* 空的源字段不会清空已有身份,避免旧表不完整行破坏已维护数据;
|
||||
* 默认只预览,只有显式传入 ``--apply`` 才写入数据库。
|
||||
|
||||
运行时采集仍从 PostgreSQL 的产品生命进程读取,不会把飞书旧表重新
|
||||
变成第二个运行时配置源。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from gyxx_flow.migration.feishu_style_config_seed import (
|
||||
BASE_TOKEN as SOURCE_BASE_TOKEN,
|
||||
)
|
||||
from gyxx_flow.migration.feishu_style_config_seed import (
|
||||
TABLE_ID as SOURCE_TABLE_ID,
|
||||
)
|
||||
from gyxx_flow.migration.feishu_style_config_seed import (
|
||||
read_source_records,
|
||||
)
|
||||
|
||||
SOURCE_URL = (
|
||||
"https://bu0zgpibak.feishu.cn/base/"
|
||||
f"{SOURCE_BASE_TOKEN}?table={SOURCE_TABLE_ID}&view=vewT7L4JUr"
|
||||
)
|
||||
SOURCE_VIEW_ID = "vewT7L4JUr"
|
||||
|
||||
PLATFORM_KEYS = {
|
||||
"天猫": "tm",
|
||||
"京东旗舰店": "jd",
|
||||
"京东": "jd",
|
||||
"京东自营": "jd_self",
|
||||
"抖音": "dy",
|
||||
}
|
||||
KNOWN_PLATFORM_KEYS = ("tm", "jd", "jd_self", "dy")
|
||||
|
||||
|
||||
class FeishuLifecycleSyncError(RuntimeError):
|
||||
"""Raised when the source or lifecycle target cannot be safely synchronized."""
|
||||
|
||||
|
||||
def _clean_list(value: Any) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
parts = value
|
||||
else:
|
||||
parts = str(value).replace(",", ",").replace("\r", ",").replace("\n", ",").split(",")
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in parts:
|
||||
text = str(item or "").strip()
|
||||
if text and text not in seen:
|
||||
seen.add(text)
|
||||
result.append(text)
|
||||
return result
|
||||
|
||||
|
||||
def _clean_style_name(value: Any) -> str:
|
||||
return " ".join(str(value or "").split())
|
||||
|
||||
|
||||
@dataclass
|
||||
class StyleIdentity:
|
||||
name: str
|
||||
brand: str = ""
|
||||
erp_codes: list[str] = field(default_factory=list)
|
||||
platform_product_ids: dict[str, list[str]] = field(
|
||||
default_factory=lambda: {key: [] for key in KNOWN_PLATFORM_KEYS}
|
||||
)
|
||||
source_record_ids: list[str] = field(default_factory=list)
|
||||
unsupported_platform_ids: dict[str, list[str]] = field(default_factory=dict)
|
||||
|
||||
def add_unique(self, field_name: str, values: list[str]) -> None:
|
||||
target = getattr(self, field_name)
|
||||
for value in values:
|
||||
if value not in target:
|
||||
target.append(value)
|
||||
|
||||
|
||||
def build_sync_plan(source_records: list[Mapping[str, Any]]) -> tuple[list[StyleIdentity], dict[str, Any]]:
|
||||
"""Group the legacy per-platform rows into lifecycle style identities."""
|
||||
|
||||
grouped: dict[str, StyleIdentity] = {}
|
||||
skipped_rows: list[dict[str, str]] = []
|
||||
unknown_platform_rows: list[dict[str, Any]] = []
|
||||
|
||||
for entry in source_records:
|
||||
config = entry.get("config") or {}
|
||||
if not isinstance(config, Mapping):
|
||||
skipped_rows.append({"record_id": str(entry.get("source_record_id") or ""), "reason": "配置结构无效"})
|
||||
continue
|
||||
source_record_id = str(entry.get("source_record_id") or "").strip()
|
||||
name = _clean_style_name(config.get("style_name"))
|
||||
if not name:
|
||||
skipped_rows.append({"record_id": source_record_id, "reason": "款式名称为空"})
|
||||
continue
|
||||
|
||||
identity = grouped.setdefault(name, StyleIdentity(name=name))
|
||||
brand = str(config.get("brand") or "").strip()
|
||||
if brand and not identity.brand:
|
||||
identity.brand = brand
|
||||
identity.add_unique("erp_codes", _clean_list(config.get("erp_codes")))
|
||||
if source_record_id and source_record_id not in identity.source_record_ids:
|
||||
identity.source_record_ids.append(source_record_id)
|
||||
|
||||
platform = str(config.get("platform") or "").strip()
|
||||
item_ids = _clean_list(config.get("item_ids"))
|
||||
key = PLATFORM_KEYS.get(platform)
|
||||
if key:
|
||||
for item_id in item_ids:
|
||||
if item_id not in identity.platform_product_ids[key]:
|
||||
identity.platform_product_ids[key].append(item_id)
|
||||
elif platform and item_ids:
|
||||
unknown = identity.unsupported_platform_ids.setdefault(platform, [])
|
||||
for item_id in item_ids:
|
||||
if item_id not in unknown:
|
||||
unknown.append(item_id)
|
||||
unknown_platform_rows.append(
|
||||
{
|
||||
"style": name,
|
||||
"platform": platform,
|
||||
"item_count": len(item_ids),
|
||||
"record_id": source_record_id,
|
||||
}
|
||||
)
|
||||
|
||||
styles = [grouped[name] for name in sorted(grouped)]
|
||||
report = {
|
||||
"source_url": SOURCE_URL,
|
||||
"source_records": len(source_records),
|
||||
"named_styles": len(styles),
|
||||
"skipped_rows": skipped_rows,
|
||||
"unknown_platform_rows": unknown_platform_rows,
|
||||
"styles_without_erp": [style.name for style in styles if not style.erp_codes],
|
||||
"styles_without_supported_platform_id": [
|
||||
style.name
|
||||
for style in styles
|
||||
if not any(style.platform_product_ids.values())
|
||||
],
|
||||
"collectable_styles": [
|
||||
style.name
|
||||
for style in styles
|
||||
if style.erp_codes and any(style.platform_product_ids.values())
|
||||
],
|
||||
}
|
||||
return styles, report
|
||||
|
||||
|
||||
def _connect(environment: Mapping[str, str] | None = None) -> psycopg.Connection[Any]:
|
||||
env = environment if environment is not None else os.environ
|
||||
dsn = str(env.get("GYXX_POSTGRES_DSN", "")).strip()
|
||||
if dsn:
|
||||
return psycopg.connect(dsn, row_factory=dict_row)
|
||||
required = ("PG_HOST", "PG_PORT", "PG_DB", "PG_USER", "PG_PASSWORD")
|
||||
missing = [name for name in required if not str(env.get(name, "")).strip()]
|
||||
if missing:
|
||||
raise FeishuLifecycleSyncError(
|
||||
"缺少 PostgreSQL 配置: " + ", ".join(missing)
|
||||
)
|
||||
try:
|
||||
port = int(env["PG_PORT"])
|
||||
except ValueError as exc:
|
||||
raise FeishuLifecycleSyncError("PG_PORT 必须是整数") from exc
|
||||
return psycopg.connect(
|
||||
host=env["PG_HOST"].strip(),
|
||||
port=port,
|
||||
dbname=env["PG_DB"].strip(),
|
||||
user=env["PG_USER"].strip(),
|
||||
password=env["PG_PASSWORD"],
|
||||
row_factory=dict_row,
|
||||
)
|
||||
|
||||
|
||||
def _require_target_schema(conn: psycopg.Connection[Any]) -> None:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_schema = 'public' AND table_name = 'cmt_styles'"
|
||||
)
|
||||
columns = {str(row["column_name"]) for row in cursor.fetchall()}
|
||||
required = {"name", "brand", "erp_style_codes", "platform_product_ids"}
|
||||
missing = sorted(required - columns)
|
||||
if missing:
|
||||
raise FeishuLifecycleSyncError(
|
||||
"cmt_styles 缺少身份字段: " + ", ".join(missing)
|
||||
)
|
||||
|
||||
|
||||
def _existing_rows(
|
||||
conn: psycopg.Connection[Any], names: list[str]
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
if not names:
|
||||
return {}
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT name, brand, erp_style_codes, platform_product_ids "
|
||||
"FROM cmt_styles WHERE name = ANY(%s)",
|
||||
(names,),
|
||||
)
|
||||
return {str(row["name"]): dict(row) for row in cursor.fetchall()}
|
||||
|
||||
|
||||
def _json_platform_ids(value: Any) -> dict[str, list[str]]:
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
value = {}
|
||||
if not isinstance(value, Mapping):
|
||||
value = {}
|
||||
# ProductLifecycleServiceImpl strictly rejects unknown JSON keys. The
|
||||
# legacy database may contain Chinese platform labels, so only carry the
|
||||
# canonical four keys into the lifecycle record.
|
||||
return {
|
||||
key: _clean_list(value.get(key))
|
||||
for key in KNOWN_PLATFORM_KEYS
|
||||
}
|
||||
|
||||
|
||||
def _merge_target_row(
|
||||
style: StyleIdentity, existing: Mapping[str, Any] | None
|
||||
) -> dict[str, Any]:
|
||||
existing = existing or {}
|
||||
erp_codes = _clean_list(existing.get("erp_style_codes"))
|
||||
if style.erp_codes:
|
||||
erp_codes = list(style.erp_codes)
|
||||
|
||||
platform_ids = _json_platform_ids(existing.get("platform_product_ids"))
|
||||
for key in KNOWN_PLATFORM_KEYS:
|
||||
if style.platform_product_ids[key]:
|
||||
platform_ids[key] = list(style.platform_product_ids[key])
|
||||
|
||||
return {
|
||||
"name": style.name,
|
||||
"brand": style.brand or str(existing.get("brand") or "").strip(),
|
||||
"erp_style_codes": erp_codes,
|
||||
"platform_product_ids": platform_ids,
|
||||
"existing": existing,
|
||||
}
|
||||
|
||||
|
||||
def build_target_rows(
|
||||
conn: psycopg.Connection[Any], styles: list[StyleIdentity]
|
||||
) -> list[dict[str, Any]]:
|
||||
existing = _existing_rows(conn, [style.name for style in styles])
|
||||
return [_merge_target_row(style, existing.get(style.name)) for style in styles]
|
||||
|
||||
|
||||
def apply_target_rows(
|
||||
conn: psycopg.Connection[Any], rows: list[Mapping[str, Any]]
|
||||
) -> int:
|
||||
with conn.cursor() as cursor:
|
||||
for row in rows:
|
||||
cursor.execute(
|
||||
"INSERT INTO cmt_styles "
|
||||
"(name, brand, erp_style_codes, platform_product_ids) "
|
||||
"VALUES (%s, %s, %s, %s::jsonb) "
|
||||
"ON CONFLICT (name) DO UPDATE SET "
|
||||
"brand = CASE WHEN EXCLUDED.brand <> '' "
|
||||
"THEN EXCLUDED.brand ELSE cmt_styles.brand END, "
|
||||
"erp_style_codes = CASE "
|
||||
"WHEN COALESCE(array_length(EXCLUDED.erp_style_codes, 1), 0) > 0 "
|
||||
"THEN EXCLUDED.erp_style_codes ELSE cmt_styles.erp_style_codes END, "
|
||||
"platform_product_ids = EXCLUDED.platform_product_ids, "
|
||||
"updated_at = CURRENT_TIMESTAMP",
|
||||
(
|
||||
row["name"],
|
||||
row["brand"],
|
||||
row["erp_style_codes"],
|
||||
json.dumps(row["platform_product_ids"], ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _print_report(report: Mapping[str, Any], rows: list[Mapping[str, Any]]) -> None:
|
||||
print(f"[INFO] 飞书源记录: {report['source_records']}")
|
||||
print(f"[INFO] 有效款式: {report['named_styles']}")
|
||||
print(f"[INFO] 将合并到 cmt_styles: {len(rows)}")
|
||||
print(f"[INFO] 可采集款式: {len(report['collectable_styles'])}")
|
||||
print(f"[INFO] ERP 缺失款式: {len(report['styles_without_erp'])}")
|
||||
print(
|
||||
"[INFO] 不支持的平台行: "
|
||||
f"{len(report['unknown_platform_rows'])}(保留在旧动态平台配置,不写入四平台身份 JSON)"
|
||||
)
|
||||
if report["skipped_rows"]:
|
||||
print(f"[WARN] 跳过空款式行: {len(report['skipped_rows'])}")
|
||||
for row in rows[:5]:
|
||||
print(
|
||||
f" - {row['name']}: erp={row['erp_style_codes']} "
|
||||
f"platform_ids={row['platform_product_ids']}"
|
||||
)
|
||||
|
||||
|
||||
def run(*, apply: bool, environment: Mapping[str, str] | None = None) -> int:
|
||||
source_records = read_source_records(
|
||||
base_token=SOURCE_BASE_TOKEN,
|
||||
table_id=SOURCE_TABLE_ID,
|
||||
view_id=SOURCE_VIEW_ID,
|
||||
)
|
||||
styles, report = build_sync_plan(source_records)
|
||||
with _connect(environment) as conn:
|
||||
_require_target_schema(conn)
|
||||
rows = build_target_rows(conn, styles)
|
||||
_print_report(report, rows)
|
||||
if not apply:
|
||||
print("[DRY-RUN] 未写入数据库;使用 --apply 执行同步")
|
||||
return 0
|
||||
applied = apply_target_rows(conn, rows)
|
||||
conn.commit()
|
||||
print(f"[OK] 产品生命进程 cmt_styles 已同步: {applied} 行")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="飞书款式配置同步到产品生命进程")
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="执行数据库写入;默认仅预览",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
return run(apply=args.apply)
|
||||
except (FeishuLifecycleSyncError, psycopg.Error) as exc:
|
||||
print(f"[FAIL] 飞书款式同步失败: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -18,7 +18,7 @@ from scrapling.fetchers import DynamicFetcher
|
||||
try:
|
||||
from . import collect_erp_yesterday_metrics as erp
|
||||
from .collect_retry_utils import retry_step
|
||||
from .config.style_config_loader import StyleConfigLoader
|
||||
from .config.style_config_loader import StyleConfigLoader, has_complete_style_identity
|
||||
from .db import (
|
||||
get_conn,
|
||||
init_schema,
|
||||
@@ -35,7 +35,7 @@ try:
|
||||
except ImportError: # Direct script execution from the module root.
|
||||
import collect_erp_yesterday_metrics as erp
|
||||
from collect_retry_utils import retry_step
|
||||
from config.style_config_loader import StyleConfigLoader
|
||||
from config.style_config_loader import StyleConfigLoader, has_complete_style_identity
|
||||
from db import get_conn, init_schema, upsert_erp_all_shop_style_daily_metrics
|
||||
from erp_login_product_analysis import (
|
||||
DEFAULT_CONFIG,
|
||||
@@ -80,14 +80,15 @@ def build_style_plans(payload: Mapping[str, Any]) -> tuple[StylePlan, ...]:
|
||||
code_owners: dict[str, str] = {}
|
||||
for raw_name, raw_config in (payload.get("styles") or {}).items():
|
||||
style_name = str(raw_name).strip()
|
||||
config = raw_config or {}
|
||||
codes = tuple(
|
||||
dict.fromkeys(
|
||||
str(code).strip()
|
||||
for code in (raw_config or {}).get("erp_codes") or ()
|
||||
for code in config.get("erp_codes") or ()
|
||||
if str(code).strip()
|
||||
)
|
||||
)
|
||||
if not style_name or not codes:
|
||||
if not style_name or not codes or not has_complete_style_identity(config):
|
||||
continue
|
||||
for code in codes:
|
||||
previous = code_owners.setdefault(code, style_name)
|
||||
|
||||
@@ -174,7 +174,7 @@ def load_styles(path: Path) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
def merge_styles_with_dynamic_config(styles: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""[已废弃] 飞书为唯一来源,本地 styles_input.json 退为兜底。保留仅为兼容旧调用。"""
|
||||
"""[已废弃] 产品生命进程是 ERP 身份唯一来源;保留仅为兼容旧调用。"""
|
||||
return styles
|
||||
|
||||
|
||||
@@ -182,64 +182,67 @@ def load_erp_styles(
|
||||
loader: StyleConfigLoader | None,
|
||||
fallback_path: Path,
|
||||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
"""飞书为唯一来源,本地 styles_input.json 仅在飞书读取失败时回退。
|
||||
"""只从产品生命进程读取 ERP 采集款式,身份源不可用时失败闭环。
|
||||
|
||||
返回 (styles, skip_report):
|
||||
styles = [{"style_name", "erp_style_codes", "brand"}]
|
||||
skip_report = {"source": "lark"|"fallback_json", "collectable": [...], "skipped": [{style, reason}], "brand_map": {...}}
|
||||
skip_report = {"source": "product_lifecycle"|"lifecycle_unavailable", ...}
|
||||
brand 字段让 collect_in_page 按品牌选店铺 (光影行星旗下店铺 vs ozko旗舰店)。
|
||||
"""
|
||||
if loader is not None:
|
||||
try:
|
||||
data = loader.get_erp_daily_styles()
|
||||
payload = loader.load()
|
||||
brand_map = data.get("brand_map", {})
|
||||
styles: list[dict[str, Any]] = []
|
||||
for name in data["collectable"]:
|
||||
codes = payload["styles"][name]["erp_codes"]
|
||||
styles.append({
|
||||
"style_name": name,
|
||||
"erp_style_codes": [str(c).strip() for c in codes if str(c).strip()],
|
||||
"brand": brand_map.get(name) or DEFAULT_BRAND,
|
||||
})
|
||||
return styles, {
|
||||
"source": "lark",
|
||||
"collectable": data["collectable"],
|
||||
"skipped": data["skipped"],
|
||||
"brand_map": brand_map,
|
||||
}
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 飞书读取失败,回退本地 {fallback_path}: {exc}")
|
||||
|
||||
styles = load_styles(fallback_path)
|
||||
for s in styles:
|
||||
s.setdefault("brand", DEFAULT_BRAND)
|
||||
return styles, {
|
||||
"source": "fallback_json",
|
||||
"collectable": [],
|
||||
"skipped": [],
|
||||
"brand_map": {s["style_name"]: s.get("brand", DEFAULT_BRAND) for s in styles},
|
||||
}
|
||||
del fallback_path # 保留参数以兼容旧调用,商品身份不再从本地文件回退。
|
||||
if loader is None:
|
||||
return [], {
|
||||
"source": "lifecycle_unavailable",
|
||||
"collectable": [],
|
||||
"skipped": [{"style": "*", "reason": "产品生命进程身份源不可用"}],
|
||||
"brand_map": {},
|
||||
}
|
||||
try:
|
||||
data = loader.get_erp_daily_styles()
|
||||
payload = loader.load()
|
||||
brand_map = data.get("brand_map", {})
|
||||
styles: list[dict[str, Any]] = []
|
||||
for name in data["collectable"]:
|
||||
codes = payload["styles"][name]["erp_codes"]
|
||||
styles.append({
|
||||
"style_name": name,
|
||||
"erp_style_codes": [str(c).strip() for c in codes if str(c).strip()],
|
||||
"brand": brand_map.get(name) or DEFAULT_BRAND,
|
||||
})
|
||||
return styles, {
|
||||
"source": "product_lifecycle",
|
||||
"collectable": data["collectable"],
|
||||
"skipped": data["skipped"],
|
||||
"brand_map": brand_map,
|
||||
}
|
||||
except Exception as exc:
|
||||
print(f"[FAIL] 产品生命进程身份源读取失败,不启动 ERP 采集: {exc}")
|
||||
return [], {
|
||||
"source": "lifecycle_unavailable",
|
||||
"collectable": [],
|
||||
"skipped": [{"style": "*", "reason": "产品生命进程身份源不可用"}],
|
||||
"brand_map": {},
|
||||
}
|
||||
|
||||
|
||||
def print_skip_summary(skip_report: dict[str, Any]) -> None:
|
||||
src = skip_report.get("source", "?")
|
||||
if not skip_report.get("skipped"):
|
||||
if src == "lark":
|
||||
if src == "product_lifecycle":
|
||||
print(f"[ERP] 来源={src}, 采集 {len(skip_report['collectable'])} 款, 跳过 0 款")
|
||||
else:
|
||||
print(f"[ERP] 来源={src}, 跳过清单未生成 (回退路径不带 reason)")
|
||||
return
|
||||
print(
|
||||
f"[ERP] 来源={src}, 采集 {len(skip_report['collectable'])} 款, "
|
||||
f"跳过 {len(skip_report['skipped'])} 款 (缺 ERP 编码 / TM 分组 ID / 飞书表地址)"
|
||||
f"跳过 {len(skip_report['skipped'])} 款 (缺 ERP 编码 / 平台商品编码 / 飞书表地址)"
|
||||
)
|
||||
for s in skip_report["skipped"]:
|
||||
print(f" - {s['style']}: {s['reason']}")
|
||||
|
||||
|
||||
def write_erp_skip_report(skip_report: dict[str, Any], date_str: str) -> Path | None:
|
||||
if skip_report.get("source") != "lark":
|
||||
if skip_report.get("source") not in {"product_lifecycle", "lifecycle_unavailable"}:
|
||||
return None
|
||||
SKIP_REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
out_path = SKIP_REPORT_DIR / f"erp_skipped_{date_str}.json"
|
||||
|
||||
@@ -17,7 +17,7 @@ import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from typing import Any, Callable, Mapping
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from gyxx_flow.adapters.workflow_config import runtime_workflow_configs
|
||||
@@ -56,6 +56,31 @@ PLATFORM_MAP = {
|
||||
"抖音": "dy",
|
||||
}
|
||||
|
||||
PRODUCT_ID_FIELDS = {
|
||||
"tm": "item_ids",
|
||||
"jd": "spus",
|
||||
"jd_self": "skus",
|
||||
"dy": "item_ids",
|
||||
}
|
||||
|
||||
|
||||
def has_complete_style_identity(
|
||||
cfg: Mapping[str, Any], platform: str | None = None
|
||||
) -> bool:
|
||||
"""判断款式是否具备商品采集所需的统一身份。"""
|
||||
|
||||
if not cfg.get("erp_codes"):
|
||||
return False
|
||||
if not any(
|
||||
bool(cfg.get(name, {}).get(ids_key))
|
||||
for name, ids_key in PRODUCT_ID_FIELDS.items()
|
||||
):
|
||||
return False
|
||||
if platform is None:
|
||||
return True
|
||||
ids_key = PRODUCT_ID_FIELDS.get(platform)
|
||||
return bool(ids_key and cfg.get(platform, {}).get(ids_key))
|
||||
|
||||
|
||||
class StyleConfigError(Exception):
|
||||
pass
|
||||
@@ -528,11 +553,11 @@ class StyleConfigLoader:
|
||||
return sorted(self.load()["styles"].keys())
|
||||
|
||||
def get_tm_style_id_map(self) -> dict[str, list[str]]:
|
||||
"""每日天猫采集列表:_aggregate 已保证满足四条件(erp/款式/item_ids/销量表)。"""
|
||||
"""天猫商品 ID 映射;只有完整身份的款式才进入采集。"""
|
||||
return {
|
||||
name: cfg["tm"]["item_ids"]
|
||||
for name, cfg in self.load()["styles"].items()
|
||||
if cfg["tm"]["item_ids"]
|
||||
if self._has_complete_identity(cfg, "tm")
|
||||
}
|
||||
|
||||
def get_tm_persona_bitable_map(self) -> dict[str, dict[str, str]]:
|
||||
@@ -540,7 +565,7 @@ class StyleConfigLoader:
|
||||
只返回同时有 tm item_ids 和 persona_bitable 的款式(三者同时存在才采)。"""
|
||||
result: dict[str, dict[str, str]] = {}
|
||||
for name, cfg in self.load()["styles"].items():
|
||||
if not cfg["tm"]["item_ids"]:
|
||||
if not self._has_complete_identity(cfg, "tm"):
|
||||
continue
|
||||
persona = cfg.get("persona_bitable") or {}
|
||||
if persona.get("base_token") and persona.get("table_id"):
|
||||
@@ -556,7 +581,7 @@ class StyleConfigLoader:
|
||||
return {
|
||||
name: cfg.get("brand", "")
|
||||
for name, cfg in self.load()["styles"].items()
|
||||
if cfg["tm"]["item_ids"]
|
||||
if self._has_complete_identity(cfg, "tm")
|
||||
}
|
||||
|
||||
def get_jd_spu_groups(self) -> list[dict]:
|
||||
@@ -564,7 +589,7 @@ class StyleConfigLoader:
|
||||
return [
|
||||
{"style": name, "spus": cfg["jd"]["spus"]}
|
||||
for name, cfg in self.load()["styles"].items()
|
||||
if cfg["jd"]["spus"]
|
||||
if self._has_complete_identity(cfg, "jd")
|
||||
]
|
||||
|
||||
def get_dy_product_groups(self) -> dict[str, list[str]]:
|
||||
@@ -572,7 +597,7 @@ class StyleConfigLoader:
|
||||
return {
|
||||
name: cfg["dy"]["item_ids"]
|
||||
for name, cfg in self.load()["styles"].items()
|
||||
if cfg["dy"]["item_ids"]
|
||||
if self._has_complete_identity(cfg, "dy")
|
||||
}
|
||||
|
||||
def get_dy_persona_bitable_map(self) -> dict[str, dict[str, str]]:
|
||||
@@ -580,7 +605,7 @@ class StyleConfigLoader:
|
||||
只返回同时有 dy item_ids 和 dy_persona_bitable 的款式。"""
|
||||
result: dict[str, dict[str, str]] = {}
|
||||
for name, cfg in self.load()["styles"].items():
|
||||
if not cfg["dy"]["item_ids"]:
|
||||
if not self._has_complete_identity(cfg, "dy"):
|
||||
continue
|
||||
persona = cfg.get("dy_persona_bitable") or {}
|
||||
if persona.get("base_token") and persona.get("table_id"):
|
||||
@@ -596,7 +621,7 @@ class StyleConfigLoader:
|
||||
只返回同时有 jd spus 和 jd_persona_bitable 的款式。"""
|
||||
result: dict[str, dict[str, str]] = {}
|
||||
for name, cfg in self.load()["styles"].items():
|
||||
if not cfg["jd"]["spus"]:
|
||||
if not self._has_complete_identity(cfg, "jd"):
|
||||
continue
|
||||
persona = cfg.get("jd_persona_bitable") or {}
|
||||
if persona.get("base_token") and persona.get("table_id"):
|
||||
@@ -611,6 +636,8 @@ class StyleConfigLoader:
|
||||
"""款式 -> 平台单品分析目标表。"""
|
||||
result: dict[str, dict[str, str]] = {}
|
||||
for name, cfg in self.load()["styles"].items():
|
||||
if not self._has_complete_identity(cfg):
|
||||
continue
|
||||
target = cfg.get("style_analysis_bitable") or {}
|
||||
if target.get("base_token") and target.get("table_id"):
|
||||
result[name] = {
|
||||
@@ -625,20 +652,22 @@ class StyleConfigLoader:
|
||||
return [
|
||||
{"style_name": name, "skus": cfg["jd_self"]["skus"]}
|
||||
for name, cfg in self.load()["styles"].items()
|
||||
if cfg["jd_self"]["skus"]
|
||||
if self._has_complete_identity(cfg, "jd_self")
|
||||
]
|
||||
|
||||
def get_erp_styles_input(self) -> list[dict]:
|
||||
return [
|
||||
{"style_name": name, "erp_style_codes": cfg["erp_codes"]}
|
||||
for name, cfg in self.load()["styles"].items()
|
||||
if cfg["erp_codes"]
|
||||
if self._has_complete_identity(cfg)
|
||||
]
|
||||
|
||||
def get_bitable_style_map(self) -> dict:
|
||||
"""生成与 bitable_style_map.json 兼容的结构。"""
|
||||
result: dict = {}
|
||||
for name, cfg in self.load()["styles"].items():
|
||||
if not self._has_complete_identity(cfg):
|
||||
continue
|
||||
sales = cfg.get("sales_bitable") or {}
|
||||
if not sales.get("base_token") or not sales.get("table_id"):
|
||||
continue
|
||||
@@ -665,6 +694,8 @@ class StyleConfigLoader:
|
||||
"""生成与 bitable_main_image_map.json 兼容的结构。"""
|
||||
result: dict = {}
|
||||
for name, cfg in self.load()["styles"].items():
|
||||
if not self._has_complete_identity(cfg, "tm"):
|
||||
continue
|
||||
main = cfg.get("main_image_bitable") or {}
|
||||
if not main.get("base_token") or not main.get("table_id"):
|
||||
continue
|
||||
@@ -686,38 +717,55 @@ class StyleConfigLoader:
|
||||
cfg.get("sales_bitable", {}).get("table_id")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _has_any_product_id(cfg: dict) -> bool:
|
||||
return any(
|
||||
bool(cfg.get(platform, {}).get(ids_key))
|
||||
for platform, ids_key in PRODUCT_ID_FIELDS.items()
|
||||
)
|
||||
|
||||
def _has_complete_identity(self, cfg: dict, platform: str | None = None) -> bool:
|
||||
"""商品采集统一门:ERP 款式编码 + 至少一个平台商品编码。"""
|
||||
return has_complete_style_identity(cfg, platform)
|
||||
|
||||
def _identity_missing(self, cfg: dict) -> list[str]:
|
||||
missing: list[str] = []
|
||||
if not cfg.get("erp_codes"):
|
||||
missing.append("ERP款式编码")
|
||||
if not self._has_any_product_id(cfg):
|
||||
missing.append("平台商品编码")
|
||||
return missing
|
||||
|
||||
def get_daily_styles(self) -> dict:
|
||||
"""每日采集门 (v2.2):
|
||||
- 款式级门: erp_codes ∩ sales_bitable 都非空 (从任意记录收集)
|
||||
- 平台级门: 该平台分组 item_ids 非空
|
||||
"""每日商品采集门:ERP 编码、销量目标和平台商品编码必须完整。
|
||||
|
||||
返回 {tm/jd/dy: {collectable: [...], skipped: [{style, reason}]}}。
|
||||
skipped 仅记录「款式级门通过、平台级缺 IDs」的款式;款式级失败不计入 skipped
|
||||
(因为该款式没有任何平台可采)。
|
||||
incomplete style 永远不会进入 collectable,并记录具体缺失字段。
|
||||
"""
|
||||
payload = self.load()
|
||||
result = {p: {"collectable": [], "skipped": []} for p in ("tm", "jd", "dy")}
|
||||
for name, cfg in payload["styles"].items():
|
||||
style_ok = bool(cfg.get("erp_codes")) and self._cfg_has_sales_bitable(cfg)
|
||||
if not style_ok:
|
||||
continue # 款式级失败,无任何平台可采
|
||||
identity_missing = self._identity_missing(cfg)
|
||||
for plat in ("tm", "jd", "dy"):
|
||||
ids_key = "spus" if plat == "jd" else "item_ids"
|
||||
if cfg.get(plat, {}).get(ids_key):
|
||||
result[plat]["collectable"].append(name)
|
||||
else:
|
||||
missing = list(identity_missing)
|
||||
if not self._cfg_has_sales_bitable(cfg):
|
||||
missing.append("销量目标地址")
|
||||
if not cfg.get(plat, {}).get(ids_key):
|
||||
missing.append(f"{plat}商品编码")
|
||||
if missing:
|
||||
result[plat]["skipped"].append(
|
||||
{"style": name, "reason": f"缺 {plat} 商品链接ID"}
|
||||
{"style": name, "reason": "缺 " + "/".join(dict.fromkeys(missing))}
|
||||
)
|
||||
else:
|
||||
result[plat]["collectable"].append(name)
|
||||
return result
|
||||
|
||||
def get_erp_daily_styles(self) -> dict:
|
||||
"""ERP daily 门:款式 + ERP 编码 + TM 分组有 item_ids + 飞书销量地址 都要有。
|
||||
"""ERP daily 门:ERP 编码、至少一个平台商品编码和销量目标都要有。
|
||||
|
||||
返回 {collectable, skipped, brand_map: {style: brand}}。
|
||||
ERP 一次跑覆盖 jd/dy/tm 三平台(用同一组 erp_codes 查三个店铺),
|
||||
所以 jd/dy 没商品 ID 也能跑,但 TM 分组必须有 item_ids
|
||||
(飞书表格里 TM 分组是「基础分组」, 款式只要在 TM 上架就算款式存在)。
|
||||
ERP 一次跑覆盖 jd/dy/tm 三个平台,平台商品编码只用于确认款式已完成商品身份配置。
|
||||
brand_map 给 collect_erp_yesterday_metrics.py 按品牌选店铺
|
||||
(光影行星旗下店铺 vs ozko旗舰店)。
|
||||
"""
|
||||
@@ -726,36 +774,31 @@ class StyleConfigLoader:
|
||||
skipped: list[dict] = []
|
||||
brand_map: dict[str, str] = {}
|
||||
for name, cfg in payload["styles"].items():
|
||||
has_tm = bool(cfg.get("tm", {}).get("item_ids"))
|
||||
if cfg.get("erp_codes") and has_tm and self._cfg_has_sales_bitable(cfg):
|
||||
missing = self._identity_missing(cfg)
|
||||
if not self._cfg_has_sales_bitable(cfg):
|
||||
missing.append("销量目标地址")
|
||||
if not missing:
|
||||
collectable.append(name)
|
||||
brand_map[name] = cfg.get("brand") or "光影行星"
|
||||
else:
|
||||
miss = []
|
||||
if not cfg.get("erp_codes"):
|
||||
miss.append("ERP款式编码")
|
||||
if not has_tm:
|
||||
miss.append("天猫分组商品ID")
|
||||
if not self._cfg_has_sales_bitable(cfg):
|
||||
miss.append("飞书多维表格地址")
|
||||
skipped.append({"style": name, "reason": "缺 " + "/".join(miss)})
|
||||
skipped.append({"style": name, "reason": "缺 " + "/".join(dict.fromkeys(missing))})
|
||||
return {"collectable": collectable, "skipped": skipped, "brand_map": brand_map}
|
||||
|
||||
def get_jd_self_styles(self) -> list[str]:
|
||||
"""京东自营周汇总门:仅 SKU 非空即可 (用户确认保持现状)。"""
|
||||
"""京东自营周汇总门:ERP 编码和京东自营 SKU 都要有。"""
|
||||
return [
|
||||
name
|
||||
for name, cfg in self.load()["styles"].items()
|
||||
if cfg.get("jd_self", {}).get("skus")
|
||||
if self._has_complete_identity(cfg, "jd_self")
|
||||
]
|
||||
|
||||
def get_main_image_styles(self) -> list[str]:
|
||||
"""主图门:天猫分组 + 主图飞书多维表格地址 同时存在。"""
|
||||
"""天猫主图门:完整身份、天猫商品编码和主图目标地址同时存在。"""
|
||||
payload = self.load()
|
||||
result: list[str] = []
|
||||
for name, cfg in payload["styles"].items():
|
||||
main = cfg.get("main_image_bitable") or {}
|
||||
if main.get("base_token") and main.get("table_id"):
|
||||
if self._has_complete_identity(cfg, "tm") and main.get("base_token") and main.get("table_id"):
|
||||
result.append(name)
|
||||
return result
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
写: check_nine_day_decline.py (去重)
|
||||
读: check_nine_day_decline.py._already_notified
|
||||
dim_style
|
||||
写: db/sync_dim_style.py (人工触发,全量扫 4 个数据源)
|
||||
写: db/sync_dim_style.py (人工触发,投影产品生命进程身份)
|
||||
读: bitable 同步 / 业务查询
|
||||
fact_platform_xlsx
|
||||
写: 暂无 caller (db/__init__.py.upsert_platform_xlsx_rows 已写好,import 脚本待补)
|
||||
|
||||
@@ -4,13 +4,10 @@
|
||||
-- 设计原则: schema 与 data/ 目录下的 JSON 产物 1:1 映射;upsert 幂等
|
||||
-- ============================================================================
|
||||
|
||||
-- 款式维度表:维护每款的 (各平台 SPU/ID + 京东自营 SKU + ERP 编码 + 飞书表地址)
|
||||
-- 数据源:
|
||||
-- jd_self_skus ← jd_self_inventory_sales_collector.STYLE_SKU_GROUPS
|
||||
-- jd_spus/dy_spus/tm_spus ← data/<plat>/<款>_<日期>/<款>_<日期>.json 的"匹配SPU"
|
||||
-- erp_codes ← daily_style_metrics.erp_style_codes (自动聚)
|
||||
-- bitable_weekly_* ← bitable_style_map.json (每款周报表地址)
|
||||
-- bitable_sku_master_* ← 飞书 SKU 主表(人工维护;表 base_token + table_id 填这里)
|
||||
-- 款式维度表:维护每款的 (各平台商品 ID + 京东自营 SKU + ERP 编码 + 飞书表地址)
|
||||
-- 身份主来源:产品生命进程 cmt_styles.platform_product_ids / erp_style_codes,
|
||||
-- 由 workflow_config.runtime_records() 统一投影;运行时不再从各采集器、JSON 产物反向拼接身份。
|
||||
-- bitable_weekly_* 仍由 bitable_style_map.json 派生;bitable_sku_master_* 仍由飞书 SKU 主表人工维护。
|
||||
CREATE TABLE IF NOT EXISTS dim_style (
|
||||
style_name TEXT PRIMARY KEY,
|
||||
jd_spus TEXT[], -- 京东 POP 商品 SPU
|
||||
@@ -226,7 +223,7 @@ ALTER TABLE dim_style ADD COLUMN IF NOT EXISTS bitable_sku_master_table TEXT;
|
||||
-- ============================================================================
|
||||
|
||||
-- dim_style
|
||||
COMMENT ON TABLE dim_style IS '款式维度表 — 维护每款 (各平台 SPU/ID + 京东自营 SKU + ERP 编码 + 飞书表地址 + 品类/上架日期/备注)。数据源: jd_self_skus、jd/dy/tm_spus、erp_codes、bitable_style_map.json、飞书 SKU 主表 (人工维护)。';
|
||||
COMMENT ON TABLE dim_style IS '款式维度表 — 维护每款 (各平台商品 ID + 京东自营 SKU + ERP 编码 + 飞书表地址 + 品类/上架日期/备注)。身份投影自产品生命进程,周报地址与 SKU 主表仍按各自来源维护。';
|
||||
COMMENT ON COLUMN dim_style.style_name IS '款式名 (主键),与 PRODUCT_STYLE_GROUPS 的 key 对齐';
|
||||
COMMENT ON COLUMN dim_style.jd_spus IS '京东 POP 商品 SPU 列表';
|
||||
COMMENT ON COLUMN dim_style.dy_spus IS '抖音商品 SPU 列表';
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""同步 dim_style:4 个数据源自动 upsert。
|
||||
|
||||
数据源:
|
||||
1) bitable_style_map.json -> bitable_weekly_url/base/table
|
||||
2) jd_self_inventory_sales_collector.STYLE_SKU_GROUPS -> jd_self_skus
|
||||
3) data/<plat>/<款>_<最近日期>/<款>_<最近日期>.json -> jd_spus / dy_spus / tm_spus
|
||||
4) daily_style_metrics.erp_style_codes -> erp_codes
|
||||
产品生命进程 + 动态配置运行时聚合 -> 商品编码、ERP 编码和飞书目标地址
|
||||
|
||||
历史文件、采集产物和硬编码仅保留在本文件的兼容函数中,不再作为同步入口,
|
||||
避免人工修改生命进程后被旧脚本反向覆盖。
|
||||
|
||||
SKU 主表的飞书地址 (bitable_sku_master_*) 由人工维护,不在自动同步范围。
|
||||
|
||||
@@ -25,7 +25,9 @@ from typing import Any, Iterable
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
sys.path.insert(0, str(PROJECT_ROOT.parents[2]))
|
||||
from runtime_paths import RAW_DATA_ROOT, runtime_script
|
||||
from gyxx_flow.adapters.workflow_config import runtime_workflow_configs # noqa: E402
|
||||
|
||||
from db import get_conn # noqa: E402
|
||||
|
||||
@@ -258,6 +260,63 @@ def _merge_records(
|
||||
return rows
|
||||
|
||||
|
||||
def load_lifecycle_records() -> list[dict[str, Any]]:
|
||||
"""读取生命进程身份与动态目标的统一运行时视图。"""
|
||||
|
||||
return runtime_workflow_configs()
|
||||
|
||||
|
||||
def _lifecycle_to_dim_rows(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
grouped: dict[str, dict[str, Any]] = {}
|
||||
platform_fields = {
|
||||
"天猫": "tm_spus",
|
||||
"京东旗舰店": "jd_spus",
|
||||
"京东": "jd_spus",
|
||||
"京东自营": "jd_self_skus",
|
||||
"抖音": "dy_spus",
|
||||
}
|
||||
for record in records:
|
||||
name = str(record.get("style_name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
row = grouped.setdefault(
|
||||
name,
|
||||
{
|
||||
"style_name": name,
|
||||
"jd_spus": [],
|
||||
"dy_spus": [],
|
||||
"tm_spus": [],
|
||||
"jd_self_skus": [],
|
||||
"erp_codes": [],
|
||||
"bitable_weekly_url": "",
|
||||
"bitable_weekly_base": "",
|
||||
"bitable_weekly_table": "",
|
||||
},
|
||||
)
|
||||
for code in record.get("erp_codes") or []:
|
||||
code = str(code).strip()
|
||||
if code and code not in row["erp_codes"]:
|
||||
row["erp_codes"].append(code)
|
||||
target_url = str(record.get("sales_bitable_url") or "").strip()
|
||||
if target_url and not row["bitable_weekly_url"]:
|
||||
row["bitable_weekly_url"] = target_url
|
||||
match = re.search(
|
||||
r"/base/([A-Za-z0-9]+)\?[^\s]*table=([A-Za-z0-9]+)",
|
||||
target_url,
|
||||
)
|
||||
if match:
|
||||
row["bitable_weekly_base"] = match.group(1)
|
||||
row["bitable_weekly_table"] = match.group(2)
|
||||
field = platform_fields.get(str(record.get("platform") or "").strip())
|
||||
if not field:
|
||||
continue
|
||||
for product_id in record.get("item_ids") or []:
|
||||
product_id = str(product_id).strip()
|
||||
if product_id and product_id not in row[field]:
|
||||
row[field].append(product_id)
|
||||
return list(grouped.values())
|
||||
|
||||
|
||||
def upsert_dim_style(conn, records: list[dict]) -> int:
|
||||
if not records:
|
||||
return 0
|
||||
@@ -311,21 +370,10 @@ def main():
|
||||
ap.add_argument("--dry-run", action="store_true", help="扫描不入库")
|
||||
args = ap.parse_args()
|
||||
|
||||
weekly = load_bitable_weekly()
|
||||
self_skus = load_jd_self_skus()
|
||||
spus_map = load_platform_spus()
|
||||
erp_map = load_erp_codes()
|
||||
runtime_records = load_lifecycle_records()
|
||||
records = _lifecycle_to_dim_rows(runtime_records)
|
||||
|
||||
# 款式名全集:合并 4 个数据源
|
||||
style_names: set[str] = set()
|
||||
style_names.update(weekly.keys())
|
||||
style_names.update(self_skus.keys())
|
||||
style_names.update(spus_map.keys())
|
||||
style_names.update(erp_map.keys())
|
||||
|
||||
records = _merge_records(sorted(style_names), weekly, self_skus, spus_map, erp_map)
|
||||
|
||||
print(f"[INFO] 数据源: weekly={len(weekly)} self_skus={len(self_skus)} spus={len(spus_map)} erp={len(erp_map)}")
|
||||
print(f"[INFO] 数据源: product_lifecycle_runtime={len(runtime_records)} 条")
|
||||
print(f"[INFO] 合并款式: {len(records)}")
|
||||
if args.dry_run:
|
||||
for r in records[:5]:
|
||||
|
||||
Reference in New Issue
Block a user