wangyunlong
6f1c193f4b
采集端与 data-hub 统一用 style_id (= dim_style.id) 关联款式，style_name 降级为显示冗余。
款式主键：
- 14 张采集表加 style_id BIGINT + style_match_method + *_style_id_idx 索引
- 一律逻辑外键：不建 FOREIGN KEY，早期 10 个 *_style_id_fk 在 schema.sql 里幂等 DROP
- loader 下发 dim_style.id；db 层 attach_style_ids / resolve_style_ids_by_product 兜底
- 新增 backfill_style_id.py 三段回填（款名 / 日报按商品 ID / 广告表按 spu_id）
- 新增视图 v_style_id_coverage、v_style_unmatched_product、v_style_unmatched_source
- 实库补录 199 个商品 ID 到 67 款 spus；日报 style_id 覆盖 49.1% -> 79.0%，
tm 广告 99.4% / jd 92.4% / dy 83.2%，其余 11 张 100%
评价采集下线：
- product_reviews 归档为 product_reviews_archived_20261101
- 差评改由 shop.{tmall,jd,dy}_negative_reviews.daily 采集，消费方读 v_negative_reviews
- 删除 import_product_reviews.py / orchestrate_review_collection.py 及相关命令与绑定
- dy_product_scraping 默认流程摘除评论采集，保留 --reviews-only
天猫广告：
- 迁移到独立表 tm_product_ad_daily_metrics，不再写 product_daily_metrics.raw_data
同批附带：
- 光合上传支持天猫新店 new_bag（仅黑曜石系列）
- 京东商智双月日历排除 -diff-month 补位格
- 万相主体搜索词覆盖 STYLE_SEARCH_KEYWORDS
- ERP 部分编码失败时用已成功编码销量兜底，避免整款归零
391 lines
12 KiB
Python
391 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from gyxx_flow.cli import EXIT_SUCCESS, main
|
|
from gyxx_flow.core.config import Settings
|
|
from gyxx_flow.script_catalog import ScriptCatalog, ScriptCatalogError
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def _write_registry(path: Path, commands: list[dict[str, object]]) -> None:
|
|
path.write_text(
|
|
json.dumps({"schema_version": 1, "commands": commands}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def test_catalog_exposes_only_explicit_commands(tmp_path: Path) -> None:
|
|
runtime = tmp_path / "content_marketing" / "runtime"
|
|
(runtime / "tools").mkdir(parents=True)
|
|
registered = runtime / "tools" / "run_job.py"
|
|
registered.write_text(
|
|
'if __name__ == "__main__":\n raise SystemExit(0)\n', encoding="utf-8"
|
|
)
|
|
(runtime / "undeclared.py").write_text(
|
|
'if __name__ == "__main__":\n raise SystemExit(0)\n', encoding="utf-8"
|
|
)
|
|
(runtime / "undeclared.bat").write_text("@echo off\n", encoding="utf-8")
|
|
(runtime / "undeclared.ps1").write_text("Write-Output ok\n", encoding="utf-8")
|
|
config = tmp_path / "commands.json"
|
|
_write_registry(
|
|
config,
|
|
[
|
|
{
|
|
"id": "content.test.run",
|
|
"module": "content_marketing",
|
|
"entry": "tools/run_job.py",
|
|
"kind": "python",
|
|
}
|
|
],
|
|
)
|
|
|
|
catalog = ScriptCatalog.load(config, roots={"content_marketing": runtime})
|
|
|
|
assert catalog.command_ids == ("content.test.run",)
|
|
assert catalog.script_ids == ("content_marketing:tools/run_job.py",)
|
|
command = catalog.get("content.test.run")
|
|
assert command.entry == "tools/run_job.py"
|
|
assert catalog.get("content_marketing:tools/run_job.py") is command
|
|
with pytest.raises(ScriptCatalogError, match="unknown command"):
|
|
catalog.get("content_marketing:undeclared.py")
|
|
|
|
|
|
def test_default_registry_covers_every_executable_workflow_entry() -> None:
|
|
catalog = ScriptCatalog.load_default()
|
|
payload = json.loads(
|
|
(PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8")
|
|
)
|
|
workflow_entries: set[str] = set()
|
|
scheduled_workflows = 0
|
|
manual_workflows = 0
|
|
for workflow in payload["workflows"]:
|
|
assert workflow["trigger"] in {"scheduled", "manual"}
|
|
if workflow["trigger"] == "scheduled":
|
|
scheduled_workflows += 1
|
|
else:
|
|
manual_workflows += 1
|
|
execution = workflow["execution"]
|
|
steps = execution.get("steps")
|
|
entries = (
|
|
[step["entry"] for step in steps]
|
|
if steps is not None
|
|
else [execution["entry"]]
|
|
)
|
|
for entry in entries:
|
|
workflow_entries.add(f"{workflow['module']}:{entry}")
|
|
|
|
assert scheduled_workflows == 40
|
|
assert manual_workflows == 0
|
|
assert len(workflow_entries) == 56
|
|
assert len(catalog.scripts) == 61
|
|
assert workflow_entries < set(catalog.script_ids)
|
|
assert {
|
|
"content.mapping.rebuild",
|
|
"content.failed.retry",
|
|
"shop.douyin_price_appeal",
|
|
"shop.jd_self_operated.collect_brand",
|
|
"shop.jd_customer_service.collect",
|
|
"shop.jd_negative_reviews.collect",
|
|
"shop.jd_after_sales_issues.collect",
|
|
"shop.douyin_customer_service.collect",
|
|
"shop.douyin_after_sales_quality.collect",
|
|
"product.backfill.run",
|
|
"product.erp_all_shop_daily",
|
|
"product.video_upload.run",
|
|
"product.jd_video_upload.run",
|
|
"supply.workflow.run",
|
|
} < set(catalog.command_ids)
|
|
assert {item.module for item in catalog.scripts} == {
|
|
"content_marketing",
|
|
"product_commerce",
|
|
"shop_intelligence",
|
|
"supply_chain",
|
|
}
|
|
assert all(item.path.is_relative_to(PROJECT_ROOT) for item in catalog.scripts)
|
|
assert catalog.get("shop.jd_self_operated.collect_brand").default_args == (
|
|
"--start-date",
|
|
"{business_date}",
|
|
"--end-date",
|
|
"{business_date}",
|
|
"--headless",
|
|
)
|
|
assert catalog.get("shop.douyin_price_appeal").default_args == (
|
|
"--price-appeal",
|
|
"--execute",
|
|
)
|
|
assert catalog.get("shop.jd_customer_service.collect").default_args == (
|
|
"--date",
|
|
"{business_date}",
|
|
"--headless",
|
|
)
|
|
assert catalog.get("shop.douyin_customer_service.collect").default_args == (
|
|
"--date",
|
|
"{business_date}",
|
|
"--headless",
|
|
)
|
|
assert catalog.get("shop.douyin_after_sales_quality.collect").default_args == (
|
|
"--date",
|
|
"{business_date}",
|
|
"--headless",
|
|
)
|
|
assert catalog.get("product.backfill.run").default_args == (
|
|
"--from",
|
|
"{business_date}",
|
|
"--to",
|
|
"{business_date}",
|
|
)
|
|
assert catalog.get("product.erp_all_shop_daily").default_args == (
|
|
"--from",
|
|
"{business_date}",
|
|
"--to",
|
|
"{business_date}",
|
|
)
|
|
assert catalog.get("product.import.tmall_ads").default_args == (
|
|
"--date",
|
|
"{business_date}",
|
|
"--execute",
|
|
)
|
|
assert catalog.get("product.tmall_wanxiang_ads.collect").default_args == (
|
|
"--date",
|
|
"{business_date}",
|
|
"--headless",
|
|
)
|
|
assert catalog.get("product.jd_ad_costs.collect").default_args == (
|
|
"--date",
|
|
"{business_date}",
|
|
"--headless",
|
|
)
|
|
assert catalog.get("product.douyin_qianchuan_ads.collect").default_args == (
|
|
"--date",
|
|
"{business_date}",
|
|
"--headless",
|
|
)
|
|
assert catalog.get("supply.workflow.run").default_args == (
|
|
"mcp-run",
|
|
"purchase-order-update",
|
|
)
|
|
assert (
|
|
catalog.get("content.marketing_report.generate").notification_workflow_id
|
|
== "content.marketing_report.daily"
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("record", "message"),
|
|
(
|
|
(
|
|
{
|
|
"id": "content.bad.path",
|
|
"module": "content_marketing",
|
|
"entry": "../outside.py",
|
|
"kind": "python",
|
|
},
|
|
"unsafe command entry",
|
|
),
|
|
(
|
|
{
|
|
"id": "content.bad.kind",
|
|
"module": "content_marketing",
|
|
"entry": "job.py",
|
|
"kind": "batch",
|
|
},
|
|
"kind does not match",
|
|
),
|
|
(
|
|
{
|
|
"id": "Content Bad",
|
|
"module": "content_marketing",
|
|
"entry": "job.py",
|
|
"kind": "python",
|
|
},
|
|
"invalid command id",
|
|
),
|
|
(
|
|
{
|
|
"id": "content.bad.args",
|
|
"module": "content_marketing",
|
|
"entry": "job.py",
|
|
"kind": "python",
|
|
"default_args": "--unsafe-shape",
|
|
},
|
|
"invalid command default_args",
|
|
),
|
|
(
|
|
{
|
|
"id": "content.bad.notification",
|
|
"module": "content_marketing",
|
|
"entry": "job.py",
|
|
"kind": "python",
|
|
"notification_workflow_id": "content.not-a-capability",
|
|
},
|
|
"invalid command notification_workflow_id",
|
|
),
|
|
),
|
|
)
|
|
def test_catalog_rejects_unsafe_or_invalid_declarations(
|
|
tmp_path: Path, record: dict[str, object], message: str
|
|
) -> None:
|
|
runtime = tmp_path / "runtime"
|
|
runtime.mkdir()
|
|
(runtime / "job.py").write_text("print('ok')\n", encoding="utf-8")
|
|
config = tmp_path / "commands.json"
|
|
_write_registry(config, [record])
|
|
|
|
with pytest.raises(ScriptCatalogError, match=message):
|
|
ScriptCatalog.load(config, roots={"content_marketing": runtime})
|
|
|
|
|
|
def test_catalog_rejects_duplicate_command_ids(tmp_path: Path) -> None:
|
|
runtime = tmp_path / "runtime"
|
|
runtime.mkdir()
|
|
for name in ("first.py", "second.py"):
|
|
(runtime / name).write_text("print('ok')\n", encoding="utf-8")
|
|
config = tmp_path / "commands.json"
|
|
_write_registry(
|
|
config,
|
|
[
|
|
{
|
|
"id": "content.test.run",
|
|
"module": "content_marketing",
|
|
"entry": "first.py",
|
|
"kind": "python",
|
|
},
|
|
{
|
|
"id": "content.test.run",
|
|
"module": "content_marketing",
|
|
"entry": "second.py",
|
|
"kind": "python",
|
|
},
|
|
],
|
|
)
|
|
|
|
with pytest.raises(ScriptCatalogError, match="duplicate command id"):
|
|
ScriptCatalog.load(config, roots={"content_marketing": runtime})
|
|
|
|
|
|
def test_cli_lists_and_dry_runs_registered_commands(tmp_path: Path) -> None:
|
|
output = io.StringIO()
|
|
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path)
|
|
|
|
assert main(["scripts", "list", "--json"], settings=settings, stdout=output) == 0
|
|
rows = json.loads(output.getvalue())
|
|
assert len(rows) == 61
|
|
assert {row["module"] for row in rows} == {
|
|
"content_marketing",
|
|
"product_commerce",
|
|
"shop_intelligence",
|
|
"supply_chain",
|
|
}
|
|
assert {
|
|
"id",
|
|
"command_id",
|
|
"legacy_script_id",
|
|
"module",
|
|
"entry",
|
|
"kind",
|
|
"default_args",
|
|
"notification_workflow_id",
|
|
} <= rows[0].keys()
|
|
|
|
output = io.StringIO()
|
|
exit_code = main(
|
|
[
|
|
"scripts",
|
|
"run",
|
|
"content.metrics.collect_collaborators",
|
|
"--date",
|
|
"2026-07-27",
|
|
],
|
|
settings=settings,
|
|
stdout=output,
|
|
)
|
|
payload = json.loads(output.getvalue())
|
|
assert exit_code == EXIT_SUCCESS
|
|
assert payload["dry_run"] is True
|
|
assert payload["command_id"] == "content.metrics.collect_collaborators"
|
|
assert payload["script_id"] == "content_marketing:run_all.py"
|
|
assert payload["steps"] == {"module_script": "skipped"}
|
|
|
|
|
|
def test_cli_accepts_legacy_script_id_alias(tmp_path: Path) -> None:
|
|
output = io.StringIO()
|
|
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path)
|
|
|
|
exit_code = main(
|
|
[
|
|
"scripts",
|
|
"run",
|
|
"content_marketing:run_all.py",
|
|
"--date",
|
|
"2026-07-27",
|
|
],
|
|
settings=settings,
|
|
stdout=output,
|
|
)
|
|
|
|
payload = json.loads(output.getvalue())
|
|
assert exit_code == EXIT_SUCCESS
|
|
assert payload["command_id"] == "content.metrics.collect_collaborators"
|
|
assert payload["script_id"] == "content_marketing:run_all.py"
|
|
|
|
|
|
def test_cli_script_effect_identity_includes_arguments(tmp_path: Path) -> None:
|
|
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path)
|
|
|
|
def dry_run(*extra: str) -> dict[str, object]:
|
|
output = io.StringIO()
|
|
exit_code = main(
|
|
[
|
|
"scripts",
|
|
"run",
|
|
"content.marketing_report.generate",
|
|
"--date",
|
|
"2026-07-27",
|
|
*extra,
|
|
],
|
|
settings=settings,
|
|
stdout=output,
|
|
)
|
|
assert exit_code == EXIT_SUCCESS
|
|
return json.loads(output.getvalue())
|
|
|
|
prompt_only = dry_run("--arg=--prompt-only")
|
|
prompt_only_again = dry_run("--arg=--prompt-only")
|
|
generate = dry_run()
|
|
|
|
assert prompt_only["workflow_id"] == prompt_only_again["workflow_id"]
|
|
assert prompt_only["workflow_id"] != generate["workflow_id"]
|
|
|
|
|
|
def test_cli_renders_command_business_date_defaults_into_effect_identity(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path)
|
|
|
|
def dry_run(business_date: str) -> dict[str, object]:
|
|
output = io.StringIO()
|
|
exit_code = main(
|
|
[
|
|
"scripts",
|
|
"run",
|
|
"product.backfill.run",
|
|
"--date",
|
|
business_date,
|
|
],
|
|
settings=settings,
|
|
stdout=output,
|
|
)
|
|
assert exit_code == EXIT_SUCCESS
|
|
return json.loads(output.getvalue())
|
|
|
|
first = dry_run("2026-07-27")
|
|
second = dry_run("2026-07-28")
|
|
|
|
assert first["workflow_id"] != second["workflow_id"]
|