feat(product): 聚水潭销售主题分析按日采集工作流

collect_erp_sales_theme_daily.py 采集 bi.erp321.com「报表→销售主题分析→按日统计」,
异步导出 + OSS 直连兜底,固定品牌+店铺口径,写 erp_sales_theme_daily_metrics
(db 层建表与 upsert 同步,每日一行幂等覆盖)。注册 product.erp_sales_theme_daily.collect
手动命令,复用 erp-account 登录绑定新增 CDP 22167;目录/集成计数 60→61,
新脚本纳入 CI 聚焦 lint 清单。
This commit is contained in:
wangyunlong committed 2026-10-14 18:29:15 +08:00
1 parent 97a486e97d
commit 9a91ce80e4
9 files changed
+1229 -6

No files matched your search

@@ -0,0 +1,191 @@
# -*- coding: utf-8 -*-
"""聚水潭销售主题分析(按日)采集脚本的纯解析与 CLI 参数测试。"""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
from gyxx_flow.modules import product_commerce
from gyxx_flow.modules.product_commerce import (
collect_erp_sales_theme_daily as collector,
)
from gyxx_flow.modules.product_commerce import db as product_db
HEADER = [
"日期", "销售单数", "销售数量", "实发数量", "实发金额", "销售金额",
"退货单数", "退货数量", "实退数量", "退货金额", "实退金额",
"净销售单数", "净销量", "净销售额", "运费收入", "运费支出", "实发单数",
]
def _write_workbook(path: Path, rows: list[list]) -> None:
import openpyxl
workbook = openpyxl.Workbook()
sheet = workbook.active
for row in rows:
sheet.append(row)
workbook.save(str(path))
workbook.close()
def test_parse_sales_theme_workbook_roundtrip(tmp_path: Path) -> None:
xlsx = tmp_path / "销售主题分析_按日_1.xlsx"
_write_workbook(
xlsx,
[
HEADER,
["2026-10-11", "649", 655, 646, "173,129.23", 175752.83,
15, 15, 0, 2677.47, "0.00", 634, 640, 173075.36, "20.00", None, 640],
["2026-10-10", 690, 691, 685, 166761.58, 168185.97,
9, 9, 0, 2656.71, 0, 681, 686, 166529.13, None, 1.6, 685],
["##", 1339, 1346, 1331, 339890.81, 343938.8,
24, 24, 0, 5334.18, 0, 1315, 1326, 339604.49, 20, 1.6, 1325],
],
)
records = collector.parse_sales_theme_workbook(xlsx)
assert [record["stat_date"] for record in records] == ["2026-10-11", "2026-10-10"]
first = records[0]
assert first["sales_order_count"] == 649
assert first["sales_qty"] == 655
assert first["shipped_amount"] == 173129.23
assert first["sales_amount"] == 175752.83
assert first["return_amount"] == 2677.47
assert first["returned_amount"] == 0.0
assert first["net_sales_order_count"] == 634
assert first["net_sales_amount"] == 173075.36
assert first["freight_income"] == 20.0
assert first["freight_expense"] is None
assert first["shipped_order_count"] == 640
# 合计行(##)必须被跳过
assert len(records) == 2
def test_parse_sales_theme_workbook_rejects_missing_date_column(tmp_path: Path) -> None:
xlsx = tmp_path / "bad.xlsx"
_write_workbook(xlsx, [["销售单数", "销售数量"], [1, 2]])
with pytest.raises(ValueError):
collector.parse_sales_theme_workbook(xlsx)
def test_parse_number_and_amount_helpers() -> None:
assert collector.parse_number("1,339") == 1339
assert collector.parse_number(None) == 0
assert collector.parse_number("") == 0
assert collector.parse_amount(None) is None
assert collector.parse_amount("") is None
assert collector.parse_amount("1.6") == 1.6
assert collector.parse_amount(2677.467) == 2677.47
class _CaptureCursor:
def __init__(self) -> None:
self.batches: list[tuple] = []
def __enter__(self):
return self
def __exit__(self, *exc_info):
return False
class _CaptureConnection:
def __init__(self) -> None:
self.cursor_obj = _CaptureCursor()
def cursor(self):
return self.cursor_obj
def test_upsert_sales_theme_dedups_and_normalizes(monkeypatch: pytest.MonkeyPatch) -> None:
captured: dict[str, object] = {}
def fake_execute_values(cur, sql, rows, template=None):
captured["sql"] = sql
captured["rows"] = list(rows)
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
records = [
{
"stat_date": "2026-10-11",
"sales_order_count": 649,
"freight_income": "20.00",
},
{
# 同日重复: 整体覆盖前一条(同 upsert_erp_all_shop 语义), 数值缺省为 0
"stat_date": "2026-10-11",
"sales_order_count": 999,
"brands": ["光影", "光影行星", "光影"],
"shops": ["天猫-光影行星旗舰店", "JD光影行星旗舰店"],
"source_file": "var/raw/erp销售主题分析日报/x.xlsx",
},
{
"stat_date": "2026-10-10",
"sales_qty": "691",
},
]
conn = _CaptureConnection()
written = product_db.upsert_erp_sales_theme_daily_metrics(conn, records)
assert written == 2
rows = captured["rows"]
assert len(rows) == 2
by_date = {row[0]: row for row in rows}
# 同日重复记录取最后一条, 数字字段缺省为 0, 金额字符串转 float
assert by_date["2026-10-11"][1] == 999
assert by_date["2026-10-10"][2] == 691
tenth = by_date["2026-10-10"]
# NOT NULL 金额列缺省(导出空单元格)落 0; 运费两列允许 NULL
assert tenth[9] == 0 # return_amount
assert tenth[15] is None # freight_income
assert tenth[16] is None # freight_expense
eleventh = by_date["2026-10-11"]
assert eleventh[-4] == "发货时间" # date_basis 默认
assert eleventh[-3] == ["光影", "光影行星"] # brands 去重排序
assert eleventh[-2] == ["JD光影行星旗舰店", "天猫-光影行星旗舰店"]
assert eleventh[-1] == "var/raw/erp销售主题分析日报/x.xlsx"
# upsert 目标表与主键在 SQL 里声明
sql = str(captured["sql"])
assert "erp_sales_theme_daily_metrics" in sql
assert "ON CONFLICT (stat_date)" in sql
def test_business_scope_constants_are_consistent() -> None:
assert len(collector.BRANDS) == 3
assert sorted(collector.BRANDS) == ["gyxx", "光影", "光影行星"]
assert len(collector.SHOP_NAMES) == len(set(collector.SHOP_NAMES)) > 0
for name in collector.SHOP_NAMES:
assert name == name.strip()
def test_resolve_window_variants() -> None:
args = collector.parse_args(["--date", "2026-10-11"])
start, end = collector.resolve_window(args)
assert (start.isoformat(), end.isoformat()) == ("2026-10-11", "2026-10-11")
args = collector.parse_args(["--date", "2026-10-11", "--days", "7"])
start, end = collector.resolve_window(args)
assert (start.isoformat(), end.isoformat()) == ("2026-10-05", "2026-10-11")
args = collector.parse_args(["--from", "2025-06-01", "--to", "2026-01-01"])
start, end = collector.resolve_window(args)
assert (start.isoformat(), end.isoformat()) == ("2025-06-01", "2026-01-01")
with pytest.raises(SystemExit):
collector.resolve_window(collector.parse_args(["--from", "2026-01-01"]))
with pytest.raises(SystemExit):
collector.resolve_window(
collector.parse_args(["--from", "2026-01-01", "--to", "2025-01-01"])
)
def test_module_uses_shared_cookie_skip_exit_code() -> None:
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
assert COOKIE_SKIP_EXIT_CODE == 75
assert sys.modules[product_commerce.__name__] is not None
+3 -3
View File
@@ -102,10 +102,10 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
data_root=project_root / "var",
)
assert len(scripts.command_ids) == 60
assert len(first.command_ids) == 157
assert len(scripts.command_ids) == 61
assert len(first.command_ids) == 158
bindings = [first.binding_for(command_id) for command_id in first.command_ids]
assert len({binding.cdp_port for binding in bindings}) == 157
assert len({binding.cdp_port for binding in bindings}) == 158
assert all(22000 <= binding.cdp_port <= 22999 for binding in bindings)
assert all(binding.cdp_url.startswith("http://127.0.0.1:") for binding in bindings)
assert bindings == [second.binding_for(item.script_id) for item in bindings]
+2 -2
View File
@@ -83,7 +83,7 @@ def test_default_registry_covers_every_executable_workflow_entry() -> None:
assert scheduled_workflows == 39
assert manual_workflows == 0
assert len(workflow_entries) == 55
assert len(catalog.scripts) == 60
assert len(catalog.scripts) == 61
assert workflow_entries < set(catalog.script_ids)
assert {
"content.mapping.rebuild",
@@ -280,7 +280,7 @@ def test_cli_lists_and_dry_runs_registered_commands(tmp_path: Path) -> None:
assert main(["scripts", "list", "--json"], settings=settings, stdout=output) == 0
rows = json.loads(output.getvalue())
assert len(rows) == 60
assert len(rows) == 61
assert {row["module"] for row in rows} == {
"content_marketing",
"product_commerce",