chore: [workflow] 提交工作区代码

This commit is contained in:
wangyunlong committed 2026-09-30 10:46:15 +08:00
1 parent 2ff588a65d
commit 01f3d7aaf7
131 files changed
+24306 -1320

No files matched your search

@@ -674,3 +674,52 @@ def test_wait_for_exported_excel_returns_immediately_when_present(monkeypatch):
)
assert chanmama._wait_for_exported_excel(0.0) == "/downloads/已有.xlsx"
def test_wait_for_exported_excel_pumps_playwright_events(monkeypatch):
"""回归: 纯 sleep 轮询不泵 Playwright 事件会让 download 事件排队,
download.save_as 到 quit 才执行,文件错过 deadline 误判超时。
等待期间必须周期性调用 driver 泵事件循环。"""
class _FakeDriver:
def __init__(self) -> None:
self.eval_calls = 0
def execute_script(self, script, *args):
self.eval_calls += 1
return None
now = [1000.0]
results = iter([None, None, "/downloads/泵事件后落盘.xlsx"])
driver = _FakeDriver()
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
monkeypatch.setattr(
chanmama.time,
"sleep",
lambda seconds: now.__setitem__(0, now[0] + seconds),
)
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
assert chanmama._wait_for_exported_excel(0.0, driver=driver) == (
"/downloads/泵事件后落盘.xlsx"
)
assert driver.eval_calls >= 3
def test_wait_for_exported_excel_tolerates_driver_errors(monkeypatch):
"""泵事件失败(页面导航中等)不应中断文件轮询。"""
class _BrokenDriver:
def execute_script(self, script, *args):
raise RuntimeError("page navigating")
monkeypatch.setattr(
chanmama,
"_find_new_excel_after",
lambda _pre_mtime: "/downloads/已有.xlsx",
)
assert chanmama._wait_for_exported_excel(0.0, driver=_BrokenDriver()) == (
"/downloads/已有.xlsx"
)
@@ -358,22 +358,32 @@ def test_scheduled_collection_has_no_automatic_retry_and_syncs_once():
)
def test_daily_collection_includes_self_operated_pipeline():
def test_daily_collection_separates_self_operated_pipeline():
steps = _workflow("content.metrics.daily").steps
entries = [step.entry for step in steps]
run_step = next(step for step in steps if step.entry == "run_all.py")
assert "--daily-scope" in run_step.args
assert "--include-self-operated" not in run_step.args
assert entries.index("run_all.py") < entries.index(
"data/tools/refresh_self_mapping.py"
)
assert entries.index("data/tools/refresh_self_mapping.py") < entries.index(
"self_bilibili_scraper.py"
)
assert entries.index("self_bilibili_scraper.py") < entries.index(
"chanmama_scraper.py"
assert entries == ["run_all.py", "data/tools/sync_metrics_to_cmt_notes.py"]
assert steps[1].args == ("--scope", "collaborators")
self_steps = _workflow("content.metrics.self_operated.daily").steps
self_entries = [step.entry for step in self_steps]
assert self_entries == [
"data/tools/refresh_self_mapping.py",
"self_bilibili_scraper.py",
"chanmama_scraper.py",
"self_xhs_scraper.py",
"data/tools/sync_metrics_to_cmt_notes.py",
"data/tools/sync_self_video_metrics.py",
]
sync_step = next(
step
for step in self_steps
if step.entry == "data/tools/sync_metrics_to_cmt_notes.py"
)
assert sync_step.args == ("--scope", "self-operated")
@pytest.mark.parametrize(
@@ -0,0 +1,661 @@
# -*- coding: utf-8 -*-
"""自营内容全量指标(抖音/B站/小红书): 解析、账号映射、入库行构造的边界测试。"""
from __future__ import annotations
import datetime as dt
import json
import os
import re
from pathlib import Path
import openpyxl
from gyxx_flow.modules.content_marketing import (
chanmama_export,
self_operated_accounts,
self_xhs_scraper,
)
from gyxx_flow.modules.content_marketing.data.tools import (
sync_self_video_metrics as sync_tool,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
PROJECT_ROOT = Path(__file__).resolve().parents[3]
TOOLS = (
PROJECT_ROOT / "src/gyxx_flow/modules/content_marketing/data/tools"
)
EXPORT_HEADERS = [
"视频名称", "蝉妈妈视频链接", "抖音视频链接", "达人昵称", "抖音号",
"视频时长", "发布时间", "预估曝光", "点赞数", "评论数", "转发数",
"预估销量", "预估销量-指数", "预估销售额", "预估销售额-指数",
]
STORE_FLAGSHIP = "光影行星箱包旗舰店"
STORE_SPECIALTY = "GYXX光影行星箱包专卖店"
XHS_SHOP = "光影行星GYXX旗舰店"
def _write_export_xlsx(path: Path, rows: list[tuple]) -> str:
workbook = openpyxl.Workbook()
sheet = workbook.active
sheet.append(EXPORT_HEADERS)
for row in rows:
sheet.append(list(row))
workbook.save(path)
workbook.close()
return str(path)
def _douyin_sample_rows() -> list[tuple]:
return [
("总计", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "0", "-", "0", "-"),
(
"轻户外,自己定义-光影行星 逐星",
"https://www.chanmama.com/awemeDetail/SbXQnFsF2tlZF5Wy7U1uG3izRQaBOwYT",
"https://www.iesdouyin.com/share/video/7687797200985869595/?region=CN",
STORE_FLAGSHIP, "gyxx168888", "1分0秒", "2026-09-21 09:39",
"3.2w+", 135, 8, 2, "12", 0, "3580.5", 0,
),
(
"图文笔记标题",
"https://www.chanmama.com/awemeDetail/M7_SEJOk9P_Q1eafatlOCAQl4F2eI3FR",
"https://www.iesdouyin.com/share/note/7687432318584143857/?region=CN",
STORE_SPECIALTY, "GYXX168168", "--", "2026-09-20 10:03",
"0", 1, 0, 0, "0", 0, "0", 0,
),
]
def test_parse_chanmama_excel_extracts_all_metric_fields(tmp_path):
xlsx = _write_export_xlsx(tmp_path / "flagship.xlsx", _douyin_sample_rows())
records, headers = chanmama_export.parse_chanmama_excel(xlsx)
assert headers == EXPORT_HEADERS
assert len(records) == 2 # 总计行被跳过
first = records[0]
assert first["creator_name"] == STORE_FLAGSHIP
assert first["exposure"] == 32000
assert first["like_count"] == 135
assert first["comment_count"] == 8
assert first["share_count"] == 2
assert first["est_sales_amount"] == 3580.5
assert chanmama_export.extract_video_id(
first["chanmama_url"], first["video_url"]
) == "7687797200985869595"
def test_self_operated_accounts_config_covers_three_platforms():
path = self_operated_accounts.accounts_config_path(PATHS.config_root)
all_accounts = self_operated_accounts.load_all_accounts(path)
assert set(all_accounts) == {"douyin", "bilibili", "xiaohongshu"}
# 用户约定的身份映射: 抖音·机能号=旗舰店, 抖音·高智号=专卖店
assert self_operated_accounts.resolve_alias(
all_accounts["douyin"], account_name=STORE_FLAGSHIP
) == "抖音·机能号"
assert self_operated_accounts.resolve_alias(
all_accounts["douyin"], platform_account_id="GYXX168168"
) == "抖音·高智号"
# 小红书账号必须带登录手机号
xhs = self_operated_accounts.find_account(
"xiaohongshu", path, account_name=XHS_SHOP
)
assert xhs is not None and re.fullmatch(r"1\d{10}", xhs["phone"])
assert self_operated_accounts.resolve_alias(
all_accounts["xiaohongshu"], account_name=XHS_SHOP
) == "小红书·自营号"
# 未知账号返回空串,不抛异常
assert self_operated_accounts.resolve_alias(
all_accounts["bilibili"], account_name="未知UP主"
) == ""
def test_load_accounts_tolerates_missing_or_broken_config(tmp_path):
assert self_operated_accounts.load_accounts(
"douyin", tmp_path / "missing.json"
) == []
broken = tmp_path / "broken.json"
broken.write_text("not-json", encoding="utf-8")
assert self_operated_accounts.load_accounts("douyin", broken) == []
empty = tmp_path / "empty.json"
empty.write_text("{}", encoding="utf-8")
assert self_operated_accounts.load_accounts("bilibili", empty) == []
with __import__("pytest").raises(ValueError):
self_operated_accounts.load_accounts("kuaishou", empty)
# ============================================================
# B站: 公开 API 全字段
# ============================================================
class _FakeBiliResponse:
def __init__(self, payload):
self._payload = payload
def json(self):
return self._payload
class _FakeBiliSession:
"""固定返回一个 B站 view API 响应的假 HTTP 会话。"""
PAYLOAD = {
"code": 0,
"data": {
"aid": 114514,
"bvid": "BV1xT4y1Q7zz",
"title": "光影行星 逐星 开箱",
"duration": 92,
"pubdate": 1761026400,
"owner": {"mid": 5201314, "name": "光影行星官方号"},
"stat": {
"view": 35821, "like": 1204, "coin": 88, "favorite": 301,
"share": 57, "danmaku": 145, "reply": 96,
},
},
}
def get(self, url, **_kwargs):
return _FakeBiliResponse(self.PAYLOAD)
def test_fetch_bili_video_extracts_full_metrics():
from gyxx_flow.modules.content_marketing.self_bilibili_scraper import (
fetch_bili_video,
format_duration,
)
info = fetch_bili_video("https://www.bilibili.com/video/BV1xT4y1Q7zz",
_FakeBiliSession())
assert info is not None
assert info["video_id"] == "BV1xT4y1Q7zz"
assert info["video_url"].endswith("/video/BV1xT4y1Q7zz")
assert info["title"] == "光影行星 逐星 开箱"
assert info["owner_name"] == "光影行星官方号"
assert info["view_count"] == 35821
assert info["like_count"] == 1204
assert info["coin_count"] == 88
assert info["favorite_count"] == 301
assert info["danmaku_count"] == 145
assert info["reply_count"] == 96
assert info["publish_time"] == dt.datetime.fromtimestamp(1761026400)
assert format_duration(92) == "1:32"
assert format_duration(3725) == "1:02:05"
assert format_duration(None) == ""
def test_collect_bilibili_rows_from_artifact(tmp_path, monkeypatch):
artifact = tmp_path / "daily.json"
artifact.write_text(
json.dumps({
"platform": "bilibili",
"business_date": "2026-09-28",
"videos": [{
"video_id": "BV1xT4y1Q7zz",
"video_url": "https://www.bilibili.com/video/BV1xT4y1Q7zz",
"title": "光影行星 逐星 开箱",
"owner_name": "光影行星官方号",
"owner_mid": "5201314",
"duration_seconds": 92,
"publish_time": "2026-09-21 09:39",
"view_count": 35821, "like_count": 1204, "coin_count": 88,
"favorite_count": 301, "share_count": 57,
"danmaku_count": 145, "reply_count": 96,
}],
}, ensure_ascii=False),
encoding="utf-8",
)
monkeypatch.setattr(sync_tool, "BILIBILI_ARTIFACT", artifact)
accounts = [
{"alias": "B站·自营号", "account_name": "光影行星官方号",
"platform_account_id": "", "phone": ""}
]
rows, stats, source = sync_tool.collect_bilibili_rows(accounts, dt.date(2026, 9, 28))
assert stats["rows"] == 1
assert source == artifact
row = rows[("bilibili", "光影行星官方号", "BV1xT4y1Q7zz")]
assert row[sync_tool.ROW_FIELDS.index("account_alias")] == "B站·自营号"
assert row[sync_tool.ROW_FIELDS.index("exposure")] == 35821
assert row[sync_tool.ROW_FIELDS.index("comment_count")] == 96 # reply→评论
assert row[sync_tool.ROW_FIELDS.index("coin_count")] == 88
assert row[sync_tool.ROW_FIELDS.index("duration")] == "1:32"
def test_collect_bilibili_rows_missing_artifact(tmp_path, monkeypatch):
monkeypatch.setattr(
sync_tool, "BILIBILI_ARTIFACT", tmp_path / "missing.json"
)
rows, stats, source = sync_tool.collect_bilibili_rows([], dt.date(2026, 9, 28))
assert rows == {} and stats["rows"] == 0 and source is None
# ============================================================
# 小红书: ARK 管理页表格解析
# ============================================================
GOODS_HEADERS = ["", "笔记信息", "挂载商品", "笔记阅读数", "笔记支付金额",
"笔记支付订单数", "发布状态", "操作"]
NORMAL_HEADERS = ["", "笔记信息", "阅读数", "点赞数", "引流主页次数",
"引流直播间次数", "发布状态", "操作"]
def test_bilibili_wbi_signing_matches_documented_vector():
from gyxx_flow.modules.content_marketing.self_bilibili_scraper import (
derive_mixin_key,
wbi_sign,
)
# B站 WBI 公开示例密钥 → 文档给定的 mixin key
mixin = derive_mixin_key(
"7cd084941338484aae1ad9425b84077c",
"4932caff0ff746eab6f01bf08b70ac45",
)
assert mixin == "ea1db124af3c7062474693fa704f4ff8"
signed = wbi_sign(
{"platform": "web", "pn": 1, "ps": 30, "mid": "1008229967",
"order": "pubdate"},
mixin,
1702204800,
)
assert signed["wts"] == 1702204800
assert re.fullmatch(r"[0-9a-f]{32}", signed["w_rid"])
# 确定性: 同参数同时间戳签名一致
again = wbi_sign(
{"mid": "1008229967", "order": "pubdate", "pn": 1, "ps": 30,
"platform": "web"},
mixin,
1702204800,
)
assert again["w_rid"] == signed["w_rid"]
def test_bilibili_artifact_covers_full_history(tmp_path, monkeypatch):
"""主页全量采集后 artifact 应含全部投稿(含飞书未登记的历史视频)。"""
artifact = tmp_path / "daily.json"
videos = [
{
"video_id": f"BV1xT4y1Q7z{i:02d}",
"video_url": f"https://www.bilibili.com/video/BV1xT4y1Q7z{i:02d}",
"title": f"视频{i}",
"owner_name": "光影行星",
"owner_mid": "1008229967",
"duration_seconds": 60 + i,
"publish_time": f"2026-09-{i + 1:02d} 09:00:00",
"view_count": 100 * i, "like_count": i, "coin_count": i,
"favorite_count": i, "share_count": i, "danmaku_count": i,
"reply_count": i,
}
for i in range(3)
]
artifact.write_text(
json.dumps({"platform": "bilibili", "business_date": "2026-09-29",
"videos": videos}, ensure_ascii=False),
encoding="utf-8",
)
monkeypatch.setattr(sync_tool, "BILIBILI_ARTIFACT", artifact)
accounts = [
{"alias": "B站·自营号", "account_name": "光影行星",
"platform_account_id": "1008229967", "phone": ""}
]
rows, stats, _source = sync_tool.collect_bilibili_rows(
accounts, dt.date(2026, 9, 29)
)
assert stats["rows"] == 3
key = ("bilibili", "光影行星", "BV1xT4y1Q7z00")
assert rows[key][sync_tool.ROW_FIELDS.index("account_alias")] == "B站·自营号"
assert rows[key][sync_tool.ROW_FIELDS.index("publish_time")] == dt.datetime(
2026, 9, 1, 9, 0
)
def test_parse_info_cell_variants():
parse = self_xhs_scraper._parse_info_cell
# 多行,发布时间带前缀(线上真实形态)
assert parse("标题A\n发布时间:2026-09-15 14:23\n手动创作") == (
"标题A", "2026-09-15 14:23", "手动创作"
)
# 单行压扁(历史兜底)
title, publish, method = parse("标题B 发布时间:2026-09-08 13:41 手动创作")
assert title == "标题B"
assert publish == "2026-09-08 13:41"
assert method == "手动创作"
# 无日期无方式的纯标题
assert parse("纯标题") == ("纯标题", "", "")
assert parse("") == ("", "", "")
def test_parse_goods_note_rows():
rows = [
{
"cells": [
"",
"斜挎包通勤好物\n2026-09-25 10:23\n手动创作",
"光影行星 极星单肩包男女…",
"1.2万",
"3,586.50",
"12",
"已发布",
"数据 编辑",
],
"href": "https://www.xiaohongshu.com/explore/66e4b2c3000000001d00f1a2",
},
{
"cells": ["", "另一条\n2026-09-20 08:00", "暂无关联商品", "356",
"—", "—", "已发布", "编辑"],
"href": "",
},
]
records, stats = self_xhs_scraper.parse_management_rows(
GOODS_HEADERS, rows, "goods"
)
assert stats["records"] == 2
first = records[0]
assert first["note_type"] == "goods"
assert first["title"] == "斜挎包通勤好物"
assert first["publish_time"] == "2026-09-25 10:23"
assert first["creation_method"] == "手动创作"
assert first["mounted_products"] == "光影行星 极星单肩包男女…"
assert first["read_count"] == 12000
assert first["payment_amount"] == 3586.50
assert first["payment_orders"] == 12
assert first["publish_status"] == "已发布"
assert first["note_id"] == "66e4b2c3000000001d00f1a2"
assert first["video_id"] == "66e4b2c3000000001d00f1a2"
# 页面原始列全量保留
assert first["raw_columns"]["笔记阅读数"] == "1.2万"
# 无链接行回退标题指纹,并跨日稳定
second = records[1]
assert second["video_id"].startswith("xhs-")
assert second["video_id"] == self_xhs_scraper.fallback_note_fingerprint(
"另一条", "2026-09-20 08:00"
)
assert second["mounted_products"] == "暂无关联商品"
assert second["payment_amount"] is None
assert second["payment_orders"] is None
def test_parse_normal_note_rows():
rows = [
{
"cells": ["", "双肩包种草\n2026-09-22 18:30\n可能是AI创作",
"6,592", "483", "35", "8", "已发布", "数据"],
"href": "https://www.xiaohongshu.com/discovery/item/66e0abc5000000001900c3b1",
},
]
records, stats = self_xhs_scraper.parse_management_rows(
NORMAL_HEADERS, rows, "normal"
)
assert stats["records"] == 1
record = records[0]
assert record["note_type"] == "normal"
assert record["read_count"] == 6592
assert record["like_count"] == 483
assert record["homepage_redirect_count"] == 35
assert record["live_redirect_count"] == 8
assert record["creation_method"] == "可能是AI创作"
assert record["video_id"] == "66e0abc5000000001900c3b1"
# 标题含「创作」字样不会被误判为创作方式标签
assert record["title"] == "双肩包种草"
def test_parse_count_and_amount_edge_cases():
assert self_xhs_scraper.parse_count("1.2万") == 12000
assert self_xhs_scraper.parse_count("6,592") == 6592
assert self_xhs_scraper.parse_count("0") == 0
assert self_xhs_scraper.parse_count("—") is None
assert self_xhs_scraper.parse_count("暂无") is None
assert self_xhs_scraper.parse_count("") is None
assert self_xhs_scraper.parse_amount("3,586.50") == 3586.50
assert self_xhs_scraper.parse_amount("0.9万") == 9000.0
assert self_xhs_scraper.parse_amount("-") is None
def test_collect_xiaohongshu_rows_from_artifact(tmp_path, monkeypatch):
artifact = tmp_path / "daily.json"
artifact.write_text(
json.dumps({
"platform": "xiaohongshu",
"business_date": "2026-09-28",
"account_name": XHS_SHOP,
"notes": [
{
"note_type": "goods",
"video_id": "66e4b2c3000000001d00f1a2",
"title": "斜挎包通勤好物",
"publish_time": "2026-09-25 10:23",
"creation_method": "手动创作",
"mounted_products": "光影行星 极星单肩包男女…",
"read_count": 12000, "like_count": 431,
"payment_amount": 3586.5, "payment_orders": 12,
"publish_status": "已发布",
"note_url": "https://www.xiaohongshu.com/explore/66e4b2c3000000001d00f1a2",
"raw_columns": {"笔记阅读数": "1.2万"},
},
],
}, ensure_ascii=False),
encoding="utf-8",
)
monkeypatch.setattr(sync_tool, "XHS_ARTIFACT", artifact)
accounts = [
{"alias": "小红书·自营号", "account_name": XHS_SHOP,
"platform_account_id": "18933246725", "phone": "18933246725"}
]
rows, stats, source = sync_tool.collect_xiaohongshu_rows(
accounts, dt.date(2026, 9, 28)
)
assert stats["rows"] == 1
row = rows[("xiaohongshu", XHS_SHOP, "66e4b2c3000000001d00f1a2")]
fields = sync_tool.ROW_FIELDS
assert row[fields.index("account_alias")] == "小红书·自营号"
assert row[fields.index("exposure")] == 12000 # 阅读数 → 主触达
assert row[fields.index("like_count")] == 431
assert row[fields.index("payment_amount")] == 3586.5
assert row[fields.index("payment_orders")] == 12
assert row[fields.index("note_type")] == "goods"
assert row[fields.index("publish_time")] == dt.datetime(2026, 9, 25, 10, 23)
assert row[fields.index("page_metrics")].obj["笔记阅读数"] == "1.2万"
def test_douyin_history_backfill_uses_file_mtime_and_filename_account(tmp_path):
"""历史回填: 按导出落盘日期各留快照,旧版12列文件按文件名前缀补账号。"""
# 旧版 12 列表头(无 达人昵称/抖音号/蝉妈妈链接)
legacy_headers = [
"视频名称", "抖音视频链接", "视频时长", "发布时间", "预估曝光",
"点赞数", "评论数", "转发数", "预估销量", "预估销量-指数",
"预估销售额", "预估销售额-指数",
]
legacy_file = tmp_path / "光影行星箱包旗舰店_视频记录_周期内发布视频_20260803-20260901.xlsx"
workbook = openpyxl.Workbook()
sheet = workbook.active
sheet.append(legacy_headers)
sheet.append([
"旧版格式视频", "https://www.iesdouyin.com/share/video/7686000000000000001/",
"46秒", "2026-08-27 16:40:56", "23000", 1321, 46, 14, "0", 0, "0", 0,
])
workbook.save(legacy_file)
workbook.close()
import os
historical = dt.datetime(2026, 9, 2, 17, 0, 0).timestamp()
os.utime(legacy_file, (historical, historical))
accounts = self_operated_accounts.load_accounts(
"douyin", self_operated_accounts.accounts_config_path(PATHS.config_root)
)
rows, stats, _sources = sync_tool.collect_douyin_rows(
accounts, dt.date(2026, 9, 29), [str(legacy_file)],
use_file_mtime=True,
)
assert stats["skip_no_identity"] == 0
assert stats["rows"] == 1
row = next(iter(rows.values()))
fields = sync_tool.ROW_FIELDS
assert row[fields.index("account_name")] == STORE_FLAGSHIP # 文件名前缀回退
assert row[fields.index("account_alias")] == "抖音·机能号"
assert row[fields.index("metric_date")] == dt.date(2026, 9, 2) # mtime 日期
assert row[fields.index("exposure")] == 23000
def test_collect_douyin_rows_keeps_video_note_type(tmp_path):
xlsx = _write_export_xlsx(tmp_path / "flagship.xlsx", _douyin_sample_rows())
accounts = self_operated_accounts.load_accounts(
"douyin", self_operated_accounts.accounts_config_path(PATHS.config_root)
)
rows, stats, _sources = sync_tool.collect_douyin_rows(
accounts, dt.date(2026, 9, 28), [xlsx]
)
assert stats["rows"] == 2
fields = sync_tool.ROW_FIELDS
video_row = rows[
("douyin", STORE_FLAGSHIP, "7687797200985869595", dt.date(2026, 9, 28))
]
note_row = rows[
("douyin", STORE_SPECIALTY, "7687432318584143857", dt.date(2026, 9, 28))
]
assert video_row[fields.index("note_type")] == "video"
assert note_row[fields.index("note_type")] == "note"
assert video_row[fields.index("account_alias")] == "抖音·机能号"
def test_make_row_validates_fields_and_truncates():
row = sync_tool.make_row(
"douyin", dt.date(2026, 9, 28), "a.xlsx",
title="超长标题" * 200, video_id="1",
)
fields = sync_tool.ROW_FIELDS
assert len(row) == len(fields)
assert row[fields.index("title")] == ("超长标题" * 200)[:512]
assert row[fields.index("account_alias")] == ""
import pytest
with pytest.raises(KeyError):
sync_tool.make_row("douyin", dt.date(2026, 9, 28), "a.xlsx",
unknown_field=1)
# ============================================================
# SQL 安全 / 表结构 / 配置登记
# ============================================================
def test_upsert_sql_is_fully_parameterized():
source = (TOOLS / "sync_self_video_metrics.py").read_text(encoding="utf-8")
matches = re.findall(
r"cur\.execute\(\s*\"\"\"\s*(INSERT\b.*?)\"\"\",\s*row,",
source,
re.DOTALL,
)
assert len(matches) == 1, "入库 upsert 必须以字面量 SQL + 参数元组形式调用"
sql = matches[0]
assert sql.count("%s") == len(sync_tool.ROW_FIELDS) # 全部参数绑定
assert "{" not in sql and ".format(" not in sql
assert "ON CONFLICT (platform, account_name, video_id, metric_date)" in sql
def test_schema_and_migrations_declare_self_video_metrics_consistently():
schema = (TOOLS / "schema_gyxx_super_data.sql").read_text(encoding="utf-8")
migration_009 = (TOOLS / "migrations" / "009_self_video_metrics.sql").read_text(
encoding="utf-8"
)
migration_010 = (
TOOLS / "migrations" / "010_self_video_metrics_platforms.sql"
).read_text(encoding="utf-8")
for sql in (schema, migration_009, migration_010):
assert "cmt_self_video_metrics" in sql
for sql in (schema, migration_009):
assert "CREATE TABLE IF NOT EXISTS cmt_self_video_metrics" in sql
assert "UNIQUE (platform, account_name, video_id, metric_date)" in sql
combined = schema + "\n" + migration_009 + "\n" + migration_010
for column in (
"account_alias", "video_id", "video_url", "publish_time",
"exposure", "like_count", "comment_count", "share_count",
"est_sales_count", "est_sales_amount", "note_type",
"creation_method", "publish_status", "mounted_products",
"homepage_redirect_count", "live_redirect_count", "payment_amount",
"payment_orders", "coin_count", "favorite_count", "danmaku_count",
"page_metrics",
):
assert re.search(rf"\b{column}\b", combined), f"{column} 缺失"
def test_workflow_commands_and_bindings_declare_self_content_pipeline():
workflows = json.loads(
(PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8")
)
workflow = next(
item
for item in workflows["workflows"]
if item["id"] == "content.metrics.self_operated.daily"
)
steps = {step["id"]: step for step in workflow["execution"]["steps"]}
assert set(steps) == {
"refresh_self_mapping", "collect_self_bilibili", "collect_self_douyin",
"collect_self_xiaohongshu", "sync", "sync_self_video_metrics",
}
assert steps["collect_self_xiaohongshu"]["entry"] == "self_xhs_scraper.py"
assert steps["collect_self_xiaohongshu"]["depends_on"] == ["collect_self_douyin"]
assert steps["sync_self_video_metrics"]["depends_on"] == [
"sync", "collect_self_xiaohongshu",
]
module_root = PROJECT_ROOT / "src/gyxx_flow/modules/content_marketing"
for entry in (
"self_xhs_scraper.py",
"data/tools/sync_self_video_metrics.py",
):
assert (module_root / entry).is_file()
commands = json.loads(
(PROJECT_ROOT / "config" / "commands.json").read_text(encoding="utf-8")
)
command_entries = {item["id"]: item for item in commands["commands"]}
assert command_entries["content.metrics.collect_self_xiaohongshu"]["entry"] == (
"self_xhs_scraper.py"
)
assert command_entries["content.metrics.sync_self_video_metrics"]["entry"] == (
"data/tools/sync_self_video_metrics.py"
)
bindings = json.loads(
(PROJECT_ROOT / "config" / "runtime-bindings.json").read_text(
encoding="utf-8"
)
)
xhs_binding = bindings["scripts"]["content.metrics.collect_self_xiaohongshu"]
assert xhs_binding["script_id"] == "content_marketing:self_xhs_scraper.py"
assert xhs_binding["required_cookie_domains"] == ["xiaohongshu.com"]
assert "XHS_ARK_ACCOUNT" in xhs_binding["credential_env_names"]
sync_binding = bindings["scripts"]["content.metrics.sync_self_video_metrics"]
assert sync_binding["cdp_port"] == 22164
ports = [entry["cdp_port"] for entry in bindings["scripts"].values()]
assert len(ports) == len(set(ports)), "CDP 端口必须唯一"
def test_metric_date_resolution_and_stale_guard(tmp_path, monkeypatch):
monkeypatch.delenv("GYXX_BUSINESS_DATE", raising=False)
assert sync_tool.runtime_business_date() is None
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-09-28")
assert sync_tool.runtime_business_date() == dt.date(2026, 9, 28)
assert sync_tool.resolve_metric_date("2026-09-01") == dt.date(2026, 9, 1)
xlsx = Path(_write_export_xlsx(tmp_path / "a.xlsx", _douyin_sample_rows()))
fresh = dt.datetime.now().timestamp()
os.utime(xlsx, (fresh, fresh))
assert sync_tool.is_current_artifact(xlsx, dt.date.today())
assert not sync_tool.is_current_artifact(xlsx, dt.date(2026, 9, 21))
def test_runtime_business_date_rejects_invalid_value(monkeypatch):
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026/09/28")
with __import__("pytest").raises(RuntimeError):
sync_tool.runtime_business_date()