chore: [workflow] 提交工作区代码
This commit is contained in:
1 parent
2ff588a65d
commit
01f3d7aaf7
131 files changed
+24306
-1320
No files matched your search
@@ -674,3 +674,52 @@ def test_wait_for_exported_excel_returns_immediately_when_present(monkeypatch):
|
||||
)
|
||||
|
||||
assert chanmama._wait_for_exported_excel(0.0) == "/downloads/已有.xlsx"
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_pumps_playwright_events(monkeypatch):
|
||||
"""回归: 纯 sleep 轮询不泵 Playwright 事件会让 download 事件排队,
|
||||
download.save_as 到 quit 才执行,文件错过 deadline 误判超时。
|
||||
等待期间必须周期性调用 driver 泵事件循环。"""
|
||||
|
||||
class _FakeDriver:
|
||||
def __init__(self) -> None:
|
||||
self.eval_calls = 0
|
||||
|
||||
def execute_script(self, script, *args):
|
||||
self.eval_calls += 1
|
||||
return None
|
||||
|
||||
now = [1000.0]
|
||||
results = iter([None, None, "/downloads/泵事件后落盘.xlsx"])
|
||||
driver = _FakeDriver()
|
||||
|
||||
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
|
||||
monkeypatch.setattr(
|
||||
chanmama.time,
|
||||
"sleep",
|
||||
lambda seconds: now.__setitem__(0, now[0] + seconds),
|
||||
)
|
||||
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
|
||||
|
||||
assert chanmama._wait_for_exported_excel(0.0, driver=driver) == (
|
||||
"/downloads/泵事件后落盘.xlsx"
|
||||
)
|
||||
assert driver.eval_calls >= 3
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_tolerates_driver_errors(monkeypatch):
|
||||
"""泵事件失败(页面导航中等)不应中断文件轮询。"""
|
||||
|
||||
class _BrokenDriver:
|
||||
def execute_script(self, script, *args):
|
||||
raise RuntimeError("page navigating")
|
||||
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_find_new_excel_after",
|
||||
lambda _pre_mtime: "/downloads/已有.xlsx",
|
||||
)
|
||||
|
||||
assert chanmama._wait_for_exported_excel(0.0, driver=_BrokenDriver()) == (
|
||||
"/downloads/已有.xlsx"
|
||||
)
|
||||
@@ -358,22 +358,32 @@ def test_scheduled_collection_has_no_automatic_retry_and_syncs_once():
|
||||
)
|
||||
|
||||
|
||||
def test_daily_collection_includes_self_operated_pipeline():
|
||||
def test_daily_collection_separates_self_operated_pipeline():
|
||||
steps = _workflow("content.metrics.daily").steps
|
||||
entries = [step.entry for step in steps]
|
||||
run_step = next(step for step in steps if step.entry == "run_all.py")
|
||||
|
||||
assert "--daily-scope" in run_step.args
|
||||
assert "--include-self-operated" not in run_step.args
|
||||
assert entries.index("run_all.py") < entries.index(
|
||||
"data/tools/refresh_self_mapping.py"
|
||||
)
|
||||
assert entries.index("data/tools/refresh_self_mapping.py") < entries.index(
|
||||
"self_bilibili_scraper.py"
|
||||
)
|
||||
assert entries.index("self_bilibili_scraper.py") < entries.index(
|
||||
"chanmama_scraper.py"
|
||||
assert entries == ["run_all.py", "data/tools/sync_metrics_to_cmt_notes.py"]
|
||||
assert steps[1].args == ("--scope", "collaborators")
|
||||
|
||||
self_steps = _workflow("content.metrics.self_operated.daily").steps
|
||||
self_entries = [step.entry for step in self_steps]
|
||||
assert self_entries == [
|
||||
"data/tools/refresh_self_mapping.py",
|
||||
"self_bilibili_scraper.py",
|
||||
"chanmama_scraper.py",
|
||||
"self_xhs_scraper.py",
|
||||
"data/tools/sync_metrics_to_cmt_notes.py",
|
||||
"data/tools/sync_self_video_metrics.py",
|
||||
]
|
||||
sync_step = next(
|
||||
step
|
||||
for step in self_steps
|
||||
if step.entry == "data/tools/sync_metrics_to_cmt_notes.py"
|
||||
)
|
||||
assert sync_step.args == ("--scope", "self-operated")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -0,0 +1,661 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""自营内容全量指标(抖音/B站/小红书): 解析、账号映射、入库行构造的边界测试。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import openpyxl
|
||||
|
||||
from gyxx_flow.modules.content_marketing import (
|
||||
chanmama_export,
|
||||
self_operated_accounts,
|
||||
self_xhs_scraper,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
sync_self_video_metrics as sync_tool,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
TOOLS = (
|
||||
PROJECT_ROOT / "src/gyxx_flow/modules/content_marketing/data/tools"
|
||||
)
|
||||
|
||||
EXPORT_HEADERS = [
|
||||
"视频名称", "蝉妈妈视频链接", "抖音视频链接", "达人昵称", "抖音号",
|
||||
"视频时长", "发布时间", "预估曝光", "点赞数", "评论数", "转发数",
|
||||
"预估销量", "预估销量-指数", "预估销售额", "预估销售额-指数",
|
||||
]
|
||||
|
||||
STORE_FLAGSHIP = "光影行星箱包旗舰店"
|
||||
STORE_SPECIALTY = "GYXX光影行星箱包专卖店"
|
||||
XHS_SHOP = "光影行星GYXX旗舰店"
|
||||
|
||||
|
||||
def _write_export_xlsx(path: Path, rows: list[tuple]) -> str:
|
||||
workbook = openpyxl.Workbook()
|
||||
sheet = workbook.active
|
||||
sheet.append(EXPORT_HEADERS)
|
||||
for row in rows:
|
||||
sheet.append(list(row))
|
||||
workbook.save(path)
|
||||
workbook.close()
|
||||
return str(path)
|
||||
|
||||
|
||||
def _douyin_sample_rows() -> list[tuple]:
|
||||
return [
|
||||
("总计", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "0", "-", "0", "-"),
|
||||
(
|
||||
"轻户外,自己定义-光影行星 逐星",
|
||||
"https://www.chanmama.com/awemeDetail/SbXQnFsF2tlZF5Wy7U1uG3izRQaBOwYT",
|
||||
"https://www.iesdouyin.com/share/video/7687797200985869595/?region=CN",
|
||||
STORE_FLAGSHIP, "gyxx168888", "1分0秒", "2026-09-21 09:39",
|
||||
"3.2w+", 135, 8, 2, "12", 0, "3580.5", 0,
|
||||
),
|
||||
(
|
||||
"图文笔记标题",
|
||||
"https://www.chanmama.com/awemeDetail/M7_SEJOk9P_Q1eafatlOCAQl4F2eI3FR",
|
||||
"https://www.iesdouyin.com/share/note/7687432318584143857/?region=CN",
|
||||
STORE_SPECIALTY, "GYXX168168", "--", "2026-09-20 10:03",
|
||||
"0", 1, 0, 0, "0", 0, "0", 0,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_chanmama_excel_extracts_all_metric_fields(tmp_path):
|
||||
xlsx = _write_export_xlsx(tmp_path / "flagship.xlsx", _douyin_sample_rows())
|
||||
|
||||
records, headers = chanmama_export.parse_chanmama_excel(xlsx)
|
||||
|
||||
assert headers == EXPORT_HEADERS
|
||||
assert len(records) == 2 # 总计行被跳过
|
||||
first = records[0]
|
||||
assert first["creator_name"] == STORE_FLAGSHIP
|
||||
assert first["exposure"] == 32000
|
||||
assert first["like_count"] == 135
|
||||
assert first["comment_count"] == 8
|
||||
assert first["share_count"] == 2
|
||||
assert first["est_sales_amount"] == 3580.5
|
||||
assert chanmama_export.extract_video_id(
|
||||
first["chanmama_url"], first["video_url"]
|
||||
) == "7687797200985869595"
|
||||
|
||||
|
||||
def test_self_operated_accounts_config_covers_three_platforms():
|
||||
path = self_operated_accounts.accounts_config_path(PATHS.config_root)
|
||||
all_accounts = self_operated_accounts.load_all_accounts(path)
|
||||
assert set(all_accounts) == {"douyin", "bilibili", "xiaohongshu"}
|
||||
# 用户约定的身份映射: 抖音·机能号=旗舰店, 抖音·高智号=专卖店
|
||||
assert self_operated_accounts.resolve_alias(
|
||||
all_accounts["douyin"], account_name=STORE_FLAGSHIP
|
||||
) == "抖音·机能号"
|
||||
assert self_operated_accounts.resolve_alias(
|
||||
all_accounts["douyin"], platform_account_id="GYXX168168"
|
||||
) == "抖音·高智号"
|
||||
# 小红书账号必须带登录手机号
|
||||
xhs = self_operated_accounts.find_account(
|
||||
"xiaohongshu", path, account_name=XHS_SHOP
|
||||
)
|
||||
assert xhs is not None and re.fullmatch(r"1\d{10}", xhs["phone"])
|
||||
assert self_operated_accounts.resolve_alias(
|
||||
all_accounts["xiaohongshu"], account_name=XHS_SHOP
|
||||
) == "小红书·自营号"
|
||||
# 未知账号返回空串,不抛异常
|
||||
assert self_operated_accounts.resolve_alias(
|
||||
all_accounts["bilibili"], account_name="未知UP主"
|
||||
) == ""
|
||||
|
||||
|
||||
def test_load_accounts_tolerates_missing_or_broken_config(tmp_path):
|
||||
assert self_operated_accounts.load_accounts(
|
||||
"douyin", tmp_path / "missing.json"
|
||||
) == []
|
||||
broken = tmp_path / "broken.json"
|
||||
broken.write_text("not-json", encoding="utf-8")
|
||||
assert self_operated_accounts.load_accounts("douyin", broken) == []
|
||||
empty = tmp_path / "empty.json"
|
||||
empty.write_text("{}", encoding="utf-8")
|
||||
assert self_operated_accounts.load_accounts("bilibili", empty) == []
|
||||
with __import__("pytest").raises(ValueError):
|
||||
self_operated_accounts.load_accounts("kuaishou", empty)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# B站: 公开 API 全字段
|
||||
# ============================================================
|
||||
class _FakeBiliResponse:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeBiliSession:
|
||||
"""固定返回一个 B站 view API 响应的假 HTTP 会话。"""
|
||||
|
||||
PAYLOAD = {
|
||||
"code": 0,
|
||||
"data": {
|
||||
"aid": 114514,
|
||||
"bvid": "BV1xT4y1Q7zz",
|
||||
"title": "光影行星 逐星 开箱",
|
||||
"duration": 92,
|
||||
"pubdate": 1761026400,
|
||||
"owner": {"mid": 5201314, "name": "光影行星官方号"},
|
||||
"stat": {
|
||||
"view": 35821, "like": 1204, "coin": 88, "favorite": 301,
|
||||
"share": 57, "danmaku": 145, "reply": 96,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def get(self, url, **_kwargs):
|
||||
return _FakeBiliResponse(self.PAYLOAD)
|
||||
|
||||
|
||||
def test_fetch_bili_video_extracts_full_metrics():
|
||||
from gyxx_flow.modules.content_marketing.self_bilibili_scraper import (
|
||||
fetch_bili_video,
|
||||
format_duration,
|
||||
)
|
||||
|
||||
info = fetch_bili_video("https://www.bilibili.com/video/BV1xT4y1Q7zz",
|
||||
_FakeBiliSession())
|
||||
assert info is not None
|
||||
assert info["video_id"] == "BV1xT4y1Q7zz"
|
||||
assert info["video_url"].endswith("/video/BV1xT4y1Q7zz")
|
||||
assert info["title"] == "光影行星 逐星 开箱"
|
||||
assert info["owner_name"] == "光影行星官方号"
|
||||
assert info["view_count"] == 35821
|
||||
assert info["like_count"] == 1204
|
||||
assert info["coin_count"] == 88
|
||||
assert info["favorite_count"] == 301
|
||||
assert info["danmaku_count"] == 145
|
||||
assert info["reply_count"] == 96
|
||||
assert info["publish_time"] == dt.datetime.fromtimestamp(1761026400)
|
||||
assert format_duration(92) == "1:32"
|
||||
assert format_duration(3725) == "1:02:05"
|
||||
assert format_duration(None) == ""
|
||||
|
||||
|
||||
def test_collect_bilibili_rows_from_artifact(tmp_path, monkeypatch):
|
||||
artifact = tmp_path / "daily.json"
|
||||
artifact.write_text(
|
||||
json.dumps({
|
||||
"platform": "bilibili",
|
||||
"business_date": "2026-09-28",
|
||||
"videos": [{
|
||||
"video_id": "BV1xT4y1Q7zz",
|
||||
"video_url": "https://www.bilibili.com/video/BV1xT4y1Q7zz",
|
||||
"title": "光影行星 逐星 开箱",
|
||||
"owner_name": "光影行星官方号",
|
||||
"owner_mid": "5201314",
|
||||
"duration_seconds": 92,
|
||||
"publish_time": "2026-09-21 09:39",
|
||||
"view_count": 35821, "like_count": 1204, "coin_count": 88,
|
||||
"favorite_count": 301, "share_count": 57,
|
||||
"danmaku_count": 145, "reply_count": 96,
|
||||
}],
|
||||
}, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(sync_tool, "BILIBILI_ARTIFACT", artifact)
|
||||
accounts = [
|
||||
{"alias": "B站·自营号", "account_name": "光影行星官方号",
|
||||
"platform_account_id": "", "phone": ""}
|
||||
]
|
||||
|
||||
rows, stats, source = sync_tool.collect_bilibili_rows(accounts, dt.date(2026, 9, 28))
|
||||
|
||||
assert stats["rows"] == 1
|
||||
assert source == artifact
|
||||
row = rows[("bilibili", "光影行星官方号", "BV1xT4y1Q7zz")]
|
||||
assert row[sync_tool.ROW_FIELDS.index("account_alias")] == "B站·自营号"
|
||||
assert row[sync_tool.ROW_FIELDS.index("exposure")] == 35821
|
||||
assert row[sync_tool.ROW_FIELDS.index("comment_count")] == 96 # reply→评论
|
||||
assert row[sync_tool.ROW_FIELDS.index("coin_count")] == 88
|
||||
assert row[sync_tool.ROW_FIELDS.index("duration")] == "1:32"
|
||||
|
||||
|
||||
def test_collect_bilibili_rows_missing_artifact(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
sync_tool, "BILIBILI_ARTIFACT", tmp_path / "missing.json"
|
||||
)
|
||||
rows, stats, source = sync_tool.collect_bilibili_rows([], dt.date(2026, 9, 28))
|
||||
assert rows == {} and stats["rows"] == 0 and source is None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 小红书: ARK 管理页表格解析
|
||||
# ============================================================
|
||||
GOODS_HEADERS = ["", "笔记信息", "挂载商品", "笔记阅读数", "笔记支付金额",
|
||||
"笔记支付订单数", "发布状态", "操作"]
|
||||
NORMAL_HEADERS = ["", "笔记信息", "阅读数", "点赞数", "引流主页次数",
|
||||
"引流直播间次数", "发布状态", "操作"]
|
||||
|
||||
|
||||
def test_bilibili_wbi_signing_matches_documented_vector():
|
||||
from gyxx_flow.modules.content_marketing.self_bilibili_scraper import (
|
||||
derive_mixin_key,
|
||||
wbi_sign,
|
||||
)
|
||||
|
||||
# B站 WBI 公开示例密钥 → 文档给定的 mixin key
|
||||
mixin = derive_mixin_key(
|
||||
"7cd084941338484aae1ad9425b84077c",
|
||||
"4932caff0ff746eab6f01bf08b70ac45",
|
||||
)
|
||||
assert mixin == "ea1db124af3c7062474693fa704f4ff8"
|
||||
|
||||
signed = wbi_sign(
|
||||
{"platform": "web", "pn": 1, "ps": 30, "mid": "1008229967",
|
||||
"order": "pubdate"},
|
||||
mixin,
|
||||
1702204800,
|
||||
)
|
||||
assert signed["wts"] == 1702204800
|
||||
assert re.fullmatch(r"[0-9a-f]{32}", signed["w_rid"])
|
||||
# 确定性: 同参数同时间戳签名一致
|
||||
again = wbi_sign(
|
||||
{"mid": "1008229967", "order": "pubdate", "pn": 1, "ps": 30,
|
||||
"platform": "web"},
|
||||
mixin,
|
||||
1702204800,
|
||||
)
|
||||
assert again["w_rid"] == signed["w_rid"]
|
||||
|
||||
|
||||
def test_bilibili_artifact_covers_full_history(tmp_path, monkeypatch):
|
||||
"""主页全量采集后 artifact 应含全部投稿(含飞书未登记的历史视频)。"""
|
||||
artifact = tmp_path / "daily.json"
|
||||
videos = [
|
||||
{
|
||||
"video_id": f"BV1xT4y1Q7z{i:02d}",
|
||||
"video_url": f"https://www.bilibili.com/video/BV1xT4y1Q7z{i:02d}",
|
||||
"title": f"视频{i}",
|
||||
"owner_name": "光影行星",
|
||||
"owner_mid": "1008229967",
|
||||
"duration_seconds": 60 + i,
|
||||
"publish_time": f"2026-09-{i + 1:02d} 09:00:00",
|
||||
"view_count": 100 * i, "like_count": i, "coin_count": i,
|
||||
"favorite_count": i, "share_count": i, "danmaku_count": i,
|
||||
"reply_count": i,
|
||||
}
|
||||
for i in range(3)
|
||||
]
|
||||
artifact.write_text(
|
||||
json.dumps({"platform": "bilibili", "business_date": "2026-09-29",
|
||||
"videos": videos}, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(sync_tool, "BILIBILI_ARTIFACT", artifact)
|
||||
accounts = [
|
||||
{"alias": "B站·自营号", "account_name": "光影行星",
|
||||
"platform_account_id": "1008229967", "phone": ""}
|
||||
]
|
||||
|
||||
rows, stats, _source = sync_tool.collect_bilibili_rows(
|
||||
accounts, dt.date(2026, 9, 29)
|
||||
)
|
||||
|
||||
assert stats["rows"] == 3
|
||||
key = ("bilibili", "光影行星", "BV1xT4y1Q7z00")
|
||||
assert rows[key][sync_tool.ROW_FIELDS.index("account_alias")] == "B站·自营号"
|
||||
assert rows[key][sync_tool.ROW_FIELDS.index("publish_time")] == dt.datetime(
|
||||
2026, 9, 1, 9, 0
|
||||
)
|
||||
|
||||
|
||||
def test_parse_info_cell_variants():
|
||||
parse = self_xhs_scraper._parse_info_cell
|
||||
# 多行,发布时间带前缀(线上真实形态)
|
||||
assert parse("标题A\n发布时间:2026-09-15 14:23\n手动创作") == (
|
||||
"标题A", "2026-09-15 14:23", "手动创作"
|
||||
)
|
||||
# 单行压扁(历史兜底)
|
||||
title, publish, method = parse("标题B 发布时间:2026-09-08 13:41 手动创作")
|
||||
assert title == "标题B"
|
||||
assert publish == "2026-09-08 13:41"
|
||||
assert method == "手动创作"
|
||||
# 无日期无方式的纯标题
|
||||
assert parse("纯标题") == ("纯标题", "", "")
|
||||
assert parse("") == ("", "", "")
|
||||
|
||||
|
||||
def test_parse_goods_note_rows():
|
||||
rows = [
|
||||
{
|
||||
"cells": [
|
||||
"",
|
||||
"斜挎包通勤好物\n2026-09-25 10:23\n手动创作",
|
||||
"光影行星 极星单肩包男女…",
|
||||
"1.2万",
|
||||
"3,586.50",
|
||||
"12",
|
||||
"已发布",
|
||||
"数据 编辑",
|
||||
],
|
||||
"href": "https://www.xiaohongshu.com/explore/66e4b2c3000000001d00f1a2",
|
||||
},
|
||||
{
|
||||
"cells": ["", "另一条\n2026-09-20 08:00", "暂无关联商品", "356",
|
||||
"—", "—", "已发布", "编辑"],
|
||||
"href": "",
|
||||
},
|
||||
]
|
||||
|
||||
records, stats = self_xhs_scraper.parse_management_rows(
|
||||
GOODS_HEADERS, rows, "goods"
|
||||
)
|
||||
|
||||
assert stats["records"] == 2
|
||||
first = records[0]
|
||||
assert first["note_type"] == "goods"
|
||||
assert first["title"] == "斜挎包通勤好物"
|
||||
assert first["publish_time"] == "2026-09-25 10:23"
|
||||
assert first["creation_method"] == "手动创作"
|
||||
assert first["mounted_products"] == "光影行星 极星单肩包男女…"
|
||||
assert first["read_count"] == 12000
|
||||
assert first["payment_amount"] == 3586.50
|
||||
assert first["payment_orders"] == 12
|
||||
assert first["publish_status"] == "已发布"
|
||||
assert first["note_id"] == "66e4b2c3000000001d00f1a2"
|
||||
assert first["video_id"] == "66e4b2c3000000001d00f1a2"
|
||||
# 页面原始列全量保留
|
||||
assert first["raw_columns"]["笔记阅读数"] == "1.2万"
|
||||
# 无链接行回退标题指纹,并跨日稳定
|
||||
second = records[1]
|
||||
assert second["video_id"].startswith("xhs-")
|
||||
assert second["video_id"] == self_xhs_scraper.fallback_note_fingerprint(
|
||||
"另一条", "2026-09-20 08:00"
|
||||
)
|
||||
assert second["mounted_products"] == "暂无关联商品"
|
||||
assert second["payment_amount"] is None
|
||||
assert second["payment_orders"] is None
|
||||
|
||||
|
||||
def test_parse_normal_note_rows():
|
||||
rows = [
|
||||
{
|
||||
"cells": ["", "双肩包种草\n2026-09-22 18:30\n可能是AI创作",
|
||||
"6,592", "483", "35", "8", "已发布", "数据"],
|
||||
"href": "https://www.xiaohongshu.com/discovery/item/66e0abc5000000001900c3b1",
|
||||
},
|
||||
]
|
||||
|
||||
records, stats = self_xhs_scraper.parse_management_rows(
|
||||
NORMAL_HEADERS, rows, "normal"
|
||||
)
|
||||
|
||||
assert stats["records"] == 1
|
||||
record = records[0]
|
||||
assert record["note_type"] == "normal"
|
||||
assert record["read_count"] == 6592
|
||||
assert record["like_count"] == 483
|
||||
assert record["homepage_redirect_count"] == 35
|
||||
assert record["live_redirect_count"] == 8
|
||||
assert record["creation_method"] == "可能是AI创作"
|
||||
assert record["video_id"] == "66e0abc5000000001900c3b1"
|
||||
# 标题含「创作」字样不会被误判为创作方式标签
|
||||
assert record["title"] == "双肩包种草"
|
||||
|
||||
|
||||
def test_parse_count_and_amount_edge_cases():
|
||||
assert self_xhs_scraper.parse_count("1.2万") == 12000
|
||||
assert self_xhs_scraper.parse_count("6,592") == 6592
|
||||
assert self_xhs_scraper.parse_count("0") == 0
|
||||
assert self_xhs_scraper.parse_count("—") is None
|
||||
assert self_xhs_scraper.parse_count("暂无") is None
|
||||
assert self_xhs_scraper.parse_count("") is None
|
||||
assert self_xhs_scraper.parse_amount("3,586.50") == 3586.50
|
||||
assert self_xhs_scraper.parse_amount("0.9万") == 9000.0
|
||||
assert self_xhs_scraper.parse_amount("-") is None
|
||||
|
||||
|
||||
def test_collect_xiaohongshu_rows_from_artifact(tmp_path, monkeypatch):
|
||||
artifact = tmp_path / "daily.json"
|
||||
artifact.write_text(
|
||||
json.dumps({
|
||||
"platform": "xiaohongshu",
|
||||
"business_date": "2026-09-28",
|
||||
"account_name": XHS_SHOP,
|
||||
"notes": [
|
||||
{
|
||||
"note_type": "goods",
|
||||
"video_id": "66e4b2c3000000001d00f1a2",
|
||||
"title": "斜挎包通勤好物",
|
||||
"publish_time": "2026-09-25 10:23",
|
||||
"creation_method": "手动创作",
|
||||
"mounted_products": "光影行星 极星单肩包男女…",
|
||||
"read_count": 12000, "like_count": 431,
|
||||
"payment_amount": 3586.5, "payment_orders": 12,
|
||||
"publish_status": "已发布",
|
||||
"note_url": "https://www.xiaohongshu.com/explore/66e4b2c3000000001d00f1a2",
|
||||
"raw_columns": {"笔记阅读数": "1.2万"},
|
||||
},
|
||||
],
|
||||
}, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(sync_tool, "XHS_ARTIFACT", artifact)
|
||||
accounts = [
|
||||
{"alias": "小红书·自营号", "account_name": XHS_SHOP,
|
||||
"platform_account_id": "18933246725", "phone": "18933246725"}
|
||||
]
|
||||
|
||||
rows, stats, source = sync_tool.collect_xiaohongshu_rows(
|
||||
accounts, dt.date(2026, 9, 28)
|
||||
)
|
||||
|
||||
assert stats["rows"] == 1
|
||||
row = rows[("xiaohongshu", XHS_SHOP, "66e4b2c3000000001d00f1a2")]
|
||||
fields = sync_tool.ROW_FIELDS
|
||||
assert row[fields.index("account_alias")] == "小红书·自营号"
|
||||
assert row[fields.index("exposure")] == 12000 # 阅读数 → 主触达
|
||||
assert row[fields.index("like_count")] == 431
|
||||
assert row[fields.index("payment_amount")] == 3586.5
|
||||
assert row[fields.index("payment_orders")] == 12
|
||||
assert row[fields.index("note_type")] == "goods"
|
||||
assert row[fields.index("publish_time")] == dt.datetime(2026, 9, 25, 10, 23)
|
||||
assert row[fields.index("page_metrics")].obj["笔记阅读数"] == "1.2万"
|
||||
|
||||
|
||||
def test_douyin_history_backfill_uses_file_mtime_and_filename_account(tmp_path):
|
||||
"""历史回填: 按导出落盘日期各留快照,旧版12列文件按文件名前缀补账号。"""
|
||||
# 旧版 12 列表头(无 达人昵称/抖音号/蝉妈妈链接)
|
||||
legacy_headers = [
|
||||
"视频名称", "抖音视频链接", "视频时长", "发布时间", "预估曝光",
|
||||
"点赞数", "评论数", "转发数", "预估销量", "预估销量-指数",
|
||||
"预估销售额", "预估销售额-指数",
|
||||
]
|
||||
legacy_file = tmp_path / "光影行星箱包旗舰店_视频记录_周期内发布视频_20260803-20260901.xlsx"
|
||||
workbook = openpyxl.Workbook()
|
||||
sheet = workbook.active
|
||||
sheet.append(legacy_headers)
|
||||
sheet.append([
|
||||
"旧版格式视频", "https://www.iesdouyin.com/share/video/7686000000000000001/",
|
||||
"46秒", "2026-08-27 16:40:56", "23000", 1321, 46, 14, "0", 0, "0", 0,
|
||||
])
|
||||
workbook.save(legacy_file)
|
||||
workbook.close()
|
||||
import os
|
||||
|
||||
historical = dt.datetime(2026, 9, 2, 17, 0, 0).timestamp()
|
||||
os.utime(legacy_file, (historical, historical))
|
||||
accounts = self_operated_accounts.load_accounts(
|
||||
"douyin", self_operated_accounts.accounts_config_path(PATHS.config_root)
|
||||
)
|
||||
|
||||
rows, stats, _sources = sync_tool.collect_douyin_rows(
|
||||
accounts, dt.date(2026, 9, 29), [str(legacy_file)],
|
||||
use_file_mtime=True,
|
||||
)
|
||||
|
||||
assert stats["skip_no_identity"] == 0
|
||||
assert stats["rows"] == 1
|
||||
row = next(iter(rows.values()))
|
||||
fields = sync_tool.ROW_FIELDS
|
||||
assert row[fields.index("account_name")] == STORE_FLAGSHIP # 文件名前缀回退
|
||||
assert row[fields.index("account_alias")] == "抖音·机能号"
|
||||
assert row[fields.index("metric_date")] == dt.date(2026, 9, 2) # mtime 日期
|
||||
assert row[fields.index("exposure")] == 23000
|
||||
|
||||
|
||||
def test_collect_douyin_rows_keeps_video_note_type(tmp_path):
|
||||
xlsx = _write_export_xlsx(tmp_path / "flagship.xlsx", _douyin_sample_rows())
|
||||
accounts = self_operated_accounts.load_accounts(
|
||||
"douyin", self_operated_accounts.accounts_config_path(PATHS.config_root)
|
||||
)
|
||||
|
||||
rows, stats, _sources = sync_tool.collect_douyin_rows(
|
||||
accounts, dt.date(2026, 9, 28), [xlsx]
|
||||
)
|
||||
|
||||
assert stats["rows"] == 2
|
||||
fields = sync_tool.ROW_FIELDS
|
||||
video_row = rows[
|
||||
("douyin", STORE_FLAGSHIP, "7687797200985869595", dt.date(2026, 9, 28))
|
||||
]
|
||||
note_row = rows[
|
||||
("douyin", STORE_SPECIALTY, "7687432318584143857", dt.date(2026, 9, 28))
|
||||
]
|
||||
assert video_row[fields.index("note_type")] == "video"
|
||||
assert note_row[fields.index("note_type")] == "note"
|
||||
assert video_row[fields.index("account_alias")] == "抖音·机能号"
|
||||
|
||||
|
||||
def test_make_row_validates_fields_and_truncates():
|
||||
row = sync_tool.make_row(
|
||||
"douyin", dt.date(2026, 9, 28), "a.xlsx",
|
||||
title="超长标题" * 200, video_id="1",
|
||||
)
|
||||
fields = sync_tool.ROW_FIELDS
|
||||
assert len(row) == len(fields)
|
||||
assert row[fields.index("title")] == ("超长标题" * 200)[:512]
|
||||
assert row[fields.index("account_alias")] == ""
|
||||
import pytest
|
||||
|
||||
with pytest.raises(KeyError):
|
||||
sync_tool.make_row("douyin", dt.date(2026, 9, 28), "a.xlsx",
|
||||
unknown_field=1)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# SQL 安全 / 表结构 / 配置登记
|
||||
# ============================================================
|
||||
def test_upsert_sql_is_fully_parameterized():
|
||||
source = (TOOLS / "sync_self_video_metrics.py").read_text(encoding="utf-8")
|
||||
matches = re.findall(
|
||||
r"cur\.execute\(\s*\"\"\"\s*(INSERT\b.*?)\"\"\",\s*row,",
|
||||
source,
|
||||
re.DOTALL,
|
||||
)
|
||||
assert len(matches) == 1, "入库 upsert 必须以字面量 SQL + 参数元组形式调用"
|
||||
sql = matches[0]
|
||||
assert sql.count("%s") == len(sync_tool.ROW_FIELDS) # 全部参数绑定
|
||||
assert "{" not in sql and ".format(" not in sql
|
||||
assert "ON CONFLICT (platform, account_name, video_id, metric_date)" in sql
|
||||
|
||||
|
||||
def test_schema_and_migrations_declare_self_video_metrics_consistently():
|
||||
schema = (TOOLS / "schema_gyxx_super_data.sql").read_text(encoding="utf-8")
|
||||
migration_009 = (TOOLS / "migrations" / "009_self_video_metrics.sql").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
migration_010 = (
|
||||
TOOLS / "migrations" / "010_self_video_metrics_platforms.sql"
|
||||
).read_text(encoding="utf-8")
|
||||
for sql in (schema, migration_009, migration_010):
|
||||
assert "cmt_self_video_metrics" in sql
|
||||
for sql in (schema, migration_009):
|
||||
assert "CREATE TABLE IF NOT EXISTS cmt_self_video_metrics" in sql
|
||||
assert "UNIQUE (platform, account_name, video_id, metric_date)" in sql
|
||||
combined = schema + "\n" + migration_009 + "\n" + migration_010
|
||||
for column in (
|
||||
"account_alias", "video_id", "video_url", "publish_time",
|
||||
"exposure", "like_count", "comment_count", "share_count",
|
||||
"est_sales_count", "est_sales_amount", "note_type",
|
||||
"creation_method", "publish_status", "mounted_products",
|
||||
"homepage_redirect_count", "live_redirect_count", "payment_amount",
|
||||
"payment_orders", "coin_count", "favorite_count", "danmaku_count",
|
||||
"page_metrics",
|
||||
):
|
||||
assert re.search(rf"\b{column}\b", combined), f"{column} 缺失"
|
||||
|
||||
|
||||
def test_workflow_commands_and_bindings_declare_self_content_pipeline():
|
||||
workflows = json.loads(
|
||||
(PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8")
|
||||
)
|
||||
workflow = next(
|
||||
item
|
||||
for item in workflows["workflows"]
|
||||
if item["id"] == "content.metrics.self_operated.daily"
|
||||
)
|
||||
steps = {step["id"]: step for step in workflow["execution"]["steps"]}
|
||||
assert set(steps) == {
|
||||
"refresh_self_mapping", "collect_self_bilibili", "collect_self_douyin",
|
||||
"collect_self_xiaohongshu", "sync", "sync_self_video_metrics",
|
||||
}
|
||||
assert steps["collect_self_xiaohongshu"]["entry"] == "self_xhs_scraper.py"
|
||||
assert steps["collect_self_xiaohongshu"]["depends_on"] == ["collect_self_douyin"]
|
||||
assert steps["sync_self_video_metrics"]["depends_on"] == [
|
||||
"sync", "collect_self_xiaohongshu",
|
||||
]
|
||||
module_root = PROJECT_ROOT / "src/gyxx_flow/modules/content_marketing"
|
||||
for entry in (
|
||||
"self_xhs_scraper.py",
|
||||
"data/tools/sync_self_video_metrics.py",
|
||||
):
|
||||
assert (module_root / entry).is_file()
|
||||
|
||||
commands = json.loads(
|
||||
(PROJECT_ROOT / "config" / "commands.json").read_text(encoding="utf-8")
|
||||
)
|
||||
command_entries = {item["id"]: item for item in commands["commands"]}
|
||||
assert command_entries["content.metrics.collect_self_xiaohongshu"]["entry"] == (
|
||||
"self_xhs_scraper.py"
|
||||
)
|
||||
assert command_entries["content.metrics.sync_self_video_metrics"]["entry"] == (
|
||||
"data/tools/sync_self_video_metrics.py"
|
||||
)
|
||||
|
||||
bindings = json.loads(
|
||||
(PROJECT_ROOT / "config" / "runtime-bindings.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
xhs_binding = bindings["scripts"]["content.metrics.collect_self_xiaohongshu"]
|
||||
assert xhs_binding["script_id"] == "content_marketing:self_xhs_scraper.py"
|
||||
assert xhs_binding["required_cookie_domains"] == ["xiaohongshu.com"]
|
||||
assert "XHS_ARK_ACCOUNT" in xhs_binding["credential_env_names"]
|
||||
sync_binding = bindings["scripts"]["content.metrics.sync_self_video_metrics"]
|
||||
assert sync_binding["cdp_port"] == 22164
|
||||
ports = [entry["cdp_port"] for entry in bindings["scripts"].values()]
|
||||
assert len(ports) == len(set(ports)), "CDP 端口必须唯一"
|
||||
|
||||
|
||||
def test_metric_date_resolution_and_stale_guard(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv("GYXX_BUSINESS_DATE", raising=False)
|
||||
assert sync_tool.runtime_business_date() is None
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-09-28")
|
||||
assert sync_tool.runtime_business_date() == dt.date(2026, 9, 28)
|
||||
assert sync_tool.resolve_metric_date("2026-09-01") == dt.date(2026, 9, 1)
|
||||
|
||||
xlsx = Path(_write_export_xlsx(tmp_path / "a.xlsx", _douyin_sample_rows()))
|
||||
fresh = dt.datetime.now().timestamp()
|
||||
os.utime(xlsx, (fresh, fresh))
|
||||
assert sync_tool.is_current_artifact(xlsx, dt.date.today())
|
||||
assert not sync_tool.is_current_artifact(xlsx, dt.date(2026, 9, 21))
|
||||
|
||||
|
||||
def test_runtime_business_date_rejects_invalid_value(monkeypatch):
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026/09/28")
|
||||
with __import__("pytest").raises(RuntimeError):
|
||||
sync_tool.runtime_business_date()
|
||||
Reference in new issue
Block a user