feat: complete production workflow migration

This commit is contained in:
2026-08-06 14:29:57 +08:00
parent 7f215e79c4
commit 8df5266abb
448 changed files with 56937 additions and 14619 deletions
@@ -0,0 +1,57 @@
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from gyxx_flow.modules.content_marketing.data.tools import (
analyze_note, # noqa: E402
)
class AnalyzeNoteWithoutCommentsTest(unittest.TestCase):
def test_no_comments_still_writes_metrics_report(self):
info = {
"title": "无评论测试笔记",
"platform": "xiaohongshu",
"source_url": "https://example.com/note",
"scraped_at": "",
"comment_count_total": 0,
"like_count": 12,
"favorite_count": 3,
"comment_count_metric": 0,
"share_count": 1,
"view_count": 1000,
"collect_count_yxyy": 0,
"style_name": "测试款",
"creator_name": "测试达人",
}
with tempfile.TemporaryDirectory() as tmpdir:
output = Path(tmpdir) / "note.txt"
argv = [
"analyze_note.py",
"--note-id",
"1",
"--no-send",
"--brand",
"光影行星",
"--output",
str(output),
]
with patch.object(
analyze_note, "load_note_from_db", return_value=(info, [])
), patch.object(
analyze_note, "resolve_layer_output", return_value=output
), patch.object(sys, "argv", argv):
result = analyze_note.main()
self.assertEqual(result, 0)
self.assertTrue(output.exists())
report = output.read_text(encoding="utf-8-sig")
self.assertIn("无有效评论可分析", report)
self.assertIn("1,000", report)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,11 @@
from gyxx_flow.modules.content_marketing.data.tools import (
batch_rescrape_douyin as batch,
)
def test_result_logged_in_reads_nested_scraper_stats():
result = {"stats": {"logged_in": True}}
assert batch.result_logged_in(result) is True
assert batch.result_logged_in({"stats": {"logged_in": False}}) is False
assert batch.result_logged_in({}) is False
@@ -0,0 +1,253 @@
import pytest
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
from gyxx_flow.modules.content_marketing import chanmama_scraper as chanmama
def test_exposure_state_clicks_text_and_zero_but_keeps_positive_values():
for value in ("更新数据", " 更新中 ", "暂无数据", "待更新", "0", "0.0", 0):
assert chanmama._classify_exposure_text(value) == "refresh"
for value in (23000, "23,000", "10w+", "8.8万+", ""):
assert chanmama._classify_exposure_text(value) == "ready"
def test_refresh_then_export_waits_once_after_all_accounts(monkeypatch):
events = []
urls = ["account-1", "account-2"]
monkeypatch.setattr(chanmama, "navigate_to_target", lambda _driver, url: events.append(("navigate", url)))
results = iter([
{"clicked": 2, "pending": 0, "zero_without_button": 0, "pages": 2},
{"clicked": 0, "pending": 1, "zero_without_button": 0, "pages": 1},
])
monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: next(results))
monkeypatch.setattr(chanmama.time, "sleep", lambda seconds: events.append(("sleep", seconds)))
monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 123.0)
monkeypatch.setattr(chanmama, "export_video_data", lambda _driver, mtime: events.append(("export", mtime)) or "x.xlsx")
monkeypatch.setattr(chanmama, "parse_chanmama_excel", lambda _path: ([{"title": "video"}], None))
records, summaries = chanmama.refresh_then_export_accounts(object(), urls, wait_seconds=600)
assert len(records) == 2
assert [item["clicked"] for item in summaries] == [2, 0]
assert events == [
("navigate", "account-1"), ("navigate", "account-2"), ("sleep", 600),
("navigate", "account-1"), ("export", 123.0),
("navigate", "account-2"), ("export", 123.0),
]
def test_refresh_then_export_skips_wait_when_nothing_needs_refresh(monkeypatch):
sleeps = []
monkeypatch.setattr(chanmama, "navigate_to_target", lambda *_args: None)
monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: {
"clicked": 0, "pending": 0, "zero_without_button": 0, "pages": 1,
})
monkeypatch.setattr(chanmama.time, "sleep", sleeps.append)
monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 0.0)
monkeypatch.setattr(chanmama, "export_video_data", lambda *_args: None)
records, _ = chanmama.refresh_then_export_accounts(object(), ["account-1"], wait_seconds=600)
assert records == []
assert sleeps == []
class _RefreshElement:
def __init__(self, text="", css_class="", element_id="node"):
self.text = text
self.css_class = css_class
self.id = element_id
def is_displayed(self):
return True
def get_attribute(self, name):
return self.css_class if name == "class" else None
def click(self):
return None
class _RefreshDriver:
def __init__(self, nodes=None):
self.nodes = list(nodes or [])
def find_elements(self, _by, selector):
return self.nodes if selector == "td .cursor-pointer" else []
def test_next_page_button_keeps_the_source_xpath_fallback():
expected = "//span[normalize-space()='下一页']/ancestor::button[1]"
button = _RefreshElement("下一页", element_id="next")
class Driver:
def find_elements(self, _by, selector):
return [button] if selector == expected else []
assert chanmama._find_next_page_button(Driver()) is button
def test_find_get_data_targets_selects_actual_cursor_pointer_only():
outer = _RefreshElement("获取数据", "detail-text", "outer")
clickable = _RefreshElement("获取数据", "cp cursor-pointer", "inner")
other = _RefreshElement("查看详情", "cursor-pointer", "other")
assert chanmama._find_get_data_targets(_RefreshDriver([outer, clickable, other])) == [clickable]
def test_refresh_zero_exposure_videos_walks_all_pages(monkeypatch):
class Button:
def __init__(self, disabled):
self.disabled = disabled
class Driver:
page = 0
def execute_script(self, _script, _element):
self.page += 1
driver = Driver()
page_results = [
{"clicked": 2, "pending": 1, "zero_without_button": 0},
{"clicked": 1, "pending": 0, "zero_without_button": 1},
]
monkeypatch.setattr(
chanmama, "_page_signature", lambda current: f"page-{current.page}"
)
monkeypatch.setattr(
chanmama,
"_refresh_current_page",
lambda current: page_results[current.page],
)
monkeypatch.setattr(
chanmama,
"_find_next_page_button",
lambda current: Button(disabled=current.page == 1),
)
monkeypatch.setattr(
chanmama,
"_pagination_element_disabled",
lambda button: button.disabled,
)
monkeypatch.setattr(chanmama.time, "sleep", lambda _seconds: None)
assert chanmama.refresh_zero_exposure_videos(driver) == {
"clicked": 3,
"pending": 1,
"zero_without_button": 1,
"unconfirmed": 0,
"pages": 2,
}
def test_missing_get_data_button_does_not_trigger_refresh_wait(monkeypatch):
sleeps = []
monkeypatch.setattr(chanmama, "navigate_to_target", lambda *_args: None)
monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: {
"clicked": 0, "pending": 0, "zero_without_button": 1, "pages": 1,
})
monkeypatch.setattr(chanmama.time, "sleep", sleeps.append)
monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 0.0)
monkeypatch.setattr(chanmama, "export_video_data", lambda *_args: None)
chanmama.refresh_then_export_accounts(object(), ["account-1"], wait_seconds=600)
assert sleeps == []
def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
events = []
class Driver:
def quit(self):
events.append("quit")
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
monkeypatch.setattr(chanmama, "save_cookies", lambda _driver: events.append("saved"))
monkeypatch.setattr(
chanmama,
"load_cookies",
lambda _driver: (_ for _ in ()).throw(AssertionError("profile must win")),
)
monkeypatch.setattr(
chanmama,
"login",
lambda _driver: (_ for _ in ()).throw(AssertionError("profile must win")),
)
monkeypatch.setattr(
chanmama,
"refresh_then_export_accounts",
lambda *_args, **_kwargs: ([], []),
)
assert chanmama.main() == 0
assert events == ["saved", "quit"]
def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
monkeypatch,
tmp_path,
):
evidence = tmp_path / "evidence.jsonl"
class Driver:
def quit(self):
pass
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(chanmama, "ACCOUNT", "")
monkeypatch.setattr(chanmama, "PASSWORD", "")
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: False)
monkeypatch.setattr(chanmama, "load_cookies", lambda _driver: False)
monkeypatch.setattr(
chanmama,
"login",
lambda _driver: (_ for _ in ()).throw(AssertionError("must not login")),
)
monkeypatch.setattr(
chanmama,
"refresh_then_export_accounts",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
AssertionError("must not scrape")
),
)
assert chanmama.main() == COOKIE_SKIP_EXIT_CODE
assert "profile/cookie session is invalid" in evidence.read_text(encoding="utf-8")
def test_acceptance_captcha_is_non_interactive(monkeypatch, tmp_path):
evidence = tmp_path / "evidence.jsonl"
class Element:
def is_displayed(self):
return True
class Driver:
current_url = "https://www.chanmama.com/login.html"
page_source = ""
def find_element(self, *_args):
return Element()
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
monkeypatch.setattr(chanmama.time, "sleep", lambda _seconds: None)
assert chanmama.handle_captcha(Driver()) is False
assert "captcha or interactive verification" in evidence.read_text(
encoding="utf-8"
)
def test_cli_preserves_cookie_skip_exit_code(monkeypatch):
monkeypatch.setattr(chanmama, "main", lambda: COOKIE_SKIP_EXIT_CODE)
with pytest.raises(SystemExit) as raised:
chanmama.cli()
assert raised.value.code == COOKIE_SKIP_EXIT_CODE
@@ -0,0 +1,840 @@
import argparse
from pathlib import Path
import pytest
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
from gyxx_flow.modules.content_marketing import collection_completeness as cc
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
from gyxx_flow.modules.content_marketing import run_all
from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
def test_normalize_title_handles_nfkc_case_and_zero_width():
assert cc.normalize_title("MacBook\u200b 通勤包!") == "macbook通勤包"
def test_coerce_url_unwraps_feishu_markdown_links():
value = "[查看笔记](https://www.xiaohongshu.com/discovery/item/abc123?x=1)"
assert cc.coerce_url(value) == "https://www.xiaohongshu.com/discovery/item/abc123?x=1"
def test_match_tasks_prefers_note_id_over_changed_title():
tasks = [{
"record_id": "r1",
"target_title": "飞书中的旧标题",
"note_url": "https://www.xiaohongshu.com/explore/abc123",
"note_id": "abc123",
}]
cards = [
{"title": "完全不同的新标题", "href": "https://www.xiaohongshu.com/explore/abc123", "note_id": "abc123"},
{"title": "飞书中的旧标题", "href": "https://www.xiaohongshu.com/explore/other", "note_id": "other"},
]
matched = cc.match_tasks_to_cards(tasks, cards, platform="xhs")
assert matched["r1"]["note_id"] == "abc123"
assert matched["r1"]["match_method"] == "content_id"
def test_match_tasks_accepts_unique_truncated_title_but_rejects_ambiguous_title():
task = {"record_id": "r1", "target_title": "男生长期主义通勤双肩包分享"}
unique = [
{"title": "男生长期主义通勤双肩包"},
{"title": "夏日轻量斜挎包"},
]
ambiguous = [
{"title": "男生长期主义通勤双肩包"},
{"title": "男生长期主义通勤双肩包"},
]
assert cc.match_tasks_to_cards([task], unique, platform="douyin")["r1"]
assert "r1" not in cc.match_tasks_to_cards([task], ambiguous, platform="douyin")
def test_title_similarity_ignores_episode_prefix_and_hashtag_suffix():
target = "第20集:Pocket4拍旋焦,无后期也能出片?"
platform_title = (
"Pocket4拍旋焦,无后期也能出片? #旋焦 #pocket4 #摄影装备 "
"无论Pocket3还是Pocket4都可以轻松拍出旋焦效果!"
)
assert cc.title_similarity(platform_title, target) >= 0.98
def test_title_similarity_rejects_different_episode_numbers():
assert cc.title_similarity(
"第19集:Pocket4拍旋焦,无后期也能出片?",
"第20集:Pocket4拍旋焦,无后期也能出片?",
) == 0.0
def test_match_tasks_rejects_duplicate_exact_titles_with_different_ids():
task = {"record_id": "r1", "target_title": "完全相同的目标标题"}
cards = [
{"title": "完全相同的目标标题", "note_id": "video-1"},
{"title": "完全相同的目标标题", "note_id": "video-2"},
]
assert "r1" not in cc.match_tasks_to_cards([task], cards, platform="douyin")
def test_one_card_is_not_reused_for_two_different_creator_tasks():
tasks = [
{
"record_id": "loose",
"target_title": "commuter backpack review today",
"note_url": "https://v.douyin.com/loose-source/",
},
{
"record_id": "exact",
"target_title": "commuter backpack review",
"note_url": "https://v.douyin.com/exact-source/",
},
]
cards = [{"title": "commuter backpack review", "note_id": "observed-1"}]
matched = cc.match_tasks_to_cards(tasks, cards, platform="douyin")
assert set(matched) == {"exact"}
def test_same_source_url_can_share_one_card_across_styles():
tasks = [
{
"record_id": "style-1",
"target_title": "同一篇跨款式笔记",
"note_url": "https://v.douyin.com/same-short-link/",
},
{
"record_id": "style-2",
"target_title": "同一篇跨款式笔记",
"note_url": "https://v.douyin.com/same-short-link/",
},
]
cards = [{"title": "同一篇跨款式笔记", "note_id": "observed-1"}]
assert set(cc.match_tasks_to_cards(tasks, cards, platform="douyin")) == {
"style-1", "style-2",
}
def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
task = {
"record_id": "r1",
"target_title": "完全相同标题",
"note_id": "video-a",
}
card = {"title": "完全相同标题", "note_id": "video-b"}
assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
cards = [
{
"title": "同一条视频",
"play_count": 19481,
"note_id": "video-1",
"source": "show_items_api",
},
{
"title": "同一条视频",
"play_count": "1.9w",
"note_id": "",
"page": 1,
},
]
deduped = cc.deduplicate_observed_cards(cards)
matched = cc.match_tasks_to_cards(
[{"record_id": "r1", "target_title": "同一条视频"}],
deduped,
platform="douyin",
)
assert len(deduped) == 1
assert matched["r1"]["note_id"] == "video-1"
def test_merge_partial_summary_keeps_existing_success_rows():
existing = {
"style": "款式A", "index": 1, "total": 2, "matched": 1, "filled": 1,
"results": [
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
{"record_id": "retry", "status": "retryable_failure", "matched": False},
],
}
partial = {
"style": "款式A", "index": 1, "total": 1, "matched": 1, "filled": 1,
"results": [
{"record_id": "retry", "status": "success", "matched": True, "write_ok": True},
],
}
merged = cc.merge_style_summary(existing, partial)
assert merged["total"] == 2
assert {row["record_id"] for row in merged["results"]} == {"ok", "retry"}
assert merged["matched"] == 2
assert merged["filled"] == 2
assert merged["complete"] is True
def test_summary_contract_detects_missing_or_write_failure():
summary = {
"total": 2,
"results": [
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
{"record_id": "bad", "status": "write_failure", "matched": True, "write_ok": False},
],
}
finalized = cc.finalize_summary(summary, dry_run=False)
assert finalized["complete"] is False
assert finalized["unresolved"] == 1
missing = cc.finalize_summary({"total": 2, "results": summary["results"][:1]}, dry_run=False)
assert missing["complete"] is False
assert missing["missing_results"] == 1
def _style_for_platform(platform: str):
return {
"index": 1,
"name": "款式A",
"base_token": "base",
"table_id": "table",
"field_map": {
"creator_name": {"field_id": "name"},
"creator_id": {"field_id": "creator_id"},
"note_title": {"field_id": "title"},
"publish_time": {"field_id": "pub"},
"note_url": {"field_id": "url"},
"platform": {"field_id": "platform"},
"read_count_7d": {"field_id": "read7"},
},
}
def test_extractors_preserve_note_url_and_content_id(monkeypatch):
xhs_row = {
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
"title": "标题", "pub": "2026-07-20", "platform": "小红书",
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
}
dy_row = {
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
"title": "标题", "pub": "2026-07-20", "platform": "抖音",
"url": "https://www.douyin.com/video/7654321", "read7": None,
}
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
xhs_task = pgy.extract_target_tasks(_style_for_platform("小红书"))[0]
dy_task = xingtu.extract_target_tasks(_style_for_platform("抖音"))[0]
assert xhs_task["note_url"].endswith("/abc123")
assert xhs_task["note_id"] == "abc123"
assert dy_task["note_url"].endswith("/7654321")
assert dy_task["note_id"] == "7654321"
@pytest.mark.parametrize(
("module", "platform", "filled_url"),
[
(pgy, "小红书", "https://www.douyin.com/video/7654321"),
(xingtu, "抖音", "http://8.99 分享文案 https://v.douyin.com/abc123/"),
],
)
def test_filled_publish_link_triggers_collection_without_platform_validation(
monkeypatch, module, platform, filled_url,
):
row = {
"record_id": "r1",
"name": "达人",
"creator_id": "creator-id",
"title": "已发布标题",
"pub": "2026-07-20",
"platform": platform,
"url": filled_url,
"read7": None,
}
monkeypatch.setattr(module, "list_records_by_table", lambda *_: [row])
task = module.extract_target_tasks(_style_for_platform(platform))[0]
assert task["input_error"] is None
def test_extractors_can_collect_missing_title_when_direct_url_has_content_id(monkeypatch):
xhs_style = _style_for_platform("小红书")
dy_style = _style_for_platform("抖音")
xhs_row = {
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
"title": None, "pub": "2026-07-20", "platform": "小红书",
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
}
dy_row = {
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
"title": None, "pub": "2026-07-20", "platform": "抖音",
"url": "https://www.douyin.com/video/7654321", "read7": None,
}
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
assert pgy.extract_target_tasks(xhs_style)[0]["note_id"] == "abc123"
assert xingtu.extract_target_tasks(dy_style)[0]["note_id"] == "7654321"
assert xhs_style["_extract_stats"]["skipped_incomplete"] == 0
assert dy_style["_extract_stats"]["skipped_incomplete"] == 0
def test_missing_critical_mapping_is_an_incomplete_summary(monkeypatch):
style = _style_for_platform("抖音")
del style["field_map"]["note_url"]
monkeypatch.setattr(
xingtu,
"list_records_by_table",
lambda *_: (_ for _ in ()).throw(AssertionError("mapping should fail first")),
)
_, summaries = xingtu.collect_tasks_across_styles([style], None)
finalized = cc.finalize_summary(summaries[1], dry_run=True)
assert "note_url" in finalized["error"]
assert finalized["complete"] is False
assert finalized["unresolved"] >= 1
class _FakeInput:
def __init__(self):
self.filled = []
def click(self):
return None
def fill(self, value):
self.filled.append(value)
def press(self, _key):
return None
class _FakeLocator:
def __init__(self, input_):
self.first = input_
def filter(self, **_kwargs):
return self
class _FakeSearchPage:
def __init__(self):
self.input = _FakeInput()
self.has_result_calls = 0
def wait_for_timeout(self, _ms):
return None
def evaluate(self, script, arg=None):
if "const rows" in script:
self.has_result_calls += 1
return self.has_result_calls >= 2
if "const inputs" in script:
return ""
raise AssertionError(script)
def locator(self, _selector):
return _FakeLocator(self.input)
def wait_for_url(self, *_args, **_kwargs):
return None
def test_pgy_search_retry_keeps_creator_id_query():
page = _FakeSearchPage()
pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
assert page.input.filled == ["xhs-id-123"]
def test_apply_matched_task_reports_write_failure(monkeypatch):
task = {
"record_id": "r1", "publish_time": object(),
"style_context": {
"index": 1, "base_token": "base", "table_id": "table", "field_map": {},
},
}
summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []}
monkeypatch.setattr(pgy, "pick_read_field", lambda *_: ("read_count_7d", "fid", "7天曝光量"))
monkeypatch.setattr(pgy, "write_back", lambda *_: False)
outcome = pgy.apply_matched_task(task, {"read_count": "123", "title": "标题"}, summary, dry_run=False)
assert outcome["status"] == "write_failure"
assert outcome["write_ok"] is False
def test_default_creator_batch_has_no_forced_hour_pause():
assert pgy.DEFAULT_BATCH_SIZE == 0
assert xingtu.DEFAULT_BATCH_SIZE == 0
def test_repeatable_style_argument_and_selection_keep_every_requested_style():
parser = argparse.ArgumentParser()
cc.add_repeatable_style_argument(parser, "styles")
args = parser.parse_args(["--style", "1", "--style", "3"])
styles = [{"index": 1}, {"index": 2}, {"index": 3}]
assert args.style == [1, 3]
assert cc.select_requested_styles(styles, args.style) == [styles[0], styles[2]]
with pytest.raises(ValueError, match="99"):
cc.select_requested_styles(styles, [1, 99])
def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
sequence = [[], [{"title": "标题", "read_count": "1"}], [{"title": "标题", "read_count": "1"}]]
monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
class Page:
waits = 0
def wait_for_timeout(self, _ms):
self.waits += 1
page = Page()
assert pgy.parse_cards_stable(page) == [{"title": "标题", "read_count": "1"}]
assert page.waits == 2
def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
pages = [
[{"title": "无关视频", "play_count": "10"}],
[{"title": "目标视频标题", "play_count": "20"}],
]
calls = {"next": 0}
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
monkeypatch.setattr(xingtu, "parse_videos_stable", lambda _page: pages.pop(0))
def next_page(_page):
calls["next"] += 1
return True
monkeypatch.setattr(xingtu, "go_next_page", next_page)
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
found, _, _ = xingtu.find_videos_for_tasks(
object(), [{"record_id": "r1", "target_title": "目标视频标题"}], max_pages=2,
)
assert found["r1"]["play_count"] == "20"
assert calls["next"] == 1
def test_xingtu_normalizes_show_items_api_video():
card = xingtu.normalize_xingtu_api_item({
"item_id": 7654321,
"item_title": "目标视频标题",
"play": 168000,
"like": 1200,
"comment": 34,
"share": 56,
"url": "https://www.douyin.com/video/7654321",
})
assert card == {
"title": "目标视频标题",
"play_count": 168000,
"like_count": 1200,
"comment_count": 34,
"share_count": 56,
"href": "https://www.douyin.com/video/7654321",
"note_id": "7654321",
"source": "show_items_api",
}
def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
class Response:
url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
def json(self):
return {
"latest_item_info": [
{"item_id": "1", "title": "视频一", "play": 10},
{"item_id": "2", "item_title": "视频二", "play": 20},
],
"latest_star_item_info": [
{"item_id": "2", "item_title": "视频二", "play": 20},
],
}
class Page:
def on(self, event, callback):
assert event == "response"
self.callback = callback
page = Page()
cards = xingtu.setup_show_items_capture(page)
page.callback(Response())
assert [card["note_id"] for card in cards] == ["1", "2"]
def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
monkeypatch.setattr(
xingtu,
"parse_videos_stable",
lambda _page: (_ for _ in ()).throw(AssertionError("DOM should not be needed")),
)
monkeypatch.setattr(
xingtu,
"is_blocked",
lambda _page: (_ for _ in ()).throw(AssertionError("page should not be needed")),
)
tasks = [{
"record_id": "r1",
"target_title": "飞书旧标题",
"note_url": "https://www.douyin.com/video/7654321",
"note_id": "7654321",
}]
api_cards = [{
"title": "平台上的新标题",
"play_count": 168000,
"note_id": "7654321",
"source": "show_items_api",
}]
found, _, page_limit_hit = xingtu.find_videos_for_tasks(
object(), tasks, max_pages=5, api_cards=api_cards,
)
assert found["r1"]["play_count"] == 168000
assert found["r1"]["match_method"] == "content_id"
assert page_limit_hit is False
def test_xingtu_api_fuzzy_candidate_does_not_preempt_later_exact_dom(monkeypatch):
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
monkeypatch.setattr(
xingtu,
"parse_videos_stable",
lambda _page: [{"title": "commuter backpack review", "note_id": "exact-card"}],
)
monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False)
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
task = {"record_id": "r1", "target_title": "commuter backpack review"}
api_cards = [{
"title": "commuter backpack review today",
"note_id": "fuzzy-card",
"source": "show_items_api",
}]
found, _, _ = xingtu.find_videos_for_tasks(
object(), [task], max_pages=2, api_cards=api_cards,
)
assert found["r1"]["note_id"] == "exact-card"
def test_browser_session_loss_detection_is_specific():
closed = RuntimeError("Target page, context or browser has been closed")
assert pgy.is_browser_session_lost(closed)
assert xingtu.is_browser_session_lost(closed)
assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
def test_xingtu_nested_session_loss_is_reraised():
closed = RuntimeError("Target page, context or browser has been closed")
with pytest.raises(RuntimeError, match="has been closed"):
xingtu.reraise_if_browser_session_lost(closed)
xingtu.reraise_if_browser_session_lost(RuntimeError("ordinary lookup miss"))
def test_xingtu_detail_and_video_search_propagate_closed_session():
closed = RuntimeError("Target page, context or browser has been closed")
class Detail:
def goto(self, *_args, **_kwargs):
raise closed
class Context:
pages = []
def new_page(self):
return Detail()
class SearchPage:
context = Context()
def evaluate(self, _script, _arg=None):
return {"found": True, "url": "https://www.xingtu.cn/ad/creator/author-homepage/douyin-video/1"}
with pytest.raises(RuntimeError, match="has been closed"):
xingtu.open_creator_detail(SearchPage(), "达人", "creator-id")
class Locator:
first = None
def __init__(self):
self.first = self
def count(self):
return 1
def wait_for(self, **_kwargs):
raise closed
class VideoPage:
def locator(self, _selector):
return Locator()
with pytest.raises(RuntimeError, match="has been closed"):
xingtu.submit_video_search(VideoPage(), "标题")
def test_pgy_fallback_detail_click_propagates_closed_session():
closed = RuntimeError("Target page, context or browser has been closed")
class Clicker:
first = None
def __init__(self):
self.first = self
def click(self, **_kwargs):
raise closed
class Context:
pages = []
class Page:
context = Context()
def evaluate(self, *_args, **_kwargs):
return {"ok": False, "reason": "no-row"}
def get_by_text(self, *_args, **_kwargs):
return Clicker()
with pytest.raises(RuntimeError, match="has been closed"):
pgy.open_blogger_detail(Page(), "达人", None)
def test_bilibili_triggered_blocked_inputs_are_visible_terminal_rows():
summary = bili.finalize_bili_summary({
"style": "款式A", "index": 1, "total_b_records": 1,
"details": [{
"record_id": "r1", "status": "blocked_input",
"matched": False, "reason": "publish_time_missing", "ok": False,
}],
})
assert summary["blocked_input"] == 1
assert summary["unresolved"] == 0
assert summary["complete"] is True
def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
style = {
"style": "款式A",
"name": "款式A",
"index": 1,
"base_token": "base",
"table_id": "table",
"field_map": {
"platform": {"field_id": "platform"},
"note_url": {"field_id": "url"},
"creator_name": {"field_id": "creator"},
"publish_time": {"field_id": "pub"},
"read_count_7d": {"field_id": "s7"},
"read_count_14d": {"field_id": "s14"},
"read_count_21d": {"field_id": "s21"},
"read_count_28d": {"field_id": "s28"},
"month_end": {"field_id": "sm"},
},
}
rows = [
{
"record_id": "no-link",
"platform": "B站",
"creator": "未发布达人",
"url": "",
"pub": "2026-07-20",
},
{
"record_id": "published",
"platform": "B站",
"creator": "已发布达人",
"url": "https://www.bilibili.com/video/BV1234567890",
"pub": "2026-07-20",
},
]
monkeypatch.setattr(bili, "list_all_records", lambda *_args: rows)
monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: 321)
monkeypatch.setattr(bili, "write_record", lambda *_args, **_kwargs: True)
summary = bili.process_style(
style,
only_record_ids=None,
dry_run=True,
delay=0,
session=object(),
state={},
force_today=bili.date(2026, 7, 23),
first_run=False,
)
assert summary["source_b_records"] == 2
assert summary["skipped_no_url"] == 1
assert summary["total_b_records"] == 1
assert summary["success"] == 1
assert summary["blocked_input"] == 0
assert [row["record_id"] for row in summary["details"]] == ["published"]
def test_bilibili_partial_retry_preserves_old_success():
existing = {
"style": "款式A", "index": 1, "total_b_records": 2,
"details": [
{"record_id": "ok", "status": "success", "ok": True},
{"record_id": "retry", "status": "retryable_failure", "ok": False},
],
}
partial = {
"style": "款式A", "index": 1, "total_b_records": 1,
"details": [{"record_id": "retry", "status": "success", "ok": True}],
}
merged = bili.merge_bili_summary(existing, partial)
assert merged["total_b_records"] == 2
assert {row["record_id"] for row in merged["details"]} == {"ok", "retry"}
assert merged["complete"] is True
def test_bilibili_top_level_error_cannot_finalize_as_complete():
finalized = bili.finalize_bili_summary({
"style": "款式A",
"index": 1,
"total_b_records": 0,
"details": [],
"error": "missing platform/url field_id",
"unresolved": 1,
"retryable_failures": 1,
"complete": False,
})
assert finalized["complete"] is False
assert finalized["unresolved"] >= 1
def test_bilibili_missing_slot_mapping_is_system_error_not_blocked_input():
summary = bili.process_style(
{
"style": "款式A",
"name": "款式A",
"index": 1,
"base_token": "base",
"table_id": "table",
"field_map": {
"platform": {"field_id": "platform"},
"note_url": {"field_id": "url"},
},
},
only_record_ids=None,
dry_run=True,
delay=0,
session=object(),
state={},
force_today=None,
first_run=False,
)
finalized = bili.finalize_bili_summary(summary, dry_run=True)
assert "publish_time" in finalized["error"]
assert "read_count_7d" in finalized["error"]
assert finalized["complete"] is False
def test_self_douyin_missing_mapping_and_write_failure_are_not_complete(monkeypatch):
missing = self_dy.scrape_one_style(
{"name": "款式A", "index": 1, "base_token": "base", "table_id": "table", "field_map": {}},
login_timeout=1,
headless=True,
only_record_ids=None,
dry_run=False,
)
assert cc.finalize_summary(missing)["complete"] is False
style = _style_for_platform("抖音")
row = {
"record_id": "r1",
"platform": "抖音",
"name": "自营达人",
"title": "目标作品",
"url": "https://www.douyin.com/video/1",
"pub": "2026-07-20",
}
class Page:
def goto(self, *_args, **_kwargs):
return None
def wait_for_timeout(self, _ms):
return None
class Session:
def __init__(self, **_kwargs):
pass
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def fetch(self, _url, page_action, wait):
page_action(Page())
monkeypatch.setattr(self_dy, "DynamicSession", Session)
monkeypatch.setattr(self_dy, "list_records_by_table", lambda *_: [row])
monkeypatch.setattr(self_dy, "dismiss_popups", lambda *_: None)
monkeypatch.setattr(self_dy, "restore_cookies", lambda *_: None)
monkeypatch.setattr(self_dy, "maybe_wait_for_login", lambda *_: None)
monkeypatch.setattr(self_dy, "save_state", lambda *_: None)
monkeypatch.setattr(
self_dy,
"scroll_and_collect_posts",
lambda *_args, **_kwargs: [{"title": "目标作品", "play_count": 123}],
)
monkeypatch.setattr(self_dy, "pick_read_field", lambda *_: ("read_count_7d", "read7", "7天曝光量"))
monkeypatch.setattr(self_dy, "write_back", lambda *_: False)
failed = cc.finalize_summary(
self_dy.scrape_one_style(
style,
login_timeout=1,
headless=True,
only_record_ids=None,
dry_run=False,
)
)
assert failed["results"][0]["status"] == "write_failure"
assert failed["complete"] is False
def test_run_all_rejects_stale_or_incomplete_summary(tmp_path: Path):
path = tmp_path / "summary.json"
path.write_text("[]", encoding="utf-8")
started_at = path.stat().st_mtime + 1
assert run_all.validate_summary_payload([], {1}, path, started_at)[0] is False
path.write_text('[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}]', encoding="utf-8")
assert run_all.validate_summary_payload(
[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}],
{1}, path, path.stat().st_mtime - 1,
)[0] is False
@@ -0,0 +1,157 @@
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
from gyxx_flow.modules.content_marketing import chanmama_scraper, weekly_summary_all
from gyxx_flow.modules.content_marketing.data.tools import (
analyze_comments,
analyze_note,
daily_report_card,
friday_relogin_parallel,
)
def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path:
evidence = tmp_path / "content-evidence.jsonl"
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False)
return evidence
def test_content_write_back_skips_before_lark_cli(
monkeypatch,
tmp_path: Path,
) -> None:
evidence = _enable_acceptance(monkeypatch, tmp_path)
def fail_if_called(*args, **kwargs):
raise AssertionError("lark-cli wrapper must not be called")
monkeypatch.setattr(chanmama_scraper, "_call_lark_json", fail_if_called)
assert chanmama_scraper._write_back(
"base_secret", "tbl_test", "rec_test", "fld_test", 123
)
payload = json.loads(evidence.read_text(encoding="utf-8"))
assert payload["operation"].endswith("record-upsert")
def test_weekly_summary_skips_table_upsert_but_reports_success(
monkeypatch,
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
monkeypatch.setattr(
weekly_summary_all,
"PATHS",
SimpleNamespace(tmp_root=tmp_path),
)
def fail_if_called(*args, **kwargs):
raise AssertionError("weekly lark-cli wrapper must not be called")
monkeypatch.setattr(weekly_summary_all, "call_lark_json", fail_if_called)
assert weekly_summary_all.write_to_target_table(
"base_secret",
"tbl_test",
"fld_time",
"fld_summary",
"style-test",
"https://example.test/doc",
False,
)
def test_daily_card_sends_only_to_wang_yunlong(
monkeypatch,
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
commands = []
def fake_run(command, **kwargs):
commands.append(command)
return SimpleNamespace(
returncode=0,
stdout=json.dumps({"message_id": "om_test"}),
stderr="",
)
monkeypatch.setattr(daily_report_card.subprocess, "run", fake_run)
results = daily_report_card.send_card_to_recipients(
{"schema": "2.0", "body": {"elements": []}},
("ou_other_a", "ou_other_b"),
"2026-08-01",
)
assert len(results) == 1
assert len(commands) == 1
command = commands[0]
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
def test_comment_summary_sends_only_to_wang_yunlong(
monkeypatch,
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
commands = []
def fake_run(command, **kwargs):
commands.append(command)
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
monkeypatch.setattr(analyze_comments.subprocess, "run", fake_run)
analyze_comments.send_feishu_summary("summary", "ou_other")
command = commands[0]
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
def test_note_report_sends_only_to_wang_yunlong(
monkeypatch,
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
commands = []
monkeypatch.setattr(analyze_note.sys, "platform", "linux")
monkeypatch.setattr(
analyze_note,
"build_card_payload",
lambda *args, **kwargs: {"schema": "2.0"},
)
def fake_run(command, **kwargs):
commands.append(command)
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
monkeypatch.setattr(analyze_note.subprocess, "run", fake_run)
analyze_note.send_feishu_report(
{},
{},
{},
[],
[],
"analysis",
"ou_other",
tmp_path / "report.md",
)
command = commands[0]
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
def test_relogin_notifications_resolve_only_to_wang_yunlong(
monkeypatch,
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
assert (
friday_relogin_parallel.recipient_for_platform("pgy", "ou_other")
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
)
@@ -0,0 +1,43 @@
from __future__ import annotations
from pathlib import Path
from gyxx_flow.modules.content_marketing import run_all
def test_run_all_rebinds_browser_environment_to_child_entry(monkeypatch) -> None:
observed: dict[str, object] = {}
def fake_rebind(target, base_environment):
observed["target"] = Path(target)
observed["base"] = dict(base_environment)
return {
**base_environment,
"GYXX_SCRIPT_ID": "content_marketing:pgy_xhs_scraper_v2.py",
"GYXX_BROWSER_CDP_PORT": "child-port",
"GYXX_BROWSER_PROFILE_DIR": "child-profile",
"GYXX_BROWSER_COOKIE_FILE": "child-cookie",
"GYXX_BROWSER_STORAGE_STATE_FILE": "child-storage",
}
monkeypatch.setattr(run_all, "environment_for_child_script", fake_rebind)
target = run_all.BASE_DIR / "pgy_xhs_scraper_v2.py"
environment = run_all.build_child_environment(
target,
{
"KEEP": "yes",
"GYXX_BROWSER_CDP_PORT": "parent-port",
"GYXX_BROWSER_PROFILE_DIR": "parent-profile",
},
)
assert observed["target"] == target
assert observed["base"]["GYXX_BROWSER_CDP_PORT"] == "parent-port"
assert environment["KEEP"] == "yes"
assert environment["GYXX_BROWSER_CDP_PORT"] == "child-port"
assert environment["GYXX_BROWSER_PROFILE_DIR"] == "child-profile"
assert environment["GYXX_BROWSER_COOKIE_FILE"] == "child-cookie"
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == "child-storage"
assert environment["PYTHONIOENCODING"] == "utf-8"
assert environment["PYTHONUNBUFFERED"] == "1"
@@ -0,0 +1,100 @@
from argparse import Namespace
from datetime import date
import pytest
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
from gyxx_flow.modules.content_marketing import run_all
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
from gyxx_flow.modules.content_marketing.daily_creator_exposure_scope import (
DEFAULT_MAX_AGE_DAYS,
DEFAULT_PUBLISHED_FROM,
EXCLUDED_STYLE_NAMES,
classify_publish_scope,
select_daily_styles,
)
def test_daily_scope_excludes_requested_styles():
styles = [
{"index": 1, "name": "保留款"},
{"index": 2, "name": "盖亚微单"},
{"index": 3, "name": "逐星GT"},
{"index": 4, "name": "晨星2"},
{"index": 5, "name": "觅光"},
]
assert [style["name"] for style in select_daily_styles(styles)] == ["保留款"]
assert EXCLUDED_STYLE_NAMES == frozenset({"盖亚微单", "逐星GT", "晨星2", "觅光"})
def test_publish_scope_starts_on_july_first_and_stops_at_30_days():
collected_on = date(2026, 7, 31)
assert classify_publish_scope(
date(2026, 6, 30),
collected_on,
published_from=DEFAULT_PUBLISHED_FROM,
max_age_days=DEFAULT_MAX_AGE_DAYS,
) == "before_publish_cutoff"
assert classify_publish_scope(
date(2026, 7, 1),
collected_on,
published_from=DEFAULT_PUBLISHED_FROM,
max_age_days=DEFAULT_MAX_AGE_DAYS,
) == "collection_window_complete"
assert classify_publish_scope(
date(2026, 7, 2),
collected_on,
published_from=DEFAULT_PUBLISHED_FROM,
max_age_days=DEFAULT_MAX_AGE_DAYS,
) is None
def test_run_all_daily_scope_passes_filters_and_no_retry_to_children():
args = Namespace(dry_run=False, style=[1, 2], daily_scope=True, no_retry=False)
command = run_all.build_process_command("pgy", args)
assert "--skip-field-prepare" in command
assert command[command.index("--published-from") + 1] == "2026-07-01"
assert command[command.index("--max-age-days") + 1] == "30"
assert "--no-retry" in command
@pytest.mark.parametrize(
("module", "platform", "url"),
[
(pgy, "小红书", "https://www.xiaohongshu.com/explore/eligible"),
(xingtu, "抖音", "https://www.douyin.com/video/eligible"),
],
)
def test_platform_task_extraction_applies_daily_publish_window(
module, platform, url, monkeypatch
):
style = {
"index": 1,
"name": "保留款",
"base_token": "base",
"table_id": "table",
"field_map": {
"creator_name": {"field_id": "creator"},
"note_title": {"field_id": "title"},
"publish_time": {"field_id": "published"},
"note_url": {"field_id": "url"},
"platform": {"field_id": "platform"},
"daily_exposure": {"field_id": "daily", "field_name": "2026-07-29曝光量"},
},
}
base = {"creator": "达人", "title": "标题", "url": url, "platform": platform}
records = [
{"record_id": "old", **base, "published": "2026-06-30"},
{"record_id": "eligible", **base, "published": "2026-07-02"},
{"record_id": "missing", **base, "published": None},
]
monkeypatch.setattr(module, "list_records_by_table", lambda *_args: records)
tasks = module.extract_target_tasks(
style,
published_from=date(2026, 7, 1),
max_age_days=30,
collection_date=date(2026, 7, 29),
)
assert [task["record_id"] for task in tasks] == ["eligible"]
assert style["_extract_stats"]["skipped_publish_scope"] == 1
assert style["_extract_stats"]["skipped_incomplete"] == 1
@@ -0,0 +1,105 @@
import unittest
from datetime import date
from gyxx_flow.modules.content_marketing.data.tools.daily_dashboard_analytics import (
build_dashboard_facts,
)
class DailyDashboardAnalyticsTests(unittest.TestCase):
def setUp(self):
self.tracking = [
{"style_name": "大流量", "visitors": 1000, "cart_users": 100, "sales": 20, "refund_orders": 2, "light": "🟡"},
{"style_name": "高转化", "visitors": 100, "cart_users": 30, "sales": 20, "refund_orders": 8, "light": "🔴"},
{"style_name": "极小样本", "visitors": 10, "cart_users": 5, "sales": 10, "refund_orders": 0, "light": "🔴"},
{"style_name": "缺数据", "visitors": None, "cart_users": None, "sales": None, "refund_orders": None, "light": "无判断"},
]
self.platform = [
{"style_name": "大流量", "platform": "天猫", "visitors": 900, "cart_users": 90, "sales": 18, "refund_orders": 2},
{"style_name": "高转化", "platform": "天猫", "visitors": 100, "cart_users": 30, "sales": 20, "refund_orders": 8},
{"style_name": "极小样本", "platform": "天猫", "visitors": 10, "cart_users": 5, "sales": 10, "refund_orders": 0},
]
self.personas = [
{
"style_name": "大流量", "platform": "天猫", "data_date": date(2026, 7, 15),
"gender_top": "女性 60%", "age_top": "26-35岁 45%",
"city_top": "一线城市 35%", "buying_power_top": "高级白领 40%",
}
]
def test_all_styles_receive_same_metrics_and_diagnosis(self):
facts = build_dashboard_facts(
"2026-07-15", self.tracking, self.platform, self.personas,
{"大流量": {"creator_count": 3, "directions": ["通勤"]}},
)
self.assertEqual(len(facts["styles"]), 4)
high = next(row for row in facts["styles"] if row["style_name"] == "高转化")
self.assertEqual(high["conversion_rate"], 0.2)
self.assertEqual(high["cart_rate"], 0.3)
self.assertEqual(high["refund_rate"], 0.4)
missing = next(row for row in facts["styles"] if row["style_name"] == "缺数据")
self.assertEqual(missing["diagnosis"], "无数据")
def test_top_conversion_excludes_tiny_visitor_base(self):
facts = build_dashboard_facts("2026-07-15", self.tracking, self.platform, self.personas, {})
self.assertEqual(facts["rankings"]["conversion"][0]["style_name"], "高转化")
self.assertEqual(facts["rankings"]["visitors"][0]["style_name"], "大流量")
self.assertEqual(facts["rankings"]["refund_rate"][0]["style_name"], "高转化")
def test_platform_winners_keep_missing_persona_explicit(self):
facts = build_dashboard_facts("2026-07-15", self.tracking, self.platform, self.personas, {})
card = facts["platform_personas"][0]
self.assertEqual(card["platform"], "天猫")
self.assertEqual(card["top_conversion"]["style_name"], "高转化")
self.assertEqual(card["top_conversion"]["persona_text"], "无数据")
self.assertEqual(card["top_visitors"]["style_name"], "大流量")
self.assertIn("推测", card["top_visitors"]["interest_hypothesis"])
self.assertIn("提转化", card["top_visitors"]["optimization"])
def test_daily_new_notes_overview_replaces_view_delta_and_creator_ranking(self):
notes = [{"note_id": 7, "style_name": "大流量", "creator": "达人A", "platform": "小红书", "view_count": 500}]
facts = build_dashboard_facts(
"2026-07-15", self.tracking, self.platform, self.personas, {}, daily_notes=notes
)
self.assertEqual(facts["daily_new_notes"], notes)
self.assertNotIn("creator_connections", facts["rankings"])
self.assertNotIn("daily_views", facts["rankings"])
def test_daily_view_ranking_and_platform_cards_receive_stored_images(self):
images = {
("高转化", "天猫"): {"local_image_path": "same.jpg", "image_platform": "天猫"},
("大流量", "天猫"): {"local_image_path": "visitor.jpg", "image_platform": "天猫"},
}
facts = build_dashboard_facts(
"2026-07-15", self.tracking, self.platform, self.personas, {},
style_images=images,
)
card = facts["platform_personas"][0]
self.assertEqual(card["top_conversion"]["local_image_path"], "same.jpg")
self.assertEqual(card["top_visitors"]["local_image_path"], "visitor.jpg")
def test_matrix_groups_by_style_category_without_total_exposure(self):
signals = {
"高转化": {"persona": True, "note_count": 3, "recent_note_exposure": 43000, "comment_count": 80,
"review_count": 10, "review_negative_rate": 0.3, "creative_ctr": 0.08},
}
facts = build_dashboard_facts(
"2026-07-15", self.tracking, self.platform, self.personas, {},
style_categories={
"大流量": "都市机能", "高转化": "智性通勤",
"极小样本": "智性通勤", "缺数据": "都市机能",
},
style_signals=signals,
)
self.assertEqual([group["style_category"] for group in facts["style_groups"]], ["都市机能", "智性通勤"])
self.assertEqual([row["style_name"] for row in facts["style_groups"][0]["styles"]], ["大流量", "缺数据"])
high = next(row for row in facts["styles"] if row["style_name"] == "高转化")
self.assertNotIn("total_note_exposure", high)
self.assertEqual(high["recent_note_exposure"], 43000)
self.assertNotIn("total_note_exposure", facts["definitions"])
self.assertIn("评价风险", high["diagnosis"])
self.assertIn("画像", high["evidence_summary"])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,127 @@
import copy
import json
from datetime import date, datetime
from pathlib import Path
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
from gyxx_flow.modules.content_marketing import feishu_mapping
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
TARGET_DATE = date(2026, 7, 28)
TARGET_NAME = "2026-07-28曝光量"
def _mapping():
return {"tables": [{"index": 1, "name": "款式A", "base_token": "base-a", "table_id": "table-a", "field_map": {}, "all_field_names": []}]}
def test_daily_exposure_field_name_uses_collection_date():
assert feishu_mapping.daily_exposure_field_name(TARGET_DATE) == TARGET_NAME
def test_build_field_map_only_maps_exact_requested_daily_field():
fmap, names = feishu_mapping.build_field_map([
{"id": "old", "name": "2026-07-27曝光量"}, {"id": "today", "name": TARGET_NAME},
], target_date=TARGET_DATE)
assert fmap["daily_exposure"] == {"field_id": "today", "field_name": TARGET_NAME}
assert names == ["2026-07-27曝光量", TARGET_NAME]
def test_ensure_daily_field_reuses_existing_field(monkeypatch):
mapping = _mapping()
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: [{"id": "field-today", "name": TARGET_NAME}])
calls = []
monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda args: calls.append(args) or {"ok": True})
result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, write=False)
assert result["ok"] is True and result["created"] == [] and calls == []
assert mapping["tables"][0]["field_map"]["daily_exposure"]["field_id"] == "field-today"
def test_ensure_daily_field_creates_number_field_then_refreshes(monkeypatch):
mapping = _mapping()
fields = iter([[], [{"id": "new-field", "name": TARGET_NAME}]])
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: next(fields))
calls = []
monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda args: calls.append(args) or {"ok": True})
result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, write=False)
assert result["ok"] is True and result["created"] == ["款式A"]
payload = calls[0][calls[0].index("--json") + 1]
assert TARGET_NAME in payload and '"type": "number"' in payload
def test_ensure_daily_field_dry_run_does_not_call_feishu(monkeypatch):
mapping = _mapping()
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: (_ for _ in ()).throw(AssertionError("must not list")))
monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda *_: (_ for _ in ()).throw(AssertionError("must not create")))
result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, dry_run=True, write=False)
assert result["ok"] is True
assert mapping["tables"][0]["field_map"]["daily_exposure"]["field_id"].startswith("dryrun:")
def test_acceptance_skips_field_create_and_keeps_sentinel_in_memory(
monkeypatch,
tmp_path: Path,
):
evidence = tmp_path / "evidence.jsonl"
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
mapping = _mapping()
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: [])
monkeypatch.setattr(
feishu_mapping,
"call_lark_json",
lambda *_: (_ for _ in ()).throw(AssertionError("must not create")),
)
result = feishu_mapping.ensure_daily_exposure_fields(
mapping,
target_date=TARGET_DATE,
write=False,
)
assert result["ok"] is True
assert result["created"] == []
assert result["skipped_creations"] == ["款式A"]
assert mapping["tables"][0]["field_map"]["daily_exposure"][
"field_id"
].startswith("acceptance-skipped:")
payload = json.loads(evidence.read_text(encoding="utf-8"))
assert payload["operation"] == "content.mapping.daily-exposure-field-create"
def test_acceptance_hydrates_cached_mapping_without_persisting_sentinel(
monkeypatch,
tmp_path: Path,
):
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
cache = tmp_path / "mapping.json"
cache.write_text(json.dumps(_mapping(), ensure_ascii=False), encoding="utf-8")
hydrated = feishu_mapping._hydrate_acceptance_daily_field(
json.loads(cache.read_text(encoding="utf-8")),
self_operated=False,
)
assert hydrated["tables"][0]["field_map"]["daily_exposure"][
"field_id"
].startswith("acceptance-skipped:")
persisted = json.loads(cache.read_text(encoding="utf-8"))
assert "daily_exposure" not in persisted["tables"][0]["field_map"]
def test_collaboration_slot_pickers_use_daily_field_without_publish_time():
fmap = {"daily_exposure": {"field_id": "today-field", "field_name": TARGET_NAME}}
expected = ("daily_exposure", "today-field", TARGET_NAME)
assert pgy.pick_read_field(fmap, None, datetime(2026, 7, 28)) == expected
assert xingtu.pick_read_field(fmap, None, datetime(2026, 7, 28)) == expected
def test_self_operated_mapping_removes_daily_field(monkeypatch):
source = {"tables": [{"field_map": {
"daily_exposure": {"field_id": "daily", "field_name": TARGET_NAME},
"read_count_7d": {"field_id": "s7", "field_name": "7天曝光量"},
}}]}
monkeypatch.setattr(feishu_mapping, "load_mapping", lambda *_args, **_kwargs: copy.deepcopy(source))
for mapping in (pgy.load_mapping(True), xingtu.load_mapping(True), bili.load_mapping(bili.DEFAULT_DATA_DIR, True)):
assert "daily_exposure" not in mapping["tables"][0]["field_map"]
@@ -0,0 +1,442 @@
import unittest
from datetime import date
from unittest.mock import patch
from gyxx_flow.modules.content_marketing import daily_marketing_report as daily
from gyxx_flow.modules.content_marketing.data.tools import analyze_comments
class DailyMarketingReportTests(unittest.TestCase):
def test_hermes_url_accepts_api_base_or_full_completion_path(self):
full_url = "http://127.0.0.1:8642/v1/chat/completions"
self.assertEqual(
analyze_comments._chat_completions_url("http://127.0.0.1:8642/v1"),
full_url,
)
self.assertEqual(
analyze_comments._chat_completions_url(full_url),
full_url,
)
def test_sales_forecast_uses_seven_day_total_and_recent_three_day_velocity(self):
forecast = daily.estimate_sales_forecast([10, 12, 14, 16, 18, 20, 22])
self.assertEqual(forecast["sales_7d_total"], 112)
self.assertEqual(forecast["sales_forecast_7d"], 140)
self.assertAlmostEqual(forecast["sales_forecast_change"], 7 / 13, places=4)
def test_sales_forecast_requires_complete_seven_day_history(self):
self.assertEqual(
daily.estimate_sales_forecast([10, 12, 14]),
{
"sales_7d_total": None,
"sales_forecast_7d": None,
"sales_forecast_change": None,
},
)
def test_default_recipients_include_all_four_daily_report_owners(self):
self.assertEqual(
daily.DEFAULT_RECIPIENT_OPEN_IDS,
(
"ou_7ad5fc8012e2f741afc5346e05ffd447",
"ou_fa8d81a16527ad06352dbecc575285b8",
"ou_2eda5eec112109ae6d19a1f6813eadcb",
"ou_89bcff110ccbb09a23548dc0fb3d880c",
),
)
def test_hermes_transport_error_is_not_accepted_as_report(self):
with self.assertRaisesRegex(RuntimeError, "Hermes 分析失败"):
daily.require_valid_hermes_report("API call failed after 3 retries: Connection error.")
self.assertEqual(daily.require_valid_hermes_report("整体经营分析\n销量保持稳定"), "整体经营分析\n销量保持稳定")
def test_light_uses_highest_absolute_visitor_or_cart_change(self):
self.assertEqual(daily.classify_light(0.05, -0.08), "🟢")
self.assertEqual(daily.classify_light(0.12, 0.02), "🟡")
self.assertEqual(daily.classify_light(-0.31, 0.01), "🔴")
self.assertEqual(daily.classify_light(None, None), "无判断")
def test_comment_signal_requires_minimum_sample(self):
comments = ["光影行星宙斯怎么买"] * 49
self.assertIsNone(daily.calculate_comment_signal("宙斯", comments))
comments.append("好看")
signal = daily.calculate_comment_signal("宙斯", comments)
self.assertEqual(signal["sample_size"], 50)
self.assertEqual(signal["brand_mention_rate"], 0.98)
self.assertEqual(signal["purchase_intent_rate"], 0.98)
def test_prompt_uses_only_the_fresh_user_instruction_and_database_facts(self):
prompt = daily.build_daily_prompt(
report_date="2026-07-15",
today_notes=[],
tracking_rows=[],
next_day_rows=[],
comment_rows=[],
)
self.assertIn("以下为数据库查询结果", prompt)
self.assertIn('"报告日期": "2026-07-15"', prompt)
self.assertIn("增长型SKU", daily.SYSTEM_PROMPT)
self.assertIn("流量机会型SKU", daily.SYSTEM_PROMPT)
self.assertIn("潜力型SKU", daily.SYSTEM_PROMPT)
self.assertIn("风险型SKU", daily.SYSTEM_PROMPT)
self.assertNotIn("A类:爆款增长SKU", daily.SYSTEM_PROMPT)
def test_prompt_uses_fashion_bag_data_analyst_role(self):
self.assertIn("专业的时尚包袋品牌电商运营数据分析师", daily.SYSTEM_PROMPT)
self.assertIn("销售、电商运营及达人营销数据", daily.SYSTEM_PROMPT)
self.assertIn("发现增长机会和风险", daily.SYSTEM_PROMPT)
self.assertIn("SKU营销运营日报", daily.SYSTEM_PROMPT)
self.assertIn("展示关键数据", daily.SYSTEM_PROMPT)
def test_fresh_prompt_contains_the_complete_requested_analysis_structure(self):
prompt = daily.SYSTEM_PROMPT
for phrase in (
"今日整体经营判断",
"SKU经营表现分析",
"达人营销效果分析",
"产品节奏判断",
"蓄水期",
"放量期",
"稳定期",
"衰退期",
"增长SKU TOP3",
"风险SKU TOP3",
"比较昨日、七日基准",
"用图表展示",
"曝光量",
"建议动作",
"对销量影响",
"所有建议必须具体到SKU",
):
self.assertIn(phrase, prompt)
for retired in ("A类:爆款增长SKU", "B类:潜力增长SKU", "D类:需要优化SKU"):
self.assertNotIn(retired, prompt)
def test_database_payload_keeps_all_available_sources(self):
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], {"reviews": [{"sample_size": 3}]})
for phrase in ("所有SKU电商运营数据", "达人营销内容数据", "评论与补充经营数据", "reviews"):
self.assertIn(phrase, prompt)
def test_note_label_ignores_placeholder_creator_name(self):
note = {"id": 7, "creator_name": "/", "title": "瑞白通勤包实测"}
self.assertEqual(daily.note_label(note), "瑞白通勤包实测")
def test_cooperation_metrics_hide_unverifiable_engagement(self):
invalid = daily._sanitize_cooperation_row({
"publish_time": None,
"exposure_count": 38000,
"engagement_count_num": 21430000,
})
self.assertIsNone(invalid["engagement_count_num"])
self.assertEqual(invalid["publish_status"], "发布时间无数据")
self.assertIn("口径异常", invalid["engagement_data_status"])
valid = daily._sanitize_cooperation_row({
"publish_time": "2026-07-14",
"exposure_count": 38000,
"engagement_count_num": 2143,
})
self.assertEqual(valid["engagement_count_num"], 2143)
self.assertEqual(valid["publish_status"], "已记录发布时间")
def test_load_report_facts_tracks_all_styles_for_report_date(self):
metrics = {
"宙斯": {"light": "🟢"},
}
with patch.object(daily, "_note_rows", return_value=[]), \
patch.object(daily, "_all_style_names", return_value=["宙斯", "蓝鹊"]) as all_styles, \
patch.object(daily, "_style_metrics", return_value=metrics) as style_metrics:
with patch.object(daily, "_load_enrichment", return_value={"platform_metrics": []}), \
patch.object(daily, "_platform_metrics", return_value=[]), \
patch.object(daily, "_persona_context", return_value=[]), \
patch.object(daily, "load_creator_connections", return_value={}), \
patch.object(daily, "load_style_images", return_value={}), \
patch.object(daily, "load_style_categories", return_value={}), \
patch.object(daily, "_cooperation_context", return_value=[]), \
patch.object(daily, "_style_comment_context", return_value=[]), \
patch.object(daily, "_review_context", return_value=[]), \
patch.object(daily, "_creative_context", return_value=[]), \
patch.object(daily, "build_dashboard_facts", return_value={"styles": ["宙斯", "蓝鹊"]}):
_, tracking, _, _, enrichment = daily.load_report_facts(date(2026, 7, 15))
all_styles.assert_called_once_with(date(2026, 7, 15))
self.assertEqual([row["style_name"] for row in tracking], ["宙斯", "蓝鹊"])
self.assertEqual(tracking[1]["light"], "无判断")
self.assertEqual(style_metrics.call_args_list[0].args[1], ["宙斯", "蓝鹊"])
self.assertEqual(enrichment["platform_metrics"], [])
self.assertEqual(enrichment["candidate_styles"], [])
self.assertEqual(enrichment["dashboard"]["styles"], ["宙斯", "蓝鹊"])
def test_report_date_uses_yesterday_notes_for_daily_new_notes(self):
notes = [
{"id": 1, "publish_date": "2026-07-15", "style_name": "宙斯", "creator_name": "达人甲", "platform": "小红书", "publish_time": "2026-07-15 10:00", "title": "今天", "view_count": 10, "engagement_rate": None},
{"id": 2, "publish_date": "2026-07-14", "style_name": "宙斯", "creator_name": "达人乙", "platform": "抖音", "publish_time": "2026-07-14 11:00", "title": "昨天", "view_count": 20, "engagement_rate": None},
]
with patch.object(daily, "_note_rows", return_value=notes), \
patch.object(daily, "_all_style_names", return_value=["宙斯"]), \
patch.object(daily, "_style_metrics", return_value={}), \
patch.object(daily, "_load_enrichment", return_value={}), \
patch.object(daily, "_platform_metrics", return_value=[]), \
patch.object(daily, "_persona_context", return_value=[]), \
patch.object(daily, "load_creator_connections", return_value={}), \
patch.object(daily, "load_style_images", return_value={}), \
patch.object(daily, "load_style_categories", return_value={}), \
patch.object(daily, "_cooperation_context", return_value=[]), \
patch.object(daily, "_style_comment_context", return_value=[]), \
patch.object(daily, "_review_context", return_value=[]), \
patch.object(daily, "_creative_context", return_value=[]), \
patch.object(daily, "_comment_rows", return_value=[]), \
patch.object(daily, "build_dashboard_facts", return_value={}) as dashboard:
daily_notes, _, _, _, _ = daily.load_report_facts(date(2026, 7, 15))
self.assertEqual([row["note_id"] for row in daily_notes], [2])
self.assertEqual([row["note_id"] for row in dashboard.call_args.kwargs["daily_notes"]], [2])
def test_prompt_requires_all_styles_without_top_ten_truncation(self):
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [])
self.assertIn("所有SKU", prompt)
self.assertNotIn("只保留异常程度最高的10个", prompt)
self.assertIn("SKU经营表现分析", daily.SYSTEM_PROMPT)
self.assertIn("加购率", daily.SYSTEM_PROMPT)
def test_prompt_uses_the_new_daily_output_contract(self):
self.assertIn("电商运营数据", daily.SYSTEM_PROMPT)
self.assertIn("达人营销数据", daily.SYSTEM_PROMPT)
self.assertNotIn("TOP20", daily.SYSTEM_PROMPT)
self.assertIn("一页A4", daily.SYSTEM_PROMPT)
self.assertIn("↑↓展示趋势", daily.SYSTEM_PROMPT)
self.assertNotIn("→稳定", daily.SYSTEM_PROMPT)
self.assertIn("电商销量表现 + 访客数 + 转化率 + 达人笔记曝光 + 用户反馈", daily.SYSTEM_PROMPT)
def test_prompt_payload_keeps_yesterday_and_seven_day_comparisons(self):
rows = [{
"style_name": "蓝鹊",
"sales": 10,
"sales_yesterday": 8,
"avg_sales_7d": 7.5,
"sales_change_yesterday": 0.25,
"sales_change": 0.3333,
}]
prompt = daily.build_daily_prompt("2026-07-16", [], rows, [], [])
self.assertIn('"sales_yesterday": 8', prompt)
self.assertIn('"avg_sales_7d": 7.5', prompt)
self.assertIn('"sales_change_yesterday": 0.25', prompt)
def test_style_metrics_calculates_yesterday_and_seven_day_changes(self):
class FakeCursor:
def __enter__(self):
return self
def __exit__(self, *args):
return False
def execute(self, query, params):
self.query = query
self.params = params
def fetchall(self):
return [(
"蓝鹊",
120, 24, 10, 2,
100, 20, 8, 1,
90, 18, 7.5, 1.2,
[4, 5, 6, 7, 8, 9, 10],
)]
class FakeConnection:
def __enter__(self):
return self
def __exit__(self, *args):
return False
def cursor(self):
return FakeCursor()
with patch.object(daily, "get_conn", return_value=FakeConnection()):
row = daily._style_metrics(date(2026, 7, 16), ["蓝鹊"])["蓝鹊"]
self.assertEqual(row["sales_yesterday"], 8)
self.assertEqual(row["sales_change_yesterday"], 0.25)
self.assertEqual(row["sales_change"], 0.3333)
self.assertEqual(row["visitor_change_yesterday"], 0.2)
self.assertEqual(row["conversion_rate"], round(10 / 120, 4))
self.assertEqual(row["conversion_rate_yesterday"], round(8 / 100, 4))
self.assertEqual(row["sales_7d_total"], 49)
self.assertEqual(row["sales_forecast_7d"], 63)
def test_prompt_does_not_reuse_retired_dashboard_sections(self):
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], {"dashboard": {
"rankings": {"sales": []},
"platform_personas": [{"platform": "天猫"}],
"daily_new_notes": [],
"styles": [{"style_name": "蓝鹊", "total_note_exposure": 1000, "evidence_summary": "经营、画像、营销笔记"}],
"definitions": {"conversion_rate": "销量/访客"},
}})
self.assertNotIn("按风格分组的综合经营矩阵", prompt)
self.assertNotIn("平台高转化与高访客画像", prompt)
self.assertNotIn("全款式经营看板摘要", prompt)
def test_report_normalization_and_all_style_validation(self):
report = daily.normalize_report(
"健康度:偏弱。36 个款式中\n蓝鹊同比增长79%\n宙斯环比下降16%",
35,
)
self.assertNotIn("同比", report)
self.assertIn("环比下降16%", report)
self.assertIn("35个款式中", report)
self.assertEqual(daily.normalize_report("销量 0 / 退款 0(无数据)"), "销量 0 / 退款 0(当日均为0")
cleaned = daily.normalize_report("投放尚未发布,带动效应快速衰减;待发布后核对")
self.assertNotIn("未发布", cleaned)
self.assertNotIn("带动效应", cleaned)
self.assertIn("发布时间无数据", cleaned)
self.assertNotIn("驱动访客", daily.normalize_report("该内容未驱动访客回升"))
daily.validate_all_styles(report, [{"style_name": "蓝鹊"}, {"style_name": "宙斯"}])
with self.assertRaisesRegex(RuntimeError, "缺少款式"):
daily.validate_all_styles(report, [{"style_name": "白鹭"}])
def test_report_normalization_removes_database_and_code_style_language(self):
raw = """数据已收到,我先用脚本做一遍聚合。
# 品牌SKU电商运营日报
• 最佳渠道:无法判定。今日 `昨日新发布内容` 为空,cooperations 仅1条,publish_time=null、engagement_data_status=无数据。
5. 数据治理与内容补位:(a)对齐 publish_time;(b)修复互动数据;(c)补齐数据。
报告字数提示:如需压缩请告知。
完整数据索引:如需导出 CSV 请告诉我。
"""
report = daily.normalize_report(raw)
self.assertTrue(report.startswith("# 品牌SKU电商运营日报"))
for forbidden in ("我先用脚本", "cooperations", "publish_time", "engagement_data_status", "null", "`", "a", "报告字数提示", "请告诉我"):
self.assertNotIn(forbidden, report)
self.assertIn("达人合作记录", report)
self.assertIn("发布时间无数据", report)
self.assertIn("互动数据无数据", report)
self.assertIn("1", report)
def test_system_prompt_forbids_database_fields_and_conversational_tail(self):
for phrase in (
"禁止输出数据库表名、字段名、JSON键名",
"禁止出现反引号、null",
"禁止描述生成过程",
"禁止在结尾追问",
):
self.assertIn(phrase, daily.SYSTEM_PROMPT)
def test_status_groups_are_replaced_with_complete_deterministic_lists(self):
report = "二、种草追踪池\n\nB. 全量状态(固定4行)\n🔴(1):蓝鹊\n\n三、昨日笔记次日追踪"
rows = [
{"style_name": "蓝鹊", "light": "🔴"},
{"style_name": "宙斯", "light": "🟡"},
{"style_name": "白鹭", "light": "🟢"},
{"style_name": "云卷2", "light": "无判断"},
]
result = daily.replace_status_groups(report, rows)
self.assertIn("B. 全量状态(4款)", result)
self.assertIn("🔴(1):蓝鹊", result)
self.assertIn("🟡(1):宙斯", result)
self.assertIn("🟢(1):白鹭", result)
self.assertIn("无判断(1):云卷2", result)
self.assertIn("三、昨日笔记次日追踪", result)
def test_status_groups_are_inserted_when_model_omits_marker(self):
report = "一、核心经营结论\n正常\n\n三、达人投放与电商同期关联\n无数据"
rows = [{"style_name": "蓝鹊", "light": "🔴"}, {"style_name": "白鹭", "light": "🟢"}]
result = daily.replace_status_groups(report, rows, "C. 补充经营信号\n- 店铺大盘:无数据")
self.assertIn("B. 全量状态(2款)", result)
self.assertIn("🔴(1):蓝鹊", result)
self.assertIn("🟢(1):白鹭", result)
self.assertLess(result.index("B. 全量状态"), result.index("三、达人投放"))
def test_prompt_passes_available_enrichment_without_restoring_old_sections(self):
enrichment = {
"platform_metrics": [{"style_name": "蓝鹊", "platform": "天猫", "sales": 14}],
"cooperations": [{"style_name": "蓝鹊", "content_direction": "通勤"}],
"personas": [{"style_name": "蓝鹊", "gender_top": "女性用户 60%"}],
"creatives": [{"style_name": "蓝鹊", "ctr": 4.2}],
"reviews": [{"style_name": "蓝鹊", "sample_size": 6}],
"shop_overview": [{"platform": "天猫", "week_end": "2026-07-12"}],
}
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], enrichment)
for source_key in ("platform_metrics", "cooperations", "personas", "creatives", "reviews", "shop_overview"):
self.assertIn(source_key, prompt)
self.assertIn("评论与补充经营数据", prompt)
self.assertNotIn("补充经营信号", prompt)
self.assertNotIn("全量状态", prompt)
def test_enrichment_section_validation_requires_available_categories(self):
enrichment = {"personas": [{}], "creatives": [{}], "reviews": [{}], "shop_overview": [{}]}
complete = "条件信号:人物画像;主图;商品评价;店铺大盘"
daily.validate_enrichment_sections(complete, enrichment)
with self.assertRaisesRegex(RuntimeError, "主图"):
daily.validate_enrichment_sections("人物画像 商品评价 店铺大盘", enrichment)
def test_build_condition_signals_covers_all_available_sources(self):
enrichment = {
"candidate_styles": ["蓝鹊"],
"personas": [{"style_name": "蓝鹊", "platform": "天猫", "data_date": "2026-07-15", "gender_top": "女性 60%"}],
"creatives": [{"style_name": "蓝鹊", "platform": "天猫", "data_date": "2026-07-12", "impressions": 1000, "clicks": 50, "ctr": 5.0}],
"reviews": [{"style_name": "蓝鹊", "sample_size": 6, "negative_count": 1, "negative_rate": 0.1667, "window_start": "2026-07-09", "window_end": "2026-07-15"}],
"shop_overview": [{"platform": "天猫", "week_end": "2026-07-12", "visitors": 10000, "deal_amount": 20000}],
}
text = daily.build_condition_signals(enrichment)
for phrase in ("C. 补充经营信号", "人物画像", "主图", "商品评价", "店铺大盘", "2026-07-12"):
self.assertIn(phrase, text)
def test_persona_top_value_ignores_invalid_percentages(self):
payload = {"buying_power": [{"name": "异常", "value": 133}, {"name": "L4", "value": 42}]}
self.assertEqual(daily._top_persona_value(payload, "buying_power"), "L4 42.0%")
def test_enriched_report_rejects_missing_platform_claim_and_invented_new_product(self):
daily.validate_enriched_report("平台拆分完整,均为同期合作")
with self.assertRaisesRegex(RuntimeError, "平台拆分"):
daily.validate_enriched_report("无平台拆分数据")
with self.assertRaisesRegex(RuntimeError, "新品"):
daily.validate_enriched_report("同期出现多个新品合作")
def test_platform_persona_section_uses_database_winners_not_model_text(self):
report = ("### 三、平台人群与营销机会\n天猫高转化款为错误款式\n\n"
"### 四、明日业务动作\n复查数据\n\n"
"三、平台人群与营销机会\n重复错误款式")
enrichment = {"dashboard": {
"daily_views_status": "无数据:未保存每日增量快照",
"platform_personas": [{
"platform": "天猫",
"top_conversion": {
"style_name": "布谷", "conversion_rate": 0.0259, "visitors": 617,
"persona_text": "无数据", "interest_hypothesis": "兴趣推测:无数据",
"optimization": "先补采人群画像",
},
"top_visitors": {
"style_name": "星云2", "conversion_rate": 0.0114, "visitors": 3504,
"persona_text": "男性用户 62.7%", "interest_hypothesis": "兴趣推测:数码3C",
"optimization": "优化详情页",
},
}],
}}
result = daily.replace_platform_persona_section(report, enrichment)
self.assertIn("转化最高为布谷(转化率2.6%", result)
self.assertIn("访客最高为星云2(访客3,504", result)
self.assertIn("兴趣推测:数码3C", result)
self.assertNotIn("错误款式", result)
self.assertNotIn("重复错误款式", result)
self.assertIn("四、明日业务动作", result)
self.assertEqual(result.count("三、平台人群与营销机会"), 1)
def test_platform_persona_section_supports_new_daily_report_boundary(self):
report = ("### 三、平台人群与营销机会\n模型内容\n\n"
"### 四、今日重点SKU分析\n重点款内容\n")
result = daily.replace_platform_persona_section(report, {"dashboard": {}})
self.assertIn("三、平台人群与营销机会", result)
self.assertIn("四、今日重点SKU分析", result)
self.assertNotIn("模型内容", result)
self.assertEqual(result.count("三、平台人群与营销机会"), 1)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,26 @@
import unittest
from pathlib import Path
from gyxx_flow.catalog import WorkflowCatalog
PROJECT_ROOT = Path(__file__).resolve().parents[3]
class DailyMarketingReportScheduleTests(unittest.TestCase):
def test_python_scheduler_runs_latest_daily_report(self):
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
workflow = next(
item
for item in catalog.workflows
if item.workflow_id == "content.marketing_report.daily"
)
schedule = catalog.schedule_for(workflow.workflow_id)
self.assertEqual(workflow.steps[0].entry, "daily_marketing_report.py")
self.assertEqual(workflow.steps[0].args, ("--send",))
self.assertEqual(schedule.kind, "daily")
self.assertEqual(schedule.at, "10:00")
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,202 @@
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from PIL import Image
from gyxx_flow.modules.content_marketing.data.tools.daily_report_card import (
build_analysis_card,
build_daily_report_card,
build_dashboard_card,
prepare_feishu_image,
send_card_to_recipients,
send_daily_report_cards,
upload_dashboard_image,
)
class DailyReportCardTests(unittest.TestCase):
def test_dashboard_and_analysis_are_separate_cards_with_full_text(self):
report = """![全域种草数据看板](charts/dashboard.png)
第一部分 整体经营分析
总销量下降15%
整体趋势判断:下滑状态。
第二部分 SKU分析排序
1. 蓝鹊:销量第一。
2. 星云2:转化领先。
3. 宙斯:访客增长。
4. 南斯:退款偏高。
5. 凌云3:需要优化。
6. 波塞冬:继续观察。
第三部分 关键预警与建议
退款订单比上升,48小时内排查。
"""
rows = [{"style_name": "蓝鹊", "light": "🔴"}]
dashboard = build_dashboard_card("2026-07-17", rows, "img_v3_test")
analysis = build_analysis_card("2026-07-17", report)
dashboard_text = json.dumps(dashboard, ensure_ascii=False)
analysis_text = json.dumps(analysis, ensure_ascii=False)
self.assertIn("img_v3_test", dashboard_text)
self.assertNotIn("总销量下降15%", dashboard_text)
self.assertNotIn("img_v3_test", analysis_text)
self.assertNotIn("![全域种草数据看板]", analysis_text)
self.assertIn("波塞冬:继续观察", analysis_text)
self.assertIn("退款订单比上升", analysis_text)
def test_build_card_uses_card_2_schema_and_dashboard(self):
report = """一、核心经营结论
1. 波塞冬edge访客明显下滑。
2. 蓝鹊销量同期上行。
二、产品经营分析
四、明日业务动作
1. 复查波塞冬edge天猫流量。
2. 跟进蓝鹊投放承接。
"""
rows = [
{"style_name": "波塞冬edge", "light": "🔴"},
{"style_name": "宙斯", "light": "🟡"},
{"style_name": "白鹭", "light": "🟢"},
]
card = build_daily_report_card("2026-07-15", report, rows, "img_v3_test")
self.assertEqual(card["schema"], "2.0")
self.assertEqual(card["config"]["width_mode"], "fill")
self.assertIn("Hermes", card["header"]["subtitle"]["content"])
elements = card["body"]["elements"]
self.assertEqual([item["tag"] for item in elements], ["column_set", "img", "markdown", "markdown"])
self.assertEqual(elements[1]["img_key"], "img_v3_test")
metrics_text = json.dumps(elements[0], ensure_ascii=False)
self.assertIn("红灯款式", metrics_text)
self.assertIn("1", metrics_text)
self.assertNotIn("二、产品经营分析", json.dumps(card, ensure_ascii=False))
def test_prepare_feishu_image_resizes_to_supported_width(self):
with tempfile.TemporaryDirectory() as tmp:
source = Path(tmp) / "source.png"
target = Path(tmp) / "target.png"
Image.new("RGB", (1600, 1000), "white").save(source)
prepare_feishu_image(source, target)
with Image.open(target) as image:
self.assertEqual(image.width, 1500)
self.assertLessEqual(image.height / image.width, 16 / 9)
def test_build_card_reads_new_management_report_headings(self):
report = """## 1. 今日经营概览
销售:↑12%,蓝鹊贡献主要增量。
整体判断:增长状态。
## 2. SKU健康排名 TOP20
## 6. 今日运营建议
1. 商品动作:复核蓝鹊库存。
2. 电商动作:优化宙斯详情页。
"""
card = build_daily_report_card("2026-07-17", report, [], "img_v3_test")
card_text = json.dumps(card, ensure_ascii=False)
self.assertIn("蓝鹊贡献主要增量", card_text)
self.assertIn("复核蓝鹊库存", card_text)
def test_build_card_reads_fresh_partial_prompt_report_headings(self):
report = """第一部分 整体经营分析
总销量下降15%
整体趋势判断:下滑状态。
第二部分 SKU分析排序
第三部分 关键预警与建议
退款订单比上升,48小时内排查。
追加增长型SKU预算。
"""
card = build_daily_report_card("2026-07-17", report, [], "img_v3_test")
card_text = json.dumps(card, ensure_ascii=False)
self.assertIn("总销量下降15%", card_text)
self.assertIn("48小时内排查", card_text)
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.subprocess.run"
)
def test_upload_dashboard_uses_named_image_file_field(self, run):
run.return_value = type("Result", (), {
"returncode": 0, "stdout": json.dumps({"image_key": "img_v3_test"}), "stderr": ""
})()
key = upload_dashboard_image(Path("charts/dashboard.png"))
self.assertEqual(key, "img_v3_test")
command = run.call_args.args[0]
self.assertEqual(command[command.index("--file") + 1], "image=./dashboard.png")
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.subprocess.run"
)
def test_send_card_to_all_recipients(self, run):
run.side_effect = [
type("Result", (), {"returncode": 0, "stdout": json.dumps({"message_id": f"om_{i}"}), "stderr": ""})()
for i in range(3)
]
recipients = ["ou_one", "ou_two", "ou_three"]
results = send_card_to_recipients({"schema": "2.0"}, recipients, "2026-07-15")
self.assertEqual([item["message_id"] for item in results], ["om_0", "om_1", "om_2"])
self.assertEqual(run.call_count, 3)
commands = [call.args[0] for call in run.call_args_list]
self.assertEqual([cmd[cmd.index("--user-id") + 1] for cmd in commands], recipients)
self.assertTrue(all("--msg-type" in cmd and "interactive" in cmd for cmd in commands))
keys = [cmd[cmd.index("--idempotency-key") + 1] for cmd in commands]
self.assertTrue(all(key.startswith("daily-2026-07-15-") for key in keys))
self.assertTrue(all(len(key) <= 50 for key in keys))
@patch(
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.send_card_to_recipients"
)
@patch(
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.upload_dashboard_image",
return_value="img_v3_test",
)
@patch(
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.prepare_feishu_image"
)
def test_daily_report_sends_dashboard_then_analysis_as_two_messages_per_recipient(
self, prepare, upload, send
):
send.side_effect = [
[{"message_id": "om_board_1"}, {"message_id": "om_board_2"}],
[{"message_id": "om_text_1"}, {"message_id": "om_text_2"}],
]
recipients = ["ou_one", "ou_two"]
results = send_daily_report_cards(
"2026-07-17",
"第一部分 整体经营分析\n完整分析内容",
[{"light": "🟢"}],
Path("charts/dashboard.png"),
recipients,
)
self.assertEqual(
[item["message_id"] for item in results],
["om_board_1", "om_board_2", "om_text_1", "om_text_2"],
)
self.assertEqual(send.call_count, 2)
first_card = send.call_args_list[0].args[0]
second_card = send.call_args_list[1].args[0]
self.assertIn("img_v3_test", json.dumps(first_card, ensure_ascii=False))
self.assertIn("完整分析内容", json.dumps(second_card, ensure_ascii=False))
self.assertEqual(send.call_args_list[0].kwargs["message_kind"], "dashboard")
self.assertEqual(send.call_args_list[1].kwargs["message_kind"], "analysis")
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,104 @@
import tempfile
import unittest
from pathlib import Path
from PIL import Image
from gyxx_flow.modules.content_marketing.data.tools.daily_report_charts import (
_split_style_groups,
build_overview_cards,
classify_style_action,
generate_dashboard,
select_chart_styles,
)
class DailyReportChartTests(unittest.TestCase):
def setUp(self):
self.rows = [
{"style_name": "大盘下滑", "light": "🔴", "visitors": 1000, "avg_visitors_7d": 2000, "cart_users": 80, "avg_cart_users_7d": 100, "visitor_change": -0.5, "cart_change": -0.2, "sales_change": -0.4, "refund_change": 0.1},
{"style_name": "正向放量", "light": "🔴", "visitors": 1500, "avg_visitors_7d": 1000, "cart_users": 180, "avg_cart_users_7d": 100, "visitor_change": 0.5, "cart_change": 0.8, "sales_change": 0.3, "refund_change": -0.1},
{"style_name": "小基数", "light": "🔴", "visitors": 20, "avg_visitors_7d": 100, "cart_users": 1, "avg_cart_users_7d": 2, "visitor_change": -0.8, "cart_change": -0.5, "sales_change": None, "refund_change": None},
{"style_name": "黄灯A", "light": "🟡", "visitors": 800, "avg_visitors_7d": 1000, "cart_users": 70, "avg_cart_users_7d": 80, "visitor_change": -0.2, "cart_change": -0.125, "sales_change": -0.1, "refund_change": 0.2},
{"style_name": "绿灯", "light": "🟢", "visitors": 500, "avg_visitors_7d": 510, "cart_users": 40, "avg_cart_users_7d": 41, "visitor_change": -0.02, "cart_change": -0.02, "sales_change": 0.0, "refund_change": 0.0},
]
def test_selection_covers_negative_positive_and_small_base(self):
names = [row["style_name"] for row in select_chart_styles(self.rows, limit=4)]
self.assertIn("大盘下滑", names)
self.assertIn("正向放量", names)
self.assertIn("小基数", names)
def test_generate_dashboard_writes_png(self):
enrichment = {"platform_metrics": [
{"style_name": "大盘下滑", "platform": "天猫", "visitors": 700},
{"style_name": "大盘下滑", "platform": "京东", "visitors": 300},
]}
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "dashboard.png"
generate_dashboard("2026-07-15", self.rows, enrichment, path)
self.assertTrue(path.exists())
self.assertGreater(path.stat().st_size, 10_000)
with Image.open(path) as image:
self.assertEqual(image.size, (1600, 2500))
def test_style_groups_are_split_into_balanced_category_columns(self):
groups = [
{"style_category": "都市机能", "styles": [{}] * 12},
{"style_category": "户外机能", "styles": [{}] * 3},
{"style_category": "都市新贵", "styles": [{}] * 3},
{"style_category": "智性通勤", "styles": [{}] * 14},
{"style_category": "都市新旅", "styles": [{}]},
{"style_category": "都市运动", "styles": [{}] * 3},
]
left, right = _split_style_groups(groups)
self.assertEqual([group["style_category"] for group in left], ["都市机能", "户外机能", "都市新贵"])
self.assertEqual([group["style_category"] for group in right], ["智性通勤", "都市新旅", "都市运动"])
groups.append({"style_category": "未分类", "styles": [{}] * 2})
left, right = _split_style_groups(groups)
self.assertEqual(sum(len(group["styles"]) for group in left), 18)
self.assertEqual(sum(len(group["styles"]) for group in right), 20)
def test_overview_compares_today_yesterday_and_seven_day_average(self):
rows = [
{
"style_name": "蓝鹊", "sales": 12, "sales_yesterday": 10, "avg_sales_7d": 8,
"visitors": 120, "visitors_yesterday": 100, "avg_visitors_7d": 90,
},
{
"style_name": "宙斯", "sales": 18, "sales_yesterday": 20, "avg_sales_7d": 15,
"visitors": 180, "visitors_yesterday": 200, "avg_visitors_7d": 150,
},
]
cards = build_overview_cards(rows, {"daily_new_notes": [{"view_count": 515}]})
sales = next(card for card in cards if card["key"] == "sales")
conversion = next(card for card in cards if card["key"] == "conversion")
content = next(card for card in cards if card["key"] == "content")
self.assertEqual((sales["today"], sales["yesterday"], sales["baseline_7d"]), (30, 30, 23))
self.assertEqual(conversion["today"], 0.1)
self.assertEqual(content["today"], 515)
self.assertIsNone(content["baseline_7d"])
def test_style_action_matches_prompt_growth_opportunity_and_risk_rules(self):
growth = classify_style_action({
"sales_change_yesterday": 0.2, "visitor_change_yesterday": 0.1,
"conversion_change_yesterday": 0.001, "refund_change_yesterday": -0.1,
})
opportunity = classify_style_action({
"sales_change_yesterday": -0.1, "visitor_change_yesterday": 0.2,
"conversion_change_yesterday": -0.01, "refund_change_yesterday": 0,
})
risk = classify_style_action({
"sales_change_yesterday": -0.2, "visitor_change_yesterday": -0.1,
"conversion_change_yesterday": -0.01, "refund_change_yesterday": 0.4,
})
self.assertEqual(growth, ("增长型", "加达人/备货"))
self.assertEqual(opportunity, ("流量机会", "优化详情页"))
self.assertEqual(risk, ("风险型", "查退款/控投放"))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,137 @@
from gyxx_flow.modules.content_marketing.data.tools import (
friday_relogin_parallel as relogin,
)
from gyxx_flow.modules.content_marketing.data.tools import relogin_transaction
def test_five_expected_platforms_are_configured():
assert set(relogin.PLATFORMS) == {
"bilibili", "douyin", "pgy", "xingtu", "xiaohongshu"
}
assert relogin.PLATFORMS["xingtu"].required_cookie == "sessionid"
assert relogin.PLATFORMS["douyin"].required_cookie == "sessionid"
def test_manual_relogin_can_select_only_xingtu():
assert relogin.resolve_platform_names(["xingtu"]) == ["xingtu"]
assert relogin.resolve_platform_names(None) == list(relogin.PLATFORMS)
def test_default_screenshot_recipients_are_routed_by_platform():
assert relogin.DEFAULT_RECIPIENTS == {
"pgy": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
"xingtu": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
"douyin": "ou_7ad5fc8012e2f741afc5346e05ffd447",
"bilibili": "ou_7ad5fc8012e2f741afc5346e05ffd447",
"xiaohongshu": "ou_7ad5fc8012e2f741afc5346e05ffd447",
}
assert relogin.recipient_for_platform("pgy") == "ou_24cc944d6e43c69c59d6560ad4e2ae6e"
assert relogin.recipient_for_platform("douyin") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
assert relogin.recipient_for_platform("bilibili") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
assert relogin.recipient_for_platform("pgy", "ou_override") == "ou_override"
def test_failed_platforms_are_grouped_by_their_responsible_recipient():
grouped = relogin.group_platforms_by_recipient(
["pgy", "douyin", "bilibili", "xingtu", "xiaohongshu"]
)
assert grouped == {
"ou_24cc944d6e43c69c59d6560ad4e2ae6e": ["pgy", "xingtu"],
"ou_7ad5fc8012e2f741afc5346e05ffd447": [
"douyin", "bilibili", "xiaohongshu"
],
}
def test_login_windows_are_classified_by_title():
assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili"
assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy"
assert relogin._platform_from_window_title("抖音授权 - Google Chrome") == "xingtu"
assert relogin._platform_from_window_title("抖音 - 记录美好生活") == "douyin"
assert relogin._platform_from_window_title("小红书 - 你的生活兴趣社区") == "xiaohongshu"
assert relogin._platform_from_window_title("Codex") is None
def test_retry_only_failed_platforms_for_at_most_three_rounds():
calls = []
def fake_round(names, attempt):
calls.append((tuple(names), attempt))
if attempt == 1:
return {name: name != "xingtu" for name in names}
if attempt == 2:
return {name: False for name in names}
return {name: True for name in names}
result = relogin.run_with_retries(
["bilibili", "pgy", "xingtu", "xiaohongshu"],
max_attempts=3,
run_round=fake_round,
)
assert calls == [
(("bilibili", "pgy", "xingtu", "xiaohongshu"), 1),
(("xingtu",), 2),
(("xingtu",), 3),
]
assert all(result.values())
def test_stops_after_third_failed_round():
calls = []
def always_fail(names, attempt):
calls.append(attempt)
return {name: False for name in names}
result = relogin.run_with_retries(
["pgy"], max_attempts=3, run_round=always_fail
)
assert calls == [1, 2, 3]
assert result == {"pgy": False}
def test_timeout_terminates_the_complete_process_tree(monkeypatch):
calls = []
class Process:
pid = 1234
def terminate(self):
calls.append("fallback-terminate")
def wait(self, timeout):
calls.append(("wait", timeout))
monkeypatch.setattr(
relogin.subprocess,
"run",
lambda command, **kwargs: calls.append((command, kwargs)),
)
relogin._terminate_process_tree(Process())
assert calls[0][0] == ["taskkill", "/PID", "1234", "/T", "/F"]
assert calls[1] == ("wait", 10)
def test_transaction_rolls_back_files_and_profile(tmp_path):
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
state = relogin_transaction.begin((cookie, storage), profile)
cookie.write_text("partial", encoding="utf-8")
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
relogin_transaction.rollback(state)
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
@@ -0,0 +1,367 @@
import json
from pathlib import Path
import pytest
from gyxx_flow.catalog import WorkflowCatalog
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
from gyxx_flow.modules.content_marketing.creator_task_grouping import (
chunk_creator_groups,
group_tasks_by_creator,
)
PROJECT_ROOT = Path(__file__).resolve().parents[3]
def _workflow(workflow_id):
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
return next(item for item in catalog.workflows if item.workflow_id == workflow_id)
def test_group_tasks_uses_creator_id_and_merges_missing_id_by_unique_name():
tasks = [
{"creator_name": "同一达人", "creator_id": " 123 ", "record_id": "a"},
{"creator_name": "同一达人 ", "creator_id": None, "record_id": "b"},
{"creator_name": "另一个达人", "creator_id": None, "record_id": "c"},
]
groups = group_tasks_by_creator(tasks)
assert len(groups) == 2
assert groups[0]["creator_id"] == "123"
assert [task["record_id"] for task in groups[0]["tasks"]] == ["a", "b"]
def test_group_tasks_does_not_guess_when_same_name_has_conflicting_ids():
tasks = [
{"creator_name": "重名达人", "creator_id": "1", "record_id": "a"},
{"creator_name": "重名达人", "creator_id": "2", "record_id": "b"},
{"creator_name": "重名达人", "creator_id": None, "record_id": "c"},
]
groups = group_tasks_by_creator(tasks)
assert len(groups) == 3
def test_zero_batch_size_means_one_unlimited_batch():
groups = [{"creator_name": str(i), "tasks": []} for i in range(3)]
assert chunk_creator_groups(groups, 0) == [groups]
def _assert_cross_style_collection(module, monkeypatch):
styles = [
{"index": 1, "name": "款式A", "base_token": "base-a", "table_id": "table-a", "field_map": {"a": 1}},
{"index": 2, "name": "款式B", "base_token": "base-b", "table_id": "table-b", "field_map": {"b": 2}},
]
def fake_extract(style, only_record_ids):
return [{
"record_id": f"record-{style['index']}",
"creator_name": "同一达人",
"creator_id": "creator-1",
"target_title": f"标题{style['index']}",
}]
monkeypatch.setattr(module, "extract_target_tasks", fake_extract)
tasks, summaries = module.collect_tasks_across_styles(styles, None)
assert len(tasks) == 2
assert len(group_tasks_by_creator(tasks)) == 1
assert tasks[0]["style_context"]["table_id"] == "table-a"
assert tasks[1]["style_context"]["table_id"] == "table-b"
assert summaries[1]["total"] == 1
assert summaries[2]["total"] == 1
def test_pgy_collects_tasks_across_styles(monkeypatch):
_assert_cross_style_collection(pgy, monkeypatch)
def test_xingtu_collects_tasks_across_styles(monkeypatch):
_assert_cross_style_collection(xingtu, monkeypatch)
def test_xingtu_record_scope_is_applied_per_style(monkeypatch):
styles = [
{"index": 1, "name": "款式A", "base_token": "a", "table_id": "a", "field_map": {}},
{"index": 2, "name": "款式B", "base_token": "b", "table_id": "b", "field_map": {}},
]
observed = {}
def fake_extract(style, only_record_ids):
observed[style["index"]] = only_record_ids
return []
monkeypatch.setattr(xingtu, "extract_target_tasks", fake_extract)
xingtu.collect_tasks_across_styles(styles, {1: {"same-id"}, 2: {"other-id"}})
assert observed == {1: {"same-id"}, 2: {"other-id"}}
def test_xingtu_load_retry_scope_keeps_style_and_record_pair(tmp_path):
summary_path = tmp_path / "summary.json"
summary_path.write_text(json.dumps([
{"index": 1, "results": [
{"record_id": "same-id", "reason": "creator_search_circuit_open"},
]},
{"index": 2, "results": [
{"record_id": "same-id", "reason": "title_unmatched"},
{"record_id": "other-id", "reason": "creator_search_circuit_open"},
]},
]), encoding="utf-8")
scope = xingtu.load_retry_scope(summary_path, "creator_search_circuit_open")
assert scope == {1: {"same-id"}, 2: {"other-id"}}
def _assert_matched_task_uses_own_writeback_context(module, metric_key, monkeypatch):
calls = []
task = {
"record_id": "record-b",
"publish_time": object(),
"style_context": {
"index": 2,
"base_token": "base-b",
"table_id": "table-b",
"field_map": {"slot": "style-b-slot"},
},
}
summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []}
monkeypatch.setattr(module, "pick_read_field", lambda fmap, publish_time: ("read_count_7d", "field-b", "7天曝光量"))
monkeypatch.setattr(module, "write_back", lambda *args: calls.append(args) or True)
module.apply_matched_task(task, {metric_key: "1.2万", "title": "命中标题"}, summary, dry_run=False)
assert calls == [("base-b", "table-b", "record-b", "field-b", 12000)]
assert summary["matched"] == 1
assert summary["filled"] == 1
def test_pgy_match_writes_to_task_style_context(monkeypatch):
_assert_matched_task_uses_own_writeback_context(pgy, "read_count", monkeypatch)
def test_xingtu_match_writes_to_task_style_context(monkeypatch):
_assert_matched_task_uses_own_writeback_context(xingtu, "play_count", monkeypatch)
def test_xingtu_search_and_open_creator_submits_only_one_search(monkeypatch):
calls = []
def fail_search(_page, creator_name, creator_id):
calls.append((creator_name, creator_id))
raise TimeoutError("no result")
monkeypatch.setattr(xingtu, "search_creator", fail_search)
monkeypatch.setattr(
xingtu,
"open_creator_detail",
lambda *_args, **_kwargs: pytest.fail("detail must not open after search failure"),
)
with pytest.raises(TimeoutError, match="no result"):
xingtu.search_and_open_creator_once(object(), "同一达人", "creator-1")
assert calls == [("同一达人", "creator-1")]
def test_xingtu_search_budget_blocks_duplicate_without_failure_circuit():
budget = xingtu.CreatorSearchBudget(max_consecutive_failures=3)
assert budget.begin(("id", "1")) is True
assert budget.begin(("id", "1")) is False
budget.failed("first")
assert budget.begin(("id", "2")) is True
budget.failed("second")
assert budget.begin(("id", "3")) is True
assert budget.failed("third") is False
assert budget.begin(("id", "4")) is True
assert budget.calls == 4
assert budget.circuit_reason is None
budget.exhaust("search_quota_exhausted")
assert budget.begin(("id", "5")) is False
def test_xingtu_wait_for_results_detects_search_quota_exhausted():
class Page:
def evaluate(self, *_args):
return "quota"
def wait_for_timeout(self, _milliseconds):
pytest.fail("quota detection must return immediately")
with pytest.raises(xingtu.SearchQuotaExhausted):
xingtu.wait_for_visible_creator_result(Page(), "达人", timeout_ms=1000)
def test_xingtu_missing_creation_tab_is_not_treated_as_global_block():
class Page:
def evaluate(self, _script):
return "no_creation_tab"
assert xingtu.is_blocked(Page()) is False
@pytest.mark.parametrize("reason", ["verify_text", "verify_iframe", "verify_div"])
def test_xingtu_explicit_verification_still_opens_global_block(reason):
class Page:
def evaluate(self, _script):
return reason
assert xingtu.is_blocked(Page()) is True
def test_xingtu_missing_creation_tab_falls_back_to_current_creator_page():
class Locator:
@property
def first(self):
return self
def click(self, **_kwargs):
raise TimeoutError("missing tab")
class Page:
def get_by_text(self, *_args, **_kwargs):
return Locator()
def evaluate(self, _script):
return False
def wait_for_timeout(self, _milliseconds):
pass
assert xingtu.open_creation_ability(Page()) is False
def test_xingtu_current_creator_page_cards_are_used_as_fallback(monkeypatch):
class Page:
def wait_for_timeout(self, _milliseconds):
pass
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
monkeypatch.setattr(
xingtu,
"parse_videos_stable",
lambda _page: [{
"title": "MacBook党必看!提升幸福感的开工好物",
"play_count": "1.2万",
"note_id": "",
}],
)
monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False)
monkeypatch.setattr(xingtu, "submit_video_search", lambda _page, _title: False)
found, _candidates, page_limit_hit = xingtu.find_videos_for_tasks(
Page(),
[{
"record_id": "record-1",
"target_title": "MacBook党必看!提升幸福感的开工好物",
}],
)
assert found["record-1"]["play_count"] == "1.2万"
assert found["record-1"]["match_method"] == "title"
assert page_limit_hit is False
@pytest.mark.parametrize(
"message",
[
"BrowserContext.new_page: Protocol error (Target.createTarget): Failed to open a new tab",
"Protocol error (Target.createTarget)",
],
)
def test_xingtu_new_tab_creation_failure_rebuilds_browser_session(message):
assert xingtu.is_browser_session_lost(RuntimeError(message)) is True
def test_scheduled_collection_has_no_automatic_retry_and_syncs_once():
steps = _workflow("content.metrics.daily").steps
entries = [step.entry for step in steps]
assert "data/tools/retry_failed.py" not in entries
assert entries.count("data/tools/sync_metrics_to_cmt_notes.py") == 1
assert entries.index("run_all.py") < entries.index(
"data/tools/sync_metrics_to_cmt_notes.py"
)
def test_daily_collection_uses_dedicated_self_operated_pipeline():
steps = _workflow("content.metrics.daily").steps
entries = [step.entry for step in steps]
run_step = next(step for step in steps if step.entry == "run_all.py")
assert "--daily-scope" in run_step.args
assert "--include-self-operated" not in run_step.args
assert "data/tools/refresh_self_mapping.py" in entries
assert "self_bilibili_scraper.py" in entries
assert "chanmama_scraper.py" in entries
refresh_index = entries.index("data/tools/refresh_self_mapping.py")
bili_index = entries.index("self_bilibili_scraper.py")
douyin_index = entries.index("chanmama_scraper.py")
assert refresh_index < bili_index < douyin_index
@pytest.mark.parametrize(
("module", "entry_url", "error_type", "operation"),
[
(pgy, pgy.HOME_URL, pgy.AcceptanceCookieSkip, "content.pugongying.login"),
(
xingtu,
xingtu.MARKET_URL,
xingtu.AcceptanceCookieSkip,
"content.xingtu.login",
),
],
)
def test_acceptance_invalid_session_skips_before_qr_login(
module,
entry_url,
error_type,
operation,
monkeypatch,
tmp_path,
):
evidence_file = tmp_path / "evidence.jsonl"
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
monkeypatch.setenv("GYXX_FEISHU_TABLE_WRITE_DISABLED", "1")
monkeypatch.setenv("GYXX_COOKIE_INVALID_SKIP", "1")
monkeypatch.setenv(
"GYXX_NOTIFICATION_RECIPIENT_OPEN_ID",
"ou_8ee224968aa26a74c7d30ba27fed5eeb",
)
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence_file))
monkeypatch.setattr(module, "is_logged_in", lambda _page: False)
monkeypatch.setattr(
module,
"wait_for_scan_login",
lambda *_args, **_kwargs: pytest.fail("QR login must remain disabled"),
)
class Page:
def __init__(self):
self.urls = []
def goto(self, url, **_kwargs):
self.urls.append(url)
def wait_for_timeout(self, _milliseconds):
pass
page = Page()
with pytest.raises(error_type):
module.ensure_login(page, login_timeout=30)
assert page.urls == [entry_url]
records = [
json.loads(line)
for line in evidence_file.read_text(encoding="utf-8").splitlines()
]
assert records[-1]["event"] == "cookie_skipped"
assert records[-1]["operation"] == operation
assert records[-1]["details"]["status"] == "SKIPPED_COOKIE"
@@ -0,0 +1,367 @@
import pytest
from gyxx_flow.modules.content_marketing import (
pgy_xhs_scraper_v2,
xingtu_scraper_v2,
)
from gyxx_flow.modules.content_marketing.data.tools import (
relogin_douyin,
relogin_pgy,
relogin_xingtu,
)
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module):
cookie = tmp_path / "cookies.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
profile.mkdir()
(profile / "state.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
cookie_backup, profile_backup = module.reset_state()
assert not cookie.exists()
assert not profile.exists()
cookie.write_text("partial-new-cookie", encoding="utf-8")
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
module.restore_previous_state(cookie_backup, profile_backup)
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module):
cookie = tmp_path / "cookies.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
profile.mkdir()
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
cookie_backup, profile_backup = module.reset_state()
cookie.write_text("new-cookie", encoding="utf-8")
profile.mkdir()
module.finish_successful_relogin(profile_backup)
assert cookie.read_text(encoding="utf-8") == "new-cookie"
assert cookie_backup.exists()
assert not profile_backup.exists()
def test_xingtu_scan_login_opens_customer_sso_and_clicks_douyin():
class FakeLocator:
def __init__(self):
self.first = self
self.clicked = False
self.evaluated = False
def count(self):
return 1
def is_visible(self):
return True
def click(self, timeout):
self.clicked = True
def evaluate(self, script):
assert script == "e => e.click()"
self.evaluated = True
class FakePage:
url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market"
def __init__(self):
self.goto_url = None
self.selectors = []
self.douyin = FakeLocator()
def goto(self, url, **kwargs):
self.goto_url = url
self.url = url
def wait_for_timeout(self, _milliseconds):
pass
def locator(self, selector):
self.selectors.append(selector)
return self.douyin
def get_by_text(self, _pattern):
return FakeLocator()
page = FakePage()
xingtu_scraper_v2.open_scan_login(page)
assert page.goto_url == xingtu_scraper_v2.CUSTOMER_LOGIN_URL
assert 'img[src*="aweme.png"]' in page.selectors
assert page.douyin.clicked
def test_xingtu_index_page_is_recognized_as_logged_in():
class Locator:
@property
def first(self):
return self
def wait_for(self, timeout):
assert timeout == 1200
class Page:
url = "https://www.xingtu.cn/ad/creator/index"
def get_by_text(self, _text, exact=False):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is True
def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie():
class Context:
def cookies(self):
return [{"name": "sessionid", "value": "stale"}]
class Page:
url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market"
context = Context()
def get_by_text(self, *_args, **_kwargs):
raise AssertionError("public homepage must be rejected before marker checks")
assert xingtu_scraper_v2.is_logged_in(Page()) is False
def test_xingtu_market_page_without_business_ui_rejects_stale_cookie():
class Locator:
@property
def first(self):
return self
def wait_for(self, **_kwargs):
raise xingtu_scraper_v2.PlaywrightTimeoutError("missing")
def count(self):
return 0
class Context:
def cookies(self):
return [{"name": "sessionid", "value": "stale"}]
class Page:
url = "https://www.xingtu.cn/ad/creator/market"
context = Context()
def get_by_text(self, *_args, **_kwargs):
return Locator()
def locator(self, *_args, **_kwargs):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is False
def test_xingtu_cookie_validation_requires_real_session_cookie(tmp_path):
cookie = tmp_path / "xingtu.json"
cookie.write_text('[{"name":"other","value":"1"}]', encoding="utf-8")
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is False
cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8")
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is True
def test_xingtu_login_only_does_not_report_success_when_page_action_was_swallowed(
tmp_path, monkeypatch
):
cookie = tmp_path / "xingtu.json"
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
class Session:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def fetch(self, *_args, **_kwargs):
# Scrapling can log a page_action error and still return a response.
return object()
monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie)
monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session())
monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None)
assert xingtu_scraper_v2.login_only(1) == 1
def test_xingtu_login_only_verifies_persisted_profile_after_oauth_context_closes(
tmp_path, monkeypatch
):
cookie = tmp_path / "xingtu.json"
class Session:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def fetch(self, *_args, **_kwargs):
raise RuntimeError("Target page, context or browser has been closed")
monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie)
monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session())
monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None)
def verify():
cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8")
return True
monkeypatch.setattr(xingtu_scraper_v2, "verify_persisted_profile_login", verify)
assert xingtu_scraper_v2.login_only(1) == 0
def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monkeypatch):
class Context:
def __init__(self):
self.pages = []
def new_page(self):
raise AssertionError("a surviving replacement page should be reused")
class Page:
def __init__(self, context, url, *, closes_on_wait=False):
self.context = context
self.url = url
self.closed = False
self.closes_on_wait = closes_on_wait
def is_closed(self):
return self.closed
def wait_for_timeout(self, _milliseconds):
if self.closes_on_wait:
self.closed = True
def goto(self, url, **_kwargs):
self.url = url
context = Context()
qr_page = Page(
context,
"https://open.douyin.com/platform/oauth/pc/auth",
closes_on_wait=True,
)
market_page = Page(context, xingtu_scraper_v2.MARKET_URL)
context.pages = [qr_page, market_page]
saved = []
monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None)
monkeypatch.setattr(
xingtu_scraper_v2, "is_scan_qr_page", lambda page: page is qr_page
)
monkeypatch.setattr(
xingtu_scraper_v2, "is_logged_in", lambda page: page is market_page
)
monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page))
result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2)
assert result is market_page
assert saved == [market_page]
def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch):
cookie = tmp_path / "xingtu_cookies.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
profile.mkdir()
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie)
monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile)
cookie_backup, profile_backup = relogin_xingtu.reset_state()
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
assert relogin_xingtu.recover_interrupted_state() is True
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
assert cookie_backup is not None
assert profile_backup is not None
def test_pgy_scan_login_clicks_qr_switch_image():
class FakeLocator:
def __init__(self):
self.first = self
self.clicked = False
self.evaluated = False
def count(self):
return 1
def is_visible(self):
return True
def click(self, timeout):
self.clicked = True
def evaluate(self, script):
assert script == "e => e.click()"
self.evaluated = True
class FakePage:
def __init__(self):
self.selectors = []
self.qr_switch = FakeLocator()
def wait_for_timeout(self, _milliseconds):
pass
def get_by_text(self, _pattern):
return FakeLocator()
def locator(self, selector):
self.selectors.append(selector)
return self.qr_switch
page = FakePage()
pgy_xhs_scraper_v2.open_scan_login(page)
assert 'img[src*="qr_code"]' in page.selectors
assert page.qr_switch.evaluated
def test_failed_douyin_relogin_restores_cookie_storage_and_profile(tmp_path, monkeypatch):
cookie = tmp_path / "douyin_cookies.json"
storage = tmp_path / "douyin_storage_state.json"
profile = tmp_path / "profile"
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(relogin_douyin, "COOKIE_FILE", cookie)
monkeypatch.setattr(relogin_douyin, "STATE_FILE", storage)
monkeypatch.setattr(relogin_douyin, "PROFILE_DIR", profile)
monkeypatch.setattr(relogin_douyin.subprocess, "call", lambda *_args, **_kwargs: 1)
monkeypatch.setattr(relogin_douyin.sys, "argv", ["relogin_douyin.py"])
assert relogin_douyin.main() == 1
assert cookie.read_text(encoding="utf-8") == '[{"name":"sessionid","value":"old"}]'
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
@@ -0,0 +1,84 @@
import io
import unittest
from pathlib import Path
from gyxx_flow.adapters import RuntimeServicePolicy
from gyxx_flow.modules.content_marketing.data.tools import db
from gyxx_flow.modules.content_marketing.data.tools import (
sync_metrics_to_cmt_notes as sync_metrics,
)
ROOT = (
Path(__file__).resolve().parents[3]
/ "src"
/ "gyxx_flow"
/ "modules"
/ "content_marketing"
)
class LocalDatabaseConfigTests(unittest.TestCase):
def test_config_requires_every_final_connection_value(self):
complete = {
"PG_HOST": "127.0.0.1",
"PG_PORT": "5432",
"PG_DB": "gyxx_super_data",
"PG_USER": "gyxx_flow",
"PG_PASSWORD": "secret",
}
self.assertEqual(
db.database_config_from_env(complete),
{
"host": "127.0.0.1",
"port": 5432,
"dbname": "gyxx_super_data",
"user": "gyxx_flow",
"password": "secret",
},
)
for missing in complete:
values = {key: value for key, value in complete.items() if key != missing}
with self.subTest(missing=missing), self.assertRaisesRegex(RuntimeError, missing):
db.database_config_from_env(values)
def test_runtime_policy_uses_explicit_cloud_postgres(self):
environment = RuntimeServicePolicy().apply(
{
"PG_HOST": "db.example.com",
"PG_PORT": "5432",
"PG_DB": "data_hub",
"PG_USER": "data_hub",
"GYXX_POSTGRES_PASSWORD": "secret",
}
)
config = db.database_config_from_env(environment)
self.assertEqual(config["host"], "db.example.com")
self.assertEqual(config["dbname"], "data_hub")
self.assertEqual(config["user"], "data_hub")
def test_database_scripts_do_not_define_remote_connection_defaults(self):
paths = (
ROOT / "data" / "tools" / "db.py",
ROOT / "data" / "tools" / "sync_metrics_to_cmt_notes.py",
ROOT / "data" / "tools" / "sync_cooperations.py",
ROOT / "data" / "tools" / "generate_creator_report.py",
)
for path in paths:
source = path.read_text(encoding="utf-8-sig")
with self.subTest(path=path.name):
self.assertNotIn("8.148.185.119", source)
self.assertNotIn("data_hub", source)
def test_sync_console_replaces_characters_unsupported_by_gbk(self):
raw = io.BytesIO()
stream = io.TextIOWrapper(raw, encoding="gbk", errors="strict")
sync_metrics.configure_console_output(stream, None)
stream.write("𝑻")
stream.flush()
self.assertTrue(raw.getvalue())
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,398 @@
import json
import os
import time
from pathlib import Path
from types import SimpleNamespace
import gyxx_flow.modules.content_marketing.data.tools.retry_failed as retry
from gyxx_flow.modules.content_marketing import run_all
def test_requested_records_must_be_successful_even_when_process_exits_zero():
data = {
"results": [
{"record_id": "a", "status": "success", "matched": True},
{"record_id": "b", "status": "retryable_failure", "matched": False},
]
}
ok, failed = retry.requested_records_succeeded(data, ["a", "b"])
assert ok is False
assert failed == ["b"]
def test_success_status_cannot_hide_write_failure_and_blocked_input_is_terminal():
contradictory = {
"results": [{
"record_id": "a", "status": "success", "matched": True,
"write_ok": False,
}]
}
assert retry.requested_records_succeeded(contradictory, ["a"]) == (False, ["a"])
blocked = {
"total": 1,
"details": [{
"record_id": "b", "status": "blocked_input", "ok": False,
"reason": "url_missing",
}],
}
assert retry.extract_failed_records(blocked, "bili") == []
def test_canonical_filename_requires_current_style_name():
style = {"index": 2, "name": "极星pro"}
assert retry.canonical_result_filename(style, "pgy", False) == "02-极星pro_v2.json"
assert retry.canonical_result_filename(style, "xt", False) == "02-极星pro_xingtu_v2.json"
assert retry.canonical_result_filename(style, "bili", True) == "02-极星pro_self_bilibili_v2.json"
def _write_json(path: Path, payload: dict | list) -> None:
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
def test_find_failed_uses_exact_current_style_file_and_ignores_legacy_name(tmp_path):
style = {"index": 2, "name": "极星pro"}
canonical = tmp_path / "02-极星pro_v2.json"
legacy = tmp_path / "02-盖亚微单_v2.json"
payload = {
"index": 2,
"total": 1,
"matched": 1,
"filled": 1,
"results": [{"record_id": "ok", "status": "success", "matched": True}],
}
_write_json(canonical, payload)
_write_json(legacy, {"index": 2, "total": 1, "matched": 0, "results": []})
old = time.time() - 48 * 3600
os.utime(legacy, (old, old))
failed = retry.find_failed(
"pgy", stale_hours=4, include_half=True,
styles=[style], v2_dir=tmp_path,
)
assert failed == {}
def test_find_failed_treats_missing_result_rows_as_incomplete(tmp_path):
style = {"index": 1, "name": "款式A"}
path = tmp_path / retry.canonical_result_filename(style, "xt", False)
_write_json(path, {"index": 1, "total": 2, "matched": 0, "filled": 0, "results": []})
failed = retry.find_failed(
"xt", stale_hours=4, include_half=True,
styles=[style], v2_dir=tmp_path,
)
assert failed[1]["reason"] == "ALL_FAILED"
assert failed[1]["failed_rids"] == []
def test_run_one_style_rechecks_payload_and_retries_exit_zero(monkeypatch, tmp_path):
style = {"index": 1, "name": "款式A"}
result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False)
calls = []
def fake_call(_cmd, cwd, env=None):
del env
calls.append(cwd)
_write_json(result_path, {
"index": 1,
"total": 1,
"matched": 0,
"filled": 0,
"results": [{
"record_id": "bad",
"status": "retryable_failure",
"matched": False,
}],
})
return 0
monkeypatch.setattr(retry.subprocess, "call", fake_call)
monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {})
monkeypatch.setattr(retry.time, "sleep", lambda _seconds: None)
ok = retry.run_one_style(
"pgy", 1, max_attempts=2, record_ids=["bad"],
style=style, v2_dir=tmp_path,
)
assert ok is False
assert len(calls) == 2
def test_run_one_style_accepts_exit_zero_only_after_requested_record_succeeds(monkeypatch, tmp_path):
style = {"index": 1, "name": "款式A"}
result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False)
def fake_call(_cmd, cwd, env=None):
del env
_write_json(result_path, {
"index": 1,
"total": 1,
"matched": 1,
"filled": 1,
"results": [{
"record_id": "ok",
"status": "success",
"matched": True,
"write_ok": True,
}],
})
return 0
monkeypatch.setattr(retry.subprocess, "call", fake_call)
monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {})
assert retry.run_one_style(
"pgy", 1, max_attempts=1, record_ids=["ok"],
style=style, v2_dir=tmp_path,
) is True
def test_self_operated_xingtu_retry_uses_self_douyin_scraper(monkeypatch, tmp_path):
style = {"index": 1, "name": "款式A"}
result_path = tmp_path / retry.canonical_result_filename(style, "xt", True)
observed = {}
def fake_call(cmd, cwd, env=None):
del env
observed["cmd"] = cmd
_write_json(result_path, {
"index": 1,
"total": 1,
"results": [{"record_id": "ok", "status": "success", "matched": True}],
})
return 0
monkeypatch.setattr(retry.subprocess, "call", fake_call)
monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {})
assert retry.run_one_style(
"xt", 1, max_attempts=1, self_operated=True,
record_ids=["ok"], style=style, v2_dir=tmp_path,
) is True
assert observed["cmd"][1].endswith("self_douyin_scraper.py")
assert "--self-operated" not in observed["cmd"]
def test_full_summary_declared_failure_cannot_be_treated_as_complete():
complete, reason = retry._summary_completed({
"total": 1,
"results": [{
"record_id": "ok",
"status": "success",
"matched": True,
"write_ok": True,
}],
"write_failures": 1,
})
assert complete is False
assert reason == "summary declares unresolved failures"
def test_retry_main_returns_nonzero_when_verified_retry_still_fails(monkeypatch):
style = {"index": 1, "name": "款式A"}
monkeypatch.setattr(retry, "load_current_styles", lambda self_operated=False: [style])
monkeypatch.setattr(
retry, "find_failed",
lambda *_args, **_kwargs: {1: {"reason": "HALF", "failed_rids": ["bad"]}},
)
monkeypatch.setattr(retry, "run_one_style", lambda *_args, **_kwargs: False)
monkeypatch.setattr(retry.sys, "argv", ["retry_failed.py", "--platform", "pgy"])
assert retry.main() == 1
def test_validate_summary_payload_rejects_stale_missing_duplicate_and_unresolved(tmp_path):
path = tmp_path / "summary.json"
valid = [{
"index": 1,
"run_started_at": time.time() + 10,
"total": 2,
"matched": 2,
"filled": 2,
"results": [
{"record_id": "a", "status": "success", "matched": True},
{"record_id": "b", "status": "success", "matched": True},
],
}]
_write_json(path, valid)
assert run_all.validate_summary_payload(
valid, {1}, path, path.stat().st_mtime - 1,
)[0] is True
assert run_all.validate_summary_payload(
valid, {1}, path, path.stat().st_mtime + 1,
)[0] is False
assert run_all.validate_summary_payload(valid, {1, 2}, path, 0)[0] is False
duplicate = [{
"index": 1, "total": 2,
"results": [
{"record_id": "a", "status": "success"},
{"record_id": "a", "status": "success"},
],
}]
assert run_all.validate_summary_payload(duplicate, {1}, path, 0)[0] is False
unresolved = [{
"index": 1, "total": 1,
"results": [{
"record_id": "a", "status": "retryable_failure", "matched": False,
}],
}]
assert run_all.validate_summary_payload(unresolved, {1}, path, 0)[0] is False
def test_validate_selected_style_allows_unrelated_rows_in_merged_global_summary(tmp_path):
path = tmp_path / "summary.json"
payload = [
{
"index": 1,
"run_started_at": time.time() + 10,
"total": 1,
"results": [{"record_id": "ok", "status": "success"}],
},
{
"index": 2,
"total": 1,
"results": [{
"record_id": "old-failure",
"status": "retryable_failure",
}],
},
]
_write_json(path, payload)
valid, reason = run_all.validate_summary_payload(
payload, {1}, path, path.stat().st_mtime - 1,
)
assert valid is True, reason
def test_validate_summary_rejects_expected_style_left_over_from_old_run(tmp_path):
path = tmp_path / "summary.json"
started_at = 100.0
payload = [
{
"index": 1,
"run_started_at": 101.0,
"total": 1,
"results": [{"record_id": "new", "status": "success"}],
},
{
"index": 2,
"run_started_at": 90.0,
"total": 1,
"results": [{"record_id": "old", "status": "success"}],
},
]
_write_json(path, payload)
valid, reason = run_all.validate_summary_payload(
payload, {1, 2}, path, started_at,
)
assert valid is False
assert "style 2" in reason
assert "current run" in reason
def test_aggregate_counts_all_unresolved_rows_as_errors():
aggregate = run_all.aggregate({
"pgy": [{
"index": 1,
"total": 3,
"matched": 1,
"filled": 1,
"results": [
{"record_id": "ok", "status": "success"},
{"record_id": "miss", "matched": False},
{"record_id": "detail", "reason": "no_detail_page"},
],
}],
})
assert aggregate["platforms"]["pgy"]["unresolved"] == 2
assert aggregate["platforms"]["pgy"]["errors"] == 2
def test_aggregate_sums_declared_failure_categories_when_unresolved_is_absent():
aggregate = run_all.aggregate({
"xt": [{
"index": 1,
"total": 2,
"matched": 2,
"results": [
{"record_id": "a", "status": "success"},
{"record_id": "b", "status": "success"},
],
"retryable_failures": 1,
"write_failures": 1,
}],
})
assert aggregate["platforms"]["xt"]["unresolved"] == 2
def test_aggregate_keeps_partial_counts_when_validation_marks_payload_invalid():
aggregate = run_all.aggregate({
"pgy": {
"error": "invalid summary: unresolved",
"payload": [{
"index": 1,
"total": 1,
"matched": 0,
"results": [{
"record_id": "bad", "status": "retryable_failure",
}],
}],
},
})
platform = aggregate["platforms"]["pgy"]
assert platform["loaded"] is True
assert platform["unresolved"] == 1
assert platform["validation_error"] == "invalid summary: unresolved"
def test_run_round_marks_exit_zero_platform_failed_when_summary_is_invalid(monkeypatch, tmp_path):
args = SimpleNamespace(dry_run=True, style=[1])
stdout = tmp_path / "stdout.log"
stderr = tmp_path / "stderr.log"
observed = {}
monkeypatch.setattr(
run_all, "start_process",
lambda *_args, **_kwargs: (object(), stdout, stderr),
)
monkeypatch.setattr(run_all, "wait_all", lambda _procs: {"pgy": 0})
def fake_load(key, self_operated=False, **kwargs):
observed.update(kwargs)
return {"error": "invalid summary: stale"}
monkeypatch.setattr(run_all, "load_summary", fake_load)
monkeypatch.setattr(run_all, "print_report", lambda *_args: None)
monkeypatch.setattr(run_all, "_atomic_write_json", lambda *_args: None)
result = run_all.run_round(args, ["pgy"])
assert result["exit_codes"] == {"pgy": 0}
assert result["_non_zero_platforms"] == ["pgy"]
assert observed["expected_indices"] == {1}
assert isinstance(observed["started_at"], float)
def test_run_all_preserves_cookie_skip_exit_code():
assert run_all.aggregate_exit_code(True, [75, 75]) == 75
assert run_all.aggregate_exit_code(True, [75, 1]) == 1
assert run_all.aggregate_exit_code(True, []) == 1
assert run_all.aggregate_exit_code(False, []) == 0
@@ -0,0 +1,10 @@
from gyxx_flow.modules.content_marketing import self_douyin_scraper
def test_self_douyin_accepts_feishu_select_and_multiselect_values():
assert self_douyin_scraper.is_douyin_platform("抖音") is True
assert self_douyin_scraper.is_douyin_platform(["抖音"]) is True
assert self_douyin_scraper.is_douyin_platform("Douyin") is True
assert self_douyin_scraper.is_douyin_platform(["小红书", "抖音"]) is True
assert self_douyin_scraper.is_douyin_platform("小红书") is False
assert self_douyin_scraper.is_douyin_platform(None) is False
@@ -0,0 +1,58 @@
import unittest
from gyxx_flow.modules.content_marketing.data.tools.sync_style_categories import (
extract_style_mapping,
normalize_style_name,
)
class SyncStyleCategoriesTests(unittest.TestCase):
def test_normalizes_feishu_lifecycle_names_to_database_names(self):
cases = {
"盖亚微单Pro生命进程(新)": "盖亚微单",
"星迹&星迹2生命进程(新)": "星迹2",
"宙斯双肩包生命进程(新)": "宙斯",
"极星Pro生命进程(新)": "极星pro",
"瑞白双肩包生命进程(新)": "瑞白",
"逐星gt生命进程(新)": "逐星GT",
"拾影相机包生命进程 (新)": "拾影斜挎相机包",
"极星托特生命进程(新)": "极星托特",
}
for source, expected in cases.items():
with self.subTest(source=source):
self.assertEqual(normalize_style_name(source), expected)
def test_extracts_mapping_and_ignores_blank_rows(self):
payload = {
"data": {
"data": [
[
"[盖亚斜挎生命进程(新)](https://example.feishu.cn/base/TokenA)",
["都市机能"],
],
[None, ["都市运动"]],
["[极星托特生命进程(新)](https://example.feishu.cn/base/TokenB)", ["智性通勤"]],
]
}
}
self.assertEqual(
extract_style_mapping(payload),
{"盖亚斜挎": "都市机能", "极星托特": "智性通勤"},
)
def test_rejects_conflicting_categories_for_same_style(self):
payload = {
"data": {
"data": [
["[布谷生命进程(新)](https://example/a)", ["都市运动"]],
["[布谷生命进程(新)](https://example/b)", ["都市机能"]],
]
}
}
with self.assertRaisesRegex(ValueError, "布谷"):
extract_style_mapping(payload)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,70 @@
import unittest
from unittest.mock import patch
from gyxx_flow.modules.content_marketing import weekly_summary_all as weekly
class WeeklyTmallPersonaTests(unittest.TestCase):
def test_missing_persona_forbids_invented_percentages(self):
text = weekly.format_tmall_persona(None)
self.assertIn("不得虚构具体比例", text)
def test_format_tmall_persona_includes_date_and_dimensions(self):
persona = {
"collect_date": "2026-07-15",
"item_id": "900176117674",
"payload": {
"gender": [
{"name": "男性用户", "value": 66.97},
{"name": "女性用户", "value": 28.26},
],
"age": [{"name": "25-29岁", "value": 26.97}],
"city_tier": [{"name": "准一线城市", "value": 31.14}],
"buying_power": [{"name": "购买力L5", "value": 30.65}],
},
}
text = weekly.format_tmall_persona(persona)
self.assertIn("2026-07-15", text)
self.assertIn("男性用户 66.97%", text)
self.assertIn("25-29岁 26.97%", text)
self.assertIn("准一线城市 31.14%", text)
self.assertIn("购买力L5 30.65%", text)
def test_generate_summary_injects_latest_tmall_persona_into_single_note_prompt(self):
persona = {
"collect_date": "2026-07-15",
"item_id": "900176117674",
"payload": {
"gender": [{"name": "男性用户", "value": 66.97}],
"age": [{"name": "25-29岁", "value": 26.97}],
"city_tier": [],
"buying_power": [],
},
}
notes = [{
"title": "测试笔记",
"platform": "douyin",
"creator_name": "达人甲",
"view_count": 100,
"like_count": 10,
"comment_count": 2,
"favorite_count": 3,
"share_count": 1,
"report_path": None,
}]
with patch.object(weekly, "get_latest_tmall_persona", return_value=persona) as get_persona, \
patch.object(weekly, "call_hermes_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
weekly.generate_summary("星云2", notes)
get_persona.assert_called_once_with("星云2")
single_prompt = call.call_args_list[0].args[1]
self.assertIn("天猫该款最新人物画像", single_prompt)
self.assertIn("男性用户 66.97%", single_prompt)
self.assertIn("内容触达人群", single_prompt)
if __name__ == "__main__":
unittest.main()