feat: complete production workflow migration
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
analyze_note, # noqa: E402
|
||||
)
|
||||
|
||||
|
||||
class AnalyzeNoteWithoutCommentsTest(unittest.TestCase):
|
||||
def test_no_comments_still_writes_metrics_report(self):
|
||||
info = {
|
||||
"title": "无评论测试笔记",
|
||||
"platform": "xiaohongshu",
|
||||
"source_url": "https://example.com/note",
|
||||
"scraped_at": "",
|
||||
"comment_count_total": 0,
|
||||
"like_count": 12,
|
||||
"favorite_count": 3,
|
||||
"comment_count_metric": 0,
|
||||
"share_count": 1,
|
||||
"view_count": 1000,
|
||||
"collect_count_yxyy": 0,
|
||||
"style_name": "测试款",
|
||||
"creator_name": "测试达人",
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output = Path(tmpdir) / "note.txt"
|
||||
argv = [
|
||||
"analyze_note.py",
|
||||
"--note-id",
|
||||
"1",
|
||||
"--no-send",
|
||||
"--brand",
|
||||
"光影行星",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
with patch.object(
|
||||
analyze_note, "load_note_from_db", return_value=(info, [])
|
||||
), patch.object(
|
||||
analyze_note, "resolve_layer_output", return_value=output
|
||||
), patch.object(sys, "argv", argv):
|
||||
result = analyze_note.main()
|
||||
|
||||
self.assertEqual(result, 0)
|
||||
self.assertTrue(output.exists())
|
||||
report = output.read_text(encoding="utf-8-sig")
|
||||
self.assertIn("无有效评论可分析", report)
|
||||
self.assertIn("1,000", report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,11 @@
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
batch_rescrape_douyin as batch,
|
||||
)
|
||||
|
||||
|
||||
def test_result_logged_in_reads_nested_scraper_stats():
|
||||
result = {"stats": {"logged_in": True}}
|
||||
|
||||
assert batch.result_logged_in(result) is True
|
||||
assert batch.result_logged_in({"stats": {"logged_in": False}}) is False
|
||||
assert batch.result_logged_in({}) is False
|
||||
@@ -0,0 +1,253 @@
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.content_marketing import chanmama_scraper as chanmama
|
||||
|
||||
|
||||
def test_exposure_state_clicks_text_and_zero_but_keeps_positive_values():
|
||||
for value in ("更新数据", " 更新中 ", "暂无数据", "待更新", "0", "0.0", 0):
|
||||
assert chanmama._classify_exposure_text(value) == "refresh"
|
||||
for value in (23000, "23,000", "10w+", "8.8万+", ""):
|
||||
assert chanmama._classify_exposure_text(value) == "ready"
|
||||
|
||||
|
||||
def test_refresh_then_export_waits_once_after_all_accounts(monkeypatch):
|
||||
events = []
|
||||
urls = ["account-1", "account-2"]
|
||||
monkeypatch.setattr(chanmama, "navigate_to_target", lambda _driver, url: events.append(("navigate", url)))
|
||||
results = iter([
|
||||
{"clicked": 2, "pending": 0, "zero_without_button": 0, "pages": 2},
|
||||
{"clicked": 0, "pending": 1, "zero_without_button": 0, "pages": 1},
|
||||
])
|
||||
monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: next(results))
|
||||
monkeypatch.setattr(chanmama.time, "sleep", lambda seconds: events.append(("sleep", seconds)))
|
||||
monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 123.0)
|
||||
monkeypatch.setattr(chanmama, "export_video_data", lambda _driver, mtime: events.append(("export", mtime)) or "x.xlsx")
|
||||
monkeypatch.setattr(chanmama, "parse_chanmama_excel", lambda _path: ([{"title": "video"}], None))
|
||||
|
||||
records, summaries = chanmama.refresh_then_export_accounts(object(), urls, wait_seconds=600)
|
||||
|
||||
assert len(records) == 2
|
||||
assert [item["clicked"] for item in summaries] == [2, 0]
|
||||
assert events == [
|
||||
("navigate", "account-1"), ("navigate", "account-2"), ("sleep", 600),
|
||||
("navigate", "account-1"), ("export", 123.0),
|
||||
("navigate", "account-2"), ("export", 123.0),
|
||||
]
|
||||
|
||||
|
||||
def test_refresh_then_export_skips_wait_when_nothing_needs_refresh(monkeypatch):
|
||||
sleeps = []
|
||||
monkeypatch.setattr(chanmama, "navigate_to_target", lambda *_args: None)
|
||||
monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: {
|
||||
"clicked": 0, "pending": 0, "zero_without_button": 0, "pages": 1,
|
||||
})
|
||||
monkeypatch.setattr(chanmama.time, "sleep", sleeps.append)
|
||||
monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 0.0)
|
||||
monkeypatch.setattr(chanmama, "export_video_data", lambda *_args: None)
|
||||
|
||||
records, _ = chanmama.refresh_then_export_accounts(object(), ["account-1"], wait_seconds=600)
|
||||
|
||||
assert records == []
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
class _RefreshElement:
|
||||
def __init__(self, text="", css_class="", element_id="node"):
|
||||
self.text = text
|
||||
self.css_class = css_class
|
||||
self.id = element_id
|
||||
|
||||
def is_displayed(self):
|
||||
return True
|
||||
|
||||
def get_attribute(self, name):
|
||||
return self.css_class if name == "class" else None
|
||||
|
||||
def click(self):
|
||||
return None
|
||||
|
||||
|
||||
class _RefreshDriver:
|
||||
def __init__(self, nodes=None):
|
||||
self.nodes = list(nodes or [])
|
||||
|
||||
def find_elements(self, _by, selector):
|
||||
return self.nodes if selector == "td .cursor-pointer" else []
|
||||
|
||||
|
||||
def test_next_page_button_keeps_the_source_xpath_fallback():
|
||||
expected = "//span[normalize-space()='下一页']/ancestor::button[1]"
|
||||
button = _RefreshElement("下一页", element_id="next")
|
||||
|
||||
class Driver:
|
||||
def find_elements(self, _by, selector):
|
||||
return [button] if selector == expected else []
|
||||
|
||||
assert chanmama._find_next_page_button(Driver()) is button
|
||||
|
||||
|
||||
def test_find_get_data_targets_selects_actual_cursor_pointer_only():
|
||||
outer = _RefreshElement("获取数据", "detail-text", "outer")
|
||||
clickable = _RefreshElement("获取数据", "cp cursor-pointer", "inner")
|
||||
other = _RefreshElement("查看详情", "cursor-pointer", "other")
|
||||
assert chanmama._find_get_data_targets(_RefreshDriver([outer, clickable, other])) == [clickable]
|
||||
|
||||
|
||||
def test_refresh_zero_exposure_videos_walks_all_pages(monkeypatch):
|
||||
class Button:
|
||||
def __init__(self, disabled):
|
||||
self.disabled = disabled
|
||||
|
||||
class Driver:
|
||||
page = 0
|
||||
|
||||
def execute_script(self, _script, _element):
|
||||
self.page += 1
|
||||
|
||||
driver = Driver()
|
||||
page_results = [
|
||||
{"clicked": 2, "pending": 1, "zero_without_button": 0},
|
||||
{"clicked": 1, "pending": 0, "zero_without_button": 1},
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
chanmama, "_page_signature", lambda current: f"page-{current.page}"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_refresh_current_page",
|
||||
lambda current: page_results[current.page],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_find_next_page_button",
|
||||
lambda current: Button(disabled=current.page == 1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_pagination_element_disabled",
|
||||
lambda button: button.disabled,
|
||||
)
|
||||
monkeypatch.setattr(chanmama.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert chanmama.refresh_zero_exposure_videos(driver) == {
|
||||
"clicked": 3,
|
||||
"pending": 1,
|
||||
"zero_without_button": 1,
|
||||
"unconfirmed": 0,
|
||||
"pages": 2,
|
||||
}
|
||||
|
||||
|
||||
def test_missing_get_data_button_does_not_trigger_refresh_wait(monkeypatch):
|
||||
sleeps = []
|
||||
monkeypatch.setattr(chanmama, "navigate_to_target", lambda *_args: None)
|
||||
monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: {
|
||||
"clicked": 0, "pending": 0, "zero_without_button": 1, "pages": 1,
|
||||
})
|
||||
monkeypatch.setattr(chanmama.time, "sleep", sleeps.append)
|
||||
monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 0.0)
|
||||
monkeypatch.setattr(chanmama, "export_video_data", lambda *_args: None)
|
||||
chanmama.refresh_then_export_accounts(object(), ["account-1"], wait_seconds=600)
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
|
||||
events = []
|
||||
|
||||
class Driver:
|
||||
def quit(self):
|
||||
events.append("quit")
|
||||
|
||||
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
|
||||
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
|
||||
monkeypatch.setattr(chanmama, "save_cookies", lambda _driver: events.append("saved"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"load_cookies",
|
||||
lambda _driver: (_ for _ in ()).throw(AssertionError("profile must win")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"login",
|
||||
lambda _driver: (_ for _ in ()).throw(AssertionError("profile must win")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"refresh_then_export_accounts",
|
||||
lambda *_args, **_kwargs: ([], []),
|
||||
)
|
||||
|
||||
assert chanmama.main() == 0
|
||||
assert events == ["saved", "quit"]
|
||||
|
||||
|
||||
def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
evidence = tmp_path / "evidence.jsonl"
|
||||
|
||||
class Driver:
|
||||
def quit(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "ACCOUNT", "")
|
||||
monkeypatch.setattr(chanmama, "PASSWORD", "")
|
||||
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
|
||||
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: False)
|
||||
monkeypatch.setattr(chanmama, "load_cookies", lambda _driver: False)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"login",
|
||||
lambda _driver: (_ for _ in ()).throw(AssertionError("must not login")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"refresh_then_export_accounts",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
AssertionError("must not scrape")
|
||||
),
|
||||
)
|
||||
|
||||
assert chanmama.main() == COOKIE_SKIP_EXIT_CODE
|
||||
assert "profile/cookie session is invalid" in evidence.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_acceptance_captcha_is_non_interactive(monkeypatch, tmp_path):
|
||||
evidence = tmp_path / "evidence.jsonl"
|
||||
|
||||
class Element:
|
||||
def is_displayed(self):
|
||||
return True
|
||||
|
||||
class Driver:
|
||||
current_url = "https://www.chanmama.com/login.html"
|
||||
page_source = ""
|
||||
|
||||
def find_element(self, *_args):
|
||||
return Element()
|
||||
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.setattr(chanmama.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert chanmama.handle_captcha(Driver()) is False
|
||||
assert "captcha or interactive verification" in evidence.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def test_cli_preserves_cookie_skip_exit_code(monkeypatch):
|
||||
monkeypatch.setattr(chanmama, "main", lambda: COOKIE_SKIP_EXIT_CODE)
|
||||
|
||||
with pytest.raises(SystemExit) as raised:
|
||||
chanmama.cli()
|
||||
|
||||
assert raised.value.code == COOKIE_SKIP_EXIT_CODE
|
||||
@@ -0,0 +1,840 @@
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
|
||||
from gyxx_flow.modules.content_marketing import collection_completeness as cc
|
||||
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
|
||||
|
||||
def test_normalize_title_handles_nfkc_case_and_zero_width():
|
||||
assert cc.normalize_title("MacBook\u200b 通勤包!") == "macbook通勤包"
|
||||
|
||||
|
||||
def test_coerce_url_unwraps_feishu_markdown_links():
|
||||
value = "[查看笔记](https://www.xiaohongshu.com/discovery/item/abc123?x=1)"
|
||||
assert cc.coerce_url(value) == "https://www.xiaohongshu.com/discovery/item/abc123?x=1"
|
||||
|
||||
|
||||
def test_match_tasks_prefers_note_id_over_changed_title():
|
||||
tasks = [{
|
||||
"record_id": "r1",
|
||||
"target_title": "飞书中的旧标题",
|
||||
"note_url": "https://www.xiaohongshu.com/explore/abc123",
|
||||
"note_id": "abc123",
|
||||
}]
|
||||
cards = [
|
||||
{"title": "完全不同的新标题", "href": "https://www.xiaohongshu.com/explore/abc123", "note_id": "abc123"},
|
||||
{"title": "飞书中的旧标题", "href": "https://www.xiaohongshu.com/explore/other", "note_id": "other"},
|
||||
]
|
||||
|
||||
matched = cc.match_tasks_to_cards(tasks, cards, platform="xhs")
|
||||
|
||||
assert matched["r1"]["note_id"] == "abc123"
|
||||
assert matched["r1"]["match_method"] == "content_id"
|
||||
|
||||
|
||||
def test_match_tasks_accepts_unique_truncated_title_but_rejects_ambiguous_title():
|
||||
task = {"record_id": "r1", "target_title": "男生长期主义通勤双肩包分享"}
|
||||
unique = [
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
{"title": "夏日轻量斜挎包"},
|
||||
]
|
||||
ambiguous = [
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
]
|
||||
|
||||
assert cc.match_tasks_to_cards([task], unique, platform="douyin")["r1"]
|
||||
assert "r1" not in cc.match_tasks_to_cards([task], ambiguous, platform="douyin")
|
||||
|
||||
|
||||
def test_title_similarity_ignores_episode_prefix_and_hashtag_suffix():
|
||||
target = "第20集:Pocket4拍旋焦,无后期也能出片?"
|
||||
platform_title = (
|
||||
"Pocket4拍旋焦,无后期也能出片? #旋焦 #pocket4 #摄影装备 "
|
||||
"无论Pocket3还是Pocket4都可以轻松拍出旋焦效果!"
|
||||
)
|
||||
|
||||
assert cc.title_similarity(platform_title, target) >= 0.98
|
||||
|
||||
|
||||
def test_title_similarity_rejects_different_episode_numbers():
|
||||
assert cc.title_similarity(
|
||||
"第19集:Pocket4拍旋焦,无后期也能出片?",
|
||||
"第20集:Pocket4拍旋焦,无后期也能出片?",
|
||||
) == 0.0
|
||||
|
||||
|
||||
def test_match_tasks_rejects_duplicate_exact_titles_with_different_ids():
|
||||
task = {"record_id": "r1", "target_title": "完全相同的目标标题"}
|
||||
cards = [
|
||||
{"title": "完全相同的目标标题", "note_id": "video-1"},
|
||||
{"title": "完全相同的目标标题", "note_id": "video-2"},
|
||||
]
|
||||
|
||||
assert "r1" not in cc.match_tasks_to_cards([task], cards, platform="douyin")
|
||||
|
||||
|
||||
def test_one_card_is_not_reused_for_two_different_creator_tasks():
|
||||
tasks = [
|
||||
{
|
||||
"record_id": "loose",
|
||||
"target_title": "commuter backpack review today",
|
||||
"note_url": "https://v.douyin.com/loose-source/",
|
||||
},
|
||||
{
|
||||
"record_id": "exact",
|
||||
"target_title": "commuter backpack review",
|
||||
"note_url": "https://v.douyin.com/exact-source/",
|
||||
},
|
||||
]
|
||||
cards = [{"title": "commuter backpack review", "note_id": "observed-1"}]
|
||||
|
||||
matched = cc.match_tasks_to_cards(tasks, cards, platform="douyin")
|
||||
|
||||
assert set(matched) == {"exact"}
|
||||
|
||||
|
||||
def test_same_source_url_can_share_one_card_across_styles():
|
||||
tasks = [
|
||||
{
|
||||
"record_id": "style-1",
|
||||
"target_title": "同一篇跨款式笔记",
|
||||
"note_url": "https://v.douyin.com/same-short-link/",
|
||||
},
|
||||
{
|
||||
"record_id": "style-2",
|
||||
"target_title": "同一篇跨款式笔记",
|
||||
"note_url": "https://v.douyin.com/same-short-link/",
|
||||
},
|
||||
]
|
||||
cards = [{"title": "同一篇跨款式笔记", "note_id": "observed-1"}]
|
||||
|
||||
assert set(cc.match_tasks_to_cards(tasks, cards, platform="douyin")) == {
|
||||
"style-1", "style-2",
|
||||
}
|
||||
|
||||
|
||||
def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
|
||||
task = {
|
||||
"record_id": "r1",
|
||||
"target_title": "完全相同标题",
|
||||
"note_id": "video-a",
|
||||
}
|
||||
card = {"title": "完全相同标题", "note_id": "video-b"}
|
||||
|
||||
assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
|
||||
|
||||
|
||||
def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
|
||||
cards = [
|
||||
{
|
||||
"title": "同一条视频",
|
||||
"play_count": 19481,
|
||||
"note_id": "video-1",
|
||||
"source": "show_items_api",
|
||||
},
|
||||
{
|
||||
"title": "同一条视频",
|
||||
"play_count": "1.9w",
|
||||
"note_id": "",
|
||||
"page": 1,
|
||||
},
|
||||
]
|
||||
|
||||
deduped = cc.deduplicate_observed_cards(cards)
|
||||
matched = cc.match_tasks_to_cards(
|
||||
[{"record_id": "r1", "target_title": "同一条视频"}],
|
||||
deduped,
|
||||
platform="douyin",
|
||||
)
|
||||
|
||||
assert len(deduped) == 1
|
||||
assert matched["r1"]["note_id"] == "video-1"
|
||||
|
||||
|
||||
def test_merge_partial_summary_keeps_existing_success_rows():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total": 2, "matched": 1, "filled": 1,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
|
||||
{"record_id": "retry", "status": "retryable_failure", "matched": False},
|
||||
],
|
||||
}
|
||||
partial = {
|
||||
"style": "款式A", "index": 1, "total": 1, "matched": 1, "filled": 1,
|
||||
"results": [
|
||||
{"record_id": "retry", "status": "success", "matched": True, "write_ok": True},
|
||||
],
|
||||
}
|
||||
|
||||
merged = cc.merge_style_summary(existing, partial)
|
||||
|
||||
assert merged["total"] == 2
|
||||
assert {row["record_id"] for row in merged["results"]} == {"ok", "retry"}
|
||||
assert merged["matched"] == 2
|
||||
assert merged["filled"] == 2
|
||||
assert merged["complete"] is True
|
||||
|
||||
|
||||
def test_summary_contract_detects_missing_or_write_failure():
|
||||
summary = {
|
||||
"total": 2,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
|
||||
{"record_id": "bad", "status": "write_failure", "matched": True, "write_ok": False},
|
||||
],
|
||||
}
|
||||
finalized = cc.finalize_summary(summary, dry_run=False)
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] == 1
|
||||
|
||||
missing = cc.finalize_summary({"total": 2, "results": summary["results"][:1]}, dry_run=False)
|
||||
assert missing["complete"] is False
|
||||
assert missing["missing_results"] == 1
|
||||
|
||||
|
||||
def _style_for_platform(platform: str):
|
||||
return {
|
||||
"index": 1,
|
||||
"name": "款式A",
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"creator_name": {"field_id": "name"},
|
||||
"creator_id": {"field_id": "creator_id"},
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"platform": {"field_id": "platform"},
|
||||
"read_count_7d": {"field_id": "read7"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_extractors_preserve_note_url_and_content_id(monkeypatch):
|
||||
xhs_row = {
|
||||
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
|
||||
"title": "标题", "pub": "2026-07-20", "platform": "小红书",
|
||||
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
|
||||
}
|
||||
dy_row = {
|
||||
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
|
||||
"title": "标题", "pub": "2026-07-20", "platform": "抖音",
|
||||
"url": "https://www.douyin.com/video/7654321", "read7": None,
|
||||
}
|
||||
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
|
||||
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
|
||||
|
||||
xhs_task = pgy.extract_target_tasks(_style_for_platform("小红书"))[0]
|
||||
dy_task = xingtu.extract_target_tasks(_style_for_platform("抖音"))[0]
|
||||
|
||||
assert xhs_task["note_url"].endswith("/abc123")
|
||||
assert xhs_task["note_id"] == "abc123"
|
||||
assert dy_task["note_url"].endswith("/7654321")
|
||||
assert dy_task["note_id"] == "7654321"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "filled_url"),
|
||||
[
|
||||
(pgy, "小红书", "https://www.douyin.com/video/7654321"),
|
||||
(xingtu, "抖音", "http://8.99 分享文案 https://v.douyin.com/abc123/"),
|
||||
],
|
||||
)
|
||||
def test_filled_publish_link_triggers_collection_without_platform_validation(
|
||||
monkeypatch, module, platform, filled_url,
|
||||
):
|
||||
row = {
|
||||
"record_id": "r1",
|
||||
"name": "达人",
|
||||
"creator_id": "creator-id",
|
||||
"title": "已发布标题",
|
||||
"pub": "2026-07-20",
|
||||
"platform": platform,
|
||||
"url": filled_url,
|
||||
"read7": None,
|
||||
}
|
||||
monkeypatch.setattr(module, "list_records_by_table", lambda *_: [row])
|
||||
|
||||
task = module.extract_target_tasks(_style_for_platform(platform))[0]
|
||||
|
||||
assert task["input_error"] is None
|
||||
|
||||
|
||||
def test_extractors_can_collect_missing_title_when_direct_url_has_content_id(monkeypatch):
|
||||
xhs_style = _style_for_platform("小红书")
|
||||
dy_style = _style_for_platform("抖音")
|
||||
xhs_row = {
|
||||
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
|
||||
"title": None, "pub": "2026-07-20", "platform": "小红书",
|
||||
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
|
||||
}
|
||||
dy_row = {
|
||||
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
|
||||
"title": None, "pub": "2026-07-20", "platform": "抖音",
|
||||
"url": "https://www.douyin.com/video/7654321", "read7": None,
|
||||
}
|
||||
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
|
||||
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
|
||||
|
||||
assert pgy.extract_target_tasks(xhs_style)[0]["note_id"] == "abc123"
|
||||
assert xingtu.extract_target_tasks(dy_style)[0]["note_id"] == "7654321"
|
||||
assert xhs_style["_extract_stats"]["skipped_incomplete"] == 0
|
||||
assert dy_style["_extract_stats"]["skipped_incomplete"] == 0
|
||||
|
||||
|
||||
def test_missing_critical_mapping_is_an_incomplete_summary(monkeypatch):
|
||||
style = _style_for_platform("抖音")
|
||||
del style["field_map"]["note_url"]
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"list_records_by_table",
|
||||
lambda *_: (_ for _ in ()).throw(AssertionError("mapping should fail first")),
|
||||
)
|
||||
|
||||
_, summaries = xingtu.collect_tasks_across_styles([style], None)
|
||||
finalized = cc.finalize_summary(summaries[1], dry_run=True)
|
||||
|
||||
assert "note_url" in finalized["error"]
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] >= 1
|
||||
|
||||
|
||||
class _FakeInput:
|
||||
def __init__(self):
|
||||
self.filled = []
|
||||
|
||||
def click(self):
|
||||
return None
|
||||
|
||||
def fill(self, value):
|
||||
self.filled.append(value)
|
||||
|
||||
def press(self, _key):
|
||||
return None
|
||||
|
||||
|
||||
class _FakeLocator:
|
||||
def __init__(self, input_):
|
||||
self.first = input_
|
||||
|
||||
def filter(self, **_kwargs):
|
||||
return self
|
||||
|
||||
|
||||
class _FakeSearchPage:
|
||||
def __init__(self):
|
||||
self.input = _FakeInput()
|
||||
self.has_result_calls = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
def evaluate(self, script, arg=None):
|
||||
if "const rows" in script:
|
||||
self.has_result_calls += 1
|
||||
return self.has_result_calls >= 2
|
||||
if "const inputs" in script:
|
||||
return ""
|
||||
raise AssertionError(script)
|
||||
|
||||
def locator(self, _selector):
|
||||
return _FakeLocator(self.input)
|
||||
|
||||
def wait_for_url(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def test_pgy_search_retry_keeps_creator_id_query():
|
||||
page = _FakeSearchPage()
|
||||
|
||||
pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
|
||||
|
||||
assert page.input.filled == ["xhs-id-123"]
|
||||
|
||||
|
||||
def test_apply_matched_task_reports_write_failure(monkeypatch):
|
||||
task = {
|
||||
"record_id": "r1", "publish_time": object(),
|
||||
"style_context": {
|
||||
"index": 1, "base_token": "base", "table_id": "table", "field_map": {},
|
||||
},
|
||||
}
|
||||
summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []}
|
||||
monkeypatch.setattr(pgy, "pick_read_field", lambda *_: ("read_count_7d", "fid", "7天曝光量"))
|
||||
monkeypatch.setattr(pgy, "write_back", lambda *_: False)
|
||||
|
||||
outcome = pgy.apply_matched_task(task, {"read_count": "123", "title": "标题"}, summary, dry_run=False)
|
||||
|
||||
assert outcome["status"] == "write_failure"
|
||||
assert outcome["write_ok"] is False
|
||||
|
||||
|
||||
def test_default_creator_batch_has_no_forced_hour_pause():
|
||||
assert pgy.DEFAULT_BATCH_SIZE == 0
|
||||
assert xingtu.DEFAULT_BATCH_SIZE == 0
|
||||
|
||||
|
||||
def test_repeatable_style_argument_and_selection_keep_every_requested_style():
|
||||
parser = argparse.ArgumentParser()
|
||||
cc.add_repeatable_style_argument(parser, "styles")
|
||||
args = parser.parse_args(["--style", "1", "--style", "3"])
|
||||
styles = [{"index": 1}, {"index": 2}, {"index": 3}]
|
||||
|
||||
assert args.style == [1, 3]
|
||||
assert cc.select_requested_styles(styles, args.style) == [styles[0], styles[2]]
|
||||
with pytest.raises(ValueError, match="99"):
|
||||
cc.select_requested_styles(styles, [1, 99])
|
||||
|
||||
|
||||
def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
|
||||
sequence = [[], [{"title": "标题", "read_count": "1"}], [{"title": "标题", "read_count": "1"}]]
|
||||
monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
|
||||
|
||||
class Page:
|
||||
waits = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
self.waits += 1
|
||||
|
||||
page = Page()
|
||||
assert pgy.parse_cards_stable(page) == [{"title": "标题", "read_count": "1"}]
|
||||
assert page.waits == 2
|
||||
|
||||
|
||||
def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
|
||||
pages = [
|
||||
[{"title": "无关视频", "play_count": "10"}],
|
||||
[{"title": "目标视频标题", "play_count": "20"}],
|
||||
]
|
||||
calls = {"next": 0}
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "parse_videos_stable", lambda _page: pages.pop(0))
|
||||
|
||||
def next_page(_page):
|
||||
calls["next"] += 1
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(xingtu, "go_next_page", next_page)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
|
||||
|
||||
found, _, _ = xingtu.find_videos_for_tasks(
|
||||
object(), [{"record_id": "r1", "target_title": "目标视频标题"}], max_pages=2,
|
||||
)
|
||||
|
||||
assert found["r1"]["play_count"] == "20"
|
||||
assert calls["next"] == 1
|
||||
|
||||
|
||||
def test_xingtu_normalizes_show_items_api_video():
|
||||
card = xingtu.normalize_xingtu_api_item({
|
||||
"item_id": 7654321,
|
||||
"item_title": "目标视频标题",
|
||||
"play": 168000,
|
||||
"like": 1200,
|
||||
"comment": 34,
|
||||
"share": 56,
|
||||
"url": "https://www.douyin.com/video/7654321",
|
||||
})
|
||||
|
||||
assert card == {
|
||||
"title": "目标视频标题",
|
||||
"play_count": 168000,
|
||||
"like_count": 1200,
|
||||
"comment_count": 34,
|
||||
"share_count": 56,
|
||||
"href": "https://www.douyin.com/video/7654321",
|
||||
"note_id": "7654321",
|
||||
"source": "show_items_api",
|
||||
}
|
||||
|
||||
|
||||
def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
|
||||
class Response:
|
||||
url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
|
||||
|
||||
def json(self):
|
||||
return {
|
||||
"latest_item_info": [
|
||||
{"item_id": "1", "title": "视频一", "play": 10},
|
||||
{"item_id": "2", "item_title": "视频二", "play": 20},
|
||||
],
|
||||
"latest_star_item_info": [
|
||||
{"item_id": "2", "item_title": "视频二", "play": 20},
|
||||
],
|
||||
}
|
||||
|
||||
class Page:
|
||||
def on(self, event, callback):
|
||||
assert event == "response"
|
||||
self.callback = callback
|
||||
|
||||
page = Page()
|
||||
cards = xingtu.setup_show_items_capture(page)
|
||||
page.callback(Response())
|
||||
|
||||
assert [card["note_id"] for card in cards] == ["1", "2"]
|
||||
|
||||
|
||||
def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: (_ for _ in ()).throw(AssertionError("DOM should not be needed")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"is_blocked",
|
||||
lambda _page: (_ for _ in ()).throw(AssertionError("page should not be needed")),
|
||||
)
|
||||
tasks = [{
|
||||
"record_id": "r1",
|
||||
"target_title": "飞书旧标题",
|
||||
"note_url": "https://www.douyin.com/video/7654321",
|
||||
"note_id": "7654321",
|
||||
}]
|
||||
api_cards = [{
|
||||
"title": "平台上的新标题",
|
||||
"play_count": 168000,
|
||||
"note_id": "7654321",
|
||||
"source": "show_items_api",
|
||||
}]
|
||||
|
||||
found, _, page_limit_hit = xingtu.find_videos_for_tasks(
|
||||
object(), tasks, max_pages=5, api_cards=api_cards,
|
||||
)
|
||||
|
||||
assert found["r1"]["play_count"] == 168000
|
||||
assert found["r1"]["match_method"] == "content_id"
|
||||
assert page_limit_hit is False
|
||||
|
||||
|
||||
def test_xingtu_api_fuzzy_candidate_does_not_preempt_later_exact_dom(monkeypatch):
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: [{"title": "commuter backpack review", "note_id": "exact-card"}],
|
||||
)
|
||||
monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
|
||||
task = {"record_id": "r1", "target_title": "commuter backpack review"}
|
||||
api_cards = [{
|
||||
"title": "commuter backpack review today",
|
||||
"note_id": "fuzzy-card",
|
||||
"source": "show_items_api",
|
||||
}]
|
||||
|
||||
found, _, _ = xingtu.find_videos_for_tasks(
|
||||
object(), [task], max_pages=2, api_cards=api_cards,
|
||||
)
|
||||
|
||||
assert found["r1"]["note_id"] == "exact-card"
|
||||
|
||||
|
||||
def test_browser_session_loss_detection_is_specific():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
assert pgy.is_browser_session_lost(closed)
|
||||
assert xingtu.is_browser_session_lost(closed)
|
||||
assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
|
||||
|
||||
|
||||
def test_xingtu_nested_session_loss_is_reraised():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.reraise_if_browser_session_lost(closed)
|
||||
xingtu.reraise_if_browser_session_lost(RuntimeError("ordinary lookup miss"))
|
||||
|
||||
|
||||
def test_xingtu_detail_and_video_search_propagate_closed_session():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
class Detail:
|
||||
def goto(self, *_args, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
def new_page(self):
|
||||
return Detail()
|
||||
|
||||
class SearchPage:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, _script, _arg=None):
|
||||
return {"found": True, "url": "https://www.xingtu.cn/ad/creator/author-homepage/douyin-video/1"}
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.open_creator_detail(SearchPage(), "达人", "creator-id")
|
||||
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class VideoPage:
|
||||
def locator(self, _selector):
|
||||
return Locator()
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.submit_video_search(VideoPage(), "标题")
|
||||
|
||||
|
||||
def test_pgy_fallback_detail_click_propagates_closed_session():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
class Clicker:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def click(self, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
class Page:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, *_args, **_kwargs):
|
||||
return {"ok": False, "reason": "no-row"}
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Clicker()
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
pgy.open_blogger_detail(Page(), "达人", None)
|
||||
|
||||
|
||||
def test_bilibili_triggered_blocked_inputs_are_visible_terminal_rows():
|
||||
summary = bili.finalize_bili_summary({
|
||||
"style": "款式A", "index": 1, "total_b_records": 1,
|
||||
"details": [{
|
||||
"record_id": "r1", "status": "blocked_input",
|
||||
"matched": False, "reason": "publish_time_missing", "ok": False,
|
||||
}],
|
||||
})
|
||||
|
||||
assert summary["blocked_input"] == 1
|
||||
assert summary["unresolved"] == 0
|
||||
assert summary["complete"] is True
|
||||
|
||||
|
||||
def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
|
||||
style = {
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"read_count_7d": {"field_id": "s7"},
|
||||
"read_count_14d": {"field_id": "s14"},
|
||||
"read_count_21d": {"field_id": "s21"},
|
||||
"read_count_28d": {"field_id": "s28"},
|
||||
"month_end": {"field_id": "sm"},
|
||||
},
|
||||
}
|
||||
rows = [
|
||||
{
|
||||
"record_id": "no-link",
|
||||
"platform": "B站",
|
||||
"creator": "未发布达人",
|
||||
"url": "",
|
||||
"pub": "2026-07-20",
|
||||
},
|
||||
{
|
||||
"record_id": "published",
|
||||
"platform": "B站",
|
||||
"creator": "已发布达人",
|
||||
"url": "https://www.bilibili.com/video/BV1234567890",
|
||||
"pub": "2026-07-20",
|
||||
},
|
||||
]
|
||||
monkeypatch.setattr(bili, "list_all_records", lambda *_args: rows)
|
||||
monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: 321)
|
||||
monkeypatch.setattr(bili, "write_record", lambda *_args, **_kwargs: True)
|
||||
|
||||
summary = bili.process_style(
|
||||
style,
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=bili.date(2026, 7, 23),
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
assert summary["source_b_records"] == 2
|
||||
assert summary["skipped_no_url"] == 1
|
||||
assert summary["total_b_records"] == 1
|
||||
assert summary["success"] == 1
|
||||
assert summary["blocked_input"] == 0
|
||||
assert [row["record_id"] for row in summary["details"]] == ["published"]
|
||||
|
||||
|
||||
def test_bilibili_partial_retry_preserves_old_success():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 2,
|
||||
"details": [
|
||||
{"record_id": "ok", "status": "success", "ok": True},
|
||||
{"record_id": "retry", "status": "retryable_failure", "ok": False},
|
||||
],
|
||||
}
|
||||
partial = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 1,
|
||||
"details": [{"record_id": "retry", "status": "success", "ok": True}],
|
||||
}
|
||||
|
||||
merged = bili.merge_bili_summary(existing, partial)
|
||||
|
||||
assert merged["total_b_records"] == 2
|
||||
assert {row["record_id"] for row in merged["details"]} == {"ok", "retry"}
|
||||
assert merged["complete"] is True
|
||||
|
||||
|
||||
def test_bilibili_top_level_error_cannot_finalize_as_complete():
|
||||
finalized = bili.finalize_bili_summary({
|
||||
"style": "款式A",
|
||||
"index": 1,
|
||||
"total_b_records": 0,
|
||||
"details": [],
|
||||
"error": "missing platform/url field_id",
|
||||
"unresolved": 1,
|
||||
"retryable_failures": 1,
|
||||
"complete": False,
|
||||
})
|
||||
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] >= 1
|
||||
|
||||
|
||||
def test_bilibili_missing_slot_mapping_is_system_error_not_blocked_input():
|
||||
summary = bili.process_style(
|
||||
{
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
},
|
||||
},
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=None,
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
finalized = bili.finalize_bili_summary(summary, dry_run=True)
|
||||
assert "publish_time" in finalized["error"]
|
||||
assert "read_count_7d" in finalized["error"]
|
||||
assert finalized["complete"] is False
|
||||
|
||||
|
||||
def test_self_douyin_missing_mapping_and_write_failure_are_not_complete(monkeypatch):
|
||||
missing = self_dy.scrape_one_style(
|
||||
{"name": "款式A", "index": 1, "base_token": "base", "table_id": "table", "field_map": {}},
|
||||
login_timeout=1,
|
||||
headless=True,
|
||||
only_record_ids=None,
|
||||
dry_run=False,
|
||||
)
|
||||
assert cc.finalize_summary(missing)["complete"] is False
|
||||
|
||||
style = _style_for_platform("抖音")
|
||||
row = {
|
||||
"record_id": "r1",
|
||||
"platform": "抖音",
|
||||
"name": "自营达人",
|
||||
"title": "目标作品",
|
||||
"url": "https://www.douyin.com/video/1",
|
||||
"pub": "2026-07-20",
|
||||
}
|
||||
|
||||
class Page:
|
||||
def goto(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
class Session:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def fetch(self, _url, page_action, wait):
|
||||
page_action(Page())
|
||||
|
||||
monkeypatch.setattr(self_dy, "DynamicSession", Session)
|
||||
monkeypatch.setattr(self_dy, "list_records_by_table", lambda *_: [row])
|
||||
monkeypatch.setattr(self_dy, "dismiss_popups", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "restore_cookies", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "maybe_wait_for_login", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "save_state", lambda *_: None)
|
||||
monkeypatch.setattr(
|
||||
self_dy,
|
||||
"scroll_and_collect_posts",
|
||||
lambda *_args, **_kwargs: [{"title": "目标作品", "play_count": 123}],
|
||||
)
|
||||
monkeypatch.setattr(self_dy, "pick_read_field", lambda *_: ("read_count_7d", "read7", "7天曝光量"))
|
||||
monkeypatch.setattr(self_dy, "write_back", lambda *_: False)
|
||||
|
||||
failed = cc.finalize_summary(
|
||||
self_dy.scrape_one_style(
|
||||
style,
|
||||
login_timeout=1,
|
||||
headless=True,
|
||||
only_record_ids=None,
|
||||
dry_run=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert failed["results"][0]["status"] == "write_failure"
|
||||
assert failed["complete"] is False
|
||||
|
||||
|
||||
def test_run_all_rejects_stale_or_incomplete_summary(tmp_path: Path):
|
||||
path = tmp_path / "summary.json"
|
||||
path.write_text("[]", encoding="utf-8")
|
||||
started_at = path.stat().st_mtime + 1
|
||||
|
||||
assert run_all.validate_summary_payload([], {1}, path, started_at)[0] is False
|
||||
|
||||
path.write_text('[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}]', encoding="utf-8")
|
||||
assert run_all.validate_summary_payload(
|
||||
[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}],
|
||||
{1}, path, path.stat().st_mtime - 1,
|
||||
)[0] is False
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.modules.content_marketing import chanmama_scraper, weekly_summary_all
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
analyze_comments,
|
||||
analyze_note,
|
||||
daily_report_card,
|
||||
friday_relogin_parallel,
|
||||
)
|
||||
|
||||
|
||||
def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path:
|
||||
evidence = tmp_path / "content-evidence.jsonl"
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False)
|
||||
return evidence
|
||||
|
||||
|
||||
def test_content_write_back_skips_before_lark_cli(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
evidence = _enable_acceptance(monkeypatch, tmp_path)
|
||||
|
||||
def fail_if_called(*args, **kwargs):
|
||||
raise AssertionError("lark-cli wrapper must not be called")
|
||||
|
||||
monkeypatch.setattr(chanmama_scraper, "_call_lark_json", fail_if_called)
|
||||
assert chanmama_scraper._write_back(
|
||||
"base_secret", "tbl_test", "rec_test", "fld_test", 123
|
||||
)
|
||||
|
||||
payload = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
assert payload["operation"].endswith("record-upsert")
|
||||
|
||||
|
||||
def test_weekly_summary_skips_table_upsert_but_reports_success(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(
|
||||
weekly_summary_all,
|
||||
"PATHS",
|
||||
SimpleNamespace(tmp_root=tmp_path),
|
||||
)
|
||||
|
||||
def fail_if_called(*args, **kwargs):
|
||||
raise AssertionError("weekly lark-cli wrapper must not be called")
|
||||
|
||||
monkeypatch.setattr(weekly_summary_all, "call_lark_json", fail_if_called)
|
||||
assert weekly_summary_all.write_to_target_table(
|
||||
"base_secret",
|
||||
"tbl_test",
|
||||
"fld_time",
|
||||
"fld_summary",
|
||||
"style-test",
|
||||
"https://example.test/doc",
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
def test_daily_card_sends_only_to_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"message_id": "om_test"}),
|
||||
stderr="",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(daily_report_card.subprocess, "run", fake_run)
|
||||
results = daily_report_card.send_card_to_recipients(
|
||||
{"schema": "2.0", "body": {"elements": []}},
|
||||
("ou_other_a", "ou_other_b"),
|
||||
"2026-08-01",
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(commands) == 1
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
|
||||
def test_comment_summary_sends_only_to_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||||
|
||||
monkeypatch.setattr(analyze_comments.subprocess, "run", fake_run)
|
||||
analyze_comments.send_feishu_summary("summary", "ou_other")
|
||||
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
|
||||
def test_note_report_sends_only_to_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
monkeypatch.setattr(analyze_note.sys, "platform", "linux")
|
||||
monkeypatch.setattr(
|
||||
analyze_note,
|
||||
"build_card_payload",
|
||||
lambda *args, **kwargs: {"schema": "2.0"},
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||||
|
||||
monkeypatch.setattr(analyze_note.subprocess, "run", fake_run)
|
||||
analyze_note.send_feishu_report(
|
||||
{},
|
||||
{},
|
||||
{},
|
||||
[],
|
||||
[],
|
||||
"analysis",
|
||||
"ou_other",
|
||||
tmp_path / "report.md",
|
||||
)
|
||||
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
|
||||
def test_relogin_notifications_resolve_only_to_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
|
||||
assert (
|
||||
friday_relogin_parallel.recipient_for_platform("pgy", "ou_other")
|
||||
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
|
||||
|
||||
def test_run_all_rebinds_browser_environment_to_child_entry(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def fake_rebind(target, base_environment):
|
||||
observed["target"] = Path(target)
|
||||
observed["base"] = dict(base_environment)
|
||||
return {
|
||||
**base_environment,
|
||||
"GYXX_SCRIPT_ID": "content_marketing:pgy_xhs_scraper_v2.py",
|
||||
"GYXX_BROWSER_CDP_PORT": "child-port",
|
||||
"GYXX_BROWSER_PROFILE_DIR": "child-profile",
|
||||
"GYXX_BROWSER_COOKIE_FILE": "child-cookie",
|
||||
"GYXX_BROWSER_STORAGE_STATE_FILE": "child-storage",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(run_all, "environment_for_child_script", fake_rebind)
|
||||
target = run_all.BASE_DIR / "pgy_xhs_scraper_v2.py"
|
||||
|
||||
environment = run_all.build_child_environment(
|
||||
target,
|
||||
{
|
||||
"KEEP": "yes",
|
||||
"GYXX_BROWSER_CDP_PORT": "parent-port",
|
||||
"GYXX_BROWSER_PROFILE_DIR": "parent-profile",
|
||||
},
|
||||
)
|
||||
|
||||
assert observed["target"] == target
|
||||
assert observed["base"]["GYXX_BROWSER_CDP_PORT"] == "parent-port"
|
||||
assert environment["KEEP"] == "yes"
|
||||
assert environment["GYXX_BROWSER_CDP_PORT"] == "child-port"
|
||||
assert environment["GYXX_BROWSER_PROFILE_DIR"] == "child-profile"
|
||||
assert environment["GYXX_BROWSER_COOKIE_FILE"] == "child-cookie"
|
||||
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == "child-storage"
|
||||
assert environment["PYTHONIOENCODING"] == "utf-8"
|
||||
assert environment["PYTHONUNBUFFERED"] == "1"
|
||||
@@ -0,0 +1,100 @@
|
||||
from argparse import Namespace
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
from gyxx_flow.modules.content_marketing.daily_creator_exposure_scope import (
|
||||
DEFAULT_MAX_AGE_DAYS,
|
||||
DEFAULT_PUBLISHED_FROM,
|
||||
EXCLUDED_STYLE_NAMES,
|
||||
classify_publish_scope,
|
||||
select_daily_styles,
|
||||
)
|
||||
|
||||
|
||||
def test_daily_scope_excludes_requested_styles():
|
||||
styles = [
|
||||
{"index": 1, "name": "保留款"},
|
||||
{"index": 2, "name": "盖亚微单"},
|
||||
{"index": 3, "name": "逐星GT"},
|
||||
{"index": 4, "name": "晨星2"},
|
||||
{"index": 5, "name": "觅光"},
|
||||
]
|
||||
assert [style["name"] for style in select_daily_styles(styles)] == ["保留款"]
|
||||
assert EXCLUDED_STYLE_NAMES == frozenset({"盖亚微单", "逐星GT", "晨星2", "觅光"})
|
||||
|
||||
|
||||
def test_publish_scope_starts_on_july_first_and_stops_at_30_days():
|
||||
collected_on = date(2026, 7, 31)
|
||||
assert classify_publish_scope(
|
||||
date(2026, 6, 30),
|
||||
collected_on,
|
||||
published_from=DEFAULT_PUBLISHED_FROM,
|
||||
max_age_days=DEFAULT_MAX_AGE_DAYS,
|
||||
) == "before_publish_cutoff"
|
||||
assert classify_publish_scope(
|
||||
date(2026, 7, 1),
|
||||
collected_on,
|
||||
published_from=DEFAULT_PUBLISHED_FROM,
|
||||
max_age_days=DEFAULT_MAX_AGE_DAYS,
|
||||
) == "collection_window_complete"
|
||||
assert classify_publish_scope(
|
||||
date(2026, 7, 2),
|
||||
collected_on,
|
||||
published_from=DEFAULT_PUBLISHED_FROM,
|
||||
max_age_days=DEFAULT_MAX_AGE_DAYS,
|
||||
) is None
|
||||
|
||||
|
||||
def test_run_all_daily_scope_passes_filters_and_no_retry_to_children():
|
||||
args = Namespace(dry_run=False, style=[1, 2], daily_scope=True, no_retry=False)
|
||||
command = run_all.build_process_command("pgy", args)
|
||||
assert "--skip-field-prepare" in command
|
||||
assert command[command.index("--published-from") + 1] == "2026-07-01"
|
||||
assert command[command.index("--max-age-days") + 1] == "30"
|
||||
assert "--no-retry" in command
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "url"),
|
||||
[
|
||||
(pgy, "小红书", "https://www.xiaohongshu.com/explore/eligible"),
|
||||
(xingtu, "抖音", "https://www.douyin.com/video/eligible"),
|
||||
],
|
||||
)
|
||||
def test_platform_task_extraction_applies_daily_publish_window(
|
||||
module, platform, url, monkeypatch
|
||||
):
|
||||
style = {
|
||||
"index": 1,
|
||||
"name": "保留款",
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "published"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"platform": {"field_id": "platform"},
|
||||
"daily_exposure": {"field_id": "daily", "field_name": "2026-07-29曝光量"},
|
||||
},
|
||||
}
|
||||
base = {"creator": "达人", "title": "标题", "url": url, "platform": platform}
|
||||
records = [
|
||||
{"record_id": "old", **base, "published": "2026-06-30"},
|
||||
{"record_id": "eligible", **base, "published": "2026-07-02"},
|
||||
{"record_id": "missing", **base, "published": None},
|
||||
]
|
||||
monkeypatch.setattr(module, "list_records_by_table", lambda *_args: records)
|
||||
tasks = module.extract_target_tasks(
|
||||
style,
|
||||
published_from=date(2026, 7, 1),
|
||||
max_age_days=30,
|
||||
collection_date=date(2026, 7, 29),
|
||||
)
|
||||
assert [task["record_id"] for task in tasks] == ["eligible"]
|
||||
assert style["_extract_stats"]["skipped_publish_scope"] == 1
|
||||
assert style["_extract_stats"]["skipped_incomplete"] == 1
|
||||
@@ -0,0 +1,105 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools.daily_dashboard_analytics import (
|
||||
build_dashboard_facts,
|
||||
)
|
||||
|
||||
|
||||
class DailyDashboardAnalyticsTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tracking = [
|
||||
{"style_name": "大流量", "visitors": 1000, "cart_users": 100, "sales": 20, "refund_orders": 2, "light": "🟡"},
|
||||
{"style_name": "高转化", "visitors": 100, "cart_users": 30, "sales": 20, "refund_orders": 8, "light": "🔴"},
|
||||
{"style_name": "极小样本", "visitors": 10, "cart_users": 5, "sales": 10, "refund_orders": 0, "light": "🔴"},
|
||||
{"style_name": "缺数据", "visitors": None, "cart_users": None, "sales": None, "refund_orders": None, "light": "无判断"},
|
||||
]
|
||||
self.platform = [
|
||||
{"style_name": "大流量", "platform": "天猫", "visitors": 900, "cart_users": 90, "sales": 18, "refund_orders": 2},
|
||||
{"style_name": "高转化", "platform": "天猫", "visitors": 100, "cart_users": 30, "sales": 20, "refund_orders": 8},
|
||||
{"style_name": "极小样本", "platform": "天猫", "visitors": 10, "cart_users": 5, "sales": 10, "refund_orders": 0},
|
||||
]
|
||||
self.personas = [
|
||||
{
|
||||
"style_name": "大流量", "platform": "天猫", "data_date": date(2026, 7, 15),
|
||||
"gender_top": "女性 60%", "age_top": "26-35岁 45%",
|
||||
"city_top": "一线城市 35%", "buying_power_top": "高级白领 40%",
|
||||
}
|
||||
]
|
||||
|
||||
def test_all_styles_receive_same_metrics_and_diagnosis(self):
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas,
|
||||
{"大流量": {"creator_count": 3, "directions": ["通勤"]}},
|
||||
)
|
||||
self.assertEqual(len(facts["styles"]), 4)
|
||||
high = next(row for row in facts["styles"] if row["style_name"] == "高转化")
|
||||
self.assertEqual(high["conversion_rate"], 0.2)
|
||||
self.assertEqual(high["cart_rate"], 0.3)
|
||||
self.assertEqual(high["refund_rate"], 0.4)
|
||||
missing = next(row for row in facts["styles"] if row["style_name"] == "缺数据")
|
||||
self.assertEqual(missing["diagnosis"], "无数据")
|
||||
|
||||
def test_top_conversion_excludes_tiny_visitor_base(self):
|
||||
facts = build_dashboard_facts("2026-07-15", self.tracking, self.platform, self.personas, {})
|
||||
self.assertEqual(facts["rankings"]["conversion"][0]["style_name"], "高转化")
|
||||
self.assertEqual(facts["rankings"]["visitors"][0]["style_name"], "大流量")
|
||||
self.assertEqual(facts["rankings"]["refund_rate"][0]["style_name"], "高转化")
|
||||
|
||||
def test_platform_winners_keep_missing_persona_explicit(self):
|
||||
facts = build_dashboard_facts("2026-07-15", self.tracking, self.platform, self.personas, {})
|
||||
card = facts["platform_personas"][0]
|
||||
self.assertEqual(card["platform"], "天猫")
|
||||
self.assertEqual(card["top_conversion"]["style_name"], "高转化")
|
||||
self.assertEqual(card["top_conversion"]["persona_text"], "无数据")
|
||||
self.assertEqual(card["top_visitors"]["style_name"], "大流量")
|
||||
self.assertIn("推测", card["top_visitors"]["interest_hypothesis"])
|
||||
self.assertIn("提转化", card["top_visitors"]["optimization"])
|
||||
|
||||
def test_daily_new_notes_overview_replaces_view_delta_and_creator_ranking(self):
|
||||
notes = [{"note_id": 7, "style_name": "大流量", "creator": "达人A", "platform": "小红书", "view_count": 500}]
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas, {}, daily_notes=notes
|
||||
)
|
||||
self.assertEqual(facts["daily_new_notes"], notes)
|
||||
self.assertNotIn("creator_connections", facts["rankings"])
|
||||
self.assertNotIn("daily_views", facts["rankings"])
|
||||
|
||||
def test_daily_view_ranking_and_platform_cards_receive_stored_images(self):
|
||||
images = {
|
||||
("高转化", "天猫"): {"local_image_path": "same.jpg", "image_platform": "天猫"},
|
||||
("大流量", "天猫"): {"local_image_path": "visitor.jpg", "image_platform": "天猫"},
|
||||
}
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas, {},
|
||||
style_images=images,
|
||||
)
|
||||
card = facts["platform_personas"][0]
|
||||
self.assertEqual(card["top_conversion"]["local_image_path"], "same.jpg")
|
||||
self.assertEqual(card["top_visitors"]["local_image_path"], "visitor.jpg")
|
||||
|
||||
def test_matrix_groups_by_style_category_without_total_exposure(self):
|
||||
signals = {
|
||||
"高转化": {"persona": True, "note_count": 3, "recent_note_exposure": 43000, "comment_count": 80,
|
||||
"review_count": 10, "review_negative_rate": 0.3, "creative_ctr": 0.08},
|
||||
}
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas, {},
|
||||
style_categories={
|
||||
"大流量": "都市机能", "高转化": "智性通勤",
|
||||
"极小样本": "智性通勤", "缺数据": "都市机能",
|
||||
},
|
||||
style_signals=signals,
|
||||
)
|
||||
self.assertEqual([group["style_category"] for group in facts["style_groups"]], ["都市机能", "智性通勤"])
|
||||
self.assertEqual([row["style_name"] for row in facts["style_groups"][0]["styles"]], ["大流量", "缺数据"])
|
||||
high = next(row for row in facts["styles"] if row["style_name"] == "高转化")
|
||||
self.assertNotIn("total_note_exposure", high)
|
||||
self.assertEqual(high["recent_note_exposure"], 43000)
|
||||
self.assertNotIn("total_note_exposure", facts["definitions"])
|
||||
self.assertIn("评价风险", high["diagnosis"])
|
||||
self.assertIn("画像", high["evidence_summary"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,127 @@
|
||||
import copy
|
||||
import json
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
|
||||
from gyxx_flow.modules.content_marketing import feishu_mapping
|
||||
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
|
||||
TARGET_DATE = date(2026, 7, 28)
|
||||
TARGET_NAME = "2026-07-28曝光量"
|
||||
|
||||
|
||||
def _mapping():
|
||||
return {"tables": [{"index": 1, "name": "款式A", "base_token": "base-a", "table_id": "table-a", "field_map": {}, "all_field_names": []}]}
|
||||
|
||||
|
||||
def test_daily_exposure_field_name_uses_collection_date():
|
||||
assert feishu_mapping.daily_exposure_field_name(TARGET_DATE) == TARGET_NAME
|
||||
|
||||
|
||||
def test_build_field_map_only_maps_exact_requested_daily_field():
|
||||
fmap, names = feishu_mapping.build_field_map([
|
||||
{"id": "old", "name": "2026-07-27曝光量"}, {"id": "today", "name": TARGET_NAME},
|
||||
], target_date=TARGET_DATE)
|
||||
assert fmap["daily_exposure"] == {"field_id": "today", "field_name": TARGET_NAME}
|
||||
assert names == ["2026-07-27曝光量", TARGET_NAME]
|
||||
|
||||
|
||||
def test_ensure_daily_field_reuses_existing_field(monkeypatch):
|
||||
mapping = _mapping()
|
||||
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: [{"id": "field-today", "name": TARGET_NAME}])
|
||||
calls = []
|
||||
monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda args: calls.append(args) or {"ok": True})
|
||||
result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, write=False)
|
||||
assert result["ok"] is True and result["created"] == [] and calls == []
|
||||
assert mapping["tables"][0]["field_map"]["daily_exposure"]["field_id"] == "field-today"
|
||||
|
||||
|
||||
def test_ensure_daily_field_creates_number_field_then_refreshes(monkeypatch):
|
||||
mapping = _mapping()
|
||||
fields = iter([[], [{"id": "new-field", "name": TARGET_NAME}]])
|
||||
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: next(fields))
|
||||
calls = []
|
||||
monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda args: calls.append(args) or {"ok": True})
|
||||
result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, write=False)
|
||||
assert result["ok"] is True and result["created"] == ["款式A"]
|
||||
payload = calls[0][calls[0].index("--json") + 1]
|
||||
assert TARGET_NAME in payload and '"type": "number"' in payload
|
||||
|
||||
|
||||
def test_ensure_daily_field_dry_run_does_not_call_feishu(monkeypatch):
|
||||
mapping = _mapping()
|
||||
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: (_ for _ in ()).throw(AssertionError("must not list")))
|
||||
monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda *_: (_ for _ in ()).throw(AssertionError("must not create")))
|
||||
result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, dry_run=True, write=False)
|
||||
assert result["ok"] is True
|
||||
assert mapping["tables"][0]["field_map"]["daily_exposure"]["field_id"].startswith("dryrun:")
|
||||
|
||||
|
||||
def test_acceptance_skips_field_create_and_keeps_sentinel_in_memory(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
):
|
||||
evidence = tmp_path / "evidence.jsonl"
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
mapping = _mapping()
|
||||
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: [])
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping,
|
||||
"call_lark_json",
|
||||
lambda *_: (_ for _ in ()).throw(AssertionError("must not create")),
|
||||
)
|
||||
|
||||
result = feishu_mapping.ensure_daily_exposure_fields(
|
||||
mapping,
|
||||
target_date=TARGET_DATE,
|
||||
write=False,
|
||||
)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["created"] == []
|
||||
assert result["skipped_creations"] == ["款式A"]
|
||||
assert mapping["tables"][0]["field_map"]["daily_exposure"][
|
||||
"field_id"
|
||||
].startswith("acceptance-skipped:")
|
||||
payload = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
assert payload["operation"] == "content.mapping.daily-exposure-field-create"
|
||||
|
||||
|
||||
def test_acceptance_hydrates_cached_mapping_without_persisting_sentinel(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
):
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
cache = tmp_path / "mapping.json"
|
||||
cache.write_text(json.dumps(_mapping(), ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
hydrated = feishu_mapping._hydrate_acceptance_daily_field(
|
||||
json.loads(cache.read_text(encoding="utf-8")),
|
||||
self_operated=False,
|
||||
)
|
||||
|
||||
assert hydrated["tables"][0]["field_map"]["daily_exposure"][
|
||||
"field_id"
|
||||
].startswith("acceptance-skipped:")
|
||||
persisted = json.loads(cache.read_text(encoding="utf-8"))
|
||||
assert "daily_exposure" not in persisted["tables"][0]["field_map"]
|
||||
|
||||
|
||||
def test_collaboration_slot_pickers_use_daily_field_without_publish_time():
|
||||
fmap = {"daily_exposure": {"field_id": "today-field", "field_name": TARGET_NAME}}
|
||||
expected = ("daily_exposure", "today-field", TARGET_NAME)
|
||||
assert pgy.pick_read_field(fmap, None, datetime(2026, 7, 28)) == expected
|
||||
assert xingtu.pick_read_field(fmap, None, datetime(2026, 7, 28)) == expected
|
||||
|
||||
|
||||
def test_self_operated_mapping_removes_daily_field(monkeypatch):
|
||||
source = {"tables": [{"field_map": {
|
||||
"daily_exposure": {"field_id": "daily", "field_name": TARGET_NAME},
|
||||
"read_count_7d": {"field_id": "s7", "field_name": "7天曝光量"},
|
||||
}}]}
|
||||
monkeypatch.setattr(feishu_mapping, "load_mapping", lambda *_args, **_kwargs: copy.deepcopy(source))
|
||||
for mapping in (pgy.load_mapping(True), xingtu.load_mapping(True), bili.load_mapping(bili.DEFAULT_DATA_DIR, True)):
|
||||
assert "daily_exposure" not in mapping["tables"][0]["field_map"]
|
||||
@@ -0,0 +1,442 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing import daily_marketing_report as daily
|
||||
from gyxx_flow.modules.content_marketing.data.tools import analyze_comments
|
||||
|
||||
|
||||
class DailyMarketingReportTests(unittest.TestCase):
|
||||
def test_hermes_url_accepts_api_base_or_full_completion_path(self):
|
||||
full_url = "http://127.0.0.1:8642/v1/chat/completions"
|
||||
|
||||
self.assertEqual(
|
||||
analyze_comments._chat_completions_url("http://127.0.0.1:8642/v1"),
|
||||
full_url,
|
||||
)
|
||||
self.assertEqual(
|
||||
analyze_comments._chat_completions_url(full_url),
|
||||
full_url,
|
||||
)
|
||||
|
||||
def test_sales_forecast_uses_seven_day_total_and_recent_three_day_velocity(self):
|
||||
forecast = daily.estimate_sales_forecast([10, 12, 14, 16, 18, 20, 22])
|
||||
|
||||
self.assertEqual(forecast["sales_7d_total"], 112)
|
||||
self.assertEqual(forecast["sales_forecast_7d"], 140)
|
||||
self.assertAlmostEqual(forecast["sales_forecast_change"], 7 / 13, places=4)
|
||||
|
||||
def test_sales_forecast_requires_complete_seven_day_history(self):
|
||||
self.assertEqual(
|
||||
daily.estimate_sales_forecast([10, 12, 14]),
|
||||
{
|
||||
"sales_7d_total": None,
|
||||
"sales_forecast_7d": None,
|
||||
"sales_forecast_change": None,
|
||||
},
|
||||
)
|
||||
|
||||
def test_default_recipients_include_all_four_daily_report_owners(self):
|
||||
self.assertEqual(
|
||||
daily.DEFAULT_RECIPIENT_OPEN_IDS,
|
||||
(
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
"ou_fa8d81a16527ad06352dbecc575285b8",
|
||||
"ou_2eda5eec112109ae6d19a1f6813eadcb",
|
||||
"ou_89bcff110ccbb09a23548dc0fb3d880c",
|
||||
),
|
||||
)
|
||||
|
||||
def test_hermes_transport_error_is_not_accepted_as_report(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "Hermes 分析失败"):
|
||||
daily.require_valid_hermes_report("API call failed after 3 retries: Connection error.")
|
||||
self.assertEqual(daily.require_valid_hermes_report("整体经营分析\n销量保持稳定"), "整体经营分析\n销量保持稳定")
|
||||
|
||||
def test_light_uses_highest_absolute_visitor_or_cart_change(self):
|
||||
self.assertEqual(daily.classify_light(0.05, -0.08), "🟢")
|
||||
self.assertEqual(daily.classify_light(0.12, 0.02), "🟡")
|
||||
self.assertEqual(daily.classify_light(-0.31, 0.01), "🔴")
|
||||
self.assertEqual(daily.classify_light(None, None), "无判断")
|
||||
|
||||
def test_comment_signal_requires_minimum_sample(self):
|
||||
comments = ["光影行星宙斯怎么买"] * 49
|
||||
self.assertIsNone(daily.calculate_comment_signal("宙斯", comments))
|
||||
|
||||
comments.append("好看")
|
||||
signal = daily.calculate_comment_signal("宙斯", comments)
|
||||
self.assertEqual(signal["sample_size"], 50)
|
||||
self.assertEqual(signal["brand_mention_rate"], 0.98)
|
||||
self.assertEqual(signal["purchase_intent_rate"], 0.98)
|
||||
|
||||
def test_prompt_uses_only_the_fresh_user_instruction_and_database_facts(self):
|
||||
prompt = daily.build_daily_prompt(
|
||||
report_date="2026-07-15",
|
||||
today_notes=[],
|
||||
tracking_rows=[],
|
||||
next_day_rows=[],
|
||||
comment_rows=[],
|
||||
)
|
||||
self.assertIn("以下为数据库查询结果", prompt)
|
||||
self.assertIn('"报告日期": "2026-07-15"', prompt)
|
||||
self.assertIn("增长型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("流量机会型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("潜力型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("风险型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertNotIn("A类:爆款增长SKU", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_prompt_uses_fashion_bag_data_analyst_role(self):
|
||||
self.assertIn("专业的时尚包袋品牌电商运营数据分析师", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("销售、电商运营及达人营销数据", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("发现增长机会和风险", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("SKU营销运营日报", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("展示关键数据", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_fresh_prompt_contains_the_complete_requested_analysis_structure(self):
|
||||
prompt = daily.SYSTEM_PROMPT
|
||||
for phrase in (
|
||||
"今日整体经营判断",
|
||||
"SKU经营表现分析",
|
||||
"达人营销效果分析",
|
||||
"产品节奏判断",
|
||||
"蓄水期",
|
||||
"放量期",
|
||||
"稳定期",
|
||||
"衰退期",
|
||||
"增长SKU TOP3",
|
||||
"风险SKU TOP3",
|
||||
"比较昨日、七日基准",
|
||||
"用图表展示",
|
||||
"曝光量",
|
||||
"建议动作",
|
||||
"对销量影响",
|
||||
"所有建议必须具体到SKU",
|
||||
):
|
||||
self.assertIn(phrase, prompt)
|
||||
for retired in ("A类:爆款增长SKU", "B类:潜力增长SKU", "D类:需要优化SKU"):
|
||||
self.assertNotIn(retired, prompt)
|
||||
|
||||
def test_database_payload_keeps_all_available_sources(self):
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], {"reviews": [{"sample_size": 3}]})
|
||||
for phrase in ("所有SKU电商运营数据", "达人营销内容数据", "评论与补充经营数据", "reviews"):
|
||||
self.assertIn(phrase, prompt)
|
||||
|
||||
def test_note_label_ignores_placeholder_creator_name(self):
|
||||
note = {"id": 7, "creator_name": "/", "title": "瑞白通勤包实测"}
|
||||
self.assertEqual(daily.note_label(note), "瑞白通勤包实测")
|
||||
|
||||
def test_cooperation_metrics_hide_unverifiable_engagement(self):
|
||||
invalid = daily._sanitize_cooperation_row({
|
||||
"publish_time": None,
|
||||
"exposure_count": 38000,
|
||||
"engagement_count_num": 21430000,
|
||||
})
|
||||
self.assertIsNone(invalid["engagement_count_num"])
|
||||
self.assertEqual(invalid["publish_status"], "发布时间无数据")
|
||||
self.assertIn("口径异常", invalid["engagement_data_status"])
|
||||
|
||||
valid = daily._sanitize_cooperation_row({
|
||||
"publish_time": "2026-07-14",
|
||||
"exposure_count": 38000,
|
||||
"engagement_count_num": 2143,
|
||||
})
|
||||
self.assertEqual(valid["engagement_count_num"], 2143)
|
||||
self.assertEqual(valid["publish_status"], "已记录发布时间")
|
||||
|
||||
def test_load_report_facts_tracks_all_styles_for_report_date(self):
|
||||
metrics = {
|
||||
"宙斯": {"light": "🟢"},
|
||||
}
|
||||
with patch.object(daily, "_note_rows", return_value=[]), \
|
||||
patch.object(daily, "_all_style_names", return_value=["宙斯", "蓝鹊"]) as all_styles, \
|
||||
patch.object(daily, "_style_metrics", return_value=metrics) as style_metrics:
|
||||
with patch.object(daily, "_load_enrichment", return_value={"platform_metrics": []}), \
|
||||
patch.object(daily, "_platform_metrics", return_value=[]), \
|
||||
patch.object(daily, "_persona_context", return_value=[]), \
|
||||
patch.object(daily, "load_creator_connections", return_value={}), \
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
patch.object(daily, "build_dashboard_facts", return_value={"styles": ["宙斯", "蓝鹊"]}):
|
||||
_, tracking, _, _, enrichment = daily.load_report_facts(date(2026, 7, 15))
|
||||
|
||||
all_styles.assert_called_once_with(date(2026, 7, 15))
|
||||
self.assertEqual([row["style_name"] for row in tracking], ["宙斯", "蓝鹊"])
|
||||
self.assertEqual(tracking[1]["light"], "无判断")
|
||||
self.assertEqual(style_metrics.call_args_list[0].args[1], ["宙斯", "蓝鹊"])
|
||||
self.assertEqual(enrichment["platform_metrics"], [])
|
||||
self.assertEqual(enrichment["candidate_styles"], [])
|
||||
self.assertEqual(enrichment["dashboard"]["styles"], ["宙斯", "蓝鹊"])
|
||||
|
||||
def test_report_date_uses_yesterday_notes_for_daily_new_notes(self):
|
||||
notes = [
|
||||
{"id": 1, "publish_date": "2026-07-15", "style_name": "宙斯", "creator_name": "达人甲", "platform": "小红书", "publish_time": "2026-07-15 10:00", "title": "今天", "view_count": 10, "engagement_rate": None},
|
||||
{"id": 2, "publish_date": "2026-07-14", "style_name": "宙斯", "creator_name": "达人乙", "platform": "抖音", "publish_time": "2026-07-14 11:00", "title": "昨天", "view_count": 20, "engagement_rate": None},
|
||||
]
|
||||
with patch.object(daily, "_note_rows", return_value=notes), \
|
||||
patch.object(daily, "_all_style_names", return_value=["宙斯"]), \
|
||||
patch.object(daily, "_style_metrics", return_value={}), \
|
||||
patch.object(daily, "_load_enrichment", return_value={}), \
|
||||
patch.object(daily, "_platform_metrics", return_value=[]), \
|
||||
patch.object(daily, "_persona_context", return_value=[]), \
|
||||
patch.object(daily, "load_creator_connections", return_value={}), \
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
patch.object(daily, "_comment_rows", return_value=[]), \
|
||||
patch.object(daily, "build_dashboard_facts", return_value={}) as dashboard:
|
||||
daily_notes, _, _, _, _ = daily.load_report_facts(date(2026, 7, 15))
|
||||
|
||||
self.assertEqual([row["note_id"] for row in daily_notes], [2])
|
||||
self.assertEqual([row["note_id"] for row in dashboard.call_args.kwargs["daily_notes"]], [2])
|
||||
|
||||
def test_prompt_requires_all_styles_without_top_ten_truncation(self):
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [])
|
||||
self.assertIn("所有SKU", prompt)
|
||||
self.assertNotIn("只保留异常程度最高的10个", prompt)
|
||||
self.assertIn("SKU经营表现分析", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("加购率", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_prompt_uses_the_new_daily_output_contract(self):
|
||||
self.assertIn("电商运营数据", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("达人营销数据", daily.SYSTEM_PROMPT)
|
||||
self.assertNotIn("TOP20", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("一页A4", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("↑↓展示趋势", daily.SYSTEM_PROMPT)
|
||||
self.assertNotIn("→稳定", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("电商销量表现 + 访客数 + 转化率 + 达人笔记曝光 + 用户反馈", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_prompt_payload_keeps_yesterday_and_seven_day_comparisons(self):
|
||||
rows = [{
|
||||
"style_name": "蓝鹊",
|
||||
"sales": 10,
|
||||
"sales_yesterday": 8,
|
||||
"avg_sales_7d": 7.5,
|
||||
"sales_change_yesterday": 0.25,
|
||||
"sales_change": 0.3333,
|
||||
}]
|
||||
prompt = daily.build_daily_prompt("2026-07-16", [], rows, [], [])
|
||||
self.assertIn('"sales_yesterday": 8', prompt)
|
||||
self.assertIn('"avg_sales_7d": 7.5', prompt)
|
||||
self.assertIn('"sales_change_yesterday": 0.25', prompt)
|
||||
|
||||
def test_style_metrics_calculates_yesterday_and_seven_day_changes(self):
|
||||
class FakeCursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def execute(self, query, params):
|
||||
self.query = query
|
||||
self.params = params
|
||||
|
||||
def fetchall(self):
|
||||
return [(
|
||||
"蓝鹊",
|
||||
120, 24, 10, 2,
|
||||
100, 20, 8, 1,
|
||||
90, 18, 7.5, 1.2,
|
||||
[4, 5, 6, 7, 8, 9, 10],
|
||||
)]
|
||||
|
||||
class FakeConnection:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return FakeCursor()
|
||||
|
||||
with patch.object(daily, "get_conn", return_value=FakeConnection()):
|
||||
row = daily._style_metrics(date(2026, 7, 16), ["蓝鹊"])["蓝鹊"]
|
||||
|
||||
self.assertEqual(row["sales_yesterday"], 8)
|
||||
self.assertEqual(row["sales_change_yesterday"], 0.25)
|
||||
self.assertEqual(row["sales_change"], 0.3333)
|
||||
self.assertEqual(row["visitor_change_yesterday"], 0.2)
|
||||
self.assertEqual(row["conversion_rate"], round(10 / 120, 4))
|
||||
self.assertEqual(row["conversion_rate_yesterday"], round(8 / 100, 4))
|
||||
self.assertEqual(row["sales_7d_total"], 49)
|
||||
self.assertEqual(row["sales_forecast_7d"], 63)
|
||||
|
||||
def test_prompt_does_not_reuse_retired_dashboard_sections(self):
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], {"dashboard": {
|
||||
"rankings": {"sales": []},
|
||||
"platform_personas": [{"platform": "天猫"}],
|
||||
"daily_new_notes": [],
|
||||
"styles": [{"style_name": "蓝鹊", "total_note_exposure": 1000, "evidence_summary": "经营、画像、营销笔记"}],
|
||||
"definitions": {"conversion_rate": "销量/访客"},
|
||||
}})
|
||||
self.assertNotIn("按风格分组的综合经营矩阵", prompt)
|
||||
self.assertNotIn("平台高转化与高访客画像", prompt)
|
||||
self.assertNotIn("全款式经营看板摘要", prompt)
|
||||
|
||||
def test_report_normalization_and_all_style_validation(self):
|
||||
report = daily.normalize_report(
|
||||
"健康度:偏弱。36 个款式中\n蓝鹊同比增长79%\n宙斯环比下降16%",
|
||||
35,
|
||||
)
|
||||
self.assertNotIn("同比", report)
|
||||
self.assertIn("环比下降16%", report)
|
||||
self.assertIn("35个款式中", report)
|
||||
self.assertEqual(daily.normalize_report("销量 0 / 退款 0(无数据)"), "销量 0 / 退款 0(当日均为0)")
|
||||
cleaned = daily.normalize_report("投放尚未发布,带动效应快速衰减;待发布后核对")
|
||||
self.assertNotIn("未发布", cleaned)
|
||||
self.assertNotIn("带动效应", cleaned)
|
||||
self.assertIn("发布时间无数据", cleaned)
|
||||
self.assertNotIn("驱动访客", daily.normalize_report("该内容未驱动访客回升"))
|
||||
daily.validate_all_styles(report, [{"style_name": "蓝鹊"}, {"style_name": "宙斯"}])
|
||||
with self.assertRaisesRegex(RuntimeError, "缺少款式"):
|
||||
daily.validate_all_styles(report, [{"style_name": "白鹭"}])
|
||||
|
||||
def test_report_normalization_removes_database_and_code_style_language(self):
|
||||
raw = """数据已收到,我先用脚本做一遍聚合。
|
||||
# 品牌SKU电商运营日报
|
||||
|
||||
• 最佳渠道:无法判定。今日 `昨日新发布内容` 为空,cooperations 仅1条,publish_time=null、engagement_data_status=无数据。
|
||||
|
||||
5. 数据治理与内容补位:(a)对齐 publish_time;(b)修复互动数据;(c)补齐数据。
|
||||
|
||||
报告字数提示:如需压缩请告知。
|
||||
完整数据索引:如需导出 CSV 请告诉我。
|
||||
"""
|
||||
report = daily.normalize_report(raw)
|
||||
|
||||
self.assertTrue(report.startswith("# 品牌SKU电商运营日报"))
|
||||
for forbidden in ("我先用脚本", "cooperations", "publish_time", "engagement_data_status", "null", "`", "(a)", "报告字数提示", "请告诉我"):
|
||||
self.assertNotIn(forbidden, report)
|
||||
self.assertIn("达人合作记录", report)
|
||||
self.assertIn("发布时间无数据", report)
|
||||
self.assertIn("互动数据无数据", report)
|
||||
self.assertIn("(1)", report)
|
||||
|
||||
def test_system_prompt_forbids_database_fields_and_conversational_tail(self):
|
||||
for phrase in (
|
||||
"禁止输出数据库表名、字段名、JSON键名",
|
||||
"禁止出现反引号、null",
|
||||
"禁止描述生成过程",
|
||||
"禁止在结尾追问",
|
||||
):
|
||||
self.assertIn(phrase, daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_status_groups_are_replaced_with_complete_deterministic_lists(self):
|
||||
report = "二、种草追踪池\n\nB. 全量状态(固定4行)\n🔴(1):蓝鹊\n\n三、昨日笔记次日追踪"
|
||||
rows = [
|
||||
{"style_name": "蓝鹊", "light": "🔴"},
|
||||
{"style_name": "宙斯", "light": "🟡"},
|
||||
{"style_name": "白鹭", "light": "🟢"},
|
||||
{"style_name": "云卷2", "light": "无判断"},
|
||||
]
|
||||
result = daily.replace_status_groups(report, rows)
|
||||
self.assertIn("B. 全量状态(4款)", result)
|
||||
self.assertIn("🔴(1):蓝鹊", result)
|
||||
self.assertIn("🟡(1):宙斯", result)
|
||||
self.assertIn("🟢(1):白鹭", result)
|
||||
self.assertIn("无判断(1):云卷2", result)
|
||||
self.assertIn("三、昨日笔记次日追踪", result)
|
||||
|
||||
def test_status_groups_are_inserted_when_model_omits_marker(self):
|
||||
report = "一、核心经营结论\n正常\n\n三、达人投放与电商同期关联\n无数据"
|
||||
rows = [{"style_name": "蓝鹊", "light": "🔴"}, {"style_name": "白鹭", "light": "🟢"}]
|
||||
result = daily.replace_status_groups(report, rows, "C. 补充经营信号\n- 店铺大盘:无数据")
|
||||
self.assertIn("B. 全量状态(2款)", result)
|
||||
self.assertIn("🔴(1):蓝鹊", result)
|
||||
self.assertIn("🟢(1):白鹭", result)
|
||||
self.assertLess(result.index("B. 全量状态"), result.index("三、达人投放"))
|
||||
|
||||
def test_prompt_passes_available_enrichment_without_restoring_old_sections(self):
|
||||
enrichment = {
|
||||
"platform_metrics": [{"style_name": "蓝鹊", "platform": "天猫", "sales": 14}],
|
||||
"cooperations": [{"style_name": "蓝鹊", "content_direction": "通勤"}],
|
||||
"personas": [{"style_name": "蓝鹊", "gender_top": "女性用户 60%"}],
|
||||
"creatives": [{"style_name": "蓝鹊", "ctr": 4.2}],
|
||||
"reviews": [{"style_name": "蓝鹊", "sample_size": 6}],
|
||||
"shop_overview": [{"platform": "天猫", "week_end": "2026-07-12"}],
|
||||
}
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], enrichment)
|
||||
for source_key in ("platform_metrics", "cooperations", "personas", "creatives", "reviews", "shop_overview"):
|
||||
self.assertIn(source_key, prompt)
|
||||
self.assertIn("评论与补充经营数据", prompt)
|
||||
self.assertNotIn("补充经营信号", prompt)
|
||||
self.assertNotIn("全量状态", prompt)
|
||||
|
||||
def test_enrichment_section_validation_requires_available_categories(self):
|
||||
enrichment = {"personas": [{}], "creatives": [{}], "reviews": [{}], "shop_overview": [{}]}
|
||||
complete = "条件信号:人物画像;主图;商品评价;店铺大盘"
|
||||
daily.validate_enrichment_sections(complete, enrichment)
|
||||
with self.assertRaisesRegex(RuntimeError, "主图"):
|
||||
daily.validate_enrichment_sections("人物画像 商品评价 店铺大盘", enrichment)
|
||||
|
||||
def test_build_condition_signals_covers_all_available_sources(self):
|
||||
enrichment = {
|
||||
"candidate_styles": ["蓝鹊"],
|
||||
"personas": [{"style_name": "蓝鹊", "platform": "天猫", "data_date": "2026-07-15", "gender_top": "女性 60%"}],
|
||||
"creatives": [{"style_name": "蓝鹊", "platform": "天猫", "data_date": "2026-07-12", "impressions": 1000, "clicks": 50, "ctr": 5.0}],
|
||||
"reviews": [{"style_name": "蓝鹊", "sample_size": 6, "negative_count": 1, "negative_rate": 0.1667, "window_start": "2026-07-09", "window_end": "2026-07-15"}],
|
||||
"shop_overview": [{"platform": "天猫", "week_end": "2026-07-12", "visitors": 10000, "deal_amount": 20000}],
|
||||
}
|
||||
text = daily.build_condition_signals(enrichment)
|
||||
for phrase in ("C. 补充经营信号", "人物画像", "主图", "商品评价", "店铺大盘", "2026-07-12"):
|
||||
self.assertIn(phrase, text)
|
||||
|
||||
def test_persona_top_value_ignores_invalid_percentages(self):
|
||||
payload = {"buying_power": [{"name": "异常", "value": 133}, {"name": "L4", "value": 42}]}
|
||||
self.assertEqual(daily._top_persona_value(payload, "buying_power"), "L4 42.0%")
|
||||
|
||||
def test_enriched_report_rejects_missing_platform_claim_and_invented_new_product(self):
|
||||
daily.validate_enriched_report("平台拆分完整,均为同期合作")
|
||||
with self.assertRaisesRegex(RuntimeError, "平台拆分"):
|
||||
daily.validate_enriched_report("无平台拆分数据")
|
||||
with self.assertRaisesRegex(RuntimeError, "新品"):
|
||||
daily.validate_enriched_report("同期出现多个新品合作")
|
||||
|
||||
def test_platform_persona_section_uses_database_winners_not_model_text(self):
|
||||
report = ("### 三、平台人群与营销机会\n天猫高转化款为错误款式\n\n"
|
||||
"### 四、明日业务动作\n复查数据\n\n"
|
||||
"三、平台人群与营销机会\n重复错误款式")
|
||||
enrichment = {"dashboard": {
|
||||
"daily_views_status": "无数据:未保存每日增量快照",
|
||||
"platform_personas": [{
|
||||
"platform": "天猫",
|
||||
"top_conversion": {
|
||||
"style_name": "布谷", "conversion_rate": 0.0259, "visitors": 617,
|
||||
"persona_text": "无数据", "interest_hypothesis": "兴趣推测:无数据",
|
||||
"optimization": "先补采人群画像",
|
||||
},
|
||||
"top_visitors": {
|
||||
"style_name": "星云2", "conversion_rate": 0.0114, "visitors": 3504,
|
||||
"persona_text": "男性用户 62.7%", "interest_hypothesis": "兴趣推测:数码3C",
|
||||
"optimization": "优化详情页",
|
||||
},
|
||||
}],
|
||||
}}
|
||||
result = daily.replace_platform_persona_section(report, enrichment)
|
||||
self.assertIn("转化最高为布谷(转化率2.6%", result)
|
||||
self.assertIn("访客最高为星云2(访客3,504", result)
|
||||
self.assertIn("兴趣推测:数码3C", result)
|
||||
self.assertNotIn("错误款式", result)
|
||||
self.assertNotIn("重复错误款式", result)
|
||||
self.assertIn("四、明日业务动作", result)
|
||||
self.assertEqual(result.count("三、平台人群与营销机会"), 1)
|
||||
|
||||
def test_platform_persona_section_supports_new_daily_report_boundary(self):
|
||||
report = ("### 三、平台人群与营销机会\n模型内容\n\n"
|
||||
"### 四、今日重点SKU分析\n重点款内容\n")
|
||||
result = daily.replace_platform_persona_section(report, {"dashboard": {}})
|
||||
self.assertIn("三、平台人群与营销机会", result)
|
||||
self.assertIn("四、今日重点SKU分析", result)
|
||||
self.assertNotIn("模型内容", result)
|
||||
self.assertEqual(result.count("三、平台人群与营销机会"), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,26 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.catalog import WorkflowCatalog
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
class DailyMarketingReportScheduleTests(unittest.TestCase):
|
||||
def test_python_scheduler_runs_latest_daily_report(self):
|
||||
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
workflow = next(
|
||||
item
|
||||
for item in catalog.workflows
|
||||
if item.workflow_id == "content.marketing_report.daily"
|
||||
)
|
||||
schedule = catalog.schedule_for(workflow.workflow_id)
|
||||
|
||||
self.assertEqual(workflow.steps[0].entry, "daily_marketing_report.py")
|
||||
self.assertEqual(workflow.steps[0].args, ("--send",))
|
||||
self.assertEqual(schedule.kind, "daily")
|
||||
self.assertEqual(schedule.at, "10:00")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,202 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools.daily_report_card import (
|
||||
build_analysis_card,
|
||||
build_daily_report_card,
|
||||
build_dashboard_card,
|
||||
prepare_feishu_image,
|
||||
send_card_to_recipients,
|
||||
send_daily_report_cards,
|
||||
upload_dashboard_image,
|
||||
)
|
||||
|
||||
|
||||
class DailyReportCardTests(unittest.TestCase):
|
||||
def test_dashboard_and_analysis_are_separate_cards_with_full_text(self):
|
||||
report = """
|
||||
|
||||
第一部分 整体经营分析
|
||||
总销量下降15%。
|
||||
整体趋势判断:下滑状态。
|
||||
|
||||
第二部分 SKU分析排序
|
||||
1. 蓝鹊:销量第一。
|
||||
2. 星云2:转化领先。
|
||||
3. 宙斯:访客增长。
|
||||
4. 南斯:退款偏高。
|
||||
5. 凌云3:需要优化。
|
||||
6. 波塞冬:继续观察。
|
||||
|
||||
第三部分 关键预警与建议
|
||||
退款订单比上升,48小时内排查。
|
||||
"""
|
||||
rows = [{"style_name": "蓝鹊", "light": "🔴"}]
|
||||
|
||||
dashboard = build_dashboard_card("2026-07-17", rows, "img_v3_test")
|
||||
analysis = build_analysis_card("2026-07-17", report)
|
||||
|
||||
dashboard_text = json.dumps(dashboard, ensure_ascii=False)
|
||||
analysis_text = json.dumps(analysis, ensure_ascii=False)
|
||||
self.assertIn("img_v3_test", dashboard_text)
|
||||
self.assertNotIn("总销量下降15%", dashboard_text)
|
||||
self.assertNotIn("img_v3_test", analysis_text)
|
||||
self.assertNotIn("![全域种草数据看板]", analysis_text)
|
||||
self.assertIn("波塞冬:继续观察", analysis_text)
|
||||
self.assertIn("退款订单比上升", analysis_text)
|
||||
|
||||
def test_build_card_uses_card_2_schema_and_dashboard(self):
|
||||
report = """一、核心经营结论
|
||||
1. 波塞冬edge访客明显下滑。
|
||||
2. 蓝鹊销量同期上行。
|
||||
|
||||
二、产品经营分析
|
||||
略
|
||||
|
||||
四、明日业务动作
|
||||
1. 复查波塞冬edge天猫流量。
|
||||
2. 跟进蓝鹊投放承接。
|
||||
"""
|
||||
rows = [
|
||||
{"style_name": "波塞冬edge", "light": "🔴"},
|
||||
{"style_name": "宙斯", "light": "🟡"},
|
||||
{"style_name": "白鹭", "light": "🟢"},
|
||||
]
|
||||
card = build_daily_report_card("2026-07-15", report, rows, "img_v3_test")
|
||||
|
||||
self.assertEqual(card["schema"], "2.0")
|
||||
self.assertEqual(card["config"]["width_mode"], "fill")
|
||||
self.assertIn("Hermes", card["header"]["subtitle"]["content"])
|
||||
elements = card["body"]["elements"]
|
||||
self.assertEqual([item["tag"] for item in elements], ["column_set", "img", "markdown", "markdown"])
|
||||
self.assertEqual(elements[1]["img_key"], "img_v3_test")
|
||||
metrics_text = json.dumps(elements[0], ensure_ascii=False)
|
||||
self.assertIn("红灯款式", metrics_text)
|
||||
self.assertIn("1", metrics_text)
|
||||
self.assertNotIn("二、产品经营分析", json.dumps(card, ensure_ascii=False))
|
||||
|
||||
def test_prepare_feishu_image_resizes_to_supported_width(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
source = Path(tmp) / "source.png"
|
||||
target = Path(tmp) / "target.png"
|
||||
Image.new("RGB", (1600, 1000), "white").save(source)
|
||||
prepare_feishu_image(source, target)
|
||||
with Image.open(target) as image:
|
||||
self.assertEqual(image.width, 1500)
|
||||
self.assertLessEqual(image.height / image.width, 16 / 9)
|
||||
|
||||
def test_build_card_reads_new_management_report_headings(self):
|
||||
report = """## 1. 今日经营概览
|
||||
销售:↑12%,蓝鹊贡献主要增量。
|
||||
整体判断:增长状态。
|
||||
|
||||
## 2. SKU健康排名 TOP20
|
||||
略
|
||||
|
||||
## 6. 今日运营建议
|
||||
1. 商品动作:复核蓝鹊库存。
|
||||
2. 电商动作:优化宙斯详情页。
|
||||
"""
|
||||
card = build_daily_report_card("2026-07-17", report, [], "img_v3_test")
|
||||
card_text = json.dumps(card, ensure_ascii=False)
|
||||
self.assertIn("蓝鹊贡献主要增量", card_text)
|
||||
self.assertIn("复核蓝鹊库存", card_text)
|
||||
|
||||
def test_build_card_reads_fresh_partial_prompt_report_headings(self):
|
||||
report = """第一部分 整体经营分析
|
||||
总销量下降15%。
|
||||
整体趋势判断:下滑状态。
|
||||
|
||||
第二部分 SKU分析排序
|
||||
略
|
||||
|
||||
第三部分 关键预警与建议
|
||||
退款订单比上升,48小时内排查。
|
||||
追加增长型SKU预算。
|
||||
"""
|
||||
card = build_daily_report_card("2026-07-17", report, [], "img_v3_test")
|
||||
card_text = json.dumps(card, ensure_ascii=False)
|
||||
self.assertIn("总销量下降15%", card_text)
|
||||
self.assertIn("48小时内排查", card_text)
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.subprocess.run"
|
||||
)
|
||||
def test_upload_dashboard_uses_named_image_file_field(self, run):
|
||||
run.return_value = type("Result", (), {
|
||||
"returncode": 0, "stdout": json.dumps({"image_key": "img_v3_test"}), "stderr": ""
|
||||
})()
|
||||
key = upload_dashboard_image(Path("charts/dashboard.png"))
|
||||
self.assertEqual(key, "img_v3_test")
|
||||
command = run.call_args.args[0]
|
||||
self.assertEqual(command[command.index("--file") + 1], "image=./dashboard.png")
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.subprocess.run"
|
||||
)
|
||||
def test_send_card_to_all_recipients(self, run):
|
||||
run.side_effect = [
|
||||
type("Result", (), {"returncode": 0, "stdout": json.dumps({"message_id": f"om_{i}"}), "stderr": ""})()
|
||||
for i in range(3)
|
||||
]
|
||||
recipients = ["ou_one", "ou_two", "ou_three"]
|
||||
results = send_card_to_recipients({"schema": "2.0"}, recipients, "2026-07-15")
|
||||
|
||||
self.assertEqual([item["message_id"] for item in results], ["om_0", "om_1", "om_2"])
|
||||
self.assertEqual(run.call_count, 3)
|
||||
commands = [call.args[0] for call in run.call_args_list]
|
||||
self.assertEqual([cmd[cmd.index("--user-id") + 1] for cmd in commands], recipients)
|
||||
self.assertTrue(all("--msg-type" in cmd and "interactive" in cmd for cmd in commands))
|
||||
keys = [cmd[cmd.index("--idempotency-key") + 1] for cmd in commands]
|
||||
self.assertTrue(all(key.startswith("daily-2026-07-15-") for key in keys))
|
||||
self.assertTrue(all(len(key) <= 50 for key in keys))
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.send_card_to_recipients"
|
||||
)
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.upload_dashboard_image",
|
||||
return_value="img_v3_test",
|
||||
)
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.prepare_feishu_image"
|
||||
)
|
||||
def test_daily_report_sends_dashboard_then_analysis_as_two_messages_per_recipient(
|
||||
self, prepare, upload, send
|
||||
):
|
||||
send.side_effect = [
|
||||
[{"message_id": "om_board_1"}, {"message_id": "om_board_2"}],
|
||||
[{"message_id": "om_text_1"}, {"message_id": "om_text_2"}],
|
||||
]
|
||||
recipients = ["ou_one", "ou_two"]
|
||||
|
||||
results = send_daily_report_cards(
|
||||
"2026-07-17",
|
||||
"第一部分 整体经营分析\n完整分析内容",
|
||||
[{"light": "🟢"}],
|
||||
Path("charts/dashboard.png"),
|
||||
recipients,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[item["message_id"] for item in results],
|
||||
["om_board_1", "om_board_2", "om_text_1", "om_text_2"],
|
||||
)
|
||||
self.assertEqual(send.call_count, 2)
|
||||
first_card = send.call_args_list[0].args[0]
|
||||
second_card = send.call_args_list[1].args[0]
|
||||
self.assertIn("img_v3_test", json.dumps(first_card, ensure_ascii=False))
|
||||
self.assertIn("完整分析内容", json.dumps(second_card, ensure_ascii=False))
|
||||
self.assertEqual(send.call_args_list[0].kwargs["message_kind"], "dashboard")
|
||||
self.assertEqual(send.call_args_list[1].kwargs["message_kind"], "analysis")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,104 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools.daily_report_charts import (
|
||||
_split_style_groups,
|
||||
build_overview_cards,
|
||||
classify_style_action,
|
||||
generate_dashboard,
|
||||
select_chart_styles,
|
||||
)
|
||||
|
||||
|
||||
class DailyReportChartTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.rows = [
|
||||
{"style_name": "大盘下滑", "light": "🔴", "visitors": 1000, "avg_visitors_7d": 2000, "cart_users": 80, "avg_cart_users_7d": 100, "visitor_change": -0.5, "cart_change": -0.2, "sales_change": -0.4, "refund_change": 0.1},
|
||||
{"style_name": "正向放量", "light": "🔴", "visitors": 1500, "avg_visitors_7d": 1000, "cart_users": 180, "avg_cart_users_7d": 100, "visitor_change": 0.5, "cart_change": 0.8, "sales_change": 0.3, "refund_change": -0.1},
|
||||
{"style_name": "小基数", "light": "🔴", "visitors": 20, "avg_visitors_7d": 100, "cart_users": 1, "avg_cart_users_7d": 2, "visitor_change": -0.8, "cart_change": -0.5, "sales_change": None, "refund_change": None},
|
||||
{"style_name": "黄灯A", "light": "🟡", "visitors": 800, "avg_visitors_7d": 1000, "cart_users": 70, "avg_cart_users_7d": 80, "visitor_change": -0.2, "cart_change": -0.125, "sales_change": -0.1, "refund_change": 0.2},
|
||||
{"style_name": "绿灯", "light": "🟢", "visitors": 500, "avg_visitors_7d": 510, "cart_users": 40, "avg_cart_users_7d": 41, "visitor_change": -0.02, "cart_change": -0.02, "sales_change": 0.0, "refund_change": 0.0},
|
||||
]
|
||||
|
||||
def test_selection_covers_negative_positive_and_small_base(self):
|
||||
names = [row["style_name"] for row in select_chart_styles(self.rows, limit=4)]
|
||||
self.assertIn("大盘下滑", names)
|
||||
self.assertIn("正向放量", names)
|
||||
self.assertIn("小基数", names)
|
||||
|
||||
def test_generate_dashboard_writes_png(self):
|
||||
enrichment = {"platform_metrics": [
|
||||
{"style_name": "大盘下滑", "platform": "天猫", "visitors": 700},
|
||||
{"style_name": "大盘下滑", "platform": "京东", "visitors": 300},
|
||||
]}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "dashboard.png"
|
||||
generate_dashboard("2026-07-15", self.rows, enrichment, path)
|
||||
self.assertTrue(path.exists())
|
||||
self.assertGreater(path.stat().st_size, 10_000)
|
||||
with Image.open(path) as image:
|
||||
self.assertEqual(image.size, (1600, 2500))
|
||||
|
||||
def test_style_groups_are_split_into_balanced_category_columns(self):
|
||||
groups = [
|
||||
{"style_category": "都市机能", "styles": [{}] * 12},
|
||||
{"style_category": "户外机能", "styles": [{}] * 3},
|
||||
{"style_category": "都市新贵", "styles": [{}] * 3},
|
||||
{"style_category": "智性通勤", "styles": [{}] * 14},
|
||||
{"style_category": "都市新旅", "styles": [{}]},
|
||||
{"style_category": "都市运动", "styles": [{}] * 3},
|
||||
]
|
||||
left, right = _split_style_groups(groups)
|
||||
self.assertEqual([group["style_category"] for group in left], ["都市机能", "户外机能", "都市新贵"])
|
||||
self.assertEqual([group["style_category"] for group in right], ["智性通勤", "都市新旅", "都市运动"])
|
||||
|
||||
groups.append({"style_category": "未分类", "styles": [{}] * 2})
|
||||
left, right = _split_style_groups(groups)
|
||||
self.assertEqual(sum(len(group["styles"]) for group in left), 18)
|
||||
self.assertEqual(sum(len(group["styles"]) for group in right), 20)
|
||||
|
||||
def test_overview_compares_today_yesterday_and_seven_day_average(self):
|
||||
rows = [
|
||||
{
|
||||
"style_name": "蓝鹊", "sales": 12, "sales_yesterday": 10, "avg_sales_7d": 8,
|
||||
"visitors": 120, "visitors_yesterday": 100, "avg_visitors_7d": 90,
|
||||
},
|
||||
{
|
||||
"style_name": "宙斯", "sales": 18, "sales_yesterday": 20, "avg_sales_7d": 15,
|
||||
"visitors": 180, "visitors_yesterday": 200, "avg_visitors_7d": 150,
|
||||
},
|
||||
]
|
||||
cards = build_overview_cards(rows, {"daily_new_notes": [{"view_count": 515}]})
|
||||
sales = next(card for card in cards if card["key"] == "sales")
|
||||
conversion = next(card for card in cards if card["key"] == "conversion")
|
||||
content = next(card for card in cards if card["key"] == "content")
|
||||
|
||||
self.assertEqual((sales["today"], sales["yesterday"], sales["baseline_7d"]), (30, 30, 23))
|
||||
self.assertEqual(conversion["today"], 0.1)
|
||||
self.assertEqual(content["today"], 515)
|
||||
self.assertIsNone(content["baseline_7d"])
|
||||
|
||||
def test_style_action_matches_prompt_growth_opportunity_and_risk_rules(self):
|
||||
growth = classify_style_action({
|
||||
"sales_change_yesterday": 0.2, "visitor_change_yesterday": 0.1,
|
||||
"conversion_change_yesterday": 0.001, "refund_change_yesterday": -0.1,
|
||||
})
|
||||
opportunity = classify_style_action({
|
||||
"sales_change_yesterday": -0.1, "visitor_change_yesterday": 0.2,
|
||||
"conversion_change_yesterday": -0.01, "refund_change_yesterday": 0,
|
||||
})
|
||||
risk = classify_style_action({
|
||||
"sales_change_yesterday": -0.2, "visitor_change_yesterday": -0.1,
|
||||
"conversion_change_yesterday": -0.01, "refund_change_yesterday": 0.4,
|
||||
})
|
||||
|
||||
self.assertEqual(growth, ("增长型", "加达人/备货"))
|
||||
self.assertEqual(opportunity, ("流量机会", "优化详情页"))
|
||||
self.assertEqual(risk, ("风险型", "查退款/控投放"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,137 @@
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
friday_relogin_parallel as relogin,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools import relogin_transaction
|
||||
|
||||
|
||||
def test_five_expected_platforms_are_configured():
|
||||
assert set(relogin.PLATFORMS) == {
|
||||
"bilibili", "douyin", "pgy", "xingtu", "xiaohongshu"
|
||||
}
|
||||
assert relogin.PLATFORMS["xingtu"].required_cookie == "sessionid"
|
||||
assert relogin.PLATFORMS["douyin"].required_cookie == "sessionid"
|
||||
|
||||
|
||||
def test_manual_relogin_can_select_only_xingtu():
|
||||
assert relogin.resolve_platform_names(["xingtu"]) == ["xingtu"]
|
||||
assert relogin.resolve_platform_names(None) == list(relogin.PLATFORMS)
|
||||
|
||||
|
||||
def test_default_screenshot_recipients_are_routed_by_platform():
|
||||
assert relogin.DEFAULT_RECIPIENTS == {
|
||||
"pgy": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
"xingtu": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
"douyin": "ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
"bilibili": "ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
"xiaohongshu": "ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
}
|
||||
assert relogin.recipient_for_platform("pgy") == "ou_24cc944d6e43c69c59d6560ad4e2ae6e"
|
||||
assert relogin.recipient_for_platform("douyin") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
|
||||
assert relogin.recipient_for_platform("bilibili") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
|
||||
assert relogin.recipient_for_platform("pgy", "ou_override") == "ou_override"
|
||||
|
||||
|
||||
def test_failed_platforms_are_grouped_by_their_responsible_recipient():
|
||||
grouped = relogin.group_platforms_by_recipient(
|
||||
["pgy", "douyin", "bilibili", "xingtu", "xiaohongshu"]
|
||||
)
|
||||
assert grouped == {
|
||||
"ou_24cc944d6e43c69c59d6560ad4e2ae6e": ["pgy", "xingtu"],
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447": [
|
||||
"douyin", "bilibili", "xiaohongshu"
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_login_windows_are_classified_by_title():
|
||||
assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili"
|
||||
assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy"
|
||||
assert relogin._platform_from_window_title("抖音授权 - Google Chrome") == "xingtu"
|
||||
assert relogin._platform_from_window_title("抖音 - 记录美好生活") == "douyin"
|
||||
assert relogin._platform_from_window_title("小红书 - 你的生活兴趣社区") == "xiaohongshu"
|
||||
assert relogin._platform_from_window_title("Codex") is None
|
||||
|
||||
|
||||
def test_retry_only_failed_platforms_for_at_most_three_rounds():
|
||||
calls = []
|
||||
|
||||
def fake_round(names, attempt):
|
||||
calls.append((tuple(names), attempt))
|
||||
if attempt == 1:
|
||||
return {name: name != "xingtu" for name in names}
|
||||
if attempt == 2:
|
||||
return {name: False for name in names}
|
||||
return {name: True for name in names}
|
||||
|
||||
result = relogin.run_with_retries(
|
||||
["bilibili", "pgy", "xingtu", "xiaohongshu"],
|
||||
max_attempts=3,
|
||||
run_round=fake_round,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(("bilibili", "pgy", "xingtu", "xiaohongshu"), 1),
|
||||
(("xingtu",), 2),
|
||||
(("xingtu",), 3),
|
||||
]
|
||||
assert all(result.values())
|
||||
|
||||
|
||||
def test_stops_after_third_failed_round():
|
||||
calls = []
|
||||
|
||||
def always_fail(names, attempt):
|
||||
calls.append(attempt)
|
||||
return {name: False for name in names}
|
||||
|
||||
result = relogin.run_with_retries(
|
||||
["pgy"], max_attempts=3, run_round=always_fail
|
||||
)
|
||||
|
||||
assert calls == [1, 2, 3]
|
||||
assert result == {"pgy": False}
|
||||
|
||||
|
||||
def test_timeout_terminates_the_complete_process_tree(monkeypatch):
|
||||
calls = []
|
||||
|
||||
class Process:
|
||||
pid = 1234
|
||||
|
||||
def terminate(self):
|
||||
calls.append("fallback-terminate")
|
||||
|
||||
def wait(self, timeout):
|
||||
calls.append(("wait", timeout))
|
||||
|
||||
monkeypatch.setattr(
|
||||
relogin.subprocess,
|
||||
"run",
|
||||
lambda command, **kwargs: calls.append((command, kwargs)),
|
||||
)
|
||||
|
||||
relogin._terminate_process_tree(Process())
|
||||
|
||||
assert calls[0][0] == ["taskkill", "/PID", "1234", "/T", "/F"]
|
||||
assert calls[1] == ("wait", 10)
|
||||
|
||||
|
||||
def test_transaction_rolls_back_files_and_profile(tmp_path):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
state = relogin_transaction.begin((cookie, storage), profile)
|
||||
cookie.write_text("partial", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
relogin_transaction.rollback(state)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
@@ -0,0 +1,367 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.catalog import WorkflowCatalog
|
||||
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
from gyxx_flow.modules.content_marketing.creator_task_grouping import (
|
||||
chunk_creator_groups,
|
||||
group_tasks_by_creator,
|
||||
)
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _workflow(workflow_id):
|
||||
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
return next(item for item in catalog.workflows if item.workflow_id == workflow_id)
|
||||
|
||||
|
||||
def test_group_tasks_uses_creator_id_and_merges_missing_id_by_unique_name():
|
||||
tasks = [
|
||||
{"creator_name": "同一达人", "creator_id": " 123 ", "record_id": "a"},
|
||||
{"creator_name": "同一达人 ", "creator_id": None, "record_id": "b"},
|
||||
{"creator_name": "另一个达人", "creator_id": None, "record_id": "c"},
|
||||
]
|
||||
|
||||
groups = group_tasks_by_creator(tasks)
|
||||
|
||||
assert len(groups) == 2
|
||||
assert groups[0]["creator_id"] == "123"
|
||||
assert [task["record_id"] for task in groups[0]["tasks"]] == ["a", "b"]
|
||||
|
||||
|
||||
def test_group_tasks_does_not_guess_when_same_name_has_conflicting_ids():
|
||||
tasks = [
|
||||
{"creator_name": "重名达人", "creator_id": "1", "record_id": "a"},
|
||||
{"creator_name": "重名达人", "creator_id": "2", "record_id": "b"},
|
||||
{"creator_name": "重名达人", "creator_id": None, "record_id": "c"},
|
||||
]
|
||||
|
||||
groups = group_tasks_by_creator(tasks)
|
||||
|
||||
assert len(groups) == 3
|
||||
|
||||
|
||||
def test_zero_batch_size_means_one_unlimited_batch():
|
||||
groups = [{"creator_name": str(i), "tasks": []} for i in range(3)]
|
||||
assert chunk_creator_groups(groups, 0) == [groups]
|
||||
|
||||
|
||||
def _assert_cross_style_collection(module, monkeypatch):
|
||||
styles = [
|
||||
{"index": 1, "name": "款式A", "base_token": "base-a", "table_id": "table-a", "field_map": {"a": 1}},
|
||||
{"index": 2, "name": "款式B", "base_token": "base-b", "table_id": "table-b", "field_map": {"b": 2}},
|
||||
]
|
||||
|
||||
def fake_extract(style, only_record_ids):
|
||||
return [{
|
||||
"record_id": f"record-{style['index']}",
|
||||
"creator_name": "同一达人",
|
||||
"creator_id": "creator-1",
|
||||
"target_title": f"标题{style['index']}",
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(module, "extract_target_tasks", fake_extract)
|
||||
tasks, summaries = module.collect_tasks_across_styles(styles, None)
|
||||
|
||||
assert len(tasks) == 2
|
||||
assert len(group_tasks_by_creator(tasks)) == 1
|
||||
assert tasks[0]["style_context"]["table_id"] == "table-a"
|
||||
assert tasks[1]["style_context"]["table_id"] == "table-b"
|
||||
assert summaries[1]["total"] == 1
|
||||
assert summaries[2]["total"] == 1
|
||||
|
||||
|
||||
def test_pgy_collects_tasks_across_styles(monkeypatch):
|
||||
_assert_cross_style_collection(pgy, monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_collects_tasks_across_styles(monkeypatch):
|
||||
_assert_cross_style_collection(xingtu, monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_record_scope_is_applied_per_style(monkeypatch):
|
||||
styles = [
|
||||
{"index": 1, "name": "款式A", "base_token": "a", "table_id": "a", "field_map": {}},
|
||||
{"index": 2, "name": "款式B", "base_token": "b", "table_id": "b", "field_map": {}},
|
||||
]
|
||||
observed = {}
|
||||
|
||||
def fake_extract(style, only_record_ids):
|
||||
observed[style["index"]] = only_record_ids
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(xingtu, "extract_target_tasks", fake_extract)
|
||||
|
||||
xingtu.collect_tasks_across_styles(styles, {1: {"same-id"}, 2: {"other-id"}})
|
||||
|
||||
assert observed == {1: {"same-id"}, 2: {"other-id"}}
|
||||
|
||||
|
||||
def test_xingtu_load_retry_scope_keeps_style_and_record_pair(tmp_path):
|
||||
summary_path = tmp_path / "summary.json"
|
||||
summary_path.write_text(json.dumps([
|
||||
{"index": 1, "results": [
|
||||
{"record_id": "same-id", "reason": "creator_search_circuit_open"},
|
||||
]},
|
||||
{"index": 2, "results": [
|
||||
{"record_id": "same-id", "reason": "title_unmatched"},
|
||||
{"record_id": "other-id", "reason": "creator_search_circuit_open"},
|
||||
]},
|
||||
]), encoding="utf-8")
|
||||
|
||||
scope = xingtu.load_retry_scope(summary_path, "creator_search_circuit_open")
|
||||
|
||||
assert scope == {1: {"same-id"}, 2: {"other-id"}}
|
||||
|
||||
|
||||
def _assert_matched_task_uses_own_writeback_context(module, metric_key, monkeypatch):
|
||||
calls = []
|
||||
task = {
|
||||
"record_id": "record-b",
|
||||
"publish_time": object(),
|
||||
"style_context": {
|
||||
"index": 2,
|
||||
"base_token": "base-b",
|
||||
"table_id": "table-b",
|
||||
"field_map": {"slot": "style-b-slot"},
|
||||
},
|
||||
}
|
||||
summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []}
|
||||
monkeypatch.setattr(module, "pick_read_field", lambda fmap, publish_time: ("read_count_7d", "field-b", "7天曝光量"))
|
||||
monkeypatch.setattr(module, "write_back", lambda *args: calls.append(args) or True)
|
||||
|
||||
module.apply_matched_task(task, {metric_key: "1.2万", "title": "命中标题"}, summary, dry_run=False)
|
||||
|
||||
assert calls == [("base-b", "table-b", "record-b", "field-b", 12000)]
|
||||
assert summary["matched"] == 1
|
||||
assert summary["filled"] == 1
|
||||
|
||||
|
||||
def test_pgy_match_writes_to_task_style_context(monkeypatch):
|
||||
_assert_matched_task_uses_own_writeback_context(pgy, "read_count", monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_match_writes_to_task_style_context(monkeypatch):
|
||||
_assert_matched_task_uses_own_writeback_context(xingtu, "play_count", monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_search_and_open_creator_submits_only_one_search(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fail_search(_page, creator_name, creator_id):
|
||||
calls.append((creator_name, creator_id))
|
||||
raise TimeoutError("no result")
|
||||
|
||||
monkeypatch.setattr(xingtu, "search_creator", fail_search)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"open_creator_detail",
|
||||
lambda *_args, **_kwargs: pytest.fail("detail must not open after search failure"),
|
||||
)
|
||||
|
||||
with pytest.raises(TimeoutError, match="no result"):
|
||||
xingtu.search_and_open_creator_once(object(), "同一达人", "creator-1")
|
||||
|
||||
assert calls == [("同一达人", "creator-1")]
|
||||
|
||||
|
||||
def test_xingtu_search_budget_blocks_duplicate_without_failure_circuit():
|
||||
budget = xingtu.CreatorSearchBudget(max_consecutive_failures=3)
|
||||
|
||||
assert budget.begin(("id", "1")) is True
|
||||
assert budget.begin(("id", "1")) is False
|
||||
budget.failed("first")
|
||||
assert budget.begin(("id", "2")) is True
|
||||
budget.failed("second")
|
||||
assert budget.begin(("id", "3")) is True
|
||||
assert budget.failed("third") is False
|
||||
assert budget.begin(("id", "4")) is True
|
||||
assert budget.calls == 4
|
||||
assert budget.circuit_reason is None
|
||||
|
||||
budget.exhaust("search_quota_exhausted")
|
||||
assert budget.begin(("id", "5")) is False
|
||||
|
||||
|
||||
def test_xingtu_wait_for_results_detects_search_quota_exhausted():
|
||||
class Page:
|
||||
def evaluate(self, *_args):
|
||||
return "quota"
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pytest.fail("quota detection must return immediately")
|
||||
|
||||
with pytest.raises(xingtu.SearchQuotaExhausted):
|
||||
xingtu.wait_for_visible_creator_result(Page(), "达人", timeout_ms=1000)
|
||||
|
||||
|
||||
def test_xingtu_missing_creation_tab_is_not_treated_as_global_block():
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return "no_creation_tab"
|
||||
|
||||
assert xingtu.is_blocked(Page()) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reason", ["verify_text", "verify_iframe", "verify_div"])
|
||||
def test_xingtu_explicit_verification_still_opens_global_block(reason):
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return reason
|
||||
|
||||
assert xingtu.is_blocked(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_missing_creation_tab_falls_back_to_current_creator_page():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def click(self, **_kwargs):
|
||||
raise TimeoutError("missing tab")
|
||||
|
||||
class Page:
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
def evaluate(self, _script):
|
||||
return False
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
assert xingtu.open_creation_ability(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_current_creator_page_cards_are_used_as_fallback(monkeypatch):
|
||||
class Page:
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: [{
|
||||
"title": "MacBook党必看!提升幸福感的开工好物",
|
||||
"play_count": "1.2万",
|
||||
"note_id": "",
|
||||
}],
|
||||
)
|
||||
monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda _page, _title: False)
|
||||
|
||||
found, _candidates, page_limit_hit = xingtu.find_videos_for_tasks(
|
||||
Page(),
|
||||
[{
|
||||
"record_id": "record-1",
|
||||
"target_title": "MacBook党必看!提升幸福感的开工好物",
|
||||
}],
|
||||
)
|
||||
|
||||
assert found["record-1"]["play_count"] == "1.2万"
|
||||
assert found["record-1"]["match_method"] == "title"
|
||||
assert page_limit_hit is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
"BrowserContext.new_page: Protocol error (Target.createTarget): Failed to open a new tab",
|
||||
"Protocol error (Target.createTarget)",
|
||||
],
|
||||
)
|
||||
def test_xingtu_new_tab_creation_failure_rebuilds_browser_session(message):
|
||||
assert xingtu.is_browser_session_lost(RuntimeError(message)) is True
|
||||
|
||||
|
||||
def test_scheduled_collection_has_no_automatic_retry_and_syncs_once():
|
||||
steps = _workflow("content.metrics.daily").steps
|
||||
entries = [step.entry for step in steps]
|
||||
|
||||
assert "data/tools/retry_failed.py" not in entries
|
||||
assert entries.count("data/tools/sync_metrics_to_cmt_notes.py") == 1
|
||||
assert entries.index("run_all.py") < entries.index(
|
||||
"data/tools/sync_metrics_to_cmt_notes.py"
|
||||
)
|
||||
|
||||
|
||||
def test_daily_collection_uses_dedicated_self_operated_pipeline():
|
||||
steps = _workflow("content.metrics.daily").steps
|
||||
entries = [step.entry for step in steps]
|
||||
run_step = next(step for step in steps if step.entry == "run_all.py")
|
||||
|
||||
assert "--daily-scope" in run_step.args
|
||||
assert "--include-self-operated" not in run_step.args
|
||||
assert "data/tools/refresh_self_mapping.py" in entries
|
||||
assert "self_bilibili_scraper.py" in entries
|
||||
assert "chanmama_scraper.py" in entries
|
||||
refresh_index = entries.index("data/tools/refresh_self_mapping.py")
|
||||
bili_index = entries.index("self_bilibili_scraper.py")
|
||||
douyin_index = entries.index("chanmama_scraper.py")
|
||||
assert refresh_index < bili_index < douyin_index
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "entry_url", "error_type", "operation"),
|
||||
[
|
||||
(pgy, pgy.HOME_URL, pgy.AcceptanceCookieSkip, "content.pugongying.login"),
|
||||
(
|
||||
xingtu,
|
||||
xingtu.MARKET_URL,
|
||||
xingtu.AcceptanceCookieSkip,
|
||||
"content.xingtu.login",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_acceptance_invalid_session_skips_before_qr_login(
|
||||
module,
|
||||
entry_url,
|
||||
error_type,
|
||||
operation,
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
evidence_file = tmp_path / "evidence.jsonl"
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_FEISHU_TABLE_WRITE_DISABLED", "1")
|
||||
monkeypatch.setenv("GYXX_COOKIE_INVALID_SKIP", "1")
|
||||
monkeypatch.setenv(
|
||||
"GYXX_NOTIFICATION_RECIPIENT_OPEN_ID",
|
||||
"ou_8ee224968aa26a74c7d30ba27fed5eeb",
|
||||
)
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence_file))
|
||||
monkeypatch.setattr(module, "is_logged_in", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"wait_for_scan_login",
|
||||
lambda *_args, **_kwargs: pytest.fail("QR login must remain disabled"),
|
||||
)
|
||||
|
||||
class Page:
|
||||
def __init__(self):
|
||||
self.urls = []
|
||||
|
||||
def goto(self, url, **_kwargs):
|
||||
self.urls.append(url)
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
page = Page()
|
||||
with pytest.raises(error_type):
|
||||
module.ensure_login(page, login_timeout=30)
|
||||
|
||||
assert page.urls == [entry_url]
|
||||
records = [
|
||||
json.loads(line)
|
||||
for line in evidence_file.read_text(encoding="utf-8").splitlines()
|
||||
]
|
||||
assert records[-1]["event"] == "cookie_skipped"
|
||||
assert records[-1]["operation"] == operation
|
||||
assert records[-1]["details"]["status"] == "SKIPPED_COOKIE"
|
||||
@@ -0,0 +1,367 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import (
|
||||
pgy_xhs_scraper_v2,
|
||||
xingtu_scraper_v2,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
relogin_douyin,
|
||||
relogin_pgy,
|
||||
relogin_xingtu,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "state.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
assert not cookie.exists()
|
||||
assert not profile.exists()
|
||||
|
||||
cookie.write_text("partial-new-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
module.restore_previous_state(cookie_backup, profile_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
cookie.write_text("new-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
module.finish_successful_relogin(profile_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "new-cookie"
|
||||
assert cookie_backup.exists()
|
||||
assert not profile_backup.exists()
|
||||
|
||||
|
||||
def test_xingtu_scan_login_opens_customer_sso_and_clicks_douyin():
|
||||
class FakeLocator:
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
self.clicked = False
|
||||
self.evaluated = False
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def is_visible(self):
|
||||
return True
|
||||
|
||||
def click(self, timeout):
|
||||
self.clicked = True
|
||||
|
||||
def evaluate(self, script):
|
||||
assert script == "e => e.click()"
|
||||
self.evaluated = True
|
||||
|
||||
class FakePage:
|
||||
url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market"
|
||||
|
||||
def __init__(self):
|
||||
self.goto_url = None
|
||||
self.selectors = []
|
||||
self.douyin = FakeLocator()
|
||||
|
||||
def goto(self, url, **kwargs):
|
||||
self.goto_url = url
|
||||
self.url = url
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
def locator(self, selector):
|
||||
self.selectors.append(selector)
|
||||
return self.douyin
|
||||
|
||||
def get_by_text(self, _pattern):
|
||||
return FakeLocator()
|
||||
|
||||
page = FakePage()
|
||||
xingtu_scraper_v2.open_scan_login(page)
|
||||
|
||||
assert page.goto_url == xingtu_scraper_v2.CUSTOMER_LOGIN_URL
|
||||
assert 'img[src*="aweme.png"]' in page.selectors
|
||||
assert page.douyin.clicked
|
||||
|
||||
|
||||
def test_xingtu_index_page_is_recognized_as_logged_in():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def wait_for(self, timeout):
|
||||
assert timeout == 1200
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/ad/creator/index"
|
||||
|
||||
def get_by_text(self, _text, exact=False):
|
||||
return Locator()
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie():
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [{"name": "sessionid", "value": "stale"}]
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market"
|
||||
context = Context()
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
raise AssertionError("public homepage must be rejected before marker checks")
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_market_page_without_business_ui_rejects_stale_cookie():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
raise xingtu_scraper_v2.PlaywrightTimeoutError("missing")
|
||||
|
||||
def count(self):
|
||||
return 0
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [{"name": "sessionid", "value": "stale"}]
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/ad/creator/market"
|
||||
context = Context()
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
def locator(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_cookie_validation_requires_real_session_cookie(tmp_path):
|
||||
cookie = tmp_path / "xingtu.json"
|
||||
cookie.write_text('[{"name":"other","value":"1"}]', encoding="utf-8")
|
||||
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is False
|
||||
|
||||
cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8")
|
||||
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is True
|
||||
|
||||
|
||||
def test_xingtu_login_only_does_not_report_success_when_page_action_was_swallowed(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
cookie = tmp_path / "xingtu.json"
|
||||
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
|
||||
|
||||
class Session:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def fetch(self, *_args, **_kwargs):
|
||||
# Scrapling can log a page_action error and still return a response.
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session())
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None)
|
||||
|
||||
assert xingtu_scraper_v2.login_only(1) == 1
|
||||
|
||||
|
||||
def test_xingtu_login_only_verifies_persisted_profile_after_oauth_context_closes(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
cookie = tmp_path / "xingtu.json"
|
||||
|
||||
class Session:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def fetch(self, *_args, **_kwargs):
|
||||
raise RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session())
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None)
|
||||
|
||||
def verify():
|
||||
cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8")
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "verify_persisted_profile_login", verify)
|
||||
|
||||
assert xingtu_scraper_v2.login_only(1) == 0
|
||||
|
||||
|
||||
def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monkeypatch):
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.pages = []
|
||||
|
||||
def new_page(self):
|
||||
raise AssertionError("a surviving replacement page should be reused")
|
||||
|
||||
class Page:
|
||||
def __init__(self, context, url, *, closes_on_wait=False):
|
||||
self.context = context
|
||||
self.url = url
|
||||
self.closed = False
|
||||
self.closes_on_wait = closes_on_wait
|
||||
|
||||
def is_closed(self):
|
||||
return self.closed
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
if self.closes_on_wait:
|
||||
self.closed = True
|
||||
|
||||
def goto(self, url, **_kwargs):
|
||||
self.url = url
|
||||
|
||||
context = Context()
|
||||
qr_page = Page(
|
||||
context,
|
||||
"https://open.douyin.com/platform/oauth/pc/auth",
|
||||
closes_on_wait=True,
|
||||
)
|
||||
market_page = Page(context, xingtu_scraper_v2.MARKET_URL)
|
||||
context.pages = [qr_page, market_page]
|
||||
saved = []
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None)
|
||||
monkeypatch.setattr(
|
||||
xingtu_scraper_v2, "is_scan_qr_page", lambda page: page is qr_page
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
xingtu_scraper_v2, "is_logged_in", lambda page: page is market_page
|
||||
)
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page))
|
||||
|
||||
result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2)
|
||||
|
||||
assert result is market_page
|
||||
assert saved == [market_page]
|
||||
|
||||
|
||||
def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch):
|
||||
cookie = tmp_path / "xingtu_cookies.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = relogin_xingtu.reset_state()
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
assert relogin_xingtu.recover_interrupted_state() is True
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
assert cookie_backup is not None
|
||||
assert profile_backup is not None
|
||||
|
||||
|
||||
def test_pgy_scan_login_clicks_qr_switch_image():
|
||||
class FakeLocator:
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
self.clicked = False
|
||||
self.evaluated = False
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def is_visible(self):
|
||||
return True
|
||||
|
||||
def click(self, timeout):
|
||||
self.clicked = True
|
||||
|
||||
def evaluate(self, script):
|
||||
assert script == "e => e.click()"
|
||||
self.evaluated = True
|
||||
|
||||
class FakePage:
|
||||
def __init__(self):
|
||||
self.selectors = []
|
||||
self.qr_switch = FakeLocator()
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
def get_by_text(self, _pattern):
|
||||
return FakeLocator()
|
||||
|
||||
def locator(self, selector):
|
||||
self.selectors.append(selector)
|
||||
return self.qr_switch
|
||||
|
||||
page = FakePage()
|
||||
pgy_xhs_scraper_v2.open_scan_login(page)
|
||||
|
||||
assert 'img[src*="qr_code"]' in page.selectors
|
||||
assert page.qr_switch.evaluated
|
||||
|
||||
|
||||
def test_failed_douyin_relogin_restores_cookie_storage_and_profile(tmp_path, monkeypatch):
|
||||
cookie = tmp_path / "douyin_cookies.json"
|
||||
storage = tmp_path / "douyin_storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(relogin_douyin, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(relogin_douyin, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(relogin_douyin, "PROFILE_DIR", profile)
|
||||
monkeypatch.setattr(relogin_douyin.subprocess, "call", lambda *_args, **_kwargs: 1)
|
||||
monkeypatch.setattr(relogin_douyin.sys, "argv", ["relogin_douyin.py"])
|
||||
|
||||
assert relogin_douyin.main() == 1
|
||||
assert cookie.read_text(encoding="utf-8") == '[{"name":"sessionid","value":"old"}]'
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
@@ -0,0 +1,84 @@
|
||||
import io
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.adapters import RuntimeServicePolicy
|
||||
from gyxx_flow.modules.content_marketing.data.tools import db
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
sync_metrics_to_cmt_notes as sync_metrics,
|
||||
)
|
||||
|
||||
ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "content_marketing"
|
||||
)
|
||||
|
||||
|
||||
class LocalDatabaseConfigTests(unittest.TestCase):
|
||||
def test_config_requires_every_final_connection_value(self):
|
||||
complete = {
|
||||
"PG_HOST": "127.0.0.1",
|
||||
"PG_PORT": "5432",
|
||||
"PG_DB": "gyxx_super_data",
|
||||
"PG_USER": "gyxx_flow",
|
||||
"PG_PASSWORD": "secret",
|
||||
}
|
||||
self.assertEqual(
|
||||
db.database_config_from_env(complete),
|
||||
{
|
||||
"host": "127.0.0.1",
|
||||
"port": 5432,
|
||||
"dbname": "gyxx_super_data",
|
||||
"user": "gyxx_flow",
|
||||
"password": "secret",
|
||||
},
|
||||
)
|
||||
for missing in complete:
|
||||
values = {key: value for key, value in complete.items() if key != missing}
|
||||
with self.subTest(missing=missing), self.assertRaisesRegex(RuntimeError, missing):
|
||||
db.database_config_from_env(values)
|
||||
|
||||
def test_runtime_policy_uses_explicit_cloud_postgres(self):
|
||||
environment = RuntimeServicePolicy().apply(
|
||||
{
|
||||
"PG_HOST": "db.example.com",
|
||||
"PG_PORT": "5432",
|
||||
"PG_DB": "data_hub",
|
||||
"PG_USER": "data_hub",
|
||||
"GYXX_POSTGRES_PASSWORD": "secret",
|
||||
}
|
||||
)
|
||||
config = db.database_config_from_env(environment)
|
||||
self.assertEqual(config["host"], "db.example.com")
|
||||
self.assertEqual(config["dbname"], "data_hub")
|
||||
self.assertEqual(config["user"], "data_hub")
|
||||
|
||||
def test_database_scripts_do_not_define_remote_connection_defaults(self):
|
||||
paths = (
|
||||
ROOT / "data" / "tools" / "db.py",
|
||||
ROOT / "data" / "tools" / "sync_metrics_to_cmt_notes.py",
|
||||
ROOT / "data" / "tools" / "sync_cooperations.py",
|
||||
ROOT / "data" / "tools" / "generate_creator_report.py",
|
||||
)
|
||||
for path in paths:
|
||||
source = path.read_text(encoding="utf-8-sig")
|
||||
with self.subTest(path=path.name):
|
||||
self.assertNotIn("8.148.185.119", source)
|
||||
self.assertNotIn("data_hub", source)
|
||||
|
||||
def test_sync_console_replaces_characters_unsupported_by_gbk(self):
|
||||
raw = io.BytesIO()
|
||||
stream = io.TextIOWrapper(raw, encoding="gbk", errors="strict")
|
||||
|
||||
sync_metrics.configure_console_output(stream, None)
|
||||
stream.write("𝑻")
|
||||
stream.flush()
|
||||
|
||||
self.assertTrue(raw.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,398 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import gyxx_flow.modules.content_marketing.data.tools.retry_failed as retry
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
|
||||
|
||||
def test_requested_records_must_be_successful_even_when_process_exits_zero():
|
||||
data = {
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success", "matched": True},
|
||||
{"record_id": "b", "status": "retryable_failure", "matched": False},
|
||||
]
|
||||
}
|
||||
|
||||
ok, failed = retry.requested_records_succeeded(data, ["a", "b"])
|
||||
|
||||
assert ok is False
|
||||
assert failed == ["b"]
|
||||
|
||||
|
||||
def test_success_status_cannot_hide_write_failure_and_blocked_input_is_terminal():
|
||||
contradictory = {
|
||||
"results": [{
|
||||
"record_id": "a", "status": "success", "matched": True,
|
||||
"write_ok": False,
|
||||
}]
|
||||
}
|
||||
assert retry.requested_records_succeeded(contradictory, ["a"]) == (False, ["a"])
|
||||
|
||||
blocked = {
|
||||
"total": 1,
|
||||
"details": [{
|
||||
"record_id": "b", "status": "blocked_input", "ok": False,
|
||||
"reason": "url_missing",
|
||||
}],
|
||||
}
|
||||
assert retry.extract_failed_records(blocked, "bili") == []
|
||||
|
||||
|
||||
def test_canonical_filename_requires_current_style_name():
|
||||
style = {"index": 2, "name": "极星pro"}
|
||||
|
||||
assert retry.canonical_result_filename(style, "pgy", False) == "02-极星pro_v2.json"
|
||||
assert retry.canonical_result_filename(style, "xt", False) == "02-极星pro_xingtu_v2.json"
|
||||
assert retry.canonical_result_filename(style, "bili", True) == "02-极星pro_self_bilibili_v2.json"
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict | list) -> None:
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def test_find_failed_uses_exact_current_style_file_and_ignores_legacy_name(tmp_path):
|
||||
style = {"index": 2, "name": "极星pro"}
|
||||
canonical = tmp_path / "02-极星pro_v2.json"
|
||||
legacy = tmp_path / "02-盖亚微单_v2.json"
|
||||
payload = {
|
||||
"index": 2,
|
||||
"total": 1,
|
||||
"matched": 1,
|
||||
"filled": 1,
|
||||
"results": [{"record_id": "ok", "status": "success", "matched": True}],
|
||||
}
|
||||
_write_json(canonical, payload)
|
||||
_write_json(legacy, {"index": 2, "total": 1, "matched": 0, "results": []})
|
||||
old = time.time() - 48 * 3600
|
||||
os.utime(legacy, (old, old))
|
||||
|
||||
failed = retry.find_failed(
|
||||
"pgy", stale_hours=4, include_half=True,
|
||||
styles=[style], v2_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert failed == {}
|
||||
|
||||
|
||||
def test_find_failed_treats_missing_result_rows_as_incomplete(tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
path = tmp_path / retry.canonical_result_filename(style, "xt", False)
|
||||
_write_json(path, {"index": 1, "total": 2, "matched": 0, "filled": 0, "results": []})
|
||||
|
||||
failed = retry.find_failed(
|
||||
"xt", stale_hours=4, include_half=True,
|
||||
styles=[style], v2_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert failed[1]["reason"] == "ALL_FAILED"
|
||||
assert failed[1]["failed_rids"] == []
|
||||
|
||||
|
||||
def test_run_one_style_rechecks_payload_and_retries_exit_zero(monkeypatch, tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False)
|
||||
calls = []
|
||||
|
||||
def fake_call(_cmd, cwd, env=None):
|
||||
del env
|
||||
calls.append(cwd)
|
||||
_write_json(result_path, {
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"matched": 0,
|
||||
"filled": 0,
|
||||
"results": [{
|
||||
"record_id": "bad",
|
||||
"status": "retryable_failure",
|
||||
"matched": False,
|
||||
}],
|
||||
})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "call", fake_call)
|
||||
monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {})
|
||||
monkeypatch.setattr(retry.time, "sleep", lambda _seconds: None)
|
||||
|
||||
ok = retry.run_one_style(
|
||||
"pgy", 1, max_attempts=2, record_ids=["bad"],
|
||||
style=style, v2_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert ok is False
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
def test_run_one_style_accepts_exit_zero_only_after_requested_record_succeeds(monkeypatch, tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False)
|
||||
|
||||
def fake_call(_cmd, cwd, env=None):
|
||||
del env
|
||||
_write_json(result_path, {
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"matched": 1,
|
||||
"filled": 1,
|
||||
"results": [{
|
||||
"record_id": "ok",
|
||||
"status": "success",
|
||||
"matched": True,
|
||||
"write_ok": True,
|
||||
}],
|
||||
})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "call", fake_call)
|
||||
monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {})
|
||||
|
||||
assert retry.run_one_style(
|
||||
"pgy", 1, max_attempts=1, record_ids=["ok"],
|
||||
style=style, v2_dir=tmp_path,
|
||||
) is True
|
||||
|
||||
|
||||
def test_self_operated_xingtu_retry_uses_self_douyin_scraper(monkeypatch, tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
result_path = tmp_path / retry.canonical_result_filename(style, "xt", True)
|
||||
observed = {}
|
||||
|
||||
def fake_call(cmd, cwd, env=None):
|
||||
del env
|
||||
observed["cmd"] = cmd
|
||||
_write_json(result_path, {
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "ok", "status": "success", "matched": True}],
|
||||
})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "call", fake_call)
|
||||
monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {})
|
||||
|
||||
assert retry.run_one_style(
|
||||
"xt", 1, max_attempts=1, self_operated=True,
|
||||
record_ids=["ok"], style=style, v2_dir=tmp_path,
|
||||
) is True
|
||||
assert observed["cmd"][1].endswith("self_douyin_scraper.py")
|
||||
assert "--self-operated" not in observed["cmd"]
|
||||
|
||||
|
||||
def test_full_summary_declared_failure_cannot_be_treated_as_complete():
|
||||
complete, reason = retry._summary_completed({
|
||||
"total": 1,
|
||||
"results": [{
|
||||
"record_id": "ok",
|
||||
"status": "success",
|
||||
"matched": True,
|
||||
"write_ok": True,
|
||||
}],
|
||||
"write_failures": 1,
|
||||
})
|
||||
|
||||
assert complete is False
|
||||
assert reason == "summary declares unresolved failures"
|
||||
|
||||
|
||||
def test_retry_main_returns_nonzero_when_verified_retry_still_fails(monkeypatch):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
monkeypatch.setattr(retry, "load_current_styles", lambda self_operated=False: [style])
|
||||
monkeypatch.setattr(
|
||||
retry, "find_failed",
|
||||
lambda *_args, **_kwargs: {1: {"reason": "HALF", "failed_rids": ["bad"]}},
|
||||
)
|
||||
monkeypatch.setattr(retry, "run_one_style", lambda *_args, **_kwargs: False)
|
||||
monkeypatch.setattr(retry.sys, "argv", ["retry_failed.py", "--platform", "pgy"])
|
||||
|
||||
assert retry.main() == 1
|
||||
|
||||
|
||||
def test_validate_summary_payload_rejects_stale_missing_duplicate_and_unresolved(tmp_path):
|
||||
path = tmp_path / "summary.json"
|
||||
valid = [{
|
||||
"index": 1,
|
||||
"run_started_at": time.time() + 10,
|
||||
"total": 2,
|
||||
"matched": 2,
|
||||
"filled": 2,
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success", "matched": True},
|
||||
{"record_id": "b", "status": "success", "matched": True},
|
||||
],
|
||||
}]
|
||||
_write_json(path, valid)
|
||||
|
||||
assert run_all.validate_summary_payload(
|
||||
valid, {1}, path, path.stat().st_mtime - 1,
|
||||
)[0] is True
|
||||
assert run_all.validate_summary_payload(
|
||||
valid, {1}, path, path.stat().st_mtime + 1,
|
||||
)[0] is False
|
||||
assert run_all.validate_summary_payload(valid, {1, 2}, path, 0)[0] is False
|
||||
|
||||
duplicate = [{
|
||||
"index": 1, "total": 2,
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success"},
|
||||
{"record_id": "a", "status": "success"},
|
||||
],
|
||||
}]
|
||||
assert run_all.validate_summary_payload(duplicate, {1}, path, 0)[0] is False
|
||||
|
||||
unresolved = [{
|
||||
"index": 1, "total": 1,
|
||||
"results": [{
|
||||
"record_id": "a", "status": "retryable_failure", "matched": False,
|
||||
}],
|
||||
}]
|
||||
assert run_all.validate_summary_payload(unresolved, {1}, path, 0)[0] is False
|
||||
|
||||
|
||||
def test_validate_selected_style_allows_unrelated_rows_in_merged_global_summary(tmp_path):
|
||||
path = tmp_path / "summary.json"
|
||||
payload = [
|
||||
{
|
||||
"index": 1,
|
||||
"run_started_at": time.time() + 10,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "ok", "status": "success"}],
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"total": 1,
|
||||
"results": [{
|
||||
"record_id": "old-failure",
|
||||
"status": "retryable_failure",
|
||||
}],
|
||||
},
|
||||
]
|
||||
_write_json(path, payload)
|
||||
|
||||
valid, reason = run_all.validate_summary_payload(
|
||||
payload, {1}, path, path.stat().st_mtime - 1,
|
||||
)
|
||||
|
||||
assert valid is True, reason
|
||||
|
||||
|
||||
def test_validate_summary_rejects_expected_style_left_over_from_old_run(tmp_path):
|
||||
path = tmp_path / "summary.json"
|
||||
started_at = 100.0
|
||||
payload = [
|
||||
{
|
||||
"index": 1,
|
||||
"run_started_at": 101.0,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "new", "status": "success"}],
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"run_started_at": 90.0,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "old", "status": "success"}],
|
||||
},
|
||||
]
|
||||
_write_json(path, payload)
|
||||
|
||||
valid, reason = run_all.validate_summary_payload(
|
||||
payload, {1, 2}, path, started_at,
|
||||
)
|
||||
|
||||
assert valid is False
|
||||
assert "style 2" in reason
|
||||
assert "current run" in reason
|
||||
|
||||
|
||||
def test_aggregate_counts_all_unresolved_rows_as_errors():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": [{
|
||||
"index": 1,
|
||||
"total": 3,
|
||||
"matched": 1,
|
||||
"filled": 1,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success"},
|
||||
{"record_id": "miss", "matched": False},
|
||||
{"record_id": "detail", "reason": "no_detail_page"},
|
||||
],
|
||||
}],
|
||||
})
|
||||
|
||||
assert aggregate["platforms"]["pgy"]["unresolved"] == 2
|
||||
assert aggregate["platforms"]["pgy"]["errors"] == 2
|
||||
|
||||
|
||||
def test_aggregate_sums_declared_failure_categories_when_unresolved_is_absent():
|
||||
aggregate = run_all.aggregate({
|
||||
"xt": [{
|
||||
"index": 1,
|
||||
"total": 2,
|
||||
"matched": 2,
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success"},
|
||||
{"record_id": "b", "status": "success"},
|
||||
],
|
||||
"retryable_failures": 1,
|
||||
"write_failures": 1,
|
||||
}],
|
||||
})
|
||||
|
||||
assert aggregate["platforms"]["xt"]["unresolved"] == 2
|
||||
|
||||
|
||||
def test_aggregate_keeps_partial_counts_when_validation_marks_payload_invalid():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": {
|
||||
"error": "invalid summary: unresolved",
|
||||
"payload": [{
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"matched": 0,
|
||||
"results": [{
|
||||
"record_id": "bad", "status": "retryable_failure",
|
||||
}],
|
||||
}],
|
||||
},
|
||||
})
|
||||
|
||||
platform = aggregate["platforms"]["pgy"]
|
||||
assert platform["loaded"] is True
|
||||
assert platform["unresolved"] == 1
|
||||
assert platform["validation_error"] == "invalid summary: unresolved"
|
||||
|
||||
|
||||
def test_run_round_marks_exit_zero_platform_failed_when_summary_is_invalid(monkeypatch, tmp_path):
|
||||
args = SimpleNamespace(dry_run=True, style=[1])
|
||||
stdout = tmp_path / "stdout.log"
|
||||
stderr = tmp_path / "stderr.log"
|
||||
observed = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
run_all, "start_process",
|
||||
lambda *_args, **_kwargs: (object(), stdout, stderr),
|
||||
)
|
||||
monkeypatch.setattr(run_all, "wait_all", lambda _procs: {"pgy": 0})
|
||||
|
||||
def fake_load(key, self_operated=False, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return {"error": "invalid summary: stale"}
|
||||
|
||||
monkeypatch.setattr(run_all, "load_summary", fake_load)
|
||||
monkeypatch.setattr(run_all, "print_report", lambda *_args: None)
|
||||
monkeypatch.setattr(run_all, "_atomic_write_json", lambda *_args: None)
|
||||
|
||||
result = run_all.run_round(args, ["pgy"])
|
||||
|
||||
assert result["exit_codes"] == {"pgy": 0}
|
||||
assert result["_non_zero_platforms"] == ["pgy"]
|
||||
assert observed["expected_indices"] == {1}
|
||||
assert isinstance(observed["started_at"], float)
|
||||
|
||||
|
||||
def test_run_all_preserves_cookie_skip_exit_code():
|
||||
assert run_all.aggregate_exit_code(True, [75, 75]) == 75
|
||||
assert run_all.aggregate_exit_code(True, [75, 1]) == 1
|
||||
assert run_all.aggregate_exit_code(True, []) == 1
|
||||
assert run_all.aggregate_exit_code(False, []) == 0
|
||||
@@ -0,0 +1,10 @@
|
||||
from gyxx_flow.modules.content_marketing import self_douyin_scraper
|
||||
|
||||
|
||||
def test_self_douyin_accepts_feishu_select_and_multiselect_values():
|
||||
assert self_douyin_scraper.is_douyin_platform("抖音") is True
|
||||
assert self_douyin_scraper.is_douyin_platform(["抖音"]) is True
|
||||
assert self_douyin_scraper.is_douyin_platform("Douyin") is True
|
||||
assert self_douyin_scraper.is_douyin_platform(["小红书", "抖音"]) is True
|
||||
assert self_douyin_scraper.is_douyin_platform("小红书") is False
|
||||
assert self_douyin_scraper.is_douyin_platform(None) is False
|
||||
@@ -0,0 +1,58 @@
|
||||
import unittest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools.sync_style_categories import (
|
||||
extract_style_mapping,
|
||||
normalize_style_name,
|
||||
)
|
||||
|
||||
|
||||
class SyncStyleCategoriesTests(unittest.TestCase):
|
||||
def test_normalizes_feishu_lifecycle_names_to_database_names(self):
|
||||
cases = {
|
||||
"盖亚微单Pro生命进程(新)": "盖亚微单",
|
||||
"星迹&星迹2生命进程(新)": "星迹2",
|
||||
"宙斯双肩包生命进程(新)": "宙斯",
|
||||
"极星Pro生命进程(新)": "极星pro",
|
||||
"瑞白双肩包生命进程(新)": "瑞白",
|
||||
"逐星gt生命进程(新)": "逐星GT",
|
||||
"拾影相机包生命进程 (新)": "拾影斜挎相机包",
|
||||
"极星托特生命进程(新)": "极星托特",
|
||||
}
|
||||
for source, expected in cases.items():
|
||||
with self.subTest(source=source):
|
||||
self.assertEqual(normalize_style_name(source), expected)
|
||||
|
||||
def test_extracts_mapping_and_ignores_blank_rows(self):
|
||||
payload = {
|
||||
"data": {
|
||||
"data": [
|
||||
[
|
||||
"[盖亚斜挎生命进程(新)](https://example.feishu.cn/base/TokenA)",
|
||||
["都市机能"],
|
||||
],
|
||||
[None, ["都市运动"]],
|
||||
["[极星托特生命进程(新)](https://example.feishu.cn/base/TokenB)", ["智性通勤"]],
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
extract_style_mapping(payload),
|
||||
{"盖亚斜挎": "都市机能", "极星托特": "智性通勤"},
|
||||
)
|
||||
|
||||
def test_rejects_conflicting_categories_for_same_style(self):
|
||||
payload = {
|
||||
"data": {
|
||||
"data": [
|
||||
["[布谷生命进程(新)](https://example/a)", ["都市运动"]],
|
||||
["[布谷生命进程(新)](https://example/b)", ["都市机能"]],
|
||||
]
|
||||
}
|
||||
}
|
||||
with self.assertRaisesRegex(ValueError, "布谷"):
|
||||
extract_style_mapping(payload)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,70 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing import weekly_summary_all as weekly
|
||||
|
||||
|
||||
class WeeklyTmallPersonaTests(unittest.TestCase):
|
||||
def test_missing_persona_forbids_invented_percentages(self):
|
||||
text = weekly.format_tmall_persona(None)
|
||||
self.assertIn("不得虚构具体比例", text)
|
||||
|
||||
def test_format_tmall_persona_includes_date_and_dimensions(self):
|
||||
persona = {
|
||||
"collect_date": "2026-07-15",
|
||||
"item_id": "900176117674",
|
||||
"payload": {
|
||||
"gender": [
|
||||
{"name": "男性用户", "value": 66.97},
|
||||
{"name": "女性用户", "value": 28.26},
|
||||
],
|
||||
"age": [{"name": "25-29岁", "value": 26.97}],
|
||||
"city_tier": [{"name": "准一线城市", "value": 31.14}],
|
||||
"buying_power": [{"name": "购买力L5", "value": 30.65}],
|
||||
},
|
||||
}
|
||||
|
||||
text = weekly.format_tmall_persona(persona)
|
||||
|
||||
self.assertIn("2026-07-15", text)
|
||||
self.assertIn("男性用户 66.97%", text)
|
||||
self.assertIn("25-29岁 26.97%", text)
|
||||
self.assertIn("准一线城市 31.14%", text)
|
||||
self.assertIn("购买力L5 30.65%", text)
|
||||
|
||||
def test_generate_summary_injects_latest_tmall_persona_into_single_note_prompt(self):
|
||||
persona = {
|
||||
"collect_date": "2026-07-15",
|
||||
"item_id": "900176117674",
|
||||
"payload": {
|
||||
"gender": [{"name": "男性用户", "value": 66.97}],
|
||||
"age": [{"name": "25-29岁", "value": 26.97}],
|
||||
"city_tier": [],
|
||||
"buying_power": [],
|
||||
},
|
||||
}
|
||||
notes = [{
|
||||
"title": "测试笔记",
|
||||
"platform": "douyin",
|
||||
"creator_name": "达人甲",
|
||||
"view_count": 100,
|
||||
"like_count": 10,
|
||||
"comment_count": 2,
|
||||
"favorite_count": 3,
|
||||
"share_count": 1,
|
||||
"report_path": None,
|
||||
}]
|
||||
|
||||
with patch.object(weekly, "get_latest_tmall_persona", return_value=persona) as get_persona, \
|
||||
patch.object(weekly, "call_hermes_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
|
||||
weekly.generate_summary("星云2", notes)
|
||||
|
||||
get_persona.assert_called_once_with("星云2")
|
||||
single_prompt = call.call_args_list[0].args[1]
|
||||
self.assertIn("天猫该款最新人物画像", single_prompt)
|
||||
self.assertIn("男性用户 66.97%", single_prompt)
|
||||
self.assertIn("内容触达人群", single_prompt)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user