import argparse from pathlib import Path import pytest from gyxx_flow.modules.content_marketing import bilibili_scraper as bili from gyxx_flow.modules.content_marketing import collection_completeness as cc from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy from gyxx_flow.modules.content_marketing import run_all from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu def test_normalize_title_handles_nfkc_case_and_zero_width(): assert cc.normalize_title("MacBook\u200b 通勤包!") == "macbook通勤包" def test_coerce_url_unwraps_feishu_markdown_links(): value = "[查看笔记](https://www.xiaohongshu.com/discovery/item/abc123?x=1)" assert cc.coerce_url(value) == "https://www.xiaohongshu.com/discovery/item/abc123?x=1" def test_match_tasks_prefers_note_id_over_changed_title(): tasks = [{ "record_id": "r1", "target_title": "飞书中的旧标题", "note_url": "https://www.xiaohongshu.com/explore/abc123", "note_id": "abc123", }] cards = [ {"title": "完全不同的新标题", "href": "https://www.xiaohongshu.com/explore/abc123", "note_id": "abc123"}, {"title": "飞书中的旧标题", "href": "https://www.xiaohongshu.com/explore/other", "note_id": "other"}, ] matched = cc.match_tasks_to_cards(tasks, cards, platform="xhs") assert matched["r1"]["note_id"] == "abc123" assert matched["r1"]["match_method"] == "content_id" def test_match_tasks_accepts_unique_truncated_title_but_rejects_ambiguous_title(): task = {"record_id": "r1", "target_title": "男生长期主义通勤双肩包分享"} unique = [ {"title": "男生长期主义通勤双肩包"}, {"title": "夏日轻量斜挎包"}, ] ambiguous = [ {"title": "男生长期主义通勤双肩包"}, {"title": "男生长期主义通勤双肩包"}, ] assert cc.match_tasks_to_cards([task], unique, platform="douyin")["r1"] assert "r1" not in cc.match_tasks_to_cards([task], ambiguous, platform="douyin") def test_title_similarity_ignores_episode_prefix_and_hashtag_suffix(): target = "第20集:Pocket4拍旋焦,无后期也能出片?" platform_title = ( "Pocket4拍旋焦,无后期也能出片? #旋焦 #pocket4 #摄影装备 " "无论Pocket3还是Pocket4都可以轻松拍出旋焦效果!" ) assert cc.title_similarity(platform_title, target) >= 0.98 def test_title_similarity_rejects_different_episode_numbers(): assert cc.title_similarity( "第19集:Pocket4拍旋焦,无后期也能出片?", "第20集:Pocket4拍旋焦,无后期也能出片?", ) == 0.0 def test_match_tasks_rejects_duplicate_exact_titles_with_different_ids(): task = {"record_id": "r1", "target_title": "完全相同的目标标题"} cards = [ {"title": "完全相同的目标标题", "note_id": "video-1"}, {"title": "完全相同的目标标题", "note_id": "video-2"}, ] assert "r1" not in cc.match_tasks_to_cards([task], cards, platform="douyin") def test_one_card_is_not_reused_for_two_different_creator_tasks(): tasks = [ { "record_id": "loose", "target_title": "commuter backpack review today", "note_url": "https://v.douyin.com/loose-source/", }, { "record_id": "exact", "target_title": "commuter backpack review", "note_url": "https://v.douyin.com/exact-source/", }, ] cards = [{"title": "commuter backpack review", "note_id": "observed-1"}] matched = cc.match_tasks_to_cards(tasks, cards, platform="douyin") assert set(matched) == {"exact"} def test_same_source_url_can_share_one_card_across_styles(): tasks = [ { "record_id": "style-1", "target_title": "同一篇跨款式笔记", "note_url": "https://v.douyin.com/same-short-link/", }, { "record_id": "style-2", "target_title": "同一篇跨款式笔记", "note_url": "https://v.douyin.com/same-short-link/", }, ] cards = [{"title": "同一篇跨款式笔记", "note_id": "observed-1"}] assert set(cc.match_tasks_to_cards(tasks, cards, platform="douyin")) == { "style-1", "style-2", } def test_known_mismatched_content_ids_cannot_fall_back_to_same_title(): task = { "record_id": "r1", "target_title": "完全相同标题", "note_id": "video-a", } card = {"title": "完全相同标题", "note_id": "video-b"} assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {} def test_xingtu_api_and_dom_duplicate_are_merged_before_matching(): cards = [ { "title": "同一条视频", "play_count": 19481, "note_id": "video-1", "source": "show_items_api", }, { "title": "同一条视频", "play_count": "1.9w", "note_id": "", "page": 1, }, ] deduped = cc.deduplicate_observed_cards(cards) matched = cc.match_tasks_to_cards( [{"record_id": "r1", "target_title": "同一条视频"}], deduped, platform="douyin", ) assert len(deduped) == 1 assert matched["r1"]["note_id"] == "video-1" def test_merge_partial_summary_keeps_existing_success_rows(): existing = { "style": "款式A", "index": 1, "total": 2, "matched": 1, "filled": 1, "results": [ {"record_id": "ok", "status": "success", "matched": True, "write_ok": True}, {"record_id": "retry", "status": "retryable_failure", "matched": False}, ], } partial = { "style": "款式A", "index": 1, "total": 1, "matched": 1, "filled": 1, "results": [ {"record_id": "retry", "status": "success", "matched": True, "write_ok": True}, ], } merged = cc.merge_style_summary(existing, partial) assert merged["total"] == 2 assert {row["record_id"] for row in merged["results"]} == {"ok", "retry"} assert merged["matched"] == 2 assert merged["filled"] == 2 assert merged["complete"] is True def test_summary_contract_detects_missing_or_write_failure(): summary = { "total": 2, "results": [ {"record_id": "ok", "status": "success", "matched": True, "write_ok": True}, {"record_id": "bad", "status": "write_failure", "matched": True, "write_ok": False}, ], } finalized = cc.finalize_summary(summary, dry_run=False) assert finalized["complete"] is False assert finalized["unresolved"] == 1 missing = cc.finalize_summary({"total": 2, "results": summary["results"][:1]}, dry_run=False) assert missing["complete"] is False assert missing["missing_results"] == 1 def _style_for_platform(platform: str): return { "index": 1, "name": "款式A", "base_token": "base", "table_id": "table", "field_map": { "creator_name": {"field_id": "name"}, "creator_id": {"field_id": "creator_id"}, "note_title": {"field_id": "title"}, "publish_time": {"field_id": "pub"}, "note_url": {"field_id": "url"}, "platform": {"field_id": "platform"}, "read_count_7d": {"field_id": "read7"}, }, } def test_extractors_preserve_note_url_and_content_id(monkeypatch): xhs_row = { "record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id", "title": "标题", "pub": "2026-07-20", "platform": "小红书", "url": "https://www.xiaohongshu.com/explore/abc123", "read7": None, } dy_row = { "record_id": "dy-r", "name": "达人", "creator_id": "dy-id", "title": "标题", "pub": "2026-07-20", "platform": "抖音", "url": "https://www.douyin.com/video/7654321", "read7": None, } monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row]) monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row]) xhs_task = pgy.extract_target_tasks(_style_for_platform("小红书"))[0] dy_task = xingtu.extract_target_tasks(_style_for_platform("抖音"))[0] assert xhs_task["note_url"].endswith("/abc123") assert xhs_task["note_id"] == "abc123" assert dy_task["note_url"].endswith("/7654321") assert dy_task["note_id"] == "7654321" @pytest.mark.parametrize( ("module", "platform", "filled_url"), [ (pgy, "小红书", "https://www.douyin.com/video/7654321"), (xingtu, "抖音", "http://8.99 分享文案 https://v.douyin.com/abc123/"), ], ) def test_filled_publish_link_triggers_collection_without_platform_validation( monkeypatch, module, platform, filled_url, ): row = { "record_id": "r1", "name": "达人", "creator_id": "creator-id", "title": "已发布标题", "pub": "2026-07-20", "platform": platform, "url": filled_url, "read7": None, } monkeypatch.setattr(module, "list_records_by_table", lambda *_: [row]) task = module.extract_target_tasks(_style_for_platform(platform))[0] assert task["input_error"] is None def test_extractors_can_collect_missing_title_when_direct_url_has_content_id(monkeypatch): xhs_style = _style_for_platform("小红书") dy_style = _style_for_platform("抖音") xhs_row = { "record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id", "title": None, "pub": "2026-07-20", "platform": "小红书", "url": "https://www.xiaohongshu.com/explore/abc123", "read7": None, } dy_row = { "record_id": "dy-r", "name": "达人", "creator_id": "dy-id", "title": None, "pub": "2026-07-20", "platform": "抖音", "url": "https://www.douyin.com/video/7654321", "read7": None, } monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row]) monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row]) assert pgy.extract_target_tasks(xhs_style)[0]["note_id"] == "abc123" assert xingtu.extract_target_tasks(dy_style)[0]["note_id"] == "7654321" assert xhs_style["_extract_stats"]["skipped_incomplete"] == 0 assert dy_style["_extract_stats"]["skipped_incomplete"] == 0 def test_missing_critical_mapping_is_an_incomplete_summary(monkeypatch): style = _style_for_platform("抖音") del style["field_map"]["note_url"] monkeypatch.setattr( xingtu, "list_records_by_table", lambda *_: (_ for _ in ()).throw(AssertionError("mapping should fail first")), ) _, summaries = xingtu.collect_tasks_across_styles([style], None) finalized = cc.finalize_summary(summaries[1], dry_run=True) assert "note_url" in finalized["error"] assert finalized["complete"] is False assert finalized["unresolved"] >= 1 class _FakeInput: def __init__(self): self.filled = [] def click(self): return None def fill(self, value): self.filled.append(value) def press(self, _key): return None class _FakeLocator: def __init__(self, input_): self.first = input_ def filter(self, **_kwargs): return self class _FakeSearchPage: def __init__(self): self.input = _FakeInput() self.has_result_calls = 0 def wait_for_timeout(self, _ms): return None def evaluate(self, script, arg=None): if "const rows" in script: self.has_result_calls += 1 return self.has_result_calls >= 2 if "const inputs" in script: return "" raise AssertionError(script) def locator(self, _selector): return _FakeLocator(self.input) def wait_for_url(self, *_args, **_kwargs): return None def test_pgy_search_retry_keeps_creator_id_query(): page = _FakeSearchPage() pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2) assert page.input.filled == ["xhs-id-123"] def test_apply_matched_task_reports_write_failure(monkeypatch): task = { "record_id": "r1", "publish_time": object(), "style_context": { "index": 1, "base_token": "base", "table_id": "table", "field_map": {}, }, } summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []} monkeypatch.setattr(pgy, "pick_read_field", lambda *_: ("read_count_7d", "fid", "7天曝光量")) monkeypatch.setattr(pgy, "write_back", lambda *_: False) outcome = pgy.apply_matched_task(task, {"read_count": "123", "title": "标题"}, summary, dry_run=False) assert outcome["status"] == "write_failure" assert outcome["write_ok"] is False def test_default_creator_batch_has_no_forced_hour_pause(): assert pgy.DEFAULT_BATCH_SIZE == 0 assert xingtu.DEFAULT_BATCH_SIZE == 0 def test_repeatable_style_argument_and_selection_keep_every_requested_style(): parser = argparse.ArgumentParser() cc.add_repeatable_style_argument(parser, "styles") args = parser.parse_args(["--style", "1", "--style", "3"]) styles = [{"index": 1}, {"index": 2}, {"index": 3}] assert args.style == [1, 3] assert cc.select_requested_styles(styles, args.style) == [styles[0], styles[2]] with pytest.raises(ValueError, match="99"): cc.select_requested_styles(styles, [1, 99]) def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch): sequence = [[], [{"title": "标题", "read_count": "1"}], [{"title": "标题", "read_count": "1"}]] monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0)) class Page: waits = 0 def wait_for_timeout(self, _ms): self.waits += 1 page = Page() assert pgy.parse_cards_stable(page) == [{"title": "标题", "read_count": "1"}] assert page.waits == 2 def test_xingtu_paginates_before_falling_back_to_search(monkeypatch): pages = [ [{"title": "无关视频", "play_count": "10"}], [{"title": "目标视频标题", "play_count": "20"}], ] calls = {"next": 0} monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False) monkeypatch.setattr(xingtu, "parse_videos_stable", lambda _page: pages.pop(0)) def next_page(_page): calls["next"] += 1 return True monkeypatch.setattr(xingtu, "go_next_page", next_page) monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False) found, _, _ = xingtu.find_videos_for_tasks( object(), [{"record_id": "r1", "target_title": "目标视频标题"}], max_pages=2, ) assert found["r1"]["play_count"] == "20" assert calls["next"] == 1 def test_xingtu_normalizes_show_items_api_video(): card = xingtu.normalize_xingtu_api_item({ "item_id": 7654321, "item_title": "目标视频标题", "play": 168000, "like": 1200, "comment": 34, "share": 56, "url": "https://www.douyin.com/video/7654321", }) assert card == { "title": "目标视频标题", "play_count": 168000, "like_count": 1200, "comment_count": 34, "share_count": 56, "href": "https://www.douyin.com/video/7654321", "note_id": "7654321", "source": "show_items_api", } def test_xingtu_show_items_capture_collects_and_deduplicates_cards(): class Response: url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2" def json(self): return { "latest_item_info": [ {"item_id": "1", "title": "视频一", "play": 10}, {"item_id": "2", "item_title": "视频二", "play": 20}, ], "latest_star_item_info": [ {"item_id": "2", "item_title": "视频二", "play": 20}, ], } class Page: def on(self, event, callback): assert event == "response" self.callback = callback page = Page() cards = xingtu.setup_show_items_capture(page) page.callback(Response()) assert [card["note_id"] for card in cards] == ["1", "2"] def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch): monkeypatch.setattr( xingtu, "parse_videos_stable", lambda _page: (_ for _ in ()).throw(AssertionError("DOM should not be needed")), ) monkeypatch.setattr( xingtu, "is_blocked", lambda _page: (_ for _ in ()).throw(AssertionError("page should not be needed")), ) tasks = [{ "record_id": "r1", "target_title": "飞书旧标题", "note_url": "https://www.douyin.com/video/7654321", "note_id": "7654321", }] api_cards = [{ "title": "平台上的新标题", "play_count": 168000, "note_id": "7654321", "source": "show_items_api", }] found, _, page_limit_hit = xingtu.find_videos_for_tasks( object(), tasks, max_pages=5, api_cards=api_cards, ) assert found["r1"]["play_count"] == 168000 assert found["r1"]["match_method"] == "content_id" assert page_limit_hit is False def test_xingtu_api_fuzzy_candidate_does_not_preempt_later_exact_dom(monkeypatch): monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False) monkeypatch.setattr( xingtu, "parse_videos_stable", lambda _page: [{"title": "commuter backpack review", "note_id": "exact-card"}], ) monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False) monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False) task = {"record_id": "r1", "target_title": "commuter backpack review"} api_cards = [{ "title": "commuter backpack review today", "note_id": "fuzzy-card", "source": "show_items_api", }] found, _, _ = xingtu.find_videos_for_tasks( object(), [task], max_pages=2, api_cards=api_cards, ) assert found["r1"]["note_id"] == "exact-card" def test_browser_session_loss_detection_is_specific(): closed = RuntimeError("Target page, context or browser has been closed") assert pgy.is_browser_session_lost(closed) assert xingtu.is_browser_session_lost(closed) assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure")) def test_xingtu_nested_session_loss_is_reraised(): closed = RuntimeError("Target page, context or browser has been closed") with pytest.raises(RuntimeError, match="has been closed"): xingtu.reraise_if_browser_session_lost(closed) xingtu.reraise_if_browser_session_lost(RuntimeError("ordinary lookup miss")) def test_xingtu_detail_and_video_search_propagate_closed_session(): closed = RuntimeError("Target page, context or browser has been closed") class Detail: def goto(self, *_args, **_kwargs): raise closed class Context: pages = [] def new_page(self): return Detail() class SearchPage: context = Context() def evaluate(self, _script, _arg=None): return {"found": True, "url": "https://www.xingtu.cn/ad/creator/author-homepage/douyin-video/1"} with pytest.raises(RuntimeError, match="has been closed"): xingtu.open_creator_detail(SearchPage(), "达人", "creator-id") class Locator: first = None def __init__(self): self.first = self def count(self): return 1 def wait_for(self, **_kwargs): raise closed class VideoPage: def locator(self, _selector): return Locator() with pytest.raises(RuntimeError, match="has been closed"): xingtu.submit_video_search(VideoPage(), "标题") def test_pgy_fallback_detail_click_propagates_closed_session(): closed = RuntimeError("Target page, context or browser has been closed") class Clicker: first = None def __init__(self): self.first = self def click(self, **_kwargs): raise closed class Context: pages = [] class Page: context = Context() def evaluate(self, *_args, **_kwargs): return {"ok": False, "reason": "no-row"} def get_by_text(self, *_args, **_kwargs): return Clicker() with pytest.raises(RuntimeError, match="has been closed"): pgy.open_blogger_detail(Page(), "达人", None) def test_bilibili_triggered_blocked_inputs_are_visible_terminal_rows(): summary = bili.finalize_bili_summary({ "style": "款式A", "index": 1, "total_b_records": 1, "details": [{ "record_id": "r1", "status": "blocked_input", "matched": False, "reason": "publish_time_missing", "ok": False, }], }) assert summary["blocked_input"] == 1 assert summary["unresolved"] == 0 assert summary["complete"] is True def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch): style = { "style": "款式A", "name": "款式A", "index": 1, "base_token": "base", "table_id": "table", "field_map": { "platform": {"field_id": "platform"}, "note_url": {"field_id": "url"}, "creator_name": {"field_id": "creator"}, "publish_time": {"field_id": "pub"}, "read_count_7d": {"field_id": "s7"}, "read_count_14d": {"field_id": "s14"}, "read_count_21d": {"field_id": "s21"}, "read_count_28d": {"field_id": "s28"}, "month_end": {"field_id": "sm"}, }, } rows = [ { "record_id": "no-link", "platform": "B站", "creator": "未发布达人", "url": "", "pub": "2026-07-20", }, { "record_id": "published", "platform": "B站", "creator": "已发布达人", "url": "https://www.bilibili.com/video/BV1234567890", "pub": "2026-07-20", }, ] monkeypatch.setattr(bili, "list_all_records", lambda *_args: rows) monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: 321) monkeypatch.setattr(bili, "write_record", lambda *_args, **_kwargs: True) summary = bili.process_style( style, only_record_ids=None, dry_run=True, delay=0, session=object(), state={}, force_today=bili.date(2026, 7, 23), first_run=False, ) assert summary["source_b_records"] == 2 assert summary["skipped_no_url"] == 1 assert summary["total_b_records"] == 1 assert summary["success"] == 1 assert summary["blocked_input"] == 0 assert [row["record_id"] for row in summary["details"]] == ["published"] def test_bilibili_partial_retry_preserves_old_success(): existing = { "style": "款式A", "index": 1, "total_b_records": 2, "details": [ {"record_id": "ok", "status": "success", "ok": True}, {"record_id": "retry", "status": "retryable_failure", "ok": False}, ], } partial = { "style": "款式A", "index": 1, "total_b_records": 1, "details": [{"record_id": "retry", "status": "success", "ok": True}], } merged = bili.merge_bili_summary(existing, partial) assert merged["total_b_records"] == 2 assert {row["record_id"] for row in merged["details"]} == {"ok", "retry"} assert merged["complete"] is True def test_bilibili_top_level_error_cannot_finalize_as_complete(): finalized = bili.finalize_bili_summary({ "style": "款式A", "index": 1, "total_b_records": 0, "details": [], "error": "missing platform/url field_id", "unresolved": 1, "retryable_failures": 1, "complete": False, }) assert finalized["complete"] is False assert finalized["unresolved"] >= 1 def test_bilibili_missing_slot_mapping_is_system_error_not_blocked_input(): summary = bili.process_style( { "style": "款式A", "name": "款式A", "index": 1, "base_token": "base", "table_id": "table", "field_map": { "platform": {"field_id": "platform"}, "note_url": {"field_id": "url"}, }, }, only_record_ids=None, dry_run=True, delay=0, session=object(), state={}, force_today=None, first_run=False, ) finalized = bili.finalize_bili_summary(summary, dry_run=True) assert "publish_time" in finalized["error"] assert "read_count_7d" in finalized["error"] assert finalized["complete"] is False def test_self_douyin_missing_mapping_and_write_failure_are_not_complete(monkeypatch): missing = self_dy.scrape_one_style( {"name": "款式A", "index": 1, "base_token": "base", "table_id": "table", "field_map": {}}, login_timeout=1, headless=True, only_record_ids=None, dry_run=False, ) assert cc.finalize_summary(missing)["complete"] is False style = _style_for_platform("抖音") row = { "record_id": "r1", "platform": "抖音", "name": "自营达人", "title": "目标作品", "url": "https://www.douyin.com/video/1", "pub": "2026-07-20", } class Page: def goto(self, *_args, **_kwargs): return None def wait_for_timeout(self, _ms): return None class Session: def __init__(self, **_kwargs): pass def __enter__(self): return self def __exit__(self, *_args): return None def fetch(self, _url, page_action, wait): page_action(Page()) monkeypatch.setattr(self_dy, "DynamicSession", Session) monkeypatch.setattr(self_dy, "list_records_by_table", lambda *_: [row]) monkeypatch.setattr(self_dy, "dismiss_popups", lambda *_: None) monkeypatch.setattr(self_dy, "restore_cookies", lambda *_: None) monkeypatch.setattr(self_dy, "maybe_wait_for_login", lambda *_: None) monkeypatch.setattr(self_dy, "save_state", lambda *_: None) monkeypatch.setattr( self_dy, "scroll_and_collect_posts", lambda *_args, **_kwargs: [{"title": "目标作品", "play_count": 123}], ) monkeypatch.setattr(self_dy, "pick_read_field", lambda *_: ("read_count_7d", "read7", "7天曝光量")) monkeypatch.setattr(self_dy, "write_back", lambda *_: False) failed = cc.finalize_summary( self_dy.scrape_one_style( style, login_timeout=1, headless=True, only_record_ids=None, dry_run=False, ) ) assert failed["results"][0]["status"] == "write_failure" assert failed["complete"] is False def test_run_all_rejects_stale_or_incomplete_summary(tmp_path: Path): path = tmp_path / "summary.json" path.write_text("[]", encoding="utf-8") started_at = path.stat().st_mtime + 1 assert run_all.validate_summary_payload([], {1}, path, started_at)[0] is False path.write_text('[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}]', encoding="utf-8") assert run_all.validate_summary_payload( [{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}], {1}, path, path.stat().st_mtime - 1, )[0] is False