feat: complete production workflow migration
This commit is contained in:
@@ -0,0 +1,840 @@
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
|
||||
from gyxx_flow.modules.content_marketing import collection_completeness as cc
|
||||
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
|
||||
|
||||
def test_normalize_title_handles_nfkc_case_and_zero_width():
|
||||
assert cc.normalize_title("ＭａｃＢｏｏｋ\u200b 通勤包！") == "macbook通勤包"
|
||||
|
||||
|
||||
def test_coerce_url_unwraps_feishu_markdown_links():
|
||||
value = "[查看笔记](https://www.xiaohongshu.com/discovery/item/abc123?x=1)"
|
||||
assert cc.coerce_url(value) == "https://www.xiaohongshu.com/discovery/item/abc123?x=1"
|
||||
|
||||
|
||||
def test_match_tasks_prefers_note_id_over_changed_title():
|
||||
tasks = [{
|
||||
"record_id": "r1",
|
||||
"target_title": "飞书中的旧标题",
|
||||
"note_url": "https://www.xiaohongshu.com/explore/abc123",
|
||||
"note_id": "abc123",
|
||||
}]
|
||||
cards = [
|
||||
{"title": "完全不同的新标题", "href": "https://www.xiaohongshu.com/explore/abc123", "note_id": "abc123"},
|
||||
{"title": "飞书中的旧标题", "href": "https://www.xiaohongshu.com/explore/other", "note_id": "other"},
|
||||
]
|
||||
|
||||
matched = cc.match_tasks_to_cards(tasks, cards, platform="xhs")
|
||||
|
||||
assert matched["r1"]["note_id"] == "abc123"
|
||||
assert matched["r1"]["match_method"] == "content_id"
|
||||
|
||||
|
||||
def test_match_tasks_accepts_unique_truncated_title_but_rejects_ambiguous_title():
|
||||
task = {"record_id": "r1", "target_title": "男生长期主义通勤双肩包分享"}
|
||||
unique = [
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
{"title": "夏日轻量斜挎包"},
|
||||
]
|
||||
ambiguous = [
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
]
|
||||
|
||||
assert cc.match_tasks_to_cards([task], unique, platform="douyin")["r1"]
|
||||
assert "r1" not in cc.match_tasks_to_cards([task], ambiguous, platform="douyin")
|
||||
|
||||
|
||||
def test_title_similarity_ignores_episode_prefix_and_hashtag_suffix():
|
||||
target = "第20集：Pocket4拍旋焦，无后期也能出片？"
|
||||
platform_title = (
|
||||
"Pocket4拍旋焦，无后期也能出片？ #旋焦 #pocket4 #摄影装备 "
|
||||
"无论Pocket3还是Pocket4都可以轻松拍出旋焦效果！"
|
||||
)
|
||||
|
||||
assert cc.title_similarity(platform_title, target) >= 0.98
|
||||
|
||||
|
||||
def test_title_similarity_rejects_different_episode_numbers():
|
||||
assert cc.title_similarity(
|
||||
"第19集：Pocket4拍旋焦，无后期也能出片？",
|
||||
"第20集：Pocket4拍旋焦，无后期也能出片？",
|
||||
) == 0.0
|
||||
|
||||
|
||||
def test_match_tasks_rejects_duplicate_exact_titles_with_different_ids():
|
||||
task = {"record_id": "r1", "target_title": "完全相同的目标标题"}
|
||||
cards = [
|
||||
{"title": "完全相同的目标标题", "note_id": "video-1"},
|
||||
{"title": "完全相同的目标标题", "note_id": "video-2"},
|
||||
]
|
||||
|
||||
assert "r1" not in cc.match_tasks_to_cards([task], cards, platform="douyin")
|
||||
|
||||
|
||||
def test_one_card_is_not_reused_for_two_different_creator_tasks():
|
||||
tasks = [
|
||||
{
|
||||
"record_id": "loose",
|
||||
"target_title": "commuter backpack review today",
|
||||
"note_url": "https://v.douyin.com/loose-source/",
|
||||
},
|
||||
{
|
||||
"record_id": "exact",
|
||||
"target_title": "commuter backpack review",
|
||||
"note_url": "https://v.douyin.com/exact-source/",
|
||||
},
|
||||
]
|
||||
cards = [{"title": "commuter backpack review", "note_id": "observed-1"}]
|
||||
|
||||
matched = cc.match_tasks_to_cards(tasks, cards, platform="douyin")
|
||||
|
||||
assert set(matched) == {"exact"}
|
||||
|
||||
|
||||
def test_same_source_url_can_share_one_card_across_styles():
|
||||
tasks = [
|
||||
{
|
||||
"record_id": "style-1",
|
||||
"target_title": "同一篇跨款式笔记",
|
||||
"note_url": "https://v.douyin.com/same-short-link/",
|
||||
},
|
||||
{
|
||||
"record_id": "style-2",
|
||||
"target_title": "同一篇跨款式笔记",
|
||||
"note_url": "https://v.douyin.com/same-short-link/",
|
||||
},
|
||||
]
|
||||
cards = [{"title": "同一篇跨款式笔记", "note_id": "observed-1"}]
|
||||
|
||||
assert set(cc.match_tasks_to_cards(tasks, cards, platform="douyin")) == {
|
||||
"style-1", "style-2",
|
||||
}
|
||||
|
||||
|
||||
def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
|
||||
task = {
|
||||
"record_id": "r1",
|
||||
"target_title": "完全相同标题",
|
||||
"note_id": "video-a",
|
||||
}
|
||||
card = {"title": "完全相同标题", "note_id": "video-b"}
|
||||
|
||||
assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
|
||||
|
||||
|
||||
def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
|
||||
cards = [
|
||||
{
|
||||
"title": "同一条视频",
|
||||
"play_count": 19481,
|
||||
"note_id": "video-1",
|
||||
"source": "show_items_api",
|
||||
},
|
||||
{
|
||||
"title": "同一条视频",
|
||||
"play_count": "1.9w",
|
||||
"note_id": "",
|
||||
"page": 1,
|
||||
},
|
||||
]
|
||||
|
||||
deduped = cc.deduplicate_observed_cards(cards)
|
||||
matched = cc.match_tasks_to_cards(
|
||||
[{"record_id": "r1", "target_title": "同一条视频"}],
|
||||
deduped,
|
||||
platform="douyin",
|
||||
)
|
||||
|
||||
assert len(deduped) == 1
|
||||
assert matched["r1"]["note_id"] == "video-1"
|
||||
|
||||
|
||||
def test_merge_partial_summary_keeps_existing_success_rows():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total": 2, "matched": 1, "filled": 1,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
|
||||
{"record_id": "retry", "status": "retryable_failure", "matched": False},
|
||||
],
|
||||
}
|
||||
partial = {
|
||||
"style": "款式A", "index": 1, "total": 1, "matched": 1, "filled": 1,
|
||||
"results": [
|
||||
{"record_id": "retry", "status": "success", "matched": True, "write_ok": True},
|
||||
],
|
||||
}
|
||||
|
||||
merged = cc.merge_style_summary(existing, partial)
|
||||
|
||||
assert merged["total"] == 2
|
||||
assert {row["record_id"] for row in merged["results"]} == {"ok", "retry"}
|
||||
assert merged["matched"] == 2
|
||||
assert merged["filled"] == 2
|
||||
assert merged["complete"] is True
|
||||
|
||||
|
||||
def test_summary_contract_detects_missing_or_write_failure():
|
||||
summary = {
|
||||
"total": 2,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
|
||||
{"record_id": "bad", "status": "write_failure", "matched": True, "write_ok": False},
|
||||
],
|
||||
}
|
||||
finalized = cc.finalize_summary(summary, dry_run=False)
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] == 1
|
||||
|
||||
missing = cc.finalize_summary({"total": 2, "results": summary["results"][:1]}, dry_run=False)
|
||||
assert missing["complete"] is False
|
||||
assert missing["missing_results"] == 1
|
||||
|
||||
|
||||
def _style_for_platform(platform: str):
|
||||
return {
|
||||
"index": 1,
|
||||
"name": "款式A",
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"creator_name": {"field_id": "name"},
|
||||
"creator_id": {"field_id": "creator_id"},
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"platform": {"field_id": "platform"},
|
||||
"read_count_7d": {"field_id": "read7"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_extractors_preserve_note_url_and_content_id(monkeypatch):
|
||||
xhs_row = {
|
||||
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
|
||||
"title": "标题", "pub": "2026-07-20", "platform": "小红书",
|
||||
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
|
||||
}
|
||||
dy_row = {
|
||||
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
|
||||
"title": "标题", "pub": "2026-07-20", "platform": "抖音",
|
||||
"url": "https://www.douyin.com/video/7654321", "read7": None,
|
||||
}
|
||||
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
|
||||
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
|
||||
|
||||
xhs_task = pgy.extract_target_tasks(_style_for_platform("小红书"))[0]
|
||||
dy_task = xingtu.extract_target_tasks(_style_for_platform("抖音"))[0]
|
||||
|
||||
assert xhs_task["note_url"].endswith("/abc123")
|
||||
assert xhs_task["note_id"] == "abc123"
|
||||
assert dy_task["note_url"].endswith("/7654321")
|
||||
assert dy_task["note_id"] == "7654321"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "filled_url"),
|
||||
[
|
||||
(pgy, "小红书", "https://www.douyin.com/video/7654321"),
|
||||
(xingtu, "抖音", "http://8.99 分享文案 https://v.douyin.com/abc123/"),
|
||||
],
|
||||
)
|
||||
def test_filled_publish_link_triggers_collection_without_platform_validation(
|
||||
monkeypatch, module, platform, filled_url,
|
||||
):
|
||||
row = {
|
||||
"record_id": "r1",
|
||||
"name": "达人",
|
||||
"creator_id": "creator-id",
|
||||
"title": "已发布标题",
|
||||
"pub": "2026-07-20",
|
||||
"platform": platform,
|
||||
"url": filled_url,
|
||||
"read7": None,
|
||||
}
|
||||
monkeypatch.setattr(module, "list_records_by_table", lambda *_: [row])
|
||||
|
||||
task = module.extract_target_tasks(_style_for_platform(platform))[0]
|
||||
|
||||
assert task["input_error"] is None
|
||||
|
||||
|
||||
def test_extractors_can_collect_missing_title_when_direct_url_has_content_id(monkeypatch):
|
||||
xhs_style = _style_for_platform("小红书")
|
||||
dy_style = _style_for_platform("抖音")
|
||||
xhs_row = {
|
||||
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
|
||||
"title": None, "pub": "2026-07-20", "platform": "小红书",
|
||||
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
|
||||
}
|
||||
dy_row = {
|
||||
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
|
||||
"title": None, "pub": "2026-07-20", "platform": "抖音",
|
||||
"url": "https://www.douyin.com/video/7654321", "read7": None,
|
||||
}
|
||||
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
|
||||
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
|
||||
|
||||
assert pgy.extract_target_tasks(xhs_style)[0]["note_id"] == "abc123"
|
||||
assert xingtu.extract_target_tasks(dy_style)[0]["note_id"] == "7654321"
|
||||
assert xhs_style["_extract_stats"]["skipped_incomplete"] == 0
|
||||
assert dy_style["_extract_stats"]["skipped_incomplete"] == 0
|
||||
|
||||
|
||||
def test_missing_critical_mapping_is_an_incomplete_summary(monkeypatch):
|
||||
style = _style_for_platform("抖音")
|
||||
del style["field_map"]["note_url"]
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"list_records_by_table",
|
||||
lambda *_: (_ for _ in ()).throw(AssertionError("mapping should fail first")),
|
||||
)
|
||||
|
||||
_, summaries = xingtu.collect_tasks_across_styles([style], None)
|
||||
finalized = cc.finalize_summary(summaries[1], dry_run=True)
|
||||
|
||||
assert "note_url" in finalized["error"]
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] >= 1
|
||||
|
||||
|
||||
class _FakeInput:
|
||||
def __init__(self):
|
||||
self.filled = []
|
||||
|
||||
def click(self):
|
||||
return None
|
||||
|
||||
def fill(self, value):
|
||||
self.filled.append(value)
|
||||
|
||||
def press(self, _key):
|
||||
return None
|
||||
|
||||
|
||||
class _FakeLocator:
|
||||
def __init__(self, input_):
|
||||
self.first = input_
|
||||
|
||||
def filter(self, **_kwargs):
|
||||
return self
|
||||
|
||||
|
||||
class _FakeSearchPage:
|
||||
def __init__(self):
|
||||
self.input = _FakeInput()
|
||||
self.has_result_calls = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
def evaluate(self, script, arg=None):
|
||||
if "const rows" in script:
|
||||
self.has_result_calls += 1
|
||||
return self.has_result_calls >= 2
|
||||
if "const inputs" in script:
|
||||
return ""
|
||||
raise AssertionError(script)
|
||||
|
||||
def locator(self, _selector):
|
||||
return _FakeLocator(self.input)
|
||||
|
||||
def wait_for_url(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def test_pgy_search_retry_keeps_creator_id_query():
|
||||
page = _FakeSearchPage()
|
||||
|
||||
pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
|
||||
|
||||
assert page.input.filled == ["xhs-id-123"]
|
||||
|
||||
|
||||
def test_apply_matched_task_reports_write_failure(monkeypatch):
|
||||
task = {
|
||||
"record_id": "r1", "publish_time": object(),
|
||||
"style_context": {
|
||||
"index": 1, "base_token": "base", "table_id": "table", "field_map": {},
|
||||
},
|
||||
}
|
||||
summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []}
|
||||
monkeypatch.setattr(pgy, "pick_read_field", lambda *_: ("read_count_7d", "fid", "7天曝光量"))
|
||||
monkeypatch.setattr(pgy, "write_back", lambda *_: False)
|
||||
|
||||
outcome = pgy.apply_matched_task(task, {"read_count": "123", "title": "标题"}, summary, dry_run=False)
|
||||
|
||||
assert outcome["status"] == "write_failure"
|
||||
assert outcome["write_ok"] is False
|
||||
|
||||
|
||||
def test_default_creator_batch_has_no_forced_hour_pause():
|
||||
assert pgy.DEFAULT_BATCH_SIZE == 0
|
||||
assert xingtu.DEFAULT_BATCH_SIZE == 0
|
||||
|
||||
|
||||
def test_repeatable_style_argument_and_selection_keep_every_requested_style():
|
||||
parser = argparse.ArgumentParser()
|
||||
cc.add_repeatable_style_argument(parser, "styles")
|
||||
args = parser.parse_args(["--style", "1", "--style", "3"])
|
||||
styles = [{"index": 1}, {"index": 2}, {"index": 3}]
|
||||
|
||||
assert args.style == [1, 3]
|
||||
assert cc.select_requested_styles(styles, args.style) == [styles[0], styles[2]]
|
||||
with pytest.raises(ValueError, match="99"):
|
||||
cc.select_requested_styles(styles, [1, 99])
|
||||
|
||||
|
||||
def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
|
||||
sequence = [[], [{"title": "标题", "read_count": "1"}], [{"title": "标题", "read_count": "1"}]]
|
||||
monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
|
||||
|
||||
class Page:
|
||||
waits = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
self.waits += 1
|
||||
|
||||
page = Page()
|
||||
assert pgy.parse_cards_stable(page) == [{"title": "标题", "read_count": "1"}]
|
||||
assert page.waits == 2
|
||||
|
||||
|
||||
def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
|
||||
pages = [
|
||||
[{"title": "无关视频", "play_count": "10"}],
|
||||
[{"title": "目标视频标题", "play_count": "20"}],
|
||||
]
|
||||
calls = {"next": 0}
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "parse_videos_stable", lambda _page: pages.pop(0))
|
||||
|
||||
def next_page(_page):
|
||||
calls["next"] += 1
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(xingtu, "go_next_page", next_page)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
|
||||
|
||||
found, _, _ = xingtu.find_videos_for_tasks(
|
||||
object(), [{"record_id": "r1", "target_title": "目标视频标题"}], max_pages=2,
|
||||
)
|
||||
|
||||
assert found["r1"]["play_count"] == "20"
|
||||
assert calls["next"] == 1
|
||||
|
||||
|
||||
def test_xingtu_normalizes_show_items_api_video():
|
||||
card = xingtu.normalize_xingtu_api_item({
|
||||
"item_id": 7654321,
|
||||
"item_title": "目标视频标题",
|
||||
"play": 168000,
|
||||
"like": 1200,
|
||||
"comment": 34,
|
||||
"share": 56,
|
||||
"url": "https://www.douyin.com/video/7654321",
|
||||
})
|
||||
|
||||
assert card == {
|
||||
"title": "目标视频标题",
|
||||
"play_count": 168000,
|
||||
"like_count": 1200,
|
||||
"comment_count": 34,
|
||||
"share_count": 56,
|
||||
"href": "https://www.douyin.com/video/7654321",
|
||||
"note_id": "7654321",
|
||||
"source": "show_items_api",
|
||||
}
|
||||
|
||||
|
||||
def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
|
||||
class Response:
|
||||
url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
|
||||
|
||||
def json(self):
|
||||
return {
|
||||
"latest_item_info": [
|
||||
{"item_id": "1", "title": "视频一", "play": 10},
|
||||
{"item_id": "2", "item_title": "视频二", "play": 20},
|
||||
],
|
||||
"latest_star_item_info": [
|
||||
{"item_id": "2", "item_title": "视频二", "play": 20},
|
||||
],
|
||||
}
|
||||
|
||||
class Page:
|
||||
def on(self, event, callback):
|
||||
assert event == "response"
|
||||
self.callback = callback
|
||||
|
||||
page = Page()
|
||||
cards = xingtu.setup_show_items_capture(page)
|
||||
page.callback(Response())
|
||||
|
||||
assert [card["note_id"] for card in cards] == ["1", "2"]
|
||||
|
||||
|
||||
def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: (_ for _ in ()).throw(AssertionError("DOM should not be needed")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"is_blocked",
|
||||
lambda _page: (_ for _ in ()).throw(AssertionError("page should not be needed")),
|
||||
)
|
||||
tasks = [{
|
||||
"record_id": "r1",
|
||||
"target_title": "飞书旧标题",
|
||||
"note_url": "https://www.douyin.com/video/7654321",
|
||||
"note_id": "7654321",
|
||||
}]
|
||||
api_cards = [{
|
||||
"title": "平台上的新标题",
|
||||
"play_count": 168000,
|
||||
"note_id": "7654321",
|
||||
"source": "show_items_api",
|
||||
}]
|
||||
|
||||
found, _, page_limit_hit = xingtu.find_videos_for_tasks(
|
||||
object(), tasks, max_pages=5, api_cards=api_cards,
|
||||
)
|
||||
|
||||
assert found["r1"]["play_count"] == 168000
|
||||
assert found["r1"]["match_method"] == "content_id"
|
||||
assert page_limit_hit is False
|
||||
|
||||
|
||||
def test_xingtu_api_fuzzy_candidate_does_not_preempt_later_exact_dom(monkeypatch):
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: [{"title": "commuter backpack review", "note_id": "exact-card"}],
|
||||
)
|
||||
monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
|
||||
task = {"record_id": "r1", "target_title": "commuter backpack review"}
|
||||
api_cards = [{
|
||||
"title": "commuter backpack review today",
|
||||
"note_id": "fuzzy-card",
|
||||
"source": "show_items_api",
|
||||
}]
|
||||
|
||||
found, _, _ = xingtu.find_videos_for_tasks(
|
||||
object(), [task], max_pages=2, api_cards=api_cards,
|
||||
)
|
||||
|
||||
assert found["r1"]["note_id"] == "exact-card"
|
||||
|
||||
|
||||
def test_browser_session_loss_detection_is_specific():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
assert pgy.is_browser_session_lost(closed)
|
||||
assert xingtu.is_browser_session_lost(closed)
|
||||
assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
|
||||
|
||||
|
||||
def test_xingtu_nested_session_loss_is_reraised():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.reraise_if_browser_session_lost(closed)
|
||||
xingtu.reraise_if_browser_session_lost(RuntimeError("ordinary lookup miss"))
|
||||
|
||||
|
||||
def test_xingtu_detail_and_video_search_propagate_closed_session():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
class Detail:
|
||||
def goto(self, *_args, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
def new_page(self):
|
||||
return Detail()
|
||||
|
||||
class SearchPage:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, _script, _arg=None):
|
||||
return {"found": True, "url": "https://www.xingtu.cn/ad/creator/author-homepage/douyin-video/1"}
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.open_creator_detail(SearchPage(), "达人", "creator-id")
|
||||
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class VideoPage:
|
||||
def locator(self, _selector):
|
||||
return Locator()
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.submit_video_search(VideoPage(), "标题")
|
||||
|
||||
|
||||
def test_pgy_fallback_detail_click_propagates_closed_session():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
class Clicker:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def click(self, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
class Page:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, *_args, **_kwargs):
|
||||
return {"ok": False, "reason": "no-row"}
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Clicker()
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
pgy.open_blogger_detail(Page(), "达人", None)
|
||||
|
||||
|
||||
def test_bilibili_triggered_blocked_inputs_are_visible_terminal_rows():
|
||||
summary = bili.finalize_bili_summary({
|
||||
"style": "款式A", "index": 1, "total_b_records": 1,
|
||||
"details": [{
|
||||
"record_id": "r1", "status": "blocked_input",
|
||||
"matched": False, "reason": "publish_time_missing", "ok": False,
|
||||
}],
|
||||
})
|
||||
|
||||
assert summary["blocked_input"] == 1
|
||||
assert summary["unresolved"] == 0
|
||||
assert summary["complete"] is True
|
||||
|
||||
|
||||
def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
|
||||
style = {
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"read_count_7d": {"field_id": "s7"},
|
||||
"read_count_14d": {"field_id": "s14"},
|
||||
"read_count_21d": {"field_id": "s21"},
|
||||
"read_count_28d": {"field_id": "s28"},
|
||||
"month_end": {"field_id": "sm"},
|
||||
},
|
||||
}
|
||||
rows = [
|
||||
{
|
||||
"record_id": "no-link",
|
||||
"platform": "B站",
|
||||
"creator": "未发布达人",
|
||||
"url": "",
|
||||
"pub": "2026-07-20",
|
||||
},
|
||||
{
|
||||
"record_id": "published",
|
||||
"platform": "B站",
|
||||
"creator": "已发布达人",
|
||||
"url": "https://www.bilibili.com/video/BV1234567890",
|
||||
"pub": "2026-07-20",
|
||||
},
|
||||
]
|
||||
monkeypatch.setattr(bili, "list_all_records", lambda *_args: rows)
|
||||
monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: 321)
|
||||
monkeypatch.setattr(bili, "write_record", lambda *_args, **_kwargs: True)
|
||||
|
||||
summary = bili.process_style(
|
||||
style,
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=bili.date(2026, 7, 23),
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
assert summary["source_b_records"] == 2
|
||||
assert summary["skipped_no_url"] == 1
|
||||
assert summary["total_b_records"] == 1
|
||||
assert summary["success"] == 1
|
||||
assert summary["blocked_input"] == 0
|
||||
assert [row["record_id"] for row in summary["details"]] == ["published"]
|
||||
|
||||
|
||||
def test_bilibili_partial_retry_preserves_old_success():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 2,
|
||||
"details": [
|
||||
{"record_id": "ok", "status": "success", "ok": True},
|
||||
{"record_id": "retry", "status": "retryable_failure", "ok": False},
|
||||
],
|
||||
}
|
||||
partial = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 1,
|
||||
"details": [{"record_id": "retry", "status": "success", "ok": True}],
|
||||
}
|
||||
|
||||
merged = bili.merge_bili_summary(existing, partial)
|
||||
|
||||
assert merged["total_b_records"] == 2
|
||||
assert {row["record_id"] for row in merged["details"]} == {"ok", "retry"}
|
||||
assert merged["complete"] is True
|
||||
|
||||
|
||||
def test_bilibili_top_level_error_cannot_finalize_as_complete():
|
||||
finalized = bili.finalize_bili_summary({
|
||||
"style": "款式A",
|
||||
"index": 1,
|
||||
"total_b_records": 0,
|
||||
"details": [],
|
||||
"error": "missing platform/url field_id",
|
||||
"unresolved": 1,
|
||||
"retryable_failures": 1,
|
||||
"complete": False,
|
||||
})
|
||||
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] >= 1
|
||||
|
||||
|
||||
def test_bilibili_missing_slot_mapping_is_system_error_not_blocked_input():
|
||||
summary = bili.process_style(
|
||||
{
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
},
|
||||
},
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=None,
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
finalized = bili.finalize_bili_summary(summary, dry_run=True)
|
||||
assert "publish_time" in finalized["error"]
|
||||
assert "read_count_7d" in finalized["error"]
|
||||
assert finalized["complete"] is False
|
||||
|
||||
|
||||
def test_self_douyin_missing_mapping_and_write_failure_are_not_complete(monkeypatch):
|
||||
missing = self_dy.scrape_one_style(
|
||||
{"name": "款式A", "index": 1, "base_token": "base", "table_id": "table", "field_map": {}},
|
||||
login_timeout=1,
|
||||
headless=True,
|
||||
only_record_ids=None,
|
||||
dry_run=False,
|
||||
)
|
||||
assert cc.finalize_summary(missing)["complete"] is False
|
||||
|
||||
style = _style_for_platform("抖音")
|
||||
row = {
|
||||
"record_id": "r1",
|
||||
"platform": "抖音",
|
||||
"name": "自营达人",
|
||||
"title": "目标作品",
|
||||
"url": "https://www.douyin.com/video/1",
|
||||
"pub": "2026-07-20",
|
||||
}
|
||||
|
||||
class Page:
|
||||
def goto(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
class Session:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def fetch(self, _url, page_action, wait):
|
||||
page_action(Page())
|
||||
|
||||
monkeypatch.setattr(self_dy, "DynamicSession", Session)
|
||||
monkeypatch.setattr(self_dy, "list_records_by_table", lambda *_: [row])
|
||||
monkeypatch.setattr(self_dy, "dismiss_popups", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "restore_cookies", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "maybe_wait_for_login", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "save_state", lambda *_: None)
|
||||
monkeypatch.setattr(
|
||||
self_dy,
|
||||
"scroll_and_collect_posts",
|
||||
lambda *_args, **_kwargs: [{"title": "目标作品", "play_count": 123}],
|
||||
)
|
||||
monkeypatch.setattr(self_dy, "pick_read_field", lambda *_: ("read_count_7d", "read7", "7天曝光量"))
|
||||
monkeypatch.setattr(self_dy, "write_back", lambda *_: False)
|
||||
|
||||
failed = cc.finalize_summary(
|
||||
self_dy.scrape_one_style(
|
||||
style,
|
||||
login_timeout=1,
|
||||
headless=True,
|
||||
only_record_ids=None,
|
||||
dry_run=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert failed["results"][0]["status"] == "write_failure"
|
||||
assert failed["complete"] is False
|
||||
|
||||
|
||||
def test_run_all_rejects_stale_or_incomplete_summary(tmp_path: Path):
|
||||
path = tmp_path / "summary.json"
|
||||
path.write_text("[]", encoding="utf-8")
|
||||
started_at = path.stat().st_mtime + 1
|
||||
|
||||
assert run_all.validate_summary_payload([], {1}, path, started_at)[0] is False
|
||||
|
||||
path.write_text('[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}]', encoding="utf-8")
|
||||
assert run_all.validate_summary_payload(
|
||||
[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}],
|
||||
{1}, path, path.stat().st_mtime - 1,
|
||||
)[0] is False
|
||||
Reference in New Issue
Block a user