feat: expand module workflows, dynamic config, notifications and console API

tests/modules/content_marketing/test_collection_completeness.py

@@ -9,6 +9,10 @@ from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
 from gyxx_flow.modules.content_marketing import run_all
 from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy
 from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
+from gyxx_flow.modules.content_marketing.serialized_page_action import (
+    is_transient_browser_error,
+    run_serialized_page_action,
+)
 
 
 def test_normalize_title_handles_nfkc_case_and_zero_width():
@@ -131,6 +135,28 @@ def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
     assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
 
 
+def test_xingtu_can_use_strong_title_when_share_id_is_canonicalized():
+    task = {
+        "record_id": "r1",
+        "target_title": "这家包颜值和实用性真挺高",
+        "note_id": "7665297310758210854",
+    }
+    card = {
+        "title": "这家包颜值和实用性真挺高 #潮流运动 #光影行星",
+        "note_id": "canonical- Xingtu-item-id",
+    }
+
+    matched = cc.match_tasks_to_cards(
+        [task],
+        [card],
+        platform="douyin",
+        allow_mismatched_content_id_title=True,
+    )
+
+    assert matched["r1"]["match_method"] == "title"
+    assert matched["r1"]["match_score"] >= 0.9
+
+
 def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
     cards = [
         {
@@ -354,8 +380,11 @@ class _FakeSearchPage:
 def test_pgy_search_retry_keeps_creator_id_query():
     page = _FakeSearchPage()
 
-    pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
+    found = pgy._ensure_search_results(
+        page, query="xhs-id-123", result_name="达人昵称", max_retries=2
+    )
 
+    assert found is True
     assert page.input.filled == ["xhs-id-123"]
 
 
@@ -376,11 +405,19 @@ def test_apply_matched_task_reports_write_failure(monkeypatch):
     assert outcome["write_ok"] is False
 
 
-def test_default_creator_batch_has_no_forced_hour_pause():
-    assert pgy.DEFAULT_BATCH_SIZE == 0
+def test_pgy_default_creator_batch_rotates_browser_before_memory_accumulates():
+    assert pgy.DEFAULT_BATCH_SIZE == 4
     assert xingtu.DEFAULT_BATCH_SIZE == 0
 
 
+@pytest.mark.parametrize("message", [
+    "Page crashed: Out of Memory",
+    "Target page, context or browser has been closed: not enough memory",
+])
+def test_pgy_treats_chromium_memory_exhaustion_as_a_recoverable_session_loss(message):
+    assert pgy.is_browser_session_lost(RuntimeError(message)) is True
+
+
 def test_repeatable_style_argument_and_selection_keep_every_requested_style():
     parser = argparse.ArgumentParser()
     cc.add_repeatable_style_argument(parser, "styles")
@@ -408,6 +445,223 @@ def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
     assert page.waits == 2
 
 
+def test_pgy_stable_cards_waits_through_a_slow_initial_render(monkeypatch):
+    card = {"title": "标题", "read_count": "1"}
+    sequence = [[], [], [], [], [], [card], [card]]
+    monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
+
+    class Page:
+        waits = 0
+
+        def wait_for_timeout(self, _ms):
+            self.waits += 1
+
+    page = Page()
+    assert pgy.parse_cards_stable(page) == [card]
+    assert page.waits == 6
+
+
+def test_pgy_card_parser_recovers_note_id_from_internal_card_link():
+    note_id = "0123456789abcdef01234567"
+
+    class Page:
+        def evaluate(self, _script):
+            return [{
+                "title": "标题",
+                "read_count": "1",
+                "href": "",
+                "note_id": "",
+                "_href_candidates": [
+                    f"https://pgy.xiaohongshu.com/internal?source_note_id={note_id}"
+                ],
+                "_data_note_ids": [],
+            }]
+
+    cards = pgy.parse_cards_on_page(Page())
+
+    assert cards == [{
+        "title": "标题",
+        "read_count": "1",
+        "href": "",
+        "note_id": note_id,
+    }]
+
+
+def test_pgy_card_parser_prefers_strict_note_detail_id_over_generic_segment():
+    note_id = "0123456789abcdef01234567"
+    internal_href = f"https://example.invalid/note/detail/{note_id}"
+
+    class Page:
+        def evaluate(self, _script):
+            return [{
+                "title": "标题",
+                "read_count": "1",
+                "href": internal_href,
+                # This is what the old generic /note/<segment> parser produced.
+                "note_id": "detail",
+                "_href_candidates": [internal_href],
+                "_data_note_ids": [],
+            }]
+
+    cards = pgy.parse_cards_on_page(Page())
+
+    assert cards[0]["note_id"] == note_id
+
+
+def test_pgy_dom_parser_reads_root_data_id_and_rejects_arbitrary_href_fallback():
+    captured = {}
+
+    class Page:
+        def evaluate(self, script):
+            captured["script"] = script
+            return []
+
+    assert pgy.parse_cards_on_page(Page()) == []
+    assert "const dataNoteNodes = [card, ...card.querySelectorAll" in captured["script"]
+    assert "const href = link?.href || '';" in captured["script"]
+    assert "hrefCandidates[0]" not in captured["script"]
+
+
+def test_pgy_arbitrary_shared_anchor_does_not_collapse_distinct_cards():
+    shared_anchor = "https://example.invalid/creator/profile"
+
+    class Page:
+        def evaluate(self, _script):
+            return [
+                {
+                    "title": "第一篇笔记",
+                    "read_count": "1",
+                    "like_count": "1",
+                    "collect_count": "1",
+                    "publish_time": "2026-08-01",
+                    "href": "",
+                    "note_id": "",
+                    "_href_candidates": [shared_anchor],
+                    "_data_note_ids": [],
+                },
+                {
+                    "title": "第二篇笔记",
+                    "read_count": "2",
+                    "like_count": "2",
+                    "collect_count": "2",
+                    "publish_time": "2026-08-02",
+                    "href": "",
+                    "note_id": "",
+                    "_href_candidates": [shared_anchor],
+                    "_data_note_ids": [],
+                },
+            ]
+
+    cards = pgy.parse_cards_on_page(Page())
+
+    assert [card["href"] for card in cards] == ["", ""]
+    assert pgy._card_observation_key(cards[0]) != pgy._card_observation_key(cards[1])
+
+
+def test_pgy_deduplicates_one_card_repeated_across_pages_before_matching(monkeypatch):
+    card = {
+        "title": "同一篇笔记",
+        "read_count": "100",
+        "like_count": "10",
+        "collect_count": "5",
+        "publish_time": "2026-08-01",
+        "href": "",
+        "note_id": "",
+    }
+    pages = [[dict(card)], [dict(card)], [dict(card)]]
+    monkeypatch.setattr(pgy, "parse_cards_stable", lambda _page: pages.pop(0))
+    monkeypatch.setattr(pgy, "go_next_page", lambda _page: bool(pages))
+
+    found, candidates, page_limit_hit = pgy.find_notes_for_tasks(
+        object(),
+        [{"record_id": "r1", "target_title": "同一篇笔记"}],
+        max_pages=4,
+    )
+
+    assert found["r1"]["read_count"] == "100"
+    assert candidates == ["同一篇笔记"]
+    assert page_limit_hit is False
+
+
+def test_pgy_missing_creator_is_blocked_but_unmatched_note_stays_retryable():
+    no_creator = pgy._missing_detail_outcome(saw_search_result=False)
+    no_unique_note = pgy._unmatched_note_outcome(
+        candidates=["平台上的其他笔记"],
+        page_limit_hit=False,
+    )
+    completed = cc.finalize_summary({
+        "total": 2,
+        "results": [
+            {"record_id": "creator", **no_creator},
+            {"record_id": "note", **no_unique_note},
+        ],
+    })
+
+    assert completed["blocked_input"] == 1
+    assert completed["retryable_failures"] == 1
+    assert completed["unresolved"] == 1
+    assert completed["complete"] is False
+    assert pgy._missing_detail_outcome(saw_search_result=True)["status"] == "retryable_failure"
+    assert pgy._unmatched_note_outcome(
+        candidates=[], page_limit_hit=False
+    )["reason"] == "empty_detail_page"
+    assert pgy._unmatched_note_outcome(
+        candidates=["候选"], page_limit_hit=True
+    )["status"] == "retryable_failure"
+
+
+@pytest.mark.parametrize(
+    "state",
+    [
+        {
+            "nextFound": True,
+            "nextDisabled": True,
+            "activePage": None,
+            "lastPage": None,
+            "numericPageCount": 0,
+        },
+        {
+            "nextFound": True,
+            "nextDisabled": False,
+            "activePage": 7,
+            "lastPage": 7,
+            "numericPageCount": 7,
+        },
+    ],
+)
+def test_pgy_known_last_page_stops_without_clicking(state):
+    class Page:
+        def evaluate(self, script):
+            if "pgy-pagination-state" in script:
+                return state
+            if "pgy-pagination-click" in script:
+                pytest.fail("terminal paginator must not be clicked")
+            return "last-page-card-signature"
+
+    assert pgy.go_next_page(Page()) is False
+
+
+def test_pgy_clicked_pagination_without_card_change_uses_source_end_semantics():
+    class Page:
+        def evaluate(self, script):
+            if "pgy-pagination-state" in script:
+                return {
+                    "nextFound": True,
+                    "nextDisabled": False,
+                    "activePage": 2,
+                    "lastPage": 7,
+                    "numericPageCount": 7,
+                }
+            if "pgy-pagination-click" in script:
+                return {"clicked": True, "text": "next"}
+            return "same-card-signature"
+
+        def wait_for_timeout(self, _ms):
+            return None
+
+    assert pgy.go_next_page(Page()) is False
+
+
 def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
     pages = [
         [{"title": "无关视频", "play_count": "10"}],
@@ -432,6 +686,125 @@ def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
     assert calls["next"] == 1
 
 
+def test_xingtu_video_search_uses_visible_input_and_source_keyboard_events():
+    events = []
+
+    class Locator:
+        first = None
+
+        def __init__(self):
+            self.first = self
+
+        def count(self):
+            return 1
+
+        def wait_for(self, **kwargs):
+            events.append(("wait", kwargs))
+
+        def click(self):
+            events.append(("click",))
+
+        def press(self, key):
+            events.append(("locator_press", key))
+
+        def fill(self, _value):
+            pytest.fail("Xingtu search must preserve source key-by-key input")
+
+    class Keyboard:
+        def type(self, value, **kwargs):
+            events.append(("type", value, kwargs))
+
+        def press(self, key):
+            events.append(("keyboard_press", key))
+
+    class Page:
+        keyboard = Keyboard()
+
+        def __init__(self):
+            self.selector = None
+            self.input = Locator()
+
+        def locator(self, selector):
+            self.selector = selector
+            return self.input
+
+    page = Page()
+
+    assert xingtu.submit_video_search(page, "目标标题") is True
+    assert ":visible" in page.selector
+    assert events == [
+        ("wait", {"state": "visible", "timeout": 5000}),
+        ("click",),
+        ("locator_press", "Control+A"),
+        ("locator_press", "Backspace"),
+        ("type", "目标标题", {"delay": 20}),
+        ("keyboard_press", "Enter"),
+    ]
+
+
+def test_xingtu_only_confirmed_creator_miss_is_blocked():
+    no_creator = xingtu._missing_detail_outcome(saw_search_result=False)
+    no_unique_video = xingtu._unmatched_video_outcome(
+        candidates=["平台上的其他视频"],
+        page_limit_hit=False,
+    )
+    completed = cc.finalize_summary({
+        "total": 2,
+        "results": [
+            {"record_id": "creator", **no_creator},
+            {"record_id": "video", **no_unique_video},
+        ],
+    })
+
+    assert completed["blocked_input"] == 1
+    assert completed["retryable_failures"] == 1
+    assert completed["unresolved"] == 1
+    assert completed["complete"] is False
+    assert no_unique_video["status"] == "retryable_failure"
+    assert xingtu._missing_detail_outcome(
+        saw_search_result=True
+    )["status"] == "retryable_failure"
+    assert xingtu._unmatched_video_outcome(
+        candidates=[], page_limit_hit=False
+    )["reason"] == "empty_detail_page"
+    assert xingtu._unmatched_video_outcome(
+        candidates=["候选"], page_limit_hit=True
+    )["status"] == "retryable_failure"
+
+
+def test_xingtu_missing_creator_result_has_distinct_business_exception():
+    class Context:
+        pages = []
+
+    class Page:
+        context = Context()
+
+        def evaluate(self, _script, argument=None):
+            if argument is not None:
+                return {"found": False}
+            return False
+
+        def wait_for_timeout(self, _milliseconds):
+            return None
+
+    with pytest.raises(xingtu.CreatorNotFound, match="未找到精确达人"):
+        xingtu.open_creator_detail(Page(), "不存在的达人", "missing-id")
+
+
+def test_xingtu_clicked_pagination_without_card_change_is_retryable_timeout():
+    class Page:
+        def evaluate(self, script):
+            if "const all" in script:
+                return {"clicked": True, "picked": "pagination"}
+            return "same-card-signature"
+
+        def wait_for_timeout(self, _milliseconds):
+            return None
+
+    with pytest.raises(xingtu.BrowserTimeoutError, match="did not change"):
+        xingtu.go_next_page(Page())
+
+
 def test_xingtu_normalizes_show_items_api_video():
     card = xingtu.normalize_xingtu_api_item({
         "item_id": 7654321,
@@ -456,6 +829,9 @@ def test_xingtu_normalizes_show_items_api_video():
 
 
 def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
+    import gc
+    import weakref
+
     class Response:
         url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
 
@@ -477,9 +853,14 @@ def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
 
     page = Page()
     cards = xingtu.setup_show_items_capture(page)
-    page.callback(Response())
+    response = Response()
+    response_ref = weakref.ref(response)
+    page.callback(response)
+    del response
+    gc.collect()
 
     assert [card["note_id"] for card in cards] == ["1", "2"]
+    assert response_ref() is None
 
 
 def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
@@ -542,9 +923,221 @@ def test_browser_session_loss_detection_is_specific():
     closed = RuntimeError("Target page, context or browser has been closed")
     assert pgy.is_browser_session_lost(closed)
     assert xingtu.is_browser_session_lost(closed)
+    assert pgy.is_browser_session_lost(RuntimeError("Page crashed"))
+    assert xingtu.is_browser_session_lost(RuntimeError("Page crashed"))
+    assert pgy.is_browser_session_lost(
+        RuntimeError("Execution context was destroyed, most likely because of a navigation.")
+    )
     assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
 
 
+def test_transient_browser_error_excludes_business_timeout():
+    assert is_transient_browser_error(
+        pgy.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded")
+    )
+    assert is_transient_browser_error(
+        RuntimeError("Page.goto: net::ERR_CONNECTION_RESET")
+    )
+    assert not is_transient_browser_error(
+        TimeoutError("Login timed out after 300 seconds.")
+    )
+
+
+def _stub_creator_session_failure(monkeypatch, tmp_path, module, error_factory):
+    style = {"index": 1, "name": "style"}
+    task = {
+        "record_id": "record-1",
+        "creator_name": "creator",
+        "creator_id": None,
+        "target_title": "target",
+        "style_context": {"index": 1},
+    }
+    summary = {
+        "index": 1,
+        "name": "style",
+        "total": 1,
+        "filled": 0,
+        "results": [],
+    }
+    group = {
+        "creator_name": "creator",
+        "creator_id": None,
+        "tasks": [task],
+    }
+    sessions = []
+
+    class Session:
+        def __init__(self, **_kwargs):
+            sessions.append(self)
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *_args):
+            return False
+
+    def fail_page_action(*_args, **_kwargs):
+        raise error_factory()
+
+    monkeypatch.setattr(
+        module,
+        "collect_tasks_across_styles",
+        lambda *_args, **_kwargs: ([task], {1: summary}),
+    )
+    monkeypatch.setattr(module, "group_tasks_by_creator", lambda _tasks: [group])
+    monkeypatch.setattr(module, "chunk_creator_groups", lambda groups, _size: [groups])
+    monkeypatch.setattr(module, "DynamicSession", Session)
+    monkeypatch.setattr(module, "run_serialized_page_action", fail_page_action)
+    monkeypatch.setattr(module, "_force_cleanup_session", lambda _session: None)
+    monkeypatch.setattr(module, "PROFILE_DIR", tmp_path / module.__name__.split(".")[-1])
+    return style, sessions
+
+
+@pytest.mark.parametrize("module", [pgy, xingtu])
+def test_creator_collectors_retry_browser_timeout_in_three_fresh_sessions(
+    monkeypatch, tmp_path, module
+):
+    style, sessions = _stub_creator_session_failure(
+        monkeypatch,
+        tmp_path,
+        module,
+        lambda: module.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded"),
+    )
+
+    summaries = module.scrape_styles([style], 1, True, None, True)
+
+    assert len(sessions) == 3
+    assert summaries[0]["complete"] is False
+    assert summaries[0]["retryable_failures"] == 1
+
+
+@pytest.mark.parametrize("module", [pgy, xingtu])
+def test_creator_collectors_do_not_retry_acceptance_terminal(
+    monkeypatch, tmp_path, module
+):
+    style, sessions = _stub_creator_session_failure(
+        monkeypatch,
+        tmp_path,
+        module,
+        lambda: module.AcceptanceCookieSkip("cookie unavailable in acceptance"),
+    )
+
+    with pytest.raises(module.AcceptanceCookieSkip):
+        module.scrape_styles([style], 1, True, None, True)
+
+    assert len(sessions) == 1
+
+
+@pytest.mark.parametrize("module", [pgy, xingtu])
+def test_creator_collectors_do_not_retry_login_timeout_terminal(
+    monkeypatch, tmp_path, module
+):
+    style, sessions = _stub_creator_session_failure(
+        monkeypatch,
+        tmp_path,
+        module,
+        lambda: TimeoutError("Login timed out after 300 seconds."),
+    )
+
+    with pytest.raises(TimeoutError, match="Login timed out"):
+        module.scrape_styles([style], 1, True, None, True)
+
+    assert len(sessions) == 1
+
+
+@pytest.mark.parametrize("error_type", [xingtu.SearchQuotaExhausted, xingtu.CaptchaBlocked])
+def test_xingtu_does_not_retry_explicit_business_terminal(
+    monkeypatch, tmp_path, error_type
+):
+    style, sessions = _stub_creator_session_failure(
+        monkeypatch,
+        tmp_path,
+        xingtu,
+        lambda: error_type("explicit platform terminal"),
+    )
+
+    with pytest.raises(error_type):
+        xingtu.scrape_styles([style], 1, True, None, True)
+
+    assert len(sessions) == 1
+
+
+@pytest.mark.parametrize("module", [pgy, xingtu])
+def test_creator_collectors_with_no_tasks_do_not_open_browser(
+    monkeypatch, module
+):
+    style = {"index": 1, "name": "style"}
+    summary = {"index": 1, "name": "style", "total": 0, "results": []}
+    monkeypatch.setattr(
+        module,
+        "collect_tasks_across_styles",
+        lambda *_args, **_kwargs: ([], {1: summary}),
+    )
+    monkeypatch.setattr(
+        module,
+        "DynamicSession",
+        lambda **_kwargs: pytest.fail("no browser session expected"),
+    )
+
+    assert module.scrape_styles([style], 1, True, None, True) == [summary]
+
+
+def test_long_page_action_does_not_retain_navigation_response():
+    import gc
+    import weakref
+
+    response_ref = None
+
+    class Response:
+        url = "https://example.test/final"
+        status = 200
+        headers = {"content-type": "text/html"}
+
+    class Page:
+        closed = False
+
+        def set_default_timeout(self, _timeout):
+            pass
+
+        def set_default_navigation_timeout(self, _timeout):
+            pass
+
+        def goto(self, *_args, **_kwargs):
+            nonlocal response_ref
+            response = Response()
+            response_ref = weakref.ref(response)
+            return response
+
+        def is_closed(self):
+            return self.closed
+
+        def close(self):
+            self.closed = True
+
+    page = Page()
+
+    class Context:
+        def new_page(self):
+            return page
+
+    class Session:
+        context = Context()
+
+    def action(_page):
+        gc.collect()
+        assert response_ref is not None
+        assert response_ref() is None
+
+    navigation = run_serialized_page_action(Session(), "https://example.test", action)
+
+    assert navigation == {
+        "url": "https://example.test/final",
+        "status": 200,
+        "headers": {"content-type": "text/html"},
+    }
+    assert page.closed is True
+
+
 def test_xingtu_nested_session_loss_is_reraised():
     closed = RuntimeError("Target page, context or browser has been closed")
     with pytest.raises(RuntimeError, match="has been closed"):
@@ -694,6 +1287,19 @@ def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
     assert [row["record_id"] for row in summary["details"]] == ["published"]
 
 
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        ("2026-07-20T00:00:00.000+08:00", bili.date(2026, 7, 20)),
+        ("2026-07-20T00:00:00+08:00", bili.date(2026, 7, 20)),
+        ("2026-07-19T16:00:00.000Z", bili.date(2026, 7, 19)),
+        ("2026-07-20", bili.date(2026, 7, 20)),
+    ],
+)
+def test_bilibili_parse_pub_date_accepts_feishu_iso_values(raw, expected):
+    assert bili.parse_pub_date(raw) == expected
+
+
 def test_bilibili_partial_retry_preserves_old_success():
     existing = {
         "style": "款式A", "index": 1, "total_b_records": 2,