feat: expand module workflows, dynamic config, notifications and console API

This commit is contained in:
2026-09-04 09:49:37 +08:00
parent 8df5266abb
commit b124d757b0
309 changed files with 89358 additions and 6232 deletions
@@ -0,0 +1,339 @@
import json
from types import SimpleNamespace
import pytest
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
from gyxx_flow.modules.content_marketing import scrapling_http
from gyxx_flow.modules.content_marketing import self_bilibili_scraper as self_bili
class FakeResponse:
def __init__(
self,
*,
status: int = 200,
payload: dict | None = None,
text: str | None = None,
content_type: str = "application/json; charset=utf-8",
json_error: ValueError | None = None,
) -> None:
self.status_code = status
self._payload = payload
self.text = json.dumps(payload) if text is None and payload is not None else (text or "")
self.content = self.text.encode("utf-8")
self.headers = {"content-type": content_type}
self._json_error = json_error
def json(self):
if self._json_error is not None:
raise self._json_error
return self._payload
class SequenceSession:
def __init__(self, outcomes):
self.outcomes = list(outcomes)
self.calls = []
def get(self, url, **kwargs):
self.calls.append((url, kwargs))
outcome = self.outcomes.pop(0)
if isinstance(outcome, BaseException):
raise outcome
return outcome
def test_scrapling_http_session_maps_fetcher_response(monkeypatch):
calls = []
def get(url, **kwargs):
calls.append((url, kwargs))
return SimpleNamespace(
status=200,
headers={"content-type": "application/json"},
url="https://www.bilibili.com/video/BV17NNw6jEPx",
body=b'{"code": 0}',
encoding="utf-8",
)
monkeypatch.setattr(scrapling_http.Fetcher, "get", get)
session = scrapling_http.ScraplingHttpSession(headers={"User-Agent": "gyxx"})
response = session.get(
"https://b23.tv/demo",
allow_redirects=True,
timeout=10,
)
assert calls == [(
"https://b23.tv/demo",
{
"headers": {"User-Agent": "gyxx"},
"follow_redirects": "all",
"retries": 1,
"timeout": 10,
},
)]
assert response.status_code == 200
assert response.json() == {"code": 0}
assert response.url.endswith("BV17NNw6jEPx")
def test_self_bilibili_uses_scrapling_user_agent_instead_of_stale_override():
session = SequenceSession([
FakeResponse(
payload={
"code": 0,
"data": {"stat": {"view": 456}, "pubdate": 1_700_000_000},
}
)
])
result = self_bili.fetch_bili_stat(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result == (456, 1_700_000_000)
assert session.calls == [(
"https://api.bilibili.com/x/web-interface/view",
{
"params": {"bvid": "BV17NNw6jEPx"},
"timeout": 10,
"headers": {"Referer": "https://www.bilibili.com/"},
},
)]
def success_response(view: int = 321) -> FakeResponse:
return FakeResponse(payload={"code": 0, "message": "OK", "data": {"stat": {"view": view}}})
def detail_success_response(view: int = 321) -> FakeResponse:
return FakeResponse(
payload={"code": 0, "message": "OK", "data": {"View": {"stat": {"view": view}}}}
)
def test_empty_body_retries_then_returns_play_count(monkeypatch, capsys):
session = SequenceSession([FakeResponse(text=""), success_response(456)])
sleeps = []
monkeypatch.setattr(bili.time, "sleep", sleeps.append)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result == 456
assert len(session.calls) == 2
assert sleeps == [1.5]
assert "[empty_body]" in capsys.readouterr().out
@pytest.mark.parametrize(
("first_response", "category"),
[
(FakeResponse(status=412, text="request blocked", content_type="text/plain"), "http_412"),
(FakeResponse(payload={"code": -509, "message": "request limit"}), "api_code_-509"),
(
FakeResponse(
text="<html>temporary gateway page</html>",
content_type="text/html",
json_error=ValueError("not json"),
),
"invalid_json",
),
],
)
def test_transient_response_classes_are_retried(
monkeypatch, capsys, first_response, category
):
session = SequenceSession([first_response, success_response()])
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result == 321
assert len(session.calls) == 2
assert f"[{category}]" in capsys.readouterr().out
def test_non_retryable_api_code_stops_after_one_attempt(monkeypatch, capsys):
session = SequenceSession(
[FakeResponse(payload={"code": -404, "message": "video not found"})]
)
monkeypatch.setattr(
bili.time,
"sleep",
lambda _seconds: pytest.fail("terminal API result must not back off"),
)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result is None
assert len(session.calls) == 1
assert "[api_code_-404]" in capsys.readouterr().out
def test_transient_failures_exhaust_three_attempts_honestly(monkeypatch, capsys):
session = SequenceSession(
[
*[FakeResponse(text="") for _ in range(3)],
FakeResponse(status=412, text="fallback blocked", content_type="text/plain"),
]
)
sleeps = []
monkeypatch.setattr(bili.time, "sleep", sleeps.append)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result is None
assert len(session.calls) == 4
assert sleeps == [1.5, 3.0]
output = capsys.readouterr().out
assert "attempt=3/3" in output
assert "[view_detail_http_412]" in output
def test_view_detail_is_used_once_after_primary_transient_exhaustion(
monkeypatch, capsys
):
session = SequenceSession(
[*[FakeResponse(text="") for _ in range(3)], detail_success_response(789)]
)
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result == 789
assert len(session.calls) == 4
assert session.calls[-1][0] == bili._VIEW_DETAIL_API_URL
assert "[FALLBACK]" in capsys.readouterr().out
def test_play_count_failure_remains_an_incomplete_summary(monkeypatch):
style = {
"style": "款式A",
"name": "款式A",
"index": 1,
"base_token": "base",
"table_id": "table",
"field_map": {
"platform": {"field_id": "platform"},
"note_url": {"field_id": "url"},
"creator_name": {"field_id": "creator"},
"publish_time": {"field_id": "pub"},
"read_count_7d": {"field_id": "s7"},
"read_count_14d": {"field_id": "s14"},
"read_count_21d": {"field_id": "s21"},
"read_count_28d": {"field_id": "s28"},
"month_end": {"field_id": "sm"},
},
}
monkeypatch.setattr(
bili,
"list_all_records",
lambda *_args: [{
"record_id": "failed",
"platform": "B站",
"creator": "达人",
"url": "https://www.bilibili.com/video/BV17NNw6jEPx",
"pub": "2026-08-08",
}],
)
monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: None)
monkeypatch.setattr(
bili,
"write_record",
lambda *_args, **_kwargs: pytest.fail("failed fetch must not write"),
)
summary = bili.process_style(
style,
only_record_ids=None,
dry_run=True,
delay=0,
session=object(),
state={},
force_today=bili.date(2026, 8, 9),
first_run=False,
)
assert summary["complete"] is False
assert summary["retryable_failures"] == 1
assert summary["details"][0]["reason"] == "play_count_fetch_failed"
def test_terminal_unavailable_detail_is_a_complete_blocked_input(monkeypatch):
style = {
"style": "款式A",
"name": "款式A",
"index": 1,
"base_token": "base",
"table_id": "table",
"field_map": {
"platform": {"field_id": "platform"},
"note_url": {"field_id": "url"},
"creator_name": {"field_id": "creator"},
"publish_time": {"field_id": "pub"},
"read_count_7d": {"field_id": "s7"},
"read_count_14d": {"field_id": "s14"},
"read_count_21d": {"field_id": "s21"},
"read_count_28d": {"field_id": "s28"},
"month_end": {"field_id": "sm"},
},
}
monkeypatch.setattr(
bili,
"list_all_records",
lambda *_args: [{
"record_id": "unavailable",
"platform": "B站",
"creator": "达人",
"url": "https://www.bilibili.com/video/BV17NNw6jEPx",
"pub": "2026-08-08",
}],
)
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(
bili,
"write_record",
lambda *_args, **_kwargs: pytest.fail("unavailable video must not write"),
)
session = SequenceSession([
*[FakeResponse(status=412, text="request blocked") for _ in range(3)],
FakeResponse(payload={"code": 62002, "message": "稿件不可见", "data": None}),
])
summary = bili.process_style(
style,
only_record_ids=None,
dry_run=True,
delay=0,
session=session,
state={},
force_today=bili.date(2026, 8, 9),
first_run=False,
)
assert len(session.calls) == 4
assert summary["complete"] is True
assert summary["blocked_input"] == 1
assert summary["retryable_failures"] == 0
assert summary["unresolved"] == 0
assert summary["details"][0]["status"] == "blocked_input"
assert summary["details"][0]["reason"] == "video_unavailable"
@@ -0,0 +1,346 @@
from __future__ import annotations
import datetime as dt
from argparse import Namespace
from pathlib import Path
import pytest
from gyxx_flow.modules.content_marketing import run_all
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
from gyxx_flow.modules.content_marketing.data.tools import retry_failed
from gyxx_flow.modules.content_marketing.data.tools import (
sync_metrics_to_cmt_notes as sync_metrics,
)
@pytest.fixture(autouse=True)
def reset_content_source():
xingtu.configure_source("xingtu")
yield
xingtu.configure_source("xingtu")
def test_buyin_source_uses_doudian_market_and_cookie_scope():
xingtu.configure_source("buyin")
assert xingtu.CONTENT_SOURCE == "buyin"
assert xingtu.MARKET_URL.startswith(
"https://buyin.jinritemai.com/dashboard/servicehall/daren-square"
)
assert xingtu.DETAIL_URL_MARK == "/dashboard/servicehall/daren-profile"
assert xingtu.BUSINESS_COOKIE_DOMAINS == ("jinritemai.com",)
def test_save_state_creates_cookie_parent_directory(tmp_path, monkeypatch):
cookie_file = tmp_path / "nested" / "cookies.json"
monkeypatch.setattr(xingtu, "DATA_DIR", tmp_path / "raw")
monkeypatch.setattr(xingtu, "COOKIE_FILE", cookie_file)
class Context:
def cookies(self):
return [{"name": "sessionid", "value": "redacted"}]
class Page:
context = Context()
xingtu.save_state(Page())
assert cookie_file.exists()
assert "sessionid" in cookie_file.read_text(encoding="utf-8")
def test_run_all_keeps_cooperation_xt_on_xingtu_for_buyin_fallback():
args = Namespace(
dry_run=False,
style=None,
daily_scope=False,
no_retry=False,
)
command = run_all.build_process_command("xt", args)
assert "--buyin" not in command
assert "--skip-field-prepare" in command
def test_retry_failed_keeps_cooperation_xt_on_xingtu_for_buyin_fallback():
assert retry_failed.SCRIPTS["xt"]["extra_args"] == []
def test_buyin_video_parser_is_dom_html_based_and_maps_viewers_to_exposure():
xingtu.configure_source("buyin")
class Page:
def __init__(self):
self.script = ""
def evaluate(self, script):
self.script = script
return [{
"title": "#达人笔记标题",
"play_count": "1,155",
"view_count": "1,155",
"exposure_count": "1,155",
}]
page = Page()
cards = xingtu.parse_videos_on_page(page)
assert cards[0]["play_count"] == "1,155"
assert cards[0]["exposure_count"] == "1,155"
assert "观看人数" in page.script
assert "innerText" in page.script
def test_buyin_history_tab_and_trade_video_filter_are_selected():
xingtu.configure_source("buyin")
events = []
filter_states = {"只看带货视频": True, "只看带货图文": True}
class Locator:
first = None
def __init__(self, name):
self.name = name
self.first = self
def count(self):
return 1
def filter(self, **kwargs):
return Locator(kwargs.get("has_text", self.name))
def click(self, **_kwargs):
events.append(f"click-{self.name}")
if self.name in filter_states:
filter_states[self.name] = False
class Page:
def get_by_text(self, text, **_kwargs):
return Locator(text)
def get_by_role(self, _role, name, **_kwargs):
return Locator(name)
def locator(self, _selector):
return Locator("selector")
def evaluate(self, script, *_args):
events.append(script)
if _args and _args[0] in filter_states:
return filter_states[_args[0]]
return False
def wait_for_timeout(self, milliseconds):
events.append(milliseconds)
assert xingtu.open_creation_ability(Page()) is True
assert "click-历史内容" in events
assert "click-近30天" in events
assert "click-视频" in events
assert "click-只看带货视频" in events
assert "click-只看带货图文" in events
assert any("scrollIntoView" in event for event in events if isinstance(event, str))
def test_buyin_creator_search_clicks_real_button_and_waits_for_changed_results(
monkeypatch,
):
xingtu.configure_source("buyin")
events = []
class Locator:
def __init__(self, kind):
self.kind = kind
self.first = self
def count(self):
return 1
def wait_for(self, **_kwargs):
return None
def click(self, **_kwargs):
events.append(f"click-{self.kind}")
def fill(self, value):
events.append(f"fill-{value}")
def press(self, key):
events.append(f"press-{key}")
class Page:
def locator(self, selector):
if selector.startswith("button.auxo-input-search-button"):
return Locator("button")
return Locator("input")
def evaluate(self, _script):
return "idle"
def get_by_role(self, *_args, **_kwargs):
raise AssertionError("Buyin should use the real search button first")
def wait_for_timeout(self, _milliseconds):
return None
monkeypatch.setattr(xingtu, "dismiss_popups", lambda _page: None)
monkeypatch.setattr(xingtu, "select_nickname_search", lambda _page: None)
monkeypatch.setattr(xingtu, "_buyin_result_page_signature", lambda _page: "before")
observed = {}
def wait_for_results(_page, _name, timeout_ms, previous_signature=None):
observed.update(timeout_ms=timeout_ms, previous_signature=previous_signature)
monkeypatch.setattr(xingtu, "wait_for_visible_creator_result", wait_for_results)
xingtu.search_creator(Page(), "linyl燕", "LinFX2327")
assert events == ["click-input", "fill-LinFX2327", "click-button"]
assert observed == {"timeout_ms": 30000, "previous_signature": "before"}
def test_buyin_search_button_waits_until_loading_finishes():
xingtu.configure_source("buyin")
states = iter(["busy", "busy", "idle"])
waits = []
class Page:
def evaluate(self, _script):
return next(states)
def wait_for_timeout(self, milliseconds):
waits.append(milliseconds)
xingtu._wait_for_buyin_search_button_idle(Page(), timeout_ms=1000)
assert waits == [250, 250]
def test_buyin_wait_distinguishes_temporary_search_throttling():
xingtu.configure_source("buyin")
class Page:
def evaluate(self, *_args):
return "throttled"
def wait_for_timeout(self, _milliseconds):
pytest.fail("throttling must return immediately")
with pytest.raises(xingtu.SearchTemporarilyThrottled, match="过于频繁"):
xingtu.wait_for_visible_creator_result(Page(), "LinFX2327", timeout_ms=1000)
def test_buyin_creator_scan_supports_single_row_without_profile_href():
script = xingtu._buyin_scan_creator_script()
assert "tr[data-row-key], [role=\"row\"][data-row-key]" in script
assert "rowKey: row.getAttribute('data-row-key')" in script
assert "via: 'single-search-result'" in script
assert "first-id-search-result" not in script
def test_buyin_next_page_waits_for_a_changed_html_signature():
xingtu.configure_source("buyin")
signatures = iter(["1/4|first-card", "2/4|second-card"])
class Page:
def evaluate(self, script):
if "const selectors" in script:
return {"clicked": True, "picked": "semi-page-next"}
return next(signatures)
def locator(self, _selector):
class Locator:
first = None
def __init__(self):
self.first = self
def click(self, **_kwargs):
return None
return Locator()
def wait_for_timeout(self, _milliseconds):
return None
assert xingtu.go_next_page(Page()) is True
def test_metric_snapshot_date_prefers_orchestrator_business_date(monkeypatch):
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-31")
assert sync_metrics._resolve_metric_date() == dt.date(2026, 8, 31)
assert sync_metrics._resolve_metric_date("2026-08-30") == dt.date(2026, 8, 30)
with pytest.raises(ValueError, match="YYYY-MM-DD"):
sync_metrics._resolve_metric_date("not-a-date")
def test_metric_snapshot_upsert_is_idempotent_by_note_and_business_date():
class Cursor:
def __init__(self):
self.calls = []
def execute(self, sql, params):
self.calls.append((" ".join(sql.split()), params))
cursor = Cursor()
sync_metrics._upsert_metric_snapshot(
cursor,
42,
{
"view_count": 1155,
"dst_platform": "douyin",
"self_operated": False,
"record_id": "rec-1",
"source_url": "https://www.douyin.com/video/42",
},
dt.date(2026, 8, 31),
)
sql, params = cursor.calls[0]
assert "INSERT INTO cmt_note_metric_snapshots" in sql
assert "ON CONFLICT (note_id, metric_date)" in sql
assert params[0:2] == (42, dt.date(2026, 8, 31))
assert params[6] == 1155
def test_snapshot_schema_and_report_query_use_daily_history():
root = Path(__file__).resolve().parents[3]
schema = (
root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "content_marketing"
/ "data"
/ "tools"
/ "schema_gyxx_super_data.sql"
).read_text(encoding="utf-8")
migration = (
root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "content_marketing"
/ "data"
/ "tools"
/ "migrations"
/ "008_note_metric_snapshots.sql"
).read_text(encoding="utf-8")
report = (
root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "content_marketing"
/ "daily_marketing_report.py"
).read_text(encoding="utf-8")
for source in (schema, migration):
assert "CREATE TABLE IF NOT EXISTS cmt_note_metric_snapshots" in source
assert "UNIQUE (note_id, metric_date)" in source
assert "COALESCE(ms.view_count, n.view_count)" in report
assert "ms.metric_date = %s" in report
@@ -1,7 +1,14 @@
from datetime import datetime
from types import SimpleNamespace
import pytest
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
from gyxx_flow.modules.content_marketing import chanmama_scraper as chanmama
from gyxx_flow.modules.content_marketing import feishu_mapping
from gyxx_flow.modules.content_marketing import (
scrapling_browser_facade as browser_facade,
)
def test_exposure_state_clicks_text_and_zero_but_keeps_positive_values():
@@ -152,6 +159,277 @@ def test_missing_get_data_button_does_not_trigger_refresh_wait(monkeypatch):
assert sleeps == []
def test_create_driver_uses_runtime_cdp_profile_and_download_bindings(monkeypatch, tmp_path):
calls = []
expected = object()
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
def start(**kwargs):
calls.append(kwargs)
return expected
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
assert chanmama.create_driver() is expected
assert calls == [{
"profile_dir": str(tmp_path / "profile"),
"download_dir": str(tmp_path / "downloads"),
"cdp_url": "http://127.0.0.1:22002",
"stealthy": True,
}]
def test_create_driver_starts_scrapling_without_cdp(monkeypatch, tmp_path):
calls = []
expected = object()
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.delenv("GYXX_BROWSER_CDP_URL", raising=False)
def start(**kwargs):
calls.append(kwargs)
return expected
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
assert chanmama.create_driver() is expected
assert calls[0]["cdp_url"] is None
assert calls[0]["stealthy"] is True
assert calls[0]["real_chrome"] is False
def test_create_driver_falls_back_when_bound_cdp_is_not_listening(
monkeypatch,
tmp_path,
):
calls = []
expected = object()
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
def start(**kwargs):
calls.append(kwargs)
if kwargs["cdp_url"]:
raise RuntimeError(
"BrowserType.connect_over_cdp: connect ECONNREFUSED "
"127.0.0.1:22002; retrieving websocket url"
)
return expected
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
assert chanmama.create_driver() is expected
assert [call["cdp_url"] for call in calls] == [
"http://127.0.0.1:22002",
None,
]
assert all(call["stealthy"] is True for call in calls)
assert calls[1]["real_chrome"] is False
def test_create_driver_keeps_explicit_remote_cdp_fail_closed(
monkeypatch,
tmp_path,
):
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://browser.internal:22002")
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
monkeypatch.setattr(
chanmama,
"ScraplingDriver",
SimpleNamespace(
start=lambda **_kwargs: (_ for _ in ()).throw(
RuntimeError("connect ECONNREFUSED browser.internal:22002")
)
),
)
with pytest.raises(RuntimeError, match="ECONNREFUSED"):
chanmama.create_driver()
def test_create_driver_does_not_hide_non_connection_cdp_failures(
monkeypatch,
tmp_path,
):
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
monkeypatch.setattr(
chanmama,
"ScraplingDriver",
SimpleNamespace(
start=lambda **_kwargs: (_ for _ in ()).throw(
RuntimeError("browser profile is corrupt")
)
),
)
with pytest.raises(RuntimeError, match="profile is corrupt"):
chanmama.create_driver()
def test_scrapling_facade_reuses_runtime_cdp_context(monkeypatch, tmp_path):
created = []
class Page:
def on(self, *_args):
return None
def add_init_script(self, script):
assert "navigator" in script
class Context:
pages = [Page()]
class Browser:
def __init__(self, **kwargs):
self.kwargs = kwargs
self.context = Context()
self.page = None
self.closed = False
created.append(self)
def start(self):
return self
def close(self):
self.closed = True
monkeypatch.setattr(browser_facade, "ScraplingBrowser", Browser)
driver = browser_facade.ScraplingDriver.start(
profile_dir=tmp_path / "profile",
download_dir=tmp_path / "downloads",
cdp_url="http://127.0.0.1:22002",
)
assert created[0].kwargs["reuse_existing_cdp_context"] is True
assert created[0].kwargs["cdp_url"] == "http://127.0.0.1:22002"
assert "user_data_dir" not in created[0].kwargs
assert "--no-sandbox" not in created[0].kwargs["extra_flags"]
assert created[0].page is created[0].context.pages[0]
driver.quit()
assert created[0].closed is True
def _one_backfill_style():
return {
"tables": [
{
"index": 1,
"name": "宙斯",
"base_token": "base",
"table_id": "table",
"field_map": {
"platform": {"field_id": "platform"},
"note_title": {"field_id": "title"},
"publish_time": {"field_id": "published"},
"read_count_7d": {
"field_id": "read-7d",
"field_name": "7日阅读量",
},
},
}
]
}
def test_backfill_mapping_or_record_list_failure_returns_false(monkeypatch, tmp_path):
monkeypatch.setattr(
feishu_mapping,
"load_mapping",
lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("mapping down")),
)
assert chanmama.backfill_self_tables([]) is False
monkeypatch.setattr(
feishu_mapping, "load_mapping", lambda **_kwargs: {"tables": []}
)
assert chanmama.backfill_self_tables([]) is False
monkeypatch.setattr(
feishu_mapping, "load_mapping", lambda **_kwargs: _one_backfill_style()
)
assert chanmama.backfill_self_tables([], only_style=999) is False
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(normalized_root=tmp_path / "normalized"),
)
monkeypatch.setattr(
chanmama,
"_list_records_by_table",
lambda *_args: (_ for _ in ()).throw(RuntimeError("base down")),
)
assert chanmama.backfill_self_tables([]) is False
def test_backfill_write_failure_is_false_but_natural_no_match_is_success(
monkeypatch, tmp_path
):
monkeypatch.setattr(
feishu_mapping, "load_mapping", lambda **_kwargs: _one_backfill_style()
)
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(normalized_root=tmp_path / "normalized"),
)
record = {
"record_id": "record-1",
"platform": "抖音",
"title": "相同标题",
"published": datetime.now(),
}
monkeypatch.setattr(
chanmama, "_list_records_by_table", lambda *_args: [record]
)
monkeypatch.setattr(chanmama, "_write_back", lambda *_args: False)
assert chanmama.backfill_self_tables(
[{"title": "相同标题", "exposure": 100}]
) is False
monkeypatch.setattr(
chanmama,
"_write_back",
lambda *_args: (_ for _ in ()).throw(
AssertionError("natural no-match must not write")
),
)
assert chanmama.backfill_self_tables(
[{"title": "完全不同", "exposure": 100}]
) is True
def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
events = []
@@ -159,7 +437,6 @@ def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
def quit(self):
events.append("quit")
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
@@ -180,10 +457,75 @@ def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
lambda *_args, **_kwargs: ([], []),
)
assert chanmama.main() == 0
assert chanmama.main() == 1
assert events == ["saved", "quit"]
def test_main_full_mode_exception_and_keyboard_interrupt_are_nonzero(monkeypatch):
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(
chanmama,
"create_driver",
lambda: (_ for _ in ()).throw(RuntimeError("driver failed")),
)
assert chanmama.main() == 1
monkeypatch.setattr(
chanmama,
"create_driver",
lambda: (_ for _ in ()).throw(KeyboardInterrupt()),
)
assert chanmama.main() == 130
def test_main_propagates_backfill_failure_after_nonempty_export(monkeypatch):
events = []
class Driver:
def quit(self):
events.append("quit")
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
monkeypatch.setattr(chanmama, "save_cookies", lambda _driver: True)
monkeypatch.setattr(
chanmama,
"refresh_then_export_accounts",
lambda *_args, **_kwargs: ([{"title": "video"}], []),
)
monkeypatch.setattr(
chanmama,
"backfill_self_tables",
lambda *_args, **_kwargs: False,
)
assert chanmama.main() == 1
assert events == ["quit"]
def test_backfill_only_propagates_backfill_failure(monkeypatch, tmp_path):
workbook = tmp_path / "existing.xlsx"
workbook.write_bytes(b"placeholder")
monkeypatch.setattr(
chanmama.sys,
"argv",
["chanmama_scraper.py", "--backfill-only", "--excel", str(workbook)],
)
monkeypatch.setattr(
chanmama,
"parse_chanmama_excel",
lambda _path: ([{"title": "video"}], []),
)
monkeypatch.setattr(
chanmama,
"backfill_self_tables",
lambda *_args, **_kwargs: False,
)
assert chanmama.main() == 1
def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
monkeypatch,
tmp_path,
@@ -196,7 +538,6 @@ def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(chanmama, "ACCOUNT", "")
monkeypatch.setattr(chanmama, "PASSWORD", "")
@@ -251,3 +592,85 @@ def test_cli_preserves_cookie_skip_exit_code(monkeypatch):
chanmama.cli()
assert raised.value.code == COOKIE_SKIP_EXIT_CODE
def test_wait_for_exported_excel_polls_until_file_appears(monkeypatch):
now = [1000.0]
sleeps: list[float] = []
results = iter([None, None, "/downloads/博主A_视频记录_周期内发布视频.xlsx"])
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
monkeypatch.setattr(
chanmama.time,
"sleep",
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
)
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
assert chanmama._wait_for_exported_excel(0.0) == (
"/downloads/博主A_视频记录_周期内发布视频.xlsx"
)
assert sleeps == [5.0, 5.0]
def test_wait_for_exported_excel_returns_none_after_timeout(monkeypatch):
now = [1000.0]
sleeps: list[float] = []
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
monkeypatch.setattr(
chanmama.time,
"sleep",
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
)
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: None)
assert (
chanmama._wait_for_exported_excel(
0.0,
timeout_seconds=12,
grace_seconds=0,
)
is None
)
assert len(sleeps) == 3 # 1000 -> 1005 -> 1010 -> 1015(>= 1012 超时)
def test_wait_for_exported_excel_allows_boundary_grace(monkeypatch):
now = [1000.0]
sleeps: list[float] = []
results = iter([None, None, "/downloads/边界后落盘.xlsx"])
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
monkeypatch.setattr(
chanmama.time,
"sleep",
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
)
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
assert chanmama._wait_for_exported_excel(
0.0,
timeout_seconds=5,
grace_seconds=5,
poll_interval_seconds=5,
) == "/downloads/边界后落盘.xlsx"
assert sleeps == [5.0, 5.0]
def test_wait_for_exported_excel_returns_immediately_when_present(monkeypatch):
now = [1000.0]
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
monkeypatch.setattr(
chanmama.time,
"sleep",
lambda _seconds: pytest.fail("不应等待已存在的文件"),
)
monkeypatch.setattr(
chanmama,
"_find_new_excel_after",
lambda _pre_mtime: "/downloads/已有.xlsx",
)
assert chanmama._wait_for_exported_excel(0.0) == "/downloads/已有.xlsx"
@@ -9,6 +9,10 @@ from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
from gyxx_flow.modules.content_marketing import run_all
from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
from gyxx_flow.modules.content_marketing.serialized_page_action import (
is_transient_browser_error,
run_serialized_page_action,
)
def test_normalize_title_handles_nfkc_case_and_zero_width():
@@ -131,6 +135,28 @@ def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
def test_xingtu_can_use_strong_title_when_share_id_is_canonicalized():
task = {
"record_id": "r1",
"target_title": "这家包颜值和实用性真挺高",
"note_id": "7665297310758210854",
}
card = {
"title": "这家包颜值和实用性真挺高 #潮流运动 #光影行星",
"note_id": "canonical- Xingtu-item-id",
}
matched = cc.match_tasks_to_cards(
[task],
[card],
platform="douyin",
allow_mismatched_content_id_title=True,
)
assert matched["r1"]["match_method"] == "title"
assert matched["r1"]["match_score"] >= 0.9
def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
cards = [
{
@@ -354,8 +380,11 @@ class _FakeSearchPage:
def test_pgy_search_retry_keeps_creator_id_query():
page = _FakeSearchPage()
pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
found = pgy._ensure_search_results(
page, query="xhs-id-123", result_name="达人昵称", max_retries=2
)
assert found is True
assert page.input.filled == ["xhs-id-123"]
@@ -376,11 +405,19 @@ def test_apply_matched_task_reports_write_failure(monkeypatch):
assert outcome["write_ok"] is False
def test_default_creator_batch_has_no_forced_hour_pause():
assert pgy.DEFAULT_BATCH_SIZE == 0
def test_pgy_default_creator_batch_rotates_browser_before_memory_accumulates():
assert pgy.DEFAULT_BATCH_SIZE == 4
assert xingtu.DEFAULT_BATCH_SIZE == 0
@pytest.mark.parametrize("message", [
"Page crashed: Out of Memory",
"Target page, context or browser has been closed: not enough memory",
])
def test_pgy_treats_chromium_memory_exhaustion_as_a_recoverable_session_loss(message):
assert pgy.is_browser_session_lost(RuntimeError(message)) is True
def test_repeatable_style_argument_and_selection_keep_every_requested_style():
parser = argparse.ArgumentParser()
cc.add_repeatable_style_argument(parser, "styles")
@@ -408,6 +445,223 @@ def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
assert page.waits == 2
def test_pgy_stable_cards_waits_through_a_slow_initial_render(monkeypatch):
card = {"title": "标题", "read_count": "1"}
sequence = [[], [], [], [], [], [card], [card]]
monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
class Page:
waits = 0
def wait_for_timeout(self, _ms):
self.waits += 1
page = Page()
assert pgy.parse_cards_stable(page) == [card]
assert page.waits == 6
def test_pgy_card_parser_recovers_note_id_from_internal_card_link():
note_id = "0123456789abcdef01234567"
class Page:
def evaluate(self, _script):
return [{
"title": "标题",
"read_count": "1",
"href": "",
"note_id": "",
"_href_candidates": [
f"https://pgy.xiaohongshu.com/internal?source_note_id={note_id}"
],
"_data_note_ids": [],
}]
cards = pgy.parse_cards_on_page(Page())
assert cards == [{
"title": "标题",
"read_count": "1",
"href": "",
"note_id": note_id,
}]
def test_pgy_card_parser_prefers_strict_note_detail_id_over_generic_segment():
note_id = "0123456789abcdef01234567"
internal_href = f"https://example.invalid/note/detail/{note_id}"
class Page:
def evaluate(self, _script):
return [{
"title": "标题",
"read_count": "1",
"href": internal_href,
# This is what the old generic /note/<segment> parser produced.
"note_id": "detail",
"_href_candidates": [internal_href],
"_data_note_ids": [],
}]
cards = pgy.parse_cards_on_page(Page())
assert cards[0]["note_id"] == note_id
def test_pgy_dom_parser_reads_root_data_id_and_rejects_arbitrary_href_fallback():
captured = {}
class Page:
def evaluate(self, script):
captured["script"] = script
return []
assert pgy.parse_cards_on_page(Page()) == []
assert "const dataNoteNodes = [card, ...card.querySelectorAll" in captured["script"]
assert "const href = link?.href || '';" in captured["script"]
assert "hrefCandidates[0]" not in captured["script"]
def test_pgy_arbitrary_shared_anchor_does_not_collapse_distinct_cards():
shared_anchor = "https://example.invalid/creator/profile"
class Page:
def evaluate(self, _script):
return [
{
"title": "第一篇笔记",
"read_count": "1",
"like_count": "1",
"collect_count": "1",
"publish_time": "2026-08-01",
"href": "",
"note_id": "",
"_href_candidates": [shared_anchor],
"_data_note_ids": [],
},
{
"title": "第二篇笔记",
"read_count": "2",
"like_count": "2",
"collect_count": "2",
"publish_time": "2026-08-02",
"href": "",
"note_id": "",
"_href_candidates": [shared_anchor],
"_data_note_ids": [],
},
]
cards = pgy.parse_cards_on_page(Page())
assert [card["href"] for card in cards] == ["", ""]
assert pgy._card_observation_key(cards[0]) != pgy._card_observation_key(cards[1])
def test_pgy_deduplicates_one_card_repeated_across_pages_before_matching(monkeypatch):
card = {
"title": "同一篇笔记",
"read_count": "100",
"like_count": "10",
"collect_count": "5",
"publish_time": "2026-08-01",
"href": "",
"note_id": "",
}
pages = [[dict(card)], [dict(card)], [dict(card)]]
monkeypatch.setattr(pgy, "parse_cards_stable", lambda _page: pages.pop(0))
monkeypatch.setattr(pgy, "go_next_page", lambda _page: bool(pages))
found, candidates, page_limit_hit = pgy.find_notes_for_tasks(
object(),
[{"record_id": "r1", "target_title": "同一篇笔记"}],
max_pages=4,
)
assert found["r1"]["read_count"] == "100"
assert candidates == ["同一篇笔记"]
assert page_limit_hit is False
def test_pgy_missing_creator_is_blocked_but_unmatched_note_stays_retryable():
no_creator = pgy._missing_detail_outcome(saw_search_result=False)
no_unique_note = pgy._unmatched_note_outcome(
candidates=["平台上的其他笔记"],
page_limit_hit=False,
)
completed = cc.finalize_summary({
"total": 2,
"results": [
{"record_id": "creator", **no_creator},
{"record_id": "note", **no_unique_note},
],
})
assert completed["blocked_input"] == 1
assert completed["retryable_failures"] == 1
assert completed["unresolved"] == 1
assert completed["complete"] is False
assert pgy._missing_detail_outcome(saw_search_result=True)["status"] == "retryable_failure"
assert pgy._unmatched_note_outcome(
candidates=[], page_limit_hit=False
)["reason"] == "empty_detail_page"
assert pgy._unmatched_note_outcome(
candidates=["候选"], page_limit_hit=True
)["status"] == "retryable_failure"
@pytest.mark.parametrize(
"state",
[
{
"nextFound": True,
"nextDisabled": True,
"activePage": None,
"lastPage": None,
"numericPageCount": 0,
},
{
"nextFound": True,
"nextDisabled": False,
"activePage": 7,
"lastPage": 7,
"numericPageCount": 7,
},
],
)
def test_pgy_known_last_page_stops_without_clicking(state):
class Page:
def evaluate(self, script):
if "pgy-pagination-state" in script:
return state
if "pgy-pagination-click" in script:
pytest.fail("terminal paginator must not be clicked")
return "last-page-card-signature"
assert pgy.go_next_page(Page()) is False
def test_pgy_clicked_pagination_without_card_change_uses_source_end_semantics():
class Page:
def evaluate(self, script):
if "pgy-pagination-state" in script:
return {
"nextFound": True,
"nextDisabled": False,
"activePage": 2,
"lastPage": 7,
"numericPageCount": 7,
}
if "pgy-pagination-click" in script:
return {"clicked": True, "text": "next"}
return "same-card-signature"
def wait_for_timeout(self, _ms):
return None
assert pgy.go_next_page(Page()) is False
def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
pages = [
[{"title": "无关视频", "play_count": "10"}],
@@ -432,6 +686,125 @@ def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
assert calls["next"] == 1
def test_xingtu_video_search_uses_visible_input_and_source_keyboard_events():
events = []
class Locator:
first = None
def __init__(self):
self.first = self
def count(self):
return 1
def wait_for(self, **kwargs):
events.append(("wait", kwargs))
def click(self):
events.append(("click",))
def press(self, key):
events.append(("locator_press", key))
def fill(self, _value):
pytest.fail("Xingtu search must preserve source key-by-key input")
class Keyboard:
def type(self, value, **kwargs):
events.append(("type", value, kwargs))
def press(self, key):
events.append(("keyboard_press", key))
class Page:
keyboard = Keyboard()
def __init__(self):
self.selector = None
self.input = Locator()
def locator(self, selector):
self.selector = selector
return self.input
page = Page()
assert xingtu.submit_video_search(page, "目标标题") is True
assert ":visible" in page.selector
assert events == [
("wait", {"state": "visible", "timeout": 5000}),
("click",),
("locator_press", "Control+A"),
("locator_press", "Backspace"),
("type", "目标标题", {"delay": 20}),
("keyboard_press", "Enter"),
]
def test_xingtu_only_confirmed_creator_miss_is_blocked():
no_creator = xingtu._missing_detail_outcome(saw_search_result=False)
no_unique_video = xingtu._unmatched_video_outcome(
candidates=["平台上的其他视频"],
page_limit_hit=False,
)
completed = cc.finalize_summary({
"total": 2,
"results": [
{"record_id": "creator", **no_creator},
{"record_id": "video", **no_unique_video},
],
})
assert completed["blocked_input"] == 1
assert completed["retryable_failures"] == 1
assert completed["unresolved"] == 1
assert completed["complete"] is False
assert no_unique_video["status"] == "retryable_failure"
assert xingtu._missing_detail_outcome(
saw_search_result=True
)["status"] == "retryable_failure"
assert xingtu._unmatched_video_outcome(
candidates=[], page_limit_hit=False
)["reason"] == "empty_detail_page"
assert xingtu._unmatched_video_outcome(
candidates=["候选"], page_limit_hit=True
)["status"] == "retryable_failure"
def test_xingtu_missing_creator_result_has_distinct_business_exception():
class Context:
pages = []
class Page:
context = Context()
def evaluate(self, _script, argument=None):
if argument is not None:
return {"found": False}
return False
def wait_for_timeout(self, _milliseconds):
return None
with pytest.raises(xingtu.CreatorNotFound, match="未找到精确达人"):
xingtu.open_creator_detail(Page(), "不存在的达人", "missing-id")
def test_xingtu_clicked_pagination_without_card_change_is_retryable_timeout():
class Page:
def evaluate(self, script):
if "const all" in script:
return {"clicked": True, "picked": "pagination"}
return "same-card-signature"
def wait_for_timeout(self, _milliseconds):
return None
with pytest.raises(xingtu.BrowserTimeoutError, match="did not change"):
xingtu.go_next_page(Page())
def test_xingtu_normalizes_show_items_api_video():
card = xingtu.normalize_xingtu_api_item({
"item_id": 7654321,
@@ -456,6 +829,9 @@ def test_xingtu_normalizes_show_items_api_video():
def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
import gc
import weakref
class Response:
url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
@@ -477,9 +853,14 @@ def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
page = Page()
cards = xingtu.setup_show_items_capture(page)
page.callback(Response())
response = Response()
response_ref = weakref.ref(response)
page.callback(response)
del response
gc.collect()
assert [card["note_id"] for card in cards] == ["1", "2"]
assert response_ref() is None
def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
@@ -542,9 +923,221 @@ def test_browser_session_loss_detection_is_specific():
closed = RuntimeError("Target page, context or browser has been closed")
assert pgy.is_browser_session_lost(closed)
assert xingtu.is_browser_session_lost(closed)
assert pgy.is_browser_session_lost(RuntimeError("Page crashed"))
assert xingtu.is_browser_session_lost(RuntimeError("Page crashed"))
assert pgy.is_browser_session_lost(
RuntimeError("Execution context was destroyed, most likely because of a navigation.")
)
assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
def test_transient_browser_error_excludes_business_timeout():
assert is_transient_browser_error(
pgy.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded")
)
assert is_transient_browser_error(
RuntimeError("Page.goto: net::ERR_CONNECTION_RESET")
)
assert not is_transient_browser_error(
TimeoutError("Login timed out after 300 seconds.")
)
def _stub_creator_session_failure(monkeypatch, tmp_path, module, error_factory):
style = {"index": 1, "name": "style"}
task = {
"record_id": "record-1",
"creator_name": "creator",
"creator_id": None,
"target_title": "target",
"style_context": {"index": 1},
}
summary = {
"index": 1,
"name": "style",
"total": 1,
"filled": 0,
"results": [],
}
group = {
"creator_name": "creator",
"creator_id": None,
"tasks": [task],
}
sessions = []
class Session:
def __init__(self, **_kwargs):
sessions.append(self)
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def fail_page_action(*_args, **_kwargs):
raise error_factory()
monkeypatch.setattr(
module,
"collect_tasks_across_styles",
lambda *_args, **_kwargs: ([task], {1: summary}),
)
monkeypatch.setattr(module, "group_tasks_by_creator", lambda _tasks: [group])
monkeypatch.setattr(module, "chunk_creator_groups", lambda groups, _size: [groups])
monkeypatch.setattr(module, "DynamicSession", Session)
monkeypatch.setattr(module, "run_serialized_page_action", fail_page_action)
monkeypatch.setattr(module, "_force_cleanup_session", lambda _session: None)
monkeypatch.setattr(module, "PROFILE_DIR", tmp_path / module.__name__.split(".")[-1])
return style, sessions
@pytest.mark.parametrize("module", [pgy, xingtu])
def test_creator_collectors_retry_browser_timeout_in_three_fresh_sessions(
monkeypatch, tmp_path, module
):
style, sessions = _stub_creator_session_failure(
monkeypatch,
tmp_path,
module,
lambda: module.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded"),
)
summaries = module.scrape_styles([style], 1, True, None, True)
assert len(sessions) == 3
assert summaries[0]["complete"] is False
assert summaries[0]["retryable_failures"] == 1
@pytest.mark.parametrize("module", [pgy, xingtu])
def test_creator_collectors_do_not_retry_acceptance_terminal(
monkeypatch, tmp_path, module
):
style, sessions = _stub_creator_session_failure(
monkeypatch,
tmp_path,
module,
lambda: module.AcceptanceCookieSkip("cookie unavailable in acceptance"),
)
with pytest.raises(module.AcceptanceCookieSkip):
module.scrape_styles([style], 1, True, None, True)
assert len(sessions) == 1
@pytest.mark.parametrize("module", [pgy, xingtu])
def test_creator_collectors_do_not_retry_login_timeout_terminal(
monkeypatch, tmp_path, module
):
style, sessions = _stub_creator_session_failure(
monkeypatch,
tmp_path,
module,
lambda: TimeoutError("Login timed out after 300 seconds."),
)
with pytest.raises(TimeoutError, match="Login timed out"):
module.scrape_styles([style], 1, True, None, True)
assert len(sessions) == 1
@pytest.mark.parametrize("error_type", [xingtu.SearchQuotaExhausted, xingtu.CaptchaBlocked])
def test_xingtu_does_not_retry_explicit_business_terminal(
monkeypatch, tmp_path, error_type
):
style, sessions = _stub_creator_session_failure(
monkeypatch,
tmp_path,
xingtu,
lambda: error_type("explicit platform terminal"),
)
with pytest.raises(error_type):
xingtu.scrape_styles([style], 1, True, None, True)
assert len(sessions) == 1
@pytest.mark.parametrize("module", [pgy, xingtu])
def test_creator_collectors_with_no_tasks_do_not_open_browser(
monkeypatch, module
):
style = {"index": 1, "name": "style"}
summary = {"index": 1, "name": "style", "total": 0, "results": []}
monkeypatch.setattr(
module,
"collect_tasks_across_styles",
lambda *_args, **_kwargs: ([], {1: summary}),
)
monkeypatch.setattr(
module,
"DynamicSession",
lambda **_kwargs: pytest.fail("no browser session expected"),
)
assert module.scrape_styles([style], 1, True, None, True) == [summary]
def test_long_page_action_does_not_retain_navigation_response():
import gc
import weakref
response_ref = None
class Response:
url = "https://example.test/final"
status = 200
headers = {"content-type": "text/html"}
class Page:
closed = False
def set_default_timeout(self, _timeout):
pass
def set_default_navigation_timeout(self, _timeout):
pass
def goto(self, *_args, **_kwargs):
nonlocal response_ref
response = Response()
response_ref = weakref.ref(response)
return response
def is_closed(self):
return self.closed
def close(self):
self.closed = True
page = Page()
class Context:
def new_page(self):
return page
class Session:
context = Context()
def action(_page):
gc.collect()
assert response_ref is not None
assert response_ref() is None
navigation = run_serialized_page_action(Session(), "https://example.test", action)
assert navigation == {
"url": "https://example.test/final",
"status": 200,
"headers": {"content-type": "text/html"},
}
assert page.closed is True
def test_xingtu_nested_session_loss_is_reraised():
closed = RuntimeError("Target page, context or browser has been closed")
with pytest.raises(RuntimeError, match="has been closed"):
@@ -694,6 +1287,19 @@ def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
assert [row["record_id"] for row in summary["details"]] == ["published"]
@pytest.mark.parametrize(
("raw", "expected"),
[
("2026-07-20T00:00:00.000+08:00", bili.date(2026, 7, 20)),
("2026-07-20T00:00:00+08:00", bili.date(2026, 7, 20)),
("2026-07-19T16:00:00.000Z", bili.date(2026, 7, 19)),
("2026-07-20", bili.date(2026, 7, 20)),
],
)
def test_bilibili_parse_pub_date_accepts_feishu_iso_values(raw, expected):
assert bili.parse_pub_date(raw) == expected
def test_bilibili_partial_retry_preserves_old_success():
existing = {
"style": "款式A", "index": 1, "total_b_records": 2,
@@ -0,0 +1,272 @@
import pytest
from gyxx_flow.adapters.scrapling import BrowserTimeoutError
from gyxx_flow.modules.content_marketing.data.tools import (
batch_rescrape_douyin as douyin_batch,
)
from gyxx_flow.modules.content_marketing.data.tools import (
batch_rescrape_xiaohongshu as xhs_batch,
)
from gyxx_flow.modules.content_marketing.data.tools.comment_batch_checkpoint import (
CommentBatchCheckpoint,
is_browser_process_lost,
)
def test_page_crash_requires_a_fresh_browser():
assert is_browser_process_lost(
RuntimeError("Page.wait_for_timeout: Page crashed")
)
assert not is_browser_process_lost(RuntimeError("ordinary parse failure"))
@pytest.mark.parametrize("batch", [xhs_batch, douyin_batch])
def test_non_acceptance_relogin_runs_platform_script(monkeypatch, batch):
class Policy:
enabled = False
calls = []
monkeypatch.setattr(batch, "current_acceptance_policy", lambda: Policy())
monkeypatch.setattr(
batch.subprocess,
"call",
lambda command, **kwargs: calls.append((command, kwargs)) or 0,
)
assert batch.do_relogin() is True
assert calls[0][0][1] == str(batch.RELOGIN_SCRIPT)
def test_comment_checkpoint_matches_both_note_id_and_url(tmp_path):
path = tmp_path / "checkpoint.json"
checkpoint = CommentBatchCheckpoint(
"douyin", business_date="2026-08-09", path=path
)
checkpoint.mark_completed(7, "https://www.douyin.com/video/7")
restored = CommentBatchCheckpoint(
"douyin", business_date="2026-08-09", path=path
)
assert restored.contains(7, "https://www.douyin.com/video/7") is True
assert restored.contains(7, "https://www.douyin.com/video/changed") is False
restored.clear()
assert not path.exists()
def test_xhs_page_crash_retries_same_note_with_new_scraper_call(monkeypatch):
calls = []
def scrape(url, *_args):
calls.append(url)
if len(calls) == 1:
raise RuntimeError("Page.wait_for_timeout: Page crashed")
return [], {"stats": {}}
monkeypatch.setattr(xhs_batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(xhs_batch.time, "sleep", lambda _seconds: None)
comments, result, error = xhs_batch.scrape_with_fresh_browser("xhs-url")
assert comments == []
assert result == {"stats": {}}
assert error is None
assert calls == ["xhs-url", "xhs-url"]
def test_douyin_page_crash_retries_same_note_with_new_scraper_call(monkeypatch):
calls = []
def scrape(url, *_args):
calls.append(url)
if len(calls) == 1:
raise RuntimeError("Page.wait_for_timeout: Page crashed")
return [], {"stats": {}}
monkeypatch.setattr(douyin_batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(douyin_batch.time, "sleep", lambda _seconds: None)
comments, result, error = douyin_batch.scrape_with_fresh_browser("dy-url")
assert comments == []
assert result == {"stats": {}}
assert error is None
assert calls == ["dy-url", "dy-url"]
@pytest.mark.parametrize(
("batch", "url"),
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
)
def test_transient_timeout_gets_three_fresh_browser_attempts(monkeypatch, batch, url):
calls = []
def scrape(scrape_url, *_args):
calls.append(scrape_url)
if len(calls) < 3:
raise BrowserTimeoutError("Page.goto: Timeout 90000ms exceeded")
return [], {"stats": {}}
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
comments, result, error = batch.scrape_with_fresh_browser(url)
assert comments == []
assert result == {"stats": {}}
assert error is None
assert calls == [url, url, url]
@pytest.mark.parametrize(
("batch", "url"),
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
)
def test_transient_network_failure_exhausts_three_attempts_honestly(
monkeypatch, batch, url
):
calls = []
def scrape(scrape_url, *_args):
calls.append(scrape_url)
raise RuntimeError("Page.goto: net::ERR_CONNECTION_RESET")
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
comments, result, error = batch.scrape_with_fresh_browser(url)
assert comments is None
assert result is None
assert error is not None
assert "ERR_CONNECTION_RESET" in str(error)
assert calls == [url, url, url]
@pytest.mark.parametrize(
("batch", "url"),
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
)
def test_terminal_parse_failure_is_not_retried(monkeypatch, batch, url):
calls = []
def scrape(scrape_url, *_args):
calls.append(scrape_url)
raise RuntimeError("ordinary terminal parse failure")
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
comments, result, error = batch.scrape_with_fresh_browser(url)
assert comments is None
assert result is None
assert error is not None
assert calls == [url]
def test_xhs_failed_note_keeps_success_checkpoint_and_fails_batch(monkeypatch):
class Checkpoint:
completed = []
cleared = False
def __init__(self, _platform):
pass
def contains(self, _note_id, _url):
return False
def mark_completed(self, note_id, url):
self.completed.append((note_id, url))
def clear(self):
self.cleared = True
rows = [
{"id": 1, "url": "xhs-1", "title": "one"},
{"id": 2, "url": "xhs-2", "title": "two"},
]
def scrape(url, *_args):
if url == "xhs-2":
raise RuntimeError("ordinary terminal failure")
return ([{"level": "comment"}], {
"title": "one",
"stats": {"logged_in": True, "note_metrics": {}},
})
monkeypatch.setattr(xhs_batch, "CommentBatchCheckpoint", Checkpoint)
monkeypatch.setattr(xhs_batch, "collect_urls", lambda: rows)
monkeypatch.setattr(xhs_batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(xhs_batch.scraper, "write_outputs", lambda *_args: (None, None))
monkeypatch.setattr(xhs_batch, "upsert_metrics", lambda *_args: None)
monkeypatch.setattr(xhs_batch.db, "replace_comments", lambda *_args, **_kwargs: None)
assert xhs_batch.main() == 1
assert Checkpoint.completed == [(1, "xhs-1")]
assert Checkpoint.cleared is False
@pytest.mark.parametrize(
("batch", "platform"),
[(xhs_batch, "xhs"), (douyin_batch, "dy")],
)
def test_incomplete_comments_preserve_db_but_remain_resume_eligible(
monkeypatch, batch, platform
):
class Checkpoint:
def __init__(self):
self.completed = []
self.cleared = False
def contains(self, _note_id, _url):
return False
def mark_completed(self, note_id, url):
self.completed.append((note_id, url))
def clear(self):
self.cleared = True
checkpoint = Checkpoint()
rows = [
{
"id": 1,
"url": f"{platform}-partial",
"title": "partial",
"comment_count": 5,
},
{
"id": 2,
"url": f"{platform}-failed",
"title": "failed",
"comment_count": 5,
},
]
def scrape(url, *_args):
if url.endswith("-failed"):
raise RuntimeError("ordinary terminal failure")
comments = [{"level": "comment"}]
stats = {
"logged_in": False,
"api_total": 5,
"note_metrics": {"comment_count": 5},
}
if batch is xhs_batch:
return comments, {"title": "partial", "stats": stats}
return comments, {"stats": stats}
monkeypatch.setattr(batch, "CommentBatchCheckpoint", lambda _platform: checkpoint)
monkeypatch.setattr(batch, "collect_urls", lambda: rows)
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(batch.scraper, "write_outputs", lambda *_args: (None, None))
monkeypatch.setattr(batch, "upsert_metrics", lambda *_args: None)
monkeypatch.setattr(
batch.db,
"replace_comments",
lambda *_args, **_kwargs: pytest.fail("incomplete comments must not replace DB"),
)
assert batch.main() == 1
assert checkpoint.completed == []
assert checkpoint.cleared is False
@@ -4,7 +4,7 @@ import json
from pathlib import Path
from types import SimpleNamespace
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
from gyxx_flow.modules.content_marketing import chanmama_scraper, weekly_summary_all
from gyxx_flow.modules.content_marketing.data.tools import (
analyze_comments,
@@ -71,17 +71,13 @@ def test_daily_card_sends_only_to_wang_yunlong(
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
commands = []
calls = []
def fake_run(command, **kwargs):
commands.append(command)
return SimpleNamespace(
returncode=0,
stdout=json.dumps({"message_id": "om_test"}),
stderr="",
)
def fake_send(**kwargs):
calls.append(kwargs)
return {"message_id": "om_test"}
monkeypatch.setattr(daily_report_card.subprocess, "run", fake_run)
monkeypatch.setattr(daily_report_card, "send_lark_bot_message", fake_send)
results = daily_report_card.send_card_to_recipients(
{"schema": "2.0", "body": {"elements": []}},
("ou_other_a", "ou_other_b"),
@@ -89,9 +85,9 @@ def test_daily_card_sends_only_to_wang_yunlong(
)
assert len(results) == 1
assert len(commands) == 1
command = commands[0]
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
assert len(calls) == 1
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["profile"] == "hermes-analyzer"
def test_comment_summary_sends_only_to_wang_yunlong(
@@ -99,17 +95,17 @@ def test_comment_summary_sends_only_to_wang_yunlong(
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
commands = []
calls = []
def fake_run(command, **kwargs):
commands.append(command)
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
def fake_send(**kwargs):
calls.append(kwargs)
return {"message_id": "om_test"}
monkeypatch.setattr(analyze_comments.subprocess, "run", fake_run)
monkeypatch.setattr(analyze_comments, "send_lark_bot_message", fake_send)
analyze_comments.send_feishu_summary("summary", "ou_other")
command = commands[0]
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["profile"] == "hermes-analyzer"
def test_note_report_sends_only_to_wang_yunlong(
@@ -117,19 +113,18 @@ def test_note_report_sends_only_to_wang_yunlong(
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
commands = []
monkeypatch.setattr(analyze_note.sys, "platform", "linux")
calls = []
monkeypatch.setattr(
analyze_note,
"build_card_payload",
lambda *args, **kwargs: {"schema": "2.0"},
)
def fake_run(command, **kwargs):
commands.append(command)
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
def fake_send(**kwargs):
calls.append(kwargs)
return {"message_id": "om_test"}
monkeypatch.setattr(analyze_note.subprocess, "run", fake_run)
monkeypatch.setattr(analyze_note, "send_lark_bot_message", fake_send)
analyze_note.send_feishu_report(
{},
{},
@@ -141,8 +136,8 @@ def test_note_report_sends_only_to_wang_yunlong(
tmp_path / "report.md",
)
command = commands[0]
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["profile"] == "hermes-analyzer"
def test_relogin_notifications_resolve_only_to_wang_yunlong(
@@ -153,5 +148,5 @@ def test_relogin_notifications_resolve_only_to_wang_yunlong(
assert (
friday_relogin_parallel.recipient_for_platform("pgy", "ou_other")
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
)
@@ -0,0 +1,596 @@
import sys
from datetime import date
from types import SimpleNamespace
import pytest
from gyxx_flow.modules.content_marketing import monthly_summary_all, weekly_summary_all
from gyxx_flow.modules.content_marketing.data.tools import db
from gyxx_flow.modules.content_marketing.data.tools import (
generate_creator_report as report,
)
from gyxx_flow.modules.content_marketing.feishu_doc_import import (
FeishuDocImportTimeout,
)
def _prepare_main(monkeypatch, tmp_path, *, doc_url, saved):
monkeypatch.setattr(report, "DATA_DIR", tmp_path)
monkeypatch.setattr(
report,
"_compute_last_month",
lambda: (date(2026, 7, 1), date(2026, 7, 31)),
)
monkeypatch.setattr(
report,
"generate_report",
lambda style_filter=None: ("report", [], {}, {}),
)
monkeypatch.setattr(report, "load_existing_creator_report", lambda *_args: None)
monkeypatch.setattr(
report, "create_feishu_doc", lambda *args, **_kwargs: doc_url
)
monkeypatch.setattr(report, "save_to_db", lambda *args: saved)
monkeypatch.setattr(sys, "argv", ["generate_creator_report.py"])
def test_main_requires_both_doc_and_database_effects(monkeypatch, tmp_path):
_prepare_main(monkeypatch, tmp_path, doc_url="https://example/doc", saved=True)
assert report.main() == 0
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=True)
assert report.main() == 1
_prepare_main(monkeypatch, tmp_path, doc_url="https://example/doc", saved=False)
assert report.main() == 1
def test_main_dry_run_does_not_require_external_effects(monkeypatch, tmp_path):
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=False)
monkeypatch.setattr(sys, "argv", ["generate_creator_report.py", "--dry-run"])
assert report.main() == 0
def _import_timeout(tmp_path):
return FeishuDocImportTimeout(
"known import ticket did not become ready",
audit_path=tmp_path / "ticket-audit.json",
)
def test_creator_report_timeout_is_persisted_as_ambiguous(monkeypatch, tmp_path):
captured = {}
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=True)
monkeypatch.setattr(
report,
"load_existing_creator_report",
lambda *_args: {"doc_url": "", "doc_title": "", "status": "doc_ambiguous"},
)
def raise_timeout(*_args, **kwargs):
captured["resume_only"] = kwargs.get("resume_only")
raise _import_timeout(tmp_path)
def capture_save(*args):
captured["doc_url"] = args[-2]
captured["status"] = args[-1]
return True
monkeypatch.setattr(report, "create_feishu_doc", raise_timeout)
monkeypatch.setattr(report, "save_to_db", capture_save)
assert report.main() == 1
assert captured == {
"doc_url": None,
"resume_only": True,
"status": "doc_ambiguous",
}
@pytest.mark.parametrize(
("module", "processor_name", "creator_name"),
(
(weekly_summary_all, "process_one_style", "create_feishu_doc"),
(
monthly_summary_all,
"process_one_style_monthly",
"create_feishu_doc_monthly",
),
),
)
def test_summary_timeout_is_persisted_as_ambiguous(
monkeypatch,
tmp_path,
module,
processor_name,
creator_name,
):
saved = {}
creator_call = {}
notes = [{"id": 1, "view_count": 10, "comment_count": 1}]
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(module, "REPORTS_DIR", tmp_path)
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
if module is weekly_summary_all:
monkeypatch.setattr(
module,
"load_existing_summary_delivery",
lambda _style: {
"doc_url": "",
"doc_title": "",
"status": "doc_ambiguous",
},
)
monkeypatch.setattr(module, "get_last_week_notes", lambda _style: notes)
monkeypatch.setattr(module, "generate_summary", lambda *_args: "summary")
else:
monkeypatch.setattr(
module,
"load_existing_monthly_summary_delivery",
lambda _style: {
"doc_url": "",
"doc_title": "",
"status": "doc_ambiguous",
},
)
monkeypatch.setattr(module, "get_last_month_notes", lambda _style: notes)
monkeypatch.setattr(module, "generate_monthly_summary", lambda *_args: "summary")
monkeypatch.setattr(module, "run_single_note_analysis", lambda _note_id: "report")
def raise_timeout(*_args, **kwargs):
creator_call["resume_only"] = kwargs.get("resume_only")
raise _import_timeout(tmp_path)
def capture_save(*args):
saved["doc_url"] = args[-3]
saved["doc_title"] = args[-2]
saved["status"] = args[-1]
return True
monkeypatch.setattr(module, creator_name, raise_timeout)
save_name = "save_to_db" if module is weekly_summary_all else "save_to_db_monthly"
monkeypatch.setattr(module, save_name, capture_save)
processor = getattr(module, processor_name)
arguments = ("宙斯", ("base", "table", "url"))
if module is weekly_summary_all:
arguments += (False,)
result = processor(*arguments)
assert result["status"] == "doc_ambiguous"
assert saved == {
"doc_url": None,
"doc_title": None,
"status": "doc_ambiguous",
}
assert creator_call == {"resume_only": True}
@pytest.mark.parametrize(
("module", "processor_name", "creator_name"),
(
(weekly_summary_all, "process_one_style", "create_feishu_doc"),
(
monthly_summary_all,
"process_one_style_monthly",
"create_feishu_doc_monthly",
),
),
)
def test_summary_pg_failure_keeps_real_document_receipt_and_fails_result(
monkeypatch,
tmp_path,
module,
processor_name,
creator_name,
):
doc_url = "https://example.feishu.cn/docx/real-document"
notes = [{"id": 1, "view_count": 10, "comment_count": 1}]
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(module, "REPORTS_DIR", tmp_path)
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
if module is weekly_summary_all:
monkeypatch.setattr(module, "load_existing_summary_delivery", lambda _style: None)
monkeypatch.setattr(module, "get_last_week_notes", lambda _style: notes)
monkeypatch.setattr(module, "generate_summary", lambda *_args: "summary")
else:
monkeypatch.setattr(
module, "load_existing_monthly_summary_delivery", lambda _style: None
)
monkeypatch.setattr(module, "get_last_month_notes", lambda _style: notes)
monkeypatch.setattr(module, "generate_monthly_summary", lambda *_args: "summary")
monkeypatch.setattr(module, "run_single_note_analysis", lambda _note_id: "report")
monkeypatch.setattr(
module, creator_name, lambda *_args, **_kwargs: (doc_url, None)
)
monkeypatch.setattr(
module, "write_to_target_table", lambda *_args, **_kwargs: True
)
save_name = "save_to_db" if module is weekly_summary_all else "save_to_db_monthly"
monkeypatch.setattr(module, save_name, lambda *_args: False)
processor = getattr(module, processor_name)
arguments = ("宙斯", ("base", "table", "url"))
if module is weekly_summary_all:
arguments += (False,)
result = processor(*arguments)
assert result["status"] == "db_failed"
assert result["doc_url"] == doc_url
assert result["target_table_written"] is True
assert not weekly_summary_all.summary_results_succeeded([result])
def test_creator_existing_period_document_is_never_imported_again(
monkeypatch, tmp_path
):
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=False)
existing_url = "https://example.feishu.cn/docx/existing-creator-report"
monkeypatch.setattr(
report,
"load_existing_creator_report",
lambda *_args: {
"doc_url": existing_url,
"doc_title": "existing",
"status": "doc_ambiguous",
},
)
def fail_import(*_args, **_kwargs):
raise AssertionError("an existing period URL must not be imported again")
monkeypatch.setattr(report, "create_feishu_doc", fail_import)
monkeypatch.setattr(
report,
"save_to_db",
lambda *_args: (_ for _ in ()).throw(
AssertionError("completed period must not be overwritten")
),
)
assert report.main() == 0
@pytest.mark.parametrize(
("module", "processor_name", "loader_name", "creator_name"),
(
(
weekly_summary_all,
"process_one_style",
"load_existing_summary_delivery",
"create_feishu_doc",
),
(
monthly_summary_all,
"process_one_style_monthly",
"load_existing_monthly_summary_delivery",
"create_feishu_doc_monthly",
),
),
)
def test_completed_summary_period_skips_all_external_work(
monkeypatch,
module,
processor_name,
loader_name,
creator_name,
):
existing_url = "https://example.feishu.cn/docx/existing-summary"
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(
module,
loader_name,
lambda _style: {
"doc_url": existing_url,
"doc_title": "existing",
"status": "ok",
},
)
def fail_work(*_args, **_kwargs):
raise AssertionError("completed summary period must be skipped")
monkeypatch.setattr(module, "find_target_fields", fail_work)
monkeypatch.setattr(module, creator_name, fail_work)
processor = getattr(module, processor_name)
arguments = ("宙斯", ("base", "table", "url"))
if module is weekly_summary_all:
arguments += (False,)
result = processor(*arguments)
assert result["status"] == "ok"
assert result["doc_url"] == existing_url
assert result["already_complete"] is True
@pytest.mark.parametrize(
("module", "processor_name", "loader_name", "creator_name", "updater_name"),
(
(
weekly_summary_all,
"process_one_style",
"load_existing_summary_delivery",
"create_feishu_doc",
"update_existing_summary_delivery_status",
),
(
monthly_summary_all,
"process_one_style_monthly",
"load_existing_monthly_summary_delivery",
"create_feishu_doc_monthly",
"update_existing_monthly_summary_delivery_status",
),
),
)
def test_failed_base_delivery_reuses_existing_document_without_import(
monkeypatch,
module,
processor_name,
loader_name,
creator_name,
updater_name,
):
existing_url = "https://example.feishu.cn/docx/existing-summary"
write_call = {}
status_call = {}
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(
module,
loader_name,
lambda _style: {
"doc_url": existing_url,
"doc_title": "existing",
"status": "feishu_write_failed",
},
)
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
def capture_write(*args, **kwargs):
write_call["args"] = args
write_call["kwargs"] = kwargs
return True
def capture_status(style, status):
status_call.update({"style": style, "status": status})
return True
def fail_import(*_args, **_kwargs):
raise AssertionError("an existing document must not be imported again")
monkeypatch.setattr(module, "write_to_target_table", capture_write)
monkeypatch.setattr(module, updater_name, capture_status)
monkeypatch.setattr(module, creator_name, fail_import)
processor = getattr(module, processor_name)
arguments = ("宙斯", ("base", "table", "url"))
if module is weekly_summary_all:
arguments += (False,)
result = processor(*arguments)
assert result["status"] == "ok"
assert result["doc_url"] == existing_url
assert result["reused_document"] is True
assert write_call["args"][5] == existing_url
assert status_call == {"style": "宙斯", "status": "ok"}
if module is monthly_summary_all:
assert write_call["kwargs"] == {
"time_label": monthly_summary_all.TIME_LABEL,
"report_label": "月度汇总",
}
def test_business_date_drives_default_report_periods(monkeypatch):
for name in ("WEEK_SINCE", "WEEK_UNTIL", "MONTH_SINCE", "MONTH_UNTIL"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-09")
assert report._compute_last_month() == (date(2026, 7, 1), date(2026, 7, 31))
assert weekly_summary_all._resolve_week_period() == (
date(2026, 7, 27),
date(2026, 8, 2),
)
assert monthly_summary_all._resolve_month_period() == (
date(2026, 7, 1),
date(2026, 7, 31),
)
def test_explicit_period_overrides_take_priority_over_business_date(monkeypatch):
monkeypatch.setenv("GYXX_BUSINESS_DATE", "not-a-date")
monkeypatch.setenv("WEEK_SINCE", "2026-06-01")
monkeypatch.setenv("WEEK_UNTIL", "2026-06-07")
monkeypatch.setenv("MONTH_SINCE", "2026-05-01")
monkeypatch.setenv("MONTH_UNTIL", "2026-05-31")
assert weekly_summary_all._resolve_week_period() == (
date(2026, 6, 1),
date(2026, 6, 7),
)
assert monthly_summary_all._resolve_month_period() == (
date(2026, 5, 1),
date(2026, 5, 31),
)
assert report._compute_last_month() == (date(2026, 5, 1), date(2026, 5, 31))
def test_invalid_business_date_and_unpaired_overrides_fail_closed(monkeypatch):
for name in ("WEEK_SINCE", "WEEK_UNTIL", "MONTH_SINCE", "MONTH_UNTIL"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026/08/09")
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
report._compute_last_month()
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
weekly_summary_all._resolve_week_period()
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
monthly_summary_all._resolve_month_period()
monkeypatch.delenv("GYXX_BUSINESS_DATE")
monkeypatch.setenv("WEEK_SINCE", "2026-06-01")
with pytest.raises(ValueError, match="configured together"):
weekly_summary_all._resolve_week_period()
monkeypatch.delenv("WEEK_SINCE")
monkeypatch.setenv("MONTH_SINCE", "2026-05-01")
with pytest.raises(ValueError, match="configured together"):
report._compute_last_month()
with pytest.raises(ValueError, match="configured together"):
monthly_summary_all._resolve_month_period()
class _RecordingCursor:
def __init__(self):
self.statements = []
def execute(self, statement, params):
self.statements.append((statement, params))
def fetchone(self):
return (7,)
class _RecordingConnection:
def __init__(self):
self.cursor_instance = _RecordingCursor()
def cursor(self):
return self.cursor_instance
def commit(self):
return None
def close(self):
return None
def __enter__(self):
return self
def __exit__(self, *_args):
return False
@pytest.mark.parametrize(
("module", "save_name", "table_name"),
(
(report, "save_to_db", "cmt_creator_report"),
(weekly_summary_all, "save_to_db", "cmt_weekly_summary"),
(monthly_summary_all, "save_to_db_monthly", "cmt_monthly_summary"),
),
)
def test_empty_upsert_values_preserve_existing_document_identity(
monkeypatch,
module,
save_name,
table_name,
):
connection = _RecordingConnection()
if module is report:
monkeypatch.setattr(report, "get_db_config", lambda: {})
monkeypatch.setattr(report.psycopg, "connect", lambda **_kwargs: connection)
saved = report.save_to_db(
"summary",
[],
{},
{},
"2026-07-01",
"2026-07-31",
None,
"doc_ambiguous",
)
else:
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(db, "get_conn", lambda: connection)
saved = getattr(module, save_name)(
"宙斯",
[],
"summary",
None,
None,
"doc_ambiguous",
)
assert saved is True
upsert = connection.cursor_instance.statements[-1][0]
assert "COALESCE" in upsert
assert "NULLIF(EXCLUDED.doc_url, '')" in upsert
assert f"{table_name}.doc_url" in upsert
assert "NULLIF(EXCLUDED.doc_title, '')" in upsert
assert f"{table_name}.doc_title" in upsert
@pytest.mark.parametrize(
(
"module",
"function_name",
"args",
"expected_title",
"source_filename",
"operation_prefix",
),
(
(
report,
"create_feishu_doc",
("report", "2026-07-01", "2026-07-31"),
"达人合作数据筛选与报价分析报告(2026-07-01~07-31",
"creator_report_doc.md",
"content-creator-report-",
),
(
weekly_summary_all,
"create_feishu_doc",
("宙斯", "summary"),
"宙斯 周笔记汇总(2026-07-27~08-02",
"weekly_summary_doc.md",
"content-weekly-summary-",
),
(
monthly_summary_all,
"create_feishu_doc_monthly",
("宙斯", "summary"),
"宙斯 月度汇总(2026-07-01~07-31",
"monthly_summary_doc.md",
"content-monthly-summary-",
),
),
)
def test_feishu_doc_import_preserves_title_body_and_root_target(
monkeypatch,
tmp_path,
module,
function_name,
args,
expected_title,
source_filename,
operation_prefix,
):
captured = {}
def fake_import(markdown, **kwargs):
captured["markdown"] = markdown
captured.update(kwargs)
return SimpleNamespace(url="https://example/doc")
monkeypatch.setattr(
module,
"PATHS",
SimpleNamespace(tmp_root=tmp_path, evidence_root=tmp_path / "evidence"),
)
monkeypatch.setattr(module, "import_markdown_document", fake_import)
if module is weekly_summary_all:
monkeypatch.setattr(module, "TIME_LABEL", "2026-07-27~08-02")
elif module is monthly_summary_all:
monkeypatch.setattr(module, "TIME_LABEL", "2026-07-01~07-31")
getattr(module, function_name)(*args)
assert captured["markdown"] == args[0 if module is report else 1]
assert captured["title"] == expected_title
assert captured["source_filename"] == source_filename
assert captured["working_directory"] == tmp_path
assert captured["audit_directory"] == tmp_path / "evidence" / "feishu_doc_import"
assert captured["identity"] == "user"
assert captured.get("folder_token") is None
assert captured["operation_id"].startswith(operation_prefix)
assert len(captured["operation_id"].rsplit("-", 1)[1]) == 40
assert captured["resume_only"] is False
@@ -1,5 +1,11 @@
import io
import sys
import tempfile
import unittest
from contextlib import redirect_stdout
from datetime import date
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from gyxx_flow.modules.content_marketing import daily_marketing_report as daily
@@ -47,6 +53,41 @@ class DailyMarketingReportTests(unittest.TestCase):
),
)
def test_disabled_dynamic_route_keeps_report_but_skips_feishu_send(self):
facts = ([], [], [], [], {})
with tempfile.TemporaryDirectory() as directory, \
patch.object(daily, "PATHS", SimpleNamespace(exports_root=Path(directory))), \
patch.object(daily, "_latest_report_date", return_value=date(2026, 8, 7)), \
patch.object(daily, "load_report_facts", return_value=facts), \
patch.object(daily, "build_daily_prompt", return_value="prompt"), \
patch.object(daily, "call_hermes_analyzer", return_value="report"), \
patch.object(daily, "require_valid_hermes_report", return_value="report"), \
patch.object(daily, "normalize_report", return_value="report"), \
patch.object(daily, "generate_dashboard"), \
patch.object(
daily,
"resolve_notification_route",
return_value=SimpleNamespace(
enabled=False,
open_ids=(),
app_profile="hermes-analyzer",
),
), \
patch.object(daily, "send_daily_report_cards") as send, \
patch.object(sys, "argv", ["daily_marketing_report.py", "--send"]):
output = io.StringIO()
with redirect_stdout(output):
self.assertEqual(daily.main(), 0)
report_path = (
Path(directory)
/ "summary"
/ "daily_marketing_report_2026-08-07.md"
)
self.assertTrue(report_path.is_file())
self.assertIn("动态配置中禁用", output.getvalue())
send.assert_not_called()
def test_hermes_transport_error_is_not_accepted_as_report(self):
with self.assertRaisesRegex(RuntimeError, "Hermes 分析失败"):
daily.require_valid_hermes_report("API call failed after 3 retries: Connection error.")
@@ -142,6 +183,25 @@ class DailyMarketingReportTests(unittest.TestCase):
self.assertEqual(valid["engagement_count_num"], 2143)
self.assertEqual(valid["publish_status"], "已记录发布时间")
def test_style_note_signal_keeps_complete_count_and_reports_metric_coverage(self):
signals = daily._build_style_signals(
["宙斯"],
[],
[
{"style_name": "宙斯", "exposure_count": 1000},
{"style_name": "宙斯", "exposure_count": None},
],
[],
[],
[],
)
self.assertEqual(signals["宙斯"]["note_count"], 2)
self.assertEqual(signals["宙斯"]["recent_note_exposure"], 1000)
self.assertEqual(signals["宙斯"]["metric_note_count"], 1)
self.assertEqual(signals["宙斯"]["metric_missing_count"], 1)
self.assertEqual(signals["宙斯"]["metric_coverage_rate"], 0.5)
def test_load_report_facts_tracks_all_styles_for_report_date(self):
metrics = {
"宙斯": {"light": "🟢"},
@@ -156,6 +216,7 @@ class DailyMarketingReportTests(unittest.TestCase):
patch.object(daily, "load_style_images", return_value={}), \
patch.object(daily, "load_style_categories", return_value={}), \
patch.object(daily, "_cooperation_context", return_value=[]), \
patch.object(daily, "_note_inventory_context", return_value=[]), \
patch.object(daily, "_style_comment_context", return_value=[]), \
patch.object(daily, "_review_context", return_value=[]), \
patch.object(daily, "_creative_context", return_value=[]), \
@@ -185,6 +246,7 @@ class DailyMarketingReportTests(unittest.TestCase):
patch.object(daily, "load_style_images", return_value={}), \
patch.object(daily, "load_style_categories", return_value={}), \
patch.object(daily, "_cooperation_context", return_value=[]), \
patch.object(daily, "_note_inventory_context", return_value=[]), \
patch.object(daily, "_style_comment_context", return_value=[]), \
patch.object(daily, "_review_context", return_value=[]), \
patch.object(daily, "_creative_context", return_value=[]), \
@@ -49,6 +49,83 @@ class DailyReportCardTests(unittest.TestCase):
self.assertNotIn("![全域种草数据看板]", analysis_text)
self.assertIn("波塞冬:继续观察", analysis_text)
self.assertIn("退款订单比上升", analysis_text)
self.assertEqual(analysis["schema"], "2.0")
self.assertEqual(analysis["config"]["width_mode"], "fill")
self.assertEqual(
[item["tag"] for item in analysis["body"]["elements"]],
["column_set", "collapsible_panel", "column_set"],
)
self.assertNotEqual(
analysis["body"]["elements"],
[{"tag": "markdown", "content": report}],
)
def test_analysis_card_structures_full_five_section_report(self):
report = """品牌SKU营销运营日报
日期:2026年8月1日
一、今日经营概览
销量:整体下降。
今日核心结论:流量收缩,转化承压。
二、SKU表现
|SKU|今日销量|综合判断|建议动作|
|---|---|---|---|
|星云2|128|继续放大|维持资源|
|星迹2|33|先修承接|优化详情页|
注:曝光为累计值。
三、重点SKU分析
星云2:经营表现健康,继续放大。
四、达人营销分析
最佳渠道:抖音。
五、明日运营建议
1. 优化星迹2详情页。
2. 补充逐星内容。
"""
card = build_analysis_card("2026-08-01", report)
serialized = json.dumps(card, ensure_ascii=False)
self.assertEqual(
[item["tag"] for item in card["body"]["elements"]],
[
"column_set",
"table",
"collapsible_panel",
"collapsible_panel",
"column_set",
],
)
self.assertEqual(len(card["body"]["elements"][1]["rows"]), 2)
sku_panel, marketing_panel = card["body"]["elements"][2:4]
self.assertTrue(sku_panel["expanded"])
self.assertTrue(marketing_panel["expanded"])
self.assertEqual(sku_panel["background_color"], "blue-50")
self.assertEqual(marketing_panel["background_color"], "violet-50")
self.assertNotEqual(
sku_panel["background_color"], marketing_panel["background_color"]
)
self.assertEqual(sku_panel["header"]["title"]["tag"], "markdown")
self.assertIn(
"**<font color='blue'>重点SKU分析</font>**",
sku_panel["header"]["title"]["content"],
)
self.assertIn(
"**<font color='violet'>达人与内容营销分析</font>**",
marketing_panel["header"]["title"]["content"],
)
for phrase in (
"流量收缩,转化承压",
"曝光为累计值",
"继续放大",
"最佳渠道:抖音",
"补充逐星内容",
):
self.assertIn(phrase, serialized)
self.assertNotIn("button", serialized)
def test_build_card_uses_card_2_schema_and_dashboard(self):
report = """一、核心经营结论
@@ -139,25 +216,99 @@ class DailyReportCardTests(unittest.TestCase):
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.subprocess.run"
"daily_report_card.send_lark_bot_message"
)
def test_send_card_to_all_recipients(self, run):
run.side_effect = [
type("Result", (), {"returncode": 0, "stdout": json.dumps({"message_id": f"om_{i}"}), "stderr": ""})()
def test_send_card_to_all_recipients(self, send):
send.side_effect = [
{"message_id": f"om_{i}"}
for i in range(3)
]
recipients = ["ou_one", "ou_two", "ou_three"]
results = send_card_to_recipients({"schema": "2.0"}, recipients, "2026-07-15")
self.assertEqual([item["message_id"] for item in results], ["om_0", "om_1", "om_2"])
self.assertEqual(run.call_count, 3)
commands = [call.args[0] for call in run.call_args_list]
self.assertEqual([cmd[cmd.index("--user-id") + 1] for cmd in commands], recipients)
self.assertTrue(all("--msg-type" in cmd and "interactive" in cmd for cmd in commands))
keys = [cmd[cmd.index("--idempotency-key") + 1] for cmd in commands]
self.assertEqual(send.call_count, 3)
calls = send.call_args_list
self.assertEqual([call.kwargs["user_id"] for call in calls], recipients)
self.assertTrue(all(call.kwargs["msg_type"] == "interactive" for call in calls))
self.assertTrue(all(call.kwargs["profile"] == "hermes-analyzer" for call in calls))
keys = [call.kwargs["idempotency_key"] for call in calls]
self.assertTrue(all(key.startswith("daily-2026-07-15-") for key in keys))
self.assertTrue(all(len(key) <= 50 for key in keys))
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.send_lark_bot_message"
)
def test_send_card_rejects_success_exit_without_message_id(self, send):
send.return_value = {"ok": True}
with self.assertRaisesRegex(RuntimeError, "message_id"):
send_card_to_recipients(
{"schema": "2.0"},
["ou_owner"],
"2026-08-09",
)
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.send_lark_bot_message"
)
def test_sender_uses_fixed_user_profile_and_hashes_full_recipient_identity(self, send):
send.side_effect = [
{"message_id": f"om_{index}"}
for index in range(2)
]
recipients = ["ou_first_sharedsuffix", "ou_second_sharedsuffix"]
send_card_to_recipients(
{"schema": "2.0"},
recipients,
"2026-08-07",
)
calls = send.call_args_list
self.assertTrue(
all(call.kwargs["profile"] == "hermes-analyzer" for call in calls)
)
keys = [
call.kwargs["idempotency_key"]
for call in calls
]
self.assertEqual(len(set(keys)), 2)
with self.assertRaisesRegex(ValueError, "hermes-analyzer"):
send_card_to_recipients(
{"schema": "2.0"},
["ou_owner"],
"2026-08-07",
app_profile="dynamic-analyzer",
)
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.send_lark_bot_message"
)
def test_stable_fingerprint_ignores_rotating_uploaded_image_key(self, send):
send.side_effect = [
{"message_id": f"om_{index}"}
for index in range(2)
]
for image_key in ("img_first_upload", "img_second_upload"):
send_card_to_recipients(
build_dashboard_card("2026-08-07", [], image_key),
["ou_owner"],
"2026-08-07",
message_kind="dashboard",
idempotency_fingerprint="same-dashboard-bytes-and-metrics",
)
keys = [
call.kwargs["idempotency_key"]
for call in send.call_args_list
]
self.assertEqual(keys[0], keys[1])
@patch(
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.send_card_to_recipients"
)
@@ -177,13 +328,16 @@ class DailyReportCardTests(unittest.TestCase):
]
recipients = ["ou_one", "ou_two"]
results = send_daily_report_cards(
"2026-07-17",
"第一部分 整体经营分析\n完整分析内容",
[{"light": "🟢"}],
Path("charts/dashboard.png"),
recipients,
)
with tempfile.TemporaryDirectory() as tmp:
chart_path = Path(tmp) / "dashboard.png"
Image.new("RGB", (4, 4), "white").save(chart_path)
results = send_daily_report_cards(
"2026-07-17",
"第一部分 整体经营分析\n完整分析内容",
[{"light": "🟢"}],
chart_path,
recipients,
)
self.assertEqual(
[item["message_id"] for item in results],
@@ -195,6 +349,10 @@ class DailyReportCardTests(unittest.TestCase):
self.assertIn("img_v3_test", json.dumps(first_card, ensure_ascii=False))
self.assertIn("完整分析内容", json.dumps(second_card, ensure_ascii=False))
self.assertEqual(send.call_args_list[0].kwargs["message_kind"], "dashboard")
self.assertIn(
"idempotency_fingerprint",
send.call_args_list[0].kwargs,
)
self.assertEqual(send.call_args_list[1].kwargs["message_kind"], "analysis")
@@ -0,0 +1,99 @@
import json
import pytest
from gyxx_flow.adapters.direct_content_llm import (
ContentLLMConfig,
ContentLLMConfigurationError,
ContentLLMResponseError,
call_content_analyzer,
load_content_llm_config,
)
def _config() -> ContentLLMConfig:
return ContentLLMConfig(
endpoint="https://api.minimaxi.com/anthropic/v1/messages",
credential="test-only-secret",
model="MiniMax-M3",
thinking_mode="disabled",
)
def _response(*blocks: dict[str, object]) -> bytes:
return json.dumps(
{
"type": "message",
"content": list(blocks),
"usage": {"input_tokens": 3, "output_tokens": 4},
}
).encode()
def test_config_uses_anthropic_endpoint_and_redacts_key(monkeypatch) -> None:
monkeypatch.setenv(
"CONTENT_ANALYSIS_LLM_BASE_URL", "https://api.minimaxi.com/anthropic"
)
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_API_KEY", "test-only-secret")
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_MODEL", "MiniMax-M3")
config = load_content_llm_config()
assert config.endpoint == "https://api.minimaxi.com/anthropic/v1/messages"
assert config.model == "MiniMax-M3"
assert "test-only-secret" not in repr(config)
def test_call_sends_anthropic_message_and_ignores_thinking_block() -> None:
observed: dict[str, object] = {}
def transport(url, headers, body, timeout):
payload = json.loads(body)
observed.update(
url=url,
headers=headers,
payload=payload,
timeout=timeout,
)
return _response(
{"type": "thinking", "thinking": "hidden reasoning"},
{"type": "text", "text": "最终报告"},
)
result = call_content_analyzer(
"system prompt",
"user prompt",
transport=transport,
config=_config(),
)
assert result == "最终报告"
assert observed["url"] == "https://api.minimaxi.com/anthropic/v1/messages"
assert observed["headers"]["X-Api-Key"] == "test-only-secret"
assert observed["headers"]["anthropic-version"] == "2023-06-01"
assert observed["payload"]["model"] == "MiniMax-M3"
assert observed["payload"]["system"] == "system prompt"
assert observed["payload"]["messages"] == [
{"role": "user", "content": "user prompt"}
]
assert observed["payload"]["thinking"] == {"type": "disabled"}
def test_call_rejects_openai_response_shape() -> None:
with pytest.raises(ContentLLMResponseError, match="response shape"):
call_content_analyzer(
"system prompt",
"user prompt",
transport=lambda *_args: b'{"choices": []}',
config=_config(),
)
def test_config_rejects_local_gateway(monkeypatch) -> None:
monkeypatch.setenv(
"CONTENT_ANALYSIS_LLM_BASE_URL", "http://127.0.0.1:8642/v1"
)
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_API_KEY", "test-only-secret")
with pytest.raises(ContentLLMConfigurationError, match="local gateway"):
load_content_llm_config()
@@ -0,0 +1,162 @@
import pytest
from gyxx_flow.modules.content_marketing import douyin_comment_scraper as scraper
from gyxx_flow.modules.content_marketing.data.tools import (
batch_rescrape_douyin as batch,
)
class _EvaluatePage:
def __init__(self, result):
self.result = result
self.calls = []
def evaluate(self, script, argument):
self.calls.append((script, argument))
return self.result
def test_bounded_fetch_uses_abort_controller_and_clears_timer(monkeypatch):
page = _EvaluatePage(
{"__gyxx_status": "ok", "payload": {"comments": []}}
)
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
payload = scraper._fetch_json_with_timeout(
page,
"https://example.invalid/comments",
deadline=1000.0,
stage="top-level pagination",
)
assert payload == {"comments": []}
script, argument = page.calls[0]
assert "AbortController" in script
assert "clearTimeout(timer)" in script
assert argument["timeoutMs"] == 30_000
def test_bounded_fetch_timeout_is_an_explicit_transient_failure(monkeypatch):
page = _EvaluatePage({"__gyxx_status": "timeout"})
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
with pytest.raises(scraper.TransientDouyinCommentTimeout) as caught:
scraper._fetch_json_with_timeout(
page,
"https://example.invalid/comments",
deadline=1000.0,
stage="reply pagination",
)
assert isinstance(caught.value, ConnectionError)
assert "reply pagination" in str(caught.value)
def test_scroll_checks_the_note_deadline_before_browser_work(monkeypatch):
page = _EvaluatePage(None)
monkeypatch.setattr(scraper.time, "monotonic", lambda: 901.0)
with pytest.raises(scraper.TransientDouyinCommentTimeout):
scraper.scroll_comments(
page,
max_scrolls=60,
idle_rounds=10,
comments_by_id={},
deadline=900.0,
)
assert page.calls == []
def test_top_and_reply_pagination_share_the_bounded_fetch(monkeypatch):
stages = []
monkeypatch.setattr(scraper.time, "monotonic", lambda: 0.0)
def fetch(_page, _url, *, deadline, stage):
assert deadline == 900.0
stages.append(stage)
return {"comments": [], "has_more": 0, "cursor": 0}
monkeypatch.setattr(scraper, "_fetch_json_with_timeout", fetch)
monkeypatch.setattr(scraper, "collect_from_payload", lambda *_args: None)
page = _EvaluatePage(None)
scraper.fetch_missing_top_comments(
page,
"source-url",
"aweme-id",
{"existing": {}},
{
"comment_url_template": "https://example.invalid/top",
"api_total": 2,
"last_cursor": 1,
},
deadline=900.0,
)
scraper.fetch_missing_replies(
page,
"source-url",
{
"parent": {
"level": "comment",
"reply_count": 1,
"comment_id": "parent-id",
}
},
{
"reply_url_template": "https://example.invalid/reply",
"aweme_id": "aweme-id",
},
deadline=900.0,
)
assert stages == ["top-level pagination", "reply pagination"]
def test_deadline_failure_retries_and_never_replaces_existing_comments(
monkeypatch,
):
attempts = []
class Checkpoint:
completed = []
cleared = False
def __init__(self, _platform):
pass
def contains(self, _note_id, _url):
return False
def mark_completed(self, note_id, url):
self.completed.append((note_id, url))
def clear(self):
self.cleared = True
def scrape(url, *_args):
attempts.append(url)
raise scraper.TransientDouyinCommentTimeout(
"Douyin comment scrape exceeded 900 seconds"
)
monkeypatch.setattr(batch, "CommentBatchCheckpoint", Checkpoint)
monkeypatch.setattr(
batch,
"collect_urls",
lambda: [{"id": 1, "url": "dy-url", "comment_count": 10}],
)
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(
batch.db,
"replace_comments",
lambda *_args, **_kwargs: pytest.fail(
"a timed-out scrape must not replace existing comments"
),
)
assert batch.main() == 1
assert attempts == ["dy-url", "dy-url", "dy-url"]
assert Checkpoint.completed == []
assert Checkpoint.cleared is False
@@ -0,0 +1,399 @@
import json
import subprocess
from types import SimpleNamespace
import pytest
from gyxx_flow.modules.content_marketing.feishu_doc_import import (
FeishuDocImportAmbiguous,
FeishuDocImportError,
FeishuDocImportTimeout,
_parse_cli_json,
deterministic_import_operation_id,
import_markdown_document,
)
def _completed(payload, *, returncode=0, prefix="", suffix=""):
return SimpleNamespace(
returncode=returncode,
stdout=f"{prefix}{json.dumps(payload)}{suffix}",
stderr="",
)
class _FakeClock:
def __init__(self):
self.now = 100.0
def monotonic(self):
return self.now
def sleep(self, seconds):
self.now += seconds
def test_import_ready_result_uses_relative_file_exact_title_and_audits_without_token(
tmp_path,
):
commands = []
title = "宙斯 周笔记汇总(2026-07-27~08-02"
markdown = "# 标题\n\n完整正文"
document_token = "VERY_SECRET_DOCUMENT_TOKEN_123456"
def fake_run(command, **kwargs):
commands.append(command)
source = command[command.index("--file") + 1]
assert source.startswith("./")
assert (kwargs["cwd"] / source.removeprefix("./")).read_text(
encoding="utf-8"
) == markdown
return _completed(
{
"scenario": "import",
"ticket": "ticket-ready",
"ready": True,
"failed": False,
"job_status": 0,
"job_status_label": "success",
"token": document_token,
"url": f"https://example.feishu.cn/docx/{document_token}",
},
prefix="lark-cli notice\n",
suffix="\ntrailing notifier",
)
result = import_markdown_document(
markdown,
title=title,
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="weekly_summary_doc.md",
runner=fake_run,
operation_id="ready-op",
)
assert result.url == f"https://example.feishu.cn/docx/{document_token}"
assert result.document_token == document_token
assert commands[0][1:3] == ["drive", "+import"]
assert commands[0][commands[0].index("--name") + 1] == title
assert commands[0][commands[0].index("--type") + 1] == "docx"
assert commands[0][commands[0].index("--as") + 1] == "user"
assert "--folder-token" not in commands[0]
audit_text = result.audit_path.read_text(encoding="utf-8")
audit = json.loads(audit_text)
assert audit["status"] == "ready"
assert audit["ticket"] == "ticket-ready"
assert audit["url"] == "https://example.feishu.cn/docx/<redacted>"
assert audit["last_result"]["url"] == audit["url"]
assert audit["document_title"] == title
assert audit["folder_target"] == "root"
assert "token" not in audit
assert "document_token" not in audit_text
assert document_token not in audit_text
assert not list((tmp_path / "work").glob("*.md"))
def test_import_persists_ticket_and_polls_task_result_until_ready(tmp_path):
responses = iter(
[
_completed(
{
"scenario": "import",
"ticket": "ticket-async",
"ready": False,
"failed": False,
"timed_out": True,
}
),
_completed(
{
"scenario": "import",
"ticket": "ticket-async",
"ready": False,
"failed": False,
"job_status": 1,
"job_status_label": "processing",
}
),
_completed(
{
"scenario": "import",
"ticket": "ticket-async",
"ready": True,
"failed": False,
"job_status": 0,
"job_status_label": "success",
"token": "doxcnAsyncDocument",
"url": "https://example.feishu.cn/docx/doxcnAsyncDocument",
}
),
]
)
commands = []
def fake_run(command, **_kwargs):
commands.append(command)
if len(commands) == 2:
in_progress = json.loads(
(tmp_path / "audit" / "async-op.json").read_text(encoding="utf-8")
)
assert in_progress["status"] == "processing"
assert in_progress["ticket"] == "ticket-async"
assert in_progress["last_result"]["ready"] is False
return next(responses)
result = import_markdown_document(
"report",
title="精确标题",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=fake_run,
sleeper=lambda _seconds: None,
operation_id="async-op",
)
assert result.ticket == "ticket-async"
assert len(commands) == 3
for command in commands[1:]:
assert command[1:3] == ["drive", "+task_result"]
assert command[command.index("--scenario") + 1] == "import"
assert command[command.index("--ticket") + 1] == "ticket-async"
assert command[command.index("--as") + 1] == "user"
audit = json.loads(result.audit_path.read_text(encoding="utf-8"))
assert audit["poll_attempts"] == 2
assert audit["ticket"] == "ticket-async"
assert audit["last_result"]["ready"] is True
def test_import_reports_failed_task_without_retrying_mutation(tmp_path):
leaked_document_token = "VERY_SECRET_FAILED_DOCUMENT_TOKEN"
leaked_open_id = "ou_SHOULD_NOT_BE_IN_AUDIT"
responses = iter(
[
_completed(
{
"ticket": "ticket-failed",
"ready": False,
"failed": False,
}
),
_completed(
{
"ticket": "ticket-failed",
"ready": False,
"failed": True,
"job_status": -1,
"job_status_label": "failed",
"job_error_msg": (
"conversion failed at "
f"https://example.feishu.cn/docx/{leaked_document_token} "
f'\"token\":\"{leaked_document_token}\" '
f'\"open_id\":\"{leaked_open_id}\"'
),
}
),
]
)
commands = []
def fake_run(command, **_kwargs):
commands.append(command)
return next(responses)
with pytest.raises(FeishuDocImportError, match="conversion failed") as raised:
import_markdown_document(
"report",
title="失败报告",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=fake_run,
sleeper=lambda _seconds: None,
operation_id="failed-op",
)
assert sum(command[1:3] == ["drive", "+import"] for command in commands) == 1
audit_text = raised.value.audit_path.read_text(encoding="utf-8")
audit = json.loads(audit_text)
assert audit["status"] == "failed"
assert audit["ticket"] == "ticket-failed"
assert audit["last_result"]["failed"] is True
assert leaked_document_token not in audit_text
assert leaked_open_id not in audit_text
def test_import_timeout_keeps_ticket_for_auditable_follow_up(tmp_path):
clock = _FakeClock()
responses = iter(
[
_completed(
{"ticket": "ticket-timeout", "ready": False, "failed": False}
),
_completed(
{
"ticket": "ticket-timeout",
"ready": False,
"failed": False,
"job_status_label": "processing",
}
),
]
)
with pytest.raises(FeishuDocImportTimeout) as raised:
import_markdown_document(
"report",
title="超时报告",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=lambda *_args, **_kwargs: next(responses),
clock=clock.monotonic,
sleeper=clock.sleep,
wait_timeout_seconds=1,
poll_interval_seconds=1,
operation_id="timeout-op",
)
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
assert audit["status"] == "timed_out"
assert audit["ticket"] == "ticket-timeout"
assert audit["poll_attempts"] == 1
def test_mutating_import_process_timeout_is_ambiguous_and_never_retried(tmp_path):
commands = []
def time_out(command, **kwargs):
commands.append(command)
assert kwargs["timeout"] == 2
raise subprocess.TimeoutExpired(command, kwargs["timeout"])
with pytest.raises(FeishuDocImportTimeout) as raised:
import_markdown_document(
"report",
title="有限超时报告",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=time_out,
import_command_timeout_seconds=2,
operation_id="command-timeout-op",
)
assert len(commands) == 1
assert commands[0][1:3] == ["drive", "+import"]
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
assert audit["status"] == "ambiguous"
assert audit["ticket"] is None
assert not list((tmp_path / "work").glob("*.md"))
def test_deterministic_audit_resumes_only_known_ticket_without_second_import(tmp_path):
audit_root = tmp_path / "audit"
audit_root.mkdir()
operation_id = "stable-report-op"
audit_path = audit_root / f"{operation_id}.json"
audit_path.write_text(
json.dumps(
{
"audit_schema": "gyxx.feishu-doc-import.v1",
"document_title": "固定周期报告",
"folder_target": "root",
"identity": "user",
"markdown_sha256": "previous-input-digest",
"operation_id": operation_id,
"poll_attempts": 1,
"status": "timed_out",
"target_type": "docx",
"ticket": "known-ticket",
}
),
encoding="utf-8",
)
commands = []
def fake_run(command, **_kwargs):
commands.append(command)
return _completed(
{
"ticket": "known-ticket",
"ready": True,
"failed": False,
"token": "doxcnRecoveredDocument",
"url": "https://example.feishu.cn/docx/doxcnRecoveredDocument",
}
)
result = import_markdown_document(
"possibly regenerated report",
title="固定周期报告",
working_directory=tmp_path / "work",
audit_directory=audit_root,
source_filename="report.md",
runner=fake_run,
operation_id=operation_id,
)
assert result.url.endswith("/doxcnRecoveredDocument")
assert len(commands) == 1
assert commands[0][1:3] == ["drive", "+task_result"]
assert all(command[1:3] != ["drive", "+import"] for command in commands)
audit_text = audit_path.read_text(encoding="utf-8")
audit = json.loads(audit_text)
assert audit["status"] == "ready"
assert audit["resume_count"] == 1
assert audit["resume_input_changed"] is True
assert "doxcnRecoveredDocument" not in audit_text
def test_resume_only_without_ticket_audit_refuses_mutating_import(tmp_path):
commands = []
with pytest.raises(FeishuDocImportAmbiguous, match="缺少可对账") as raised:
import_markdown_document(
"report",
title="待对账报告",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=lambda command, **_kwargs: commands.append(command),
operation_id="missing-ticket-op",
resume_only=True,
)
assert commands == []
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
assert audit["status"] == "ambiguous"
assert audit["ticket"] is None
def test_deterministic_operation_id_is_stable_and_period_specific():
first = deterministic_import_operation_id(
"content-weekly-summary", "宙斯", "2026-07-27", "2026-08-02"
)
same = deterministic_import_operation_id(
"content-weekly-summary", "宙斯", "2026-07-27", "2026-08-02"
)
other_style = deterministic_import_operation_id(
"content-weekly-summary", "赫拉", "2026-07-27", "2026-08-02"
)
assert first == same
assert first != other_style
assert first.startswith("content-weekly-summary-")
assert len(first.rsplit("-", 1)[1]) == 40
def test_cli_json_parser_accepts_noise_and_rejects_invalid_output():
assert _parse_cli_json(
'notice {"notice":"update"}\n{"ready":false,"ticket":"abc"}\ntrailing'
) == {
"ready": False,
"ticket": "abc",
}
with pytest.raises(ValueError, match="no JSON"):
_parse_cli_json("plain text", "also plain text")
@@ -1,3 +1,7 @@
from types import SimpleNamespace
import pytest
from gyxx_flow.modules.content_marketing.data.tools import (
friday_relogin_parallel as relogin,
)
@@ -43,6 +47,434 @@ def test_failed_platforms_are_grouped_by_their_responsible_recipient():
}
def test_legacy_route_keeps_the_existing_per_platform_recipients(monkeypatch):
observed = {}
def fake_resolver(workflow_id, defaults):
observed["workflow_id"] = workflow_id
observed["defaults"] = defaults
return relogin.ResolvedNotificationRoute(
configured=False,
enabled=True,
app_profile=None,
open_ids=tuple(defaults),
)
monkeypatch.setattr(relogin, "resolve_notification_route", fake_resolver)
route = relogin.resolve_weekly_notification_route()
assert observed == {
"workflow_id": "content.relogin.weekly",
"defaults": (
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
"ou_7ad5fc8012e2f741afc5346e05ffd447",
),
}
assert relogin.recipients_for_platform("pgy", route) == (
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
)
assert relogin.recipients_for_platform("douyin", route) == (
"ou_7ad5fc8012e2f741afc5346e05ffd447",
)
def test_disabled_route_still_runs_relogin_without_sending(monkeypatch):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=False,
app_profile="hermes-analyzer",
open_ids=(),
)
rounds = []
monkeypatch.setattr(
relogin,
"current_acceptance_policy",
lambda: SimpleNamespace(enabled=False),
)
monkeypatch.setattr(
relogin,
"resolve_weekly_notification_route",
lambda _override=None: route,
)
def fake_round_factory(_args, resolved_route, notification_failures):
assert resolved_route is route
assert notification_failures == []
def run_round(names, attempt):
rounds.append((tuple(names), attempt))
return {name: True for name in names}
return run_round
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
monkeypatch.setattr(
relogin,
"_send_lark",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
AssertionError("disabled route must not send Lark messages")
),
)
monkeypatch.setattr(
relogin.sys,
"argv",
["friday_relogin_parallel.py", "--platform", "pgy"],
)
assert relogin.main() == 0
assert rounds == [(("pgy",), 1)]
def test_enabled_route_sends_qr_text_and_image_to_every_recipient(
monkeypatch,
tmp_path,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
sent = []
image = tmp_path / "qrcode.png"
monkeypatch.setattr(
relogin,
"_send_lark",
lambda recipient, text=None, image=None, app_profile=None: sent.append(
(recipient, text, image, app_profile)
),
)
failures = relogin._send_qr_notification(
route,
"pgy",
override=None,
text="请扫码",
image=image,
)
assert failures == ()
assert sent == [
("ou_first123", "请扫码", None, "hermes-analyzer"),
("ou_first123", None, image, "hermes-analyzer"),
("ou_second456", "请扫码", None, "hermes-analyzer"),
("ou_second456", None, image, "hermes-analyzer"),
]
assert relogin.group_platforms_by_notification_recipient(
["pgy", "douyin"], route
) == {
"ou_first123": ["pgy", "douyin"],
"ou_second456": ["pgy", "douyin"],
}
def test_qr_delivery_aggregates_errors_and_attempts_every_recipient(
monkeypatch,
tmp_path,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
attempts = []
image = tmp_path / "qrcode.png"
def fake_send(recipient, text=None, image=None, app_profile=None):
stage = "qr_text" if text is not None else "qr_image"
attempts.append((recipient, stage, app_profile))
if (recipient, stage) == ("ou_first123", "qr_text"):
raise RuntimeError("first text rejected")
if (recipient, stage) == ("ou_second456", "qr_image"):
raise RuntimeError("second image rejected")
monkeypatch.setattr(relogin, "_send_lark", fake_send)
failures = relogin._send_qr_notification(
route,
"pgy",
override=None,
text="请扫码",
image=image,
)
assert attempts == [
("ou_first123", "qr_text", "hermes-analyzer"),
("ou_first123", "qr_image", "hermes-analyzer"),
("ou_second456", "qr_text", "hermes-analyzer"),
("ou_second456", "qr_image", "hermes-analyzer"),
]
assert failures == (
relogin.NotificationDeliveryFailure(
stage="qr_text",
platform="pgy",
recipient="ou_first123",
detail="first text rejected",
),
relogin.NotificationDeliveryFailure(
stage="qr_image",
platform="pgy",
recipient="ou_second456",
detail="second image rejected",
),
)
def test_failure_summary_aggregates_every_recipient_error(monkeypatch):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
attempts = []
def fail_send(recipient, **_kwargs):
attempts.append(recipient)
raise RuntimeError(f"rejected {recipient}")
monkeypatch.setattr(relogin, "_send_lark", fail_send)
failures = relogin._send_failure_summaries(
route,
["pgy", "douyin"],
override=None,
max_attempts=3,
)
assert attempts == ["ou_first123", "ou_second456"]
assert [failure.stage for failure in failures] == [
"failure_summary",
"failure_summary",
]
assert [failure.recipient for failure in failures] == attempts
assert [failure.platform for failure in failures] == [
"pgy,douyin",
"pgy,douyin",
]
def test_main_returns_failure_and_reports_all_notification_errors(
monkeypatch,
capsys,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
monkeypatch.setattr(
relogin,
"current_acceptance_policy",
lambda: SimpleNamespace(enabled=False),
)
monkeypatch.setattr(
relogin,
"resolve_weekly_notification_route",
lambda _override=None: route,
)
def fake_round_factory(_args, _route, notification_failures):
notification_failures.extend(
(
relogin.NotificationDeliveryFailure(
"qr_text", "pgy", "ou_first123", "text rejected"
),
relogin.NotificationDeliveryFailure(
"qr_image", "pgy", "ou_second456", "image rejected"
),
)
)
return lambda names, _attempt: {name: True for name in names}
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
monkeypatch.setattr(
relogin.sys,
"argv",
["friday_relogin_parallel.py", "--platform", "pgy"],
)
assert relogin.main() == 2
output = capsys.readouterr().out
assert "Final relogin status: {'pgy': True}" in output
assert "Final notification status: failed (2 deliveries)" in output
assert "recipient=ou_first123 error=text rejected" in output
assert "recipient=ou_second456 error=image rejected" in output
def test_round_completes_relogin_after_notification_failure(monkeypatch, tmp_path):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
args = SimpleNamespace(
round_timeout=10,
screenshot_delay=0,
no_send=False,
recipient=None,
max_attempts=1,
)
failures = []
attempts = []
waits = []
class Process:
pid = 1234
def wait(self, timeout):
waits.append(timeout)
return 0
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
monkeypatch.setattr(
relogin,
"binding_from_environment",
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
)
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(relogin, "_login_windows", lambda: {"pgy": 101})
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
monkeypatch.setattr(relogin, "_capture_window", lambda _hwnd, _output: None)
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: True)
def fake_send(recipient, text=None, image=None, **_kwargs):
stage = "qr_text" if text is not None else "qr_image"
attempts.append((recipient, stage))
if (recipient, stage) == ("ou_first123", "qr_text"):
raise RuntimeError("temporary send failure")
monkeypatch.setattr(relogin, "_send_lark", fake_send)
run_round = relogin._run_round_factory(args, route, failures)
assert run_round(["pgy"], 1) == {"pgy": True}
assert waits == [100]
assert attempts == [
("ou_first123", "qr_text"),
("ou_first123", "qr_image"),
("ou_second456", "qr_text"),
("ou_second456", "qr_image"),
]
assert failures == [
relogin.NotificationDeliveryFailure(
stage="qr_text",
platform="pgy",
recipient="ou_first123",
detail="temporary send failure",
)
]
@pytest.mark.parametrize(
("returncode", "cookie_valid", "expected_status", "expected_failure_count"),
(
(0, True, True, 0),
(1, False, False, 4),
),
)
def test_missing_qr_window_is_classified_after_relogin_result(
monkeypatch,
tmp_path,
returncode,
cookie_valid,
expected_status,
expected_failure_count,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
args = SimpleNamespace(
round_timeout=10,
screenshot_delay=0,
no_send=False,
recipient=None,
max_attempts=1,
)
failures = []
class Process:
pid = 1234
def wait(self, timeout):
assert timeout == 100
return returncode
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
monkeypatch.setattr(
relogin,
"binding_from_environment",
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
)
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(relogin, "_login_windows", lambda: {})
monkeypatch.setattr(relogin, "_window_for_process", lambda _pid: None)
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: cookie_valid)
monkeypatch.setattr(
relogin,
"_send_lark",
lambda *_args, **_kwargs: pytest.fail(
"no QR window must not start a send before child classification"
),
)
run_round = relogin._run_round_factory(args, route, failures)
assert run_round(["pgy"], 1) == {"pgy": expected_status}
assert len(failures) == expected_failure_count
assert {failure.stage for failure in failures} <= {"qr_text", "qr_image"}
assert all(
failure.detail == "no visible QR window and relogin did not succeed"
for failure in failures
)
def test_lark_send_uses_shared_analyzer_user_sender_for_text_and_image(
monkeypatch,
tmp_path,
):
calls = []
image = tmp_path / "qrcode.png"
image.write_bytes(b"png")
monkeypatch.setattr(relogin, "ANALYZER_DIR", tmp_path)
monkeypatch.setenv("HERMES_ANALYZER_TOKEN", "must-not-leak")
def fake_send(**kwargs):
calls.append(kwargs)
return {"message_id": f"om_{len(calls)}"}
monkeypatch.setattr(relogin, "send_lark_bot_message", fake_send)
receipts = relogin._send_lark(
"ou_recipient123",
text="登录提醒",
image=image,
)
assert [receipt["message_id"] for receipt in receipts] == ["om_1", "om_2"]
assert calls[0]["user_id"] == "ou_recipient123"
assert calls[0]["text"] == "登录提醒"
assert "image" not in calls[0]
assert calls[1]["user_id"] == "ou_recipient123"
assert calls[1]["image"] == image
assert "text" not in calls[1]
assert all(call["profile"] == "hermes-analyzer" for call in calls)
assert all("HERMES_ANALYZER_TOKEN" not in call["env"] for call in calls)
def test_login_windows_are_classified_by_title():
assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili"
assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy"
@@ -169,6 +169,73 @@ def test_xingtu_search_and_open_creator_submits_only_one_search(monkeypatch):
assert calls == [("同一达人", "creator-1")]
def test_creator_search_attempts_id_then_nickname_and_skips_blank_id():
assert xingtu._creator_search_attempts("Rm.", "Ruan97") == [
("ID", "Ruan97", "Ruan97"),
("昵称", "Rm.", None),
]
assert xingtu._creator_search_attempts("Rm.", " ") == [
("昵称", "Rm.", None),
]
def test_xingtu_primary_retries_only_unmatched_tasks_through_buyin(monkeypatch):
xingtu.configure_source("xingtu")
styles = [{"index": 1, "name": "款式A"}]
context = {"index": 1}
tasks = [
{"record_id": "hit", "creator_name": "达人A", "creator_id": "id-a", "style_context": context},
{"record_id": "miss", "creator_name": "达人B", "creator_id": "id-b", "style_context": context},
]
summaries = {
1: {
"style": "款式A",
"index": 1,
"total": 2,
"matched": 0,
"filled": 0,
"skipped_no_pubtime": 0,
"results": [],
}
}
calls = []
def fake_collect(*_args, **_kwargs):
return tasks, summaries
def fake_scrape(*_args, **kwargs):
source = xingtu.CONTENT_SOURCE
selected = kwargs["task_override"]
calls.append((source, [task["record_id"] for task in selected], kwargs.get("replace_existing", False)))
for task in selected:
matched = source == "buyin" or task["record_id"] == "hit"
xingtu.upsert_result(summaries[1], {
"record_id": task["record_id"],
"matched": matched,
"status": xingtu.SUCCESS if matched else xingtu.RETRYABLE_FAILURE,
"reason": None if matched else "title_unmatched",
})
return [summaries[1]]
monkeypatch.setattr(xingtu, "collect_tasks_across_styles", fake_collect)
monkeypatch.setattr(xingtu, "scrape_styles", fake_scrape)
try:
result = xingtu.scrape_styles_with_fallback(
styles, 1, False, None, True,
)
finally:
xingtu.configure_source("xingtu")
assert calls == [
("xingtu", ["hit", "miss"], False),
("buyin", ["miss"], True),
]
rows = {row["record_id"]: row for row in result[0]["results"]}
assert rows["hit"]["collection_source"] == "xingtu"
assert rows["miss"]["collection_source"] == "buyin_fallback"
assert rows["miss"]["fallback_from"] == "xingtu"
def test_xingtu_search_budget_blocks_duplicate_without_failure_circuit():
budget = xingtu.CreatorSearchBudget(max_consecutive_failures=3)
@@ -291,20 +358,22 @@ def test_scheduled_collection_has_no_automatic_retry_and_syncs_once():
)
def test_daily_collection_uses_dedicated_self_operated_pipeline():
def test_daily_collection_includes_self_operated_pipeline():
steps = _workflow("content.metrics.daily").steps
entries = [step.entry for step in steps]
run_step = next(step for step in steps if step.entry == "run_all.py")
assert "--daily-scope" in run_step.args
assert "--include-self-operated" not in run_step.args
assert "data/tools/refresh_self_mapping.py" in entries
assert "self_bilibili_scraper.py" in entries
assert "chanmama_scraper.py" in entries
refresh_index = entries.index("data/tools/refresh_self_mapping.py")
bili_index = entries.index("self_bilibili_scraper.py")
douyin_index = entries.index("chanmama_scraper.py")
assert refresh_index < bili_index < douyin_index
assert entries.index("run_all.py") < entries.index(
"data/tools/refresh_self_mapping.py"
)
assert entries.index("data/tools/refresh_self_mapping.py") < entries.index(
"self_bilibili_scraper.py"
)
assert entries.index("self_bilibili_scraper.py") < entries.index(
"chanmama_scraper.py"
)
@pytest.mark.parametrize(
@@ -0,0 +1,194 @@
from __future__ import annotations
from gyxx_flow.modules.content_marketing.data.tools import sync_notes_master as sync
def _table() -> dict:
return {
"name": "宙斯",
"base_token": "base-token",
"table_id": "tbl-source",
"field_map": {
"note_title": {"field_id": "title"},
"publish_time": {"field_id": "published"},
"note_url": {"field_id": "url"},
"platform": {"field_id": "platform"},
"creator_name": {"field_id": "creator"},
},
}
def test_parse_records_counts_only_complete_note_rows() -> None:
rows = sync.parse_records(
_table(),
[
{
"record_id": "complete",
"title": "通勤双肩包实测",
"published": "2026-08-11 09:20:00",
"url": "[笔记](https://www.xiaohongshu.com/explore/abc)",
"platform": "小红书",
"creator": "达人甲",
},
{
"record_id": "missing-publish-time",
"title": "只有标题和链接",
"published": None,
"url": "https://www.douyin.com/video/123",
"platform": "抖音",
"creator": "达人乙",
},
],
self_operated=False,
)
assert rows[0].is_countable is True
assert rows[0].platform == "xiaohongshu"
assert rows[0].publish_time == "2026-08-11 09:20:00+08:00"
assert rows[0].url == "https://www.xiaohongshu.com/explore/abc"
assert rows[1].is_countable is False
def test_platform_can_be_inferred_from_url_when_source_cell_is_empty() -> None:
rows = sync.parse_records(
_table(),
[{
"record_id": "bili",
"title": "B站开箱",
"published": "2026-08-10",
"url": "https://www.bilibili.com/video/BV1test",
"platform": "",
"creator": "自营号",
}],
self_operated=True,
)
assert rows[0].platform == "bilibili"
assert rows[0].is_countable is True
def test_unrecognized_source_text_is_bounded_to_database_columns() -> None:
rows = sync.parse_records(
_table(),
[{
"record_id": "unexpected-long-values",
"title": "" * 600,
"published": "2026-08-10",
"url": "not-a-platform-url",
"platform": "unexpected-platform-value-" * 4,
"creator": "" * 300,
}],
self_operated=False,
)
assert len(rows[0].platform) == 32
assert len(rows[0].creator_name) == 255
assert len(rows[0].title) == 512
def test_collaboration_fields_parsed_only_for_collaborator_tables() -> None:
table = _table()
table["field_map"].update(
{
"cost": {"field_id": "cost"},
"tracking_no": {"field_id": "tracking"},
"is_paid": {"field_id": "paid"},
"follower_count": {"field_id": "followers"},
}
)
records = [{
"record_id": "coop-1",
"title": "合作笔记",
"published": "2026-08-11",
"url": "https://www.douyin.com/video/9",
"platform": "抖音",
"creator": "达人丙",
"cost": 1200,
"tracking": "SF123",
"paid": True,
"followers": "3.7万",
}]
coop_row = sync.parse_records(table, records, self_operated=False)[0]
assert coop_row.coop_attrs["cooperation_cost"] == 1200.0
assert coop_row.coop_attrs["tracking_number"] == "SF123"
assert coop_row.coop_attrs["is_paid"] is True
assert coop_row.creator_attrs["follower_count_num"] == 37000
self_row = sync.parse_records(table, records, self_operated=True)[0]
assert self_row.coop_attrs == {}
assert self_row.creator_attrs == {}
def test_dry_run_fetches_both_scopes_without_opening_database(monkeypatch) -> None:
collaborator = {**_table(), "name": "合作款"}
self_operated = {**_table(), "name": "自营款", "table_id": "tbl-self"}
monkeypatch.setattr(
sync,
"load_sources",
lambda **_kwargs: [(False, collaborator), (True, self_operated)],
)
monkeypatch.setattr(
sync.feishu_mapping,
"_record_list_all",
lambda *_args: [{
"record_id": "one",
"title": "完整笔记",
"published": "2026-08-11",
"url": "https://www.douyin.com/video/1",
"platform": "抖音",
"creator": "达人",
}],
)
monkeypatch.setattr(
sync.psycopg,
"connect",
lambda **_kwargs: (_ for _ in ()).throw(
AssertionError("dry-run must not open PostgreSQL")
),
)
summary = sync.synchronize()
assert summary == {
"tables": 2,
"rows": 2,
"creators": 0,
"countable": 2,
"incomplete": 0,
"deactivated": 0,
}
def test_sync_table_deactivates_only_rows_missing_from_same_source() -> None:
class Cursor:
rowcount = 3
def __init__(self) -> None:
self.calls = []
def execute(self, sql, params=None):
self.calls.append((" ".join(sql.split()), params))
def fetchone(self):
return (42,)
cur = Cursor()
rows = [
sync.MasterRow(
record_id="rec-1",
platform="xiaohongshu",
creator_name="达人",
title="标题",
publish_time="2026-08-11 00:00:00+08:00",
url="https://www.xiaohongshu.com/explore/1",
)
]
result = sync.sync_table(cur, _table(), self_operated=False, rows=rows)
assert result["countable"] == 1
deactivate_sql, params = cur.calls[-1]
assert "source_base_token = %s AND source_table_id = %s" in deactivate_sql
assert "NOT (feishu_record_id = ANY(%s))" in deactivate_sql
assert params == ("base-token", "tbl-source", ["rec-1"])
@@ -15,25 +15,31 @@ from gyxx_flow.modules.content_marketing.data.tools import (
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module):
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "state.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
cookie_backup, profile_backup = module.reset_state()
cookie_backup, profile_backup, state_backup = module.reset_state()
assert not cookie.exists()
assert not storage.exists()
assert not profile.exists()
cookie.write_text("partial-new-cookie", encoding="utf-8")
storage.write_text("partial-new-storage", encoding="utf-8")
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
module.restore_previous_state(cookie_backup, profile_backup)
module.restore_previous_state(cookie_backup, profile_backup, state_backup)
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
@@ -41,21 +47,27 @@ def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, modul
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module):
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
cookie_backup, profile_backup = module.reset_state()
cookie_backup, profile_backup, state_backup = module.reset_state()
cookie.write_text("new-cookie", encoding="utf-8")
storage.write_text("new-storage", encoding="utf-8")
profile.mkdir()
module.finish_successful_relogin(profile_backup)
assert cookie.read_text(encoding="utf-8") == "new-cookie"
assert storage.read_text(encoding="utf-8") == "new-storage"
assert cookie_backup.exists()
assert state_backup.exists()
assert not profile_backup.exists()
@@ -127,6 +139,82 @@ def test_xingtu_index_page_is_recognized_as_logged_in():
assert xingtu_scraper_v2.is_logged_in(Page()) is True
def test_xingtu_index_page_with_xingtu_session_is_logged_in_without_market_ui():
class Locator:
@property
def first(self):
return self
def count(self):
return 0
def wait_for(self, timeout):
raise xingtu_scraper_v2.BrowserTimeoutError("not on market page")
def is_visible(self):
return False
class Context:
def cookies(self):
return [
{
"name": "sessionid",
"value": "fresh",
"domain": ".xingtu.cn",
}
]
class Page:
url = "https://www.xingtu.cn/ad/creator/index"
context = Context()
def get_by_text(self, *_args, **_kwargs):
return Locator()
def locator(self, *_args, **_kwargs):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is True
def test_xingtu_index_page_without_xingtu_session_is_not_logged_in():
class Locator:
@property
def first(self):
return self
def count(self):
return 0
def wait_for(self, timeout):
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
def is_visible(self):
return False
class Context:
def cookies(self):
return [
{
"name": "sessionid",
"value": "douyin-only",
"domain": ".douyin.com",
}
]
class Page:
url = "https://www.xingtu.cn/ad/creator/index"
context = Context()
def get_by_text(self, *_args, **_kwargs):
return Locator()
def locator(self, *_args, **_kwargs):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is False
def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie():
class Context:
def cookies(self):
@@ -149,7 +237,7 @@ def test_xingtu_market_page_without_business_ui_rejects_stale_cookie():
return self
def wait_for(self, **_kwargs):
raise xingtu_scraper_v2.PlaywrightTimeoutError("missing")
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
def count(self):
return 0
@@ -282,25 +370,64 @@ def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monke
assert saved == [market_page]
def test_xingtu_scan_login_detects_business_tab_while_qr_tab_stays_open(monkeypatch):
class Context:
def __init__(self):
self.pages = []
class Page:
def __init__(self, context, url):
self.context = context
self.url = url
def is_closed(self):
return False
def wait_for_timeout(self, _milliseconds):
raise AssertionError("authenticated tab should be detected immediately")
context = Context()
qr_page = Page(context, "https://open.douyin.com/platform/oauth/pc/auth")
index_page = Page(context, "https://www.xingtu.cn/ad/creator/index")
context.pages = [qr_page, index_page]
saved = []
monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None)
monkeypatch.setattr(
xingtu_scraper_v2, "is_logged_in", lambda page: page is index_page
)
monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page))
result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2)
assert result is index_page
assert saved == [index_page]
def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch):
cookie = tmp_path / "xingtu_cookies.json"
storage = tmp_path / "xingtu_storage_state.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie)
monkeypatch.setattr(relogin_xingtu, "STATE_FILE", storage)
monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile)
cookie_backup, profile_backup = relogin_xingtu.reset_state()
cookie_backup, profile_backup, state_backup = relogin_xingtu.reset_state()
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
assert relogin_xingtu.recover_interrupted_state() is True
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
assert cookie_backup is not None
assert state_backup is not None
assert profile_backup is not None
@@ -0,0 +1,410 @@
from __future__ import annotations
import json
from pathlib import Path
from types import ModuleType
import pytest
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
from gyxx_flow.modules.content_marketing.data.tools import (
relogin_bilibili,
relogin_douyin,
relogin_pgy,
relogin_runtime_paths,
relogin_xiaohongshu,
relogin_xingtu,
)
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
CONSUMER_BINDING_IDS,
child_browser_environment,
resolve_relogin_runtime_paths,
sync_relogin_state_to_consumers,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
PROJECT_ROOT = Path(__file__).resolve().parents[3]
def test_relogin_paths_prefer_injected_runtime_binding(tmp_path: Path) -> None:
cookie = tmp_path / "binding" / "cookies.json"
storage = tmp_path / "binding" / "storage_state.json"
profile = tmp_path / "binding" / "profile"
resolved = resolve_relogin_runtime_paths(
"pgy",
{
"GYXX_BROWSER_COOKIE_FILE": str(cookie),
"GYXX_BROWSER_STORAGE_STATE_FILE": str(storage),
"GYXX_BROWSER_PROFILE_DIR": str(profile),
},
)
assert resolved.cookie_file == cookie.resolve()
assert resolved.storage_state_file == storage.resolve()
assert resolved.profile_dir == profile.resolve()
def test_relogin_paths_keep_legacy_standalone_layout() -> None:
resolved = resolve_relogin_runtime_paths("xiaohongshu", {})
assert resolved.cookie_file == (
PATHS.state_root / "cookies/xiaohongshu_cookies.json"
).resolve()
assert resolved.storage_state_file == (
PATHS.state_root / "cookies/xiaohongshu_storage_state.json"
).resolve()
assert resolved.profile_dir == (
PATHS.state_root / "browser-profiles/xiaohongshu"
).resolve()
def test_child_login_environment_preserves_wrapper_binding(tmp_path: Path) -> None:
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
environment = child_browser_environment(
cookie_file=cookie,
storage_state_file=storage,
profile_dir=profile,
environment={"GYXX_MODULE_ID": "content_marketing", "KEEP": "1"},
)
assert environment["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(
storage.resolve()
)
assert environment["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
assert "GYXX_MODULE_ID" not in environment
assert environment["KEEP"] == "1"
@pytest.mark.parametrize(
("module", "platform", "login_cookie", "argv"),
[
(
relogin_bilibili,
"bilibili",
"SESSDATA",
["relogin_bilibili.py"],
),
(relogin_douyin, "douyin", "sessionid", ["relogin_douyin.py"]),
(relogin_pgy, "pgy", "web_session", ["relogin_pgy.py"]),
(
relogin_xiaohongshu,
"xiaohongshu",
"web_session",
["relogin_xiaohongshu.py"],
),
(
relogin_xingtu,
"xingtu",
"sessionid",
["relogin_xingtu.py", "--force"],
),
],
)
def test_relogin_wrapper_validates_the_same_binding_written_by_child(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
module: ModuleType,
platform: str,
login_cookie: str,
argv: list[str],
) -> None:
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
monkeypatch.setattr(module.sys, "argv", argv)
monkeypatch.setenv("GYXX_MODULE_ID", "content_marketing")
if module is relogin_xingtu:
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
synced: list[tuple[str, Path, Path]] = []
def fake_sync(
synced_platform: str,
*,
cookie_file: Path,
storage_state_file: Path,
) -> tuple[Path, ...]:
synced.append(
(synced_platform, cookie_file.resolve(), storage_state_file.resolve())
)
return ()
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fake_sync)
def fake_call(
_command: list[str],
*,
cwd: str,
env: dict[str, str],
) -> int:
assert cwd == str(module.PROJECT_DIR)
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
assert env["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(storage.resolve())
assert env["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
assert "GYXX_MODULE_ID" not in env
cookie.write_text(
json.dumps([{"name": login_cookie, "value": "fresh"}]),
encoding="utf-8",
)
storage.write_text("{}", encoding="utf-8")
profile.mkdir()
return 0
monkeypatch.setattr(module.subprocess, "call", fake_call)
assert module.main() == 0
assert cookie.is_file()
assert storage.is_file()
assert profile.is_dir()
assert synced == [(platform, cookie.resolve(), storage.resolve())]
@pytest.mark.parametrize(
("module", "platform", "login_cookie", "cookie_domain", "argv"),
[
(
relogin_bilibili,
"bilibili",
"SESSDATA",
".bilibili.com",
["relogin_bilibili.py"],
),
(
relogin_douyin,
"douyin",
"sessionid",
".douyin.com",
["relogin_douyin.py"],
),
(
relogin_pgy,
"pgy",
"web_session",
".xiaohongshu.com",
["relogin_pgy.py"],
),
(
relogin_xiaohongshu,
"xiaohongshu",
"web_session",
".xiaohongshu.com",
["relogin_xiaohongshu.py"],
),
(
relogin_xingtu,
"xingtu",
"sessionid",
".xingtu.cn",
["relogin_xingtu.py", "--force"],
),
],
)
def test_relogin_wrapper_restores_previous_state_when_publish_fails(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
module: ModuleType,
platform: str,
login_cookie: str,
cookie_domain: str,
argv: list[str],
) -> None:
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
old_cookie = '[{"name":"old","value":"old-session"}]'
old_storage = '{"cookies":[{"name":"old"}]}'
cookie.write_text(old_cookie, encoding="utf-8")
storage.write_text(old_storage, encoding="utf-8")
profile.mkdir()
(profile / "old-state.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
monkeypatch.setattr(module.sys, "argv", argv)
if module is relogin_xingtu:
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
def fake_call(
_command: list[str],
*,
cwd: str,
env: dict[str, str],
) -> int:
assert cwd == str(module.PROJECT_DIR)
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
cookie.write_text(
json.dumps(
[
{
"name": login_cookie,
"value": "fresh-session",
"domain": cookie_domain,
}
]
),
encoding="utf-8",
)
storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
profile.mkdir()
return 0
def fail_publish(
synced_platform: str,
*,
cookie_file: Path,
storage_state_file: Path,
) -> tuple[Path, ...]:
assert synced_platform == platform
assert cookie_file == cookie
assert storage_state_file == storage
raise OSError("simulated consumer publish failure")
monkeypatch.setattr(module.subprocess, "call", fake_call)
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fail_publish)
assert module.main() == 1
assert cookie.read_text(encoding="utf-8") == old_cookie
assert storage.read_text(encoding="utf-8") == old_storage
assert (profile / "old-state.txt").read_text(encoding="utf-8") == (
"old-profile"
)
@pytest.mark.parametrize(
("platform", "cookie_name", "cookie_domain"),
[
("bilibili", "SESSDATA", ".bilibili.com"),
("douyin", "sessionid", ".douyin.com"),
("pgy", "web_session", ".xiaohongshu.com"),
("xiaohongshu", "web_session", ".xiaohongshu.com"),
("xingtu", "sessionid", ".xingtu.cn"),
],
)
def test_successful_relogin_atomically_updates_unique_consumer_state(
tmp_path: Path,
platform: str,
cookie_name: str,
cookie_domain: str,
) -> None:
data_root = tmp_path / "data"
source_cookie = tmp_path / "relogin" / "cookies.json"
source_storage = tmp_path / "relogin" / "storage_state.json"
source_cookie.parent.mkdir(parents=True)
fresh_cookie = [
{
"name": cookie_name,
"value": "fresh-session",
"domain": cookie_domain,
}
]
fresh_storage = {"cookies": fresh_cookie, "origins": []}
source_cookie.write_text(json.dumps(fresh_cookie), encoding="utf-8")
source_storage.write_text(json.dumps(fresh_storage), encoding="utf-8")
catalog = RuntimeIntegrationCatalog.load_default(
project_root=PROJECT_ROOT,
data_root=data_root,
)
consumer_id = CONSUMER_BINDING_IDS[platform][0]
consumer = catalog.binding_for(consumer_id)
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
consumer.cookie_file.write_text('[{"name":"old"}]', encoding="utf-8")
consumer.storage_state_file.write_text(
'{"cookies":[{"name":"old"}]}',
encoding="utf-8",
)
destinations = sync_relogin_state_to_consumers(
platform,
cookie_file=source_cookie,
storage_state_file=source_storage,
environment={
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
"GYXX_DATA_ROOT": str(data_root),
},
)
assert source_cookie.resolve() != consumer.cookie_file.resolve()
assert source_storage.resolve() != consumer.storage_state_file.resolve()
assert destinations == (consumer.cookie_file.resolve(),)
assert json.loads(consumer.cookie_file.read_text(encoding="utf-8")) == (
fresh_cookie
)
assert json.loads(
consumer.storage_state_file.read_text(encoding="utf-8")
) == fresh_storage
def test_consumer_state_is_rolled_back_if_atomic_publish_fails(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
data_root = tmp_path / "data"
source_cookie = tmp_path / "relogin" / "cookies.json"
source_storage = tmp_path / "relogin" / "storage_state.json"
source_cookie.parent.mkdir(parents=True)
source_cookie.write_text(
json.dumps(
[
{
"name": "SESSDATA",
"value": "fresh-session",
"domain": ".bilibili.com",
}
]
),
encoding="utf-8",
)
source_storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
catalog = RuntimeIntegrationCatalog.load_default(
project_root=PROJECT_ROOT,
data_root=data_root,
)
consumer = catalog.binding_for(CONSUMER_BINDING_IDS["bilibili"][0])
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
old_cookie = '[{"name":"SESSDATA","value":"old"}]'
old_storage = '{"cookies":[{"name":"old"}]}'
consumer.cookie_file.write_text(old_cookie, encoding="utf-8")
consumer.storage_state_file.write_text(old_storage, encoding="utf-8")
original_replace = relogin_runtime_paths.os.replace
failed = False
def fail_second_replace(source: Path | str, target: Path | str) -> None:
nonlocal failed
source_path = Path(source)
target_path = Path(target)
if (
not failed
and source_path.suffix == ".tmp"
and target_path == consumer.storage_state_file
):
failed = True
raise OSError("simulated storage publish failure")
original_replace(source, target)
monkeypatch.setattr(relogin_runtime_paths.os, "replace", fail_second_replace)
with pytest.raises(OSError, match="simulated storage publish failure"):
sync_relogin_state_to_consumers(
"bilibili",
cookie_file=source_cookie,
storage_state_file=source_storage,
environment={
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
"GYXX_DATA_ROOT": str(data_root),
},
)
assert consumer.cookie_file.read_text(encoding="utf-8") == old_cookie
assert consumer.storage_state_file.read_text(encoding="utf-8") == old_storage
@@ -1,6 +1,7 @@
import io
import unittest
from pathlib import Path
from unittest.mock import patch
from gyxx_flow.adapters import RuntimeServicePolicy
from gyxx_flow.modules.content_marketing.data.tools import db
@@ -60,7 +61,7 @@ class LocalDatabaseConfigTests(unittest.TestCase):
paths = (
ROOT / "data" / "tools" / "db.py",
ROOT / "data" / "tools" / "sync_metrics_to_cmt_notes.py",
ROOT / "data" / "tools" / "sync_cooperations.py",
ROOT / "data" / "tools" / "sync_notes_master.py",
ROOT / "data" / "tools" / "generate_creator_report.py",
)
for path in paths:
@@ -79,6 +80,33 @@ class LocalDatabaseConfigTests(unittest.TestCase):
self.assertTrue(raw.getvalue())
def test_explicit_insert_missing_preserves_default_sync_contract(self):
with (
patch("sys.argv", ["sync_metrics_to_cmt_notes.py", "--insert-missing"]),
patch.object(
sync_metrics,
"sync",
return_value={"identity_ambiguous": 0},
) as sync_call,
):
sync_metrics.main()
sync_call.assert_called_once_with(dry_run=False, insert_missing=True)
def test_default_sync_exits_nonzero_when_identity_is_ambiguous(self):
with (
patch("sys.argv", ["sync_metrics_to_cmt_notes.py"]),
patch.object(
sync_metrics,
"sync",
return_value={"identity_ambiguous": 2},
),
self.assertRaises(SystemExit) as exc_info,
):
sync_metrics.main()
self.assertEqual(exc_info.exception.code, 1)
if __name__ == "__main__":
unittest.main()
@@ -1,6 +1,7 @@
import json
import os
import time
from datetime import date
from pathlib import Path
from types import SimpleNamespace
@@ -39,6 +40,54 @@ def test_success_status_cannot_hide_write_failure_and_blocked_input_is_terminal(
}],
}
assert retry.extract_failed_records(blocked, "bili") == []
assert retry.requested_records_succeeded(blocked, ["b"]) == (True, [])
def test_requested_records_require_exactly_one_resolved_blocked_terminal():
duplicate_blocked = {
"results": [
{
"record_id": "b",
"status": "blocked_input",
"matched": False,
"reason": "url_missing",
},
{
"record_id": "b",
"status": "blocked_input",
"matched": False,
"reason": "url_missing",
},
],
}
assert retry.requested_records_succeeded(duplicate_blocked, ["b"]) == (
False,
["b"],
)
def test_blocked_input_cannot_hide_explicit_execution_or_write_failure():
for explicit_failure in (
{"write_ok": False},
{"write_error": "database rejected update"},
{"error": "browser session crashed"},
):
blocked = {
"results": [{
"record_id": "b",
"status": "blocked_input",
"matched": False,
"reason": "url_missing",
**explicit_failure,
}],
}
assert retry.requested_records_succeeded(blocked, ["b"]) == (
False,
["b"],
)
assert retry.extract_failed_records(blocked, "bili") == ["b"]
def test_canonical_filename_requires_current_style_name():
@@ -305,6 +354,87 @@ def test_validate_summary_rejects_expected_style_left_over_from_old_run(tmp_path
assert "current run" in reason
def test_build_retry_targets_narrows_retryable_rows_and_expands_structural_gaps():
targets = run_all.build_retry_targets(
{
"bili": [{
"index": 1,
"total_b_records": 2,
"details": [
{"record_id": "ok", "status": "success", "ok": True},
{"record_id": "bad", "status": "retryable_failure", "ok": False},
],
}],
"pgy": [{
"index": 1,
"total": 2,
"results": [{"record_id": "only-one", "status": "success"}],
}],
"xt": [{
"index": 1,
"total": 1,
"results": [{
"record_id": "terminal",
"status": "blocked_input",
"matched": False,
}],
}],
},
{1},
)
assert targets == {
"bili": {1: {"bad"}},
"pgy": {1: None},
}
def test_retry_manifest_round_trips_record_and_full_style_scopes(monkeypatch, tmp_path):
monkeypatch.setattr(run_all, "RETRY_MANIFEST_DIR", tmp_path)
targets = {"bili": {1: {"r2", "r1"}}, "pgy": {3: None}}
run_all._write_retry_manifest(
date(2026, 9, 2), targets, source_slot="01:00"
)
assert run_all._load_retry_manifest(date(2026, 9, 2)) == targets
def test_retry_process_command_passes_only_target_records():
args = SimpleNamespace(
dry_run=False,
style=[1, 2],
daily_scope=True,
no_retry=False,
)
command = run_all.build_process_command(
"pgy", args, retry_targets={2: {"record-b", "record-a"}}
)
assert command[command.index("--style") + 1] == "2"
assert command.count("--style") == 1
assert command[command.index("--record") + 1] == "record-a"
assert command.count("--record") == 2
assert "--no-retry" in command
def test_retry_process_command_uses_full_style_when_scope_is_structurally_incomplete():
args = SimpleNamespace(
dry_run=False,
style=[1, 2],
daily_scope=True,
no_retry=False,
)
command = run_all.build_process_command(
"xt", args, retry_targets={1: None}
)
assert command[command.index("--style") + 1] == "1"
assert "--record" not in command
def test_aggregate_counts_all_unresolved_rows_as_errors():
aggregate = run_all.aggregate({
"pgy": [{
@@ -363,6 +493,40 @@ def test_aggregate_keeps_partial_counts_when_validation_marks_payload_invalid():
assert platform["validation_error"] == "invalid summary: unresolved"
def test_aggregate_fails_closed_for_stale_payload():
aggregate = run_all.aggregate({
"pgy": {
"error": "invalid summary: summary file is stale",
"payload": [{
"index": 1,
"total": 141,
"matched": 101,
"results": [],
}],
},
})
platform = aggregate["platforms"]["pgy"]
assert platform["loaded"] is False
assert platform["error"] == "invalid summary: summary file is stale"
assert aggregate["totals"]["tasks"] == 0
def test_aggregate_filters_merged_history_to_current_styles():
aggregate = run_all.aggregate(
{
"xt": [
{"index": 1, "total": 2, "matched": 1, "results": []},
{"index": 2, "total": 99, "matched": 99, "results": []},
],
},
expected_indices={1},
)
assert aggregate["platforms"]["xt"]["styles"] == 1
assert aggregate["platforms"]["xt"]["tasks"] == 2
def test_run_round_marks_exit_zero_platform_failed_when_summary_is_invalid(monkeypatch, tmp_path):
args = SimpleNamespace(dry_run=True, style=[1])
stdout = tmp_path / "stdout.log"
@@ -0,0 +1,110 @@
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
from gyxx_flow.modules.content_marketing import weekly_summary_all
def test_target_table_payload_uses_temp_cwd_and_relative_reference(
monkeypatch,
tmp_path: Path,
) -> None:
payload_root = tmp_path / "payloads"
payload_root.mkdir()
working_directory = tmp_path / "working-directory"
working_directory.mkdir()
monkeypatch.chdir(working_directory)
monkeypatch.setattr(
weekly_summary_all,
"PATHS",
SimpleNamespace(tmp_root=payload_root),
)
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
calls: list[tuple[list[str], Path | None]] = []
def fake_call_lark_json(
args: list[str], *, cwd: Path | None = None
) -> dict[str, object]:
calls.append((args, cwd))
if args[1] == "+record-list":
assert cwd is None
return {
"ok": True,
"data": {
"record_id_list": [],
"data": [],
"field_id_list": [],
},
}
assert cwd == payload_root
assert cwd != Path.cwd()
payload_reference = args[args.index("--json") + 1]
assert payload_reference.startswith("@./")
payload_path = cwd / payload_reference.removeprefix("@./")
assert not Path(payload_reference[1:]).is_absolute()
assert json.loads(payload_path.read_text(encoding="utf-8")) == {
"fld_time": "2026-07-01~07-31",
"fld_summary": (
"[凌云air 月度汇总(2026-07-01~07-31]"
"(https://example.test/doc)"
),
}
return {"ok": True}
monkeypatch.setattr(weekly_summary_all, "call_lark_json", fake_call_lark_json)
assert weekly_summary_all.write_to_target_table(
"base-test",
"table-test",
"fld_time",
"fld_summary",
"凌云air",
"https://example.test/doc",
False,
time_label="2026-07-01~07-31",
report_label="月度汇总",
)
assert [command[1] for command, _cwd in calls] == [
"+record-list",
"+record-upsert",
]
def test_call_lark_json_logs_only_redacted_failure_detail(
monkeypatch,
capsys,
tmp_path: Path,
) -> None:
raw_secret = "sensitive-token-value"
raw_open_id = "sensitive-open-id"
captured_kwargs: dict[str, object] = {}
def fake_run(*_args, **kwargs):
captured_kwargs.update(kwargs)
return SimpleNamespace(
returncode=1,
stdout="",
stderr=(
f"request failed; base_token={raw_secret}; "
f"open_id={raw_open_id}; https://example.test/private"
),
)
monkeypatch.setattr(weekly_summary_all.subprocess, "run", fake_run)
response = weekly_summary_all.call_lark_json(
["base", "+record-upsert"], cwd=tmp_path
)
captured = capsys.readouterr().out
assert captured_kwargs["cwd"] == tmp_path
assert response["ok"] is False
assert raw_secret not in response["error"]
assert raw_open_id not in response["error"]
assert raw_secret not in captured
assert raw_open_id not in captured
assert "[REDACTED]" in captured
assert "[URL]" in captured
File diff suppressed because it is too large Load Diff
@@ -5,6 +5,32 @@ from gyxx_flow.modules.content_marketing import weekly_summary_all as weekly
class WeeklyTmallPersonaTests(unittest.TestCase):
def test_summary_result_requires_all_real_external_effects(self):
self.assertTrue(
weekly.summary_results_succeeded(
[{"status": "ok"}, {"status": "no_notes"}]
)
)
for status in (
"no_fields",
"no_reports",
"llm_failed",
"doc_failed",
"doc_ambiguous",
"db_failed",
"write_failed",
"exception",
):
with self.subTest(status=status):
self.assertFalse(
weekly.summary_results_succeeded([{"status": status}])
)
def test_dry_run_status_is_only_successful_in_dry_run_mode(self):
results = [{"status": "dry_run"}]
self.assertFalse(weekly.summary_results_succeeded(results))
self.assertTrue(weekly.summary_results_succeeded(results, dry_run=True))
def test_missing_persona_forbids_invented_percentages(self):
text = weekly.format_tmall_persona(None)
self.assertIn("不得虚构具体比例", text)
@@ -56,7 +82,7 @@ class WeeklyTmallPersonaTests(unittest.TestCase):
}]
with patch.object(weekly, "get_latest_tmall_persona", return_value=persona) as get_persona, \
patch.object(weekly, "call_hermes_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
patch.object(weekly, "call_content_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
weekly.generate_summary("星云2", notes)
get_persona.assert_called_once_with("星云2")
@@ -0,0 +1,108 @@
import pytest
from gyxx_flow.modules.content_marketing import xiaohongshu_comment_scraper as scraper
class _EvaluatePage:
def __init__(self, result):
self.result = result
self.calls = []
def evaluate(self, script, argument):
self.calls.append((script, argument))
return self.result
def test_bounded_fetch_uses_abort_controller_and_clears_timer(monkeypatch):
page = _EvaluatePage(
{"__gyxx_status": "ok", "payload": {"comments": []}}
)
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
payload = scraper._fetch_json_with_timeout(
page,
"https://example.invalid/comments",
deadline=1000.0,
stage="top-level pagination",
)
assert payload == {"comments": []}
script, argument = page.calls[0]
assert "AbortController" in script
assert "clearTimeout(timer)" in script
assert argument["timeoutMs"] == 30_000
def test_bounded_fetch_timeout_is_an_explicit_transient_failure(monkeypatch):
page = _EvaluatePage({"__gyxx_status": "timeout"})
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
with pytest.raises(scraper.TransientXiaohongshuCommentTimeout) as caught:
scraper._fetch_json_with_timeout(
page,
"https://example.invalid/comments",
deadline=1000.0,
stage="reply pagination",
)
assert isinstance(caught.value, ConnectionError)
assert "reply pagination" in str(caught.value)
def test_scroll_checks_the_note_deadline_before_browser_work(monkeypatch):
page = _EvaluatePage(None)
monkeypatch.setattr(scraper.time, "monotonic", lambda: 901.0)
with pytest.raises(scraper.TransientXiaohongshuCommentTimeout):
scraper.scroll_comments(
page,
max_scrolls=30,
idle_rounds=4,
comments_by_id={},
deadline=900.0,
)
assert page.calls == []
def test_top_and_reply_pagination_share_the_bounded_fetch(monkeypatch):
stages = []
monkeypatch.setattr(scraper.time, "monotonic", lambda: 0.0)
def fetch(_page, _url, *, deadline, stage):
assert deadline == 900.0
stages.append(stage)
return {"comments": [], "has_more": 0, "cursor": 0}
monkeypatch.setattr(scraper, "_fetch_json_with_timeout", fetch)
monkeypatch.setattr(scraper, "collect_from_payload", lambda *_args: None)
page = _EvaluatePage(None)
scraper.fetch_missing_top_comments(
page,
"source-url",
{},
{
"comment_url_template": "https://example.invalid/top",
"api_total": 2,
"last_cursor": 1,
},
deadline=900.0,
)
scraper.fetch_missing_replies(
page,
"source-url",
{
"parent": {
"level": "comment",
"reply_count": 1,
"comment_id": "parent-id",
}
},
{
"sub_url_template": "https://example.invalid/reply",
},
deadline=900.0,
)
assert stages == ["top-level pagination", "reply pagination"]