feat: expand module workflows, dynamic config, notifications and console API
This commit is contained in:
@@ -0,0 +1,339 @@
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
|
||||
from gyxx_flow.modules.content_marketing import scrapling_http
|
||||
from gyxx_flow.modules.content_marketing import self_bilibili_scraper as self_bili
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
status: int = 200,
|
||||
payload: dict | None = None,
|
||||
text: str | None = None,
|
||||
content_type: str = "application/json; charset=utf-8",
|
||||
json_error: ValueError | None = None,
|
||||
) -> None:
|
||||
self.status_code = status
|
||||
self._payload = payload
|
||||
self.text = json.dumps(payload) if text is None and payload is not None else (text or "")
|
||||
self.content = self.text.encode("utf-8")
|
||||
self.headers = {"content-type": content_type}
|
||||
self._json_error = json_error
|
||||
|
||||
def json(self):
|
||||
if self._json_error is not None:
|
||||
raise self._json_error
|
||||
return self._payload
|
||||
|
||||
|
||||
class SequenceSession:
|
||||
def __init__(self, outcomes):
|
||||
self.outcomes = list(outcomes)
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
self.calls.append((url, kwargs))
|
||||
outcome = self.outcomes.pop(0)
|
||||
if isinstance(outcome, BaseException):
|
||||
raise outcome
|
||||
return outcome
|
||||
|
||||
|
||||
def test_scrapling_http_session_maps_fetcher_response(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def get(url, **kwargs):
|
||||
calls.append((url, kwargs))
|
||||
return SimpleNamespace(
|
||||
status=200,
|
||||
headers={"content-type": "application/json"},
|
||||
url="https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
body=b'{"code": 0}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(scrapling_http.Fetcher, "get", get)
|
||||
session = scrapling_http.ScraplingHttpSession(headers={"User-Agent": "gyxx"})
|
||||
|
||||
response = session.get(
|
||||
"https://b23.tv/demo",
|
||||
allow_redirects=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
assert calls == [(
|
||||
"https://b23.tv/demo",
|
||||
{
|
||||
"headers": {"User-Agent": "gyxx"},
|
||||
"follow_redirects": "all",
|
||||
"retries": 1,
|
||||
"timeout": 10,
|
||||
},
|
||||
)]
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"code": 0}
|
||||
assert response.url.endswith("BV17NNw6jEPx")
|
||||
|
||||
|
||||
def test_self_bilibili_uses_scrapling_user_agent_instead_of_stale_override():
|
||||
session = SequenceSession([
|
||||
FakeResponse(
|
||||
payload={
|
||||
"code": 0,
|
||||
"data": {"stat": {"view": 456}, "pubdate": 1_700_000_000},
|
||||
}
|
||||
)
|
||||
])
|
||||
|
||||
result = self_bili.fetch_bili_stat(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result == (456, 1_700_000_000)
|
||||
assert session.calls == [(
|
||||
"https://api.bilibili.com/x/web-interface/view",
|
||||
{
|
||||
"params": {"bvid": "BV17NNw6jEPx"},
|
||||
"timeout": 10,
|
||||
"headers": {"Referer": "https://www.bilibili.com/"},
|
||||
},
|
||||
)]
|
||||
|
||||
|
||||
def success_response(view: int = 321) -> FakeResponse:
|
||||
return FakeResponse(payload={"code": 0, "message": "OK", "data": {"stat": {"view": view}}})
|
||||
|
||||
|
||||
def detail_success_response(view: int = 321) -> FakeResponse:
|
||||
return FakeResponse(
|
||||
payload={"code": 0, "message": "OK", "data": {"View": {"stat": {"view": view}}}}
|
||||
)
|
||||
|
||||
|
||||
def test_empty_body_retries_then_returns_play_count(monkeypatch, capsys):
|
||||
session = SequenceSession([FakeResponse(text=""), success_response(456)])
|
||||
sleeps = []
|
||||
monkeypatch.setattr(bili.time, "sleep", sleeps.append)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result == 456
|
||||
assert len(session.calls) == 2
|
||||
assert sleeps == [1.5]
|
||||
assert "[empty_body]" in capsys.readouterr().out
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("first_response", "category"),
|
||||
[
|
||||
(FakeResponse(status=412, text="request blocked", content_type="text/plain"), "http_412"),
|
||||
(FakeResponse(payload={"code": -509, "message": "request limit"}), "api_code_-509"),
|
||||
(
|
||||
FakeResponse(
|
||||
text="<html>temporary gateway page</html>",
|
||||
content_type="text/html",
|
||||
json_error=ValueError("not json"),
|
||||
),
|
||||
"invalid_json",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_transient_response_classes_are_retried(
|
||||
monkeypatch, capsys, first_response, category
|
||||
):
|
||||
session = SequenceSession([first_response, success_response()])
|
||||
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result == 321
|
||||
assert len(session.calls) == 2
|
||||
assert f"[{category}]" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_non_retryable_api_code_stops_after_one_attempt(monkeypatch, capsys):
|
||||
session = SequenceSession(
|
||||
[FakeResponse(payload={"code": -404, "message": "video not found"})]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
bili.time,
|
||||
"sleep",
|
||||
lambda _seconds: pytest.fail("terminal API result must not back off"),
|
||||
)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert len(session.calls) == 1
|
||||
assert "[api_code_-404]" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_transient_failures_exhaust_three_attempts_honestly(monkeypatch, capsys):
|
||||
session = SequenceSession(
|
||||
[
|
||||
*[FakeResponse(text="") for _ in range(3)],
|
||||
FakeResponse(status=412, text="fallback blocked", content_type="text/plain"),
|
||||
]
|
||||
)
|
||||
sleeps = []
|
||||
monkeypatch.setattr(bili.time, "sleep", sleeps.append)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert len(session.calls) == 4
|
||||
assert sleeps == [1.5, 3.0]
|
||||
output = capsys.readouterr().out
|
||||
assert "attempt=3/3" in output
|
||||
assert "[view_detail_http_412]" in output
|
||||
|
||||
|
||||
def test_view_detail_is_used_once_after_primary_transient_exhaustion(
|
||||
monkeypatch, capsys
|
||||
):
|
||||
session = SequenceSession(
|
||||
[*[FakeResponse(text="") for _ in range(3)], detail_success_response(789)]
|
||||
)
|
||||
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result == 789
|
||||
assert len(session.calls) == 4
|
||||
assert session.calls[-1][0] == bili._VIEW_DETAIL_API_URL
|
||||
assert "[FALLBACK]" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_play_count_failure_remains_an_incomplete_summary(monkeypatch):
|
||||
style = {
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"read_count_7d": {"field_id": "s7"},
|
||||
"read_count_14d": {"field_id": "s14"},
|
||||
"read_count_21d": {"field_id": "s21"},
|
||||
"read_count_28d": {"field_id": "s28"},
|
||||
"month_end": {"field_id": "sm"},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
bili,
|
||||
"list_all_records",
|
||||
lambda *_args: [{
|
||||
"record_id": "failed",
|
||||
"platform": "B站",
|
||||
"creator": "达人",
|
||||
"url": "https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
"pub": "2026-08-08",
|
||||
}],
|
||||
)
|
||||
monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
bili,
|
||||
"write_record",
|
||||
lambda *_args, **_kwargs: pytest.fail("failed fetch must not write"),
|
||||
)
|
||||
|
||||
summary = bili.process_style(
|
||||
style,
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=bili.date(2026, 8, 9),
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
assert summary["complete"] is False
|
||||
assert summary["retryable_failures"] == 1
|
||||
assert summary["details"][0]["reason"] == "play_count_fetch_failed"
|
||||
|
||||
|
||||
def test_terminal_unavailable_detail_is_a_complete_blocked_input(monkeypatch):
|
||||
style = {
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"read_count_7d": {"field_id": "s7"},
|
||||
"read_count_14d": {"field_id": "s14"},
|
||||
"read_count_21d": {"field_id": "s21"},
|
||||
"read_count_28d": {"field_id": "s28"},
|
||||
"month_end": {"field_id": "sm"},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
bili,
|
||||
"list_all_records",
|
||||
lambda *_args: [{
|
||||
"record_id": "unavailable",
|
||||
"platform": "B站",
|
||||
"creator": "达人",
|
||||
"url": "https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
"pub": "2026-08-08",
|
||||
}],
|
||||
)
|
||||
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(
|
||||
bili,
|
||||
"write_record",
|
||||
lambda *_args, **_kwargs: pytest.fail("unavailable video must not write"),
|
||||
)
|
||||
session = SequenceSession([
|
||||
*[FakeResponse(status=412, text="request blocked") for _ in range(3)],
|
||||
FakeResponse(payload={"code": 62002, "message": "稿件不可见", "data": None}),
|
||||
])
|
||||
|
||||
summary = bili.process_style(
|
||||
style,
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=session,
|
||||
state={},
|
||||
force_today=bili.date(2026, 8, 9),
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
assert len(session.calls) == 4
|
||||
assert summary["complete"] is True
|
||||
assert summary["blocked_input"] == 1
|
||||
assert summary["retryable_failures"] == 0
|
||||
assert summary["unresolved"] == 0
|
||||
assert summary["details"][0]["status"] == "blocked_input"
|
||||
assert summary["details"][0]["reason"] == "video_unavailable"
|
||||
@@ -0,0 +1,346 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from argparse import Namespace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
from gyxx_flow.modules.content_marketing.data.tools import retry_failed
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
sync_metrics_to_cmt_notes as sync_metrics,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_content_source():
|
||||
xingtu.configure_source("xingtu")
|
||||
yield
|
||||
xingtu.configure_source("xingtu")
|
||||
|
||||
|
||||
def test_buyin_source_uses_doudian_market_and_cookie_scope():
|
||||
xingtu.configure_source("buyin")
|
||||
|
||||
assert xingtu.CONTENT_SOURCE == "buyin"
|
||||
assert xingtu.MARKET_URL.startswith(
|
||||
"https://buyin.jinritemai.com/dashboard/servicehall/daren-square"
|
||||
)
|
||||
assert xingtu.DETAIL_URL_MARK == "/dashboard/servicehall/daren-profile"
|
||||
assert xingtu.BUSINESS_COOKIE_DOMAINS == ("jinritemai.com",)
|
||||
|
||||
|
||||
def test_save_state_creates_cookie_parent_directory(tmp_path, monkeypatch):
|
||||
cookie_file = tmp_path / "nested" / "cookies.json"
|
||||
monkeypatch.setattr(xingtu, "DATA_DIR", tmp_path / "raw")
|
||||
monkeypatch.setattr(xingtu, "COOKIE_FILE", cookie_file)
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [{"name": "sessionid", "value": "redacted"}]
|
||||
|
||||
class Page:
|
||||
context = Context()
|
||||
|
||||
xingtu.save_state(Page())
|
||||
|
||||
assert cookie_file.exists()
|
||||
assert "sessionid" in cookie_file.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_run_all_keeps_cooperation_xt_on_xingtu_for_buyin_fallback():
|
||||
args = Namespace(
|
||||
dry_run=False,
|
||||
style=None,
|
||||
daily_scope=False,
|
||||
no_retry=False,
|
||||
)
|
||||
|
||||
command = run_all.build_process_command("xt", args)
|
||||
|
||||
assert "--buyin" not in command
|
||||
assert "--skip-field-prepare" in command
|
||||
|
||||
|
||||
def test_retry_failed_keeps_cooperation_xt_on_xingtu_for_buyin_fallback():
|
||||
assert retry_failed.SCRIPTS["xt"]["extra_args"] == []
|
||||
|
||||
|
||||
def test_buyin_video_parser_is_dom_html_based_and_maps_viewers_to_exposure():
|
||||
xingtu.configure_source("buyin")
|
||||
|
||||
class Page:
|
||||
def __init__(self):
|
||||
self.script = ""
|
||||
|
||||
def evaluate(self, script):
|
||||
self.script = script
|
||||
return [{
|
||||
"title": "#达人笔记标题",
|
||||
"play_count": "1,155",
|
||||
"view_count": "1,155",
|
||||
"exposure_count": "1,155",
|
||||
}]
|
||||
|
||||
page = Page()
|
||||
cards = xingtu.parse_videos_on_page(page)
|
||||
|
||||
assert cards[0]["play_count"] == "1,155"
|
||||
assert cards[0]["exposure_count"] == "1,155"
|
||||
assert "观看人数" in page.script
|
||||
assert "innerText" in page.script
|
||||
|
||||
|
||||
def test_buyin_history_tab_and_trade_video_filter_are_selected():
|
||||
xingtu.configure_source("buyin")
|
||||
events = []
|
||||
filter_states = {"只看带货视频": True, "只看带货图文": True}
|
||||
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def filter(self, **kwargs):
|
||||
return Locator(kwargs.get("has_text", self.name))
|
||||
|
||||
def click(self, **_kwargs):
|
||||
events.append(f"click-{self.name}")
|
||||
if self.name in filter_states:
|
||||
filter_states[self.name] = False
|
||||
|
||||
class Page:
|
||||
def get_by_text(self, text, **_kwargs):
|
||||
return Locator(text)
|
||||
|
||||
def get_by_role(self, _role, name, **_kwargs):
|
||||
return Locator(name)
|
||||
|
||||
def locator(self, _selector):
|
||||
return Locator("selector")
|
||||
|
||||
def evaluate(self, script, *_args):
|
||||
events.append(script)
|
||||
if _args and _args[0] in filter_states:
|
||||
return filter_states[_args[0]]
|
||||
return False
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
events.append(milliseconds)
|
||||
|
||||
assert xingtu.open_creation_ability(Page()) is True
|
||||
assert "click-历史内容" in events
|
||||
assert "click-近30天" in events
|
||||
assert "click-视频" in events
|
||||
assert "click-只看带货视频" in events
|
||||
assert "click-只看带货图文" in events
|
||||
assert any("scrollIntoView" in event for event in events if isinstance(event, str))
|
||||
|
||||
|
||||
def test_buyin_creator_search_clicks_real_button_and_waits_for_changed_results(
|
||||
monkeypatch,
|
||||
):
|
||||
xingtu.configure_source("buyin")
|
||||
events = []
|
||||
|
||||
class Locator:
|
||||
def __init__(self, kind):
|
||||
self.kind = kind
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
return None
|
||||
|
||||
def click(self, **_kwargs):
|
||||
events.append(f"click-{self.kind}")
|
||||
|
||||
def fill(self, value):
|
||||
events.append(f"fill-{value}")
|
||||
|
||||
def press(self, key):
|
||||
events.append(f"press-{key}")
|
||||
|
||||
class Page:
|
||||
def locator(self, selector):
|
||||
if selector.startswith("button.auxo-input-search-button"):
|
||||
return Locator("button")
|
||||
return Locator("input")
|
||||
|
||||
def evaluate(self, _script):
|
||||
return "idle"
|
||||
|
||||
def get_by_role(self, *_args, **_kwargs):
|
||||
raise AssertionError("Buyin should use the real search button first")
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(xingtu, "dismiss_popups", lambda _page: None)
|
||||
monkeypatch.setattr(xingtu, "select_nickname_search", lambda _page: None)
|
||||
monkeypatch.setattr(xingtu, "_buyin_result_page_signature", lambda _page: "before")
|
||||
observed = {}
|
||||
|
||||
def wait_for_results(_page, _name, timeout_ms, previous_signature=None):
|
||||
observed.update(timeout_ms=timeout_ms, previous_signature=previous_signature)
|
||||
|
||||
monkeypatch.setattr(xingtu, "wait_for_visible_creator_result", wait_for_results)
|
||||
|
||||
xingtu.search_creator(Page(), "linyl燕", "LinFX2327")
|
||||
|
||||
assert events == ["click-input", "fill-LinFX2327", "click-button"]
|
||||
assert observed == {"timeout_ms": 30000, "previous_signature": "before"}
|
||||
|
||||
|
||||
def test_buyin_search_button_waits_until_loading_finishes():
|
||||
xingtu.configure_source("buyin")
|
||||
states = iter(["busy", "busy", "idle"])
|
||||
waits = []
|
||||
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return next(states)
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
waits.append(milliseconds)
|
||||
|
||||
xingtu._wait_for_buyin_search_button_idle(Page(), timeout_ms=1000)
|
||||
|
||||
assert waits == [250, 250]
|
||||
|
||||
|
||||
def test_buyin_wait_distinguishes_temporary_search_throttling():
|
||||
xingtu.configure_source("buyin")
|
||||
|
||||
class Page:
|
||||
def evaluate(self, *_args):
|
||||
return "throttled"
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pytest.fail("throttling must return immediately")
|
||||
|
||||
with pytest.raises(xingtu.SearchTemporarilyThrottled, match="过于频繁"):
|
||||
xingtu.wait_for_visible_creator_result(Page(), "LinFX2327", timeout_ms=1000)
|
||||
|
||||
|
||||
def test_buyin_creator_scan_supports_single_row_without_profile_href():
|
||||
script = xingtu._buyin_scan_creator_script()
|
||||
|
||||
assert "tr[data-row-key], [role=\"row\"][data-row-key]" in script
|
||||
assert "rowKey: row.getAttribute('data-row-key')" in script
|
||||
assert "via: 'single-search-result'" in script
|
||||
assert "first-id-search-result" not in script
|
||||
|
||||
|
||||
def test_buyin_next_page_waits_for_a_changed_html_signature():
|
||||
xingtu.configure_source("buyin")
|
||||
signatures = iter(["1/4|first-card", "2/4|second-card"])
|
||||
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
if "const selectors" in script:
|
||||
return {"clicked": True, "picked": "semi-page-next"}
|
||||
return next(signatures)
|
||||
|
||||
def locator(self, _selector):
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def click(self, **_kwargs):
|
||||
return None
|
||||
|
||||
return Locator()
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
assert xingtu.go_next_page(Page()) is True
|
||||
|
||||
|
||||
def test_metric_snapshot_date_prefers_orchestrator_business_date(monkeypatch):
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-31")
|
||||
|
||||
assert sync_metrics._resolve_metric_date() == dt.date(2026, 8, 31)
|
||||
assert sync_metrics._resolve_metric_date("2026-08-30") == dt.date(2026, 8, 30)
|
||||
with pytest.raises(ValueError, match="YYYY-MM-DD"):
|
||||
sync_metrics._resolve_metric_date("not-a-date")
|
||||
|
||||
|
||||
def test_metric_snapshot_upsert_is_idempotent_by_note_and_business_date():
|
||||
class Cursor:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.calls.append((" ".join(sql.split()), params))
|
||||
|
||||
cursor = Cursor()
|
||||
sync_metrics._upsert_metric_snapshot(
|
||||
cursor,
|
||||
42,
|
||||
{
|
||||
"view_count": 1155,
|
||||
"dst_platform": "douyin",
|
||||
"self_operated": False,
|
||||
"record_id": "rec-1",
|
||||
"source_url": "https://www.douyin.com/video/42",
|
||||
},
|
||||
dt.date(2026, 8, 31),
|
||||
)
|
||||
|
||||
sql, params = cursor.calls[0]
|
||||
assert "INSERT INTO cmt_note_metric_snapshots" in sql
|
||||
assert "ON CONFLICT (note_id, metric_date)" in sql
|
||||
assert params[0:2] == (42, dt.date(2026, 8, 31))
|
||||
assert params[6] == 1155
|
||||
|
||||
|
||||
def test_snapshot_schema_and_report_query_use_daily_history():
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
schema = (
|
||||
root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "content_marketing"
|
||||
/ "data"
|
||||
/ "tools"
|
||||
/ "schema_gyxx_super_data.sql"
|
||||
).read_text(encoding="utf-8")
|
||||
migration = (
|
||||
root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "content_marketing"
|
||||
/ "data"
|
||||
/ "tools"
|
||||
/ "migrations"
|
||||
/ "008_note_metric_snapshots.sql"
|
||||
).read_text(encoding="utf-8")
|
||||
report = (
|
||||
root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "content_marketing"
|
||||
/ "daily_marketing_report.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
for source in (schema, migration):
|
||||
assert "CREATE TABLE IF NOT EXISTS cmt_note_metric_snapshots" in source
|
||||
assert "UNIQUE (note_id, metric_date)" in source
|
||||
assert "COALESCE(ms.view_count, n.view_count)" in report
|
||||
assert "ms.metric_date = %s" in report
|
||||
@@ -1,7 +1,14 @@
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.content_marketing import chanmama_scraper as chanmama
|
||||
from gyxx_flow.modules.content_marketing import feishu_mapping
|
||||
from gyxx_flow.modules.content_marketing import (
|
||||
scrapling_browser_facade as browser_facade,
|
||||
)
|
||||
|
||||
|
||||
def test_exposure_state_clicks_text_and_zero_but_keeps_positive_values():
|
||||
@@ -152,6 +159,277 @@ def test_missing_get_data_button_does_not_trigger_refresh_wait(monkeypatch):
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
def test_create_driver_uses_runtime_cdp_profile_and_download_bindings(monkeypatch, tmp_path):
|
||||
calls = []
|
||||
expected = object()
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
|
||||
|
||||
def start(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
|
||||
|
||||
assert chanmama.create_driver() is expected
|
||||
assert calls == [{
|
||||
"profile_dir": str(tmp_path / "profile"),
|
||||
"download_dir": str(tmp_path / "downloads"),
|
||||
"cdp_url": "http://127.0.0.1:22002",
|
||||
"stealthy": True,
|
||||
}]
|
||||
|
||||
|
||||
def test_create_driver_starts_scrapling_without_cdp(monkeypatch, tmp_path):
|
||||
calls = []
|
||||
expected = object()
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.delenv("GYXX_BROWSER_CDP_URL", raising=False)
|
||||
|
||||
def start(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
|
||||
|
||||
assert chanmama.create_driver() is expected
|
||||
assert calls[0]["cdp_url"] is None
|
||||
assert calls[0]["stealthy"] is True
|
||||
assert calls[0]["real_chrome"] is False
|
||||
|
||||
|
||||
def test_create_driver_falls_back_when_bound_cdp_is_not_listening(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
calls = []
|
||||
expected = object()
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
|
||||
|
||||
def start(**kwargs):
|
||||
calls.append(kwargs)
|
||||
if kwargs["cdp_url"]:
|
||||
raise RuntimeError(
|
||||
"BrowserType.connect_over_cdp: connect ECONNREFUSED "
|
||||
"127.0.0.1:22002; retrieving websocket url"
|
||||
)
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
|
||||
|
||||
assert chanmama.create_driver() is expected
|
||||
assert [call["cdp_url"] for call in calls] == [
|
||||
"http://127.0.0.1:22002",
|
||||
None,
|
||||
]
|
||||
assert all(call["stealthy"] is True for call in calls)
|
||||
assert calls[1]["real_chrome"] is False
|
||||
|
||||
|
||||
def test_create_driver_keeps_explicit_remote_cdp_fail_closed(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://browser.internal:22002")
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"ScraplingDriver",
|
||||
SimpleNamespace(
|
||||
start=lambda **_kwargs: (_ for _ in ()).throw(
|
||||
RuntimeError("connect ECONNREFUSED browser.internal:22002")
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="ECONNREFUSED"):
|
||||
chanmama.create_driver()
|
||||
|
||||
|
||||
def test_create_driver_does_not_hide_non_connection_cdp_failures(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"ScraplingDriver",
|
||||
SimpleNamespace(
|
||||
start=lambda **_kwargs: (_ for _ in ()).throw(
|
||||
RuntimeError("browser profile is corrupt")
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="profile is corrupt"):
|
||||
chanmama.create_driver()
|
||||
|
||||
|
||||
def test_scrapling_facade_reuses_runtime_cdp_context(monkeypatch, tmp_path):
|
||||
created = []
|
||||
|
||||
class Page:
|
||||
def on(self, *_args):
|
||||
return None
|
||||
|
||||
def add_init_script(self, script):
|
||||
assert "navigator" in script
|
||||
|
||||
class Context:
|
||||
pages = [Page()]
|
||||
|
||||
class Browser:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.context = Context()
|
||||
self.page = None
|
||||
self.closed = False
|
||||
created.append(self)
|
||||
|
||||
def start(self):
|
||||
return self
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
monkeypatch.setattr(browser_facade, "ScraplingBrowser", Browser)
|
||||
|
||||
driver = browser_facade.ScraplingDriver.start(
|
||||
profile_dir=tmp_path / "profile",
|
||||
download_dir=tmp_path / "downloads",
|
||||
cdp_url="http://127.0.0.1:22002",
|
||||
)
|
||||
|
||||
assert created[0].kwargs["reuse_existing_cdp_context"] is True
|
||||
assert created[0].kwargs["cdp_url"] == "http://127.0.0.1:22002"
|
||||
assert "user_data_dir" not in created[0].kwargs
|
||||
assert "--no-sandbox" not in created[0].kwargs["extra_flags"]
|
||||
assert created[0].page is created[0].context.pages[0]
|
||||
driver.quit()
|
||||
assert created[0].closed is True
|
||||
|
||||
|
||||
def _one_backfill_style():
|
||||
return {
|
||||
"tables": [
|
||||
{
|
||||
"index": 1,
|
||||
"name": "宙斯",
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "published"},
|
||||
"read_count_7d": {
|
||||
"field_id": "read-7d",
|
||||
"field_name": "7日阅读量",
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def test_backfill_mapping_or_record_list_failure_returns_false(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping,
|
||||
"load_mapping",
|
||||
lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("mapping down")),
|
||||
)
|
||||
assert chanmama.backfill_self_tables([]) is False
|
||||
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping, "load_mapping", lambda **_kwargs: {"tables": []}
|
||||
)
|
||||
assert chanmama.backfill_self_tables([]) is False
|
||||
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping, "load_mapping", lambda **_kwargs: _one_backfill_style()
|
||||
)
|
||||
assert chanmama.backfill_self_tables([], only_style=999) is False
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(normalized_root=tmp_path / "normalized"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_list_records_by_table",
|
||||
lambda *_args: (_ for _ in ()).throw(RuntimeError("base down")),
|
||||
)
|
||||
|
||||
assert chanmama.backfill_self_tables([]) is False
|
||||
|
||||
|
||||
def test_backfill_write_failure_is_false_but_natural_no_match_is_success(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping, "load_mapping", lambda **_kwargs: _one_backfill_style()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(normalized_root=tmp_path / "normalized"),
|
||||
)
|
||||
record = {
|
||||
"record_id": "record-1",
|
||||
"platform": "抖音",
|
||||
"title": "相同标题",
|
||||
"published": datetime.now(),
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
chanmama, "_list_records_by_table", lambda *_args: [record]
|
||||
)
|
||||
monkeypatch.setattr(chanmama, "_write_back", lambda *_args: False)
|
||||
|
||||
assert chanmama.backfill_self_tables(
|
||||
[{"title": "相同标题", "exposure": 100}]
|
||||
) is False
|
||||
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_write_back",
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
AssertionError("natural no-match must not write")
|
||||
),
|
||||
)
|
||||
assert chanmama.backfill_self_tables(
|
||||
[{"title": "完全不同", "exposure": 100}]
|
||||
) is True
|
||||
|
||||
|
||||
def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
|
||||
events = []
|
||||
|
||||
@@ -159,7 +437,6 @@ def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
|
||||
def quit(self):
|
||||
events.append("quit")
|
||||
|
||||
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
|
||||
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
|
||||
@@ -180,10 +457,75 @@ def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
|
||||
lambda *_args, **_kwargs: ([], []),
|
||||
)
|
||||
|
||||
assert chanmama.main() == 0
|
||||
assert chanmama.main() == 1
|
||||
assert events == ["saved", "quit"]
|
||||
|
||||
|
||||
def test_main_full_mode_exception_and_keyboard_interrupt_are_nonzero(monkeypatch):
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"create_driver",
|
||||
lambda: (_ for _ in ()).throw(RuntimeError("driver failed")),
|
||||
)
|
||||
assert chanmama.main() == 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"create_driver",
|
||||
lambda: (_ for _ in ()).throw(KeyboardInterrupt()),
|
||||
)
|
||||
assert chanmama.main() == 130
|
||||
|
||||
|
||||
def test_main_propagates_backfill_failure_after_nonempty_export(monkeypatch):
|
||||
events = []
|
||||
|
||||
class Driver:
|
||||
def quit(self):
|
||||
events.append("quit")
|
||||
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
|
||||
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
|
||||
monkeypatch.setattr(chanmama, "save_cookies", lambda _driver: True)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"refresh_then_export_accounts",
|
||||
lambda *_args, **_kwargs: ([{"title": "video"}], []),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"backfill_self_tables",
|
||||
lambda *_args, **_kwargs: False,
|
||||
)
|
||||
|
||||
assert chanmama.main() == 1
|
||||
assert events == ["quit"]
|
||||
|
||||
|
||||
def test_backfill_only_propagates_backfill_failure(monkeypatch, tmp_path):
|
||||
workbook = tmp_path / "existing.xlsx"
|
||||
workbook.write_bytes(b"placeholder")
|
||||
monkeypatch.setattr(
|
||||
chanmama.sys,
|
||||
"argv",
|
||||
["chanmama_scraper.py", "--backfill-only", "--excel", str(workbook)],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"parse_chanmama_excel",
|
||||
lambda _path: ([{"title": "video"}], []),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"backfill_self_tables",
|
||||
lambda *_args, **_kwargs: False,
|
||||
)
|
||||
|
||||
assert chanmama.main() == 1
|
||||
|
||||
|
||||
def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
@@ -196,7 +538,6 @@ def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
|
||||
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "ACCOUNT", "")
|
||||
monkeypatch.setattr(chanmama, "PASSWORD", "")
|
||||
@@ -251,3 +592,85 @@ def test_cli_preserves_cookie_skip_exit_code(monkeypatch):
|
||||
chanmama.cli()
|
||||
|
||||
assert raised.value.code == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_polls_until_file_appears(monkeypatch):
|
||||
now = [1000.0]
|
||||
sleeps: list[float] = []
|
||||
results = iter([None, None, "/downloads/博主A_视频记录_周期内发布视频.xlsx"])
|
||||
|
||||
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
|
||||
monkeypatch.setattr(
|
||||
chanmama.time,
|
||||
"sleep",
|
||||
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
|
||||
)
|
||||
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
|
||||
|
||||
assert chanmama._wait_for_exported_excel(0.0) == (
|
||||
"/downloads/博主A_视频记录_周期内发布视频.xlsx"
|
||||
)
|
||||
assert sleeps == [5.0, 5.0]
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_returns_none_after_timeout(monkeypatch):
|
||||
now = [1000.0]
|
||||
sleeps: list[float] = []
|
||||
|
||||
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
|
||||
monkeypatch.setattr(
|
||||
chanmama.time,
|
||||
"sleep",
|
||||
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
|
||||
)
|
||||
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: None)
|
||||
|
||||
assert (
|
||||
chanmama._wait_for_exported_excel(
|
||||
0.0,
|
||||
timeout_seconds=12,
|
||||
grace_seconds=0,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert len(sleeps) == 3 # 1000 -> 1005 -> 1010 -> 1015(>= 1012 超时)
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_allows_boundary_grace(monkeypatch):
|
||||
now = [1000.0]
|
||||
sleeps: list[float] = []
|
||||
results = iter([None, None, "/downloads/边界后落盘.xlsx"])
|
||||
|
||||
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
|
||||
monkeypatch.setattr(
|
||||
chanmama.time,
|
||||
"sleep",
|
||||
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
|
||||
)
|
||||
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
|
||||
|
||||
assert chanmama._wait_for_exported_excel(
|
||||
0.0,
|
||||
timeout_seconds=5,
|
||||
grace_seconds=5,
|
||||
poll_interval_seconds=5,
|
||||
) == "/downloads/边界后落盘.xlsx"
|
||||
assert sleeps == [5.0, 5.0]
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_returns_immediately_when_present(monkeypatch):
|
||||
now = [1000.0]
|
||||
|
||||
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
|
||||
monkeypatch.setattr(
|
||||
chanmama.time,
|
||||
"sleep",
|
||||
lambda _seconds: pytest.fail("不应等待已存在的文件"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_find_new_excel_after",
|
||||
lambda _pre_mtime: "/downloads/已有.xlsx",
|
||||
)
|
||||
|
||||
assert chanmama._wait_for_exported_excel(0.0) == "/downloads/已有.xlsx"
|
||||
|
||||
@@ -9,6 +9,10 @@ from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
from gyxx_flow.modules.content_marketing.serialized_page_action import (
|
||||
is_transient_browser_error,
|
||||
run_serialized_page_action,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_title_handles_nfkc_case_and_zero_width():
|
||||
@@ -131,6 +135,28 @@ def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
|
||||
assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
|
||||
|
||||
|
||||
def test_xingtu_can_use_strong_title_when_share_id_is_canonicalized():
|
||||
task = {
|
||||
"record_id": "r1",
|
||||
"target_title": "这家包颜值和实用性真挺高",
|
||||
"note_id": "7665297310758210854",
|
||||
}
|
||||
card = {
|
||||
"title": "这家包颜值和实用性真挺高 #潮流运动 #光影行星",
|
||||
"note_id": "canonical- Xingtu-item-id",
|
||||
}
|
||||
|
||||
matched = cc.match_tasks_to_cards(
|
||||
[task],
|
||||
[card],
|
||||
platform="douyin",
|
||||
allow_mismatched_content_id_title=True,
|
||||
)
|
||||
|
||||
assert matched["r1"]["match_method"] == "title"
|
||||
assert matched["r1"]["match_score"] >= 0.9
|
||||
|
||||
|
||||
def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
|
||||
cards = [
|
||||
{
|
||||
@@ -354,8 +380,11 @@ class _FakeSearchPage:
|
||||
def test_pgy_search_retry_keeps_creator_id_query():
|
||||
page = _FakeSearchPage()
|
||||
|
||||
pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
|
||||
found = pgy._ensure_search_results(
|
||||
page, query="xhs-id-123", result_name="达人昵称", max_retries=2
|
||||
)
|
||||
|
||||
assert found is True
|
||||
assert page.input.filled == ["xhs-id-123"]
|
||||
|
||||
|
||||
@@ -376,11 +405,19 @@ def test_apply_matched_task_reports_write_failure(monkeypatch):
|
||||
assert outcome["write_ok"] is False
|
||||
|
||||
|
||||
def test_default_creator_batch_has_no_forced_hour_pause():
|
||||
assert pgy.DEFAULT_BATCH_SIZE == 0
|
||||
def test_pgy_default_creator_batch_rotates_browser_before_memory_accumulates():
|
||||
assert pgy.DEFAULT_BATCH_SIZE == 4
|
||||
assert xingtu.DEFAULT_BATCH_SIZE == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("message", [
|
||||
"Page crashed: Out of Memory",
|
||||
"Target page, context or browser has been closed: not enough memory",
|
||||
])
|
||||
def test_pgy_treats_chromium_memory_exhaustion_as_a_recoverable_session_loss(message):
|
||||
assert pgy.is_browser_session_lost(RuntimeError(message)) is True
|
||||
|
||||
|
||||
def test_repeatable_style_argument_and_selection_keep_every_requested_style():
|
||||
parser = argparse.ArgumentParser()
|
||||
cc.add_repeatable_style_argument(parser, "styles")
|
||||
@@ -408,6 +445,223 @@ def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
|
||||
assert page.waits == 2
|
||||
|
||||
|
||||
def test_pgy_stable_cards_waits_through_a_slow_initial_render(monkeypatch):
|
||||
card = {"title": "标题", "read_count": "1"}
|
||||
sequence = [[], [], [], [], [], [card], [card]]
|
||||
monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
|
||||
|
||||
class Page:
|
||||
waits = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
self.waits += 1
|
||||
|
||||
page = Page()
|
||||
assert pgy.parse_cards_stable(page) == [card]
|
||||
assert page.waits == 6
|
||||
|
||||
|
||||
def test_pgy_card_parser_recovers_note_id_from_internal_card_link():
|
||||
note_id = "0123456789abcdef01234567"
|
||||
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return [{
|
||||
"title": "标题",
|
||||
"read_count": "1",
|
||||
"href": "",
|
||||
"note_id": "",
|
||||
"_href_candidates": [
|
||||
f"https://pgy.xiaohongshu.com/internal?source_note_id={note_id}"
|
||||
],
|
||||
"_data_note_ids": [],
|
||||
}]
|
||||
|
||||
cards = pgy.parse_cards_on_page(Page())
|
||||
|
||||
assert cards == [{
|
||||
"title": "标题",
|
||||
"read_count": "1",
|
||||
"href": "",
|
||||
"note_id": note_id,
|
||||
}]
|
||||
|
||||
|
||||
def test_pgy_card_parser_prefers_strict_note_detail_id_over_generic_segment():
|
||||
note_id = "0123456789abcdef01234567"
|
||||
internal_href = f"https://example.invalid/note/detail/{note_id}"
|
||||
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return [{
|
||||
"title": "标题",
|
||||
"read_count": "1",
|
||||
"href": internal_href,
|
||||
# This is what the old generic /note/<segment> parser produced.
|
||||
"note_id": "detail",
|
||||
"_href_candidates": [internal_href],
|
||||
"_data_note_ids": [],
|
||||
}]
|
||||
|
||||
cards = pgy.parse_cards_on_page(Page())
|
||||
|
||||
assert cards[0]["note_id"] == note_id
|
||||
|
||||
|
||||
def test_pgy_dom_parser_reads_root_data_id_and_rejects_arbitrary_href_fallback():
|
||||
captured = {}
|
||||
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
captured["script"] = script
|
||||
return []
|
||||
|
||||
assert pgy.parse_cards_on_page(Page()) == []
|
||||
assert "const dataNoteNodes = [card, ...card.querySelectorAll" in captured["script"]
|
||||
assert "const href = link?.href || '';" in captured["script"]
|
||||
assert "hrefCandidates[0]" not in captured["script"]
|
||||
|
||||
|
||||
def test_pgy_arbitrary_shared_anchor_does_not_collapse_distinct_cards():
|
||||
shared_anchor = "https://example.invalid/creator/profile"
|
||||
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return [
|
||||
{
|
||||
"title": "第一篇笔记",
|
||||
"read_count": "1",
|
||||
"like_count": "1",
|
||||
"collect_count": "1",
|
||||
"publish_time": "2026-08-01",
|
||||
"href": "",
|
||||
"note_id": "",
|
||||
"_href_candidates": [shared_anchor],
|
||||
"_data_note_ids": [],
|
||||
},
|
||||
{
|
||||
"title": "第二篇笔记",
|
||||
"read_count": "2",
|
||||
"like_count": "2",
|
||||
"collect_count": "2",
|
||||
"publish_time": "2026-08-02",
|
||||
"href": "",
|
||||
"note_id": "",
|
||||
"_href_candidates": [shared_anchor],
|
||||
"_data_note_ids": [],
|
||||
},
|
||||
]
|
||||
|
||||
cards = pgy.parse_cards_on_page(Page())
|
||||
|
||||
assert [card["href"] for card in cards] == ["", ""]
|
||||
assert pgy._card_observation_key(cards[0]) != pgy._card_observation_key(cards[1])
|
||||
|
||||
|
||||
def test_pgy_deduplicates_one_card_repeated_across_pages_before_matching(monkeypatch):
|
||||
card = {
|
||||
"title": "同一篇笔记",
|
||||
"read_count": "100",
|
||||
"like_count": "10",
|
||||
"collect_count": "5",
|
||||
"publish_time": "2026-08-01",
|
||||
"href": "",
|
||||
"note_id": "",
|
||||
}
|
||||
pages = [[dict(card)], [dict(card)], [dict(card)]]
|
||||
monkeypatch.setattr(pgy, "parse_cards_stable", lambda _page: pages.pop(0))
|
||||
monkeypatch.setattr(pgy, "go_next_page", lambda _page: bool(pages))
|
||||
|
||||
found, candidates, page_limit_hit = pgy.find_notes_for_tasks(
|
||||
object(),
|
||||
[{"record_id": "r1", "target_title": "同一篇笔记"}],
|
||||
max_pages=4,
|
||||
)
|
||||
|
||||
assert found["r1"]["read_count"] == "100"
|
||||
assert candidates == ["同一篇笔记"]
|
||||
assert page_limit_hit is False
|
||||
|
||||
|
||||
def test_pgy_missing_creator_is_blocked_but_unmatched_note_stays_retryable():
|
||||
no_creator = pgy._missing_detail_outcome(saw_search_result=False)
|
||||
no_unique_note = pgy._unmatched_note_outcome(
|
||||
candidates=["平台上的其他笔记"],
|
||||
page_limit_hit=False,
|
||||
)
|
||||
completed = cc.finalize_summary({
|
||||
"total": 2,
|
||||
"results": [
|
||||
{"record_id": "creator", **no_creator},
|
||||
{"record_id": "note", **no_unique_note},
|
||||
],
|
||||
})
|
||||
|
||||
assert completed["blocked_input"] == 1
|
||||
assert completed["retryable_failures"] == 1
|
||||
assert completed["unresolved"] == 1
|
||||
assert completed["complete"] is False
|
||||
assert pgy._missing_detail_outcome(saw_search_result=True)["status"] == "retryable_failure"
|
||||
assert pgy._unmatched_note_outcome(
|
||||
candidates=[], page_limit_hit=False
|
||||
)["reason"] == "empty_detail_page"
|
||||
assert pgy._unmatched_note_outcome(
|
||||
candidates=["候选"], page_limit_hit=True
|
||||
)["status"] == "retryable_failure"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state",
|
||||
[
|
||||
{
|
||||
"nextFound": True,
|
||||
"nextDisabled": True,
|
||||
"activePage": None,
|
||||
"lastPage": None,
|
||||
"numericPageCount": 0,
|
||||
},
|
||||
{
|
||||
"nextFound": True,
|
||||
"nextDisabled": False,
|
||||
"activePage": 7,
|
||||
"lastPage": 7,
|
||||
"numericPageCount": 7,
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_pgy_known_last_page_stops_without_clicking(state):
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
if "pgy-pagination-state" in script:
|
||||
return state
|
||||
if "pgy-pagination-click" in script:
|
||||
pytest.fail("terminal paginator must not be clicked")
|
||||
return "last-page-card-signature"
|
||||
|
||||
assert pgy.go_next_page(Page()) is False
|
||||
|
||||
|
||||
def test_pgy_clicked_pagination_without_card_change_uses_source_end_semantics():
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
if "pgy-pagination-state" in script:
|
||||
return {
|
||||
"nextFound": True,
|
||||
"nextDisabled": False,
|
||||
"activePage": 2,
|
||||
"lastPage": 7,
|
||||
"numericPageCount": 7,
|
||||
}
|
||||
if "pgy-pagination-click" in script:
|
||||
return {"clicked": True, "text": "next"}
|
||||
return "same-card-signature"
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
assert pgy.go_next_page(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
|
||||
pages = [
|
||||
[{"title": "无关视频", "play_count": "10"}],
|
||||
@@ -432,6 +686,125 @@ def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
|
||||
assert calls["next"] == 1
|
||||
|
||||
|
||||
def test_xingtu_video_search_uses_visible_input_and_source_keyboard_events():
|
||||
events = []
|
||||
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def wait_for(self, **kwargs):
|
||||
events.append(("wait", kwargs))
|
||||
|
||||
def click(self):
|
||||
events.append(("click",))
|
||||
|
||||
def press(self, key):
|
||||
events.append(("locator_press", key))
|
||||
|
||||
def fill(self, _value):
|
||||
pytest.fail("Xingtu search must preserve source key-by-key input")
|
||||
|
||||
class Keyboard:
|
||||
def type(self, value, **kwargs):
|
||||
events.append(("type", value, kwargs))
|
||||
|
||||
def press(self, key):
|
||||
events.append(("keyboard_press", key))
|
||||
|
||||
class Page:
|
||||
keyboard = Keyboard()
|
||||
|
||||
def __init__(self):
|
||||
self.selector = None
|
||||
self.input = Locator()
|
||||
|
||||
def locator(self, selector):
|
||||
self.selector = selector
|
||||
return self.input
|
||||
|
||||
page = Page()
|
||||
|
||||
assert xingtu.submit_video_search(page, "目标标题") is True
|
||||
assert ":visible" in page.selector
|
||||
assert events == [
|
||||
("wait", {"state": "visible", "timeout": 5000}),
|
||||
("click",),
|
||||
("locator_press", "Control+A"),
|
||||
("locator_press", "Backspace"),
|
||||
("type", "目标标题", {"delay": 20}),
|
||||
("keyboard_press", "Enter"),
|
||||
]
|
||||
|
||||
|
||||
def test_xingtu_only_confirmed_creator_miss_is_blocked():
|
||||
no_creator = xingtu._missing_detail_outcome(saw_search_result=False)
|
||||
no_unique_video = xingtu._unmatched_video_outcome(
|
||||
candidates=["平台上的其他视频"],
|
||||
page_limit_hit=False,
|
||||
)
|
||||
completed = cc.finalize_summary({
|
||||
"total": 2,
|
||||
"results": [
|
||||
{"record_id": "creator", **no_creator},
|
||||
{"record_id": "video", **no_unique_video},
|
||||
],
|
||||
})
|
||||
|
||||
assert completed["blocked_input"] == 1
|
||||
assert completed["retryable_failures"] == 1
|
||||
assert completed["unresolved"] == 1
|
||||
assert completed["complete"] is False
|
||||
assert no_unique_video["status"] == "retryable_failure"
|
||||
assert xingtu._missing_detail_outcome(
|
||||
saw_search_result=True
|
||||
)["status"] == "retryable_failure"
|
||||
assert xingtu._unmatched_video_outcome(
|
||||
candidates=[], page_limit_hit=False
|
||||
)["reason"] == "empty_detail_page"
|
||||
assert xingtu._unmatched_video_outcome(
|
||||
candidates=["候选"], page_limit_hit=True
|
||||
)["status"] == "retryable_failure"
|
||||
|
||||
|
||||
def test_xingtu_missing_creator_result_has_distinct_business_exception():
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
class Page:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, _script, argument=None):
|
||||
if argument is not None:
|
||||
return {"found": False}
|
||||
return False
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
with pytest.raises(xingtu.CreatorNotFound, match="未找到精确达人"):
|
||||
xingtu.open_creator_detail(Page(), "不存在的达人", "missing-id")
|
||||
|
||||
|
||||
def test_xingtu_clicked_pagination_without_card_change_is_retryable_timeout():
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
if "const all" in script:
|
||||
return {"clicked": True, "picked": "pagination"}
|
||||
return "same-card-signature"
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
with pytest.raises(xingtu.BrowserTimeoutError, match="did not change"):
|
||||
xingtu.go_next_page(Page())
|
||||
|
||||
|
||||
def test_xingtu_normalizes_show_items_api_video():
|
||||
card = xingtu.normalize_xingtu_api_item({
|
||||
"item_id": 7654321,
|
||||
@@ -456,6 +829,9 @@ def test_xingtu_normalizes_show_items_api_video():
|
||||
|
||||
|
||||
def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
class Response:
|
||||
url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
|
||||
|
||||
@@ -477,9 +853,14 @@ def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
|
||||
|
||||
page = Page()
|
||||
cards = xingtu.setup_show_items_capture(page)
|
||||
page.callback(Response())
|
||||
response = Response()
|
||||
response_ref = weakref.ref(response)
|
||||
page.callback(response)
|
||||
del response
|
||||
gc.collect()
|
||||
|
||||
assert [card["note_id"] for card in cards] == ["1", "2"]
|
||||
assert response_ref() is None
|
||||
|
||||
|
||||
def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
|
||||
@@ -542,9 +923,221 @@ def test_browser_session_loss_detection_is_specific():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
assert pgy.is_browser_session_lost(closed)
|
||||
assert xingtu.is_browser_session_lost(closed)
|
||||
assert pgy.is_browser_session_lost(RuntimeError("Page crashed"))
|
||||
assert xingtu.is_browser_session_lost(RuntimeError("Page crashed"))
|
||||
assert pgy.is_browser_session_lost(
|
||||
RuntimeError("Execution context was destroyed, most likely because of a navigation.")
|
||||
)
|
||||
assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
|
||||
|
||||
|
||||
def test_transient_browser_error_excludes_business_timeout():
|
||||
assert is_transient_browser_error(
|
||||
pgy.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded")
|
||||
)
|
||||
assert is_transient_browser_error(
|
||||
RuntimeError("Page.goto: net::ERR_CONNECTION_RESET")
|
||||
)
|
||||
assert not is_transient_browser_error(
|
||||
TimeoutError("Login timed out after 300 seconds.")
|
||||
)
|
||||
|
||||
|
||||
def _stub_creator_session_failure(monkeypatch, tmp_path, module, error_factory):
|
||||
style = {"index": 1, "name": "style"}
|
||||
task = {
|
||||
"record_id": "record-1",
|
||||
"creator_name": "creator",
|
||||
"creator_id": None,
|
||||
"target_title": "target",
|
||||
"style_context": {"index": 1},
|
||||
}
|
||||
summary = {
|
||||
"index": 1,
|
||||
"name": "style",
|
||||
"total": 1,
|
||||
"filled": 0,
|
||||
"results": [],
|
||||
}
|
||||
group = {
|
||||
"creator_name": "creator",
|
||||
"creator_id": None,
|
||||
"tasks": [task],
|
||||
}
|
||||
sessions = []
|
||||
|
||||
class Session:
|
||||
def __init__(self, **_kwargs):
|
||||
sessions.append(self)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def fail_page_action(*_args, **_kwargs):
|
||||
raise error_factory()
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"collect_tasks_across_styles",
|
||||
lambda *_args, **_kwargs: ([task], {1: summary}),
|
||||
)
|
||||
monkeypatch.setattr(module, "group_tasks_by_creator", lambda _tasks: [group])
|
||||
monkeypatch.setattr(module, "chunk_creator_groups", lambda groups, _size: [groups])
|
||||
monkeypatch.setattr(module, "DynamicSession", Session)
|
||||
monkeypatch.setattr(module, "run_serialized_page_action", fail_page_action)
|
||||
monkeypatch.setattr(module, "_force_cleanup_session", lambda _session: None)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", tmp_path / module.__name__.split(".")[-1])
|
||||
return style, sessions
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [pgy, xingtu])
|
||||
def test_creator_collectors_retry_browser_timeout_in_three_fresh_sessions(
|
||||
monkeypatch, tmp_path, module
|
||||
):
|
||||
style, sessions = _stub_creator_session_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
lambda: module.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded"),
|
||||
)
|
||||
|
||||
summaries = module.scrape_styles([style], 1, True, None, True)
|
||||
|
||||
assert len(sessions) == 3
|
||||
assert summaries[0]["complete"] is False
|
||||
assert summaries[0]["retryable_failures"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [pgy, xingtu])
|
||||
def test_creator_collectors_do_not_retry_acceptance_terminal(
|
||||
monkeypatch, tmp_path, module
|
||||
):
|
||||
style, sessions = _stub_creator_session_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
lambda: module.AcceptanceCookieSkip("cookie unavailable in acceptance"),
|
||||
)
|
||||
|
||||
with pytest.raises(module.AcceptanceCookieSkip):
|
||||
module.scrape_styles([style], 1, True, None, True)
|
||||
|
||||
assert len(sessions) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [pgy, xingtu])
|
||||
def test_creator_collectors_do_not_retry_login_timeout_terminal(
|
||||
monkeypatch, tmp_path, module
|
||||
):
|
||||
style, sessions = _stub_creator_session_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
lambda: TimeoutError("Login timed out after 300 seconds."),
|
||||
)
|
||||
|
||||
with pytest.raises(TimeoutError, match="Login timed out"):
|
||||
module.scrape_styles([style], 1, True, None, True)
|
||||
|
||||
assert len(sessions) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_type", [xingtu.SearchQuotaExhausted, xingtu.CaptchaBlocked])
|
||||
def test_xingtu_does_not_retry_explicit_business_terminal(
|
||||
monkeypatch, tmp_path, error_type
|
||||
):
|
||||
style, sessions = _stub_creator_session_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
xingtu,
|
||||
lambda: error_type("explicit platform terminal"),
|
||||
)
|
||||
|
||||
with pytest.raises(error_type):
|
||||
xingtu.scrape_styles([style], 1, True, None, True)
|
||||
|
||||
assert len(sessions) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [pgy, xingtu])
|
||||
def test_creator_collectors_with_no_tasks_do_not_open_browser(
|
||||
monkeypatch, module
|
||||
):
|
||||
style = {"index": 1, "name": "style"}
|
||||
summary = {"index": 1, "name": "style", "total": 0, "results": []}
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"collect_tasks_across_styles",
|
||||
lambda *_args, **_kwargs: ([], {1: summary}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"DynamicSession",
|
||||
lambda **_kwargs: pytest.fail("no browser session expected"),
|
||||
)
|
||||
|
||||
assert module.scrape_styles([style], 1, True, None, True) == [summary]
|
||||
|
||||
|
||||
def test_long_page_action_does_not_retain_navigation_response():
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
response_ref = None
|
||||
|
||||
class Response:
|
||||
url = "https://example.test/final"
|
||||
status = 200
|
||||
headers = {"content-type": "text/html"}
|
||||
|
||||
class Page:
|
||||
closed = False
|
||||
|
||||
def set_default_timeout(self, _timeout):
|
||||
pass
|
||||
|
||||
def set_default_navigation_timeout(self, _timeout):
|
||||
pass
|
||||
|
||||
def goto(self, *_args, **_kwargs):
|
||||
nonlocal response_ref
|
||||
response = Response()
|
||||
response_ref = weakref.ref(response)
|
||||
return response
|
||||
|
||||
def is_closed(self):
|
||||
return self.closed
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
page = Page()
|
||||
|
||||
class Context:
|
||||
def new_page(self):
|
||||
return page
|
||||
|
||||
class Session:
|
||||
context = Context()
|
||||
|
||||
def action(_page):
|
||||
gc.collect()
|
||||
assert response_ref is not None
|
||||
assert response_ref() is None
|
||||
|
||||
navigation = run_serialized_page_action(Session(), "https://example.test", action)
|
||||
|
||||
assert navigation == {
|
||||
"url": "https://example.test/final",
|
||||
"status": 200,
|
||||
"headers": {"content-type": "text/html"},
|
||||
}
|
||||
assert page.closed is True
|
||||
|
||||
|
||||
def test_xingtu_nested_session_loss_is_reraised():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
@@ -694,6 +1287,19 @@ def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
|
||||
assert [row["record_id"] for row in summary["details"]] == ["published"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("2026-07-20T00:00:00.000+08:00", bili.date(2026, 7, 20)),
|
||||
("2026-07-20T00:00:00+08:00", bili.date(2026, 7, 20)),
|
||||
("2026-07-19T16:00:00.000Z", bili.date(2026, 7, 19)),
|
||||
("2026-07-20", bili.date(2026, 7, 20)),
|
||||
],
|
||||
)
|
||||
def test_bilibili_parse_pub_date_accepts_feishu_iso_values(raw, expected):
|
||||
assert bili.parse_pub_date(raw) == expected
|
||||
|
||||
|
||||
def test_bilibili_partial_retry_preserves_old_success():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 2,
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters.scrapling import BrowserTimeoutError
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
batch_rescrape_douyin as douyin_batch,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
batch_rescrape_xiaohongshu as xhs_batch,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools.comment_batch_checkpoint import (
|
||||
CommentBatchCheckpoint,
|
||||
is_browser_process_lost,
|
||||
)
|
||||
|
||||
|
||||
def test_page_crash_requires_a_fresh_browser():
|
||||
assert is_browser_process_lost(
|
||||
RuntimeError("Page.wait_for_timeout: Page crashed")
|
||||
)
|
||||
assert not is_browser_process_lost(RuntimeError("ordinary parse failure"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch", [xhs_batch, douyin_batch])
|
||||
def test_non_acceptance_relogin_runs_platform_script(monkeypatch, batch):
|
||||
class Policy:
|
||||
enabled = False
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(batch, "current_acceptance_policy", lambda: Policy())
|
||||
monkeypatch.setattr(
|
||||
batch.subprocess,
|
||||
"call",
|
||||
lambda command, **kwargs: calls.append((command, kwargs)) or 0,
|
||||
)
|
||||
|
||||
assert batch.do_relogin() is True
|
||||
assert calls[0][0][1] == str(batch.RELOGIN_SCRIPT)
|
||||
|
||||
|
||||
def test_comment_checkpoint_matches_both_note_id_and_url(tmp_path):
|
||||
path = tmp_path / "checkpoint.json"
|
||||
checkpoint = CommentBatchCheckpoint(
|
||||
"douyin", business_date="2026-08-09", path=path
|
||||
)
|
||||
checkpoint.mark_completed(7, "https://www.douyin.com/video/7")
|
||||
|
||||
restored = CommentBatchCheckpoint(
|
||||
"douyin", business_date="2026-08-09", path=path
|
||||
)
|
||||
assert restored.contains(7, "https://www.douyin.com/video/7") is True
|
||||
assert restored.contains(7, "https://www.douyin.com/video/changed") is False
|
||||
|
||||
restored.clear()
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
def test_xhs_page_crash_retries_same_note_with_new_scraper_call(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def scrape(url, *_args):
|
||||
calls.append(url)
|
||||
if len(calls) == 1:
|
||||
raise RuntimeError("Page.wait_for_timeout: Page crashed")
|
||||
return [], {"stats": {}}
|
||||
|
||||
monkeypatch.setattr(xhs_batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(xhs_batch.time, "sleep", lambda _seconds: None)
|
||||
|
||||
comments, result, error = xhs_batch.scrape_with_fresh_browser("xhs-url")
|
||||
|
||||
assert comments == []
|
||||
assert result == {"stats": {}}
|
||||
assert error is None
|
||||
assert calls == ["xhs-url", "xhs-url"]
|
||||
|
||||
|
||||
def test_douyin_page_crash_retries_same_note_with_new_scraper_call(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def scrape(url, *_args):
|
||||
calls.append(url)
|
||||
if len(calls) == 1:
|
||||
raise RuntimeError("Page.wait_for_timeout: Page crashed")
|
||||
return [], {"stats": {}}
|
||||
|
||||
monkeypatch.setattr(douyin_batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(douyin_batch.time, "sleep", lambda _seconds: None)
|
||||
|
||||
comments, result, error = douyin_batch.scrape_with_fresh_browser("dy-url")
|
||||
|
||||
assert comments == []
|
||||
assert result == {"stats": {}}
|
||||
assert error is None
|
||||
assert calls == ["dy-url", "dy-url"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch", "url"),
|
||||
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
|
||||
)
|
||||
def test_transient_timeout_gets_three_fresh_browser_attempts(monkeypatch, batch, url):
|
||||
calls = []
|
||||
|
||||
def scrape(scrape_url, *_args):
|
||||
calls.append(scrape_url)
|
||||
if len(calls) < 3:
|
||||
raise BrowserTimeoutError("Page.goto: Timeout 90000ms exceeded")
|
||||
return [], {"stats": {}}
|
||||
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
|
||||
|
||||
comments, result, error = batch.scrape_with_fresh_browser(url)
|
||||
|
||||
assert comments == []
|
||||
assert result == {"stats": {}}
|
||||
assert error is None
|
||||
assert calls == [url, url, url]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch", "url"),
|
||||
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
|
||||
)
|
||||
def test_transient_network_failure_exhausts_three_attempts_honestly(
|
||||
monkeypatch, batch, url
|
||||
):
|
||||
calls = []
|
||||
|
||||
def scrape(scrape_url, *_args):
|
||||
calls.append(scrape_url)
|
||||
raise RuntimeError("Page.goto: net::ERR_CONNECTION_RESET")
|
||||
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
|
||||
|
||||
comments, result, error = batch.scrape_with_fresh_browser(url)
|
||||
|
||||
assert comments is None
|
||||
assert result is None
|
||||
assert error is not None
|
||||
assert "ERR_CONNECTION_RESET" in str(error)
|
||||
assert calls == [url, url, url]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch", "url"),
|
||||
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
|
||||
)
|
||||
def test_terminal_parse_failure_is_not_retried(monkeypatch, batch, url):
|
||||
calls = []
|
||||
|
||||
def scrape(scrape_url, *_args):
|
||||
calls.append(scrape_url)
|
||||
raise RuntimeError("ordinary terminal parse failure")
|
||||
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
|
||||
comments, result, error = batch.scrape_with_fresh_browser(url)
|
||||
|
||||
assert comments is None
|
||||
assert result is None
|
||||
assert error is not None
|
||||
assert calls == [url]
|
||||
|
||||
|
||||
def test_xhs_failed_note_keeps_success_checkpoint_and_fails_batch(monkeypatch):
|
||||
class Checkpoint:
|
||||
completed = []
|
||||
cleared = False
|
||||
|
||||
def __init__(self, _platform):
|
||||
pass
|
||||
|
||||
def contains(self, _note_id, _url):
|
||||
return False
|
||||
|
||||
def mark_completed(self, note_id, url):
|
||||
self.completed.append((note_id, url))
|
||||
|
||||
def clear(self):
|
||||
self.cleared = True
|
||||
|
||||
rows = [
|
||||
{"id": 1, "url": "xhs-1", "title": "one"},
|
||||
{"id": 2, "url": "xhs-2", "title": "two"},
|
||||
]
|
||||
|
||||
def scrape(url, *_args):
|
||||
if url == "xhs-2":
|
||||
raise RuntimeError("ordinary terminal failure")
|
||||
return ([{"level": "comment"}], {
|
||||
"title": "one",
|
||||
"stats": {"logged_in": True, "note_metrics": {}},
|
||||
})
|
||||
|
||||
monkeypatch.setattr(xhs_batch, "CommentBatchCheckpoint", Checkpoint)
|
||||
monkeypatch.setattr(xhs_batch, "collect_urls", lambda: rows)
|
||||
monkeypatch.setattr(xhs_batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(xhs_batch.scraper, "write_outputs", lambda *_args: (None, None))
|
||||
monkeypatch.setattr(xhs_batch, "upsert_metrics", lambda *_args: None)
|
||||
monkeypatch.setattr(xhs_batch.db, "replace_comments", lambda *_args, **_kwargs: None)
|
||||
|
||||
assert xhs_batch.main() == 1
|
||||
assert Checkpoint.completed == [(1, "xhs-1")]
|
||||
assert Checkpoint.cleared is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch", "platform"),
|
||||
[(xhs_batch, "xhs"), (douyin_batch, "dy")],
|
||||
)
|
||||
def test_incomplete_comments_preserve_db_but_remain_resume_eligible(
|
||||
monkeypatch, batch, platform
|
||||
):
|
||||
class Checkpoint:
|
||||
def __init__(self):
|
||||
self.completed = []
|
||||
self.cleared = False
|
||||
|
||||
def contains(self, _note_id, _url):
|
||||
return False
|
||||
|
||||
def mark_completed(self, note_id, url):
|
||||
self.completed.append((note_id, url))
|
||||
|
||||
def clear(self):
|
||||
self.cleared = True
|
||||
|
||||
checkpoint = Checkpoint()
|
||||
rows = [
|
||||
{
|
||||
"id": 1,
|
||||
"url": f"{platform}-partial",
|
||||
"title": "partial",
|
||||
"comment_count": 5,
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"url": f"{platform}-failed",
|
||||
"title": "failed",
|
||||
"comment_count": 5,
|
||||
},
|
||||
]
|
||||
|
||||
def scrape(url, *_args):
|
||||
if url.endswith("-failed"):
|
||||
raise RuntimeError("ordinary terminal failure")
|
||||
comments = [{"level": "comment"}]
|
||||
stats = {
|
||||
"logged_in": False,
|
||||
"api_total": 5,
|
||||
"note_metrics": {"comment_count": 5},
|
||||
}
|
||||
if batch is xhs_batch:
|
||||
return comments, {"title": "partial", "stats": stats}
|
||||
return comments, {"stats": stats}
|
||||
|
||||
monkeypatch.setattr(batch, "CommentBatchCheckpoint", lambda _platform: checkpoint)
|
||||
monkeypatch.setattr(batch, "collect_urls", lambda: rows)
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(batch.scraper, "write_outputs", lambda *_args: (None, None))
|
||||
monkeypatch.setattr(batch, "upsert_metrics", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
batch.db,
|
||||
"replace_comments",
|
||||
lambda *_args, **_kwargs: pytest.fail("incomplete comments must not replace DB"),
|
||||
)
|
||||
|
||||
assert batch.main() == 1
|
||||
assert checkpoint.completed == []
|
||||
assert checkpoint.cleared is False
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.modules.content_marketing import chanmama_scraper, weekly_summary_all
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
analyze_comments,
|
||||
@@ -71,17 +71,13 @@ def test_daily_card_sends_only_to_wang_yunlong(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"message_id": "om_test"}),
|
||||
stderr="",
|
||||
)
|
||||
def fake_send(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
monkeypatch.setattr(daily_report_card.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(daily_report_card, "send_lark_bot_message", fake_send)
|
||||
results = daily_report_card.send_card_to_recipients(
|
||||
{"schema": "2.0", "body": {"elements": []}},
|
||||
("ou_other_a", "ou_other_b"),
|
||||
@@ -89,9 +85,9 @@ def test_daily_card_sends_only_to_wang_yunlong(
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(commands) == 1
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["profile"] == "hermes-analyzer"
|
||||
|
||||
|
||||
def test_comment_summary_sends_only_to_wang_yunlong(
|
||||
@@ -99,17 +95,17 @@ def test_comment_summary_sends_only_to_wang_yunlong(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||||
def fake_send(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
monkeypatch.setattr(analyze_comments.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(analyze_comments, "send_lark_bot_message", fake_send)
|
||||
analyze_comments.send_feishu_summary("summary", "ou_other")
|
||||
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["profile"] == "hermes-analyzer"
|
||||
|
||||
|
||||
def test_note_report_sends_only_to_wang_yunlong(
|
||||
@@ -117,19 +113,18 @@ def test_note_report_sends_only_to_wang_yunlong(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
monkeypatch.setattr(analyze_note.sys, "platform", "linux")
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
analyze_note,
|
||||
"build_card_payload",
|
||||
lambda *args, **kwargs: {"schema": "2.0"},
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||||
def fake_send(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
monkeypatch.setattr(analyze_note.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(analyze_note, "send_lark_bot_message", fake_send)
|
||||
analyze_note.send_feishu_report(
|
||||
{},
|
||||
{},
|
||||
@@ -141,8 +136,8 @@ def test_note_report_sends_only_to_wang_yunlong(
|
||||
tmp_path / "report.md",
|
||||
)
|
||||
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["profile"] == "hermes-analyzer"
|
||||
|
||||
|
||||
def test_relogin_notifications_resolve_only_to_wang_yunlong(
|
||||
@@ -153,5 +148,5 @@ def test_relogin_notifications_resolve_only_to_wang_yunlong(
|
||||
|
||||
assert (
|
||||
friday_relogin_parallel.recipient_for_platform("pgy", "ou_other")
|
||||
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
)
|
||||
|
||||
@@ -0,0 +1,596 @@
|
||||
import sys
|
||||
from datetime import date
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import monthly_summary_all, weekly_summary_all
|
||||
from gyxx_flow.modules.content_marketing.data.tools import db
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
generate_creator_report as report,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.feishu_doc_import import (
|
||||
FeishuDocImportTimeout,
|
||||
)
|
||||
|
||||
|
||||
def _prepare_main(monkeypatch, tmp_path, *, doc_url, saved):
|
||||
monkeypatch.setattr(report, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"_compute_last_month",
|
||||
lambda: (date(2026, 7, 1), date(2026, 7, 31)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"generate_report",
|
||||
lambda style_filter=None: ("report", [], {}, {}),
|
||||
)
|
||||
monkeypatch.setattr(report, "load_existing_creator_report", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
report, "create_feishu_doc", lambda *args, **_kwargs: doc_url
|
||||
)
|
||||
monkeypatch.setattr(report, "save_to_db", lambda *args: saved)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_creator_report.py"])
|
||||
|
||||
|
||||
def test_main_requires_both_doc_and_database_effects(monkeypatch, tmp_path):
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url="https://example/doc", saved=True)
|
||||
assert report.main() == 0
|
||||
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=True)
|
||||
assert report.main() == 1
|
||||
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url="https://example/doc", saved=False)
|
||||
assert report.main() == 1
|
||||
|
||||
|
||||
def test_main_dry_run_does_not_require_external_effects(monkeypatch, tmp_path):
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=False)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_creator_report.py", "--dry-run"])
|
||||
assert report.main() == 0
|
||||
|
||||
|
||||
def _import_timeout(tmp_path):
|
||||
return FeishuDocImportTimeout(
|
||||
"known import ticket did not become ready",
|
||||
audit_path=tmp_path / "ticket-audit.json",
|
||||
)
|
||||
|
||||
|
||||
def test_creator_report_timeout_is_persisted_as_ambiguous(monkeypatch, tmp_path):
|
||||
captured = {}
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=True)
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"load_existing_creator_report",
|
||||
lambda *_args: {"doc_url": "", "doc_title": "", "status": "doc_ambiguous"},
|
||||
)
|
||||
|
||||
def raise_timeout(*_args, **kwargs):
|
||||
captured["resume_only"] = kwargs.get("resume_only")
|
||||
raise _import_timeout(tmp_path)
|
||||
|
||||
def capture_save(*args):
|
||||
captured["doc_url"] = args[-2]
|
||||
captured["status"] = args[-1]
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(report, "create_feishu_doc", raise_timeout)
|
||||
monkeypatch.setattr(report, "save_to_db", capture_save)
|
||||
|
||||
assert report.main() == 1
|
||||
assert captured == {
|
||||
"doc_url": None,
|
||||
"resume_only": True,
|
||||
"status": "doc_ambiguous",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "processor_name", "creator_name"),
|
||||
(
|
||||
(weekly_summary_all, "process_one_style", "create_feishu_doc"),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"process_one_style_monthly",
|
||||
"create_feishu_doc_monthly",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_summary_timeout_is_persisted_as_ambiguous(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
processor_name,
|
||||
creator_name,
|
||||
):
|
||||
saved = {}
|
||||
creator_call = {}
|
||||
notes = [{"id": 1, "view_count": 10, "comment_count": 1}]
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(module, "REPORTS_DIR", tmp_path)
|
||||
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
|
||||
if module is weekly_summary_all:
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"load_existing_summary_delivery",
|
||||
lambda _style: {
|
||||
"doc_url": "",
|
||||
"doc_title": "",
|
||||
"status": "doc_ambiguous",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(module, "get_last_week_notes", lambda _style: notes)
|
||||
monkeypatch.setattr(module, "generate_summary", lambda *_args: "summary")
|
||||
else:
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"load_existing_monthly_summary_delivery",
|
||||
lambda _style: {
|
||||
"doc_url": "",
|
||||
"doc_title": "",
|
||||
"status": "doc_ambiguous",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(module, "get_last_month_notes", lambda _style: notes)
|
||||
monkeypatch.setattr(module, "generate_monthly_summary", lambda *_args: "summary")
|
||||
monkeypatch.setattr(module, "run_single_note_analysis", lambda _note_id: "report")
|
||||
|
||||
def raise_timeout(*_args, **kwargs):
|
||||
creator_call["resume_only"] = kwargs.get("resume_only")
|
||||
raise _import_timeout(tmp_path)
|
||||
|
||||
def capture_save(*args):
|
||||
saved["doc_url"] = args[-3]
|
||||
saved["doc_title"] = args[-2]
|
||||
saved["status"] = args[-1]
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(module, creator_name, raise_timeout)
|
||||
save_name = "save_to_db" if module is weekly_summary_all else "save_to_db_monthly"
|
||||
monkeypatch.setattr(module, save_name, capture_save)
|
||||
|
||||
processor = getattr(module, processor_name)
|
||||
arguments = ("宙斯", ("base", "table", "url"))
|
||||
if module is weekly_summary_all:
|
||||
arguments += (False,)
|
||||
result = processor(*arguments)
|
||||
|
||||
assert result["status"] == "doc_ambiguous"
|
||||
assert saved == {
|
||||
"doc_url": None,
|
||||
"doc_title": None,
|
||||
"status": "doc_ambiguous",
|
||||
}
|
||||
assert creator_call == {"resume_only": True}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "processor_name", "creator_name"),
|
||||
(
|
||||
(weekly_summary_all, "process_one_style", "create_feishu_doc"),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"process_one_style_monthly",
|
||||
"create_feishu_doc_monthly",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_summary_pg_failure_keeps_real_document_receipt_and_fails_result(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
processor_name,
|
||||
creator_name,
|
||||
):
|
||||
doc_url = "https://example.feishu.cn/docx/real-document"
|
||||
notes = [{"id": 1, "view_count": 10, "comment_count": 1}]
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(module, "REPORTS_DIR", tmp_path)
|
||||
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
|
||||
if module is weekly_summary_all:
|
||||
monkeypatch.setattr(module, "load_existing_summary_delivery", lambda _style: None)
|
||||
monkeypatch.setattr(module, "get_last_week_notes", lambda _style: notes)
|
||||
monkeypatch.setattr(module, "generate_summary", lambda *_args: "summary")
|
||||
else:
|
||||
monkeypatch.setattr(
|
||||
module, "load_existing_monthly_summary_delivery", lambda _style: None
|
||||
)
|
||||
monkeypatch.setattr(module, "get_last_month_notes", lambda _style: notes)
|
||||
monkeypatch.setattr(module, "generate_monthly_summary", lambda *_args: "summary")
|
||||
monkeypatch.setattr(module, "run_single_note_analysis", lambda _note_id: "report")
|
||||
monkeypatch.setattr(
|
||||
module, creator_name, lambda *_args, **_kwargs: (doc_url, None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module, "write_to_target_table", lambda *_args, **_kwargs: True
|
||||
)
|
||||
save_name = "save_to_db" if module is weekly_summary_all else "save_to_db_monthly"
|
||||
monkeypatch.setattr(module, save_name, lambda *_args: False)
|
||||
|
||||
processor = getattr(module, processor_name)
|
||||
arguments = ("宙斯", ("base", "table", "url"))
|
||||
if module is weekly_summary_all:
|
||||
arguments += (False,)
|
||||
result = processor(*arguments)
|
||||
|
||||
assert result["status"] == "db_failed"
|
||||
assert result["doc_url"] == doc_url
|
||||
assert result["target_table_written"] is True
|
||||
assert not weekly_summary_all.summary_results_succeeded([result])
|
||||
|
||||
|
||||
def test_creator_existing_period_document_is_never_imported_again(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=False)
|
||||
existing_url = "https://example.feishu.cn/docx/existing-creator-report"
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"load_existing_creator_report",
|
||||
lambda *_args: {
|
||||
"doc_url": existing_url,
|
||||
"doc_title": "existing",
|
||||
"status": "doc_ambiguous",
|
||||
},
|
||||
)
|
||||
|
||||
def fail_import(*_args, **_kwargs):
|
||||
raise AssertionError("an existing period URL must not be imported again")
|
||||
|
||||
monkeypatch.setattr(report, "create_feishu_doc", fail_import)
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"save_to_db",
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
AssertionError("completed period must not be overwritten")
|
||||
),
|
||||
)
|
||||
|
||||
assert report.main() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "processor_name", "loader_name", "creator_name"),
|
||||
(
|
||||
(
|
||||
weekly_summary_all,
|
||||
"process_one_style",
|
||||
"load_existing_summary_delivery",
|
||||
"create_feishu_doc",
|
||||
),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"process_one_style_monthly",
|
||||
"load_existing_monthly_summary_delivery",
|
||||
"create_feishu_doc_monthly",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_completed_summary_period_skips_all_external_work(
|
||||
monkeypatch,
|
||||
module,
|
||||
processor_name,
|
||||
loader_name,
|
||||
creator_name,
|
||||
):
|
||||
existing_url = "https://example.feishu.cn/docx/existing-summary"
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
loader_name,
|
||||
lambda _style: {
|
||||
"doc_url": existing_url,
|
||||
"doc_title": "existing",
|
||||
"status": "ok",
|
||||
},
|
||||
)
|
||||
|
||||
def fail_work(*_args, **_kwargs):
|
||||
raise AssertionError("completed summary period must be skipped")
|
||||
|
||||
monkeypatch.setattr(module, "find_target_fields", fail_work)
|
||||
monkeypatch.setattr(module, creator_name, fail_work)
|
||||
processor = getattr(module, processor_name)
|
||||
arguments = ("宙斯", ("base", "table", "url"))
|
||||
if module is weekly_summary_all:
|
||||
arguments += (False,)
|
||||
|
||||
result = processor(*arguments)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["doc_url"] == existing_url
|
||||
assert result["already_complete"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "processor_name", "loader_name", "creator_name", "updater_name"),
|
||||
(
|
||||
(
|
||||
weekly_summary_all,
|
||||
"process_one_style",
|
||||
"load_existing_summary_delivery",
|
||||
"create_feishu_doc",
|
||||
"update_existing_summary_delivery_status",
|
||||
),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"process_one_style_monthly",
|
||||
"load_existing_monthly_summary_delivery",
|
||||
"create_feishu_doc_monthly",
|
||||
"update_existing_monthly_summary_delivery_status",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_failed_base_delivery_reuses_existing_document_without_import(
|
||||
monkeypatch,
|
||||
module,
|
||||
processor_name,
|
||||
loader_name,
|
||||
creator_name,
|
||||
updater_name,
|
||||
):
|
||||
existing_url = "https://example.feishu.cn/docx/existing-summary"
|
||||
write_call = {}
|
||||
status_call = {}
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
loader_name,
|
||||
lambda _style: {
|
||||
"doc_url": existing_url,
|
||||
"doc_title": "existing",
|
||||
"status": "feishu_write_failed",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
|
||||
|
||||
def capture_write(*args, **kwargs):
|
||||
write_call["args"] = args
|
||||
write_call["kwargs"] = kwargs
|
||||
return True
|
||||
|
||||
def capture_status(style, status):
|
||||
status_call.update({"style": style, "status": status})
|
||||
return True
|
||||
|
||||
def fail_import(*_args, **_kwargs):
|
||||
raise AssertionError("an existing document must not be imported again")
|
||||
|
||||
monkeypatch.setattr(module, "write_to_target_table", capture_write)
|
||||
monkeypatch.setattr(module, updater_name, capture_status)
|
||||
monkeypatch.setattr(module, creator_name, fail_import)
|
||||
processor = getattr(module, processor_name)
|
||||
arguments = ("宙斯", ("base", "table", "url"))
|
||||
if module is weekly_summary_all:
|
||||
arguments += (False,)
|
||||
|
||||
result = processor(*arguments)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["doc_url"] == existing_url
|
||||
assert result["reused_document"] is True
|
||||
assert write_call["args"][5] == existing_url
|
||||
assert status_call == {"style": "宙斯", "status": "ok"}
|
||||
if module is monthly_summary_all:
|
||||
assert write_call["kwargs"] == {
|
||||
"time_label": monthly_summary_all.TIME_LABEL,
|
||||
"report_label": "月度汇总",
|
||||
}
|
||||
|
||||
|
||||
def test_business_date_drives_default_report_periods(monkeypatch):
|
||||
for name in ("WEEK_SINCE", "WEEK_UNTIL", "MONTH_SINCE", "MONTH_UNTIL"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-09")
|
||||
|
||||
assert report._compute_last_month() == (date(2026, 7, 1), date(2026, 7, 31))
|
||||
assert weekly_summary_all._resolve_week_period() == (
|
||||
date(2026, 7, 27),
|
||||
date(2026, 8, 2),
|
||||
)
|
||||
assert monthly_summary_all._resolve_month_period() == (
|
||||
date(2026, 7, 1),
|
||||
date(2026, 7, 31),
|
||||
)
|
||||
|
||||
|
||||
def test_explicit_period_overrides_take_priority_over_business_date(monkeypatch):
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "not-a-date")
|
||||
monkeypatch.setenv("WEEK_SINCE", "2026-06-01")
|
||||
monkeypatch.setenv("WEEK_UNTIL", "2026-06-07")
|
||||
monkeypatch.setenv("MONTH_SINCE", "2026-05-01")
|
||||
monkeypatch.setenv("MONTH_UNTIL", "2026-05-31")
|
||||
|
||||
assert weekly_summary_all._resolve_week_period() == (
|
||||
date(2026, 6, 1),
|
||||
date(2026, 6, 7),
|
||||
)
|
||||
assert monthly_summary_all._resolve_month_period() == (
|
||||
date(2026, 5, 1),
|
||||
date(2026, 5, 31),
|
||||
)
|
||||
assert report._compute_last_month() == (date(2026, 5, 1), date(2026, 5, 31))
|
||||
|
||||
|
||||
def test_invalid_business_date_and_unpaired_overrides_fail_closed(monkeypatch):
|
||||
for name in ("WEEK_SINCE", "WEEK_UNTIL", "MONTH_SINCE", "MONTH_UNTIL"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026/08/09")
|
||||
|
||||
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
|
||||
report._compute_last_month()
|
||||
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
|
||||
weekly_summary_all._resolve_week_period()
|
||||
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
|
||||
monthly_summary_all._resolve_month_period()
|
||||
|
||||
monkeypatch.delenv("GYXX_BUSINESS_DATE")
|
||||
monkeypatch.setenv("WEEK_SINCE", "2026-06-01")
|
||||
with pytest.raises(ValueError, match="configured together"):
|
||||
weekly_summary_all._resolve_week_period()
|
||||
monkeypatch.delenv("WEEK_SINCE")
|
||||
monkeypatch.setenv("MONTH_SINCE", "2026-05-01")
|
||||
with pytest.raises(ValueError, match="configured together"):
|
||||
report._compute_last_month()
|
||||
with pytest.raises(ValueError, match="configured together"):
|
||||
monthly_summary_all._resolve_month_period()
|
||||
|
||||
|
||||
class _RecordingCursor:
|
||||
def __init__(self):
|
||||
self.statements = []
|
||||
|
||||
def execute(self, statement, params):
|
||||
self.statements.append((statement, params))
|
||||
|
||||
def fetchone(self):
|
||||
return (7,)
|
||||
|
||||
|
||||
class _RecordingConnection:
|
||||
def __init__(self):
|
||||
self.cursor_instance = _RecordingCursor()
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_instance
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "save_name", "table_name"),
|
||||
(
|
||||
(report, "save_to_db", "cmt_creator_report"),
|
||||
(weekly_summary_all, "save_to_db", "cmt_weekly_summary"),
|
||||
(monthly_summary_all, "save_to_db_monthly", "cmt_monthly_summary"),
|
||||
),
|
||||
)
|
||||
def test_empty_upsert_values_preserve_existing_document_identity(
|
||||
monkeypatch,
|
||||
module,
|
||||
save_name,
|
||||
table_name,
|
||||
):
|
||||
connection = _RecordingConnection()
|
||||
if module is report:
|
||||
monkeypatch.setattr(report, "get_db_config", lambda: {})
|
||||
monkeypatch.setattr(report.psycopg, "connect", lambda **_kwargs: connection)
|
||||
saved = report.save_to_db(
|
||||
"summary",
|
||||
[],
|
||||
{},
|
||||
{},
|
||||
"2026-07-01",
|
||||
"2026-07-31",
|
||||
None,
|
||||
"doc_ambiguous",
|
||||
)
|
||||
else:
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(db, "get_conn", lambda: connection)
|
||||
saved = getattr(module, save_name)(
|
||||
"宙斯",
|
||||
[],
|
||||
"summary",
|
||||
None,
|
||||
None,
|
||||
"doc_ambiguous",
|
||||
)
|
||||
|
||||
assert saved is True
|
||||
upsert = connection.cursor_instance.statements[-1][0]
|
||||
assert "COALESCE" in upsert
|
||||
assert "NULLIF(EXCLUDED.doc_url, '')" in upsert
|
||||
assert f"{table_name}.doc_url" in upsert
|
||||
assert "NULLIF(EXCLUDED.doc_title, '')" in upsert
|
||||
assert f"{table_name}.doc_title" in upsert
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"module",
|
||||
"function_name",
|
||||
"args",
|
||||
"expected_title",
|
||||
"source_filename",
|
||||
"operation_prefix",
|
||||
),
|
||||
(
|
||||
(
|
||||
report,
|
||||
"create_feishu_doc",
|
||||
("report", "2026-07-01", "2026-07-31"),
|
||||
"达人合作数据筛选与报价分析报告(2026-07-01~07-31)",
|
||||
"creator_report_doc.md",
|
||||
"content-creator-report-",
|
||||
),
|
||||
(
|
||||
weekly_summary_all,
|
||||
"create_feishu_doc",
|
||||
("宙斯", "summary"),
|
||||
"宙斯 周笔记汇总(2026-07-27~08-02)",
|
||||
"weekly_summary_doc.md",
|
||||
"content-weekly-summary-",
|
||||
),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"create_feishu_doc_monthly",
|
||||
("宙斯", "summary"),
|
||||
"宙斯 月度汇总(2026-07-01~07-31)",
|
||||
"monthly_summary_doc.md",
|
||||
"content-monthly-summary-",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_feishu_doc_import_preserves_title_body_and_root_target(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
function_name,
|
||||
args,
|
||||
expected_title,
|
||||
source_filename,
|
||||
operation_prefix,
|
||||
):
|
||||
captured = {}
|
||||
|
||||
def fake_import(markdown, **kwargs):
|
||||
captured["markdown"] = markdown
|
||||
captured.update(kwargs)
|
||||
return SimpleNamespace(url="https://example/doc")
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"PATHS",
|
||||
SimpleNamespace(tmp_root=tmp_path, evidence_root=tmp_path / "evidence"),
|
||||
)
|
||||
monkeypatch.setattr(module, "import_markdown_document", fake_import)
|
||||
if module is weekly_summary_all:
|
||||
monkeypatch.setattr(module, "TIME_LABEL", "2026-07-27~08-02")
|
||||
elif module is monthly_summary_all:
|
||||
monkeypatch.setattr(module, "TIME_LABEL", "2026-07-01~07-31")
|
||||
|
||||
getattr(module, function_name)(*args)
|
||||
|
||||
assert captured["markdown"] == args[0 if module is report else 1]
|
||||
assert captured["title"] == expected_title
|
||||
assert captured["source_filename"] == source_filename
|
||||
assert captured["working_directory"] == tmp_path
|
||||
assert captured["audit_directory"] == tmp_path / "evidence" / "feishu_doc_import"
|
||||
assert captured["identity"] == "user"
|
||||
assert captured.get("folder_token") is None
|
||||
assert captured["operation_id"].startswith(operation_prefix)
|
||||
assert len(captured["operation_id"].rsplit("-", 1)[1]) == 40
|
||||
assert captured["resume_only"] is False
|
||||
@@ -1,5 +1,11 @@
|
||||
import io
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing import daily_marketing_report as daily
|
||||
@@ -47,6 +53,41 @@ class DailyMarketingReportTests(unittest.TestCase):
|
||||
),
|
||||
)
|
||||
|
||||
def test_disabled_dynamic_route_keeps_report_but_skips_feishu_send(self):
|
||||
facts = ([], [], [], [], {})
|
||||
with tempfile.TemporaryDirectory() as directory, \
|
||||
patch.object(daily, "PATHS", SimpleNamespace(exports_root=Path(directory))), \
|
||||
patch.object(daily, "_latest_report_date", return_value=date(2026, 8, 7)), \
|
||||
patch.object(daily, "load_report_facts", return_value=facts), \
|
||||
patch.object(daily, "build_daily_prompt", return_value="prompt"), \
|
||||
patch.object(daily, "call_hermes_analyzer", return_value="report"), \
|
||||
patch.object(daily, "require_valid_hermes_report", return_value="report"), \
|
||||
patch.object(daily, "normalize_report", return_value="report"), \
|
||||
patch.object(daily, "generate_dashboard"), \
|
||||
patch.object(
|
||||
daily,
|
||||
"resolve_notification_route",
|
||||
return_value=SimpleNamespace(
|
||||
enabled=False,
|
||||
open_ids=(),
|
||||
app_profile="hermes-analyzer",
|
||||
),
|
||||
), \
|
||||
patch.object(daily, "send_daily_report_cards") as send, \
|
||||
patch.object(sys, "argv", ["daily_marketing_report.py", "--send"]):
|
||||
output = io.StringIO()
|
||||
with redirect_stdout(output):
|
||||
self.assertEqual(daily.main(), 0)
|
||||
|
||||
report_path = (
|
||||
Path(directory)
|
||||
/ "summary"
|
||||
/ "daily_marketing_report_2026-08-07.md"
|
||||
)
|
||||
self.assertTrue(report_path.is_file())
|
||||
self.assertIn("动态配置中禁用", output.getvalue())
|
||||
send.assert_not_called()
|
||||
|
||||
def test_hermes_transport_error_is_not_accepted_as_report(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "Hermes 分析失败"):
|
||||
daily.require_valid_hermes_report("API call failed after 3 retries: Connection error.")
|
||||
@@ -142,6 +183,25 @@ class DailyMarketingReportTests(unittest.TestCase):
|
||||
self.assertEqual(valid["engagement_count_num"], 2143)
|
||||
self.assertEqual(valid["publish_status"], "已记录发布时间")
|
||||
|
||||
def test_style_note_signal_keeps_complete_count_and_reports_metric_coverage(self):
|
||||
signals = daily._build_style_signals(
|
||||
["宙斯"],
|
||||
[],
|
||||
[
|
||||
{"style_name": "宙斯", "exposure_count": 1000},
|
||||
{"style_name": "宙斯", "exposure_count": None},
|
||||
],
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
)
|
||||
|
||||
self.assertEqual(signals["宙斯"]["note_count"], 2)
|
||||
self.assertEqual(signals["宙斯"]["recent_note_exposure"], 1000)
|
||||
self.assertEqual(signals["宙斯"]["metric_note_count"], 1)
|
||||
self.assertEqual(signals["宙斯"]["metric_missing_count"], 1)
|
||||
self.assertEqual(signals["宙斯"]["metric_coverage_rate"], 0.5)
|
||||
|
||||
def test_load_report_facts_tracks_all_styles_for_report_date(self):
|
||||
metrics = {
|
||||
"宙斯": {"light": "🟢"},
|
||||
@@ -156,6 +216,7 @@ class DailyMarketingReportTests(unittest.TestCase):
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_note_inventory_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
@@ -185,6 +246,7 @@ class DailyMarketingReportTests(unittest.TestCase):
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_note_inventory_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
|
||||
@@ -49,6 +49,83 @@ class DailyReportCardTests(unittest.TestCase):
|
||||
self.assertNotIn("![全域种草数据看板]", analysis_text)
|
||||
self.assertIn("波塞冬:继续观察", analysis_text)
|
||||
self.assertIn("退款订单比上升", analysis_text)
|
||||
self.assertEqual(analysis["schema"], "2.0")
|
||||
self.assertEqual(analysis["config"]["width_mode"], "fill")
|
||||
self.assertEqual(
|
||||
[item["tag"] for item in analysis["body"]["elements"]],
|
||||
["column_set", "collapsible_panel", "column_set"],
|
||||
)
|
||||
self.assertNotEqual(
|
||||
analysis["body"]["elements"],
|
||||
[{"tag": "markdown", "content": report}],
|
||||
)
|
||||
|
||||
def test_analysis_card_structures_full_five_section_report(self):
|
||||
report = """品牌SKU营销运营日报
|
||||
日期:2026年8月1日
|
||||
|
||||
一、今日经营概览
|
||||
销量:整体下降。
|
||||
今日核心结论:流量收缩,转化承压。
|
||||
|
||||
二、SKU表现
|
||||
|SKU|今日销量|综合判断|建议动作|
|
||||
|---|---|---|---|
|
||||
|星云2|128|继续放大|维持资源|
|
||||
|星迹2|33|先修承接|优化详情页|
|
||||
|
||||
注:曝光为累计值。
|
||||
|
||||
三、重点SKU分析
|
||||
星云2:经营表现健康,继续放大。
|
||||
|
||||
四、达人营销分析
|
||||
最佳渠道:抖音。
|
||||
|
||||
五、明日运营建议
|
||||
1. 优化星迹2详情页。
|
||||
2. 补充逐星内容。
|
||||
"""
|
||||
card = build_analysis_card("2026-08-01", report)
|
||||
serialized = json.dumps(card, ensure_ascii=False)
|
||||
|
||||
self.assertEqual(
|
||||
[item["tag"] for item in card["body"]["elements"]],
|
||||
[
|
||||
"column_set",
|
||||
"table",
|
||||
"collapsible_panel",
|
||||
"collapsible_panel",
|
||||
"column_set",
|
||||
],
|
||||
)
|
||||
self.assertEqual(len(card["body"]["elements"][1]["rows"]), 2)
|
||||
sku_panel, marketing_panel = card["body"]["elements"][2:4]
|
||||
self.assertTrue(sku_panel["expanded"])
|
||||
self.assertTrue(marketing_panel["expanded"])
|
||||
self.assertEqual(sku_panel["background_color"], "blue-50")
|
||||
self.assertEqual(marketing_panel["background_color"], "violet-50")
|
||||
self.assertNotEqual(
|
||||
sku_panel["background_color"], marketing_panel["background_color"]
|
||||
)
|
||||
self.assertEqual(sku_panel["header"]["title"]["tag"], "markdown")
|
||||
self.assertIn(
|
||||
"**<font color='blue'>重点SKU分析</font>**",
|
||||
sku_panel["header"]["title"]["content"],
|
||||
)
|
||||
self.assertIn(
|
||||
"**<font color='violet'>达人与内容营销分析</font>**",
|
||||
marketing_panel["header"]["title"]["content"],
|
||||
)
|
||||
for phrase in (
|
||||
"流量收缩,转化承压",
|
||||
"曝光为累计值",
|
||||
"继续放大",
|
||||
"最佳渠道:抖音",
|
||||
"补充逐星内容",
|
||||
):
|
||||
self.assertIn(phrase, serialized)
|
||||
self.assertNotIn("button", serialized)
|
||||
|
||||
def test_build_card_uses_card_2_schema_and_dashboard(self):
|
||||
report = """一、核心经营结论
|
||||
@@ -139,25 +216,99 @@ class DailyReportCardTests(unittest.TestCase):
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.subprocess.run"
|
||||
"daily_report_card.send_lark_bot_message"
|
||||
)
|
||||
def test_send_card_to_all_recipients(self, run):
|
||||
run.side_effect = [
|
||||
type("Result", (), {"returncode": 0, "stdout": json.dumps({"message_id": f"om_{i}"}), "stderr": ""})()
|
||||
def test_send_card_to_all_recipients(self, send):
|
||||
send.side_effect = [
|
||||
{"message_id": f"om_{i}"}
|
||||
for i in range(3)
|
||||
]
|
||||
recipients = ["ou_one", "ou_two", "ou_three"]
|
||||
results = send_card_to_recipients({"schema": "2.0"}, recipients, "2026-07-15")
|
||||
|
||||
self.assertEqual([item["message_id"] for item in results], ["om_0", "om_1", "om_2"])
|
||||
self.assertEqual(run.call_count, 3)
|
||||
commands = [call.args[0] for call in run.call_args_list]
|
||||
self.assertEqual([cmd[cmd.index("--user-id") + 1] for cmd in commands], recipients)
|
||||
self.assertTrue(all("--msg-type" in cmd and "interactive" in cmd for cmd in commands))
|
||||
keys = [cmd[cmd.index("--idempotency-key") + 1] for cmd in commands]
|
||||
self.assertEqual(send.call_count, 3)
|
||||
calls = send.call_args_list
|
||||
self.assertEqual([call.kwargs["user_id"] for call in calls], recipients)
|
||||
self.assertTrue(all(call.kwargs["msg_type"] == "interactive" for call in calls))
|
||||
self.assertTrue(all(call.kwargs["profile"] == "hermes-analyzer" for call in calls))
|
||||
keys = [call.kwargs["idempotency_key"] for call in calls]
|
||||
self.assertTrue(all(key.startswith("daily-2026-07-15-") for key in keys))
|
||||
self.assertTrue(all(len(key) <= 50 for key in keys))
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.send_lark_bot_message"
|
||||
)
|
||||
def test_send_card_rejects_success_exit_without_message_id(self, send):
|
||||
send.return_value = {"ok": True}
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "message_id"):
|
||||
send_card_to_recipients(
|
||||
{"schema": "2.0"},
|
||||
["ou_owner"],
|
||||
"2026-08-09",
|
||||
)
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.send_lark_bot_message"
|
||||
)
|
||||
def test_sender_uses_fixed_user_profile_and_hashes_full_recipient_identity(self, send):
|
||||
send.side_effect = [
|
||||
{"message_id": f"om_{index}"}
|
||||
for index in range(2)
|
||||
]
|
||||
recipients = ["ou_first_sharedsuffix", "ou_second_sharedsuffix"]
|
||||
|
||||
send_card_to_recipients(
|
||||
{"schema": "2.0"},
|
||||
recipients,
|
||||
"2026-08-07",
|
||||
)
|
||||
|
||||
calls = send.call_args_list
|
||||
self.assertTrue(
|
||||
all(call.kwargs["profile"] == "hermes-analyzer" for call in calls)
|
||||
)
|
||||
keys = [
|
||||
call.kwargs["idempotency_key"]
|
||||
for call in calls
|
||||
]
|
||||
self.assertEqual(len(set(keys)), 2)
|
||||
with self.assertRaisesRegex(ValueError, "hermes-analyzer"):
|
||||
send_card_to_recipients(
|
||||
{"schema": "2.0"},
|
||||
["ou_owner"],
|
||||
"2026-08-07",
|
||||
app_profile="dynamic-analyzer",
|
||||
)
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.send_lark_bot_message"
|
||||
)
|
||||
def test_stable_fingerprint_ignores_rotating_uploaded_image_key(self, send):
|
||||
send.side_effect = [
|
||||
{"message_id": f"om_{index}"}
|
||||
for index in range(2)
|
||||
]
|
||||
|
||||
for image_key in ("img_first_upload", "img_second_upload"):
|
||||
send_card_to_recipients(
|
||||
build_dashboard_card("2026-08-07", [], image_key),
|
||||
["ou_owner"],
|
||||
"2026-08-07",
|
||||
message_kind="dashboard",
|
||||
idempotency_fingerprint="same-dashboard-bytes-and-metrics",
|
||||
)
|
||||
|
||||
keys = [
|
||||
call.kwargs["idempotency_key"]
|
||||
for call in send.call_args_list
|
||||
]
|
||||
self.assertEqual(keys[0], keys[1])
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.send_card_to_recipients"
|
||||
)
|
||||
@@ -177,13 +328,16 @@ class DailyReportCardTests(unittest.TestCase):
|
||||
]
|
||||
recipients = ["ou_one", "ou_two"]
|
||||
|
||||
results = send_daily_report_cards(
|
||||
"2026-07-17",
|
||||
"第一部分 整体经营分析\n完整分析内容",
|
||||
[{"light": "🟢"}],
|
||||
Path("charts/dashboard.png"),
|
||||
recipients,
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
chart_path = Path(tmp) / "dashboard.png"
|
||||
Image.new("RGB", (4, 4), "white").save(chart_path)
|
||||
results = send_daily_report_cards(
|
||||
"2026-07-17",
|
||||
"第一部分 整体经营分析\n完整分析内容",
|
||||
[{"light": "🟢"}],
|
||||
chart_path,
|
||||
recipients,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[item["message_id"] for item in results],
|
||||
@@ -195,6 +349,10 @@ class DailyReportCardTests(unittest.TestCase):
|
||||
self.assertIn("img_v3_test", json.dumps(first_card, ensure_ascii=False))
|
||||
self.assertIn("完整分析内容", json.dumps(second_card, ensure_ascii=False))
|
||||
self.assertEqual(send.call_args_list[0].kwargs["message_kind"], "dashboard")
|
||||
self.assertIn(
|
||||
"idempotency_fingerprint",
|
||||
send.call_args_list[0].kwargs,
|
||||
)
|
||||
self.assertEqual(send.call_args_list[1].kwargs["message_kind"], "analysis")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters.direct_content_llm import (
|
||||
ContentLLMConfig,
|
||||
ContentLLMConfigurationError,
|
||||
ContentLLMResponseError,
|
||||
call_content_analyzer,
|
||||
load_content_llm_config,
|
||||
)
|
||||
|
||||
|
||||
def _config() -> ContentLLMConfig:
|
||||
return ContentLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/anthropic/v1/messages",
|
||||
credential="test-only-secret",
|
||||
model="MiniMax-M3",
|
||||
thinking_mode="disabled",
|
||||
)
|
||||
|
||||
|
||||
def _response(*blocks: dict[str, object]) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "message",
|
||||
"content": list(blocks),
|
||||
"usage": {"input_tokens": 3, "output_tokens": 4},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def test_config_uses_anthropic_endpoint_and_redacts_key(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"CONTENT_ANALYSIS_LLM_BASE_URL", "https://api.minimaxi.com/anthropic"
|
||||
)
|
||||
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_API_KEY", "test-only-secret")
|
||||
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_MODEL", "MiniMax-M3")
|
||||
|
||||
config = load_content_llm_config()
|
||||
|
||||
assert config.endpoint == "https://api.minimaxi.com/anthropic/v1/messages"
|
||||
assert config.model == "MiniMax-M3"
|
||||
assert "test-only-secret" not in repr(config)
|
||||
|
||||
|
||||
def test_call_sends_anthropic_message_and_ignores_thinking_block() -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def transport(url, headers, body, timeout):
|
||||
payload = json.loads(body)
|
||||
observed.update(
|
||||
url=url,
|
||||
headers=headers,
|
||||
payload=payload,
|
||||
timeout=timeout,
|
||||
)
|
||||
return _response(
|
||||
{"type": "thinking", "thinking": "hidden reasoning"},
|
||||
{"type": "text", "text": "最终报告"},
|
||||
)
|
||||
|
||||
result = call_content_analyzer(
|
||||
"system prompt",
|
||||
"user prompt",
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert result == "最终报告"
|
||||
assert observed["url"] == "https://api.minimaxi.com/anthropic/v1/messages"
|
||||
assert observed["headers"]["X-Api-Key"] == "test-only-secret"
|
||||
assert observed["headers"]["anthropic-version"] == "2023-06-01"
|
||||
assert observed["payload"]["model"] == "MiniMax-M3"
|
||||
assert observed["payload"]["system"] == "system prompt"
|
||||
assert observed["payload"]["messages"] == [
|
||||
{"role": "user", "content": "user prompt"}
|
||||
]
|
||||
assert observed["payload"]["thinking"] == {"type": "disabled"}
|
||||
|
||||
|
||||
def test_call_rejects_openai_response_shape() -> None:
|
||||
with pytest.raises(ContentLLMResponseError, match="response shape"):
|
||||
call_content_analyzer(
|
||||
"system prompt",
|
||||
"user prompt",
|
||||
transport=lambda *_args: b'{"choices": []}',
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_config_rejects_local_gateway(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"CONTENT_ANALYSIS_LLM_BASE_URL", "http://127.0.0.1:8642/v1"
|
||||
)
|
||||
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_API_KEY", "test-only-secret")
|
||||
|
||||
with pytest.raises(ContentLLMConfigurationError, match="local gateway"):
|
||||
load_content_llm_config()
|
||||
@@ -0,0 +1,162 @@
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import douyin_comment_scraper as scraper
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
batch_rescrape_douyin as batch,
|
||||
)
|
||||
|
||||
|
||||
class _EvaluatePage:
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
self.calls = []
|
||||
|
||||
def evaluate(self, script, argument):
|
||||
self.calls.append((script, argument))
|
||||
return self.result
|
||||
|
||||
|
||||
def test_bounded_fetch_uses_abort_controller_and_clears_timer(monkeypatch):
|
||||
page = _EvaluatePage(
|
||||
{"__gyxx_status": "ok", "payload": {"comments": []}}
|
||||
)
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
|
||||
|
||||
payload = scraper._fetch_json_with_timeout(
|
||||
page,
|
||||
"https://example.invalid/comments",
|
||||
deadline=1000.0,
|
||||
stage="top-level pagination",
|
||||
)
|
||||
|
||||
assert payload == {"comments": []}
|
||||
script, argument = page.calls[0]
|
||||
assert "AbortController" in script
|
||||
assert "clearTimeout(timer)" in script
|
||||
assert argument["timeoutMs"] == 30_000
|
||||
|
||||
|
||||
def test_bounded_fetch_timeout_is_an_explicit_transient_failure(monkeypatch):
|
||||
page = _EvaluatePage({"__gyxx_status": "timeout"})
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
|
||||
|
||||
with pytest.raises(scraper.TransientDouyinCommentTimeout) as caught:
|
||||
scraper._fetch_json_with_timeout(
|
||||
page,
|
||||
"https://example.invalid/comments",
|
||||
deadline=1000.0,
|
||||
stage="reply pagination",
|
||||
)
|
||||
|
||||
assert isinstance(caught.value, ConnectionError)
|
||||
assert "reply pagination" in str(caught.value)
|
||||
|
||||
|
||||
def test_scroll_checks_the_note_deadline_before_browser_work(monkeypatch):
|
||||
page = _EvaluatePage(None)
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 901.0)
|
||||
|
||||
with pytest.raises(scraper.TransientDouyinCommentTimeout):
|
||||
scraper.scroll_comments(
|
||||
page,
|
||||
max_scrolls=60,
|
||||
idle_rounds=10,
|
||||
comments_by_id={},
|
||||
deadline=900.0,
|
||||
)
|
||||
|
||||
assert page.calls == []
|
||||
|
||||
|
||||
def test_top_and_reply_pagination_share_the_bounded_fetch(monkeypatch):
|
||||
stages = []
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 0.0)
|
||||
|
||||
def fetch(_page, _url, *, deadline, stage):
|
||||
assert deadline == 900.0
|
||||
stages.append(stage)
|
||||
return {"comments": [], "has_more": 0, "cursor": 0}
|
||||
|
||||
monkeypatch.setattr(scraper, "_fetch_json_with_timeout", fetch)
|
||||
monkeypatch.setattr(scraper, "collect_from_payload", lambda *_args: None)
|
||||
page = _EvaluatePage(None)
|
||||
|
||||
scraper.fetch_missing_top_comments(
|
||||
page,
|
||||
"source-url",
|
||||
"aweme-id",
|
||||
{"existing": {}},
|
||||
{
|
||||
"comment_url_template": "https://example.invalid/top",
|
||||
"api_total": 2,
|
||||
"last_cursor": 1,
|
||||
},
|
||||
deadline=900.0,
|
||||
)
|
||||
scraper.fetch_missing_replies(
|
||||
page,
|
||||
"source-url",
|
||||
{
|
||||
"parent": {
|
||||
"level": "comment",
|
||||
"reply_count": 1,
|
||||
"comment_id": "parent-id",
|
||||
}
|
||||
},
|
||||
{
|
||||
"reply_url_template": "https://example.invalid/reply",
|
||||
"aweme_id": "aweme-id",
|
||||
},
|
||||
deadline=900.0,
|
||||
)
|
||||
|
||||
assert stages == ["top-level pagination", "reply pagination"]
|
||||
|
||||
|
||||
def test_deadline_failure_retries_and_never_replaces_existing_comments(
|
||||
monkeypatch,
|
||||
):
|
||||
attempts = []
|
||||
|
||||
class Checkpoint:
|
||||
completed = []
|
||||
cleared = False
|
||||
|
||||
def __init__(self, _platform):
|
||||
pass
|
||||
|
||||
def contains(self, _note_id, _url):
|
||||
return False
|
||||
|
||||
def mark_completed(self, note_id, url):
|
||||
self.completed.append((note_id, url))
|
||||
|
||||
def clear(self):
|
||||
self.cleared = True
|
||||
|
||||
def scrape(url, *_args):
|
||||
attempts.append(url)
|
||||
raise scraper.TransientDouyinCommentTimeout(
|
||||
"Douyin comment scrape exceeded 900 seconds"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(batch, "CommentBatchCheckpoint", Checkpoint)
|
||||
monkeypatch.setattr(
|
||||
batch,
|
||||
"collect_urls",
|
||||
lambda: [{"id": 1, "url": "dy-url", "comment_count": 10}],
|
||||
)
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(
|
||||
batch.db,
|
||||
"replace_comments",
|
||||
lambda *_args, **_kwargs: pytest.fail(
|
||||
"a timed-out scrape must not replace existing comments"
|
||||
),
|
||||
)
|
||||
|
||||
assert batch.main() == 1
|
||||
assert attempts == ["dy-url", "dy-url", "dy-url"]
|
||||
assert Checkpoint.completed == []
|
||||
assert Checkpoint.cleared is False
|
||||
@@ -0,0 +1,399 @@
|
||||
import json
|
||||
import subprocess
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.feishu_doc_import import (
|
||||
FeishuDocImportAmbiguous,
|
||||
FeishuDocImportError,
|
||||
FeishuDocImportTimeout,
|
||||
_parse_cli_json,
|
||||
deterministic_import_operation_id,
|
||||
import_markdown_document,
|
||||
)
|
||||
|
||||
|
||||
def _completed(payload, *, returncode=0, prefix="", suffix=""):
|
||||
return SimpleNamespace(
|
||||
returncode=returncode,
|
||||
stdout=f"{prefix}{json.dumps(payload)}{suffix}",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
|
||||
class _FakeClock:
|
||||
def __init__(self):
|
||||
self.now = 100.0
|
||||
|
||||
def monotonic(self):
|
||||
return self.now
|
||||
|
||||
def sleep(self, seconds):
|
||||
self.now += seconds
|
||||
|
||||
|
||||
def test_import_ready_result_uses_relative_file_exact_title_and_audits_without_token(
|
||||
tmp_path,
|
||||
):
|
||||
commands = []
|
||||
title = "宙斯 周笔记汇总(2026-07-27~08-02)"
|
||||
markdown = "# 标题\n\n完整正文"
|
||||
document_token = "VERY_SECRET_DOCUMENT_TOKEN_123456"
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
source = command[command.index("--file") + 1]
|
||||
assert source.startswith("./")
|
||||
assert (kwargs["cwd"] / source.removeprefix("./")).read_text(
|
||||
encoding="utf-8"
|
||||
) == markdown
|
||||
return _completed(
|
||||
{
|
||||
"scenario": "import",
|
||||
"ticket": "ticket-ready",
|
||||
"ready": True,
|
||||
"failed": False,
|
||||
"job_status": 0,
|
||||
"job_status_label": "success",
|
||||
"token": document_token,
|
||||
"url": f"https://example.feishu.cn/docx/{document_token}",
|
||||
},
|
||||
prefix="lark-cli notice\n",
|
||||
suffix="\ntrailing notifier",
|
||||
)
|
||||
|
||||
result = import_markdown_document(
|
||||
markdown,
|
||||
title=title,
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="weekly_summary_doc.md",
|
||||
runner=fake_run,
|
||||
operation_id="ready-op",
|
||||
)
|
||||
|
||||
assert result.url == f"https://example.feishu.cn/docx/{document_token}"
|
||||
assert result.document_token == document_token
|
||||
assert commands[0][1:3] == ["drive", "+import"]
|
||||
assert commands[0][commands[0].index("--name") + 1] == title
|
||||
assert commands[0][commands[0].index("--type") + 1] == "docx"
|
||||
assert commands[0][commands[0].index("--as") + 1] == "user"
|
||||
assert "--folder-token" not in commands[0]
|
||||
|
||||
audit_text = result.audit_path.read_text(encoding="utf-8")
|
||||
audit = json.loads(audit_text)
|
||||
assert audit["status"] == "ready"
|
||||
assert audit["ticket"] == "ticket-ready"
|
||||
assert audit["url"] == "https://example.feishu.cn/docx/<redacted>"
|
||||
assert audit["last_result"]["url"] == audit["url"]
|
||||
assert audit["document_title"] == title
|
||||
assert audit["folder_target"] == "root"
|
||||
assert "token" not in audit
|
||||
assert "document_token" not in audit_text
|
||||
assert document_token not in audit_text
|
||||
assert not list((tmp_path / "work").glob("*.md"))
|
||||
|
||||
|
||||
def test_import_persists_ticket_and_polls_task_result_until_ready(tmp_path):
|
||||
responses = iter(
|
||||
[
|
||||
_completed(
|
||||
{
|
||||
"scenario": "import",
|
||||
"ticket": "ticket-async",
|
||||
"ready": False,
|
||||
"failed": False,
|
||||
"timed_out": True,
|
||||
}
|
||||
),
|
||||
_completed(
|
||||
{
|
||||
"scenario": "import",
|
||||
"ticket": "ticket-async",
|
||||
"ready": False,
|
||||
"failed": False,
|
||||
"job_status": 1,
|
||||
"job_status_label": "processing",
|
||||
}
|
||||
),
|
||||
_completed(
|
||||
{
|
||||
"scenario": "import",
|
||||
"ticket": "ticket-async",
|
||||
"ready": True,
|
||||
"failed": False,
|
||||
"job_status": 0,
|
||||
"job_status_label": "success",
|
||||
"token": "doxcnAsyncDocument",
|
||||
"url": "https://example.feishu.cn/docx/doxcnAsyncDocument",
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
commands.append(command)
|
||||
if len(commands) == 2:
|
||||
in_progress = json.loads(
|
||||
(tmp_path / "audit" / "async-op.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert in_progress["status"] == "processing"
|
||||
assert in_progress["ticket"] == "ticket-async"
|
||||
assert in_progress["last_result"]["ready"] is False
|
||||
return next(responses)
|
||||
|
||||
result = import_markdown_document(
|
||||
"report",
|
||||
title="精确标题",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=fake_run,
|
||||
sleeper=lambda _seconds: None,
|
||||
operation_id="async-op",
|
||||
)
|
||||
|
||||
assert result.ticket == "ticket-async"
|
||||
assert len(commands) == 3
|
||||
for command in commands[1:]:
|
||||
assert command[1:3] == ["drive", "+task_result"]
|
||||
assert command[command.index("--scenario") + 1] == "import"
|
||||
assert command[command.index("--ticket") + 1] == "ticket-async"
|
||||
assert command[command.index("--as") + 1] == "user"
|
||||
audit = json.loads(result.audit_path.read_text(encoding="utf-8"))
|
||||
assert audit["poll_attempts"] == 2
|
||||
assert audit["ticket"] == "ticket-async"
|
||||
assert audit["last_result"]["ready"] is True
|
||||
|
||||
|
||||
def test_import_reports_failed_task_without_retrying_mutation(tmp_path):
|
||||
leaked_document_token = "VERY_SECRET_FAILED_DOCUMENT_TOKEN"
|
||||
leaked_open_id = "ou_SHOULD_NOT_BE_IN_AUDIT"
|
||||
responses = iter(
|
||||
[
|
||||
_completed(
|
||||
{
|
||||
"ticket": "ticket-failed",
|
||||
"ready": False,
|
||||
"failed": False,
|
||||
}
|
||||
),
|
||||
_completed(
|
||||
{
|
||||
"ticket": "ticket-failed",
|
||||
"ready": False,
|
||||
"failed": True,
|
||||
"job_status": -1,
|
||||
"job_status_label": "failed",
|
||||
"job_error_msg": (
|
||||
"conversion failed at "
|
||||
f"https://example.feishu.cn/docx/{leaked_document_token} "
|
||||
f'\"token\":\"{leaked_document_token}\" '
|
||||
f'\"open_id\":\"{leaked_open_id}\"'
|
||||
),
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
commands.append(command)
|
||||
return next(responses)
|
||||
|
||||
with pytest.raises(FeishuDocImportError, match="conversion failed") as raised:
|
||||
import_markdown_document(
|
||||
"report",
|
||||
title="失败报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=fake_run,
|
||||
sleeper=lambda _seconds: None,
|
||||
operation_id="failed-op",
|
||||
)
|
||||
|
||||
assert sum(command[1:3] == ["drive", "+import"] for command in commands) == 1
|
||||
audit_text = raised.value.audit_path.read_text(encoding="utf-8")
|
||||
audit = json.loads(audit_text)
|
||||
assert audit["status"] == "failed"
|
||||
assert audit["ticket"] == "ticket-failed"
|
||||
assert audit["last_result"]["failed"] is True
|
||||
assert leaked_document_token not in audit_text
|
||||
assert leaked_open_id not in audit_text
|
||||
|
||||
|
||||
def test_import_timeout_keeps_ticket_for_auditable_follow_up(tmp_path):
|
||||
clock = _FakeClock()
|
||||
responses = iter(
|
||||
[
|
||||
_completed(
|
||||
{"ticket": "ticket-timeout", "ready": False, "failed": False}
|
||||
),
|
||||
_completed(
|
||||
{
|
||||
"ticket": "ticket-timeout",
|
||||
"ready": False,
|
||||
"failed": False,
|
||||
"job_status_label": "processing",
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(FeishuDocImportTimeout) as raised:
|
||||
import_markdown_document(
|
||||
"report",
|
||||
title="超时报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=lambda *_args, **_kwargs: next(responses),
|
||||
clock=clock.monotonic,
|
||||
sleeper=clock.sleep,
|
||||
wait_timeout_seconds=1,
|
||||
poll_interval_seconds=1,
|
||||
operation_id="timeout-op",
|
||||
)
|
||||
|
||||
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
|
||||
assert audit["status"] == "timed_out"
|
||||
assert audit["ticket"] == "ticket-timeout"
|
||||
assert audit["poll_attempts"] == 1
|
||||
|
||||
|
||||
def test_mutating_import_process_timeout_is_ambiguous_and_never_retried(tmp_path):
|
||||
commands = []
|
||||
|
||||
def time_out(command, **kwargs):
|
||||
commands.append(command)
|
||||
assert kwargs["timeout"] == 2
|
||||
raise subprocess.TimeoutExpired(command, kwargs["timeout"])
|
||||
|
||||
with pytest.raises(FeishuDocImportTimeout) as raised:
|
||||
import_markdown_document(
|
||||
"report",
|
||||
title="有限超时报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=time_out,
|
||||
import_command_timeout_seconds=2,
|
||||
operation_id="command-timeout-op",
|
||||
)
|
||||
|
||||
assert len(commands) == 1
|
||||
assert commands[0][1:3] == ["drive", "+import"]
|
||||
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
|
||||
assert audit["status"] == "ambiguous"
|
||||
assert audit["ticket"] is None
|
||||
assert not list((tmp_path / "work").glob("*.md"))
|
||||
|
||||
|
||||
def test_deterministic_audit_resumes_only_known_ticket_without_second_import(tmp_path):
|
||||
audit_root = tmp_path / "audit"
|
||||
audit_root.mkdir()
|
||||
operation_id = "stable-report-op"
|
||||
audit_path = audit_root / f"{operation_id}.json"
|
||||
audit_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"audit_schema": "gyxx.feishu-doc-import.v1",
|
||||
"document_title": "固定周期报告",
|
||||
"folder_target": "root",
|
||||
"identity": "user",
|
||||
"markdown_sha256": "previous-input-digest",
|
||||
"operation_id": operation_id,
|
||||
"poll_attempts": 1,
|
||||
"status": "timed_out",
|
||||
"target_type": "docx",
|
||||
"ticket": "known-ticket",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
commands.append(command)
|
||||
return _completed(
|
||||
{
|
||||
"ticket": "known-ticket",
|
||||
"ready": True,
|
||||
"failed": False,
|
||||
"token": "doxcnRecoveredDocument",
|
||||
"url": "https://example.feishu.cn/docx/doxcnRecoveredDocument",
|
||||
}
|
||||
)
|
||||
|
||||
result = import_markdown_document(
|
||||
"possibly regenerated report",
|
||||
title="固定周期报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=audit_root,
|
||||
source_filename="report.md",
|
||||
runner=fake_run,
|
||||
operation_id=operation_id,
|
||||
)
|
||||
|
||||
assert result.url.endswith("/doxcnRecoveredDocument")
|
||||
assert len(commands) == 1
|
||||
assert commands[0][1:3] == ["drive", "+task_result"]
|
||||
assert all(command[1:3] != ["drive", "+import"] for command in commands)
|
||||
audit_text = audit_path.read_text(encoding="utf-8")
|
||||
audit = json.loads(audit_text)
|
||||
assert audit["status"] == "ready"
|
||||
assert audit["resume_count"] == 1
|
||||
assert audit["resume_input_changed"] is True
|
||||
assert "doxcnRecoveredDocument" not in audit_text
|
||||
|
||||
|
||||
def test_resume_only_without_ticket_audit_refuses_mutating_import(tmp_path):
|
||||
commands = []
|
||||
|
||||
with pytest.raises(FeishuDocImportAmbiguous, match="缺少可对账") as raised:
|
||||
import_markdown_document(
|
||||
"report",
|
||||
title="待对账报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=lambda command, **_kwargs: commands.append(command),
|
||||
operation_id="missing-ticket-op",
|
||||
resume_only=True,
|
||||
)
|
||||
|
||||
assert commands == []
|
||||
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
|
||||
assert audit["status"] == "ambiguous"
|
||||
assert audit["ticket"] is None
|
||||
|
||||
|
||||
def test_deterministic_operation_id_is_stable_and_period_specific():
|
||||
first = deterministic_import_operation_id(
|
||||
"content-weekly-summary", "宙斯", "2026-07-27", "2026-08-02"
|
||||
)
|
||||
same = deterministic_import_operation_id(
|
||||
"content-weekly-summary", "宙斯", "2026-07-27", "2026-08-02"
|
||||
)
|
||||
other_style = deterministic_import_operation_id(
|
||||
"content-weekly-summary", "赫拉", "2026-07-27", "2026-08-02"
|
||||
)
|
||||
|
||||
assert first == same
|
||||
assert first != other_style
|
||||
assert first.startswith("content-weekly-summary-")
|
||||
assert len(first.rsplit("-", 1)[1]) == 40
|
||||
|
||||
|
||||
def test_cli_json_parser_accepts_noise_and_rejects_invalid_output():
|
||||
assert _parse_cli_json(
|
||||
'notice {"notice":"update"}\n{"ready":false,"ticket":"abc"}\ntrailing'
|
||||
) == {
|
||||
"ready": False,
|
||||
"ticket": "abc",
|
||||
}
|
||||
with pytest.raises(ValueError, match="no JSON"):
|
||||
_parse_cli_json("plain text", "also plain text")
|
||||
@@ -1,3 +1,7 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
friday_relogin_parallel as relogin,
|
||||
)
|
||||
@@ -43,6 +47,434 @@ def test_failed_platforms_are_grouped_by_their_responsible_recipient():
|
||||
}
|
||||
|
||||
|
||||
def test_legacy_route_keeps_the_existing_per_platform_recipients(monkeypatch):
|
||||
observed = {}
|
||||
|
||||
def fake_resolver(workflow_id, defaults):
|
||||
observed["workflow_id"] = workflow_id
|
||||
observed["defaults"] = defaults
|
||||
return relogin.ResolvedNotificationRoute(
|
||||
configured=False,
|
||||
enabled=True,
|
||||
app_profile=None,
|
||||
open_ids=tuple(defaults),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(relogin, "resolve_notification_route", fake_resolver)
|
||||
|
||||
route = relogin.resolve_weekly_notification_route()
|
||||
|
||||
assert observed == {
|
||||
"workflow_id": "content.relogin.weekly",
|
||||
"defaults": (
|
||||
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
),
|
||||
}
|
||||
assert relogin.recipients_for_platform("pgy", route) == (
|
||||
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
)
|
||||
assert relogin.recipients_for_platform("douyin", route) == (
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
)
|
||||
|
||||
|
||||
def test_disabled_route_still_runs_relogin_without_sending(monkeypatch):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=False,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=(),
|
||||
)
|
||||
rounds = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"resolve_weekly_notification_route",
|
||||
lambda _override=None: route,
|
||||
)
|
||||
|
||||
def fake_round_factory(_args, resolved_route, notification_failures):
|
||||
assert resolved_route is route
|
||||
assert notification_failures == []
|
||||
|
||||
def run_round(names, attempt):
|
||||
rounds.append((tuple(names), attempt))
|
||||
return {name: True for name in names}
|
||||
|
||||
return run_round
|
||||
|
||||
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"_send_lark",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
AssertionError("disabled route must not send Lark messages")
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
relogin.sys,
|
||||
"argv",
|
||||
["friday_relogin_parallel.py", "--platform", "pgy"],
|
||||
)
|
||||
|
||||
assert relogin.main() == 0
|
||||
assert rounds == [(("pgy",), 1)]
|
||||
|
||||
|
||||
def test_enabled_route_sends_qr_text_and_image_to_every_recipient(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
sent = []
|
||||
image = tmp_path / "qrcode.png"
|
||||
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"_send_lark",
|
||||
lambda recipient, text=None, image=None, app_profile=None: sent.append(
|
||||
(recipient, text, image, app_profile)
|
||||
),
|
||||
)
|
||||
|
||||
failures = relogin._send_qr_notification(
|
||||
route,
|
||||
"pgy",
|
||||
override=None,
|
||||
text="请扫码",
|
||||
image=image,
|
||||
)
|
||||
|
||||
assert failures == ()
|
||||
assert sent == [
|
||||
("ou_first123", "请扫码", None, "hermes-analyzer"),
|
||||
("ou_first123", None, image, "hermes-analyzer"),
|
||||
("ou_second456", "请扫码", None, "hermes-analyzer"),
|
||||
("ou_second456", None, image, "hermes-analyzer"),
|
||||
]
|
||||
assert relogin.group_platforms_by_notification_recipient(
|
||||
["pgy", "douyin"], route
|
||||
) == {
|
||||
"ou_first123": ["pgy", "douyin"],
|
||||
"ou_second456": ["pgy", "douyin"],
|
||||
}
|
||||
|
||||
|
||||
def test_qr_delivery_aggregates_errors_and_attempts_every_recipient(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
attempts = []
|
||||
image = tmp_path / "qrcode.png"
|
||||
|
||||
def fake_send(recipient, text=None, image=None, app_profile=None):
|
||||
stage = "qr_text" if text is not None else "qr_image"
|
||||
attempts.append((recipient, stage, app_profile))
|
||||
if (recipient, stage) == ("ou_first123", "qr_text"):
|
||||
raise RuntimeError("first text rejected")
|
||||
if (recipient, stage) == ("ou_second456", "qr_image"):
|
||||
raise RuntimeError("second image rejected")
|
||||
|
||||
monkeypatch.setattr(relogin, "_send_lark", fake_send)
|
||||
|
||||
failures = relogin._send_qr_notification(
|
||||
route,
|
||||
"pgy",
|
||||
override=None,
|
||||
text="请扫码",
|
||||
image=image,
|
||||
)
|
||||
|
||||
assert attempts == [
|
||||
("ou_first123", "qr_text", "hermes-analyzer"),
|
||||
("ou_first123", "qr_image", "hermes-analyzer"),
|
||||
("ou_second456", "qr_text", "hermes-analyzer"),
|
||||
("ou_second456", "qr_image", "hermes-analyzer"),
|
||||
]
|
||||
assert failures == (
|
||||
relogin.NotificationDeliveryFailure(
|
||||
stage="qr_text",
|
||||
platform="pgy",
|
||||
recipient="ou_first123",
|
||||
detail="first text rejected",
|
||||
),
|
||||
relogin.NotificationDeliveryFailure(
|
||||
stage="qr_image",
|
||||
platform="pgy",
|
||||
recipient="ou_second456",
|
||||
detail="second image rejected",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_failure_summary_aggregates_every_recipient_error(monkeypatch):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
attempts = []
|
||||
|
||||
def fail_send(recipient, **_kwargs):
|
||||
attempts.append(recipient)
|
||||
raise RuntimeError(f"rejected {recipient}")
|
||||
|
||||
monkeypatch.setattr(relogin, "_send_lark", fail_send)
|
||||
|
||||
failures = relogin._send_failure_summaries(
|
||||
route,
|
||||
["pgy", "douyin"],
|
||||
override=None,
|
||||
max_attempts=3,
|
||||
)
|
||||
|
||||
assert attempts == ["ou_first123", "ou_second456"]
|
||||
assert [failure.stage for failure in failures] == [
|
||||
"failure_summary",
|
||||
"failure_summary",
|
||||
]
|
||||
assert [failure.recipient for failure in failures] == attempts
|
||||
assert [failure.platform for failure in failures] == [
|
||||
"pgy,douyin",
|
||||
"pgy,douyin",
|
||||
]
|
||||
|
||||
|
||||
def test_main_returns_failure_and_reports_all_notification_errors(
|
||||
monkeypatch,
|
||||
capsys,
|
||||
):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"resolve_weekly_notification_route",
|
||||
lambda _override=None: route,
|
||||
)
|
||||
|
||||
def fake_round_factory(_args, _route, notification_failures):
|
||||
notification_failures.extend(
|
||||
(
|
||||
relogin.NotificationDeliveryFailure(
|
||||
"qr_text", "pgy", "ou_first123", "text rejected"
|
||||
),
|
||||
relogin.NotificationDeliveryFailure(
|
||||
"qr_image", "pgy", "ou_second456", "image rejected"
|
||||
),
|
||||
)
|
||||
)
|
||||
return lambda names, _attempt: {name: True for name in names}
|
||||
|
||||
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
|
||||
monkeypatch.setattr(
|
||||
relogin.sys,
|
||||
"argv",
|
||||
["friday_relogin_parallel.py", "--platform", "pgy"],
|
||||
)
|
||||
|
||||
assert relogin.main() == 2
|
||||
output = capsys.readouterr().out
|
||||
assert "Final relogin status: {'pgy': True}" in output
|
||||
assert "Final notification status: failed (2 deliveries)" in output
|
||||
assert "recipient=ou_first123 error=text rejected" in output
|
||||
assert "recipient=ou_second456 error=image rejected" in output
|
||||
|
||||
|
||||
def test_round_completes_relogin_after_notification_failure(monkeypatch, tmp_path):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
args = SimpleNamespace(
|
||||
round_timeout=10,
|
||||
screenshot_delay=0,
|
||||
no_send=False,
|
||||
recipient=None,
|
||||
max_attempts=1,
|
||||
)
|
||||
failures = []
|
||||
attempts = []
|
||||
waits = []
|
||||
|
||||
class Process:
|
||||
pid = 1234
|
||||
|
||||
def wait(self, timeout):
|
||||
waits.append(timeout)
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
|
||||
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"binding_from_environment",
|
||||
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
|
||||
)
|
||||
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(relogin, "_login_windows", lambda: {"pgy": 101})
|
||||
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
|
||||
monkeypatch.setattr(relogin, "_capture_window", lambda _hwnd, _output: None)
|
||||
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: True)
|
||||
|
||||
def fake_send(recipient, text=None, image=None, **_kwargs):
|
||||
stage = "qr_text" if text is not None else "qr_image"
|
||||
attempts.append((recipient, stage))
|
||||
if (recipient, stage) == ("ou_first123", "qr_text"):
|
||||
raise RuntimeError("temporary send failure")
|
||||
|
||||
monkeypatch.setattr(relogin, "_send_lark", fake_send)
|
||||
|
||||
run_round = relogin._run_round_factory(args, route, failures)
|
||||
|
||||
assert run_round(["pgy"], 1) == {"pgy": True}
|
||||
assert waits == [100]
|
||||
assert attempts == [
|
||||
("ou_first123", "qr_text"),
|
||||
("ou_first123", "qr_image"),
|
||||
("ou_second456", "qr_text"),
|
||||
("ou_second456", "qr_image"),
|
||||
]
|
||||
assert failures == [
|
||||
relogin.NotificationDeliveryFailure(
|
||||
stage="qr_text",
|
||||
platform="pgy",
|
||||
recipient="ou_first123",
|
||||
detail="temporary send failure",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("returncode", "cookie_valid", "expected_status", "expected_failure_count"),
|
||||
(
|
||||
(0, True, True, 0),
|
||||
(1, False, False, 4),
|
||||
),
|
||||
)
|
||||
def test_missing_qr_window_is_classified_after_relogin_result(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
returncode,
|
||||
cookie_valid,
|
||||
expected_status,
|
||||
expected_failure_count,
|
||||
):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
args = SimpleNamespace(
|
||||
round_timeout=10,
|
||||
screenshot_delay=0,
|
||||
no_send=False,
|
||||
recipient=None,
|
||||
max_attempts=1,
|
||||
)
|
||||
failures = []
|
||||
|
||||
class Process:
|
||||
pid = 1234
|
||||
|
||||
def wait(self, timeout):
|
||||
assert timeout == 100
|
||||
return returncode
|
||||
|
||||
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
|
||||
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"binding_from_environment",
|
||||
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
|
||||
)
|
||||
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(relogin, "_login_windows", lambda: {})
|
||||
monkeypatch.setattr(relogin, "_window_for_process", lambda _pid: None)
|
||||
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
|
||||
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: cookie_valid)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"_send_lark",
|
||||
lambda *_args, **_kwargs: pytest.fail(
|
||||
"no QR window must not start a send before child classification"
|
||||
),
|
||||
)
|
||||
|
||||
run_round = relogin._run_round_factory(args, route, failures)
|
||||
|
||||
assert run_round(["pgy"], 1) == {"pgy": expected_status}
|
||||
assert len(failures) == expected_failure_count
|
||||
assert {failure.stage for failure in failures} <= {"qr_text", "qr_image"}
|
||||
assert all(
|
||||
failure.detail == "no visible QR window and relogin did not succeed"
|
||||
for failure in failures
|
||||
)
|
||||
|
||||
|
||||
def test_lark_send_uses_shared_analyzer_user_sender_for_text_and_image(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
calls = []
|
||||
image = tmp_path / "qrcode.png"
|
||||
image.write_bytes(b"png")
|
||||
|
||||
monkeypatch.setattr(relogin, "ANALYZER_DIR", tmp_path)
|
||||
monkeypatch.setenv("HERMES_ANALYZER_TOKEN", "must-not-leak")
|
||||
|
||||
def fake_send(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return {"message_id": f"om_{len(calls)}"}
|
||||
|
||||
monkeypatch.setattr(relogin, "send_lark_bot_message", fake_send)
|
||||
|
||||
receipts = relogin._send_lark(
|
||||
"ou_recipient123",
|
||||
text="登录提醒",
|
||||
image=image,
|
||||
)
|
||||
|
||||
assert [receipt["message_id"] for receipt in receipts] == ["om_1", "om_2"]
|
||||
assert calls[0]["user_id"] == "ou_recipient123"
|
||||
assert calls[0]["text"] == "登录提醒"
|
||||
assert "image" not in calls[0]
|
||||
assert calls[1]["user_id"] == "ou_recipient123"
|
||||
assert calls[1]["image"] == image
|
||||
assert "text" not in calls[1]
|
||||
assert all(call["profile"] == "hermes-analyzer" for call in calls)
|
||||
assert all("HERMES_ANALYZER_TOKEN" not in call["env"] for call in calls)
|
||||
|
||||
|
||||
def test_login_windows_are_classified_by_title():
|
||||
assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili"
|
||||
assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy"
|
||||
|
||||
@@ -169,6 +169,73 @@ def test_xingtu_search_and_open_creator_submits_only_one_search(monkeypatch):
|
||||
assert calls == [("同一达人", "creator-1")]
|
||||
|
||||
|
||||
def test_creator_search_attempts_id_then_nickname_and_skips_blank_id():
|
||||
assert xingtu._creator_search_attempts("Rm.", "Ruan97") == [
|
||||
("ID", "Ruan97", "Ruan97"),
|
||||
("昵称", "Rm.", None),
|
||||
]
|
||||
assert xingtu._creator_search_attempts("Rm.", " ") == [
|
||||
("昵称", "Rm.", None),
|
||||
]
|
||||
|
||||
|
||||
def test_xingtu_primary_retries_only_unmatched_tasks_through_buyin(monkeypatch):
|
||||
xingtu.configure_source("xingtu")
|
||||
styles = [{"index": 1, "name": "款式A"}]
|
||||
context = {"index": 1}
|
||||
tasks = [
|
||||
{"record_id": "hit", "creator_name": "达人A", "creator_id": "id-a", "style_context": context},
|
||||
{"record_id": "miss", "creator_name": "达人B", "creator_id": "id-b", "style_context": context},
|
||||
]
|
||||
summaries = {
|
||||
1: {
|
||||
"style": "款式A",
|
||||
"index": 1,
|
||||
"total": 2,
|
||||
"matched": 0,
|
||||
"filled": 0,
|
||||
"skipped_no_pubtime": 0,
|
||||
"results": [],
|
||||
}
|
||||
}
|
||||
calls = []
|
||||
|
||||
def fake_collect(*_args, **_kwargs):
|
||||
return tasks, summaries
|
||||
|
||||
def fake_scrape(*_args, **kwargs):
|
||||
source = xingtu.CONTENT_SOURCE
|
||||
selected = kwargs["task_override"]
|
||||
calls.append((source, [task["record_id"] for task in selected], kwargs.get("replace_existing", False)))
|
||||
for task in selected:
|
||||
matched = source == "buyin" or task["record_id"] == "hit"
|
||||
xingtu.upsert_result(summaries[1], {
|
||||
"record_id": task["record_id"],
|
||||
"matched": matched,
|
||||
"status": xingtu.SUCCESS if matched else xingtu.RETRYABLE_FAILURE,
|
||||
"reason": None if matched else "title_unmatched",
|
||||
})
|
||||
return [summaries[1]]
|
||||
|
||||
monkeypatch.setattr(xingtu, "collect_tasks_across_styles", fake_collect)
|
||||
monkeypatch.setattr(xingtu, "scrape_styles", fake_scrape)
|
||||
try:
|
||||
result = xingtu.scrape_styles_with_fallback(
|
||||
styles, 1, False, None, True,
|
||||
)
|
||||
finally:
|
||||
xingtu.configure_source("xingtu")
|
||||
|
||||
assert calls == [
|
||||
("xingtu", ["hit", "miss"], False),
|
||||
("buyin", ["miss"], True),
|
||||
]
|
||||
rows = {row["record_id"]: row for row in result[0]["results"]}
|
||||
assert rows["hit"]["collection_source"] == "xingtu"
|
||||
assert rows["miss"]["collection_source"] == "buyin_fallback"
|
||||
assert rows["miss"]["fallback_from"] == "xingtu"
|
||||
|
||||
|
||||
def test_xingtu_search_budget_blocks_duplicate_without_failure_circuit():
|
||||
budget = xingtu.CreatorSearchBudget(max_consecutive_failures=3)
|
||||
|
||||
@@ -291,20 +358,22 @@ def test_scheduled_collection_has_no_automatic_retry_and_syncs_once():
|
||||
)
|
||||
|
||||
|
||||
def test_daily_collection_uses_dedicated_self_operated_pipeline():
|
||||
def test_daily_collection_includes_self_operated_pipeline():
|
||||
steps = _workflow("content.metrics.daily").steps
|
||||
entries = [step.entry for step in steps]
|
||||
run_step = next(step for step in steps if step.entry == "run_all.py")
|
||||
|
||||
assert "--daily-scope" in run_step.args
|
||||
assert "--include-self-operated" not in run_step.args
|
||||
assert "data/tools/refresh_self_mapping.py" in entries
|
||||
assert "self_bilibili_scraper.py" in entries
|
||||
assert "chanmama_scraper.py" in entries
|
||||
refresh_index = entries.index("data/tools/refresh_self_mapping.py")
|
||||
bili_index = entries.index("self_bilibili_scraper.py")
|
||||
douyin_index = entries.index("chanmama_scraper.py")
|
||||
assert refresh_index < bili_index < douyin_index
|
||||
assert entries.index("run_all.py") < entries.index(
|
||||
"data/tools/refresh_self_mapping.py"
|
||||
)
|
||||
assert entries.index("data/tools/refresh_self_mapping.py") < entries.index(
|
||||
"self_bilibili_scraper.py"
|
||||
)
|
||||
assert entries.index("self_bilibili_scraper.py") < entries.index(
|
||||
"chanmama_scraper.py"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools import sync_notes_master as sync
|
||||
|
||||
|
||||
def _table() -> dict:
|
||||
return {
|
||||
"name": "宙斯",
|
||||
"base_token": "base-token",
|
||||
"table_id": "tbl-source",
|
||||
"field_map": {
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "published"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"platform": {"field_id": "platform"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_parse_records_counts_only_complete_note_rows() -> None:
|
||||
rows = sync.parse_records(
|
||||
_table(),
|
||||
[
|
||||
{
|
||||
"record_id": "complete",
|
||||
"title": "通勤双肩包实测",
|
||||
"published": "2026-08-11 09:20:00",
|
||||
"url": "[笔记](https://www.xiaohongshu.com/explore/abc)",
|
||||
"platform": "小红书",
|
||||
"creator": "达人甲",
|
||||
},
|
||||
{
|
||||
"record_id": "missing-publish-time",
|
||||
"title": "只有标题和链接",
|
||||
"published": None,
|
||||
"url": "https://www.douyin.com/video/123",
|
||||
"platform": "抖音",
|
||||
"creator": "达人乙",
|
||||
},
|
||||
],
|
||||
self_operated=False,
|
||||
)
|
||||
|
||||
assert rows[0].is_countable is True
|
||||
assert rows[0].platform == "xiaohongshu"
|
||||
assert rows[0].publish_time == "2026-08-11 09:20:00+08:00"
|
||||
assert rows[0].url == "https://www.xiaohongshu.com/explore/abc"
|
||||
assert rows[1].is_countable is False
|
||||
|
||||
|
||||
def test_platform_can_be_inferred_from_url_when_source_cell_is_empty() -> None:
|
||||
rows = sync.parse_records(
|
||||
_table(),
|
||||
[{
|
||||
"record_id": "bili",
|
||||
"title": "B站开箱",
|
||||
"published": "2026-08-10",
|
||||
"url": "https://www.bilibili.com/video/BV1test",
|
||||
"platform": "",
|
||||
"creator": "自营号",
|
||||
}],
|
||||
self_operated=True,
|
||||
)
|
||||
|
||||
assert rows[0].platform == "bilibili"
|
||||
assert rows[0].is_countable is True
|
||||
|
||||
|
||||
def test_unrecognized_source_text_is_bounded_to_database_columns() -> None:
|
||||
rows = sync.parse_records(
|
||||
_table(),
|
||||
[{
|
||||
"record_id": "unexpected-long-values",
|
||||
"title": "题" * 600,
|
||||
"published": "2026-08-10",
|
||||
"url": "not-a-platform-url",
|
||||
"platform": "unexpected-platform-value-" * 4,
|
||||
"creator": "创" * 300,
|
||||
}],
|
||||
self_operated=False,
|
||||
)
|
||||
|
||||
assert len(rows[0].platform) == 32
|
||||
assert len(rows[0].creator_name) == 255
|
||||
assert len(rows[0].title) == 512
|
||||
|
||||
|
||||
def test_collaboration_fields_parsed_only_for_collaborator_tables() -> None:
|
||||
table = _table()
|
||||
table["field_map"].update(
|
||||
{
|
||||
"cost": {"field_id": "cost"},
|
||||
"tracking_no": {"field_id": "tracking"},
|
||||
"is_paid": {"field_id": "paid"},
|
||||
"follower_count": {"field_id": "followers"},
|
||||
}
|
||||
)
|
||||
records = [{
|
||||
"record_id": "coop-1",
|
||||
"title": "合作笔记",
|
||||
"published": "2026-08-11",
|
||||
"url": "https://www.douyin.com/video/9",
|
||||
"platform": "抖音",
|
||||
"creator": "达人丙",
|
||||
"cost": 1200,
|
||||
"tracking": "SF123",
|
||||
"paid": True,
|
||||
"followers": "3.7万",
|
||||
}]
|
||||
|
||||
coop_row = sync.parse_records(table, records, self_operated=False)[0]
|
||||
assert coop_row.coop_attrs["cooperation_cost"] == 1200.0
|
||||
assert coop_row.coop_attrs["tracking_number"] == "SF123"
|
||||
assert coop_row.coop_attrs["is_paid"] is True
|
||||
assert coop_row.creator_attrs["follower_count_num"] == 37000
|
||||
|
||||
self_row = sync.parse_records(table, records, self_operated=True)[0]
|
||||
assert self_row.coop_attrs == {}
|
||||
assert self_row.creator_attrs == {}
|
||||
|
||||
|
||||
def test_dry_run_fetches_both_scopes_without_opening_database(monkeypatch) -> None:
|
||||
collaborator = {**_table(), "name": "合作款"}
|
||||
self_operated = {**_table(), "name": "自营款", "table_id": "tbl-self"}
|
||||
monkeypatch.setattr(
|
||||
sync,
|
||||
"load_sources",
|
||||
lambda **_kwargs: [(False, collaborator), (True, self_operated)],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync.feishu_mapping,
|
||||
"_record_list_all",
|
||||
lambda *_args: [{
|
||||
"record_id": "one",
|
||||
"title": "完整笔记",
|
||||
"published": "2026-08-11",
|
||||
"url": "https://www.douyin.com/video/1",
|
||||
"platform": "抖音",
|
||||
"creator": "达人",
|
||||
}],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync.psycopg,
|
||||
"connect",
|
||||
lambda **_kwargs: (_ for _ in ()).throw(
|
||||
AssertionError("dry-run must not open PostgreSQL")
|
||||
),
|
||||
)
|
||||
|
||||
summary = sync.synchronize()
|
||||
|
||||
assert summary == {
|
||||
"tables": 2,
|
||||
"rows": 2,
|
||||
"creators": 0,
|
||||
"countable": 2,
|
||||
"incomplete": 0,
|
||||
"deactivated": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_sync_table_deactivates_only_rows_missing_from_same_source() -> None:
|
||||
class Cursor:
|
||||
rowcount = 3
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = []
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
self.calls.append((" ".join(sql.split()), params))
|
||||
|
||||
def fetchone(self):
|
||||
return (42,)
|
||||
|
||||
cur = Cursor()
|
||||
rows = [
|
||||
sync.MasterRow(
|
||||
record_id="rec-1",
|
||||
platform="xiaohongshu",
|
||||
creator_name="达人",
|
||||
title="标题",
|
||||
publish_time="2026-08-11 00:00:00+08:00",
|
||||
url="https://www.xiaohongshu.com/explore/1",
|
||||
)
|
||||
]
|
||||
|
||||
result = sync.sync_table(cur, _table(), self_operated=False, rows=rows)
|
||||
|
||||
assert result["countable"] == 1
|
||||
deactivate_sql, params = cur.calls[-1]
|
||||
assert "source_base_token = %s AND source_table_id = %s" in deactivate_sql
|
||||
assert "NOT (feishu_record_id = ANY(%s))" in deactivate_sql
|
||||
assert params == ("base-token", "tbl-source", ["rec-1"])
|
||||
@@ -15,25 +15,31 @@ from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "state.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
cookie_backup, profile_backup, state_backup = module.reset_state()
|
||||
assert not cookie.exists()
|
||||
assert not storage.exists()
|
||||
assert not profile.exists()
|
||||
|
||||
cookie.write_text("partial-new-cookie", encoding="utf-8")
|
||||
storage.write_text("partial-new-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
module.restore_previous_state(cookie_backup, profile_backup)
|
||||
module.restore_previous_state(cookie_backup, profile_backup, state_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
|
||||
@@ -41,21 +47,27 @@ def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, modul
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
cookie_backup, profile_backup, state_backup = module.reset_state()
|
||||
cookie.write_text("new-cookie", encoding="utf-8")
|
||||
storage.write_text("new-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
module.finish_successful_relogin(profile_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "new-cookie"
|
||||
assert storage.read_text(encoding="utf-8") == "new-storage"
|
||||
assert cookie_backup.exists()
|
||||
assert state_backup.exists()
|
||||
assert not profile_backup.exists()
|
||||
|
||||
|
||||
@@ -127,6 +139,82 @@ def test_xingtu_index_page_is_recognized_as_logged_in():
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_index_page_with_xingtu_session_is_logged_in_without_market_ui():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def count(self):
|
||||
return 0
|
||||
|
||||
def wait_for(self, timeout):
|
||||
raise xingtu_scraper_v2.BrowserTimeoutError("not on market page")
|
||||
|
||||
def is_visible(self):
|
||||
return False
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [
|
||||
{
|
||||
"name": "sessionid",
|
||||
"value": "fresh",
|
||||
"domain": ".xingtu.cn",
|
||||
}
|
||||
]
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/ad/creator/index"
|
||||
context = Context()
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
def locator(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_index_page_without_xingtu_session_is_not_logged_in():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def count(self):
|
||||
return 0
|
||||
|
||||
def wait_for(self, timeout):
|
||||
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
|
||||
|
||||
def is_visible(self):
|
||||
return False
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [
|
||||
{
|
||||
"name": "sessionid",
|
||||
"value": "douyin-only",
|
||||
"domain": ".douyin.com",
|
||||
}
|
||||
]
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/ad/creator/index"
|
||||
context = Context()
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
def locator(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie():
|
||||
class Context:
|
||||
def cookies(self):
|
||||
@@ -149,7 +237,7 @@ def test_xingtu_market_page_without_business_ui_rejects_stale_cookie():
|
||||
return self
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
raise xingtu_scraper_v2.PlaywrightTimeoutError("missing")
|
||||
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
|
||||
|
||||
def count(self):
|
||||
return 0
|
||||
@@ -282,25 +370,64 @@ def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monke
|
||||
assert saved == [market_page]
|
||||
|
||||
|
||||
def test_xingtu_scan_login_detects_business_tab_while_qr_tab_stays_open(monkeypatch):
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.pages = []
|
||||
|
||||
class Page:
|
||||
def __init__(self, context, url):
|
||||
self.context = context
|
||||
self.url = url
|
||||
|
||||
def is_closed(self):
|
||||
return False
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
raise AssertionError("authenticated tab should be detected immediately")
|
||||
|
||||
context = Context()
|
||||
qr_page = Page(context, "https://open.douyin.com/platform/oauth/pc/auth")
|
||||
index_page = Page(context, "https://www.xingtu.cn/ad/creator/index")
|
||||
context.pages = [qr_page, index_page]
|
||||
saved = []
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None)
|
||||
monkeypatch.setattr(
|
||||
xingtu_scraper_v2, "is_logged_in", lambda page: page is index_page
|
||||
)
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page))
|
||||
|
||||
result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2)
|
||||
|
||||
assert result is index_page
|
||||
assert saved == [index_page]
|
||||
|
||||
|
||||
def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch):
|
||||
cookie = tmp_path / "xingtu_cookies.json"
|
||||
storage = tmp_path / "xingtu_storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(relogin_xingtu, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = relogin_xingtu.reset_state()
|
||||
cookie_backup, profile_backup, state_backup = relogin_xingtu.reset_state()
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
assert relogin_xingtu.recover_interrupted_state() is True
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
assert cookie_backup is not None
|
||||
assert state_backup is not None
|
||||
assert profile_backup is not None
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
relogin_bilibili,
|
||||
relogin_douyin,
|
||||
relogin_pgy,
|
||||
relogin_runtime_paths,
|
||||
relogin_xiaohongshu,
|
||||
relogin_xingtu,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
|
||||
CONSUMER_BINDING_IDS,
|
||||
child_browser_environment,
|
||||
resolve_relogin_runtime_paths,
|
||||
sync_relogin_state_to_consumers,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def test_relogin_paths_prefer_injected_runtime_binding(tmp_path: Path) -> None:
|
||||
cookie = tmp_path / "binding" / "cookies.json"
|
||||
storage = tmp_path / "binding" / "storage_state.json"
|
||||
profile = tmp_path / "binding" / "profile"
|
||||
|
||||
resolved = resolve_relogin_runtime_paths(
|
||||
"pgy",
|
||||
{
|
||||
"GYXX_BROWSER_COOKIE_FILE": str(cookie),
|
||||
"GYXX_BROWSER_STORAGE_STATE_FILE": str(storage),
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(profile),
|
||||
},
|
||||
)
|
||||
|
||||
assert resolved.cookie_file == cookie.resolve()
|
||||
assert resolved.storage_state_file == storage.resolve()
|
||||
assert resolved.profile_dir == profile.resolve()
|
||||
|
||||
|
||||
def test_relogin_paths_keep_legacy_standalone_layout() -> None:
|
||||
resolved = resolve_relogin_runtime_paths("xiaohongshu", {})
|
||||
|
||||
assert resolved.cookie_file == (
|
||||
PATHS.state_root / "cookies/xiaohongshu_cookies.json"
|
||||
).resolve()
|
||||
assert resolved.storage_state_file == (
|
||||
PATHS.state_root / "cookies/xiaohongshu_storage_state.json"
|
||||
).resolve()
|
||||
assert resolved.profile_dir == (
|
||||
PATHS.state_root / "browser-profiles/xiaohongshu"
|
||||
).resolve()
|
||||
|
||||
|
||||
def test_child_login_environment_preserves_wrapper_binding(tmp_path: Path) -> None:
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
|
||||
environment = child_browser_environment(
|
||||
cookie_file=cookie,
|
||||
storage_state_file=storage,
|
||||
profile_dir=profile,
|
||||
environment={"GYXX_MODULE_ID": "content_marketing", "KEEP": "1"},
|
||||
)
|
||||
|
||||
assert environment["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
|
||||
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(
|
||||
storage.resolve()
|
||||
)
|
||||
assert environment["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
|
||||
assert "GYXX_MODULE_ID" not in environment
|
||||
assert environment["KEEP"] == "1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "login_cookie", "argv"),
|
||||
[
|
||||
(
|
||||
relogin_bilibili,
|
||||
"bilibili",
|
||||
"SESSDATA",
|
||||
["relogin_bilibili.py"],
|
||||
),
|
||||
(relogin_douyin, "douyin", "sessionid", ["relogin_douyin.py"]),
|
||||
(relogin_pgy, "pgy", "web_session", ["relogin_pgy.py"]),
|
||||
(
|
||||
relogin_xiaohongshu,
|
||||
"xiaohongshu",
|
||||
"web_session",
|
||||
["relogin_xiaohongshu.py"],
|
||||
),
|
||||
(
|
||||
relogin_xingtu,
|
||||
"xingtu",
|
||||
"sessionid",
|
||||
["relogin_xingtu.py", "--force"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_relogin_wrapper_validates_the_same_binding_written_by_child(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
module: ModuleType,
|
||||
platform: str,
|
||||
login_cookie: str,
|
||||
argv: list[str],
|
||||
) -> None:
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
monkeypatch.setattr(module.sys, "argv", argv)
|
||||
monkeypatch.setenv("GYXX_MODULE_ID", "content_marketing")
|
||||
if module is relogin_xingtu:
|
||||
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
|
||||
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
|
||||
synced: list[tuple[str, Path, Path]] = []
|
||||
|
||||
def fake_sync(
|
||||
synced_platform: str,
|
||||
*,
|
||||
cookie_file: Path,
|
||||
storage_state_file: Path,
|
||||
) -> tuple[Path, ...]:
|
||||
synced.append(
|
||||
(synced_platform, cookie_file.resolve(), storage_state_file.resolve())
|
||||
)
|
||||
return ()
|
||||
|
||||
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fake_sync)
|
||||
|
||||
def fake_call(
|
||||
_command: list[str],
|
||||
*,
|
||||
cwd: str,
|
||||
env: dict[str, str],
|
||||
) -> int:
|
||||
assert cwd == str(module.PROJECT_DIR)
|
||||
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
|
||||
assert env["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(storage.resolve())
|
||||
assert env["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
|
||||
assert "GYXX_MODULE_ID" not in env
|
||||
cookie.write_text(
|
||||
json.dumps([{"name": login_cookie, "value": "fresh"}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
storage.write_text("{}", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(module.subprocess, "call", fake_call)
|
||||
|
||||
assert module.main() == 0
|
||||
assert cookie.is_file()
|
||||
assert storage.is_file()
|
||||
assert profile.is_dir()
|
||||
assert synced == [(platform, cookie.resolve(), storage.resolve())]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "login_cookie", "cookie_domain", "argv"),
|
||||
[
|
||||
(
|
||||
relogin_bilibili,
|
||||
"bilibili",
|
||||
"SESSDATA",
|
||||
".bilibili.com",
|
||||
["relogin_bilibili.py"],
|
||||
),
|
||||
(
|
||||
relogin_douyin,
|
||||
"douyin",
|
||||
"sessionid",
|
||||
".douyin.com",
|
||||
["relogin_douyin.py"],
|
||||
),
|
||||
(
|
||||
relogin_pgy,
|
||||
"pgy",
|
||||
"web_session",
|
||||
".xiaohongshu.com",
|
||||
["relogin_pgy.py"],
|
||||
),
|
||||
(
|
||||
relogin_xiaohongshu,
|
||||
"xiaohongshu",
|
||||
"web_session",
|
||||
".xiaohongshu.com",
|
||||
["relogin_xiaohongshu.py"],
|
||||
),
|
||||
(
|
||||
relogin_xingtu,
|
||||
"xingtu",
|
||||
"sessionid",
|
||||
".xingtu.cn",
|
||||
["relogin_xingtu.py", "--force"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_relogin_wrapper_restores_previous_state_when_publish_fails(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
module: ModuleType,
|
||||
platform: str,
|
||||
login_cookie: str,
|
||||
cookie_domain: str,
|
||||
argv: list[str],
|
||||
) -> None:
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
old_cookie = '[{"name":"old","value":"old-session"}]'
|
||||
old_storage = '{"cookies":[{"name":"old"}]}'
|
||||
cookie.write_text(old_cookie, encoding="utf-8")
|
||||
storage.write_text(old_storage, encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old-state.txt").write_text("old-profile", encoding="utf-8")
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
monkeypatch.setattr(module.sys, "argv", argv)
|
||||
if module is relogin_xingtu:
|
||||
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
|
||||
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
|
||||
|
||||
def fake_call(
|
||||
_command: list[str],
|
||||
*,
|
||||
cwd: str,
|
||||
env: dict[str, str],
|
||||
) -> int:
|
||||
assert cwd == str(module.PROJECT_DIR)
|
||||
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
|
||||
cookie.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"name": login_cookie,
|
||||
"value": "fresh-session",
|
||||
"domain": cookie_domain,
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
|
||||
profile.mkdir()
|
||||
return 0
|
||||
|
||||
def fail_publish(
|
||||
synced_platform: str,
|
||||
*,
|
||||
cookie_file: Path,
|
||||
storage_state_file: Path,
|
||||
) -> tuple[Path, ...]:
|
||||
assert synced_platform == platform
|
||||
assert cookie_file == cookie
|
||||
assert storage_state_file == storage
|
||||
raise OSError("simulated consumer publish failure")
|
||||
|
||||
monkeypatch.setattr(module.subprocess, "call", fake_call)
|
||||
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fail_publish)
|
||||
|
||||
assert module.main() == 1
|
||||
assert cookie.read_text(encoding="utf-8") == old_cookie
|
||||
assert storage.read_text(encoding="utf-8") == old_storage
|
||||
assert (profile / "old-state.txt").read_text(encoding="utf-8") == (
|
||||
"old-profile"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "cookie_name", "cookie_domain"),
|
||||
[
|
||||
("bilibili", "SESSDATA", ".bilibili.com"),
|
||||
("douyin", "sessionid", ".douyin.com"),
|
||||
("pgy", "web_session", ".xiaohongshu.com"),
|
||||
("xiaohongshu", "web_session", ".xiaohongshu.com"),
|
||||
("xingtu", "sessionid", ".xingtu.cn"),
|
||||
],
|
||||
)
|
||||
def test_successful_relogin_atomically_updates_unique_consumer_state(
|
||||
tmp_path: Path,
|
||||
platform: str,
|
||||
cookie_name: str,
|
||||
cookie_domain: str,
|
||||
) -> None:
|
||||
data_root = tmp_path / "data"
|
||||
source_cookie = tmp_path / "relogin" / "cookies.json"
|
||||
source_storage = tmp_path / "relogin" / "storage_state.json"
|
||||
source_cookie.parent.mkdir(parents=True)
|
||||
fresh_cookie = [
|
||||
{
|
||||
"name": cookie_name,
|
||||
"value": "fresh-session",
|
||||
"domain": cookie_domain,
|
||||
}
|
||||
]
|
||||
fresh_storage = {"cookies": fresh_cookie, "origins": []}
|
||||
source_cookie.write_text(json.dumps(fresh_cookie), encoding="utf-8")
|
||||
source_storage.write_text(json.dumps(fresh_storage), encoding="utf-8")
|
||||
|
||||
catalog = RuntimeIntegrationCatalog.load_default(
|
||||
project_root=PROJECT_ROOT,
|
||||
data_root=data_root,
|
||||
)
|
||||
consumer_id = CONSUMER_BINDING_IDS[platform][0]
|
||||
consumer = catalog.binding_for(consumer_id)
|
||||
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
consumer.cookie_file.write_text('[{"name":"old"}]', encoding="utf-8")
|
||||
consumer.storage_state_file.write_text(
|
||||
'{"cookies":[{"name":"old"}]}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
destinations = sync_relogin_state_to_consumers(
|
||||
platform,
|
||||
cookie_file=source_cookie,
|
||||
storage_state_file=source_storage,
|
||||
environment={
|
||||
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
|
||||
"GYXX_DATA_ROOT": str(data_root),
|
||||
},
|
||||
)
|
||||
|
||||
assert source_cookie.resolve() != consumer.cookie_file.resolve()
|
||||
assert source_storage.resolve() != consumer.storage_state_file.resolve()
|
||||
assert destinations == (consumer.cookie_file.resolve(),)
|
||||
assert json.loads(consumer.cookie_file.read_text(encoding="utf-8")) == (
|
||||
fresh_cookie
|
||||
)
|
||||
assert json.loads(
|
||||
consumer.storage_state_file.read_text(encoding="utf-8")
|
||||
) == fresh_storage
|
||||
|
||||
|
||||
def test_consumer_state_is_rolled_back_if_atomic_publish_fails(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
data_root = tmp_path / "data"
|
||||
source_cookie = tmp_path / "relogin" / "cookies.json"
|
||||
source_storage = tmp_path / "relogin" / "storage_state.json"
|
||||
source_cookie.parent.mkdir(parents=True)
|
||||
source_cookie.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"name": "SESSDATA",
|
||||
"value": "fresh-session",
|
||||
"domain": ".bilibili.com",
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
source_storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
|
||||
|
||||
catalog = RuntimeIntegrationCatalog.load_default(
|
||||
project_root=PROJECT_ROOT,
|
||||
data_root=data_root,
|
||||
)
|
||||
consumer = catalog.binding_for(CONSUMER_BINDING_IDS["bilibili"][0])
|
||||
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
old_cookie = '[{"name":"SESSDATA","value":"old"}]'
|
||||
old_storage = '{"cookies":[{"name":"old"}]}'
|
||||
consumer.cookie_file.write_text(old_cookie, encoding="utf-8")
|
||||
consumer.storage_state_file.write_text(old_storage, encoding="utf-8")
|
||||
|
||||
original_replace = relogin_runtime_paths.os.replace
|
||||
failed = False
|
||||
|
||||
def fail_second_replace(source: Path | str, target: Path | str) -> None:
|
||||
nonlocal failed
|
||||
source_path = Path(source)
|
||||
target_path = Path(target)
|
||||
if (
|
||||
not failed
|
||||
and source_path.suffix == ".tmp"
|
||||
and target_path == consumer.storage_state_file
|
||||
):
|
||||
failed = True
|
||||
raise OSError("simulated storage publish failure")
|
||||
original_replace(source, target)
|
||||
|
||||
monkeypatch.setattr(relogin_runtime_paths.os, "replace", fail_second_replace)
|
||||
|
||||
with pytest.raises(OSError, match="simulated storage publish failure"):
|
||||
sync_relogin_state_to_consumers(
|
||||
"bilibili",
|
||||
cookie_file=source_cookie,
|
||||
storage_state_file=source_storage,
|
||||
environment={
|
||||
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
|
||||
"GYXX_DATA_ROOT": str(data_root),
|
||||
},
|
||||
)
|
||||
|
||||
assert consumer.cookie_file.read_text(encoding="utf-8") == old_cookie
|
||||
assert consumer.storage_state_file.read_text(encoding="utf-8") == old_storage
|
||||
@@ -1,6 +1,7 @@
|
||||
import io
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.adapters import RuntimeServicePolicy
|
||||
from gyxx_flow.modules.content_marketing.data.tools import db
|
||||
@@ -60,7 +61,7 @@ class LocalDatabaseConfigTests(unittest.TestCase):
|
||||
paths = (
|
||||
ROOT / "data" / "tools" / "db.py",
|
||||
ROOT / "data" / "tools" / "sync_metrics_to_cmt_notes.py",
|
||||
ROOT / "data" / "tools" / "sync_cooperations.py",
|
||||
ROOT / "data" / "tools" / "sync_notes_master.py",
|
||||
ROOT / "data" / "tools" / "generate_creator_report.py",
|
||||
)
|
||||
for path in paths:
|
||||
@@ -79,6 +80,33 @@ class LocalDatabaseConfigTests(unittest.TestCase):
|
||||
|
||||
self.assertTrue(raw.getvalue())
|
||||
|
||||
def test_explicit_insert_missing_preserves_default_sync_contract(self):
|
||||
with (
|
||||
patch("sys.argv", ["sync_metrics_to_cmt_notes.py", "--insert-missing"]),
|
||||
patch.object(
|
||||
sync_metrics,
|
||||
"sync",
|
||||
return_value={"identity_ambiguous": 0},
|
||||
) as sync_call,
|
||||
):
|
||||
sync_metrics.main()
|
||||
|
||||
sync_call.assert_called_once_with(dry_run=False, insert_missing=True)
|
||||
|
||||
def test_default_sync_exits_nonzero_when_identity_is_ambiguous(self):
|
||||
with (
|
||||
patch("sys.argv", ["sync_metrics_to_cmt_notes.py"]),
|
||||
patch.object(
|
||||
sync_metrics,
|
||||
"sync",
|
||||
return_value={"identity_ambiguous": 2},
|
||||
),
|
||||
self.assertRaises(SystemExit) as exc_info,
|
||||
):
|
||||
sync_metrics.main()
|
||||
|
||||
self.assertEqual(exc_info.exception.code, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -39,6 +40,54 @@ def test_success_status_cannot_hide_write_failure_and_blocked_input_is_terminal(
|
||||
}],
|
||||
}
|
||||
assert retry.extract_failed_records(blocked, "bili") == []
|
||||
assert retry.requested_records_succeeded(blocked, ["b"]) == (True, [])
|
||||
|
||||
|
||||
def test_requested_records_require_exactly_one_resolved_blocked_terminal():
|
||||
duplicate_blocked = {
|
||||
"results": [
|
||||
{
|
||||
"record_id": "b",
|
||||
"status": "blocked_input",
|
||||
"matched": False,
|
||||
"reason": "url_missing",
|
||||
},
|
||||
{
|
||||
"record_id": "b",
|
||||
"status": "blocked_input",
|
||||
"matched": False,
|
||||
"reason": "url_missing",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
assert retry.requested_records_succeeded(duplicate_blocked, ["b"]) == (
|
||||
False,
|
||||
["b"],
|
||||
)
|
||||
|
||||
|
||||
def test_blocked_input_cannot_hide_explicit_execution_or_write_failure():
|
||||
for explicit_failure in (
|
||||
{"write_ok": False},
|
||||
{"write_error": "database rejected update"},
|
||||
{"error": "browser session crashed"},
|
||||
):
|
||||
blocked = {
|
||||
"results": [{
|
||||
"record_id": "b",
|
||||
"status": "blocked_input",
|
||||
"matched": False,
|
||||
"reason": "url_missing",
|
||||
**explicit_failure,
|
||||
}],
|
||||
}
|
||||
|
||||
assert retry.requested_records_succeeded(blocked, ["b"]) == (
|
||||
False,
|
||||
["b"],
|
||||
)
|
||||
assert retry.extract_failed_records(blocked, "bili") == ["b"]
|
||||
|
||||
|
||||
def test_canonical_filename_requires_current_style_name():
|
||||
@@ -305,6 +354,87 @@ def test_validate_summary_rejects_expected_style_left_over_from_old_run(tmp_path
|
||||
assert "current run" in reason
|
||||
|
||||
|
||||
def test_build_retry_targets_narrows_retryable_rows_and_expands_structural_gaps():
|
||||
targets = run_all.build_retry_targets(
|
||||
{
|
||||
"bili": [{
|
||||
"index": 1,
|
||||
"total_b_records": 2,
|
||||
"details": [
|
||||
{"record_id": "ok", "status": "success", "ok": True},
|
||||
{"record_id": "bad", "status": "retryable_failure", "ok": False},
|
||||
],
|
||||
}],
|
||||
"pgy": [{
|
||||
"index": 1,
|
||||
"total": 2,
|
||||
"results": [{"record_id": "only-one", "status": "success"}],
|
||||
}],
|
||||
"xt": [{
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"results": [{
|
||||
"record_id": "terminal",
|
||||
"status": "blocked_input",
|
||||
"matched": False,
|
||||
}],
|
||||
}],
|
||||
},
|
||||
{1},
|
||||
)
|
||||
|
||||
assert targets == {
|
||||
"bili": {1: {"bad"}},
|
||||
"pgy": {1: None},
|
||||
}
|
||||
|
||||
|
||||
def test_retry_manifest_round_trips_record_and_full_style_scopes(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(run_all, "RETRY_MANIFEST_DIR", tmp_path)
|
||||
targets = {"bili": {1: {"r2", "r1"}}, "pgy": {3: None}}
|
||||
|
||||
run_all._write_retry_manifest(
|
||||
date(2026, 9, 2), targets, source_slot="01:00"
|
||||
)
|
||||
|
||||
assert run_all._load_retry_manifest(date(2026, 9, 2)) == targets
|
||||
|
||||
|
||||
def test_retry_process_command_passes_only_target_records():
|
||||
args = SimpleNamespace(
|
||||
dry_run=False,
|
||||
style=[1, 2],
|
||||
daily_scope=True,
|
||||
no_retry=False,
|
||||
)
|
||||
|
||||
command = run_all.build_process_command(
|
||||
"pgy", args, retry_targets={2: {"record-b", "record-a"}}
|
||||
)
|
||||
|
||||
assert command[command.index("--style") + 1] == "2"
|
||||
assert command.count("--style") == 1
|
||||
assert command[command.index("--record") + 1] == "record-a"
|
||||
assert command.count("--record") == 2
|
||||
assert "--no-retry" in command
|
||||
|
||||
|
||||
def test_retry_process_command_uses_full_style_when_scope_is_structurally_incomplete():
|
||||
args = SimpleNamespace(
|
||||
dry_run=False,
|
||||
style=[1, 2],
|
||||
daily_scope=True,
|
||||
no_retry=False,
|
||||
)
|
||||
|
||||
command = run_all.build_process_command(
|
||||
"xt", args, retry_targets={1: None}
|
||||
)
|
||||
|
||||
assert command[command.index("--style") + 1] == "1"
|
||||
assert "--record" not in command
|
||||
|
||||
|
||||
def test_aggregate_counts_all_unresolved_rows_as_errors():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": [{
|
||||
@@ -363,6 +493,40 @@ def test_aggregate_keeps_partial_counts_when_validation_marks_payload_invalid():
|
||||
assert platform["validation_error"] == "invalid summary: unresolved"
|
||||
|
||||
|
||||
def test_aggregate_fails_closed_for_stale_payload():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": {
|
||||
"error": "invalid summary: summary file is stale",
|
||||
"payload": [{
|
||||
"index": 1,
|
||||
"total": 141,
|
||||
"matched": 101,
|
||||
"results": [],
|
||||
}],
|
||||
},
|
||||
})
|
||||
|
||||
platform = aggregate["platforms"]["pgy"]
|
||||
assert platform["loaded"] is False
|
||||
assert platform["error"] == "invalid summary: summary file is stale"
|
||||
assert aggregate["totals"]["tasks"] == 0
|
||||
|
||||
|
||||
def test_aggregate_filters_merged_history_to_current_styles():
|
||||
aggregate = run_all.aggregate(
|
||||
{
|
||||
"xt": [
|
||||
{"index": 1, "total": 2, "matched": 1, "results": []},
|
||||
{"index": 2, "total": 99, "matched": 99, "results": []},
|
||||
],
|
||||
},
|
||||
expected_indices={1},
|
||||
)
|
||||
|
||||
assert aggregate["platforms"]["xt"]["styles"] == 1
|
||||
assert aggregate["platforms"]["xt"]["tasks"] == 2
|
||||
|
||||
|
||||
def test_run_round_marks_exit_zero_platform_failed_when_summary_is_invalid(monkeypatch, tmp_path):
|
||||
args = SimpleNamespace(dry_run=True, style=[1])
|
||||
stdout = tmp_path / "stdout.log"
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.modules.content_marketing import weekly_summary_all
|
||||
|
||||
|
||||
def test_target_table_payload_uses_temp_cwd_and_relative_reference(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
payload_root = tmp_path / "payloads"
|
||||
payload_root.mkdir()
|
||||
working_directory = tmp_path / "working-directory"
|
||||
working_directory.mkdir()
|
||||
monkeypatch.chdir(working_directory)
|
||||
monkeypatch.setattr(
|
||||
weekly_summary_all,
|
||||
"PATHS",
|
||||
SimpleNamespace(tmp_root=payload_root),
|
||||
)
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
|
||||
calls: list[tuple[list[str], Path | None]] = []
|
||||
|
||||
def fake_call_lark_json(
|
||||
args: list[str], *, cwd: Path | None = None
|
||||
) -> dict[str, object]:
|
||||
calls.append((args, cwd))
|
||||
if args[1] == "+record-list":
|
||||
assert cwd is None
|
||||
return {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"record_id_list": [],
|
||||
"data": [],
|
||||
"field_id_list": [],
|
||||
},
|
||||
}
|
||||
assert cwd == payload_root
|
||||
assert cwd != Path.cwd()
|
||||
payload_reference = args[args.index("--json") + 1]
|
||||
assert payload_reference.startswith("@./")
|
||||
payload_path = cwd / payload_reference.removeprefix("@./")
|
||||
assert not Path(payload_reference[1:]).is_absolute()
|
||||
assert json.loads(payload_path.read_text(encoding="utf-8")) == {
|
||||
"fld_time": "2026-07-01~07-31",
|
||||
"fld_summary": (
|
||||
"[凌云air 月度汇总(2026-07-01~07-31)]"
|
||||
"(https://example.test/doc)"
|
||||
),
|
||||
}
|
||||
return {"ok": True}
|
||||
|
||||
monkeypatch.setattr(weekly_summary_all, "call_lark_json", fake_call_lark_json)
|
||||
|
||||
assert weekly_summary_all.write_to_target_table(
|
||||
"base-test",
|
||||
"table-test",
|
||||
"fld_time",
|
||||
"fld_summary",
|
||||
"凌云air",
|
||||
"https://example.test/doc",
|
||||
False,
|
||||
time_label="2026-07-01~07-31",
|
||||
report_label="月度汇总",
|
||||
)
|
||||
assert [command[1] for command, _cwd in calls] == [
|
||||
"+record-list",
|
||||
"+record-upsert",
|
||||
]
|
||||
|
||||
|
||||
def test_call_lark_json_logs_only_redacted_failure_detail(
|
||||
monkeypatch,
|
||||
capsys,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
raw_secret = "sensitive-token-value"
|
||||
raw_open_id = "sensitive-open-id"
|
||||
captured_kwargs: dict[str, object] = {}
|
||||
|
||||
def fake_run(*_args, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return SimpleNamespace(
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr=(
|
||||
f"request failed; base_token={raw_secret}; "
|
||||
f"open_id={raw_open_id}; https://example.test/private"
|
||||
),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(weekly_summary_all.subprocess, "run", fake_run)
|
||||
|
||||
response = weekly_summary_all.call_lark_json(
|
||||
["base", "+record-upsert"], cwd=tmp_path
|
||||
)
|
||||
|
||||
captured = capsys.readouterr().out
|
||||
assert captured_kwargs["cwd"] == tmp_path
|
||||
assert response["ok"] is False
|
||||
assert raw_secret not in response["error"]
|
||||
assert raw_open_id not in response["error"]
|
||||
assert raw_secret not in captured
|
||||
assert raw_open_id not in captured
|
||||
assert "[REDACTED]" in captured
|
||||
assert "[URL]" in captured
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,32 @@ from gyxx_flow.modules.content_marketing import weekly_summary_all as weekly
|
||||
|
||||
|
||||
class WeeklyTmallPersonaTests(unittest.TestCase):
|
||||
def test_summary_result_requires_all_real_external_effects(self):
|
||||
self.assertTrue(
|
||||
weekly.summary_results_succeeded(
|
||||
[{"status": "ok"}, {"status": "no_notes"}]
|
||||
)
|
||||
)
|
||||
for status in (
|
||||
"no_fields",
|
||||
"no_reports",
|
||||
"llm_failed",
|
||||
"doc_failed",
|
||||
"doc_ambiguous",
|
||||
"db_failed",
|
||||
"write_failed",
|
||||
"exception",
|
||||
):
|
||||
with self.subTest(status=status):
|
||||
self.assertFalse(
|
||||
weekly.summary_results_succeeded([{"status": status}])
|
||||
)
|
||||
|
||||
def test_dry_run_status_is_only_successful_in_dry_run_mode(self):
|
||||
results = [{"status": "dry_run"}]
|
||||
self.assertFalse(weekly.summary_results_succeeded(results))
|
||||
self.assertTrue(weekly.summary_results_succeeded(results, dry_run=True))
|
||||
|
||||
def test_missing_persona_forbids_invented_percentages(self):
|
||||
text = weekly.format_tmall_persona(None)
|
||||
self.assertIn("不得虚构具体比例", text)
|
||||
@@ -56,7 +82,7 @@ class WeeklyTmallPersonaTests(unittest.TestCase):
|
||||
}]
|
||||
|
||||
with patch.object(weekly, "get_latest_tmall_persona", return_value=persona) as get_persona, \
|
||||
patch.object(weekly, "call_hermes_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
|
||||
patch.object(weekly, "call_content_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
|
||||
weekly.generate_summary("星云2", notes)
|
||||
|
||||
get_persona.assert_called_once_with("星云2")
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import xiaohongshu_comment_scraper as scraper
|
||||
|
||||
|
||||
class _EvaluatePage:
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
self.calls = []
|
||||
|
||||
def evaluate(self, script, argument):
|
||||
self.calls.append((script, argument))
|
||||
return self.result
|
||||
|
||||
|
||||
def test_bounded_fetch_uses_abort_controller_and_clears_timer(monkeypatch):
|
||||
page = _EvaluatePage(
|
||||
{"__gyxx_status": "ok", "payload": {"comments": []}}
|
||||
)
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
|
||||
|
||||
payload = scraper._fetch_json_with_timeout(
|
||||
page,
|
||||
"https://example.invalid/comments",
|
||||
deadline=1000.0,
|
||||
stage="top-level pagination",
|
||||
)
|
||||
|
||||
assert payload == {"comments": []}
|
||||
script, argument = page.calls[0]
|
||||
assert "AbortController" in script
|
||||
assert "clearTimeout(timer)" in script
|
||||
assert argument["timeoutMs"] == 30_000
|
||||
|
||||
|
||||
def test_bounded_fetch_timeout_is_an_explicit_transient_failure(monkeypatch):
|
||||
page = _EvaluatePage({"__gyxx_status": "timeout"})
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
|
||||
|
||||
with pytest.raises(scraper.TransientXiaohongshuCommentTimeout) as caught:
|
||||
scraper._fetch_json_with_timeout(
|
||||
page,
|
||||
"https://example.invalid/comments",
|
||||
deadline=1000.0,
|
||||
stage="reply pagination",
|
||||
)
|
||||
|
||||
assert isinstance(caught.value, ConnectionError)
|
||||
assert "reply pagination" in str(caught.value)
|
||||
|
||||
|
||||
def test_scroll_checks_the_note_deadline_before_browser_work(monkeypatch):
|
||||
page = _EvaluatePage(None)
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 901.0)
|
||||
|
||||
with pytest.raises(scraper.TransientXiaohongshuCommentTimeout):
|
||||
scraper.scroll_comments(
|
||||
page,
|
||||
max_scrolls=30,
|
||||
idle_rounds=4,
|
||||
comments_by_id={},
|
||||
deadline=900.0,
|
||||
)
|
||||
|
||||
assert page.calls == []
|
||||
|
||||
|
||||
def test_top_and_reply_pagination_share_the_bounded_fetch(monkeypatch):
|
||||
stages = []
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 0.0)
|
||||
|
||||
def fetch(_page, _url, *, deadline, stage):
|
||||
assert deadline == 900.0
|
||||
stages.append(stage)
|
||||
return {"comments": [], "has_more": 0, "cursor": 0}
|
||||
|
||||
monkeypatch.setattr(scraper, "_fetch_json_with_timeout", fetch)
|
||||
monkeypatch.setattr(scraper, "collect_from_payload", lambda *_args: None)
|
||||
page = _EvaluatePage(None)
|
||||
|
||||
scraper.fetch_missing_top_comments(
|
||||
page,
|
||||
"source-url",
|
||||
{},
|
||||
{
|
||||
"comment_url_template": "https://example.invalid/top",
|
||||
"api_total": 2,
|
||||
"last_cursor": 1,
|
||||
},
|
||||
deadline=900.0,
|
||||
)
|
||||
scraper.fetch_missing_replies(
|
||||
page,
|
||||
"source-url",
|
||||
{
|
||||
"parent": {
|
||||
"level": "comment",
|
||||
"reply_count": 1,
|
||||
"comment_id": "parent-id",
|
||||
}
|
||||
},
|
||||
{
|
||||
"sub_url_template": "https://example.invalid/reply",
|
||||
},
|
||||
deadline=900.0,
|
||||
)
|
||||
|
||||
assert stages == ["top-level pagination", "reply pagination"]
|
||||
Reference in New Issue
Block a user