feat: expand module workflows, dynamic config, notifications and console API

This commit is contained in:
2026-09-04 09:49:37 +08:00
parent 8df5266abb
commit b124d757b0
309 changed files with 89358 additions and 6232 deletions
@@ -0,0 +1,339 @@
import json
from types import SimpleNamespace
import pytest
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
from gyxx_flow.modules.content_marketing import scrapling_http
from gyxx_flow.modules.content_marketing import self_bilibili_scraper as self_bili
class FakeResponse:
def __init__(
self,
*,
status: int = 200,
payload: dict | None = None,
text: str | None = None,
content_type: str = "application/json; charset=utf-8",
json_error: ValueError | None = None,
) -> None:
self.status_code = status
self._payload = payload
self.text = json.dumps(payload) if text is None and payload is not None else (text or "")
self.content = self.text.encode("utf-8")
self.headers = {"content-type": content_type}
self._json_error = json_error
def json(self):
if self._json_error is not None:
raise self._json_error
return self._payload
class SequenceSession:
def __init__(self, outcomes):
self.outcomes = list(outcomes)
self.calls = []
def get(self, url, **kwargs):
self.calls.append((url, kwargs))
outcome = self.outcomes.pop(0)
if isinstance(outcome, BaseException):
raise outcome
return outcome
def test_scrapling_http_session_maps_fetcher_response(monkeypatch):
calls = []
def get(url, **kwargs):
calls.append((url, kwargs))
return SimpleNamespace(
status=200,
headers={"content-type": "application/json"},
url="https://www.bilibili.com/video/BV17NNw6jEPx",
body=b'{"code": 0}',
encoding="utf-8",
)
monkeypatch.setattr(scrapling_http.Fetcher, "get", get)
session = scrapling_http.ScraplingHttpSession(headers={"User-Agent": "gyxx"})
response = session.get(
"https://b23.tv/demo",
allow_redirects=True,
timeout=10,
)
assert calls == [(
"https://b23.tv/demo",
{
"headers": {"User-Agent": "gyxx"},
"follow_redirects": "all",
"retries": 1,
"timeout": 10,
},
)]
assert response.status_code == 200
assert response.json() == {"code": 0}
assert response.url.endswith("BV17NNw6jEPx")
def test_self_bilibili_uses_scrapling_user_agent_instead_of_stale_override():
session = SequenceSession([
FakeResponse(
payload={
"code": 0,
"data": {"stat": {"view": 456}, "pubdate": 1_700_000_000},
}
)
])
result = self_bili.fetch_bili_stat(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result == (456, 1_700_000_000)
assert session.calls == [(
"https://api.bilibili.com/x/web-interface/view",
{
"params": {"bvid": "BV17NNw6jEPx"},
"timeout": 10,
"headers": {"Referer": "https://www.bilibili.com/"},
},
)]
def success_response(view: int = 321) -> FakeResponse:
return FakeResponse(payload={"code": 0, "message": "OK", "data": {"stat": {"view": view}}})
def detail_success_response(view: int = 321) -> FakeResponse:
return FakeResponse(
payload={"code": 0, "message": "OK", "data": {"View": {"stat": {"view": view}}}}
)
def test_empty_body_retries_then_returns_play_count(monkeypatch, capsys):
session = SequenceSession([FakeResponse(text=""), success_response(456)])
sleeps = []
monkeypatch.setattr(bili.time, "sleep", sleeps.append)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result == 456
assert len(session.calls) == 2
assert sleeps == [1.5]
assert "[empty_body]" in capsys.readouterr().out
@pytest.mark.parametrize(
("first_response", "category"),
[
(FakeResponse(status=412, text="request blocked", content_type="text/plain"), "http_412"),
(FakeResponse(payload={"code": -509, "message": "request limit"}), "api_code_-509"),
(
FakeResponse(
text="<html>temporary gateway page</html>",
content_type="text/html",
json_error=ValueError("not json"),
),
"invalid_json",
),
],
)
def test_transient_response_classes_are_retried(
monkeypatch, capsys, first_response, category
):
session = SequenceSession([first_response, success_response()])
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result == 321
assert len(session.calls) == 2
assert f"[{category}]" in capsys.readouterr().out
def test_non_retryable_api_code_stops_after_one_attempt(monkeypatch, capsys):
session = SequenceSession(
[FakeResponse(payload={"code": -404, "message": "video not found"})]
)
monkeypatch.setattr(
bili.time,
"sleep",
lambda _seconds: pytest.fail("terminal API result must not back off"),
)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result is None
assert len(session.calls) == 1
assert "[api_code_-404]" in capsys.readouterr().out
def test_transient_failures_exhaust_three_attempts_honestly(monkeypatch, capsys):
session = SequenceSession(
[
*[FakeResponse(text="") for _ in range(3)],
FakeResponse(status=412, text="fallback blocked", content_type="text/plain"),
]
)
sleeps = []
monkeypatch.setattr(bili.time, "sleep", sleeps.append)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result is None
assert len(session.calls) == 4
assert sleeps == [1.5, 3.0]
output = capsys.readouterr().out
assert "attempt=3/3" in output
assert "[view_detail_http_412]" in output
def test_view_detail_is_used_once_after_primary_transient_exhaustion(
monkeypatch, capsys
):
session = SequenceSession(
[*[FakeResponse(text="") for _ in range(3)], detail_success_response(789)]
)
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
result = bili.fetch_play_count(
"https://www.bilibili.com/video/BV17NNw6jEPx",
session,
)
assert result == 789
assert len(session.calls) == 4
assert session.calls[-1][0] == bili._VIEW_DETAIL_API_URL
assert "[FALLBACK]" in capsys.readouterr().out
def test_play_count_failure_remains_an_incomplete_summary(monkeypatch):
style = {
"style": "款式A",
"name": "款式A",
"index": 1,
"base_token": "base",
"table_id": "table",
"field_map": {
"platform": {"field_id": "platform"},
"note_url": {"field_id": "url"},
"creator_name": {"field_id": "creator"},
"publish_time": {"field_id": "pub"},
"read_count_7d": {"field_id": "s7"},
"read_count_14d": {"field_id": "s14"},
"read_count_21d": {"field_id": "s21"},
"read_count_28d": {"field_id": "s28"},
"month_end": {"field_id": "sm"},
},
}
monkeypatch.setattr(
bili,
"list_all_records",
lambda *_args: [{
"record_id": "failed",
"platform": "B站",
"creator": "达人",
"url": "https://www.bilibili.com/video/BV17NNw6jEPx",
"pub": "2026-08-08",
}],
)
monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: None)
monkeypatch.setattr(
bili,
"write_record",
lambda *_args, **_kwargs: pytest.fail("failed fetch must not write"),
)
summary = bili.process_style(
style,
only_record_ids=None,
dry_run=True,
delay=0,
session=object(),
state={},
force_today=bili.date(2026, 8, 9),
first_run=False,
)
assert summary["complete"] is False
assert summary["retryable_failures"] == 1
assert summary["details"][0]["reason"] == "play_count_fetch_failed"
def test_terminal_unavailable_detail_is_a_complete_blocked_input(monkeypatch):
style = {
"style": "款式A",
"name": "款式A",
"index": 1,
"base_token": "base",
"table_id": "table",
"field_map": {
"platform": {"field_id": "platform"},
"note_url": {"field_id": "url"},
"creator_name": {"field_id": "creator"},
"publish_time": {"field_id": "pub"},
"read_count_7d": {"field_id": "s7"},
"read_count_14d": {"field_id": "s14"},
"read_count_21d": {"field_id": "s21"},
"read_count_28d": {"field_id": "s28"},
"month_end": {"field_id": "sm"},
},
}
monkeypatch.setattr(
bili,
"list_all_records",
lambda *_args: [{
"record_id": "unavailable",
"platform": "B站",
"creator": "达人",
"url": "https://www.bilibili.com/video/BV17NNw6jEPx",
"pub": "2026-08-08",
}],
)
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(
bili,
"write_record",
lambda *_args, **_kwargs: pytest.fail("unavailable video must not write"),
)
session = SequenceSession([
*[FakeResponse(status=412, text="request blocked") for _ in range(3)],
FakeResponse(payload={"code": 62002, "message": "稿件不可见", "data": None}),
])
summary = bili.process_style(
style,
only_record_ids=None,
dry_run=True,
delay=0,
session=session,
state={},
force_today=bili.date(2026, 8, 9),
first_run=False,
)
assert len(session.calls) == 4
assert summary["complete"] is True
assert summary["blocked_input"] == 1
assert summary["retryable_failures"] == 0
assert summary["unresolved"] == 0
assert summary["details"][0]["status"] == "blocked_input"
assert summary["details"][0]["reason"] == "video_unavailable"
@@ -0,0 +1,346 @@
from __future__ import annotations
import datetime as dt
from argparse import Namespace
from pathlib import Path
import pytest
from gyxx_flow.modules.content_marketing import run_all
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
from gyxx_flow.modules.content_marketing.data.tools import retry_failed
from gyxx_flow.modules.content_marketing.data.tools import (
sync_metrics_to_cmt_notes as sync_metrics,
)
@pytest.fixture(autouse=True)
def reset_content_source():
xingtu.configure_source("xingtu")
yield
xingtu.configure_source("xingtu")
def test_buyin_source_uses_doudian_market_and_cookie_scope():
xingtu.configure_source("buyin")
assert xingtu.CONTENT_SOURCE == "buyin"
assert xingtu.MARKET_URL.startswith(
"https://buyin.jinritemai.com/dashboard/servicehall/daren-square"
)
assert xingtu.DETAIL_URL_MARK == "/dashboard/servicehall/daren-profile"
assert xingtu.BUSINESS_COOKIE_DOMAINS == ("jinritemai.com",)
def test_save_state_creates_cookie_parent_directory(tmp_path, monkeypatch):
cookie_file = tmp_path / "nested" / "cookies.json"
monkeypatch.setattr(xingtu, "DATA_DIR", tmp_path / "raw")
monkeypatch.setattr(xingtu, "COOKIE_FILE", cookie_file)
class Context:
def cookies(self):
return [{"name": "sessionid", "value": "redacted"}]
class Page:
context = Context()
xingtu.save_state(Page())
assert cookie_file.exists()
assert "sessionid" in cookie_file.read_text(encoding="utf-8")
def test_run_all_keeps_cooperation_xt_on_xingtu_for_buyin_fallback():
args = Namespace(
dry_run=False,
style=None,
daily_scope=False,
no_retry=False,
)
command = run_all.build_process_command("xt", args)
assert "--buyin" not in command
assert "--skip-field-prepare" in command
def test_retry_failed_keeps_cooperation_xt_on_xingtu_for_buyin_fallback():
assert retry_failed.SCRIPTS["xt"]["extra_args"] == []
def test_buyin_video_parser_is_dom_html_based_and_maps_viewers_to_exposure():
xingtu.configure_source("buyin")
class Page:
def __init__(self):
self.script = ""
def evaluate(self, script):
self.script = script
return [{
"title": "#达人笔记标题",
"play_count": "1,155",
"view_count": "1,155",
"exposure_count": "1,155",
}]
page = Page()
cards = xingtu.parse_videos_on_page(page)
assert cards[0]["play_count"] == "1,155"
assert cards[0]["exposure_count"] == "1,155"
assert "观看人数" in page.script
assert "innerText" in page.script
def test_buyin_history_tab_and_trade_video_filter_are_selected():
xingtu.configure_source("buyin")
events = []
filter_states = {"只看带货视频": True, "只看带货图文": True}
class Locator:
first = None
def __init__(self, name):
self.name = name
self.first = self
def count(self):
return 1
def filter(self, **kwargs):
return Locator(kwargs.get("has_text", self.name))
def click(self, **_kwargs):
events.append(f"click-{self.name}")
if self.name in filter_states:
filter_states[self.name] = False
class Page:
def get_by_text(self, text, **_kwargs):
return Locator(text)
def get_by_role(self, _role, name, **_kwargs):
return Locator(name)
def locator(self, _selector):
return Locator("selector")
def evaluate(self, script, *_args):
events.append(script)
if _args and _args[0] in filter_states:
return filter_states[_args[0]]
return False
def wait_for_timeout(self, milliseconds):
events.append(milliseconds)
assert xingtu.open_creation_ability(Page()) is True
assert "click-历史内容" in events
assert "click-近30天" in events
assert "click-视频" in events
assert "click-只看带货视频" in events
assert "click-只看带货图文" in events
assert any("scrollIntoView" in event for event in events if isinstance(event, str))
def test_buyin_creator_search_clicks_real_button_and_waits_for_changed_results(
monkeypatch,
):
xingtu.configure_source("buyin")
events = []
class Locator:
def __init__(self, kind):
self.kind = kind
self.first = self
def count(self):
return 1
def wait_for(self, **_kwargs):
return None
def click(self, **_kwargs):
events.append(f"click-{self.kind}")
def fill(self, value):
events.append(f"fill-{value}")
def press(self, key):
events.append(f"press-{key}")
class Page:
def locator(self, selector):
if selector.startswith("button.auxo-input-search-button"):
return Locator("button")
return Locator("input")
def evaluate(self, _script):
return "idle"
def get_by_role(self, *_args, **_kwargs):
raise AssertionError("Buyin should use the real search button first")
def wait_for_timeout(self, _milliseconds):
return None
monkeypatch.setattr(xingtu, "dismiss_popups", lambda _page: None)
monkeypatch.setattr(xingtu, "select_nickname_search", lambda _page: None)
monkeypatch.setattr(xingtu, "_buyin_result_page_signature", lambda _page: "before")
observed = {}
def wait_for_results(_page, _name, timeout_ms, previous_signature=None):
observed.update(timeout_ms=timeout_ms, previous_signature=previous_signature)
monkeypatch.setattr(xingtu, "wait_for_visible_creator_result", wait_for_results)
xingtu.search_creator(Page(), "linyl燕", "LinFX2327")
assert events == ["click-input", "fill-LinFX2327", "click-button"]
assert observed == {"timeout_ms": 30000, "previous_signature": "before"}
def test_buyin_search_button_waits_until_loading_finishes():
xingtu.configure_source("buyin")
states = iter(["busy", "busy", "idle"])
waits = []
class Page:
def evaluate(self, _script):
return next(states)
def wait_for_timeout(self, milliseconds):
waits.append(milliseconds)
xingtu._wait_for_buyin_search_button_idle(Page(), timeout_ms=1000)
assert waits == [250, 250]
def test_buyin_wait_distinguishes_temporary_search_throttling():
xingtu.configure_source("buyin")
class Page:
def evaluate(self, *_args):
return "throttled"
def wait_for_timeout(self, _milliseconds):
pytest.fail("throttling must return immediately")
with pytest.raises(xingtu.SearchTemporarilyThrottled, match="过于频繁"):
xingtu.wait_for_visible_creator_result(Page(), "LinFX2327", timeout_ms=1000)
def test_buyin_creator_scan_supports_single_row_without_profile_href():
script = xingtu._buyin_scan_creator_script()
assert "tr[data-row-key], [role=\"row\"][data-row-key]" in script
assert "rowKey: row.getAttribute('data-row-key')" in script
assert "via: 'single-search-result'" in script
assert "first-id-search-result" not in script
def test_buyin_next_page_waits_for_a_changed_html_signature():
xingtu.configure_source("buyin")
signatures = iter(["1/4|first-card", "2/4|second-card"])
class Page:
def evaluate(self, script):
if "const selectors" in script:
return {"clicked": True, "picked": "semi-page-next"}
return next(signatures)
def locator(self, _selector):
class Locator:
first = None
def __init__(self):
self.first = self
def click(self, **_kwargs):
return None
return Locator()
def wait_for_timeout(self, _milliseconds):
return None
assert xingtu.go_next_page(Page()) is True
def test_metric_snapshot_date_prefers_orchestrator_business_date(monkeypatch):
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-31")
assert sync_metrics._resolve_metric_date() == dt.date(2026, 8, 31)
assert sync_metrics._resolve_metric_date("2026-08-30") == dt.date(2026, 8, 30)
with pytest.raises(ValueError, match="YYYY-MM-DD"):
sync_metrics._resolve_metric_date("not-a-date")
def test_metric_snapshot_upsert_is_idempotent_by_note_and_business_date():
class Cursor:
def __init__(self):
self.calls = []
def execute(self, sql, params):
self.calls.append((" ".join(sql.split()), params))
cursor = Cursor()
sync_metrics._upsert_metric_snapshot(
cursor,
42,
{
"view_count": 1155,
"dst_platform": "douyin",
"self_operated": False,
"record_id": "rec-1",
"source_url": "https://www.douyin.com/video/42",
},
dt.date(2026, 8, 31),
)
sql, params = cursor.calls[0]
assert "INSERT INTO cmt_note_metric_snapshots" in sql
assert "ON CONFLICT (note_id, metric_date)" in sql
assert params[0:2] == (42, dt.date(2026, 8, 31))
assert params[6] == 1155
def test_snapshot_schema_and_report_query_use_daily_history():
root = Path(__file__).resolve().parents[3]
schema = (
root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "content_marketing"
/ "data"
/ "tools"
/ "schema_gyxx_super_data.sql"
).read_text(encoding="utf-8")
migration = (
root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "content_marketing"
/ "data"
/ "tools"
/ "migrations"
/ "008_note_metric_snapshots.sql"
).read_text(encoding="utf-8")
report = (
root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "content_marketing"
/ "daily_marketing_report.py"
).read_text(encoding="utf-8")
for source in (schema, migration):
assert "CREATE TABLE IF NOT EXISTS cmt_note_metric_snapshots" in source
assert "UNIQUE (note_id, metric_date)" in source
assert "COALESCE(ms.view_count, n.view_count)" in report
assert "ms.metric_date = %s" in report
@@ -1,7 +1,14 @@
from datetime import datetime
from types import SimpleNamespace
import pytest
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
from gyxx_flow.modules.content_marketing import chanmama_scraper as chanmama
from gyxx_flow.modules.content_marketing import feishu_mapping
from gyxx_flow.modules.content_marketing import (
scrapling_browser_facade as browser_facade,
)
def test_exposure_state_clicks_text_and_zero_but_keeps_positive_values():
@@ -152,6 +159,277 @@ def test_missing_get_data_button_does_not_trigger_refresh_wait(monkeypatch):
assert sleeps == []
def test_create_driver_uses_runtime_cdp_profile_and_download_bindings(monkeypatch, tmp_path):
calls = []
expected = object()
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
def start(**kwargs):
calls.append(kwargs)
return expected
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
assert chanmama.create_driver() is expected
assert calls == [{
"profile_dir": str(tmp_path / "profile"),
"download_dir": str(tmp_path / "downloads"),
"cdp_url": "http://127.0.0.1:22002",
"stealthy": True,
}]
def test_create_driver_starts_scrapling_without_cdp(monkeypatch, tmp_path):
calls = []
expected = object()
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.delenv("GYXX_BROWSER_CDP_URL", raising=False)
def start(**kwargs):
calls.append(kwargs)
return expected
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
assert chanmama.create_driver() is expected
assert calls[0]["cdp_url"] is None
assert calls[0]["stealthy"] is True
assert calls[0]["real_chrome"] is False
def test_create_driver_falls_back_when_bound_cdp_is_not_listening(
monkeypatch,
tmp_path,
):
calls = []
expected = object()
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
def start(**kwargs):
calls.append(kwargs)
if kwargs["cdp_url"]:
raise RuntimeError(
"BrowserType.connect_over_cdp: connect ECONNREFUSED "
"127.0.0.1:22002; retrieving websocket url"
)
return expected
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
assert chanmama.create_driver() is expected
assert [call["cdp_url"] for call in calls] == [
"http://127.0.0.1:22002",
None,
]
assert all(call["stealthy"] is True for call in calls)
assert calls[1]["real_chrome"] is False
def test_create_driver_keeps_explicit_remote_cdp_fail_closed(
monkeypatch,
tmp_path,
):
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://browser.internal:22002")
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
monkeypatch.setattr(
chanmama,
"ScraplingDriver",
SimpleNamespace(
start=lambda **_kwargs: (_ for _ in ()).throw(
RuntimeError("connect ECONNREFUSED browser.internal:22002")
)
),
)
with pytest.raises(RuntimeError, match="ECONNREFUSED"):
chanmama.create_driver()
def test_create_driver_does_not_hide_non_connection_cdp_failures(
monkeypatch,
tmp_path,
):
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
)
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
monkeypatch.setattr(
chanmama,
"ScraplingDriver",
SimpleNamespace(
start=lambda **_kwargs: (_ for _ in ()).throw(
RuntimeError("browser profile is corrupt")
)
),
)
with pytest.raises(RuntimeError, match="profile is corrupt"):
chanmama.create_driver()
def test_scrapling_facade_reuses_runtime_cdp_context(monkeypatch, tmp_path):
created = []
class Page:
def on(self, *_args):
return None
def add_init_script(self, script):
assert "navigator" in script
class Context:
pages = [Page()]
class Browser:
def __init__(self, **kwargs):
self.kwargs = kwargs
self.context = Context()
self.page = None
self.closed = False
created.append(self)
def start(self):
return self
def close(self):
self.closed = True
monkeypatch.setattr(browser_facade, "ScraplingBrowser", Browser)
driver = browser_facade.ScraplingDriver.start(
profile_dir=tmp_path / "profile",
download_dir=tmp_path / "downloads",
cdp_url="http://127.0.0.1:22002",
)
assert created[0].kwargs["reuse_existing_cdp_context"] is True
assert created[0].kwargs["cdp_url"] == "http://127.0.0.1:22002"
assert "user_data_dir" not in created[0].kwargs
assert "--no-sandbox" not in created[0].kwargs["extra_flags"]
assert created[0].page is created[0].context.pages[0]
driver.quit()
assert created[0].closed is True
def _one_backfill_style():
return {
"tables": [
{
"index": 1,
"name": "宙斯",
"base_token": "base",
"table_id": "table",
"field_map": {
"platform": {"field_id": "platform"},
"note_title": {"field_id": "title"},
"publish_time": {"field_id": "published"},
"read_count_7d": {
"field_id": "read-7d",
"field_name": "7日阅读量",
},
},
}
]
}
def test_backfill_mapping_or_record_list_failure_returns_false(monkeypatch, tmp_path):
monkeypatch.setattr(
feishu_mapping,
"load_mapping",
lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("mapping down")),
)
assert chanmama.backfill_self_tables([]) is False
monkeypatch.setattr(
feishu_mapping, "load_mapping", lambda **_kwargs: {"tables": []}
)
assert chanmama.backfill_self_tables([]) is False
monkeypatch.setattr(
feishu_mapping, "load_mapping", lambda **_kwargs: _one_backfill_style()
)
assert chanmama.backfill_self_tables([], only_style=999) is False
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(normalized_root=tmp_path / "normalized"),
)
monkeypatch.setattr(
chanmama,
"_list_records_by_table",
lambda *_args: (_ for _ in ()).throw(RuntimeError("base down")),
)
assert chanmama.backfill_self_tables([]) is False
def test_backfill_write_failure_is_false_but_natural_no_match_is_success(
monkeypatch, tmp_path
):
monkeypatch.setattr(
feishu_mapping, "load_mapping", lambda **_kwargs: _one_backfill_style()
)
monkeypatch.setattr(
chanmama,
"PATHS",
SimpleNamespace(normalized_root=tmp_path / "normalized"),
)
record = {
"record_id": "record-1",
"platform": "抖音",
"title": "相同标题",
"published": datetime.now(),
}
monkeypatch.setattr(
chanmama, "_list_records_by_table", lambda *_args: [record]
)
monkeypatch.setattr(chanmama, "_write_back", lambda *_args: False)
assert chanmama.backfill_self_tables(
[{"title": "相同标题", "exposure": 100}]
) is False
monkeypatch.setattr(
chanmama,
"_write_back",
lambda *_args: (_ for _ in ()).throw(
AssertionError("natural no-match must not write")
),
)
assert chanmama.backfill_self_tables(
[{"title": "完全不同", "exposure": 100}]
) is True
def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
events = []
@@ -159,7 +437,6 @@ def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
def quit(self):
events.append("quit")
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
@@ -180,10 +457,75 @@ def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
lambda *_args, **_kwargs: ([], []),
)
assert chanmama.main() == 0
assert chanmama.main() == 1
assert events == ["saved", "quit"]
def test_main_full_mode_exception_and_keyboard_interrupt_are_nonzero(monkeypatch):
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(
chanmama,
"create_driver",
lambda: (_ for _ in ()).throw(RuntimeError("driver failed")),
)
assert chanmama.main() == 1
monkeypatch.setattr(
chanmama,
"create_driver",
lambda: (_ for _ in ()).throw(KeyboardInterrupt()),
)
assert chanmama.main() == 130
def test_main_propagates_backfill_failure_after_nonempty_export(monkeypatch):
events = []
class Driver:
def quit(self):
events.append("quit")
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
monkeypatch.setattr(chanmama, "save_cookies", lambda _driver: True)
monkeypatch.setattr(
chanmama,
"refresh_then_export_accounts",
lambda *_args, **_kwargs: ([{"title": "video"}], []),
)
monkeypatch.setattr(
chanmama,
"backfill_self_tables",
lambda *_args, **_kwargs: False,
)
assert chanmama.main() == 1
assert events == ["quit"]
def test_backfill_only_propagates_backfill_failure(monkeypatch, tmp_path):
workbook = tmp_path / "existing.xlsx"
workbook.write_bytes(b"placeholder")
monkeypatch.setattr(
chanmama.sys,
"argv",
["chanmama_scraper.py", "--backfill-only", "--excel", str(workbook)],
)
monkeypatch.setattr(
chanmama,
"parse_chanmama_excel",
lambda _path: ([{"title": "video"}], []),
)
monkeypatch.setattr(
chanmama,
"backfill_self_tables",
lambda *_args, **_kwargs: False,
)
assert chanmama.main() == 1
def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
monkeypatch,
tmp_path,
@@ -196,7 +538,6 @@ def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
monkeypatch.setattr(chanmama, "ACCOUNT", "")
monkeypatch.setattr(chanmama, "PASSWORD", "")
@@ -251,3 +592,85 @@ def test_cli_preserves_cookie_skip_exit_code(monkeypatch):
chanmama.cli()
assert raised.value.code == COOKIE_SKIP_EXIT_CODE
def test_wait_for_exported_excel_polls_until_file_appears(monkeypatch):
now = [1000.0]
sleeps: list[float] = []
results = iter([None, None, "/downloads/博主A_视频记录_周期内发布视频.xlsx"])
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
monkeypatch.setattr(
chanmama.time,
"sleep",
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
)
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
assert chanmama._wait_for_exported_excel(0.0) == (
"/downloads/博主A_视频记录_周期内发布视频.xlsx"
)
assert sleeps == [5.0, 5.0]
def test_wait_for_exported_excel_returns_none_after_timeout(monkeypatch):
now = [1000.0]
sleeps: list[float] = []
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
monkeypatch.setattr(
chanmama.time,
"sleep",
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
)
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: None)
assert (
chanmama._wait_for_exported_excel(
0.0,
timeout_seconds=12,
grace_seconds=0,
)
is None
)
assert len(sleeps) == 3 # 1000 -> 1005 -> 1010 -> 1015(>= 1012 超时)
def test_wait_for_exported_excel_allows_boundary_grace(monkeypatch):
now = [1000.0]
sleeps: list[float] = []
results = iter([None, None, "/downloads/边界后落盘.xlsx"])
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
monkeypatch.setattr(
chanmama.time,
"sleep",
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
)
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
assert chanmama._wait_for_exported_excel(
0.0,
timeout_seconds=5,
grace_seconds=5,
poll_interval_seconds=5,
) == "/downloads/边界后落盘.xlsx"
assert sleeps == [5.0, 5.0]
def test_wait_for_exported_excel_returns_immediately_when_present(monkeypatch):
now = [1000.0]
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
monkeypatch.setattr(
chanmama.time,
"sleep",
lambda _seconds: pytest.fail("不应等待已存在的文件"),
)
monkeypatch.setattr(
chanmama,
"_find_new_excel_after",
lambda _pre_mtime: "/downloads/已有.xlsx",
)
assert chanmama._wait_for_exported_excel(0.0) == "/downloads/已有.xlsx"
@@ -9,6 +9,10 @@ from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
from gyxx_flow.modules.content_marketing import run_all
from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
from gyxx_flow.modules.content_marketing.serialized_page_action import (
is_transient_browser_error,
run_serialized_page_action,
)
def test_normalize_title_handles_nfkc_case_and_zero_width():
@@ -131,6 +135,28 @@ def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
def test_xingtu_can_use_strong_title_when_share_id_is_canonicalized():
task = {
"record_id": "r1",
"target_title": "这家包颜值和实用性真挺高",
"note_id": "7665297310758210854",
}
card = {
"title": "这家包颜值和实用性真挺高 #潮流运动 #光影行星",
"note_id": "canonical- Xingtu-item-id",
}
matched = cc.match_tasks_to_cards(
[task],
[card],
platform="douyin",
allow_mismatched_content_id_title=True,
)
assert matched["r1"]["match_method"] == "title"
assert matched["r1"]["match_score"] >= 0.9
def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
cards = [
{
@@ -354,8 +380,11 @@ class _FakeSearchPage:
def test_pgy_search_retry_keeps_creator_id_query():
page = _FakeSearchPage()
pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
found = pgy._ensure_search_results(
page, query="xhs-id-123", result_name="达人昵称", max_retries=2
)
assert found is True
assert page.input.filled == ["xhs-id-123"]
@@ -376,11 +405,19 @@ def test_apply_matched_task_reports_write_failure(monkeypatch):
assert outcome["write_ok"] is False
def test_default_creator_batch_has_no_forced_hour_pause():
assert pgy.DEFAULT_BATCH_SIZE == 0
def test_pgy_default_creator_batch_rotates_browser_before_memory_accumulates():
assert pgy.DEFAULT_BATCH_SIZE == 4
assert xingtu.DEFAULT_BATCH_SIZE == 0
@pytest.mark.parametrize("message", [
"Page crashed: Out of Memory",
"Target page, context or browser has been closed: not enough memory",
])
def test_pgy_treats_chromium_memory_exhaustion_as_a_recoverable_session_loss(message):
assert pgy.is_browser_session_lost(RuntimeError(message)) is True
def test_repeatable_style_argument_and_selection_keep_every_requested_style():
parser = argparse.ArgumentParser()
cc.add_repeatable_style_argument(parser, "styles")
@@ -408,6 +445,223 @@ def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
assert page.waits == 2
def test_pgy_stable_cards_waits_through_a_slow_initial_render(monkeypatch):
card = {"title": "标题", "read_count": "1"}
sequence = [[], [], [], [], [], [card], [card]]
monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
class Page:
waits = 0
def wait_for_timeout(self, _ms):
self.waits += 1
page = Page()
assert pgy.parse_cards_stable(page) == [card]
assert page.waits == 6
def test_pgy_card_parser_recovers_note_id_from_internal_card_link():
note_id = "0123456789abcdef01234567"
class Page:
def evaluate(self, _script):
return [{
"title": "标题",
"read_count": "1",
"href": "",
"note_id": "",
"_href_candidates": [
f"https://pgy.xiaohongshu.com/internal?source_note_id={note_id}"
],
"_data_note_ids": [],
}]
cards = pgy.parse_cards_on_page(Page())
assert cards == [{
"title": "标题",
"read_count": "1",
"href": "",
"note_id": note_id,
}]
def test_pgy_card_parser_prefers_strict_note_detail_id_over_generic_segment():
note_id = "0123456789abcdef01234567"
internal_href = f"https://example.invalid/note/detail/{note_id}"
class Page:
def evaluate(self, _script):
return [{
"title": "标题",
"read_count": "1",
"href": internal_href,
# This is what the old generic /note/<segment> parser produced.
"note_id": "detail",
"_href_candidates": [internal_href],
"_data_note_ids": [],
}]
cards = pgy.parse_cards_on_page(Page())
assert cards[0]["note_id"] == note_id
def test_pgy_dom_parser_reads_root_data_id_and_rejects_arbitrary_href_fallback():
captured = {}
class Page:
def evaluate(self, script):
captured["script"] = script
return []
assert pgy.parse_cards_on_page(Page()) == []
assert "const dataNoteNodes = [card, ...card.querySelectorAll" in captured["script"]
assert "const href = link?.href || '';" in captured["script"]
assert "hrefCandidates[0]" not in captured["script"]
def test_pgy_arbitrary_shared_anchor_does_not_collapse_distinct_cards():
shared_anchor = "https://example.invalid/creator/profile"
class Page:
def evaluate(self, _script):
return [
{
"title": "第一篇笔记",
"read_count": "1",
"like_count": "1",
"collect_count": "1",
"publish_time": "2026-08-01",
"href": "",
"note_id": "",
"_href_candidates": [shared_anchor],
"_data_note_ids": [],
},
{
"title": "第二篇笔记",
"read_count": "2",
"like_count": "2",
"collect_count": "2",
"publish_time": "2026-08-02",
"href": "",
"note_id": "",
"_href_candidates": [shared_anchor],
"_data_note_ids": [],
},
]
cards = pgy.parse_cards_on_page(Page())
assert [card["href"] for card in cards] == ["", ""]
assert pgy._card_observation_key(cards[0]) != pgy._card_observation_key(cards[1])
def test_pgy_deduplicates_one_card_repeated_across_pages_before_matching(monkeypatch):
card = {
"title": "同一篇笔记",
"read_count": "100",
"like_count": "10",
"collect_count": "5",
"publish_time": "2026-08-01",
"href": "",
"note_id": "",
}
pages = [[dict(card)], [dict(card)], [dict(card)]]
monkeypatch.setattr(pgy, "parse_cards_stable", lambda _page: pages.pop(0))
monkeypatch.setattr(pgy, "go_next_page", lambda _page: bool(pages))
found, candidates, page_limit_hit = pgy.find_notes_for_tasks(
object(),
[{"record_id": "r1", "target_title": "同一篇笔记"}],
max_pages=4,
)
assert found["r1"]["read_count"] == "100"
assert candidates == ["同一篇笔记"]
assert page_limit_hit is False
def test_pgy_missing_creator_is_blocked_but_unmatched_note_stays_retryable():
no_creator = pgy._missing_detail_outcome(saw_search_result=False)
no_unique_note = pgy._unmatched_note_outcome(
candidates=["平台上的其他笔记"],
page_limit_hit=False,
)
completed = cc.finalize_summary({
"total": 2,
"results": [
{"record_id": "creator", **no_creator},
{"record_id": "note", **no_unique_note},
],
})
assert completed["blocked_input"] == 1
assert completed["retryable_failures"] == 1
assert completed["unresolved"] == 1
assert completed["complete"] is False
assert pgy._missing_detail_outcome(saw_search_result=True)["status"] == "retryable_failure"
assert pgy._unmatched_note_outcome(
candidates=[], page_limit_hit=False
)["reason"] == "empty_detail_page"
assert pgy._unmatched_note_outcome(
candidates=["候选"], page_limit_hit=True
)["status"] == "retryable_failure"
@pytest.mark.parametrize(
"state",
[
{
"nextFound": True,
"nextDisabled": True,
"activePage": None,
"lastPage": None,
"numericPageCount": 0,
},
{
"nextFound": True,
"nextDisabled": False,
"activePage": 7,
"lastPage": 7,
"numericPageCount": 7,
},
],
)
def test_pgy_known_last_page_stops_without_clicking(state):
class Page:
def evaluate(self, script):
if "pgy-pagination-state" in script:
return state
if "pgy-pagination-click" in script:
pytest.fail("terminal paginator must not be clicked")
return "last-page-card-signature"
assert pgy.go_next_page(Page()) is False
def test_pgy_clicked_pagination_without_card_change_uses_source_end_semantics():
class Page:
def evaluate(self, script):
if "pgy-pagination-state" in script:
return {
"nextFound": True,
"nextDisabled": False,
"activePage": 2,
"lastPage": 7,
"numericPageCount": 7,
}
if "pgy-pagination-click" in script:
return {"clicked": True, "text": "next"}
return "same-card-signature"
def wait_for_timeout(self, _ms):
return None
assert pgy.go_next_page(Page()) is False
def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
pages = [
[{"title": "无关视频", "play_count": "10"}],
@@ -432,6 +686,125 @@ def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
assert calls["next"] == 1
def test_xingtu_video_search_uses_visible_input_and_source_keyboard_events():
events = []
class Locator:
first = None
def __init__(self):
self.first = self
def count(self):
return 1
def wait_for(self, **kwargs):
events.append(("wait", kwargs))
def click(self):
events.append(("click",))
def press(self, key):
events.append(("locator_press", key))
def fill(self, _value):
pytest.fail("Xingtu search must preserve source key-by-key input")
class Keyboard:
def type(self, value, **kwargs):
events.append(("type", value, kwargs))
def press(self, key):
events.append(("keyboard_press", key))
class Page:
keyboard = Keyboard()
def __init__(self):
self.selector = None
self.input = Locator()
def locator(self, selector):
self.selector = selector
return self.input
page = Page()
assert xingtu.submit_video_search(page, "目标标题") is True
assert ":visible" in page.selector
assert events == [
("wait", {"state": "visible", "timeout": 5000}),
("click",),
("locator_press", "Control+A"),
("locator_press", "Backspace"),
("type", "目标标题", {"delay": 20}),
("keyboard_press", "Enter"),
]
def test_xingtu_only_confirmed_creator_miss_is_blocked():
no_creator = xingtu._missing_detail_outcome(saw_search_result=False)
no_unique_video = xingtu._unmatched_video_outcome(
candidates=["平台上的其他视频"],
page_limit_hit=False,
)
completed = cc.finalize_summary({
"total": 2,
"results": [
{"record_id": "creator", **no_creator},
{"record_id": "video", **no_unique_video},
],
})
assert completed["blocked_input"] == 1
assert completed["retryable_failures"] == 1
assert completed["unresolved"] == 1
assert completed["complete"] is False
assert no_unique_video["status"] == "retryable_failure"
assert xingtu._missing_detail_outcome(
saw_search_result=True
)["status"] == "retryable_failure"
assert xingtu._unmatched_video_outcome(
candidates=[], page_limit_hit=False
)["reason"] == "empty_detail_page"
assert xingtu._unmatched_video_outcome(
candidates=["候选"], page_limit_hit=True
)["status"] == "retryable_failure"
def test_xingtu_missing_creator_result_has_distinct_business_exception():
class Context:
pages = []
class Page:
context = Context()
def evaluate(self, _script, argument=None):
if argument is not None:
return {"found": False}
return False
def wait_for_timeout(self, _milliseconds):
return None
with pytest.raises(xingtu.CreatorNotFound, match="未找到精确达人"):
xingtu.open_creator_detail(Page(), "不存在的达人", "missing-id")
def test_xingtu_clicked_pagination_without_card_change_is_retryable_timeout():
class Page:
def evaluate(self, script):
if "const all" in script:
return {"clicked": True, "picked": "pagination"}
return "same-card-signature"
def wait_for_timeout(self, _milliseconds):
return None
with pytest.raises(xingtu.BrowserTimeoutError, match="did not change"):
xingtu.go_next_page(Page())
def test_xingtu_normalizes_show_items_api_video():
card = xingtu.normalize_xingtu_api_item({
"item_id": 7654321,
@@ -456,6 +829,9 @@ def test_xingtu_normalizes_show_items_api_video():
def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
import gc
import weakref
class Response:
url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
@@ -477,9 +853,14 @@ def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
page = Page()
cards = xingtu.setup_show_items_capture(page)
page.callback(Response())
response = Response()
response_ref = weakref.ref(response)
page.callback(response)
del response
gc.collect()
assert [card["note_id"] for card in cards] == ["1", "2"]
assert response_ref() is None
def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
@@ -542,9 +923,221 @@ def test_browser_session_loss_detection_is_specific():
closed = RuntimeError("Target page, context or browser has been closed")
assert pgy.is_browser_session_lost(closed)
assert xingtu.is_browser_session_lost(closed)
assert pgy.is_browser_session_lost(RuntimeError("Page crashed"))
assert xingtu.is_browser_session_lost(RuntimeError("Page crashed"))
assert pgy.is_browser_session_lost(
RuntimeError("Execution context was destroyed, most likely because of a navigation.")
)
assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
def test_transient_browser_error_excludes_business_timeout():
assert is_transient_browser_error(
pgy.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded")
)
assert is_transient_browser_error(
RuntimeError("Page.goto: net::ERR_CONNECTION_RESET")
)
assert not is_transient_browser_error(
TimeoutError("Login timed out after 300 seconds.")
)
def _stub_creator_session_failure(monkeypatch, tmp_path, module, error_factory):
style = {"index": 1, "name": "style"}
task = {
"record_id": "record-1",
"creator_name": "creator",
"creator_id": None,
"target_title": "target",
"style_context": {"index": 1},
}
summary = {
"index": 1,
"name": "style",
"total": 1,
"filled": 0,
"results": [],
}
group = {
"creator_name": "creator",
"creator_id": None,
"tasks": [task],
}
sessions = []
class Session:
def __init__(self, **_kwargs):
sessions.append(self)
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def fail_page_action(*_args, **_kwargs):
raise error_factory()
monkeypatch.setattr(
module,
"collect_tasks_across_styles",
lambda *_args, **_kwargs: ([task], {1: summary}),
)
monkeypatch.setattr(module, "group_tasks_by_creator", lambda _tasks: [group])
monkeypatch.setattr(module, "chunk_creator_groups", lambda groups, _size: [groups])
monkeypatch.setattr(module, "DynamicSession", Session)
monkeypatch.setattr(module, "run_serialized_page_action", fail_page_action)
monkeypatch.setattr(module, "_force_cleanup_session", lambda _session: None)
monkeypatch.setattr(module, "PROFILE_DIR", tmp_path / module.__name__.split(".")[-1])
return style, sessions
@pytest.mark.parametrize("module", [pgy, xingtu])
def test_creator_collectors_retry_browser_timeout_in_three_fresh_sessions(
monkeypatch, tmp_path, module
):
style, sessions = _stub_creator_session_failure(
monkeypatch,
tmp_path,
module,
lambda: module.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded"),
)
summaries = module.scrape_styles([style], 1, True, None, True)
assert len(sessions) == 3
assert summaries[0]["complete"] is False
assert summaries[0]["retryable_failures"] == 1
@pytest.mark.parametrize("module", [pgy, xingtu])
def test_creator_collectors_do_not_retry_acceptance_terminal(
monkeypatch, tmp_path, module
):
style, sessions = _stub_creator_session_failure(
monkeypatch,
tmp_path,
module,
lambda: module.AcceptanceCookieSkip("cookie unavailable in acceptance"),
)
with pytest.raises(module.AcceptanceCookieSkip):
module.scrape_styles([style], 1, True, None, True)
assert len(sessions) == 1
@pytest.mark.parametrize("module", [pgy, xingtu])
def test_creator_collectors_do_not_retry_login_timeout_terminal(
monkeypatch, tmp_path, module
):
style, sessions = _stub_creator_session_failure(
monkeypatch,
tmp_path,
module,
lambda: TimeoutError("Login timed out after 300 seconds."),
)
with pytest.raises(TimeoutError, match="Login timed out"):
module.scrape_styles([style], 1, True, None, True)
assert len(sessions) == 1
@pytest.mark.parametrize("error_type", [xingtu.SearchQuotaExhausted, xingtu.CaptchaBlocked])
def test_xingtu_does_not_retry_explicit_business_terminal(
monkeypatch, tmp_path, error_type
):
style, sessions = _stub_creator_session_failure(
monkeypatch,
tmp_path,
xingtu,
lambda: error_type("explicit platform terminal"),
)
with pytest.raises(error_type):
xingtu.scrape_styles([style], 1, True, None, True)
assert len(sessions) == 1
@pytest.mark.parametrize("module", [pgy, xingtu])
def test_creator_collectors_with_no_tasks_do_not_open_browser(
monkeypatch, module
):
style = {"index": 1, "name": "style"}
summary = {"index": 1, "name": "style", "total": 0, "results": []}
monkeypatch.setattr(
module,
"collect_tasks_across_styles",
lambda *_args, **_kwargs: ([], {1: summary}),
)
monkeypatch.setattr(
module,
"DynamicSession",
lambda **_kwargs: pytest.fail("no browser session expected"),
)
assert module.scrape_styles([style], 1, True, None, True) == [summary]
def test_long_page_action_does_not_retain_navigation_response():
import gc
import weakref
response_ref = None
class Response:
url = "https://example.test/final"
status = 200
headers = {"content-type": "text/html"}
class Page:
closed = False
def set_default_timeout(self, _timeout):
pass
def set_default_navigation_timeout(self, _timeout):
pass
def goto(self, *_args, **_kwargs):
nonlocal response_ref
response = Response()
response_ref = weakref.ref(response)
return response
def is_closed(self):
return self.closed
def close(self):
self.closed = True
page = Page()
class Context:
def new_page(self):
return page
class Session:
context = Context()
def action(_page):
gc.collect()
assert response_ref is not None
assert response_ref() is None
navigation = run_serialized_page_action(Session(), "https://example.test", action)
assert navigation == {
"url": "https://example.test/final",
"status": 200,
"headers": {"content-type": "text/html"},
}
assert page.closed is True
def test_xingtu_nested_session_loss_is_reraised():
closed = RuntimeError("Target page, context or browser has been closed")
with pytest.raises(RuntimeError, match="has been closed"):
@@ -694,6 +1287,19 @@ def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
assert [row["record_id"] for row in summary["details"]] == ["published"]
@pytest.mark.parametrize(
("raw", "expected"),
[
("2026-07-20T00:00:00.000+08:00", bili.date(2026, 7, 20)),
("2026-07-20T00:00:00+08:00", bili.date(2026, 7, 20)),
("2026-07-19T16:00:00.000Z", bili.date(2026, 7, 19)),
("2026-07-20", bili.date(2026, 7, 20)),
],
)
def test_bilibili_parse_pub_date_accepts_feishu_iso_values(raw, expected):
assert bili.parse_pub_date(raw) == expected
def test_bilibili_partial_retry_preserves_old_success():
existing = {
"style": "款式A", "index": 1, "total_b_records": 2,
@@ -0,0 +1,272 @@
import pytest
from gyxx_flow.adapters.scrapling import BrowserTimeoutError
from gyxx_flow.modules.content_marketing.data.tools import (
batch_rescrape_douyin as douyin_batch,
)
from gyxx_flow.modules.content_marketing.data.tools import (
batch_rescrape_xiaohongshu as xhs_batch,
)
from gyxx_flow.modules.content_marketing.data.tools.comment_batch_checkpoint import (
CommentBatchCheckpoint,
is_browser_process_lost,
)
def test_page_crash_requires_a_fresh_browser():
assert is_browser_process_lost(
RuntimeError("Page.wait_for_timeout: Page crashed")
)
assert not is_browser_process_lost(RuntimeError("ordinary parse failure"))
@pytest.mark.parametrize("batch", [xhs_batch, douyin_batch])
def test_non_acceptance_relogin_runs_platform_script(monkeypatch, batch):
class Policy:
enabled = False
calls = []
monkeypatch.setattr(batch, "current_acceptance_policy", lambda: Policy())
monkeypatch.setattr(
batch.subprocess,
"call",
lambda command, **kwargs: calls.append((command, kwargs)) or 0,
)
assert batch.do_relogin() is True
assert calls[0][0][1] == str(batch.RELOGIN_SCRIPT)
def test_comment_checkpoint_matches_both_note_id_and_url(tmp_path):
path = tmp_path / "checkpoint.json"
checkpoint = CommentBatchCheckpoint(
"douyin", business_date="2026-08-09", path=path
)
checkpoint.mark_completed(7, "https://www.douyin.com/video/7")
restored = CommentBatchCheckpoint(
"douyin", business_date="2026-08-09", path=path
)
assert restored.contains(7, "https://www.douyin.com/video/7") is True
assert restored.contains(7, "https://www.douyin.com/video/changed") is False
restored.clear()
assert not path.exists()
def test_xhs_page_crash_retries_same_note_with_new_scraper_call(monkeypatch):
calls = []
def scrape(url, *_args):
calls.append(url)
if len(calls) == 1:
raise RuntimeError("Page.wait_for_timeout: Page crashed")
return [], {"stats": {}}
monkeypatch.setattr(xhs_batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(xhs_batch.time, "sleep", lambda _seconds: None)
comments, result, error = xhs_batch.scrape_with_fresh_browser("xhs-url")
assert comments == []
assert result == {"stats": {}}
assert error is None
assert calls == ["xhs-url", "xhs-url"]
def test_douyin_page_crash_retries_same_note_with_new_scraper_call(monkeypatch):
calls = []
def scrape(url, *_args):
calls.append(url)
if len(calls) == 1:
raise RuntimeError("Page.wait_for_timeout: Page crashed")
return [], {"stats": {}}
monkeypatch.setattr(douyin_batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(douyin_batch.time, "sleep", lambda _seconds: None)
comments, result, error = douyin_batch.scrape_with_fresh_browser("dy-url")
assert comments == []
assert result == {"stats": {}}
assert error is None
assert calls == ["dy-url", "dy-url"]
@pytest.mark.parametrize(
("batch", "url"),
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
)
def test_transient_timeout_gets_three_fresh_browser_attempts(monkeypatch, batch, url):
calls = []
def scrape(scrape_url, *_args):
calls.append(scrape_url)
if len(calls) < 3:
raise BrowserTimeoutError("Page.goto: Timeout 90000ms exceeded")
return [], {"stats": {}}
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
comments, result, error = batch.scrape_with_fresh_browser(url)
assert comments == []
assert result == {"stats": {}}
assert error is None
assert calls == [url, url, url]
@pytest.mark.parametrize(
("batch", "url"),
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
)
def test_transient_network_failure_exhausts_three_attempts_honestly(
monkeypatch, batch, url
):
calls = []
def scrape(scrape_url, *_args):
calls.append(scrape_url)
raise RuntimeError("Page.goto: net::ERR_CONNECTION_RESET")
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
comments, result, error = batch.scrape_with_fresh_browser(url)
assert comments is None
assert result is None
assert error is not None
assert "ERR_CONNECTION_RESET" in str(error)
assert calls == [url, url, url]
@pytest.mark.parametrize(
("batch", "url"),
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
)
def test_terminal_parse_failure_is_not_retried(monkeypatch, batch, url):
calls = []
def scrape(scrape_url, *_args):
calls.append(scrape_url)
raise RuntimeError("ordinary terminal parse failure")
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
comments, result, error = batch.scrape_with_fresh_browser(url)
assert comments is None
assert result is None
assert error is not None
assert calls == [url]
def test_xhs_failed_note_keeps_success_checkpoint_and_fails_batch(monkeypatch):
class Checkpoint:
completed = []
cleared = False
def __init__(self, _platform):
pass
def contains(self, _note_id, _url):
return False
def mark_completed(self, note_id, url):
self.completed.append((note_id, url))
def clear(self):
self.cleared = True
rows = [
{"id": 1, "url": "xhs-1", "title": "one"},
{"id": 2, "url": "xhs-2", "title": "two"},
]
def scrape(url, *_args):
if url == "xhs-2":
raise RuntimeError("ordinary terminal failure")
return ([{"level": "comment"}], {
"title": "one",
"stats": {"logged_in": True, "note_metrics": {}},
})
monkeypatch.setattr(xhs_batch, "CommentBatchCheckpoint", Checkpoint)
monkeypatch.setattr(xhs_batch, "collect_urls", lambda: rows)
monkeypatch.setattr(xhs_batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(xhs_batch.scraper, "write_outputs", lambda *_args: (None, None))
monkeypatch.setattr(xhs_batch, "upsert_metrics", lambda *_args: None)
monkeypatch.setattr(xhs_batch.db, "replace_comments", lambda *_args, **_kwargs: None)
assert xhs_batch.main() == 1
assert Checkpoint.completed == [(1, "xhs-1")]
assert Checkpoint.cleared is False
@pytest.mark.parametrize(
("batch", "platform"),
[(xhs_batch, "xhs"), (douyin_batch, "dy")],
)
def test_incomplete_comments_preserve_db_but_remain_resume_eligible(
monkeypatch, batch, platform
):
class Checkpoint:
def __init__(self):
self.completed = []
self.cleared = False
def contains(self, _note_id, _url):
return False
def mark_completed(self, note_id, url):
self.completed.append((note_id, url))
def clear(self):
self.cleared = True
checkpoint = Checkpoint()
rows = [
{
"id": 1,
"url": f"{platform}-partial",
"title": "partial",
"comment_count": 5,
},
{
"id": 2,
"url": f"{platform}-failed",
"title": "failed",
"comment_count": 5,
},
]
def scrape(url, *_args):
if url.endswith("-failed"):
raise RuntimeError("ordinary terminal failure")
comments = [{"level": "comment"}]
stats = {
"logged_in": False,
"api_total": 5,
"note_metrics": {"comment_count": 5},
}
if batch is xhs_batch:
return comments, {"title": "partial", "stats": stats}
return comments, {"stats": stats}
monkeypatch.setattr(batch, "CommentBatchCheckpoint", lambda _platform: checkpoint)
monkeypatch.setattr(batch, "collect_urls", lambda: rows)
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(batch.scraper, "write_outputs", lambda *_args: (None, None))
monkeypatch.setattr(batch, "upsert_metrics", lambda *_args: None)
monkeypatch.setattr(
batch.db,
"replace_comments",
lambda *_args, **_kwargs: pytest.fail("incomplete comments must not replace DB"),
)
assert batch.main() == 1
assert checkpoint.completed == []
assert checkpoint.cleared is False
@@ -4,7 +4,7 @@ import json
from pathlib import Path
from types import SimpleNamespace
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
from gyxx_flow.modules.content_marketing import chanmama_scraper, weekly_summary_all
from gyxx_flow.modules.content_marketing.data.tools import (
analyze_comments,
@@ -71,17 +71,13 @@ def test_daily_card_sends_only_to_wang_yunlong(
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
commands = []
calls = []
def fake_run(command, **kwargs):
commands.append(command)
return SimpleNamespace(
returncode=0,
stdout=json.dumps({"message_id": "om_test"}),
stderr="",
)
def fake_send(**kwargs):
calls.append(kwargs)
return {"message_id": "om_test"}
monkeypatch.setattr(daily_report_card.subprocess, "run", fake_run)
monkeypatch.setattr(daily_report_card, "send_lark_bot_message", fake_send)
results = daily_report_card.send_card_to_recipients(
{"schema": "2.0", "body": {"elements": []}},
("ou_other_a", "ou_other_b"),
@@ -89,9 +85,9 @@ def test_daily_card_sends_only_to_wang_yunlong(
)
assert len(results) == 1
assert len(commands) == 1
command = commands[0]
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
assert len(calls) == 1
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["profile"] == "hermes-analyzer"
def test_comment_summary_sends_only_to_wang_yunlong(
@@ -99,17 +95,17 @@ def test_comment_summary_sends_only_to_wang_yunlong(
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
commands = []
calls = []
def fake_run(command, **kwargs):
commands.append(command)
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
def fake_send(**kwargs):
calls.append(kwargs)
return {"message_id": "om_test"}
monkeypatch.setattr(analyze_comments.subprocess, "run", fake_run)
monkeypatch.setattr(analyze_comments, "send_lark_bot_message", fake_send)
analyze_comments.send_feishu_summary("summary", "ou_other")
command = commands[0]
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["profile"] == "hermes-analyzer"
def test_note_report_sends_only_to_wang_yunlong(
@@ -117,19 +113,18 @@ def test_note_report_sends_only_to_wang_yunlong(
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
commands = []
monkeypatch.setattr(analyze_note.sys, "platform", "linux")
calls = []
monkeypatch.setattr(
analyze_note,
"build_card_payload",
lambda *args, **kwargs: {"schema": "2.0"},
)
def fake_run(command, **kwargs):
commands.append(command)
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
def fake_send(**kwargs):
calls.append(kwargs)
return {"message_id": "om_test"}
monkeypatch.setattr(analyze_note.subprocess, "run", fake_run)
monkeypatch.setattr(analyze_note, "send_lark_bot_message", fake_send)
analyze_note.send_feishu_report(
{},
{},
@@ -141,8 +136,8 @@ def test_note_report_sends_only_to_wang_yunlong(
tmp_path / "report.md",
)
command = commands[0]
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
assert calls[0]["profile"] == "hermes-analyzer"
def test_relogin_notifications_resolve_only_to_wang_yunlong(
@@ -153,5 +148,5 @@ def test_relogin_notifications_resolve_only_to_wang_yunlong(
assert (
friday_relogin_parallel.recipient_for_platform("pgy", "ou_other")
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
)
@@ -0,0 +1,596 @@
import sys
from datetime import date
from types import SimpleNamespace
import pytest
from gyxx_flow.modules.content_marketing import monthly_summary_all, weekly_summary_all
from gyxx_flow.modules.content_marketing.data.tools import db
from gyxx_flow.modules.content_marketing.data.tools import (
generate_creator_report as report,
)
from gyxx_flow.modules.content_marketing.feishu_doc_import import (
FeishuDocImportTimeout,
)
def _prepare_main(monkeypatch, tmp_path, *, doc_url, saved):
monkeypatch.setattr(report, "DATA_DIR", tmp_path)
monkeypatch.setattr(
report,
"_compute_last_month",
lambda: (date(2026, 7, 1), date(2026, 7, 31)),
)
monkeypatch.setattr(
report,
"generate_report",
lambda style_filter=None: ("report", [], {}, {}),
)
monkeypatch.setattr(report, "load_existing_creator_report", lambda *_args: None)
monkeypatch.setattr(
report, "create_feishu_doc", lambda *args, **_kwargs: doc_url
)
monkeypatch.setattr(report, "save_to_db", lambda *args: saved)
monkeypatch.setattr(sys, "argv", ["generate_creator_report.py"])
def test_main_requires_both_doc_and_database_effects(monkeypatch, tmp_path):
_prepare_main(monkeypatch, tmp_path, doc_url="https://example/doc", saved=True)
assert report.main() == 0
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=True)
assert report.main() == 1
_prepare_main(monkeypatch, tmp_path, doc_url="https://example/doc", saved=False)
assert report.main() == 1
def test_main_dry_run_does_not_require_external_effects(monkeypatch, tmp_path):
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=False)
monkeypatch.setattr(sys, "argv", ["generate_creator_report.py", "--dry-run"])
assert report.main() == 0
def _import_timeout(tmp_path):
return FeishuDocImportTimeout(
"known import ticket did not become ready",
audit_path=tmp_path / "ticket-audit.json",
)
def test_creator_report_timeout_is_persisted_as_ambiguous(monkeypatch, tmp_path):
captured = {}
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=True)
monkeypatch.setattr(
report,
"load_existing_creator_report",
lambda *_args: {"doc_url": "", "doc_title": "", "status": "doc_ambiguous"},
)
def raise_timeout(*_args, **kwargs):
captured["resume_only"] = kwargs.get("resume_only")
raise _import_timeout(tmp_path)
def capture_save(*args):
captured["doc_url"] = args[-2]
captured["status"] = args[-1]
return True
monkeypatch.setattr(report, "create_feishu_doc", raise_timeout)
monkeypatch.setattr(report, "save_to_db", capture_save)
assert report.main() == 1
assert captured == {
"doc_url": None,
"resume_only": True,
"status": "doc_ambiguous",
}
@pytest.mark.parametrize(
("module", "processor_name", "creator_name"),
(
(weekly_summary_all, "process_one_style", "create_feishu_doc"),
(
monthly_summary_all,
"process_one_style_monthly",
"create_feishu_doc_monthly",
),
),
)
def test_summary_timeout_is_persisted_as_ambiguous(
monkeypatch,
tmp_path,
module,
processor_name,
creator_name,
):
saved = {}
creator_call = {}
notes = [{"id": 1, "view_count": 10, "comment_count": 1}]
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(module, "REPORTS_DIR", tmp_path)
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
if module is weekly_summary_all:
monkeypatch.setattr(
module,
"load_existing_summary_delivery",
lambda _style: {
"doc_url": "",
"doc_title": "",
"status": "doc_ambiguous",
},
)
monkeypatch.setattr(module, "get_last_week_notes", lambda _style: notes)
monkeypatch.setattr(module, "generate_summary", lambda *_args: "summary")
else:
monkeypatch.setattr(
module,
"load_existing_monthly_summary_delivery",
lambda _style: {
"doc_url": "",
"doc_title": "",
"status": "doc_ambiguous",
},
)
monkeypatch.setattr(module, "get_last_month_notes", lambda _style: notes)
monkeypatch.setattr(module, "generate_monthly_summary", lambda *_args: "summary")
monkeypatch.setattr(module, "run_single_note_analysis", lambda _note_id: "report")
def raise_timeout(*_args, **kwargs):
creator_call["resume_only"] = kwargs.get("resume_only")
raise _import_timeout(tmp_path)
def capture_save(*args):
saved["doc_url"] = args[-3]
saved["doc_title"] = args[-2]
saved["status"] = args[-1]
return True
monkeypatch.setattr(module, creator_name, raise_timeout)
save_name = "save_to_db" if module is weekly_summary_all else "save_to_db_monthly"
monkeypatch.setattr(module, save_name, capture_save)
processor = getattr(module, processor_name)
arguments = ("宙斯", ("base", "table", "url"))
if module is weekly_summary_all:
arguments += (False,)
result = processor(*arguments)
assert result["status"] == "doc_ambiguous"
assert saved == {
"doc_url": None,
"doc_title": None,
"status": "doc_ambiguous",
}
assert creator_call == {"resume_only": True}
@pytest.mark.parametrize(
("module", "processor_name", "creator_name"),
(
(weekly_summary_all, "process_one_style", "create_feishu_doc"),
(
monthly_summary_all,
"process_one_style_monthly",
"create_feishu_doc_monthly",
),
),
)
def test_summary_pg_failure_keeps_real_document_receipt_and_fails_result(
monkeypatch,
tmp_path,
module,
processor_name,
creator_name,
):
doc_url = "https://example.feishu.cn/docx/real-document"
notes = [{"id": 1, "view_count": 10, "comment_count": 1}]
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(module, "REPORTS_DIR", tmp_path)
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
if module is weekly_summary_all:
monkeypatch.setattr(module, "load_existing_summary_delivery", lambda _style: None)
monkeypatch.setattr(module, "get_last_week_notes", lambda _style: notes)
monkeypatch.setattr(module, "generate_summary", lambda *_args: "summary")
else:
monkeypatch.setattr(
module, "load_existing_monthly_summary_delivery", lambda _style: None
)
monkeypatch.setattr(module, "get_last_month_notes", lambda _style: notes)
monkeypatch.setattr(module, "generate_monthly_summary", lambda *_args: "summary")
monkeypatch.setattr(module, "run_single_note_analysis", lambda _note_id: "report")
monkeypatch.setattr(
module, creator_name, lambda *_args, **_kwargs: (doc_url, None)
)
monkeypatch.setattr(
module, "write_to_target_table", lambda *_args, **_kwargs: True
)
save_name = "save_to_db" if module is weekly_summary_all else "save_to_db_monthly"
monkeypatch.setattr(module, save_name, lambda *_args: False)
processor = getattr(module, processor_name)
arguments = ("宙斯", ("base", "table", "url"))
if module is weekly_summary_all:
arguments += (False,)
result = processor(*arguments)
assert result["status"] == "db_failed"
assert result["doc_url"] == doc_url
assert result["target_table_written"] is True
assert not weekly_summary_all.summary_results_succeeded([result])
def test_creator_existing_period_document_is_never_imported_again(
monkeypatch, tmp_path
):
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=False)
existing_url = "https://example.feishu.cn/docx/existing-creator-report"
monkeypatch.setattr(
report,
"load_existing_creator_report",
lambda *_args: {
"doc_url": existing_url,
"doc_title": "existing",
"status": "doc_ambiguous",
},
)
def fail_import(*_args, **_kwargs):
raise AssertionError("an existing period URL must not be imported again")
monkeypatch.setattr(report, "create_feishu_doc", fail_import)
monkeypatch.setattr(
report,
"save_to_db",
lambda *_args: (_ for _ in ()).throw(
AssertionError("completed period must not be overwritten")
),
)
assert report.main() == 0
@pytest.mark.parametrize(
("module", "processor_name", "loader_name", "creator_name"),
(
(
weekly_summary_all,
"process_one_style",
"load_existing_summary_delivery",
"create_feishu_doc",
),
(
monthly_summary_all,
"process_one_style_monthly",
"load_existing_monthly_summary_delivery",
"create_feishu_doc_monthly",
),
),
)
def test_completed_summary_period_skips_all_external_work(
monkeypatch,
module,
processor_name,
loader_name,
creator_name,
):
existing_url = "https://example.feishu.cn/docx/existing-summary"
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(
module,
loader_name,
lambda _style: {
"doc_url": existing_url,
"doc_title": "existing",
"status": "ok",
},
)
def fail_work(*_args, **_kwargs):
raise AssertionError("completed summary period must be skipped")
monkeypatch.setattr(module, "find_target_fields", fail_work)
monkeypatch.setattr(module, creator_name, fail_work)
processor = getattr(module, processor_name)
arguments = ("宙斯", ("base", "table", "url"))
if module is weekly_summary_all:
arguments += (False,)
result = processor(*arguments)
assert result["status"] == "ok"
assert result["doc_url"] == existing_url
assert result["already_complete"] is True
@pytest.mark.parametrize(
("module", "processor_name", "loader_name", "creator_name", "updater_name"),
(
(
weekly_summary_all,
"process_one_style",
"load_existing_summary_delivery",
"create_feishu_doc",
"update_existing_summary_delivery_status",
),
(
monthly_summary_all,
"process_one_style_monthly",
"load_existing_monthly_summary_delivery",
"create_feishu_doc_monthly",
"update_existing_monthly_summary_delivery_status",
),
),
)
def test_failed_base_delivery_reuses_existing_document_without_import(
monkeypatch,
module,
processor_name,
loader_name,
creator_name,
updater_name,
):
existing_url = "https://example.feishu.cn/docx/existing-summary"
write_call = {}
status_call = {}
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(
module,
loader_name,
lambda _style: {
"doc_url": existing_url,
"doc_title": "existing",
"status": "feishu_write_failed",
},
)
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
def capture_write(*args, **kwargs):
write_call["args"] = args
write_call["kwargs"] = kwargs
return True
def capture_status(style, status):
status_call.update({"style": style, "status": status})
return True
def fail_import(*_args, **_kwargs):
raise AssertionError("an existing document must not be imported again")
monkeypatch.setattr(module, "write_to_target_table", capture_write)
monkeypatch.setattr(module, updater_name, capture_status)
monkeypatch.setattr(module, creator_name, fail_import)
processor = getattr(module, processor_name)
arguments = ("宙斯", ("base", "table", "url"))
if module is weekly_summary_all:
arguments += (False,)
result = processor(*arguments)
assert result["status"] == "ok"
assert result["doc_url"] == existing_url
assert result["reused_document"] is True
assert write_call["args"][5] == existing_url
assert status_call == {"style": "宙斯", "status": "ok"}
if module is monthly_summary_all:
assert write_call["kwargs"] == {
"time_label": monthly_summary_all.TIME_LABEL,
"report_label": "月度汇总",
}
def test_business_date_drives_default_report_periods(monkeypatch):
for name in ("WEEK_SINCE", "WEEK_UNTIL", "MONTH_SINCE", "MONTH_UNTIL"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-09")
assert report._compute_last_month() == (date(2026, 7, 1), date(2026, 7, 31))
assert weekly_summary_all._resolve_week_period() == (
date(2026, 7, 27),
date(2026, 8, 2),
)
assert monthly_summary_all._resolve_month_period() == (
date(2026, 7, 1),
date(2026, 7, 31),
)
def test_explicit_period_overrides_take_priority_over_business_date(monkeypatch):
monkeypatch.setenv("GYXX_BUSINESS_DATE", "not-a-date")
monkeypatch.setenv("WEEK_SINCE", "2026-06-01")
monkeypatch.setenv("WEEK_UNTIL", "2026-06-07")
monkeypatch.setenv("MONTH_SINCE", "2026-05-01")
monkeypatch.setenv("MONTH_UNTIL", "2026-05-31")
assert weekly_summary_all._resolve_week_period() == (
date(2026, 6, 1),
date(2026, 6, 7),
)
assert monthly_summary_all._resolve_month_period() == (
date(2026, 5, 1),
date(2026, 5, 31),
)
assert report._compute_last_month() == (date(2026, 5, 1), date(2026, 5, 31))
def test_invalid_business_date_and_unpaired_overrides_fail_closed(monkeypatch):
for name in ("WEEK_SINCE", "WEEK_UNTIL", "MONTH_SINCE", "MONTH_UNTIL"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026/08/09")
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
report._compute_last_month()
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
weekly_summary_all._resolve_week_period()
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
monthly_summary_all._resolve_month_period()
monkeypatch.delenv("GYXX_BUSINESS_DATE")
monkeypatch.setenv("WEEK_SINCE", "2026-06-01")
with pytest.raises(ValueError, match="configured together"):
weekly_summary_all._resolve_week_period()
monkeypatch.delenv("WEEK_SINCE")
monkeypatch.setenv("MONTH_SINCE", "2026-05-01")
with pytest.raises(ValueError, match="configured together"):
report._compute_last_month()
with pytest.raises(ValueError, match="configured together"):
monthly_summary_all._resolve_month_period()
class _RecordingCursor:
def __init__(self):
self.statements = []
def execute(self, statement, params):
self.statements.append((statement, params))
def fetchone(self):
return (7,)
class _RecordingConnection:
def __init__(self):
self.cursor_instance = _RecordingCursor()
def cursor(self):
return self.cursor_instance
def commit(self):
return None
def close(self):
return None
def __enter__(self):
return self
def __exit__(self, *_args):
return False
@pytest.mark.parametrize(
("module", "save_name", "table_name"),
(
(report, "save_to_db", "cmt_creator_report"),
(weekly_summary_all, "save_to_db", "cmt_weekly_summary"),
(monthly_summary_all, "save_to_db_monthly", "cmt_monthly_summary"),
),
)
def test_empty_upsert_values_preserve_existing_document_identity(
monkeypatch,
module,
save_name,
table_name,
):
connection = _RecordingConnection()
if module is report:
monkeypatch.setattr(report, "get_db_config", lambda: {})
monkeypatch.setattr(report.psycopg, "connect", lambda **_kwargs: connection)
saved = report.save_to_db(
"summary",
[],
{},
{},
"2026-07-01",
"2026-07-31",
None,
"doc_ambiguous",
)
else:
monkeypatch.setattr(module, "dry_run_global", False)
monkeypatch.setattr(db, "get_conn", lambda: connection)
saved = getattr(module, save_name)(
"宙斯",
[],
"summary",
None,
None,
"doc_ambiguous",
)
assert saved is True
upsert = connection.cursor_instance.statements[-1][0]
assert "COALESCE" in upsert
assert "NULLIF(EXCLUDED.doc_url, '')" in upsert
assert f"{table_name}.doc_url" in upsert
assert "NULLIF(EXCLUDED.doc_title, '')" in upsert
assert f"{table_name}.doc_title" in upsert
@pytest.mark.parametrize(
(
"module",
"function_name",
"args",
"expected_title",
"source_filename",
"operation_prefix",
),
(
(
report,
"create_feishu_doc",
("report", "2026-07-01", "2026-07-31"),
"达人合作数据筛选与报价分析报告(2026-07-01~07-31",
"creator_report_doc.md",
"content-creator-report-",
),
(
weekly_summary_all,
"create_feishu_doc",
("宙斯", "summary"),
"宙斯 周笔记汇总(2026-07-27~08-02",
"weekly_summary_doc.md",
"content-weekly-summary-",
),
(
monthly_summary_all,
"create_feishu_doc_monthly",
("宙斯", "summary"),
"宙斯 月度汇总(2026-07-01~07-31",
"monthly_summary_doc.md",
"content-monthly-summary-",
),
),
)
def test_feishu_doc_import_preserves_title_body_and_root_target(
monkeypatch,
tmp_path,
module,
function_name,
args,
expected_title,
source_filename,
operation_prefix,
):
captured = {}
def fake_import(markdown, **kwargs):
captured["markdown"] = markdown
captured.update(kwargs)
return SimpleNamespace(url="https://example/doc")
monkeypatch.setattr(
module,
"PATHS",
SimpleNamespace(tmp_root=tmp_path, evidence_root=tmp_path / "evidence"),
)
monkeypatch.setattr(module, "import_markdown_document", fake_import)
if module is weekly_summary_all:
monkeypatch.setattr(module, "TIME_LABEL", "2026-07-27~08-02")
elif module is monthly_summary_all:
monkeypatch.setattr(module, "TIME_LABEL", "2026-07-01~07-31")
getattr(module, function_name)(*args)
assert captured["markdown"] == args[0 if module is report else 1]
assert captured["title"] == expected_title
assert captured["source_filename"] == source_filename
assert captured["working_directory"] == tmp_path
assert captured["audit_directory"] == tmp_path / "evidence" / "feishu_doc_import"
assert captured["identity"] == "user"
assert captured.get("folder_token") is None
assert captured["operation_id"].startswith(operation_prefix)
assert len(captured["operation_id"].rsplit("-", 1)[1]) == 40
assert captured["resume_only"] is False
@@ -1,5 +1,11 @@
import io
import sys
import tempfile
import unittest
from contextlib import redirect_stdout
from datetime import date
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from gyxx_flow.modules.content_marketing import daily_marketing_report as daily
@@ -47,6 +53,41 @@ class DailyMarketingReportTests(unittest.TestCase):
),
)
def test_disabled_dynamic_route_keeps_report_but_skips_feishu_send(self):
facts = ([], [], [], [], {})
with tempfile.TemporaryDirectory() as directory, \
patch.object(daily, "PATHS", SimpleNamespace(exports_root=Path(directory))), \
patch.object(daily, "_latest_report_date", return_value=date(2026, 8, 7)), \
patch.object(daily, "load_report_facts", return_value=facts), \
patch.object(daily, "build_daily_prompt", return_value="prompt"), \
patch.object(daily, "call_hermes_analyzer", return_value="report"), \
patch.object(daily, "require_valid_hermes_report", return_value="report"), \
patch.object(daily, "normalize_report", return_value="report"), \
patch.object(daily, "generate_dashboard"), \
patch.object(
daily,
"resolve_notification_route",
return_value=SimpleNamespace(
enabled=False,
open_ids=(),
app_profile="hermes-analyzer",
),
), \
patch.object(daily, "send_daily_report_cards") as send, \
patch.object(sys, "argv", ["daily_marketing_report.py", "--send"]):
output = io.StringIO()
with redirect_stdout(output):
self.assertEqual(daily.main(), 0)
report_path = (
Path(directory)
/ "summary"
/ "daily_marketing_report_2026-08-07.md"
)
self.assertTrue(report_path.is_file())
self.assertIn("动态配置中禁用", output.getvalue())
send.assert_not_called()
def test_hermes_transport_error_is_not_accepted_as_report(self):
with self.assertRaisesRegex(RuntimeError, "Hermes 分析失败"):
daily.require_valid_hermes_report("API call failed after 3 retries: Connection error.")
@@ -142,6 +183,25 @@ class DailyMarketingReportTests(unittest.TestCase):
self.assertEqual(valid["engagement_count_num"], 2143)
self.assertEqual(valid["publish_status"], "已记录发布时间")
def test_style_note_signal_keeps_complete_count_and_reports_metric_coverage(self):
signals = daily._build_style_signals(
["宙斯"],
[],
[
{"style_name": "宙斯", "exposure_count": 1000},
{"style_name": "宙斯", "exposure_count": None},
],
[],
[],
[],
)
self.assertEqual(signals["宙斯"]["note_count"], 2)
self.assertEqual(signals["宙斯"]["recent_note_exposure"], 1000)
self.assertEqual(signals["宙斯"]["metric_note_count"], 1)
self.assertEqual(signals["宙斯"]["metric_missing_count"], 1)
self.assertEqual(signals["宙斯"]["metric_coverage_rate"], 0.5)
def test_load_report_facts_tracks_all_styles_for_report_date(self):
metrics = {
"宙斯": {"light": "🟢"},
@@ -156,6 +216,7 @@ class DailyMarketingReportTests(unittest.TestCase):
patch.object(daily, "load_style_images", return_value={}), \
patch.object(daily, "load_style_categories", return_value={}), \
patch.object(daily, "_cooperation_context", return_value=[]), \
patch.object(daily, "_note_inventory_context", return_value=[]), \
patch.object(daily, "_style_comment_context", return_value=[]), \
patch.object(daily, "_review_context", return_value=[]), \
patch.object(daily, "_creative_context", return_value=[]), \
@@ -185,6 +246,7 @@ class DailyMarketingReportTests(unittest.TestCase):
patch.object(daily, "load_style_images", return_value={}), \
patch.object(daily, "load_style_categories", return_value={}), \
patch.object(daily, "_cooperation_context", return_value=[]), \
patch.object(daily, "_note_inventory_context", return_value=[]), \
patch.object(daily, "_style_comment_context", return_value=[]), \
patch.object(daily, "_review_context", return_value=[]), \
patch.object(daily, "_creative_context", return_value=[]), \
@@ -49,6 +49,83 @@ class DailyReportCardTests(unittest.TestCase):
self.assertNotIn("![全域种草数据看板]", analysis_text)
self.assertIn("波塞冬:继续观察", analysis_text)
self.assertIn("退款订单比上升", analysis_text)
self.assertEqual(analysis["schema"], "2.0")
self.assertEqual(analysis["config"]["width_mode"], "fill")
self.assertEqual(
[item["tag"] for item in analysis["body"]["elements"]],
["column_set", "collapsible_panel", "column_set"],
)
self.assertNotEqual(
analysis["body"]["elements"],
[{"tag": "markdown", "content": report}],
)
def test_analysis_card_structures_full_five_section_report(self):
report = """品牌SKU营销运营日报
日期:2026年8月1日
一、今日经营概览
销量:整体下降。
今日核心结论:流量收缩,转化承压。
二、SKU表现
|SKU|今日销量|综合判断|建议动作|
|---|---|---|---|
|星云2|128|继续放大|维持资源|
|星迹2|33|先修承接|优化详情页|
注:曝光为累计值。
三、重点SKU分析
星云2:经营表现健康,继续放大。
四、达人营销分析
最佳渠道:抖音。
五、明日运营建议
1. 优化星迹2详情页。
2. 补充逐星内容。
"""
card = build_analysis_card("2026-08-01", report)
serialized = json.dumps(card, ensure_ascii=False)
self.assertEqual(
[item["tag"] for item in card["body"]["elements"]],
[
"column_set",
"table",
"collapsible_panel",
"collapsible_panel",
"column_set",
],
)
self.assertEqual(len(card["body"]["elements"][1]["rows"]), 2)
sku_panel, marketing_panel = card["body"]["elements"][2:4]
self.assertTrue(sku_panel["expanded"])
self.assertTrue(marketing_panel["expanded"])
self.assertEqual(sku_panel["background_color"], "blue-50")
self.assertEqual(marketing_panel["background_color"], "violet-50")
self.assertNotEqual(
sku_panel["background_color"], marketing_panel["background_color"]
)
self.assertEqual(sku_panel["header"]["title"]["tag"], "markdown")
self.assertIn(
"**<font color='blue'>重点SKU分析</font>**",
sku_panel["header"]["title"]["content"],
)
self.assertIn(
"**<font color='violet'>达人与内容营销分析</font>**",
marketing_panel["header"]["title"]["content"],
)
for phrase in (
"流量收缩,转化承压",
"曝光为累计值",
"继续放大",
"最佳渠道:抖音",
"补充逐星内容",
):
self.assertIn(phrase, serialized)
self.assertNotIn("button", serialized)
def test_build_card_uses_card_2_schema_and_dashboard(self):
report = """一、核心经营结论
@@ -139,25 +216,99 @@ class DailyReportCardTests(unittest.TestCase):
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.subprocess.run"
"daily_report_card.send_lark_bot_message"
)
def test_send_card_to_all_recipients(self, run):
run.side_effect = [
type("Result", (), {"returncode": 0, "stdout": json.dumps({"message_id": f"om_{i}"}), "stderr": ""})()
def test_send_card_to_all_recipients(self, send):
send.side_effect = [
{"message_id": f"om_{i}"}
for i in range(3)
]
recipients = ["ou_one", "ou_two", "ou_three"]
results = send_card_to_recipients({"schema": "2.0"}, recipients, "2026-07-15")
self.assertEqual([item["message_id"] for item in results], ["om_0", "om_1", "om_2"])
self.assertEqual(run.call_count, 3)
commands = [call.args[0] for call in run.call_args_list]
self.assertEqual([cmd[cmd.index("--user-id") + 1] for cmd in commands], recipients)
self.assertTrue(all("--msg-type" in cmd and "interactive" in cmd for cmd in commands))
keys = [cmd[cmd.index("--idempotency-key") + 1] for cmd in commands]
self.assertEqual(send.call_count, 3)
calls = send.call_args_list
self.assertEqual([call.kwargs["user_id"] for call in calls], recipients)
self.assertTrue(all(call.kwargs["msg_type"] == "interactive" for call in calls))
self.assertTrue(all(call.kwargs["profile"] == "hermes-analyzer" for call in calls))
keys = [call.kwargs["idempotency_key"] for call in calls]
self.assertTrue(all(key.startswith("daily-2026-07-15-") for key in keys))
self.assertTrue(all(len(key) <= 50 for key in keys))
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.send_lark_bot_message"
)
def test_send_card_rejects_success_exit_without_message_id(self, send):
send.return_value = {"ok": True}
with self.assertRaisesRegex(RuntimeError, "message_id"):
send_card_to_recipients(
{"schema": "2.0"},
["ou_owner"],
"2026-08-09",
)
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.send_lark_bot_message"
)
def test_sender_uses_fixed_user_profile_and_hashes_full_recipient_identity(self, send):
send.side_effect = [
{"message_id": f"om_{index}"}
for index in range(2)
]
recipients = ["ou_first_sharedsuffix", "ou_second_sharedsuffix"]
send_card_to_recipients(
{"schema": "2.0"},
recipients,
"2026-08-07",
)
calls = send.call_args_list
self.assertTrue(
all(call.kwargs["profile"] == "hermes-analyzer" for call in calls)
)
keys = [
call.kwargs["idempotency_key"]
for call in calls
]
self.assertEqual(len(set(keys)), 2)
with self.assertRaisesRegex(ValueError, "hermes-analyzer"):
send_card_to_recipients(
{"schema": "2.0"},
["ou_owner"],
"2026-08-07",
app_profile="dynamic-analyzer",
)
@patch(
"gyxx_flow.modules.content_marketing.data.tools."
"daily_report_card.send_lark_bot_message"
)
def test_stable_fingerprint_ignores_rotating_uploaded_image_key(self, send):
send.side_effect = [
{"message_id": f"om_{index}"}
for index in range(2)
]
for image_key in ("img_first_upload", "img_second_upload"):
send_card_to_recipients(
build_dashboard_card("2026-08-07", [], image_key),
["ou_owner"],
"2026-08-07",
message_kind="dashboard",
idempotency_fingerprint="same-dashboard-bytes-and-metrics",
)
keys = [
call.kwargs["idempotency_key"]
for call in send.call_args_list
]
self.assertEqual(keys[0], keys[1])
@patch(
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.send_card_to_recipients"
)
@@ -177,13 +328,16 @@ class DailyReportCardTests(unittest.TestCase):
]
recipients = ["ou_one", "ou_two"]
results = send_daily_report_cards(
"2026-07-17",
"第一部分 整体经营分析\n完整分析内容",
[{"light": "🟢"}],
Path("charts/dashboard.png"),
recipients,
)
with tempfile.TemporaryDirectory() as tmp:
chart_path = Path(tmp) / "dashboard.png"
Image.new("RGB", (4, 4), "white").save(chart_path)
results = send_daily_report_cards(
"2026-07-17",
"第一部分 整体经营分析\n完整分析内容",
[{"light": "🟢"}],
chart_path,
recipients,
)
self.assertEqual(
[item["message_id"] for item in results],
@@ -195,6 +349,10 @@ class DailyReportCardTests(unittest.TestCase):
self.assertIn("img_v3_test", json.dumps(first_card, ensure_ascii=False))
self.assertIn("完整分析内容", json.dumps(second_card, ensure_ascii=False))
self.assertEqual(send.call_args_list[0].kwargs["message_kind"], "dashboard")
self.assertIn(
"idempotency_fingerprint",
send.call_args_list[0].kwargs,
)
self.assertEqual(send.call_args_list[1].kwargs["message_kind"], "analysis")
@@ -0,0 +1,99 @@
import json
import pytest
from gyxx_flow.adapters.direct_content_llm import (
ContentLLMConfig,
ContentLLMConfigurationError,
ContentLLMResponseError,
call_content_analyzer,
load_content_llm_config,
)
def _config() -> ContentLLMConfig:
return ContentLLMConfig(
endpoint="https://api.minimaxi.com/anthropic/v1/messages",
credential="test-only-secret",
model="MiniMax-M3",
thinking_mode="disabled",
)
def _response(*blocks: dict[str, object]) -> bytes:
return json.dumps(
{
"type": "message",
"content": list(blocks),
"usage": {"input_tokens": 3, "output_tokens": 4},
}
).encode()
def test_config_uses_anthropic_endpoint_and_redacts_key(monkeypatch) -> None:
monkeypatch.setenv(
"CONTENT_ANALYSIS_LLM_BASE_URL", "https://api.minimaxi.com/anthropic"
)
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_API_KEY", "test-only-secret")
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_MODEL", "MiniMax-M3")
config = load_content_llm_config()
assert config.endpoint == "https://api.minimaxi.com/anthropic/v1/messages"
assert config.model == "MiniMax-M3"
assert "test-only-secret" not in repr(config)
def test_call_sends_anthropic_message_and_ignores_thinking_block() -> None:
observed: dict[str, object] = {}
def transport(url, headers, body, timeout):
payload = json.loads(body)
observed.update(
url=url,
headers=headers,
payload=payload,
timeout=timeout,
)
return _response(
{"type": "thinking", "thinking": "hidden reasoning"},
{"type": "text", "text": "最终报告"},
)
result = call_content_analyzer(
"system prompt",
"user prompt",
transport=transport,
config=_config(),
)
assert result == "最终报告"
assert observed["url"] == "https://api.minimaxi.com/anthropic/v1/messages"
assert observed["headers"]["X-Api-Key"] == "test-only-secret"
assert observed["headers"]["anthropic-version"] == "2023-06-01"
assert observed["payload"]["model"] == "MiniMax-M3"
assert observed["payload"]["system"] == "system prompt"
assert observed["payload"]["messages"] == [
{"role": "user", "content": "user prompt"}
]
assert observed["payload"]["thinking"] == {"type": "disabled"}
def test_call_rejects_openai_response_shape() -> None:
with pytest.raises(ContentLLMResponseError, match="response shape"):
call_content_analyzer(
"system prompt",
"user prompt",
transport=lambda *_args: b'{"choices": []}',
config=_config(),
)
def test_config_rejects_local_gateway(monkeypatch) -> None:
monkeypatch.setenv(
"CONTENT_ANALYSIS_LLM_BASE_URL", "http://127.0.0.1:8642/v1"
)
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_API_KEY", "test-only-secret")
with pytest.raises(ContentLLMConfigurationError, match="local gateway"):
load_content_llm_config()
@@ -0,0 +1,162 @@
import pytest
from gyxx_flow.modules.content_marketing import douyin_comment_scraper as scraper
from gyxx_flow.modules.content_marketing.data.tools import (
batch_rescrape_douyin as batch,
)
class _EvaluatePage:
def __init__(self, result):
self.result = result
self.calls = []
def evaluate(self, script, argument):
self.calls.append((script, argument))
return self.result
def test_bounded_fetch_uses_abort_controller_and_clears_timer(monkeypatch):
page = _EvaluatePage(
{"__gyxx_status": "ok", "payload": {"comments": []}}
)
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
payload = scraper._fetch_json_with_timeout(
page,
"https://example.invalid/comments",
deadline=1000.0,
stage="top-level pagination",
)
assert payload == {"comments": []}
script, argument = page.calls[0]
assert "AbortController" in script
assert "clearTimeout(timer)" in script
assert argument["timeoutMs"] == 30_000
def test_bounded_fetch_timeout_is_an_explicit_transient_failure(monkeypatch):
page = _EvaluatePage({"__gyxx_status": "timeout"})
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
with pytest.raises(scraper.TransientDouyinCommentTimeout) as caught:
scraper._fetch_json_with_timeout(
page,
"https://example.invalid/comments",
deadline=1000.0,
stage="reply pagination",
)
assert isinstance(caught.value, ConnectionError)
assert "reply pagination" in str(caught.value)
def test_scroll_checks_the_note_deadline_before_browser_work(monkeypatch):
page = _EvaluatePage(None)
monkeypatch.setattr(scraper.time, "monotonic", lambda: 901.0)
with pytest.raises(scraper.TransientDouyinCommentTimeout):
scraper.scroll_comments(
page,
max_scrolls=60,
idle_rounds=10,
comments_by_id={},
deadline=900.0,
)
assert page.calls == []
def test_top_and_reply_pagination_share_the_bounded_fetch(monkeypatch):
stages = []
monkeypatch.setattr(scraper.time, "monotonic", lambda: 0.0)
def fetch(_page, _url, *, deadline, stage):
assert deadline == 900.0
stages.append(stage)
return {"comments": [], "has_more": 0, "cursor": 0}
monkeypatch.setattr(scraper, "_fetch_json_with_timeout", fetch)
monkeypatch.setattr(scraper, "collect_from_payload", lambda *_args: None)
page = _EvaluatePage(None)
scraper.fetch_missing_top_comments(
page,
"source-url",
"aweme-id",
{"existing": {}},
{
"comment_url_template": "https://example.invalid/top",
"api_total": 2,
"last_cursor": 1,
},
deadline=900.0,
)
scraper.fetch_missing_replies(
page,
"source-url",
{
"parent": {
"level": "comment",
"reply_count": 1,
"comment_id": "parent-id",
}
},
{
"reply_url_template": "https://example.invalid/reply",
"aweme_id": "aweme-id",
},
deadline=900.0,
)
assert stages == ["top-level pagination", "reply pagination"]
def test_deadline_failure_retries_and_never_replaces_existing_comments(
monkeypatch,
):
attempts = []
class Checkpoint:
completed = []
cleared = False
def __init__(self, _platform):
pass
def contains(self, _note_id, _url):
return False
def mark_completed(self, note_id, url):
self.completed.append((note_id, url))
def clear(self):
self.cleared = True
def scrape(url, *_args):
attempts.append(url)
raise scraper.TransientDouyinCommentTimeout(
"Douyin comment scrape exceeded 900 seconds"
)
monkeypatch.setattr(batch, "CommentBatchCheckpoint", Checkpoint)
monkeypatch.setattr(
batch,
"collect_urls",
lambda: [{"id": 1, "url": "dy-url", "comment_count": 10}],
)
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(
batch.db,
"replace_comments",
lambda *_args, **_kwargs: pytest.fail(
"a timed-out scrape must not replace existing comments"
),
)
assert batch.main() == 1
assert attempts == ["dy-url", "dy-url", "dy-url"]
assert Checkpoint.completed == []
assert Checkpoint.cleared is False
@@ -0,0 +1,399 @@
import json
import subprocess
from types import SimpleNamespace
import pytest
from gyxx_flow.modules.content_marketing.feishu_doc_import import (
FeishuDocImportAmbiguous,
FeishuDocImportError,
FeishuDocImportTimeout,
_parse_cli_json,
deterministic_import_operation_id,
import_markdown_document,
)
def _completed(payload, *, returncode=0, prefix="", suffix=""):
return SimpleNamespace(
returncode=returncode,
stdout=f"{prefix}{json.dumps(payload)}{suffix}",
stderr="",
)
class _FakeClock:
def __init__(self):
self.now = 100.0
def monotonic(self):
return self.now
def sleep(self, seconds):
self.now += seconds
def test_import_ready_result_uses_relative_file_exact_title_and_audits_without_token(
tmp_path,
):
commands = []
title = "宙斯 周笔记汇总(2026-07-27~08-02"
markdown = "# 标题\n\n完整正文"
document_token = "VERY_SECRET_DOCUMENT_TOKEN_123456"
def fake_run(command, **kwargs):
commands.append(command)
source = command[command.index("--file") + 1]
assert source.startswith("./")
assert (kwargs["cwd"] / source.removeprefix("./")).read_text(
encoding="utf-8"
) == markdown
return _completed(
{
"scenario": "import",
"ticket": "ticket-ready",
"ready": True,
"failed": False,
"job_status": 0,
"job_status_label": "success",
"token": document_token,
"url": f"https://example.feishu.cn/docx/{document_token}",
},
prefix="lark-cli notice\n",
suffix="\ntrailing notifier",
)
result = import_markdown_document(
markdown,
title=title,
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="weekly_summary_doc.md",
runner=fake_run,
operation_id="ready-op",
)
assert result.url == f"https://example.feishu.cn/docx/{document_token}"
assert result.document_token == document_token
assert commands[0][1:3] == ["drive", "+import"]
assert commands[0][commands[0].index("--name") + 1] == title
assert commands[0][commands[0].index("--type") + 1] == "docx"
assert commands[0][commands[0].index("--as") + 1] == "user"
assert "--folder-token" not in commands[0]
audit_text = result.audit_path.read_text(encoding="utf-8")
audit = json.loads(audit_text)
assert audit["status"] == "ready"
assert audit["ticket"] == "ticket-ready"
assert audit["url"] == "https://example.feishu.cn/docx/<redacted>"
assert audit["last_result"]["url"] == audit["url"]
assert audit["document_title"] == title
assert audit["folder_target"] == "root"
assert "token" not in audit
assert "document_token" not in audit_text
assert document_token not in audit_text
assert not list((tmp_path / "work").glob("*.md"))
def test_import_persists_ticket_and_polls_task_result_until_ready(tmp_path):
responses = iter(
[
_completed(
{
"scenario": "import",
"ticket": "ticket-async",
"ready": False,
"failed": False,
"timed_out": True,
}
),
_completed(
{
"scenario": "import",
"ticket": "ticket-async",
"ready": False,
"failed": False,
"job_status": 1,
"job_status_label": "processing",
}
),
_completed(
{
"scenario": "import",
"ticket": "ticket-async",
"ready": True,
"failed": False,
"job_status": 0,
"job_status_label": "success",
"token": "doxcnAsyncDocument",
"url": "https://example.feishu.cn/docx/doxcnAsyncDocument",
}
),
]
)
commands = []
def fake_run(command, **_kwargs):
commands.append(command)
if len(commands) == 2:
in_progress = json.loads(
(tmp_path / "audit" / "async-op.json").read_text(encoding="utf-8")
)
assert in_progress["status"] == "processing"
assert in_progress["ticket"] == "ticket-async"
assert in_progress["last_result"]["ready"] is False
return next(responses)
result = import_markdown_document(
"report",
title="精确标题",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=fake_run,
sleeper=lambda _seconds: None,
operation_id="async-op",
)
assert result.ticket == "ticket-async"
assert len(commands) == 3
for command in commands[1:]:
assert command[1:3] == ["drive", "+task_result"]
assert command[command.index("--scenario") + 1] == "import"
assert command[command.index("--ticket") + 1] == "ticket-async"
assert command[command.index("--as") + 1] == "user"
audit = json.loads(result.audit_path.read_text(encoding="utf-8"))
assert audit["poll_attempts"] == 2
assert audit["ticket"] == "ticket-async"
assert audit["last_result"]["ready"] is True
def test_import_reports_failed_task_without_retrying_mutation(tmp_path):
leaked_document_token = "VERY_SECRET_FAILED_DOCUMENT_TOKEN"
leaked_open_id = "ou_SHOULD_NOT_BE_IN_AUDIT"
responses = iter(
[
_completed(
{
"ticket": "ticket-failed",
"ready": False,
"failed": False,
}
),
_completed(
{
"ticket": "ticket-failed",
"ready": False,
"failed": True,
"job_status": -1,
"job_status_label": "failed",
"job_error_msg": (
"conversion failed at "
f"https://example.feishu.cn/docx/{leaked_document_token} "
f'\"token\":\"{leaked_document_token}\" '
f'\"open_id\":\"{leaked_open_id}\"'
),
}
),
]
)
commands = []
def fake_run(command, **_kwargs):
commands.append(command)
return next(responses)
with pytest.raises(FeishuDocImportError, match="conversion failed") as raised:
import_markdown_document(
"report",
title="失败报告",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=fake_run,
sleeper=lambda _seconds: None,
operation_id="failed-op",
)
assert sum(command[1:3] == ["drive", "+import"] for command in commands) == 1
audit_text = raised.value.audit_path.read_text(encoding="utf-8")
audit = json.loads(audit_text)
assert audit["status"] == "failed"
assert audit["ticket"] == "ticket-failed"
assert audit["last_result"]["failed"] is True
assert leaked_document_token not in audit_text
assert leaked_open_id not in audit_text
def test_import_timeout_keeps_ticket_for_auditable_follow_up(tmp_path):
clock = _FakeClock()
responses = iter(
[
_completed(
{"ticket": "ticket-timeout", "ready": False, "failed": False}
),
_completed(
{
"ticket": "ticket-timeout",
"ready": False,
"failed": False,
"job_status_label": "processing",
}
),
]
)
with pytest.raises(FeishuDocImportTimeout) as raised:
import_markdown_document(
"report",
title="超时报告",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=lambda *_args, **_kwargs: next(responses),
clock=clock.monotonic,
sleeper=clock.sleep,
wait_timeout_seconds=1,
poll_interval_seconds=1,
operation_id="timeout-op",
)
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
assert audit["status"] == "timed_out"
assert audit["ticket"] == "ticket-timeout"
assert audit["poll_attempts"] == 1
def test_mutating_import_process_timeout_is_ambiguous_and_never_retried(tmp_path):
commands = []
def time_out(command, **kwargs):
commands.append(command)
assert kwargs["timeout"] == 2
raise subprocess.TimeoutExpired(command, kwargs["timeout"])
with pytest.raises(FeishuDocImportTimeout) as raised:
import_markdown_document(
"report",
title="有限超时报告",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=time_out,
import_command_timeout_seconds=2,
operation_id="command-timeout-op",
)
assert len(commands) == 1
assert commands[0][1:3] == ["drive", "+import"]
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
assert audit["status"] == "ambiguous"
assert audit["ticket"] is None
assert not list((tmp_path / "work").glob("*.md"))
def test_deterministic_audit_resumes_only_known_ticket_without_second_import(tmp_path):
audit_root = tmp_path / "audit"
audit_root.mkdir()
operation_id = "stable-report-op"
audit_path = audit_root / f"{operation_id}.json"
audit_path.write_text(
json.dumps(
{
"audit_schema": "gyxx.feishu-doc-import.v1",
"document_title": "固定周期报告",
"folder_target": "root",
"identity": "user",
"markdown_sha256": "previous-input-digest",
"operation_id": operation_id,
"poll_attempts": 1,
"status": "timed_out",
"target_type": "docx",
"ticket": "known-ticket",
}
),
encoding="utf-8",
)
commands = []
def fake_run(command, **_kwargs):
commands.append(command)
return _completed(
{
"ticket": "known-ticket",
"ready": True,
"failed": False,
"token": "doxcnRecoveredDocument",
"url": "https://example.feishu.cn/docx/doxcnRecoveredDocument",
}
)
result = import_markdown_document(
"possibly regenerated report",
title="固定周期报告",
working_directory=tmp_path / "work",
audit_directory=audit_root,
source_filename="report.md",
runner=fake_run,
operation_id=operation_id,
)
assert result.url.endswith("/doxcnRecoveredDocument")
assert len(commands) == 1
assert commands[0][1:3] == ["drive", "+task_result"]
assert all(command[1:3] != ["drive", "+import"] for command in commands)
audit_text = audit_path.read_text(encoding="utf-8")
audit = json.loads(audit_text)
assert audit["status"] == "ready"
assert audit["resume_count"] == 1
assert audit["resume_input_changed"] is True
assert "doxcnRecoveredDocument" not in audit_text
def test_resume_only_without_ticket_audit_refuses_mutating_import(tmp_path):
commands = []
with pytest.raises(FeishuDocImportAmbiguous, match="缺少可对账") as raised:
import_markdown_document(
"report",
title="待对账报告",
working_directory=tmp_path / "work",
audit_directory=tmp_path / "audit",
source_filename="report.md",
runner=lambda command, **_kwargs: commands.append(command),
operation_id="missing-ticket-op",
resume_only=True,
)
assert commands == []
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
assert audit["status"] == "ambiguous"
assert audit["ticket"] is None
def test_deterministic_operation_id_is_stable_and_period_specific():
first = deterministic_import_operation_id(
"content-weekly-summary", "宙斯", "2026-07-27", "2026-08-02"
)
same = deterministic_import_operation_id(
"content-weekly-summary", "宙斯", "2026-07-27", "2026-08-02"
)
other_style = deterministic_import_operation_id(
"content-weekly-summary", "赫拉", "2026-07-27", "2026-08-02"
)
assert first == same
assert first != other_style
assert first.startswith("content-weekly-summary-")
assert len(first.rsplit("-", 1)[1]) == 40
def test_cli_json_parser_accepts_noise_and_rejects_invalid_output():
assert _parse_cli_json(
'notice {"notice":"update"}\n{"ready":false,"ticket":"abc"}\ntrailing'
) == {
"ready": False,
"ticket": "abc",
}
with pytest.raises(ValueError, match="no JSON"):
_parse_cli_json("plain text", "also plain text")
@@ -1,3 +1,7 @@
from types import SimpleNamespace
import pytest
from gyxx_flow.modules.content_marketing.data.tools import (
friday_relogin_parallel as relogin,
)
@@ -43,6 +47,434 @@ def test_failed_platforms_are_grouped_by_their_responsible_recipient():
}
def test_legacy_route_keeps_the_existing_per_platform_recipients(monkeypatch):
observed = {}
def fake_resolver(workflow_id, defaults):
observed["workflow_id"] = workflow_id
observed["defaults"] = defaults
return relogin.ResolvedNotificationRoute(
configured=False,
enabled=True,
app_profile=None,
open_ids=tuple(defaults),
)
monkeypatch.setattr(relogin, "resolve_notification_route", fake_resolver)
route = relogin.resolve_weekly_notification_route()
assert observed == {
"workflow_id": "content.relogin.weekly",
"defaults": (
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
"ou_7ad5fc8012e2f741afc5346e05ffd447",
),
}
assert relogin.recipients_for_platform("pgy", route) == (
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
)
assert relogin.recipients_for_platform("douyin", route) == (
"ou_7ad5fc8012e2f741afc5346e05ffd447",
)
def test_disabled_route_still_runs_relogin_without_sending(monkeypatch):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=False,
app_profile="hermes-analyzer",
open_ids=(),
)
rounds = []
monkeypatch.setattr(
relogin,
"current_acceptance_policy",
lambda: SimpleNamespace(enabled=False),
)
monkeypatch.setattr(
relogin,
"resolve_weekly_notification_route",
lambda _override=None: route,
)
def fake_round_factory(_args, resolved_route, notification_failures):
assert resolved_route is route
assert notification_failures == []
def run_round(names, attempt):
rounds.append((tuple(names), attempt))
return {name: True for name in names}
return run_round
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
monkeypatch.setattr(
relogin,
"_send_lark",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
AssertionError("disabled route must not send Lark messages")
),
)
monkeypatch.setattr(
relogin.sys,
"argv",
["friday_relogin_parallel.py", "--platform", "pgy"],
)
assert relogin.main() == 0
assert rounds == [(("pgy",), 1)]
def test_enabled_route_sends_qr_text_and_image_to_every_recipient(
monkeypatch,
tmp_path,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
sent = []
image = tmp_path / "qrcode.png"
monkeypatch.setattr(
relogin,
"_send_lark",
lambda recipient, text=None, image=None, app_profile=None: sent.append(
(recipient, text, image, app_profile)
),
)
failures = relogin._send_qr_notification(
route,
"pgy",
override=None,
text="请扫码",
image=image,
)
assert failures == ()
assert sent == [
("ou_first123", "请扫码", None, "hermes-analyzer"),
("ou_first123", None, image, "hermes-analyzer"),
("ou_second456", "请扫码", None, "hermes-analyzer"),
("ou_second456", None, image, "hermes-analyzer"),
]
assert relogin.group_platforms_by_notification_recipient(
["pgy", "douyin"], route
) == {
"ou_first123": ["pgy", "douyin"],
"ou_second456": ["pgy", "douyin"],
}
def test_qr_delivery_aggregates_errors_and_attempts_every_recipient(
monkeypatch,
tmp_path,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
attempts = []
image = tmp_path / "qrcode.png"
def fake_send(recipient, text=None, image=None, app_profile=None):
stage = "qr_text" if text is not None else "qr_image"
attempts.append((recipient, stage, app_profile))
if (recipient, stage) == ("ou_first123", "qr_text"):
raise RuntimeError("first text rejected")
if (recipient, stage) == ("ou_second456", "qr_image"):
raise RuntimeError("second image rejected")
monkeypatch.setattr(relogin, "_send_lark", fake_send)
failures = relogin._send_qr_notification(
route,
"pgy",
override=None,
text="请扫码",
image=image,
)
assert attempts == [
("ou_first123", "qr_text", "hermes-analyzer"),
("ou_first123", "qr_image", "hermes-analyzer"),
("ou_second456", "qr_text", "hermes-analyzer"),
("ou_second456", "qr_image", "hermes-analyzer"),
]
assert failures == (
relogin.NotificationDeliveryFailure(
stage="qr_text",
platform="pgy",
recipient="ou_first123",
detail="first text rejected",
),
relogin.NotificationDeliveryFailure(
stage="qr_image",
platform="pgy",
recipient="ou_second456",
detail="second image rejected",
),
)
def test_failure_summary_aggregates_every_recipient_error(monkeypatch):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
attempts = []
def fail_send(recipient, **_kwargs):
attempts.append(recipient)
raise RuntimeError(f"rejected {recipient}")
monkeypatch.setattr(relogin, "_send_lark", fail_send)
failures = relogin._send_failure_summaries(
route,
["pgy", "douyin"],
override=None,
max_attempts=3,
)
assert attempts == ["ou_first123", "ou_second456"]
assert [failure.stage for failure in failures] == [
"failure_summary",
"failure_summary",
]
assert [failure.recipient for failure in failures] == attempts
assert [failure.platform for failure in failures] == [
"pgy,douyin",
"pgy,douyin",
]
def test_main_returns_failure_and_reports_all_notification_errors(
monkeypatch,
capsys,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
monkeypatch.setattr(
relogin,
"current_acceptance_policy",
lambda: SimpleNamespace(enabled=False),
)
monkeypatch.setattr(
relogin,
"resolve_weekly_notification_route",
lambda _override=None: route,
)
def fake_round_factory(_args, _route, notification_failures):
notification_failures.extend(
(
relogin.NotificationDeliveryFailure(
"qr_text", "pgy", "ou_first123", "text rejected"
),
relogin.NotificationDeliveryFailure(
"qr_image", "pgy", "ou_second456", "image rejected"
),
)
)
return lambda names, _attempt: {name: True for name in names}
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
monkeypatch.setattr(
relogin.sys,
"argv",
["friday_relogin_parallel.py", "--platform", "pgy"],
)
assert relogin.main() == 2
output = capsys.readouterr().out
assert "Final relogin status: {'pgy': True}" in output
assert "Final notification status: failed (2 deliveries)" in output
assert "recipient=ou_first123 error=text rejected" in output
assert "recipient=ou_second456 error=image rejected" in output
def test_round_completes_relogin_after_notification_failure(monkeypatch, tmp_path):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
args = SimpleNamespace(
round_timeout=10,
screenshot_delay=0,
no_send=False,
recipient=None,
max_attempts=1,
)
failures = []
attempts = []
waits = []
class Process:
pid = 1234
def wait(self, timeout):
waits.append(timeout)
return 0
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
monkeypatch.setattr(
relogin,
"binding_from_environment",
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
)
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(relogin, "_login_windows", lambda: {"pgy": 101})
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
monkeypatch.setattr(relogin, "_capture_window", lambda _hwnd, _output: None)
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: True)
def fake_send(recipient, text=None, image=None, **_kwargs):
stage = "qr_text" if text is not None else "qr_image"
attempts.append((recipient, stage))
if (recipient, stage) == ("ou_first123", "qr_text"):
raise RuntimeError("temporary send failure")
monkeypatch.setattr(relogin, "_send_lark", fake_send)
run_round = relogin._run_round_factory(args, route, failures)
assert run_round(["pgy"], 1) == {"pgy": True}
assert waits == [100]
assert attempts == [
("ou_first123", "qr_text"),
("ou_first123", "qr_image"),
("ou_second456", "qr_text"),
("ou_second456", "qr_image"),
]
assert failures == [
relogin.NotificationDeliveryFailure(
stage="qr_text",
platform="pgy",
recipient="ou_first123",
detail="temporary send failure",
)
]
@pytest.mark.parametrize(
("returncode", "cookie_valid", "expected_status", "expected_failure_count"),
(
(0, True, True, 0),
(1, False, False, 4),
),
)
def test_missing_qr_window_is_classified_after_relogin_result(
monkeypatch,
tmp_path,
returncode,
cookie_valid,
expected_status,
expected_failure_count,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
args = SimpleNamespace(
round_timeout=10,
screenshot_delay=0,
no_send=False,
recipient=None,
max_attempts=1,
)
failures = []
class Process:
pid = 1234
def wait(self, timeout):
assert timeout == 100
return returncode
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
monkeypatch.setattr(
relogin,
"binding_from_environment",
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
)
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(relogin, "_login_windows", lambda: {})
monkeypatch.setattr(relogin, "_window_for_process", lambda _pid: None)
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: cookie_valid)
monkeypatch.setattr(
relogin,
"_send_lark",
lambda *_args, **_kwargs: pytest.fail(
"no QR window must not start a send before child classification"
),
)
run_round = relogin._run_round_factory(args, route, failures)
assert run_round(["pgy"], 1) == {"pgy": expected_status}
assert len(failures) == expected_failure_count
assert {failure.stage for failure in failures} <= {"qr_text", "qr_image"}
assert all(
failure.detail == "no visible QR window and relogin did not succeed"
for failure in failures
)
def test_lark_send_uses_shared_analyzer_user_sender_for_text_and_image(
monkeypatch,
tmp_path,
):
calls = []
image = tmp_path / "qrcode.png"
image.write_bytes(b"png")
monkeypatch.setattr(relogin, "ANALYZER_DIR", tmp_path)
monkeypatch.setenv("HERMES_ANALYZER_TOKEN", "must-not-leak")
def fake_send(**kwargs):
calls.append(kwargs)
return {"message_id": f"om_{len(calls)}"}
monkeypatch.setattr(relogin, "send_lark_bot_message", fake_send)
receipts = relogin._send_lark(
"ou_recipient123",
text="登录提醒",
image=image,
)
assert [receipt["message_id"] for receipt in receipts] == ["om_1", "om_2"]
assert calls[0]["user_id"] == "ou_recipient123"
assert calls[0]["text"] == "登录提醒"
assert "image" not in calls[0]
assert calls[1]["user_id"] == "ou_recipient123"
assert calls[1]["image"] == image
assert "text" not in calls[1]
assert all(call["profile"] == "hermes-analyzer" for call in calls)
assert all("HERMES_ANALYZER_TOKEN" not in call["env"] for call in calls)
def test_login_windows_are_classified_by_title():
assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili"
assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy"
@@ -169,6 +169,73 @@ def test_xingtu_search_and_open_creator_submits_only_one_search(monkeypatch):
assert calls == [("同一达人", "creator-1")]
def test_creator_search_attempts_id_then_nickname_and_skips_blank_id():
assert xingtu._creator_search_attempts("Rm.", "Ruan97") == [
("ID", "Ruan97", "Ruan97"),
("昵称", "Rm.", None),
]
assert xingtu._creator_search_attempts("Rm.", " ") == [
("昵称", "Rm.", None),
]
def test_xingtu_primary_retries_only_unmatched_tasks_through_buyin(monkeypatch):
xingtu.configure_source("xingtu")
styles = [{"index": 1, "name": "款式A"}]
context = {"index": 1}
tasks = [
{"record_id": "hit", "creator_name": "达人A", "creator_id": "id-a", "style_context": context},
{"record_id": "miss", "creator_name": "达人B", "creator_id": "id-b", "style_context": context},
]
summaries = {
1: {
"style": "款式A",
"index": 1,
"total": 2,
"matched": 0,
"filled": 0,
"skipped_no_pubtime": 0,
"results": [],
}
}
calls = []
def fake_collect(*_args, **_kwargs):
return tasks, summaries
def fake_scrape(*_args, **kwargs):
source = xingtu.CONTENT_SOURCE
selected = kwargs["task_override"]
calls.append((source, [task["record_id"] for task in selected], kwargs.get("replace_existing", False)))
for task in selected:
matched = source == "buyin" or task["record_id"] == "hit"
xingtu.upsert_result(summaries[1], {
"record_id": task["record_id"],
"matched": matched,
"status": xingtu.SUCCESS if matched else xingtu.RETRYABLE_FAILURE,
"reason": None if matched else "title_unmatched",
})
return [summaries[1]]
monkeypatch.setattr(xingtu, "collect_tasks_across_styles", fake_collect)
monkeypatch.setattr(xingtu, "scrape_styles", fake_scrape)
try:
result = xingtu.scrape_styles_with_fallback(
styles, 1, False, None, True,
)
finally:
xingtu.configure_source("xingtu")
assert calls == [
("xingtu", ["hit", "miss"], False),
("buyin", ["miss"], True),
]
rows = {row["record_id"]: row for row in result[0]["results"]}
assert rows["hit"]["collection_source"] == "xingtu"
assert rows["miss"]["collection_source"] == "buyin_fallback"
assert rows["miss"]["fallback_from"] == "xingtu"
def test_xingtu_search_budget_blocks_duplicate_without_failure_circuit():
budget = xingtu.CreatorSearchBudget(max_consecutive_failures=3)
@@ -291,20 +358,22 @@ def test_scheduled_collection_has_no_automatic_retry_and_syncs_once():
)
def test_daily_collection_uses_dedicated_self_operated_pipeline():
def test_daily_collection_includes_self_operated_pipeline():
steps = _workflow("content.metrics.daily").steps
entries = [step.entry for step in steps]
run_step = next(step for step in steps if step.entry == "run_all.py")
assert "--daily-scope" in run_step.args
assert "--include-self-operated" not in run_step.args
assert "data/tools/refresh_self_mapping.py" in entries
assert "self_bilibili_scraper.py" in entries
assert "chanmama_scraper.py" in entries
refresh_index = entries.index("data/tools/refresh_self_mapping.py")
bili_index = entries.index("self_bilibili_scraper.py")
douyin_index = entries.index("chanmama_scraper.py")
assert refresh_index < bili_index < douyin_index
assert entries.index("run_all.py") < entries.index(
"data/tools/refresh_self_mapping.py"
)
assert entries.index("data/tools/refresh_self_mapping.py") < entries.index(
"self_bilibili_scraper.py"
)
assert entries.index("self_bilibili_scraper.py") < entries.index(
"chanmama_scraper.py"
)
@pytest.mark.parametrize(
@@ -0,0 +1,194 @@
from __future__ import annotations
from gyxx_flow.modules.content_marketing.data.tools import sync_notes_master as sync
def _table() -> dict:
return {
"name": "宙斯",
"base_token": "base-token",
"table_id": "tbl-source",
"field_map": {
"note_title": {"field_id": "title"},
"publish_time": {"field_id": "published"},
"note_url": {"field_id": "url"},
"platform": {"field_id": "platform"},
"creator_name": {"field_id": "creator"},
},
}
def test_parse_records_counts_only_complete_note_rows() -> None:
rows = sync.parse_records(
_table(),
[
{
"record_id": "complete",
"title": "通勤双肩包实测",
"published": "2026-08-11 09:20:00",
"url": "[笔记](https://www.xiaohongshu.com/explore/abc)",
"platform": "小红书",
"creator": "达人甲",
},
{
"record_id": "missing-publish-time",
"title": "只有标题和链接",
"published": None,
"url": "https://www.douyin.com/video/123",
"platform": "抖音",
"creator": "达人乙",
},
],
self_operated=False,
)
assert rows[0].is_countable is True
assert rows[0].platform == "xiaohongshu"
assert rows[0].publish_time == "2026-08-11 09:20:00+08:00"
assert rows[0].url == "https://www.xiaohongshu.com/explore/abc"
assert rows[1].is_countable is False
def test_platform_can_be_inferred_from_url_when_source_cell_is_empty() -> None:
rows = sync.parse_records(
_table(),
[{
"record_id": "bili",
"title": "B站开箱",
"published": "2026-08-10",
"url": "https://www.bilibili.com/video/BV1test",
"platform": "",
"creator": "自营号",
}],
self_operated=True,
)
assert rows[0].platform == "bilibili"
assert rows[0].is_countable is True
def test_unrecognized_source_text_is_bounded_to_database_columns() -> None:
rows = sync.parse_records(
_table(),
[{
"record_id": "unexpected-long-values",
"title": "" * 600,
"published": "2026-08-10",
"url": "not-a-platform-url",
"platform": "unexpected-platform-value-" * 4,
"creator": "" * 300,
}],
self_operated=False,
)
assert len(rows[0].platform) == 32
assert len(rows[0].creator_name) == 255
assert len(rows[0].title) == 512
def test_collaboration_fields_parsed_only_for_collaborator_tables() -> None:
table = _table()
table["field_map"].update(
{
"cost": {"field_id": "cost"},
"tracking_no": {"field_id": "tracking"},
"is_paid": {"field_id": "paid"},
"follower_count": {"field_id": "followers"},
}
)
records = [{
"record_id": "coop-1",
"title": "合作笔记",
"published": "2026-08-11",
"url": "https://www.douyin.com/video/9",
"platform": "抖音",
"creator": "达人丙",
"cost": 1200,
"tracking": "SF123",
"paid": True,
"followers": "3.7万",
}]
coop_row = sync.parse_records(table, records, self_operated=False)[0]
assert coop_row.coop_attrs["cooperation_cost"] == 1200.0
assert coop_row.coop_attrs["tracking_number"] == "SF123"
assert coop_row.coop_attrs["is_paid"] is True
assert coop_row.creator_attrs["follower_count_num"] == 37000
self_row = sync.parse_records(table, records, self_operated=True)[0]
assert self_row.coop_attrs == {}
assert self_row.creator_attrs == {}
def test_dry_run_fetches_both_scopes_without_opening_database(monkeypatch) -> None:
collaborator = {**_table(), "name": "合作款"}
self_operated = {**_table(), "name": "自营款", "table_id": "tbl-self"}
monkeypatch.setattr(
sync,
"load_sources",
lambda **_kwargs: [(False, collaborator), (True, self_operated)],
)
monkeypatch.setattr(
sync.feishu_mapping,
"_record_list_all",
lambda *_args: [{
"record_id": "one",
"title": "完整笔记",
"published": "2026-08-11",
"url": "https://www.douyin.com/video/1",
"platform": "抖音",
"creator": "达人",
}],
)
monkeypatch.setattr(
sync.psycopg,
"connect",
lambda **_kwargs: (_ for _ in ()).throw(
AssertionError("dry-run must not open PostgreSQL")
),
)
summary = sync.synchronize()
assert summary == {
"tables": 2,
"rows": 2,
"creators": 0,
"countable": 2,
"incomplete": 0,
"deactivated": 0,
}
def test_sync_table_deactivates_only_rows_missing_from_same_source() -> None:
class Cursor:
rowcount = 3
def __init__(self) -> None:
self.calls = []
def execute(self, sql, params=None):
self.calls.append((" ".join(sql.split()), params))
def fetchone(self):
return (42,)
cur = Cursor()
rows = [
sync.MasterRow(
record_id="rec-1",
platform="xiaohongshu",
creator_name="达人",
title="标题",
publish_time="2026-08-11 00:00:00+08:00",
url="https://www.xiaohongshu.com/explore/1",
)
]
result = sync.sync_table(cur, _table(), self_operated=False, rows=rows)
assert result["countable"] == 1
deactivate_sql, params = cur.calls[-1]
assert "source_base_token = %s AND source_table_id = %s" in deactivate_sql
assert "NOT (feishu_record_id = ANY(%s))" in deactivate_sql
assert params == ("base-token", "tbl-source", ["rec-1"])
@@ -15,25 +15,31 @@ from gyxx_flow.modules.content_marketing.data.tools import (
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module):
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "state.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
cookie_backup, profile_backup = module.reset_state()
cookie_backup, profile_backup, state_backup = module.reset_state()
assert not cookie.exists()
assert not storage.exists()
assert not profile.exists()
cookie.write_text("partial-new-cookie", encoding="utf-8")
storage.write_text("partial-new-storage", encoding="utf-8")
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
module.restore_previous_state(cookie_backup, profile_backup)
module.restore_previous_state(cookie_backup, profile_backup, state_backup)
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
@@ -41,21 +47,27 @@ def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, modul
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module):
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
cookie_backup, profile_backup = module.reset_state()
cookie_backup, profile_backup, state_backup = module.reset_state()
cookie.write_text("new-cookie", encoding="utf-8")
storage.write_text("new-storage", encoding="utf-8")
profile.mkdir()
module.finish_successful_relogin(profile_backup)
assert cookie.read_text(encoding="utf-8") == "new-cookie"
assert storage.read_text(encoding="utf-8") == "new-storage"
assert cookie_backup.exists()
assert state_backup.exists()
assert not profile_backup.exists()
@@ -127,6 +139,82 @@ def test_xingtu_index_page_is_recognized_as_logged_in():
assert xingtu_scraper_v2.is_logged_in(Page()) is True
def test_xingtu_index_page_with_xingtu_session_is_logged_in_without_market_ui():
class Locator:
@property
def first(self):
return self
def count(self):
return 0
def wait_for(self, timeout):
raise xingtu_scraper_v2.BrowserTimeoutError("not on market page")
def is_visible(self):
return False
class Context:
def cookies(self):
return [
{
"name": "sessionid",
"value": "fresh",
"domain": ".xingtu.cn",
}
]
class Page:
url = "https://www.xingtu.cn/ad/creator/index"
context = Context()
def get_by_text(self, *_args, **_kwargs):
return Locator()
def locator(self, *_args, **_kwargs):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is True
def test_xingtu_index_page_without_xingtu_session_is_not_logged_in():
class Locator:
@property
def first(self):
return self
def count(self):
return 0
def wait_for(self, timeout):
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
def is_visible(self):
return False
class Context:
def cookies(self):
return [
{
"name": "sessionid",
"value": "douyin-only",
"domain": ".douyin.com",
}
]
class Page:
url = "https://www.xingtu.cn/ad/creator/index"
context = Context()
def get_by_text(self, *_args, **_kwargs):
return Locator()
def locator(self, *_args, **_kwargs):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is False
def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie():
class Context:
def cookies(self):
@@ -149,7 +237,7 @@ def test_xingtu_market_page_without_business_ui_rejects_stale_cookie():
return self
def wait_for(self, **_kwargs):
raise xingtu_scraper_v2.PlaywrightTimeoutError("missing")
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
def count(self):
return 0
@@ -282,25 +370,64 @@ def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monke
assert saved == [market_page]
def test_xingtu_scan_login_detects_business_tab_while_qr_tab_stays_open(monkeypatch):
class Context:
def __init__(self):
self.pages = []
class Page:
def __init__(self, context, url):
self.context = context
self.url = url
def is_closed(self):
return False
def wait_for_timeout(self, _milliseconds):
raise AssertionError("authenticated tab should be detected immediately")
context = Context()
qr_page = Page(context, "https://open.douyin.com/platform/oauth/pc/auth")
index_page = Page(context, "https://www.xingtu.cn/ad/creator/index")
context.pages = [qr_page, index_page]
saved = []
monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None)
monkeypatch.setattr(
xingtu_scraper_v2, "is_logged_in", lambda page: page is index_page
)
monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page))
result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2)
assert result is index_page
assert saved == [index_page]
def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch):
cookie = tmp_path / "xingtu_cookies.json"
storage = tmp_path / "xingtu_storage_state.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie)
monkeypatch.setattr(relogin_xingtu, "STATE_FILE", storage)
monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile)
cookie_backup, profile_backup = relogin_xingtu.reset_state()
cookie_backup, profile_backup, state_backup = relogin_xingtu.reset_state()
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
assert relogin_xingtu.recover_interrupted_state() is True
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
assert cookie_backup is not None
assert state_backup is not None
assert profile_backup is not None
@@ -0,0 +1,410 @@
from __future__ import annotations
import json
from pathlib import Path
from types import ModuleType
import pytest
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
from gyxx_flow.modules.content_marketing.data.tools import (
relogin_bilibili,
relogin_douyin,
relogin_pgy,
relogin_runtime_paths,
relogin_xiaohongshu,
relogin_xingtu,
)
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
CONSUMER_BINDING_IDS,
child_browser_environment,
resolve_relogin_runtime_paths,
sync_relogin_state_to_consumers,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
PROJECT_ROOT = Path(__file__).resolve().parents[3]
def test_relogin_paths_prefer_injected_runtime_binding(tmp_path: Path) -> None:
cookie = tmp_path / "binding" / "cookies.json"
storage = tmp_path / "binding" / "storage_state.json"
profile = tmp_path / "binding" / "profile"
resolved = resolve_relogin_runtime_paths(
"pgy",
{
"GYXX_BROWSER_COOKIE_FILE": str(cookie),
"GYXX_BROWSER_STORAGE_STATE_FILE": str(storage),
"GYXX_BROWSER_PROFILE_DIR": str(profile),
},
)
assert resolved.cookie_file == cookie.resolve()
assert resolved.storage_state_file == storage.resolve()
assert resolved.profile_dir == profile.resolve()
def test_relogin_paths_keep_legacy_standalone_layout() -> None:
resolved = resolve_relogin_runtime_paths("xiaohongshu", {})
assert resolved.cookie_file == (
PATHS.state_root / "cookies/xiaohongshu_cookies.json"
).resolve()
assert resolved.storage_state_file == (
PATHS.state_root / "cookies/xiaohongshu_storage_state.json"
).resolve()
assert resolved.profile_dir == (
PATHS.state_root / "browser-profiles/xiaohongshu"
).resolve()
def test_child_login_environment_preserves_wrapper_binding(tmp_path: Path) -> None:
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
environment = child_browser_environment(
cookie_file=cookie,
storage_state_file=storage,
profile_dir=profile,
environment={"GYXX_MODULE_ID": "content_marketing", "KEEP": "1"},
)
assert environment["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(
storage.resolve()
)
assert environment["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
assert "GYXX_MODULE_ID" not in environment
assert environment["KEEP"] == "1"
@pytest.mark.parametrize(
("module", "platform", "login_cookie", "argv"),
[
(
relogin_bilibili,
"bilibili",
"SESSDATA",
["relogin_bilibili.py"],
),
(relogin_douyin, "douyin", "sessionid", ["relogin_douyin.py"]),
(relogin_pgy, "pgy", "web_session", ["relogin_pgy.py"]),
(
relogin_xiaohongshu,
"xiaohongshu",
"web_session",
["relogin_xiaohongshu.py"],
),
(
relogin_xingtu,
"xingtu",
"sessionid",
["relogin_xingtu.py", "--force"],
),
],
)
def test_relogin_wrapper_validates_the_same_binding_written_by_child(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
module: ModuleType,
platform: str,
login_cookie: str,
argv: list[str],
) -> None:
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
monkeypatch.setattr(module.sys, "argv", argv)
monkeypatch.setenv("GYXX_MODULE_ID", "content_marketing")
if module is relogin_xingtu:
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
synced: list[tuple[str, Path, Path]] = []
def fake_sync(
synced_platform: str,
*,
cookie_file: Path,
storage_state_file: Path,
) -> tuple[Path, ...]:
synced.append(
(synced_platform, cookie_file.resolve(), storage_state_file.resolve())
)
return ()
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fake_sync)
def fake_call(
_command: list[str],
*,
cwd: str,
env: dict[str, str],
) -> int:
assert cwd == str(module.PROJECT_DIR)
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
assert env["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(storage.resolve())
assert env["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
assert "GYXX_MODULE_ID" not in env
cookie.write_text(
json.dumps([{"name": login_cookie, "value": "fresh"}]),
encoding="utf-8",
)
storage.write_text("{}", encoding="utf-8")
profile.mkdir()
return 0
monkeypatch.setattr(module.subprocess, "call", fake_call)
assert module.main() == 0
assert cookie.is_file()
assert storage.is_file()
assert profile.is_dir()
assert synced == [(platform, cookie.resolve(), storage.resolve())]
@pytest.mark.parametrize(
("module", "platform", "login_cookie", "cookie_domain", "argv"),
[
(
relogin_bilibili,
"bilibili",
"SESSDATA",
".bilibili.com",
["relogin_bilibili.py"],
),
(
relogin_douyin,
"douyin",
"sessionid",
".douyin.com",
["relogin_douyin.py"],
),
(
relogin_pgy,
"pgy",
"web_session",
".xiaohongshu.com",
["relogin_pgy.py"],
),
(
relogin_xiaohongshu,
"xiaohongshu",
"web_session",
".xiaohongshu.com",
["relogin_xiaohongshu.py"],
),
(
relogin_xingtu,
"xingtu",
"sessionid",
".xingtu.cn",
["relogin_xingtu.py", "--force"],
),
],
)
def test_relogin_wrapper_restores_previous_state_when_publish_fails(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
module: ModuleType,
platform: str,
login_cookie: str,
cookie_domain: str,
argv: list[str],
) -> None:
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
old_cookie = '[{"name":"old","value":"old-session"}]'
old_storage = '{"cookies":[{"name":"old"}]}'
cookie.write_text(old_cookie, encoding="utf-8")
storage.write_text(old_storage, encoding="utf-8")
profile.mkdir()
(profile / "old-state.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
monkeypatch.setattr(module.sys, "argv", argv)
if module is relogin_xingtu:
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
def fake_call(
_command: list[str],
*,
cwd: str,
env: dict[str, str],
) -> int:
assert cwd == str(module.PROJECT_DIR)
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
cookie.write_text(
json.dumps(
[
{
"name": login_cookie,
"value": "fresh-session",
"domain": cookie_domain,
}
]
),
encoding="utf-8",
)
storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
profile.mkdir()
return 0
def fail_publish(
synced_platform: str,
*,
cookie_file: Path,
storage_state_file: Path,
) -> tuple[Path, ...]:
assert synced_platform == platform
assert cookie_file == cookie
assert storage_state_file == storage
raise OSError("simulated consumer publish failure")
monkeypatch.setattr(module.subprocess, "call", fake_call)
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fail_publish)
assert module.main() == 1
assert cookie.read_text(encoding="utf-8") == old_cookie
assert storage.read_text(encoding="utf-8") == old_storage
assert (profile / "old-state.txt").read_text(encoding="utf-8") == (
"old-profile"
)
@pytest.mark.parametrize(
("platform", "cookie_name", "cookie_domain"),
[
("bilibili", "SESSDATA", ".bilibili.com"),
("douyin", "sessionid", ".douyin.com"),
("pgy", "web_session", ".xiaohongshu.com"),
("xiaohongshu", "web_session", ".xiaohongshu.com"),
("xingtu", "sessionid", ".xingtu.cn"),
],
)
def test_successful_relogin_atomically_updates_unique_consumer_state(
tmp_path: Path,
platform: str,
cookie_name: str,
cookie_domain: str,
) -> None:
data_root = tmp_path / "data"
source_cookie = tmp_path / "relogin" / "cookies.json"
source_storage = tmp_path / "relogin" / "storage_state.json"
source_cookie.parent.mkdir(parents=True)
fresh_cookie = [
{
"name": cookie_name,
"value": "fresh-session",
"domain": cookie_domain,
}
]
fresh_storage = {"cookies": fresh_cookie, "origins": []}
source_cookie.write_text(json.dumps(fresh_cookie), encoding="utf-8")
source_storage.write_text(json.dumps(fresh_storage), encoding="utf-8")
catalog = RuntimeIntegrationCatalog.load_default(
project_root=PROJECT_ROOT,
data_root=data_root,
)
consumer_id = CONSUMER_BINDING_IDS[platform][0]
consumer = catalog.binding_for(consumer_id)
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
consumer.cookie_file.write_text('[{"name":"old"}]', encoding="utf-8")
consumer.storage_state_file.write_text(
'{"cookies":[{"name":"old"}]}',
encoding="utf-8",
)
destinations = sync_relogin_state_to_consumers(
platform,
cookie_file=source_cookie,
storage_state_file=source_storage,
environment={
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
"GYXX_DATA_ROOT": str(data_root),
},
)
assert source_cookie.resolve() != consumer.cookie_file.resolve()
assert source_storage.resolve() != consumer.storage_state_file.resolve()
assert destinations == (consumer.cookie_file.resolve(),)
assert json.loads(consumer.cookie_file.read_text(encoding="utf-8")) == (
fresh_cookie
)
assert json.loads(
consumer.storage_state_file.read_text(encoding="utf-8")
) == fresh_storage
def test_consumer_state_is_rolled_back_if_atomic_publish_fails(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
data_root = tmp_path / "data"
source_cookie = tmp_path / "relogin" / "cookies.json"
source_storage = tmp_path / "relogin" / "storage_state.json"
source_cookie.parent.mkdir(parents=True)
source_cookie.write_text(
json.dumps(
[
{
"name": "SESSDATA",
"value": "fresh-session",
"domain": ".bilibili.com",
}
]
),
encoding="utf-8",
)
source_storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
catalog = RuntimeIntegrationCatalog.load_default(
project_root=PROJECT_ROOT,
data_root=data_root,
)
consumer = catalog.binding_for(CONSUMER_BINDING_IDS["bilibili"][0])
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
old_cookie = '[{"name":"SESSDATA","value":"old"}]'
old_storage = '{"cookies":[{"name":"old"}]}'
consumer.cookie_file.write_text(old_cookie, encoding="utf-8")
consumer.storage_state_file.write_text(old_storage, encoding="utf-8")
original_replace = relogin_runtime_paths.os.replace
failed = False
def fail_second_replace(source: Path | str, target: Path | str) -> None:
nonlocal failed
source_path = Path(source)
target_path = Path(target)
if (
not failed
and source_path.suffix == ".tmp"
and target_path == consumer.storage_state_file
):
failed = True
raise OSError("simulated storage publish failure")
original_replace(source, target)
monkeypatch.setattr(relogin_runtime_paths.os, "replace", fail_second_replace)
with pytest.raises(OSError, match="simulated storage publish failure"):
sync_relogin_state_to_consumers(
"bilibili",
cookie_file=source_cookie,
storage_state_file=source_storage,
environment={
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
"GYXX_DATA_ROOT": str(data_root),
},
)
assert consumer.cookie_file.read_text(encoding="utf-8") == old_cookie
assert consumer.storage_state_file.read_text(encoding="utf-8") == old_storage
@@ -1,6 +1,7 @@
import io
import unittest
from pathlib import Path
from unittest.mock import patch
from gyxx_flow.adapters import RuntimeServicePolicy
from gyxx_flow.modules.content_marketing.data.tools import db
@@ -60,7 +61,7 @@ class LocalDatabaseConfigTests(unittest.TestCase):
paths = (
ROOT / "data" / "tools" / "db.py",
ROOT / "data" / "tools" / "sync_metrics_to_cmt_notes.py",
ROOT / "data" / "tools" / "sync_cooperations.py",
ROOT / "data" / "tools" / "sync_notes_master.py",
ROOT / "data" / "tools" / "generate_creator_report.py",
)
for path in paths:
@@ -79,6 +80,33 @@ class LocalDatabaseConfigTests(unittest.TestCase):
self.assertTrue(raw.getvalue())
def test_explicit_insert_missing_preserves_default_sync_contract(self):
with (
patch("sys.argv", ["sync_metrics_to_cmt_notes.py", "--insert-missing"]),
patch.object(
sync_metrics,
"sync",
return_value={"identity_ambiguous": 0},
) as sync_call,
):
sync_metrics.main()
sync_call.assert_called_once_with(dry_run=False, insert_missing=True)
def test_default_sync_exits_nonzero_when_identity_is_ambiguous(self):
with (
patch("sys.argv", ["sync_metrics_to_cmt_notes.py"]),
patch.object(
sync_metrics,
"sync",
return_value={"identity_ambiguous": 2},
),
self.assertRaises(SystemExit) as exc_info,
):
sync_metrics.main()
self.assertEqual(exc_info.exception.code, 1)
if __name__ == "__main__":
unittest.main()
@@ -1,6 +1,7 @@
import json
import os
import time
from datetime import date
from pathlib import Path
from types import SimpleNamespace
@@ -39,6 +40,54 @@ def test_success_status_cannot_hide_write_failure_and_blocked_input_is_terminal(
}],
}
assert retry.extract_failed_records(blocked, "bili") == []
assert retry.requested_records_succeeded(blocked, ["b"]) == (True, [])
def test_requested_records_require_exactly_one_resolved_blocked_terminal():
duplicate_blocked = {
"results": [
{
"record_id": "b",
"status": "blocked_input",
"matched": False,
"reason": "url_missing",
},
{
"record_id": "b",
"status": "blocked_input",
"matched": False,
"reason": "url_missing",
},
],
}
assert retry.requested_records_succeeded(duplicate_blocked, ["b"]) == (
False,
["b"],
)
def test_blocked_input_cannot_hide_explicit_execution_or_write_failure():
for explicit_failure in (
{"write_ok": False},
{"write_error": "database rejected update"},
{"error": "browser session crashed"},
):
blocked = {
"results": [{
"record_id": "b",
"status": "blocked_input",
"matched": False,
"reason": "url_missing",
**explicit_failure,
}],
}
assert retry.requested_records_succeeded(blocked, ["b"]) == (
False,
["b"],
)
assert retry.extract_failed_records(blocked, "bili") == ["b"]
def test_canonical_filename_requires_current_style_name():
@@ -305,6 +354,87 @@ def test_validate_summary_rejects_expected_style_left_over_from_old_run(tmp_path
assert "current run" in reason
def test_build_retry_targets_narrows_retryable_rows_and_expands_structural_gaps():
targets = run_all.build_retry_targets(
{
"bili": [{
"index": 1,
"total_b_records": 2,
"details": [
{"record_id": "ok", "status": "success", "ok": True},
{"record_id": "bad", "status": "retryable_failure", "ok": False},
],
}],
"pgy": [{
"index": 1,
"total": 2,
"results": [{"record_id": "only-one", "status": "success"}],
}],
"xt": [{
"index": 1,
"total": 1,
"results": [{
"record_id": "terminal",
"status": "blocked_input",
"matched": False,
}],
}],
},
{1},
)
assert targets == {
"bili": {1: {"bad"}},
"pgy": {1: None},
}
def test_retry_manifest_round_trips_record_and_full_style_scopes(monkeypatch, tmp_path):
monkeypatch.setattr(run_all, "RETRY_MANIFEST_DIR", tmp_path)
targets = {"bili": {1: {"r2", "r1"}}, "pgy": {3: None}}
run_all._write_retry_manifest(
date(2026, 9, 2), targets, source_slot="01:00"
)
assert run_all._load_retry_manifest(date(2026, 9, 2)) == targets
def test_retry_process_command_passes_only_target_records():
args = SimpleNamespace(
dry_run=False,
style=[1, 2],
daily_scope=True,
no_retry=False,
)
command = run_all.build_process_command(
"pgy", args, retry_targets={2: {"record-b", "record-a"}}
)
assert command[command.index("--style") + 1] == "2"
assert command.count("--style") == 1
assert command[command.index("--record") + 1] == "record-a"
assert command.count("--record") == 2
assert "--no-retry" in command
def test_retry_process_command_uses_full_style_when_scope_is_structurally_incomplete():
args = SimpleNamespace(
dry_run=False,
style=[1, 2],
daily_scope=True,
no_retry=False,
)
command = run_all.build_process_command(
"xt", args, retry_targets={1: None}
)
assert command[command.index("--style") + 1] == "1"
assert "--record" not in command
def test_aggregate_counts_all_unresolved_rows_as_errors():
aggregate = run_all.aggregate({
"pgy": [{
@@ -363,6 +493,40 @@ def test_aggregate_keeps_partial_counts_when_validation_marks_payload_invalid():
assert platform["validation_error"] == "invalid summary: unresolved"
def test_aggregate_fails_closed_for_stale_payload():
aggregate = run_all.aggregate({
"pgy": {
"error": "invalid summary: summary file is stale",
"payload": [{
"index": 1,
"total": 141,
"matched": 101,
"results": [],
}],
},
})
platform = aggregate["platforms"]["pgy"]
assert platform["loaded"] is False
assert platform["error"] == "invalid summary: summary file is stale"
assert aggregate["totals"]["tasks"] == 0
def test_aggregate_filters_merged_history_to_current_styles():
aggregate = run_all.aggregate(
{
"xt": [
{"index": 1, "total": 2, "matched": 1, "results": []},
{"index": 2, "total": 99, "matched": 99, "results": []},
],
},
expected_indices={1},
)
assert aggregate["platforms"]["xt"]["styles"] == 1
assert aggregate["platforms"]["xt"]["tasks"] == 2
def test_run_round_marks_exit_zero_platform_failed_when_summary_is_invalid(monkeypatch, tmp_path):
args = SimpleNamespace(dry_run=True, style=[1])
stdout = tmp_path / "stdout.log"
@@ -0,0 +1,110 @@
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
from gyxx_flow.modules.content_marketing import weekly_summary_all
def test_target_table_payload_uses_temp_cwd_and_relative_reference(
monkeypatch,
tmp_path: Path,
) -> None:
payload_root = tmp_path / "payloads"
payload_root.mkdir()
working_directory = tmp_path / "working-directory"
working_directory.mkdir()
monkeypatch.chdir(working_directory)
monkeypatch.setattr(
weekly_summary_all,
"PATHS",
SimpleNamespace(tmp_root=payload_root),
)
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
calls: list[tuple[list[str], Path | None]] = []
def fake_call_lark_json(
args: list[str], *, cwd: Path | None = None
) -> dict[str, object]:
calls.append((args, cwd))
if args[1] == "+record-list":
assert cwd is None
return {
"ok": True,
"data": {
"record_id_list": [],
"data": [],
"field_id_list": [],
},
}
assert cwd == payload_root
assert cwd != Path.cwd()
payload_reference = args[args.index("--json") + 1]
assert payload_reference.startswith("@./")
payload_path = cwd / payload_reference.removeprefix("@./")
assert not Path(payload_reference[1:]).is_absolute()
assert json.loads(payload_path.read_text(encoding="utf-8")) == {
"fld_time": "2026-07-01~07-31",
"fld_summary": (
"[凌云air 月度汇总(2026-07-01~07-31]"
"(https://example.test/doc)"
),
}
return {"ok": True}
monkeypatch.setattr(weekly_summary_all, "call_lark_json", fake_call_lark_json)
assert weekly_summary_all.write_to_target_table(
"base-test",
"table-test",
"fld_time",
"fld_summary",
"凌云air",
"https://example.test/doc",
False,
time_label="2026-07-01~07-31",
report_label="月度汇总",
)
assert [command[1] for command, _cwd in calls] == [
"+record-list",
"+record-upsert",
]
def test_call_lark_json_logs_only_redacted_failure_detail(
monkeypatch,
capsys,
tmp_path: Path,
) -> None:
raw_secret = "sensitive-token-value"
raw_open_id = "sensitive-open-id"
captured_kwargs: dict[str, object] = {}
def fake_run(*_args, **kwargs):
captured_kwargs.update(kwargs)
return SimpleNamespace(
returncode=1,
stdout="",
stderr=(
f"request failed; base_token={raw_secret}; "
f"open_id={raw_open_id}; https://example.test/private"
),
)
monkeypatch.setattr(weekly_summary_all.subprocess, "run", fake_run)
response = weekly_summary_all.call_lark_json(
["base", "+record-upsert"], cwd=tmp_path
)
captured = capsys.readouterr().out
assert captured_kwargs["cwd"] == tmp_path
assert response["ok"] is False
assert raw_secret not in response["error"]
assert raw_open_id not in response["error"]
assert raw_secret not in captured
assert raw_open_id not in captured
assert "[REDACTED]" in captured
assert "[URL]" in captured
File diff suppressed because it is too large Load Diff
@@ -5,6 +5,32 @@ from gyxx_flow.modules.content_marketing import weekly_summary_all as weekly
class WeeklyTmallPersonaTests(unittest.TestCase):
def test_summary_result_requires_all_real_external_effects(self):
self.assertTrue(
weekly.summary_results_succeeded(
[{"status": "ok"}, {"status": "no_notes"}]
)
)
for status in (
"no_fields",
"no_reports",
"llm_failed",
"doc_failed",
"doc_ambiguous",
"db_failed",
"write_failed",
"exception",
):
with self.subTest(status=status):
self.assertFalse(
weekly.summary_results_succeeded([{"status": status}])
)
def test_dry_run_status_is_only_successful_in_dry_run_mode(self):
results = [{"status": "dry_run"}]
self.assertFalse(weekly.summary_results_succeeded(results))
self.assertTrue(weekly.summary_results_succeeded(results, dry_run=True))
def test_missing_persona_forbids_invented_percentages(self):
text = weekly.format_tmall_persona(None)
self.assertIn("不得虚构具体比例", text)
@@ -56,7 +82,7 @@ class WeeklyTmallPersonaTests(unittest.TestCase):
}]
with patch.object(weekly, "get_latest_tmall_persona", return_value=persona) as get_persona, \
patch.object(weekly, "call_hermes_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
patch.object(weekly, "call_content_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
weekly.generate_summary("星云2", notes)
get_persona.assert_called_once_with("星云2")
@@ -0,0 +1,108 @@
import pytest
from gyxx_flow.modules.content_marketing import xiaohongshu_comment_scraper as scraper
class _EvaluatePage:
def __init__(self, result):
self.result = result
self.calls = []
def evaluate(self, script, argument):
self.calls.append((script, argument))
return self.result
def test_bounded_fetch_uses_abort_controller_and_clears_timer(monkeypatch):
page = _EvaluatePage(
{"__gyxx_status": "ok", "payload": {"comments": []}}
)
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
payload = scraper._fetch_json_with_timeout(
page,
"https://example.invalid/comments",
deadline=1000.0,
stage="top-level pagination",
)
assert payload == {"comments": []}
script, argument = page.calls[0]
assert "AbortController" in script
assert "clearTimeout(timer)" in script
assert argument["timeoutMs"] == 30_000
def test_bounded_fetch_timeout_is_an_explicit_transient_failure(monkeypatch):
page = _EvaluatePage({"__gyxx_status": "timeout"})
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
with pytest.raises(scraper.TransientXiaohongshuCommentTimeout) as caught:
scraper._fetch_json_with_timeout(
page,
"https://example.invalid/comments",
deadline=1000.0,
stage="reply pagination",
)
assert isinstance(caught.value, ConnectionError)
assert "reply pagination" in str(caught.value)
def test_scroll_checks_the_note_deadline_before_browser_work(monkeypatch):
page = _EvaluatePage(None)
monkeypatch.setattr(scraper.time, "monotonic", lambda: 901.0)
with pytest.raises(scraper.TransientXiaohongshuCommentTimeout):
scraper.scroll_comments(
page,
max_scrolls=30,
idle_rounds=4,
comments_by_id={},
deadline=900.0,
)
assert page.calls == []
def test_top_and_reply_pagination_share_the_bounded_fetch(monkeypatch):
stages = []
monkeypatch.setattr(scraper.time, "monotonic", lambda: 0.0)
def fetch(_page, _url, *, deadline, stage):
assert deadline == 900.0
stages.append(stage)
return {"comments": [], "has_more": 0, "cursor": 0}
monkeypatch.setattr(scraper, "_fetch_json_with_timeout", fetch)
monkeypatch.setattr(scraper, "collect_from_payload", lambda *_args: None)
page = _EvaluatePage(None)
scraper.fetch_missing_top_comments(
page,
"source-url",
{},
{
"comment_url_template": "https://example.invalid/top",
"api_total": 2,
"last_cursor": 1,
},
deadline=900.0,
)
scraper.fetch_missing_replies(
page,
"source-url",
{
"parent": {
"level": "comment",
"reply_count": 1,
"comment_id": "parent-id",
}
},
{
"sub_url_template": "https://example.invalid/reply",
},
deadline=900.0,
)
assert stages == ["top-level pagination", "reply pagination"]
@@ -3,11 +3,11 @@ from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
from types import ModuleType
from types import ModuleType, SimpleNamespace
import pytest
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
PROJECT_ROOT = Path(__file__).resolve().parents[3]
COMMAND_PATH = (
@@ -56,7 +56,7 @@ def test_alert_command_uses_canonical_acceptance_recipient(
"--end-date",
"2026-08-01",
"--openid",
WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID,
ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID,
)
@@ -72,3 +72,52 @@ def test_alert_command_accepts_canonical_production_recipient(
assert module.main() == 0
assert observed["argv"][-1] == "ou_configured"
def test_alert_command_passes_all_dynamic_recipients(
monkeypatch: pytest.MonkeyPatch,
) -> None:
observed: dict[str, object] = {}
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01")
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
module = _load_command(monkeypatch, observed)
monkeypatch.setattr(
module,
"resolve_notification_route",
lambda *_args, **_kwargs: SimpleNamespace(
enabled=True,
open_ids=("ou_first", "ou_second"),
),
)
assert module.main() == 0
assert observed["argv"] == (
sys.argv[0],
"--end-date",
"2026-08-01",
"--openid",
"ou_first",
"--openid",
"ou_second",
)
def test_alert_command_uses_safe_no_notify_when_route_is_disabled(
monkeypatch: pytest.MonkeyPatch,
) -> None:
observed: dict[str, object] = {}
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01")
module = _load_command(monkeypatch, observed)
monkeypatch.setattr(
module,
"resolve_notification_route",
lambda *_args, **_kwargs: SimpleNamespace(enabled=False, open_ids=()),
)
assert module.main() == 0
assert observed["argv"] == (
sys.argv[0],
"--end-date",
"2026-08-01",
"--no-notify",
)
@@ -53,6 +53,11 @@ def _prepare_main(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
)
monkeypatch.setattr(decline, "managed_data_path", lambda value: Path(value))
monkeypatch.setattr(decline, "append_log", lambda line: None)
monkeypatch.setattr(
decline,
"_persist_detected_events",
lambda events, *_args: len(events),
)
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
@@ -142,27 +147,73 @@ def test_segment_loader_uses_postgres_range_sum_on_one_connection(
]
@pytest.mark.parametrize(
("response", "expected"),
[
({"id": "chatcmpl-not-a-receipt"}, None),
({"message_id": "om_direct123"}, "om_direct123"),
({"data": {"message_id": "om_nested123"}}, "om_nested123"),
(
{"choices": [{"message": {"content": "飞书回执 om_content123"}}]},
"om_content123",
),
(
{"choices": [{"message": {"content": "发送成功,但没有回执"}}]},
None,
),
],
)
def test_extract_feishu_message_id_requires_real_receipt_shape(
response: dict,
expected: str | None,
def test_build_decline_notification_message_is_deterministic() -> None:
message = decline.build_decline_notification_message(
[_event()],
end_date="2026-08-01",
window_days=9,
)
assert "销量连续下滑告警(截至 2026-08-01,3 段 × 3 天)" in message
assert "[tm] 测试款" in message
assert "300 → 210 → 90" in message
assert "累计 -70.0%" in message
assert "ERP" in message
def test_alert_delivery_uuid_is_stable_and_recipient_scoped() -> None:
first = decline._notification_delivery_uuid(
[_event()],
end_date="2026-08-01",
window_days=9,
openid="ou_first",
)
same = decline._notification_delivery_uuid(
[_event()],
end_date="2026-08-01",
window_days=9,
openid="ou_first",
)
second = decline._notification_delivery_uuid(
[_event()],
end_date="2026-08-01",
window_days=9,
openid="ou_second",
)
assert first == same
assert first != second
def test_decline_notification_uses_shared_lark_user_sender(
monkeypatch: pytest.MonkeyPatch,
) -> None:
assert decline.extract_feishu_message_id(response) == expected
captured: dict[str, object] = {}
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
def fake_send(**kwargs):
captured.update(kwargs)
return {"message_id": "om_real123"}
monkeypatch.setattr(decline, "send_lark_bot_message", fake_send)
response = decline.send_decline_notification(
[_event()],
"2026-08-01",
9,
openid="ou_owner",
)
assert response["message_id"] == "om_real123"
assert captured["user_id"] == "ou_owner"
assert captured["profile"] == "hermes-analyzer"
assert captured["idempotency_key"] == decline._notification_delivery_uuid(
[_event()],
end_date="2026-08-01",
window_days=9,
openid="ou_owner",
)
assert "测试款" in str(captured["text"])
def test_main_fails_when_postgres_detection_read_fails(
@@ -205,12 +256,12 @@ def test_main_dedup_query_failure_is_fail_closed(
lambda *args: (_ for _ in ()).throw(RuntimeError("db timeout")),
)
notify = pytest.fail
monkeypatch.setattr(decline, "notify_hermes", notify)
monkeypatch.setattr(decline, "send_decline_notification", notify)
assert decline.main() == 5
def test_main_does_not_accept_arbitrary_non_error_hermes_text(
def test_main_does_not_accept_lark_response_without_real_message_id(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
@@ -219,11 +270,8 @@ def test_main_does_not_accept_arbitrary_non_error_hermes_text(
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
monkeypatch.setattr(
decline,
"notify_hermes",
lambda *args, **kwargs: {
"id": "chatcmpl-123",
"choices": [{"message": {"content": "发送成功"}}],
},
"send_decline_notification",
lambda *args, **kwargs: {"ok": True},
)
persist = pytest.fail
monkeypatch.setattr(decline, "_persist_events", persist)
@@ -240,7 +288,7 @@ def test_main_returns_nonzero_when_receipt_persistence_fails(
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
monkeypatch.setattr(
decline,
"notify_hermes",
"send_decline_notification",
lambda *args, **kwargs: {"message_id": "om_real123"},
)
monkeypatch.setattr(
@@ -263,7 +311,7 @@ def test_main_success_requires_and_persists_feishu_receipt(
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
monkeypatch.setattr(
decline,
"notify_hermes",
"send_decline_notification",
lambda *args, **kwargs: {"message_id": "om_real123"},
)
@@ -281,6 +329,174 @@ def test_main_success_requires_and_persists_feishu_receipt(
assert "persistence=committed history_rows=1" in output
def test_main_no_notify_still_detects_hits_and_succeeds(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
) -> None:
_prepare_main(monkeypatch, tmp_path)
monkeypatch.setattr(
sys,
"argv",
[
"check_nine_day_decline.py",
"--end-date",
"2026-08-01",
"--no-notify",
"--data-root",
str(tmp_path),
],
)
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
persisted: list[tuple[list[dict], str, int]] = []
monkeypatch.setattr(
decline,
"_persist_detected_events",
lambda events, end_date, window_days: (
persisted.append((events, end_date, window_days)) or len(events)
),
)
monkeypatch.setattr(
decline,
"_already_notified",
lambda *args: pytest.fail("disabled notification must not query receipts"),
)
monkeypatch.setattr(
decline,
"send_decline_notification",
lambda *args, **kwargs: pytest.fail("disabled notification must not send"),
)
assert decline.main() == 0
assert persisted == [([_event()], "2026-08-01", 9)]
output = capsys.readouterr().out
assert "events=1 notification=disabled persistence=committed" in output
assert "detection_rows=1" in output
def test_main_blocks_notification_when_detection_output_cannot_be_persisted(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_prepare_main(monkeypatch, tmp_path)
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
monkeypatch.setattr(
decline,
"_persist_detected_events",
lambda *_args: (_ for _ in ()).throw(RuntimeError("write failed")),
)
monkeypatch.setattr(
decline,
"send_decline_notification",
lambda *args, **kwargs: pytest.fail("must not send without durable output"),
)
assert decline.main() == 5
def test_main_sends_and_persists_each_recipient_independently(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_prepare_main(monkeypatch, tmp_path)
monkeypatch.setattr(
sys,
"argv",
[
"check_nine_day_decline.py",
"--end-date",
"2026-08-01",
"--openid",
"ou_first",
"--openid",
"ou_second",
"--data-root",
str(tmp_path),
],
)
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
dedup_queries: list[str] = []
persisted: list[tuple[str, str]] = []
def fake_already_notified(_style, _date, recipient):
dedup_queries.append(recipient)
return False
def fake_notify(*_args, openid, **_kwargs):
return {"message_id": f"om_{openid.removeprefix('ou_')}"}
def fake_persist(*_args, openid, message_id, **_kwargs):
persisted.append((openid, message_id))
return 1
monkeypatch.setattr(decline, "_already_notified", fake_already_notified)
monkeypatch.setattr(decline, "send_decline_notification", fake_notify)
monkeypatch.setattr(decline, "_persist_events", fake_persist)
assert decline.main() == 0
assert dedup_queries == ["ou_first", "ou_second"]
assert persisted == [
("ou_first", "om_first"),
("ou_second", "om_second"),
]
def test_alert_schema_and_upsert_are_recipient_scoped() -> None:
repository_root = Path(__file__).resolve().parents[3]
schema = (
repository_root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "product_commerce"
/ "db"
/ "schema.sql"
).read_text(encoding="utf-8")
db_module = (
repository_root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "product_commerce"
/ "db"
/ "__init__.py"
).read_text(encoding="utf-8")
assert "UNIQUE (style_name, trigger_date, recipient)" in schema
assert "DROP CONSTRAINT IF EXISTS fact_alert_history_style_name_trigger_date_key" in schema
assert "ON CONFLICT (style_name, trigger_date, recipient)" in db_module
assert "def ensure_alert_history_recipient_scope" in db_module
assert "fact_alert_history_style_name_trigger_date_key" in db_module
def test_detected_output_migrates_recipient_scope_before_upsert(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_db = ModuleType("db")
connection = object()
calls: list[tuple[str, object]] = []
class ConnectionContext:
def __enter__(self):
return connection
def __exit__(self, *args):
return False
fake_db.get_conn = ConnectionContext # type: ignore[attr-defined]
fake_db.ensure_alert_history_recipient_scope = ( # type: ignore[attr-defined]
lambda conn: calls.append(("migrate", conn))
)
fake_db.upsert_decline_event = ( # type: ignore[attr-defined]
lambda conn, event: calls.append(("upsert", (conn, event)))
)
monkeypatch.setitem(sys.modules, "db", fake_db)
assert decline._persist_detected_events([_event()], "2026-08-01", 9) == 1
assert calls[0] == ("migrate", connection)
assert calls[1][0] == "upsert"
class _FailedCollector:
def __init__(self, *args):
self.pages: list[tuple] = []
@@ -7,16 +7,10 @@ from types import SimpleNamespace
import collect_erp_yesterday_metrics as erp
import collect_jd_persona_to_bitable as jd_persona
from runtime_paths import playwright_launch_options
import erp_login_product_analysis as erp_login
from runtime_paths import scrapling_session_options
class _Chromium:
def __init__(self) -> None:
self.kwargs = None
def launch_persistent_context(self, **kwargs):
self.kwargs = kwargs
return object()
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
def test_jd_persona_launches_the_managed_persistent_profile(
@@ -25,13 +19,20 @@ def test_jd_persona_launches_the_managed_persistent_profile(
) -> None:
profile = tmp_path / "jd-profile"
monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile))
chromium = _Chromium()
captured: dict[str, object] = {}
context = jd_persona._launch_jd_context(chromium, headless=True)
def browser_factory(**options):
captured.update(options)
return object()
assert context is not None
assert chromium.kwargs["user_data_dir"] == str(profile.resolve())
assert chromium.kwargs["headless"] is True
monkeypatch.setattr(jd_persona, "ScraplingBrowser", browser_factory)
browser = jd_persona._jd_browser(headless=True)
assert browser is not None
assert captured["user_data_dir"] == str(profile.resolve())
assert captured["headless"] is True
assert captured["retries"] == 1
def test_jd_persona_reuses_logged_in_page_without_credentials() -> None:
@@ -61,6 +62,36 @@ def test_jd_persona_reuses_logged_in_page_without_credentials() -> None:
assert calls == []
def test_jd_persona_waits_for_slow_shop_shell_before_login_fallback() -> None:
calls: list[tuple[str, str]] = []
bodies = iter(("", "", "", "", "京东商家后台 商品明细"))
waits: list[int] = []
class Page:
url = "https://shop.jd.com/jdm/home"
def goto(self, *_args, **_kwargs):
return None
def wait_for_timeout(self, milliseconds):
waits.append(milliseconds)
def locator(self, _selector):
return SimpleNamespace(
inner_text=lambda **_kwargs: next(bodies),
)
jd_persona._ensure_jd_session(
Page(),
shop="",
password="",
step_login=lambda _page, shop, password: calls.append((shop, password)),
)
assert calls == []
assert waits == [2000, 1000, 1000, 1000, 1000]
def test_erp_uses_managed_profile_when_bound_cdp_is_not_running(
monkeypatch,
tmp_path: Path,
@@ -95,6 +126,62 @@ def test_erp_attaches_only_to_the_live_managed_cdp(monkeypatch) -> None:
assert user_data_dir is None
def test_erp_login_navigates_before_probing_initial_blank_page(
monkeypatch,
) -> None:
events: list[tuple[str, str]] = []
class Page:
url = "about:blank"
context = object()
def goto(self, url, **_kwargs):
events.append(("goto", url))
self.url = url
def wait_for_timeout(self, _milliseconds):
return None
monkeypatch.setattr(erp_login, "restore_erp_runtime_state", lambda _context: None)
monkeypatch.setattr(erp_login, "has_valid_session", lambda _page: False)
monkeypatch.setattr(erp_login, "is_authenticated_session", lambda _page: False)
monkeypatch.setattr(erp_login, "first_visible", lambda *_args: object())
monkeypatch.setattr(
erp_login,
"fill_first_visible",
lambda _page, _selectors, _value, label: events.append(("fill", label)) or True,
)
monkeypatch.setattr(erp_login, "check_visible_checkboxes", lambda _page: None)
monkeypatch.setattr(erp_login, "click_login_button", lambda _page: events.append(("click", "login")))
monkeypatch.setattr(erp_login, "wait_for_manual_captcha", lambda *_args, **_kwargs: None)
monkeypatch.setattr(erp_login, "wait_until_logged_in", lambda *_args, **_kwargs: True)
monkeypatch.setattr(erp_login, "persist_erp_runtime_state", lambda _context: 1)
state: dict[str, object] = {}
action = erp_login.make_page_action(
login_url="https://erp.test/login.aspx",
username="user",
password="pw",
home_url="https://erp.test/epaas",
timeout_ms=1000,
captcha_checks=0,
captcha_interval_seconds=1,
hold_seconds=0,
login_only=True,
state=state,
)
action(Page())
assert events[:4] == [
("goto", "https://erp.test/login.aspx"),
("fill", "username"),
("fill", "password"),
("click", "login"),
]
assert state == {"ok": True, "final_url": "https://erp.test/login.aspx"}
def test_erp_main_fails_before_browser_when_no_style_is_collectable(monkeypatch) -> None:
args = SimpleNamespace(
config="unused.json",
@@ -192,6 +279,110 @@ def test_erp_main_fails_when_collector_summary_contains_failures(
assert erp.main() == 1
def test_erp_main_maps_login_recovery_to_cookie_skip(
monkeypatch,
tmp_path: Path,
) -> None:
args = SimpleNamespace(
config="unused.json",
target_date=date(2026, 8, 12),
mode="daily",
refresh_config=False,
input="unused-styles.json",
data_dir=str(tmp_path),
platform="all",
no_cdp=False,
timeout_ms=1000,
limit=0,
report_timeout=1,
report_attempts=1,
include_slow_codes=False,
hold_seconds=0,
headless=True,
)
styles = [
{
"style_name": "style-a",
"erp_style_codes": ["erp-a"],
"brand": erp.DEFAULT_BRAND,
}
]
monkeypatch.setattr(erp, "parse_args", lambda: args)
monkeypatch.setattr(
erp,
"load_config",
lambda _path: {
"login": {
"url": "https://erp.test/login",
"username": "placeholder",
"password": "placeholder",
}
},
)
monkeypatch.setattr(
erp,
"load_erp_styles",
lambda *_args: (
styles,
{"source": "lark", "collectable": ["style-a"], "skipped": []},
),
)
monkeypatch.setattr(erp, "print_skip_summary", lambda _report: None)
monkeypatch.setattr(erp, "write_erp_skip_report", lambda *_args: None)
monkeypatch.setattr(erp, "managed_data_path", lambda _path: tmp_path)
monkeypatch.setattr(erp, "resolve_browser_session", lambda _config: (None, None))
def fake_login_action(**kwargs):
state = kwargs["state"]
def action(_page):
state["ok"] = False
state["error"] = "Login did not complete; captcha required"
return action
monkeypatch.setattr(erp, "make_page_action", fake_login_action)
monkeypatch.setattr(
erp.DynamicFetcher,
"fetch",
lambda _url, **kwargs: kwargs["page_action"](object()),
)
assert erp.main() == COOKIE_SKIP_EXIT_CODE
def test_erp_login_waits_for_a_delayed_captcha(monkeypatch) -> None:
captcha_states = iter([False, False, True, True, False])
waits: list[int] = []
class Page:
url = "https://erp.test/login"
def wait_for_timeout(self, milliseconds):
waits.append(milliseconds)
monkeypatch.setattr(
erp_login,
"captcha_exists",
lambda _page: next(captcha_states),
)
monotonic_values = iter([0.0, 0.1, 0.2])
monkeypatch.setattr(
erp_login.time,
"monotonic",
lambda: next(monotonic_values),
)
erp_login.wait_for_manual_captcha(
Page(),
checks=3,
interval_seconds=5,
login_url="https://erp.test/login",
)
assert waits == [500, 500, 5000, 5000]
def test_erp_date_inputs_are_required_and_read_back() -> None:
source = inspect.getsource(erp.set_platform_and_code)
@@ -200,8 +391,11 @@ def test_erp_date_inputs_are_required_and_read_back() -> None:
assert "date range did not persist" in source
def test_playwright_channel_is_optional_and_environment_driven() -> None:
assert playwright_launch_options({}) == {}
assert playwright_launch_options({"GYXX_PLAYWRIGHT_CHANNEL": "chrome"}) == {
"channel": "chrome"
def test_scrapling_real_chrome_is_optional_and_environment_driven() -> None:
assert scrapling_session_options({}) == {}
assert scrapling_session_options({"GYXX_PLAYWRIGHT_CHANNEL": "chrome"}) == {
"real_chrome": True
}
assert scrapling_session_options({"GYXX_SCRAPLING_REAL_CHROME": "false"}) == {
"real_chrome": False
}
@@ -0,0 +1,92 @@
from __future__ import annotations
from pathlib import Path
import collect_douyin_qianchuan_ads as collector
def test_qianchuan_browser_uses_scrapling_stealth_and_persistent_state(
monkeypatch, tmp_path: Path
) -> None:
captured: dict[str, object] = {}
class _FakeBrowser:
def __init__(self, **options: object) -> None:
captured.update(options)
cookie_file = tmp_path / "cookies.json"
storage_state_file = tmp_path / "storage_state.json"
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_state_file))
monkeypatch.setattr(collector, "ScraplingBrowser", _FakeBrowser)
browser = collector._start_browser(True, str(tmp_path / "profile"))
assert isinstance(browser, _FakeBrowser)
assert captured["stealthy"] is True
assert captured["real_chrome"] is False
assert captured["headless"] is True
assert captured["user_data_dir"] == str((tmp_path / "profile").resolve())
assert captured["cookie_file"] == str(cookie_file)
assert captured["storage_state_file"] == str(storage_state_file)
assert captured["extra_flags"] == ["--no-proxy-server"]
assert captured["additional_args"] == {
"viewport": {"width": 1920, "height": 1080},
"accept_downloads": True,
}
def test_calendar_cell_prefers_actual_month_when_adjacent_month_duplicates() -> None:
class _Locator:
def __init__(self, count: int) -> None:
self._count = count
def count(self) -> int:
return self._count
class _Page:
def locator(self, selector: str) -> _Locator:
if ".ovui-date__cell--inview" in selector:
return _Locator(1)
return _Locator(2)
cell = collector._calendar_cell(_Page(), "2026-08-30")
assert cell.count() == 1
def test_calendar_cell_falls_back_when_picker_has_no_inview_class() -> None:
class _Locator:
def __init__(self, count: int) -> None:
self._count = count
def count(self) -> int:
return self._count
class _Page:
def locator(self, selector: str) -> _Locator:
if ".ovui-date__cell--inview" in selector:
return _Locator(0)
return _Locator(1)
cell = collector._calendar_cell(_Page(), "2026-08-30")
assert cell.count() == 1
def test_report_ready_wait_guards_missing_body() -> None:
captured: dict[str, object] = {}
class _Page:
def wait_for_function(self, expression: str, **options: object) -> None:
captured["expression"] = expression
captured.update(options)
page = _Page()
collector._wait_for_report_ready(page)
assert captured == {
"expression": collector.REPORT_READY_EXPRESSION,
"arg": list(collector._required_page_markers()),
"timeout": 90_000,
}
@@ -0,0 +1,175 @@
from __future__ import annotations
import zipfile
from datetime import date
from pathlib import Path
import collect_jd_ad_costs as collector
import pytest
def test_login_starts_from_jd_merchant_login_page() -> None:
assert collector.JD_LOGIN_URL == (
"https://passport.shop.jd.com/login/index.action/jdm?"
"ReturnUrl=https%3A%2F%2Fshop.jd.com%2Fjdm%2Fhome"
)
def test_ensure_logged_in_navigates_to_login_entry_first(monkeypatch) -> None:
visited: list[str] = []
monkeypatch.setattr(
collector,
"safe_goto",
lambda _page, url: visited.append(url),
)
monkeypatch.setattr(collector, "_login_complete", lambda _page: True)
collector.ensure_logged_in(object(), "光影行星研发", "secret")
assert visited == [collector.JD_LOGIN_URL]
def test_ensure_logged_in_reuses_valid_cookie_session(monkeypatch) -> None:
visited: list[str] = []
class Page:
def wait_for_timeout(self, _milliseconds: int) -> None:
return None
states = iter((False, True))
monkeypatch.setattr(
collector,
"safe_goto",
lambda _page, url: visited.append(url),
)
monkeypatch.setattr(collector, "_login_complete", lambda _page: next(states))
collector.ensure_logged_in(Page(), "光影行星研发", "secret")
assert visited == [collector.JD_LOGIN_URL, collector.JD_HOME_URL]
def test_login_failure_reason_exposes_jd_credential_error() -> None:
class Page:
def locator(self, _selector):
return self
def inner_text(self, timeout):
del timeout
return "账号名与密码不匹配"
assert collector._login_failure_reason(Page()) == "账号名与密码不匹配"
def test_safe_goto_stops_when_browser_context_is_closed() -> None:
class Page:
def goto(self, *_args, **_kwargs) -> None:
raise RuntimeError("Target page, context or browser has been closed")
def is_closed(self) -> bool:
return True
with pytest.raises(collector.JDAdCostCollectionError, match="上下文意外关闭"):
collector.safe_goto(Page(), "https://jzt.jd.com/custom-report/#/download")
def test_jd_ad_costs_uses_bundled_chromium_despite_global_chrome_override(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: dict[str, object] = {}
def build_browser(**options):
captured.update(options)
return object()
monkeypatch.setattr(collector, "ScraplingBrowser", build_browser)
monkeypatch.setenv("GYXX_SCRAPLING_REAL_CHROME", "true")
collector._start_browser(False, "D:/gyxx-flow/var/state/browser-profiles/jd-ad-costs")
assert captured["real_chrome"] is False
@pytest.mark.parametrize(
("text", "report_type", "expected"),
[
("营销概况_矩阵分析_SKU维度_2026-08-23 报告已生成", "non_full", "ready"),
("营销概况_下钻趋势图_2026-08-23 报表已生成", "full", "ready"),
("营销概况_矩阵分析_SKU维度_2026-08-23 生成中", "non_full", "pending"),
("营销概况_下钻趋势图_2026-08-22 报告已生成", "full", "absent"),
(
"光影行星研发_营销概况_矩阵分析_SKU维度_2026-08-17至2026-08-17"
" 报表已生成 创建日期 2026-08-23",
"non_full",
"absent",
),
(
"光影行星研发_营销概况_下钻趋势图_2026-06-01至2026-08-23"
" 报表已生成 创建日期 2026-08-23",
"full",
"absent",
),
],
)
def test_download_center_state_is_report_type_and_date_scoped(
text: str, report_type: collector.ReportType, expected: collector.ReportState
) -> None:
assert (
collector._report_state_from_text(text, report_type, date(2026, 8, 23))
== expected
)
def test_download_center_total_treats_empty_table_as_zero() -> None:
class Body:
def inner_text(self, timeout):
del timeout
return "下载报表 暂无数据"
class Page:
def locator(self, selector):
assert selector == "body"
return Body()
assert collector._download_center_total(Page()) == 0
def test_clear_download_center_deletes_each_page_until_empty(monkeypatch) -> None:
totals = iter((61, 51, 41, 31, 21, 11, 1, 0))
deleted_batches: list[int] = []
monkeypatch.setattr(collector, "_wait_for_download_center_table", lambda _page: None)
monkeypatch.setattr(collector, "_download_center_total", lambda _page: next(totals))
monkeypatch.setattr(
collector,
"_select_all_download_center_rows",
lambda _page: deleted_batches.append(1) or 10,
)
monkeypatch.setattr(
collector, "_delete_selected_download_center_rows", lambda _page: None
)
monkeypatch.setattr(collector, "_wait_for", lambda *args, **kwargs: None)
monkeypatch.setattr(collector, "_reload_download_center", lambda _page: None)
collector._clear_download_center(object())
assert len(deleted_batches) == 7
def test_safe_extract_zip_rejects_path_traversal(tmp_path: Path) -> None:
archive = tmp_path / "report.zip"
with zipfile.ZipFile(archive, "w") as handle:
handle.writestr("../outside.csv", "bad")
with pytest.raises(collector.JDAdCostCollectionError, match="路径越界"):
collector._safe_extract_zip(archive, tmp_path / "extracted")
def test_safe_extract_zip_keeps_csv_inside_output(tmp_path: Path) -> None:
archive = tmp_path / "report.zip"
with zipfile.ZipFile(archive, "w") as handle:
handle.writestr("nested/full.csv", "日期,'当前时间',SPU ID\n")
files = collector._safe_extract_zip(archive, tmp_path / "extracted")
assert files == [(tmp_path / "extracted" / "nested" / "full.csv").resolve()]
@@ -0,0 +1,180 @@
from __future__ import annotations
import json
import zipfile
from datetime import date
from pathlib import Path
import collect_tmall_wanxiang_ads as collector
import pytest
PROJECT_ROOT = Path(__file__).resolve().parents[3]
def test_item_promotion_url_contains_explicit_target_date() -> None:
url = collector.item_promotion_url(date(2026, 8, 17))
assert "#!/report/item_promotion" in url
assert "startTime=2026-08-17" in url
assert "endTime=2026-08-17" in url
@pytest.mark.parametrize(
("text", "expected"),
[
("商品报表 2026-08-18 生成成功", "ready"),
("商品报表 2026-08-18 生成中", "pending"),
("商品报表 2026-08-18 生成失败", "failed"),
("商品报表 2026-08-17 生成成功", "absent"),
# 数据日期范围不匹配,仅创建日期含目标日:不能误判为同日报表
(
"生成成功 商品报表_20260818_120200 15天累计数据 2026-08-11至2026-08-11 2026-08-18",
"absent",
),
# 数据日期范围覆盖目标日:可复用
(
"生成成功 商品报表_20260819_113811 15天累计数据 2026-08-18至2026-08-18 2026-08-19",
"ready",
),
],)
def test_report_task_state_is_date_scoped_and_conservative(
text: str, expected: collector.ReportTaskState
) -> None:
assert collector._report_task_state_from_text(text, date(2026, 8, 18)) == expected
def test_scheduled_wanxiang_workflow_does_not_force_duplicate_request() -> None:
payload = json.loads(
(PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8")
)
workflow = next(
item for item in payload["workflows"] if item["id"] == "product.ecommerce_costs.daily"
)
download_step = next(
step for step in workflow["execution"]["steps"] if step["id"] == "tmall_download"
)
assert "--force-request" not in download_step["args"]
def test_request_report_submits_outer_dialog_only_once(
monkeypatch: pytest.MonkeyPatch,
) -> None:
page = object()
modal = {"open": True}
confirm_calls = 0
selected_dates: list[date] = []
monkeypatch.setattr(collector, "_wait_for_text", lambda *args, **kwargs: None)
monkeypatch.setattr(collector, "_click_visible_text", lambda *args, **kwargs: True)
monkeypatch.setattr(
collector,
"_select_date_range",
lambda _page, target_date: selected_dates.append(target_date),
)
monkeypatch.setattr(collector, "_modal_visible", lambda _page: modal["open"])
def confirm(_page: object) -> bool:
nonlocal confirm_calls
confirm_calls += 1
modal["open"] = False
return True
monkeypatch.setattr(collector, "_click_download_dialog_confirm", confirm)
collector.request_report(page, date(2026, 8, 18))
assert confirm_calls == 1
assert selected_dates == [date(2026, 8, 18)]
def test_run_waits_for_existing_pending_task_instead_of_requesting_another(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
class _Session:
def close(self) -> None:
pass
session = _Session()
page = object()
calls: list[bool] = []
report_zip = tmp_path / "report.zip"
extracted_csv = tmp_path / "report.csv"
monkeypatch.setattr(
collector,
"_start_session",
lambda _headless, _user_data_dir: (session, object(), page),
)
monkeypatch.setattr(collector, "ensure_logged_in", lambda *args, **kwargs: None)
monkeypatch.setattr(
collector,
"download_ready_report",
lambda _page, _date, _output, *, timeout_seconds, wait_for_generation: (
calls.append(wait_for_generation)
or (None if len(calls) == 1 else report_zip)
),
)
monkeypatch.setattr(collector, "_report_task_state", lambda *_args: "pending")
monkeypatch.setattr(
collector,
"extract_report_zip",
lambda _zip_path, _target_date: [extracted_csv],
)
monkeypatch.setattr(
collector,
"request_report",
lambda *_args, **_kwargs: pytest.fail("must not submit a second task"),
)
args = collector.build_parser().parse_args(
["--date", "2026-08-18", "--output", str(tmp_path)]
)
# Content validation only re-requests when the reused report does not
# cover the target date; a matching CSV must keep the reuse path.
extracted_csv.write_text("日期,主体ID\n2026-08-18,1\n", encoding="utf-8")
assert collector.run(args) == [extracted_csv]
assert calls == [False, True]
class _Page:
def __init__(self, url: str, body: str) -> None:
self.url = url
self._body = body
def locator(self, selector: str):
assert selector == "body"
return self
def inner_text(self, timeout: int = 0) -> str:
return self._body
def test_login_success_accepts_taobao_seller_center_before_redirect() -> None:
assert collector._is_taobao_login_complete(
_Page("https://myseller.taobao.com/home", "物流监控 违规预警")
)
assert not collector._is_taobao_login_complete(
_Page("https://loginmyseller.taobao.com/", "请输入登录密码")
)
def test_safe_extract_zip_returns_csv_and_writes_manifest(tmp_path: Path) -> None:
archive_path = tmp_path / "商品报表.zip"
with zipfile.ZipFile(archive_path, "w") as archive:
archive.writestr("商品报表_20260817.csv", "日期,主体ID\n")
archive.writestr("说明.txt", "万相台商品报表")
csv_files = collector.extract_report_zip(archive_path, date(2026, 8, 17))
assert [path.name for path in csv_files] == ["商品报表_20260817.csv"]
manifest = (tmp_path / "manifest.json").read_text(encoding="utf-8")
assert '"target_date": "2026-08-17"' in manifest
assert (tmp_path / "extracted" / "商品报表_20260817.csv").is_file()
def test_safe_extract_zip_rejects_path_traversal(tmp_path: Path) -> None:
archive_path = tmp_path / "unsafe.zip"
with zipfile.ZipFile(archive_path, "w") as archive:
archive.writestr("../escape.csv", "bad")
with pytest.raises(collector.WanxiangDownloadError, match="路径越界"):
collector._safe_extract_zip(archive_path, tmp_path / "out")
@@ -13,6 +13,11 @@ import orchestrate_daily_collection as daily
import pytest
from commands import import_daily as import_daily_command
from gyxx_flow.core.exit_codes import (
COOKIE_SKIP_EXIT_CODE,
NON_RETRYABLE_EXIT_CODE,
)
class _Context:
def __init__(self, value):
@@ -93,6 +98,153 @@ def test_selected_platform_collector_failure_stops_daily_aggregation(monkeypatch
target_date="2026-08-03",
)
def test_cookie_skip_is_not_retried_by_daily_orchestrator(monkeypatch) -> None:
calls = 0
def fake_run(*_args, **_kwargs):
nonlocal calls
calls += 1
return {"name": "erp", "code": COOKIE_SKIP_EXIT_CODE}
monkeypatch.setattr(daily, "run_step", fake_run)
monkeypatch.setattr(
daily.time,
"sleep",
lambda *_args: pytest.fail("cookie recovery must not enter retry cooldown"),
)
with pytest.raises(daily.CookiePreflightSkipped, match="erp requires"):
daily.run_step_with_retry("erp", ["python", "collector.py"])
assert calls == 1
def test_non_retryable_collector_result_is_not_retried(monkeypatch) -> None:
calls = 0
def fake_run(*_args, **_kwargs):
nonlocal calls
calls += 1
return {"name": "tm", "code": NON_RETRYABLE_EXIT_CODE}
monkeypatch.setattr(daily, "run_step", fake_run)
monkeypatch.setattr(
daily.time,
"sleep",
lambda *_args: pytest.fail("deterministic validation must not retry"),
)
result = daily.run_step_with_retry(
"tm",
["python", "collector.py"],
critical=False,
)
assert result["code"] == NON_RETRYABLE_EXIT_CODE
assert calls == 1
def test_daily_main_propagates_cookie_skip_exit_code(monkeypatch) -> None:
monkeypatch.setattr(
daily,
"run_collect",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
daily.CookiePreflightSkipped("erp requires browser login")
),
)
monkeypatch.setattr(
daily.argparse.ArgumentParser,
"parse_args",
lambda _parser: SimpleNamespace(
date=None,
target_date="2026-08-12",
stages=["collect"],
skip_erp=False,
skip_platforms=False,
platforms="jd,dy,tm",
serial_platforms=False,
skip_insert=False,
skip_reviews_import=False,
dry_run=False,
erp_mode="daily",
max_run_retries=2,
),
)
assert daily.main() == COOKIE_SKIP_EXIT_CODE
def test_erp_mode_auto_uses_daily_only_for_yesterday() -> None:
today = datetime(2026, 8, 13).date()
assert daily.resolve_erp_mode("2026-08-12", today=today) == "daily"
assert daily.resolve_erp_mode("2026-08-11", today=today) == "backfill"
assert daily.resolve_erp_mode("2026-08-13", today=today) == "backfill"
assert daily.is_historical_target("2026-08-12", today=today) is False
assert daily.is_historical_target("2026-08-11", today=today) is True
@pytest.mark.parametrize("mode", ["daily", "weekly", "backfill"])
def test_explicit_erp_mode_overrides_automatic_selection(mode) -> None:
assert (
daily.resolve_erp_mode(
"2026-08-11",
requested_mode=mode,
today=datetime(2026, 8, 13).date(),
)
== mode
)
def test_historical_daily_collection_passes_backfill_mode_to_erp(
monkeypatch,
) -> None:
args = SimpleNamespace(
skip_erp=False,
skip_platforms=True,
skip_reviews_import=True,
dry_run=False,
erp_mode="auto",
)
calls = []
monkeypatch.setattr(
daily,
"resolve_erp_mode",
lambda *_args, **_kwargs: "backfill",
)
monkeypatch.setattr(
daily,
"run_step_with_retry",
lambda *args, **kwargs: calls.append((args, kwargs))
or {"name": "erp", "code": 0},
)
daily.run_collect(args, "2026-08-11", ["jd", "dy", "tm"], failed=None)
command = calls[0][0][1]
assert command[-4:] == ["--target-date", "2026-08-11", "--mode", "backfill"]
def test_historical_tm_collection_uses_backfill_child(monkeypatch) -> None:
calls: list[tuple[str, list[str]]] = []
def fake_run(name, command, **_kwargs):
calls.append((name, command))
return {"name": name, "code": 0}
monkeypatch.setattr(daily, "run_step_with_retry", fake_run)
monkeypatch.setattr(daily, "verify_platform_artifacts", lambda *_args: 1)
daily.run_platforms(
parallel=False,
platforms=["tm"],
target_date="2026-08-11",
historical=True,
)
assert calls[0][1][-3:] == ["--backfill", "--target-date", "2026-08-11"]
def test_platform_selection_keeps_omitted_collectors_optional(monkeypatch) -> None:
calls: list[str] = []
@@ -217,6 +369,63 @@ def test_platform_artifact_verifier_accepts_fresh_valid_product(
assert daily.verify_platform_artifacts("jd", "2026-08-03", marker_ns) == 1
def test_dy_verifier_accepts_source_date_format_and_legitimate_zero_style(
monkeypatch,
tmp_path,
) -> None:
class Loader:
def get_daily_styles(self):
return {
"dy": {
"collectable": ["matched-style", "zero-style"],
"skipped": [],
}
}
monkeypatch.setattr(daily, "StyleConfigLoader", Loader)
monkeypatch.setattr(daily, "DATA_ROOT", tmp_path)
marker_ns = time.time_ns()
for style, matched in (("matched-style", 2), ("zero-style", 0)):
path = daily._platform_artifact_path("dy", style, "2026-08-03")
path.parent.mkdir(parents=True)
path.write_text(
json.dumps({
"date_range": "2026/08/03 - 2026/08/03",
"matched_row_count": matched,
"download_file": "_downloads/2026-08-03/report.xlsx",
}),
encoding="utf-8",
)
assert daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns) == 2
def test_platform_artifact_verifier_rejects_an_export_with_no_product_matches(
monkeypatch,
tmp_path,
) -> None:
class Loader:
def get_daily_styles(self):
return {"dy": {"collectable": ["zero-style"], "skipped": []}}
monkeypatch.setattr(daily, "StyleConfigLoader", Loader)
monkeypatch.setattr(daily, "DATA_ROOT", tmp_path)
marker_ns = time.time_ns()
path = daily._platform_artifact_path("dy", "zero-style", "2026-08-03")
path.parent.mkdir(parents=True)
path.write_text(
json.dumps({
"date_range": "2026/08/03 - 2026/08/03",
"matched_row_count": 0,
"download_file": "_downloads/2026-08-03/report.xlsx",
}),
encoding="utf-8",
)
with pytest.raises(RuntimeError, match="matched no configured products"):
daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns)
def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
args = SimpleNamespace(
skip_erp=True,
@@ -240,6 +449,27 @@ def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
assert calls[0][1]["critical"] is False
def test_daily_import_stage_uses_the_collected_business_date(monkeypatch) -> None:
args = SimpleNamespace(dry_run=True, platforms="jd,tm")
calls = []
monkeypatch.setattr(
daily,
"run_step_with_retry",
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0},
)
daily.run_import(args, "2026-08-03", failed=None)
assert calls[0][0][0] == "import_product_daily"
assert calls[0][0][1][-4:] == [
"--date",
"2026-08-03",
"--platforms",
"jd,tm",
]
assert calls[0][1]["dry_run"] is True
def test_formal_aggregation_fails_when_postgres_write_is_not_fresh(monkeypatch) -> None:
marker = datetime(2026, 8, 4, 3, 0, tzinfo=timezone.utc)
args = SimpleNamespace(dry_run=False)
@@ -374,6 +604,26 @@ def test_import_fails_when_selected_platform_has_no_target_directory(monkeypatch
assert "dy: 没有匹配所选日期范围的报表目录" in capsys.readouterr().err
def test_import_deduplicates_rerun_rows_before_one_upsert_batch(capsys) -> None:
first = {
**_record(),
"source_file": "old-report.xlsx",
"visitors": 10,
}
latest = {
**_record(),
"source_file": "latest-report.xlsx",
"visitors": 20,
}
result = product_import._deduplicate_records_for_upsert([first, latest])
assert len(result) == 1
assert result[0]["source_file"] == "latest-report.xlsx"
assert result[0]["visitors"] == 20
assert "丢弃重复原始行: 1 行" in capsys.readouterr().out
def test_import_fails_when_target_directory_has_no_input_file(monkeypatch, capsys) -> None:
target_dir = Path("2026-08-03")
monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [target_dir])
@@ -0,0 +1,133 @@
from __future__ import annotations
import db
import pytest
from db import InterfaceError, get_conn
class FakeCursor:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def execute(self, _sql):
return None
def fetchone(self):
return (1,)
class FakeConnection:
def __init__(self, alive: bool) -> None:
self.alive = alive
self.closed = False
self.committed = False
self.rolled_back = False
def rollback(self) -> None:
if not self.alive:
raise RuntimeError("connection already closed")
self.rolled_back = True
def cursor(self):
if not self.alive:
raise RuntimeError("connection already closed")
return FakeCursor()
def commit(self) -> None:
self.committed = True
def close(self) -> None:
self.closed = True
class FakePool:
def __init__(self, connections):
self.connections = list(connections)
self.returned: list[tuple[object, bool]] = []
def getconn(self):
if not self.connections:
raise RuntimeError("pool exhausted")
return self.connections.pop(0)
def putconn(self, conn, close: bool = False) -> None:
self.returned.append((conn, close))
def test_get_conn_reuses_healthy_pooled_connection(
monkeypatch: pytest.MonkeyPatch,
) -> None:
connection = FakeConnection(alive=True)
fake_pool = FakePool([connection])
monkeypatch.setattr(db, "_get_pool", lambda: fake_pool)
with get_conn() as acquired:
assert acquired is connection
assert connection.committed is True
assert fake_pool.returned == [(connection, False)]
def test_get_conn_discards_stale_connection_and_borrows_fresh(
monkeypatch: pytest.MonkeyPatch,
) -> None:
stale = FakeConnection(alive=False)
fresh = FakeConnection(alive=True)
fake_pool = FakePool([stale, fresh])
monkeypatch.setattr(db, "_get_pool", lambda: fake_pool)
with get_conn() as acquired:
assert acquired is fresh
assert stale.closed is True
assert fresh.committed is True
assert fake_pool.returned == [(stale, True), (fresh, False)]
def test_get_conn_fails_closed_when_all_pooled_connections_are_dead(
monkeypatch: pytest.MonkeyPatch,
) -> None:
first = FakeConnection(alive=False)
second = FakeConnection(alive=False)
fake_pool = FakePool([first, second])
monkeypatch.setattr(db, "_get_pool", lambda: fake_pool)
with pytest.raises(InterfaceError):
with get_conn():
pass
assert first.closed is True
assert second.closed is True
assert fake_pool.returned == [(first, True), (second, True)]
def test_get_pool_retries_transient_initial_connection(
monkeypatch: pytest.MonkeyPatch,
) -> None:
attempts: list[str] = []
def fake_pool(**_kwargs):
attempts.append("connect")
if len(attempts) == 1:
raise db.OperationalError("transient connection failure")
return object()
for name, value in {
"PG_HOST": "db.example.test",
"PG_PORT": "5432",
"PG_DB": "warehouse",
"PG_USER": "collector",
"PG_PASSWORD": "unit-test-only",
}.items():
monkeypatch.setenv(name, value)
monkeypatch.setattr(db, "_pool", None)
monkeypatch.setattr(db.pool, "SimpleConnectionPool", fake_pool)
monkeypatch.setattr(db.time, "sleep", lambda _seconds: attempts.append("sleep"))
result = db._get_pool(connect_attempts=2, retry_delay_seconds=0)
assert result is not None
assert attempts == ["connect", "sleep", "connect"]
@@ -0,0 +1,183 @@
from __future__ import annotations
import json
import pytest
from gyxx_flow.modules.product_commerce.direct_llm import (
DirectLLMConfig,
DirectLLMConfigurationError,
DirectLLMResponseError,
DirectLLMTransientError,
chat_completion,
load_direct_llm_config,
)
def _config() -> DirectLLMConfig:
return DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="test-only-secret",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
)
def _response(content: str) -> bytes:
return json.dumps(
{
"choices": [{"message": {"content": content}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 4},
}
).encode()
def test_direct_chat_uses_provider_endpoint_and_model_without_echoing_secret() -> None:
observed: dict[str, object] = {}
def transport(url, headers, body, timeout):
payload = json.loads(body)
observed.update(
url=url,
model=payload["model"],
messages=payload["messages"],
reasoning_split=payload["reasoning_split"],
timeout=timeout,
authorization=headers["Authorization"],
)
return _response('{"ok":true}')
content, usage = chat_completion(
[{"role": "user", "content": "hello"}],
model="MiniMax-M3",
timeout=17,
transport=transport,
config=_config(),
)
assert content == '{"ok":true}'
assert usage == {"prompt_tokens": 3, "completion_tokens": 4}
assert observed["url"] == "https://api.minimaxi.com/v1/chat/completions"
assert observed["model"] == "MiniMax-M3"
assert observed["reasoning_split"] is True
assert observed["timeout"] == 17.0
assert observed["authorization"] == "Bearer test-only-secret"
def test_direct_chat_can_disable_minimax_thinking_for_structured_video_tasks() -> None:
observed: dict[str, object] = {}
def transport(_url, _headers, body, _timeout):
observed.update(json.loads(body))
return _response("ok")
chat_completion(
[{"role": "user", "content": "return one token"}],
thinking_mode="disabled",
transport=transport,
config=_config(),
)
assert observed["thinking"] == {"type": "disabled"}
def test_direct_chat_rejects_unknown_thinking_mode() -> None:
with pytest.raises(DirectLLMConfigurationError, match="thinking mode"):
chat_completion(
[{"role": "user", "content": "hello"}],
thinking_mode="sometimes",
transport=lambda *_args: _response("unused"),
config=_config(),
)
def test_config_reads_shared_minimax_environment(monkeypatch) -> None:
monkeypatch.delenv("GYXX_DIRECT_LLM_BASE_URL", raising=False)
monkeypatch.delenv("GYXX_DIRECT_LLM_API_KEY", raising=False)
monkeypatch.delenv("GYXX_DIRECT_LLM_MODEL", raising=False)
monkeypatch.setenv("STYLE_ANALYSIS_LLM_BASE_URL", "https://api.minimaxi.com/v1")
monkeypatch.setenv("STYLE_ANALYSIS_LLM_API_KEY", "test-only-secret")
monkeypatch.setenv("STYLE_ANALYSIS_LLM_MODEL", "MiniMax-M3")
config = load_direct_llm_config()
assert config.endpoint == "https://api.minimaxi.com/v1/chat/completions"
assert config.title_model == "MiniMax-M3"
assert config.vision_model == "MiniMax-M3"
assert "test-only-secret" not in repr(config)
def test_config_rejects_legacy_local_gateway(monkeypatch) -> None:
monkeypatch.setenv(
"GYXX_DIRECT_LLM_BASE_URL",
"http://127.0.0.1:8642/v1",
)
monkeypatch.setenv("GYXX_DIRECT_LLM_API_KEY", "test-only-secret")
with pytest.raises(DirectLLMConfigurationError, match="local gateway"):
load_direct_llm_config()
def test_invalid_response_is_fail_closed() -> None:
with pytest.raises(DirectLLMResponseError, match="invalid JSON"):
chat_completion(
[{"role": "user", "content": "hello"}],
transport=lambda *_args: b"not-json",
config=_config(),
)
def test_direct_chat_retries_transient_provider_overload(monkeypatch) -> None:
calls = 0
def transport(_url, _headers, _body, _timeout):
nonlocal calls
calls += 1
if calls < 3:
raise DirectLLMTransientError(
"direct model request failed with HTTP 529"
)
return _response("recovered")
delays: list[float] = []
monkeypatch.setattr(
"gyxx_flow.modules.product_commerce.direct_llm.time.sleep",
delays.append,
)
content, _usage = chat_completion(
[{"role": "user", "content": "hello"}],
transport=transport,
config=_config(),
)
assert content == "recovered"
assert calls == 3
assert delays == [1.0, 3.0]
def test_direct_chat_preserves_transient_error_after_bounded_retries(
monkeypatch,
) -> None:
calls = 0
def transport(_url, _headers, _body, _timeout):
nonlocal calls
calls += 1
raise DirectLLMTransientError(
"direct model request failed with HTTP 529"
)
monkeypatch.setattr(
"gyxx_flow.modules.product_commerce.direct_llm.time.sleep",
lambda _seconds: None,
)
with pytest.raises(DirectLLMTransientError, match="HTTP 529"):
chat_completion(
[{"role": "user", "content": "hello"}],
transport=transport,
config=_config(),
)
assert calls == 3
@@ -0,0 +1,154 @@
from __future__ import annotations
import base64
import json
from pathlib import Path
from PIL import Image
from gyxx_flow.modules.product_commerce.direct_llm import DirectLLMConfig
from gyxx_flow.modules.product_commerce.direct_visual_color import (
DirectVisualColorClassifier,
VisualCandidate,
)
def _image(path: Path, color: str) -> Path:
Image.new("RGB", (12, 12), color).save(path, format="PNG")
return path
def _response(classification: dict[str, object]) -> bytes:
return json.dumps(
{
"choices": [
{
"message": {
"content": json.dumps(classification),
}
}
]
}
).encode()
def test_direct_visual_classifier_sends_images_to_minimax_m3(tmp_path) -> None:
source = _image(tmp_path / "source.png", "black")
candidate = _image(tmp_path / "candidate.png", "black")
observed: dict[str, object] = {}
def transport(url, headers, body, timeout):
payload = json.loads(body)
image_parts = [
part
for part in payload["messages"][1]["content"]
if part["type"] == "image_url"
]
observed.update(
url=url,
model=payload["model"],
thinking=payload["thinking"],
timeout=timeout,
image_count=len(image_parts),
data_url_prefix=image_parts[0]["image_url"]["url"][:22],
has_auth=headers["Authorization"].startswith("Bearer "),
)
return _response(
{
"source_color": "black",
"candidate_colors": {"candidate": "black"},
"visual_match": {"unique": False, "key": None},
}
)
result = DirectVisualColorClassifier(
transport=transport,
config_resolver=lambda: DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="test-only-secret",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
),
).classify([source], [VisualCandidate("candidate", candidate)])
assert result.source_color == "black"
assert result.selected_key == "candidate"
assert observed == {
"url": "https://api.minimaxi.com/v1/chat/completions",
"model": "MiniMax-M3",
"thinking": {"type": "disabled"},
"timeout": 120.0,
"image_count": 2,
"data_url_prefix": "data:image/png;base64,",
"has_auth": True,
}
def test_direct_visual_payload_uses_external_provider_endpoint(tmp_path) -> None:
source = _image(tmp_path / "source.png", "black")
candidate = _image(tmp_path / "candidate.png", "black")
captured: dict[str, object] = {}
def transport(url, _headers, body, _timeout):
captured["url"] = url
payload = json.loads(body)
image_url = next(
part["image_url"]["url"]
for part in payload["messages"][1]["content"]
if part["type"] == "image_url"
)
assert base64.b64decode(image_url.split(",", 1)[1])
return _response(
{
"source_color": "unknown",
"candidate_colors": {"candidate": "black"},
"visual_match": {"unique": False, "key": None},
}
)
DirectVisualColorClassifier(
transport=transport,
config_resolver=lambda: DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="test-only-secret",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
),
).classify([source], [VisualCandidate("candidate", candidate)])
assert captured["url"] == "https://api.minimaxi.com/v1/chat/completions"
assert "8642" not in str(captured["url"])
def test_direct_visual_classifier_accepts_model_reasoning_wrapper(tmp_path) -> None:
source = _image(tmp_path / "source.png", "black")
candidate = _image(tmp_path / "candidate.png", "black")
classification = {
"source_color": "black",
"candidate_colors": {"candidate": "black"},
"visual_match": {"unique": False, "key": None},
}
def transport(_url, _headers, _body, _timeout):
content = (
"<think>分析图片后再给出严格结果。</think>\n"
"```json\n"
+ json.dumps(classification)
+ "\n```"
)
return json.dumps(
{"choices": [{"message": {"content": content}}]}
).encode()
result = DirectVisualColorClassifier(
transport=transport,
config_resolver=lambda: DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="test-only-secret",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
),
).classify([source], [VisualCandidate("candidate", candidate)])
assert result.source_color == "black"
assert result.selected_key == "candidate"
@@ -0,0 +1,524 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import pytest
from PIL import Image
from gyxx_flow.modules.product_commerce.direct_llm import DirectLLMConfig
from gyxx_flow.modules.product_commerce.direct_visual_color import (
COARSE_COLORS,
MAX_CANDIDATES,
DirectVisualColorClassifier,
DirectVisualColorError,
VisualCandidate,
normalize_coarse_color,
)
def _image(path: Path, color: str) -> Path:
Image.new("RGB", (12, 12), color).save(path, format="PNG")
return path
def _response(classification: dict[str, Any]) -> bytes:
return json.dumps(
{
"choices": [
{
"message": {
"role": "assistant",
"content": json.dumps(classification),
}
}
]
}
).encode()
def _classifier(response: bytes, observed: dict[str, Any] | None = None):
def transport(url, headers, body, timeout):
if observed is not None:
payload = json.loads(body)
image_parts = [
part
for part in payload["messages"][1]["content"]
if part["type"] == "image_url"
]
observed.update(
{
"url": url,
"timeout": timeout,
"model": payload["model"],
"image_count": len(image_parts),
"all_data_urls": all(
part["image_url"]["url"].startswith(
"data:image/png;base64,"
)
for part in image_parts
),
"has_authorization": headers.get("Authorization", "").startswith(
"Bearer "
),
}
)
return response
return DirectVisualColorClassifier(
transport=transport,
config_resolver=lambda: DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="test-only-credential",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
),
)
def test_unique_coarse_color_match_uses_local_images_as_data_urls(tmp_path) -> None:
source = _image(tmp_path / "source.png", "saddlebrown")
black = _image(tmp_path / "black.png", "black")
brown = _image(tmp_path / "brown.png", "peru")
observed: dict[str, Any] = {}
classifier = _classifier(
_response(
{
"source_color": "brown",
"candidate_colors": {"sku-black": "black", "sku-brown": "tan"},
"visual_match": {"unique": False, "key": None},
}
),
observed,
)
result = classifier.classify(
[source],
[
VisualCandidate("sku-black", black),
VisualCandidate("sku-brown", brown),
],
)
assert result.source_color == "brown"
assert result.candidate_colors == {"sku-black": "black", "sku-brown": "brown"}
assert result.selected_key == "sku-brown"
assert result.selection_basis == "coarse_color"
assert observed == {
"url": "https://api.minimaxi.com/v1/chat/completions",
"timeout": 120.0,
"model": "MiniMax-M3",
"image_count": 3,
"all_data_urls": True,
"has_authorization": True,
}
def test_same_color_tie_allows_one_visual_similarity_match(tmp_path) -> None:
source_a = _image(tmp_path / "source-a.png", "navy")
source_b = _image(tmp_path / "source-b.png", "blue")
first = _image(tmp_path / "first.png", "navy")
second = _image(tmp_path / "second.png", "blue")
classifier = _classifier(
_response(
{
"source_color": "navy",
"candidate_colors": {"first": "blue", "second": "blue"},
"visual_match": {"unique": True, "key": "second"},
}
)
)
result = classifier.classify(
[source_a, source_b],
[VisualCandidate("first", first), VisualCandidate("second", second)],
)
assert result.source_color == "blue"
assert result.selected_key == "second"
assert result.selection_basis == "visual_similarity"
@pytest.mark.parametrize(
"classification",
[
{
"source_color": "blue",
"candidate_colors": {"first": "blue", "second": "blue"},
"visual_match": {"unique": True, "key": "invented"},
},
{
"source_color": "blue",
"candidate_colors": {"first": "blue", "invented": "blue"},
"visual_match": {"unique": False, "key": None},
},
{
"source_color": "blue",
"candidate_colors": {"first": "blue"},
"visual_match": {"unique": False, "key": None},
},
],
)
def test_unknown_or_missing_candidate_key_fails_closed(
tmp_path,
classification,
) -> None:
source = _image(tmp_path / "source.png", "blue")
first = _image(tmp_path / "first.png", "blue")
second = _image(tmp_path / "second.png", "blue")
with pytest.raises(DirectVisualColorError, match="candidate key"):
_classifier(_response(classification)).classify(
[source],
[VisualCandidate("first", first), VisualCandidate("second", second)],
)
def test_one_character_candidate_key_typo_fails_closed(tmp_path) -> None:
source = _image(tmp_path / "source.png", "black")
candidate = _image(tmp_path / "candidate.png", "black")
with pytest.raises(DirectVisualColorError, match="unknown candidate key"):
_classifier(
_response(
{
"source_color": "black",
"candidate_colors": {"card-eb51a74da3a4": "black"},
"visual_match": {"unique": False, "key": None},
}
)
).classify(
[source],
[VisualCandidate("card-eb51a74da3a3", candidate)],
)
def test_visual_match_cannot_bypass_coarse_color_filter(tmp_path) -> None:
source = _image(tmp_path / "source.png", "red")
red = _image(tmp_path / "red.png", "red")
blue = _image(tmp_path / "blue.png", "blue")
with pytest.raises(DirectVisualColorError, match="inconsistent"):
_classifier(
_response(
{
"source_color": "red",
"candidate_colors": {"red": "red", "blue": "blue"},
"visual_match": {"unique": True, "key": "blue"},
}
)
).classify(
[source],
[VisualCandidate("red", red), VisualCandidate("blue", blue)],
)
@pytest.mark.parametrize(
"assistant_content",
[
"not-json",
'```json\n{"source_color":"black"}\n```',
'{"source_color": NaN}',
'{"source_color":"black","candidate_colors":{},"visual_match":{},"extra":1}',
],
)
def test_non_strict_model_json_fails_closed(tmp_path, assistant_content) -> None:
source = _image(tmp_path / "source.png", "black")
candidate = _image(tmp_path / "candidate.png", "black")
response = json.dumps(
{"choices": [{"message": {"content": assistant_content}}]}
).encode()
with pytest.raises(DirectVisualColorError):
_classifier(response).classify(
[source],
[VisualCandidate("candidate", candidate)],
)
def test_non_string_source_color_fails_closed(tmp_path) -> None:
source = _image(tmp_path / "source.png", "black")
candidate = _image(tmp_path / "candidate.png", "black")
with pytest.raises(DirectVisualColorError, match="source color"):
_classifier(
_response(
{
"source_color": None,
"candidate_colors": {"candidate": "black"},
"visual_match": {"unique": False, "key": None},
}
)
).classify(
[source],
[VisualCandidate("candidate", candidate)],
)
def test_image_count_limits_fail_before_transport(tmp_path) -> None:
image = _image(tmp_path / "image.png", "black")
called = False
def transport(*_args):
nonlocal called
called = True
raise AssertionError("must not call transport")
classifier = DirectVisualColorClassifier(
transport=transport,
config_resolver=lambda: DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="test-only-credential",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
),
)
with pytest.raises(DirectVisualColorError, match="between 1 and 2"):
classifier.classify([], [VisualCandidate("one", image)])
with pytest.raises(DirectVisualColorError, match="between 1 and 2"):
classifier.classify([image, image, image, image], [VisualCandidate("one", image)])
with pytest.raises(DirectVisualColorError, match="safe limit"):
classifier.classify(
[image],
[VisualCandidate(f"candidate-{index}", image) for index in range(MAX_CANDIDATES + 1)],
)
assert called is False
def test_http_failure_never_echoes_secret_or_image_payload(tmp_path) -> None:
source = _image(tmp_path / "source.png", "black")
candidate = _image(tmp_path / "candidate.png", "black")
leaked_marker = "data:image/png;base64,SHOULD-NOT-LEAK"
def transport(*_args):
raise RuntimeError(f"secret-value {leaked_marker}")
classifier = DirectVisualColorClassifier(
transport=transport,
config_resolver=lambda: DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="secret-value",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
),
)
with pytest.raises(DirectVisualColorError) as captured:
classifier.classify(
[source],
[VisualCandidate("candidate", candidate)],
)
message = str(captured.value)
assert "secret-value" not in message
assert "base64" not in message
assert captured.value.__cause__ is None
def test_missing_or_invalid_image_fails_before_transport(tmp_path) -> None:
invalid = tmp_path / "invalid.png"
invalid.write_text("not an image", encoding="utf-8")
candidate = _image(tmp_path / "candidate.png", "black")
with pytest.raises(DirectVisualColorError, match="invalid"):
_classifier(b"unused").classify(
[invalid],
[VisualCandidate("candidate", candidate)],
)
def test_color_normalization_is_closed() -> None:
assert normalize_coarse_color("Grey") == "unknown"
assert normalize_coarse_color("light gray") == "white"
assert normalize_coarse_color("\u6d45\u7070") == "white"
assert normalize_coarse_color("silver-white") == "white"
assert normalize_coarse_color("dark gray") == "black"
assert normalize_coarse_color("\u70ad\u7070") == "black"
assert normalize_coarse_color("\u6df1\u7070") == "black"
assert normalize_coarse_color("\u5361\u5176") == "brown"
assert normalize_coarse_color("navy") == "blue"
assert normalize_coarse_color("transparent") == "unknown"
assert normalize_coarse_color(None) == "unknown"
def test_gray_model_output_is_unknown_and_never_matches(tmp_path) -> None:
source = _image(tmp_path / "source.png", "gray")
candidate = _image(tmp_path / "candidate.png", "white")
result = _classifier(
_response(
{
"source_color": "gray",
"candidate_colors": {"candidate": "white"},
"visual_match": {"unique": False, "key": None},
}
)
).classify([source], [VisualCandidate("candidate", candidate)])
assert result.source_color == "unknown"
assert result.candidate_colors == {"candidate": "white"}
assert result.selected_key is None
assert result.selection_basis == "unresolved"
def test_light_neutral_matches_white_but_remains_distinct_from_black(tmp_path) -> None:
source = _image(tmp_path / "source.png", "#dddddd")
candidate = _image(tmp_path / "candidate.png", "white")
light_result = _classifier(
_response(
{
"source_color": "light gray",
"candidate_colors": {"candidate": "white"},
"visual_match": {"unique": False, "key": None},
}
)
).classify([source], [VisualCandidate("candidate", candidate)])
dark_result = _classifier(
_response(
{
"source_color": "black",
"candidate_colors": {"candidate": "light gray"},
"visual_match": {"unique": False, "key": None},
}
)
).classify([source], [VisualCandidate("candidate", candidate)])
assert light_result.source_color == "white"
assert light_result.selected_key == "candidate"
assert dark_result.source_color == "black"
assert dark_result.candidate_colors == {"candidate": "white"}
assert dark_result.selected_key is None
def test_prompt_and_schema_split_neutrals_by_brightness(tmp_path) -> None:
source = _image(tmp_path / "source.png", "#dddddd")
candidate = _image(tmp_path / "candidate.png", "white")
observed: dict[str, Any] = {}
def transport(_url, _headers, body, _timeout):
payload = json.loads(body)
observed["instruction"] = payload["messages"][1]["content"][0]["text"]
return _response(
{
"source_color": "white",
"candidate_colors": {"candidate": "white"},
"visual_match": {"unique": False, "key": None},
}
)
result = DirectVisualColorClassifier(
transport=transport,
config_resolver=lambda: DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="test-only-credential",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
),
).classify([source], [VisualCandidate("candidate", candidate)])
instruction = str(observed["instruction"])
schema = json.loads(instruction.rsplit("shape: ", maxsplit=1)[1])
allowed_value = "one of [" + ", ".join(COARSE_COLORS) + "]"
assert "gray" not in COARSE_COLORS
assert schema["source_color"] == allowed_value
assert schema["candidate_colors"] == {"candidate": allowed_value}
assert "silver-white, and light gray" in instruction
assert "black includes black, charcoal, and dark gray" in instruction
assert "Never output gray as a standalone color" in instruction
assert result.source_color == "white"
assert result.selected_key == "candidate"
def test_prompt_classifies_only_bag_body_and_ignores_clothing_and_overlays(
tmp_path,
) -> None:
source = _image(tmp_path / "source.png", "black")
candidate = _image(tmp_path / "candidate.png", "black")
observed: dict[str, str] = {}
def transport(_url, _headers, body, _timeout):
payload = json.loads(body)
observed["system"] = payload["messages"][0]["content"]
observed["instruction"] = payload["messages"][1]["content"][0]["text"]
return _response(
{
"source_color": "black",
"candidate_colors": {"candidate": "black"},
"visual_match": {"unique": False, "key": None},
}
)
result = DirectVisualColorClassifier(
transport=transport,
config_resolver=lambda: DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="test-only-credential",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
),
).classify([source], [VisualCandidate("candidate", candidate)])
instruction = observed["instruction"]
assert "Classify the BAG itself, not the whole image" in instruction
assert "only the dominant color of the bag's main body panels" in instruction
assert "product color is never the color of a person or their clothing" in instruction
assert "jackets, shirts, pants" in instruction
assert "text, prices, badges" in instruction
assert "black bag on a light-gray jacket is black" in instruction
assert "black or navy bag on green clothing" in instruction
assert "Never classify the overall image palette" in instruction
assert "return unknown" in instruction
assert "When that count is 0 or 1" in instruction
assert 'visual_match MUST be exactly {"unique":false,"key":null}' in instruction
assert "never put the sole same-color candidate" in instruction
assert "Only when the count is 2 or more" in instruction
assert "fail-closed product image classifier" in observed["system"]
assert result.source_color == "black"
assert result.selected_key == "candidate"
def test_prompt_repeats_exact_opaque_candidate_key_allowlist(tmp_path) -> None:
source = _image(tmp_path / "source.png", "black")
first = _image(tmp_path / "first.png", "black")
second = _image(tmp_path / "second.png", "white")
exact_keys = ("card-eb51a74da3a3", "card-eb51a74da3af")
observed: dict[str, str] = {}
def transport(_url, _headers, body, _timeout):
payload = json.loads(body)
observed["instruction"] = payload["messages"][1]["content"][0]["text"]
return _response(
{
"source_color": "black",
"candidate_colors": {
exact_keys[0]: "black",
exact_keys[1]: "white",
},
"visual_match": {"unique": False, "key": None},
}
)
result = DirectVisualColorClassifier(
transport=transport,
config_resolver=lambda: DirectLLMConfig(
endpoint="https://api.minimaxi.com/v1/chat/completions",
credential="test-only-credential",
title_model="MiniMax-M3",
vision_model="MiniMax-M3",
),
).classify(
[source],
[VisualCandidate(exact_keys[0], first), VisualCandidate(exact_keys[1], second)],
)
instruction = observed["instruction"]
allowlist = json.dumps(list(exact_keys), separators=(",", ":"))
assert instruction.count(allowlist) == 2
assert "Candidate keys are opaque, immutable identifiers" in instruction
assert "byte-for-byte exactly as provided" in instruction
assert "do not edit, invent, autocomplete, normalize, truncate, or repair" in instruction
assert "each provided candidate key exactly once and no other keys" in instruction
assert result.selected_key == exact_keys[0]
@@ -1,4 +1,6 @@
import unittest
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import MagicMock, patch
from collect_dy_market_rank import (
@@ -7,12 +9,14 @@ from collect_dy_market_rank import (
build_dy_category_table_xml,
build_dy_feishu_batch_xml,
build_dy_feishu_intro_xml,
install_saved_cookies,
is_excluded_product_title,
navigate_to_product_rank,
parse_market_product_rank_html,
parse_money_range_upper,
payment_metric_header_index,
require_payment_metric_header,
snapshot_rank_page,
)
from collect_sycm_market_rank import build_feishu_batch_xml
@@ -54,6 +58,24 @@ SAMPLE_HTML = """
class DyMarketRankTests(unittest.TestCase):
def test_saved_cookies_refresh_existing_context(self):
context = MagicMock()
cookies = [{"name": "session", "value": "secret"}]
self.assertEqual(install_saved_cookies(context, cookies), 1)
context.add_cookies.assert_called_once_with(cookies)
def test_rank_timeout_snapshot_persists_dom(self):
page = MagicMock()
page.content.return_value = "<html><body>新版榜单</body></html>"
with TemporaryDirectory() as temp_dir:
with patch("collect_dy_market_rank.DEBUG_DIR", Path(temp_dir)):
path = snapshot_rank_page(page, "header timeout")
self.assertTrue(path.is_file())
self.assertIn("新版榜单", path.read_text(encoding="utf-8"))
def test_title_blacklist_filters_children_primary_and_middle_school(self):
self.assertTrue(is_excluded_product_title("儿童轻便双肩包"))
self.assertTrue(is_excluded_product_title("小学生护脊书包"))
@@ -0,0 +1,191 @@
from __future__ import annotations
from datetime import date
from pathlib import Path
import pytest
from gyxx_flow.modules.product_commerce import backfill_erp_all_shop_daily as backfill
from gyxx_flow.modules.product_commerce.db import (
upsert_erp_all_shop_style_daily_metrics,
)
def test_default_range_is_yesterday_only() -> None:
assert backfill.DEFAULT_FROM_DATE == backfill.DEFAULT_TO_DATE
assert backfill.DEFAULT_FROM_DATE == date.today() - backfill.timedelta(days=1)
def test_historical_range_is_inclusive_and_covers_103_days() -> None:
dates = backfill.inclusive_dates(date(2026, 5, 1), date(2026, 8, 11))
assert dates[0] == date(2026, 5, 1)
assert dates[-1] == date(2026, 8, 11)
assert len(dates) == 103
def test_invalid_or_oversized_range_is_rejected() -> None:
with pytest.raises(ValueError, match="--to"):
backfill.inclusive_dates(date(2026, 8, 12), date(2026, 8, 11))
with pytest.raises(ValueError, match="366"):
backfill.inclusive_dates(date(2025, 1, 1), date(2026, 8, 11))
def test_style_plan_rejects_one_erp_code_owned_by_two_styles() -> None:
with pytest.raises(ValueError, match="belongs to both"):
backfill.build_style_plans(
{
"styles": {
"款式甲": {"erp_codes": ["10416"]},
"款式乙": {"erp_codes": ["10416"]},
}
}
)
def test_all_shop_filter_selects_every_shop_and_batches_all_codes() -> None:
class Frame:
def __init__(self) -> None:
self.script = ""
self.args = {}
def evaluate(self, script, args):
self.script = script
self.args = args
return {
"selected_shop_count": 38,
"selection_mode": "all",
}
frame = Frame()
result = backfill.set_all_shop_daily_filters(
frame,
report_date=date(2026, 5, 1),
erp_codes=("10416", "10455"),
)
assert frame.args == {
"reportDate": "2026-05-01",
"erpCodes": ["10416", "10455"],
}
assert 'input[name="shop_id"]' in frame.script
assert "=== '全选'" in frame.script
assert "erpCodes.join(',')" in frame.script
assert "shopLabels" not in frame.script
assert result == {"selected_shop_count": 38, "selection_mode": "all"}
def test_style_records_sum_product_rows_and_keep_zero_styles() -> None:
plans = (
backfill.StylePlan("款式甲", ("10416", "10455")),
backfill.StylePlan("款式乙", ("10504",)),
)
rows = [
{"erp_style_code": "10416", "sales": 12, "return_qty": 2},
{"erp_style_code": "10416", "sales": 8, "return_qty": 1},
{"erp_style_code": "10455", "sales": 5, "return_qty": 3},
]
records = backfill.aggregate_style_records(
date(2026, 5, 1),
plans,
rows,
selected_shop_count=38,
source="raw/product_commerce/evidence.json",
)
assert records[0] == {
"metric_date": date(2026, 5, 1),
"style_name": "款式甲",
"sales": 25,
"return_qty": 6,
"erp_style_codes": ["10416", "10455"],
"selected_shop_count": 38,
"matched_product_rows": 3,
"source": "raw/product_commerce/evidence.json",
}
assert records[1]["sales"] == 0
assert records[1]["return_qty"] == 0
assert records[1]["matched_product_rows"] == 0
def test_range_collector_queries_once_per_date_not_once_per_style(monkeypatch) -> None:
filter_frame = object()
monkeypatch.setattr(
backfill.erp, "find_report_frames", lambda _page: (filter_frame, object())
)
calls: list[date] = []
def collect_one(_page, _frame, _plans, report_date, **_kwargs):
calls.append(report_date)
return (
{"selected_shop_count": 38},
{"rows": [], "frame_url": f"https://example/{report_date}", "signature": str(report_date)},
)
monkeypatch.setattr(backfill, "collect_one_date", collect_one)
persisted: list[date] = []
backfill.collect_range_in_page(
object(),
(
backfill.StylePlan("款式甲", ("10416", "10455")),
backfill.StylePlan("款式乙", ("10504",)),
),
(date(2026, 5, 1), date(2026, 5, 2)),
persist=lambda report_date, _selection, _report: persisted.append(report_date),
)
assert calls == [date(2026, 5, 1), date(2026, 5, 2)]
assert persisted == calls
def test_database_upsert_uses_date_and_style_conflict_key(monkeypatch) -> None:
captured = {}
class Cursor:
def __enter__(self):
return self
def __exit__(self, *_args):
return None
class Connection:
def cursor(self):
return Cursor()
def fake_execute_values(cursor, sql, rows):
captured.update(cursor=cursor, sql=sql, rows=rows)
monkeypatch.setattr(
"gyxx_flow.modules.product_commerce.db.execute_values", fake_execute_values
)
written = upsert_erp_all_shop_style_daily_metrics(
Connection(),
[
{
"metric_date": date(2026, 5, 1),
"style_name": "款式甲",
"sales": 25,
"return_qty": 6,
"erp_style_codes": ["10455", "10416"],
"selected_shop_count": 38,
"matched_product_rows": 3,
"source": "raw/evidence.json",
}
],
)
assert written == 1
assert "ON CONFLICT (metric_date, style_name)" in captured["sql"]
assert captured["rows"][0][2:4] == (25, 6)
assert captured["rows"][0][4] == ["10416", "10455"]
def test_schema_keeps_all_shop_metrics_separate_from_platform_metrics() -> None:
schema = (
Path(backfill.__file__).parent / "db" / "schema.sql"
).read_text(encoding="utf-8")
assert "CREATE TABLE IF NOT EXISTS erp_all_shop_style_daily_metrics" in schema
assert "PRIMARY KEY (metric_date, style_name)" in schema
assert "return_qty" in schema
@@ -5,22 +5,97 @@ from unittest.mock import Mock, patch
from upload_video_to_guanghe import (
GuangHeUploader,
TmallTitleGenerationError,
_match_tm_item_ids_from_product_titles,
_select_video_product_names,
_tm_product_title_search_pattern,
download_videos,
)
def test_tm_product_title_search_pattern_uses_model_token() -> None:
assert _tm_product_title_search_pattern("拾影单肩包") == "%拾影%"
assert _tm_product_title_search_pattern("星云mini") == "%星云mini%"
def test_multi_style_record_prefers_style_named_by_attachment() -> None:
assert _select_video_product_names(
["阿波罗2", "宙斯3"], "宙斯3 7.30.mp4"
) == ["宙斯3"]
assert _select_video_product_names(
["阿波罗2", "宙斯3"], "7月30日.mp4"
) == ["阿波罗2", "宙斯3"]
def test_attachment_style_matching_handles_combos_and_overlapping_aliases() -> None:
assert _select_video_product_names(
["阿波罗2", "宙斯3", "星云2", "逐星"],
"宙斯3+阿波罗2+星云2+逐星.m4v",
) == ["宙斯3", "阿波罗2", "星云2", "逐星"]
assert _select_video_product_names(
["极星双肩", "极星双肩2"], "极星双肩2 新新新.mp4"
) == ["极星双肩2"]
class GuangHeMetadataTests(unittest.TestCase):
def test_formal_title_rejection_uses_non_browser_failure_type(self):
GuangHeUploader.configure_catalog_product_titles(
{"极星双肩2": ["极星双肩2双肩包轻便电脑包"]}
)
uploader = GuangHeUploader("", "", require_llm_titles=True)
try:
with patch(
"upload_video_to_guanghe.JdVideoTitleGenerator.generate",
side_effect=RuntimeError("候选不合格"),
):
with self.assertRaisesRegex(
TmallTitleGenerationError, "候选不合格"
):
uploader._generate_title(["极星双肩2"], video_key="token")
finally:
GuangHeUploader.configure_catalog_product_titles({})
def test_formal_collection_title_forbids_specific_style_names(self):
GuangHeUploader.configure_catalog_product_titles(
{
"宙斯3": ["宙斯3双肩包男士大容量"],
"阿波罗2": ["阿波罗2电脑包手提轻便"],
}
)
uploader = GuangHeUploader("", "", require_llm_titles=True)
try:
with patch(
"upload_video_to_guanghe.JdVideoTitleGenerator"
) as generator_class:
generator = generator_class.return_value
generator.generate.return_value = (
"多款通勤包袋,大容量轻便按场景切换"
)
title = uploader._generate_title(
["宙斯3", "阿波罗2"], video_key="combo-token"
)
self.assertEqual(title, "多款通勤包袋,大容量轻便按场景切换")
self.assertTrue(generator_class.call_args.kwargs["generic_collection"])
self.assertFalse(generator_class.call_args.kwargs["require_product_name"])
self.assertEqual(generator_class.call_args.kwargs["llm_timeout"], 300)
generator.generate.assert_called_once()
self.assertEqual(
generator.generate.call_args.args[0], ["宙斯3", "阿波罗2"]
)
finally:
GuangHeUploader.configure_catalog_product_titles({})
def test_downloads_with_same_attachment_name_are_isolated_by_record(self):
attachment = {"file_token": "file-token", "name": "same-name.mp4"}
with TemporaryDirectory() as tmp_dir, patch(
"upload_video_to_guanghe.DOWNLOAD_DIR", Path(tmp_dir)
), patch("upload_video_to_guanghe.run_lark") as run_lark_mock:
def fake_download(args):
output_dir = Path(args[args.index("--output") + 1])
def fake_download(args, *, cwd=None, timeout=None):
self.assertEqual(timeout, 600)
output_dir = Path(cwd)
output_dir.mkdir(parents=True, exist_ok=True)
(output_dir / attachment["name"]).write_bytes(b"video")
(output_dir / args[args.index("--output") + 1]).write_bytes(b"video")
return 0, "", ""
run_lark_mock.side_effect = fake_download
@@ -28,8 +103,9 @@ class GuangHeMetadataTests(unittest.TestCase):
first = download_videos("record-one", [attachment])
second = download_videos("record-two", [attachment])
self.assertEqual(first[0].parent.name, "record-one")
self.assertEqual(second[0].parent.name, "record-two")
self.assertEqual(first[0].parent.parent.name, "record-one")
self.assertEqual(second[0].parent.parent.name, "record-two")
self.assertEqual(first[0].parent.name, "file-token")
self.assertNotEqual(first[0], second[0])
def test_unresolved_style_uses_exact_database_product_title_match(self):
@@ -81,14 +157,20 @@ class GuangHeMetadataTests(unittest.TestCase):
def test_topic_result_card_requires_topic_and_activity_statistics(self):
self.assertTrue(
GuangHeUploader._topic_result_card_text_matches(
"# 在淘宝种草一夏 小二推荐 作品数469.5万 参与数16.2万",
"在淘宝种草一夏",
"# 我的夏日焕新清单 小二推荐 作品数469.5万 参与数16.2万",
"我的夏日焕新清单",
)
)
self.assertTrue(
GuangHeUploader._topic_result_card_text_matches(
"# 秋上新开拍了 时尚 作品数198.3万 参与数11.3万",
"秋上新",
)
)
self.assertFalse(
GuangHeUploader._topic_result_card_text_matches(
"在淘宝种草一夏",
"在淘宝种草一夏",
"我的夏日焕新清单",
"我的夏日焕新清单",
)
)
self.assertTrue(GuangHeUploader._topic_card_box_is_clickable(620, 120))
@@ -160,13 +242,12 @@ class GuangHeMetadataTests(unittest.TestCase):
self.assertGreaterEqual(len(title), 26)
self.assertLessEqual(len(title), 30)
def test_same_style_multiple_videos_get_distinct_titles_when_hermes_repeats(self):
def test_same_style_multiple_videos_get_distinct_titles_when_model_repeats(self):
uploader = object.__new__(GuangHeUploader)
repeated = "光影行星宙斯双肩包科学分区收纳轻量通勤出行"
with (
patch.object(uploader, "_search_product_features", return_value=""),
patch("upload_video_to_guanghe.HAS_HERMES", True),
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
patch("upload_video_to_guanghe.call_direct_llm", return_value=(repeated, {})),
):
first = uploader._generate_title(["宙斯"], video_index=1, video_total=2)
second = uploader._generate_title(["宙斯"], video_index=2, video_total=2)
@@ -175,13 +256,12 @@ class GuangHeMetadataTests(unittest.TestCase):
self.assertTrue(first.startswith("宙斯双肩包,"))
self.assertTrue(second.startswith("宙斯双肩包,"))
def test_same_style_repeated_hermes_copy_gets_low_similarity_bodies(self):
def test_same_style_repeated_model_copy_gets_low_similarity_bodies(self):
uploader = object.__new__(GuangHeUploader)
repeated = "光影行星宙斯双肩包,科学分区收纳,轻量通勤出行!"
with (
patch.object(uploader, "_search_product_features", return_value=""),
patch("upload_video_to_guanghe.HAS_HERMES", True),
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
patch("upload_video_to_guanghe.call_direct_llm", return_value=(repeated, {})),
):
titles = [
uploader._generate_title(["宙斯"], video_index=index, video_total=4)
@@ -202,8 +282,7 @@ class GuangHeMetadataTests(unittest.TestCase):
repeated = "科学分区收纳,轻装通勤出行"
with (
patch.object(uploader, "_search_product_features", return_value=""),
patch("upload_video_to_guanghe.HAS_HERMES", True),
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
patch("upload_video_to_guanghe.call_direct_llm", return_value=(repeated, {})),
):
first = uploader._generate_title(["宙斯"], video_index=1)
second = uploader._generate_title(["星云2"], video_index=1)
@@ -339,9 +418,8 @@ class GuangHeMetadataTests(unittest.TestCase):
uploader = object.__new__(GuangHeUploader)
with (
patch.object(uploader, "_search_product_features", return_value=""),
patch("upload_video_to_guanghe.HAS_HERMES", True),
patch(
"upload_video_to_guanghe._call_hermes",
"upload_video_to_guanghe.call_direct_llm",
return_value=("光影行星宙斯双肩包大容量笔记本电脑轻量通勤", {}),
),
):
@@ -350,7 +428,7 @@ class GuangHeMetadataTests(unittest.TestCase):
self.assertTrue(title.startswith("宙斯双肩包,"))
self.assertNotIn("包袋双肩包", title)
def test_generated_poseidon_title_cannot_be_changed_to_backpack_by_hermes(self):
def test_generated_poseidon_title_cannot_be_changed_to_backpack_by_model(self):
uploader = object.__new__(GuangHeUploader)
with (
patch.object(
@@ -358,9 +436,8 @@ class GuangHeMetadataTests(unittest.TestCase):
"_search_product_features",
return_value="双肩包、电脑包、托特包、大容量",
),
patch("upload_video_to_guanghe.HAS_HERMES", True),
patch(
"upload_video_to_guanghe._call_hermes",
"upload_video_to_guanghe.call_direct_llm",
return_value=(
"光影行星波塞冬edge双肩包暴雨通勤防泼水电脑仓从容出行",
{},
@@ -439,8 +516,8 @@ class GuangHeMetadataTests(unittest.TestCase):
with self.subTest(product_names=product_names):
uploader = object.__new__(GuangHeUploader)
calls = []
uploader._pick_content_labels = Mock(
side_effect=lambda labels: calls.append(("labels", list(labels)))
uploader._pick_content_label = Mock(
side_effect=lambda label: calls.append(("label", label))
)
uploader._pick_topic = Mock(
side_effect=lambda topic: calls.append(("topic", topic))
@@ -452,8 +529,8 @@ class GuangHeMetadataTests(unittest.TestCase):
self.assertEqual(
calls,
[
("labels", expected_labels),
("topic", "在淘宝种草一夏"),
*[("label", label) for label in expected_labels],
("topic", "我的夏日焕新清单"),
],
)
@@ -461,8 +538,55 @@ class GuangHeMetadataTests(unittest.TestCase):
self.assertEqual(GuangHeUploader.FIXED_CONTENT_LABELS, ("商品展示",))
self.assertNotIn("户外", GuangHeUploader.FIXED_CONTENT_LABELS)
def test_missing_user_selected_topic_stops_metadata_preparation(self):
uploader = object.__new__(GuangHeUploader)
uploader._pick_content_label = Mock()
uploader._pick_topic = Mock(
side_effect=RuntimeError("没有找到固定话题: #我的夏日焕新清单")
)
uploader._snapshot = Mock()
with self.assertRaisesRegex(RuntimeError, "没有找到固定话题"):
uploader._prepare_publish_metadata(["极星双肩"])
self.assertEqual(
uploader._pick_content_label.call_args_list,
[unittest.mock.call("双肩包"), unittest.mock.call("商品展示")],
)
def test_optional_content_label_failures_do_not_skip_required_topic(self):
uploader = object.__new__(GuangHeUploader)
uploader.page = Mock()
uploader._pick_content_label = Mock(
side_effect=[
RuntimeError("内容标签没有精确匹配项: #双肩包"),
RuntimeError("内容标签没有精确匹配项: #商品展示"),
]
)
uploader._pick_topic = Mock()
uploader._snapshot = Mock()
uploader._prepare_publish_metadata(["极星双肩"])
self.assertEqual(
uploader._pick_content_label.call_args_list,
[unittest.mock.call("双肩包"), unittest.mock.call("商品展示")],
)
uploader._pick_topic.assert_called_once_with("我的夏日焕新清单")
def test_metadata_uses_operator_topic_keyword(self):
uploader = object.__new__(GuangHeUploader)
calls = []
uploader._pick_content_label = Mock()
uploader._pick_topic = Mock(side_effect=lambda topic: calls.append(topic))
uploader._snapshot = Mock()
uploader._prepare_publish_metadata(["极星双肩"], topic_keyword="秋上新")
self.assertEqual(calls, ["秋上新"])
def test_fixed_topic_is_not_a_recommended_first_result(self):
self.assertEqual(GuangHeUploader.FIXED_TOPIC, "在淘宝种草一夏")
self.assertEqual(GuangHeUploader.FIXED_TOPIC, "我的夏日焕新清单")
if __name__ == "__main__":
@@ -0,0 +1,175 @@
from __future__ import annotations
from datetime import date
from decimal import Decimal
import db as product_db
import import_douyin_qianchuan_ads as importer
from openpyxl import Workbook
HEADERS = [
"商品ID",
"商品名称",
"日期",
"整体展示次数",
"整体点击次数",
"整体点击率",
"整体转化率",
"整体消耗(元)",
"整体成交金额(元)",
"整体支付ROI",
"整体成交订单成本(元)",
"用户实际支付金额(元)",
"电商平台补贴金额(元)",
"净成交ROI",
"净成交金额(元)",
"净成交订单成本(元)",
"净成交金额结算率",
"1小时内退款率",
]
def _workbook(path) -> None:
workbook = Workbook()
sheet = workbook.active
sheet.append(HEADERS)
sheet.append(
[
"3725242433383039375",
"测试商品",
"全部",
"12,345",
"678",
"5.49%",
"1.00%",
"100.00",
"900.00",
"9.00",
"25.00",
"850.00",
"50.00",
"8.00",
"800.00",
"30.00",
"88.89%",
"11.11%",
]
)
sheet.append(
[
"3725242433383039375",
"测试商品",
"2026-08-23",
"12,345",
"678",
"5.49%",
"1.00%",
"100.00",
"900.00",
"9.00",
"25.00",
"850.00",
"50.00",
"8.00",
"800.00",
"30.00",
"88.89%",
"11.11%",
]
)
workbook.save(path)
def test_parse_workbook_keeps_long_product_id_and_skips_total(tmp_path) -> None:
path = tmp_path / "qianchuan.xlsx"
_workbook(path)
records = importer.parse_workbook(path)
assert len(records) == 1
assert records[0]["stat_date"] == date(2026, 8, 23)
assert records[0]["product_id"] == "3725242433383039375"
assert records[0]["overall_impressions"] == 12345
assert records[0]["spend"] == Decimal("100.00")
assert records[0]["refund_rate_1h"] == Decimal("11.11")
def test_upsert_uses_advertiser_date_product_business_key(monkeypatch) -> None:
executed = {}
class Cursor:
def execute(self, sql, params=None):
executed["lookup_sql"] = sql
executed["lookup_params"] = params
def fetchall(self):
return [("测试款式", ["3725242433383039375"])]
def __enter__(self):
return self
def __exit__(self, exc_type, exc, traceback):
return None
class Connection:
def cursor(self):
return Cursor()
def fake_execute_values(cursor, sql, rows):
executed.update(sql=sql, rows=rows)
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
record = {
"stat_date": date(2026, 8, 23),
"advertiser_id": importer.ADVERTISER_ID,
"product_id": "3725242433383039375",
"product_name": "测试商品",
"spend": Decimal("100.00"),
}
assert product_db.upsert_douyin_product_ad_daily_metrics(
Connection(), [record, record]
) == 1
assert "ON CONFLICT (stat_date, advertiser_id, product_id)" in executed["sql"]
assert len(executed["rows"]) == 1
assert "style_name" in executed["sql"]
assert executed["rows"][0][4] == "测试款式"
assert executed["rows"][0][5] == "product_id"
def test_upsert_falls_back_to_exact_product_name_style_match(monkeypatch) -> None:
executed = {}
class Cursor:
def execute(self, sql, params=None):
executed["lookup_sql"] = sql
executed["lookup_params"] = params
def fetchall(self):
return [("测试款式", [])]
def __enter__(self):
return self
def __exit__(self, exc_type, exc, traceback):
return None
class Connection:
def cursor(self):
return Cursor()
def fake_execute_values(cursor, sql, rows):
executed.update(sql=sql, rows=rows)
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
record = {
"stat_date": date(2026, 8, 23),
"advertiser_id": importer.ADVERTISER_ID,
"product_id": "unmapped-product-id",
"product_name": "GYXX/测试款式3双肩包2026新款",
"spend": Decimal("100.00"),
}
assert product_db.upsert_douyin_product_ad_daily_metrics(Connection(), [record]) == 1
assert executed["rows"][0][4] == "测试款式"
assert executed["rows"][0][5] == "product_name"
@@ -0,0 +1,215 @@
from __future__ import annotations
from datetime import date
from pathlib import Path
import db as product_db
import import_jd_ad_costs as importer
def test_jd_exports_only_import_current_block_and_ignore_comparison_rows(
tmp_path: Path,
) -> None:
full = tmp_path / "full.csv"
full.write_text(
"查询条件,广告主PIN: 光影行星旗舰店霖; 时间: 2026-08-23\n"
"日期,'当前时间',SPU ID,商品计划名称,SpuImgUrl,花费,全站投产比\n"
"20260823~20260823,当前时间,spu-1,盖亚款式,img,12.30,4.50\n"
"汇总,,,汇总,,12.30,4.50\n"
"20260822~20260822,对比时间,spu-old,旧款,img,99.00,9.00\n",
encoding="utf-8-sig",
)
sku = tmp_path / "sku.csv"
sku.write_text(
"查询条件,广告主PIN: 光影行星旗舰店霖; 时间: 2026-08-23\n"
"SKU ID,SKU 名称,点击时间,'当前时间',花费,展现数,点击数,投产比\n"
"sku-1,光影行星 盖亚款式 黑色,20260823~20260823,当前时间,3.20,100,5,2.50\n"
"sku-old,光影行星 盖亚款式 黑色,20260822~20260822,对比时间,99.00,100,5,9.00\n",
encoding="utf-8-sig",
)
full_records = importer.parse_full_csv(full, date(2026, 8, 23))
sku_records = importer.parse_non_full_csv(sku, date(2026, 8, 23))
assert [record["spu_id"] for record in full_records] == ["spu-1"]
assert [record["sku_id"] for record in sku_records] == ["sku-1"]
assert full_records[0]["metrics"]["source"]["current_marker"] == "当前时间"
assert full_records[0]["spend"] == importer.Decimal("12.30")
assert sku_records[0]["metrics"]["clicks"] == 5
def test_sku_name_matches_dim_style_spu_and_keeps_audit_fields(tmp_path: Path) -> None:
report = tmp_path / "sku.csv"
report.write_text(
"查询条件,广告主PIN: 光影行星旗舰店霖\n"
"SKU ID,SKU 名称,点击时间,'当前时间',花费,投产比\n"
"sku-1,光影行星(GYXX)盖亚C1斜挎包 曜夜黑,20260823~20260823,当前时间,3.20,2.50\n",
encoding="utf-8-sig",
)
sku_records = importer.parse_non_full_csv(report, date(2026, 8, 23))
matched = importer.apply_sku_matches(
sku_records,
style_catalog=[{"style_name": "盖亚斜挎", "jd_spus": ["spu-1"]}],
full_records=[{"spu_id": "spu-1", "spu_name": "盖亚C1斜挎包"}],
)
assert matched[0]["match_status"] == "matched"
assert matched[0]["spu_id"] == "spu-1"
assert matched[0]["style_name"] == "盖亚斜挎"
assert matched[0]["metrics"]["match"]["reason"]
def test_full_spu_matches_dim_style_exactly() -> None:
full_records = [
{
"spu_id": "spu-1",
"style_name": None,
"match_status": "direct",
"match_reason": "full-site SPU report",
"metrics": {},
}
]
matched = importer.apply_full_style_matches(
full_records,
style_catalog=[
{"style_name": "盖亚斜挎", "jd_spus": ["spu-1"]},
],
)
assert matched[0]["style_name"] == "盖亚斜挎"
assert matched[0]["match_status"] == "direct"
assert matched[0]["metrics"]["match"]["style_name"] == "盖亚斜挎"
def test_full_spu_name_matches_base_style_when_spu_mapping_is_missing() -> None:
full_records = [
{
"spu_id": "spu-2",
"spu_name": "光影行星(GYXX)盖亚斜挎包3单反相机包",
"style_name": None,
"match_status": "direct",
"match_reason": "full-site SPU report",
"metrics": {},
}
]
matched = importer.apply_full_style_matches(
full_records,
style_catalog=[
{"style_name": "盖亚斜挎", "jd_spus": []},
{"style_name": "盖亚双肩3", "jd_spus": []},
],
spu_catalog=[
{"spu_id": "spu-2", "product_no": "盖亚斜挎3"},
],
)
assert matched[0]["style_name"] == "盖亚斜挎"
assert matched[0]["metrics"]["match"]["reason"] == (
"full-site product_no matched style name"
)
def test_sku_name_uses_latest_jd_spu_catalog_when_full_report_lacks_spu() -> None:
sku_records = [
{
"sku_name": "光影行星(GYXX)阿波罗2斜挎包男士通勤便携单肩背包 曜夜黑",
"metrics": {},
}
]
matched = importer.apply_sku_matches(
sku_records,
style_catalog=[
{"style_name": "阿波罗2", "jd_spus": ["spu-1", "spu-2"]}
],
full_records=[],
spu_catalog=[
{
"spu_id": "spu-1",
"spu_name": "光影行星(GYXX)阿波罗2斜挎包男士通勤便携单肩背包",
},
{"spu_id": "spu-2", "spu_name": "另一款商品"},
],
)
assert matched[0]["match_status"] == "matched"
assert matched[0]["spu_id"] == "spu-1"
class _Cursor:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
class _Connection:
def cursor(self):
return _Cursor()
def test_jd_upsert_deduplicates_same_logical_row(monkeypatch) -> None:
captured: dict[str, object] = {}
def fake_execute_values(_cursor, sql, rows, *, template=None):
captured["sql"] = sql
captured["rows"] = rows
captured["template"] = template
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
record = {
"stat_date": date(2026, 8, 23),
"advertiser_pin": "光影行星旗舰店霖",
"shop_account": "光影行星研发",
"report_type": "full",
"row_key": "spu:spu-1",
"spu_id": "spu-1",
"spu_name": "盖亚",
"match_status": "direct",
"spend": importer.Decimal("12.30"),
"reported_roi": importer.Decimal("4.50"),
"metrics": {"spend": "12.30"},
"source_file": "full.csv",
"source_row_number": 3,
}
assert product_db.upsert_jd_product_ad_daily_metrics(
_Connection(), [record, {**record, "spend": importer.Decimal("13.30")}]
) == 1
assert len(captured["rows"]) == 1
assert captured["rows"][0][11] == importer.Decimal("13.30")
assert "jd_product_ad_daily_metrics" in str(captured["sql"])
def test_jd_dry_run_accepts_explicit_desktop_style_files(tmp_path: Path, capsys) -> None:
full = tmp_path / "full.csv"
full.write_text(
"条件,广告主PIN: 光影行星旗舰店霖\n"
"日期,'当前时间',SPU ID,商品计划名称,花费\n"
"20260823~20260823,当前时间,spu-1,盖亚,1.00\n",
encoding="utf-8-sig",
)
sku = tmp_path / "sku.csv"
sku.write_text(
"条件,广告主PIN: 光影行星旗舰店霖\n"
"SKU ID,SKU 名称,点击时间,'当前时间',花费\n"
"sku-1,盖亚黑色,20260823~20260823,当前时间,1.00\n",
encoding="utf-8-sig",
)
assert importer.main(
[
"--date",
"2026-08-23",
"--full-file",
str(full),
"--non-full-file",
str(sku),
]
) == 0
output = capsys.readouterr().out
assert "全站 SPU 1" in output
assert "非全站 SKU 1" in output
assert "DRY-RUN" in output
@@ -0,0 +1,187 @@
from __future__ import annotations
import json
from pathlib import Path
import db as product_db
import import_tmall_product_ads as importer
class _Cursor:
def execute(self, sql, params=None):
self.sql = sql
self.params = params
def fetchall(self):
return [("极星", ["1001"])]
def __enter__(self):
return self
def __exit__(self, *_args):
return False
class _Connection:
def cursor(self):
return _Cursor()
def test_parse_tmall_product_ad_csv_merges_product_rows(tmp_path: Path) -> None:
report = tmp_path / "商品报表.csv"
report.write_text(
"日期,主体ID,主体类型,主体名称,花费,直接成交金额,间接成交金额,总成交金额,投入产出比,展现量\n"
"2026-08-17,1001,商品,极星单肩包,90.25,730.02,0,730.02,8.09,1000\n"
"2026-08-17,1001,商品,极星单肩包,9.75,0,70,70,7.18,200\n"
"2026-08-17,plan-1,计划,测试计划,100,1000,0,1000,10,500\n",
encoding="utf-8-sig",
)
records = importer.parse_tmall_product_ad_csv(report)
assert len(records) == 1
metrics = records[0]["ad_metrics"]
assert records[0]["product_id"] == "1001"
assert metrics["spend"] == "100.00"
assert metrics["attributed_deal_amount"] == "800.02"
assert metrics["roi"] == "8.00"
assert metrics["impressions"] == 1200
assert records[0]["duplicate_row_count"] == 2
def test_parse_gb18030_export(tmp_path: Path) -> None:
report = tmp_path / "商品报表.csv"
report.write_bytes(
(
"日期,主体ID,主体类型,主体名称,花费,总成交金额,投入产出比\n"
"2026-08-17,1001,商品,极星单肩包,90.25,730.02,8.09\n"
).encode("gb18030")
)
records = importer.parse_tmall_product_ad_csv(report)
assert records[0]["product_name"] == "极星单肩包"
assert records[0]["ad_metrics"]["roi"] == "8.09"
def test_parse_export_null_markers_as_missing(tmp_path: Path) -> None:
report = tmp_path / "商品报表.csv"
report.write_text(
"日期,主体ID,主体类型,主体名称,花费,总成交金额,平均点击花费,投入产出比\n"
"2026-08-17,1001,商品,极星单肩包,90.25,730.02,\\N,\\N\n",
encoding="utf-8",
)
records = importer.parse_tmall_product_ad_csv(report)
assert records[0]["ad_metrics"] == {
"spend": "90.25",
"attributed_deal_amount": "730.02",
"roi": "8.09",
}
def test_find_report_files_uses_current_manifest_csv(monkeypatch, tmp_path: Path) -> None:
target_date = importer.date(2026, 8, 17)
date_root = tmp_path / target_date.isoformat()
extracted = date_root / "extracted"
extracted.mkdir(parents=True)
old_csv = extracted / "商品报表_旧.csv"
current_csv = extracted / "商品报表_新.csv"
old_csv.write_text("old", encoding="utf-8")
current_csv.write_text("current", encoding="utf-8")
(date_root / "manifest.json").write_text(
json.dumps({"csv_files": ["extracted\\商品报表_新.csv"]}),
encoding="utf-8",
)
monkeypatch.setattr(importer, "REPORT_ROOT", tmp_path)
assert importer.find_report_files(target_date) == [current_csv.resolve()]
def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
captured: dict[str, object] = {}
def fake_execute_values(_cursor, sql, rows, *, template=None):
captured["sql"] = sql
captured["rows"] = rows
captured["template"] = template
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
written = product_db.upsert_tmall_product_ad_raw_metrics(
_Connection(),
[
{
"stat_date": importer.date(2026, 8, 17),
"product_id": "1001",
"product_name": "极星单肩包",
"ad_metrics": {
"spend": "90.25",
"attributed_deal_amount": "730.02",
"roi": "8.09",
},
"source_file": "商品报表.csv",
}
],
)
assert written == 1
sql = str(captured["sql"])
assert "INSERT INTO product_daily_metrics" in sql
assert "jsonb_set" in sql
assert "tm_product_ad_daily_metrics" not in sql
assert captured["template"] == "('tm', %s, %s, %s, %s::jsonb, %s, NOW())"
row = captured["rows"][0]
payload = json.loads(row[3])
assert payload["tm_ad_metrics"]["spend"] == "90.25"
assert payload["tm_ad_metrics"]["style_name"] == "极星"
assert payload["tm_ad_metrics"]["style_match_method"] == "product_id"
assert row[1] == "1001"
def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> None:
captured: dict[str, object] = {}
class _NameFallbackCursor(_Cursor):
def fetchall(self):
return [("极星", [])]
class _NameFallbackConnection:
def cursor(self):
return _NameFallbackCursor()
def fake_execute_values(_cursor, _sql, rows, *, template=None):
captured["rows"] = rows
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
written = product_db.upsert_tmall_product_ad_raw_metrics(
_NameFallbackConnection(),
[
{
"stat_date": importer.date(2026, 8, 17),
"product_id": "unmapped-id",
"product_name": "GYXX/极星单肩包2026新款",
"ad_metrics": {"spend": "12.00"},
"source_file": "商品报表.csv",
}
],
)
assert written == 1
row = captured["rows"][0]
payload = json.loads(row[3])
assert payload["tm_ad_metrics"]["style_name"] == "极星"
assert payload["tm_ad_metrics"]["style_match_method"] == "product_name"
def test_import_date_filter_and_dry_run(tmp_path: Path, capsys) -> None:
report = tmp_path / "商品报表.csv"
report.write_text(
"日期,主体ID,主体类型,主体名称,花费,总成交金额\n"
"2026-08-16,1001,商品,旧日期,1,2\n"
"2026-08-17,1002,商品,目标日期,3,9\n",
encoding="utf-8",
)
assert importer.main(["--file", str(report), "--date", "2026-08-17"]) == 0
assert "DRY-RUN" in capsys.readouterr().out
@@ -7,7 +7,9 @@ from collect_jd_market_rank import (
build_jd_feishu_batch_xml,
build_jd_feishu_category_xml,
build_jd_feishu_intro_xml,
category_label_is_selected,
category_selection_matches,
category_ui_label_candidates,
exact_category_label_pattern,
group_products_by_category,
high_resolution_jd_image_url,
@@ -15,6 +17,7 @@ from collect_jd_market_rank import (
parse_jd_market_rank_html,
parse_money_range_upper,
resize_feishu_table_xml,
select_category_path,
select_category_paths,
)
@@ -80,14 +83,13 @@ class JdMarketRankTests(unittest.TestCase):
set(specs),
{"旅行箱", "运动包", "双肩包", "斜挎包", "胸包", "手机包", "电脑包"},
)
self.assertEqual(specs["旅行箱"]["paths"], (("功能箱包", "行李箱"),))
self.assertEqual(specs["运动包"]["paths"], (("功能箱包", "休闲运动包"),))
self.assertEqual(
specs["双肩"]["paths"],
(("", "双肩包"), ("男包", "男士双肩")),
)
self.assertEqual(specs["手机"]["paths"], (("", "男士手机"),))
self.assertEqual(specs["电脑包"]["paths"], (("功能箱包", "电脑包"),))
self.assertEqual(specs["旅行箱"]["path"], ("功能箱包", "行李箱"))
self.assertEqual(specs["运动包"]["path"], ("功能箱包", "休闲运动包"))
self.assertEqual(specs["双肩包"]["path"], ("男包", "男士双肩包"))
self.assertEqual(specs["斜挎"]["path"], ("男包", "男士单肩/斜挎包"))
self.assertEqual(specs[""]["path"], ("男包", "男士腰包/胸"))
self.assertEqual(specs["手机包"]["path"], ("男包", "男士手机包"))
self.assertEqual(specs["电脑"]["path"], ("功能箱", "电脑"))
def test_category_label_matching_is_exact_not_contains(self):
pattern = exact_category_label_pattern("双肩包")
@@ -96,6 +98,116 @@ class JdMarketRankTests(unittest.TestCase):
self.assertIsNone(pattern.fullmatch("男士双肩包"))
self.assertIsNone(pattern.fullmatch("双肩包配件"))
def test_live_ui_alias_keeps_the_source_business_category(self):
self.assertEqual(
category_ui_label_candidates("男士双肩包"),
("男士双肩包", "双肩包"),
)
self.assertEqual(
category_ui_label_candidates("男士手机包"),
("男士手机包",),
)
def test_category_parent_hover_uses_the_dropdown_option_not_its_label(self):
page = MagicMock()
trigger = MagicMock()
parent_option = MagicMock()
child_option = MagicMock()
with (
patch("collect_jd_market_rank._visible", return_value=trigger),
patch(
"collect_jd_market_rank._selected_category_labels",
side_effect=[("功能箱包",), ("男包 > 双肩包",)],
),
patch("collect_jd_market_rank._first_product_key", return_value="old"),
patch(
"collect_jd_market_rank._exact_visible_category_option",
return_value=parent_option,
) as resolve_option,
patch(
"collect_jd_market_rank._wait_for_category_option",
return_value=(child_option, "双肩包"),
) as wait_for_child,
patch(
"collect_jd_market_rank._click_category_option"
) as click_option,
patch("collect_jd_market_rank._wait_for_table_change"),
):
select_category_path(page, ("男包", "男士双肩包"))
resolve_option.assert_called_once_with(page, "男包")
wait_for_child.assert_called_once_with(
page,
("男士双肩包", "双肩包"),
)
parent_option.hover.assert_called_once_with(timeout=5_000)
click_option.assert_called_once_with(child_option, timeout_ms=10_000)
def test_same_selected_parent_opens_directly_on_child_options(self):
page = MagicMock()
trigger = MagicMock()
child_option = MagicMock()
with (
patch("collect_jd_market_rank._visible", return_value=trigger),
patch(
"collect_jd_market_rank._selected_category_labels",
side_effect=[
("功能箱包",),
("功能箱包 > 休闲运动包",),
],
),
patch("collect_jd_market_rank._first_product_key", return_value="old"),
patch(
"collect_jd_market_rank._exact_visible_category_option",
return_value=None,
) as resolve_option,
patch(
"collect_jd_market_rank._wait_for_category_option",
return_value=(child_option, "休闲运动包"),
),
patch(
"collect_jd_market_rank._click_category_option"
) as click_option,
patch("collect_jd_market_rank._wait_for_table_change"),
):
select_category_path(page, ("功能箱包", "休闲运动包"))
resolve_option.assert_called_once_with(page, "功能箱包")
click_option.assert_called_once_with(child_option, timeout_ms=10_000)
def test_cross_parent_switch_clicks_parent_when_hover_keeps_stale_children(self):
page = MagicMock()
trigger = MagicMock()
parent_option = MagicMock()
child_option = MagicMock()
with (
patch("collect_jd_market_rank._visible", return_value=trigger),
patch(
"collect_jd_market_rank._selected_category_labels",
side_effect=[("功能箱包",), ("男包 > 双肩包",)],
),
patch("collect_jd_market_rank._first_product_key", return_value="old"),
patch(
"collect_jd_market_rank._exact_visible_category_option",
return_value=parent_option,
),
patch(
"collect_jd_market_rank._wait_for_category_option",
side_effect=[(None, None), (child_option, "双肩包")],
) as wait_for_child,
patch(
"collect_jd_market_rank._click_category_option"
) as click_option,
patch("collect_jd_market_rank._wait_for_table_change"),
):
select_category_path(page, ("男包", "男士双肩包"))
self.assertEqual(wait_for_child.call_count, 2)
self.assertEqual(
[item.args[0] for item in click_option.call_args_list],
[parent_option, child_option],
)
def test_selected_category_verification_accepts_only_exact_display_forms(self):
path = ("男包", "双肩包")
@@ -104,6 +216,10 @@ class JdMarketRankTests(unittest.TestCase):
self.assertFalse(category_selection_matches(("男士双肩包",), path))
self.assertFalse(category_selection_matches(("双肩包配件",), path))
def test_selected_parent_matching_is_exact(self):
self.assertTrue(category_label_is_selected((" 功能 箱包 ",), "功能箱包"))
self.assertFalse(category_label_is_selected(("功能箱包配件",), "功能箱包"))
def test_category_candidates_fall_back_only_after_explicit_failure(self):
page = MagicMock()
paths = (("男包", "双肩包"), ("男包", "男士双肩包"))
@@ -0,0 +1,150 @@
from __future__ import annotations
import math
import pytest
from PIL import Image
from gyxx_flow.modules.product_commerce import jd_video_frames
@pytest.mark.parametrize(
("duration", "max_frames", "expected"),
[
(2.0, 3, (1.0,)),
(3.0, 3, (1.0, 2.0)),
(7.5, 3, (2.5, 5.0)),
(8.0, 3, (1.6, 4.0, 6.4)),
(8.0, 2, (4.0, 6.4)),
(48.88, 2, (24.44, 39.104)),
(48.88, 1, (24.44,)),
],
)
def test_choose_reference_frame_times(
duration: float,
max_frames: int,
expected: tuple[float, ...],
) -> None:
assert jd_video_frames.choose_reference_frame_times(
duration,
max_frames=max_frames,
) == expected
@pytest.mark.parametrize("duration", [0.0, -1.0, math.nan, math.inf])
def test_choose_reference_frame_times_rejects_invalid_duration(
duration: float,
) -> None:
with pytest.raises(
jd_video_frames.VideoFrameExtractionError,
match="视频时长无效",
):
jd_video_frames.choose_reference_frame_times(duration)
class _FakeReader:
def __init__(self, values: list[object], closed: list[bool]) -> None:
self._values = iter(values)
self._closed = closed
def __iter__(self):
return self
def __next__(self):
return next(self._values)
def close(self) -> None:
self._closed.append(True)
def test_extract_reference_frames_writes_decodable_jpegs(
tmp_path,
monkeypatch,
) -> None:
source = tmp_path / "source.mp4"
source.write_bytes(b"video-placeholder")
output_root = tmp_path / "allowed"
output_dir = output_root / "probe"
closed: list[bool] = []
frame = bytes(range(12))
def read_frames(_path, *, pix_fmt, input_params, output_params):
assert pix_fmt == "rgb24"
assert output_params == ["-frames:v", "1"]
metadata = {"duration": 12.0, "size": (2, 2)}
values = [metadata] if not input_params else [metadata, frame]
return _FakeReader(values, closed)
monkeypatch.setattr(jd_video_frames.imageio_ffmpeg, "read_frames", read_frames)
result = jd_video_frames.extract_reference_frames(
source,
output_dir=output_dir,
allowed_output_root=output_root,
timestamps_seconds=(3.0, 6.0),
max_frames=2,
max_edge=256,
)
assert [item.timestamp_seconds for item in result] == [3.0, 6.0]
assert len(closed) == 3
assert all(item.path.is_file() for item in result)
assert all(len(item.sha256) == 64 for item in result)
with Image.open(result[0].path) as image:
assert image.mode == "RGB"
assert image.size == (2, 2)
def test_extract_reference_frames_cleans_partial_outputs_on_failure(
tmp_path,
monkeypatch,
) -> None:
source = tmp_path / "source.mp4"
source.write_bytes(b"video-placeholder")
output_root = tmp_path / "allowed"
output_dir = output_root / "probe"
calls = 0
def read_frames(_path, *, pix_fmt, input_params, output_params):
nonlocal calls
calls += 1
metadata = {"duration": 12.0, "size": (2, 2)}
if not input_params:
return _FakeReader([metadata], [])
if calls == 2:
return _FakeReader([metadata, bytes(range(12))], [])
return _FakeReader([metadata, b"truncated"], [])
monkeypatch.setattr(jd_video_frames.imageio_ffmpeg, "read_frames", read_frames)
with pytest.raises(
jd_video_frames.VideoFrameExtractionError,
match="RGB 帧长度无效",
):
jd_video_frames.extract_reference_frames(
source,
output_dir=output_dir,
allowed_output_root=output_root,
timestamps_seconds=(3.0, 6.0),
max_frames=2,
max_edge=256,
)
assert list(output_dir.glob("*")) == []
def test_extract_reference_frames_rejects_output_outside_allowed_root(
tmp_path,
) -> None:
source = tmp_path / "source.mp4"
source.write_bytes(b"video-placeholder")
with pytest.raises(
jd_video_frames.VideoFrameExtractionError,
match="越出允许的临时目录",
):
jd_video_frames.extract_reference_frames(
source,
output_dir=tmp_path / "outside",
allowed_output_root=tmp_path / "allowed",
)
File diff suppressed because it is too large Load Diff
@@ -1,6 +1,6 @@
from __future__ import annotations
import json
import uuid
from datetime import date, datetime
from unittest.mock import MagicMock, patch
@@ -50,53 +50,6 @@ def test_build_notification_message_supports_partial_platform_success():
assert "当前成功平台 1 个" in message
def test_build_hermes_payload_requires_direct_message_to_recipient():
payload = notification.build_hermes_payload(
recipient_open_id="ou_wangyunlong",
message="三个链接",
)
encoded = json.dumps(payload, ensure_ascii=False)
assert "ou_wangyunlong" in encoded
assert "飞书私聊" in encoded
assert "三个链接" in encoded
def test_extracts_real_feishu_message_id_from_hermes_completion():
response = {
"id": "api-response-id",
"choices": [
{
"message": {
"content": "发送成功,消息ID om_x100b696f1169f0a0b1f9c311e8dc323"
}
}
],
}
assert (
notification.extract_feishu_message_id(response)
== "om_x100b696f1169f0a0b1f9c311e8dc323"
)
assert notification.hermes_send_succeeded(response) is True
def test_hermes_success_text_without_real_message_id_is_not_success():
response = {
"choices": [{"message": {"content": "发送成功,但没有返回消息回执"}}]
}
assert notification.extract_feishu_message_id(response) is None
assert notification.hermes_send_succeeded(response) is False
def test_extracts_nested_feishu_message_id_and_accepts_safe_suffixes():
response = {"data": {"message_id": "om_real-id_123"}}
assert notification.extract_feishu_message_id(response) == "om_real-id_123"
assert notification.hermes_send_succeeded(response) is True
def test_notify_skips_when_same_day_recipient_was_already_sent():
conn = MagicMock()
with (
@@ -104,7 +57,7 @@ def test_notify_skips_when_same_day_recipient_was_already_sent():
patch.object(notification, "ensure_market_rank_notification_table"),
patch.object(notification, "get_market_rank_reports_for_date", return_value=REPORTS),
patch.object(notification, "market_rank_notification_was_sent", return_value=True),
patch.object(notification, "post_hermes") as post_hermes,
patch.object(notification, "send_lark_bot_message") as send_message,
):
get_conn.return_value.__enter__.return_value = conn
result = notification.notify_market_rank_reports(
@@ -114,21 +67,12 @@ def test_notify_skips_when_same_day_recipient_was_already_sent():
assert result["status"] == "skipped"
assert result["reason"] == "already_sent"
post_hermes.assert_not_called()
send_message.assert_not_called()
def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
def test_notify_uses_lark_user_sender_and_persists_success():
conn = MagicMock()
response = {
"id": "hermes-response-id",
"choices": [
{
"message": {
"content": "发送成功,消息ID om_x100b696f1169f0a0b1f9c311e8dc323"
}
}
],
}
response = {"message_id": "om_x100b696f1169f0a0b1f9c311e8dc323"}
with (
patch.object(notification, "get_conn") as get_conn,
patch.object(notification, "ensure_market_rank_notification_table"),
@@ -138,7 +82,11 @@ def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
"get_market_rank_reports_for_date",
return_value=REPORTS,
) as get_reports,
patch.object(notification, "post_hermes", return_value=response) as post_hermes,
patch.object(
notification,
"send_lark_bot_message",
return_value=response,
) as send_message,
patch.object(notification, "upsert_market_rank_notification") as upsert,
):
get_conn.return_value.__enter__.return_value = conn
@@ -156,9 +104,16 @@ def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
notification.PLATFORM_ORDER,
updated_after=cutoff,
)
post_hermes.assert_called_once()
send_message.assert_called_once()
sent = send_message.call_args.kwargs
assert sent["user_id"] == "ou_wangyunlong"
assert sent["profile"] == "hermes-analyzer"
assert "天猫:https://example.com/tm" in sent["text"]
uuid.UUID(sent["idempotency_key"])
saved = upsert.call_args.args[1]
assert saved["status"] == "sent"
assert saved["hermes_message_id"] == response["message_id"]
assert result["message_id"] == response["message_id"]
assert saved["platform_links"] == {
"tm": "https://example.com/tm",
"jd": "https://example.com/jd",
@@ -168,15 +123,17 @@ def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
def test_notify_sends_available_links_when_other_platforms_are_missing():
conn = MagicMock()
response = {
"choices": [{"message": {"content": "发送成功,消息ID om_partial123"}}]
}
response = {"message_id": "om_partial123"}
with (
patch.object(notification, "get_conn") as get_conn,
patch.object(notification, "ensure_market_rank_notification_table"),
patch.object(notification, "market_rank_notification_was_sent", return_value=False),
patch.object(notification, "get_market_rank_reports_for_date", return_value=REPORTS[:2]),
patch.object(notification, "post_hermes", return_value=response) as post_hermes,
patch.object(
notification,
"send_lark_bot_message",
return_value=response,
) as send_message,
patch.object(notification, "upsert_market_rank_notification") as upsert,
):
get_conn.return_value.__enter__.return_value = conn
@@ -187,10 +144,41 @@ def test_notify_sends_available_links_when_other_platforms_are_missing():
assert result["status"] == "sent"
assert result["missing_platforms"] == ["dy"]
post_hermes.assert_called_once()
send_message.assert_called_once()
assert upsert.call_args.args[1]["status"] == "sent"
def test_notify_archives_lark_user_send_failure() -> None:
conn = MagicMock()
with (
patch.object(notification, "get_conn") as get_conn,
patch.object(notification, "ensure_market_rank_notification_table"),
patch.object(notification, "market_rank_notification_was_sent", return_value=False),
patch.object(
notification,
"get_market_rank_reports_for_date",
return_value=REPORTS,
),
patch.object(
notification,
"send_lark_bot_message",
side_effect=RuntimeError("lark unavailable"),
),
patch.object(notification, "upsert_market_rank_notification") as upsert,
):
get_conn.return_value.__enter__.return_value = conn
result = notification.notify_market_rank_reports(
report_date=date(2026, 7, 26),
recipient_open_id="ou_wangyunlong",
)
assert result["status"] == "failed"
assert "lark unavailable" in result["error"]
saved = upsert.call_args.args[1]
assert saved["status"] == "failed"
assert saved["hermes_message_id"] is None
def test_notify_skips_only_when_current_run_has_no_links():
conn = MagicMock()
with (
@@ -198,7 +186,7 @@ def test_notify_skips_only_when_current_run_has_no_links():
patch.object(notification, "ensure_market_rank_notification_table"),
patch.object(notification, "get_market_rank_reports_for_date", return_value=[]),
patch.object(notification, "market_rank_notification_was_sent") as was_sent,
patch.object(notification, "post_hermes") as post_hermes,
patch.object(notification, "send_lark_bot_message") as send_message,
patch.object(notification, "upsert_market_rank_notification") as upsert,
):
get_conn.return_value.__enter__.return_value = conn
@@ -210,5 +198,5 @@ def test_notify_skips_only_when_current_run_has_no_links():
assert result["status"] == "skipped"
assert result["reason"] == "no_platform_reports"
was_sent.assert_not_called()
post_hermes.assert_not_called()
send_message.assert_not_called()
assert upsert.call_args.args[1]["status"] == "incomplete"
@@ -5,6 +5,7 @@ import sys
from datetime import date, timedelta
from pathlib import Path
from threading import Barrier
from types import SimpleNamespace
import orchestrate_market_rank_collection as workflow
import pytest
@@ -219,6 +220,50 @@ def test_workflow_notifies_after_all_parallel_steps_finish(tmp_path):
assert saved["notification"]["status"] == "sent"
def test_workflow_notifies_each_dynamic_recipient_and_aggregates_results(tmp_path):
notifications: list[str] = []
def successful_executor(*, platform, command, log_path, dry_run):
return {
"platform": platform,
"label": workflow.PLATFORM_LABELS[platform],
"command": command,
"log_path": str(log_path),
"started_at": "2026-08-07T10:00:00",
"finished_at": "2026-08-07T10:01:00",
"duration_seconds": 60.0,
"exit_code": 0,
"status": "success",
}
def fake_notifier(**kwargs):
recipient = kwargs["recipient_open_id"]
notifications.append(recipient)
return {"status": "sent", "recipient_open_id": recipient}
summary = workflow.execute_workflow(
workflow.DEFAULT_PLATFORMS,
run_id="multi-notify-run",
log_dir=tmp_path,
executor=successful_executor,
report_date=date.today(),
notify_open_ids=("ou_first", "ou_second"),
notifier=fake_notifier,
)
assert notifications == ["ou_first", "ou_second"]
assert summary["notification"] == {
"status": "sent",
"recipient_count": 2,
"deliveries": [
{"status": "sent", "recipient_open_id": "ou_first"},
{"status": "sent", "recipient_open_id": "ou_second"},
],
}
assert summary["notifications"] == summary["notification"]["deliveries"]
assert summary["exit_code"] == 0
def test_partial_current_run_links_can_still_send_successfully(tmp_path):
def successful_executor(*, platform, command, log_path, dry_run):
return {
@@ -249,6 +294,36 @@ def test_partial_current_run_links_can_still_send_successfully(tmp_path):
assert summary["status"] == "success"
def test_no_current_run_links_is_a_valid_no_notification_outcome(tmp_path):
def successful_executor(*, platform, command, log_path, dry_run):
return {
"platform": platform,
"command": command,
"log_path": str(log_path),
"started_at": "2026-08-07T10:00:00",
"finished_at": "2026-08-07T10:01:00",
"duration_seconds": 60.0,
"exit_code": 0,
"status": "success",
}
summary = workflow.execute_workflow(
workflow.DEFAULT_PLATFORMS,
run_id="no-current-links-run",
log_dir=tmp_path,
executor=successful_executor,
notify_open_id="ou_owner",
notifier=lambda **_kwargs: {
"status": "skipped",
"reason": "no_platform_reports",
},
)
assert summary["notification"]["reason"] == "no_platform_reports"
assert summary["exit_code"] == 0
assert summary["status"] == "success"
def test_platform_parser_deduplicates_while_preserving_order():
assert workflow.parse_platforms("dy,tm,dy,jd") == ("dy", "tm", "jd")
@@ -289,6 +364,41 @@ def test_main_allows_explicit_no_notify_without_recipient(monkeypatch, tmp_path)
assert observed["notify_open_id"] is None
def test_main_passes_all_dynamic_route_recipients(monkeypatch, tmp_path):
observed = {}
def fake_execute(*args, **kwargs):
observed.update(kwargs)
return {"exit_code": 0}
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
monkeypatch.setattr(workflow, "DEFAULT_NOTIFY_OPEN_ID", "")
monkeypatch.setattr(
workflow,
"resolve_notification_route",
lambda *_args, **_kwargs: SimpleNamespace(
enabled=True,
open_ids=("ou_first", "ou_second"),
),
)
monkeypatch.setattr(workflow, "execute_workflow", fake_execute)
monkeypatch.setattr(workflow, "managed_data_path", Path)
monkeypatch.setattr(
sys,
"argv",
[
"orchestrate_market_rank_collection.py",
"--dry-run",
"--log-dir",
str(tmp_path),
],
)
assert workflow.main() == 0
assert observed["notify_open_ids"] == ("ou_first", "ou_second")
assert observed["notify_open_id"] is None
def test_weekly_launcher_uses_market_rank_orchestrator():
launcher = (
HISTORY_ROOT / "launchers_reference" / "run_weekly_market_rank.bat"
@@ -19,6 +19,26 @@ WORKERS = (tm_worker, dy_worker, jd_worker)
BUSINESS_DATE = date(2026, 8, 4)
@pytest.mark.parametrize("worker", WORKERS)
def test_missing_time_select_option_is_treated_as_no_existing_record(monkeypatch, worker) -> None:
monkeypatch.setattr(
worker.subprocess,
"run",
lambda *_a, **_k: SimpleNamespace(
returncode=1,
stdout="",
stderr=json.dumps(
{
"ok": False,
"error": {"code": 800030005, "message": "not_found"},
}
),
),
)
assert worker.find_record_id_by_time("bas_test", "tbl_test", "8.13") is None
@pytest.mark.parametrize("worker", WORKERS)
def test_platform_collector_failure_forces_nonzero(worker) -> None:
assert worker.combined_result_exit_code(
@@ -78,6 +98,30 @@ def test_formal_base_upsert_accepts_verified_response_record_id(monkeypatch, wor
assert message == "record_id=rec12345678"
@pytest.mark.parametrize("worker", WORKERS)
def test_formal_base_update_reuses_valid_requested_record_id(monkeypatch, worker) -> None:
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
monkeypatch.setattr(
worker.subprocess,
"run",
lambda *_a, **_k: SimpleNamespace(
returncode=0,
stdout=json.dumps({"ok": True, "data": {"updated": True}}),
stderr="",
),
)
ok, message = worker.upsert_record(
"bas_test",
"tbl_test",
{"时间": "8.4", "男性比例": 0.5},
record_id="rec_existing123",
)
assert ok is True
assert message == "record_id=rec_existing123"
@pytest.mark.parametrize("worker", WORKERS)
def test_acceptance_skip_is_not_formal_success(monkeypatch, worker) -> None:
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: True)
@@ -265,6 +309,21 @@ def test_zero_distributions_are_not_valid_persona() -> None:
) is False
def test_dy_persistent_profile_is_not_overwritten_by_exported_cookies() -> None:
assert dy_collector.persistent_profile_launch_options("bound-profile") == {
"user_data_dir": "bound-profile"
}
def test_dy_profileless_fallback_can_restore_exported_cookies() -> None:
cookies = [{"name": "session", "value": "new", "domain": ".jinritemai.com"}]
assert dy_collector.cookie_fallback_launch_options(cookies) == {
"cookies": cookies
}
assert dy_collector.cookie_fallback_launch_options(None) == {}
def test_dmp_skips_are_not_valid_style_results() -> None:
styles = {"款A": ["123"]}
assert dmp.records_exit_code(
@@ -317,3 +376,24 @@ def test_tm_collector_command_redacts_credentials() -> None:
assert "secret-user" not in rendered
assert "secret-pass" not in rendered
assert rendered.count("***") == 2
def test_jd_persona_login_home_navigation_does_not_wait_for_networkidle() -> None:
project_root = Path(__file__).resolve().parents[3]
source_path = (
project_root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "product_commerce"
/ "vendors"
/ "jd-data-flow"
/ "jd_data_collector.py"
)
source = source_path.read_text(encoding="utf-8")
step_login = source[source.index("def step_login") :]
home_block = step_login[step_login.index("shop.jd.com/jdm/home") :]
assert 'wait_until="networkidle"' not in home_block[:400]
assert 'wait_until="domcontentloaded"' in home_block[:400]
assert "timeout=60_000" in home_block[:400]
@@ -15,6 +15,46 @@ class PersonaLauncherTests(unittest.TestCase):
def test_all_platforms_success_makes_launcher_success(self):
self.assertEqual(run_daily_persona.combined_exit_code({"tm": 0, "dy": 0, "jd": 0}), 0)
def test_partial_platform_results_are_accepted_when_real_rows_were_written(self):
results = {"tm": 2, "dy": 2, "jd": 2}
summaries = {
"tm": {"ok": 31, "skipped": 0, "failed": 4, "total": 35},
"dy": {"ok": 2, "skipped": 0, "failed": 30, "total": 31},
"jd": {"ok": 18, "skipped": 0, "failed": 13, "total": 31},
}
self.assertEqual(run_daily_persona.combined_exit_code(results, summaries), 0)
def test_partial_platform_without_real_rows_still_fails(self):
results = {"tm": 2, "dy": 2, "jd": 2}
summaries = {
"tm": {"ok": 31},
"dy": {"ok": 0},
"jd": {"ok": 18},
}
self.assertEqual(run_daily_persona.combined_exit_code(results, summaries), 2)
def test_result_summary_is_scoped_to_current_log_segment(self):
from tempfile import TemporaryDirectory
with TemporaryDirectory() as directory:
path = Path(directory) / "persona.log"
path.write_text(
"[DONE] ok=0 permission_skipped=0 failed=31 total=31\n",
encoding="utf-8",
)
offset = path.stat().st_size
with path.open("a", encoding="utf-8") as stream:
stream.write(
"[DONE] ok=18 permission_skipped=0 failed=13 total=31\n"
)
self.assertEqual(
run_daily_persona.read_result_summary(path, offset),
{"ok": 18, "skipped": 0, "failed": 13, "total": 31},
)
def test_launcher_passes_business_date_to_each_worker(self):
command = run_daily_persona.build_child_command(
Path("collect_persona_to_bitable.py"),
@@ -1,6 +1,5 @@
from __future__ import annotations
import io
import json
from pathlib import Path
from types import SimpleNamespace
@@ -11,7 +10,7 @@ import lark_cli_runtime
import market_rank_hermes_notification as market_notification
import pytest
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path:
@@ -103,39 +102,38 @@ def test_product_shared_lark_cli_allows_safe_profile_override(monkeypatch) -> No
]
def test_decline_hermes_prompt_forces_wang_yunlong(
def test_decline_lark_user_send_forces_acceptance_recipient(
monkeypatch,
tmp_path: Path,
) -> None:
_enable_acceptance(monkeypatch, tmp_path)
monkeypatch.setattr(decline, "HERMES_API_KEY", "test-only")
captured = {}
class Response:
def __enter__(self):
return self
def fake_send(**kwargs):
captured.update(kwargs)
return {"message_id": "om_test"}
def __exit__(self, *args):
return False
def read(self):
return json.dumps({"choices": [{"message": {"content": "ok"}}]}).encode()
def fake_urlopen(request, timeout):
captured["body"] = json.load(io.BytesIO(request.data))
return Response()
monkeypatch.setattr(decline.urllib.request, "urlopen", fake_urlopen)
decline.notify_hermes(
[{"style_name": "test", "platform": "tm"}],
monkeypatch.setattr(decline, "send_lark_bot_message", fake_send)
decline.send_decline_notification(
[
{
"style_name": "test",
"platform": "tm",
"seg1_sales": 30,
"seg2_sales": 20,
"seg3_sales": 10,
"drop_pct_1to2": 33.3,
"drop_pct_2to3": 50.0,
"drop_pct_1to3": 66.7,
}
],
"2026-08-01",
9,
openid="ou_someone_else",
)
encoded = json.dumps(captured["body"], ensure_ascii=False)
assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded
assert "ou_someone_else" not in encoded
assert captured["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
assert captured["profile"] == "hermes-analyzer"
def test_market_rank_notification_forces_wang_yunlong(
@@ -175,14 +173,14 @@ def test_market_rank_notification_forces_wang_yunlong(
lambda *args, **kwargs: False,
)
def fake_post(payload):
captured["payload"] = payload
def fake_send(**kwargs):
captured["send"] = kwargs
return {"message_id": "om_test"}
def fake_archive(conn, record):
captured["record"] = record
monkeypatch.setattr(market_notification, "post_hermes", fake_post)
monkeypatch.setattr(market_notification, "send_lark_bot_message", fake_send)
monkeypatch.setattr(
market_notification,
"upsert_market_rank_notification",
@@ -194,11 +192,13 @@ def test_market_rank_notification_forces_wang_yunlong(
recipient_open_id="ou_someone_else",
)
encoded = json.dumps(captured["payload"], ensure_ascii=False)
assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded
assert "ou_someone_else" not in encoded
assert result["recipient_open_id"] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
assert (
captured["send"]["user_id"]
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
)
assert captured["send"]["profile"] == "hermes-analyzer"
assert result["recipient_open_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
assert (
captured["record"]["recipient_open_id"]
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
)
@@ -0,0 +1,46 @@
from __future__ import annotations
from datetime import date
import db as product_db
class _Cursor:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
class _Connection:
def cursor(self):
return _Cursor()
def test_product_daily_upsert_keeps_existing_ad_metrics(monkeypatch) -> None:
captured: dict[str, object] = {}
def fake_execute_values(_cursor, sql, rows):
captured["sql"] = sql
captured["rows"] = rows
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
written = product_db.upsert_product_daily_metrics(
_Connection(),
[{
"platform": "tm",
"stat_date": date(2026, 8, 17),
"product_id": "1001",
"product_name": "测试商品",
"raw_data": {"paid_amount": "100.00"},
"source_file": "商品经营日报.xls",
}],
)
assert written == 1
assert (
"COALESCE(product_daily_metrics.raw_data, '{}'::jsonb)\n"
" || EXCLUDED.raw_data"
) in str(captured["sql"])
@@ -8,6 +8,7 @@ import backfill_collect
import backfill_one_day
import collect_dy_persona_to_bitable as dy_persona_worker
import collect_persona_to_bitable as tm_persona_worker
import dy_product_scraping as dy_product
import orchestrate_daily_collection as daily
import orchestrate_market_rank_collection as market_rank
import orchestrate_review_collection as reviews
@@ -115,6 +116,40 @@ def test_daily_run_step_rebinds_to_command_target(monkeypatch) -> None:
_assert_child_binding(observed["environment"])
def test_daily_command_logging_redacts_sensitive_values() -> None:
command = [
"python",
"collector.py",
"--account",
"secret-user",
"--password=placeholder",
"--api-key",
"example",
"--target-date",
"2026-08-04",
]
rendered = daily.redacted_command(command)
assert "secret-user" not in rendered
assert "placeholder" not in rendered
assert "example" not in rendered
assert rendered.count("[REDACTED]") == 3
assert "2026-08-04" in rendered
def test_daily_saved_cookie_binding_refreshes_existing_profile(tmp_path) -> None:
profile = tmp_path / "profile"
cookie_db = profile / "Default" / "Network" / "Cookies"
cookie_db.parent.mkdir(parents=True)
cookie_db.write_bytes(b"x" * 5000)
cookies = [{"name": "session", "value": "secret", "domain": ".jinritemai.com"}]
assert dy_product._saved_cookie_launch_options(str(profile), cookies) == {
"cookies": cookies
}
def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None:
observed: dict[str, object] = {}
monkeypatch.setattr(
@@ -13,12 +13,20 @@ def _workflow(workflow_id: str) -> dict:
return next(item for item in catalog["workflows"] if item["id"] == workflow_id)
def _schedule(workflow_id: str) -> dict:
schedules = json.loads(
(REPOSITORY_ROOT / "config" / "schedules.json").read_text(encoding="utf-8")
)
return next(item for item in schedules["schedules"] if item["workflow_id"] == workflow_id)
def test_product_workflows_forward_the_selected_business_date() -> None:
persona = _workflow("product.persona.daily")["execution"]["steps"][0]
style = _workflow("product.style_analysis.interval")["execution"]["steps"][0]
main_image = _workflow("product.main_image.weekly")["execution"]["steps"]
market_rank = _workflow("product.market_rank")["execution"]["steps"][0]
monthly_sales = _workflow("product.sales_sheet.daily")["execution"]["steps"][0]
all_shop_daily = _workflow("product.erp_all_shop_daily")["execution"]["steps"][0]
assert persona["args"] == ["--date", "{business_date}"]
assert style["args"][-2:] == ["--end-date", "{business_date}"]
@@ -33,3 +41,46 @@ def test_product_workflows_forward_the_selected_business_date() -> None:
"--execute",
"--allow-missing-erp-as-zero",
]
assert all_shop_daily["args"] == [
"--from",
"{business_date}",
"--to",
"{business_date}",
"--execute",
"--force",
]
def test_ecommerce_costs_workflow_runs_three_platforms_in_parallel() -> None:
workflow = _workflow("product.ecommerce_costs.daily")
steps = workflow["execution"]["steps"]
by_id = {step["id"]: step for step in steps}
assert [step["id"] for step in steps] == [
"tmall_download",
"tmall_import",
"jd_download",
"jd_import",
"douyin_download",
"douyin_import",
]
for platform, collector, importer in (
("tmall", "collect_tmall_wanxiang_ads.py", "import_tmall_product_ads.py"),
("jd", "collect_jd_ad_costs.py", "import_jd_ad_costs.py"),
("douyin", "collect_douyin_qianchuan_ads.py", "import_douyin_qianchuan_ads.py"),
):
download = by_id[f"{platform}_download"]
import_step = by_id[f"{platform}_import"]
assert download["entry"] == collector
assert download["args"] == ["--date", "{business_date}", "--headless"]
assert download.get("depends_on", []) == []
assert import_step["entry"] == importer
assert import_step["args"] == ["--date", "{business_date}", "--execute"]
assert import_step["depends_on"] == [f"{platform}_download"]
assert "点击下载报表" in by_id["tmall_download"]["description"]
assert "当前时间" in by_id["jd_import"]["description"]
assert "SKU" in by_id["jd_import"]["description"]
schedule = _schedule("product.ecommerce_costs.daily")
assert schedule["at"] == "10:00"
assert schedule["business_date_offset_days"] == -1
@@ -1,5 +1,6 @@
import json
import os
import re
import tempfile
import unittest
from datetime import date
@@ -7,7 +8,7 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import analyze_style_with_hermes as analysis
import analyze_style as analysis
import orchestrate_daily_collection as orchestrator
from config.style_config_loader import StyleConfigLoader
@@ -140,6 +141,60 @@ class StyleAnalysisWindowTests(unittest.TestCase):
for heading in headings:
self.assertIn(heading, analysis.SYSTEM)
def test_system_prompt_and_validator_require_pure_chinese_report(self):
self.assertIn("纯中文硬约束", analysis.SYSTEM)
self.assertIn("不得出现英文单词", analysis.SYSTEM)
valid_report = "\n".join([
*analysis.REPORT_HEADINGS,
*analysis.REPORT_SUBHEADINGS,
"**P0**", "**重点**", "**重点**", "**重点**", "**重点**",
"**重点**", "**重点**", "**重点**",
])
normalized, error = analysis.validate_report_markdown(valid_report)
self.assertIsNone(error)
self.assertIn("**P0**", normalized)
english_report = valid_report.replace("**P0**", "English text")
_, error = analysis.validate_report_markdown(english_report)
self.assertEqual(error, "MiniMax 报告包含英文内容,必须改为纯中文")
def test_validator_removes_separate_reasoning_before_language_check(self):
valid_report = "\n".join([
"<think>English reasoning</think>",
*analysis.REPORT_HEADINGS,
*analysis.REPORT_SUBHEADINGS,
"**P0**", "**重点**", "**重点**", "**重点**", "**重点**",
"**重点**", "**重点**", "**重点**",
])
normalized, error = analysis.validate_report_markdown(valid_report)
self.assertIsNone(error)
self.assertNotIn("English reasoning", normalized)
def test_localization_fallback_converts_source_english_to_chinese(self):
report = "英文前缀\nSwitchLite、SKU、IP、ROI、A/B、Apollo、notes_recent_top8、**P0**"
localized = analysis._localize_report_latin(report)
latin_tokens = set(re.findall(r"[A-Za-z][A-Za-z0-9_-]*", localized))
self.assertEqual(latin_tokens, {"P0"})
self.assertIn("游戏主机", localized)
self.assertIn("规格", localized)
self.assertIn("地域", localized)
self.assertIn("投入产出比", localized)
self.assertIn("甲/乙", localized)
self.assertIn("**P0**", localized)
def test_direct_checkpoint_namespace_isolated_for_language_gate_revision(self):
self.assertEqual(analysis.CHECKPOINT_NAMESPACE, "direct_minimax_v2")
self.assertIn("direct_minimax_v2", str(analysis.StyleAnalysisCheckpointSpec.for_report(
"测试款",
date(2026, 8, 1),
date(2026, 8, 3),
3,
"测试款 单款式多维度分析 2026-08-01~2026-08-03",
"# 内容",
).path()))
def test_plain_seven_section_titles_are_normalized_to_markdown(self):
report = """盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)
@@ -164,6 +219,99 @@ class StyleAnalysisWindowTests(unittest.TestCase):
self.assertTrue(normalized.startswith("# 盖亚斜挎"))
self.assertTrue(analysis.has_exact_report_headings(normalized))
def test_number_variants_and_bare_total_title_are_canonicalized(self):
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
for index, heading in enumerate(analysis.REPORT_HEADINGS):
subject = heading.split("", 1)[1]
raw_heading = heading
if index == 4:
raw_heading = f"## 六、{subject}"
elif index == 6:
raw_heading = "## 总评"
blocks.extend([
raw_heading,
*analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3],
"**重点**",
])
normalized = analysis.normalize_report_markdown("\n".join(blocks))
self.assertEqual(
[
line.strip()
for line in normalized.splitlines()
if line.strip().startswith("## ")
],
analysis.REPORT_HEADINGS,
)
def test_priority_action_heading_is_inserted_for_unheaded_priority_lines(self):
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
for index, heading in enumerate(analysis.REPORT_HEADINGS[:-1]):
blocks.extend([
heading,
*analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3],
"**重点**",
])
blocks.extend([
analysis.REPORT_HEADINGS[-1],
"### 核心结论",
"**当前核心问题是转化偏低。**",
"### 转化链路",
"曝光到访客再到成交。",
"- **P0**48小时内完成整改。",
])
normalized = analysis.normalize_report_markdown("\n".join(blocks))
self.assertTrue(analysis.has_clear_report_hierarchy(normalized))
self.assertIn("### 优先级行动", normalized)
def test_minimax_adapter_disables_tools_and_uses_extended_timeout(self):
captured = {}
def fake_call(system, user, **kwargs):
captured.update(system=system, user=user, kwargs=kwargs)
return "报告", {}
with (
patch.object(analysis, "call_direct_llm", fake_call),
):
content, _ = analysis.call_minimax("system", "user")
self.assertEqual(content, "报告")
self.assertEqual(captured["system"], "system")
self.assertEqual(captured["user"], "user")
self.assertEqual(captured["kwargs"]["model"], analysis.MINIMAX_MODEL)
self.assertEqual(captured["kwargs"]["temperature"], 0.3)
self.assertEqual(captured["kwargs"]["timeout"], analysis.MINIMAX_TIMEOUT_SECONDS)
self.assertEqual(captured["kwargs"]["max_tokens"], analysis.MINIMAX_MAX_TOKENS)
self.assertEqual(captured["kwargs"]["thinking_mode"], "disabled")
def test_invalid_report_is_regenerated_once_before_failure(self):
valid_lines = []
for index, heading in enumerate(analysis.REPORT_HEADINGS):
valid_lines.extend([
heading,
*analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3],
"**重点**",
])
valid_report = "\n".join(valid_lines)
with (
patch.object(analysis, "MINIMAX_FORMAT_RETRIES", 1),
patch.object(
analysis,
"call_minimax",
side_effect=[("不完整报告", {"attempt": 1}), (valid_report, {"attempt": 2})],
) as call,
):
report, usage = analysis.generate_validated_report("user prompt")
self.assertEqual(report, analysis.normalize_report_markdown(valid_report))
self.assertEqual(usage, {"attempt": 2})
self.assertEqual(call.call_count, 2)
def test_report_gets_clear_subheadings_and_key_emphasis(self):
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
for heading in analysis.REPORT_HEADINGS[:-1]:
@@ -190,6 +338,32 @@ class StyleAnalysisWindowTests(unittest.TestCase):
self.assertIn("**P0**", normalized)
self.assertIn("**24小时内**", normalized)
def test_report_normalization_is_idempotent_for_checkpoint_recovery(self):
blocks = ["逐星GT 单款式多维度分析报告 (2026-08-01~2026-08-03)"]
for heading in analysis.REPORT_HEADINGS[:-1]:
blocks.extend([
heading,
"### 数据快照",
"**三日访客 100**。",
"### 核心判断",
"**当前转化偏低。**",
"### 优化建议",
"P13天内完成优化。",
])
blocks.extend([
analysis.REPORT_HEADINGS[-1],
"### 核心结论",
"**当前核心问题是转化偏低。**",
"### 转化链路",
"曝光到访客再到成交。",
"### 优先级行动",
"- **P0(48小时内)**:完成整改。",
])
normalized = analysis.normalize_report_markdown("\n".join(blocks))
self.assertEqual(analysis.normalize_report_markdown(normalized), normalized)
def test_extra_helpful_h3_does_not_reject_complete_required_hierarchy(self):
required = "\n".join(analysis.REPORT_SUBHEADINGS)
report = required.replace(
@@ -212,6 +386,78 @@ class StyleAnalysisWindowTests(unittest.TestCase):
self.assertEqual(missing, [date(2026, 7, 15)])
def test_batch_style_selection_requires_a_complete_calendar_window(self):
class Cursor:
def __init__(self, rows):
self.rows = rows
self.sql = ""
self.params = None
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def execute(self, sql, params):
self.sql = sql
self.params = params
def fetchall(self):
return self.rows
class Connection:
def __init__(self, cursor):
self.cursor_value = cursor
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def cursor(self):
return self.cursor_value
cursor = Cursor([("完整款", 100)])
with patch.object(analysis, "get_conn", return_value=Connection(cursor)):
styles = analysis.fetch_styles_with_data(date(2026, 7, 15), 3)
self.assertEqual(styles, ["完整款"])
self.assertIn("HAVING COUNT(DISTINCT metric_date) = %s", cursor.sql)
self.assertEqual(
cursor.params,
(date(2026, 7, 13), date(2026, 7, 15), 3),
)
def test_incomplete_batch_styles_are_reported_with_present_dates(self):
cursor = type(
"Cursor",
(),
{
"__enter__": lambda self: self,
"__exit__": lambda self, exc_type, exc, tb: False,
"execute": lambda self, sql, params: setattr(self, "params", params),
"fetchall": lambda self: [("云栖", [date(2026, 7, 15)])],
},
)()
connection = type(
"Connection",
(),
{
"__enter__": lambda self: self,
"__exit__": lambda self, exc_type, exc, tb: False,
"cursor": lambda self: cursor,
},
)()
with patch.object(analysis, "get_conn", return_value=connection):
incomplete = analysis.fetch_styles_with_incomplete_data(
date(2026, 7, 15), 3,
)
self.assertEqual(incomplete, {"云栖": [date(2026, 7, 15)]})
class LarkEnvironmentTests(unittest.TestCase):
def test_legacy_lark_config_is_used_when_default_is_missing(self):
@@ -607,10 +853,6 @@ class TargetBaseWriteTests(unittest.TestCase):
{"name": "时间", "type": "select"},
{"name": "笔记分析汇总", "type": "text"},
]}},
{"data": {"field": {
"id": "fld_time", "name": "时间", "type": "select",
"multiple": False, "options": [{"name": "2026-07-13~07-15"}],
}}},
{"data": {
"data": [[['2026-07-13~07-15'], "old link"]],
"fields": ["时间", "笔记分析汇总"],
@@ -627,33 +869,117 @@ class TargetBaseWriteTests(unittest.TestCase):
)
self.assertEqual(record_id, "rec_existing")
upsert_args = run.call_args_list[3].args[0]
upsert_args = run.call_args_list[2].args[0]
self.assertIn("--record-id", upsert_args)
self.assertIn("rec_existing", upsert_args)
payload = json.loads(upsert_args[upsert_args.index("--json") + 1])
self.assertIn("笔记分析汇总", payload)
def test_missing_period_is_appended_without_dropping_existing_options(self):
def test_missing_period_is_created_without_rewriting_select_field(self):
responses = [
{"data": {"field": {
"id": "fld_time", "name": "时间", "type": "select",
"multiple": False, "options": [{"name": "7月第二周", "hue": "Blue"}],
}}},
{"ok": True},
{"data": {"fields": [
{"name": "时间", "type": "select"},
{"name": "笔记分析汇总", "type": "text"},
]}},
{"data": {
"data": [],
"fields": ["时间", "笔记分析汇总"],
"record_id_list": [],
}},
{"ok": True, "data": {"record_id": "rec_created"}},
]
with patch.object(analysis, "run_lark_cli", side_effect=responses) as run:
analysis.ensure_select_option(
record_id = analysis.write_analysis_link_to_base(
{"base_token": "base", "table_id": "table"},
"时间", "2026-07-13~07-15", "盖亚斜挎",
"2026-08-01~08-03",
"https://example.feishu.cn/docx/doc",
"盖亚斜挎",
)
update_args = run.call_args_list[1].args[0]
definition = json.loads(update_args[update_args.index("--json") + 1])
self.assertEqual(record_id, "rec_created")
self.assertEqual(
[option["name"] for option in definition["options"]],
["7月第二周", "2026-07-13~07-15"],
[call.args[0][1] for call in run.call_args_list],
["+field-list", "+record-list", "+record-upsert"],
)
self.assertIn("--yes", update_args)
payload_args = run.call_args_list[2].args[0]
payload = json.loads(payload_args[payload_args.index("--json") + 1])
self.assertEqual(payload["时间"], "2026-08-01~08-03")
def test_timeout_reconciles_remote_upsert_without_second_write(self):
document_url = "https://example.feishu.cn/docx/doc"
responses = [
{"data": {"fields": [
{"name": "时间", "type": "select"},
{"name": "笔记分析汇总", "type": "text"},
]}},
{"data": {
"data": [],
"fields": ["时间", "笔记分析汇总"],
"record_id_list": [],
}},
RuntimeError(
'lark-cli failed (exit=4): {"error":{"type":"network",'
'"subtype":"timeout"}}'
),
{"data": {
"data": [[
["2026-08-01~08-03"],
f"[盖亚斜挎 三日单品分析]({document_url})",
]],
"fields": ["时间", "笔记分析汇总"],
"record_id_list": ["rec_reconciled"],
}},
]
with (
patch.object(analysis, "run_lark_cli", side_effect=responses) as run,
patch.object(analysis, "BASE_WRITE_RECONCILE_DELAYS_SECONDS", (0,)),
):
record_id = analysis.write_analysis_link_to_base(
{"base_token": "base", "table_id": "table"},
"2026-08-01~08-03",
document_url,
"盖亚斜挎",
)
self.assertEqual(record_id, "rec_reconciled")
self.assertEqual(
[call.args[0][1] for call in run.call_args_list],
["+field-list", "+record-list", "+record-upsert", "+record-list"],
)
self.assertEqual(run.call_args_list[2].kwargs["timeout"], 180)
def test_timeout_reconciliation_requires_the_expected_report_link(self):
responses = [
{"data": {"fields": [
{"name": "时间", "type": "select"},
{"name": "笔记分析汇总", "type": "text"},
]}},
{"data": {
"data": [],
"fields": ["时间", "笔记分析汇总"],
"record_id_list": [],
}},
RuntimeError("API call failed: server time out error"),
{"data": {
"data": [[
["2026-08-01~08-03"],
"[盖亚斜挎 三日单品分析](https://example.feishu.cn/docx/old)",
]],
"fields": ["时间", "笔记分析汇总"],
"record_id_list": ["rec_old"],
}},
]
with (
patch.object(analysis, "run_lark_cli", side_effect=responses),
patch.object(analysis, "BASE_WRITE_RECONCILE_DELAYS_SECONDS", (0,)),
):
with self.assertRaisesRegex(RuntimeError, "server time out error"):
analysis.write_analysis_link_to_base(
{"base_token": "base", "table_id": "table"},
"2026-08-01~08-03",
"https://example.feishu.cn/docx/new",
"盖亚斜挎",
)
class OrchestratorTests(unittest.TestCase):
@@ -663,7 +989,7 @@ class OrchestratorTests(unittest.TestCase):
orchestrator.run_style_analysis(args, "2026-07-15", failed=None)
command = run.call_args.args[1]
self.assertIn("analyze_style_with_hermes.py", [Path(item).name for item in command])
self.assertIn("analyze_style.py", [Path(item).name for item in command])
self.assertIn("--all-styles", command)
self.assertEqual(command[command.index("--days") + 1], "3")
self.assertEqual(command[command.index("--min-interval-days") + 1], "3")
@@ -39,25 +39,32 @@ def test_non_tm_erp_code_is_not_used_when_tm_code_is_missing() -> None:
assert "天猫分组未填写" in output.getvalue()
def test_exact_style_field_wins_over_style_content_prefix(monkeypatch) -> None:
loader = StyleConfigLoader()
monkeypatch.setattr(
loader,
"_run_lark",
lambda _args: {
"data": {
"fields": ["款式内容", "款式", "ERP款式编码", "平台"],
"data": [["营销文案", "盖世m1", "10416,10455", ["天猫"]]],
def test_database_row_uses_explicit_style_name_not_style_content() -> None:
loader = StyleConfigLoader(
records_provider=lambda: [
{
"style_content": "营销文案",
"style_name": "盖世m1",
"erp_codes": ["10416", "10455"],
"platform": "天猫",
}
},
]
)
records = loader._load_records_from_lark()
records = loader._load_records_from_database()
assert records == [
{
"style": "盖世m1",
"erp_codes": "10416,10455",
"erp_codes": ["10416", "10455"],
"platform": "天猫",
"item_ids": [],
"brand": "",
"sales_bitable_url": "",
"main_image_bitable_url": "",
"persona_bitable_url": "",
"dy_persona_bitable_url": "",
"jd_persona_bitable_url": "",
"style_analysis_bitable_url": "",
}
]
@@ -14,6 +14,7 @@ from collect_sycm_market_rank import (
collect_all_rank_pages,
create_feishu_doc,
detect_risk_control_text,
dismiss_floating_overlays,
ensure_market_rank_login,
group_products_by_category,
inspect_feishu_batch_tables,
@@ -49,6 +50,15 @@ SAMPLE_HTML = """
class SycmMarketRankTests(unittest.TestCase):
def test_floating_overlays_are_removed_before_pagination(self):
page = MagicMock()
page.evaluate.return_value = 3
self.assertEqual(dismiss_floating_overlays(page), 3)
script = page.evaluate.call_args.args[0]
self.assertIn("scenario-widget", script)
self.assertIn("aes-survey-hanging", script)
def test_expired_login_without_credentials_fails_clearly(self):
with (
patch("collect_sycm_market_rank.sycm.ACCOUNT", ""),
@@ -1,5 +1,6 @@
from __future__ import annotations
import io
from datetime import date
import pytest
@@ -17,6 +18,7 @@ from gyxx_flow.modules.product_commerce.sync_monthly_sales_sheet import (
MonthlySalesConfig,
ProductTarget,
SheetInfo,
_reconfigure_stream,
_write_rows,
build_parser,
choose_template,
@@ -43,6 +45,17 @@ def test_month_contract_uses_natural_month_and_existing_title_convention() -> No
assert month_sheet_title(month) == "26年8月"
def test_reconfigure_stream_switches_gbk_stream_to_utf8() -> None:
raw = io.BytesIO()
stream = io.TextIOWrapper(raw, encoding="gbk", errors="strict")
_reconfigure_stream(stream)
stream.write("商品\u200c\n")
stream.flush()
assert "商品\u200c".encode("utf-8") in raw.getvalue()
def test_erp_month_interval_uses_inclusive_natural_month_for_day_input() -> None:
assert monthly_report_interval(parse_month("2026-07-02")) == (
"2026-07-01",
@@ -0,0 +1,298 @@
from __future__ import annotations
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
import taobao_sycm_collect as collect
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
@pytest.fixture(autouse=True)
def stub_collection_chrome(monkeypatch):
"""Keep wrapper tests from starting a real browser on collector ports."""
monkeypatch.setattr(collect, "is_chrome_debug_running", lambda _port: False)
monkeypatch.setattr(
collect,
"launch_chrome_with_debug",
lambda *_args, **_kwargs: True,
)
def test_tm_identity_markers_require_the_complete_store_name(monkeypatch) -> None:
monkeypatch.delenv("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS", raising=False)
assert collect is not None
# The child collector owns the actual page identity check; import it here
# so this regression test also guards its brand fallback markers.
import taobao_sycm_products as products
monkeypatch.setattr(products, "BRAND", "ozko")
monkeypatch.setattr(products, "ACCOUNT", "")
assert products._login_identity_markers() == ("ozko旗舰店", "OZKO旗舰店")
monkeypatch.setattr(products, "BRAND", "光影行星")
assert products._login_identity_markers() == ("光影行星旗舰店",)
def test_dynamic_tm_brand_route_selects_the_requested_account_vault(
monkeypatch,
) -> None:
calls: list[str] = []
class FakeCatalog:
def environment_for_account(self, account_id, base_environment):
calls.append(account_id)
return {
**base_environment,
"GYXX_ACCOUNT_ID": account_id,
"GYXX_ACCOUNT_COOKIE_FILE": f"{account_id}-cookies.json",
"GYXX_ACCOUNT_STORAGE_STATE_FILE": f"{account_id}-state.json",
}
monkeypatch.setattr(
collect.RuntimeIntegrationCatalog,
"load_default",
lambda **_kwargs: FakeCatalog(),
)
environment = collect._account_environment(
"tmall-ozko",
{
"GYXX_PROJECT_ROOT": "D:/project",
"GYXX_DATA_ROOT": "D:/data",
"KEEP": "yes",
},
)
assert calls == ["tmall-ozko"]
assert environment["KEEP"] == "yes"
assert environment["GYXX_ACCOUNT_ID"] == "tmall-ozko"
assert environment["GYXX_ACCOUNT_COOKIE_FILE"] == "tmall-ozko-cookies.json"
@pytest.mark.parametrize(
("extra_args", "expects_backfill"),
[
([], False),
(["--backfill"], True),
],
)
def test_each_tm_brand_is_injected_into_its_own_child_profile(
monkeypatch,
tmp_path: Path,
extra_args: list[str],
expects_backfill: bool,
) -> None:
profile_root = tmp_path / "tm-profile"
environments: dict[str, dict[str, str]] = {}
ports: dict[str, str] = {}
launched_ports: list[int] = []
child_scripts: list[str] = []
child_commands: dict[str, list[str]] = {}
binding_scripts: list[str] = []
monkeypatch.setattr(
collect,
"get_tm_style_id_map",
lambda: {"极星pro": ["main-id"], "布谷": ["ozko-id"]},
)
monkeypatch.setattr(
collect,
"get_tm_brand_map",
lambda: {"极星pro": "光影行星", "布谷": "ozko"},
)
monkeypatch.setattr(
collect,
"get_daily_styles",
lambda: {
"tm": {
"collectable": ["极星pro", "布谷"],
"skipped": [],
}
},
)
def fake_child_environment(script, *_args):
binding_scripts.append(Path(script).name)
return {
"GYXX_BROWSER_PROFILE_DIR": str(profile_root),
"GYXX_BROWSER_CDP_PORT": "22096",
}
monkeypatch.setattr(collect, "environment_for_child_script", fake_child_environment)
monkeypatch.setattr(collect, "_kill_port_listeners", lambda _port: 0)
monkeypatch.setattr(
collect,
"launch_chrome_with_debug",
lambda *_args, **kwargs: launched_ports.append(kwargs["debug_port"]) or True,
)
def fake_run(command, **kwargs):
brand = command[command.index("--brand") + 1]
environments[brand] = kwargs["env"]
ports[brand] = kwargs["env"]["GYXX_BROWSER_CDP_PORT"]
child_scripts.append(Path(command[1]).name)
child_commands[brand] = command
return SimpleNamespace(returncode=0)
monkeypatch.setattr(collect.subprocess, "run", fake_run)
monkeypatch.setattr(
sys,
"argv",
[
"taobao_sycm_collect.py",
"--target-date",
"2026-08-06",
*extra_args,
],
)
assert collect.main() == 0
assert child_scripts == ["taobao_sycm_products.py", "taobao_sycm_products.py"]
assert binding_scripts == ["taobao_sycm_products.py"]
for brand, expected_profile in {
"光影行星": profile_root,
"ozko": profile_root.parent / f"{profile_root.name}_ozko_tmall-ozko",
}.items():
command = child_commands[brand]
assert ("--backfill" in command) is expects_backfill
assert command[command.index("--cdp-url") + 1] == (
"http://127.0.0.1:" + ports[brand]
)
assert command[command.index("--brand-user-data-dir") + 1] == str(
expected_profile
)
assert environments["光影行星"]["GYXX_BROWSER_PROFILE_DIR"] == str(
profile_root
)
assert environments["光影行星"]["TM_USER_DATA_DIR"] == str(profile_root)
assert environments["ozko"]["GYXX_BROWSER_PROFILE_DIR"] == str(
profile_root.parent / f"{profile_root.name}_ozko_tmall-ozko"
)
assert environments["ozko"]["TM_USER_DATA_DIR"] == str(
profile_root.parent / f"{profile_root.name}_ozko_tmall-ozko"
)
assert ports["光影行星"] != ports["ozko"]
assert sorted(launched_ports) == [22196, 22197]
def test_cookie_recovery_marks_all_tm_brands_for_relogin(monkeypatch, tmp_path: Path) -> None:
calls: list[str] = []
monkeypatch.setattr(collect, "get_tm_style_id_map", lambda: {"a": ["1"], "b": ["2"]})
monkeypatch.setattr(collect, "get_tm_brand_map", lambda: {"a": "ozko", "b": "光影行星"})
monkeypatch.setattr(
collect,
"get_daily_styles",
lambda: {"tm": {"collectable": ["a", "b"], "skipped": []}},
)
monkeypatch.setattr(
collect,
"environment_for_child_script",
lambda *_args: {
"GYXX_BROWSER_PROFILE_DIR": str(tmp_path / "profile"),
"GYXX_BROWSER_CDP_PORT": "22096",
},
)
def fake_run(command, **_kwargs):
calls.append(command[command.index("--brand") + 1])
return SimpleNamespace(returncode=COOKIE_SKIP_EXIT_CODE)
monkeypatch.setattr(collect.subprocess, "run", fake_run)
assert collect.main(["--target-date", "2026-08-11", "--backfill"]) == COOKIE_SKIP_EXIT_CODE
assert sorted(calls) == ["ozko", "光影行星"]
def test_transient_brand_child_failure_is_retried_once(monkeypatch, tmp_path: Path) -> None:
calls: list[str] = []
monkeypatch.setattr(collect, "get_tm_style_id_map", lambda: {"a": ["1"]})
monkeypatch.setattr(collect, "get_tm_brand_map", lambda: {"a": "ozko"})
monkeypatch.setattr(
collect,
"get_daily_styles",
lambda: {"tm": {"collectable": ["a"], "skipped": []}},
)
monkeypatch.setattr(
collect,
"environment_for_child_script",
lambda *_args: {
"GYXX_BROWSER_PROFILE_DIR": str(tmp_path / "profile"),
"GYXX_BROWSER_CDP_PORT": "22096",
},
)
monkeypatch.setattr(collect, "_kill_port_listeners", lambda _port: 0)
def fake_run(command, **_kwargs):
calls.append(command[command.index("--brand") + 1])
return SimpleNamespace(returncode=1 if len(calls) == 1 else 0)
monkeypatch.setattr(collect.subprocess, "run", fake_run)
assert collect.main(["--target-date", "2026-08-11"]) == 0
assert calls == ["ozko", "ozko"]
def test_account_routed_brand_uses_a_fresh_collection_profile(monkeypatch, tmp_path: Path) -> None:
profiles: list[str] = []
monkeypatch.setattr(collect, "get_tm_style_id_map", lambda: {"布谷": ["1"]})
monkeypatch.setattr(collect, "get_tm_brand_map", lambda: {"布谷": "ozko"})
monkeypatch.setattr(
collect,
"get_daily_styles",
lambda: {"tm": {"collectable": ["布谷"], "skipped": []}},
)
monkeypatch.setattr(
collect,
"environment_for_child_script",
lambda *_args: {
"GYXX_PROJECT_ROOT": "D:/project",
"GYXX_DATA_ROOT": "D:/data",
"GYXX_BROWSER_PROFILE_DIR": str(tmp_path / "legacy-profile"),
"GYXX_BROWSER_CDP_PORT": "22096",
"GYXX_BROWSER_CDP_URL": "http://127.0.0.1:22096",
},
)
monkeypatch.setattr(
collect,
"_account_environment",
lambda account_id, environment: {
**environment,
"GYXX_ACCOUNT_ID": account_id,
"GYXX_ACCOUNT_PROFILE_DIR": str(tmp_path / "account" / "profile"),
"GYXX_ACCOUNT_CDP_PORT": "22097",
"GYXX_ACCOUNT_CDP_URL": "http://127.0.0.1:22097",
"GYXX_ACCOUNT_COLLECTION_CDP_PORT": "22143",
"GYXX_ACCOUNT_COLLECTION_CDP_URL": "http://127.0.0.1:22143",
"GYXX_ACCOUNT_COOKIE_FILE": "vault-cookies.json",
"GYXX_ACCOUNT_STORAGE_STATE_FILE": "vault-state.json",
"GYXX_ACCOUNT_REQUIRED_COOKIE_DOMAINS": "[\"taobao.com\"]",
"GYXX_ACCOUNT_REQUIRED_COOKIE_NAMES": "[\"cookie2\"]",
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES": "[]",
"GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS": "[\"ozko\"]",
},
)
monkeypatch.setattr(collect, "_kill_port_listeners", lambda _port: 0)
def fake_run(command, **kwargs):
profiles.append(kwargs["env"]["TM_USER_DATA_DIR"])
assert kwargs["env"]["GYXX_BROWSER_CDP_PORT"] == "22143"
assert kwargs["env"]["GYXX_BROWSER_CDP_URL"] == (
"http://127.0.0.1:22143"
)
assert kwargs["env"]["AUTOFLOW_CDP_PORT"] == "22143"
assert kwargs["env"]["GYXX_ACCOUNT_CDP_PORT"] == "22097"
assert command[command.index("--cdp-url") + 1] == (
"http://127.0.0.1:22143"
)
return SimpleNamespace(returncode=0)
monkeypatch.setattr(collect.subprocess, "run", fake_run)
assert collect.main(["--target-date", "2026-08-11"]) == 0
assert len(profiles) == 1
assert Path(profiles[0]).parent == tmp_path / "account" / "collector-runs"
assert Path(profiles[0]).name.startswith("tmall-ozko-2026-08-11-")
@@ -4,6 +4,11 @@ from pathlib import Path
import taobao_sycm_products as tm
from gyxx_flow.core.exit_codes import (
COOKIE_SKIP_EXIT_CODE,
NON_RETRYABLE_EXIT_CODE,
)
class _Session:
def __init__(self) -> None:
@@ -14,12 +19,171 @@ class _Session:
self.closed = True
class _LoginPage:
url = "https://sycm.taobao.com/cc/item_rank"
def __init__(self, state: dict[str, bool]) -> None:
self.state = state
def evaluate(self, *_args):
return self.state
def test_tm_login_state_requires_the_configured_shop(monkeypatch) -> None:
monkeypatch.setattr(tm, "ACCOUNT", "光影行星旗舰店:operator")
wrong_shop = _LoginPage(
{"hasStoreName": False, "hasTopNav": True, "hasAvatar": True}
)
requested_shop = _LoginPage(
{"hasStoreName": True, "hasTopNav": True, "hasAvatar": True}
)
assert tm.is_logged_in(wrong_shop) is False
assert tm.is_logged_in(requested_shop) is True
def test_tm_account_vault_replaces_stale_profile_cookies(monkeypatch, tmp_path: Path) -> None:
class Context:
def __init__(self) -> None:
self.cleared = False
def clear_cookies(self) -> None:
self.cleared = True
restored: list[tuple[str, str]] = []
context = Context()
cookie_file = tmp_path / "vault-cookies.json"
storage_state_file = tmp_path / "vault-state.json"
cookie_file.write_text("[]", encoding="utf-8")
storage_state_file.write_text("{}", encoding="utf-8")
monkeypatch.setenv("GYXX_ACCOUNT_ID", "tmall-ozko")
monkeypatch.setattr(
tm,
"_runtime_state_paths",
lambda: ((str(cookie_file), str(storage_state_file)), ("", "")),
)
monkeypatch.setattr(
tm,
"restore_browser_state",
lambda _context, *, cookie_file, storage_state_file: restored.append(
(cookie_file, storage_state_file)
),
)
tm._restore_runtime_account_state(context)
assert context.cleared is True
assert restored == [(str(cookie_file), str(storage_state_file))]
def test_tm_main_fails_when_browser_connection_is_missing(monkeypatch) -> None:
monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (None, None))
assert tm.main(["--target-date", "2026-08-03"]) == 1
def test_tm_backfill_delegates_inside_the_daily_browser_binding(monkeypatch) -> None:
import taobao_sycm_collect_backfill as backfill
delegated: list[str] = []
monkeypatch.setattr(
backfill,
"main",
lambda argv=None: delegated.extend(argv or []) or 0,
)
assert tm.main([
"--target-date",
"2026-08-03",
"--brand-user-data-dir",
"brand-profile",
"--brand",
"ozko",
"--backfill",
]) == 0
assert delegated[:4] == [
"--target-date",
"2026-08-03",
"--brand-user-data-dir",
"brand-profile",
]
assert delegated[-2:] == ["--brand", "ozko"]
def test_tm_backfill_returns_cookie_recovery_code_without_credentials(
monkeypatch,
) -> None:
import taobao_sycm_collect_backfill as backfill
session = _Session()
monkeypatch.setattr(
backfill,
"start_stealthy_session",
lambda *_args: (session, object()),
)
monkeypatch.setattr(
backfill,
"ensure_logged_in",
lambda *_args: (_ for _ in ()).throw(
RuntimeError("登录态失效且未配置 SYCM_ACCOUNT/SYCM_PASSWORD")
),
)
assert backfill.main(["--target-date", "2026-08-03"]) == COOKIE_SKIP_EXIT_CODE
assert session.closed is True
def test_tm_manual_login_waits_for_the_operator_and_closes_session(
monkeypatch,
) -> None:
session = _Session()
page = type("Page", (), {"goto": lambda *_args, **_kwargs: None})()
monkeypatch.setattr(
tm,
"connect_native_manual_login",
lambda _profile_dir: (session, page),
)
monkeypatch.setattr(tm, "is_logged_in", lambda _page: True)
assert tm.main([
"--brand-user-data-dir",
"ozko-profile",
"--manual-login",
"--login-timeout-seconds",
"1",
]) == 0
assert tm.USER_DATA_DIR == "ozko-profile"
assert session.closed is True
def test_tm_login_target_returns_to_the_product_page() -> None:
assert "_target=https%3A%2F%2Fsycm.taobao.com%2Fcc%2Fitem_rank" in tm.LOGIN_URL
assert "/custom/login" not in tm.LOGIN_URL.split("_target=", 1)[1]
def test_tm_native_manual_login_starts_with_one_blank_tab(monkeypatch) -> None:
launched: list[tuple[str, str]] = []
monkeypatch.setattr(
tm,
"launch_chrome_with_debug",
lambda profile, start_url=tm.LOGIN_URL: launched.append((profile, start_url))
or True,
)
monkeypatch.setattr(
tm,
"connect_browser",
lambda **kwargs: ("session", kwargs["cdp_url"]),
)
monkeypatch.setattr(tm, "DEBUG_PORT", 22096)
assert tm.connect_native_manual_login("ozko-profile") == (
"session",
"http://127.0.0.1:22096",
)
assert launched == [("ozko-profile", "about:blank")]
def test_tm_main_fails_when_login_does_not_complete(monkeypatch) -> None:
session = _Session()
page = object()
@@ -73,6 +237,9 @@ def test_tm_main_fails_when_download_has_no_valid_style(
monkeypatch.setattr(tm, "export_excel_to_json_and_md", lambda _path: (None, None))
monkeypatch.setattr(tm, "summarize_styles_from_excel", lambda _path: [])
monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0)
persisted: list[bool] = []
monkeypatch.setattr(tm, "_persist_runtime_account_state", lambda _context: persisted.append(True))
assert tm.main(["--target-date", "2026-08-03"]) == 1
assert tm.main(["--target-date", "2026-08-03"]) == NON_RETRYABLE_EXIT_CODE
assert session.closed is True
assert persisted == []
@@ -1,11 +1,73 @@
import unittest
from pathlib import Path
from unittest.mock import Mock
from unittest.mock import Mock, patch
import taobao_dmp_item_crowd_insight_screenshots as dmp
class TmPersonaRecoveryTests(unittest.TestCase):
def test_cdp_connection_uses_adapter_existing_context_mode(self):
page = Mock()
context = Mock()
context.new_page.return_value = page
session = Mock()
session.context = context
browser_factory = Mock()
browser_factory.return_value.start.return_value = session
with patch.object(dmp, "ScraplingBrowser", browser_factory):
actual_session, actual_page = dmp._connect_cdp()
browser_factory.assert_called_once_with(
stealthy=True,
reuse_existing_cdp_context=True,
cdp_url=dmp.CDP_URL,
timeout=60000,
network_idle=False,
disable_resources=False,
block_ads=False,
)
self.assertIs(actual_session, session)
self.assertIs(actual_page, page)
page.set_viewport_size.assert_called_once_with(
{"width": 1920, "height": 1080}
)
def test_item_insight_spa_route_is_authenticated_before_widgets_render(self):
page = Mock()
page.url = "https://dmp.taobao.com/index_new.html#!/items/item-insight"
with (
patch.object(dmp, "login_form_visible", return_value=False),
patch.object(dmp, "dmp_app_ready", return_value=False),
patch.object(dmp, "page_text", return_value=""),
):
self.assertTrue(dmp.is_logged_in(page))
def test_login_redirect_is_not_authenticated_when_spa_text_leaks(self):
page = Mock()
page.url = (
"https://dmp.taobao.com/login.html?mxredirectUrl="
"https%3A%2F%2Fdmp.taobao.com%2Findex_new.html%23!%2Fitems%2Fitem-insight"
)
with (
patch.object(dmp, "login_form_visible", return_value=False),
patch.object(dmp, "dmp_app_ready", return_value=False),
patch.object(dmp, "page_text", return_value="单品信息 单品人群洞察"),
):
self.assertFalse(dmp.is_logged_in(page))
def test_login_redirect_is_not_ready_when_spa_text_leaks(self):
page = Mock()
page.url = (
"https://dmp.taobao.com/login.html?mxredirectUrl="
"https%3A%2F%2Fdmp.taobao.com%2Findex_new.html%23!%2Fitems%2Fitem-insight"
)
self.assertFalse(dmp.dmp_app_ready(page))
page.evaluate.assert_not_called()
def test_zero_chart_skip_is_not_platform_success(self):
self.assertEqual(
dmp.records_exit_code(
@@ -0,0 +1,548 @@
from __future__ import annotations
import json
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
import pytest
import tmall_baibu_apply_common as baibu
def _entry() -> baibu.BaibuEntry:
return baibu.BaibuEntry(
entry_id="demo",
name="测试入口",
account_id="tmall-shop",
account_marker="测试店铺",
template_url="https://feishu.example/wiki/template",
apply_url="https://myseller.example/apply",
script="demo.py",
)
def test_latest_batch_import_status_parses_processing_and_complete_summary() -> None:
processing = baibu._latest_batch_import_status(
"您于2026-08-26 09:23:37执行了商品批量导入操作,正在处理中。"
)
assert processing is not None
assert processing.state == "processing"
completed = baibu._latest_batch_import_status(
"最近操作:您于2026-08-26 09:23:37执行了商品批量导入操作,"
"总数量20件,已成功20件,失败0件,待查看0件。"
)
assert completed is not None
assert completed.state == "success"
assert completed.detail == "总数量20件,已成功20件,失败0件,待查看0件"
def test_latest_batch_import_status_marks_partial_summary_as_failure() -> None:
result = baibu._latest_batch_import_status(
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
"总数量95件,已成功8件,失败87件,待查看0件。"
)
assert result is not None
assert result.state == "failure"
assert "失败87件" in result.detail
def test_credentials_are_loaded_from_account_declared_environment_names(
monkeypatch,
) -> None:
monkeypatch.setenv(
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
'["TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD"]',
)
monkeypatch.delenv("GYXX_BROWSER_CREDENTIAL_ENV_NAMES", raising=False)
monkeypatch.setenv("TMALL_BAG_ACCOUNT", "demo-account")
monkeypatch.setenv("TMALL_BAG_PASSWORD", "demo-password")
credentials, names = baibu._credentials_from_environment()
assert names == ("TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD")
assert credentials is not None
assert credentials.username == "demo-account"
assert credentials.password == "demo-password"
assert credentials.username_env == "TMALL_BAG_ACCOUNT"
assert credentials.password_env == "TMALL_BAG_PASSWORD"
def test_credentials_prefer_the_tmall_shop_pair_over_legacy_pairs(
monkeypatch,
) -> None:
monkeypatch.setenv(
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
'["TMALL_SHOP_ACCOUNT", "TMALL_SHOP_PASSWORD", '
'"SYCM_ACCOUNT", "SYCM_PASSWORD"]',
)
monkeypatch.setenv("TMALL_SHOP_ACCOUNT", "完整店铺名:子账号")
monkeypatch.setenv("TMALL_SHOP_PASSWORD", "shop-password")
monkeypatch.setenv("SYCM_ACCOUNT", "legacy-account")
monkeypatch.setenv("SYCM_PASSWORD", "legacy-password")
credentials, names = baibu._credentials_from_environment()
assert names == (
"TMALL_SHOP_ACCOUNT",
"TMALL_SHOP_PASSWORD",
"SYCM_ACCOUNT",
"SYCM_PASSWORD",
)
assert credentials is not None
assert credentials.username == "完整店铺名:子账号"
assert credentials.username_env == "TMALL_SHOP_ACCOUNT"
assert credentials.password_env == "TMALL_SHOP_PASSWORD"
class _LoginLocator:
def __init__(self, *, visible: bool = True) -> None:
self.visible = visible
self.filled: list[str] = []
self.click_count = 0
def count(self) -> int:
return 1 if self.visible else 0
def nth(self, _index: int):
return self
def is_visible(self) -> bool:
return self.visible
def is_enabled(self) -> bool:
return True
def fill(self, value: str) -> None:
self.filled.append(value)
def click(self, **_kwargs) -> None:
self.click_count += 1
class _LoginFrame:
def __init__(self) -> None:
self.username = _LoginLocator()
self.password = _LoginLocator()
self.submit = _LoginLocator()
def locator(self, selector: str):
return {
"input[placeholder='账号名/邮箱/手机号']": self.username,
"input[placeholder='请输入登录密码']": self.password,
"button[type='submit']": _LoginLocator(visible=False),
"input[type='submit']": _LoginLocator(visible=False),
}.get(selector, _LoginLocator(visible=False))
def get_by_role(self, role: str, *, name: str, exact: bool):
if role == "button" and name == "登录" and exact:
return self.submit
return _LoginLocator(visible=False)
def get_by_text(self, label: str, *, exact: bool):
return _LoginLocator(visible=False)
def test_login_if_needed_fills_the_password_form_inside_an_iframe(monkeypatch) -> None:
monkeypatch.setenv(
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
'["TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD"]',
)
monkeypatch.setenv("TMALL_BAG_ACCOUNT", "demo-account")
monkeypatch.setenv("TMALL_BAG_PASSWORD", "demo-password")
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
frame = _LoginFrame()
page = SimpleNamespace(
url="https://loginmyseller.taobao.com/?redirect_url=https%3A%2F%2Fqn.taobao.com",
frames=[frame],
)
attempted = baibu._login_if_needed(page=page, entry=_entry())
assert attempted is True
assert frame.username.filled == ["demo-account"]
assert frame.password.filled == ["demo-password"]
assert frame.submit.click_count == 1
def test_manual_login_waits_without_filling_the_password_form(monkeypatch) -> None:
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
frame = _LoginFrame()
page = SimpleNamespace(
url="https://loginmyseller.taobao.com/",
frames=[frame],
)
attempted = baibu._login_if_needed(
page=page,
entry=_entry(),
manual_login=True,
)
assert attempted is True
assert frame.username.filled == []
assert frame.password.filled == []
assert frame.submit.click_count == 0
def test_login_if_needed_reports_missing_credentials_without_filling(monkeypatch) -> None:
monkeypatch.setenv(
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
'["TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD"]',
)
monkeypatch.delenv("TMALL_BAG_ACCOUNT", raising=False)
monkeypatch.delenv("TMALL_BAG_PASSWORD", raising=False)
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
frame = _LoginFrame()
page = SimpleNamespace(
url="https://loginmyseller.taobao.com/",
frames=[frame],
)
try:
baibu._login_if_needed(page=page, entry=_entry())
except baibu.BaibuApplyError as exc:
assert "TMALL_BAG_ACCOUNT" in str(exc)
assert "TMALL_BAG_PASSWORD" in str(exc)
else:
raise AssertionError("missing credentials must stop before filling")
assert frame.username.filled == []
assert frame.password.filled == []
def test_wait_for_authenticated_stops_on_explicit_login_error(monkeypatch) -> None:
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
monkeypatch.setattr(baibu, "_page_text", lambda _page: "账密错误")
with pytest.raises(baibu.BaibuApplyError, match="账密错误"):
baibu._wait_for_authenticated(
_entry(),
SimpleNamespace(url="https://loginmyseller.taobao.com/"),
login_attempted=True,
)
def test_login_second_factor_marker_is_detected(monkeypatch) -> None:
monkeypatch.setattr(baibu, "_page_text", lambda _page: "你正在使用手机短信验证身份")
assert baibu._login_second_factor_marker(SimpleNamespace()) == "短信验证"
def test_persist_account_state_filters_cookies_to_the_account_domains(
monkeypatch,
tmp_path: Path,
) -> None:
account_cookie_file = tmp_path / "state" / "accounts" / "tmall-bag" / "cookies.json"
account_storage_file = (
tmp_path / "state" / "accounts" / "tmall-bag" / "storage_state.json"
)
monkeypatch.setenv("GYXX_ACCOUNT_COOKIE_FILE", str(account_cookie_file))
monkeypatch.setenv("GYXX_ACCOUNT_STORAGE_STATE_FILE", str(account_storage_file))
monkeypatch.setenv(
"GYXX_ACCOUNT_REQUIRED_COOKIE_DOMAINS",
'["taobao.com", "alimama.com"]',
)
monkeypatch.setenv(
"GYXX_ACCOUNT_REQUIRED_COOKIE_NAMES",
'["unb", "cookie2", "_tb_token_"]',
)
cookies = [
{"name": "unb", "domain": ".taobao.com", "value": "valid"},
{"name": "other", "domain": "example.com", "value": "unrelated"},
]
class Context:
def cookies(self):
return cookies
def storage_state(self):
return {
"cookies": cookies,
"origins": [
{"origin": "https://qn.taobao.com", "localStorage": []},
{"origin": "https://example.com", "localStorage": []},
],
}
browser = SimpleNamespace(context=Context())
assert baibu._persist_account_state(browser, _entry()) is True
saved_cookies = json.loads(account_cookie_file.read_text(encoding="utf-8"))
saved_state = json.loads(account_storage_file.read_text(encoding="utf-8"))
assert saved_cookies == [cookies[0]]
assert saved_state["cookies"] == [cookies[0]]
assert saved_state["origins"] == [
{"origin": "https://qn.taobao.com", "localStorage": []}
]
def test_upload_template_supports_tmall_dialog_import_button(
monkeypatch,
tmp_path: Path,
) -> None:
calls: list[tuple[str, ...]] = []
class Chooser:
def __init__(self) -> None:
self.files: str | None = None
def set_files(self, value: str) -> None:
self.files = value
chooser = Chooser()
class ExpectChooser:
def __enter__(self):
return SimpleNamespace(value=chooser)
def __exit__(self, *_args) -> None:
return None
monkeypatch.setattr(baibu, "_file_inputs", lambda _page: [])
def click(_page, labels):
labels = tuple(labels)
calls.append(labels)
return "导入表格" in labels
monkeypatch.setattr(baibu, "_click_text", click)
page = SimpleNamespace(
expect_file_chooser=lambda **_kwargs: ExpectChooser(),
)
template = tmp_path / "template.xlsx"
template.write_bytes(b"xlsx")
baibu._upload_template(page, template)
assert calls[0] == ("导入表格",)
assert chooser.files == str(template)
def test_open_batch_import_refuses_an_active_previous_operation(monkeypatch) -> None:
page = SimpleNamespace()
monkeypatch.setattr(
baibu,
"_page_text",
lambda _page: "您于2026-08-26 09:23:37执行了商品批量导入操作,正在处理中。",
)
monkeypatch.setattr(baibu, "_refresh_batch_results", lambda _page: True)
try:
baibu._open_batch_import(page, _entry())
except baibu.BaibuApplyError as exc:
assert "正在处理" in str(exc)
else:
raise AssertionError("active batch import must not be submitted again")
def test_open_batch_import_waits_for_hydrated_previous_summary(monkeypatch) -> None:
page = SimpleNamespace()
texts = iter(
[
"测试店铺 商品批量导入",
"测试店铺 最近操作:您于2026-08-26 16:53:04执行了商品批量导入操作,"
"总数量2件,已成功2件,失败0件,待查看0件。",
]
)
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
monkeypatch.setattr(
baibu,
"_click_text",
lambda _page, _labels: True,
)
previous = baibu._open_batch_import(page, _entry())
assert previous is not None
assert previous.state == "success"
def test_wait_import_result_refreshes_and_accepts_new_success_summary(
monkeypatch,
) -> None:
texts = iter(
[
"您于2026-08-26 16:53:04执行了商品批量导入操作,正在处理中。",
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
"总数量2件,已成功2件,失败0件,待查看0件。",
]
)
refreshes: list[bool] = []
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
monkeypatch.setattr(
baibu,
"_refresh_batch_results",
lambda _page: refreshes.append(True) or True,
)
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
monkeypatch.setattr(baibu, "_RESULT_REFRESH_INTERVAL_SECONDS", 0)
baibu._wait_import_result(SimpleNamespace(), _entry())
assert refreshes
def test_wait_import_result_accepts_new_failure_summary_as_submitted(
monkeypatch,
) -> None:
texts = iter(
[
"您于2026-08-26 16:53:04执行了商品批量导入操作,正在处理中。",
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
"总数量95件,已成功0件,失败95件,待查看0件。",
]
)
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
monkeypatch.setattr(baibu, "_refresh_batch_results", lambda _page: True)
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
monkeypatch.setattr(baibu, "_RESULT_REFRESH_INTERVAL_SECONDS", 0)
# Tmall's row-level result is informational; submitting the batch is the
# success condition for this workflow.
baibu._wait_import_result(SimpleNamespace(), _entry())
def test_wait_import_result_ignores_stale_summary_until_signature_changes(
monkeypatch,
) -> None:
previous = baibu._latest_batch_import_status(
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
"总数量95件,已成功8件,失败87件,待查看0件。"
)
assert previous is not None
texts = iter(
[
"最近操作:您于2026-08-26 16:53:04执行了商品批量导入操作,"
"总数量95件,已成功8件,失败87件,待查看0件。",
"您于2026-08-26 17:30:01执行了商品批量导入操作,正在处理中。",
"您于2026-08-26 17:30:01执行了商品批量导入操作,"
"总数量2件,已成功2件,失败0件,待查看0件。",
]
)
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
monkeypatch.setattr(baibu, "_refresh_batch_results", lambda _page: True)
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
monkeypatch.setattr(baibu, "_RESULT_REFRESH_INTERVAL_SECONDS", 0)
baibu._wait_import_result(
SimpleNamespace(),
_entry(),
previous_operation=previous,
)
def test_complete_draft_products_runs_required_sequence_until_count_is_zero(
monkeypatch,
) -> None:
entry = replace(_entry(), complete_drafts_after_import=True)
page = SimpleNamespace()
page_texts = iter(["草稿 (2)", "草稿 (1)", "草稿 (0)"])
events: list[str] = []
clicked: list[tuple[str, ...]] = []
opened: list[bool] = []
monkeypatch.setattr(
baibu,
"_click_draft_tab",
lambda _page: events.append("draft") or True,
)
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(page_texts))
monkeypatch.setattr(baibu, "_visible_text_count", lambda _page, _label: 1)
monkeypatch.setattr(baibu, "_has_visible_text", lambda _page, _labels: True)
monkeypatch.setattr(
baibu,
"_open_draft_editor",
lambda _page, _entry, *, first: events.append(
f"open:{first}"
) or opened.append(first),
)
monkeypatch.setattr(
baibu,
"_click_text_until_found",
lambda _page, labels, **_kwargs: events.append(next(iter(labels)))
or clicked.append(tuple(labels)),
)
baibu._complete_draft_products(page, entry)
assert opened == [True, False]
assert events == [
"draft",
"open:True",
"",
"下一步",
"draft",
"open:False",
"",
"下一步",
"draft",
]
assert clicked == [
("",),
("下一步",),
("",),
("下一步",),
]
def test_complete_draft_products_is_disabled_for_other_entries(monkeypatch) -> None:
monkeypatch.setattr(
baibu,
"_click_draft_tab",
lambda _page: pytest.fail("其他入口不得进入草稿处理流程"),
)
baibu._complete_draft_products(SimpleNamespace(), _entry())
def test_complete_draft_products_accepts_empty_data_page(monkeypatch) -> None:
entry = replace(_entry(), complete_drafts_after_import=True)
page = SimpleNamespace()
monkeypatch.setattr(baibu, "_click_draft_tab", lambda _page: True)
monkeypatch.setattr(baibu, "_page_text", lambda _page: "草稿 数据为空")
monkeypatch.setattr(baibu, "_visible_text_count", lambda _page, _label: 0)
baibu._complete_draft_products(page, entry)
def test_return_to_draft_list_falls_back_to_browser_history(monkeypatch) -> None:
events: list[str] = []
class Page:
def go_back(self, **_kwargs) -> None:
events.append("back")
monkeypatch.setattr(
baibu,
"_click_draft_tab",
lambda _page: events.append("draft") or False,
)
monkeypatch.setattr(baibu, "_close_editor", lambda _page: False)
monkeypatch.setattr(
baibu,
"_has_visible_text",
lambda _page, labels: "完善商品" in labels and "back" in events,
)
baibu._return_to_draft_list(Page(), _entry())
assert events == ["draft", "back"]
def test_tmall_config_enables_draft_completion_for_only_old_all_3c() -> None:
old_all_3c = baibu._load_entry("old_all_3c")
old_all_bag = baibu._load_entry("old_all_bag")
new_all_bag = baibu._load_entry("new_all_bag")
assert old_all_3c.complete_drafts_after_import is True
assert old_all_bag.complete_drafts_after_import is False
assert new_all_bag.complete_drafts_after_import is False
@@ -0,0 +1,572 @@
from __future__ import annotations
import sys
from collections import defaultdict
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
import upload_video_to_guanghe as tmall
def _record(
record_id: str,
attachments: list[dict[str, Any]],
*,
product_names: str = "极星双肩",
record_date: object = None,
) -> dict[str, Any]:
return {
"record_id": record_id,
"代拍产品": product_names,
"代拍附件": attachments,
"日期": record_date,
"品牌": ["光影行星"],
"性别": [""],
tmall.TM_UPLOAD_FIELD: False,
}
def _one_candidate(*, store: str = tmall.STORE_LUGGAGE):
candidates, issues = tmall.discover_tmall_publication_candidates(
[
_record(
"record-one",
[
{
"type": "file",
"name": "video.mp4",
"file_token": "video-token",
"size": 123,
}
],
)
],
store=store,
)
assert issues == []
return candidates[0]
def test_catalog_matching_accepts_variant_after_bag_stem() -> None:
assert tmall._match_tm_item_ids_from_product_titles(
["盖亚斜挎3"],
[
(
"1059764111499",
"GYXX/光影行星盖亚斜挎包3单反相机包男款微单摄影包",
)
],
) == {"盖亚斜挎3": ["1059764111499"]}
def test_catalog_search_pattern_keeps_variant_alias_rows_in_scope() -> None:
assert tmall._tm_product_title_search_pattern("盖亚斜挎3") == "%盖亚斜挎%"
def test_invalid_compatible_video_cache_is_rebuilt(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
source = tmp_path / "source.mov"
source.write_bytes(b"source")
cached = source.with_name("source_compatible.mp4")
cached.write_bytes(b"truncated-cache")
class _Reader:
def __init__(self, metadata: dict[str, Any]) -> None:
self.metadata = metadata
def __iter__(self) -> _Reader:
return self
def __next__(self) -> dict[str, Any]:
return self.metadata
def close(self) -> None:
return None
class _FakeImageioFfmpeg:
@staticmethod
def get_ffmpeg_exe() -> str:
return "ffmpeg"
@staticmethod
def read_frames(path: str) -> _Reader:
if path == str(cached):
raise OSError("moov atom not found")
return _Reader({"codec": "hevc", "source_size": (1920, 1080)})
monkeypatch.setitem(sys.modules, "imageio_ffmpeg", _FakeImageioFfmpeg)
def fake_run(command: list[str], **_kwargs: Any) -> SimpleNamespace:
Path(command[-1]).write_bytes(b"rebuilt")
return SimpleNamespace(returncode=0, stderr="", stdout="")
monkeypatch.setattr(tmall.subprocess, "run", fake_run)
assert tmall.ensure_upload_compatible_video(source) == cached
assert cached.read_bytes() == b"rebuilt"
def test_discovery_creates_one_candidate_per_exact_video_attachment() -> None:
candidates, issues = tmall.discover_tmall_publication_candidates(
[
_record(
"record-with-two-videos",
[
{
"type": "file",
"name": "same-name.mp4",
"file_token": "video-token-a",
"size": "101",
},
{
"type": "file",
"name": "same-name.mp4",
"file_token": "video-token-b",
"size": 202,
},
{
"type": "file",
"name": "cover.jpg",
"file_token": "image-token",
},
{
"type": "file",
"name": "generic-attachment",
"file_token": "generic-file-token",
},
],
record_date=None,
),
_record(
"record-marked-skip",
[
{
"type": "file",
"name": "must-not-upload.mp4",
"file_token": "skipped-token",
}
],
product_names="极星双肩 / 未到货",
),
],
store=tmall.STORE_LUGGAGE,
)
assert {
(candidate.source_record_id, candidate.video_file_token)
for candidate in candidates
} == {
("record-with-two-videos", "video-token-a"),
("record-with-two-videos", "video-token-b"),
}
assert {candidate.video_file_name for candidate in candidates} == {
"same-name.mp4"
}
assert all(candidate.source_record_date is None for candidate in candidates)
assert [
(issue.source_record_id, issue.reason) for issue in issues
] == [("record-marked-skip", "代拍产品包含未到货/不发标记")]
def test_flagship_and_luggage_use_distinct_account_scopes() -> None:
flagship = _one_candidate(store=tmall.STORE_FLAGSHIP)
luggage = _one_candidate(store=tmall.STORE_LUGGAGE)
assert flagship.target_account_key == "guanghe-flagship"
assert luggage.target_account_key == "guanghe-luggage"
assert flagship.target_account_key != luggage.target_account_key
assert tmall.STORE_ACCOUNT_KEYS == {
tmall.STORE_FLAGSHIP: flagship.target_account_key,
tmall.STORE_LUGGAGE: luggage.target_account_key,
}
def test_luggage_runtime_uses_new_store_account_label_without_changing_password_key(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("GUANGHE_LUGGAGE_USERNAME", "光影行星鑫华达专卖店:龙虾仔")
monkeypatch.setenv("GUANGHE_LUGGAGE_PASSWORD", "same-runtime-secret")
store_name, username, password, _profile = tmall._store_runtime(tmall.STORE_LUGGAGE)
assert store_name == "光影行星鑫华达专卖店:龙虾仔"
assert username == "光影行星鑫华达专卖店:龙虾仔"
assert password == "same-runtime-secret"
def test_account_identity_markers_accept_store_name_with_login_nickname() -> None:
assert tmall.GuangHeUploader._account_identity_markers(
"光影行星鑫华达专卖店:龙虾仔"
) == (
"光影行星鑫华达专卖店:龙虾仔",
"光影行星鑫华达专卖店",
)
def test_flagship_creator_identity_uses_visible_name_and_account_number() -> None:
assert tmall._expected_creator_identity_markers(
tmall.STORE_FLAGSHIP,
"光影行星旗舰店",
) == ("光影行星天猫官方", "逛逛号:3301283916")
def test_creator_identity_requires_every_configured_marker() -> None:
class Body:
def __init__(self, text: str) -> None:
self.text = text
def inner_text(self, **_kwargs: object) -> str:
return self.text
class Page:
def __init__(self, text: str) -> None:
self.text = text
def locator(self, selector: str) -> Body:
assert selector == "body"
return Body(self.text)
uploader = object.__new__(tmall.GuangHeUploader)
uploader.expected_account_markers = (
"光影行星天猫官方",
"逛逛号:3301283916",
)
uploader.page = Page("光影行星天猫官方\n逛逛号:3301283916\n账号正常")
assert uploader._page_matches_expected_account()
uploader.page = Page("光影行星天猫官方\n账号正常")
assert not uploader._page_matches_expected_account()
def test_wait_for_existing_login_allows_account_card_to_render(
monkeypatch: pytest.MonkeyPatch,
) -> None:
uploader = object.__new__(tmall.GuangHeUploader)
states = iter((False, True))
uploader._sync_page = lambda: None
uploader._is_logged_in = lambda: next(states)
monkeypatch.setattr(tmall.time, "monotonic", lambda: 0.0)
monkeypatch.setattr(tmall.time, "sleep", lambda _seconds: None)
assert uploader._wait_for_existing_login(timeout_seconds=8)
def test_legacy_backfill_has_25_exact_attachment_matches_and_fails_on_drift() -> None:
entries = tmall.load_legacy_publication_backfill()
assert len(entries) == 25
attachments_by_record: dict[str, list[dict[str, Any]]] = defaultdict(list)
for entry in entries:
attachments_by_record[entry["source_record_id"]].append(
{
"type": "file",
"name": entry["video_file_name"],
"file_token": entry["video_file_token"],
}
)
records = [
_record(record_id, attachments)
for record_id, attachments in attachments_by_record.items()
]
candidates, issues = tmall.discover_tmall_publication_candidates(
records,
store=tmall.STORE_LUGGAGE,
)
assert issues == []
matched = tmall._legacy_backfill_for_candidates(
tmall.STORE_LUGGAGE,
candidates,
)
assert len(matched) == 25
assert {
(
candidate.source_record_id,
candidate.video_file_token,
candidate.video_file_name,
)
for candidate, _entry in matched
} == {
(
entry["source_record_id"],
entry["video_file_token"],
entry["video_file_name"],
)
for entry in entries
}
drifted = [
replace(candidates[0], video_file_token="drifted-token"),
*candidates[1:],
]
with pytest.raises(RuntimeError, match="record_id/file_token/文件名不一致"):
tmall._legacy_backfill_for_candidates(
tmall.STORE_LUGGAGE,
drifted,
)
def test_parse_args_is_preview_by_default_and_requires_explicit_execute(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("GYXX_VIDEO_UPLOAD_STORE", raising=False)
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py"])
preview = tmall.parse_args()
assert preview.execute is False
assert preview.store == tmall.STORE_FLAGSHIP
monkeypatch.setattr(
sys,
"argv",
["upload_video_to_guanghe.py", "--execute"],
)
execution = tmall.parse_args()
assert execution.execute is True
def test_parse_args_uses_frontend_selected_store_from_runtime_environment(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("GYXX_VIDEO_UPLOAD_STORE", tmall.STORE_LUGGAGE)
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py", "--execute"])
execution = tmall.parse_args()
assert execution.store == tmall.STORE_LUGGAGE
def test_parse_args_uses_configured_topic_keyword_and_removes_spacing(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv(tmall.TMALL_TOPIC_KEYWORD_ENV, " 秋 上新 ")
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py"])
execution = tmall.parse_args()
assert execution.topic_keyword == "秋上新"
def test_parse_args_rejects_invalid_runtime_store(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("GYXX_VIDEO_UPLOAD_STORE", "unknown")
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py"])
with pytest.raises(ValueError, match="GYXX_VIDEO_UPLOAD_STORE"):
tmall.parse_args()
def test_tmall_pre_submit_failures_are_automatically_retryable() -> None:
assert tmall._is_retryable_pre_submit_failure(
tmall.TmallTitleGenerationError("no valid candidate"),
1,
)
assert tmall._is_retryable_pre_submit_failure(RuntimeError("timed out"), 1)
assert not tmall._is_retryable_pre_submit_failure(RuntimeError("timed out"), 3)
def test_tmall_strict_title_requests_a_candidate_pool_and_classifies_validation(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: dict[str, object] = {}
class ValidationGenerator:
def __init__(self, **kwargs: object) -> None:
captured.update(kwargs)
def generate(self, *_args: object, **_kwargs: object) -> str:
raise tmall.TitleValidationError("no valid candidate")
monkeypatch.setattr(tmall, "JdVideoTitleGenerator", ValidationGenerator)
uploader = object.__new__(tmall.GuangHeUploader)
uploader.require_llm_titles = True
uploader._used_titles_global = []
uploader._catalog_product_titles = lambda _name: ("极星双肩包轻便电脑包",)
with pytest.raises(tmall.TmallTitleGenerationError, match="no valid candidate"):
uploader._generate_title(["极星双肩"], video_key="video-token")
assert captured["candidate_count"] == tmall.TMALL_TITLE_CANDIDATE_COUNT
assert int(captured["candidate_count"]) > 1
assert captured["allow_verified_fallback"] is True
def test_tmall_strict_title_keeps_model_request_failures_retryable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class RequestGenerator:
def __init__(self, **_kwargs: object) -> None:
pass
def generate(self, *_args: object, **_kwargs: object) -> str:
raise tmall.TitleGenerationError("timed out")
monkeypatch.setattr(tmall, "JdVideoTitleGenerator", RequestGenerator)
uploader = object.__new__(tmall.GuangHeUploader)
uploader.require_llm_titles = True
uploader._used_titles_global = []
uploader._catalog_product_titles = lambda _name: ("极星双肩包轻便电脑包",)
with pytest.raises(tmall.TitleGenerationError, match="timed out"):
uploader._generate_title(["极星双肩"], video_key="video-token")
def test_execute_with_no_selected_store_candidates_does_not_start_browser(
monkeypatch: pytest.MonkeyPatch,
) -> None:
args = SimpleNamespace(
execute=True,
republish_from_date=None,
video_name=None,
exclude_video_name=[],
record_id=None,
)
monkeypatch.setattr(
tmall,
"_store_runtime",
lambda _store: ("光影行星鑫华达专卖店:龙虾仔", "configured", "configured", Path("profile")),
)
monkeypatch.setattr(tmall, "select_pending_records", lambda *_args, **_kwargs: [])
monkeypatch.setattr(
tmall,
"discover_tmall_publication_candidates",
lambda *_args, **_kwargs: ([], []),
)
monkeypatch.setattr(
tmall,
"_legacy_backfill_for_candidates",
lambda *_args, **_kwargs: [],
)
class UnexpectedUploader:
def __init__(self, *_args: Any, **_kwargs: Any) -> None:
raise AssertionError("筛选为空时不应启动浏览器")
monkeypatch.setattr(tmall, "GuangHeUploader", UnexpectedUploader)
result = tmall._run_store(
args,
store=tmall.STORE_LUGGAGE,
records=[],
luggage_style_whitelist={"极星双肩"},
)
assert result == tmall.EXIT_OK
class _FakeLocator:
def __init__(
self,
*,
count: int = 1,
on_click: Any = None,
) -> None:
self._count = count
self._on_click = on_click
@property
def first(self) -> _FakeLocator:
return self
def count(self) -> int:
return self._count
def fill(self, _value: str) -> None:
return None
def click(self, **_kwargs: Any) -> None:
if self._on_click is not None:
self._on_click()
def test_publish_video_persists_submit_boundary_before_publish_click(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
events: list[str] = []
upload_timeouts: list[int] = []
uploader = object.__new__(tmall.GuangHeUploader)
uploader.page = SimpleNamespace(url="https://creator.guanghe.taobao.com/page/")
uploader._upload_video_file = lambda _path: None
uploader._wait_upload_done = lambda **kwargs: (upload_timeouts.append(kwargs["timeout"]) or True)
uploader._snapshot = lambda _name: None
uploader._generate_title = lambda *_args, **_kwargs: "极星双肩包视频标题"
uploader._prepare_publish_metadata = lambda _products, **_kwargs: None
uploader._link_product = lambda _products, _item_ids: True
def locate(selector: str) -> _FakeLocator:
if selector == "text=立即发布":
return _FakeLocator(on_click=lambda: events.append("publish-click"))
if selector == "text=发布成功":
return _FakeLocator(count=1)
if selector.startswith("text=") and selector not in {
"text=内容无需标注",
}:
return _FakeLocator(count=0)
return _FakeLocator(count=1)
uploader._loc = locate
monkeypatch.setattr(tmall.time, "sleep", lambda _seconds: None)
def before_submit(payload: Any) -> None:
assert payload["video_file_name"] == "video.mp4"
events.append("before-submit")
published = uploader.publish_video(
tmp_path / "video.mp4",
["极星双肩"],
before_submit=before_submit,
)
assert published is True
assert events == ["before-submit", "publish-click"]
assert upload_timeouts == [tmall.TMALL_VIDEO_UPLOAD_TIMEOUT_SECONDS]
class _SeedCursor:
def __init__(self) -> None:
self.executions: list[tuple[str, tuple[Any, ...]]] = []
def __enter__(self) -> _SeedCursor:
return self
def __exit__(self, *_args: object) -> None:
return None
def execute(self, sql: str, params: tuple[Any, ...]) -> None:
self.executions.append((sql, params))
def fetchone(self) -> tuple[int]:
return (1,)
class _SeedConnection:
def __init__(self) -> None:
self.cursor_instance = _SeedCursor()
def cursor(self) -> _SeedCursor:
return self.cursor_instance
def test_legacy_seed_inserts_published_without_overwriting_conflicts() -> None:
connection = _SeedConnection()
candidate = _one_candidate()
inserted = tmall.seed_legacy_published_candidates(
connection,
[(candidate, {"evidence": {"publish_success_line": 72}})],
run_id="test-run",
)
assert inserted == 1
sql, params = connection.cursor_instance.executions[0]
normalized_sql = " ".join(sql.split())
assert "'published'" in normalized_sql
assert "ON CONFLICT (" in normalized_sql
assert ") DO NOTHING RETURNING id" in normalized_sql
assert params[-3:-1] == ("test-run", "test-run")
@@ -0,0 +1,163 @@
from __future__ import annotations
import inspect
from datetime import date
from pathlib import Path
from typing import get_type_hints
import pytest
from gyxx_flow.modules.product_commerce.jd_video_publication_db import (
CHANNEL_TMALL,
VIDEO_PUBLICATION_DDL,
ClaimedVideoPublication,
VideoPublicationCandidate,
VideoPublicationState,
claim_next_video_publication,
finish_video_publication,
finish_video_publication_rejected,
load_video_publication_states,
)
class FakeCursor:
def __init__(self, rows: list[tuple[object, ...]]) -> None:
self.rows = rows
self.executions: list[tuple[str, tuple[object, ...]]] = []
self.rowcount = 1
def __enter__(self) -> FakeCursor:
return self
def __exit__(self, *_args: object) -> None:
return None
def execute(self, query: str, params: tuple[object, ...]) -> None:
self.executions.append((query, params))
def fetchall(self) -> list[tuple[object, ...]]:
return self.rows
class FakeConnection:
def __init__(self, cursor: FakeCursor) -> None:
self.fake_cursor = cursor
def cursor(self) -> FakeCursor:
return self.fake_cursor
def test_load_video_publication_states_scopes_query_and_maps_attachments() -> None:
cursor = FakeCursor(
[
("rec_1", "video_1", 101, "published", False),
("rec_2", "video_2", 102, "failed", True),
]
)
states = load_video_publication_states(
FakeConnection(cursor),
CHANNEL_TMALL,
"tmall-luggage-store",
"base_token",
"table_id",
)
assert cursor.executions[0][1] == (
"tmall",
"tmall-luggage-store",
"base_token",
"table_id",
)
assert "FROM fact_video_publications" in cursor.executions[0][0]
assert states == {
("rec_1", "video_1"): VideoPublicationState(101, "published", False),
("rec_2", "video_2"): VideoPublicationState(102, "failed", True),
}
def test_load_video_publication_states_does_not_hide_database_errors() -> None:
class MissingTableCursor(FakeCursor):
def execute(self, query: str, params: tuple[object, ...]) -> None:
raise RuntimeError("fact_video_publications does not exist")
with pytest.raises(RuntimeError, match="does not exist"):
load_video_publication_states(
FakeConnection(MissingTableCursor([])),
CHANNEL_TMALL,
"tmall-flagship-store",
"base_token",
"table_id",
)
def test_shared_ledger_preserves_missing_source_dates() -> None:
assert get_type_hints(VideoPublicationCandidate)["source_record_date"] == date | None
assert get_type_hints(ClaimedVideoPublication)["source_record_date"] == date | None
assert "source_record_date DATE," in VIDEO_PUBLICATION_DDL
assert "ALTER COLUMN source_record_date DROP NOT NULL;" in VIDEO_PUBLICATION_DDL
claim_source = inspect.getsource(claim_next_video_publication)
assert "source_snapshot ->> 'source_view_position'" in claim_source
assert "source_record_date NULLS LAST" in claim_source
schema_path = (
Path(__file__).resolve().parents[3]
/ "src"
/ "gyxx_flow"
/ "modules"
/ "product_commerce"
/ "db"
/ "schema.sql"
)
schema = schema_path.read_text(encoding="utf-8")
assert "source_record_date DATE," in schema
assert "ALTER COLUMN source_record_date DROP NOT NULL;" in schema
def _sample_claim() -> ClaimedVideoPublication:
return ClaimedVideoPublication(
publication_id=764,
attempt_no=1,
source_record_id="rec_1",
source_record_date=None,
video_file_token="video_1",
video_file_name="video.mp4",
cover_file_token=None,
cover_file_name=None,
source_snapshot={},
publish_payload={},
)
def test_published_and_ambiguous_outcomes_are_not_retryable() -> None:
published_cursor = FakeCursor([])
finish_video_publication(
FakeConnection(published_cursor),
_sample_claim(),
status="published",
retryable=True,
)
assert published_cursor.executions[0][1][0:2] == ("published", False)
ambiguous_cursor = FakeCursor([])
finish_video_publication(
FakeConnection(ambiguous_cursor),
_sample_claim(),
status="ambiguous",
retryable=True,
)
assert ambiguous_cursor.executions[0][1][0:2] == ("ambiguous", False)
def test_clear_remote_rejection_is_retryable_but_keeps_submit_audit() -> None:
cursor = FakeCursor([])
finish_video_publication_rejected(
FakeConnection(cursor),
_sample_claim(),
error="平台明确拒绝",
evidence={"form_visible": True},
)
params = cursor.executions[0][1]
assert params[0:2] == ("failed", True)
assert params[3] is True
@@ -14,21 +14,17 @@ from gyxx_flow.modules.shop_intelligence.collectors import (
)
class _PlaywrightContext:
class _ScraplingBrowserContext:
def __init__(self, page: object) -> None:
self.page = page
def __enter__(self):
context = SimpleNamespace(
pages=[self.page],
new_page=lambda: self.page,
close=lambda: None,
)
browser = SimpleNamespace(
new_context=lambda **kwargs: context,
close=lambda: None,
)
chromium = SimpleNamespace(launch=lambda **kwargs: browser)
return SimpleNamespace(chromium=chromium)
return SimpleNamespace(context=context)
def __exit__(self, exc_type, exc, traceback) -> None:
return None
@@ -39,8 +35,8 @@ def test_jd_shop_collector_returns_nonzero_when_collection_fails(
) -> None:
monkeypatch.setattr(
jd_data_collector,
"sync_playwright",
lambda: _PlaywrightContext(object()),
"ScraplingBrowser",
lambda **_kwargs: _ScraplingBrowserContext(object()),
)
monkeypatch.setattr(
jd_data_collector,
@@ -88,8 +84,8 @@ def test_jd_peer_collector_returns_nonzero_when_collection_fails(
) -> None:
monkeypatch.setattr(
jd_peer_store_data_collector,
"sync_playwright",
lambda: _PlaywrightContext(object()),
"ScraplingBrowser",
lambda **_kwargs: _ScraplingBrowserContext(object()),
)
monkeypatch.setattr(
jd_peer_store_data_collector,
@@ -254,6 +254,53 @@ class _LateShopIdentityPage:
self.wait_calls += 1
class _RetryNavigationPage(_LateShopIdentityPage):
def __init__(self, *, failures: int, url: str | None = None) -> None:
super().__init__()
self.failures = failures
self.goto_calls = 0
if url is not None:
self.url = url
def goto(self, *args: object, **kwargs: object) -> None:
del args, kwargs
self.goto_calls += 1
if self.goto_calls <= self.failures:
raise RuntimeError(
"Page.goto: net::ERR_ABORTED at "
f"{appeal.CAMPAIGN_SQUARE_URL} Call log: - navigating"
)
class _FeedbackTextControl:
def __init__(self, text: str) -> None:
self.text_value = text
def count(self) -> int:
return 1
def nth(self, index: int) -> _FeedbackTextControl:
assert index == 0
return self
def is_visible(self, timeout: int | None = None) -> bool:
del timeout
return True
def inner_text(self, timeout: int | None = None) -> str:
del timeout
return self.text_value
class _FeedbackPage:
def __init__(self, items: list[_VisibleControl]) -> None:
self.items = items
def locator(self, selector: str) -> _LocatorCollection:
assert "toast" in selector
return _LocatorCollection(self.items)
class _FakeClock:
def __init__(self) -> None:
self.now = 0.0
@@ -319,6 +366,38 @@ class _DynamicPanel:
return _EmptyLocator()
class _TransientTextPanel(_DynamicPanel):
def __init__(
self,
values: tuple[str, ...],
*,
transient_failures: int,
) -> None:
super().__init__(values, action_count=0)
self.transient_failures = transient_failures
def inner_text(self, timeout: int | None = None) -> str:
del timeout
self.read_calls += 1
if self.transient_failures > 0:
self.transient_failures -= 1
raise RuntimeError("drawer locator detached during transition")
return next(self.values, self.latest)
class _DetachedFeedbackControl(_TransientTextPanel):
def count(self) -> int:
return 1
def nth(self, index: int) -> _DetachedFeedbackControl:
assert index == 0
return self
def is_visible(self, timeout: int | None = None) -> bool:
del timeout
return True
class _PanelNode(_VisibleControl):
def __init__(self, name: str, box: dict[str, float]) -> None:
super().__init__()
@@ -465,6 +544,90 @@ def test_shop_identity_can_render_after_campaign_content(
assert page.wait_calls == 1
def test_navigation_recovers_after_first_abort(
monkeypatch: pytest.MonkeyPatch,
) -> None:
page = _RetryNavigationPage(failures=1)
monkeypatch.setattr(
appeal,
"_body_text",
lambda page: "活动广场 待办事项 待改价 9 光影行星箱包旗舰店",
)
appeal._navigate_to_campaign_square( # noqa: SLF001
page,
expected_shop_name=appeal.EXPECTED_SHOP_NAME,
)
assert page.goto_calls == 2
assert page.wait_calls == 1
def test_navigation_aborted_to_login_page_reports_session_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
page = _RetryNavigationPage(
failures=2,
url=(
"https://fxg.jinritemai.com/login/common"
"?extra=%7B%22target_url%22%3A%22https%3A%2F%2F"
"fxg.jinritemai.com%2Fffa%2Fmerchant%2Fcampaign-square%22%7D"
),
)
monkeypatch.setattr(appeal, "_body_text", lambda page: "")
with pytest.raises(appeal.PriceAppealSessionError, match="Cookie 已失效"):
appeal._navigate_to_campaign_square( # noqa: SLF001
page,
expected_shop_name=appeal.EXPECTED_SHOP_NAME,
)
assert page.goto_calls == 2
def test_navigation_aborted_twice_reports_navigation_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
page = _RetryNavigationPage(failures=2)
monkeypatch.setattr(appeal, "_body_text", lambda page: "")
with pytest.raises(appeal.PriceAppealError, match="活动广场导航失败") as exc:
appeal._navigate_to_campaign_square( # noqa: SLF001
page,
expected_shop_name=appeal.EXPECTED_SHOP_NAME,
)
assert page.goto_calls == 2
assert "ERR_ABORTED" in str(exc.value)
def test_pending_content_signature_returns_none_while_drawer_detached(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
panel = _TransientTextPanel(
("待改价 报名信息 商品 ID: 12345678",),
transient_failures=100,
)
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
assert appeal._pending_content_signature(panel) is None # noqa: SLF001
def test_visible_feedback_texts_skips_detached_feedback_node(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
detached = _DetachedFeedbackControl(("提交成功",), transient_failures=100)
healthy = _FeedbackTextControl(" 提交成功 ")
page = _FeedbackPage([detached, healthy])
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
assert appeal._visible_feedback_texts(page) == ("提交成功",)
def test_pending_count_can_render_after_campaign_content(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -801,6 +964,111 @@ def test_item_panel_waits_for_async_product_and_sku_content(
assert page.wait_calls == 2
def test_item_panel_reacquires_after_transient_drawer_detach(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
page = _PendingMetricPage([], clock=clock)
expected_item_id = "3774363265803616739"
detached_panel = _TransientTextPanel(
("选择申诉SKU",),
transient_failures=1,
)
ready_panel = _DynamicPanel(
(
"选择申诉SKU 商品 ID: 3774363265803616739 "
"SKU规格 ID: 3619462493048834 原价 ¥839",
),
action_count=0,
)
panels = iter((detached_panel, ready_panel))
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False)
monkeypatch.setattr(
appeal,
"_find_active_panel",
lambda page, markers: next(panels, ready_panel),
)
result = appeal._wait_for_item_panel_ready( # noqa: SLF001
page,
("选择申诉SKU",),
expected_item_id=expected_item_id,
timeout_seconds=2,
)
assert result is ready_panel
assert detached_panel.read_calls == 2
assert ready_panel.read_calls == 1
assert page.wait_calls == 1
def test_item_panel_persistent_read_failure_still_fails_closed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
page = _PendingMetricPage([], clock=clock)
panel = _TransientTextPanel(
("选择申诉SKU",),
transient_failures=100,
)
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False)
monkeypatch.setattr(
appeal,
"_find_active_panel",
lambda page, markers: panel,
)
with pytest.raises(appeal.PriceAppealError, match="未加载目标商品 ID") as exc:
appeal._wait_for_item_panel_ready( # noqa: SLF001
page,
("选择申诉SKU",),
expected_item_id="3774363265803616739",
timeout_seconds=1,
)
assert isinstance(exc.value.__cause__, appeal.PriceAppealError)
assert str(exc.value.__cause__) == "无法读取活动抽屉内容"
def test_verify_item_identity_recovers_after_transient_detach(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
panel = _TransientTextPanel(
(
"选择申诉SKU 商品 ID: 3774363265803616739 "
"SKU规格 ID: 3619462493048834 原价 ¥839",
),
transient_failures=1,
)
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
appeal._verify_item_identity(panel, "3774363265803616739") # noqa: SLF001
assert panel.read_calls == 2
def test_verify_item_identity_persistent_read_failure_fails_closed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
panel = _TransientTextPanel(
("选择申诉SKU",),
transient_failures=100,
)
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
with pytest.raises(appeal.PriceAppealError, match="与待处理行不一致") as exc:
appeal._verify_item_identity(panel, "3774363265803616739") # noqa: SLF001
assert isinstance(exc.value.__cause__, appeal.PriceAppealError)
assert str(exc.value.__cause__) == "无法读取活动抽屉内容"
def test_pending_panel_with_headers_only_never_becomes_ready(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -67,6 +67,37 @@ def test_market_category_option_uses_visible_titled_menu_item() -> None:
assert option.clicked
class _UnstableOption(_Locator):
def scroll_into_view_if_needed(self, **_kwargs) -> None:
return None
def click(self, **_kwargs) -> None:
raise TimeoutError(
"Locator.click: Timeout 5000ms exceeded ... element is not stable"
)
def test_market_category_option_falls_back_to_native_click_when_unstable() -> None:
option = _UnstableOption()
page = SimpleNamespace(
locator=lambda _selector: option,
evaluate=lambda *_args, **_kwargs: True,
)
assert collector._click_market_category_option(page, "双肩包")
assert option.waited
def test_market_category_option_fails_closed_when_native_click_also_fails() -> None:
option = _UnstableOption()
page = SimpleNamespace(
locator=lambda _selector: option,
evaluate=lambda *_args, **_kwargs: False,
)
assert not collector._click_market_category_option(page, "双肩包")
def test_select_market_category_requires_path_and_fresh_table(monkeypatch) -> None:
expected = collector._expected_market_category_path("双肩包")
paths = iter([collector._expected_market_category_path("腰包/胸包"), expected])
@@ -0,0 +1,106 @@
from __future__ import annotations
from datetime import date
import pytest
from gyxx_flow.modules.shop_intelligence.collectors import jd_data_collector as collector
class _FlakyCellLocator:
def __init__(self) -> None:
self.click_calls = 0
def scroll_into_view_if_needed(self, **_kwargs) -> None:
return None
def click(self, **_kwargs) -> None:
self.click_calls += 1
raise TimeoutError("element is not visible")
class _DatePickerPage:
"""Stub page: date already unset, month aligned, cell found, then results."""
def __init__(self, native_click_result: bool) -> None:
self.cell = _FlakyCellLocator()
self.native_click_result = native_click_result
self.calls = 0
self.native_click_script: str | None = None
def evaluate(self, script: str, arg: object = None):
del arg
self.calls += 1
if self.calls == 1:
assert "textContent" in script # already-set check
return False
if self.calls == 2:
assert "targetYear" in script # month navigation
return {"status": "aligned"}
if self.calls == 3:
assert "gridcell" in script # locate target day cell
return {"status": "found_cell", "id": "__jd_target"}
assert self.calls == 4
assert "document.getElementById" in script # native click fallback
self.native_click_script = script
return self.native_click_result
def locator(self, selector: str) -> _FlakyCellLocator:
assert selector == "#__jd_target"
return self.cell
def wait_for_timeout(self, _milliseconds: int) -> None:
return None
def _prepare(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
collector,
"get_last_week_range",
lambda: (date(2026, 8, 10), date(2026, 8, 16)),
)
monkeypatch.setattr(collector, "save_date_picker_debug", lambda *_args, **_kwargs: None)
def test_old_datepicker_cell_click_falls_back_to_native_click(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_prepare(monkeypatch)
page = _DatePickerPage(native_click_result=True)
collector.select_last_week_in_old_datepicker(page)
assert page.cell.click_calls == 1
assert page.native_click_script is not None
assert page.calls == 4
def test_old_datepicker_fails_closed_when_native_click_also_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_prepare(monkeypatch)
page = _DatePickerPage(native_click_result=False)
with pytest.raises(RuntimeError, match="旧版日期控件未能选择上周日期"):
collector.select_last_week_in_old_datepicker(page)
assert page.cell.click_calls == 1
assert page.native_click_script is not None
def test_old_datepicker_happy_path_uses_playwright_click(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_prepare(monkeypatch)
page = _DatePickerPage(native_click_result=True)
class StableCell(_FlakyCellLocator):
def click(self, **_kwargs) -> None:
self.click_calls += 1
page.cell = StableCell()
collector.select_last_week_in_old_datepicker(page)
assert page.cell.click_calls == 1
assert page.native_click_script is None
@@ -48,6 +48,7 @@ def test_category_menu_closes_stale_portals_before_opening_selector() -> None:
class Trigger:
def __init__(self, page) -> None:
self.page = page
self.first = self
def click(self, **_kwargs) -> None:
self.page.menu_count = 1
@@ -255,3 +256,129 @@ def test_failed_category_switch_never_extracts_previous_table(
assert "类目切换或榜单刷新失败" in result["男士双肩包"]["error"]
assert result["男士单肩/斜挎包"]["rank"] == "7"
assert extracted == ["called"]
class _SearchBox:
def __init__(self) -> None:
self.calls: list[str] = []
def is_visible(self, **_kwargs) -> bool:
return True
def click(self, **_kwargs) -> None:
self.calls.append("click")
def fill(self, value: str, **_kwargs) -> None:
self.calls.append(f"fill:{value}")
def press(self, key: str, **_kwargs) -> None:
self.calls.append(f"press:{key}")
class _OneLocator:
def __init__(self, item: _SearchBox) -> None:
self.item = item
@property
def first(self) -> _OneLocator:
return self
def count(self) -> int:
return 1
def is_visible(self, **_kwargs) -> bool:
return self.item.is_visible()
def click(self, **_kwargs) -> None:
self.item.click()
def fill(self, value: str, **_kwargs) -> None:
self.item.fill(value)
def press(self, key: str, **_kwargs) -> None:
self.item.press(key)
class _EmptyLocator:
@property
def first(self) -> _EmptyLocator:
return self
def count(self) -> int:
return 0
def is_visible(self, **_kwargs) -> bool:
raise TimeoutError("no elements matched")
def click(self, **_kwargs) -> None:
raise TimeoutError("no elements matched")
class _SearchPage:
def __init__(self, box: _SearchBox, *, section_result: str | None) -> None:
self.box = box
self.section_result = section_result
self.evaluate_calls = 0
def evaluate(self, _script: str, _arg: object = None):
self.evaluate_calls += 1
return self.section_result
def locator(self, selector: str) -> _OneLocator | _EmptyLocator:
if selector.startswith("#__peer_search_input"):
return _OneLocator(self.box)
if selector == "input":
return _OneLocator(self.box)
if "店铺" in selector:
return _OneLocator(self.box)
return _EmptyLocator()
def wait_for_timeout(self, _milliseconds: int) -> None:
return None
def test_search_shop_prefers_trade_rank_section_input() -> None:
box = _SearchBox()
page = _SearchPage(box, section_result="__peer_search_input_1")
collector.search_shop(page, "Bellroy")
assert page.evaluate_calls == 1
assert box.calls == [
"click",
"fill:",
"fill:Bellroy",
"press:Enter",
]
def test_search_shop_falls_back_to_legacy_selectors_without_section() -> None:
box = _SearchBox()
page = _SearchPage(box, section_result=None)
collector.search_shop(page, "Bellroy")
assert page.evaluate_calls == 1
assert box.calls == [
"click",
"fill:",
"fill:Bellroy",
"press:Enter",
]
def test_locate_peer_search_input_returns_none_when_evaluate_fails() -> None:
box = _SearchBox()
class FailingPage(_SearchPage):
def evaluate(self, _script: str, _arg: object = None):
raise RuntimeError("page crashed")
assert collector._locate_peer_search_input(FailingPage(box, section_result="x")) is None
def test_locate_peer_search_input_returns_none_without_section() -> None:
box = _SearchBox()
page = _SearchPage(box, section_result=None)
assert collector._locate_peer_search_input(page) is None
@@ -27,7 +27,7 @@ def test_browser_runtime_defaults_to_bundled_chromium(monkeypatch):
def test_browser_runtime_can_use_installed_chrome(monkeypatch):
monkeypatch.setenv("GYXX_BROWSER_CHANNEL", "chrome")
assert browser_runtime_options() == {"channel": "chrome"}
assert browser_runtime_options() == {"real_chrome": True}
class DateRangeTests(unittest.TestCase):
@@ -455,6 +455,36 @@ def test_shop_nonpositive_persistence_stops_feishu_write(
assert errors == [f"{platform_code} DB 持久化失败: 未写入任何记录"]
@pytest.mark.parametrize(
("platform", "persist_name"),
[
(run_shop.JD_PLATFORM, "persist_jd_shop"),
(run_shop.DY_PLATFORM, "persist_dy_shop"),
],
)
def test_shop_skip_feishu_preserves_database_write_without_calling_writer(
monkeypatch,
platform: str,
persist_name: str,
) -> None:
monkeypatch.setattr(run_shop, "ensure_ai_analysis", lambda *_args: "")
monkeypatch.setattr(run_shop, persist_name, lambda _path: 1)
monkeypatch.setattr(
run_shop,
"write_shop",
lambda *_args: pytest.fail("weekly shop workflow must not write Feishu"),
)
errors = run_shop._persist_and_write(
platform,
True,
"D:/fresh-shop.json",
skip_feishu=True,
)
assert errors == []
@pytest.mark.parametrize(
("platform", "persist_name", "platform_code"),
[
@@ -625,19 +655,16 @@ def test_jd_shop_uses_bound_profile_port_and_persists_state(
context = Context()
class Chromium:
def launch_persistent_context(self, **kwargs):
class Scrapling:
def __init__(self, **kwargs):
observed.update(kwargs)
return context
class Playwright:
chromium = Chromium()
self.context = context
def __enter__(self):
return self
def __exit__(self, *_args) -> None:
return None
context.close()
cookie_file = tmp_path / "state" / "cookies.json"
storage_file = tmp_path / "state" / "storage.json"
@@ -646,7 +673,7 @@ def test_jd_shop_uses_bound_profile_port_and_persists_state(
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22105")
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
monkeypatch.setattr(jd_data_collector, "sync_playwright", Playwright)
monkeypatch.setattr(jd_data_collector, "ScraplingBrowser", Scrapling)
monkeypatch.setattr(jd_data_collector, "step_login", lambda *_args: None)
monkeypatch.setattr(
jd_data_collector,
@@ -670,8 +697,13 @@ def test_jd_shop_uses_bound_profile_port_and_persists_state(
assert jd_data_collector.main(["--password", "test-only"]) == 0
assert observed["user_data_dir"] == str(profile)
assert "--remote-debugging-port=22105" in observed["args"]
assert "--no-proxy-server" in observed["args"]
assert "--remote-debugging-port=22105" in observed["extra_flags"]
assert "--no-proxy-server" in observed["extra_flags"]
assert observed["additional_args"]["viewport"] == {
"width": 1920,
"height": 1080,
}
assert observed["retries"] == 1
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "session"
assert storage_file.is_file()
assert context.closed is True
@@ -699,19 +731,16 @@ def test_jd_peer_uses_bound_profile_port_and_persists_state(
context = Context()
class Chromium:
def launch_persistent_context(self, **kwargs):
class Scrapling:
def __init__(self, **kwargs):
observed.update(kwargs)
return context
class Playwright:
chromium = Chromium()
self.context = context
def __enter__(self):
return self
def __exit__(self, *_args) -> None:
return None
context.close()
cookie_file = tmp_path / "state" / "cookies.json"
storage_file = tmp_path / "state" / "storage.json"
@@ -720,7 +749,11 @@ def test_jd_peer_uses_bound_profile_port_and_persists_state(
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22106")
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
monkeypatch.setattr(jd_peer_store_data_collector, "sync_playwright", Playwright)
monkeypatch.setattr(
jd_peer_store_data_collector,
"ScraplingBrowser",
Scrapling,
)
monkeypatch.setattr(jd_peer_store_data_collector, "step_login", lambda *_args: None)
monkeypatch.setattr(
jd_peer_store_data_collector,
@@ -765,7 +798,12 @@ def test_jd_peer_uses_bound_profile_port_and_persists_state(
assert jd_peer_store_data_collector.main(["--password", "test-only"]) == 0
assert observed["user_data_dir"] == str(profile)
assert "--remote-debugging-port=22106" in observed["args"]
assert "--remote-debugging-port=22106" in observed["extra_flags"]
assert observed["additional_args"]["viewport"] == {
"width": 1920,
"height": 1080,
}
assert observed["retries"] == 1
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "peer-session"
assert storage_file.is_file()
assert context.closed is True
@@ -133,6 +133,12 @@ TARGET_AST_SHA256_OVERRIDES = {
("jd_data_collector.py", "select_last_week_any_day"): (
"8f6aed11b39d8e19ec4146e9b32eee96f5977fdbef694311534346bacec5d847"
),
# Reviewed after production runs showed Playwright's actionability check
# rejecting the legacy date-picker cell ("element is not visible") while the
# panel re-renders; a native click on the same node is the approved fallback.
("jd_data_collector.py", "select_last_week_in_old_datepicker"): (
"43d1056eeac1faeff32f0c01d0867e6673d2ab57999a14a3ebeb1b51c40075d4"
),
("jd_data_collector.py", "step_shop_star_and_trade"): (
"77aa873543861f1fad63d0bd1ace60528ab20592b0bffef3d46ad192f2103161"
),