feat: expand module workflows, dynamic config, notifications and console API
This commit is contained in:
@@ -0,0 +1,339 @@
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
|
||||
from gyxx_flow.modules.content_marketing import scrapling_http
|
||||
from gyxx_flow.modules.content_marketing import self_bilibili_scraper as self_bili
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
status: int = 200,
|
||||
payload: dict | None = None,
|
||||
text: str | None = None,
|
||||
content_type: str = "application/json; charset=utf-8",
|
||||
json_error: ValueError | None = None,
|
||||
) -> None:
|
||||
self.status_code = status
|
||||
self._payload = payload
|
||||
self.text = json.dumps(payload) if text is None and payload is not None else (text or "")
|
||||
self.content = self.text.encode("utf-8")
|
||||
self.headers = {"content-type": content_type}
|
||||
self._json_error = json_error
|
||||
|
||||
def json(self):
|
||||
if self._json_error is not None:
|
||||
raise self._json_error
|
||||
return self._payload
|
||||
|
||||
|
||||
class SequenceSession:
|
||||
def __init__(self, outcomes):
|
||||
self.outcomes = list(outcomes)
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
self.calls.append((url, kwargs))
|
||||
outcome = self.outcomes.pop(0)
|
||||
if isinstance(outcome, BaseException):
|
||||
raise outcome
|
||||
return outcome
|
||||
|
||||
|
||||
def test_scrapling_http_session_maps_fetcher_response(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def get(url, **kwargs):
|
||||
calls.append((url, kwargs))
|
||||
return SimpleNamespace(
|
||||
status=200,
|
||||
headers={"content-type": "application/json"},
|
||||
url="https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
body=b'{"code": 0}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(scrapling_http.Fetcher, "get", get)
|
||||
session = scrapling_http.ScraplingHttpSession(headers={"User-Agent": "gyxx"})
|
||||
|
||||
response = session.get(
|
||||
"https://b23.tv/demo",
|
||||
allow_redirects=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
assert calls == [(
|
||||
"https://b23.tv/demo",
|
||||
{
|
||||
"headers": {"User-Agent": "gyxx"},
|
||||
"follow_redirects": "all",
|
||||
"retries": 1,
|
||||
"timeout": 10,
|
||||
},
|
||||
)]
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"code": 0}
|
||||
assert response.url.endswith("BV17NNw6jEPx")
|
||||
|
||||
|
||||
def test_self_bilibili_uses_scrapling_user_agent_instead_of_stale_override():
|
||||
session = SequenceSession([
|
||||
FakeResponse(
|
||||
payload={
|
||||
"code": 0,
|
||||
"data": {"stat": {"view": 456}, "pubdate": 1_700_000_000},
|
||||
}
|
||||
)
|
||||
])
|
||||
|
||||
result = self_bili.fetch_bili_stat(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result == (456, 1_700_000_000)
|
||||
assert session.calls == [(
|
||||
"https://api.bilibili.com/x/web-interface/view",
|
||||
{
|
||||
"params": {"bvid": "BV17NNw6jEPx"},
|
||||
"timeout": 10,
|
||||
"headers": {"Referer": "https://www.bilibili.com/"},
|
||||
},
|
||||
)]
|
||||
|
||||
|
||||
def success_response(view: int = 321) -> FakeResponse:
|
||||
return FakeResponse(payload={"code": 0, "message": "OK", "data": {"stat": {"view": view}}})
|
||||
|
||||
|
||||
def detail_success_response(view: int = 321) -> FakeResponse:
|
||||
return FakeResponse(
|
||||
payload={"code": 0, "message": "OK", "data": {"View": {"stat": {"view": view}}}}
|
||||
)
|
||||
|
||||
|
||||
def test_empty_body_retries_then_returns_play_count(monkeypatch, capsys):
|
||||
session = SequenceSession([FakeResponse(text=""), success_response(456)])
|
||||
sleeps = []
|
||||
monkeypatch.setattr(bili.time, "sleep", sleeps.append)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result == 456
|
||||
assert len(session.calls) == 2
|
||||
assert sleeps == [1.5]
|
||||
assert "[empty_body]" in capsys.readouterr().out
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("first_response", "category"),
|
||||
[
|
||||
(FakeResponse(status=412, text="request blocked", content_type="text/plain"), "http_412"),
|
||||
(FakeResponse(payload={"code": -509, "message": "request limit"}), "api_code_-509"),
|
||||
(
|
||||
FakeResponse(
|
||||
text="<html>temporary gateway page</html>",
|
||||
content_type="text/html",
|
||||
json_error=ValueError("not json"),
|
||||
),
|
||||
"invalid_json",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_transient_response_classes_are_retried(
|
||||
monkeypatch, capsys, first_response, category
|
||||
):
|
||||
session = SequenceSession([first_response, success_response()])
|
||||
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result == 321
|
||||
assert len(session.calls) == 2
|
||||
assert f"[{category}]" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_non_retryable_api_code_stops_after_one_attempt(monkeypatch, capsys):
|
||||
session = SequenceSession(
|
||||
[FakeResponse(payload={"code": -404, "message": "video not found"})]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
bili.time,
|
||||
"sleep",
|
||||
lambda _seconds: pytest.fail("terminal API result must not back off"),
|
||||
)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert len(session.calls) == 1
|
||||
assert "[api_code_-404]" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_transient_failures_exhaust_three_attempts_honestly(monkeypatch, capsys):
|
||||
session = SequenceSession(
|
||||
[
|
||||
*[FakeResponse(text="") for _ in range(3)],
|
||||
FakeResponse(status=412, text="fallback blocked", content_type="text/plain"),
|
||||
]
|
||||
)
|
||||
sleeps = []
|
||||
monkeypatch.setattr(bili.time, "sleep", sleeps.append)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert len(session.calls) == 4
|
||||
assert sleeps == [1.5, 3.0]
|
||||
output = capsys.readouterr().out
|
||||
assert "attempt=3/3" in output
|
||||
assert "[view_detail_http_412]" in output
|
||||
|
||||
|
||||
def test_view_detail_is_used_once_after_primary_transient_exhaustion(
|
||||
monkeypatch, capsys
|
||||
):
|
||||
session = SequenceSession(
|
||||
[*[FakeResponse(text="") for _ in range(3)], detail_success_response(789)]
|
||||
)
|
||||
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
|
||||
|
||||
result = bili.fetch_play_count(
|
||||
"https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
session,
|
||||
)
|
||||
|
||||
assert result == 789
|
||||
assert len(session.calls) == 4
|
||||
assert session.calls[-1][0] == bili._VIEW_DETAIL_API_URL
|
||||
assert "[FALLBACK]" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_play_count_failure_remains_an_incomplete_summary(monkeypatch):
|
||||
style = {
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"read_count_7d": {"field_id": "s7"},
|
||||
"read_count_14d": {"field_id": "s14"},
|
||||
"read_count_21d": {"field_id": "s21"},
|
||||
"read_count_28d": {"field_id": "s28"},
|
||||
"month_end": {"field_id": "sm"},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
bili,
|
||||
"list_all_records",
|
||||
lambda *_args: [{
|
||||
"record_id": "failed",
|
||||
"platform": "B站",
|
||||
"creator": "达人",
|
||||
"url": "https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
"pub": "2026-08-08",
|
||||
}],
|
||||
)
|
||||
monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
bili,
|
||||
"write_record",
|
||||
lambda *_args, **_kwargs: pytest.fail("failed fetch must not write"),
|
||||
)
|
||||
|
||||
summary = bili.process_style(
|
||||
style,
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=bili.date(2026, 8, 9),
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
assert summary["complete"] is False
|
||||
assert summary["retryable_failures"] == 1
|
||||
assert summary["details"][0]["reason"] == "play_count_fetch_failed"
|
||||
|
||||
|
||||
def test_terminal_unavailable_detail_is_a_complete_blocked_input(monkeypatch):
|
||||
style = {
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"read_count_7d": {"field_id": "s7"},
|
||||
"read_count_14d": {"field_id": "s14"},
|
||||
"read_count_21d": {"field_id": "s21"},
|
||||
"read_count_28d": {"field_id": "s28"},
|
||||
"month_end": {"field_id": "sm"},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
bili,
|
||||
"list_all_records",
|
||||
lambda *_args: [{
|
||||
"record_id": "unavailable",
|
||||
"platform": "B站",
|
||||
"creator": "达人",
|
||||
"url": "https://www.bilibili.com/video/BV17NNw6jEPx",
|
||||
"pub": "2026-08-08",
|
||||
}],
|
||||
)
|
||||
monkeypatch.setattr(bili.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(
|
||||
bili,
|
||||
"write_record",
|
||||
lambda *_args, **_kwargs: pytest.fail("unavailable video must not write"),
|
||||
)
|
||||
session = SequenceSession([
|
||||
*[FakeResponse(status=412, text="request blocked") for _ in range(3)],
|
||||
FakeResponse(payload={"code": 62002, "message": "稿件不可见", "data": None}),
|
||||
])
|
||||
|
||||
summary = bili.process_style(
|
||||
style,
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=session,
|
||||
state={},
|
||||
force_today=bili.date(2026, 8, 9),
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
assert len(session.calls) == 4
|
||||
assert summary["complete"] is True
|
||||
assert summary["blocked_input"] == 1
|
||||
assert summary["retryable_failures"] == 0
|
||||
assert summary["unresolved"] == 0
|
||||
assert summary["details"][0]["status"] == "blocked_input"
|
||||
assert summary["details"][0]["reason"] == "video_unavailable"
|
||||
@@ -0,0 +1,346 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from argparse import Namespace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
from gyxx_flow.modules.content_marketing.data.tools import retry_failed
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
sync_metrics_to_cmt_notes as sync_metrics,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_content_source():
|
||||
xingtu.configure_source("xingtu")
|
||||
yield
|
||||
xingtu.configure_source("xingtu")
|
||||
|
||||
|
||||
def test_buyin_source_uses_doudian_market_and_cookie_scope():
|
||||
xingtu.configure_source("buyin")
|
||||
|
||||
assert xingtu.CONTENT_SOURCE == "buyin"
|
||||
assert xingtu.MARKET_URL.startswith(
|
||||
"https://buyin.jinritemai.com/dashboard/servicehall/daren-square"
|
||||
)
|
||||
assert xingtu.DETAIL_URL_MARK == "/dashboard/servicehall/daren-profile"
|
||||
assert xingtu.BUSINESS_COOKIE_DOMAINS == ("jinritemai.com",)
|
||||
|
||||
|
||||
def test_save_state_creates_cookie_parent_directory(tmp_path, monkeypatch):
|
||||
cookie_file = tmp_path / "nested" / "cookies.json"
|
||||
monkeypatch.setattr(xingtu, "DATA_DIR", tmp_path / "raw")
|
||||
monkeypatch.setattr(xingtu, "COOKIE_FILE", cookie_file)
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [{"name": "sessionid", "value": "redacted"}]
|
||||
|
||||
class Page:
|
||||
context = Context()
|
||||
|
||||
xingtu.save_state(Page())
|
||||
|
||||
assert cookie_file.exists()
|
||||
assert "sessionid" in cookie_file.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_run_all_keeps_cooperation_xt_on_xingtu_for_buyin_fallback():
|
||||
args = Namespace(
|
||||
dry_run=False,
|
||||
style=None,
|
||||
daily_scope=False,
|
||||
no_retry=False,
|
||||
)
|
||||
|
||||
command = run_all.build_process_command("xt", args)
|
||||
|
||||
assert "--buyin" not in command
|
||||
assert "--skip-field-prepare" in command
|
||||
|
||||
|
||||
def test_retry_failed_keeps_cooperation_xt_on_xingtu_for_buyin_fallback():
|
||||
assert retry_failed.SCRIPTS["xt"]["extra_args"] == []
|
||||
|
||||
|
||||
def test_buyin_video_parser_is_dom_html_based_and_maps_viewers_to_exposure():
|
||||
xingtu.configure_source("buyin")
|
||||
|
||||
class Page:
|
||||
def __init__(self):
|
||||
self.script = ""
|
||||
|
||||
def evaluate(self, script):
|
||||
self.script = script
|
||||
return [{
|
||||
"title": "#达人笔记标题",
|
||||
"play_count": "1,155",
|
||||
"view_count": "1,155",
|
||||
"exposure_count": "1,155",
|
||||
}]
|
||||
|
||||
page = Page()
|
||||
cards = xingtu.parse_videos_on_page(page)
|
||||
|
||||
assert cards[0]["play_count"] == "1,155"
|
||||
assert cards[0]["exposure_count"] == "1,155"
|
||||
assert "观看人数" in page.script
|
||||
assert "innerText" in page.script
|
||||
|
||||
|
||||
def test_buyin_history_tab_and_trade_video_filter_are_selected():
|
||||
xingtu.configure_source("buyin")
|
||||
events = []
|
||||
filter_states = {"只看带货视频": True, "只看带货图文": True}
|
||||
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def filter(self, **kwargs):
|
||||
return Locator(kwargs.get("has_text", self.name))
|
||||
|
||||
def click(self, **_kwargs):
|
||||
events.append(f"click-{self.name}")
|
||||
if self.name in filter_states:
|
||||
filter_states[self.name] = False
|
||||
|
||||
class Page:
|
||||
def get_by_text(self, text, **_kwargs):
|
||||
return Locator(text)
|
||||
|
||||
def get_by_role(self, _role, name, **_kwargs):
|
||||
return Locator(name)
|
||||
|
||||
def locator(self, _selector):
|
||||
return Locator("selector")
|
||||
|
||||
def evaluate(self, script, *_args):
|
||||
events.append(script)
|
||||
if _args and _args[0] in filter_states:
|
||||
return filter_states[_args[0]]
|
||||
return False
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
events.append(milliseconds)
|
||||
|
||||
assert xingtu.open_creation_ability(Page()) is True
|
||||
assert "click-历史内容" in events
|
||||
assert "click-近30天" in events
|
||||
assert "click-视频" in events
|
||||
assert "click-只看带货视频" in events
|
||||
assert "click-只看带货图文" in events
|
||||
assert any("scrollIntoView" in event for event in events if isinstance(event, str))
|
||||
|
||||
|
||||
def test_buyin_creator_search_clicks_real_button_and_waits_for_changed_results(
|
||||
monkeypatch,
|
||||
):
|
||||
xingtu.configure_source("buyin")
|
||||
events = []
|
||||
|
||||
class Locator:
|
||||
def __init__(self, kind):
|
||||
self.kind = kind
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
return None
|
||||
|
||||
def click(self, **_kwargs):
|
||||
events.append(f"click-{self.kind}")
|
||||
|
||||
def fill(self, value):
|
||||
events.append(f"fill-{value}")
|
||||
|
||||
def press(self, key):
|
||||
events.append(f"press-{key}")
|
||||
|
||||
class Page:
|
||||
def locator(self, selector):
|
||||
if selector.startswith("button.auxo-input-search-button"):
|
||||
return Locator("button")
|
||||
return Locator("input")
|
||||
|
||||
def evaluate(self, _script):
|
||||
return "idle"
|
||||
|
||||
def get_by_role(self, *_args, **_kwargs):
|
||||
raise AssertionError("Buyin should use the real search button first")
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(xingtu, "dismiss_popups", lambda _page: None)
|
||||
monkeypatch.setattr(xingtu, "select_nickname_search", lambda _page: None)
|
||||
monkeypatch.setattr(xingtu, "_buyin_result_page_signature", lambda _page: "before")
|
||||
observed = {}
|
||||
|
||||
def wait_for_results(_page, _name, timeout_ms, previous_signature=None):
|
||||
observed.update(timeout_ms=timeout_ms, previous_signature=previous_signature)
|
||||
|
||||
monkeypatch.setattr(xingtu, "wait_for_visible_creator_result", wait_for_results)
|
||||
|
||||
xingtu.search_creator(Page(), "linyl燕", "LinFX2327")
|
||||
|
||||
assert events == ["click-input", "fill-LinFX2327", "click-button"]
|
||||
assert observed == {"timeout_ms": 30000, "previous_signature": "before"}
|
||||
|
||||
|
||||
def test_buyin_search_button_waits_until_loading_finishes():
|
||||
xingtu.configure_source("buyin")
|
||||
states = iter(["busy", "busy", "idle"])
|
||||
waits = []
|
||||
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return next(states)
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
waits.append(milliseconds)
|
||||
|
||||
xingtu._wait_for_buyin_search_button_idle(Page(), timeout_ms=1000)
|
||||
|
||||
assert waits == [250, 250]
|
||||
|
||||
|
||||
def test_buyin_wait_distinguishes_temporary_search_throttling():
|
||||
xingtu.configure_source("buyin")
|
||||
|
||||
class Page:
|
||||
def evaluate(self, *_args):
|
||||
return "throttled"
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pytest.fail("throttling must return immediately")
|
||||
|
||||
with pytest.raises(xingtu.SearchTemporarilyThrottled, match="过于频繁"):
|
||||
xingtu.wait_for_visible_creator_result(Page(), "LinFX2327", timeout_ms=1000)
|
||||
|
||||
|
||||
def test_buyin_creator_scan_supports_single_row_without_profile_href():
|
||||
script = xingtu._buyin_scan_creator_script()
|
||||
|
||||
assert "tr[data-row-key], [role=\"row\"][data-row-key]" in script
|
||||
assert "rowKey: row.getAttribute('data-row-key')" in script
|
||||
assert "via: 'single-search-result'" in script
|
||||
assert "first-id-search-result" not in script
|
||||
|
||||
|
||||
def test_buyin_next_page_waits_for_a_changed_html_signature():
|
||||
xingtu.configure_source("buyin")
|
||||
signatures = iter(["1/4|first-card", "2/4|second-card"])
|
||||
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
if "const selectors" in script:
|
||||
return {"clicked": True, "picked": "semi-page-next"}
|
||||
return next(signatures)
|
||||
|
||||
def locator(self, _selector):
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def click(self, **_kwargs):
|
||||
return None
|
||||
|
||||
return Locator()
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
assert xingtu.go_next_page(Page()) is True
|
||||
|
||||
|
||||
def test_metric_snapshot_date_prefers_orchestrator_business_date(monkeypatch):
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-31")
|
||||
|
||||
assert sync_metrics._resolve_metric_date() == dt.date(2026, 8, 31)
|
||||
assert sync_metrics._resolve_metric_date("2026-08-30") == dt.date(2026, 8, 30)
|
||||
with pytest.raises(ValueError, match="YYYY-MM-DD"):
|
||||
sync_metrics._resolve_metric_date("not-a-date")
|
||||
|
||||
|
||||
def test_metric_snapshot_upsert_is_idempotent_by_note_and_business_date():
|
||||
class Cursor:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.calls.append((" ".join(sql.split()), params))
|
||||
|
||||
cursor = Cursor()
|
||||
sync_metrics._upsert_metric_snapshot(
|
||||
cursor,
|
||||
42,
|
||||
{
|
||||
"view_count": 1155,
|
||||
"dst_platform": "douyin",
|
||||
"self_operated": False,
|
||||
"record_id": "rec-1",
|
||||
"source_url": "https://www.douyin.com/video/42",
|
||||
},
|
||||
dt.date(2026, 8, 31),
|
||||
)
|
||||
|
||||
sql, params = cursor.calls[0]
|
||||
assert "INSERT INTO cmt_note_metric_snapshots" in sql
|
||||
assert "ON CONFLICT (note_id, metric_date)" in sql
|
||||
assert params[0:2] == (42, dt.date(2026, 8, 31))
|
||||
assert params[6] == 1155
|
||||
|
||||
|
||||
def test_snapshot_schema_and_report_query_use_daily_history():
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
schema = (
|
||||
root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "content_marketing"
|
||||
/ "data"
|
||||
/ "tools"
|
||||
/ "schema_gyxx_super_data.sql"
|
||||
).read_text(encoding="utf-8")
|
||||
migration = (
|
||||
root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "content_marketing"
|
||||
/ "data"
|
||||
/ "tools"
|
||||
/ "migrations"
|
||||
/ "008_note_metric_snapshots.sql"
|
||||
).read_text(encoding="utf-8")
|
||||
report = (
|
||||
root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "content_marketing"
|
||||
/ "daily_marketing_report.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
for source in (schema, migration):
|
||||
assert "CREATE TABLE IF NOT EXISTS cmt_note_metric_snapshots" in source
|
||||
assert "UNIQUE (note_id, metric_date)" in source
|
||||
assert "COALESCE(ms.view_count, n.view_count)" in report
|
||||
assert "ms.metric_date = %s" in report
|
||||
@@ -1,7 +1,14 @@
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.content_marketing import chanmama_scraper as chanmama
|
||||
from gyxx_flow.modules.content_marketing import feishu_mapping
|
||||
from gyxx_flow.modules.content_marketing import (
|
||||
scrapling_browser_facade as browser_facade,
|
||||
)
|
||||
|
||||
|
||||
def test_exposure_state_clicks_text_and_zero_but_keeps_positive_values():
|
||||
@@ -152,6 +159,277 @@ def test_missing_get_data_button_does_not_trigger_refresh_wait(monkeypatch):
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
def test_create_driver_uses_runtime_cdp_profile_and_download_bindings(monkeypatch, tmp_path):
|
||||
calls = []
|
||||
expected = object()
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
|
||||
|
||||
def start(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
|
||||
|
||||
assert chanmama.create_driver() is expected
|
||||
assert calls == [{
|
||||
"profile_dir": str(tmp_path / "profile"),
|
||||
"download_dir": str(tmp_path / "downloads"),
|
||||
"cdp_url": "http://127.0.0.1:22002",
|
||||
"stealthy": True,
|
||||
}]
|
||||
|
||||
|
||||
def test_create_driver_starts_scrapling_without_cdp(monkeypatch, tmp_path):
|
||||
calls = []
|
||||
expected = object()
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.delenv("GYXX_BROWSER_CDP_URL", raising=False)
|
||||
|
||||
def start(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
|
||||
|
||||
assert chanmama.create_driver() is expected
|
||||
assert calls[0]["cdp_url"] is None
|
||||
assert calls[0]["stealthy"] is True
|
||||
assert calls[0]["real_chrome"] is False
|
||||
|
||||
|
||||
def test_create_driver_falls_back_when_bound_cdp_is_not_listening(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
calls = []
|
||||
expected = object()
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
|
||||
|
||||
def start(**kwargs):
|
||||
calls.append(kwargs)
|
||||
if kwargs["cdp_url"]:
|
||||
raise RuntimeError(
|
||||
"BrowserType.connect_over_cdp: connect ECONNREFUSED "
|
||||
"127.0.0.1:22002; retrieving websocket url"
|
||||
)
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(chanmama, "ScraplingDriver", SimpleNamespace(start=start))
|
||||
|
||||
assert chanmama.create_driver() is expected
|
||||
assert [call["cdp_url"] for call in calls] == [
|
||||
"http://127.0.0.1:22002",
|
||||
None,
|
||||
]
|
||||
assert all(call["stealthy"] is True for call in calls)
|
||||
assert calls[1]["real_chrome"] is False
|
||||
|
||||
|
||||
def test_create_driver_keeps_explicit_remote_cdp_fail_closed(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://browser.internal:22002")
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22002")
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"ScraplingDriver",
|
||||
SimpleNamespace(
|
||||
start=lambda **_kwargs: (_ for _ in ()).throw(
|
||||
RuntimeError("connect ECONNREFUSED browser.internal:22002")
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="ECONNREFUSED"):
|
||||
chanmama.create_driver()
|
||||
|
||||
|
||||
def test_create_driver_does_not_hide_non_connection_cdp_failures(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
monkeypatch.setattr(chanmama, "DOWNLOAD_DIR", str(tmp_path / "downloads"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(browser_profile_dir=tmp_path / "profile"),
|
||||
)
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_URL", "http://127.0.0.1:22002")
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"ScraplingDriver",
|
||||
SimpleNamespace(
|
||||
start=lambda **_kwargs: (_ for _ in ()).throw(
|
||||
RuntimeError("browser profile is corrupt")
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="profile is corrupt"):
|
||||
chanmama.create_driver()
|
||||
|
||||
|
||||
def test_scrapling_facade_reuses_runtime_cdp_context(monkeypatch, tmp_path):
|
||||
created = []
|
||||
|
||||
class Page:
|
||||
def on(self, *_args):
|
||||
return None
|
||||
|
||||
def add_init_script(self, script):
|
||||
assert "navigator" in script
|
||||
|
||||
class Context:
|
||||
pages = [Page()]
|
||||
|
||||
class Browser:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.context = Context()
|
||||
self.page = None
|
||||
self.closed = False
|
||||
created.append(self)
|
||||
|
||||
def start(self):
|
||||
return self
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
monkeypatch.setattr(browser_facade, "ScraplingBrowser", Browser)
|
||||
|
||||
driver = browser_facade.ScraplingDriver.start(
|
||||
profile_dir=tmp_path / "profile",
|
||||
download_dir=tmp_path / "downloads",
|
||||
cdp_url="http://127.0.0.1:22002",
|
||||
)
|
||||
|
||||
assert created[0].kwargs["reuse_existing_cdp_context"] is True
|
||||
assert created[0].kwargs["cdp_url"] == "http://127.0.0.1:22002"
|
||||
assert "user_data_dir" not in created[0].kwargs
|
||||
assert "--no-sandbox" not in created[0].kwargs["extra_flags"]
|
||||
assert created[0].page is created[0].context.pages[0]
|
||||
driver.quit()
|
||||
assert created[0].closed is True
|
||||
|
||||
|
||||
def _one_backfill_style():
|
||||
return {
|
||||
"tables": [
|
||||
{
|
||||
"index": 1,
|
||||
"name": "宙斯",
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "published"},
|
||||
"read_count_7d": {
|
||||
"field_id": "read-7d",
|
||||
"field_name": "7日阅读量",
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def test_backfill_mapping_or_record_list_failure_returns_false(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping,
|
||||
"load_mapping",
|
||||
lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("mapping down")),
|
||||
)
|
||||
assert chanmama.backfill_self_tables([]) is False
|
||||
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping, "load_mapping", lambda **_kwargs: {"tables": []}
|
||||
)
|
||||
assert chanmama.backfill_self_tables([]) is False
|
||||
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping, "load_mapping", lambda **_kwargs: _one_backfill_style()
|
||||
)
|
||||
assert chanmama.backfill_self_tables([], only_style=999) is False
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(normalized_root=tmp_path / "normalized"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_list_records_by_table",
|
||||
lambda *_args: (_ for _ in ()).throw(RuntimeError("base down")),
|
||||
)
|
||||
|
||||
assert chanmama.backfill_self_tables([]) is False
|
||||
|
||||
|
||||
def test_backfill_write_failure_is_false_but_natural_no_match_is_success(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping, "load_mapping", lambda **_kwargs: _one_backfill_style()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"PATHS",
|
||||
SimpleNamespace(normalized_root=tmp_path / "normalized"),
|
||||
)
|
||||
record = {
|
||||
"record_id": "record-1",
|
||||
"platform": "抖音",
|
||||
"title": "相同标题",
|
||||
"published": datetime.now(),
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
chanmama, "_list_records_by_table", lambda *_args: [record]
|
||||
)
|
||||
monkeypatch.setattr(chanmama, "_write_back", lambda *_args: False)
|
||||
|
||||
assert chanmama.backfill_self_tables(
|
||||
[{"title": "相同标题", "exposure": 100}]
|
||||
) is False
|
||||
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_write_back",
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
AssertionError("natural no-match must not write")
|
||||
),
|
||||
)
|
||||
assert chanmama.backfill_self_tables(
|
||||
[{"title": "完全不同", "exposure": 100}]
|
||||
) is True
|
||||
|
||||
|
||||
def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
|
||||
events = []
|
||||
|
||||
@@ -159,7 +437,6 @@ def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
|
||||
def quit(self):
|
||||
events.append("quit")
|
||||
|
||||
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
|
||||
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
|
||||
@@ -180,10 +457,75 @@ def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
|
||||
lambda *_args, **_kwargs: ([], []),
|
||||
)
|
||||
|
||||
assert chanmama.main() == 0
|
||||
assert chanmama.main() == 1
|
||||
assert events == ["saved", "quit"]
|
||||
|
||||
|
||||
def test_main_full_mode_exception_and_keyboard_interrupt_are_nonzero(monkeypatch):
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"create_driver",
|
||||
lambda: (_ for _ in ()).throw(RuntimeError("driver failed")),
|
||||
)
|
||||
assert chanmama.main() == 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"create_driver",
|
||||
lambda: (_ for _ in ()).throw(KeyboardInterrupt()),
|
||||
)
|
||||
assert chanmama.main() == 130
|
||||
|
||||
|
||||
def test_main_propagates_backfill_failure_after_nonempty_export(monkeypatch):
|
||||
events = []
|
||||
|
||||
class Driver:
|
||||
def quit(self):
|
||||
events.append("quit")
|
||||
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
|
||||
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
|
||||
monkeypatch.setattr(chanmama, "save_cookies", lambda _driver: True)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"refresh_then_export_accounts",
|
||||
lambda *_args, **_kwargs: ([{"title": "video"}], []),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"backfill_self_tables",
|
||||
lambda *_args, **_kwargs: False,
|
||||
)
|
||||
|
||||
assert chanmama.main() == 1
|
||||
assert events == ["quit"]
|
||||
|
||||
|
||||
def test_backfill_only_propagates_backfill_failure(monkeypatch, tmp_path):
|
||||
workbook = tmp_path / "existing.xlsx"
|
||||
workbook.write_bytes(b"placeholder")
|
||||
monkeypatch.setattr(
|
||||
chanmama.sys,
|
||||
"argv",
|
||||
["chanmama_scraper.py", "--backfill-only", "--excel", str(workbook)],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"parse_chanmama_excel",
|
||||
lambda _path: ([{"title": "video"}], []),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"backfill_self_tables",
|
||||
lambda *_args, **_kwargs: False,
|
||||
)
|
||||
|
||||
assert chanmama.main() == 1
|
||||
|
||||
|
||||
def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
@@ -196,7 +538,6 @@ def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
|
||||
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "ACCOUNT", "")
|
||||
monkeypatch.setattr(chanmama, "PASSWORD", "")
|
||||
@@ -251,3 +592,85 @@ def test_cli_preserves_cookie_skip_exit_code(monkeypatch):
|
||||
chanmama.cli()
|
||||
|
||||
assert raised.value.code == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_polls_until_file_appears(monkeypatch):
|
||||
now = [1000.0]
|
||||
sleeps: list[float] = []
|
||||
results = iter([None, None, "/downloads/博主A_视频记录_周期内发布视频.xlsx"])
|
||||
|
||||
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
|
||||
monkeypatch.setattr(
|
||||
chanmama.time,
|
||||
"sleep",
|
||||
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
|
||||
)
|
||||
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
|
||||
|
||||
assert chanmama._wait_for_exported_excel(0.0) == (
|
||||
"/downloads/博主A_视频记录_周期内发布视频.xlsx"
|
||||
)
|
||||
assert sleeps == [5.0, 5.0]
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_returns_none_after_timeout(monkeypatch):
|
||||
now = [1000.0]
|
||||
sleeps: list[float] = []
|
||||
|
||||
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
|
||||
monkeypatch.setattr(
|
||||
chanmama.time,
|
||||
"sleep",
|
||||
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
|
||||
)
|
||||
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: None)
|
||||
|
||||
assert (
|
||||
chanmama._wait_for_exported_excel(
|
||||
0.0,
|
||||
timeout_seconds=12,
|
||||
grace_seconds=0,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert len(sleeps) == 3 # 1000 -> 1005 -> 1010 -> 1015(>= 1012 超时)
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_allows_boundary_grace(monkeypatch):
|
||||
now = [1000.0]
|
||||
sleeps: list[float] = []
|
||||
results = iter([None, None, "/downloads/边界后落盘.xlsx"])
|
||||
|
||||
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
|
||||
monkeypatch.setattr(
|
||||
chanmama.time,
|
||||
"sleep",
|
||||
lambda seconds: (sleeps.append(seconds), now.__setitem__(0, now[0] + seconds)),
|
||||
)
|
||||
monkeypatch.setattr(chanmama, "_find_new_excel_after", lambda _pre_mtime: next(results))
|
||||
|
||||
assert chanmama._wait_for_exported_excel(
|
||||
0.0,
|
||||
timeout_seconds=5,
|
||||
grace_seconds=5,
|
||||
poll_interval_seconds=5,
|
||||
) == "/downloads/边界后落盘.xlsx"
|
||||
assert sleeps == [5.0, 5.0]
|
||||
|
||||
|
||||
def test_wait_for_exported_excel_returns_immediately_when_present(monkeypatch):
|
||||
now = [1000.0]
|
||||
|
||||
monkeypatch.setattr(chanmama.time, "time", lambda: now[0])
|
||||
monkeypatch.setattr(
|
||||
chanmama.time,
|
||||
"sleep",
|
||||
lambda _seconds: pytest.fail("不应等待已存在的文件"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_find_new_excel_after",
|
||||
lambda _pre_mtime: "/downloads/已有.xlsx",
|
||||
)
|
||||
|
||||
assert chanmama._wait_for_exported_excel(0.0) == "/downloads/已有.xlsx"
|
||||
|
||||
@@ -9,6 +9,10 @@ from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
from gyxx_flow.modules.content_marketing.serialized_page_action import (
|
||||
is_transient_browser_error,
|
||||
run_serialized_page_action,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_title_handles_nfkc_case_and_zero_width():
|
||||
@@ -131,6 +135,28 @@ def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
|
||||
assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
|
||||
|
||||
|
||||
def test_xingtu_can_use_strong_title_when_share_id_is_canonicalized():
|
||||
task = {
|
||||
"record_id": "r1",
|
||||
"target_title": "这家包颜值和实用性真挺高",
|
||||
"note_id": "7665297310758210854",
|
||||
}
|
||||
card = {
|
||||
"title": "这家包颜值和实用性真挺高 #潮流运动 #光影行星",
|
||||
"note_id": "canonical- Xingtu-item-id",
|
||||
}
|
||||
|
||||
matched = cc.match_tasks_to_cards(
|
||||
[task],
|
||||
[card],
|
||||
platform="douyin",
|
||||
allow_mismatched_content_id_title=True,
|
||||
)
|
||||
|
||||
assert matched["r1"]["match_method"] == "title"
|
||||
assert matched["r1"]["match_score"] >= 0.9
|
||||
|
||||
|
||||
def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
|
||||
cards = [
|
||||
{
|
||||
@@ -354,8 +380,11 @@ class _FakeSearchPage:
|
||||
def test_pgy_search_retry_keeps_creator_id_query():
|
||||
page = _FakeSearchPage()
|
||||
|
||||
pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
|
||||
found = pgy._ensure_search_results(
|
||||
page, query="xhs-id-123", result_name="达人昵称", max_retries=2
|
||||
)
|
||||
|
||||
assert found is True
|
||||
assert page.input.filled == ["xhs-id-123"]
|
||||
|
||||
|
||||
@@ -376,11 +405,19 @@ def test_apply_matched_task_reports_write_failure(monkeypatch):
|
||||
assert outcome["write_ok"] is False
|
||||
|
||||
|
||||
def test_default_creator_batch_has_no_forced_hour_pause():
|
||||
assert pgy.DEFAULT_BATCH_SIZE == 0
|
||||
def test_pgy_default_creator_batch_rotates_browser_before_memory_accumulates():
|
||||
assert pgy.DEFAULT_BATCH_SIZE == 4
|
||||
assert xingtu.DEFAULT_BATCH_SIZE == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("message", [
|
||||
"Page crashed: Out of Memory",
|
||||
"Target page, context or browser has been closed: not enough memory",
|
||||
])
|
||||
def test_pgy_treats_chromium_memory_exhaustion_as_a_recoverable_session_loss(message):
|
||||
assert pgy.is_browser_session_lost(RuntimeError(message)) is True
|
||||
|
||||
|
||||
def test_repeatable_style_argument_and_selection_keep_every_requested_style():
|
||||
parser = argparse.ArgumentParser()
|
||||
cc.add_repeatable_style_argument(parser, "styles")
|
||||
@@ -408,6 +445,223 @@ def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
|
||||
assert page.waits == 2
|
||||
|
||||
|
||||
def test_pgy_stable_cards_waits_through_a_slow_initial_render(monkeypatch):
|
||||
card = {"title": "标题", "read_count": "1"}
|
||||
sequence = [[], [], [], [], [], [card], [card]]
|
||||
monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
|
||||
|
||||
class Page:
|
||||
waits = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
self.waits += 1
|
||||
|
||||
page = Page()
|
||||
assert pgy.parse_cards_stable(page) == [card]
|
||||
assert page.waits == 6
|
||||
|
||||
|
||||
def test_pgy_card_parser_recovers_note_id_from_internal_card_link():
|
||||
note_id = "0123456789abcdef01234567"
|
||||
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return [{
|
||||
"title": "标题",
|
||||
"read_count": "1",
|
||||
"href": "",
|
||||
"note_id": "",
|
||||
"_href_candidates": [
|
||||
f"https://pgy.xiaohongshu.com/internal?source_note_id={note_id}"
|
||||
],
|
||||
"_data_note_ids": [],
|
||||
}]
|
||||
|
||||
cards = pgy.parse_cards_on_page(Page())
|
||||
|
||||
assert cards == [{
|
||||
"title": "标题",
|
||||
"read_count": "1",
|
||||
"href": "",
|
||||
"note_id": note_id,
|
||||
}]
|
||||
|
||||
|
||||
def test_pgy_card_parser_prefers_strict_note_detail_id_over_generic_segment():
|
||||
note_id = "0123456789abcdef01234567"
|
||||
internal_href = f"https://example.invalid/note/detail/{note_id}"
|
||||
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return [{
|
||||
"title": "标题",
|
||||
"read_count": "1",
|
||||
"href": internal_href,
|
||||
# This is what the old generic /note/<segment> parser produced.
|
||||
"note_id": "detail",
|
||||
"_href_candidates": [internal_href],
|
||||
"_data_note_ids": [],
|
||||
}]
|
||||
|
||||
cards = pgy.parse_cards_on_page(Page())
|
||||
|
||||
assert cards[0]["note_id"] == note_id
|
||||
|
||||
|
||||
def test_pgy_dom_parser_reads_root_data_id_and_rejects_arbitrary_href_fallback():
|
||||
captured = {}
|
||||
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
captured["script"] = script
|
||||
return []
|
||||
|
||||
assert pgy.parse_cards_on_page(Page()) == []
|
||||
assert "const dataNoteNodes = [card, ...card.querySelectorAll" in captured["script"]
|
||||
assert "const href = link?.href || '';" in captured["script"]
|
||||
assert "hrefCandidates[0]" not in captured["script"]
|
||||
|
||||
|
||||
def test_pgy_arbitrary_shared_anchor_does_not_collapse_distinct_cards():
|
||||
shared_anchor = "https://example.invalid/creator/profile"
|
||||
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return [
|
||||
{
|
||||
"title": "第一篇笔记",
|
||||
"read_count": "1",
|
||||
"like_count": "1",
|
||||
"collect_count": "1",
|
||||
"publish_time": "2026-08-01",
|
||||
"href": "",
|
||||
"note_id": "",
|
||||
"_href_candidates": [shared_anchor],
|
||||
"_data_note_ids": [],
|
||||
},
|
||||
{
|
||||
"title": "第二篇笔记",
|
||||
"read_count": "2",
|
||||
"like_count": "2",
|
||||
"collect_count": "2",
|
||||
"publish_time": "2026-08-02",
|
||||
"href": "",
|
||||
"note_id": "",
|
||||
"_href_candidates": [shared_anchor],
|
||||
"_data_note_ids": [],
|
||||
},
|
||||
]
|
||||
|
||||
cards = pgy.parse_cards_on_page(Page())
|
||||
|
||||
assert [card["href"] for card in cards] == ["", ""]
|
||||
assert pgy._card_observation_key(cards[0]) != pgy._card_observation_key(cards[1])
|
||||
|
||||
|
||||
def test_pgy_deduplicates_one_card_repeated_across_pages_before_matching(monkeypatch):
|
||||
card = {
|
||||
"title": "同一篇笔记",
|
||||
"read_count": "100",
|
||||
"like_count": "10",
|
||||
"collect_count": "5",
|
||||
"publish_time": "2026-08-01",
|
||||
"href": "",
|
||||
"note_id": "",
|
||||
}
|
||||
pages = [[dict(card)], [dict(card)], [dict(card)]]
|
||||
monkeypatch.setattr(pgy, "parse_cards_stable", lambda _page: pages.pop(0))
|
||||
monkeypatch.setattr(pgy, "go_next_page", lambda _page: bool(pages))
|
||||
|
||||
found, candidates, page_limit_hit = pgy.find_notes_for_tasks(
|
||||
object(),
|
||||
[{"record_id": "r1", "target_title": "同一篇笔记"}],
|
||||
max_pages=4,
|
||||
)
|
||||
|
||||
assert found["r1"]["read_count"] == "100"
|
||||
assert candidates == ["同一篇笔记"]
|
||||
assert page_limit_hit is False
|
||||
|
||||
|
||||
def test_pgy_missing_creator_is_blocked_but_unmatched_note_stays_retryable():
|
||||
no_creator = pgy._missing_detail_outcome(saw_search_result=False)
|
||||
no_unique_note = pgy._unmatched_note_outcome(
|
||||
candidates=["平台上的其他笔记"],
|
||||
page_limit_hit=False,
|
||||
)
|
||||
completed = cc.finalize_summary({
|
||||
"total": 2,
|
||||
"results": [
|
||||
{"record_id": "creator", **no_creator},
|
||||
{"record_id": "note", **no_unique_note},
|
||||
],
|
||||
})
|
||||
|
||||
assert completed["blocked_input"] == 1
|
||||
assert completed["retryable_failures"] == 1
|
||||
assert completed["unresolved"] == 1
|
||||
assert completed["complete"] is False
|
||||
assert pgy._missing_detail_outcome(saw_search_result=True)["status"] == "retryable_failure"
|
||||
assert pgy._unmatched_note_outcome(
|
||||
candidates=[], page_limit_hit=False
|
||||
)["reason"] == "empty_detail_page"
|
||||
assert pgy._unmatched_note_outcome(
|
||||
candidates=["候选"], page_limit_hit=True
|
||||
)["status"] == "retryable_failure"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state",
|
||||
[
|
||||
{
|
||||
"nextFound": True,
|
||||
"nextDisabled": True,
|
||||
"activePage": None,
|
||||
"lastPage": None,
|
||||
"numericPageCount": 0,
|
||||
},
|
||||
{
|
||||
"nextFound": True,
|
||||
"nextDisabled": False,
|
||||
"activePage": 7,
|
||||
"lastPage": 7,
|
||||
"numericPageCount": 7,
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_pgy_known_last_page_stops_without_clicking(state):
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
if "pgy-pagination-state" in script:
|
||||
return state
|
||||
if "pgy-pagination-click" in script:
|
||||
pytest.fail("terminal paginator must not be clicked")
|
||||
return "last-page-card-signature"
|
||||
|
||||
assert pgy.go_next_page(Page()) is False
|
||||
|
||||
|
||||
def test_pgy_clicked_pagination_without_card_change_uses_source_end_semantics():
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
if "pgy-pagination-state" in script:
|
||||
return {
|
||||
"nextFound": True,
|
||||
"nextDisabled": False,
|
||||
"activePage": 2,
|
||||
"lastPage": 7,
|
||||
"numericPageCount": 7,
|
||||
}
|
||||
if "pgy-pagination-click" in script:
|
||||
return {"clicked": True, "text": "next"}
|
||||
return "same-card-signature"
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
assert pgy.go_next_page(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
|
||||
pages = [
|
||||
[{"title": "无关视频", "play_count": "10"}],
|
||||
@@ -432,6 +686,125 @@ def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
|
||||
assert calls["next"] == 1
|
||||
|
||||
|
||||
def test_xingtu_video_search_uses_visible_input_and_source_keyboard_events():
|
||||
events = []
|
||||
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def wait_for(self, **kwargs):
|
||||
events.append(("wait", kwargs))
|
||||
|
||||
def click(self):
|
||||
events.append(("click",))
|
||||
|
||||
def press(self, key):
|
||||
events.append(("locator_press", key))
|
||||
|
||||
def fill(self, _value):
|
||||
pytest.fail("Xingtu search must preserve source key-by-key input")
|
||||
|
||||
class Keyboard:
|
||||
def type(self, value, **kwargs):
|
||||
events.append(("type", value, kwargs))
|
||||
|
||||
def press(self, key):
|
||||
events.append(("keyboard_press", key))
|
||||
|
||||
class Page:
|
||||
keyboard = Keyboard()
|
||||
|
||||
def __init__(self):
|
||||
self.selector = None
|
||||
self.input = Locator()
|
||||
|
||||
def locator(self, selector):
|
||||
self.selector = selector
|
||||
return self.input
|
||||
|
||||
page = Page()
|
||||
|
||||
assert xingtu.submit_video_search(page, "目标标题") is True
|
||||
assert ":visible" in page.selector
|
||||
assert events == [
|
||||
("wait", {"state": "visible", "timeout": 5000}),
|
||||
("click",),
|
||||
("locator_press", "Control+A"),
|
||||
("locator_press", "Backspace"),
|
||||
("type", "目标标题", {"delay": 20}),
|
||||
("keyboard_press", "Enter"),
|
||||
]
|
||||
|
||||
|
||||
def test_xingtu_only_confirmed_creator_miss_is_blocked():
|
||||
no_creator = xingtu._missing_detail_outcome(saw_search_result=False)
|
||||
no_unique_video = xingtu._unmatched_video_outcome(
|
||||
candidates=["平台上的其他视频"],
|
||||
page_limit_hit=False,
|
||||
)
|
||||
completed = cc.finalize_summary({
|
||||
"total": 2,
|
||||
"results": [
|
||||
{"record_id": "creator", **no_creator},
|
||||
{"record_id": "video", **no_unique_video},
|
||||
],
|
||||
})
|
||||
|
||||
assert completed["blocked_input"] == 1
|
||||
assert completed["retryable_failures"] == 1
|
||||
assert completed["unresolved"] == 1
|
||||
assert completed["complete"] is False
|
||||
assert no_unique_video["status"] == "retryable_failure"
|
||||
assert xingtu._missing_detail_outcome(
|
||||
saw_search_result=True
|
||||
)["status"] == "retryable_failure"
|
||||
assert xingtu._unmatched_video_outcome(
|
||||
candidates=[], page_limit_hit=False
|
||||
)["reason"] == "empty_detail_page"
|
||||
assert xingtu._unmatched_video_outcome(
|
||||
candidates=["候选"], page_limit_hit=True
|
||||
)["status"] == "retryable_failure"
|
||||
|
||||
|
||||
def test_xingtu_missing_creator_result_has_distinct_business_exception():
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
class Page:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, _script, argument=None):
|
||||
if argument is not None:
|
||||
return {"found": False}
|
||||
return False
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
with pytest.raises(xingtu.CreatorNotFound, match="未找到精确达人"):
|
||||
xingtu.open_creator_detail(Page(), "不存在的达人", "missing-id")
|
||||
|
||||
|
||||
def test_xingtu_clicked_pagination_without_card_change_is_retryable_timeout():
|
||||
class Page:
|
||||
def evaluate(self, script):
|
||||
if "const all" in script:
|
||||
return {"clicked": True, "picked": "pagination"}
|
||||
return "same-card-signature"
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
with pytest.raises(xingtu.BrowserTimeoutError, match="did not change"):
|
||||
xingtu.go_next_page(Page())
|
||||
|
||||
|
||||
def test_xingtu_normalizes_show_items_api_video():
|
||||
card = xingtu.normalize_xingtu_api_item({
|
||||
"item_id": 7654321,
|
||||
@@ -456,6 +829,9 @@ def test_xingtu_normalizes_show_items_api_video():
|
||||
|
||||
|
||||
def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
class Response:
|
||||
url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
|
||||
|
||||
@@ -477,9 +853,14 @@ def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
|
||||
|
||||
page = Page()
|
||||
cards = xingtu.setup_show_items_capture(page)
|
||||
page.callback(Response())
|
||||
response = Response()
|
||||
response_ref = weakref.ref(response)
|
||||
page.callback(response)
|
||||
del response
|
||||
gc.collect()
|
||||
|
||||
assert [card["note_id"] for card in cards] == ["1", "2"]
|
||||
assert response_ref() is None
|
||||
|
||||
|
||||
def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
|
||||
@@ -542,9 +923,221 @@ def test_browser_session_loss_detection_is_specific():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
assert pgy.is_browser_session_lost(closed)
|
||||
assert xingtu.is_browser_session_lost(closed)
|
||||
assert pgy.is_browser_session_lost(RuntimeError("Page crashed"))
|
||||
assert xingtu.is_browser_session_lost(RuntimeError("Page crashed"))
|
||||
assert pgy.is_browser_session_lost(
|
||||
RuntimeError("Execution context was destroyed, most likely because of a navigation.")
|
||||
)
|
||||
assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
|
||||
|
||||
|
||||
def test_transient_browser_error_excludes_business_timeout():
|
||||
assert is_transient_browser_error(
|
||||
pgy.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded")
|
||||
)
|
||||
assert is_transient_browser_error(
|
||||
RuntimeError("Page.goto: net::ERR_CONNECTION_RESET")
|
||||
)
|
||||
assert not is_transient_browser_error(
|
||||
TimeoutError("Login timed out after 300 seconds.")
|
||||
)
|
||||
|
||||
|
||||
def _stub_creator_session_failure(monkeypatch, tmp_path, module, error_factory):
|
||||
style = {"index": 1, "name": "style"}
|
||||
task = {
|
||||
"record_id": "record-1",
|
||||
"creator_name": "creator",
|
||||
"creator_id": None,
|
||||
"target_title": "target",
|
||||
"style_context": {"index": 1},
|
||||
}
|
||||
summary = {
|
||||
"index": 1,
|
||||
"name": "style",
|
||||
"total": 1,
|
||||
"filled": 0,
|
||||
"results": [],
|
||||
}
|
||||
group = {
|
||||
"creator_name": "creator",
|
||||
"creator_id": None,
|
||||
"tasks": [task],
|
||||
}
|
||||
sessions = []
|
||||
|
||||
class Session:
|
||||
def __init__(self, **_kwargs):
|
||||
sessions.append(self)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def fail_page_action(*_args, **_kwargs):
|
||||
raise error_factory()
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"collect_tasks_across_styles",
|
||||
lambda *_args, **_kwargs: ([task], {1: summary}),
|
||||
)
|
||||
monkeypatch.setattr(module, "group_tasks_by_creator", lambda _tasks: [group])
|
||||
monkeypatch.setattr(module, "chunk_creator_groups", lambda groups, _size: [groups])
|
||||
monkeypatch.setattr(module, "DynamicSession", Session)
|
||||
monkeypatch.setattr(module, "run_serialized_page_action", fail_page_action)
|
||||
monkeypatch.setattr(module, "_force_cleanup_session", lambda _session: None)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", tmp_path / module.__name__.split(".")[-1])
|
||||
return style, sessions
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [pgy, xingtu])
|
||||
def test_creator_collectors_retry_browser_timeout_in_three_fresh_sessions(
|
||||
monkeypatch, tmp_path, module
|
||||
):
|
||||
style, sessions = _stub_creator_session_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
lambda: module.BrowserTimeoutError("Page.goto: Timeout 60000ms exceeded"),
|
||||
)
|
||||
|
||||
summaries = module.scrape_styles([style], 1, True, None, True)
|
||||
|
||||
assert len(sessions) == 3
|
||||
assert summaries[0]["complete"] is False
|
||||
assert summaries[0]["retryable_failures"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [pgy, xingtu])
|
||||
def test_creator_collectors_do_not_retry_acceptance_terminal(
|
||||
monkeypatch, tmp_path, module
|
||||
):
|
||||
style, sessions = _stub_creator_session_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
lambda: module.AcceptanceCookieSkip("cookie unavailable in acceptance"),
|
||||
)
|
||||
|
||||
with pytest.raises(module.AcceptanceCookieSkip):
|
||||
module.scrape_styles([style], 1, True, None, True)
|
||||
|
||||
assert len(sessions) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [pgy, xingtu])
|
||||
def test_creator_collectors_do_not_retry_login_timeout_terminal(
|
||||
monkeypatch, tmp_path, module
|
||||
):
|
||||
style, sessions = _stub_creator_session_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
lambda: TimeoutError("Login timed out after 300 seconds."),
|
||||
)
|
||||
|
||||
with pytest.raises(TimeoutError, match="Login timed out"):
|
||||
module.scrape_styles([style], 1, True, None, True)
|
||||
|
||||
assert len(sessions) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_type", [xingtu.SearchQuotaExhausted, xingtu.CaptchaBlocked])
|
||||
def test_xingtu_does_not_retry_explicit_business_terminal(
|
||||
monkeypatch, tmp_path, error_type
|
||||
):
|
||||
style, sessions = _stub_creator_session_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
xingtu,
|
||||
lambda: error_type("explicit platform terminal"),
|
||||
)
|
||||
|
||||
with pytest.raises(error_type):
|
||||
xingtu.scrape_styles([style], 1, True, None, True)
|
||||
|
||||
assert len(sessions) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [pgy, xingtu])
|
||||
def test_creator_collectors_with_no_tasks_do_not_open_browser(
|
||||
monkeypatch, module
|
||||
):
|
||||
style = {"index": 1, "name": "style"}
|
||||
summary = {"index": 1, "name": "style", "total": 0, "results": []}
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"collect_tasks_across_styles",
|
||||
lambda *_args, **_kwargs: ([], {1: summary}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"DynamicSession",
|
||||
lambda **_kwargs: pytest.fail("no browser session expected"),
|
||||
)
|
||||
|
||||
assert module.scrape_styles([style], 1, True, None, True) == [summary]
|
||||
|
||||
|
||||
def test_long_page_action_does_not_retain_navigation_response():
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
response_ref = None
|
||||
|
||||
class Response:
|
||||
url = "https://example.test/final"
|
||||
status = 200
|
||||
headers = {"content-type": "text/html"}
|
||||
|
||||
class Page:
|
||||
closed = False
|
||||
|
||||
def set_default_timeout(self, _timeout):
|
||||
pass
|
||||
|
||||
def set_default_navigation_timeout(self, _timeout):
|
||||
pass
|
||||
|
||||
def goto(self, *_args, **_kwargs):
|
||||
nonlocal response_ref
|
||||
response = Response()
|
||||
response_ref = weakref.ref(response)
|
||||
return response
|
||||
|
||||
def is_closed(self):
|
||||
return self.closed
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
page = Page()
|
||||
|
||||
class Context:
|
||||
def new_page(self):
|
||||
return page
|
||||
|
||||
class Session:
|
||||
context = Context()
|
||||
|
||||
def action(_page):
|
||||
gc.collect()
|
||||
assert response_ref is not None
|
||||
assert response_ref() is None
|
||||
|
||||
navigation = run_serialized_page_action(Session(), "https://example.test", action)
|
||||
|
||||
assert navigation == {
|
||||
"url": "https://example.test/final",
|
||||
"status": 200,
|
||||
"headers": {"content-type": "text/html"},
|
||||
}
|
||||
assert page.closed is True
|
||||
|
||||
|
||||
def test_xingtu_nested_session_loss_is_reraised():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
@@ -694,6 +1287,19 @@ def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
|
||||
assert [row["record_id"] for row in summary["details"]] == ["published"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("2026-07-20T00:00:00.000+08:00", bili.date(2026, 7, 20)),
|
||||
("2026-07-20T00:00:00+08:00", bili.date(2026, 7, 20)),
|
||||
("2026-07-19T16:00:00.000Z", bili.date(2026, 7, 19)),
|
||||
("2026-07-20", bili.date(2026, 7, 20)),
|
||||
],
|
||||
)
|
||||
def test_bilibili_parse_pub_date_accepts_feishu_iso_values(raw, expected):
|
||||
assert bili.parse_pub_date(raw) == expected
|
||||
|
||||
|
||||
def test_bilibili_partial_retry_preserves_old_success():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 2,
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters.scrapling import BrowserTimeoutError
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
batch_rescrape_douyin as douyin_batch,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
batch_rescrape_xiaohongshu as xhs_batch,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools.comment_batch_checkpoint import (
|
||||
CommentBatchCheckpoint,
|
||||
is_browser_process_lost,
|
||||
)
|
||||
|
||||
|
||||
def test_page_crash_requires_a_fresh_browser():
|
||||
assert is_browser_process_lost(
|
||||
RuntimeError("Page.wait_for_timeout: Page crashed")
|
||||
)
|
||||
assert not is_browser_process_lost(RuntimeError("ordinary parse failure"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch", [xhs_batch, douyin_batch])
|
||||
def test_non_acceptance_relogin_runs_platform_script(monkeypatch, batch):
|
||||
class Policy:
|
||||
enabled = False
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(batch, "current_acceptance_policy", lambda: Policy())
|
||||
monkeypatch.setattr(
|
||||
batch.subprocess,
|
||||
"call",
|
||||
lambda command, **kwargs: calls.append((command, kwargs)) or 0,
|
||||
)
|
||||
|
||||
assert batch.do_relogin() is True
|
||||
assert calls[0][0][1] == str(batch.RELOGIN_SCRIPT)
|
||||
|
||||
|
||||
def test_comment_checkpoint_matches_both_note_id_and_url(tmp_path):
|
||||
path = tmp_path / "checkpoint.json"
|
||||
checkpoint = CommentBatchCheckpoint(
|
||||
"douyin", business_date="2026-08-09", path=path
|
||||
)
|
||||
checkpoint.mark_completed(7, "https://www.douyin.com/video/7")
|
||||
|
||||
restored = CommentBatchCheckpoint(
|
||||
"douyin", business_date="2026-08-09", path=path
|
||||
)
|
||||
assert restored.contains(7, "https://www.douyin.com/video/7") is True
|
||||
assert restored.contains(7, "https://www.douyin.com/video/changed") is False
|
||||
|
||||
restored.clear()
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
def test_xhs_page_crash_retries_same_note_with_new_scraper_call(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def scrape(url, *_args):
|
||||
calls.append(url)
|
||||
if len(calls) == 1:
|
||||
raise RuntimeError("Page.wait_for_timeout: Page crashed")
|
||||
return [], {"stats": {}}
|
||||
|
||||
monkeypatch.setattr(xhs_batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(xhs_batch.time, "sleep", lambda _seconds: None)
|
||||
|
||||
comments, result, error = xhs_batch.scrape_with_fresh_browser("xhs-url")
|
||||
|
||||
assert comments == []
|
||||
assert result == {"stats": {}}
|
||||
assert error is None
|
||||
assert calls == ["xhs-url", "xhs-url"]
|
||||
|
||||
|
||||
def test_douyin_page_crash_retries_same_note_with_new_scraper_call(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def scrape(url, *_args):
|
||||
calls.append(url)
|
||||
if len(calls) == 1:
|
||||
raise RuntimeError("Page.wait_for_timeout: Page crashed")
|
||||
return [], {"stats": {}}
|
||||
|
||||
monkeypatch.setattr(douyin_batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(douyin_batch.time, "sleep", lambda _seconds: None)
|
||||
|
||||
comments, result, error = douyin_batch.scrape_with_fresh_browser("dy-url")
|
||||
|
||||
assert comments == []
|
||||
assert result == {"stats": {}}
|
||||
assert error is None
|
||||
assert calls == ["dy-url", "dy-url"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch", "url"),
|
||||
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
|
||||
)
|
||||
def test_transient_timeout_gets_three_fresh_browser_attempts(monkeypatch, batch, url):
|
||||
calls = []
|
||||
|
||||
def scrape(scrape_url, *_args):
|
||||
calls.append(scrape_url)
|
||||
if len(calls) < 3:
|
||||
raise BrowserTimeoutError("Page.goto: Timeout 90000ms exceeded")
|
||||
return [], {"stats": {}}
|
||||
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
|
||||
|
||||
comments, result, error = batch.scrape_with_fresh_browser(url)
|
||||
|
||||
assert comments == []
|
||||
assert result == {"stats": {}}
|
||||
assert error is None
|
||||
assert calls == [url, url, url]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch", "url"),
|
||||
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
|
||||
)
|
||||
def test_transient_network_failure_exhausts_three_attempts_honestly(
|
||||
monkeypatch, batch, url
|
||||
):
|
||||
calls = []
|
||||
|
||||
def scrape(scrape_url, *_args):
|
||||
calls.append(scrape_url)
|
||||
raise RuntimeError("Page.goto: net::ERR_CONNECTION_RESET")
|
||||
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
|
||||
|
||||
comments, result, error = batch.scrape_with_fresh_browser(url)
|
||||
|
||||
assert comments is None
|
||||
assert result is None
|
||||
assert error is not None
|
||||
assert "ERR_CONNECTION_RESET" in str(error)
|
||||
assert calls == [url, url, url]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch", "url"),
|
||||
[(xhs_batch, "xhs-url"), (douyin_batch, "dy-url")],
|
||||
)
|
||||
def test_terminal_parse_failure_is_not_retried(monkeypatch, batch, url):
|
||||
calls = []
|
||||
|
||||
def scrape(scrape_url, *_args):
|
||||
calls.append(scrape_url)
|
||||
raise RuntimeError("ordinary terminal parse failure")
|
||||
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
|
||||
comments, result, error = batch.scrape_with_fresh_browser(url)
|
||||
|
||||
assert comments is None
|
||||
assert result is None
|
||||
assert error is not None
|
||||
assert calls == [url]
|
||||
|
||||
|
||||
def test_xhs_failed_note_keeps_success_checkpoint_and_fails_batch(monkeypatch):
|
||||
class Checkpoint:
|
||||
completed = []
|
||||
cleared = False
|
||||
|
||||
def __init__(self, _platform):
|
||||
pass
|
||||
|
||||
def contains(self, _note_id, _url):
|
||||
return False
|
||||
|
||||
def mark_completed(self, note_id, url):
|
||||
self.completed.append((note_id, url))
|
||||
|
||||
def clear(self):
|
||||
self.cleared = True
|
||||
|
||||
rows = [
|
||||
{"id": 1, "url": "xhs-1", "title": "one"},
|
||||
{"id": 2, "url": "xhs-2", "title": "two"},
|
||||
]
|
||||
|
||||
def scrape(url, *_args):
|
||||
if url == "xhs-2":
|
||||
raise RuntimeError("ordinary terminal failure")
|
||||
return ([{"level": "comment"}], {
|
||||
"title": "one",
|
||||
"stats": {"logged_in": True, "note_metrics": {}},
|
||||
})
|
||||
|
||||
monkeypatch.setattr(xhs_batch, "CommentBatchCheckpoint", Checkpoint)
|
||||
monkeypatch.setattr(xhs_batch, "collect_urls", lambda: rows)
|
||||
monkeypatch.setattr(xhs_batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(xhs_batch.scraper, "write_outputs", lambda *_args: (None, None))
|
||||
monkeypatch.setattr(xhs_batch, "upsert_metrics", lambda *_args: None)
|
||||
monkeypatch.setattr(xhs_batch.db, "replace_comments", lambda *_args, **_kwargs: None)
|
||||
|
||||
assert xhs_batch.main() == 1
|
||||
assert Checkpoint.completed == [(1, "xhs-1")]
|
||||
assert Checkpoint.cleared is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch", "platform"),
|
||||
[(xhs_batch, "xhs"), (douyin_batch, "dy")],
|
||||
)
|
||||
def test_incomplete_comments_preserve_db_but_remain_resume_eligible(
|
||||
monkeypatch, batch, platform
|
||||
):
|
||||
class Checkpoint:
|
||||
def __init__(self):
|
||||
self.completed = []
|
||||
self.cleared = False
|
||||
|
||||
def contains(self, _note_id, _url):
|
||||
return False
|
||||
|
||||
def mark_completed(self, note_id, url):
|
||||
self.completed.append((note_id, url))
|
||||
|
||||
def clear(self):
|
||||
self.cleared = True
|
||||
|
||||
checkpoint = Checkpoint()
|
||||
rows = [
|
||||
{
|
||||
"id": 1,
|
||||
"url": f"{platform}-partial",
|
||||
"title": "partial",
|
||||
"comment_count": 5,
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"url": f"{platform}-failed",
|
||||
"title": "failed",
|
||||
"comment_count": 5,
|
||||
},
|
||||
]
|
||||
|
||||
def scrape(url, *_args):
|
||||
if url.endswith("-failed"):
|
||||
raise RuntimeError("ordinary terminal failure")
|
||||
comments = [{"level": "comment"}]
|
||||
stats = {
|
||||
"logged_in": False,
|
||||
"api_total": 5,
|
||||
"note_metrics": {"comment_count": 5},
|
||||
}
|
||||
if batch is xhs_batch:
|
||||
return comments, {"title": "partial", "stats": stats}
|
||||
return comments, {"stats": stats}
|
||||
|
||||
monkeypatch.setattr(batch, "CommentBatchCheckpoint", lambda _platform: checkpoint)
|
||||
monkeypatch.setattr(batch, "collect_urls", lambda: rows)
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(batch.scraper, "write_outputs", lambda *_args: (None, None))
|
||||
monkeypatch.setattr(batch, "upsert_metrics", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
batch.db,
|
||||
"replace_comments",
|
||||
lambda *_args, **_kwargs: pytest.fail("incomplete comments must not replace DB"),
|
||||
)
|
||||
|
||||
assert batch.main() == 1
|
||||
assert checkpoint.completed == []
|
||||
assert checkpoint.cleared is False
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.modules.content_marketing import chanmama_scraper, weekly_summary_all
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
analyze_comments,
|
||||
@@ -71,17 +71,13 @@ def test_daily_card_sends_only_to_wang_yunlong(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"message_id": "om_test"}),
|
||||
stderr="",
|
||||
)
|
||||
def fake_send(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
monkeypatch.setattr(daily_report_card.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(daily_report_card, "send_lark_bot_message", fake_send)
|
||||
results = daily_report_card.send_card_to_recipients(
|
||||
{"schema": "2.0", "body": {"elements": []}},
|
||||
("ou_other_a", "ou_other_b"),
|
||||
@@ -89,9 +85,9 @@ def test_daily_card_sends_only_to_wang_yunlong(
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(commands) == 1
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["profile"] == "hermes-analyzer"
|
||||
|
||||
|
||||
def test_comment_summary_sends_only_to_wang_yunlong(
|
||||
@@ -99,17 +95,17 @@ def test_comment_summary_sends_only_to_wang_yunlong(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||||
def fake_send(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
monkeypatch.setattr(analyze_comments.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(analyze_comments, "send_lark_bot_message", fake_send)
|
||||
analyze_comments.send_feishu_summary("summary", "ou_other")
|
||||
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["profile"] == "hermes-analyzer"
|
||||
|
||||
|
||||
def test_note_report_sends_only_to_wang_yunlong(
|
||||
@@ -117,19 +113,18 @@ def test_note_report_sends_only_to_wang_yunlong(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
monkeypatch.setattr(analyze_note.sys, "platform", "linux")
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
analyze_note,
|
||||
"build_card_payload",
|
||||
lambda *args, **kwargs: {"schema": "2.0"},
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||||
def fake_send(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
monkeypatch.setattr(analyze_note.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(analyze_note, "send_lark_bot_message", fake_send)
|
||||
analyze_note.send_feishu_report(
|
||||
{},
|
||||
{},
|
||||
@@ -141,8 +136,8 @@ def test_note_report_sends_only_to_wang_yunlong(
|
||||
tmp_path / "report.md",
|
||||
)
|
||||
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert calls[0]["profile"] == "hermes-analyzer"
|
||||
|
||||
|
||||
def test_relogin_notifications_resolve_only_to_wang_yunlong(
|
||||
@@ -153,5 +148,5 @@ def test_relogin_notifications_resolve_only_to_wang_yunlong(
|
||||
|
||||
assert (
|
||||
friday_relogin_parallel.recipient_for_platform("pgy", "ou_other")
|
||||
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
)
|
||||
|
||||
@@ -0,0 +1,596 @@
|
||||
import sys
|
||||
from datetime import date
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import monthly_summary_all, weekly_summary_all
|
||||
from gyxx_flow.modules.content_marketing.data.tools import db
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
generate_creator_report as report,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.feishu_doc_import import (
|
||||
FeishuDocImportTimeout,
|
||||
)
|
||||
|
||||
|
||||
def _prepare_main(monkeypatch, tmp_path, *, doc_url, saved):
|
||||
monkeypatch.setattr(report, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"_compute_last_month",
|
||||
lambda: (date(2026, 7, 1), date(2026, 7, 31)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"generate_report",
|
||||
lambda style_filter=None: ("report", [], {}, {}),
|
||||
)
|
||||
monkeypatch.setattr(report, "load_existing_creator_report", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
report, "create_feishu_doc", lambda *args, **_kwargs: doc_url
|
||||
)
|
||||
monkeypatch.setattr(report, "save_to_db", lambda *args: saved)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_creator_report.py"])
|
||||
|
||||
|
||||
def test_main_requires_both_doc_and_database_effects(monkeypatch, tmp_path):
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url="https://example/doc", saved=True)
|
||||
assert report.main() == 0
|
||||
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=True)
|
||||
assert report.main() == 1
|
||||
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url="https://example/doc", saved=False)
|
||||
assert report.main() == 1
|
||||
|
||||
|
||||
def test_main_dry_run_does_not_require_external_effects(monkeypatch, tmp_path):
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=False)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_creator_report.py", "--dry-run"])
|
||||
assert report.main() == 0
|
||||
|
||||
|
||||
def _import_timeout(tmp_path):
|
||||
return FeishuDocImportTimeout(
|
||||
"known import ticket did not become ready",
|
||||
audit_path=tmp_path / "ticket-audit.json",
|
||||
)
|
||||
|
||||
|
||||
def test_creator_report_timeout_is_persisted_as_ambiguous(monkeypatch, tmp_path):
|
||||
captured = {}
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=True)
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"load_existing_creator_report",
|
||||
lambda *_args: {"doc_url": "", "doc_title": "", "status": "doc_ambiguous"},
|
||||
)
|
||||
|
||||
def raise_timeout(*_args, **kwargs):
|
||||
captured["resume_only"] = kwargs.get("resume_only")
|
||||
raise _import_timeout(tmp_path)
|
||||
|
||||
def capture_save(*args):
|
||||
captured["doc_url"] = args[-2]
|
||||
captured["status"] = args[-1]
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(report, "create_feishu_doc", raise_timeout)
|
||||
monkeypatch.setattr(report, "save_to_db", capture_save)
|
||||
|
||||
assert report.main() == 1
|
||||
assert captured == {
|
||||
"doc_url": None,
|
||||
"resume_only": True,
|
||||
"status": "doc_ambiguous",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "processor_name", "creator_name"),
|
||||
(
|
||||
(weekly_summary_all, "process_one_style", "create_feishu_doc"),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"process_one_style_monthly",
|
||||
"create_feishu_doc_monthly",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_summary_timeout_is_persisted_as_ambiguous(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
processor_name,
|
||||
creator_name,
|
||||
):
|
||||
saved = {}
|
||||
creator_call = {}
|
||||
notes = [{"id": 1, "view_count": 10, "comment_count": 1}]
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(module, "REPORTS_DIR", tmp_path)
|
||||
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
|
||||
if module is weekly_summary_all:
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"load_existing_summary_delivery",
|
||||
lambda _style: {
|
||||
"doc_url": "",
|
||||
"doc_title": "",
|
||||
"status": "doc_ambiguous",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(module, "get_last_week_notes", lambda _style: notes)
|
||||
monkeypatch.setattr(module, "generate_summary", lambda *_args: "summary")
|
||||
else:
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"load_existing_monthly_summary_delivery",
|
||||
lambda _style: {
|
||||
"doc_url": "",
|
||||
"doc_title": "",
|
||||
"status": "doc_ambiguous",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(module, "get_last_month_notes", lambda _style: notes)
|
||||
monkeypatch.setattr(module, "generate_monthly_summary", lambda *_args: "summary")
|
||||
monkeypatch.setattr(module, "run_single_note_analysis", lambda _note_id: "report")
|
||||
|
||||
def raise_timeout(*_args, **kwargs):
|
||||
creator_call["resume_only"] = kwargs.get("resume_only")
|
||||
raise _import_timeout(tmp_path)
|
||||
|
||||
def capture_save(*args):
|
||||
saved["doc_url"] = args[-3]
|
||||
saved["doc_title"] = args[-2]
|
||||
saved["status"] = args[-1]
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(module, creator_name, raise_timeout)
|
||||
save_name = "save_to_db" if module is weekly_summary_all else "save_to_db_monthly"
|
||||
monkeypatch.setattr(module, save_name, capture_save)
|
||||
|
||||
processor = getattr(module, processor_name)
|
||||
arguments = ("宙斯", ("base", "table", "url"))
|
||||
if module is weekly_summary_all:
|
||||
arguments += (False,)
|
||||
result = processor(*arguments)
|
||||
|
||||
assert result["status"] == "doc_ambiguous"
|
||||
assert saved == {
|
||||
"doc_url": None,
|
||||
"doc_title": None,
|
||||
"status": "doc_ambiguous",
|
||||
}
|
||||
assert creator_call == {"resume_only": True}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "processor_name", "creator_name"),
|
||||
(
|
||||
(weekly_summary_all, "process_one_style", "create_feishu_doc"),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"process_one_style_monthly",
|
||||
"create_feishu_doc_monthly",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_summary_pg_failure_keeps_real_document_receipt_and_fails_result(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
processor_name,
|
||||
creator_name,
|
||||
):
|
||||
doc_url = "https://example.feishu.cn/docx/real-document"
|
||||
notes = [{"id": 1, "view_count": 10, "comment_count": 1}]
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(module, "REPORTS_DIR", tmp_path)
|
||||
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
|
||||
if module is weekly_summary_all:
|
||||
monkeypatch.setattr(module, "load_existing_summary_delivery", lambda _style: None)
|
||||
monkeypatch.setattr(module, "get_last_week_notes", lambda _style: notes)
|
||||
monkeypatch.setattr(module, "generate_summary", lambda *_args: "summary")
|
||||
else:
|
||||
monkeypatch.setattr(
|
||||
module, "load_existing_monthly_summary_delivery", lambda _style: None
|
||||
)
|
||||
monkeypatch.setattr(module, "get_last_month_notes", lambda _style: notes)
|
||||
monkeypatch.setattr(module, "generate_monthly_summary", lambda *_args: "summary")
|
||||
monkeypatch.setattr(module, "run_single_note_analysis", lambda _note_id: "report")
|
||||
monkeypatch.setattr(
|
||||
module, creator_name, lambda *_args, **_kwargs: (doc_url, None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module, "write_to_target_table", lambda *_args, **_kwargs: True
|
||||
)
|
||||
save_name = "save_to_db" if module is weekly_summary_all else "save_to_db_monthly"
|
||||
monkeypatch.setattr(module, save_name, lambda *_args: False)
|
||||
|
||||
processor = getattr(module, processor_name)
|
||||
arguments = ("宙斯", ("base", "table", "url"))
|
||||
if module is weekly_summary_all:
|
||||
arguments += (False,)
|
||||
result = processor(*arguments)
|
||||
|
||||
assert result["status"] == "db_failed"
|
||||
assert result["doc_url"] == doc_url
|
||||
assert result["target_table_written"] is True
|
||||
assert not weekly_summary_all.summary_results_succeeded([result])
|
||||
|
||||
|
||||
def test_creator_existing_period_document_is_never_imported_again(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
_prepare_main(monkeypatch, tmp_path, doc_url=None, saved=False)
|
||||
existing_url = "https://example.feishu.cn/docx/existing-creator-report"
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"load_existing_creator_report",
|
||||
lambda *_args: {
|
||||
"doc_url": existing_url,
|
||||
"doc_title": "existing",
|
||||
"status": "doc_ambiguous",
|
||||
},
|
||||
)
|
||||
|
||||
def fail_import(*_args, **_kwargs):
|
||||
raise AssertionError("an existing period URL must not be imported again")
|
||||
|
||||
monkeypatch.setattr(report, "create_feishu_doc", fail_import)
|
||||
monkeypatch.setattr(
|
||||
report,
|
||||
"save_to_db",
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
AssertionError("completed period must not be overwritten")
|
||||
),
|
||||
)
|
||||
|
||||
assert report.main() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "processor_name", "loader_name", "creator_name"),
|
||||
(
|
||||
(
|
||||
weekly_summary_all,
|
||||
"process_one_style",
|
||||
"load_existing_summary_delivery",
|
||||
"create_feishu_doc",
|
||||
),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"process_one_style_monthly",
|
||||
"load_existing_monthly_summary_delivery",
|
||||
"create_feishu_doc_monthly",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_completed_summary_period_skips_all_external_work(
|
||||
monkeypatch,
|
||||
module,
|
||||
processor_name,
|
||||
loader_name,
|
||||
creator_name,
|
||||
):
|
||||
existing_url = "https://example.feishu.cn/docx/existing-summary"
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
loader_name,
|
||||
lambda _style: {
|
||||
"doc_url": existing_url,
|
||||
"doc_title": "existing",
|
||||
"status": "ok",
|
||||
},
|
||||
)
|
||||
|
||||
def fail_work(*_args, **_kwargs):
|
||||
raise AssertionError("completed summary period must be skipped")
|
||||
|
||||
monkeypatch.setattr(module, "find_target_fields", fail_work)
|
||||
monkeypatch.setattr(module, creator_name, fail_work)
|
||||
processor = getattr(module, processor_name)
|
||||
arguments = ("宙斯", ("base", "table", "url"))
|
||||
if module is weekly_summary_all:
|
||||
arguments += (False,)
|
||||
|
||||
result = processor(*arguments)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["doc_url"] == existing_url
|
||||
assert result["already_complete"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "processor_name", "loader_name", "creator_name", "updater_name"),
|
||||
(
|
||||
(
|
||||
weekly_summary_all,
|
||||
"process_one_style",
|
||||
"load_existing_summary_delivery",
|
||||
"create_feishu_doc",
|
||||
"update_existing_summary_delivery_status",
|
||||
),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"process_one_style_monthly",
|
||||
"load_existing_monthly_summary_delivery",
|
||||
"create_feishu_doc_monthly",
|
||||
"update_existing_monthly_summary_delivery_status",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_failed_base_delivery_reuses_existing_document_without_import(
|
||||
monkeypatch,
|
||||
module,
|
||||
processor_name,
|
||||
loader_name,
|
||||
creator_name,
|
||||
updater_name,
|
||||
):
|
||||
existing_url = "https://example.feishu.cn/docx/existing-summary"
|
||||
write_call = {}
|
||||
status_call = {}
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
loader_name,
|
||||
lambda _style: {
|
||||
"doc_url": existing_url,
|
||||
"doc_title": "existing",
|
||||
"status": "feishu_write_failed",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(module, "find_target_fields", lambda *_args: ("time", "summary"))
|
||||
|
||||
def capture_write(*args, **kwargs):
|
||||
write_call["args"] = args
|
||||
write_call["kwargs"] = kwargs
|
||||
return True
|
||||
|
||||
def capture_status(style, status):
|
||||
status_call.update({"style": style, "status": status})
|
||||
return True
|
||||
|
||||
def fail_import(*_args, **_kwargs):
|
||||
raise AssertionError("an existing document must not be imported again")
|
||||
|
||||
monkeypatch.setattr(module, "write_to_target_table", capture_write)
|
||||
monkeypatch.setattr(module, updater_name, capture_status)
|
||||
monkeypatch.setattr(module, creator_name, fail_import)
|
||||
processor = getattr(module, processor_name)
|
||||
arguments = ("宙斯", ("base", "table", "url"))
|
||||
if module is weekly_summary_all:
|
||||
arguments += (False,)
|
||||
|
||||
result = processor(*arguments)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["doc_url"] == existing_url
|
||||
assert result["reused_document"] is True
|
||||
assert write_call["args"][5] == existing_url
|
||||
assert status_call == {"style": "宙斯", "status": "ok"}
|
||||
if module is monthly_summary_all:
|
||||
assert write_call["kwargs"] == {
|
||||
"time_label": monthly_summary_all.TIME_LABEL,
|
||||
"report_label": "月度汇总",
|
||||
}
|
||||
|
||||
|
||||
def test_business_date_drives_default_report_periods(monkeypatch):
|
||||
for name in ("WEEK_SINCE", "WEEK_UNTIL", "MONTH_SINCE", "MONTH_UNTIL"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-09")
|
||||
|
||||
assert report._compute_last_month() == (date(2026, 7, 1), date(2026, 7, 31))
|
||||
assert weekly_summary_all._resolve_week_period() == (
|
||||
date(2026, 7, 27),
|
||||
date(2026, 8, 2),
|
||||
)
|
||||
assert monthly_summary_all._resolve_month_period() == (
|
||||
date(2026, 7, 1),
|
||||
date(2026, 7, 31),
|
||||
)
|
||||
|
||||
|
||||
def test_explicit_period_overrides_take_priority_over_business_date(monkeypatch):
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "not-a-date")
|
||||
monkeypatch.setenv("WEEK_SINCE", "2026-06-01")
|
||||
monkeypatch.setenv("WEEK_UNTIL", "2026-06-07")
|
||||
monkeypatch.setenv("MONTH_SINCE", "2026-05-01")
|
||||
monkeypatch.setenv("MONTH_UNTIL", "2026-05-31")
|
||||
|
||||
assert weekly_summary_all._resolve_week_period() == (
|
||||
date(2026, 6, 1),
|
||||
date(2026, 6, 7),
|
||||
)
|
||||
assert monthly_summary_all._resolve_month_period() == (
|
||||
date(2026, 5, 1),
|
||||
date(2026, 5, 31),
|
||||
)
|
||||
assert report._compute_last_month() == (date(2026, 5, 1), date(2026, 5, 31))
|
||||
|
||||
|
||||
def test_invalid_business_date_and_unpaired_overrides_fail_closed(monkeypatch):
|
||||
for name in ("WEEK_SINCE", "WEEK_UNTIL", "MONTH_SINCE", "MONTH_UNTIL"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026/08/09")
|
||||
|
||||
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
|
||||
report._compute_last_month()
|
||||
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
|
||||
weekly_summary_all._resolve_week_period()
|
||||
with pytest.raises(ValueError, match="GYXX_BUSINESS_DATE"):
|
||||
monthly_summary_all._resolve_month_period()
|
||||
|
||||
monkeypatch.delenv("GYXX_BUSINESS_DATE")
|
||||
monkeypatch.setenv("WEEK_SINCE", "2026-06-01")
|
||||
with pytest.raises(ValueError, match="configured together"):
|
||||
weekly_summary_all._resolve_week_period()
|
||||
monkeypatch.delenv("WEEK_SINCE")
|
||||
monkeypatch.setenv("MONTH_SINCE", "2026-05-01")
|
||||
with pytest.raises(ValueError, match="configured together"):
|
||||
report._compute_last_month()
|
||||
with pytest.raises(ValueError, match="configured together"):
|
||||
monthly_summary_all._resolve_month_period()
|
||||
|
||||
|
||||
class _RecordingCursor:
|
||||
def __init__(self):
|
||||
self.statements = []
|
||||
|
||||
def execute(self, statement, params):
|
||||
self.statements.append((statement, params))
|
||||
|
||||
def fetchone(self):
|
||||
return (7,)
|
||||
|
||||
|
||||
class _RecordingConnection:
|
||||
def __init__(self):
|
||||
self.cursor_instance = _RecordingCursor()
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_instance
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "save_name", "table_name"),
|
||||
(
|
||||
(report, "save_to_db", "cmt_creator_report"),
|
||||
(weekly_summary_all, "save_to_db", "cmt_weekly_summary"),
|
||||
(monthly_summary_all, "save_to_db_monthly", "cmt_monthly_summary"),
|
||||
),
|
||||
)
|
||||
def test_empty_upsert_values_preserve_existing_document_identity(
|
||||
monkeypatch,
|
||||
module,
|
||||
save_name,
|
||||
table_name,
|
||||
):
|
||||
connection = _RecordingConnection()
|
||||
if module is report:
|
||||
monkeypatch.setattr(report, "get_db_config", lambda: {})
|
||||
monkeypatch.setattr(report.psycopg, "connect", lambda **_kwargs: connection)
|
||||
saved = report.save_to_db(
|
||||
"summary",
|
||||
[],
|
||||
{},
|
||||
{},
|
||||
"2026-07-01",
|
||||
"2026-07-31",
|
||||
None,
|
||||
"doc_ambiguous",
|
||||
)
|
||||
else:
|
||||
monkeypatch.setattr(module, "dry_run_global", False)
|
||||
monkeypatch.setattr(db, "get_conn", lambda: connection)
|
||||
saved = getattr(module, save_name)(
|
||||
"宙斯",
|
||||
[],
|
||||
"summary",
|
||||
None,
|
||||
None,
|
||||
"doc_ambiguous",
|
||||
)
|
||||
|
||||
assert saved is True
|
||||
upsert = connection.cursor_instance.statements[-1][0]
|
||||
assert "COALESCE" in upsert
|
||||
assert "NULLIF(EXCLUDED.doc_url, '')" in upsert
|
||||
assert f"{table_name}.doc_url" in upsert
|
||||
assert "NULLIF(EXCLUDED.doc_title, '')" in upsert
|
||||
assert f"{table_name}.doc_title" in upsert
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"module",
|
||||
"function_name",
|
||||
"args",
|
||||
"expected_title",
|
||||
"source_filename",
|
||||
"operation_prefix",
|
||||
),
|
||||
(
|
||||
(
|
||||
report,
|
||||
"create_feishu_doc",
|
||||
("report", "2026-07-01", "2026-07-31"),
|
||||
"达人合作数据筛选与报价分析报告(2026-07-01~07-31)",
|
||||
"creator_report_doc.md",
|
||||
"content-creator-report-",
|
||||
),
|
||||
(
|
||||
weekly_summary_all,
|
||||
"create_feishu_doc",
|
||||
("宙斯", "summary"),
|
||||
"宙斯 周笔记汇总(2026-07-27~08-02)",
|
||||
"weekly_summary_doc.md",
|
||||
"content-weekly-summary-",
|
||||
),
|
||||
(
|
||||
monthly_summary_all,
|
||||
"create_feishu_doc_monthly",
|
||||
("宙斯", "summary"),
|
||||
"宙斯 月度汇总(2026-07-01~07-31)",
|
||||
"monthly_summary_doc.md",
|
||||
"content-monthly-summary-",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_feishu_doc_import_preserves_title_body_and_root_target(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
module,
|
||||
function_name,
|
||||
args,
|
||||
expected_title,
|
||||
source_filename,
|
||||
operation_prefix,
|
||||
):
|
||||
captured = {}
|
||||
|
||||
def fake_import(markdown, **kwargs):
|
||||
captured["markdown"] = markdown
|
||||
captured.update(kwargs)
|
||||
return SimpleNamespace(url="https://example/doc")
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"PATHS",
|
||||
SimpleNamespace(tmp_root=tmp_path, evidence_root=tmp_path / "evidence"),
|
||||
)
|
||||
monkeypatch.setattr(module, "import_markdown_document", fake_import)
|
||||
if module is weekly_summary_all:
|
||||
monkeypatch.setattr(module, "TIME_LABEL", "2026-07-27~08-02")
|
||||
elif module is monthly_summary_all:
|
||||
monkeypatch.setattr(module, "TIME_LABEL", "2026-07-01~07-31")
|
||||
|
||||
getattr(module, function_name)(*args)
|
||||
|
||||
assert captured["markdown"] == args[0 if module is report else 1]
|
||||
assert captured["title"] == expected_title
|
||||
assert captured["source_filename"] == source_filename
|
||||
assert captured["working_directory"] == tmp_path
|
||||
assert captured["audit_directory"] == tmp_path / "evidence" / "feishu_doc_import"
|
||||
assert captured["identity"] == "user"
|
||||
assert captured.get("folder_token") is None
|
||||
assert captured["operation_id"].startswith(operation_prefix)
|
||||
assert len(captured["operation_id"].rsplit("-", 1)[1]) == 40
|
||||
assert captured["resume_only"] is False
|
||||
@@ -1,5 +1,11 @@
|
||||
import io
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing import daily_marketing_report as daily
|
||||
@@ -47,6 +53,41 @@ class DailyMarketingReportTests(unittest.TestCase):
|
||||
),
|
||||
)
|
||||
|
||||
def test_disabled_dynamic_route_keeps_report_but_skips_feishu_send(self):
|
||||
facts = ([], [], [], [], {})
|
||||
with tempfile.TemporaryDirectory() as directory, \
|
||||
patch.object(daily, "PATHS", SimpleNamespace(exports_root=Path(directory))), \
|
||||
patch.object(daily, "_latest_report_date", return_value=date(2026, 8, 7)), \
|
||||
patch.object(daily, "load_report_facts", return_value=facts), \
|
||||
patch.object(daily, "build_daily_prompt", return_value="prompt"), \
|
||||
patch.object(daily, "call_hermes_analyzer", return_value="report"), \
|
||||
patch.object(daily, "require_valid_hermes_report", return_value="report"), \
|
||||
patch.object(daily, "normalize_report", return_value="report"), \
|
||||
patch.object(daily, "generate_dashboard"), \
|
||||
patch.object(
|
||||
daily,
|
||||
"resolve_notification_route",
|
||||
return_value=SimpleNamespace(
|
||||
enabled=False,
|
||||
open_ids=(),
|
||||
app_profile="hermes-analyzer",
|
||||
),
|
||||
), \
|
||||
patch.object(daily, "send_daily_report_cards") as send, \
|
||||
patch.object(sys, "argv", ["daily_marketing_report.py", "--send"]):
|
||||
output = io.StringIO()
|
||||
with redirect_stdout(output):
|
||||
self.assertEqual(daily.main(), 0)
|
||||
|
||||
report_path = (
|
||||
Path(directory)
|
||||
/ "summary"
|
||||
/ "daily_marketing_report_2026-08-07.md"
|
||||
)
|
||||
self.assertTrue(report_path.is_file())
|
||||
self.assertIn("动态配置中禁用", output.getvalue())
|
||||
send.assert_not_called()
|
||||
|
||||
def test_hermes_transport_error_is_not_accepted_as_report(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "Hermes 分析失败"):
|
||||
daily.require_valid_hermes_report("API call failed after 3 retries: Connection error.")
|
||||
@@ -142,6 +183,25 @@ class DailyMarketingReportTests(unittest.TestCase):
|
||||
self.assertEqual(valid["engagement_count_num"], 2143)
|
||||
self.assertEqual(valid["publish_status"], "已记录发布时间")
|
||||
|
||||
def test_style_note_signal_keeps_complete_count_and_reports_metric_coverage(self):
|
||||
signals = daily._build_style_signals(
|
||||
["宙斯"],
|
||||
[],
|
||||
[
|
||||
{"style_name": "宙斯", "exposure_count": 1000},
|
||||
{"style_name": "宙斯", "exposure_count": None},
|
||||
],
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
)
|
||||
|
||||
self.assertEqual(signals["宙斯"]["note_count"], 2)
|
||||
self.assertEqual(signals["宙斯"]["recent_note_exposure"], 1000)
|
||||
self.assertEqual(signals["宙斯"]["metric_note_count"], 1)
|
||||
self.assertEqual(signals["宙斯"]["metric_missing_count"], 1)
|
||||
self.assertEqual(signals["宙斯"]["metric_coverage_rate"], 0.5)
|
||||
|
||||
def test_load_report_facts_tracks_all_styles_for_report_date(self):
|
||||
metrics = {
|
||||
"宙斯": {"light": "🟢"},
|
||||
@@ -156,6 +216,7 @@ class DailyMarketingReportTests(unittest.TestCase):
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_note_inventory_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
@@ -185,6 +246,7 @@ class DailyMarketingReportTests(unittest.TestCase):
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_note_inventory_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
|
||||
@@ -49,6 +49,83 @@ class DailyReportCardTests(unittest.TestCase):
|
||||
self.assertNotIn("![全域种草数据看板]", analysis_text)
|
||||
self.assertIn("波塞冬:继续观察", analysis_text)
|
||||
self.assertIn("退款订单比上升", analysis_text)
|
||||
self.assertEqual(analysis["schema"], "2.0")
|
||||
self.assertEqual(analysis["config"]["width_mode"], "fill")
|
||||
self.assertEqual(
|
||||
[item["tag"] for item in analysis["body"]["elements"]],
|
||||
["column_set", "collapsible_panel", "column_set"],
|
||||
)
|
||||
self.assertNotEqual(
|
||||
analysis["body"]["elements"],
|
||||
[{"tag": "markdown", "content": report}],
|
||||
)
|
||||
|
||||
def test_analysis_card_structures_full_five_section_report(self):
|
||||
report = """品牌SKU营销运营日报
|
||||
日期:2026年8月1日
|
||||
|
||||
一、今日经营概览
|
||||
销量:整体下降。
|
||||
今日核心结论:流量收缩,转化承压。
|
||||
|
||||
二、SKU表现
|
||||
|SKU|今日销量|综合判断|建议动作|
|
||||
|---|---|---|---|
|
||||
|星云2|128|继续放大|维持资源|
|
||||
|星迹2|33|先修承接|优化详情页|
|
||||
|
||||
注:曝光为累计值。
|
||||
|
||||
三、重点SKU分析
|
||||
星云2:经营表现健康,继续放大。
|
||||
|
||||
四、达人营销分析
|
||||
最佳渠道:抖音。
|
||||
|
||||
五、明日运营建议
|
||||
1. 优化星迹2详情页。
|
||||
2. 补充逐星内容。
|
||||
"""
|
||||
card = build_analysis_card("2026-08-01", report)
|
||||
serialized = json.dumps(card, ensure_ascii=False)
|
||||
|
||||
self.assertEqual(
|
||||
[item["tag"] for item in card["body"]["elements"]],
|
||||
[
|
||||
"column_set",
|
||||
"table",
|
||||
"collapsible_panel",
|
||||
"collapsible_panel",
|
||||
"column_set",
|
||||
],
|
||||
)
|
||||
self.assertEqual(len(card["body"]["elements"][1]["rows"]), 2)
|
||||
sku_panel, marketing_panel = card["body"]["elements"][2:4]
|
||||
self.assertTrue(sku_panel["expanded"])
|
||||
self.assertTrue(marketing_panel["expanded"])
|
||||
self.assertEqual(sku_panel["background_color"], "blue-50")
|
||||
self.assertEqual(marketing_panel["background_color"], "violet-50")
|
||||
self.assertNotEqual(
|
||||
sku_panel["background_color"], marketing_panel["background_color"]
|
||||
)
|
||||
self.assertEqual(sku_panel["header"]["title"]["tag"], "markdown")
|
||||
self.assertIn(
|
||||
"**<font color='blue'>重点SKU分析</font>**",
|
||||
sku_panel["header"]["title"]["content"],
|
||||
)
|
||||
self.assertIn(
|
||||
"**<font color='violet'>达人与内容营销分析</font>**",
|
||||
marketing_panel["header"]["title"]["content"],
|
||||
)
|
||||
for phrase in (
|
||||
"流量收缩,转化承压",
|
||||
"曝光为累计值",
|
||||
"继续放大",
|
||||
"最佳渠道:抖音",
|
||||
"补充逐星内容",
|
||||
):
|
||||
self.assertIn(phrase, serialized)
|
||||
self.assertNotIn("button", serialized)
|
||||
|
||||
def test_build_card_uses_card_2_schema_and_dashboard(self):
|
||||
report = """一、核心经营结论
|
||||
@@ -139,25 +216,99 @@ class DailyReportCardTests(unittest.TestCase):
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.subprocess.run"
|
||||
"daily_report_card.send_lark_bot_message"
|
||||
)
|
||||
def test_send_card_to_all_recipients(self, run):
|
||||
run.side_effect = [
|
||||
type("Result", (), {"returncode": 0, "stdout": json.dumps({"message_id": f"om_{i}"}), "stderr": ""})()
|
||||
def test_send_card_to_all_recipients(self, send):
|
||||
send.side_effect = [
|
||||
{"message_id": f"om_{i}"}
|
||||
for i in range(3)
|
||||
]
|
||||
recipients = ["ou_one", "ou_two", "ou_three"]
|
||||
results = send_card_to_recipients({"schema": "2.0"}, recipients, "2026-07-15")
|
||||
|
||||
self.assertEqual([item["message_id"] for item in results], ["om_0", "om_1", "om_2"])
|
||||
self.assertEqual(run.call_count, 3)
|
||||
commands = [call.args[0] for call in run.call_args_list]
|
||||
self.assertEqual([cmd[cmd.index("--user-id") + 1] for cmd in commands], recipients)
|
||||
self.assertTrue(all("--msg-type" in cmd and "interactive" in cmd for cmd in commands))
|
||||
keys = [cmd[cmd.index("--idempotency-key") + 1] for cmd in commands]
|
||||
self.assertEqual(send.call_count, 3)
|
||||
calls = send.call_args_list
|
||||
self.assertEqual([call.kwargs["user_id"] for call in calls], recipients)
|
||||
self.assertTrue(all(call.kwargs["msg_type"] == "interactive" for call in calls))
|
||||
self.assertTrue(all(call.kwargs["profile"] == "hermes-analyzer" for call in calls))
|
||||
keys = [call.kwargs["idempotency_key"] for call in calls]
|
||||
self.assertTrue(all(key.startswith("daily-2026-07-15-") for key in keys))
|
||||
self.assertTrue(all(len(key) <= 50 for key in keys))
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.send_lark_bot_message"
|
||||
)
|
||||
def test_send_card_rejects_success_exit_without_message_id(self, send):
|
||||
send.return_value = {"ok": True}
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "message_id"):
|
||||
send_card_to_recipients(
|
||||
{"schema": "2.0"},
|
||||
["ou_owner"],
|
||||
"2026-08-09",
|
||||
)
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.send_lark_bot_message"
|
||||
)
|
||||
def test_sender_uses_fixed_user_profile_and_hashes_full_recipient_identity(self, send):
|
||||
send.side_effect = [
|
||||
{"message_id": f"om_{index}"}
|
||||
for index in range(2)
|
||||
]
|
||||
recipients = ["ou_first_sharedsuffix", "ou_second_sharedsuffix"]
|
||||
|
||||
send_card_to_recipients(
|
||||
{"schema": "2.0"},
|
||||
recipients,
|
||||
"2026-08-07",
|
||||
)
|
||||
|
||||
calls = send.call_args_list
|
||||
self.assertTrue(
|
||||
all(call.kwargs["profile"] == "hermes-analyzer" for call in calls)
|
||||
)
|
||||
keys = [
|
||||
call.kwargs["idempotency_key"]
|
||||
for call in calls
|
||||
]
|
||||
self.assertEqual(len(set(keys)), 2)
|
||||
with self.assertRaisesRegex(ValueError, "hermes-analyzer"):
|
||||
send_card_to_recipients(
|
||||
{"schema": "2.0"},
|
||||
["ou_owner"],
|
||||
"2026-08-07",
|
||||
app_profile="dynamic-analyzer",
|
||||
)
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.send_lark_bot_message"
|
||||
)
|
||||
def test_stable_fingerprint_ignores_rotating_uploaded_image_key(self, send):
|
||||
send.side_effect = [
|
||||
{"message_id": f"om_{index}"}
|
||||
for index in range(2)
|
||||
]
|
||||
|
||||
for image_key in ("img_first_upload", "img_second_upload"):
|
||||
send_card_to_recipients(
|
||||
build_dashboard_card("2026-08-07", [], image_key),
|
||||
["ou_owner"],
|
||||
"2026-08-07",
|
||||
message_kind="dashboard",
|
||||
idempotency_fingerprint="same-dashboard-bytes-and-metrics",
|
||||
)
|
||||
|
||||
keys = [
|
||||
call.kwargs["idempotency_key"]
|
||||
for call in send.call_args_list
|
||||
]
|
||||
self.assertEqual(keys[0], keys[1])
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.send_card_to_recipients"
|
||||
)
|
||||
@@ -177,13 +328,16 @@ class DailyReportCardTests(unittest.TestCase):
|
||||
]
|
||||
recipients = ["ou_one", "ou_two"]
|
||||
|
||||
results = send_daily_report_cards(
|
||||
"2026-07-17",
|
||||
"第一部分 整体经营分析\n完整分析内容",
|
||||
[{"light": "🟢"}],
|
||||
Path("charts/dashboard.png"),
|
||||
recipients,
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
chart_path = Path(tmp) / "dashboard.png"
|
||||
Image.new("RGB", (4, 4), "white").save(chart_path)
|
||||
results = send_daily_report_cards(
|
||||
"2026-07-17",
|
||||
"第一部分 整体经营分析\n完整分析内容",
|
||||
[{"light": "🟢"}],
|
||||
chart_path,
|
||||
recipients,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[item["message_id"] for item in results],
|
||||
@@ -195,6 +349,10 @@ class DailyReportCardTests(unittest.TestCase):
|
||||
self.assertIn("img_v3_test", json.dumps(first_card, ensure_ascii=False))
|
||||
self.assertIn("完整分析内容", json.dumps(second_card, ensure_ascii=False))
|
||||
self.assertEqual(send.call_args_list[0].kwargs["message_kind"], "dashboard")
|
||||
self.assertIn(
|
||||
"idempotency_fingerprint",
|
||||
send.call_args_list[0].kwargs,
|
||||
)
|
||||
self.assertEqual(send.call_args_list[1].kwargs["message_kind"], "analysis")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters.direct_content_llm import (
|
||||
ContentLLMConfig,
|
||||
ContentLLMConfigurationError,
|
||||
ContentLLMResponseError,
|
||||
call_content_analyzer,
|
||||
load_content_llm_config,
|
||||
)
|
||||
|
||||
|
||||
def _config() -> ContentLLMConfig:
|
||||
return ContentLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/anthropic/v1/messages",
|
||||
credential="test-only-secret",
|
||||
model="MiniMax-M3",
|
||||
thinking_mode="disabled",
|
||||
)
|
||||
|
||||
|
||||
def _response(*blocks: dict[str, object]) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "message",
|
||||
"content": list(blocks),
|
||||
"usage": {"input_tokens": 3, "output_tokens": 4},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def test_config_uses_anthropic_endpoint_and_redacts_key(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"CONTENT_ANALYSIS_LLM_BASE_URL", "https://api.minimaxi.com/anthropic"
|
||||
)
|
||||
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_API_KEY", "test-only-secret")
|
||||
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_MODEL", "MiniMax-M3")
|
||||
|
||||
config = load_content_llm_config()
|
||||
|
||||
assert config.endpoint == "https://api.minimaxi.com/anthropic/v1/messages"
|
||||
assert config.model == "MiniMax-M3"
|
||||
assert "test-only-secret" not in repr(config)
|
||||
|
||||
|
||||
def test_call_sends_anthropic_message_and_ignores_thinking_block() -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def transport(url, headers, body, timeout):
|
||||
payload = json.loads(body)
|
||||
observed.update(
|
||||
url=url,
|
||||
headers=headers,
|
||||
payload=payload,
|
||||
timeout=timeout,
|
||||
)
|
||||
return _response(
|
||||
{"type": "thinking", "thinking": "hidden reasoning"},
|
||||
{"type": "text", "text": "最终报告"},
|
||||
)
|
||||
|
||||
result = call_content_analyzer(
|
||||
"system prompt",
|
||||
"user prompt",
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert result == "最终报告"
|
||||
assert observed["url"] == "https://api.minimaxi.com/anthropic/v1/messages"
|
||||
assert observed["headers"]["X-Api-Key"] == "test-only-secret"
|
||||
assert observed["headers"]["anthropic-version"] == "2023-06-01"
|
||||
assert observed["payload"]["model"] == "MiniMax-M3"
|
||||
assert observed["payload"]["system"] == "system prompt"
|
||||
assert observed["payload"]["messages"] == [
|
||||
{"role": "user", "content": "user prompt"}
|
||||
]
|
||||
assert observed["payload"]["thinking"] == {"type": "disabled"}
|
||||
|
||||
|
||||
def test_call_rejects_openai_response_shape() -> None:
|
||||
with pytest.raises(ContentLLMResponseError, match="response shape"):
|
||||
call_content_analyzer(
|
||||
"system prompt",
|
||||
"user prompt",
|
||||
transport=lambda *_args: b'{"choices": []}',
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_config_rejects_local_gateway(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"CONTENT_ANALYSIS_LLM_BASE_URL", "http://127.0.0.1:8642/v1"
|
||||
)
|
||||
monkeypatch.setenv("CONTENT_ANALYSIS_LLM_API_KEY", "test-only-secret")
|
||||
|
||||
with pytest.raises(ContentLLMConfigurationError, match="local gateway"):
|
||||
load_content_llm_config()
|
||||
@@ -0,0 +1,162 @@
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import douyin_comment_scraper as scraper
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
batch_rescrape_douyin as batch,
|
||||
)
|
||||
|
||||
|
||||
class _EvaluatePage:
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
self.calls = []
|
||||
|
||||
def evaluate(self, script, argument):
|
||||
self.calls.append((script, argument))
|
||||
return self.result
|
||||
|
||||
|
||||
def test_bounded_fetch_uses_abort_controller_and_clears_timer(monkeypatch):
|
||||
page = _EvaluatePage(
|
||||
{"__gyxx_status": "ok", "payload": {"comments": []}}
|
||||
)
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
|
||||
|
||||
payload = scraper._fetch_json_with_timeout(
|
||||
page,
|
||||
"https://example.invalid/comments",
|
||||
deadline=1000.0,
|
||||
stage="top-level pagination",
|
||||
)
|
||||
|
||||
assert payload == {"comments": []}
|
||||
script, argument = page.calls[0]
|
||||
assert "AbortController" in script
|
||||
assert "clearTimeout(timer)" in script
|
||||
assert argument["timeoutMs"] == 30_000
|
||||
|
||||
|
||||
def test_bounded_fetch_timeout_is_an_explicit_transient_failure(monkeypatch):
|
||||
page = _EvaluatePage({"__gyxx_status": "timeout"})
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
|
||||
|
||||
with pytest.raises(scraper.TransientDouyinCommentTimeout) as caught:
|
||||
scraper._fetch_json_with_timeout(
|
||||
page,
|
||||
"https://example.invalid/comments",
|
||||
deadline=1000.0,
|
||||
stage="reply pagination",
|
||||
)
|
||||
|
||||
assert isinstance(caught.value, ConnectionError)
|
||||
assert "reply pagination" in str(caught.value)
|
||||
|
||||
|
||||
def test_scroll_checks_the_note_deadline_before_browser_work(monkeypatch):
|
||||
page = _EvaluatePage(None)
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 901.0)
|
||||
|
||||
with pytest.raises(scraper.TransientDouyinCommentTimeout):
|
||||
scraper.scroll_comments(
|
||||
page,
|
||||
max_scrolls=60,
|
||||
idle_rounds=10,
|
||||
comments_by_id={},
|
||||
deadline=900.0,
|
||||
)
|
||||
|
||||
assert page.calls == []
|
||||
|
||||
|
||||
def test_top_and_reply_pagination_share_the_bounded_fetch(monkeypatch):
|
||||
stages = []
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 0.0)
|
||||
|
||||
def fetch(_page, _url, *, deadline, stage):
|
||||
assert deadline == 900.0
|
||||
stages.append(stage)
|
||||
return {"comments": [], "has_more": 0, "cursor": 0}
|
||||
|
||||
monkeypatch.setattr(scraper, "_fetch_json_with_timeout", fetch)
|
||||
monkeypatch.setattr(scraper, "collect_from_payload", lambda *_args: None)
|
||||
page = _EvaluatePage(None)
|
||||
|
||||
scraper.fetch_missing_top_comments(
|
||||
page,
|
||||
"source-url",
|
||||
"aweme-id",
|
||||
{"existing": {}},
|
||||
{
|
||||
"comment_url_template": "https://example.invalid/top",
|
||||
"api_total": 2,
|
||||
"last_cursor": 1,
|
||||
},
|
||||
deadline=900.0,
|
||||
)
|
||||
scraper.fetch_missing_replies(
|
||||
page,
|
||||
"source-url",
|
||||
{
|
||||
"parent": {
|
||||
"level": "comment",
|
||||
"reply_count": 1,
|
||||
"comment_id": "parent-id",
|
||||
}
|
||||
},
|
||||
{
|
||||
"reply_url_template": "https://example.invalid/reply",
|
||||
"aweme_id": "aweme-id",
|
||||
},
|
||||
deadline=900.0,
|
||||
)
|
||||
|
||||
assert stages == ["top-level pagination", "reply pagination"]
|
||||
|
||||
|
||||
def test_deadline_failure_retries_and_never_replaces_existing_comments(
|
||||
monkeypatch,
|
||||
):
|
||||
attempts = []
|
||||
|
||||
class Checkpoint:
|
||||
completed = []
|
||||
cleared = False
|
||||
|
||||
def __init__(self, _platform):
|
||||
pass
|
||||
|
||||
def contains(self, _note_id, _url):
|
||||
return False
|
||||
|
||||
def mark_completed(self, note_id, url):
|
||||
self.completed.append((note_id, url))
|
||||
|
||||
def clear(self):
|
||||
self.cleared = True
|
||||
|
||||
def scrape(url, *_args):
|
||||
attempts.append(url)
|
||||
raise scraper.TransientDouyinCommentTimeout(
|
||||
"Douyin comment scrape exceeded 900 seconds"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(batch, "CommentBatchCheckpoint", Checkpoint)
|
||||
monkeypatch.setattr(
|
||||
batch,
|
||||
"collect_urls",
|
||||
lambda: [{"id": 1, "url": "dy-url", "comment_count": 10}],
|
||||
)
|
||||
monkeypatch.setattr(batch.scraper, "scrape_comments", scrape)
|
||||
monkeypatch.setattr(batch.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(
|
||||
batch.db,
|
||||
"replace_comments",
|
||||
lambda *_args, **_kwargs: pytest.fail(
|
||||
"a timed-out scrape must not replace existing comments"
|
||||
),
|
||||
)
|
||||
|
||||
assert batch.main() == 1
|
||||
assert attempts == ["dy-url", "dy-url", "dy-url"]
|
||||
assert Checkpoint.completed == []
|
||||
assert Checkpoint.cleared is False
|
||||
@@ -0,0 +1,399 @@
|
||||
import json
|
||||
import subprocess
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.feishu_doc_import import (
|
||||
FeishuDocImportAmbiguous,
|
||||
FeishuDocImportError,
|
||||
FeishuDocImportTimeout,
|
||||
_parse_cli_json,
|
||||
deterministic_import_operation_id,
|
||||
import_markdown_document,
|
||||
)
|
||||
|
||||
|
||||
def _completed(payload, *, returncode=0, prefix="", suffix=""):
|
||||
return SimpleNamespace(
|
||||
returncode=returncode,
|
||||
stdout=f"{prefix}{json.dumps(payload)}{suffix}",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
|
||||
class _FakeClock:
|
||||
def __init__(self):
|
||||
self.now = 100.0
|
||||
|
||||
def monotonic(self):
|
||||
return self.now
|
||||
|
||||
def sleep(self, seconds):
|
||||
self.now += seconds
|
||||
|
||||
|
||||
def test_import_ready_result_uses_relative_file_exact_title_and_audits_without_token(
|
||||
tmp_path,
|
||||
):
|
||||
commands = []
|
||||
title = "宙斯 周笔记汇总(2026-07-27~08-02)"
|
||||
markdown = "# 标题\n\n完整正文"
|
||||
document_token = "VERY_SECRET_DOCUMENT_TOKEN_123456"
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
source = command[command.index("--file") + 1]
|
||||
assert source.startswith("./")
|
||||
assert (kwargs["cwd"] / source.removeprefix("./")).read_text(
|
||||
encoding="utf-8"
|
||||
) == markdown
|
||||
return _completed(
|
||||
{
|
||||
"scenario": "import",
|
||||
"ticket": "ticket-ready",
|
||||
"ready": True,
|
||||
"failed": False,
|
||||
"job_status": 0,
|
||||
"job_status_label": "success",
|
||||
"token": document_token,
|
||||
"url": f"https://example.feishu.cn/docx/{document_token}",
|
||||
},
|
||||
prefix="lark-cli notice\n",
|
||||
suffix="\ntrailing notifier",
|
||||
)
|
||||
|
||||
result = import_markdown_document(
|
||||
markdown,
|
||||
title=title,
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="weekly_summary_doc.md",
|
||||
runner=fake_run,
|
||||
operation_id="ready-op",
|
||||
)
|
||||
|
||||
assert result.url == f"https://example.feishu.cn/docx/{document_token}"
|
||||
assert result.document_token == document_token
|
||||
assert commands[0][1:3] == ["drive", "+import"]
|
||||
assert commands[0][commands[0].index("--name") + 1] == title
|
||||
assert commands[0][commands[0].index("--type") + 1] == "docx"
|
||||
assert commands[0][commands[0].index("--as") + 1] == "user"
|
||||
assert "--folder-token" not in commands[0]
|
||||
|
||||
audit_text = result.audit_path.read_text(encoding="utf-8")
|
||||
audit = json.loads(audit_text)
|
||||
assert audit["status"] == "ready"
|
||||
assert audit["ticket"] == "ticket-ready"
|
||||
assert audit["url"] == "https://example.feishu.cn/docx/<redacted>"
|
||||
assert audit["last_result"]["url"] == audit["url"]
|
||||
assert audit["document_title"] == title
|
||||
assert audit["folder_target"] == "root"
|
||||
assert "token" not in audit
|
||||
assert "document_token" not in audit_text
|
||||
assert document_token not in audit_text
|
||||
assert not list((tmp_path / "work").glob("*.md"))
|
||||
|
||||
|
||||
def test_import_persists_ticket_and_polls_task_result_until_ready(tmp_path):
|
||||
responses = iter(
|
||||
[
|
||||
_completed(
|
||||
{
|
||||
"scenario": "import",
|
||||
"ticket": "ticket-async",
|
||||
"ready": False,
|
||||
"failed": False,
|
||||
"timed_out": True,
|
||||
}
|
||||
),
|
||||
_completed(
|
||||
{
|
||||
"scenario": "import",
|
||||
"ticket": "ticket-async",
|
||||
"ready": False,
|
||||
"failed": False,
|
||||
"job_status": 1,
|
||||
"job_status_label": "processing",
|
||||
}
|
||||
),
|
||||
_completed(
|
||||
{
|
||||
"scenario": "import",
|
||||
"ticket": "ticket-async",
|
||||
"ready": True,
|
||||
"failed": False,
|
||||
"job_status": 0,
|
||||
"job_status_label": "success",
|
||||
"token": "doxcnAsyncDocument",
|
||||
"url": "https://example.feishu.cn/docx/doxcnAsyncDocument",
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
commands.append(command)
|
||||
if len(commands) == 2:
|
||||
in_progress = json.loads(
|
||||
(tmp_path / "audit" / "async-op.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert in_progress["status"] == "processing"
|
||||
assert in_progress["ticket"] == "ticket-async"
|
||||
assert in_progress["last_result"]["ready"] is False
|
||||
return next(responses)
|
||||
|
||||
result = import_markdown_document(
|
||||
"report",
|
||||
title="精确标题",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=fake_run,
|
||||
sleeper=lambda _seconds: None,
|
||||
operation_id="async-op",
|
||||
)
|
||||
|
||||
assert result.ticket == "ticket-async"
|
||||
assert len(commands) == 3
|
||||
for command in commands[1:]:
|
||||
assert command[1:3] == ["drive", "+task_result"]
|
||||
assert command[command.index("--scenario") + 1] == "import"
|
||||
assert command[command.index("--ticket") + 1] == "ticket-async"
|
||||
assert command[command.index("--as") + 1] == "user"
|
||||
audit = json.loads(result.audit_path.read_text(encoding="utf-8"))
|
||||
assert audit["poll_attempts"] == 2
|
||||
assert audit["ticket"] == "ticket-async"
|
||||
assert audit["last_result"]["ready"] is True
|
||||
|
||||
|
||||
def test_import_reports_failed_task_without_retrying_mutation(tmp_path):
|
||||
leaked_document_token = "VERY_SECRET_FAILED_DOCUMENT_TOKEN"
|
||||
leaked_open_id = "ou_SHOULD_NOT_BE_IN_AUDIT"
|
||||
responses = iter(
|
||||
[
|
||||
_completed(
|
||||
{
|
||||
"ticket": "ticket-failed",
|
||||
"ready": False,
|
||||
"failed": False,
|
||||
}
|
||||
),
|
||||
_completed(
|
||||
{
|
||||
"ticket": "ticket-failed",
|
||||
"ready": False,
|
||||
"failed": True,
|
||||
"job_status": -1,
|
||||
"job_status_label": "failed",
|
||||
"job_error_msg": (
|
||||
"conversion failed at "
|
||||
f"https://example.feishu.cn/docx/{leaked_document_token} "
|
||||
f'\"token\":\"{leaked_document_token}\" '
|
||||
f'\"open_id\":\"{leaked_open_id}\"'
|
||||
),
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
commands.append(command)
|
||||
return next(responses)
|
||||
|
||||
with pytest.raises(FeishuDocImportError, match="conversion failed") as raised:
|
||||
import_markdown_document(
|
||||
"report",
|
||||
title="失败报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=fake_run,
|
||||
sleeper=lambda _seconds: None,
|
||||
operation_id="failed-op",
|
||||
)
|
||||
|
||||
assert sum(command[1:3] == ["drive", "+import"] for command in commands) == 1
|
||||
audit_text = raised.value.audit_path.read_text(encoding="utf-8")
|
||||
audit = json.loads(audit_text)
|
||||
assert audit["status"] == "failed"
|
||||
assert audit["ticket"] == "ticket-failed"
|
||||
assert audit["last_result"]["failed"] is True
|
||||
assert leaked_document_token not in audit_text
|
||||
assert leaked_open_id not in audit_text
|
||||
|
||||
|
||||
def test_import_timeout_keeps_ticket_for_auditable_follow_up(tmp_path):
|
||||
clock = _FakeClock()
|
||||
responses = iter(
|
||||
[
|
||||
_completed(
|
||||
{"ticket": "ticket-timeout", "ready": False, "failed": False}
|
||||
),
|
||||
_completed(
|
||||
{
|
||||
"ticket": "ticket-timeout",
|
||||
"ready": False,
|
||||
"failed": False,
|
||||
"job_status_label": "processing",
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(FeishuDocImportTimeout) as raised:
|
||||
import_markdown_document(
|
||||
"report",
|
||||
title="超时报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=lambda *_args, **_kwargs: next(responses),
|
||||
clock=clock.monotonic,
|
||||
sleeper=clock.sleep,
|
||||
wait_timeout_seconds=1,
|
||||
poll_interval_seconds=1,
|
||||
operation_id="timeout-op",
|
||||
)
|
||||
|
||||
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
|
||||
assert audit["status"] == "timed_out"
|
||||
assert audit["ticket"] == "ticket-timeout"
|
||||
assert audit["poll_attempts"] == 1
|
||||
|
||||
|
||||
def test_mutating_import_process_timeout_is_ambiguous_and_never_retried(tmp_path):
|
||||
commands = []
|
||||
|
||||
def time_out(command, **kwargs):
|
||||
commands.append(command)
|
||||
assert kwargs["timeout"] == 2
|
||||
raise subprocess.TimeoutExpired(command, kwargs["timeout"])
|
||||
|
||||
with pytest.raises(FeishuDocImportTimeout) as raised:
|
||||
import_markdown_document(
|
||||
"report",
|
||||
title="有限超时报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=time_out,
|
||||
import_command_timeout_seconds=2,
|
||||
operation_id="command-timeout-op",
|
||||
)
|
||||
|
||||
assert len(commands) == 1
|
||||
assert commands[0][1:3] == ["drive", "+import"]
|
||||
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
|
||||
assert audit["status"] == "ambiguous"
|
||||
assert audit["ticket"] is None
|
||||
assert not list((tmp_path / "work").glob("*.md"))
|
||||
|
||||
|
||||
def test_deterministic_audit_resumes_only_known_ticket_without_second_import(tmp_path):
|
||||
audit_root = tmp_path / "audit"
|
||||
audit_root.mkdir()
|
||||
operation_id = "stable-report-op"
|
||||
audit_path = audit_root / f"{operation_id}.json"
|
||||
audit_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"audit_schema": "gyxx.feishu-doc-import.v1",
|
||||
"document_title": "固定周期报告",
|
||||
"folder_target": "root",
|
||||
"identity": "user",
|
||||
"markdown_sha256": "previous-input-digest",
|
||||
"operation_id": operation_id,
|
||||
"poll_attempts": 1,
|
||||
"status": "timed_out",
|
||||
"target_type": "docx",
|
||||
"ticket": "known-ticket",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
commands.append(command)
|
||||
return _completed(
|
||||
{
|
||||
"ticket": "known-ticket",
|
||||
"ready": True,
|
||||
"failed": False,
|
||||
"token": "doxcnRecoveredDocument",
|
||||
"url": "https://example.feishu.cn/docx/doxcnRecoveredDocument",
|
||||
}
|
||||
)
|
||||
|
||||
result = import_markdown_document(
|
||||
"possibly regenerated report",
|
||||
title="固定周期报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=audit_root,
|
||||
source_filename="report.md",
|
||||
runner=fake_run,
|
||||
operation_id=operation_id,
|
||||
)
|
||||
|
||||
assert result.url.endswith("/doxcnRecoveredDocument")
|
||||
assert len(commands) == 1
|
||||
assert commands[0][1:3] == ["drive", "+task_result"]
|
||||
assert all(command[1:3] != ["drive", "+import"] for command in commands)
|
||||
audit_text = audit_path.read_text(encoding="utf-8")
|
||||
audit = json.loads(audit_text)
|
||||
assert audit["status"] == "ready"
|
||||
assert audit["resume_count"] == 1
|
||||
assert audit["resume_input_changed"] is True
|
||||
assert "doxcnRecoveredDocument" not in audit_text
|
||||
|
||||
|
||||
def test_resume_only_without_ticket_audit_refuses_mutating_import(tmp_path):
|
||||
commands = []
|
||||
|
||||
with pytest.raises(FeishuDocImportAmbiguous, match="缺少可对账") as raised:
|
||||
import_markdown_document(
|
||||
"report",
|
||||
title="待对账报告",
|
||||
working_directory=tmp_path / "work",
|
||||
audit_directory=tmp_path / "audit",
|
||||
source_filename="report.md",
|
||||
runner=lambda command, **_kwargs: commands.append(command),
|
||||
operation_id="missing-ticket-op",
|
||||
resume_only=True,
|
||||
)
|
||||
|
||||
assert commands == []
|
||||
audit = json.loads(raised.value.audit_path.read_text(encoding="utf-8"))
|
||||
assert audit["status"] == "ambiguous"
|
||||
assert audit["ticket"] is None
|
||||
|
||||
|
||||
def test_deterministic_operation_id_is_stable_and_period_specific():
|
||||
first = deterministic_import_operation_id(
|
||||
"content-weekly-summary", "宙斯", "2026-07-27", "2026-08-02"
|
||||
)
|
||||
same = deterministic_import_operation_id(
|
||||
"content-weekly-summary", "宙斯", "2026-07-27", "2026-08-02"
|
||||
)
|
||||
other_style = deterministic_import_operation_id(
|
||||
"content-weekly-summary", "赫拉", "2026-07-27", "2026-08-02"
|
||||
)
|
||||
|
||||
assert first == same
|
||||
assert first != other_style
|
||||
assert first.startswith("content-weekly-summary-")
|
||||
assert len(first.rsplit("-", 1)[1]) == 40
|
||||
|
||||
|
||||
def test_cli_json_parser_accepts_noise_and_rejects_invalid_output():
|
||||
assert _parse_cli_json(
|
||||
'notice {"notice":"update"}\n{"ready":false,"ticket":"abc"}\ntrailing'
|
||||
) == {
|
||||
"ready": False,
|
||||
"ticket": "abc",
|
||||
}
|
||||
with pytest.raises(ValueError, match="no JSON"):
|
||||
_parse_cli_json("plain text", "also plain text")
|
||||
@@ -1,3 +1,7 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
friday_relogin_parallel as relogin,
|
||||
)
|
||||
@@ -43,6 +47,434 @@ def test_failed_platforms_are_grouped_by_their_responsible_recipient():
|
||||
}
|
||||
|
||||
|
||||
def test_legacy_route_keeps_the_existing_per_platform_recipients(monkeypatch):
|
||||
observed = {}
|
||||
|
||||
def fake_resolver(workflow_id, defaults):
|
||||
observed["workflow_id"] = workflow_id
|
||||
observed["defaults"] = defaults
|
||||
return relogin.ResolvedNotificationRoute(
|
||||
configured=False,
|
||||
enabled=True,
|
||||
app_profile=None,
|
||||
open_ids=tuple(defaults),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(relogin, "resolve_notification_route", fake_resolver)
|
||||
|
||||
route = relogin.resolve_weekly_notification_route()
|
||||
|
||||
assert observed == {
|
||||
"workflow_id": "content.relogin.weekly",
|
||||
"defaults": (
|
||||
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
),
|
||||
}
|
||||
assert relogin.recipients_for_platform("pgy", route) == (
|
||||
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
)
|
||||
assert relogin.recipients_for_platform("douyin", route) == (
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
)
|
||||
|
||||
|
||||
def test_disabled_route_still_runs_relogin_without_sending(monkeypatch):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=False,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=(),
|
||||
)
|
||||
rounds = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"resolve_weekly_notification_route",
|
||||
lambda _override=None: route,
|
||||
)
|
||||
|
||||
def fake_round_factory(_args, resolved_route, notification_failures):
|
||||
assert resolved_route is route
|
||||
assert notification_failures == []
|
||||
|
||||
def run_round(names, attempt):
|
||||
rounds.append((tuple(names), attempt))
|
||||
return {name: True for name in names}
|
||||
|
||||
return run_round
|
||||
|
||||
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"_send_lark",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
AssertionError("disabled route must not send Lark messages")
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
relogin.sys,
|
||||
"argv",
|
||||
["friday_relogin_parallel.py", "--platform", "pgy"],
|
||||
)
|
||||
|
||||
assert relogin.main() == 0
|
||||
assert rounds == [(("pgy",), 1)]
|
||||
|
||||
|
||||
def test_enabled_route_sends_qr_text_and_image_to_every_recipient(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
sent = []
|
||||
image = tmp_path / "qrcode.png"
|
||||
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"_send_lark",
|
||||
lambda recipient, text=None, image=None, app_profile=None: sent.append(
|
||||
(recipient, text, image, app_profile)
|
||||
),
|
||||
)
|
||||
|
||||
failures = relogin._send_qr_notification(
|
||||
route,
|
||||
"pgy",
|
||||
override=None,
|
||||
text="请扫码",
|
||||
image=image,
|
||||
)
|
||||
|
||||
assert failures == ()
|
||||
assert sent == [
|
||||
("ou_first123", "请扫码", None, "hermes-analyzer"),
|
||||
("ou_first123", None, image, "hermes-analyzer"),
|
||||
("ou_second456", "请扫码", None, "hermes-analyzer"),
|
||||
("ou_second456", None, image, "hermes-analyzer"),
|
||||
]
|
||||
assert relogin.group_platforms_by_notification_recipient(
|
||||
["pgy", "douyin"], route
|
||||
) == {
|
||||
"ou_first123": ["pgy", "douyin"],
|
||||
"ou_second456": ["pgy", "douyin"],
|
||||
}
|
||||
|
||||
|
||||
def test_qr_delivery_aggregates_errors_and_attempts_every_recipient(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
attempts = []
|
||||
image = tmp_path / "qrcode.png"
|
||||
|
||||
def fake_send(recipient, text=None, image=None, app_profile=None):
|
||||
stage = "qr_text" if text is not None else "qr_image"
|
||||
attempts.append((recipient, stage, app_profile))
|
||||
if (recipient, stage) == ("ou_first123", "qr_text"):
|
||||
raise RuntimeError("first text rejected")
|
||||
if (recipient, stage) == ("ou_second456", "qr_image"):
|
||||
raise RuntimeError("second image rejected")
|
||||
|
||||
monkeypatch.setattr(relogin, "_send_lark", fake_send)
|
||||
|
||||
failures = relogin._send_qr_notification(
|
||||
route,
|
||||
"pgy",
|
||||
override=None,
|
||||
text="请扫码",
|
||||
image=image,
|
||||
)
|
||||
|
||||
assert attempts == [
|
||||
("ou_first123", "qr_text", "hermes-analyzer"),
|
||||
("ou_first123", "qr_image", "hermes-analyzer"),
|
||||
("ou_second456", "qr_text", "hermes-analyzer"),
|
||||
("ou_second456", "qr_image", "hermes-analyzer"),
|
||||
]
|
||||
assert failures == (
|
||||
relogin.NotificationDeliveryFailure(
|
||||
stage="qr_text",
|
||||
platform="pgy",
|
||||
recipient="ou_first123",
|
||||
detail="first text rejected",
|
||||
),
|
||||
relogin.NotificationDeliveryFailure(
|
||||
stage="qr_image",
|
||||
platform="pgy",
|
||||
recipient="ou_second456",
|
||||
detail="second image rejected",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_failure_summary_aggregates_every_recipient_error(monkeypatch):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
attempts = []
|
||||
|
||||
def fail_send(recipient, **_kwargs):
|
||||
attempts.append(recipient)
|
||||
raise RuntimeError(f"rejected {recipient}")
|
||||
|
||||
monkeypatch.setattr(relogin, "_send_lark", fail_send)
|
||||
|
||||
failures = relogin._send_failure_summaries(
|
||||
route,
|
||||
["pgy", "douyin"],
|
||||
override=None,
|
||||
max_attempts=3,
|
||||
)
|
||||
|
||||
assert attempts == ["ou_first123", "ou_second456"]
|
||||
assert [failure.stage for failure in failures] == [
|
||||
"failure_summary",
|
||||
"failure_summary",
|
||||
]
|
||||
assert [failure.recipient for failure in failures] == attempts
|
||||
assert [failure.platform for failure in failures] == [
|
||||
"pgy,douyin",
|
||||
"pgy,douyin",
|
||||
]
|
||||
|
||||
|
||||
def test_main_returns_failure_and_reports_all_notification_errors(
|
||||
monkeypatch,
|
||||
capsys,
|
||||
):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"resolve_weekly_notification_route",
|
||||
lambda _override=None: route,
|
||||
)
|
||||
|
||||
def fake_round_factory(_args, _route, notification_failures):
|
||||
notification_failures.extend(
|
||||
(
|
||||
relogin.NotificationDeliveryFailure(
|
||||
"qr_text", "pgy", "ou_first123", "text rejected"
|
||||
),
|
||||
relogin.NotificationDeliveryFailure(
|
||||
"qr_image", "pgy", "ou_second456", "image rejected"
|
||||
),
|
||||
)
|
||||
)
|
||||
return lambda names, _attempt: {name: True for name in names}
|
||||
|
||||
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
|
||||
monkeypatch.setattr(
|
||||
relogin.sys,
|
||||
"argv",
|
||||
["friday_relogin_parallel.py", "--platform", "pgy"],
|
||||
)
|
||||
|
||||
assert relogin.main() == 2
|
||||
output = capsys.readouterr().out
|
||||
assert "Final relogin status: {'pgy': True}" in output
|
||||
assert "Final notification status: failed (2 deliveries)" in output
|
||||
assert "recipient=ou_first123 error=text rejected" in output
|
||||
assert "recipient=ou_second456 error=image rejected" in output
|
||||
|
||||
|
||||
def test_round_completes_relogin_after_notification_failure(monkeypatch, tmp_path):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
args = SimpleNamespace(
|
||||
round_timeout=10,
|
||||
screenshot_delay=0,
|
||||
no_send=False,
|
||||
recipient=None,
|
||||
max_attempts=1,
|
||||
)
|
||||
failures = []
|
||||
attempts = []
|
||||
waits = []
|
||||
|
||||
class Process:
|
||||
pid = 1234
|
||||
|
||||
def wait(self, timeout):
|
||||
waits.append(timeout)
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
|
||||
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"binding_from_environment",
|
||||
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
|
||||
)
|
||||
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(relogin, "_login_windows", lambda: {"pgy": 101})
|
||||
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
|
||||
monkeypatch.setattr(relogin, "_capture_window", lambda _hwnd, _output: None)
|
||||
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: True)
|
||||
|
||||
def fake_send(recipient, text=None, image=None, **_kwargs):
|
||||
stage = "qr_text" if text is not None else "qr_image"
|
||||
attempts.append((recipient, stage))
|
||||
if (recipient, stage) == ("ou_first123", "qr_text"):
|
||||
raise RuntimeError("temporary send failure")
|
||||
|
||||
monkeypatch.setattr(relogin, "_send_lark", fake_send)
|
||||
|
||||
run_round = relogin._run_round_factory(args, route, failures)
|
||||
|
||||
assert run_round(["pgy"], 1) == {"pgy": True}
|
||||
assert waits == [100]
|
||||
assert attempts == [
|
||||
("ou_first123", "qr_text"),
|
||||
("ou_first123", "qr_image"),
|
||||
("ou_second456", "qr_text"),
|
||||
("ou_second456", "qr_image"),
|
||||
]
|
||||
assert failures == [
|
||||
relogin.NotificationDeliveryFailure(
|
||||
stage="qr_text",
|
||||
platform="pgy",
|
||||
recipient="ou_first123",
|
||||
detail="temporary send failure",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("returncode", "cookie_valid", "expected_status", "expected_failure_count"),
|
||||
(
|
||||
(0, True, True, 0),
|
||||
(1, False, False, 4),
|
||||
),
|
||||
)
|
||||
def test_missing_qr_window_is_classified_after_relogin_result(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
returncode,
|
||||
cookie_valid,
|
||||
expected_status,
|
||||
expected_failure_count,
|
||||
):
|
||||
route = relogin.ResolvedNotificationRoute(
|
||||
configured=True,
|
||||
enabled=True,
|
||||
app_profile="hermes-analyzer",
|
||||
open_ids=("ou_first123", "ou_second456"),
|
||||
)
|
||||
args = SimpleNamespace(
|
||||
round_timeout=10,
|
||||
screenshot_delay=0,
|
||||
no_send=False,
|
||||
recipient=None,
|
||||
max_attempts=1,
|
||||
)
|
||||
failures = []
|
||||
|
||||
class Process:
|
||||
pid = 1234
|
||||
|
||||
def wait(self, timeout):
|
||||
assert timeout == 100
|
||||
return returncode
|
||||
|
||||
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
|
||||
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"binding_from_environment",
|
||||
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
|
||||
)
|
||||
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(relogin, "_login_windows", lambda: {})
|
||||
monkeypatch.setattr(relogin, "_window_for_process", lambda _pid: None)
|
||||
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
|
||||
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: cookie_valid)
|
||||
monkeypatch.setattr(
|
||||
relogin,
|
||||
"_send_lark",
|
||||
lambda *_args, **_kwargs: pytest.fail(
|
||||
"no QR window must not start a send before child classification"
|
||||
),
|
||||
)
|
||||
|
||||
run_round = relogin._run_round_factory(args, route, failures)
|
||||
|
||||
assert run_round(["pgy"], 1) == {"pgy": expected_status}
|
||||
assert len(failures) == expected_failure_count
|
||||
assert {failure.stage for failure in failures} <= {"qr_text", "qr_image"}
|
||||
assert all(
|
||||
failure.detail == "no visible QR window and relogin did not succeed"
|
||||
for failure in failures
|
||||
)
|
||||
|
||||
|
||||
def test_lark_send_uses_shared_analyzer_user_sender_for_text_and_image(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
calls = []
|
||||
image = tmp_path / "qrcode.png"
|
||||
image.write_bytes(b"png")
|
||||
|
||||
monkeypatch.setattr(relogin, "ANALYZER_DIR", tmp_path)
|
||||
monkeypatch.setenv("HERMES_ANALYZER_TOKEN", "must-not-leak")
|
||||
|
||||
def fake_send(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return {"message_id": f"om_{len(calls)}"}
|
||||
|
||||
monkeypatch.setattr(relogin, "send_lark_bot_message", fake_send)
|
||||
|
||||
receipts = relogin._send_lark(
|
||||
"ou_recipient123",
|
||||
text="登录提醒",
|
||||
image=image,
|
||||
)
|
||||
|
||||
assert [receipt["message_id"] for receipt in receipts] == ["om_1", "om_2"]
|
||||
assert calls[0]["user_id"] == "ou_recipient123"
|
||||
assert calls[0]["text"] == "登录提醒"
|
||||
assert "image" not in calls[0]
|
||||
assert calls[1]["user_id"] == "ou_recipient123"
|
||||
assert calls[1]["image"] == image
|
||||
assert "text" not in calls[1]
|
||||
assert all(call["profile"] == "hermes-analyzer" for call in calls)
|
||||
assert all("HERMES_ANALYZER_TOKEN" not in call["env"] for call in calls)
|
||||
|
||||
|
||||
def test_login_windows_are_classified_by_title():
|
||||
assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili"
|
||||
assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy"
|
||||
|
||||
@@ -169,6 +169,73 @@ def test_xingtu_search_and_open_creator_submits_only_one_search(monkeypatch):
|
||||
assert calls == [("同一达人", "creator-1")]
|
||||
|
||||
|
||||
def test_creator_search_attempts_id_then_nickname_and_skips_blank_id():
|
||||
assert xingtu._creator_search_attempts("Rm.", "Ruan97") == [
|
||||
("ID", "Ruan97", "Ruan97"),
|
||||
("昵称", "Rm.", None),
|
||||
]
|
||||
assert xingtu._creator_search_attempts("Rm.", " ") == [
|
||||
("昵称", "Rm.", None),
|
||||
]
|
||||
|
||||
|
||||
def test_xingtu_primary_retries_only_unmatched_tasks_through_buyin(monkeypatch):
|
||||
xingtu.configure_source("xingtu")
|
||||
styles = [{"index": 1, "name": "款式A"}]
|
||||
context = {"index": 1}
|
||||
tasks = [
|
||||
{"record_id": "hit", "creator_name": "达人A", "creator_id": "id-a", "style_context": context},
|
||||
{"record_id": "miss", "creator_name": "达人B", "creator_id": "id-b", "style_context": context},
|
||||
]
|
||||
summaries = {
|
||||
1: {
|
||||
"style": "款式A",
|
||||
"index": 1,
|
||||
"total": 2,
|
||||
"matched": 0,
|
||||
"filled": 0,
|
||||
"skipped_no_pubtime": 0,
|
||||
"results": [],
|
||||
}
|
||||
}
|
||||
calls = []
|
||||
|
||||
def fake_collect(*_args, **_kwargs):
|
||||
return tasks, summaries
|
||||
|
||||
def fake_scrape(*_args, **kwargs):
|
||||
source = xingtu.CONTENT_SOURCE
|
||||
selected = kwargs["task_override"]
|
||||
calls.append((source, [task["record_id"] for task in selected], kwargs.get("replace_existing", False)))
|
||||
for task in selected:
|
||||
matched = source == "buyin" or task["record_id"] == "hit"
|
||||
xingtu.upsert_result(summaries[1], {
|
||||
"record_id": task["record_id"],
|
||||
"matched": matched,
|
||||
"status": xingtu.SUCCESS if matched else xingtu.RETRYABLE_FAILURE,
|
||||
"reason": None if matched else "title_unmatched",
|
||||
})
|
||||
return [summaries[1]]
|
||||
|
||||
monkeypatch.setattr(xingtu, "collect_tasks_across_styles", fake_collect)
|
||||
monkeypatch.setattr(xingtu, "scrape_styles", fake_scrape)
|
||||
try:
|
||||
result = xingtu.scrape_styles_with_fallback(
|
||||
styles, 1, False, None, True,
|
||||
)
|
||||
finally:
|
||||
xingtu.configure_source("xingtu")
|
||||
|
||||
assert calls == [
|
||||
("xingtu", ["hit", "miss"], False),
|
||||
("buyin", ["miss"], True),
|
||||
]
|
||||
rows = {row["record_id"]: row for row in result[0]["results"]}
|
||||
assert rows["hit"]["collection_source"] == "xingtu"
|
||||
assert rows["miss"]["collection_source"] == "buyin_fallback"
|
||||
assert rows["miss"]["fallback_from"] == "xingtu"
|
||||
|
||||
|
||||
def test_xingtu_search_budget_blocks_duplicate_without_failure_circuit():
|
||||
budget = xingtu.CreatorSearchBudget(max_consecutive_failures=3)
|
||||
|
||||
@@ -291,20 +358,22 @@ def test_scheduled_collection_has_no_automatic_retry_and_syncs_once():
|
||||
)
|
||||
|
||||
|
||||
def test_daily_collection_uses_dedicated_self_operated_pipeline():
|
||||
def test_daily_collection_includes_self_operated_pipeline():
|
||||
steps = _workflow("content.metrics.daily").steps
|
||||
entries = [step.entry for step in steps]
|
||||
run_step = next(step for step in steps if step.entry == "run_all.py")
|
||||
|
||||
assert "--daily-scope" in run_step.args
|
||||
assert "--include-self-operated" not in run_step.args
|
||||
assert "data/tools/refresh_self_mapping.py" in entries
|
||||
assert "self_bilibili_scraper.py" in entries
|
||||
assert "chanmama_scraper.py" in entries
|
||||
refresh_index = entries.index("data/tools/refresh_self_mapping.py")
|
||||
bili_index = entries.index("self_bilibili_scraper.py")
|
||||
douyin_index = entries.index("chanmama_scraper.py")
|
||||
assert refresh_index < bili_index < douyin_index
|
||||
assert entries.index("run_all.py") < entries.index(
|
||||
"data/tools/refresh_self_mapping.py"
|
||||
)
|
||||
assert entries.index("data/tools/refresh_self_mapping.py") < entries.index(
|
||||
"self_bilibili_scraper.py"
|
||||
)
|
||||
assert entries.index("self_bilibili_scraper.py") < entries.index(
|
||||
"chanmama_scraper.py"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools import sync_notes_master as sync
|
||||
|
||||
|
||||
def _table() -> dict:
|
||||
return {
|
||||
"name": "宙斯",
|
||||
"base_token": "base-token",
|
||||
"table_id": "tbl-source",
|
||||
"field_map": {
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "published"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"platform": {"field_id": "platform"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_parse_records_counts_only_complete_note_rows() -> None:
|
||||
rows = sync.parse_records(
|
||||
_table(),
|
||||
[
|
||||
{
|
||||
"record_id": "complete",
|
||||
"title": "通勤双肩包实测",
|
||||
"published": "2026-08-11 09:20:00",
|
||||
"url": "[笔记](https://www.xiaohongshu.com/explore/abc)",
|
||||
"platform": "小红书",
|
||||
"creator": "达人甲",
|
||||
},
|
||||
{
|
||||
"record_id": "missing-publish-time",
|
||||
"title": "只有标题和链接",
|
||||
"published": None,
|
||||
"url": "https://www.douyin.com/video/123",
|
||||
"platform": "抖音",
|
||||
"creator": "达人乙",
|
||||
},
|
||||
],
|
||||
self_operated=False,
|
||||
)
|
||||
|
||||
assert rows[0].is_countable is True
|
||||
assert rows[0].platform == "xiaohongshu"
|
||||
assert rows[0].publish_time == "2026-08-11 09:20:00+08:00"
|
||||
assert rows[0].url == "https://www.xiaohongshu.com/explore/abc"
|
||||
assert rows[1].is_countable is False
|
||||
|
||||
|
||||
def test_platform_can_be_inferred_from_url_when_source_cell_is_empty() -> None:
|
||||
rows = sync.parse_records(
|
||||
_table(),
|
||||
[{
|
||||
"record_id": "bili",
|
||||
"title": "B站开箱",
|
||||
"published": "2026-08-10",
|
||||
"url": "https://www.bilibili.com/video/BV1test",
|
||||
"platform": "",
|
||||
"creator": "自营号",
|
||||
}],
|
||||
self_operated=True,
|
||||
)
|
||||
|
||||
assert rows[0].platform == "bilibili"
|
||||
assert rows[0].is_countable is True
|
||||
|
||||
|
||||
def test_unrecognized_source_text_is_bounded_to_database_columns() -> None:
|
||||
rows = sync.parse_records(
|
||||
_table(),
|
||||
[{
|
||||
"record_id": "unexpected-long-values",
|
||||
"title": "题" * 600,
|
||||
"published": "2026-08-10",
|
||||
"url": "not-a-platform-url",
|
||||
"platform": "unexpected-platform-value-" * 4,
|
||||
"creator": "创" * 300,
|
||||
}],
|
||||
self_operated=False,
|
||||
)
|
||||
|
||||
assert len(rows[0].platform) == 32
|
||||
assert len(rows[0].creator_name) == 255
|
||||
assert len(rows[0].title) == 512
|
||||
|
||||
|
||||
def test_collaboration_fields_parsed_only_for_collaborator_tables() -> None:
|
||||
table = _table()
|
||||
table["field_map"].update(
|
||||
{
|
||||
"cost": {"field_id": "cost"},
|
||||
"tracking_no": {"field_id": "tracking"},
|
||||
"is_paid": {"field_id": "paid"},
|
||||
"follower_count": {"field_id": "followers"},
|
||||
}
|
||||
)
|
||||
records = [{
|
||||
"record_id": "coop-1",
|
||||
"title": "合作笔记",
|
||||
"published": "2026-08-11",
|
||||
"url": "https://www.douyin.com/video/9",
|
||||
"platform": "抖音",
|
||||
"creator": "达人丙",
|
||||
"cost": 1200,
|
||||
"tracking": "SF123",
|
||||
"paid": True,
|
||||
"followers": "3.7万",
|
||||
}]
|
||||
|
||||
coop_row = sync.parse_records(table, records, self_operated=False)[0]
|
||||
assert coop_row.coop_attrs["cooperation_cost"] == 1200.0
|
||||
assert coop_row.coop_attrs["tracking_number"] == "SF123"
|
||||
assert coop_row.coop_attrs["is_paid"] is True
|
||||
assert coop_row.creator_attrs["follower_count_num"] == 37000
|
||||
|
||||
self_row = sync.parse_records(table, records, self_operated=True)[0]
|
||||
assert self_row.coop_attrs == {}
|
||||
assert self_row.creator_attrs == {}
|
||||
|
||||
|
||||
def test_dry_run_fetches_both_scopes_without_opening_database(monkeypatch) -> None:
|
||||
collaborator = {**_table(), "name": "合作款"}
|
||||
self_operated = {**_table(), "name": "自营款", "table_id": "tbl-self"}
|
||||
monkeypatch.setattr(
|
||||
sync,
|
||||
"load_sources",
|
||||
lambda **_kwargs: [(False, collaborator), (True, self_operated)],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync.feishu_mapping,
|
||||
"_record_list_all",
|
||||
lambda *_args: [{
|
||||
"record_id": "one",
|
||||
"title": "完整笔记",
|
||||
"published": "2026-08-11",
|
||||
"url": "https://www.douyin.com/video/1",
|
||||
"platform": "抖音",
|
||||
"creator": "达人",
|
||||
}],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync.psycopg,
|
||||
"connect",
|
||||
lambda **_kwargs: (_ for _ in ()).throw(
|
||||
AssertionError("dry-run must not open PostgreSQL")
|
||||
),
|
||||
)
|
||||
|
||||
summary = sync.synchronize()
|
||||
|
||||
assert summary == {
|
||||
"tables": 2,
|
||||
"rows": 2,
|
||||
"creators": 0,
|
||||
"countable": 2,
|
||||
"incomplete": 0,
|
||||
"deactivated": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_sync_table_deactivates_only_rows_missing_from_same_source() -> None:
|
||||
class Cursor:
|
||||
rowcount = 3
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = []
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
self.calls.append((" ".join(sql.split()), params))
|
||||
|
||||
def fetchone(self):
|
||||
return (42,)
|
||||
|
||||
cur = Cursor()
|
||||
rows = [
|
||||
sync.MasterRow(
|
||||
record_id="rec-1",
|
||||
platform="xiaohongshu",
|
||||
creator_name="达人",
|
||||
title="标题",
|
||||
publish_time="2026-08-11 00:00:00+08:00",
|
||||
url="https://www.xiaohongshu.com/explore/1",
|
||||
)
|
||||
]
|
||||
|
||||
result = sync.sync_table(cur, _table(), self_operated=False, rows=rows)
|
||||
|
||||
assert result["countable"] == 1
|
||||
deactivate_sql, params = cur.calls[-1]
|
||||
assert "source_base_token = %s AND source_table_id = %s" in deactivate_sql
|
||||
assert "NOT (feishu_record_id = ANY(%s))" in deactivate_sql
|
||||
assert params == ("base-token", "tbl-source", ["rec-1"])
|
||||
@@ -15,25 +15,31 @@ from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "state.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
cookie_backup, profile_backup, state_backup = module.reset_state()
|
||||
assert not cookie.exists()
|
||||
assert not storage.exists()
|
||||
assert not profile.exists()
|
||||
|
||||
cookie.write_text("partial-new-cookie", encoding="utf-8")
|
||||
storage.write_text("partial-new-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
module.restore_previous_state(cookie_backup, profile_backup)
|
||||
module.restore_previous_state(cookie_backup, profile_backup, state_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
|
||||
@@ -41,21 +47,27 @@ def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, modul
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
cookie_backup, profile_backup, state_backup = module.reset_state()
|
||||
cookie.write_text("new-cookie", encoding="utf-8")
|
||||
storage.write_text("new-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
module.finish_successful_relogin(profile_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "new-cookie"
|
||||
assert storage.read_text(encoding="utf-8") == "new-storage"
|
||||
assert cookie_backup.exists()
|
||||
assert state_backup.exists()
|
||||
assert not profile_backup.exists()
|
||||
|
||||
|
||||
@@ -127,6 +139,82 @@ def test_xingtu_index_page_is_recognized_as_logged_in():
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_index_page_with_xingtu_session_is_logged_in_without_market_ui():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def count(self):
|
||||
return 0
|
||||
|
||||
def wait_for(self, timeout):
|
||||
raise xingtu_scraper_v2.BrowserTimeoutError("not on market page")
|
||||
|
||||
def is_visible(self):
|
||||
return False
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [
|
||||
{
|
||||
"name": "sessionid",
|
||||
"value": "fresh",
|
||||
"domain": ".xingtu.cn",
|
||||
}
|
||||
]
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/ad/creator/index"
|
||||
context = Context()
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
def locator(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_index_page_without_xingtu_session_is_not_logged_in():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def count(self):
|
||||
return 0
|
||||
|
||||
def wait_for(self, timeout):
|
||||
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
|
||||
|
||||
def is_visible(self):
|
||||
return False
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [
|
||||
{
|
||||
"name": "sessionid",
|
||||
"value": "douyin-only",
|
||||
"domain": ".douyin.com",
|
||||
}
|
||||
]
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/ad/creator/index"
|
||||
context = Context()
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
def locator(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie():
|
||||
class Context:
|
||||
def cookies(self):
|
||||
@@ -149,7 +237,7 @@ def test_xingtu_market_page_without_business_ui_rejects_stale_cookie():
|
||||
return self
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
raise xingtu_scraper_v2.PlaywrightTimeoutError("missing")
|
||||
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
|
||||
|
||||
def count(self):
|
||||
return 0
|
||||
@@ -282,25 +370,64 @@ def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monke
|
||||
assert saved == [market_page]
|
||||
|
||||
|
||||
def test_xingtu_scan_login_detects_business_tab_while_qr_tab_stays_open(monkeypatch):
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.pages = []
|
||||
|
||||
class Page:
|
||||
def __init__(self, context, url):
|
||||
self.context = context
|
||||
self.url = url
|
||||
|
||||
def is_closed(self):
|
||||
return False
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
raise AssertionError("authenticated tab should be detected immediately")
|
||||
|
||||
context = Context()
|
||||
qr_page = Page(context, "https://open.douyin.com/platform/oauth/pc/auth")
|
||||
index_page = Page(context, "https://www.xingtu.cn/ad/creator/index")
|
||||
context.pages = [qr_page, index_page]
|
||||
saved = []
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None)
|
||||
monkeypatch.setattr(
|
||||
xingtu_scraper_v2, "is_logged_in", lambda page: page is index_page
|
||||
)
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page))
|
||||
|
||||
result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2)
|
||||
|
||||
assert result is index_page
|
||||
assert saved == [index_page]
|
||||
|
||||
|
||||
def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch):
|
||||
cookie = tmp_path / "xingtu_cookies.json"
|
||||
storage = tmp_path / "xingtu_storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(relogin_xingtu, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = relogin_xingtu.reset_state()
|
||||
cookie_backup, profile_backup, state_backup = relogin_xingtu.reset_state()
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
assert relogin_xingtu.recover_interrupted_state() is True
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
assert cookie_backup is not None
|
||||
assert state_backup is not None
|
||||
assert profile_backup is not None
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
relogin_bilibili,
|
||||
relogin_douyin,
|
||||
relogin_pgy,
|
||||
relogin_runtime_paths,
|
||||
relogin_xiaohongshu,
|
||||
relogin_xingtu,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
|
||||
CONSUMER_BINDING_IDS,
|
||||
child_browser_environment,
|
||||
resolve_relogin_runtime_paths,
|
||||
sync_relogin_state_to_consumers,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def test_relogin_paths_prefer_injected_runtime_binding(tmp_path: Path) -> None:
|
||||
cookie = tmp_path / "binding" / "cookies.json"
|
||||
storage = tmp_path / "binding" / "storage_state.json"
|
||||
profile = tmp_path / "binding" / "profile"
|
||||
|
||||
resolved = resolve_relogin_runtime_paths(
|
||||
"pgy",
|
||||
{
|
||||
"GYXX_BROWSER_COOKIE_FILE": str(cookie),
|
||||
"GYXX_BROWSER_STORAGE_STATE_FILE": str(storage),
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(profile),
|
||||
},
|
||||
)
|
||||
|
||||
assert resolved.cookie_file == cookie.resolve()
|
||||
assert resolved.storage_state_file == storage.resolve()
|
||||
assert resolved.profile_dir == profile.resolve()
|
||||
|
||||
|
||||
def test_relogin_paths_keep_legacy_standalone_layout() -> None:
|
||||
resolved = resolve_relogin_runtime_paths("xiaohongshu", {})
|
||||
|
||||
assert resolved.cookie_file == (
|
||||
PATHS.state_root / "cookies/xiaohongshu_cookies.json"
|
||||
).resolve()
|
||||
assert resolved.storage_state_file == (
|
||||
PATHS.state_root / "cookies/xiaohongshu_storage_state.json"
|
||||
).resolve()
|
||||
assert resolved.profile_dir == (
|
||||
PATHS.state_root / "browser-profiles/xiaohongshu"
|
||||
).resolve()
|
||||
|
||||
|
||||
def test_child_login_environment_preserves_wrapper_binding(tmp_path: Path) -> None:
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
|
||||
environment = child_browser_environment(
|
||||
cookie_file=cookie,
|
||||
storage_state_file=storage,
|
||||
profile_dir=profile,
|
||||
environment={"GYXX_MODULE_ID": "content_marketing", "KEEP": "1"},
|
||||
)
|
||||
|
||||
assert environment["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
|
||||
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(
|
||||
storage.resolve()
|
||||
)
|
||||
assert environment["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
|
||||
assert "GYXX_MODULE_ID" not in environment
|
||||
assert environment["KEEP"] == "1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "login_cookie", "argv"),
|
||||
[
|
||||
(
|
||||
relogin_bilibili,
|
||||
"bilibili",
|
||||
"SESSDATA",
|
||||
["relogin_bilibili.py"],
|
||||
),
|
||||
(relogin_douyin, "douyin", "sessionid", ["relogin_douyin.py"]),
|
||||
(relogin_pgy, "pgy", "web_session", ["relogin_pgy.py"]),
|
||||
(
|
||||
relogin_xiaohongshu,
|
||||
"xiaohongshu",
|
||||
"web_session",
|
||||
["relogin_xiaohongshu.py"],
|
||||
),
|
||||
(
|
||||
relogin_xingtu,
|
||||
"xingtu",
|
||||
"sessionid",
|
||||
["relogin_xingtu.py", "--force"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_relogin_wrapper_validates_the_same_binding_written_by_child(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
module: ModuleType,
|
||||
platform: str,
|
||||
login_cookie: str,
|
||||
argv: list[str],
|
||||
) -> None:
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
monkeypatch.setattr(module.sys, "argv", argv)
|
||||
monkeypatch.setenv("GYXX_MODULE_ID", "content_marketing")
|
||||
if module is relogin_xingtu:
|
||||
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
|
||||
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
|
||||
synced: list[tuple[str, Path, Path]] = []
|
||||
|
||||
def fake_sync(
|
||||
synced_platform: str,
|
||||
*,
|
||||
cookie_file: Path,
|
||||
storage_state_file: Path,
|
||||
) -> tuple[Path, ...]:
|
||||
synced.append(
|
||||
(synced_platform, cookie_file.resolve(), storage_state_file.resolve())
|
||||
)
|
||||
return ()
|
||||
|
||||
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fake_sync)
|
||||
|
||||
def fake_call(
|
||||
_command: list[str],
|
||||
*,
|
||||
cwd: str,
|
||||
env: dict[str, str],
|
||||
) -> int:
|
||||
assert cwd == str(module.PROJECT_DIR)
|
||||
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
|
||||
assert env["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(storage.resolve())
|
||||
assert env["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
|
||||
assert "GYXX_MODULE_ID" not in env
|
||||
cookie.write_text(
|
||||
json.dumps([{"name": login_cookie, "value": "fresh"}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
storage.write_text("{}", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(module.subprocess, "call", fake_call)
|
||||
|
||||
assert module.main() == 0
|
||||
assert cookie.is_file()
|
||||
assert storage.is_file()
|
||||
assert profile.is_dir()
|
||||
assert synced == [(platform, cookie.resolve(), storage.resolve())]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "login_cookie", "cookie_domain", "argv"),
|
||||
[
|
||||
(
|
||||
relogin_bilibili,
|
||||
"bilibili",
|
||||
"SESSDATA",
|
||||
".bilibili.com",
|
||||
["relogin_bilibili.py"],
|
||||
),
|
||||
(
|
||||
relogin_douyin,
|
||||
"douyin",
|
||||
"sessionid",
|
||||
".douyin.com",
|
||||
["relogin_douyin.py"],
|
||||
),
|
||||
(
|
||||
relogin_pgy,
|
||||
"pgy",
|
||||
"web_session",
|
||||
".xiaohongshu.com",
|
||||
["relogin_pgy.py"],
|
||||
),
|
||||
(
|
||||
relogin_xiaohongshu,
|
||||
"xiaohongshu",
|
||||
"web_session",
|
||||
".xiaohongshu.com",
|
||||
["relogin_xiaohongshu.py"],
|
||||
),
|
||||
(
|
||||
relogin_xingtu,
|
||||
"xingtu",
|
||||
"sessionid",
|
||||
".xingtu.cn",
|
||||
["relogin_xingtu.py", "--force"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_relogin_wrapper_restores_previous_state_when_publish_fails(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
module: ModuleType,
|
||||
platform: str,
|
||||
login_cookie: str,
|
||||
cookie_domain: str,
|
||||
argv: list[str],
|
||||
) -> None:
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
old_cookie = '[{"name":"old","value":"old-session"}]'
|
||||
old_storage = '{"cookies":[{"name":"old"}]}'
|
||||
cookie.write_text(old_cookie, encoding="utf-8")
|
||||
storage.write_text(old_storage, encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old-state.txt").write_text("old-profile", encoding="utf-8")
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
monkeypatch.setattr(module.sys, "argv", argv)
|
||||
if module is relogin_xingtu:
|
||||
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
|
||||
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
|
||||
|
||||
def fake_call(
|
||||
_command: list[str],
|
||||
*,
|
||||
cwd: str,
|
||||
env: dict[str, str],
|
||||
) -> int:
|
||||
assert cwd == str(module.PROJECT_DIR)
|
||||
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
|
||||
cookie.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"name": login_cookie,
|
||||
"value": "fresh-session",
|
||||
"domain": cookie_domain,
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
|
||||
profile.mkdir()
|
||||
return 0
|
||||
|
||||
def fail_publish(
|
||||
synced_platform: str,
|
||||
*,
|
||||
cookie_file: Path,
|
||||
storage_state_file: Path,
|
||||
) -> tuple[Path, ...]:
|
||||
assert synced_platform == platform
|
||||
assert cookie_file == cookie
|
||||
assert storage_state_file == storage
|
||||
raise OSError("simulated consumer publish failure")
|
||||
|
||||
monkeypatch.setattr(module.subprocess, "call", fake_call)
|
||||
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fail_publish)
|
||||
|
||||
assert module.main() == 1
|
||||
assert cookie.read_text(encoding="utf-8") == old_cookie
|
||||
assert storage.read_text(encoding="utf-8") == old_storage
|
||||
assert (profile / "old-state.txt").read_text(encoding="utf-8") == (
|
||||
"old-profile"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "cookie_name", "cookie_domain"),
|
||||
[
|
||||
("bilibili", "SESSDATA", ".bilibili.com"),
|
||||
("douyin", "sessionid", ".douyin.com"),
|
||||
("pgy", "web_session", ".xiaohongshu.com"),
|
||||
("xiaohongshu", "web_session", ".xiaohongshu.com"),
|
||||
("xingtu", "sessionid", ".xingtu.cn"),
|
||||
],
|
||||
)
|
||||
def test_successful_relogin_atomically_updates_unique_consumer_state(
|
||||
tmp_path: Path,
|
||||
platform: str,
|
||||
cookie_name: str,
|
||||
cookie_domain: str,
|
||||
) -> None:
|
||||
data_root = tmp_path / "data"
|
||||
source_cookie = tmp_path / "relogin" / "cookies.json"
|
||||
source_storage = tmp_path / "relogin" / "storage_state.json"
|
||||
source_cookie.parent.mkdir(parents=True)
|
||||
fresh_cookie = [
|
||||
{
|
||||
"name": cookie_name,
|
||||
"value": "fresh-session",
|
||||
"domain": cookie_domain,
|
||||
}
|
||||
]
|
||||
fresh_storage = {"cookies": fresh_cookie, "origins": []}
|
||||
source_cookie.write_text(json.dumps(fresh_cookie), encoding="utf-8")
|
||||
source_storage.write_text(json.dumps(fresh_storage), encoding="utf-8")
|
||||
|
||||
catalog = RuntimeIntegrationCatalog.load_default(
|
||||
project_root=PROJECT_ROOT,
|
||||
data_root=data_root,
|
||||
)
|
||||
consumer_id = CONSUMER_BINDING_IDS[platform][0]
|
||||
consumer = catalog.binding_for(consumer_id)
|
||||
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
consumer.cookie_file.write_text('[{"name":"old"}]', encoding="utf-8")
|
||||
consumer.storage_state_file.write_text(
|
||||
'{"cookies":[{"name":"old"}]}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
destinations = sync_relogin_state_to_consumers(
|
||||
platform,
|
||||
cookie_file=source_cookie,
|
||||
storage_state_file=source_storage,
|
||||
environment={
|
||||
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
|
||||
"GYXX_DATA_ROOT": str(data_root),
|
||||
},
|
||||
)
|
||||
|
||||
assert source_cookie.resolve() != consumer.cookie_file.resolve()
|
||||
assert source_storage.resolve() != consumer.storage_state_file.resolve()
|
||||
assert destinations == (consumer.cookie_file.resolve(),)
|
||||
assert json.loads(consumer.cookie_file.read_text(encoding="utf-8")) == (
|
||||
fresh_cookie
|
||||
)
|
||||
assert json.loads(
|
||||
consumer.storage_state_file.read_text(encoding="utf-8")
|
||||
) == fresh_storage
|
||||
|
||||
|
||||
def test_consumer_state_is_rolled_back_if_atomic_publish_fails(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
data_root = tmp_path / "data"
|
||||
source_cookie = tmp_path / "relogin" / "cookies.json"
|
||||
source_storage = tmp_path / "relogin" / "storage_state.json"
|
||||
source_cookie.parent.mkdir(parents=True)
|
||||
source_cookie.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"name": "SESSDATA",
|
||||
"value": "fresh-session",
|
||||
"domain": ".bilibili.com",
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
source_storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
|
||||
|
||||
catalog = RuntimeIntegrationCatalog.load_default(
|
||||
project_root=PROJECT_ROOT,
|
||||
data_root=data_root,
|
||||
)
|
||||
consumer = catalog.binding_for(CONSUMER_BINDING_IDS["bilibili"][0])
|
||||
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
old_cookie = '[{"name":"SESSDATA","value":"old"}]'
|
||||
old_storage = '{"cookies":[{"name":"old"}]}'
|
||||
consumer.cookie_file.write_text(old_cookie, encoding="utf-8")
|
||||
consumer.storage_state_file.write_text(old_storage, encoding="utf-8")
|
||||
|
||||
original_replace = relogin_runtime_paths.os.replace
|
||||
failed = False
|
||||
|
||||
def fail_second_replace(source: Path | str, target: Path | str) -> None:
|
||||
nonlocal failed
|
||||
source_path = Path(source)
|
||||
target_path = Path(target)
|
||||
if (
|
||||
not failed
|
||||
and source_path.suffix == ".tmp"
|
||||
and target_path == consumer.storage_state_file
|
||||
):
|
||||
failed = True
|
||||
raise OSError("simulated storage publish failure")
|
||||
original_replace(source, target)
|
||||
|
||||
monkeypatch.setattr(relogin_runtime_paths.os, "replace", fail_second_replace)
|
||||
|
||||
with pytest.raises(OSError, match="simulated storage publish failure"):
|
||||
sync_relogin_state_to_consumers(
|
||||
"bilibili",
|
||||
cookie_file=source_cookie,
|
||||
storage_state_file=source_storage,
|
||||
environment={
|
||||
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
|
||||
"GYXX_DATA_ROOT": str(data_root),
|
||||
},
|
||||
)
|
||||
|
||||
assert consumer.cookie_file.read_text(encoding="utf-8") == old_cookie
|
||||
assert consumer.storage_state_file.read_text(encoding="utf-8") == old_storage
|
||||
@@ -1,6 +1,7 @@
|
||||
import io
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.adapters import RuntimeServicePolicy
|
||||
from gyxx_flow.modules.content_marketing.data.tools import db
|
||||
@@ -60,7 +61,7 @@ class LocalDatabaseConfigTests(unittest.TestCase):
|
||||
paths = (
|
||||
ROOT / "data" / "tools" / "db.py",
|
||||
ROOT / "data" / "tools" / "sync_metrics_to_cmt_notes.py",
|
||||
ROOT / "data" / "tools" / "sync_cooperations.py",
|
||||
ROOT / "data" / "tools" / "sync_notes_master.py",
|
||||
ROOT / "data" / "tools" / "generate_creator_report.py",
|
||||
)
|
||||
for path in paths:
|
||||
@@ -79,6 +80,33 @@ class LocalDatabaseConfigTests(unittest.TestCase):
|
||||
|
||||
self.assertTrue(raw.getvalue())
|
||||
|
||||
def test_explicit_insert_missing_preserves_default_sync_contract(self):
|
||||
with (
|
||||
patch("sys.argv", ["sync_metrics_to_cmt_notes.py", "--insert-missing"]),
|
||||
patch.object(
|
||||
sync_metrics,
|
||||
"sync",
|
||||
return_value={"identity_ambiguous": 0},
|
||||
) as sync_call,
|
||||
):
|
||||
sync_metrics.main()
|
||||
|
||||
sync_call.assert_called_once_with(dry_run=False, insert_missing=True)
|
||||
|
||||
def test_default_sync_exits_nonzero_when_identity_is_ambiguous(self):
|
||||
with (
|
||||
patch("sys.argv", ["sync_metrics_to_cmt_notes.py"]),
|
||||
patch.object(
|
||||
sync_metrics,
|
||||
"sync",
|
||||
return_value={"identity_ambiguous": 2},
|
||||
),
|
||||
self.assertRaises(SystemExit) as exc_info,
|
||||
):
|
||||
sync_metrics.main()
|
||||
|
||||
self.assertEqual(exc_info.exception.code, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -39,6 +40,54 @@ def test_success_status_cannot_hide_write_failure_and_blocked_input_is_terminal(
|
||||
}],
|
||||
}
|
||||
assert retry.extract_failed_records(blocked, "bili") == []
|
||||
assert retry.requested_records_succeeded(blocked, ["b"]) == (True, [])
|
||||
|
||||
|
||||
def test_requested_records_require_exactly_one_resolved_blocked_terminal():
|
||||
duplicate_blocked = {
|
||||
"results": [
|
||||
{
|
||||
"record_id": "b",
|
||||
"status": "blocked_input",
|
||||
"matched": False,
|
||||
"reason": "url_missing",
|
||||
},
|
||||
{
|
||||
"record_id": "b",
|
||||
"status": "blocked_input",
|
||||
"matched": False,
|
||||
"reason": "url_missing",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
assert retry.requested_records_succeeded(duplicate_blocked, ["b"]) == (
|
||||
False,
|
||||
["b"],
|
||||
)
|
||||
|
||||
|
||||
def test_blocked_input_cannot_hide_explicit_execution_or_write_failure():
|
||||
for explicit_failure in (
|
||||
{"write_ok": False},
|
||||
{"write_error": "database rejected update"},
|
||||
{"error": "browser session crashed"},
|
||||
):
|
||||
blocked = {
|
||||
"results": [{
|
||||
"record_id": "b",
|
||||
"status": "blocked_input",
|
||||
"matched": False,
|
||||
"reason": "url_missing",
|
||||
**explicit_failure,
|
||||
}],
|
||||
}
|
||||
|
||||
assert retry.requested_records_succeeded(blocked, ["b"]) == (
|
||||
False,
|
||||
["b"],
|
||||
)
|
||||
assert retry.extract_failed_records(blocked, "bili") == ["b"]
|
||||
|
||||
|
||||
def test_canonical_filename_requires_current_style_name():
|
||||
@@ -305,6 +354,87 @@ def test_validate_summary_rejects_expected_style_left_over_from_old_run(tmp_path
|
||||
assert "current run" in reason
|
||||
|
||||
|
||||
def test_build_retry_targets_narrows_retryable_rows_and_expands_structural_gaps():
|
||||
targets = run_all.build_retry_targets(
|
||||
{
|
||||
"bili": [{
|
||||
"index": 1,
|
||||
"total_b_records": 2,
|
||||
"details": [
|
||||
{"record_id": "ok", "status": "success", "ok": True},
|
||||
{"record_id": "bad", "status": "retryable_failure", "ok": False},
|
||||
],
|
||||
}],
|
||||
"pgy": [{
|
||||
"index": 1,
|
||||
"total": 2,
|
||||
"results": [{"record_id": "only-one", "status": "success"}],
|
||||
}],
|
||||
"xt": [{
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"results": [{
|
||||
"record_id": "terminal",
|
||||
"status": "blocked_input",
|
||||
"matched": False,
|
||||
}],
|
||||
}],
|
||||
},
|
||||
{1},
|
||||
)
|
||||
|
||||
assert targets == {
|
||||
"bili": {1: {"bad"}},
|
||||
"pgy": {1: None},
|
||||
}
|
||||
|
||||
|
||||
def test_retry_manifest_round_trips_record_and_full_style_scopes(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(run_all, "RETRY_MANIFEST_DIR", tmp_path)
|
||||
targets = {"bili": {1: {"r2", "r1"}}, "pgy": {3: None}}
|
||||
|
||||
run_all._write_retry_manifest(
|
||||
date(2026, 9, 2), targets, source_slot="01:00"
|
||||
)
|
||||
|
||||
assert run_all._load_retry_manifest(date(2026, 9, 2)) == targets
|
||||
|
||||
|
||||
def test_retry_process_command_passes_only_target_records():
|
||||
args = SimpleNamespace(
|
||||
dry_run=False,
|
||||
style=[1, 2],
|
||||
daily_scope=True,
|
||||
no_retry=False,
|
||||
)
|
||||
|
||||
command = run_all.build_process_command(
|
||||
"pgy", args, retry_targets={2: {"record-b", "record-a"}}
|
||||
)
|
||||
|
||||
assert command[command.index("--style") + 1] == "2"
|
||||
assert command.count("--style") == 1
|
||||
assert command[command.index("--record") + 1] == "record-a"
|
||||
assert command.count("--record") == 2
|
||||
assert "--no-retry" in command
|
||||
|
||||
|
||||
def test_retry_process_command_uses_full_style_when_scope_is_structurally_incomplete():
|
||||
args = SimpleNamespace(
|
||||
dry_run=False,
|
||||
style=[1, 2],
|
||||
daily_scope=True,
|
||||
no_retry=False,
|
||||
)
|
||||
|
||||
command = run_all.build_process_command(
|
||||
"xt", args, retry_targets={1: None}
|
||||
)
|
||||
|
||||
assert command[command.index("--style") + 1] == "1"
|
||||
assert "--record" not in command
|
||||
|
||||
|
||||
def test_aggregate_counts_all_unresolved_rows_as_errors():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": [{
|
||||
@@ -363,6 +493,40 @@ def test_aggregate_keeps_partial_counts_when_validation_marks_payload_invalid():
|
||||
assert platform["validation_error"] == "invalid summary: unresolved"
|
||||
|
||||
|
||||
def test_aggregate_fails_closed_for_stale_payload():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": {
|
||||
"error": "invalid summary: summary file is stale",
|
||||
"payload": [{
|
||||
"index": 1,
|
||||
"total": 141,
|
||||
"matched": 101,
|
||||
"results": [],
|
||||
}],
|
||||
},
|
||||
})
|
||||
|
||||
platform = aggregate["platforms"]["pgy"]
|
||||
assert platform["loaded"] is False
|
||||
assert platform["error"] == "invalid summary: summary file is stale"
|
||||
assert aggregate["totals"]["tasks"] == 0
|
||||
|
||||
|
||||
def test_aggregate_filters_merged_history_to_current_styles():
|
||||
aggregate = run_all.aggregate(
|
||||
{
|
||||
"xt": [
|
||||
{"index": 1, "total": 2, "matched": 1, "results": []},
|
||||
{"index": 2, "total": 99, "matched": 99, "results": []},
|
||||
],
|
||||
},
|
||||
expected_indices={1},
|
||||
)
|
||||
|
||||
assert aggregate["platforms"]["xt"]["styles"] == 1
|
||||
assert aggregate["platforms"]["xt"]["tasks"] == 2
|
||||
|
||||
|
||||
def test_run_round_marks_exit_zero_platform_failed_when_summary_is_invalid(monkeypatch, tmp_path):
|
||||
args = SimpleNamespace(dry_run=True, style=[1])
|
||||
stdout = tmp_path / "stdout.log"
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.modules.content_marketing import weekly_summary_all
|
||||
|
||||
|
||||
def test_target_table_payload_uses_temp_cwd_and_relative_reference(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
payload_root = tmp_path / "payloads"
|
||||
payload_root.mkdir()
|
||||
working_directory = tmp_path / "working-directory"
|
||||
working_directory.mkdir()
|
||||
monkeypatch.chdir(working_directory)
|
||||
monkeypatch.setattr(
|
||||
weekly_summary_all,
|
||||
"PATHS",
|
||||
SimpleNamespace(tmp_root=payload_root),
|
||||
)
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
|
||||
calls: list[tuple[list[str], Path | None]] = []
|
||||
|
||||
def fake_call_lark_json(
|
||||
args: list[str], *, cwd: Path | None = None
|
||||
) -> dict[str, object]:
|
||||
calls.append((args, cwd))
|
||||
if args[1] == "+record-list":
|
||||
assert cwd is None
|
||||
return {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"record_id_list": [],
|
||||
"data": [],
|
||||
"field_id_list": [],
|
||||
},
|
||||
}
|
||||
assert cwd == payload_root
|
||||
assert cwd != Path.cwd()
|
||||
payload_reference = args[args.index("--json") + 1]
|
||||
assert payload_reference.startswith("@./")
|
||||
payload_path = cwd / payload_reference.removeprefix("@./")
|
||||
assert not Path(payload_reference[1:]).is_absolute()
|
||||
assert json.loads(payload_path.read_text(encoding="utf-8")) == {
|
||||
"fld_time": "2026-07-01~07-31",
|
||||
"fld_summary": (
|
||||
"[凌云air 月度汇总(2026-07-01~07-31)]"
|
||||
"(https://example.test/doc)"
|
||||
),
|
||||
}
|
||||
return {"ok": True}
|
||||
|
||||
monkeypatch.setattr(weekly_summary_all, "call_lark_json", fake_call_lark_json)
|
||||
|
||||
assert weekly_summary_all.write_to_target_table(
|
||||
"base-test",
|
||||
"table-test",
|
||||
"fld_time",
|
||||
"fld_summary",
|
||||
"凌云air",
|
||||
"https://example.test/doc",
|
||||
False,
|
||||
time_label="2026-07-01~07-31",
|
||||
report_label="月度汇总",
|
||||
)
|
||||
assert [command[1] for command, _cwd in calls] == [
|
||||
"+record-list",
|
||||
"+record-upsert",
|
||||
]
|
||||
|
||||
|
||||
def test_call_lark_json_logs_only_redacted_failure_detail(
|
||||
monkeypatch,
|
||||
capsys,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
raw_secret = "sensitive-token-value"
|
||||
raw_open_id = "sensitive-open-id"
|
||||
captured_kwargs: dict[str, object] = {}
|
||||
|
||||
def fake_run(*_args, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return SimpleNamespace(
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr=(
|
||||
f"request failed; base_token={raw_secret}; "
|
||||
f"open_id={raw_open_id}; https://example.test/private"
|
||||
),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(weekly_summary_all.subprocess, "run", fake_run)
|
||||
|
||||
response = weekly_summary_all.call_lark_json(
|
||||
["base", "+record-upsert"], cwd=tmp_path
|
||||
)
|
||||
|
||||
captured = capsys.readouterr().out
|
||||
assert captured_kwargs["cwd"] == tmp_path
|
||||
assert response["ok"] is False
|
||||
assert raw_secret not in response["error"]
|
||||
assert raw_open_id not in response["error"]
|
||||
assert raw_secret not in captured
|
||||
assert raw_open_id not in captured
|
||||
assert "[REDACTED]" in captured
|
||||
assert "[URL]" in captured
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,32 @@ from gyxx_flow.modules.content_marketing import weekly_summary_all as weekly
|
||||
|
||||
|
||||
class WeeklyTmallPersonaTests(unittest.TestCase):
|
||||
def test_summary_result_requires_all_real_external_effects(self):
|
||||
self.assertTrue(
|
||||
weekly.summary_results_succeeded(
|
||||
[{"status": "ok"}, {"status": "no_notes"}]
|
||||
)
|
||||
)
|
||||
for status in (
|
||||
"no_fields",
|
||||
"no_reports",
|
||||
"llm_failed",
|
||||
"doc_failed",
|
||||
"doc_ambiguous",
|
||||
"db_failed",
|
||||
"write_failed",
|
||||
"exception",
|
||||
):
|
||||
with self.subTest(status=status):
|
||||
self.assertFalse(
|
||||
weekly.summary_results_succeeded([{"status": status}])
|
||||
)
|
||||
|
||||
def test_dry_run_status_is_only_successful_in_dry_run_mode(self):
|
||||
results = [{"status": "dry_run"}]
|
||||
self.assertFalse(weekly.summary_results_succeeded(results))
|
||||
self.assertTrue(weekly.summary_results_succeeded(results, dry_run=True))
|
||||
|
||||
def test_missing_persona_forbids_invented_percentages(self):
|
||||
text = weekly.format_tmall_persona(None)
|
||||
self.assertIn("不得虚构具体比例", text)
|
||||
@@ -56,7 +82,7 @@ class WeeklyTmallPersonaTests(unittest.TestCase):
|
||||
}]
|
||||
|
||||
with patch.object(weekly, "get_latest_tmall_persona", return_value=persona) as get_persona, \
|
||||
patch.object(weekly, "call_hermes_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
|
||||
patch.object(weekly, "call_content_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
|
||||
weekly.generate_summary("星云2", notes)
|
||||
|
||||
get_persona.assert_called_once_with("星云2")
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import xiaohongshu_comment_scraper as scraper
|
||||
|
||||
|
||||
class _EvaluatePage:
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
self.calls = []
|
||||
|
||||
def evaluate(self, script, argument):
|
||||
self.calls.append((script, argument))
|
||||
return self.result
|
||||
|
||||
|
||||
def test_bounded_fetch_uses_abort_controller_and_clears_timer(monkeypatch):
|
||||
page = _EvaluatePage(
|
||||
{"__gyxx_status": "ok", "payload": {"comments": []}}
|
||||
)
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
|
||||
|
||||
payload = scraper._fetch_json_with_timeout(
|
||||
page,
|
||||
"https://example.invalid/comments",
|
||||
deadline=1000.0,
|
||||
stage="top-level pagination",
|
||||
)
|
||||
|
||||
assert payload == {"comments": []}
|
||||
script, argument = page.calls[0]
|
||||
assert "AbortController" in script
|
||||
assert "clearTimeout(timer)" in script
|
||||
assert argument["timeoutMs"] == 30_000
|
||||
|
||||
|
||||
def test_bounded_fetch_timeout_is_an_explicit_transient_failure(monkeypatch):
|
||||
page = _EvaluatePage({"__gyxx_status": "timeout"})
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 100.0)
|
||||
|
||||
with pytest.raises(scraper.TransientXiaohongshuCommentTimeout) as caught:
|
||||
scraper._fetch_json_with_timeout(
|
||||
page,
|
||||
"https://example.invalid/comments",
|
||||
deadline=1000.0,
|
||||
stage="reply pagination",
|
||||
)
|
||||
|
||||
assert isinstance(caught.value, ConnectionError)
|
||||
assert "reply pagination" in str(caught.value)
|
||||
|
||||
|
||||
def test_scroll_checks_the_note_deadline_before_browser_work(monkeypatch):
|
||||
page = _EvaluatePage(None)
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 901.0)
|
||||
|
||||
with pytest.raises(scraper.TransientXiaohongshuCommentTimeout):
|
||||
scraper.scroll_comments(
|
||||
page,
|
||||
max_scrolls=30,
|
||||
idle_rounds=4,
|
||||
comments_by_id={},
|
||||
deadline=900.0,
|
||||
)
|
||||
|
||||
assert page.calls == []
|
||||
|
||||
|
||||
def test_top_and_reply_pagination_share_the_bounded_fetch(monkeypatch):
|
||||
stages = []
|
||||
monkeypatch.setattr(scraper.time, "monotonic", lambda: 0.0)
|
||||
|
||||
def fetch(_page, _url, *, deadline, stage):
|
||||
assert deadline == 900.0
|
||||
stages.append(stage)
|
||||
return {"comments": [], "has_more": 0, "cursor": 0}
|
||||
|
||||
monkeypatch.setattr(scraper, "_fetch_json_with_timeout", fetch)
|
||||
monkeypatch.setattr(scraper, "collect_from_payload", lambda *_args: None)
|
||||
page = _EvaluatePage(None)
|
||||
|
||||
scraper.fetch_missing_top_comments(
|
||||
page,
|
||||
"source-url",
|
||||
{},
|
||||
{
|
||||
"comment_url_template": "https://example.invalid/top",
|
||||
"api_total": 2,
|
||||
"last_cursor": 1,
|
||||
},
|
||||
deadline=900.0,
|
||||
)
|
||||
scraper.fetch_missing_replies(
|
||||
page,
|
||||
"source-url",
|
||||
{
|
||||
"parent": {
|
||||
"level": "comment",
|
||||
"reply_count": 1,
|
||||
"comment_id": "parent-id",
|
||||
}
|
||||
},
|
||||
{
|
||||
"sub_url_template": "https://example.invalid/reply",
|
||||
},
|
||||
deadline=900.0,
|
||||
)
|
||||
|
||||
assert stages == ["top-level pagination", "reply pagination"]
|
||||
@@ -3,11 +3,11 @@ from __future__ import annotations
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
COMMAND_PATH = (
|
||||
@@ -56,7 +56,7 @@ def test_alert_command_uses_canonical_acceptance_recipient(
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID,
|
||||
ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID,
|
||||
)
|
||||
|
||||
|
||||
@@ -72,3 +72,52 @@ def test_alert_command_accepts_canonical_production_recipient(
|
||||
|
||||
assert module.main() == 0
|
||||
assert observed["argv"][-1] == "ou_configured"
|
||||
|
||||
|
||||
def test_alert_command_passes_all_dynamic_recipients(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01")
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
module = _load_command(monkeypatch, observed)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"resolve_notification_route",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
enabled=True,
|
||||
open_ids=("ou_first", "ou_second"),
|
||||
),
|
||||
)
|
||||
|
||||
assert module.main() == 0
|
||||
assert observed["argv"] == (
|
||||
sys.argv[0],
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
"ou_first",
|
||||
"--openid",
|
||||
"ou_second",
|
||||
)
|
||||
|
||||
|
||||
def test_alert_command_uses_safe_no_notify_when_route_is_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01")
|
||||
module = _load_command(monkeypatch, observed)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"resolve_notification_route",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(enabled=False, open_ids=()),
|
||||
)
|
||||
|
||||
assert module.main() == 0
|
||||
assert observed["argv"] == (
|
||||
sys.argv[0],
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--no-notify",
|
||||
)
|
||||
|
||||
@@ -53,6 +53,11 @@ def _prepare_main(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
)
|
||||
monkeypatch.setattr(decline, "managed_data_path", lambda value: Path(value))
|
||||
monkeypatch.setattr(decline, "append_log", lambda line: None)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_persist_detected_events",
|
||||
lambda events, *_args: len(events),
|
||||
)
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
|
||||
|
||||
@@ -142,27 +147,73 @@ def test_segment_loader_uses_postgres_range_sum_on_one_connection(
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("response", "expected"),
|
||||
[
|
||||
({"id": "chatcmpl-not-a-receipt"}, None),
|
||||
({"message_id": "om_direct123"}, "om_direct123"),
|
||||
({"data": {"message_id": "om_nested123"}}, "om_nested123"),
|
||||
(
|
||||
{"choices": [{"message": {"content": "飞书回执 om_content123"}}]},
|
||||
"om_content123",
|
||||
),
|
||||
(
|
||||
{"choices": [{"message": {"content": "发送成功,但没有回执"}}]},
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_feishu_message_id_requires_real_receipt_shape(
|
||||
response: dict,
|
||||
expected: str | None,
|
||||
def test_build_decline_notification_message_is_deterministic() -> None:
|
||||
message = decline.build_decline_notification_message(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
)
|
||||
|
||||
assert "销量连续下滑告警(截至 2026-08-01,3 段 × 3 天)" in message
|
||||
assert "[tm] 测试款" in message
|
||||
assert "300 → 210 → 90" in message
|
||||
assert "累计 -70.0%" in message
|
||||
assert "ERP" in message
|
||||
|
||||
|
||||
def test_alert_delivery_uuid_is_stable_and_recipient_scoped() -> None:
|
||||
first = decline._notification_delivery_uuid(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
openid="ou_first",
|
||||
)
|
||||
same = decline._notification_delivery_uuid(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
openid="ou_first",
|
||||
)
|
||||
second = decline._notification_delivery_uuid(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
openid="ou_second",
|
||||
)
|
||||
|
||||
assert first == same
|
||||
assert first != second
|
||||
|
||||
|
||||
def test_decline_notification_uses_shared_lark_user_sender(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
assert decline.extract_feishu_message_id(response) == expected
|
||||
captured: dict[str, object] = {}
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
|
||||
def fake_send(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"message_id": "om_real123"}
|
||||
|
||||
monkeypatch.setattr(decline, "send_lark_bot_message", fake_send)
|
||||
|
||||
response = decline.send_decline_notification(
|
||||
[_event()],
|
||||
"2026-08-01",
|
||||
9,
|
||||
openid="ou_owner",
|
||||
)
|
||||
|
||||
assert response["message_id"] == "om_real123"
|
||||
assert captured["user_id"] == "ou_owner"
|
||||
assert captured["profile"] == "hermes-analyzer"
|
||||
assert captured["idempotency_key"] == decline._notification_delivery_uuid(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
openid="ou_owner",
|
||||
)
|
||||
assert "测试款" in str(captured["text"])
|
||||
|
||||
|
||||
def test_main_fails_when_postgres_detection_read_fails(
|
||||
@@ -205,12 +256,12 @@ def test_main_dedup_query_failure_is_fail_closed(
|
||||
lambda *args: (_ for _ in ()).throw(RuntimeError("db timeout")),
|
||||
)
|
||||
notify = pytest.fail
|
||||
monkeypatch.setattr(decline, "notify_hermes", notify)
|
||||
monkeypatch.setattr(decline, "send_decline_notification", notify)
|
||||
|
||||
assert decline.main() == 5
|
||||
|
||||
|
||||
def test_main_does_not_accept_arbitrary_non_error_hermes_text(
|
||||
def test_main_does_not_accept_lark_response_without_real_message_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
@@ -219,11 +270,8 @@ def test_main_does_not_accept_arbitrary_non_error_hermes_text(
|
||||
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"notify_hermes",
|
||||
lambda *args, **kwargs: {
|
||||
"id": "chatcmpl-123",
|
||||
"choices": [{"message": {"content": "发送成功"}}],
|
||||
},
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: {"ok": True},
|
||||
)
|
||||
persist = pytest.fail
|
||||
monkeypatch.setattr(decline, "_persist_events", persist)
|
||||
@@ -240,7 +288,7 @@ def test_main_returns_nonzero_when_receipt_persistence_fails(
|
||||
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"notify_hermes",
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: {"message_id": "om_real123"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
@@ -263,7 +311,7 @@ def test_main_success_requires_and_persists_feishu_receipt(
|
||||
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"notify_hermes",
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: {"message_id": "om_real123"},
|
||||
)
|
||||
|
||||
@@ -281,6 +329,174 @@ def test_main_success_requires_and_persists_feishu_receipt(
|
||||
assert "persistence=committed history_rows=1" in output
|
||||
|
||||
|
||||
def test_main_no_notify_still_detects_hits_and_succeeds(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"check_nine_day_decline.py",
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--no-notify",
|
||||
"--data-root",
|
||||
str(tmp_path),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
persisted: list[tuple[list[dict], str, int]] = []
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_persist_detected_events",
|
||||
lambda events, end_date, window_days: (
|
||||
persisted.append((events, end_date, window_days)) or len(events)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_already_notified",
|
||||
lambda *args: pytest.fail("disabled notification must not query receipts"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: pytest.fail("disabled notification must not send"),
|
||||
)
|
||||
|
||||
assert decline.main() == 0
|
||||
assert persisted == [([_event()], "2026-08-01", 9)]
|
||||
output = capsys.readouterr().out
|
||||
assert "events=1 notification=disabled persistence=committed" in output
|
||||
assert "detection_rows=1" in output
|
||||
|
||||
|
||||
def test_main_blocks_notification_when_detection_output_cannot_be_persisted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_persist_detected_events",
|
||||
lambda *_args: (_ for _ in ()).throw(RuntimeError("write failed")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: pytest.fail("must not send without durable output"),
|
||||
)
|
||||
|
||||
assert decline.main() == 5
|
||||
|
||||
|
||||
def test_main_sends_and_persists_each_recipient_independently(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"check_nine_day_decline.py",
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
"ou_first",
|
||||
"--openid",
|
||||
"ou_second",
|
||||
"--data-root",
|
||||
str(tmp_path),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
dedup_queries: list[str] = []
|
||||
persisted: list[tuple[str, str]] = []
|
||||
|
||||
def fake_already_notified(_style, _date, recipient):
|
||||
dedup_queries.append(recipient)
|
||||
return False
|
||||
|
||||
def fake_notify(*_args, openid, **_kwargs):
|
||||
return {"message_id": f"om_{openid.removeprefix('ou_')}"}
|
||||
|
||||
def fake_persist(*_args, openid, message_id, **_kwargs):
|
||||
persisted.append((openid, message_id))
|
||||
return 1
|
||||
|
||||
monkeypatch.setattr(decline, "_already_notified", fake_already_notified)
|
||||
monkeypatch.setattr(decline, "send_decline_notification", fake_notify)
|
||||
monkeypatch.setattr(decline, "_persist_events", fake_persist)
|
||||
|
||||
assert decline.main() == 0
|
||||
assert dedup_queries == ["ou_first", "ou_second"]
|
||||
assert persisted == [
|
||||
("ou_first", "om_first"),
|
||||
("ou_second", "om_second"),
|
||||
]
|
||||
|
||||
|
||||
def test_alert_schema_and_upsert_are_recipient_scoped() -> None:
|
||||
repository_root = Path(__file__).resolve().parents[3]
|
||||
schema = (
|
||||
repository_root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "db"
|
||||
/ "schema.sql"
|
||||
).read_text(encoding="utf-8")
|
||||
db_module = (
|
||||
repository_root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "db"
|
||||
/ "__init__.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "UNIQUE (style_name, trigger_date, recipient)" in schema
|
||||
assert "DROP CONSTRAINT IF EXISTS fact_alert_history_style_name_trigger_date_key" in schema
|
||||
assert "ON CONFLICT (style_name, trigger_date, recipient)" in db_module
|
||||
assert "def ensure_alert_history_recipient_scope" in db_module
|
||||
assert "fact_alert_history_style_name_trigger_date_key" in db_module
|
||||
|
||||
|
||||
def test_detected_output_migrates_recipient_scope_before_upsert(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
fake_db = ModuleType("db")
|
||||
connection = object()
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
class ConnectionContext:
|
||||
def __enter__(self):
|
||||
return connection
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
fake_db.get_conn = ConnectionContext # type: ignore[attr-defined]
|
||||
fake_db.ensure_alert_history_recipient_scope = ( # type: ignore[attr-defined]
|
||||
lambda conn: calls.append(("migrate", conn))
|
||||
)
|
||||
fake_db.upsert_decline_event = ( # type: ignore[attr-defined]
|
||||
lambda conn, event: calls.append(("upsert", (conn, event)))
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "db", fake_db)
|
||||
|
||||
assert decline._persist_detected_events([_event()], "2026-08-01", 9) == 1
|
||||
assert calls[0] == ("migrate", connection)
|
||||
assert calls[1][0] == "upsert"
|
||||
|
||||
|
||||
class _FailedCollector:
|
||||
def __init__(self, *args):
|
||||
self.pages: list[tuple] = []
|
||||
|
||||
@@ -7,16 +7,10 @@ from types import SimpleNamespace
|
||||
|
||||
import collect_erp_yesterday_metrics as erp
|
||||
import collect_jd_persona_to_bitable as jd_persona
|
||||
from runtime_paths import playwright_launch_options
|
||||
import erp_login_product_analysis as erp_login
|
||||
from runtime_paths import scrapling_session_options
|
||||
|
||||
|
||||
class _Chromium:
|
||||
def __init__(self) -> None:
|
||||
self.kwargs = None
|
||||
|
||||
def launch_persistent_context(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
return object()
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_jd_persona_launches_the_managed_persistent_profile(
|
||||
@@ -25,13 +19,20 @@ def test_jd_persona_launches_the_managed_persistent_profile(
|
||||
) -> None:
|
||||
profile = tmp_path / "jd-profile"
|
||||
monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile))
|
||||
chromium = _Chromium()
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
context = jd_persona._launch_jd_context(chromium, headless=True)
|
||||
def browser_factory(**options):
|
||||
captured.update(options)
|
||||
return object()
|
||||
|
||||
assert context is not None
|
||||
assert chromium.kwargs["user_data_dir"] == str(profile.resolve())
|
||||
assert chromium.kwargs["headless"] is True
|
||||
monkeypatch.setattr(jd_persona, "ScraplingBrowser", browser_factory)
|
||||
|
||||
browser = jd_persona._jd_browser(headless=True)
|
||||
|
||||
assert browser is not None
|
||||
assert captured["user_data_dir"] == str(profile.resolve())
|
||||
assert captured["headless"] is True
|
||||
assert captured["retries"] == 1
|
||||
|
||||
|
||||
def test_jd_persona_reuses_logged_in_page_without_credentials() -> None:
|
||||
@@ -61,6 +62,36 @@ def test_jd_persona_reuses_logged_in_page_without_credentials() -> None:
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_jd_persona_waits_for_slow_shop_shell_before_login_fallback() -> None:
|
||||
calls: list[tuple[str, str]] = []
|
||||
bodies = iter(("", "", "", "", "京东商家后台 商品明细"))
|
||||
waits: list[int] = []
|
||||
|
||||
class Page:
|
||||
url = "https://shop.jd.com/jdm/home"
|
||||
|
||||
def goto(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
waits.append(milliseconds)
|
||||
|
||||
def locator(self, _selector):
|
||||
return SimpleNamespace(
|
||||
inner_text=lambda **_kwargs: next(bodies),
|
||||
)
|
||||
|
||||
jd_persona._ensure_jd_session(
|
||||
Page(),
|
||||
shop="",
|
||||
password="",
|
||||
step_login=lambda _page, shop, password: calls.append((shop, password)),
|
||||
)
|
||||
|
||||
assert calls == []
|
||||
assert waits == [2000, 1000, 1000, 1000, 1000]
|
||||
|
||||
|
||||
def test_erp_uses_managed_profile_when_bound_cdp_is_not_running(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
@@ -95,6 +126,62 @@ def test_erp_attaches_only_to_the_live_managed_cdp(monkeypatch) -> None:
|
||||
assert user_data_dir is None
|
||||
|
||||
|
||||
def test_erp_login_navigates_before_probing_initial_blank_page(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
events: list[tuple[str, str]] = []
|
||||
|
||||
class Page:
|
||||
url = "about:blank"
|
||||
context = object()
|
||||
|
||||
def goto(self, url, **_kwargs):
|
||||
events.append(("goto", url))
|
||||
self.url = url
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(erp_login, "restore_erp_runtime_state", lambda _context: None)
|
||||
monkeypatch.setattr(erp_login, "has_valid_session", lambda _page: False)
|
||||
monkeypatch.setattr(erp_login, "is_authenticated_session", lambda _page: False)
|
||||
monkeypatch.setattr(erp_login, "first_visible", lambda *_args: object())
|
||||
monkeypatch.setattr(
|
||||
erp_login,
|
||||
"fill_first_visible",
|
||||
lambda _page, _selectors, _value, label: events.append(("fill", label)) or True,
|
||||
)
|
||||
monkeypatch.setattr(erp_login, "check_visible_checkboxes", lambda _page: None)
|
||||
monkeypatch.setattr(erp_login, "click_login_button", lambda _page: events.append(("click", "login")))
|
||||
monkeypatch.setattr(erp_login, "wait_for_manual_captcha", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr(erp_login, "wait_until_logged_in", lambda *_args, **_kwargs: True)
|
||||
monkeypatch.setattr(erp_login, "persist_erp_runtime_state", lambda _context: 1)
|
||||
|
||||
state: dict[str, object] = {}
|
||||
action = erp_login.make_page_action(
|
||||
login_url="https://erp.test/login.aspx",
|
||||
username="user",
|
||||
password="pw",
|
||||
home_url="https://erp.test/epaas",
|
||||
timeout_ms=1000,
|
||||
captcha_checks=0,
|
||||
captcha_interval_seconds=1,
|
||||
hold_seconds=0,
|
||||
login_only=True,
|
||||
state=state,
|
||||
)
|
||||
|
||||
action(Page())
|
||||
|
||||
assert events[:4] == [
|
||||
("goto", "https://erp.test/login.aspx"),
|
||||
("fill", "username"),
|
||||
("fill", "password"),
|
||||
("click", "login"),
|
||||
]
|
||||
assert state == {"ok": True, "final_url": "https://erp.test/login.aspx"}
|
||||
|
||||
|
||||
def test_erp_main_fails_before_browser_when_no_style_is_collectable(monkeypatch) -> None:
|
||||
args = SimpleNamespace(
|
||||
config="unused.json",
|
||||
@@ -192,6 +279,110 @@ def test_erp_main_fails_when_collector_summary_contains_failures(
|
||||
assert erp.main() == 1
|
||||
|
||||
|
||||
def test_erp_main_maps_login_recovery_to_cookie_skip(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
args = SimpleNamespace(
|
||||
config="unused.json",
|
||||
target_date=date(2026, 8, 12),
|
||||
mode="daily",
|
||||
refresh_config=False,
|
||||
input="unused-styles.json",
|
||||
data_dir=str(tmp_path),
|
||||
platform="all",
|
||||
no_cdp=False,
|
||||
timeout_ms=1000,
|
||||
limit=0,
|
||||
report_timeout=1,
|
||||
report_attempts=1,
|
||||
include_slow_codes=False,
|
||||
hold_seconds=0,
|
||||
headless=True,
|
||||
)
|
||||
styles = [
|
||||
{
|
||||
"style_name": "style-a",
|
||||
"erp_style_codes": ["erp-a"],
|
||||
"brand": erp.DEFAULT_BRAND,
|
||||
}
|
||||
]
|
||||
monkeypatch.setattr(erp, "parse_args", lambda: args)
|
||||
monkeypatch.setattr(
|
||||
erp,
|
||||
"load_config",
|
||||
lambda _path: {
|
||||
"login": {
|
||||
"url": "https://erp.test/login",
|
||||
"username": "placeholder",
|
||||
"password": "placeholder",
|
||||
}
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
erp,
|
||||
"load_erp_styles",
|
||||
lambda *_args: (
|
||||
styles,
|
||||
{"source": "lark", "collectable": ["style-a"], "skipped": []},
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(erp, "print_skip_summary", lambda _report: None)
|
||||
monkeypatch.setattr(erp, "write_erp_skip_report", lambda *_args: None)
|
||||
monkeypatch.setattr(erp, "managed_data_path", lambda _path: tmp_path)
|
||||
monkeypatch.setattr(erp, "resolve_browser_session", lambda _config: (None, None))
|
||||
|
||||
def fake_login_action(**kwargs):
|
||||
state = kwargs["state"]
|
||||
|
||||
def action(_page):
|
||||
state["ok"] = False
|
||||
state["error"] = "Login did not complete; captcha required"
|
||||
|
||||
return action
|
||||
|
||||
monkeypatch.setattr(erp, "make_page_action", fake_login_action)
|
||||
monkeypatch.setattr(
|
||||
erp.DynamicFetcher,
|
||||
"fetch",
|
||||
lambda _url, **kwargs: kwargs["page_action"](object()),
|
||||
)
|
||||
|
||||
assert erp.main() == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_erp_login_waits_for_a_delayed_captcha(monkeypatch) -> None:
|
||||
captcha_states = iter([False, False, True, True, False])
|
||||
waits: list[int] = []
|
||||
|
||||
class Page:
|
||||
url = "https://erp.test/login"
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
waits.append(milliseconds)
|
||||
|
||||
monkeypatch.setattr(
|
||||
erp_login,
|
||||
"captcha_exists",
|
||||
lambda _page: next(captcha_states),
|
||||
)
|
||||
monotonic_values = iter([0.0, 0.1, 0.2])
|
||||
monkeypatch.setattr(
|
||||
erp_login.time,
|
||||
"monotonic",
|
||||
lambda: next(monotonic_values),
|
||||
)
|
||||
|
||||
erp_login.wait_for_manual_captcha(
|
||||
Page(),
|
||||
checks=3,
|
||||
interval_seconds=5,
|
||||
login_url="https://erp.test/login",
|
||||
)
|
||||
|
||||
assert waits == [500, 500, 5000, 5000]
|
||||
|
||||
|
||||
def test_erp_date_inputs_are_required_and_read_back() -> None:
|
||||
source = inspect.getsource(erp.set_platform_and_code)
|
||||
|
||||
@@ -200,8 +391,11 @@ def test_erp_date_inputs_are_required_and_read_back() -> None:
|
||||
assert "date range did not persist" in source
|
||||
|
||||
|
||||
def test_playwright_channel_is_optional_and_environment_driven() -> None:
|
||||
assert playwright_launch_options({}) == {}
|
||||
assert playwright_launch_options({"GYXX_PLAYWRIGHT_CHANNEL": "chrome"}) == {
|
||||
"channel": "chrome"
|
||||
def test_scrapling_real_chrome_is_optional_and_environment_driven() -> None:
|
||||
assert scrapling_session_options({}) == {}
|
||||
assert scrapling_session_options({"GYXX_PLAYWRIGHT_CHANNEL": "chrome"}) == {
|
||||
"real_chrome": True
|
||||
}
|
||||
assert scrapling_session_options({"GYXX_SCRAPLING_REAL_CHROME": "false"}) == {
|
||||
"real_chrome": False
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import collect_douyin_qianchuan_ads as collector
|
||||
|
||||
|
||||
def test_qianchuan_browser_uses_scrapling_stealth_and_persistent_state(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class _FakeBrowser:
|
||||
def __init__(self, **options: object) -> None:
|
||||
captured.update(options)
|
||||
|
||||
cookie_file = tmp_path / "cookies.json"
|
||||
storage_state_file = tmp_path / "storage_state.json"
|
||||
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
|
||||
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_state_file))
|
||||
monkeypatch.setattr(collector, "ScraplingBrowser", _FakeBrowser)
|
||||
|
||||
browser = collector._start_browser(True, str(tmp_path / "profile"))
|
||||
|
||||
assert isinstance(browser, _FakeBrowser)
|
||||
assert captured["stealthy"] is True
|
||||
assert captured["real_chrome"] is False
|
||||
assert captured["headless"] is True
|
||||
assert captured["user_data_dir"] == str((tmp_path / "profile").resolve())
|
||||
assert captured["cookie_file"] == str(cookie_file)
|
||||
assert captured["storage_state_file"] == str(storage_state_file)
|
||||
assert captured["extra_flags"] == ["--no-proxy-server"]
|
||||
assert captured["additional_args"] == {
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"accept_downloads": True,
|
||||
}
|
||||
|
||||
|
||||
def test_calendar_cell_prefers_actual_month_when_adjacent_month_duplicates() -> None:
|
||||
class _Locator:
|
||||
def __init__(self, count: int) -> None:
|
||||
self._count = count
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count
|
||||
|
||||
class _Page:
|
||||
def locator(self, selector: str) -> _Locator:
|
||||
if ".ovui-date__cell--inview" in selector:
|
||||
return _Locator(1)
|
||||
return _Locator(2)
|
||||
|
||||
cell = collector._calendar_cell(_Page(), "2026-08-30")
|
||||
|
||||
assert cell.count() == 1
|
||||
|
||||
|
||||
def test_calendar_cell_falls_back_when_picker_has_no_inview_class() -> None:
|
||||
class _Locator:
|
||||
def __init__(self, count: int) -> None:
|
||||
self._count = count
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count
|
||||
|
||||
class _Page:
|
||||
def locator(self, selector: str) -> _Locator:
|
||||
if ".ovui-date__cell--inview" in selector:
|
||||
return _Locator(0)
|
||||
return _Locator(1)
|
||||
|
||||
cell = collector._calendar_cell(_Page(), "2026-08-30")
|
||||
|
||||
assert cell.count() == 1
|
||||
|
||||
|
||||
def test_report_ready_wait_guards_missing_body() -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class _Page:
|
||||
def wait_for_function(self, expression: str, **options: object) -> None:
|
||||
captured["expression"] = expression
|
||||
captured.update(options)
|
||||
|
||||
page = _Page()
|
||||
collector._wait_for_report_ready(page)
|
||||
|
||||
assert captured == {
|
||||
"expression": collector.REPORT_READY_EXPRESSION,
|
||||
"arg": list(collector._required_page_markers()),
|
||||
"timeout": 90_000,
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import zipfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import collect_jd_ad_costs as collector
|
||||
import pytest
|
||||
|
||||
|
||||
def test_login_starts_from_jd_merchant_login_page() -> None:
|
||||
assert collector.JD_LOGIN_URL == (
|
||||
"https://passport.shop.jd.com/login/index.action/jdm?"
|
||||
"ReturnUrl=https%3A%2F%2Fshop.jd.com%2Fjdm%2Fhome"
|
||||
)
|
||||
|
||||
|
||||
def test_ensure_logged_in_navigates_to_login_entry_first(monkeypatch) -> None:
|
||||
visited: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"safe_goto",
|
||||
lambda _page, url: visited.append(url),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_login_complete", lambda _page: True)
|
||||
|
||||
collector.ensure_logged_in(object(), "光影行星研发", "secret")
|
||||
|
||||
assert visited == [collector.JD_LOGIN_URL]
|
||||
|
||||
|
||||
def test_ensure_logged_in_reuses_valid_cookie_session(monkeypatch) -> None:
|
||||
visited: list[str] = []
|
||||
|
||||
class Page:
|
||||
def wait_for_timeout(self, _milliseconds: int) -> None:
|
||||
return None
|
||||
|
||||
states = iter((False, True))
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"safe_goto",
|
||||
lambda _page, url: visited.append(url),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_login_complete", lambda _page: next(states))
|
||||
|
||||
collector.ensure_logged_in(Page(), "光影行星研发", "secret")
|
||||
|
||||
assert visited == [collector.JD_LOGIN_URL, collector.JD_HOME_URL]
|
||||
|
||||
|
||||
def test_login_failure_reason_exposes_jd_credential_error() -> None:
|
||||
class Page:
|
||||
def locator(self, _selector):
|
||||
return self
|
||||
|
||||
def inner_text(self, timeout):
|
||||
del timeout
|
||||
return "账号名与密码不匹配"
|
||||
|
||||
assert collector._login_failure_reason(Page()) == "账号名与密码不匹配"
|
||||
|
||||
|
||||
def test_safe_goto_stops_when_browser_context_is_closed() -> None:
|
||||
class Page:
|
||||
def goto(self, *_args, **_kwargs) -> None:
|
||||
raise RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
def is_closed(self) -> bool:
|
||||
return True
|
||||
|
||||
with pytest.raises(collector.JDAdCostCollectionError, match="上下文意外关闭"):
|
||||
collector.safe_goto(Page(), "https://jzt.jd.com/custom-report/#/download")
|
||||
|
||||
|
||||
def test_jd_ad_costs_uses_bundled_chromium_despite_global_chrome_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def build_browser(**options):
|
||||
captured.update(options)
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(collector, "ScraplingBrowser", build_browser)
|
||||
monkeypatch.setenv("GYXX_SCRAPLING_REAL_CHROME", "true")
|
||||
|
||||
collector._start_browser(False, "D:/gyxx-flow/var/state/browser-profiles/jd-ad-costs")
|
||||
|
||||
assert captured["real_chrome"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "report_type", "expected"),
|
||||
[
|
||||
("营销概况_矩阵分析_SKU维度_2026-08-23 报告已生成", "non_full", "ready"),
|
||||
("营销概况_下钻趋势图_2026-08-23 报表已生成", "full", "ready"),
|
||||
("营销概况_矩阵分析_SKU维度_2026-08-23 生成中", "non_full", "pending"),
|
||||
("营销概况_下钻趋势图_2026-08-22 报告已生成", "full", "absent"),
|
||||
(
|
||||
"光影行星研发_营销概况_矩阵分析_SKU维度_2026-08-17至2026-08-17"
|
||||
" 报表已生成 创建日期 2026-08-23",
|
||||
"non_full",
|
||||
"absent",
|
||||
),
|
||||
(
|
||||
"光影行星研发_营销概况_下钻趋势图_2026-06-01至2026-08-23"
|
||||
" 报表已生成 创建日期 2026-08-23",
|
||||
"full",
|
||||
"absent",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_download_center_state_is_report_type_and_date_scoped(
|
||||
text: str, report_type: collector.ReportType, expected: collector.ReportState
|
||||
) -> None:
|
||||
assert (
|
||||
collector._report_state_from_text(text, report_type, date(2026, 8, 23))
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
def test_download_center_total_treats_empty_table_as_zero() -> None:
|
||||
class Body:
|
||||
def inner_text(self, timeout):
|
||||
del timeout
|
||||
return "下载报表 暂无数据"
|
||||
|
||||
class Page:
|
||||
def locator(self, selector):
|
||||
assert selector == "body"
|
||||
return Body()
|
||||
|
||||
assert collector._download_center_total(Page()) == 0
|
||||
|
||||
|
||||
def test_clear_download_center_deletes_each_page_until_empty(monkeypatch) -> None:
|
||||
totals = iter((61, 51, 41, 31, 21, 11, 1, 0))
|
||||
deleted_batches: list[int] = []
|
||||
|
||||
monkeypatch.setattr(collector, "_wait_for_download_center_table", lambda _page: None)
|
||||
monkeypatch.setattr(collector, "_download_center_total", lambda _page: next(totals))
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_all_download_center_rows",
|
||||
lambda _page: deleted_batches.append(1) or 10,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector, "_delete_selected_download_center_rows", lambda _page: None
|
||||
)
|
||||
monkeypatch.setattr(collector, "_wait_for", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(collector, "_reload_download_center", lambda _page: None)
|
||||
|
||||
collector._clear_download_center(object())
|
||||
|
||||
assert len(deleted_batches) == 7
|
||||
|
||||
|
||||
def test_safe_extract_zip_rejects_path_traversal(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "report.zip"
|
||||
with zipfile.ZipFile(archive, "w") as handle:
|
||||
handle.writestr("../outside.csv", "bad")
|
||||
|
||||
with pytest.raises(collector.JDAdCostCollectionError, match="路径越界"):
|
||||
collector._safe_extract_zip(archive, tmp_path / "extracted")
|
||||
|
||||
|
||||
def test_safe_extract_zip_keeps_csv_inside_output(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "report.zip"
|
||||
with zipfile.ZipFile(archive, "w") as handle:
|
||||
handle.writestr("nested/full.csv", "日期,'当前时间',SPU ID\n")
|
||||
|
||||
files = collector._safe_extract_zip(archive, tmp_path / "extracted")
|
||||
|
||||
assert files == [(tmp_path / "extracted" / "nested" / "full.csv").resolve()]
|
||||
@@ -0,0 +1,180 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import zipfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import collect_tmall_wanxiang_ads as collector
|
||||
import pytest
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def test_item_promotion_url_contains_explicit_target_date() -> None:
|
||||
url = collector.item_promotion_url(date(2026, 8, 17))
|
||||
assert "#!/report/item_promotion" in url
|
||||
assert "startTime=2026-08-17" in url
|
||||
assert "endTime=2026-08-17" in url
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
("商品报表 2026-08-18 生成成功", "ready"),
|
||||
("商品报表 2026-08-18 生成中", "pending"),
|
||||
("商品报表 2026-08-18 生成失败", "failed"),
|
||||
("商品报表 2026-08-17 生成成功", "absent"),
|
||||
# 数据日期范围不匹配,仅创建日期含目标日:不能误判为同日报表
|
||||
(
|
||||
"生成成功 商品报表_20260818_120200 15天累计数据 2026-08-11至2026-08-11 2026-08-18",
|
||||
"absent",
|
||||
),
|
||||
# 数据日期范围覆盖目标日:可复用
|
||||
(
|
||||
"生成成功 商品报表_20260819_113811 15天累计数据 2026-08-18至2026-08-18 2026-08-19",
|
||||
"ready",
|
||||
),
|
||||
],)
|
||||
def test_report_task_state_is_date_scoped_and_conservative(
|
||||
text: str, expected: collector.ReportTaskState
|
||||
) -> None:
|
||||
assert collector._report_task_state_from_text(text, date(2026, 8, 18)) == expected
|
||||
|
||||
|
||||
def test_scheduled_wanxiang_workflow_does_not_force_duplicate_request() -> None:
|
||||
payload = json.loads(
|
||||
(PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8")
|
||||
)
|
||||
workflow = next(
|
||||
item for item in payload["workflows"] if item["id"] == "product.ecommerce_costs.daily"
|
||||
)
|
||||
download_step = next(
|
||||
step for step in workflow["execution"]["steps"] if step["id"] == "tmall_download"
|
||||
)
|
||||
assert "--force-request" not in download_step["args"]
|
||||
|
||||
|
||||
def test_request_report_submits_outer_dialog_only_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
page = object()
|
||||
modal = {"open": True}
|
||||
confirm_calls = 0
|
||||
selected_dates: list[date] = []
|
||||
|
||||
monkeypatch.setattr(collector, "_wait_for_text", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(collector, "_click_visible_text", lambda *args, **kwargs: True)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_date_range",
|
||||
lambda _page, target_date: selected_dates.append(target_date),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_modal_visible", lambda _page: modal["open"])
|
||||
|
||||
def confirm(_page: object) -> bool:
|
||||
nonlocal confirm_calls
|
||||
confirm_calls += 1
|
||||
modal["open"] = False
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(collector, "_click_download_dialog_confirm", confirm)
|
||||
|
||||
collector.request_report(page, date(2026, 8, 18))
|
||||
|
||||
assert confirm_calls == 1
|
||||
assert selected_dates == [date(2026, 8, 18)]
|
||||
|
||||
|
||||
def test_run_waits_for_existing_pending_task_instead_of_requesting_another(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
class _Session:
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
session = _Session()
|
||||
page = object()
|
||||
calls: list[bool] = []
|
||||
report_zip = tmp_path / "report.zip"
|
||||
extracted_csv = tmp_path / "report.csv"
|
||||
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_start_session",
|
||||
lambda _headless, _user_data_dir: (session, object(), page),
|
||||
)
|
||||
monkeypatch.setattr(collector, "ensure_logged_in", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"download_ready_report",
|
||||
lambda _page, _date, _output, *, timeout_seconds, wait_for_generation: (
|
||||
calls.append(wait_for_generation)
|
||||
or (None if len(calls) == 1 else report_zip)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_report_task_state", lambda *_args: "pending")
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"extract_report_zip",
|
||||
lambda _zip_path, _target_date: [extracted_csv],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"request_report",
|
||||
lambda *_args, **_kwargs: pytest.fail("must not submit a second task"),
|
||||
)
|
||||
|
||||
args = collector.build_parser().parse_args(
|
||||
["--date", "2026-08-18", "--output", str(tmp_path)]
|
||||
)
|
||||
# Content validation only re-requests when the reused report does not
|
||||
# cover the target date; a matching CSV must keep the reuse path.
|
||||
extracted_csv.write_text("日期,主体ID\n2026-08-18,1\n", encoding="utf-8")
|
||||
assert collector.run(args) == [extracted_csv]
|
||||
assert calls == [False, True]
|
||||
|
||||
|
||||
class _Page:
|
||||
def __init__(self, url: str, body: str) -> None:
|
||||
self.url = url
|
||||
self._body = body
|
||||
|
||||
def locator(self, selector: str):
|
||||
assert selector == "body"
|
||||
return self
|
||||
|
||||
def inner_text(self, timeout: int = 0) -> str:
|
||||
return self._body
|
||||
|
||||
|
||||
def test_login_success_accepts_taobao_seller_center_before_redirect() -> None:
|
||||
assert collector._is_taobao_login_complete(
|
||||
_Page("https://myseller.taobao.com/home", "物流监控 违规预警")
|
||||
)
|
||||
assert not collector._is_taobao_login_complete(
|
||||
_Page("https://loginmyseller.taobao.com/", "请输入登录密码")
|
||||
)
|
||||
|
||||
|
||||
def test_safe_extract_zip_returns_csv_and_writes_manifest(tmp_path: Path) -> None:
|
||||
archive_path = tmp_path / "商品报表.zip"
|
||||
with zipfile.ZipFile(archive_path, "w") as archive:
|
||||
archive.writestr("商品报表_20260817.csv", "日期,主体ID\n")
|
||||
archive.writestr("说明.txt", "万相台商品报表")
|
||||
|
||||
csv_files = collector.extract_report_zip(archive_path, date(2026, 8, 17))
|
||||
|
||||
assert [path.name for path in csv_files] == ["商品报表_20260817.csv"]
|
||||
manifest = (tmp_path / "manifest.json").read_text(encoding="utf-8")
|
||||
assert '"target_date": "2026-08-17"' in manifest
|
||||
assert (tmp_path / "extracted" / "商品报表_20260817.csv").is_file()
|
||||
|
||||
|
||||
def test_safe_extract_zip_rejects_path_traversal(tmp_path: Path) -> None:
|
||||
archive_path = tmp_path / "unsafe.zip"
|
||||
with zipfile.ZipFile(archive_path, "w") as archive:
|
||||
archive.writestr("../escape.csv", "bad")
|
||||
|
||||
with pytest.raises(collector.WanxiangDownloadError, match="路径越界"):
|
||||
collector._safe_extract_zip(archive_path, tmp_path / "out")
|
||||
@@ -13,6 +13,11 @@ import orchestrate_daily_collection as daily
|
||||
import pytest
|
||||
from commands import import_daily as import_daily_command
|
||||
|
||||
from gyxx_flow.core.exit_codes import (
|
||||
COOKIE_SKIP_EXIT_CODE,
|
||||
NON_RETRYABLE_EXIT_CODE,
|
||||
)
|
||||
|
||||
|
||||
class _Context:
|
||||
def __init__(self, value):
|
||||
@@ -93,6 +98,153 @@ def test_selected_platform_collector_failure_stops_daily_aggregation(monkeypatch
|
||||
target_date="2026-08-03",
|
||||
)
|
||||
|
||||
|
||||
def test_cookie_skip_is_not_retried_by_daily_orchestrator(monkeypatch) -> None:
|
||||
calls = 0
|
||||
|
||||
def fake_run(*_args, **_kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return {"name": "erp", "code": COOKIE_SKIP_EXIT_CODE}
|
||||
|
||||
monkeypatch.setattr(daily, "run_step", fake_run)
|
||||
monkeypatch.setattr(
|
||||
daily.time,
|
||||
"sleep",
|
||||
lambda *_args: pytest.fail("cookie recovery must not enter retry cooldown"),
|
||||
)
|
||||
|
||||
with pytest.raises(daily.CookiePreflightSkipped, match="erp requires"):
|
||||
daily.run_step_with_retry("erp", ["python", "collector.py"])
|
||||
|
||||
assert calls == 1
|
||||
|
||||
|
||||
def test_non_retryable_collector_result_is_not_retried(monkeypatch) -> None:
|
||||
calls = 0
|
||||
|
||||
def fake_run(*_args, **_kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return {"name": "tm", "code": NON_RETRYABLE_EXIT_CODE}
|
||||
|
||||
monkeypatch.setattr(daily, "run_step", fake_run)
|
||||
monkeypatch.setattr(
|
||||
daily.time,
|
||||
"sleep",
|
||||
lambda *_args: pytest.fail("deterministic validation must not retry"),
|
||||
)
|
||||
|
||||
result = daily.run_step_with_retry(
|
||||
"tm",
|
||||
["python", "collector.py"],
|
||||
critical=False,
|
||||
)
|
||||
|
||||
assert result["code"] == NON_RETRYABLE_EXIT_CODE
|
||||
assert calls == 1
|
||||
|
||||
|
||||
def test_daily_main_propagates_cookie_skip_exit_code(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_collect",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
daily.CookiePreflightSkipped("erp requires browser login")
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
daily.argparse.ArgumentParser,
|
||||
"parse_args",
|
||||
lambda _parser: SimpleNamespace(
|
||||
date=None,
|
||||
target_date="2026-08-12",
|
||||
stages=["collect"],
|
||||
skip_erp=False,
|
||||
skip_platforms=False,
|
||||
platforms="jd,dy,tm",
|
||||
serial_platforms=False,
|
||||
skip_insert=False,
|
||||
skip_reviews_import=False,
|
||||
dry_run=False,
|
||||
erp_mode="daily",
|
||||
max_run_retries=2,
|
||||
),
|
||||
)
|
||||
|
||||
assert daily.main() == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_erp_mode_auto_uses_daily_only_for_yesterday() -> None:
|
||||
today = datetime(2026, 8, 13).date()
|
||||
|
||||
assert daily.resolve_erp_mode("2026-08-12", today=today) == "daily"
|
||||
assert daily.resolve_erp_mode("2026-08-11", today=today) == "backfill"
|
||||
assert daily.resolve_erp_mode("2026-08-13", today=today) == "backfill"
|
||||
assert daily.is_historical_target("2026-08-12", today=today) is False
|
||||
assert daily.is_historical_target("2026-08-11", today=today) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["daily", "weekly", "backfill"])
|
||||
def test_explicit_erp_mode_overrides_automatic_selection(mode) -> None:
|
||||
assert (
|
||||
daily.resolve_erp_mode(
|
||||
"2026-08-11",
|
||||
requested_mode=mode,
|
||||
today=datetime(2026, 8, 13).date(),
|
||||
)
|
||||
== mode
|
||||
)
|
||||
|
||||
|
||||
def test_historical_daily_collection_passes_backfill_mode_to_erp(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
args = SimpleNamespace(
|
||||
skip_erp=False,
|
||||
skip_platforms=True,
|
||||
skip_reviews_import=True,
|
||||
dry_run=False,
|
||||
erp_mode="auto",
|
||||
)
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"resolve_erp_mode",
|
||||
lambda *_args, **_kwargs: "backfill",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs))
|
||||
or {"name": "erp", "code": 0},
|
||||
)
|
||||
|
||||
daily.run_collect(args, "2026-08-11", ["jd", "dy", "tm"], failed=None)
|
||||
|
||||
command = calls[0][0][1]
|
||||
assert command[-4:] == ["--target-date", "2026-08-11", "--mode", "backfill"]
|
||||
|
||||
|
||||
def test_historical_tm_collection_uses_backfill_child(monkeypatch) -> None:
|
||||
calls: list[tuple[str, list[str]]] = []
|
||||
|
||||
def fake_run(name, command, **_kwargs):
|
||||
calls.append((name, command))
|
||||
return {"name": name, "code": 0}
|
||||
|
||||
monkeypatch.setattr(daily, "run_step_with_retry", fake_run)
|
||||
monkeypatch.setattr(daily, "verify_platform_artifacts", lambda *_args: 1)
|
||||
|
||||
daily.run_platforms(
|
||||
parallel=False,
|
||||
platforms=["tm"],
|
||||
target_date="2026-08-11",
|
||||
historical=True,
|
||||
)
|
||||
|
||||
assert calls[0][1][-3:] == ["--backfill", "--target-date", "2026-08-11"]
|
||||
|
||||
def test_platform_selection_keeps_omitted_collectors_optional(monkeypatch) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
@@ -217,6 +369,63 @@ def test_platform_artifact_verifier_accepts_fresh_valid_product(
|
||||
assert daily.verify_platform_artifacts("jd", "2026-08-03", marker_ns) == 1
|
||||
|
||||
|
||||
def test_dy_verifier_accepts_source_date_format_and_legitimate_zero_style(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
class Loader:
|
||||
def get_daily_styles(self):
|
||||
return {
|
||||
"dy": {
|
||||
"collectable": ["matched-style", "zero-style"],
|
||||
"skipped": [],
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(daily, "StyleConfigLoader", Loader)
|
||||
monkeypatch.setattr(daily, "DATA_ROOT", tmp_path)
|
||||
marker_ns = time.time_ns()
|
||||
for style, matched in (("matched-style", 2), ("zero-style", 0)):
|
||||
path = daily._platform_artifact_path("dy", style, "2026-08-03")
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(
|
||||
json.dumps({
|
||||
"date_range": "2026/08/03 - 2026/08/03",
|
||||
"matched_row_count": matched,
|
||||
"download_file": "_downloads/2026-08-03/report.xlsx",
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns) == 2
|
||||
|
||||
|
||||
def test_platform_artifact_verifier_rejects_an_export_with_no_product_matches(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
class Loader:
|
||||
def get_daily_styles(self):
|
||||
return {"dy": {"collectable": ["zero-style"], "skipped": []}}
|
||||
|
||||
monkeypatch.setattr(daily, "StyleConfigLoader", Loader)
|
||||
monkeypatch.setattr(daily, "DATA_ROOT", tmp_path)
|
||||
marker_ns = time.time_ns()
|
||||
path = daily._platform_artifact_path("dy", "zero-style", "2026-08-03")
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(
|
||||
json.dumps({
|
||||
"date_range": "2026/08/03 - 2026/08/03",
|
||||
"matched_row_count": 0,
|
||||
"download_file": "_downloads/2026-08-03/report.xlsx",
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="matched no configured products"):
|
||||
daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns)
|
||||
|
||||
|
||||
def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
|
||||
args = SimpleNamespace(
|
||||
skip_erp=True,
|
||||
@@ -240,6 +449,27 @@ def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
|
||||
assert calls[0][1]["critical"] is False
|
||||
|
||||
|
||||
def test_daily_import_stage_uses_the_collected_business_date(monkeypatch) -> None:
|
||||
args = SimpleNamespace(dry_run=True, platforms="jd,tm")
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0},
|
||||
)
|
||||
|
||||
daily.run_import(args, "2026-08-03", failed=None)
|
||||
|
||||
assert calls[0][0][0] == "import_product_daily"
|
||||
assert calls[0][0][1][-4:] == [
|
||||
"--date",
|
||||
"2026-08-03",
|
||||
"--platforms",
|
||||
"jd,tm",
|
||||
]
|
||||
assert calls[0][1]["dry_run"] is True
|
||||
|
||||
|
||||
def test_formal_aggregation_fails_when_postgres_write_is_not_fresh(monkeypatch) -> None:
|
||||
marker = datetime(2026, 8, 4, 3, 0, tzinfo=timezone.utc)
|
||||
args = SimpleNamespace(dry_run=False)
|
||||
@@ -374,6 +604,26 @@ def test_import_fails_when_selected_platform_has_no_target_directory(monkeypatch
|
||||
assert "dy: 没有匹配所选日期范围的报表目录" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_import_deduplicates_rerun_rows_before_one_upsert_batch(capsys) -> None:
|
||||
first = {
|
||||
**_record(),
|
||||
"source_file": "old-report.xlsx",
|
||||
"visitors": 10,
|
||||
}
|
||||
latest = {
|
||||
**_record(),
|
||||
"source_file": "latest-report.xlsx",
|
||||
"visitors": 20,
|
||||
}
|
||||
|
||||
result = product_import._deduplicate_records_for_upsert([first, latest])
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["source_file"] == "latest-report.xlsx"
|
||||
assert result[0]["visitors"] == 20
|
||||
assert "丢弃重复原始行: 1 行" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_import_fails_when_target_directory_has_no_input_file(monkeypatch, capsys) -> None:
|
||||
target_dir = Path("2026-08-03")
|
||||
monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [target_dir])
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import db
|
||||
import pytest
|
||||
from db import InterfaceError, get_conn
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def execute(self, _sql):
|
||||
return None
|
||||
|
||||
def fetchone(self):
|
||||
return (1,)
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self, alive: bool) -> None:
|
||||
self.alive = alive
|
||||
self.closed = False
|
||||
self.committed = False
|
||||
self.rolled_back = False
|
||||
|
||||
def rollback(self) -> None:
|
||||
if not self.alive:
|
||||
raise RuntimeError("connection already closed")
|
||||
self.rolled_back = True
|
||||
|
||||
def cursor(self):
|
||||
if not self.alive:
|
||||
raise RuntimeError("connection already closed")
|
||||
return FakeCursor()
|
||||
|
||||
def commit(self) -> None:
|
||||
self.committed = True
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class FakePool:
|
||||
def __init__(self, connections):
|
||||
self.connections = list(connections)
|
||||
self.returned: list[tuple[object, bool]] = []
|
||||
|
||||
def getconn(self):
|
||||
if not self.connections:
|
||||
raise RuntimeError("pool exhausted")
|
||||
return self.connections.pop(0)
|
||||
|
||||
def putconn(self, conn, close: bool = False) -> None:
|
||||
self.returned.append((conn, close))
|
||||
|
||||
|
||||
def test_get_conn_reuses_healthy_pooled_connection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
connection = FakeConnection(alive=True)
|
||||
fake_pool = FakePool([connection])
|
||||
monkeypatch.setattr(db, "_get_pool", lambda: fake_pool)
|
||||
|
||||
with get_conn() as acquired:
|
||||
assert acquired is connection
|
||||
|
||||
assert connection.committed is True
|
||||
assert fake_pool.returned == [(connection, False)]
|
||||
|
||||
|
||||
def test_get_conn_discards_stale_connection_and_borrows_fresh(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
stale = FakeConnection(alive=False)
|
||||
fresh = FakeConnection(alive=True)
|
||||
fake_pool = FakePool([stale, fresh])
|
||||
monkeypatch.setattr(db, "_get_pool", lambda: fake_pool)
|
||||
|
||||
with get_conn() as acquired:
|
||||
assert acquired is fresh
|
||||
|
||||
assert stale.closed is True
|
||||
assert fresh.committed is True
|
||||
assert fake_pool.returned == [(stale, True), (fresh, False)]
|
||||
|
||||
|
||||
def test_get_conn_fails_closed_when_all_pooled_connections_are_dead(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
first = FakeConnection(alive=False)
|
||||
second = FakeConnection(alive=False)
|
||||
fake_pool = FakePool([first, second])
|
||||
monkeypatch.setattr(db, "_get_pool", lambda: fake_pool)
|
||||
|
||||
with pytest.raises(InterfaceError):
|
||||
with get_conn():
|
||||
pass
|
||||
|
||||
assert first.closed is True
|
||||
assert second.closed is True
|
||||
assert fake_pool.returned == [(first, True), (second, True)]
|
||||
|
||||
|
||||
def test_get_pool_retries_transient_initial_connection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
attempts: list[str] = []
|
||||
|
||||
def fake_pool(**_kwargs):
|
||||
attempts.append("connect")
|
||||
if len(attempts) == 1:
|
||||
raise db.OperationalError("transient connection failure")
|
||||
return object()
|
||||
|
||||
for name, value in {
|
||||
"PG_HOST": "db.example.test",
|
||||
"PG_PORT": "5432",
|
||||
"PG_DB": "warehouse",
|
||||
"PG_USER": "collector",
|
||||
"PG_PASSWORD": "unit-test-only",
|
||||
}.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.setattr(db, "_pool", None)
|
||||
monkeypatch.setattr(db.pool, "SimpleConnectionPool", fake_pool)
|
||||
monkeypatch.setattr(db.time, "sleep", lambda _seconds: attempts.append("sleep"))
|
||||
|
||||
result = db._get_pool(connect_attempts=2, retry_delay_seconds=0)
|
||||
|
||||
assert result is not None
|
||||
assert attempts == ["connect", "sleep", "connect"]
|
||||
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.product_commerce.direct_llm import (
|
||||
DirectLLMConfig,
|
||||
DirectLLMConfigurationError,
|
||||
DirectLLMResponseError,
|
||||
DirectLLMTransientError,
|
||||
chat_completion,
|
||||
load_direct_llm_config,
|
||||
)
|
||||
|
||||
|
||||
def _config() -> DirectLLMConfig:
|
||||
return DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-secret",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
)
|
||||
|
||||
|
||||
def _response(content: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"choices": [{"message": {"content": content}}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 4},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def test_direct_chat_uses_provider_endpoint_and_model_without_echoing_secret() -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def transport(url, headers, body, timeout):
|
||||
payload = json.loads(body)
|
||||
observed.update(
|
||||
url=url,
|
||||
model=payload["model"],
|
||||
messages=payload["messages"],
|
||||
reasoning_split=payload["reasoning_split"],
|
||||
timeout=timeout,
|
||||
authorization=headers["Authorization"],
|
||||
)
|
||||
return _response('{"ok":true}')
|
||||
|
||||
content, usage = chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
model="MiniMax-M3",
|
||||
timeout=17,
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert content == '{"ok":true}'
|
||||
assert usage == {"prompt_tokens": 3, "completion_tokens": 4}
|
||||
assert observed["url"] == "https://api.minimaxi.com/v1/chat/completions"
|
||||
assert observed["model"] == "MiniMax-M3"
|
||||
assert observed["reasoning_split"] is True
|
||||
assert observed["timeout"] == 17.0
|
||||
assert observed["authorization"] == "Bearer test-only-secret"
|
||||
|
||||
|
||||
def test_direct_chat_can_disable_minimax_thinking_for_structured_video_tasks() -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def transport(_url, _headers, body, _timeout):
|
||||
observed.update(json.loads(body))
|
||||
return _response("ok")
|
||||
|
||||
chat_completion(
|
||||
[{"role": "user", "content": "return one token"}],
|
||||
thinking_mode="disabled",
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert observed["thinking"] == {"type": "disabled"}
|
||||
|
||||
|
||||
def test_direct_chat_rejects_unknown_thinking_mode() -> None:
|
||||
with pytest.raises(DirectLLMConfigurationError, match="thinking mode"):
|
||||
chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
thinking_mode="sometimes",
|
||||
transport=lambda *_args: _response("unused"),
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_config_reads_shared_minimax_environment(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GYXX_DIRECT_LLM_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("GYXX_DIRECT_LLM_API_KEY", raising=False)
|
||||
monkeypatch.delenv("GYXX_DIRECT_LLM_MODEL", raising=False)
|
||||
monkeypatch.setenv("STYLE_ANALYSIS_LLM_BASE_URL", "https://api.minimaxi.com/v1")
|
||||
monkeypatch.setenv("STYLE_ANALYSIS_LLM_API_KEY", "test-only-secret")
|
||||
monkeypatch.setenv("STYLE_ANALYSIS_LLM_MODEL", "MiniMax-M3")
|
||||
|
||||
config = load_direct_llm_config()
|
||||
|
||||
assert config.endpoint == "https://api.minimaxi.com/v1/chat/completions"
|
||||
assert config.title_model == "MiniMax-M3"
|
||||
assert config.vision_model == "MiniMax-M3"
|
||||
assert "test-only-secret" not in repr(config)
|
||||
|
||||
|
||||
def test_config_rejects_legacy_local_gateway(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_DIRECT_LLM_BASE_URL",
|
||||
"http://127.0.0.1:8642/v1",
|
||||
)
|
||||
monkeypatch.setenv("GYXX_DIRECT_LLM_API_KEY", "test-only-secret")
|
||||
|
||||
with pytest.raises(DirectLLMConfigurationError, match="local gateway"):
|
||||
load_direct_llm_config()
|
||||
|
||||
|
||||
def test_invalid_response_is_fail_closed() -> None:
|
||||
with pytest.raises(DirectLLMResponseError, match="invalid JSON"):
|
||||
chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
transport=lambda *_args: b"not-json",
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_direct_chat_retries_transient_provider_overload(monkeypatch) -> None:
|
||||
calls = 0
|
||||
|
||||
def transport(_url, _headers, _body, _timeout):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls < 3:
|
||||
raise DirectLLMTransientError(
|
||||
"direct model request failed with HTTP 529"
|
||||
)
|
||||
return _response("recovered")
|
||||
|
||||
delays: list[float] = []
|
||||
monkeypatch.setattr(
|
||||
"gyxx_flow.modules.product_commerce.direct_llm.time.sleep",
|
||||
delays.append,
|
||||
)
|
||||
|
||||
content, _usage = chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert content == "recovered"
|
||||
assert calls == 3
|
||||
assert delays == [1.0, 3.0]
|
||||
|
||||
|
||||
def test_direct_chat_preserves_transient_error_after_bounded_retries(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
calls = 0
|
||||
|
||||
def transport(_url, _headers, _body, _timeout):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise DirectLLMTransientError(
|
||||
"direct model request failed with HTTP 529"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"gyxx_flow.modules.product_commerce.direct_llm.time.sleep",
|
||||
lambda _seconds: None,
|
||||
)
|
||||
|
||||
with pytest.raises(DirectLLMTransientError, match="HTTP 529"):
|
||||
chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert calls == 3
|
||||
@@ -0,0 +1,154 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.product_commerce.direct_llm import DirectLLMConfig
|
||||
from gyxx_flow.modules.product_commerce.direct_visual_color import (
|
||||
DirectVisualColorClassifier,
|
||||
VisualCandidate,
|
||||
)
|
||||
|
||||
|
||||
def _image(path: Path, color: str) -> Path:
|
||||
Image.new("RGB", (12, 12), color).save(path, format="PNG")
|
||||
return path
|
||||
|
||||
|
||||
def _response(classification: dict[str, object]) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps(classification),
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def test_direct_visual_classifier_sends_images_to_minimax_m3(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def transport(url, headers, body, timeout):
|
||||
payload = json.loads(body)
|
||||
image_parts = [
|
||||
part
|
||||
for part in payload["messages"][1]["content"]
|
||||
if part["type"] == "image_url"
|
||||
]
|
||||
observed.update(
|
||||
url=url,
|
||||
model=payload["model"],
|
||||
thinking=payload["thinking"],
|
||||
timeout=timeout,
|
||||
image_count=len(image_parts),
|
||||
data_url_prefix=image_parts[0]["image_url"]["url"][:22],
|
||||
has_auth=headers["Authorization"].startswith("Bearer "),
|
||||
)
|
||||
return _response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-secret",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert result.source_color == "black"
|
||||
assert result.selected_key == "candidate"
|
||||
assert observed == {
|
||||
"url": "https://api.minimaxi.com/v1/chat/completions",
|
||||
"model": "MiniMax-M3",
|
||||
"thinking": {"type": "disabled"},
|
||||
"timeout": 120.0,
|
||||
"image_count": 2,
|
||||
"data_url_prefix": "data:image/png;base64,",
|
||||
"has_auth": True,
|
||||
}
|
||||
|
||||
|
||||
def test_direct_visual_payload_uses_external_provider_endpoint(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def transport(url, _headers, body, _timeout):
|
||||
captured["url"] = url
|
||||
payload = json.loads(body)
|
||||
image_url = next(
|
||||
part["image_url"]["url"]
|
||||
for part in payload["messages"][1]["content"]
|
||||
if part["type"] == "image_url"
|
||||
)
|
||||
assert base64.b64decode(image_url.split(",", 1)[1])
|
||||
return _response(
|
||||
{
|
||||
"source_color": "unknown",
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-secret",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert captured["url"] == "https://api.minimaxi.com/v1/chat/completions"
|
||||
assert "8642" not in str(captured["url"])
|
||||
|
||||
|
||||
def test_direct_visual_classifier_accepts_model_reasoning_wrapper(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
classification = {
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
|
||||
def transport(_url, _headers, _body, _timeout):
|
||||
content = (
|
||||
"<think>分析图片后再给出严格结果。</think>\n"
|
||||
"```json\n"
|
||||
+ json.dumps(classification)
|
||||
+ "\n```"
|
||||
)
|
||||
return json.dumps(
|
||||
{"choices": [{"message": {"content": content}}]}
|
||||
).encode()
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-secret",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert result.source_color == "black"
|
||||
assert result.selected_key == "candidate"
|
||||
@@ -0,0 +1,524 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.product_commerce.direct_llm import DirectLLMConfig
|
||||
from gyxx_flow.modules.product_commerce.direct_visual_color import (
|
||||
COARSE_COLORS,
|
||||
MAX_CANDIDATES,
|
||||
DirectVisualColorClassifier,
|
||||
DirectVisualColorError,
|
||||
VisualCandidate,
|
||||
normalize_coarse_color,
|
||||
)
|
||||
|
||||
|
||||
def _image(path: Path, color: str) -> Path:
|
||||
Image.new("RGB", (12, 12), color).save(path, format="PNG")
|
||||
return path
|
||||
|
||||
|
||||
def _response(classification: dict[str, Any]) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": json.dumps(classification),
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _classifier(response: bytes, observed: dict[str, Any] | None = None):
|
||||
def transport(url, headers, body, timeout):
|
||||
if observed is not None:
|
||||
payload = json.loads(body)
|
||||
image_parts = [
|
||||
part
|
||||
for part in payload["messages"][1]["content"]
|
||||
if part["type"] == "image_url"
|
||||
]
|
||||
observed.update(
|
||||
{
|
||||
"url": url,
|
||||
"timeout": timeout,
|
||||
"model": payload["model"],
|
||||
"image_count": len(image_parts),
|
||||
"all_data_urls": all(
|
||||
part["image_url"]["url"].startswith(
|
||||
"data:image/png;base64,"
|
||||
)
|
||||
for part in image_parts
|
||||
),
|
||||
"has_authorization": headers.get("Authorization", "").startswith(
|
||||
"Bearer "
|
||||
),
|
||||
}
|
||||
)
|
||||
return response
|
||||
|
||||
return DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_unique_coarse_color_match_uses_local_images_as_data_urls(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "saddlebrown")
|
||||
black = _image(tmp_path / "black.png", "black")
|
||||
brown = _image(tmp_path / "brown.png", "peru")
|
||||
observed: dict[str, Any] = {}
|
||||
classifier = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "brown",
|
||||
"candidate_colors": {"sku-black": "black", "sku-brown": "tan"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
),
|
||||
observed,
|
||||
)
|
||||
|
||||
result = classifier.classify(
|
||||
[source],
|
||||
[
|
||||
VisualCandidate("sku-black", black),
|
||||
VisualCandidate("sku-brown", brown),
|
||||
],
|
||||
)
|
||||
|
||||
assert result.source_color == "brown"
|
||||
assert result.candidate_colors == {"sku-black": "black", "sku-brown": "brown"}
|
||||
assert result.selected_key == "sku-brown"
|
||||
assert result.selection_basis == "coarse_color"
|
||||
assert observed == {
|
||||
"url": "https://api.minimaxi.com/v1/chat/completions",
|
||||
"timeout": 120.0,
|
||||
"model": "MiniMax-M3",
|
||||
"image_count": 3,
|
||||
"all_data_urls": True,
|
||||
"has_authorization": True,
|
||||
}
|
||||
|
||||
|
||||
def test_same_color_tie_allows_one_visual_similarity_match(tmp_path) -> None:
|
||||
source_a = _image(tmp_path / "source-a.png", "navy")
|
||||
source_b = _image(tmp_path / "source-b.png", "blue")
|
||||
first = _image(tmp_path / "first.png", "navy")
|
||||
second = _image(tmp_path / "second.png", "blue")
|
||||
classifier = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "navy",
|
||||
"candidate_colors": {"first": "blue", "second": "blue"},
|
||||
"visual_match": {"unique": True, "key": "second"},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = classifier.classify(
|
||||
[source_a, source_b],
|
||||
[VisualCandidate("first", first), VisualCandidate("second", second)],
|
||||
)
|
||||
|
||||
assert result.source_color == "blue"
|
||||
assert result.selected_key == "second"
|
||||
assert result.selection_basis == "visual_similarity"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"classification",
|
||||
[
|
||||
{
|
||||
"source_color": "blue",
|
||||
"candidate_colors": {"first": "blue", "second": "blue"},
|
||||
"visual_match": {"unique": True, "key": "invented"},
|
||||
},
|
||||
{
|
||||
"source_color": "blue",
|
||||
"candidate_colors": {"first": "blue", "invented": "blue"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
},
|
||||
{
|
||||
"source_color": "blue",
|
||||
"candidate_colors": {"first": "blue"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_unknown_or_missing_candidate_key_fails_closed(
|
||||
tmp_path,
|
||||
classification,
|
||||
) -> None:
|
||||
source = _image(tmp_path / "source.png", "blue")
|
||||
first = _image(tmp_path / "first.png", "blue")
|
||||
second = _image(tmp_path / "second.png", "blue")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="candidate key"):
|
||||
_classifier(_response(classification)).classify(
|
||||
[source],
|
||||
[VisualCandidate("first", first), VisualCandidate("second", second)],
|
||||
)
|
||||
|
||||
|
||||
def test_one_character_candidate_key_typo_fails_closed(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="unknown candidate key"):
|
||||
_classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"card-eb51a74da3a4": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify(
|
||||
[source],
|
||||
[VisualCandidate("card-eb51a74da3a3", candidate)],
|
||||
)
|
||||
|
||||
|
||||
def test_visual_match_cannot_bypass_coarse_color_filter(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "red")
|
||||
red = _image(tmp_path / "red.png", "red")
|
||||
blue = _image(tmp_path / "blue.png", "blue")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="inconsistent"):
|
||||
_classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "red",
|
||||
"candidate_colors": {"red": "red", "blue": "blue"},
|
||||
"visual_match": {"unique": True, "key": "blue"},
|
||||
}
|
||||
)
|
||||
).classify(
|
||||
[source],
|
||||
[VisualCandidate("red", red), VisualCandidate("blue", blue)],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"assistant_content",
|
||||
[
|
||||
"not-json",
|
||||
'```json\n{"source_color":"black"}\n```',
|
||||
'{"source_color": NaN}',
|
||||
'{"source_color":"black","candidate_colors":{},"visual_match":{},"extra":1}',
|
||||
],
|
||||
)
|
||||
def test_non_strict_model_json_fails_closed(tmp_path, assistant_content) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
response = json.dumps(
|
||||
{"choices": [{"message": {"content": assistant_content}}]}
|
||||
).encode()
|
||||
|
||||
with pytest.raises(DirectVisualColorError):
|
||||
_classifier(response).classify(
|
||||
[source],
|
||||
[VisualCandidate("candidate", candidate)],
|
||||
)
|
||||
|
||||
|
||||
def test_non_string_source_color_fails_closed(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="source color"):
|
||||
_classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": None,
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify(
|
||||
[source],
|
||||
[VisualCandidate("candidate", candidate)],
|
||||
)
|
||||
|
||||
|
||||
def test_image_count_limits_fail_before_transport(tmp_path) -> None:
|
||||
image = _image(tmp_path / "image.png", "black")
|
||||
called = False
|
||||
|
||||
def transport(*_args):
|
||||
nonlocal called
|
||||
called = True
|
||||
raise AssertionError("must not call transport")
|
||||
|
||||
classifier = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
)
|
||||
with pytest.raises(DirectVisualColorError, match="between 1 and 2"):
|
||||
classifier.classify([], [VisualCandidate("one", image)])
|
||||
with pytest.raises(DirectVisualColorError, match="between 1 and 2"):
|
||||
classifier.classify([image, image, image, image], [VisualCandidate("one", image)])
|
||||
with pytest.raises(DirectVisualColorError, match="safe limit"):
|
||||
classifier.classify(
|
||||
[image],
|
||||
[VisualCandidate(f"candidate-{index}", image) for index in range(MAX_CANDIDATES + 1)],
|
||||
)
|
||||
assert called is False
|
||||
|
||||
|
||||
def test_http_failure_never_echoes_secret_or_image_payload(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
leaked_marker = "data:image/png;base64,SHOULD-NOT-LEAK"
|
||||
|
||||
def transport(*_args):
|
||||
raise RuntimeError(f"secret-value {leaked_marker}")
|
||||
|
||||
classifier = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="secret-value",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
)
|
||||
with pytest.raises(DirectVisualColorError) as captured:
|
||||
classifier.classify(
|
||||
[source],
|
||||
[VisualCandidate("candidate", candidate)],
|
||||
)
|
||||
|
||||
message = str(captured.value)
|
||||
assert "secret-value" not in message
|
||||
assert "base64" not in message
|
||||
assert captured.value.__cause__ is None
|
||||
|
||||
|
||||
def test_missing_or_invalid_image_fails_before_transport(tmp_path) -> None:
|
||||
invalid = tmp_path / "invalid.png"
|
||||
invalid.write_text("not an image", encoding="utf-8")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="invalid"):
|
||||
_classifier(b"unused").classify(
|
||||
[invalid],
|
||||
[VisualCandidate("candidate", candidate)],
|
||||
)
|
||||
|
||||
|
||||
def test_color_normalization_is_closed() -> None:
|
||||
assert normalize_coarse_color("Grey") == "unknown"
|
||||
assert normalize_coarse_color("light gray") == "white"
|
||||
assert normalize_coarse_color("\u6d45\u7070") == "white"
|
||||
assert normalize_coarse_color("silver-white") == "white"
|
||||
assert normalize_coarse_color("dark gray") == "black"
|
||||
assert normalize_coarse_color("\u70ad\u7070") == "black"
|
||||
assert normalize_coarse_color("\u6df1\u7070") == "black"
|
||||
assert normalize_coarse_color("\u5361\u5176") == "brown"
|
||||
assert normalize_coarse_color("navy") == "blue"
|
||||
assert normalize_coarse_color("transparent") == "unknown"
|
||||
assert normalize_coarse_color(None) == "unknown"
|
||||
|
||||
|
||||
def test_gray_model_output_is_unknown_and_never_matches(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "gray")
|
||||
candidate = _image(tmp_path / "candidate.png", "white")
|
||||
|
||||
result = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "gray",
|
||||
"candidate_colors": {"candidate": "white"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert result.source_color == "unknown"
|
||||
assert result.candidate_colors == {"candidate": "white"}
|
||||
assert result.selected_key is None
|
||||
assert result.selection_basis == "unresolved"
|
||||
|
||||
|
||||
def test_light_neutral_matches_white_but_remains_distinct_from_black(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "#dddddd")
|
||||
candidate = _image(tmp_path / "candidate.png", "white")
|
||||
|
||||
light_result = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "light gray",
|
||||
"candidate_colors": {"candidate": "white"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
dark_result = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"candidate": "light gray"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert light_result.source_color == "white"
|
||||
assert light_result.selected_key == "candidate"
|
||||
assert dark_result.source_color == "black"
|
||||
assert dark_result.candidate_colors == {"candidate": "white"}
|
||||
assert dark_result.selected_key is None
|
||||
|
||||
|
||||
def test_prompt_and_schema_split_neutrals_by_brightness(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "#dddddd")
|
||||
candidate = _image(tmp_path / "candidate.png", "white")
|
||||
observed: dict[str, Any] = {}
|
||||
|
||||
def transport(_url, _headers, body, _timeout):
|
||||
payload = json.loads(body)
|
||||
observed["instruction"] = payload["messages"][1]["content"][0]["text"]
|
||||
return _response(
|
||||
{
|
||||
"source_color": "white",
|
||||
"candidate_colors": {"candidate": "white"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
instruction = str(observed["instruction"])
|
||||
schema = json.loads(instruction.rsplit("shape: ", maxsplit=1)[1])
|
||||
allowed_value = "one of [" + ", ".join(COARSE_COLORS) + "]"
|
||||
assert "gray" not in COARSE_COLORS
|
||||
assert schema["source_color"] == allowed_value
|
||||
assert schema["candidate_colors"] == {"candidate": allowed_value}
|
||||
assert "silver-white, and light gray" in instruction
|
||||
assert "black includes black, charcoal, and dark gray" in instruction
|
||||
assert "Never output gray as a standalone color" in instruction
|
||||
assert result.source_color == "white"
|
||||
assert result.selected_key == "candidate"
|
||||
|
||||
|
||||
def test_prompt_classifies_only_bag_body_and_ignores_clothing_and_overlays(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
observed: dict[str, str] = {}
|
||||
|
||||
def transport(_url, _headers, body, _timeout):
|
||||
payload = json.loads(body)
|
||||
observed["system"] = payload["messages"][0]["content"]
|
||||
observed["instruction"] = payload["messages"][1]["content"][0]["text"]
|
||||
return _response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
instruction = observed["instruction"]
|
||||
assert "Classify the BAG itself, not the whole image" in instruction
|
||||
assert "only the dominant color of the bag's main body panels" in instruction
|
||||
assert "product color is never the color of a person or their clothing" in instruction
|
||||
assert "jackets, shirts, pants" in instruction
|
||||
assert "text, prices, badges" in instruction
|
||||
assert "black bag on a light-gray jacket is black" in instruction
|
||||
assert "black or navy bag on green clothing" in instruction
|
||||
assert "Never classify the overall image palette" in instruction
|
||||
assert "return unknown" in instruction
|
||||
assert "When that count is 0 or 1" in instruction
|
||||
assert 'visual_match MUST be exactly {"unique":false,"key":null}' in instruction
|
||||
assert "never put the sole same-color candidate" in instruction
|
||||
assert "Only when the count is 2 or more" in instruction
|
||||
assert "fail-closed product image classifier" in observed["system"]
|
||||
assert result.source_color == "black"
|
||||
assert result.selected_key == "candidate"
|
||||
|
||||
|
||||
def test_prompt_repeats_exact_opaque_candidate_key_allowlist(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
first = _image(tmp_path / "first.png", "black")
|
||||
second = _image(tmp_path / "second.png", "white")
|
||||
exact_keys = ("card-eb51a74da3a3", "card-eb51a74da3af")
|
||||
observed: dict[str, str] = {}
|
||||
|
||||
def transport(_url, _headers, body, _timeout):
|
||||
payload = json.loads(body)
|
||||
observed["instruction"] = payload["messages"][1]["content"][0]["text"]
|
||||
return _response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {
|
||||
exact_keys[0]: "black",
|
||||
exact_keys[1]: "white",
|
||||
},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify(
|
||||
[source],
|
||||
[VisualCandidate(exact_keys[0], first), VisualCandidate(exact_keys[1], second)],
|
||||
)
|
||||
|
||||
instruction = observed["instruction"]
|
||||
allowlist = json.dumps(list(exact_keys), separators=(",", ":"))
|
||||
assert instruction.count(allowlist) == 2
|
||||
assert "Candidate keys are opaque, immutable identifiers" in instruction
|
||||
assert "byte-for-byte exactly as provided" in instruction
|
||||
assert "do not edit, invent, autocomplete, normalize, truncate, or repair" in instruction
|
||||
assert "each provided candidate key exactly once and no other keys" in instruction
|
||||
assert result.selected_key == exact_keys[0]
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from collect_dy_market_rank import (
|
||||
@@ -7,12 +9,14 @@ from collect_dy_market_rank import (
|
||||
build_dy_category_table_xml,
|
||||
build_dy_feishu_batch_xml,
|
||||
build_dy_feishu_intro_xml,
|
||||
install_saved_cookies,
|
||||
is_excluded_product_title,
|
||||
navigate_to_product_rank,
|
||||
parse_market_product_rank_html,
|
||||
parse_money_range_upper,
|
||||
payment_metric_header_index,
|
||||
require_payment_metric_header,
|
||||
snapshot_rank_page,
|
||||
)
|
||||
from collect_sycm_market_rank import build_feishu_batch_xml
|
||||
|
||||
@@ -54,6 +58,24 @@ SAMPLE_HTML = """
|
||||
|
||||
|
||||
class DyMarketRankTests(unittest.TestCase):
|
||||
def test_saved_cookies_refresh_existing_context(self):
|
||||
context = MagicMock()
|
||||
cookies = [{"name": "session", "value": "secret"}]
|
||||
|
||||
self.assertEqual(install_saved_cookies(context, cookies), 1)
|
||||
context.add_cookies.assert_called_once_with(cookies)
|
||||
|
||||
def test_rank_timeout_snapshot_persists_dom(self):
|
||||
page = MagicMock()
|
||||
page.content.return_value = "<html><body>新版榜单</body></html>"
|
||||
|
||||
with TemporaryDirectory() as temp_dir:
|
||||
with patch("collect_dy_market_rank.DEBUG_DIR", Path(temp_dir)):
|
||||
path = snapshot_rank_page(page, "header timeout")
|
||||
|
||||
self.assertTrue(path.is_file())
|
||||
self.assertIn("新版榜单", path.read_text(encoding="utf-8"))
|
||||
|
||||
def test_title_blacklist_filters_children_primary_and_middle_school(self):
|
||||
self.assertTrue(is_excluded_product_title("儿童轻便双肩包"))
|
||||
self.assertTrue(is_excluded_product_title("小学生护脊书包"))
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.product_commerce import backfill_erp_all_shop_daily as backfill
|
||||
from gyxx_flow.modules.product_commerce.db import (
|
||||
upsert_erp_all_shop_style_daily_metrics,
|
||||
)
|
||||
|
||||
|
||||
def test_default_range_is_yesterday_only() -> None:
|
||||
assert backfill.DEFAULT_FROM_DATE == backfill.DEFAULT_TO_DATE
|
||||
assert backfill.DEFAULT_FROM_DATE == date.today() - backfill.timedelta(days=1)
|
||||
|
||||
|
||||
def test_historical_range_is_inclusive_and_covers_103_days() -> None:
|
||||
dates = backfill.inclusive_dates(date(2026, 5, 1), date(2026, 8, 11))
|
||||
|
||||
assert dates[0] == date(2026, 5, 1)
|
||||
assert dates[-1] == date(2026, 8, 11)
|
||||
assert len(dates) == 103
|
||||
|
||||
|
||||
def test_invalid_or_oversized_range_is_rejected() -> None:
|
||||
with pytest.raises(ValueError, match="--to"):
|
||||
backfill.inclusive_dates(date(2026, 8, 12), date(2026, 8, 11))
|
||||
with pytest.raises(ValueError, match="366"):
|
||||
backfill.inclusive_dates(date(2025, 1, 1), date(2026, 8, 11))
|
||||
|
||||
|
||||
def test_style_plan_rejects_one_erp_code_owned_by_two_styles() -> None:
|
||||
with pytest.raises(ValueError, match="belongs to both"):
|
||||
backfill.build_style_plans(
|
||||
{
|
||||
"styles": {
|
||||
"款式甲": {"erp_codes": ["10416"]},
|
||||
"款式乙": {"erp_codes": ["10416"]},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_all_shop_filter_selects_every_shop_and_batches_all_codes() -> None:
|
||||
class Frame:
|
||||
def __init__(self) -> None:
|
||||
self.script = ""
|
||||
self.args = {}
|
||||
|
||||
def evaluate(self, script, args):
|
||||
self.script = script
|
||||
self.args = args
|
||||
return {
|
||||
"selected_shop_count": 38,
|
||||
"selection_mode": "all",
|
||||
}
|
||||
|
||||
frame = Frame()
|
||||
result = backfill.set_all_shop_daily_filters(
|
||||
frame,
|
||||
report_date=date(2026, 5, 1),
|
||||
erp_codes=("10416", "10455"),
|
||||
)
|
||||
|
||||
assert frame.args == {
|
||||
"reportDate": "2026-05-01",
|
||||
"erpCodes": ["10416", "10455"],
|
||||
}
|
||||
assert 'input[name="shop_id"]' in frame.script
|
||||
assert "=== '全选'" in frame.script
|
||||
assert "erpCodes.join(',')" in frame.script
|
||||
assert "shopLabels" not in frame.script
|
||||
assert result == {"selected_shop_count": 38, "selection_mode": "all"}
|
||||
|
||||
|
||||
def test_style_records_sum_product_rows_and_keep_zero_styles() -> None:
|
||||
plans = (
|
||||
backfill.StylePlan("款式甲", ("10416", "10455")),
|
||||
backfill.StylePlan("款式乙", ("10504",)),
|
||||
)
|
||||
rows = [
|
||||
{"erp_style_code": "10416", "sales": 12, "return_qty": 2},
|
||||
{"erp_style_code": "10416", "sales": 8, "return_qty": 1},
|
||||
{"erp_style_code": "10455", "sales": 5, "return_qty": 3},
|
||||
]
|
||||
|
||||
records = backfill.aggregate_style_records(
|
||||
date(2026, 5, 1),
|
||||
plans,
|
||||
rows,
|
||||
selected_shop_count=38,
|
||||
source="raw/product_commerce/evidence.json",
|
||||
)
|
||||
|
||||
assert records[0] == {
|
||||
"metric_date": date(2026, 5, 1),
|
||||
"style_name": "款式甲",
|
||||
"sales": 25,
|
||||
"return_qty": 6,
|
||||
"erp_style_codes": ["10416", "10455"],
|
||||
"selected_shop_count": 38,
|
||||
"matched_product_rows": 3,
|
||||
"source": "raw/product_commerce/evidence.json",
|
||||
}
|
||||
assert records[1]["sales"] == 0
|
||||
assert records[1]["return_qty"] == 0
|
||||
assert records[1]["matched_product_rows"] == 0
|
||||
|
||||
|
||||
def test_range_collector_queries_once_per_date_not_once_per_style(monkeypatch) -> None:
|
||||
filter_frame = object()
|
||||
monkeypatch.setattr(
|
||||
backfill.erp, "find_report_frames", lambda _page: (filter_frame, object())
|
||||
)
|
||||
calls: list[date] = []
|
||||
|
||||
def collect_one(_page, _frame, _plans, report_date, **_kwargs):
|
||||
calls.append(report_date)
|
||||
return (
|
||||
{"selected_shop_count": 38},
|
||||
{"rows": [], "frame_url": f"https://example/{report_date}", "signature": str(report_date)},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(backfill, "collect_one_date", collect_one)
|
||||
persisted: list[date] = []
|
||||
backfill.collect_range_in_page(
|
||||
object(),
|
||||
(
|
||||
backfill.StylePlan("款式甲", ("10416", "10455")),
|
||||
backfill.StylePlan("款式乙", ("10504",)),
|
||||
),
|
||||
(date(2026, 5, 1), date(2026, 5, 2)),
|
||||
persist=lambda report_date, _selection, _report: persisted.append(report_date),
|
||||
)
|
||||
|
||||
assert calls == [date(2026, 5, 1), date(2026, 5, 2)]
|
||||
assert persisted == calls
|
||||
|
||||
|
||||
def test_database_upsert_uses_date_and_style_conflict_key(monkeypatch) -> None:
|
||||
captured = {}
|
||||
|
||||
class Cursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
class Connection:
|
||||
def cursor(self):
|
||||
return Cursor()
|
||||
|
||||
def fake_execute_values(cursor, sql, rows):
|
||||
captured.update(cursor=cursor, sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"gyxx_flow.modules.product_commerce.db.execute_values", fake_execute_values
|
||||
)
|
||||
written = upsert_erp_all_shop_style_daily_metrics(
|
||||
Connection(),
|
||||
[
|
||||
{
|
||||
"metric_date": date(2026, 5, 1),
|
||||
"style_name": "款式甲",
|
||||
"sales": 25,
|
||||
"return_qty": 6,
|
||||
"erp_style_codes": ["10455", "10416"],
|
||||
"selected_shop_count": 38,
|
||||
"matched_product_rows": 3,
|
||||
"source": "raw/evidence.json",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
assert "ON CONFLICT (metric_date, style_name)" in captured["sql"]
|
||||
assert captured["rows"][0][2:4] == (25, 6)
|
||||
assert captured["rows"][0][4] == ["10416", "10455"]
|
||||
|
||||
|
||||
def test_schema_keeps_all_shop_metrics_separate_from_platform_metrics() -> None:
|
||||
schema = (
|
||||
Path(backfill.__file__).parent / "db" / "schema.sql"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "CREATE TABLE IF NOT EXISTS erp_all_shop_style_daily_metrics" in schema
|
||||
assert "PRIMARY KEY (metric_date, style_name)" in schema
|
||||
assert "return_qty" in schema
|
||||
@@ -5,22 +5,97 @@ from unittest.mock import Mock, patch
|
||||
|
||||
from upload_video_to_guanghe import (
|
||||
GuangHeUploader,
|
||||
TmallTitleGenerationError,
|
||||
_match_tm_item_ids_from_product_titles,
|
||||
_select_video_product_names,
|
||||
_tm_product_title_search_pattern,
|
||||
download_videos,
|
||||
)
|
||||
|
||||
|
||||
def test_tm_product_title_search_pattern_uses_model_token() -> None:
|
||||
assert _tm_product_title_search_pattern("拾影单肩包") == "%拾影%"
|
||||
assert _tm_product_title_search_pattern("星云mini") == "%星云mini%"
|
||||
|
||||
|
||||
def test_multi_style_record_prefers_style_named_by_attachment() -> None:
|
||||
assert _select_video_product_names(
|
||||
["阿波罗2", "宙斯3"], "宙斯3 7.30.mp4"
|
||||
) == ["宙斯3"]
|
||||
assert _select_video_product_names(
|
||||
["阿波罗2", "宙斯3"], "7月30日.mp4"
|
||||
) == ["阿波罗2", "宙斯3"]
|
||||
|
||||
|
||||
def test_attachment_style_matching_handles_combos_and_overlapping_aliases() -> None:
|
||||
assert _select_video_product_names(
|
||||
["阿波罗2", "宙斯3", "星云2", "逐星"],
|
||||
"宙斯3+阿波罗2+星云2+逐星.m4v",
|
||||
) == ["宙斯3", "阿波罗2", "星云2", "逐星"]
|
||||
assert _select_video_product_names(
|
||||
["极星双肩", "极星双肩2"], "极星双肩2 新新新.mp4"
|
||||
) == ["极星双肩2"]
|
||||
|
||||
|
||||
class GuangHeMetadataTests(unittest.TestCase):
|
||||
def test_formal_title_rejection_uses_non_browser_failure_type(self):
|
||||
GuangHeUploader.configure_catalog_product_titles(
|
||||
{"极星双肩2": ["极星双肩2双肩包轻便电脑包"]}
|
||||
)
|
||||
uploader = GuangHeUploader("", "", require_llm_titles=True)
|
||||
try:
|
||||
with patch(
|
||||
"upload_video_to_guanghe.JdVideoTitleGenerator.generate",
|
||||
side_effect=RuntimeError("候选不合格"),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
TmallTitleGenerationError, "候选不合格"
|
||||
):
|
||||
uploader._generate_title(["极星双肩2"], video_key="token")
|
||||
finally:
|
||||
GuangHeUploader.configure_catalog_product_titles({})
|
||||
|
||||
def test_formal_collection_title_forbids_specific_style_names(self):
|
||||
GuangHeUploader.configure_catalog_product_titles(
|
||||
{
|
||||
"宙斯3": ["宙斯3双肩包男士大容量"],
|
||||
"阿波罗2": ["阿波罗2电脑包手提轻便"],
|
||||
}
|
||||
)
|
||||
uploader = GuangHeUploader("", "", require_llm_titles=True)
|
||||
try:
|
||||
with patch(
|
||||
"upload_video_to_guanghe.JdVideoTitleGenerator"
|
||||
) as generator_class:
|
||||
generator = generator_class.return_value
|
||||
generator.generate.return_value = (
|
||||
"多款通勤包袋,大容量轻便按场景切换"
|
||||
)
|
||||
title = uploader._generate_title(
|
||||
["宙斯3", "阿波罗2"], video_key="combo-token"
|
||||
)
|
||||
self.assertEqual(title, "多款通勤包袋,大容量轻便按场景切换")
|
||||
self.assertTrue(generator_class.call_args.kwargs["generic_collection"])
|
||||
self.assertFalse(generator_class.call_args.kwargs["require_product_name"])
|
||||
self.assertEqual(generator_class.call_args.kwargs["llm_timeout"], 300)
|
||||
generator.generate.assert_called_once()
|
||||
self.assertEqual(
|
||||
generator.generate.call_args.args[0], ["宙斯3", "阿波罗2"]
|
||||
)
|
||||
finally:
|
||||
GuangHeUploader.configure_catalog_product_titles({})
|
||||
|
||||
def test_downloads_with_same_attachment_name_are_isolated_by_record(self):
|
||||
attachment = {"file_token": "file-token", "name": "same-name.mp4"}
|
||||
|
||||
with TemporaryDirectory() as tmp_dir, patch(
|
||||
"upload_video_to_guanghe.DOWNLOAD_DIR", Path(tmp_dir)
|
||||
), patch("upload_video_to_guanghe.run_lark") as run_lark_mock:
|
||||
def fake_download(args):
|
||||
output_dir = Path(args[args.index("--output") + 1])
|
||||
def fake_download(args, *, cwd=None, timeout=None):
|
||||
self.assertEqual(timeout, 600)
|
||||
output_dir = Path(cwd)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / attachment["name"]).write_bytes(b"video")
|
||||
(output_dir / args[args.index("--output") + 1]).write_bytes(b"video")
|
||||
return 0, "", ""
|
||||
|
||||
run_lark_mock.side_effect = fake_download
|
||||
@@ -28,8 +103,9 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
first = download_videos("record-one", [attachment])
|
||||
second = download_videos("record-two", [attachment])
|
||||
|
||||
self.assertEqual(first[0].parent.name, "record-one")
|
||||
self.assertEqual(second[0].parent.name, "record-two")
|
||||
self.assertEqual(first[0].parent.parent.name, "record-one")
|
||||
self.assertEqual(second[0].parent.parent.name, "record-two")
|
||||
self.assertEqual(first[0].parent.name, "file-token")
|
||||
self.assertNotEqual(first[0], second[0])
|
||||
|
||||
def test_unresolved_style_uses_exact_database_product_title_match(self):
|
||||
@@ -81,14 +157,20 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
def test_topic_result_card_requires_topic_and_activity_statistics(self):
|
||||
self.assertTrue(
|
||||
GuangHeUploader._topic_result_card_text_matches(
|
||||
"# 在淘宝种草一夏 小二推荐 作品数469.5万 参与数16.2万",
|
||||
"在淘宝种草一夏",
|
||||
"# 我的夏日焕新清单 小二推荐 作品数469.5万 参与数16.2万",
|
||||
"我的夏日焕新清单",
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
GuangHeUploader._topic_result_card_text_matches(
|
||||
"# 秋上新开拍了 时尚 作品数198.3万 参与数11.3万",
|
||||
"秋上新",
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
GuangHeUploader._topic_result_card_text_matches(
|
||||
"在淘宝种草一夏",
|
||||
"在淘宝种草一夏",
|
||||
"我的夏日焕新清单",
|
||||
"我的夏日焕新清单",
|
||||
)
|
||||
)
|
||||
self.assertTrue(GuangHeUploader._topic_card_box_is_clickable(620, 120))
|
||||
@@ -160,13 +242,12 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertGreaterEqual(len(title), 26)
|
||||
self.assertLessEqual(len(title), 30)
|
||||
|
||||
def test_same_style_multiple_videos_get_distinct_titles_when_hermes_repeats(self):
|
||||
def test_same_style_multiple_videos_get_distinct_titles_when_model_repeats(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
repeated = "光影行星宙斯双肩包科学分区收纳轻量通勤出行"
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
|
||||
patch("upload_video_to_guanghe.call_direct_llm", return_value=(repeated, {})),
|
||||
):
|
||||
first = uploader._generate_title(["宙斯"], video_index=1, video_total=2)
|
||||
second = uploader._generate_title(["宙斯"], video_index=2, video_total=2)
|
||||
@@ -175,13 +256,12 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertTrue(first.startswith("宙斯双肩包,"))
|
||||
self.assertTrue(second.startswith("宙斯双肩包,"))
|
||||
|
||||
def test_same_style_repeated_hermes_copy_gets_low_similarity_bodies(self):
|
||||
def test_same_style_repeated_model_copy_gets_low_similarity_bodies(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
repeated = "光影行星宙斯双肩包,科学分区收纳,轻量通勤出行!"
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
|
||||
patch("upload_video_to_guanghe.call_direct_llm", return_value=(repeated, {})),
|
||||
):
|
||||
titles = [
|
||||
uploader._generate_title(["宙斯"], video_index=index, video_total=4)
|
||||
@@ -202,8 +282,7 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
repeated = "科学分区收纳,轻装通勤出行"
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
|
||||
patch("upload_video_to_guanghe.call_direct_llm", return_value=(repeated, {})),
|
||||
):
|
||||
first = uploader._generate_title(["宙斯"], video_index=1)
|
||||
second = uploader._generate_title(["星云2"], video_index=1)
|
||||
@@ -339,9 +418,8 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch(
|
||||
"upload_video_to_guanghe._call_hermes",
|
||||
"upload_video_to_guanghe.call_direct_llm",
|
||||
return_value=("光影行星宙斯双肩包大容量笔记本电脑轻量通勤", {}),
|
||||
),
|
||||
):
|
||||
@@ -350,7 +428,7 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertTrue(title.startswith("宙斯双肩包,"))
|
||||
self.assertNotIn("包袋双肩包", title)
|
||||
|
||||
def test_generated_poseidon_title_cannot_be_changed_to_backpack_by_hermes(self):
|
||||
def test_generated_poseidon_title_cannot_be_changed_to_backpack_by_model(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
with (
|
||||
patch.object(
|
||||
@@ -358,9 +436,8 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
"_search_product_features",
|
||||
return_value="双肩包、电脑包、托特包、大容量",
|
||||
),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch(
|
||||
"upload_video_to_guanghe._call_hermes",
|
||||
"upload_video_to_guanghe.call_direct_llm",
|
||||
return_value=(
|
||||
"光影行星波塞冬edge双肩包暴雨通勤防泼水电脑仓从容出行",
|
||||
{},
|
||||
@@ -439,8 +516,8 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
with self.subTest(product_names=product_names):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
calls = []
|
||||
uploader._pick_content_labels = Mock(
|
||||
side_effect=lambda labels: calls.append(("labels", list(labels)))
|
||||
uploader._pick_content_label = Mock(
|
||||
side_effect=lambda label: calls.append(("label", label))
|
||||
)
|
||||
uploader._pick_topic = Mock(
|
||||
side_effect=lambda topic: calls.append(("topic", topic))
|
||||
@@ -452,8 +529,8 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
calls,
|
||||
[
|
||||
("labels", expected_labels),
|
||||
("topic", "在淘宝种草一夏"),
|
||||
*[("label", label) for label in expected_labels],
|
||||
("topic", "我的夏日焕新清单"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -461,8 +538,55 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertEqual(GuangHeUploader.FIXED_CONTENT_LABELS, ("商品展示",))
|
||||
self.assertNotIn("户外", GuangHeUploader.FIXED_CONTENT_LABELS)
|
||||
|
||||
def test_missing_user_selected_topic_stops_metadata_preparation(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
uploader._pick_content_label = Mock()
|
||||
uploader._pick_topic = Mock(
|
||||
side_effect=RuntimeError("没有找到固定话题: #我的夏日焕新清单")
|
||||
)
|
||||
uploader._snapshot = Mock()
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "没有找到固定话题"):
|
||||
uploader._prepare_publish_metadata(["极星双肩"])
|
||||
|
||||
self.assertEqual(
|
||||
uploader._pick_content_label.call_args_list,
|
||||
[unittest.mock.call("双肩包"), unittest.mock.call("商品展示")],
|
||||
)
|
||||
|
||||
def test_optional_content_label_failures_do_not_skip_required_topic(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
uploader.page = Mock()
|
||||
uploader._pick_content_label = Mock(
|
||||
side_effect=[
|
||||
RuntimeError("内容标签没有精确匹配项: #双肩包"),
|
||||
RuntimeError("内容标签没有精确匹配项: #商品展示"),
|
||||
]
|
||||
)
|
||||
uploader._pick_topic = Mock()
|
||||
uploader._snapshot = Mock()
|
||||
|
||||
uploader._prepare_publish_metadata(["极星双肩"])
|
||||
|
||||
self.assertEqual(
|
||||
uploader._pick_content_label.call_args_list,
|
||||
[unittest.mock.call("双肩包"), unittest.mock.call("商品展示")],
|
||||
)
|
||||
uploader._pick_topic.assert_called_once_with("我的夏日焕新清单")
|
||||
|
||||
def test_metadata_uses_operator_topic_keyword(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
calls = []
|
||||
uploader._pick_content_label = Mock()
|
||||
uploader._pick_topic = Mock(side_effect=lambda topic: calls.append(topic))
|
||||
uploader._snapshot = Mock()
|
||||
|
||||
uploader._prepare_publish_metadata(["极星双肩"], topic_keyword="秋上新")
|
||||
|
||||
self.assertEqual(calls, ["秋上新"])
|
||||
|
||||
def test_fixed_topic_is_not_a_recommended_first_result(self):
|
||||
self.assertEqual(GuangHeUploader.FIXED_TOPIC, "在淘宝种草一夏")
|
||||
self.assertEqual(GuangHeUploader.FIXED_TOPIC, "我的夏日焕新清单")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
import db as product_db
|
||||
import import_douyin_qianchuan_ads as importer
|
||||
from openpyxl import Workbook
|
||||
|
||||
HEADERS = [
|
||||
"商品ID",
|
||||
"商品名称",
|
||||
"日期",
|
||||
"整体展示次数",
|
||||
"整体点击次数",
|
||||
"整体点击率",
|
||||
"整体转化率",
|
||||
"整体消耗(元)",
|
||||
"整体成交金额(元)",
|
||||
"整体支付ROI",
|
||||
"整体成交订单成本(元)",
|
||||
"用户实际支付金额(元)",
|
||||
"电商平台补贴金额(元)",
|
||||
"净成交ROI",
|
||||
"净成交金额(元)",
|
||||
"净成交订单成本(元)",
|
||||
"净成交金额结算率",
|
||||
"1小时内退款率",
|
||||
]
|
||||
|
||||
|
||||
def _workbook(path) -> None:
|
||||
workbook = Workbook()
|
||||
sheet = workbook.active
|
||||
sheet.append(HEADERS)
|
||||
sheet.append(
|
||||
[
|
||||
"3725242433383039375",
|
||||
"测试商品",
|
||||
"全部",
|
||||
"12,345",
|
||||
"678",
|
||||
"5.49%",
|
||||
"1.00%",
|
||||
"100.00",
|
||||
"900.00",
|
||||
"9.00",
|
||||
"25.00",
|
||||
"850.00",
|
||||
"50.00",
|
||||
"8.00",
|
||||
"800.00",
|
||||
"30.00",
|
||||
"88.89%",
|
||||
"11.11%",
|
||||
]
|
||||
)
|
||||
sheet.append(
|
||||
[
|
||||
"3725242433383039375",
|
||||
"测试商品",
|
||||
"2026-08-23",
|
||||
"12,345",
|
||||
"678",
|
||||
"5.49%",
|
||||
"1.00%",
|
||||
"100.00",
|
||||
"900.00",
|
||||
"9.00",
|
||||
"25.00",
|
||||
"850.00",
|
||||
"50.00",
|
||||
"8.00",
|
||||
"800.00",
|
||||
"30.00",
|
||||
"88.89%",
|
||||
"11.11%",
|
||||
]
|
||||
)
|
||||
workbook.save(path)
|
||||
|
||||
|
||||
def test_parse_workbook_keeps_long_product_id_and_skips_total(tmp_path) -> None:
|
||||
path = tmp_path / "qianchuan.xlsx"
|
||||
_workbook(path)
|
||||
|
||||
records = importer.parse_workbook(path)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["stat_date"] == date(2026, 8, 23)
|
||||
assert records[0]["product_id"] == "3725242433383039375"
|
||||
assert records[0]["overall_impressions"] == 12345
|
||||
assert records[0]["spend"] == Decimal("100.00")
|
||||
assert records[0]["refund_rate_1h"] == Decimal("11.11")
|
||||
|
||||
|
||||
def test_upsert_uses_advertiser_date_product_business_key(monkeypatch) -> None:
|
||||
executed = {}
|
||||
|
||||
class Cursor:
|
||||
def execute(self, sql, params=None):
|
||||
executed["lookup_sql"] = sql
|
||||
executed["lookup_params"] = params
|
||||
|
||||
def fetchall(self):
|
||||
return [("测试款式", ["3725242433383039375"])]
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback):
|
||||
return None
|
||||
|
||||
class Connection:
|
||||
def cursor(self):
|
||||
return Cursor()
|
||||
|
||||
def fake_execute_values(cursor, sql, rows):
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
"product_id": "3725242433383039375",
|
||||
"product_name": "测试商品",
|
||||
"spend": Decimal("100.00"),
|
||||
}
|
||||
|
||||
assert product_db.upsert_douyin_product_ad_daily_metrics(
|
||||
Connection(), [record, record]
|
||||
) == 1
|
||||
assert "ON CONFLICT (stat_date, advertiser_id, product_id)" in executed["sql"]
|
||||
assert len(executed["rows"]) == 1
|
||||
assert "style_name" in executed["sql"]
|
||||
assert executed["rows"][0][4] == "测试款式"
|
||||
assert executed["rows"][0][5] == "product_id"
|
||||
|
||||
|
||||
def test_upsert_falls_back_to_exact_product_name_style_match(monkeypatch) -> None:
|
||||
executed = {}
|
||||
|
||||
class Cursor:
|
||||
def execute(self, sql, params=None):
|
||||
executed["lookup_sql"] = sql
|
||||
executed["lookup_params"] = params
|
||||
|
||||
def fetchall(self):
|
||||
return [("测试款式", [])]
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback):
|
||||
return None
|
||||
|
||||
class Connection:
|
||||
def cursor(self):
|
||||
return Cursor()
|
||||
|
||||
def fake_execute_values(cursor, sql, rows):
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
"product_id": "unmapped-product-id",
|
||||
"product_name": "GYXX/测试款式3双肩包2026新款",
|
||||
"spend": Decimal("100.00"),
|
||||
}
|
||||
|
||||
assert product_db.upsert_douyin_product_ad_daily_metrics(Connection(), [record]) == 1
|
||||
assert executed["rows"][0][4] == "测试款式"
|
||||
assert executed["rows"][0][5] == "product_name"
|
||||
@@ -0,0 +1,215 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import db as product_db
|
||||
import import_jd_ad_costs as importer
|
||||
|
||||
|
||||
def test_jd_exports_only_import_current_block_and_ignore_comparison_rows(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
full = tmp_path / "full.csv"
|
||||
full.write_text(
|
||||
"查询条件,广告主PIN: 光影行星旗舰店霖; 时间: 2026-08-23\n"
|
||||
"日期,'当前时间',SPU ID,商品计划名称,SpuImgUrl,花费,全站投产比\n"
|
||||
"20260823~20260823,当前时间,spu-1,盖亚款式,img,12.30,4.50\n"
|
||||
"汇总,,,汇总,,12.30,4.50\n"
|
||||
"20260822~20260822,对比时间,spu-old,旧款,img,99.00,9.00\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
sku = tmp_path / "sku.csv"
|
||||
sku.write_text(
|
||||
"查询条件,广告主PIN: 光影行星旗舰店霖; 时间: 2026-08-23\n"
|
||||
"SKU ID,SKU 名称,点击时间,'当前时间',花费,展现数,点击数,投产比\n"
|
||||
"sku-1,光影行星 盖亚款式 黑色,20260823~20260823,当前时间,3.20,100,5,2.50\n"
|
||||
"sku-old,光影行星 盖亚款式 黑色,20260822~20260822,对比时间,99.00,100,5,9.00\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
|
||||
full_records = importer.parse_full_csv(full, date(2026, 8, 23))
|
||||
sku_records = importer.parse_non_full_csv(sku, date(2026, 8, 23))
|
||||
|
||||
assert [record["spu_id"] for record in full_records] == ["spu-1"]
|
||||
assert [record["sku_id"] for record in sku_records] == ["sku-1"]
|
||||
assert full_records[0]["metrics"]["source"]["current_marker"] == "当前时间"
|
||||
assert full_records[0]["spend"] == importer.Decimal("12.30")
|
||||
assert sku_records[0]["metrics"]["clicks"] == 5
|
||||
|
||||
|
||||
def test_sku_name_matches_dim_style_spu_and_keeps_audit_fields(tmp_path: Path) -> None:
|
||||
report = tmp_path / "sku.csv"
|
||||
report.write_text(
|
||||
"查询条件,广告主PIN: 光影行星旗舰店霖\n"
|
||||
"SKU ID,SKU 名称,点击时间,'当前时间',花费,投产比\n"
|
||||
"sku-1,光影行星(GYXX)盖亚C1斜挎包 曜夜黑,20260823~20260823,当前时间,3.20,2.50\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
sku_records = importer.parse_non_full_csv(report, date(2026, 8, 23))
|
||||
matched = importer.apply_sku_matches(
|
||||
sku_records,
|
||||
style_catalog=[{"style_name": "盖亚斜挎", "jd_spus": ["spu-1"]}],
|
||||
full_records=[{"spu_id": "spu-1", "spu_name": "盖亚C1斜挎包"}],
|
||||
)
|
||||
|
||||
assert matched[0]["match_status"] == "matched"
|
||||
assert matched[0]["spu_id"] == "spu-1"
|
||||
assert matched[0]["style_name"] == "盖亚斜挎"
|
||||
assert matched[0]["metrics"]["match"]["reason"]
|
||||
|
||||
|
||||
def test_full_spu_matches_dim_style_exactly() -> None:
|
||||
full_records = [
|
||||
{
|
||||
"spu_id": "spu-1",
|
||||
"style_name": None,
|
||||
"match_status": "direct",
|
||||
"match_reason": "full-site SPU report",
|
||||
"metrics": {},
|
||||
}
|
||||
]
|
||||
|
||||
matched = importer.apply_full_style_matches(
|
||||
full_records,
|
||||
style_catalog=[
|
||||
{"style_name": "盖亚斜挎", "jd_spus": ["spu-1"]},
|
||||
],
|
||||
)
|
||||
|
||||
assert matched[0]["style_name"] == "盖亚斜挎"
|
||||
assert matched[0]["match_status"] == "direct"
|
||||
assert matched[0]["metrics"]["match"]["style_name"] == "盖亚斜挎"
|
||||
|
||||
|
||||
def test_full_spu_name_matches_base_style_when_spu_mapping_is_missing() -> None:
|
||||
full_records = [
|
||||
{
|
||||
"spu_id": "spu-2",
|
||||
"spu_name": "光影行星(GYXX)盖亚斜挎包3单反相机包",
|
||||
"style_name": None,
|
||||
"match_status": "direct",
|
||||
"match_reason": "full-site SPU report",
|
||||
"metrics": {},
|
||||
}
|
||||
]
|
||||
|
||||
matched = importer.apply_full_style_matches(
|
||||
full_records,
|
||||
style_catalog=[
|
||||
{"style_name": "盖亚斜挎", "jd_spus": []},
|
||||
{"style_name": "盖亚双肩3", "jd_spus": []},
|
||||
],
|
||||
spu_catalog=[
|
||||
{"spu_id": "spu-2", "product_no": "盖亚斜挎3"},
|
||||
],
|
||||
)
|
||||
|
||||
assert matched[0]["style_name"] == "盖亚斜挎"
|
||||
assert matched[0]["metrics"]["match"]["reason"] == (
|
||||
"full-site product_no matched style name"
|
||||
)
|
||||
|
||||
|
||||
def test_sku_name_uses_latest_jd_spu_catalog_when_full_report_lacks_spu() -> None:
|
||||
sku_records = [
|
||||
{
|
||||
"sku_name": "光影行星(GYXX)阿波罗2斜挎包男士通勤便携单肩背包 曜夜黑",
|
||||
"metrics": {},
|
||||
}
|
||||
]
|
||||
matched = importer.apply_sku_matches(
|
||||
sku_records,
|
||||
style_catalog=[
|
||||
{"style_name": "阿波罗2", "jd_spus": ["spu-1", "spu-2"]}
|
||||
],
|
||||
full_records=[],
|
||||
spu_catalog=[
|
||||
{
|
||||
"spu_id": "spu-1",
|
||||
"spu_name": "光影行星(GYXX)阿波罗2斜挎包男士通勤便携单肩背包",
|
||||
},
|
||||
{"spu_id": "spu-2", "spu_name": "另一款商品"},
|
||||
],
|
||||
)
|
||||
|
||||
assert matched[0]["match_status"] == "matched"
|
||||
assert matched[0]["spu_id"] == "spu-1"
|
||||
|
||||
|
||||
class _Cursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
class _Connection:
|
||||
def cursor(self):
|
||||
return _Cursor()
|
||||
|
||||
|
||||
def test_jd_upsert_deduplicates_same_logical_row(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows, *, template=None):
|
||||
captured["sql"] = sql
|
||||
captured["rows"] = rows
|
||||
captured["template"] = template
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_pin": "光影行星旗舰店霖",
|
||||
"shop_account": "光影行星研发",
|
||||
"report_type": "full",
|
||||
"row_key": "spu:spu-1",
|
||||
"spu_id": "spu-1",
|
||||
"spu_name": "盖亚",
|
||||
"match_status": "direct",
|
||||
"spend": importer.Decimal("12.30"),
|
||||
"reported_roi": importer.Decimal("4.50"),
|
||||
"metrics": {"spend": "12.30"},
|
||||
"source_file": "full.csv",
|
||||
"source_row_number": 3,
|
||||
}
|
||||
|
||||
assert product_db.upsert_jd_product_ad_daily_metrics(
|
||||
_Connection(), [record, {**record, "spend": importer.Decimal("13.30")}]
|
||||
) == 1
|
||||
assert len(captured["rows"]) == 1
|
||||
assert captured["rows"][0][11] == importer.Decimal("13.30")
|
||||
assert "jd_product_ad_daily_metrics" in str(captured["sql"])
|
||||
|
||||
|
||||
def test_jd_dry_run_accepts_explicit_desktop_style_files(tmp_path: Path, capsys) -> None:
|
||||
full = tmp_path / "full.csv"
|
||||
full.write_text(
|
||||
"条件,广告主PIN: 光影行星旗舰店霖\n"
|
||||
"日期,'当前时间',SPU ID,商品计划名称,花费\n"
|
||||
"20260823~20260823,当前时间,spu-1,盖亚,1.00\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
sku = tmp_path / "sku.csv"
|
||||
sku.write_text(
|
||||
"条件,广告主PIN: 光影行星旗舰店霖\n"
|
||||
"SKU ID,SKU 名称,点击时间,'当前时间',花费\n"
|
||||
"sku-1,盖亚黑色,20260823~20260823,当前时间,1.00\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
|
||||
assert importer.main(
|
||||
[
|
||||
"--date",
|
||||
"2026-08-23",
|
||||
"--full-file",
|
||||
str(full),
|
||||
"--non-full-file",
|
||||
str(sku),
|
||||
]
|
||||
) == 0
|
||||
output = capsys.readouterr().out
|
||||
assert "全站 SPU 1" in output
|
||||
assert "非全站 SKU 1" in output
|
||||
assert "DRY-RUN" in output
|
||||
@@ -0,0 +1,187 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import db as product_db
|
||||
import import_tmall_product_ads as importer
|
||||
|
||||
|
||||
class _Cursor:
|
||||
def execute(self, sql, params=None):
|
||||
self.sql = sql
|
||||
self.params = params
|
||||
|
||||
def fetchall(self):
|
||||
return [("极星", ["1001"])]
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
class _Connection:
|
||||
def cursor(self):
|
||||
return _Cursor()
|
||||
|
||||
|
||||
def test_parse_tmall_product_ad_csv_merges_product_rows(tmp_path: Path) -> None:
|
||||
report = tmp_path / "商品报表.csv"
|
||||
report.write_text(
|
||||
"日期,主体ID,主体类型,主体名称,花费,直接成交金额,间接成交金额,总成交金额,投入产出比,展现量\n"
|
||||
"2026-08-17,1001,商品,极星单肩包,90.25,730.02,0,730.02,8.09,1000\n"
|
||||
"2026-08-17,1001,商品,极星单肩包,9.75,0,70,70,7.18,200\n"
|
||||
"2026-08-17,plan-1,计划,测试计划,100,1000,0,1000,10,500\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
|
||||
records = importer.parse_tmall_product_ad_csv(report)
|
||||
|
||||
assert len(records) == 1
|
||||
metrics = records[0]["ad_metrics"]
|
||||
assert records[0]["product_id"] == "1001"
|
||||
assert metrics["spend"] == "100.00"
|
||||
assert metrics["attributed_deal_amount"] == "800.02"
|
||||
assert metrics["roi"] == "8.00"
|
||||
assert metrics["impressions"] == 1200
|
||||
assert records[0]["duplicate_row_count"] == 2
|
||||
|
||||
|
||||
def test_parse_gb18030_export(tmp_path: Path) -> None:
|
||||
report = tmp_path / "商品报表.csv"
|
||||
report.write_bytes(
|
||||
(
|
||||
"日期,主体ID,主体类型,主体名称,花费,总成交金额,投入产出比\n"
|
||||
"2026-08-17,1001,商品,极星单肩包,90.25,730.02,8.09\n"
|
||||
).encode("gb18030")
|
||||
)
|
||||
|
||||
records = importer.parse_tmall_product_ad_csv(report)
|
||||
|
||||
assert records[0]["product_name"] == "极星单肩包"
|
||||
assert records[0]["ad_metrics"]["roi"] == "8.09"
|
||||
|
||||
|
||||
def test_parse_export_null_markers_as_missing(tmp_path: Path) -> None:
|
||||
report = tmp_path / "商品报表.csv"
|
||||
report.write_text(
|
||||
"日期,主体ID,主体类型,主体名称,花费,总成交金额,平均点击花费,投入产出比\n"
|
||||
"2026-08-17,1001,商品,极星单肩包,90.25,730.02,\\N,\\N\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
records = importer.parse_tmall_product_ad_csv(report)
|
||||
|
||||
assert records[0]["ad_metrics"] == {
|
||||
"spend": "90.25",
|
||||
"attributed_deal_amount": "730.02",
|
||||
"roi": "8.09",
|
||||
}
|
||||
|
||||
|
||||
def test_find_report_files_uses_current_manifest_csv(monkeypatch, tmp_path: Path) -> None:
|
||||
target_date = importer.date(2026, 8, 17)
|
||||
date_root = tmp_path / target_date.isoformat()
|
||||
extracted = date_root / "extracted"
|
||||
extracted.mkdir(parents=True)
|
||||
old_csv = extracted / "商品报表_旧.csv"
|
||||
current_csv = extracted / "商品报表_新.csv"
|
||||
old_csv.write_text("old", encoding="utf-8")
|
||||
current_csv.write_text("current", encoding="utf-8")
|
||||
(date_root / "manifest.json").write_text(
|
||||
json.dumps({"csv_files": ["extracted\\商品报表_新.csv"]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(importer, "REPORT_ROOT", tmp_path)
|
||||
|
||||
assert importer.find_report_files(target_date) == [current_csv.resolve()]
|
||||
|
||||
|
||||
def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows, *, template=None):
|
||||
captured["sql"] = sql
|
||||
captured["rows"] = rows
|
||||
captured["template"] = template
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
_Connection(),
|
||||
[
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "极星单肩包",
|
||||
"ad_metrics": {
|
||||
"spend": "90.25",
|
||||
"attributed_deal_amount": "730.02",
|
||||
"roi": "8.09",
|
||||
},
|
||||
"source_file": "商品报表.csv",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
sql = str(captured["sql"])
|
||||
assert "INSERT INTO product_daily_metrics" in sql
|
||||
assert "jsonb_set" in sql
|
||||
assert "tm_product_ad_daily_metrics" not in sql
|
||||
assert captured["template"] == "('tm', %s, %s, %s, %s::jsonb, %s, NOW())"
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["spend"] == "90.25"
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_id"
|
||||
assert row[1] == "1001"
|
||||
|
||||
|
||||
def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class _NameFallbackCursor(_Cursor):
|
||||
def fetchall(self):
|
||||
return [("极星", [])]
|
||||
|
||||
class _NameFallbackConnection:
|
||||
def cursor(self):
|
||||
return _NameFallbackCursor()
|
||||
|
||||
def fake_execute_values(_cursor, _sql, rows, *, template=None):
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
_NameFallbackConnection(),
|
||||
[
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "unmapped-id",
|
||||
"product_name": "GYXX/极星单肩包2026新款",
|
||||
"ad_metrics": {"spend": "12.00"},
|
||||
"source_file": "商品报表.csv",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_name"
|
||||
|
||||
|
||||
def test_import_date_filter_and_dry_run(tmp_path: Path, capsys) -> None:
|
||||
report = tmp_path / "商品报表.csv"
|
||||
report.write_text(
|
||||
"日期,主体ID,主体类型,主体名称,花费,总成交金额\n"
|
||||
"2026-08-16,1001,商品,旧日期,1,2\n"
|
||||
"2026-08-17,1002,商品,目标日期,3,9\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert importer.main(["--file", str(report), "--date", "2026-08-17"]) == 0
|
||||
assert "DRY-RUN" in capsys.readouterr().out
|
||||
@@ -7,7 +7,9 @@ from collect_jd_market_rank import (
|
||||
build_jd_feishu_batch_xml,
|
||||
build_jd_feishu_category_xml,
|
||||
build_jd_feishu_intro_xml,
|
||||
category_label_is_selected,
|
||||
category_selection_matches,
|
||||
category_ui_label_candidates,
|
||||
exact_category_label_pattern,
|
||||
group_products_by_category,
|
||||
high_resolution_jd_image_url,
|
||||
@@ -15,6 +17,7 @@ from collect_jd_market_rank import (
|
||||
parse_jd_market_rank_html,
|
||||
parse_money_range_upper,
|
||||
resize_feishu_table_xml,
|
||||
select_category_path,
|
||||
select_category_paths,
|
||||
)
|
||||
|
||||
@@ -80,14 +83,13 @@ class JdMarketRankTests(unittest.TestCase):
|
||||
set(specs),
|
||||
{"旅行箱", "运动包", "双肩包", "斜挎包", "胸包", "手机包", "电脑包"},
|
||||
)
|
||||
self.assertEqual(specs["旅行箱"]["paths"], (("功能箱包", "行李箱"),))
|
||||
self.assertEqual(specs["运动包"]["paths"], (("功能箱包", "休闲运动包"),))
|
||||
self.assertEqual(
|
||||
specs["双肩包"]["paths"],
|
||||
(("男包", "双肩包"), ("男包", "男士双肩包")),
|
||||
)
|
||||
self.assertEqual(specs["手机包"]["paths"], (("男包", "男士手机包"),))
|
||||
self.assertEqual(specs["电脑包"]["paths"], (("功能箱包", "电脑包"),))
|
||||
self.assertEqual(specs["旅行箱"]["path"], ("功能箱包", "行李箱"))
|
||||
self.assertEqual(specs["运动包"]["path"], ("功能箱包", "休闲运动包"))
|
||||
self.assertEqual(specs["双肩包"]["path"], ("男包", "男士双肩包"))
|
||||
self.assertEqual(specs["斜挎包"]["path"], ("男包", "男士单肩/斜挎包"))
|
||||
self.assertEqual(specs["胸包"]["path"], ("男包", "男士腰包/胸包"))
|
||||
self.assertEqual(specs["手机包"]["path"], ("男包", "男士手机包"))
|
||||
self.assertEqual(specs["电脑包"]["path"], ("功能箱包", "电脑包"))
|
||||
|
||||
def test_category_label_matching_is_exact_not_contains(self):
|
||||
pattern = exact_category_label_pattern("双肩包")
|
||||
@@ -96,6 +98,116 @@ class JdMarketRankTests(unittest.TestCase):
|
||||
self.assertIsNone(pattern.fullmatch("男士双肩包"))
|
||||
self.assertIsNone(pattern.fullmatch("双肩包配件"))
|
||||
|
||||
def test_live_ui_alias_keeps_the_source_business_category(self):
|
||||
self.assertEqual(
|
||||
category_ui_label_candidates("男士双肩包"),
|
||||
("男士双肩包", "双肩包"),
|
||||
)
|
||||
self.assertEqual(
|
||||
category_ui_label_candidates("男士手机包"),
|
||||
("男士手机包",),
|
||||
)
|
||||
|
||||
def test_category_parent_hover_uses_the_dropdown_option_not_its_label(self):
|
||||
page = MagicMock()
|
||||
trigger = MagicMock()
|
||||
parent_option = MagicMock()
|
||||
child_option = MagicMock()
|
||||
with (
|
||||
patch("collect_jd_market_rank._visible", return_value=trigger),
|
||||
patch(
|
||||
"collect_jd_market_rank._selected_category_labels",
|
||||
side_effect=[("功能箱包",), ("男包 > 双肩包",)],
|
||||
),
|
||||
patch("collect_jd_market_rank._first_product_key", return_value="old"),
|
||||
patch(
|
||||
"collect_jd_market_rank._exact_visible_category_option",
|
||||
return_value=parent_option,
|
||||
) as resolve_option,
|
||||
patch(
|
||||
"collect_jd_market_rank._wait_for_category_option",
|
||||
return_value=(child_option, "双肩包"),
|
||||
) as wait_for_child,
|
||||
patch(
|
||||
"collect_jd_market_rank._click_category_option"
|
||||
) as click_option,
|
||||
patch("collect_jd_market_rank._wait_for_table_change"),
|
||||
):
|
||||
select_category_path(page, ("男包", "男士双肩包"))
|
||||
|
||||
resolve_option.assert_called_once_with(page, "男包")
|
||||
wait_for_child.assert_called_once_with(
|
||||
page,
|
||||
("男士双肩包", "双肩包"),
|
||||
)
|
||||
parent_option.hover.assert_called_once_with(timeout=5_000)
|
||||
click_option.assert_called_once_with(child_option, timeout_ms=10_000)
|
||||
|
||||
def test_same_selected_parent_opens_directly_on_child_options(self):
|
||||
page = MagicMock()
|
||||
trigger = MagicMock()
|
||||
child_option = MagicMock()
|
||||
with (
|
||||
patch("collect_jd_market_rank._visible", return_value=trigger),
|
||||
patch(
|
||||
"collect_jd_market_rank._selected_category_labels",
|
||||
side_effect=[
|
||||
("功能箱包",),
|
||||
("功能箱包 > 休闲运动包",),
|
||||
],
|
||||
),
|
||||
patch("collect_jd_market_rank._first_product_key", return_value="old"),
|
||||
patch(
|
||||
"collect_jd_market_rank._exact_visible_category_option",
|
||||
return_value=None,
|
||||
) as resolve_option,
|
||||
patch(
|
||||
"collect_jd_market_rank._wait_for_category_option",
|
||||
return_value=(child_option, "休闲运动包"),
|
||||
),
|
||||
patch(
|
||||
"collect_jd_market_rank._click_category_option"
|
||||
) as click_option,
|
||||
patch("collect_jd_market_rank._wait_for_table_change"),
|
||||
):
|
||||
select_category_path(page, ("功能箱包", "休闲运动包"))
|
||||
|
||||
resolve_option.assert_called_once_with(page, "功能箱包")
|
||||
click_option.assert_called_once_with(child_option, timeout_ms=10_000)
|
||||
|
||||
def test_cross_parent_switch_clicks_parent_when_hover_keeps_stale_children(self):
|
||||
page = MagicMock()
|
||||
trigger = MagicMock()
|
||||
parent_option = MagicMock()
|
||||
child_option = MagicMock()
|
||||
with (
|
||||
patch("collect_jd_market_rank._visible", return_value=trigger),
|
||||
patch(
|
||||
"collect_jd_market_rank._selected_category_labels",
|
||||
side_effect=[("功能箱包",), ("男包 > 双肩包",)],
|
||||
),
|
||||
patch("collect_jd_market_rank._first_product_key", return_value="old"),
|
||||
patch(
|
||||
"collect_jd_market_rank._exact_visible_category_option",
|
||||
return_value=parent_option,
|
||||
),
|
||||
patch(
|
||||
"collect_jd_market_rank._wait_for_category_option",
|
||||
side_effect=[(None, None), (child_option, "双肩包")],
|
||||
) as wait_for_child,
|
||||
patch(
|
||||
"collect_jd_market_rank._click_category_option"
|
||||
) as click_option,
|
||||
patch("collect_jd_market_rank._wait_for_table_change"),
|
||||
):
|
||||
select_category_path(page, ("男包", "男士双肩包"))
|
||||
|
||||
self.assertEqual(wait_for_child.call_count, 2)
|
||||
self.assertEqual(
|
||||
[item.args[0] for item in click_option.call_args_list],
|
||||
[parent_option, child_option],
|
||||
)
|
||||
|
||||
def test_selected_category_verification_accepts_only_exact_display_forms(self):
|
||||
path = ("男包", "双肩包")
|
||||
|
||||
@@ -104,6 +216,10 @@ class JdMarketRankTests(unittest.TestCase):
|
||||
self.assertFalse(category_selection_matches(("男士双肩包",), path))
|
||||
self.assertFalse(category_selection_matches(("双肩包配件",), path))
|
||||
|
||||
def test_selected_parent_matching_is_exact(self):
|
||||
self.assertTrue(category_label_is_selected((" 功能 箱包 ",), "功能箱包"))
|
||||
self.assertFalse(category_label_is_selected(("功能箱包配件",), "功能箱包"))
|
||||
|
||||
def test_category_candidates_fall_back_only_after_explicit_failure(self):
|
||||
page = MagicMock()
|
||||
paths = (("男包", "双肩包"), ("男包", "男士双肩包"))
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.product_commerce import jd_video_frames
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("duration", "max_frames", "expected"),
|
||||
[
|
||||
(2.0, 3, (1.0,)),
|
||||
(3.0, 3, (1.0, 2.0)),
|
||||
(7.5, 3, (2.5, 5.0)),
|
||||
(8.0, 3, (1.6, 4.0, 6.4)),
|
||||
(8.0, 2, (4.0, 6.4)),
|
||||
(48.88, 2, (24.44, 39.104)),
|
||||
(48.88, 1, (24.44,)),
|
||||
],
|
||||
)
|
||||
def test_choose_reference_frame_times(
|
||||
duration: float,
|
||||
max_frames: int,
|
||||
expected: tuple[float, ...],
|
||||
) -> None:
|
||||
assert jd_video_frames.choose_reference_frame_times(
|
||||
duration,
|
||||
max_frames=max_frames,
|
||||
) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("duration", [0.0, -1.0, math.nan, math.inf])
|
||||
def test_choose_reference_frame_times_rejects_invalid_duration(
|
||||
duration: float,
|
||||
) -> None:
|
||||
with pytest.raises(
|
||||
jd_video_frames.VideoFrameExtractionError,
|
||||
match="视频时长无效",
|
||||
):
|
||||
jd_video_frames.choose_reference_frame_times(duration)
|
||||
|
||||
|
||||
class _FakeReader:
|
||||
def __init__(self, values: list[object], closed: list[bool]) -> None:
|
||||
self._values = iter(values)
|
||||
self._closed = closed
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
return next(self._values)
|
||||
|
||||
def close(self) -> None:
|
||||
self._closed.append(True)
|
||||
|
||||
|
||||
def test_extract_reference_frames_writes_decodable_jpegs(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
source = tmp_path / "source.mp4"
|
||||
source.write_bytes(b"video-placeholder")
|
||||
output_root = tmp_path / "allowed"
|
||||
output_dir = output_root / "probe"
|
||||
closed: list[bool] = []
|
||||
frame = bytes(range(12))
|
||||
|
||||
def read_frames(_path, *, pix_fmt, input_params, output_params):
|
||||
assert pix_fmt == "rgb24"
|
||||
assert output_params == ["-frames:v", "1"]
|
||||
metadata = {"duration": 12.0, "size": (2, 2)}
|
||||
values = [metadata] if not input_params else [metadata, frame]
|
||||
return _FakeReader(values, closed)
|
||||
|
||||
monkeypatch.setattr(jd_video_frames.imageio_ffmpeg, "read_frames", read_frames)
|
||||
|
||||
result = jd_video_frames.extract_reference_frames(
|
||||
source,
|
||||
output_dir=output_dir,
|
||||
allowed_output_root=output_root,
|
||||
timestamps_seconds=(3.0, 6.0),
|
||||
max_frames=2,
|
||||
max_edge=256,
|
||||
)
|
||||
|
||||
assert [item.timestamp_seconds for item in result] == [3.0, 6.0]
|
||||
assert len(closed) == 3
|
||||
assert all(item.path.is_file() for item in result)
|
||||
assert all(len(item.sha256) == 64 for item in result)
|
||||
with Image.open(result[0].path) as image:
|
||||
assert image.mode == "RGB"
|
||||
assert image.size == (2, 2)
|
||||
|
||||
|
||||
def test_extract_reference_frames_cleans_partial_outputs_on_failure(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
source = tmp_path / "source.mp4"
|
||||
source.write_bytes(b"video-placeholder")
|
||||
output_root = tmp_path / "allowed"
|
||||
output_dir = output_root / "probe"
|
||||
calls = 0
|
||||
|
||||
def read_frames(_path, *, pix_fmt, input_params, output_params):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
metadata = {"duration": 12.0, "size": (2, 2)}
|
||||
if not input_params:
|
||||
return _FakeReader([metadata], [])
|
||||
if calls == 2:
|
||||
return _FakeReader([metadata, bytes(range(12))], [])
|
||||
return _FakeReader([metadata, b"truncated"], [])
|
||||
|
||||
monkeypatch.setattr(jd_video_frames.imageio_ffmpeg, "read_frames", read_frames)
|
||||
|
||||
with pytest.raises(
|
||||
jd_video_frames.VideoFrameExtractionError,
|
||||
match="RGB 帧长度无效",
|
||||
):
|
||||
jd_video_frames.extract_reference_frames(
|
||||
source,
|
||||
output_dir=output_dir,
|
||||
allowed_output_root=output_root,
|
||||
timestamps_seconds=(3.0, 6.0),
|
||||
max_frames=2,
|
||||
max_edge=256,
|
||||
)
|
||||
|
||||
assert list(output_dir.glob("*")) == []
|
||||
|
||||
|
||||
def test_extract_reference_frames_rejects_output_outside_allowed_root(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
source = tmp_path / "source.mp4"
|
||||
source.write_bytes(b"video-placeholder")
|
||||
|
||||
with pytest.raises(
|
||||
jd_video_frames.VideoFrameExtractionError,
|
||||
match="越出允许的临时目录",
|
||||
):
|
||||
jd_video_frames.extract_reference_frames(
|
||||
source,
|
||||
output_dir=tmp_path / "outside",
|
||||
allowed_output_root=tmp_path / "allowed",
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import date, datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -50,53 +50,6 @@ def test_build_notification_message_supports_partial_platform_success():
|
||||
assert "当前成功平台 1 个" in message
|
||||
|
||||
|
||||
def test_build_hermes_payload_requires_direct_message_to_recipient():
|
||||
payload = notification.build_hermes_payload(
|
||||
recipient_open_id="ou_wangyunlong",
|
||||
message="三个链接",
|
||||
)
|
||||
|
||||
encoded = json.dumps(payload, ensure_ascii=False)
|
||||
assert "ou_wangyunlong" in encoded
|
||||
assert "飞书私聊" in encoded
|
||||
assert "三个链接" in encoded
|
||||
|
||||
|
||||
def test_extracts_real_feishu_message_id_from_hermes_completion():
|
||||
response = {
|
||||
"id": "api-response-id",
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "发送成功,消息ID om_x100b696f1169f0a0b1f9c311e8dc323"
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
assert (
|
||||
notification.extract_feishu_message_id(response)
|
||||
== "om_x100b696f1169f0a0b1f9c311e8dc323"
|
||||
)
|
||||
assert notification.hermes_send_succeeded(response) is True
|
||||
|
||||
|
||||
def test_hermes_success_text_without_real_message_id_is_not_success():
|
||||
response = {
|
||||
"choices": [{"message": {"content": "发送成功,但没有返回消息回执"}}]
|
||||
}
|
||||
|
||||
assert notification.extract_feishu_message_id(response) is None
|
||||
assert notification.hermes_send_succeeded(response) is False
|
||||
|
||||
|
||||
def test_extracts_nested_feishu_message_id_and_accepts_safe_suffixes():
|
||||
response = {"data": {"message_id": "om_real-id_123"}}
|
||||
|
||||
assert notification.extract_feishu_message_id(response) == "om_real-id_123"
|
||||
assert notification.hermes_send_succeeded(response) is True
|
||||
|
||||
|
||||
def test_notify_skips_when_same_day_recipient_was_already_sent():
|
||||
conn = MagicMock()
|
||||
with (
|
||||
@@ -104,7 +57,7 @@ def test_notify_skips_when_same_day_recipient_was_already_sent():
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "get_market_rank_reports_for_date", return_value=REPORTS),
|
||||
patch.object(notification, "market_rank_notification_was_sent", return_value=True),
|
||||
patch.object(notification, "post_hermes") as post_hermes,
|
||||
patch.object(notification, "send_lark_bot_message") as send_message,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
result = notification.notify_market_rank_reports(
|
||||
@@ -114,21 +67,12 @@ def test_notify_skips_when_same_day_recipient_was_already_sent():
|
||||
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "already_sent"
|
||||
post_hermes.assert_not_called()
|
||||
send_message.assert_not_called()
|
||||
|
||||
|
||||
def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
|
||||
def test_notify_uses_lark_user_sender_and_persists_success():
|
||||
conn = MagicMock()
|
||||
response = {
|
||||
"id": "hermes-response-id",
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "发送成功,消息ID om_x100b696f1169f0a0b1f9c311e8dc323"
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
response = {"message_id": "om_x100b696f1169f0a0b1f9c311e8dc323"}
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
@@ -138,7 +82,11 @@ def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
|
||||
"get_market_rank_reports_for_date",
|
||||
return_value=REPORTS,
|
||||
) as get_reports,
|
||||
patch.object(notification, "post_hermes", return_value=response) as post_hermes,
|
||||
patch.object(
|
||||
notification,
|
||||
"send_lark_bot_message",
|
||||
return_value=response,
|
||||
) as send_message,
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
@@ -156,9 +104,16 @@ def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
|
||||
notification.PLATFORM_ORDER,
|
||||
updated_after=cutoff,
|
||||
)
|
||||
post_hermes.assert_called_once()
|
||||
send_message.assert_called_once()
|
||||
sent = send_message.call_args.kwargs
|
||||
assert sent["user_id"] == "ou_wangyunlong"
|
||||
assert sent["profile"] == "hermes-analyzer"
|
||||
assert "天猫:https://example.com/tm" in sent["text"]
|
||||
uuid.UUID(sent["idempotency_key"])
|
||||
saved = upsert.call_args.args[1]
|
||||
assert saved["status"] == "sent"
|
||||
assert saved["hermes_message_id"] == response["message_id"]
|
||||
assert result["message_id"] == response["message_id"]
|
||||
assert saved["platform_links"] == {
|
||||
"tm": "https://example.com/tm",
|
||||
"jd": "https://example.com/jd",
|
||||
@@ -168,15 +123,17 @@ def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
|
||||
|
||||
def test_notify_sends_available_links_when_other_platforms_are_missing():
|
||||
conn = MagicMock()
|
||||
response = {
|
||||
"choices": [{"message": {"content": "发送成功,消息ID om_partial123"}}]
|
||||
}
|
||||
response = {"message_id": "om_partial123"}
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "market_rank_notification_was_sent", return_value=False),
|
||||
patch.object(notification, "get_market_rank_reports_for_date", return_value=REPORTS[:2]),
|
||||
patch.object(notification, "post_hermes", return_value=response) as post_hermes,
|
||||
patch.object(
|
||||
notification,
|
||||
"send_lark_bot_message",
|
||||
return_value=response,
|
||||
) as send_message,
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
@@ -187,10 +144,41 @@ def test_notify_sends_available_links_when_other_platforms_are_missing():
|
||||
|
||||
assert result["status"] == "sent"
|
||||
assert result["missing_platforms"] == ["dy"]
|
||||
post_hermes.assert_called_once()
|
||||
send_message.assert_called_once()
|
||||
assert upsert.call_args.args[1]["status"] == "sent"
|
||||
|
||||
|
||||
def test_notify_archives_lark_user_send_failure() -> None:
|
||||
conn = MagicMock()
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "market_rank_notification_was_sent", return_value=False),
|
||||
patch.object(
|
||||
notification,
|
||||
"get_market_rank_reports_for_date",
|
||||
return_value=REPORTS,
|
||||
),
|
||||
patch.object(
|
||||
notification,
|
||||
"send_lark_bot_message",
|
||||
side_effect=RuntimeError("lark unavailable"),
|
||||
),
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
result = notification.notify_market_rank_reports(
|
||||
report_date=date(2026, 7, 26),
|
||||
recipient_open_id="ou_wangyunlong",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert "lark unavailable" in result["error"]
|
||||
saved = upsert.call_args.args[1]
|
||||
assert saved["status"] == "failed"
|
||||
assert saved["hermes_message_id"] is None
|
||||
|
||||
|
||||
def test_notify_skips_only_when_current_run_has_no_links():
|
||||
conn = MagicMock()
|
||||
with (
|
||||
@@ -198,7 +186,7 @@ def test_notify_skips_only_when_current_run_has_no_links():
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "get_market_rank_reports_for_date", return_value=[]),
|
||||
patch.object(notification, "market_rank_notification_was_sent") as was_sent,
|
||||
patch.object(notification, "post_hermes") as post_hermes,
|
||||
patch.object(notification, "send_lark_bot_message") as send_message,
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
@@ -210,5 +198,5 @@ def test_notify_skips_only_when_current_run_has_no_links():
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "no_platform_reports"
|
||||
was_sent.assert_not_called()
|
||||
post_hermes.assert_not_called()
|
||||
send_message.assert_not_called()
|
||||
assert upsert.call_args.args[1]["status"] == "incomplete"
|
||||
|
||||
@@ -5,6 +5,7 @@ import sys
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
from threading import Barrier
|
||||
from types import SimpleNamespace
|
||||
|
||||
import orchestrate_market_rank_collection as workflow
|
||||
import pytest
|
||||
@@ -219,6 +220,50 @@ def test_workflow_notifies_after_all_parallel_steps_finish(tmp_path):
|
||||
assert saved["notification"]["status"] == "sent"
|
||||
|
||||
|
||||
def test_workflow_notifies_each_dynamic_recipient_and_aggregates_results(tmp_path):
|
||||
notifications: list[str] = []
|
||||
|
||||
def successful_executor(*, platform, command, log_path, dry_run):
|
||||
return {
|
||||
"platform": platform,
|
||||
"label": workflow.PLATFORM_LABELS[platform],
|
||||
"command": command,
|
||||
"log_path": str(log_path),
|
||||
"started_at": "2026-08-07T10:00:00",
|
||||
"finished_at": "2026-08-07T10:01:00",
|
||||
"duration_seconds": 60.0,
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
def fake_notifier(**kwargs):
|
||||
recipient = kwargs["recipient_open_id"]
|
||||
notifications.append(recipient)
|
||||
return {"status": "sent", "recipient_open_id": recipient}
|
||||
|
||||
summary = workflow.execute_workflow(
|
||||
workflow.DEFAULT_PLATFORMS,
|
||||
run_id="multi-notify-run",
|
||||
log_dir=tmp_path,
|
||||
executor=successful_executor,
|
||||
report_date=date.today(),
|
||||
notify_open_ids=("ou_first", "ou_second"),
|
||||
notifier=fake_notifier,
|
||||
)
|
||||
|
||||
assert notifications == ["ou_first", "ou_second"]
|
||||
assert summary["notification"] == {
|
||||
"status": "sent",
|
||||
"recipient_count": 2,
|
||||
"deliveries": [
|
||||
{"status": "sent", "recipient_open_id": "ou_first"},
|
||||
{"status": "sent", "recipient_open_id": "ou_second"},
|
||||
],
|
||||
}
|
||||
assert summary["notifications"] == summary["notification"]["deliveries"]
|
||||
assert summary["exit_code"] == 0
|
||||
|
||||
|
||||
def test_partial_current_run_links_can_still_send_successfully(tmp_path):
|
||||
def successful_executor(*, platform, command, log_path, dry_run):
|
||||
return {
|
||||
@@ -249,6 +294,36 @@ def test_partial_current_run_links_can_still_send_successfully(tmp_path):
|
||||
assert summary["status"] == "success"
|
||||
|
||||
|
||||
def test_no_current_run_links_is_a_valid_no_notification_outcome(tmp_path):
|
||||
def successful_executor(*, platform, command, log_path, dry_run):
|
||||
return {
|
||||
"platform": platform,
|
||||
"command": command,
|
||||
"log_path": str(log_path),
|
||||
"started_at": "2026-08-07T10:00:00",
|
||||
"finished_at": "2026-08-07T10:01:00",
|
||||
"duration_seconds": 60.0,
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
summary = workflow.execute_workflow(
|
||||
workflow.DEFAULT_PLATFORMS,
|
||||
run_id="no-current-links-run",
|
||||
log_dir=tmp_path,
|
||||
executor=successful_executor,
|
||||
notify_open_id="ou_owner",
|
||||
notifier=lambda **_kwargs: {
|
||||
"status": "skipped",
|
||||
"reason": "no_platform_reports",
|
||||
},
|
||||
)
|
||||
|
||||
assert summary["notification"]["reason"] == "no_platform_reports"
|
||||
assert summary["exit_code"] == 0
|
||||
assert summary["status"] == "success"
|
||||
|
||||
|
||||
def test_platform_parser_deduplicates_while_preserving_order():
|
||||
assert workflow.parse_platforms("dy,tm,dy,jd") == ("dy", "tm", "jd")
|
||||
|
||||
@@ -289,6 +364,41 @@ def test_main_allows_explicit_no_notify_without_recipient(monkeypatch, tmp_path)
|
||||
assert observed["notify_open_id"] is None
|
||||
|
||||
|
||||
def test_main_passes_all_dynamic_route_recipients(monkeypatch, tmp_path):
|
||||
observed = {}
|
||||
|
||||
def fake_execute(*args, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return {"exit_code": 0}
|
||||
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
monkeypatch.setattr(workflow, "DEFAULT_NOTIFY_OPEN_ID", "")
|
||||
monkeypatch.setattr(
|
||||
workflow,
|
||||
"resolve_notification_route",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
enabled=True,
|
||||
open_ids=("ou_first", "ou_second"),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(workflow, "execute_workflow", fake_execute)
|
||||
monkeypatch.setattr(workflow, "managed_data_path", Path)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"orchestrate_market_rank_collection.py",
|
||||
"--dry-run",
|
||||
"--log-dir",
|
||||
str(tmp_path),
|
||||
],
|
||||
)
|
||||
|
||||
assert workflow.main() == 0
|
||||
assert observed["notify_open_ids"] == ("ou_first", "ou_second")
|
||||
assert observed["notify_open_id"] is None
|
||||
|
||||
|
||||
def test_weekly_launcher_uses_market_rank_orchestrator():
|
||||
launcher = (
|
||||
HISTORY_ROOT / "launchers_reference" / "run_weekly_market_rank.bat"
|
||||
|
||||
@@ -19,6 +19,26 @@ WORKERS = (tm_worker, dy_worker, jd_worker)
|
||||
BUSINESS_DATE = date(2026, 8, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_missing_time_select_option_is_treated_as_no_existing_record(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: SimpleNamespace(
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr=json.dumps(
|
||||
{
|
||||
"ok": False,
|
||||
"error": {"code": 800030005, "message": "not_found"},
|
||||
}
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
assert worker.find_record_id_by_time("bas_test", "tbl_test", "8.13") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_platform_collector_failure_forces_nonzero(worker) -> None:
|
||||
assert worker.combined_result_exit_code(
|
||||
@@ -78,6 +98,30 @@ def test_formal_base_upsert_accepts_verified_response_record_id(monkeypatch, wor
|
||||
assert message == "record_id=rec12345678"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_formal_base_update_reuses_valid_requested_record_id(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"ok": True, "data": {"updated": True}}),
|
||||
stderr="",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message = worker.upsert_record(
|
||||
"bas_test",
|
||||
"tbl_test",
|
||||
{"时间": "8.4", "男性比例": 0.5},
|
||||
record_id="rec_existing123",
|
||||
)
|
||||
|
||||
assert ok is True
|
||||
assert message == "record_id=rec_existing123"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_acceptance_skip_is_not_formal_success(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: True)
|
||||
@@ -265,6 +309,21 @@ def test_zero_distributions_are_not_valid_persona() -> None:
|
||||
) is False
|
||||
|
||||
|
||||
def test_dy_persistent_profile_is_not_overwritten_by_exported_cookies() -> None:
|
||||
assert dy_collector.persistent_profile_launch_options("bound-profile") == {
|
||||
"user_data_dir": "bound-profile"
|
||||
}
|
||||
|
||||
|
||||
def test_dy_profileless_fallback_can_restore_exported_cookies() -> None:
|
||||
cookies = [{"name": "session", "value": "new", "domain": ".jinritemai.com"}]
|
||||
|
||||
assert dy_collector.cookie_fallback_launch_options(cookies) == {
|
||||
"cookies": cookies
|
||||
}
|
||||
assert dy_collector.cookie_fallback_launch_options(None) == {}
|
||||
|
||||
|
||||
def test_dmp_skips_are_not_valid_style_results() -> None:
|
||||
styles = {"款A": ["123"]}
|
||||
assert dmp.records_exit_code(
|
||||
@@ -317,3 +376,24 @@ def test_tm_collector_command_redacts_credentials() -> None:
|
||||
assert "secret-user" not in rendered
|
||||
assert "secret-pass" not in rendered
|
||||
assert rendered.count("***") == 2
|
||||
|
||||
|
||||
def test_jd_persona_login_home_navigation_does_not_wait_for_networkidle() -> None:
|
||||
project_root = Path(__file__).resolve().parents[3]
|
||||
source_path = (
|
||||
project_root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "vendors"
|
||||
/ "jd-data-flow"
|
||||
/ "jd_data_collector.py"
|
||||
)
|
||||
source = source_path.read_text(encoding="utf-8")
|
||||
|
||||
step_login = source[source.index("def step_login") :]
|
||||
home_block = step_login[step_login.index("shop.jd.com/jdm/home") :]
|
||||
assert 'wait_until="networkidle"' not in home_block[:400]
|
||||
assert 'wait_until="domcontentloaded"' in home_block[:400]
|
||||
assert "timeout=60_000" in home_block[:400]
|
||||
|
||||
@@ -15,6 +15,46 @@ class PersonaLauncherTests(unittest.TestCase):
|
||||
def test_all_platforms_success_makes_launcher_success(self):
|
||||
self.assertEqual(run_daily_persona.combined_exit_code({"tm": 0, "dy": 0, "jd": 0}), 0)
|
||||
|
||||
def test_partial_platform_results_are_accepted_when_real_rows_were_written(self):
|
||||
results = {"tm": 2, "dy": 2, "jd": 2}
|
||||
summaries = {
|
||||
"tm": {"ok": 31, "skipped": 0, "failed": 4, "total": 35},
|
||||
"dy": {"ok": 2, "skipped": 0, "failed": 30, "total": 31},
|
||||
"jd": {"ok": 18, "skipped": 0, "failed": 13, "total": 31},
|
||||
}
|
||||
|
||||
self.assertEqual(run_daily_persona.combined_exit_code(results, summaries), 0)
|
||||
|
||||
def test_partial_platform_without_real_rows_still_fails(self):
|
||||
results = {"tm": 2, "dy": 2, "jd": 2}
|
||||
summaries = {
|
||||
"tm": {"ok": 31},
|
||||
"dy": {"ok": 0},
|
||||
"jd": {"ok": 18},
|
||||
}
|
||||
|
||||
self.assertEqual(run_daily_persona.combined_exit_code(results, summaries), 2)
|
||||
|
||||
def test_result_summary_is_scoped_to_current_log_segment(self):
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
with TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "persona.log"
|
||||
path.write_text(
|
||||
"[DONE] ok=0 permission_skipped=0 failed=31 total=31\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
offset = path.stat().st_size
|
||||
with path.open("a", encoding="utf-8") as stream:
|
||||
stream.write(
|
||||
"[DONE] ok=18 permission_skipped=0 failed=13 total=31\n"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
run_daily_persona.read_result_summary(path, offset),
|
||||
{"ok": 18, "skipped": 0, "failed": 13, "total": 31},
|
||||
)
|
||||
|
||||
def test_launcher_passes_business_date_to_each_worker(self):
|
||||
command = run_daily_persona.build_child_command(
|
||||
Path("collect_persona_to_bitable.py"),
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -11,7 +10,7 @@ import lark_cli_runtime
|
||||
import market_rank_hermes_notification as market_notification
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
|
||||
def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path:
|
||||
@@ -103,39 +102,38 @@ def test_product_shared_lark_cli_allows_safe_profile_override(monkeypatch) -> No
|
||||
]
|
||||
|
||||
|
||||
def test_decline_hermes_prompt_forces_wang_yunlong(
|
||||
def test_decline_lark_user_send_forces_acceptance_recipient(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(decline, "HERMES_API_KEY", "test-only")
|
||||
captured = {}
|
||||
|
||||
class Response:
|
||||
def __enter__(self):
|
||||
return self
|
||||
def fake_send(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return json.dumps({"choices": [{"message": {"content": "ok"}}]}).encode()
|
||||
|
||||
def fake_urlopen(request, timeout):
|
||||
captured["body"] = json.load(io.BytesIO(request.data))
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr(decline.urllib.request, "urlopen", fake_urlopen)
|
||||
decline.notify_hermes(
|
||||
[{"style_name": "test", "platform": "tm"}],
|
||||
monkeypatch.setattr(decline, "send_lark_bot_message", fake_send)
|
||||
decline.send_decline_notification(
|
||||
[
|
||||
{
|
||||
"style_name": "test",
|
||||
"platform": "tm",
|
||||
"seg1_sales": 30,
|
||||
"seg2_sales": 20,
|
||||
"seg3_sales": 10,
|
||||
"drop_pct_1to2": 33.3,
|
||||
"drop_pct_2to3": 50.0,
|
||||
"drop_pct_1to3": 66.7,
|
||||
}
|
||||
],
|
||||
"2026-08-01",
|
||||
9,
|
||||
openid="ou_someone_else",
|
||||
)
|
||||
|
||||
encoded = json.dumps(captured["body"], ensure_ascii=False)
|
||||
assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded
|
||||
assert "ou_someone_else" not in encoded
|
||||
assert captured["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert captured["profile"] == "hermes-analyzer"
|
||||
|
||||
|
||||
def test_market_rank_notification_forces_wang_yunlong(
|
||||
@@ -175,14 +173,14 @@ def test_market_rank_notification_forces_wang_yunlong(
|
||||
lambda *args, **kwargs: False,
|
||||
)
|
||||
|
||||
def fake_post(payload):
|
||||
captured["payload"] = payload
|
||||
def fake_send(**kwargs):
|
||||
captured["send"] = kwargs
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
def fake_archive(conn, record):
|
||||
captured["record"] = record
|
||||
|
||||
monkeypatch.setattr(market_notification, "post_hermes", fake_post)
|
||||
monkeypatch.setattr(market_notification, "send_lark_bot_message", fake_send)
|
||||
monkeypatch.setattr(
|
||||
market_notification,
|
||||
"upsert_market_rank_notification",
|
||||
@@ -194,11 +192,13 @@ def test_market_rank_notification_forces_wang_yunlong(
|
||||
recipient_open_id="ou_someone_else",
|
||||
)
|
||||
|
||||
encoded = json.dumps(captured["payload"], ensure_ascii=False)
|
||||
assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded
|
||||
assert "ou_someone_else" not in encoded
|
||||
assert result["recipient_open_id"] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert (
|
||||
captured["send"]["user_id"]
|
||||
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
)
|
||||
assert captured["send"]["profile"] == "hermes-analyzer"
|
||||
assert result["recipient_open_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert (
|
||||
captured["record"]["recipient_open_id"]
|
||||
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import db as product_db
|
||||
|
||||
|
||||
class _Cursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
class _Connection:
|
||||
def cursor(self):
|
||||
return _Cursor()
|
||||
|
||||
|
||||
def test_product_daily_upsert_keeps_existing_ad_metrics(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows):
|
||||
captured["sql"] = sql
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
|
||||
written = product_db.upsert_product_daily_metrics(
|
||||
_Connection(),
|
||||
[{
|
||||
"platform": "tm",
|
||||
"stat_date": date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "测试商品",
|
||||
"raw_data": {"paid_amount": "100.00"},
|
||||
"source_file": "商品经营日报.xls",
|
||||
}],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
assert (
|
||||
"COALESCE(product_daily_metrics.raw_data, '{}'::jsonb)\n"
|
||||
" || EXCLUDED.raw_data"
|
||||
) in str(captured["sql"])
|
||||
@@ -8,6 +8,7 @@ import backfill_collect
|
||||
import backfill_one_day
|
||||
import collect_dy_persona_to_bitable as dy_persona_worker
|
||||
import collect_persona_to_bitable as tm_persona_worker
|
||||
import dy_product_scraping as dy_product
|
||||
import orchestrate_daily_collection as daily
|
||||
import orchestrate_market_rank_collection as market_rank
|
||||
import orchestrate_review_collection as reviews
|
||||
@@ -115,6 +116,40 @@ def test_daily_run_step_rebinds_to_command_target(monkeypatch) -> None:
|
||||
_assert_child_binding(observed["environment"])
|
||||
|
||||
|
||||
def test_daily_command_logging_redacts_sensitive_values() -> None:
|
||||
command = [
|
||||
"python",
|
||||
"collector.py",
|
||||
"--account",
|
||||
"secret-user",
|
||||
"--password=placeholder",
|
||||
"--api-key",
|
||||
"example",
|
||||
"--target-date",
|
||||
"2026-08-04",
|
||||
]
|
||||
|
||||
rendered = daily.redacted_command(command)
|
||||
|
||||
assert "secret-user" not in rendered
|
||||
assert "placeholder" not in rendered
|
||||
assert "example" not in rendered
|
||||
assert rendered.count("[REDACTED]") == 3
|
||||
assert "2026-08-04" in rendered
|
||||
|
||||
|
||||
def test_daily_saved_cookie_binding_refreshes_existing_profile(tmp_path) -> None:
|
||||
profile = tmp_path / "profile"
|
||||
cookie_db = profile / "Default" / "Network" / "Cookies"
|
||||
cookie_db.parent.mkdir(parents=True)
|
||||
cookie_db.write_bytes(b"x" * 5000)
|
||||
cookies = [{"name": "session", "value": "secret", "domain": ".jinritemai.com"}]
|
||||
|
||||
assert dy_product._saved_cookie_launch_options(str(profile), cookies) == {
|
||||
"cookies": cookies
|
||||
}
|
||||
|
||||
|
||||
def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -13,12 +13,20 @@ def _workflow(workflow_id: str) -> dict:
|
||||
return next(item for item in catalog["workflows"] if item["id"] == workflow_id)
|
||||
|
||||
|
||||
def _schedule(workflow_id: str) -> dict:
|
||||
schedules = json.loads(
|
||||
(REPOSITORY_ROOT / "config" / "schedules.json").read_text(encoding="utf-8")
|
||||
)
|
||||
return next(item for item in schedules["schedules"] if item["workflow_id"] == workflow_id)
|
||||
|
||||
|
||||
def test_product_workflows_forward_the_selected_business_date() -> None:
|
||||
persona = _workflow("product.persona.daily")["execution"]["steps"][0]
|
||||
style = _workflow("product.style_analysis.interval")["execution"]["steps"][0]
|
||||
main_image = _workflow("product.main_image.weekly")["execution"]["steps"]
|
||||
market_rank = _workflow("product.market_rank")["execution"]["steps"][0]
|
||||
monthly_sales = _workflow("product.sales_sheet.daily")["execution"]["steps"][0]
|
||||
all_shop_daily = _workflow("product.erp_all_shop_daily")["execution"]["steps"][0]
|
||||
|
||||
assert persona["args"] == ["--date", "{business_date}"]
|
||||
assert style["args"][-2:] == ["--end-date", "{business_date}"]
|
||||
@@ -33,3 +41,46 @@ def test_product_workflows_forward_the_selected_business_date() -> None:
|
||||
"--execute",
|
||||
"--allow-missing-erp-as-zero",
|
||||
]
|
||||
assert all_shop_daily["args"] == [
|
||||
"--from",
|
||||
"{business_date}",
|
||||
"--to",
|
||||
"{business_date}",
|
||||
"--execute",
|
||||
"--force",
|
||||
]
|
||||
|
||||
|
||||
def test_ecommerce_costs_workflow_runs_three_platforms_in_parallel() -> None:
|
||||
workflow = _workflow("product.ecommerce_costs.daily")
|
||||
steps = workflow["execution"]["steps"]
|
||||
by_id = {step["id"]: step for step in steps}
|
||||
|
||||
assert [step["id"] for step in steps] == [
|
||||
"tmall_download",
|
||||
"tmall_import",
|
||||
"jd_download",
|
||||
"jd_import",
|
||||
"douyin_download",
|
||||
"douyin_import",
|
||||
]
|
||||
for platform, collector, importer in (
|
||||
("tmall", "collect_tmall_wanxiang_ads.py", "import_tmall_product_ads.py"),
|
||||
("jd", "collect_jd_ad_costs.py", "import_jd_ad_costs.py"),
|
||||
("douyin", "collect_douyin_qianchuan_ads.py", "import_douyin_qianchuan_ads.py"),
|
||||
):
|
||||
download = by_id[f"{platform}_download"]
|
||||
import_step = by_id[f"{platform}_import"]
|
||||
assert download["entry"] == collector
|
||||
assert download["args"] == ["--date", "{business_date}", "--headless"]
|
||||
assert download.get("depends_on", []) == []
|
||||
assert import_step["entry"] == importer
|
||||
assert import_step["args"] == ["--date", "{business_date}", "--execute"]
|
||||
assert import_step["depends_on"] == [f"{platform}_download"]
|
||||
|
||||
assert "点击下载报表" in by_id["tmall_download"]["description"]
|
||||
assert "当前时间" in by_id["jd_import"]["description"]
|
||||
assert "SKU" in by_id["jd_import"]["description"]
|
||||
schedule = _schedule("product.ecommerce_costs.daily")
|
||||
assert schedule["at"] == "10:00"
|
||||
assert schedule["business_date_offset_days"] == -1
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import date
|
||||
@@ -7,7 +8,7 @@ from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import analyze_style_with_hermes as analysis
|
||||
import analyze_style as analysis
|
||||
import orchestrate_daily_collection as orchestrator
|
||||
from config.style_config_loader import StyleConfigLoader
|
||||
|
||||
@@ -140,6 +141,60 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
for heading in headings:
|
||||
self.assertIn(heading, analysis.SYSTEM)
|
||||
|
||||
def test_system_prompt_and_validator_require_pure_chinese_report(self):
|
||||
self.assertIn("纯中文硬约束", analysis.SYSTEM)
|
||||
self.assertIn("不得出现英文单词", analysis.SYSTEM)
|
||||
|
||||
valid_report = "\n".join([
|
||||
*analysis.REPORT_HEADINGS,
|
||||
*analysis.REPORT_SUBHEADINGS,
|
||||
"**P0**", "**重点**", "**重点**", "**重点**", "**重点**",
|
||||
"**重点**", "**重点**", "**重点**",
|
||||
])
|
||||
normalized, error = analysis.validate_report_markdown(valid_report)
|
||||
self.assertIsNone(error)
|
||||
self.assertIn("**P0**", normalized)
|
||||
|
||||
english_report = valid_report.replace("**P0**", "English text")
|
||||
_, error = analysis.validate_report_markdown(english_report)
|
||||
self.assertEqual(error, "MiniMax 报告包含英文内容,必须改为纯中文")
|
||||
|
||||
def test_validator_removes_separate_reasoning_before_language_check(self):
|
||||
valid_report = "\n".join([
|
||||
"<think>English reasoning</think>",
|
||||
*analysis.REPORT_HEADINGS,
|
||||
*analysis.REPORT_SUBHEADINGS,
|
||||
"**P0**", "**重点**", "**重点**", "**重点**", "**重点**",
|
||||
"**重点**", "**重点**", "**重点**",
|
||||
])
|
||||
normalized, error = analysis.validate_report_markdown(valid_report)
|
||||
self.assertIsNone(error)
|
||||
self.assertNotIn("English reasoning", normalized)
|
||||
|
||||
def test_localization_fallback_converts_source_english_to_chinese(self):
|
||||
report = "英文前缀\nSwitchLite、SKU、IP、ROI、A/B、Apollo、notes_recent_top8、**P0**"
|
||||
localized = analysis._localize_report_latin(report)
|
||||
|
||||
latin_tokens = set(re.findall(r"[A-Za-z][A-Za-z0-9_-]*", localized))
|
||||
self.assertEqual(latin_tokens, {"P0"})
|
||||
self.assertIn("游戏主机", localized)
|
||||
self.assertIn("规格", localized)
|
||||
self.assertIn("地域", localized)
|
||||
self.assertIn("投入产出比", localized)
|
||||
self.assertIn("甲/乙", localized)
|
||||
self.assertIn("**P0**", localized)
|
||||
|
||||
def test_direct_checkpoint_namespace_isolated_for_language_gate_revision(self):
|
||||
self.assertEqual(analysis.CHECKPOINT_NAMESPACE, "direct_minimax_v2")
|
||||
self.assertIn("direct_minimax_v2", str(analysis.StyleAnalysisCheckpointSpec.for_report(
|
||||
"测试款",
|
||||
date(2026, 8, 1),
|
||||
date(2026, 8, 3),
|
||||
3,
|
||||
"测试款 单款式多维度分析 2026-08-01~2026-08-03",
|
||||
"# 内容",
|
||||
).path()))
|
||||
|
||||
def test_plain_seven_section_titles_are_normalized_to_markdown(self):
|
||||
report = """盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)
|
||||
|
||||
@@ -164,6 +219,99 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
self.assertTrue(normalized.startswith("# 盖亚斜挎"))
|
||||
self.assertTrue(analysis.has_exact_report_headings(normalized))
|
||||
|
||||
def test_number_variants_and_bare_total_title_are_canonicalized(self):
|
||||
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
|
||||
for index, heading in enumerate(analysis.REPORT_HEADINGS):
|
||||
subject = heading.split("、", 1)[1]
|
||||
raw_heading = heading
|
||||
if index == 4:
|
||||
raw_heading = f"## 六、{subject}"
|
||||
elif index == 6:
|
||||
raw_heading = "## 总评"
|
||||
blocks.extend([
|
||||
raw_heading,
|
||||
*analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3],
|
||||
"**重点**",
|
||||
])
|
||||
|
||||
normalized = analysis.normalize_report_markdown("\n".join(blocks))
|
||||
|
||||
self.assertEqual(
|
||||
[
|
||||
line.strip()
|
||||
for line in normalized.splitlines()
|
||||
if line.strip().startswith("## ")
|
||||
],
|
||||
analysis.REPORT_HEADINGS,
|
||||
)
|
||||
|
||||
def test_priority_action_heading_is_inserted_for_unheaded_priority_lines(self):
|
||||
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
|
||||
for index, heading in enumerate(analysis.REPORT_HEADINGS[:-1]):
|
||||
blocks.extend([
|
||||
heading,
|
||||
*analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3],
|
||||
"**重点**",
|
||||
])
|
||||
blocks.extend([
|
||||
analysis.REPORT_HEADINGS[-1],
|
||||
"### 核心结论",
|
||||
"**当前核心问题是转化偏低。**",
|
||||
"### 转化链路",
|
||||
"曝光到访客再到成交。",
|
||||
"- **P0**:48小时内完成整改。",
|
||||
])
|
||||
|
||||
normalized = analysis.normalize_report_markdown("\n".join(blocks))
|
||||
|
||||
self.assertTrue(analysis.has_clear_report_hierarchy(normalized))
|
||||
self.assertIn("### 优先级行动", normalized)
|
||||
|
||||
def test_minimax_adapter_disables_tools_and_uses_extended_timeout(self):
|
||||
captured = {}
|
||||
|
||||
def fake_call(system, user, **kwargs):
|
||||
captured.update(system=system, user=user, kwargs=kwargs)
|
||||
return "报告", {}
|
||||
|
||||
with (
|
||||
patch.object(analysis, "call_direct_llm", fake_call),
|
||||
):
|
||||
content, _ = analysis.call_minimax("system", "user")
|
||||
|
||||
self.assertEqual(content, "报告")
|
||||
self.assertEqual(captured["system"], "system")
|
||||
self.assertEqual(captured["user"], "user")
|
||||
self.assertEqual(captured["kwargs"]["model"], analysis.MINIMAX_MODEL)
|
||||
self.assertEqual(captured["kwargs"]["temperature"], 0.3)
|
||||
self.assertEqual(captured["kwargs"]["timeout"], analysis.MINIMAX_TIMEOUT_SECONDS)
|
||||
self.assertEqual(captured["kwargs"]["max_tokens"], analysis.MINIMAX_MAX_TOKENS)
|
||||
self.assertEqual(captured["kwargs"]["thinking_mode"], "disabled")
|
||||
|
||||
def test_invalid_report_is_regenerated_once_before_failure(self):
|
||||
valid_lines = []
|
||||
for index, heading in enumerate(analysis.REPORT_HEADINGS):
|
||||
valid_lines.extend([
|
||||
heading,
|
||||
*analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3],
|
||||
"**重点**",
|
||||
])
|
||||
valid_report = "\n".join(valid_lines)
|
||||
|
||||
with (
|
||||
patch.object(analysis, "MINIMAX_FORMAT_RETRIES", 1),
|
||||
patch.object(
|
||||
analysis,
|
||||
"call_minimax",
|
||||
side_effect=[("不完整报告", {"attempt": 1}), (valid_report, {"attempt": 2})],
|
||||
) as call,
|
||||
):
|
||||
report, usage = analysis.generate_validated_report("user prompt")
|
||||
|
||||
self.assertEqual(report, analysis.normalize_report_markdown(valid_report))
|
||||
self.assertEqual(usage, {"attempt": 2})
|
||||
self.assertEqual(call.call_count, 2)
|
||||
|
||||
def test_report_gets_clear_subheadings_and_key_emphasis(self):
|
||||
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
|
||||
for heading in analysis.REPORT_HEADINGS[:-1]:
|
||||
@@ -190,6 +338,32 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
self.assertIn("**P0**", normalized)
|
||||
self.assertIn("**24小时内**", normalized)
|
||||
|
||||
def test_report_normalization_is_idempotent_for_checkpoint_recovery(self):
|
||||
blocks = ["逐星GT 单款式多维度分析报告 (2026-08-01~2026-08-03)"]
|
||||
for heading in analysis.REPORT_HEADINGS[:-1]:
|
||||
blocks.extend([
|
||||
heading,
|
||||
"### 数据快照",
|
||||
"**三日访客 100**。",
|
||||
"### 核心判断",
|
||||
"**当前转化偏低。**",
|
||||
"### 优化建议",
|
||||
"P1,3天内完成优化。",
|
||||
])
|
||||
blocks.extend([
|
||||
analysis.REPORT_HEADINGS[-1],
|
||||
"### 核心结论",
|
||||
"**当前核心问题是转化偏低。**",
|
||||
"### 转化链路",
|
||||
"曝光到访客再到成交。",
|
||||
"### 优先级行动",
|
||||
"- **P0(48小时内)**:完成整改。",
|
||||
])
|
||||
|
||||
normalized = analysis.normalize_report_markdown("\n".join(blocks))
|
||||
|
||||
self.assertEqual(analysis.normalize_report_markdown(normalized), normalized)
|
||||
|
||||
def test_extra_helpful_h3_does_not_reject_complete_required_hierarchy(self):
|
||||
required = "\n".join(analysis.REPORT_SUBHEADINGS)
|
||||
report = required.replace(
|
||||
@@ -212,6 +386,78 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(missing, [date(2026, 7, 15)])
|
||||
|
||||
def test_batch_style_selection_requires_a_complete_calendar_window(self):
|
||||
class Cursor:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
self.sql = ""
|
||||
self.params = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.sql = sql
|
||||
self.params = params
|
||||
|
||||
def fetchall(self):
|
||||
return self.rows
|
||||
|
||||
class Connection:
|
||||
def __init__(self, cursor):
|
||||
self.cursor_value = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_value
|
||||
|
||||
cursor = Cursor([("完整款", 100)])
|
||||
with patch.object(analysis, "get_conn", return_value=Connection(cursor)):
|
||||
styles = analysis.fetch_styles_with_data(date(2026, 7, 15), 3)
|
||||
|
||||
self.assertEqual(styles, ["完整款"])
|
||||
self.assertIn("HAVING COUNT(DISTINCT metric_date) = %s", cursor.sql)
|
||||
self.assertEqual(
|
||||
cursor.params,
|
||||
(date(2026, 7, 13), date(2026, 7, 15), 3),
|
||||
)
|
||||
|
||||
def test_incomplete_batch_styles_are_reported_with_present_dates(self):
|
||||
cursor = type(
|
||||
"Cursor",
|
||||
(),
|
||||
{
|
||||
"__enter__": lambda self: self,
|
||||
"__exit__": lambda self, exc_type, exc, tb: False,
|
||||
"execute": lambda self, sql, params: setattr(self, "params", params),
|
||||
"fetchall": lambda self: [("云栖", [date(2026, 7, 15)])],
|
||||
},
|
||||
)()
|
||||
connection = type(
|
||||
"Connection",
|
||||
(),
|
||||
{
|
||||
"__enter__": lambda self: self,
|
||||
"__exit__": lambda self, exc_type, exc, tb: False,
|
||||
"cursor": lambda self: cursor,
|
||||
},
|
||||
)()
|
||||
|
||||
with patch.object(analysis, "get_conn", return_value=connection):
|
||||
incomplete = analysis.fetch_styles_with_incomplete_data(
|
||||
date(2026, 7, 15), 3,
|
||||
)
|
||||
|
||||
self.assertEqual(incomplete, {"云栖": [date(2026, 7, 15)]})
|
||||
|
||||
|
||||
class LarkEnvironmentTests(unittest.TestCase):
|
||||
def test_legacy_lark_config_is_used_when_default_is_missing(self):
|
||||
@@ -607,10 +853,6 @@ class TargetBaseWriteTests(unittest.TestCase):
|
||||
{"name": "时间", "type": "select"},
|
||||
{"name": "笔记分析汇总", "type": "text"},
|
||||
]}},
|
||||
{"data": {"field": {
|
||||
"id": "fld_time", "name": "时间", "type": "select",
|
||||
"multiple": False, "options": [{"name": "2026-07-13~07-15"}],
|
||||
}}},
|
||||
{"data": {
|
||||
"data": [[['2026-07-13~07-15'], "old link"]],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
@@ -627,33 +869,117 @@ class TargetBaseWriteTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(record_id, "rec_existing")
|
||||
upsert_args = run.call_args_list[3].args[0]
|
||||
upsert_args = run.call_args_list[2].args[0]
|
||||
self.assertIn("--record-id", upsert_args)
|
||||
self.assertIn("rec_existing", upsert_args)
|
||||
payload = json.loads(upsert_args[upsert_args.index("--json") + 1])
|
||||
self.assertIn("笔记分析汇总", payload)
|
||||
|
||||
def test_missing_period_is_appended_without_dropping_existing_options(self):
|
||||
def test_missing_period_is_created_without_rewriting_select_field(self):
|
||||
responses = [
|
||||
{"data": {"field": {
|
||||
"id": "fld_time", "name": "时间", "type": "select",
|
||||
"multiple": False, "options": [{"name": "7月第二周", "hue": "Blue"}],
|
||||
}}},
|
||||
{"ok": True},
|
||||
{"data": {"fields": [
|
||||
{"name": "时间", "type": "select"},
|
||||
{"name": "笔记分析汇总", "type": "text"},
|
||||
]}},
|
||||
{"data": {
|
||||
"data": [],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": [],
|
||||
}},
|
||||
{"ok": True, "data": {"record_id": "rec_created"}},
|
||||
]
|
||||
with patch.object(analysis, "run_lark_cli", side_effect=responses) as run:
|
||||
analysis.ensure_select_option(
|
||||
record_id = analysis.write_analysis_link_to_base(
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"时间", "2026-07-13~07-15", "盖亚斜挎",
|
||||
"2026-08-01~08-03",
|
||||
"https://example.feishu.cn/docx/doc",
|
||||
"盖亚斜挎",
|
||||
)
|
||||
|
||||
update_args = run.call_args_list[1].args[0]
|
||||
definition = json.loads(update_args[update_args.index("--json") + 1])
|
||||
self.assertEqual(record_id, "rec_created")
|
||||
self.assertEqual(
|
||||
[option["name"] for option in definition["options"]],
|
||||
["7月第二周", "2026-07-13~07-15"],
|
||||
[call.args[0][1] for call in run.call_args_list],
|
||||
["+field-list", "+record-list", "+record-upsert"],
|
||||
)
|
||||
self.assertIn("--yes", update_args)
|
||||
payload_args = run.call_args_list[2].args[0]
|
||||
payload = json.loads(payload_args[payload_args.index("--json") + 1])
|
||||
self.assertEqual(payload["时间"], "2026-08-01~08-03")
|
||||
|
||||
def test_timeout_reconciles_remote_upsert_without_second_write(self):
|
||||
document_url = "https://example.feishu.cn/docx/doc"
|
||||
responses = [
|
||||
{"data": {"fields": [
|
||||
{"name": "时间", "type": "select"},
|
||||
{"name": "笔记分析汇总", "type": "text"},
|
||||
]}},
|
||||
{"data": {
|
||||
"data": [],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": [],
|
||||
}},
|
||||
RuntimeError(
|
||||
'lark-cli failed (exit=4): {"error":{"type":"network",'
|
||||
'"subtype":"timeout"}}'
|
||||
),
|
||||
{"data": {
|
||||
"data": [[
|
||||
["2026-08-01~08-03"],
|
||||
f"[盖亚斜挎 三日单品分析]({document_url})",
|
||||
]],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": ["rec_reconciled"],
|
||||
}},
|
||||
]
|
||||
with (
|
||||
patch.object(analysis, "run_lark_cli", side_effect=responses) as run,
|
||||
patch.object(analysis, "BASE_WRITE_RECONCILE_DELAYS_SECONDS", (0,)),
|
||||
):
|
||||
record_id = analysis.write_analysis_link_to_base(
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"2026-08-01~08-03",
|
||||
document_url,
|
||||
"盖亚斜挎",
|
||||
)
|
||||
|
||||
self.assertEqual(record_id, "rec_reconciled")
|
||||
self.assertEqual(
|
||||
[call.args[0][1] for call in run.call_args_list],
|
||||
["+field-list", "+record-list", "+record-upsert", "+record-list"],
|
||||
)
|
||||
self.assertEqual(run.call_args_list[2].kwargs["timeout"], 180)
|
||||
|
||||
def test_timeout_reconciliation_requires_the_expected_report_link(self):
|
||||
responses = [
|
||||
{"data": {"fields": [
|
||||
{"name": "时间", "type": "select"},
|
||||
{"name": "笔记分析汇总", "type": "text"},
|
||||
]}},
|
||||
{"data": {
|
||||
"data": [],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": [],
|
||||
}},
|
||||
RuntimeError("API call failed: server time out error"),
|
||||
{"data": {
|
||||
"data": [[
|
||||
["2026-08-01~08-03"],
|
||||
"[盖亚斜挎 三日单品分析](https://example.feishu.cn/docx/old)",
|
||||
]],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": ["rec_old"],
|
||||
}},
|
||||
]
|
||||
with (
|
||||
patch.object(analysis, "run_lark_cli", side_effect=responses),
|
||||
patch.object(analysis, "BASE_WRITE_RECONCILE_DELAYS_SECONDS", (0,)),
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "server time out error"):
|
||||
analysis.write_analysis_link_to_base(
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"2026-08-01~08-03",
|
||||
"https://example.feishu.cn/docx/new",
|
||||
"盖亚斜挎",
|
||||
)
|
||||
|
||||
|
||||
class OrchestratorTests(unittest.TestCase):
|
||||
@@ -663,7 +989,7 @@ class OrchestratorTests(unittest.TestCase):
|
||||
orchestrator.run_style_analysis(args, "2026-07-15", failed=None)
|
||||
|
||||
command = run.call_args.args[1]
|
||||
self.assertIn("analyze_style_with_hermes.py", [Path(item).name for item in command])
|
||||
self.assertIn("analyze_style.py", [Path(item).name for item in command])
|
||||
self.assertIn("--all-styles", command)
|
||||
self.assertEqual(command[command.index("--days") + 1], "3")
|
||||
self.assertEqual(command[command.index("--min-interval-days") + 1], "3")
|
||||
|
||||
@@ -39,25 +39,32 @@ def test_non_tm_erp_code_is_not_used_when_tm_code_is_missing() -> None:
|
||||
assert "天猫分组未填写" in output.getvalue()
|
||||
|
||||
|
||||
def test_exact_style_field_wins_over_style_content_prefix(monkeypatch) -> None:
|
||||
loader = StyleConfigLoader()
|
||||
monkeypatch.setattr(
|
||||
loader,
|
||||
"_run_lark",
|
||||
lambda _args: {
|
||||
"data": {
|
||||
"fields": ["款式内容", "款式", "ERP款式编码", "平台"],
|
||||
"data": [["营销文案", "盖世m1", "10416,10455", ["天猫"]]],
|
||||
def test_database_row_uses_explicit_style_name_not_style_content() -> None:
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"style_content": "营销文案",
|
||||
"style_name": "盖世m1",
|
||||
"erp_codes": ["10416", "10455"],
|
||||
"platform": "天猫",
|
||||
}
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
records = loader._load_records_from_lark()
|
||||
records = loader._load_records_from_database()
|
||||
|
||||
assert records == [
|
||||
{
|
||||
"style": "盖世m1",
|
||||
"erp_codes": "10416,10455",
|
||||
"erp_codes": ["10416", "10455"],
|
||||
"platform": "天猫",
|
||||
"item_ids": [],
|
||||
"brand": "",
|
||||
"sales_bitable_url": "",
|
||||
"main_image_bitable_url": "",
|
||||
"persona_bitable_url": "",
|
||||
"dy_persona_bitable_url": "",
|
||||
"jd_persona_bitable_url": "",
|
||||
"style_analysis_bitable_url": "",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -14,6 +14,7 @@ from collect_sycm_market_rank import (
|
||||
collect_all_rank_pages,
|
||||
create_feishu_doc,
|
||||
detect_risk_control_text,
|
||||
dismiss_floating_overlays,
|
||||
ensure_market_rank_login,
|
||||
group_products_by_category,
|
||||
inspect_feishu_batch_tables,
|
||||
@@ -49,6 +50,15 @@ SAMPLE_HTML = """
|
||||
|
||||
|
||||
class SycmMarketRankTests(unittest.TestCase):
|
||||
def test_floating_overlays_are_removed_before_pagination(self):
|
||||
page = MagicMock()
|
||||
page.evaluate.return_value = 3
|
||||
|
||||
self.assertEqual(dismiss_floating_overlays(page), 3)
|
||||
script = page.evaluate.call_args.args[0]
|
||||
self.assertIn("scenario-widget", script)
|
||||
self.assertIn("aes-survey-hanging", script)
|
||||
|
||||
def test_expired_login_without_credentials_fails_clearly(self):
|
||||
with (
|
||||
patch("collect_sycm_market_rank.sycm.ACCOUNT", ""),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
@@ -17,6 +18,7 @@ from gyxx_flow.modules.product_commerce.sync_monthly_sales_sheet import (
|
||||
MonthlySalesConfig,
|
||||
ProductTarget,
|
||||
SheetInfo,
|
||||
_reconfigure_stream,
|
||||
_write_rows,
|
||||
build_parser,
|
||||
choose_template,
|
||||
@@ -43,6 +45,17 @@ def test_month_contract_uses_natural_month_and_existing_title_convention() -> No
|
||||
assert month_sheet_title(month) == "26年8月"
|
||||
|
||||
|
||||
def test_reconfigure_stream_switches_gbk_stream_to_utf8() -> None:
|
||||
raw = io.BytesIO()
|
||||
stream = io.TextIOWrapper(raw, encoding="gbk", errors="strict")
|
||||
|
||||
_reconfigure_stream(stream)
|
||||
stream.write("商品\u200c名\n")
|
||||
stream.flush()
|
||||
|
||||
assert "商品\u200c名".encode("utf-8") in raw.getvalue()
|
||||
|
||||
|
||||
def test_erp_month_interval_uses_inclusive_natural_month_for_day_input() -> None:
|
||||
assert monthly_report_interval(parse_month("2026-07-02")) == (
|
||||
"2026-07-01",
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import taobao_sycm_collect as collect
|
||||
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def stub_collection_chrome(monkeypatch):
|
||||
"""Keep wrapper tests from starting a real browser on collector ports."""
|
||||
|
||||
monkeypatch.setattr(collect, "is_chrome_debug_running", lambda _port: False)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"launch_chrome_with_debug",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
|
||||
|
||||
def test_tm_identity_markers_require_the_complete_store_name(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS", raising=False)
|
||||
|
||||
assert collect is not None
|
||||
# The child collector owns the actual page identity check; import it here
|
||||
# so this regression test also guards its brand fallback markers.
|
||||
import taobao_sycm_products as products
|
||||
|
||||
monkeypatch.setattr(products, "BRAND", "ozko")
|
||||
monkeypatch.setattr(products, "ACCOUNT", "")
|
||||
assert products._login_identity_markers() == ("ozko旗舰店", "OZKO旗舰店")
|
||||
|
||||
monkeypatch.setattr(products, "BRAND", "光影行星")
|
||||
assert products._login_identity_markers() == ("光影行星旗舰店",)
|
||||
|
||||
|
||||
def test_dynamic_tm_brand_route_selects_the_requested_account_vault(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
class FakeCatalog:
|
||||
def environment_for_account(self, account_id, base_environment):
|
||||
calls.append(account_id)
|
||||
return {
|
||||
**base_environment,
|
||||
"GYXX_ACCOUNT_ID": account_id,
|
||||
"GYXX_ACCOUNT_COOKIE_FILE": f"{account_id}-cookies.json",
|
||||
"GYXX_ACCOUNT_STORAGE_STATE_FILE": f"{account_id}-state.json",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
collect.RuntimeIntegrationCatalog,
|
||||
"load_default",
|
||||
lambda **_kwargs: FakeCatalog(),
|
||||
)
|
||||
|
||||
environment = collect._account_environment(
|
||||
"tmall-ozko",
|
||||
{
|
||||
"GYXX_PROJECT_ROOT": "D:/project",
|
||||
"GYXX_DATA_ROOT": "D:/data",
|
||||
"KEEP": "yes",
|
||||
},
|
||||
)
|
||||
|
||||
assert calls == ["tmall-ozko"]
|
||||
assert environment["KEEP"] == "yes"
|
||||
assert environment["GYXX_ACCOUNT_ID"] == "tmall-ozko"
|
||||
assert environment["GYXX_ACCOUNT_COOKIE_FILE"] == "tmall-ozko-cookies.json"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extra_args", "expects_backfill"),
|
||||
[
|
||||
([], False),
|
||||
(["--backfill"], True),
|
||||
],
|
||||
)
|
||||
def test_each_tm_brand_is_injected_into_its_own_child_profile(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
extra_args: list[str],
|
||||
expects_backfill: bool,
|
||||
) -> None:
|
||||
profile_root = tmp_path / "tm-profile"
|
||||
environments: dict[str, dict[str, str]] = {}
|
||||
ports: dict[str, str] = {}
|
||||
launched_ports: list[int] = []
|
||||
child_scripts: list[str] = []
|
||||
child_commands: dict[str, list[str]] = {}
|
||||
binding_scripts: list[str] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_tm_style_id_map",
|
||||
lambda: {"极星pro": ["main-id"], "布谷": ["ozko-id"]},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_tm_brand_map",
|
||||
lambda: {"极星pro": "光影行星", "布谷": "ozko"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_daily_styles",
|
||||
lambda: {
|
||||
"tm": {
|
||||
"collectable": ["极星pro", "布谷"],
|
||||
"skipped": [],
|
||||
}
|
||||
},
|
||||
)
|
||||
def fake_child_environment(script, *_args):
|
||||
binding_scripts.append(Path(script).name)
|
||||
return {
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(profile_root),
|
||||
"GYXX_BROWSER_CDP_PORT": "22096",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(collect, "environment_for_child_script", fake_child_environment)
|
||||
monkeypatch.setattr(collect, "_kill_port_listeners", lambda _port: 0)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"launch_chrome_with_debug",
|
||||
lambda *_args, **kwargs: launched_ports.append(kwargs["debug_port"]) or True,
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
brand = command[command.index("--brand") + 1]
|
||||
environments[brand] = kwargs["env"]
|
||||
ports[brand] = kwargs["env"]["GYXX_BROWSER_CDP_PORT"]
|
||||
child_scripts.append(Path(command[1]).name)
|
||||
child_commands[brand] = command
|
||||
return SimpleNamespace(returncode=0)
|
||||
|
||||
monkeypatch.setattr(collect.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"taobao_sycm_collect.py",
|
||||
"--target-date",
|
||||
"2026-08-06",
|
||||
*extra_args,
|
||||
],
|
||||
)
|
||||
|
||||
assert collect.main() == 0
|
||||
assert child_scripts == ["taobao_sycm_products.py", "taobao_sycm_products.py"]
|
||||
assert binding_scripts == ["taobao_sycm_products.py"]
|
||||
for brand, expected_profile in {
|
||||
"光影行星": profile_root,
|
||||
"ozko": profile_root.parent / f"{profile_root.name}_ozko_tmall-ozko",
|
||||
}.items():
|
||||
command = child_commands[brand]
|
||||
assert ("--backfill" in command) is expects_backfill
|
||||
assert command[command.index("--cdp-url") + 1] == (
|
||||
"http://127.0.0.1:" + ports[brand]
|
||||
)
|
||||
assert command[command.index("--brand-user-data-dir") + 1] == str(
|
||||
expected_profile
|
||||
)
|
||||
assert environments["光影行星"]["GYXX_BROWSER_PROFILE_DIR"] == str(
|
||||
profile_root
|
||||
)
|
||||
assert environments["光影行星"]["TM_USER_DATA_DIR"] == str(profile_root)
|
||||
assert environments["ozko"]["GYXX_BROWSER_PROFILE_DIR"] == str(
|
||||
profile_root.parent / f"{profile_root.name}_ozko_tmall-ozko"
|
||||
)
|
||||
assert environments["ozko"]["TM_USER_DATA_DIR"] == str(
|
||||
profile_root.parent / f"{profile_root.name}_ozko_tmall-ozko"
|
||||
)
|
||||
assert ports["光影行星"] != ports["ozko"]
|
||||
assert sorted(launched_ports) == [22196, 22197]
|
||||
|
||||
|
||||
def test_cookie_recovery_marks_all_tm_brands_for_relogin(monkeypatch, tmp_path: Path) -> None:
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(collect, "get_tm_style_id_map", lambda: {"a": ["1"], "b": ["2"]})
|
||||
monkeypatch.setattr(collect, "get_tm_brand_map", lambda: {"a": "ozko", "b": "光影行星"})
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_daily_styles",
|
||||
lambda: {"tm": {"collectable": ["a", "b"], "skipped": []}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"environment_for_child_script",
|
||||
lambda *_args: {
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(tmp_path / "profile"),
|
||||
"GYXX_BROWSER_CDP_PORT": "22096",
|
||||
},
|
||||
)
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
calls.append(command[command.index("--brand") + 1])
|
||||
return SimpleNamespace(returncode=COOKIE_SKIP_EXIT_CODE)
|
||||
|
||||
monkeypatch.setattr(collect.subprocess, "run", fake_run)
|
||||
|
||||
assert collect.main(["--target-date", "2026-08-11", "--backfill"]) == COOKIE_SKIP_EXIT_CODE
|
||||
assert sorted(calls) == ["ozko", "光影行星"]
|
||||
|
||||
|
||||
def test_transient_brand_child_failure_is_retried_once(monkeypatch, tmp_path: Path) -> None:
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(collect, "get_tm_style_id_map", lambda: {"a": ["1"]})
|
||||
monkeypatch.setattr(collect, "get_tm_brand_map", lambda: {"a": "ozko"})
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_daily_styles",
|
||||
lambda: {"tm": {"collectable": ["a"], "skipped": []}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"environment_for_child_script",
|
||||
lambda *_args: {
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(tmp_path / "profile"),
|
||||
"GYXX_BROWSER_CDP_PORT": "22096",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(collect, "_kill_port_listeners", lambda _port: 0)
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
calls.append(command[command.index("--brand") + 1])
|
||||
return SimpleNamespace(returncode=1 if len(calls) == 1 else 0)
|
||||
|
||||
monkeypatch.setattr(collect.subprocess, "run", fake_run)
|
||||
|
||||
assert collect.main(["--target-date", "2026-08-11"]) == 0
|
||||
assert calls == ["ozko", "ozko"]
|
||||
|
||||
|
||||
def test_account_routed_brand_uses_a_fresh_collection_profile(monkeypatch, tmp_path: Path) -> None:
|
||||
profiles: list[str] = []
|
||||
monkeypatch.setattr(collect, "get_tm_style_id_map", lambda: {"布谷": ["1"]})
|
||||
monkeypatch.setattr(collect, "get_tm_brand_map", lambda: {"布谷": "ozko"})
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_daily_styles",
|
||||
lambda: {"tm": {"collectable": ["布谷"], "skipped": []}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"environment_for_child_script",
|
||||
lambda *_args: {
|
||||
"GYXX_PROJECT_ROOT": "D:/project",
|
||||
"GYXX_DATA_ROOT": "D:/data",
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(tmp_path / "legacy-profile"),
|
||||
"GYXX_BROWSER_CDP_PORT": "22096",
|
||||
"GYXX_BROWSER_CDP_URL": "http://127.0.0.1:22096",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"_account_environment",
|
||||
lambda account_id, environment: {
|
||||
**environment,
|
||||
"GYXX_ACCOUNT_ID": account_id,
|
||||
"GYXX_ACCOUNT_PROFILE_DIR": str(tmp_path / "account" / "profile"),
|
||||
"GYXX_ACCOUNT_CDP_PORT": "22097",
|
||||
"GYXX_ACCOUNT_CDP_URL": "http://127.0.0.1:22097",
|
||||
"GYXX_ACCOUNT_COLLECTION_CDP_PORT": "22143",
|
||||
"GYXX_ACCOUNT_COLLECTION_CDP_URL": "http://127.0.0.1:22143",
|
||||
"GYXX_ACCOUNT_COOKIE_FILE": "vault-cookies.json",
|
||||
"GYXX_ACCOUNT_STORAGE_STATE_FILE": "vault-state.json",
|
||||
"GYXX_ACCOUNT_REQUIRED_COOKIE_DOMAINS": "[\"taobao.com\"]",
|
||||
"GYXX_ACCOUNT_REQUIRED_COOKIE_NAMES": "[\"cookie2\"]",
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES": "[]",
|
||||
"GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS": "[\"ozko\"]",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(collect, "_kill_port_listeners", lambda _port: 0)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
profiles.append(kwargs["env"]["TM_USER_DATA_DIR"])
|
||||
assert kwargs["env"]["GYXX_BROWSER_CDP_PORT"] == "22143"
|
||||
assert kwargs["env"]["GYXX_BROWSER_CDP_URL"] == (
|
||||
"http://127.0.0.1:22143"
|
||||
)
|
||||
assert kwargs["env"]["AUTOFLOW_CDP_PORT"] == "22143"
|
||||
assert kwargs["env"]["GYXX_ACCOUNT_CDP_PORT"] == "22097"
|
||||
assert command[command.index("--cdp-url") + 1] == (
|
||||
"http://127.0.0.1:22143"
|
||||
)
|
||||
return SimpleNamespace(returncode=0)
|
||||
|
||||
monkeypatch.setattr(collect.subprocess, "run", fake_run)
|
||||
|
||||
assert collect.main(["--target-date", "2026-08-11"]) == 0
|
||||
assert len(profiles) == 1
|
||||
assert Path(profiles[0]).parent == tmp_path / "account" / "collector-runs"
|
||||
assert Path(profiles[0]).name.startswith("tmall-ozko-2026-08-11-")
|
||||
@@ -4,6 +4,11 @@ from pathlib import Path
|
||||
|
||||
import taobao_sycm_products as tm
|
||||
|
||||
from gyxx_flow.core.exit_codes import (
|
||||
COOKIE_SKIP_EXIT_CODE,
|
||||
NON_RETRYABLE_EXIT_CODE,
|
||||
)
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self) -> None:
|
||||
@@ -14,12 +19,171 @@ class _Session:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _LoginPage:
|
||||
url = "https://sycm.taobao.com/cc/item_rank"
|
||||
|
||||
def __init__(self, state: dict[str, bool]) -> None:
|
||||
self.state = state
|
||||
|
||||
def evaluate(self, *_args):
|
||||
return self.state
|
||||
|
||||
|
||||
def test_tm_login_state_requires_the_configured_shop(monkeypatch) -> None:
|
||||
monkeypatch.setattr(tm, "ACCOUNT", "光影行星旗舰店:operator")
|
||||
|
||||
wrong_shop = _LoginPage(
|
||||
{"hasStoreName": False, "hasTopNav": True, "hasAvatar": True}
|
||||
)
|
||||
requested_shop = _LoginPage(
|
||||
{"hasStoreName": True, "hasTopNav": True, "hasAvatar": True}
|
||||
)
|
||||
|
||||
assert tm.is_logged_in(wrong_shop) is False
|
||||
assert tm.is_logged_in(requested_shop) is True
|
||||
|
||||
|
||||
def test_tm_account_vault_replaces_stale_profile_cookies(monkeypatch, tmp_path: Path) -> None:
|
||||
class Context:
|
||||
def __init__(self) -> None:
|
||||
self.cleared = False
|
||||
|
||||
def clear_cookies(self) -> None:
|
||||
self.cleared = True
|
||||
|
||||
restored: list[tuple[str, str]] = []
|
||||
context = Context()
|
||||
cookie_file = tmp_path / "vault-cookies.json"
|
||||
storage_state_file = tmp_path / "vault-state.json"
|
||||
cookie_file.write_text("[]", encoding="utf-8")
|
||||
storage_state_file.write_text("{}", encoding="utf-8")
|
||||
monkeypatch.setenv("GYXX_ACCOUNT_ID", "tmall-ozko")
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"_runtime_state_paths",
|
||||
lambda: ((str(cookie_file), str(storage_state_file)), ("", "")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"restore_browser_state",
|
||||
lambda _context, *, cookie_file, storage_state_file: restored.append(
|
||||
(cookie_file, storage_state_file)
|
||||
),
|
||||
)
|
||||
|
||||
tm._restore_runtime_account_state(context)
|
||||
|
||||
assert context.cleared is True
|
||||
assert restored == [(str(cookie_file), str(storage_state_file))]
|
||||
|
||||
|
||||
def test_tm_main_fails_when_browser_connection_is_missing(monkeypatch) -> None:
|
||||
monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (None, None))
|
||||
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == 1
|
||||
|
||||
|
||||
def test_tm_backfill_delegates_inside_the_daily_browser_binding(monkeypatch) -> None:
|
||||
import taobao_sycm_collect_backfill as backfill
|
||||
|
||||
delegated: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
backfill,
|
||||
"main",
|
||||
lambda argv=None: delegated.extend(argv or []) or 0,
|
||||
)
|
||||
|
||||
assert tm.main([
|
||||
"--target-date",
|
||||
"2026-08-03",
|
||||
"--brand-user-data-dir",
|
||||
"brand-profile",
|
||||
"--brand",
|
||||
"ozko",
|
||||
"--backfill",
|
||||
]) == 0
|
||||
assert delegated[:4] == [
|
||||
"--target-date",
|
||||
"2026-08-03",
|
||||
"--brand-user-data-dir",
|
||||
"brand-profile",
|
||||
]
|
||||
assert delegated[-2:] == ["--brand", "ozko"]
|
||||
|
||||
|
||||
def test_tm_backfill_returns_cookie_recovery_code_without_credentials(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
import taobao_sycm_collect_backfill as backfill
|
||||
|
||||
session = _Session()
|
||||
monkeypatch.setattr(
|
||||
backfill,
|
||||
"start_stealthy_session",
|
||||
lambda *_args: (session, object()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
backfill,
|
||||
"ensure_logged_in",
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
RuntimeError("登录态失效且未配置 SYCM_ACCOUNT/SYCM_PASSWORD")
|
||||
),
|
||||
)
|
||||
|
||||
assert backfill.main(["--target-date", "2026-08-03"]) == COOKIE_SKIP_EXIT_CODE
|
||||
assert session.closed is True
|
||||
|
||||
|
||||
def test_tm_manual_login_waits_for_the_operator_and_closes_session(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
session = _Session()
|
||||
page = type("Page", (), {"goto": lambda *_args, **_kwargs: None})()
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"connect_native_manual_login",
|
||||
lambda _profile_dir: (session, page),
|
||||
)
|
||||
monkeypatch.setattr(tm, "is_logged_in", lambda _page: True)
|
||||
|
||||
assert tm.main([
|
||||
"--brand-user-data-dir",
|
||||
"ozko-profile",
|
||||
"--manual-login",
|
||||
"--login-timeout-seconds",
|
||||
"1",
|
||||
]) == 0
|
||||
assert tm.USER_DATA_DIR == "ozko-profile"
|
||||
assert session.closed is True
|
||||
|
||||
|
||||
def test_tm_login_target_returns_to_the_product_page() -> None:
|
||||
assert "_target=https%3A%2F%2Fsycm.taobao.com%2Fcc%2Fitem_rank" in tm.LOGIN_URL
|
||||
assert "/custom/login" not in tm.LOGIN_URL.split("_target=", 1)[1]
|
||||
|
||||
|
||||
def test_tm_native_manual_login_starts_with_one_blank_tab(monkeypatch) -> None:
|
||||
launched: list[tuple[str, str]] = []
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"launch_chrome_with_debug",
|
||||
lambda profile, start_url=tm.LOGIN_URL: launched.append((profile, start_url))
|
||||
or True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"connect_browser",
|
||||
lambda **kwargs: ("session", kwargs["cdp_url"]),
|
||||
)
|
||||
monkeypatch.setattr(tm, "DEBUG_PORT", 22096)
|
||||
|
||||
assert tm.connect_native_manual_login("ozko-profile") == (
|
||||
"session",
|
||||
"http://127.0.0.1:22096",
|
||||
)
|
||||
assert launched == [("ozko-profile", "about:blank")]
|
||||
|
||||
|
||||
def test_tm_main_fails_when_login_does_not_complete(monkeypatch) -> None:
|
||||
session = _Session()
|
||||
page = object()
|
||||
@@ -73,6 +237,9 @@ def test_tm_main_fails_when_download_has_no_valid_style(
|
||||
monkeypatch.setattr(tm, "export_excel_to_json_and_md", lambda _path: (None, None))
|
||||
monkeypatch.setattr(tm, "summarize_styles_from_excel", lambda _path: [])
|
||||
monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0)
|
||||
persisted: list[bool] = []
|
||||
monkeypatch.setattr(tm, "_persist_runtime_account_state", lambda _context: persisted.append(True))
|
||||
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == 1
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == NON_RETRYABLE_EXIT_CODE
|
||||
assert session.closed is True
|
||||
assert persisted == []
|
||||
|
||||
@@ -1,11 +1,73 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import taobao_dmp_item_crowd_insight_screenshots as dmp
|
||||
|
||||
|
||||
class TmPersonaRecoveryTests(unittest.TestCase):
|
||||
def test_cdp_connection_uses_adapter_existing_context_mode(self):
|
||||
page = Mock()
|
||||
context = Mock()
|
||||
context.new_page.return_value = page
|
||||
session = Mock()
|
||||
session.context = context
|
||||
browser_factory = Mock()
|
||||
browser_factory.return_value.start.return_value = session
|
||||
|
||||
with patch.object(dmp, "ScraplingBrowser", browser_factory):
|
||||
actual_session, actual_page = dmp._connect_cdp()
|
||||
|
||||
browser_factory.assert_called_once_with(
|
||||
stealthy=True,
|
||||
reuse_existing_cdp_context=True,
|
||||
cdp_url=dmp.CDP_URL,
|
||||
timeout=60000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
block_ads=False,
|
||||
)
|
||||
self.assertIs(actual_session, session)
|
||||
self.assertIs(actual_page, page)
|
||||
page.set_viewport_size.assert_called_once_with(
|
||||
{"width": 1920, "height": 1080}
|
||||
)
|
||||
|
||||
def test_item_insight_spa_route_is_authenticated_before_widgets_render(self):
|
||||
page = Mock()
|
||||
page.url = "https://dmp.taobao.com/index_new.html#!/items/item-insight"
|
||||
|
||||
with (
|
||||
patch.object(dmp, "login_form_visible", return_value=False),
|
||||
patch.object(dmp, "dmp_app_ready", return_value=False),
|
||||
patch.object(dmp, "page_text", return_value=""),
|
||||
):
|
||||
self.assertTrue(dmp.is_logged_in(page))
|
||||
|
||||
def test_login_redirect_is_not_authenticated_when_spa_text_leaks(self):
|
||||
page = Mock()
|
||||
page.url = (
|
||||
"https://dmp.taobao.com/login.html?mxredirectUrl="
|
||||
"https%3A%2F%2Fdmp.taobao.com%2Findex_new.html%23!%2Fitems%2Fitem-insight"
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(dmp, "login_form_visible", return_value=False),
|
||||
patch.object(dmp, "dmp_app_ready", return_value=False),
|
||||
patch.object(dmp, "page_text", return_value="单品信息 单品人群洞察"),
|
||||
):
|
||||
self.assertFalse(dmp.is_logged_in(page))
|
||||
|
||||
def test_login_redirect_is_not_ready_when_spa_text_leaks(self):
|
||||
page = Mock()
|
||||
page.url = (
|
||||
"https://dmp.taobao.com/login.html?mxredirectUrl="
|
||||
"https%3A%2F%2Fdmp.taobao.com%2Findex_new.html%23!%2Fitems%2Fitem-insight"
|
||||
)
|
||||
|
||||
self.assertFalse(dmp.dmp_app_ready(page))
|
||||
page.evaluate.assert_not_called()
|
||||
|
||||
def test_zero_chart_skip_is_not_platform_success(self):
|
||||
self.assertEqual(
|
||||
dmp.records_exit_code(
|
||||
|
||||
@@ -0,0 +1,548 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import tmall_baibu_apply_common as baibu
|
||||
|
||||
|
||||
def _entry() -> baibu.BaibuEntry:
|
||||
return baibu.BaibuEntry(
|
||||
entry_id="demo",
|
||||
name="测试入口",
|
||||
account_id="tmall-shop",
|
||||
account_marker="测试店铺",
|
||||
template_url="https://feishu.example/wiki/template",
|
||||
apply_url="https://myseller.example/apply",
|
||||
script="demo.py",
|
||||
)
|
||||
|
||||
|
||||
def test_latest_batch_import_status_parses_processing_and_complete_summary() -> None:
|
||||
processing = baibu._latest_batch_import_status(
|
||||
"您于2026-08-26 09:23:37执行了商品批量导入操作,正在处理中。"
|
||||
)
|
||||
assert processing is not None
|
||||
assert processing.state == "processing"
|
||||
|
||||
completed = baibu._latest_batch_import_status(
|
||||
"最近操作:您于2026-08-26 09:23:37执行了商品批量导入操作,"
|
||||
"总数量20件,已成功20件,失败0件,待查看0件。"
|
||||
)
|
||||
assert completed is not None
|
||||
assert completed.state == "success"
|
||||
assert completed.detail == "总数量20件,已成功20件,失败0件,待查看0件"
|
||||
|
||||
|
||||
def test_latest_batch_import_status_marks_partial_summary_as_failure() -> None:
|
||||
result = baibu._latest_batch_import_status(
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量95件,已成功8件,失败87件,待查看0件。"
|
||||
)
|
||||
assert result is not None
|
||||
assert result.state == "failure"
|
||||
assert "失败87件" in result.detail
|
||||
|
||||
|
||||
def test_credentials_are_loaded_from_account_declared_environment_names(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
|
||||
'["TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD"]',
|
||||
)
|
||||
monkeypatch.delenv("GYXX_BROWSER_CREDENTIAL_ENV_NAMES", raising=False)
|
||||
monkeypatch.setenv("TMALL_BAG_ACCOUNT", "demo-account")
|
||||
monkeypatch.setenv("TMALL_BAG_PASSWORD", "demo-password")
|
||||
|
||||
credentials, names = baibu._credentials_from_environment()
|
||||
|
||||
assert names == ("TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD")
|
||||
assert credentials is not None
|
||||
assert credentials.username == "demo-account"
|
||||
assert credentials.password == "demo-password"
|
||||
assert credentials.username_env == "TMALL_BAG_ACCOUNT"
|
||||
assert credentials.password_env == "TMALL_BAG_PASSWORD"
|
||||
|
||||
|
||||
def test_credentials_prefer_the_tmall_shop_pair_over_legacy_pairs(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
|
||||
'["TMALL_SHOP_ACCOUNT", "TMALL_SHOP_PASSWORD", '
|
||||
'"SYCM_ACCOUNT", "SYCM_PASSWORD"]',
|
||||
)
|
||||
monkeypatch.setenv("TMALL_SHOP_ACCOUNT", "完整店铺名:子账号")
|
||||
monkeypatch.setenv("TMALL_SHOP_PASSWORD", "shop-password")
|
||||
monkeypatch.setenv("SYCM_ACCOUNT", "legacy-account")
|
||||
monkeypatch.setenv("SYCM_PASSWORD", "legacy-password")
|
||||
|
||||
credentials, names = baibu._credentials_from_environment()
|
||||
|
||||
assert names == (
|
||||
"TMALL_SHOP_ACCOUNT",
|
||||
"TMALL_SHOP_PASSWORD",
|
||||
"SYCM_ACCOUNT",
|
||||
"SYCM_PASSWORD",
|
||||
)
|
||||
assert credentials is not None
|
||||
assert credentials.username == "完整店铺名:子账号"
|
||||
assert credentials.username_env == "TMALL_SHOP_ACCOUNT"
|
||||
assert credentials.password_env == "TMALL_SHOP_PASSWORD"
|
||||
|
||||
|
||||
class _LoginLocator:
|
||||
def __init__(self, *, visible: bool = True) -> None:
|
||||
self.visible = visible
|
||||
self.filled: list[str] = []
|
||||
self.click_count = 0
|
||||
|
||||
def count(self) -> int:
|
||||
return 1 if self.visible else 0
|
||||
|
||||
def nth(self, _index: int):
|
||||
return self
|
||||
|
||||
def is_visible(self) -> bool:
|
||||
return self.visible
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
return True
|
||||
|
||||
def fill(self, value: str) -> None:
|
||||
self.filled.append(value)
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.click_count += 1
|
||||
|
||||
|
||||
class _LoginFrame:
|
||||
def __init__(self) -> None:
|
||||
self.username = _LoginLocator()
|
||||
self.password = _LoginLocator()
|
||||
self.submit = _LoginLocator()
|
||||
|
||||
def locator(self, selector: str):
|
||||
return {
|
||||
"input[placeholder='账号名/邮箱/手机号']": self.username,
|
||||
"input[placeholder='请输入登录密码']": self.password,
|
||||
"button[type='submit']": _LoginLocator(visible=False),
|
||||
"input[type='submit']": _LoginLocator(visible=False),
|
||||
}.get(selector, _LoginLocator(visible=False))
|
||||
|
||||
def get_by_role(self, role: str, *, name: str, exact: bool):
|
||||
if role == "button" and name == "登录" and exact:
|
||||
return self.submit
|
||||
return _LoginLocator(visible=False)
|
||||
|
||||
def get_by_text(self, label: str, *, exact: bool):
|
||||
return _LoginLocator(visible=False)
|
||||
|
||||
|
||||
def test_login_if_needed_fills_the_password_form_inside_an_iframe(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
|
||||
'["TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD"]',
|
||||
)
|
||||
monkeypatch.setenv("TMALL_BAG_ACCOUNT", "demo-account")
|
||||
monkeypatch.setenv("TMALL_BAG_PASSWORD", "demo-password")
|
||||
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
|
||||
frame = _LoginFrame()
|
||||
page = SimpleNamespace(
|
||||
url="https://loginmyseller.taobao.com/?redirect_url=https%3A%2F%2Fqn.taobao.com",
|
||||
frames=[frame],
|
||||
)
|
||||
|
||||
attempted = baibu._login_if_needed(page=page, entry=_entry())
|
||||
|
||||
assert attempted is True
|
||||
assert frame.username.filled == ["demo-account"]
|
||||
assert frame.password.filled == ["demo-password"]
|
||||
assert frame.submit.click_count == 1
|
||||
|
||||
|
||||
def test_manual_login_waits_without_filling_the_password_form(monkeypatch) -> None:
|
||||
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
|
||||
frame = _LoginFrame()
|
||||
page = SimpleNamespace(
|
||||
url="https://loginmyseller.taobao.com/",
|
||||
frames=[frame],
|
||||
)
|
||||
|
||||
attempted = baibu._login_if_needed(
|
||||
page=page,
|
||||
entry=_entry(),
|
||||
manual_login=True,
|
||||
)
|
||||
|
||||
assert attempted is True
|
||||
assert frame.username.filled == []
|
||||
assert frame.password.filled == []
|
||||
assert frame.submit.click_count == 0
|
||||
|
||||
|
||||
def test_login_if_needed_reports_missing_credentials_without_filling(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
|
||||
'["TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD"]',
|
||||
)
|
||||
monkeypatch.delenv("TMALL_BAG_ACCOUNT", raising=False)
|
||||
monkeypatch.delenv("TMALL_BAG_PASSWORD", raising=False)
|
||||
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
|
||||
frame = _LoginFrame()
|
||||
page = SimpleNamespace(
|
||||
url="https://loginmyseller.taobao.com/",
|
||||
frames=[frame],
|
||||
)
|
||||
|
||||
try:
|
||||
baibu._login_if_needed(page=page, entry=_entry())
|
||||
except baibu.BaibuApplyError as exc:
|
||||
assert "TMALL_BAG_ACCOUNT" in str(exc)
|
||||
assert "TMALL_BAG_PASSWORD" in str(exc)
|
||||
else:
|
||||
raise AssertionError("missing credentials must stop before filling")
|
||||
|
||||
assert frame.username.filled == []
|
||||
assert frame.password.filled == []
|
||||
|
||||
|
||||
def test_wait_for_authenticated_stops_on_explicit_login_error(monkeypatch) -> None:
|
||||
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: "账密错误")
|
||||
|
||||
with pytest.raises(baibu.BaibuApplyError, match="账密错误"):
|
||||
baibu._wait_for_authenticated(
|
||||
_entry(),
|
||||
SimpleNamespace(url="https://loginmyseller.taobao.com/"),
|
||||
login_attempted=True,
|
||||
)
|
||||
|
||||
|
||||
def test_login_second_factor_marker_is_detected(monkeypatch) -> None:
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: "你正在使用手机短信验证身份")
|
||||
|
||||
assert baibu._login_second_factor_marker(SimpleNamespace()) == "短信验证"
|
||||
|
||||
|
||||
def test_persist_account_state_filters_cookies_to_the_account_domains(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
account_cookie_file = tmp_path / "state" / "accounts" / "tmall-bag" / "cookies.json"
|
||||
account_storage_file = (
|
||||
tmp_path / "state" / "accounts" / "tmall-bag" / "storage_state.json"
|
||||
)
|
||||
monkeypatch.setenv("GYXX_ACCOUNT_COOKIE_FILE", str(account_cookie_file))
|
||||
monkeypatch.setenv("GYXX_ACCOUNT_STORAGE_STATE_FILE", str(account_storage_file))
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_REQUIRED_COOKIE_DOMAINS",
|
||||
'["taobao.com", "alimama.com"]',
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_REQUIRED_COOKIE_NAMES",
|
||||
'["unb", "cookie2", "_tb_token_"]',
|
||||
)
|
||||
|
||||
cookies = [
|
||||
{"name": "unb", "domain": ".taobao.com", "value": "valid"},
|
||||
{"name": "other", "domain": "example.com", "value": "unrelated"},
|
||||
]
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return cookies
|
||||
|
||||
def storage_state(self):
|
||||
return {
|
||||
"cookies": cookies,
|
||||
"origins": [
|
||||
{"origin": "https://qn.taobao.com", "localStorage": []},
|
||||
{"origin": "https://example.com", "localStorage": []},
|
||||
],
|
||||
}
|
||||
|
||||
browser = SimpleNamespace(context=Context())
|
||||
|
||||
assert baibu._persist_account_state(browser, _entry()) is True
|
||||
|
||||
saved_cookies = json.loads(account_cookie_file.read_text(encoding="utf-8"))
|
||||
saved_state = json.loads(account_storage_file.read_text(encoding="utf-8"))
|
||||
assert saved_cookies == [cookies[0]]
|
||||
assert saved_state["cookies"] == [cookies[0]]
|
||||
assert saved_state["origins"] == [
|
||||
{"origin": "https://qn.taobao.com", "localStorage": []}
|
||||
]
|
||||
|
||||
|
||||
def test_upload_template_supports_tmall_dialog_import_button(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
calls: list[tuple[str, ...]] = []
|
||||
|
||||
class Chooser:
|
||||
def __init__(self) -> None:
|
||||
self.files: str | None = None
|
||||
|
||||
def set_files(self, value: str) -> None:
|
||||
self.files = value
|
||||
|
||||
chooser = Chooser()
|
||||
|
||||
class ExpectChooser:
|
||||
def __enter__(self):
|
||||
return SimpleNamespace(value=chooser)
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(baibu, "_file_inputs", lambda _page: [])
|
||||
|
||||
def click(_page, labels):
|
||||
labels = tuple(labels)
|
||||
calls.append(labels)
|
||||
return "导入表格" in labels
|
||||
|
||||
monkeypatch.setattr(baibu, "_click_text", click)
|
||||
page = SimpleNamespace(
|
||||
expect_file_chooser=lambda **_kwargs: ExpectChooser(),
|
||||
)
|
||||
template = tmp_path / "template.xlsx"
|
||||
template.write_bytes(b"xlsx")
|
||||
|
||||
baibu._upload_template(page, template)
|
||||
|
||||
assert calls[0] == ("导入表格",)
|
||||
assert chooser.files == str(template)
|
||||
|
||||
|
||||
def test_open_batch_import_refuses_an_active_previous_operation(monkeypatch) -> None:
|
||||
page = SimpleNamespace()
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_page_text",
|
||||
lambda _page: "您于2026-08-26 09:23:37执行了商品批量导入操作,正在处理中。",
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_refresh_batch_results", lambda _page: True)
|
||||
|
||||
try:
|
||||
baibu._open_batch_import(page, _entry())
|
||||
except baibu.BaibuApplyError as exc:
|
||||
assert "正在处理" in str(exc)
|
||||
else:
|
||||
raise AssertionError("active batch import must not be submitted again")
|
||||
|
||||
|
||||
def test_open_batch_import_waits_for_hydrated_previous_summary(monkeypatch) -> None:
|
||||
page = SimpleNamespace()
|
||||
texts = iter(
|
||||
[
|
||||
"测试店铺 商品批量导入",
|
||||
"测试店铺 最近操作:您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量2件,已成功2件,失败0件,待查看0件。",
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
|
||||
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_text",
|
||||
lambda _page, _labels: True,
|
||||
)
|
||||
|
||||
previous = baibu._open_batch_import(page, _entry())
|
||||
|
||||
assert previous is not None
|
||||
assert previous.state == "success"
|
||||
|
||||
|
||||
def test_wait_import_result_refreshes_and_accepts_new_success_summary(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
texts = iter(
|
||||
[
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,正在处理中。",
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量2件,已成功2件,失败0件,待查看0件。",
|
||||
]
|
||||
)
|
||||
refreshes: list[bool] = []
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_refresh_batch_results",
|
||||
lambda _page: refreshes.append(True) or True,
|
||||
)
|
||||
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(baibu, "_RESULT_REFRESH_INTERVAL_SECONDS", 0)
|
||||
|
||||
baibu._wait_import_result(SimpleNamespace(), _entry())
|
||||
|
||||
assert refreshes
|
||||
|
||||
|
||||
def test_wait_import_result_accepts_new_failure_summary_as_submitted(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
texts = iter(
|
||||
[
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,正在处理中。",
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量95件,已成功0件,失败95件,待查看0件。",
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
|
||||
monkeypatch.setattr(baibu, "_refresh_batch_results", lambda _page: True)
|
||||
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(baibu, "_RESULT_REFRESH_INTERVAL_SECONDS", 0)
|
||||
|
||||
# Tmall's row-level result is informational; submitting the batch is the
|
||||
# success condition for this workflow.
|
||||
baibu._wait_import_result(SimpleNamespace(), _entry())
|
||||
|
||||
|
||||
def test_wait_import_result_ignores_stale_summary_until_signature_changes(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
previous = baibu._latest_batch_import_status(
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量95件,已成功8件,失败87件,待查看0件。"
|
||||
)
|
||||
assert previous is not None
|
||||
texts = iter(
|
||||
[
|
||||
"最近操作:您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量95件,已成功8件,失败87件,待查看0件。",
|
||||
"您于2026-08-26 17:30:01执行了商品批量导入操作,正在处理中。",
|
||||
"您于2026-08-26 17:30:01执行了商品批量导入操作,"
|
||||
"总数量2件,已成功2件,失败0件,待查看0件。",
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
|
||||
monkeypatch.setattr(baibu, "_refresh_batch_results", lambda _page: True)
|
||||
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(baibu, "_RESULT_REFRESH_INTERVAL_SECONDS", 0)
|
||||
|
||||
baibu._wait_import_result(
|
||||
SimpleNamespace(),
|
||||
_entry(),
|
||||
previous_operation=previous,
|
||||
)
|
||||
|
||||
|
||||
def test_complete_draft_products_runs_required_sequence_until_count_is_zero(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
entry = replace(_entry(), complete_drafts_after_import=True)
|
||||
page = SimpleNamespace()
|
||||
page_texts = iter(["草稿 (2)", "草稿 (1)", "草稿 (0)"])
|
||||
events: list[str] = []
|
||||
clicked: list[tuple[str, ...]] = []
|
||||
opened: list[bool] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_draft_tab",
|
||||
lambda _page: events.append("draft") or True,
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(page_texts))
|
||||
monkeypatch.setattr(baibu, "_visible_text_count", lambda _page, _label: 1)
|
||||
monkeypatch.setattr(baibu, "_has_visible_text", lambda _page, _labels: True)
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_open_draft_editor",
|
||||
lambda _page, _entry, *, first: events.append(
|
||||
f"open:{first}"
|
||||
) or opened.append(first),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_text_until_found",
|
||||
lambda _page, labels, **_kwargs: events.append(next(iter(labels)))
|
||||
or clicked.append(tuple(labels)),
|
||||
)
|
||||
|
||||
baibu._complete_draft_products(page, entry)
|
||||
|
||||
assert opened == [True, False]
|
||||
assert events == [
|
||||
"draft",
|
||||
"open:True",
|
||||
"否",
|
||||
"下一步",
|
||||
"draft",
|
||||
"open:False",
|
||||
"否",
|
||||
"下一步",
|
||||
"draft",
|
||||
]
|
||||
assert clicked == [
|
||||
("否",),
|
||||
("下一步",),
|
||||
("否",),
|
||||
("下一步",),
|
||||
]
|
||||
|
||||
|
||||
def test_complete_draft_products_is_disabled_for_other_entries(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_draft_tab",
|
||||
lambda _page: pytest.fail("其他入口不得进入草稿处理流程"),
|
||||
)
|
||||
|
||||
baibu._complete_draft_products(SimpleNamespace(), _entry())
|
||||
|
||||
|
||||
def test_complete_draft_products_accepts_empty_data_page(monkeypatch) -> None:
|
||||
entry = replace(_entry(), complete_drafts_after_import=True)
|
||||
page = SimpleNamespace()
|
||||
|
||||
monkeypatch.setattr(baibu, "_click_draft_tab", lambda _page: True)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: "草稿 数据为空")
|
||||
monkeypatch.setattr(baibu, "_visible_text_count", lambda _page, _label: 0)
|
||||
|
||||
baibu._complete_draft_products(page, entry)
|
||||
|
||||
|
||||
def test_return_to_draft_list_falls_back_to_browser_history(monkeypatch) -> None:
|
||||
events: list[str] = []
|
||||
|
||||
class Page:
|
||||
def go_back(self, **_kwargs) -> None:
|
||||
events.append("back")
|
||||
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_draft_tab",
|
||||
lambda _page: events.append("draft") or False,
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_close_editor", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_has_visible_text",
|
||||
lambda _page, labels: "完善商品" in labels and "back" in events,
|
||||
)
|
||||
|
||||
baibu._return_to_draft_list(Page(), _entry())
|
||||
|
||||
assert events == ["draft", "back"]
|
||||
|
||||
|
||||
def test_tmall_config_enables_draft_completion_for_only_old_all_3c() -> None:
|
||||
old_all_3c = baibu._load_entry("old_all_3c")
|
||||
old_all_bag = baibu._load_entry("old_all_bag")
|
||||
new_all_bag = baibu._load_entry("new_all_bag")
|
||||
|
||||
assert old_all_3c.complete_drafts_after_import is True
|
||||
assert old_all_bag.complete_drafts_after_import is False
|
||||
assert new_all_bag.complete_drafts_after_import is False
|
||||
@@ -0,0 +1,572 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import upload_video_to_guanghe as tmall
|
||||
|
||||
|
||||
def _record(
|
||||
record_id: str,
|
||||
attachments: list[dict[str, Any]],
|
||||
*,
|
||||
product_names: str = "极星双肩",
|
||||
record_date: object = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"record_id": record_id,
|
||||
"代拍产品": product_names,
|
||||
"代拍附件": attachments,
|
||||
"日期": record_date,
|
||||
"品牌": ["光影行星"],
|
||||
"性别": ["女"],
|
||||
tmall.TM_UPLOAD_FIELD: False,
|
||||
}
|
||||
|
||||
|
||||
def _one_candidate(*, store: str = tmall.STORE_LUGGAGE):
|
||||
candidates, issues = tmall.discover_tmall_publication_candidates(
|
||||
[
|
||||
_record(
|
||||
"record-one",
|
||||
[
|
||||
{
|
||||
"type": "file",
|
||||
"name": "video.mp4",
|
||||
"file_token": "video-token",
|
||||
"size": 123,
|
||||
}
|
||||
],
|
||||
)
|
||||
],
|
||||
store=store,
|
||||
)
|
||||
assert issues == []
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def test_catalog_matching_accepts_variant_after_bag_stem() -> None:
|
||||
assert tmall._match_tm_item_ids_from_product_titles(
|
||||
["盖亚斜挎3"],
|
||||
[
|
||||
(
|
||||
"1059764111499",
|
||||
"GYXX/光影行星盖亚斜挎包3单反相机包男款微单摄影包",
|
||||
)
|
||||
],
|
||||
) == {"盖亚斜挎3": ["1059764111499"]}
|
||||
|
||||
|
||||
def test_catalog_search_pattern_keeps_variant_alias_rows_in_scope() -> None:
|
||||
assert tmall._tm_product_title_search_pattern("盖亚斜挎3") == "%盖亚斜挎%"
|
||||
|
||||
|
||||
def test_invalid_compatible_video_cache_is_rebuilt(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source = tmp_path / "source.mov"
|
||||
source.write_bytes(b"source")
|
||||
cached = source.with_name("source_compatible.mp4")
|
||||
cached.write_bytes(b"truncated-cache")
|
||||
|
||||
class _Reader:
|
||||
def __init__(self, metadata: dict[str, Any]) -> None:
|
||||
self.metadata = metadata
|
||||
|
||||
def __iter__(self) -> _Reader:
|
||||
return self
|
||||
|
||||
def __next__(self) -> dict[str, Any]:
|
||||
return self.metadata
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
class _FakeImageioFfmpeg:
|
||||
@staticmethod
|
||||
def get_ffmpeg_exe() -> str:
|
||||
return "ffmpeg"
|
||||
|
||||
@staticmethod
|
||||
def read_frames(path: str) -> _Reader:
|
||||
if path == str(cached):
|
||||
raise OSError("moov atom not found")
|
||||
return _Reader({"codec": "hevc", "source_size": (1920, 1080)})
|
||||
|
||||
monkeypatch.setitem(sys.modules, "imageio_ffmpeg", _FakeImageioFfmpeg)
|
||||
|
||||
def fake_run(command: list[str], **_kwargs: Any) -> SimpleNamespace:
|
||||
Path(command[-1]).write_bytes(b"rebuilt")
|
||||
return SimpleNamespace(returncode=0, stderr="", stdout="")
|
||||
|
||||
monkeypatch.setattr(tmall.subprocess, "run", fake_run)
|
||||
|
||||
assert tmall.ensure_upload_compatible_video(source) == cached
|
||||
assert cached.read_bytes() == b"rebuilt"
|
||||
|
||||
|
||||
def test_discovery_creates_one_candidate_per_exact_video_attachment() -> None:
|
||||
candidates, issues = tmall.discover_tmall_publication_candidates(
|
||||
[
|
||||
_record(
|
||||
"record-with-two-videos",
|
||||
[
|
||||
{
|
||||
"type": "file",
|
||||
"name": "same-name.mp4",
|
||||
"file_token": "video-token-a",
|
||||
"size": "101",
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"name": "same-name.mp4",
|
||||
"file_token": "video-token-b",
|
||||
"size": 202,
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"name": "cover.jpg",
|
||||
"file_token": "image-token",
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"name": "generic-attachment",
|
||||
"file_token": "generic-file-token",
|
||||
},
|
||||
],
|
||||
record_date=None,
|
||||
),
|
||||
_record(
|
||||
"record-marked-skip",
|
||||
[
|
||||
{
|
||||
"type": "file",
|
||||
"name": "must-not-upload.mp4",
|
||||
"file_token": "skipped-token",
|
||||
}
|
||||
],
|
||||
product_names="极星双肩 / 未到货",
|
||||
),
|
||||
],
|
||||
store=tmall.STORE_LUGGAGE,
|
||||
)
|
||||
|
||||
assert {
|
||||
(candidate.source_record_id, candidate.video_file_token)
|
||||
for candidate in candidates
|
||||
} == {
|
||||
("record-with-two-videos", "video-token-a"),
|
||||
("record-with-two-videos", "video-token-b"),
|
||||
}
|
||||
assert {candidate.video_file_name for candidate in candidates} == {
|
||||
"same-name.mp4"
|
||||
}
|
||||
assert all(candidate.source_record_date is None for candidate in candidates)
|
||||
assert [
|
||||
(issue.source_record_id, issue.reason) for issue in issues
|
||||
] == [("record-marked-skip", "代拍产品包含未到货/不发标记")]
|
||||
|
||||
|
||||
def test_flagship_and_luggage_use_distinct_account_scopes() -> None:
|
||||
flagship = _one_candidate(store=tmall.STORE_FLAGSHIP)
|
||||
luggage = _one_candidate(store=tmall.STORE_LUGGAGE)
|
||||
|
||||
assert flagship.target_account_key == "guanghe-flagship"
|
||||
assert luggage.target_account_key == "guanghe-luggage"
|
||||
assert flagship.target_account_key != luggage.target_account_key
|
||||
assert tmall.STORE_ACCOUNT_KEYS == {
|
||||
tmall.STORE_FLAGSHIP: flagship.target_account_key,
|
||||
tmall.STORE_LUGGAGE: luggage.target_account_key,
|
||||
}
|
||||
|
||||
|
||||
def test_luggage_runtime_uses_new_store_account_label_without_changing_password_key(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GUANGHE_LUGGAGE_USERNAME", "光影行星鑫华达专卖店:龙虾仔")
|
||||
monkeypatch.setenv("GUANGHE_LUGGAGE_PASSWORD", "same-runtime-secret")
|
||||
|
||||
store_name, username, password, _profile = tmall._store_runtime(tmall.STORE_LUGGAGE)
|
||||
|
||||
assert store_name == "光影行星鑫华达专卖店:龙虾仔"
|
||||
assert username == "光影行星鑫华达专卖店:龙虾仔"
|
||||
assert password == "same-runtime-secret"
|
||||
|
||||
|
||||
def test_account_identity_markers_accept_store_name_with_login_nickname() -> None:
|
||||
assert tmall.GuangHeUploader._account_identity_markers(
|
||||
"光影行星鑫华达专卖店:龙虾仔"
|
||||
) == (
|
||||
"光影行星鑫华达专卖店:龙虾仔",
|
||||
"光影行星鑫华达专卖店",
|
||||
)
|
||||
|
||||
|
||||
def test_flagship_creator_identity_uses_visible_name_and_account_number() -> None:
|
||||
assert tmall._expected_creator_identity_markers(
|
||||
tmall.STORE_FLAGSHIP,
|
||||
"光影行星旗舰店",
|
||||
) == ("光影行星天猫官方", "逛逛号:3301283916")
|
||||
|
||||
|
||||
def test_creator_identity_requires_every_configured_marker() -> None:
|
||||
class Body:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
|
||||
def inner_text(self, **_kwargs: object) -> str:
|
||||
return self.text
|
||||
|
||||
class Page:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
|
||||
def locator(self, selector: str) -> Body:
|
||||
assert selector == "body"
|
||||
return Body(self.text)
|
||||
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
uploader.expected_account_markers = (
|
||||
"光影行星天猫官方",
|
||||
"逛逛号:3301283916",
|
||||
)
|
||||
uploader.page = Page("光影行星天猫官方\n逛逛号:3301283916\n账号正常")
|
||||
assert uploader._page_matches_expected_account()
|
||||
|
||||
uploader.page = Page("光影行星天猫官方\n账号正常")
|
||||
assert not uploader._page_matches_expected_account()
|
||||
|
||||
|
||||
def test_wait_for_existing_login_allows_account_card_to_render(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
states = iter((False, True))
|
||||
uploader._sync_page = lambda: None
|
||||
uploader._is_logged_in = lambda: next(states)
|
||||
monkeypatch.setattr(tmall.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(tmall.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert uploader._wait_for_existing_login(timeout_seconds=8)
|
||||
|
||||
|
||||
def test_legacy_backfill_has_25_exact_attachment_matches_and_fails_on_drift() -> None:
|
||||
entries = tmall.load_legacy_publication_backfill()
|
||||
assert len(entries) == 25
|
||||
|
||||
attachments_by_record: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for entry in entries:
|
||||
attachments_by_record[entry["source_record_id"]].append(
|
||||
{
|
||||
"type": "file",
|
||||
"name": entry["video_file_name"],
|
||||
"file_token": entry["video_file_token"],
|
||||
}
|
||||
)
|
||||
records = [
|
||||
_record(record_id, attachments)
|
||||
for record_id, attachments in attachments_by_record.items()
|
||||
]
|
||||
candidates, issues = tmall.discover_tmall_publication_candidates(
|
||||
records,
|
||||
store=tmall.STORE_LUGGAGE,
|
||||
)
|
||||
|
||||
assert issues == []
|
||||
matched = tmall._legacy_backfill_for_candidates(
|
||||
tmall.STORE_LUGGAGE,
|
||||
candidates,
|
||||
)
|
||||
assert len(matched) == 25
|
||||
assert {
|
||||
(
|
||||
candidate.source_record_id,
|
||||
candidate.video_file_token,
|
||||
candidate.video_file_name,
|
||||
)
|
||||
for candidate, _entry in matched
|
||||
} == {
|
||||
(
|
||||
entry["source_record_id"],
|
||||
entry["video_file_token"],
|
||||
entry["video_file_name"],
|
||||
)
|
||||
for entry in entries
|
||||
}
|
||||
|
||||
drifted = [
|
||||
replace(candidates[0], video_file_token="drifted-token"),
|
||||
*candidates[1:],
|
||||
]
|
||||
with pytest.raises(RuntimeError, match="record_id/file_token/文件名不一致"):
|
||||
tmall._legacy_backfill_for_candidates(
|
||||
tmall.STORE_LUGGAGE,
|
||||
drifted,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_args_is_preview_by_default_and_requires_explicit_execute(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("GYXX_VIDEO_UPLOAD_STORE", raising=False)
|
||||
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py"])
|
||||
preview = tmall.parse_args()
|
||||
assert preview.execute is False
|
||||
assert preview.store == tmall.STORE_FLAGSHIP
|
||||
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["upload_video_to_guanghe.py", "--execute"],
|
||||
)
|
||||
execution = tmall.parse_args()
|
||||
assert execution.execute is True
|
||||
|
||||
|
||||
def test_parse_args_uses_frontend_selected_store_from_runtime_environment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GYXX_VIDEO_UPLOAD_STORE", tmall.STORE_LUGGAGE)
|
||||
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py", "--execute"])
|
||||
|
||||
execution = tmall.parse_args()
|
||||
|
||||
assert execution.store == tmall.STORE_LUGGAGE
|
||||
|
||||
|
||||
def test_parse_args_uses_configured_topic_keyword_and_removes_spacing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv(tmall.TMALL_TOPIC_KEYWORD_ENV, " 秋 上新 ")
|
||||
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py"])
|
||||
|
||||
execution = tmall.parse_args()
|
||||
|
||||
assert execution.topic_keyword == "秋上新"
|
||||
|
||||
|
||||
def test_parse_args_rejects_invalid_runtime_store(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GYXX_VIDEO_UPLOAD_STORE", "unknown")
|
||||
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py"])
|
||||
|
||||
with pytest.raises(ValueError, match="GYXX_VIDEO_UPLOAD_STORE"):
|
||||
tmall.parse_args()
|
||||
|
||||
|
||||
def test_tmall_pre_submit_failures_are_automatically_retryable() -> None:
|
||||
assert tmall._is_retryable_pre_submit_failure(
|
||||
tmall.TmallTitleGenerationError("no valid candidate"),
|
||||
1,
|
||||
)
|
||||
assert tmall._is_retryable_pre_submit_failure(RuntimeError("timed out"), 1)
|
||||
assert not tmall._is_retryable_pre_submit_failure(RuntimeError("timed out"), 3)
|
||||
|
||||
|
||||
def test_tmall_strict_title_requests_a_candidate_pool_and_classifies_validation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class ValidationGenerator:
|
||||
def __init__(self, **kwargs: object) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
def generate(self, *_args: object, **_kwargs: object) -> str:
|
||||
raise tmall.TitleValidationError("no valid candidate")
|
||||
|
||||
monkeypatch.setattr(tmall, "JdVideoTitleGenerator", ValidationGenerator)
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
uploader.require_llm_titles = True
|
||||
uploader._used_titles_global = []
|
||||
uploader._catalog_product_titles = lambda _name: ("极星双肩包轻便电脑包",)
|
||||
|
||||
with pytest.raises(tmall.TmallTitleGenerationError, match="no valid candidate"):
|
||||
uploader._generate_title(["极星双肩"], video_key="video-token")
|
||||
|
||||
assert captured["candidate_count"] == tmall.TMALL_TITLE_CANDIDATE_COUNT
|
||||
assert int(captured["candidate_count"]) > 1
|
||||
assert captured["allow_verified_fallback"] is True
|
||||
|
||||
|
||||
def test_tmall_strict_title_keeps_model_request_failures_retryable(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class RequestGenerator:
|
||||
def __init__(self, **_kwargs: object) -> None:
|
||||
pass
|
||||
|
||||
def generate(self, *_args: object, **_kwargs: object) -> str:
|
||||
raise tmall.TitleGenerationError("timed out")
|
||||
|
||||
monkeypatch.setattr(tmall, "JdVideoTitleGenerator", RequestGenerator)
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
uploader.require_llm_titles = True
|
||||
uploader._used_titles_global = []
|
||||
uploader._catalog_product_titles = lambda _name: ("极星双肩包轻便电脑包",)
|
||||
|
||||
with pytest.raises(tmall.TitleGenerationError, match="timed out"):
|
||||
uploader._generate_title(["极星双肩"], video_key="video-token")
|
||||
|
||||
|
||||
def test_execute_with_no_selected_store_candidates_does_not_start_browser(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
args = SimpleNamespace(
|
||||
execute=True,
|
||||
republish_from_date=None,
|
||||
video_name=None,
|
||||
exclude_video_name=[],
|
||||
record_id=None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tmall,
|
||||
"_store_runtime",
|
||||
lambda _store: ("光影行星鑫华达专卖店:龙虾仔", "configured", "configured", Path("profile")),
|
||||
)
|
||||
monkeypatch.setattr(tmall, "select_pending_records", lambda *_args, **_kwargs: [])
|
||||
monkeypatch.setattr(
|
||||
tmall,
|
||||
"discover_tmall_publication_candidates",
|
||||
lambda *_args, **_kwargs: ([], []),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tmall,
|
||||
"_legacy_backfill_for_candidates",
|
||||
lambda *_args, **_kwargs: [],
|
||||
)
|
||||
|
||||
class UnexpectedUploader:
|
||||
def __init__(self, *_args: Any, **_kwargs: Any) -> None:
|
||||
raise AssertionError("筛选为空时不应启动浏览器")
|
||||
|
||||
monkeypatch.setattr(tmall, "GuangHeUploader", UnexpectedUploader)
|
||||
|
||||
result = tmall._run_store(
|
||||
args,
|
||||
store=tmall.STORE_LUGGAGE,
|
||||
records=[],
|
||||
luggage_style_whitelist={"极星双肩"},
|
||||
)
|
||||
|
||||
assert result == tmall.EXIT_OK
|
||||
|
||||
|
||||
class _FakeLocator:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
count: int = 1,
|
||||
on_click: Any = None,
|
||||
) -> None:
|
||||
self._count = count
|
||||
self._on_click = on_click
|
||||
|
||||
@property
|
||||
def first(self) -> _FakeLocator:
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count
|
||||
|
||||
def fill(self, _value: str) -> None:
|
||||
return None
|
||||
|
||||
def click(self, **_kwargs: Any) -> None:
|
||||
if self._on_click is not None:
|
||||
self._on_click()
|
||||
|
||||
|
||||
def test_publish_video_persists_submit_boundary_before_publish_click(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
events: list[str] = []
|
||||
upload_timeouts: list[int] = []
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
uploader.page = SimpleNamespace(url="https://creator.guanghe.taobao.com/page/")
|
||||
uploader._upload_video_file = lambda _path: None
|
||||
uploader._wait_upload_done = lambda **kwargs: (upload_timeouts.append(kwargs["timeout"]) or True)
|
||||
uploader._snapshot = lambda _name: None
|
||||
uploader._generate_title = lambda *_args, **_kwargs: "极星双肩包视频标题"
|
||||
uploader._prepare_publish_metadata = lambda _products, **_kwargs: None
|
||||
uploader._link_product = lambda _products, _item_ids: True
|
||||
|
||||
def locate(selector: str) -> _FakeLocator:
|
||||
if selector == "text=立即发布":
|
||||
return _FakeLocator(on_click=lambda: events.append("publish-click"))
|
||||
if selector == "text=发布成功":
|
||||
return _FakeLocator(count=1)
|
||||
if selector.startswith("text=") and selector not in {
|
||||
"text=内容无需标注",
|
||||
}:
|
||||
return _FakeLocator(count=0)
|
||||
return _FakeLocator(count=1)
|
||||
|
||||
uploader._loc = locate
|
||||
monkeypatch.setattr(tmall.time, "sleep", lambda _seconds: None)
|
||||
|
||||
def before_submit(payload: Any) -> None:
|
||||
assert payload["video_file_name"] == "video.mp4"
|
||||
events.append("before-submit")
|
||||
|
||||
published = uploader.publish_video(
|
||||
tmp_path / "video.mp4",
|
||||
["极星双肩"],
|
||||
before_submit=before_submit,
|
||||
)
|
||||
|
||||
assert published is True
|
||||
assert events == ["before-submit", "publish-click"]
|
||||
assert upload_timeouts == [tmall.TMALL_VIDEO_UPLOAD_TIMEOUT_SECONDS]
|
||||
|
||||
|
||||
class _SeedCursor:
|
||||
def __init__(self) -> None:
|
||||
self.executions: list[tuple[str, tuple[Any, ...]]] = []
|
||||
|
||||
def __enter__(self) -> _SeedCursor:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
def execute(self, sql: str, params: tuple[Any, ...]) -> None:
|
||||
self.executions.append((sql, params))
|
||||
|
||||
def fetchone(self) -> tuple[int]:
|
||||
return (1,)
|
||||
|
||||
|
||||
class _SeedConnection:
|
||||
def __init__(self) -> None:
|
||||
self.cursor_instance = _SeedCursor()
|
||||
|
||||
def cursor(self) -> _SeedCursor:
|
||||
return self.cursor_instance
|
||||
|
||||
|
||||
def test_legacy_seed_inserts_published_without_overwriting_conflicts() -> None:
|
||||
connection = _SeedConnection()
|
||||
candidate = _one_candidate()
|
||||
|
||||
inserted = tmall.seed_legacy_published_candidates(
|
||||
connection,
|
||||
[(candidate, {"evidence": {"publish_success_line": 72}})],
|
||||
run_id="test-run",
|
||||
)
|
||||
|
||||
assert inserted == 1
|
||||
sql, params = connection.cursor_instance.executions[0]
|
||||
normalized_sql = " ".join(sql.split())
|
||||
assert "'published'" in normalized_sql
|
||||
assert "ON CONFLICT (" in normalized_sql
|
||||
assert ") DO NOTHING RETURNING id" in normalized_sql
|
||||
assert params[-3:-1] == ("test-run", "test-run")
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import get_type_hints
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.product_commerce.jd_video_publication_db import (
|
||||
CHANNEL_TMALL,
|
||||
VIDEO_PUBLICATION_DDL,
|
||||
ClaimedVideoPublication,
|
||||
VideoPublicationCandidate,
|
||||
VideoPublicationState,
|
||||
claim_next_video_publication,
|
||||
finish_video_publication,
|
||||
finish_video_publication_rejected,
|
||||
load_video_publication_states,
|
||||
)
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, rows: list[tuple[object, ...]]) -> None:
|
||||
self.rows = rows
|
||||
self.executions: list[tuple[str, tuple[object, ...]]] = []
|
||||
self.rowcount = 1
|
||||
|
||||
def __enter__(self) -> FakeCursor:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
def execute(self, query: str, params: tuple[object, ...]) -> None:
|
||||
self.executions.append((query, params))
|
||||
|
||||
def fetchall(self) -> list[tuple[object, ...]]:
|
||||
return self.rows
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self, cursor: FakeCursor) -> None:
|
||||
self.fake_cursor = cursor
|
||||
|
||||
def cursor(self) -> FakeCursor:
|
||||
return self.fake_cursor
|
||||
|
||||
|
||||
def test_load_video_publication_states_scopes_query_and_maps_attachments() -> None:
|
||||
cursor = FakeCursor(
|
||||
[
|
||||
("rec_1", "video_1", 101, "published", False),
|
||||
("rec_2", "video_2", 102, "failed", True),
|
||||
]
|
||||
)
|
||||
|
||||
states = load_video_publication_states(
|
||||
FakeConnection(cursor),
|
||||
CHANNEL_TMALL,
|
||||
"tmall-luggage-store",
|
||||
"base_token",
|
||||
"table_id",
|
||||
)
|
||||
|
||||
assert cursor.executions[0][1] == (
|
||||
"tmall",
|
||||
"tmall-luggage-store",
|
||||
"base_token",
|
||||
"table_id",
|
||||
)
|
||||
assert "FROM fact_video_publications" in cursor.executions[0][0]
|
||||
assert states == {
|
||||
("rec_1", "video_1"): VideoPublicationState(101, "published", False),
|
||||
("rec_2", "video_2"): VideoPublicationState(102, "failed", True),
|
||||
}
|
||||
|
||||
|
||||
def test_load_video_publication_states_does_not_hide_database_errors() -> None:
|
||||
class MissingTableCursor(FakeCursor):
|
||||
def execute(self, query: str, params: tuple[object, ...]) -> None:
|
||||
raise RuntimeError("fact_video_publications does not exist")
|
||||
|
||||
with pytest.raises(RuntimeError, match="does not exist"):
|
||||
load_video_publication_states(
|
||||
FakeConnection(MissingTableCursor([])),
|
||||
CHANNEL_TMALL,
|
||||
"tmall-flagship-store",
|
||||
"base_token",
|
||||
"table_id",
|
||||
)
|
||||
|
||||
|
||||
def test_shared_ledger_preserves_missing_source_dates() -> None:
|
||||
assert get_type_hints(VideoPublicationCandidate)["source_record_date"] == date | None
|
||||
assert get_type_hints(ClaimedVideoPublication)["source_record_date"] == date | None
|
||||
assert "source_record_date DATE," in VIDEO_PUBLICATION_DDL
|
||||
assert "ALTER COLUMN source_record_date DROP NOT NULL;" in VIDEO_PUBLICATION_DDL
|
||||
claim_source = inspect.getsource(claim_next_video_publication)
|
||||
assert "source_snapshot ->> 'source_view_position'" in claim_source
|
||||
assert "source_record_date NULLS LAST" in claim_source
|
||||
|
||||
schema_path = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "db"
|
||||
/ "schema.sql"
|
||||
)
|
||||
schema = schema_path.read_text(encoding="utf-8")
|
||||
assert "source_record_date DATE," in schema
|
||||
assert "ALTER COLUMN source_record_date DROP NOT NULL;" in schema
|
||||
|
||||
|
||||
def _sample_claim() -> ClaimedVideoPublication:
|
||||
return ClaimedVideoPublication(
|
||||
publication_id=764,
|
||||
attempt_no=1,
|
||||
source_record_id="rec_1",
|
||||
source_record_date=None,
|
||||
video_file_token="video_1",
|
||||
video_file_name="video.mp4",
|
||||
cover_file_token=None,
|
||||
cover_file_name=None,
|
||||
source_snapshot={},
|
||||
publish_payload={},
|
||||
)
|
||||
|
||||
|
||||
def test_published_and_ambiguous_outcomes_are_not_retryable() -> None:
|
||||
published_cursor = FakeCursor([])
|
||||
finish_video_publication(
|
||||
FakeConnection(published_cursor),
|
||||
_sample_claim(),
|
||||
status="published",
|
||||
retryable=True,
|
||||
)
|
||||
assert published_cursor.executions[0][1][0:2] == ("published", False)
|
||||
|
||||
ambiguous_cursor = FakeCursor([])
|
||||
finish_video_publication(
|
||||
FakeConnection(ambiguous_cursor),
|
||||
_sample_claim(),
|
||||
status="ambiguous",
|
||||
retryable=True,
|
||||
)
|
||||
assert ambiguous_cursor.executions[0][1][0:2] == ("ambiguous", False)
|
||||
|
||||
|
||||
def test_clear_remote_rejection_is_retryable_but_keeps_submit_audit() -> None:
|
||||
cursor = FakeCursor([])
|
||||
finish_video_publication_rejected(
|
||||
FakeConnection(cursor),
|
||||
_sample_claim(),
|
||||
error="平台明确拒绝",
|
||||
evidence={"form_visible": True},
|
||||
)
|
||||
|
||||
params = cursor.executions[0][1]
|
||||
assert params[0:2] == ("failed", True)
|
||||
assert params[3] is True
|
||||
@@ -14,21 +14,17 @@ from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
)
|
||||
|
||||
|
||||
class _PlaywrightContext:
|
||||
class _ScraplingBrowserContext:
|
||||
def __init__(self, page: object) -> None:
|
||||
self.page = page
|
||||
|
||||
def __enter__(self):
|
||||
context = SimpleNamespace(
|
||||
pages=[self.page],
|
||||
new_page=lambda: self.page,
|
||||
close=lambda: None,
|
||||
)
|
||||
browser = SimpleNamespace(
|
||||
new_context=lambda **kwargs: context,
|
||||
close=lambda: None,
|
||||
)
|
||||
chromium = SimpleNamespace(launch=lambda **kwargs: browser)
|
||||
return SimpleNamespace(chromium=chromium)
|
||||
return SimpleNamespace(context=context)
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback) -> None:
|
||||
return None
|
||||
@@ -39,8 +35,8 @@ def test_jd_shop_collector_returns_nonzero_when_collection_fails(
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"sync_playwright",
|
||||
lambda: _PlaywrightContext(object()),
|
||||
"ScraplingBrowser",
|
||||
lambda **_kwargs: _ScraplingBrowserContext(object()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
@@ -88,8 +84,8 @@ def test_jd_peer_collector_returns_nonzero_when_collection_fails(
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"sync_playwright",
|
||||
lambda: _PlaywrightContext(object()),
|
||||
"ScraplingBrowser",
|
||||
lambda **_kwargs: _ScraplingBrowserContext(object()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
|
||||
@@ -254,6 +254,53 @@ class _LateShopIdentityPage:
|
||||
self.wait_calls += 1
|
||||
|
||||
|
||||
class _RetryNavigationPage(_LateShopIdentityPage):
|
||||
def __init__(self, *, failures: int, url: str | None = None) -> None:
|
||||
super().__init__()
|
||||
self.failures = failures
|
||||
self.goto_calls = 0
|
||||
if url is not None:
|
||||
self.url = url
|
||||
|
||||
def goto(self, *args: object, **kwargs: object) -> None:
|
||||
del args, kwargs
|
||||
self.goto_calls += 1
|
||||
if self.goto_calls <= self.failures:
|
||||
raise RuntimeError(
|
||||
"Page.goto: net::ERR_ABORTED at "
|
||||
f"{appeal.CAMPAIGN_SQUARE_URL} Call log: - navigating"
|
||||
)
|
||||
|
||||
|
||||
class _FeedbackTextControl:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text_value = text
|
||||
|
||||
def count(self) -> int:
|
||||
return 1
|
||||
|
||||
def nth(self, index: int) -> _FeedbackTextControl:
|
||||
assert index == 0
|
||||
return self
|
||||
|
||||
def is_visible(self, timeout: int | None = None) -> bool:
|
||||
del timeout
|
||||
return True
|
||||
|
||||
def inner_text(self, timeout: int | None = None) -> str:
|
||||
del timeout
|
||||
return self.text_value
|
||||
|
||||
|
||||
class _FeedbackPage:
|
||||
def __init__(self, items: list[_VisibleControl]) -> None:
|
||||
self.items = items
|
||||
|
||||
def locator(self, selector: str) -> _LocatorCollection:
|
||||
assert "toast" in selector
|
||||
return _LocatorCollection(self.items)
|
||||
|
||||
|
||||
class _FakeClock:
|
||||
def __init__(self) -> None:
|
||||
self.now = 0.0
|
||||
@@ -319,6 +366,38 @@ class _DynamicPanel:
|
||||
return _EmptyLocator()
|
||||
|
||||
|
||||
class _TransientTextPanel(_DynamicPanel):
|
||||
def __init__(
|
||||
self,
|
||||
values: tuple[str, ...],
|
||||
*,
|
||||
transient_failures: int,
|
||||
) -> None:
|
||||
super().__init__(values, action_count=0)
|
||||
self.transient_failures = transient_failures
|
||||
|
||||
def inner_text(self, timeout: int | None = None) -> str:
|
||||
del timeout
|
||||
self.read_calls += 1
|
||||
if self.transient_failures > 0:
|
||||
self.transient_failures -= 1
|
||||
raise RuntimeError("drawer locator detached during transition")
|
||||
return next(self.values, self.latest)
|
||||
|
||||
|
||||
class _DetachedFeedbackControl(_TransientTextPanel):
|
||||
def count(self) -> int:
|
||||
return 1
|
||||
|
||||
def nth(self, index: int) -> _DetachedFeedbackControl:
|
||||
assert index == 0
|
||||
return self
|
||||
|
||||
def is_visible(self, timeout: int | None = None) -> bool:
|
||||
del timeout
|
||||
return True
|
||||
|
||||
|
||||
class _PanelNode(_VisibleControl):
|
||||
def __init__(self, name: str, box: dict[str, float]) -> None:
|
||||
super().__init__()
|
||||
@@ -465,6 +544,90 @@ def test_shop_identity_can_render_after_campaign_content(
|
||||
assert page.wait_calls == 1
|
||||
|
||||
|
||||
def test_navigation_recovers_after_first_abort(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
page = _RetryNavigationPage(failures=1)
|
||||
monkeypatch.setattr(
|
||||
appeal,
|
||||
"_body_text",
|
||||
lambda page: "活动广场 待办事项 待改价 9 光影行星箱包旗舰店",
|
||||
)
|
||||
|
||||
appeal._navigate_to_campaign_square( # noqa: SLF001
|
||||
page,
|
||||
expected_shop_name=appeal.EXPECTED_SHOP_NAME,
|
||||
)
|
||||
|
||||
assert page.goto_calls == 2
|
||||
assert page.wait_calls == 1
|
||||
|
||||
|
||||
def test_navigation_aborted_to_login_page_reports_session_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
page = _RetryNavigationPage(
|
||||
failures=2,
|
||||
url=(
|
||||
"https://fxg.jinritemai.com/login/common"
|
||||
"?extra=%7B%22target_url%22%3A%22https%3A%2F%2F"
|
||||
"fxg.jinritemai.com%2Fffa%2Fmerchant%2Fcampaign-square%22%7D"
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(appeal, "_body_text", lambda page: "")
|
||||
|
||||
with pytest.raises(appeal.PriceAppealSessionError, match="Cookie 已失效"):
|
||||
appeal._navigate_to_campaign_square( # noqa: SLF001
|
||||
page,
|
||||
expected_shop_name=appeal.EXPECTED_SHOP_NAME,
|
||||
)
|
||||
|
||||
assert page.goto_calls == 2
|
||||
|
||||
|
||||
def test_navigation_aborted_twice_reports_navigation_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
page = _RetryNavigationPage(failures=2)
|
||||
monkeypatch.setattr(appeal, "_body_text", lambda page: "")
|
||||
|
||||
with pytest.raises(appeal.PriceAppealError, match="活动广场导航失败") as exc:
|
||||
appeal._navigate_to_campaign_square( # noqa: SLF001
|
||||
page,
|
||||
expected_shop_name=appeal.EXPECTED_SHOP_NAME,
|
||||
)
|
||||
|
||||
assert page.goto_calls == 2
|
||||
assert "ERR_ABORTED" in str(exc.value)
|
||||
|
||||
|
||||
def test_pending_content_signature_returns_none_while_drawer_detached(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
clock = _FakeClock()
|
||||
panel = _TransientTextPanel(
|
||||
("待改价 报名信息 商品 ID: 12345678",),
|
||||
transient_failures=100,
|
||||
)
|
||||
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
|
||||
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
|
||||
|
||||
assert appeal._pending_content_signature(panel) is None # noqa: SLF001
|
||||
|
||||
|
||||
def test_visible_feedback_texts_skips_detached_feedback_node(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
clock = _FakeClock()
|
||||
detached = _DetachedFeedbackControl(("提交成功",), transient_failures=100)
|
||||
healthy = _FeedbackTextControl(" 提交成功 ")
|
||||
page = _FeedbackPage([detached, healthy])
|
||||
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
|
||||
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
|
||||
|
||||
assert appeal._visible_feedback_texts(page) == ("提交成功",)
|
||||
|
||||
|
||||
def test_pending_count_can_render_after_campaign_content(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
@@ -801,6 +964,111 @@ def test_item_panel_waits_for_async_product_and_sku_content(
|
||||
assert page.wait_calls == 2
|
||||
|
||||
|
||||
def test_item_panel_reacquires_after_transient_drawer_detach(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
clock = _FakeClock()
|
||||
page = _PendingMetricPage([], clock=clock)
|
||||
expected_item_id = "3774363265803616739"
|
||||
detached_panel = _TransientTextPanel(
|
||||
("选择申诉SKU",),
|
||||
transient_failures=1,
|
||||
)
|
||||
ready_panel = _DynamicPanel(
|
||||
(
|
||||
"选择申诉SKU 商品 ID: 3774363265803616739 "
|
||||
"SKU规格 ID: 3619462493048834 原价 ¥839",
|
||||
),
|
||||
action_count=0,
|
||||
)
|
||||
panels = iter((detached_panel, ready_panel))
|
||||
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
|
||||
monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False)
|
||||
monkeypatch.setattr(
|
||||
appeal,
|
||||
"_find_active_panel",
|
||||
lambda page, markers: next(panels, ready_panel),
|
||||
)
|
||||
|
||||
result = appeal._wait_for_item_panel_ready( # noqa: SLF001
|
||||
page,
|
||||
("选择申诉SKU",),
|
||||
expected_item_id=expected_item_id,
|
||||
timeout_seconds=2,
|
||||
)
|
||||
|
||||
assert result is ready_panel
|
||||
assert detached_panel.read_calls == 2
|
||||
assert ready_panel.read_calls == 1
|
||||
assert page.wait_calls == 1
|
||||
|
||||
|
||||
def test_item_panel_persistent_read_failure_still_fails_closed(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
clock = _FakeClock()
|
||||
page = _PendingMetricPage([], clock=clock)
|
||||
panel = _TransientTextPanel(
|
||||
("选择申诉SKU",),
|
||||
transient_failures=100,
|
||||
)
|
||||
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
|
||||
monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False)
|
||||
monkeypatch.setattr(
|
||||
appeal,
|
||||
"_find_active_panel",
|
||||
lambda page, markers: panel,
|
||||
)
|
||||
|
||||
with pytest.raises(appeal.PriceAppealError, match="未加载目标商品 ID") as exc:
|
||||
appeal._wait_for_item_panel_ready( # noqa: SLF001
|
||||
page,
|
||||
("选择申诉SKU",),
|
||||
expected_item_id="3774363265803616739",
|
||||
timeout_seconds=1,
|
||||
)
|
||||
|
||||
assert isinstance(exc.value.__cause__, appeal.PriceAppealError)
|
||||
assert str(exc.value.__cause__) == "无法读取活动抽屉内容"
|
||||
|
||||
|
||||
def test_verify_item_identity_recovers_after_transient_detach(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
clock = _FakeClock()
|
||||
panel = _TransientTextPanel(
|
||||
(
|
||||
"选择申诉SKU 商品 ID: 3774363265803616739 "
|
||||
"SKU规格 ID: 3619462493048834 原价 ¥839",
|
||||
),
|
||||
transient_failures=1,
|
||||
)
|
||||
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
|
||||
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
|
||||
|
||||
appeal._verify_item_identity(panel, "3774363265803616739") # noqa: SLF001
|
||||
|
||||
assert panel.read_calls == 2
|
||||
|
||||
|
||||
def test_verify_item_identity_persistent_read_failure_fails_closed(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
clock = _FakeClock()
|
||||
panel = _TransientTextPanel(
|
||||
("选择申诉SKU",),
|
||||
transient_failures=100,
|
||||
)
|
||||
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
|
||||
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
|
||||
|
||||
with pytest.raises(appeal.PriceAppealError, match="与待处理行不一致") as exc:
|
||||
appeal._verify_item_identity(panel, "3774363265803616739") # noqa: SLF001
|
||||
|
||||
assert isinstance(exc.value.__cause__, appeal.PriceAppealError)
|
||||
assert str(exc.value.__cause__) == "无法读取活动抽屉内容"
|
||||
|
||||
|
||||
def test_pending_panel_with_headers_only_never_becomes_ready(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
@@ -67,6 +67,37 @@ def test_market_category_option_uses_visible_titled_menu_item() -> None:
|
||||
assert option.clicked
|
||||
|
||||
|
||||
class _UnstableOption(_Locator):
|
||||
def scroll_into_view_if_needed(self, **_kwargs) -> None:
|
||||
return None
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
raise TimeoutError(
|
||||
"Locator.click: Timeout 5000ms exceeded ... element is not stable"
|
||||
)
|
||||
|
||||
|
||||
def test_market_category_option_falls_back_to_native_click_when_unstable() -> None:
|
||||
option = _UnstableOption()
|
||||
page = SimpleNamespace(
|
||||
locator=lambda _selector: option,
|
||||
evaluate=lambda *_args, **_kwargs: True,
|
||||
)
|
||||
|
||||
assert collector._click_market_category_option(page, "双肩包")
|
||||
assert option.waited
|
||||
|
||||
|
||||
def test_market_category_option_fails_closed_when_native_click_also_fails() -> None:
|
||||
option = _UnstableOption()
|
||||
page = SimpleNamespace(
|
||||
locator=lambda _selector: option,
|
||||
evaluate=lambda *_args, **_kwargs: False,
|
||||
)
|
||||
|
||||
assert not collector._click_market_category_option(page, "双肩包")
|
||||
|
||||
|
||||
def test_select_market_category_requires_path_and_fresh_table(monkeypatch) -> None:
|
||||
expected = collector._expected_market_category_path("双肩包")
|
||||
paths = iter([collector._expected_market_category_path("腰包/胸包"), expected])
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import jd_data_collector as collector
|
||||
|
||||
|
||||
class _FlakyCellLocator:
|
||||
def __init__(self) -> None:
|
||||
self.click_calls = 0
|
||||
|
||||
def scroll_into_view_if_needed(self, **_kwargs) -> None:
|
||||
return None
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.click_calls += 1
|
||||
raise TimeoutError("element is not visible")
|
||||
|
||||
|
||||
class _DatePickerPage:
|
||||
"""Stub page: date already unset, month aligned, cell found, then results."""
|
||||
|
||||
def __init__(self, native_click_result: bool) -> None:
|
||||
self.cell = _FlakyCellLocator()
|
||||
self.native_click_result = native_click_result
|
||||
self.calls = 0
|
||||
self.native_click_script: str | None = None
|
||||
|
||||
def evaluate(self, script: str, arg: object = None):
|
||||
del arg
|
||||
self.calls += 1
|
||||
if self.calls == 1:
|
||||
assert "textContent" in script # already-set check
|
||||
return False
|
||||
if self.calls == 2:
|
||||
assert "targetYear" in script # month navigation
|
||||
return {"status": "aligned"}
|
||||
if self.calls == 3:
|
||||
assert "gridcell" in script # locate target day cell
|
||||
return {"status": "found_cell", "id": "__jd_target"}
|
||||
assert self.calls == 4
|
||||
assert "document.getElementById" in script # native click fallback
|
||||
self.native_click_script = script
|
||||
return self.native_click_result
|
||||
|
||||
def locator(self, selector: str) -> _FlakyCellLocator:
|
||||
assert selector == "#__jd_target"
|
||||
return self.cell
|
||||
|
||||
def wait_for_timeout(self, _milliseconds: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _prepare(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"get_last_week_range",
|
||||
lambda: (date(2026, 8, 10), date(2026, 8, 16)),
|
||||
)
|
||||
monkeypatch.setattr(collector, "save_date_picker_debug", lambda *_args, **_kwargs: None)
|
||||
|
||||
|
||||
def test_old_datepicker_cell_click_falls_back_to_native_click(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_prepare(monkeypatch)
|
||||
page = _DatePickerPage(native_click_result=True)
|
||||
|
||||
collector.select_last_week_in_old_datepicker(page)
|
||||
|
||||
assert page.cell.click_calls == 1
|
||||
assert page.native_click_script is not None
|
||||
assert page.calls == 4
|
||||
|
||||
|
||||
def test_old_datepicker_fails_closed_when_native_click_also_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_prepare(monkeypatch)
|
||||
page = _DatePickerPage(native_click_result=False)
|
||||
|
||||
with pytest.raises(RuntimeError, match="旧版日期控件未能选择上周日期"):
|
||||
collector.select_last_week_in_old_datepicker(page)
|
||||
|
||||
assert page.cell.click_calls == 1
|
||||
assert page.native_click_script is not None
|
||||
|
||||
|
||||
def test_old_datepicker_happy_path_uses_playwright_click(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_prepare(monkeypatch)
|
||||
page = _DatePickerPage(native_click_result=True)
|
||||
|
||||
class StableCell(_FlakyCellLocator):
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.click_calls += 1
|
||||
|
||||
page.cell = StableCell()
|
||||
|
||||
collector.select_last_week_in_old_datepicker(page)
|
||||
|
||||
assert page.cell.click_calls == 1
|
||||
assert page.native_click_script is None
|
||||
@@ -48,6 +48,7 @@ def test_category_menu_closes_stale_portals_before_opening_selector() -> None:
|
||||
class Trigger:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
self.first = self
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.page.menu_count = 1
|
||||
@@ -255,3 +256,129 @@ def test_failed_category_switch_never_extracts_previous_table(
|
||||
assert "类目切换或榜单刷新失败" in result["男士双肩包"]["error"]
|
||||
assert result["男士单肩/斜挎包"]["rank"] == "7"
|
||||
assert extracted == ["called"]
|
||||
|
||||
|
||||
class _SearchBox:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[str] = []
|
||||
|
||||
def is_visible(self, **_kwargs) -> bool:
|
||||
return True
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.calls.append("click")
|
||||
|
||||
def fill(self, value: str, **_kwargs) -> None:
|
||||
self.calls.append(f"fill:{value}")
|
||||
|
||||
def press(self, key: str, **_kwargs) -> None:
|
||||
self.calls.append(f"press:{key}")
|
||||
|
||||
|
||||
class _OneLocator:
|
||||
def __init__(self, item: _SearchBox) -> None:
|
||||
self.item = item
|
||||
|
||||
@property
|
||||
def first(self) -> _OneLocator:
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return 1
|
||||
|
||||
def is_visible(self, **_kwargs) -> bool:
|
||||
return self.item.is_visible()
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.item.click()
|
||||
|
||||
def fill(self, value: str, **_kwargs) -> None:
|
||||
self.item.fill(value)
|
||||
|
||||
def press(self, key: str, **_kwargs) -> None:
|
||||
self.item.press(key)
|
||||
|
||||
|
||||
class _EmptyLocator:
|
||||
@property
|
||||
def first(self) -> _EmptyLocator:
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return 0
|
||||
|
||||
def is_visible(self, **_kwargs) -> bool:
|
||||
raise TimeoutError("no elements matched")
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
raise TimeoutError("no elements matched")
|
||||
|
||||
|
||||
class _SearchPage:
|
||||
def __init__(self, box: _SearchBox, *, section_result: str | None) -> None:
|
||||
self.box = box
|
||||
self.section_result = section_result
|
||||
self.evaluate_calls = 0
|
||||
|
||||
def evaluate(self, _script: str, _arg: object = None):
|
||||
self.evaluate_calls += 1
|
||||
return self.section_result
|
||||
|
||||
def locator(self, selector: str) -> _OneLocator | _EmptyLocator:
|
||||
if selector.startswith("#__peer_search_input"):
|
||||
return _OneLocator(self.box)
|
||||
if selector == "input":
|
||||
return _OneLocator(self.box)
|
||||
if "店铺" in selector:
|
||||
return _OneLocator(self.box)
|
||||
return _EmptyLocator()
|
||||
|
||||
def wait_for_timeout(self, _milliseconds: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_search_shop_prefers_trade_rank_section_input() -> None:
|
||||
box = _SearchBox()
|
||||
page = _SearchPage(box, section_result="__peer_search_input_1")
|
||||
|
||||
collector.search_shop(page, "Bellroy")
|
||||
|
||||
assert page.evaluate_calls == 1
|
||||
assert box.calls == [
|
||||
"click",
|
||||
"fill:",
|
||||
"fill:Bellroy",
|
||||
"press:Enter",
|
||||
]
|
||||
|
||||
|
||||
def test_search_shop_falls_back_to_legacy_selectors_without_section() -> None:
|
||||
box = _SearchBox()
|
||||
page = _SearchPage(box, section_result=None)
|
||||
|
||||
collector.search_shop(page, "Bellroy")
|
||||
|
||||
assert page.evaluate_calls == 1
|
||||
assert box.calls == [
|
||||
"click",
|
||||
"fill:",
|
||||
"fill:Bellroy",
|
||||
"press:Enter",
|
||||
]
|
||||
|
||||
|
||||
def test_locate_peer_search_input_returns_none_when_evaluate_fails() -> None:
|
||||
box = _SearchBox()
|
||||
|
||||
class FailingPage(_SearchPage):
|
||||
def evaluate(self, _script: str, _arg: object = None):
|
||||
raise RuntimeError("page crashed")
|
||||
|
||||
assert collector._locate_peer_search_input(FailingPage(box, section_result="x")) is None
|
||||
|
||||
|
||||
def test_locate_peer_search_input_returns_none_without_section() -> None:
|
||||
box = _SearchBox()
|
||||
page = _SearchPage(box, section_result=None)
|
||||
|
||||
assert collector._locate_peer_search_input(page) is None
|
||||
|
||||
@@ -27,7 +27,7 @@ def test_browser_runtime_defaults_to_bundled_chromium(monkeypatch):
|
||||
def test_browser_runtime_can_use_installed_chrome(monkeypatch):
|
||||
monkeypatch.setenv("GYXX_BROWSER_CHANNEL", "chrome")
|
||||
|
||||
assert browser_runtime_options() == {"channel": "chrome"}
|
||||
assert browser_runtime_options() == {"real_chrome": True}
|
||||
|
||||
|
||||
class DateRangeTests(unittest.TestCase):
|
||||
|
||||
@@ -455,6 +455,36 @@ def test_shop_nonpositive_persistence_stops_feishu_write(
|
||||
assert errors == [f"{platform_code} DB 持久化失败: 未写入任何记录"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "persist_name"),
|
||||
[
|
||||
(run_shop.JD_PLATFORM, "persist_jd_shop"),
|
||||
(run_shop.DY_PLATFORM, "persist_dy_shop"),
|
||||
],
|
||||
)
|
||||
def test_shop_skip_feishu_preserves_database_write_without_calling_writer(
|
||||
monkeypatch,
|
||||
platform: str,
|
||||
persist_name: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(run_shop, "ensure_ai_analysis", lambda *_args: "")
|
||||
monkeypatch.setattr(run_shop, persist_name, lambda _path: 1)
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"write_shop",
|
||||
lambda *_args: pytest.fail("weekly shop workflow must not write Feishu"),
|
||||
)
|
||||
|
||||
errors = run_shop._persist_and_write(
|
||||
platform,
|
||||
True,
|
||||
"D:/fresh-shop.json",
|
||||
skip_feishu=True,
|
||||
)
|
||||
|
||||
assert errors == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "persist_name", "platform_code"),
|
||||
[
|
||||
@@ -625,19 +655,16 @@ def test_jd_shop_uses_bound_profile_port_and_persists_state(
|
||||
|
||||
context = Context()
|
||||
|
||||
class Chromium:
|
||||
def launch_persistent_context(self, **kwargs):
|
||||
class Scrapling:
|
||||
def __init__(self, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return context
|
||||
|
||||
class Playwright:
|
||||
chromium = Chromium()
|
||||
self.context = context
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
context.close()
|
||||
|
||||
cookie_file = tmp_path / "state" / "cookies.json"
|
||||
storage_file = tmp_path / "state" / "storage.json"
|
||||
@@ -646,7 +673,7 @@ def test_jd_shop_uses_bound_profile_port_and_persists_state(
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22105")
|
||||
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
|
||||
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
|
||||
monkeypatch.setattr(jd_data_collector, "sync_playwright", Playwright)
|
||||
monkeypatch.setattr(jd_data_collector, "ScraplingBrowser", Scrapling)
|
||||
monkeypatch.setattr(jd_data_collector, "step_login", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
@@ -670,8 +697,13 @@ def test_jd_shop_uses_bound_profile_port_and_persists_state(
|
||||
|
||||
assert jd_data_collector.main(["--password", "test-only"]) == 0
|
||||
assert observed["user_data_dir"] == str(profile)
|
||||
assert "--remote-debugging-port=22105" in observed["args"]
|
||||
assert "--no-proxy-server" in observed["args"]
|
||||
assert "--remote-debugging-port=22105" in observed["extra_flags"]
|
||||
assert "--no-proxy-server" in observed["extra_flags"]
|
||||
assert observed["additional_args"]["viewport"] == {
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
}
|
||||
assert observed["retries"] == 1
|
||||
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "session"
|
||||
assert storage_file.is_file()
|
||||
assert context.closed is True
|
||||
@@ -699,19 +731,16 @@ def test_jd_peer_uses_bound_profile_port_and_persists_state(
|
||||
|
||||
context = Context()
|
||||
|
||||
class Chromium:
|
||||
def launch_persistent_context(self, **kwargs):
|
||||
class Scrapling:
|
||||
def __init__(self, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return context
|
||||
|
||||
class Playwright:
|
||||
chromium = Chromium()
|
||||
self.context = context
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
context.close()
|
||||
|
||||
cookie_file = tmp_path / "state" / "cookies.json"
|
||||
storage_file = tmp_path / "state" / "storage.json"
|
||||
@@ -720,7 +749,11 @@ def test_jd_peer_uses_bound_profile_port_and_persists_state(
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22106")
|
||||
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
|
||||
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
|
||||
monkeypatch.setattr(jd_peer_store_data_collector, "sync_playwright", Playwright)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"ScraplingBrowser",
|
||||
Scrapling,
|
||||
)
|
||||
monkeypatch.setattr(jd_peer_store_data_collector, "step_login", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
@@ -765,7 +798,12 @@ def test_jd_peer_uses_bound_profile_port_and_persists_state(
|
||||
|
||||
assert jd_peer_store_data_collector.main(["--password", "test-only"]) == 0
|
||||
assert observed["user_data_dir"] == str(profile)
|
||||
assert "--remote-debugging-port=22106" in observed["args"]
|
||||
assert "--remote-debugging-port=22106" in observed["extra_flags"]
|
||||
assert observed["additional_args"]["viewport"] == {
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
}
|
||||
assert observed["retries"] == 1
|
||||
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "peer-session"
|
||||
assert storage_file.is_file()
|
||||
assert context.closed is True
|
||||
|
||||
@@ -133,6 +133,12 @@ TARGET_AST_SHA256_OVERRIDES = {
|
||||
("jd_data_collector.py", "select_last_week_any_day"): (
|
||||
"8f6aed11b39d8e19ec4146e9b32eee96f5977fdbef694311534346bacec5d847"
|
||||
),
|
||||
# Reviewed after production runs showed Playwright's actionability check
|
||||
# rejecting the legacy date-picker cell ("element is not visible") while the
|
||||
# panel re-renders; a native click on the same node is the approved fallback.
|
||||
("jd_data_collector.py", "select_last_week_in_old_datepicker"): (
|
||||
"43d1056eeac1faeff32f0c01d0867e6673d2ab57999a14a3ebeb1b51c40075d4"
|
||||
),
|
||||
("jd_data_collector.py", "step_shop_star_and_trade"): (
|
||||
"77aa873543861f1fad63d0bd1ace60528ab20592b0bffef3d46ad192f2103161"
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user