feat: expand module workflows, dynamic config, notifications and console API
This commit is contained in:
@@ -3,11 +3,11 @@ from __future__ import annotations
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
COMMAND_PATH = (
|
||||
@@ -56,7 +56,7 @@ def test_alert_command_uses_canonical_acceptance_recipient(
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID,
|
||||
ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID,
|
||||
)
|
||||
|
||||
|
||||
@@ -72,3 +72,52 @@ def test_alert_command_accepts_canonical_production_recipient(
|
||||
|
||||
assert module.main() == 0
|
||||
assert observed["argv"][-1] == "ou_configured"
|
||||
|
||||
|
||||
def test_alert_command_passes_all_dynamic_recipients(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01")
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
module = _load_command(monkeypatch, observed)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"resolve_notification_route",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
enabled=True,
|
||||
open_ids=("ou_first", "ou_second"),
|
||||
),
|
||||
)
|
||||
|
||||
assert module.main() == 0
|
||||
assert observed["argv"] == (
|
||||
sys.argv[0],
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
"ou_first",
|
||||
"--openid",
|
||||
"ou_second",
|
||||
)
|
||||
|
||||
|
||||
def test_alert_command_uses_safe_no_notify_when_route_is_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01")
|
||||
module = _load_command(monkeypatch, observed)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"resolve_notification_route",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(enabled=False, open_ids=()),
|
||||
)
|
||||
|
||||
assert module.main() == 0
|
||||
assert observed["argv"] == (
|
||||
sys.argv[0],
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--no-notify",
|
||||
)
|
||||
|
||||
@@ -53,6 +53,11 @@ def _prepare_main(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
)
|
||||
monkeypatch.setattr(decline, "managed_data_path", lambda value: Path(value))
|
||||
monkeypatch.setattr(decline, "append_log", lambda line: None)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_persist_detected_events",
|
||||
lambda events, *_args: len(events),
|
||||
)
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
|
||||
|
||||
@@ -142,27 +147,73 @@ def test_segment_loader_uses_postgres_range_sum_on_one_connection(
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("response", "expected"),
|
||||
[
|
||||
({"id": "chatcmpl-not-a-receipt"}, None),
|
||||
({"message_id": "om_direct123"}, "om_direct123"),
|
||||
({"data": {"message_id": "om_nested123"}}, "om_nested123"),
|
||||
(
|
||||
{"choices": [{"message": {"content": "飞书回执 om_content123"}}]},
|
||||
"om_content123",
|
||||
),
|
||||
(
|
||||
{"choices": [{"message": {"content": "发送成功,但没有回执"}}]},
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_feishu_message_id_requires_real_receipt_shape(
|
||||
response: dict,
|
||||
expected: str | None,
|
||||
def test_build_decline_notification_message_is_deterministic() -> None:
|
||||
message = decline.build_decline_notification_message(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
)
|
||||
|
||||
assert "销量连续下滑告警(截至 2026-08-01,3 段 × 3 天)" in message
|
||||
assert "[tm] 测试款" in message
|
||||
assert "300 → 210 → 90" in message
|
||||
assert "累计 -70.0%" in message
|
||||
assert "ERP" in message
|
||||
|
||||
|
||||
def test_alert_delivery_uuid_is_stable_and_recipient_scoped() -> None:
|
||||
first = decline._notification_delivery_uuid(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
openid="ou_first",
|
||||
)
|
||||
same = decline._notification_delivery_uuid(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
openid="ou_first",
|
||||
)
|
||||
second = decline._notification_delivery_uuid(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
openid="ou_second",
|
||||
)
|
||||
|
||||
assert first == same
|
||||
assert first != second
|
||||
|
||||
|
||||
def test_decline_notification_uses_shared_lark_user_sender(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
assert decline.extract_feishu_message_id(response) == expected
|
||||
captured: dict[str, object] = {}
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
|
||||
def fake_send(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"message_id": "om_real123"}
|
||||
|
||||
monkeypatch.setattr(decline, "send_lark_bot_message", fake_send)
|
||||
|
||||
response = decline.send_decline_notification(
|
||||
[_event()],
|
||||
"2026-08-01",
|
||||
9,
|
||||
openid="ou_owner",
|
||||
)
|
||||
|
||||
assert response["message_id"] == "om_real123"
|
||||
assert captured["user_id"] == "ou_owner"
|
||||
assert captured["profile"] == "hermes-analyzer"
|
||||
assert captured["idempotency_key"] == decline._notification_delivery_uuid(
|
||||
[_event()],
|
||||
end_date="2026-08-01",
|
||||
window_days=9,
|
||||
openid="ou_owner",
|
||||
)
|
||||
assert "测试款" in str(captured["text"])
|
||||
|
||||
|
||||
def test_main_fails_when_postgres_detection_read_fails(
|
||||
@@ -205,12 +256,12 @@ def test_main_dedup_query_failure_is_fail_closed(
|
||||
lambda *args: (_ for _ in ()).throw(RuntimeError("db timeout")),
|
||||
)
|
||||
notify = pytest.fail
|
||||
monkeypatch.setattr(decline, "notify_hermes", notify)
|
||||
monkeypatch.setattr(decline, "send_decline_notification", notify)
|
||||
|
||||
assert decline.main() == 5
|
||||
|
||||
|
||||
def test_main_does_not_accept_arbitrary_non_error_hermes_text(
|
||||
def test_main_does_not_accept_lark_response_without_real_message_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
@@ -219,11 +270,8 @@ def test_main_does_not_accept_arbitrary_non_error_hermes_text(
|
||||
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"notify_hermes",
|
||||
lambda *args, **kwargs: {
|
||||
"id": "chatcmpl-123",
|
||||
"choices": [{"message": {"content": "发送成功"}}],
|
||||
},
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: {"ok": True},
|
||||
)
|
||||
persist = pytest.fail
|
||||
monkeypatch.setattr(decline, "_persist_events", persist)
|
||||
@@ -240,7 +288,7 @@ def test_main_returns_nonzero_when_receipt_persistence_fails(
|
||||
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"notify_hermes",
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: {"message_id": "om_real123"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
@@ -263,7 +311,7 @@ def test_main_success_requires_and_persists_feishu_receipt(
|
||||
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"notify_hermes",
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: {"message_id": "om_real123"},
|
||||
)
|
||||
|
||||
@@ -281,6 +329,174 @@ def test_main_success_requires_and_persists_feishu_receipt(
|
||||
assert "persistence=committed history_rows=1" in output
|
||||
|
||||
|
||||
def test_main_no_notify_still_detects_hits_and_succeeds(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"check_nine_day_decline.py",
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--no-notify",
|
||||
"--data-root",
|
||||
str(tmp_path),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
persisted: list[tuple[list[dict], str, int]] = []
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_persist_detected_events",
|
||||
lambda events, end_date, window_days: (
|
||||
persisted.append((events, end_date, window_days)) or len(events)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_already_notified",
|
||||
lambda *args: pytest.fail("disabled notification must not query receipts"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: pytest.fail("disabled notification must not send"),
|
||||
)
|
||||
|
||||
assert decline.main() == 0
|
||||
assert persisted == [([_event()], "2026-08-01", 9)]
|
||||
output = capsys.readouterr().out
|
||||
assert "events=1 notification=disabled persistence=committed" in output
|
||||
assert "detection_rows=1" in output
|
||||
|
||||
|
||||
def test_main_blocks_notification_when_detection_output_cannot_be_persisted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_persist_detected_events",
|
||||
lambda *_args: (_ for _ in ()).throw(RuntimeError("write failed")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"send_decline_notification",
|
||||
lambda *args, **kwargs: pytest.fail("must not send without durable output"),
|
||||
)
|
||||
|
||||
assert decline.main() == 5
|
||||
|
||||
|
||||
def test_main_sends_and_persists_each_recipient_independently(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"check_nine_day_decline.py",
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
"ou_first",
|
||||
"--openid",
|
||||
"ou_second",
|
||||
"--data-root",
|
||||
str(tmp_path),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
dedup_queries: list[str] = []
|
||||
persisted: list[tuple[str, str]] = []
|
||||
|
||||
def fake_already_notified(_style, _date, recipient):
|
||||
dedup_queries.append(recipient)
|
||||
return False
|
||||
|
||||
def fake_notify(*_args, openid, **_kwargs):
|
||||
return {"message_id": f"om_{openid.removeprefix('ou_')}"}
|
||||
|
||||
def fake_persist(*_args, openid, message_id, **_kwargs):
|
||||
persisted.append((openid, message_id))
|
||||
return 1
|
||||
|
||||
monkeypatch.setattr(decline, "_already_notified", fake_already_notified)
|
||||
monkeypatch.setattr(decline, "send_decline_notification", fake_notify)
|
||||
monkeypatch.setattr(decline, "_persist_events", fake_persist)
|
||||
|
||||
assert decline.main() == 0
|
||||
assert dedup_queries == ["ou_first", "ou_second"]
|
||||
assert persisted == [
|
||||
("ou_first", "om_first"),
|
||||
("ou_second", "om_second"),
|
||||
]
|
||||
|
||||
|
||||
def test_alert_schema_and_upsert_are_recipient_scoped() -> None:
|
||||
repository_root = Path(__file__).resolve().parents[3]
|
||||
schema = (
|
||||
repository_root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "db"
|
||||
/ "schema.sql"
|
||||
).read_text(encoding="utf-8")
|
||||
db_module = (
|
||||
repository_root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "db"
|
||||
/ "__init__.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "UNIQUE (style_name, trigger_date, recipient)" in schema
|
||||
assert "DROP CONSTRAINT IF EXISTS fact_alert_history_style_name_trigger_date_key" in schema
|
||||
assert "ON CONFLICT (style_name, trigger_date, recipient)" in db_module
|
||||
assert "def ensure_alert_history_recipient_scope" in db_module
|
||||
assert "fact_alert_history_style_name_trigger_date_key" in db_module
|
||||
|
||||
|
||||
def test_detected_output_migrates_recipient_scope_before_upsert(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
fake_db = ModuleType("db")
|
||||
connection = object()
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
class ConnectionContext:
|
||||
def __enter__(self):
|
||||
return connection
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
fake_db.get_conn = ConnectionContext # type: ignore[attr-defined]
|
||||
fake_db.ensure_alert_history_recipient_scope = ( # type: ignore[attr-defined]
|
||||
lambda conn: calls.append(("migrate", conn))
|
||||
)
|
||||
fake_db.upsert_decline_event = ( # type: ignore[attr-defined]
|
||||
lambda conn, event: calls.append(("upsert", (conn, event)))
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "db", fake_db)
|
||||
|
||||
assert decline._persist_detected_events([_event()], "2026-08-01", 9) == 1
|
||||
assert calls[0] == ("migrate", connection)
|
||||
assert calls[1][0] == "upsert"
|
||||
|
||||
|
||||
class _FailedCollector:
|
||||
def __init__(self, *args):
|
||||
self.pages: list[tuple] = []
|
||||
|
||||
@@ -7,16 +7,10 @@ from types import SimpleNamespace
|
||||
|
||||
import collect_erp_yesterday_metrics as erp
|
||||
import collect_jd_persona_to_bitable as jd_persona
|
||||
from runtime_paths import playwright_launch_options
|
||||
import erp_login_product_analysis as erp_login
|
||||
from runtime_paths import scrapling_session_options
|
||||
|
||||
|
||||
class _Chromium:
|
||||
def __init__(self) -> None:
|
||||
self.kwargs = None
|
||||
|
||||
def launch_persistent_context(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
return object()
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_jd_persona_launches_the_managed_persistent_profile(
|
||||
@@ -25,13 +19,20 @@ def test_jd_persona_launches_the_managed_persistent_profile(
|
||||
) -> None:
|
||||
profile = tmp_path / "jd-profile"
|
||||
monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile))
|
||||
chromium = _Chromium()
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
context = jd_persona._launch_jd_context(chromium, headless=True)
|
||||
def browser_factory(**options):
|
||||
captured.update(options)
|
||||
return object()
|
||||
|
||||
assert context is not None
|
||||
assert chromium.kwargs["user_data_dir"] == str(profile.resolve())
|
||||
assert chromium.kwargs["headless"] is True
|
||||
monkeypatch.setattr(jd_persona, "ScraplingBrowser", browser_factory)
|
||||
|
||||
browser = jd_persona._jd_browser(headless=True)
|
||||
|
||||
assert browser is not None
|
||||
assert captured["user_data_dir"] == str(profile.resolve())
|
||||
assert captured["headless"] is True
|
||||
assert captured["retries"] == 1
|
||||
|
||||
|
||||
def test_jd_persona_reuses_logged_in_page_without_credentials() -> None:
|
||||
@@ -61,6 +62,36 @@ def test_jd_persona_reuses_logged_in_page_without_credentials() -> None:
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_jd_persona_waits_for_slow_shop_shell_before_login_fallback() -> None:
|
||||
calls: list[tuple[str, str]] = []
|
||||
bodies = iter(("", "", "", "", "京东商家后台 商品明细"))
|
||||
waits: list[int] = []
|
||||
|
||||
class Page:
|
||||
url = "https://shop.jd.com/jdm/home"
|
||||
|
||||
def goto(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
waits.append(milliseconds)
|
||||
|
||||
def locator(self, _selector):
|
||||
return SimpleNamespace(
|
||||
inner_text=lambda **_kwargs: next(bodies),
|
||||
)
|
||||
|
||||
jd_persona._ensure_jd_session(
|
||||
Page(),
|
||||
shop="",
|
||||
password="",
|
||||
step_login=lambda _page, shop, password: calls.append((shop, password)),
|
||||
)
|
||||
|
||||
assert calls == []
|
||||
assert waits == [2000, 1000, 1000, 1000, 1000]
|
||||
|
||||
|
||||
def test_erp_uses_managed_profile_when_bound_cdp_is_not_running(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
@@ -95,6 +126,62 @@ def test_erp_attaches_only_to_the_live_managed_cdp(monkeypatch) -> None:
|
||||
assert user_data_dir is None
|
||||
|
||||
|
||||
def test_erp_login_navigates_before_probing_initial_blank_page(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
events: list[tuple[str, str]] = []
|
||||
|
||||
class Page:
|
||||
url = "about:blank"
|
||||
context = object()
|
||||
|
||||
def goto(self, url, **_kwargs):
|
||||
events.append(("goto", url))
|
||||
self.url = url
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(erp_login, "restore_erp_runtime_state", lambda _context: None)
|
||||
monkeypatch.setattr(erp_login, "has_valid_session", lambda _page: False)
|
||||
monkeypatch.setattr(erp_login, "is_authenticated_session", lambda _page: False)
|
||||
monkeypatch.setattr(erp_login, "first_visible", lambda *_args: object())
|
||||
monkeypatch.setattr(
|
||||
erp_login,
|
||||
"fill_first_visible",
|
||||
lambda _page, _selectors, _value, label: events.append(("fill", label)) or True,
|
||||
)
|
||||
monkeypatch.setattr(erp_login, "check_visible_checkboxes", lambda _page: None)
|
||||
monkeypatch.setattr(erp_login, "click_login_button", lambda _page: events.append(("click", "login")))
|
||||
monkeypatch.setattr(erp_login, "wait_for_manual_captcha", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr(erp_login, "wait_until_logged_in", lambda *_args, **_kwargs: True)
|
||||
monkeypatch.setattr(erp_login, "persist_erp_runtime_state", lambda _context: 1)
|
||||
|
||||
state: dict[str, object] = {}
|
||||
action = erp_login.make_page_action(
|
||||
login_url="https://erp.test/login.aspx",
|
||||
username="user",
|
||||
password="pw",
|
||||
home_url="https://erp.test/epaas",
|
||||
timeout_ms=1000,
|
||||
captcha_checks=0,
|
||||
captcha_interval_seconds=1,
|
||||
hold_seconds=0,
|
||||
login_only=True,
|
||||
state=state,
|
||||
)
|
||||
|
||||
action(Page())
|
||||
|
||||
assert events[:4] == [
|
||||
("goto", "https://erp.test/login.aspx"),
|
||||
("fill", "username"),
|
||||
("fill", "password"),
|
||||
("click", "login"),
|
||||
]
|
||||
assert state == {"ok": True, "final_url": "https://erp.test/login.aspx"}
|
||||
|
||||
|
||||
def test_erp_main_fails_before_browser_when_no_style_is_collectable(monkeypatch) -> None:
|
||||
args = SimpleNamespace(
|
||||
config="unused.json",
|
||||
@@ -192,6 +279,110 @@ def test_erp_main_fails_when_collector_summary_contains_failures(
|
||||
assert erp.main() == 1
|
||||
|
||||
|
||||
def test_erp_main_maps_login_recovery_to_cookie_skip(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
args = SimpleNamespace(
|
||||
config="unused.json",
|
||||
target_date=date(2026, 8, 12),
|
||||
mode="daily",
|
||||
refresh_config=False,
|
||||
input="unused-styles.json",
|
||||
data_dir=str(tmp_path),
|
||||
platform="all",
|
||||
no_cdp=False,
|
||||
timeout_ms=1000,
|
||||
limit=0,
|
||||
report_timeout=1,
|
||||
report_attempts=1,
|
||||
include_slow_codes=False,
|
||||
hold_seconds=0,
|
||||
headless=True,
|
||||
)
|
||||
styles = [
|
||||
{
|
||||
"style_name": "style-a",
|
||||
"erp_style_codes": ["erp-a"],
|
||||
"brand": erp.DEFAULT_BRAND,
|
||||
}
|
||||
]
|
||||
monkeypatch.setattr(erp, "parse_args", lambda: args)
|
||||
monkeypatch.setattr(
|
||||
erp,
|
||||
"load_config",
|
||||
lambda _path: {
|
||||
"login": {
|
||||
"url": "https://erp.test/login",
|
||||
"username": "placeholder",
|
||||
"password": "placeholder",
|
||||
}
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
erp,
|
||||
"load_erp_styles",
|
||||
lambda *_args: (
|
||||
styles,
|
||||
{"source": "lark", "collectable": ["style-a"], "skipped": []},
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(erp, "print_skip_summary", lambda _report: None)
|
||||
monkeypatch.setattr(erp, "write_erp_skip_report", lambda *_args: None)
|
||||
monkeypatch.setattr(erp, "managed_data_path", lambda _path: tmp_path)
|
||||
monkeypatch.setattr(erp, "resolve_browser_session", lambda _config: (None, None))
|
||||
|
||||
def fake_login_action(**kwargs):
|
||||
state = kwargs["state"]
|
||||
|
||||
def action(_page):
|
||||
state["ok"] = False
|
||||
state["error"] = "Login did not complete; captcha required"
|
||||
|
||||
return action
|
||||
|
||||
monkeypatch.setattr(erp, "make_page_action", fake_login_action)
|
||||
monkeypatch.setattr(
|
||||
erp.DynamicFetcher,
|
||||
"fetch",
|
||||
lambda _url, **kwargs: kwargs["page_action"](object()),
|
||||
)
|
||||
|
||||
assert erp.main() == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_erp_login_waits_for_a_delayed_captcha(monkeypatch) -> None:
|
||||
captcha_states = iter([False, False, True, True, False])
|
||||
waits: list[int] = []
|
||||
|
||||
class Page:
|
||||
url = "https://erp.test/login"
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
waits.append(milliseconds)
|
||||
|
||||
monkeypatch.setattr(
|
||||
erp_login,
|
||||
"captcha_exists",
|
||||
lambda _page: next(captcha_states),
|
||||
)
|
||||
monotonic_values = iter([0.0, 0.1, 0.2])
|
||||
monkeypatch.setattr(
|
||||
erp_login.time,
|
||||
"monotonic",
|
||||
lambda: next(monotonic_values),
|
||||
)
|
||||
|
||||
erp_login.wait_for_manual_captcha(
|
||||
Page(),
|
||||
checks=3,
|
||||
interval_seconds=5,
|
||||
login_url="https://erp.test/login",
|
||||
)
|
||||
|
||||
assert waits == [500, 500, 5000, 5000]
|
||||
|
||||
|
||||
def test_erp_date_inputs_are_required_and_read_back() -> None:
|
||||
source = inspect.getsource(erp.set_platform_and_code)
|
||||
|
||||
@@ -200,8 +391,11 @@ def test_erp_date_inputs_are_required_and_read_back() -> None:
|
||||
assert "date range did not persist" in source
|
||||
|
||||
|
||||
def test_playwright_channel_is_optional_and_environment_driven() -> None:
|
||||
assert playwright_launch_options({}) == {}
|
||||
assert playwright_launch_options({"GYXX_PLAYWRIGHT_CHANNEL": "chrome"}) == {
|
||||
"channel": "chrome"
|
||||
def test_scrapling_real_chrome_is_optional_and_environment_driven() -> None:
|
||||
assert scrapling_session_options({}) == {}
|
||||
assert scrapling_session_options({"GYXX_PLAYWRIGHT_CHANNEL": "chrome"}) == {
|
||||
"real_chrome": True
|
||||
}
|
||||
assert scrapling_session_options({"GYXX_SCRAPLING_REAL_CHROME": "false"}) == {
|
||||
"real_chrome": False
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import collect_douyin_qianchuan_ads as collector
|
||||
|
||||
|
||||
def test_qianchuan_browser_uses_scrapling_stealth_and_persistent_state(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class _FakeBrowser:
|
||||
def __init__(self, **options: object) -> None:
|
||||
captured.update(options)
|
||||
|
||||
cookie_file = tmp_path / "cookies.json"
|
||||
storage_state_file = tmp_path / "storage_state.json"
|
||||
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
|
||||
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_state_file))
|
||||
monkeypatch.setattr(collector, "ScraplingBrowser", _FakeBrowser)
|
||||
|
||||
browser = collector._start_browser(True, str(tmp_path / "profile"))
|
||||
|
||||
assert isinstance(browser, _FakeBrowser)
|
||||
assert captured["stealthy"] is True
|
||||
assert captured["real_chrome"] is False
|
||||
assert captured["headless"] is True
|
||||
assert captured["user_data_dir"] == str((tmp_path / "profile").resolve())
|
||||
assert captured["cookie_file"] == str(cookie_file)
|
||||
assert captured["storage_state_file"] == str(storage_state_file)
|
||||
assert captured["extra_flags"] == ["--no-proxy-server"]
|
||||
assert captured["additional_args"] == {
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"accept_downloads": True,
|
||||
}
|
||||
|
||||
|
||||
def test_calendar_cell_prefers_actual_month_when_adjacent_month_duplicates() -> None:
|
||||
class _Locator:
|
||||
def __init__(self, count: int) -> None:
|
||||
self._count = count
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count
|
||||
|
||||
class _Page:
|
||||
def locator(self, selector: str) -> _Locator:
|
||||
if ".ovui-date__cell--inview" in selector:
|
||||
return _Locator(1)
|
||||
return _Locator(2)
|
||||
|
||||
cell = collector._calendar_cell(_Page(), "2026-08-30")
|
||||
|
||||
assert cell.count() == 1
|
||||
|
||||
|
||||
def test_calendar_cell_falls_back_when_picker_has_no_inview_class() -> None:
|
||||
class _Locator:
|
||||
def __init__(self, count: int) -> None:
|
||||
self._count = count
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count
|
||||
|
||||
class _Page:
|
||||
def locator(self, selector: str) -> _Locator:
|
||||
if ".ovui-date__cell--inview" in selector:
|
||||
return _Locator(0)
|
||||
return _Locator(1)
|
||||
|
||||
cell = collector._calendar_cell(_Page(), "2026-08-30")
|
||||
|
||||
assert cell.count() == 1
|
||||
|
||||
|
||||
def test_report_ready_wait_guards_missing_body() -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class _Page:
|
||||
def wait_for_function(self, expression: str, **options: object) -> None:
|
||||
captured["expression"] = expression
|
||||
captured.update(options)
|
||||
|
||||
page = _Page()
|
||||
collector._wait_for_report_ready(page)
|
||||
|
||||
assert captured == {
|
||||
"expression": collector.REPORT_READY_EXPRESSION,
|
||||
"arg": list(collector._required_page_markers()),
|
||||
"timeout": 90_000,
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import zipfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import collect_jd_ad_costs as collector
|
||||
import pytest
|
||||
|
||||
|
||||
def test_login_starts_from_jd_merchant_login_page() -> None:
|
||||
assert collector.JD_LOGIN_URL == (
|
||||
"https://passport.shop.jd.com/login/index.action/jdm?"
|
||||
"ReturnUrl=https%3A%2F%2Fshop.jd.com%2Fjdm%2Fhome"
|
||||
)
|
||||
|
||||
|
||||
def test_ensure_logged_in_navigates_to_login_entry_first(monkeypatch) -> None:
|
||||
visited: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"safe_goto",
|
||||
lambda _page, url: visited.append(url),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_login_complete", lambda _page: True)
|
||||
|
||||
collector.ensure_logged_in(object(), "光影行星研发", "secret")
|
||||
|
||||
assert visited == [collector.JD_LOGIN_URL]
|
||||
|
||||
|
||||
def test_ensure_logged_in_reuses_valid_cookie_session(monkeypatch) -> None:
|
||||
visited: list[str] = []
|
||||
|
||||
class Page:
|
||||
def wait_for_timeout(self, _milliseconds: int) -> None:
|
||||
return None
|
||||
|
||||
states = iter((False, True))
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"safe_goto",
|
||||
lambda _page, url: visited.append(url),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_login_complete", lambda _page: next(states))
|
||||
|
||||
collector.ensure_logged_in(Page(), "光影行星研发", "secret")
|
||||
|
||||
assert visited == [collector.JD_LOGIN_URL, collector.JD_HOME_URL]
|
||||
|
||||
|
||||
def test_login_failure_reason_exposes_jd_credential_error() -> None:
|
||||
class Page:
|
||||
def locator(self, _selector):
|
||||
return self
|
||||
|
||||
def inner_text(self, timeout):
|
||||
del timeout
|
||||
return "账号名与密码不匹配"
|
||||
|
||||
assert collector._login_failure_reason(Page()) == "账号名与密码不匹配"
|
||||
|
||||
|
||||
def test_safe_goto_stops_when_browser_context_is_closed() -> None:
|
||||
class Page:
|
||||
def goto(self, *_args, **_kwargs) -> None:
|
||||
raise RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
def is_closed(self) -> bool:
|
||||
return True
|
||||
|
||||
with pytest.raises(collector.JDAdCostCollectionError, match="上下文意外关闭"):
|
||||
collector.safe_goto(Page(), "https://jzt.jd.com/custom-report/#/download")
|
||||
|
||||
|
||||
def test_jd_ad_costs_uses_bundled_chromium_despite_global_chrome_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def build_browser(**options):
|
||||
captured.update(options)
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(collector, "ScraplingBrowser", build_browser)
|
||||
monkeypatch.setenv("GYXX_SCRAPLING_REAL_CHROME", "true")
|
||||
|
||||
collector._start_browser(False, "D:/gyxx-flow/var/state/browser-profiles/jd-ad-costs")
|
||||
|
||||
assert captured["real_chrome"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "report_type", "expected"),
|
||||
[
|
||||
("营销概况_矩阵分析_SKU维度_2026-08-23 报告已生成", "non_full", "ready"),
|
||||
("营销概况_下钻趋势图_2026-08-23 报表已生成", "full", "ready"),
|
||||
("营销概况_矩阵分析_SKU维度_2026-08-23 生成中", "non_full", "pending"),
|
||||
("营销概况_下钻趋势图_2026-08-22 报告已生成", "full", "absent"),
|
||||
(
|
||||
"光影行星研发_营销概况_矩阵分析_SKU维度_2026-08-17至2026-08-17"
|
||||
" 报表已生成 创建日期 2026-08-23",
|
||||
"non_full",
|
||||
"absent",
|
||||
),
|
||||
(
|
||||
"光影行星研发_营销概况_下钻趋势图_2026-06-01至2026-08-23"
|
||||
" 报表已生成 创建日期 2026-08-23",
|
||||
"full",
|
||||
"absent",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_download_center_state_is_report_type_and_date_scoped(
|
||||
text: str, report_type: collector.ReportType, expected: collector.ReportState
|
||||
) -> None:
|
||||
assert (
|
||||
collector._report_state_from_text(text, report_type, date(2026, 8, 23))
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
def test_download_center_total_treats_empty_table_as_zero() -> None:
|
||||
class Body:
|
||||
def inner_text(self, timeout):
|
||||
del timeout
|
||||
return "下载报表 暂无数据"
|
||||
|
||||
class Page:
|
||||
def locator(self, selector):
|
||||
assert selector == "body"
|
||||
return Body()
|
||||
|
||||
assert collector._download_center_total(Page()) == 0
|
||||
|
||||
|
||||
def test_clear_download_center_deletes_each_page_until_empty(monkeypatch) -> None:
|
||||
totals = iter((61, 51, 41, 31, 21, 11, 1, 0))
|
||||
deleted_batches: list[int] = []
|
||||
|
||||
monkeypatch.setattr(collector, "_wait_for_download_center_table", lambda _page: None)
|
||||
monkeypatch.setattr(collector, "_download_center_total", lambda _page: next(totals))
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_all_download_center_rows",
|
||||
lambda _page: deleted_batches.append(1) or 10,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector, "_delete_selected_download_center_rows", lambda _page: None
|
||||
)
|
||||
monkeypatch.setattr(collector, "_wait_for", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(collector, "_reload_download_center", lambda _page: None)
|
||||
|
||||
collector._clear_download_center(object())
|
||||
|
||||
assert len(deleted_batches) == 7
|
||||
|
||||
|
||||
def test_safe_extract_zip_rejects_path_traversal(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "report.zip"
|
||||
with zipfile.ZipFile(archive, "w") as handle:
|
||||
handle.writestr("../outside.csv", "bad")
|
||||
|
||||
with pytest.raises(collector.JDAdCostCollectionError, match="路径越界"):
|
||||
collector._safe_extract_zip(archive, tmp_path / "extracted")
|
||||
|
||||
|
||||
def test_safe_extract_zip_keeps_csv_inside_output(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "report.zip"
|
||||
with zipfile.ZipFile(archive, "w") as handle:
|
||||
handle.writestr("nested/full.csv", "日期,'当前时间',SPU ID\n")
|
||||
|
||||
files = collector._safe_extract_zip(archive, tmp_path / "extracted")
|
||||
|
||||
assert files == [(tmp_path / "extracted" / "nested" / "full.csv").resolve()]
|
||||
@@ -0,0 +1,180 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import zipfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import collect_tmall_wanxiang_ads as collector
|
||||
import pytest
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def test_item_promotion_url_contains_explicit_target_date() -> None:
|
||||
url = collector.item_promotion_url(date(2026, 8, 17))
|
||||
assert "#!/report/item_promotion" in url
|
||||
assert "startTime=2026-08-17" in url
|
||||
assert "endTime=2026-08-17" in url
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
("商品报表 2026-08-18 生成成功", "ready"),
|
||||
("商品报表 2026-08-18 生成中", "pending"),
|
||||
("商品报表 2026-08-18 生成失败", "failed"),
|
||||
("商品报表 2026-08-17 生成成功", "absent"),
|
||||
# 数据日期范围不匹配,仅创建日期含目标日:不能误判为同日报表
|
||||
(
|
||||
"生成成功 商品报表_20260818_120200 15天累计数据 2026-08-11至2026-08-11 2026-08-18",
|
||||
"absent",
|
||||
),
|
||||
# 数据日期范围覆盖目标日:可复用
|
||||
(
|
||||
"生成成功 商品报表_20260819_113811 15天累计数据 2026-08-18至2026-08-18 2026-08-19",
|
||||
"ready",
|
||||
),
|
||||
],)
|
||||
def test_report_task_state_is_date_scoped_and_conservative(
|
||||
text: str, expected: collector.ReportTaskState
|
||||
) -> None:
|
||||
assert collector._report_task_state_from_text(text, date(2026, 8, 18)) == expected
|
||||
|
||||
|
||||
def test_scheduled_wanxiang_workflow_does_not_force_duplicate_request() -> None:
|
||||
payload = json.loads(
|
||||
(PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8")
|
||||
)
|
||||
workflow = next(
|
||||
item for item in payload["workflows"] if item["id"] == "product.ecommerce_costs.daily"
|
||||
)
|
||||
download_step = next(
|
||||
step for step in workflow["execution"]["steps"] if step["id"] == "tmall_download"
|
||||
)
|
||||
assert "--force-request" not in download_step["args"]
|
||||
|
||||
|
||||
def test_request_report_submits_outer_dialog_only_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
page = object()
|
||||
modal = {"open": True}
|
||||
confirm_calls = 0
|
||||
selected_dates: list[date] = []
|
||||
|
||||
monkeypatch.setattr(collector, "_wait_for_text", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(collector, "_click_visible_text", lambda *args, **kwargs: True)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_date_range",
|
||||
lambda _page, target_date: selected_dates.append(target_date),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_modal_visible", lambda _page: modal["open"])
|
||||
|
||||
def confirm(_page: object) -> bool:
|
||||
nonlocal confirm_calls
|
||||
confirm_calls += 1
|
||||
modal["open"] = False
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(collector, "_click_download_dialog_confirm", confirm)
|
||||
|
||||
collector.request_report(page, date(2026, 8, 18))
|
||||
|
||||
assert confirm_calls == 1
|
||||
assert selected_dates == [date(2026, 8, 18)]
|
||||
|
||||
|
||||
def test_run_waits_for_existing_pending_task_instead_of_requesting_another(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
class _Session:
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
session = _Session()
|
||||
page = object()
|
||||
calls: list[bool] = []
|
||||
report_zip = tmp_path / "report.zip"
|
||||
extracted_csv = tmp_path / "report.csv"
|
||||
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_start_session",
|
||||
lambda _headless, _user_data_dir: (session, object(), page),
|
||||
)
|
||||
monkeypatch.setattr(collector, "ensure_logged_in", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"download_ready_report",
|
||||
lambda _page, _date, _output, *, timeout_seconds, wait_for_generation: (
|
||||
calls.append(wait_for_generation)
|
||||
or (None if len(calls) == 1 else report_zip)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_report_task_state", lambda *_args: "pending")
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"extract_report_zip",
|
||||
lambda _zip_path, _target_date: [extracted_csv],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"request_report",
|
||||
lambda *_args, **_kwargs: pytest.fail("must not submit a second task"),
|
||||
)
|
||||
|
||||
args = collector.build_parser().parse_args(
|
||||
["--date", "2026-08-18", "--output", str(tmp_path)]
|
||||
)
|
||||
# Content validation only re-requests when the reused report does not
|
||||
# cover the target date; a matching CSV must keep the reuse path.
|
||||
extracted_csv.write_text("日期,主体ID\n2026-08-18,1\n", encoding="utf-8")
|
||||
assert collector.run(args) == [extracted_csv]
|
||||
assert calls == [False, True]
|
||||
|
||||
|
||||
class _Page:
|
||||
def __init__(self, url: str, body: str) -> None:
|
||||
self.url = url
|
||||
self._body = body
|
||||
|
||||
def locator(self, selector: str):
|
||||
assert selector == "body"
|
||||
return self
|
||||
|
||||
def inner_text(self, timeout: int = 0) -> str:
|
||||
return self._body
|
||||
|
||||
|
||||
def test_login_success_accepts_taobao_seller_center_before_redirect() -> None:
|
||||
assert collector._is_taobao_login_complete(
|
||||
_Page("https://myseller.taobao.com/home", "物流监控 违规预警")
|
||||
)
|
||||
assert not collector._is_taobao_login_complete(
|
||||
_Page("https://loginmyseller.taobao.com/", "请输入登录密码")
|
||||
)
|
||||
|
||||
|
||||
def test_safe_extract_zip_returns_csv_and_writes_manifest(tmp_path: Path) -> None:
|
||||
archive_path = tmp_path / "商品报表.zip"
|
||||
with zipfile.ZipFile(archive_path, "w") as archive:
|
||||
archive.writestr("商品报表_20260817.csv", "日期,主体ID\n")
|
||||
archive.writestr("说明.txt", "万相台商品报表")
|
||||
|
||||
csv_files = collector.extract_report_zip(archive_path, date(2026, 8, 17))
|
||||
|
||||
assert [path.name for path in csv_files] == ["商品报表_20260817.csv"]
|
||||
manifest = (tmp_path / "manifest.json").read_text(encoding="utf-8")
|
||||
assert '"target_date": "2026-08-17"' in manifest
|
||||
assert (tmp_path / "extracted" / "商品报表_20260817.csv").is_file()
|
||||
|
||||
|
||||
def test_safe_extract_zip_rejects_path_traversal(tmp_path: Path) -> None:
|
||||
archive_path = tmp_path / "unsafe.zip"
|
||||
with zipfile.ZipFile(archive_path, "w") as archive:
|
||||
archive.writestr("../escape.csv", "bad")
|
||||
|
||||
with pytest.raises(collector.WanxiangDownloadError, match="路径越界"):
|
||||
collector._safe_extract_zip(archive_path, tmp_path / "out")
|
||||
@@ -13,6 +13,11 @@ import orchestrate_daily_collection as daily
|
||||
import pytest
|
||||
from commands import import_daily as import_daily_command
|
||||
|
||||
from gyxx_flow.core.exit_codes import (
|
||||
COOKIE_SKIP_EXIT_CODE,
|
||||
NON_RETRYABLE_EXIT_CODE,
|
||||
)
|
||||
|
||||
|
||||
class _Context:
|
||||
def __init__(self, value):
|
||||
@@ -93,6 +98,153 @@ def test_selected_platform_collector_failure_stops_daily_aggregation(monkeypatch
|
||||
target_date="2026-08-03",
|
||||
)
|
||||
|
||||
|
||||
def test_cookie_skip_is_not_retried_by_daily_orchestrator(monkeypatch) -> None:
|
||||
calls = 0
|
||||
|
||||
def fake_run(*_args, **_kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return {"name": "erp", "code": COOKIE_SKIP_EXIT_CODE}
|
||||
|
||||
monkeypatch.setattr(daily, "run_step", fake_run)
|
||||
monkeypatch.setattr(
|
||||
daily.time,
|
||||
"sleep",
|
||||
lambda *_args: pytest.fail("cookie recovery must not enter retry cooldown"),
|
||||
)
|
||||
|
||||
with pytest.raises(daily.CookiePreflightSkipped, match="erp requires"):
|
||||
daily.run_step_with_retry("erp", ["python", "collector.py"])
|
||||
|
||||
assert calls == 1
|
||||
|
||||
|
||||
def test_non_retryable_collector_result_is_not_retried(monkeypatch) -> None:
|
||||
calls = 0
|
||||
|
||||
def fake_run(*_args, **_kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return {"name": "tm", "code": NON_RETRYABLE_EXIT_CODE}
|
||||
|
||||
monkeypatch.setattr(daily, "run_step", fake_run)
|
||||
monkeypatch.setattr(
|
||||
daily.time,
|
||||
"sleep",
|
||||
lambda *_args: pytest.fail("deterministic validation must not retry"),
|
||||
)
|
||||
|
||||
result = daily.run_step_with_retry(
|
||||
"tm",
|
||||
["python", "collector.py"],
|
||||
critical=False,
|
||||
)
|
||||
|
||||
assert result["code"] == NON_RETRYABLE_EXIT_CODE
|
||||
assert calls == 1
|
||||
|
||||
|
||||
def test_daily_main_propagates_cookie_skip_exit_code(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_collect",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
daily.CookiePreflightSkipped("erp requires browser login")
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
daily.argparse.ArgumentParser,
|
||||
"parse_args",
|
||||
lambda _parser: SimpleNamespace(
|
||||
date=None,
|
||||
target_date="2026-08-12",
|
||||
stages=["collect"],
|
||||
skip_erp=False,
|
||||
skip_platforms=False,
|
||||
platforms="jd,dy,tm",
|
||||
serial_platforms=False,
|
||||
skip_insert=False,
|
||||
skip_reviews_import=False,
|
||||
dry_run=False,
|
||||
erp_mode="daily",
|
||||
max_run_retries=2,
|
||||
),
|
||||
)
|
||||
|
||||
assert daily.main() == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_erp_mode_auto_uses_daily_only_for_yesterday() -> None:
|
||||
today = datetime(2026, 8, 13).date()
|
||||
|
||||
assert daily.resolve_erp_mode("2026-08-12", today=today) == "daily"
|
||||
assert daily.resolve_erp_mode("2026-08-11", today=today) == "backfill"
|
||||
assert daily.resolve_erp_mode("2026-08-13", today=today) == "backfill"
|
||||
assert daily.is_historical_target("2026-08-12", today=today) is False
|
||||
assert daily.is_historical_target("2026-08-11", today=today) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["daily", "weekly", "backfill"])
|
||||
def test_explicit_erp_mode_overrides_automatic_selection(mode) -> None:
|
||||
assert (
|
||||
daily.resolve_erp_mode(
|
||||
"2026-08-11",
|
||||
requested_mode=mode,
|
||||
today=datetime(2026, 8, 13).date(),
|
||||
)
|
||||
== mode
|
||||
)
|
||||
|
||||
|
||||
def test_historical_daily_collection_passes_backfill_mode_to_erp(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
args = SimpleNamespace(
|
||||
skip_erp=False,
|
||||
skip_platforms=True,
|
||||
skip_reviews_import=True,
|
||||
dry_run=False,
|
||||
erp_mode="auto",
|
||||
)
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"resolve_erp_mode",
|
||||
lambda *_args, **_kwargs: "backfill",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs))
|
||||
or {"name": "erp", "code": 0},
|
||||
)
|
||||
|
||||
daily.run_collect(args, "2026-08-11", ["jd", "dy", "tm"], failed=None)
|
||||
|
||||
command = calls[0][0][1]
|
||||
assert command[-4:] == ["--target-date", "2026-08-11", "--mode", "backfill"]
|
||||
|
||||
|
||||
def test_historical_tm_collection_uses_backfill_child(monkeypatch) -> None:
|
||||
calls: list[tuple[str, list[str]]] = []
|
||||
|
||||
def fake_run(name, command, **_kwargs):
|
||||
calls.append((name, command))
|
||||
return {"name": name, "code": 0}
|
||||
|
||||
monkeypatch.setattr(daily, "run_step_with_retry", fake_run)
|
||||
monkeypatch.setattr(daily, "verify_platform_artifacts", lambda *_args: 1)
|
||||
|
||||
daily.run_platforms(
|
||||
parallel=False,
|
||||
platforms=["tm"],
|
||||
target_date="2026-08-11",
|
||||
historical=True,
|
||||
)
|
||||
|
||||
assert calls[0][1][-3:] == ["--backfill", "--target-date", "2026-08-11"]
|
||||
|
||||
def test_platform_selection_keeps_omitted_collectors_optional(monkeypatch) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
@@ -217,6 +369,63 @@ def test_platform_artifact_verifier_accepts_fresh_valid_product(
|
||||
assert daily.verify_platform_artifacts("jd", "2026-08-03", marker_ns) == 1
|
||||
|
||||
|
||||
def test_dy_verifier_accepts_source_date_format_and_legitimate_zero_style(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
class Loader:
|
||||
def get_daily_styles(self):
|
||||
return {
|
||||
"dy": {
|
||||
"collectable": ["matched-style", "zero-style"],
|
||||
"skipped": [],
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(daily, "StyleConfigLoader", Loader)
|
||||
monkeypatch.setattr(daily, "DATA_ROOT", tmp_path)
|
||||
marker_ns = time.time_ns()
|
||||
for style, matched in (("matched-style", 2), ("zero-style", 0)):
|
||||
path = daily._platform_artifact_path("dy", style, "2026-08-03")
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(
|
||||
json.dumps({
|
||||
"date_range": "2026/08/03 - 2026/08/03",
|
||||
"matched_row_count": matched,
|
||||
"download_file": "_downloads/2026-08-03/report.xlsx",
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns) == 2
|
||||
|
||||
|
||||
def test_platform_artifact_verifier_rejects_an_export_with_no_product_matches(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
class Loader:
|
||||
def get_daily_styles(self):
|
||||
return {"dy": {"collectable": ["zero-style"], "skipped": []}}
|
||||
|
||||
monkeypatch.setattr(daily, "StyleConfigLoader", Loader)
|
||||
monkeypatch.setattr(daily, "DATA_ROOT", tmp_path)
|
||||
marker_ns = time.time_ns()
|
||||
path = daily._platform_artifact_path("dy", "zero-style", "2026-08-03")
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(
|
||||
json.dumps({
|
||||
"date_range": "2026/08/03 - 2026/08/03",
|
||||
"matched_row_count": 0,
|
||||
"download_file": "_downloads/2026-08-03/report.xlsx",
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="matched no configured products"):
|
||||
daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns)
|
||||
|
||||
|
||||
def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
|
||||
args = SimpleNamespace(
|
||||
skip_erp=True,
|
||||
@@ -240,6 +449,27 @@ def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
|
||||
assert calls[0][1]["critical"] is False
|
||||
|
||||
|
||||
def test_daily_import_stage_uses_the_collected_business_date(monkeypatch) -> None:
|
||||
args = SimpleNamespace(dry_run=True, platforms="jd,tm")
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0},
|
||||
)
|
||||
|
||||
daily.run_import(args, "2026-08-03", failed=None)
|
||||
|
||||
assert calls[0][0][0] == "import_product_daily"
|
||||
assert calls[0][0][1][-4:] == [
|
||||
"--date",
|
||||
"2026-08-03",
|
||||
"--platforms",
|
||||
"jd,tm",
|
||||
]
|
||||
assert calls[0][1]["dry_run"] is True
|
||||
|
||||
|
||||
def test_formal_aggregation_fails_when_postgres_write_is_not_fresh(monkeypatch) -> None:
|
||||
marker = datetime(2026, 8, 4, 3, 0, tzinfo=timezone.utc)
|
||||
args = SimpleNamespace(dry_run=False)
|
||||
@@ -374,6 +604,26 @@ def test_import_fails_when_selected_platform_has_no_target_directory(monkeypatch
|
||||
assert "dy: 没有匹配所选日期范围的报表目录" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_import_deduplicates_rerun_rows_before_one_upsert_batch(capsys) -> None:
|
||||
first = {
|
||||
**_record(),
|
||||
"source_file": "old-report.xlsx",
|
||||
"visitors": 10,
|
||||
}
|
||||
latest = {
|
||||
**_record(),
|
||||
"source_file": "latest-report.xlsx",
|
||||
"visitors": 20,
|
||||
}
|
||||
|
||||
result = product_import._deduplicate_records_for_upsert([first, latest])
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["source_file"] == "latest-report.xlsx"
|
||||
assert result[0]["visitors"] == 20
|
||||
assert "丢弃重复原始行: 1 行" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_import_fails_when_target_directory_has_no_input_file(monkeypatch, capsys) -> None:
|
||||
target_dir = Path("2026-08-03")
|
||||
monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [target_dir])
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import db
|
||||
import pytest
|
||||
from db import InterfaceError, get_conn
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def execute(self, _sql):
|
||||
return None
|
||||
|
||||
def fetchone(self):
|
||||
return (1,)
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self, alive: bool) -> None:
|
||||
self.alive = alive
|
||||
self.closed = False
|
||||
self.committed = False
|
||||
self.rolled_back = False
|
||||
|
||||
def rollback(self) -> None:
|
||||
if not self.alive:
|
||||
raise RuntimeError("connection already closed")
|
||||
self.rolled_back = True
|
||||
|
||||
def cursor(self):
|
||||
if not self.alive:
|
||||
raise RuntimeError("connection already closed")
|
||||
return FakeCursor()
|
||||
|
||||
def commit(self) -> None:
|
||||
self.committed = True
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class FakePool:
|
||||
def __init__(self, connections):
|
||||
self.connections = list(connections)
|
||||
self.returned: list[tuple[object, bool]] = []
|
||||
|
||||
def getconn(self):
|
||||
if not self.connections:
|
||||
raise RuntimeError("pool exhausted")
|
||||
return self.connections.pop(0)
|
||||
|
||||
def putconn(self, conn, close: bool = False) -> None:
|
||||
self.returned.append((conn, close))
|
||||
|
||||
|
||||
def test_get_conn_reuses_healthy_pooled_connection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
connection = FakeConnection(alive=True)
|
||||
fake_pool = FakePool([connection])
|
||||
monkeypatch.setattr(db, "_get_pool", lambda: fake_pool)
|
||||
|
||||
with get_conn() as acquired:
|
||||
assert acquired is connection
|
||||
|
||||
assert connection.committed is True
|
||||
assert fake_pool.returned == [(connection, False)]
|
||||
|
||||
|
||||
def test_get_conn_discards_stale_connection_and_borrows_fresh(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
stale = FakeConnection(alive=False)
|
||||
fresh = FakeConnection(alive=True)
|
||||
fake_pool = FakePool([stale, fresh])
|
||||
monkeypatch.setattr(db, "_get_pool", lambda: fake_pool)
|
||||
|
||||
with get_conn() as acquired:
|
||||
assert acquired is fresh
|
||||
|
||||
assert stale.closed is True
|
||||
assert fresh.committed is True
|
||||
assert fake_pool.returned == [(stale, True), (fresh, False)]
|
||||
|
||||
|
||||
def test_get_conn_fails_closed_when_all_pooled_connections_are_dead(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
first = FakeConnection(alive=False)
|
||||
second = FakeConnection(alive=False)
|
||||
fake_pool = FakePool([first, second])
|
||||
monkeypatch.setattr(db, "_get_pool", lambda: fake_pool)
|
||||
|
||||
with pytest.raises(InterfaceError):
|
||||
with get_conn():
|
||||
pass
|
||||
|
||||
assert first.closed is True
|
||||
assert second.closed is True
|
||||
assert fake_pool.returned == [(first, True), (second, True)]
|
||||
|
||||
|
||||
def test_get_pool_retries_transient_initial_connection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
attempts: list[str] = []
|
||||
|
||||
def fake_pool(**_kwargs):
|
||||
attempts.append("connect")
|
||||
if len(attempts) == 1:
|
||||
raise db.OperationalError("transient connection failure")
|
||||
return object()
|
||||
|
||||
for name, value in {
|
||||
"PG_HOST": "db.example.test",
|
||||
"PG_PORT": "5432",
|
||||
"PG_DB": "warehouse",
|
||||
"PG_USER": "collector",
|
||||
"PG_PASSWORD": "unit-test-only",
|
||||
}.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.setattr(db, "_pool", None)
|
||||
monkeypatch.setattr(db.pool, "SimpleConnectionPool", fake_pool)
|
||||
monkeypatch.setattr(db.time, "sleep", lambda _seconds: attempts.append("sleep"))
|
||||
|
||||
result = db._get_pool(connect_attempts=2, retry_delay_seconds=0)
|
||||
|
||||
assert result is not None
|
||||
assert attempts == ["connect", "sleep", "connect"]
|
||||
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.product_commerce.direct_llm import (
|
||||
DirectLLMConfig,
|
||||
DirectLLMConfigurationError,
|
||||
DirectLLMResponseError,
|
||||
DirectLLMTransientError,
|
||||
chat_completion,
|
||||
load_direct_llm_config,
|
||||
)
|
||||
|
||||
|
||||
def _config() -> DirectLLMConfig:
|
||||
return DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-secret",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
)
|
||||
|
||||
|
||||
def _response(content: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"choices": [{"message": {"content": content}}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 4},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def test_direct_chat_uses_provider_endpoint_and_model_without_echoing_secret() -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def transport(url, headers, body, timeout):
|
||||
payload = json.loads(body)
|
||||
observed.update(
|
||||
url=url,
|
||||
model=payload["model"],
|
||||
messages=payload["messages"],
|
||||
reasoning_split=payload["reasoning_split"],
|
||||
timeout=timeout,
|
||||
authorization=headers["Authorization"],
|
||||
)
|
||||
return _response('{"ok":true}')
|
||||
|
||||
content, usage = chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
model="MiniMax-M3",
|
||||
timeout=17,
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert content == '{"ok":true}'
|
||||
assert usage == {"prompt_tokens": 3, "completion_tokens": 4}
|
||||
assert observed["url"] == "https://api.minimaxi.com/v1/chat/completions"
|
||||
assert observed["model"] == "MiniMax-M3"
|
||||
assert observed["reasoning_split"] is True
|
||||
assert observed["timeout"] == 17.0
|
||||
assert observed["authorization"] == "Bearer test-only-secret"
|
||||
|
||||
|
||||
def test_direct_chat_can_disable_minimax_thinking_for_structured_video_tasks() -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def transport(_url, _headers, body, _timeout):
|
||||
observed.update(json.loads(body))
|
||||
return _response("ok")
|
||||
|
||||
chat_completion(
|
||||
[{"role": "user", "content": "return one token"}],
|
||||
thinking_mode="disabled",
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert observed["thinking"] == {"type": "disabled"}
|
||||
|
||||
|
||||
def test_direct_chat_rejects_unknown_thinking_mode() -> None:
|
||||
with pytest.raises(DirectLLMConfigurationError, match="thinking mode"):
|
||||
chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
thinking_mode="sometimes",
|
||||
transport=lambda *_args: _response("unused"),
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_config_reads_shared_minimax_environment(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GYXX_DIRECT_LLM_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("GYXX_DIRECT_LLM_API_KEY", raising=False)
|
||||
monkeypatch.delenv("GYXX_DIRECT_LLM_MODEL", raising=False)
|
||||
monkeypatch.setenv("STYLE_ANALYSIS_LLM_BASE_URL", "https://api.minimaxi.com/v1")
|
||||
monkeypatch.setenv("STYLE_ANALYSIS_LLM_API_KEY", "test-only-secret")
|
||||
monkeypatch.setenv("STYLE_ANALYSIS_LLM_MODEL", "MiniMax-M3")
|
||||
|
||||
config = load_direct_llm_config()
|
||||
|
||||
assert config.endpoint == "https://api.minimaxi.com/v1/chat/completions"
|
||||
assert config.title_model == "MiniMax-M3"
|
||||
assert config.vision_model == "MiniMax-M3"
|
||||
assert "test-only-secret" not in repr(config)
|
||||
|
||||
|
||||
def test_config_rejects_legacy_local_gateway(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_DIRECT_LLM_BASE_URL",
|
||||
"http://127.0.0.1:8642/v1",
|
||||
)
|
||||
monkeypatch.setenv("GYXX_DIRECT_LLM_API_KEY", "test-only-secret")
|
||||
|
||||
with pytest.raises(DirectLLMConfigurationError, match="local gateway"):
|
||||
load_direct_llm_config()
|
||||
|
||||
|
||||
def test_invalid_response_is_fail_closed() -> None:
|
||||
with pytest.raises(DirectLLMResponseError, match="invalid JSON"):
|
||||
chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
transport=lambda *_args: b"not-json",
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_direct_chat_retries_transient_provider_overload(monkeypatch) -> None:
|
||||
calls = 0
|
||||
|
||||
def transport(_url, _headers, _body, _timeout):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls < 3:
|
||||
raise DirectLLMTransientError(
|
||||
"direct model request failed with HTTP 529"
|
||||
)
|
||||
return _response("recovered")
|
||||
|
||||
delays: list[float] = []
|
||||
monkeypatch.setattr(
|
||||
"gyxx_flow.modules.product_commerce.direct_llm.time.sleep",
|
||||
delays.append,
|
||||
)
|
||||
|
||||
content, _usage = chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert content == "recovered"
|
||||
assert calls == 3
|
||||
assert delays == [1.0, 3.0]
|
||||
|
||||
|
||||
def test_direct_chat_preserves_transient_error_after_bounded_retries(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
calls = 0
|
||||
|
||||
def transport(_url, _headers, _body, _timeout):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise DirectLLMTransientError(
|
||||
"direct model request failed with HTTP 529"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"gyxx_flow.modules.product_commerce.direct_llm.time.sleep",
|
||||
lambda _seconds: None,
|
||||
)
|
||||
|
||||
with pytest.raises(DirectLLMTransientError, match="HTTP 529"):
|
||||
chat_completion(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
transport=transport,
|
||||
config=_config(),
|
||||
)
|
||||
|
||||
assert calls == 3
|
||||
@@ -0,0 +1,154 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.product_commerce.direct_llm import DirectLLMConfig
|
||||
from gyxx_flow.modules.product_commerce.direct_visual_color import (
|
||||
DirectVisualColorClassifier,
|
||||
VisualCandidate,
|
||||
)
|
||||
|
||||
|
||||
def _image(path: Path, color: str) -> Path:
|
||||
Image.new("RGB", (12, 12), color).save(path, format="PNG")
|
||||
return path
|
||||
|
||||
|
||||
def _response(classification: dict[str, object]) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps(classification),
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def test_direct_visual_classifier_sends_images_to_minimax_m3(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def transport(url, headers, body, timeout):
|
||||
payload = json.loads(body)
|
||||
image_parts = [
|
||||
part
|
||||
for part in payload["messages"][1]["content"]
|
||||
if part["type"] == "image_url"
|
||||
]
|
||||
observed.update(
|
||||
url=url,
|
||||
model=payload["model"],
|
||||
thinking=payload["thinking"],
|
||||
timeout=timeout,
|
||||
image_count=len(image_parts),
|
||||
data_url_prefix=image_parts[0]["image_url"]["url"][:22],
|
||||
has_auth=headers["Authorization"].startswith("Bearer "),
|
||||
)
|
||||
return _response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-secret",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert result.source_color == "black"
|
||||
assert result.selected_key == "candidate"
|
||||
assert observed == {
|
||||
"url": "https://api.minimaxi.com/v1/chat/completions",
|
||||
"model": "MiniMax-M3",
|
||||
"thinking": {"type": "disabled"},
|
||||
"timeout": 120.0,
|
||||
"image_count": 2,
|
||||
"data_url_prefix": "data:image/png;base64,",
|
||||
"has_auth": True,
|
||||
}
|
||||
|
||||
|
||||
def test_direct_visual_payload_uses_external_provider_endpoint(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def transport(url, _headers, body, _timeout):
|
||||
captured["url"] = url
|
||||
payload = json.loads(body)
|
||||
image_url = next(
|
||||
part["image_url"]["url"]
|
||||
for part in payload["messages"][1]["content"]
|
||||
if part["type"] == "image_url"
|
||||
)
|
||||
assert base64.b64decode(image_url.split(",", 1)[1])
|
||||
return _response(
|
||||
{
|
||||
"source_color": "unknown",
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-secret",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert captured["url"] == "https://api.minimaxi.com/v1/chat/completions"
|
||||
assert "8642" not in str(captured["url"])
|
||||
|
||||
|
||||
def test_direct_visual_classifier_accepts_model_reasoning_wrapper(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
classification = {
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
|
||||
def transport(_url, _headers, _body, _timeout):
|
||||
content = (
|
||||
"<think>分析图片后再给出严格结果。</think>\n"
|
||||
"```json\n"
|
||||
+ json.dumps(classification)
|
||||
+ "\n```"
|
||||
)
|
||||
return json.dumps(
|
||||
{"choices": [{"message": {"content": content}}]}
|
||||
).encode()
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-secret",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert result.source_color == "black"
|
||||
assert result.selected_key == "candidate"
|
||||
@@ -0,0 +1,524 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.product_commerce.direct_llm import DirectLLMConfig
|
||||
from gyxx_flow.modules.product_commerce.direct_visual_color import (
|
||||
COARSE_COLORS,
|
||||
MAX_CANDIDATES,
|
||||
DirectVisualColorClassifier,
|
||||
DirectVisualColorError,
|
||||
VisualCandidate,
|
||||
normalize_coarse_color,
|
||||
)
|
||||
|
||||
|
||||
def _image(path: Path, color: str) -> Path:
|
||||
Image.new("RGB", (12, 12), color).save(path, format="PNG")
|
||||
return path
|
||||
|
||||
|
||||
def _response(classification: dict[str, Any]) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": json.dumps(classification),
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _classifier(response: bytes, observed: dict[str, Any] | None = None):
|
||||
def transport(url, headers, body, timeout):
|
||||
if observed is not None:
|
||||
payload = json.loads(body)
|
||||
image_parts = [
|
||||
part
|
||||
for part in payload["messages"][1]["content"]
|
||||
if part["type"] == "image_url"
|
||||
]
|
||||
observed.update(
|
||||
{
|
||||
"url": url,
|
||||
"timeout": timeout,
|
||||
"model": payload["model"],
|
||||
"image_count": len(image_parts),
|
||||
"all_data_urls": all(
|
||||
part["image_url"]["url"].startswith(
|
||||
"data:image/png;base64,"
|
||||
)
|
||||
for part in image_parts
|
||||
),
|
||||
"has_authorization": headers.get("Authorization", "").startswith(
|
||||
"Bearer "
|
||||
),
|
||||
}
|
||||
)
|
||||
return response
|
||||
|
||||
return DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_unique_coarse_color_match_uses_local_images_as_data_urls(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "saddlebrown")
|
||||
black = _image(tmp_path / "black.png", "black")
|
||||
brown = _image(tmp_path / "brown.png", "peru")
|
||||
observed: dict[str, Any] = {}
|
||||
classifier = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "brown",
|
||||
"candidate_colors": {"sku-black": "black", "sku-brown": "tan"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
),
|
||||
observed,
|
||||
)
|
||||
|
||||
result = classifier.classify(
|
||||
[source],
|
||||
[
|
||||
VisualCandidate("sku-black", black),
|
||||
VisualCandidate("sku-brown", brown),
|
||||
],
|
||||
)
|
||||
|
||||
assert result.source_color == "brown"
|
||||
assert result.candidate_colors == {"sku-black": "black", "sku-brown": "brown"}
|
||||
assert result.selected_key == "sku-brown"
|
||||
assert result.selection_basis == "coarse_color"
|
||||
assert observed == {
|
||||
"url": "https://api.minimaxi.com/v1/chat/completions",
|
||||
"timeout": 120.0,
|
||||
"model": "MiniMax-M3",
|
||||
"image_count": 3,
|
||||
"all_data_urls": True,
|
||||
"has_authorization": True,
|
||||
}
|
||||
|
||||
|
||||
def test_same_color_tie_allows_one_visual_similarity_match(tmp_path) -> None:
|
||||
source_a = _image(tmp_path / "source-a.png", "navy")
|
||||
source_b = _image(tmp_path / "source-b.png", "blue")
|
||||
first = _image(tmp_path / "first.png", "navy")
|
||||
second = _image(tmp_path / "second.png", "blue")
|
||||
classifier = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "navy",
|
||||
"candidate_colors": {"first": "blue", "second": "blue"},
|
||||
"visual_match": {"unique": True, "key": "second"},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
result = classifier.classify(
|
||||
[source_a, source_b],
|
||||
[VisualCandidate("first", first), VisualCandidate("second", second)],
|
||||
)
|
||||
|
||||
assert result.source_color == "blue"
|
||||
assert result.selected_key == "second"
|
||||
assert result.selection_basis == "visual_similarity"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"classification",
|
||||
[
|
||||
{
|
||||
"source_color": "blue",
|
||||
"candidate_colors": {"first": "blue", "second": "blue"},
|
||||
"visual_match": {"unique": True, "key": "invented"},
|
||||
},
|
||||
{
|
||||
"source_color": "blue",
|
||||
"candidate_colors": {"first": "blue", "invented": "blue"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
},
|
||||
{
|
||||
"source_color": "blue",
|
||||
"candidate_colors": {"first": "blue"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_unknown_or_missing_candidate_key_fails_closed(
|
||||
tmp_path,
|
||||
classification,
|
||||
) -> None:
|
||||
source = _image(tmp_path / "source.png", "blue")
|
||||
first = _image(tmp_path / "first.png", "blue")
|
||||
second = _image(tmp_path / "second.png", "blue")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="candidate key"):
|
||||
_classifier(_response(classification)).classify(
|
||||
[source],
|
||||
[VisualCandidate("first", first), VisualCandidate("second", second)],
|
||||
)
|
||||
|
||||
|
||||
def test_one_character_candidate_key_typo_fails_closed(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="unknown candidate key"):
|
||||
_classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"card-eb51a74da3a4": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify(
|
||||
[source],
|
||||
[VisualCandidate("card-eb51a74da3a3", candidate)],
|
||||
)
|
||||
|
||||
|
||||
def test_visual_match_cannot_bypass_coarse_color_filter(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "red")
|
||||
red = _image(tmp_path / "red.png", "red")
|
||||
blue = _image(tmp_path / "blue.png", "blue")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="inconsistent"):
|
||||
_classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "red",
|
||||
"candidate_colors": {"red": "red", "blue": "blue"},
|
||||
"visual_match": {"unique": True, "key": "blue"},
|
||||
}
|
||||
)
|
||||
).classify(
|
||||
[source],
|
||||
[VisualCandidate("red", red), VisualCandidate("blue", blue)],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"assistant_content",
|
||||
[
|
||||
"not-json",
|
||||
'```json\n{"source_color":"black"}\n```',
|
||||
'{"source_color": NaN}',
|
||||
'{"source_color":"black","candidate_colors":{},"visual_match":{},"extra":1}',
|
||||
],
|
||||
)
|
||||
def test_non_strict_model_json_fails_closed(tmp_path, assistant_content) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
response = json.dumps(
|
||||
{"choices": [{"message": {"content": assistant_content}}]}
|
||||
).encode()
|
||||
|
||||
with pytest.raises(DirectVisualColorError):
|
||||
_classifier(response).classify(
|
||||
[source],
|
||||
[VisualCandidate("candidate", candidate)],
|
||||
)
|
||||
|
||||
|
||||
def test_non_string_source_color_fails_closed(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="source color"):
|
||||
_classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": None,
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify(
|
||||
[source],
|
||||
[VisualCandidate("candidate", candidate)],
|
||||
)
|
||||
|
||||
|
||||
def test_image_count_limits_fail_before_transport(tmp_path) -> None:
|
||||
image = _image(tmp_path / "image.png", "black")
|
||||
called = False
|
||||
|
||||
def transport(*_args):
|
||||
nonlocal called
|
||||
called = True
|
||||
raise AssertionError("must not call transport")
|
||||
|
||||
classifier = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
)
|
||||
with pytest.raises(DirectVisualColorError, match="between 1 and 2"):
|
||||
classifier.classify([], [VisualCandidate("one", image)])
|
||||
with pytest.raises(DirectVisualColorError, match="between 1 and 2"):
|
||||
classifier.classify([image, image, image, image], [VisualCandidate("one", image)])
|
||||
with pytest.raises(DirectVisualColorError, match="safe limit"):
|
||||
classifier.classify(
|
||||
[image],
|
||||
[VisualCandidate(f"candidate-{index}", image) for index in range(MAX_CANDIDATES + 1)],
|
||||
)
|
||||
assert called is False
|
||||
|
||||
|
||||
def test_http_failure_never_echoes_secret_or_image_payload(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
leaked_marker = "data:image/png;base64,SHOULD-NOT-LEAK"
|
||||
|
||||
def transport(*_args):
|
||||
raise RuntimeError(f"secret-value {leaked_marker}")
|
||||
|
||||
classifier = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="secret-value",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
)
|
||||
with pytest.raises(DirectVisualColorError) as captured:
|
||||
classifier.classify(
|
||||
[source],
|
||||
[VisualCandidate("candidate", candidate)],
|
||||
)
|
||||
|
||||
message = str(captured.value)
|
||||
assert "secret-value" not in message
|
||||
assert "base64" not in message
|
||||
assert captured.value.__cause__ is None
|
||||
|
||||
|
||||
def test_missing_or_invalid_image_fails_before_transport(tmp_path) -> None:
|
||||
invalid = tmp_path / "invalid.png"
|
||||
invalid.write_text("not an image", encoding="utf-8")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
|
||||
with pytest.raises(DirectVisualColorError, match="invalid"):
|
||||
_classifier(b"unused").classify(
|
||||
[invalid],
|
||||
[VisualCandidate("candidate", candidate)],
|
||||
)
|
||||
|
||||
|
||||
def test_color_normalization_is_closed() -> None:
|
||||
assert normalize_coarse_color("Grey") == "unknown"
|
||||
assert normalize_coarse_color("light gray") == "white"
|
||||
assert normalize_coarse_color("\u6d45\u7070") == "white"
|
||||
assert normalize_coarse_color("silver-white") == "white"
|
||||
assert normalize_coarse_color("dark gray") == "black"
|
||||
assert normalize_coarse_color("\u70ad\u7070") == "black"
|
||||
assert normalize_coarse_color("\u6df1\u7070") == "black"
|
||||
assert normalize_coarse_color("\u5361\u5176") == "brown"
|
||||
assert normalize_coarse_color("navy") == "blue"
|
||||
assert normalize_coarse_color("transparent") == "unknown"
|
||||
assert normalize_coarse_color(None) == "unknown"
|
||||
|
||||
|
||||
def test_gray_model_output_is_unknown_and_never_matches(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "gray")
|
||||
candidate = _image(tmp_path / "candidate.png", "white")
|
||||
|
||||
result = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "gray",
|
||||
"candidate_colors": {"candidate": "white"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert result.source_color == "unknown"
|
||||
assert result.candidate_colors == {"candidate": "white"}
|
||||
assert result.selected_key is None
|
||||
assert result.selection_basis == "unresolved"
|
||||
|
||||
|
||||
def test_light_neutral_matches_white_but_remains_distinct_from_black(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "#dddddd")
|
||||
candidate = _image(tmp_path / "candidate.png", "white")
|
||||
|
||||
light_result = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "light gray",
|
||||
"candidate_colors": {"candidate": "white"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
dark_result = _classifier(
|
||||
_response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"candidate": "light gray"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
assert light_result.source_color == "white"
|
||||
assert light_result.selected_key == "candidate"
|
||||
assert dark_result.source_color == "black"
|
||||
assert dark_result.candidate_colors == {"candidate": "white"}
|
||||
assert dark_result.selected_key is None
|
||||
|
||||
|
||||
def test_prompt_and_schema_split_neutrals_by_brightness(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "#dddddd")
|
||||
candidate = _image(tmp_path / "candidate.png", "white")
|
||||
observed: dict[str, Any] = {}
|
||||
|
||||
def transport(_url, _headers, body, _timeout):
|
||||
payload = json.loads(body)
|
||||
observed["instruction"] = payload["messages"][1]["content"][0]["text"]
|
||||
return _response(
|
||||
{
|
||||
"source_color": "white",
|
||||
"candidate_colors": {"candidate": "white"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
instruction = str(observed["instruction"])
|
||||
schema = json.loads(instruction.rsplit("shape: ", maxsplit=1)[1])
|
||||
allowed_value = "one of [" + ", ".join(COARSE_COLORS) + "]"
|
||||
assert "gray" not in COARSE_COLORS
|
||||
assert schema["source_color"] == allowed_value
|
||||
assert schema["candidate_colors"] == {"candidate": allowed_value}
|
||||
assert "silver-white, and light gray" in instruction
|
||||
assert "black includes black, charcoal, and dark gray" in instruction
|
||||
assert "Never output gray as a standalone color" in instruction
|
||||
assert result.source_color == "white"
|
||||
assert result.selected_key == "candidate"
|
||||
|
||||
|
||||
def test_prompt_classifies_only_bag_body_and_ignores_clothing_and_overlays(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
candidate = _image(tmp_path / "candidate.png", "black")
|
||||
observed: dict[str, str] = {}
|
||||
|
||||
def transport(_url, _headers, body, _timeout):
|
||||
payload = json.loads(body)
|
||||
observed["system"] = payload["messages"][0]["content"]
|
||||
observed["instruction"] = payload["messages"][1]["content"][0]["text"]
|
||||
return _response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {"candidate": "black"},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify([source], [VisualCandidate("candidate", candidate)])
|
||||
|
||||
instruction = observed["instruction"]
|
||||
assert "Classify the BAG itself, not the whole image" in instruction
|
||||
assert "only the dominant color of the bag's main body panels" in instruction
|
||||
assert "product color is never the color of a person or their clothing" in instruction
|
||||
assert "jackets, shirts, pants" in instruction
|
||||
assert "text, prices, badges" in instruction
|
||||
assert "black bag on a light-gray jacket is black" in instruction
|
||||
assert "black or navy bag on green clothing" in instruction
|
||||
assert "Never classify the overall image palette" in instruction
|
||||
assert "return unknown" in instruction
|
||||
assert "When that count is 0 or 1" in instruction
|
||||
assert 'visual_match MUST be exactly {"unique":false,"key":null}' in instruction
|
||||
assert "never put the sole same-color candidate" in instruction
|
||||
assert "Only when the count is 2 or more" in instruction
|
||||
assert "fail-closed product image classifier" in observed["system"]
|
||||
assert result.source_color == "black"
|
||||
assert result.selected_key == "candidate"
|
||||
|
||||
|
||||
def test_prompt_repeats_exact_opaque_candidate_key_allowlist(tmp_path) -> None:
|
||||
source = _image(tmp_path / "source.png", "black")
|
||||
first = _image(tmp_path / "first.png", "black")
|
||||
second = _image(tmp_path / "second.png", "white")
|
||||
exact_keys = ("card-eb51a74da3a3", "card-eb51a74da3af")
|
||||
observed: dict[str, str] = {}
|
||||
|
||||
def transport(_url, _headers, body, _timeout):
|
||||
payload = json.loads(body)
|
||||
observed["instruction"] = payload["messages"][1]["content"][0]["text"]
|
||||
return _response(
|
||||
{
|
||||
"source_color": "black",
|
||||
"candidate_colors": {
|
||||
exact_keys[0]: "black",
|
||||
exact_keys[1]: "white",
|
||||
},
|
||||
"visual_match": {"unique": False, "key": None},
|
||||
}
|
||||
)
|
||||
|
||||
result = DirectVisualColorClassifier(
|
||||
transport=transport,
|
||||
config_resolver=lambda: DirectLLMConfig(
|
||||
endpoint="https://api.minimaxi.com/v1/chat/completions",
|
||||
credential="test-only-credential",
|
||||
title_model="MiniMax-M3",
|
||||
vision_model="MiniMax-M3",
|
||||
),
|
||||
).classify(
|
||||
[source],
|
||||
[VisualCandidate(exact_keys[0], first), VisualCandidate(exact_keys[1], second)],
|
||||
)
|
||||
|
||||
instruction = observed["instruction"]
|
||||
allowlist = json.dumps(list(exact_keys), separators=(",", ":"))
|
||||
assert instruction.count(allowlist) == 2
|
||||
assert "Candidate keys are opaque, immutable identifiers" in instruction
|
||||
assert "byte-for-byte exactly as provided" in instruction
|
||||
assert "do not edit, invent, autocomplete, normalize, truncate, or repair" in instruction
|
||||
assert "each provided candidate key exactly once and no other keys" in instruction
|
||||
assert result.selected_key == exact_keys[0]
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from collect_dy_market_rank import (
|
||||
@@ -7,12 +9,14 @@ from collect_dy_market_rank import (
|
||||
build_dy_category_table_xml,
|
||||
build_dy_feishu_batch_xml,
|
||||
build_dy_feishu_intro_xml,
|
||||
install_saved_cookies,
|
||||
is_excluded_product_title,
|
||||
navigate_to_product_rank,
|
||||
parse_market_product_rank_html,
|
||||
parse_money_range_upper,
|
||||
payment_metric_header_index,
|
||||
require_payment_metric_header,
|
||||
snapshot_rank_page,
|
||||
)
|
||||
from collect_sycm_market_rank import build_feishu_batch_xml
|
||||
|
||||
@@ -54,6 +58,24 @@ SAMPLE_HTML = """
|
||||
|
||||
|
||||
class DyMarketRankTests(unittest.TestCase):
|
||||
def test_saved_cookies_refresh_existing_context(self):
|
||||
context = MagicMock()
|
||||
cookies = [{"name": "session", "value": "secret"}]
|
||||
|
||||
self.assertEqual(install_saved_cookies(context, cookies), 1)
|
||||
context.add_cookies.assert_called_once_with(cookies)
|
||||
|
||||
def test_rank_timeout_snapshot_persists_dom(self):
|
||||
page = MagicMock()
|
||||
page.content.return_value = "<html><body>新版榜单</body></html>"
|
||||
|
||||
with TemporaryDirectory() as temp_dir:
|
||||
with patch("collect_dy_market_rank.DEBUG_DIR", Path(temp_dir)):
|
||||
path = snapshot_rank_page(page, "header timeout")
|
||||
|
||||
self.assertTrue(path.is_file())
|
||||
self.assertIn("新版榜单", path.read_text(encoding="utf-8"))
|
||||
|
||||
def test_title_blacklist_filters_children_primary_and_middle_school(self):
|
||||
self.assertTrue(is_excluded_product_title("儿童轻便双肩包"))
|
||||
self.assertTrue(is_excluded_product_title("小学生护脊书包"))
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.product_commerce import backfill_erp_all_shop_daily as backfill
|
||||
from gyxx_flow.modules.product_commerce.db import (
|
||||
upsert_erp_all_shop_style_daily_metrics,
|
||||
)
|
||||
|
||||
|
||||
def test_default_range_is_yesterday_only() -> None:
|
||||
assert backfill.DEFAULT_FROM_DATE == backfill.DEFAULT_TO_DATE
|
||||
assert backfill.DEFAULT_FROM_DATE == date.today() - backfill.timedelta(days=1)
|
||||
|
||||
|
||||
def test_historical_range_is_inclusive_and_covers_103_days() -> None:
|
||||
dates = backfill.inclusive_dates(date(2026, 5, 1), date(2026, 8, 11))
|
||||
|
||||
assert dates[0] == date(2026, 5, 1)
|
||||
assert dates[-1] == date(2026, 8, 11)
|
||||
assert len(dates) == 103
|
||||
|
||||
|
||||
def test_invalid_or_oversized_range_is_rejected() -> None:
|
||||
with pytest.raises(ValueError, match="--to"):
|
||||
backfill.inclusive_dates(date(2026, 8, 12), date(2026, 8, 11))
|
||||
with pytest.raises(ValueError, match="366"):
|
||||
backfill.inclusive_dates(date(2025, 1, 1), date(2026, 8, 11))
|
||||
|
||||
|
||||
def test_style_plan_rejects_one_erp_code_owned_by_two_styles() -> None:
|
||||
with pytest.raises(ValueError, match="belongs to both"):
|
||||
backfill.build_style_plans(
|
||||
{
|
||||
"styles": {
|
||||
"款式甲": {"erp_codes": ["10416"]},
|
||||
"款式乙": {"erp_codes": ["10416"]},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_all_shop_filter_selects_every_shop_and_batches_all_codes() -> None:
|
||||
class Frame:
|
||||
def __init__(self) -> None:
|
||||
self.script = ""
|
||||
self.args = {}
|
||||
|
||||
def evaluate(self, script, args):
|
||||
self.script = script
|
||||
self.args = args
|
||||
return {
|
||||
"selected_shop_count": 38,
|
||||
"selection_mode": "all",
|
||||
}
|
||||
|
||||
frame = Frame()
|
||||
result = backfill.set_all_shop_daily_filters(
|
||||
frame,
|
||||
report_date=date(2026, 5, 1),
|
||||
erp_codes=("10416", "10455"),
|
||||
)
|
||||
|
||||
assert frame.args == {
|
||||
"reportDate": "2026-05-01",
|
||||
"erpCodes": ["10416", "10455"],
|
||||
}
|
||||
assert 'input[name="shop_id"]' in frame.script
|
||||
assert "=== '全选'" in frame.script
|
||||
assert "erpCodes.join(',')" in frame.script
|
||||
assert "shopLabels" not in frame.script
|
||||
assert result == {"selected_shop_count": 38, "selection_mode": "all"}
|
||||
|
||||
|
||||
def test_style_records_sum_product_rows_and_keep_zero_styles() -> None:
|
||||
plans = (
|
||||
backfill.StylePlan("款式甲", ("10416", "10455")),
|
||||
backfill.StylePlan("款式乙", ("10504",)),
|
||||
)
|
||||
rows = [
|
||||
{"erp_style_code": "10416", "sales": 12, "return_qty": 2},
|
||||
{"erp_style_code": "10416", "sales": 8, "return_qty": 1},
|
||||
{"erp_style_code": "10455", "sales": 5, "return_qty": 3},
|
||||
]
|
||||
|
||||
records = backfill.aggregate_style_records(
|
||||
date(2026, 5, 1),
|
||||
plans,
|
||||
rows,
|
||||
selected_shop_count=38,
|
||||
source="raw/product_commerce/evidence.json",
|
||||
)
|
||||
|
||||
assert records[0] == {
|
||||
"metric_date": date(2026, 5, 1),
|
||||
"style_name": "款式甲",
|
||||
"sales": 25,
|
||||
"return_qty": 6,
|
||||
"erp_style_codes": ["10416", "10455"],
|
||||
"selected_shop_count": 38,
|
||||
"matched_product_rows": 3,
|
||||
"source": "raw/product_commerce/evidence.json",
|
||||
}
|
||||
assert records[1]["sales"] == 0
|
||||
assert records[1]["return_qty"] == 0
|
||||
assert records[1]["matched_product_rows"] == 0
|
||||
|
||||
|
||||
def test_range_collector_queries_once_per_date_not_once_per_style(monkeypatch) -> None:
|
||||
filter_frame = object()
|
||||
monkeypatch.setattr(
|
||||
backfill.erp, "find_report_frames", lambda _page: (filter_frame, object())
|
||||
)
|
||||
calls: list[date] = []
|
||||
|
||||
def collect_one(_page, _frame, _plans, report_date, **_kwargs):
|
||||
calls.append(report_date)
|
||||
return (
|
||||
{"selected_shop_count": 38},
|
||||
{"rows": [], "frame_url": f"https://example/{report_date}", "signature": str(report_date)},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(backfill, "collect_one_date", collect_one)
|
||||
persisted: list[date] = []
|
||||
backfill.collect_range_in_page(
|
||||
object(),
|
||||
(
|
||||
backfill.StylePlan("款式甲", ("10416", "10455")),
|
||||
backfill.StylePlan("款式乙", ("10504",)),
|
||||
),
|
||||
(date(2026, 5, 1), date(2026, 5, 2)),
|
||||
persist=lambda report_date, _selection, _report: persisted.append(report_date),
|
||||
)
|
||||
|
||||
assert calls == [date(2026, 5, 1), date(2026, 5, 2)]
|
||||
assert persisted == calls
|
||||
|
||||
|
||||
def test_database_upsert_uses_date_and_style_conflict_key(monkeypatch) -> None:
|
||||
captured = {}
|
||||
|
||||
class Cursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
class Connection:
|
||||
def cursor(self):
|
||||
return Cursor()
|
||||
|
||||
def fake_execute_values(cursor, sql, rows):
|
||||
captured.update(cursor=cursor, sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"gyxx_flow.modules.product_commerce.db.execute_values", fake_execute_values
|
||||
)
|
||||
written = upsert_erp_all_shop_style_daily_metrics(
|
||||
Connection(),
|
||||
[
|
||||
{
|
||||
"metric_date": date(2026, 5, 1),
|
||||
"style_name": "款式甲",
|
||||
"sales": 25,
|
||||
"return_qty": 6,
|
||||
"erp_style_codes": ["10455", "10416"],
|
||||
"selected_shop_count": 38,
|
||||
"matched_product_rows": 3,
|
||||
"source": "raw/evidence.json",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
assert "ON CONFLICT (metric_date, style_name)" in captured["sql"]
|
||||
assert captured["rows"][0][2:4] == (25, 6)
|
||||
assert captured["rows"][0][4] == ["10416", "10455"]
|
||||
|
||||
|
||||
def test_schema_keeps_all_shop_metrics_separate_from_platform_metrics() -> None:
|
||||
schema = (
|
||||
Path(backfill.__file__).parent / "db" / "schema.sql"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "CREATE TABLE IF NOT EXISTS erp_all_shop_style_daily_metrics" in schema
|
||||
assert "PRIMARY KEY (metric_date, style_name)" in schema
|
||||
assert "return_qty" in schema
|
||||
@@ -5,22 +5,97 @@ from unittest.mock import Mock, patch
|
||||
|
||||
from upload_video_to_guanghe import (
|
||||
GuangHeUploader,
|
||||
TmallTitleGenerationError,
|
||||
_match_tm_item_ids_from_product_titles,
|
||||
_select_video_product_names,
|
||||
_tm_product_title_search_pattern,
|
||||
download_videos,
|
||||
)
|
||||
|
||||
|
||||
def test_tm_product_title_search_pattern_uses_model_token() -> None:
|
||||
assert _tm_product_title_search_pattern("拾影单肩包") == "%拾影%"
|
||||
assert _tm_product_title_search_pattern("星云mini") == "%星云mini%"
|
||||
|
||||
|
||||
def test_multi_style_record_prefers_style_named_by_attachment() -> None:
|
||||
assert _select_video_product_names(
|
||||
["阿波罗2", "宙斯3"], "宙斯3 7.30.mp4"
|
||||
) == ["宙斯3"]
|
||||
assert _select_video_product_names(
|
||||
["阿波罗2", "宙斯3"], "7月30日.mp4"
|
||||
) == ["阿波罗2", "宙斯3"]
|
||||
|
||||
|
||||
def test_attachment_style_matching_handles_combos_and_overlapping_aliases() -> None:
|
||||
assert _select_video_product_names(
|
||||
["阿波罗2", "宙斯3", "星云2", "逐星"],
|
||||
"宙斯3+阿波罗2+星云2+逐星.m4v",
|
||||
) == ["宙斯3", "阿波罗2", "星云2", "逐星"]
|
||||
assert _select_video_product_names(
|
||||
["极星双肩", "极星双肩2"], "极星双肩2 新新新.mp4"
|
||||
) == ["极星双肩2"]
|
||||
|
||||
|
||||
class GuangHeMetadataTests(unittest.TestCase):
|
||||
def test_formal_title_rejection_uses_non_browser_failure_type(self):
|
||||
GuangHeUploader.configure_catalog_product_titles(
|
||||
{"极星双肩2": ["极星双肩2双肩包轻便电脑包"]}
|
||||
)
|
||||
uploader = GuangHeUploader("", "", require_llm_titles=True)
|
||||
try:
|
||||
with patch(
|
||||
"upload_video_to_guanghe.JdVideoTitleGenerator.generate",
|
||||
side_effect=RuntimeError("候选不合格"),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
TmallTitleGenerationError, "候选不合格"
|
||||
):
|
||||
uploader._generate_title(["极星双肩2"], video_key="token")
|
||||
finally:
|
||||
GuangHeUploader.configure_catalog_product_titles({})
|
||||
|
||||
def test_formal_collection_title_forbids_specific_style_names(self):
|
||||
GuangHeUploader.configure_catalog_product_titles(
|
||||
{
|
||||
"宙斯3": ["宙斯3双肩包男士大容量"],
|
||||
"阿波罗2": ["阿波罗2电脑包手提轻便"],
|
||||
}
|
||||
)
|
||||
uploader = GuangHeUploader("", "", require_llm_titles=True)
|
||||
try:
|
||||
with patch(
|
||||
"upload_video_to_guanghe.JdVideoTitleGenerator"
|
||||
) as generator_class:
|
||||
generator = generator_class.return_value
|
||||
generator.generate.return_value = (
|
||||
"多款通勤包袋,大容量轻便按场景切换"
|
||||
)
|
||||
title = uploader._generate_title(
|
||||
["宙斯3", "阿波罗2"], video_key="combo-token"
|
||||
)
|
||||
self.assertEqual(title, "多款通勤包袋,大容量轻便按场景切换")
|
||||
self.assertTrue(generator_class.call_args.kwargs["generic_collection"])
|
||||
self.assertFalse(generator_class.call_args.kwargs["require_product_name"])
|
||||
self.assertEqual(generator_class.call_args.kwargs["llm_timeout"], 300)
|
||||
generator.generate.assert_called_once()
|
||||
self.assertEqual(
|
||||
generator.generate.call_args.args[0], ["宙斯3", "阿波罗2"]
|
||||
)
|
||||
finally:
|
||||
GuangHeUploader.configure_catalog_product_titles({})
|
||||
|
||||
def test_downloads_with_same_attachment_name_are_isolated_by_record(self):
|
||||
attachment = {"file_token": "file-token", "name": "same-name.mp4"}
|
||||
|
||||
with TemporaryDirectory() as tmp_dir, patch(
|
||||
"upload_video_to_guanghe.DOWNLOAD_DIR", Path(tmp_dir)
|
||||
), patch("upload_video_to_guanghe.run_lark") as run_lark_mock:
|
||||
def fake_download(args):
|
||||
output_dir = Path(args[args.index("--output") + 1])
|
||||
def fake_download(args, *, cwd=None, timeout=None):
|
||||
self.assertEqual(timeout, 600)
|
||||
output_dir = Path(cwd)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / attachment["name"]).write_bytes(b"video")
|
||||
(output_dir / args[args.index("--output") + 1]).write_bytes(b"video")
|
||||
return 0, "", ""
|
||||
|
||||
run_lark_mock.side_effect = fake_download
|
||||
@@ -28,8 +103,9 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
first = download_videos("record-one", [attachment])
|
||||
second = download_videos("record-two", [attachment])
|
||||
|
||||
self.assertEqual(first[0].parent.name, "record-one")
|
||||
self.assertEqual(second[0].parent.name, "record-two")
|
||||
self.assertEqual(first[0].parent.parent.name, "record-one")
|
||||
self.assertEqual(second[0].parent.parent.name, "record-two")
|
||||
self.assertEqual(first[0].parent.name, "file-token")
|
||||
self.assertNotEqual(first[0], second[0])
|
||||
|
||||
def test_unresolved_style_uses_exact_database_product_title_match(self):
|
||||
@@ -81,14 +157,20 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
def test_topic_result_card_requires_topic_and_activity_statistics(self):
|
||||
self.assertTrue(
|
||||
GuangHeUploader._topic_result_card_text_matches(
|
||||
"# 在淘宝种草一夏 小二推荐 作品数469.5万 参与数16.2万",
|
||||
"在淘宝种草一夏",
|
||||
"# 我的夏日焕新清单 小二推荐 作品数469.5万 参与数16.2万",
|
||||
"我的夏日焕新清单",
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
GuangHeUploader._topic_result_card_text_matches(
|
||||
"# 秋上新开拍了 时尚 作品数198.3万 参与数11.3万",
|
||||
"秋上新",
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
GuangHeUploader._topic_result_card_text_matches(
|
||||
"在淘宝种草一夏",
|
||||
"在淘宝种草一夏",
|
||||
"我的夏日焕新清单",
|
||||
"我的夏日焕新清单",
|
||||
)
|
||||
)
|
||||
self.assertTrue(GuangHeUploader._topic_card_box_is_clickable(620, 120))
|
||||
@@ -160,13 +242,12 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertGreaterEqual(len(title), 26)
|
||||
self.assertLessEqual(len(title), 30)
|
||||
|
||||
def test_same_style_multiple_videos_get_distinct_titles_when_hermes_repeats(self):
|
||||
def test_same_style_multiple_videos_get_distinct_titles_when_model_repeats(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
repeated = "光影行星宙斯双肩包科学分区收纳轻量通勤出行"
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
|
||||
patch("upload_video_to_guanghe.call_direct_llm", return_value=(repeated, {})),
|
||||
):
|
||||
first = uploader._generate_title(["宙斯"], video_index=1, video_total=2)
|
||||
second = uploader._generate_title(["宙斯"], video_index=2, video_total=2)
|
||||
@@ -175,13 +256,12 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertTrue(first.startswith("宙斯双肩包,"))
|
||||
self.assertTrue(second.startswith("宙斯双肩包,"))
|
||||
|
||||
def test_same_style_repeated_hermes_copy_gets_low_similarity_bodies(self):
|
||||
def test_same_style_repeated_model_copy_gets_low_similarity_bodies(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
repeated = "光影行星宙斯双肩包,科学分区收纳,轻量通勤出行!"
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
|
||||
patch("upload_video_to_guanghe.call_direct_llm", return_value=(repeated, {})),
|
||||
):
|
||||
titles = [
|
||||
uploader._generate_title(["宙斯"], video_index=index, video_total=4)
|
||||
@@ -202,8 +282,7 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
repeated = "科学分区收纳,轻装通勤出行"
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
|
||||
patch("upload_video_to_guanghe.call_direct_llm", return_value=(repeated, {})),
|
||||
):
|
||||
first = uploader._generate_title(["宙斯"], video_index=1)
|
||||
second = uploader._generate_title(["星云2"], video_index=1)
|
||||
@@ -339,9 +418,8 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch(
|
||||
"upload_video_to_guanghe._call_hermes",
|
||||
"upload_video_to_guanghe.call_direct_llm",
|
||||
return_value=("光影行星宙斯双肩包大容量笔记本电脑轻量通勤", {}),
|
||||
),
|
||||
):
|
||||
@@ -350,7 +428,7 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertTrue(title.startswith("宙斯双肩包,"))
|
||||
self.assertNotIn("包袋双肩包", title)
|
||||
|
||||
def test_generated_poseidon_title_cannot_be_changed_to_backpack_by_hermes(self):
|
||||
def test_generated_poseidon_title_cannot_be_changed_to_backpack_by_model(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
with (
|
||||
patch.object(
|
||||
@@ -358,9 +436,8 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
"_search_product_features",
|
||||
return_value="双肩包、电脑包、托特包、大容量",
|
||||
),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch(
|
||||
"upload_video_to_guanghe._call_hermes",
|
||||
"upload_video_to_guanghe.call_direct_llm",
|
||||
return_value=(
|
||||
"光影行星波塞冬edge双肩包暴雨通勤防泼水电脑仓从容出行",
|
||||
{},
|
||||
@@ -439,8 +516,8 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
with self.subTest(product_names=product_names):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
calls = []
|
||||
uploader._pick_content_labels = Mock(
|
||||
side_effect=lambda labels: calls.append(("labels", list(labels)))
|
||||
uploader._pick_content_label = Mock(
|
||||
side_effect=lambda label: calls.append(("label", label))
|
||||
)
|
||||
uploader._pick_topic = Mock(
|
||||
side_effect=lambda topic: calls.append(("topic", topic))
|
||||
@@ -452,8 +529,8 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
calls,
|
||||
[
|
||||
("labels", expected_labels),
|
||||
("topic", "在淘宝种草一夏"),
|
||||
*[("label", label) for label in expected_labels],
|
||||
("topic", "我的夏日焕新清单"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -461,8 +538,55 @@ class GuangHeMetadataTests(unittest.TestCase):
|
||||
self.assertEqual(GuangHeUploader.FIXED_CONTENT_LABELS, ("商品展示",))
|
||||
self.assertNotIn("户外", GuangHeUploader.FIXED_CONTENT_LABELS)
|
||||
|
||||
def test_missing_user_selected_topic_stops_metadata_preparation(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
uploader._pick_content_label = Mock()
|
||||
uploader._pick_topic = Mock(
|
||||
side_effect=RuntimeError("没有找到固定话题: #我的夏日焕新清单")
|
||||
)
|
||||
uploader._snapshot = Mock()
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "没有找到固定话题"):
|
||||
uploader._prepare_publish_metadata(["极星双肩"])
|
||||
|
||||
self.assertEqual(
|
||||
uploader._pick_content_label.call_args_list,
|
||||
[unittest.mock.call("双肩包"), unittest.mock.call("商品展示")],
|
||||
)
|
||||
|
||||
def test_optional_content_label_failures_do_not_skip_required_topic(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
uploader.page = Mock()
|
||||
uploader._pick_content_label = Mock(
|
||||
side_effect=[
|
||||
RuntimeError("内容标签没有精确匹配项: #双肩包"),
|
||||
RuntimeError("内容标签没有精确匹配项: #商品展示"),
|
||||
]
|
||||
)
|
||||
uploader._pick_topic = Mock()
|
||||
uploader._snapshot = Mock()
|
||||
|
||||
uploader._prepare_publish_metadata(["极星双肩"])
|
||||
|
||||
self.assertEqual(
|
||||
uploader._pick_content_label.call_args_list,
|
||||
[unittest.mock.call("双肩包"), unittest.mock.call("商品展示")],
|
||||
)
|
||||
uploader._pick_topic.assert_called_once_with("我的夏日焕新清单")
|
||||
|
||||
def test_metadata_uses_operator_topic_keyword(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
calls = []
|
||||
uploader._pick_content_label = Mock()
|
||||
uploader._pick_topic = Mock(side_effect=lambda topic: calls.append(topic))
|
||||
uploader._snapshot = Mock()
|
||||
|
||||
uploader._prepare_publish_metadata(["极星双肩"], topic_keyword="秋上新")
|
||||
|
||||
self.assertEqual(calls, ["秋上新"])
|
||||
|
||||
def test_fixed_topic_is_not_a_recommended_first_result(self):
|
||||
self.assertEqual(GuangHeUploader.FIXED_TOPIC, "在淘宝种草一夏")
|
||||
self.assertEqual(GuangHeUploader.FIXED_TOPIC, "我的夏日焕新清单")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
import db as product_db
|
||||
import import_douyin_qianchuan_ads as importer
|
||||
from openpyxl import Workbook
|
||||
|
||||
HEADERS = [
|
||||
"商品ID",
|
||||
"商品名称",
|
||||
"日期",
|
||||
"整体展示次数",
|
||||
"整体点击次数",
|
||||
"整体点击率",
|
||||
"整体转化率",
|
||||
"整体消耗(元)",
|
||||
"整体成交金额(元)",
|
||||
"整体支付ROI",
|
||||
"整体成交订单成本(元)",
|
||||
"用户实际支付金额(元)",
|
||||
"电商平台补贴金额(元)",
|
||||
"净成交ROI",
|
||||
"净成交金额(元)",
|
||||
"净成交订单成本(元)",
|
||||
"净成交金额结算率",
|
||||
"1小时内退款率",
|
||||
]
|
||||
|
||||
|
||||
def _workbook(path) -> None:
|
||||
workbook = Workbook()
|
||||
sheet = workbook.active
|
||||
sheet.append(HEADERS)
|
||||
sheet.append(
|
||||
[
|
||||
"3725242433383039375",
|
||||
"测试商品",
|
||||
"全部",
|
||||
"12,345",
|
||||
"678",
|
||||
"5.49%",
|
||||
"1.00%",
|
||||
"100.00",
|
||||
"900.00",
|
||||
"9.00",
|
||||
"25.00",
|
||||
"850.00",
|
||||
"50.00",
|
||||
"8.00",
|
||||
"800.00",
|
||||
"30.00",
|
||||
"88.89%",
|
||||
"11.11%",
|
||||
]
|
||||
)
|
||||
sheet.append(
|
||||
[
|
||||
"3725242433383039375",
|
||||
"测试商品",
|
||||
"2026-08-23",
|
||||
"12,345",
|
||||
"678",
|
||||
"5.49%",
|
||||
"1.00%",
|
||||
"100.00",
|
||||
"900.00",
|
||||
"9.00",
|
||||
"25.00",
|
||||
"850.00",
|
||||
"50.00",
|
||||
"8.00",
|
||||
"800.00",
|
||||
"30.00",
|
||||
"88.89%",
|
||||
"11.11%",
|
||||
]
|
||||
)
|
||||
workbook.save(path)
|
||||
|
||||
|
||||
def test_parse_workbook_keeps_long_product_id_and_skips_total(tmp_path) -> None:
|
||||
path = tmp_path / "qianchuan.xlsx"
|
||||
_workbook(path)
|
||||
|
||||
records = importer.parse_workbook(path)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["stat_date"] == date(2026, 8, 23)
|
||||
assert records[0]["product_id"] == "3725242433383039375"
|
||||
assert records[0]["overall_impressions"] == 12345
|
||||
assert records[0]["spend"] == Decimal("100.00")
|
||||
assert records[0]["refund_rate_1h"] == Decimal("11.11")
|
||||
|
||||
|
||||
def test_upsert_uses_advertiser_date_product_business_key(monkeypatch) -> None:
|
||||
executed = {}
|
||||
|
||||
class Cursor:
|
||||
def execute(self, sql, params=None):
|
||||
executed["lookup_sql"] = sql
|
||||
executed["lookup_params"] = params
|
||||
|
||||
def fetchall(self):
|
||||
return [("测试款式", ["3725242433383039375"])]
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback):
|
||||
return None
|
||||
|
||||
class Connection:
|
||||
def cursor(self):
|
||||
return Cursor()
|
||||
|
||||
def fake_execute_values(cursor, sql, rows):
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
"product_id": "3725242433383039375",
|
||||
"product_name": "测试商品",
|
||||
"spend": Decimal("100.00"),
|
||||
}
|
||||
|
||||
assert product_db.upsert_douyin_product_ad_daily_metrics(
|
||||
Connection(), [record, record]
|
||||
) == 1
|
||||
assert "ON CONFLICT (stat_date, advertiser_id, product_id)" in executed["sql"]
|
||||
assert len(executed["rows"]) == 1
|
||||
assert "style_name" in executed["sql"]
|
||||
assert executed["rows"][0][4] == "测试款式"
|
||||
assert executed["rows"][0][5] == "product_id"
|
||||
|
||||
|
||||
def test_upsert_falls_back_to_exact_product_name_style_match(monkeypatch) -> None:
|
||||
executed = {}
|
||||
|
||||
class Cursor:
|
||||
def execute(self, sql, params=None):
|
||||
executed["lookup_sql"] = sql
|
||||
executed["lookup_params"] = params
|
||||
|
||||
def fetchall(self):
|
||||
return [("测试款式", [])]
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback):
|
||||
return None
|
||||
|
||||
class Connection:
|
||||
def cursor(self):
|
||||
return Cursor()
|
||||
|
||||
def fake_execute_values(cursor, sql, rows):
|
||||
executed.update(sql=sql, rows=rows)
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_id": importer.ADVERTISER_ID,
|
||||
"product_id": "unmapped-product-id",
|
||||
"product_name": "GYXX/测试款式3双肩包2026新款",
|
||||
"spend": Decimal("100.00"),
|
||||
}
|
||||
|
||||
assert product_db.upsert_douyin_product_ad_daily_metrics(Connection(), [record]) == 1
|
||||
assert executed["rows"][0][4] == "测试款式"
|
||||
assert executed["rows"][0][5] == "product_name"
|
||||
@@ -0,0 +1,215 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import db as product_db
|
||||
import import_jd_ad_costs as importer
|
||||
|
||||
|
||||
def test_jd_exports_only_import_current_block_and_ignore_comparison_rows(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
full = tmp_path / "full.csv"
|
||||
full.write_text(
|
||||
"查询条件,广告主PIN: 光影行星旗舰店霖; 时间: 2026-08-23\n"
|
||||
"日期,'当前时间',SPU ID,商品计划名称,SpuImgUrl,花费,全站投产比\n"
|
||||
"20260823~20260823,当前时间,spu-1,盖亚款式,img,12.30,4.50\n"
|
||||
"汇总,,,汇总,,12.30,4.50\n"
|
||||
"20260822~20260822,对比时间,spu-old,旧款,img,99.00,9.00\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
sku = tmp_path / "sku.csv"
|
||||
sku.write_text(
|
||||
"查询条件,广告主PIN: 光影行星旗舰店霖; 时间: 2026-08-23\n"
|
||||
"SKU ID,SKU 名称,点击时间,'当前时间',花费,展现数,点击数,投产比\n"
|
||||
"sku-1,光影行星 盖亚款式 黑色,20260823~20260823,当前时间,3.20,100,5,2.50\n"
|
||||
"sku-old,光影行星 盖亚款式 黑色,20260822~20260822,对比时间,99.00,100,5,9.00\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
|
||||
full_records = importer.parse_full_csv(full, date(2026, 8, 23))
|
||||
sku_records = importer.parse_non_full_csv(sku, date(2026, 8, 23))
|
||||
|
||||
assert [record["spu_id"] for record in full_records] == ["spu-1"]
|
||||
assert [record["sku_id"] for record in sku_records] == ["sku-1"]
|
||||
assert full_records[0]["metrics"]["source"]["current_marker"] == "当前时间"
|
||||
assert full_records[0]["spend"] == importer.Decimal("12.30")
|
||||
assert sku_records[0]["metrics"]["clicks"] == 5
|
||||
|
||||
|
||||
def test_sku_name_matches_dim_style_spu_and_keeps_audit_fields(tmp_path: Path) -> None:
|
||||
report = tmp_path / "sku.csv"
|
||||
report.write_text(
|
||||
"查询条件,广告主PIN: 光影行星旗舰店霖\n"
|
||||
"SKU ID,SKU 名称,点击时间,'当前时间',花费,投产比\n"
|
||||
"sku-1,光影行星(GYXX)盖亚C1斜挎包 曜夜黑,20260823~20260823,当前时间,3.20,2.50\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
sku_records = importer.parse_non_full_csv(report, date(2026, 8, 23))
|
||||
matched = importer.apply_sku_matches(
|
||||
sku_records,
|
||||
style_catalog=[{"style_name": "盖亚斜挎", "jd_spus": ["spu-1"]}],
|
||||
full_records=[{"spu_id": "spu-1", "spu_name": "盖亚C1斜挎包"}],
|
||||
)
|
||||
|
||||
assert matched[0]["match_status"] == "matched"
|
||||
assert matched[0]["spu_id"] == "spu-1"
|
||||
assert matched[0]["style_name"] == "盖亚斜挎"
|
||||
assert matched[0]["metrics"]["match"]["reason"]
|
||||
|
||||
|
||||
def test_full_spu_matches_dim_style_exactly() -> None:
|
||||
full_records = [
|
||||
{
|
||||
"spu_id": "spu-1",
|
||||
"style_name": None,
|
||||
"match_status": "direct",
|
||||
"match_reason": "full-site SPU report",
|
||||
"metrics": {},
|
||||
}
|
||||
]
|
||||
|
||||
matched = importer.apply_full_style_matches(
|
||||
full_records,
|
||||
style_catalog=[
|
||||
{"style_name": "盖亚斜挎", "jd_spus": ["spu-1"]},
|
||||
],
|
||||
)
|
||||
|
||||
assert matched[0]["style_name"] == "盖亚斜挎"
|
||||
assert matched[0]["match_status"] == "direct"
|
||||
assert matched[0]["metrics"]["match"]["style_name"] == "盖亚斜挎"
|
||||
|
||||
|
||||
def test_full_spu_name_matches_base_style_when_spu_mapping_is_missing() -> None:
|
||||
full_records = [
|
||||
{
|
||||
"spu_id": "spu-2",
|
||||
"spu_name": "光影行星(GYXX)盖亚斜挎包3单反相机包",
|
||||
"style_name": None,
|
||||
"match_status": "direct",
|
||||
"match_reason": "full-site SPU report",
|
||||
"metrics": {},
|
||||
}
|
||||
]
|
||||
|
||||
matched = importer.apply_full_style_matches(
|
||||
full_records,
|
||||
style_catalog=[
|
||||
{"style_name": "盖亚斜挎", "jd_spus": []},
|
||||
{"style_name": "盖亚双肩3", "jd_spus": []},
|
||||
],
|
||||
spu_catalog=[
|
||||
{"spu_id": "spu-2", "product_no": "盖亚斜挎3"},
|
||||
],
|
||||
)
|
||||
|
||||
assert matched[0]["style_name"] == "盖亚斜挎"
|
||||
assert matched[0]["metrics"]["match"]["reason"] == (
|
||||
"full-site product_no matched style name"
|
||||
)
|
||||
|
||||
|
||||
def test_sku_name_uses_latest_jd_spu_catalog_when_full_report_lacks_spu() -> None:
|
||||
sku_records = [
|
||||
{
|
||||
"sku_name": "光影行星(GYXX)阿波罗2斜挎包男士通勤便携单肩背包 曜夜黑",
|
||||
"metrics": {},
|
||||
}
|
||||
]
|
||||
matched = importer.apply_sku_matches(
|
||||
sku_records,
|
||||
style_catalog=[
|
||||
{"style_name": "阿波罗2", "jd_spus": ["spu-1", "spu-2"]}
|
||||
],
|
||||
full_records=[],
|
||||
spu_catalog=[
|
||||
{
|
||||
"spu_id": "spu-1",
|
||||
"spu_name": "光影行星(GYXX)阿波罗2斜挎包男士通勤便携单肩背包",
|
||||
},
|
||||
{"spu_id": "spu-2", "spu_name": "另一款商品"},
|
||||
],
|
||||
)
|
||||
|
||||
assert matched[0]["match_status"] == "matched"
|
||||
assert matched[0]["spu_id"] == "spu-1"
|
||||
|
||||
|
||||
class _Cursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
class _Connection:
|
||||
def cursor(self):
|
||||
return _Cursor()
|
||||
|
||||
|
||||
def test_jd_upsert_deduplicates_same_logical_row(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows, *, template=None):
|
||||
captured["sql"] = sql
|
||||
captured["rows"] = rows
|
||||
captured["template"] = template
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
record = {
|
||||
"stat_date": date(2026, 8, 23),
|
||||
"advertiser_pin": "光影行星旗舰店霖",
|
||||
"shop_account": "光影行星研发",
|
||||
"report_type": "full",
|
||||
"row_key": "spu:spu-1",
|
||||
"spu_id": "spu-1",
|
||||
"spu_name": "盖亚",
|
||||
"match_status": "direct",
|
||||
"spend": importer.Decimal("12.30"),
|
||||
"reported_roi": importer.Decimal("4.50"),
|
||||
"metrics": {"spend": "12.30"},
|
||||
"source_file": "full.csv",
|
||||
"source_row_number": 3,
|
||||
}
|
||||
|
||||
assert product_db.upsert_jd_product_ad_daily_metrics(
|
||||
_Connection(), [record, {**record, "spend": importer.Decimal("13.30")}]
|
||||
) == 1
|
||||
assert len(captured["rows"]) == 1
|
||||
assert captured["rows"][0][11] == importer.Decimal("13.30")
|
||||
assert "jd_product_ad_daily_metrics" in str(captured["sql"])
|
||||
|
||||
|
||||
def test_jd_dry_run_accepts_explicit_desktop_style_files(tmp_path: Path, capsys) -> None:
|
||||
full = tmp_path / "full.csv"
|
||||
full.write_text(
|
||||
"条件,广告主PIN: 光影行星旗舰店霖\n"
|
||||
"日期,'当前时间',SPU ID,商品计划名称,花费\n"
|
||||
"20260823~20260823,当前时间,spu-1,盖亚,1.00\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
sku = tmp_path / "sku.csv"
|
||||
sku.write_text(
|
||||
"条件,广告主PIN: 光影行星旗舰店霖\n"
|
||||
"SKU ID,SKU 名称,点击时间,'当前时间',花费\n"
|
||||
"sku-1,盖亚黑色,20260823~20260823,当前时间,1.00\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
|
||||
assert importer.main(
|
||||
[
|
||||
"--date",
|
||||
"2026-08-23",
|
||||
"--full-file",
|
||||
str(full),
|
||||
"--non-full-file",
|
||||
str(sku),
|
||||
]
|
||||
) == 0
|
||||
output = capsys.readouterr().out
|
||||
assert "全站 SPU 1" in output
|
||||
assert "非全站 SKU 1" in output
|
||||
assert "DRY-RUN" in output
|
||||
@@ -0,0 +1,187 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import db as product_db
|
||||
import import_tmall_product_ads as importer
|
||||
|
||||
|
||||
class _Cursor:
|
||||
def execute(self, sql, params=None):
|
||||
self.sql = sql
|
||||
self.params = params
|
||||
|
||||
def fetchall(self):
|
||||
return [("极星", ["1001"])]
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
class _Connection:
|
||||
def cursor(self):
|
||||
return _Cursor()
|
||||
|
||||
|
||||
def test_parse_tmall_product_ad_csv_merges_product_rows(tmp_path: Path) -> None:
|
||||
report = tmp_path / "商品报表.csv"
|
||||
report.write_text(
|
||||
"日期,主体ID,主体类型,主体名称,花费,直接成交金额,间接成交金额,总成交金额,投入产出比,展现量\n"
|
||||
"2026-08-17,1001,商品,极星单肩包,90.25,730.02,0,730.02,8.09,1000\n"
|
||||
"2026-08-17,1001,商品,极星单肩包,9.75,0,70,70,7.18,200\n"
|
||||
"2026-08-17,plan-1,计划,测试计划,100,1000,0,1000,10,500\n",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
|
||||
records = importer.parse_tmall_product_ad_csv(report)
|
||||
|
||||
assert len(records) == 1
|
||||
metrics = records[0]["ad_metrics"]
|
||||
assert records[0]["product_id"] == "1001"
|
||||
assert metrics["spend"] == "100.00"
|
||||
assert metrics["attributed_deal_amount"] == "800.02"
|
||||
assert metrics["roi"] == "8.00"
|
||||
assert metrics["impressions"] == 1200
|
||||
assert records[0]["duplicate_row_count"] == 2
|
||||
|
||||
|
||||
def test_parse_gb18030_export(tmp_path: Path) -> None:
|
||||
report = tmp_path / "商品报表.csv"
|
||||
report.write_bytes(
|
||||
(
|
||||
"日期,主体ID,主体类型,主体名称,花费,总成交金额,投入产出比\n"
|
||||
"2026-08-17,1001,商品,极星单肩包,90.25,730.02,8.09\n"
|
||||
).encode("gb18030")
|
||||
)
|
||||
|
||||
records = importer.parse_tmall_product_ad_csv(report)
|
||||
|
||||
assert records[0]["product_name"] == "极星单肩包"
|
||||
assert records[0]["ad_metrics"]["roi"] == "8.09"
|
||||
|
||||
|
||||
def test_parse_export_null_markers_as_missing(tmp_path: Path) -> None:
|
||||
report = tmp_path / "商品报表.csv"
|
||||
report.write_text(
|
||||
"日期,主体ID,主体类型,主体名称,花费,总成交金额,平均点击花费,投入产出比\n"
|
||||
"2026-08-17,1001,商品,极星单肩包,90.25,730.02,\\N,\\N\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
records = importer.parse_tmall_product_ad_csv(report)
|
||||
|
||||
assert records[0]["ad_metrics"] == {
|
||||
"spend": "90.25",
|
||||
"attributed_deal_amount": "730.02",
|
||||
"roi": "8.09",
|
||||
}
|
||||
|
||||
|
||||
def test_find_report_files_uses_current_manifest_csv(monkeypatch, tmp_path: Path) -> None:
|
||||
target_date = importer.date(2026, 8, 17)
|
||||
date_root = tmp_path / target_date.isoformat()
|
||||
extracted = date_root / "extracted"
|
||||
extracted.mkdir(parents=True)
|
||||
old_csv = extracted / "商品报表_旧.csv"
|
||||
current_csv = extracted / "商品报表_新.csv"
|
||||
old_csv.write_text("old", encoding="utf-8")
|
||||
current_csv.write_text("current", encoding="utf-8")
|
||||
(date_root / "manifest.json").write_text(
|
||||
json.dumps({"csv_files": ["extracted\\商品报表_新.csv"]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(importer, "REPORT_ROOT", tmp_path)
|
||||
|
||||
assert importer.find_report_files(target_date) == [current_csv.resolve()]
|
||||
|
||||
|
||||
def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows, *, template=None):
|
||||
captured["sql"] = sql
|
||||
captured["rows"] = rows
|
||||
captured["template"] = template
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
_Connection(),
|
||||
[
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "极星单肩包",
|
||||
"ad_metrics": {
|
||||
"spend": "90.25",
|
||||
"attributed_deal_amount": "730.02",
|
||||
"roi": "8.09",
|
||||
},
|
||||
"source_file": "商品报表.csv",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
sql = str(captured["sql"])
|
||||
assert "INSERT INTO product_daily_metrics" in sql
|
||||
assert "jsonb_set" in sql
|
||||
assert "tm_product_ad_daily_metrics" not in sql
|
||||
assert captured["template"] == "('tm', %s, %s, %s, %s::jsonb, %s, NOW())"
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["spend"] == "90.25"
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_id"
|
||||
assert row[1] == "1001"
|
||||
|
||||
|
||||
def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class _NameFallbackCursor(_Cursor):
|
||||
def fetchall(self):
|
||||
return [("极星", [])]
|
||||
|
||||
class _NameFallbackConnection:
|
||||
def cursor(self):
|
||||
return _NameFallbackCursor()
|
||||
|
||||
def fake_execute_values(_cursor, _sql, rows, *, template=None):
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
written = product_db.upsert_tmall_product_ad_raw_metrics(
|
||||
_NameFallbackConnection(),
|
||||
[
|
||||
{
|
||||
"stat_date": importer.date(2026, 8, 17),
|
||||
"product_id": "unmapped-id",
|
||||
"product_name": "GYXX/极星单肩包2026新款",
|
||||
"ad_metrics": {"spend": "12.00"},
|
||||
"source_file": "商品报表.csv",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
row = captured["rows"][0]
|
||||
payload = json.loads(row[3])
|
||||
assert payload["tm_ad_metrics"]["style_name"] == "极星"
|
||||
assert payload["tm_ad_metrics"]["style_match_method"] == "product_name"
|
||||
|
||||
|
||||
def test_import_date_filter_and_dry_run(tmp_path: Path, capsys) -> None:
|
||||
report = tmp_path / "商品报表.csv"
|
||||
report.write_text(
|
||||
"日期,主体ID,主体类型,主体名称,花费,总成交金额\n"
|
||||
"2026-08-16,1001,商品,旧日期,1,2\n"
|
||||
"2026-08-17,1002,商品,目标日期,3,9\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert importer.main(["--file", str(report), "--date", "2026-08-17"]) == 0
|
||||
assert "DRY-RUN" in capsys.readouterr().out
|
||||
@@ -7,7 +7,9 @@ from collect_jd_market_rank import (
|
||||
build_jd_feishu_batch_xml,
|
||||
build_jd_feishu_category_xml,
|
||||
build_jd_feishu_intro_xml,
|
||||
category_label_is_selected,
|
||||
category_selection_matches,
|
||||
category_ui_label_candidates,
|
||||
exact_category_label_pattern,
|
||||
group_products_by_category,
|
||||
high_resolution_jd_image_url,
|
||||
@@ -15,6 +17,7 @@ from collect_jd_market_rank import (
|
||||
parse_jd_market_rank_html,
|
||||
parse_money_range_upper,
|
||||
resize_feishu_table_xml,
|
||||
select_category_path,
|
||||
select_category_paths,
|
||||
)
|
||||
|
||||
@@ -80,14 +83,13 @@ class JdMarketRankTests(unittest.TestCase):
|
||||
set(specs),
|
||||
{"旅行箱", "运动包", "双肩包", "斜挎包", "胸包", "手机包", "电脑包"},
|
||||
)
|
||||
self.assertEqual(specs["旅行箱"]["paths"], (("功能箱包", "行李箱"),))
|
||||
self.assertEqual(specs["运动包"]["paths"], (("功能箱包", "休闲运动包"),))
|
||||
self.assertEqual(
|
||||
specs["双肩包"]["paths"],
|
||||
(("男包", "双肩包"), ("男包", "男士双肩包")),
|
||||
)
|
||||
self.assertEqual(specs["手机包"]["paths"], (("男包", "男士手机包"),))
|
||||
self.assertEqual(specs["电脑包"]["paths"], (("功能箱包", "电脑包"),))
|
||||
self.assertEqual(specs["旅行箱"]["path"], ("功能箱包", "行李箱"))
|
||||
self.assertEqual(specs["运动包"]["path"], ("功能箱包", "休闲运动包"))
|
||||
self.assertEqual(specs["双肩包"]["path"], ("男包", "男士双肩包"))
|
||||
self.assertEqual(specs["斜挎包"]["path"], ("男包", "男士单肩/斜挎包"))
|
||||
self.assertEqual(specs["胸包"]["path"], ("男包", "男士腰包/胸包"))
|
||||
self.assertEqual(specs["手机包"]["path"], ("男包", "男士手机包"))
|
||||
self.assertEqual(specs["电脑包"]["path"], ("功能箱包", "电脑包"))
|
||||
|
||||
def test_category_label_matching_is_exact_not_contains(self):
|
||||
pattern = exact_category_label_pattern("双肩包")
|
||||
@@ -96,6 +98,116 @@ class JdMarketRankTests(unittest.TestCase):
|
||||
self.assertIsNone(pattern.fullmatch("男士双肩包"))
|
||||
self.assertIsNone(pattern.fullmatch("双肩包配件"))
|
||||
|
||||
def test_live_ui_alias_keeps_the_source_business_category(self):
|
||||
self.assertEqual(
|
||||
category_ui_label_candidates("男士双肩包"),
|
||||
("男士双肩包", "双肩包"),
|
||||
)
|
||||
self.assertEqual(
|
||||
category_ui_label_candidates("男士手机包"),
|
||||
("男士手机包",),
|
||||
)
|
||||
|
||||
def test_category_parent_hover_uses_the_dropdown_option_not_its_label(self):
|
||||
page = MagicMock()
|
||||
trigger = MagicMock()
|
||||
parent_option = MagicMock()
|
||||
child_option = MagicMock()
|
||||
with (
|
||||
patch("collect_jd_market_rank._visible", return_value=trigger),
|
||||
patch(
|
||||
"collect_jd_market_rank._selected_category_labels",
|
||||
side_effect=[("功能箱包",), ("男包 > 双肩包",)],
|
||||
),
|
||||
patch("collect_jd_market_rank._first_product_key", return_value="old"),
|
||||
patch(
|
||||
"collect_jd_market_rank._exact_visible_category_option",
|
||||
return_value=parent_option,
|
||||
) as resolve_option,
|
||||
patch(
|
||||
"collect_jd_market_rank._wait_for_category_option",
|
||||
return_value=(child_option, "双肩包"),
|
||||
) as wait_for_child,
|
||||
patch(
|
||||
"collect_jd_market_rank._click_category_option"
|
||||
) as click_option,
|
||||
patch("collect_jd_market_rank._wait_for_table_change"),
|
||||
):
|
||||
select_category_path(page, ("男包", "男士双肩包"))
|
||||
|
||||
resolve_option.assert_called_once_with(page, "男包")
|
||||
wait_for_child.assert_called_once_with(
|
||||
page,
|
||||
("男士双肩包", "双肩包"),
|
||||
)
|
||||
parent_option.hover.assert_called_once_with(timeout=5_000)
|
||||
click_option.assert_called_once_with(child_option, timeout_ms=10_000)
|
||||
|
||||
def test_same_selected_parent_opens_directly_on_child_options(self):
|
||||
page = MagicMock()
|
||||
trigger = MagicMock()
|
||||
child_option = MagicMock()
|
||||
with (
|
||||
patch("collect_jd_market_rank._visible", return_value=trigger),
|
||||
patch(
|
||||
"collect_jd_market_rank._selected_category_labels",
|
||||
side_effect=[
|
||||
("功能箱包",),
|
||||
("功能箱包 > 休闲运动包",),
|
||||
],
|
||||
),
|
||||
patch("collect_jd_market_rank._first_product_key", return_value="old"),
|
||||
patch(
|
||||
"collect_jd_market_rank._exact_visible_category_option",
|
||||
return_value=None,
|
||||
) as resolve_option,
|
||||
patch(
|
||||
"collect_jd_market_rank._wait_for_category_option",
|
||||
return_value=(child_option, "休闲运动包"),
|
||||
),
|
||||
patch(
|
||||
"collect_jd_market_rank._click_category_option"
|
||||
) as click_option,
|
||||
patch("collect_jd_market_rank._wait_for_table_change"),
|
||||
):
|
||||
select_category_path(page, ("功能箱包", "休闲运动包"))
|
||||
|
||||
resolve_option.assert_called_once_with(page, "功能箱包")
|
||||
click_option.assert_called_once_with(child_option, timeout_ms=10_000)
|
||||
|
||||
def test_cross_parent_switch_clicks_parent_when_hover_keeps_stale_children(self):
|
||||
page = MagicMock()
|
||||
trigger = MagicMock()
|
||||
parent_option = MagicMock()
|
||||
child_option = MagicMock()
|
||||
with (
|
||||
patch("collect_jd_market_rank._visible", return_value=trigger),
|
||||
patch(
|
||||
"collect_jd_market_rank._selected_category_labels",
|
||||
side_effect=[("功能箱包",), ("男包 > 双肩包",)],
|
||||
),
|
||||
patch("collect_jd_market_rank._first_product_key", return_value="old"),
|
||||
patch(
|
||||
"collect_jd_market_rank._exact_visible_category_option",
|
||||
return_value=parent_option,
|
||||
),
|
||||
patch(
|
||||
"collect_jd_market_rank._wait_for_category_option",
|
||||
side_effect=[(None, None), (child_option, "双肩包")],
|
||||
) as wait_for_child,
|
||||
patch(
|
||||
"collect_jd_market_rank._click_category_option"
|
||||
) as click_option,
|
||||
patch("collect_jd_market_rank._wait_for_table_change"),
|
||||
):
|
||||
select_category_path(page, ("男包", "男士双肩包"))
|
||||
|
||||
self.assertEqual(wait_for_child.call_count, 2)
|
||||
self.assertEqual(
|
||||
[item.args[0] for item in click_option.call_args_list],
|
||||
[parent_option, child_option],
|
||||
)
|
||||
|
||||
def test_selected_category_verification_accepts_only_exact_display_forms(self):
|
||||
path = ("男包", "双肩包")
|
||||
|
||||
@@ -104,6 +216,10 @@ class JdMarketRankTests(unittest.TestCase):
|
||||
self.assertFalse(category_selection_matches(("男士双肩包",), path))
|
||||
self.assertFalse(category_selection_matches(("双肩包配件",), path))
|
||||
|
||||
def test_selected_parent_matching_is_exact(self):
|
||||
self.assertTrue(category_label_is_selected((" 功能 箱包 ",), "功能箱包"))
|
||||
self.assertFalse(category_label_is_selected(("功能箱包配件",), "功能箱包"))
|
||||
|
||||
def test_category_candidates_fall_back_only_after_explicit_failure(self):
|
||||
page = MagicMock()
|
||||
paths = (("男包", "双肩包"), ("男包", "男士双肩包"))
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.product_commerce import jd_video_frames
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("duration", "max_frames", "expected"),
|
||||
[
|
||||
(2.0, 3, (1.0,)),
|
||||
(3.0, 3, (1.0, 2.0)),
|
||||
(7.5, 3, (2.5, 5.0)),
|
||||
(8.0, 3, (1.6, 4.0, 6.4)),
|
||||
(8.0, 2, (4.0, 6.4)),
|
||||
(48.88, 2, (24.44, 39.104)),
|
||||
(48.88, 1, (24.44,)),
|
||||
],
|
||||
)
|
||||
def test_choose_reference_frame_times(
|
||||
duration: float,
|
||||
max_frames: int,
|
||||
expected: tuple[float, ...],
|
||||
) -> None:
|
||||
assert jd_video_frames.choose_reference_frame_times(
|
||||
duration,
|
||||
max_frames=max_frames,
|
||||
) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("duration", [0.0, -1.0, math.nan, math.inf])
|
||||
def test_choose_reference_frame_times_rejects_invalid_duration(
|
||||
duration: float,
|
||||
) -> None:
|
||||
with pytest.raises(
|
||||
jd_video_frames.VideoFrameExtractionError,
|
||||
match="视频时长无效",
|
||||
):
|
||||
jd_video_frames.choose_reference_frame_times(duration)
|
||||
|
||||
|
||||
class _FakeReader:
|
||||
def __init__(self, values: list[object], closed: list[bool]) -> None:
|
||||
self._values = iter(values)
|
||||
self._closed = closed
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
return next(self._values)
|
||||
|
||||
def close(self) -> None:
|
||||
self._closed.append(True)
|
||||
|
||||
|
||||
def test_extract_reference_frames_writes_decodable_jpegs(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
source = tmp_path / "source.mp4"
|
||||
source.write_bytes(b"video-placeholder")
|
||||
output_root = tmp_path / "allowed"
|
||||
output_dir = output_root / "probe"
|
||||
closed: list[bool] = []
|
||||
frame = bytes(range(12))
|
||||
|
||||
def read_frames(_path, *, pix_fmt, input_params, output_params):
|
||||
assert pix_fmt == "rgb24"
|
||||
assert output_params == ["-frames:v", "1"]
|
||||
metadata = {"duration": 12.0, "size": (2, 2)}
|
||||
values = [metadata] if not input_params else [metadata, frame]
|
||||
return _FakeReader(values, closed)
|
||||
|
||||
monkeypatch.setattr(jd_video_frames.imageio_ffmpeg, "read_frames", read_frames)
|
||||
|
||||
result = jd_video_frames.extract_reference_frames(
|
||||
source,
|
||||
output_dir=output_dir,
|
||||
allowed_output_root=output_root,
|
||||
timestamps_seconds=(3.0, 6.0),
|
||||
max_frames=2,
|
||||
max_edge=256,
|
||||
)
|
||||
|
||||
assert [item.timestamp_seconds for item in result] == [3.0, 6.0]
|
||||
assert len(closed) == 3
|
||||
assert all(item.path.is_file() for item in result)
|
||||
assert all(len(item.sha256) == 64 for item in result)
|
||||
with Image.open(result[0].path) as image:
|
||||
assert image.mode == "RGB"
|
||||
assert image.size == (2, 2)
|
||||
|
||||
|
||||
def test_extract_reference_frames_cleans_partial_outputs_on_failure(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
source = tmp_path / "source.mp4"
|
||||
source.write_bytes(b"video-placeholder")
|
||||
output_root = tmp_path / "allowed"
|
||||
output_dir = output_root / "probe"
|
||||
calls = 0
|
||||
|
||||
def read_frames(_path, *, pix_fmt, input_params, output_params):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
metadata = {"duration": 12.0, "size": (2, 2)}
|
||||
if not input_params:
|
||||
return _FakeReader([metadata], [])
|
||||
if calls == 2:
|
||||
return _FakeReader([metadata, bytes(range(12))], [])
|
||||
return _FakeReader([metadata, b"truncated"], [])
|
||||
|
||||
monkeypatch.setattr(jd_video_frames.imageio_ffmpeg, "read_frames", read_frames)
|
||||
|
||||
with pytest.raises(
|
||||
jd_video_frames.VideoFrameExtractionError,
|
||||
match="RGB 帧长度无效",
|
||||
):
|
||||
jd_video_frames.extract_reference_frames(
|
||||
source,
|
||||
output_dir=output_dir,
|
||||
allowed_output_root=output_root,
|
||||
timestamps_seconds=(3.0, 6.0),
|
||||
max_frames=2,
|
||||
max_edge=256,
|
||||
)
|
||||
|
||||
assert list(output_dir.glob("*")) == []
|
||||
|
||||
|
||||
def test_extract_reference_frames_rejects_output_outside_allowed_root(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
source = tmp_path / "source.mp4"
|
||||
source.write_bytes(b"video-placeholder")
|
||||
|
||||
with pytest.raises(
|
||||
jd_video_frames.VideoFrameExtractionError,
|
||||
match="越出允许的临时目录",
|
||||
):
|
||||
jd_video_frames.extract_reference_frames(
|
||||
source,
|
||||
output_dir=tmp_path / "outside",
|
||||
allowed_output_root=tmp_path / "allowed",
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import date, datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -50,53 +50,6 @@ def test_build_notification_message_supports_partial_platform_success():
|
||||
assert "当前成功平台 1 个" in message
|
||||
|
||||
|
||||
def test_build_hermes_payload_requires_direct_message_to_recipient():
|
||||
payload = notification.build_hermes_payload(
|
||||
recipient_open_id="ou_wangyunlong",
|
||||
message="三个链接",
|
||||
)
|
||||
|
||||
encoded = json.dumps(payload, ensure_ascii=False)
|
||||
assert "ou_wangyunlong" in encoded
|
||||
assert "飞书私聊" in encoded
|
||||
assert "三个链接" in encoded
|
||||
|
||||
|
||||
def test_extracts_real_feishu_message_id_from_hermes_completion():
|
||||
response = {
|
||||
"id": "api-response-id",
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "发送成功,消息ID om_x100b696f1169f0a0b1f9c311e8dc323"
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
assert (
|
||||
notification.extract_feishu_message_id(response)
|
||||
== "om_x100b696f1169f0a0b1f9c311e8dc323"
|
||||
)
|
||||
assert notification.hermes_send_succeeded(response) is True
|
||||
|
||||
|
||||
def test_hermes_success_text_without_real_message_id_is_not_success():
|
||||
response = {
|
||||
"choices": [{"message": {"content": "发送成功,但没有返回消息回执"}}]
|
||||
}
|
||||
|
||||
assert notification.extract_feishu_message_id(response) is None
|
||||
assert notification.hermes_send_succeeded(response) is False
|
||||
|
||||
|
||||
def test_extracts_nested_feishu_message_id_and_accepts_safe_suffixes():
|
||||
response = {"data": {"message_id": "om_real-id_123"}}
|
||||
|
||||
assert notification.extract_feishu_message_id(response) == "om_real-id_123"
|
||||
assert notification.hermes_send_succeeded(response) is True
|
||||
|
||||
|
||||
def test_notify_skips_when_same_day_recipient_was_already_sent():
|
||||
conn = MagicMock()
|
||||
with (
|
||||
@@ -104,7 +57,7 @@ def test_notify_skips_when_same_day_recipient_was_already_sent():
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "get_market_rank_reports_for_date", return_value=REPORTS),
|
||||
patch.object(notification, "market_rank_notification_was_sent", return_value=True),
|
||||
patch.object(notification, "post_hermes") as post_hermes,
|
||||
patch.object(notification, "send_lark_bot_message") as send_message,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
result = notification.notify_market_rank_reports(
|
||||
@@ -114,21 +67,12 @@ def test_notify_skips_when_same_day_recipient_was_already_sent():
|
||||
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "already_sent"
|
||||
post_hermes.assert_not_called()
|
||||
send_message.assert_not_called()
|
||||
|
||||
|
||||
def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
|
||||
def test_notify_uses_lark_user_sender_and_persists_success():
|
||||
conn = MagicMock()
|
||||
response = {
|
||||
"id": "hermes-response-id",
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "发送成功,消息ID om_x100b696f1169f0a0b1f9c311e8dc323"
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
response = {"message_id": "om_x100b696f1169f0a0b1f9c311e8dc323"}
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
@@ -138,7 +82,11 @@ def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
|
||||
"get_market_rank_reports_for_date",
|
||||
return_value=REPORTS,
|
||||
) as get_reports,
|
||||
patch.object(notification, "post_hermes", return_value=response) as post_hermes,
|
||||
patch.object(
|
||||
notification,
|
||||
"send_lark_bot_message",
|
||||
return_value=response,
|
||||
) as send_message,
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
@@ -156,9 +104,16 @@ def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
|
||||
notification.PLATFORM_ORDER,
|
||||
updated_after=cutoff,
|
||||
)
|
||||
post_hermes.assert_called_once()
|
||||
send_message.assert_called_once()
|
||||
sent = send_message.call_args.kwargs
|
||||
assert sent["user_id"] == "ou_wangyunlong"
|
||||
assert sent["profile"] == "hermes-analyzer"
|
||||
assert "天猫:https://example.com/tm" in sent["text"]
|
||||
uuid.UUID(sent["idempotency_key"])
|
||||
saved = upsert.call_args.args[1]
|
||||
assert saved["status"] == "sent"
|
||||
assert saved["hermes_message_id"] == response["message_id"]
|
||||
assert result["message_id"] == response["message_id"]
|
||||
assert saved["platform_links"] == {
|
||||
"tm": "https://example.com/tm",
|
||||
"jd": "https://example.com/jd",
|
||||
@@ -168,15 +123,17 @@ def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
|
||||
|
||||
def test_notify_sends_available_links_when_other_platforms_are_missing():
|
||||
conn = MagicMock()
|
||||
response = {
|
||||
"choices": [{"message": {"content": "发送成功,消息ID om_partial123"}}]
|
||||
}
|
||||
response = {"message_id": "om_partial123"}
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "market_rank_notification_was_sent", return_value=False),
|
||||
patch.object(notification, "get_market_rank_reports_for_date", return_value=REPORTS[:2]),
|
||||
patch.object(notification, "post_hermes", return_value=response) as post_hermes,
|
||||
patch.object(
|
||||
notification,
|
||||
"send_lark_bot_message",
|
||||
return_value=response,
|
||||
) as send_message,
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
@@ -187,10 +144,41 @@ def test_notify_sends_available_links_when_other_platforms_are_missing():
|
||||
|
||||
assert result["status"] == "sent"
|
||||
assert result["missing_platforms"] == ["dy"]
|
||||
post_hermes.assert_called_once()
|
||||
send_message.assert_called_once()
|
||||
assert upsert.call_args.args[1]["status"] == "sent"
|
||||
|
||||
|
||||
def test_notify_archives_lark_user_send_failure() -> None:
|
||||
conn = MagicMock()
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "market_rank_notification_was_sent", return_value=False),
|
||||
patch.object(
|
||||
notification,
|
||||
"get_market_rank_reports_for_date",
|
||||
return_value=REPORTS,
|
||||
),
|
||||
patch.object(
|
||||
notification,
|
||||
"send_lark_bot_message",
|
||||
side_effect=RuntimeError("lark unavailable"),
|
||||
),
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
result = notification.notify_market_rank_reports(
|
||||
report_date=date(2026, 7, 26),
|
||||
recipient_open_id="ou_wangyunlong",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert "lark unavailable" in result["error"]
|
||||
saved = upsert.call_args.args[1]
|
||||
assert saved["status"] == "failed"
|
||||
assert saved["hermes_message_id"] is None
|
||||
|
||||
|
||||
def test_notify_skips_only_when_current_run_has_no_links():
|
||||
conn = MagicMock()
|
||||
with (
|
||||
@@ -198,7 +186,7 @@ def test_notify_skips_only_when_current_run_has_no_links():
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "get_market_rank_reports_for_date", return_value=[]),
|
||||
patch.object(notification, "market_rank_notification_was_sent") as was_sent,
|
||||
patch.object(notification, "post_hermes") as post_hermes,
|
||||
patch.object(notification, "send_lark_bot_message") as send_message,
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
@@ -210,5 +198,5 @@ def test_notify_skips_only_when_current_run_has_no_links():
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "no_platform_reports"
|
||||
was_sent.assert_not_called()
|
||||
post_hermes.assert_not_called()
|
||||
send_message.assert_not_called()
|
||||
assert upsert.call_args.args[1]["status"] == "incomplete"
|
||||
|
||||
@@ -5,6 +5,7 @@ import sys
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
from threading import Barrier
|
||||
from types import SimpleNamespace
|
||||
|
||||
import orchestrate_market_rank_collection as workflow
|
||||
import pytest
|
||||
@@ -219,6 +220,50 @@ def test_workflow_notifies_after_all_parallel_steps_finish(tmp_path):
|
||||
assert saved["notification"]["status"] == "sent"
|
||||
|
||||
|
||||
def test_workflow_notifies_each_dynamic_recipient_and_aggregates_results(tmp_path):
|
||||
notifications: list[str] = []
|
||||
|
||||
def successful_executor(*, platform, command, log_path, dry_run):
|
||||
return {
|
||||
"platform": platform,
|
||||
"label": workflow.PLATFORM_LABELS[platform],
|
||||
"command": command,
|
||||
"log_path": str(log_path),
|
||||
"started_at": "2026-08-07T10:00:00",
|
||||
"finished_at": "2026-08-07T10:01:00",
|
||||
"duration_seconds": 60.0,
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
def fake_notifier(**kwargs):
|
||||
recipient = kwargs["recipient_open_id"]
|
||||
notifications.append(recipient)
|
||||
return {"status": "sent", "recipient_open_id": recipient}
|
||||
|
||||
summary = workflow.execute_workflow(
|
||||
workflow.DEFAULT_PLATFORMS,
|
||||
run_id="multi-notify-run",
|
||||
log_dir=tmp_path,
|
||||
executor=successful_executor,
|
||||
report_date=date.today(),
|
||||
notify_open_ids=("ou_first", "ou_second"),
|
||||
notifier=fake_notifier,
|
||||
)
|
||||
|
||||
assert notifications == ["ou_first", "ou_second"]
|
||||
assert summary["notification"] == {
|
||||
"status": "sent",
|
||||
"recipient_count": 2,
|
||||
"deliveries": [
|
||||
{"status": "sent", "recipient_open_id": "ou_first"},
|
||||
{"status": "sent", "recipient_open_id": "ou_second"},
|
||||
],
|
||||
}
|
||||
assert summary["notifications"] == summary["notification"]["deliveries"]
|
||||
assert summary["exit_code"] == 0
|
||||
|
||||
|
||||
def test_partial_current_run_links_can_still_send_successfully(tmp_path):
|
||||
def successful_executor(*, platform, command, log_path, dry_run):
|
||||
return {
|
||||
@@ -249,6 +294,36 @@ def test_partial_current_run_links_can_still_send_successfully(tmp_path):
|
||||
assert summary["status"] == "success"
|
||||
|
||||
|
||||
def test_no_current_run_links_is_a_valid_no_notification_outcome(tmp_path):
|
||||
def successful_executor(*, platform, command, log_path, dry_run):
|
||||
return {
|
||||
"platform": platform,
|
||||
"command": command,
|
||||
"log_path": str(log_path),
|
||||
"started_at": "2026-08-07T10:00:00",
|
||||
"finished_at": "2026-08-07T10:01:00",
|
||||
"duration_seconds": 60.0,
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
summary = workflow.execute_workflow(
|
||||
workflow.DEFAULT_PLATFORMS,
|
||||
run_id="no-current-links-run",
|
||||
log_dir=tmp_path,
|
||||
executor=successful_executor,
|
||||
notify_open_id="ou_owner",
|
||||
notifier=lambda **_kwargs: {
|
||||
"status": "skipped",
|
||||
"reason": "no_platform_reports",
|
||||
},
|
||||
)
|
||||
|
||||
assert summary["notification"]["reason"] == "no_platform_reports"
|
||||
assert summary["exit_code"] == 0
|
||||
assert summary["status"] == "success"
|
||||
|
||||
|
||||
def test_platform_parser_deduplicates_while_preserving_order():
|
||||
assert workflow.parse_platforms("dy,tm,dy,jd") == ("dy", "tm", "jd")
|
||||
|
||||
@@ -289,6 +364,41 @@ def test_main_allows_explicit_no_notify_without_recipient(monkeypatch, tmp_path)
|
||||
assert observed["notify_open_id"] is None
|
||||
|
||||
|
||||
def test_main_passes_all_dynamic_route_recipients(monkeypatch, tmp_path):
|
||||
observed = {}
|
||||
|
||||
def fake_execute(*args, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return {"exit_code": 0}
|
||||
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
monkeypatch.setattr(workflow, "DEFAULT_NOTIFY_OPEN_ID", "")
|
||||
monkeypatch.setattr(
|
||||
workflow,
|
||||
"resolve_notification_route",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
enabled=True,
|
||||
open_ids=("ou_first", "ou_second"),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(workflow, "execute_workflow", fake_execute)
|
||||
monkeypatch.setattr(workflow, "managed_data_path", Path)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"orchestrate_market_rank_collection.py",
|
||||
"--dry-run",
|
||||
"--log-dir",
|
||||
str(tmp_path),
|
||||
],
|
||||
)
|
||||
|
||||
assert workflow.main() == 0
|
||||
assert observed["notify_open_ids"] == ("ou_first", "ou_second")
|
||||
assert observed["notify_open_id"] is None
|
||||
|
||||
|
||||
def test_weekly_launcher_uses_market_rank_orchestrator():
|
||||
launcher = (
|
||||
HISTORY_ROOT / "launchers_reference" / "run_weekly_market_rank.bat"
|
||||
|
||||
@@ -19,6 +19,26 @@ WORKERS = (tm_worker, dy_worker, jd_worker)
|
||||
BUSINESS_DATE = date(2026, 8, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_missing_time_select_option_is_treated_as_no_existing_record(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: SimpleNamespace(
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr=json.dumps(
|
||||
{
|
||||
"ok": False,
|
||||
"error": {"code": 800030005, "message": "not_found"},
|
||||
}
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
assert worker.find_record_id_by_time("bas_test", "tbl_test", "8.13") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_platform_collector_failure_forces_nonzero(worker) -> None:
|
||||
assert worker.combined_result_exit_code(
|
||||
@@ -78,6 +98,30 @@ def test_formal_base_upsert_accepts_verified_response_record_id(monkeypatch, wor
|
||||
assert message == "record_id=rec12345678"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_formal_base_update_reuses_valid_requested_record_id(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"ok": True, "data": {"updated": True}}),
|
||||
stderr="",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message = worker.upsert_record(
|
||||
"bas_test",
|
||||
"tbl_test",
|
||||
{"时间": "8.4", "男性比例": 0.5},
|
||||
record_id="rec_existing123",
|
||||
)
|
||||
|
||||
assert ok is True
|
||||
assert message == "record_id=rec_existing123"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_acceptance_skip_is_not_formal_success(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: True)
|
||||
@@ -265,6 +309,21 @@ def test_zero_distributions_are_not_valid_persona() -> None:
|
||||
) is False
|
||||
|
||||
|
||||
def test_dy_persistent_profile_is_not_overwritten_by_exported_cookies() -> None:
|
||||
assert dy_collector.persistent_profile_launch_options("bound-profile") == {
|
||||
"user_data_dir": "bound-profile"
|
||||
}
|
||||
|
||||
|
||||
def test_dy_profileless_fallback_can_restore_exported_cookies() -> None:
|
||||
cookies = [{"name": "session", "value": "new", "domain": ".jinritemai.com"}]
|
||||
|
||||
assert dy_collector.cookie_fallback_launch_options(cookies) == {
|
||||
"cookies": cookies
|
||||
}
|
||||
assert dy_collector.cookie_fallback_launch_options(None) == {}
|
||||
|
||||
|
||||
def test_dmp_skips_are_not_valid_style_results() -> None:
|
||||
styles = {"款A": ["123"]}
|
||||
assert dmp.records_exit_code(
|
||||
@@ -317,3 +376,24 @@ def test_tm_collector_command_redacts_credentials() -> None:
|
||||
assert "secret-user" not in rendered
|
||||
assert "secret-pass" not in rendered
|
||||
assert rendered.count("***") == 2
|
||||
|
||||
|
||||
def test_jd_persona_login_home_navigation_does_not_wait_for_networkidle() -> None:
|
||||
project_root = Path(__file__).resolve().parents[3]
|
||||
source_path = (
|
||||
project_root
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "vendors"
|
||||
/ "jd-data-flow"
|
||||
/ "jd_data_collector.py"
|
||||
)
|
||||
source = source_path.read_text(encoding="utf-8")
|
||||
|
||||
step_login = source[source.index("def step_login") :]
|
||||
home_block = step_login[step_login.index("shop.jd.com/jdm/home") :]
|
||||
assert 'wait_until="networkidle"' not in home_block[:400]
|
||||
assert 'wait_until="domcontentloaded"' in home_block[:400]
|
||||
assert "timeout=60_000" in home_block[:400]
|
||||
|
||||
@@ -15,6 +15,46 @@ class PersonaLauncherTests(unittest.TestCase):
|
||||
def test_all_platforms_success_makes_launcher_success(self):
|
||||
self.assertEqual(run_daily_persona.combined_exit_code({"tm": 0, "dy": 0, "jd": 0}), 0)
|
||||
|
||||
def test_partial_platform_results_are_accepted_when_real_rows_were_written(self):
|
||||
results = {"tm": 2, "dy": 2, "jd": 2}
|
||||
summaries = {
|
||||
"tm": {"ok": 31, "skipped": 0, "failed": 4, "total": 35},
|
||||
"dy": {"ok": 2, "skipped": 0, "failed": 30, "total": 31},
|
||||
"jd": {"ok": 18, "skipped": 0, "failed": 13, "total": 31},
|
||||
}
|
||||
|
||||
self.assertEqual(run_daily_persona.combined_exit_code(results, summaries), 0)
|
||||
|
||||
def test_partial_platform_without_real_rows_still_fails(self):
|
||||
results = {"tm": 2, "dy": 2, "jd": 2}
|
||||
summaries = {
|
||||
"tm": {"ok": 31},
|
||||
"dy": {"ok": 0},
|
||||
"jd": {"ok": 18},
|
||||
}
|
||||
|
||||
self.assertEqual(run_daily_persona.combined_exit_code(results, summaries), 2)
|
||||
|
||||
def test_result_summary_is_scoped_to_current_log_segment(self):
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
with TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "persona.log"
|
||||
path.write_text(
|
||||
"[DONE] ok=0 permission_skipped=0 failed=31 total=31\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
offset = path.stat().st_size
|
||||
with path.open("a", encoding="utf-8") as stream:
|
||||
stream.write(
|
||||
"[DONE] ok=18 permission_skipped=0 failed=13 total=31\n"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
run_daily_persona.read_result_summary(path, offset),
|
||||
{"ok": 18, "skipped": 0, "failed": 13, "total": 31},
|
||||
)
|
||||
|
||||
def test_launcher_passes_business_date_to_each_worker(self):
|
||||
command = run_daily_persona.build_child_command(
|
||||
Path("collect_persona_to_bitable.py"),
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -11,7 +10,7 @@ import lark_cli_runtime
|
||||
import market_rank_hermes_notification as market_notification
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.adapters import ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
|
||||
def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path:
|
||||
@@ -103,39 +102,38 @@ def test_product_shared_lark_cli_allows_safe_profile_override(monkeypatch) -> No
|
||||
]
|
||||
|
||||
|
||||
def test_decline_hermes_prompt_forces_wang_yunlong(
|
||||
def test_decline_lark_user_send_forces_acceptance_recipient(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(decline, "HERMES_API_KEY", "test-only")
|
||||
captured = {}
|
||||
|
||||
class Response:
|
||||
def __enter__(self):
|
||||
return self
|
||||
def fake_send(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return json.dumps({"choices": [{"message": {"content": "ok"}}]}).encode()
|
||||
|
||||
def fake_urlopen(request, timeout):
|
||||
captured["body"] = json.load(io.BytesIO(request.data))
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr(decline.urllib.request, "urlopen", fake_urlopen)
|
||||
decline.notify_hermes(
|
||||
[{"style_name": "test", "platform": "tm"}],
|
||||
monkeypatch.setattr(decline, "send_lark_bot_message", fake_send)
|
||||
decline.send_decline_notification(
|
||||
[
|
||||
{
|
||||
"style_name": "test",
|
||||
"platform": "tm",
|
||||
"seg1_sales": 30,
|
||||
"seg2_sales": 20,
|
||||
"seg3_sales": 10,
|
||||
"drop_pct_1to2": 33.3,
|
||||
"drop_pct_2to3": 50.0,
|
||||
"drop_pct_1to3": 66.7,
|
||||
}
|
||||
],
|
||||
"2026-08-01",
|
||||
9,
|
||||
openid="ou_someone_else",
|
||||
)
|
||||
|
||||
encoded = json.dumps(captured["body"], ensure_ascii=False)
|
||||
assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded
|
||||
assert "ou_someone_else" not in encoded
|
||||
assert captured["user_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert captured["profile"] == "hermes-analyzer"
|
||||
|
||||
|
||||
def test_market_rank_notification_forces_wang_yunlong(
|
||||
@@ -175,14 +173,14 @@ def test_market_rank_notification_forces_wang_yunlong(
|
||||
lambda *args, **kwargs: False,
|
||||
)
|
||||
|
||||
def fake_post(payload):
|
||||
captured["payload"] = payload
|
||||
def fake_send(**kwargs):
|
||||
captured["send"] = kwargs
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
def fake_archive(conn, record):
|
||||
captured["record"] = record
|
||||
|
||||
monkeypatch.setattr(market_notification, "post_hermes", fake_post)
|
||||
monkeypatch.setattr(market_notification, "send_lark_bot_message", fake_send)
|
||||
monkeypatch.setattr(
|
||||
market_notification,
|
||||
"upsert_market_rank_notification",
|
||||
@@ -194,11 +192,13 @@ def test_market_rank_notification_forces_wang_yunlong(
|
||||
recipient_open_id="ou_someone_else",
|
||||
)
|
||||
|
||||
encoded = json.dumps(captured["payload"], ensure_ascii=False)
|
||||
assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded
|
||||
assert "ou_someone_else" not in encoded
|
||||
assert result["recipient_open_id"] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert (
|
||||
captured["send"]["user_id"]
|
||||
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
)
|
||||
assert captured["send"]["profile"] == "hermes-analyzer"
|
||||
assert result["recipient_open_id"] == ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert (
|
||||
captured["record"]["recipient_open_id"]
|
||||
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
== ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import db as product_db
|
||||
|
||||
|
||||
class _Cursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
class _Connection:
|
||||
def cursor(self):
|
||||
return _Cursor()
|
||||
|
||||
|
||||
def test_product_daily_upsert_keeps_existing_ad_metrics(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_execute_values(_cursor, sql, rows):
|
||||
captured["sql"] = sql
|
||||
captured["rows"] = rows
|
||||
|
||||
monkeypatch.setattr(product_db, "execute_values", fake_execute_values)
|
||||
|
||||
written = product_db.upsert_product_daily_metrics(
|
||||
_Connection(),
|
||||
[{
|
||||
"platform": "tm",
|
||||
"stat_date": date(2026, 8, 17),
|
||||
"product_id": "1001",
|
||||
"product_name": "测试商品",
|
||||
"raw_data": {"paid_amount": "100.00"},
|
||||
"source_file": "商品经营日报.xls",
|
||||
}],
|
||||
)
|
||||
|
||||
assert written == 1
|
||||
assert (
|
||||
"COALESCE(product_daily_metrics.raw_data, '{}'::jsonb)\n"
|
||||
" || EXCLUDED.raw_data"
|
||||
) in str(captured["sql"])
|
||||
@@ -8,6 +8,7 @@ import backfill_collect
|
||||
import backfill_one_day
|
||||
import collect_dy_persona_to_bitable as dy_persona_worker
|
||||
import collect_persona_to_bitable as tm_persona_worker
|
||||
import dy_product_scraping as dy_product
|
||||
import orchestrate_daily_collection as daily
|
||||
import orchestrate_market_rank_collection as market_rank
|
||||
import orchestrate_review_collection as reviews
|
||||
@@ -115,6 +116,40 @@ def test_daily_run_step_rebinds_to_command_target(monkeypatch) -> None:
|
||||
_assert_child_binding(observed["environment"])
|
||||
|
||||
|
||||
def test_daily_command_logging_redacts_sensitive_values() -> None:
|
||||
command = [
|
||||
"python",
|
||||
"collector.py",
|
||||
"--account",
|
||||
"secret-user",
|
||||
"--password=placeholder",
|
||||
"--api-key",
|
||||
"example",
|
||||
"--target-date",
|
||||
"2026-08-04",
|
||||
]
|
||||
|
||||
rendered = daily.redacted_command(command)
|
||||
|
||||
assert "secret-user" not in rendered
|
||||
assert "placeholder" not in rendered
|
||||
assert "example" not in rendered
|
||||
assert rendered.count("[REDACTED]") == 3
|
||||
assert "2026-08-04" in rendered
|
||||
|
||||
|
||||
def test_daily_saved_cookie_binding_refreshes_existing_profile(tmp_path) -> None:
|
||||
profile = tmp_path / "profile"
|
||||
cookie_db = profile / "Default" / "Network" / "Cookies"
|
||||
cookie_db.parent.mkdir(parents=True)
|
||||
cookie_db.write_bytes(b"x" * 5000)
|
||||
cookies = [{"name": "session", "value": "secret", "domain": ".jinritemai.com"}]
|
||||
|
||||
assert dy_product._saved_cookie_launch_options(str(profile), cookies) == {
|
||||
"cookies": cookies
|
||||
}
|
||||
|
||||
|
||||
def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -13,12 +13,20 @@ def _workflow(workflow_id: str) -> dict:
|
||||
return next(item for item in catalog["workflows"] if item["id"] == workflow_id)
|
||||
|
||||
|
||||
def _schedule(workflow_id: str) -> dict:
|
||||
schedules = json.loads(
|
||||
(REPOSITORY_ROOT / "config" / "schedules.json").read_text(encoding="utf-8")
|
||||
)
|
||||
return next(item for item in schedules["schedules"] if item["workflow_id"] == workflow_id)
|
||||
|
||||
|
||||
def test_product_workflows_forward_the_selected_business_date() -> None:
|
||||
persona = _workflow("product.persona.daily")["execution"]["steps"][0]
|
||||
style = _workflow("product.style_analysis.interval")["execution"]["steps"][0]
|
||||
main_image = _workflow("product.main_image.weekly")["execution"]["steps"]
|
||||
market_rank = _workflow("product.market_rank")["execution"]["steps"][0]
|
||||
monthly_sales = _workflow("product.sales_sheet.daily")["execution"]["steps"][0]
|
||||
all_shop_daily = _workflow("product.erp_all_shop_daily")["execution"]["steps"][0]
|
||||
|
||||
assert persona["args"] == ["--date", "{business_date}"]
|
||||
assert style["args"][-2:] == ["--end-date", "{business_date}"]
|
||||
@@ -33,3 +41,46 @@ def test_product_workflows_forward_the_selected_business_date() -> None:
|
||||
"--execute",
|
||||
"--allow-missing-erp-as-zero",
|
||||
]
|
||||
assert all_shop_daily["args"] == [
|
||||
"--from",
|
||||
"{business_date}",
|
||||
"--to",
|
||||
"{business_date}",
|
||||
"--execute",
|
||||
"--force",
|
||||
]
|
||||
|
||||
|
||||
def test_ecommerce_costs_workflow_runs_three_platforms_in_parallel() -> None:
|
||||
workflow = _workflow("product.ecommerce_costs.daily")
|
||||
steps = workflow["execution"]["steps"]
|
||||
by_id = {step["id"]: step for step in steps}
|
||||
|
||||
assert [step["id"] for step in steps] == [
|
||||
"tmall_download",
|
||||
"tmall_import",
|
||||
"jd_download",
|
||||
"jd_import",
|
||||
"douyin_download",
|
||||
"douyin_import",
|
||||
]
|
||||
for platform, collector, importer in (
|
||||
("tmall", "collect_tmall_wanxiang_ads.py", "import_tmall_product_ads.py"),
|
||||
("jd", "collect_jd_ad_costs.py", "import_jd_ad_costs.py"),
|
||||
("douyin", "collect_douyin_qianchuan_ads.py", "import_douyin_qianchuan_ads.py"),
|
||||
):
|
||||
download = by_id[f"{platform}_download"]
|
||||
import_step = by_id[f"{platform}_import"]
|
||||
assert download["entry"] == collector
|
||||
assert download["args"] == ["--date", "{business_date}", "--headless"]
|
||||
assert download.get("depends_on", []) == []
|
||||
assert import_step["entry"] == importer
|
||||
assert import_step["args"] == ["--date", "{business_date}", "--execute"]
|
||||
assert import_step["depends_on"] == [f"{platform}_download"]
|
||||
|
||||
assert "点击下载报表" in by_id["tmall_download"]["description"]
|
||||
assert "当前时间" in by_id["jd_import"]["description"]
|
||||
assert "SKU" in by_id["jd_import"]["description"]
|
||||
schedule = _schedule("product.ecommerce_costs.daily")
|
||||
assert schedule["at"] == "10:00"
|
||||
assert schedule["business_date_offset_days"] == -1
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import date
|
||||
@@ -7,7 +8,7 @@ from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import analyze_style_with_hermes as analysis
|
||||
import analyze_style as analysis
|
||||
import orchestrate_daily_collection as orchestrator
|
||||
from config.style_config_loader import StyleConfigLoader
|
||||
|
||||
@@ -140,6 +141,60 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
for heading in headings:
|
||||
self.assertIn(heading, analysis.SYSTEM)
|
||||
|
||||
def test_system_prompt_and_validator_require_pure_chinese_report(self):
|
||||
self.assertIn("纯中文硬约束", analysis.SYSTEM)
|
||||
self.assertIn("不得出现英文单词", analysis.SYSTEM)
|
||||
|
||||
valid_report = "\n".join([
|
||||
*analysis.REPORT_HEADINGS,
|
||||
*analysis.REPORT_SUBHEADINGS,
|
||||
"**P0**", "**重点**", "**重点**", "**重点**", "**重点**",
|
||||
"**重点**", "**重点**", "**重点**",
|
||||
])
|
||||
normalized, error = analysis.validate_report_markdown(valid_report)
|
||||
self.assertIsNone(error)
|
||||
self.assertIn("**P0**", normalized)
|
||||
|
||||
english_report = valid_report.replace("**P0**", "English text")
|
||||
_, error = analysis.validate_report_markdown(english_report)
|
||||
self.assertEqual(error, "MiniMax 报告包含英文内容,必须改为纯中文")
|
||||
|
||||
def test_validator_removes_separate_reasoning_before_language_check(self):
|
||||
valid_report = "\n".join([
|
||||
"<think>English reasoning</think>",
|
||||
*analysis.REPORT_HEADINGS,
|
||||
*analysis.REPORT_SUBHEADINGS,
|
||||
"**P0**", "**重点**", "**重点**", "**重点**", "**重点**",
|
||||
"**重点**", "**重点**", "**重点**",
|
||||
])
|
||||
normalized, error = analysis.validate_report_markdown(valid_report)
|
||||
self.assertIsNone(error)
|
||||
self.assertNotIn("English reasoning", normalized)
|
||||
|
||||
def test_localization_fallback_converts_source_english_to_chinese(self):
|
||||
report = "英文前缀\nSwitchLite、SKU、IP、ROI、A/B、Apollo、notes_recent_top8、**P0**"
|
||||
localized = analysis._localize_report_latin(report)
|
||||
|
||||
latin_tokens = set(re.findall(r"[A-Za-z][A-Za-z0-9_-]*", localized))
|
||||
self.assertEqual(latin_tokens, {"P0"})
|
||||
self.assertIn("游戏主机", localized)
|
||||
self.assertIn("规格", localized)
|
||||
self.assertIn("地域", localized)
|
||||
self.assertIn("投入产出比", localized)
|
||||
self.assertIn("甲/乙", localized)
|
||||
self.assertIn("**P0**", localized)
|
||||
|
||||
def test_direct_checkpoint_namespace_isolated_for_language_gate_revision(self):
|
||||
self.assertEqual(analysis.CHECKPOINT_NAMESPACE, "direct_minimax_v2")
|
||||
self.assertIn("direct_minimax_v2", str(analysis.StyleAnalysisCheckpointSpec.for_report(
|
||||
"测试款",
|
||||
date(2026, 8, 1),
|
||||
date(2026, 8, 3),
|
||||
3,
|
||||
"测试款 单款式多维度分析 2026-08-01~2026-08-03",
|
||||
"# 内容",
|
||||
).path()))
|
||||
|
||||
def test_plain_seven_section_titles_are_normalized_to_markdown(self):
|
||||
report = """盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)
|
||||
|
||||
@@ -164,6 +219,99 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
self.assertTrue(normalized.startswith("# 盖亚斜挎"))
|
||||
self.assertTrue(analysis.has_exact_report_headings(normalized))
|
||||
|
||||
def test_number_variants_and_bare_total_title_are_canonicalized(self):
|
||||
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
|
||||
for index, heading in enumerate(analysis.REPORT_HEADINGS):
|
||||
subject = heading.split("、", 1)[1]
|
||||
raw_heading = heading
|
||||
if index == 4:
|
||||
raw_heading = f"## 六、{subject}"
|
||||
elif index == 6:
|
||||
raw_heading = "## 总评"
|
||||
blocks.extend([
|
||||
raw_heading,
|
||||
*analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3],
|
||||
"**重点**",
|
||||
])
|
||||
|
||||
normalized = analysis.normalize_report_markdown("\n".join(blocks))
|
||||
|
||||
self.assertEqual(
|
||||
[
|
||||
line.strip()
|
||||
for line in normalized.splitlines()
|
||||
if line.strip().startswith("## ")
|
||||
],
|
||||
analysis.REPORT_HEADINGS,
|
||||
)
|
||||
|
||||
def test_priority_action_heading_is_inserted_for_unheaded_priority_lines(self):
|
||||
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
|
||||
for index, heading in enumerate(analysis.REPORT_HEADINGS[:-1]):
|
||||
blocks.extend([
|
||||
heading,
|
||||
*analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3],
|
||||
"**重点**",
|
||||
])
|
||||
blocks.extend([
|
||||
analysis.REPORT_HEADINGS[-1],
|
||||
"### 核心结论",
|
||||
"**当前核心问题是转化偏低。**",
|
||||
"### 转化链路",
|
||||
"曝光到访客再到成交。",
|
||||
"- **P0**:48小时内完成整改。",
|
||||
])
|
||||
|
||||
normalized = analysis.normalize_report_markdown("\n".join(blocks))
|
||||
|
||||
self.assertTrue(analysis.has_clear_report_hierarchy(normalized))
|
||||
self.assertIn("### 优先级行动", normalized)
|
||||
|
||||
def test_minimax_adapter_disables_tools_and_uses_extended_timeout(self):
|
||||
captured = {}
|
||||
|
||||
def fake_call(system, user, **kwargs):
|
||||
captured.update(system=system, user=user, kwargs=kwargs)
|
||||
return "报告", {}
|
||||
|
||||
with (
|
||||
patch.object(analysis, "call_direct_llm", fake_call),
|
||||
):
|
||||
content, _ = analysis.call_minimax("system", "user")
|
||||
|
||||
self.assertEqual(content, "报告")
|
||||
self.assertEqual(captured["system"], "system")
|
||||
self.assertEqual(captured["user"], "user")
|
||||
self.assertEqual(captured["kwargs"]["model"], analysis.MINIMAX_MODEL)
|
||||
self.assertEqual(captured["kwargs"]["temperature"], 0.3)
|
||||
self.assertEqual(captured["kwargs"]["timeout"], analysis.MINIMAX_TIMEOUT_SECONDS)
|
||||
self.assertEqual(captured["kwargs"]["max_tokens"], analysis.MINIMAX_MAX_TOKENS)
|
||||
self.assertEqual(captured["kwargs"]["thinking_mode"], "disabled")
|
||||
|
||||
def test_invalid_report_is_regenerated_once_before_failure(self):
|
||||
valid_lines = []
|
||||
for index, heading in enumerate(analysis.REPORT_HEADINGS):
|
||||
valid_lines.extend([
|
||||
heading,
|
||||
*analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3],
|
||||
"**重点**",
|
||||
])
|
||||
valid_report = "\n".join(valid_lines)
|
||||
|
||||
with (
|
||||
patch.object(analysis, "MINIMAX_FORMAT_RETRIES", 1),
|
||||
patch.object(
|
||||
analysis,
|
||||
"call_minimax",
|
||||
side_effect=[("不完整报告", {"attempt": 1}), (valid_report, {"attempt": 2})],
|
||||
) as call,
|
||||
):
|
||||
report, usage = analysis.generate_validated_report("user prompt")
|
||||
|
||||
self.assertEqual(report, analysis.normalize_report_markdown(valid_report))
|
||||
self.assertEqual(usage, {"attempt": 2})
|
||||
self.assertEqual(call.call_count, 2)
|
||||
|
||||
def test_report_gets_clear_subheadings_and_key_emphasis(self):
|
||||
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
|
||||
for heading in analysis.REPORT_HEADINGS[:-1]:
|
||||
@@ -190,6 +338,32 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
self.assertIn("**P0**", normalized)
|
||||
self.assertIn("**24小时内**", normalized)
|
||||
|
||||
def test_report_normalization_is_idempotent_for_checkpoint_recovery(self):
|
||||
blocks = ["逐星GT 单款式多维度分析报告 (2026-08-01~2026-08-03)"]
|
||||
for heading in analysis.REPORT_HEADINGS[:-1]:
|
||||
blocks.extend([
|
||||
heading,
|
||||
"### 数据快照",
|
||||
"**三日访客 100**。",
|
||||
"### 核心判断",
|
||||
"**当前转化偏低。**",
|
||||
"### 优化建议",
|
||||
"P1,3天内完成优化。",
|
||||
])
|
||||
blocks.extend([
|
||||
analysis.REPORT_HEADINGS[-1],
|
||||
"### 核心结论",
|
||||
"**当前核心问题是转化偏低。**",
|
||||
"### 转化链路",
|
||||
"曝光到访客再到成交。",
|
||||
"### 优先级行动",
|
||||
"- **P0(48小时内)**:完成整改。",
|
||||
])
|
||||
|
||||
normalized = analysis.normalize_report_markdown("\n".join(blocks))
|
||||
|
||||
self.assertEqual(analysis.normalize_report_markdown(normalized), normalized)
|
||||
|
||||
def test_extra_helpful_h3_does_not_reject_complete_required_hierarchy(self):
|
||||
required = "\n".join(analysis.REPORT_SUBHEADINGS)
|
||||
report = required.replace(
|
||||
@@ -212,6 +386,78 @@ class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(missing, [date(2026, 7, 15)])
|
||||
|
||||
def test_batch_style_selection_requires_a_complete_calendar_window(self):
|
||||
class Cursor:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
self.sql = ""
|
||||
self.params = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.sql = sql
|
||||
self.params = params
|
||||
|
||||
def fetchall(self):
|
||||
return self.rows
|
||||
|
||||
class Connection:
|
||||
def __init__(self, cursor):
|
||||
self.cursor_value = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_value
|
||||
|
||||
cursor = Cursor([("完整款", 100)])
|
||||
with patch.object(analysis, "get_conn", return_value=Connection(cursor)):
|
||||
styles = analysis.fetch_styles_with_data(date(2026, 7, 15), 3)
|
||||
|
||||
self.assertEqual(styles, ["完整款"])
|
||||
self.assertIn("HAVING COUNT(DISTINCT metric_date) = %s", cursor.sql)
|
||||
self.assertEqual(
|
||||
cursor.params,
|
||||
(date(2026, 7, 13), date(2026, 7, 15), 3),
|
||||
)
|
||||
|
||||
def test_incomplete_batch_styles_are_reported_with_present_dates(self):
|
||||
cursor = type(
|
||||
"Cursor",
|
||||
(),
|
||||
{
|
||||
"__enter__": lambda self: self,
|
||||
"__exit__": lambda self, exc_type, exc, tb: False,
|
||||
"execute": lambda self, sql, params: setattr(self, "params", params),
|
||||
"fetchall": lambda self: [("云栖", [date(2026, 7, 15)])],
|
||||
},
|
||||
)()
|
||||
connection = type(
|
||||
"Connection",
|
||||
(),
|
||||
{
|
||||
"__enter__": lambda self: self,
|
||||
"__exit__": lambda self, exc_type, exc, tb: False,
|
||||
"cursor": lambda self: cursor,
|
||||
},
|
||||
)()
|
||||
|
||||
with patch.object(analysis, "get_conn", return_value=connection):
|
||||
incomplete = analysis.fetch_styles_with_incomplete_data(
|
||||
date(2026, 7, 15), 3,
|
||||
)
|
||||
|
||||
self.assertEqual(incomplete, {"云栖": [date(2026, 7, 15)]})
|
||||
|
||||
|
||||
class LarkEnvironmentTests(unittest.TestCase):
|
||||
def test_legacy_lark_config_is_used_when_default_is_missing(self):
|
||||
@@ -607,10 +853,6 @@ class TargetBaseWriteTests(unittest.TestCase):
|
||||
{"name": "时间", "type": "select"},
|
||||
{"name": "笔记分析汇总", "type": "text"},
|
||||
]}},
|
||||
{"data": {"field": {
|
||||
"id": "fld_time", "name": "时间", "type": "select",
|
||||
"multiple": False, "options": [{"name": "2026-07-13~07-15"}],
|
||||
}}},
|
||||
{"data": {
|
||||
"data": [[['2026-07-13~07-15'], "old link"]],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
@@ -627,33 +869,117 @@ class TargetBaseWriteTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(record_id, "rec_existing")
|
||||
upsert_args = run.call_args_list[3].args[0]
|
||||
upsert_args = run.call_args_list[2].args[0]
|
||||
self.assertIn("--record-id", upsert_args)
|
||||
self.assertIn("rec_existing", upsert_args)
|
||||
payload = json.loads(upsert_args[upsert_args.index("--json") + 1])
|
||||
self.assertIn("笔记分析汇总", payload)
|
||||
|
||||
def test_missing_period_is_appended_without_dropping_existing_options(self):
|
||||
def test_missing_period_is_created_without_rewriting_select_field(self):
|
||||
responses = [
|
||||
{"data": {"field": {
|
||||
"id": "fld_time", "name": "时间", "type": "select",
|
||||
"multiple": False, "options": [{"name": "7月第二周", "hue": "Blue"}],
|
||||
}}},
|
||||
{"ok": True},
|
||||
{"data": {"fields": [
|
||||
{"name": "时间", "type": "select"},
|
||||
{"name": "笔记分析汇总", "type": "text"},
|
||||
]}},
|
||||
{"data": {
|
||||
"data": [],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": [],
|
||||
}},
|
||||
{"ok": True, "data": {"record_id": "rec_created"}},
|
||||
]
|
||||
with patch.object(analysis, "run_lark_cli", side_effect=responses) as run:
|
||||
analysis.ensure_select_option(
|
||||
record_id = analysis.write_analysis_link_to_base(
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"时间", "2026-07-13~07-15", "盖亚斜挎",
|
||||
"2026-08-01~08-03",
|
||||
"https://example.feishu.cn/docx/doc",
|
||||
"盖亚斜挎",
|
||||
)
|
||||
|
||||
update_args = run.call_args_list[1].args[0]
|
||||
definition = json.loads(update_args[update_args.index("--json") + 1])
|
||||
self.assertEqual(record_id, "rec_created")
|
||||
self.assertEqual(
|
||||
[option["name"] for option in definition["options"]],
|
||||
["7月第二周", "2026-07-13~07-15"],
|
||||
[call.args[0][1] for call in run.call_args_list],
|
||||
["+field-list", "+record-list", "+record-upsert"],
|
||||
)
|
||||
self.assertIn("--yes", update_args)
|
||||
payload_args = run.call_args_list[2].args[0]
|
||||
payload = json.loads(payload_args[payload_args.index("--json") + 1])
|
||||
self.assertEqual(payload["时间"], "2026-08-01~08-03")
|
||||
|
||||
def test_timeout_reconciles_remote_upsert_without_second_write(self):
|
||||
document_url = "https://example.feishu.cn/docx/doc"
|
||||
responses = [
|
||||
{"data": {"fields": [
|
||||
{"name": "时间", "type": "select"},
|
||||
{"name": "笔记分析汇总", "type": "text"},
|
||||
]}},
|
||||
{"data": {
|
||||
"data": [],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": [],
|
||||
}},
|
||||
RuntimeError(
|
||||
'lark-cli failed (exit=4): {"error":{"type":"network",'
|
||||
'"subtype":"timeout"}}'
|
||||
),
|
||||
{"data": {
|
||||
"data": [[
|
||||
["2026-08-01~08-03"],
|
||||
f"[盖亚斜挎 三日单品分析]({document_url})",
|
||||
]],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": ["rec_reconciled"],
|
||||
}},
|
||||
]
|
||||
with (
|
||||
patch.object(analysis, "run_lark_cli", side_effect=responses) as run,
|
||||
patch.object(analysis, "BASE_WRITE_RECONCILE_DELAYS_SECONDS", (0,)),
|
||||
):
|
||||
record_id = analysis.write_analysis_link_to_base(
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"2026-08-01~08-03",
|
||||
document_url,
|
||||
"盖亚斜挎",
|
||||
)
|
||||
|
||||
self.assertEqual(record_id, "rec_reconciled")
|
||||
self.assertEqual(
|
||||
[call.args[0][1] for call in run.call_args_list],
|
||||
["+field-list", "+record-list", "+record-upsert", "+record-list"],
|
||||
)
|
||||
self.assertEqual(run.call_args_list[2].kwargs["timeout"], 180)
|
||||
|
||||
def test_timeout_reconciliation_requires_the_expected_report_link(self):
|
||||
responses = [
|
||||
{"data": {"fields": [
|
||||
{"name": "时间", "type": "select"},
|
||||
{"name": "笔记分析汇总", "type": "text"},
|
||||
]}},
|
||||
{"data": {
|
||||
"data": [],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": [],
|
||||
}},
|
||||
RuntimeError("API call failed: server time out error"),
|
||||
{"data": {
|
||||
"data": [[
|
||||
["2026-08-01~08-03"],
|
||||
"[盖亚斜挎 三日单品分析](https://example.feishu.cn/docx/old)",
|
||||
]],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": ["rec_old"],
|
||||
}},
|
||||
]
|
||||
with (
|
||||
patch.object(analysis, "run_lark_cli", side_effect=responses),
|
||||
patch.object(analysis, "BASE_WRITE_RECONCILE_DELAYS_SECONDS", (0,)),
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "server time out error"):
|
||||
analysis.write_analysis_link_to_base(
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"2026-08-01~08-03",
|
||||
"https://example.feishu.cn/docx/new",
|
||||
"盖亚斜挎",
|
||||
)
|
||||
|
||||
|
||||
class OrchestratorTests(unittest.TestCase):
|
||||
@@ -663,7 +989,7 @@ class OrchestratorTests(unittest.TestCase):
|
||||
orchestrator.run_style_analysis(args, "2026-07-15", failed=None)
|
||||
|
||||
command = run.call_args.args[1]
|
||||
self.assertIn("analyze_style_with_hermes.py", [Path(item).name for item in command])
|
||||
self.assertIn("analyze_style.py", [Path(item).name for item in command])
|
||||
self.assertIn("--all-styles", command)
|
||||
self.assertEqual(command[command.index("--days") + 1], "3")
|
||||
self.assertEqual(command[command.index("--min-interval-days") + 1], "3")
|
||||
|
||||
@@ -39,25 +39,32 @@ def test_non_tm_erp_code_is_not_used_when_tm_code_is_missing() -> None:
|
||||
assert "天猫分组未填写" in output.getvalue()
|
||||
|
||||
|
||||
def test_exact_style_field_wins_over_style_content_prefix(monkeypatch) -> None:
|
||||
loader = StyleConfigLoader()
|
||||
monkeypatch.setattr(
|
||||
loader,
|
||||
"_run_lark",
|
||||
lambda _args: {
|
||||
"data": {
|
||||
"fields": ["款式内容", "款式", "ERP款式编码", "平台"],
|
||||
"data": [["营销文案", "盖世m1", "10416,10455", ["天猫"]]],
|
||||
def test_database_row_uses_explicit_style_name_not_style_content() -> None:
|
||||
loader = StyleConfigLoader(
|
||||
records_provider=lambda: [
|
||||
{
|
||||
"style_content": "营销文案",
|
||||
"style_name": "盖世m1",
|
||||
"erp_codes": ["10416", "10455"],
|
||||
"platform": "天猫",
|
||||
}
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
records = loader._load_records_from_lark()
|
||||
records = loader._load_records_from_database()
|
||||
|
||||
assert records == [
|
||||
{
|
||||
"style": "盖世m1",
|
||||
"erp_codes": "10416,10455",
|
||||
"erp_codes": ["10416", "10455"],
|
||||
"platform": "天猫",
|
||||
"item_ids": [],
|
||||
"brand": "",
|
||||
"sales_bitable_url": "",
|
||||
"main_image_bitable_url": "",
|
||||
"persona_bitable_url": "",
|
||||
"dy_persona_bitable_url": "",
|
||||
"jd_persona_bitable_url": "",
|
||||
"style_analysis_bitable_url": "",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -14,6 +14,7 @@ from collect_sycm_market_rank import (
|
||||
collect_all_rank_pages,
|
||||
create_feishu_doc,
|
||||
detect_risk_control_text,
|
||||
dismiss_floating_overlays,
|
||||
ensure_market_rank_login,
|
||||
group_products_by_category,
|
||||
inspect_feishu_batch_tables,
|
||||
@@ -49,6 +50,15 @@ SAMPLE_HTML = """
|
||||
|
||||
|
||||
class SycmMarketRankTests(unittest.TestCase):
|
||||
def test_floating_overlays_are_removed_before_pagination(self):
|
||||
page = MagicMock()
|
||||
page.evaluate.return_value = 3
|
||||
|
||||
self.assertEqual(dismiss_floating_overlays(page), 3)
|
||||
script = page.evaluate.call_args.args[0]
|
||||
self.assertIn("scenario-widget", script)
|
||||
self.assertIn("aes-survey-hanging", script)
|
||||
|
||||
def test_expired_login_without_credentials_fails_clearly(self):
|
||||
with (
|
||||
patch("collect_sycm_market_rank.sycm.ACCOUNT", ""),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
@@ -17,6 +18,7 @@ from gyxx_flow.modules.product_commerce.sync_monthly_sales_sheet import (
|
||||
MonthlySalesConfig,
|
||||
ProductTarget,
|
||||
SheetInfo,
|
||||
_reconfigure_stream,
|
||||
_write_rows,
|
||||
build_parser,
|
||||
choose_template,
|
||||
@@ -43,6 +45,17 @@ def test_month_contract_uses_natural_month_and_existing_title_convention() -> No
|
||||
assert month_sheet_title(month) == "26年8月"
|
||||
|
||||
|
||||
def test_reconfigure_stream_switches_gbk_stream_to_utf8() -> None:
|
||||
raw = io.BytesIO()
|
||||
stream = io.TextIOWrapper(raw, encoding="gbk", errors="strict")
|
||||
|
||||
_reconfigure_stream(stream)
|
||||
stream.write("商品\u200c名\n")
|
||||
stream.flush()
|
||||
|
||||
assert "商品\u200c名".encode("utf-8") in raw.getvalue()
|
||||
|
||||
|
||||
def test_erp_month_interval_uses_inclusive_natural_month_for_day_input() -> None:
|
||||
assert monthly_report_interval(parse_month("2026-07-02")) == (
|
||||
"2026-07-01",
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import taobao_sycm_collect as collect
|
||||
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def stub_collection_chrome(monkeypatch):
|
||||
"""Keep wrapper tests from starting a real browser on collector ports."""
|
||||
|
||||
monkeypatch.setattr(collect, "is_chrome_debug_running", lambda _port: False)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"launch_chrome_with_debug",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
|
||||
|
||||
def test_tm_identity_markers_require_the_complete_store_name(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS", raising=False)
|
||||
|
||||
assert collect is not None
|
||||
# The child collector owns the actual page identity check; import it here
|
||||
# so this regression test also guards its brand fallback markers.
|
||||
import taobao_sycm_products as products
|
||||
|
||||
monkeypatch.setattr(products, "BRAND", "ozko")
|
||||
monkeypatch.setattr(products, "ACCOUNT", "")
|
||||
assert products._login_identity_markers() == ("ozko旗舰店", "OZKO旗舰店")
|
||||
|
||||
monkeypatch.setattr(products, "BRAND", "光影行星")
|
||||
assert products._login_identity_markers() == ("光影行星旗舰店",)
|
||||
|
||||
|
||||
def test_dynamic_tm_brand_route_selects_the_requested_account_vault(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
class FakeCatalog:
|
||||
def environment_for_account(self, account_id, base_environment):
|
||||
calls.append(account_id)
|
||||
return {
|
||||
**base_environment,
|
||||
"GYXX_ACCOUNT_ID": account_id,
|
||||
"GYXX_ACCOUNT_COOKIE_FILE": f"{account_id}-cookies.json",
|
||||
"GYXX_ACCOUNT_STORAGE_STATE_FILE": f"{account_id}-state.json",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
collect.RuntimeIntegrationCatalog,
|
||||
"load_default",
|
||||
lambda **_kwargs: FakeCatalog(),
|
||||
)
|
||||
|
||||
environment = collect._account_environment(
|
||||
"tmall-ozko",
|
||||
{
|
||||
"GYXX_PROJECT_ROOT": "D:/project",
|
||||
"GYXX_DATA_ROOT": "D:/data",
|
||||
"KEEP": "yes",
|
||||
},
|
||||
)
|
||||
|
||||
assert calls == ["tmall-ozko"]
|
||||
assert environment["KEEP"] == "yes"
|
||||
assert environment["GYXX_ACCOUNT_ID"] == "tmall-ozko"
|
||||
assert environment["GYXX_ACCOUNT_COOKIE_FILE"] == "tmall-ozko-cookies.json"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extra_args", "expects_backfill"),
|
||||
[
|
||||
([], False),
|
||||
(["--backfill"], True),
|
||||
],
|
||||
)
|
||||
def test_each_tm_brand_is_injected_into_its_own_child_profile(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
extra_args: list[str],
|
||||
expects_backfill: bool,
|
||||
) -> None:
|
||||
profile_root = tmp_path / "tm-profile"
|
||||
environments: dict[str, dict[str, str]] = {}
|
||||
ports: dict[str, str] = {}
|
||||
launched_ports: list[int] = []
|
||||
child_scripts: list[str] = []
|
||||
child_commands: dict[str, list[str]] = {}
|
||||
binding_scripts: list[str] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_tm_style_id_map",
|
||||
lambda: {"极星pro": ["main-id"], "布谷": ["ozko-id"]},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_tm_brand_map",
|
||||
lambda: {"极星pro": "光影行星", "布谷": "ozko"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_daily_styles",
|
||||
lambda: {
|
||||
"tm": {
|
||||
"collectable": ["极星pro", "布谷"],
|
||||
"skipped": [],
|
||||
}
|
||||
},
|
||||
)
|
||||
def fake_child_environment(script, *_args):
|
||||
binding_scripts.append(Path(script).name)
|
||||
return {
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(profile_root),
|
||||
"GYXX_BROWSER_CDP_PORT": "22096",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(collect, "environment_for_child_script", fake_child_environment)
|
||||
monkeypatch.setattr(collect, "_kill_port_listeners", lambda _port: 0)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"launch_chrome_with_debug",
|
||||
lambda *_args, **kwargs: launched_ports.append(kwargs["debug_port"]) or True,
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
brand = command[command.index("--brand") + 1]
|
||||
environments[brand] = kwargs["env"]
|
||||
ports[brand] = kwargs["env"]["GYXX_BROWSER_CDP_PORT"]
|
||||
child_scripts.append(Path(command[1]).name)
|
||||
child_commands[brand] = command
|
||||
return SimpleNamespace(returncode=0)
|
||||
|
||||
monkeypatch.setattr(collect.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"taobao_sycm_collect.py",
|
||||
"--target-date",
|
||||
"2026-08-06",
|
||||
*extra_args,
|
||||
],
|
||||
)
|
||||
|
||||
assert collect.main() == 0
|
||||
assert child_scripts == ["taobao_sycm_products.py", "taobao_sycm_products.py"]
|
||||
assert binding_scripts == ["taobao_sycm_products.py"]
|
||||
for brand, expected_profile in {
|
||||
"光影行星": profile_root,
|
||||
"ozko": profile_root.parent / f"{profile_root.name}_ozko_tmall-ozko",
|
||||
}.items():
|
||||
command = child_commands[brand]
|
||||
assert ("--backfill" in command) is expects_backfill
|
||||
assert command[command.index("--cdp-url") + 1] == (
|
||||
"http://127.0.0.1:" + ports[brand]
|
||||
)
|
||||
assert command[command.index("--brand-user-data-dir") + 1] == str(
|
||||
expected_profile
|
||||
)
|
||||
assert environments["光影行星"]["GYXX_BROWSER_PROFILE_DIR"] == str(
|
||||
profile_root
|
||||
)
|
||||
assert environments["光影行星"]["TM_USER_DATA_DIR"] == str(profile_root)
|
||||
assert environments["ozko"]["GYXX_BROWSER_PROFILE_DIR"] == str(
|
||||
profile_root.parent / f"{profile_root.name}_ozko_tmall-ozko"
|
||||
)
|
||||
assert environments["ozko"]["TM_USER_DATA_DIR"] == str(
|
||||
profile_root.parent / f"{profile_root.name}_ozko_tmall-ozko"
|
||||
)
|
||||
assert ports["光影行星"] != ports["ozko"]
|
||||
assert sorted(launched_ports) == [22196, 22197]
|
||||
|
||||
|
||||
def test_cookie_recovery_marks_all_tm_brands_for_relogin(monkeypatch, tmp_path: Path) -> None:
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(collect, "get_tm_style_id_map", lambda: {"a": ["1"], "b": ["2"]})
|
||||
monkeypatch.setattr(collect, "get_tm_brand_map", lambda: {"a": "ozko", "b": "光影行星"})
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_daily_styles",
|
||||
lambda: {"tm": {"collectable": ["a", "b"], "skipped": []}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"environment_for_child_script",
|
||||
lambda *_args: {
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(tmp_path / "profile"),
|
||||
"GYXX_BROWSER_CDP_PORT": "22096",
|
||||
},
|
||||
)
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
calls.append(command[command.index("--brand") + 1])
|
||||
return SimpleNamespace(returncode=COOKIE_SKIP_EXIT_CODE)
|
||||
|
||||
monkeypatch.setattr(collect.subprocess, "run", fake_run)
|
||||
|
||||
assert collect.main(["--target-date", "2026-08-11", "--backfill"]) == COOKIE_SKIP_EXIT_CODE
|
||||
assert sorted(calls) == ["ozko", "光影行星"]
|
||||
|
||||
|
||||
def test_transient_brand_child_failure_is_retried_once(monkeypatch, tmp_path: Path) -> None:
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(collect, "get_tm_style_id_map", lambda: {"a": ["1"]})
|
||||
monkeypatch.setattr(collect, "get_tm_brand_map", lambda: {"a": "ozko"})
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_daily_styles",
|
||||
lambda: {"tm": {"collectable": ["a"], "skipped": []}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"environment_for_child_script",
|
||||
lambda *_args: {
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(tmp_path / "profile"),
|
||||
"GYXX_BROWSER_CDP_PORT": "22096",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(collect, "_kill_port_listeners", lambda _port: 0)
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
calls.append(command[command.index("--brand") + 1])
|
||||
return SimpleNamespace(returncode=1 if len(calls) == 1 else 0)
|
||||
|
||||
monkeypatch.setattr(collect.subprocess, "run", fake_run)
|
||||
|
||||
assert collect.main(["--target-date", "2026-08-11"]) == 0
|
||||
assert calls == ["ozko", "ozko"]
|
||||
|
||||
|
||||
def test_account_routed_brand_uses_a_fresh_collection_profile(monkeypatch, tmp_path: Path) -> None:
|
||||
profiles: list[str] = []
|
||||
monkeypatch.setattr(collect, "get_tm_style_id_map", lambda: {"布谷": ["1"]})
|
||||
monkeypatch.setattr(collect, "get_tm_brand_map", lambda: {"布谷": "ozko"})
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"get_daily_styles",
|
||||
lambda: {"tm": {"collectable": ["布谷"], "skipped": []}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"environment_for_child_script",
|
||||
lambda *_args: {
|
||||
"GYXX_PROJECT_ROOT": "D:/project",
|
||||
"GYXX_DATA_ROOT": "D:/data",
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(tmp_path / "legacy-profile"),
|
||||
"GYXX_BROWSER_CDP_PORT": "22096",
|
||||
"GYXX_BROWSER_CDP_URL": "http://127.0.0.1:22096",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collect,
|
||||
"_account_environment",
|
||||
lambda account_id, environment: {
|
||||
**environment,
|
||||
"GYXX_ACCOUNT_ID": account_id,
|
||||
"GYXX_ACCOUNT_PROFILE_DIR": str(tmp_path / "account" / "profile"),
|
||||
"GYXX_ACCOUNT_CDP_PORT": "22097",
|
||||
"GYXX_ACCOUNT_CDP_URL": "http://127.0.0.1:22097",
|
||||
"GYXX_ACCOUNT_COLLECTION_CDP_PORT": "22143",
|
||||
"GYXX_ACCOUNT_COLLECTION_CDP_URL": "http://127.0.0.1:22143",
|
||||
"GYXX_ACCOUNT_COOKIE_FILE": "vault-cookies.json",
|
||||
"GYXX_ACCOUNT_STORAGE_STATE_FILE": "vault-state.json",
|
||||
"GYXX_ACCOUNT_REQUIRED_COOKIE_DOMAINS": "[\"taobao.com\"]",
|
||||
"GYXX_ACCOUNT_REQUIRED_COOKIE_NAMES": "[\"cookie2\"]",
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES": "[]",
|
||||
"GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS": "[\"ozko\"]",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(collect, "_kill_port_listeners", lambda _port: 0)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
profiles.append(kwargs["env"]["TM_USER_DATA_DIR"])
|
||||
assert kwargs["env"]["GYXX_BROWSER_CDP_PORT"] == "22143"
|
||||
assert kwargs["env"]["GYXX_BROWSER_CDP_URL"] == (
|
||||
"http://127.0.0.1:22143"
|
||||
)
|
||||
assert kwargs["env"]["AUTOFLOW_CDP_PORT"] == "22143"
|
||||
assert kwargs["env"]["GYXX_ACCOUNT_CDP_PORT"] == "22097"
|
||||
assert command[command.index("--cdp-url") + 1] == (
|
||||
"http://127.0.0.1:22143"
|
||||
)
|
||||
return SimpleNamespace(returncode=0)
|
||||
|
||||
monkeypatch.setattr(collect.subprocess, "run", fake_run)
|
||||
|
||||
assert collect.main(["--target-date", "2026-08-11"]) == 0
|
||||
assert len(profiles) == 1
|
||||
assert Path(profiles[0]).parent == tmp_path / "account" / "collector-runs"
|
||||
assert Path(profiles[0]).name.startswith("tmall-ozko-2026-08-11-")
|
||||
@@ -4,6 +4,11 @@ from pathlib import Path
|
||||
|
||||
import taobao_sycm_products as tm
|
||||
|
||||
from gyxx_flow.core.exit_codes import (
|
||||
COOKIE_SKIP_EXIT_CODE,
|
||||
NON_RETRYABLE_EXIT_CODE,
|
||||
)
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self) -> None:
|
||||
@@ -14,12 +19,171 @@ class _Session:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _LoginPage:
|
||||
url = "https://sycm.taobao.com/cc/item_rank"
|
||||
|
||||
def __init__(self, state: dict[str, bool]) -> None:
|
||||
self.state = state
|
||||
|
||||
def evaluate(self, *_args):
|
||||
return self.state
|
||||
|
||||
|
||||
def test_tm_login_state_requires_the_configured_shop(monkeypatch) -> None:
|
||||
monkeypatch.setattr(tm, "ACCOUNT", "光影行星旗舰店:operator")
|
||||
|
||||
wrong_shop = _LoginPage(
|
||||
{"hasStoreName": False, "hasTopNav": True, "hasAvatar": True}
|
||||
)
|
||||
requested_shop = _LoginPage(
|
||||
{"hasStoreName": True, "hasTopNav": True, "hasAvatar": True}
|
||||
)
|
||||
|
||||
assert tm.is_logged_in(wrong_shop) is False
|
||||
assert tm.is_logged_in(requested_shop) is True
|
||||
|
||||
|
||||
def test_tm_account_vault_replaces_stale_profile_cookies(monkeypatch, tmp_path: Path) -> None:
|
||||
class Context:
|
||||
def __init__(self) -> None:
|
||||
self.cleared = False
|
||||
|
||||
def clear_cookies(self) -> None:
|
||||
self.cleared = True
|
||||
|
||||
restored: list[tuple[str, str]] = []
|
||||
context = Context()
|
||||
cookie_file = tmp_path / "vault-cookies.json"
|
||||
storage_state_file = tmp_path / "vault-state.json"
|
||||
cookie_file.write_text("[]", encoding="utf-8")
|
||||
storage_state_file.write_text("{}", encoding="utf-8")
|
||||
monkeypatch.setenv("GYXX_ACCOUNT_ID", "tmall-ozko")
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"_runtime_state_paths",
|
||||
lambda: ((str(cookie_file), str(storage_state_file)), ("", "")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"restore_browser_state",
|
||||
lambda _context, *, cookie_file, storage_state_file: restored.append(
|
||||
(cookie_file, storage_state_file)
|
||||
),
|
||||
)
|
||||
|
||||
tm._restore_runtime_account_state(context)
|
||||
|
||||
assert context.cleared is True
|
||||
assert restored == [(str(cookie_file), str(storage_state_file))]
|
||||
|
||||
|
||||
def test_tm_main_fails_when_browser_connection_is_missing(monkeypatch) -> None:
|
||||
monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (None, None))
|
||||
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == 1
|
||||
|
||||
|
||||
def test_tm_backfill_delegates_inside_the_daily_browser_binding(monkeypatch) -> None:
|
||||
import taobao_sycm_collect_backfill as backfill
|
||||
|
||||
delegated: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
backfill,
|
||||
"main",
|
||||
lambda argv=None: delegated.extend(argv or []) or 0,
|
||||
)
|
||||
|
||||
assert tm.main([
|
||||
"--target-date",
|
||||
"2026-08-03",
|
||||
"--brand-user-data-dir",
|
||||
"brand-profile",
|
||||
"--brand",
|
||||
"ozko",
|
||||
"--backfill",
|
||||
]) == 0
|
||||
assert delegated[:4] == [
|
||||
"--target-date",
|
||||
"2026-08-03",
|
||||
"--brand-user-data-dir",
|
||||
"brand-profile",
|
||||
]
|
||||
assert delegated[-2:] == ["--brand", "ozko"]
|
||||
|
||||
|
||||
def test_tm_backfill_returns_cookie_recovery_code_without_credentials(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
import taobao_sycm_collect_backfill as backfill
|
||||
|
||||
session = _Session()
|
||||
monkeypatch.setattr(
|
||||
backfill,
|
||||
"start_stealthy_session",
|
||||
lambda *_args: (session, object()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
backfill,
|
||||
"ensure_logged_in",
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
RuntimeError("登录态失效且未配置 SYCM_ACCOUNT/SYCM_PASSWORD")
|
||||
),
|
||||
)
|
||||
|
||||
assert backfill.main(["--target-date", "2026-08-03"]) == COOKIE_SKIP_EXIT_CODE
|
||||
assert session.closed is True
|
||||
|
||||
|
||||
def test_tm_manual_login_waits_for_the_operator_and_closes_session(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
session = _Session()
|
||||
page = type("Page", (), {"goto": lambda *_args, **_kwargs: None})()
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"connect_native_manual_login",
|
||||
lambda _profile_dir: (session, page),
|
||||
)
|
||||
monkeypatch.setattr(tm, "is_logged_in", lambda _page: True)
|
||||
|
||||
assert tm.main([
|
||||
"--brand-user-data-dir",
|
||||
"ozko-profile",
|
||||
"--manual-login",
|
||||
"--login-timeout-seconds",
|
||||
"1",
|
||||
]) == 0
|
||||
assert tm.USER_DATA_DIR == "ozko-profile"
|
||||
assert session.closed is True
|
||||
|
||||
|
||||
def test_tm_login_target_returns_to_the_product_page() -> None:
|
||||
assert "_target=https%3A%2F%2Fsycm.taobao.com%2Fcc%2Fitem_rank" in tm.LOGIN_URL
|
||||
assert "/custom/login" not in tm.LOGIN_URL.split("_target=", 1)[1]
|
||||
|
||||
|
||||
def test_tm_native_manual_login_starts_with_one_blank_tab(monkeypatch) -> None:
|
||||
launched: list[tuple[str, str]] = []
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"launch_chrome_with_debug",
|
||||
lambda profile, start_url=tm.LOGIN_URL: launched.append((profile, start_url))
|
||||
or True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tm,
|
||||
"connect_browser",
|
||||
lambda **kwargs: ("session", kwargs["cdp_url"]),
|
||||
)
|
||||
monkeypatch.setattr(tm, "DEBUG_PORT", 22096)
|
||||
|
||||
assert tm.connect_native_manual_login("ozko-profile") == (
|
||||
"session",
|
||||
"http://127.0.0.1:22096",
|
||||
)
|
||||
assert launched == [("ozko-profile", "about:blank")]
|
||||
|
||||
|
||||
def test_tm_main_fails_when_login_does_not_complete(monkeypatch) -> None:
|
||||
session = _Session()
|
||||
page = object()
|
||||
@@ -73,6 +237,9 @@ def test_tm_main_fails_when_download_has_no_valid_style(
|
||||
monkeypatch.setattr(tm, "export_excel_to_json_and_md", lambda _path: (None, None))
|
||||
monkeypatch.setattr(tm, "summarize_styles_from_excel", lambda _path: [])
|
||||
monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0)
|
||||
persisted: list[bool] = []
|
||||
monkeypatch.setattr(tm, "_persist_runtime_account_state", lambda _context: persisted.append(True))
|
||||
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == 1
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == NON_RETRYABLE_EXIT_CODE
|
||||
assert session.closed is True
|
||||
assert persisted == []
|
||||
|
||||
@@ -1,11 +1,73 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import taobao_dmp_item_crowd_insight_screenshots as dmp
|
||||
|
||||
|
||||
class TmPersonaRecoveryTests(unittest.TestCase):
|
||||
def test_cdp_connection_uses_adapter_existing_context_mode(self):
|
||||
page = Mock()
|
||||
context = Mock()
|
||||
context.new_page.return_value = page
|
||||
session = Mock()
|
||||
session.context = context
|
||||
browser_factory = Mock()
|
||||
browser_factory.return_value.start.return_value = session
|
||||
|
||||
with patch.object(dmp, "ScraplingBrowser", browser_factory):
|
||||
actual_session, actual_page = dmp._connect_cdp()
|
||||
|
||||
browser_factory.assert_called_once_with(
|
||||
stealthy=True,
|
||||
reuse_existing_cdp_context=True,
|
||||
cdp_url=dmp.CDP_URL,
|
||||
timeout=60000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
block_ads=False,
|
||||
)
|
||||
self.assertIs(actual_session, session)
|
||||
self.assertIs(actual_page, page)
|
||||
page.set_viewport_size.assert_called_once_with(
|
||||
{"width": 1920, "height": 1080}
|
||||
)
|
||||
|
||||
def test_item_insight_spa_route_is_authenticated_before_widgets_render(self):
|
||||
page = Mock()
|
||||
page.url = "https://dmp.taobao.com/index_new.html#!/items/item-insight"
|
||||
|
||||
with (
|
||||
patch.object(dmp, "login_form_visible", return_value=False),
|
||||
patch.object(dmp, "dmp_app_ready", return_value=False),
|
||||
patch.object(dmp, "page_text", return_value=""),
|
||||
):
|
||||
self.assertTrue(dmp.is_logged_in(page))
|
||||
|
||||
def test_login_redirect_is_not_authenticated_when_spa_text_leaks(self):
|
||||
page = Mock()
|
||||
page.url = (
|
||||
"https://dmp.taobao.com/login.html?mxredirectUrl="
|
||||
"https%3A%2F%2Fdmp.taobao.com%2Findex_new.html%23!%2Fitems%2Fitem-insight"
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(dmp, "login_form_visible", return_value=False),
|
||||
patch.object(dmp, "dmp_app_ready", return_value=False),
|
||||
patch.object(dmp, "page_text", return_value="单品信息 单品人群洞察"),
|
||||
):
|
||||
self.assertFalse(dmp.is_logged_in(page))
|
||||
|
||||
def test_login_redirect_is_not_ready_when_spa_text_leaks(self):
|
||||
page = Mock()
|
||||
page.url = (
|
||||
"https://dmp.taobao.com/login.html?mxredirectUrl="
|
||||
"https%3A%2F%2Fdmp.taobao.com%2Findex_new.html%23!%2Fitems%2Fitem-insight"
|
||||
)
|
||||
|
||||
self.assertFalse(dmp.dmp_app_ready(page))
|
||||
page.evaluate.assert_not_called()
|
||||
|
||||
def test_zero_chart_skip_is_not_platform_success(self):
|
||||
self.assertEqual(
|
||||
dmp.records_exit_code(
|
||||
|
||||
@@ -0,0 +1,548 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import tmall_baibu_apply_common as baibu
|
||||
|
||||
|
||||
def _entry() -> baibu.BaibuEntry:
|
||||
return baibu.BaibuEntry(
|
||||
entry_id="demo",
|
||||
name="测试入口",
|
||||
account_id="tmall-shop",
|
||||
account_marker="测试店铺",
|
||||
template_url="https://feishu.example/wiki/template",
|
||||
apply_url="https://myseller.example/apply",
|
||||
script="demo.py",
|
||||
)
|
||||
|
||||
|
||||
def test_latest_batch_import_status_parses_processing_and_complete_summary() -> None:
|
||||
processing = baibu._latest_batch_import_status(
|
||||
"您于2026-08-26 09:23:37执行了商品批量导入操作,正在处理中。"
|
||||
)
|
||||
assert processing is not None
|
||||
assert processing.state == "processing"
|
||||
|
||||
completed = baibu._latest_batch_import_status(
|
||||
"最近操作:您于2026-08-26 09:23:37执行了商品批量导入操作,"
|
||||
"总数量20件,已成功20件,失败0件,待查看0件。"
|
||||
)
|
||||
assert completed is not None
|
||||
assert completed.state == "success"
|
||||
assert completed.detail == "总数量20件,已成功20件,失败0件,待查看0件"
|
||||
|
||||
|
||||
def test_latest_batch_import_status_marks_partial_summary_as_failure() -> None:
|
||||
result = baibu._latest_batch_import_status(
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量95件,已成功8件,失败87件,待查看0件。"
|
||||
)
|
||||
assert result is not None
|
||||
assert result.state == "failure"
|
||||
assert "失败87件" in result.detail
|
||||
|
||||
|
||||
def test_credentials_are_loaded_from_account_declared_environment_names(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
|
||||
'["TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD"]',
|
||||
)
|
||||
monkeypatch.delenv("GYXX_BROWSER_CREDENTIAL_ENV_NAMES", raising=False)
|
||||
monkeypatch.setenv("TMALL_BAG_ACCOUNT", "demo-account")
|
||||
monkeypatch.setenv("TMALL_BAG_PASSWORD", "demo-password")
|
||||
|
||||
credentials, names = baibu._credentials_from_environment()
|
||||
|
||||
assert names == ("TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD")
|
||||
assert credentials is not None
|
||||
assert credentials.username == "demo-account"
|
||||
assert credentials.password == "demo-password"
|
||||
assert credentials.username_env == "TMALL_BAG_ACCOUNT"
|
||||
assert credentials.password_env == "TMALL_BAG_PASSWORD"
|
||||
|
||||
|
||||
def test_credentials_prefer_the_tmall_shop_pair_over_legacy_pairs(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
|
||||
'["TMALL_SHOP_ACCOUNT", "TMALL_SHOP_PASSWORD", '
|
||||
'"SYCM_ACCOUNT", "SYCM_PASSWORD"]',
|
||||
)
|
||||
monkeypatch.setenv("TMALL_SHOP_ACCOUNT", "完整店铺名:子账号")
|
||||
monkeypatch.setenv("TMALL_SHOP_PASSWORD", "shop-password")
|
||||
monkeypatch.setenv("SYCM_ACCOUNT", "legacy-account")
|
||||
monkeypatch.setenv("SYCM_PASSWORD", "legacy-password")
|
||||
|
||||
credentials, names = baibu._credentials_from_environment()
|
||||
|
||||
assert names == (
|
||||
"TMALL_SHOP_ACCOUNT",
|
||||
"TMALL_SHOP_PASSWORD",
|
||||
"SYCM_ACCOUNT",
|
||||
"SYCM_PASSWORD",
|
||||
)
|
||||
assert credentials is not None
|
||||
assert credentials.username == "完整店铺名:子账号"
|
||||
assert credentials.username_env == "TMALL_SHOP_ACCOUNT"
|
||||
assert credentials.password_env == "TMALL_SHOP_PASSWORD"
|
||||
|
||||
|
||||
class _LoginLocator:
|
||||
def __init__(self, *, visible: bool = True) -> None:
|
||||
self.visible = visible
|
||||
self.filled: list[str] = []
|
||||
self.click_count = 0
|
||||
|
||||
def count(self) -> int:
|
||||
return 1 if self.visible else 0
|
||||
|
||||
def nth(self, _index: int):
|
||||
return self
|
||||
|
||||
def is_visible(self) -> bool:
|
||||
return self.visible
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
return True
|
||||
|
||||
def fill(self, value: str) -> None:
|
||||
self.filled.append(value)
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.click_count += 1
|
||||
|
||||
|
||||
class _LoginFrame:
|
||||
def __init__(self) -> None:
|
||||
self.username = _LoginLocator()
|
||||
self.password = _LoginLocator()
|
||||
self.submit = _LoginLocator()
|
||||
|
||||
def locator(self, selector: str):
|
||||
return {
|
||||
"input[placeholder='账号名/邮箱/手机号']": self.username,
|
||||
"input[placeholder='请输入登录密码']": self.password,
|
||||
"button[type='submit']": _LoginLocator(visible=False),
|
||||
"input[type='submit']": _LoginLocator(visible=False),
|
||||
}.get(selector, _LoginLocator(visible=False))
|
||||
|
||||
def get_by_role(self, role: str, *, name: str, exact: bool):
|
||||
if role == "button" and name == "登录" and exact:
|
||||
return self.submit
|
||||
return _LoginLocator(visible=False)
|
||||
|
||||
def get_by_text(self, label: str, *, exact: bool):
|
||||
return _LoginLocator(visible=False)
|
||||
|
||||
|
||||
def test_login_if_needed_fills_the_password_form_inside_an_iframe(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
|
||||
'["TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD"]',
|
||||
)
|
||||
monkeypatch.setenv("TMALL_BAG_ACCOUNT", "demo-account")
|
||||
monkeypatch.setenv("TMALL_BAG_PASSWORD", "demo-password")
|
||||
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
|
||||
frame = _LoginFrame()
|
||||
page = SimpleNamespace(
|
||||
url="https://loginmyseller.taobao.com/?redirect_url=https%3A%2F%2Fqn.taobao.com",
|
||||
frames=[frame],
|
||||
)
|
||||
|
||||
attempted = baibu._login_if_needed(page=page, entry=_entry())
|
||||
|
||||
assert attempted is True
|
||||
assert frame.username.filled == ["demo-account"]
|
||||
assert frame.password.filled == ["demo-password"]
|
||||
assert frame.submit.click_count == 1
|
||||
|
||||
|
||||
def test_manual_login_waits_without_filling_the_password_form(monkeypatch) -> None:
|
||||
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
|
||||
frame = _LoginFrame()
|
||||
page = SimpleNamespace(
|
||||
url="https://loginmyseller.taobao.com/",
|
||||
frames=[frame],
|
||||
)
|
||||
|
||||
attempted = baibu._login_if_needed(
|
||||
page=page,
|
||||
entry=_entry(),
|
||||
manual_login=True,
|
||||
)
|
||||
|
||||
assert attempted is True
|
||||
assert frame.username.filled == []
|
||||
assert frame.password.filled == []
|
||||
assert frame.submit.click_count == 0
|
||||
|
||||
|
||||
def test_login_if_needed_reports_missing_credentials_without_filling(monkeypatch) -> None:
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES",
|
||||
'["TMALL_BAG_ACCOUNT", "TMALL_BAG_PASSWORD"]',
|
||||
)
|
||||
monkeypatch.delenv("TMALL_BAG_ACCOUNT", raising=False)
|
||||
monkeypatch.delenv("TMALL_BAG_PASSWORD", raising=False)
|
||||
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
|
||||
frame = _LoginFrame()
|
||||
page = SimpleNamespace(
|
||||
url="https://loginmyseller.taobao.com/",
|
||||
frames=[frame],
|
||||
)
|
||||
|
||||
try:
|
||||
baibu._login_if_needed(page=page, entry=_entry())
|
||||
except baibu.BaibuApplyError as exc:
|
||||
assert "TMALL_BAG_ACCOUNT" in str(exc)
|
||||
assert "TMALL_BAG_PASSWORD" in str(exc)
|
||||
else:
|
||||
raise AssertionError("missing credentials must stop before filling")
|
||||
|
||||
assert frame.username.filled == []
|
||||
assert frame.password.filled == []
|
||||
|
||||
|
||||
def test_wait_for_authenticated_stops_on_explicit_login_error(monkeypatch) -> None:
|
||||
monkeypatch.setattr(baibu, "_authenticated", lambda _entry, _page: False)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: "账密错误")
|
||||
|
||||
with pytest.raises(baibu.BaibuApplyError, match="账密错误"):
|
||||
baibu._wait_for_authenticated(
|
||||
_entry(),
|
||||
SimpleNamespace(url="https://loginmyseller.taobao.com/"),
|
||||
login_attempted=True,
|
||||
)
|
||||
|
||||
|
||||
def test_login_second_factor_marker_is_detected(monkeypatch) -> None:
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: "你正在使用手机短信验证身份")
|
||||
|
||||
assert baibu._login_second_factor_marker(SimpleNamespace()) == "短信验证"
|
||||
|
||||
|
||||
def test_persist_account_state_filters_cookies_to_the_account_domains(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
account_cookie_file = tmp_path / "state" / "accounts" / "tmall-bag" / "cookies.json"
|
||||
account_storage_file = (
|
||||
tmp_path / "state" / "accounts" / "tmall-bag" / "storage_state.json"
|
||||
)
|
||||
monkeypatch.setenv("GYXX_ACCOUNT_COOKIE_FILE", str(account_cookie_file))
|
||||
monkeypatch.setenv("GYXX_ACCOUNT_STORAGE_STATE_FILE", str(account_storage_file))
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_REQUIRED_COOKIE_DOMAINS",
|
||||
'["taobao.com", "alimama.com"]',
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
"GYXX_ACCOUNT_REQUIRED_COOKIE_NAMES",
|
||||
'["unb", "cookie2", "_tb_token_"]',
|
||||
)
|
||||
|
||||
cookies = [
|
||||
{"name": "unb", "domain": ".taobao.com", "value": "valid"},
|
||||
{"name": "other", "domain": "example.com", "value": "unrelated"},
|
||||
]
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return cookies
|
||||
|
||||
def storage_state(self):
|
||||
return {
|
||||
"cookies": cookies,
|
||||
"origins": [
|
||||
{"origin": "https://qn.taobao.com", "localStorage": []},
|
||||
{"origin": "https://example.com", "localStorage": []},
|
||||
],
|
||||
}
|
||||
|
||||
browser = SimpleNamespace(context=Context())
|
||||
|
||||
assert baibu._persist_account_state(browser, _entry()) is True
|
||||
|
||||
saved_cookies = json.loads(account_cookie_file.read_text(encoding="utf-8"))
|
||||
saved_state = json.loads(account_storage_file.read_text(encoding="utf-8"))
|
||||
assert saved_cookies == [cookies[0]]
|
||||
assert saved_state["cookies"] == [cookies[0]]
|
||||
assert saved_state["origins"] == [
|
||||
{"origin": "https://qn.taobao.com", "localStorage": []}
|
||||
]
|
||||
|
||||
|
||||
def test_upload_template_supports_tmall_dialog_import_button(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
calls: list[tuple[str, ...]] = []
|
||||
|
||||
class Chooser:
|
||||
def __init__(self) -> None:
|
||||
self.files: str | None = None
|
||||
|
||||
def set_files(self, value: str) -> None:
|
||||
self.files = value
|
||||
|
||||
chooser = Chooser()
|
||||
|
||||
class ExpectChooser:
|
||||
def __enter__(self):
|
||||
return SimpleNamespace(value=chooser)
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(baibu, "_file_inputs", lambda _page: [])
|
||||
|
||||
def click(_page, labels):
|
||||
labels = tuple(labels)
|
||||
calls.append(labels)
|
||||
return "导入表格" in labels
|
||||
|
||||
monkeypatch.setattr(baibu, "_click_text", click)
|
||||
page = SimpleNamespace(
|
||||
expect_file_chooser=lambda **_kwargs: ExpectChooser(),
|
||||
)
|
||||
template = tmp_path / "template.xlsx"
|
||||
template.write_bytes(b"xlsx")
|
||||
|
||||
baibu._upload_template(page, template)
|
||||
|
||||
assert calls[0] == ("导入表格",)
|
||||
assert chooser.files == str(template)
|
||||
|
||||
|
||||
def test_open_batch_import_refuses_an_active_previous_operation(monkeypatch) -> None:
|
||||
page = SimpleNamespace()
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_page_text",
|
||||
lambda _page: "您于2026-08-26 09:23:37执行了商品批量导入操作,正在处理中。",
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_refresh_batch_results", lambda _page: True)
|
||||
|
||||
try:
|
||||
baibu._open_batch_import(page, _entry())
|
||||
except baibu.BaibuApplyError as exc:
|
||||
assert "正在处理" in str(exc)
|
||||
else:
|
||||
raise AssertionError("active batch import must not be submitted again")
|
||||
|
||||
|
||||
def test_open_batch_import_waits_for_hydrated_previous_summary(monkeypatch) -> None:
|
||||
page = SimpleNamespace()
|
||||
texts = iter(
|
||||
[
|
||||
"测试店铺 商品批量导入",
|
||||
"测试店铺 最近操作:您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量2件,已成功2件,失败0件,待查看0件。",
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
|
||||
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_text",
|
||||
lambda _page, _labels: True,
|
||||
)
|
||||
|
||||
previous = baibu._open_batch_import(page, _entry())
|
||||
|
||||
assert previous is not None
|
||||
assert previous.state == "success"
|
||||
|
||||
|
||||
def test_wait_import_result_refreshes_and_accepts_new_success_summary(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
texts = iter(
|
||||
[
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,正在处理中。",
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量2件,已成功2件,失败0件,待查看0件。",
|
||||
]
|
||||
)
|
||||
refreshes: list[bool] = []
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_refresh_batch_results",
|
||||
lambda _page: refreshes.append(True) or True,
|
||||
)
|
||||
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(baibu, "_RESULT_REFRESH_INTERVAL_SECONDS", 0)
|
||||
|
||||
baibu._wait_import_result(SimpleNamespace(), _entry())
|
||||
|
||||
assert refreshes
|
||||
|
||||
|
||||
def test_wait_import_result_accepts_new_failure_summary_as_submitted(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
texts = iter(
|
||||
[
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,正在处理中。",
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量95件,已成功0件,失败95件,待查看0件。",
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
|
||||
monkeypatch.setattr(baibu, "_refresh_batch_results", lambda _page: True)
|
||||
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(baibu, "_RESULT_REFRESH_INTERVAL_SECONDS", 0)
|
||||
|
||||
# Tmall's row-level result is informational; submitting the batch is the
|
||||
# success condition for this workflow.
|
||||
baibu._wait_import_result(SimpleNamespace(), _entry())
|
||||
|
||||
|
||||
def test_wait_import_result_ignores_stale_summary_until_signature_changes(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
previous = baibu._latest_batch_import_status(
|
||||
"您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量95件,已成功8件,失败87件,待查看0件。"
|
||||
)
|
||||
assert previous is not None
|
||||
texts = iter(
|
||||
[
|
||||
"最近操作:您于2026-08-26 16:53:04执行了商品批量导入操作,"
|
||||
"总数量95件,已成功8件,失败87件,待查看0件。",
|
||||
"您于2026-08-26 17:30:01执行了商品批量导入操作,正在处理中。",
|
||||
"您于2026-08-26 17:30:01执行了商品批量导入操作,"
|
||||
"总数量2件,已成功2件,失败0件,待查看0件。",
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(texts))
|
||||
monkeypatch.setattr(baibu, "_refresh_batch_results", lambda _page: True)
|
||||
monkeypatch.setattr(baibu.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(baibu.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(baibu, "_RESULT_REFRESH_INTERVAL_SECONDS", 0)
|
||||
|
||||
baibu._wait_import_result(
|
||||
SimpleNamespace(),
|
||||
_entry(),
|
||||
previous_operation=previous,
|
||||
)
|
||||
|
||||
|
||||
def test_complete_draft_products_runs_required_sequence_until_count_is_zero(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
entry = replace(_entry(), complete_drafts_after_import=True)
|
||||
page = SimpleNamespace()
|
||||
page_texts = iter(["草稿 (2)", "草稿 (1)", "草稿 (0)"])
|
||||
events: list[str] = []
|
||||
clicked: list[tuple[str, ...]] = []
|
||||
opened: list[bool] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_draft_tab",
|
||||
lambda _page: events.append("draft") or True,
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: next(page_texts))
|
||||
monkeypatch.setattr(baibu, "_visible_text_count", lambda _page, _label: 1)
|
||||
monkeypatch.setattr(baibu, "_has_visible_text", lambda _page, _labels: True)
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_open_draft_editor",
|
||||
lambda _page, _entry, *, first: events.append(
|
||||
f"open:{first}"
|
||||
) or opened.append(first),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_text_until_found",
|
||||
lambda _page, labels, **_kwargs: events.append(next(iter(labels)))
|
||||
or clicked.append(tuple(labels)),
|
||||
)
|
||||
|
||||
baibu._complete_draft_products(page, entry)
|
||||
|
||||
assert opened == [True, False]
|
||||
assert events == [
|
||||
"draft",
|
||||
"open:True",
|
||||
"否",
|
||||
"下一步",
|
||||
"draft",
|
||||
"open:False",
|
||||
"否",
|
||||
"下一步",
|
||||
"draft",
|
||||
]
|
||||
assert clicked == [
|
||||
("否",),
|
||||
("下一步",),
|
||||
("否",),
|
||||
("下一步",),
|
||||
]
|
||||
|
||||
|
||||
def test_complete_draft_products_is_disabled_for_other_entries(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_draft_tab",
|
||||
lambda _page: pytest.fail("其他入口不得进入草稿处理流程"),
|
||||
)
|
||||
|
||||
baibu._complete_draft_products(SimpleNamespace(), _entry())
|
||||
|
||||
|
||||
def test_complete_draft_products_accepts_empty_data_page(monkeypatch) -> None:
|
||||
entry = replace(_entry(), complete_drafts_after_import=True)
|
||||
page = SimpleNamespace()
|
||||
|
||||
monkeypatch.setattr(baibu, "_click_draft_tab", lambda _page: True)
|
||||
monkeypatch.setattr(baibu, "_page_text", lambda _page: "草稿 数据为空")
|
||||
monkeypatch.setattr(baibu, "_visible_text_count", lambda _page, _label: 0)
|
||||
|
||||
baibu._complete_draft_products(page, entry)
|
||||
|
||||
|
||||
def test_return_to_draft_list_falls_back_to_browser_history(monkeypatch) -> None:
|
||||
events: list[str] = []
|
||||
|
||||
class Page:
|
||||
def go_back(self, **_kwargs) -> None:
|
||||
events.append("back")
|
||||
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_click_draft_tab",
|
||||
lambda _page: events.append("draft") or False,
|
||||
)
|
||||
monkeypatch.setattr(baibu, "_close_editor", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
baibu,
|
||||
"_has_visible_text",
|
||||
lambda _page, labels: "完善商品" in labels and "back" in events,
|
||||
)
|
||||
|
||||
baibu._return_to_draft_list(Page(), _entry())
|
||||
|
||||
assert events == ["draft", "back"]
|
||||
|
||||
|
||||
def test_tmall_config_enables_draft_completion_for_only_old_all_3c() -> None:
|
||||
old_all_3c = baibu._load_entry("old_all_3c")
|
||||
old_all_bag = baibu._load_entry("old_all_bag")
|
||||
new_all_bag = baibu._load_entry("new_all_bag")
|
||||
|
||||
assert old_all_3c.complete_drafts_after_import is True
|
||||
assert old_all_bag.complete_drafts_after_import is False
|
||||
assert new_all_bag.complete_drafts_after_import is False
|
||||
@@ -0,0 +1,572 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import upload_video_to_guanghe as tmall
|
||||
|
||||
|
||||
def _record(
|
||||
record_id: str,
|
||||
attachments: list[dict[str, Any]],
|
||||
*,
|
||||
product_names: str = "极星双肩",
|
||||
record_date: object = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"record_id": record_id,
|
||||
"代拍产品": product_names,
|
||||
"代拍附件": attachments,
|
||||
"日期": record_date,
|
||||
"品牌": ["光影行星"],
|
||||
"性别": ["女"],
|
||||
tmall.TM_UPLOAD_FIELD: False,
|
||||
}
|
||||
|
||||
|
||||
def _one_candidate(*, store: str = tmall.STORE_LUGGAGE):
|
||||
candidates, issues = tmall.discover_tmall_publication_candidates(
|
||||
[
|
||||
_record(
|
||||
"record-one",
|
||||
[
|
||||
{
|
||||
"type": "file",
|
||||
"name": "video.mp4",
|
||||
"file_token": "video-token",
|
||||
"size": 123,
|
||||
}
|
||||
],
|
||||
)
|
||||
],
|
||||
store=store,
|
||||
)
|
||||
assert issues == []
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def test_catalog_matching_accepts_variant_after_bag_stem() -> None:
|
||||
assert tmall._match_tm_item_ids_from_product_titles(
|
||||
["盖亚斜挎3"],
|
||||
[
|
||||
(
|
||||
"1059764111499",
|
||||
"GYXX/光影行星盖亚斜挎包3单反相机包男款微单摄影包",
|
||||
)
|
||||
],
|
||||
) == {"盖亚斜挎3": ["1059764111499"]}
|
||||
|
||||
|
||||
def test_catalog_search_pattern_keeps_variant_alias_rows_in_scope() -> None:
|
||||
assert tmall._tm_product_title_search_pattern("盖亚斜挎3") == "%盖亚斜挎%"
|
||||
|
||||
|
||||
def test_invalid_compatible_video_cache_is_rebuilt(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source = tmp_path / "source.mov"
|
||||
source.write_bytes(b"source")
|
||||
cached = source.with_name("source_compatible.mp4")
|
||||
cached.write_bytes(b"truncated-cache")
|
||||
|
||||
class _Reader:
|
||||
def __init__(self, metadata: dict[str, Any]) -> None:
|
||||
self.metadata = metadata
|
||||
|
||||
def __iter__(self) -> _Reader:
|
||||
return self
|
||||
|
||||
def __next__(self) -> dict[str, Any]:
|
||||
return self.metadata
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
class _FakeImageioFfmpeg:
|
||||
@staticmethod
|
||||
def get_ffmpeg_exe() -> str:
|
||||
return "ffmpeg"
|
||||
|
||||
@staticmethod
|
||||
def read_frames(path: str) -> _Reader:
|
||||
if path == str(cached):
|
||||
raise OSError("moov atom not found")
|
||||
return _Reader({"codec": "hevc", "source_size": (1920, 1080)})
|
||||
|
||||
monkeypatch.setitem(sys.modules, "imageio_ffmpeg", _FakeImageioFfmpeg)
|
||||
|
||||
def fake_run(command: list[str], **_kwargs: Any) -> SimpleNamespace:
|
||||
Path(command[-1]).write_bytes(b"rebuilt")
|
||||
return SimpleNamespace(returncode=0, stderr="", stdout="")
|
||||
|
||||
monkeypatch.setattr(tmall.subprocess, "run", fake_run)
|
||||
|
||||
assert tmall.ensure_upload_compatible_video(source) == cached
|
||||
assert cached.read_bytes() == b"rebuilt"
|
||||
|
||||
|
||||
def test_discovery_creates_one_candidate_per_exact_video_attachment() -> None:
|
||||
candidates, issues = tmall.discover_tmall_publication_candidates(
|
||||
[
|
||||
_record(
|
||||
"record-with-two-videos",
|
||||
[
|
||||
{
|
||||
"type": "file",
|
||||
"name": "same-name.mp4",
|
||||
"file_token": "video-token-a",
|
||||
"size": "101",
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"name": "same-name.mp4",
|
||||
"file_token": "video-token-b",
|
||||
"size": 202,
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"name": "cover.jpg",
|
||||
"file_token": "image-token",
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"name": "generic-attachment",
|
||||
"file_token": "generic-file-token",
|
||||
},
|
||||
],
|
||||
record_date=None,
|
||||
),
|
||||
_record(
|
||||
"record-marked-skip",
|
||||
[
|
||||
{
|
||||
"type": "file",
|
||||
"name": "must-not-upload.mp4",
|
||||
"file_token": "skipped-token",
|
||||
}
|
||||
],
|
||||
product_names="极星双肩 / 未到货",
|
||||
),
|
||||
],
|
||||
store=tmall.STORE_LUGGAGE,
|
||||
)
|
||||
|
||||
assert {
|
||||
(candidate.source_record_id, candidate.video_file_token)
|
||||
for candidate in candidates
|
||||
} == {
|
||||
("record-with-two-videos", "video-token-a"),
|
||||
("record-with-two-videos", "video-token-b"),
|
||||
}
|
||||
assert {candidate.video_file_name for candidate in candidates} == {
|
||||
"same-name.mp4"
|
||||
}
|
||||
assert all(candidate.source_record_date is None for candidate in candidates)
|
||||
assert [
|
||||
(issue.source_record_id, issue.reason) for issue in issues
|
||||
] == [("record-marked-skip", "代拍产品包含未到货/不发标记")]
|
||||
|
||||
|
||||
def test_flagship_and_luggage_use_distinct_account_scopes() -> None:
|
||||
flagship = _one_candidate(store=tmall.STORE_FLAGSHIP)
|
||||
luggage = _one_candidate(store=tmall.STORE_LUGGAGE)
|
||||
|
||||
assert flagship.target_account_key == "guanghe-flagship"
|
||||
assert luggage.target_account_key == "guanghe-luggage"
|
||||
assert flagship.target_account_key != luggage.target_account_key
|
||||
assert tmall.STORE_ACCOUNT_KEYS == {
|
||||
tmall.STORE_FLAGSHIP: flagship.target_account_key,
|
||||
tmall.STORE_LUGGAGE: luggage.target_account_key,
|
||||
}
|
||||
|
||||
|
||||
def test_luggage_runtime_uses_new_store_account_label_without_changing_password_key(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GUANGHE_LUGGAGE_USERNAME", "光影行星鑫华达专卖店:龙虾仔")
|
||||
monkeypatch.setenv("GUANGHE_LUGGAGE_PASSWORD", "same-runtime-secret")
|
||||
|
||||
store_name, username, password, _profile = tmall._store_runtime(tmall.STORE_LUGGAGE)
|
||||
|
||||
assert store_name == "光影行星鑫华达专卖店:龙虾仔"
|
||||
assert username == "光影行星鑫华达专卖店:龙虾仔"
|
||||
assert password == "same-runtime-secret"
|
||||
|
||||
|
||||
def test_account_identity_markers_accept_store_name_with_login_nickname() -> None:
|
||||
assert tmall.GuangHeUploader._account_identity_markers(
|
||||
"光影行星鑫华达专卖店:龙虾仔"
|
||||
) == (
|
||||
"光影行星鑫华达专卖店:龙虾仔",
|
||||
"光影行星鑫华达专卖店",
|
||||
)
|
||||
|
||||
|
||||
def test_flagship_creator_identity_uses_visible_name_and_account_number() -> None:
|
||||
assert tmall._expected_creator_identity_markers(
|
||||
tmall.STORE_FLAGSHIP,
|
||||
"光影行星旗舰店",
|
||||
) == ("光影行星天猫官方", "逛逛号:3301283916")
|
||||
|
||||
|
||||
def test_creator_identity_requires_every_configured_marker() -> None:
|
||||
class Body:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
|
||||
def inner_text(self, **_kwargs: object) -> str:
|
||||
return self.text
|
||||
|
||||
class Page:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
|
||||
def locator(self, selector: str) -> Body:
|
||||
assert selector == "body"
|
||||
return Body(self.text)
|
||||
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
uploader.expected_account_markers = (
|
||||
"光影行星天猫官方",
|
||||
"逛逛号:3301283916",
|
||||
)
|
||||
uploader.page = Page("光影行星天猫官方\n逛逛号:3301283916\n账号正常")
|
||||
assert uploader._page_matches_expected_account()
|
||||
|
||||
uploader.page = Page("光影行星天猫官方\n账号正常")
|
||||
assert not uploader._page_matches_expected_account()
|
||||
|
||||
|
||||
def test_wait_for_existing_login_allows_account_card_to_render(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
states = iter((False, True))
|
||||
uploader._sync_page = lambda: None
|
||||
uploader._is_logged_in = lambda: next(states)
|
||||
monkeypatch.setattr(tmall.time, "monotonic", lambda: 0.0)
|
||||
monkeypatch.setattr(tmall.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert uploader._wait_for_existing_login(timeout_seconds=8)
|
||||
|
||||
|
||||
def test_legacy_backfill_has_25_exact_attachment_matches_and_fails_on_drift() -> None:
|
||||
entries = tmall.load_legacy_publication_backfill()
|
||||
assert len(entries) == 25
|
||||
|
||||
attachments_by_record: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for entry in entries:
|
||||
attachments_by_record[entry["source_record_id"]].append(
|
||||
{
|
||||
"type": "file",
|
||||
"name": entry["video_file_name"],
|
||||
"file_token": entry["video_file_token"],
|
||||
}
|
||||
)
|
||||
records = [
|
||||
_record(record_id, attachments)
|
||||
for record_id, attachments in attachments_by_record.items()
|
||||
]
|
||||
candidates, issues = tmall.discover_tmall_publication_candidates(
|
||||
records,
|
||||
store=tmall.STORE_LUGGAGE,
|
||||
)
|
||||
|
||||
assert issues == []
|
||||
matched = tmall._legacy_backfill_for_candidates(
|
||||
tmall.STORE_LUGGAGE,
|
||||
candidates,
|
||||
)
|
||||
assert len(matched) == 25
|
||||
assert {
|
||||
(
|
||||
candidate.source_record_id,
|
||||
candidate.video_file_token,
|
||||
candidate.video_file_name,
|
||||
)
|
||||
for candidate, _entry in matched
|
||||
} == {
|
||||
(
|
||||
entry["source_record_id"],
|
||||
entry["video_file_token"],
|
||||
entry["video_file_name"],
|
||||
)
|
||||
for entry in entries
|
||||
}
|
||||
|
||||
drifted = [
|
||||
replace(candidates[0], video_file_token="drifted-token"),
|
||||
*candidates[1:],
|
||||
]
|
||||
with pytest.raises(RuntimeError, match="record_id/file_token/文件名不一致"):
|
||||
tmall._legacy_backfill_for_candidates(
|
||||
tmall.STORE_LUGGAGE,
|
||||
drifted,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_args_is_preview_by_default_and_requires_explicit_execute(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("GYXX_VIDEO_UPLOAD_STORE", raising=False)
|
||||
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py"])
|
||||
preview = tmall.parse_args()
|
||||
assert preview.execute is False
|
||||
assert preview.store == tmall.STORE_FLAGSHIP
|
||||
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["upload_video_to_guanghe.py", "--execute"],
|
||||
)
|
||||
execution = tmall.parse_args()
|
||||
assert execution.execute is True
|
||||
|
||||
|
||||
def test_parse_args_uses_frontend_selected_store_from_runtime_environment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GYXX_VIDEO_UPLOAD_STORE", tmall.STORE_LUGGAGE)
|
||||
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py", "--execute"])
|
||||
|
||||
execution = tmall.parse_args()
|
||||
|
||||
assert execution.store == tmall.STORE_LUGGAGE
|
||||
|
||||
|
||||
def test_parse_args_uses_configured_topic_keyword_and_removes_spacing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv(tmall.TMALL_TOPIC_KEYWORD_ENV, " 秋 上新 ")
|
||||
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py"])
|
||||
|
||||
execution = tmall.parse_args()
|
||||
|
||||
assert execution.topic_keyword == "秋上新"
|
||||
|
||||
|
||||
def test_parse_args_rejects_invalid_runtime_store(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GYXX_VIDEO_UPLOAD_STORE", "unknown")
|
||||
monkeypatch.setattr(sys, "argv", ["upload_video_to_guanghe.py"])
|
||||
|
||||
with pytest.raises(ValueError, match="GYXX_VIDEO_UPLOAD_STORE"):
|
||||
tmall.parse_args()
|
||||
|
||||
|
||||
def test_tmall_pre_submit_failures_are_automatically_retryable() -> None:
|
||||
assert tmall._is_retryable_pre_submit_failure(
|
||||
tmall.TmallTitleGenerationError("no valid candidate"),
|
||||
1,
|
||||
)
|
||||
assert tmall._is_retryable_pre_submit_failure(RuntimeError("timed out"), 1)
|
||||
assert not tmall._is_retryable_pre_submit_failure(RuntimeError("timed out"), 3)
|
||||
|
||||
|
||||
def test_tmall_strict_title_requests_a_candidate_pool_and_classifies_validation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class ValidationGenerator:
|
||||
def __init__(self, **kwargs: object) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
def generate(self, *_args: object, **_kwargs: object) -> str:
|
||||
raise tmall.TitleValidationError("no valid candidate")
|
||||
|
||||
monkeypatch.setattr(tmall, "JdVideoTitleGenerator", ValidationGenerator)
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
uploader.require_llm_titles = True
|
||||
uploader._used_titles_global = []
|
||||
uploader._catalog_product_titles = lambda _name: ("极星双肩包轻便电脑包",)
|
||||
|
||||
with pytest.raises(tmall.TmallTitleGenerationError, match="no valid candidate"):
|
||||
uploader._generate_title(["极星双肩"], video_key="video-token")
|
||||
|
||||
assert captured["candidate_count"] == tmall.TMALL_TITLE_CANDIDATE_COUNT
|
||||
assert int(captured["candidate_count"]) > 1
|
||||
assert captured["allow_verified_fallback"] is True
|
||||
|
||||
|
||||
def test_tmall_strict_title_keeps_model_request_failures_retryable(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class RequestGenerator:
|
||||
def __init__(self, **_kwargs: object) -> None:
|
||||
pass
|
||||
|
||||
def generate(self, *_args: object, **_kwargs: object) -> str:
|
||||
raise tmall.TitleGenerationError("timed out")
|
||||
|
||||
monkeypatch.setattr(tmall, "JdVideoTitleGenerator", RequestGenerator)
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
uploader.require_llm_titles = True
|
||||
uploader._used_titles_global = []
|
||||
uploader._catalog_product_titles = lambda _name: ("极星双肩包轻便电脑包",)
|
||||
|
||||
with pytest.raises(tmall.TitleGenerationError, match="timed out"):
|
||||
uploader._generate_title(["极星双肩"], video_key="video-token")
|
||||
|
||||
|
||||
def test_execute_with_no_selected_store_candidates_does_not_start_browser(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
args = SimpleNamespace(
|
||||
execute=True,
|
||||
republish_from_date=None,
|
||||
video_name=None,
|
||||
exclude_video_name=[],
|
||||
record_id=None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tmall,
|
||||
"_store_runtime",
|
||||
lambda _store: ("光影行星鑫华达专卖店:龙虾仔", "configured", "configured", Path("profile")),
|
||||
)
|
||||
monkeypatch.setattr(tmall, "select_pending_records", lambda *_args, **_kwargs: [])
|
||||
monkeypatch.setattr(
|
||||
tmall,
|
||||
"discover_tmall_publication_candidates",
|
||||
lambda *_args, **_kwargs: ([], []),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tmall,
|
||||
"_legacy_backfill_for_candidates",
|
||||
lambda *_args, **_kwargs: [],
|
||||
)
|
||||
|
||||
class UnexpectedUploader:
|
||||
def __init__(self, *_args: Any, **_kwargs: Any) -> None:
|
||||
raise AssertionError("筛选为空时不应启动浏览器")
|
||||
|
||||
monkeypatch.setattr(tmall, "GuangHeUploader", UnexpectedUploader)
|
||||
|
||||
result = tmall._run_store(
|
||||
args,
|
||||
store=tmall.STORE_LUGGAGE,
|
||||
records=[],
|
||||
luggage_style_whitelist={"极星双肩"},
|
||||
)
|
||||
|
||||
assert result == tmall.EXIT_OK
|
||||
|
||||
|
||||
class _FakeLocator:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
count: int = 1,
|
||||
on_click: Any = None,
|
||||
) -> None:
|
||||
self._count = count
|
||||
self._on_click = on_click
|
||||
|
||||
@property
|
||||
def first(self) -> _FakeLocator:
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count
|
||||
|
||||
def fill(self, _value: str) -> None:
|
||||
return None
|
||||
|
||||
def click(self, **_kwargs: Any) -> None:
|
||||
if self._on_click is not None:
|
||||
self._on_click()
|
||||
|
||||
|
||||
def test_publish_video_persists_submit_boundary_before_publish_click(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
events: list[str] = []
|
||||
upload_timeouts: list[int] = []
|
||||
uploader = object.__new__(tmall.GuangHeUploader)
|
||||
uploader.page = SimpleNamespace(url="https://creator.guanghe.taobao.com/page/")
|
||||
uploader._upload_video_file = lambda _path: None
|
||||
uploader._wait_upload_done = lambda **kwargs: (upload_timeouts.append(kwargs["timeout"]) or True)
|
||||
uploader._snapshot = lambda _name: None
|
||||
uploader._generate_title = lambda *_args, **_kwargs: "极星双肩包视频标题"
|
||||
uploader._prepare_publish_metadata = lambda _products, **_kwargs: None
|
||||
uploader._link_product = lambda _products, _item_ids: True
|
||||
|
||||
def locate(selector: str) -> _FakeLocator:
|
||||
if selector == "text=立即发布":
|
||||
return _FakeLocator(on_click=lambda: events.append("publish-click"))
|
||||
if selector == "text=发布成功":
|
||||
return _FakeLocator(count=1)
|
||||
if selector.startswith("text=") and selector not in {
|
||||
"text=内容无需标注",
|
||||
}:
|
||||
return _FakeLocator(count=0)
|
||||
return _FakeLocator(count=1)
|
||||
|
||||
uploader._loc = locate
|
||||
monkeypatch.setattr(tmall.time, "sleep", lambda _seconds: None)
|
||||
|
||||
def before_submit(payload: Any) -> None:
|
||||
assert payload["video_file_name"] == "video.mp4"
|
||||
events.append("before-submit")
|
||||
|
||||
published = uploader.publish_video(
|
||||
tmp_path / "video.mp4",
|
||||
["极星双肩"],
|
||||
before_submit=before_submit,
|
||||
)
|
||||
|
||||
assert published is True
|
||||
assert events == ["before-submit", "publish-click"]
|
||||
assert upload_timeouts == [tmall.TMALL_VIDEO_UPLOAD_TIMEOUT_SECONDS]
|
||||
|
||||
|
||||
class _SeedCursor:
|
||||
def __init__(self) -> None:
|
||||
self.executions: list[tuple[str, tuple[Any, ...]]] = []
|
||||
|
||||
def __enter__(self) -> _SeedCursor:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
def execute(self, sql: str, params: tuple[Any, ...]) -> None:
|
||||
self.executions.append((sql, params))
|
||||
|
||||
def fetchone(self) -> tuple[int]:
|
||||
return (1,)
|
||||
|
||||
|
||||
class _SeedConnection:
|
||||
def __init__(self) -> None:
|
||||
self.cursor_instance = _SeedCursor()
|
||||
|
||||
def cursor(self) -> _SeedCursor:
|
||||
return self.cursor_instance
|
||||
|
||||
|
||||
def test_legacy_seed_inserts_published_without_overwriting_conflicts() -> None:
|
||||
connection = _SeedConnection()
|
||||
candidate = _one_candidate()
|
||||
|
||||
inserted = tmall.seed_legacy_published_candidates(
|
||||
connection,
|
||||
[(candidate, {"evidence": {"publish_success_line": 72}})],
|
||||
run_id="test-run",
|
||||
)
|
||||
|
||||
assert inserted == 1
|
||||
sql, params = connection.cursor_instance.executions[0]
|
||||
normalized_sql = " ".join(sql.split())
|
||||
assert "'published'" in normalized_sql
|
||||
assert "ON CONFLICT (" in normalized_sql
|
||||
assert ") DO NOTHING RETURNING id" in normalized_sql
|
||||
assert params[-3:-1] == ("test-run", "test-run")
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import get_type_hints
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.product_commerce.jd_video_publication_db import (
|
||||
CHANNEL_TMALL,
|
||||
VIDEO_PUBLICATION_DDL,
|
||||
ClaimedVideoPublication,
|
||||
VideoPublicationCandidate,
|
||||
VideoPublicationState,
|
||||
claim_next_video_publication,
|
||||
finish_video_publication,
|
||||
finish_video_publication_rejected,
|
||||
load_video_publication_states,
|
||||
)
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, rows: list[tuple[object, ...]]) -> None:
|
||||
self.rows = rows
|
||||
self.executions: list[tuple[str, tuple[object, ...]]] = []
|
||||
self.rowcount = 1
|
||||
|
||||
def __enter__(self) -> FakeCursor:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
def execute(self, query: str, params: tuple[object, ...]) -> None:
|
||||
self.executions.append((query, params))
|
||||
|
||||
def fetchall(self) -> list[tuple[object, ...]]:
|
||||
return self.rows
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self, cursor: FakeCursor) -> None:
|
||||
self.fake_cursor = cursor
|
||||
|
||||
def cursor(self) -> FakeCursor:
|
||||
return self.fake_cursor
|
||||
|
||||
|
||||
def test_load_video_publication_states_scopes_query_and_maps_attachments() -> None:
|
||||
cursor = FakeCursor(
|
||||
[
|
||||
("rec_1", "video_1", 101, "published", False),
|
||||
("rec_2", "video_2", 102, "failed", True),
|
||||
]
|
||||
)
|
||||
|
||||
states = load_video_publication_states(
|
||||
FakeConnection(cursor),
|
||||
CHANNEL_TMALL,
|
||||
"tmall-luggage-store",
|
||||
"base_token",
|
||||
"table_id",
|
||||
)
|
||||
|
||||
assert cursor.executions[0][1] == (
|
||||
"tmall",
|
||||
"tmall-luggage-store",
|
||||
"base_token",
|
||||
"table_id",
|
||||
)
|
||||
assert "FROM fact_video_publications" in cursor.executions[0][0]
|
||||
assert states == {
|
||||
("rec_1", "video_1"): VideoPublicationState(101, "published", False),
|
||||
("rec_2", "video_2"): VideoPublicationState(102, "failed", True),
|
||||
}
|
||||
|
||||
|
||||
def test_load_video_publication_states_does_not_hide_database_errors() -> None:
|
||||
class MissingTableCursor(FakeCursor):
|
||||
def execute(self, query: str, params: tuple[object, ...]) -> None:
|
||||
raise RuntimeError("fact_video_publications does not exist")
|
||||
|
||||
with pytest.raises(RuntimeError, match="does not exist"):
|
||||
load_video_publication_states(
|
||||
FakeConnection(MissingTableCursor([])),
|
||||
CHANNEL_TMALL,
|
||||
"tmall-flagship-store",
|
||||
"base_token",
|
||||
"table_id",
|
||||
)
|
||||
|
||||
|
||||
def test_shared_ledger_preserves_missing_source_dates() -> None:
|
||||
assert get_type_hints(VideoPublicationCandidate)["source_record_date"] == date | None
|
||||
assert get_type_hints(ClaimedVideoPublication)["source_record_date"] == date | None
|
||||
assert "source_record_date DATE," in VIDEO_PUBLICATION_DDL
|
||||
assert "ALTER COLUMN source_record_date DROP NOT NULL;" in VIDEO_PUBLICATION_DDL
|
||||
claim_source = inspect.getsource(claim_next_video_publication)
|
||||
assert "source_snapshot ->> 'source_view_position'" in claim_source
|
||||
assert "source_record_date NULLS LAST" in claim_source
|
||||
|
||||
schema_path = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "db"
|
||||
/ "schema.sql"
|
||||
)
|
||||
schema = schema_path.read_text(encoding="utf-8")
|
||||
assert "source_record_date DATE," in schema
|
||||
assert "ALTER COLUMN source_record_date DROP NOT NULL;" in schema
|
||||
|
||||
|
||||
def _sample_claim() -> ClaimedVideoPublication:
|
||||
return ClaimedVideoPublication(
|
||||
publication_id=764,
|
||||
attempt_no=1,
|
||||
source_record_id="rec_1",
|
||||
source_record_date=None,
|
||||
video_file_token="video_1",
|
||||
video_file_name="video.mp4",
|
||||
cover_file_token=None,
|
||||
cover_file_name=None,
|
||||
source_snapshot={},
|
||||
publish_payload={},
|
||||
)
|
||||
|
||||
|
||||
def test_published_and_ambiguous_outcomes_are_not_retryable() -> None:
|
||||
published_cursor = FakeCursor([])
|
||||
finish_video_publication(
|
||||
FakeConnection(published_cursor),
|
||||
_sample_claim(),
|
||||
status="published",
|
||||
retryable=True,
|
||||
)
|
||||
assert published_cursor.executions[0][1][0:2] == ("published", False)
|
||||
|
||||
ambiguous_cursor = FakeCursor([])
|
||||
finish_video_publication(
|
||||
FakeConnection(ambiguous_cursor),
|
||||
_sample_claim(),
|
||||
status="ambiguous",
|
||||
retryable=True,
|
||||
)
|
||||
assert ambiguous_cursor.executions[0][1][0:2] == ("ambiguous", False)
|
||||
|
||||
|
||||
def test_clear_remote_rejection_is_retryable_but_keeps_submit_audit() -> None:
|
||||
cursor = FakeCursor([])
|
||||
finish_video_publication_rejected(
|
||||
FakeConnection(cursor),
|
||||
_sample_claim(),
|
||||
error="平台明确拒绝",
|
||||
evidence={"form_visible": True},
|
||||
)
|
||||
|
||||
params = cursor.executions[0][1]
|
||||
assert params[0:2] == ("failed", True)
|
||||
assert params[3] is True
|
||||
Reference in New Issue
Block a user