feat: complete production workflow migration
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.browser_session import (
|
||||
install_jd_dom_compatibility,
|
||||
reuse_or_run_jd_login,
|
||||
)
|
||||
|
||||
|
||||
class _AccountInput:
|
||||
def __init__(self, *, visible: bool) -> None:
|
||||
self._visible = visible
|
||||
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return int(self._visible)
|
||||
|
||||
def is_visible(self, *, timeout: int) -> bool:
|
||||
assert timeout == 1_000
|
||||
return self._visible
|
||||
|
||||
|
||||
class _Context:
|
||||
def __init__(self) -> None:
|
||||
self.scripts: list[str] = []
|
||||
|
||||
def add_init_script(self, *, script: str) -> None:
|
||||
self.scripts.append(script)
|
||||
|
||||
|
||||
def test_jd_session_adapter_reuses_authenticated_profile() -> None:
|
||||
inputs = {"shop": "account", "credential": "credential"}
|
||||
page = SimpleNamespace(
|
||||
url="https://shop.jd.com/jdm/home",
|
||||
goto=lambda *_args, **_kwargs: None,
|
||||
wait_for_timeout=lambda *_args: None,
|
||||
locator=lambda _selector: _AccountInput(visible=False),
|
||||
)
|
||||
|
||||
result = reuse_or_run_jd_login(
|
||||
page,
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
AssertionError("authenticated profile must not repeat source login")
|
||||
),
|
||||
shop=inputs["shop"],
|
||||
password=inputs["credential"],
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_jd_session_adapter_invokes_source_login_when_form_is_visible() -> None:
|
||||
calls: list[tuple[object, str, str]] = []
|
||||
inputs = {"shop": "account", "credential": "credential"}
|
||||
page = SimpleNamespace(
|
||||
url="https://shop.jd.com/jdm/home",
|
||||
goto=lambda *_args, **_kwargs: None,
|
||||
wait_for_timeout=lambda *_args: None,
|
||||
locator=lambda _selector: _AccountInput(visible=True),
|
||||
)
|
||||
|
||||
reuse_or_run_jd_login(
|
||||
page,
|
||||
lambda current_page, shop, password: calls.append(
|
||||
(current_page, shop, password)
|
||||
),
|
||||
shop=inputs["shop"],
|
||||
password=inputs["credential"],
|
||||
)
|
||||
|
||||
assert calls == [(page, "account", "credential")]
|
||||
|
||||
|
||||
def test_jd_dom_compatibility_aliases_current_and_future_pages() -> None:
|
||||
context = _Context()
|
||||
evaluated: list[str] = []
|
||||
page = SimpleNamespace(
|
||||
context=context,
|
||||
evaluate=lambda script: evaluated.append(script),
|
||||
)
|
||||
|
||||
install_jd_dom_compatibility(page)
|
||||
|
||||
assert len(context.scripts) == 1
|
||||
assert evaluated == context.scripts
|
||||
assert "jmtd-date-picker-combo" in context.scripts[0]
|
||||
assert "jmt-combo-date-picker" in context.scripts[0]
|
||||
assert "jmt-date-picker" in context.scripts[0]
|
||||
assert "classList.remove('jmt-date-picker')" in context.scripts[0]
|
||||
assert "element.dataset.eventContent === 'realtime'" in context.scripts[0]
|
||||
assert "gyxx-disabled-week-number" in context.scripts[0]
|
||||
assert "if (!document.documentElement)" in context.scripts[0]
|
||||
assert "setTimeout(installObserver, 0)" in context.scripts[0]
|
||||
assert "childList: true" in context.scripts[0]
|
||||
assert "attributes: true" not in context.scripts[0]
|
||||
assert 'data-event-content="week"' in context.scripts[0]
|
||||
assert "currentWeekTag.click()" in context.scripts[0]
|
||||
assert "__gyxxJdWeekActivationAt" in context.scripts[0]
|
||||
assert "gyxxWeekActivationAttempted" in context.scripts[0]
|
||||
assert ").find(visible)" in context.scripts[0]
|
||||
assert "gyxxWeekAdapterInstalled" in context.scripts[0]
|
||||
assert "addEventListener('mouseenter'" in context.scripts[0]
|
||||
@@ -0,0 +1,386 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
dy_store_competitor_store_scraping as dy_collector,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_data_collector,
|
||||
jd_peer_store_data_collector,
|
||||
)
|
||||
|
||||
|
||||
class _PlaywrightContext:
|
||||
def __init__(self, page: object) -> None:
|
||||
self.page = page
|
||||
|
||||
def __enter__(self):
|
||||
context = SimpleNamespace(
|
||||
new_page=lambda: self.page,
|
||||
close=lambda: None,
|
||||
)
|
||||
browser = SimpleNamespace(
|
||||
new_context=lambda **kwargs: context,
|
||||
close=lambda: None,
|
||||
)
|
||||
chromium = SimpleNamespace(launch=lambda **kwargs: browser)
|
||||
return SimpleNamespace(chromium=chromium)
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_jd_shop_collector_returns_nonzero_when_collection_fails(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"sync_playwright",
|
||||
lambda: _PlaywrightContext(object()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"step_login",
|
||||
lambda *args: (_ for _ in ()).throw(RuntimeError("page changed")),
|
||||
)
|
||||
|
||||
assert jd_data_collector.main([]) == 1
|
||||
|
||||
|
||||
def test_jd_shop_login_preserves_source_account_password_flow() -> None:
|
||||
events: list[tuple[str, object]] = []
|
||||
|
||||
class Field:
|
||||
def fill(self, value: str) -> None:
|
||||
events.append(("fill", value))
|
||||
|
||||
def click(self) -> None:
|
||||
events.append(("click", "login"))
|
||||
|
||||
fields = {
|
||||
'input[placeholder="请输入账号名/邮箱"]': Field(),
|
||||
'input[placeholder="请输入登录密码"]': Field(),
|
||||
'button:has-text("立即登录")': Field(),
|
||||
}
|
||||
page = SimpleNamespace(
|
||||
goto=lambda url, **kwargs: events.append(("goto", (url, kwargs))),
|
||||
wait_for_timeout=lambda timeout: events.append(("wait", timeout)),
|
||||
wait_for_load_state=lambda *_args, **_kwargs: None,
|
||||
locator=lambda selector: fields[selector],
|
||||
evaluate=lambda script: events.append(("evaluate", script)),
|
||||
keyboard=SimpleNamespace(
|
||||
press=lambda key: events.append(("key", key)),
|
||||
),
|
||||
)
|
||||
|
||||
assert jd_data_collector.step_login(page, "account", "credential") is None
|
||||
assert ("fill", "account") in events
|
||||
assert ("fill", "credential") in events
|
||||
assert ("click", "login") in events
|
||||
|
||||
|
||||
def test_jd_peer_collector_returns_nonzero_when_collection_fails(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"sync_playwright",
|
||||
lambda: _PlaywrightContext(object()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_login",
|
||||
lambda *args: (_ for _ in ()).throw(RuntimeError("page changed")),
|
||||
)
|
||||
|
||||
assert jd_peer_store_data_collector.main([]) == 1
|
||||
|
||||
|
||||
def test_dy_callback_preserves_failure_for_main_process(monkeypatch) -> None:
|
||||
failure = RuntimeError("date control changed")
|
||||
persisted = []
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda page: (_ for _ in ()).throw(failure),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_save_browser_state",
|
||||
lambda *args: persisted.append(args),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="date control changed"):
|
||||
dy_collector._page_automation(object())
|
||||
|
||||
assert dy_collector._automation_error is failure
|
||||
assert persisted == []
|
||||
|
||||
|
||||
def test_dy_outer_callback_persists_final_cross_domain_state(monkeypatch) -> None:
|
||||
events = []
|
||||
page = SimpleNamespace(
|
||||
url="https://fxg.jinritemai.com/ffa/eco/experience-score"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda actual_page: events.append(("source", actual_page)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_save_browser_state",
|
||||
lambda actual_page, stage: events.append(("persist", actual_page, stage)),
|
||||
)
|
||||
|
||||
dy_collector._page_automation(page)
|
||||
|
||||
assert events == [
|
||||
("source", page),
|
||||
("persist", page, "after cross-domain stages"),
|
||||
]
|
||||
assert dy_collector._automation_error is None
|
||||
|
||||
|
||||
def test_dy_final_state_snapshot_keeps_all_cross_domain_cookies(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
cookie_file = tmp_path / "cookies.json"
|
||||
storage_file = tmp_path / "storage_state.json"
|
||||
cookies = [
|
||||
{"name": "compass", "value": "one", "domain": ".jinritemai.com"},
|
||||
{"name": "qianchuan", "value": "two", "domain": ".oceanengine.com"},
|
||||
{"name": "fxg", "value": "three", "domain": "fxg.jinritemai.com"},
|
||||
]
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return cookies
|
||||
|
||||
def storage_state(self, *, path: str) -> None:
|
||||
with open(path, "w", encoding="utf-8") as stream:
|
||||
json.dump({"cookies": cookies, "origins": []}, stream)
|
||||
|
||||
monkeypatch.setattr(dy_collector, "COOKIES_FILE", str(cookie_file))
|
||||
monkeypatch.setattr(dy_collector, "STORAGE_STATE_FILE", str(storage_file))
|
||||
|
||||
dy_collector._save_browser_state(
|
||||
SimpleNamespace(context=Context()),
|
||||
"after cross-domain stages",
|
||||
)
|
||||
|
||||
saved_cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
|
||||
saved_storage = json.loads(storage_file.read_text(encoding="utf-8"))
|
||||
assert {item["domain"] for item in saved_cookies} == {
|
||||
".jinritemai.com",
|
||||
".oceanengine.com",
|
||||
"fxg.jinritemai.com",
|
||||
}
|
||||
assert saved_storage["cookies"] == cookies
|
||||
|
||||
|
||||
def test_dy_main_returns_nonzero_when_fetcher_swallows_callback_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=True),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "OUTPUT_DIR", str(tmp_path / "module"))
|
||||
monkeypatch.setattr(dy_collector, "DATA_DIR", str(tmp_path / "data"))
|
||||
monkeypatch.setattr(dy_collector, "STORE_DATA_DIR", str(tmp_path / "store"))
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"COMPETITOR_DATA_DIR",
|
||||
str(tmp_path / "competitor"),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "USER_DATA_DIR", str(tmp_path / "profile"))
|
||||
monkeypatch.setattr(dy_collector, "_load_cookies", lambda: [])
|
||||
monkeypatch.setattr(dy_collector, "enable_adaptive_fetchers", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda page: (_ for _ in ()).throw(RuntimeError("date control changed")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"save_results",
|
||||
lambda data: (_ for _ in ()).throw(
|
||||
AssertionError("failed automation must not save fallback data")
|
||||
),
|
||||
)
|
||||
|
||||
def fake_fetch(url, *, page_action, **kwargs):
|
||||
try:
|
||||
page_action(object())
|
||||
except RuntimeError:
|
||||
pass
|
||||
return SimpleNamespace(url=url, status=200)
|
||||
|
||||
monkeypatch.setattr(dy_collector.DynamicFetcher, "fetch", fake_fetch)
|
||||
|
||||
assert dy_collector.main() == 1
|
||||
|
||||
|
||||
def test_dy_main_treats_final_login_response_as_cookie_skip(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=True),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "OUTPUT_DIR", str(tmp_path / "module"))
|
||||
monkeypatch.setattr(dy_collector, "DATA_DIR", str(tmp_path / "data"))
|
||||
monkeypatch.setattr(dy_collector, "STORE_DATA_DIR", str(tmp_path / "store"))
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"COMPETITOR_DATA_DIR",
|
||||
str(tmp_path / "competitor"),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "USER_DATA_DIR", str(tmp_path / "profile"))
|
||||
monkeypatch.setattr(dy_collector, "_load_cookies", lambda: [])
|
||||
monkeypatch.setattr(dy_collector, "enable_adaptive_fetchers", lambda: None)
|
||||
monkeypatch.setattr(dy_collector, "_session_expired_exit_code", lambda: 75)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda page: (_ for _ in ()).throw(RuntimeError("navigation aborted")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"save_results",
|
||||
lambda data: (_ for _ in ()).throw(
|
||||
AssertionError("expired session must not save fallback data")
|
||||
),
|
||||
)
|
||||
|
||||
def fake_fetch(url, *, page_action, **kwargs):
|
||||
try:
|
||||
page_action(object())
|
||||
except RuntimeError:
|
||||
pass
|
||||
return SimpleNamespace(
|
||||
url="https://compass.jinritemai.com/login?roleType=shop",
|
||||
status=200,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dy_collector.DynamicFetcher, "fetch", fake_fetch)
|
||||
|
||||
assert dy_collector.main() == 75
|
||||
assert isinstance(dy_collector._automation_error, RuntimeError)
|
||||
|
||||
|
||||
def test_dy_production_keeps_source_success_after_optional_page_redirect(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "OUTPUT_DIR", str(tmp_path / "module"))
|
||||
monkeypatch.setattr(dy_collector, "DATA_DIR", str(tmp_path / "data"))
|
||||
monkeypatch.setattr(dy_collector, "STORE_DATA_DIR", str(tmp_path / "store"))
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"COMPETITOR_DATA_DIR",
|
||||
str(tmp_path / "competitor"),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "USER_DATA_DIR", str(tmp_path / "profile"))
|
||||
monkeypatch.setattr(dy_collector, "_load_cookies", lambda: [])
|
||||
monkeypatch.setattr(dy_collector, "enable_adaptive_fetchers", lambda: None)
|
||||
monkeypatch.setattr(dy_collector, "_results_saved", True)
|
||||
monkeypatch.setattr(dy_collector, "_automation_error", None)
|
||||
monkeypatch.setattr(dy_collector, "_page_automation_impl", lambda page: None)
|
||||
|
||||
def fake_fetch(url, *, page_action, **kwargs):
|
||||
page_action(SimpleNamespace(url=url))
|
||||
return SimpleNamespace(
|
||||
url="https://fxg.jinritemai.com/login/common",
|
||||
status=200,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dy_collector.DynamicFetcher, "fetch", fake_fetch)
|
||||
|
||||
assert dy_collector.main() == 0
|
||||
|
||||
|
||||
def test_dy_login_redirect_is_not_treated_as_business_page() -> None:
|
||||
assert dy_collector._is_login_page_url(
|
||||
"https://fxg.jinritemai.com/login/common"
|
||||
)
|
||||
assert not dy_collector._is_login_page_url(
|
||||
"https://compass.jinritemai.com/shop/business-part"
|
||||
)
|
||||
|
||||
|
||||
def test_dy_login_form_is_detected_when_spa_url_does_not_change() -> None:
|
||||
counts = {
|
||||
'input[name="mobile"]': 1,
|
||||
'input[name="mobilecaptcha"]': 1,
|
||||
".account-center-action-button": 1,
|
||||
}
|
||||
page = SimpleNamespace(
|
||||
locator=lambda selector: SimpleNamespace(count=lambda: counts[selector])
|
||||
)
|
||||
|
||||
assert dy_collector._has_interactive_login_form(page)
|
||||
|
||||
|
||||
def test_dy_acceptance_login_redirect_returns_cookie_skip(monkeypatch) -> None:
|
||||
recorded = []
|
||||
|
||||
class Policy:
|
||||
enabled = True
|
||||
|
||||
def record(self, event, **payload):
|
||||
recorded.append((event, payload))
|
||||
|
||||
monkeypatch.setattr(dy_collector, "current_acceptance_policy", lambda: Policy())
|
||||
|
||||
assert dy_collector._session_expired_exit_code() == 75
|
||||
assert recorded[0][0] == "cookie_skipped"
|
||||
assert recorded[0][1]["details"]["status"] == "SKIPPED_COOKIE"
|
||||
|
||||
|
||||
def test_dy_session_expiry_is_classified_by_outer_callback(monkeypatch) -> None:
|
||||
class Policy:
|
||||
enabled = True
|
||||
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: Policy(),
|
||||
)
|
||||
page = SimpleNamespace(
|
||||
url="https://compass.jinritemai.com/login?roleType=shop",
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "_save_browser_debug", lambda *args: None)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
AssertionError("expired session must not enter source business logic")
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
dy_collector.BrowserSessionExpiredError,
|
||||
match="redirected to an interactive login page",
|
||||
):
|
||||
dy_collector._page_automation(page)
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.db import db as shop_db
|
||||
|
||||
|
||||
class _Cursor:
|
||||
rowcount = 1
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _Connection:
|
||||
def __init__(self) -> None:
|
||||
self.cursor_instance = _Cursor()
|
||||
|
||||
def cursor(self) -> _Cursor:
|
||||
return self.cursor_instance
|
||||
|
||||
|
||||
def test_dynamic_batch_upsert_reports_all_input_records(monkeypatch) -> None:
|
||||
records = [
|
||||
{"week_start": "2026-07-27", "brand": "A", "rank": 1},
|
||||
{"week_start": "2026-07-27", "brand": "B", "rank": 2},
|
||||
{"week_start": "2026-07-27", "brand": "C", "rank": 3},
|
||||
]
|
||||
executed = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
shop_db,
|
||||
"_get_table_columns",
|
||||
lambda conn, table: {"week_start", "brand", "rank"},
|
||||
)
|
||||
|
||||
def fake_execute_batch(cursor, sql, data, *, page_size):
|
||||
executed.update(
|
||||
cursor=cursor,
|
||||
sql=sql,
|
||||
data=data,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(shop_db.psycopg2.extras, "execute_batch", fake_execute_batch)
|
||||
|
||||
connection = _Connection()
|
||||
persisted = shop_db._dynamic_batch_upsert(
|
||||
connection,
|
||||
"jd_peer_store_ranking",
|
||||
records,
|
||||
("week_start", "brand"),
|
||||
)
|
||||
|
||||
assert connection.cursor_instance.rowcount == 1
|
||||
assert persisted == len(records)
|
||||
assert executed["cursor"] is connection.cursor_instance
|
||||
assert executed["data"] == [
|
||||
["2026-07-27", "A", 1],
|
||||
["2026-07-27", "B", 2],
|
||||
["2026-07-27", "C", 3],
|
||||
]
|
||||
assert executed["page_size"] == 200
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,196 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
dy_store_competitor_store_scraping as collector,
|
||||
)
|
||||
|
||||
|
||||
class _Locator:
|
||||
def __init__(self, *, count: int = 1) -> None:
|
||||
self._count = count
|
||||
self.clicked = False
|
||||
self.waited = False
|
||||
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.clicked = True
|
||||
|
||||
def wait_for(self, **_kwargs) -> None:
|
||||
self.waited = True
|
||||
|
||||
|
||||
def test_market_category_controls_are_bound_to_visible_cascader() -> None:
|
||||
selectors: list[str] = []
|
||||
closed_menu = _Locator(count=0)
|
||||
trigger = _Locator()
|
||||
opened_menu = _Locator()
|
||||
|
||||
class Page:
|
||||
def locator(self, selector: str):
|
||||
selectors.append(selector)
|
||||
if selector == ".aurora-cascader-menu:visible":
|
||||
return closed_menu if selectors.count(selector) == 1 else opened_menu
|
||||
if selector == ".aurora-form-item-control .aurora-cascader:visible":
|
||||
return trigger
|
||||
raise AssertionError(selector)
|
||||
|
||||
assert collector._click_market_category_dropdown(Page())
|
||||
assert trigger.clicked
|
||||
assert opened_menu.waited
|
||||
|
||||
|
||||
def test_market_category_option_uses_visible_titled_menu_item() -> None:
|
||||
located: list[str] = []
|
||||
option = _Locator()
|
||||
page = SimpleNamespace(
|
||||
locator=lambda selector: located.append(selector) or option,
|
||||
)
|
||||
|
||||
assert collector._click_market_category_option(page, "双肩包")
|
||||
assert located == [
|
||||
'.aurora-cascader-menu '
|
||||
'li.aurora-cascader-menu-item[role="menuitemcheckbox"]'
|
||||
'[title="双肩包"]:visible'
|
||||
]
|
||||
assert option.waited
|
||||
assert option.clicked
|
||||
|
||||
|
||||
def test_select_market_category_requires_path_and_fresh_table(monkeypatch) -> None:
|
||||
expected = collector._expected_market_category_path("双肩包")
|
||||
paths = iter([collector._expected_market_category_path("腰包/胸包"), expected])
|
||||
signatures = iter(["old rows", "new rows"])
|
||||
clicked: list[str] = []
|
||||
refresh_checks: list[tuple[str, str]] = []
|
||||
page = SimpleNamespace(
|
||||
evaluate=lambda *_args, **_kwargs: None,
|
||||
wait_for_timeout=lambda *_args, **_kwargs: None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_selected_market_category_path",
|
||||
lambda _page: next(paths),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_market_table_signature",
|
||||
lambda _page: next(signatures),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_click_market_category_dropdown",
|
||||
lambda _page: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_click_market_category_option",
|
||||
lambda _page, label: clicked.append(label) or True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_wait_for_market_category_refresh",
|
||||
lambda _page, *, expected_path, previous_signature: refresh_checks.append(
|
||||
(expected_path, previous_signature)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_wait_after_action", lambda *_args, **_kwargs: None)
|
||||
|
||||
collector._select_market_category(page, "双肩包")
|
||||
|
||||
assert clicked == ["鞋靴箱包", "箱包", "时尚箱包", "双肩包"]
|
||||
assert refresh_checks == [(expected, "old rows")]
|
||||
|
||||
|
||||
def test_select_market_category_rejects_stale_table(monkeypatch) -> None:
|
||||
expected = collector._expected_market_category_path("双肩包")
|
||||
paths = iter([collector._expected_market_category_path("腰包/胸包"), expected])
|
||||
page = SimpleNamespace(
|
||||
evaluate=lambda *_args, **_kwargs: None,
|
||||
wait_for_timeout=lambda *_args, **_kwargs: None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_selected_market_category_path",
|
||||
lambda _page: next(paths),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_market_table_signature",
|
||||
lambda _page: "same rows",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_click_market_category_dropdown",
|
||||
lambda _page: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_click_market_category_option",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_wait_for_market_category_refresh",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(collector, "_wait_after_action", lambda *_args, **_kwargs: None)
|
||||
|
||||
with pytest.raises(
|
||||
collector.MarketCategorySelectionError,
|
||||
match="still contains stale rows",
|
||||
):
|
||||
collector._select_market_category(page, "双肩包")
|
||||
|
||||
|
||||
def test_market_ranking_collection_never_extracts_after_selection_failure(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
extracted: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_previous_week_range",
|
||||
lambda: (date(2026, 7, 27), date(2026, 8, 2)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_open_market_rank_shop_page",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_market_previous_week",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_market_category",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
collector.MarketCategorySelectionError("category did not change")
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_extract_market_ranking",
|
||||
lambda _page, category: extracted.append(category),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
collector.MarketCategorySelectionError,
|
||||
match="category did not change",
|
||||
):
|
||||
collector._collect_market_rankings(object())
|
||||
|
||||
assert extracted == []
|
||||
@@ -0,0 +1,359 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence import lark_cli
|
||||
from gyxx_flow.modules.shop_intelligence.config import (
|
||||
DY_PEER_BASE_FIELDS,
|
||||
DY_PEER_CAT_MAP,
|
||||
JD_PEER_BASE_FIELDS,
|
||||
JD_PEER_BRAND_MAP,
|
||||
JD_PEER_CATEGORY_MAP,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.writers import peer_store_writer
|
||||
|
||||
|
||||
def test_shop_idempotency_reads_all_pages_and_normalizes_month(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
offsets: list[int] = []
|
||||
|
||||
def fake_run(_node, _script, args, **_kwargs):
|
||||
payload = json.loads(args[-1])
|
||||
offsets.append(payload["offset"])
|
||||
if payload["offset"] == 0:
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "platform", "month"],
|
||||
"data": [["7月第5周（7.27-8.02）", ["JD"], ["2026.7月"]]],
|
||||
"record_id_list": ["other-platform"],
|
||||
"has_more": True,
|
||||
}
|
||||
}
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "platform", "month"],
|
||||
"data": [["7月第5周（7.27-8.02）", ["DY"], ["2026.7月"]]],
|
||||
"record_id_list": ["target"],
|
||||
"has_more": False,
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(lark_cli, "run_lark_cli", fake_run)
|
||||
|
||||
record_id = lark_cli.find_existing_record(
|
||||
"node",
|
||||
"script",
|
||||
"base",
|
||||
"table",
|
||||
{"时间": "time", "平台": "platform", "月份": "month"},
|
||||
"7月第5周（7.27-8.2）",
|
||||
"DY",
|
||||
month_str="2026.07月",
|
||||
)
|
||||
|
||||
assert record_id == "target"
|
||||
assert offsets == [0, 1]
|
||||
|
||||
|
||||
def test_shop_idempotency_fails_closed_when_identity_index_is_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
lark_cli,
|
||||
"run_lark_cli",
|
||||
lambda *_args, **_kwargs: {
|
||||
"data": {
|
||||
"fields": ["time", "month"],
|
||||
"data": [],
|
||||
"record_id_list": [],
|
||||
"has_more": False,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="平台"):
|
||||
lark_cli.find_existing_record(
|
||||
"node",
|
||||
"script",
|
||||
"base",
|
||||
"table",
|
||||
{"时间": "time", "平台": "platform", "月份": "month"},
|
||||
"7月第5周（7.27-8.2）",
|
||||
"DY",
|
||||
month_str="2026.07月",
|
||||
)
|
||||
|
||||
|
||||
def test_record_search_pagination_has_a_hard_page_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(lark_cli, "_MAX_RECORD_SEARCH_PAGES", 1)
|
||||
monkeypatch.setattr(
|
||||
lark_cli,
|
||||
"run_lark_cli",
|
||||
lambda *_args, **_kwargs: {
|
||||
"data": {
|
||||
"fields": ["time"],
|
||||
"data": [["7月第5周（7.27-8.2）"]],
|
||||
"record_id_list": ["record-1"],
|
||||
"has_more": True,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="安全上限"):
|
||||
lark_cli.search_record_pages(
|
||||
"node",
|
||||
"script",
|
||||
"base",
|
||||
"table",
|
||||
{
|
||||
"keyword": "7月第5周",
|
||||
"search_fields": ["time"],
|
||||
"select_fields": ["time"],
|
||||
"limit": 1,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_peer_idempotency_reads_all_pages_and_requires_identity_indexes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
offsets: list[int] = []
|
||||
|
||||
def fake_run(_node, _script, args, **_kwargs):
|
||||
payload = json.loads(args[-1])
|
||||
offsets.append(payload["offset"])
|
||||
if payload["offset"] == 0:
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "brand", "month"],
|
||||
"data": [["7月第5周（7.27-8.02）", ["other"], ["2026.7月"]]],
|
||||
"record_id_list": ["other-brand"],
|
||||
"has_more": True,
|
||||
}
|
||||
}
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "brand", "month"],
|
||||
"data": [["7月第5周（7.27-8.02）", ["NIID"], ["2026.7月"]]],
|
||||
"record_id_list": ["target"],
|
||||
"has_more": False,
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(lark_cli, "run_lark_cli", fake_run)
|
||||
|
||||
record_id = peer_store_writer._find_existing_peer_record(
|
||||
"node",
|
||||
"script",
|
||||
"table",
|
||||
{"时间": "time", "竞店名称": "brand", "月份": "month"},
|
||||
"7月第5周（7.27-8.2）",
|
||||
"NIID",
|
||||
"2026.07月",
|
||||
)
|
||||
|
||||
assert record_id == "target"
|
||||
assert offsets == [0, 1]
|
||||
|
||||
monkeypatch.setattr(
|
||||
lark_cli,
|
||||
"run_lark_cli",
|
||||
lambda *_args, **_kwargs: {
|
||||
"data": {
|
||||
"fields": ["time", "month"],
|
||||
"data": [],
|
||||
"record_id_list": [],
|
||||
"has_more": False,
|
||||
}
|
||||
},
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="竞店名称"):
|
||||
peer_store_writer._find_existing_peer_record(
|
||||
"node",
|
||||
"script",
|
||||
"table",
|
||||
{"时间": "time", "竞店名称": "brand", "月份": "month"},
|
||||
"7月第5周（7.27-8.2）",
|
||||
"NIID",
|
||||
"2026.07月",
|
||||
)
|
||||
|
||||
|
||||
def test_jd_rerun_explicitly_clears_only_na_category(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
brand_json_name = next(iter(JD_PEER_BRAND_MAP))
|
||||
category_names = list(JD_PEER_CATEGORY_MAP)
|
||||
artifact = tmp_path / "jd-peer.json"
|
||||
artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"数据周期": "2026-07-27 ~ 2026-08-02",
|
||||
"数据": {
|
||||
brand_json_name: {
|
||||
category_names[0]: {"rank": "N/A"},
|
||||
category_names[1]: {"rank": "7"},
|
||||
category_names[2]: {"rank": "3"},
|
||||
}
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
writes: list[dict] = []
|
||||
monkeypatch.setattr(peer_store_writer, "validate_peer_payload", lambda *_args: (True, ""))
|
||||
monkeypatch.setattr(peer_store_writer, "resolve_lark_cli", lambda: ("node", "script"))
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_month_options_cache",
|
||||
lambda *_args: {"2026.07月": "2026.07月"},
|
||||
)
|
||||
monkeypatch.setattr(peer_store_writer, "_find_existing_peer_record", lambda *_args: "existing")
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_upsert_peer_record",
|
||||
lambda *_args: writes.append(_args[-1]) or {"data": {"record_id_list": ["existing"]}},
|
||||
)
|
||||
|
||||
assert peer_store_writer._write_jd_peer_store(str(artifact)) is True
|
||||
assert len(writes) == 1
|
||||
assert writes[0][JD_PEER_CATEGORY_MAP[category_names[0]]] is None
|
||||
assert writes[0][JD_PEER_CATEGORY_MAP[category_names[1]]] == 7
|
||||
assert writes[0][JD_PEER_CATEGORY_MAP[category_names[2]]] == 3
|
||||
|
||||
|
||||
def test_dy_brand_iteration_uses_both_category_brand_sets(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "dy-peer.json"
|
||||
artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"date_range": "2026/07/27 - 2026/08/02",
|
||||
"results": {
|
||||
DY_PEER_CAT_MAP["斜挎包"]: {"alpha": {"rank": "11"}},
|
||||
DY_PEER_CAT_MAP["双肩包"]: {"beta": {"rank": "22"}},
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
writes: list[dict] = []
|
||||
monkeypatch.setattr(peer_store_writer, "validate_peer_payload", lambda *_args: (True, ""))
|
||||
monkeypatch.setattr(peer_store_writer, "resolve_lark_cli", lambda: ("node", "script"))
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_brand_options_cache",
|
||||
lambda *_args: {"alpha": "Alpha", "beta": "Beta"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_month_options_cache",
|
||||
lambda *_args: {"2026.07月": "2026.07月"},
|
||||
)
|
||||
monkeypatch.setattr(peer_store_writer, "_find_existing_peer_record", lambda *_args: "existing")
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_upsert_peer_record",
|
||||
lambda *_args: writes.append(_args[-1]) or {"data": {"record_id_list": ["existing"]}},
|
||||
)
|
||||
|
||||
assert peer_store_writer._write_dy_peer_store(str(artifact)) is True
|
||||
assert [row[DY_PEER_BASE_FIELDS["竞店名称"]] for row in writes] == [
|
||||
"Alpha",
|
||||
"Beta",
|
||||
]
|
||||
|
||||
|
||||
def test_dy_local_shop_validation_failure_propagates_false(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
peer_artifact = tmp_path / "dy-peer.json"
|
||||
peer_artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"date_range": "2026/07/27 - 2026/08/02",
|
||||
"results": {
|
||||
DY_PEER_CAT_MAP["斜挎包"]: {"alpha": {"rank": "11"}},
|
||||
DY_PEER_CAT_MAP["双肩包"]: {"alpha": {"rank": "22"}},
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
shop_artifact = tmp_path / "dy-shop.json"
|
||||
shop_artifact.write_text("{}", encoding="utf-8")
|
||||
monkeypatch.setattr(peer_store_writer, "validate_peer_payload", lambda *_args: (True, ""))
|
||||
monkeypatch.setattr(peer_store_writer, "resolve_lark_cli", lambda: ("node", "script"))
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_brand_options_cache",
|
||||
lambda *_args: {"alpha": "Alpha"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_month_options_cache",
|
||||
lambda *_args: {"2026.07月": "2026.07月"},
|
||||
)
|
||||
monkeypatch.setattr(peer_store_writer, "_find_existing_peer_record", lambda *_args: "existing")
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_upsert_peer_record",
|
||||
lambda *_args: {"data": {"record_id_list": ["existing"]}},
|
||||
)
|
||||
monkeypatch.setattr(peer_store_writer, "_write_dy_local_shop_row", lambda *_args: False)
|
||||
|
||||
assert (
|
||||
peer_store_writer._write_dy_peer_store(
|
||||
str(peer_artifact),
|
||||
str(shop_artifact),
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_dy_local_shop_rankings_return_false_without_upsert(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "shop.json"
|
||||
artifact.write_text(json.dumps({"market_rankings": {}}), encoding="utf-8")
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"validate_market_rankings",
|
||||
lambda *_args: (False, "missing rankings"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_upsert_peer_record",
|
||||
lambda *_args: pytest.fail("invalid local rankings must not be written"),
|
||||
)
|
||||
|
||||
assert (
|
||||
peer_store_writer._write_dy_local_shop_row(
|
||||
"node",
|
||||
"script",
|
||||
"7月第5周（7.27-8.2）",
|
||||
"2026.07月",
|
||||
str(artifact),
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_configured_peer_identity_fields_are_non_empty() -> None:
|
||||
for fields in (JD_PEER_BASE_FIELDS, DY_PEER_BASE_FIELDS):
|
||||
assert fields["时间"]
|
||||
assert fields["月份"]
|
||||
assert fields["竞店名称"]
|
||||
@@ -0,0 +1,257 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_peer_store_data_collector as collector,
|
||||
)
|
||||
|
||||
|
||||
class _HiddenMenuLocator:
|
||||
def count(self) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
class _Page:
|
||||
def __init__(self) -> None:
|
||||
self.keyboard = SimpleNamespace(press=lambda _key: None)
|
||||
|
||||
def locator(self, _selector: str) -> _HiddenMenuLocator:
|
||||
return _HiddenMenuLocator()
|
||||
|
||||
def wait_for_timeout(self, _timeout_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_category_path_normalization_is_exact() -> None:
|
||||
assert collector._normalize_category_path(" 男包 → 男士双肩包 ") == (
|
||||
"男包 > 男士双肩包"
|
||||
)
|
||||
assert collector._expected_category_path("男士双肩包", "男包") == (
|
||||
"男包 > 男士双肩包"
|
||||
)
|
||||
assert collector._expected_category_path("男包", None) == "男包"
|
||||
|
||||
|
||||
def test_category_menu_closes_stale_portals_before_opening_selector() -> None:
|
||||
class Menus:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
self.first = self
|
||||
|
||||
def count(self) -> int:
|
||||
return self.page.menu_count
|
||||
|
||||
def wait_for(self, **_kwargs) -> None:
|
||||
return None
|
||||
|
||||
class Trigger:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.page.menu_count = 1
|
||||
|
||||
class Selector:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def locator(self, _selector: str) -> Trigger:
|
||||
return Trigger(self.page)
|
||||
|
||||
class Page:
|
||||
def __init__(self) -> None:
|
||||
self.menu_count = 2
|
||||
self.escapes = 0
|
||||
self.keyboard = SimpleNamespace(press=self.press)
|
||||
|
||||
def press(self, key: str) -> None:
|
||||
assert key == "Escape"
|
||||
self.escapes += 1
|
||||
self.menu_count = 0
|
||||
|
||||
def locator(self, _selector: str) -> Menus:
|
||||
return Menus(self)
|
||||
|
||||
def wait_for_timeout(self, _timeout_ms: int) -> None:
|
||||
return None
|
||||
|
||||
page = Page()
|
||||
|
||||
menu = collector._visible_category_menu(page, Selector(page))
|
||||
|
||||
assert menu.count() == 1
|
||||
assert page.escapes == 1
|
||||
|
||||
|
||||
def test_category_menu_targets_only_outer_dropdown_panel() -> None:
|
||||
source = __import__("inspect").getsource(collector._visible_category_menu)
|
||||
|
||||
assert ".jmtd-dropdown-panel.jmtd-dropdown-lists:visible" in source
|
||||
|
||||
|
||||
def test_successful_category_switch_closes_its_dropdown_portal(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
class Menus:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def count(self) -> int:
|
||||
return self.page.menu_count
|
||||
|
||||
class Trigger:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.page.menu_count = 0
|
||||
|
||||
class Selector:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def locator(self, _selector: str) -> Trigger:
|
||||
return Trigger(self.page)
|
||||
|
||||
class Page:
|
||||
def __init__(self) -> None:
|
||||
self.menu_count = 1
|
||||
self.keyboard = SimpleNamespace(press=lambda _key: None)
|
||||
|
||||
def locator(self, _selector: str) -> Menus:
|
||||
return Menus(self)
|
||||
|
||||
def wait_for_timeout(self, _timeout_ms: int) -> None:
|
||||
return None
|
||||
|
||||
page = Page()
|
||||
selector = Selector(page)
|
||||
monkeypatch.setattr(collector, "_visible_category_selector", lambda _page: selector)
|
||||
monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "before")
|
||||
monkeypatch.setattr(
|
||||
collector, "_selected_category_path", lambda _selector: "男包 > 男士双肩包"
|
||||
)
|
||||
|
||||
assert collector.switch_category(page, "男士双肩包", "男包") is True
|
||||
assert page.menu_count == 0
|
||||
|
||||
|
||||
def test_ranking_refresh_accepts_only_explicit_empty_state(monkeypatch) -> None:
|
||||
monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "")
|
||||
monkeypatch.setattr(
|
||||
collector, "_ranking_table_is_explicitly_empty", lambda _page: True
|
||||
)
|
||||
|
||||
assert collector._wait_for_ranking_table_refresh(
|
||||
object(), "previous", timeout_ms=1
|
||||
) is True
|
||||
|
||||
|
||||
def test_ranking_refresh_rejects_unverified_empty_state(monkeypatch) -> None:
|
||||
monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "")
|
||||
monkeypatch.setattr(
|
||||
collector, "_ranking_table_is_explicitly_empty", lambda _page: False
|
||||
)
|
||||
monkeypatch.setattr(collector.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert collector._wait_for_ranking_table_refresh(
|
||||
object(), "previous", timeout_ms=0
|
||||
) is False
|
||||
|
||||
|
||||
def test_switch_category_requires_selected_state_and_table_refresh(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
page = _Page()
|
||||
selector = object()
|
||||
menu = object()
|
||||
observed: list[object] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_visible_category_selector",
|
||||
lambda actual_page: selector if actual_page is page else None,
|
||||
)
|
||||
monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "before")
|
||||
monkeypatch.setattr(collector, "_selected_category_path", lambda _selector: "男包")
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_visible_category_menu",
|
||||
lambda actual_page, actual_selector: (
|
||||
menu if (actual_page, actual_selector) == (page, selector) else None
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_category_option",
|
||||
lambda actual_page, actual_menu, category, parent: observed.append(
|
||||
(actual_page, actual_menu, category, parent)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_wait_for_selected_category",
|
||||
lambda actual_selector, expected: (
|
||||
observed.append((actual_selector, expected, "selected")) or True
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_wait_for_ranking_table_refresh",
|
||||
lambda actual_page, previous: (
|
||||
observed.append((actual_page, previous, "refreshed")) or True
|
||||
),
|
||||
)
|
||||
|
||||
assert collector.switch_category(page, "男士双肩包", "男包") is True
|
||||
assert observed == [
|
||||
(page, menu, "男士双肩包", "男包"),
|
||||
(selector, "男包 > 男士双肩包", "selected"),
|
||||
(page, "before", "refreshed"),
|
||||
]
|
||||
|
||||
|
||||
def test_failed_category_switch_never_extracts_previous_table(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
page = _Page()
|
||||
extracted: list[str] = []
|
||||
|
||||
monkeypatch.setattr(collector, "SHOPS", ["peer-shop"])
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"CATEGORIES",
|
||||
[("男士双肩包", "男包"), ("男士单肩/斜挎包", "男包")],
|
||||
)
|
||||
monkeypatch.setattr(collector, "select_last_week_any_day", lambda _page: None)
|
||||
monkeypatch.setattr(collector, "search_shop", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"switch_category",
|
||||
lambda _page, category, _parent: category == "男士单肩/斜挎包",
|
||||
)
|
||||
|
||||
def extract_rows(_page):
|
||||
extracted.append("called")
|
||||
return [
|
||||
{
|
||||
"rank": "7",
|
||||
"shop": "peer-shop",
|
||||
"amount": "￥1万~￥2万",
|
||||
"orders": "10~50",
|
||||
"followers": "3",
|
||||
"visitors": "1,000~2,000",
|
||||
"searchClicks": "600~800",
|
||||
}
|
||||
]
|
||||
|
||||
monkeypatch.setattr(collector, "extract_ranking_table", extract_rows)
|
||||
|
||||
result = collector.step_collect_competitor_data(page)["peer-shop"]
|
||||
|
||||
assert result["男士双肩包"]["rank"] == "N/A"
|
||||
assert "禁止复用" not in result["男士双肩包"]["error"]
|
||||
assert "类目切换或榜单刷新失败" in result["男士双肩包"]["error"]
|
||||
assert result["男士单肩/斜挎包"]["rank"] == "7"
|
||||
assert extracted == ["called"]
|
||||
@@ -0,0 +1,225 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_self_operated_brand_daily as brand_daily,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_brand_daily import (
|
||||
browser_runtime_options,
|
||||
build_db_records,
|
||||
is_authenticated_shop_url,
|
||||
iter_dates,
|
||||
parse_decimal,
|
||||
parse_percent,
|
||||
resolve_account_identifier,
|
||||
)
|
||||
|
||||
|
||||
def test_browser_runtime_defaults_to_bundled_chromium(monkeypatch):
|
||||
monkeypatch.delenv("GYXX_BROWSER_CHANNEL", raising=False)
|
||||
|
||||
assert browser_runtime_options() == {}
|
||||
|
||||
|
||||
def test_browser_runtime_can_use_installed_chrome(monkeypatch):
|
||||
monkeypatch.setenv("GYXX_BROWSER_CHANNEL", "chrome")
|
||||
|
||||
assert browser_runtime_options() == {"channel": "chrome"}
|
||||
|
||||
|
||||
class DateRangeTests(unittest.TestCase):
|
||||
def test_iter_dates_is_inclusive(self):
|
||||
self.assertEqual(
|
||||
list(iter_dates(date(2026, 6, 29), date(2026, 7, 2))),
|
||||
[
|
||||
date(2026, 6, 29),
|
||||
date(2026, 6, 30),
|
||||
date(2026, 7, 1),
|
||||
date(2026, 7, 2),
|
||||
],
|
||||
)
|
||||
|
||||
def test_iter_dates_rejects_reverse_range(self):
|
||||
with self.assertRaisesRegex(ValueError, "start_date"):
|
||||
list(iter_dates(date(2026, 7, 2), date(2026, 7, 1)))
|
||||
|
||||
|
||||
class LoginStateTests(unittest.TestCase):
|
||||
def test_shop_page_is_authenticated(self):
|
||||
self.assertTrue(
|
||||
is_authenticated_shop_url(
|
||||
"https://shop.jd.com/jdm/vc/data-management/performance/brandPerformance"
|
||||
)
|
||||
)
|
||||
|
||||
def test_passport_and_security_pages_are_not_authenticated(self):
|
||||
self.assertFalse(
|
||||
is_authenticated_shop_url(
|
||||
"https://passport.shop.jd.com/login/index.action/jdm"
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
is_authenticated_shop_url("https://aq.jd.com/certified/index?p=abc")
|
||||
)
|
||||
|
||||
|
||||
class _FakeLocator:
|
||||
def __init__(self, value=None):
|
||||
self.value = value
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return int(self.value is not None)
|
||||
|
||||
def get_attribute(self, name):
|
||||
assert name == "title"
|
||||
return self.value
|
||||
|
||||
def inner_text(self):
|
||||
return self.value or ""
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self, values, cookies=None):
|
||||
self.values = values
|
||||
self.context = SimpleNamespace(cookies=lambda: cookies or [])
|
||||
|
||||
def locator(self, selector):
|
||||
return _FakeLocator(self.values.get(selector))
|
||||
|
||||
|
||||
class AccountIdentifierTests(unittest.TestCase):
|
||||
def test_configured_account_takes_priority(self):
|
||||
self.assertEqual(
|
||||
resolve_account_identifier(_FakePage({}), " gyxx2022 "),
|
||||
"gyxx2022",
|
||||
)
|
||||
|
||||
def test_cookie_session_uses_signed_in_pin(self):
|
||||
page = _FakePage(
|
||||
{
|
||||
".shop-menu-accountV1__account-dropdown .content-pin[title]": (
|
||||
"gyxx2022"
|
||||
),
|
||||
".shop-menu-accountV1__right-account-top-name[title]": (
|
||||
"惠州鑫华达科技有限公司"
|
||||
),
|
||||
}
|
||||
)
|
||||
self.assertEqual(resolve_account_identifier(page, ""), "gyxx2022")
|
||||
|
||||
def test_cookie_session_falls_back_to_company_name(self):
|
||||
page = _FakePage(
|
||||
{
|
||||
".shop-menu-accountV1__right-account-top-name[title]": (
|
||||
"惠州鑫华达科技有限公司"
|
||||
)
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
resolve_account_identifier(page, ""),
|
||||
"惠州鑫华达科技有限公司",
|
||||
)
|
||||
|
||||
def test_cookie_session_uses_jd_pin_cookie_without_account_menu(self):
|
||||
page = _FakePage(
|
||||
{},
|
||||
cookies=[
|
||||
{
|
||||
"name": "pin",
|
||||
"value": "gyxx%32%30%32%32",
|
||||
"domain": ".jd.com",
|
||||
}
|
||||
],
|
||||
)
|
||||
self.assertEqual(resolve_account_identifier(page, ""), "gyxx2022")
|
||||
|
||||
def test_cookie_session_requires_stable_identifier(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "无法识别当前京麦账号"):
|
||||
resolve_account_identifier(_FakePage({}), "")
|
||||
|
||||
|
||||
class ValueParsingTests(unittest.TestCase):
|
||||
def test_parse_decimal_handles_commas_and_currency(self):
|
||||
self.assertEqual(parse_decimal("¥34,605.80"), Decimal("34605.80"))
|
||||
self.assertEqual(parse_decimal("122.00"), Decimal("122.00"))
|
||||
|
||||
def test_parse_decimal_preserves_negative_values(self):
|
||||
self.assertEqual(parse_decimal("-2,173.20"), Decimal("-2173.20"))
|
||||
|
||||
def test_parse_decimal_returns_none_for_placeholder(self):
|
||||
for value in (None, "", "-", "--", "—", "N/A"):
|
||||
self.assertIsNone(parse_decimal(value))
|
||||
|
||||
def test_parse_percent_converts_display_percent_to_ratio(self):
|
||||
self.assertEqual(parse_percent("6.25%"), Decimal("0.0625"))
|
||||
self.assertEqual(parse_percent("0.06"), Decimal("0.06"))
|
||||
|
||||
|
||||
class RecordBuildingTests(unittest.TestCase):
|
||||
def test_build_db_records_maps_all_columns(self):
|
||||
rows = [
|
||||
{
|
||||
"品牌": "光影行星（GYXX）",
|
||||
"财务销量": "122.00",
|
||||
"收入": "34,605.80",
|
||||
"毛利": "2,173.20",
|
||||
"毛利率": "6.00%",
|
||||
"库存金额": "586,149.83",
|
||||
}
|
||||
]
|
||||
|
||||
records = build_db_records(
|
||||
account="gyxx2022",
|
||||
data_date=date(2026, 7, 29),
|
||||
rows=rows,
|
||||
source_url="https://shop.jd.com/example",
|
||||
)
|
||||
|
||||
self.assertEqual(len(records), 1)
|
||||
record = records[0]
|
||||
self.assertEqual(record["account"], "gyxx2022")
|
||||
self.assertEqual(record["data_date"], date(2026, 7, 29))
|
||||
self.assertEqual(record["brand"], "光影行星（GYXX）")
|
||||
self.assertEqual(record["financial_sales"], Decimal("122.00"))
|
||||
self.assertEqual(record["revenue"], Decimal("34605.80"))
|
||||
self.assertEqual(record["gross_profit"], Decimal("2173.20"))
|
||||
self.assertEqual(record["gross_margin"], Decimal("0.06"))
|
||||
self.assertEqual(record["inventory_amount"], Decimal("586149.83"))
|
||||
self.assertEqual(record["raw_row"]["收入"], "34,605.80")
|
||||
|
||||
def test_build_db_records_skips_empty_placeholder_row(self):
|
||||
records = build_db_records(
|
||||
account="gyxx2022",
|
||||
data_date=date(2026, 7, 29),
|
||||
rows=[{"品牌": "-", "收入": "-"}],
|
||||
source_url="https://shop.jd.com/example",
|
||||
)
|
||||
self.assertEqual(records, [])
|
||||
|
||||
|
||||
def test_acceptance_preflight_skips_before_argument_parsing(monkeypatch):
|
||||
class FakePolicy:
|
||||
enabled = True
|
||||
|
||||
def preflight_cookie(self, binding, *, environment):
|
||||
return SimpleNamespace(should_skip=True, reason="browser state is missing")
|
||||
|
||||
monkeypatch.setattr(brand_daily, "current_acceptance_policy", lambda: FakePolicy())
|
||||
monkeypatch.setattr(brand_daily, "binding_from_environment", lambda: "binding")
|
||||
monkeypatch.setattr(
|
||||
brand_daily,
|
||||
"parse_args",
|
||||
lambda argv: (_ for _ in ()).throw(
|
||||
AssertionError("arguments must not be parsed after a cookie skip")
|
||||
),
|
||||
)
|
||||
|
||||
assert brand_daily.main([]) == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,108 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_self_operated_product_daily as product_daily,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_product_daily import (
|
||||
build_product_db_records,
|
||||
parse_total_count,
|
||||
validate_collected_rows,
|
||||
)
|
||||
|
||||
|
||||
class TotalCountTests(unittest.TestCase):
|
||||
def test_parse_total_count(self):
|
||||
self.assertEqual(parse_total_count("共50条"), 50)
|
||||
self.assertEqual(parse_total_count("共 1,234 条"), 1234)
|
||||
|
||||
def test_parse_total_count_rejects_unknown_text(self):
|
||||
with self.assertRaisesRegex(ValueError, "总条数"):
|
||||
parse_total_count("暂无数据")
|
||||
|
||||
|
||||
class ProductRecordTests(unittest.TestCase):
|
||||
def test_maps_all_product_columns(self):
|
||||
rows = [
|
||||
{
|
||||
"商品编码": "100232520894",
|
||||
"商品名称": "光影行星测试商品",
|
||||
"财务销量": "1.00",
|
||||
"收入": "145.14",
|
||||
"成本": "196.60",
|
||||
"毛利": "-51.46",
|
||||
"优惠券抵减数": "50.68",
|
||||
"积分抵减数": "0",
|
||||
"满返满减额": "0",
|
||||
"库存周转天数": "70.55",
|
||||
}
|
||||
]
|
||||
|
||||
records = build_product_db_records(
|
||||
account="gyxx2022",
|
||||
data_date=date(2026, 7, 30),
|
||||
rows=rows,
|
||||
source_url="https://shop.jd.com/example",
|
||||
)
|
||||
|
||||
self.assertEqual(len(records), 1)
|
||||
record = records[0]
|
||||
self.assertEqual(record["product_code"], "100232520894")
|
||||
self.assertEqual(record["product_name"], "光影行星测试商品")
|
||||
self.assertEqual(record["financial_sales"], Decimal("1.00"))
|
||||
self.assertEqual(record["revenue"], Decimal("145.14"))
|
||||
self.assertEqual(record["cost"], Decimal("196.60"))
|
||||
self.assertEqual(record["gross_profit"], Decimal("-51.46"))
|
||||
self.assertEqual(record["coupon_deduction"], Decimal("50.68"))
|
||||
self.assertEqual(record["points_deduction"], Decimal("0"))
|
||||
self.assertEqual(record["promotion_deduction"], Decimal("0"))
|
||||
self.assertEqual(record["inventory_turnover_days"], Decimal("70.55"))
|
||||
|
||||
def test_skips_row_without_product_code(self):
|
||||
self.assertEqual(
|
||||
build_product_db_records(
|
||||
"gyxx2022",
|
||||
date(2026, 7, 30),
|
||||
[{"商品编码": "-", "商品名称": "占位行"}],
|
||||
"https://shop.jd.com/example",
|
||||
),
|
||||
[],
|
||||
)
|
||||
|
||||
|
||||
class PaginationValidationTests(unittest.TestCase):
|
||||
def test_accepts_complete_unique_rows(self):
|
||||
rows = [{"商品编码": "1"}, {"商品编码": "2"}]
|
||||
self.assertEqual(validate_collected_rows(rows, 2), rows)
|
||||
|
||||
def test_rejects_duplicate_product_codes(self):
|
||||
rows = [{"商品编码": "1"}, {"商品编码": "1"}]
|
||||
with self.assertRaisesRegex(RuntimeError, "重复"):
|
||||
validate_collected_rows(rows, 2)
|
||||
|
||||
def test_rejects_missing_rows(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "总条数"):
|
||||
validate_collected_rows([{"商品编码": "1"}], 2)
|
||||
|
||||
|
||||
def test_acceptance_preflight_skips_before_argument_parsing(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
product_daily,
|
||||
"preflight_acceptance_login",
|
||||
lambda: COOKIE_SKIP_EXIT_CODE,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
product_daily,
|
||||
"parse_args",
|
||||
lambda argv: (_ for _ in ()).throw(
|
||||
AssertionError("arguments must not be parsed after a cookie skip")
|
||||
),
|
||||
)
|
||||
|
||||
assert product_daily.main([]) == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,517 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.catalog import WorkflowCatalog
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule
|
||||
from gyxx_flow.modules.shop_intelligence.config import DY_PLATFORM, JD_PLATFORM
|
||||
from gyxx_flow.modules.shop_intelligence.db import db as shop_db
|
||||
from gyxx_flow.modules.shop_intelligence.quality import validate_peer_payload
|
||||
|
||||
run_peer_store = import_module(
|
||||
"gyxx_flow.modules.shop_intelligence.runners.run_peer_store"
|
||||
)
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
JD_BRANDS = (
|
||||
"Bellroy官方旗舰店",
|
||||
"tomtoc官方旗舰店",
|
||||
"探迹者TAJEZZO官方旗舰店",
|
||||
"BAGSMART旗舰店",
|
||||
"GROTTO个乐官方旗舰店",
|
||||
"NIID箱包旗舰店",
|
||||
"光影行星 GYXX官方旗舰店",
|
||||
)
|
||||
JD_CATEGORIES = (
|
||||
"男士包袋",
|
||||
"男士双肩包",
|
||||
"男士单肩/斜挎包",
|
||||
"电脑包",
|
||||
"男士腰包/胸包",
|
||||
"旅行包",
|
||||
"休闲运动包",
|
||||
)
|
||||
DY_BRANDS = (
|
||||
"Bellroy",
|
||||
"tomtoc",
|
||||
"BAGSMART",
|
||||
"GROTTO",
|
||||
"NIID",
|
||||
"reeyee",
|
||||
"VAOPER",
|
||||
"探迹者",
|
||||
)
|
||||
DY_CATEGORIES = ("单肩包/斜挎包", "双肩包", "腰包/胸包")
|
||||
|
||||
|
||||
def _jd_ranked_entry(brand: str, seed: int) -> dict:
|
||||
return {
|
||||
"rank": str(seed + 1),
|
||||
"shop": brand,
|
||||
"amount": f"￥{seed + 10}万~￥{seed + 20}万",
|
||||
"orders": f"{seed + 20}~{seed + 30}",
|
||||
"followers": str(seed + 40),
|
||||
"visitors": f"{seed + 50}~{seed + 60}",
|
||||
"searchClicks": str(seed + 70),
|
||||
}
|
||||
|
||||
|
||||
def _jd_unranked_entry(brand: str) -> dict:
|
||||
return {
|
||||
"rank": "N/A",
|
||||
"amount": "N/A",
|
||||
"orders": "N/A",
|
||||
"followers": "N/A",
|
||||
"visitors": "N/A",
|
||||
"searchClicks": "N/A",
|
||||
"error": f"当前类目榜单未找到店铺: {brand}",
|
||||
}
|
||||
|
||||
|
||||
def _jd_payload() -> dict:
|
||||
return {
|
||||
"数据": {
|
||||
brand: {
|
||||
category: _jd_ranked_entry(brand, brand_index * 20 + category_index)
|
||||
for category_index, category in enumerate(JD_CATEGORIES)
|
||||
}
|
||||
for brand_index, brand in enumerate(JD_BRANDS)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _dy_range(seed: int) -> dict:
|
||||
return {
|
||||
"min": float(seed),
|
||||
"max": float(seed + 1),
|
||||
"open_ended": False,
|
||||
"raw_values": [str(seed)],
|
||||
}
|
||||
|
||||
|
||||
def _dy_entry(
|
||||
brand: str,
|
||||
category: str,
|
||||
seed: int,
|
||||
*,
|
||||
ranked: bool = True,
|
||||
matched: bool = True,
|
||||
) -> dict:
|
||||
rank = str(seed + 1) if ranked else ""
|
||||
stores = (
|
||||
[
|
||||
{
|
||||
"shop_name": f"{brand}旗舰店",
|
||||
"rank": rank,
|
||||
"row_text": f"{brand}-{seed}",
|
||||
}
|
||||
]
|
||||
if matched
|
||||
else []
|
||||
)
|
||||
return {
|
||||
"brand": brand,
|
||||
"category": category,
|
||||
"matched_shop_count": len(stores),
|
||||
"rank": rank,
|
||||
"rank_display": rank or "未进入前200名",
|
||||
"payment_amount": _dy_range(seed + 10),
|
||||
"order_count": _dy_range(seed + 20),
|
||||
"product_click_users": _dy_range(seed + 30),
|
||||
"stores": stores,
|
||||
"captured_at": "2026-08-04T10:00:00",
|
||||
}
|
||||
|
||||
|
||||
def _dy_payload() -> dict:
|
||||
return {
|
||||
"categories": list(DY_CATEGORIES),
|
||||
"brands": list(DY_BRANDS),
|
||||
"results": {
|
||||
category: {
|
||||
brand: _dy_entry(brand, category, category_index * 100 + brand_index)
|
||||
for brand_index, brand in enumerate(DY_BRANDS)
|
||||
}
|
||||
for category_index, category in enumerate(DY_CATEGORIES)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_runner_rejects_success_without_fresh_peer_artifact(monkeypatch) -> None:
|
||||
marker = ("D:/old-peer.json", 1, 10)
|
||||
monkeypatch.setattr(run_peer_store, "_artifact_marker", lambda _platform: marker)
|
||||
|
||||
result = run_peer_store._run_with_fresh_artifact(
|
||||
JD_PLATFORM,
|
||||
lambda: (JD_PLATFORM, 0, "collector output"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "no fresh peer-store file" in result[2]
|
||||
|
||||
|
||||
def test_runner_passes_only_the_fresh_peer_artifact(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "new-peer.json"
|
||||
artifact.write_text(
|
||||
json.dumps(_jd_payload(), ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
marker = (str(artifact), artifact.stat().st_mtime_ns, artifact.stat().st_size)
|
||||
markers = iter((None, marker))
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_artifact_marker",
|
||||
lambda _platform: next(markers),
|
||||
)
|
||||
|
||||
result = run_peer_store._run_with_fresh_artifact(
|
||||
JD_PLATFORM,
|
||||
lambda: (JD_PLATFORM, 0, "ok"),
|
||||
)
|
||||
|
||||
assert result == (JD_PLATFORM, 0, "ok", str(artifact))
|
||||
|
||||
|
||||
def test_runner_rejects_identical_jd_payload_copied_across_categories(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "bad-peer.json"
|
||||
payload = _jd_payload()
|
||||
repeated = _jd_ranked_entry(JD_BRANDS[0], 22)
|
||||
for category in JD_CATEGORIES:
|
||||
payload["数据"][JD_BRANDS[0]][category] = repeated
|
||||
artifact.write_text(
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
marker = (str(artifact), artifact.stat().st_mtime_ns, artifact.stat().st_size)
|
||||
markers = iter((None, marker))
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_artifact_marker",
|
||||
lambda _platform: next(markers),
|
||||
)
|
||||
|
||||
result = run_peer_store._run_with_fresh_artifact(
|
||||
JD_PLATFORM,
|
||||
lambda: (JD_PLATFORM, 0, "ok"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "identical payload copied across categories" in result[2]
|
||||
|
||||
|
||||
def test_quality_gate_allows_one_unranked_brand_when_other_brands_differ() -> None:
|
||||
payload = _jd_payload()
|
||||
brand = JD_BRANDS[0]
|
||||
payload["数据"][brand] = {
|
||||
category: _jd_unranked_entry(brand) for category in JD_CATEGORIES
|
||||
}
|
||||
|
||||
assert validate_peer_payload(JD_PLATFORM, payload) == (True, "")
|
||||
|
||||
|
||||
def test_quality_gate_rejects_explicit_category_switch_error() -> None:
|
||||
payload = _jd_payload()
|
||||
payload["数据"][JD_BRANDS[0]][JD_CATEGORIES[1]] = {
|
||||
**_jd_unranked_entry(JD_BRANDS[0]),
|
||||
"error": "类目切换或榜单刷新失败",
|
||||
}
|
||||
|
||||
valid, reason = validate_peer_payload(JD_PLATFORM, payload)
|
||||
|
||||
assert valid is False
|
||||
assert "类目切换或榜单刷新失败" in reason
|
||||
|
||||
|
||||
def test_direct_jd_peer_persistence_rejects_repeated_payload_before_connecting(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "bad-peer-direct.json"
|
||||
payload = _jd_payload()
|
||||
repeated = _jd_ranked_entry(JD_BRANDS[-2], 22)
|
||||
for category in JD_CATEGORIES:
|
||||
payload["数据"][JD_BRANDS[-2]][category] = repeated
|
||||
artifact.write_text(
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(shop_db, "DB_ENABLED", True)
|
||||
monkeypatch.setattr(
|
||||
shop_db,
|
||||
"get_conn",
|
||||
lambda: pytest.fail("invalid payload must not connect to PostgreSQL"),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="JD peer-store data quality check failed"):
|
||||
shop_db.persist_jd_peer_store(str(artifact))
|
||||
|
||||
|
||||
def test_quality_gate_requires_exact_jd_brand_and_category_sets() -> None:
|
||||
missing_brand = _jd_payload()
|
||||
missing_brand["数据"].pop(JD_BRANDS[-1])
|
||||
valid, reason = validate_peer_payload(JD_PLATFORM, missing_brand)
|
||||
assert valid is False
|
||||
assert "brand set mismatch" in reason
|
||||
|
||||
missing_category = _jd_payload()
|
||||
missing_category["数据"][JD_BRANDS[0]].pop(JD_CATEGORIES[-1])
|
||||
valid, reason = validate_peer_payload(JD_PLATFORM, missing_category)
|
||||
assert valid is False
|
||||
assert "category set mismatch" in reason
|
||||
|
||||
|
||||
def test_quality_gate_rejects_malformed_jd_ranking_entry() -> None:
|
||||
payload = _jd_payload()
|
||||
payload["数据"][JD_BRANDS[0]][JD_CATEGORIES[0]] = "23"
|
||||
|
||||
valid, reason = validate_peer_payload(JD_PLATFORM, payload)
|
||||
|
||||
assert valid is False
|
||||
assert "ranking entry is not an object" in reason
|
||||
|
||||
|
||||
def test_quality_gate_allows_verified_dy_unranked_results() -> None:
|
||||
payload = _dy_payload()
|
||||
for category in DY_CATEGORIES:
|
||||
for brand_index, brand in enumerate(DY_BRANDS):
|
||||
payload["results"][category][brand] = _dy_entry(
|
||||
brand,
|
||||
category,
|
||||
brand_index,
|
||||
ranked=False,
|
||||
matched=False,
|
||||
)
|
||||
|
||||
assert validate_peer_payload(DY_PLATFORM, payload) == (True, "")
|
||||
|
||||
|
||||
def test_quality_gate_rejects_identical_dy_data_in_two_categories() -> None:
|
||||
payload = _dy_payload()
|
||||
first_category, second_category = DY_CATEGORIES[:2]
|
||||
for brand in DY_BRANDS:
|
||||
repeated = deepcopy(payload["results"][first_category][brand])
|
||||
repeated["category"] = second_category
|
||||
payload["results"][second_category][brand] = repeated
|
||||
|
||||
valid, reason = validate_peer_payload(DY_PLATFORM, payload)
|
||||
|
||||
assert valid is False
|
||||
assert "identical payload copied across categories" in reason
|
||||
|
||||
|
||||
def test_quality_gate_requires_exact_dy_sets_and_entry_structure() -> None:
|
||||
payload = _dy_payload()
|
||||
payload["results"][DY_CATEGORIES[0]].pop(DY_BRANDS[-1])
|
||||
valid, reason = validate_peer_payload(DY_PLATFORM, payload)
|
||||
assert valid is False
|
||||
assert "brand set mismatch" in reason
|
||||
|
||||
payload = _dy_payload()
|
||||
payload["results"][DY_CATEGORIES[0]][DY_BRANDS[0]].pop("stores")
|
||||
valid, reason = validate_peer_payload(DY_PLATFORM, payload)
|
||||
assert valid is False
|
||||
assert "ranking fields missing stores" in reason
|
||||
|
||||
|
||||
def test_runner_propagates_cookie_skip_without_writes(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["run_peer_store", "--platform", "dy", "--skip-feishu"],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"setup_runner_logging",
|
||||
lambda _name: "test.log",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"skip_feishu_for_acceptance",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_run_with_fresh_artifact",
|
||||
lambda platform, _collector: (
|
||||
platform,
|
||||
COOKIE_SKIP_EXIT_CODE,
|
||||
"expired",
|
||||
"",
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_persist_and_write",
|
||||
lambda *_args, **_kwargs: pytest.fail("skipped platform must not write"),
|
||||
)
|
||||
|
||||
assert run_peer_store.main() == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_persist_zero_is_reported_as_failure_and_feishu_can_be_skipped(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(run_peer_store, "persist_jd_peer_store", lambda _path: 0)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"write_peer_store",
|
||||
lambda *_args, **_kwargs: pytest.fail("Feishu must remain skipped"),
|
||||
)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
JD_PLATFORM,
|
||||
True,
|
||||
"D:/fresh-peer.json",
|
||||
skip_feishu=True,
|
||||
)
|
||||
|
||||
assert errors == [f"{JD_PLATFORM} DB 持久化失败: 未写入任何记录"]
|
||||
|
||||
|
||||
def test_persist_zero_stops_peer_feishu_write(monkeypatch) -> None:
|
||||
monkeypatch.setattr(run_peer_store, "persist_jd_peer_store", lambda _path: 0)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"write_peer_store",
|
||||
lambda *_args, **_kwargs: pytest.fail("Feishu must not run after DB failure"),
|
||||
)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
JD_PLATFORM,
|
||||
True,
|
||||
"D:/fresh-peer.json",
|
||||
)
|
||||
|
||||
assert errors == [f"{JD_PLATFORM} DB 持久化失败: 未写入任何记录"]
|
||||
|
||||
|
||||
def test_dy_peer_rejects_shop_json_from_another_week_before_writes(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
peer_json = tmp_path / "peer.json"
|
||||
peer_json.write_text(
|
||||
json.dumps({"date_range": "2026/07/27 - 2026/08/02"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
shop_json = tmp_path / "shop.json"
|
||||
shop_json.write_text(
|
||||
json.dumps({"date_range": "2026/07/20 - 2026/07/26"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_latest_shop_json_for_dy",
|
||||
lambda: str(shop_json),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"persist_dy_peer_store",
|
||||
lambda _path: pytest.fail("mismatched snapshots must fail before PG"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"write_peer_store",
|
||||
lambda *_args, **_kwargs: pytest.fail("mismatched snapshots must not write Feishu"),
|
||||
)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
DY_PLATFORM,
|
||||
True,
|
||||
str(peer_json),
|
||||
)
|
||||
|
||||
assert len(errors) == 1
|
||||
assert "周区间不一致" in errors[0]
|
||||
|
||||
|
||||
def test_dy_peer_passes_matching_shop_json_to_writer(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
peer_json = tmp_path / "peer.json"
|
||||
peer_json.write_text(
|
||||
json.dumps({"date_range": "2026/07/27 - 2026/08/02"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
shop_json = tmp_path / "shop.json"
|
||||
shop_json.write_text(
|
||||
json.dumps({"date_range": "2026-07-27 ~ 2026-08-02"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
observed = {}
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_latest_shop_json_for_dy",
|
||||
lambda: str(shop_json),
|
||||
)
|
||||
monkeypatch.setattr(run_peer_store, "persist_dy_peer_store", lambda _path: 3)
|
||||
|
||||
def fake_write(platform, actual_peer_json, actual_shop_json):
|
||||
observed.update(
|
||||
platform=platform,
|
||||
peer_json=actual_peer_json,
|
||||
shop_json=actual_shop_json,
|
||||
)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(run_peer_store, "write_peer_store", fake_write)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
DY_PLATFORM,
|
||||
True,
|
||||
str(peer_json),
|
||||
)
|
||||
|
||||
assert errors == []
|
||||
assert observed == {
|
||||
"platform": DY_PLATFORM,
|
||||
"peer_json": str(peer_json),
|
||||
"shop_json": str(shop_json),
|
||||
}
|
||||
|
||||
|
||||
def test_false_feishu_writer_result_is_reported(monkeypatch) -> None:
|
||||
monkeypatch.setattr(run_peer_store, "persist_jd_peer_store", lambda _path: 3)
|
||||
monkeypatch.setattr(run_peer_store, "write_peer_store", lambda *_args: False)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
JD_PLATFORM,
|
||||
True,
|
||||
"D:/fresh-peer.json",
|
||||
)
|
||||
|
||||
assert errors == [f"{JD_PLATFORM} 竞店飞书写入失败: writer 返回 False"]
|
||||
|
||||
|
||||
def test_competitor_platforms_are_independent_graph_steps() -> None:
|
||||
module = ShopIntelligenceModule.from_catalog(
|
||||
WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
)
|
||||
competitor = next(
|
||||
definition
|
||||
for definition in module.workflow_definitions()
|
||||
if definition.workflow_id == "shop.competitor.weekly"
|
||||
)
|
||||
|
||||
assert tuple(step.step_id for step in competitor.steps) == ("jd", "dy")
|
||||
assert all(not step.depends_on for step in competitor.steps)
|
||||
assert tuple(step.action.command_args for step in competitor.steps) == (
|
||||
("--platform", "jd"),
|
||||
("--platform", "dy"),
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.runners.run_jd_self_operated_daily import (
|
||||
build_collector_args,
|
||||
previous_day,
|
||||
)
|
||||
|
||||
|
||||
class TargetDateTests(unittest.TestCase):
|
||||
def test_previous_day_handles_month_boundary(self):
|
||||
self.assertEqual(previous_day(date(2026, 8, 1)), date(2026, 7, 31))
|
||||
|
||||
def test_previous_day_handles_year_boundary(self):
|
||||
self.assertEqual(previous_day(date(2027, 1, 1)), date(2026, 12, 31))
|
||||
|
||||
|
||||
class CollectorArgsTests(unittest.TestCase):
|
||||
def test_builds_single_day_headless_args(self):
|
||||
self.assertEqual(
|
||||
build_collector_args(date(2026, 7, 30), headless=True),
|
||||
[
|
||||
"--start-date",
|
||||
"2026-07-30",
|
||||
"--end-date",
|
||||
"2026-07-30",
|
||||
"--headless",
|
||||
],
|
||||
)
|
||||
|
||||
def test_headful_args_omit_headless_flag(self):
|
||||
self.assertNotIn(
|
||||
"--headless",
|
||||
build_collector_args(date(2026, 7, 30), headless=False),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence import lark_cli
|
||||
|
||||
|
||||
def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path:
|
||||
evidence = tmp_path / "shop-evidence.jsonl"
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False)
|
||||
return evidence
|
||||
|
||||
|
||||
def test_shop_shared_lark_cli_physically_skips_bitable_write(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
evidence = _enable_acceptance(monkeypatch, tmp_path)
|
||||
|
||||
def fail_if_spawned(*args, **kwargs):
|
||||
raise AssertionError("lark-cli subprocess must not start")
|
||||
|
||||
monkeypatch.setattr(lark_cli.subprocess, "run", fail_if_spawned)
|
||||
result = lark_cli.run_lark_cli(
|
||||
"node",
|
||||
"run.js",
|
||||
["base", "+record-batch-update", "--table-id", "tbl_test"],
|
||||
)
|
||||
|
||||
assert result["acceptance_skipped"] is True
|
||||
payload = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
assert payload["event"] == "feishu_write_skipped"
|
||||
assert payload["operation"].endswith("record-batch-update")
|
||||
|
||||
|
||||
def test_shop_shared_lark_cli_keeps_production_write_behavior(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
calls.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="")
|
||||
|
||||
monkeypatch.setattr(lark_cli.subprocess, "run", fake_run)
|
||||
result = lark_cli.run_lark_cli(
|
||||
"node",
|
||||
"run.js",
|
||||
["base", "+record-batch-create", "--table-id", "tbl_test"],
|
||||
)
|
||||
|
||||
assert result == {"ok": True}
|
||||
assert len(calls) == 1
|
||||
@@ -0,0 +1,916 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import json
|
||||
import sys
|
||||
from datetime import date
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence import lark_cli as shop_lark_cli
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_data_collector,
|
||||
jd_peer_store_data_collector,
|
||||
jd_self_operated_brand_daily,
|
||||
taobao_sycm,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.db import db as shop_db
|
||||
from gyxx_flow.modules.shop_intelligence.lark_cli import (
|
||||
format_week,
|
||||
normalize_week_label,
|
||||
week_search_prefix,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.quality import validate_shop_payload
|
||||
from gyxx_flow.modules.shop_intelligence.writers import shop_base_writer
|
||||
|
||||
run_shop = import_module("gyxx_flow.modules.shop_intelligence.runners.run_shop")
|
||||
|
||||
|
||||
def test_jd_playwright_wait_for_function_passes_argument_by_keyword() -> None:
|
||||
tree = ast.parse(
|
||||
Path(jd_data_collector.__file__).read_text(encoding="utf-8-sig")
|
||||
)
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "wait_for_function"
|
||||
]
|
||||
|
||||
assert calls
|
||||
assert all(len(call.args) == 1 for call in calls)
|
||||
|
||||
|
||||
def test_jd_cross_month_week_uses_visible_iso_week_row() -> None:
|
||||
source = inspect.getsource(jd_data_collector.select_last_week_any_day)
|
||||
|
||||
assert "target_year = last_sunday.year" in source
|
||||
assert "jmtd-date-picker-header-btn-prev-month" in source
|
||||
assert "jmtd-date-picker-week-number-cell" in source
|
||||
assert "clicked_iso_week_row_" in source
|
||||
assert "jmtd-date-picker-combo-tag-echo" in source
|
||||
|
||||
|
||||
def test_jd_navigation_retries_transient_failures() -> None:
|
||||
class Page:
|
||||
def __init__(self) -> None:
|
||||
self.attempts = 0
|
||||
self.waits: list[int] = []
|
||||
|
||||
def goto(self, *_args, **_kwargs) -> None:
|
||||
self.attempts += 1
|
||||
if self.attempts < 3:
|
||||
raise RuntimeError("net::ERR_PROXY_CONNECTION_FAILED")
|
||||
|
||||
def wait_for_timeout(self, timeout_ms: int) -> None:
|
||||
self.waits.append(timeout_ms)
|
||||
|
||||
page = Page()
|
||||
jd_data_collector.goto_with_retry(page, "https://example.invalid")
|
||||
|
||||
assert page.attempts == 3
|
||||
assert page.waits == [3000, 6000]
|
||||
|
||||
|
||||
def test_jd_waits_for_summary_card_hydration_after_week_switch() -> None:
|
||||
source = inspect.getsource(jd_data_collector.step_shop_star_and_trade)
|
||||
|
||||
assert "page.wait_for_timeout(8000)" in source
|
||||
assert ".sz-summary-item" in source
|
||||
assert ".sz-summary-item-index [data-value]" in source
|
||||
assert "real-time-module-right-shop-level-content-my-level-data-value" in source
|
||||
|
||||
|
||||
def test_jd_star_accepts_current_bare_score_value() -> None:
|
||||
assert jd_data_collector._extract_jd_star("5.0") == "5.0星"
|
||||
assert jd_data_collector._extract_jd_star("5.0详情") == "5.0星"
|
||||
assert jd_data_collector._extract_jd_star("未获取到") == "未获取到"
|
||||
|
||||
|
||||
def test_week_label_is_canonical_and_historical_padding_is_equivalent() -> None:
|
||||
week_label, month_label, *_ = format_week("2026-07-27", "2026-08-02")
|
||||
|
||||
assert week_label == "7月第5周（7.27-8.2）"
|
||||
assert month_label == "2026.07月"
|
||||
assert normalize_week_label("7月第5周（7.27-8.02）") == normalize_week_label(
|
||||
week_label
|
||||
)
|
||||
assert normalize_week_label(" 7月第5周 ( 7.27 - 8.02 ) ") == normalize_week_label(
|
||||
week_label
|
||||
)
|
||||
assert week_search_prefix("7月第5周（7.27-8.02）") == "7月第5周"
|
||||
|
||||
|
||||
def test_find_existing_record_matches_padding_but_not_another_year(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
observed = {}
|
||||
|
||||
def fake_run(_node, _script, args, **_kwargs):
|
||||
observed["payload"] = json.loads(args[-1])
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "platform", "month"],
|
||||
"data": [
|
||||
["7月第5周（7.27-8.02）", ["DY"], ["2025.07月"]],
|
||||
["7月第5周（7.27-8.02）", ["DY"], ["2026.07月"]],
|
||||
],
|
||||
"record_id_list": ["old-year", "same-week"],
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(shop_lark_cli, "run_lark_cli", fake_run)
|
||||
fields = {
|
||||
"时间": "time",
|
||||
"平台": "platform",
|
||||
"月份": "month",
|
||||
}
|
||||
|
||||
record_id = shop_lark_cli.find_existing_record(
|
||||
"node",
|
||||
"script",
|
||||
"base",
|
||||
"table",
|
||||
fields,
|
||||
"7月第5周（7.27-8.2）",
|
||||
"DY",
|
||||
month_str="2026.07月",
|
||||
)
|
||||
|
||||
assert record_id == "same-week"
|
||||
assert observed["payload"]["keyword"] == "7月第5周"
|
||||
|
||||
|
||||
def test_dy_shop_feishu_sales_uses_confirmed_net_sales_amount() -> None:
|
||||
extracted = shop_base_writer._extract_dy_metrics(
|
||||
{
|
||||
"date_range": "2026/07/27 - 2026/08/02",
|
||||
"metrics": {
|
||||
"成交金额": "¥413,988.46",
|
||||
"成交退款金额": "¥119,095.48",
|
||||
},
|
||||
"derived_metrics": {"raw": {"销售额": 294892.98}},
|
||||
"experience_score": {"score": "95"},
|
||||
}
|
||||
)
|
||||
|
||||
assert extracted["sales_amount"] == 294892.98
|
||||
assert extracted["star_rating"] == 95.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_value",
|
||||
["加载失败", "未采集到", "¥", "abc", float("nan"), -0.01, "5星"],
|
||||
)
|
||||
def test_jd_shop_quality_rejects_invalid_or_negative_sales(invalid_value) -> None:
|
||||
valid, reason = validate_shop_payload(
|
||||
run_shop.JD_PLATFORM,
|
||||
{
|
||||
"transaction": {"成交金额": {"value": invalid_value}},
|
||||
"star_rating_raw": "5.0星",
|
||||
},
|
||||
)
|
||||
|
||||
assert valid is False
|
||||
assert "成交金额" in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_value",
|
||||
["加载失败", "未采集到", "¥", "abc", float("nan"), -0.01, 5.01, "¥4.9"],
|
||||
)
|
||||
def test_jd_shop_quality_rejects_invalid_or_out_of_range_score(
|
||||
invalid_value,
|
||||
) -> None:
|
||||
valid, reason = validate_shop_payload(
|
||||
run_shop.JD_PLATFORM,
|
||||
{
|
||||
"transaction": {"成交金额": {"value": "￥311,671.16"}},
|
||||
"star_rating_raw": invalid_value,
|
||||
},
|
||||
)
|
||||
|
||||
assert valid is False
|
||||
assert "店铺评分" in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "invalid_value", "reason_text"),
|
||||
[
|
||||
("gross", "加载失败", "成交金额"),
|
||||
("gross", "¥", "成交金额"),
|
||||
("gross", float("nan"), "成交金额"),
|
||||
("gross", -1, "成交金额"),
|
||||
("net", "abc", "净销售额"),
|
||||
("net", float("nan"), "净销售额"),
|
||||
("net", -1, "净销售额"),
|
||||
("score", "未采集到", "店铺评分"),
|
||||
("score", float("nan"), "店铺评分"),
|
||||
("score", -1, "店铺评分"),
|
||||
("score", 100.01, "店铺评分"),
|
||||
],
|
||||
)
|
||||
def test_dy_shop_quality_rejects_invalid_metrics(
|
||||
field,
|
||||
invalid_value,
|
||||
reason_text,
|
||||
) -> None:
|
||||
gross = "¥413,988.46"
|
||||
net = 294892.98
|
||||
score = "95分"
|
||||
if field == "gross":
|
||||
gross = invalid_value
|
||||
elif field == "net":
|
||||
net = invalid_value
|
||||
else:
|
||||
score = invalid_value
|
||||
|
||||
valid, reason = validate_shop_payload(
|
||||
run_shop.DY_PLATFORM,
|
||||
{
|
||||
"metrics": {"成交金额": gross},
|
||||
"derived_metrics": {"raw": {"销售额": net}},
|
||||
"experience_score": {"score": score},
|
||||
},
|
||||
)
|
||||
|
||||
assert valid is False
|
||||
assert reason_text in reason
|
||||
|
||||
|
||||
def test_shop_quality_accepts_strict_real_metric_formats() -> None:
|
||||
assert validate_shop_payload(
|
||||
run_shop.JD_PLATFORM,
|
||||
{
|
||||
"transaction": {"成交金额": {"value": "￥311,671.16"}},
|
||||
"star_rating_raw": "5.0星",
|
||||
},
|
||||
) == (True, "")
|
||||
assert validate_shop_payload(
|
||||
run_shop.JD_PLATFORM,
|
||||
{
|
||||
"transaction": {"成交金额": {"value": "￥264,650.81"}},
|
||||
"star_rating_raw": (
|
||||
"店铺星级星级运营店铺星级我的店铺星级4.7详情较昨日持平"
|
||||
),
|
||||
},
|
||||
) == (True, "")
|
||||
assert validate_shop_payload(
|
||||
run_shop.DY_PLATFORM,
|
||||
{
|
||||
"metrics": {"成交金额": "¥413,988.46"},
|
||||
"derived_metrics": {"raw": {"销售额": 294892.98}},
|
||||
"experience_score": {"score": "95分"},
|
||||
},
|
||||
) == (True, "")
|
||||
|
||||
|
||||
def test_jd_shop_feishu_rejects_missing_metrics_instead_of_writing_zero() -> None:
|
||||
with pytest.raises(ValueError, match="缺少有效成交金额、店铺评分"):
|
||||
shop_base_writer._extract_jd_metrics(
|
||||
{
|
||||
"week_start": "2026-07-27",
|
||||
"week_end": "2026-08-02",
|
||||
"star_rating_raw": "未获取到",
|
||||
"transaction": {"成交金额": {"value": "N/A"}},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_jd_current_dom_metric_text_and_star_are_normalized() -> None:
|
||||
assert jd_data_collector._normalize_jd_metric_text(
|
||||
"￥\n3\n1\n1\n,\n6\n7\n1\n.\n1\n6"
|
||||
) == "￥311,671.16"
|
||||
assert jd_data_collector._extract_jd_star(
|
||||
"店铺星级\n星级运营\n我的店铺星级\n5.0详情\n较昨日持平"
|
||||
) == "5.0星"
|
||||
|
||||
|
||||
def test_jd_week_event_suffix_uses_iso_week() -> None:
|
||||
assert jd_data_collector._week_event_suffix(
|
||||
jd_data_collector.datetime(2026, 7, 27)
|
||||
) == "2026-31"
|
||||
|
||||
|
||||
def test_runner_rejects_success_without_fresh_artifact(monkeypatch) -> None:
|
||||
marker = ("D:/old.json", 1, 10)
|
||||
monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: marker)
|
||||
|
||||
result = run_shop._run_with_fresh_artifact(
|
||||
"JD",
|
||||
lambda: ("JD", 0, "collector output"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "no fresh metrics file" in result[2]
|
||||
|
||||
|
||||
def test_runner_passes_only_the_new_artifact(monkeypatch) -> None:
|
||||
markers = iter((None, ("D:/new.json", 2, 20)))
|
||||
monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: next(markers))
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"_validate_metrics_artifact",
|
||||
lambda _platform, _path: (True, ""),
|
||||
)
|
||||
|
||||
result = run_shop._run_with_fresh_artifact(
|
||||
"JD",
|
||||
lambda: ("JD", 0, "ok"),
|
||||
)
|
||||
|
||||
assert result == ("JD", 0, "ok", "D:/new.json")
|
||||
|
||||
|
||||
def test_runner_rejects_fresh_but_empty_metrics_artifact(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "dy.json"
|
||||
artifact.write_text('{"metrics": {}}', encoding="utf-8")
|
||||
markers = iter((None, (str(artifact), 2, artifact.stat().st_size)))
|
||||
monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: next(markers))
|
||||
|
||||
result = run_shop._run_with_fresh_artifact(
|
||||
run_shop.DY_PLATFORM,
|
||||
lambda: (run_shop.DY_PLATFORM, 0, "collector output"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "shop metrics are empty" in result[2]
|
||||
|
||||
|
||||
def test_runner_rejects_jd_placeholder_metrics_before_external_writes(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "jd.json"
|
||||
artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"star_rating_raw": "未获取到",
|
||||
"transaction": {"成交金额": {"value": "N/A"}},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
marker = (str(artifact), artifact.stat().st_mtime_ns, artifact.stat().st_size)
|
||||
markers = iter((None, marker))
|
||||
monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: next(markers))
|
||||
|
||||
result = run_shop._run_with_fresh_artifact(
|
||||
run_shop.JD_PLATFORM,
|
||||
lambda: (run_shop.JD_PLATFORM, 0, "collector output"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "missing valid 成交金额/店铺评分" in result[2]
|
||||
|
||||
|
||||
def test_direct_jd_persistence_rejects_placeholder_metrics_before_connecting(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "jd-invalid.json"
|
||||
artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"star_rating_raw": "未获取到",
|
||||
"transaction": {"成交金额": {"value": "N/A"}},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(shop_db, "DB_ENABLED", True)
|
||||
monkeypatch.setattr(
|
||||
shop_db,
|
||||
"get_conn",
|
||||
lambda: pytest.fail("invalid payload must not connect to PostgreSQL"),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="JD shop data quality check failed"):
|
||||
shop_db.persist_jd_shop(str(artifact))
|
||||
|
||||
|
||||
def test_jd_pg_transaction_value_does_not_depend_on_item_key_order() -> None:
|
||||
metrics = {
|
||||
"shop": "test-shop",
|
||||
"week_start": "2026-07-27",
|
||||
"week_end": "2026-08-02",
|
||||
"star_rating_raw": "5.0",
|
||||
"transaction": {
|
||||
"成交金额": {
|
||||
"环比": "-19.8%",
|
||||
"同行同级均值": "¥999.00",
|
||||
"value": "¥311,671.16",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
row = shop_db._build_jd_shop_row(metrics)
|
||||
|
||||
assert row is not None
|
||||
assert row["deal_amount"] == 311671.16
|
||||
assert row["deal_amount_qoq"] == "-19.8%"
|
||||
assert row["deal_amount_peer_avg"] == "¥999.00"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "persist_name", "platform_code"),
|
||||
[
|
||||
(run_shop.JD_PLATFORM, "persist_jd_shop", "JD"),
|
||||
(run_shop.DY_PLATFORM, "persist_dy_shop", "DY"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("persisted_count", [0, -1])
|
||||
def test_shop_nonpositive_persistence_stops_feishu_write(
|
||||
monkeypatch,
|
||||
platform: str,
|
||||
persist_name: str,
|
||||
platform_code: str,
|
||||
persisted_count: int,
|
||||
) -> None:
|
||||
monkeypatch.setattr(run_shop, "ensure_ai_analysis", lambda *_args: "")
|
||||
monkeypatch.setattr(run_shop, persist_name, lambda _path: persisted_count)
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"write_shop",
|
||||
lambda *_args: pytest.fail("Feishu must not run after DB failure"),
|
||||
)
|
||||
|
||||
errors = run_shop._persist_and_write(platform, True, "D:/fresh-shop.json")
|
||||
|
||||
assert errors == [f"{platform_code} DB 持久化失败: 未写入任何记录"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "persist_name", "platform_code"),
|
||||
[
|
||||
(run_shop.JD_PLATFORM, "persist_jd_shop", "JD"),
|
||||
(run_shop.DY_PLATFORM, "persist_dy_shop", "DY"),
|
||||
],
|
||||
)
|
||||
def test_shop_false_writer_result_is_reported(
|
||||
monkeypatch,
|
||||
platform: str,
|
||||
persist_name: str,
|
||||
platform_code: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(run_shop, "ensure_ai_analysis", lambda *_args: "")
|
||||
monkeypatch.setattr(run_shop, persist_name, lambda _path: 1)
|
||||
monkeypatch.setattr(run_shop, "write_shop", lambda *_args: False)
|
||||
|
||||
errors = run_shop._persist_and_write(platform, True, "D:/fresh-shop.json")
|
||||
|
||||
assert errors == [f"{platform_code} 飞书写入失败: writer 返回 False"]
|
||||
|
||||
|
||||
def test_runner_propagates_cookie_skip_without_writes(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["run_shop", "--platform", "dy", "--skip-feishu"],
|
||||
)
|
||||
monkeypatch.setattr(run_shop, "setup_runner_logging", lambda _name: "test.log")
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"skip_feishu_for_acceptance",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"_run_with_fresh_artifact",
|
||||
lambda platform, _collector: (
|
||||
platform,
|
||||
COOKIE_SKIP_EXIT_CODE,
|
||||
"expired",
|
||||
"",
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"_persist_and_write",
|
||||
lambda *_args, **_kwargs: pytest.fail("skipped platform must not write"),
|
||||
)
|
||||
|
||||
assert run_shop.main() == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_shop_analysis_uses_local_analyzer_hermes(monkeypatch) -> None:
|
||||
observed = {}
|
||||
|
||||
class Response:
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict:
|
||||
return {"choices": [{"message": {"content": "analysis"}}]}
|
||||
|
||||
class Session:
|
||||
trust_env = True
|
||||
|
||||
def post(self, url, **kwargs):
|
||||
observed["session"] = self
|
||||
observed["url"] = url
|
||||
observed.update(kwargs)
|
||||
return Response()
|
||||
|
||||
monkeypatch.setenv(
|
||||
"HERMES_ANALYZER_URL",
|
||||
"http://127.0.0.1:8642/v1/chat/completions",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_ANALYZER_TOKEN", "test-token")
|
||||
monkeypatch.setattr(shop_base_writer.requests, "Session", Session)
|
||||
|
||||
assert shop_base_writer._call_hermes("test prompt") == "analysis"
|
||||
assert observed["url"].startswith("http://127.0.0.1:8642/")
|
||||
assert observed["session"].trust_env is False
|
||||
assert observed["headers"]["Authorization"] == "Bearer test-token"
|
||||
|
||||
|
||||
def test_shop_analysis_falls_back_to_local_cli_when_gateway_requires_auth(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
class Response:
|
||||
status_code = 401
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
error = shop_base_writer.requests.HTTPError("unauthorized")
|
||||
error.response = self
|
||||
raise error
|
||||
|
||||
class Session:
|
||||
trust_env = True
|
||||
|
||||
def post(self, *_args, **_kwargs):
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr(shop_base_writer.requests, "Session", Session)
|
||||
monkeypatch.setattr(
|
||||
shop_base_writer,
|
||||
"_call_hermes_cli",
|
||||
lambda prompt, *, timeout: f"cli:{prompt}:{timeout}",
|
||||
)
|
||||
|
||||
assert shop_base_writer._call_hermes("prompt", timeout=12) == "cli:prompt:12"
|
||||
|
||||
|
||||
def test_shop_analysis_reads_analyzer_profile_gateway_key(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
profile = tmp_path / "profiles" / "data-analyzer"
|
||||
profile.mkdir(parents=True)
|
||||
(profile / ".env").write_text(
|
||||
"API_SERVER_KEY=profile-key\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
observed = {}
|
||||
|
||||
class Response:
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict:
|
||||
return {"choices": [{"message": {"content": "analysis"}}]}
|
||||
|
||||
class Session:
|
||||
trust_env = True
|
||||
|
||||
def post(self, _url, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return Response()
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
monkeypatch.delenv("HERMES_ANALYZER_TOKEN", raising=False)
|
||||
monkeypatch.delenv("GYXX_HERMES_API_KEY", raising=False)
|
||||
monkeypatch.delenv("HERMES_API_KEY", raising=False)
|
||||
monkeypatch.setattr(shop_base_writer.requests, "Session", Session)
|
||||
|
||||
assert shop_base_writer._call_hermes("prompt") == "analysis"
|
||||
assert observed["headers"]["Authorization"] == "Bearer profile-key"
|
||||
|
||||
|
||||
def test_jd_shop_uses_bound_profile_port_and_persists_state(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
page = object()
|
||||
observed = {}
|
||||
|
||||
class Context:
|
||||
pages = [page]
|
||||
closed = False
|
||||
|
||||
def cookies(self):
|
||||
return [{"name": "session", "value": "saved"}]
|
||||
|
||||
def storage_state(self, *, path: str) -> None:
|
||||
Path(path).write_text('{"cookies": []}', encoding="utf-8")
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
context = Context()
|
||||
|
||||
class Chromium:
|
||||
def launch_persistent_context(self, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return context
|
||||
|
||||
class Playwright:
|
||||
chromium = Chromium()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
|
||||
cookie_file = tmp_path / "state" / "cookies.json"
|
||||
storage_file = tmp_path / "state" / "storage.json"
|
||||
profile = tmp_path / "profile"
|
||||
monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile))
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22105")
|
||||
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
|
||||
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
|
||||
monkeypatch.setattr(jd_data_collector, "sync_playwright", Playwright)
|
||||
monkeypatch.setattr(jd_data_collector, "step_login", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"step_enter_shangzhi",
|
||||
lambda *_args: page,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"step_shop_star_and_trade",
|
||||
lambda *_args: ({}, {}),
|
||||
)
|
||||
monkeypatch.setattr(jd_data_collector, "step_deal_summary", lambda *_args: {})
|
||||
monkeypatch.setattr(jd_data_collector, "step_merchant_ranking", lambda *_args: {})
|
||||
monkeypatch.setattr(jd_data_collector, "step_jzt_non_full_site", lambda *_args: {})
|
||||
monkeypatch.setattr(jd_data_collector, "step_jzt_full_site", lambda *_args: {})
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"generate_report",
|
||||
lambda *_args: str(tmp_path / "report.json"),
|
||||
)
|
||||
|
||||
assert jd_data_collector.main(["--password", "test-only"]) == 0
|
||||
assert observed["user_data_dir"] == str(profile)
|
||||
assert "--remote-debugging-port=22105" in observed["args"]
|
||||
assert "--no-proxy-server" in observed["args"]
|
||||
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "session"
|
||||
assert storage_file.is_file()
|
||||
assert context.closed is True
|
||||
|
||||
|
||||
def test_jd_peer_uses_bound_profile_port_and_persists_state(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
page = object()
|
||||
observed = {}
|
||||
|
||||
class Context:
|
||||
pages = [page]
|
||||
closed = False
|
||||
|
||||
def cookies(self):
|
||||
return [{"name": "peer-session", "value": "saved"}]
|
||||
|
||||
def storage_state(self, *, path: str) -> None:
|
||||
Path(path).write_text('{"cookies": []}', encoding="utf-8")
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
context = Context()
|
||||
|
||||
class Chromium:
|
||||
def launch_persistent_context(self, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return context
|
||||
|
||||
class Playwright:
|
||||
chromium = Chromium()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
|
||||
cookie_file = tmp_path / "state" / "cookies.json"
|
||||
storage_file = tmp_path / "state" / "storage.json"
|
||||
profile = tmp_path / "profile"
|
||||
monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile))
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22106")
|
||||
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
|
||||
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
|
||||
monkeypatch.setattr(jd_peer_store_data_collector, "sync_playwright", Playwright)
|
||||
monkeypatch.setattr(jd_peer_store_data_collector, "step_login", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_enter_shangzhi",
|
||||
lambda *_args: page,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_return_old_version",
|
||||
lambda *_args: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_goto_vender_ranks",
|
||||
lambda *_args: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_collect_competitor_data",
|
||||
lambda *_args: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"save_results",
|
||||
lambda *_args: str(tmp_path / "peer.json"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_goto_product_ranks",
|
||||
lambda *_args: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"collect_new_products",
|
||||
lambda *_args: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"save_new_product_results",
|
||||
lambda *_args: None,
|
||||
)
|
||||
|
||||
assert jd_peer_store_data_collector.main(["--password", "test-only"]) == 0
|
||||
assert observed["user_data_dir"] == str(profile)
|
||||
assert "--remote-debugging-port=22106" in observed["args"]
|
||||
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "peer-session"
|
||||
assert storage_file.is_file()
|
||||
assert context.closed is True
|
||||
|
||||
|
||||
def test_jd_peer_selects_week_once_before_category_loop(monkeypatch) -> None:
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"SHOPS",
|
||||
["peer-shop"],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"CATEGORIES",
|
||||
[("男士双肩包", "男包")],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"select_last_week_any_day",
|
||||
lambda _page: calls.append("initial-week"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"search_shop",
|
||||
lambda *_args: calls.append("search"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"switch_category",
|
||||
lambda *_args: calls.append("category"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"reselect_last_week",
|
||||
lambda _page: calls.append("reselect-week"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"extract_ranking_table",
|
||||
lambda _page: [],
|
||||
)
|
||||
|
||||
page = SimpleNamespace(wait_for_timeout=lambda *_args: None)
|
||||
jd_peer_store_data_collector.step_collect_competitor_data(page)
|
||||
|
||||
assert calls == ["initial-week", "search", "category"]
|
||||
|
||||
|
||||
def test_jd_peer_product_rank_navigation_does_not_wait_for_network_idle() -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
class Page:
|
||||
def goto(self, url: str, **kwargs) -> None:
|
||||
observed["url"] = url
|
||||
observed.update(kwargs)
|
||||
|
||||
def wait_for_timeout(self, timeout_ms: int) -> None:
|
||||
observed["settle_ms"] = timeout_ms
|
||||
|
||||
jd_peer_store_data_collector.step_goto_product_ranks(Page())
|
||||
|
||||
assert observed == {
|
||||
"url": "https://sz.jd.com/szweb/sz/view/industryMarket/productRanks.html",
|
||||
"wait_until": "domcontentloaded",
|
||||
"timeout": 60000,
|
||||
"settle_ms": 8000,
|
||||
}
|
||||
|
||||
|
||||
def test_jd_self_operated_reports_date_not_yet_available() -> None:
|
||||
page = MagicMock()
|
||||
mode = MagicMock()
|
||||
mode.count.return_value = 0
|
||||
date_picker = MagicMock()
|
||||
dropdown = MagicMock()
|
||||
enabled_target = MagicMock()
|
||||
disabled_target = MagicMock()
|
||||
selectable = MagicMock()
|
||||
enabled_target.count.return_value = 0
|
||||
disabled_target.count.return_value = 1
|
||||
selectable.evaluate_all.return_value = [
|
||||
"2026-08-01",
|
||||
"2026-08-02",
|
||||
None,
|
||||
]
|
||||
|
||||
def locate(selector: str):
|
||||
if selector == ".jd-select-selection-item":
|
||||
return SimpleNamespace(first=mode)
|
||||
if selector == ".jd-picker-range":
|
||||
return SimpleNamespace(first=date_picker)
|
||||
if selector == ".jd-picker-dropdown:visible":
|
||||
return SimpleNamespace(first=dropdown)
|
||||
raise AssertionError(selector)
|
||||
|
||||
def locate_dropdown(selector: str):
|
||||
if selector == (
|
||||
"td[title].jd-picker-cell-in-view:not(.jd-picker-cell-disabled)"
|
||||
):
|
||||
return selectable
|
||||
if selector.endswith(":not(.jd-picker-cell-disabled)"):
|
||||
return enabled_target
|
||||
if selector.endswith(".jd-picker-cell-disabled"):
|
||||
return disabled_target
|
||||
raise AssertionError(selector)
|
||||
|
||||
page.locator.side_effect = locate
|
||||
dropdown.locator.side_effect = locate_dropdown
|
||||
|
||||
with pytest.raises(RuntimeError, match="当前最晚可选日期为 2026-08-02"):
|
||||
jd_self_operated_brand_daily.set_single_day(
|
||||
page,
|
||||
date(2026, 8, 3),
|
||||
)
|
||||
|
||||
|
||||
def test_tmall_returns_failure_when_browser_cannot_start(monkeypatch) -> None:
|
||||
monkeypatch.setattr(taobao_sycm, "sync_cookies_from_daily_profile", lambda *_args: 0)
|
||||
monkeypatch.setattr(taobao_sycm, "connect_browser", lambda: (None, None))
|
||||
|
||||
assert taobao_sycm.main() == 1
|
||||
|
||||
|
||||
def test_tmall_profile_seed_preserves_cookie_encryption_state(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source"
|
||||
target = tmp_path / "target"
|
||||
(source / "Default" / "Network").mkdir(parents=True)
|
||||
(source / "Local State").write_text("encryption-state", encoding="utf-8")
|
||||
(source / "Default" / "Network" / "Cookies").write_bytes(b"cookie-db")
|
||||
|
||||
copied = taobao_sycm.sync_cookies_from_daily_profile(source, target)
|
||||
|
||||
assert copied >= 2
|
||||
assert (target / "Local State").read_text(encoding="utf-8") == "encryption-state"
|
||||
assert (target / "Default" / "Network" / "Cookies").read_bytes() == b"cookie-db"
|
||||
|
||||
|
||||
def test_tmall_close_always_releases_bound_cdp(monkeypatch) -> None:
|
||||
closed = []
|
||||
killed = []
|
||||
monkeypatch.setattr(taobao_sycm, "DEBUG_PORT", 22107)
|
||||
monkeypatch.setattr(taobao_sycm, "_kill_port_listeners", killed.append)
|
||||
|
||||
taobao_sycm._close_session(SimpleNamespace(close=lambda: closed.append(True)))
|
||||
|
||||
assert closed == [True]
|
||||
assert killed == [22107]
|
||||
@@ -0,0 +1,208 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence.runners import utils
|
||||
|
||||
|
||||
def test_runner_rebinds_browser_environment_after_caller_overrides(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def fake_rebind(target, base_environment):
|
||||
observed["target"] = Path(target)
|
||||
observed["base"] = dict(base_environment)
|
||||
return {
|
||||
**base_environment,
|
||||
"GYXX_SCRIPT_ID": "shop_intelligence:collectors/example.py",
|
||||
"GYXX_BROWSER_CDP_PORT": "child-port",
|
||||
"GYXX_BROWSER_PROFILE_DIR": "child-profile",
|
||||
"GYXX_BROWSER_COOKIE_FILE": "child-cookie",
|
||||
"GYXX_BROWSER_STORAGE_STATE_FILE": "child-storage",
|
||||
}
|
||||
|
||||
def fake_run(command, *, environment, timeout):
|
||||
observed["command"] = command
|
||||
observed["environment"] = environment
|
||||
observed["timeout"] = timeout
|
||||
return subprocess.CompletedProcess(command, 0, stdout=b"ok", stderr=b"")
|
||||
|
||||
monkeypatch.setattr(utils, "environment_for_child_script", fake_rebind)
|
||||
monkeypatch.setattr(utils, "_run_managed_command", fake_run)
|
||||
target = str(Path(utils.PROJECT_ROOT) / "collectors" / "example.py")
|
||||
|
||||
code, output = utils.run_collector(
|
||||
"example",
|
||||
target,
|
||||
[],
|
||||
env={"KEEP": "yes", "GYXX_BROWSER_CDP_PORT": "caller-port"},
|
||||
)
|
||||
|
||||
environment = observed["environment"]
|
||||
assert code == 0
|
||||
assert output == "ok"
|
||||
assert observed["target"] == Path(target)
|
||||
assert observed["base"]["GYXX_BROWSER_CDP_PORT"] == "caller-port"
|
||||
assert environment["KEEP"] == "yes"
|
||||
assert environment["GYXX_BROWSER_CDP_PORT"] == "child-port"
|
||||
assert environment["GYXX_BROWSER_PROFILE_DIR"] == "child-profile"
|
||||
assert environment["GYXX_BROWSER_COOKIE_FILE"] == "child-cookie"
|
||||
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == "child-storage"
|
||||
assert observed["timeout"] == 3600
|
||||
|
||||
|
||||
def test_runner_skips_invalid_cookie_before_starting_child(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def fake_rebind(target, base_environment):
|
||||
return {
|
||||
**base_environment,
|
||||
"GYXX_WORKFLOW_ACCEPTANCE": "1",
|
||||
"GYXX_SCRIPT_ID": "shop_intelligence:collectors/example.py",
|
||||
}
|
||||
|
||||
class FakePolicy:
|
||||
enabled = True
|
||||
|
||||
def preflight_cookie(self, binding, *, environment):
|
||||
observed["binding"] = binding
|
||||
observed["environment"] = environment
|
||||
return SimpleNamespace(should_skip=True, reason="browser state is missing")
|
||||
|
||||
monkeypatch.setattr(utils, "environment_for_child_script", fake_rebind)
|
||||
monkeypatch.setattr(utils, "binding_from_environment", lambda env: "binding")
|
||||
monkeypatch.setattr(utils, "current_acceptance_policy", lambda env: FakePolicy())
|
||||
monkeypatch.setattr(
|
||||
utils,
|
||||
"_run_managed_command",
|
||||
lambda *args, **kwargs: (_ for _ in ()).throw(
|
||||
AssertionError("child process must not start")
|
||||
),
|
||||
)
|
||||
|
||||
code, output = utils.run_collector(
|
||||
"example",
|
||||
str(Path(utils.PROJECT_ROOT) / "collectors" / "example.py"),
|
||||
[],
|
||||
)
|
||||
|
||||
assert code == COOKIE_SKIP_EXIT_CODE
|
||||
assert output == "browser state is missing"
|
||||
assert observed["binding"] == "binding"
|
||||
assert observed["environment"]["GYXX_WORKFLOW_ACCEPTANCE"] == "1"
|
||||
|
||||
|
||||
def test_runner_timeout_terminates_collector_grandchild_before_return(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
identity_file = tmp_path / "grandchild.json"
|
||||
marker_file = tmp_path / "grandchild-finished.txt"
|
||||
grandchild_code = (
|
||||
"import pathlib,time; "
|
||||
"time.sleep(2.8); "
|
||||
f"pathlib.Path({str(marker_file)!r}).write_text('alive', encoding='utf-8')"
|
||||
)
|
||||
collector = tmp_path / "collector.py"
|
||||
collector.write_text(
|
||||
"\n".join(
|
||||
(
|
||||
"import json, pathlib, psutil, subprocess, sys, time",
|
||||
f"child = subprocess.Popen([sys.executable, '-c', {grandchild_code!r}], close_fds=False)",
|
||||
f"pathlib.Path({str(identity_file)!r}).write_text(json.dumps({{'pid': child.pid, 'created': psutil.Process(child.pid).create_time()}}), encoding='utf-8')",
|
||||
"time.sleep(10)",
|
||||
)
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
utils,
|
||||
"environment_for_child_script",
|
||||
lambda _target, environment: environment,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
utils,
|
||||
"current_acceptance_policy",
|
||||
lambda _environment: SimpleNamespace(enabled=False),
|
||||
)
|
||||
|
||||
code, output = utils.run_collector(
|
||||
"timeout-probe",
|
||||
str(collector),
|
||||
[],
|
||||
env=dict(os.environ),
|
||||
timeout=2,
|
||||
)
|
||||
|
||||
assert code == 1
|
||||
assert "timeout-probe" in output
|
||||
assert identity_file.is_file()
|
||||
identity = json.loads(identity_file.read_text(encoding="utf-8"))
|
||||
try:
|
||||
descendant = psutil.Process(identity["pid"])
|
||||
same_process_is_live = (
|
||||
descendant.create_time()
|
||||
== pytest.approx(identity["created"], abs=0.01)
|
||||
and descendant.is_running()
|
||||
and descendant.status() != psutil.STATUS_ZOMBIE
|
||||
)
|
||||
except psutil.Error:
|
||||
same_process_is_live = False
|
||||
assert same_process_is_live is False
|
||||
time.sleep(1.0)
|
||||
assert marker_file.exists() is False
|
||||
|
||||
|
||||
def test_runner_redacts_sensitive_command_arguments() -> None:
|
||||
command = [
|
||||
"python",
|
||||
"collector.py",
|
||||
"--password",
|
||||
"placeholder",
|
||||
"--api-key=placeholder",
|
||||
"--output",
|
||||
"safe-path",
|
||||
]
|
||||
|
||||
assert utils._redact_command(command) == [
|
||||
"python",
|
||||
"collector.py",
|
||||
"--password",
|
||||
"[REDACTED]",
|
||||
"--api-key=[REDACTED]",
|
||||
"--output",
|
||||
"safe-path",
|
||||
]
|
||||
|
||||
|
||||
def test_acceptance_skips_entire_feishu_writer_path(monkeypatch) -> None:
|
||||
observed: list[str] = []
|
||||
|
||||
class FakePolicy:
|
||||
def skip_feishu_write(self, operation):
|
||||
observed.append(operation)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(utils, "current_acceptance_policy", lambda: FakePolicy())
|
||||
|
||||
assert utils.skip_feishu_for_acceptance("shop.writer", requested=False) is True
|
||||
assert observed == ["shop.writer"]
|
||||
|
||||
|
||||
def test_explicit_feishu_skip_does_not_need_policy(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
utils,
|
||||
"current_acceptance_policy",
|
||||
lambda: (_ for _ in ()).throw(AssertionError("policy must not be loaded")),
|
||||
)
|
||||
|
||||
assert utils.skip_feishu_for_acceptance("shop.writer", requested=True) is True
|
||||
@@ -0,0 +1,316 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
TARGET_COLLECTOR_ROOT = (
|
||||
PROJECT_ROOT
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "shop_intelligence"
|
||||
/ "collectors"
|
||||
)
|
||||
SOURCE_COLLECTOR_ROOT = Path(
|
||||
os.getenv("GYXX_SHOP_SOURCE_ROOT", r"D:\shop-data-flow")
|
||||
) / "collectors"
|
||||
|
||||
# These functions implement page navigation, date/category selection, extraction,
|
||||
# and iteration order. Portability adaptations belong in imports, configuration,
|
||||
# persistence, browser-session setup, and main/runner boundaries instead.
|
||||
CORE_BUSINESS_FUNCTIONS = {
|
||||
"jd_data_collector.py": (
|
||||
"select_last_week_any_day",
|
||||
"select_last_week_in_old_datepicker",
|
||||
"step_enter_shangzhi",
|
||||
"step_jzt_full_site",
|
||||
"step_jzt_non_full_site",
|
||||
"step_merchant_ranking",
|
||||
"step_shop_star_and_trade",
|
||||
"switch_category",
|
||||
),
|
||||
"jd_peer_store_data_collector.py": (
|
||||
"clear_and_search_brand",
|
||||
"collect_new_products",
|
||||
"reselect_last_week",
|
||||
"search_product_by_name",
|
||||
"select_last_week_any_day",
|
||||
"select_product_category",
|
||||
"step_enter_shangzhi",
|
||||
),
|
||||
"dy_store_competitor_store_scraping.py": (
|
||||
"_check_fxg_login",
|
||||
"_collect_competitor_rankings",
|
||||
"_collect_experience_score",
|
||||
"_ensure_fxg_login",
|
||||
"_extract_experience_score",
|
||||
"_open_market_rank_shop_page",
|
||||
"_select_menu_and_date",
|
||||
),
|
||||
}
|
||||
|
||||
# SHA-256 of ast.dump(function, include_attributes=False) from the reviewed
|
||||
# D:\shop-data-flow source. A source change must be reviewed before updating this
|
||||
# snapshot; copying a new hash merely to make the test pass defeats the guard.
|
||||
SOURCE_AST_SHA256 = {
|
||||
("jd_data_collector.py", "select_last_week_any_day"): (
|
||||
"0c85d64e6d47b08a5971caec3afe391271fd6bc94d86bab65f6ebae9655b8a6f"
|
||||
),
|
||||
("jd_data_collector.py", "select_last_week_in_old_datepicker"): (
|
||||
"ff542b968c43b92304ccb15a2282acf3d64ab7258dee020bae51f1c7f5f5f973"
|
||||
),
|
||||
("jd_data_collector.py", "step_enter_shangzhi"): (
|
||||
"6ba9e2df11ecbf0aae88fb2ed3cab8abe5157121418a0426384f8c43368b9362"
|
||||
),
|
||||
("jd_data_collector.py", "step_jzt_full_site"): (
|
||||
"0cf4276039e96650879ba6fff8822322245dc581ad3020785ed6cb8a84e11e8f"
|
||||
),
|
||||
("jd_data_collector.py", "step_jzt_non_full_site"): (
|
||||
"2a911597995c4d9a2cc47b106d6a8c5a811932eea0d83e928b0dcafcdb6864a5"
|
||||
),
|
||||
("jd_data_collector.py", "step_merchant_ranking"): (
|
||||
"027626587724aa224fca6146dae61ea505374e78ca36603c96769b36725dfac7"
|
||||
),
|
||||
("jd_data_collector.py", "step_shop_star_and_trade"): (
|
||||
"2e54324b6091e961eb6fbc704aca2867a91ce829709c2824e8a6ee8fa2260790"
|
||||
),
|
||||
("jd_data_collector.py", "switch_category"): (
|
||||
"f6dfdc7025958cbb85476c31b4930f2b57a8f67316c3000a8e38f5d49acb6aef"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "clear_and_search_brand"): (
|
||||
"32327c849483a44aa8c2b8cab8210f8e80767d3e2b86fd6789d855669062dc00"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "collect_new_products"): (
|
||||
"c2cbf6528a756d4fb26f5b8794177cbcb37790584f85660eff0a851ecffc8f58"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "reselect_last_week"): (
|
||||
"5897bd8afb8bf0604daace2a13628ea002f753afd8de7956246a4ee18e89cbd9"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "search_product_by_name"): (
|
||||
"2fd758156e716d2c666b8c1c1d9edb0c72cad83c56c4c44b09900bd7eb0bd940"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "select_last_week_any_day"): (
|
||||
"0a9b5fad4ea111e7b2afe39241508b5d036b84006091188b95ac85b652e7e3f9"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "select_product_category"): (
|
||||
"ff947df3e6e455176d723c93fb2af06ab92a72ac9174f4a5ae2db306201d5182"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "step_enter_shangzhi"): (
|
||||
"d6bfb283c00c842a129d8f81f0da61cae752e421e2dafaffdb87fd20603f0f2d"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_collect_competitor_rankings"): (
|
||||
"de55dded8a9341b955e000e4fafb8b4914533b25e06dc2aca930535836cfc0b5"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_check_fxg_login"): (
|
||||
"1928b38f6c167971139b147565b51e727e65a74e49ba21fe40d47711a7641e6a"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_collect_experience_score"): (
|
||||
"4609852591005c5b3ba8f9704b4db49918173f846176d9e50da6d4c8ee07914a"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_ensure_fxg_login"): (
|
||||
"ae69c5c58563c6cdb97fd6af380a24e2d275ec6623038a58b5e15a7bda2777f4"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_extract_experience_score"): (
|
||||
"5a28dbc7fa627fea384dae3d5f9b85fb0542b585c7312918ef355810209089df"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_open_market_rank_shop_page"): (
|
||||
"b60acaeaf3b1073144f3f570eb539ad87945fd92f2577cf338b3b496ec542d35"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_select_menu_and_date"): (
|
||||
"4ffb5e89e7a7af3d0af126adf420399a0017fba8e49b9185266baf7307e3e549"
|
||||
),
|
||||
}
|
||||
|
||||
# Deliberately reviewed target-only compatibility fixes. The source project stays
|
||||
# read-only; these hashes ensure the migrated collector cannot drift beyond the
|
||||
# current JD DOM adaptation without another explicit review.
|
||||
TARGET_AST_SHA256_OVERRIDES = {
|
||||
("jd_data_collector.py", "select_last_week_any_day"): (
|
||||
"8f6aed11b39d8e19ec4146e9b32eee96f5977fdbef694311534346bacec5d847"
|
||||
),
|
||||
("jd_data_collector.py", "step_shop_star_and_trade"): (
|
||||
"77aa873543861f1fad63d0bd1ace60528ab20592b0bffef3d46ad192f2103161"
|
||||
),
|
||||
("jd_data_collector.py", "step_merchant_ranking"): (
|
||||
"187cd3cfffcb97a5bf9f1ed4dde0d117ee65f82c1cf431df4c7c9c9ee2a7b282"
|
||||
),
|
||||
("jd_data_collector.py", "step_jzt_non_full_site"): (
|
||||
"1bd53eba5d0123e63d2e628d9772c0efa57fc8d2f87d79a4f3c41a311132952d"
|
||||
),
|
||||
}
|
||||
|
||||
# These two source functions were reviewed after a production run proved that
|
||||
# a failed category click silently copied the previous category's table. They
|
||||
# intentionally diverge from D:\shop-data-flow and retain their own reviewed
|
||||
# target snapshots so the defect cannot be reintroduced by a future sync.
|
||||
REVIEWED_DEFECT_FIX_AST_SHA256 = {
|
||||
("jd_peer_store_data_collector.py", "step_collect_competitor_data"): (
|
||||
"bf6836516dc3e70d00c145acbdfa00b6ddeee480f9927d7236e82ba8b1430ff2"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "switch_category"): (
|
||||
"bb4dc034447666682e19dc8cce05903c92d53852669ac93cc6810b18ad310d13"
|
||||
),
|
||||
}
|
||||
|
||||
FUNCTION_CASES = tuple(
|
||||
(filename, function_name)
|
||||
for filename, function_names in CORE_BUSINESS_FUNCTIONS.items()
|
||||
for function_name in function_names
|
||||
)
|
||||
|
||||
|
||||
def _call_name(node: ast.AST) -> str | None:
|
||||
if not isinstance(node, ast.Call):
|
||||
return None
|
||||
if isinstance(node.func, ast.Name):
|
||||
return node.func.id
|
||||
if isinstance(node.func, ast.Attribute):
|
||||
return node.func.attr
|
||||
return None
|
||||
|
||||
|
||||
def _is_session_expiry_guard(node: ast.stmt) -> bool:
|
||||
if not isinstance(node, ast.If) or node.orelse:
|
||||
return False
|
||||
detector_names = {"_is_login_page_url", "_has_interactive_login_form"}
|
||||
detectors = {
|
||||
name
|
||||
for child in ast.walk(node.test)
|
||||
if (name := _call_name(child)) is not None
|
||||
}
|
||||
raises_session_expired = any(
|
||||
isinstance(child, ast.Raise)
|
||||
and child.exc is not None
|
||||
and _call_name(child.exc) == "BrowserSessionExpiredError"
|
||||
for statement in node.body
|
||||
for child in ast.walk(statement)
|
||||
)
|
||||
return bool(detectors & detector_names) and raises_session_expired
|
||||
|
||||
|
||||
def _is_navigation_session_wrapper(node: ast.Try) -> bool:
|
||||
if len(node.body) != 1 or node.orelse or node.finalbody or len(node.handlers) != 1:
|
||||
return False
|
||||
navigation = node.body[0]
|
||||
if not (
|
||||
isinstance(navigation, ast.Expr)
|
||||
and isinstance(navigation.value, ast.Call)
|
||||
and _call_name(navigation.value) == "goto"
|
||||
):
|
||||
return False
|
||||
handler_body = node.handlers[0].body
|
||||
return (
|
||||
len(handler_body) == 2
|
||||
and _is_session_expiry_guard(handler_body[0])
|
||||
and isinstance(handler_body[1], ast.Raise)
|
||||
and handler_body[1].exc is None
|
||||
)
|
||||
|
||||
|
||||
class _ApprovedBoundaryNormalizer(ast.NodeTransformer):
|
||||
"""Remove only the approved cookie/session-expiry adapter statements."""
|
||||
|
||||
def visit_If(self, node: ast.If) -> ast.AST | None: # noqa: N802
|
||||
if _is_session_expiry_guard(node):
|
||||
return None
|
||||
return self.generic_visit(node)
|
||||
|
||||
def visit_Try(self, node: ast.Try) -> ast.AST | list[ast.AST]: # noqa: N802
|
||||
if _is_navigation_session_wrapper(node):
|
||||
return [self.visit(statement) for statement in node.body]
|
||||
return self.generic_visit(node)
|
||||
|
||||
|
||||
def _function_ast(path: Path, function_name: str) -> str:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path))
|
||||
matches = [
|
||||
node
|
||||
for node in tree.body
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
and node.name == function_name
|
||||
]
|
||||
assert len(matches) == 1, f"expected exactly one top-level {function_name} in {path}"
|
||||
normalized = _ApprovedBoundaryNormalizer().visit(matches[0])
|
||||
assert isinstance(normalized, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
return ast.dump(normalized, annotate_fields=True, include_attributes=False)
|
||||
|
||||
|
||||
def _ast_sha256(function_ast: str) -> str:
|
||||
return hashlib.sha256(function_ast.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def test_ast_snapshot_covers_exactly_the_reviewed_business_functions() -> None:
|
||||
assert set(SOURCE_AST_SHA256) == set(FUNCTION_CASES)
|
||||
assert set(SOURCE_AST_SHA256).isdisjoint(REVIEWED_DEFECT_FIX_AST_SHA256)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "function_name"),
|
||||
REVIEWED_DEFECT_FIX_AST_SHA256,
|
||||
ids=lambda value: value,
|
||||
)
|
||||
def test_reviewed_local_defect_fixes_match_target_snapshot(
|
||||
filename: str,
|
||||
function_name: str,
|
||||
) -> None:
|
||||
target_ast = _function_ast(TARGET_COLLECTOR_ROOT / filename, function_name)
|
||||
|
||||
assert _ast_sha256(target_ast) == REVIEWED_DEFECT_FIX_AST_SHA256[
|
||||
(filename, function_name)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "function_name"),
|
||||
FUNCTION_CASES,
|
||||
ids=lambda value: value,
|
||||
)
|
||||
def test_migrated_core_business_logic_matches_reviewed_ast_snapshot(
|
||||
filename: str,
|
||||
function_name: str,
|
||||
) -> None:
|
||||
target_ast = _function_ast(TARGET_COLLECTOR_ROOT / filename, function_name)
|
||||
|
||||
expected = TARGET_AST_SHA256_OVERRIDES.get(
|
||||
(filename, function_name),
|
||||
SOURCE_AST_SHA256[(filename, function_name)],
|
||||
)
|
||||
assert _ast_sha256(target_ast) == expected, (
|
||||
f"{filename}:{function_name} changed the reviewed source business logic; "
|
||||
"only explicitly reviewed target compatibility adapters may differ"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not SOURCE_COLLECTOR_ROOT.is_dir(),
|
||||
reason="local D:\\shop-data-flow source project is unavailable",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "function_name"),
|
||||
FUNCTION_CASES,
|
||||
ids=lambda value: value,
|
||||
)
|
||||
def test_reviewed_snapshot_and_migration_match_current_local_source(
|
||||
filename: str,
|
||||
function_name: str,
|
||||
) -> None:
|
||||
source_ast = _function_ast(SOURCE_COLLECTOR_ROOT / filename, function_name)
|
||||
target_ast = _function_ast(TARGET_COLLECTOR_ROOT / filename, function_name)
|
||||
expected = SOURCE_AST_SHA256[(filename, function_name)]
|
||||
|
||||
assert _ast_sha256(source_ast) == expected, (
|
||||
f"{filename}:{function_name} changed in D:\\shop-data-flow; review the "
|
||||
"source change before deliberately refreshing this snapshot"
|
||||
)
|
||||
override = TARGET_AST_SHA256_OVERRIDES.get((filename, function_name))
|
||||
if override:
|
||||
assert _ast_sha256(target_ast) == override
|
||||
else:
|
||||
assert target_ast == source_ast, (
|
||||
f"{filename}:{function_name} no longer matches the current source project"
|
||||
)
|
||||
Reference in New Issue
Block a user