feat: expand module workflows, dynamic config, notifications and console API

This commit is contained in:
2026-09-04 09:49:37 +08:00
parent 8df5266abb
commit b124d757b0
309 changed files with 89358 additions and 6232 deletions
@@ -14,21 +14,17 @@ from gyxx_flow.modules.shop_intelligence.collectors import (
)
class _PlaywrightContext:
class _ScraplingBrowserContext:
def __init__(self, page: object) -> None:
self.page = page
def __enter__(self):
context = SimpleNamespace(
pages=[self.page],
new_page=lambda: self.page,
close=lambda: None,
)
browser = SimpleNamespace(
new_context=lambda **kwargs: context,
close=lambda: None,
)
chromium = SimpleNamespace(launch=lambda **kwargs: browser)
return SimpleNamespace(chromium=chromium)
return SimpleNamespace(context=context)
def __exit__(self, exc_type, exc, traceback) -> None:
return None
@@ -39,8 +35,8 @@ def test_jd_shop_collector_returns_nonzero_when_collection_fails(
) -> None:
monkeypatch.setattr(
jd_data_collector,
"sync_playwright",
lambda: _PlaywrightContext(object()),
"ScraplingBrowser",
lambda **_kwargs: _ScraplingBrowserContext(object()),
)
monkeypatch.setattr(
jd_data_collector,
@@ -88,8 +84,8 @@ def test_jd_peer_collector_returns_nonzero_when_collection_fails(
) -> None:
monkeypatch.setattr(
jd_peer_store_data_collector,
"sync_playwright",
lambda: _PlaywrightContext(object()),
"ScraplingBrowser",
lambda **_kwargs: _ScraplingBrowserContext(object()),
)
monkeypatch.setattr(
jd_peer_store_data_collector,
@@ -254,6 +254,53 @@ class _LateShopIdentityPage:
self.wait_calls += 1
class _RetryNavigationPage(_LateShopIdentityPage):
def __init__(self, *, failures: int, url: str | None = None) -> None:
super().__init__()
self.failures = failures
self.goto_calls = 0
if url is not None:
self.url = url
def goto(self, *args: object, **kwargs: object) -> None:
del args, kwargs
self.goto_calls += 1
if self.goto_calls <= self.failures:
raise RuntimeError(
"Page.goto: net::ERR_ABORTED at "
f"{appeal.CAMPAIGN_SQUARE_URL} Call log: - navigating"
)
class _FeedbackTextControl:
def __init__(self, text: str) -> None:
self.text_value = text
def count(self) -> int:
return 1
def nth(self, index: int) -> _FeedbackTextControl:
assert index == 0
return self
def is_visible(self, timeout: int | None = None) -> bool:
del timeout
return True
def inner_text(self, timeout: int | None = None) -> str:
del timeout
return self.text_value
class _FeedbackPage:
def __init__(self, items: list[_VisibleControl]) -> None:
self.items = items
def locator(self, selector: str) -> _LocatorCollection:
assert "toast" in selector
return _LocatorCollection(self.items)
class _FakeClock:
def __init__(self) -> None:
self.now = 0.0
@@ -319,6 +366,38 @@ class _DynamicPanel:
return _EmptyLocator()
class _TransientTextPanel(_DynamicPanel):
def __init__(
self,
values: tuple[str, ...],
*,
transient_failures: int,
) -> None:
super().__init__(values, action_count=0)
self.transient_failures = transient_failures
def inner_text(self, timeout: int | None = None) -> str:
del timeout
self.read_calls += 1
if self.transient_failures > 0:
self.transient_failures -= 1
raise RuntimeError("drawer locator detached during transition")
return next(self.values, self.latest)
class _DetachedFeedbackControl(_TransientTextPanel):
def count(self) -> int:
return 1
def nth(self, index: int) -> _DetachedFeedbackControl:
assert index == 0
return self
def is_visible(self, timeout: int | None = None) -> bool:
del timeout
return True
class _PanelNode(_VisibleControl):
def __init__(self, name: str, box: dict[str, float]) -> None:
super().__init__()
@@ -465,6 +544,90 @@ def test_shop_identity_can_render_after_campaign_content(
assert page.wait_calls == 1
def test_navigation_recovers_after_first_abort(
monkeypatch: pytest.MonkeyPatch,
) -> None:
page = _RetryNavigationPage(failures=1)
monkeypatch.setattr(
appeal,
"_body_text",
lambda page: "活动广场 待办事项 待改价 9 光影行星箱包旗舰店",
)
appeal._navigate_to_campaign_square( # noqa: SLF001
page,
expected_shop_name=appeal.EXPECTED_SHOP_NAME,
)
assert page.goto_calls == 2
assert page.wait_calls == 1
def test_navigation_aborted_to_login_page_reports_session_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
page = _RetryNavigationPage(
failures=2,
url=(
"https://fxg.jinritemai.com/login/common"
"?extra=%7B%22target_url%22%3A%22https%3A%2F%2F"
"fxg.jinritemai.com%2Fffa%2Fmerchant%2Fcampaign-square%22%7D"
),
)
monkeypatch.setattr(appeal, "_body_text", lambda page: "")
with pytest.raises(appeal.PriceAppealSessionError, match="Cookie 已失效"):
appeal._navigate_to_campaign_square( # noqa: SLF001
page,
expected_shop_name=appeal.EXPECTED_SHOP_NAME,
)
assert page.goto_calls == 2
def test_navigation_aborted_twice_reports_navigation_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
page = _RetryNavigationPage(failures=2)
monkeypatch.setattr(appeal, "_body_text", lambda page: "")
with pytest.raises(appeal.PriceAppealError, match="活动广场导航失败") as exc:
appeal._navigate_to_campaign_square( # noqa: SLF001
page,
expected_shop_name=appeal.EXPECTED_SHOP_NAME,
)
assert page.goto_calls == 2
assert "ERR_ABORTED" in str(exc.value)
def test_pending_content_signature_returns_none_while_drawer_detached(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
panel = _TransientTextPanel(
("待改价 报名信息 商品 ID: 12345678",),
transient_failures=100,
)
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
assert appeal._pending_content_signature(panel) is None # noqa: SLF001
def test_visible_feedback_texts_skips_detached_feedback_node(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
detached = _DetachedFeedbackControl(("提交成功",), transient_failures=100)
healthy = _FeedbackTextControl(" 提交成功 ")
page = _FeedbackPage([detached, healthy])
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
assert appeal._visible_feedback_texts(page) == ("提交成功",)
def test_pending_count_can_render_after_campaign_content(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -801,6 +964,111 @@ def test_item_panel_waits_for_async_product_and_sku_content(
assert page.wait_calls == 2
def test_item_panel_reacquires_after_transient_drawer_detach(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
page = _PendingMetricPage([], clock=clock)
expected_item_id = "3774363265803616739"
detached_panel = _TransientTextPanel(
("选择申诉SKU",),
transient_failures=1,
)
ready_panel = _DynamicPanel(
(
"选择申诉SKU 商品 ID: 3774363265803616739 "
"SKU规格 ID: 3619462493048834 原价 ¥839",
),
action_count=0,
)
panels = iter((detached_panel, ready_panel))
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False)
monkeypatch.setattr(
appeal,
"_find_active_panel",
lambda page, markers: next(panels, ready_panel),
)
result = appeal._wait_for_item_panel_ready( # noqa: SLF001
page,
("选择申诉SKU",),
expected_item_id=expected_item_id,
timeout_seconds=2,
)
assert result is ready_panel
assert detached_panel.read_calls == 2
assert ready_panel.read_calls == 1
assert page.wait_calls == 1
def test_item_panel_persistent_read_failure_still_fails_closed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
page = _PendingMetricPage([], clock=clock)
panel = _TransientTextPanel(
("选择申诉SKU",),
transient_failures=100,
)
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False)
monkeypatch.setattr(
appeal,
"_find_active_panel",
lambda page, markers: panel,
)
with pytest.raises(appeal.PriceAppealError, match="未加载目标商品 ID") as exc:
appeal._wait_for_item_panel_ready( # noqa: SLF001
page,
("选择申诉SKU",),
expected_item_id="3774363265803616739",
timeout_seconds=1,
)
assert isinstance(exc.value.__cause__, appeal.PriceAppealError)
assert str(exc.value.__cause__) == "无法读取活动抽屉内容"
def test_verify_item_identity_recovers_after_transient_detach(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
panel = _TransientTextPanel(
(
"选择申诉SKU 商品 ID: 3774363265803616739 "
"SKU规格 ID: 3619462493048834 原价 ¥839",
),
transient_failures=1,
)
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
appeal._verify_item_identity(panel, "3774363265803616739") # noqa: SLF001
assert panel.read_calls == 2
def test_verify_item_identity_persistent_read_failure_fails_closed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = _FakeClock()
panel = _TransientTextPanel(
("选择申诉SKU",),
transient_failures=100,
)
monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic)
monkeypatch.setattr(appeal.time, "sleep", clock.advance)
with pytest.raises(appeal.PriceAppealError, match="与待处理行不一致") as exc:
appeal._verify_item_identity(panel, "3774363265803616739") # noqa: SLF001
assert isinstance(exc.value.__cause__, appeal.PriceAppealError)
assert str(exc.value.__cause__) == "无法读取活动抽屉内容"
def test_pending_panel_with_headers_only_never_becomes_ready(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -67,6 +67,37 @@ def test_market_category_option_uses_visible_titled_menu_item() -> None:
assert option.clicked
class _UnstableOption(_Locator):
def scroll_into_view_if_needed(self, **_kwargs) -> None:
return None
def click(self, **_kwargs) -> None:
raise TimeoutError(
"Locator.click: Timeout 5000ms exceeded ... element is not stable"
)
def test_market_category_option_falls_back_to_native_click_when_unstable() -> None:
option = _UnstableOption()
page = SimpleNamespace(
locator=lambda _selector: option,
evaluate=lambda *_args, **_kwargs: True,
)
assert collector._click_market_category_option(page, "双肩包")
assert option.waited
def test_market_category_option_fails_closed_when_native_click_also_fails() -> None:
option = _UnstableOption()
page = SimpleNamespace(
locator=lambda _selector: option,
evaluate=lambda *_args, **_kwargs: False,
)
assert not collector._click_market_category_option(page, "双肩包")
def test_select_market_category_requires_path_and_fresh_table(monkeypatch) -> None:
expected = collector._expected_market_category_path("双肩包")
paths = iter([collector._expected_market_category_path("腰包/胸包"), expected])
@@ -0,0 +1,106 @@
from __future__ import annotations
from datetime import date
import pytest
from gyxx_flow.modules.shop_intelligence.collectors import jd_data_collector as collector
class _FlakyCellLocator:
def __init__(self) -> None:
self.click_calls = 0
def scroll_into_view_if_needed(self, **_kwargs) -> None:
return None
def click(self, **_kwargs) -> None:
self.click_calls += 1
raise TimeoutError("element is not visible")
class _DatePickerPage:
"""Stub page: date already unset, month aligned, cell found, then results."""
def __init__(self, native_click_result: bool) -> None:
self.cell = _FlakyCellLocator()
self.native_click_result = native_click_result
self.calls = 0
self.native_click_script: str | None = None
def evaluate(self, script: str, arg: object = None):
del arg
self.calls += 1
if self.calls == 1:
assert "textContent" in script # already-set check
return False
if self.calls == 2:
assert "targetYear" in script # month navigation
return {"status": "aligned"}
if self.calls == 3:
assert "gridcell" in script # locate target day cell
return {"status": "found_cell", "id": "__jd_target"}
assert self.calls == 4
assert "document.getElementById" in script # native click fallback
self.native_click_script = script
return self.native_click_result
def locator(self, selector: str) -> _FlakyCellLocator:
assert selector == "#__jd_target"
return self.cell
def wait_for_timeout(self, _milliseconds: int) -> None:
return None
def _prepare(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
collector,
"get_last_week_range",
lambda: (date(2026, 8, 10), date(2026, 8, 16)),
)
monkeypatch.setattr(collector, "save_date_picker_debug", lambda *_args, **_kwargs: None)
def test_old_datepicker_cell_click_falls_back_to_native_click(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_prepare(monkeypatch)
page = _DatePickerPage(native_click_result=True)
collector.select_last_week_in_old_datepicker(page)
assert page.cell.click_calls == 1
assert page.native_click_script is not None
assert page.calls == 4
def test_old_datepicker_fails_closed_when_native_click_also_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_prepare(monkeypatch)
page = _DatePickerPage(native_click_result=False)
with pytest.raises(RuntimeError, match="旧版日期控件未能选择上周日期"):
collector.select_last_week_in_old_datepicker(page)
assert page.cell.click_calls == 1
assert page.native_click_script is not None
def test_old_datepicker_happy_path_uses_playwright_click(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_prepare(monkeypatch)
page = _DatePickerPage(native_click_result=True)
class StableCell(_FlakyCellLocator):
def click(self, **_kwargs) -> None:
self.click_calls += 1
page.cell = StableCell()
collector.select_last_week_in_old_datepicker(page)
assert page.cell.click_calls == 1
assert page.native_click_script is None
@@ -48,6 +48,7 @@ def test_category_menu_closes_stale_portals_before_opening_selector() -> None:
class Trigger:
def __init__(self, page) -> None:
self.page = page
self.first = self
def click(self, **_kwargs) -> None:
self.page.menu_count = 1
@@ -255,3 +256,129 @@ def test_failed_category_switch_never_extracts_previous_table(
assert "类目切换或榜单刷新失败" in result["男士双肩包"]["error"]
assert result["男士单肩/斜挎包"]["rank"] == "7"
assert extracted == ["called"]
class _SearchBox:
def __init__(self) -> None:
self.calls: list[str] = []
def is_visible(self, **_kwargs) -> bool:
return True
def click(self, **_kwargs) -> None:
self.calls.append("click")
def fill(self, value: str, **_kwargs) -> None:
self.calls.append(f"fill:{value}")
def press(self, key: str, **_kwargs) -> None:
self.calls.append(f"press:{key}")
class _OneLocator:
def __init__(self, item: _SearchBox) -> None:
self.item = item
@property
def first(self) -> _OneLocator:
return self
def count(self) -> int:
return 1
def is_visible(self, **_kwargs) -> bool:
return self.item.is_visible()
def click(self, **_kwargs) -> None:
self.item.click()
def fill(self, value: str, **_kwargs) -> None:
self.item.fill(value)
def press(self, key: str, **_kwargs) -> None:
self.item.press(key)
class _EmptyLocator:
@property
def first(self) -> _EmptyLocator:
return self
def count(self) -> int:
return 0
def is_visible(self, **_kwargs) -> bool:
raise TimeoutError("no elements matched")
def click(self, **_kwargs) -> None:
raise TimeoutError("no elements matched")
class _SearchPage:
def __init__(self, box: _SearchBox, *, section_result: str | None) -> None:
self.box = box
self.section_result = section_result
self.evaluate_calls = 0
def evaluate(self, _script: str, _arg: object = None):
self.evaluate_calls += 1
return self.section_result
def locator(self, selector: str) -> _OneLocator | _EmptyLocator:
if selector.startswith("#__peer_search_input"):
return _OneLocator(self.box)
if selector == "input":
return _OneLocator(self.box)
if "店铺" in selector:
return _OneLocator(self.box)
return _EmptyLocator()
def wait_for_timeout(self, _milliseconds: int) -> None:
return None
def test_search_shop_prefers_trade_rank_section_input() -> None:
box = _SearchBox()
page = _SearchPage(box, section_result="__peer_search_input_1")
collector.search_shop(page, "Bellroy")
assert page.evaluate_calls == 1
assert box.calls == [
"click",
"fill:",
"fill:Bellroy",
"press:Enter",
]
def test_search_shop_falls_back_to_legacy_selectors_without_section() -> None:
box = _SearchBox()
page = _SearchPage(box, section_result=None)
collector.search_shop(page, "Bellroy")
assert page.evaluate_calls == 1
assert box.calls == [
"click",
"fill:",
"fill:Bellroy",
"press:Enter",
]
def test_locate_peer_search_input_returns_none_when_evaluate_fails() -> None:
box = _SearchBox()
class FailingPage(_SearchPage):
def evaluate(self, _script: str, _arg: object = None):
raise RuntimeError("page crashed")
assert collector._locate_peer_search_input(FailingPage(box, section_result="x")) is None
def test_locate_peer_search_input_returns_none_without_section() -> None:
box = _SearchBox()
page = _SearchPage(box, section_result=None)
assert collector._locate_peer_search_input(page) is None
@@ -27,7 +27,7 @@ def test_browser_runtime_defaults_to_bundled_chromium(monkeypatch):
def test_browser_runtime_can_use_installed_chrome(monkeypatch):
monkeypatch.setenv("GYXX_BROWSER_CHANNEL", "chrome")
assert browser_runtime_options() == {"channel": "chrome"}
assert browser_runtime_options() == {"real_chrome": True}
class DateRangeTests(unittest.TestCase):
@@ -455,6 +455,36 @@ def test_shop_nonpositive_persistence_stops_feishu_write(
assert errors == [f"{platform_code} DB 持久化失败: 未写入任何记录"]
@pytest.mark.parametrize(
("platform", "persist_name"),
[
(run_shop.JD_PLATFORM, "persist_jd_shop"),
(run_shop.DY_PLATFORM, "persist_dy_shop"),
],
)
def test_shop_skip_feishu_preserves_database_write_without_calling_writer(
monkeypatch,
platform: str,
persist_name: str,
) -> None:
monkeypatch.setattr(run_shop, "ensure_ai_analysis", lambda *_args: "")
monkeypatch.setattr(run_shop, persist_name, lambda _path: 1)
monkeypatch.setattr(
run_shop,
"write_shop",
lambda *_args: pytest.fail("weekly shop workflow must not write Feishu"),
)
errors = run_shop._persist_and_write(
platform,
True,
"D:/fresh-shop.json",
skip_feishu=True,
)
assert errors == []
@pytest.mark.parametrize(
("platform", "persist_name", "platform_code"),
[
@@ -625,19 +655,16 @@ def test_jd_shop_uses_bound_profile_port_and_persists_state(
context = Context()
class Chromium:
def launch_persistent_context(self, **kwargs):
class Scrapling:
def __init__(self, **kwargs):
observed.update(kwargs)
return context
class Playwright:
chromium = Chromium()
self.context = context
def __enter__(self):
return self
def __exit__(self, *_args) -> None:
return None
context.close()
cookie_file = tmp_path / "state" / "cookies.json"
storage_file = tmp_path / "state" / "storage.json"
@@ -646,7 +673,7 @@ def test_jd_shop_uses_bound_profile_port_and_persists_state(
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22105")
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
monkeypatch.setattr(jd_data_collector, "sync_playwright", Playwright)
monkeypatch.setattr(jd_data_collector, "ScraplingBrowser", Scrapling)
monkeypatch.setattr(jd_data_collector, "step_login", lambda *_args: None)
monkeypatch.setattr(
jd_data_collector,
@@ -670,8 +697,13 @@ def test_jd_shop_uses_bound_profile_port_and_persists_state(
assert jd_data_collector.main(["--password", "test-only"]) == 0
assert observed["user_data_dir"] == str(profile)
assert "--remote-debugging-port=22105" in observed["args"]
assert "--no-proxy-server" in observed["args"]
assert "--remote-debugging-port=22105" in observed["extra_flags"]
assert "--no-proxy-server" in observed["extra_flags"]
assert observed["additional_args"]["viewport"] == {
"width": 1920,
"height": 1080,
}
assert observed["retries"] == 1
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "session"
assert storage_file.is_file()
assert context.closed is True
@@ -699,19 +731,16 @@ def test_jd_peer_uses_bound_profile_port_and_persists_state(
context = Context()
class Chromium:
def launch_persistent_context(self, **kwargs):
class Scrapling:
def __init__(self, **kwargs):
observed.update(kwargs)
return context
class Playwright:
chromium = Chromium()
self.context = context
def __enter__(self):
return self
def __exit__(self, *_args) -> None:
return None
context.close()
cookie_file = tmp_path / "state" / "cookies.json"
storage_file = tmp_path / "state" / "storage.json"
@@ -720,7 +749,11 @@ def test_jd_peer_uses_bound_profile_port_and_persists_state(
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22106")
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
monkeypatch.setattr(jd_peer_store_data_collector, "sync_playwright", Playwright)
monkeypatch.setattr(
jd_peer_store_data_collector,
"ScraplingBrowser",
Scrapling,
)
monkeypatch.setattr(jd_peer_store_data_collector, "step_login", lambda *_args: None)
monkeypatch.setattr(
jd_peer_store_data_collector,
@@ -765,7 +798,12 @@ def test_jd_peer_uses_bound_profile_port_and_persists_state(
assert jd_peer_store_data_collector.main(["--password", "test-only"]) == 0
assert observed["user_data_dir"] == str(profile)
assert "--remote-debugging-port=22106" in observed["args"]
assert "--remote-debugging-port=22106" in observed["extra_flags"]
assert observed["additional_args"]["viewport"] == {
"width": 1920,
"height": 1080,
}
assert observed["retries"] == 1
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "peer-session"
assert storage_file.is_file()
assert context.closed is True
@@ -133,6 +133,12 @@ TARGET_AST_SHA256_OVERRIDES = {
("jd_data_collector.py", "select_last_week_any_day"): (
"8f6aed11b39d8e19ec4146e9b32eee96f5977fdbef694311534346bacec5d847"
),
# Reviewed after production runs showed Playwright's actionability check
# rejecting the legacy date-picker cell ("element is not visible") while the
# panel re-renders; a native click on the same node is the approved fallback.
("jd_data_collector.py", "select_last_week_in_old_datepicker"): (
"43d1056eeac1faeff32f0c01d0867e6673d2ab57999a14a3ebeb1b51c40075d4"
),
("jd_data_collector.py", "step_shop_star_and_trade"): (
"77aa873543861f1fad63d0bd1ace60528ab20592b0bffef3d46ad192f2103161"
),