feat(cookie-gate): 调度器恢复重跑接线与控制台健康自检

调度器维护循环挂载 CookieGateRecoveryManager,登录态恢复后按原业务日期自动
重跑被阻断工作流并飞书确认;工作流级 required_accounts 声明接入目录解析;
采集器抛 CookieExpiredError 退出 75;控制台新增 /api/health 自检端点与进程内
看门狗,配合 deploy/watchdog 双层保活;安装脚本同步。
This commit is contained in:
wangyunlong committed 2026-10-09 13:45:29 +08:00
1 parent 78ea4641ce
commit 130f06b320
13 files changed
+318 -77

No files matched your search

+5 -39
View File
@@ -10,9 +10,6 @@ from types import SimpleNamespace
import pytest
from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE
from gyxx_flow.modules.content_marketing.data.tools import (
friday_relogin_parallel as weekly_relogin,
)
@pytest.fixture
@@ -75,29 +72,6 @@ def _import_supply_script(module_name: str):
sys.stderr = stderr
def test_weekly_relogin_skips_the_entire_qr_workflow(
acceptance_environment: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
monkeypatch.setattr(
weekly_relogin.subprocess,
"Popen",
_forbidden("acceptance mode must not launch relogin subprocesses"),
)
monkeypatch.setattr(
weekly_relogin,
"_send_lark",
_forbidden("acceptance mode must not send QR screenshots"),
)
assert weekly_relogin.main() == COOKIE_SKIP_EXIT_CODE
assert "SKIPPED_COOKIE" in capsys.readouterr().out
assert "content.relogin.weekly" in acceptance_environment.read_text(
encoding="utf-8"
)
@pytest.mark.parametrize(
"module_name",
[
@@ -106,25 +80,17 @@ def test_weekly_relogin_skips_the_entire_qr_workflow(
"batch_rescrape_bilibili",
],
)
def test_batch_rescrape_relogin_never_starts_a_qr_process(
def test_batch_rescrape_has_no_relogin_capability(
module_name: str,
acceptance_environment: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
"""扫码重登能力已从批量补采工具中移除,登录统一由账号 vault 管理。"""
module = importlib.import_module(
f"gyxx_flow.modules.content_marketing.data.tools.{module_name}"
)
monkeypatch.setattr(
module.subprocess,
"call",
_forbidden("acceptance mode must not invoke a relogin script"),
)
assert module.do_relogin() is False
assert "SKIPPED_COOKIE" in capsys.readouterr().out
evidence = acceptance_environment.read_text(encoding="utf-8")
assert f"content.{module_name}.relogin" in evidence
assert not hasattr(module, "do_relogin")
assert not hasattr(module, "RELOGIN_SCRIPT")
@pytest.mark.parametrize(
+4 -6
View File
@@ -18,8 +18,8 @@ def test_catalog_maps_39_project_scheduled_tasks() -> None:
assert len(scheduled) == 39
assert len({item.source_task_name for item in scheduled}) == 39
assert Counter(item.module for item in scheduled) == {
"content_marketing": 10,
"product_commerce": 13,
"content_marketing": 9,
"product_commerce": 14,
"shop_intelligence": 12,
"supply_chain": 4,
}
@@ -80,7 +80,6 @@ def test_jd_video_upload_is_an_idempotent_scheduled_workflow() -> None:
assert step.args == (
"--anchor-record-id",
"recvrXTRWWFXeJ",
"--manual-login",
"--execute",
)
assert step.replay_policy == "idempotent"
@@ -192,7 +191,7 @@ def test_every_scheduled_workflow_has_one_valid_asia_shanghai_schedule() -> None
assert schedule.workflow_id == workflow.workflow_id
assert schedule.kind in {"daily", "weekly", "monthly", "interval_days"}
assert schedule.at.count(":") == 1
assert sum(schedule.enabled for schedule in catalog.schedules) == 27
assert sum(schedule.enabled for schedule in catalog.schedules) == 28
customer_service = catalog.schedule_for("shop.jd_customer_service.daily")
assert customer_service.at == "03:00"
assert customer_service.enabled is True
@@ -265,8 +264,7 @@ def test_scheduled_workflows_use_python_entries_and_content_tasks_are_graphs() -
"data/tools/sync_self_video_metrics.py"
)
assert catalog.schedule_for(self_operated.workflow_id).effective_times == (
"01:00",
"05:00",
"06:00",
)
backfill = next(
+1 -1
View File
@@ -54,7 +54,7 @@ def test_default_cli_registry_exposes_all_project_scheduled_workflows(
scheduled_ids = {
entry.workflow_id for entry in registry.catalog.scheduled_workflows()
}
assert len(scheduled_ids) == 35
assert len(scheduled_ids) == 39
assert scheduled_ids == {
schedule.workflow_id for schedule in registry.catalog.schedules
}
+93 -19
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import http.client
import json
import os
import re
import shutil
import subprocess
@@ -311,7 +312,7 @@ def test_notification_config_persists_app_scoped_multi_recipient_routes(
)
initial = service.notification_config()
assert len(initial["people"]) == 16
assert len(initial["people"]) == 18
assert initial["available_app_profiles"] == ["hermes-analyzer"]
assert [app["profile"] for app in initial["apps"]] == ["hermes-analyzer"]
wang = next(person for person in initial["people"] if person["id"] == "wang_yunlong")
@@ -321,7 +322,12 @@ def test_notification_config_persists_app_scoped_multi_recipient_routes(
assert wang["bindings"]["hermes-analyzer"]["open_id"] == (
"ou_7ad5fc8012e2f741afc5346e05ffd447"
)
assert all(route["configured"] is False for route in initial["routes"])
stock_alert_route = next(
route
for route in initial["routes"]
if route["workflow_id"] == "product.online_stock_alert.daily"
)
assert stock_alert_route["configured"] is True
routes = [dict(route) for route in initial["routes"]]
daily = next(
@@ -1566,7 +1572,7 @@ def test_overview_exposes_only_verified_scheduled_runs_as_cancellable(
"schema_version": 1,
"updated_at": "2026-08-14T02:00:00+00:00",
"jobs": {
"content.relogin.weekly": {
"content.notes_master.daily": {
"status": "running",
"pid": 4310,
"business_date": "2026-08-14",
@@ -1589,7 +1595,7 @@ def test_overview_exposes_only_verified_scheduled_runs_as_cancellable(
workflow_id: str,
_job: dict[str, Any],
) -> tuple[bool, Any, float | None]:
if workflow_id == "content.relogin.weekly":
if workflow_id == "content.notes_master.daily":
return True, VerifiedProcess(), 1_800_000_000.0
return False, None, None
@@ -1600,7 +1606,7 @@ def test_overview_exposes_only_verified_scheduled_runs_as_cancellable(
).overview()
workflows = {item["id"]: item for item in overview["workflows"]}
verified = workflows["content.relogin.weekly"]["scheduler_job"]
verified = workflows["content.notes_master.daily"]["scheduler_job"]
assert verified["status"] == "running"
assert verified["cancellable"] is True
assert "pid" not in verified
@@ -1614,7 +1620,7 @@ def test_service_cancels_verified_scheduled_run_and_finalizes_journal(
monkeypatch: pytest.MonkeyPatch,
console_settings: Settings,
) -> None:
workflow_id = "content.relogin.weekly"
workflow_id = "content.notes_master.daily"
_write_scheduler_state(
console_settings,
{
@@ -1906,24 +1912,13 @@ def test_content_production_preflight_rejects_non_production_modes(
relogin = next(
item
for item in catalog.workflows
if item.workflow_id == "content.relogin.weekly"
if item.workflow_id == "content.summary.weekly"
)
with pytest.raises(ConsoleRequestError, match=message):
console_module._validate_content_production_runtime(relogin, {flag: "1"})
def test_content_relogin_preflight_does_not_require_postgres() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
relogin = next(
item
for item in catalog.workflows
if item.workflow_id == "content.relogin.weekly"
)
console_module._validate_content_production_runtime(relogin, {})
@pytest.mark.parametrize(
"workflow_id",
["content.notes_master.daily", "content.metrics.daily"],
@@ -3297,7 +3292,7 @@ def test_http_notification_config_requires_revision_and_supports_multi_recipient
assert status == HTTPStatus.OK
_assert_security_headers(response_headers)
config = json.loads(body)
assert len(config["people"]) == 16
assert len(config["people"]) == 18
assert {
route["workflow_id"] for route in config["routes"]
} == set(console_module.NOTIFICATION_CAPABILITIES)
@@ -3616,3 +3611,82 @@ def test_douyin_negative_reviews_range_uses_single_run_with_env_dates(
{"business_date": "2026-09-19"},
)
assert "business_date_end" not in single
def test_api_health_reports_process_and_scheduler_state(
console_settings: Settings,
) -> None:
with _running_server(console_settings, FakeConsoleLauncher()) as server:
status, _headers, content = _http_request(server, "GET", "/api/health")
assert status == HTTPStatus.OK
payload = json.loads(content)
assert payload["ok"] is True
assert payload["pid"] == os.getpid()
assert payload["uptime_seconds"] >= 0
# 测试环境没有调度器实例锁,health 只报状态不做任何拉起。
assert payload["scheduler_running"] is False
assert payload["scheduler_pid"] is None
def test_api_health_requires_no_token(console_settings: Settings) -> None:
# 自检探针端点供进程内看门狗与保活脚本使用,必须绕过访问令牌。
with _running_server(
console_settings, FakeConsoleLauncher(), token="x" * 24
) as server:
status, _headers, _content = _http_request(server, "GET", "/api/health")
assert status == HTTPStatus.OK
def test_probe_console_health_against_real_server(
console_settings: Settings,
) -> None:
with _running_server(console_settings, FakeConsoleLauncher()) as server:
assert console_module._probe_console_health(server.server_port, 2.0) is True
assert console_module._probe_console_health(1, 0.2) is False
def test_console_watchdog_exits_after_consecutive_probe_failures() -> None:
stopped = threading.Event()
exits: list[int] = []
probes: list[int] = []
def always_failing(port: int, _timeout: float) -> bool:
probes.append(port)
return False
console_module._console_watchdog(
8765,
interval_seconds=0.01,
failure_limit=3,
probe_timeout_seconds=0.01,
stopped=stopped,
probe=always_failing,
exit_process=exits.append,
sleep=lambda _seconds: None,
)
assert exits == [console_module.CONSOLE_WATCHDOG_EXIT_CODE]
assert probes == [8765, 8765, 8765]
def test_console_watchdog_resets_failure_count_after_successful_probe() -> None:
stopped = threading.Event()
exits: list[int] = []
results = iter([False, False, True, False, False, False])
def flaky_probe(_port: int, _timeout: float) -> bool:
return next(results)
console_module._console_watchdog(
8765,
interval_seconds=0.01,
failure_limit=3,
probe_timeout_seconds=0.01,
stopped=stopped,
probe=flaky_probe,
exit_process=exits.append,
sleep=lambda _seconds: None,
)
# 成功一次清零计数:2 次失败 + 1 次成功 + 3 次失败后才退出。
assert exits == [console_module.CONSOLE_WATCHDOG_EXIT_CODE]
+3 -3
View File
@@ -103,9 +103,9 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
)
assert len(scripts.command_ids) == 60
assert len(first.command_ids) == 163
assert len(first.command_ids) == 157
bindings = [first.binding_for(command_id) for command_id in first.command_ids]
assert len({binding.cdp_port for binding in bindings}) == 163
assert len({binding.cdp_port for binding in bindings}) == 157
assert all(22000 <= binding.cdp_port <= 22999 for binding in bindings)
assert all(binding.cdp_url.startswith("http://127.0.0.1:") for binding in bindings)
assert bindings == [second.binding_for(item.script_id) for item in bindings]
@@ -450,7 +450,7 @@ def test_project_accounts_cover_all_configured_platform_login_identities() -> No
assert all(
catalog.bindings_for_account(account.account_id)
for account in catalog.accounts
if account.account_id != "tmall-ozko"
if account.account_id not in {"tmall-ozko", "xingtu"}
)
assert catalog.bindings_for_account("tmall-ozko") == ()
assert catalog.account_for("jd-shop").credential_env_names == (
+1 -1
View File
@@ -54,7 +54,7 @@ def test_python_scheduler_status_is_project_owned_and_cross_platform(tmp_path: P
assert exit_code == 0
payload = json.loads(output.getvalue())
assert payload["timezone"] == "Asia/Shanghai"
assert payload["scheduled_count"] == 35
assert payload["scheduled_count"] == 39
assert payload["state"]["schema_version"] == 1
+3 -2
View File
@@ -206,16 +206,17 @@ def test_misfire_older_than_grace_is_not_launched(tmp_path: Path) -> None:
def test_configured_schedules_disable_cookie_refresh_job() -> None:
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
assert len(catalog.schedules) == 35
assert len(catalog.schedules) == 39
assert [
schedule.workflow_id for schedule in catalog.schedules if not schedule.enabled
] == [
"content.metrics.backfill",
"content.marketing_report.daily",
"content.relogin.weekly",
"content.summary.weekly",
"content.summary.monthly",
"content.creator_report.monthly",
"shop.tmall_negative_reviews.daily",
"shop.tmall_customer_service.daily",
"product.alert.daily",
"product.style_analysis.interval",
"supply.purchase_confirmation.daily",
+2 -2
View File
@@ -47,8 +47,8 @@ def test_deployment_documents_one_scheduler_and_external_data_root() -> None:
assert "GYXX_DATA_ROOT" in guide
assert "/var/lib/gyxx-flow" in guide
assert "Windows Task Scheduler" in guide
assert len(payload["schedules"]) == 35
assert sum(item.get("enabled", True) for item in payload["schedules"]) == 25
assert len(payload["schedules"]) == 39
assert sum(item.get("enabled", True) for item in payload["schedules"]) == 28
def test_three_canonical_documents_cover_architecture_deployment_and_operations() -> None: