feat(cookie-gate): 调度器恢复重跑接线与控制台健康自检
调度器维护循环挂载 CookieGateRecoveryManager,登录态恢复后按原业务日期自动 重跑被阻断工作流并飞书确认;工作流级 required_accounts 声明接入目录解析; 采集器抛 CookieExpiredError 退出 75;控制台新增 /api/health 自检端点与进程内 看门狗,配合 deploy/watchdog 双层保活;安装脚本同步。
This commit is contained in:
1 parent
78ea4641ce
commit
130f06b320
13 files changed
+318
-77
No files matched your search
+93
-19
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -311,7 +312,7 @@ def test_notification_config_persists_app_scoped_multi_recipient_routes(
|
||||
)
|
||||
initial = service.notification_config()
|
||||
|
||||
assert len(initial["people"]) == 16
|
||||
assert len(initial["people"]) == 18
|
||||
assert initial["available_app_profiles"] == ["hermes-analyzer"]
|
||||
assert [app["profile"] for app in initial["apps"]] == ["hermes-analyzer"]
|
||||
wang = next(person for person in initial["people"] if person["id"] == "wang_yunlong")
|
||||
@@ -321,7 +322,12 @@ def test_notification_config_persists_app_scoped_multi_recipient_routes(
|
||||
assert wang["bindings"]["hermes-analyzer"]["open_id"] == (
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447"
|
||||
)
|
||||
assert all(route["configured"] is False for route in initial["routes"])
|
||||
stock_alert_route = next(
|
||||
route
|
||||
for route in initial["routes"]
|
||||
if route["workflow_id"] == "product.online_stock_alert.daily"
|
||||
)
|
||||
assert stock_alert_route["configured"] is True
|
||||
|
||||
routes = [dict(route) for route in initial["routes"]]
|
||||
daily = next(
|
||||
@@ -1566,7 +1572,7 @@ def test_overview_exposes_only_verified_scheduled_runs_as_cancellable(
|
||||
"schema_version": 1,
|
||||
"updated_at": "2026-08-14T02:00:00+00:00",
|
||||
"jobs": {
|
||||
"content.relogin.weekly": {
|
||||
"content.notes_master.daily": {
|
||||
"status": "running",
|
||||
"pid": 4310,
|
||||
"business_date": "2026-08-14",
|
||||
@@ -1589,7 +1595,7 @@ def test_overview_exposes_only_verified_scheduled_runs_as_cancellable(
|
||||
workflow_id: str,
|
||||
_job: dict[str, Any],
|
||||
) -> tuple[bool, Any, float | None]:
|
||||
if workflow_id == "content.relogin.weekly":
|
||||
if workflow_id == "content.notes_master.daily":
|
||||
return True, VerifiedProcess(), 1_800_000_000.0
|
||||
return False, None, None
|
||||
|
||||
@@ -1600,7 +1606,7 @@ def test_overview_exposes_only_verified_scheduled_runs_as_cancellable(
|
||||
).overview()
|
||||
workflows = {item["id"]: item for item in overview["workflows"]}
|
||||
|
||||
verified = workflows["content.relogin.weekly"]["scheduler_job"]
|
||||
verified = workflows["content.notes_master.daily"]["scheduler_job"]
|
||||
assert verified["status"] == "running"
|
||||
assert verified["cancellable"] is True
|
||||
assert "pid" not in verified
|
||||
@@ -1614,7 +1620,7 @@ def test_service_cancels_verified_scheduled_run_and_finalizes_journal(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
console_settings: Settings,
|
||||
) -> None:
|
||||
workflow_id = "content.relogin.weekly"
|
||||
workflow_id = "content.notes_master.daily"
|
||||
_write_scheduler_state(
|
||||
console_settings,
|
||||
{
|
||||
@@ -1906,24 +1912,13 @@ def test_content_production_preflight_rejects_non_production_modes(
|
||||
relogin = next(
|
||||
item
|
||||
for item in catalog.workflows
|
||||
if item.workflow_id == "content.relogin.weekly"
|
||||
if item.workflow_id == "content.summary.weekly"
|
||||
)
|
||||
|
||||
with pytest.raises(ConsoleRequestError, match=message):
|
||||
console_module._validate_content_production_runtime(relogin, {flag: "1"})
|
||||
|
||||
|
||||
def test_content_relogin_preflight_does_not_require_postgres() -> None:
|
||||
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
relogin = next(
|
||||
item
|
||||
for item in catalog.workflows
|
||||
if item.workflow_id == "content.relogin.weekly"
|
||||
)
|
||||
|
||||
console_module._validate_content_production_runtime(relogin, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"workflow_id",
|
||||
["content.notes_master.daily", "content.metrics.daily"],
|
||||
@@ -3297,7 +3292,7 @@ def test_http_notification_config_requires_revision_and_supports_multi_recipient
|
||||
assert status == HTTPStatus.OK
|
||||
_assert_security_headers(response_headers)
|
||||
config = json.loads(body)
|
||||
assert len(config["people"]) == 16
|
||||
assert len(config["people"]) == 18
|
||||
assert {
|
||||
route["workflow_id"] for route in config["routes"]
|
||||
} == set(console_module.NOTIFICATION_CAPABILITIES)
|
||||
@@ -3616,3 +3611,82 @@ def test_douyin_negative_reviews_range_uses_single_run_with_env_dates(
|
||||
{"business_date": "2026-09-19"},
|
||||
)
|
||||
assert "business_date_end" not in single
|
||||
|
||||
|
||||
def test_api_health_reports_process_and_scheduler_state(
|
||||
console_settings: Settings,
|
||||
) -> None:
|
||||
with _running_server(console_settings, FakeConsoleLauncher()) as server:
|
||||
status, _headers, content = _http_request(server, "GET", "/api/health")
|
||||
assert status == HTTPStatus.OK
|
||||
payload = json.loads(content)
|
||||
assert payload["ok"] is True
|
||||
assert payload["pid"] == os.getpid()
|
||||
assert payload["uptime_seconds"] >= 0
|
||||
# 测试环境没有调度器实例锁,health 只报状态不做任何拉起。
|
||||
assert payload["scheduler_running"] is False
|
||||
assert payload["scheduler_pid"] is None
|
||||
|
||||
|
||||
def test_api_health_requires_no_token(console_settings: Settings) -> None:
|
||||
# 自检探针端点供进程内看门狗与保活脚本使用,必须绕过访问令牌。
|
||||
with _running_server(
|
||||
console_settings, FakeConsoleLauncher(), token="x" * 24
|
||||
) as server:
|
||||
status, _headers, _content = _http_request(server, "GET", "/api/health")
|
||||
assert status == HTTPStatus.OK
|
||||
|
||||
|
||||
def test_probe_console_health_against_real_server(
|
||||
console_settings: Settings,
|
||||
) -> None:
|
||||
with _running_server(console_settings, FakeConsoleLauncher()) as server:
|
||||
assert console_module._probe_console_health(server.server_port, 2.0) is True
|
||||
assert console_module._probe_console_health(1, 0.2) is False
|
||||
|
||||
|
||||
def test_console_watchdog_exits_after_consecutive_probe_failures() -> None:
|
||||
stopped = threading.Event()
|
||||
exits: list[int] = []
|
||||
probes: list[int] = []
|
||||
|
||||
def always_failing(port: int, _timeout: float) -> bool:
|
||||
probes.append(port)
|
||||
return False
|
||||
|
||||
console_module._console_watchdog(
|
||||
8765,
|
||||
interval_seconds=0.01,
|
||||
failure_limit=3,
|
||||
probe_timeout_seconds=0.01,
|
||||
stopped=stopped,
|
||||
probe=always_failing,
|
||||
exit_process=exits.append,
|
||||
sleep=lambda _seconds: None,
|
||||
)
|
||||
|
||||
assert exits == [console_module.CONSOLE_WATCHDOG_EXIT_CODE]
|
||||
assert probes == [8765, 8765, 8765]
|
||||
|
||||
|
||||
def test_console_watchdog_resets_failure_count_after_successful_probe() -> None:
|
||||
stopped = threading.Event()
|
||||
exits: list[int] = []
|
||||
results = iter([False, False, True, False, False, False])
|
||||
|
||||
def flaky_probe(_port: int, _timeout: float) -> bool:
|
||||
return next(results)
|
||||
|
||||
console_module._console_watchdog(
|
||||
8765,
|
||||
interval_seconds=0.01,
|
||||
failure_limit=3,
|
||||
probe_timeout_seconds=0.01,
|
||||
stopped=stopped,
|
||||
probe=flaky_probe,
|
||||
exit_process=exits.append,
|
||||
sleep=lambda _seconds: None,
|
||||
)
|
||||
|
||||
# 成功一次清零计数:2 次失败 + 1 次成功 + 3 次失败后才退出。
|
||||
assert exits == [console_module.CONSOLE_WATCHDOG_EXIT_CODE]
|
||||
Reference in new issue
Block a user