feat(cookie-gate): 调度器恢复重跑接线与控制台健康自检

调度器维护循环挂载 CookieGateRecoveryManager,登录态恢复后按原业务日期自动
重跑被阻断工作流并飞书确认;工作流级 required_accounts 声明接入目录解析;
采集器抛 CookieExpiredError 退出 75;控制台新增 /api/health 自检端点与进程内
看门狗,配合 deploy/watchdog 双层保活;安装脚本同步。
This commit is contained in:
wangyunlong committed 2026-10-09 13:45:29 +08:00
1 parent 78ea4641ce
commit 130f06b320
13 files changed
+318 -77

No files matched your search

+19
View File
@@ -168,3 +168,22 @@ def looks_like_login_url(url: str) -> bool:
path = parsed.path.casefold()
markers = ("login", "passport", "signin", "sign-in")
return any(marker in hostname or marker in path for marker in markers)
class CookieExpiredError(RuntimeError):
"""A platform login state expired and no automatic recovery exists.
Collectors raise this instead of waiting for a human QR/SMS login: the
account cookie gate has already blocked unverified accounts before the
run, so reaching this point means the session died mid-run. The CLI maps
it to ``COOKIE_SKIP_EXIT_CODE`` so the engine records a cookie skip.
"""
def cookie_expired_message(account_id: str, platform: str) -> str:
"""One standard, operator-actionable expiry message across platforms."""
return (
f"{platform} 登录态已失效:请执行 `uv run gyxx accounts login {account_id}` "
"完成扫码/验证码重新登录(或等待飞书通知后处理)"
)
+16
View File
@@ -79,6 +79,10 @@ class WorkflowEntry:
steps: tuple[WorkflowStepEntry, ...] = ()
notification_workflow_id: str | None = None
topic_config: WorkflowTopicConfig | None = None
# Orchestrator entries whose own binding carries no runtime account while
# the spawned child collectors do; the cookie gate probes these before
# the workflow runs.
required_accounts: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
@@ -218,6 +222,9 @@ def _parse_workflow(item: Any) -> WorkflowEntry:
if trigger == "scheduled" and (not isinstance(task_name, str) or not task_name):
raise CatalogError(f"scheduled workflow requires source task name: {workflow_id}")
topic_config = _parse_topic_config(item.get("topic_config"), workflow_id)
required_accounts = _parse_required_accounts(
item.get("required_accounts", []), workflow_id
)
return WorkflowEntry(
workflow_id=workflow_id,
module=module,
@@ -229,9 +236,18 @@ def _parse_workflow(item: Any) -> WorkflowEntry:
note=item.get("note"),
steps=steps,
topic_config=topic_config,
required_accounts=required_accounts,
)
def _parse_required_accounts(value: Any, workflow_id: str) -> tuple[str, ...]:
if not isinstance(value, list) or not all(
isinstance(item, str) and _WORKFLOW_ID.fullmatch(item) for item in value
):
raise CatalogError(f"invalid required_accounts for {workflow_id}")
return tuple(dict.fromkeys(value))
def _parse_topic_config(value: Any, workflow_id: str) -> WorkflowTopicConfig | None:
if value is None:
return None
+41 -1
View File
@@ -703,7 +703,19 @@ def _python_scheduler(
if enable_account_keepalive and not dry_run:
from gyxx_flow.accounts import AccountKeepaliveManager
maintenance = AccountKeepaliveManager(settings).tick
keepalive_manager = AccountKeepaliveManager(settings)
recovery_manager = _build_cookie_recovery_manager(
settings, launcher=launcher, keepalive=keepalive_manager
)
def maintenance() -> None: # noqa: D107 - scheduler maintenance chain
keepalive_manager.tick()
if recovery_manager is not None:
try:
recovery_manager.tick()
except Exception: # noqa: BLE001 - 自动重跑不得影响调度心跳
pass
return PythonScheduler(
catalog,
settings.data_root,
@@ -717,6 +729,29 @@ def _python_scheduler(
)
def _build_cookie_recovery_manager(
settings: Settings,
*,
launcher: object,
keepalive: object,
):
"""Build the cookie-gate recovery rerun manager; None when disabled."""
from gyxx_flow.cookie_gate import (
CookieGate,
CookieGateRecoveryManager,
gate_enabled,
)
if not gate_enabled():
return None
try:
gate = CookieGate(settings, keepalive=keepalive) # type: ignore[arg-type]
except Exception: # noqa: BLE001 - 无集成配置时没有可恢复的账号
return None
return CookieGateRecoveryManager(gate, launcher=launcher)
def _schedule_run_command(
arguments: argparse.Namespace,
catalog: WorkflowCatalog,
@@ -885,6 +920,11 @@ def _execute_once(
dry_run: bool,
settings: Settings,
) -> tuple[int, dict[str, object]]:
if not dry_run:
# 每账号 Cookie 健康门控:dry-run 不注入,也不做任何探测/通知。
from gyxx_flow.cookie_gate import apply_cookie_gates
workflow = apply_cookie_gates(workflow, settings=settings)
context = RunContext.create(
workflow.workflow_id,
business_date.isoformat(),
+118 -2
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import calendar
import hashlib
import hmac
import http.client
import ipaddress
import json
import os
@@ -15,6 +16,7 @@ import sys
import tempfile
import threading
import time
import traceback
import unicodedata
import uuid
from contextlib import suppress
@@ -24,7 +26,7 @@ from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from importlib import resources
from pathlib import Path
from typing import Any, Mapping, Protocol, Sequence, TextIO, cast
from typing import Any, Callable, Mapping, Protocol, Sequence, TextIO, cast
from urllib.parse import parse_qs, unquote, urlsplit
from zoneinfo import ZoneInfo
@@ -171,7 +173,6 @@ _WORKFLOW_NAMES = {
"content.metrics.daily": "合作达人内容指标日报采集",
"content.metrics.self_operated.daily": "自营内容指标日报采集",
"content.marketing_report.daily": "营销日报生成与发送",
"content.relogin.weekly": "内容平台登录态刷新",
"content.creator_report.monthly": "达人月报生成",
"content.summary.monthly": "内容月度汇总",
"content.comments.weekly": "多平台评论周采集",
@@ -3736,6 +3737,7 @@ class WorkflowConsoleHTTPServer(ThreadingHTTPServer):
self.console_service = service
self.console_token = token
self.bind_host = bind_host
self.started_at = time.time()
super().__init__(server_address, WorkflowConsoleRequestHandler)
def handle_error(
@@ -3796,6 +3798,29 @@ class WorkflowConsoleRequestHandler(BaseHTTPRequestHandler):
if method in {"GET", "HEAD"} and path == "/assets/app.js":
self._static("app.js", "text/javascript; charset=utf-8", head=method == "HEAD")
return
if method == "GET" and path == "/api/health":
# 自检探针端点:不鉴权、不触外部依赖,只反映进程与调度锁存活,
# 供进程内看门狗与 deploy/watchdog 保活脚本探测。
try:
scheduler = _scheduler_process_from_lock(
self.console_server.console_service.settings
)
except Exception:
scheduler = None
self._json(
HTTPStatus.OK,
{
"ok": True,
"pid": os.getpid(),
"started_at": self.console_server.started_at,
"uptime_seconds": round(
time.time() - self.console_server.started_at, 1
),
"scheduler_running": scheduler is not None,
"scheduler_pid": scheduler.pid if scheduler else None,
},
)
return
if method == "GET" and path == "/api/overview":
self._require_api_access(mutation=False)
self._json(HTTPStatus.OK, self.console_server.console_service.overview())
@@ -4020,6 +4045,9 @@ class WorkflowConsoleRequestHandler(BaseHTTPRequestHandler):
except (BrokenPipeError, ConnectionResetError):
return
except Exception:
# 前端只拿到固定文案;完整堆栈落到 stderr(随启动重定向进
# var/logs/console-restarts 或服务日志),否则故障无法定位。
traceback.print_exc()
self._json_error(HTTPStatus.INTERNAL_SERVER_ERROR, "控制台处理请求失败")
def _require_api_access(self, *, mutation: bool) -> None:
@@ -4190,6 +4218,92 @@ def create_console_server(
)
CONSOLE_WATCHDOG_EXIT_CODE = 70
def _probe_console_health(port: int, timeout_seconds: float) -> bool:
# 只连本机回环端口的固定路径 /api/health,探测自身 HTTP 服务往返。
connection = http.client.HTTPConnection("127.0.0.1", port, timeout=timeout_seconds)
try:
connection.request("GET", "/api/health")
response = connection.getresponse()
response.read()
return response.status == HTTPStatus.OK
except Exception:
return False
finally:
connection.close()
def _console_watchdog(
port: int,
*,
interval_seconds: float,
failure_limit: int,
probe_timeout_seconds: float,
stopped: threading.Event,
probe: Callable[[int, float], bool] | None = None,
exit_process: Callable[[int], None] = os._exit,
sleep: Callable[[float], None] = time.sleep,
) -> None:
"""进程内自检:周期探测自身 /api/health,连续失败则退出交给守护拉起。
HTTP accept 线程活着但工作线程全部卡死(锁竞争/句柄耗尽)时,外部
静态页能打开而 API 全挂——正是"控制台暂时不可用"的形态;只有真实
走一遍 HTTP 往返才能暴露,所以探针必须打 socket 而不是查内存状态。
"""
run_probe = probe or _probe_console_health
failures = 0
while not stopped.is_set():
if run_probe(port, probe_timeout_seconds):
failures = 0
else:
failures += 1
print(
f"⚠️ 控制台自检失败 {failures}/{failure_limit}: 127.0.0.1:{port}/api/health",
file=sys.stderr,
flush=True,
)
if failures >= failure_limit:
print(
"❌ 控制台自检连续失败,进程退出等待守护脚本拉起",
file=sys.stderr,
flush=True,
)
exit_process(CONSOLE_WATCHDOG_EXIT_CODE)
return
sleep(interval_seconds)
def _start_console_watchdog(server: WorkflowConsoleHTTPServer) -> threading.Event:
interval = max(
5.0, float(os.environ.get("GYXX_CONSOLE_WATCHDOG_INTERVAL_SECONDS", "30"))
)
limit = int(os.environ.get("GYXX_CONSOLE_WATCHDOG_FAILURES", "3"))
enabled = (
os.environ.get("GYXX_CONSOLE_WATCHDOG", "1").strip().lower()
not in {"0", "false", "no", "off"}
and limit > 0
)
stopped = threading.Event()
if not enabled:
return stopped
thread = threading.Thread(
target=_console_watchdog,
args=(server.server_port,),
kwargs={
"interval_seconds": interval,
"failure_limit": limit,
"probe_timeout_seconds": min(5.0, interval),
"stopped": stopped,
},
name="gyxx-console-watchdog",
daemon=True,
)
thread.start()
return stopped
def serve_console(
settings: Settings,
*,
@@ -4211,11 +4325,13 @@ def serve_console(
display_host = "127.0.0.1" if host in {"0.0.0.0", "::"} else host
output.write(f"GYXX Flow console: http://{display_host}:{server.server_port}\n")
output.flush()
watchdog_stopped = _start_console_watchdog(server)
try:
server.serve_forever(poll_interval=0.5)
except KeyboardInterrupt:
pass
finally:
watchdog_stopped.set()
server.server_close()
return 0