feat(cookie-gate): 补充 cookie_gate 网关模块与 deploy/watchdog 保活脚本
cookie_gate.py(账号三级门控/阻断记录 pending_reruns/恢复重跑管理器)为此前 接线提交遗漏的模块文件;deploy/watchdog 为控制台与调度器的进程外保活脚本。
This commit is contained in:
1 parent
dc8be88e60
commit
36fe7b81f8
3 files changed
+1223
No files matched your search
@@ -0,0 +1,336 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""控制台与调度器的进程外保活(登录自启动,随用户会话常驻)。
|
||||||
|
|
||||||
|
职责(不承载任何业务调度规则,业务调度仍由 `gyxx schedule run` 唯一负责):
|
||||||
|
1. 控制台:探测 http://127.0.0.1:8765/api/health——进程不在则拉起;
|
||||||
|
进程在但探测连续失败(wedged)则结束该进程树后拉起。控制台进程内
|
||||||
|
另有看门狗(console.py `_console_watchdog`)自检失败会主动退出,此处
|
||||||
|
负责把退出后的空窗补上,两层互为补充。
|
||||||
|
2. 调度器:读 `var/state/scheduler/service.lock` 的 PID 判断存活,锁主
|
||||||
|
已死则按 install.ps1 同款参数拉起 `python -m gyxx_flow schedule run`;
|
||||||
|
调度器自身单实例锁保证不会双跑。
|
||||||
|
3. 节流:每个目标有冷却间隔;短时间反复重启触发熔断,只记录不再拉起,
|
||||||
|
避免故障风暴。
|
||||||
|
|
||||||
|
用法:
|
||||||
|
gyxx_keepalive.py # 常驻循环(Startup 的 vbs 以 pythonw 启动)
|
||||||
|
gyxx_keepalive.py --once # 单轮探测+处置后退出,供验收/手动巡检
|
||||||
|
|
||||||
|
日志:var/logs/keepalive/keepalive.log(追加)。
|
||||||
|
自身单实例:var/state/keepalive.lock(PID 存活检测,重复启动直接退出)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import ctypes
|
||||||
|
import http.client
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
ENV_FILES = [
|
||||||
|
Path(os.getenv("GYXX_RUNTIME_ENV_FILE", r"D:\product-collector-analyze-flow\.env")),
|
||||||
|
Path(
|
||||||
|
os.getenv(
|
||||||
|
"GYXX_RUNTIME_JUSHUITAN_ENV_FILE",
|
||||||
|
r"D:\product-collector-analyze-flow\private-secrets\jushuitan.env",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
]
|
||||||
|
CONSOLE_HOST = "127.0.0.1"
|
||||||
|
CONSOLE_PORT = int(os.getenv("GYXX_CONSOLE_PORT", "8765"))
|
||||||
|
LOOP_INTERVAL_SECONDS = float(os.getenv("GYXX_KEEPALIVE_INTERVAL_SECONDS", "30"))
|
||||||
|
PROBE_TIMEOUT_SECONDS = 5.0
|
||||||
|
PROBE_FAILURE_LIMIT = 3
|
||||||
|
RESTART_COOLDOWN_SECONDS = float(os.getenv("GYXX_KEEPALIVE_COOLDOWN_SECONDS", "120"))
|
||||||
|
TRIP_MAX_RESTARTS = 5
|
||||||
|
TRIP_WINDOW_SECONDS = 30 * 60
|
||||||
|
TRIP_PAUSE_SECONDS = 60 * 60
|
||||||
|
|
||||||
|
DATA_ROOT = PROJECT_ROOT / "var"
|
||||||
|
LOG_PATH = DATA_ROOT / "logs" / "keepalive" / "keepalive.log"
|
||||||
|
LOCK_PATH = DATA_ROOT / "state" / "keepalive.lock"
|
||||||
|
CONSOLE_LOG_DIR = DATA_ROOT / "logs" / "console-restarts"
|
||||||
|
SCHEDULER_LOG_DIR = DATA_ROOT / "logs" / "scheduler-service"
|
||||||
|
SCHEDULER_SERVICE_LOCK = DATA_ROOT / "state" / "scheduler" / "service.lock"
|
||||||
|
|
||||||
|
|
||||||
|
def _log(event: str, detail: str = "") -> None:
|
||||||
|
stamp = datetime.now(timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
line = f"{stamp} {event} {detail}".rstrip()
|
||||||
|
LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with LOG_PATH.open("a", encoding="utf-8", newline="\n") as stream:
|
||||||
|
stream.write(line + "\n")
|
||||||
|
print(line, flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _pid_alive(pid: int) -> bool:
|
||||||
|
if pid <= 0:
|
||||||
|
return False
|
||||||
|
kernel32 = ctypes.windll.kernel32
|
||||||
|
handle = kernel32.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION
|
||||||
|
if not handle:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
exit_code = ctypes.c_ulong()
|
||||||
|
if kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)):
|
||||||
|
return exit_code.value == 259 # STILL_ACTIVE
|
||||||
|
return False
|
||||||
|
finally:
|
||||||
|
kernel32.CloseHandle(handle)
|
||||||
|
|
||||||
|
|
||||||
|
def _acquire_self_lock() -> bool:
|
||||||
|
LOCK_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if LOCK_PATH.is_file():
|
||||||
|
try:
|
||||||
|
existing = json.loads(LOCK_PATH.read_text(encoding="utf-8"))
|
||||||
|
if _pid_alive(int(existing.get("pid", 0))):
|
||||||
|
return False
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
LOCK_PATH.write_text(
|
||||||
|
json.dumps({"pid": os.getpid(), "started_at": datetime.now(timezone.utc).isoformat()}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _console_env_files() -> list[Path]:
|
||||||
|
return [path for path in ENV_FILES if path.is_file()]
|
||||||
|
|
||||||
|
|
||||||
|
def _spawn(argv: list[str], log_dir: Path, log_prefix: str) -> int:
|
||||||
|
log_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
log_path = log_dir / f"{log_prefix}-{stamp}.log"
|
||||||
|
kwargs: dict[str, object] = {}
|
||||||
|
if os.name == "nt":
|
||||||
|
kwargs["creationflags"] = (
|
||||||
|
subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
kwargs["start_new_session"] = True
|
||||||
|
with log_path.open("a", encoding="utf-8", newline="\n") as stream:
|
||||||
|
process = subprocess.Popen(
|
||||||
|
argv,
|
||||||
|
cwd=str(PROJECT_ROOT),
|
||||||
|
env=os.environ,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=stream,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
shell=False,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
return process.pid
|
||||||
|
|
||||||
|
|
||||||
|
def _console_argv() -> list[str]:
|
||||||
|
argv = [
|
||||||
|
str(PROJECT_ROOT / ".venv" / "Scripts" / "python.exe"),
|
||||||
|
"-m",
|
||||||
|
"gyxx_flow",
|
||||||
|
"console",
|
||||||
|
"--host",
|
||||||
|
CONSOLE_HOST,
|
||||||
|
"--port",
|
||||||
|
str(CONSOLE_PORT),
|
||||||
|
]
|
||||||
|
for env_file in _console_env_files():
|
||||||
|
argv.extend(("--env-file", str(env_file)))
|
||||||
|
return argv
|
||||||
|
|
||||||
|
|
||||||
|
def _scheduler_argv() -> list[str]:
|
||||||
|
argv = [
|
||||||
|
str(PROJECT_ROOT / ".venv" / "Scripts" / "python.exe"),
|
||||||
|
"-m",
|
||||||
|
"gyxx_flow",
|
||||||
|
"schedule",
|
||||||
|
"run",
|
||||||
|
]
|
||||||
|
for env_file in _console_env_files():
|
||||||
|
argv.extend(("--env-file", str(env_file)))
|
||||||
|
return argv
|
||||||
|
|
||||||
|
|
||||||
|
def _probe_console() -> bool:
|
||||||
|
try:
|
||||||
|
connection = http.client.HTTPConnection(CONSOLE_HOST, CONSOLE_PORT, timeout=PROBE_TIMEOUT_SECONDS)
|
||||||
|
try:
|
||||||
|
connection.request("GET", "/api/health")
|
||||||
|
response = connection.getresponse()
|
||||||
|
response.read()
|
||||||
|
return response.status == 200
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _port_owner_pid(port: int) -> int | None:
|
||||||
|
try:
|
||||||
|
output = subprocess.run(
|
||||||
|
["netstat", "-ano", "-p", "tcp"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=10,
|
||||||
|
).stdout
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
needle = f"{CONSOLE_HOST}:{port}"
|
||||||
|
for line in output.splitlines():
|
||||||
|
parts = line.split()
|
||||||
|
if (
|
||||||
|
len(parts) >= 5
|
||||||
|
and parts[0].upper() == "TCP"
|
||||||
|
and parts[1] == needle
|
||||||
|
and parts[3].upper() == "LISTENING"
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
return int(parts[4])
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _kill_pid_tree(pid: int) -> None:
|
||||||
|
if pid <= 0 or pid == os.getpid():
|
||||||
|
return
|
||||||
|
subprocess.run(
|
||||||
|
["taskkill", "/PID", str(pid), "/T", "/F"],
|
||||||
|
capture_output=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _scheduler_lock_pid() -> int | None:
|
||||||
|
try:
|
||||||
|
payload = json.loads(SCHEDULER_SERVICE_LOCK.read_text(encoding="utf-8"))
|
||||||
|
return int(payload.get("pid", 0)) or None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class _TargetState:
|
||||||
|
"""记录单目标的连续失败计数、冷却与熔断状态。"""
|
||||||
|
|
||||||
|
def __init__(self, name: str) -> None:
|
||||||
|
self.name = name
|
||||||
|
self.failures = 0
|
||||||
|
self.last_restart_at = 0.0
|
||||||
|
self.recent_restarts: list[float] = []
|
||||||
|
self.tripped_at: float | None = None
|
||||||
|
|
||||||
|
def tripped(self, now: float) -> bool:
|
||||||
|
if self.tripped_at is None:
|
||||||
|
return False
|
||||||
|
if now - self.tripped_at >= TRIP_PAUSE_SECONDS:
|
||||||
|
self.tripped_at = None
|
||||||
|
self.recent_restarts.clear()
|
||||||
|
_log("keepalive-trip-reset", f"target={self.name} 熔断窗口结束恢复守护")
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def may_restart(self, now: float) -> bool:
|
||||||
|
if self.tripped(now):
|
||||||
|
return False
|
||||||
|
if now - self.last_restart_at < RESTART_COOLDOWN_SECONDS:
|
||||||
|
return False
|
||||||
|
self.recent_restarts = [
|
||||||
|
stamp for stamp in self.recent_restarts if now - stamp < TRIP_WINDOW_SECONDS
|
||||||
|
]
|
||||||
|
if len(self.recent_restarts) >= TRIP_MAX_RESTARTS:
|
||||||
|
self.tripped_at = now
|
||||||
|
_log(
|
||||||
|
"keepalive-tripped",
|
||||||
|
f"target={self.name} {TRIP_MAX_RESTARTS} 次重启/{TRIP_WINDOW_SECONDS // 60}分钟 "
|
||||||
|
f"暂停拉起{TRIP_PAUSE_SECONDS // 60}分钟,请人工检查日志",
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def record_restart(self, now: float) -> None:
|
||||||
|
self.last_restart_at = now
|
||||||
|
self.recent_restarts.append(now)
|
||||||
|
|
||||||
|
|
||||||
|
def run_cycle(console: _TargetState, scheduler: _TargetState) -> None:
|
||||||
|
now = time.time()
|
||||||
|
|
||||||
|
# ---- 控制台 ----
|
||||||
|
if _probe_console():
|
||||||
|
console.failures = 0
|
||||||
|
else:
|
||||||
|
console.failures += 1
|
||||||
|
owner = _port_owner_pid(CONSOLE_PORT)
|
||||||
|
_log(
|
||||||
|
"console-probe-failed",
|
||||||
|
f"{console.failures}/{PROBE_FAILURE_LIMIT} port_owner_pid={owner}",
|
||||||
|
)
|
||||||
|
if console.failures >= PROBE_FAILURE_LIMIT and console.may_restart(now):
|
||||||
|
if owner is not None and _pid_alive(owner):
|
||||||
|
_log("console-kill-wedged", f"pid={owner} 探测持续失败,结束进程树")
|
||||||
|
_kill_pid_tree(owner)
|
||||||
|
time.sleep(3)
|
||||||
|
pid = _spawn(_console_argv(), CONSOLE_LOG_DIR, "keepalive-console")
|
||||||
|
console.record_restart(time.time())
|
||||||
|
console.failures = 0
|
||||||
|
_log("console-restarted", f"pid={pid}")
|
||||||
|
|
||||||
|
# ---- 调度器 ----
|
||||||
|
scheduler_pid = _scheduler_lock_pid()
|
||||||
|
scheduler_alive = scheduler_pid is not None and _pid_alive(scheduler_pid)
|
||||||
|
if not scheduler_alive and scheduler.may_restart(now):
|
||||||
|
pid = _spawn(_scheduler_argv(), SCHEDULER_LOG_DIR, "keepalive-start")
|
||||||
|
scheduler.record_restart(time.time())
|
||||||
|
_log(
|
||||||
|
"scheduler-restarted",
|
||||||
|
f"pid={pid} prev_lock_pid={scheduler_pid}",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="GYXX Flow 控制台/调度器保活")
|
||||||
|
parser.add_argument("--once", action="store_true", help="单轮探测+处置后退出")
|
||||||
|
arguments = parser.parse_args(argv)
|
||||||
|
|
||||||
|
if not _acquire_self_lock():
|
||||||
|
_log("keepalive-duplicate-exit", "已有保活进程在运行")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
console = _TargetState("console")
|
||||||
|
scheduler = _TargetState("scheduler")
|
||||||
|
missing_env = [str(path) for path in ENV_FILES if not path.is_file()]
|
||||||
|
_log(
|
||||||
|
"keepalive-start",
|
||||||
|
f"pid={os.getpid()} once={arguments.once}"
|
||||||
|
+ (f" missing_env={missing_env}" if missing_env else ""),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
run_cycle(console, scheduler)
|
||||||
|
if arguments.once:
|
||||||
|
_log("keepalive-once-done", "")
|
||||||
|
return 0
|
||||||
|
time.sleep(LOOP_INTERVAL_SECONDS)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
if LOCK_PATH.is_file():
|
||||||
|
payload = json.loads(LOCK_PATH.read_text(encoding="utf-8"))
|
||||||
|
if int(payload.get("pid", 0)) == os.getpid():
|
||||||
|
LOCK_PATH.unlink()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
' GYXX Flow 保活登录自启动:无窗口启动 deploy/watchdog/gyxx_keepalive.py,
|
||||||
|
' 由它负责拉起控制台(含进程内自检看门狗)与调度器。
|
||||||
|
' 安装:复制到 %APPDATA%\Microsoft\Windows\Start Menu\Programs\Startup\
|
||||||
|
CreateObject("WScript.Shell").Run _
|
||||||
|
"""D:\gyxx-flow\.venv\Scripts\pythonw.exe"" ""D:\gyxx-flow\deploy\watchdog\gyxx_keepalive.py""", 0, False
|
||||||
@@ -0,0 +1,882 @@
|
|||||||
|
"""Per-account cookie health gate injected before workflow collection steps.
|
||||||
|
|
||||||
|
Every runtime account a workflow depends on is checked before the collection
|
||||||
|
steps that use it may run. Accounts fall into three gate modes derived from
|
||||||
|
``config/runtime-bindings.json``:
|
||||||
|
|
||||||
|
* ``blocking`` -- QR-code / SMS-only accounts (no credential environment and
|
||||||
|
no configured login command). The gate verifies the login state and, when
|
||||||
|
it has expired, fails the gate step so every dependent step is skipped with
|
||||||
|
``dependency-failed`` and sends a Feishu alert asking the login owner to
|
||||||
|
re-login interactively via ``gyxx accounts login <account_id>``.
|
||||||
|
* ``recover`` -- accounts with a vault-managed credential login command
|
||||||
|
(``login_mode`` ``C``). The keepalive manager re-runs the credential login
|
||||||
|
automatically; only a failed recovery blocks the workflow and alerts.
|
||||||
|
* ``defer`` -- accounts whose member scripts embed username/password login
|
||||||
|
flows. These keep their script-level login behaviour unchanged and are not
|
||||||
|
gated at all.
|
||||||
|
|
||||||
|
The per-account decision is persisted as a small state machine under
|
||||||
|
``state/accounts/<id>/cookie_state.json`` (``healthy`` <-> ``awaiting_login``)
|
||||||
|
so repeat failures alert at most once per notify interval instead of once per
|
||||||
|
workflow run.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass, replace
|
||||||
|
from datetime import date, datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from gyxx_flow.accounts import (
|
||||||
|
AccountKeepaliveManager,
|
||||||
|
_keepalive_requires_relogin,
|
||||||
|
_read_keepalive_status,
|
||||||
|
_vault_cookie_valid,
|
||||||
|
)
|
||||||
|
from gyxx_flow.adapters.browser import BrowserCookieStore
|
||||||
|
from gyxx_flow.adapters.feishu_messages import send_lark_bot_message
|
||||||
|
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
|
||||||
|
from gyxx_flow.catalog import WorkflowCatalog
|
||||||
|
from gyxx_flow.core.artifacts import atomic_write_json
|
||||||
|
from gyxx_flow.core.config import Settings
|
||||||
|
from gyxx_flow.core.context import RunContext
|
||||||
|
from gyxx_flow.notification_routing import (
|
||||||
|
ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID,
|
||||||
|
ANALYZER_NOTIFICATION_APP_PROFILE,
|
||||||
|
NotificationRoutingStore,
|
||||||
|
)
|
||||||
|
from gyxx_flow.workflow.model import StepDefinition, WorkflowDefinition
|
||||||
|
from gyxx_flow.workflow.steps import StepExecution
|
||||||
|
|
||||||
|
GATE_MODE_BLOCKING = "blocking"
|
||||||
|
GATE_MODE_RECOVER = "recover"
|
||||||
|
GATE_MODE_DEFER = "defer"
|
||||||
|
|
||||||
|
# 飞书通知的默认接收人:按手机号在 config/notification-routing.json 的
|
||||||
|
# people 段解析 open_id;解析失败时回退到同名 analyzer 常量收件人。
|
||||||
|
DEFAULT_LOGIN_ALERT_MOBILE = "+8613663785145"
|
||||||
|
|
||||||
|
GYXX_COOKIE_GATE = "GYXX_COOKIE_GATE"
|
||||||
|
GYXX_COOKIE_GATE_MAX_STATUS_AGE_SECONDS = "GYXX_COOKIE_GATE_MAX_STATUS_AGE_SECONDS"
|
||||||
|
GYXX_LOGIN_ALERT_OPEN_IDS = "GYXX_LOGIN_ALERT_OPEN_IDS"
|
||||||
|
GYXX_LOGIN_ALERT_APP_PROFILE = "GYXX_LOGIN_ALERT_APP_PROFILE"
|
||||||
|
GYXX_LOGIN_ALERT_NOTIFY_INTERVAL_SECONDS = "GYXX_LOGIN_ALERT_NOTIFY_INTERVAL_SECONDS"
|
||||||
|
|
||||||
|
_GATE_STEP_PREFIX = "cookie-gate"
|
||||||
|
_GATE_STATE_FILENAME = "cookie_state.json"
|
||||||
|
_GATE_TIMEOUT_SECONDS = 1800.0
|
||||||
|
_DEFAULT_MAX_STATUS_AGE_SECONDS = 3600.0
|
||||||
|
_MIN_MAX_STATUS_AGE_SECONDS = 300.0
|
||||||
|
_DEFAULT_NOTIFY_INTERVAL_SECONDS = 1800.0
|
||||||
|
_DISABLED_VALUES = frozenset({"0", "false", "off", "no", "disabled"})
|
||||||
|
# 自动重跑:待重跑清单上限、健康探活节流、同账号重跑启动冷却。
|
||||||
|
_MAX_PENDING_RERUNS = 20
|
||||||
|
_RECOVERY_PROBE_INTERVAL_SECONDS = 300.0
|
||||||
|
_RECOVERY_LAUNCH_COOLDOWN_SECONDS = 300.0
|
||||||
|
_RECOVERY_HISTORY_LIMIT = 20
|
||||||
|
|
||||||
|
STATE_HEALTHY = "healthy"
|
||||||
|
STATE_AWAITING_LOGIN = "awaiting_login"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class GateVerdict:
|
||||||
|
"""One account's gate decision, recorded in the run journal on failure."""
|
||||||
|
|
||||||
|
account_id: str
|
||||||
|
mode: str
|
||||||
|
blocked: bool
|
||||||
|
message: str
|
||||||
|
recovered: bool = False
|
||||||
|
notified: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class CookieGateStepAction:
|
||||||
|
"""ExecutableStep that turns one account verdict into a step outcome."""
|
||||||
|
|
||||||
|
gate: "CookieGate"
|
||||||
|
account_id: str
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
context: RunContext,
|
||||||
|
timeout_seconds: float | None,
|
||||||
|
dry_run: bool,
|
||||||
|
) -> StepExecution:
|
||||||
|
if dry_run: # pragma: no cover - the engine skips dry-run steps first
|
||||||
|
return StepExecution(exit_code=0, skipped=True, reason="dry-run")
|
||||||
|
verdict = self.gate.check(self.account_id, context=context)
|
||||||
|
if verdict.blocked:
|
||||||
|
return StepExecution(
|
||||||
|
exit_code=1,
|
||||||
|
error=verdict.message,
|
||||||
|
reason="cookie-expired",
|
||||||
|
)
|
||||||
|
return StepExecution(exit_code=0)
|
||||||
|
|
||||||
|
|
||||||
|
class CookieGate:
|
||||||
|
"""Resolve, probe, and gate the accounts one workflow depends on."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
settings: Settings,
|
||||||
|
*,
|
||||||
|
catalog: RuntimeIntegrationCatalog | None = None,
|
||||||
|
keepalive: AccountKeepaliveManager | None = None,
|
||||||
|
environment: Mapping[str, str] | None = None,
|
||||||
|
now: Callable[[], datetime] | None = None,
|
||||||
|
sender: Callable[..., dict[str, Any]] | None = None,
|
||||||
|
) -> None: # noqa: D107 - simple wiring
|
||||||
|
self.settings = settings
|
||||||
|
self.catalog = catalog or RuntimeIntegrationCatalog.load_default(
|
||||||
|
project_root=settings.project_root,
|
||||||
|
data_root=settings.data_root,
|
||||||
|
)
|
||||||
|
self.keepalive = keepalive or AccountKeepaliveManager(
|
||||||
|
settings, catalog=self.catalog
|
||||||
|
)
|
||||||
|
self._environment = dict(os.environ if environment is None else environment)
|
||||||
|
self._now = now or (lambda: datetime.now(timezone.utc))
|
||||||
|
self._sender = sender or send_lark_bot_message
|
||||||
|
self._workflow_catalog: WorkflowCatalog | None = None
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Account resolution
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def gate_mode(self, account_id: str) -> str:
|
||||||
|
account = self.catalog.account_for(account_id)
|
||||||
|
if str(account.login_command_id or "").strip():
|
||||||
|
return GATE_MODE_RECOVER
|
||||||
|
if account.credential_env_names or account.login_mode == "B":
|
||||||
|
return GATE_MODE_DEFER
|
||||||
|
return GATE_MODE_BLOCKING
|
||||||
|
|
||||||
|
def collect_workflow_accounts(
|
||||||
|
self,
|
||||||
|
workflow: WorkflowDefinition,
|
||||||
|
) -> dict[str, set[str]]:
|
||||||
|
"""Map each gated account to the step ids that depend on it.
|
||||||
|
|
||||||
|
Accounts resolve two ways: directly, from each step binding's
|
||||||
|
``account``; and declaratively, from the catalog workflow's
|
||||||
|
``required_accounts`` (used by orchestrator entries whose own binding
|
||||||
|
has no account while the spawned child collectors do).
|
||||||
|
"""
|
||||||
|
|
||||||
|
accounts: dict[str, set[str]] = {}
|
||||||
|
for step in workflow.steps:
|
||||||
|
account_id = self._account_for_step(step)
|
||||||
|
if account_id:
|
||||||
|
accounts.setdefault(account_id, set()).add(step.step_id)
|
||||||
|
for account_id in self._declared_required_accounts(workflow.workflow_id):
|
||||||
|
self.catalog.account_for(account_id) # unknown id: loud config error
|
||||||
|
accounts.setdefault(account_id, set()).update(
|
||||||
|
step.step_id for step in workflow.steps
|
||||||
|
)
|
||||||
|
# Defer-mode accounts keep their script-level credential login flows
|
||||||
|
# and are intentionally not gated.
|
||||||
|
return {
|
||||||
|
account_id: step_ids
|
||||||
|
for account_id, step_ids in accounts.items()
|
||||||
|
if self.gate_mode(account_id) != GATE_MODE_DEFER
|
||||||
|
}
|
||||||
|
|
||||||
|
def _account_for_step(self, step: StepDefinition) -> str | None:
|
||||||
|
action = step.action
|
||||||
|
entry = getattr(action, "entry", None)
|
||||||
|
module_id = getattr(entry, "module", "")
|
||||||
|
command_entry = getattr(action, "command_entry", None) or getattr(
|
||||||
|
entry, "entry", ""
|
||||||
|
)
|
||||||
|
if not module_id or not command_entry:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
binding = self.catalog.binding_for(f"{module_id}:{command_entry}")
|
||||||
|
except Exception: # noqa: BLE001 - unbound entries run ungated
|
||||||
|
return None
|
||||||
|
return binding.account or None
|
||||||
|
|
||||||
|
def workflow_catalog(self) -> WorkflowCatalog:
|
||||||
|
"""Load (once) the catalog used for declared account and rerun lookups."""
|
||||||
|
|
||||||
|
if self._workflow_catalog is None:
|
||||||
|
try:
|
||||||
|
self._workflow_catalog = WorkflowCatalog.load(
|
||||||
|
self.settings.project_root / "config"
|
||||||
|
)
|
||||||
|
except Exception: # noqa: BLE001 - catalog issues surface elsewhere
|
||||||
|
self._workflow_catalog = WorkflowCatalog(
|
||||||
|
timezone="UTC", workflows=(), schedules=()
|
||||||
|
)
|
||||||
|
return self._workflow_catalog
|
||||||
|
|
||||||
|
def _declared_required_accounts(self, workflow_id: str) -> tuple[str, ...]:
|
||||||
|
for entry in self.workflow_catalog().workflows:
|
||||||
|
if entry.workflow_id == workflow_id:
|
||||||
|
return tuple(entry.required_accounts)
|
||||||
|
return ()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# State machine
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def check(self, account_id: str, *, context: RunContext) -> GateVerdict:
|
||||||
|
account = self.catalog.account_for(account_id)
|
||||||
|
mode = self.gate_mode(account_id)
|
||||||
|
store = BrowserCookieStore(account.cookie_file, account.storage_state_file)
|
||||||
|
static_valid = _vault_cookie_valid(account, store)
|
||||||
|
status = _read_keepalive_status(account)
|
||||||
|
if static_valid and self._keepalive_success_is_fresh(status):
|
||||||
|
self._write_state(account, STATE_HEALTHY, probe="fresh-keepalive")
|
||||||
|
return GateVerdict(
|
||||||
|
account_id=account_id,
|
||||||
|
mode=mode,
|
||||||
|
blocked=False,
|
||||||
|
message=f"账号 {account_id} 登录态有效(keepalive 近期验证通过)",
|
||||||
|
)
|
||||||
|
if self._acceptance_enabled():
|
||||||
|
# 验收模式不开浏览器:按静态证据判定,且不投递飞书通知。
|
||||||
|
if static_valid and not _keepalive_requires_relogin(status):
|
||||||
|
return GateVerdict(
|
||||||
|
account_id=account_id,
|
||||||
|
mode=mode,
|
||||||
|
blocked=False,
|
||||||
|
message=f"账号 {account_id} 静态 cookie 校验通过(验收模式)",
|
||||||
|
)
|
||||||
|
return self._expire(
|
||||||
|
account,
|
||||||
|
mode,
|
||||||
|
context,
|
||||||
|
detail="验收模式下静态 cookie 校验未通过",
|
||||||
|
notify=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
outcome = self.keepalive.refresh_account(account_id)
|
||||||
|
probe_status = str(outcome.get("status", "error"))
|
||||||
|
if probe_status == "success":
|
||||||
|
recovered = bool(outcome.get("login_recovered"))
|
||||||
|
self._write_state(
|
||||||
|
account,
|
||||||
|
STATE_HEALTHY,
|
||||||
|
probe="success",
|
||||||
|
recovered=recovered,
|
||||||
|
)
|
||||||
|
message = (
|
||||||
|
f"账号 {account_id} 登录态有效(自动账密重登成功)"
|
||||||
|
if recovered
|
||||||
|
else f"账号 {account_id} 登录态有效(探活通过)"
|
||||||
|
)
|
||||||
|
return GateVerdict(
|
||||||
|
account_id=account_id,
|
||||||
|
mode=mode,
|
||||||
|
blocked=False,
|
||||||
|
message=message,
|
||||||
|
recovered=recovered,
|
||||||
|
)
|
||||||
|
if probe_status == "busy":
|
||||||
|
if static_valid and not _keepalive_requires_relogin(status):
|
||||||
|
return GateVerdict(
|
||||||
|
account_id=account_id,
|
||||||
|
mode=mode,
|
||||||
|
blocked=False,
|
||||||
|
message=f"账号 {account_id} 探活锁忙,静态 cookie 校验通过",
|
||||||
|
)
|
||||||
|
return self._expire(
|
||||||
|
account,
|
||||||
|
mode,
|
||||||
|
context,
|
||||||
|
detail="探活锁被占用且静态 cookie 校验未通过",
|
||||||
|
)
|
||||||
|
if probe_status in {"expired", "not_logged_in"}:
|
||||||
|
evidence = _recovery_evidence(_read_keepalive_status(account))
|
||||||
|
detail = "cookie 已过期,无法正常采集"
|
||||||
|
if probe_status == "not_logged_in" and evidence:
|
||||||
|
detail = f"登录态缺失;{evidence}"
|
||||||
|
elif evidence:
|
||||||
|
detail = f"cookie 已过期;{evidence}"
|
||||||
|
return self._expire(account, mode, context, detail=detail)
|
||||||
|
# timeout / error / http_error / disabled:探测结果不可信时回退静态证据。
|
||||||
|
if static_valid and not _keepalive_requires_relogin(status):
|
||||||
|
return GateVerdict(
|
||||||
|
account_id=account_id,
|
||||||
|
mode=mode,
|
||||||
|
blocked=False,
|
||||||
|
message=f"账号 {account_id} 探活异常({probe_status}),静态 cookie 校验通过",
|
||||||
|
)
|
||||||
|
return self._expire(
|
||||||
|
account,
|
||||||
|
mode,
|
||||||
|
context,
|
||||||
|
detail=f"探活异常({probe_status})且静态 cookie 校验未通过",
|
||||||
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Expiry, notification, persisted state
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _expire(
|
||||||
|
self,
|
||||||
|
account: Any,
|
||||||
|
mode: str,
|
||||||
|
context: RunContext,
|
||||||
|
*,
|
||||||
|
detail: str,
|
||||||
|
notify: bool | None = None,
|
||||||
|
) -> GateVerdict:
|
||||||
|
account_id = str(account.account_id)
|
||||||
|
if notify is None:
|
||||||
|
notify = not context.shadow and not self._acceptance_enabled()
|
||||||
|
state = self._read_state(account)
|
||||||
|
notified = False
|
||||||
|
notification_note = ""
|
||||||
|
if notify and not self._within_notify_interval(state):
|
||||||
|
recipients = self._alert_recipients()
|
||||||
|
message = _expiry_message(account_id, mode, context, detail)
|
||||||
|
for open_id, profile in recipients:
|
||||||
|
try:
|
||||||
|
self._sender(
|
||||||
|
user_id=open_id,
|
||||||
|
text=message,
|
||||||
|
idempotency_key=f"cookie-gate:{account_id}:{context.run_id}",
|
||||||
|
profile=profile,
|
||||||
|
)
|
||||||
|
notified = True
|
||||||
|
except Exception as exc: # noqa: BLE001 - alerting must not mask the verdict
|
||||||
|
notification_note = f"{type(exc).__name__}: {exc}"[:300]
|
||||||
|
if recipients and notified:
|
||||||
|
state["last_notified_at"] = _now_iso(self._now)
|
||||||
|
state.pop("notification_error", None)
|
||||||
|
elif notification_note:
|
||||||
|
state["notification_error"] = notification_note
|
||||||
|
state["state"] = STATE_AWAITING_LOGIN
|
||||||
|
state["updated_at"] = _now_iso(self._now)
|
||||||
|
state["last_probe_status"] = "expired"
|
||||||
|
state["last_blocked_run"] = context.run_id
|
||||||
|
if not context.shadow and not self._acceptance_enabled():
|
||||||
|
# 记录待自动重跑清单:登录态恢复后由调度器维护循环按原业务日期重跑。
|
||||||
|
state["pending_reruns"] = _record_pending_rerun(
|
||||||
|
state.get("pending_reruns"), context, _now_iso(self._now)
|
||||||
|
)
|
||||||
|
self._save_state(account, state)
|
||||||
|
message = _expiry_message(account_id, mode, context, detail)
|
||||||
|
if notified:
|
||||||
|
message += " 已通过飞书通知登录负责人。"
|
||||||
|
elif notification_note:
|
||||||
|
message += f" 飞书通知发送失败({notification_note})。"
|
||||||
|
return GateVerdict(
|
||||||
|
account_id=account_id,
|
||||||
|
mode=mode,
|
||||||
|
blocked=True,
|
||||||
|
message=message,
|
||||||
|
notified=notified,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _write_state(
|
||||||
|
self,
|
||||||
|
account: Any,
|
||||||
|
state_name: str,
|
||||||
|
*,
|
||||||
|
probe: str,
|
||||||
|
recovered: bool = False,
|
||||||
|
) -> None:
|
||||||
|
state = self._read_state(account)
|
||||||
|
if (
|
||||||
|
state_name == STATE_HEALTHY
|
||||||
|
and state.get("state") == STATE_HEALTHY
|
||||||
|
and state.get("last_probe_status") in {"fresh-keepalive", "success"}
|
||||||
|
and not recovered
|
||||||
|
):
|
||||||
|
return # 持续健康时避免每次运行都重写状态文件
|
||||||
|
state["state"] = state_name
|
||||||
|
state["updated_at"] = _now_iso(self._now)
|
||||||
|
state["last_probe_status"] = probe
|
||||||
|
if state_name == STATE_HEALTHY:
|
||||||
|
state["last_healthy_at"] = _now_iso(self._now)
|
||||||
|
if recovered:
|
||||||
|
state["last_recovered_at"] = _now_iso(self._now)
|
||||||
|
state.pop("notification_error", None)
|
||||||
|
self._save_state(account, state)
|
||||||
|
|
||||||
|
def _state_path(self, account: Any) -> Path:
|
||||||
|
return account.cookie_file.with_name(_GATE_STATE_FILENAME)
|
||||||
|
|
||||||
|
def _read_state(self, account: Any) -> dict[str, object]:
|
||||||
|
path = self._state_path(account)
|
||||||
|
if not path.exists():
|
||||||
|
return {"account_id": str(account.account_id)}
|
||||||
|
try:
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return {"account_id": str(account.account_id)}
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return {"account_id": str(account.account_id)}
|
||||||
|
return dict(payload)
|
||||||
|
|
||||||
|
def _save_state(self, account: Any, state: Mapping[str, object]) -> None:
|
||||||
|
payload = {
|
||||||
|
"account_id": str(account.account_id),
|
||||||
|
**state,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
atomic_write_json(self._state_path(account), payload)
|
||||||
|
except OSError: # pragma: no cover - state file is advisory only
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _within_notify_interval(self, state: Mapping[str, object]) -> bool:
|
||||||
|
raw = state.get("last_notified_at")
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
last = datetime.fromisoformat(raw)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
now = self._now()
|
||||||
|
if last.tzinfo is None:
|
||||||
|
last = last.replace(tzinfo=timezone.utc)
|
||||||
|
interval = _env_int(
|
||||||
|
self._environment,
|
||||||
|
GYXX_LOGIN_ALERT_NOTIFY_INTERVAL_SECONDS,
|
||||||
|
_DEFAULT_NOTIFY_INTERVAL_SECONDS,
|
||||||
|
minimum=0,
|
||||||
|
)
|
||||||
|
return (now - last).total_seconds() < interval
|
||||||
|
|
||||||
|
def _keepalive_success_is_fresh(self, status: Mapping[str, object] | None) -> bool:
|
||||||
|
if not isinstance(status, Mapping) or status.get("status") != "success":
|
||||||
|
return False
|
||||||
|
raw = status.get("last_success_at")
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
last = datetime.fromisoformat(raw)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
now = self._now()
|
||||||
|
if last.tzinfo is None:
|
||||||
|
last = last.replace(tzinfo=timezone.utc)
|
||||||
|
max_age = _env_float(
|
||||||
|
self._environment,
|
||||||
|
GYXX_COOKIE_GATE_MAX_STATUS_AGE_SECONDS,
|
||||||
|
_DEFAULT_MAX_STATUS_AGE_SECONDS,
|
||||||
|
minimum=_MIN_MAX_STATUS_AGE_SECONDS,
|
||||||
|
)
|
||||||
|
return (now - last).total_seconds() <= max_age
|
||||||
|
|
||||||
|
def _alert_recipients(self) -> list[tuple[str, str]]:
|
||||||
|
profile = (
|
||||||
|
self._environment.get(GYXX_LOGIN_ALERT_APP_PROFILE, "").strip()
|
||||||
|
or ANALYZER_NOTIFICATION_APP_PROFILE
|
||||||
|
)
|
||||||
|
override = [
|
||||||
|
value.strip()
|
||||||
|
for value in self._environment.get(GYXX_LOGIN_ALERT_OPEN_IDS, "").split(",")
|
||||||
|
if value.strip()
|
||||||
|
]
|
||||||
|
if override:
|
||||||
|
return [(open_id, profile) for open_id in override]
|
||||||
|
try:
|
||||||
|
payload, _revision = NotificationRoutingStore(self.settings).snapshot()
|
||||||
|
for person in payload["people"].values():
|
||||||
|
if person.get("mobile") == DEFAULT_LOGIN_ALERT_MOBILE:
|
||||||
|
binding = person.get("bindings", {}).get(profile)
|
||||||
|
if binding:
|
||||||
|
return [(str(binding["open_id"]), profile)]
|
||||||
|
except Exception: # noqa: BLE001 - fall back to the constant recipient
|
||||||
|
pass
|
||||||
|
return [(ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID, profile)]
|
||||||
|
|
||||||
|
def _acceptance_enabled(self) -> bool:
|
||||||
|
value = self._environment.get("GYXX_WORKFLOW_ACCEPTANCE", "")
|
||||||
|
return bool(value and value.strip().casefold() in {"1", "true", "yes", "on"})
|
||||||
|
|
||||||
|
|
||||||
|
def gate_enabled(environment: Mapping[str, str] | None = None) -> bool:
|
||||||
|
values = os.environ if environment is None else environment
|
||||||
|
return values.get(GYXX_COOKIE_GATE, "").strip().casefold() not in _DISABLED_VALUES
|
||||||
|
|
||||||
|
|
||||||
|
def apply_cookie_gates(
|
||||||
|
workflow: WorkflowDefinition,
|
||||||
|
*,
|
||||||
|
settings: Settings,
|
||||||
|
environment: Mapping[str, str] | None = None,
|
||||||
|
gate: CookieGate | None = None,
|
||||||
|
) -> WorkflowDefinition:
|
||||||
|
"""Inject one critical gate step per gated account in front of its users.
|
||||||
|
|
||||||
|
Gate steps are graph sources; each step that uses an account gains the
|
||||||
|
account's gate as an extra dependency. A failed gate therefore skips its
|
||||||
|
dependents with ``dependency-failed`` through the regular engine, while
|
||||||
|
unrelated branches keep running. Operator slices (``--rerun-step`` /
|
||||||
|
``--resume-from``) are produced before this injection and stay gate-free.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not workflow.steps:
|
||||||
|
return workflow
|
||||||
|
if gate is None:
|
||||||
|
try:
|
||||||
|
gate = CookieGate(
|
||||||
|
settings,
|
||||||
|
environment=os.environ if environment is None else environment,
|
||||||
|
)
|
||||||
|
except Exception: # noqa: BLE001 - 无集成配置时不门控;步骤执行仍会显式报错
|
||||||
|
return workflow
|
||||||
|
account_steps = gate.collect_workflow_accounts(workflow)
|
||||||
|
if not account_steps:
|
||||||
|
return workflow
|
||||||
|
existing_ids = {step.step_id for step in workflow.steps}
|
||||||
|
gate_steps: list[StepDefinition] = []
|
||||||
|
for account_id in sorted(account_steps):
|
||||||
|
step_id = f"{_GATE_STEP_PREFIX}.{account_id}"
|
||||||
|
if step_id in existing_ids:
|
||||||
|
continue
|
||||||
|
gate_steps.append(
|
||||||
|
StepDefinition(
|
||||||
|
step_id,
|
||||||
|
CookieGateStepAction(gate, account_id),
|
||||||
|
timeout_seconds=_GATE_TIMEOUT_SECONDS,
|
||||||
|
max_attempts=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not gate_steps:
|
||||||
|
return workflow
|
||||||
|
patched: list[StepDefinition] = []
|
||||||
|
for step in workflow.steps:
|
||||||
|
dependencies = list(dict.fromkeys(step.depends_on))
|
||||||
|
changed = False
|
||||||
|
for account_id, step_ids in account_steps.items():
|
||||||
|
gate_id = f"{_GATE_STEP_PREFIX}.{account_id}"
|
||||||
|
if step.step_id in step_ids and gate_id not in dependencies:
|
||||||
|
dependencies.append(gate_id)
|
||||||
|
changed = True
|
||||||
|
if changed:
|
||||||
|
step = replace(step, depends_on=tuple(dependencies))
|
||||||
|
patched.append(step)
|
||||||
|
return WorkflowDefinition(
|
||||||
|
workflow.workflow_id,
|
||||||
|
(*gate_steps, *patched),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CookieGateRecoveryManager:
|
||||||
|
"""Re-run workflows blocked by expired cookies once the login recovers.
|
||||||
|
|
||||||
|
Runs inside the resident scheduler's maintenance loop. For every account
|
||||||
|
with a non-empty ``pending_reruns`` list, it re-verifies the login state
|
||||||
|
(fresh keepalive success first, then a throttled live probe); on recovery
|
||||||
|
it relaunches each recorded workflow with its original business date
|
||||||
|
through the scheduler's own launcher, clears the pending list, and sends a
|
||||||
|
Feishu confirmation to the login owner.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
gate: CookieGate,
|
||||||
|
*,
|
||||||
|
launcher: Any,
|
||||||
|
probe_interval_seconds: float = _RECOVERY_PROBE_INTERVAL_SECONDS,
|
||||||
|
launch_cooldown_seconds: float = _RECOVERY_LAUNCH_COOLDOWN_SECONDS,
|
||||||
|
) -> None:
|
||||||
|
self.gate = gate
|
||||||
|
self._launcher = launcher
|
||||||
|
self._probe_interval = probe_interval_seconds
|
||||||
|
self._launch_cooldown = launch_cooldown_seconds
|
||||||
|
|
||||||
|
def tick(self) -> list[dict[str, object]]:
|
||||||
|
"""Attempt recovery reruns once; returns redacted outcomes per account."""
|
||||||
|
|
||||||
|
outcomes: list[dict[str, object]] = []
|
||||||
|
for account in self.gate.catalog.accounts:
|
||||||
|
state = self.gate._read_state(account)
|
||||||
|
pending = state.get("pending_reruns")
|
||||||
|
if not isinstance(pending, list) or not pending:
|
||||||
|
continue
|
||||||
|
now = self.gate._now()
|
||||||
|
if not self._launch_cooldown_passed(state, now):
|
||||||
|
continue
|
||||||
|
if not self._account_recovered(account, state, now):
|
||||||
|
continue
|
||||||
|
launched, remaining = self._launch_pending(account, pending, now)
|
||||||
|
state["pending_reruns"] = remaining
|
||||||
|
state["updated_at"] = _now_iso(self.gate._now)
|
||||||
|
state["last_recovered_at"] = state["updated_at"]
|
||||||
|
if not remaining:
|
||||||
|
state["state"] = STATE_HEALTHY
|
||||||
|
state["last_probe_status"] = "recovered-rerun"
|
||||||
|
if launched:
|
||||||
|
state["last_rerun_launch_at"] = state["updated_at"]
|
||||||
|
history = [
|
||||||
|
item
|
||||||
|
for item in state.get("rerun_history", [])
|
||||||
|
if isinstance(item, dict)
|
||||||
|
]
|
||||||
|
history.extend(launched)
|
||||||
|
state["rerun_history"] = history[-_RECOVERY_HISTORY_LIMIT:]
|
||||||
|
self.gate._save_state(account, state)
|
||||||
|
if launched:
|
||||||
|
self._notify_recovered(str(account.account_id), launched)
|
||||||
|
outcomes.append(
|
||||||
|
{
|
||||||
|
"account_id": str(account.account_id),
|
||||||
|
"launched": launched,
|
||||||
|
"pending": len(remaining),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return outcomes
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# internals
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _launch_cooldown_passed(
|
||||||
|
self, state: Mapping[str, object], now: datetime
|
||||||
|
) -> bool:
|
||||||
|
raw = state.get("last_rerun_launch_at")
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
last = datetime.fromisoformat(raw)
|
||||||
|
except ValueError:
|
||||||
|
return True
|
||||||
|
if last.tzinfo is None:
|
||||||
|
last = last.replace(tzinfo=timezone.utc)
|
||||||
|
return (now - last).total_seconds() >= self._launch_cooldown
|
||||||
|
|
||||||
|
def _account_recovered(
|
||||||
|
self,
|
||||||
|
account: Any,
|
||||||
|
state: dict[str, object],
|
||||||
|
now: datetime,
|
||||||
|
) -> bool:
|
||||||
|
"""Fresh keepalive success first; otherwise one throttled live probe."""
|
||||||
|
|
||||||
|
store = BrowserCookieStore(account.cookie_file, account.storage_state_file)
|
||||||
|
if _vault_cookie_valid(account, store) and self.gate._keepalive_success_is_fresh(
|
||||||
|
_read_keepalive_status(account)
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
raw = state.get("last_recovery_probe_at")
|
||||||
|
if isinstance(raw, str):
|
||||||
|
try:
|
||||||
|
last_probe = datetime.fromisoformat(raw)
|
||||||
|
except ValueError:
|
||||||
|
last_probe = None
|
||||||
|
if last_probe is not None:
|
||||||
|
if last_probe.tzinfo is None:
|
||||||
|
last_probe = last_probe.replace(tzinfo=timezone.utc)
|
||||||
|
if (now - last_probe).total_seconds() < self._probe_interval:
|
||||||
|
return False
|
||||||
|
state["last_recovery_probe_at"] = _now_iso(self.gate._now)
|
||||||
|
self.gate._save_state(account, state)
|
||||||
|
try:
|
||||||
|
outcome = self.gate.keepalive.refresh_account(str(account.account_id))
|
||||||
|
except Exception: # noqa: BLE001 - 恢复探活失败留给下一轮
|
||||||
|
return False
|
||||||
|
return str(outcome.get("status", "error")) == "success"
|
||||||
|
|
||||||
|
def _launch_pending(
|
||||||
|
self,
|
||||||
|
account: Any,
|
||||||
|
pending: list[object],
|
||||||
|
now: datetime,
|
||||||
|
) -> tuple[list[dict[str, object]], list[dict[str, object]]]:
|
||||||
|
known_ids = {
|
||||||
|
entry.workflow_id for entry in self.gate.workflow_catalog().workflows
|
||||||
|
}
|
||||||
|
launched: list[dict[str, object]] = []
|
||||||
|
remaining: list[dict[str, object]] = []
|
||||||
|
for item in pending:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue # 控制台脚本等临时运行 id:不在 catalog 中,直接丢弃
|
||||||
|
workflow_id = item.get("workflow_id")
|
||||||
|
business_date = item.get("business_date")
|
||||||
|
if not isinstance(workflow_id, str) or not isinstance(business_date, str):
|
||||||
|
continue
|
||||||
|
if workflow_id not in known_ids:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
process = self._launcher.launch(
|
||||||
|
workflow_id, now, date.fromisoformat(business_date)
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001 - 保留 pending,下一轮重试
|
||||||
|
remaining.append(
|
||||||
|
{
|
||||||
|
"workflow_id": workflow_id,
|
||||||
|
"business_date": business_date,
|
||||||
|
"error": f"{type(exc).__name__}: {exc}"[:200],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
close = getattr(self._launcher, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
try:
|
||||||
|
close(process)
|
||||||
|
except Exception: # noqa: BLE001 - 关闭日志句柄失败不影响重跑
|
||||||
|
pass
|
||||||
|
launched.append(
|
||||||
|
{
|
||||||
|
"workflow_id": workflow_id,
|
||||||
|
"business_date": business_date,
|
||||||
|
"pid": int(getattr(process, "pid", 0) or 0),
|
||||||
|
"launched_at": _now_iso(self.gate._now),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return launched, remaining
|
||||||
|
|
||||||
|
def _notify_recovered(
|
||||||
|
self,
|
||||||
|
account_id: str,
|
||||||
|
launched: list[dict[str, object]],
|
||||||
|
) -> None:
|
||||||
|
lines = [
|
||||||
|
f"- {item['workflow_id']}(业务日期 {item['business_date']})"
|
||||||
|
for item in launched
|
||||||
|
]
|
||||||
|
message = (
|
||||||
|
"【GYXX 登录态已恢复】账号 "
|
||||||
|
f"{account_id} 登录态已恢复,已自动重跑被阻断的工作流:\n"
|
||||||
|
+ "\n".join(lines)
|
||||||
|
+ "\n如本次重跑仍有失败,请查看运行日志或工作流控制台。"
|
||||||
|
)
|
||||||
|
key = "-".join(
|
||||||
|
[account_id, *(str(item["workflow_id"]) for item in launched)]
|
||||||
|
)
|
||||||
|
for open_id, profile in self.gate._alert_recipients():
|
||||||
|
try:
|
||||||
|
self.gate._sender(
|
||||||
|
user_id=open_id,
|
||||||
|
text=message,
|
||||||
|
idempotency_key=f"cookie-recovery:{key}",
|
||||||
|
profile=profile,
|
||||||
|
)
|
||||||
|
except Exception: # noqa: BLE001 - 确认通知失败不影响已发起的重跑
|
||||||
|
continue
|
||||||
|
|
||||||
|
|
||||||
|
def _record_pending_rerun(
|
||||||
|
current: object,
|
||||||
|
context: RunContext,
|
||||||
|
blocked_at: str,
|
||||||
|
) -> list[dict[str, object]]:
|
||||||
|
"""Append one blocked run to the pending list, deduped and bounded."""
|
||||||
|
|
||||||
|
pending: list[dict[str, object]] = [
|
||||||
|
dict(item)
|
||||||
|
for item in (current if isinstance(current, list) else [])
|
||||||
|
if isinstance(item, dict)
|
||||||
|
]
|
||||||
|
entry = {
|
||||||
|
"workflow_id": context.workflow_id,
|
||||||
|
"business_date": context.business_date.isoformat(),
|
||||||
|
"blocked_run_id": context.run_id,
|
||||||
|
"blocked_at": blocked_at,
|
||||||
|
}
|
||||||
|
pending = [
|
||||||
|
item
|
||||||
|
for item in pending
|
||||||
|
if not (
|
||||||
|
item.get("workflow_id") == entry["workflow_id"]
|
||||||
|
and item.get("business_date") == entry["business_date"]
|
||||||
|
)
|
||||||
|
]
|
||||||
|
pending.append(entry)
|
||||||
|
return pending[-_MAX_PENDING_RERUNS:]
|
||||||
|
|
||||||
|
|
||||||
|
def _recovery_evidence(status: Mapping[str, object] | None) -> str:
|
||||||
|
if not isinstance(status, Mapping):
|
||||||
|
return ""
|
||||||
|
if not status.get("login_attempted"):
|
||||||
|
return ""
|
||||||
|
reason = status.get("login_reason")
|
||||||
|
error_type = status.get("login_error_type")
|
||||||
|
exit_code = status.get("login_exit_code")
|
||||||
|
parts = ["自动账密重登已尝试但未成功"]
|
||||||
|
if isinstance(reason, str) and reason:
|
||||||
|
parts.append(reason)
|
||||||
|
if isinstance(error_type, str) and error_type:
|
||||||
|
parts.append(f"错误={error_type}")
|
||||||
|
if isinstance(exit_code, int) and not isinstance(exit_code, bool):
|
||||||
|
parts.append(f"退出码={exit_code}")
|
||||||
|
return "(" + ",".join(parts) + ")"
|
||||||
|
|
||||||
|
|
||||||
|
def _expiry_message(
|
||||||
|
account_id: str,
|
||||||
|
mode: str,
|
||||||
|
context: RunContext,
|
||||||
|
detail: str,
|
||||||
|
) -> str:
|
||||||
|
mode_note = (
|
||||||
|
"账号密码自动重登失败,需人工处理(如验证码)"
|
||||||
|
if mode == GATE_MODE_RECOVER
|
||||||
|
else "该平台仅支持扫码/验证码登录,无法自动恢复"
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
"【GYXX 登录态失效】工作流被阻断,需要人工重新登录\n"
|
||||||
|
f"工作流: {context.workflow_id}(业务日期 {context.business_date.isoformat()})\n"
|
||||||
|
f"账号: {account_id}\n"
|
||||||
|
f"原因: {detail};{mode_note}\n"
|
||||||
|
f"请在服务器上执行: uv run gyxx accounts login {account_id}\n"
|
||||||
|
"(将打开浏览器窗口,完成扫码或验证码登录后自动保存并同步到全部成员脚本;"
|
||||||
|
"受影响的采集步骤本次已跳过,登录完成后下次运行自动恢复)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _now_iso(now: Callable[[], datetime]) -> str:
|
||||||
|
return now().astimezone(timezone.utc).isoformat(timespec="seconds")
|
||||||
|
|
||||||
|
|
||||||
|
def _env_int(
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
name: str,
|
||||||
|
default: int,
|
||||||
|
*,
|
||||||
|
minimum: int,
|
||||||
|
) -> int:
|
||||||
|
raw = environment.get(name, "").strip()
|
||||||
|
try:
|
||||||
|
value = int(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return default
|
||||||
|
return max(minimum, value)
|
||||||
|
|
||||||
|
|
||||||
|
def _env_float(
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
name: str,
|
||||||
|
default: float,
|
||||||
|
*,
|
||||||
|
minimum: float,
|
||||||
|
) -> float:
|
||||||
|
raw = environment.get(name, "").strip()
|
||||||
|
try:
|
||||||
|
value = float(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return default
|
||||||
|
return max(minimum, value)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"DEFAULT_LOGIN_ALERT_MOBILE",
|
||||||
|
"GATE_MODE_BLOCKING",
|
||||||
|
"GATE_MODE_DEFER",
|
||||||
|
"GATE_MODE_RECOVER",
|
||||||
|
"GYXX_COOKIE_GATE",
|
||||||
|
"GYXX_LOGIN_ALERT_OPEN_IDS",
|
||||||
|
"CookieGate",
|
||||||
|
"CookieGateRecoveryManager",
|
||||||
|
"CookieGateStepAction",
|
||||||
|
"GateVerdict",
|
||||||
|
"apply_cookie_gates",
|
||||||
|
"gate_enabled",
|
||||||
|
]
|
||||||
Reference in new issue
Block a user