feat(cookie-gate): 补充 cookie_gate 网关模块与 deploy/watchdog 保活脚本

cookie_gate.py(账号三级门控/阻断记录 pending_reruns/恢复重跑管理器)为此前
接线提交遗漏的模块文件;deploy/watchdog 为控制台与调度器的进程外保活脚本。
This commit is contained in:
wangyunlong committed 2026-10-09 13:45:56 +08:00
1 parent dc8be88e60
commit 36fe7b81f8
3 files changed
+1223

No files matched your search

+336
View File
@@ -0,0 +1,336 @@
# -*- coding: utf-8 -*-
"""控制台与调度器的进程外保活(登录自启动,随用户会话常驻)。
职责(不承载任何业务调度规则,业务调度仍由 `gyxx schedule run` 唯一负责):
1. 控制台:探测 http://127.0.0.1:8765/api/health——进程不在则拉起;
进程在但探测连续失败(wedged)则结束该进程树后拉起。控制台进程内
另有看门狗(console.py `_console_watchdog`)自检失败会主动退出,此处
负责把退出后的空窗补上,两层互为补充。
2. 调度器:读 `var/state/scheduler/service.lock` 的 PID 判断存活,锁主
已死则按 install.ps1 同款参数拉起 `python -m gyxx_flow schedule run`;
调度器自身单实例锁保证不会双跑。
3. 节流:每个目标有冷却间隔;短时间反复重启触发熔断,只记录不再拉起,
避免故障风暴。
用法:
gyxx_keepalive.py # 常驻循环(Startup 的 vbs 以 pythonw 启动)
gyxx_keepalive.py --once # 单轮探测+处置后退出,供验收/手动巡检
日志:var/logs/keepalive/keepalive.log(追加)。
自身单实例:var/state/keepalive.lock(PID 存活检测,重复启动直接退出)。
"""
from __future__ import annotations
import argparse
import ctypes
import http.client
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
ENV_FILES = [
Path(os.getenv("GYXX_RUNTIME_ENV_FILE", r"D:\product-collector-analyze-flow\.env")),
Path(
os.getenv(
"GYXX_RUNTIME_JUSHUITAN_ENV_FILE",
r"D:\product-collector-analyze-flow\private-secrets\jushuitan.env",
)
),
]
CONSOLE_HOST = "127.0.0.1"
CONSOLE_PORT = int(os.getenv("GYXX_CONSOLE_PORT", "8765"))
LOOP_INTERVAL_SECONDS = float(os.getenv("GYXX_KEEPALIVE_INTERVAL_SECONDS", "30"))
PROBE_TIMEOUT_SECONDS = 5.0
PROBE_FAILURE_LIMIT = 3
RESTART_COOLDOWN_SECONDS = float(os.getenv("GYXX_KEEPALIVE_COOLDOWN_SECONDS", "120"))
TRIP_MAX_RESTARTS = 5
TRIP_WINDOW_SECONDS = 30 * 60
TRIP_PAUSE_SECONDS = 60 * 60
DATA_ROOT = PROJECT_ROOT / "var"
LOG_PATH = DATA_ROOT / "logs" / "keepalive" / "keepalive.log"
LOCK_PATH = DATA_ROOT / "state" / "keepalive.lock"
CONSOLE_LOG_DIR = DATA_ROOT / "logs" / "console-restarts"
SCHEDULER_LOG_DIR = DATA_ROOT / "logs" / "scheduler-service"
SCHEDULER_SERVICE_LOCK = DATA_ROOT / "state" / "scheduler" / "service.lock"
def _log(event: str, detail: str = "") -> None:
stamp = datetime.now(timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S")
line = f"{stamp} {event} {detail}".rstrip()
LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
with LOG_PATH.open("a", encoding="utf-8", newline="\n") as stream:
stream.write(line + "\n")
print(line, flush=True)
def _pid_alive(pid: int) -> bool:
if pid <= 0:
return False
kernel32 = ctypes.windll.kernel32
handle = kernel32.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION
if not handle:
return False
try:
exit_code = ctypes.c_ulong()
if kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)):
return exit_code.value == 259 # STILL_ACTIVE
return False
finally:
kernel32.CloseHandle(handle)
def _acquire_self_lock() -> bool:
LOCK_PATH.parent.mkdir(parents=True, exist_ok=True)
if LOCK_PATH.is_file():
try:
existing = json.loads(LOCK_PATH.read_text(encoding="utf-8"))
if _pid_alive(int(existing.get("pid", 0))):
return False
except Exception:
pass
LOCK_PATH.write_text(
json.dumps({"pid": os.getpid(), "started_at": datetime.now(timezone.utc).isoformat()}),
encoding="utf-8",
)
return True
def _console_env_files() -> list[Path]:
return [path for path in ENV_FILES if path.is_file()]
def _spawn(argv: list[str], log_dir: Path, log_prefix: str) -> int:
log_dir.mkdir(parents=True, exist_ok=True)
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
log_path = log_dir / f"{log_prefix}-{stamp}.log"
kwargs: dict[str, object] = {}
if os.name == "nt":
kwargs["creationflags"] = (
subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP
)
else:
kwargs["start_new_session"] = True
with log_path.open("a", encoding="utf-8", newline="\n") as stream:
process = subprocess.Popen(
argv,
cwd=str(PROJECT_ROOT),
env=os.environ,
stdin=subprocess.DEVNULL,
stdout=stream,
stderr=subprocess.STDOUT,
shell=False,
**kwargs,
)
return process.pid
def _console_argv() -> list[str]:
argv = [
str(PROJECT_ROOT / ".venv" / "Scripts" / "python.exe"),
"-m",
"gyxx_flow",
"console",
"--host",
CONSOLE_HOST,
"--port",
str(CONSOLE_PORT),
]
for env_file in _console_env_files():
argv.extend(("--env-file", str(env_file)))
return argv
def _scheduler_argv() -> list[str]:
argv = [
str(PROJECT_ROOT / ".venv" / "Scripts" / "python.exe"),
"-m",
"gyxx_flow",
"schedule",
"run",
]
for env_file in _console_env_files():
argv.extend(("--env-file", str(env_file)))
return argv
def _probe_console() -> bool:
try:
connection = http.client.HTTPConnection(CONSOLE_HOST, CONSOLE_PORT, timeout=PROBE_TIMEOUT_SECONDS)
try:
connection.request("GET", "/api/health")
response = connection.getresponse()
response.read()
return response.status == 200
finally:
connection.close()
except Exception:
return False
def _port_owner_pid(port: int) -> int | None:
try:
output = subprocess.run(
["netstat", "-ano", "-p", "tcp"],
capture_output=True,
text=True,
timeout=10,
).stdout
except Exception:
return None
needle = f"{CONSOLE_HOST}:{port}"
for line in output.splitlines():
parts = line.split()
if (
len(parts) >= 5
and parts[0].upper() == "TCP"
and parts[1] == needle
and parts[3].upper() == "LISTENING"
):
try:
return int(parts[4])
except ValueError:
return None
return None
def _kill_pid_tree(pid: int) -> None:
if pid <= 0 or pid == os.getpid():
return
subprocess.run(
["taskkill", "/PID", str(pid), "/T", "/F"],
capture_output=True,
timeout=30,
)
def _scheduler_lock_pid() -> int | None:
try:
payload = json.loads(SCHEDULER_SERVICE_LOCK.read_text(encoding="utf-8"))
return int(payload.get("pid", 0)) or None
except Exception:
return None
class _TargetState:
"""记录单目标的连续失败计数、冷却与熔断状态。"""
def __init__(self, name: str) -> None:
self.name = name
self.failures = 0
self.last_restart_at = 0.0
self.recent_restarts: list[float] = []
self.tripped_at: float | None = None
def tripped(self, now: float) -> bool:
if self.tripped_at is None:
return False
if now - self.tripped_at >= TRIP_PAUSE_SECONDS:
self.tripped_at = None
self.recent_restarts.clear()
_log("keepalive-trip-reset", f"target={self.name} 熔断窗口结束恢复守护")
return False
return True
def may_restart(self, now: float) -> bool:
if self.tripped(now):
return False
if now - self.last_restart_at < RESTART_COOLDOWN_SECONDS:
return False
self.recent_restarts = [
stamp for stamp in self.recent_restarts if now - stamp < TRIP_WINDOW_SECONDS
]
if len(self.recent_restarts) >= TRIP_MAX_RESTARTS:
self.tripped_at = now
_log(
"keepalive-tripped",
f"target={self.name} {TRIP_MAX_RESTARTS} 次重启/{TRIP_WINDOW_SECONDS // 60}分钟 "
f"暂停拉起{TRIP_PAUSE_SECONDS // 60}分钟,请人工检查日志",
)
return False
return True
def record_restart(self, now: float) -> None:
self.last_restart_at = now
self.recent_restarts.append(now)
def run_cycle(console: _TargetState, scheduler: _TargetState) -> None:
now = time.time()
# ---- 控制台 ----
if _probe_console():
console.failures = 0
else:
console.failures += 1
owner = _port_owner_pid(CONSOLE_PORT)
_log(
"console-probe-failed",
f"{console.failures}/{PROBE_FAILURE_LIMIT} port_owner_pid={owner}",
)
if console.failures >= PROBE_FAILURE_LIMIT and console.may_restart(now):
if owner is not None and _pid_alive(owner):
_log("console-kill-wedged", f"pid={owner} 探测持续失败,结束进程树")
_kill_pid_tree(owner)
time.sleep(3)
pid = _spawn(_console_argv(), CONSOLE_LOG_DIR, "keepalive-console")
console.record_restart(time.time())
console.failures = 0
_log("console-restarted", f"pid={pid}")
# ---- 调度器 ----
scheduler_pid = _scheduler_lock_pid()
scheduler_alive = scheduler_pid is not None and _pid_alive(scheduler_pid)
if not scheduler_alive and scheduler.may_restart(now):
pid = _spawn(_scheduler_argv(), SCHEDULER_LOG_DIR, "keepalive-start")
scheduler.record_restart(time.time())
_log(
"scheduler-restarted",
f"pid={pid} prev_lock_pid={scheduler_pid}",
)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="GYXX Flow 控制台/调度器保活")
parser.add_argument("--once", action="store_true", help="单轮探测+处置后退出")
arguments = parser.parse_args(argv)
if not _acquire_self_lock():
_log("keepalive-duplicate-exit", "已有保活进程在运行")
return 0
console = _TargetState("console")
scheduler = _TargetState("scheduler")
missing_env = [str(path) for path in ENV_FILES if not path.is_file()]
_log(
"keepalive-start",
f"pid={os.getpid()} once={arguments.once}"
+ (f" missing_env={missing_env}" if missing_env else ""),
)
try:
while True:
run_cycle(console, scheduler)
if arguments.once:
_log("keepalive-once-done", "")
return 0
time.sleep(LOOP_INTERVAL_SECONDS)
except KeyboardInterrupt:
return 0
finally:
try:
if LOCK_PATH.is_file():
payload = json.loads(LOCK_PATH.read_text(encoding="utf-8"))
if int(payload.get("pid", 0)) == os.getpid():
LOCK_PATH.unlink()
except Exception:
pass
if __name__ == "__main__":
sys.exit(main())
+5
View File
@@ -0,0 +1,5 @@
' GYXX Flow 保活登录自启动:无窗口启动 deploy/watchdog/gyxx_keepalive.py,
' 由它负责拉起控制台(含进程内自检看门狗)与调度器。
' 安装:复制到 %APPDATA%\Microsoft\Windows\Start Menu\Programs\Startup\
CreateObject("WScript.Shell").Run _
"""D:\gyxx-flow\.venv\Scripts\pythonw.exe"" ""D:\gyxx-flow\deploy\watchdog\gyxx_keepalive.py""", 0, False
+882
View File
@@ -0,0 +1,882 @@
"""Per-account cookie health gate injected before workflow collection steps.
Every runtime account a workflow depends on is checked before the collection
steps that use it may run. Accounts fall into three gate modes derived from
``config/runtime-bindings.json``:
* ``blocking`` -- QR-code / SMS-only accounts (no credential environment and
no configured login command). The gate verifies the login state and, when
it has expired, fails the gate step so every dependent step is skipped with
``dependency-failed`` and sends a Feishu alert asking the login owner to
re-login interactively via ``gyxx accounts login <account_id>``.
* ``recover`` -- accounts with a vault-managed credential login command
(``login_mode`` ``C``). The keepalive manager re-runs the credential login
automatically; only a failed recovery blocks the workflow and alerts.
* ``defer`` -- accounts whose member scripts embed username/password login
flows. These keep their script-level login behaviour unchanged and are not
gated at all.
The per-account decision is persisted as a small state machine under
``state/accounts/<id>/cookie_state.json`` (``healthy`` <-> ``awaiting_login``)
so repeat failures alert at most once per notify interval instead of once per
workflow run.
"""
from __future__ import annotations
import json
import os
from collections.abc import Callable
from dataclasses import dataclass, replace
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any, Mapping
from gyxx_flow.accounts import (
AccountKeepaliveManager,
_keepalive_requires_relogin,
_read_keepalive_status,
_vault_cookie_valid,
)
from gyxx_flow.adapters.browser import BrowserCookieStore
from gyxx_flow.adapters.feishu_messages import send_lark_bot_message
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
from gyxx_flow.catalog import WorkflowCatalog
from gyxx_flow.core.artifacts import atomic_write_json
from gyxx_flow.core.config import Settings
from gyxx_flow.core.context import RunContext
from gyxx_flow.notification_routing import (
ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID,
ANALYZER_NOTIFICATION_APP_PROFILE,
NotificationRoutingStore,
)
from gyxx_flow.workflow.model import StepDefinition, WorkflowDefinition
from gyxx_flow.workflow.steps import StepExecution
GATE_MODE_BLOCKING = "blocking"
GATE_MODE_RECOVER = "recover"
GATE_MODE_DEFER = "defer"
# 飞书通知的默认接收人:按手机号在 config/notification-routing.json 的
# people 段解析 open_id;解析失败时回退到同名 analyzer 常量收件人。
DEFAULT_LOGIN_ALERT_MOBILE = "+8613663785145"
GYXX_COOKIE_GATE = "GYXX_COOKIE_GATE"
GYXX_COOKIE_GATE_MAX_STATUS_AGE_SECONDS = "GYXX_COOKIE_GATE_MAX_STATUS_AGE_SECONDS"
GYXX_LOGIN_ALERT_OPEN_IDS = "GYXX_LOGIN_ALERT_OPEN_IDS"
GYXX_LOGIN_ALERT_APP_PROFILE = "GYXX_LOGIN_ALERT_APP_PROFILE"
GYXX_LOGIN_ALERT_NOTIFY_INTERVAL_SECONDS = "GYXX_LOGIN_ALERT_NOTIFY_INTERVAL_SECONDS"
_GATE_STEP_PREFIX = "cookie-gate"
_GATE_STATE_FILENAME = "cookie_state.json"
_GATE_TIMEOUT_SECONDS = 1800.0
_DEFAULT_MAX_STATUS_AGE_SECONDS = 3600.0
_MIN_MAX_STATUS_AGE_SECONDS = 300.0
_DEFAULT_NOTIFY_INTERVAL_SECONDS = 1800.0
_DISABLED_VALUES = frozenset({"0", "false", "off", "no", "disabled"})
# 自动重跑:待重跑清单上限、健康探活节流、同账号重跑启动冷却。
_MAX_PENDING_RERUNS = 20
_RECOVERY_PROBE_INTERVAL_SECONDS = 300.0
_RECOVERY_LAUNCH_COOLDOWN_SECONDS = 300.0
_RECOVERY_HISTORY_LIMIT = 20
STATE_HEALTHY = "healthy"
STATE_AWAITING_LOGIN = "awaiting_login"
@dataclass(frozen=True, slots=True)
class GateVerdict:
"""One account's gate decision, recorded in the run journal on failure."""
account_id: str
mode: str
blocked: bool
message: str
recovered: bool = False
notified: bool = False
@dataclass(frozen=True, slots=True)
class CookieGateStepAction:
"""ExecutableStep that turns one account verdict into a step outcome."""
gate: "CookieGate"
account_id: str
def execute(
self,
*,
context: RunContext,
timeout_seconds: float | None,
dry_run: bool,
) -> StepExecution:
if dry_run: # pragma: no cover - the engine skips dry-run steps first
return StepExecution(exit_code=0, skipped=True, reason="dry-run")
verdict = self.gate.check(self.account_id, context=context)
if verdict.blocked:
return StepExecution(
exit_code=1,
error=verdict.message,
reason="cookie-expired",
)
return StepExecution(exit_code=0)
class CookieGate:
"""Resolve, probe, and gate the accounts one workflow depends on."""
def __init__(
self,
settings: Settings,
*,
catalog: RuntimeIntegrationCatalog | None = None,
keepalive: AccountKeepaliveManager | None = None,
environment: Mapping[str, str] | None = None,
now: Callable[[], datetime] | None = None,
sender: Callable[..., dict[str, Any]] | None = None,
) -> None: # noqa: D107 - simple wiring
self.settings = settings
self.catalog = catalog or RuntimeIntegrationCatalog.load_default(
project_root=settings.project_root,
data_root=settings.data_root,
)
self.keepalive = keepalive or AccountKeepaliveManager(
settings, catalog=self.catalog
)
self._environment = dict(os.environ if environment is None else environment)
self._now = now or (lambda: datetime.now(timezone.utc))
self._sender = sender or send_lark_bot_message
self._workflow_catalog: WorkflowCatalog | None = None
# ------------------------------------------------------------------
# Account resolution
# ------------------------------------------------------------------
def gate_mode(self, account_id: str) -> str:
account = self.catalog.account_for(account_id)
if str(account.login_command_id or "").strip():
return GATE_MODE_RECOVER
if account.credential_env_names or account.login_mode == "B":
return GATE_MODE_DEFER
return GATE_MODE_BLOCKING
def collect_workflow_accounts(
self,
workflow: WorkflowDefinition,
) -> dict[str, set[str]]:
"""Map each gated account to the step ids that depend on it.
Accounts resolve two ways: directly, from each step binding's
``account``; and declaratively, from the catalog workflow's
``required_accounts`` (used by orchestrator entries whose own binding
has no account while the spawned child collectors do).
"""
accounts: dict[str, set[str]] = {}
for step in workflow.steps:
account_id = self._account_for_step(step)
if account_id:
accounts.setdefault(account_id, set()).add(step.step_id)
for account_id in self._declared_required_accounts(workflow.workflow_id):
self.catalog.account_for(account_id) # unknown id: loud config error
accounts.setdefault(account_id, set()).update(
step.step_id for step in workflow.steps
)
# Defer-mode accounts keep their script-level credential login flows
# and are intentionally not gated.
return {
account_id: step_ids
for account_id, step_ids in accounts.items()
if self.gate_mode(account_id) != GATE_MODE_DEFER
}
def _account_for_step(self, step: StepDefinition) -> str | None:
action = step.action
entry = getattr(action, "entry", None)
module_id = getattr(entry, "module", "")
command_entry = getattr(action, "command_entry", None) or getattr(
entry, "entry", ""
)
if not module_id or not command_entry:
return None
try:
binding = self.catalog.binding_for(f"{module_id}:{command_entry}")
except Exception: # noqa: BLE001 - unbound entries run ungated
return None
return binding.account or None
def workflow_catalog(self) -> WorkflowCatalog:
"""Load (once) the catalog used for declared account and rerun lookups."""
if self._workflow_catalog is None:
try:
self._workflow_catalog = WorkflowCatalog.load(
self.settings.project_root / "config"
)
except Exception: # noqa: BLE001 - catalog issues surface elsewhere
self._workflow_catalog = WorkflowCatalog(
timezone="UTC", workflows=(), schedules=()
)
return self._workflow_catalog
def _declared_required_accounts(self, workflow_id: str) -> tuple[str, ...]:
for entry in self.workflow_catalog().workflows:
if entry.workflow_id == workflow_id:
return tuple(entry.required_accounts)
return ()
# ------------------------------------------------------------------
# State machine
# ------------------------------------------------------------------
def check(self, account_id: str, *, context: RunContext) -> GateVerdict:
account = self.catalog.account_for(account_id)
mode = self.gate_mode(account_id)
store = BrowserCookieStore(account.cookie_file, account.storage_state_file)
static_valid = _vault_cookie_valid(account, store)
status = _read_keepalive_status(account)
if static_valid and self._keepalive_success_is_fresh(status):
self._write_state(account, STATE_HEALTHY, probe="fresh-keepalive")
return GateVerdict(
account_id=account_id,
mode=mode,
blocked=False,
message=f"账号 {account_id} 登录态有效(keepalive 近期验证通过)",
)
if self._acceptance_enabled():
# 验收模式不开浏览器:按静态证据判定,且不投递飞书通知。
if static_valid and not _keepalive_requires_relogin(status):
return GateVerdict(
account_id=account_id,
mode=mode,
blocked=False,
message=f"账号 {account_id} 静态 cookie 校验通过(验收模式)",
)
return self._expire(
account,
mode,
context,
detail="验收模式下静态 cookie 校验未通过",
notify=False,
)
outcome = self.keepalive.refresh_account(account_id)
probe_status = str(outcome.get("status", "error"))
if probe_status == "success":
recovered = bool(outcome.get("login_recovered"))
self._write_state(
account,
STATE_HEALTHY,
probe="success",
recovered=recovered,
)
message = (
f"账号 {account_id} 登录态有效(自动账密重登成功)"
if recovered
else f"账号 {account_id} 登录态有效(探活通过)"
)
return GateVerdict(
account_id=account_id,
mode=mode,
blocked=False,
message=message,
recovered=recovered,
)
if probe_status == "busy":
if static_valid and not _keepalive_requires_relogin(status):
return GateVerdict(
account_id=account_id,
mode=mode,
blocked=False,
message=f"账号 {account_id} 探活锁忙,静态 cookie 校验通过",
)
return self._expire(
account,
mode,
context,
detail="探活锁被占用且静态 cookie 校验未通过",
)
if probe_status in {"expired", "not_logged_in"}:
evidence = _recovery_evidence(_read_keepalive_status(account))
detail = "cookie 已过期,无法正常采集"
if probe_status == "not_logged_in" and evidence:
detail = f"登录态缺失;{evidence}"
elif evidence:
detail = f"cookie 已过期;{evidence}"
return self._expire(account, mode, context, detail=detail)
# timeout / error / http_error / disabled:探测结果不可信时回退静态证据。
if static_valid and not _keepalive_requires_relogin(status):
return GateVerdict(
account_id=account_id,
mode=mode,
blocked=False,
message=f"账号 {account_id} 探活异常({probe_status}),静态 cookie 校验通过",
)
return self._expire(
account,
mode,
context,
detail=f"探活异常({probe_status})且静态 cookie 校验未通过",
)
# ------------------------------------------------------------------
# Expiry, notification, persisted state
# ------------------------------------------------------------------
def _expire(
self,
account: Any,
mode: str,
context: RunContext,
*,
detail: str,
notify: bool | None = None,
) -> GateVerdict:
account_id = str(account.account_id)
if notify is None:
notify = not context.shadow and not self._acceptance_enabled()
state = self._read_state(account)
notified = False
notification_note = ""
if notify and not self._within_notify_interval(state):
recipients = self._alert_recipients()
message = _expiry_message(account_id, mode, context, detail)
for open_id, profile in recipients:
try:
self._sender(
user_id=open_id,
text=message,
idempotency_key=f"cookie-gate:{account_id}:{context.run_id}",
profile=profile,
)
notified = True
except Exception as exc: # noqa: BLE001 - alerting must not mask the verdict
notification_note = f"{type(exc).__name__}: {exc}"[:300]
if recipients and notified:
state["last_notified_at"] = _now_iso(self._now)
state.pop("notification_error", None)
elif notification_note:
state["notification_error"] = notification_note
state["state"] = STATE_AWAITING_LOGIN
state["updated_at"] = _now_iso(self._now)
state["last_probe_status"] = "expired"
state["last_blocked_run"] = context.run_id
if not context.shadow and not self._acceptance_enabled():
# 记录待自动重跑清单:登录态恢复后由调度器维护循环按原业务日期重跑。
state["pending_reruns"] = _record_pending_rerun(
state.get("pending_reruns"), context, _now_iso(self._now)
)
self._save_state(account, state)
message = _expiry_message(account_id, mode, context, detail)
if notified:
message += " 已通过飞书通知登录负责人。"
elif notification_note:
message += f" 飞书通知发送失败({notification_note})。"
return GateVerdict(
account_id=account_id,
mode=mode,
blocked=True,
message=message,
notified=notified,
)
def _write_state(
self,
account: Any,
state_name: str,
*,
probe: str,
recovered: bool = False,
) -> None:
state = self._read_state(account)
if (
state_name == STATE_HEALTHY
and state.get("state") == STATE_HEALTHY
and state.get("last_probe_status") in {"fresh-keepalive", "success"}
and not recovered
):
return # 持续健康时避免每次运行都重写状态文件
state["state"] = state_name
state["updated_at"] = _now_iso(self._now)
state["last_probe_status"] = probe
if state_name == STATE_HEALTHY:
state["last_healthy_at"] = _now_iso(self._now)
if recovered:
state["last_recovered_at"] = _now_iso(self._now)
state.pop("notification_error", None)
self._save_state(account, state)
def _state_path(self, account: Any) -> Path:
return account.cookie_file.with_name(_GATE_STATE_FILENAME)
def _read_state(self, account: Any) -> dict[str, object]:
path = self._state_path(account)
if not path.exists():
return {"account_id": str(account.account_id)}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {"account_id": str(account.account_id)}
if not isinstance(payload, dict):
return {"account_id": str(account.account_id)}
return dict(payload)
def _save_state(self, account: Any, state: Mapping[str, object]) -> None:
payload = {
"account_id": str(account.account_id),
**state,
}
try:
atomic_write_json(self._state_path(account), payload)
except OSError: # pragma: no cover - state file is advisory only
pass
def _within_notify_interval(self, state: Mapping[str, object]) -> bool:
raw = state.get("last_notified_at")
if not isinstance(raw, str):
return False
try:
last = datetime.fromisoformat(raw)
except ValueError:
return False
now = self._now()
if last.tzinfo is None:
last = last.replace(tzinfo=timezone.utc)
interval = _env_int(
self._environment,
GYXX_LOGIN_ALERT_NOTIFY_INTERVAL_SECONDS,
_DEFAULT_NOTIFY_INTERVAL_SECONDS,
minimum=0,
)
return (now - last).total_seconds() < interval
def _keepalive_success_is_fresh(self, status: Mapping[str, object] | None) -> bool:
if not isinstance(status, Mapping) or status.get("status") != "success":
return False
raw = status.get("last_success_at")
if not isinstance(raw, str):
return False
try:
last = datetime.fromisoformat(raw)
except ValueError:
return False
now = self._now()
if last.tzinfo is None:
last = last.replace(tzinfo=timezone.utc)
max_age = _env_float(
self._environment,
GYXX_COOKIE_GATE_MAX_STATUS_AGE_SECONDS,
_DEFAULT_MAX_STATUS_AGE_SECONDS,
minimum=_MIN_MAX_STATUS_AGE_SECONDS,
)
return (now - last).total_seconds() <= max_age
def _alert_recipients(self) -> list[tuple[str, str]]:
profile = (
self._environment.get(GYXX_LOGIN_ALERT_APP_PROFILE, "").strip()
or ANALYZER_NOTIFICATION_APP_PROFILE
)
override = [
value.strip()
for value in self._environment.get(GYXX_LOGIN_ALERT_OPEN_IDS, "").split(",")
if value.strip()
]
if override:
return [(open_id, profile) for open_id in override]
try:
payload, _revision = NotificationRoutingStore(self.settings).snapshot()
for person in payload["people"].values():
if person.get("mobile") == DEFAULT_LOGIN_ALERT_MOBILE:
binding = person.get("bindings", {}).get(profile)
if binding:
return [(str(binding["open_id"]), profile)]
except Exception: # noqa: BLE001 - fall back to the constant recipient
pass
return [(ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID, profile)]
def _acceptance_enabled(self) -> bool:
value = self._environment.get("GYXX_WORKFLOW_ACCEPTANCE", "")
return bool(value and value.strip().casefold() in {"1", "true", "yes", "on"})
def gate_enabled(environment: Mapping[str, str] | None = None) -> bool:
values = os.environ if environment is None else environment
return values.get(GYXX_COOKIE_GATE, "").strip().casefold() not in _DISABLED_VALUES
def apply_cookie_gates(
workflow: WorkflowDefinition,
*,
settings: Settings,
environment: Mapping[str, str] | None = None,
gate: CookieGate | None = None,
) -> WorkflowDefinition:
"""Inject one critical gate step per gated account in front of its users.
Gate steps are graph sources; each step that uses an account gains the
account's gate as an extra dependency. A failed gate therefore skips its
dependents with ``dependency-failed`` through the regular engine, while
unrelated branches keep running. Operator slices (``--rerun-step`` /
``--resume-from``) are produced before this injection and stay gate-free.
"""
if not workflow.steps:
return workflow
if gate is None:
try:
gate = CookieGate(
settings,
environment=os.environ if environment is None else environment,
)
except Exception: # noqa: BLE001 - 无集成配置时不门控;步骤执行仍会显式报错
return workflow
account_steps = gate.collect_workflow_accounts(workflow)
if not account_steps:
return workflow
existing_ids = {step.step_id for step in workflow.steps}
gate_steps: list[StepDefinition] = []
for account_id in sorted(account_steps):
step_id = f"{_GATE_STEP_PREFIX}.{account_id}"
if step_id in existing_ids:
continue
gate_steps.append(
StepDefinition(
step_id,
CookieGateStepAction(gate, account_id),
timeout_seconds=_GATE_TIMEOUT_SECONDS,
max_attempts=1,
)
)
if not gate_steps:
return workflow
patched: list[StepDefinition] = []
for step in workflow.steps:
dependencies = list(dict.fromkeys(step.depends_on))
changed = False
for account_id, step_ids in account_steps.items():
gate_id = f"{_GATE_STEP_PREFIX}.{account_id}"
if step.step_id in step_ids and gate_id not in dependencies:
dependencies.append(gate_id)
changed = True
if changed:
step = replace(step, depends_on=tuple(dependencies))
patched.append(step)
return WorkflowDefinition(
workflow.workflow_id,
(*gate_steps, *patched),
)
class CookieGateRecoveryManager:
"""Re-run workflows blocked by expired cookies once the login recovers.
Runs inside the resident scheduler's maintenance loop. For every account
with a non-empty ``pending_reruns`` list, it re-verifies the login state
(fresh keepalive success first, then a throttled live probe); on recovery
it relaunches each recorded workflow with its original business date
through the scheduler's own launcher, clears the pending list, and sends a
Feishu confirmation to the login owner.
"""
def __init__(
self,
gate: CookieGate,
*,
launcher: Any,
probe_interval_seconds: float = _RECOVERY_PROBE_INTERVAL_SECONDS,
launch_cooldown_seconds: float = _RECOVERY_LAUNCH_COOLDOWN_SECONDS,
) -> None:
self.gate = gate
self._launcher = launcher
self._probe_interval = probe_interval_seconds
self._launch_cooldown = launch_cooldown_seconds
def tick(self) -> list[dict[str, object]]:
"""Attempt recovery reruns once; returns redacted outcomes per account."""
outcomes: list[dict[str, object]] = []
for account in self.gate.catalog.accounts:
state = self.gate._read_state(account)
pending = state.get("pending_reruns")
if not isinstance(pending, list) or not pending:
continue
now = self.gate._now()
if not self._launch_cooldown_passed(state, now):
continue
if not self._account_recovered(account, state, now):
continue
launched, remaining = self._launch_pending(account, pending, now)
state["pending_reruns"] = remaining
state["updated_at"] = _now_iso(self.gate._now)
state["last_recovered_at"] = state["updated_at"]
if not remaining:
state["state"] = STATE_HEALTHY
state["last_probe_status"] = "recovered-rerun"
if launched:
state["last_rerun_launch_at"] = state["updated_at"]
history = [
item
for item in state.get("rerun_history", [])
if isinstance(item, dict)
]
history.extend(launched)
state["rerun_history"] = history[-_RECOVERY_HISTORY_LIMIT:]
self.gate._save_state(account, state)
if launched:
self._notify_recovered(str(account.account_id), launched)
outcomes.append(
{
"account_id": str(account.account_id),
"launched": launched,
"pending": len(remaining),
}
)
return outcomes
# ------------------------------------------------------------------
# internals
# ------------------------------------------------------------------
def _launch_cooldown_passed(
self, state: Mapping[str, object], now: datetime
) -> bool:
raw = state.get("last_rerun_launch_at")
if not isinstance(raw, str):
return True
try:
last = datetime.fromisoformat(raw)
except ValueError:
return True
if last.tzinfo is None:
last = last.replace(tzinfo=timezone.utc)
return (now - last).total_seconds() >= self._launch_cooldown
def _account_recovered(
self,
account: Any,
state: dict[str, object],
now: datetime,
) -> bool:
"""Fresh keepalive success first; otherwise one throttled live probe."""
store = BrowserCookieStore(account.cookie_file, account.storage_state_file)
if _vault_cookie_valid(account, store) and self.gate._keepalive_success_is_fresh(
_read_keepalive_status(account)
):
return True
raw = state.get("last_recovery_probe_at")
if isinstance(raw, str):
try:
last_probe = datetime.fromisoformat(raw)
except ValueError:
last_probe = None
if last_probe is not None:
if last_probe.tzinfo is None:
last_probe = last_probe.replace(tzinfo=timezone.utc)
if (now - last_probe).total_seconds() < self._probe_interval:
return False
state["last_recovery_probe_at"] = _now_iso(self.gate._now)
self.gate._save_state(account, state)
try:
outcome = self.gate.keepalive.refresh_account(str(account.account_id))
except Exception: # noqa: BLE001 - 恢复探活失败留给下一轮
return False
return str(outcome.get("status", "error")) == "success"
def _launch_pending(
self,
account: Any,
pending: list[object],
now: datetime,
) -> tuple[list[dict[str, object]], list[dict[str, object]]]:
known_ids = {
entry.workflow_id for entry in self.gate.workflow_catalog().workflows
}
launched: list[dict[str, object]] = []
remaining: list[dict[str, object]] = []
for item in pending:
if not isinstance(item, dict):
continue # 控制台脚本等临时运行 id:不在 catalog 中,直接丢弃
workflow_id = item.get("workflow_id")
business_date = item.get("business_date")
if not isinstance(workflow_id, str) or not isinstance(business_date, str):
continue
if workflow_id not in known_ids:
continue
try:
process = self._launcher.launch(
workflow_id, now, date.fromisoformat(business_date)
)
except Exception as exc: # noqa: BLE001 - 保留 pending,下一轮重试
remaining.append(
{
"workflow_id": workflow_id,
"business_date": business_date,
"error": f"{type(exc).__name__}: {exc}"[:200],
}
)
continue
close = getattr(self._launcher, "close", None)
if callable(close):
try:
close(process)
except Exception: # noqa: BLE001 - 关闭日志句柄失败不影响重跑
pass
launched.append(
{
"workflow_id": workflow_id,
"business_date": business_date,
"pid": int(getattr(process, "pid", 0) or 0),
"launched_at": _now_iso(self.gate._now),
}
)
return launched, remaining
def _notify_recovered(
self,
account_id: str,
launched: list[dict[str, object]],
) -> None:
lines = [
f"- {item['workflow_id']}(业务日期 {item['business_date']})"
for item in launched
]
message = (
"【GYXX 登录态已恢复】账号 "
f"{account_id} 登录态已恢复,已自动重跑被阻断的工作流:\n"
+ "\n".join(lines)
+ "\n如本次重跑仍有失败,请查看运行日志或工作流控制台。"
)
key = "-".join(
[account_id, *(str(item["workflow_id"]) for item in launched)]
)
for open_id, profile in self.gate._alert_recipients():
try:
self.gate._sender(
user_id=open_id,
text=message,
idempotency_key=f"cookie-recovery:{key}",
profile=profile,
)
except Exception: # noqa: BLE001 - 确认通知失败不影响已发起的重跑
continue
def _record_pending_rerun(
current: object,
context: RunContext,
blocked_at: str,
) -> list[dict[str, object]]:
"""Append one blocked run to the pending list, deduped and bounded."""
pending: list[dict[str, object]] = [
dict(item)
for item in (current if isinstance(current, list) else [])
if isinstance(item, dict)
]
entry = {
"workflow_id": context.workflow_id,
"business_date": context.business_date.isoformat(),
"blocked_run_id": context.run_id,
"blocked_at": blocked_at,
}
pending = [
item
for item in pending
if not (
item.get("workflow_id") == entry["workflow_id"]
and item.get("business_date") == entry["business_date"]
)
]
pending.append(entry)
return pending[-_MAX_PENDING_RERUNS:]
def _recovery_evidence(status: Mapping[str, object] | None) -> str:
if not isinstance(status, Mapping):
return ""
if not status.get("login_attempted"):
return ""
reason = status.get("login_reason")
error_type = status.get("login_error_type")
exit_code = status.get("login_exit_code")
parts = ["自动账密重登已尝试但未成功"]
if isinstance(reason, str) and reason:
parts.append(reason)
if isinstance(error_type, str) and error_type:
parts.append(f"错误={error_type}")
if isinstance(exit_code, int) and not isinstance(exit_code, bool):
parts.append(f"退出码={exit_code}")
return "(" + ",".join(parts) + ")"
def _expiry_message(
account_id: str,
mode: str,
context: RunContext,
detail: str,
) -> str:
mode_note = (
"账号密码自动重登失败,需人工处理(如验证码)"
if mode == GATE_MODE_RECOVER
else "该平台仅支持扫码/验证码登录,无法自动恢复"
)
return (
"【GYXX 登录态失效】工作流被阻断,需要人工重新登录\n"
f"工作流: {context.workflow_id}(业务日期 {context.business_date.isoformat()})\n"
f"账号: {account_id}\n"
f"原因: {detail};{mode_note}\n"
f"请在服务器上执行: uv run gyxx accounts login {account_id}\n"
"(将打开浏览器窗口,完成扫码或验证码登录后自动保存并同步到全部成员脚本;"
"受影响的采集步骤本次已跳过,登录完成后下次运行自动恢复)"
)
def _now_iso(now: Callable[[], datetime]) -> str:
return now().astimezone(timezone.utc).isoformat(timespec="seconds")
def _env_int(
environment: Mapping[str, str],
name: str,
default: int,
*,
minimum: int,
) -> int:
raw = environment.get(name, "").strip()
try:
value = int(raw)
except (TypeError, ValueError):
return default
return max(minimum, value)
def _env_float(
environment: Mapping[str, str],
name: str,
default: float,
*,
minimum: float,
) -> float:
raw = environment.get(name, "").strip()
try:
value = float(raw)
except (TypeError, ValueError):
return default
return max(minimum, value)
__all__ = [
"DEFAULT_LOGIN_ALERT_MOBILE",
"GATE_MODE_BLOCKING",
"GATE_MODE_DEFER",
"GATE_MODE_RECOVER",
"GYXX_COOKIE_GATE",
"GYXX_LOGIN_ALERT_OPEN_IDS",
"CookieGate",
"CookieGateRecoveryManager",
"CookieGateStepAction",
"GateVerdict",
"apply_cookie_gates",
"gate_enabled",
]