wangyunlong
36fe7b81f8
cookie_gate.py(账号三级门控/阻断记录 pending_reruns/恢复重跑管理器)为此前 接线提交遗漏的模块文件；deploy/watchdog 为控制台与调度器的进程外保活脚本。
883 lines
33 KiB
Python
883 lines
33 KiB
Python
"""Per-account cookie health gate injected before workflow collection steps.
|
||
|
||
Every runtime account a workflow depends on is checked before the collection
|
||
steps that use it may run. Accounts fall into three gate modes derived from
|
||
``config/runtime-bindings.json``:
|
||
|
||
* ``blocking`` -- QR-code / SMS-only accounts (no credential environment and
|
||
no configured login command). The gate verifies the login state and, when
|
||
it has expired, fails the gate step so every dependent step is skipped with
|
||
``dependency-failed`` and sends a Feishu alert asking the login owner to
|
||
re-login interactively via ``gyxx accounts login <account_id>``.
|
||
* ``recover`` -- accounts with a vault-managed credential login command
|
||
(``login_mode`` ``C``). The keepalive manager re-runs the credential login
|
||
automatically; only a failed recovery blocks the workflow and alerts.
|
||
* ``defer`` -- accounts whose member scripts embed username/password login
|
||
flows. These keep their script-level login behaviour unchanged and are not
|
||
gated at all.
|
||
|
||
The per-account decision is persisted as a small state machine under
|
||
``state/accounts/<id>/cookie_state.json`` (``healthy`` <-> ``awaiting_login``)
|
||
so repeat failures alert at most once per notify interval instead of once per
|
||
workflow run.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
from collections.abc import Callable
|
||
from dataclasses import dataclass, replace
|
||
from datetime import date, datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any, Mapping
|
||
|
||
from gyxx_flow.accounts import (
|
||
AccountKeepaliveManager,
|
||
_keepalive_requires_relogin,
|
||
_read_keepalive_status,
|
||
_vault_cookie_valid,
|
||
)
|
||
from gyxx_flow.adapters.browser import BrowserCookieStore
|
||
from gyxx_flow.adapters.feishu_messages import send_lark_bot_message
|
||
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
|
||
from gyxx_flow.catalog import WorkflowCatalog
|
||
from gyxx_flow.core.artifacts import atomic_write_json
|
||
from gyxx_flow.core.config import Settings
|
||
from gyxx_flow.core.context import RunContext
|
||
from gyxx_flow.notification_routing import (
|
||
ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID,
|
||
ANALYZER_NOTIFICATION_APP_PROFILE,
|
||
NotificationRoutingStore,
|
||
)
|
||
from gyxx_flow.workflow.model import StepDefinition, WorkflowDefinition
|
||
from gyxx_flow.workflow.steps import StepExecution
|
||
|
||
GATE_MODE_BLOCKING = "blocking"
|
||
GATE_MODE_RECOVER = "recover"
|
||
GATE_MODE_DEFER = "defer"
|
||
|
||
# 飞书通知的默认接收人：按手机号在 config/notification-routing.json 的
|
||
# people 段解析 open_id；解析失败时回退到同名 analyzer 常量收件人。
|
||
DEFAULT_LOGIN_ALERT_MOBILE = "+8613663785145"
|
||
|
||
GYXX_COOKIE_GATE = "GYXX_COOKIE_GATE"
|
||
GYXX_COOKIE_GATE_MAX_STATUS_AGE_SECONDS = "GYXX_COOKIE_GATE_MAX_STATUS_AGE_SECONDS"
|
||
GYXX_LOGIN_ALERT_OPEN_IDS = "GYXX_LOGIN_ALERT_OPEN_IDS"
|
||
GYXX_LOGIN_ALERT_APP_PROFILE = "GYXX_LOGIN_ALERT_APP_PROFILE"
|
||
GYXX_LOGIN_ALERT_NOTIFY_INTERVAL_SECONDS = "GYXX_LOGIN_ALERT_NOTIFY_INTERVAL_SECONDS"
|
||
|
||
_GATE_STEP_PREFIX = "cookie-gate"
|
||
_GATE_STATE_FILENAME = "cookie_state.json"
|
||
_GATE_TIMEOUT_SECONDS = 1800.0
|
||
_DEFAULT_MAX_STATUS_AGE_SECONDS = 3600.0
|
||
_MIN_MAX_STATUS_AGE_SECONDS = 300.0
|
||
_DEFAULT_NOTIFY_INTERVAL_SECONDS = 1800.0
|
||
_DISABLED_VALUES = frozenset({"0", "false", "off", "no", "disabled"})
|
||
# 自动重跑：待重跑清单上限、健康探活节流、同账号重跑启动冷却。
|
||
_MAX_PENDING_RERUNS = 20
|
||
_RECOVERY_PROBE_INTERVAL_SECONDS = 300.0
|
||
_RECOVERY_LAUNCH_COOLDOWN_SECONDS = 300.0
|
||
_RECOVERY_HISTORY_LIMIT = 20
|
||
|
||
STATE_HEALTHY = "healthy"
|
||
STATE_AWAITING_LOGIN = "awaiting_login"
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class GateVerdict:
|
||
"""One account's gate decision, recorded in the run journal on failure."""
|
||
|
||
account_id: str
|
||
mode: str
|
||
blocked: bool
|
||
message: str
|
||
recovered: bool = False
|
||
notified: bool = False
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class CookieGateStepAction:
|
||
"""ExecutableStep that turns one account verdict into a step outcome."""
|
||
|
||
gate: "CookieGate"
|
||
account_id: str
|
||
|
||
def execute(
|
||
self,
|
||
*,
|
||
context: RunContext,
|
||
timeout_seconds: float | None,
|
||
dry_run: bool,
|
||
) -> StepExecution:
|
||
if dry_run: # pragma: no cover - the engine skips dry-run steps first
|
||
return StepExecution(exit_code=0, skipped=True, reason="dry-run")
|
||
verdict = self.gate.check(self.account_id, context=context)
|
||
if verdict.blocked:
|
||
return StepExecution(
|
||
exit_code=1,
|
||
error=verdict.message,
|
||
reason="cookie-expired",
|
||
)
|
||
return StepExecution(exit_code=0)
|
||
|
||
|
||
class CookieGate:
|
||
"""Resolve, probe, and gate the accounts one workflow depends on."""
|
||
|
||
def __init__(
|
||
self,
|
||
settings: Settings,
|
||
*,
|
||
catalog: RuntimeIntegrationCatalog | None = None,
|
||
keepalive: AccountKeepaliveManager | None = None,
|
||
environment: Mapping[str, str] | None = None,
|
||
now: Callable[[], datetime] | None = None,
|
||
sender: Callable[..., dict[str, Any]] | None = None,
|
||
) -> None: # noqa: D107 - simple wiring
|
||
self.settings = settings
|
||
self.catalog = catalog or RuntimeIntegrationCatalog.load_default(
|
||
project_root=settings.project_root,
|
||
data_root=settings.data_root,
|
||
)
|
||
self.keepalive = keepalive or AccountKeepaliveManager(
|
||
settings, catalog=self.catalog
|
||
)
|
||
self._environment = dict(os.environ if environment is None else environment)
|
||
self._now = now or (lambda: datetime.now(timezone.utc))
|
||
self._sender = sender or send_lark_bot_message
|
||
self._workflow_catalog: WorkflowCatalog | None = None
|
||
|
||
# ------------------------------------------------------------------
|
||
# Account resolution
|
||
# ------------------------------------------------------------------
|
||
|
||
def gate_mode(self, account_id: str) -> str:
|
||
account = self.catalog.account_for(account_id)
|
||
if str(account.login_command_id or "").strip():
|
||
return GATE_MODE_RECOVER
|
||
if account.credential_env_names or account.login_mode == "B":
|
||
return GATE_MODE_DEFER
|
||
return GATE_MODE_BLOCKING
|
||
|
||
def collect_workflow_accounts(
|
||
self,
|
||
workflow: WorkflowDefinition,
|
||
) -> dict[str, set[str]]:
|
||
"""Map each gated account to the step ids that depend on it.
|
||
|
||
Accounts resolve two ways: directly, from each step binding's
|
||
``account``; and declaratively, from the catalog workflow's
|
||
``required_accounts`` (used by orchestrator entries whose own binding
|
||
has no account while the spawned child collectors do).
|
||
"""
|
||
|
||
accounts: dict[str, set[str]] = {}
|
||
for step in workflow.steps:
|
||
account_id = self._account_for_step(step)
|
||
if account_id:
|
||
accounts.setdefault(account_id, set()).add(step.step_id)
|
||
for account_id in self._declared_required_accounts(workflow.workflow_id):
|
||
self.catalog.account_for(account_id) # unknown id: loud config error
|
||
accounts.setdefault(account_id, set()).update(
|
||
step.step_id for step in workflow.steps
|
||
)
|
||
# Defer-mode accounts keep their script-level credential login flows
|
||
# and are intentionally not gated.
|
||
return {
|
||
account_id: step_ids
|
||
for account_id, step_ids in accounts.items()
|
||
if self.gate_mode(account_id) != GATE_MODE_DEFER
|
||
}
|
||
|
||
def _account_for_step(self, step: StepDefinition) -> str | None:
|
||
action = step.action
|
||
entry = getattr(action, "entry", None)
|
||
module_id = getattr(entry, "module", "")
|
||
command_entry = getattr(action, "command_entry", None) or getattr(
|
||
entry, "entry", ""
|
||
)
|
||
if not module_id or not command_entry:
|
||
return None
|
||
try:
|
||
binding = self.catalog.binding_for(f"{module_id}:{command_entry}")
|
||
except Exception: # noqa: BLE001 - unbound entries run ungated
|
||
return None
|
||
return binding.account or None
|
||
|
||
def workflow_catalog(self) -> WorkflowCatalog:
|
||
"""Load (once) the catalog used for declared account and rerun lookups."""
|
||
|
||
if self._workflow_catalog is None:
|
||
try:
|
||
self._workflow_catalog = WorkflowCatalog.load(
|
||
self.settings.project_root / "config"
|
||
)
|
||
except Exception: # noqa: BLE001 - catalog issues surface elsewhere
|
||
self._workflow_catalog = WorkflowCatalog(
|
||
timezone="UTC", workflows=(), schedules=()
|
||
)
|
||
return self._workflow_catalog
|
||
|
||
def _declared_required_accounts(self, workflow_id: str) -> tuple[str, ...]:
|
||
for entry in self.workflow_catalog().workflows:
|
||
if entry.workflow_id == workflow_id:
|
||
return tuple(entry.required_accounts)
|
||
return ()
|
||
|
||
# ------------------------------------------------------------------
|
||
# State machine
|
||
# ------------------------------------------------------------------
|
||
|
||
def check(self, account_id: str, *, context: RunContext) -> GateVerdict:
|
||
account = self.catalog.account_for(account_id)
|
||
mode = self.gate_mode(account_id)
|
||
store = BrowserCookieStore(account.cookie_file, account.storage_state_file)
|
||
static_valid = _vault_cookie_valid(account, store)
|
||
status = _read_keepalive_status(account)
|
||
if static_valid and self._keepalive_success_is_fresh(status):
|
||
self._write_state(account, STATE_HEALTHY, probe="fresh-keepalive")
|
||
return GateVerdict(
|
||
account_id=account_id,
|
||
mode=mode,
|
||
blocked=False,
|
||
message=f"账号 {account_id} 登录态有效（keepalive 近期验证通过）",
|
||
)
|
||
if self._acceptance_enabled():
|
||
# 验收模式不开浏览器：按静态证据判定，且不投递飞书通知。
|
||
if static_valid and not _keepalive_requires_relogin(status):
|
||
return GateVerdict(
|
||
account_id=account_id,
|
||
mode=mode,
|
||
blocked=False,
|
||
message=f"账号 {account_id} 静态 cookie 校验通过（验收模式）",
|
||
)
|
||
return self._expire(
|
||
account,
|
||
mode,
|
||
context,
|
||
detail="验收模式下静态 cookie 校验未通过",
|
||
notify=False,
|
||
)
|
||
|
||
outcome = self.keepalive.refresh_account(account_id)
|
||
probe_status = str(outcome.get("status", "error"))
|
||
if probe_status == "success":
|
||
recovered = bool(outcome.get("login_recovered"))
|
||
self._write_state(
|
||
account,
|
||
STATE_HEALTHY,
|
||
probe="success",
|
||
recovered=recovered,
|
||
)
|
||
message = (
|
||
f"账号 {account_id} 登录态有效（自动账密重登成功）"
|
||
if recovered
|
||
else f"账号 {account_id} 登录态有效（探活通过）"
|
||
)
|
||
return GateVerdict(
|
||
account_id=account_id,
|
||
mode=mode,
|
||
blocked=False,
|
||
message=message,
|
||
recovered=recovered,
|
||
)
|
||
if probe_status == "busy":
|
||
if static_valid and not _keepalive_requires_relogin(status):
|
||
return GateVerdict(
|
||
account_id=account_id,
|
||
mode=mode,
|
||
blocked=False,
|
||
message=f"账号 {account_id} 探活锁忙，静态 cookie 校验通过",
|
||
)
|
||
return self._expire(
|
||
account,
|
||
mode,
|
||
context,
|
||
detail="探活锁被占用且静态 cookie 校验未通过",
|
||
)
|
||
if probe_status in {"expired", "not_logged_in"}:
|
||
evidence = _recovery_evidence(_read_keepalive_status(account))
|
||
detail = "cookie 已过期，无法正常采集"
|
||
if probe_status == "not_logged_in" and evidence:
|
||
detail = f"登录态缺失；{evidence}"
|
||
elif evidence:
|
||
detail = f"cookie 已过期；{evidence}"
|
||
return self._expire(account, mode, context, detail=detail)
|
||
# timeout / error / http_error / disabled：探测结果不可信时回退静态证据。
|
||
if static_valid and not _keepalive_requires_relogin(status):
|
||
return GateVerdict(
|
||
account_id=account_id,
|
||
mode=mode,
|
||
blocked=False,
|
||
message=f"账号 {account_id} 探活异常（{probe_status}），静态 cookie 校验通过",
|
||
)
|
||
return self._expire(
|
||
account,
|
||
mode,
|
||
context,
|
||
detail=f"探活异常（{probe_status}）且静态 cookie 校验未通过",
|
||
)
|
||
|
||
# ------------------------------------------------------------------
|
||
# Expiry, notification, persisted state
|
||
# ------------------------------------------------------------------
|
||
|
||
def _expire(
|
||
self,
|
||
account: Any,
|
||
mode: str,
|
||
context: RunContext,
|
||
*,
|
||
detail: str,
|
||
notify: bool | None = None,
|
||
) -> GateVerdict:
|
||
account_id = str(account.account_id)
|
||
if notify is None:
|
||
notify = not context.shadow and not self._acceptance_enabled()
|
||
state = self._read_state(account)
|
||
notified = False
|
||
notification_note = ""
|
||
if notify and not self._within_notify_interval(state):
|
||
recipients = self._alert_recipients()
|
||
message = _expiry_message(account_id, mode, context, detail)
|
||
for open_id, profile in recipients:
|
||
try:
|
||
self._sender(
|
||
user_id=open_id,
|
||
text=message,
|
||
idempotency_key=f"cookie-gate:{account_id}:{context.run_id}",
|
||
profile=profile,
|
||
)
|
||
notified = True
|
||
except Exception as exc: # noqa: BLE001 - alerting must not mask the verdict
|
||
notification_note = f"{type(exc).__name__}: {exc}"[:300]
|
||
if recipients and notified:
|
||
state["last_notified_at"] = _now_iso(self._now)
|
||
state.pop("notification_error", None)
|
||
elif notification_note:
|
||
state["notification_error"] = notification_note
|
||
state["state"] = STATE_AWAITING_LOGIN
|
||
state["updated_at"] = _now_iso(self._now)
|
||
state["last_probe_status"] = "expired"
|
||
state["last_blocked_run"] = context.run_id
|
||
if not context.shadow and not self._acceptance_enabled():
|
||
# 记录待自动重跑清单：登录态恢复后由调度器维护循环按原业务日期重跑。
|
||
state["pending_reruns"] = _record_pending_rerun(
|
||
state.get("pending_reruns"), context, _now_iso(self._now)
|
||
)
|
||
self._save_state(account, state)
|
||
message = _expiry_message(account_id, mode, context, detail)
|
||
if notified:
|
||
message += " 已通过飞书通知登录负责人。"
|
||
elif notification_note:
|
||
message += f" 飞书通知发送失败（{notification_note}）。"
|
||
return GateVerdict(
|
||
account_id=account_id,
|
||
mode=mode,
|
||
blocked=True,
|
||
message=message,
|
||
notified=notified,
|
||
)
|
||
|
||
def _write_state(
|
||
self,
|
||
account: Any,
|
||
state_name: str,
|
||
*,
|
||
probe: str,
|
||
recovered: bool = False,
|
||
) -> None:
|
||
state = self._read_state(account)
|
||
if (
|
||
state_name == STATE_HEALTHY
|
||
and state.get("state") == STATE_HEALTHY
|
||
and state.get("last_probe_status") in {"fresh-keepalive", "success"}
|
||
and not recovered
|
||
):
|
||
return # 持续健康时避免每次运行都重写状态文件
|
||
state["state"] = state_name
|
||
state["updated_at"] = _now_iso(self._now)
|
||
state["last_probe_status"] = probe
|
||
if state_name == STATE_HEALTHY:
|
||
state["last_healthy_at"] = _now_iso(self._now)
|
||
if recovered:
|
||
state["last_recovered_at"] = _now_iso(self._now)
|
||
state.pop("notification_error", None)
|
||
self._save_state(account, state)
|
||
|
||
def _state_path(self, account: Any) -> Path:
|
||
return account.cookie_file.with_name(_GATE_STATE_FILENAME)
|
||
|
||
def _read_state(self, account: Any) -> dict[str, object]:
|
||
path = self._state_path(account)
|
||
if not path.exists():
|
||
return {"account_id": str(account.account_id)}
|
||
try:
|
||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError):
|
||
return {"account_id": str(account.account_id)}
|
||
if not isinstance(payload, dict):
|
||
return {"account_id": str(account.account_id)}
|
||
return dict(payload)
|
||
|
||
def _save_state(self, account: Any, state: Mapping[str, object]) -> None:
|
||
payload = {
|
||
"account_id": str(account.account_id),
|
||
**state,
|
||
}
|
||
try:
|
||
atomic_write_json(self._state_path(account), payload)
|
||
except OSError: # pragma: no cover - state file is advisory only
|
||
pass
|
||
|
||
def _within_notify_interval(self, state: Mapping[str, object]) -> bool:
|
||
raw = state.get("last_notified_at")
|
||
if not isinstance(raw, str):
|
||
return False
|
||
try:
|
||
last = datetime.fromisoformat(raw)
|
||
except ValueError:
|
||
return False
|
||
now = self._now()
|
||
if last.tzinfo is None:
|
||
last = last.replace(tzinfo=timezone.utc)
|
||
interval = _env_int(
|
||
self._environment,
|
||
GYXX_LOGIN_ALERT_NOTIFY_INTERVAL_SECONDS,
|
||
_DEFAULT_NOTIFY_INTERVAL_SECONDS,
|
||
minimum=0,
|
||
)
|
||
return (now - last).total_seconds() < interval
|
||
|
||
def _keepalive_success_is_fresh(self, status: Mapping[str, object] | None) -> bool:
|
||
if not isinstance(status, Mapping) or status.get("status") != "success":
|
||
return False
|
||
raw = status.get("last_success_at")
|
||
if not isinstance(raw, str):
|
||
return False
|
||
try:
|
||
last = datetime.fromisoformat(raw)
|
||
except ValueError:
|
||
return False
|
||
now = self._now()
|
||
if last.tzinfo is None:
|
||
last = last.replace(tzinfo=timezone.utc)
|
||
max_age = _env_float(
|
||
self._environment,
|
||
GYXX_COOKIE_GATE_MAX_STATUS_AGE_SECONDS,
|
||
_DEFAULT_MAX_STATUS_AGE_SECONDS,
|
||
minimum=_MIN_MAX_STATUS_AGE_SECONDS,
|
||
)
|
||
return (now - last).total_seconds() <= max_age
|
||
|
||
def _alert_recipients(self) -> list[tuple[str, str]]:
|
||
profile = (
|
||
self._environment.get(GYXX_LOGIN_ALERT_APP_PROFILE, "").strip()
|
||
or ANALYZER_NOTIFICATION_APP_PROFILE
|
||
)
|
||
override = [
|
||
value.strip()
|
||
for value in self._environment.get(GYXX_LOGIN_ALERT_OPEN_IDS, "").split(",")
|
||
if value.strip()
|
||
]
|
||
if override:
|
||
return [(open_id, profile) for open_id in override]
|
||
try:
|
||
payload, _revision = NotificationRoutingStore(self.settings).snapshot()
|
||
for person in payload["people"].values():
|
||
if person.get("mobile") == DEFAULT_LOGIN_ALERT_MOBILE:
|
||
binding = person.get("bindings", {}).get(profile)
|
||
if binding:
|
||
return [(str(binding["open_id"]), profile)]
|
||
except Exception: # noqa: BLE001 - fall back to the constant recipient
|
||
pass
|
||
return [(ANALYZER_ACCEPTANCE_RECIPIENT_OPEN_ID, profile)]
|
||
|
||
def _acceptance_enabled(self) -> bool:
|
||
value = self._environment.get("GYXX_WORKFLOW_ACCEPTANCE", "")
|
||
return bool(value and value.strip().casefold() in {"1", "true", "yes", "on"})
|
||
|
||
|
||
def gate_enabled(environment: Mapping[str, str] | None = None) -> bool:
|
||
values = os.environ if environment is None else environment
|
||
return values.get(GYXX_COOKIE_GATE, "").strip().casefold() not in _DISABLED_VALUES
|
||
|
||
|
||
def apply_cookie_gates(
|
||
workflow: WorkflowDefinition,
|
||
*,
|
||
settings: Settings,
|
||
environment: Mapping[str, str] | None = None,
|
||
gate: CookieGate | None = None,
|
||
) -> WorkflowDefinition:
|
||
"""Inject one critical gate step per gated account in front of its users.
|
||
|
||
Gate steps are graph sources; each step that uses an account gains the
|
||
account's gate as an extra dependency. A failed gate therefore skips its
|
||
dependents with ``dependency-failed`` through the regular engine, while
|
||
unrelated branches keep running. Operator slices (``--rerun-step`` /
|
||
``--resume-from``) are produced before this injection and stay gate-free.
|
||
"""
|
||
|
||
if not workflow.steps:
|
||
return workflow
|
||
if gate is None:
|
||
try:
|
||
gate = CookieGate(
|
||
settings,
|
||
environment=os.environ if environment is None else environment,
|
||
)
|
||
except Exception: # noqa: BLE001 - 无集成配置时不门控；步骤执行仍会显式报错
|
||
return workflow
|
||
account_steps = gate.collect_workflow_accounts(workflow)
|
||
if not account_steps:
|
||
return workflow
|
||
existing_ids = {step.step_id for step in workflow.steps}
|
||
gate_steps: list[StepDefinition] = []
|
||
for account_id in sorted(account_steps):
|
||
step_id = f"{_GATE_STEP_PREFIX}.{account_id}"
|
||
if step_id in existing_ids:
|
||
continue
|
||
gate_steps.append(
|
||
StepDefinition(
|
||
step_id,
|
||
CookieGateStepAction(gate, account_id),
|
||
timeout_seconds=_GATE_TIMEOUT_SECONDS,
|
||
max_attempts=1,
|
||
)
|
||
)
|
||
if not gate_steps:
|
||
return workflow
|
||
patched: list[StepDefinition] = []
|
||
for step in workflow.steps:
|
||
dependencies = list(dict.fromkeys(step.depends_on))
|
||
changed = False
|
||
for account_id, step_ids in account_steps.items():
|
||
gate_id = f"{_GATE_STEP_PREFIX}.{account_id}"
|
||
if step.step_id in step_ids and gate_id not in dependencies:
|
||
dependencies.append(gate_id)
|
||
changed = True
|
||
if changed:
|
||
step = replace(step, depends_on=tuple(dependencies))
|
||
patched.append(step)
|
||
return WorkflowDefinition(
|
||
workflow.workflow_id,
|
||
(*gate_steps, *patched),
|
||
)
|
||
|
||
|
||
class CookieGateRecoveryManager:
|
||
"""Re-run workflows blocked by expired cookies once the login recovers.
|
||
|
||
Runs inside the resident scheduler's maintenance loop. For every account
|
||
with a non-empty ``pending_reruns`` list, it re-verifies the login state
|
||
(fresh keepalive success first, then a throttled live probe); on recovery
|
||
it relaunches each recorded workflow with its original business date
|
||
through the scheduler's own launcher, clears the pending list, and sends a
|
||
Feishu confirmation to the login owner.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
gate: CookieGate,
|
||
*,
|
||
launcher: Any,
|
||
probe_interval_seconds: float = _RECOVERY_PROBE_INTERVAL_SECONDS,
|
||
launch_cooldown_seconds: float = _RECOVERY_LAUNCH_COOLDOWN_SECONDS,
|
||
) -> None:
|
||
self.gate = gate
|
||
self._launcher = launcher
|
||
self._probe_interval = probe_interval_seconds
|
||
self._launch_cooldown = launch_cooldown_seconds
|
||
|
||
def tick(self) -> list[dict[str, object]]:
|
||
"""Attempt recovery reruns once; returns redacted outcomes per account."""
|
||
|
||
outcomes: list[dict[str, object]] = []
|
||
for account in self.gate.catalog.accounts:
|
||
state = self.gate._read_state(account)
|
||
pending = state.get("pending_reruns")
|
||
if not isinstance(pending, list) or not pending:
|
||
continue
|
||
now = self.gate._now()
|
||
if not self._launch_cooldown_passed(state, now):
|
||
continue
|
||
if not self._account_recovered(account, state, now):
|
||
continue
|
||
launched, remaining = self._launch_pending(account, pending, now)
|
||
state["pending_reruns"] = remaining
|
||
state["updated_at"] = _now_iso(self.gate._now)
|
||
state["last_recovered_at"] = state["updated_at"]
|
||
if not remaining:
|
||
state["state"] = STATE_HEALTHY
|
||
state["last_probe_status"] = "recovered-rerun"
|
||
if launched:
|
||
state["last_rerun_launch_at"] = state["updated_at"]
|
||
history = [
|
||
item
|
||
for item in state.get("rerun_history", [])
|
||
if isinstance(item, dict)
|
||
]
|
||
history.extend(launched)
|
||
state["rerun_history"] = history[-_RECOVERY_HISTORY_LIMIT:]
|
||
self.gate._save_state(account, state)
|
||
if launched:
|
||
self._notify_recovered(str(account.account_id), launched)
|
||
outcomes.append(
|
||
{
|
||
"account_id": str(account.account_id),
|
||
"launched": launched,
|
||
"pending": len(remaining),
|
||
}
|
||
)
|
||
return outcomes
|
||
|
||
# ------------------------------------------------------------------
|
||
# internals
|
||
# ------------------------------------------------------------------
|
||
|
||
def _launch_cooldown_passed(
|
||
self, state: Mapping[str, object], now: datetime
|
||
) -> bool:
|
||
raw = state.get("last_rerun_launch_at")
|
||
if not isinstance(raw, str):
|
||
return True
|
||
try:
|
||
last = datetime.fromisoformat(raw)
|
||
except ValueError:
|
||
return True
|
||
if last.tzinfo is None:
|
||
last = last.replace(tzinfo=timezone.utc)
|
||
return (now - last).total_seconds() >= self._launch_cooldown
|
||
|
||
def _account_recovered(
|
||
self,
|
||
account: Any,
|
||
state: dict[str, object],
|
||
now: datetime,
|
||
) -> bool:
|
||
"""Fresh keepalive success first; otherwise one throttled live probe."""
|
||
|
||
store = BrowserCookieStore(account.cookie_file, account.storage_state_file)
|
||
if _vault_cookie_valid(account, store) and self.gate._keepalive_success_is_fresh(
|
||
_read_keepalive_status(account)
|
||
):
|
||
return True
|
||
raw = state.get("last_recovery_probe_at")
|
||
if isinstance(raw, str):
|
||
try:
|
||
last_probe = datetime.fromisoformat(raw)
|
||
except ValueError:
|
||
last_probe = None
|
||
if last_probe is not None:
|
||
if last_probe.tzinfo is None:
|
||
last_probe = last_probe.replace(tzinfo=timezone.utc)
|
||
if (now - last_probe).total_seconds() < self._probe_interval:
|
||
return False
|
||
state["last_recovery_probe_at"] = _now_iso(self.gate._now)
|
||
self.gate._save_state(account, state)
|
||
try:
|
||
outcome = self.gate.keepalive.refresh_account(str(account.account_id))
|
||
except Exception: # noqa: BLE001 - 恢复探活失败留给下一轮
|
||
return False
|
||
return str(outcome.get("status", "error")) == "success"
|
||
|
||
def _launch_pending(
|
||
self,
|
||
account: Any,
|
||
pending: list[object],
|
||
now: datetime,
|
||
) -> tuple[list[dict[str, object]], list[dict[str, object]]]:
|
||
known_ids = {
|
||
entry.workflow_id for entry in self.gate.workflow_catalog().workflows
|
||
}
|
||
launched: list[dict[str, object]] = []
|
||
remaining: list[dict[str, object]] = []
|
||
for item in pending:
|
||
if not isinstance(item, dict):
|
||
continue # 控制台脚本等临时运行 id：不在 catalog 中，直接丢弃
|
||
workflow_id = item.get("workflow_id")
|
||
business_date = item.get("business_date")
|
||
if not isinstance(workflow_id, str) or not isinstance(business_date, str):
|
||
continue
|
||
if workflow_id not in known_ids:
|
||
continue
|
||
try:
|
||
process = self._launcher.launch(
|
||
workflow_id, now, date.fromisoformat(business_date)
|
||
)
|
||
except Exception as exc: # noqa: BLE001 - 保留 pending，下一轮重试
|
||
remaining.append(
|
||
{
|
||
"workflow_id": workflow_id,
|
||
"business_date": business_date,
|
||
"error": f"{type(exc).__name__}: {exc}"[:200],
|
||
}
|
||
)
|
||
continue
|
||
close = getattr(self._launcher, "close", None)
|
||
if callable(close):
|
||
try:
|
||
close(process)
|
||
except Exception: # noqa: BLE001 - 关闭日志句柄失败不影响重跑
|
||
pass
|
||
launched.append(
|
||
{
|
||
"workflow_id": workflow_id,
|
||
"business_date": business_date,
|
||
"pid": int(getattr(process, "pid", 0) or 0),
|
||
"launched_at": _now_iso(self.gate._now),
|
||
}
|
||
)
|
||
return launched, remaining
|
||
|
||
def _notify_recovered(
|
||
self,
|
||
account_id: str,
|
||
launched: list[dict[str, object]],
|
||
) -> None:
|
||
lines = [
|
||
f"- {item['workflow_id']}（业务日期 {item['business_date']}）"
|
||
for item in launched
|
||
]
|
||
message = (
|
||
"【GYXX 登录态已恢复】账号 "
|
||
f"{account_id} 登录态已恢复，已自动重跑被阻断的工作流：\n"
|
||
+ "\n".join(lines)
|
||
+ "\n如本次重跑仍有失败，请查看运行日志或工作流控制台。"
|
||
)
|
||
key = "-".join(
|
||
[account_id, *(str(item["workflow_id"]) for item in launched)]
|
||
)
|
||
for open_id, profile in self.gate._alert_recipients():
|
||
try:
|
||
self.gate._sender(
|
||
user_id=open_id,
|
||
text=message,
|
||
idempotency_key=f"cookie-recovery:{key}",
|
||
profile=profile,
|
||
)
|
||
except Exception: # noqa: BLE001 - 确认通知失败不影响已发起的重跑
|
||
continue
|
||
|
||
|
||
def _record_pending_rerun(
|
||
current: object,
|
||
context: RunContext,
|
||
blocked_at: str,
|
||
) -> list[dict[str, object]]:
|
||
"""Append one blocked run to the pending list, deduped and bounded."""
|
||
|
||
pending: list[dict[str, object]] = [
|
||
dict(item)
|
||
for item in (current if isinstance(current, list) else [])
|
||
if isinstance(item, dict)
|
||
]
|
||
entry = {
|
||
"workflow_id": context.workflow_id,
|
||
"business_date": context.business_date.isoformat(),
|
||
"blocked_run_id": context.run_id,
|
||
"blocked_at": blocked_at,
|
||
}
|
||
pending = [
|
||
item
|
||
for item in pending
|
||
if not (
|
||
item.get("workflow_id") == entry["workflow_id"]
|
||
and item.get("business_date") == entry["business_date"]
|
||
)
|
||
]
|
||
pending.append(entry)
|
||
return pending[-_MAX_PENDING_RERUNS:]
|
||
|
||
|
||
def _recovery_evidence(status: Mapping[str, object] | None) -> str:
|
||
if not isinstance(status, Mapping):
|
||
return ""
|
||
if not status.get("login_attempted"):
|
||
return ""
|
||
reason = status.get("login_reason")
|
||
error_type = status.get("login_error_type")
|
||
exit_code = status.get("login_exit_code")
|
||
parts = ["自动账密重登已尝试但未成功"]
|
||
if isinstance(reason, str) and reason:
|
||
parts.append(reason)
|
||
if isinstance(error_type, str) and error_type:
|
||
parts.append(f"错误={error_type}")
|
||
if isinstance(exit_code, int) and not isinstance(exit_code, bool):
|
||
parts.append(f"退出码={exit_code}")
|
||
return "（" + "，".join(parts) + "）"
|
||
|
||
|
||
def _expiry_message(
|
||
account_id: str,
|
||
mode: str,
|
||
context: RunContext,
|
||
detail: str,
|
||
) -> str:
|
||
mode_note = (
|
||
"账号密码自动重登失败，需人工处理（如验证码）"
|
||
if mode == GATE_MODE_RECOVER
|
||
else "该平台仅支持扫码/验证码登录，无法自动恢复"
|
||
)
|
||
return (
|
||
"【GYXX 登录态失效】工作流被阻断，需要人工重新登录\n"
|
||
f"工作流: {context.workflow_id}（业务日期 {context.business_date.isoformat()}）\n"
|
||
f"账号: {account_id}\n"
|
||
f"原因: {detail}；{mode_note}\n"
|
||
f"请在服务器上执行: uv run gyxx accounts login {account_id}\n"
|
||
"（将打开浏览器窗口，完成扫码或验证码登录后自动保存并同步到全部成员脚本；"
|
||
"受影响的采集步骤本次已跳过，登录完成后下次运行自动恢复）"
|
||
)
|
||
|
||
|
||
def _now_iso(now: Callable[[], datetime]) -> str:
|
||
return now().astimezone(timezone.utc).isoformat(timespec="seconds")
|
||
|
||
|
||
def _env_int(
|
||
environment: Mapping[str, str],
|
||
name: str,
|
||
default: int,
|
||
*,
|
||
minimum: int,
|
||
) -> int:
|
||
raw = environment.get(name, "").strip()
|
||
try:
|
||
value = int(raw)
|
||
except (TypeError, ValueError):
|
||
return default
|
||
return max(minimum, value)
|
||
|
||
|
||
def _env_float(
|
||
environment: Mapping[str, str],
|
||
name: str,
|
||
default: float,
|
||
*,
|
||
minimum: float,
|
||
) -> float:
|
||
raw = environment.get(name, "").strip()
|
||
try:
|
||
value = float(raw)
|
||
except (TypeError, ValueError):
|
||
return default
|
||
return max(minimum, value)
|
||
|
||
|
||
__all__ = [
|
||
"DEFAULT_LOGIN_ALERT_MOBILE",
|
||
"GATE_MODE_BLOCKING",
|
||
"GATE_MODE_DEFER",
|
||
"GATE_MODE_RECOVER",
|
||
"GYXX_COOKIE_GATE",
|
||
"GYXX_LOGIN_ALERT_OPEN_IDS",
|
||
"CookieGate",
|
||
"CookieGateRecoveryManager",
|
||
"CookieGateStepAction",
|
||
"GateVerdict",
|
||
"apply_cookie_gates",
|
||
"gate_enabled",
|
||
]
|