from __future__ import annotations import json from datetime import date, datetime, timezone from pathlib import Path from typing import Any import pytest from gyxx_flow.ops.audit import ( SCHEMA_SQL, AuditStore, split_step_key, summarize_cookies, ) from gyxx_flow.ops.index import RunIndex DOUYIN_COOKIES: list[dict[str, Any]] = [ { "name": "sessionid", "value": "AAA", "domain": ".jinritemai.com", "expires": 1766000000.0, }, { "name": "sid_tt", "value": "BBB", "domain": ".jinritemai.com", "expires": 1766100000.0, }, { "name": "unrelated", "value": "CCC", "domain": ".example.com", "expires": 1700000000.0, }, { "name": "session", "value": "DDD", "domain": ".jinritemai.com", "expires": -1, }, ] class _FakeCursor: def __init__(self, log: list[Any], rows: list[dict[str, Any]]) -> None: self._log = log self._rows = rows def __enter__(self) -> "_FakeCursor": return self def __exit__(self, *_: object) -> bool: return False def execute(self, sql: str, params: Any = None) -> None: self._log.append(("execute", sql, params)) def executemany(self, sql: str, rows: Any) -> None: self._log.append(("executemany", sql, list(rows))) def fetchall(self) -> list[dict[str, Any]]: return self._rows def fetchone(self) -> dict[str, Any] | None: # 默认模拟「ops_audit_schema 尚不存在」,从而走首次建表分支。 return None def close(self) -> None: return None class _FakeConnection: def __init__(self, log: list[Any], rows: list[dict[str, Any]] | None = None) -> None: self._log = log self._rows = rows or [] def cursor(self) -> _FakeCursor: return _FakeCursor(self._log, self._rows) def commit(self) -> None: self._log.append(("commit",)) def close(self) -> None: self._log.append(("close",)) def _store_with_fake( tmp_path: Path, *, environment: dict[str, str] | None = None, rows: list[dict[str, Any]] | None = None, ) -> tuple[AuditStore, list[Any]]: log: list[Any] = [] store = AuditStore( tmp_path, environment=environment or {"GYXX_POSTGRES_DSN": "postgresql://fake/audit"}, connect=lambda: _FakeConnection(log, rows), ) return store, log # ---------------------------------------------------------------------- # cookie 快照 # ---------------------------------------------------------------------- def test_summarize_cookies_keeps_only_required_cookies() -> None: snapshot = summarize_cookies( DOUYIN_COOKIES, required_domains=("jinritemai.com",), required_names=("sessionid", "sid_tt", "session"), ) assert snapshot.cookie_count == 4 assert snapshot.required_count == 3 # 最早的必需 cookie 过期时间:1766000000,而不是无关 cookie 的 1700000000 assert snapshot.declared_expires_at == datetime.fromtimestamp( 1766000000.0, tz=timezone.utc ) def test_summarize_cookies_ignores_session_cookies_for_expiry() -> None: snapshot = summarize_cookies( DOUYIN_COOKIES, required_domains=("jinritemai.com",), required_names=("session",), ) assert snapshot.required_count == 1 assert snapshot.declared_expires_at is None def test_summarize_cookies_fingerprint_tracks_value_changes_only() -> None: baseline = summarize_cookies(DOUYIN_COOKIES, required_domains=("jinritemai.com",)) reordered = summarize_cookies( list(reversed(DOUYIN_COOKIES)), required_domains=("jinritemai.com",) ) replaced = [dict(cookie) for cookie in DOUYIN_COOKIES] replaced[0]["value"] = "ROTATED" assert baseline.fingerprint # 非空 assert baseline.fingerprint == reordered.fingerprint assert baseline.fingerprint != summarize_cookies( replaced, required_domains=("jinritemai.com",) ).fingerprint def test_summarize_cookies_without_requirements_counts_everything() -> None: snapshot = summarize_cookies(DOUYIN_COOKIES) assert snapshot.required_count == 4 assert snapshot.declared_expires_at == datetime.fromtimestamp( 1700000000.0, tz=timezone.utc ) def test_summarize_cookies_has_no_plaintext_value_in_fingerprint() -> None: snapshot = summarize_cookies(DOUYIN_COOKIES, required_domains=("jinritemai.com",)) assert "AAA" not in snapshot.fingerprint assert len(snapshot.fingerprint) == 32 def test_split_step_key_handles_retry_suffix() -> None: assert split_step_key("sync.attempt-2") == ("sync", 2) assert split_step_key("collect_collaborators") == ("collect_collaborators", 1) assert split_step_key("a.attempt-x") == ("a.attempt-x", 1) # ---------------------------------------------------------------------- # 写入装配 # ---------------------------------------------------------------------- def test_record_run_upserts_run_and_steps(tmp_path: Path) -> None: store, log = _store_with_fake(tmp_path) ok = store.record_run( { "workflow_id": "content.metrics.daily", "run_id": "content.metrics.daily__20261018__abc", "business_date": "2026-10-18", "status": "success", "mode": "dry_run", "shadow": False, "started_at": "2026-10-19T01:41:43+00:00", "ended_at": "2026-10-19T01:41:48+00:00", "error": None, "journal_path": "runs/content.metrics.daily/run.json", }, { "sync.attempt-1": { "status": "success", "exit_code": 0, "started_at": "2026-10-19T01:41:44+00:00", "ended_at": "2026-10-19T01:41:46+00:00", }, "collect.attempt-1": {"status": "failed", "exit_code": 4, "error": "boom"}, }, ) assert ok is True statements = [entry for entry in log if entry[0] == "execute"] assert any("CREATE TABLE IF NOT EXISTS ops_workflow_runs" in item[1] for item in statements) insert = next(item for item in statements if "INSERT INTO ops_workflow_runs" in item[1]) params = insert[2] assert params["business_date"] == date(2026, 10, 18) assert params["duration_seconds"] == 5 assert params["step_total"] == 2 assert params["step_success"] == 1 assert params["step_failed"] == 1 batch = next(item for item in log if item[0] == "executemany") rows = {item["step_id"]: item for item in batch[2]} assert rows["sync"]["attempt"] == 1 assert rows["sync"]["duration_seconds"] == 2 assert rows["collect"]["exit_code"] == 4 assert rows["collect"]["error"] == "boom" def test_record_run_rejects_incomplete_payload(tmp_path: Path) -> None: store, log = _store_with_fake(tmp_path) assert store.record_run({"workflow_id": "x"}) is False assert not [entry for entry in log if entry[0] == "execute"] def test_record_cookie_event_writes_value_free_snapshot(tmp_path: Path) -> None: store, log = _store_with_fake(tmp_path) snapshot = summarize_cookies(DOUYIN_COOKIES, required_domains=("jinritemai.com",)) ok = store.record_cookie_event( account_id="douyin-shop", state="healthy", snapshot=snapshot, probe_status="success", probe_source="keepalive", workflow_id="content.metrics.daily", run_id="run-1", ) assert ok is True insert = next( item for item in log if item[0] == "execute" and "INSERT INTO ops_account_cookie_events" in item[1] ) params = insert[2] assert params["account_id"] == "douyin-shop" assert params["state"] == "healthy" assert params["fingerprint"] == snapshot.fingerprint assert params["declared_expires_at"] == snapshot.declared_expires_at # 明文值不得出现在任何参数里 assert "AAA" not in json.dumps(params, default=str) def test_record_cookie_event_rejects_unknown_state(tmp_path: Path) -> None: store, _ = _store_with_fake(tmp_path) with pytest.raises(ValueError): store.record_cookie_event(account_id="a", state="definitely-not-valid") # ---------------------------------------------------------------------- # 降级与熔断 # ---------------------------------------------------------------------- def test_missing_dsn_never_raises_and_logs_failure(tmp_path: Path) -> None: store = AuditStore(tmp_path, environment={}) assert store.record_run( { "workflow_id": "content.metrics.daily", "run_id": "r1", "business_date": "2026-10-18", "status": "success", } ) is False assert "审计数据库配置缺失" in store.last_error failure_log = tmp_path / "logs" / "ops-audit" / "audit-failures.log" assert failure_log.exists() assert json.loads(failure_log.read_text(encoding="utf-8").splitlines()[0])["operation"] == ( "record_run" ) def test_audit_can_be_disabled_without_touching_the_database(tmp_path: Path) -> None: store, log = _store_with_fake( tmp_path, environment={"GYXX_POSTGRES_DSN": "postgresql://fake", "GYXX_OPS_AUDIT": "off"} ) assert store.enabled is False assert store.record_cookie_event(account_id="a", state="healthy") is False assert log == [] def test_consecutive_failures_open_the_circuit(tmp_path: Path) -> None: attempts = 0 def _boom() -> Any: nonlocal attempts attempts += 1 raise OSError("database unreachable") store = AuditStore( tmp_path, environment={"GYXX_POSTGRES_DSN": "postgresql://fake", "GYXX_OPS_AUDIT_FAILURE_COOLDOWN": "60"}, connect=_boom, ) payload = { "workflow_id": "w", "run_id": "r", "business_date": "2026-10-18", "status": "success", } for _ in range(3): store.record_run(payload) assert attempts == 3 # 熔断生效后不再尝试连接,避免每个工作流都卡在 TCP 超时上 assert store.record_run(payload) is False assert attempts == 3 # ---------------------------------------------------------------------- # 与运行索引的集成 # ---------------------------------------------------------------------- def test_run_index_still_works_when_audit_is_unavailable(tmp_path: Path) -> None: from gyxx_flow.core.context import RunContext from gyxx_flow.core.layout import DataLayout from gyxx_flow.core.records import RunJournal layout = DataLayout(tmp_path) context = RunContext.create("content.metrics.daily", "2026-10-18") journal = RunJournal.create(layout, context, mode="dry_run") journal.finalize("success") record = RunIndex(tmp_path).index_journal(journal) assert record.workflow_id == "content.metrics.daily" assert record.status == "success" # 运行索引文件仍然写出,审计失败不影响主链路 assert (tmp_path / "state" / "ops" / "run-index" / f"{record.run_id}.json").exists() def test_schema_sql_defines_both_audit_surfaces() -> None: assert "ops_workflow_runs" in SCHEMA_SQL assert "ops_workflow_run_steps" in SCHEMA_SQL assert "ops_account_cookie_events" in SCHEMA_SQL assert "ops_account_cookie_lifetimes" in SCHEMA_SQL assert "ops_account_cookie_stats" in SCHEMA_SQL