From 78ea4641cedc96fad64110f84035d63e96c06385 Mon Sep 17 00:00:00 2001 From: wangyunlong <3080433063@qq.com> Date: Fri, 9 Oct 2026 13:45:07 +0800 Subject: [PATCH] =?UTF-8?q?refactor(login):=20=E7=A7=BB=E9=99=A4=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E5=86=85=E6=89=AB=E7=A0=81=E9=87=8D=E7=99=BB=EF=BC=8C?= =?UTF-8?q?=E7=99=BB=E5=BD=95=E7=BB=9F=E4=B8=80=E4=BA=A4=E7=BB=99=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=20vault=20=E4=B8=8E=20cookie-gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 删除 relogin_*/friday_relogin_parallel 子进程及其测试；批量补采、评论采集、 星图/蒲公英/B站/抖音自营与抖店竞品脚本不再内嵌扫码等待，登录态失效统一由 cookie-gate 阻断并走 gyxx accounts login 恢复；runtime-bindings 与模块注册 同步移除 content.relogin.weekly。 --- config/runtime-bindings.json | 113 +-- .../modules/content_marketing/__init__.py | 5 +- .../bilibili_comment_scraper.py | 57 +- .../data/tools/batch_rescrape_bilibili.py | 47 +- .../data/tools/batch_rescrape_douyin.py | 49 +- .../data/tools/batch_rescrape_xiaohongshu.py | 45 +- .../data/tools/friday_relogin_parallel.py | 666 ------------------ .../data/tools/relogin_bilibili.py | 113 --- .../data/tools/relogin_douyin.py | 121 ---- .../data/tools/relogin_pgy.py | 148 ---- .../data/tools/relogin_runtime_paths.py | 279 -------- .../data/tools/relogin_transaction.py | 47 -- .../data/tools/relogin_xiaohongshu.py | 116 --- .../data/tools/relogin_xingtu.py | 350 --------- .../douyin_comment_scraper.py | 68 +- .../content_marketing/pgy_xhs_scraper_v2.py | 115 +-- .../self_bilibili_scraper.py | 61 +- .../content_marketing/self_douyin_scraper.py | 57 +- .../xiaohongshu_comment_scraper.py | 68 +- .../content_marketing/xingtu_scraper_v2.py | 291 +------- .../dy_store_competitor_store_scraping.py | 35 +- .../collectors/dy_negative_reviews.py | 27 +- .../dy_store_competitor_store_scraping.py | 35 +- src/gyxx_flow/notification_routing.py | 1 - .../test_comment_batch_recovery.py | 17 +- .../test_friday_relogin_parallel.py | 569 --------------- .../test_relogin_rollback.py | 494 ------------- .../test_relogin_runtime_bindings.py | 410 ----------- 28 files changed, 135 insertions(+), 4269 deletions(-) delete mode 100644 src/gyxx_flow/modules/content_marketing/data/tools/friday_relogin_parallel.py delete mode 100644 src/gyxx_flow/modules/content_marketing/data/tools/relogin_bilibili.py delete mode 100644 src/gyxx_flow/modules/content_marketing/data/tools/relogin_douyin.py delete mode 100644 src/gyxx_flow/modules/content_marketing/data/tools/relogin_pgy.py delete mode 100644 src/gyxx_flow/modules/content_marketing/data/tools/relogin_runtime_paths.py delete mode 100644 src/gyxx_flow/modules/content_marketing/data/tools/relogin_transaction.py delete mode 100644 src/gyxx_flow/modules/content_marketing/data/tools/relogin_xiaohongshu.py delete mode 100644 src/gyxx_flow/modules/content_marketing/data/tools/relogin_xingtu.py delete mode 100644 tests/modules/content_marketing/test_friday_relogin_parallel.py delete mode 100644 tests/modules/content_marketing/test_relogin_rollback.py delete mode 100644 tests/modules/content_marketing/test_relogin_runtime_bindings.py diff --git a/config/runtime-bindings.json b/config/runtime-bindings.json index 953b555..50bcbdc 100644 --- a/config/runtime-bindings.json +++ b/config/runtime-bindings.json @@ -312,8 +312,6 @@ }, "members": [ "content.comments.collect_xiaohongshu", - "content_marketing:data/tools/relogin_pgy.py", - "content_marketing:data/tools/relogin_xiaohongshu.py", "content_marketing:pgy_xhs_scraper_v2.py" ] }, @@ -340,7 +338,6 @@ }, "members": [ "content.comments.collect_douyin", - "content_marketing:data/tools/relogin_douyin.py", "content_marketing:self_douyin_scraper.py" ] }, @@ -368,12 +365,15 @@ "real_chrome": true }, "members": [ - "content_marketing:data/tools/relogin_xingtu.py" ] }, "chanmama": { "cdp_port": 22207, "login_mode": "D", + "credential_env_names": [ + "CHANMAMA_ACCOUNT", + "CHANMAMA_PASSWORD" + ], "required_cookie_domains": [ "chanmama.com" ], @@ -411,8 +411,7 @@ }, "members": [ "content.comments.collect_bilibili", - "content.metrics.collect_self_bilibili", - "content_marketing:data/tools/relogin_bilibili.py" + "content.metrics.collect_self_bilibili" ] } }, @@ -582,23 +581,6 @@ ], "cdp_port": 22016 }, - "content_marketing:data/tools/friday_relogin.bat": { - "script_id": "content_marketing:data/tools/friday_relogin.bat", - "state_key": "content_marketing:data/tools/friday_relogin.bat", - "aliases": [ - "content_marketing:data/tools/friday_relogin.bat" - ], - "cdp_port": 22017 - }, - "content.login.refresh": { - "script_id": "content_marketing:data/tools/friday_relogin_parallel.py", - "state_key": "content_marketing:data/tools/friday_relogin_parallel.py", - "aliases": [ - "content_marketing:data/tools/friday_relogin_parallel.py" - ], - "cdp_port": 22018, - "login_mode": "D" - }, "content.creator_report.generate": { "script_id": "content_marketing:data/tools/generate_creator_report.py", "state_key": "content_marketing:data/tools/generate_creator_report.py", @@ -679,83 +661,6 @@ ], "cdp_port": 22135 }, - "content_marketing:data/tools/relogin_bilibili.py": { - "script_id": "content_marketing:data/tools/relogin_bilibili.py", - "state_key": "content_marketing:data/tools/relogin_bilibili.py", - "aliases": [ - "content_marketing:data/tools/relogin_bilibili.py" - ], - "cdp_port": 22028, - "login_mode": "D", - "required_cookie_domains": [ - "bilibili.com" - ], - "required_cookie_names": [ - "SESSDATA" - ] - }, - "content_marketing:data/tools/relogin_douyin.py": { - "script_id": "content_marketing:data/tools/relogin_douyin.py", - "state_key": "content_marketing:data/tools/relogin_douyin.py", - "aliases": [ - "content_marketing:data/tools/relogin_douyin.py" - ], - "cdp_port": 22029, - "login_mode": "D", - "required_cookie_domains": [ - "douyin.com" - ], - "required_cookie_names": [ - "sessionid" - ] - }, - "content_marketing:data/tools/relogin_pgy.py": { - "script_id": "content_marketing:data/tools/relogin_pgy.py", - "state_key": "content_marketing:data/tools/relogin_pgy.py", - "aliases": [ - "content_marketing:data/tools/relogin_pgy.py" - ], - "cdp_port": 22030, - "login_mode": "D", - "required_cookie_domains": [ - "xiaohongshu.com" - ] - }, - "content_marketing:data/tools/relogin_xiaohongshu.py": { - "script_id": "content_marketing:data/tools/relogin_xiaohongshu.py", - "state_key": "content_marketing:data/tools/relogin_xiaohongshu.py", - "aliases": [ - "content_marketing:data/tools/relogin_xiaohongshu.py" - ], - "cdp_port": 22031, - "login_mode": "D", - "required_cookie_domains": [ - "xiaohongshu.com" - ], - "required_cookie_names": [ - "web_session" - ] - }, - "content_marketing:data/tools/relogin_xingtu.py": { - "script_id": "content_marketing:data/tools/relogin_xingtu.py", - "state_key": "content_marketing:data/tools/relogin_xingtu.py", - "aliases": [ - "content_marketing:data/tools/relogin_xingtu.py" - ], - "cdp_port": 22032, - "login_mode": "D", - "required_cookie_domains": [ - "douyin.com", - "oceanengine.com", - "xingtu.cn" - ], - "required_cookie_names": [ - "sessionid", - "sessionid_ss", - "sid_tt", - "sid_guard" - ] - }, "content.failed.retry": { "script_id": "content_marketing:data/tools/retry_failed.py", "state_key": "content_marketing:data/tools/retry_failed.py", @@ -1441,6 +1346,14 @@ "erp321.com" ] }, + "product.hk_anpei_public_inventory.refresh": { + "script_id": "product_commerce:public_sku_inventory_sheet.py", + "state_key": "product_commerce:public_sku_inventory_sheet.py", + "aliases": [ + "product_commerce:public_sku_inventory_sheet.py" + ], + "cdp_port": 22166 + }, "product_commerce:scripts/insert_jd_main_image_records.py": { "script_id": "product_commerce:scripts/insert_jd_main_image_records.py", "state_key": "product_commerce:scripts/insert_jd_main_image_records.py", diff --git a/src/gyxx_flow/modules/content_marketing/__init__.py b/src/gyxx_flow/modules/content_marketing/__init__.py index be07d0a..b474705 100644 --- a/src/gyxx_flow/modules/content_marketing/__init__.py +++ b/src/gyxx_flow/modules/content_marketing/__init__.py @@ -1,4 +1,4 @@ -"""Content metrics, creator, comment, report, and login workflows.""" +"""Content metrics, creator, comment, and report workflows.""" from __future__ import annotations @@ -14,7 +14,6 @@ CONTENT_SCHEDULED_WORKFLOW_IDS = ( "content.metrics.self_operated.daily", "content.metrics.backfill", "content.marketing_report.daily", - "content.relogin.weekly", "content.creator_report.monthly", "content.summary.monthly", "content.notes_master.daily", @@ -28,7 +27,6 @@ CONTENT_COMMENT_TIMEOUT_SECONDS = 6 * 60 * 60 # weakening the module-wide write lock. CONTENT_NOTES_MASTER_LOCK_TIMEOUT_SECONDS = 6 * 60 * 60 CONTENT_RESOURCE = "module:content_marketing" -_RELOGIN_WORKFLOW_ID = "content.relogin.weekly" _PARALLEL_COMMENT_WORKFLOW_ID = "content.comments.weekly" # Keep both long-running daily metrics graphs on workflow-scoped resources so # their collection steps do not block unrelated content workflows. @@ -94,7 +92,6 @@ class ContentMarketingModule: ), resource=workflow_resource, command_factory=command_factory, - official_notification=workflow_id == _RELOGIN_WORKFLOW_ID, independent_step_resources=( workflow_id in { _PARALLEL_COMMENT_WORKFLOW_ID, diff --git a/src/gyxx_flow/modules/content_marketing/bilibili_comment_scraper.py b/src/gyxx_flow/modules/content_marketing/bilibili_comment_scraper.py index 9e413ca..139694b 100644 --- a/src/gyxx_flow/modules/content_marketing/bilibili_comment_scraper.py +++ b/src/gyxx_flow/modules/content_marketing/bilibili_comment_scraper.py @@ -77,23 +77,15 @@ def is_logged_in(page: Page) -> bool: def maybe_wait_for_login(page: Page, login_timeout: int) -> None: - """检测登录态,未登录就等用户扫码。B 站登录页 URL 含 passport.bilibili.com/login。""" + """检测登录态；未登录立即降级为游客态，不再等待人工扫码。 + + 登录统一由账号 vault 管理（gyxx accounts login bilibili-content）， + 由工作流执行前的 cookie 网关保证登录态。 + """ if is_logged_in(page): return - log("B 站未登录 (无 SESSDATA cookie)。请在弹出的浏览器窗口扫码登录。") - # 跳到登录页,确保用户能看到二维码 - try: - page.goto("https://passport.bilibili.com/login", wait_until="domcontentloaded", timeout=30000) - except Exception as exc: - log(f" 跳登录页失败 (继续等当前页扫码): {exc}") - deadline = time.time() + login_timeout - while time.time() < deadline: - if is_logged_in(page): - log("登录成功,SESSDATA 已写入") - save_state(page) - return - page.wait_for_timeout(2000) - log(f" [WARN] 等待 {login_timeout}s 仍未登录,继续以游客态采集 (评论只会拿到热门 3-5 条)") + log(" [WARN] B 站未登录 (无 SESSDATA cookie)，降级为游客态采集 (评论只会拿到热门 3-5 条)。" + "如需全量评论请执行 gyxx accounts login bilibili-content") def safe_filename(value: str) -> str: @@ -437,48 +429,17 @@ def write_outputs(url: str, comments: list[dict[str, Any]], result: dict[str, An return json_path, csv_path -def login_only(login_timeout: int) -> int: - """只触发登录(headed),登录完成立刻退出。给 relogin_bilibili.py 用。""" - def action(page: Page) -> None: - page.goto("https://passport.bilibili.com/login", wait_until="domcontentloaded", timeout=30000) - page.wait_for_timeout(1500) - maybe_wait_for_login(page, login_timeout) - - PROFILE_DIR.mkdir(parents=True, exist_ok=True) - with DynamicSession( - headless=False, - real_chrome=True, - user_data_dir=str(PROFILE_DIR), - locale="zh-CN", - timezone_id="Asia/Shanghai", - timeout=90000, - network_idle=False, - disable_resources=False, - google_search=False, - page_setup=restore_cookies, - max_pages=1, - ) as session: - session.fetch("https://passport.bilibili.com/login", page_action=action, wait=1000) - log("[login-only] 完成") - return 0 - - def main() -> int: parser = argparse.ArgumentParser(description="Scrape Bilibili video comments with a real browser and API.") - parser.add_argument("url", help="Bilibili video URL (传 dummy 'login' 配合 --login-only 也可)") + parser.add_argument("url", help="Bilibili video URL") parser.add_argument("--max-reply-pages", type=int, default=200, help="max sub-reply pages per top-level comment") parser.add_argument("--max-pages", type=int, default=200, help="最多抓取一级评论页数") parser.add_argument("--delay", type=float, default=0.25, help="接口请求间隔秒数") parser.add_argument("--headless", action="store_true", help="无头模式") parser.add_argument("--login-timeout", type=int, default=0, - help="未登录时等待扫码秒数(>0 启用登录检测;默认 0 沿用游客态)") - parser.add_argument("--login-only", action="store_true", - help="只触发登录(headed)然后退出,给 relogin_bilibili 用") + help="保留参数（登录等待已移除；未登录将直接以游客态采集）") args = parser.parse_args() - if args.login_only: - return login_only(args.login_timeout if args.login_timeout > 0 else 300) - comments, result = scrape_comments( args.url, args.max_pages, args.delay, args.headless, args.max_reply_pages, args.login_timeout, diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_bilibili.py b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_bilibili.py index fd299bf..7756ead 100644 --- a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_bilibili.py +++ b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_bilibili.py @@ -18,7 +18,6 @@ from gyxx_flow.modules.content_marketing.runtime_paths import PATHS _TOOLS_DIR = PATHS.tools_root _PROJECT_ROOT = PATHS.module_root -RELOGIN_SCRIPT = _TOOLS_DIR / "relogin_bilibili.py" try: sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined] @@ -81,39 +80,6 @@ def comment_replace_policy(comments: list[dict], stats: dict, metrics: dict) -> return False, False, "空评论且无法确认平台为 0 评论" -def do_relogin() -> bool: - """Run relogin_bilibili.py (headed, waits for QR scan). Returns True if successful.""" - acceptance_policy = current_acceptance_policy() - if acceptance_policy.enabled: - acceptance_policy.record( - "cookie_skipped", - operation="content.batch_rescrape_bilibili.relogin", - details={ - "status": "SKIPPED_COOKIE", - "reason": "cookie is invalid and QR relogin is disabled for acceptance", - }, - ) - log( - " [SKIPPED_COOKIE] Cookie invalid; acceptance mode will not " - "launch relogin or wait for a QR scan" - ) - return False - log(" Cookie 可能过期,触发自动重登 (请扫码)...") - try: - rc = subprocess.call( - [sys.executable, str(RELOGIN_SCRIPT)], - cwd=str(_PROJECT_ROOT), - ) - if rc == 0: - log(" 重登成功,继续采集") - return True - else: - log(f" 重登失败 (exit={rc})") - return False - except Exception as exc: - log(f" 重登脚本执行异常: {exc}") - return False - def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: rows = collect_urls() @@ -126,8 +92,6 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: total_comments = 0 consecutive_fail = 0 MAX_CONSECUTIVE_FAIL = 3 - relogin_used = 0 - MAX_RELOGINS = 1 t0 = time.time() for i, row in enumerate(rows, 1): note_id = row["id"] @@ -152,11 +116,12 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: break if comments is None: consecutive_fail += 1 - if consecutive_fail >= MAX_CONSECUTIVE_FAIL and relogin_used < MAX_RELOGINS: - if do_relogin(): - relogin_used += 1 - consecutive_fail = 0 - continue + if consecutive_fail >= MAX_CONSECUTIVE_FAIL: + # 登录统一由账号 vault 管理；连续失败只提示，不再触发扫码重登。 + log( + " 连续失败达到上限；若为登录态失效请执行 " + "uv run gyxx accounts login bilibili-content" + ) if current_acceptance_policy().enabled: return COOKIE_SKIP_EXIT_CODE continue diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_douyin.py b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_douyin.py index ed0bb15..357c0b6 100644 --- a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_douyin.py +++ b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_douyin.py @@ -26,7 +26,6 @@ from gyxx_flow.modules.content_marketing.serialized_page_action import ( _TOOLS_DIR = PATHS.tools_root _PROJECT_ROOT = PATHS.module_root -RELOGIN_SCRIPT = _TOOLS_DIR / "relogin_douyin.py" try: sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined] @@ -104,38 +103,6 @@ def result_logged_in(result: dict | None) -> bool: return bool(nested_stats.get("logged_in")) if isinstance(nested_stats, dict) else False -def do_relogin() -> bool: - """Run relogin_douyin.py (headed, waits for QR scan). Returns True if successful.""" - acceptance_policy = current_acceptance_policy() - if acceptance_policy.enabled: - acceptance_policy.record( - "cookie_skipped", - operation="content.batch_rescrape_douyin.relogin", - details={ - "status": "SKIPPED_COOKIE", - "reason": "cookie is invalid and QR relogin is disabled for acceptance", - }, - ) - log( - " [SKIPPED_COOKIE] Cookie invalid; acceptance mode will not " - "launch relogin or wait for a QR scan" - ) - return False - log(" Cookie 可能过期,触发自动重登 (请扫码)...") - try: - rc = subprocess.call( - [sys.executable, str(RELOGIN_SCRIPT)], - cwd=str(_PROJECT_ROOT), - ) - if rc == 0: - log(" 重登成功,继续采集") - return True - log(f" 重登失败 (exit={rc})") - return False - except Exception as exc: - log(f" 重登脚本执行异常: {exc}") - return False - def scrape_with_fresh_browser( url: str, @@ -203,8 +170,6 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: COOLDOWN_SECONDS = 300 cooldown_used = 0 MAX_COOLDOWNS = 3 - relogin_used = 0 - MAX_RELOGINS = 1 # 每轮 batch 最多自动重登1次 t0 = time.time() for i, row in enumerate(rows, 1): note_id = row["id"] @@ -214,16 +179,10 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: if comments is None: consecutive_fail += 1 - # 连续失败 + 可能 cookie 过期 → 尝试自动重登 - if consecutive_fail >= MAX_CONSECUTIVE_FAIL and relogin_used < MAX_RELOGINS: - if do_relogin(): - relogin_used += 1 - consecutive_fail = 0 - comments, stats, scrape_error = scrape_with_fresh_browser(url) - else: - # 重登失败,走冷却逻辑；验收模式必须直接跳过，不能等待。 - if current_acceptance_policy().enabled: - return COOKIE_SKIP_EXIT_CODE + # 登录统一由账号 vault 管理；不再触发扫码重登，验收模式直接跳过。 + if consecutive_fail >= MAX_CONSECUTIVE_FAIL: + if current_acceptance_policy().enabled: + return COOKIE_SKIP_EXIT_CODE if comments is None and consecutive_fail >= MAX_CONSECUTIVE_FAIL: if cooldown_used < MAX_COOLDOWNS: diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_xiaohongshu.py b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_xiaohongshu.py index 79ca84c..9891fe4 100644 --- a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_xiaohongshu.py +++ b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_xiaohongshu.py @@ -25,7 +25,6 @@ from gyxx_flow.modules.content_marketing.serialized_page_action import ( _TOOLS_DIR = PATHS.tools_root _PROJECT_ROOT = PATHS.module_root -RELOGIN_SCRIPT = _TOOLS_DIR / "relogin_xiaohongshu.py" try: sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined] @@ -87,38 +86,6 @@ def comment_replace_policy(comments: list[dict], stats: dict, metrics: dict) -> return False, False, "空评论且无法确认平台为 0 评论" -def do_relogin() -> bool: - """Run relogin_xiaohongshu.py (headed, waits for QR scan). Returns True if successful.""" - acceptance_policy = current_acceptance_policy() - if acceptance_policy.enabled: - acceptance_policy.record( - "cookie_skipped", - operation="content.batch_rescrape_xiaohongshu.relogin", - details={ - "status": "SKIPPED_COOKIE", - "reason": "cookie is invalid and QR relogin is disabled for acceptance", - }, - ) - log( - " [SKIPPED_COOKIE] Cookie invalid; acceptance mode will not " - "launch relogin or wait for a QR scan" - ) - return False - log(" Cookie 可能过期,触发自动重登 (请扫码)...") - try: - rc = subprocess.call( - [sys.executable, str(RELOGIN_SCRIPT)], - cwd=str(_PROJECT_ROOT), - ) - if rc == 0: - log(" 重登成功,继续采集") - return True - log(f" 重登失败 (exit={rc})") - return False - except Exception as exc: - log(f" 重登脚本执行异常: {exc}") - return False - def scrape_with_fresh_browser( url: str, @@ -173,8 +140,6 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: total_comments = 0 consecutive_fail = 0 MAX_CONSECUTIVE_FAIL = 3 - relogin_used = 0 - MAX_RELOGINS = 1 t0 = time.time() for i, row in enumerate(rows, 1): note_id = row["id"] @@ -183,13 +148,13 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: comments, result, scrape_error = scrape_with_fresh_browser(url) if comments is None: consecutive_fail += 1 - if consecutive_fail >= MAX_CONSECUTIVE_FAIL and relogin_used < MAX_RELOGINS: - if do_relogin(): - relogin_used += 1 - consecutive_fail = 0 - comments, result, scrape_error = scrape_with_fresh_browser(url) + if consecutive_fail >= MAX_CONSECUTIVE_FAIL: if current_acceptance_policy().enabled: return COOKIE_SKIP_EXIT_CODE + log( + " 连续失败达到上限；若为登录态失效请执行 " + "uv run gyxx accounts login xiaohongshu-content" + ) if comments is None: log(f" ERROR: {scrape_error}") fail += 1 diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/friday_relogin_parallel.py b/src/gyxx_flow/modules/content_marketing/data/tools/friday_relogin_parallel.py deleted file mode 100644 index a6b72ec..0000000 --- a/src/gyxx_flow/modules/content_marketing/data/tools/friday_relogin_parallel.py +++ /dev/null @@ -1,666 +0,0 @@ -"""Launch five QR relogins in parallel, notify via analyzer Lark, retry 3 times.""" - -from __future__ import annotations - -import argparse -import ctypes -import datetime as dt -import os -import subprocess -import sys -import time -from ctypes import wintypes -from dataclasses import dataclass -from pathlib import Path -from typing import Callable - -import psutil - -from gyxx_flow.adapters import ( - ANALYZER_NOTIFICATION_APP_PROFILE, - COOKIE_SKIP_EXIT_CODE, - ResolvedNotificationRoute, - binding_from_environment, - current_acceptance_policy, - environment_for_child_script, - resolve_notification_route, - send_lark_bot_message, -) -from gyxx_flow.modules.content_marketing.runtime_paths import PATHS - -PROJECT_DIR = PATHS.module_root -ANALYZER_DIR = PATHS.tmp_root / "relogin" -WORKFLOW_ID = "content.relogin.weekly" -MARKETING_RECIPIENT = "ou_24cc944d6e43c69c59d6560ad4e2ae6e" -TECHNICAL_RECIPIENT = "ou_7ad5fc8012e2f741afc5346e05ffd447" - - -@dataclass(frozen=True) -class Platform: - label: str - command: tuple[str, ...] - required_cookie: str - - -@dataclass(frozen=True, slots=True) -class NotificationDeliveryFailure: - """One failed recipient delivery that must affect the workflow outcome.""" - - stage: str - platform: str - recipient: str - detail: str - - -PLATFORMS = { - "douyin": Platform( - "\u6296\u97f3", ("data/tools/relogin_douyin.py",), - "sessionid", - ), - "bilibili": Platform( - "B\u7ad9", ("data/tools/relogin_bilibili.py",), - "SESSDATA", - ), - "pgy": Platform( - "\u84b2\u516c\u82f1", ("data/tools/relogin_pgy.py",), - "", - ), - "xingtu": Platform( - "\u661f\u56fe", ("data/tools/relogin_xingtu.py", "--force"), - "sessionid", - ), - "xiaohongshu": Platform( - "\u5c0f\u7ea2\u4e66", ("data/tools/relogin_xiaohongshu.py",), - "web_session", - ), -} - -DEFAULT_RECIPIENTS = { - "pgy": MARKETING_RECIPIENT, - "xingtu": MARKETING_RECIPIENT, - "douyin": TECHNICAL_RECIPIENT, - "bilibili": TECHNICAL_RECIPIENT, - "xiaohongshu": TECHNICAL_RECIPIENT, -} - - -def resolve_platform_names(requested: list[str] | None) -> list[str]: - """Resolve an optional manual platform subset while preserving order.""" - if not requested: - return list(PLATFORMS) - names: list[str] = [] - for name in requested: - if name not in PLATFORMS: - raise ValueError(f"unknown platform: {name}") - if name not in names: - names.append(name) - return names - - -def recipient_for_platform(platform_name: str, override: str | None = None) -> str: - """Return the responsible recipient, unless a manual all-platform override is set.""" - from gyxx_flow.adapters.acceptance_policy import resolve_notification_recipients - - if override: - recipient = override - else: - try: - recipient = DEFAULT_RECIPIENTS[platform_name] - except KeyError as exc: - raise ValueError( - f"no Lark recipient configured for platform: {platform_name}" - ) from exc - return resolve_notification_recipients( - (recipient,), - app_profile=ANALYZER_NOTIFICATION_APP_PROFILE, - )[0] - - -def resolve_weekly_notification_route( - override: str | None = None, -) -> ResolvedNotificationRoute: - """Resolve the scheduled workflow route while retaining legacy defaults.""" - defaults = ( - (override,) - if override - else tuple(dict.fromkeys(DEFAULT_RECIPIENTS.values())) - ) - return resolve_notification_route(WORKFLOW_ID, defaults) - - -def recipients_for_platform( - platform_name: str, - route: ResolvedNotificationRoute, - override: str | None = None, -) -> tuple[str, ...]: - """Return dynamic recipients, or the original per-platform recipient.""" - if not route.enabled: - return () - if route.configured: - return route.open_ids - return (recipient_for_platform(platform_name, override),) - - -def group_platforms_by_recipient( - platform_names: list[str], override: str | None = None, -) -> dict[str, list[str]]: - grouped: dict[str, list[str]] = {} - for name in platform_names: - grouped.setdefault(recipient_for_platform(name, override), []).append(name) - return grouped - - -def group_platforms_by_notification_recipient( - platform_names: list[str], - route: ResolvedNotificationRoute, - override: str | None = None, -) -> dict[str, list[str]]: - """Group failed platforms for legacy or configured multi-recipient delivery.""" - if not route.enabled: - return {} - if route.configured: - return {recipient: list(platform_names) for recipient in route.open_ids} - return group_platforms_by_recipient(platform_names, override) - - -def run_with_retries( - platform_names: list[str], - *, - max_attempts: int, - run_round: Callable[[list[str], int], dict[str, bool]], -) -> dict[str, bool]: - status = {name: False for name in platform_names} - pending = list(platform_names) - for attempt in range(1, max_attempts + 1): - if not pending: - break - round_result = run_round(pending, attempt) - for name in pending: - status[name] = bool(round_result.get(name)) - pending = [name for name in pending if not status[name]] - return status - - -def _valid_cookie(platform: Platform, cookie_file: Path) -> bool: - if not cookie_file.is_file(): - return False - if not platform.required_cookie: - return cookie_file.stat().st_size > 10 - try: - import json - values = json.loads(cookie_file.read_text(encoding="utf-8")) - return any( - item.get("name") == platform.required_cookie and item.get("value") - for item in values if isinstance(item, dict) - ) - except Exception: - return False - - -def _descendant_pids(pid: int) -> set[int]: - try: - return {child.pid for child in psutil.Process(pid).children(recursive=True)} - except (psutil.Error, OSError): - return set() - - -def _terminate_process_tree(process: subprocess.Popen) -> None: - """Terminate the whole Windows subprocess tree, including Chrome/Scrapling.""" - try: - subprocess.run( - ["taskkill", "/PID", str(process.pid), "/T", "/F"], - capture_output=True, - text=True, - check=False, - ) - except OSError: - process.terminate() - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - process.kill() - - -def _window_for_process(pid: int) -> int | None: - pids = _descendant_pids(pid) | {pid} - handles: list[int] = [] - user32 = ctypes.windll.user32 - callback_type = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p) - - def callback(hwnd, _lparam): - process_id = ctypes.c_ulong() - user32.GetWindowThreadProcessId(hwnd, ctypes.byref(process_id)) - if process_id.value in pids and user32.IsWindowVisible(hwnd): - handles.append(int(hwnd)) - return True - - user32.EnumWindows(callback_type(callback), 0) - return handles[0] if handles else None - - -def _platform_from_window_title(title: str) -> str | None: - """Identify a login window without relying on Chrome's process tree.""" - if "\u84b2\u516c\u82f1" in title: - return "pgy" - if "\u6296\u97f3\u6388\u6743" in title or "\u5de8\u91cf\u661f\u56fe" in title: - return "xingtu" - if "\u6296\u97f3" in title: - return "douyin" - if "bilibili" in title.lower() or "\u8d26\u53f7\u767b\u5f55" in title: - return "bilibili" - if "\u5c0f\u7ea2\u4e66" in title: - return "xiaohongshu" - return None - - -def _login_windows() -> dict[str, int]: - """Map visible Chrome login windows to platforms by their page titles.""" - user32 = ctypes.windll.user32 - found: dict[str, int] = {} - callback_type = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p) - - def callback(hwnd, _lparam): - if not user32.IsWindowVisible(hwnd): - return True - length = user32.GetWindowTextLengthW(hwnd) - if length <= 0: - return True - buffer = ctypes.create_unicode_buffer(length + 1) - user32.GetWindowTextW(hwnd, buffer, length + 1) - platform = _platform_from_window_title(buffer.value) - if platform: - found[platform] = int(hwnd) - return True - - user32.EnumWindows(callback_type(callback), 0) - return found - - -def _capture_window(hwnd: int, output: Path) -> None: - """Maximize and render one window directly, even when it is not foreground.""" - from PIL import Image - - user32 = ctypes.windll.user32 - gdi32 = ctypes.windll.gdi32 - user32.ShowWindow(hwnd, 3) # SW_MAXIMIZE - time.sleep(1) - rect = wintypes.RECT() - user32.GetWindowRect(hwnd, ctypes.byref(rect)) - width, height = rect.right - rect.left, rect.bottom - rect.top - if width <= 0 or height <= 0: - raise RuntimeError(f"invalid window size: {width}x{height}") - - window_dc = user32.GetWindowDC(hwnd) - memory_dc = gdi32.CreateCompatibleDC(window_dc) - bitmap = gdi32.CreateCompatibleBitmap(window_dc, width, height) - previous = gdi32.SelectObject(memory_dc, bitmap) - try: - if not user32.PrintWindow(hwnd, memory_dc, 2): # PW_RENDERFULLCONTENT - raise RuntimeError("PrintWindow failed") - - class BitmapInfoHeader(ctypes.Structure): - _fields_ = [ - ("biSize", wintypes.DWORD), ("biWidth", wintypes.LONG), - ("biHeight", wintypes.LONG), ("biPlanes", wintypes.WORD), - ("biBitCount", wintypes.WORD), ("biCompression", wintypes.DWORD), - ("biSizeImage", wintypes.DWORD), - ("biXPelsPerMeter", wintypes.LONG), - ("biYPelsPerMeter", wintypes.LONG), - ("biClrUsed", wintypes.DWORD), - ("biClrImportant", wintypes.DWORD), - ] - - byte_count = width * height * 4 - pixels = ctypes.create_string_buffer(byte_count) - header = BitmapInfoHeader( - ctypes.sizeof(BitmapInfoHeader), width, -height, 1, 32, 0, - byte_count, 0, 0, 0, 0, - ) - rows = gdi32.GetDIBits( - memory_dc, bitmap, 0, height, pixels, ctypes.byref(header), 0 - ) - if rows != height: - raise RuntimeError(f"GetDIBits returned {rows}/{height} rows") - image = Image.frombuffer( - "RGB", (width, height), pixels, "raw", "BGRX", 0, 1 - ).copy() - finally: - gdi32.SelectObject(memory_dc, previous) - gdi32.DeleteObject(bitmap) - gdi32.DeleteDC(memory_dc) - user32.ReleaseDC(hwnd, window_dc) - - output.parent.mkdir(parents=True, exist_ok=True) - image.save(output, "PNG") - - -def _tile_windows(handles: list[int]) -> None: - """Place four login windows in a 2x2 grid so screenshots cannot overlap.""" - user32 = ctypes.windll.user32 - screen_w = user32.GetSystemMetrics(0) - screen_h = user32.GetSystemMetrics(1) - cell_w = max(640, screen_w // 2) - cell_h = max(450, screen_h // 2) - for index, hwnd in enumerate(handles): - x = (index % 2) * cell_w - y = (index // 2) * cell_h - user32.ShowWindow(hwnd, 9) # SW_RESTORE - user32.MoveWindow(hwnd, x, y, cell_w, cell_h, True) - - -def _send_lark( - recipient: str, - text: str | None = None, - image: Path | None = None, - *, - app_profile: str = ANALYZER_NOTIFICATION_APP_PROFILE, -) -> tuple[dict[str, object], ...]: - """Send text and/or image through the analyzer Hermes bot.""" - if app_profile != ANALYZER_NOTIFICATION_APP_PROFILE: - raise ValueError( - "relogin notifications must use the hermes-analyzer lark-cli profile" - ) - lark_env = { - key: value for key, value in os.environ.items() - if not key.upper().startswith("HERMES_") - } - receipts: list[dict[str, object]] = [] - if text: - ANALYZER_DIR.mkdir(parents=True, exist_ok=True) - receipts.append(send_lark_bot_message( - user_id=recipient, - text=text, - profile=ANALYZER_NOTIFICATION_APP_PROFILE, - cwd=ANALYZER_DIR, - env=lark_env, - )) - if image: - receipts.append(send_lark_bot_message( - user_id=recipient, - image=image, - profile=ANALYZER_NOTIFICATION_APP_PROFILE, - cwd=image.parent, - env=lark_env, - )) - return tuple(receipts) - - -def _send_qr_notification( - route: ResolvedNotificationRoute, - platform_name: str, - *, - override: str | None, - text: str, - image: Path, -) -> tuple[NotificationDeliveryFailure, ...]: - """Attempt every QR delivery and return all recipient-specific failures.""" - profile = route.app_profile or ANALYZER_NOTIFICATION_APP_PROFILE - failures: list[NotificationDeliveryFailure] = [] - for recipient in recipients_for_platform(platform_name, route, override): - for stage, kwargs in ( - ("qr_text", {"text": text}), - ("qr_image", {"image": image}), - ): - try: - _send_lark(recipient, app_profile=profile, **kwargs) - except Exception as exc: - failures.append( - NotificationDeliveryFailure( - stage=stage, - platform=platform_name, - recipient=recipient, - detail=str(exc) or type(exc).__name__, - ) - ) - return tuple(failures) - - -def _unavailable_qr_delivery_failures( - route: ResolvedNotificationRoute, - platform_name: str, - *, - override: str | None, - detail: str, -) -> tuple[NotificationDeliveryFailure, ...]: - """Record both intended QR deliveries when no screenshot can be sent.""" - return tuple( - NotificationDeliveryFailure( - stage=stage, - platform=platform_name, - recipient=recipient, - detail=detail, - ) - for recipient in recipients_for_platform(platform_name, route, override) - for stage in ("qr_text", "qr_image") - ) - - -def _send_failure_summaries( - route: ResolvedNotificationRoute, - failed_platforms: list[str], - *, - override: str | None, - max_attempts: int, -) -> tuple[NotificationDeliveryFailure, ...]: - """Attempt every final failure summary and return all delivery failures.""" - groups = group_platforms_by_notification_recipient( - failed_platforms, - route, - override, - ) - profile = route.app_profile or ANALYZER_NOTIFICATION_APP_PROFILE - failures: list[NotificationDeliveryFailure] = [] - for recipient, names in groups.items(): - labels = "\u3001".join(PLATFORMS[name].label for name in names) - try: - _send_lark( - recipient, - text=( - f"\u4ee5\u4e0b\u5e73\u53f0\u8fde\u7eed {max_attempts} \u6b21\u672a\u626b\u7801\uff0c" - "\u5df2\u6062\u590d\u5e76\u7ee7\u4f7f\u7528\u65e7 Cookie\uff1a" - f"{labels}" - ), - app_profile=profile, - ) - except Exception as exc: - failures.append( - NotificationDeliveryFailure( - stage="failure_summary", - platform=",".join(names), - recipient=recipient, - detail=str(exc) or type(exc).__name__, - ) - ) - return tuple(failures) - - -def _run_round_factory( - args, - notification_route: ResolvedNotificationRoute | None, - notification_failures: list[NotificationDeliveryFailure], -): - python = sys.executable - screenshot_root = PATHS.tmp_root / "qrcode/friday_relogin" - - def run_round(names: list[str], attempt: int) -> dict[str, bool]: - started: dict[str, subprocess.Popen] = {} - cookie_files: dict[str, Path] = {} - missing_qr_windows: set[str] = set() - stamp = dt.datetime.now().strftime("%Y%m%d_%H%M%S") - for name in names: - platform = PLATFORMS[name] - cmd = [python, *platform.command, "--login-timeout", str(args.round_timeout)] - target = PROJECT_DIR / platform.command[0] - environment = environment_for_child_script(target, os.environ) - cookie_files[name] = binding_from_environment(environment).cookie_file - started[name] = subprocess.Popen( - cmd, - cwd=PROJECT_DIR, - env=environment, - ) - - time.sleep(args.screenshot_delay) - title_windows = _login_windows() - windows = { - name: title_windows.get(name) or _window_for_process(process.pid) - for name, process in started.items() - } - _tile_windows([hwnd for hwnd in windows.values() if hwnd]) - time.sleep(2) - for name, process in started.items(): - platform = PLATFORMS[name] - hwnd = windows[name] - if not hwnd: - print(f"[{name}] no visible browser window for QR screenshot", flush=True) - missing_qr_windows.add(name) - continue - output = screenshot_root / f"{stamp}_{name}_round{attempt}.png" - try: - _capture_window(hwnd, output) - print(f"[{name}] QR screenshot: {output}", flush=True) - except Exception as exc: - print(f"[{name}] screenshot failed: {exc}", flush=True) - if not args.no_send and notification_route is not None: - notification_failures.extend( - _unavailable_qr_delivery_failures( - notification_route, - name, - override=args.recipient, - detail=( - "QR screenshot capture failed: " - f"{str(exc) or type(exc).__name__}" - ), - ) - ) - continue - if not args.no_send and notification_route is not None: - notification_failures.extend( - _send_qr_notification( - notification_route, - name, - override=args.recipient, - text=( - f"{platform.label} \u767b\u5f55\u4e8c\u7ef4\u7801" - f"\uff08\u7b2c {attempt}/{args.max_attempts} \u6b21\uff09\uff0c" - "\u8bf7\u5728 3 \u5206\u949f\u5185\u626b\u7801\u3002" - ), - image=output, - ) - ) - - # Put the still-open login windows back into a compact layout after - # taking full-size screenshots one at a time. - _tile_windows([hwnd for hwnd in windows.values() if hwnd]) - - results: dict[str, bool] = {} - for name, process in started.items(): - try: - rc = process.wait(timeout=args.round_timeout + 90) - except subprocess.TimeoutExpired: - _terminate_process_tree(process) - rc = 1 - results[name] = rc == 0 and _valid_cookie( - PLATFORMS[name], - cookie_files[name], - ) - if ( - not results[name] - and name in missing_qr_windows - and not args.no_send - and notification_route is not None - ): - notification_failures.extend( - _unavailable_qr_delivery_failures( - notification_route, - name, - override=args.recipient, - detail=( - "no visible QR window and relogin did not succeed" - ), - ) - ) - print(f"[{name}] round={attempt} exit={rc} success={results[name]}", flush=True) - return results - - return run_round - - -def main() -> int: - acceptance_policy = current_acceptance_policy() - if acceptance_policy.enabled: - acceptance_policy.record( - "cookie_skipped", - operation="content.relogin.weekly", - details={ - "status": "SKIPPED_COOKIE", - "reason": "acceptance runs never launch QR relogin workflows", - }, - ) - print( - "[SKIPPED_COOKIE] Acceptance mode does not launch login windows, " - "reset cookies/profiles, or wait for QR scans.", - flush=True, - ) - return COOKIE_SKIP_EXIT_CODE - - parser = argparse.ArgumentParser(description="Friday five-platform parallel QR relogin") - parser.add_argument( - "--recipient", default=None, - help="Override the configured recipients and send every platform to one open_id", - ) - parser.add_argument("--max-attempts", type=int, default=3) - parser.add_argument("--round-timeout", type=int, default=300) - parser.add_argument("--screenshot-delay", type=int, default=60) - parser.add_argument("--no-send", action="store_true", help="Do not send Lark messages") - parser.add_argument( - "--platform", action="append", choices=sorted(PLATFORMS), - help="Only relogin the selected platform; may be repeated", - ) - args = parser.parse_args() - - platform_names = resolve_platform_names(args.platform) - notification_route = ( - None if args.no_send else resolve_weekly_notification_route(args.recipient) - ) - notification_failures: list[NotificationDeliveryFailure] = [] - status = run_with_retries( - platform_names, max_attempts=args.max_attempts, - run_round=_run_round_factory( - args, - notification_route, - notification_failures, - ), - ) - failed = [name for name, ok in status.items() if not ok] - if failed and not args.no_send and notification_route is not None: - notification_failures.extend( - _send_failure_summaries( - notification_route, - failed, - override=args.recipient, - max_attempts=args.max_attempts, - ) - ) - print(f"Final relogin status: {status}", flush=True) - if notification_failures: - print( - f"Final notification status: failed ({len(notification_failures)} deliveries)", - flush=True, - ) - for failure in notification_failures: - print( - "[NOTIFICATION_ERROR] " - f"stage={failure.stage} platform={failure.platform} " - f"recipient={failure.recipient} error={failure.detail}", - flush=True, - ) - return 2 - if notification_route is not None and notification_route.enabled: - print("Final notification status: delivered", flush=True) - else: - print("Final notification status: skipped", flush=True) - return 1 if failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_bilibili.py b/src/gyxx_flow/modules/content_marketing/data/tools/relogin_bilibili.py deleted file mode 100644 index a28f2e1..0000000 --- a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_bilibili.py +++ /dev/null @@ -1,113 +0,0 @@ -"""每周一重登 B 站。 - -强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。 -任务计划: 每周一 09:00 - -跑法: - python data/tools/relogin_bilibili.py # 实际跑(会弹二维码) - python data/tools/relogin_bilibili.py --dry-run # 只看会做什么 -""" - -import argparse -import datetime as dt -import subprocess -import sys - -from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import ( - child_browser_environment, - resolve_relogin_runtime_paths, - sync_relogin_state_to_consumers, -) -from gyxx_flow.modules.content_marketing.data.tools.relogin_transaction import ( - begin, - commit, - rollback, -) -from gyxx_flow.modules.content_marketing.runtime_paths import PATHS - -PROJECT_DIR = PATHS.module_root -RUNTIME_PATHS = resolve_relogin_runtime_paths("bilibili") -COOKIE_FILE = RUNTIME_PATHS.cookie_file -STATE_FILE = RUNTIME_PATHS.storage_state_file -PROFILE_DIR = RUNTIME_PATHS.profile_dir -SCRAPER = PROJECT_DIR / "bilibili_comment_scraper.py" - - -def reset_state(): - """删 cookie JSON + storage_state + 清浏览器 profile。""" - state = begin((COOKIE_FILE, STATE_FILE), PROFILE_DIR) - print(" [OK] 旧 Cookie/Profile 已暂存,失败时将自动恢复") - return state - - -def main() -> int: - ap = argparse.ArgumentParser(description="每周一重登 B 站") - ap.add_argument("--login-timeout", type=int, default=300, help="扫码超时(秒)") - ap.add_argument("--dry-run", action="store_true", help="只打印计划,不删不跑") - args = ap.parse_args() - - print("== Relogin: B 站 ==") - print(f" 今天: {dt.date.today()} ({dt.date.today().strftime('%A')})") - print(f" cookie: {COOKIE_FILE}") - print(f" profile: {PROFILE_DIR}") - print() - - if args.dry_run: - print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码") - return 0 - - old_state = reset_state() - cmd = [ - sys.executable, - str(SCRAPER), - "login", # dummy url (login-only 模式不需要真实 url) - "--login-only", - "--login-timeout", - str(args.login_timeout), - ] - print(f"\n [RUN] {' '.join(cmd)}") - try: - rc = subprocess.call( - cmd, - cwd=str(PROJECT_DIR), - env=child_browser_environment( - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - profile_dir=PROFILE_DIR, - ), - ) - except BaseException: - rollback(old_state) - raise - print(f"\n exit={rc}") - login_ok = False - if rc == 0: - # 验证 SESSDATA 是否真写入了 - try: - import json - cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8")) - sessdata = any(c.get("name") == "SESSDATA" and c.get("value") for c in cookies) - login_ok = bool(sessdata) - print(f" SESSDATA 写入: {'OK' if sessdata else 'FAIL — 没有看到 SESSDATA,登录可能没成功'}") - except Exception as exc: - print(f" [WARN] 验证 cookie 时出错: {exc}") - if rc != 0 or not login_ok: - rollback(old_state) - print(" [ROLLBACK] 未完成扫码,已恢复旧登录状态") - return rc or 1 - try: - sync_relogin_state_to_consumers( - "bilibili", - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - ) - except Exception as exc: - rollback(old_state) - print(f" [ROLLBACK] 同步采集器登录态失败: {exc}") - return 1 - commit(old_state) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_douyin.py b/src/gyxx_flow/modules/content_marketing/data/tools/relogin_douyin.py deleted file mode 100644 index 3c37e30..0000000 --- a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_douyin.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Force a fresh Douyin login while preserving the previous usable session.""" - -from __future__ import annotations - -import argparse -import datetime as dt -import json -import subprocess -import sys - -from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import ( - child_browser_environment, - resolve_relogin_runtime_paths, - sync_relogin_state_to_consumers, -) -from gyxx_flow.modules.content_marketing.runtime_paths import PATHS - -try: - from .relogin_transaction import begin, commit, rollback -except ImportError: # Direct script execution from data/tools. - from gyxx_flow.modules.content_marketing.data.tools.relogin_transaction import ( - begin, - commit, - rollback, - ) - -PROJECT_DIR = PATHS.module_root -RUNTIME_PATHS = resolve_relogin_runtime_paths("douyin") -COOKIE_FILE = RUNTIME_PATHS.cookie_file -STATE_FILE = RUNTIME_PATHS.storage_state_file -PROFILE_DIR = RUNTIME_PATHS.profile_dir -SCRAPER = PROJECT_DIR / "douyin_comment_scraper.py" - -LOGIN_COOKIE = "sessionid" - - -def reset_state(): - state = begin((COOKIE_FILE, STATE_FILE), PROFILE_DIR) - print(" [OK] Old Cookie/Profile staged; failures will roll back automatically") - return state - - -def is_cookie_valid() -> bool: - if not COOKIE_FILE.exists(): - return False - try: - cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8")) - return any( - isinstance(cookie, dict) - and cookie.get("name") == LOGIN_COOKIE - and cookie.get("value") - for cookie in cookies - ) - except Exception: - return False - - -def main() -> int: - parser = argparse.ArgumentParser(description="Relogin Douyin") - parser.add_argument("--login-timeout", type=int, default=300) - parser.add_argument("--dry-run", action="store_true") - args = parser.parse_args() - - print("== Relogin: Douyin ==") - print(f" date: {dt.date.today()}") - print(f" cookie: {COOKIE_FILE}") - print(f" profile: {PROFILE_DIR}") - print(f" current cookie: {'present' if is_cookie_valid() else 'missing/invalid'}") - - if args.dry_run: - print("[DRY-RUN] Would stage Cookie/Profile and open the QR login window") - return 0 - - old_state = reset_state() - command = [ - sys.executable, - str(SCRAPER), - "login", - "--login-only", - "--login-timeout", - str(args.login_timeout), - ] - print(f"\n [RUN] {' '.join(command)}") - try: - return_code = subprocess.call( - command, - cwd=str(PROJECT_DIR), - env=child_browser_environment( - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - profile_dir=PROFILE_DIR, - ), - ) - except BaseException: - rollback(old_state) - raise - - login_ok = return_code == 0 and is_cookie_valid() - print(f"\n exit={return_code}") - print(f" {LOGIN_COOKIE} written: {'OK' if login_ok else 'FAIL'}") - if not login_ok: - rollback(old_state) - print(" [ROLLBACK] Login incomplete; restored previous Cookie/Profile") - return return_code or 1 - try: - sync_relogin_state_to_consumers( - "douyin", - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - ) - except Exception as exc: - rollback(old_state) - print(f" [ROLLBACK] Failed to sync collector login state: {exc}") - return 1 - - commit(old_state) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_pgy.py b/src/gyxx_flow/modules/content_marketing/data/tools/relogin_pgy.py deleted file mode 100644 index efa721b..0000000 --- a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_pgy.py +++ /dev/null @@ -1,148 +0,0 @@ -"""每周一重登小红书蒲公英。 - -强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。 -任务计划: 每周一 09:00 - -跑法: - python data/tools/relogin_pgy.py # 实际跑(会弹二维码) - python data/tools/relogin_pgy.py --dry-run # 只看会做什么 -""" - -import argparse -import datetime as dt -import shutil -import subprocess -import sys -from pathlib import Path - -from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import ( - child_browser_environment, - resolve_relogin_runtime_paths, - sync_relogin_state_to_consumers, -) -from gyxx_flow.modules.content_marketing.runtime_paths import PATHS - -PROJECT_DIR = PATHS.module_root -RUNTIME_PATHS = resolve_relogin_runtime_paths("pgy") -COOKIE_FILE = RUNTIME_PATHS.cookie_file -STATE_FILE = RUNTIME_PATHS.storage_state_file -PROFILE_DIR = RUNTIME_PATHS.profile_dir -SCRAPER = PROJECT_DIR / "pgy_xhs_scraper_v2.py" - - -def reset_state() -> tuple[Path | None, Path | None, Path | None]: - """Temporarily move the old login state aside so it can be restored.""" - ts = dt.datetime.now().strftime("%Y%m%d_%H%M%S_%f") - cookie_backup = None - state_backup = None - profile_backup = None - if COOKIE_FILE.exists(): - backup = COOKIE_FILE.with_suffix(f".json.bak.{ts}") - COOKIE_FILE.rename(backup) - cookie_backup = backup - print(f" [OK] 备份 cookie -> {backup.name}") - else: - print(f" [INFO] {COOKIE_FILE.name} 不存在,跳过") - if STATE_FILE.exists(): - state_backup = STATE_FILE.with_suffix(f".json.bak.{ts}") - STATE_FILE.rename(state_backup) - print(f" [OK] 备份 storage state -> {state_backup.name}") - else: - print(f" [INFO] {STATE_FILE.name} 不存在,跳过") - if PROFILE_DIR.exists(): - profile_backup = PROFILE_DIR.with_name(f"{PROFILE_DIR.name}.bak.{ts}") - PROFILE_DIR.rename(profile_backup) - print(f" [OK] 暂存 {PROFILE_DIR.name}/ -> {profile_backup.name}/") - else: - print(f" [INFO] {PROFILE_DIR.name}/ 不存在,跳过") - return cookie_backup, profile_backup, state_backup - - -def restore_previous_state( - cookie_backup: Path | None, - profile_backup: Path | None, - state_backup: Path | None = None, -) -> None: - """Discard an incomplete login and restore the last known-good state.""" - if COOKIE_FILE.exists(): - COOKIE_FILE.unlink() - if cookie_backup and cookie_backup.exists(): - cookie_backup.rename(COOKIE_FILE) - print(f" [ROLLBACK] 已恢复旧 cookie: {COOKIE_FILE.name}") - if STATE_FILE.exists(): - STATE_FILE.unlink() - if state_backup and state_backup.exists(): - state_backup.rename(STATE_FILE) - print(f" [ROLLBACK] 已恢复旧 storage state: {STATE_FILE.name}") - if PROFILE_DIR.exists(): - shutil.rmtree(PROFILE_DIR) - if profile_backup and profile_backup.exists(): - profile_backup.rename(PROFILE_DIR) - print(f" [ROLLBACK] 已恢复旧浏览器状态: {PROFILE_DIR.name}/") - - -def finish_successful_relogin(profile_backup: Path | None) -> None: - """Keep the new state and remove the bulky old profile backup.""" - if profile_backup and profile_backup.exists(): - shutil.rmtree(profile_backup) - - -def main() -> int: - ap = argparse.ArgumentParser(description="每周一重登小红书蒲公英") - ap.add_argument("--login-timeout", type=int, default=300, help="扫码超时(秒)") - ap.add_argument("--dry-run", action="store_true", help="只打印计划,不删不跑") - args = ap.parse_args() - - print("== Relogin: 小红书蒲公英 ==") - print(f" 今天: {dt.date.today()} ({dt.date.today().strftime('%A')})") - print(f" cookie: {COOKIE_FILE}") - print(f" profile: {PROFILE_DIR}") - print() - - if args.dry_run: - print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码") - return 0 - - cookie_backup, profile_backup, state_backup = reset_state() - cmd = [ - sys.executable, - str(SCRAPER), - "--login-only", - "--login-timeout", - str(args.login_timeout), - ] - print(f"\n [RUN] {' '.join(cmd)}") - try: - rc = subprocess.call( - cmd, - cwd=str(PROJECT_DIR), - env=child_browser_environment( - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - profile_dir=PROFILE_DIR, - ), - ) - except BaseException: - restore_previous_state(cookie_backup, profile_backup, state_backup) - raise - print(f"\n exit={rc}") - if rc != 0 or not COOKIE_FILE.exists(): - print(" [WARN] 新登录未完成,恢复旧登录状态") - restore_previous_state(cookie_backup, profile_backup, state_backup) - return rc or 1 - try: - sync_relogin_state_to_consumers( - "pgy", - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - ) - except Exception as exc: - restore_previous_state(cookie_backup, profile_backup, state_backup) - print(f" [ROLLBACK] 同步采集器登录态失败: {exc}") - return 1 - finish_successful_relogin(profile_backup) - return rc - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_runtime_paths.py b/src/gyxx_flow/modules/content_marketing/data/tools/relogin_runtime_paths.py deleted file mode 100644 index fa84f0c..0000000 --- a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_runtime_paths.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Resolve one relogin wrapper's browser state contract. - -The workflow runtime injects a unique browser binding into each relogin -wrapper. Standalone legacy launches do not have that environment, so they -retain the original per-platform files below the content-marketing state root. -""" - -from __future__ import annotations - -import json -import os -import re -import shutil -import tempfile -import uuid -from dataclasses import dataclass -from pathlib import Path -from typing import Mapping - -from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog -from gyxx_flow.modules.content_marketing.runtime_paths import PATHS - -_PLATFORM = re.compile(r"^[a-z][a-z0-9_-]{0,31}$") - -CONSUMER_BINDING_IDS: Mapping[str, tuple[str, ...]] = { - "bilibili": ("content.comments.collect_bilibili",), - "douyin": ("content.comments.collect_douyin",), - "pgy": ("content_marketing:pgy_xhs_scraper_v2.py",), - "xiaohongshu": ("content.comments.collect_xiaohongshu",), - # v2 now reads the Buyin/Doudian site and is owned by the douyin-shop - # account. Keep the legacy Xingtu relogin isolated from that vault. - "buyin": ("content_marketing:xingtu_scraper_v2.py",), - "xingtu": ("content_marketing:xingtu_scraper.py",), -} - - -@dataclass(frozen=True) -class ReloginRuntimePaths: - cookie_file: Path - storage_state_file: Path - profile_dir: Path - - -def resolve_relogin_runtime_paths( - platform: str, - environment: Mapping[str, str] | None = None, -) -> ReloginRuntimePaths: - """Prefer the injected binding and fall back to original platform paths.""" - - if not _PLATFORM.fullmatch(platform): - raise ValueError("invalid relogin platform") - values = os.environ if environment is None else environment - legacy_cookie = PATHS.state_root / "cookies" / f"{platform}_cookies.json" - legacy_storage = ( - PATHS.state_root / "cookies" / f"{platform}_storage_state.json" - ) - legacy_profile = PATHS.state_root / "browser-profiles" / platform - return ReloginRuntimePaths( - cookie_file=_bound_path( - values, - "GYXX_BROWSER_COOKIE_FILE", - legacy_cookie, - ), - storage_state_file=_bound_path( - values, - "GYXX_BROWSER_STORAGE_STATE_FILE", - legacy_storage, - ), - profile_dir=_bound_path( - values, - "GYXX_BROWSER_PROFILE_DIR", - legacy_profile, - ), - ) - - -def child_browser_environment( - *, - cookie_file: Path, - storage_state_file: Path, - profile_dir: Path, - environment: Mapping[str, str] | None = None, -) -> dict[str, str]: - """Keep the nested login scraper on the wrapper's injected binding.""" - - result = dict(os.environ if environment is None else environment) - result.update( - { - "GYXX_BROWSER_COOKIE_FILE": str(cookie_file.resolve()), - "GYXX_BROWSER_STORAGE_STATE_FILE": str( - storage_state_file.resolve() - ), - "GYXX_BROWSER_PROFILE_DIR": str(profile_dir.resolve()), - } - ) - # content_marketing/sitecustomize.py normally rebinds a nested Python - # script from argv[0]. A relogin scraper is an implementation detail of - # this wrapper and must write the wrapper binding that the parent monitors. - result.pop("GYXX_MODULE_ID", None) - return result - - -def sync_relogin_state_to_consumers( - platform: str, - *, - cookie_file: Path, - storage_state_file: Path, - environment: Mapping[str, str] | None = None, -) -> tuple[Path, ...]: - """Atomically publish a verified relogin state to unique consumer paths.""" - - try: - consumer_ids = CONSUMER_BINDING_IDS[platform] - except KeyError as exc: - raise ValueError("invalid relogin platform") from exc - values = os.environ if environment is None else environment - project_root = values.get("GYXX_PROJECT_ROOT", "").strip() - data_root = values.get("GYXX_DATA_ROOT", "").strip() - if not project_root and not data_root: - # Original standalone launches use the legacy per-platform state files - # directly, so there is no separate managed consumer to publish to. - return () - if not project_root or not data_root: - raise RuntimeError("managed relogin state sync requires project and data roots") - - catalog = RuntimeIntegrationCatalog.load_default( - project_root=project_root, - data_root=data_root, - ) - source_cookie = cookie_file.expanduser().resolve() - source_storage = storage_state_file.expanduser().resolve() - cookie_payload = _read_cookie_payload(source_cookie) - storage_available = source_storage.is_file() - if storage_available: - _read_storage_state_payload(source_storage) - - source_paths = {source_cookie, source_storage} - destination_paths: set[Path] = set() - destinations: list[Path] = [] - updates: list[tuple[Path, Path]] = [] - for consumer_id in consumer_ids: - binding = catalog.binding_for(consumer_id) - target_cookie = binding.cookie_file.resolve() - target_storage = binding.storage_state_file.resolve() - targets = (target_cookie, target_storage) - if any(target in source_paths for target in targets) or any( - target in destination_paths for target in targets - ): - raise RuntimeError( - "relogin and consumer browser state paths must be unique" - ) - destination_paths.update(targets) - _validate_cookie_requirements( - cookie_payload, - required_domains=binding.required_cookie_domains, - required_names=binding.required_cookie_names, - ) - updates.append((source_cookie, target_cookie)) - if storage_available: - updates.append((source_storage, target_storage)) - destinations.append(target_cookie) - _atomic_replace_files(updates) - return tuple(destinations) - - -def _bound_path( - environment: Mapping[str, str], - name: str, - fallback: Path, -) -> Path: - configured = environment.get(name, "").strip() - return ( - Path(configured).expanduser().resolve() - if configured - else fallback.resolve() - ) - - -def _read_cookie_payload(path: Path) -> list[dict[str, object]]: - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, ValueError) as exc: - raise RuntimeError("relogin cookie state is missing or invalid") from exc - if not isinstance(payload, list) or not payload or not all( - isinstance(cookie, dict) for cookie in payload - ): - raise RuntimeError("relogin cookie state is missing or invalid") - return payload - - -def _read_storage_state_payload(path: Path) -> dict[str, object]: - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, ValueError) as exc: - raise RuntimeError("relogin storage state is invalid") from exc - if not isinstance(payload, dict): - raise RuntimeError("relogin storage state is invalid") - return payload - - -def _validate_cookie_requirements( - cookies: list[dict[str, object]], - *, - required_domains: tuple[str, ...], - required_names: tuple[str, ...], -) -> None: - usable = [cookie for cookie in cookies if cookie.get("value")] - if not usable: - raise RuntimeError("relogin cookie state has no usable cookies") - if required_domains and not any( - any( - str(cookie.get("domain", "")).lstrip(".").casefold() - .endswith(domain.casefold().lstrip(".")) - for domain in required_domains - ) - for cookie in usable - ): - raise RuntimeError("relogin cookie state does not match consumer domain") - if required_names and not any( - str(cookie.get("name", "")) in required_names for cookie in usable - ): - raise RuntimeError("relogin cookie state misses the consumer login cookie") - - -def _atomic_replace_files(updates: list[tuple[Path, Path]]) -> None: - """Replace a small related file set and restore every old target on error.""" - - transaction = uuid.uuid4().hex - staged: list[tuple[Path, Path]] = [] - backups: dict[Path, Path | None] = {} - replaced: list[Path] = [] - try: - for source, target in updates: - target.parent.mkdir(parents=True, exist_ok=True) - handle, raw_temp = tempfile.mkstemp( - prefix=f".{target.name}.{transaction}.", - suffix=".tmp", - dir=target.parent, - ) - temp = Path(raw_temp) - with os.fdopen(handle, "wb") as stream: - stream.write(source.read_bytes()) - stream.flush() - os.fsync(stream.fileno()) - staged.append((temp, target)) - backup = target.with_name(f".{target.name}.{transaction}.bak") - if target.is_file(): - shutil.copy2(target, backup) - backups[target] = backup - else: - backups[target] = None - - for temp, target in staged: - os.replace(temp, target) - replaced.append(target) - except BaseException: - for target in reversed(replaced): - backup = backups.get(target) - if backup is not None and backup.exists(): - os.replace(backup, target) - elif target.exists(): - target.unlink() - raise - finally: - for temp, _target in staged: - if temp.exists(): - temp.unlink() - for backup in backups.values(): - if backup is not None and backup.exists(): - backup.unlink() - - -__all__ = [ - "CONSUMER_BINDING_IDS", - "ReloginRuntimePaths", - "child_browser_environment", - "resolve_relogin_runtime_paths", - "sync_relogin_state_to_consumers", -] diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_transaction.py b/src/gyxx_flow/modules/content_marketing/data/tools/relogin_transaction.py deleted file mode 100644 index 49bac92..0000000 --- a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_transaction.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Transactional backup/rollback helpers for interactive relogin scripts.""" - -from __future__ import annotations - -import datetime as dt -import shutil -from dataclasses import dataclass -from pathlib import Path - - -@dataclass -class LoginStateBackup: - files: dict[Path, Path] - profile: Path - profile_backup: Path | None - - -def begin(files: tuple[Path, ...], profile: Path) -> LoginStateBackup: - stamp = dt.datetime.now().strftime("%Y%m%d_%H%M%S_%f") - backups: dict[Path, Path] = {} - for source in files: - if source.exists(): - backup = source.with_name(f"{source.name}.bak.{stamp}") - source.rename(backup) - backups[source] = backup - profile_backup = None - if profile.exists(): - profile_backup = profile.with_name(f"{profile.name}.bak.{stamp}") - profile.rename(profile_backup) - return LoginStateBackup(backups, profile, profile_backup) - - -def rollback(state: LoginStateBackup) -> None: - for source, backup in state.files.items(): - if source.exists(): - source.unlink() - if backup.exists(): - backup.rename(source) - if state.profile.exists(): - shutil.rmtree(state.profile) - if state.profile_backup and state.profile_backup.exists(): - state.profile_backup.rename(state.profile) - - -def commit(state: LoginStateBackup) -> None: - if state.profile_backup and state.profile_backup.exists(): - shutil.rmtree(state.profile_backup) diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_xiaohongshu.py b/src/gyxx_flow/modules/content_marketing/data/tools/relogin_xiaohongshu.py deleted file mode 100644 index 2c3f633..0000000 --- a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_xiaohongshu.py +++ /dev/null @@ -1,116 +0,0 @@ -"""每周重登小红书。 - -强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。 -跑法: - python data/tools/relogin_xiaohongshu.py # 实际跑(会弹二维码) - python data/tools/relogin_xiaohongshu.py --dry-run # 只看会做什么 -""" - -import argparse -import datetime as dt -import json -import subprocess -import sys - -from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import ( - child_browser_environment, - resolve_relogin_runtime_paths, - sync_relogin_state_to_consumers, -) -from gyxx_flow.modules.content_marketing.data.tools.relogin_transaction import ( - begin, - commit, - rollback, -) -from gyxx_flow.modules.content_marketing.runtime_paths import PATHS - -PROJECT_DIR = PATHS.module_root -RUNTIME_PATHS = resolve_relogin_runtime_paths("xiaohongshu") -COOKIE_FILE = RUNTIME_PATHS.cookie_file -STATE_FILE = RUNTIME_PATHS.storage_state_file -PROFILE_DIR = RUNTIME_PATHS.profile_dir -SCRAPER = PROJECT_DIR / "xiaohongshu_comment_scraper.py" - -LOGIN_COOKIE = "web_session" - - -def reset_state(): - state = begin((COOKIE_FILE, STATE_FILE), PROFILE_DIR) - print(" [OK] 旧 Cookie/Profile 已暂存,失败时将自动恢复") - return state - - -def is_cookie_valid() -> bool: - if not COOKIE_FILE.exists(): - return False - try: - cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8")) - return any(c.get("name") == LOGIN_COOKIE and c.get("value") for c in cookies) - except Exception: - return False - - -def main() -> int: - ap = argparse.ArgumentParser(description="重登小红书") - ap.add_argument("--login-timeout", type=int, default=300, help="扫码超时(秒)") - ap.add_argument("--dry-run", action="store_true", help="只打印计划,不删不跑") - args = ap.parse_args() - - print("== Relogin: 小红书 ==") - print(f" 今天: {dt.date.today()}") - print(f" cookie: {COOKIE_FILE}") - print(f" profile: {PROFILE_DIR}") - print(f" 当前 cookie 状态: {'有效' if is_cookie_valid() else '无效/不存在'}") - print() - - if args.dry_run: - print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码") - return 0 - - old_state = reset_state() - cmd = [ - sys.executable, - str(SCRAPER), - "login", - "--login-only", - "--login-timeout", - str(args.login_timeout), - ] - print(f"\n [RUN] {' '.join(cmd)}") - try: - rc = subprocess.call( - cmd, - cwd=str(PROJECT_DIR), - env=child_browser_environment( - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - profile_dir=PROFILE_DIR, - ), - ) - except BaseException: - rollback(old_state) - raise - print(f"\n exit={rc}") - login_ok = rc == 0 and is_cookie_valid() - if rc == 0: - print(f" {LOGIN_COOKIE} 写入: {'OK' if login_ok else 'FAIL — 没有看到 web_session,登录可能没成功'}") - if not login_ok: - rollback(old_state) - print(" [ROLLBACK] 未完成扫码,已恢复旧登录状态") - return rc or 1 - try: - sync_relogin_state_to_consumers( - "xiaohongshu", - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - ) - except Exception as exc: - rollback(old_state) - print(f" [ROLLBACK] 同步采集器登录态失败: {exc}") - return 1 - commit(old_state) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_xingtu.py b/src/gyxx_flow/modules/content_marketing/data/tools/relogin_xingtu.py deleted file mode 100644 index 1f155c9..0000000 --- a/src/gyxx_flow/modules/content_marketing/data/tools/relogin_xingtu.py +++ /dev/null @@ -1,350 +0,0 @@ -"""每 3 个周一重登一次抖音/星图。 - -强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。 -星图登录完成后,自动把登录态 cookie 同步到抖音评论 scraper,避免再扫一次码。 - -任务计划: 每周一 09:00 (脚本内部判断是否到了该重登的周一,不到就退出) - -3 周循环基准: 任意一个周一,后续 (今天 - BASE_DATE) // 7 % 3 == 0 才是星图周。 -改 BASE_DATE 就改了重登周期。 - -跑法: - python data/tools/relogin_xingtu.py # 实际跑 - python data/tools/relogin_xingtu.py --dry-run # 看今天是不是星图周 - python data/tools/relogin_xingtu.py --force # 忽略 3 周判断,强制重登 -""" - -import argparse -import datetime as dt -import json -import os -import shutil -import subprocess -import sys -from pathlib import Path - -import psutil - -from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import ( - child_browser_environment, - resolve_relogin_runtime_paths, - sync_relogin_state_to_consumers, -) -from gyxx_flow.modules.content_marketing.runtime_paths import PATHS - -PROJECT_DIR = PATHS.module_root -RUNTIME_PATHS = resolve_relogin_runtime_paths("xingtu") -COOKIE_FILE = RUNTIME_PATHS.cookie_file -STATE_FILE = RUNTIME_PATHS.storage_state_file -PROFILE_DIR = RUNTIME_PATHS.profile_dir -SCRAPER = PROJECT_DIR / "xingtu_scraper_v2.py" - -# 抖音评论 scraper 的 cookie 文件 (登录态共享目标) -DY_COOKIE_FILE = PATHS.state_root / "cookies/douyin_cookies.json" -DY_STATE_FILE = PATHS.state_root / "cookies/douyin_storage_state.json" -DY_PROFILE_DIR = PATHS.state_root / "browser-profiles/douyin" - -# 需要同步的抖音登录 cookie 名 -DY_LOGIN_COOKIES = {"sessionid", "sessionid_ss", "sid_guard", "sid_tt", "passport_csrf_token"} - -# 3 周循环起点(必须是周一)。星图 cookie ~22 天,每 3 周(21 天)重登正好。 -BASE_DATE = dt.date(2026, 6, 22) -LOGIN_COOKIE_NAMES = {"sessionid", "sessionid_ss", "sid_tt", "sid_guard"} - - -def transaction_file() -> Path: - return COOKIE_FILE.with_name(".xingtu_relogin_transaction.json") - - -def _write_transaction( - cookie_backup: Path | None, - profile_backup: Path | None, - state_backup: Path | None, -) -> None: - marker = transaction_file() - marker.parent.mkdir(parents=True, exist_ok=True) - temp = marker.with_name(f"{marker.name}.tmp") - temp.write_text(json.dumps({ - "cookie_backup": str(cookie_backup) if cookie_backup else "", - "state_backup": str(state_backup) if state_backup else "", - "profile_backup": str(profile_backup) if profile_backup else "", - }, ensure_ascii=False, indent=2), encoding="utf-8") - os.replace(temp, marker) - - -def _clear_transaction() -> None: - marker = transaction_file() - if marker.exists(): - marker.unlink() - - -def _remove_path(path: Path) -> None: - if path.is_dir(): - shutil.rmtree(path) - elif path.exists(): - path.unlink() - - -def recover_interrupted_state() -> bool: - """Restore a relogin transaction that was killed after moving old state.""" - marker = transaction_file() - if not marker.exists(): - return False - try: - state = json.loads(marker.read_text(encoding="utf-8")) - except Exception: - return False - - cookie_backup = Path(state["cookie_backup"]) if state.get("cookie_backup") else None - state_backup = Path(state["state_backup"]) if state.get("state_backup") else None - profile_backup = Path(state["profile_backup"]) if state.get("profile_backup") else None - terminate_profile_processes() - if cookie_backup and cookie_backup.exists(): - _remove_path(COOKIE_FILE) - cookie_backup.rename(COOKIE_FILE) - if state_backup and state_backup.exists(): - _remove_path(STATE_FILE) - state_backup.rename(STATE_FILE) - if profile_backup and profile_backup.exists(): - _remove_path(PROFILE_DIR) - profile_backup.rename(PROFILE_DIR) - _clear_transaction() - print(" [RECOVER] 已恢复上次被中断的星图登录状态") - return True - - -def has_valid_login_cookie_file(path: Path | None = None) -> bool: - path = path or COOKIE_FILE - try: - cookies = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return False - return any( - isinstance(cookie, dict) - and cookie.get("name") in LOGIN_COOKIE_NAMES - and cookie.get("value") - for cookie in cookies - ) - - -def terminate_profile_processes() -> None: - """Stop only browser processes using Xingtu's dedicated profile.""" - profile_text = str(PROFILE_DIR.resolve()).casefold() - matches = [] - for process in psutil.process_iter(["pid", "cmdline"]): - try: - command = " ".join(process.info.get("cmdline") or []).casefold() - if process.pid != os.getpid() and profile_text in command: - matches.append(process) - except (psutil.Error, OSError): - continue - for process in matches: - try: - process.terminate() - except (psutil.Error, OSError): - pass - _gone, alive = psutil.wait_procs(matches, timeout=5) - for process in alive: - try: - process.kill() - except (psutil.Error, OSError): - pass - - -def is_xingtu_week(today: dt.date, base: dt.date) -> bool: - days = (today - base).days - if days < 0: - return False - return (days // 7) % 3 == 0 - - -def reset_state() -> tuple[Path | None, Path | None, Path | None]: - """Temporarily move the old login state aside so it can be restored.""" - ts = dt.datetime.now().strftime("%Y%m%d_%H%M%S_%f") - cookie_backup = None - state_backup = None - profile_backup = None - if COOKIE_FILE.exists(): - cookie_backup = COOKIE_FILE.with_suffix(f".json.bak.{ts}") - if STATE_FILE.exists(): - state_backup = STATE_FILE.with_suffix(f".json.bak.{ts}") - if PROFILE_DIR.exists(): - profile_backup = PROFILE_DIR.with_name(f"{PROFILE_DIR.name}.bak.{ts}") - _write_transaction(cookie_backup, profile_backup, state_backup) - - if cookie_backup: - COOKIE_FILE.rename(cookie_backup) - print(f" [OK] 备份 cookie -> {cookie_backup.name}") - else: - print(f" [INFO] {COOKIE_FILE.name} 不存在,跳过") - if state_backup: - STATE_FILE.rename(state_backup) - print(f" [OK] 备份 storage state -> {state_backup.name}") - else: - print(f" [INFO] {STATE_FILE.name} 不存在,跳过") - if profile_backup: - PROFILE_DIR.rename(profile_backup) - print(f" [OK] 暂存 {PROFILE_DIR.name}/ -> {profile_backup.name}/") - else: - print(f" [INFO] {PROFILE_DIR.name}/ 不存在,跳过") - return cookie_backup, profile_backup, state_backup - - -def restore_previous_state( - cookie_backup: Path | None, - profile_backup: Path | None, - state_backup: Path | None = None, -) -> None: - """Discard an incomplete login and restore the last known-good state.""" - terminate_profile_processes() - if COOKIE_FILE.exists(): - COOKIE_FILE.unlink() - if cookie_backup and cookie_backup.exists(): - cookie_backup.rename(COOKIE_FILE) - print(f" [ROLLBACK] 已恢复旧 cookie: {COOKIE_FILE.name}") - _remove_path(STATE_FILE) - if state_backup and state_backup.exists(): - state_backup.rename(STATE_FILE) - print(f" [ROLLBACK] 已恢复旧 storage state: {STATE_FILE.name}") - if PROFILE_DIR.exists(): - shutil.rmtree(PROFILE_DIR) - if profile_backup and profile_backup.exists(): - profile_backup.rename(PROFILE_DIR) - print(f" [ROLLBACK] 已恢复旧浏览器状态: {PROFILE_DIR.name}/") - _clear_transaction() - - -def finish_successful_relogin(profile_backup: Path | None) -> None: - """Keep the new state and remove the bulky old profile backup.""" - if profile_backup and profile_backup.exists(): - shutil.rmtree(profile_backup) - _clear_transaction() - - -def sync_to_douyin_scraper() -> None: - """把星图的登录态 cookie 同步到抖音评论 scraper,避免再扫一次码。""" - if not COOKIE_FILE.exists(): - print(" [SKIP] 星图 cookie 不存在,无法同步") - return - - try: - xingtu_cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8")) - except Exception as exc: - print(f" [SKIP] 读取星图 cookie 失败: {exc}") - return - - # 提取登录态 cookie - login_cookies = [c for c in xingtu_cookies if isinstance(c, dict) and c.get("name") in DY_LOGIN_COOKIES] - if not login_cookies: - print(" [SKIP] 星图 cookie 中没有找到登录态,跳过同步") - return - - # 清除抖音评论 scraper 的旧 profile (强制刷新) - if DY_PROFILE_DIR.exists(): - shutil.rmtree(DY_PROFILE_DIR) - print(f" [OK] 清空 {DY_PROFILE_DIR.name}/") - - # 读取或创建抖音评论 cookie 文件 - dy_cookies = [] - if DY_COOKIE_FILE.exists(): - try: - dy_cookies = json.loads(DY_COOKIE_FILE.read_text(encoding="utf-8")) - except Exception: - dy_cookies = [] - - # 合并: 同名 cookie 覆盖,新 cookie 追加 - dy_by_name = {c.get("name"): c for c in dy_cookies if isinstance(c, dict)} - for c in login_cookies: - # 修正 domain: 星图的 cookie domain 可能是 .douyin.com 子域,评论采集需要 .douyin.com - domain = c.get("domain", "") - if "xingtu" in domain or "oceanengine" in domain: - c["domain"] = ".douyin.com" - dy_by_name[c["name"]] = c - - DY_COOKIE_FILE.parent.mkdir(parents=True, exist_ok=True) - DY_COOKIE_FILE.write_text( - json.dumps(list(dy_by_name.values()), ensure_ascii=False, indent=2), - encoding="utf-8", - ) - print(f" [OK] 同步 {len(login_cookies)} 个登录 cookie 到 {DY_COOKIE_FILE.name}") - - -def main() -> int: - ap = argparse.ArgumentParser(description="每 3 周周一重登星图") - ap.add_argument("--base-date", default=BASE_DATE.isoformat(), - help="3 周循环起点(必须是周一),格式 YYYY-MM-DD") - ap.add_argument("--login-timeout", type=int, default=300) - ap.add_argument("--force", action="store_true", help="忽略 3 周判断,强制重登") - ap.add_argument("--dry-run", action="store_true", help="只打印计划") - args = ap.parse_args() - - today = dt.date.today() - base = dt.date.fromisoformat(args.base_date) - is_xw = is_xingtu_week(today, base) - - print("== Relogin: 抖音/星图 ==") - print(f" 今天: {today} ({today.strftime('%A')})") - print(f" 3 周基准: {base}") - print(f" 星图周? {'是' if is_xw else '否'}") - print() - - if not is_xw and not args.force: - print("今天不是星图周,跳过 (传 --force 强制重登)") - return 0 - - if args.dry_run: - print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码") - return 0 - - recover_interrupted_state() - terminate_profile_processes() - cookie_backup, profile_backup, state_backup = reset_state() - cmd = [ - sys.executable, - str(SCRAPER), - "--login-only", - "--login-timeout", - str(args.login_timeout), - ] - print(f"\n [RUN] {' '.join(cmd)}") - try: - rc = subprocess.call( - cmd, - cwd=str(PROJECT_DIR), - env=child_browser_environment( - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - profile_dir=PROFILE_DIR, - ), - ) - except BaseException: - restore_previous_state(cookie_backup, profile_backup, state_backup) - raise - finally: - terminate_profile_processes() - print(f"\n exit={rc}") - - if rc != 0 or not has_valid_login_cookie_file(): - print(" [WARN] 新登录未完成,恢复旧登录状态") - restore_previous_state(cookie_backup, profile_backup, state_backup) - return rc or 1 - try: - sync_relogin_state_to_consumers( - "xingtu", - cookie_file=COOKIE_FILE, - storage_state_file=STATE_FILE, - ) - except Exception as exc: - restore_previous_state(cookie_backup, profile_backup, state_backup) - print(f" [ROLLBACK] 同步采集器登录态失败: {exc}") - return 1 - - finish_successful_relogin(profile_backup) - if rc == 0: - sync_to_douyin_scraper() - - return rc - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/gyxx_flow/modules/content_marketing/douyin_comment_scraper.py b/src/gyxx_flow/modules/content_marketing/douyin_comment_scraper.py index 63cc254..83b9364 100644 --- a/src/gyxx_flow/modules/content_marketing/douyin_comment_scraper.py +++ b/src/gyxx_flow/modules/content_marketing/douyin_comment_scraper.py @@ -273,18 +273,15 @@ def maybe_wait_for_login( *, deadline: float | None = None, ) -> None: - login_deadline = time.monotonic() + login_timeout - while time.monotonic() < login_deadline: - _check_note_deadline(deadline, "login wait") - try: - body = page.locator("body").inner_text(timeout=5000) - except Exception: - body = "" - if "扫码登录" not in body and "登录后" not in body: - return - log("Douyin may require login. Please finish login in the opened Chrome window.") - page.wait_for_timeout(3000) - _check_note_deadline(deadline, "login wait") + # 登录统一由账号 vault 管理（gyxx accounts login douyin-content）， + # 由工作流执行前的 cookie 网关保证；运行中失效立即降级，不再等待人工扫码。 + try: + body = page.locator("body").inner_text(timeout=5000) + except Exception: + body = "" + if "扫码登录" in body or "登录后" in body: + log("Douyin login state missing; collecting degraded samples. " + "Run `gyxx accounts login douyin-content` to restore full comments.") def is_logged_in(page: Page) -> bool: @@ -833,55 +830,16 @@ def write_outputs(url: str, comments: list[dict[str, Any]], result: dict[str, An return json_path, csv_path -def login_only(login_timeout: int) -> int: - """只触发登录(headed),登录完成立刻退出。给 relogin_douyin.py 用。""" - def action(page: Page) -> None: - page.goto("https://www.douyin.com/", wait_until="domcontentloaded", timeout=90000) - page.wait_for_timeout(2500) - # 主动触发登录弹窗(点登录按钮) - try: - login_btn = page.get_by_text("登录", exact=True).first - if login_btn.count(): - login_btn.click(timeout=3000) - page.wait_for_timeout(1500) - except Exception: - pass - maybe_wait_for_login(page, login_timeout) - save_state(page) - - PROFILE_DIR.mkdir(parents=True, exist_ok=True) - with ScraplingBrowser( - headless=False, - real_chrome=True, - user_data_dir=str(PROFILE_DIR), - locale="zh-CN", - timezone_id="Asia/Shanghai", - timeout=90000, - network_idle=False, - disable_resources=False, - google_search=False, - page_setup=restore_cookies, - max_pages=1, - ) as browser: - browser.run_fetch_action("https://www.douyin.com/", action, wait=1000) - log("[login-only] 完成") - return 0 - - def main() -> int: parser = argparse.ArgumentParser(description="Scrape Douyin video comments via Scrapling browser and XHR interception.") - parser.add_argument("url", help="Douyin video/user URL (传 'login' 配合 --login-only)") - parser.add_argument("--login-timeout", type=int, default=300, help="等待扫码/登录秒数") + parser.add_argument("url", help="Douyin video/user URL") + parser.add_argument("--login-timeout", type=int, default=300, + help="保留参数（登录等待已移除；未登录将直接降级采集）") parser.add_argument("--max-scrolls", type=int, default=120, help="最多滚动加载次数") parser.add_argument("--idle-rounds", type=int, default=8, help="连续几轮无新评论后停止") - parser.add_argument("--headless", action="store_true", help="无头模式；首次登录不要使用") - parser.add_argument("--login-only", action="store_true", - help="只触发登录(headed)然后退出,给 relogin_douyin 用") + parser.add_argument("--headless", action="store_true", help="无头模式") args = parser.parse_args() - if args.login_only: - return login_only(args.login_timeout) - comments, result = scrape_comments(args.url, args.login_timeout, args.max_scrolls, args.idle_rounds, args.headless) json_path, csv_path = write_outputs(args.url, comments, result) log(f"Final URL: {result.get('final_url', '')}") diff --git a/src/gyxx_flow/modules/content_marketing/pgy_xhs_scraper_v2.py b/src/gyxx_flow/modules/content_marketing/pgy_xhs_scraper_v2.py index 86c851e..9b6b004 100644 --- a/src/gyxx_flow/modules/content_marketing/pgy_xhs_scraper_v2.py +++ b/src/gyxx_flow/modules/content_marketing/pgy_xhs_scraper_v2.py @@ -38,6 +38,7 @@ from gyxx_flow.adapters.scrapling import ( BrowserTimeoutError, # noqa: F401 is_browser_timeout_error, ) +from gyxx_flow.adapters.browser import CookieExpiredError, cookie_expired_message from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE from gyxx_flow.modules.content_marketing.collection_completeness import ( BLOCKED_INPUT, @@ -333,56 +334,6 @@ def is_logged_in(page: Page) -> bool: return False -def open_scan_login(page: Page) -> None: - """Open the login dialog and prefer its QR-code login tab.""" - for pattern in (r"^登录$|账号登录",): - try: - button = page.get_by_text(re.compile(pattern)).first - if button.count() and button.is_visible(): - button.click(timeout=2000) - page.wait_for_timeout(500) - except Exception: - pass - - # The login dialog uses an image-only folded-corner switch. Its bundled - # asset name is stable even though the generated CSS class is not. - try: - qr_switch = page.locator('img[src*="qr_code"]').first - if qr_switch.count() and qr_switch.is_visible(): - # The browser engine's actionability check is blocked by the folded-corner - # overlay. A native DOM click triggers the image's real handler. - qr_switch.evaluate("e => e.click()") - page.wait_for_timeout(1000) - return - except Exception: - pass - - for pattern in (r"扫码登录|二维码登录|扫码",): - try: - button = page.get_by_text(re.compile(pattern)).first - if button.count() and button.is_visible(): - button.click(timeout=2000) - page.wait_for_timeout(500) - except Exception: - pass - - -def wait_for_scan_login(page: Page, timeout_seconds: int) -> None: - log("Not logged in. 请在打开的 Chrome 中扫码登录。") - open_scan_login(page) - deadline = time.time() + timeout_seconds - next_login_click = time.time() + 5 - while time.time() < deadline: - if is_logged_in(page): - save_state(page) - return - if time.time() >= next_login_click: - open_scan_login(page) - next_login_click = time.time() + 5 - page.wait_for_timeout(2000) - raise TimeoutError(f"Login timed out after {timeout_seconds} seconds.") - - def ensure_login(page: Page, login_timeout: int) -> None: page.goto(HOME_URL, wait_until="domcontentloaded", timeout=60000) page.wait_for_timeout(2500) @@ -402,20 +353,11 @@ def ensure_login(page: Page, login_timeout: int) -> None: ) log(f"[SKIPPED_COOKIE] {reason}") raise AcceptanceCookieSkip(reason) - page.goto(LOGIN_URL, wait_until="domcontentloaded", timeout=60000) - page.wait_for_timeout(1500) - wait_for_scan_login(page, login_timeout) - page.goto(HOME_URL, wait_until="domcontentloaded", timeout=60000) - page.wait_for_load_state("domcontentloaded") - # 关闭弹窗 - for text in ("跳过", "不再提示", "我知道了"): - try: - b = page.get_by_text(text, exact=True).first - if b.count(): - b.click(timeout=1500) - page.wait_for_timeout(800) - except Exception: - pass + # 登录统一由账号 vault 管理（gyxx accounts login xiaohongshu-content）， + # 采集脚本不再内嵌扫码等待；运行中失效直接快速失败。 + reason = cookie_expired_message("xiaohongshu-content", "小红书蒲公英") + log(f"[COOKIE_EXPIRED] {reason}") + raise CookieExpiredError(reason) def _is_login_timeout(exc: BaseException) -> bool: @@ -1933,7 +1875,7 @@ def scrape_styles(styles: list[dict], login_timeout: int, headless: bool, action, page_setup=restore_cookies, ) - except AcceptanceCookieSkip: + except (AcceptanceCookieSkip, CookieExpiredError): raise except Exception as exc: session_lost = is_browser_session_lost(exc) @@ -2044,39 +1986,6 @@ def parse_int(s) -> int | None: return int(num) -def login_only(login_timeout: int) -> int: - """只触发登录(headed),登录完成立刻退出。给 weekly_relogin.py 用。""" - def action(page): - ensure_login(page, login_timeout) - return page - - session = None - try: - with DynamicSession( - headless=False, - real_chrome=True, - user_data_dir=str(PROFILE_DIR), - locale="zh-CN", - timezone_id="Asia/Shanghai", - timeout=60000, - network_idle=False, - disable_resources=False, - google_search=False, - page_setup=restore_cookies, - max_pages=2, - ) as session: - session.fetch(HOME_URL, page_action=action, wait=1000) - except Exception as exc: - if "TargetClosedError" in type(exc).__name__ or "has been closed" in str(exc): - log(f" [WARN] 浏览器清理时报错: {type(exc).__name__}") - else: - raise - finally: - _force_cleanup_session(session) - log("[login-only] 登录完成,退出") - return 0 - - def main() -> int: parser = argparse.ArgumentParser(description="小红书蒲公英 V2: 多维表格驱动精准采集+回填") add_repeatable_style_argument(parser, "只跑指定款式编号(可多次)") @@ -2090,8 +1999,6 @@ def main() -> int: parser.add_argument("--skip-done", action="store_true", help="跳过已完成的款式(从进度文件读)") parser.add_argument("--inter-style-pause", type=int, default=5, help="兼容旧参数；全局按达人采集后不再按款式暂停") - parser.add_argument("--login-only", action="store_true", - help="只触发登录(headed),登录完成立刻退出,给 weekly_relogin 用") parser.add_argument("--self-operated", action="store_true", help="抓取自营达人表格(而非合作达人)") parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE, @@ -2108,12 +2015,6 @@ def main() -> int: help=argparse.SUPPRESS) args = parser.parse_args() - if args.login_only: - try: - return login_only(args.login_timeout) - except AcceptanceCookieSkip: - return COOKIE_SKIP_EXIT_CODE - mapping = load_mapping(self_operated=args.self_operated) styles = mapping["tables"] if args.style: @@ -2213,7 +2114,7 @@ def main() -> int: published_from=args.published_from, max_age_days=args.max_age_days, no_retry=args.no_retry, ) - except AcceptanceCookieSkip: + except (AcceptanceCookieSkip, CookieExpiredError): return COOKIE_SKIP_EXIT_CODE except Exception as exc: if not _is_login_timeout(exc): diff --git a/src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py b/src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py index bcfd72e..a24d774 100644 --- a/src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py +++ b/src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py @@ -363,8 +363,8 @@ def log(msg: str) -> None: # ============================================================ -# B站登录态: Cookie 持久化 + 扫码登录引导 -# 空间投稿列表需要登录态,首次用 --login-only 扫码,之后调度运行复用。 +# B站登录态: Cookie 持久化；登录统一由账号 vault 管理 +# (gyxx accounts login bilibili-content)，脚本内不再提供扫码登录。 # ============================================================ BILI_COOKIE_FILE = PATHS.browser_cookie_file @@ -389,56 +389,6 @@ def save_state(page) -> None: log(f"已保存 Cookie ({len(cookies)} 条): {BILI_COOKIE_FILE}") -def _has_login_cookies(page) -> bool: - names = {cookie.get("name") for cookie in page.context.cookies()} - return "SESSDATA" in names and "DedeUserID" in names - - -def login_only(login_timeout: int = 300) -> int: - """打开 B站扫码登录页,等待人工扫码后保存 Cookie。""" - from scrapling.fetchers import StealthySession - - with StealthySession( - headless=False, - real_chrome=True, - user_data_dir=str(PATHS.browser_profile_dir), - locale="zh-CN", - timezone_id="Asia/Shanghai", - timeout=60_000, - network_idle=False, - disable_resources=False, - google_search=False, - max_pages=2, - page_setup=restore_cookies, - ) as session: - page = ( - session.context.pages[0] - if session.context.pages - else session.context.new_page() - ) - page.goto("https://passport.bilibili.com/login", - wait_until="domcontentloaded", timeout=60_000) - page.wait_for_timeout(2000) - if _has_login_cookies(page): - log("✅ 已有 B站登录态,无需扫码") - save_state(page) - return 0 - print("\n" + "=" * 60) - print("🔐 请在打开的浏览器中用哔哩哔哩 App 扫码登录") - print(" 登录成功后脚本会自动继续并保存 Cookie") - print("=" * 60, flush=True) - deadline = time.time() + login_timeout - while time.time() < deadline: - if _has_login_cookies(page): - log("✅ 检测到登录成功") - page.wait_for_timeout(2000) - save_state(page) - return 0 - page.wait_for_timeout(2000) - log(f"❌ 等待扫码超时({login_timeout}s),未保存登录态") - return 1 - - # ============================================================ # 飞书多维表格 # ============================================================ @@ -846,17 +796,10 @@ def main() -> int: description="自营 B 站采集: 公开 API 全量视频指标 + UP 主主页全量投稿 + 飞书曝光回填") parser.add_argument("--style", type=int, help="只跑指定款式编号") parser.add_argument("--dry-run", action="store_true", help="不写回飞书") - parser.add_argument("--login-only", action="store_true", - help="只打开 B站扫码登录并保存 Cookie(首次部署引导)") - parser.add_argument("--login-timeout", type=int, default=300, - help="等待人工扫码的秒数") parser.add_argument("--delay", type=float, default=0.3, help="每条请求间隔秒数(防风控)") args = parser.parse_args() - if args.login_only: - return login_only(args.login_timeout) - from gyxx_flow.modules.content_marketing import feishu_mapping try: mapping = feishu_mapping.load_mapping(self_operated=True) diff --git a/src/gyxx_flow/modules/content_marketing/self_douyin_scraper.py b/src/gyxx_flow/modules/content_marketing/self_douyin_scraper.py index 0419c3e..000ed98 100644 --- a/src/gyxx_flow/modules/content_marketing/self_douyin_scraper.py +++ b/src/gyxx_flow/modules/content_marketing/self_douyin_scraper.py @@ -27,6 +27,8 @@ from typing import Any from scrapling.fetchers import DynamicSession from gyxx_flow.adapters.scrapling import BrowserPage as Page +from gyxx_flow.adapters.browser import CookieExpiredError, cookie_expired_message +from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE from gyxx_flow.modules.content_marketing.collection_completeness import ( BLOCKED_INPUT, @@ -261,17 +263,16 @@ def save_state(page: Page) -> None: def maybe_wait_for_login(page: Page, login_timeout: int) -> None: - deadline = time.time() + login_timeout - while time.time() < deadline: - try: - body = page.locator("body").inner_text(timeout=5000) - except Exception: - body = "" - if "扫码登录" not in body and "登录后" not in body: - return - log("抖音需要登录,请在打开的 Chrome 窗口中完成登录") - page.wait_for_timeout(3000) - log("[WARN] 登录等待超时") + # 登录统一由账号 vault 管理（gyxx accounts login douyin-content）， + # 采集过程遇到登录态失效立即快速失败，不再等待人工扫码。 + try: + body = page.locator("body").inner_text(timeout=5000) + except Exception: + body = "" + if "扫码登录" in body or "登录后" in body: + reason = cookie_expired_message("douyin-content", "自营抖音") + log(f"[COOKIE_EXPIRED] {reason}") + raise CookieExpiredError(reason) def dismiss_popups(page: Page) -> None: @@ -605,33 +606,6 @@ def scrape_one_style(style: dict, login_timeout: int, headless: bool, return summary -def login_only(login_timeout: int) -> int: - """只触发登录""" - login_done = False - - def page_action(page: Page): - restore_cookies(page) - page.goto("https://www.douyin.com/", wait_until="domcontentloaded", timeout=60000) - page.wait_for_timeout(2000) - maybe_wait_for_login(page, login_timeout) - save_state(page) - log("登录完成") - - PROFILE_DIR.mkdir(parents=True, exist_ok=True) - with DynamicSession( - headless=False, - real_chrome=True, - user_data_dir=str(PROFILE_DIR), - locale="zh-CN", - timezone_id="Asia/Shanghai", - timeout=60000, - page_setup=restore_cookies, - ) as session: - session.fetch("https://www.douyin.com/", page_action=page_action, wait=1000) - - return 0 if login_done else 1 - - # ============================================================ # main # ============================================================ @@ -642,12 +616,8 @@ def main() -> int: parser.add_argument("--headless", action="store_true") parser.add_argument("--login-timeout", type=int, default=300) parser.add_argument("--dry-run", action="store_true", help="不写回多维表格") - parser.add_argument("--login-only", action="store_true", help="只触发登录") args = parser.parse_args() - if args.login_only: - return login_only(args.login_timeout) - mapping = load_mapping() styles = mapping["tables"] if args.style: @@ -703,6 +673,9 @@ def main() -> int: if __name__ == "__main__": try: raise SystemExit(main()) + except CookieExpiredError as exc: + print(f"[COOKIE_EXPIRED] {exc}", file=sys.stderr) + raise SystemExit(COOKIE_SKIP_EXIT_CODE) except KeyboardInterrupt: print("\nInterrupted.", file=sys.stderr) raise SystemExit(130) diff --git a/src/gyxx_flow/modules/content_marketing/xiaohongshu_comment_scraper.py b/src/gyxx_flow/modules/content_marketing/xiaohongshu_comment_scraper.py index 16f8e92..cc4e839 100644 --- a/src/gyxx_flow/modules/content_marketing/xiaohongshu_comment_scraper.py +++ b/src/gyxx_flow/modules/content_marketing/xiaohongshu_comment_scraper.py @@ -223,18 +223,15 @@ def maybe_wait_for_login( *, deadline: float | None = None, ) -> None: - login_deadline = time.monotonic() + max(0, login_timeout) - while time.monotonic() < login_deadline: - _check_note_deadline(deadline, "login wait") - try: - body = page.locator("body").inner_text(timeout=5000) - except Exception: - body = "" - if "扫码登录" not in body and "登录后" not in body and "验证码" not in body: - return - log("Xiaohongshu may require login. Please finish login in the opened Chrome window.") - page.wait_for_timeout(3000) - _check_note_deadline(deadline, "login wait") + # 登录统一由账号 vault 管理（gyxx accounts login xiaohongshu-content）， + # 由工作流执行前的 cookie 网关保证；运行中失效立即降级，不再等待人工扫码。 + try: + body = page.locator("body").inner_text(timeout=5000) + except Exception: + body = "" + if "扫码登录" in body or "登录后" in body or "验证码" in body: + log("Xiaohongshu login state missing; collecting degraded samples. " + "Run `gyxx accounts login xiaohongshu-content` to restore full comments.") def is_logged_in(page: Page) -> bool: @@ -751,55 +748,16 @@ def write_outputs(url: str, comments: list[dict[str, Any]], result: dict[str, An return json_path, csv_path -def login_only(login_timeout: int) -> int: - """只触发登录(headed),登录完成立刻退出。给 relogin_xiaohongshu.py 用。""" - def action(page: Page) -> None: - page.goto("https://www.xiaohongshu.com/", wait_until="domcontentloaded", timeout=90000) - page.wait_for_timeout(2500) - # 主动点登录按钮触发二维码 - try: - login_btn = page.get_by_text("登录", exact=True).first - if login_btn.count(): - login_btn.click(timeout=3000) - page.wait_for_timeout(1500) - except Exception: - pass - maybe_wait_for_login(page, login_timeout) - save_state(page) - - PROFILE_DIR.mkdir(parents=True, exist_ok=True) - with ScraplingBrowser( - headless=False, - real_chrome=True, - user_data_dir=str(PROFILE_DIR), - locale="zh-CN", - timezone_id="Asia/Shanghai", - timeout=90000, - network_idle=False, - disable_resources=False, - google_search=False, - page_setup=restore_cookies, - max_pages=1, - ) as browser: - browser.run_fetch_action("https://www.xiaohongshu.com/", action, wait=1000) - log("[login-only] 完成") - return 0 - - def main() -> int: parser = argparse.ArgumentParser(description="Scrape Xiaohongshu note comments with a real browser.") - parser.add_argument("url", help="Xiaohongshu explore/note URL (传 'login' 配合 --login-only)") - parser.add_argument("--login-timeout", type=int, default=300, help="等待扫码/登录秒数") + parser.add_argument("url", help="Xiaohongshu explore/note URL") + parser.add_argument("--login-timeout", type=int, default=300, + help="保留参数（登录等待已移除；未登录将直接降级采集）") parser.add_argument("--max-scrolls", type=int, default=120, help="最多滚动加载次数") parser.add_argument("--idle-rounds", type=int, default=8, help="连续几轮无新评论后停止") - parser.add_argument("--headless", action="store_true", help="无头模式；首次登录不要使用") - parser.add_argument("--login-only", action="store_true", - help="只触发登录(headed)然后退出,给 relogin_xiaohongshu 用") + parser.add_argument("--headless", action="store_true", help="无头模式") args = parser.parse_args() - if args.login_only: - return login_only(args.login_timeout) - comments, result = scrape_comments(args.url, args.login_timeout, args.max_scrolls, args.idle_rounds, args.headless) json_path, csv_path = write_outputs(args.url, comments, result) log(f"Final URL: {result.get('final_url', '')}") diff --git a/src/gyxx_flow/modules/content_marketing/xingtu_scraper_v2.py b/src/gyxx_flow/modules/content_marketing/xingtu_scraper_v2.py index 1d663bc..8b1e44b 100644 --- a/src/gyxx_flow/modules/content_marketing/xingtu_scraper_v2.py +++ b/src/gyxx_flow/modules/content_marketing/xingtu_scraper_v2.py @@ -33,6 +33,7 @@ from gyxx_flow.adapters.scrapling import ( BrowserTimeoutError, is_browser_timeout_error, ) +from gyxx_flow.adapters.browser import CookieExpiredError, cookie_expired_message from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE from gyxx_flow.modules.content_marketing.collection_completeness import ( BLOCKED_INPUT, @@ -537,183 +538,6 @@ def is_logged_in(page: Page) -> bool: return False -def open_scan_login(page: Page) -> None: - """Open the selected platform's login dialog without handling passwords.""" - if CONTENT_SOURCE == "buyin": - if not is_business_page(page) or "/dashboard/servicehall/daren-square" not in page.url: - try: - page.goto(CUSTOMER_LOGIN_URL, wait_until="domcontentloaded", timeout=60000) - page.wait_for_timeout(1500) - except Exception: - pass - for pattern in (r"^登录$", r"扫码登录|二维码登录|抖音登录|使用抖音登录"): - try: - button = page.get_by_text(re.compile(pattern)).first - if button.count() and button.is_visible(): - button.click(timeout=3000) - page.wait_for_timeout(800) - return - except Exception as exc: - reraise_if_browser_session_lost(exc) - return - - if "sso.oceanengine.com/xingtu/login" not in page.url: - try: - page.goto(CUSTOMER_LOGIN_URL, wait_until="domcontentloaded", timeout=60000) - page.wait_for_timeout(1500) - except Exception: - pass - - # The Douyin login icon on Xingtu's customer SSO page has a stable - # aweme.png source. Clicking it opens the QR authorization page directly. - try: - douyin = page.locator('img[src*="aweme.png"]').first - if douyin.count() and douyin.is_visible(): - douyin.click(timeout=3000) - page.wait_for_timeout(1500) - return - except Exception: - pass - - for pattern in (r"^登录$|立即登录", r"抖音扫码登录|扫码登录|二维码登录|扫码"): - try: - button = page.get_by_text(re.compile(pattern)).first - if button.count() and button.is_visible(): - button.click(timeout=2000) - page.wait_for_timeout(500) - except Exception: - pass - - -def is_scan_qr_page(page: Page) -> bool: - """Return True once Douyin's QR authorization page is visible. - - The QR page lives outside xingtu.cn. Treating that URL as a page that still - needs opening causes the login helper to navigate back and click again every - few seconds, continuously refreshing the QR code. - """ - url = page.url.lower() - if any(host in url for host in ("open.douyin.com", "sso.douyin.com")): - return True - for selector in ( - 'img[src*="qrcode"]', - 'img[src*="qr_code"]', - 'canvas', - ): - try: - locator = page.locator(selector).first - if locator.count() and locator.is_visible(): - return True - except Exception: - pass - return False - - -def resolve_live_page(page: Page, *, create_if_missing: bool = False) -> Page | None: - """Return a usable page after OAuth closes or replaces its QR tab.""" - try: - if not page.is_closed(): - return page - except Exception: - pass - - try: - context = page.context - for candidate in reversed(list(context.pages)): - try: - if not candidate.is_closed(): - return candidate - except Exception: - continue - if create_if_missing: - return context.new_page() - except Exception: - return None - return None - - -def find_logged_in_page(page: Page) -> Page | None: - """Find an authenticated business tab even while the OAuth tab stays open.""" - candidates: list[Page] = [] - try: - candidates.extend(reversed(list(page.context.pages))) - except Exception: - pass - candidates.append(page) - - seen: set[int] = set() - for candidate in candidates: - marker = id(candidate) - if marker in seen: - continue - seen.add(marker) - try: - if candidate.is_closed() or not is_business_page(candidate): - continue - if is_logged_in(candidate): - return candidate - except Exception: - continue - return None - - -def wait_for_scan_login(page: Page, timeout_seconds: int) -> Page: - log("Not logged in. 请在打开的 Chrome 中扫码登录。") - current_page = page - open_scan_login(current_page) - deadline = time.time() + timeout_seconds - next_login_click = time.time() + 5 - while time.time() < deadline: - authenticated = find_logged_in_page(current_page) - if authenticated is not None: - save_state(authenticated) - return authenticated - - replacement = resolve_live_page(current_page, create_if_missing=True) - if replacement is None: - raise RuntimeError("Xingtu browser context closed during OAuth login") - if replacement is not current_page: - current_page = replacement - try: - current_page.goto( - MARKET_URL, wait_until="domcontentloaded", timeout=60000 - ) - current_page.wait_for_timeout(2000) - except Exception: - pass - - if ( - CONTENT_SOURCE == "xingtu" - and "redirect_uri=/ad/creator/market" in current_page.url - and not is_scan_qr_page(current_page) - ): - try: - current_page.goto( - MARKET_URL, wait_until="domcontentloaded", timeout=60000 - ) - current_page.wait_for_timeout(2000) - except Exception: - pass - authenticated = find_logged_in_page(current_page) - if authenticated is not None: - save_state(authenticated) - return authenticated - if ( - time.time() >= next_login_click - and not is_scan_qr_page(current_page) - and not is_business_page(current_page) - ): - open_scan_login(current_page) - next_login_click = time.time() + 5 - try: - current_page.wait_for_timeout(2000) - except Exception: - # OAuth may close the QR tab after authorization. The next loop - # follows the surviving tab in the same browser context. - continue - raise TimeoutError(f"Login timed out after {timeout_seconds} seconds.") - - def ensure_login(page: Page, login_timeout: int) -> Page: page.goto(MARKET_URL, wait_until="domcontentloaded", timeout=60000) page.wait_for_timeout(3000) @@ -735,12 +559,11 @@ def ensure_login(page: Page, login_timeout: int) -> Page: ) log(f"[SKIPPED_COOKIE] {reason}") raise AcceptanceCookieSkip(reason) - page = wait_for_scan_login(page, login_timeout) - page.goto(MARKET_URL, wait_until="domcontentloaded", timeout=60000) - page.wait_for_timeout(3000) - if not is_logged_in(page): - page = wait_for_scan_login(page, login_timeout) - return page + # 登录统一由账号 vault 管理（gyxx accounts login douyin-shop）， + # 采集脚本不再内嵌扫码等待；运行中失效直接快速失败。 + reason = cookie_expired_message("douyin-shop", f"{CONTENT_SOURCE}达人广场") + log(f"[COOKIE_EXPIRED] {reason}") + raise CookieExpiredError(reason) def dismiss_popups(page: Page) -> None: @@ -3736,7 +3559,7 @@ def scrape_styles_with_fallback( summary_override=summaries, replace_existing=True, ) - except AcceptanceCookieSkip as exc: + except (AcceptanceCookieSkip, CookieExpiredError) as exc: log(f" [WARN] 抖店补采登录态不可用，保留星图结果: {exc}") _mark_fallback_not_processed( fallback_tasks, summaries, "buyin_cookie_skipped" @@ -3805,96 +3628,6 @@ def _force_cleanup_session(session) -> None: pass -def login_only(login_timeout: int) -> int: - """只触发登录(headed),登录完成立刻退出。给 weekly_relogin.py 用。""" - login_completed = False - - def action(page): - nonlocal login_completed - page = ensure_login(page, login_timeout) - if not is_logged_in(page): - raise RuntimeError(f"{CONTENT_SOURCE} 登录状态验证失败") - save_state(page) - login_completed = True - return page - - session = None - try: - with DynamicSession( - headless=False, - real_chrome=True, - user_data_dir=str(PROFILE_DIR), - locale="zh-CN", - timezone_id="Asia/Shanghai", - timeout=60000, - network_idle=False, - disable_resources=False, - google_search=False, - page_setup=restore_cookies, - max_pages=2, - ) as session: - session.fetch(MARKET_URL, page_action=action, wait=1000) - except Exception as exc: - err_msg = str(exc).lower() - if "closed" in err_msg or "context" in err_msg or "target" in err_msg: - log(f"[login-only] 浏览器会话关闭异常: {exc}") - else: - log(f"[login-only] 登录执行失败: {exc}") - finally: - _force_cleanup_session(session) - - # Douyin OAuth may close the only QR tab (and therefore the temporary - # browser context) after a successful mobile confirmation. The durable - # Chromium profile can already contain the new session even though the - # page_action never got a chance to export cookies. Reopen the same profile - # once, verify the selected business page, then export its cookies. - if not login_completed or not has_valid_login_cookie_file(): - log(f"[login-only] 扫码窗口已结束,重新打开{CONTENT_SOURCE}验证持久会话") - login_completed = verify_persisted_profile_login() - if not login_completed or not has_valid_login_cookie_file(): - log("[login-only] 登录未完成或Cookie缺少有效会话") - return 1 - log("[login-only] 登录完成,退出") - return 0 - - -def verify_persisted_profile_login() -> bool: - """Reopen the persistent profile after OAuth closes its only tab.""" - verified = False - - def action(page: Page): - nonlocal verified - page.goto(MARKET_URL, wait_until="domcontentloaded", timeout=60000) - page.wait_for_timeout(3000) - if not is_logged_in(page): - raise RuntimeError(f"persisted {CONTENT_SOURCE} profile is not authenticated") - save_state(page) - verified = True - return page - - session = None - try: - with DynamicSession( - headless=False, - real_chrome=True, - user_data_dir=str(PROFILE_DIR), - locale="zh-CN", - timezone_id="Asia/Shanghai", - timeout=60000, - network_idle=False, - disable_resources=False, - google_search=False, - page_setup=restore_cookies, - max_pages=1, - ) as session: - session.fetch(MARKET_URL, page_action=action, wait=1000) - except Exception as exc: - log(f"[login-only] 持久会话验证失败: {exc}") - finally: - _force_cleanup_session(session) - return verified and has_valid_login_cookie_file() - - def load_retry_scope(summary_path: Path, reason: str) -> dict[int, set[str]]: """Load exact style/record pairs for a reason-limited retry.""" try: @@ -3926,8 +3659,6 @@ def main() -> int: parser.add_argument("--login-timeout", type=int, default=300) parser.add_argument("--dry-run", action="store_true") parser.add_argument("--data-dir", default=str(PATHS.normalized_root)) - parser.add_argument("--login-only", action="store_true", - help="只触发登录(headed),登录完成立刻退出,给 weekly_relogin 用") parser.add_argument( "--buyin", action="store_true", @@ -3952,12 +3683,6 @@ def main() -> int: "buyin" if args.buyin else os.environ.get("GYXX_CONTENT_SOURCE", "xingtu") ) - if args.login_only: - try: - return login_only(args.login_timeout) - except AcceptanceCookieSkip: - return COOKIE_SKIP_EXIT_CODE - mapping = load_mapping(self_operated=args.self_operated) styles = mapping["tables"] if args.style: @@ -4038,7 +3763,7 @@ def main() -> int: max_age_days=args.max_age_days, no_retry=args.no_retry, ) - except AcceptanceCookieSkip: + except (AcceptanceCookieSkip, CookieExpiredError): return COOKIE_SKIP_EXIT_CODE persist_summaries(all_summaries) for s, summary in zip(styles, all_summaries): diff --git a/src/gyxx_flow/modules/product_commerce/vendors/dy-data-flow/dy_store_competitor_store_scraping.py b/src/gyxx_flow/modules/product_commerce/vendors/dy-data-flow/dy_store_competitor_store_scraping.py index 1ea3298..7a35932 100644 --- a/src/gyxx_flow/modules/product_commerce/vendors/dy-data-flow/dy_store_competitor_store_scraping.py +++ b/src/gyxx_flow/modules/product_commerce/vendors/dy-data-flow/dy_store_competitor_store_scraping.py @@ -223,23 +223,16 @@ def _check_login_status(page) -> bool: def _ensure_login(page): - print("\n" + "=" * 60) - print("【请登录】请在弹出的浏览器窗口中完成登录") - print("提示：扫码/短信/密码均可，登录成功后页面会自动跳转") - print("=" * 60 + "\n") - - for i in range(LOGIN_TIMEOUT): - page = _active_page(page) - if _check_login_status(page): - print(f"[OK] Login success! URL: {page.url}") - page.wait_for_timeout(3000) - return page - if i % 5 == 0: - print(f"[WAIT] Waiting for login... {i}s / {LOGIN_TIMEOUT}s") - time.sleep(1) - - print(f"[ERR] Login timeout ({LOGIN_TIMEOUT}s)") - return None + # 登录统一由账号 vault 管理（gyxx accounts login douyin-shop）， + # 采集脚本不再弹出浏览器等待人工扫码/短信登录。 + print( + "[COOKIE_EXPIRED] 抖音罗盘登录态已失效：" + "请执行 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + raise RuntimeError( + "抖音罗盘登录态已失效；请执行 uv run gyxx accounts login douyin-shop " + "刷新账号登录态后重跑" + ) def _select_menu_and_date(page) -> None: @@ -1927,12 +1920,8 @@ def _page_automation(page) -> None: # 1. Check and wait for login if not _check_login_status(page): - logged_page = _ensure_login(page) - if not logged_page: - print("[ERR] Login failed, will try save cookies anyway...") - else: - page = logged_page - _maximize_page(page) + # 登录态失效时 _ensure_login 会直接抛 RuntimeError + _ensure_login(page) # 2. Save state after login (best effort) _save_browser_state(page, "after login") diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/dy_negative_reviews.py b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_negative_reviews.py index fb44786..2af1786 100644 --- a/src/gyxx_flow/modules/shop_intelligence/collectors/dy_negative_reviews.py +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_negative_reviews.py @@ -226,23 +226,16 @@ def ensure_login(page: Page, *, headless: bool, login_timeout: int) -> None: print("[SKIPPED_COOKIE] 抖店会话跳转到了登录页") raise DouyinSessionExpiredError("抖店会话跳转到了登录页") - if headless: - raise RuntimeError( - "抖音 Cookie 已失效且当前为无头模式，无法扫码登录；" - "请先执行 uv run gyxx accounts login douyin-shop 刷新账号登录态" - ) - - print("=" * 60) - print("【请登录】请在弹出的浏览器窗口中完成抖店登录") - print("=" * 60) - deadline = time.monotonic() + login_timeout - while time.monotonic() < deadline: - if not douyin_common.is_login_page(page): - print(f" [OK] 登录成功: {page.url}") - page.wait_for_timeout(3_000) - return - page.wait_for_timeout(1_000) - raise RuntimeError(f"登录在 {login_timeout} 秒内未完成，当前页面: {page.url}") + # 登录统一由账号 vault 管理（gyxx accounts login douyin-shop）， + # 采集脚本不再弹出浏览器等待人工扫码/验证码。 + print( + "[COOKIE_EXPIRED] 抖店 Cookie 已失效：" + "请执行 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + raise DouyinSessionExpiredError( + "抖音 Cookie 已失效；请执行 uv run gyxx accounts login douyin-shop " + "刷新账号登录态后重跑" + ) class DouyinSessionExpiredError(RuntimeError): diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py index 47b4c40..e468344 100644 --- a/src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py @@ -262,23 +262,16 @@ def _check_login_status(page) -> bool: def _ensure_login(page): - print("\n" + "=" * 60) - print("【请登录】请在弹出的浏览器窗口中完成登录") - print("提示：扫码/短信/密码均可，登录成功后页面会自动跳转") - print("=" * 60 + "\n") - - for i in range(LOGIN_TIMEOUT): - page = _active_page(page) - if _check_login_status(page): - print(f"[OK] Login success! URL: {page.url}") - page.wait_for_timeout(3000) - return page - if i % 5 == 0: - print(f"[WAIT] Waiting for login... {i}s / {LOGIN_TIMEOUT}s") - time.sleep(1) - - print(f"[ERR] Login timeout ({LOGIN_TIMEOUT}s)") - return None + # 登录统一由账号 vault 管理（gyxx accounts login douyin-shop）， + # 采集脚本不再弹出浏览器等待人工扫码/短信登录。 + print( + "[COOKIE_EXPIRED] 抖音罗盘登录态已失效：" + "请执行 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + raise BrowserSessionExpiredError( + "抖音罗盘登录态已失效；请执行 uv run gyxx accounts login douyin-shop " + "刷新账号登录态后重跑" + ) def _select_menu_and_date(page) -> None: @@ -2325,12 +2318,8 @@ def _page_automation_impl(page) -> None: # 1. Check and wait for login if not _check_login_status(page): - logged_page = _ensure_login(page) - if not logged_page: - print("[ERR] Login failed, will try save cookies anyway...") - else: - page = logged_page - _maximize_page(page) + # If the login state is expired, _ensure_login will directly raise BrowserSessionExpiredError + _ensure_login(page) # 2. Save state after login (best effort) _save_browser_state(page, "after login") diff --git a/src/gyxx_flow/notification_routing.py b/src/gyxx_flow/notification_routing.py index 3fb5e28..8a93f75 100644 --- a/src/gyxx_flow/notification_routing.py +++ b/src/gyxx_flow/notification_routing.py @@ -52,7 +52,6 @@ MAX_NOTIFICATION_BINDINGS_PER_PERSON = 16 NOTIFICATION_CAPABILITIES: Mapping[str, str] = MappingProxyType( { "content.marketing_report.daily": "正式执行且启用发送时投递最终营销日报。", - "content.relogin.weekly": "刷新期间投递各平台扫码二维码，并在平台失败时投递失败汇总。", "product.alert.daily": "命中销量下滑规则且尚未重复通知时投递告警。", "product.market_rank": "存在本次新生成的平台报告链接且尚未重复通知时投递。", "product.online_stock_alert.daily": "线上商品当前库存低于 3 时向配置接收人发送库存预警。", diff --git a/tests/modules/content_marketing/test_comment_batch_recovery.py b/tests/modules/content_marketing/test_comment_batch_recovery.py index 3c18eb0..43df2ad 100644 --- a/tests/modules/content_marketing/test_comment_batch_recovery.py +++ b/tests/modules/content_marketing/test_comment_batch_recovery.py @@ -21,20 +21,11 @@ def test_page_crash_requires_a_fresh_browser(): @pytest.mark.parametrize("batch", [xhs_batch, douyin_batch]) -def test_non_acceptance_relogin_runs_platform_script(monkeypatch, batch): - class Policy: - enabled = False +def test_batch_tools_no_longer_embed_relogin(batch): + """扫码重登已统一到账号 vault（gyxx accounts login），工具内不再提供。""" - calls = [] - monkeypatch.setattr(batch, "current_acceptance_policy", lambda: Policy()) - monkeypatch.setattr( - batch.subprocess, - "call", - lambda command, **kwargs: calls.append((command, kwargs)) or 0, - ) - - assert batch.do_relogin() is True - assert calls[0][0][1] == str(batch.RELOGIN_SCRIPT) + assert not hasattr(batch, "do_relogin") + assert not hasattr(batch, "RELOGIN_SCRIPT") def test_comment_checkpoint_matches_both_note_id_and_url(tmp_path): diff --git a/tests/modules/content_marketing/test_friday_relogin_parallel.py b/tests/modules/content_marketing/test_friday_relogin_parallel.py deleted file mode 100644 index ac565b9..0000000 --- a/tests/modules/content_marketing/test_friday_relogin_parallel.py +++ /dev/null @@ -1,569 +0,0 @@ -from types import SimpleNamespace - -import pytest - -from gyxx_flow.modules.content_marketing.data.tools import ( - friday_relogin_parallel as relogin, -) -from gyxx_flow.modules.content_marketing.data.tools import relogin_transaction - - -def test_five_expected_platforms_are_configured(): - assert set(relogin.PLATFORMS) == { - "bilibili", "douyin", "pgy", "xingtu", "xiaohongshu" - } - assert relogin.PLATFORMS["xingtu"].required_cookie == "sessionid" - assert relogin.PLATFORMS["douyin"].required_cookie == "sessionid" - - -def test_manual_relogin_can_select_only_xingtu(): - assert relogin.resolve_platform_names(["xingtu"]) == ["xingtu"] - assert relogin.resolve_platform_names(None) == list(relogin.PLATFORMS) - - -def test_default_screenshot_recipients_are_routed_by_platform(): - assert relogin.DEFAULT_RECIPIENTS == { - "pgy": "ou_24cc944d6e43c69c59d6560ad4e2ae6e", - "xingtu": "ou_24cc944d6e43c69c59d6560ad4e2ae6e", - "douyin": "ou_7ad5fc8012e2f741afc5346e05ffd447", - "bilibili": "ou_7ad5fc8012e2f741afc5346e05ffd447", - "xiaohongshu": "ou_7ad5fc8012e2f741afc5346e05ffd447", - } - assert relogin.recipient_for_platform("pgy") == "ou_24cc944d6e43c69c59d6560ad4e2ae6e" - assert relogin.recipient_for_platform("douyin") == "ou_7ad5fc8012e2f741afc5346e05ffd447" - assert relogin.recipient_for_platform("bilibili") == "ou_7ad5fc8012e2f741afc5346e05ffd447" - assert relogin.recipient_for_platform("pgy", "ou_override") == "ou_override" - - -def test_failed_platforms_are_grouped_by_their_responsible_recipient(): - grouped = relogin.group_platforms_by_recipient( - ["pgy", "douyin", "bilibili", "xingtu", "xiaohongshu"] - ) - assert grouped == { - "ou_24cc944d6e43c69c59d6560ad4e2ae6e": ["pgy", "xingtu"], - "ou_7ad5fc8012e2f741afc5346e05ffd447": [ - "douyin", "bilibili", "xiaohongshu" - ], - } - - -def test_legacy_route_keeps_the_existing_per_platform_recipients(monkeypatch): - observed = {} - - def fake_resolver(workflow_id, defaults): - observed["workflow_id"] = workflow_id - observed["defaults"] = defaults - return relogin.ResolvedNotificationRoute( - configured=False, - enabled=True, - app_profile=None, - open_ids=tuple(defaults), - ) - - monkeypatch.setattr(relogin, "resolve_notification_route", fake_resolver) - - route = relogin.resolve_weekly_notification_route() - - assert observed == { - "workflow_id": "content.relogin.weekly", - "defaults": ( - "ou_24cc944d6e43c69c59d6560ad4e2ae6e", - "ou_7ad5fc8012e2f741afc5346e05ffd447", - ), - } - assert relogin.recipients_for_platform("pgy", route) == ( - "ou_24cc944d6e43c69c59d6560ad4e2ae6e", - ) - assert relogin.recipients_for_platform("douyin", route) == ( - "ou_7ad5fc8012e2f741afc5346e05ffd447", - ) - - -def test_disabled_route_still_runs_relogin_without_sending(monkeypatch): - route = relogin.ResolvedNotificationRoute( - configured=True, - enabled=False, - app_profile="hermes-analyzer", - open_ids=(), - ) - rounds = [] - - monkeypatch.setattr( - relogin, - "current_acceptance_policy", - lambda: SimpleNamespace(enabled=False), - ) - monkeypatch.setattr( - relogin, - "resolve_weekly_notification_route", - lambda _override=None: route, - ) - - def fake_round_factory(_args, resolved_route, notification_failures): - assert resolved_route is route - assert notification_failures == [] - - def run_round(names, attempt): - rounds.append((tuple(names), attempt)) - return {name: True for name in names} - - return run_round - - monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory) - monkeypatch.setattr( - relogin, - "_send_lark", - lambda *_args, **_kwargs: (_ for _ in ()).throw( - AssertionError("disabled route must not send Lark messages") - ), - ) - monkeypatch.setattr( - relogin.sys, - "argv", - ["friday_relogin_parallel.py", "--platform", "pgy"], - ) - - assert relogin.main() == 0 - assert rounds == [(("pgy",), 1)] - - -def test_enabled_route_sends_qr_text_and_image_to_every_recipient( - monkeypatch, - tmp_path, -): - route = relogin.ResolvedNotificationRoute( - configured=True, - enabled=True, - app_profile="hermes-analyzer", - open_ids=("ou_first123", "ou_second456"), - ) - sent = [] - image = tmp_path / "qrcode.png" - - monkeypatch.setattr( - relogin, - "_send_lark", - lambda recipient, text=None, image=None, app_profile=None: sent.append( - (recipient, text, image, app_profile) - ), - ) - - failures = relogin._send_qr_notification( - route, - "pgy", - override=None, - text="请扫码", - image=image, - ) - - assert failures == () - assert sent == [ - ("ou_first123", "请扫码", None, "hermes-analyzer"), - ("ou_first123", None, image, "hermes-analyzer"), - ("ou_second456", "请扫码", None, "hermes-analyzer"), - ("ou_second456", None, image, "hermes-analyzer"), - ] - assert relogin.group_platforms_by_notification_recipient( - ["pgy", "douyin"], route - ) == { - "ou_first123": ["pgy", "douyin"], - "ou_second456": ["pgy", "douyin"], - } - - -def test_qr_delivery_aggregates_errors_and_attempts_every_recipient( - monkeypatch, - tmp_path, -): - route = relogin.ResolvedNotificationRoute( - configured=True, - enabled=True, - app_profile="hermes-analyzer", - open_ids=("ou_first123", "ou_second456"), - ) - attempts = [] - image = tmp_path / "qrcode.png" - - def fake_send(recipient, text=None, image=None, app_profile=None): - stage = "qr_text" if text is not None else "qr_image" - attempts.append((recipient, stage, app_profile)) - if (recipient, stage) == ("ou_first123", "qr_text"): - raise RuntimeError("first text rejected") - if (recipient, stage) == ("ou_second456", "qr_image"): - raise RuntimeError("second image rejected") - - monkeypatch.setattr(relogin, "_send_lark", fake_send) - - failures = relogin._send_qr_notification( - route, - "pgy", - override=None, - text="请扫码", - image=image, - ) - - assert attempts == [ - ("ou_first123", "qr_text", "hermes-analyzer"), - ("ou_first123", "qr_image", "hermes-analyzer"), - ("ou_second456", "qr_text", "hermes-analyzer"), - ("ou_second456", "qr_image", "hermes-analyzer"), - ] - assert failures == ( - relogin.NotificationDeliveryFailure( - stage="qr_text", - platform="pgy", - recipient="ou_first123", - detail="first text rejected", - ), - relogin.NotificationDeliveryFailure( - stage="qr_image", - platform="pgy", - recipient="ou_second456", - detail="second image rejected", - ), - ) - - -def test_failure_summary_aggregates_every_recipient_error(monkeypatch): - route = relogin.ResolvedNotificationRoute( - configured=True, - enabled=True, - app_profile="hermes-analyzer", - open_ids=("ou_first123", "ou_second456"), - ) - attempts = [] - - def fail_send(recipient, **_kwargs): - attempts.append(recipient) - raise RuntimeError(f"rejected {recipient}") - - monkeypatch.setattr(relogin, "_send_lark", fail_send) - - failures = relogin._send_failure_summaries( - route, - ["pgy", "douyin"], - override=None, - max_attempts=3, - ) - - assert attempts == ["ou_first123", "ou_second456"] - assert [failure.stage for failure in failures] == [ - "failure_summary", - "failure_summary", - ] - assert [failure.recipient for failure in failures] == attempts - assert [failure.platform for failure in failures] == [ - "pgy,douyin", - "pgy,douyin", - ] - - -def test_main_returns_failure_and_reports_all_notification_errors( - monkeypatch, - capsys, -): - route = relogin.ResolvedNotificationRoute( - configured=True, - enabled=True, - app_profile="hermes-analyzer", - open_ids=("ou_first123", "ou_second456"), - ) - monkeypatch.setattr( - relogin, - "current_acceptance_policy", - lambda: SimpleNamespace(enabled=False), - ) - monkeypatch.setattr( - relogin, - "resolve_weekly_notification_route", - lambda _override=None: route, - ) - - def fake_round_factory(_args, _route, notification_failures): - notification_failures.extend( - ( - relogin.NotificationDeliveryFailure( - "qr_text", "pgy", "ou_first123", "text rejected" - ), - relogin.NotificationDeliveryFailure( - "qr_image", "pgy", "ou_second456", "image rejected" - ), - ) - ) - return lambda names, _attempt: {name: True for name in names} - - monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory) - monkeypatch.setattr( - relogin.sys, - "argv", - ["friday_relogin_parallel.py", "--platform", "pgy"], - ) - - assert relogin.main() == 2 - output = capsys.readouterr().out - assert "Final relogin status: {'pgy': True}" in output - assert "Final notification status: failed (2 deliveries)" in output - assert "recipient=ou_first123 error=text rejected" in output - assert "recipient=ou_second456 error=image rejected" in output - - -def test_round_completes_relogin_after_notification_failure(monkeypatch, tmp_path): - route = relogin.ResolvedNotificationRoute( - configured=True, - enabled=True, - app_profile="hermes-analyzer", - open_ids=("ou_first123", "ou_second456"), - ) - args = SimpleNamespace( - round_timeout=10, - screenshot_delay=0, - no_send=False, - recipient=None, - max_attempts=1, - ) - failures = [] - attempts = [] - waits = [] - - class Process: - pid = 1234 - - def wait(self, timeout): - waits.append(timeout) - return 0 - - monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process()) - monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {}) - monkeypatch.setattr( - relogin, - "binding_from_environment", - lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"), - ) - monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None) - monkeypatch.setattr(relogin, "_login_windows", lambda: {"pgy": 101}) - monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None) - monkeypatch.setattr(relogin, "_capture_window", lambda _hwnd, _output: None) - monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: True) - - def fake_send(recipient, text=None, image=None, **_kwargs): - stage = "qr_text" if text is not None else "qr_image" - attempts.append((recipient, stage)) - if (recipient, stage) == ("ou_first123", "qr_text"): - raise RuntimeError("temporary send failure") - - monkeypatch.setattr(relogin, "_send_lark", fake_send) - - run_round = relogin._run_round_factory(args, route, failures) - - assert run_round(["pgy"], 1) == {"pgy": True} - assert waits == [100] - assert attempts == [ - ("ou_first123", "qr_text"), - ("ou_first123", "qr_image"), - ("ou_second456", "qr_text"), - ("ou_second456", "qr_image"), - ] - assert failures == [ - relogin.NotificationDeliveryFailure( - stage="qr_text", - platform="pgy", - recipient="ou_first123", - detail="temporary send failure", - ) - ] - - -@pytest.mark.parametrize( - ("returncode", "cookie_valid", "expected_status", "expected_failure_count"), - ( - (0, True, True, 0), - (1, False, False, 4), - ), -) -def test_missing_qr_window_is_classified_after_relogin_result( - monkeypatch, - tmp_path, - returncode, - cookie_valid, - expected_status, - expected_failure_count, -): - route = relogin.ResolvedNotificationRoute( - configured=True, - enabled=True, - app_profile="hermes-analyzer", - open_ids=("ou_first123", "ou_second456"), - ) - args = SimpleNamespace( - round_timeout=10, - screenshot_delay=0, - no_send=False, - recipient=None, - max_attempts=1, - ) - failures = [] - - class Process: - pid = 1234 - - def wait(self, timeout): - assert timeout == 100 - return returncode - - monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process()) - monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {}) - monkeypatch.setattr( - relogin, - "binding_from_environment", - lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"), - ) - monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None) - monkeypatch.setattr(relogin, "_login_windows", lambda: {}) - monkeypatch.setattr(relogin, "_window_for_process", lambda _pid: None) - monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None) - monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: cookie_valid) - monkeypatch.setattr( - relogin, - "_send_lark", - lambda *_args, **_kwargs: pytest.fail( - "no QR window must not start a send before child classification" - ), - ) - - run_round = relogin._run_round_factory(args, route, failures) - - assert run_round(["pgy"], 1) == {"pgy": expected_status} - assert len(failures) == expected_failure_count - assert {failure.stage for failure in failures} <= {"qr_text", "qr_image"} - assert all( - failure.detail == "no visible QR window and relogin did not succeed" - for failure in failures - ) - - -def test_lark_send_uses_shared_analyzer_user_sender_for_text_and_image( - monkeypatch, - tmp_path, -): - calls = [] - image = tmp_path / "qrcode.png" - image.write_bytes(b"png") - - monkeypatch.setattr(relogin, "ANALYZER_DIR", tmp_path) - monkeypatch.setenv("HERMES_ANALYZER_TOKEN", "must-not-leak") - - def fake_send(**kwargs): - calls.append(kwargs) - return {"message_id": f"om_{len(calls)}"} - - monkeypatch.setattr(relogin, "send_lark_bot_message", fake_send) - - receipts = relogin._send_lark( - "ou_recipient123", - text="登录提醒", - image=image, - ) - - assert [receipt["message_id"] for receipt in receipts] == ["om_1", "om_2"] - assert calls[0]["user_id"] == "ou_recipient123" - assert calls[0]["text"] == "登录提醒" - assert "image" not in calls[0] - assert calls[1]["user_id"] == "ou_recipient123" - assert calls[1]["image"] == image - assert "text" not in calls[1] - assert all(call["profile"] == "hermes-analyzer" for call in calls) - assert all("HERMES_ANALYZER_TOKEN" not in call["env"] for call in calls) - - -def test_login_windows_are_classified_by_title(): - assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili" - assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy" - assert relogin._platform_from_window_title("抖音授权 - Google Chrome") == "xingtu" - assert relogin._platform_from_window_title("抖音 - 记录美好生活") == "douyin" - assert relogin._platform_from_window_title("小红书 - 你的生活兴趣社区") == "xiaohongshu" - assert relogin._platform_from_window_title("Codex") is None - - -def test_retry_only_failed_platforms_for_at_most_three_rounds(): - calls = [] - - def fake_round(names, attempt): - calls.append((tuple(names), attempt)) - if attempt == 1: - return {name: name != "xingtu" for name in names} - if attempt == 2: - return {name: False for name in names} - return {name: True for name in names} - - result = relogin.run_with_retries( - ["bilibili", "pgy", "xingtu", "xiaohongshu"], - max_attempts=3, - run_round=fake_round, - ) - - assert calls == [ - (("bilibili", "pgy", "xingtu", "xiaohongshu"), 1), - (("xingtu",), 2), - (("xingtu",), 3), - ] - assert all(result.values()) - - -def test_stops_after_third_failed_round(): - calls = [] - - def always_fail(names, attempt): - calls.append(attempt) - return {name: False for name in names} - - result = relogin.run_with_retries( - ["pgy"], max_attempts=3, run_round=always_fail - ) - - assert calls == [1, 2, 3] - assert result == {"pgy": False} - - -def test_timeout_terminates_the_complete_process_tree(monkeypatch): - calls = [] - - class Process: - pid = 1234 - - def terminate(self): - calls.append("fallback-terminate") - - def wait(self, timeout): - calls.append(("wait", timeout)) - - monkeypatch.setattr( - relogin.subprocess, - "run", - lambda command, **kwargs: calls.append((command, kwargs)), - ) - - relogin._terminate_process_tree(Process()) - - assert calls[0][0] == ["taskkill", "/PID", "1234", "/T", "/F"] - assert calls[1] == ("wait", 10) - - -def test_transaction_rolls_back_files_and_profile(tmp_path): - cookie = tmp_path / "cookies.json" - storage = tmp_path / "storage.json" - profile = tmp_path / "profile" - cookie.write_text("old-cookie", encoding="utf-8") - storage.write_text("old-storage", encoding="utf-8") - profile.mkdir() - (profile / "old.txt").write_text("old-profile", encoding="utf-8") - - state = relogin_transaction.begin((cookie, storage), profile) - cookie.write_text("partial", encoding="utf-8") - profile.mkdir() - (profile / "partial.txt").write_text("partial", encoding="utf-8") - relogin_transaction.rollback(state) - - assert cookie.read_text(encoding="utf-8") == "old-cookie" - assert storage.read_text(encoding="utf-8") == "old-storage" - assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile" - assert not (profile / "partial.txt").exists() diff --git a/tests/modules/content_marketing/test_relogin_rollback.py b/tests/modules/content_marketing/test_relogin_rollback.py deleted file mode 100644 index 35df412..0000000 --- a/tests/modules/content_marketing/test_relogin_rollback.py +++ /dev/null @@ -1,494 +0,0 @@ - -import pytest - -from gyxx_flow.modules.content_marketing import ( - pgy_xhs_scraper_v2, - xingtu_scraper_v2, -) -from gyxx_flow.modules.content_marketing.data.tools import ( - relogin_douyin, - relogin_pgy, - relogin_xingtu, -) - - -@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu]) -def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module): - cookie = tmp_path / "cookies.json" - storage = tmp_path / "storage_state.json" - profile = tmp_path / "profile" - cookie.write_text("old-cookie", encoding="utf-8") - storage.write_text("old-storage", encoding="utf-8") - profile.mkdir() - (profile / "state.txt").write_text("old-profile", encoding="utf-8") - - monkeypatch.setattr(module, "COOKIE_FILE", cookie) - monkeypatch.setattr(module, "STATE_FILE", storage) - monkeypatch.setattr(module, "PROFILE_DIR", profile) - - cookie_backup, profile_backup, state_backup = module.reset_state() - assert not cookie.exists() - assert not storage.exists() - assert not profile.exists() - - cookie.write_text("partial-new-cookie", encoding="utf-8") - storage.write_text("partial-new-storage", encoding="utf-8") - profile.mkdir() - (profile / "partial.txt").write_text("partial", encoding="utf-8") - - module.restore_previous_state(cookie_backup, profile_backup, state_backup) - - assert cookie.read_text(encoding="utf-8") == "old-cookie" - assert storage.read_text(encoding="utf-8") == "old-storage" - assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile" - assert not (profile / "partial.txt").exists() - - -@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu]) -def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module): - cookie = tmp_path / "cookies.json" - storage = tmp_path / "storage_state.json" - profile = tmp_path / "profile" - cookie.write_text("old-cookie", encoding="utf-8") - storage.write_text("old-storage", encoding="utf-8") - profile.mkdir() - - monkeypatch.setattr(module, "COOKIE_FILE", cookie) - monkeypatch.setattr(module, "STATE_FILE", storage) - monkeypatch.setattr(module, "PROFILE_DIR", profile) - - cookie_backup, profile_backup, state_backup = module.reset_state() - cookie.write_text("new-cookie", encoding="utf-8") - storage.write_text("new-storage", encoding="utf-8") - profile.mkdir() - - module.finish_successful_relogin(profile_backup) - - assert cookie.read_text(encoding="utf-8") == "new-cookie" - assert storage.read_text(encoding="utf-8") == "new-storage" - assert cookie_backup.exists() - assert state_backup.exists() - assert not profile_backup.exists() - - -def test_xingtu_scan_login_opens_customer_sso_and_clicks_douyin(): - class FakeLocator: - def __init__(self): - self.first = self - self.clicked = False - self.evaluated = False - - def count(self): - return 1 - - def is_visible(self): - return True - - def click(self, timeout): - self.clicked = True - - def evaluate(self, script): - assert script == "e => e.click()" - self.evaluated = True - - class FakePage: - url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market" - - def __init__(self): - self.goto_url = None - self.selectors = [] - self.douyin = FakeLocator() - - def goto(self, url, **kwargs): - self.goto_url = url - self.url = url - - def wait_for_timeout(self, _milliseconds): - pass - - def locator(self, selector): - self.selectors.append(selector) - return self.douyin - - def get_by_text(self, _pattern): - return FakeLocator() - - page = FakePage() - xingtu_scraper_v2.open_scan_login(page) - - assert page.goto_url == xingtu_scraper_v2.CUSTOMER_LOGIN_URL - assert 'img[src*="aweme.png"]' in page.selectors - assert page.douyin.clicked - - -def test_xingtu_index_page_is_recognized_as_logged_in(): - class Locator: - @property - def first(self): - return self - - def wait_for(self, timeout): - assert timeout == 1200 - - class Page: - url = "https://www.xingtu.cn/ad/creator/index" - - def get_by_text(self, _text, exact=False): - return Locator() - - assert xingtu_scraper_v2.is_logged_in(Page()) is True - - -def test_xingtu_index_page_with_xingtu_session_is_logged_in_without_market_ui(): - class Locator: - @property - def first(self): - return self - - def count(self): - return 0 - - def wait_for(self, timeout): - raise xingtu_scraper_v2.BrowserTimeoutError("not on market page") - - def is_visible(self): - return False - - class Context: - def cookies(self): - return [ - { - "name": "sessionid", - "value": "fresh", - "domain": ".xingtu.cn", - } - ] - - class Page: - url = "https://www.xingtu.cn/ad/creator/index" - context = Context() - - def get_by_text(self, *_args, **_kwargs): - return Locator() - - def locator(self, *_args, **_kwargs): - return Locator() - - assert xingtu_scraper_v2.is_logged_in(Page()) is True - - -def test_xingtu_index_page_without_xingtu_session_is_not_logged_in(): - class Locator: - @property - def first(self): - return self - - def count(self): - return 0 - - def wait_for(self, timeout): - raise xingtu_scraper_v2.BrowserTimeoutError("missing") - - def is_visible(self): - return False - - class Context: - def cookies(self): - return [ - { - "name": "sessionid", - "value": "douyin-only", - "domain": ".douyin.com", - } - ] - - class Page: - url = "https://www.xingtu.cn/ad/creator/index" - context = Context() - - def get_by_text(self, *_args, **_kwargs): - return Locator() - - def locator(self, *_args, **_kwargs): - return Locator() - - assert xingtu_scraper_v2.is_logged_in(Page()) is False - - -def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie(): - class Context: - def cookies(self): - return [{"name": "sessionid", "value": "stale"}] - - class Page: - url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market" - context = Context() - - def get_by_text(self, *_args, **_kwargs): - raise AssertionError("public homepage must be rejected before marker checks") - - assert xingtu_scraper_v2.is_logged_in(Page()) is False - - -def test_xingtu_market_page_without_business_ui_rejects_stale_cookie(): - class Locator: - @property - def first(self): - return self - - def wait_for(self, **_kwargs): - raise xingtu_scraper_v2.BrowserTimeoutError("missing") - - def count(self): - return 0 - - class Context: - def cookies(self): - return [{"name": "sessionid", "value": "stale"}] - - class Page: - url = "https://www.xingtu.cn/ad/creator/market" - context = Context() - - def get_by_text(self, *_args, **_kwargs): - return Locator() - - def locator(self, *_args, **_kwargs): - return Locator() - - assert xingtu_scraper_v2.is_logged_in(Page()) is False - - -def test_xingtu_cookie_validation_requires_real_session_cookie(tmp_path): - cookie = tmp_path / "xingtu.json" - cookie.write_text('[{"name":"other","value":"1"}]', encoding="utf-8") - assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is False - - cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8") - assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is True - - -def test_xingtu_login_only_does_not_report_success_when_page_action_was_swallowed( - tmp_path, monkeypatch -): - cookie = tmp_path / "xingtu.json" - cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8") - - class Session: - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def fetch(self, *_args, **_kwargs): - # Scrapling can log a page_action error and still return a response. - return object() - - monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie) - monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session()) - monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None) - - assert xingtu_scraper_v2.login_only(1) == 1 - - -def test_xingtu_login_only_verifies_persisted_profile_after_oauth_context_closes( - tmp_path, monkeypatch -): - cookie = tmp_path / "xingtu.json" - - class Session: - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def fetch(self, *_args, **_kwargs): - raise RuntimeError("Target page, context or browser has been closed") - - monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie) - monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session()) - monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None) - - def verify(): - cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8") - return True - - monkeypatch.setattr(xingtu_scraper_v2, "verify_persisted_profile_login", verify) - - assert xingtu_scraper_v2.login_only(1) == 0 - - -def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monkeypatch): - class Context: - def __init__(self): - self.pages = [] - - def new_page(self): - raise AssertionError("a surviving replacement page should be reused") - - class Page: - def __init__(self, context, url, *, closes_on_wait=False): - self.context = context - self.url = url - self.closed = False - self.closes_on_wait = closes_on_wait - - def is_closed(self): - return self.closed - - def wait_for_timeout(self, _milliseconds): - if self.closes_on_wait: - self.closed = True - - def goto(self, url, **_kwargs): - self.url = url - - context = Context() - qr_page = Page( - context, - "https://open.douyin.com/platform/oauth/pc/auth", - closes_on_wait=True, - ) - market_page = Page(context, xingtu_scraper_v2.MARKET_URL) - context.pages = [qr_page, market_page] - saved = [] - - monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None) - monkeypatch.setattr( - xingtu_scraper_v2, "is_scan_qr_page", lambda page: page is qr_page - ) - monkeypatch.setattr( - xingtu_scraper_v2, "is_logged_in", lambda page: page is market_page - ) - monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page)) - - result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2) - - assert result is market_page - assert saved == [market_page] - - -def test_xingtu_scan_login_detects_business_tab_while_qr_tab_stays_open(monkeypatch): - class Context: - def __init__(self): - self.pages = [] - - class Page: - def __init__(self, context, url): - self.context = context - self.url = url - - def is_closed(self): - return False - - def wait_for_timeout(self, _milliseconds): - raise AssertionError("authenticated tab should be detected immediately") - - context = Context() - qr_page = Page(context, "https://open.douyin.com/platform/oauth/pc/auth") - index_page = Page(context, "https://www.xingtu.cn/ad/creator/index") - context.pages = [qr_page, index_page] - saved = [] - - monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None) - monkeypatch.setattr( - xingtu_scraper_v2, "is_logged_in", lambda page: page is index_page - ) - monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page)) - - result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2) - - assert result is index_page - assert saved == [index_page] - - -def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch): - cookie = tmp_path / "xingtu_cookies.json" - storage = tmp_path / "xingtu_storage_state.json" - profile = tmp_path / "profile" - cookie.write_text("old-cookie", encoding="utf-8") - storage.write_text("old-storage", encoding="utf-8") - profile.mkdir() - (profile / "old.txt").write_text("old-profile", encoding="utf-8") - - monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie) - monkeypatch.setattr(relogin_xingtu, "STATE_FILE", storage) - monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile) - - cookie_backup, profile_backup, state_backup = relogin_xingtu.reset_state() - profile.mkdir() - (profile / "partial.txt").write_text("partial", encoding="utf-8") - - assert relogin_xingtu.recover_interrupted_state() is True - assert cookie.read_text(encoding="utf-8") == "old-cookie" - assert storage.read_text(encoding="utf-8") == "old-storage" - assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile" - assert not (profile / "partial.txt").exists() - assert cookie_backup is not None - assert state_backup is not None - assert profile_backup is not None - - -def test_pgy_scan_login_clicks_qr_switch_image(): - class FakeLocator: - def __init__(self): - self.first = self - self.clicked = False - self.evaluated = False - - def count(self): - return 1 - - def is_visible(self): - return True - - def click(self, timeout): - self.clicked = True - - def evaluate(self, script): - assert script == "e => e.click()" - self.evaluated = True - - class FakePage: - def __init__(self): - self.selectors = [] - self.qr_switch = FakeLocator() - - def wait_for_timeout(self, _milliseconds): - pass - - def get_by_text(self, _pattern): - return FakeLocator() - - def locator(self, selector): - self.selectors.append(selector) - return self.qr_switch - - page = FakePage() - pgy_xhs_scraper_v2.open_scan_login(page) - - assert 'img[src*="qr_code"]' in page.selectors - assert page.qr_switch.evaluated - - -def test_failed_douyin_relogin_restores_cookie_storage_and_profile(tmp_path, monkeypatch): - cookie = tmp_path / "douyin_cookies.json" - storage = tmp_path / "douyin_storage_state.json" - profile = tmp_path / "profile" - cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8") - storage.write_text("old-storage", encoding="utf-8") - profile.mkdir() - (profile / "old.txt").write_text("old-profile", encoding="utf-8") - - monkeypatch.setattr(relogin_douyin, "COOKIE_FILE", cookie) - monkeypatch.setattr(relogin_douyin, "STATE_FILE", storage) - monkeypatch.setattr(relogin_douyin, "PROFILE_DIR", profile) - monkeypatch.setattr(relogin_douyin.subprocess, "call", lambda *_args, **_kwargs: 1) - monkeypatch.setattr(relogin_douyin.sys, "argv", ["relogin_douyin.py"]) - - assert relogin_douyin.main() == 1 - assert cookie.read_text(encoding="utf-8") == '[{"name":"sessionid","value":"old"}]' - assert storage.read_text(encoding="utf-8") == "old-storage" - assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile" diff --git a/tests/modules/content_marketing/test_relogin_runtime_bindings.py b/tests/modules/content_marketing/test_relogin_runtime_bindings.py deleted file mode 100644 index dfa2df2..0000000 --- a/tests/modules/content_marketing/test_relogin_runtime_bindings.py +++ /dev/null @@ -1,410 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from types import ModuleType - -import pytest - -from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog -from gyxx_flow.modules.content_marketing.data.tools import ( - relogin_bilibili, - relogin_douyin, - relogin_pgy, - relogin_runtime_paths, - relogin_xiaohongshu, - relogin_xingtu, -) -from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import ( - CONSUMER_BINDING_IDS, - child_browser_environment, - resolve_relogin_runtime_paths, - sync_relogin_state_to_consumers, -) -from gyxx_flow.modules.content_marketing.runtime_paths import PATHS - -PROJECT_ROOT = Path(__file__).resolve().parents[3] - - -def test_relogin_paths_prefer_injected_runtime_binding(tmp_path: Path) -> None: - cookie = tmp_path / "binding" / "cookies.json" - storage = tmp_path / "binding" / "storage_state.json" - profile = tmp_path / "binding" / "profile" - - resolved = resolve_relogin_runtime_paths( - "pgy", - { - "GYXX_BROWSER_COOKIE_FILE": str(cookie), - "GYXX_BROWSER_STORAGE_STATE_FILE": str(storage), - "GYXX_BROWSER_PROFILE_DIR": str(profile), - }, - ) - - assert resolved.cookie_file == cookie.resolve() - assert resolved.storage_state_file == storage.resolve() - assert resolved.profile_dir == profile.resolve() - - -def test_relogin_paths_keep_legacy_standalone_layout() -> None: - resolved = resolve_relogin_runtime_paths("xiaohongshu", {}) - - assert resolved.cookie_file == ( - PATHS.state_root / "cookies/xiaohongshu_cookies.json" - ).resolve() - assert resolved.storage_state_file == ( - PATHS.state_root / "cookies/xiaohongshu_storage_state.json" - ).resolve() - assert resolved.profile_dir == ( - PATHS.state_root / "browser-profiles/xiaohongshu" - ).resolve() - - -def test_child_login_environment_preserves_wrapper_binding(tmp_path: Path) -> None: - cookie = tmp_path / "cookies.json" - storage = tmp_path / "storage_state.json" - profile = tmp_path / "profile" - - environment = child_browser_environment( - cookie_file=cookie, - storage_state_file=storage, - profile_dir=profile, - environment={"GYXX_MODULE_ID": "content_marketing", "KEEP": "1"}, - ) - - assert environment["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve()) - assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == str( - storage.resolve() - ) - assert environment["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve()) - assert "GYXX_MODULE_ID" not in environment - assert environment["KEEP"] == "1" - - -@pytest.mark.parametrize( - ("module", "platform", "login_cookie", "argv"), - [ - ( - relogin_bilibili, - "bilibili", - "SESSDATA", - ["relogin_bilibili.py"], - ), - (relogin_douyin, "douyin", "sessionid", ["relogin_douyin.py"]), - (relogin_pgy, "pgy", "web_session", ["relogin_pgy.py"]), - ( - relogin_xiaohongshu, - "xiaohongshu", - "web_session", - ["relogin_xiaohongshu.py"], - ), - ( - relogin_xingtu, - "xingtu", - "sessionid", - ["relogin_xingtu.py", "--force"], - ), - ], -) -def test_relogin_wrapper_validates_the_same_binding_written_by_child( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - module: ModuleType, - platform: str, - login_cookie: str, - argv: list[str], -) -> None: - cookie = tmp_path / "cookies.json" - storage = tmp_path / "storage_state.json" - profile = tmp_path / "profile" - monkeypatch.setattr(module, "COOKIE_FILE", cookie) - monkeypatch.setattr(module, "STATE_FILE", storage) - monkeypatch.setattr(module, "PROFILE_DIR", profile) - monkeypatch.setattr(module.sys, "argv", argv) - monkeypatch.setenv("GYXX_MODULE_ID", "content_marketing") - if module is relogin_xingtu: - monkeypatch.setattr(module, "terminate_profile_processes", lambda: None) - monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None) - synced: list[tuple[str, Path, Path]] = [] - - def fake_sync( - synced_platform: str, - *, - cookie_file: Path, - storage_state_file: Path, - ) -> tuple[Path, ...]: - synced.append( - (synced_platform, cookie_file.resolve(), storage_state_file.resolve()) - ) - return () - - monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fake_sync) - - def fake_call( - _command: list[str], - *, - cwd: str, - env: dict[str, str], - ) -> int: - assert cwd == str(module.PROJECT_DIR) - assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve()) - assert env["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(storage.resolve()) - assert env["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve()) - assert "GYXX_MODULE_ID" not in env - cookie.write_text( - json.dumps([{"name": login_cookie, "value": "fresh"}]), - encoding="utf-8", - ) - storage.write_text("{}", encoding="utf-8") - profile.mkdir() - return 0 - - monkeypatch.setattr(module.subprocess, "call", fake_call) - - assert module.main() == 0 - assert cookie.is_file() - assert storage.is_file() - assert profile.is_dir() - assert synced == [(platform, cookie.resolve(), storage.resolve())] - - -@pytest.mark.parametrize( - ("module", "platform", "login_cookie", "cookie_domain", "argv"), - [ - ( - relogin_bilibili, - "bilibili", - "SESSDATA", - ".bilibili.com", - ["relogin_bilibili.py"], - ), - ( - relogin_douyin, - "douyin", - "sessionid", - ".douyin.com", - ["relogin_douyin.py"], - ), - ( - relogin_pgy, - "pgy", - "web_session", - ".xiaohongshu.com", - ["relogin_pgy.py"], - ), - ( - relogin_xiaohongshu, - "xiaohongshu", - "web_session", - ".xiaohongshu.com", - ["relogin_xiaohongshu.py"], - ), - ( - relogin_xingtu, - "xingtu", - "sessionid", - ".xingtu.cn", - ["relogin_xingtu.py", "--force"], - ), - ], -) -def test_relogin_wrapper_restores_previous_state_when_publish_fails( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - module: ModuleType, - platform: str, - login_cookie: str, - cookie_domain: str, - argv: list[str], -) -> None: - cookie = tmp_path / "cookies.json" - storage = tmp_path / "storage_state.json" - profile = tmp_path / "profile" - old_cookie = '[{"name":"old","value":"old-session"}]' - old_storage = '{"cookies":[{"name":"old"}]}' - cookie.write_text(old_cookie, encoding="utf-8") - storage.write_text(old_storage, encoding="utf-8") - profile.mkdir() - (profile / "old-state.txt").write_text("old-profile", encoding="utf-8") - monkeypatch.setattr(module, "COOKIE_FILE", cookie) - monkeypatch.setattr(module, "STATE_FILE", storage) - monkeypatch.setattr(module, "PROFILE_DIR", profile) - monkeypatch.setattr(module.sys, "argv", argv) - if module is relogin_xingtu: - monkeypatch.setattr(module, "terminate_profile_processes", lambda: None) - monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None) - - def fake_call( - _command: list[str], - *, - cwd: str, - env: dict[str, str], - ) -> int: - assert cwd == str(module.PROJECT_DIR) - assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve()) - cookie.write_text( - json.dumps( - [ - { - "name": login_cookie, - "value": "fresh-session", - "domain": cookie_domain, - } - ] - ), - encoding="utf-8", - ) - storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8") - profile.mkdir() - return 0 - - def fail_publish( - synced_platform: str, - *, - cookie_file: Path, - storage_state_file: Path, - ) -> tuple[Path, ...]: - assert synced_platform == platform - assert cookie_file == cookie - assert storage_state_file == storage - raise OSError("simulated consumer publish failure") - - monkeypatch.setattr(module.subprocess, "call", fake_call) - monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fail_publish) - - assert module.main() == 1 - assert cookie.read_text(encoding="utf-8") == old_cookie - assert storage.read_text(encoding="utf-8") == old_storage - assert (profile / "old-state.txt").read_text(encoding="utf-8") == ( - "old-profile" - ) - - -@pytest.mark.parametrize( - ("platform", "cookie_name", "cookie_domain"), - [ - ("bilibili", "SESSDATA", ".bilibili.com"), - ("douyin", "sessionid", ".douyin.com"), - ("pgy", "web_session", ".xiaohongshu.com"), - ("xiaohongshu", "web_session", ".xiaohongshu.com"), - ("xingtu", "sessionid", ".xingtu.cn"), - ], -) -def test_successful_relogin_atomically_updates_unique_consumer_state( - tmp_path: Path, - platform: str, - cookie_name: str, - cookie_domain: str, -) -> None: - data_root = tmp_path / "data" - source_cookie = tmp_path / "relogin" / "cookies.json" - source_storage = tmp_path / "relogin" / "storage_state.json" - source_cookie.parent.mkdir(parents=True) - fresh_cookie = [ - { - "name": cookie_name, - "value": "fresh-session", - "domain": cookie_domain, - } - ] - fresh_storage = {"cookies": fresh_cookie, "origins": []} - source_cookie.write_text(json.dumps(fresh_cookie), encoding="utf-8") - source_storage.write_text(json.dumps(fresh_storage), encoding="utf-8") - - catalog = RuntimeIntegrationCatalog.load_default( - project_root=PROJECT_ROOT, - data_root=data_root, - ) - consumer_id = CONSUMER_BINDING_IDS[platform][0] - consumer = catalog.binding_for(consumer_id) - consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True) - consumer.cookie_file.write_text('[{"name":"old"}]', encoding="utf-8") - consumer.storage_state_file.write_text( - '{"cookies":[{"name":"old"}]}', - encoding="utf-8", - ) - - destinations = sync_relogin_state_to_consumers( - platform, - cookie_file=source_cookie, - storage_state_file=source_storage, - environment={ - "GYXX_PROJECT_ROOT": str(PROJECT_ROOT), - "GYXX_DATA_ROOT": str(data_root), - }, - ) - - assert source_cookie.resolve() != consumer.cookie_file.resolve() - assert source_storage.resolve() != consumer.storage_state_file.resolve() - assert destinations == (consumer.cookie_file.resolve(),) - assert json.loads(consumer.cookie_file.read_text(encoding="utf-8")) == ( - fresh_cookie - ) - assert json.loads( - consumer.storage_state_file.read_text(encoding="utf-8") - ) == fresh_storage - - -def test_consumer_state_is_rolled_back_if_atomic_publish_fails( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - data_root = tmp_path / "data" - source_cookie = tmp_path / "relogin" / "cookies.json" - source_storage = tmp_path / "relogin" / "storage_state.json" - source_cookie.parent.mkdir(parents=True) - source_cookie.write_text( - json.dumps( - [ - { - "name": "SESSDATA", - "value": "fresh-session", - "domain": ".bilibili.com", - } - ] - ), - encoding="utf-8", - ) - source_storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8") - - catalog = RuntimeIntegrationCatalog.load_default( - project_root=PROJECT_ROOT, - data_root=data_root, - ) - consumer = catalog.binding_for(CONSUMER_BINDING_IDS["bilibili"][0]) - consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True) - old_cookie = '[{"name":"SESSDATA","value":"old"}]' - old_storage = '{"cookies":[{"name":"old"}]}' - consumer.cookie_file.write_text(old_cookie, encoding="utf-8") - consumer.storage_state_file.write_text(old_storage, encoding="utf-8") - - original_replace = relogin_runtime_paths.os.replace - failed = False - - def fail_second_replace(source: Path | str, target: Path | str) -> None: - nonlocal failed - source_path = Path(source) - target_path = Path(target) - if ( - not failed - and source_path.suffix == ".tmp" - and target_path == consumer.storage_state_file - ): - failed = True - raise OSError("simulated storage publish failure") - original_replace(source, target) - - monkeypatch.setattr(relogin_runtime_paths.os, "replace", fail_second_replace) - - with pytest.raises(OSError, match="simulated storage publish failure"): - sync_relogin_state_to_consumers( - "bilibili", - cookie_file=source_cookie, - storage_state_file=source_storage, - environment={ - "GYXX_PROJECT_ROOT": str(PROJECT_ROOT), - "GYXX_DATA_ROOT": str(data_root), - }, - ) - - assert consumer.cookie_file.read_text(encoding="utf-8") == old_cookie - assert consumer.storage_state_file.read_text(encoding="utf-8") == old_storage