wangyunlong/gyxx-flow
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(login): 移除脚本内扫码重登，登录统一交给账号 vault 与 cookie-gate

Browse Source 
删除 relogin_*/friday_relogin_parallel 子进程及其测试；批量补采、评论采集、
星图/蒲公英/B站/抖音自营与抖店竞品脚本不再内嵌扫码等待，登录态失效统一由
cookie-gate 阻断并走 gyxx accounts login 恢复；runtime-bindings 与模块注册
同步移除 content.relogin.weekly。
This commit is contained in:
wangyunlong committed 2026-10-09 13:45:19 +08:00
1 parent 01f3d7aaf7
commit 78ea4641ce
28 files changed
+120 -4254
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

config/runtime-bindings.json
+13 -100
View File
@@ -312,8 +312,6 @@
},
"members": [
"content.comments.collect_xiaohongshu",
"content_marketing:data/tools/relogin_pgy.py",
"content_marketing:data/tools/relogin_xiaohongshu.py",
"content_marketing:pgy_xhs_scraper_v2.py"
]
},
@@ -340,7 +338,6 @@
},
"members": [
"content.comments.collect_douyin",
"content_marketing:data/tools/relogin_douyin.py",
"content_marketing:self_douyin_scraper.py"
]
},
@@ -368,12 +365,15 @@
"real_chrome": true
},
"members": [
"content_marketing:data/tools/relogin_xingtu.py"
]
},
"chanmama": {
"cdp_port": 22207,
"login_mode": "D",
"credential_env_names": [
"CHANMAMA_ACCOUNT",
"CHANMAMA_PASSWORD"
],
"required_cookie_domains": [
"chanmama.com"
],
@@ -411,8 +411,7 @@
},
"members": [
"content.comments.collect_bilibili",
"content.metrics.collect_self_bilibili",
"content_marketing:data/tools/relogin_bilibili.py"
"content.metrics.collect_self_bilibili"
]
}
},
@@ -582,23 +581,6 @@
],
"cdp_port": 22016
},
"content_marketing:data/tools/friday_relogin.bat": {
"script_id": "content_marketing:data/tools/friday_relogin.bat",
"state_key": "content_marketing:data/tools/friday_relogin.bat",
"aliases": [
"content_marketing:data/tools/friday_relogin.bat"
],
"cdp_port": 22017
},
"content.login.refresh": {
"script_id": "content_marketing:data/tools/friday_relogin_parallel.py",
"state_key": "content_marketing:data/tools/friday_relogin_parallel.py",
"aliases": [
"content_marketing:data/tools/friday_relogin_parallel.py"
],
"cdp_port": 22018,
"login_mode": "D"
},
"content.creator_report.generate": {
"script_id": "content_marketing:data/tools/generate_creator_report.py",
"state_key": "content_marketing:data/tools/generate_creator_report.py",
@@ -679,83 +661,6 @@
],
"cdp_port": 22135
},
"content_marketing:data/tools/relogin_bilibili.py": {
"script_id": "content_marketing:data/tools/relogin_bilibili.py",
"state_key": "content_marketing:data/tools/relogin_bilibili.py",
"aliases": [
"content_marketing:data/tools/relogin_bilibili.py"
],
"cdp_port": 22028,
"login_mode": "D",
"required_cookie_domains": [
"bilibili.com"
],
"required_cookie_names": [
"SESSDATA"
]
},
"content_marketing:data/tools/relogin_douyin.py": {
"script_id": "content_marketing:data/tools/relogin_douyin.py",
"state_key": "content_marketing:data/tools/relogin_douyin.py",
"aliases": [
"content_marketing:data/tools/relogin_douyin.py"
],
"cdp_port": 22029,
"login_mode": "D",
"required_cookie_domains": [
"douyin.com"
],
"required_cookie_names": [
"sessionid"
]
},
"content_marketing:data/tools/relogin_pgy.py": {
"script_id": "content_marketing:data/tools/relogin_pgy.py",
"state_key": "content_marketing:data/tools/relogin_pgy.py",
"aliases": [
"content_marketing:data/tools/relogin_pgy.py"
],
"cdp_port": 22030,
"login_mode": "D",
"required_cookie_domains": [
"xiaohongshu.com"
]
},
"content_marketing:data/tools/relogin_xiaohongshu.py": {
"script_id": "content_marketing:data/tools/relogin_xiaohongshu.py",
"state_key": "content_marketing:data/tools/relogin_xiaohongshu.py",
"aliases": [
"content_marketing:data/tools/relogin_xiaohongshu.py"
],
"cdp_port": 22031,
"login_mode": "D",
"required_cookie_domains": [
"xiaohongshu.com"
],
"required_cookie_names": [
"web_session"
]
},
"content_marketing:data/tools/relogin_xingtu.py": {
"script_id": "content_marketing:data/tools/relogin_xingtu.py",
"state_key": "content_marketing:data/tools/relogin_xingtu.py",
"aliases": [
"content_marketing:data/tools/relogin_xingtu.py"
],
"cdp_port": 22032,
"login_mode": "D",
"required_cookie_domains": [
"douyin.com",
"oceanengine.com",
"xingtu.cn"
],
"required_cookie_names": [
"sessionid",
"sessionid_ss",
"sid_tt",
"sid_guard"
]
},
"content.failed.retry": {
"script_id": "content_marketing:data/tools/retry_failed.py",
"state_key": "content_marketing:data/tools/retry_failed.py",
@@ -1441,6 +1346,14 @@
"erp321.com"
]
},
"product.hk_anpei_public_inventory.refresh": {
"script_id": "product_commerce:public_sku_inventory_sheet.py",
"state_key": "product_commerce:public_sku_inventory_sheet.py",
"aliases": [
"product_commerce:public_sku_inventory_sheet.py"
],
"cdp_port": 22166
},
"product_commerce:scripts/insert_jd_main_image_records.py": {
"script_id": "product_commerce:scripts/insert_jd_main_image_records.py",
"state_key": "product_commerce:scripts/insert_jd_main_image_records.py",
src/gyxx_flow/modules/content_marketing/__init__.py
+1 -4
View File
@@ -1,4 +1,4 @@
"""Content metrics, creator, comment, report, and login workflows."""
"""Content metrics, creator, comment, and report workflows."""
from __future__ import annotations
@@ -14,7 +14,6 @@ CONTENT_SCHEDULED_WORKFLOW_IDS = (
"content.metrics.self_operated.daily",
"content.metrics.backfill",
"content.marketing_report.daily",
"content.relogin.weekly",
"content.creator_report.monthly",
"content.summary.monthly",
"content.notes_master.daily",
@@ -28,7 +27,6 @@ CONTENT_COMMENT_TIMEOUT_SECONDS = 6 * 60 * 60
# weakening the module-wide write lock.
CONTENT_NOTES_MASTER_LOCK_TIMEOUT_SECONDS = 6 * 60 * 60
CONTENT_RESOURCE = "module:content_marketing"
_RELOGIN_WORKFLOW_ID = "content.relogin.weekly"
_PARALLEL_COMMENT_WORKFLOW_ID = "content.comments.weekly"
# Keep both long-running daily metrics graphs on workflow-scoped resources so
# their collection steps do not block unrelated content workflows.
@@ -94,7 +92,6 @@ class ContentMarketingModule:
),
resource=workflow_resource,
command_factory=command_factory,
official_notification=workflow_id == _RELOGIN_WORKFLOW_ID,
independent_step_resources=(
workflow_id in {
_PARALLEL_COMMENT_WORKFLOW_ID,
src/gyxx_flow/modules/content_marketing/bilibili_comment_scraper.py
+9 -48
View File
@@ -77,23 +77,15 @@ def is_logged_in(page: Page) -> bool:
def maybe_wait_for_login(page: Page, login_timeout: int) -> None:
"""检测登录态,未登录就等用户扫码。B 站登录页 URL 含 passport.bilibili.com/login。"""
"""检测登录态；未登录立即降级为游客态，不再等待人工扫码。
登录统一由账号 vault 管理（gyxx accounts login bilibili-content），
由工作流执行前的 cookie 网关保证登录态。
"""
if is_logged_in(page):
return
log("B 站未登录 (无 SESSDATA cookie)。请在弹出的浏览器窗口扫码登录。")
# 跳到登录页,确保用户能看到二维码
try:
page.goto("https://passport.bilibili.com/login", wait_until="domcontentloaded", timeout=30000)
except Exception as exc:
log(f" 跳登录页失败 (继续等当前页扫码): {exc}")
deadline = time.time() + login_timeout
while time.time() < deadline:
if is_logged_in(page):
log("登录成功,SESSDATA 已写入")
save_state(page)
return
page.wait_for_timeout(2000)
log(f" [WARN] 等待 {login_timeout}s 仍未登录,继续以游客态采集 (评论只会拿到热门 3-5 条)")
log(" [WARN] B 站未登录 (无 SESSDATA cookie)，降级为游客态采集 (评论只会拿到热门 3-5 条)。"
"如需全量评论请执行 gyxx accounts login bilibili-content")
def safe_filename(value: str) -> str:
@@ -437,48 +429,17 @@ def write_outputs(url: str, comments: list[dict[str, Any]], result: dict[str, An
return json_path, csv_path
def login_only(login_timeout: int) -> int:
"""只触发登录(headed),登录完成立刻退出。给 relogin_bilibili.py 用。"""
def action(page: Page) -> None:
page.goto("https://passport.bilibili.com/login", wait_until="domcontentloaded", timeout=30000)
page.wait_for_timeout(1500)
maybe_wait_for_login(page, login_timeout)
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
with DynamicSession(
headless=False,
real_chrome=True,
user_data_dir=str(PROFILE_DIR),
locale="zh-CN",
timezone_id="Asia/Shanghai",
timeout=90000,
network_idle=False,
disable_resources=False,
google_search=False,
page_setup=restore_cookies,
max_pages=1,
) as session:
session.fetch("https://passport.bilibili.com/login", page_action=action, wait=1000)
log("[login-only] 完成")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description="Scrape Bilibili video comments with a real browser and API.")
parser.add_argument("url", help="Bilibili video URL (传 dummy 'login' 配合 --login-only 也可)")
parser.add_argument("url", help="Bilibili video URL")
parser.add_argument("--max-reply-pages", type=int, default=200, help="max sub-reply pages per top-level comment")
parser.add_argument("--max-pages", type=int, default=200, help="最多抓取一级评论页数")
parser.add_argument("--delay", type=float, default=0.25, help="接口请求间隔秒数")
parser.add_argument("--headless", action="store_true", help="无头模式")
parser.add_argument("--login-timeout", type=int, default=0,
help="未登录时等待扫码秒数(>0 启用登录检测;默认 0 沿用游客态)")
parser.add_argument("--login-only", action="store_true",
help="只触发登录(headed)然后退出,给 relogin_bilibili 用")
help="保留参数（登录等待已移除；未登录将直接以游客态采集）")
args = parser.parse_args()
if args.login_only:
return login_only(args.login_timeout if args.login_timeout > 0 else 300)
comments, result = scrape_comments(
args.url, args.max_pages, args.delay, args.headless,
args.max_reply_pages, args.login_timeout,
src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_bilibili.py
+6 -41
View File
@@ -18,7 +18,6 @@ from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
_TOOLS_DIR = PATHS.tools_root
_PROJECT_ROOT = PATHS.module_root
RELOGIN_SCRIPT = _TOOLS_DIR / "relogin_bilibili.py"
try:
sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
@@ -81,39 +80,6 @@ def comment_replace_policy(comments: list[dict], stats: dict, metrics: dict) ->
return False, False, "空评论且无法确认平台为 0 评论"
def do_relogin() -> bool:
"""Run relogin_bilibili.py (headed, waits for QR scan). Returns True if successful."""
acceptance_policy = current_acceptance_policy()
if acceptance_policy.enabled:
acceptance_policy.record(
"cookie_skipped",
operation="content.batch_rescrape_bilibili.relogin",
details={
"status": "SKIPPED_COOKIE",
"reason": "cookie is invalid and QR relogin is disabled for acceptance",
},
)
log(
" [SKIPPED_COOKIE] Cookie invalid; acceptance mode will not "
"launch relogin or wait for a QR scan"
)
return False
log(" Cookie 可能过期,触发自动重登 (请扫码)...")
try:
rc = subprocess.call(
[sys.executable, str(RELOGIN_SCRIPT)],
cwd=str(_PROJECT_ROOT),
)
if rc == 0:
log(" 重登成功,继续采集")
return True
else:
log(f" 重登失败 (exit={rc})")
return False
except Exception as exc:
log(f" 重登脚本执行异常: {exc}")
return False
def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
rows = collect_urls()
@@ -126,8 +92,6 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
total_comments = 0
consecutive_fail = 0
MAX_CONSECUTIVE_FAIL = 3
relogin_used = 0
MAX_RELOGINS = 1
t0 = time.time()
for i, row in enumerate(rows, 1):
note_id = row["id"]
@@ -152,11 +116,12 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
break
if comments is None:
consecutive_fail += 1
if consecutive_fail >= MAX_CONSECUTIVE_FAIL and relogin_used < MAX_RELOGINS:
if do_relogin():
relogin_used += 1
consecutive_fail = 0
continue
if consecutive_fail >= MAX_CONSECUTIVE_FAIL:
# 登录统一由账号 vault 管理；连续失败只提示，不再触发扫码重登。
log(
" 连续失败达到上限；若为登录态失效请执行 "
"uv run gyxx accounts login bilibili-content"
)
if current_acceptance_policy().enabled:
return COOKIE_SKIP_EXIT_CODE
continue
src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_douyin.py
+2 -43
View File
@@ -26,7 +26,6 @@ from gyxx_flow.modules.content_marketing.serialized_page_action import (
_TOOLS_DIR = PATHS.tools_root
_PROJECT_ROOT = PATHS.module_root
RELOGIN_SCRIPT = _TOOLS_DIR / "relogin_douyin.py"
try:
sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
@@ -104,38 +103,6 @@ def result_logged_in(result: dict | None) -> bool:
return bool(nested_stats.get("logged_in")) if isinstance(nested_stats, dict) else False
def do_relogin() -> bool:
"""Run relogin_douyin.py (headed, waits for QR scan). Returns True if successful."""
acceptance_policy = current_acceptance_policy()
if acceptance_policy.enabled:
acceptance_policy.record(
"cookie_skipped",
operation="content.batch_rescrape_douyin.relogin",
details={
"status": "SKIPPED_COOKIE",
"reason": "cookie is invalid and QR relogin is disabled for acceptance",
},
)
log(
" [SKIPPED_COOKIE] Cookie invalid; acceptance mode will not "
"launch relogin or wait for a QR scan"
)
return False
log(" Cookie 可能过期,触发自动重登 (请扫码)...")
try:
rc = subprocess.call(
[sys.executable, str(RELOGIN_SCRIPT)],
cwd=str(_PROJECT_ROOT),
)
if rc == 0:
log(" 重登成功,继续采集")
return True
log(f" 重登失败 (exit={rc})")
return False
except Exception as exc:
log(f" 重登脚本执行异常: {exc}")
return False
def scrape_with_fresh_browser(
url: str,
@@ -203,8 +170,6 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
COOLDOWN_SECONDS = 300
cooldown_used = 0
MAX_COOLDOWNS = 3
relogin_used = 0
MAX_RELOGINS = 1 # 每轮 batch 最多自动重登1次
t0 = time.time()
for i, row in enumerate(rows, 1):
note_id = row["id"]
@@ -214,14 +179,8 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
if comments is None:
consecutive_fail += 1
# 连续失败 + 可能 cookie 过期 → 尝试自动重登
if consecutive_fail >= MAX_CONSECUTIVE_FAIL and relogin_used < MAX_RELOGINS:
if do_relogin():
relogin_used += 1
consecutive_fail = 0
comments, stats, scrape_error = scrape_with_fresh_browser(url)
else:
# 重登失败,走冷却逻辑；验收模式必须直接跳过，不能等待。
# 登录统一由账号 vault 管理；不再触发扫码重登，验收模式直接跳过。
if consecutive_fail >= MAX_CONSECUTIVE_FAIL:
if current_acceptance_policy().enabled:
return COOKIE_SKIP_EXIT_CODE
src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_xiaohongshu.py
+5 -40
View File
@@ -25,7 +25,6 @@ from gyxx_flow.modules.content_marketing.serialized_page_action import (
_TOOLS_DIR = PATHS.tools_root
_PROJECT_ROOT = PATHS.module_root
RELOGIN_SCRIPT = _TOOLS_DIR / "relogin_xiaohongshu.py"
try:
sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
@@ -87,38 +86,6 @@ def comment_replace_policy(comments: list[dict], stats: dict, metrics: dict) ->
return False, False, "空评论且无法确认平台为 0 评论"
def do_relogin() -> bool:
"""Run relogin_xiaohongshu.py (headed, waits for QR scan). Returns True if successful."""
acceptance_policy = current_acceptance_policy()
if acceptance_policy.enabled:
acceptance_policy.record(
"cookie_skipped",
operation="content.batch_rescrape_xiaohongshu.relogin",
details={
"status": "SKIPPED_COOKIE",
"reason": "cookie is invalid and QR relogin is disabled for acceptance",
},
)
log(
" [SKIPPED_COOKIE] Cookie invalid; acceptance mode will not "
"launch relogin or wait for a QR scan"
)
return False
log(" Cookie 可能过期,触发自动重登 (请扫码)...")
try:
rc = subprocess.call(
[sys.executable, str(RELOGIN_SCRIPT)],
cwd=str(_PROJECT_ROOT),
)
if rc == 0:
log(" 重登成功,继续采集")
return True
log(f" 重登失败 (exit={rc})")
return False
except Exception as exc:
log(f" 重登脚本执行异常: {exc}")
return False
def scrape_with_fresh_browser(
url: str,
@@ -173,8 +140,6 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
total_comments = 0
consecutive_fail = 0
MAX_CONSECUTIVE_FAIL = 3
relogin_used = 0
MAX_RELOGINS = 1
t0 = time.time()
for i, row in enumerate(rows, 1):
note_id = row["id"]
@@ -183,13 +148,13 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
comments, result, scrape_error = scrape_with_fresh_browser(url)
if comments is None:
consecutive_fail += 1
if consecutive_fail >= MAX_CONSECUTIVE_FAIL and relogin_used < MAX_RELOGINS:
if do_relogin():
relogin_used += 1
consecutive_fail = 0
comments, result, scrape_error = scrape_with_fresh_browser(url)
if consecutive_fail >= MAX_CONSECUTIVE_FAIL:
if current_acceptance_policy().enabled:
return COOKIE_SKIP_EXIT_CODE
log(
" 连续失败达到上限；若为登录态失效请执行 "
"uv run gyxx accounts login xiaohongshu-content"
)
if comments is None:
log(f" ERROR: {scrape_error}")
fail += 1
src/gyxx_flow/modules/content_marketing/data/tools/friday_relogin_parallel.py
-666
View File
@@ -1,666 +0,0 @@
"""Launch five QR relogins in parallel, notify via analyzer Lark, retry 3 times."""
from __future__ import annotations
import argparse
import ctypes
import datetime as dt
import os
import subprocess
import sys
import time
from ctypes import wintypes
from dataclasses import dataclass
from pathlib import Path
from typing import Callable
import psutil
from gyxx_flow.adapters import (
ANALYZER_NOTIFICATION_APP_PROFILE,
COOKIE_SKIP_EXIT_CODE,
ResolvedNotificationRoute,
binding_from_environment,
current_acceptance_policy,
environment_for_child_script,
resolve_notification_route,
send_lark_bot_message,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
PROJECT_DIR = PATHS.module_root
ANALYZER_DIR = PATHS.tmp_root / "relogin"
WORKFLOW_ID = "content.relogin.weekly"
MARKETING_RECIPIENT = "ou_24cc944d6e43c69c59d6560ad4e2ae6e"
TECHNICAL_RECIPIENT = "ou_7ad5fc8012e2f741afc5346e05ffd447"
@dataclass(frozen=True)
class Platform:
label: str
command: tuple[str, ...]
required_cookie: str
@dataclass(frozen=True, slots=True)
class NotificationDeliveryFailure:
"""One failed recipient delivery that must affect the workflow outcome."""
stage: str
platform: str
recipient: str
detail: str
PLATFORMS = {
"douyin": Platform(
"\u6296\u97f3", ("data/tools/relogin_douyin.py",),
"sessionid",
),
"bilibili": Platform(
"B\u7ad9", ("data/tools/relogin_bilibili.py",),
"SESSDATA",
),
"pgy": Platform(
"\u84b2\u516c\u82f1", ("data/tools/relogin_pgy.py",),
"",
),
"xingtu": Platform(
"\u661f\u56fe", ("data/tools/relogin_xingtu.py", "--force"),
"sessionid",
),
"xiaohongshu": Platform(
"\u5c0f\u7ea2\u4e66", ("data/tools/relogin_xiaohongshu.py",),
"web_session",
),
}
DEFAULT_RECIPIENTS = {
"pgy": MARKETING_RECIPIENT,
"xingtu": MARKETING_RECIPIENT,
"douyin": TECHNICAL_RECIPIENT,
"bilibili": TECHNICAL_RECIPIENT,
"xiaohongshu": TECHNICAL_RECIPIENT,
}
def resolve_platform_names(requested: list[str] | None) -> list[str]:
"""Resolve an optional manual platform subset while preserving order."""
if not requested:
return list(PLATFORMS)
names: list[str] = []
for name in requested:
if name not in PLATFORMS:
raise ValueError(f"unknown platform: {name}")
if name not in names:
names.append(name)
return names
def recipient_for_platform(platform_name: str, override: str | None = None) -> str:
"""Return the responsible recipient, unless a manual all-platform override is set."""
from gyxx_flow.adapters.acceptance_policy import resolve_notification_recipients
if override:
recipient = override
else:
try:
recipient = DEFAULT_RECIPIENTS[platform_name]
except KeyError as exc:
raise ValueError(
f"no Lark recipient configured for platform: {platform_name}"
) from exc
return resolve_notification_recipients(
(recipient,),
app_profile=ANALYZER_NOTIFICATION_APP_PROFILE,
)[0]
def resolve_weekly_notification_route(
override: str | None = None,
) -> ResolvedNotificationRoute:
"""Resolve the scheduled workflow route while retaining legacy defaults."""
defaults = (
(override,)
if override
else tuple(dict.fromkeys(DEFAULT_RECIPIENTS.values()))
)
return resolve_notification_route(WORKFLOW_ID, defaults)
def recipients_for_platform(
platform_name: str,
route: ResolvedNotificationRoute,
override: str | None = None,
) -> tuple[str, ...]:
"""Return dynamic recipients, or the original per-platform recipient."""
if not route.enabled:
return ()
if route.configured:
return route.open_ids
return (recipient_for_platform(platform_name, override),)
def group_platforms_by_recipient(
platform_names: list[str], override: str | None = None,
) -> dict[str, list[str]]:
grouped: dict[str, list[str]] = {}
for name in platform_names:
grouped.setdefault(recipient_for_platform(name, override), []).append(name)
return grouped
def group_platforms_by_notification_recipient(
platform_names: list[str],
route: ResolvedNotificationRoute,
override: str | None = None,
) -> dict[str, list[str]]:
"""Group failed platforms for legacy or configured multi-recipient delivery."""
if not route.enabled:
return {}
if route.configured:
return {recipient: list(platform_names) for recipient in route.open_ids}
return group_platforms_by_recipient(platform_names, override)
def run_with_retries(
platform_names: list[str],
*,
max_attempts: int,
run_round: Callable[[list[str], int], dict[str, bool]],
) -> dict[str, bool]:
status = {name: False for name in platform_names}
pending = list(platform_names)
for attempt in range(1, max_attempts + 1):
if not pending:
break
round_result = run_round(pending, attempt)
for name in pending:
status[name] = bool(round_result.get(name))
pending = [name for name in pending if not status[name]]
return status
def _valid_cookie(platform: Platform, cookie_file: Path) -> bool:
if not cookie_file.is_file():
return False
if not platform.required_cookie:
return cookie_file.stat().st_size > 10
try:
import json
values = json.loads(cookie_file.read_text(encoding="utf-8"))
return any(
item.get("name") == platform.required_cookie and item.get("value")
for item in values if isinstance(item, dict)
)
except Exception:
return False
def _descendant_pids(pid: int) -> set[int]:
try:
return {child.pid for child in psutil.Process(pid).children(recursive=True)}
except (psutil.Error, OSError):
return set()
def _terminate_process_tree(process: subprocess.Popen) -> None:
"""Terminate the whole Windows subprocess tree, including Chrome/Scrapling."""
try:
subprocess.run(
["taskkill", "/PID", str(process.pid), "/T", "/F"],
capture_output=True,
text=True,
check=False,
)
except OSError:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
def _window_for_process(pid: int) -> int | None:
pids = _descendant_pids(pid) | {pid}
handles: list[int] = []
user32 = ctypes.windll.user32
callback_type = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p)
def callback(hwnd, _lparam):
process_id = ctypes.c_ulong()
user32.GetWindowThreadProcessId(hwnd, ctypes.byref(process_id))
if process_id.value in pids and user32.IsWindowVisible(hwnd):
handles.append(int(hwnd))
return True
user32.EnumWindows(callback_type(callback), 0)
return handles[0] if handles else None
def _platform_from_window_title(title: str) -> str | None:
"""Identify a login window without relying on Chrome's process tree."""
if "\u84b2\u516c\u82f1" in title:
return "pgy"
if "\u6296\u97f3\u6388\u6743" in title or "\u5de8\u91cf\u661f\u56fe" in title:
return "xingtu"
if "\u6296\u97f3" in title:
return "douyin"
if "bilibili" in title.lower() or "\u8d26\u53f7\u767b\u5f55" in title:
return "bilibili"
if "\u5c0f\u7ea2\u4e66" in title:
return "xiaohongshu"
return None
def _login_windows() -> dict[str, int]:
"""Map visible Chrome login windows to platforms by their page titles."""
user32 = ctypes.windll.user32
found: dict[str, int] = {}
callback_type = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p)
def callback(hwnd, _lparam):
if not user32.IsWindowVisible(hwnd):
return True
length = user32.GetWindowTextLengthW(hwnd)
if length <= 0:
return True
buffer = ctypes.create_unicode_buffer(length + 1)
user32.GetWindowTextW(hwnd, buffer, length + 1)
platform = _platform_from_window_title(buffer.value)
if platform:
found[platform] = int(hwnd)
return True
user32.EnumWindows(callback_type(callback), 0)
return found
def _capture_window(hwnd: int, output: Path) -> None:
"""Maximize and render one window directly, even when it is not foreground."""
from PIL import Image
user32 = ctypes.windll.user32
gdi32 = ctypes.windll.gdi32
user32.ShowWindow(hwnd, 3) # SW_MAXIMIZE
time.sleep(1)
rect = wintypes.RECT()
user32.GetWindowRect(hwnd, ctypes.byref(rect))
width, height = rect.right - rect.left, rect.bottom - rect.top
if width <= 0 or height <= 0:
raise RuntimeError(f"invalid window size: {width}x{height}")
window_dc = user32.GetWindowDC(hwnd)
memory_dc = gdi32.CreateCompatibleDC(window_dc)
bitmap = gdi32.CreateCompatibleBitmap(window_dc, width, height)
previous = gdi32.SelectObject(memory_dc, bitmap)
try:
if not user32.PrintWindow(hwnd, memory_dc, 2): # PW_RENDERFULLCONTENT
raise RuntimeError("PrintWindow failed")
class BitmapInfoHeader(ctypes.Structure):
_fields_ = [
("biSize", wintypes.DWORD), ("biWidth", wintypes.LONG),
("biHeight", wintypes.LONG), ("biPlanes", wintypes.WORD),
("biBitCount", wintypes.WORD), ("biCompression", wintypes.DWORD),
("biSizeImage", wintypes.DWORD),
("biXPelsPerMeter", wintypes.LONG),
("biYPelsPerMeter", wintypes.LONG),
("biClrUsed", wintypes.DWORD),
("biClrImportant", wintypes.DWORD),
]
byte_count = width * height * 4
pixels = ctypes.create_string_buffer(byte_count)
header = BitmapInfoHeader(
ctypes.sizeof(BitmapInfoHeader), width, -height, 1, 32, 0,
byte_count, 0, 0, 0, 0,
)
rows = gdi32.GetDIBits(
memory_dc, bitmap, 0, height, pixels, ctypes.byref(header), 0
)
if rows != height:
raise RuntimeError(f"GetDIBits returned {rows}/{height} rows")
image = Image.frombuffer(
"RGB", (width, height), pixels, "raw", "BGRX", 0, 1
).copy()
finally:
gdi32.SelectObject(memory_dc, previous)
gdi32.DeleteObject(bitmap)
gdi32.DeleteDC(memory_dc)
user32.ReleaseDC(hwnd, window_dc)
output.parent.mkdir(parents=True, exist_ok=True)
image.save(output, "PNG")
def _tile_windows(handles: list[int]) -> None:
"""Place four login windows in a 2x2 grid so screenshots cannot overlap."""
user32 = ctypes.windll.user32
screen_w = user32.GetSystemMetrics(0)
screen_h = user32.GetSystemMetrics(1)
cell_w = max(640, screen_w // 2)
cell_h = max(450, screen_h // 2)
for index, hwnd in enumerate(handles):
x = (index % 2) * cell_w
y = (index // 2) * cell_h
user32.ShowWindow(hwnd, 9) # SW_RESTORE
user32.MoveWindow(hwnd, x, y, cell_w, cell_h, True)
def _send_lark(
recipient: str,
text: str | None = None,
image: Path | None = None,
*,
app_profile: str = ANALYZER_NOTIFICATION_APP_PROFILE,
) -> tuple[dict[str, object], ...]:
"""Send text and/or image through the analyzer Hermes bot."""
if app_profile != ANALYZER_NOTIFICATION_APP_PROFILE:
raise ValueError(
"relogin notifications must use the hermes-analyzer lark-cli profile"
)
lark_env = {
key: value for key, value in os.environ.items()
if not key.upper().startswith("HERMES_")
}
receipts: list[dict[str, object]] = []
if text:
ANALYZER_DIR.mkdir(parents=True, exist_ok=True)
receipts.append(send_lark_bot_message(
user_id=recipient,
text=text,
profile=ANALYZER_NOTIFICATION_APP_PROFILE,
cwd=ANALYZER_DIR,
env=lark_env,
))
if image:
receipts.append(send_lark_bot_message(
user_id=recipient,
image=image,
profile=ANALYZER_NOTIFICATION_APP_PROFILE,
cwd=image.parent,
env=lark_env,
))
return tuple(receipts)
def _send_qr_notification(
route: ResolvedNotificationRoute,
platform_name: str,
*,
override: str | None,
text: str,
image: Path,
) -> tuple[NotificationDeliveryFailure, ...]:
"""Attempt every QR delivery and return all recipient-specific failures."""
profile = route.app_profile or ANALYZER_NOTIFICATION_APP_PROFILE
failures: list[NotificationDeliveryFailure] = []
for recipient in recipients_for_platform(platform_name, route, override):
for stage, kwargs in (
("qr_text", {"text": text}),
("qr_image", {"image": image}),
):
try:
_send_lark(recipient, app_profile=profile, **kwargs)
except Exception as exc:
failures.append(
NotificationDeliveryFailure(
stage=stage,
platform=platform_name,
recipient=recipient,
detail=str(exc) or type(exc).__name__,
)
)
return tuple(failures)
def _unavailable_qr_delivery_failures(
route: ResolvedNotificationRoute,
platform_name: str,
*,
override: str | None,
detail: str,
) -> tuple[NotificationDeliveryFailure, ...]:
"""Record both intended QR deliveries when no screenshot can be sent."""
return tuple(
NotificationDeliveryFailure(
stage=stage,
platform=platform_name,
recipient=recipient,
detail=detail,
)
for recipient in recipients_for_platform(platform_name, route, override)
for stage in ("qr_text", "qr_image")
)
def _send_failure_summaries(
route: ResolvedNotificationRoute,
failed_platforms: list[str],
*,
override: str | None,
max_attempts: int,
) -> tuple[NotificationDeliveryFailure, ...]:
"""Attempt every final failure summary and return all delivery failures."""
groups = group_platforms_by_notification_recipient(
failed_platforms,
route,
override,
)
profile = route.app_profile or ANALYZER_NOTIFICATION_APP_PROFILE
failures: list[NotificationDeliveryFailure] = []
for recipient, names in groups.items():
labels = "\u3001".join(PLATFORMS[name].label for name in names)
try:
_send_lark(
recipient,
text=(
f"\u4ee5\u4e0b\u5e73\u53f0\u8fde\u7eed {max_attempts} \u6b21\u672a\u626b\u7801\uff0c"
"\u5df2\u6062\u590d\u5e76\u7ee7\u4f7f\u7528\u65e7 Cookie\uff1a"
f"{labels}"
),
app_profile=profile,
)
except Exception as exc:
failures.append(
NotificationDeliveryFailure(
stage="failure_summary",
platform=",".join(names),
recipient=recipient,
detail=str(exc) or type(exc).__name__,
)
)
return tuple(failures)
def _run_round_factory(
args,
notification_route: ResolvedNotificationRoute | None,
notification_failures: list[NotificationDeliveryFailure],
):
python = sys.executable
screenshot_root = PATHS.tmp_root / "qrcode/friday_relogin"
def run_round(names: list[str], attempt: int) -> dict[str, bool]:
started: dict[str, subprocess.Popen] = {}
cookie_files: dict[str, Path] = {}
missing_qr_windows: set[str] = set()
stamp = dt.datetime.now().strftime("%Y%m%d_%H%M%S")
for name in names:
platform = PLATFORMS[name]
cmd = [python, *platform.command, "--login-timeout", str(args.round_timeout)]
target = PROJECT_DIR / platform.command[0]
environment = environment_for_child_script(target, os.environ)
cookie_files[name] = binding_from_environment(environment).cookie_file
started[name] = subprocess.Popen(
cmd,
cwd=PROJECT_DIR,
env=environment,
)
time.sleep(args.screenshot_delay)
title_windows = _login_windows()
windows = {
name: title_windows.get(name) or _window_for_process(process.pid)
for name, process in started.items()
}
_tile_windows([hwnd for hwnd in windows.values() if hwnd])
time.sleep(2)
for name, process in started.items():
platform = PLATFORMS[name]
hwnd = windows[name]
if not hwnd:
print(f"[{name}] no visible browser window for QR screenshot", flush=True)
missing_qr_windows.add(name)
continue
output = screenshot_root / f"{stamp}_{name}_round{attempt}.png"
try:
_capture_window(hwnd, output)
print(f"[{name}] QR screenshot: {output}", flush=True)
except Exception as exc:
print(f"[{name}] screenshot failed: {exc}", flush=True)
if not args.no_send and notification_route is not None:
notification_failures.extend(
_unavailable_qr_delivery_failures(
notification_route,
name,
override=args.recipient,
detail=(
"QR screenshot capture failed: "
f"{str(exc) or type(exc).__name__}"
),
)
)
continue
if not args.no_send and notification_route is not None:
notification_failures.extend(
_send_qr_notification(
notification_route,
name,
override=args.recipient,
text=(
f"{platform.label} \u767b\u5f55\u4e8c\u7ef4\u7801"
f"\uff08\u7b2c {attempt}/{args.max_attempts} \u6b21\uff09\uff0c"
"\u8bf7\u5728 3 \u5206\u949f\u5185\u626b\u7801\u3002"
),
image=output,
)
)
# Put the still-open login windows back into a compact layout after
# taking full-size screenshots one at a time.
_tile_windows([hwnd for hwnd in windows.values() if hwnd])
results: dict[str, bool] = {}
for name, process in started.items():
try:
rc = process.wait(timeout=args.round_timeout + 90)
except subprocess.TimeoutExpired:
_terminate_process_tree(process)
rc = 1
results[name] = rc == 0 and _valid_cookie(
PLATFORMS[name],
cookie_files[name],
)
if (
not results[name]
and name in missing_qr_windows
and not args.no_send
and notification_route is not None
):
notification_failures.extend(
_unavailable_qr_delivery_failures(
notification_route,
name,
override=args.recipient,
detail=(
"no visible QR window and relogin did not succeed"
),
)
)
print(f"[{name}] round={attempt} exit={rc} success={results[name]}", flush=True)
return results
return run_round
def main() -> int:
acceptance_policy = current_acceptance_policy()
if acceptance_policy.enabled:
acceptance_policy.record(
"cookie_skipped",
operation="content.relogin.weekly",
details={
"status": "SKIPPED_COOKIE",
"reason": "acceptance runs never launch QR relogin workflows",
},
)
print(
"[SKIPPED_COOKIE] Acceptance mode does not launch login windows, "
"reset cookies/profiles, or wait for QR scans.",
flush=True,
)
return COOKIE_SKIP_EXIT_CODE
parser = argparse.ArgumentParser(description="Friday five-platform parallel QR relogin")
parser.add_argument(
"--recipient", default=None,
help="Override the configured recipients and send every platform to one open_id",
)
parser.add_argument("--max-attempts", type=int, default=3)
parser.add_argument("--round-timeout", type=int, default=300)
parser.add_argument("--screenshot-delay", type=int, default=60)
parser.add_argument("--no-send", action="store_true", help="Do not send Lark messages")
parser.add_argument(
"--platform", action="append", choices=sorted(PLATFORMS),
help="Only relogin the selected platform; may be repeated",
)
args = parser.parse_args()
platform_names = resolve_platform_names(args.platform)
notification_route = (
None if args.no_send else resolve_weekly_notification_route(args.recipient)
)
notification_failures: list[NotificationDeliveryFailure] = []
status = run_with_retries(
platform_names, max_attempts=args.max_attempts,
run_round=_run_round_factory(
args,
notification_route,
notification_failures,
),
)
failed = [name for name, ok in status.items() if not ok]
if failed and not args.no_send and notification_route is not None:
notification_failures.extend(
_send_failure_summaries(
notification_route,
failed,
override=args.recipient,
max_attempts=args.max_attempts,
)
)
print(f"Final relogin status: {status}", flush=True)
if notification_failures:
print(
f"Final notification status: failed ({len(notification_failures)} deliveries)",
flush=True,
)
for failure in notification_failures:
print(
"[NOTIFICATION_ERROR] "
f"stage={failure.stage} platform={failure.platform} "
f"recipient={failure.recipient} error={failure.detail}",
flush=True,
)
return 2
if notification_route is not None and notification_route.enabled:
print("Final notification status: delivered", flush=True)
else:
print("Final notification status: skipped", flush=True)
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())
src/gyxx_flow/modules/content_marketing/data/tools/relogin_bilibili.py
-113
View File
@@ -1,113 +0,0 @@
"""每周一重登 B 站。
强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。
任务计划: 每周一 09:00
跑法:
python data/tools/relogin_bilibili.py # 实际跑(会弹二维码)
python data/tools/relogin_bilibili.py --dry-run # 只看会做什么
"""
import argparse
import datetime as dt
import subprocess
import sys
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
child_browser_environment,
resolve_relogin_runtime_paths,
sync_relogin_state_to_consumers,
)
from gyxx_flow.modules.content_marketing.data.tools.relogin_transaction import (
begin,
commit,
rollback,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
PROJECT_DIR = PATHS.module_root
RUNTIME_PATHS = resolve_relogin_runtime_paths("bilibili")
COOKIE_FILE = RUNTIME_PATHS.cookie_file
STATE_FILE = RUNTIME_PATHS.storage_state_file
PROFILE_DIR = RUNTIME_PATHS.profile_dir
SCRAPER = PROJECT_DIR / "bilibili_comment_scraper.py"
def reset_state():
"""删 cookie JSON + storage_state + 清浏览器 profile。"""
state = begin((COOKIE_FILE, STATE_FILE), PROFILE_DIR)
print(" [OK] 旧 Cookie/Profile 已暂存,失败时将自动恢复")
return state
def main() -> int:
ap = argparse.ArgumentParser(description="每周一重登 B 站")
ap.add_argument("--login-timeout", type=int, default=300, help="扫码超时(秒)")
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不删不跑")
args = ap.parse_args()
print("== Relogin: B 站 ==")
print(f" 今天: {dt.date.today()} ({dt.date.today().strftime('%A')})")
print(f" cookie: {COOKIE_FILE}")
print(f" profile: {PROFILE_DIR}")
print()
if args.dry_run:
print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码")
return 0
old_state = reset_state()
cmd = [
sys.executable,
str(SCRAPER),
"login", # dummy url (login-only 模式不需要真实 url)
"--login-only",
"--login-timeout",
str(args.login_timeout),
]
print(f"\n [RUN] {' '.join(cmd)}")
try:
rc = subprocess.call(
cmd,
cwd=str(PROJECT_DIR),
env=child_browser_environment(
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
profile_dir=PROFILE_DIR,
),
)
except BaseException:
rollback(old_state)
raise
print(f"\n exit={rc}")
login_ok = False
if rc == 0:
# 验证 SESSDATA 是否真写入了
try:
import json
cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
sessdata = any(c.get("name") == "SESSDATA" and c.get("value") for c in cookies)
login_ok = bool(sessdata)
print(f" SESSDATA 写入: {'OK' if sessdata else 'FAIL — 没有看到 SESSDATA,登录可能没成功'}")
except Exception as exc:
print(f" [WARN] 验证 cookie 时出错: {exc}")
if rc != 0 or not login_ok:
rollback(old_state)
print(" [ROLLBACK] 未完成扫码,已恢复旧登录状态")
return rc or 1
try:
sync_relogin_state_to_consumers(
"bilibili",
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
)
except Exception as exc:
rollback(old_state)
print(f" [ROLLBACK] 同步采集器登录态失败: {exc}")
return 1
commit(old_state)
return 0
if __name__ == "__main__":
sys.exit(main())
src/gyxx_flow/modules/content_marketing/data/tools/relogin_douyin.py
-121
View File
@@ -1,121 +0,0 @@
"""Force a fresh Douyin login while preserving the previous usable session."""
from __future__ import annotations
import argparse
import datetime as dt
import json
import subprocess
import sys
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
child_browser_environment,
resolve_relogin_runtime_paths,
sync_relogin_state_to_consumers,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
try:
from .relogin_transaction import begin, commit, rollback
except ImportError: # Direct script execution from data/tools.
from gyxx_flow.modules.content_marketing.data.tools.relogin_transaction import (
begin,
commit,
rollback,
)
PROJECT_DIR = PATHS.module_root
RUNTIME_PATHS = resolve_relogin_runtime_paths("douyin")
COOKIE_FILE = RUNTIME_PATHS.cookie_file
STATE_FILE = RUNTIME_PATHS.storage_state_file
PROFILE_DIR = RUNTIME_PATHS.profile_dir
SCRAPER = PROJECT_DIR / "douyin_comment_scraper.py"
LOGIN_COOKIE = "sessionid"
def reset_state():
state = begin((COOKIE_FILE, STATE_FILE), PROFILE_DIR)
print(" [OK] Old Cookie/Profile staged; failures will roll back automatically")
return state
def is_cookie_valid() -> bool:
if not COOKIE_FILE.exists():
return False
try:
cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
return any(
isinstance(cookie, dict)
and cookie.get("name") == LOGIN_COOKIE
and cookie.get("value")
for cookie in cookies
)
except Exception:
return False
def main() -> int:
parser = argparse.ArgumentParser(description="Relogin Douyin")
parser.add_argument("--login-timeout", type=int, default=300)
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
print("== Relogin: Douyin ==")
print(f" date: {dt.date.today()}")
print(f" cookie: {COOKIE_FILE}")
print(f" profile: {PROFILE_DIR}")
print(f" current cookie: {'present' if is_cookie_valid() else 'missing/invalid'}")
if args.dry_run:
print("[DRY-RUN] Would stage Cookie/Profile and open the QR login window")
return 0
old_state = reset_state()
command = [
sys.executable,
str(SCRAPER),
"login",
"--login-only",
"--login-timeout",
str(args.login_timeout),
]
print(f"\n [RUN] {' '.join(command)}")
try:
return_code = subprocess.call(
command,
cwd=str(PROJECT_DIR),
env=child_browser_environment(
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
profile_dir=PROFILE_DIR,
),
)
except BaseException:
rollback(old_state)
raise
login_ok = return_code == 0 and is_cookie_valid()
print(f"\n exit={return_code}")
print(f" {LOGIN_COOKIE} written: {'OK' if login_ok else 'FAIL'}")
if not login_ok:
rollback(old_state)
print(" [ROLLBACK] Login incomplete; restored previous Cookie/Profile")
return return_code or 1
try:
sync_relogin_state_to_consumers(
"douyin",
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
)
except Exception as exc:
rollback(old_state)
print(f" [ROLLBACK] Failed to sync collector login state: {exc}")
return 1
commit(old_state)
return 0
if __name__ == "__main__":
raise SystemExit(main())
src/gyxx_flow/modules/content_marketing/data/tools/relogin_pgy.py
-148
View File
@@ -1,148 +0,0 @@
"""每周一重登小红书蒲公英。
强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。
任务计划: 每周一 09:00
跑法:
python data/tools/relogin_pgy.py # 实际跑(会弹二维码)
python data/tools/relogin_pgy.py --dry-run # 只看会做什么
"""
import argparse
import datetime as dt
import shutil
import subprocess
import sys
from pathlib import Path
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
child_browser_environment,
resolve_relogin_runtime_paths,
sync_relogin_state_to_consumers,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
PROJECT_DIR = PATHS.module_root
RUNTIME_PATHS = resolve_relogin_runtime_paths("pgy")
COOKIE_FILE = RUNTIME_PATHS.cookie_file
STATE_FILE = RUNTIME_PATHS.storage_state_file
PROFILE_DIR = RUNTIME_PATHS.profile_dir
SCRAPER = PROJECT_DIR / "pgy_xhs_scraper_v2.py"
def reset_state() -> tuple[Path | None, Path | None, Path | None]:
"""Temporarily move the old login state aside so it can be restored."""
ts = dt.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
cookie_backup = None
state_backup = None
profile_backup = None
if COOKIE_FILE.exists():
backup = COOKIE_FILE.with_suffix(f".json.bak.{ts}")
COOKIE_FILE.rename(backup)
cookie_backup = backup
print(f" [OK] 备份 cookie -> {backup.name}")
else:
print(f" [INFO] {COOKIE_FILE.name} 不存在,跳过")
if STATE_FILE.exists():
state_backup = STATE_FILE.with_suffix(f".json.bak.{ts}")
STATE_FILE.rename(state_backup)
print(f" [OK] 备份 storage state -> {state_backup.name}")
else:
print(f" [INFO] {STATE_FILE.name} 不存在,跳过")
if PROFILE_DIR.exists():
profile_backup = PROFILE_DIR.with_name(f"{PROFILE_DIR.name}.bak.{ts}")
PROFILE_DIR.rename(profile_backup)
print(f" [OK] 暂存 {PROFILE_DIR.name}/ -> {profile_backup.name}/")
else:
print(f" [INFO] {PROFILE_DIR.name}/ 不存在,跳过")
return cookie_backup, profile_backup, state_backup
def restore_previous_state(
cookie_backup: Path | None,
profile_backup: Path | None,
state_backup: Path | None = None,
) -> None:
"""Discard an incomplete login and restore the last known-good state."""
if COOKIE_FILE.exists():
COOKIE_FILE.unlink()
if cookie_backup and cookie_backup.exists():
cookie_backup.rename(COOKIE_FILE)
print(f" [ROLLBACK] 已恢复旧 cookie: {COOKIE_FILE.name}")
if STATE_FILE.exists():
STATE_FILE.unlink()
if state_backup and state_backup.exists():
state_backup.rename(STATE_FILE)
print(f" [ROLLBACK] 已恢复旧 storage state: {STATE_FILE.name}")
if PROFILE_DIR.exists():
shutil.rmtree(PROFILE_DIR)
if profile_backup and profile_backup.exists():
profile_backup.rename(PROFILE_DIR)
print(f" [ROLLBACK] 已恢复旧浏览器状态: {PROFILE_DIR.name}/")
def finish_successful_relogin(profile_backup: Path | None) -> None:
"""Keep the new state and remove the bulky old profile backup."""
if profile_backup and profile_backup.exists():
shutil.rmtree(profile_backup)
def main() -> int:
ap = argparse.ArgumentParser(description="每周一重登小红书蒲公英")
ap.add_argument("--login-timeout", type=int, default=300, help="扫码超时(秒)")
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不删不跑")
args = ap.parse_args()
print("== Relogin: 小红书蒲公英 ==")
print(f" 今天: {dt.date.today()} ({dt.date.today().strftime('%A')})")
print(f" cookie: {COOKIE_FILE}")
print(f" profile: {PROFILE_DIR}")
print()
if args.dry_run:
print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码")
return 0
cookie_backup, profile_backup, state_backup = reset_state()
cmd = [
sys.executable,
str(SCRAPER),
"--login-only",
"--login-timeout",
str(args.login_timeout),
]
print(f"\n [RUN] {' '.join(cmd)}")
try:
rc = subprocess.call(
cmd,
cwd=str(PROJECT_DIR),
env=child_browser_environment(
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
profile_dir=PROFILE_DIR,
),
)
except BaseException:
restore_previous_state(cookie_backup, profile_backup, state_backup)
raise
print(f"\n exit={rc}")
if rc != 0 or not COOKIE_FILE.exists():
print(" [WARN] 新登录未完成,恢复旧登录状态")
restore_previous_state(cookie_backup, profile_backup, state_backup)
return rc or 1
try:
sync_relogin_state_to_consumers(
"pgy",
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
)
except Exception as exc:
restore_previous_state(cookie_backup, profile_backup, state_backup)
print(f" [ROLLBACK] 同步采集器登录态失败: {exc}")
return 1
finish_successful_relogin(profile_backup)
return rc
if __name__ == "__main__":
sys.exit(main())
src/gyxx_flow/modules/content_marketing/data/tools/relogin_runtime_paths.py
-279
View File
@@ -1,279 +0,0 @@
"""Resolve one relogin wrapper's browser state contract.
The workflow runtime injects a unique browser binding into each relogin
wrapper. Standalone legacy launches do not have that environment, so they
retain the original per-platform files below the content-marketing state root.
"""
from __future__ import annotations
import json
import os
import re
import shutil
import tempfile
import uuid
from dataclasses import dataclass
from pathlib import Path
from typing import Mapping
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
_PLATFORM = re.compile(r"^[a-z][a-z0-9_-]{0,31}$")
CONSUMER_BINDING_IDS: Mapping[str, tuple[str, ...]] = {
"bilibili": ("content.comments.collect_bilibili",),
"douyin": ("content.comments.collect_douyin",),
"pgy": ("content_marketing:pgy_xhs_scraper_v2.py",),
"xiaohongshu": ("content.comments.collect_xiaohongshu",),
# v2 now reads the Buyin/Doudian site and is owned by the douyin-shop
# account. Keep the legacy Xingtu relogin isolated from that vault.
"buyin": ("content_marketing:xingtu_scraper_v2.py",),
"xingtu": ("content_marketing:xingtu_scraper.py",),
}
@dataclass(frozen=True)
class ReloginRuntimePaths:
cookie_file: Path
storage_state_file: Path
profile_dir: Path
def resolve_relogin_runtime_paths(
platform: str,
environment: Mapping[str, str] | None = None,
) -> ReloginRuntimePaths:
"""Prefer the injected binding and fall back to original platform paths."""
if not _PLATFORM.fullmatch(platform):
raise ValueError("invalid relogin platform")
values = os.environ if environment is None else environment
legacy_cookie = PATHS.state_root / "cookies" / f"{platform}_cookies.json"
legacy_storage = (
PATHS.state_root / "cookies" / f"{platform}_storage_state.json"
)
legacy_profile = PATHS.state_root / "browser-profiles" / platform
return ReloginRuntimePaths(
cookie_file=_bound_path(
values,
"GYXX_BROWSER_COOKIE_FILE",
legacy_cookie,
),
storage_state_file=_bound_path(
values,
"GYXX_BROWSER_STORAGE_STATE_FILE",
legacy_storage,
),
profile_dir=_bound_path(
values,
"GYXX_BROWSER_PROFILE_DIR",
legacy_profile,
),
)
def child_browser_environment(
*,
cookie_file: Path,
storage_state_file: Path,
profile_dir: Path,
environment: Mapping[str, str] | None = None,
) -> dict[str, str]:
"""Keep the nested login scraper on the wrapper's injected binding."""
result = dict(os.environ if environment is None else environment)
result.update(
{
"GYXX_BROWSER_COOKIE_FILE": str(cookie_file.resolve()),
"GYXX_BROWSER_STORAGE_STATE_FILE": str(
storage_state_file.resolve()
),
"GYXX_BROWSER_PROFILE_DIR": str(profile_dir.resolve()),
}
)
# content_marketing/sitecustomize.py normally rebinds a nested Python
# script from argv[0]. A relogin scraper is an implementation detail of
# this wrapper and must write the wrapper binding that the parent monitors.
result.pop("GYXX_MODULE_ID", None)
return result
def sync_relogin_state_to_consumers(
platform: str,
*,
cookie_file: Path,
storage_state_file: Path,
environment: Mapping[str, str] | None = None,
) -> tuple[Path, ...]:
"""Atomically publish a verified relogin state to unique consumer paths."""
try:
consumer_ids = CONSUMER_BINDING_IDS[platform]
except KeyError as exc:
raise ValueError("invalid relogin platform") from exc
values = os.environ if environment is None else environment
project_root = values.get("GYXX_PROJECT_ROOT", "").strip()
data_root = values.get("GYXX_DATA_ROOT", "").strip()
if not project_root and not data_root:
# Original standalone launches use the legacy per-platform state files
# directly, so there is no separate managed consumer to publish to.
return ()
if not project_root or not data_root:
raise RuntimeError("managed relogin state sync requires project and data roots")
catalog = RuntimeIntegrationCatalog.load_default(
project_root=project_root,
data_root=data_root,
)
source_cookie = cookie_file.expanduser().resolve()
source_storage = storage_state_file.expanduser().resolve()
cookie_payload = _read_cookie_payload(source_cookie)
storage_available = source_storage.is_file()
if storage_available:
_read_storage_state_payload(source_storage)
source_paths = {source_cookie, source_storage}
destination_paths: set[Path] = set()
destinations: list[Path] = []
updates: list[tuple[Path, Path]] = []
for consumer_id in consumer_ids:
binding = catalog.binding_for(consumer_id)
target_cookie = binding.cookie_file.resolve()
target_storage = binding.storage_state_file.resolve()
targets = (target_cookie, target_storage)
if any(target in source_paths for target in targets) or any(
target in destination_paths for target in targets
):
raise RuntimeError(
"relogin and consumer browser state paths must be unique"
)
destination_paths.update(targets)
_validate_cookie_requirements(
cookie_payload,
required_domains=binding.required_cookie_domains,
required_names=binding.required_cookie_names,
)
updates.append((source_cookie, target_cookie))
if storage_available:
updates.append((source_storage, target_storage))
destinations.append(target_cookie)
_atomic_replace_files(updates)
return tuple(destinations)
def _bound_path(
environment: Mapping[str, str],
name: str,
fallback: Path,
) -> Path:
configured = environment.get(name, "").strip()
return (
Path(configured).expanduser().resolve()
if configured
else fallback.resolve()
)
def _read_cookie_payload(path: Path) -> list[dict[str, object]]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, ValueError) as exc:
raise RuntimeError("relogin cookie state is missing or invalid") from exc
if not isinstance(payload, list) or not payload or not all(
isinstance(cookie, dict) for cookie in payload
):
raise RuntimeError("relogin cookie state is missing or invalid")
return payload
def _read_storage_state_payload(path: Path) -> dict[str, object]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, ValueError) as exc:
raise RuntimeError("relogin storage state is invalid") from exc
if not isinstance(payload, dict):
raise RuntimeError("relogin storage state is invalid")
return payload
def _validate_cookie_requirements(
cookies: list[dict[str, object]],
*,
required_domains: tuple[str, ...],
required_names: tuple[str, ...],
) -> None:
usable = [cookie for cookie in cookies if cookie.get("value")]
if not usable:
raise RuntimeError("relogin cookie state has no usable cookies")
if required_domains and not any(
any(
str(cookie.get("domain", "")).lstrip(".").casefold()
.endswith(domain.casefold().lstrip("."))
for domain in required_domains
)
for cookie in usable
):
raise RuntimeError("relogin cookie state does not match consumer domain")
if required_names and not any(
str(cookie.get("name", "")) in required_names for cookie in usable
):
raise RuntimeError("relogin cookie state misses the consumer login cookie")
def _atomic_replace_files(updates: list[tuple[Path, Path]]) -> None:
"""Replace a small related file set and restore every old target on error."""
transaction = uuid.uuid4().hex
staged: list[tuple[Path, Path]] = []
backups: dict[Path, Path | None] = {}
replaced: list[Path] = []
try:
for source, target in updates:
target.parent.mkdir(parents=True, exist_ok=True)
handle, raw_temp = tempfile.mkstemp(
prefix=f".{target.name}.{transaction}.",
suffix=".tmp",
dir=target.parent,
)
temp = Path(raw_temp)
with os.fdopen(handle, "wb") as stream:
stream.write(source.read_bytes())
stream.flush()
os.fsync(stream.fileno())
staged.append((temp, target))
backup = target.with_name(f".{target.name}.{transaction}.bak")
if target.is_file():
shutil.copy2(target, backup)
backups[target] = backup
else:
backups[target] = None
for temp, target in staged:
os.replace(temp, target)
replaced.append(target)
except BaseException:
for target in reversed(replaced):
backup = backups.get(target)
if backup is not None and backup.exists():
os.replace(backup, target)
elif target.exists():
target.unlink()
raise
finally:
for temp, _target in staged:
if temp.exists():
temp.unlink()
for backup in backups.values():
if backup is not None and backup.exists():
backup.unlink()
__all__ = [
"CONSUMER_BINDING_IDS",
"ReloginRuntimePaths",
"child_browser_environment",
"resolve_relogin_runtime_paths",
"sync_relogin_state_to_consumers",
]
src/gyxx_flow/modules/content_marketing/data/tools/relogin_transaction.py
-47
View File
@@ -1,47 +0,0 @@
"""Transactional backup/rollback helpers for interactive relogin scripts."""
from __future__ import annotations
import datetime as dt
import shutil
from dataclasses import dataclass
from pathlib import Path
@dataclass
class LoginStateBackup:
files: dict[Path, Path]
profile: Path
profile_backup: Path | None
def begin(files: tuple[Path, ...], profile: Path) -> LoginStateBackup:
stamp = dt.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
backups: dict[Path, Path] = {}
for source in files:
if source.exists():
backup = source.with_name(f"{source.name}.bak.{stamp}")
source.rename(backup)
backups[source] = backup
profile_backup = None
if profile.exists():
profile_backup = profile.with_name(f"{profile.name}.bak.{stamp}")
profile.rename(profile_backup)
return LoginStateBackup(backups, profile, profile_backup)
def rollback(state: LoginStateBackup) -> None:
for source, backup in state.files.items():
if source.exists():
source.unlink()
if backup.exists():
backup.rename(source)
if state.profile.exists():
shutil.rmtree(state.profile)
if state.profile_backup and state.profile_backup.exists():
state.profile_backup.rename(state.profile)
def commit(state: LoginStateBackup) -> None:
if state.profile_backup and state.profile_backup.exists():
shutil.rmtree(state.profile_backup)
src/gyxx_flow/modules/content_marketing/data/tools/relogin_xiaohongshu.py
-116
View File
@@ -1,116 +0,0 @@
"""每周重登小红书。
强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。
跑法:
python data/tools/relogin_xiaohongshu.py # 实际跑(会弹二维码)
python data/tools/relogin_xiaohongshu.py --dry-run # 只看会做什么
"""
import argparse
import datetime as dt
import json
import subprocess
import sys
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
child_browser_environment,
resolve_relogin_runtime_paths,
sync_relogin_state_to_consumers,
)
from gyxx_flow.modules.content_marketing.data.tools.relogin_transaction import (
begin,
commit,
rollback,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
PROJECT_DIR = PATHS.module_root
RUNTIME_PATHS = resolve_relogin_runtime_paths("xiaohongshu")
COOKIE_FILE = RUNTIME_PATHS.cookie_file
STATE_FILE = RUNTIME_PATHS.storage_state_file
PROFILE_DIR = RUNTIME_PATHS.profile_dir
SCRAPER = PROJECT_DIR / "xiaohongshu_comment_scraper.py"
LOGIN_COOKIE = "web_session"
def reset_state():
state = begin((COOKIE_FILE, STATE_FILE), PROFILE_DIR)
print(" [OK] 旧 Cookie/Profile 已暂存,失败时将自动恢复")
return state
def is_cookie_valid() -> bool:
if not COOKIE_FILE.exists():
return False
try:
cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
return any(c.get("name") == LOGIN_COOKIE and c.get("value") for c in cookies)
except Exception:
return False
def main() -> int:
ap = argparse.ArgumentParser(description="重登小红书")
ap.add_argument("--login-timeout", type=int, default=300, help="扫码超时(秒)")
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不删不跑")
args = ap.parse_args()
print("== Relogin: 小红书 ==")
print(f" 今天: {dt.date.today()}")
print(f" cookie: {COOKIE_FILE}")
print(f" profile: {PROFILE_DIR}")
print(f" 当前 cookie 状态: {'有效' if is_cookie_valid() else '无效/不存在'}")
print()
if args.dry_run:
print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码")
return 0
old_state = reset_state()
cmd = [
sys.executable,
str(SCRAPER),
"login",
"--login-only",
"--login-timeout",
str(args.login_timeout),
]
print(f"\n [RUN] {' '.join(cmd)}")
try:
rc = subprocess.call(
cmd,
cwd=str(PROJECT_DIR),
env=child_browser_environment(
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
profile_dir=PROFILE_DIR,
),
)
except BaseException:
rollback(old_state)
raise
print(f"\n exit={rc}")
login_ok = rc == 0 and is_cookie_valid()
if rc == 0:
print(f" {LOGIN_COOKIE} 写入: {'OK' if login_ok else 'FAIL — 没有看到 web_session,登录可能没成功'}")
if not login_ok:
rollback(old_state)
print(" [ROLLBACK] 未完成扫码,已恢复旧登录状态")
return rc or 1
try:
sync_relogin_state_to_consumers(
"xiaohongshu",
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
)
except Exception as exc:
rollback(old_state)
print(f" [ROLLBACK] 同步采集器登录态失败: {exc}")
return 1
commit(old_state)
return 0
if __name__ == "__main__":
sys.exit(main())
src/gyxx_flow/modules/content_marketing/data/tools/relogin_xingtu.py
-350
View File
@@ -1,350 +0,0 @@
"""每 3 个周一重登一次抖音/星图。
强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。
星图登录完成后,自动把登录态 cookie 同步到抖音评论 scraper,避免再扫一次码。
任务计划: 每周一 09:00 (脚本内部判断是否到了该重登的周一,不到就退出)
3 周循环基准: 任意一个周一,后续 (今天 - BASE_DATE) // 7 % 3 == 0 才是星图周。
改 BASE_DATE 就改了重登周期。
跑法:
python data/tools/relogin_xingtu.py # 实际跑
python data/tools/relogin_xingtu.py --dry-run # 看今天是不是星图周
python data/tools/relogin_xingtu.py --force # 忽略 3 周判断,强制重登
"""
import argparse
import datetime as dt
import json
import os
import shutil
import subprocess
import sys
from pathlib import Path
import psutil
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
child_browser_environment,
resolve_relogin_runtime_paths,
sync_relogin_state_to_consumers,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
PROJECT_DIR = PATHS.module_root
RUNTIME_PATHS = resolve_relogin_runtime_paths("xingtu")
COOKIE_FILE = RUNTIME_PATHS.cookie_file
STATE_FILE = RUNTIME_PATHS.storage_state_file
PROFILE_DIR = RUNTIME_PATHS.profile_dir
SCRAPER = PROJECT_DIR / "xingtu_scraper_v2.py"
# 抖音评论 scraper 的 cookie 文件 (登录态共享目标)
DY_COOKIE_FILE = PATHS.state_root / "cookies/douyin_cookies.json"
DY_STATE_FILE = PATHS.state_root / "cookies/douyin_storage_state.json"
DY_PROFILE_DIR = PATHS.state_root / "browser-profiles/douyin"
# 需要同步的抖音登录 cookie 名
DY_LOGIN_COOKIES = {"sessionid", "sessionid_ss", "sid_guard", "sid_tt", "passport_csrf_token"}
# 3 周循环起点(必须是周一)。星图 cookie ~22 天,每 3 周(21 天)重登正好。
BASE_DATE = dt.date(2026, 6, 22)
LOGIN_COOKIE_NAMES = {"sessionid", "sessionid_ss", "sid_tt", "sid_guard"}
def transaction_file() -> Path:
return COOKIE_FILE.with_name(".xingtu_relogin_transaction.json")
def _write_transaction(
cookie_backup: Path | None,
profile_backup: Path | None,
state_backup: Path | None,
) -> None:
marker = transaction_file()
marker.parent.mkdir(parents=True, exist_ok=True)
temp = marker.with_name(f"{marker.name}.tmp")
temp.write_text(json.dumps({
"cookie_backup": str(cookie_backup) if cookie_backup else "",
"state_backup": str(state_backup) if state_backup else "",
"profile_backup": str(profile_backup) if profile_backup else "",
}, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(temp, marker)
def _clear_transaction() -> None:
marker = transaction_file()
if marker.exists():
marker.unlink()
def _remove_path(path: Path) -> None:
if path.is_dir():
shutil.rmtree(path)
elif path.exists():
path.unlink()
def recover_interrupted_state() -> bool:
"""Restore a relogin transaction that was killed after moving old state."""
marker = transaction_file()
if not marker.exists():
return False
try:
state = json.loads(marker.read_text(encoding="utf-8"))
except Exception:
return False
cookie_backup = Path(state["cookie_backup"]) if state.get("cookie_backup") else None
state_backup = Path(state["state_backup"]) if state.get("state_backup") else None
profile_backup = Path(state["profile_backup"]) if state.get("profile_backup") else None
terminate_profile_processes()
if cookie_backup and cookie_backup.exists():
_remove_path(COOKIE_FILE)
cookie_backup.rename(COOKIE_FILE)
if state_backup and state_backup.exists():
_remove_path(STATE_FILE)
state_backup.rename(STATE_FILE)
if profile_backup and profile_backup.exists():
_remove_path(PROFILE_DIR)
profile_backup.rename(PROFILE_DIR)
_clear_transaction()
print(" [RECOVER] 已恢复上次被中断的星图登录状态")
return True
def has_valid_login_cookie_file(path: Path | None = None) -> bool:
path = path or COOKIE_FILE
try:
cookies = json.loads(path.read_text(encoding="utf-8"))
except Exception:
return False
return any(
isinstance(cookie, dict)
and cookie.get("name") in LOGIN_COOKIE_NAMES
and cookie.get("value")
for cookie in cookies
)
def terminate_profile_processes() -> None:
"""Stop only browser processes using Xingtu's dedicated profile."""
profile_text = str(PROFILE_DIR.resolve()).casefold()
matches = []
for process in psutil.process_iter(["pid", "cmdline"]):
try:
command = " ".join(process.info.get("cmdline") or []).casefold()
if process.pid != os.getpid() and profile_text in command:
matches.append(process)
except (psutil.Error, OSError):
continue
for process in matches:
try:
process.terminate()
except (psutil.Error, OSError):
pass
_gone, alive = psutil.wait_procs(matches, timeout=5)
for process in alive:
try:
process.kill()
except (psutil.Error, OSError):
pass
def is_xingtu_week(today: dt.date, base: dt.date) -> bool:
days = (today - base).days
if days < 0:
return False
return (days // 7) % 3 == 0
def reset_state() -> tuple[Path | None, Path | None, Path | None]:
"""Temporarily move the old login state aside so it can be restored."""
ts = dt.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
cookie_backup = None
state_backup = None
profile_backup = None
if COOKIE_FILE.exists():
cookie_backup = COOKIE_FILE.with_suffix(f".json.bak.{ts}")
if STATE_FILE.exists():
state_backup = STATE_FILE.with_suffix(f".json.bak.{ts}")
if PROFILE_DIR.exists():
profile_backup = PROFILE_DIR.with_name(f"{PROFILE_DIR.name}.bak.{ts}")
_write_transaction(cookie_backup, profile_backup, state_backup)
if cookie_backup:
COOKIE_FILE.rename(cookie_backup)
print(f" [OK] 备份 cookie -> {cookie_backup.name}")
else:
print(f" [INFO] {COOKIE_FILE.name} 不存在,跳过")
if state_backup:
STATE_FILE.rename(state_backup)
print(f" [OK] 备份 storage state -> {state_backup.name}")
else:
print(f" [INFO] {STATE_FILE.name} 不存在,跳过")
if profile_backup:
PROFILE_DIR.rename(profile_backup)
print(f" [OK] 暂存 {PROFILE_DIR.name}/ -> {profile_backup.name}/")
else:
print(f" [INFO] {PROFILE_DIR.name}/ 不存在,跳过")
return cookie_backup, profile_backup, state_backup
def restore_previous_state(
cookie_backup: Path | None,
profile_backup: Path | None,
state_backup: Path | None = None,
) -> None:
"""Discard an incomplete login and restore the last known-good state."""
terminate_profile_processes()
if COOKIE_FILE.exists():
COOKIE_FILE.unlink()
if cookie_backup and cookie_backup.exists():
cookie_backup.rename(COOKIE_FILE)
print(f" [ROLLBACK] 已恢复旧 cookie: {COOKIE_FILE.name}")
_remove_path(STATE_FILE)
if state_backup and state_backup.exists():
state_backup.rename(STATE_FILE)
print(f" [ROLLBACK] 已恢复旧 storage state: {STATE_FILE.name}")
if PROFILE_DIR.exists():
shutil.rmtree(PROFILE_DIR)
if profile_backup and profile_backup.exists():
profile_backup.rename(PROFILE_DIR)
print(f" [ROLLBACK] 已恢复旧浏览器状态: {PROFILE_DIR.name}/")
_clear_transaction()
def finish_successful_relogin(profile_backup: Path | None) -> None:
"""Keep the new state and remove the bulky old profile backup."""
if profile_backup and profile_backup.exists():
shutil.rmtree(profile_backup)
_clear_transaction()
def sync_to_douyin_scraper() -> None:
"""把星图的登录态 cookie 同步到抖音评论 scraper,避免再扫一次码。"""
if not COOKIE_FILE.exists():
print(" [SKIP] 星图 cookie 不存在,无法同步")
return
try:
xingtu_cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
except Exception as exc:
print(f" [SKIP] 读取星图 cookie 失败: {exc}")
return
# 提取登录态 cookie
login_cookies = [c for c in xingtu_cookies if isinstance(c, dict) and c.get("name") in DY_LOGIN_COOKIES]
if not login_cookies:
print(" [SKIP] 星图 cookie 中没有找到登录态,跳过同步")
return
# 清除抖音评论 scraper 的旧 profile (强制刷新)
if DY_PROFILE_DIR.exists():
shutil.rmtree(DY_PROFILE_DIR)
print(f" [OK] 清空 {DY_PROFILE_DIR.name}/")
# 读取或创建抖音评论 cookie 文件
dy_cookies = []
if DY_COOKIE_FILE.exists():
try:
dy_cookies = json.loads(DY_COOKIE_FILE.read_text(encoding="utf-8"))
except Exception:
dy_cookies = []
# 合并: 同名 cookie 覆盖,新 cookie 追加
dy_by_name = {c.get("name"): c for c in dy_cookies if isinstance(c, dict)}
for c in login_cookies:
# 修正 domain: 星图的 cookie domain 可能是 .douyin.com 子域,评论采集需要 .douyin.com
domain = c.get("domain", "")
if "xingtu" in domain or "oceanengine" in domain:
c["domain"] = ".douyin.com"
dy_by_name[c["name"]] = c
DY_COOKIE_FILE.parent.mkdir(parents=True, exist_ok=True)
DY_COOKIE_FILE.write_text(
json.dumps(list(dy_by_name.values()), ensure_ascii=False, indent=2),
encoding="utf-8",
)
print(f" [OK] 同步 {len(login_cookies)} 个登录 cookie 到 {DY_COOKIE_FILE.name}")
def main() -> int:
ap = argparse.ArgumentParser(description="每 3 周周一重登星图")
ap.add_argument("--base-date", default=BASE_DATE.isoformat(),
help="3 周循环起点(必须是周一),格式 YYYY-MM-DD")
ap.add_argument("--login-timeout", type=int, default=300)
ap.add_argument("--force", action="store_true", help="忽略 3 周判断,强制重登")
ap.add_argument("--dry-run", action="store_true", help="只打印计划")
args = ap.parse_args()
today = dt.date.today()
base = dt.date.fromisoformat(args.base_date)
is_xw = is_xingtu_week(today, base)
print("== Relogin: 抖音/星图 ==")
print(f" 今天: {today} ({today.strftime('%A')})")
print(f" 3 周基准: {base}")
print(f" 星图周? {'是' if is_xw else '否'}")
print()
if not is_xw and not args.force:
print("今天不是星图周,跳过 (传 --force 强制重登)")
return 0
if args.dry_run:
print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码")
return 0
recover_interrupted_state()
terminate_profile_processes()
cookie_backup, profile_backup, state_backup = reset_state()
cmd = [
sys.executable,
str(SCRAPER),
"--login-only",
"--login-timeout",
str(args.login_timeout),
]
print(f"\n [RUN] {' '.join(cmd)}")
try:
rc = subprocess.call(
cmd,
cwd=str(PROJECT_DIR),
env=child_browser_environment(
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
profile_dir=PROFILE_DIR,
),
)
except BaseException:
restore_previous_state(cookie_backup, profile_backup, state_backup)
raise
finally:
terminate_profile_processes()
print(f"\n exit={rc}")
if rc != 0 or not has_valid_login_cookie_file():
print(" [WARN] 新登录未完成,恢复旧登录状态")
restore_previous_state(cookie_backup, profile_backup, state_backup)
return rc or 1
try:
sync_relogin_state_to_consumers(
"xingtu",
cookie_file=COOKIE_FILE,
storage_state_file=STATE_FILE,
)
except Exception as exc:
restore_previous_state(cookie_backup, profile_backup, state_backup)
print(f" [ROLLBACK] 同步采集器登录态失败: {exc}")
return 1
finish_successful_relogin(profile_backup)
if rc == 0:
sync_to_douyin_scraper()
return rc
if __name__ == "__main__":
sys.exit(main())
src/gyxx_flow/modules/content_marketing/douyin_comment_scraper.py
+9 -51
View File
@@ -273,18 +273,15 @@ def maybe_wait_for_login(
*,
deadline: float | None = None,
) -> None:
login_deadline = time.monotonic() + login_timeout
while time.monotonic() < login_deadline:
_check_note_deadline(deadline, "login wait")
# 登录统一由账号 vault 管理（gyxx accounts login douyin-content），
# 由工作流执行前的 cookie 网关保证；运行中失效立即降级，不再等待人工扫码。
try:
body = page.locator("body").inner_text(timeout=5000)
except Exception:
body = ""
if "扫码登录" not in body and "登录后" not in body:
return
log("Douyin may require login. Please finish login in the opened Chrome window.")
page.wait_for_timeout(3000)
_check_note_deadline(deadline, "login wait")
if "扫码登录" in body or "登录后" in body:
log("Douyin login state missing; collecting degraded samples. "
"Run `gyxx accounts login douyin-content` to restore full comments.")
def is_logged_in(page: Page) -> bool:
@@ -833,55 +830,16 @@ def write_outputs(url: str, comments: list[dict[str, Any]], result: dict[str, An
return json_path, csv_path
def login_only(login_timeout: int) -> int:
"""只触发登录(headed),登录完成立刻退出。给 relogin_douyin.py 用。"""
def action(page: Page) -> None:
page.goto("https://www.douyin.com/", wait_until="domcontentloaded", timeout=90000)
page.wait_for_timeout(2500)
# 主动触发登录弹窗(点登录按钮)
try:
login_btn = page.get_by_text("登录", exact=True).first
if login_btn.count():
login_btn.click(timeout=3000)
page.wait_for_timeout(1500)
except Exception:
pass
maybe_wait_for_login(page, login_timeout)
save_state(page)
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
with ScraplingBrowser(
headless=False,
real_chrome=True,
user_data_dir=str(PROFILE_DIR),
locale="zh-CN",
timezone_id="Asia/Shanghai",
timeout=90000,
network_idle=False,
disable_resources=False,
google_search=False,
page_setup=restore_cookies,
max_pages=1,
) as browser:
browser.run_fetch_action("https://www.douyin.com/", action, wait=1000)
log("[login-only] 完成")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description="Scrape Douyin video comments via Scrapling browser and XHR interception.")
parser.add_argument("url", help="Douyin video/user URL (传 'login' 配合 --login-only)")
parser.add_argument("--login-timeout", type=int, default=300, help="等待扫码/登录秒数")
parser.add_argument("url", help="Douyin video/user URL")
parser.add_argument("--login-timeout", type=int, default=300,
help="保留参数（登录等待已移除；未登录将直接降级采集）")
parser.add_argument("--max-scrolls", type=int, default=120, help="最多滚动加载次数")
parser.add_argument("--idle-rounds", type=int, default=8, help="连续几轮无新评论后停止")
parser.add_argument("--headless", action="store_true", help="无头模式；首次登录不要使用")
parser.add_argument("--login-only", action="store_true",
help="只触发登录(headed)然后退出,给 relogin_douyin 用")
parser.add_argument("--headless", action="store_true", help="无头模式")
args = parser.parse_args()
if args.login_only:
return login_only(args.login_timeout)
comments, result = scrape_comments(args.url, args.login_timeout, args.max_scrolls, args.idle_rounds, args.headless)
json_path, csv_path = write_outputs(args.url, comments, result)
log(f"Final URL: {result.get('final_url', '')}")
src/gyxx_flow/modules/content_marketing/pgy_xhs_scraper_v2.py
+8 -107
View File
@@ -38,6 +38,7 @@ from gyxx_flow.adapters.scrapling import (
BrowserTimeoutError, # noqa: F401
is_browser_timeout_error,
)
from gyxx_flow.adapters.browser import CookieExpiredError, cookie_expired_message
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
from gyxx_flow.modules.content_marketing.collection_completeness import (
BLOCKED_INPUT,
@@ -333,56 +334,6 @@ def is_logged_in(page: Page) -> bool:
return False
def open_scan_login(page: Page) -> None:
"""Open the login dialog and prefer its QR-code login tab."""
for pattern in (r"^登录$|账号登录",):
try:
button = page.get_by_text(re.compile(pattern)).first
if button.count() and button.is_visible():
button.click(timeout=2000)
page.wait_for_timeout(500)
except Exception:
pass
# The login dialog uses an image-only folded-corner switch. Its bundled
# asset name is stable even though the generated CSS class is not.
try:
qr_switch = page.locator('img[src*="qr_code"]').first
if qr_switch.count() and qr_switch.is_visible():
# The browser engine's actionability check is blocked by the folded-corner
# overlay. A native DOM click triggers the image's real handler.
qr_switch.evaluate("e => e.click()")
page.wait_for_timeout(1000)
return
except Exception:
pass
for pattern in (r"扫码登录|二维码登录|扫码",):
try:
button = page.get_by_text(re.compile(pattern)).first
if button.count() and button.is_visible():
button.click(timeout=2000)
page.wait_for_timeout(500)
except Exception:
pass
def wait_for_scan_login(page: Page, timeout_seconds: int) -> None:
log("Not logged in. 请在打开的 Chrome 中扫码登录。")
open_scan_login(page)
deadline = time.time() + timeout_seconds
next_login_click = time.time() + 5
while time.time() < deadline:
if is_logged_in(page):
save_state(page)
return
if time.time() >= next_login_click:
open_scan_login(page)
next_login_click = time.time() + 5
page.wait_for_timeout(2000)
raise TimeoutError(f"Login timed out after {timeout_seconds} seconds.")
def ensure_login(page: Page, login_timeout: int) -> None:
page.goto(HOME_URL, wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(2500)
@@ -402,20 +353,11 @@ def ensure_login(page: Page, login_timeout: int) -> None:
)
log(f"[SKIPPED_COOKIE] {reason}")
raise AcceptanceCookieSkip(reason)
page.goto(LOGIN_URL, wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(1500)
wait_for_scan_login(page, login_timeout)
page.goto(HOME_URL, wait_until="domcontentloaded", timeout=60000)
page.wait_for_load_state("domcontentloaded")
# 关闭弹窗
for text in ("跳过", "不再提示", "我知道了"):
try:
b = page.get_by_text(text, exact=True).first
if b.count():
b.click(timeout=1500)
page.wait_for_timeout(800)
except Exception:
pass
# 登录统一由账号 vault 管理（gyxx accounts login xiaohongshu-content），
# 采集脚本不再内嵌扫码等待；运行中失效直接快速失败。
reason = cookie_expired_message("xiaohongshu-content", "小红书蒲公英")
log(f"[COOKIE_EXPIRED] {reason}")
raise CookieExpiredError(reason)
def _is_login_timeout(exc: BaseException) -> bool:
@@ -1933,7 +1875,7 @@ def scrape_styles(styles: list[dict], login_timeout: int, headless: bool,
action,
page_setup=restore_cookies,
)
except AcceptanceCookieSkip:
except (AcceptanceCookieSkip, CookieExpiredError):
raise
except Exception as exc:
session_lost = is_browser_session_lost(exc)
@@ -2044,39 +1986,6 @@ def parse_int(s) -> int | None:
return int(num)
def login_only(login_timeout: int) -> int:
"""只触发登录(headed),登录完成立刻退出。给 weekly_relogin.py 用。"""
def action(page):
ensure_login(page, login_timeout)
return page
session = None
try:
with DynamicSession(
headless=False,
real_chrome=True,
user_data_dir=str(PROFILE_DIR),
locale="zh-CN",
timezone_id="Asia/Shanghai",
timeout=60000,
network_idle=False,
disable_resources=False,
google_search=False,
page_setup=restore_cookies,
max_pages=2,
) as session:
session.fetch(HOME_URL, page_action=action, wait=1000)
except Exception as exc:
if "TargetClosedError" in type(exc).__name__ or "has been closed" in str(exc):
log(f" [WARN] 浏览器清理时报错: {type(exc).__name__}")
else:
raise
finally:
_force_cleanup_session(session)
log("[login-only] 登录完成,退出")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description="小红书蒲公英 V2: 多维表格驱动精准采集+回填")
add_repeatable_style_argument(parser, "只跑指定款式编号(可多次)")
@@ -2090,8 +1999,6 @@ def main() -> int:
parser.add_argument("--skip-done", action="store_true", help="跳过已完成的款式(从进度文件读)")
parser.add_argument("--inter-style-pause", type=int, default=5,
help="兼容旧参数；全局按达人采集后不再按款式暂停")
parser.add_argument("--login-only", action="store_true",
help="只触发登录(headed),登录完成立刻退出,给 weekly_relogin 用")
parser.add_argument("--self-operated", action="store_true",
help="抓取自营达人表格(而非合作达人)")
parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE,
@@ -2108,12 +2015,6 @@ def main() -> int:
help=argparse.SUPPRESS)
args = parser.parse_args()
if args.login_only:
try:
return login_only(args.login_timeout)
except AcceptanceCookieSkip:
return COOKIE_SKIP_EXIT_CODE
mapping = load_mapping(self_operated=args.self_operated)
styles = mapping["tables"]
if args.style:
@@ -2213,7 +2114,7 @@ def main() -> int:
published_from=args.published_from,
max_age_days=args.max_age_days, no_retry=args.no_retry,
)
except AcceptanceCookieSkip:
except (AcceptanceCookieSkip, CookieExpiredError):
return COOKIE_SKIP_EXIT_CODE
except Exception as exc:
if not _is_login_timeout(exc):
src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py
+2 -59
View File
@@ -363,8 +363,8 @@ def log(msg: str) -> None:
# ============================================================
# B站登录态: Cookie 持久化 + 扫码登录引导
# 空间投稿列表需要登录态,首次用 --login-only 扫码,之后调度运行复用。
# B站登录态: Cookie 持久化；登录统一由账号 vault 管理
# (gyxx accounts login bilibili-content)，脚本内不再提供扫码登录。
# ============================================================
BILI_COOKIE_FILE = PATHS.browser_cookie_file
@@ -389,56 +389,6 @@ def save_state(page) -> None:
log(f"已保存 Cookie ({len(cookies)} 条): {BILI_COOKIE_FILE}")
def _has_login_cookies(page) -> bool:
names = {cookie.get("name") for cookie in page.context.cookies()}
return "SESSDATA" in names and "DedeUserID" in names
def login_only(login_timeout: int = 300) -> int:
"""打开 B站扫码登录页,等待人工扫码后保存 Cookie。"""
from scrapling.fetchers import StealthySession
with StealthySession(
headless=False,
real_chrome=True,
user_data_dir=str(PATHS.browser_profile_dir),
locale="zh-CN",
timezone_id="Asia/Shanghai",
timeout=60_000,
network_idle=False,
disable_resources=False,
google_search=False,
max_pages=2,
page_setup=restore_cookies,
) as session:
page = (
session.context.pages[0]
if session.context.pages
else session.context.new_page()
)
page.goto("https://passport.bilibili.com/login",
wait_until="domcontentloaded", timeout=60_000)
page.wait_for_timeout(2000)
if _has_login_cookies(page):
log("✅ 已有 B站登录态,无需扫码")
save_state(page)
return 0
print("\n" + "=" * 60)
print("🔐 请在打开的浏览器中用哔哩哔哩 App 扫码登录")
print(" 登录成功后脚本会自动继续并保存 Cookie")
print("=" * 60, flush=True)
deadline = time.time() + login_timeout
while time.time() < deadline:
if _has_login_cookies(page):
log("✅ 检测到登录成功")
page.wait_for_timeout(2000)
save_state(page)
return 0
page.wait_for_timeout(2000)
log(f"❌ 等待扫码超时({login_timeout}s),未保存登录态")
return 1
# ============================================================
# 飞书多维表格
# ============================================================
@@ -846,17 +796,10 @@ def main() -> int:
description="自营 B 站采集: 公开 API 全量视频指标 + UP 主主页全量投稿 + 飞书曝光回填")
parser.add_argument("--style", type=int, help="只跑指定款式编号")
parser.add_argument("--dry-run", action="store_true", help="不写回飞书")
parser.add_argument("--login-only", action="store_true",
help="只打开 B站扫码登录并保存 Cookie(首次部署引导)")
parser.add_argument("--login-timeout", type=int, default=300,
help="等待人工扫码的秒数")
parser.add_argument("--delay", type=float, default=0.3,
help="每条请求间隔秒数(防风控)")
args = parser.parse_args()
if args.login_only:
return login_only(args.login_timeout)
from gyxx_flow.modules.content_marketing import feishu_mapping
try:
mapping = feishu_mapping.load_mapping(self_operated=True)
src/gyxx_flow/modules/content_marketing/self_douyin_scraper.py
+11 -38
View File
@@ -27,6 +27,8 @@ from typing import Any
from scrapling.fetchers import DynamicSession
from gyxx_flow.adapters.scrapling import BrowserPage as Page
from gyxx_flow.adapters.browser import CookieExpiredError, cookie_expired_message
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
from gyxx_flow.modules.content_marketing.collection_completeness import (
BLOCKED_INPUT,
@@ -261,17 +263,16 @@ def save_state(page: Page) -> None:
def maybe_wait_for_login(page: Page, login_timeout: int) -> None:
deadline = time.time() + login_timeout
while time.time() < deadline:
# 登录统一由账号 vault 管理（gyxx accounts login douyin-content），
# 采集过程遇到登录态失效立即快速失败，不再等待人工扫码。
try:
body = page.locator("body").inner_text(timeout=5000)
except Exception:
body = ""
if "扫码登录" not in body and "登录后" not in body:
return
log("抖音需要登录,请在打开的 Chrome 窗口中完成登录")
page.wait_for_timeout(3000)
log("[WARN] 登录等待超时")
if "扫码登录" in body or "登录后" in body:
reason = cookie_expired_message("douyin-content", "自营抖音")
log(f"[COOKIE_EXPIRED] {reason}")
raise CookieExpiredError(reason)
def dismiss_popups(page: Page) -> None:
@@ -605,33 +606,6 @@ def scrape_one_style(style: dict, login_timeout: int, headless: bool,
return summary
def login_only(login_timeout: int) -> int:
"""只触发登录"""
login_done = False
def page_action(page: Page):
restore_cookies(page)
page.goto("https://www.douyin.com/", wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(2000)
maybe_wait_for_login(page, login_timeout)
save_state(page)
log("登录完成")
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
with DynamicSession(
headless=False,
real_chrome=True,
user_data_dir=str(PROFILE_DIR),
locale="zh-CN",
timezone_id="Asia/Shanghai",
timeout=60000,
page_setup=restore_cookies,
) as session:
session.fetch("https://www.douyin.com/", page_action=page_action, wait=1000)
return 0 if login_done else 1
# ============================================================
# main
# ============================================================
@@ -642,12 +616,8 @@ def main() -> int:
parser.add_argument("--headless", action="store_true")
parser.add_argument("--login-timeout", type=int, default=300)
parser.add_argument("--dry-run", action="store_true", help="不写回多维表格")
parser.add_argument("--login-only", action="store_true", help="只触发登录")
args = parser.parse_args()
if args.login_only:
return login_only(args.login_timeout)
mapping = load_mapping()
styles = mapping["tables"]
if args.style:
@@ -703,6 +673,9 @@ def main() -> int:
if __name__ == "__main__":
try:
raise SystemExit(main())
except CookieExpiredError as exc:
print(f"[COOKIE_EXPIRED] {exc}", file=sys.stderr)
raise SystemExit(COOKIE_SKIP_EXIT_CODE)
except KeyboardInterrupt:
print("\nInterrupted.", file=sys.stderr)
raise SystemExit(130)
src/gyxx_flow/modules/content_marketing/xiaohongshu_comment_scraper.py
+9 -51
View File
@@ -223,18 +223,15 @@ def maybe_wait_for_login(
*,
deadline: float | None = None,
) -> None:
login_deadline = time.monotonic() + max(0, login_timeout)
while time.monotonic() < login_deadline:
_check_note_deadline(deadline, "login wait")
# 登录统一由账号 vault 管理（gyxx accounts login xiaohongshu-content），
# 由工作流执行前的 cookie 网关保证；运行中失效立即降级，不再等待人工扫码。
try:
body = page.locator("body").inner_text(timeout=5000)
except Exception:
body = ""
if "扫码登录" not in body and "登录后" not in body and "验证码" not in body:
return
log("Xiaohongshu may require login. Please finish login in the opened Chrome window.")
page.wait_for_timeout(3000)
_check_note_deadline(deadline, "login wait")
if "扫码登录" in body or "登录后" in body or "验证码" in body:
log("Xiaohongshu login state missing; collecting degraded samples. "
"Run `gyxx accounts login xiaohongshu-content` to restore full comments.")
def is_logged_in(page: Page) -> bool:
@@ -751,55 +748,16 @@ def write_outputs(url: str, comments: list[dict[str, Any]], result: dict[str, An
return json_path, csv_path
def login_only(login_timeout: int) -> int:
"""只触发登录(headed),登录完成立刻退出。给 relogin_xiaohongshu.py 用。"""
def action(page: Page) -> None:
page.goto("https://www.xiaohongshu.com/", wait_until="domcontentloaded", timeout=90000)
page.wait_for_timeout(2500)
# 主动点登录按钮触发二维码
try:
login_btn = page.get_by_text("登录", exact=True).first
if login_btn.count():
login_btn.click(timeout=3000)
page.wait_for_timeout(1500)
except Exception:
pass
maybe_wait_for_login(page, login_timeout)
save_state(page)
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
with ScraplingBrowser(
headless=False,
real_chrome=True,
user_data_dir=str(PROFILE_DIR),
locale="zh-CN",
timezone_id="Asia/Shanghai",
timeout=90000,
network_idle=False,
disable_resources=False,
google_search=False,
page_setup=restore_cookies,
max_pages=1,
) as browser:
browser.run_fetch_action("https://www.xiaohongshu.com/", action, wait=1000)
log("[login-only] 完成")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description="Scrape Xiaohongshu note comments with a real browser.")
parser.add_argument("url", help="Xiaohongshu explore/note URL (传 'login' 配合 --login-only)")
parser.add_argument("--login-timeout", type=int, default=300, help="等待扫码/登录秒数")
parser.add_argument("url", help="Xiaohongshu explore/note URL")
parser.add_argument("--login-timeout", type=int, default=300,
help="保留参数（登录等待已移除；未登录将直接降级采集）")
parser.add_argument("--max-scrolls", type=int, default=120, help="最多滚动加载次数")
parser.add_argument("--idle-rounds", type=int, default=8, help="连续几轮无新评论后停止")
parser.add_argument("--headless", action="store_true", help="无头模式；首次登录不要使用")
parser.add_argument("--login-only", action="store_true",
help="只触发登录(headed)然后退出,给 relogin_xiaohongshu 用")
parser.add_argument("--headless", action="store_true", help="无头模式")
args = parser.parse_args()
if args.login_only:
return login_only(args.login_timeout)
comments, result = scrape_comments(args.url, args.login_timeout, args.max_scrolls, args.idle_rounds, args.headless)
json_path, csv_path = write_outputs(args.url, comments, result)
log(f"Final URL: {result.get('final_url', '')}")
src/gyxx_flow/modules/content_marketing/xingtu_scraper_v2.py
+8 -283
View File
@@ -33,6 +33,7 @@ from gyxx_flow.adapters.scrapling import (
BrowserTimeoutError,
is_browser_timeout_error,
)
from gyxx_flow.adapters.browser import CookieExpiredError, cookie_expired_message
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
from gyxx_flow.modules.content_marketing.collection_completeness import (
BLOCKED_INPUT,
@@ -537,183 +538,6 @@ def is_logged_in(page: Page) -> bool:
return False
def open_scan_login(page: Page) -> None:
"""Open the selected platform's login dialog without handling passwords."""
if CONTENT_SOURCE == "buyin":
if not is_business_page(page) or "/dashboard/servicehall/daren-square" not in page.url:
try:
page.goto(CUSTOMER_LOGIN_URL, wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(1500)
except Exception:
pass
for pattern in (r"^登录$", r"扫码登录|二维码登录|抖音登录|使用抖音登录"):
try:
button = page.get_by_text(re.compile(pattern)).first
if button.count() and button.is_visible():
button.click(timeout=3000)
page.wait_for_timeout(800)
return
except Exception as exc:
reraise_if_browser_session_lost(exc)
return
if "sso.oceanengine.com/xingtu/login" not in page.url:
try:
page.goto(CUSTOMER_LOGIN_URL, wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(1500)
except Exception:
pass
# The Douyin login icon on Xingtu's customer SSO page has a stable
# aweme.png source. Clicking it opens the QR authorization page directly.
try:
douyin = page.locator('img[src*="aweme.png"]').first
if douyin.count() and douyin.is_visible():
douyin.click(timeout=3000)
page.wait_for_timeout(1500)
return
except Exception:
pass
for pattern in (r"^登录$|立即登录", r"抖音扫码登录|扫码登录|二维码登录|扫码"):
try:
button = page.get_by_text(re.compile(pattern)).first
if button.count() and button.is_visible():
button.click(timeout=2000)
page.wait_for_timeout(500)
except Exception:
pass
def is_scan_qr_page(page: Page) -> bool:
"""Return True once Douyin's QR authorization page is visible.
The QR page lives outside xingtu.cn. Treating that URL as a page that still
needs opening causes the login helper to navigate back and click again every
few seconds, continuously refreshing the QR code.
"""
url = page.url.lower()
if any(host in url for host in ("open.douyin.com", "sso.douyin.com")):
return True
for selector in (
'img[src*="qrcode"]',
'img[src*="qr_code"]',
'canvas',
):
try:
locator = page.locator(selector).first
if locator.count() and locator.is_visible():
return True
except Exception:
pass
return False
def resolve_live_page(page: Page, *, create_if_missing: bool = False) -> Page | None:
"""Return a usable page after OAuth closes or replaces its QR tab."""
try:
if not page.is_closed():
return page
except Exception:
pass
try:
context = page.context
for candidate in reversed(list(context.pages)):
try:
if not candidate.is_closed():
return candidate
except Exception:
continue
if create_if_missing:
return context.new_page()
except Exception:
return None
return None
def find_logged_in_page(page: Page) -> Page | None:
"""Find an authenticated business tab even while the OAuth tab stays open."""
candidates: list[Page] = []
try:
candidates.extend(reversed(list(page.context.pages)))
except Exception:
pass
candidates.append(page)
seen: set[int] = set()
for candidate in candidates:
marker = id(candidate)
if marker in seen:
continue
seen.add(marker)
try:
if candidate.is_closed() or not is_business_page(candidate):
continue
if is_logged_in(candidate):
return candidate
except Exception:
continue
return None
def wait_for_scan_login(page: Page, timeout_seconds: int) -> Page:
log("Not logged in. 请在打开的 Chrome 中扫码登录。")
current_page = page
open_scan_login(current_page)
deadline = time.time() + timeout_seconds
next_login_click = time.time() + 5
while time.time() < deadline:
authenticated = find_logged_in_page(current_page)
if authenticated is not None:
save_state(authenticated)
return authenticated
replacement = resolve_live_page(current_page, create_if_missing=True)
if replacement is None:
raise RuntimeError("Xingtu browser context closed during OAuth login")
if replacement is not current_page:
current_page = replacement
try:
current_page.goto(
MARKET_URL, wait_until="domcontentloaded", timeout=60000
)
current_page.wait_for_timeout(2000)
except Exception:
pass
if (
CONTENT_SOURCE == "xingtu"
and "redirect_uri=/ad/creator/market" in current_page.url
and not is_scan_qr_page(current_page)
):
try:
current_page.goto(
MARKET_URL, wait_until="domcontentloaded", timeout=60000
)
current_page.wait_for_timeout(2000)
except Exception:
pass
authenticated = find_logged_in_page(current_page)
if authenticated is not None:
save_state(authenticated)
return authenticated
if (
time.time() >= next_login_click
and not is_scan_qr_page(current_page)
and not is_business_page(current_page)
):
open_scan_login(current_page)
next_login_click = time.time() + 5
try:
current_page.wait_for_timeout(2000)
except Exception:
# OAuth may close the QR tab after authorization. The next loop
# follows the surviving tab in the same browser context.
continue
raise TimeoutError(f"Login timed out after {timeout_seconds} seconds.")
def ensure_login(page: Page, login_timeout: int) -> Page:
page.goto(MARKET_URL, wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(3000)
@@ -735,12 +559,11 @@ def ensure_login(page: Page, login_timeout: int) -> Page:
)
log(f"[SKIPPED_COOKIE] {reason}")
raise AcceptanceCookieSkip(reason)
page = wait_for_scan_login(page, login_timeout)
page.goto(MARKET_URL, wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(3000)
if not is_logged_in(page):
page = wait_for_scan_login(page, login_timeout)
return page
# 登录统一由账号 vault 管理（gyxx accounts login douyin-shop），
# 采集脚本不再内嵌扫码等待；运行中失效直接快速失败。
reason = cookie_expired_message("douyin-shop", f"{CONTENT_SOURCE}达人广场")
log(f"[COOKIE_EXPIRED] {reason}")
raise CookieExpiredError(reason)
def dismiss_popups(page: Page) -> None:
@@ -3736,7 +3559,7 @@ def scrape_styles_with_fallback(
summary_override=summaries,
replace_existing=True,
)
except AcceptanceCookieSkip as exc:
except (AcceptanceCookieSkip, CookieExpiredError) as exc:
log(f" [WARN] 抖店补采登录态不可用，保留星图结果: {exc}")
_mark_fallback_not_processed(
fallback_tasks, summaries, "buyin_cookie_skipped"
@@ -3805,96 +3628,6 @@ def _force_cleanup_session(session) -> None:
pass
def login_only(login_timeout: int) -> int:
"""只触发登录(headed),登录完成立刻退出。给 weekly_relogin.py 用。"""
login_completed = False
def action(page):
nonlocal login_completed
page = ensure_login(page, login_timeout)
if not is_logged_in(page):
raise RuntimeError(f"{CONTENT_SOURCE} 登录状态验证失败")
save_state(page)
login_completed = True
return page
session = None
try:
with DynamicSession(
headless=False,
real_chrome=True,
user_data_dir=str(PROFILE_DIR),
locale="zh-CN",
timezone_id="Asia/Shanghai",
timeout=60000,
network_idle=False,
disable_resources=False,
google_search=False,
page_setup=restore_cookies,
max_pages=2,
) as session:
session.fetch(MARKET_URL, page_action=action, wait=1000)
except Exception as exc:
err_msg = str(exc).lower()
if "closed" in err_msg or "context" in err_msg or "target" in err_msg:
log(f"[login-only] 浏览器会话关闭异常: {exc}")
else:
log(f"[login-only] 登录执行失败: {exc}")
finally:
_force_cleanup_session(session)
# Douyin OAuth may close the only QR tab (and therefore the temporary
# browser context) after a successful mobile confirmation. The durable
# Chromium profile can already contain the new session even though the
# page_action never got a chance to export cookies. Reopen the same profile
# once, verify the selected business page, then export its cookies.
if not login_completed or not has_valid_login_cookie_file():
log(f"[login-only] 扫码窗口已结束,重新打开{CONTENT_SOURCE}验证持久会话")
login_completed = verify_persisted_profile_login()
if not login_completed or not has_valid_login_cookie_file():
log("[login-only] 登录未完成或Cookie缺少有效会话")
return 1
log("[login-only] 登录完成,退出")
return 0
def verify_persisted_profile_login() -> bool:
"""Reopen the persistent profile after OAuth closes its only tab."""
verified = False
def action(page: Page):
nonlocal verified
page.goto(MARKET_URL, wait_until="domcontentloaded", timeout=60000)
page.wait_for_timeout(3000)
if not is_logged_in(page):
raise RuntimeError(f"persisted {CONTENT_SOURCE} profile is not authenticated")
save_state(page)
verified = True
return page
session = None
try:
with DynamicSession(
headless=False,
real_chrome=True,
user_data_dir=str(PROFILE_DIR),
locale="zh-CN",
timezone_id="Asia/Shanghai",
timeout=60000,
network_idle=False,
disable_resources=False,
google_search=False,
page_setup=restore_cookies,
max_pages=1,
) as session:
session.fetch(MARKET_URL, page_action=action, wait=1000)
except Exception as exc:
log(f"[login-only] 持久会话验证失败: {exc}")
finally:
_force_cleanup_session(session)
return verified and has_valid_login_cookie_file()
def load_retry_scope(summary_path: Path, reason: str) -> dict[int, set[str]]:
"""Load exact style/record pairs for a reason-limited retry."""
try:
@@ -3926,8 +3659,6 @@ def main() -> int:
parser.add_argument("--login-timeout", type=int, default=300)
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--data-dir", default=str(PATHS.normalized_root))
parser.add_argument("--login-only", action="store_true",
help="只触发登录(headed),登录完成立刻退出,给 weekly_relogin 用")
parser.add_argument(
"--buyin",
action="store_true",
@@ -3952,12 +3683,6 @@ def main() -> int:
"buyin" if args.buyin else os.environ.get("GYXX_CONTENT_SOURCE", "xingtu")
)
if args.login_only:
try:
return login_only(args.login_timeout)
except AcceptanceCookieSkip:
return COOKIE_SKIP_EXIT_CODE
mapping = load_mapping(self_operated=args.self_operated)
styles = mapping["tables"]
if args.style:
@@ -4038,7 +3763,7 @@ def main() -> int:
max_age_days=args.max_age_days,
no_retry=args.no_retry,
)
except AcceptanceCookieSkip:
except (AcceptanceCookieSkip, CookieExpiredError):
return COOKIE_SKIP_EXIT_CODE
persist_summaries(all_summaries)
for s, summary in zip(styles, all_summaries):
src/gyxx_flow/modules/product_commerce/vendors/dy-data-flow/dy_store_competitor_store_scraping.py
Vendored
+12 -23
View File
@@ -223,23 +223,16 @@ def _check_login_status(page) -> bool:
def _ensure_login(page):
print("\n" + "=" * 60)
print("【请登录】请在弹出的浏览器窗口中完成登录")
print("提示：扫码/短信/密码均可，登录成功后页面会自动跳转")
print("=" * 60 + "\n")
for i in range(LOGIN_TIMEOUT):
page = _active_page(page)
if _check_login_status(page):
print(f"[OK] Login success! URL: {page.url}")
page.wait_for_timeout(3000)
return page
if i % 5 == 0:
print(f"[WAIT] Waiting for login... {i}s / {LOGIN_TIMEOUT}s")
time.sleep(1)
print(f"[ERR] Login timeout ({LOGIN_TIMEOUT}s)")
return None
# 登录统一由账号 vault 管理（gyxx accounts login douyin-shop），
# 采集脚本不再弹出浏览器等待人工扫码/短信登录。
print(
"[COOKIE_EXPIRED] 抖音罗盘登录态已失效："
"请执行 uv run gyxx accounts login douyin-shop 刷新账号登录态"
)
raise RuntimeError(
"抖音罗盘登录态已失效；请执行 uv run gyxx accounts login douyin-shop "
"刷新账号登录态后重跑"
)
def _select_menu_and_date(page) -> None:
@@ -1927,12 +1920,8 @@ def _page_automation(page) -> None:
# 1. Check and wait for login
if not _check_login_status(page):
logged_page = _ensure_login(page)
if not logged_page:
print("[ERR] Login failed, will try save cookies anyway...")
else:
page = logged_page
_maximize_page(page)
# 登录态失效时 _ensure_login 会直接抛 RuntimeError
_ensure_login(page)
# 2. Save state after login (best effort)
_save_browser_state(page, "after login")
src/gyxx_flow/modules/shop_intelligence/collectors/dy_negative_reviews.py
+9 -16
View File
@@ -226,23 +226,16 @@ def ensure_login(page: Page, *, headless: bool, login_timeout: int) -> None:
print("[SKIPPED_COOKIE] 抖店会话跳转到了登录页")
raise DouyinSessionExpiredError("抖店会话跳转到了登录页")
if headless:
raise RuntimeError(
"抖音 Cookie 已失效且当前为无头模式，无法扫码登录；"
"请先执行 uv run gyxx accounts login douyin-shop 刷新账号登录态"
# 登录统一由账号 vault 管理（gyxx accounts login douyin-shop），
# 采集脚本不再弹出浏览器等待人工扫码/验证码。
print(
"[COOKIE_EXPIRED] 抖店 Cookie 已失效："
"请执行 uv run gyxx accounts login douyin-shop 刷新账号登录态"
)
raise DouyinSessionExpiredError(
"抖音 Cookie 已失效；请执行 uv run gyxx accounts login douyin-shop "
"刷新账号登录态后重跑"
)
print("=" * 60)
print("【请登录】请在弹出的浏览器窗口中完成抖店登录")
print("=" * 60)
deadline = time.monotonic() + login_timeout
while time.monotonic() < deadline:
if not douyin_common.is_login_page(page):
print(f" [OK] 登录成功: {page.url}")
page.wait_for_timeout(3_000)
return
page.wait_for_timeout(1_000)
raise RuntimeError(f"登录在 {login_timeout} 秒内未完成，当前页面: {page.url}")
class DouyinSessionExpiredError(RuntimeError):
src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py
+12 -23
View File
@@ -262,23 +262,16 @@ def _check_login_status(page) -> bool:
def _ensure_login(page):
print("\n" + "=" * 60)
print("【请登录】请在弹出的浏览器窗口中完成登录")
print("提示：扫码/短信/密码均可，登录成功后页面会自动跳转")
print("=" * 60 + "\n")
for i in range(LOGIN_TIMEOUT):
page = _active_page(page)
if _check_login_status(page):
print(f"[OK] Login success! URL: {page.url}")
page.wait_for_timeout(3000)
return page
if i % 5 == 0:
print(f"[WAIT] Waiting for login... {i}s / {LOGIN_TIMEOUT}s")
time.sleep(1)
print(f"[ERR] Login timeout ({LOGIN_TIMEOUT}s)")
return None
# 登录统一由账号 vault 管理（gyxx accounts login douyin-shop），
# 采集脚本不再弹出浏览器等待人工扫码/短信登录。
print(
"[COOKIE_EXPIRED] 抖音罗盘登录态已失效："
"请执行 uv run gyxx accounts login douyin-shop 刷新账号登录态"
)
raise BrowserSessionExpiredError(
"抖音罗盘登录态已失效；请执行 uv run gyxx accounts login douyin-shop "
"刷新账号登录态后重跑"
)
def _select_menu_and_date(page) -> None:
@@ -2325,12 +2318,8 @@ def _page_automation_impl(page) -> None:
# 1. Check and wait for login
if not _check_login_status(page):
logged_page = _ensure_login(page)
if not logged_page:
print("[ERR] Login failed, will try save cookies anyway...")
else:
page = logged_page
_maximize_page(page)
# If the login state is expired, _ensure_login will directly raise BrowserSessionExpiredError
_ensure_login(page)
# 2. Save state after login (best effort)
_save_browser_state(page, "after login")
src/gyxx_flow/notification_routing.py
-1
View File
@@ -52,7 +52,6 @@ MAX_NOTIFICATION_BINDINGS_PER_PERSON = 16
NOTIFICATION_CAPABILITIES: Mapping[str, str] = MappingProxyType(
{
"content.marketing_report.daily": "正式执行且启用发送时投递最终营销日报。",
"content.relogin.weekly": "刷新期间投递各平台扫码二维码，并在平台失败时投递失败汇总。",
"product.alert.daily": "命中销量下滑规则且尚未重复通知时投递告警。",
"product.market_rank": "存在本次新生成的平台报告链接且尚未重复通知时投递。",
"product.online_stock_alert.daily": "线上商品当前库存低于 3 时向配置接收人发送库存预警。",
tests/modules/content_marketing/test_comment_batch_recovery.py
+4 -13
View File
@@ -21,20 +21,11 @@ def test_page_crash_requires_a_fresh_browser():
@pytest.mark.parametrize("batch", [xhs_batch, douyin_batch])
def test_non_acceptance_relogin_runs_platform_script(monkeypatch, batch):
class Policy:
enabled = False
def test_batch_tools_no_longer_embed_relogin(batch):
"""扫码重登已统一到账号 vault（gyxx accounts login），工具内不再提供。"""
calls = []
monkeypatch.setattr(batch, "current_acceptance_policy", lambda: Policy())
monkeypatch.setattr(
batch.subprocess,
"call",
lambda command, **kwargs: calls.append((command, kwargs)) or 0,
)
assert batch.do_relogin() is True
assert calls[0][0][1] == str(batch.RELOGIN_SCRIPT)
assert not hasattr(batch, "do_relogin")
assert not hasattr(batch, "RELOGIN_SCRIPT")
def test_comment_checkpoint_matches_both_note_id_and_url(tmp_path):
tests/modules/content_marketing/test_friday_relogin_parallel.py
-569
View File
@@ -1,569 +0,0 @@
from types import SimpleNamespace
import pytest
from gyxx_flow.modules.content_marketing.data.tools import (
friday_relogin_parallel as relogin,
)
from gyxx_flow.modules.content_marketing.data.tools import relogin_transaction
def test_five_expected_platforms_are_configured():
assert set(relogin.PLATFORMS) == {
"bilibili", "douyin", "pgy", "xingtu", "xiaohongshu"
}
assert relogin.PLATFORMS["xingtu"].required_cookie == "sessionid"
assert relogin.PLATFORMS["douyin"].required_cookie == "sessionid"
def test_manual_relogin_can_select_only_xingtu():
assert relogin.resolve_platform_names(["xingtu"]) == ["xingtu"]
assert relogin.resolve_platform_names(None) == list(relogin.PLATFORMS)
def test_default_screenshot_recipients_are_routed_by_platform():
assert relogin.DEFAULT_RECIPIENTS == {
"pgy": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
"xingtu": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
"douyin": "ou_7ad5fc8012e2f741afc5346e05ffd447",
"bilibili": "ou_7ad5fc8012e2f741afc5346e05ffd447",
"xiaohongshu": "ou_7ad5fc8012e2f741afc5346e05ffd447",
}
assert relogin.recipient_for_platform("pgy") == "ou_24cc944d6e43c69c59d6560ad4e2ae6e"
assert relogin.recipient_for_platform("douyin") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
assert relogin.recipient_for_platform("bilibili") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
assert relogin.recipient_for_platform("pgy", "ou_override") == "ou_override"
def test_failed_platforms_are_grouped_by_their_responsible_recipient():
grouped = relogin.group_platforms_by_recipient(
["pgy", "douyin", "bilibili", "xingtu", "xiaohongshu"]
)
assert grouped == {
"ou_24cc944d6e43c69c59d6560ad4e2ae6e": ["pgy", "xingtu"],
"ou_7ad5fc8012e2f741afc5346e05ffd447": [
"douyin", "bilibili", "xiaohongshu"
],
}
def test_legacy_route_keeps_the_existing_per_platform_recipients(monkeypatch):
observed = {}
def fake_resolver(workflow_id, defaults):
observed["workflow_id"] = workflow_id
observed["defaults"] = defaults
return relogin.ResolvedNotificationRoute(
configured=False,
enabled=True,
app_profile=None,
open_ids=tuple(defaults),
)
monkeypatch.setattr(relogin, "resolve_notification_route", fake_resolver)
route = relogin.resolve_weekly_notification_route()
assert observed == {
"workflow_id": "content.relogin.weekly",
"defaults": (
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
"ou_7ad5fc8012e2f741afc5346e05ffd447",
),
}
assert relogin.recipients_for_platform("pgy", route) == (
"ou_24cc944d6e43c69c59d6560ad4e2ae6e",
)
assert relogin.recipients_for_platform("douyin", route) == (
"ou_7ad5fc8012e2f741afc5346e05ffd447",
)
def test_disabled_route_still_runs_relogin_without_sending(monkeypatch):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=False,
app_profile="hermes-analyzer",
open_ids=(),
)
rounds = []
monkeypatch.setattr(
relogin,
"current_acceptance_policy",
lambda: SimpleNamespace(enabled=False),
)
monkeypatch.setattr(
relogin,
"resolve_weekly_notification_route",
lambda _override=None: route,
)
def fake_round_factory(_args, resolved_route, notification_failures):
assert resolved_route is route
assert notification_failures == []
def run_round(names, attempt):
rounds.append((tuple(names), attempt))
return {name: True for name in names}
return run_round
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
monkeypatch.setattr(
relogin,
"_send_lark",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
AssertionError("disabled route must not send Lark messages")
),
)
monkeypatch.setattr(
relogin.sys,
"argv",
["friday_relogin_parallel.py", "--platform", "pgy"],
)
assert relogin.main() == 0
assert rounds == [(("pgy",), 1)]
def test_enabled_route_sends_qr_text_and_image_to_every_recipient(
monkeypatch,
tmp_path,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
sent = []
image = tmp_path / "qrcode.png"
monkeypatch.setattr(
relogin,
"_send_lark",
lambda recipient, text=None, image=None, app_profile=None: sent.append(
(recipient, text, image, app_profile)
),
)
failures = relogin._send_qr_notification(
route,
"pgy",
override=None,
text="请扫码",
image=image,
)
assert failures == ()
assert sent == [
("ou_first123", "请扫码", None, "hermes-analyzer"),
("ou_first123", None, image, "hermes-analyzer"),
("ou_second456", "请扫码", None, "hermes-analyzer"),
("ou_second456", None, image, "hermes-analyzer"),
]
assert relogin.group_platforms_by_notification_recipient(
["pgy", "douyin"], route
) == {
"ou_first123": ["pgy", "douyin"],
"ou_second456": ["pgy", "douyin"],
}
def test_qr_delivery_aggregates_errors_and_attempts_every_recipient(
monkeypatch,
tmp_path,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
attempts = []
image = tmp_path / "qrcode.png"
def fake_send(recipient, text=None, image=None, app_profile=None):
stage = "qr_text" if text is not None else "qr_image"
attempts.append((recipient, stage, app_profile))
if (recipient, stage) == ("ou_first123", "qr_text"):
raise RuntimeError("first text rejected")
if (recipient, stage) == ("ou_second456", "qr_image"):
raise RuntimeError("second image rejected")
monkeypatch.setattr(relogin, "_send_lark", fake_send)
failures = relogin._send_qr_notification(
route,
"pgy",
override=None,
text="请扫码",
image=image,
)
assert attempts == [
("ou_first123", "qr_text", "hermes-analyzer"),
("ou_first123", "qr_image", "hermes-analyzer"),
("ou_second456", "qr_text", "hermes-analyzer"),
("ou_second456", "qr_image", "hermes-analyzer"),
]
assert failures == (
relogin.NotificationDeliveryFailure(
stage="qr_text",
platform="pgy",
recipient="ou_first123",
detail="first text rejected",
),
relogin.NotificationDeliveryFailure(
stage="qr_image",
platform="pgy",
recipient="ou_second456",
detail="second image rejected",
),
)
def test_failure_summary_aggregates_every_recipient_error(monkeypatch):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
attempts = []
def fail_send(recipient, **_kwargs):
attempts.append(recipient)
raise RuntimeError(f"rejected {recipient}")
monkeypatch.setattr(relogin, "_send_lark", fail_send)
failures = relogin._send_failure_summaries(
route,
["pgy", "douyin"],
override=None,
max_attempts=3,
)
assert attempts == ["ou_first123", "ou_second456"]
assert [failure.stage for failure in failures] == [
"failure_summary",
"failure_summary",
]
assert [failure.recipient for failure in failures] == attempts
assert [failure.platform for failure in failures] == [
"pgy,douyin",
"pgy,douyin",
]
def test_main_returns_failure_and_reports_all_notification_errors(
monkeypatch,
capsys,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
monkeypatch.setattr(
relogin,
"current_acceptance_policy",
lambda: SimpleNamespace(enabled=False),
)
monkeypatch.setattr(
relogin,
"resolve_weekly_notification_route",
lambda _override=None: route,
)
def fake_round_factory(_args, _route, notification_failures):
notification_failures.extend(
(
relogin.NotificationDeliveryFailure(
"qr_text", "pgy", "ou_first123", "text rejected"
),
relogin.NotificationDeliveryFailure(
"qr_image", "pgy", "ou_second456", "image rejected"
),
)
)
return lambda names, _attempt: {name: True for name in names}
monkeypatch.setattr(relogin, "_run_round_factory", fake_round_factory)
monkeypatch.setattr(
relogin.sys,
"argv",
["friday_relogin_parallel.py", "--platform", "pgy"],
)
assert relogin.main() == 2
output = capsys.readouterr().out
assert "Final relogin status: {'pgy': True}" in output
assert "Final notification status: failed (2 deliveries)" in output
assert "recipient=ou_first123 error=text rejected" in output
assert "recipient=ou_second456 error=image rejected" in output
def test_round_completes_relogin_after_notification_failure(monkeypatch, tmp_path):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
args = SimpleNamespace(
round_timeout=10,
screenshot_delay=0,
no_send=False,
recipient=None,
max_attempts=1,
)
failures = []
attempts = []
waits = []
class Process:
pid = 1234
def wait(self, timeout):
waits.append(timeout)
return 0
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
monkeypatch.setattr(
relogin,
"binding_from_environment",
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
)
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(relogin, "_login_windows", lambda: {"pgy": 101})
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
monkeypatch.setattr(relogin, "_capture_window", lambda _hwnd, _output: None)
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: True)
def fake_send(recipient, text=None, image=None, **_kwargs):
stage = "qr_text" if text is not None else "qr_image"
attempts.append((recipient, stage))
if (recipient, stage) == ("ou_first123", "qr_text"):
raise RuntimeError("temporary send failure")
monkeypatch.setattr(relogin, "_send_lark", fake_send)
run_round = relogin._run_round_factory(args, route, failures)
assert run_round(["pgy"], 1) == {"pgy": True}
assert waits == [100]
assert attempts == [
("ou_first123", "qr_text"),
("ou_first123", "qr_image"),
("ou_second456", "qr_text"),
("ou_second456", "qr_image"),
]
assert failures == [
relogin.NotificationDeliveryFailure(
stage="qr_text",
platform="pgy",
recipient="ou_first123",
detail="temporary send failure",
)
]
@pytest.mark.parametrize(
("returncode", "cookie_valid", "expected_status", "expected_failure_count"),
(
(0, True, True, 0),
(1, False, False, 4),
),
)
def test_missing_qr_window_is_classified_after_relogin_result(
monkeypatch,
tmp_path,
returncode,
cookie_valid,
expected_status,
expected_failure_count,
):
route = relogin.ResolvedNotificationRoute(
configured=True,
enabled=True,
app_profile="hermes-analyzer",
open_ids=("ou_first123", "ou_second456"),
)
args = SimpleNamespace(
round_timeout=10,
screenshot_delay=0,
no_send=False,
recipient=None,
max_attempts=1,
)
failures = []
class Process:
pid = 1234
def wait(self, timeout):
assert timeout == 100
return returncode
monkeypatch.setattr(relogin.subprocess, "Popen", lambda *_args, **_kwargs: Process())
monkeypatch.setattr(relogin, "environment_for_child_script", lambda *_args: {})
monkeypatch.setattr(
relogin,
"binding_from_environment",
lambda _environment: SimpleNamespace(cookie_file=tmp_path / "cookies.json"),
)
monkeypatch.setattr(relogin.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(relogin, "_login_windows", lambda: {})
monkeypatch.setattr(relogin, "_window_for_process", lambda _pid: None)
monkeypatch.setattr(relogin, "_tile_windows", lambda _handles: None)
monkeypatch.setattr(relogin, "_valid_cookie", lambda *_args: cookie_valid)
monkeypatch.setattr(
relogin,
"_send_lark",
lambda *_args, **_kwargs: pytest.fail(
"no QR window must not start a send before child classification"
),
)
run_round = relogin._run_round_factory(args, route, failures)
assert run_round(["pgy"], 1) == {"pgy": expected_status}
assert len(failures) == expected_failure_count
assert {failure.stage for failure in failures} <= {"qr_text", "qr_image"}
assert all(
failure.detail == "no visible QR window and relogin did not succeed"
for failure in failures
)
def test_lark_send_uses_shared_analyzer_user_sender_for_text_and_image(
monkeypatch,
tmp_path,
):
calls = []
image = tmp_path / "qrcode.png"
image.write_bytes(b"png")
monkeypatch.setattr(relogin, "ANALYZER_DIR", tmp_path)
monkeypatch.setenv("HERMES_ANALYZER_TOKEN", "must-not-leak")
def fake_send(**kwargs):
calls.append(kwargs)
return {"message_id": f"om_{len(calls)}"}
monkeypatch.setattr(relogin, "send_lark_bot_message", fake_send)
receipts = relogin._send_lark(
"ou_recipient123",
text="登录提醒",
image=image,
)
assert [receipt["message_id"] for receipt in receipts] == ["om_1", "om_2"]
assert calls[0]["user_id"] == "ou_recipient123"
assert calls[0]["text"] == "登录提醒"
assert "image" not in calls[0]
assert calls[1]["user_id"] == "ou_recipient123"
assert calls[1]["image"] == image
assert "text" not in calls[1]
assert all(call["profile"] == "hermes-analyzer" for call in calls)
assert all("HERMES_ANALYZER_TOKEN" not in call["env"] for call in calls)
def test_login_windows_are_classified_by_title():
assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili"
assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy"
assert relogin._platform_from_window_title("抖音授权 - Google Chrome") == "xingtu"
assert relogin._platform_from_window_title("抖音 - 记录美好生活") == "douyin"
assert relogin._platform_from_window_title("小红书 - 你的生活兴趣社区") == "xiaohongshu"
assert relogin._platform_from_window_title("Codex") is None
def test_retry_only_failed_platforms_for_at_most_three_rounds():
calls = []
def fake_round(names, attempt):
calls.append((tuple(names), attempt))
if attempt == 1:
return {name: name != "xingtu" for name in names}
if attempt == 2:
return {name: False for name in names}
return {name: True for name in names}
result = relogin.run_with_retries(
["bilibili", "pgy", "xingtu", "xiaohongshu"],
max_attempts=3,
run_round=fake_round,
)
assert calls == [
(("bilibili", "pgy", "xingtu", "xiaohongshu"), 1),
(("xingtu",), 2),
(("xingtu",), 3),
]
assert all(result.values())
def test_stops_after_third_failed_round():
calls = []
def always_fail(names, attempt):
calls.append(attempt)
return {name: False for name in names}
result = relogin.run_with_retries(
["pgy"], max_attempts=3, run_round=always_fail
)
assert calls == [1, 2, 3]
assert result == {"pgy": False}
def test_timeout_terminates_the_complete_process_tree(monkeypatch):
calls = []
class Process:
pid = 1234
def terminate(self):
calls.append("fallback-terminate")
def wait(self, timeout):
calls.append(("wait", timeout))
monkeypatch.setattr(
relogin.subprocess,
"run",
lambda command, **kwargs: calls.append((command, kwargs)),
)
relogin._terminate_process_tree(Process())
assert calls[0][0] == ["taskkill", "/PID", "1234", "/T", "/F"]
assert calls[1] == ("wait", 10)
def test_transaction_rolls_back_files_and_profile(tmp_path):
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
state = relogin_transaction.begin((cookie, storage), profile)
cookie.write_text("partial", encoding="utf-8")
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
relogin_transaction.rollback(state)
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
tests/modules/content_marketing/test_relogin_rollback.py
-494
View File
@@ -1,494 +0,0 @@
import pytest
from gyxx_flow.modules.content_marketing import (
pgy_xhs_scraper_v2,
xingtu_scraper_v2,
)
from gyxx_flow.modules.content_marketing.data.tools import (
relogin_douyin,
relogin_pgy,
relogin_xingtu,
)
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module):
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "state.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
cookie_backup, profile_backup, state_backup = module.reset_state()
assert not cookie.exists()
assert not storage.exists()
assert not profile.exists()
cookie.write_text("partial-new-cookie", encoding="utf-8")
storage.write_text("partial-new-storage", encoding="utf-8")
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
module.restore_previous_state(cookie_backup, profile_backup, state_backup)
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module):
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
cookie_backup, profile_backup, state_backup = module.reset_state()
cookie.write_text("new-cookie", encoding="utf-8")
storage.write_text("new-storage", encoding="utf-8")
profile.mkdir()
module.finish_successful_relogin(profile_backup)
assert cookie.read_text(encoding="utf-8") == "new-cookie"
assert storage.read_text(encoding="utf-8") == "new-storage"
assert cookie_backup.exists()
assert state_backup.exists()
assert not profile_backup.exists()
def test_xingtu_scan_login_opens_customer_sso_and_clicks_douyin():
class FakeLocator:
def __init__(self):
self.first = self
self.clicked = False
self.evaluated = False
def count(self):
return 1
def is_visible(self):
return True
def click(self, timeout):
self.clicked = True
def evaluate(self, script):
assert script == "e => e.click()"
self.evaluated = True
class FakePage:
url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market"
def __init__(self):
self.goto_url = None
self.selectors = []
self.douyin = FakeLocator()
def goto(self, url, **kwargs):
self.goto_url = url
self.url = url
def wait_for_timeout(self, _milliseconds):
pass
def locator(self, selector):
self.selectors.append(selector)
return self.douyin
def get_by_text(self, _pattern):
return FakeLocator()
page = FakePage()
xingtu_scraper_v2.open_scan_login(page)
assert page.goto_url == xingtu_scraper_v2.CUSTOMER_LOGIN_URL
assert 'img[src*="aweme.png"]' in page.selectors
assert page.douyin.clicked
def test_xingtu_index_page_is_recognized_as_logged_in():
class Locator:
@property
def first(self):
return self
def wait_for(self, timeout):
assert timeout == 1200
class Page:
url = "https://www.xingtu.cn/ad/creator/index"
def get_by_text(self, _text, exact=False):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is True
def test_xingtu_index_page_with_xingtu_session_is_logged_in_without_market_ui():
class Locator:
@property
def first(self):
return self
def count(self):
return 0
def wait_for(self, timeout):
raise xingtu_scraper_v2.BrowserTimeoutError("not on market page")
def is_visible(self):
return False
class Context:
def cookies(self):
return [
{
"name": "sessionid",
"value": "fresh",
"domain": ".xingtu.cn",
}
]
class Page:
url = "https://www.xingtu.cn/ad/creator/index"
context = Context()
def get_by_text(self, *_args, **_kwargs):
return Locator()
def locator(self, *_args, **_kwargs):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is True
def test_xingtu_index_page_without_xingtu_session_is_not_logged_in():
class Locator:
@property
def first(self):
return self
def count(self):
return 0
def wait_for(self, timeout):
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
def is_visible(self):
return False
class Context:
def cookies(self):
return [
{
"name": "sessionid",
"value": "douyin-only",
"domain": ".douyin.com",
}
]
class Page:
url = "https://www.xingtu.cn/ad/creator/index"
context = Context()
def get_by_text(self, *_args, **_kwargs):
return Locator()
def locator(self, *_args, **_kwargs):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is False
def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie():
class Context:
def cookies(self):
return [{"name": "sessionid", "value": "stale"}]
class Page:
url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market"
context = Context()
def get_by_text(self, *_args, **_kwargs):
raise AssertionError("public homepage must be rejected before marker checks")
assert xingtu_scraper_v2.is_logged_in(Page()) is False
def test_xingtu_market_page_without_business_ui_rejects_stale_cookie():
class Locator:
@property
def first(self):
return self
def wait_for(self, **_kwargs):
raise xingtu_scraper_v2.BrowserTimeoutError("missing")
def count(self):
return 0
class Context:
def cookies(self):
return [{"name": "sessionid", "value": "stale"}]
class Page:
url = "https://www.xingtu.cn/ad/creator/market"
context = Context()
def get_by_text(self, *_args, **_kwargs):
return Locator()
def locator(self, *_args, **_kwargs):
return Locator()
assert xingtu_scraper_v2.is_logged_in(Page()) is False
def test_xingtu_cookie_validation_requires_real_session_cookie(tmp_path):
cookie = tmp_path / "xingtu.json"
cookie.write_text('[{"name":"other","value":"1"}]', encoding="utf-8")
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is False
cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8")
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is True
def test_xingtu_login_only_does_not_report_success_when_page_action_was_swallowed(
tmp_path, monkeypatch
):
cookie = tmp_path / "xingtu.json"
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
class Session:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def fetch(self, *_args, **_kwargs):
# Scrapling can log a page_action error and still return a response.
return object()
monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie)
monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session())
monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None)
assert xingtu_scraper_v2.login_only(1) == 1
def test_xingtu_login_only_verifies_persisted_profile_after_oauth_context_closes(
tmp_path, monkeypatch
):
cookie = tmp_path / "xingtu.json"
class Session:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def fetch(self, *_args, **_kwargs):
raise RuntimeError("Target page, context or browser has been closed")
monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie)
monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session())
monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None)
def verify():
cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8")
return True
monkeypatch.setattr(xingtu_scraper_v2, "verify_persisted_profile_login", verify)
assert xingtu_scraper_v2.login_only(1) == 0
def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monkeypatch):
class Context:
def __init__(self):
self.pages = []
def new_page(self):
raise AssertionError("a surviving replacement page should be reused")
class Page:
def __init__(self, context, url, *, closes_on_wait=False):
self.context = context
self.url = url
self.closed = False
self.closes_on_wait = closes_on_wait
def is_closed(self):
return self.closed
def wait_for_timeout(self, _milliseconds):
if self.closes_on_wait:
self.closed = True
def goto(self, url, **_kwargs):
self.url = url
context = Context()
qr_page = Page(
context,
"https://open.douyin.com/platform/oauth/pc/auth",
closes_on_wait=True,
)
market_page = Page(context, xingtu_scraper_v2.MARKET_URL)
context.pages = [qr_page, market_page]
saved = []
monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None)
monkeypatch.setattr(
xingtu_scraper_v2, "is_scan_qr_page", lambda page: page is qr_page
)
monkeypatch.setattr(
xingtu_scraper_v2, "is_logged_in", lambda page: page is market_page
)
monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page))
result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2)
assert result is market_page
assert saved == [market_page]
def test_xingtu_scan_login_detects_business_tab_while_qr_tab_stays_open(monkeypatch):
class Context:
def __init__(self):
self.pages = []
class Page:
def __init__(self, context, url):
self.context = context
self.url = url
def is_closed(self):
return False
def wait_for_timeout(self, _milliseconds):
raise AssertionError("authenticated tab should be detected immediately")
context = Context()
qr_page = Page(context, "https://open.douyin.com/platform/oauth/pc/auth")
index_page = Page(context, "https://www.xingtu.cn/ad/creator/index")
context.pages = [qr_page, index_page]
saved = []
monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None)
monkeypatch.setattr(
xingtu_scraper_v2, "is_logged_in", lambda page: page is index_page
)
monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page))
result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2)
assert result is index_page
assert saved == [index_page]
def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch):
cookie = tmp_path / "xingtu_cookies.json"
storage = tmp_path / "xingtu_storage_state.json"
profile = tmp_path / "profile"
cookie.write_text("old-cookie", encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie)
monkeypatch.setattr(relogin_xingtu, "STATE_FILE", storage)
monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile)
cookie_backup, profile_backup, state_backup = relogin_xingtu.reset_state()
profile.mkdir()
(profile / "partial.txt").write_text("partial", encoding="utf-8")
assert relogin_xingtu.recover_interrupted_state() is True
assert cookie.read_text(encoding="utf-8") == "old-cookie"
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
assert not (profile / "partial.txt").exists()
assert cookie_backup is not None
assert state_backup is not None
assert profile_backup is not None
def test_pgy_scan_login_clicks_qr_switch_image():
class FakeLocator:
def __init__(self):
self.first = self
self.clicked = False
self.evaluated = False
def count(self):
return 1
def is_visible(self):
return True
def click(self, timeout):
self.clicked = True
def evaluate(self, script):
assert script == "e => e.click()"
self.evaluated = True
class FakePage:
def __init__(self):
self.selectors = []
self.qr_switch = FakeLocator()
def wait_for_timeout(self, _milliseconds):
pass
def get_by_text(self, _pattern):
return FakeLocator()
def locator(self, selector):
self.selectors.append(selector)
return self.qr_switch
page = FakePage()
pgy_xhs_scraper_v2.open_scan_login(page)
assert 'img[src*="qr_code"]' in page.selectors
assert page.qr_switch.evaluated
def test_failed_douyin_relogin_restores_cookie_storage_and_profile(tmp_path, monkeypatch):
cookie = tmp_path / "douyin_cookies.json"
storage = tmp_path / "douyin_storage_state.json"
profile = tmp_path / "profile"
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
storage.write_text("old-storage", encoding="utf-8")
profile.mkdir()
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(relogin_douyin, "COOKIE_FILE", cookie)
monkeypatch.setattr(relogin_douyin, "STATE_FILE", storage)
monkeypatch.setattr(relogin_douyin, "PROFILE_DIR", profile)
monkeypatch.setattr(relogin_douyin.subprocess, "call", lambda *_args, **_kwargs: 1)
monkeypatch.setattr(relogin_douyin.sys, "argv", ["relogin_douyin.py"])
assert relogin_douyin.main() == 1
assert cookie.read_text(encoding="utf-8") == '[{"name":"sessionid","value":"old"}]'
assert storage.read_text(encoding="utf-8") == "old-storage"
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
tests/modules/content_marketing/test_relogin_runtime_bindings.py
-410
View File
@@ -1,410 +0,0 @@
from __future__ import annotations
import json
from pathlib import Path
from types import ModuleType
import pytest
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
from gyxx_flow.modules.content_marketing.data.tools import (
relogin_bilibili,
relogin_douyin,
relogin_pgy,
relogin_runtime_paths,
relogin_xiaohongshu,
relogin_xingtu,
)
from gyxx_flow.modules.content_marketing.data.tools.relogin_runtime_paths import (
CONSUMER_BINDING_IDS,
child_browser_environment,
resolve_relogin_runtime_paths,
sync_relogin_state_to_consumers,
)
from gyxx_flow.modules.content_marketing.runtime_paths import PATHS
PROJECT_ROOT = Path(__file__).resolve().parents[3]
def test_relogin_paths_prefer_injected_runtime_binding(tmp_path: Path) -> None:
cookie = tmp_path / "binding" / "cookies.json"
storage = tmp_path / "binding" / "storage_state.json"
profile = tmp_path / "binding" / "profile"
resolved = resolve_relogin_runtime_paths(
"pgy",
{
"GYXX_BROWSER_COOKIE_FILE": str(cookie),
"GYXX_BROWSER_STORAGE_STATE_FILE": str(storage),
"GYXX_BROWSER_PROFILE_DIR": str(profile),
},
)
assert resolved.cookie_file == cookie.resolve()
assert resolved.storage_state_file == storage.resolve()
assert resolved.profile_dir == profile.resolve()
def test_relogin_paths_keep_legacy_standalone_layout() -> None:
resolved = resolve_relogin_runtime_paths("xiaohongshu", {})
assert resolved.cookie_file == (
PATHS.state_root / "cookies/xiaohongshu_cookies.json"
).resolve()
assert resolved.storage_state_file == (
PATHS.state_root / "cookies/xiaohongshu_storage_state.json"
).resolve()
assert resolved.profile_dir == (
PATHS.state_root / "browser-profiles/xiaohongshu"
).resolve()
def test_child_login_environment_preserves_wrapper_binding(tmp_path: Path) -> None:
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
environment = child_browser_environment(
cookie_file=cookie,
storage_state_file=storage,
profile_dir=profile,
environment={"GYXX_MODULE_ID": "content_marketing", "KEEP": "1"},
)
assert environment["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(
storage.resolve()
)
assert environment["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
assert "GYXX_MODULE_ID" not in environment
assert environment["KEEP"] == "1"
@pytest.mark.parametrize(
("module", "platform", "login_cookie", "argv"),
[
(
relogin_bilibili,
"bilibili",
"SESSDATA",
["relogin_bilibili.py"],
),
(relogin_douyin, "douyin", "sessionid", ["relogin_douyin.py"]),
(relogin_pgy, "pgy", "web_session", ["relogin_pgy.py"]),
(
relogin_xiaohongshu,
"xiaohongshu",
"web_session",
["relogin_xiaohongshu.py"],
),
(
relogin_xingtu,
"xingtu",
"sessionid",
["relogin_xingtu.py", "--force"],
),
],
)
def test_relogin_wrapper_validates_the_same_binding_written_by_child(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
module: ModuleType,
platform: str,
login_cookie: str,
argv: list[str],
) -> None:
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
monkeypatch.setattr(module.sys, "argv", argv)
monkeypatch.setenv("GYXX_MODULE_ID", "content_marketing")
if module is relogin_xingtu:
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
synced: list[tuple[str, Path, Path]] = []
def fake_sync(
synced_platform: str,
*,
cookie_file: Path,
storage_state_file: Path,
) -> tuple[Path, ...]:
synced.append(
(synced_platform, cookie_file.resolve(), storage_state_file.resolve())
)
return ()
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fake_sync)
def fake_call(
_command: list[str],
*,
cwd: str,
env: dict[str, str],
) -> int:
assert cwd == str(module.PROJECT_DIR)
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
assert env["GYXX_BROWSER_STORAGE_STATE_FILE"] == str(storage.resolve())
assert env["GYXX_BROWSER_PROFILE_DIR"] == str(profile.resolve())
assert "GYXX_MODULE_ID" not in env
cookie.write_text(
json.dumps([{"name": login_cookie, "value": "fresh"}]),
encoding="utf-8",
)
storage.write_text("{}", encoding="utf-8")
profile.mkdir()
return 0
monkeypatch.setattr(module.subprocess, "call", fake_call)
assert module.main() == 0
assert cookie.is_file()
assert storage.is_file()
assert profile.is_dir()
assert synced == [(platform, cookie.resolve(), storage.resolve())]
@pytest.mark.parametrize(
("module", "platform", "login_cookie", "cookie_domain", "argv"),
[
(
relogin_bilibili,
"bilibili",
"SESSDATA",
".bilibili.com",
["relogin_bilibili.py"],
),
(
relogin_douyin,
"douyin",
"sessionid",
".douyin.com",
["relogin_douyin.py"],
),
(
relogin_pgy,
"pgy",
"web_session",
".xiaohongshu.com",
["relogin_pgy.py"],
),
(
relogin_xiaohongshu,
"xiaohongshu",
"web_session",
".xiaohongshu.com",
["relogin_xiaohongshu.py"],
),
(
relogin_xingtu,
"xingtu",
"sessionid",
".xingtu.cn",
["relogin_xingtu.py", "--force"],
),
],
)
def test_relogin_wrapper_restores_previous_state_when_publish_fails(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
module: ModuleType,
platform: str,
login_cookie: str,
cookie_domain: str,
argv: list[str],
) -> None:
cookie = tmp_path / "cookies.json"
storage = tmp_path / "storage_state.json"
profile = tmp_path / "profile"
old_cookie = '[{"name":"old","value":"old-session"}]'
old_storage = '{"cookies":[{"name":"old"}]}'
cookie.write_text(old_cookie, encoding="utf-8")
storage.write_text(old_storage, encoding="utf-8")
profile.mkdir()
(profile / "old-state.txt").write_text("old-profile", encoding="utf-8")
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
monkeypatch.setattr(module, "STATE_FILE", storage)
monkeypatch.setattr(module, "PROFILE_DIR", profile)
monkeypatch.setattr(module.sys, "argv", argv)
if module is relogin_xingtu:
monkeypatch.setattr(module, "terminate_profile_processes", lambda: None)
monkeypatch.setattr(module, "sync_to_douyin_scraper", lambda: None)
def fake_call(
_command: list[str],
*,
cwd: str,
env: dict[str, str],
) -> int:
assert cwd == str(module.PROJECT_DIR)
assert env["GYXX_BROWSER_COOKIE_FILE"] == str(cookie.resolve())
cookie.write_text(
json.dumps(
[
{
"name": login_cookie,
"value": "fresh-session",
"domain": cookie_domain,
}
]
),
encoding="utf-8",
)
storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
profile.mkdir()
return 0
def fail_publish(
synced_platform: str,
*,
cookie_file: Path,
storage_state_file: Path,
) -> tuple[Path, ...]:
assert synced_platform == platform
assert cookie_file == cookie
assert storage_state_file == storage
raise OSError("simulated consumer publish failure")
monkeypatch.setattr(module.subprocess, "call", fake_call)
monkeypatch.setattr(module, "sync_relogin_state_to_consumers", fail_publish)
assert module.main() == 1
assert cookie.read_text(encoding="utf-8") == old_cookie
assert storage.read_text(encoding="utf-8") == old_storage
assert (profile / "old-state.txt").read_text(encoding="utf-8") == (
"old-profile"
)
@pytest.mark.parametrize(
("platform", "cookie_name", "cookie_domain"),
[
("bilibili", "SESSDATA", ".bilibili.com"),
("douyin", "sessionid", ".douyin.com"),
("pgy", "web_session", ".xiaohongshu.com"),
("xiaohongshu", "web_session", ".xiaohongshu.com"),
("xingtu", "sessionid", ".xingtu.cn"),
],
)
def test_successful_relogin_atomically_updates_unique_consumer_state(
tmp_path: Path,
platform: str,
cookie_name: str,
cookie_domain: str,
) -> None:
data_root = tmp_path / "data"
source_cookie = tmp_path / "relogin" / "cookies.json"
source_storage = tmp_path / "relogin" / "storage_state.json"
source_cookie.parent.mkdir(parents=True)
fresh_cookie = [
{
"name": cookie_name,
"value": "fresh-session",
"domain": cookie_domain,
}
]
fresh_storage = {"cookies": fresh_cookie, "origins": []}
source_cookie.write_text(json.dumps(fresh_cookie), encoding="utf-8")
source_storage.write_text(json.dumps(fresh_storage), encoding="utf-8")
catalog = RuntimeIntegrationCatalog.load_default(
project_root=PROJECT_ROOT,
data_root=data_root,
)
consumer_id = CONSUMER_BINDING_IDS[platform][0]
consumer = catalog.binding_for(consumer_id)
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
consumer.cookie_file.write_text('[{"name":"old"}]', encoding="utf-8")
consumer.storage_state_file.write_text(
'{"cookies":[{"name":"old"}]}',
encoding="utf-8",
)
destinations = sync_relogin_state_to_consumers(
platform,
cookie_file=source_cookie,
storage_state_file=source_storage,
environment={
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
"GYXX_DATA_ROOT": str(data_root),
},
)
assert source_cookie.resolve() != consumer.cookie_file.resolve()
assert source_storage.resolve() != consumer.storage_state_file.resolve()
assert destinations == (consumer.cookie_file.resolve(),)
assert json.loads(consumer.cookie_file.read_text(encoding="utf-8")) == (
fresh_cookie
)
assert json.loads(
consumer.storage_state_file.read_text(encoding="utf-8")
) == fresh_storage
def test_consumer_state_is_rolled_back_if_atomic_publish_fails(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
data_root = tmp_path / "data"
source_cookie = tmp_path / "relogin" / "cookies.json"
source_storage = tmp_path / "relogin" / "storage_state.json"
source_cookie.parent.mkdir(parents=True)
source_cookie.write_text(
json.dumps(
[
{
"name": "SESSDATA",
"value": "fresh-session",
"domain": ".bilibili.com",
}
]
),
encoding="utf-8",
)
source_storage.write_text('{"cookies":[],"origins":[]}', encoding="utf-8")
catalog = RuntimeIntegrationCatalog.load_default(
project_root=PROJECT_ROOT,
data_root=data_root,
)
consumer = catalog.binding_for(CONSUMER_BINDING_IDS["bilibili"][0])
consumer.cookie_file.parent.mkdir(parents=True, exist_ok=True)
old_cookie = '[{"name":"SESSDATA","value":"old"}]'
old_storage = '{"cookies":[{"name":"old"}]}'
consumer.cookie_file.write_text(old_cookie, encoding="utf-8")
consumer.storage_state_file.write_text(old_storage, encoding="utf-8")
original_replace = relogin_runtime_paths.os.replace
failed = False
def fail_second_replace(source: Path | str, target: Path | str) -> None:
nonlocal failed
source_path = Path(source)
target_path = Path(target)
if (
not failed
and source_path.suffix == ".tmp"
and target_path == consumer.storage_state_file
):
failed = True
raise OSError("simulated storage publish failure")
original_replace(source, target)
monkeypatch.setattr(relogin_runtime_paths.os, "replace", fail_second_replace)
with pytest.raises(OSError, match="simulated storage publish failure"):
sync_relogin_state_to_consumers(
"bilibili",
cookie_file=source_cookie,
storage_state_file=source_storage,
environment={
"GYXX_PROJECT_ROOT": str(PROJECT_ROOT),
"GYXX_DATA_ROOT": str(data_root),
},
)
assert consumer.cookie_file.read_text(encoding="utf-8") == old_cookie
assert consumer.storage_state_file.read_text(encoding="utf-8") == old_storage