fix(chanmama): 导出确认弹窗被浮层清理误删与代理出口拉黑修复
浮层清理只删可见浮层——页面常驻 90+ 个 display:none 弹窗模板(含导出确认框 模板),误删后 Vue 只在游离节点上"打开"弹窗,页面上仅剩灰色遮罩与滚动锁, 导出等待 1320s 必超时;导出等待改为逐秒弹窗状态机(可点即点/点不到先关/卡死 强删并补点导出),导出失败落截图与 HTML 诊断;浏览器强制直连并切换真 Chrome 非 stealthy 模式(StealthySession 下蝉妈妈前端会静默压制弹窗渲染);导出按钮 原生可信点击优先,防 @click 绑定在内层元素时 JS 合成点击无效。
This commit is contained in:
1 parent
130f06b320
commit
41f448420d
3 files changed
+687
-55
No files matched your search
@@ -52,6 +52,11 @@ REFRESH_WAIT_SECONDS = int(os.getenv("CHANMAMA_REFRESH_WAIT_SECONDS", "600"))
|
||||
CHANMAMA_LIST_READY_TIMEOUT_SECONDS = int(
|
||||
os.getenv("CHANMAMA_LIST_READY_TIMEOUT_SECONDS", "45")
|
||||
)
|
||||
# 公告/活动浮层可能盖住视频列表导致渲染判定一直不满足;渲染等待期间按
|
||||
# 该间隔无交互清理浮层(派发 ESC + 移除 fixed/absolute 浮层节点)。
|
||||
CHANMAMA_OVERLAY_DISMISS_INTERVAL_SECONDS = float(
|
||||
os.getenv("CHANMAMA_OVERLAY_DISMISS_INTERVAL_SECONDS", "10")
|
||||
)
|
||||
# 蝉妈妈导出接口实测可能在 15 分钟边界后才把 Excel 写入下载目录;把轮询
|
||||
# 上限做成可配置,并默认留出 20 分钟,避免文件已生成却被脚本判定为失败。
|
||||
CHANMAMA_EXPORT_WAIT_SECONDS = int(
|
||||
@@ -99,16 +104,20 @@ def create_driver():
|
||||
profile_dir = str(PATHS.browser_profile_dir)
|
||||
os.makedirs(profile_dir, exist_ok=True)
|
||||
cdp_url = os.getenv("GYXX_BROWSER_CDP_URL") or None
|
||||
# 蝉妈妈是国内站且其 WAF 会拉黑代理出口(Clash 节点 TLS 直接 reset,
|
||||
# 导出 API 全断、确认弹窗永远不弹);强制浏览器直连。
|
||||
direct_flags = ["--no-proxy-server"]
|
||||
# StealthySession(隐藏指纹补丁)下蝉妈妈前端会静默压制功能: 页面能开
|
||||
# 但导出确认弹窗不渲染(只剩空壳遮罩)。真 Chrome + 非 stealthy + 直连
|
||||
# 已实测弹窗完整渲染(含取消/导出按钮),不再使用 stealthy。
|
||||
try:
|
||||
return ScraplingDriver.start(
|
||||
profile_dir=profile_dir,
|
||||
download_dir=DOWNLOAD_DIR,
|
||||
cdp_url=cdp_url,
|
||||
stealthy=True,
|
||||
# 蝉妈妈新版 Vue 页面与 StealthySession + 系统 Chrome 组合
|
||||
# 不兼容:HTML 能返回,但 #app 不会挂载。使用 Scrapling
|
||||
# 自带 Chromium 仍保留隐藏指纹会话,且不影响已连接的 CDP。
|
||||
**({"real_chrome": False} if cdp_url is None else {}),
|
||||
stealthy=False,
|
||||
real_chrome=True,
|
||||
extra_flags=direct_flags,
|
||||
)
|
||||
except Exception as exc:
|
||||
if (
|
||||
@@ -125,8 +134,9 @@ def create_driver():
|
||||
profile_dir=profile_dir,
|
||||
download_dir=DOWNLOAD_DIR,
|
||||
cdp_url=None,
|
||||
stealthy=True,
|
||||
real_chrome=False,
|
||||
stealthy=False,
|
||||
real_chrome=True,
|
||||
extra_flags=direct_flags,
|
||||
)
|
||||
|
||||
|
||||
@@ -297,7 +307,10 @@ def handle_captcha(driver) -> bool:
|
||||
def verify_login(driver):
|
||||
"""验证是否已登录
|
||||
|
||||
通过访问目标页面,检查是否出现登录相关的元素
|
||||
通过访问目标页面,检查是否出现登录相关的元素。
|
||||
cookie 失效时蝉妈妈会把目标页重定向到 login.html,但重定向常在
|
||||
首屏渲染后数秒才发生——等待 URL 稳定后再判定,避免误判已登录
|
||||
(误判会跳过重新登录,导致后续导航一直超时)。
|
||||
"""
|
||||
try:
|
||||
# 跳转到目标页面
|
||||
@@ -305,10 +318,12 @@ def verify_login(driver):
|
||||
WebDriverWait(driver, 12).until(
|
||||
EC.presence_of_element_located((By.TAG_NAME, "body"))
|
||||
)
|
||||
time.sleep(3)
|
||||
# cookie 失效时的重定向在首屏渲染后 3~20 秒才发生,固定等满 10 秒
|
||||
# 再判定,不能因 URL 短暂未变而提前判为已登录
|
||||
time.sleep(10)
|
||||
cur_lower = driver.current_url.lower()
|
||||
|
||||
# 检查是否跳转回登录页 / 注册页(cookie 失效时蝉妈妈会重定向到 login.html 或 register.html)
|
||||
cur_lower = driver.current_url.lower()
|
||||
if "login" in cur_lower or "register" in cur_lower:
|
||||
print(f"🔒 检测到被重定向到登录/注册页 ({driver.current_url}),cookie 已失效")
|
||||
return False
|
||||
@@ -458,6 +473,180 @@ def login(driver):
|
||||
return False
|
||||
|
||||
|
||||
def _dismiss_overlays(driver) -> int:
|
||||
"""无交互清理遮挡浮层:派发 ESC 并移除 fixed/absolute 定位的弹层。"""
|
||||
try:
|
||||
removed = int(driver.execute_script(_OVERLAY_DISMISS_SCRIPT) or 0)
|
||||
except Exception as error:
|
||||
print(f"⚠️ 浮层清理脚本执行失败(忽略): {error}")
|
||||
return 0
|
||||
if removed:
|
||||
print(f"🧹 已清理 {removed} 个遮挡浮层")
|
||||
return removed
|
||||
|
||||
|
||||
def _dismiss_export_stage_overlays(driver) -> int:
|
||||
"""导出阶段清理弹窗(ESC + 移除弹层节点)。
|
||||
|
||||
只在弹窗不可确认时调用;返回移除的弹窗数。
|
||||
"""
|
||||
try:
|
||||
removed = int(driver.execute_script(_EXPORT_STAGE_DISMISS_SCRIPT) or 0)
|
||||
except Exception as error:
|
||||
print(f"⚠️ 导出阶段弹窗清理脚本执行失败(忽略): {error}")
|
||||
return 0
|
||||
if removed > 0:
|
||||
print(f"🧹 导出阶段清理 {removed} 个遮挡弹窗")
|
||||
return removed
|
||||
|
||||
|
||||
def _export_dialog_state(driver) -> str:
|
||||
"""探测导出阶段弹窗状态: confirmable / present / none。"""
|
||||
try:
|
||||
state = driver.execute_script(_EXPORT_DIALOG_STATE_SCRIPT)
|
||||
except Exception:
|
||||
return "none"
|
||||
return state if state in {"confirmable", "present", "none"} else "none"
|
||||
|
||||
|
||||
def _log_export_dialog_texts(driver) -> None:
|
||||
"""把当前可见弹窗的类名与文字打进日志,用于定位弹窗身份。"""
|
||||
try:
|
||||
texts = driver.execute_script(_EXPORT_DIALOG_TEXT_SCRIPT) or []
|
||||
except Exception:
|
||||
return
|
||||
for i, text in enumerate(texts[:4], 1):
|
||||
print(f"🪟 弹窗{i}内容: {text}")
|
||||
|
||||
|
||||
def _click_export_confirm(driver) -> bool:
|
||||
"""点击导出确认弹窗中的【导出】按钮,成功返回 True。"""
|
||||
# 关键:用 contains(.,'导出') 匹配后代文本("导出"在 <span> 内)
|
||||
confirm_selectors = [
|
||||
# 精确:主按钮(primary) + 含"导出"文字
|
||||
"//div[contains(@class,'el-dialog') or @role='dialog']//button[contains(.,'导出')]",
|
||||
# 弹窗内任意含"导出"的按钮
|
||||
"//div[contains(@class,'el-dialog') or @role='dialog']//button[contains(@class,'el-button--primary')][contains(.,'导出')]",
|
||||
# 兼容"确定"/"确认"
|
||||
"//div[contains(@class,'el-dialog') or @role='dialog']//button[contains(.,'确定')]",
|
||||
"//div[contains(@class,'el-dialog') or @role='dialog']//button[contains(.,'确认')]",
|
||||
]
|
||||
for sel in confirm_selectors:
|
||||
try:
|
||||
btn = driver.find_element(By.XPATH, sel)
|
||||
if btn.is_displayed():
|
||||
# 原生点击优先(可触发内层元素的 Vue @click),失败再退 JS 合成点击
|
||||
try:
|
||||
btn.click()
|
||||
except Exception:
|
||||
driver.execute_script("arguments[0].click();", btn)
|
||||
print(f"✅ 已在弹窗中点击导出按钮: {sel}")
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
|
||||
def _reclick_export_control(driver) -> bool:
|
||||
"""重新触发一次导出按钮(第一次点击可能已被弹窗吞掉)。
|
||||
|
||||
优先 Playwright 原生点击(在元素命中点派发可信鼠标事件,能触发绑定
|
||||
在按钮内层 span/div 上的 Vue @click);JS 合成 click 的目标是按钮自身,
|
||||
内层监听器不会触发——蝉妈妈改版后导出点击无声失效就是这个原因。
|
||||
"""
|
||||
if _native_click_export_button(driver):
|
||||
return True
|
||||
try:
|
||||
return bool(driver.execute_script(_CLICK_EXPORT_CONTROL_SCRIPT))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
_EXPORT_BUTTON_XPATHS = [
|
||||
"//button[contains(.,'导出数据')]",
|
||||
"//button[contains(.,'批量导出')]",
|
||||
"//button[contains(.,'导出视频')]",
|
||||
"//*[contains(@class,'export') and (self::button or self::a or @role='button')]",
|
||||
]
|
||||
|
||||
|
||||
def _native_click_export_button(driver) -> bool:
|
||||
"""用原生可信点击触发导出按钮,成功返回 True。"""
|
||||
for xp in _EXPORT_BUTTON_XPATHS:
|
||||
try:
|
||||
elements = driver.find_elements(By.XPATH, xp)
|
||||
except Exception:
|
||||
continue
|
||||
for el in elements:
|
||||
try:
|
||||
if el.is_displayed():
|
||||
el.click()
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
|
||||
def _wait_video_list_ready(driver, timeout_seconds: float) -> None:
|
||||
"""等待视频列表渲染就绪,期间周期性清理遮挡浮层。
|
||||
|
||||
语义与 WebDriverWait.until 保持一致:浏览器侧的暂时性异常只记录
|
||||
不中断,到达 deadline 后抛 TimeoutException。
|
||||
"""
|
||||
timeout = max(0.0, float(timeout_seconds))
|
||||
deadline = time.monotonic() + timeout
|
||||
next_dismiss_at = 0.0
|
||||
last_error: Exception | None = None
|
||||
while True:
|
||||
try:
|
||||
if driver.execute_script(_VIDEO_LIST_READY_SCRIPT):
|
||||
return
|
||||
except Exception as error:
|
||||
last_error = error
|
||||
now = time.monotonic()
|
||||
if now >= deadline:
|
||||
detail = f": {last_error}" if last_error else ""
|
||||
raise TimeoutException(
|
||||
f"condition was not met within {timeout}s{detail}"
|
||||
) from last_error
|
||||
if now >= next_dismiss_at:
|
||||
_dismiss_overlays(driver)
|
||||
next_dismiss_at = now + CHANMAMA_OVERLAY_DISMISS_INTERVAL_SECONDS
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def _dump_render_failure(driver, url: str) -> None:
|
||||
"""渲染失败时落截图与页面 HTML,供事后定位是浮层/验证码还是改版。"""
|
||||
try:
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
diag_dir = Path(DOWNLOAD_DIR) / "_diag"
|
||||
diag_dir.mkdir(parents=True, exist_ok=True)
|
||||
screenshot = diag_dir / f"render_failure_{stamp}.png"
|
||||
driver.save_screenshot(str(screenshot))
|
||||
(diag_dir / f"render_failure_{stamp}.html").write_text(
|
||||
driver.page_source, encoding="utf-8"
|
||||
)
|
||||
print(f"🩺 渲染失败诊断已保存: {screenshot}")
|
||||
except Exception as error:
|
||||
print(f"⚠️ 渲染失败诊断保存失败: {error}")
|
||||
|
||||
|
||||
def _dump_export_failure(driver) -> None:
|
||||
"""导出失败时落截图与页面 HTML,供事后定位弹窗/按钮的实际形态。"""
|
||||
try:
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
diag_dir = Path(DOWNLOAD_DIR) / "_diag"
|
||||
diag_dir.mkdir(parents=True, exist_ok=True)
|
||||
screenshot = diag_dir / f"export_failure_{stamp}.png"
|
||||
driver.save_screenshot(str(screenshot))
|
||||
(diag_dir / f"export_failure_{stamp}.html").write_text(
|
||||
driver.page_source, encoding="utf-8"
|
||||
)
|
||||
print(f"🩺 导出失败诊断已保存: {screenshot}")
|
||||
except Exception as error:
|
||||
print(f"⚠️ 导出失败诊断保存失败: {error}")
|
||||
|
||||
|
||||
def navigate_to_target(driver, url):
|
||||
"""跳转到目标博主视频页面"""
|
||||
last_error = None
|
||||
@@ -475,12 +664,11 @@ def navigate_to_target(driver, url):
|
||||
)
|
||||
time.sleep(3) # 等待页面数据加载
|
||||
# 截图中的页面虽然 URL 正确,但正文尚未渲染;在交给导出逻辑
|
||||
# 前确认虚拟视频列表已经出现,空白页则有限重载一次。
|
||||
WebDriverWait(driver, CHANMAMA_LIST_READY_TIMEOUT_SECONDS).until(
|
||||
lambda current: bool(
|
||||
current.execute_script(_VIDEO_LIST_READY_SCRIPT)
|
||||
)
|
||||
)
|
||||
# 前确认虚拟视频列表已经出现,空白页则有限重载一次。浮层多在
|
||||
# 首屏挂载后才弹出:进等待前先清一轮,等待期间按间隔再清,
|
||||
# 避免延迟弹出的公告/活动浮层永久挡住渲染判定。
|
||||
_dismiss_overlays(driver)
|
||||
_wait_video_list_ready(driver, CHANMAMA_LIST_READY_TIMEOUT_SECONDS)
|
||||
print(f"✅ 已到达并完成渲染目标页面: {current_url}")
|
||||
return True
|
||||
except Exception as exc:
|
||||
@@ -499,6 +687,7 @@ def navigate_to_target(driver, url):
|
||||
except Exception as reset_exc:
|
||||
print(f"⚠️ 蝉妈妈页面重置失败,继续重试目标页: {reset_exc}")
|
||||
time.sleep(1)
|
||||
_dump_render_failure(driver, url)
|
||||
raise RuntimeError(f"蝉妈妈目标页加载失败: {url}; {last_error}") from last_error
|
||||
|
||||
|
||||
@@ -539,6 +728,206 @@ def _find_video_table(driver):
|
||||
return WebDriverWait(driver, 15).until(locate)
|
||||
|
||||
|
||||
# 蝉妈妈公告/活动弹窗以 fixed/absolute 浮层盖住页面,且类名不稳定、未必
|
||||
# 有关闭按钮。这里不点叉:派发 ESC 键事件(多数组件库监听 document 关闭
|
||||
# 弹窗),再移除定位浮层节点并解除其对 html/body 的滚动锁;已知类名未
|
||||
# 命中时兜底移除 z-index 高且覆盖大半屏的 fixed 层。列表区
|
||||
# (.aweme-record-wrapper/.list-box/.list-auth-wrap-block) 内的元素永不命中。
|
||||
# .v-modal 是 ElementUI 直接挂在 <body> 下的灰色遮罩,与弹窗本体分离,
|
||||
# 必须显式清理,否则弹窗删掉后页面仍然灰屏、无法滚动;滚动锁 class
|
||||
# (el-popup-parent--hidden 等)同理。
|
||||
# 关键约束:只移除"可见"的浮层。页面常驻 90+ 个 display:none 的隐藏弹窗
|
||||
# 模板(含导出确认框的模板)——删掉它们后 Vue 只会在游离节点上"打开"弹窗,
|
||||
# 页面上永远渲染不出来(导出只剩灰遮罩),因此 display:none/零尺寸节点
|
||||
# 一律跳过。
|
||||
_OVERLAY_DISMISS_SCRIPT = r"""
|
||||
() => {
|
||||
const esc = new KeyboardEvent('keydown', {
|
||||
key: 'Escape', code: 'Escape', keyCode: 27, which: 27, bubbles: true,
|
||||
});
|
||||
document.dispatchEvent(esc);
|
||||
window.dispatchEvent(esc);
|
||||
let removed = 0;
|
||||
const seen = new Set();
|
||||
const isOverlay = (el) => {
|
||||
const style = window.getComputedStyle(el);
|
||||
return style.position === 'fixed' || style.position === 'absolute';
|
||||
};
|
||||
const drop = (el) => {
|
||||
if (seen.has(el) || !isOverlay(el)) return;
|
||||
const style = window.getComputedStyle(el);
|
||||
if (style.display === 'none') return;
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width === 0 || rect.height === 0) return;
|
||||
if (el.closest('.aweme-record-wrapper,.list-box,.list-auth-wrap-block')) return;
|
||||
seen.add(el);
|
||||
el.remove();
|
||||
removed += 1;
|
||||
};
|
||||
const selectors = [
|
||||
'.el-dialog__wrapper', '.el-overlay', '.el-message-box__wrapper',
|
||||
'.ant-modal-mask', '.ant-modal-wrap', '.modal-mask', '.modal-dialog',
|
||||
'.v-modal', '.v-modal-inner',
|
||||
'[class*="popup"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="mask"]',
|
||||
];
|
||||
for (const selector of selectors) {
|
||||
document.querySelectorAll(selector).forEach(drop);
|
||||
}
|
||||
if (removed === 0) {
|
||||
document.querySelectorAll('body *').forEach((el) => {
|
||||
const style = window.getComputedStyle(el);
|
||||
const zIndex = Number.parseInt(style.zIndex, 10);
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (Number.isFinite(zIndex) && zIndex > 500
|
||||
&& rect.height > window.innerHeight * 0.5) {
|
||||
drop(el);
|
||||
}
|
||||
});
|
||||
}
|
||||
document.body.classList.remove(
|
||||
'el-popup-parent--hidden', 'el-popup-parent--overflow-hidden', 'overflow-hidden'
|
||||
);
|
||||
document.documentElement.classList.remove(
|
||||
'el-popup-parent--hidden', 'el-popup-parent--overflow-hidden', 'overflow-hidden'
|
||||
);
|
||||
document.documentElement.style.overflow = '';
|
||||
document.body.style.overflow = '';
|
||||
document.body.style.paddingRight = '';
|
||||
return removed;
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
# 导出阶段专用清理:与 _OVERLAY_DISMISS_SCRIPT 同一套清理思路,但
|
||||
# 调用时机由 _export_dialog_state 决定——只在弹窗"没有可点的确认按钮"时
|
||||
# 才执行(确认框可点时直接点导出,绝不清)。三件事:
|
||||
# 1) 先 ESC + 点弹窗自带的关闭(×)按钮,走框架的正常关闭路径;
|
||||
# 2) 再移除残留的弹层节点——必须包含 .v-modal:ElementUI 的灰色遮罩直接
|
||||
# 挂在 <body> 下、与弹窗本体分离,漏掉它就是"删了弹窗页面仍灰屏";
|
||||
# 3) 最后解除滚动锁:清掉 body/html 上的 el-popup-parent--hidden 等
|
||||
# 框架加的 overflow 锁 class 与内联样式,否则页面删完遮罩也不能滚动。
|
||||
_EXPORT_STAGE_DISMISS_SCRIPT = r"""
|
||||
() => {
|
||||
const visible = (el) => {
|
||||
const style = window.getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
return style.display !== 'none' && style.visibility !== 'hidden'
|
||||
&& style.opacity !== '0' && rect.width > 0 && rect.height > 0;
|
||||
};
|
||||
const esc = new KeyboardEvent('keydown', {
|
||||
key: 'Escape', code: 'Escape', keyCode: 27, which: 27, bubbles: true,
|
||||
});
|
||||
document.dispatchEvent(esc);
|
||||
window.dispatchEvent(esc);
|
||||
// 先走正常关闭:点弹窗自己的 × / 关闭按钮
|
||||
let closed = 0;
|
||||
document.querySelectorAll(
|
||||
'.el-dialog__wrapper [class*="close"],.el-message-box__wrapper [class*="close"],'
|
||||
+ '.ant-modal-wrap [class*="close"],[class*="dialog"] [class*="close"],'
|
||||
+ '[class*="modal"] [class*="close"],[class*="popup"] [class*="close"]'
|
||||
).forEach((el) => {
|
||||
if (visible(el)) { el.click(); closed += 1; }
|
||||
});
|
||||
let removed = 0;
|
||||
const seen = new Set();
|
||||
const drop = (el) => {
|
||||
if (seen.has(el)) return;
|
||||
const style = window.getComputedStyle(el);
|
||||
if (style.position !== 'fixed' && style.position !== 'absolute') return;
|
||||
if (style.display === 'none') return;
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width === 0 || rect.height === 0) return;
|
||||
if (el.closest('.aweme-record-wrapper,.list-box,.list-auth-wrap-block')) return;
|
||||
seen.add(el);
|
||||
el.remove();
|
||||
removed += 1;
|
||||
};
|
||||
const selectors = [
|
||||
'.el-dialog__wrapper', '.el-overlay', '.el-message-box__wrapper',
|
||||
'.ant-modal-mask', '.ant-modal-wrap', '.modal-mask', '.modal-dialog',
|
||||
'.v-modal', '.v-modal-inner',
|
||||
'[class*="popup"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="mask"]',
|
||||
];
|
||||
for (const selector of selectors) {
|
||||
document.querySelectorAll(selector).forEach(drop);
|
||||
}
|
||||
if (removed === 0) {
|
||||
document.querySelectorAll('body *').forEach((el) => {
|
||||
const style = window.getComputedStyle(el);
|
||||
const zIndex = Number.parseInt(style.zIndex, 10);
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (Number.isFinite(zIndex) && zIndex > 500
|
||||
&& rect.height > window.innerHeight * 0.5) {
|
||||
drop(el);
|
||||
}
|
||||
});
|
||||
}
|
||||
// 解除滚动锁:框架加的锁 class + 内联 overflow 一起清
|
||||
document.body.classList.remove(
|
||||
'el-popup-parent--hidden', 'el-popup-parent--overflow-hidden', 'overflow-hidden'
|
||||
);
|
||||
document.documentElement.classList.remove(
|
||||
'el-popup-parent--hidden', 'el-popup-parent--overflow-hidden', 'overflow-hidden'
|
||||
);
|
||||
document.documentElement.style.overflow = '';
|
||||
document.body.style.overflow = '';
|
||||
document.body.style.paddingRight = '';
|
||||
return removed + closed;
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
# 导出阶段弹窗状态机:'confirmable'=有可见弹窗且弹窗里有可点的
|
||||
# 导出/确定/确认按钮(直接点,不能清);'present'=弹窗可见但没有可点
|
||||
# 的确认按钮(空壳遮罩/公告,ESC 关不掉就强删);'none'=没有弹窗。
|
||||
_EXPORT_DIALOG_STATE_SCRIPT = r"""
|
||||
() => {
|
||||
const visible = (el) => {
|
||||
const style = window.getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
return style.display !== 'none' && style.visibility !== 'hidden'
|
||||
&& style.opacity !== '0' && rect.width > 0 && rect.height > 0;
|
||||
};
|
||||
const roots = [...document.querySelectorAll(
|
||||
'.el-dialog__wrapper,.el-message-box__wrapper,.el-overlay,'
|
||||
+ '.ant-modal-wrap,.ant-modal-mask,[role="dialog"],'
|
||||
+ '[class*="dialog"],[class*="modal"],[class*="popup"]'
|
||||
)].filter(visible);
|
||||
for (const root of roots) {
|
||||
const confirmable = [...root.querySelectorAll(
|
||||
'button,a,[role="button"],[class*="confirm"],[class*="primary"]'
|
||||
)].some((el) => {
|
||||
if (!visible(el)) return false;
|
||||
const text = (el.innerText || el.value || '').replace(/\s+/g, '');
|
||||
return /导出|确定|确认/.test(text);
|
||||
});
|
||||
if (confirmable) return 'confirmable';
|
||||
}
|
||||
return roots.length ? 'present' : 'none';
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
# 抓取当前可见弹窗的文字内容(每窗截前 160 字),写进日志用于定位弹窗身份。
|
||||
_EXPORT_DIALOG_TEXT_SCRIPT = r"""
|
||||
() => {
|
||||
const visible = (el) => {
|
||||
const style = window.getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
return style.display !== 'none' && style.visibility !== 'hidden'
|
||||
&& style.opacity !== '0' && rect.width > 0 && rect.height > 0;
|
||||
};
|
||||
return [...document.querySelectorAll(
|
||||
'.el-dialog__wrapper,.el-message-box__wrapper,.el-overlay,'
|
||||
+ '.ant-modal-wrap,[role="dialog"],[class*="dialog"],[class*="modal"],[class*="popup"]'
|
||||
)].filter(visible).map((el) => {
|
||||
const text = (el.innerText || '').replace(/\s+/g, ' ').trim();
|
||||
const classes = String(el.className || '').slice(0, 80);
|
||||
return `[${classes}] ${text.slice(0, 160)}`;
|
||||
});
|
||||
}
|
||||
"""
|
||||
|
||||
_VIDEO_LIST_READY_SCRIPT = r"""
|
||||
() => {
|
||||
const visible = (el) => {
|
||||
@@ -909,8 +1298,39 @@ def _max_excel_mtime():
|
||||
|
||||
|
||||
def refresh_then_export_accounts(driver, urls, wait_seconds=REFRESH_WAIT_SECONDS):
|
||||
"""两阶段执行:先刷新所有账号全部分页,再等待并重新导出。"""
|
||||
"""两阶段执行:先刷新所有账号全部分页,再等待并重新导出。
|
||||
|
||||
运行中途登录态失效(目标页被重定向到登录页)时,自动用账号密码重新
|
||||
登录并重试当前账号一次;整套运行最多自动重登一次,避免死循环。
|
||||
"""
|
||||
refresh_summaries = []
|
||||
relogin_state = {"attempted": False}
|
||||
|
||||
def _relogin_if_expired(exc: BaseException) -> bool:
|
||||
"""登录态失效类异常时自动重登一次;返回是否已恢复登录。"""
|
||||
if relogin_state["attempted"] or "登录态失效" not in str(exc):
|
||||
return False
|
||||
relogin_state["attempted"] = True
|
||||
print("🔐 检测到登录态失效,尝试账号密码自动重新登录...")
|
||||
try:
|
||||
ok = login(driver)
|
||||
except Exception as login_exc:
|
||||
print(f"⚠️ 自动重新登录异常: {login_exc}")
|
||||
return False
|
||||
if ok:
|
||||
print("✅ 自动重新登录成功,重试当前账号")
|
||||
return bool(ok)
|
||||
|
||||
def _empty_summary(exc: BaseException) -> dict:
|
||||
return {
|
||||
"clicked": 0,
|
||||
"pending": 0,
|
||||
"zero_without_button": 0,
|
||||
"unconfirmed": 0,
|
||||
"pages": 0,
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
for index, url in enumerate(urls, 1):
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f"🔄 刷新阶段 博主 {index}/{len(urls)}: {url}")
|
||||
@@ -919,15 +1339,16 @@ def refresh_then_export_accounts(driver, urls, wait_seconds=REFRESH_WAIT_SECONDS
|
||||
navigate_to_target(driver, url)
|
||||
summary = refresh_zero_exposure_videos(driver)
|
||||
except Exception as exc:
|
||||
print(f"⚠️ 博主 {index} 刷新扫描异常: {exc},继续下一个")
|
||||
summary = {
|
||||
"clicked": 0,
|
||||
"pending": 0,
|
||||
"zero_without_button": 0,
|
||||
"unconfirmed": 0,
|
||||
"pages": 0,
|
||||
"error": str(exc),
|
||||
}
|
||||
if _relogin_if_expired(exc):
|
||||
try:
|
||||
navigate_to_target(driver, url)
|
||||
summary = refresh_zero_exposure_videos(driver)
|
||||
except Exception as retry_exc:
|
||||
print(f"⚠️ 博主 {index} 重登后刷新仍失败: {retry_exc},继续下一个")
|
||||
summary = _empty_summary(retry_exc)
|
||||
else:
|
||||
print(f"⚠️ 博主 {index} 刷新扫描异常: {exc},继续下一个")
|
||||
summary = _empty_summary(exc)
|
||||
refresh_summaries.append(summary)
|
||||
|
||||
should_wait = any(
|
||||
@@ -956,7 +1377,21 @@ def refresh_then_export_accounts(driver, urls, wait_seconds=REFRESH_WAIT_SECONDS
|
||||
print(f" 解析出 {len(records)} 条视频记录")
|
||||
all_records.extend(records)
|
||||
except Exception as exc:
|
||||
print(f"⚠️ 博主 {index} 导出异常: {exc},跳过")
|
||||
if _relogin_if_expired(exc):
|
||||
try:
|
||||
navigate_to_target(driver, url)
|
||||
pre_mtime = _max_excel_mtime()
|
||||
excel_path = export_video_data(driver, pre_mtime)
|
||||
if not excel_path:
|
||||
print(f"⚠️ 博主 {index} 重登后导出失败,跳过")
|
||||
continue
|
||||
records, _ = parse_chanmama_export(excel_path)
|
||||
print(f" 解析出 {len(records)} 条视频记录(重登后)")
|
||||
all_records.extend(records)
|
||||
except Exception as retry_exc:
|
||||
print(f"⚠️ 博主 {index} 重登后导出仍失败: {retry_exc},跳过")
|
||||
else:
|
||||
print(f"⚠️ 博主 {index} 导出异常: {exc},跳过")
|
||||
|
||||
return all_records, refresh_summaries
|
||||
|
||||
@@ -978,6 +1413,10 @@ def export_video_data(driver, pre_mtime: float = 0):
|
||||
except TimeoutException:
|
||||
print("⚠️ 视频列表加载超时,继续尝试导出...")
|
||||
|
||||
# 刷新/翻页阶段可能延迟弹出公告/活动弹窗,吞掉导出点击或盖住确认框;
|
||||
# 点导出按钮前先清一次(确认框受保护)。
|
||||
_dismiss_export_stage_overlays(driver)
|
||||
|
||||
# 方法1: 先在可见 DOM 中找真正的交互节点。新版按钮文字常在嵌套
|
||||
# span/div 内,旧的 contains(text()) 和第一个隐藏 locator 都会漏掉它。
|
||||
export_clicked = False
|
||||
@@ -989,8 +1428,11 @@ def export_video_data(driver, pre_mtime: float = 0):
|
||||
except TimeoutException:
|
||||
control = None
|
||||
if control:
|
||||
clicked = driver.execute_script(_CLICK_EXPORT_CONTROL_SCRIPT)
|
||||
export_clicked = bool(clicked)
|
||||
# 原生点击优先(可触发内层元素的 Vue @click),JS 合成点击兜底
|
||||
export_clicked = _native_click_export_button(driver)
|
||||
if not export_clicked:
|
||||
clicked = driver.execute_script(_CLICK_EXPORT_CONTROL_SCRIPT)
|
||||
export_clicked = bool(clicked)
|
||||
if export_clicked:
|
||||
print(f"✅ 已点击导出按钮: {control.get('text') or control.get('className')}")
|
||||
|
||||
@@ -1032,36 +1474,38 @@ def export_video_data(driver, pre_mtime: float = 0):
|
||||
return scrape_table_data(driver)
|
||||
|
||||
# 等待导出确认弹窗出现并点击【导出】按钮
|
||||
# 蝉妈妈弹窗标题"您即将进行数据导出",按钮为 橙色"导出"(文字在 <span> 内)
|
||||
# 蝉妈妈弹窗标题"您即将进行数据导出",按钮为 橙色"导出"(文字在 <span> 内)。
|
||||
# 逐秒探测弹窗状态:可确认→立即点导出;挂着但没有可点的确认按钮
|
||||
# (空壳遮罩/公告弹窗)→先 ESC+移除正常关,持续 5 秒以上就补点导出按钮
|
||||
# (第一次点击可能已被弹窗吞掉);没有弹窗→常规清理并定期补点导出。
|
||||
print("⏳ 等待导出确认弹窗...")
|
||||
export_confirmed = False
|
||||
for wait_sec in range(20):
|
||||
unconfirmable_since: float | None = None
|
||||
for wait_sec in range(36):
|
||||
time.sleep(1)
|
||||
try:
|
||||
# 关键:用 contains(.,'导出') 匹配后代文本("导出"在 <span> 内)
|
||||
confirm_selectors = [
|
||||
# 精确:主按钮(primary) + 含"导出"文字
|
||||
"//div[contains(@class,'el-dialog') or @role='dialog']//button[contains(.,'导出')]",
|
||||
# 弹窗内任意含"导出"的按钮
|
||||
"//div[contains(@class,'el-dialog') or @role='dialog']//button[contains(@class,'el-button--primary')][contains(.,'导出')]",
|
||||
# 兼容"确定"/"确认"
|
||||
"//div[contains(@class,'el-dialog') or @role='dialog']//button[contains(.,'确定')]",
|
||||
"//div[contains(@class,'el-dialog') or @role='dialog']//button[contains(.,'确认')]",
|
||||
]
|
||||
for sel in confirm_selectors:
|
||||
try:
|
||||
btn = driver.find_element(By.XPATH, sel)
|
||||
if btn.is_displayed():
|
||||
driver.execute_script("arguments[0].click();", btn)
|
||||
export_confirmed = True
|
||||
print(f"✅ 已在弹窗中点击导出按钮: {sel}")
|
||||
break
|
||||
except NoSuchElementException:
|
||||
continue
|
||||
if export_confirmed:
|
||||
state = _export_dialog_state(driver)
|
||||
if state == "confirmable":
|
||||
unconfirmable_since = None
|
||||
if _click_export_confirm(driver):
|
||||
export_confirmed = True
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
if state == "present":
|
||||
now = time.monotonic()
|
||||
if unconfirmable_since is None:
|
||||
_log_export_dialog_texts(driver)
|
||||
print("ℹ️ 弹窗可见但没有可点的确认按钮,尝试关闭")
|
||||
unconfirmable_since = now
|
||||
elif now - unconfirmable_since > 5:
|
||||
if _reclick_export_control(driver):
|
||||
print("🔁 清掉卡死弹窗后补点导出按钮")
|
||||
unconfirmable_since = None
|
||||
_dismiss_export_stage_overlays(driver)
|
||||
continue
|
||||
unconfirmable_since = None
|
||||
_dismiss_export_stage_overlays(driver)
|
||||
if wait_sec in (10, 20, 30) and _reclick_export_control(driver):
|
||||
print("🔁 确认框迟迟未出现,补点导出按钮")
|
||||
|
||||
if not export_confirmed:
|
||||
try:
|
||||
@@ -1093,9 +1537,20 @@ def export_video_data(driver, pre_mtime: float = 0):
|
||||
"⚠️ 未找到本次下载的 Excel文件(等待 "
|
||||
f"{CHANMAMA_EXPORT_WAIT_SECONDS + max(CHANMAMA_EXPORT_GRACE_SECONDS, 0)} 秒超时)"
|
||||
)
|
||||
_dump_export_failure(driver)
|
||||
return None
|
||||
|
||||
|
||||
def _download_complete(path: str) -> bool:
|
||||
"""判断下载文件是否已写完整(zip EOCD 在文件尾,截断时 is_zipfile 为假)。"""
|
||||
import zipfile
|
||||
|
||||
try:
|
||||
return zipfile.is_zipfile(path)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _wait_for_exported_excel(
|
||||
pre_mtime: float,
|
||||
*,
|
||||
@@ -1125,7 +1580,13 @@ def _wait_for_exported_excel(
|
||||
except Exception:
|
||||
pass
|
||||
new_path = _find_new_excel_after(pre_mtime)
|
||||
if new_path is not None:
|
||||
# 文件出现 ≠ 下载完成: 当前浏览器可能以最终名边下边写,截断的
|
||||
# zip 会让解析报 "File is not a zip file"。zip 的 EOCD 记录在
|
||||
# 文件末尾,is_zipfile 通过即认为下载已完整;路径不存在(测试桩)
|
||||
# 时保持旧行为直接返回。
|
||||
if new_path is not None and (
|
||||
not os.path.exists(new_path) or _download_complete(new_path)
|
||||
):
|
||||
return new_path
|
||||
now = time.time()
|
||||
if now >= deadline and not grace_logged and grace_deadline > deadline:
|
||||
|
||||
@@ -137,6 +137,7 @@ class ScraplingDriver:
|
||||
cdp_url: str | None = None,
|
||||
stealthy: bool = False,
|
||||
real_chrome: bool = True,
|
||||
extra_flags: list[str] | None = None,
|
||||
) -> ScraplingDriver:
|
||||
options: dict[str, Any] = {
|
||||
"headless": False,
|
||||
@@ -152,6 +153,7 @@ class ScraplingDriver:
|
||||
"extra_flags": [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-dev-shm-usage",
|
||||
*(extra_flags or []),
|
||||
],
|
||||
}
|
||||
reuse_existing_context = bool(cdp_url)
|
||||
@@ -211,6 +213,9 @@ class ScraplingDriver:
|
||||
[script, raw_arguments],
|
||||
)
|
||||
|
||||
def save_screenshot(self, path: str | os.PathLike[str]) -> bool:
|
||||
return bool(self._page.screenshot(path=str(Path(path))))
|
||||
|
||||
def get_cookies(self) -> list[dict[str, Any]]:
|
||||
context = self._owner.context
|
||||
return list(context.cookies()) if context is not None else []
|
||||
|
||||
Reference in new issue
Block a user