feat(product): 京东新版商智竞店采集改造
入口跳转、两段式日期选择、搜索定位与表格解析适配新版商智;配合类目下拉与 XHR 拦截兜底。
This commit is contained in:
1 parent
b0e1b5066d
commit
df11e195b7
1 file changed
+422
-294
+422
-294
@@ -32,10 +32,9 @@ from gyxx_flow.modules.shop_intelligence.browser_session import (
|
|||||||
)
|
)
|
||||||
from gyxx_flow.modules.shop_intelligence.collectors.jd_data_collector import (
|
from gyxx_flow.modules.shop_intelligence.collectors.jd_data_collector import (
|
||||||
close_profit_plan_popup_if_present,
|
close_profit_plan_popup_if_present,
|
||||||
select_last_week_in_old_datepicker,
|
|
||||||
)
|
)
|
||||||
from gyxx_flow.modules.shop_intelligence.config import JD_PEER_STORE_OUTPUT
|
from gyxx_flow.modules.shop_intelligence.config import JD_PEER_STORE_OUTPUT
|
||||||
from gyxx_flow.modules.shop_intelligence.paths import resolve_raw_output
|
from gyxx_flow.modules.shop_intelligence.paths import TMP_ROOT, resolve_raw_output
|
||||||
|
|
||||||
# ── 配置 ─────────────────────────────────────────────────────────────────────
|
# ── 配置 ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -67,7 +66,11 @@ CATEGORY_OUTPUT_NAMES = {
|
|||||||
"男包": "男士包袋",
|
"男包": "男士包袋",
|
||||||
}
|
}
|
||||||
|
|
||||||
VENDER_RANKS_URL = "https://jdsz.jd.com/szweb/view/industryMarket/venderRanks.html"
|
# 2026-10 京东下线旧版商智:venderRanks 旧入口 404,官方公告要求迁移新版
|
||||||
|
# jdsz.jd.com。新版等价页面是「行业榜单」的店铺榜 tab。
|
||||||
|
INDUSTRY_TOP_URL = "https://jdsz.jd.com/szweb/view/industry/industry-top.html"
|
||||||
|
# 兼容旧引用
|
||||||
|
VENDER_RANKS_URL = INDUSTRY_TOP_URL
|
||||||
|
|
||||||
# 新品采集配置
|
# 新品采集配置
|
||||||
NEW_PRODUCT_SHOPS = [
|
NEW_PRODUCT_SHOPS = [
|
||||||
@@ -398,97 +401,262 @@ def get_last_week_range():
|
|||||||
return last_monday, last_sunday
|
return last_monday, last_sunday
|
||||||
|
|
||||||
|
|
||||||
|
def _dump_peer_filter_debug(page: Page, tag: str) -> None:
|
||||||
|
"""日期/类目操作失败时保存现场,便于按页面实际 DOM 校准选择器。"""
|
||||||
|
try:
|
||||||
|
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||||
|
debug_dir = TMP_ROOT / "debug" / "jd_peer_filter"
|
||||||
|
debug_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
page.screenshot(path=str(debug_dir / f"{stamp}_{tag}.png"))
|
||||||
|
html_path = debug_dir / f"{stamp}_{tag}.html"
|
||||||
|
html_path.write_text(page.content(), encoding="utf-8")
|
||||||
|
print(f" [DEBUG] 现场已保存: {html_path}")
|
||||||
|
except Exception as exc:
|
||||||
|
print(f" [DEBUG] 现场保存失败: {exc}")
|
||||||
|
|
||||||
|
|
||||||
|
def _click_custom_day_cell(page: Page, date_str: str) -> None:
|
||||||
|
"""在自定义日期面板中点击指定日期格(按月份块消歧,真实坐标点击)。
|
||||||
|
|
||||||
|
面板为单月迷你日历:日格只带日号文本,须先定位「只含一个月份标题且
|
||||||
|
含日历格」的月份块,再在其中匹配日号;组件对合成点击不稳定,统一用
|
||||||
|
bounding_box 中心坐标做真实鼠标点击。
|
||||||
|
"""
|
||||||
|
center = page.evaluate(
|
||||||
|
"""(dateStr) => {
|
||||||
|
const [year, month, day] = dateStr.split('-');
|
||||||
|
const wantTitle = `${Number(year)}年${String(Number(month)).padStart(2, '0')}月`;
|
||||||
|
const dayText = String(Number(day));
|
||||||
|
const isMonthTitle = (text) => /^\\d{4}年\\d{2}月$/.test((text || '').trim());
|
||||||
|
const titleSpans = Array.from(document.querySelectorAll('span'))
|
||||||
|
.filter(el => el.getClientRects().length && isMonthTitle(el.innerText));
|
||||||
|
for (const span of titleSpans) {
|
||||||
|
if ((span.innerText || '').trim() !== wantTitle) continue;
|
||||||
|
let node = span.parentElement;
|
||||||
|
for (let depth = 0; node && depth < 10; depth += 1, node = node.parentElement) {
|
||||||
|
const titles = Array.from(node.querySelectorAll('span'))
|
||||||
|
.filter(s => isMonthTitle(s.innerText));
|
||||||
|
if (titles.length > 1) break;
|
||||||
|
if (titles.length === 1
|
||||||
|
&& node.querySelector('td.jmt-date-picker-calendar-cell')) {
|
||||||
|
const cells = Array.from(
|
||||||
|
node.querySelectorAll('td.jmt-date-picker-calendar-cell')
|
||||||
|
).filter(el => el.getBoundingClientRect().width > 0
|
||||||
|
&& (el.innerText || '').trim() === dayText);
|
||||||
|
if (cells.length) {
|
||||||
|
const rect = cells[0].getBoundingClientRect();
|
||||||
|
return {x: Math.round(rect.x + rect.width / 2),
|
||||||
|
y: Math.round(rect.y + rect.height / 2)};
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}""",
|
||||||
|
date_str,
|
||||||
|
)
|
||||||
|
if not center:
|
||||||
|
_dump_peer_filter_debug(page, f"day_not_found_{date_str}")
|
||||||
|
raise RuntimeError(f"自定义日期面板未找到日期格: {date_str}")
|
||||||
|
page.mouse.click(center["x"], center["y"])
|
||||||
|
|
||||||
|
|
||||||
|
def _open_custom_date_panel(page: Page, *, retries: int = 3) -> bool:
|
||||||
|
"""点击「自定义」标签打开日历;带开面板验证与重试。
|
||||||
|
|
||||||
|
面板行为实测有两种形态且跨运行不定:迷你单月(选起点即自动关闭)
|
||||||
|
与常驻双月(选完不关)。tag 点击在面板刚自动关闭后可能需要缓一下
|
||||||
|
再点,故带月标题可见性验证与重试。
|
||||||
|
"""
|
||||||
|
tag = page.locator(
|
||||||
|
"li.jmt-combo-date-picker-list-tag", has_text="自定义"
|
||||||
|
).first
|
||||||
|
for _attempt in range(retries):
|
||||||
|
try:
|
||||||
|
box = tag.bounding_box(timeout=5000)
|
||||||
|
except Exception:
|
||||||
|
box = None
|
||||||
|
if not box:
|
||||||
|
_dump_peer_filter_debug(page, "custom_tag_missing")
|
||||||
|
raise RuntimeError("未找到「自定义」时间按钮")
|
||||||
|
page.mouse.click(
|
||||||
|
box["x"] + box["width"] / 2, box["y"] + box["height"] / 2
|
||||||
|
)
|
||||||
|
page.wait_for_timeout(2000)
|
||||||
|
if _panel_months_visible(page):
|
||||||
|
return True
|
||||||
|
page.wait_for_timeout(1000)
|
||||||
|
return _panel_months_visible(page)
|
||||||
|
|
||||||
|
|
||||||
|
def _panel_months_visible(page) -> bool:
|
||||||
|
try:
|
||||||
|
return bool(
|
||||||
|
page.evaluate(
|
||||||
|
"() => Array.from(document.querySelectorAll('span'))"
|
||||||
|
".some(el => el.getClientRects().length"
|
||||||
|
" && /^\\d{4}年\\d{2}月$/.test((el.innerText || '').trim()))"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_month_visible(page: Page, month_title: str) -> None:
|
||||||
|
"""单月日历翻月:目标月份不可见时点「后一月」箭头(至多 12 次)。"""
|
||||||
|
|
||||||
|
def visible_months():
|
||||||
|
return page.evaluate(
|
||||||
|
"""() => Array.from(document.querySelectorAll('span'))
|
||||||
|
.filter(el => el.getClientRects().length)
|
||||||
|
.map(el => (el.innerText || '').trim())
|
||||||
|
.filter(t => /^\\d{4}年\\d{2}月$/.test(t))"""
|
||||||
|
)
|
||||||
|
|
||||||
|
for _ in range(12):
|
||||||
|
if month_title in visible_months():
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
arrow = page.locator('[data-event-content="后一月"]')
|
||||||
|
if arrow.count() == 0:
|
||||||
|
return
|
||||||
|
box = arrow.first.bounding_box(timeout=3000)
|
||||||
|
except Exception:
|
||||||
|
return
|
||||||
|
if not box:
|
||||||
|
return
|
||||||
|
page.mouse.click(
|
||||||
|
box["x"] + box["width"] / 2, box["y"] + box["height"] / 2
|
||||||
|
)
|
||||||
|
page.wait_for_timeout(1200)
|
||||||
|
|
||||||
|
|
||||||
|
def _custom_panel_state(page) -> str:
|
||||||
|
"""调试用:当前日期面板的可见月份与「当前:」显示。"""
|
||||||
|
try:
|
||||||
|
return str(
|
||||||
|
page.evaluate(
|
||||||
|
"""() => {
|
||||||
|
const months = Array.from(document.querySelectorAll('span'))
|
||||||
|
.filter(el => el.getClientRects().length)
|
||||||
|
.map(el => (el.innerText || '').trim())
|
||||||
|
.filter(t => /^\\d{4}年\\d{2}月$/.test(t));
|
||||||
|
const current = (document.body.innerText.match(/当前:[^\\n]{0,40}/) || [''])[0];
|
||||||
|
return {months: [...new Set(months)], current};
|
||||||
|
}"""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
return f"<state-error {exc}>"
|
||||||
|
|
||||||
|
|
||||||
|
def _custom_range_applied(page, start_mmdd: str) -> bool:
|
||||||
|
try:
|
||||||
|
body_text = page.locator("body").inner_text(timeout=3000)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
return start_mmdd in body_text.replace(" ", "")
|
||||||
|
|
||||||
|
|
||||||
def select_last_week_any_day(page):
|
def select_last_week_any_day(page):
|
||||||
"""点击日期选择器 → 按周查询 → 点上周任意一天(自动选中整周)"""
|
"""新版行业榜单:自定义区间选择上周一~上周日(状态机,3 轮重试)。
|
||||||
|
|
||||||
|
面板有两种形态且跨运行不定:
|
||||||
|
- 迷你单月:点击起点后自动关闭;重开面板点终点,选完立即生效。
|
||||||
|
- 常驻双月:起止选完面板不关;点面板外部(店铺搜索框)应用;
|
||||||
|
Escape 会取消选择。
|
||||||
|
统一状态机:选起点 → 面板仍开则同会话选终点,已关则重开选终点 →
|
||||||
|
未生效则点外部应用 → 仍未生效整轮重来。
|
||||||
|
"""
|
||||||
print(" 选择上周时间范围...")
|
print(" 选择上周时间范围...")
|
||||||
|
last_monday, last_sunday = get_last_week_range()
|
||||||
|
start_str = last_monday.strftime("%Y-%m-%d")
|
||||||
|
end_str = last_sunday.strftime("%Y-%m-%d")
|
||||||
|
start_month = f"{last_monday.year}年{last_monday.month:02d}月"
|
||||||
|
end_month = f"{last_sunday.year}年{last_sunday.month:02d}月"
|
||||||
|
start_mmdd = last_monday.strftime("%m-%d")
|
||||||
|
|
||||||
# 1. 点击日期选择器(显示"按天查询..."的元素)
|
for attempt in range(1, 4):
|
||||||
|
print(f" [state] 第 {attempt} 轮: {_custom_panel_state(page)}")
|
||||||
|
if not _open_custom_date_panel(page):
|
||||||
|
print(" [state] 面板未能打开,重试")
|
||||||
|
continue
|
||||||
|
_ensure_month_visible(page, start_month)
|
||||||
try:
|
try:
|
||||||
date_picker = page.locator('.jmtd-date-picker .jmtd-base-input-top').first
|
_click_custom_day_cell(page, start_str)
|
||||||
date_picker.wait_for(state="visible", timeout=5000)
|
except RuntimeError:
|
||||||
date_picker.click()
|
continue
|
||||||
print(" [OK] 点击日期选择器")
|
page.wait_for_timeout(1500)
|
||||||
except Exception as e:
|
print(f" [state] 点击开始后: {_custom_panel_state(page)}")
|
||||||
print(f" [WARN] 点击日期选择器失败: {e}")
|
|
||||||
# 兜底:点击包含"按天查询"文本的元素
|
if not _panel_months_visible(page):
|
||||||
|
# 迷你面板形态:起点选择后面板已自动关闭,重开选终点
|
||||||
|
page.wait_for_timeout(1200)
|
||||||
|
if not _open_custom_date_panel(page):
|
||||||
|
continue
|
||||||
|
_ensure_month_visible(page, end_month)
|
||||||
try:
|
try:
|
||||||
page.locator('text=按天查询').first.click()
|
_click_custom_day_cell(page, end_str)
|
||||||
print(" [OK] 使用文本定位点击日期选择器")
|
except RuntimeError:
|
||||||
|
continue
|
||||||
|
page.wait_for_timeout(2500)
|
||||||
|
print(f" [state] 点击结束后: {_custom_panel_state(page)}")
|
||||||
|
|
||||||
|
if _custom_range_applied(page, start_mmdd):
|
||||||
|
print(f" [OK] 已选择上周区间: {start_str} ~ {end_str}")
|
||||||
|
return
|
||||||
|
# 常驻双月形态:点面板外部应用(Escape 是取消,绝不能用)
|
||||||
|
try:
|
||||||
|
page.locator(
|
||||||
|
'input[placeholder="请输入店铺名称"]'
|
||||||
|
).first.click(timeout=4000)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
page.mouse.click(30, 400)
|
||||||
page.wait_for_timeout(2000)
|
|
||||||
|
|
||||||
# 2. 点击"按周查询"
|
|
||||||
try:
|
|
||||||
week_switch = page.locator('.jmtd-date-picker-switch-panel-item', has_text="按周查询").first
|
|
||||||
week_switch.wait_for(state="visible", timeout=5000)
|
|
||||||
week_switch.click()
|
|
||||||
print(" [OK] 点击按周查询")
|
|
||||||
except Exception as e:
|
|
||||||
print(f" [WARN] 点击按周查询失败: {e}")
|
|
||||||
# 兜底:用 JS 点击
|
|
||||||
page.evaluate("""() => {
|
|
||||||
const items = document.querySelectorAll('.jmtd-date-picker-switch-panel-item');
|
|
||||||
for (const item of items) {
|
|
||||||
if (item.textContent.includes('按周查询')) {
|
|
||||||
item.click();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}""")
|
|
||||||
page.wait_for_timeout(2000)
|
|
||||||
|
|
||||||
# 3. 等待日历弹窗并选择上周某一天
|
|
||||||
calendar = page.locator('.jmtd-date-picker-popup, .jmtd-date-picker-panel, [class*="picker-panel"], [class*="calendar-panel"]')
|
|
||||||
try:
|
|
||||||
calendar.wait_for(state="visible", timeout=5000)
|
|
||||||
except Exception:
|
|
||||||
print(" [WARN] 未检测到日历弹窗,尝试继续...")
|
|
||||||
page.wait_for_timeout(3000)
|
page.wait_for_timeout(3000)
|
||||||
|
if _custom_range_applied(page, start_mmdd):
|
||||||
|
print(f" [OK] 已选择上周区间(点外部应用): {start_str} ~ {end_str}")
|
||||||
|
return
|
||||||
|
print(f" [state] 本轮未生效: {_custom_panel_state(page)}")
|
||||||
|
|
||||||
select_last_week_in_old_datepicker(page, "peer_store")
|
_dump_peer_filter_debug(page, "range_not_applied")
|
||||||
|
raise RuntimeError(
|
||||||
try:
|
f"自定义日期区间未生效: 期望 {start_str} ~ {end_str}"
|
||||||
calendar.wait_for(state="hidden", timeout=5000)
|
)
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
page.wait_for_timeout(5000)
|
|
||||||
print(" 已切换到上周数据")
|
|
||||||
|
|
||||||
|
|
||||||
def reselect_last_week(page):
|
|
||||||
"""切换类目后重新选择上周日期(点击日期选择器 → 按周查询 → 点日期)"""
|
|
||||||
# 重新点击日期选择器
|
|
||||||
try:
|
|
||||||
date_picker = page.locator('.jmtd-date-picker .jmtd-base-input-top').first
|
|
||||||
date_picker.click()
|
|
||||||
except Exception:
|
|
||||||
page.locator('text=按天查询').first.click()
|
|
||||||
page.wait_for_timeout(1500)
|
|
||||||
|
|
||||||
# 点击按周查询
|
|
||||||
try:
|
|
||||||
week_switch = page.locator('.jmtd-date-picker-switch-panel-item', has_text="按周查询").first
|
|
||||||
week_switch.click()
|
def _selector_option_center(page: Page, label: str, *, exclude_toolbar: bool = True):
|
||||||
except Exception:
|
"""在下拉浮层里按精确文本定位 jmt-selector 选项,返回真实坐标。
|
||||||
page.evaluate("""() => {
|
|
||||||
const items = document.querySelectorAll('.jmtd-date-picker-switch-panel-item');
|
筛选栏控件本身也是 button.jmt-selector(显示当前选中值),必须
|
||||||
for (const item of items) {
|
排除,否则会把控件当选项点到。
|
||||||
if (item.textContent.includes('按周查询')) {
|
"""
|
||||||
item.click();
|
return page.evaluate(
|
||||||
return true;
|
"""(label) => {
|
||||||
|
const visible = (el) => {
|
||||||
|
const r = el.getBoundingClientRect();
|
||||||
|
return r.width > 0 && r.height > 0;
|
||||||
|
};
|
||||||
|
const buttons = Array.from(
|
||||||
|
document.querySelectorAll('div.jmt-selector')
|
||||||
|
).filter(visible);
|
||||||
|
for (const btn of buttons) {
|
||||||
|
if (btn.closest('.industry-top-rank-toolbar')) continue;
|
||||||
|
const lab = btn.querySelector('.jmt-label');
|
||||||
|
if (lab && (lab.innerText || '').trim() === label) {
|
||||||
|
const r = btn.getBoundingClientRect();
|
||||||
|
return {x: Math.round(r.x + r.width / 2),
|
||||||
|
y: Math.round(r.y + r.height / 2)};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}""")
|
return null;
|
||||||
page.wait_for_timeout(1500)
|
}""",
|
||||||
|
label,
|
||||||
select_last_week_in_old_datepicker(page, "peer_store_reselect")
|
)
|
||||||
page.wait_for_timeout(8000)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def switch_category(
|
def switch_category(
|
||||||
@@ -496,113 +664,174 @@ def switch_category(
|
|||||||
category_name: str,
|
category_name: str,
|
||||||
parent_category: str | None = None,
|
parent_category: str | None = None,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""切换商家榜单类目;只有选中态和榜单刷新均验证后才成功。"""
|
"""新版行业榜单:jmt-selector 层级下拉选类目 → 点「查询」→ 校验表格。
|
||||||
|
|
||||||
|
一级选项直接点击;二级选项先 hover 一级展开子面板再点击。下拉的
|
||||||
|
选项与筛选栏控件是同款 button.jmt-selector,靠排除工具栏区分。
|
||||||
|
"""
|
||||||
label = f"{parent_category} → {category_name}" if parent_category else category_name
|
label = f"{parent_category} → {category_name}" if parent_category else category_name
|
||||||
print(f" 切换类目到: {label}")
|
print(f" 切换类目到: {label}")
|
||||||
expected_path = _expected_category_path(category_name, parent_category)
|
# 类目下拉的触发行为极不稳定(单击/双击/坐标点击时开时不开),
|
||||||
selector = None
|
# 用多策略轮换:单击→轮询;双击→轮询;再单击→轮询,任何一刻
|
||||||
|
# 选项出现即继续。
|
||||||
|
trigger = page.locator(
|
||||||
|
".jmt--form-item-main", has_text="类目"
|
||||||
|
).first
|
||||||
|
|
||||||
|
def _dropdown_ready() -> bool:
|
||||||
|
return bool(
|
||||||
|
page.evaluate(
|
||||||
|
"() => Array.from(document.querySelectorAll('div.jmt-selector'))"
|
||||||
|
".some(el => el.getBoundingClientRect().width > 0"
|
||||||
|
" && !el.closest('.industry-top-rank-toolbar'))"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
opened = False
|
||||||
|
for action in ("click", "dblclick", "click"):
|
||||||
try:
|
try:
|
||||||
selector = _visible_category_selector(page)
|
if action == "click":
|
||||||
previous_fingerprint = _ranking_table_fingerprint(page)
|
trigger.click(timeout=6000)
|
||||||
current_path = _selected_category_path(selector)
|
else:
|
||||||
if current_path == expected_path:
|
trigger.dblclick(timeout=6000)
|
||||||
if not previous_fingerprint:
|
|
||||||
print(f" [WARN] 当前类目是 {label},但榜单表格为空")
|
|
||||||
return False
|
|
||||||
print(f" [OK] 当前已是 {label},榜单表格可用")
|
|
||||||
return True
|
|
||||||
|
|
||||||
menu = _visible_category_menu(page, selector)
|
|
||||||
_select_category_option(
|
|
||||||
page,
|
|
||||||
menu,
|
|
||||||
category_name,
|
|
||||||
parent_category,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not _wait_for_selected_category(selector, expected_path):
|
|
||||||
actual_path = _selected_category_path(selector)
|
|
||||||
print(
|
|
||||||
f" [WARN] 类目选中态校验失败: "
|
|
||||||
f"expected={expected_path}, actual={actual_path}"
|
|
||||||
)
|
|
||||||
return False
|
|
||||||
if not _wait_for_ranking_table_refresh(page, previous_fingerprint):
|
|
||||||
print(f" [WARN] 已选中 {label},但榜单表格未确认刷新")
|
|
||||||
return False
|
|
||||||
|
|
||||||
print(f" [OK] 已切换到 {label},榜单表格已刷新")
|
|
||||||
return True
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
print(f" [WARN] 切换类目失败: {exc}")
|
print(f" [WARN] 类目控件{action}失败: {exc}")
|
||||||
|
_dump_peer_filter_debug(page, "category_control_missing")
|
||||||
return False
|
return False
|
||||||
finally:
|
for _ in range(6):
|
||||||
|
page.wait_for_timeout(500)
|
||||||
|
if _dropdown_ready():
|
||||||
|
opened = True
|
||||||
|
break
|
||||||
|
if opened:
|
||||||
|
break
|
||||||
|
if not opened:
|
||||||
|
print(" [WARN] 类目下拉未能打开")
|
||||||
|
_dump_peer_filter_debug(page, "category_dropdown_not_open")
|
||||||
|
return False
|
||||||
|
|
||||||
|
selected = False
|
||||||
|
if parent_category and parent_category != category_name:
|
||||||
|
parent_center = _selector_option_center(page, parent_category)
|
||||||
|
if parent_center:
|
||||||
|
page.mouse.move(parent_center["x"], parent_center["y"])
|
||||||
|
page.wait_for_timeout(1500)
|
||||||
|
leaf_center = _selector_option_center(page, category_name)
|
||||||
|
if leaf_center:
|
||||||
|
page.mouse.click(leaf_center["x"], leaf_center["y"])
|
||||||
|
selected = True
|
||||||
|
else:
|
||||||
|
option_center = _selector_option_center(page, category_name)
|
||||||
|
if option_center:
|
||||||
|
page.mouse.click(option_center["x"], option_center["y"])
|
||||||
|
selected = True
|
||||||
|
if not selected:
|
||||||
|
print(f" [WARN] 未找到类目选项 {label}")
|
||||||
|
_dump_peer_filter_debug(page, f"category_option_missing_{category_name}")
|
||||||
try:
|
try:
|
||||||
menus = page.locator(
|
|
||||||
".jmtd-dropdown-panel.jmtd-dropdown-lists:visible"
|
|
||||||
)
|
|
||||||
if menus.count() and selector is not None:
|
|
||||||
selector.locator(
|
|
||||||
".jmtd-base-input-top:visible"
|
|
||||||
).click(timeout=3000)
|
|
||||||
page.wait_for_timeout(300)
|
|
||||||
if menus.count():
|
|
||||||
page.keyboard.press("Escape")
|
page.keyboard.press("Escape")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
return False
|
||||||
|
page.wait_for_timeout(1500)
|
||||||
|
|
||||||
|
# 应用筛选并等待榜单刷新
|
||||||
|
try:
|
||||||
|
query_box = page.locator("button", has_text="查询").first.bounding_box(
|
||||||
|
timeout=4000
|
||||||
|
)
|
||||||
|
if query_box:
|
||||||
|
page.mouse.click(
|
||||||
|
query_box["x"] + query_box["width"] / 2,
|
||||||
|
query_box["y"] + query_box["height"] / 2,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass # 部分面板选择后立即生效
|
||||||
|
page.wait_for_timeout(4500)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 注意不能用共享 extract():它会把结果字符串化导致布尔失真
|
||||||
|
table_present = bool(
|
||||||
|
page.evaluate(
|
||||||
|
"() => Array.from(document.querySelectorAll('table'))"
|
||||||
|
".some(t => (t.innerText || '').includes('排名'))"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
table_present = False
|
||||||
|
if not table_present:
|
||||||
|
print(" [WARN] 类目切换后未见榜单表格,禁止复用旧数据")
|
||||||
|
_dump_peer_filter_debug(page, f"table_missing_{category_name}")
|
||||||
|
return False
|
||||||
|
print(f" [OK] 已切换到 {label},榜单表格已刷新")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def extract_ranking_table(page):
|
def extract_ranking_table(page):
|
||||||
"""提取商家榜单表格数据"""
|
"""提取店铺榜表格;按表头行映射列,字段名沿用旧口径。
|
||||||
ranking_data = extract(page, """() => {
|
|
||||||
const rows = document.querySelectorAll('.jmtd-table-main tbody tr');
|
新版列:排名/店铺信息/成交金额/成交单量/成交商品件数/访客数/浏览量/
|
||||||
|
搜索点击次数;旧字段 followers(关注人数)新版已无此列,占位 '--'。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# 直接 evaluate 取结构化列表(共享 extract() 会把结果字符串化)
|
||||||
|
ranking_data = page.evaluate(
|
||||||
|
"""() => {
|
||||||
|
const table = Array.from(document.querySelectorAll('table'))
|
||||||
|
.find(t => (t.innerText || '').includes('排名'));
|
||||||
|
if (!table) return [];
|
||||||
|
const trs = Array.from(table.querySelectorAll('tr'));
|
||||||
|
let headers = null;
|
||||||
const result = [];
|
const result = [];
|
||||||
for (const row of rows) {
|
for (const tr of trs) {
|
||||||
const cells = row.querySelectorAll('td');
|
const cells = Array.from(tr.querySelectorAll('th,td'))
|
||||||
if (cells.length < 7) continue;
|
.map(c => (c.innerText || '').trim());
|
||||||
|
if (!cells.length) continue;
|
||||||
// 排名提取:优先数字文本,其次检查 1/2/3 名图标 class
|
if (headers === null) {
|
||||||
let rank = '';
|
if (cells.includes('排名')) headers = cells;
|
||||||
const rankNumEl = cells[0].querySelector('.jmtd-table-formatter-rank-number');
|
continue;
|
||||||
if (rankNumEl) {
|
|
||||||
rank = rankNumEl.textContent.trim();
|
|
||||||
}
|
}
|
||||||
if (!rank) {
|
if (cells.includes('排名')) continue;
|
||||||
const rankIconEl = cells[0].querySelector('.jmtd-table-formatter-rank');
|
const item = {};
|
||||||
if (rankIconEl) {
|
cells.forEach((value, index) => {
|
||||||
const iconMatch = rankIconEl.className.match(/jmtd-table-formatter-rank-icon-(\\d)/);
|
const header = headers[index] || `col${index}`;
|
||||||
if (iconMatch) {
|
item[header] = value;
|
||||||
rank = iconMatch[1];
|
});
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!rank) {
|
|
||||||
rank = cells[0].textContent.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
const shopLink = cells[1].querySelector('a');
|
|
||||||
const shop = shopLink ? shopLink.textContent.trim() : cells[1].textContent.trim();
|
|
||||||
|
|
||||||
const item = {
|
|
||||||
rank: rank,
|
|
||||||
shop: shop,
|
|
||||||
amount: cells[2]?.textContent?.trim() || '',
|
|
||||||
orders: cells[3]?.textContent?.trim() || '',
|
|
||||||
followers: cells[4]?.textContent?.trim() || '',
|
|
||||||
visitors: cells[5]?.textContent?.trim() || '',
|
|
||||||
searchClicks: cells[6]?.textContent?.trim() || '',
|
|
||||||
};
|
|
||||||
result.push(item);
|
result.push(item);
|
||||||
}
|
}
|
||||||
return JSON.stringify(result);
|
return result;
|
||||||
}""", "[]")
|
}"""
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
ranking_data = []
|
||||||
|
if not isinstance(ranking_data, list):
|
||||||
|
ranking_data = []
|
||||||
|
|
||||||
rows = []
|
rows = []
|
||||||
if isinstance(ranking_data, str):
|
header_alias = {
|
||||||
try:
|
"排名": "rank",
|
||||||
rows = json.loads(ranking_data)
|
"店铺信息": "shop",
|
||||||
except json.JSONDecodeError:
|
"成交金额": "amount",
|
||||||
rows = []
|
"成交单量": "orders",
|
||||||
|
"成交商品件数": "items",
|
||||||
|
"访客数": "visitors",
|
||||||
|
"浏览量": "views",
|
||||||
|
"搜索点击次数": "searchClicks",
|
||||||
|
}
|
||||||
|
for raw in ranking_data if isinstance(ranking_data, list) else []:
|
||||||
|
shop_text = str(raw.get("店铺信息", "")).splitlines()
|
||||||
|
item = {
|
||||||
|
"rank": str(raw.get("排名", "")).strip(),
|
||||||
|
"shop": shop_text[0].strip() if shop_text else "",
|
||||||
|
"amount": str(raw.get("成交金额", "")).strip(),
|
||||||
|
"orders": str(raw.get("成交单量", "")).strip(),
|
||||||
|
"followers": "--",
|
||||||
|
"visitors": str(raw.get("访客数", "")).strip(),
|
||||||
|
"searchClicks": str(raw.get("搜索点击次数", "")).strip(),
|
||||||
|
"items": str(raw.get("成交商品件数", "")).strip(),
|
||||||
|
"views": str(raw.get("浏览量", "")).strip(),
|
||||||
|
}
|
||||||
|
rows.append(item)
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@@ -1086,9 +1315,13 @@ def step_return_old_version(page: Page):
|
|||||||
|
|
||||||
|
|
||||||
def step_goto_vender_ranks(page: Page):
|
def step_goto_vender_ranks(page: Page):
|
||||||
print("[Step 4] 跳转到新版商家榜单...")
|
print("[Step 4] 跳转到新版「行业榜单-店铺榜」...")
|
||||||
page.goto(VENDER_RANKS_URL, wait_until="domcontentloaded", timeout=60000)
|
page.goto(INDUSTRY_TOP_URL, wait_until="domcontentloaded", timeout=60000)
|
||||||
page.wait_for_timeout(8000)
|
page.wait_for_timeout(8000)
|
||||||
|
current_url = page.url or ""
|
||||||
|
if "error.html" in current_url or "industry-top" not in current_url:
|
||||||
|
_dump_peer_filter_debug(page, "industry_top_bad_landing")
|
||||||
|
raise RuntimeError(f"行业榜单页跳转异常(京东侧可能再次改版): {current_url}")
|
||||||
body_text = page.locator("body").inner_text(timeout=10000)
|
body_text = page.locator("body").inner_text(timeout=10000)
|
||||||
permission_markers = (
|
permission_markers = (
|
||||||
"当前身份下暂无此菜单",
|
"当前身份下暂无此菜单",
|
||||||
@@ -1098,140 +1331,35 @@ def step_goto_vender_ranks(page: Page):
|
|||||||
)
|
)
|
||||||
if any(marker in body_text for marker in permission_markers):
|
if any(marker in body_text for marker in permission_markers):
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"京东商智新版商家榜单当前账号无菜单或权限,"
|
"京东商智新版行业榜单当前账号无菜单或权限,"
|
||||||
f"请确认该账号已开通榜单模块: {page.url}"
|
f"请确认该账号已开通榜单模块: {current_url}"
|
||||||
)
|
)
|
||||||
print(f" 已进入新版商家榜单: {page.url}")
|
if "店铺榜" not in body_text:
|
||||||
|
_dump_peer_filter_debug(page, "shop_rank_tab_missing")
|
||||||
|
raise RuntimeError(f"行业榜单页缺少「店铺榜」入口: {current_url}")
|
||||||
|
page.locator("text=店铺榜").first.click(timeout=10000)
|
||||||
|
page.wait_for_timeout(5000)
|
||||||
|
print(f" 已进入店铺榜: {page.url}")
|
||||||
|
|
||||||
|
|
||||||
# ── 搜索店铺 ──────────────────────────────────────────────────────────────────
|
# ── 搜索店铺 ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
def _locate_peer_search_input(page: Page):
|
|
||||||
"""Locate the shop search box inside the 交易榜单 section.
|
|
||||||
|
|
||||||
The vender-ranks page also has a global search box at the very top; generic
|
|
||||||
selectors (and the ``input`` first-match fallback) can hit that one and then
|
|
||||||
search nothing. Prefer the input that lives in the same container hierarchy
|
|
||||||
as the visible 交易榜单 heading.
|
|
||||||
"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
temp_id = page.evaluate(
|
|
||||||
r"""() => {
|
|
||||||
const visible = (el) => {
|
|
||||||
if (!el) return false;
|
|
||||||
const style = window.getComputedStyle(el);
|
|
||||||
const rect = el.getBoundingClientRect();
|
|
||||||
return style.display !== 'none' && style.visibility !== 'hidden'
|
|
||||||
&& rect.width > 0 && rect.height > 0;
|
|
||||||
};
|
|
||||||
const isSearchish = (input) => {
|
|
||||||
const type = (input.type || '').toLowerCase();
|
|
||||||
if (['hidden', 'checkbox', 'radio', 'submit', 'button'].includes(type)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// 全局 AI 搜索框(找功能/找指标/找商品)不是店铺搜索框,永远不要选中
|
|
||||||
if (input.classList.contains('ai-search-input')) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// 交易榜单区块的店铺搜索框(无 placeholder,靠 class 精确命中)
|
|
||||||
if (input.classList.contains('jmtd-input-input')
|
|
||||||
|| input.classList.contains('jmt-input-input')) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
const placeholder = (input.placeholder || '').trim();
|
|
||||||
const parentText = input.closest('.jmtd-base-input')?.textContent || '';
|
|
||||||
return Boolean(placeholder) || /搜索|查询|店铺|商家/.test(parentText);
|
|
||||||
};
|
|
||||||
const headings = Array.from(document.querySelectorAll('*'))
|
|
||||||
.filter((el) => visible(el) && (el.textContent || '').trim() === '交易榜单');
|
|
||||||
for (const heading of headings) {
|
|
||||||
let current = heading.parentElement;
|
|
||||||
for (let depth = 0; current && depth < 15; depth += 1, current = current.parentElement) {
|
|
||||||
const inputs = Array.from(current.querySelectorAll('input'))
|
|
||||||
.filter(visible)
|
|
||||||
.filter(isSearchish);
|
|
||||||
if (inputs.length === 1) {
|
|
||||||
const tempId = '__peer_search_input_' + Date.now() + '_' + Math.random().toString(36).slice(2);
|
|
||||||
inputs[0].setAttribute('id', tempId);
|
|
||||||
return tempId;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}"""
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
return None
|
|
||||||
if not isinstance(temp_id, str) or not temp_id:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
candidate = page.locator(f"#{temp_id}").first
|
|
||||||
if candidate.count() == 1 and candidate.is_visible(timeout=2_000):
|
|
||||||
print(" [OK] 已定位交易榜单区块内的店铺搜索框")
|
|
||||||
return candidate
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def search_shop(page: Page, shop_name: str):
|
def search_shop(page: Page, shop_name: str):
|
||||||
"""在商家榜单页面搜索指定店铺"""
|
"""新版店铺榜:固定 placeholder 的店铺搜索框过滤榜单。"""
|
||||||
print(f" 搜索店铺: {shop_name}")
|
print(f" 搜索店铺: {shop_name}")
|
||||||
|
search_input = page.locator('input[placeholder="请输入店铺名称"]').first
|
||||||
# 优先定位“交易榜单”区块内的店铺搜索框,避免命中页面顶部全局搜索框
|
|
||||||
search_input = _locate_peer_search_input(page)
|
|
||||||
|
|
||||||
# 尝试多种搜索框选择器
|
|
||||||
if search_input is None:
|
|
||||||
for selector in [
|
|
||||||
'.jmtd-base-input input.jmtd-input-input',
|
|
||||||
'.jmtd-base-input input.jmt-input-input',
|
|
||||||
'input[placeholder*="搜索"]',
|
|
||||||
'input[placeholder*="店铺"]',
|
|
||||||
'input[placeholder*="商家"]',
|
|
||||||
'input[type="search"]',
|
|
||||||
'.search-input input',
|
|
||||||
'.jmtd-search input',
|
|
||||||
]:
|
|
||||||
try:
|
try:
|
||||||
el = page.locator(selector).first
|
search_input.wait_for(state="visible", timeout=10000)
|
||||||
if el.is_visible(timeout=2000):
|
except Exception as exc:
|
||||||
# 排除全局 AI 搜索框,避免把品牌名输到"找功能/找指标/找商品"
|
_dump_peer_filter_debug(page, "shop_search_input_missing")
|
||||||
if el.evaluate("(input) => input.classList.contains('ai-search-input')"):
|
raise RuntimeError(f"未找到店铺搜索框: {exc}") from exc
|
||||||
continue
|
|
||||||
search_input = el
|
|
||||||
break
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not search_input:
|
|
||||||
# 兜底:优先 jmtd-input-input,其次任意 input(仍排除 AI 搜索框)
|
|
||||||
try:
|
|
||||||
fallback = page.locator('input.jmtd-input-input, input.jmt-input-input').first
|
|
||||||
if fallback.is_visible(timeout=2000):
|
|
||||||
search_input = fallback
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
if not search_input:
|
|
||||||
search_input = page.locator('input').first
|
|
||||||
print(" [WARN] 未找到明确的搜索框,使用第一个 input")
|
|
||||||
|
|
||||||
# 清空并输入店铺名
|
|
||||||
search_input.click()
|
search_input.click()
|
||||||
search_input.fill("")
|
search_input.fill("")
|
||||||
page.wait_for_timeout(500)
|
page.wait_for_timeout(300)
|
||||||
search_input.fill(shop_name)
|
search_input.fill(shop_name)
|
||||||
page.wait_for_timeout(1000)
|
page.wait_for_timeout(800)
|
||||||
|
|
||||||
# 尝试点击搜索按钮或按回车
|
|
||||||
try:
|
|
||||||
search_btn = page.locator('button:has-text("搜索"), button:has-text("查询"), .search-btn, .jmtd-search-btn').first
|
|
||||||
search_btn.click(timeout=3000)
|
|
||||||
except Exception:
|
|
||||||
search_input.press("Enter")
|
search_input.press("Enter")
|
||||||
|
page.wait_for_timeout(4000)
|
||||||
page.wait_for_timeout(8000)
|
|
||||||
print(f" [OK] 搜索完成: {shop_name}")
|
print(f" [OK] 搜索完成: {shop_name}")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user