feat(product): 京东新版商智竞店采集改造
入口跳转、两段式日期选择、搜索定位与表格解析适配新版商智；配合类目下拉与 XHR 拦截兜底。
This commit is contained in:
wangyunlong committed
1 parent
b0e1b5066d
commit
df11e195b7
1 file changed
+422
-294
+422
-294
@@ -32,10 +32,9 @@ from gyxx_flow.modules.shop_intelligence.browser_session import (
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.collectors.jd_data_collector import (
|
||||
close_profit_plan_popup_if_present,
|
||||
select_last_week_in_old_datepicker,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.config import JD_PEER_STORE_OUTPUT
|
||||
from gyxx_flow.modules.shop_intelligence.paths import resolve_raw_output
|
||||
from gyxx_flow.modules.shop_intelligence.paths import TMP_ROOT, resolve_raw_output
|
||||
|
||||
# ── 配置 ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -67,7 +66,11 @@ CATEGORY_OUTPUT_NAMES = {
|
||||
"男包": "男士包袋",
|
||||
}
|
||||
|
||||
VENDER_RANKS_URL = "https://jdsz.jd.com/szweb/view/industryMarket/venderRanks.html"
|
||||
# 2026-10 京东下线旧版商智：venderRanks 旧入口 404，官方公告要求迁移新版
|
||||
# jdsz.jd.com。新版等价页面是「行业榜单」的店铺榜 tab。
|
||||
INDUSTRY_TOP_URL = "https://jdsz.jd.com/szweb/view/industry/industry-top.html"
|
||||
# 兼容旧引用
|
||||
VENDER_RANKS_URL = INDUSTRY_TOP_URL
|
||||
|
||||
# 新品采集配置
|
||||
NEW_PRODUCT_SHOPS = [
|
||||
@@ -398,97 +401,262 @@ def get_last_week_range():
|
||||
return last_monday, last_sunday
|
||||
|
||||
|
||||
def _dump_peer_filter_debug(page: Page, tag: str) -> None:
|
||||
"""日期/类目操作失败时保存现场，便于按页面实际 DOM 校准选择器。"""
|
||||
try:
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
debug_dir = TMP_ROOT / "debug" / "jd_peer_filter"
|
||||
debug_dir.mkdir(parents=True, exist_ok=True)
|
||||
page.screenshot(path=str(debug_dir / f"{stamp}_{tag}.png"))
|
||||
html_path = debug_dir / f"{stamp}_{tag}.html"
|
||||
html_path.write_text(page.content(), encoding="utf-8")
|
||||
print(f" [DEBUG] 现场已保存: {html_path}")
|
||||
except Exception as exc:
|
||||
print(f" [DEBUG] 现场保存失败: {exc}")
|
||||
|
||||
|
||||
def _click_custom_day_cell(page: Page, date_str: str) -> None:
|
||||
"""在自定义日期面板中点击指定日期格（按月份块消歧，真实坐标点击）。
|
||||
|
||||
面板为单月迷你日历：日格只带日号文本，须先定位「只含一个月份标题且
|
||||
含日历格」的月份块，再在其中匹配日号；组件对合成点击不稳定，统一用
|
||||
bounding_box 中心坐标做真实鼠标点击。
|
||||
"""
|
||||
center = page.evaluate(
|
||||
"""(dateStr) => {
|
||||
const [year, month, day] = dateStr.split('-');
|
||||
const wantTitle = `${Number(year)}年${String(Number(month)).padStart(2, '0')}月`;
|
||||
const dayText = String(Number(day));
|
||||
const isMonthTitle = (text) => /^\\d{4}年\\d{2}月$/.test((text || '').trim());
|
||||
const titleSpans = Array.from(document.querySelectorAll('span'))
|
||||
.filter(el => el.getClientRects().length && isMonthTitle(el.innerText));
|
||||
for (const span of titleSpans) {
|
||||
if ((span.innerText || '').trim() !== wantTitle) continue;
|
||||
let node = span.parentElement;
|
||||
for (let depth = 0; node && depth < 10; depth += 1, node = node.parentElement) {
|
||||
const titles = Array.from(node.querySelectorAll('span'))
|
||||
.filter(s => isMonthTitle(s.innerText));
|
||||
if (titles.length > 1) break;
|
||||
if (titles.length === 1
|
||||
&& node.querySelector('td.jmt-date-picker-calendar-cell')) {
|
||||
const cells = Array.from(
|
||||
node.querySelectorAll('td.jmt-date-picker-calendar-cell')
|
||||
).filter(el => el.getBoundingClientRect().width > 0
|
||||
&& (el.innerText || '').trim() === dayText);
|
||||
if (cells.length) {
|
||||
const rect = cells[0].getBoundingClientRect();
|
||||
return {x: Math.round(rect.x + rect.width / 2),
|
||||
y: Math.round(rect.y + rect.height / 2)};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}""",
|
||||
date_str,
|
||||
)
|
||||
if not center:
|
||||
_dump_peer_filter_debug(page, f"day_not_found_{date_str}")
|
||||
raise RuntimeError(f"自定义日期面板未找到日期格: {date_str}")
|
||||
page.mouse.click(center["x"], center["y"])
|
||||
|
||||
|
||||
def _open_custom_date_panel(page: Page, *, retries: int = 3) -> bool:
|
||||
"""点击「自定义」标签打开日历；带开面板验证与重试。
|
||||
|
||||
面板行为实测有两种形态且跨运行不定：迷你单月（选起点即自动关闭）
|
||||
与常驻双月（选完不关）。tag 点击在面板刚自动关闭后可能需要缓一下
|
||||
再点，故带月标题可见性验证与重试。
|
||||
"""
|
||||
tag = page.locator(
|
||||
"li.jmt-combo-date-picker-list-tag", has_text="自定义"
|
||||
).first
|
||||
for _attempt in range(retries):
|
||||
try:
|
||||
box = tag.bounding_box(timeout=5000)
|
||||
except Exception:
|
||||
box = None
|
||||
if not box:
|
||||
_dump_peer_filter_debug(page, "custom_tag_missing")
|
||||
raise RuntimeError("未找到「自定义」时间按钮")
|
||||
page.mouse.click(
|
||||
box["x"] + box["width"] / 2, box["y"] + box["height"] / 2
|
||||
)
|
||||
page.wait_for_timeout(2000)
|
||||
if _panel_months_visible(page):
|
||||
return True
|
||||
page.wait_for_timeout(1000)
|
||||
return _panel_months_visible(page)
|
||||
|
||||
|
||||
def _panel_months_visible(page) -> bool:
|
||||
try:
|
||||
return bool(
|
||||
page.evaluate(
|
||||
"() => Array.from(document.querySelectorAll('span'))"
|
||||
".some(el => el.getClientRects().length"
|
||||
" && /^\\d{4}年\\d{2}月$/.test((el.innerText || '').trim()))"
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _ensure_month_visible(page: Page, month_title: str) -> None:
|
||||
"""单月日历翻月：目标月份不可见时点「后一月」箭头（至多 12 次）。"""
|
||||
|
||||
def visible_months():
|
||||
return page.evaluate(
|
||||
"""() => Array.from(document.querySelectorAll('span'))
|
||||
.filter(el => el.getClientRects().length)
|
||||
.map(el => (el.innerText || '').trim())
|
||||
.filter(t => /^\\d{4}年\\d{2}月$/.test(t))"""
|
||||
)
|
||||
|
||||
for _ in range(12):
|
||||
if month_title in visible_months():
|
||||
return
|
||||
try:
|
||||
arrow = page.locator('[data-event-content="后一月"]')
|
||||
if arrow.count() == 0:
|
||||
return
|
||||
box = arrow.first.bounding_box(timeout=3000)
|
||||
except Exception:
|
||||
return
|
||||
if not box:
|
||||
return
|
||||
page.mouse.click(
|
||||
box["x"] + box["width"] / 2, box["y"] + box["height"] / 2
|
||||
)
|
||||
page.wait_for_timeout(1200)
|
||||
|
||||
|
||||
def _custom_panel_state(page) -> str:
|
||||
"""调试用：当前日期面板的可见月份与「当前：」显示。"""
|
||||
try:
|
||||
return str(
|
||||
page.evaluate(
|
||||
"""() => {
|
||||
const months = Array.from(document.querySelectorAll('span'))
|
||||
.filter(el => el.getClientRects().length)
|
||||
.map(el => (el.innerText || '').trim())
|
||||
.filter(t => /^\\d{4}年\\d{2}月$/.test(t));
|
||||
const current = (document.body.innerText.match(/当前：[^\\n]{0,40}/) || [''])[0];
|
||||
return {months: [...new Set(months)], current};
|
||||
}"""
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
return f"<state-error {exc}>"
|
||||
|
||||
|
||||
def _custom_range_applied(page, start_mmdd: str) -> bool:
|
||||
try:
|
||||
body_text = page.locator("body").inner_text(timeout=3000)
|
||||
except Exception:
|
||||
return False
|
||||
return start_mmdd in body_text.replace(" ", "")
|
||||
|
||||
|
||||
def select_last_week_any_day(page):
|
||||
"""点击日期选择器 → 按周查询 → 点上周任意一天（自动选中整周）"""
|
||||
"""新版行业榜单：自定义区间选择上周一~上周日（状态机，3 轮重试）。
|
||||
|
||||
面板有两种形态且跨运行不定：
|
||||
- 迷你单月：点击起点后自动关闭；重开面板点终点，选完立即生效。
|
||||
- 常驻双月：起止选完面板不关；点面板外部（店铺搜索框）应用；
|
||||
Escape 会取消选择。
|
||||
统一状态机：选起点 → 面板仍开则同会话选终点，已关则重开选终点 →
|
||||
未生效则点外部应用 → 仍未生效整轮重来。
|
||||
"""
|
||||
print(" 选择上周时间范围...")
|
||||
last_monday, last_sunday = get_last_week_range()
|
||||
start_str = last_monday.strftime("%Y-%m-%d")
|
||||
end_str = last_sunday.strftime("%Y-%m-%d")
|
||||
start_month = f"{last_monday.year}年{last_monday.month:02d}月"
|
||||
end_month = f"{last_sunday.year}年{last_sunday.month:02d}月"
|
||||
start_mmdd = last_monday.strftime("%m-%d")
|
||||
|
||||
# 1. 点击日期选择器（显示"按天查询..."的元素）
|
||||
for attempt in range(1, 4):
|
||||
print(f" [state] 第 {attempt} 轮: {_custom_panel_state(page)}")
|
||||
if not _open_custom_date_panel(page):
|
||||
print(" [state] 面板未能打开，重试")
|
||||
continue
|
||||
_ensure_month_visible(page, start_month)
|
||||
try:
|
||||
date_picker = page.locator('.jmtd-date-picker .jmtd-base-input-top').first
|
||||
date_picker.wait_for(state="visible", timeout=5000)
|
||||
date_picker.click()
|
||||
print(" [OK] 点击日期选择器")
|
||||
except Exception as e:
|
||||
print(f" [WARN] 点击日期选择器失败: {e}")
|
||||
# 兜底：点击包含"按天查询"文本的元素
|
||||
_click_custom_day_cell(page, start_str)
|
||||
except RuntimeError:
|
||||
continue
|
||||
page.wait_for_timeout(1500)
|
||||
print(f" [state] 点击开始后: {_custom_panel_state(page)}")
|
||||
|
||||
if not _panel_months_visible(page):
|
||||
# 迷你面板形态：起点选择后面板已自动关闭，重开选终点
|
||||
page.wait_for_timeout(1200)
|
||||
if not _open_custom_date_panel(page):
|
||||
continue
|
||||
_ensure_month_visible(page, end_month)
|
||||
try:
|
||||
page.locator('text=按天查询').first.click()
|
||||
print(" [OK] 使用文本定位点击日期选择器")
|
||||
_click_custom_day_cell(page, end_str)
|
||||
except RuntimeError:
|
||||
continue
|
||||
page.wait_for_timeout(2500)
|
||||
print(f" [state] 点击结束后: {_custom_panel_state(page)}")
|
||||
|
||||
if _custom_range_applied(page, start_mmdd):
|
||||
print(f" [OK] 已选择上周区间: {start_str} ~ {end_str}")
|
||||
return
|
||||
# 常驻双月形态：点面板外部应用（Escape 是取消，绝不能用）
|
||||
try:
|
||||
page.locator(
|
||||
'input[placeholder="请输入店铺名称"]'
|
||||
).first.click(timeout=4000)
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
# 2. 点击"按周查询"
|
||||
try:
|
||||
week_switch = page.locator('.jmtd-date-picker-switch-panel-item', has_text="按周查询").first
|
||||
week_switch.wait_for(state="visible", timeout=5000)
|
||||
week_switch.click()
|
||||
print(" [OK] 点击按周查询")
|
||||
except Exception as e:
|
||||
print(f" [WARN] 点击按周查询失败: {e}")
|
||||
# 兜底：用 JS 点击
|
||||
page.evaluate("""() => {
|
||||
const items = document.querySelectorAll('.jmtd-date-picker-switch-panel-item');
|
||||
for (const item of items) {
|
||||
if (item.textContent.includes('按周查询')) {
|
||||
item.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}""")
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
# 3. 等待日历弹窗并选择上周某一天
|
||||
calendar = page.locator('.jmtd-date-picker-popup, .jmtd-date-picker-panel, [class*="picker-panel"], [class*="calendar-panel"]')
|
||||
try:
|
||||
calendar.wait_for(state="visible", timeout=5000)
|
||||
except Exception:
|
||||
print(" [WARN] 未检测到日历弹窗,尝试继续...")
|
||||
page.mouse.click(30, 400)
|
||||
page.wait_for_timeout(3000)
|
||||
if _custom_range_applied(page, start_mmdd):
|
||||
print(f" [OK] 已选择上周区间(点外部应用): {start_str} ~ {end_str}")
|
||||
return
|
||||
print(f" [state] 本轮未生效: {_custom_panel_state(page)}")
|
||||
|
||||
select_last_week_in_old_datepicker(page, "peer_store")
|
||||
|
||||
try:
|
||||
calendar.wait_for(state="hidden", timeout=5000)
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(5000)
|
||||
print(" 已切换到上周数据")
|
||||
_dump_peer_filter_debug(page, "range_not_applied")
|
||||
raise RuntimeError(
|
||||
f"自定义日期区间未生效: 期望 {start_str} ~ {end_str}"
|
||||
)
|
||||
|
||||
|
||||
def reselect_last_week(page):
|
||||
"""切换类目后重新选择上周日期（点击日期选择器 → 按周查询 → 点日期）"""
|
||||
# 重新点击日期选择器
|
||||
try:
|
||||
date_picker = page.locator('.jmtd-date-picker .jmtd-base-input-top').first
|
||||
date_picker.click()
|
||||
except Exception:
|
||||
page.locator('text=按天查询').first.click()
|
||||
page.wait_for_timeout(1500)
|
||||
|
||||
# 点击按周查询
|
||||
try:
|
||||
week_switch = page.locator('.jmtd-date-picker-switch-panel-item', has_text="按周查询").first
|
||||
week_switch.click()
|
||||
except Exception:
|
||||
page.evaluate("""() => {
|
||||
const items = document.querySelectorAll('.jmtd-date-picker-switch-panel-item');
|
||||
for (const item of items) {
|
||||
if (item.textContent.includes('按周查询')) {
|
||||
item.click();
|
||||
return true;
|
||||
|
||||
|
||||
|
||||
def _selector_option_center(page: Page, label: str, *, exclude_toolbar: bool = True):
|
||||
"""在下拉浮层里按精确文本定位 jmt-selector 选项，返回真实坐标。
|
||||
|
||||
筛选栏控件本身也是 button.jmt-selector（显示当前选中值），必须
|
||||
排除，否则会把控件当选项点到。
|
||||
"""
|
||||
return page.evaluate(
|
||||
"""(label) => {
|
||||
const visible = (el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
return r.width > 0 && r.height > 0;
|
||||
};
|
||||
const buttons = Array.from(
|
||||
document.querySelectorAll('div.jmt-selector')
|
||||
).filter(visible);
|
||||
for (const btn of buttons) {
|
||||
if (btn.closest('.industry-top-rank-toolbar')) continue;
|
||||
const lab = btn.querySelector('.jmt-label');
|
||||
if (lab && (lab.innerText || '').trim() === label) {
|
||||
const r = btn.getBoundingClientRect();
|
||||
return {x: Math.round(r.x + r.width / 2),
|
||||
y: Math.round(r.y + r.height / 2)};
|
||||
}
|
||||
}
|
||||
}""")
|
||||
page.wait_for_timeout(1500)
|
||||
|
||||
select_last_week_in_old_datepicker(page, "peer_store_reselect")
|
||||
page.wait_for_timeout(8000)
|
||||
|
||||
|
||||
|
||||
|
||||
return null;
|
||||
}""",
|
||||
label,
|
||||
)
|
||||
|
||||
|
||||
def switch_category(
|
||||
@@ -496,113 +664,174 @@ def switch_category(
|
||||
category_name: str,
|
||||
parent_category: str | None = None,
|
||||
) -> bool:
|
||||
"""切换商家榜单类目；只有选中态和榜单刷新均验证后才成功。"""
|
||||
"""新版行业榜单：jmt-selector 层级下拉选类目 → 点「查询」→ 校验表格。
|
||||
|
||||
一级选项直接点击；二级选项先 hover 一级展开子面板再点击。下拉的
|
||||
选项与筛选栏控件是同款 button.jmt-selector，靠排除工具栏区分。
|
||||
"""
|
||||
label = f"{parent_category} → {category_name}" if parent_category else category_name
|
||||
print(f" 切换类目到: {label}")
|
||||
expected_path = _expected_category_path(category_name, parent_category)
|
||||
selector = None
|
||||
# 类目下拉的触发行为极不稳定（单击/双击/坐标点击时开时不开），
|
||||
# 用多策略轮换：单击→轮询；双击→轮询；再单击→轮询，任何一刻
|
||||
# 选项出现即继续。
|
||||
trigger = page.locator(
|
||||
".jmt--form-item-main", has_text="类目"
|
||||
).first
|
||||
|
||||
def _dropdown_ready() -> bool:
|
||||
return bool(
|
||||
page.evaluate(
|
||||
"() => Array.from(document.querySelectorAll('div.jmt-selector'))"
|
||||
".some(el => el.getBoundingClientRect().width > 0"
|
||||
" && !el.closest('.industry-top-rank-toolbar'))"
|
||||
)
|
||||
)
|
||||
|
||||
opened = False
|
||||
for action in ("click", "dblclick", "click"):
|
||||
try:
|
||||
selector = _visible_category_selector(page)
|
||||
previous_fingerprint = _ranking_table_fingerprint(page)
|
||||
current_path = _selected_category_path(selector)
|
||||
if current_path == expected_path:
|
||||
if not previous_fingerprint:
|
||||
print(f" [WARN] 当前类目是 {label}，但榜单表格为空")
|
||||
return False
|
||||
print(f" [OK] 当前已是 {label}，榜单表格可用")
|
||||
return True
|
||||
|
||||
menu = _visible_category_menu(page, selector)
|
||||
_select_category_option(
|
||||
page,
|
||||
menu,
|
||||
category_name,
|
||||
parent_category,
|
||||
)
|
||||
|
||||
if not _wait_for_selected_category(selector, expected_path):
|
||||
actual_path = _selected_category_path(selector)
|
||||
print(
|
||||
f" [WARN] 类目选中态校验失败: "
|
||||
f"expected={expected_path}, actual={actual_path}"
|
||||
)
|
||||
return False
|
||||
if not _wait_for_ranking_table_refresh(page, previous_fingerprint):
|
||||
print(f" [WARN] 已选中 {label}，但榜单表格未确认刷新")
|
||||
return False
|
||||
|
||||
print(f" [OK] 已切换到 {label}，榜单表格已刷新")
|
||||
return True
|
||||
if action == "click":
|
||||
trigger.click(timeout=6000)
|
||||
else:
|
||||
trigger.dblclick(timeout=6000)
|
||||
except Exception as exc:
|
||||
print(f" [WARN] 切换类目失败: {exc}")
|
||||
print(f" [WARN] 类目控件{action}失败: {exc}")
|
||||
_dump_peer_filter_debug(page, "category_control_missing")
|
||||
return False
|
||||
finally:
|
||||
for _ in range(6):
|
||||
page.wait_for_timeout(500)
|
||||
if _dropdown_ready():
|
||||
opened = True
|
||||
break
|
||||
if opened:
|
||||
break
|
||||
if not opened:
|
||||
print(" [WARN] 类目下拉未能打开")
|
||||
_dump_peer_filter_debug(page, "category_dropdown_not_open")
|
||||
return False
|
||||
|
||||
selected = False
|
||||
if parent_category and parent_category != category_name:
|
||||
parent_center = _selector_option_center(page, parent_category)
|
||||
if parent_center:
|
||||
page.mouse.move(parent_center["x"], parent_center["y"])
|
||||
page.wait_for_timeout(1500)
|
||||
leaf_center = _selector_option_center(page, category_name)
|
||||
if leaf_center:
|
||||
page.mouse.click(leaf_center["x"], leaf_center["y"])
|
||||
selected = True
|
||||
else:
|
||||
option_center = _selector_option_center(page, category_name)
|
||||
if option_center:
|
||||
page.mouse.click(option_center["x"], option_center["y"])
|
||||
selected = True
|
||||
if not selected:
|
||||
print(f" [WARN] 未找到类目选项 {label}")
|
||||
_dump_peer_filter_debug(page, f"category_option_missing_{category_name}")
|
||||
try:
|
||||
menus = page.locator(
|
||||
".jmtd-dropdown-panel.jmtd-dropdown-lists:visible"
|
||||
)
|
||||
if menus.count() and selector is not None:
|
||||
selector.locator(
|
||||
".jmtd-base-input-top:visible"
|
||||
).click(timeout=3000)
|
||||
page.wait_for_timeout(300)
|
||||
if menus.count():
|
||||
page.keyboard.press("Escape")
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
page.wait_for_timeout(1500)
|
||||
|
||||
# 应用筛选并等待榜单刷新
|
||||
try:
|
||||
query_box = page.locator("button", has_text="查询").first.bounding_box(
|
||||
timeout=4000
|
||||
)
|
||||
if query_box:
|
||||
page.mouse.click(
|
||||
query_box["x"] + query_box["width"] / 2,
|
||||
query_box["y"] + query_box["height"] / 2,
|
||||
)
|
||||
except Exception:
|
||||
pass # 部分面板选择后立即生效
|
||||
page.wait_for_timeout(4500)
|
||||
|
||||
try:
|
||||
# 注意不能用共享 extract()：它会把结果字符串化导致布尔失真
|
||||
table_present = bool(
|
||||
page.evaluate(
|
||||
"() => Array.from(document.querySelectorAll('table'))"
|
||||
".some(t => (t.innerText || '').includes('排名'))"
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
table_present = False
|
||||
if not table_present:
|
||||
print(" [WARN] 类目切换后未见榜单表格，禁止复用旧数据")
|
||||
_dump_peer_filter_debug(page, f"table_missing_{category_name}")
|
||||
return False
|
||||
print(f" [OK] 已切换到 {label}，榜单表格已刷新")
|
||||
return True
|
||||
|
||||
|
||||
def extract_ranking_table(page):
|
||||
"""提取商家榜单表格数据"""
|
||||
ranking_data = extract(page, """() => {
|
||||
const rows = document.querySelectorAll('.jmtd-table-main tbody tr');
|
||||
"""提取店铺榜表格；按表头行映射列，字段名沿用旧口径。
|
||||
|
||||
新版列：排名/店铺信息/成交金额/成交单量/成交商品件数/访客数/浏览量/
|
||||
搜索点击次数；旧字段 followers（关注人数）新版已无此列，占位 '--'。
|
||||
"""
|
||||
try:
|
||||
# 直接 evaluate 取结构化列表（共享 extract() 会把结果字符串化）
|
||||
ranking_data = page.evaluate(
|
||||
"""() => {
|
||||
const table = Array.from(document.querySelectorAll('table'))
|
||||
.find(t => (t.innerText || '').includes('排名'));
|
||||
if (!table) return [];
|
||||
const trs = Array.from(table.querySelectorAll('tr'));
|
||||
let headers = null;
|
||||
const result = [];
|
||||
for (const row of rows) {
|
||||
const cells = row.querySelectorAll('td');
|
||||
if (cells.length < 7) continue;
|
||||
|
||||
// 排名提取：优先数字文本，其次检查 1/2/3 名图标 class
|
||||
let rank = '';
|
||||
const rankNumEl = cells[0].querySelector('.jmtd-table-formatter-rank-number');
|
||||
if (rankNumEl) {
|
||||
rank = rankNumEl.textContent.trim();
|
||||
for (const tr of trs) {
|
||||
const cells = Array.from(tr.querySelectorAll('th,td'))
|
||||
.map(c => (c.innerText || '').trim());
|
||||
if (!cells.length) continue;
|
||||
if (headers === null) {
|
||||
if (cells.includes('排名')) headers = cells;
|
||||
continue;
|
||||
}
|
||||
if (!rank) {
|
||||
const rankIconEl = cells[0].querySelector('.jmtd-table-formatter-rank');
|
||||
if (rankIconEl) {
|
||||
const iconMatch = rankIconEl.className.match(/jmtd-table-formatter-rank-icon-(\\d)/);
|
||||
if (iconMatch) {
|
||||
rank = iconMatch[1];
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!rank) {
|
||||
rank = cells[0].textContent.trim();
|
||||
}
|
||||
|
||||
const shopLink = cells[1].querySelector('a');
|
||||
const shop = shopLink ? shopLink.textContent.trim() : cells[1].textContent.trim();
|
||||
|
||||
const item = {
|
||||
rank: rank,
|
||||
shop: shop,
|
||||
amount: cells[2]?.textContent?.trim() || '',
|
||||
orders: cells[3]?.textContent?.trim() || '',
|
||||
followers: cells[4]?.textContent?.trim() || '',
|
||||
visitors: cells[5]?.textContent?.trim() || '',
|
||||
searchClicks: cells[6]?.textContent?.trim() || '',
|
||||
};
|
||||
if (cells.includes('排名')) continue;
|
||||
const item = {};
|
||||
cells.forEach((value, index) => {
|
||||
const header = headers[index] || `col${index}`;
|
||||
item[header] = value;
|
||||
});
|
||||
result.push(item);
|
||||
}
|
||||
return JSON.stringify(result);
|
||||
}""", "[]")
|
||||
return result;
|
||||
}"""
|
||||
)
|
||||
except Exception:
|
||||
ranking_data = []
|
||||
if not isinstance(ranking_data, list):
|
||||
ranking_data = []
|
||||
|
||||
rows = []
|
||||
if isinstance(ranking_data, str):
|
||||
try:
|
||||
rows = json.loads(ranking_data)
|
||||
except json.JSONDecodeError:
|
||||
rows = []
|
||||
header_alias = {
|
||||
"排名": "rank",
|
||||
"店铺信息": "shop",
|
||||
"成交金额": "amount",
|
||||
"成交单量": "orders",
|
||||
"成交商品件数": "items",
|
||||
"访客数": "visitors",
|
||||
"浏览量": "views",
|
||||
"搜索点击次数": "searchClicks",
|
||||
}
|
||||
for raw in ranking_data if isinstance(ranking_data, list) else []:
|
||||
shop_text = str(raw.get("店铺信息", "")).splitlines()
|
||||
item = {
|
||||
"rank": str(raw.get("排名", "")).strip(),
|
||||
"shop": shop_text[0].strip() if shop_text else "",
|
||||
"amount": str(raw.get("成交金额", "")).strip(),
|
||||
"orders": str(raw.get("成交单量", "")).strip(),
|
||||
"followers": "--",
|
||||
"visitors": str(raw.get("访客数", "")).strip(),
|
||||
"searchClicks": str(raw.get("搜索点击次数", "")).strip(),
|
||||
"items": str(raw.get("成交商品件数", "")).strip(),
|
||||
"views": str(raw.get("浏览量", "")).strip(),
|
||||
}
|
||||
rows.append(item)
|
||||
return rows
|
||||
|
||||
|
||||
@@ -1086,9 +1315,13 @@ def step_return_old_version(page: Page):
|
||||
|
||||
|
||||
def step_goto_vender_ranks(page: Page):
|
||||
print("[Step 4] 跳转到新版商家榜单...")
|
||||
page.goto(VENDER_RANKS_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
print("[Step 4] 跳转到新版「行业榜单-店铺榜」...")
|
||||
page.goto(INDUSTRY_TOP_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_timeout(8000)
|
||||
current_url = page.url or ""
|
||||
if "error.html" in current_url or "industry-top" not in current_url:
|
||||
_dump_peer_filter_debug(page, "industry_top_bad_landing")
|
||||
raise RuntimeError(f"行业榜单页跳转异常（京东侧可能再次改版）: {current_url}")
|
||||
body_text = page.locator("body").inner_text(timeout=10000)
|
||||
permission_markers = (
|
||||
"当前身份下暂无此菜单",
|
||||
@@ -1098,140 +1331,35 @@ def step_goto_vender_ranks(page: Page):
|
||||
)
|
||||
if any(marker in body_text for marker in permission_markers):
|
||||
raise RuntimeError(
|
||||
"京东商智新版商家榜单当前账号无菜单或权限，"
|
||||
f"请确认该账号已开通榜单模块: {page.url}"
|
||||
"京东商智新版行业榜单当前账号无菜单或权限，"
|
||||
f"请确认该账号已开通榜单模块: {current_url}"
|
||||
)
|
||||
print(f" 已进入新版商家榜单: {page.url}")
|
||||
if "店铺榜" not in body_text:
|
||||
_dump_peer_filter_debug(page, "shop_rank_tab_missing")
|
||||
raise RuntimeError(f"行业榜单页缺少「店铺榜」入口: {current_url}")
|
||||
page.locator("text=店铺榜").first.click(timeout=10000)
|
||||
page.wait_for_timeout(5000)
|
||||
print(f" 已进入店铺榜: {page.url}")
|
||||
|
||||
|
||||
# ── 搜索店铺 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _locate_peer_search_input(page: Page):
|
||||
"""Locate the shop search box inside the 交易榜单 section.
|
||||
|
||||
The vender-ranks page also has a global search box at the very top; generic
|
||||
selectors (and the ``input`` first-match fallback) can hit that one and then
|
||||
search nothing. Prefer the input that lives in the same container hierarchy
|
||||
as the visible 交易榜单 heading.
|
||||
"""
|
||||
|
||||
try:
|
||||
temp_id = page.evaluate(
|
||||
r"""() => {
|
||||
const visible = (el) => {
|
||||
if (!el) return false;
|
||||
const style = window.getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
return style.display !== 'none' && style.visibility !== 'hidden'
|
||||
&& rect.width > 0 && rect.height > 0;
|
||||
};
|
||||
const isSearchish = (input) => {
|
||||
const type = (input.type || '').toLowerCase();
|
||||
if (['hidden', 'checkbox', 'radio', 'submit', 'button'].includes(type)) {
|
||||
return false;
|
||||
}
|
||||
// 全局 AI 搜索框(找功能/找指标/找商品)不是店铺搜索框,永远不要选中
|
||||
if (input.classList.contains('ai-search-input')) {
|
||||
return false;
|
||||
}
|
||||
// 交易榜单区块的店铺搜索框(无 placeholder,靠 class 精确命中)
|
||||
if (input.classList.contains('jmtd-input-input')
|
||||
|| input.classList.contains('jmt-input-input')) {
|
||||
return true;
|
||||
}
|
||||
const placeholder = (input.placeholder || '').trim();
|
||||
const parentText = input.closest('.jmtd-base-input')?.textContent || '';
|
||||
return Boolean(placeholder) || /搜索|查询|店铺|商家/.test(parentText);
|
||||
};
|
||||
const headings = Array.from(document.querySelectorAll('*'))
|
||||
.filter((el) => visible(el) && (el.textContent || '').trim() === '交易榜单');
|
||||
for (const heading of headings) {
|
||||
let current = heading.parentElement;
|
||||
for (let depth = 0; current && depth < 15; depth += 1, current = current.parentElement) {
|
||||
const inputs = Array.from(current.querySelectorAll('input'))
|
||||
.filter(visible)
|
||||
.filter(isSearchish);
|
||||
if (inputs.length === 1) {
|
||||
const tempId = '__peer_search_input_' + Date.now() + '_' + Math.random().toString(36).slice(2);
|
||||
inputs[0].setAttribute('id', tempId);
|
||||
return tempId;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}"""
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(temp_id, str) or not temp_id:
|
||||
return None
|
||||
try:
|
||||
candidate = page.locator(f"#{temp_id}").first
|
||||
if candidate.count() == 1 and candidate.is_visible(timeout=2_000):
|
||||
print(" [OK] 已定位交易榜单区块内的店铺搜索框")
|
||||
return candidate
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def search_shop(page: Page, shop_name: str):
|
||||
"""在商家榜单页面搜索指定店铺"""
|
||||
"""新版店铺榜：固定 placeholder 的店铺搜索框过滤榜单。"""
|
||||
print(f" 搜索店铺: {shop_name}")
|
||||
|
||||
# 优先定位“交易榜单”区块内的店铺搜索框,避免命中页面顶部全局搜索框
|
||||
search_input = _locate_peer_search_input(page)
|
||||
|
||||
# 尝试多种搜索框选择器
|
||||
if search_input is None:
|
||||
for selector in [
|
||||
'.jmtd-base-input input.jmtd-input-input',
|
||||
'.jmtd-base-input input.jmt-input-input',
|
||||
'input[placeholder*="搜索"]',
|
||||
'input[placeholder*="店铺"]',
|
||||
'input[placeholder*="商家"]',
|
||||
'input[type="search"]',
|
||||
'.search-input input',
|
||||
'.jmtd-search input',
|
||||
]:
|
||||
search_input = page.locator('input[placeholder="请输入店铺名称"]').first
|
||||
try:
|
||||
el = page.locator(selector).first
|
||||
if el.is_visible(timeout=2000):
|
||||
# 排除全局 AI 搜索框,避免把品牌名输到"找功能/找指标/找商品"
|
||||
if el.evaluate("(input) => input.classList.contains('ai-search-input')"):
|
||||
continue
|
||||
search_input = el
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not search_input:
|
||||
# 兜底：优先 jmtd-input-input,其次任意 input（仍排除 AI 搜索框）
|
||||
try:
|
||||
fallback = page.locator('input.jmtd-input-input, input.jmt-input-input').first
|
||||
if fallback.is_visible(timeout=2000):
|
||||
search_input = fallback
|
||||
except Exception:
|
||||
pass
|
||||
if not search_input:
|
||||
search_input = page.locator('input').first
|
||||
print(" [WARN] 未找到明确的搜索框,使用第一个 input")
|
||||
|
||||
# 清空并输入店铺名
|
||||
search_input.wait_for(state="visible", timeout=10000)
|
||||
except Exception as exc:
|
||||
_dump_peer_filter_debug(page, "shop_search_input_missing")
|
||||
raise RuntimeError(f"未找到店铺搜索框: {exc}") from exc
|
||||
search_input.click()
|
||||
search_input.fill("")
|
||||
page.wait_for_timeout(500)
|
||||
page.wait_for_timeout(300)
|
||||
search_input.fill(shop_name)
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
# 尝试点击搜索按钮或按回车
|
||||
try:
|
||||
search_btn = page.locator('button:has-text("搜索"), button:has-text("查询"), .search-btn, .jmtd-search-btn').first
|
||||
search_btn.click(timeout=3000)
|
||||
except Exception:
|
||||
page.wait_for_timeout(800)
|
||||
search_input.press("Enter")
|
||||
|
||||
page.wait_for_timeout(8000)
|
||||
page.wait_for_timeout(4000)
|
||||
print(f" [OK] 搜索完成: {shop_name}")
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user