feat: consolidate legacy workflows into gyxx-flow
This commit is contained in:
wangyunlong committed
commit
c23b62a8c8
374 files changed
+132990
No files matched your search
@@ -0,0 +1,77 @@
|
||||
"""Business-module contracts, registry, and default composition root."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .content_marketing import ContentMarketingModule
|
||||
from .contracts import BusinessModule
|
||||
from .product_commerce import ProductCommerceModule
|
||||
from .registry import ModuleRegistry
|
||||
from .shop_intelligence import ShopIntelligenceModule
|
||||
from .supply_chain import SupplyChainModule
|
||||
|
||||
EXPECTED_MODULE_IDS = (
|
||||
"content_marketing",
|
||||
"product_commerce",
|
||||
"shop_intelligence",
|
||||
"supply_chain",
|
||||
)
|
||||
|
||||
|
||||
def create_default_registry(
|
||||
*,
|
||||
catalog: Any | None = None,
|
||||
shop_command_factory: Any | None = None,
|
||||
supply_command_factory: Any | None = None,
|
||||
content_command_factory: Any | None = None,
|
||||
product_command_factory: Any | None = None,
|
||||
) -> ModuleRegistry:
|
||||
"""Compose built-in modules, registering only explicitly supplied migrations."""
|
||||
|
||||
shop_module = (
|
||||
ShopIntelligenceModule()
|
||||
if catalog is None
|
||||
else ShopIntelligenceModule.from_catalog(
|
||||
catalog,
|
||||
command_factory=shop_command_factory,
|
||||
)
|
||||
)
|
||||
supply_module = (
|
||||
SupplyChainModule()
|
||||
if catalog is None
|
||||
else SupplyChainModule.from_catalog(
|
||||
catalog, command_factory=supply_command_factory
|
||||
)
|
||||
)
|
||||
content_module = (
|
||||
ContentMarketingModule()
|
||||
if catalog is None
|
||||
else ContentMarketingModule.from_catalog(
|
||||
catalog, command_factory=content_command_factory
|
||||
)
|
||||
)
|
||||
product_module = (
|
||||
ProductCommerceModule()
|
||||
if catalog is None
|
||||
else ProductCommerceModule.from_catalog(
|
||||
catalog, command_factory=product_command_factory
|
||||
)
|
||||
)
|
||||
|
||||
return ModuleRegistry(
|
||||
(
|
||||
content_module,
|
||||
product_module,
|
||||
shop_module,
|
||||
supply_module,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BusinessModule",
|
||||
"EXPECTED_MODULE_IDS",
|
||||
"ModuleRegistry",
|
||||
"create_default_registry",
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Content metrics, creator, comment, report, and login workflows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
from gyxx_flow.adapters.native import DeferredModuleCommandStep, ModuleCommandFactory
|
||||
from gyxx_flow.workflow import StepDefinition, WorkflowDefinition
|
||||
|
||||
CONTENT_SCHEDULED_WORKFLOW_IDS = (
|
||||
"content.metrics.daily",
|
||||
"content.marketing_report.daily",
|
||||
"content.relogin.weekly",
|
||||
"content.self_operated.weekly",
|
||||
"content.creator_report.monthly",
|
||||
"content.summary.monthly",
|
||||
"content.cooperations.daily",
|
||||
"content.comments.weekly",
|
||||
"content.summary.weekly",
|
||||
)
|
||||
CONTENT_MANUAL_WORKFLOW_IDS = (
|
||||
"content.mapping.refresh",
|
||||
"content.retry_failed",
|
||||
"content.metrics.backfill",
|
||||
)
|
||||
CONTENT_WORKFLOW_IDS = (*CONTENT_SCHEDULED_WORKFLOW_IDS, *CONTENT_MANUAL_WORKFLOW_IDS)
|
||||
CONTENT_TIMEOUT_SECONDS = 4 * 60 * 60
|
||||
CONTENT_RESOURCE = "module:content_marketing"
|
||||
_RELOGIN_WORKFLOW_ID = "content.relogin.weekly"
|
||||
|
||||
|
||||
class ContentMarketingModule:
|
||||
"""Content-marketing business module composition boundary."""
|
||||
|
||||
module_id = "content_marketing"
|
||||
|
||||
def __init__(self, definitions: Iterable[WorkflowDefinition] = ()) -> None:
|
||||
self._definitions = tuple(definitions)
|
||||
|
||||
@classmethod
|
||||
def from_catalog(
|
||||
cls,
|
||||
catalog: Any,
|
||||
*,
|
||||
command_factory: ModuleCommandFactory | None = None,
|
||||
) -> "ContentMarketingModule":
|
||||
"""Build the scheduled content workflows from validated catalog metadata.
|
||||
|
||||
Each launcher resolves from source owned by this installed module.
|
||||
Registration remains inert and dry-run never starts a subprocess.
|
||||
"""
|
||||
|
||||
entries = {
|
||||
entry.workflow_id: entry
|
||||
for entry in catalog.workflows
|
||||
if entry.module == cls.module_id and entry.trigger in {"scheduled", "manual"}
|
||||
}
|
||||
expected = set(CONTENT_WORKFLOW_IDS)
|
||||
if set(entries) != expected:
|
||||
raise ValueError(
|
||||
"content catalog workflows do not match the migration contract: "
|
||||
f"expected={sorted(expected)}, actual={sorted(entries)}"
|
||||
)
|
||||
definitions: list[WorkflowDefinition] = []
|
||||
for workflow_id in CONTENT_WORKFLOW_IDS:
|
||||
entry = entries[workflow_id]
|
||||
action = (
|
||||
DeferredModuleCommandStep(entry)
|
||||
if command_factory is None
|
||||
else DeferredModuleCommandStep(entry, command_factory=command_factory)
|
||||
)
|
||||
definitions.append(
|
||||
WorkflowDefinition(
|
||||
workflow_id,
|
||||
(
|
||||
StepDefinition(
|
||||
"module_run",
|
||||
action,
|
||||
timeout_seconds=CONTENT_TIMEOUT_SECONDS,
|
||||
max_attempts=1,
|
||||
resources=(CONTENT_RESOURCE,),
|
||||
production_sink=True,
|
||||
official_notification=workflow_id
|
||||
== _RELOGIN_WORKFLOW_ID,
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
return cls(definitions)
|
||||
|
||||
def workflow_definitions(self) -> tuple[WorkflowDefinition, ...]:
|
||||
return self._definitions
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CONTENT_WORKFLOW_IDS",
|
||||
"CONTENT_MANUAL_WORKFLOW_IDS",
|
||||
"CONTENT_SCHEDULED_WORKFLOW_IDS",
|
||||
"CONTENT_RESOURCE",
|
||||
"CONTENT_TIMEOUT_SECONDS",
|
||||
"ContentMarketingModule",
|
||||
]
|
||||
@@ -0,0 +1,500 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
|
||||
|
||||
from playwright.sync_api import Page
|
||||
from scrapling.fetchers import DynamicSession
|
||||
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
STATE_DIR = PATHS.state_root
|
||||
NOTES_DIR = PATHS.raw_root / "notes" / "bilibili"
|
||||
COOKIE_FILE = PATHS.browser_cookie_file
|
||||
PROFILE_DIR = PATHS.browser_profile_dir
|
||||
|
||||
BV_RE = re.compile(r"BV[0-9A-Za-z]{10}")
|
||||
AV_RE = re.compile(r"(?:av|aid=)(\d+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {message}", flush=True)
|
||||
|
||||
|
||||
def load_cookies() -> list[dict[str, Any]]:
|
||||
if not COOKIE_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except json.JSONDecodeError:
|
||||
log(f"Cookie file is invalid, ignoring: {COOKIE_FILE}")
|
||||
return []
|
||||
|
||||
|
||||
def restore_cookies(page: Page) -> None:
|
||||
cookies = load_cookies()
|
||||
if not cookies:
|
||||
return
|
||||
try:
|
||||
page.context.add_cookies(cookies)
|
||||
log(f"Loaded cookies: {COOKIE_FILE}")
|
||||
except Exception as exc:
|
||||
log(f"Failed to load cookies, will continue with browser profile: {exc}")
|
||||
|
||||
|
||||
def save_state(page: Page) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
COOKIE_FILE.write_text(
|
||||
json.dumps(page.context.cookies(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
PATHS.browser_storage_state_file.write_text(
|
||||
json.dumps(page.context.storage_state(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
log(f"Saved cookies: {COOKIE_FILE}")
|
||||
|
||||
|
||||
def is_logged_in(page: Page) -> bool:
|
||||
"""B 站登录态靠 SESSDATA cookie,有它才能调评论 API 拿全量。
|
||||
没有 SESSDATA 时 API 只返回热门 3-5 条。
|
||||
"""
|
||||
for c in page.context.cookies():
|
||||
if c.get("name") == "SESSDATA" and c.get("value"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def maybe_wait_for_login(page: Page, login_timeout: int) -> None:
|
||||
"""检测登录态,未登录就等用户扫码。B 站登录页 URL 含 passport.bilibili.com/login。"""
|
||||
if is_logged_in(page):
|
||||
return
|
||||
log("B 站未登录 (无 SESSDATA cookie)。请在弹出的浏览器窗口扫码登录。")
|
||||
# 跳到登录页,确保用户能看到二维码
|
||||
try:
|
||||
page.goto("https://passport.bilibili.com/login", wait_until="domcontentloaded", timeout=30000)
|
||||
except Exception as exc:
|
||||
log(f" 跳登录页失败 (继续等当前页扫码): {exc}")
|
||||
deadline = time.time() + login_timeout
|
||||
while time.time() < deadline:
|
||||
if is_logged_in(page):
|
||||
log("登录成功,SESSDATA 已写入")
|
||||
save_state(page)
|
||||
return
|
||||
page.wait_for_timeout(2000)
|
||||
log(f" [WARN] 等待 {login_timeout}s 仍未登录,继续以游客态采集 (评论只会拿到热门 3-5 条)")
|
||||
|
||||
|
||||
def safe_filename(value: str) -> str:
|
||||
value = re.sub(r"[<>:\"/\\\\|?*\\s]+", "_", value).strip("_")
|
||||
return value[:90] or "bilibili_comments"
|
||||
|
||||
|
||||
def extract_bvid(url: str) -> str:
|
||||
match = BV_RE.search(url)
|
||||
return match.group(0) if match else ""
|
||||
|
||||
|
||||
def extract_aid(url: str) -> str:
|
||||
match = AV_RE.search(url)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def ts_to_iso(value: Any) -> str:
|
||||
if not isinstance(value, (int, float)) or not value:
|
||||
return ""
|
||||
try:
|
||||
return datetime.fromtimestamp(value).isoformat(timespec="seconds")
|
||||
except Exception:
|
||||
return str(value)
|
||||
|
||||
|
||||
def with_query_params(url: str, updates: dict[str, Any]) -> str:
|
||||
parts = urlsplit(url)
|
||||
params = dict((key, values[-1]) for key, values in parse_qs(parts.query, keep_blank_values=True).items())
|
||||
for key, value in updates.items():
|
||||
params[key] = str(value)
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(params), parts.fragment))
|
||||
|
||||
|
||||
def parse_reply(reply: dict[str, Any], source_url: str, level: str, parent_rpid: str = "") -> dict[str, Any]:
|
||||
member = reply.get("member") or {}
|
||||
content = reply.get("content") or {}
|
||||
return {
|
||||
"source_url": source_url,
|
||||
"level": level,
|
||||
"rpid": str(reply.get("rpid") or ""),
|
||||
"parent_rpid": str(parent_rpid or reply.get("parent") or ""),
|
||||
"root_rpid": str(reply.get("root") or ""),
|
||||
"mid": str(member.get("mid") or reply.get("mid") or ""),
|
||||
"nickname": member.get("uname") or "",
|
||||
"sex": member.get("sex") or "",
|
||||
"message": content.get("message") or "",
|
||||
"like_count": reply.get("like") or 0,
|
||||
"reply_count": reply.get("rcount") or 0,
|
||||
"location": reply.get("reply_control", {}).get("location") or "",
|
||||
"created_at": ts_to_iso(reply.get("ctime")),
|
||||
"raw_ctime": reply.get("ctime") or "",
|
||||
}
|
||||
|
||||
|
||||
def merge_reply(row: dict[str, Any], comments_by_id: dict[str, dict[str, Any]]) -> None:
|
||||
key = row.get("rpid") or f"{row.get('level')}:{len(comments_by_id)}"
|
||||
comments_by_id[str(key)] = row
|
||||
|
||||
|
||||
def collect_main_payload(payload: dict[str, Any], source_url: str, comments_by_id: dict[str, dict[str, Any]]) -> None:
|
||||
data = payload.get("data") if isinstance(payload.get("data"), dict) else payload
|
||||
replies = data.get("replies") or []
|
||||
if not isinstance(replies, list):
|
||||
return
|
||||
for reply in replies:
|
||||
if not isinstance(reply, dict):
|
||||
continue
|
||||
row = parse_reply(reply, source_url, "comment")
|
||||
merge_reply(row, comments_by_id)
|
||||
child_replies = reply.get("replies") or []
|
||||
if isinstance(child_replies, list):
|
||||
for child in child_replies:
|
||||
if isinstance(child, dict):
|
||||
child_row = parse_reply(child, source_url, "reply", row.get("rpid") or "")
|
||||
merge_reply(child_row, comments_by_id)
|
||||
|
||||
|
||||
def collect_reply_payload(payload: dict[str, Any], source_url: str, parent_rpid: str, comments_by_id: dict[str, dict[str, Any]]) -> None:
|
||||
data = payload.get("data") if isinstance(payload.get("data"), dict) else payload
|
||||
replies = data.get("replies") or []
|
||||
if not isinstance(replies, list):
|
||||
return
|
||||
for reply in replies:
|
||||
if isinstance(reply, dict):
|
||||
row = parse_reply(reply, source_url, "reply", parent_rpid)
|
||||
merge_reply(row, comments_by_id)
|
||||
|
||||
|
||||
def api_get(page: Page, url: str) -> dict[str, Any]:
|
||||
return page.evaluate(
|
||||
"""
|
||||
async (url) => {
|
||||
const res = await fetch(url, {
|
||||
credentials: 'include',
|
||||
headers: { 'accept': 'application/json, text/plain, */*' }
|
||||
});
|
||||
return await res.json();
|
||||
}
|
||||
""",
|
||||
url,
|
||||
)
|
||||
|
||||
|
||||
def resolve_video_info(page: Page, url: str) -> dict[str, Any]:
|
||||
bvid = extract_bvid(url)
|
||||
aid = extract_aid(url)
|
||||
if not bvid and not aid:
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=90000)
|
||||
page.wait_for_timeout(3000)
|
||||
bvid = extract_bvid(page.url) or extract_bvid(page.locator("body").inner_text(timeout=5000))
|
||||
aid = extract_aid(page.url)
|
||||
|
||||
params = {}
|
||||
if bvid:
|
||||
params["bvid"] = bvid
|
||||
elif aid:
|
||||
params["aid"] = aid
|
||||
else:
|
||||
raise RuntimeError("Could not find BV or AV id from URL.")
|
||||
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view?" + urlencode(params)
|
||||
payload = api_get(page, api_url)
|
||||
if payload.get("code") != 0:
|
||||
raise RuntimeError(f"Failed to resolve video info: {payload}")
|
||||
|
||||
data = payload.get("data") or {}
|
||||
stat = data.get("stat") or {}
|
||||
return {
|
||||
"aid": str(data.get("aid") or aid),
|
||||
"bvid": data.get("bvid") or bvid,
|
||||
"title": data.get("title") or "",
|
||||
"owner_mid": str((data.get("owner") or {}).get("mid") or ""),
|
||||
"owner_name": (data.get("owner") or {}).get("name") or "",
|
||||
"note_metrics": {
|
||||
"like_count": stat.get("like", ""),
|
||||
"favorite_count": stat.get("favorite", ""),
|
||||
"share_count": stat.get("share", ""),
|
||||
"comment_count": stat.get("reply", ""),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def fetch_main_comments(
|
||||
page: Page,
|
||||
source_url: str,
|
||||
oid: str,
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
max_pages: int,
|
||||
delay: float,
|
||||
) -> dict[str, Any]:
|
||||
stats = {"api_hits": 0, "api_total": None, "last_page": 0}
|
||||
page_num = 1
|
||||
page_size = 20
|
||||
|
||||
while page_num <= max_pages:
|
||||
api_url = (
|
||||
"https://api.bilibili.com/x/v2/reply?"
|
||||
+ urlencode({"type": 1, "oid": oid, "pn": page_num, "ps": page_size, "sort": 2})
|
||||
)
|
||||
payload = api_get(page, api_url)
|
||||
stats["api_hits"] += 1
|
||||
if payload.get("code") != 0:
|
||||
log(f"Main comment API returned code={payload.get('code')}: {payload.get('message')}")
|
||||
break
|
||||
|
||||
data = payload.get("data") or {}
|
||||
page_info = data.get("page") or {}
|
||||
stats["api_total"] = page_info.get("count", stats.get("api_total"))
|
||||
stats["last_page"] = page_num
|
||||
|
||||
before = len(comments_by_id)
|
||||
collect_main_payload(payload, source_url, comments_by_id)
|
||||
loaded = len(comments_by_id) - before
|
||||
log(f"Main page {page_num}: +{loaded}, total {len(comments_by_id)}")
|
||||
|
||||
replies = data.get("replies") or []
|
||||
if not replies:
|
||||
break
|
||||
if page_info.get("count") and len([r for r in comments_by_id.values() if r.get("level") == "comment"]) >= int(page_info["count"]):
|
||||
break
|
||||
|
||||
page_num += 1
|
||||
if delay:
|
||||
page.wait_for_timeout(int(delay * 1000))
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def fetch_sub_replies(
|
||||
page: Page,
|
||||
source_url: str,
|
||||
oid: str,
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
delay: float,
|
||||
max_reply_pages: int,
|
||||
) -> int:
|
||||
parents = [
|
||||
row for row in list(comments_by_id.values())
|
||||
if row.get("level") == "comment" and int(row.get("reply_count") or 0) > 0
|
||||
]
|
||||
if not parents:
|
||||
return 0
|
||||
|
||||
log(f"Fetching sub replies for {len(parents)} comments.")
|
||||
merged = 0
|
||||
for parent in parents:
|
||||
root = parent.get("rpid")
|
||||
if not root:
|
||||
continue
|
||||
pn = 1
|
||||
ps = 20
|
||||
while pn <= max_reply_pages:
|
||||
api_url = (
|
||||
"https://api.bilibili.com/x/v2/reply/reply?"
|
||||
+ urlencode({"type": 1, "oid": oid, "root": root, "pn": pn, "ps": ps})
|
||||
)
|
||||
payload = api_get(page, api_url)
|
||||
if payload.get("code") != 0:
|
||||
break
|
||||
data = payload.get("data") or {}
|
||||
before = len(comments_by_id)
|
||||
collect_reply_payload(payload, source_url, str(root), comments_by_id)
|
||||
merged += max(0, len(comments_by_id) - before)
|
||||
|
||||
replies = data.get("replies") or []
|
||||
page_info = data.get("page") or {}
|
||||
if not replies:
|
||||
break
|
||||
if page_info.get("count") and pn * ps >= int(page_info["count"]):
|
||||
break
|
||||
pn += 1
|
||||
if delay:
|
||||
page.wait_for_timeout(int(delay * 1000))
|
||||
log(f"Sub replies merged: {merged}; total {len(comments_by_id)}")
|
||||
return merged
|
||||
|
||||
|
||||
def scrape_comments(
|
||||
url: str,
|
||||
max_pages: int,
|
||||
delay: float,
|
||||
headless: bool,
|
||||
max_reply_pages: int = 200,
|
||||
login_timeout: int = 0,
|
||||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
result: dict[str, Any] = {"comments": [], "stats": {}, "video": {}}
|
||||
|
||||
def action(page: Page) -> None:
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=90000)
|
||||
page.wait_for_timeout(2500)
|
||||
save_state(page)
|
||||
|
||||
# 登录检测:未登录时评论 API 只返回热门 3-5 条
|
||||
if login_timeout > 0:
|
||||
maybe_wait_for_login(page, login_timeout)
|
||||
# 登录后回到视频页(可能在登录页)
|
||||
if not page.url.startswith("https://www.bilibili.com/video/"):
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=90000)
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
video = resolve_video_info(page, url)
|
||||
oid = video["aid"]
|
||||
comments_by_id: dict[str, dict[str, Any]] = {}
|
||||
stats = fetch_main_comments(page, url, oid, comments_by_id, max_pages=max_pages, delay=delay)
|
||||
sub_count = fetch_sub_replies(page, url, oid, comments_by_id, delay=delay, max_reply_pages=max_reply_pages)
|
||||
save_state(page)
|
||||
|
||||
stats["sub_replies_merged"] = sub_count
|
||||
stats["max_reply_pages"] = max_reply_pages
|
||||
stats["top_level_count"] = len([row for row in comments_by_id.values() if row.get("level") == "comment"])
|
||||
stats["reply_count"] = len([row for row in comments_by_id.values() if row.get("level") == "reply"])
|
||||
stats["note_metrics"] = video.get("note_metrics", {})
|
||||
stats["logged_in"] = is_logged_in(page)
|
||||
result["comments"] = list(comments_by_id.values())
|
||||
result["stats"] = stats
|
||||
result["video"] = video
|
||||
result["final_url"] = page.url
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=headless,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
session.fetch(url, page_action=action, wait=1000)
|
||||
|
||||
return result["comments"], result
|
||||
|
||||
|
||||
def write_outputs(url: str, comments: list[dict[str, Any]], result: dict[str, Any]) -> tuple[Path, Path]:
|
||||
NOTES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
bvid = result.get("video", {}).get("bvid") or extract_bvid(url) or safe_filename(url)
|
||||
json_path = NOTES_DIR / f"bilibili_comments_{bvid}.json"
|
||||
csv_path = NOTES_DIR / f"bilibili_comments_{bvid}.csv"
|
||||
if not comments and json_path.exists() and csv_path.exists():
|
||||
log(f"No comments scraped; keeping existing output files: {json_path}, {csv_path}")
|
||||
return json_path, csv_path
|
||||
|
||||
payload = {
|
||||
"source_url": url,
|
||||
"title": result.get("video", {}).get("title", ""),
|
||||
"final_url": result.get("final_url", ""),
|
||||
"video": result.get("video", {}),
|
||||
"scraped_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"count": len(comments),
|
||||
"stats": result.get("stats", {}),
|
||||
"comments": comments,
|
||||
}
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
fieldnames = [
|
||||
"source_url",
|
||||
"level",
|
||||
"rpid",
|
||||
"parent_rpid",
|
||||
"root_rpid",
|
||||
"mid",
|
||||
"nickname",
|
||||
"sex",
|
||||
"message",
|
||||
"like_count",
|
||||
"reply_count",
|
||||
"location",
|
||||
"created_at",
|
||||
"raw_ctime",
|
||||
]
|
||||
with csv_path.open("w", newline="", encoding="utf-8-sig") as file:
|
||||
writer = csv.DictWriter(file, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(comments)
|
||||
|
||||
return json_path, csv_path
|
||||
|
||||
|
||||
def login_only(login_timeout: int) -> int:
|
||||
"""只触发登录(headed),登录完成立刻退出。给 relogin_bilibili.py 用。"""
|
||||
def action(page: Page) -> None:
|
||||
page.goto("https://passport.bilibili.com/login", wait_until="domcontentloaded", timeout=30000)
|
||||
page.wait_for_timeout(1500)
|
||||
maybe_wait_for_login(page, login_timeout)
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=False,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
session.fetch("https://passport.bilibili.com/login", page_action=action, wait=1000)
|
||||
log("[login-only] 完成")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Scrape Bilibili video comments with a real browser and API.")
|
||||
parser.add_argument("url", help="Bilibili video URL (传 dummy 'login' 配合 --login-only 也可)")
|
||||
parser.add_argument("--max-reply-pages", type=int, default=200, help="max sub-reply pages per top-level comment")
|
||||
parser.add_argument("--max-pages", type=int, default=200, help="最多抓取一级评论页数")
|
||||
parser.add_argument("--delay", type=float, default=0.25, help="接口请求间隔秒数")
|
||||
parser.add_argument("--headless", action="store_true", help="无头模式")
|
||||
parser.add_argument("--login-timeout", type=int, default=0,
|
||||
help="未登录时等待扫码秒数(>0 启用登录检测;默认 0 沿用游客态)")
|
||||
parser.add_argument("--login-only", action="store_true",
|
||||
help="只触发登录(headed)然后退出,给 relogin_bilibili 用")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.login_only:
|
||||
return login_only(args.login_timeout if args.login_timeout > 0 else 300)
|
||||
|
||||
comments, result = scrape_comments(
|
||||
args.url, args.max_pages, args.delay, args.headless,
|
||||
args.max_reply_pages, args.login_timeout,
|
||||
)
|
||||
json_path, csv_path = write_outputs(args.url, comments, result)
|
||||
log(f"Final URL: {result.get('final_url', '')}")
|
||||
log(f"Video: {result.get('video', {})}")
|
||||
log(f"Stats: {result.get('stats', {})}")
|
||||
log(f"Scraped {len(comments)} comments")
|
||||
log(f"JSON: {json_path}")
|
||||
log(f"CSV: {csv_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,854 @@
|
||||
"""
|
||||
B 站抓取脚本: 多维表格驱动,按"发布时间到今天的天数差"自动选 7/14/21/28/月槽位
|
||||
|
||||
核心规则:
|
||||
- 槽位由 (今天 - 发布时间) 的天数差决定:
|
||||
0-7 天 → 7天曝光量
|
||||
8-14 天 → 14天曝光量
|
||||
15-21 天 → 21天曝光量
|
||||
22-28 天 → 28天曝光量
|
||||
29+ 天 → 月底曝光量
|
||||
- 同槽覆盖语义: 同一槽多次跑会更新为最新播放量
|
||||
- 抓取: B 站公开 API (api.bilibili.com/x/web-interface/view) 拿 stat.view
|
||||
不用登录,不用 cookie
|
||||
- 字段 ID 全部从 款式_多维表格_对照.json 的 field_map 动态读取
|
||||
- 平台枚举: B站 / b站 / B 站 都接受 (大小写无关)
|
||||
|
||||
用法:
|
||||
# 跑全 24 款式 (默认)
|
||||
python bilibili_scraper.py
|
||||
|
||||
# 试跑不写
|
||||
python bilibili_scraper.py --dry-run
|
||||
|
||||
# 只跑某个款式
|
||||
python bilibili_scraper.py --style 1
|
||||
|
||||
# 只跑某几条
|
||||
python bilibili_scraper.py --record recXXXXX
|
||||
|
||||
# 模拟任意日期 (调试用)
|
||||
python bilibili_scraper.py --today 2025-08-15 --dry-run
|
||||
|
||||
# 防风控:每条请求间隔 0.5s
|
||||
python bilibili_scraper.py --delay 0.5
|
||||
|
||||
# 首次回填: 所有 B 站记录全部填 7天曝光量 (忽略发布时间)
|
||||
python bilibili_scraper.py --first-run
|
||||
|
||||
# 重置运行状态
|
||||
python bilibili_scraper.py --reset-state
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, date
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import (
|
||||
PATHS,
|
||||
resolve_layer_output,
|
||||
)
|
||||
|
||||
import requests
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.collection_completeness import (
|
||||
add_repeatable_style_argument,
|
||||
atomic_write_json,
|
||||
merge_global_summaries,
|
||||
select_requested_styles,
|
||||
)
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DEFAULT_DATA_DIR = PATHS.normalized_root
|
||||
V2_DIR = PATHS.normalized_root / "v2_results"
|
||||
BILIBILI_CHECKPOINT_DIR = PATHS.state_root / "checkpoints" / "bilibili"
|
||||
MAPPING_PATH = PATHS.normalized_root / "mappings" / "款式_多维表格_对照.json"
|
||||
STATE_FILENAME = "weekly_run_state.json"
|
||||
|
||||
LARK_CLI = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd")
|
||||
if not os.path.exists(LARK_CLI):
|
||||
# Scheduled tasks run as SYSTEM; %APPDATA% is not Administrator's dir
|
||||
LARK_CLI = "lark-cli.cmd"
|
||||
|
||||
# 字段 ID 全部从 data/config/款式_多维表格_对照.json 的 field_map 动态读
|
||||
# 下面是 logical key(各款式表结构一致)
|
||||
KEY_PLATFORM = "platform" # 投放平台 (select)
|
||||
KEY_URL = "note_url" # 发布笔记链接 (text)
|
||||
KEY_TITLE = "note_title" # 发布笔记标题 (text)
|
||||
KEY_CREATOR = "creator_name" # 达人名称 (text)
|
||||
KEY_CREATOR_ID = "creator_id" # 达人id (text,选填)
|
||||
KEY_PUBTIME = "publish_time" # 发布时间 (datetime)
|
||||
KEY_MONTH = "month" # 月份 (select)
|
||||
KEY_PARENT = "parent_record" # 父记录 (link,自关联)
|
||||
KEY_WEEK = "publish_week" # 周发布 (select)
|
||||
|
||||
# 5 档曝光量槽位: 7/14/21/28 天 + 月底(第 5 周用) - logical name -> 天数
|
||||
SLOTS = [
|
||||
("read_count_7d", 7),
|
||||
("read_count_14d", 14),
|
||||
("read_count_21d", 21),
|
||||
("read_count_28d", 28),
|
||||
("month_end", "月底"),
|
||||
]
|
||||
|
||||
# 月份枚举(选项里已有)
|
||||
MONTH_OPTIONS = ["6月", "7月", "8月", "9月", "10月"]
|
||||
|
||||
# B 站 API 相关
|
||||
_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
|
||||
_BV_RE = re.compile(r"BV[0-9A-Za-z]{10}")
|
||||
_AV_RE = re.compile(r"av(\d+)", re.IGNORECASE)
|
||||
_B23_RE = re.compile(r"https?://b23\.tv/\S+", re.IGNORECASE)
|
||||
_MD_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
|
||||
_URL_RE = re.compile(r"https?://[^\s)>\]]+")
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
safe = str(msg).encode("gbk", errors="replace").decode("gbk", errors="replace")
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {safe}", flush=True)
|
||||
|
||||
|
||||
def finalize_bili_summary(summary: dict, *, dry_run: bool = False) -> dict:
|
||||
"""Attach one terminal state per Bilibili record and exact completeness counters."""
|
||||
out = dict(summary)
|
||||
unique: list[dict] = []
|
||||
positions: dict[str, int] = {}
|
||||
duplicates = 0
|
||||
for raw in out.get("details") or []:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
row = dict(raw)
|
||||
rid = str(row.get("record_id") or "")
|
||||
if rid and rid in positions:
|
||||
unique[positions[rid]] = row
|
||||
duplicates += 1
|
||||
else:
|
||||
if rid:
|
||||
positions[rid] = len(unique)
|
||||
unique.append(row)
|
||||
total = int(out.get("total_b_records") or 0)
|
||||
success = sum(row.get("status") == "success" for row in unique)
|
||||
blocked = sum(row.get("status") == "blocked_input" for row in unique)
|
||||
retryable = sum(row.get("status") == "retryable_failure" for row in unique)
|
||||
write_failures = sum(row.get("status") == "write_failure" for row in unique)
|
||||
missing = max(0, total - len(unique))
|
||||
unresolved = retryable + write_failures + missing
|
||||
top_level_error = bool(out.get("error"))
|
||||
if top_level_error:
|
||||
retryable = max(retryable, 1)
|
||||
unresolved = max(unresolved, 1)
|
||||
out.update({
|
||||
"details": unique,
|
||||
"updated": success,
|
||||
"skipped": blocked + retryable + write_failures,
|
||||
"success": success,
|
||||
"blocked_input": blocked,
|
||||
"retryable_failures": retryable,
|
||||
"write_failures": write_failures,
|
||||
"missing_results": missing,
|
||||
"duplicate_results": duplicates,
|
||||
"unresolved": unresolved,
|
||||
"complete": (
|
||||
not top_level_error
|
||||
and unresolved == 0
|
||||
and duplicates == 0
|
||||
and len(unique) == total
|
||||
),
|
||||
"dry_run": bool(dry_run),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def merge_bili_summary(existing: dict | None, partial: dict, *, dry_run: bool = False) -> dict:
|
||||
"""Merge a --record retry without deleting previously successful rows."""
|
||||
if not existing:
|
||||
return finalize_bili_summary(partial, dry_run=dry_run)
|
||||
combined = dict(existing)
|
||||
for key, value in partial.items():
|
||||
if key != "details":
|
||||
combined[key] = value
|
||||
rows: list[dict] = []
|
||||
positions: dict[str, int] = {}
|
||||
for source in (existing.get("details") or [], partial.get("details") or []):
|
||||
for raw in source:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
row = dict(raw)
|
||||
rid = str(row.get("record_id") or "")
|
||||
if rid and rid in positions:
|
||||
rows[positions[rid]] = row
|
||||
else:
|
||||
if rid:
|
||||
positions[rid] = len(rows)
|
||||
rows.append(row)
|
||||
combined["details"] = rows
|
||||
combined["total_b_records"] = max(
|
||||
int(existing.get("total_b_records") or 0),
|
||||
int(partial.get("total_b_records") or 0),
|
||||
len(rows),
|
||||
)
|
||||
return finalize_bili_summary(combined, dry_run=dry_run)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 飞书多维表格
|
||||
# ============================================================
|
||||
def call_lark_json(args: list[str]) -> dict:
|
||||
cmd = [LARK_CLI] + args
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(
|
||||
cmd, cwd=PATHS.tmp_root, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace",
|
||||
)
|
||||
text = proc.stdout
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
return {"ok": False, "error": f"no json: {text[:300]}"}
|
||||
try:
|
||||
return json.loads(text[start:], strict=False)
|
||||
except json.JSONDecodeError as e:
|
||||
return {"ok": False, "error": f"json decode: {e}"}
|
||||
|
||||
|
||||
def load_mapping(data_dir: Path, self_operated: bool = False) -> dict:
|
||||
# 动态:从「合作达人」/「自营达人」多维表格地址表读取款式→各表地址,实时拉字段重建对照表
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping
|
||||
return feishu_mapping.load_mapping(None, self_operated=self_operated)
|
||||
|
||||
|
||||
def list_all_records(base_token: str, table_id: str) -> list[dict]:
|
||||
"""拉一张表所有记录,自动分页"""
|
||||
all_records = []
|
||||
offset = 0
|
||||
while True:
|
||||
resp = call_lark_json([
|
||||
"base", "+record-list",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--limit", "200",
|
||||
"--offset", str(offset),
|
||||
])
|
||||
if not resp.get("ok"):
|
||||
raise RuntimeError(f"record-list 失败: {resp.get('error')}")
|
||||
d = resp["data"]
|
||||
ids = d["field_id_list"]
|
||||
rids = d["record_id_list"]
|
||||
rows = d["data"]
|
||||
for i, row in enumerate(rows):
|
||||
rec = {"record_id": rids[i]}
|
||||
for j, fid in enumerate(ids):
|
||||
rec[fid] = row[j] if j < len(row) else None
|
||||
all_records.append(rec)
|
||||
offset += len(rows)
|
||||
if not d.get("has_more", False) or len(rows) == 0:
|
||||
break
|
||||
return all_records
|
||||
|
||||
|
||||
def _write_payload_to_cwd(payload: dict, name: str = "_bili_payload.json") -> str:
|
||||
"""把 payload 写到当前工作目录,返回相对路径名(lark-cli --json @file 必须是 cwd 相对路径名,不是绝对路径)"""
|
||||
payload_file = PATHS.tmp_root / name
|
||||
payload_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload_file.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
return name # 只返回文件名,而不是绝对路径
|
||||
|
||||
|
||||
def write_record(base_token: str, table_id: str, record_id: str,
|
||||
fields: dict, dry_run: bool) -> bool:
|
||||
"""更新一条已有记录"""
|
||||
if dry_run:
|
||||
log(f" [DRY-RUN] 写入: {json.dumps(fields, ensure_ascii=False)[:200]}")
|
||||
return True
|
||||
pf = _write_payload_to_cwd(fields)
|
||||
cmd = [LARK_CLI, "base", "+record-upsert",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--record-id", record_id,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--json", f"@{pf}"]
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(
|
||||
cmd, cwd=PATHS.tmp_root, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace",
|
||||
)
|
||||
text = proc.stdout
|
||||
if proc.stderr:
|
||||
# lark-cli 在 stderr 也会输出,真实响应可能在 stderr
|
||||
if "ok" in proc.stderr and (text == "" or not text.lstrip().startswith("{")):
|
||||
text = proc.stderr
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
log(f" [ERROR] 写记录失败: no json. stdout={proc.stdout[:200]} stderr={proc.stderr[:200]}")
|
||||
return False
|
||||
try:
|
||||
resp = json.loads(text[start:], strict=False)
|
||||
except json.JSONDecodeError as e:
|
||||
log(f" [ERROR] 写记录失败: json decode {e}. text={text[:200]}")
|
||||
return False
|
||||
if not resp.get("ok"):
|
||||
log(f" [ERROR] 写记录失败: {resp.get('error')}")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def create_record(base_token: str, table_id: str,
|
||||
fields: dict, dry_run: bool) -> str | None:
|
||||
"""创建新记录,返回 record_id"""
|
||||
if dry_run:
|
||||
log(f" [DRY-RUN] 新建: {json.dumps(fields, ensure_ascii=False)[:200]}")
|
||||
return "rec_dryrun"
|
||||
pf = _write_payload_to_cwd(fields)
|
||||
cmd = [LARK_CLI, "base", "+record-upsert",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--json", f"@{pf}"]
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(
|
||||
cmd, cwd=PATHS.tmp_root, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace",
|
||||
)
|
||||
text = proc.stdout
|
||||
if proc.stderr:
|
||||
if "ok" in proc.stderr and (text == "" or not text.lstrip().startswith("{")):
|
||||
text = proc.stderr
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
log(f" [ERROR] 创建记录失败: no json. stderr={proc.stderr[:200]}")
|
||||
return None
|
||||
try:
|
||||
resp = json.loads(text[start:], strict=False)
|
||||
except json.JSONDecodeError as e:
|
||||
log(f" [ERROR] 创建记录失败: json decode {e}. text={text[:200]}")
|
||||
return None
|
||||
if not resp.get("ok"):
|
||||
log(f" [ERROR] 创建记录失败: {resp.get('error')}")
|
||||
return None
|
||||
rids = resp.get("data", {}).get("record", {}).get("record_id_list", [])
|
||||
return rids[0] if rids else None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# B 站播放量抓取
|
||||
# ============================================================
|
||||
def extract_url(text) -> str | None:
|
||||
"""从 note_url 字段里抽真实 URL,处理 Markdown / 纯文本 / 短链"""
|
||||
if not text:
|
||||
return None
|
||||
if isinstance(text, list):
|
||||
text = text[0] if text else None
|
||||
if not text:
|
||||
return None
|
||||
s = str(text).strip()
|
||||
m = _MD_LINK_RE.search(s)
|
||||
if m:
|
||||
return m.group(2).strip()
|
||||
m = _URL_RE.search(s)
|
||||
if m:
|
||||
return m.group(0).strip()
|
||||
return None
|
||||
|
||||
|
||||
def resolve_b23(url: str, session: requests.Session) -> str | None:
|
||||
try:
|
||||
r = session.get(url, allow_redirects=True, timeout=10, headers={"User-Agent": _UA})
|
||||
return r.url
|
||||
except Exception as exc:
|
||||
log(f" [WARN] b23 解析失败: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_play_count(url: str, session: requests.Session) -> int | None:
|
||||
real = url
|
||||
if _B23_RE.match(url):
|
||||
resolved = resolve_b23(url, session)
|
||||
if not resolved:
|
||||
return None
|
||||
real = resolved
|
||||
bv = _BV_RE.search(real)
|
||||
av = _AV_RE.search(real)
|
||||
if not bv and not av:
|
||||
return None
|
||||
params = {"bvid": bv.group(0)} if bv else {"aid": av.group(1)}
|
||||
try:
|
||||
r = session.get("https://api.bilibili.com/x/web-interface/view",
|
||||
params=params, timeout=10,
|
||||
headers={"User-Agent": _UA, "Referer": "https://www.bilibili.com/"})
|
||||
j = r.json()
|
||||
if j.get("code") != 0:
|
||||
return None
|
||||
v = j.get("data", {}).get("stat", {}).get("view")
|
||||
return int(v) if v is not None else None
|
||||
except Exception as exc:
|
||||
log(f" [WARN] B 站 API: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 记录关系 + 槽位推进
|
||||
# ============================================================
|
||||
def get_parent_id(rec: dict, fid: str) -> str | None:
|
||||
if not fid:
|
||||
return None
|
||||
p = rec.get(fid)
|
||||
if isinstance(p, list) and p:
|
||||
item = p[0]
|
||||
if isinstance(item, dict):
|
||||
return item.get("id")
|
||||
if isinstance(item, str):
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def get_month(rec: dict, fid: str) -> str | None:
|
||||
if not fid:
|
||||
return None
|
||||
m = rec.get(fid)
|
||||
if isinstance(m, list) and m:
|
||||
return m[0]
|
||||
if isinstance(m, str):
|
||||
return m
|
||||
return None
|
||||
|
||||
|
||||
def month_to_num(m: str) -> int:
|
||||
m = re.match(r"(\d+)月", m or "")
|
||||
return int(m.group(1)) if m else 0
|
||||
|
||||
|
||||
def num_to_month(n: int) -> str:
|
||||
return f"{n}月"
|
||||
|
||||
|
||||
def get_week_label(rec: dict, fid: str) -> str | None:
|
||||
"""从记录的"周发布"字段提取文本(e.g. '7月第二周')"""
|
||||
if not fid:
|
||||
return None
|
||||
w = rec.get(fid)
|
||||
if isinstance(w, list) and w:
|
||||
return w[0]
|
||||
if isinstance(w, str):
|
||||
return w
|
||||
return None
|
||||
|
||||
|
||||
def compute_week_slot(today: date) -> tuple[str, str, int | str]:
|
||||
"""
|
||||
根据今天日期,返回 (week_label, slot_logical_name, slot_key):
|
||||
- 1-7 号 → "X月第一周" → 7天
|
||||
- 8-14 号 → "X月第二周" → 14天
|
||||
- 15-21 号 → "X月第三周" → 21天
|
||||
- 22-28 号 → "X月第四周" → 28天
|
||||
- 29-31 号 → "X月第五周" → 月底曝光量
|
||||
注: 返回的是 logical name (如 'read_count_7d'),需要从 fmap 查 field_id
|
||||
"""
|
||||
day = today.day
|
||||
month = today.month
|
||||
month_label = num_to_month(month)
|
||||
if day <= 7:
|
||||
return f"{month_label}第一周", SLOTS[0][0], SLOTS[0][1]
|
||||
elif day <= 14:
|
||||
return f"{month_label}第二周", SLOTS[1][0], SLOTS[1][1]
|
||||
elif day <= 21:
|
||||
return f"{month_label}第三周", SLOTS[2][0], SLOTS[2][1]
|
||||
elif day <= 28:
|
||||
return f"{month_label}第四周", SLOTS[3][0], SLOTS[3][1]
|
||||
else:
|
||||
return f"{month_label}第五周", SLOTS[4][0], SLOTS[4][1]
|
||||
|
||||
|
||||
def next_slot_in_record(rec: dict, slot_fids: list[str]) -> tuple[str | None, int | str | None]:
|
||||
"""
|
||||
返回 (slot_fid, slot_key) — 该记录下一个该填的槽
|
||||
判定: None / 空字符串 / 0 都视为"未填"
|
||||
5 槽都填了返回 (None, None)
|
||||
slot_fids: 5 档槽位的实际 field_id 列表(顺序对应 SLOTS)
|
||||
"""
|
||||
for fid, days in zip(slot_fids, [s[1] for s in SLOTS]):
|
||||
v = rec.get(fid)
|
||||
if v is None:
|
||||
return fid, days
|
||||
if isinstance(v, str) and v.strip() == "":
|
||||
return fid, days
|
||||
try:
|
||||
if float(v) == 0:
|
||||
return fid, days
|
||||
except (TypeError, ValueError):
|
||||
return fid, days
|
||||
return None, None
|
||||
|
||||
|
||||
def get_slot_fid(fmap: dict, logical_name: str) -> str | None:
|
||||
"""从 fmap 查某个 logical 槽位(如 'read_count_7d')的实际 field_id"""
|
||||
return fmap.get(logical_name, {}).get("field_id")
|
||||
|
||||
|
||||
def build_chains(records: list[dict], fid_parent: str, fid_month: str, fid_pubtime: str) -> dict[str, list[dict]]:
|
||||
"""
|
||||
把记录组织成"链": key=链根 rid(顶层父记录 rid,即父字段为空的)
|
||||
value=按月份升序的子记录列表(含根本身)
|
||||
"""
|
||||
by_id = {r["record_id"]: r for r in records}
|
||||
roots: dict[str, list[dict]] = {}
|
||||
|
||||
for r in records:
|
||||
cur = r
|
||||
while True:
|
||||
pid = get_parent_id(cur, fid=fid_parent)
|
||||
if not pid:
|
||||
root_id = cur["record_id"]
|
||||
break
|
||||
if pid not in by_id:
|
||||
root_id = cur["record_id"] # 父不在表内,自己当根
|
||||
break
|
||||
cur = by_id[pid]
|
||||
roots.setdefault(root_id, []).append(r)
|
||||
|
||||
for k in roots:
|
||||
roots[k].sort(key=lambda x: (
|
||||
month_to_num(get_month(x, fid=fid_month) or "0月"),
|
||||
x.get(fid_pubtime) if fid_pubtime else 0
|
||||
))
|
||||
return roots
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 状态持久化(运行历史,可重置)
|
||||
# ============================================================
|
||||
def load_state(data_dir: Path) -> dict:
|
||||
del data_dir
|
||||
p = BILIBILI_CHECKPOINT_DIR / STATE_FILENAME
|
||||
if not p.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(data_dir: Path, state: dict) -> None:
|
||||
del data_dir
|
||||
p = BILIBILI_CHECKPOINT_DIR / STATE_FILENAME
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_write_json(p, state)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 单款式处理
|
||||
# ============================================================
|
||||
def process_style(style: dict, only_record_ids: set[str] | None,
|
||||
dry_run: bool, delay: float, session: requests.Session,
|
||||
state: dict, force_today: date | None,
|
||||
first_run: bool = False) -> dict:
|
||||
base_token = style["base_token"]
|
||||
table_id = style["table_id"]
|
||||
fmap = style.get("field_map", {})
|
||||
|
||||
# 从本款 field_map 解析出实际字段 ID
|
||||
fid_platform = fmap.get(KEY_PLATFORM, {}).get("field_id", "")
|
||||
fid_url = fmap.get(KEY_URL, {}).get("field_id", "")
|
||||
fid_creator = fmap.get(KEY_CREATOR, {}).get("field_id", "")
|
||||
fid_pubtime = fmap.get(KEY_PUBTIME, {}).get("field_id", "")
|
||||
fid_month = fmap.get(KEY_MONTH, {}).get("field_id", "")
|
||||
fid_parent = fmap.get(KEY_PARENT, {}).get("field_id", "")
|
||||
fid_week = fmap.get(KEY_WEEK, {}).get("field_id", "")
|
||||
|
||||
# 5 档槽位的 field_id 列表
|
||||
slot_fids = [get_slot_fid(fmap, ln) or "" for ln, _ in SLOTS]
|
||||
|
||||
log(f"\n========== [{style['index']:02d}] {style['name']} (B 站) ==========")
|
||||
log(f" 字段映射: platform={fid_platform} url={fid_url} creator={fid_creator or '(无)'} pubtime={fid_pubtime or '(无)'}")
|
||||
missing_mapping: list[str] = []
|
||||
if not fid_platform:
|
||||
missing_mapping.append("platform")
|
||||
if not fid_url:
|
||||
missing_mapping.append("note_url")
|
||||
if first_run:
|
||||
if not slot_fids[0]:
|
||||
missing_mapping.append("read_count_7d")
|
||||
else:
|
||||
if not fid_pubtime:
|
||||
missing_mapping.append("publish_time")
|
||||
missing_mapping.extend(
|
||||
logical
|
||||
for field_id, (logical, _label) in zip(slot_fids, SLOTS)
|
||||
if not field_id
|
||||
)
|
||||
if missing_mapping:
|
||||
error = f"missing field mapping: {','.join(missing_mapping)}"
|
||||
log(f" [ERROR] {error},跳过该款式")
|
||||
return {"style": style["name"], "index": style["index"],
|
||||
"total_b_records": 0, "updated": 0, "skipped": 0, "details": [],
|
||||
"run_started_at": time.time(),
|
||||
"error": error, "unresolved": 1,
|
||||
"retryable_failures": 1, "complete": False}
|
||||
|
||||
all_records = list_all_records(base_token, table_id)
|
||||
# 只看 B 站记录 (大小写无关)
|
||||
def is_bili_platform(v) -> bool:
|
||||
if not v:
|
||||
return False
|
||||
s = str(v).lower()
|
||||
return "b站" in s or "b 站" in s
|
||||
|
||||
b_records = [r for r in all_records
|
||||
if r.get(fid_platform) and is_bili_platform(r.get(fid_platform))]
|
||||
log(f" 全表 {len(all_records)} 条,B 站源记录 {len(b_records)} 条")
|
||||
|
||||
if only_record_ids:
|
||||
b_records = [r for r in b_records if r["record_id"] in only_record_ids]
|
||||
log(f" --record 过滤后,B 站源记录剩 {len(b_records)} 条")
|
||||
|
||||
# 业务触发条件:只有填了发布笔记链接的记录才进入采集清单。
|
||||
# 无链接通常表示尚未发布，不是 blocked_input，也不进入完整度分母。
|
||||
source_b_records = len(b_records)
|
||||
b_records = [r for r in b_records if extract_url(r.get(fid_url))]
|
||||
skipped_no_url = source_b_records - len(b_records)
|
||||
log(f" 发布链接过滤: 待采 {len(b_records)} 条,未触发 {skipped_no_url} 条")
|
||||
|
||||
today = force_today or date.today()
|
||||
style_state = state.setdefault(style["name"], {})
|
||||
summary = {
|
||||
"style": style["name"], "index": style["index"],
|
||||
"run_started_at": time.time(),
|
||||
"source_b_records": source_b_records,
|
||||
"skipped_no_url": skipped_no_url,
|
||||
"total_b_records": len(b_records), "updated": 0, "skipped": 0,
|
||||
"details": [],
|
||||
}
|
||||
|
||||
for rec in b_records:
|
||||
rid = rec["record_id"]
|
||||
creator = rec.get(fid_creator) or "?"
|
||||
pubtime_raw = rec.get(fid_pubtime)
|
||||
url = extract_url(rec.get(fid_url))
|
||||
if not url: # 防御性保护；正常情况下已在待采清单生成前过滤。
|
||||
continue
|
||||
|
||||
# 解析发布时间
|
||||
pub_date = parse_pub_date(pubtime_raw)
|
||||
if pub_date is None and not first_run:
|
||||
log(f" [SKIP] {rid[:10]}.. {creator} 发布时间为空,无法判断槽位")
|
||||
summary["skipped"] += 1
|
||||
summary["details"].append({
|
||||
"record_id": rid, "creator": creator, "url": url,
|
||||
"status": "blocked_input", "matched": False,
|
||||
"reason": "publish_time_missing", "ok": False,
|
||||
})
|
||||
continue
|
||||
|
||||
# 算槽位: (今天 - 发布日) 天数差
|
||||
if first_run:
|
||||
target_slot_fid, target_slot_key = slot_fids[0] or None, SLOTS[0][1] if slot_fids[0] else None
|
||||
else:
|
||||
days_elapsed = (today - pub_date).days
|
||||
target_slot_fid, target_slot_key = pick_slot_by_days(days_elapsed, slot_fids)
|
||||
if target_slot_fid is None:
|
||||
if first_run:
|
||||
log(f" [SKIP] {rid[:10]}.. {creator} 7天曝光量字段缺失")
|
||||
else:
|
||||
log(f" [SKIP] {rid[:10]}.. {creator} 对应槽位字段缺失或发布时间 {pub_date} 在 {today} 之后")
|
||||
summary["skipped"] += 1
|
||||
summary["details"].append({
|
||||
"record_id": rid, "creator": creator, "url": url,
|
||||
"publish_date": str(pub_date) if pub_date else None,
|
||||
"status": "blocked_input", "matched": False,
|
||||
"reason": "slot_missing_or_future_publish_time", "ok": False,
|
||||
})
|
||||
continue
|
||||
|
||||
# 抓播放量
|
||||
play = fetch_play_count(url, session)
|
||||
if play is None:
|
||||
log(f" [SKIP] {rid[:10]}.. {creator} 抓不到播放量")
|
||||
summary["skipped"] += 1
|
||||
summary["details"].append({
|
||||
"record_id": rid, "creator": creator, "url": url,
|
||||
"publish_date": str(pub_date) if pub_date else None,
|
||||
"status": "retryable_failure", "matched": False,
|
||||
"reason": "play_count_fetch_failed", "ok": False,
|
||||
})
|
||||
continue
|
||||
|
||||
# 覆盖写入 (同槽覆盖语义,数字更新)
|
||||
action = "first_run_fill" if first_run else "update_slot"
|
||||
log(f" {rid[:10]}.. {creator} 发布={pub_date} 距今={(today - pub_date).days if pub_date else '-'}天"
|
||||
f" → 写 {target_slot_key}天曝光量={play}")
|
||||
ok = write_record(base_token, table_id, rid,
|
||||
{target_slot_fid: play}, dry_run)
|
||||
if ok:
|
||||
summary["updated"] += 1
|
||||
style_state[rid] = {"last_run": str(today), "slot": target_slot_key}
|
||||
summary["details"].append({
|
||||
"record_id": rid, "creator": creator,
|
||||
"publish_date": str(pub_date) if pub_date else None,
|
||||
"days_elapsed": (today - pub_date).days if pub_date else None,
|
||||
"slot": target_slot_key, "play_count": play,
|
||||
"action": action, "ok": ok,
|
||||
"matched": True,
|
||||
"write_ok": ok if not dry_run else None,
|
||||
"status": "success" if ok else "write_failure",
|
||||
**({} if ok else {"reason": "write_back_failed"}),
|
||||
})
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
|
||||
state[style["name"]] = style_state
|
||||
return finalize_bili_summary(summary, dry_run=dry_run)
|
||||
|
||||
|
||||
# 槽位:天数差 → (slot_fid, slot_key)
|
||||
# 0-7天→7天, 8-14→14天, 15-21→21天, 22-28→28天, 29+→月底
|
||||
def pick_slot_by_days(days: int, slot_fids: list[str]) -> tuple[str | None, int | str | None]:
|
||||
if days < 0:
|
||||
return None, None
|
||||
if days <= 7:
|
||||
fid = slot_fids[0]
|
||||
elif days <= 14:
|
||||
fid = slot_fids[1]
|
||||
elif days <= 21:
|
||||
fid = slot_fids[2]
|
||||
elif days <= 28:
|
||||
fid = slot_fids[3]
|
||||
else:
|
||||
fid = slot_fids[4]
|
||||
if not fid:
|
||||
return None, None
|
||||
return fid, SLOTS[slot_fids.index(fid)][1]
|
||||
|
||||
|
||||
def parse_pub_date(v) -> date | None:
|
||||
"""发布时间字段可能是毫秒时间戳 (int) 或 'YYYY-MM-DD HH:MM:SS' 字符串"""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, (int, float)):
|
||||
# 毫秒时间戳
|
||||
if v > 1e12:
|
||||
v = v / 1000
|
||||
try:
|
||||
return datetime.fromtimestamp(v).date()
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(v, str):
|
||||
s = v.strip()
|
||||
if not s:
|
||||
return None
|
||||
# 试常见格式
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d", "%Y/%m/%d %H:%M:%S"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# main
|
||||
# ============================================================
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="B 站抓取+周更: 自动推进 7/14/21/28/月曝光量槽 + 跨月开新子记录")
|
||||
add_repeatable_style_argument(parser, "只跑指定款式编号(可多次)")
|
||||
parser.add_argument("--record", action="append", help="只跑指定 record_id(可多次)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="只抓不写")
|
||||
parser.add_argument("--today", help="强制 today 日期,格式 YYYY-MM-DD (调试用)")
|
||||
parser.add_argument("--delay", type=float, default=0.3,
|
||||
help="每条请求间隔秒数(防风控,默认 0.3s)")
|
||||
parser.add_argument("--data-dir", default=str(DEFAULT_DATA_DIR),
|
||||
help=f"数据目录(默认 {DEFAULT_DATA_DIR})")
|
||||
parser.add_argument("--reset-state", action="store_true", help="重置运行状态")
|
||||
parser.add_argument("--first-run", action="store_true",
|
||||
help="首次回填: 所有 B 站记录全部填 7天曝光量,跳过周次逻辑")
|
||||
parser.add_argument("--self-operated", action="store_true",
|
||||
help="抓取自营达人表格(而非合作达人)")
|
||||
args = parser.parse_args()
|
||||
|
||||
data_dir = resolve_layer_output(
|
||||
args.data_dir, layer_root=PATHS.normalized_root, field="--data-dir"
|
||||
)
|
||||
mapping = load_mapping(data_dir, self_operated=args.self_operated)
|
||||
styles = mapping["tables"]
|
||||
if args.style:
|
||||
try:
|
||||
styles = select_requested_styles(styles, args.style)
|
||||
except ValueError as exc:
|
||||
print(str(exc)); return 1
|
||||
|
||||
only_rids = set(args.record) if args.record else None
|
||||
force_today = None
|
||||
if args.today:
|
||||
force_today = date.fromisoformat(args.today)
|
||||
|
||||
state_path = BILIBILI_CHECKPOINT_DIR / STATE_FILENAME
|
||||
if args.reset_state and state_path.exists():
|
||||
state_path.unlink()
|
||||
log("状态已重置")
|
||||
|
||||
state = load_state(data_dir)
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": _UA})
|
||||
|
||||
out_dir = V2_DIR if data_dir == PATHS.normalized_root.resolve() else data_dir / "v2_results"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
all_summaries = []
|
||||
self_sfx = "_self" if args.self_operated else ""
|
||||
final = out_dir / f"_all_summaries{self_sfx}_bilibili.json"
|
||||
for s in styles:
|
||||
summary = process_style(s, only_rids, args.dry_run, args.delay,
|
||||
session, state, force_today, args.first_run)
|
||||
# 传了 --record 但本款式没匹配到 → 跳过 (不落空盘,不计入总汇总)
|
||||
if only_rids and summary["total_b_records"] == 0 and summary["updated"] == 0 \
|
||||
and summary["skipped"] == 0:
|
||||
continue
|
||||
# 每款式落盘
|
||||
out_file = out_dir / f"{s['index']:02d}-{s['name']}{self_sfx}_bilibili_v2.json"
|
||||
if only_rids and out_file.exists():
|
||||
try:
|
||||
previous = json.loads(out_file.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
previous = None
|
||||
summary = merge_bili_summary(previous, summary, dry_run=args.dry_run)
|
||||
else:
|
||||
summary = finalize_bili_summary(summary, dry_run=args.dry_run)
|
||||
all_summaries.append(summary)
|
||||
atomic_write_json(out_file, summary)
|
||||
log(f" [{s['index']:02d} {s['name']}] B站记录={summary['total_b_records']} "
|
||||
f"更新={summary['updated']} 跳过={summary['skipped']} "
|
||||
f"未解决={summary.get('unresolved', 0)} 落盘: {out_file.name}")
|
||||
|
||||
if not args.dry_run:
|
||||
save_state(data_dir, state)
|
||||
log(f" 状态已存: {state_path}")
|
||||
|
||||
# 总汇总按款式合并，单款/局部补跑不能删除其他款式。
|
||||
try:
|
||||
existing_global = json.loads(final.read_text(encoding="utf-8")) if final.exists() else []
|
||||
except Exception:
|
||||
existing_global = []
|
||||
atomic_write_json(final, merge_global_summaries(existing_global, all_summaries))
|
||||
log(f"\n=== 全部完成 ===")
|
||||
log(f"汇总: {final}")
|
||||
incomplete = [s.get("index") for s in all_summaries if not s.get("complete")]
|
||||
if incomplete:
|
||||
log(f"[ERROR] 以下款式仍有未解决采集/写回结果: {incomplete}")
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,583 @@
|
||||
"""Shared completeness, matching, and atomic-result helpers for V2 scrapers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
|
||||
SUCCESS = "success"
|
||||
BLOCKED_INPUT = "blocked_input"
|
||||
RETRYABLE_FAILURE = "retryable_failure"
|
||||
WRITE_FAILURE = "write_failure"
|
||||
TERMINAL_STATUSES = {SUCCESS, BLOCKED_INPUT, RETRYABLE_FAILURE, WRITE_FAILURE}
|
||||
|
||||
|
||||
def add_repeatable_style_argument(parser: Any, help_text: str) -> Any:
|
||||
"""Add the shared repeatable --style option used by every platform CLI."""
|
||||
return parser.add_argument(
|
||||
"--style", type=int, action="append", help=help_text,
|
||||
)
|
||||
|
||||
|
||||
def select_requested_styles(styles: Iterable[dict], requested: Iterable[int] | None) -> list[dict]:
|
||||
"""Select every requested style and fail when any requested index is unknown."""
|
||||
style_list = list(styles)
|
||||
if not requested:
|
||||
return style_list
|
||||
requested_set = {int(index) for index in requested}
|
||||
selected = [style for style in style_list if int(style.get("index", -1)) in requested_set]
|
||||
found = {int(style.get("index", -1)) for style in selected}
|
||||
missing = sorted(requested_set - found)
|
||||
if missing:
|
||||
raise ValueError(f"未找到款式: {','.join(str(index) for index in missing)}")
|
||||
return selected
|
||||
|
||||
|
||||
def atomic_write_json(path: str | Path, value: Any) -> None:
|
||||
"""Write JSON through a sibling temporary file and atomically replace target."""
|
||||
target = Path(path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = target.with_suffix(target.suffix + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, ensure_ascii=False, indent=2)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp, target)
|
||||
|
||||
|
||||
def coerce_url(value: Any) -> str:
|
||||
"""Extract a URL from common Feishu URL-field shapes."""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
markdown = re.search(r"\[[^\]]*\]\((https?://[^)]+)\)", text, flags=re.IGNORECASE)
|
||||
if markdown:
|
||||
return markdown.group(1).strip()
|
||||
embedded = re.search(r"https?://[^\s)>\]]+", text, flags=re.IGNORECASE)
|
||||
return embedded.group(0).strip() if embedded else text
|
||||
if isinstance(value, dict):
|
||||
for key in ("link", "url", "href", "text"):
|
||||
candidate = value.get(key)
|
||||
if isinstance(candidate, str) and candidate.strip():
|
||||
return candidate.strip()
|
||||
return ""
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
candidate = coerce_url(item)
|
||||
if candidate:
|
||||
return candidate
|
||||
return ""
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def extract_content_id(url_value: Any, platform: str) -> str | None:
|
||||
"""Extract a stable Xiaohongshu note ID or Douyin video ID from a URL."""
|
||||
url = coerce_url(url_value)
|
||||
if not url:
|
||||
return None
|
||||
platform_key = platform.casefold()
|
||||
if platform_key in {"xhs", "xiaohongshu", "pgy"}:
|
||||
match = re.search(r"/(?:explore|discovery/item|note)/([A-Za-z0-9_-]+)", url)
|
||||
return match.group(1) if match else None
|
||||
if platform_key in {"douyin", "xingtu", "xt"}:
|
||||
match = re.search(r"/video/(\d+)", url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
try:
|
||||
query = parse_qs(urlparse(url).query)
|
||||
except ValueError:
|
||||
return None
|
||||
for key in ("modal_id", "item_ids", "item_id", "video_id"):
|
||||
values = query.get(key) or []
|
||||
if values:
|
||||
id_match = re.search(r"\d+", str(values[0]))
|
||||
if id_match:
|
||||
return id_match.group(0)
|
||||
return None
|
||||
|
||||
|
||||
def validate_platform_url(url_value: Any, platform: str) -> tuple[bool, str | None]:
|
||||
"""Validate URL host without rejecting known short-link domains."""
|
||||
url = coerce_url(url_value)
|
||||
if not url:
|
||||
return False, "url_missing"
|
||||
try:
|
||||
host = (urlparse(url).hostname or "").casefold()
|
||||
except ValueError:
|
||||
return False, "url_invalid"
|
||||
if not host:
|
||||
return False, "url_invalid"
|
||||
key = platform.casefold()
|
||||
allowed = {
|
||||
"xhs": ("xiaohongshu.com", "xhslink.com"),
|
||||
"pgy": ("xiaohongshu.com", "xhslink.com"),
|
||||
"xiaohongshu": ("xiaohongshu.com", "xhslink.com"),
|
||||
"douyin": ("douyin.com", "iesdouyin.com"),
|
||||
"xingtu": ("douyin.com", "iesdouyin.com"),
|
||||
"xt": ("douyin.com", "iesdouyin.com"),
|
||||
}.get(key, ())
|
||||
if allowed and not any(host == domain or host.endswith("." + domain) for domain in allowed):
|
||||
return False, "platform_url_mismatch"
|
||||
return True, None
|
||||
|
||||
|
||||
def normalize_title(value: Any) -> str:
|
||||
"""Unicode/case normalization followed by punctuation and whitespace removal."""
|
||||
text = unicodedata.normalize("NFKC", str(value or "")).casefold()
|
||||
text = re.sub(r"[\u200b-\u200f\u2060\ufeff]", "", text)
|
||||
return re.sub(r"[\W_]+", "", text, flags=re.UNICODE)
|
||||
|
||||
|
||||
def _title_variants(value: Any) -> list[str]:
|
||||
"""Build safe title forms for platform-added episode prefixes and hashtags."""
|
||||
text = unicodedata.normalize("NFKC", str(value or "")).casefold()
|
||||
text = re.sub(r"[\u200b-\u200f\u2060\ufeff]", "", text).strip()
|
||||
if not text:
|
||||
return []
|
||||
|
||||
raw_variants = [text]
|
||||
without_episode = re.sub(
|
||||
r"^\s*[【\[（(]?\s*第\s*[0-9一二三四五六七八九十百零两]+\s*"
|
||||
r"[集期篇回弹]\s*[】\]）)]?\s*[:：\-—、.]?\s*",
|
||||
"",
|
||||
text,
|
||||
)
|
||||
without_episode = re.sub(
|
||||
r"^\s*(?:ep(?:isode)?|part)\s*\d+\s*[:：\-—、.]?\s*",
|
||||
"",
|
||||
without_episode,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
if without_episode != text:
|
||||
raw_variants.append(without_episode)
|
||||
|
||||
for candidate in list(raw_variants):
|
||||
before_hashtag = re.split(r"[#＃]", candidate, maxsplit=1)[0].strip()
|
||||
if before_hashtag and before_hashtag != candidate:
|
||||
raw_variants.append(before_hashtag)
|
||||
|
||||
variants: list[str] = []
|
||||
for candidate in raw_variants:
|
||||
normalized = normalize_title(candidate)
|
||||
if normalized and normalized not in variants:
|
||||
variants.append(normalized)
|
||||
return variants
|
||||
|
||||
|
||||
def _episode_marker(value: Any) -> str | None:
|
||||
text = unicodedata.normalize("NFKC", str(value or "")).casefold().strip()
|
||||
chinese = re.match(
|
||||
r"^[【\[（(]?\s*第\s*([0-9一二三四五六七八九十百零两]+)\s*[集期篇回弹]",
|
||||
text,
|
||||
)
|
||||
english = re.match(r"^(?:ep(?:isode)?|part)\s*(\d+)", text, flags=re.IGNORECASE)
|
||||
match = chinese or english
|
||||
if not match:
|
||||
return None
|
||||
marker = match.group(1)
|
||||
return str(int(marker)) if marker.isdigit() else marker
|
||||
|
||||
|
||||
def _normalized_title_similarity(card: str, target: str) -> float:
|
||||
if not card or not target:
|
||||
return 0.0
|
||||
if card == target:
|
||||
return 1.0
|
||||
shorter, longer = sorted((card, target), key=len)
|
||||
if len(shorter) >= 6 and shorter in longer:
|
||||
coverage = len(shorter) / max(len(longer), 1)
|
||||
return min(0.98, 0.93 + 0.05 * coverage)
|
||||
ratio = SequenceMatcher(None, card, target, autojunk=False).ratio()
|
||||
# Prefixes are common when the UI truncates a title.
|
||||
common_prefix = os.path.commonprefix([card, target])
|
||||
if len(common_prefix) >= 8:
|
||||
ratio = max(ratio, 0.78 + min(0.12, len(common_prefix) / max(len(card), len(target)) * 0.12))
|
||||
return ratio
|
||||
|
||||
|
||||
def title_similarity(card_title: Any, target_title: Any) -> float:
|
||||
"""Return a conservative 0..1 score suitable for unique-best matching."""
|
||||
card_episode = _episode_marker(card_title)
|
||||
target_episode = _episode_marker(target_title)
|
||||
if card_episode and target_episode and card_episode != target_episode:
|
||||
return 0.0
|
||||
card_variants = _title_variants(card_title)
|
||||
target_variants = _title_variants(target_title)
|
||||
if not card_variants or not target_variants:
|
||||
return 0.0
|
||||
return max(
|
||||
_normalized_title_similarity(card, target)
|
||||
for card in card_variants
|
||||
for target in target_variants
|
||||
)
|
||||
|
||||
|
||||
def _metric_number(value: Any) -> float | None:
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
text = str(value or "").strip().replace(",", "")
|
||||
match = re.match(r"^([0-9]+(?:\.[0-9]+)?)\s*(万|w)?", text, flags=re.IGNORECASE)
|
||||
if not match:
|
||||
return None
|
||||
number = float(match.group(1))
|
||||
return number * 10000 if match.group(2) else number
|
||||
|
||||
|
||||
def deduplicate_observed_cards(cards: Iterable[dict]) -> list[dict]:
|
||||
"""Merge API/DOM observations of the same video while preserving distinct IDs."""
|
||||
unique: list[dict] = []
|
||||
for raw in cards:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
card = dict(raw)
|
||||
card_id = str(card.get("note_id") or "")
|
||||
card_url = coerce_url(card.get("href") or card.get("url")).rstrip("/")
|
||||
duplicate_index: int | None = None
|
||||
for index, old in enumerate(unique):
|
||||
old_id = str(old.get("note_id") or "")
|
||||
old_url = coerce_url(old.get("href") or old.get("url")).rstrip("/")
|
||||
if card_id and old_id:
|
||||
if card_id == old_id:
|
||||
duplicate_index = index
|
||||
if duplicate_index is not None or card_id != old_id:
|
||||
if duplicate_index is not None:
|
||||
break
|
||||
continue
|
||||
if card_url and old_url and card_url == old_url:
|
||||
duplicate_index = index
|
||||
break
|
||||
|
||||
one_is_api = (
|
||||
card.get("source") == "show_items_api"
|
||||
or old.get("source") == "show_items_api"
|
||||
)
|
||||
one_missing_id = not card_id or not old_id
|
||||
same_title = (
|
||||
normalize_title(card.get("title"))
|
||||
and normalize_title(card.get("title")) == normalize_title(old.get("title"))
|
||||
)
|
||||
card_play = _metric_number(card.get("play_count"))
|
||||
old_play = _metric_number(old.get("play_count"))
|
||||
metric_compatible = (
|
||||
card_play is None
|
||||
or old_play is None
|
||||
or abs(card_play - old_play) / max(card_play, old_play, 1.0) <= 0.08
|
||||
)
|
||||
if one_is_api and one_missing_id and same_title and metric_compatible:
|
||||
duplicate_index = index
|
||||
break
|
||||
|
||||
if duplicate_index is None:
|
||||
unique.append(card)
|
||||
continue
|
||||
|
||||
old = unique[duplicate_index]
|
||||
card_quality = int(bool(card.get("note_id"))) + int(card.get("source") == "show_items_api")
|
||||
old_quality = int(bool(old.get("note_id"))) + int(old.get("source") == "show_items_api")
|
||||
preferred, fallback = (card, old) if card_quality > old_quality else (old, card)
|
||||
merged = dict(fallback)
|
||||
merged.update({key: value for key, value in preferred.items() if value not in (None, "")})
|
||||
unique[duplicate_index] = merged
|
||||
return unique
|
||||
|
||||
|
||||
def match_tasks_to_cards(
|
||||
tasks: Iterable[dict],
|
||||
cards: Iterable[dict],
|
||||
platform: str,
|
||||
*,
|
||||
min_score: float = 0.78,
|
||||
ambiguity_margin: float = 0.035,
|
||||
) -> dict[str, dict]:
|
||||
"""Match by ID first, then safely allocate each observed card to one source note."""
|
||||
card_list = [dict(card) for card in cards if isinstance(card, dict)]
|
||||
for card in card_list:
|
||||
if not card.get("note_id"):
|
||||
card["note_id"] = extract_content_id(card.get("href") or card.get("url"), platform)
|
||||
|
||||
task_list = [dict(task) for task in tasks if isinstance(task, dict) and task.get("record_id")]
|
||||
|
||||
def task_source_key(task: dict, content_id: Any = None) -> tuple[str, str]:
|
||||
stable_id = content_id or task.get("note_id") or extract_content_id(task.get("note_url"), platform)
|
||||
if stable_id:
|
||||
return "id", str(stable_id)
|
||||
url = coerce_url(task.get("note_url")).rstrip("/")
|
||||
if url:
|
||||
return "url", url
|
||||
# Without a source URL, never let separate records reuse one fuzzy match.
|
||||
return "record", str(task.get("record_id"))
|
||||
|
||||
def task_allows_card(task: dict, card: dict) -> bool:
|
||||
task_id = task.get("note_id") or extract_content_id(task.get("note_url"), platform)
|
||||
card_id = card.get("note_id")
|
||||
return not task_id or not card_id or str(task_id) == str(card_id)
|
||||
|
||||
def card_key(card: dict, index: int) -> tuple[str, str]:
|
||||
if card.get("note_id"):
|
||||
return "id", str(card["note_id"])
|
||||
url = coerce_url(card.get("href") or card.get("url")).rstrip("/")
|
||||
return ("url", url) if url else ("index", str(index))
|
||||
|
||||
indexed_cards = [(index, card, card_key(card, index)) for index, card in enumerate(card_list)]
|
||||
matched: dict[str, dict] = {}
|
||||
claimed: dict[tuple[str, str], tuple[str, str]] = {}
|
||||
remaining: list[dict] = []
|
||||
|
||||
for task in task_list:
|
||||
record_id = str(task.get("record_id") or "")
|
||||
content_id = task.get("note_id") or extract_content_id(task.get("note_url"), platform)
|
||||
if content_id:
|
||||
id_matches = [
|
||||
(index, card, key)
|
||||
for index, card, key in indexed_cards
|
||||
if str(card.get("note_id") or "") == str(content_id)
|
||||
]
|
||||
if id_matches:
|
||||
_, raw_card, key = id_matches[0]
|
||||
source_key = task_source_key(task, content_id)
|
||||
owner = claimed.get(key)
|
||||
if owner is not None and owner != source_key:
|
||||
remaining.append(task)
|
||||
continue
|
||||
claimed[key] = source_key
|
||||
selected = dict(raw_card)
|
||||
selected.update({"match_method": "content_id", "match_score": 1.0})
|
||||
matched[record_id] = selected
|
||||
continue
|
||||
remaining.append(task)
|
||||
|
||||
# If two different source URLs have the exact same title, title alone cannot
|
||||
# identify which record owns the observed card. Leave both unresolved.
|
||||
conflicted_cards: set[tuple[str, str]] = set()
|
||||
for _, card, key in indexed_cards:
|
||||
exact_sources = {
|
||||
task_source_key(task)
|
||||
for task in remaining
|
||||
if task_allows_card(task, card)
|
||||
if title_similarity(card.get("title"), task.get("target_title")) == 1.0
|
||||
}
|
||||
if len(exact_sources) > 1:
|
||||
conflicted_cards.add(key)
|
||||
|
||||
def task_best_score(task: dict) -> float:
|
||||
return max(
|
||||
(
|
||||
title_similarity(card.get("title"), task.get("target_title"))
|
||||
for _, card, key in indexed_cards
|
||||
if key not in conflicted_cards
|
||||
and task_allows_card(task, card)
|
||||
),
|
||||
default=0.0,
|
||||
)
|
||||
|
||||
# Exact/strongest tasks claim first, so a looser title cannot steal the one
|
||||
# card needed by a later exact task.
|
||||
remaining.sort(key=lambda task: (-task_best_score(task), str(task.get("record_id"))))
|
||||
|
||||
for task in remaining:
|
||||
record_id = str(task.get("record_id") or "")
|
||||
source_key = task_source_key(task)
|
||||
|
||||
scored = sorted(
|
||||
(
|
||||
(title_similarity(card.get("title"), task.get("target_title")), index, card, key)
|
||||
for index, card, key in indexed_cards
|
||||
if key not in conflicted_cards
|
||||
and task_allows_card(task, card)
|
||||
and (claimed.get(key) is None or claimed.get(key) == source_key)
|
||||
),
|
||||
key=lambda item: (-item[0], item[1]),
|
||||
)
|
||||
if not scored or scored[0][0] < min_score:
|
||||
continue
|
||||
best_score, _, best_card, best_key = scored[0]
|
||||
second_score = scored[1][0] if len(scored) > 1 else 0.0
|
||||
if (
|
||||
second_score >= min_score
|
||||
and best_score - second_score < ambiguity_margin
|
||||
and (best_score < 1.0 or second_score == 1.0)
|
||||
):
|
||||
second_card = scored[1][2]
|
||||
best_id = str(best_card.get("note_id") or "")
|
||||
second_id = str(second_card.get("note_id") or "")
|
||||
if not best_id or best_id != second_id:
|
||||
continue
|
||||
claimed[best_key] = source_key
|
||||
selected = dict(best_card)
|
||||
selected.update({"match_method": "title", "match_score": round(best_score, 4)})
|
||||
matched[record_id] = selected
|
||||
return matched
|
||||
|
||||
|
||||
def result_status(row: dict) -> str:
|
||||
"""Read new status values and conservatively classify legacy result rows."""
|
||||
explicit = row.get("status")
|
||||
if explicit in TERMINAL_STATUSES:
|
||||
return explicit
|
||||
if row.get("write_ok") is False or row.get("write_error"):
|
||||
return WRITE_FAILURE
|
||||
if row.get("matched") is False or row.get("ok") is False or row.get("error") or row.get("reason"):
|
||||
return RETRYABLE_FAILURE
|
||||
if row.get("matched") is True or row.get("ok") is True:
|
||||
return SUCCESS
|
||||
metric_keys = ("read_count", "play_count", "view_count", "updated_value")
|
||||
if any(row.get(key) is not None for key in metric_keys):
|
||||
return SUCCESS
|
||||
return RETRYABLE_FAILURE
|
||||
|
||||
|
||||
def upsert_result(summary: dict, row: dict) -> None:
|
||||
"""Replace the previous result for one record instead of appending duplicates."""
|
||||
results = summary.setdefault("results", [])
|
||||
record_id = row.get("record_id")
|
||||
if record_id:
|
||||
for index, old in enumerate(results):
|
||||
if isinstance(old, dict) and old.get("record_id") == record_id:
|
||||
results[index] = row
|
||||
return
|
||||
results.append(row)
|
||||
|
||||
|
||||
def finalize_summary(summary: dict, *, dry_run: bool = False) -> dict:
|
||||
"""Deduplicate results, recalculate counters, and attach completeness fields."""
|
||||
out = dict(summary)
|
||||
unique: list[dict] = []
|
||||
positions: dict[str, int] = {}
|
||||
duplicate_results = 0
|
||||
for raw in out.get("results") or []:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
row = dict(raw)
|
||||
status = result_status(row)
|
||||
if status == SUCCESS and not dry_run and row.get("write_ok") is False:
|
||||
status = WRITE_FAILURE
|
||||
row["status"] = status
|
||||
if status == SUCCESS:
|
||||
row.setdefault("matched", True)
|
||||
elif status != BLOCKED_INPUT:
|
||||
row.setdefault("matched", False)
|
||||
rid = str(row.get("record_id") or "")
|
||||
if rid and rid in positions:
|
||||
unique[positions[rid]] = row
|
||||
duplicate_results += 1
|
||||
else:
|
||||
if rid:
|
||||
positions[rid] = len(unique)
|
||||
unique.append(row)
|
||||
|
||||
total = int(out.get("total") or 0)
|
||||
success = sum(result_status(row) == SUCCESS for row in unique)
|
||||
blocked = sum(result_status(row) == BLOCKED_INPUT for row in unique)
|
||||
retryable = sum(result_status(row) == RETRYABLE_FAILURE for row in unique)
|
||||
write_failures = sum(result_status(row) == WRITE_FAILURE for row in unique)
|
||||
missing = max(0, total - len(unique))
|
||||
unresolved = retryable + write_failures + missing
|
||||
top_level_error = bool(out.get("error"))
|
||||
if top_level_error:
|
||||
retryable = max(retryable, 1)
|
||||
unresolved = max(unresolved, 1)
|
||||
if dry_run:
|
||||
filled = 0
|
||||
else:
|
||||
explicit_write_rows = [row for row in unique if result_status(row) == SUCCESS and "write_ok" in row]
|
||||
if explicit_write_rows:
|
||||
filled = sum(row.get("write_ok") is True for row in explicit_write_rows)
|
||||
# Preserve legacy successes that predate per-row write status.
|
||||
filled += sum(
|
||||
1 for row in unique
|
||||
if result_status(row) == SUCCESS and "write_ok" not in row
|
||||
)
|
||||
else:
|
||||
filled = min(int(out.get("filled") or 0), success)
|
||||
|
||||
out.update({
|
||||
"results": unique,
|
||||
"matched": success + write_failures,
|
||||
"filled": filled,
|
||||
"success": success,
|
||||
"blocked_input": blocked,
|
||||
"retryable_failures": retryable,
|
||||
"write_failures": write_failures,
|
||||
"missing_results": missing,
|
||||
"duplicate_results": duplicate_results,
|
||||
"unresolved": unresolved,
|
||||
"complete": (
|
||||
not top_level_error
|
||||
and unresolved == 0
|
||||
and duplicate_results == 0
|
||||
and len(unique) == total
|
||||
),
|
||||
"dry_run": bool(dry_run),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def merge_style_summary(existing: dict | None, partial: dict, *, dry_run: bool = False) -> dict:
|
||||
"""Merge a record-limited retry into the canonical per-style summary."""
|
||||
if not existing:
|
||||
return finalize_summary(partial, dry_run=dry_run)
|
||||
combined = dict(existing)
|
||||
for key, value in partial.items():
|
||||
if key != "results":
|
||||
combined[key] = value
|
||||
rows: list[dict] = []
|
||||
positions: dict[str, int] = {}
|
||||
for source in (existing.get("results") or [], partial.get("results") or []):
|
||||
for raw in source:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
row = dict(raw)
|
||||
rid = str(row.get("record_id") or "")
|
||||
if rid and rid in positions:
|
||||
rows[positions[rid]] = row
|
||||
else:
|
||||
if rid:
|
||||
positions[rid] = len(rows)
|
||||
rows.append(row)
|
||||
combined["results"] = rows
|
||||
combined["total"] = max(
|
||||
int(existing.get("total") or 0),
|
||||
int(partial.get("total") or 0),
|
||||
len(rows),
|
||||
)
|
||||
return finalize_summary(combined, dry_run=dry_run)
|
||||
|
||||
|
||||
def merge_global_summaries(existing: Any, updates: Iterable[dict]) -> list[dict]:
|
||||
"""Merge selected style summaries into an existing all-style summary by index."""
|
||||
merged: dict[Any, dict] = {}
|
||||
order: list[Any] = []
|
||||
for row in existing if isinstance(existing, list) else []:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
key = row.get("index", row.get("style"))
|
||||
if key not in merged:
|
||||
order.append(key)
|
||||
merged[key] = row
|
||||
for row in updates:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
key = row.get("index", row.get("style"))
|
||||
if key not in merged:
|
||||
order.append(key)
|
||||
merged[key] = row
|
||||
return [merged[key] for key in sorted(order, key=lambda value: (not isinstance(value, int), value))]
|
||||
|
||||
|
||||
def requested_records_succeeded(summary: dict, record_ids: Iterable[str]) -> tuple[bool, list[str]]:
|
||||
"""Validate that every requested record exists and has a successful terminal state."""
|
||||
by_id = {
|
||||
str(row.get("record_id")): row
|
||||
for row in (summary.get("results") or summary.get("details") or [])
|
||||
if isinstance(row, dict) and row.get("record_id")
|
||||
}
|
||||
failed = [str(rid) for rid in record_ids if result_status(by_id.get(str(rid), {})) != SUCCESS]
|
||||
return not failed, failed
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Shared helpers for grouping cross-style scrape tasks by creator."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict, defaultdict
|
||||
|
||||
|
||||
def _clean_creator_id(value) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def _clean_creator_name(value) -> str:
|
||||
return "".join(str(value or "").split()).casefold()
|
||||
|
||||
|
||||
def group_tasks_by_creator(tasks: list[dict]) -> list[dict]:
|
||||
"""Group tasks globally, preferring creator ID and falling back to name.
|
||||
|
||||
A name-only task joins an ID group when that normalized name maps to exactly
|
||||
one ID in the current run. Conflicting IDs remain separate.
|
||||
"""
|
||||
ids_by_name: dict[str, set[str]] = defaultdict(set)
|
||||
for task in tasks:
|
||||
creator_id = _clean_creator_id(task.get("creator_id"))
|
||||
creator_name = _clean_creator_name(task.get("creator_name"))
|
||||
if creator_id and creator_name:
|
||||
ids_by_name[creator_name].add(creator_id)
|
||||
|
||||
groups: OrderedDict[tuple[str, str], dict] = OrderedDict()
|
||||
for task in tasks:
|
||||
creator_id = _clean_creator_id(task.get("creator_id"))
|
||||
creator_name_key = _clean_creator_name(task.get("creator_name"))
|
||||
if not creator_id and len(ids_by_name.get(creator_name_key, ())) == 1:
|
||||
creator_id = next(iter(ids_by_name[creator_name_key]))
|
||||
key = ("id", creator_id) if creator_id else ("name", creator_name_key)
|
||||
if key not in groups:
|
||||
groups[key] = {
|
||||
"creator_name": str(task.get("creator_name") or "").strip(),
|
||||
"creator_id": creator_id or None,
|
||||
"tasks": [],
|
||||
}
|
||||
groups[key]["tasks"].append(task)
|
||||
return list(groups.values())
|
||||
|
||||
|
||||
def chunk_creator_groups(groups: list[dict], batch_size: int) -> list[list[dict]]:
|
||||
"""Split creator groups into batches; non-positive means unlimited."""
|
||||
if not groups:
|
||||
return []
|
||||
if batch_size <= 0:
|
||||
return [groups]
|
||||
return [groups[i:i + batch_size] for i in range(0, len(groups), batch_size)]
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,8 @@
|
||||
# 分析脚本配置模板
|
||||
# 复制为 analyze.env 并填真实值 (analyze.env 已被 .gitignore 排除)
|
||||
|
||||
# Hermes 分析端 API Token
|
||||
HERMES_ANALYZER_TOKEN=your_token_here
|
||||
|
||||
# 飞书接收人 open_id (不设则必须传 --no-send)
|
||||
FEISHU_TARGET_OPEN_ID=ou_xxxxxxxxxxxxxxxxx
|
||||
@@ -0,0 +1,8 @@
|
||||
# PG 连接信息模板
|
||||
# 复制为 db.env 并填真实值 (db.env 已被 .gitignore 排除)
|
||||
|
||||
PG_HOST=8.148.185.119
|
||||
PG_PORT=5432
|
||||
PG_DB=data_hub
|
||||
PG_USER=data_hub
|
||||
PG_PASSWORD=${GYXX_PG_PASSWORD}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,863 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""评论采集数据 → 本地关键词过滤 + Hermes 舆情分析 → 项目 data/summary 汇总 + 飞书通知。
|
||||
|
||||
使用本地 Hermes 分析端：
|
||||
- URL: http://127.0.0.1:8642/v1（~/.hermes/profiles/data-analyzer/.env 里的 API_SERVER_PORT）
|
||||
- Model: mimo-v2.5-pro
|
||||
- Profile: data-analyzer
|
||||
|
||||
流程：
|
||||
1. 读取项目采集汇总的评论文件（默认取 data/summary/ 下最新 CSV，支持 txt/json/csv）。
|
||||
2. 用本地 Python 关键词规则（飞书文档 6 层体系）过滤评论。
|
||||
3. 用 Hermes 分析端 + 内置舆情分析提示词生成汇总报告。
|
||||
4. 将汇总写入 data/summary/评论舆论分析汇总.txt。
|
||||
5. 通过 lark-cli --profile hermes-analyzer 发送汇总给目标 open_id。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
from gyxx_flow.adapters import RuntimeServicePolicy
|
||||
import random
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import (
|
||||
PATHS,
|
||||
resolve_layer_output,
|
||||
)
|
||||
|
||||
# Path setup
|
||||
_TOOLS_DIR = PATHS.tools_root
|
||||
_PROJECT_ROOT = PATHS.module_root
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
_analyze_env = PATHS.state_root / "config/analyze.env"
|
||||
if _analyze_env.exists():
|
||||
load_dotenv(_analyze_env, override=False)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
PROJECT_ROOT = _PROJECT_ROOT
|
||||
DATA_DIR = PATHS.raw_root
|
||||
SUMMARY_DIR = PATHS.exports_root / "summary"
|
||||
|
||||
|
||||
def _latest_summary_csv() -> Path:
|
||||
"""自动选取 data/summary/ 下最新的汇总 CSV（项目采集输出）。"""
|
||||
if not SUMMARY_DIR.exists():
|
||||
return SUMMARY_DIR / "*.csv"
|
||||
csv_files = sorted(SUMMARY_DIR.glob("*.csv"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
return csv_files[0] if csv_files else SUMMARY_DIR / "*.csv"
|
||||
|
||||
|
||||
DEFAULT_INPUT = _latest_summary_csv()
|
||||
OUTPUT_PATH = SUMMARY_DIR / "评论舆论分析汇总.txt"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hermes 分析端配置
|
||||
# 实际 API Server 端口见 ~/.hermes/profiles/data-analyzer/.env 中的 API_SERVER_PORT（当前为 8642）
|
||||
# ---------------------------------------------------------------------------
|
||||
os.environ.update(RuntimeServicePolicy().apply(os.environ))
|
||||
HERMES_ANALYZER_URL = os.environ["HERMES_ANALYZER_URL"]
|
||||
HERMES_ANALYZER_TOKEN = os.getenv("HERMES_ANALYZER_TOKEN", "")
|
||||
HERMES_ANALYZER_MODEL = os.getenv("HERMES_ANALYZER_MODEL", "mimo-v2.5-pro")
|
||||
|
||||
TARGET_OPEN_ID = os.getenv("FEISHU_TARGET_OPEN_ID", "")
|
||||
|
||||
# 飞书长消息截断阈值（留有余量）
|
||||
FEISHU_TEXT_LIMIT = 6000
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 本地代码过滤关键词（飞书文档《评论区关键词抓取》6 层体系）
|
||||
# ---------------------------------------------------------------------------
|
||||
# 流程：全量采集评论 → 本地关键词过滤 → 对有效评论进行舆情分析汇总
|
||||
CODE_FILTER_KEYWORDS = {
|
||||
# 第1层：购买意向信号（衡量种草效率）
|
||||
"purchase_intent": [
|
||||
# 直接购买
|
||||
"下单", "已买", "入手", "冲了", "付款", "等收货",
|
||||
# 搜索意向
|
||||
"怎么买", "链接", "哪里买", "搜什么", "淘宝", "店铺",
|
||||
# 价格评估
|
||||
"多少钱", "价格", "划算", "贵不贵", "优惠", "值得买",
|
||||
# 犹豫对比（用双字词避免单字"比"误匹配[比心]等表情）
|
||||
"纠结", "哪个好", "容量够吗", "犹豫", "对比", "比较", "和哪个",
|
||||
],
|
||||
# 第2层：产品功能反馈（正向）
|
||||
"positive_feature": [
|
||||
# 轻量化（"轻"用双字避免误匹配"年轻人"等）
|
||||
"很轻", "超轻", "挺轻", "不重", "没感觉", "毫无负担", "背着不累",
|
||||
# 收纳秩序
|
||||
"分区", "隔层", "收纳", "能装", "各就各位",
|
||||
# 快取便利（"快"用双字避免误匹配"快乐"等）
|
||||
"很方便", "顺手", "很快", "单手", "侧袋", "不用翻",
|
||||
# 品质感
|
||||
"质感", "材质", "面料", "五金", "走线", "高级",
|
||||
],
|
||||
# 第3层：产品功能反馈（负向）
|
||||
"negative_feature": [
|
||||
# 重量负担（"重"用双字避免误匹配"重要"等）
|
||||
"很重", "太重", "超重", "偏重", "背不动", "肩膀疼",
|
||||
# 收纳不足
|
||||
"不够装", "太小", "放不下", "没地方", "容量",
|
||||
# 品质问题
|
||||
"线头", "脱线", "划痕", "掉色", "起皱", "做工",
|
||||
# 使用不便（"卡""紧"用双字避免误匹配）
|
||||
"不好拿", "卡住", "太紧", "难开", "不方便",
|
||||
],
|
||||
# 第4层：用户场景（判断内容与目标用户是否匹配）
|
||||
"user_scenario": [
|
||||
# 通勤
|
||||
"上班", "地铁", "挤公交", "通勤", "工位", "办公室",
|
||||
# 旅行/周末
|
||||
"旅行", "出差", "周末", "短途", "出游", "打包",
|
||||
# 社交
|
||||
"约会", "聚会", "探店", "咖啡馆", "拍照",
|
||||
# 户外
|
||||
"徒步", "爬山", "露营", "户外",
|
||||
],
|
||||
# 第5层：用户审美与风格认同
|
||||
"aesthetic": [
|
||||
# 颜值设计
|
||||
"好看", "颜值", "设计", "高级", "喜欢这个",
|
||||
# 百搭搭配（"配"用双字避免误匹配）
|
||||
"百搭", "好搭", "搭衣服", "搭配", "风格",
|
||||
# 子风格认同（"潮"用双字避免误匹配）
|
||||
"机能", "潮流", "潮牌", "科技感", "克制", "简约", "秩序",
|
||||
],
|
||||
# 第6层：品牌情感（长期心智指标）
|
||||
"brand_emotion": [
|
||||
# 推荐意愿
|
||||
"推荐", "安利", "值得", "满意", "惊喜",
|
||||
# 品牌认同
|
||||
"光影行星", "关注", "忠实", "一直用", "老用户",
|
||||
# 复购意向
|
||||
"再买", "第二个", "送人", "给家人", "还会",
|
||||
],
|
||||
}
|
||||
|
||||
# 包包品类通用词：用于判断评论是否与"我方产品"相关。
|
||||
# 除了品牌名/款式名，评论中出现这些词也视为与我方产品相关。
|
||||
BAG_CATEGORY_TERPS = [
|
||||
# 包类通用
|
||||
"包", "背包", "双肩包", "单肩包", "斜挎包", "手提包", "托特包",
|
||||
"邮差包", "胸包", "腰包", "手包", "拎包",
|
||||
# 容量/装载
|
||||
"能装", "装得下", "容量", "放电脑", "放平板", "放手机",
|
||||
"装东西", "装不下", "够装", "收纳",
|
||||
# 背负/功能
|
||||
"背着", "背过", "背起来", "背负", "肩带", "减负",
|
||||
"防泼水", "防水", "防震", "减震",
|
||||
# 材质/做工
|
||||
"面料", "材质", "走线", "五金", "拉链", "做工",
|
||||
# 通勤/出行场景中的包
|
||||
"通勤包", "出差包", "旅行包", "电脑包", "书包",
|
||||
"机车包", "头盔包", "健身包", "运动包",
|
||||
]
|
||||
|
||||
# 纯寒暄/无信息词：只含这些词（+标点）且未命中任何关键词的评论丢弃
|
||||
# 注意：不要把文档中的有效关键词（如"好看"）放进来，否则会被误丢
|
||||
CODE_FILTER_NOISE_WORDS = {
|
||||
"好", "棒", "支持", "来了", "赞", "厉害", "不错", "优秀",
|
||||
"喜欢", "可爱", "666", "牛", "太强了", "真棒", "很好",
|
||||
}
|
||||
|
||||
# 判断纯表情/无意义评论：只要包含中文、英文或数字就不是纯表情
|
||||
_HAS_MEANINGFUL_CHAR_RE = re.compile(r"[\u4e00-\u9fa5a-zA-Z0-9]")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 内置提示词（已硬编码，运行时不读取 md 文件）
|
||||
# ---------------------------------------------------------------------------
|
||||
ANALYSIS_SYSTEM_PROMPT = """# 角色：小红书评论舆情分析专家
|
||||
|
||||
你是一位专注小红书生态5年的舆情分析师，擅长将非结构化评论转化为结构化洞察，并给出可落地的口碑管理策略。
|
||||
|
||||
## 核心能力
|
||||
|
||||
1. **情绪分层**：正向（赞美/推荐/回购/共鸣）、负向（质量/服务/价格/竞品贬低）、中性（咨询/建议/客观描述）；识别情绪烈度（强烈推荐 vs 还行 vs 非常失望）。
|
||||
2. **主题聚类**：产品（材质/容量/颜色/重量/五金）、服务（发货/包装/客服/售后）、内容（种草来源/预期落差）、场景（通勤/旅行/约会/健身）、竞品（对比品牌/话术）。
|
||||
3. **用户画像**：推断年龄（学生党/上班族/宝妈）、消费力（价格敏感型 vs 直接下单型）、使用场景、审美偏好；分类用户类型（理性测评/感性颜值/价格敏感）。
|
||||
4. **风险预警**：≥3人提及同问题 → 品控/服务漏洞；出现"平替""不如XX" → 定位模糊；出现"退货""过敏""坏了""假货" → 严重投诉。分级：P0(安全/假货/严重质量)、P1(批量负面但可控)、P2(偶发吐槽)。
|
||||
5. **竞品对比**：正/负面率对比、被夸/被骂维度对比、差异化口碑优势与薄弱环节。
|
||||
6. **营销归因**：评论热点与近期投放/促销/新品动作联动，给出口碑效果归因图。
|
||||
7. **优化建议**：产品端（高频吐槽改进）、内容端（详情页/笔记关键信息优化）、客服端（FAQ/话术优化）、运营端（种草卖点提炼/补救内容设计）。
|
||||
8. **报告输出**：标准化舆情周报/月报（数据概览+主题分析+画像+预警+竞品+建议）。
|
||||
|
||||
## 工作原则
|
||||
|
||||
- 数据说话：每条结论附数据支撑（如"63%负评集中在尺寸偏小"）
|
||||
- 客观中立：不过度美化也不过度恐慌
|
||||
- 行动导向：给出"下周一就能做的3件事"
|
||||
- 长线思维：关注长期品牌口碑健康指标
|
||||
- 用户视角：读出用户"没说出来但藏在语气里的情绪"
|
||||
|
||||
## 输出格式
|
||||
|
||||
收到评论数据后，按以下结构输出：
|
||||
|
||||
### 一、数据概览
|
||||
- 评论总数 / 有效评论数
|
||||
- 情绪分布：正面X% / 中性X% / 负面X%
|
||||
- 互动概览：平均点赞/回复数、高互动Top3
|
||||
|
||||
### 二、核心发现（Top 3）
|
||||
每条附数据支撑
|
||||
|
||||
### 三、分维度深度分析
|
||||
| 排名 | 主题标签 | 提及次数 | 正向占比 | 负向占比 | 典型评论 |
|
||||
|------|----------|----------|----------|----------|----------|
|
||||
| 1 | 材质 | 45 | 80% | 10% | "皮料很软" |
|
||||
|
||||
- **情绪趋势**：正/负/中性随时间波动，是否有拐点
|
||||
- **用户画像**：高频身份、核心场景、主要决策因素
|
||||
- **风险预警**：P0/P1/P2 分级清单
|
||||
|
||||
### 四、归因分析
|
||||
正向评论高峰由哪些营销动作带来？负面集中反馈由哪些动作引发？
|
||||
|
||||
### 五、优化建议（按优先级）
|
||||
- **P0（立即）**：问题 → 行动 → 责任方
|
||||
- **P1（本周）**：问题 → 行动 → 责任方
|
||||
- **P2（下月）**：问题 → 行动 → 责任方
|
||||
|
||||
### 六、可复用资产
|
||||
- 评论分类标签体系
|
||||
- 情绪打分规则
|
||||
- 舆情周报模板
|
||||
|
||||
## 约束
|
||||
|
||||
- 数据量<50条时注明"样本量较小，结论仅供参考"
|
||||
- 涉及隐私的评论需脱敏处理
|
||||
- 所有建议必须标注责任方（产品/内容/客服/运营）"""
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {message}", flush=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 文件读取
|
||||
# ---------------------------------------------------------------------------
|
||||
def load_text(path: Path) -> str:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"找不到文件: {path}")
|
||||
for enc in ("utf-8-sig", "utf-8", "gbk"):
|
||||
try:
|
||||
return path.read_text(encoding=enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
def _extract_comment_text(item: Any) -> str:
|
||||
if isinstance(item, str):
|
||||
return item.strip()
|
||||
if isinstance(item, dict):
|
||||
for key in ("comment_text", "content", "text", "message", "comment"):
|
||||
val = item.get(key)
|
||||
if val is not None:
|
||||
return str(val).strip()
|
||||
return str(item).strip()
|
||||
|
||||
|
||||
def load_comments(path: Path) -> list[str]:
|
||||
"""从 txt/json/csv 加载评论文本列表。"""
|
||||
text = load_text(path)
|
||||
ext = path.suffix.lower()
|
||||
|
||||
if ext == ".json":
|
||||
data = json.loads(text)
|
||||
if isinstance(data, list):
|
||||
return [_extract_comment_text(item) for item in data if _extract_comment_text(item)]
|
||||
if isinstance(data, dict):
|
||||
for key in ("comments", "data", "list", "rows"):
|
||||
if key in data and isinstance(data[key], list):
|
||||
return [_extract_comment_text(item) for item in data[key] if _extract_comment_text(item)]
|
||||
raise ValueError(f"不支持的 JSON 结构: {path}")
|
||||
|
||||
if ext == ".csv":
|
||||
rows = list(csv.DictReader(text.splitlines()))
|
||||
candidates = ("comment_text", "content", "text", "message", "评论内容", "评论", "comment")
|
||||
comments: list[str] = []
|
||||
for row in rows:
|
||||
for key in candidates:
|
||||
val = row.get(key)
|
||||
if val is not None and str(val).strip():
|
||||
comments.append(str(val).strip())
|
||||
break
|
||||
return comments
|
||||
|
||||
# 默认：每行一条评论
|
||||
return [line.strip() for line in text.splitlines() if line.strip()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 按款式分批处理（用于项目汇总 CSV）
|
||||
# ---------------------------------------------------------------------------
|
||||
def load_style_groups(path: Path, max_per_style: int | None = None) -> dict[str, list[str]]:
|
||||
"""从 CSV 中按款式分组读取评论，可选每款式抽样上限。"""
|
||||
text = load_text(path)
|
||||
rows = list(csv.DictReader(text.splitlines()))
|
||||
|
||||
candidates = ("comment_text", "content", "text", "message", "评论内容", "评论", "comment")
|
||||
style_key_candidates = ("style_name", "款式", "style", "产品")
|
||||
|
||||
style_col: str | None = None
|
||||
for row in rows[:1]:
|
||||
for key in style_key_candidates:
|
||||
if key in row:
|
||||
style_col = key
|
||||
break
|
||||
|
||||
groups: dict[str, list[str]] = {}
|
||||
for row in rows:
|
||||
style = row.get(style_col, "未知款式") if style_col else "未知款式"
|
||||
style = style.strip() or "未知款式"
|
||||
for key in candidates:
|
||||
val = row.get(key)
|
||||
if val is not None and str(val).strip():
|
||||
groups.setdefault(style, []).append(str(val).strip())
|
||||
break
|
||||
|
||||
if max_per_style:
|
||||
rng = random.Random(42)
|
||||
for style in groups:
|
||||
if len(groups[style]) > max_per_style:
|
||||
groups[style] = rng.sample(groups[style], max_per_style)
|
||||
|
||||
return dict(sorted(groups.items(), key=lambda item: -len(item[1])))
|
||||
|
||||
|
||||
def analyze_style(style: str, comments: list[str]) -> tuple[str, int, int]:
|
||||
"""对单个款式的评论进行过滤 + 分析，返回（汇总文本, 原始数, 过滤后数）。"""
|
||||
log(f"【{style}】开始处理（{len(comments)} 条）...")
|
||||
filtered = code_filter_comments(comments)
|
||||
summary = analyze_comments(filtered)
|
||||
return summary, len(comments), len(filtered)
|
||||
|
||||
|
||||
def build_batch_report(style_results: list[tuple[str, str, int, int]]) -> str:
|
||||
"""把所有款式汇总合并成一份总报告。"""
|
||||
total_raw = sum(raw for _, _, raw, _ in style_results)
|
||||
total_filtered = sum(filt for _, _, _, filt in style_results)
|
||||
|
||||
lines = [
|
||||
"# 多款式评论舆情汇总报告",
|
||||
"",
|
||||
f"- 款式总数：{len(style_results)}",
|
||||
f"- 评论总数（原始）：{total_raw}",
|
||||
f"- 有效评论总数（过滤后）：{total_filtered}",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
for style, summary, raw, filtered in style_results:
|
||||
lines.append(f"## 款式：{style}")
|
||||
lines.append(f"原始评论：{raw} 条 | 过滤后：{filtered} 条")
|
||||
lines.append("")
|
||||
lines.append(summary)
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hermes 分析端调用
|
||||
# ---------------------------------------------------------------------------
|
||||
def call_hermes_analyzer(system_prompt: str, user_content: str) -> str:
|
||||
url = HERMES_ANALYZER_URL.rstrip("/") + "/chat/completions"
|
||||
payload = {
|
||||
"model": HERMES_ANALYZER_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_content},
|
||||
],
|
||||
}
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
headers={
|
||||
"Authorization": f"Bearer {HERMES_ANALYZER_TOKEN}",
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=1800) as response:
|
||||
result = json.loads(response.read().decode("utf-8", errors="replace"))
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(f"Hermes HTTP {exc.code}: {body}") from exc
|
||||
|
||||
if "choices" not in result or not result["choices"]:
|
||||
raise RuntimeError(f"Hermes 返回异常: {result}")
|
||||
return str(result["choices"][0]["message"]["content"])
|
||||
|
||||
|
||||
def _is_pure_emoji(text: str) -> bool:
|
||||
"""判断是否为纯表情/无意义评论（不含中文、英文、数字）。"""
|
||||
return not _HAS_MEANINGFUL_CHAR_RE.search(text)
|
||||
|
||||
|
||||
# 抖音/B站表情格式：[比心] [舔屏] 等，匹配前需去掉避免误触发关键词
|
||||
_EMOJI_BRACKET_RE = re.compile(r'\[[^\[\]]{1,10}\]')
|
||||
|
||||
|
||||
def _strip_emoji_brackets(text: str) -> str:
|
||||
"""去掉 [xxx] 格式的表情符号，避免表情文本误触发关键词匹配。"""
|
||||
return _EMOJI_BRACKET_RE.sub('', text).strip()
|
||||
|
||||
|
||||
def _matches_code_keywords(text: str) -> bool:
|
||||
"""本地关键词匹配，命中任意有效关键词即保留。"""
|
||||
clean = _strip_emoji_brackets(text)
|
||||
if not clean:
|
||||
return False
|
||||
for keywords in CODE_FILTER_KEYWORDS.values():
|
||||
for kw in keywords:
|
||||
if kw in clean:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def classify_comment_keywords(text: str) -> list[str]:
|
||||
"""返回一条评论命中的所有关键词类别（用于统计分布）。"""
|
||||
clean = _strip_emoji_brackets(text)
|
||||
if not clean:
|
||||
return []
|
||||
categories: list[str] = []
|
||||
for category, keywords in CODE_FILTER_KEYWORDS.items():
|
||||
for kw in keywords:
|
||||
if kw in clean:
|
||||
categories.append(category)
|
||||
break
|
||||
return categories
|
||||
|
||||
|
||||
def _is_noise_only(text: str) -> bool:
|
||||
"""判断是否为纯寒暄/无信息评论（仅由常见寒暄词和标点组成）。"""
|
||||
# 去掉常见寒暄词
|
||||
cleaned = text
|
||||
for w in sorted(CODE_FILTER_NOISE_WORDS, key=len, reverse=True):
|
||||
cleaned = cleaned.replace(w, "")
|
||||
# 再去掉常见标点、空格、数字
|
||||
cleaned = re.sub(r"[\s，,。！!？?~～.·\d]+", "", cleaned).strip()
|
||||
return not cleaned
|
||||
|
||||
|
||||
def code_filter_comments(comments: list[str]) -> list[str]:
|
||||
"""用本地代码规则快速过滤评论，不调用大模型。"""
|
||||
if not comments:
|
||||
log("没有评论需要过滤")
|
||||
return []
|
||||
|
||||
log(f"使用本地代码规则进行评论过滤（共 {len(comments)} 条）...")
|
||||
seen: set[str] = set()
|
||||
kept: list[str] = []
|
||||
for text in comments:
|
||||
t = text.strip()
|
||||
if not t or t in seen:
|
||||
continue
|
||||
if _is_pure_emoji(t):
|
||||
continue
|
||||
if _matches_code_keywords(t):
|
||||
seen.add(t)
|
||||
kept.append(t)
|
||||
continue
|
||||
# 未命中关键词，但属于短句纯寒暄，也丢弃
|
||||
if len(t) <= 12 and _is_noise_only(t):
|
||||
continue
|
||||
# 其余未命中关键词的评论丢弃
|
||||
log(f"代码过滤完成：{len(comments)} 条 → {len(kept)} 条")
|
||||
return kept
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM 评论清洗(语义过滤,替代/补充关键词过滤)
|
||||
# ---------------------------------------------------------------------------
|
||||
COMMENT_CLEAN_SYSTEM_PROMPT = """你是"光影行星"品牌的评论清洗助手。光影行星主要经营双肩包、单肩包、斜挎包、摄影包、电脑包、手提包及相关包袋配件。你的任务是过滤无效评论，并从评论中提取对品牌经营、产品优化或用户服务有价值的内容。
|
||||
|
||||
有效评论包括：
|
||||
1. 产品评价：涉及外观、颜色、尺寸、容量、重量、材质、做工、气味、拉链、肩带、扣具、分区、收纳、防水性、耐用性、舒适度、便携性等。
|
||||
2. 使用体验：涉及通勤、旅行、摄影、户外、上学、商务、电脑收纳、相机收纳等真实使用场景。
|
||||
3. 产品咨询：询问尺寸、颜色、容量、适配设备、材质、库存、价格、使用方法、清洁保养或不同款式区别。
|
||||
4. 购买意向：明确表达想买、准备下单、等待补货、询问购买链接或推荐款式。
|
||||
5. 产品问题与售后反馈：涉及破损、开线、掉色、拉链故障、尺寸不符、配件缺失、使用异常、退换货原因等。
|
||||
6. 品牌反馈：明确提到光影行星，对品牌设计、质量、价格、服务或产品系列提出评价和建议。
|
||||
|
||||
无效评论包括：
|
||||
1. 只有表情、数字、标点或无意义字符。
|
||||
2. "哈哈哈""路过""支持一下""不错不错"等没有具体信息的泛泛表达。
|
||||
3. 广告、引流、联系方式、刷单、抽奖、互关或推广其他商品。
|
||||
4. 与光影行星或包袋产品完全无关的内容。
|
||||
5. 明显复制粘贴、重复发送或机器生成的灌水内容。
|
||||
6. 单纯辱骂、攻击他人且没有任何产品事实或具体问题。
|
||||
7. 无法理解、语义残缺且不能确定评论意图的内容。
|
||||
8. 只讨论主播、视频背景、音乐或其他无关内容。
|
||||
9. 仅询问快递到了哪里，且不包含产品或售后问题。
|
||||
10. 没有真实内容的默认好评或模板评价。
|
||||
|
||||
判定原则：
|
||||
1. 负面评论只要包含具体产品问题，就属于有效评论，不能因为评价负面而删除。
|
||||
2. 评论同时包含有效和无效内容时，只提取其中与光影行星及包袋产品有关的有效部分。
|
||||
3. 不得修改评论原意，不得虚构用户没有提到的信息。
|
||||
4. 可以删除表情、重复语句、无意义语气词和无关内容。
|
||||
5. "很好""不错""喜欢"等评论，如果没有说明具体原因，判定为无效。
|
||||
6. "容量很大""肩带舒服""拉链不好用"等虽然简短，但包含具体产品信息，应判定为有效。
|
||||
7. 无法确定时，优先保留，但标记为"需要人工确认"。
|
||||
8. 所有"有效"类别(产品评价/使用体验/产品咨询/购买意向/售后问题/品牌反馈)都必须限定在"光影行星品牌"或"包袋类产品"范围内。如果评论只询问/讨论笔记里出现的其他物品(如镜子、指甲刀、U型枕、蚊虫膏、行李箱、自拍支架、护照收纳袋、登山杖等非包袋物品),即使有购买意向或产品咨询,也判定为无效。
|
||||
9. 评论同时提到包袋和其他物品时,只看包袋相关部分:若包袋部分有具体信息(评价/咨询/购买意向等),按有效处理;若包袋部分只是被顺带提及或无具体信息,判定为无效。
|
||||
|
||||
每条评论按照以下格式输出（不要输出其它内容）：
|
||||
评论编号：<编号>
|
||||
是否有效：有效/无效/需要人工确认
|
||||
有效评论：<提取并清洗后的有效内容；无有效内容则填写"无">
|
||||
评论类型：产品评价/使用体验/产品咨询/购买意向/售后问题/品牌反馈/无效内容
|
||||
情感倾向：正面/中性/负面/无法判断
|
||||
涉及维度：外观、颜色、尺寸、容量、材质、做工、舒适度、收纳、防水、耐用、价格、配件、售后、其他
|
||||
无效原因：<有效评论填写"无"；无效评论说明具体原因>"""
|
||||
|
||||
|
||||
_CLEAN_FIELD_RE = re.compile(
|
||||
r"(是否有效|有效评论|评论类型|情感倾向|涉及维度|无效原因)\s*[：:]\s*(.*)"
|
||||
)
|
||||
|
||||
|
||||
def parse_cleaning_response(response: str) -> list[dict]:
|
||||
"""解析 LLM 结构化输出。每条评论 7 行,按"评论编号："分块。
|
||||
返回 [{valid, cleaned, type, sentiment, dimensions, reason}, ...]
|
||||
"""
|
||||
items: list[dict] = []
|
||||
# 按"评论编号：<数字>"分块,跳过首块(分割后的空块)
|
||||
blocks = re.split(r"评论编号\s*[：:]\s*\d+", response)
|
||||
for block in blocks[1:]:
|
||||
item = {
|
||||
"valid": "无效",
|
||||
"cleaned": "",
|
||||
"type": "无效内容",
|
||||
"sentiment": "无法判断",
|
||||
"dimensions": "其他",
|
||||
"reason": "无",
|
||||
}
|
||||
for line in block.split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
m = _CLEAN_FIELD_RE.match(line)
|
||||
if not m:
|
||||
continue
|
||||
key, val = m.group(1), m.group(2).strip()
|
||||
if key == "是否有效":
|
||||
item["valid"] = val
|
||||
elif key == "有效评论":
|
||||
if val and val != "无":
|
||||
item["cleaned"] = val
|
||||
elif key == "评论类型":
|
||||
item["type"] = val
|
||||
elif key == "情感倾向":
|
||||
item["sentiment"] = val
|
||||
elif key == "涉及维度":
|
||||
item["dimensions"] = val
|
||||
elif key == "无效原因":
|
||||
item["reason"] = val
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def _trivial_prefilter(comments: list[str]) -> list[str]:
|
||||
"""轻量预过滤:只去掉空、纯表情、重复,其余全留(给 LLM 判断)。"""
|
||||
seen: set[str] = set()
|
||||
kept: list[str] = []
|
||||
for text in comments:
|
||||
t = str(text or "").strip()
|
||||
if not t or t in seen:
|
||||
continue
|
||||
if _is_pure_emoji(t):
|
||||
continue
|
||||
seen.add(t)
|
||||
kept.append(t)
|
||||
return kept
|
||||
|
||||
|
||||
def _clean_one_batch(batch: list[str], batch_idx: int,
|
||||
total_batches: int) -> tuple[int, list[dict]]:
|
||||
"""清洗一批评论,返回 (batch_idx, results)。
|
||||
并发安全:不写共享状态,只返回结果。
|
||||
"""
|
||||
lines = [f"评论编号：{i+1}\n评论内容：{text}" for i, text in enumerate(batch, 1)]
|
||||
user_content = (f"请按规则清洗以下 {len(batch)} 条评论,逐条输出结构化结果。\n\n"
|
||||
+ "\n\n".join(lines))
|
||||
results: list[dict] = []
|
||||
try:
|
||||
response = call_hermes_analyzer(COMMENT_CLEAN_SYSTEM_PROMPT, user_content)
|
||||
parsed = parse_cleaning_response(response)
|
||||
for i, item in enumerate(parsed):
|
||||
if i < len(batch):
|
||||
item["original"] = batch[i]
|
||||
if len(parsed) < len(batch):
|
||||
for i in range(len(parsed), len(batch)):
|
||||
results.append({
|
||||
"original": batch[i],
|
||||
"valid": "需要人工确认",
|
||||
"cleaned": batch[i],
|
||||
"type": "无效内容",
|
||||
"sentiment": "无法判断",
|
||||
"dimensions": "其他",
|
||||
"reason": "LLM 响应未解析到该条",
|
||||
})
|
||||
results.extend(parsed)
|
||||
except Exception as exc:
|
||||
for text in batch:
|
||||
results.append({
|
||||
"original": text,
|
||||
"valid": "需要人工确认",
|
||||
"cleaned": text,
|
||||
"type": "无效内容",
|
||||
"sentiment": "无法判断",
|
||||
"dimensions": "其他",
|
||||
"reason": f"LLM 调用失败: {exc}",
|
||||
})
|
||||
return batch_idx, results
|
||||
|
||||
|
||||
def llm_clean_comments_detailed(comments: list[str],
|
||||
batch_size: int = 30,
|
||||
max_workers: int = 6,
|
||||
include_invalid: bool = False) -> list[dict]:
|
||||
"""用 LLM 对评论做语义清洗(并发批处理)。
|
||||
返回 [{original, valid, cleaned, type, sentiment, dimensions, reason}, ...]
|
||||
默认只返回 valid != "无效" 的评论;传 include_invalid=True 返回全部。
|
||||
max_workers: 并发批数(默认 6,Hermes 本地端可承受)
|
||||
"""
|
||||
if not comments:
|
||||
return []
|
||||
prefiltered = _trivial_prefilter(comments)
|
||||
log(f"LLM 清洗:预过滤 {len(comments)} -> {len(prefiltered)} 条(去空/去重/去纯表情)")
|
||||
|
||||
total = len(prefiltered)
|
||||
total_batches = (total + batch_size - 1) // batch_size
|
||||
|
||||
# 切分批次
|
||||
batches: list[tuple[int, list[str]]] = []
|
||||
for batch_idx in range(total_batches):
|
||||
start = batch_idx * batch_size
|
||||
end = min(start + batch_size, total)
|
||||
batches.append((batch_idx, prefiltered[start:end]))
|
||||
|
||||
# 并发跑
|
||||
results_by_idx: dict[int, list[dict]] = {}
|
||||
completed = 0
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
future_to_idx = {executor.submit(_clean_one_batch, batch, idx, total_batches): idx
|
||||
for idx, batch in batches}
|
||||
for future in concurrent.futures.as_completed(future_to_idx):
|
||||
idx = future_to_idx[future]
|
||||
try:
|
||||
batch_idx, batch_results = future.result()
|
||||
results_by_idx[batch_idx] = batch_results
|
||||
except Exception as exc:
|
||||
log(f" [WARN] 批次 {idx+1} 异常: {exc}")
|
||||
results_by_idx[idx] = []
|
||||
completed += 1
|
||||
if completed % 3 == 0 or completed == total_batches:
|
||||
log(f" 进度: {completed}/{total_batches} 批完成")
|
||||
|
||||
# 按原顺序合并
|
||||
all_results: list[dict] = []
|
||||
for idx in range(total_batches):
|
||||
all_results.extend(results_by_idx.get(idx, []))
|
||||
|
||||
kept = [r for r in all_results if r.get("valid") != "无效"]
|
||||
valid_count = sum(1 for r in all_results if r.get("valid") == "有效")
|
||||
unsure_count = sum(1 for r in all_results if r.get("valid") == "需要人工确认")
|
||||
invalid_count = sum(1 for r in all_results if r.get("valid") == "无效")
|
||||
log(f"LLM 清洗完成: 共 {len(all_results)} 条 -> 有效 {valid_count} / "
|
||||
f"需人工 {unsure_count} / 无效 {invalid_count}")
|
||||
return all_results if include_invalid else kept
|
||||
|
||||
|
||||
def llm_clean_comments(comments: list[str], batch_size: int = 30,
|
||||
use_cleaned: bool = True, max_workers: int = 6) -> list[str]:
|
||||
"""LLM 清洗评论,返回清洗后的评论文本列表(只含有效+需要人工确认)。
|
||||
use_cleaned=True 返回清洗后内容,False 返回原文。"""
|
||||
details = llm_clean_comments_detailed(comments, batch_size=batch_size,
|
||||
max_workers=max_workers)
|
||||
out: list[str] = []
|
||||
for d in details:
|
||||
text = d.get("cleaned") if use_cleaned else d.get("original")
|
||||
if not text:
|
||||
text = d.get("original", "")
|
||||
if text:
|
||||
out.append(text)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 舆情分析
|
||||
# ---------------------------------------------------------------------------
|
||||
def analyze_comments(comments: list[str]) -> str:
|
||||
if not comments:
|
||||
return "无有效评论可分析。"
|
||||
|
||||
log(f"调用 Hermes 分析端生成舆情汇总（共 {len(comments)} 条）...")
|
||||
lines = [f"{i + 1}. {text}" for i, text in enumerate(comments)]
|
||||
user_content = (
|
||||
f"共有 {len(comments)} 条过滤后的有效评论，请根据角色设定输出完整的舆情分析汇总报告。\n\n"
|
||||
+ "\n".join(lines)
|
||||
)
|
||||
return call_hermes_analyzer(ANALYSIS_SYSTEM_PROMPT, user_content)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 输出 & 飞书通知
|
||||
# ---------------------------------------------------------------------------
|
||||
def write_summary(summary: str, path: Path) -> Path:
|
||||
path.write_text(summary, encoding="utf-8-sig")
|
||||
log(f"汇总报告已写入: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def send_feishu_summary(summary: str, open_id: str) -> dict[str, Any]:
|
||||
"""通过 lark-cli --profile hermes-analyzer 以分析端应用身份发送汇总。"""
|
||||
text = summary
|
||||
if len(text) > FEISHU_TEXT_LIMIT:
|
||||
text = text[:FEISHU_TEXT_LIMIT] + "\n\n...（内容过长，已截断，完整内容见项目 data/summary 汇总文件）"
|
||||
|
||||
command = [
|
||||
"lark-cli",
|
||||
"--profile", "hermes-analyzer",
|
||||
"im", "+messages-send",
|
||||
"--user-id", open_id,
|
||||
"--text", text,
|
||||
"--as", "bot",
|
||||
"--format", "json",
|
||||
]
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"lark-cli 发送失败 (exit {completed.returncode}):\n"
|
||||
f"stdout: {completed.stdout}\nstderr: {completed.stderr}"
|
||||
)
|
||||
try:
|
||||
return json.loads(completed.stdout)
|
||||
except json.JSONDecodeError:
|
||||
return {"raw": completed.stdout}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="评论 → Hermes 过滤/分析 → 桌面汇总 + 飞书通知")
|
||||
parser.add_argument("--input", type=Path, default=DEFAULT_INPUT, help="原始评论文件路径")
|
||||
parser.add_argument("--output", type=Path, default=OUTPUT_PATH, help="汇总输出文件路径")
|
||||
parser.add_argument("--open-id", default=TARGET_OPEN_ID, help="飞书接收人 open_id")
|
||||
parser.add_argument("--no-send", action="store_true", help="只生成汇总文件，不发送飞书")
|
||||
parser.add_argument("--batch-by-style", action="store_true", help="CSV 按款式分批分析（用于项目汇总 CSV）")
|
||||
parser.add_argument("--max-per-style", type=int, default=300, help="每款式最多分析多少条评论（防上下文超长）")
|
||||
args = parser.parse_args()
|
||||
args.output = resolve_layer_output(
|
||||
args.output, layer_root=PATHS.exports_root, field="--output"
|
||||
)
|
||||
|
||||
if not args.no_send:
|
||||
if not HERMES_ANALYZER_TOKEN:
|
||||
log("[ERROR] HERMES_ANALYZER_TOKEN 未设置。请配置 data/config/analyze.env 或传 --no-send")
|
||||
return 1
|
||||
if not args.open_id:
|
||||
log("[ERROR] 飞书接收人 open_id 未设置。请配置 FEISHU_TARGET_OPEN_ID 或传 --open-id 或 --no-send")
|
||||
return 1
|
||||
|
||||
if args.batch_by_style:
|
||||
log(f"按款式分批读取 CSV: {args.input}")
|
||||
try:
|
||||
style_groups = load_style_groups(args.input, args.max_per_style)
|
||||
except Exception as exc:
|
||||
log(f"读取 CSV 分组失败: {exc}")
|
||||
return 1
|
||||
log(f"共 {len(style_groups)} 个款式")
|
||||
if not style_groups:
|
||||
log("没有读取到任何评论，流程结束")
|
||||
return 0
|
||||
|
||||
style_results: list[tuple[str, str, int, int]] = []
|
||||
for style, comments in style_groups.items():
|
||||
try:
|
||||
summary, raw_count, filtered_count = analyze_style(style, comments)
|
||||
style_results.append((style, summary, raw_count, filtered_count))
|
||||
except Exception as exc:
|
||||
log(f"【{style}】处理失败: {exc}")
|
||||
style_results.append((style, f"处理失败: {exc}", len(comments), 0))
|
||||
|
||||
summary = build_batch_report(style_results)
|
||||
else:
|
||||
log(f"读取原始评论文件: {args.input}")
|
||||
try:
|
||||
comments = load_comments(args.input)
|
||||
except Exception as exc:
|
||||
log(f"读取评论失败: {exc}")
|
||||
return 1
|
||||
log(f"读取到 {len(comments)} 条评论")
|
||||
|
||||
if not comments:
|
||||
log("没有读取到任何评论，流程结束")
|
||||
return 0
|
||||
|
||||
try:
|
||||
filtered = code_filter_comments(comments)
|
||||
except Exception as exc:
|
||||
log(f"评论过滤失败: {exc}")
|
||||
return 1
|
||||
|
||||
try:
|
||||
summary = analyze_comments(filtered)
|
||||
except Exception as exc:
|
||||
log(f"舆情分析失败: {exc}")
|
||||
return 1
|
||||
|
||||
try:
|
||||
write_summary(summary, args.output)
|
||||
except Exception as exc:
|
||||
log(f"写入汇总文件失败: {exc}")
|
||||
return 1
|
||||
|
||||
if args.no_send:
|
||||
log("已指定 --no-send，跳过飞书发送")
|
||||
else:
|
||||
try:
|
||||
result = send_feishu_summary(summary, args.open_id)
|
||||
log(f"飞书消息发送成功: {result}")
|
||||
except Exception as exc:
|
||||
log(f"飞书消息发送失败（汇总文件仍已生成）: {exc}")
|
||||
return 1
|
||||
|
||||
log("全部完成")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\n已中断", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,868 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""单篇笔记分析报告生成器。
|
||||
|
||||
读取项目采集的单篇笔记 JSON（data/notes/{platform}/...json），
|
||||
结合笔记标题、点赞/收藏/评论/分享等基础指标，以及评论内容，
|
||||
生成本地代码过滤 + Hermes 舆情分析的单篇笔记分析报告。
|
||||
|
||||
输出保存到：data/summary/单篇笔记分析报告.txt
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import (
|
||||
PATHS,
|
||||
resolve_layer_output,
|
||||
)
|
||||
|
||||
# Path setup
|
||||
_TOOLS_DIR = PATHS.tools_root
|
||||
_PROJECT_ROOT = PATHS.module_root
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.analyze_comments import (
|
||||
HERMES_ANALYZER_MODEL,
|
||||
HERMES_ANALYZER_TOKEN,
|
||||
HERMES_ANALYZER_URL,
|
||||
FEISHU_TEXT_LIMIT,
|
||||
TARGET_OPEN_ID,
|
||||
BAG_CATEGORY_TERPS,
|
||||
code_filter_comments,
|
||||
llm_clean_comments,
|
||||
llm_clean_comments_detailed,
|
||||
classify_comment_keywords,
|
||||
call_hermes_analyzer,
|
||||
)
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import db
|
||||
|
||||
PROJECT_ROOT = _PROJECT_ROOT
|
||||
DATA_DIR = PATHS.raw_root
|
||||
SUMMARY_DIR = PATHS.exports_root / "summary"
|
||||
NOTES_DIR = PATHS.raw_root / "notes"
|
||||
|
||||
OUTPUT_PATH = SUMMARY_DIR / "单篇笔记分析报告.txt"
|
||||
|
||||
CATEGORY_LABELS = {
|
||||
"purchase_intent": "购买意向",
|
||||
"positive_feature": "正向功能反馈",
|
||||
"negative_feature": "负向功能反馈",
|
||||
"user_scenario": "用户场景",
|
||||
"aesthetic": "审美与风格",
|
||||
"brand_emotion": "品牌情感",
|
||||
}
|
||||
|
||||
# 单篇笔记分析：送入 LLM 的有效评论上限
|
||||
# 超过此数量时按关键词多样性抽样，保留最有信息量的评论
|
||||
MAX_COMMENTS_FOR_LLM = 80
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 单篇笔记分析的 LLM 提示词
|
||||
# ---------------------------------------------------------------------------
|
||||
SINGLE_NOTE_PROMPT = """你是一位品牌投放复盘分析师。请基于以下单篇笔记的数据与评论区内容，按指定结构输出分析报告。
|
||||
|
||||
【笔记信息】
|
||||
- 标题：{title}
|
||||
- 平台：{platform}
|
||||
- 品牌/产品：{brand} / {product}
|
||||
- 点赞：{likes} | 收藏：{favs} | 评论：{comments} | 分享：{shares}
|
||||
- 曝光/播放量：{views}（如有）
|
||||
|
||||
【有效评论列表】
|
||||
{有效评论原文，逐条列出}
|
||||
|
||||
【输出要求】
|
||||
请按以下7个模块输出结论，每个模块直接给出判断和建议，不重复提及数据来源或指标定义，引用评论用「」包裹原文：
|
||||
|
||||
模块1｜笔记表现归因
|
||||
- 基于互动率结构（评论/收藏/转发）判断这条笔记的传播属性：是认知留存型、社交裂变型还是搜索截流型？
|
||||
- 结合曝光量（如有）判断流量效率是否匹配投放预期。
|
||||
- 一句话总结：这条笔记"成也何处，败也何处"。
|
||||
|
||||
模块2｜评论质量分层
|
||||
- 将评论按"有效种草 / 连带兴趣 / 无效互动"三档归类，给出各档条数与占比。
|
||||
- 无效互动的典型表现是什么？（如纯求链接、纯@好友、无意义表情）
|
||||
- 有效评论中，高频关键词指向哪些用户决策阶段？（规格确认、价值判断、竞品对比、冲动下单、售后反馈）
|
||||
|
||||
模块3｜品牌安全与控评建议
|
||||
- 评论区是否存在负面舆情集中点？若有，具体是什么问题？影响面多大？
|
||||
- 是否出现竞品高频被提及？提及场景是"对比纠结"还是"被替代"？
|
||||
- 给出具体的控评/引导话术建议，至少2条可挂置顶或评论区回复的原文。
|
||||
|
||||
模块4｜脚本有效性拆解
|
||||
- 标题钩子是否有效？是吸引点击还是引发质疑？
|
||||
- 卖点传达是否精准？评论区讨论的卖点与笔记主推卖点是否一致？如果偏离，偏离方向是什么？
|
||||
- 用户是否按预期行动？（如搜索品牌词、进店铺、点击评论区链接等）
|
||||
- 内容调性是否与品牌心智匹配？若匹配度不足，差异点在哪？
|
||||
|
||||
模块5｜达人匹配度评估
|
||||
- 该达人的粉丝画像（性别、年龄、兴趣）与我方目标客群的重合度如何？
|
||||
- 达人的内容风格（硬核测评/温柔种草/剧情演绎/颜值展示）是否适合承载本产品的核心卖点？
|
||||
- 如果匹配度不足，是"换人"还是"改脚本"？具体建议。
|
||||
|
||||
模块6｜可沉淀的选题资产
|
||||
- 从评论区提取至少2条可作为下一期内容选题的真实用户需求或疑问。
|
||||
- 每条选题需附：内容方向+标题参考+核心脚本建议。
|
||||
|
||||
模块7｜下一篇笔记可执行优化清单
|
||||
- 脚本层面（前3秒钩子、卖点顺序、画面设计、结尾引导）逐项给出可落地的修改点。
|
||||
- 评论区运营层面（预埋评论、置顶引导、舆情应对）给出具体话术。
|
||||
- 投放策略层面（达人选择、内容方向、流量助推）给出调整建议。
|
||||
|
||||
【输出格式要求】
|
||||
- 不要出现"根据数据""数据显示""通过分析可知"等来源类表述。
|
||||
- 直接给结论和建议，每条建议须具体可执行，避免"加强""优化"等空泛词汇。
|
||||
- 如我方产品在笔记中被边缘化，须在模块4末尾明确加粗标注"⚠️ 我方产品边缘化，非核心种草"。
|
||||
- 整体篇幅控制在1500-2000字，结论前置，层次分明。"""
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {message}", flush=True)
|
||||
|
||||
|
||||
def detect_platform(source_url: str) -> str:
|
||||
url = source_url.lower()
|
||||
if "bilibili" in url or "b23.tv" in url or url.startswith("bv"):
|
||||
return "bilibili"
|
||||
if "douyin" in url:
|
||||
return "douyin"
|
||||
if "xiaohongshu" in url or "xhslink" in url:
|
||||
return "xiaohongshu"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def safe_div(numerator: Any, denominator: Any) -> float:
|
||||
try:
|
||||
n = float(numerator)
|
||||
d = float(denominator)
|
||||
if d == 0:
|
||||
return 0.0
|
||||
return n / d
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def load_note_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"找不到文件: {path}")
|
||||
for enc in ("utf-8-sig", "utf-8", "gbk"):
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding=enc))
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
except json.JSONDecodeError:
|
||||
raise
|
||||
else:
|
||||
data = json.loads(path.read_text(encoding="utf-8", errors="replace"))
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"不支持的 JSON 结构: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def extract_note_info(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""从笔记 JSON 中提取标题、URL、指标等。"""
|
||||
source_url = data.get("source_url") or data.get("final_url") or ""
|
||||
platform = detect_platform(source_url)
|
||||
|
||||
# 标题：优先顶层 title，兼容 bilibili 旧数据的 video.title
|
||||
title = data.get("title") or ""
|
||||
if not title and isinstance(data.get("video"), dict):
|
||||
title = data["video"].get("title", "")
|
||||
|
||||
stats = data.get("stats") or {}
|
||||
metrics = stats.get("note_metrics") or {}
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"platform": platform,
|
||||
"source_url": source_url,
|
||||
"scraped_at": data.get("scraped_at", ""),
|
||||
"comment_count_total": data.get("count", 0),
|
||||
"like_count": metrics.get("like_count", ""),
|
||||
"favorite_count": metrics.get("favorite_count", ""),
|
||||
"comment_count_metric": metrics.get("comment_count", ""),
|
||||
"share_count": metrics.get("share_count", ""),
|
||||
"view_count": metrics.get("view_count", ""),
|
||||
"collect_count_yxyy": metrics.get("collect_count", ""),
|
||||
"style_name": "",
|
||||
"creator_name": "",
|
||||
}
|
||||
|
||||
|
||||
def extract_comment_texts(data: dict[str, Any]) -> list[str]:
|
||||
"""从笔记 JSON 中提取评论文本列表。"""
|
||||
comments = data.get("comments") or []
|
||||
texts: list[str] = []
|
||||
for c in comments:
|
||||
if not isinstance(c, dict):
|
||||
continue
|
||||
for key in ("content", "text", "message"):
|
||||
val = c.get(key)
|
||||
if val is not None:
|
||||
text = str(val).strip()
|
||||
if text:
|
||||
texts.append(text)
|
||||
break
|
||||
return texts
|
||||
|
||||
|
||||
def load_note_from_db(note_id: int | None = None, url: str | None = None) -> tuple[dict[str, Any], list[str]]:
|
||||
"""从数据库读取笔记信息和评论文本。
|
||||
|
||||
返回 (note_info_dict, comment_texts_list)。
|
||||
只查 cmt_notes（含 cmt_styles / cmt_creators JOIN）+ cmt_comments。
|
||||
"""
|
||||
if note_id is None and not url:
|
||||
raise ValueError("必须提供 note_id 或 url")
|
||||
|
||||
if note_id is not None:
|
||||
note_row = db.get_note_by_id(note_id)
|
||||
else:
|
||||
note_row = db.get_note_by_url(url)
|
||||
|
||||
if not note_row:
|
||||
raise ValueError(f"数据库中找不到笔记: note_id={note_id}, url={url}")
|
||||
|
||||
note_id = note_row["id"]
|
||||
comment_rows = db.get_comments_by_note_id(note_id)
|
||||
|
||||
info = {
|
||||
"title": note_row.get("title") or "",
|
||||
"platform": note_row.get("platform") or "",
|
||||
"source_url": note_row.get("url") or "",
|
||||
"scraped_at": note_row.get("scraped_at").isoformat(timespec="seconds") if note_row.get("scraped_at") else "",
|
||||
"comment_count_total": len(comment_rows),
|
||||
"like_count": note_row.get("like_count", ""),
|
||||
"favorite_count": note_row.get("favorite_count", ""),
|
||||
"comment_count_metric": note_row.get("comment_count", ""),
|
||||
"share_count": note_row.get("share_count", ""),
|
||||
"view_count": note_row.get("view_count", ""),
|
||||
"collect_count_yxyy": note_row.get("collect_count", ""),
|
||||
"style_name": note_row.get("style_name") or "",
|
||||
"creator_name": note_row.get("creator_name") or "",
|
||||
}
|
||||
|
||||
texts = [str(row.get("content") or "").strip() for row in comment_rows if str(row.get("content") or "").strip()]
|
||||
return info, texts
|
||||
|
||||
|
||||
def _to_number(value: Any) -> float:
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return 0.0
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def compute_metrics(info: dict[str, Any]) -> dict[str, Any]:
|
||||
"""计算互动率等衍生指标。所有互动率统一以「曝光量」为分母，避免多口径混用。"""
|
||||
likes = info["like_count"]
|
||||
favorites = info["favorite_count"]
|
||||
comments = info["comment_count_metric"] or info["comment_count_total"]
|
||||
shares = info["share_count"]
|
||||
views_raw = info.get("view_count")
|
||||
|
||||
likes_num = _to_number(likes)
|
||||
favorites_num = _to_number(favorites)
|
||||
comments_num = _to_number(comments)
|
||||
shares_num = _to_number(shares)
|
||||
views_num = _to_number(views_raw)
|
||||
|
||||
return {
|
||||
"view_count": views_num,
|
||||
"like_rate": safe_div(likes_num, views_num), # 点赞率
|
||||
"comment_rate": safe_div(comments_num, views_num), # 评论率（评论/曝光，全报告唯一口径）
|
||||
"favorite_rate": safe_div(favorites_num, views_num), # 收藏率
|
||||
"share_rate": safe_div(shares_num, views_num), # 分享率
|
||||
"engagement_rate": safe_div(likes_num + comments_num + favorites_num + shares_num, views_num), # 综合互动率
|
||||
"favorite_to_comment": safe_div(favorites_num, comments_num), # 收藏评论比（独立口径）
|
||||
}
|
||||
|
||||
|
||||
def keyword_distribution(comments: list[str], target_terms: list[str] | None = None) -> dict[str, Any]:
|
||||
"""统计关键词命中分布，区分「全评论」与「我方产品相关」。
|
||||
|
||||
target_terms：品牌名/款式名列表；提供时把命中文本先筛出再统计，得到「我方产品相关」分布。
|
||||
匹配规则：评论中出现品牌名/款式名，或出现 BAG_CATEGORY_TERPS 中的包包品类词，均视为"我方产品相关"。
|
||||
返回:
|
||||
{
|
||||
"overall": {category: count, ...}, # 全评论 6 类命中数
|
||||
"relevant": {category: count, ...}, # 我方产品相关子集的 6 类命中数
|
||||
"relevant_total": int, # 我方产品相关评论条数
|
||||
}
|
||||
"""
|
||||
terms = [t for t in (target_terms or []) if t]
|
||||
bag_terms = BAG_CATEGORY_TERPS
|
||||
overall: dict[str, int] = {key: 0 for key in CATEGORY_LABELS}
|
||||
relevant: dict[str, int] = {key: 0 for key in CATEGORY_LABELS}
|
||||
relevant_total = 0
|
||||
for text in comments:
|
||||
cats = classify_comment_keywords(text)
|
||||
for c in cats:
|
||||
overall[c] += 1
|
||||
is_relevant = False
|
||||
if terms and any(t in text for t in terms):
|
||||
is_relevant = True
|
||||
elif any(t in text for t in bag_terms):
|
||||
is_relevant = True
|
||||
if is_relevant:
|
||||
relevant_total += 1
|
||||
for c in cats:
|
||||
relevant[c] += 1
|
||||
return {
|
||||
"overall": overall,
|
||||
"relevant": relevant,
|
||||
"relevant_total": relevant_total,
|
||||
}
|
||||
|
||||
|
||||
def _brand_block(brand: str, style_name: str, target_term: str) -> str:
|
||||
if not (brand or style_name):
|
||||
return ""
|
||||
return f"""
|
||||
|
||||
【实体隔离】
|
||||
本次分析的目标：品牌「{brand or '未指定'}」、产品「{style_name or '未指定'}」。本笔记可能涉及多品牌联合推广。
|
||||
请严格将分析对象限定为我方品牌/产品：
|
||||
- 仅将「{target_term}」相关的信息计入种草指标
|
||||
- 笔记/评论中涉及的其他品牌或产品，仅作为场景搭配或联合推广的陪衬
|
||||
- 评论区数据先按以下三类分类再纳入分析：
|
||||
· 有效种草：明确提及「{target_term}」的卖点/外观/体验，或询问其价格/链接
|
||||
· 连带兴趣：既讨论了「{target_term}」也讨论了其他产品
|
||||
· 无效/他品互动：仅讨论其他产品或完全无关闲聊
|
||||
- 情感倾向、痛点需求、卖点提取，仅基于「有效种草」与「连带兴趣」生成；「无效/他品互动」不计入种草指标
|
||||
- 关键词分布按「全评论」与「我方产品相关」两个口径分别统计
|
||||
"""
|
||||
|
||||
|
||||
def _data_table_markdown(info: dict[str, Any], metrics: dict[str, Any], raw_total: int, filtered_total: int) -> str:
|
||||
"""构造数据表 markdown（用于提示词 / 报告 / 卡片）。"""
|
||||
has_view = bool(metrics.get("view_count"))
|
||||
def pct(v: float) -> str:
|
||||
return f"{v * 100:.2f}%" if v else "—"
|
||||
def num(v: Any) -> str:
|
||||
if v in ("", None):
|
||||
return "—"
|
||||
try:
|
||||
return f"{int(float(v)):,}"
|
||||
except (TypeError, ValueError):
|
||||
return str(v)
|
||||
|
||||
likes = info.get("like_count")
|
||||
favorites = info.get("favorite_count")
|
||||
comments = info.get("comment_count_metric") or info.get("comment_count_total")
|
||||
shares = info.get("share_count")
|
||||
valid_ratio = safe_div(filtered_total, raw_total)
|
||||
|
||||
lines = [
|
||||
"| 维度 | 指标 | 数值 |",
|
||||
"| --- | --- | --- |",
|
||||
f"| 基础 | 点赞 | {num(likes)} |",
|
||||
f"| 基础 | 收藏 | {num(favorites)} |",
|
||||
f"| 基础 | 评论 | {num(comments)} |",
|
||||
f"| 基础 | 分享 | {num(shares)} |",
|
||||
f"| 传播 | 曝光量 | {num(metrics.get('view_count')) if has_view else '未采集'} |",
|
||||
f"| 互动率 | 点赞率 | {pct(metrics.get('like_rate', 0))} |",
|
||||
f"| 互动率 | 评论率 | {pct(metrics.get('comment_rate', 0))} |",
|
||||
f"| 互动率 | 收藏率 | {pct(metrics.get('favorite_rate', 0))} |",
|
||||
f"| 互动率 | 分享率 | {pct(metrics.get('share_rate', 0))} |",
|
||||
f"| 互动率 | 综合互动率 | {pct(metrics.get('engagement_rate', 0))} |",
|
||||
f"| 互动率 | 收藏评论比 | {metrics.get('favorite_to_comment', 0):.2f} |",
|
||||
f"| 质量 | 有效率 | {pct(valid_ratio)} |",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _keyword_distribution_markdown(
|
||||
distribution: dict[str, Any],
|
||||
total_filtered: int,
|
||||
) -> str:
|
||||
"""构造关键词分布 markdown，含「全评论」与「我方产品相关」两段。"""
|
||||
overall = distribution.get("overall", {}) if isinstance(distribution, dict) else {}
|
||||
relevant = distribution.get("relevant", {}) if isinstance(distribution, dict) else {}
|
||||
relevant_total = distribution.get("relevant_total", 0) if isinstance(distribution, dict) else 0
|
||||
if not overall:
|
||||
return "（无关键词分布数据）"
|
||||
|
||||
def fmt(d: dict[str, int]) -> str:
|
||||
return "\n".join(f"- {CATEGORY_LABELS[k]}：{d.get(k, 0)} 条" for k in CATEGORY_LABELS)
|
||||
|
||||
lines = [
|
||||
f"### 全评论（{total_filtered} 条）",
|
||||
fmt(overall),
|
||||
"",
|
||||
f"### 我方产品相关（{relevant_total} 条）",
|
||||
fmt(relevant) if relevant_total else "_（无评论明确提及我方产品/品牌）_",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _sample_comments(comments: list[str], max_count: int) -> list[str]:
|
||||
"""当评论数超过上限时，按关键词多样性抽样保留最有信息量的评论。
|
||||
|
||||
策略：优先保留命中更多关键词类别的评论，同类别数时保留更长的评论。
|
||||
"""
|
||||
scored: list[tuple[int, int, int, str]] = [] # (负类别数, 负长度, 原始索引, 文本)
|
||||
for i, text in enumerate(comments):
|
||||
cats = classify_comment_keywords(text)
|
||||
scored.append((-len(cats), -len(text), i, text))
|
||||
scored.sort()
|
||||
return [text for _, _, _, text in scored[:max_count]]
|
||||
|
||||
|
||||
def build_note_analysis_prompt(
|
||||
info: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
distribution: dict[str, Any],
|
||||
comments: list[str],
|
||||
) -> str:
|
||||
"""构造 LLM 输入的 user_content。"""
|
||||
brand = info.get("brand") or ""
|
||||
style_name = info.get("style_name") or ""
|
||||
target_label = " / ".join(x for x in (brand, style_name) if x) if (brand or style_name) else "未指定"
|
||||
target_term = brand or style_name
|
||||
|
||||
# 评论数量截断：超过上限时按关键词多样性抽样
|
||||
total_count = len(comments)
|
||||
if total_count > MAX_COMMENTS_FOR_LLM:
|
||||
comments = _sample_comments(comments, MAX_COMMENTS_FOR_LLM)
|
||||
|
||||
lines: list[str] = []
|
||||
if total_count > MAX_COMMENTS_FOR_LLM:
|
||||
lines.append(f"共有 {total_count} 条过滤后的有效评论（已抽样 {len(comments)} 条），请基于以下单篇笔记的信息和评论内容，按要求输出分析报告。")
|
||||
else:
|
||||
lines.append(f"共有 {len(comments)} 条过滤后的有效评论，请基于以下单篇笔记的信息和评论内容，按要求输出分析报告。")
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"## 单篇笔记基础信息",
|
||||
f"- 标题：{info['title'] or '未获取'}",
|
||||
f"- 平台：{info['platform']}",
|
||||
f"- 链接：{info['source_url']}",
|
||||
f"- 品牌/产品：{target_label}",
|
||||
f"- 点赞数：{info['like_count']}",
|
||||
f"- 收藏数：{info['favorite_count']}",
|
||||
f"- 评论数：{info['comment_count_metric'] or info['comment_count_total']}",
|
||||
f"- 分享数：{info['share_count']}",
|
||||
"",
|
||||
])
|
||||
|
||||
if brand or style_name:
|
||||
lines.extend([
|
||||
"## 实体隔离规则（产品隔离）",
|
||||
f"本次分析的目标：品牌「{brand}」、产品「{style_name}」。本笔记可能涉及多品牌联合推广，请严格将分析对象限定为我方品牌/产品。",
|
||||
f"- 仅将「{target_term}」相关的信息计入种草指标",
|
||||
"- 笔记/评论中涉及的其他品牌或产品，仅作为场景搭配或联合推广的陪衬",
|
||||
"- 评论区按以下三类分类后再纳入分析：",
|
||||
f" · 有效种草：明确提及「{target_term}」的卖点/外观/使用体验，或询问其价格/链接",
|
||||
f" · 连带兴趣：既讨论了「{target_term}」也讨论了其他产品",
|
||||
" · 无效/他品互动：仅讨论其他产品或完全无关闲聊",
|
||||
"- 情感倾向、痛点需求、卖点提取，仅基于「有效种草」与「连带兴趣」生成；「无效/他品互动」不计入种草指标",
|
||||
"",
|
||||
])
|
||||
|
||||
lines.extend([
|
||||
"## 数据表（已计算好，可直接引用）",
|
||||
_data_table_markdown(info, metrics, info.get("comment_count_total", 0), len(comments)),
|
||||
"",
|
||||
"## 评论关键词分布（按口径分别给出）",
|
||||
_keyword_distribution_markdown(distribution, len(comments)),
|
||||
"",
|
||||
"## 有效评论内容",
|
||||
])
|
||||
for text in comments:
|
||||
lines.append(f"- {text}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def analyze_note(
|
||||
info: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
distribution: dict[str, Any],
|
||||
comments: list[str],
|
||||
) -> str:
|
||||
"""调用 Hermes 生成单篇笔记分析。"""
|
||||
if not comments:
|
||||
return "该笔记无有效评论可分析。"
|
||||
|
||||
log(f"调用 Hermes 生成分析报告（{info['title'] or info['source_url']}，{len(comments)} 条有效评论）...")
|
||||
user_content = build_note_analysis_prompt(info, metrics, distribution, comments)
|
||||
|
||||
brand = info.get("brand") or ""
|
||||
style_name = info.get("style_name") or ""
|
||||
target_term = brand or style_name
|
||||
brand_block = _brand_block(brand, style_name, target_term)
|
||||
|
||||
system_prompt = SINGLE_NOTE_PROMPT + brand_block
|
||||
|
||||
return call_hermes_analyzer(system_prompt, user_content)
|
||||
|
||||
|
||||
def build_report(
|
||||
info: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
distribution: dict[str, Any],
|
||||
raw_comments: list[str],
|
||||
filtered_comments: list[str],
|
||||
analysis: str,
|
||||
) -> str:
|
||||
"""构造本地报告文件。"""
|
||||
return _build_report_concise(info, metrics, distribution, raw_comments, filtered_comments, analysis)
|
||||
|
||||
|
||||
def _build_report_concise(
|
||||
info: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
distribution: dict[str, Any],
|
||||
raw_comments: list[str],
|
||||
filtered_comments: list[str],
|
||||
analysis: str,
|
||||
) -> str:
|
||||
brand = info.get("brand") or ""
|
||||
style_name = info.get("style_name") or ""
|
||||
target_label = " / ".join(x for x in (brand, style_name) if x) if (brand or style_name) else "未指定"
|
||||
|
||||
lines = [
|
||||
"# 单篇笔记分析报告（投放复盘版）",
|
||||
"",
|
||||
f"**标题**：{info['title'] or '未获取'}",
|
||||
f"**平台**：{info['platform']}",
|
||||
f"**品牌/产品**：{target_label}",
|
||||
f"**达人**：{info.get('creator_name') or '未指定'}",
|
||||
f"**链接**：{info['source_url']}",
|
||||
"",
|
||||
"## 一、数据表",
|
||||
_data_table_markdown(info, metrics, len(raw_comments), len(filtered_comments)),
|
||||
"",
|
||||
"## 二、关键词分布",
|
||||
_keyword_distribution_markdown(distribution, len(filtered_comments)),
|
||||
"",
|
||||
"## 三、投放复盘",
|
||||
analysis,
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_report(report: str, path: Path) -> Path:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(report, encoding="utf-8-sig")
|
||||
log(f"报告已写入: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _md_field(label: str, value: str, is_short: bool = True) -> dict[str, Any]:
|
||||
return {
|
||||
"is_short": is_short,
|
||||
"text": {"tag": "lark_md", "content": f"**{label}**\n{value}"},
|
||||
}
|
||||
|
||||
|
||||
def _section_header(title: str) -> dict[str, Any]:
|
||||
return {
|
||||
"tag": "div",
|
||||
"text": {"tag": "lark_md", "content": f"**{title}**"},
|
||||
}
|
||||
|
||||
|
||||
def build_card_payload(
|
||||
info: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
distribution: dict[str, Any],
|
||||
raw_comments: list[str],
|
||||
filtered_comments: list[str],
|
||||
analysis: str,
|
||||
report_path: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""构造飞书 interactive 卡片 JSON。
|
||||
|
||||
布局（精简版默认）：
|
||||
header（蓝色）+ 基本信息 + 数据表 + 关键词分布 + 投放复盘 + 报告路径
|
||||
"""
|
||||
brand = info.get("brand") or ""
|
||||
style_name = info.get("style_name") or ""
|
||||
creator_name = info.get("creator_name") or ""
|
||||
target_label = " / ".join(x for x in (brand, style_name) if x) if (brand or style_name) else "未指定"
|
||||
title = info.get("title") or "未获取"
|
||||
platform = info.get("platform") or "unknown"
|
||||
source_url = info.get("source_url") or ""
|
||||
|
||||
elements: list[dict[str, Any]] = []
|
||||
|
||||
# 基本信息
|
||||
elements.append({
|
||||
"tag": "div",
|
||||
"fields": [
|
||||
_md_field("标题", title, is_short=False),
|
||||
_md_field("品牌 / 产品", target_label),
|
||||
_md_field("平台", platform),
|
||||
_md_field("达人", creator_name or "未指定"),
|
||||
],
|
||||
})
|
||||
if source_url:
|
||||
elements.append({
|
||||
"tag": "div",
|
||||
"text": {"tag": "lark_md", "content": f"🔗 [打开笔记原文]({source_url})"},
|
||||
})
|
||||
|
||||
# 数据表
|
||||
elements.append({"tag": "hr"})
|
||||
elements.append(_section_header("📋 数据表"))
|
||||
elements.append({
|
||||
"tag": "div",
|
||||
"text": {"tag": "lark_md", "content": _data_table_markdown(info, metrics, len(raw_comments), len(filtered_comments))},
|
||||
})
|
||||
|
||||
# 关键词分布（拆分两个口径）
|
||||
elements.append(_section_header("🔑 关键词分布"))
|
||||
elements.append({
|
||||
"tag": "div",
|
||||
"text": {"tag": "lark_md", "content": _keyword_distribution_markdown(distribution, len(filtered_comments))},
|
||||
})
|
||||
|
||||
# 投放复盘（LLM 输出）
|
||||
elements.append(_section_header("💡 投放复盘"))
|
||||
analysis_text = analysis or "（无分析结果）"
|
||||
if len(analysis_text) > 4500:
|
||||
analysis_text = analysis_text[:4500] + "\n\n...（内容过长，已截断，完整分析见报告文件）"
|
||||
elements.append({
|
||||
"tag": "div",
|
||||
"text": {"tag": "lark_md", "content": analysis_text},
|
||||
})
|
||||
|
||||
return {
|
||||
"config": {"wide_screen_mode": True},
|
||||
"header": {
|
||||
"template": "blue",
|
||||
"title": {
|
||||
"tag": "plain_text",
|
||||
"content": "📊 单篇笔记分析报告",
|
||||
},
|
||||
},
|
||||
"elements": elements,
|
||||
}
|
||||
|
||||
|
||||
def _fmt_int(value: Any) -> str:
|
||||
if value is None or value == "":
|
||||
return "—"
|
||||
try:
|
||||
return f"{int(float(value)):,}"
|
||||
except (TypeError, ValueError):
|
||||
return str(value)
|
||||
|
||||
|
||||
def send_feishu_report(
|
||||
info: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
distribution: dict[str, int],
|
||||
raw_comments: list[str],
|
||||
filtered_comments: list[str],
|
||||
analysis: str,
|
||||
open_id: str,
|
||||
report_path: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""通过 lark-cli 以飞书 interactive 卡片格式发送报告。
|
||||
|
||||
卡片包含：标题/平台/达人基础信息、6 个分节指标 + LLM 舆情洞察。
|
||||
Windows 下直接走 lark-cli.cmd 会撞到命令行长度上限且对 &|<>^% 转义敏感，
|
||||
改为调 node + run.js，args 用 list 传，由 node 自己处理。
|
||||
"""
|
||||
card = build_card_payload(
|
||||
info, metrics, distribution,
|
||||
raw_comments, filtered_comments, analysis,
|
||||
report_path,
|
||||
)
|
||||
content_json = json.dumps(card, ensure_ascii=False)
|
||||
|
||||
if sys.platform.startswith("win"):
|
||||
# Windows 下走 lark-cli.cmd 会撞到命令行长度上限（~8K）且对 &|<>^% 转义敏感；
|
||||
# 直接调 node + run.js，args 用 list 传，由 node 自己处理，避开所有 cmd 转义问题。
|
||||
run_js = (
|
||||
Path(os.environ.get("APPDATA", str(Path.home())))
|
||||
/ "npm"
|
||||
/ "node_modules"
|
||||
/ "@larksuite"
|
||||
/ "cli"
|
||||
/ "scripts"
|
||||
/ "run.js"
|
||||
)
|
||||
if not run_js.exists():
|
||||
raise RuntimeError(f"找不到 lark-cli 入口: {run_js}")
|
||||
command = [
|
||||
"node", str(run_js),
|
||||
"--profile", "hermes-analyzer",
|
||||
"im", "+messages-send",
|
||||
"--user-id", open_id,
|
||||
"--content", content_json,
|
||||
"--msg-type", "interactive",
|
||||
"--as", "bot",
|
||||
"--format", "json",
|
||||
]
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
stdout = completed.stdout
|
||||
stderr = completed.stderr
|
||||
else:
|
||||
command = [
|
||||
"lark-cli",
|
||||
"--profile", "hermes-analyzer",
|
||||
"im", "+messages-send",
|
||||
"--user-id", open_id,
|
||||
"--content", content_json,
|
||||
"--msg-type", "interactive",
|
||||
"--as", "bot",
|
||||
"--format", "json",
|
||||
]
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
stdout = completed.stdout
|
||||
stderr = completed.stderr
|
||||
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"lark-cli 发送失败 (exit {completed.returncode}):\n"
|
||||
f"stdout: {stdout}\nstderr: {stderr}"
|
||||
)
|
||||
try:
|
||||
return json.loads(stdout)
|
||||
except json.JSONDecodeError:
|
||||
return {"raw": stdout}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="单篇笔记 → 本地过滤 + Hermes 分析 → 项目 data/summary 报告")
|
||||
source_group = parser.add_mutually_exclusive_group(required=True)
|
||||
source_group.add_argument("--input", type=Path, help="单篇笔记 JSON 文件路径")
|
||||
source_group.add_argument("--note-id", type=int, help="数据库 cmt_notes.id")
|
||||
source_group.add_argument("--url", type=str, help="数据库 cmt_notes.url")
|
||||
parser.add_argument("--output", type=Path, default=OUTPUT_PATH, help="报告输出路径")
|
||||
parser.add_argument("--open-id", default=TARGET_OPEN_ID, help="飞书接收人 open_id")
|
||||
parser.add_argument("--brand", default="", help="我方品牌名（用于实体隔离/产品隔离规则；不传则只用款式名）")
|
||||
parser.add_argument("--no-send", action="store_true", help="只生成报告文件，不发送飞书")
|
||||
parser.add_argument("--llm-filter", action="store_true",
|
||||
help="用 LLM 语义清洗评论(替代关键词过滤,保留更多有效评论,但慢)")
|
||||
parser.add_argument("--batch-size", type=int, default=30,
|
||||
help="LLM 清洗的批大小(默认 30 条/批)")
|
||||
parser.add_argument("--max-workers", type=int, default=6,
|
||||
help="LLM 清洗的并发批数(默认 6)")
|
||||
args = parser.parse_args()
|
||||
args.output = resolve_layer_output(
|
||||
args.output, layer_root=PATHS.exports_root, field="--output"
|
||||
)
|
||||
|
||||
if not args.no_send:
|
||||
if not HERMES_ANALYZER_TOKEN:
|
||||
log("[ERROR] HERMES_ANALYZER_TOKEN 未设置。请配置 data/config/analyze.env 或传 --no-send")
|
||||
return 1
|
||||
if not args.open_id:
|
||||
log("[ERROR] 飞书接收人 open_id 未设置。请配置 FEISHU_TARGET_OPEN_ID 或传 --open-id 或 --no-send")
|
||||
return 1
|
||||
|
||||
if args.input:
|
||||
log(f"读取单篇笔记 JSON: {args.input}")
|
||||
try:
|
||||
data = load_note_json(args.input)
|
||||
except Exception as exc:
|
||||
log(f"读取失败: {exc}")
|
||||
return 1
|
||||
info = extract_note_info(data)
|
||||
raw_comments = extract_comment_texts(data)
|
||||
elif args.note_id or args.url:
|
||||
try:
|
||||
info, raw_comments = load_note_from_db(note_id=args.note_id, url=args.url)
|
||||
except Exception as exc:
|
||||
log(f"从数据库读取失败: {exc}")
|
||||
return 1
|
||||
else:
|
||||
log("必须提供 --input、--note-id 或 --url")
|
||||
return 1
|
||||
|
||||
log(f"笔记标题: {info['title'] or '未获取'} | 平台: {info['platform']} | 总评论: {info['comment_count_total']}")
|
||||
|
||||
if not raw_comments:
|
||||
# 无评论只代表评论洞察不可用，不应让整篇笔记及其曝光/互动指标
|
||||
# 从周汇总中消失。后续会生成数据型报告，并明确标注无评论。
|
||||
log("该笔记没有评论内容，跳过评论洞察并继续生成数据型报告")
|
||||
|
||||
# 先把 brand 注入到 info，再算关键词分布（需要 target_terms 拆分相关/全部）
|
||||
if args.brand:
|
||||
info["brand"] = args.brand
|
||||
else:
|
||||
info.setdefault("brand", "")
|
||||
|
||||
metrics = compute_metrics(info)
|
||||
try:
|
||||
if args.llm_filter:
|
||||
log(f"使用 LLM 语义清洗评论(批大小 {args.batch_size}, 并发 {args.max_workers})...")
|
||||
filtered_comments = llm_clean_comments(raw_comments,
|
||||
batch_size=args.batch_size,
|
||||
max_workers=args.max_workers)
|
||||
else:
|
||||
filtered_comments = code_filter_comments(raw_comments)
|
||||
except Exception as exc:
|
||||
log(f"评论过滤失败: {exc}")
|
||||
return 1
|
||||
|
||||
target_terms = [t for t in (info.get("brand"), info.get("style_name")) if t]
|
||||
distribution = keyword_distribution(filtered_comments, target_terms=target_terms)
|
||||
log(f"关键词分布: 全评论 {len(filtered_comments)} 条，我方产品相关 {distribution['relevant_total']} 条")
|
||||
|
||||
try:
|
||||
analysis = analyze_note(info, metrics, distribution, filtered_comments)
|
||||
except Exception as exc:
|
||||
log(f"笔记分析失败: {exc}")
|
||||
return 1
|
||||
|
||||
report = build_report(info, metrics, distribution, raw_comments, filtered_comments, analysis)
|
||||
|
||||
try:
|
||||
write_report(report, args.output)
|
||||
except Exception as exc:
|
||||
log(f"写入报告失败: {exc}")
|
||||
return 1
|
||||
|
||||
if args.no_send:
|
||||
log("已指定 --no-send，跳过飞书发送")
|
||||
else:
|
||||
# 卡片永远走精简版（给老板/团队 30 秒看完的复盘视图）
|
||||
try:
|
||||
result = send_feishu_report(
|
||||
info, metrics, distribution,
|
||||
raw_comments, filtered_comments, analysis,
|
||||
args.open_id, args.output,
|
||||
)
|
||||
log(f"飞书卡片发送成功: {result}")
|
||||
except Exception as exc:
|
||||
log(f"飞书消息发送失败（报告文件仍已生成）: {exc}")
|
||||
return 1
|
||||
|
||||
log("全部完成")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\n已中断", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
"""Batch re-scrape all Bilibili videos from DB using the browser scraper.
|
||||
|
||||
Updates:
|
||||
- note_metrics (like / favorite / comment / share counts, title)
|
||||
- cmt_comments table (replace existing)
|
||||
- JSON/CSV files in data/notes/bilibili/
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
_TOOLS_DIR = PATHS.tools_root
|
||||
_PROJECT_ROOT = PATHS.module_root
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import db
|
||||
from gyxx_flow.modules.content_marketing.runtime import bilibili_comment_scraper as scraper
|
||||
|
||||
RELOGIN_SCRIPT = _TOOLS_DIR / "relogin_bilibili.py"
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
||||
|
||||
|
||||
def collect_urls() -> list[dict]:
|
||||
with db.get_dict_cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, url, title
|
||||
FROM cmt_notes
|
||||
WHERE platform = 'bilibili' AND url IS NOT NULL AND url != ''
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
def upsert_metrics(note_id: int, title: str, metrics: dict) -> None:
|
||||
db.update_note_metrics(
|
||||
note_id,
|
||||
like_count=metrics.get("like_count") or None,
|
||||
favorite_count=metrics.get("favorite_count") or None,
|
||||
comment_count=metrics.get("comment_count") or None,
|
||||
share_count=metrics.get("share_count") or None,
|
||||
title=title or None,
|
||||
)
|
||||
|
||||
|
||||
def _to_int(value) -> int | None:
|
||||
try:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def comment_replace_policy(comments: list[dict], stats: dict, metrics: dict) -> tuple[bool, bool, str]:
|
||||
"""Return (should_replace, allow_empty, reason)."""
|
||||
logged_in = bool(stats.get("logged_in"))
|
||||
expected = _to_int(metrics.get("comment_count"))
|
||||
if expected is None:
|
||||
expected = _to_int(stats.get("api_total"))
|
||||
|
||||
if comments:
|
||||
if not logged_in and (expected is None or expected > len(comments)):
|
||||
return False, False, "未登录且可能只采到热门评论"
|
||||
return True, False, ""
|
||||
|
||||
if expected == 0:
|
||||
return True, True, "平台显示 0 评论"
|
||||
return False, False, "空评论且无法确认平台为 0 评论"
|
||||
|
||||
|
||||
def do_relogin() -> bool:
|
||||
"""Run relogin_bilibili.py (headed, waits for QR scan). Returns True if successful."""
|
||||
log(" Cookie 可能过期,触发自动重登 (请扫码)...")
|
||||
try:
|
||||
rc = subprocess.call(
|
||||
[sys.executable, str(RELOGIN_SCRIPT)],
|
||||
cwd=str(_PROJECT_ROOT),
|
||||
)
|
||||
if rc == 0:
|
||||
log(" 重登成功,继续采集")
|
||||
return True
|
||||
else:
|
||||
log(f" 重登失败 (exit={rc})")
|
||||
return False
|
||||
except Exception as exc:
|
||||
log(f" 重登脚本执行异常: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
|
||||
rows = collect_urls()
|
||||
if only_ids:
|
||||
rows = [r for r in rows if r["id"] in set(only_ids)]
|
||||
log(f"Total bilibili notes to scrape: {len(rows)}")
|
||||
|
||||
ok = 0
|
||||
fail = 0
|
||||
total_comments = 0
|
||||
consecutive_fail = 0
|
||||
MAX_CONSECUTIVE_FAIL = 3
|
||||
relogin_used = 0
|
||||
MAX_RELOGINS = 1
|
||||
t0 = time.time()
|
||||
for i, row in enumerate(rows, 1):
|
||||
note_id = row["id"]
|
||||
url = row["url"]
|
||||
log(f"[{i}/{len(rows)}] id={note_id} url={url}")
|
||||
comments = None
|
||||
result = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
comments, result = scraper.scrape_comments(
|
||||
url, max_pages=200, delay=0.25, headless=True,
|
||||
max_reply_pages=200, login_timeout=0,
|
||||
)
|
||||
break
|
||||
except Exception as exc:
|
||||
if attempt < 2 and "has been closed" in str(exc):
|
||||
log(f" Browser closed, retrying ({attempt+2}/3)...")
|
||||
time.sleep(3)
|
||||
else:
|
||||
log(f" ERROR: {exc}")
|
||||
fail += 1
|
||||
break
|
||||
if comments is None:
|
||||
consecutive_fail += 1
|
||||
if consecutive_fail >= MAX_CONSECUTIVE_FAIL and relogin_used < MAX_RELOGINS:
|
||||
if do_relogin():
|
||||
relogin_used += 1
|
||||
consecutive_fail = 0
|
||||
continue
|
||||
continue
|
||||
|
||||
consecutive_fail = 0
|
||||
|
||||
video = result.get("video", {}) or {}
|
||||
title = video.get("title", "") or result.get("title", "")
|
||||
stats = result.get("stats", {}) or {}
|
||||
metrics = stats.get("note_metrics", {}) or {}
|
||||
logged_in = bool(stats.get("logged_in"))
|
||||
|
||||
json_path, csv_path = scraper.write_outputs(url, comments, result)
|
||||
top = len([c for c in comments if c["level"] == "comment"])
|
||||
replies = len([c for c in comments if c["level"] == "reply"])
|
||||
warn = "" if logged_in else " [WARN] 未登录,只采到热门评论"
|
||||
log(
|
||||
f" ok: title={title[:40]!r} | {len(comments)} comments ({top}+{replies}) | "
|
||||
f"likes={metrics.get('like_count')}, favs={metrics.get('favorite_count')}, "
|
||||
f"cmts={metrics.get('comment_count')}"
|
||||
+ warn
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
try:
|
||||
upsert_metrics(note_id, title, metrics)
|
||||
except Exception as exc:
|
||||
log(f" DB metric update error: {exc}")
|
||||
try:
|
||||
should_replace, allow_empty, reason = comment_replace_policy(comments, stats, metrics)
|
||||
if should_replace:
|
||||
db.replace_comments(note_id, "bilibili", comments, allow_empty=allow_empty)
|
||||
else:
|
||||
log(f" skip DB comment replace: {reason}")
|
||||
except Exception as exc:
|
||||
log(f" DB comment replace error: {exc}")
|
||||
|
||||
ok += 1
|
||||
total_comments += len(comments)
|
||||
|
||||
elapsed = time.time() - t0
|
||||
log(
|
||||
f"DONE: {ok} ok, {fail} fail, {total_comments} comments total, "
|
||||
f"{elapsed:.1f}s ({elapsed / max(ok, 1):.1f}s per note)"
|
||||
)
|
||||
return 0 if fail == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:]
|
||||
only_ids = []
|
||||
dry_run = False
|
||||
for a in args:
|
||||
if a == "--dry-run":
|
||||
dry_run = True
|
||||
elif a.isdigit():
|
||||
only_ids.append(int(a))
|
||||
if only_ids:
|
||||
log(f"Restricted to ids: {only_ids}")
|
||||
if dry_run:
|
||||
log("DRY RUN: will not touch DB")
|
||||
raise SystemExit(main(only_ids or None, dry_run))
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Batch re-scrape all Douyin notes from DB using the browser-based scraper.
|
||||
|
||||
Updates:
|
||||
- note_metrics (like / favorite / comment / share counts)
|
||||
- cmt_comments table (replace existing)
|
||||
- JSON/CSV files in data/notes/douyin/
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
# Path setup: db.py is in data/tools/, scrapers are in project root
|
||||
_TOOLS_DIR = PATHS.tools_root
|
||||
_PROJECT_ROOT = PATHS.module_root
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import db
|
||||
from gyxx_flow.modules.content_marketing.runtime import douyin_comment_scraper as scraper
|
||||
|
||||
RELOGIN_SCRIPT = _TOOLS_DIR / "relogin_douyin.py"
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
||||
|
||||
|
||||
def collect_urls() -> list[dict]:
|
||||
with db.get_dict_cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, url, title, comment_count
|
||||
FROM cmt_notes
|
||||
WHERE platform = 'douyin' AND url IS NOT NULL AND url != ''
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
|
||||
fast = [r for r in rows if (r.get("comment_count") or 0) <= 200]
|
||||
slow = [r for r in rows if (r.get("comment_count") or 0) > 200]
|
||||
if fast:
|
||||
log(f"Sorted: {len(fast)} notes ≤200 comments (fast first), {len(slow)} notes >200 (slow)")
|
||||
return fast + slow
|
||||
|
||||
|
||||
def upsert_metrics(note_id: int, metrics: dict) -> None:
|
||||
db.update_note_metrics(
|
||||
note_id,
|
||||
like_count=metrics.get("like_count") or None,
|
||||
favorite_count=metrics.get("favorite_count") or None,
|
||||
comment_count=metrics.get("comment_count") or None,
|
||||
share_count=metrics.get("share_count") or None,
|
||||
title=metrics.get("title") or None,
|
||||
)
|
||||
|
||||
|
||||
def _to_int(value) -> int | None:
|
||||
try:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def comment_replace_policy(comments: list[dict], result: dict, metrics: dict) -> tuple[bool, bool, str]:
|
||||
"""Return (should_replace, allow_empty, reason)."""
|
||||
stats = result.get("stats", {}) if isinstance(result, dict) else {}
|
||||
logged_in = bool(stats.get("logged_in"))
|
||||
expected = _to_int(stats.get("api_total"))
|
||||
if expected is None:
|
||||
expected = _to_int(metrics.get("comment_count"))
|
||||
|
||||
if comments:
|
||||
if not logged_in and (expected is None or expected > len(comments)):
|
||||
return False, False, "未登录且可能只采到部分评论"
|
||||
return True, False, ""
|
||||
|
||||
if expected == 0:
|
||||
return True, True, "平台显示 0 评论"
|
||||
return False, False, "空评论且无法确认平台为 0 评论"
|
||||
|
||||
|
||||
def result_logged_in(result: dict | None) -> bool:
|
||||
"""Read the login flag from the scraper result's nested stats payload."""
|
||||
if not isinstance(result, dict):
|
||||
return False
|
||||
nested_stats = result.get("stats")
|
||||
return bool(nested_stats.get("logged_in")) if isinstance(nested_stats, dict) else False
|
||||
|
||||
|
||||
def do_relogin() -> bool:
|
||||
"""Run relogin_douyin.py (headed, waits for QR scan). Returns True if successful."""
|
||||
log(" Cookie 可能过期,触发自动重登 (请扫码)...")
|
||||
try:
|
||||
rc = subprocess.call(
|
||||
[sys.executable, str(RELOGIN_SCRIPT)],
|
||||
cwd=str(_PROJECT_ROOT),
|
||||
)
|
||||
if rc == 0:
|
||||
log(" 重登成功,继续采集")
|
||||
return True
|
||||
else:
|
||||
log(f" 重登失败 (exit={rc})")
|
||||
return False
|
||||
except Exception as exc:
|
||||
log(f" 重登脚本执行异常: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
|
||||
rows = collect_urls()
|
||||
if only_ids:
|
||||
rows = [r for r in rows if r["id"] in set(only_ids)]
|
||||
log(f"Total douyin notes to scrape: {len(rows)}")
|
||||
|
||||
ok = 0
|
||||
fail = 0
|
||||
total_comments = 0
|
||||
consecutive_fail = 0
|
||||
MAX_CONSECUTIVE_FAIL = 3
|
||||
COOLDOWN_SECONDS = 300
|
||||
cooldown_used = 0
|
||||
MAX_COOLDOWNS = 3
|
||||
relogin_used = 0
|
||||
MAX_RELOGINS = 1 # 每轮 batch 最多自动重登1次
|
||||
t0 = time.time()
|
||||
for i, row in enumerate(rows, 1):
|
||||
note_id = row["id"]
|
||||
url = row["url"]
|
||||
log(f"[{i}/{len(rows)}] id={note_id} url={url}")
|
||||
comments = None
|
||||
stats = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
comments, stats = scraper.scrape_comments(url, 300, 60, 10, True)
|
||||
break
|
||||
except Exception as exc:
|
||||
msg = str(exc)
|
||||
retryable = attempt < 2 and (
|
||||
"has been closed" in msg
|
||||
or "page not loaded" in msg
|
||||
or "Cannot extract aweme_id" in msg
|
||||
)
|
||||
if retryable:
|
||||
wait_s = 3 * (attempt + 1)
|
||||
label = (
|
||||
"Browser closed" if "has been closed" in msg
|
||||
else "Page not loaded" if "page not loaded" in msg
|
||||
else "aweme_id failed"
|
||||
)
|
||||
log(f" {label}, retrying ({attempt+2}/3) after {wait_s}s...")
|
||||
time.sleep(wait_s)
|
||||
else:
|
||||
log(f" ERROR: {exc}")
|
||||
fail += 1
|
||||
break
|
||||
|
||||
# Check logged_in status even on success
|
||||
logged_in = result_logged_in(stats)
|
||||
|
||||
if comments is None:
|
||||
consecutive_fail += 1
|
||||
# 连续失败 + 可能 cookie 过期 → 尝试自动重登
|
||||
if consecutive_fail >= MAX_CONSECUTIVE_FAIL and relogin_used < MAX_RELOGINS:
|
||||
if do_relogin():
|
||||
relogin_used += 1
|
||||
consecutive_fail = 0
|
||||
continue # retry the current URL in next iteration
|
||||
else:
|
||||
# 重登失败,走冷却逻辑
|
||||
pass
|
||||
|
||||
if consecutive_fail >= MAX_CONSECUTIVE_FAIL:
|
||||
if cooldown_used < MAX_COOLDOWNS:
|
||||
cooldown_used += 1
|
||||
log(
|
||||
f" {consecutive_fail} consecutive failures, cooling down "
|
||||
f"{COOLDOWN_SECONDS}s (cooldown {cooldown_used}/{MAX_COOLDOWNS})..."
|
||||
)
|
||||
time.sleep(COOLDOWN_SECONDS)
|
||||
consecutive_fail = 0
|
||||
continue
|
||||
else:
|
||||
log(
|
||||
f" {MAX_COOLDOWNS} cooldowns exhausted, aborting batch; "
|
||||
f"{len(rows) - i} notes skipped"
|
||||
)
|
||||
break
|
||||
continue
|
||||
|
||||
consecutive_fail = 0
|
||||
|
||||
json_path, csv_path = scraper.write_outputs(url, comments, stats)
|
||||
metrics = stats.get("stats", {}).get("note_metrics", {})
|
||||
top = len([c for c in comments if c["level"] == "comment"])
|
||||
replies = len([c for c in comments if c["level"] == "reply"])
|
||||
warn = "" if logged_in else " [WARN] 未登录,可能只采到部分评论"
|
||||
log(
|
||||
f" ok: {len(comments)} comments ({top} top + {replies} replies) | "
|
||||
f"likes={metrics.get('like_count')}, favs={metrics.get('favorite_count')}, "
|
||||
f"cmts={metrics.get('comment_count')}, shares={metrics.get('share_count')}"
|
||||
+ warn
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
try:
|
||||
upsert_metrics(note_id, metrics)
|
||||
except Exception as exc:
|
||||
log(f" DB metric update error: {exc}")
|
||||
try:
|
||||
should_replace, allow_empty, reason = comment_replace_policy(comments, stats, metrics)
|
||||
if should_replace:
|
||||
db.replace_comments(note_id, "douyin", comments, allow_empty=allow_empty)
|
||||
else:
|
||||
log(f" skip DB comment replace: {reason}")
|
||||
except Exception as exc:
|
||||
log(f" DB comment replace error: {exc}")
|
||||
|
||||
ok += 1
|
||||
total_comments += len(comments)
|
||||
|
||||
elapsed = time.time() - t0
|
||||
log(
|
||||
f"DONE: {ok} ok, {fail} fail, {total_comments} comments total, "
|
||||
f"{elapsed:.1f}s ({elapsed / max(ok, 1):.1f}s per note)"
|
||||
)
|
||||
return 0 if fail == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:]
|
||||
only_ids = []
|
||||
dry_run = False
|
||||
for a in args:
|
||||
if a == "--dry-run":
|
||||
dry_run = True
|
||||
elif a.isdigit():
|
||||
only_ids.append(int(a))
|
||||
if only_ids:
|
||||
log(f"Restricted to ids: {only_ids}")
|
||||
if dry_run:
|
||||
log("DRY RUN: will not touch DB")
|
||||
raise SystemExit(main(only_ids or None, dry_run))
|
||||
+200
@@ -0,0 +1,200 @@
|
||||
"""Batch re-scrape all Xiaohongshu notes from DB using the browser scraper.
|
||||
|
||||
Updates:
|
||||
- note_metrics (like / favorite / comment / share counts, title)
|
||||
- cmt_comments table (replace existing)
|
||||
- JSON/CSV files in data/notes/xiaohongshu/
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
_TOOLS_DIR = PATHS.tools_root
|
||||
_PROJECT_ROOT = PATHS.module_root
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import db
|
||||
from gyxx_flow.modules.content_marketing.runtime import xiaohongshu_comment_scraper as scraper
|
||||
|
||||
RELOGIN_SCRIPT = _TOOLS_DIR / "relogin_xiaohongshu.py"
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
||||
|
||||
|
||||
def collect_urls() -> list[dict]:
|
||||
with db.get_dict_cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, url, title
|
||||
FROM cmt_notes
|
||||
WHERE platform = 'xiaohongshu' AND url IS NOT NULL AND url != ''
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
def upsert_metrics(note_id: int, title: str, metrics: dict) -> None:
|
||||
db.update_note_metrics(
|
||||
note_id,
|
||||
like_count=metrics.get("like_count") or None,
|
||||
favorite_count=metrics.get("favorite_count") or None,
|
||||
comment_count=metrics.get("comment_count") or None,
|
||||
share_count=metrics.get("share_count") or None,
|
||||
title=title or None,
|
||||
)
|
||||
|
||||
|
||||
def _to_int(value) -> int | None:
|
||||
try:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def comment_replace_policy(comments: list[dict], stats: dict, metrics: dict) -> tuple[bool, bool, str]:
|
||||
"""Return (should_replace, allow_empty, reason)."""
|
||||
logged_in = bool(stats.get("logged_in"))
|
||||
expected = _to_int(metrics.get("comment_count"))
|
||||
if expected is None:
|
||||
expected = _to_int(stats.get("api_total"))
|
||||
|
||||
if comments:
|
||||
if not logged_in and (expected is None or expected > len(comments)):
|
||||
return False, False, "未登录且可能只采到部分评论"
|
||||
return True, False, ""
|
||||
if expected == 0:
|
||||
return True, True, "平台显示 0 评论"
|
||||
return False, False, "空评论且无法确认平台为 0 评论"
|
||||
|
||||
|
||||
def do_relogin() -> bool:
|
||||
"""Run relogin_xiaohongshu.py (headed, waits for QR scan). Returns True if successful."""
|
||||
log(" Cookie 可能过期,触发自动重登 (请扫码)...")
|
||||
try:
|
||||
rc = subprocess.call(
|
||||
[sys.executable, str(RELOGIN_SCRIPT)],
|
||||
cwd=str(_PROJECT_ROOT),
|
||||
)
|
||||
if rc == 0:
|
||||
log(" 重登成功,继续采集")
|
||||
return True
|
||||
else:
|
||||
log(f" 重登失败 (exit={rc})")
|
||||
return False
|
||||
except Exception as exc:
|
||||
log(f" 重登脚本执行异常: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int:
|
||||
rows = collect_urls()
|
||||
if only_ids:
|
||||
rows = [r for r in rows if r["id"] in set(only_ids)]
|
||||
log(f"Total xiaohongshu notes to scrape: {len(rows)}")
|
||||
|
||||
ok = 0
|
||||
fail = 0
|
||||
total_comments = 0
|
||||
consecutive_fail = 0
|
||||
MAX_CONSECUTIVE_FAIL = 3
|
||||
relogin_used = 0
|
||||
MAX_RELOGINS = 1
|
||||
t0 = time.time()
|
||||
for i, row in enumerate(rows, 1):
|
||||
note_id = row["id"]
|
||||
url = row["url"]
|
||||
log(f"[{i}/{len(rows)}] id={note_id} url={url}")
|
||||
comments = None
|
||||
result = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
comments, result = scraper.scrape_comments(url, 60, 30, 4, True)
|
||||
break
|
||||
except Exception as exc:
|
||||
if attempt < 2 and "has been closed" in str(exc):
|
||||
log(f" Browser closed, retrying ({attempt+2}/3)...")
|
||||
time.sleep(3)
|
||||
else:
|
||||
log(f" ERROR: {exc}")
|
||||
fail += 1
|
||||
break
|
||||
if comments is None:
|
||||
consecutive_fail += 1
|
||||
if consecutive_fail >= MAX_CONSECUTIVE_FAIL and relogin_used < MAX_RELOGINS:
|
||||
if do_relogin():
|
||||
relogin_used += 1
|
||||
consecutive_fail = 0
|
||||
continue
|
||||
# No relogin or relogin failed → skip this note
|
||||
continue
|
||||
|
||||
consecutive_fail = 0
|
||||
|
||||
title = result.get("title", "")
|
||||
stats = result.get("stats", {})
|
||||
metrics = stats.get("note_metrics", {}) or {}
|
||||
logged_in = bool(stats.get("logged_in"))
|
||||
|
||||
json_path, csv_path = scraper.write_outputs(url, comments, result)
|
||||
top = len([c for c in comments if c["level"] == "comment"])
|
||||
replies = len([c for c in comments if c["level"] == "reply"])
|
||||
warn = "" if logged_in else " [WARN] 未登录,可能只采到部分评论"
|
||||
log(
|
||||
f" ok: title={title[:40]!r} | {len(comments)} comments ({top}+{replies}) | "
|
||||
f"likes={metrics.get('like_count')}, favs={metrics.get('favorite_count')}, "
|
||||
f"cmts={metrics.get('comment_count')}"
|
||||
+ warn
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
try:
|
||||
upsert_metrics(note_id, title, metrics)
|
||||
except Exception as exc:
|
||||
log(f" DB metric update error: {exc}")
|
||||
try:
|
||||
should_replace, allow_empty, reason = comment_replace_policy(comments, stats, metrics)
|
||||
if should_replace:
|
||||
db.replace_comments(note_id, "xiaohongshu", comments, allow_empty=allow_empty)
|
||||
else:
|
||||
log(f" skip DB comment replace: {reason}")
|
||||
except Exception as exc:
|
||||
log(f" DB comment replace error: {exc}")
|
||||
|
||||
ok += 1
|
||||
total_comments += len(comments)
|
||||
|
||||
elapsed = time.time() - t0
|
||||
log(
|
||||
f"DONE: {ok} ok, {fail} fail, {total_comments} comments total, "
|
||||
f"{elapsed:.1f}s ({elapsed / max(ok, 1):.1f}s per note)"
|
||||
)
|
||||
return 0 if fail == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:]
|
||||
only_ids = []
|
||||
dry_run = False
|
||||
for a in args:
|
||||
if a == "--dry-run":
|
||||
dry_run = True
|
||||
elif a.isdigit():
|
||||
only_ids.append(int(a))
|
||||
if only_ids:
|
||||
log(f"Restricted to ids: {only_ids}")
|
||||
if dry_run:
|
||||
log("DRY RUN: will not touch DB")
|
||||
raise SystemExit(main(only_ids or None, dry_run))
|
||||
@@ -0,0 +1,42 @@
|
||||
"""统计 24 张表里 creator_id / creator_name 填充率"""
|
||||
import json
|
||||
import glob
|
||||
from pathlib import Path
|
||||
|
||||
per_table = []
|
||||
total_records = total_with_id = total_with_name = 0
|
||||
|
||||
for path in sorted(
|
||||
glob.glob("data/feishu_tables/[0-9]*.json"),
|
||||
key=lambda p: int(Path(p).stem.split("-")[0]),
|
||||
):
|
||||
d = json.load(open(path, encoding="utf-8"))
|
||||
name = d.get("name")
|
||||
fid_to_name = d.get("field_id_to_name", {})
|
||||
name_fid = id_fid = None
|
||||
for fid, fname in fid_to_name.items():
|
||||
cname = (fname or "").rstrip("?").rstrip()
|
||||
if cname in ("达人名称", "达人昵称"):
|
||||
name_fid = fid
|
||||
elif cname in ("达人id", "达人id号", "达人ID", "星图ID"):
|
||||
id_fid = fid
|
||||
n_with_id = n_with_name = 0
|
||||
n_records = len(d.get("records", []))
|
||||
for rec in d.get("records", []):
|
||||
if id_fid and rec.get(id_fid):
|
||||
n_with_id += 1
|
||||
if name_fid and rec.get(name_fid):
|
||||
n_with_name += 1
|
||||
per_table.append((name, n_records, n_with_id, n_with_name))
|
||||
total_records += n_records
|
||||
total_with_id += n_with_id
|
||||
total_with_name += n_with_name
|
||||
|
||||
print(f"{'款式':<14} {'总记录':>5} {'有ID':>5} {'有名称':>6} {'填充率(ID)':>10}")
|
||||
for name, n, nid, nnm in per_table:
|
||||
rate = f"{100*nid/n:.1f}%" if n else "-"
|
||||
print(f"{name:<14} {n:>5} {nid:>5} {nnm:>6} {rate:>10}")
|
||||
print()
|
||||
print(f"总计 {total_records} 条")
|
||||
print(f" 有 creator_id: {total_with_id} ({100*total_with_id/total_records:.1f}%)")
|
||||
print(f" 有 creator_name:{total_with_name} ({100*total_with_name/total_records:.1f}%)")
|
||||
@@ -0,0 +1,179 @@
|
||||
"""查看每个平台 24 款式的采集状态。
|
||||
|
||||
判定规则(对单个平台):
|
||||
OK - v2 JSON 文件存在,且 mtime 在 --stale-hours 小时内
|
||||
STALE - v2 JSON 文件存在,但 mtime 超过 --stale-hours 小时(需要重跑)
|
||||
MISSING - v2 JSON 文件不存在(进程崩溃/被中断)
|
||||
EMPTY - v2 JSON 存在但 matched=0、filled=0、total=0 (该款式确实没数据)
|
||||
HALF - v2 JSON 存在但 results 里至少有一条 error (部分采集成功)
|
||||
|
||||
用法:
|
||||
python data/tools/check_status.py # 看三个平台
|
||||
python data/tools/check_status.py --platform xt # 只看星图
|
||||
python data/tools/check_status.py --stale-hours 4 # 4 小时以上算陈旧
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
PROJECT_DIR = PATHS.module_root
|
||||
V2_DIR = PATHS.normalized_root / "v2_results"
|
||||
|
||||
# 共享工具
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.v2_filename import parse_v2_filename # noqa: E402
|
||||
|
||||
PLATFORMS = {
|
||||
"bili": {"name": "B 站", "label": "bili"},
|
||||
"pgy": {"name": "小红书蒲公英", "label": "xhs"},
|
||||
"xt": {"name": "星图", "label": "xt"},
|
||||
}
|
||||
|
||||
# 款式总数 1-24
|
||||
STYLE_INDICES = list(range(1, 25))
|
||||
|
||||
|
||||
def scan_platform(platform_key: str, stale_hours: float) -> dict[int, dict]:
|
||||
"""扫一个平台 v2_results,返回 {index: {status, mtime, matched, ...}}"""
|
||||
label = PLATFORMS[platform_key]["label"]
|
||||
found: dict[int, dict] = {}
|
||||
for path in V2_DIR.glob("*.json"):
|
||||
idx, name, plat, _ = parse_v2_filename(path.name)
|
||||
if plat != label or idx is None:
|
||||
continue
|
||||
mtime = dt.datetime.fromtimestamp(path.stat().st_mtime)
|
||||
age_h = (dt.datetime.now() - mtime).total_seconds() / 3600.0
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
data = {}
|
||||
matched = data.get("matched") or data.get("updated") or 0
|
||||
filled = data.get("filled") or matched
|
||||
total = (data.get("total") or data.get("total_b_records") or 0)
|
||||
results = data.get("results") or data.get("details") or []
|
||||
has_error = any(isinstance(r, dict) and r.get("error") for r in results)
|
||||
|
||||
if total == 0 and matched == 0 and not has_error:
|
||||
status = "EMPTY"
|
||||
elif has_error and matched > 0:
|
||||
status = "HALF"
|
||||
elif has_error:
|
||||
status = "HALF"
|
||||
elif age_h > stale_hours:
|
||||
status = "STALE"
|
||||
else:
|
||||
status = "OK"
|
||||
found[idx] = {
|
||||
"status": status,
|
||||
"name": name,
|
||||
"path": path.name,
|
||||
"mtime": mtime,
|
||||
"age_h": age_h,
|
||||
"matched": matched,
|
||||
"filled": filled,
|
||||
"total": total,
|
||||
"has_error": has_error,
|
||||
}
|
||||
return found
|
||||
|
||||
|
||||
def color(s: str, code: str) -> str:
|
||||
if not sys.stdout.isatty():
|
||||
return s
|
||||
return f"\033[{code}m{s}\033[0m"
|
||||
|
||||
|
||||
STATUS_COLOR = {
|
||||
"OK": ("32",), # green
|
||||
"STALE": ("33",), # yellow
|
||||
"MISSING": ("31",), # red
|
||||
"EMPTY": ("90",), # grey
|
||||
"HALF": ("33",), # yellow
|
||||
}
|
||||
|
||||
|
||||
def fmt_status(s: str) -> str:
|
||||
code = STATUS_COLOR.get(s, ("",))[0]
|
||||
label_map = {
|
||||
"OK": " OK ",
|
||||
"STALE": " STALE ",
|
||||
"MISSING": "MISSING",
|
||||
"EMPTY": " EMPTY ",
|
||||
"HALF": " HALF ",
|
||||
}
|
||||
return color(label_map.get(s, s), code)
|
||||
|
||||
|
||||
def print_platform(platform_key: str, info: dict, stale_hours: float) -> tuple[int, int, int]:
|
||||
name = info["name"]
|
||||
by_idx = scan_platform(platform_key, stale_hours)
|
||||
counts = {"OK": 0, "STALE": 0, "MISSING": 0, "EMPTY": 0, "HALF": 0}
|
||||
print(f"\n{'=' * 78}")
|
||||
print(f" {name} (stale > {stale_hours}h)")
|
||||
print(f"{'=' * 78}")
|
||||
print(f" {'款式':>4} {'状态':<8} {'名称':<14} {'matched':>7} {'total':>5} "
|
||||
f"{'err':>3} {'mtime':<19} {'age':>5}")
|
||||
print(" " + "-" * 74)
|
||||
for idx in STYLE_INDICES:
|
||||
if idx in by_idx:
|
||||
r = by_idx[idx]
|
||||
counts[r["status"]] += 1
|
||||
err_n = sum(1 for x in (r.get("path"),) if x and "error" in x) # placeholder
|
||||
# 计算 has_error 计数
|
||||
try:
|
||||
data = json.loads((V2_DIR / r["path"]).read_text(encoding="utf-8"))
|
||||
results = data.get("results") or data.get("details") or []
|
||||
err_n = sum(1 for x in results if isinstance(x, dict) and x.get("error"))
|
||||
except Exception:
|
||||
err_n = 0
|
||||
mtime_str = r["mtime"].strftime("%Y-%m-%d %H:%M:%S")
|
||||
print(f" {idx:>4} {fmt_status(r['status'])} {r['name']:<14} "
|
||||
f"{r['matched']:>7} {r['total']:>5} {err_n:>3} "
|
||||
f"{mtime_str:<19} {r['age_h']:>4.1f}h")
|
||||
else:
|
||||
counts["MISSING"] += 1
|
||||
print(f" {idx:>4} {fmt_status('MISSING')} {'-':<14} {'-':>7} {'-':>5} "
|
||||
f"{'-':>3} {'-':<19} {'-':>5}")
|
||||
print(" " + "-" * 74)
|
||||
total = sum(counts.values())
|
||||
print(f" 小计: OK={counts['OK']} STALE={counts['STALE']} "
|
||||
f"MISSING={counts['MISSING']} EMPTY={counts['EMPTY']} "
|
||||
f"HALF={counts['HALF']} / 共 {total}")
|
||||
# 待补跑 = MISSING + STALE + HALF(只算有 matched>0 的)
|
||||
need_retry = counts["MISSING"] + counts["STALE"]
|
||||
print(f" 待补跑: {need_retry} 个款式")
|
||||
return counts["MISSING"] + counts["STALE"], counts["HALF"], total
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="查看各平台 24 款式采集状态")
|
||||
ap.add_argument("--platform", type=str, default="bili,pgy,xt",
|
||||
help="逗号分隔平台键 bili/pgy/xt,默认全看")
|
||||
ap.add_argument("--stale-hours", type=float, default=26.0,
|
||||
help="超过多少小时算 STALE(默认 26h = 每日一次)")
|
||||
args = ap.parse_args()
|
||||
|
||||
keys = [k.strip() for k in args.platform.split(",") if k.strip()]
|
||||
for k in keys:
|
||||
if k not in PLATFORMS:
|
||||
print(f"[ERROR] 未知平台: {k} 可选: {','.join(PLATFORMS.keys())}")
|
||||
return 1
|
||||
|
||||
print(f"扫描目录: {V2_DIR}")
|
||||
grand_retry = grand_half = grand_total = 0
|
||||
for k in keys:
|
||||
retry_n, half_n, total_n = print_platform(k, PLATFORMS[k], args.stale_hours)
|
||||
grand_retry += retry_n
|
||||
grand_half += half_n
|
||||
grand_total += total_n
|
||||
print(f"\n{'=' * 78}")
|
||||
print(f" 汇总: 待补跑 {grand_retry} 款 (HALF {grand_half} 款) / 共 {grand_total} 款")
|
||||
print(f" 补跑命令: python data/tools/retry_failed.py")
|
||||
print(f"{'=' * 78}\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,188 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import (
|
||||
PATHS,
|
||||
resolve_layer_output,
|
||||
)
|
||||
|
||||
# Path setup: scrapers are in project root, feishu_comment_batch is in data/tools/
|
||||
_TOOLS_DIR = PATHS.tools_root
|
||||
_PROJECT_ROOT = PATHS.module_root
|
||||
|
||||
from scrapling.fetchers import DynamicSession
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime import bilibili_comment_scraper
|
||||
from gyxx_flow.modules.content_marketing.runtime import douyin_comment_scraper
|
||||
from gyxx_flow.modules.content_marketing.runtime import xiaohongshu_comment_scraper
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.feishu_comment_batch import detect_platform, normalize_note_url
|
||||
|
||||
|
||||
def unique_note_urls(source_csv: Path, platform: str) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
urls: list[str] = []
|
||||
with source_csv.open(encoding="utf-8-sig", newline="") as file:
|
||||
for row in csv.DictReader(file):
|
||||
url = normalize_note_url(row.get("note_url") or row.get("笔记链接") or "")
|
||||
if not url or detect_platform(url) != platform or url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
urls.append(url)
|
||||
return urls
|
||||
|
||||
|
||||
def collect_douyin(urls: list[str], headless: bool) -> list[dict[str, Any]]:
|
||||
"""Collect Douyin note metrics via browser (Playwright + XHR interception)."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
|
||||
def make_action(url: str):
|
||||
def action(page):
|
||||
page.wait_for_timeout(7000)
|
||||
metrics = douyin_comment_scraper.extract_note_metrics(page)
|
||||
rows.append({
|
||||
"platform": "douyin",
|
||||
"note_url": url,
|
||||
"like_count": metrics.get("like_count", ""),
|
||||
"favorite_count": metrics.get("favorite_count", ""),
|
||||
"share_count": metrics.get("share_count", ""),
|
||||
"comment_count": metrics.get("comment_count", ""),
|
||||
})
|
||||
print(f"douyin {len(rows)}/{len(urls)} {metrics} {url}", flush=True)
|
||||
|
||||
return action
|
||||
|
||||
with DynamicSession(
|
||||
headless=headless,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(douyin_comment_scraper.PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=douyin_comment_scraper.restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
for url in urls:
|
||||
try:
|
||||
normalized = douyin_comment_scraper.normalize_douyin_url(url)
|
||||
session.fetch(normalized, page_action=make_action(url), wait=300)
|
||||
except Exception as exc:
|
||||
rows.append({"platform": "douyin", "note_url": url, "like_count": "", "favorite_count": "", "share_count": "", "comment_count": "", "error": repr(exc)})
|
||||
print(f"douyin failed {url} {exc!r}", flush=True)
|
||||
return rows
|
||||
|
||||
|
||||
def collect_xiaohongshu(urls: list[str], headless: bool) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
|
||||
def make_action(url: str):
|
||||
def action(page):
|
||||
page.wait_for_timeout(7000)
|
||||
metrics = xiaohongshu_comment_scraper.extract_note_metrics(page)
|
||||
rows.append({"platform": "xiaohongshu", "note_url": url, **metrics})
|
||||
print(f"xiaohongshu {len(rows)}/{len(urls)} {metrics} {url}", flush=True)
|
||||
|
||||
return action
|
||||
|
||||
with DynamicSession(
|
||||
headless=headless,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(xiaohongshu_comment_scraper.PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=xiaohongshu_comment_scraper.restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
for url in urls:
|
||||
try:
|
||||
session.fetch(url, page_action=make_action(url), wait=300)
|
||||
except Exception as exc:
|
||||
rows.append({"platform": "xiaohongshu", "note_url": url, "like_count": "", "favorite_count": "", "share_count": "", "comment_count": "", "error": repr(exc)})
|
||||
print(f"xiaohongshu failed {url} {exc!r}", flush=True)
|
||||
return rows
|
||||
|
||||
|
||||
def collect_bilibili(urls: list[str], headless: bool) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
|
||||
def make_action(url: str):
|
||||
def action(page):
|
||||
page.wait_for_timeout(1500)
|
||||
video = bilibili_comment_scraper.resolve_video_info(page, url)
|
||||
metrics = video.get("note_metrics") or {}
|
||||
rows.append({"platform": "bilibili", "note_url": url, **metrics})
|
||||
print(f"bilibili {len(rows)}/{len(urls)} {metrics} {url}", flush=True)
|
||||
|
||||
return action
|
||||
|
||||
with DynamicSession(
|
||||
headless=headless,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(bilibili_comment_scraper.PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=bilibili_comment_scraper.restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
for url in urls:
|
||||
try:
|
||||
session.fetch(url, page_action=make_action(url), wait=300)
|
||||
except Exception as exc:
|
||||
rows.append({"platform": "bilibili", "note_url": url, "like_count": "", "favorite_count": "", "share_count": "", "comment_count": "", "error": repr(exc)})
|
||||
print(f"bilibili failed {url} {exc!r}", flush=True)
|
||||
return rows
|
||||
|
||||
|
||||
def write_rows(rows: list[dict[str, Any]], output: Path) -> None:
|
||||
output.parent.mkdir(exist_ok=True)
|
||||
fields = ["platform", "note_url", "like_count", "favorite_count", "share_count", "comment_count", "error"]
|
||||
with output.open("w", encoding="utf-8-sig", newline="") as file:
|
||||
writer = csv.DictWriter(file, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow({field: row.get(field, "") for field in fields})
|
||||
output.with_suffix(".json").write_text(json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-csv", required=True)
|
||||
parser.add_argument("--platform", choices=["douyin", "xiaohongshu", "bilibili"], required=True)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--headless", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
source_csv = Path(args.source_csv)
|
||||
output = resolve_layer_output(
|
||||
args.output, layer_root=PATHS.normalized_root, field="--output"
|
||||
)
|
||||
urls = unique_note_urls(source_csv, args.platform)
|
||||
print(f"{args.platform}: {len(urls)} unique notes", flush=True)
|
||||
|
||||
if args.platform == "douyin":
|
||||
rows = collect_douyin(urls, args.headless)
|
||||
elif args.platform == "xiaohongshu":
|
||||
rows = collect_xiaohongshu(urls, args.headless)
|
||||
else:
|
||||
rows = collect_bilibili(urls, args.headless)
|
||||
|
||||
write_rows(rows, output)
|
||||
print(f"wrote {output}", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+321
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""构建全款式同权经营看板所需的确定性指标与榜单。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import date, timedelta
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
try:
|
||||
from .db import get_conn
|
||||
except ImportError:
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
|
||||
|
||||
MIN_CONVERSION_RANK_VISITORS = 30
|
||||
MIN_REFUND_RANK_SALES = 5
|
||||
ROOT = PATHS.module_root
|
||||
IMAGE_ROOTS = [
|
||||
ROOT,
|
||||
Path(os.environ.get("GYXX_PRODUCT_ASSET_ROOT", str(ROOT))),
|
||||
]
|
||||
PLATFORM_IMAGE_CODES = {"天猫": "tm", "京东": "jd", "抖音电商": "dy"}
|
||||
STYLE_CATEGORY_ORDER = ("都市机能", "户外机能", "都市新贵", "智性通勤", "都市新旅", "都市运动", "未分类")
|
||||
|
||||
|
||||
def load_style_categories(style_names: list[str]) -> dict[str, str]:
|
||||
if not style_names:
|
||||
return {}
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT name, COALESCE(NULLIF(BTRIM(style_category), ''), '未分类')
|
||||
FROM cmt_styles
|
||||
WHERE name = ANY(%s)
|
||||
""",
|
||||
(style_names,),
|
||||
)
|
||||
return {name: category for name, category in cur.fetchall()}
|
||||
|
||||
|
||||
def load_creator_connections(metric_date: date, style_names: list[str]) -> dict[str, dict[str, Any]]:
|
||||
if not style_names:
|
||||
return {}
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT s.name,
|
||||
COUNT(DISTINCT c.creator_id) FILTER (WHERE c.creator_id IS NOT NULL) creator_count,
|
||||
ARRAY_REMOVE(ARRAY_AGG(DISTINCT NULLIF(BTRIM(c.content_direction), '')), NULL) directions
|
||||
FROM cmt_styles s
|
||||
LEFT JOIN cmt_cooperations c
|
||||
ON c.style_id = s.id
|
||||
AND COALESCE(c.publish_time::date, c.cooperation_date) BETWEEN %s AND %s
|
||||
WHERE s.name = ANY(%s)
|
||||
GROUP BY s.name
|
||||
""",
|
||||
(metric_date - timedelta(days=29), metric_date, style_names),
|
||||
)
|
||||
return {
|
||||
style_name: {"creator_count": creator_count or 0, "directions": directions or []}
|
||||
for style_name, creator_count, directions in cur.fetchall()
|
||||
}
|
||||
|
||||
|
||||
def load_style_images(style_names: list[str]) -> dict[tuple[str, str], dict[str, Any]]:
|
||||
"""为三平台画像卡统一选择同款天猫高点击率主图。"""
|
||||
if not style_names:
|
||||
return {}
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT style_name, platform, collect_date, local_image_path, creative_image_url,
|
||||
COALESCE(impressions, 0) impressions, click_rate
|
||||
FROM main_image_creatives
|
||||
WHERE style_name = ANY(%s)
|
||||
AND platform = 'tm'
|
||||
AND image_downloaded = TRUE
|
||||
AND local_image_path IS NOT NULL
|
||||
AND COALESCE(impressions, 0) >= 100
|
||||
ORDER BY COALESCE(click_rate, 0) DESC, COALESCE(impressions, 0) DESC, collect_date DESC
|
||||
""",
|
||||
(style_names,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
by_style: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for style_name, platform, collect_date, local_path, url, impressions, click_rate in rows:
|
||||
relative = Path(str(local_path))
|
||||
absolute = next((root / relative for root in IMAGE_ROOTS if (root / relative).exists()), None)
|
||||
if absolute is None:
|
||||
continue
|
||||
by_style[style_name].append({
|
||||
"local_image_path": str(absolute),
|
||||
"image_url": url,
|
||||
"image_platform": {"tm": "天猫", "jd": "京东", "dy": "抖音电商"}.get(platform, platform),
|
||||
"image_date": collect_date,
|
||||
"image_impressions": impressions,
|
||||
"image_ctr": float(click_rate) if click_rate is not None else None,
|
||||
})
|
||||
result: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for style_name in style_names:
|
||||
candidates = by_style.get(style_name, [])
|
||||
for target in ("天猫", "京东", "抖音电商"):
|
||||
if candidates:
|
||||
result[(style_name, target)] = candidates[0]
|
||||
return result
|
||||
|
||||
|
||||
def _rate(numerator: Any, denominator: Any) -> float | None:
|
||||
if numerator is None or denominator is None or denominator <= 0:
|
||||
return None
|
||||
return round(float(numerator) / float(denominator), 4)
|
||||
|
||||
|
||||
def _percentile(values: list[float], fraction: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
index = min(len(ordered) - 1, max(0, round((len(ordered) - 1) * fraction)))
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def _persona_text(persona: dict[str, Any] | None) -> str:
|
||||
if not persona:
|
||||
return "无数据"
|
||||
labels = [persona.get(key) for key in ("gender_top", "age_top", "city_top", "buying_power_top") if persona.get(key)]
|
||||
return "、".join(labels) if labels else "无数据"
|
||||
|
||||
|
||||
def _interest_hypothesis(persona: dict[str, Any] | None, directions: list[str]) -> str:
|
||||
if not persona:
|
||||
return "无数据"
|
||||
source = " ".join(str(value or "") for value in persona.values())
|
||||
interests: list[str] = []
|
||||
if any(term in source for term in ("女性", "女")):
|
||||
interests.extend(["穿搭", "健身", "旅行摄影"])
|
||||
if any(term in source for term in ("男性", "男")):
|
||||
interests.extend(["数码3C", "城市通勤", "户外出行"])
|
||||
if any(term in source for term in ("18-24", "18-25", "25-29")):
|
||||
interests.extend(["潮流内容", "社交分享"])
|
||||
if any(term in source for term in ("26-35", "30-34", "35-39")):
|
||||
interests.extend(["职场效率", "短途出行"])
|
||||
if any(term in source for term in ("56岁以上", "46-55", "45-49", "50")):
|
||||
interests.extend(["品质消费", "舒适出行"])
|
||||
if any(term in source for term in ("土豪", "高级白领", "L4", "L5")):
|
||||
interests.extend(["品质材质", "设计感"])
|
||||
interests.extend(direction for direction in directions if direction)
|
||||
unique = list(dict.fromkeys(interests))[:4]
|
||||
return "兴趣推测：" + "、".join(unique) if unique else "兴趣推测：无足够标签"
|
||||
|
||||
|
||||
def _optimization(role: str, has_persona: bool) -> str:
|
||||
persona_action = "按画像扩展相似人群" if has_persona else "先补采人群画像"
|
||||
if role == "conversion":
|
||||
return f"高转化扩量：{persona_action}，逐步增加流量并监控退款订单比"
|
||||
return f"高访客提转化：{persona_action}，优化主图/详情页/利益点并做加购再营销"
|
||||
|
||||
|
||||
def _diagnose(row: dict[str, Any], visitor_q75: float, conversion_median: float, cart_median: float, refund_q75: float) -> str:
|
||||
if row.get("visitors") is None:
|
||||
return "无数据"
|
||||
conversion = row.get("conversion_rate")
|
||||
refund = row.get("refund_rate")
|
||||
cart_rate = row.get("cart_rate")
|
||||
if refund is not None and refund >= max(0.3, refund_q75):
|
||||
return "退款压力"
|
||||
if row["visitors"] >= visitor_q75 and conversion is not None and conversion >= conversion_median:
|
||||
return "高流量高转化"
|
||||
if row["visitors"] >= visitor_q75:
|
||||
return "高流量·转化待提"
|
||||
if conversion is not None and conversion >= conversion_median * 1.25:
|
||||
return "低流量·高转化可放量"
|
||||
if cart_rate is not None and cart_rate < cart_median * 0.7:
|
||||
return "加购偏弱"
|
||||
return "常规观察"
|
||||
|
||||
|
||||
def build_dashboard_facts(
|
||||
report_date: str,
|
||||
tracking_rows: list[dict[str, Any]],
|
||||
platform_metrics: list[dict[str, Any]],
|
||||
personas: list[dict[str, Any]],
|
||||
connections: dict[str, dict[str, Any]],
|
||||
style_images: dict[tuple[str, str], dict[str, Any]] | None = None,
|
||||
daily_notes: list[dict[str, Any]] | None = None,
|
||||
style_categories: dict[str, str] | None = None,
|
||||
style_signals: dict[str, dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
styles = []
|
||||
for source in tracking_rows:
|
||||
row = dict(source)
|
||||
row["cart_rate"] = _rate(row.get("cart_users"), row.get("visitors"))
|
||||
row["conversion_rate"] = _rate(row.get("sales"), row.get("visitors"))
|
||||
row["refund_rate"] = _rate(row.get("refund_orders"), row.get("sales"))
|
||||
connection = connections.get(row["style_name"], {})
|
||||
row["creator_connections_30d"] = int(connection.get("creator_count") or 0)
|
||||
row["content_directions_30d"] = connection.get("directions") or []
|
||||
row["style_category"] = (style_categories or {}).get(row["style_name"]) or "未分类"
|
||||
row["multi_source_signals"] = (style_signals or {}).get(row["style_name"], {})
|
||||
row["recent_note_exposure"] = row["multi_source_signals"].get("recent_note_exposure")
|
||||
styles.append(row)
|
||||
|
||||
visitors = [float(row["visitors"]) for row in styles if row.get("visitors") is not None]
|
||||
conversions = [float(row["conversion_rate"]) for row in styles if row.get("conversion_rate") is not None]
|
||||
carts = [float(row["cart_rate"]) for row in styles if row.get("cart_rate") is not None]
|
||||
refunds = [float(row["refund_rate"]) for row in styles if row.get("refund_rate") is not None]
|
||||
visitor_q75 = _percentile(visitors, 0.75)
|
||||
conversion_median = _percentile(conversions, 0.5)
|
||||
cart_median = _percentile(carts, 0.5)
|
||||
refund_q75 = _percentile(refunds, 0.75)
|
||||
for row in styles:
|
||||
base = _diagnose(row, visitor_q75, conversion_median, cart_median, refund_q75)
|
||||
signal = row["multi_source_signals"]
|
||||
evidence = ["经营"]
|
||||
if signal.get("persona"):
|
||||
evidence.append("画像")
|
||||
if signal.get("note_count"):
|
||||
evidence.append("营销笔记")
|
||||
if signal.get("comment_count"):
|
||||
evidence.append("评论")
|
||||
if signal.get("review_count"):
|
||||
evidence.append("商品评价")
|
||||
if signal.get("creative_ctr") is not None:
|
||||
evidence.append("主图")
|
||||
row["evidence_summary"] = "、".join(evidence)
|
||||
if (signal.get("review_negative_rate") or 0) >= 0.20:
|
||||
row["diagnosis"] = f"{base}·评价风险" if base != "无数据" else "评价风险"
|
||||
elif (signal.get("comment_negative_rate") or 0) >= 0.20:
|
||||
row["diagnosis"] = f"{base}·评论风险" if base != "无数据" else "评论风险"
|
||||
else:
|
||||
row["diagnosis"] = base
|
||||
|
||||
category_rank = {name: index for index, name in enumerate(STYLE_CATEGORY_ORDER)}
|
||||
ordered_styles = sorted(
|
||||
styles,
|
||||
key=lambda row: (
|
||||
category_rank.get(row["style_category"], len(category_rank)),
|
||||
row["style_category"],
|
||||
row["style_name"],
|
||||
),
|
||||
)
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in ordered_styles:
|
||||
grouped[row["style_category"]].append(row)
|
||||
style_groups = [
|
||||
{"style_category": category, "styles": grouped[category]}
|
||||
for category in sorted(grouped, key=lambda value: (category_rank.get(value, len(category_rank)), value))
|
||||
]
|
||||
|
||||
def top(metric: str, eligible=lambda row: True) -> list[dict[str, Any]]:
|
||||
rows = [row for row in styles if row.get(metric) is not None and eligible(row)]
|
||||
return sorted(rows, key=lambda row: (row[metric], row.get("visitors") or 0), reverse=True)[:5]
|
||||
|
||||
rankings = {
|
||||
"conversion": top("conversion_rate", lambda row: (row.get("visitors") or 0) >= MIN_CONVERSION_RANK_VISITORS),
|
||||
"visitors": top("visitors"),
|
||||
"cart_users": top("cart_users"),
|
||||
"sales": top("sales"),
|
||||
"refund_rate": top("refund_rate", lambda row: (row.get("sales") or 0) >= MIN_REFUND_RANK_SALES),
|
||||
}
|
||||
|
||||
persona_index = {(row.get("style_name"), row.get("platform")): row for row in personas}
|
||||
directions = {row["style_name"]: row.get("content_directions_30d", []) for row in styles}
|
||||
platform_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for source in platform_metrics:
|
||||
row = dict(source)
|
||||
row["conversion_rate"] = _rate(row.get("sales"), row.get("visitors"))
|
||||
platform_groups[row.get("platform")].append(row)
|
||||
|
||||
platform_personas = []
|
||||
preferred_platforms = ["天猫", "京东", "抖音电商"]
|
||||
ordered_platforms = [name for name in preferred_platforms if name in platform_groups]
|
||||
ordered_platforms.extend(sorted(name for name in platform_groups if name not in preferred_platforms))
|
||||
for platform in ordered_platforms:
|
||||
rows = platform_groups[platform]
|
||||
conversion_eligible = [row for row in rows if (row.get("visitors") or 0) >= MIN_CONVERSION_RANK_VISITORS and row.get("conversion_rate") is not None]
|
||||
conversion_winner = max(conversion_eligible, key=lambda row: row["conversion_rate"]) if conversion_eligible else None
|
||||
visitor_winner = max(rows, key=lambda row: row.get("visitors") or -1) if rows else None
|
||||
|
||||
def winner_payload(winner: dict[str, Any] | None, role: str) -> dict[str, Any] | None:
|
||||
if not winner:
|
||||
return None
|
||||
persona = persona_index.get((winner["style_name"], platform))
|
||||
return {
|
||||
**winner,
|
||||
**((style_images or {}).get((winner["style_name"], platform)) or {}),
|
||||
"persona_text": _persona_text(persona),
|
||||
"persona_date": persona.get("data_date") if persona else None,
|
||||
"interest_hypothesis": _interest_hypothesis(persona, directions.get(winner["style_name"], [])),
|
||||
"optimization": _optimization(role, bool(persona)),
|
||||
}
|
||||
|
||||
platform_personas.append({
|
||||
"platform": platform,
|
||||
"top_conversion": winner_payload(conversion_winner, "conversion"),
|
||||
"top_visitors": winner_payload(visitor_winner, "visitors"),
|
||||
})
|
||||
|
||||
return {
|
||||
"report_date": report_date,
|
||||
"styles": ordered_styles,
|
||||
"style_groups": style_groups,
|
||||
"rankings": rankings,
|
||||
"platform_personas": platform_personas,
|
||||
"daily_new_notes": sorted(daily_notes or [], key=lambda row: row.get("view_count") or 0, reverse=True),
|
||||
"definitions": {
|
||||
"conversion_rate": "销量/访客；榜单访客门槛≥30",
|
||||
"cart_rate": "加购人数/访客",
|
||||
"refund_rate": "退款订单/当日销量，可能含跨期退款",
|
||||
"creator_connections": "近30天去重达人数量",
|
||||
"sales_7d_total": "报告日及前6日累计销量",
|
||||
"sales_forecast_7d": "最近3日日均销量×7；趋势相对前4日日均，仅作短期估算",
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Daily SKU marketing operations report: Hermes analysis + Feishu dashboard and full report
|
||||
REM Register: schtasks /Create /SC DAILY /TN YingxiaoYunying_DailyMarketingReport /TR %PROJECT_DIR%\data\tools\daily_marketing_report.bat /ST 10:00 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing\scheduled
|
||||
set PYTHON=%GYXX_PYTHON%
|
||||
where %PYTHON% >nul 2>&1 || set PYTHON=python
|
||||
|
||||
set PYTHONIOENCODING=utf-8
|
||||
set LARK_CLI_NO_PROXY=1
|
||||
set LARKSUITE_CLI_NO_UPDATE_NOTIFIER=1
|
||||
set LARKSUITE_CLI_NO_SKILLS_NOTIFIER=1
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
|
||||
set TS=%date:~0,4%%date:~5,2%%date:~8,2%_%time:~0,2%%time:~3,2%%time:~6,2%
|
||||
set TS=%TS: =0%
|
||||
set LOG_FILE=%LOG_DIR%\daily_marketing_report_%TS%.log
|
||||
|
||||
echo === Daily marketing report started at %date% %time% === > "%LOG_FILE%"
|
||||
echo PROJECT_DIR=%PROJECT_DIR% >> "%LOG_FILE%"
|
||||
|
||||
REM Wait for today's daily_run to finish (poll every 60s, max 90 min), so that
|
||||
REM cmt_notes is synced before the report reads yesterday's new notes.
|
||||
set TODAY=%date:~0,4%%date:~5,2%%date:~8,2%
|
||||
set /a WAITED=0
|
||||
:wait_daily_run
|
||||
set DAILY_DONE=
|
||||
for %%F in ("%LOG_DIR%\daily_run_%TODAY%_*.log") do findstr /C:"Daily run finished" /C:"Daily run skipped" "%%F" >nul 2>&1 && set DAILY_DONE=1
|
||||
if defined DAILY_DONE goto daily_run_ready
|
||||
if %WAITED% GEQ 5400 goto daily_run_timeout
|
||||
timeout /t 60 /nobreak >nul
|
||||
set /a WAITED+=60
|
||||
goto wait_daily_run
|
||||
:daily_run_timeout
|
||||
echo WARN: daily_run not finished after 90 min wait, generating report anyway >> "%LOG_FILE%"
|
||||
:daily_run_ready
|
||||
|
||||
cd /d "%PROJECT_DIR%"
|
||||
call %PYTHON% -u -X utf8 daily_marketing_report.py --send >> "%LOG_FILE%" 2>&1
|
||||
set RC=%ERRORLEVEL%
|
||||
|
||||
echo daily_marketing_report exit=%RC% >> "%LOG_FILE%"
|
||||
echo === Daily marketing report finished at %date% %time% (rc=%RC%) === >> "%LOG_FILE%"
|
||||
|
||||
endlocal & exit /b %RC%
|
||||
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""构造并发送时尚包袋品牌经营日报飞书 Card 2.0。"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from PIL import Image
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
|
||||
LARK_PROFILE = "hermes-analyzer"
|
||||
LARK_CLI = shutil.which("lark-cli.cmd") or shutil.which("lark-cli") or "lark-cli"
|
||||
MAX_CARD_IMAGE_WIDTH = 1500
|
||||
|
||||
|
||||
def _lark_command() -> list[str]:
|
||||
run_js = (
|
||||
Path(os.environ.get("APPDATA", str(Path.home())))
|
||||
/ "npm" / "node_modules" / "@larksuite" / "cli" / "scripts" / "run.js"
|
||||
)
|
||||
if not run_js.exists():
|
||||
run_js = Path(os.path.expandvars(r"%APPDATA%\npm\node_modules\@larksuite\cli\scripts\run.js"))
|
||||
if run_js.exists():
|
||||
return [shutil.which("node") or "node", str(run_js)]
|
||||
return [LARK_CLI]
|
||||
|
||||
|
||||
def _section_lines(report: str, start: str, end: str | None, limit: int = 3) -> list[str]:
|
||||
start_match = re.search(rf"(?m)^\s*#*\s*{re.escape(start)}\s*$", report)
|
||||
if not start_match:
|
||||
return []
|
||||
tail = report[start_match.end():]
|
||||
if end:
|
||||
end_match = re.search(rf"(?m)^\s*#*\s*{re.escape(end)}", tail)
|
||||
if end_match:
|
||||
tail = tail[:end_match.start()]
|
||||
lines = []
|
||||
for raw in tail.splitlines():
|
||||
value = re.sub(r"^\s*\d+[\.、]\s*", "", raw).strip()
|
||||
if not value or value.startswith("![") or value.startswith("|"):
|
||||
continue
|
||||
lines.append(value[:180])
|
||||
if len(lines) >= limit:
|
||||
break
|
||||
return lines
|
||||
|
||||
|
||||
def _metric_column(value: int, label: str, emphasis: bool = False) -> dict[str, Any]:
|
||||
number_size = "heading-1" if emphasis else "heading-2"
|
||||
return {
|
||||
"tag": "column",
|
||||
"width": "weighted",
|
||||
"weight": 1,
|
||||
"background_style": "grey-50",
|
||||
"padding": "12px",
|
||||
"vertical_spacing": "2px",
|
||||
"elements": [
|
||||
{"tag": "markdown", "content": str(value), "text_size": number_size, "text_align": "center"},
|
||||
{"tag": "markdown", "content": f"<font color='grey'>{label}</font>", "text_size": "notation", "text_align": "center"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_dashboard_card(
|
||||
report_date: str,
|
||||
tracking_rows: list[dict[str, Any]],
|
||||
image_key: str,
|
||||
) -> dict[str, Any]:
|
||||
"""构造只包含核心灯号和看板图片的第一条消息。"""
|
||||
counts = Counter(row.get("light", "无判断") for row in tracking_rows)
|
||||
return {
|
||||
"schema": "2.0",
|
||||
"config": {
|
||||
"update_multi": True,
|
||||
"width_mode": "fill",
|
||||
"enable_forward": True,
|
||||
"summary": {"content": f"{report_date} 时尚包袋品牌经营看板"},
|
||||
},
|
||||
"header": {
|
||||
"title": {"tag": "plain_text", "content": "时尚包袋品牌经营看板"},
|
||||
"subtitle": {"tag": "plain_text", "content": f"{report_date} · Hermes 自动分析"},
|
||||
"template": "blue",
|
||||
"icon": {"tag": "standard_icon", "token": "chart_colorful"},
|
||||
"text_tag_list": [
|
||||
{"tag": "text_tag", "text": {"tag": "plain_text", "content": "数据看板"}, "color": "blue"}
|
||||
],
|
||||
},
|
||||
"body": {
|
||||
"direction": "vertical",
|
||||
"padding": "12px 12px 20px 12px",
|
||||
"vertical_spacing": "large",
|
||||
"elements": [
|
||||
{
|
||||
"tag": "column_set",
|
||||
"flex_mode": "none",
|
||||
"horizontal_spacing": "12px",
|
||||
"columns": [
|
||||
_metric_column(counts.get("🔴", 0), "红灯款式", emphasis=True),
|
||||
_metric_column(counts.get("🟡", 0), "黄灯款式"),
|
||||
_metric_column(counts.get("🟢", 0), "绿灯款式"),
|
||||
],
|
||||
},
|
||||
{
|
||||
"tag": "img",
|
||||
"img_key": image_key,
|
||||
"alt": {"tag": "plain_text", "content": "全域种草数据看板"},
|
||||
"scale_type": "fit_horizontal",
|
||||
"corner_radius": "8px",
|
||||
"preview": True,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _report_without_dashboard_reference(report: str) -> str:
|
||||
return re.sub(
|
||||
r"(?m)^\s*!\[全域种草数据看板\]\([^\n]+\)\s*\n?",
|
||||
"",
|
||||
report,
|
||||
count=1,
|
||||
).strip()
|
||||
|
||||
|
||||
def build_analysis_card(report_date: str, report: str) -> dict[str, Any]:
|
||||
"""构造包含完整日报正文的第二条消息，不做行数或字数裁剪。"""
|
||||
analysis = _report_without_dashboard_reference(report) or "无数据"
|
||||
return {
|
||||
"schema": "2.0",
|
||||
"config": {
|
||||
"update_multi": True,
|
||||
"width_mode": "fill",
|
||||
"enable_forward": True,
|
||||
"summary": {"content": f"{report_date} SKU营销运营日报分析"},
|
||||
},
|
||||
"header": {
|
||||
"title": {"tag": "plain_text", "content": "SKU营销运营日报分析"},
|
||||
"subtitle": {"tag": "plain_text", "content": f"{report_date} · 完整文字版"},
|
||||
"template": "turquoise",
|
||||
"icon": {"tag": "standard_icon", "token": "doc_colorful"},
|
||||
"text_tag_list": [
|
||||
{"tag": "text_tag", "text": {"tag": "plain_text", "content": "完整分析"}, "color": "turquoise"}
|
||||
],
|
||||
},
|
||||
"body": {
|
||||
"direction": "vertical",
|
||||
"padding": "12px 16px 20px 16px",
|
||||
"vertical_spacing": "medium",
|
||||
"elements": [{"tag": "markdown", "content": analysis}],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_daily_report_card(
|
||||
report_date: str,
|
||||
report: str,
|
||||
tracking_rows: list[dict[str, Any]],
|
||||
image_key: str,
|
||||
) -> dict[str, Any]:
|
||||
counts = Counter(row.get("light", "无判断") for row in tracking_rows)
|
||||
conclusions = _section_lines(report, "1. 今日经营概览", "2.", 5)
|
||||
if not conclusions:
|
||||
conclusions = _section_lines(report, "一、今日经营概览", "二、", 5)
|
||||
if not conclusions:
|
||||
conclusions = _section_lines(report, "一、核心经营结论", "二、", 3)
|
||||
if not conclusions:
|
||||
conclusions = _section_lines(report, "第一部分 整体经营分析", "第二部分 SKU分析排序", 5)
|
||||
actions = _section_lines(report, "6. 今日运营建议", None, 5)
|
||||
if not actions:
|
||||
actions = _section_lines(report, "七、明日业务动作", None, 5)
|
||||
if not actions:
|
||||
actions = _section_lines(report, "四、明日业务动作", None, 3)
|
||||
if not actions:
|
||||
actions = _section_lines(report, "第三部分 关键预警与建议", None, 5)
|
||||
conclusion_text = "\n".join(f"{idx}. {line}" for idx, line in enumerate(conclusions, 1)) or "无数据"
|
||||
action_text = "\n".join(f"{idx}. {line}" for idx, line in enumerate(actions, 1)) or "无数据"
|
||||
|
||||
return {
|
||||
"schema": "2.0",
|
||||
"config": {
|
||||
"update_multi": True,
|
||||
"width_mode": "fill",
|
||||
"enable_forward": True,
|
||||
"summary": {"content": f"{report_date} 时尚包袋品牌经营日报"},
|
||||
},
|
||||
"header": {
|
||||
"title": {"tag": "plain_text", "content": "时尚包袋品牌经营日报"},
|
||||
"subtitle": {"tag": "plain_text", "content": f"{report_date} · Hermes 自动分析"},
|
||||
"template": "blue",
|
||||
"icon": {"tag": "standard_icon", "token": "chart_colorful"},
|
||||
"text_tag_list": [
|
||||
{"tag": "text_tag", "text": {"tag": "plain_text", "content": "自动日报"}, "color": "blue"}
|
||||
],
|
||||
},
|
||||
"body": {
|
||||
"direction": "vertical",
|
||||
"padding": "12px 12px 20px 12px",
|
||||
"vertical_spacing": "large",
|
||||
"elements": [
|
||||
{
|
||||
"tag": "column_set",
|
||||
"flex_mode": "none",
|
||||
"horizontal_spacing": "12px",
|
||||
"columns": [
|
||||
_metric_column(counts.get("🔴", 0), "红灯款式", emphasis=True),
|
||||
_metric_column(counts.get("🟡", 0), "黄灯款式"),
|
||||
_metric_column(counts.get("🟢", 0), "绿灯款式"),
|
||||
],
|
||||
},
|
||||
{
|
||||
"tag": "img",
|
||||
"img_key": image_key,
|
||||
"alt": {"tag": "plain_text", "content": "全域种草数据看板"},
|
||||
"scale_type": "fit_horizontal",
|
||||
"corner_radius": "8px",
|
||||
"preview": True,
|
||||
},
|
||||
{"tag": "markdown", "content": f"**今日经营概览**\n{conclusion_text}"},
|
||||
{"tag": "markdown", "content": f"**今日运营建议**\n{action_text}"},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def prepare_feishu_image(source: Path, target: Path) -> Path:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with Image.open(source) as image:
|
||||
image = image.convert("RGB")
|
||||
if image.width > MAX_CARD_IMAGE_WIDTH:
|
||||
height = round(image.height * MAX_CARD_IMAGE_WIDTH / image.width)
|
||||
image = image.resize((MAX_CARD_IMAGE_WIDTH, height), Image.Resampling.LANCZOS)
|
||||
image.save(target, format="PNG", optimize=True)
|
||||
return target
|
||||
|
||||
|
||||
def upload_dashboard_image(image_path: Path) -> str:
|
||||
command = [
|
||||
*_lark_command(), "--profile", LARK_PROFILE,
|
||||
"im", "images", "create",
|
||||
"--data", json.dumps({"image_type": "message"}),
|
||||
"--file", f"image=./{image_path.name}",
|
||||
"--as", "bot", "--format", "json",
|
||||
]
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=image_path.parent,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(f"飞书看板图片上传失败: {completed.stderr or completed.stdout}")
|
||||
payload = json.loads(completed.stdout)
|
||||
image_key = payload.get("image_key") or (payload.get("data") or {}).get("image_key")
|
||||
if not image_key:
|
||||
raise RuntimeError("飞书看板图片上传成功但未返回 image_key")
|
||||
return image_key
|
||||
|
||||
|
||||
def send_card_to_recipients(
|
||||
card: dict[str, Any],
|
||||
recipients: Iterable[str],
|
||||
report_date: str,
|
||||
*,
|
||||
message_kind: str = "combined",
|
||||
) -> list[dict[str, Any]]:
|
||||
content = json.dumps(card, ensure_ascii=False, separators=(",", ":"))
|
||||
content_digest = hashlib.sha256(content.encode("utf-8")).hexdigest()[:10]
|
||||
results = []
|
||||
for open_id in recipients:
|
||||
kind_token = re.sub(r"[^a-z0-9]", "", message_kind.lower())[:1] or "m"
|
||||
idempotency_key = f"daily-{report_date}-{kind_token}-{content_digest}-{open_id[-6:]}"
|
||||
command = [
|
||||
*_lark_command(), "--profile", LARK_PROFILE,
|
||||
"im", "+messages-send",
|
||||
"--user-id", open_id,
|
||||
"--msg-type", "interactive",
|
||||
"--content", content,
|
||||
"--idempotency-key", idempotency_key,
|
||||
"--as", "bot", "--format", "json",
|
||||
]
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(f"飞书卡片发送失败({open_id}): {completed.stderr or completed.stdout}")
|
||||
results.append(json.loads(completed.stdout))
|
||||
return results
|
||||
|
||||
|
||||
def send_daily_report_cards(
|
||||
report_date: str,
|
||||
report: str,
|
||||
tracking_rows: list[dict[str, Any]],
|
||||
chart_path: Path,
|
||||
recipients: Iterable[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
prepared_dir = PATHS.tmp_root / "daily_report_card"
|
||||
prepared_dir.mkdir(parents=True, exist_ok=True)
|
||||
prepared = prepared_dir / f"{chart_path.stem}_feishu.png"
|
||||
prepare_feishu_image(chart_path, prepared)
|
||||
try:
|
||||
image_key = upload_dashboard_image(prepared)
|
||||
finally:
|
||||
prepared.unlink(missing_ok=True)
|
||||
recipients = tuple(recipients)
|
||||
dashboard_card = build_dashboard_card(report_date, tracking_rows, image_key)
|
||||
analysis_card = build_analysis_card(report_date, report)
|
||||
dashboard_results = send_card_to_recipients(
|
||||
dashboard_card, recipients, report_date, message_kind="dashboard"
|
||||
)
|
||||
analysis_results = send_card_to_recipients(
|
||||
analysis_card, recipients, report_date, message_kind="analysis"
|
||||
)
|
||||
return dashboard_results + analysis_results
|
||||
@@ -0,0 +1,518 @@
|
||||
"""使用 Pillow 生成全款式同权经营日报长图看板。"""
|
||||
|
||||
from pathlib import Path
|
||||
from textwrap import wrap
|
||||
from typing import Any, Callable
|
||||
|
||||
from PIL import Image, ImageDraw, ImageFont, ImageOps
|
||||
|
||||
try:
|
||||
from .daily_dashboard_analytics import build_dashboard_facts
|
||||
except ImportError:
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.daily_dashboard_analytics import build_dashboard_facts
|
||||
|
||||
|
||||
WIDTH, HEIGHT = 1600, 2500
|
||||
BG = "#F7F8FA"
|
||||
PANEL = "#FFFFFF"
|
||||
TEXT = "#18212F"
|
||||
MUTED = "#667085"
|
||||
GRID = "#E4E7EC"
|
||||
RED = "#E5484D"
|
||||
YELLOW = "#E8A317"
|
||||
GREEN = "#30A46C"
|
||||
BLUE = "#3B82F6"
|
||||
ORANGE = "#F97316"
|
||||
PURPLE = "#8B5CF6"
|
||||
LIGHT_BLUE = "#EFF6FF"
|
||||
LIGHT_RED = "#FFF1F1"
|
||||
LIGHT_YELLOW = "#FFF8E6"
|
||||
PLATFORM_COLORS = {"天猫": RED, "京东": BLUE, "抖音电商": TEXT}
|
||||
|
||||
|
||||
def _font(size: int, bold: bool = False):
|
||||
candidates = [
|
||||
Path(r"C:\Windows\Fonts\msyhbd.ttc" if bold else r"C:\Windows\Fonts\msyh.ttc"),
|
||||
Path(r"C:\Windows\Fonts\simhei.ttf"),
|
||||
]
|
||||
for path in candidates:
|
||||
if path.exists():
|
||||
return ImageFont.truetype(str(path), size)
|
||||
return ImageFont.load_default()
|
||||
|
||||
|
||||
def _pct(value: Any) -> float:
|
||||
try:
|
||||
return float(value or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _fmt_int(value: Any) -> str:
|
||||
if value is None:
|
||||
return "无数据"
|
||||
return f"{int(value):,}"
|
||||
|
||||
|
||||
def _fmt_rate(value: Any) -> str:
|
||||
if value is None:
|
||||
return "无数据"
|
||||
return f"{float(value):.1%}"
|
||||
|
||||
|
||||
def _sum_known(rows: list[dict[str, Any]], key: str) -> float | None:
|
||||
values = [float(row[key]) for row in rows if row.get(key) is not None]
|
||||
return sum(values) if values else None
|
||||
|
||||
|
||||
def _ratio(numerator: float | None, denominator: float | None) -> float | None:
|
||||
if numerator is None or denominator is None or denominator <= 0:
|
||||
return None
|
||||
return numerator / denominator
|
||||
|
||||
|
||||
def build_overview_cards(rows: list[dict[str, Any]], dashboard: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
today_sales = _sum_known(rows, "sales")
|
||||
yesterday_sales = _sum_known(rows, "sales_yesterday")
|
||||
baseline_sales = _sum_known(rows, "avg_sales_7d")
|
||||
today_visitors = _sum_known(rows, "visitors")
|
||||
yesterday_visitors = _sum_known(rows, "visitors_yesterday")
|
||||
baseline_visitors = _sum_known(rows, "avg_visitors_7d")
|
||||
notes = dashboard.get("daily_new_notes", [])
|
||||
note_views = [float(row["view_count"]) for row in notes if row.get("view_count") is not None]
|
||||
return [
|
||||
{
|
||||
"key": "sales", "label": "销量",
|
||||
"today": today_sales, "yesterday": yesterday_sales, "baseline_7d": baseline_sales,
|
||||
"formatter": _fmt_int,
|
||||
},
|
||||
{
|
||||
"key": "visitors", "label": "访客",
|
||||
"today": today_visitors, "yesterday": yesterday_visitors, "baseline_7d": baseline_visitors,
|
||||
"formatter": _fmt_int,
|
||||
},
|
||||
{
|
||||
"key": "conversion", "label": "整体转化率",
|
||||
"today": _ratio(today_sales, today_visitors),
|
||||
"yesterday": _ratio(yesterday_sales, yesterday_visitors),
|
||||
"baseline_7d": _ratio(baseline_sales, baseline_visitors),
|
||||
"formatter": _fmt_rate,
|
||||
},
|
||||
{
|
||||
"key": "content", "label": "新发内容曝光",
|
||||
"today": sum(note_views) if note_views else None,
|
||||
"yesterday": None, "baseline_7d": None,
|
||||
"formatter": _fmt_int,
|
||||
"note": f"昨日新发 {len(notes)} 篇｜历史曝光快照无数据",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def classify_style_action(row: dict[str, Any]) -> tuple[str, str]:
|
||||
sales = row.get("sales_change_yesterday")
|
||||
visitors = row.get("visitor_change_yesterday")
|
||||
conversion = row.get("conversion_change_yesterday")
|
||||
refund = row.get("refund_change_yesterday")
|
||||
if sales is not None and conversion is not None and refund is not None and sales < 0 and conversion < 0 and refund > 0:
|
||||
return "风险型", "查退款/控投放"
|
||||
if refund is not None and refund >= 0.30:
|
||||
return "风险型", "查退款/控投放"
|
||||
if sales is not None and visitors is not None and conversion is not None and sales > 0 and visitors > 0 and conversion >= 0:
|
||||
return "增长型", "加达人/备货"
|
||||
if visitors is not None and conversion is not None and visitors > 0 and conversion < 0:
|
||||
return "流量机会", "优化详情页"
|
||||
if (row.get("cart_change_yesterday") or 0) >= 0.20 or (row.get("recent_note_exposure") or 0) > 0:
|
||||
return "潜力型", "加曝光测试"
|
||||
if row.get("sales") is None:
|
||||
return "无数据", "补齐数据"
|
||||
return "稳定型", "维护转化"
|
||||
|
||||
|
||||
def select_chart_styles(rows: list[dict[str, Any]], limit: int = 6) -> list[dict[str, Any]]:
|
||||
"""保留旧调用兼容；新看板不会再只展示这些款式。"""
|
||||
anomalies = [row for row in rows if row.get("light") in {"🔴", "🟡"} and row.get("visitors") is not None]
|
||||
|
||||
def visitor_delta(row):
|
||||
return float(row.get("visitors") or 0) - float(row.get("avg_visitors_7d") or 0)
|
||||
|
||||
def impact(row):
|
||||
vd = abs(visitor_delta(row))
|
||||
cd = abs(float(row.get("cart_users") or 0) - float(row.get("avg_cart_users_7d") or 0))
|
||||
return vd + cd * 10
|
||||
|
||||
selected, used = [], set()
|
||||
groups = [
|
||||
sorted((row for row in anomalies if visitor_delta(row) < 0), key=lambda row: abs(visitor_delta(row)), reverse=True),
|
||||
sorted((row for row in anomalies if visitor_delta(row) > 0 or _pct(row.get("cart_change")) > 0), key=impact, reverse=True),
|
||||
sorted((row for row in anomalies if float(row.get("visitors") or 0) < 200), key=lambda row: max(abs(_pct(row.get("visitor_change"))), abs(_pct(row.get("cart_change")))), reverse=True),
|
||||
]
|
||||
for candidates, target in zip(groups, (2, 2, 1)):
|
||||
added = 0
|
||||
for row in candidates:
|
||||
if row["style_name"] in used:
|
||||
continue
|
||||
selected.append(row)
|
||||
used.add(row["style_name"])
|
||||
added += 1
|
||||
if added >= target or len(selected) >= limit:
|
||||
break
|
||||
for row in sorted(anomalies, key=impact, reverse=True):
|
||||
if len(selected) >= limit:
|
||||
break
|
||||
if row["style_name"] not in used:
|
||||
selected.append(row)
|
||||
used.add(row["style_name"])
|
||||
return selected[:limit]
|
||||
|
||||
|
||||
def _panel(draw: ImageDraw.ImageDraw, box: tuple[int, int, int, int], title: str, title_size: int = 23) -> None:
|
||||
draw.rounded_rectangle(box, radius=18, fill=PANEL)
|
||||
draw.text((box[0] + 20, box[1] + 16), title, font=_font(title_size, True), fill=TEXT)
|
||||
|
||||
|
||||
def _ranking_panel(
|
||||
draw: ImageDraw.ImageDraw,
|
||||
box: tuple[int, int, int, int],
|
||||
title: str,
|
||||
rows: list[dict[str, Any]],
|
||||
value_key: str,
|
||||
formatter: Callable[[Any], str],
|
||||
empty_text: str = "无数据",
|
||||
) -> None:
|
||||
_panel(draw, box, title, 21)
|
||||
x0, y0, x1, _ = box
|
||||
if not rows:
|
||||
lines = wrap(empty_text, width=22) or ["无数据"]
|
||||
for idx, line in enumerate(lines[:4]):
|
||||
draw.text((x0 + 20, y0 + 76 + idx * 28), line, font=_font(16), fill=MUTED)
|
||||
return
|
||||
max_value = max(float(row.get(value_key) or 0) for row in rows) or 1
|
||||
for idx, row in enumerate(rows[:5], 1):
|
||||
y = y0 + 63 + (idx - 1) * 37
|
||||
draw.text((x0 + 20, y), str(idx), font=_font(16, True), fill=BLUE)
|
||||
draw.text((x0 + 48, y), str(row.get("display_name") or row.get("style_name") or "无数据")[:11], font=_font(16), fill=TEXT)
|
||||
bar_x, bar_w = x0 + 190, x1 - x0 - 285
|
||||
width = max(2, bar_w * float(row.get(value_key) or 0) / max_value)
|
||||
draw.rounded_rectangle((bar_x, y + 5, bar_x + bar_w, y + 15), radius=5, fill=GRID)
|
||||
draw.rounded_rectangle((bar_x, y + 5, bar_x + width, y + 15), radius=5, fill=BLUE)
|
||||
draw.text((x1 - 84, y - 1), formatter(row.get(value_key)), font=_font(15, True), fill=TEXT)
|
||||
|
||||
|
||||
def _matrix_header(draw: ImageDraw.ImageDraw, x: int, y: int) -> None:
|
||||
headers = [
|
||||
("款式", 0), ("今日", 108), ("近7日", 150), ("访客", 200),
|
||||
("转化", 254), ("曝光", 310), ("预测7日", 366), ("判断", 442), ("动作", 536),
|
||||
]
|
||||
for label, offset in headers:
|
||||
draw.text((x + offset, y), label, font=_font(14, True), fill=MUTED)
|
||||
|
||||
|
||||
def _split_style_groups(style_groups: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
if len(style_groups) < 2:
|
||||
return style_groups, []
|
||||
total = sum(len(group.get("styles", [])) for group in style_groups)
|
||||
running = 0
|
||||
best_index = 1
|
||||
best_gap = total
|
||||
for index, group in enumerate(style_groups[:-1], 1):
|
||||
running += len(group.get("styles", []))
|
||||
gap = abs(total - running * 2)
|
||||
if gap < best_gap:
|
||||
best_index, best_gap = index, gap
|
||||
return style_groups[:best_index], style_groups[best_index:]
|
||||
|
||||
|
||||
def _style_matrix(draw: ImageDraw.ImageDraw, box: tuple[int, int, int, int], style_groups: list[dict[str, Any]]) -> None:
|
||||
total_styles = sum(len(group.get("styles", [])) for group in style_groups)
|
||||
_panel(draw, box, f"三、全部 {total_styles} 款SKU表现｜按风格分组", 25)
|
||||
x0, y0, x1, _ = box
|
||||
draw.text(
|
||||
(x1 - 760, y0 + 20),
|
||||
"近7日含报告日｜预测=近3日日均×7，箭头较前4日日均｜仅作趋势估算",
|
||||
font=_font(13),
|
||||
fill=MUTED,
|
||||
)
|
||||
groups = _split_style_groups(style_groups)
|
||||
column_boxes = [(x0 + 20, x0 + 750), (x0 + 790, x1 - 20)]
|
||||
for category_groups, (left, right) in zip(groups, column_boxes):
|
||||
header_y = y0 + 70
|
||||
draw.rounded_rectangle((left, header_y - 8, right, header_y + 28), radius=8, fill=LIGHT_BLUE)
|
||||
_matrix_header(draw, left + 10, header_y)
|
||||
cursor_y = header_y + 38
|
||||
for category_group in category_groups:
|
||||
category = category_group.get("style_category") or "未分类"
|
||||
rows = category_group.get("styles", [])
|
||||
draw.rounded_rectangle((left, cursor_y, right, cursor_y + 26), radius=7, fill=LIGHT_YELLOW)
|
||||
draw.text((left + 12, cursor_y + 3), f"{category}（{len(rows)}款）", font=_font(14, True), fill=ORANGE)
|
||||
cursor_y += 31
|
||||
for idx, row in enumerate(rows):
|
||||
y = cursor_y + idx * 38
|
||||
if idx % 2:
|
||||
draw.rectangle((left, y - 4, right, y + 28), fill="#FAFBFC")
|
||||
light_color = {"🔴": RED, "🟡": YELLOW, "🟢": GREEN}.get(row.get("light"), MUTED)
|
||||
draw.ellipse((left + 8, y + 4, left + 18, y + 14), fill=light_color)
|
||||
values = [
|
||||
(str(row.get("style_name") or "无数据")[:7], 24, TEXT),
|
||||
(_fmt_int(row.get("sales")), 114, TEXT),
|
||||
(_fmt_int(row.get("sales_7d_total")), 156, TEXT),
|
||||
(_fmt_int(row.get("visitors")), 206, TEXT),
|
||||
(_fmt_rate(row.get("conversion_rate")), 260, TEXT),
|
||||
(_fmt_int(row.get("recent_note_exposure")), 316, PURPLE if row.get("recent_note_exposure") else MUTED),
|
||||
]
|
||||
forecast = row.get("sales_forecast_7d")
|
||||
forecast_change = row.get("sales_forecast_change")
|
||||
if forecast is None:
|
||||
forecast_text, forecast_color = "无数据", MUTED
|
||||
elif forecast_change is None:
|
||||
forecast_text, forecast_color = f"·{_fmt_int(forecast)}", MUTED
|
||||
elif forecast_change > 0:
|
||||
forecast_text, forecast_color = f"↑{_fmt_int(forecast)}", GREEN
|
||||
elif forecast_change < 0:
|
||||
forecast_text, forecast_color = f"↓{_fmt_int(forecast)}", RED
|
||||
else:
|
||||
forecast_text, forecast_color = f"→{_fmt_int(forecast)}", MUTED
|
||||
diagnosis, action = classify_style_action(row)
|
||||
values.extend([
|
||||
(forecast_text, 372, forecast_color),
|
||||
(diagnosis, 448, RED if diagnosis == "风险型" else GREEN if diagnosis == "增长型" else TEXT),
|
||||
(action, 542, TEXT),
|
||||
])
|
||||
for value, offset, color in values:
|
||||
draw.text((left + offset, y), str(value)[:10], font=_font(12, offset in (24, 448, 542)), fill=color)
|
||||
cursor_y += len(rows) * 38 + 7
|
||||
|
||||
|
||||
def _change_text(current: float | None, baseline: float | None) -> tuple[str, str]:
|
||||
if current is None or baseline is None or baseline == 0:
|
||||
return "无数据", MUTED
|
||||
change = (current - baseline) / baseline
|
||||
if change > 0:
|
||||
return f"↑{change:.1%}", GREEN
|
||||
if change < 0:
|
||||
return f"↓{abs(change):.1%}", RED
|
||||
return "持平", MUTED
|
||||
|
||||
|
||||
def _overview_section(draw: ImageDraw.ImageDraw, box: tuple[int, int, int, int], cards: list[dict[str, Any]]) -> None:
|
||||
_panel(draw, box, "一、今日经营概览｜今日 vs 昨日 vs 7日均值", 25)
|
||||
x0, y0, x1, _ = box
|
||||
gap = 14
|
||||
card_width = (x1 - x0 - 40 - gap * 3) // 4
|
||||
for idx, card in enumerate(cards[:4]):
|
||||
left = x0 + 20 + idx * (card_width + gap)
|
||||
right = left + card_width
|
||||
top = y0 + 62
|
||||
draw.rounded_rectangle((left, top, right, box[3] - 18), radius=13, fill=LIGHT_BLUE if idx < 3 else "#F8F5FF")
|
||||
draw.text((left + 14, top + 12), card["label"], font=_font(18, True), fill=BLUE if idx < 3 else PURPLE)
|
||||
formatter = card["formatter"]
|
||||
draw.text((left + 14, top + 45), formatter(card.get("today")), font=_font(28, True), fill=TEXT)
|
||||
vs_yesterday, color_y = _change_text(card.get("today"), card.get("yesterday"))
|
||||
vs_baseline, color_7 = _change_text(card.get("today"), card.get("baseline_7d"))
|
||||
draw.text((left + 14, top + 86), f"较昨日 {vs_yesterday}", font=_font(14, True), fill=color_y)
|
||||
draw.text((left + 154, top + 86), f"较7日 {vs_baseline}", font=_font(14, True), fill=color_7)
|
||||
|
||||
values = [("今日", card.get("today"), BLUE), ("昨日", card.get("yesterday"), MUTED), ("7日", card.get("baseline_7d"), YELLOW)]
|
||||
known = [float(value) for _, value, _ in values if value is not None]
|
||||
maximum = max(known) if known else 1
|
||||
for row_index, (label, value, color) in enumerate(values):
|
||||
y = top + 124 + row_index * 30
|
||||
draw.text((left + 14, y), label, font=_font(13), fill=MUTED)
|
||||
draw.rounded_rectangle((left + 56, y + 4, right - 78, y + 14), radius=5, fill=GRID)
|
||||
if value is not None:
|
||||
bar_width = max(2, (right - left - 148) * float(value) / maximum)
|
||||
draw.rounded_rectangle((left + 56, y + 4, left + 56 + bar_width, y + 14), radius=5, fill=color)
|
||||
draw.text((right - 70, y - 1), formatter(value), font=_font(12, True), fill=TEXT if value is not None else MUTED)
|
||||
if card.get("note"):
|
||||
draw.text((left + 14, top + 220), str(card["note"])[:30], font=_font(12), fill=MUTED)
|
||||
|
||||
|
||||
def _focus_styles(styles: list[dict[str, Any]], target: str) -> list[dict[str, Any]]:
|
||||
candidates = []
|
||||
for row in styles:
|
||||
if target == "增长型":
|
||||
sales_change = row.get("sales_change")
|
||||
visitor_change = row.get("visitor_change")
|
||||
refund_rate = row.get("refund_rate")
|
||||
if sales_change is None or sales_change <= 0 or (visitor_change is not None and visitor_change < -0.05):
|
||||
continue
|
||||
if refund_rate is not None and refund_rate >= 0.40:
|
||||
continue
|
||||
score = float(sales_change) + max(float(visitor_change or 0), 0)
|
||||
action = "加达人/备货" if (row.get("conversion_change_7d") or 0) >= -0.002 else "加曝光测试"
|
||||
else:
|
||||
sales_change = row.get("sales_change")
|
||||
conversion_change = row.get("conversion_change_7d")
|
||||
refund_change = row.get("refund_change")
|
||||
refund_rate = row.get("refund_rate")
|
||||
strict_risk = (
|
||||
sales_change is not None and sales_change < 0
|
||||
and conversion_change is not None and conversion_change < 0
|
||||
and refund_change is not None and refund_change > 0
|
||||
)
|
||||
if not strict_risk and not (sales_change is not None and sales_change < 0 and (refund_rate or 0) >= 0.40):
|
||||
continue
|
||||
score = abs(min(float(sales_change or 0), 0)) + max(float(refund_change or 0), 0) + float(refund_rate or 0)
|
||||
action = "查退款/控投放"
|
||||
candidates.append((score, {**row, "prompt_diagnosis": target, "prompt_action": action}))
|
||||
return [row for _, row in sorted(candidates, key=lambda item: item[0], reverse=True)[:3]]
|
||||
|
||||
|
||||
def _focus_panel(draw: ImageDraw.ImageDraw, box: tuple[int, int, int, int], title: str, rows: list[dict[str, Any]], risk: bool = False) -> None:
|
||||
_panel(draw, box, title, 22)
|
||||
x0, y0, x1, _ = box
|
||||
accent = RED if risk else GREEN
|
||||
if not rows:
|
||||
draw.text((x0 + 20, y0 + 74), "无符合规则的SKU", font=_font(16), fill=MUTED)
|
||||
return
|
||||
for idx, row in enumerate(rows, 1):
|
||||
y = y0 + 62 + (idx - 1) * 65
|
||||
draw.rounded_rectangle((x0 + 18, y, x1 - 18, y + 54), radius=10, fill=LIGHT_RED if risk else "#EFFAF4")
|
||||
draw.text((x0 + 30, y + 14), str(idx), font=_font(17, True), fill=accent)
|
||||
draw.text((x0 + 60, y + 8), str(row.get("style_name") or "无数据")[:12], font=_font(17, True), fill=TEXT)
|
||||
sales_text, sales_color = _change_text(row.get("sales"), row.get("avg_sales_7d"))
|
||||
visitor_text, visitor_color = _change_text(row.get("visitors"), row.get("avg_visitors_7d"))
|
||||
draw.text((x0 + 210, y + 10), f"销量较7日 {sales_text}", font=_font(13, True), fill=sales_color)
|
||||
draw.text((x0 + 350, y + 10), f"访客较7日 {visitor_text}", font=_font(13, True), fill=visitor_color)
|
||||
draw.text((x0 + 500, y + 10), row.get("prompt_action") or "无数据", font=_font(14, True), fill=accent)
|
||||
draw.text((x0 + 60, y + 32), f"转化 {_fmt_rate(row.get('conversion_rate'))}｜曝光 {_fmt_int(row.get('recent_note_exposure'))}", font=_font(12), fill=MUTED)
|
||||
|
||||
|
||||
def _channel_section(
|
||||
draw: ImageDraw.ImageDraw,
|
||||
box: tuple[int, int, int, int],
|
||||
notes: list[dict[str, Any]],
|
||||
styles: list[dict[str, Any]],
|
||||
) -> None:
|
||||
_panel(draw, box, "四、达人营销效果｜昨日新发布内容", 25)
|
||||
x0, y0, x1, _ = box
|
||||
platforms = [("小红书", RED), ("B站", BLUE), ("抖音", TEXT)]
|
||||
gap = 18
|
||||
width = (x1 - x0 - 40 - gap * 2) // 3
|
||||
for idx, (platform, color) in enumerate(platforms):
|
||||
items = [row for row in notes if row.get("platform") == platform]
|
||||
left = x0 + 20 + idx * (width + gap)
|
||||
top = y0 + 66
|
||||
draw.rounded_rectangle((left, top, left + width, top + 230), radius=13, fill=LIGHT_BLUE)
|
||||
draw.text((left + 16, top + 14), platform, font=_font(20, True), fill=color)
|
||||
exposure = sum(int(row.get("view_count") or 0) for row in items) if items else None
|
||||
draw.text((left + 16, top + 54), f"新发 {len(items)} 篇", font=_font(19, True), fill=TEXT)
|
||||
draw.text((left + 150, top + 56), f"曝光 {_fmt_int(exposure)}", font=_font(16, True), fill=TEXT)
|
||||
if items:
|
||||
best = max(items, key=lambda row: row.get("view_count") or 0)
|
||||
_draw_wrapped(draw, f"最佳：{best.get('style_name') or '无款式'} · {best.get('creator') or '无达人'}", left + 16, top + 96, 28, 2, TEXT, 15, True)
|
||||
_draw_wrapped(draw, best.get("title") or "标题无数据", left + 16, top + 146, 32, 2, MUTED, 13)
|
||||
else:
|
||||
draw.text((left + 16, top + 100), "无新发布内容", font=_font(15), fill=MUTED)
|
||||
|
||||
best_note = max(notes, key=lambda row: row.get("view_count") or 0) if notes else None
|
||||
best_channel = best_note.get("platform") if best_note else "无数据"
|
||||
best_style = best_note.get("style_name") if best_note else "无数据"
|
||||
comment_samples = sum(int((row.get("multi_source_signals") or {}).get("comment_count") or 0) for row in styles)
|
||||
summary_y = y0 + 322
|
||||
summary_items = [
|
||||
("最佳渠道", best_channel or "无数据", "优先复盘可量化内容"),
|
||||
("最佳内容SKU", best_style or "无数据", f"曝光 {_fmt_int(best_note.get('view_count') if best_note else None)}"),
|
||||
("内容反馈", f"新发 {len(notes)} 篇", "跨渠道曝光仍不完整"),
|
||||
("用户洞察", f"评论样本 {comment_samples}", "样本不足则不推断"),
|
||||
]
|
||||
summary_width = (x1 - x0 - 40 - 14 * 3) // 4
|
||||
for idx, (label, value, note) in enumerate(summary_items):
|
||||
left = x0 + 20 + idx * (summary_width + 14)
|
||||
draw.rounded_rectangle((left, summary_y, left + summary_width, summary_y + 150), radius=12, fill="#F8F5FF")
|
||||
draw.text((left + 14, summary_y + 14), label, font=_font(15, True), fill=PURPLE)
|
||||
draw.text((left + 14, summary_y + 48), str(value)[:18], font=_font(20, True), fill=TEXT)
|
||||
_draw_wrapped(draw, note, left + 14, summary_y + 86, 24, 2, MUTED, 13)
|
||||
draw.text((x0 + 20, box[3] - 38), "说明：曝光为数据库保存的当前快照；缺少历史快照时不推算单日播放增量。", font=_font(14), fill=MUTED)
|
||||
|
||||
|
||||
def _draw_wrapped(
|
||||
draw: ImageDraw.ImageDraw,
|
||||
text: str,
|
||||
x: int,
|
||||
y: int,
|
||||
width: int = 28,
|
||||
max_lines: int = 3,
|
||||
color: str = TEXT,
|
||||
size: int = 14,
|
||||
bold: bool = False,
|
||||
) -> int:
|
||||
lines = wrap(str(text or "无数据"), width=width)[:max_lines] or ["无数据"]
|
||||
for line in lines:
|
||||
draw.text((x, y), line, font=_font(size, bold), fill=color)
|
||||
y += size + 8
|
||||
return y
|
||||
|
||||
|
||||
def _paste_product_image(canvas: Image.Image, draw: ImageDraw.ImageDraw, winner: dict[str, Any], x: int, y: int) -> None:
|
||||
size = 112
|
||||
path = winner.get("local_image_path")
|
||||
if path:
|
||||
try:
|
||||
with Image.open(path) as source:
|
||||
thumb = ImageOps.fit(source.convert("RGB"), (size, size), method=Image.Resampling.LANCZOS)
|
||||
mask = Image.new("L", (size, size), 0)
|
||||
ImageDraw.Draw(mask).rounded_rectangle((0, 0, size, size), radius=10, fill=255)
|
||||
canvas.paste(thumb, (x, y), mask)
|
||||
ctr = winner.get("image_ctr")
|
||||
ctr_text = f" CTR {float(ctr):.1f}%" if ctr is not None else ""
|
||||
draw.text((x, y + size + 7), f"图片：天猫{ctr_text}", font=_font(12), fill=MUTED)
|
||||
return
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
draw.rounded_rectangle((x, y, x + size, y + size), radius=10, fill=PANEL, outline=GRID)
|
||||
draw.text((x + 17, y + 46), "图片无数据", font=_font(13), fill=MUTED)
|
||||
|
||||
|
||||
def _winner_block(canvas: Image.Image, draw: ImageDraw.ImageDraw, x: int, y: int, width: int, title: str, winner: dict[str, Any] | None, role: str) -> int:
|
||||
draw.rounded_rectangle((x, y, x + width, y + 260), radius=12, fill=LIGHT_BLUE if role == "conversion" else "#F8F5FF")
|
||||
draw.text((x + 14, y + 12), title, font=_font(17, True), fill=BLUE if role == "conversion" else PURPLE)
|
||||
if not winner:
|
||||
draw.text((x + 14, y + 50), "无数据", font=_font(15), fill=MUTED)
|
||||
return y + 260
|
||||
metric = _fmt_rate(winner.get("conversion_rate")) if role == "conversion" else _fmt_int(winner.get("visitors"))
|
||||
unit = "转化率" if role == "conversion" else "访客"
|
||||
draw.text((x + 14, y + 45), f"{winner['style_name']}｜{unit} {metric}", font=_font(16, True), fill=TEXT)
|
||||
_paste_product_image(canvas, draw, winner, x + 14, y + 80)
|
||||
text_x = x + 140
|
||||
persona_date = f"（{winner.get('persona_date')}）" if winner.get("persona_date") else ""
|
||||
cursor = _draw_wrapped(draw, f"画像{persona_date}：{winner.get('persona_text') or '无数据'}", text_x, y + 78, 20, 3, TEXT, 13)
|
||||
cursor = _draw_wrapped(draw, winner.get("interest_hypothesis") or "无数据", text_x, cursor + 2, 20, 2, MUTED, 13)
|
||||
_draw_wrapped(draw, winner.get("optimization") or "无数据", text_x, cursor + 2, 20, 3, TEXT, 13)
|
||||
return y + 260
|
||||
|
||||
|
||||
def _persona_section(canvas: Image.Image, draw: ImageDraw.ImageDraw, box: tuple[int, int, int, int], cards: list[dict[str, Any]]) -> None:
|
||||
_panel(draw, box, "三平台高转化 / 高访客款式画像与增长空间", 25)
|
||||
x0, y0, x1, _ = box
|
||||
draw.text((x1 - 570, y0 + 20), "兴趣爱好均为画像与内容方向的推测，不是数据库原始标签", font=_font(14), fill=MUTED)
|
||||
gap = 16
|
||||
card_width = (x1 - x0 - 40 - gap * 2) // 3
|
||||
for idx, card in enumerate(cards[:3]):
|
||||
x = x0 + 20 + idx * (card_width + gap)
|
||||
draw.text((x, y0 + 66), card.get("platform") or "无数据", font=_font(21, True), fill=PLATFORM_COLORS.get(card.get("platform"), BLUE))
|
||||
next_y = _winner_block(canvas, draw, x, y0 + 103, card_width, "转化最高", card.get("top_conversion"), "conversion")
|
||||
_winner_block(canvas, draw, x, next_y + 14, card_width, "访客最高", card.get("top_visitors"), "visitors")
|
||||
|
||||
|
||||
def generate_dashboard(report_date: str, rows: list[dict[str, Any]], enrichment: dict[str, Any], output_path: Path) -> Path:
|
||||
output_path = Path(output_path)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
dashboard = enrichment.get("dashboard") or build_dashboard_facts(
|
||||
report_date, rows, enrichment.get("platform_metrics", []), enrichment.get("personas", []), {}
|
||||
)
|
||||
|
||||
image = Image.new("RGB", (WIDTH, HEIGHT), BG)
|
||||
draw = ImageDraw.Draw(image)
|
||||
draw.text((42, 24), f"{report_date} 品牌SKU营销运营看板", font=_font(36, True), fill=TEXT)
|
||||
draw.text((42, 74), "经营双基准｜近7日销量与未来趋势预估｜全款式同权｜达人曝光缺失不推算", font=_font(18), fill=MUTED)
|
||||
|
||||
overview_cards = build_overview_cards(rows, dashboard)
|
||||
_overview_section(draw, (32, 112, 1568, 442), overview_cards)
|
||||
styles = dashboard.get("styles", [])
|
||||
_focus_panel(draw, (32, 464, 790, 730), "二、增长SKU TOP3", _focus_styles(styles, "增长型"))
|
||||
_focus_panel(draw, (810, 464, 1568, 730), "风险SKU TOP3", _focus_styles(styles, "风险型"), risk=True)
|
||||
style_groups = dashboard.get("style_groups") or [{"style_category": "未分类", "styles": dashboard.get("styles", [])}]
|
||||
_style_matrix(draw, (32, 752, 1568, 1780), style_groups)
|
||||
_channel_section(draw, (32, 1802, 1568, 2468), dashboard.get("daily_new_notes", []), styles)
|
||||
image.save(output_path, format="PNG", optimize=True)
|
||||
return output_path
|
||||
@@ -0,0 +1,61 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Daily task: run 3-platform V2 scrape + sync metrics to cmt_notes (INSERT missing notes every day)
|
||||
REM Register: schtasks /Create /SC DAILY /TN YingxiaoYunying_DailyRun /TR %PROJECT_DIR%\data\tools\daily_run.bat /ST 06:00 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing\scheduled
|
||||
REM Task Scheduler (SYSTEM account) has no user PATH -> lock to venv python
|
||||
set PYTHON=%GYXX_PYTHON%
|
||||
where %PYTHON% >nul 2>&1 || set PYTHON=python
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
|
||||
set TS=%date:~0,4%%date:~5,2%%date:~8,2%_%time:~0,2%%time:~3,2%%time:~6,2%
|
||||
set TS=%TS: =0%
|
||||
|
||||
set LOG_FILE=%LOG_DIR%\daily_run_%TS%.log
|
||||
|
||||
echo === Daily run started at %date% %time% === > "%LOG_FILE%"
|
||||
echo PROJECT_DIR=%PROJECT_DIR% >> "%LOG_FILE%"
|
||||
|
||||
cd /d "%PROJECT_DIR%"
|
||||
|
||||
echo --- [step 1] run_all.py (includes feishu_mapping refresh) --- >> "%LOG_FILE%"
|
||||
call %PYTHON% run_all.py >> "%LOG_FILE%" 2>&1
|
||||
set RC_RUN=%ERRORLEVEL%
|
||||
echo run_all exit=%RC_RUN% >> "%LOG_FILE%"
|
||||
|
||||
echo --- [step 2] sync_metrics_to_cmt_notes.py (主采集后首次落库) --- >> "%LOG_FILE%"
|
||||
call %PYTHON% data\tools\sync_metrics_to_cmt_notes.py >> "%LOG_FILE%" 2>&1
|
||||
set RC_SYNC_PRE=%ERRORLEVEL%
|
||||
echo sync_pre exit=%RC_SYNC_PRE% >> "%LOG_FILE%"
|
||||
|
||||
echo --- [step 3] retry_failed.py (B站 + 蒲公英; 星图每日只搜索一轮) --- >> "%LOG_FILE%"
|
||||
call %PYTHON% data\tools\retry_failed.py --platform bili,pgy --max-attempts 1 --stale-hours 4 >> "%LOG_FILE%" 2>&1
|
||||
set RC_RETRY=%ERRORLEVEL%
|
||||
echo retry_failed exit=%RC_RETRY% >> "%LOG_FILE%"
|
||||
|
||||
echo --- [step 4] sync_metrics_to_cmt_notes.py (补跑后再次落库) --- >> "%LOG_FILE%"
|
||||
call %PYTHON% data\tools\sync_metrics_to_cmt_notes.py >> "%LOG_FILE%" 2>&1
|
||||
set RC_SYNC_POST=%ERRORLEVEL%
|
||||
echo sync_post exit=%RC_SYNC_POST% >> "%LOG_FILE%"
|
||||
|
||||
set FINAL_RC=0
|
||||
if not "%RC_RUN%"=="0" set FINAL_RC=1
|
||||
if not "%RC_SYNC_PRE%"=="0" set FINAL_RC=1
|
||||
if not "%RC_RETRY%"=="0" set FINAL_RC=1
|
||||
if not "%RC_SYNC_POST%"=="0" set FINAL_RC=1
|
||||
|
||||
echo === Daily run finished at %date% %time% (run=%RC_RUN%, sync_pre=%RC_SYNC_PRE%, retry=%RC_RETRY%, sync_post=%RC_SYNC_POST%, final=%FINAL_RC%) === >> "%LOG_FILE%"
|
||||
|
||||
endlocal & exit /b %FINAL_RC%
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Monday task: run 3-platform V2 + sync metrics to cmt_notes
|
||||
REM Register: schtasks /Create /SC WEEKLY /D MON /TN YingxiaoYunying_MondayBackfill /TR %PROJECT_DIR%\data\tools\daily_run_with_backfill.bat /ST 06:00 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing\scheduled
|
||||
REM Task Scheduler (SYSTEM account) has no user PATH -> lock to venv python
|
||||
set PYTHON=%GYXX_PYTHON%
|
||||
where %PYTHON% >nul 2>&1 || set PYTHON=python
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
|
||||
set TS=%date:~0,4%%date:~5,2%%date:~8,2%_%time:~0,2%%time:~3,2%%time:~6,2%
|
||||
set TS=%TS: =0%
|
||||
|
||||
set LOG_FILE=%LOG_DIR%\monday_run_%TS%.log
|
||||
|
||||
echo === Monday run started at %date% %time% === > "%LOG_FILE%"
|
||||
echo PROJECT_DIR=%PROJECT_DIR% >> "%LOG_FILE%"
|
||||
|
||||
cd /d "%PROJECT_DIR%"
|
||||
|
||||
echo --- [step 1] run_all.py (includes feishu_mapping refresh) --- >> "%LOG_FILE%"
|
||||
call %PYTHON% run_all.py >> "%LOG_FILE%" 2>&1
|
||||
set RC_RUN=%ERRORLEVEL%
|
||||
echo run_all exit=%RC_RUN% >> "%LOG_FILE%"
|
||||
|
||||
echo --- [step 2] sync_metrics_to_cmt_notes.py (主采集后首次落库) --- >> "%LOG_FILE%"
|
||||
call %PYTHON% data\tools\sync_metrics_to_cmt_notes.py >> "%LOG_FILE%" 2>&1
|
||||
set RC_SYNC_PRE=%ERRORLEVEL%
|
||||
echo sync_pre exit=%RC_SYNC_PRE% >> "%LOG_FILE%"
|
||||
|
||||
echo --- [step 3] retry_failed.py --- >> "%LOG_FILE%"
|
||||
call %PYTHON% data\tools\retry_failed.py --max-attempts 1 --stale-hours 4 >> "%LOG_FILE%" 2>&1
|
||||
set RC_RETRY=%ERRORLEVEL%
|
||||
echo retry_failed exit=%RC_RETRY% >> "%LOG_FILE%"
|
||||
|
||||
echo --- [step 4] sync_metrics_to_cmt_notes.py (补跑后再次落库) --- >> "%LOG_FILE%"
|
||||
call %PYTHON% data\tools\sync_metrics_to_cmt_notes.py >> "%LOG_FILE%" 2>&1
|
||||
set RC_SYNC_POST=%ERRORLEVEL%
|
||||
echo sync_post exit=%RC_SYNC_POST% >> "%LOG_FILE%"
|
||||
|
||||
set FINAL_RC=0
|
||||
if not "%RC_RUN%"=="0" set FINAL_RC=1
|
||||
if not "%RC_SYNC_PRE%"=="0" set FINAL_RC=1
|
||||
if not "%RC_RETRY%"=="0" set FINAL_RC=1
|
||||
if not "%RC_SYNC_POST%"=="0" set FINAL_RC=1
|
||||
|
||||
echo === Monday run finished at %date% %time% (run=%RC_RUN%, sync_pre=%RC_SYNC_PRE%, retry=%RC_RETRY%, sync_post=%RC_SYNC_POST%, final=%FINAL_RC%) === >> "%LOG_FILE%"
|
||||
|
||||
endlocal & exit /b %FINAL_RC%
|
||||
@@ -0,0 +1,515 @@
|
||||
"""
|
||||
db.py - PostgreSQL 连接池 + cmt_* 表 CRUD
|
||||
|
||||
统一使用 psycopg3，合并了原 yingxiaoyunying 诊断功能 + comment-data-collector 的完整 CRUD。
|
||||
|
||||
用法:
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import make_pool, get_conn, get_dict_conn
|
||||
python db.py ping # 测试连接
|
||||
python db.py info # 显示 cmt_* 表行数
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator, Mapping
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
from gyxx_flow.adapters import RuntimeServicePolicy
|
||||
|
||||
try:
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
except ImportError:
|
||||
print("[FATAL] 请先 pip install 'psycopg[binary]>=3.1' psycopg-pool", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
except ImportError:
|
||||
print("[FATAL] 请先 pip install python-dotenv", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
BASE_DIR = PATHS.tools_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
ENV_PATH = PATHS.state_root / "config/db.env"
|
||||
ENV_EXAMPLE = PATHS.config_root / "db.env.example"
|
||||
|
||||
|
||||
def load_env() -> None:
|
||||
if not ENV_PATH.exists():
|
||||
print(f"[FATAL] 找不到 {ENV_PATH}", file=sys.stderr)
|
||||
print(f" 请复制 {ENV_EXAMPLE} -> {ENV_PATH} 并填连接信息", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
load_dotenv(ENV_PATH, override=False)
|
||||
os.environ.update(RuntimeServicePolicy().apply(os.environ))
|
||||
|
||||
|
||||
def database_config_from_env(environ: Mapping[str, str]) -> dict[str, Any]:
|
||||
"""从环境变量构造连接配置；缺项立即失败，绝不回退到本地库。"""
|
||||
required = ("PG_HOST", "PG_PORT", "PG_DB", "PG_USER", "PG_PASSWORD")
|
||||
missing = [key for key in required if not str(environ.get(key, "")).strip()]
|
||||
if missing:
|
||||
raise RuntimeError(f"数据库配置缺失: {', '.join(missing)}")
|
||||
try:
|
||||
port = int(environ["PG_PORT"])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RuntimeError("数据库配置无效: PG_PORT 必须是整数") from exc
|
||||
return {
|
||||
"host": environ["PG_HOST"].strip(),
|
||||
"port": port,
|
||||
"dbname": environ["PG_DB"].strip(),
|
||||
"user": environ["PG_USER"].strip(),
|
||||
"password": environ["PG_PASSWORD"],
|
||||
}
|
||||
|
||||
|
||||
def get_db_config() -> dict[str, Any]:
|
||||
load_env()
|
||||
return database_config_from_env(os.environ)
|
||||
|
||||
|
||||
def make_pool(min_size: int = 1, max_size: int = 5) -> ConnectionPool:
|
||||
conninfo = psycopg.conninfo.make_conninfo(**get_db_config())
|
||||
return ConnectionPool(conninfo, min_size=min_size, max_size=max_size, open=True)
|
||||
|
||||
|
||||
# 全局连接池（懒初始化）
|
||||
_pool: ConnectionPool | None = None
|
||||
|
||||
|
||||
def _get_pool() -> ConnectionPool:
|
||||
global _pool
|
||||
if _pool is None or _pool.closed:
|
||||
_pool = make_pool()
|
||||
return _pool
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_conn(pool: ConnectionPool | None = None) -> Iterator[psycopg.Connection]:
|
||||
p = pool or _get_pool()
|
||||
with p.connection() as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_dict_conn(pool: ConnectionPool | None = None) -> Iterator[psycopg.Connection]:
|
||||
"""返回连接，cursor 需自行指定 row_factory=dict_row。替代 psycopg2 的 RealDictCursor。"""
|
||||
p = pool or _get_pool()
|
||||
with p.connection() as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_dict_cursor(pool: ConnectionPool | None = None):
|
||||
"""便捷函数：返回 dict_row cursor，自动 commit/rollback。"""
|
||||
with get_dict_conn(pool) as conn:
|
||||
cur = conn.cursor(row_factory=dict_row)
|
||||
try:
|
||||
yield cur
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
cur.close()
|
||||
|
||||
|
||||
def now() -> datetime:
|
||||
return datetime.now()
|
||||
|
||||
|
||||
def show_info(pool: ConnectionPool) -> None:
|
||||
"""显示 cmt_* 表行数"""
|
||||
with get_conn(pool) as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
for tbl in ("cmt_styles", "cmt_creators", "cmt_style_creators", "cmt_notes", "cmt_comments"):
|
||||
cur.execute(f"SELECT COUNT(*) AS n FROM {tbl}")
|
||||
n = cur.fetchone()["n"]
|
||||
print(f" {tbl:<25} {n:>8} 行")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Styles
|
||||
# ---------------------------------------------------------------------------
|
||||
def upsert_style(
|
||||
name: str,
|
||||
platform: str = "",
|
||||
feishu_base_url: str = "",
|
||||
feishu_base_token: str = "",
|
||||
feishu_table_id: str = "",
|
||||
feishu_view_id: str = "",
|
||||
) -> int:
|
||||
sql = """
|
||||
INSERT INTO cmt_styles (name, platform, feishu_base_url, feishu_base_token, feishu_table_id, feishu_view_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (name) DO UPDATE SET
|
||||
platform = EXCLUDED.platform,
|
||||
feishu_base_url = EXCLUDED.feishu_base_url,
|
||||
feishu_base_token = EXCLUDED.feishu_base_token,
|
||||
feishu_table_id = EXCLUDED.feishu_table_id,
|
||||
feishu_view_id = EXCLUDED.feishu_view_id,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
RETURNING id
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (name, platform, feishu_base_url, feishu_base_token, feishu_table_id, feishu_view_id))
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def get_style_id(name: str) -> int | None:
|
||||
with get_dict_conn() as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute("SELECT id FROM cmt_styles WHERE name = %s", (name,))
|
||||
row = cur.fetchone()
|
||||
return row["id"] if row else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Creators
|
||||
# ---------------------------------------------------------------------------
|
||||
def upsert_creator(name: str) -> int:
|
||||
if not name:
|
||||
raise ValueError("creator name cannot be empty")
|
||||
sql = """
|
||||
INSERT INTO cmt_creators (name)
|
||||
VALUES (%s)
|
||||
ON CONFLICT (name) DO UPDATE SET
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
RETURNING id
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (name,))
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def link_style_creator(style_id: int, creator_id: int) -> None:
|
||||
sql = """
|
||||
INSERT INTO cmt_style_creators (style_id, creator_id)
|
||||
VALUES (%s, %s)
|
||||
ON CONFLICT (style_id, creator_id) DO NOTHING
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (style_id, creator_id))
|
||||
conn.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Notes
|
||||
# ---------------------------------------------------------------------------
|
||||
def upsert_note(
|
||||
style_id: int,
|
||||
creator_id: int,
|
||||
platform: str,
|
||||
url: str,
|
||||
title: str = "",
|
||||
feishu_record_id: str = "",
|
||||
) -> int:
|
||||
sql = """
|
||||
INSERT INTO cmt_notes (style_id, creator_id, platform, title, url, feishu_record_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (url) DO UPDATE SET
|
||||
style_id = EXCLUDED.style_id,
|
||||
creator_id = EXCLUDED.creator_id,
|
||||
platform = EXCLUDED.platform,
|
||||
title = EXCLUDED.title,
|
||||
feishu_record_id = EXCLUDED.feishu_record_id,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
RETURNING id
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (style_id, creator_id, platform, title, url, feishu_record_id))
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def update_note_metrics(
|
||||
note_id: int,
|
||||
like_count: int | None,
|
||||
favorite_count: int | None,
|
||||
comment_count: int | None,
|
||||
share_count: int | None,
|
||||
title: str | None = None,
|
||||
) -> None:
|
||||
sql = """
|
||||
UPDATE cmt_notes
|
||||
SET like_count = %s,
|
||||
favorite_count = %s,
|
||||
comment_count = %s,
|
||||
share_count = %s,
|
||||
title = COALESCE(%s, title),
|
||||
scraped_at = CURRENT_TIMESTAMP,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = %s
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (like_count, favorite_count, comment_count, share_count, title, note_id))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def update_note_view_metrics(
|
||||
note_id: int,
|
||||
view_count: int | None,
|
||||
collect_count: int | None,
|
||||
) -> None:
|
||||
"""更新单笔记的曝光量/播放量、收藏量(来自 yingxiaoyunying 每日采集)。"""
|
||||
sql = """
|
||||
UPDATE cmt_notes
|
||||
SET view_count = %s,
|
||||
collect_count = %s,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = %s
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (view_count, collect_count, note_id))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_note_id_by_url(url: str) -> int | None:
|
||||
with get_dict_conn() as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute("SELECT id FROM cmt_notes WHERE url = %s", (url,))
|
||||
row = cur.fetchone()
|
||||
return row["id"] if row else None
|
||||
|
||||
|
||||
def get_note_by_id(note_id: int) -> dict[str, Any] | None:
|
||||
with get_dict_conn() as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT n.*, s.name AS style_name, c.name AS creator_name
|
||||
FROM cmt_notes n
|
||||
LEFT JOIN cmt_styles s ON n.style_id = s.id
|
||||
LEFT JOIN cmt_creators c ON n.creator_id = c.id
|
||||
WHERE n.id = %s
|
||||
""",
|
||||
(note_id,),
|
||||
)
|
||||
return cur.fetchone()
|
||||
|
||||
|
||||
def get_note_by_url(url: str) -> dict[str, Any] | None:
|
||||
with get_dict_conn() as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT n.*, s.name AS style_name, c.name AS creator_name
|
||||
FROM cmt_notes n
|
||||
LEFT JOIN cmt_styles s ON n.style_id = s.id
|
||||
LEFT JOIN cmt_creators c ON n.creator_id = c.id
|
||||
WHERE n.url = %s
|
||||
""",
|
||||
(url,),
|
||||
)
|
||||
return cur.fetchone()
|
||||
|
||||
|
||||
def get_notes_by_platform(
|
||||
platform: str,
|
||||
only_with_comments: bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List all notes for a platform with style/creator joins + comment count."""
|
||||
sql = """
|
||||
SELECT n.*, s.name AS style_name, c.name AS creator_name,
|
||||
(SELECT COUNT(*) FROM cmt_comments WHERE note_id = n.id) AS comment_total
|
||||
FROM cmt_notes n
|
||||
LEFT JOIN cmt_styles s ON n.style_id = s.id
|
||||
LEFT JOIN cmt_creators c ON n.creator_id = c.id
|
||||
WHERE n.platform = %s
|
||||
"""
|
||||
if only_with_comments:
|
||||
sql += " AND (SELECT COUNT(*) FROM cmt_comments WHERE note_id = n.id) > 0"
|
||||
sql += " ORDER BY n.id"
|
||||
with get_dict_conn() as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(sql, (platform,))
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
def get_all_styles_with_notes() -> list[dict[str, Any]]:
|
||||
"""List all styles that have at least one note, with per-style rollup counts."""
|
||||
sql = """
|
||||
SELECT s.*,
|
||||
COUNT(n.id) AS note_total,
|
||||
COUNT(*) FILTER (WHERE EXISTS(SELECT 1 FROM cmt_comments WHERE note_id = n.id)) AS note_with_comments
|
||||
FROM cmt_styles s
|
||||
LEFT JOIN cmt_notes n ON n.style_id = s.id
|
||||
GROUP BY s.id
|
||||
HAVING COUNT(n.id) > 0
|
||||
ORDER BY note_total DESC, s.name ASC
|
||||
"""
|
||||
with get_dict_conn() as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(sql)
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
def get_notes_by_style(
|
||||
style_id: int,
|
||||
only_with_comments: bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List all notes for a single style, same field set as get_notes_by_platform."""
|
||||
sql = """
|
||||
SELECT n.*, s.name AS style_name, c.name AS creator_name,
|
||||
(SELECT COUNT(*) FROM cmt_comments WHERE note_id = n.id) AS comment_total
|
||||
FROM cmt_notes n
|
||||
LEFT JOIN cmt_styles s ON n.style_id = s.id
|
||||
LEFT JOIN cmt_creators c ON n.creator_id = c.id
|
||||
WHERE n.style_id = %s
|
||||
"""
|
||||
if only_with_comments:
|
||||
sql += " AND (SELECT COUNT(*) FROM cmt_comments WHERE note_id = n.id) > 0"
|
||||
sql += " ORDER BY n.id ASC"
|
||||
with get_dict_conn() as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(sql, (style_id,))
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
def get_comments_by_note_id(note_id: int) -> list[dict[str, Any]]:
|
||||
with get_dict_conn() as conn:
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT * FROM cmt_comments
|
||||
WHERE note_id = %s
|
||||
ORDER BY comment_created_at ASC, id ASC
|
||||
""",
|
||||
(note_id,),
|
||||
)
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Comments
|
||||
# ---------------------------------------------------------------------------
|
||||
def replace_comments(note_id: int, platform: str, comments: list[dict[str, Any]], allow_empty: bool = False) -> int:
|
||||
"""Delete existing comments for the note and insert the latest batch."""
|
||||
if not comments and not allow_empty:
|
||||
raise ValueError("refusing to replace existing comments with an empty scrape result")
|
||||
|
||||
scraped_at = now()
|
||||
|
||||
insert_sql = """
|
||||
INSERT INTO cmt_comments (
|
||||
note_id, platform, level, platform_comment_id, parent_comment_id,
|
||||
reply_to_comment_id, user_id, nickname, content, like_count,
|
||||
reply_count, ip_location, comment_created_at, raw_create_time, scraped_at
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (note_id, platform_comment_id) DO UPDATE SET
|
||||
level = EXCLUDED.level,
|
||||
parent_comment_id = EXCLUDED.parent_comment_id,
|
||||
reply_to_comment_id = EXCLUDED.reply_to_comment_id,
|
||||
user_id = EXCLUDED.user_id,
|
||||
nickname = EXCLUDED.nickname,
|
||||
content = EXCLUDED.content,
|
||||
like_count = EXCLUDED.like_count,
|
||||
reply_count = EXCLUDED.reply_count,
|
||||
ip_location = EXCLUDED.ip_location,
|
||||
comment_created_at = EXCLUDED.comment_created_at,
|
||||
raw_create_time = EXCLUDED.raw_create_time,
|
||||
scraped_at = EXCLUDED.scraped_at,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
|
||||
def to_int(value: Any) -> int | None:
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
def parse_dt(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
ts = value / 1000 if value > 10_000_000_000 else value
|
||||
try:
|
||||
return datetime.fromtimestamp(ts)
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S%z"):
|
||||
try:
|
||||
return datetime.strptime(value[:19], fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
rows = []
|
||||
for c in comments:
|
||||
platform_comment_id = str(c.get("comment_id") or c.get("cid") or c.get("rpid") or "")
|
||||
parent = str(c.get("parent_comment_id") or c.get("parent_rpid") or "")
|
||||
reply_to = str(c.get("reply_to_comment_id") or c.get("reply_id") or c.get("reply_to_reply_id") or "")
|
||||
if not reply_to and parent:
|
||||
reply_to = parent
|
||||
rows.append((
|
||||
note_id,
|
||||
platform,
|
||||
c.get("level") or "comment",
|
||||
platform_comment_id,
|
||||
parent,
|
||||
reply_to,
|
||||
str(c.get("user_id") or c.get("mid") or c.get("sec_uid") or ""),
|
||||
str(c.get("nickname") or c.get("uname") or ""),
|
||||
str(c.get("content") or c.get("text") or c.get("message") or ""),
|
||||
to_int(c.get("like_count") or c.get("digg_count") or 0),
|
||||
to_int(c.get("reply_count") or 0),
|
||||
str(c.get("ip_location") or c.get("ip_label") or c.get("location") or ""),
|
||||
parse_dt(c.get("created_at") or c.get("raw_create_time")),
|
||||
str(c.get("raw_create_time") or ""),
|
||||
scraped_at,
|
||||
))
|
||||
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM cmt_comments WHERE note_id = %s", (note_id,))
|
||||
if rows:
|
||||
# Use executemany for bulk insert (psycopg3 native)
|
||||
cur.executemany(insert_sql, rows)
|
||||
conn.commit()
|
||||
return len(rows)
|
||||
|
||||
|
||||
def test_connection() -> bool:
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT version()")
|
||||
print(cur.fetchone()[0])
|
||||
return True
|
||||
|
||||
|
||||
def cli():
|
||||
if len(sys.argv) < 2:
|
||||
print("用法: python db.py [ping|info]")
|
||||
sys.exit(1)
|
||||
cmd = sys.argv[1]
|
||||
pool = make_pool()
|
||||
try:
|
||||
if cmd == "ping":
|
||||
with get_conn(pool) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT version()")
|
||||
print(cur.fetchone()[0])
|
||||
elif cmd == "info":
|
||||
show_info(pool)
|
||||
else:
|
||||
print(f"未知命令: {cmd}")
|
||||
sys.exit(1)
|
||||
finally:
|
||||
pool.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -0,0 +1,716 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from multiprocessing import Pool
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
# Path setup: db.py is in data/tools/, scrapers are in project root
|
||||
_TOOLS_DIR = PATHS.tools_root
|
||||
_PROJECT_ROOT = PATHS.module_root
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime import bilibili_comment_scraper
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import db
|
||||
from gyxx_flow.modules.content_marketing.runtime import douyin_comment_scraper
|
||||
from gyxx_flow.modules.content_marketing.runtime import xiaohongshu_comment_scraper
|
||||
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
SUMMARY_DIR = PATHS.exports_root / "summary"
|
||||
|
||||
DEFAULT_INDEX_BASE_TOKEN = "TtoCb1NuQaDy3NsZWTpc0GIvnph"
|
||||
DEFAULT_INDEX_TABLE_ID = "tblKCjplVAFrRwMC"
|
||||
DEFAULT_INDEX_VIEW_ID = "vewvy33xEk"
|
||||
|
||||
STYLE_NAME_FIELD = "款式"
|
||||
STYLE_BASE_FIELD = "合作达人多维表格地址"
|
||||
INDEX_PLATFORM_FIELD = "平台"
|
||||
NOTE_URL_FIELD = "发布笔记链接"
|
||||
NOTE_PLATFORM_FIELD = "投放平台"
|
||||
CREATOR_FIELD = "达人名称"
|
||||
TITLE_FIELD = "发布笔记标题"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BaseRef:
|
||||
url: str
|
||||
base_token: str
|
||||
table_id: str
|
||||
view_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StyleTask:
|
||||
record_id: str
|
||||
style_name: str
|
||||
platform: str
|
||||
base_ref: BaseRef
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NoteTask:
|
||||
style_name: str
|
||||
style_record_id: str
|
||||
note_record_id: str
|
||||
platform_hint: str
|
||||
creator_name: str
|
||||
title: str
|
||||
note_url: str
|
||||
source_base_url: str
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {message}", flush=True)
|
||||
|
||||
|
||||
def run_lark(args: list[str]) -> dict[str, Any]:
|
||||
lark_cli = shutil.which("lark-cli") or shutil.which("lark-cli.cmd") or "lark-cli"
|
||||
command = [lark_cli, "base", *args, "--format", "json", "--as", "user"]
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=BASE_DIR,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
capture_output=True,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"lark-cli failed ({completed.returncode}): {' '.join(command)}\n"
|
||||
f"stdout:\n{completed.stdout}\n\nstderr:\n{completed.stderr}"
|
||||
)
|
||||
try:
|
||||
return json.loads(completed.stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"Failed to parse lark-cli JSON output: {exc}\n{completed.stdout}") from exc
|
||||
|
||||
|
||||
def list_records(base_token: str, table_id: str, view_id: str = "", limit: int = 200) -> list[dict[str, Any]]:
|
||||
offset = 0
|
||||
records: list[dict[str, Any]] = []
|
||||
while True:
|
||||
args = [
|
||||
"+record-list",
|
||||
"--base-token",
|
||||
base_token,
|
||||
"--table-id",
|
||||
table_id,
|
||||
"--limit",
|
||||
str(limit),
|
||||
"--offset",
|
||||
str(offset),
|
||||
]
|
||||
if view_id:
|
||||
args.extend(["--view-id", view_id])
|
||||
|
||||
payload = run_lark(args)
|
||||
if not payload.get("ok"):
|
||||
raise RuntimeError(f"lark-cli returned ok=false: {payload}")
|
||||
|
||||
data = payload.get("data") or {}
|
||||
field_names = data.get("fields") or []
|
||||
record_ids = data.get("record_id_list") or []
|
||||
rows = data.get("data") or []
|
||||
|
||||
for index, row in enumerate(rows):
|
||||
record = dict(zip(field_names, row))
|
||||
if index < len(record_ids):
|
||||
record["_record_id"] = record_ids[index]
|
||||
records.append(record)
|
||||
|
||||
if not data.get("has_more") or not rows:
|
||||
return records
|
||||
offset += len(rows)
|
||||
|
||||
|
||||
def first_text(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
if isinstance(value, list):
|
||||
parts = [first_text(item) for item in value]
|
||||
return ",".join(part for part in parts if part)
|
||||
if isinstance(value, dict):
|
||||
for key in ("text", "name", "link", "url"):
|
||||
text = first_text(value.get(key))
|
||||
if text:
|
||||
return text
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def extract_urls(value: Any) -> list[str]:
|
||||
text = first_text(value)
|
||||
if not text:
|
||||
return []
|
||||
markdown_urls = re.findall(r"\]\((https?://[^)\s]+)\)", text)
|
||||
plain_urls = re.findall(r"https?://[^\s)\]>]+", text)
|
||||
urls = markdown_urls + plain_urls
|
||||
seen: set[str] = set()
|
||||
unique: list[str] = []
|
||||
for url in urls:
|
||||
url = url.replace("&", "&").strip()
|
||||
if url not in seen:
|
||||
seen.add(url)
|
||||
unique.append(url)
|
||||
return unique
|
||||
|
||||
|
||||
def normalize_note_url(url: str) -> str:
|
||||
url = url.replace("&", "&").strip()
|
||||
for marker in ("复制", "打开", "，", "。", " "):
|
||||
if marker in url:
|
||||
url = url.split(marker, 1)[0]
|
||||
|
||||
parsed = urlparse(url)
|
||||
host = parsed.netloc.lower()
|
||||
query = parse_qs(parsed.query)
|
||||
modal_id = (query.get("modal_id") or [""])[0]
|
||||
if "douyin.com" in host and modal_id.isdigit():
|
||||
return f"https://www.douyin.com/video/{modal_id}"
|
||||
|
||||
path_parts = [part for part in parsed.path.split("/") if part]
|
||||
if host == "v.douyin.com" and path_parts:
|
||||
return f"{parsed.scheme}://{parsed.netloc}/{path_parts[0]}/"
|
||||
if host in {"xhslink.com", "www.xhslink.com"} and len(path_parts) >= 2:
|
||||
return f"{parsed.scheme}://{parsed.netloc}/{path_parts[0]}/{path_parts[1]}"
|
||||
if host == "b23.tv" and path_parts:
|
||||
return f"{parsed.scheme}://{parsed.netloc}/{path_parts[0]}"
|
||||
return url
|
||||
|
||||
|
||||
def is_supported_note_url(url: str) -> bool:
|
||||
host = urlparse(url).netloc.lower()
|
||||
return any(
|
||||
domain in host
|
||||
for domain in (
|
||||
"douyin.com",
|
||||
"xiaohongshu.com",
|
||||
"xhslink.com",
|
||||
"bilibili.com",
|
||||
"b23.tv",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def parse_base_ref(url: str) -> BaseRef | None:
|
||||
parsed = urlparse(url)
|
||||
parts = [part for part in parsed.path.split("/") if part]
|
||||
if len(parts) < 2 or parts[0] != "base":
|
||||
return None
|
||||
query = parse_qs(parsed.query)
|
||||
table_id = (query.get("table") or [""])[0]
|
||||
view_id = (query.get("view") or [""])[0]
|
||||
if not table_id:
|
||||
return None
|
||||
return BaseRef(url=url, base_token=parts[1], table_id=table_id, view_id=view_id)
|
||||
|
||||
|
||||
def detect_platform(url: str, platform_hint: str = "") -> str:
|
||||
lowered = url.lower()
|
||||
if "xiaohongshu.com" in lowered or "xhslink.com" in lowered:
|
||||
return "xiaohongshu"
|
||||
if "douyin.com" in lowered:
|
||||
return "douyin"
|
||||
if "bilibili.com" in lowered or "b23.tv" in lowered:
|
||||
return "bilibili"
|
||||
|
||||
return "unknown"
|
||||
|
||||
|
||||
def load_style_tasks(
|
||||
index_base_token: str,
|
||||
index_table_id: str,
|
||||
index_view_id: str,
|
||||
index_platform: str,
|
||||
max_styles: int | None,
|
||||
) -> list[StyleTask]:
|
||||
records = list_records(index_base_token, index_table_id, index_view_id)
|
||||
tasks: list[StyleTask] = []
|
||||
for record in records:
|
||||
platform = first_text(record.get(INDEX_PLATFORM_FIELD))
|
||||
if index_platform and platform != index_platform:
|
||||
continue
|
||||
style_name = first_text(record.get(STYLE_NAME_FIELD))
|
||||
urls = extract_urls(record.get(STYLE_BASE_FIELD))
|
||||
if not urls:
|
||||
continue
|
||||
base_ref = parse_base_ref(urls[0])
|
||||
if not base_ref:
|
||||
log(f"Skip style without valid Base link: {style_name or record.get('_record_id')}")
|
||||
continue
|
||||
tasks.append(
|
||||
StyleTask(
|
||||
record_id=first_text(record.get("_record_id")),
|
||||
style_name=style_name or first_text(record.get("_record_id")) or "unknown_style",
|
||||
platform=platform,
|
||||
base_ref=base_ref,
|
||||
)
|
||||
)
|
||||
if max_styles and len(tasks) >= max_styles:
|
||||
break
|
||||
return tasks
|
||||
|
||||
|
||||
def load_note_tasks(style_tasks: list[StyleTask], max_notes_per_style: int | None) -> list[NoteTask]:
|
||||
note_tasks: list[NoteTask] = []
|
||||
for style in style_tasks:
|
||||
log(f"Read notes: {style.style_name}")
|
||||
records = list_records(style.base_ref.base_token, style.base_ref.table_id, style.base_ref.view_id)
|
||||
count_for_style = 0
|
||||
for record in records:
|
||||
urls = extract_urls(record.get(NOTE_URL_FIELD))
|
||||
if not urls:
|
||||
continue
|
||||
platform_hint = first_text(record.get(NOTE_PLATFORM_FIELD))
|
||||
for url in urls:
|
||||
url = normalize_note_url(url)
|
||||
if not is_supported_note_url(url):
|
||||
log(f"Skip unsupported URL: {style.style_name} | {url}")
|
||||
continue
|
||||
note_tasks.append(
|
||||
NoteTask(
|
||||
style_name=style.style_name,
|
||||
style_record_id=style.record_id,
|
||||
note_record_id=first_text(record.get("_record_id")),
|
||||
platform_hint=platform_hint,
|
||||
creator_name=first_text(record.get(CREATOR_FIELD)),
|
||||
title=first_text(record.get(TITLE_FIELD)),
|
||||
note_url=url,
|
||||
source_base_url=style.base_ref.url,
|
||||
)
|
||||
)
|
||||
count_for_style += 1
|
||||
if max_notes_per_style and count_for_style >= max_notes_per_style:
|
||||
break
|
||||
if max_notes_per_style and count_for_style >= max_notes_per_style:
|
||||
break
|
||||
log(f"Found {count_for_style} note links: {style.style_name}")
|
||||
return note_tasks
|
||||
|
||||
|
||||
def normalize_comment_rows(
|
||||
task: NoteTask,
|
||||
platform: str,
|
||||
comments: list[dict[str, Any]],
|
||||
note_metrics: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for comment in comments:
|
||||
comment_text = first_text(comment.get("text") or comment.get("content") or comment.get("message"))
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"style_name": task.style_name,
|
||||
"creator_name": task.creator_name,
|
||||
"note_title": task.title,
|
||||
"platform": platform,
|
||||
"note_url": task.note_url,
|
||||
"comment_text": comment_text,
|
||||
"comment_count": note_metrics.get("comment_count", ""),
|
||||
"like_count": note_metrics.get("like_count", ""),
|
||||
"favorite_count": note_metrics.get("favorite_count", ""),
|
||||
"share_count": note_metrics.get("share_count", ""),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Database sync helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def sync_styles_to_db(style_tasks: list[StyleTask]) -> dict[str, int]:
|
||||
"""Upsert styles and return {style_name: style_id}."""
|
||||
style_ids: dict[str, int] = {}
|
||||
for style in style_tasks:
|
||||
style_id = db.upsert_style(
|
||||
name=style.style_name,
|
||||
platform=style.platform,
|
||||
feishu_base_url=style.base_ref.url,
|
||||
feishu_base_token=style.base_ref.base_token,
|
||||
feishu_table_id=style.base_ref.table_id,
|
||||
feishu_view_id=style.base_ref.view_id,
|
||||
)
|
||||
style_ids[style.style_name] = style_id
|
||||
log(f"Synced {len(style_ids)} styles to database")
|
||||
return style_ids
|
||||
|
||||
|
||||
def sync_notes_to_db(note_tasks: list[NoteTask], style_ids: dict[str, int]) -> dict[str, int]:
|
||||
"""Upsert creators and notes, link styles and creators. Return {url: note_id}."""
|
||||
creator_ids: dict[str, int] = {}
|
||||
note_ids: dict[str, int] = {}
|
||||
for task in note_tasks:
|
||||
style_id = style_ids.get(task.style_name)
|
||||
if not style_id:
|
||||
log(f"Skip DB sync for note without known style: {task.note_url}")
|
||||
continue
|
||||
|
||||
creator_name = task.creator_name or "unknown_creator"
|
||||
if creator_name not in creator_ids:
|
||||
creator_ids[creator_name] = db.upsert_creator(creator_name)
|
||||
creator_id = creator_ids[creator_name]
|
||||
db.link_style_creator(style_id, creator_id)
|
||||
|
||||
platform = detect_platform(task.note_url, task.platform_hint)
|
||||
note_id = db.upsert_note(
|
||||
style_id=style_id,
|
||||
creator_id=creator_id,
|
||||
platform=platform,
|
||||
url=task.note_url,
|
||||
title=task.title,
|
||||
feishu_record_id=task.note_record_id,
|
||||
)
|
||||
note_ids[task.note_url] = note_id
|
||||
log(f"Synced {len(note_ids)} notes to database")
|
||||
return note_ids
|
||||
|
||||
|
||||
def save_scrape_to_db(
|
||||
task: NoteTask,
|
||||
platform: str,
|
||||
comments: list[dict[str, Any]],
|
||||
note_metrics: dict[str, Any],
|
||||
note_ids: dict[str, int],
|
||||
) -> int:
|
||||
"""Update note metrics and replace comments in DB."""
|
||||
note_id = note_ids.get(task.note_url)
|
||||
if not note_id:
|
||||
return 0
|
||||
db.update_note_metrics(
|
||||
note_id=note_id,
|
||||
like_count=_to_int(note_metrics.get("like_count")),
|
||||
favorite_count=_to_int(note_metrics.get("favorite_count")),
|
||||
comment_count=_to_int(note_metrics.get("comment_count")),
|
||||
share_count=_to_int(note_metrics.get("share_count")),
|
||||
)
|
||||
allow_empty = _to_int(note_metrics.get("comment_count")) == 0
|
||||
if not comments and not allow_empty:
|
||||
log(f"Skip replacing comments for {task.note_url}: empty scrape result without confirmed zero comments")
|
||||
return 0
|
||||
inserted = db.replace_comments(note_id, platform, comments, allow_empty=allow_empty)
|
||||
return inserted
|
||||
|
||||
|
||||
def _to_int(value: Any) -> int | None:
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
return int(float(value))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def scrape_note(
|
||||
task: NoteTask,
|
||||
args: argparse.Namespace,
|
||||
note_ids: dict[str, int] | None = None,
|
||||
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
platform = detect_platform(task.note_url, task.platform_hint)
|
||||
started_at = datetime.now().isoformat(timespec="seconds")
|
||||
result: dict[str, Any] = {
|
||||
"started_at": started_at,
|
||||
"style_name": task.style_name,
|
||||
"style_record_id": task.style_record_id,
|
||||
"note_record_id": task.note_record_id,
|
||||
"platform_hint": task.platform_hint,
|
||||
"platform": platform,
|
||||
"creator_name": task.creator_name,
|
||||
"title": task.title,
|
||||
"note_url": task.note_url,
|
||||
"source_base_url": task.source_base_url,
|
||||
"status": "pending",
|
||||
"comment_count": 0,
|
||||
"json_path": "",
|
||||
"csv_path": "",
|
||||
"error": "",
|
||||
}
|
||||
|
||||
if platform == "unknown":
|
||||
result["status"] = "skipped"
|
||||
result["error"] = "Unsupported note URL"
|
||||
return result, []
|
||||
|
||||
if args.dry_run:
|
||||
result["status"] = "dry_run"
|
||||
return result, []
|
||||
|
||||
try:
|
||||
if platform == "xiaohongshu":
|
||||
comments, result = xiaohongshu_comment_scraper.scrape_comments(
|
||||
task.note_url,
|
||||
args.login_timeout,
|
||||
args.max_scrolls,
|
||||
args.idle_rounds,
|
||||
args.headless,
|
||||
)
|
||||
json_path, csv_path = xiaohongshu_comment_scraper.write_outputs(task.note_url, comments, result)
|
||||
note_metrics = (result.get("stats") or {}).get("note_metrics", {})
|
||||
elif platform == "douyin":
|
||||
comments, result = douyin_comment_scraper.scrape_comments(
|
||||
task.note_url,
|
||||
args.login_timeout,
|
||||
args.max_scrolls,
|
||||
args.idle_rounds,
|
||||
args.headless,
|
||||
)
|
||||
json_path, csv_path = douyin_comment_scraper.write_outputs(task.note_url, comments, result)
|
||||
note_metrics = (result.get("stats") or {}).get("note_metrics", {})
|
||||
else:
|
||||
comments, result = bilibili_comment_scraper.scrape_comments(
|
||||
task.note_url,
|
||||
args.bilibili_max_pages,
|
||||
args.bilibili_delay,
|
||||
args.headless,
|
||||
args.bilibili_max_reply_pages,
|
||||
)
|
||||
json_path, csv_path = bilibili_comment_scraper.write_outputs(task.note_url, comments, result)
|
||||
note_metrics = (result.get("stats") or {}).get("note_metrics", {})
|
||||
|
||||
result["status"] = "ok"
|
||||
result["comment_count"] = len(comments)
|
||||
result["json_path"] = str(json_path)
|
||||
result["csv_path"] = str(csv_path)
|
||||
comment_rows = normalize_comment_rows(task, platform, comments, note_metrics)
|
||||
|
||||
if not args.no_db and note_ids:
|
||||
try:
|
||||
db_inserted = save_scrape_to_db(task, platform, comments, note_metrics, note_ids)
|
||||
log(f"Saved {db_inserted} comments to DB for {task.note_url}")
|
||||
except Exception as exc:
|
||||
log(f"Failed to save comments to DB: {exc}")
|
||||
except Exception as exc:
|
||||
result["status"] = "failed"
|
||||
result["error"] = repr(exc)
|
||||
comment_rows = []
|
||||
|
||||
result["finished_at"] = datetime.now().isoformat(timespec="seconds")
|
||||
return result, comment_rows
|
||||
|
||||
|
||||
def scrape_notes_for_platform(
|
||||
platform: str,
|
||||
tasks: list[NoteTask],
|
||||
args_dict: dict[str, Any],
|
||||
note_ids: dict[str, int] | None,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
"""Scrape all notes for one platform in a separate process/browser."""
|
||||
results: list[dict[str, Any]] = []
|
||||
comment_rows: list[dict[str, Any]] = []
|
||||
|
||||
class SimpleArgs:
|
||||
def __init__(self, data: dict[str, Any]):
|
||||
self.__dict__.update(data)
|
||||
|
||||
platform_args = SimpleArgs(args_dict)
|
||||
|
||||
for index, task in enumerate(tasks, 1):
|
||||
log(f"[{platform} {index}/{len(tasks)}] {task.style_name} | {task.note_url}")
|
||||
result, rows = scrape_note(task, platform_args, note_ids)
|
||||
results.append(result)
|
||||
comment_rows.extend(rows)
|
||||
if platform_args.between_notes_delay > 0 and index < len(tasks):
|
||||
time.sleep(platform_args.between_notes_delay)
|
||||
|
||||
return results, comment_rows
|
||||
|
||||
|
||||
def current_week_label() -> str:
|
||||
"""Return a label like '2026_06_week4' for the current date."""
|
||||
now = datetime.now()
|
||||
week_of_month = (now.day - 1) // 7 + 1
|
||||
return f"{now.year}_{now.month:02d}_week{week_of_month}"
|
||||
|
||||
|
||||
def write_manifest(rows: list[dict[str, Any]]) -> tuple[Path, Path]:
|
||||
SUMMARY_DIR.mkdir(exist_ok=True)
|
||||
label = current_week_label()
|
||||
json_path = SUMMARY_DIR / f"feishu_comment_batch_{label}.json"
|
||||
csv_path = SUMMARY_DIR / f"feishu_comment_batch_{label}.csv"
|
||||
json_path.write_text(json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
fieldnames = [
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"style_name",
|
||||
"style_record_id",
|
||||
"note_record_id",
|
||||
"platform_hint",
|
||||
"platform",
|
||||
"creator_name",
|
||||
"title",
|
||||
"note_url",
|
||||
"source_base_url",
|
||||
"status",
|
||||
"comment_count",
|
||||
"json_path",
|
||||
"csv_path",
|
||||
"error",
|
||||
]
|
||||
with csv_path.open("w", newline="", encoding="utf-8-sig") as file:
|
||||
writer = csv.DictWriter(file, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
return json_path, csv_path
|
||||
|
||||
|
||||
def write_comment_summary(rows: list[dict[str, Any]]) -> tuple[Path, Path]:
|
||||
SUMMARY_DIR.mkdir(exist_ok=True)
|
||||
label = current_week_label()
|
||||
json_path = SUMMARY_DIR / f"feishu_comments_{label}.json"
|
||||
csv_path = SUMMARY_DIR / f"feishu_comments_{label}.csv"
|
||||
|
||||
note_counts: dict[tuple[str, str, str, str, str], int] = {}
|
||||
for row in rows:
|
||||
key = (
|
||||
first_text(row.get("style_name")),
|
||||
first_text(row.get("creator_name")),
|
||||
first_text(row.get("note_title")),
|
||||
first_text(row.get("platform")),
|
||||
first_text(row.get("note_url")),
|
||||
)
|
||||
note_counts[key] = note_counts.get(key, 0) + 1
|
||||
for row in rows:
|
||||
key = (
|
||||
first_text(row.get("style_name")),
|
||||
first_text(row.get("creator_name")),
|
||||
first_text(row.get("note_title")),
|
||||
first_text(row.get("platform")),
|
||||
first_text(row.get("note_url")),
|
||||
)
|
||||
if row.get("comment_count") in ("", None):
|
||||
row["comment_count"] = note_counts.get(key, 0)
|
||||
|
||||
json_path.write_text(json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
columns = [
|
||||
("style_name", "\u6b3e\u5f0f"),
|
||||
("creator_name", "\u535a\u4e3b\u540d"),
|
||||
("note_title", "\u7b14\u8bb0\u540d"),
|
||||
("platform", "\u5e73\u53f0"),
|
||||
("note_url", "\u7b14\u8bb0\u94fe\u63a5"),
|
||||
("comment_text", "\u8bc4\u8bba\u5185\u5bb9"),
|
||||
("comment_count", "\u8bc4\u8bba\u6570"),
|
||||
("like_count", "\u70b9\u8d5e\u6570"),
|
||||
("favorite_count", "\u6536\u85cf\u6570"),
|
||||
("share_count", "\u5206\u4eab\u91cf"),
|
||||
]
|
||||
with csv_path.open("w", newline="", encoding="utf-8-sig") as file:
|
||||
writer = csv.DictWriter(file, fieldnames=[label for _, label in columns])
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow({label: row.get(key, "") for key, label in columns})
|
||||
return json_path, csv_path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Read Feishu Base note links and batch scrape comments.")
|
||||
parser.add_argument("--index-base-token", default=DEFAULT_INDEX_BASE_TOKEN)
|
||||
parser.add_argument("--index-table-id", default=DEFAULT_INDEX_TABLE_ID)
|
||||
parser.add_argument("--index-view-id", default=DEFAULT_INDEX_VIEW_ID)
|
||||
parser.add_argument("--index-platform", default="天猫", help="only read this platform group from the style index; use empty string for all")
|
||||
parser.add_argument("--max-styles", type=int, default=None, help="limit styles for testing")
|
||||
parser.add_argument("--max-notes-per-style", type=int, default=None, help="limit note links per style for testing")
|
||||
parser.add_argument("--platform", choices=["all", "xiaohongshu", "douyin", "bilibili"], default="all")
|
||||
parser.add_argument("--dry-run", action="store_true", help="only read Feishu and list tasks; do not scrape comments")
|
||||
parser.add_argument("--headless", action="store_true")
|
||||
parser.add_argument("--login-timeout", type=int, default=300)
|
||||
parser.add_argument("--max-scrolls", type=int, default=120)
|
||||
parser.add_argument("--idle-rounds", type=int, default=8)
|
||||
parser.add_argument("--bilibili-max-pages", type=int, default=200)
|
||||
parser.add_argument("--bilibili-max-reply-pages", type=int, default=200)
|
||||
parser.add_argument("--bilibili-delay", type=float, default=0.25)
|
||||
parser.add_argument("--between-notes-delay", type=float, default=1.0)
|
||||
parser.add_argument("--no-db", action="store_true", help="do not write to PostgreSQL database")
|
||||
parser.add_argument("--parallel", action="store_true", default=True, help="run one browser per platform in parallel (default)")
|
||||
parser.add_argument("--no-parallel", action="store_true", help="disable parallel scraping")
|
||||
args = parser.parse_args()
|
||||
|
||||
style_tasks = load_style_tasks(
|
||||
args.index_base_token,
|
||||
args.index_table_id,
|
||||
args.index_view_id,
|
||||
args.index_platform,
|
||||
args.max_styles,
|
||||
)
|
||||
log(f"Found {len(style_tasks)} style Base links")
|
||||
note_tasks = load_note_tasks(style_tasks, args.max_notes_per_style)
|
||||
if args.platform != "all":
|
||||
note_tasks = [task for task in note_tasks if detect_platform(task.note_url, task.platform_hint) == args.platform]
|
||||
log(f"Prepared {len(note_tasks)} note scrape tasks")
|
||||
|
||||
style_ids: dict[str, int] = {}
|
||||
note_ids: dict[str, int] = {}
|
||||
if not args.no_db:
|
||||
style_ids = sync_styles_to_db(style_tasks)
|
||||
note_ids = sync_notes_to_db(note_tasks, style_ids)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
comment_rows: list[dict[str, Any]] = []
|
||||
|
||||
parallel = args.parallel and not args.no_parallel and len(note_tasks) > 1
|
||||
if parallel:
|
||||
platform_groups: dict[str, list[NoteTask]] = {}
|
||||
for task in note_tasks:
|
||||
platform = detect_platform(task.note_url, task.platform_hint)
|
||||
platform_groups.setdefault(platform, []).append(task)
|
||||
|
||||
active_groups = {p: t for p, t in platform_groups.items() if p in ("xiaohongshu", "douyin", "bilibili")}
|
||||
if len(active_groups) > 1:
|
||||
log(f"Running parallel scraping for platforms: {list(active_groups.keys())}")
|
||||
args_dict = vars(args)
|
||||
worker_inputs = [
|
||||
(platform, tasks, args_dict, note_ids if not args.no_db else None)
|
||||
for platform, tasks in active_groups.items()
|
||||
]
|
||||
with Pool(processes=len(worker_inputs)) as pool:
|
||||
worker_outputs = pool.starmap(scrape_notes_for_platform, worker_inputs)
|
||||
for worker_results, worker_rows in worker_outputs:
|
||||
results.extend(worker_results)
|
||||
comment_rows.extend(worker_rows)
|
||||
else:
|
||||
parallel = False
|
||||
|
||||
if not parallel:
|
||||
for index, task in enumerate(note_tasks, 1):
|
||||
platform = detect_platform(task.note_url, task.platform_hint)
|
||||
log(f"[{index}/{len(note_tasks)}] {task.style_name} | {platform} | {task.note_url}")
|
||||
result, rows = scrape_note(task, args, note_ids if not args.no_db else None)
|
||||
results.append(result)
|
||||
comment_rows.extend(rows)
|
||||
if args.between_notes_delay > 0 and index < len(note_tasks):
|
||||
time.sleep(args.between_notes_delay)
|
||||
|
||||
json_path, csv_path = write_manifest(results)
|
||||
comments_json_path, comments_csv_path = write_comment_summary(comment_rows)
|
||||
ok_count = sum(1 for row in results if row.get("status") == "ok")
|
||||
failed_count = sum(1 for row in results if row.get("status") == "failed")
|
||||
skipped_count = sum(1 for row in results if row.get("status") in {"skipped", "dry_run"})
|
||||
log(f"Done. ok={ok_count}, failed={failed_count}, skipped/dry_run={skipped_count}")
|
||||
log(f"Collected comment rows: {len(comment_rows)}")
|
||||
log(f"Manifest JSON: {json_path}")
|
||||
log(f"Manifest CSV: {csv_path}")
|
||||
log(f"Comments JSON: {comments_json_path}")
|
||||
log(f"Comments CSV: {comments_csv_path}")
|
||||
return 0 if failed_count == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,787 @@
|
||||
"""Feishu Docx (v2 blocks API) writer for note analysis reports.
|
||||
|
||||
Wraps `lark-cli docs +create/+update --api-version v2` with:
|
||||
- atomic state persistence in data/state/feishu_doc.json
|
||||
- XML escaping + XML-native builders (data table, keyword distribution, etc.)
|
||||
- a node + run.js Windows pattern (mirrors analyze_note.send_feishu_report) to avoid
|
||||
cmd 8K argv limit and `&|<>^%` escaping pitfalls on Windows
|
||||
- permission smoke test before any real write
|
||||
|
||||
All lark-cli calls use `--profile hermes-analyzer --as bot --format json`.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
STATE_DIR = PATHS.state_root
|
||||
TMP_DIR = PATHS.tmp_root / "feishu_doc_writer"
|
||||
|
||||
STATE_FILE = STATE_DIR / "feishu_doc.json"
|
||||
|
||||
PROFILE = "hermes-analyzer"
|
||||
IDENTITY = "bot"
|
||||
|
||||
FEISHU_BASE_URL = "https://bu0zgpibak.feishu.cn"
|
||||
|
||||
# Cap for the LLM analysis text inside a doc section (matches analyze_comments).
|
||||
DOC_ANALYSIS_LIMIT = 6000
|
||||
|
||||
# Below this size we pass --content inline; above we write to a file and use --content @<rel>.
|
||||
INLINE_CONTENT_LIMIT = 7000
|
||||
|
||||
# Per skill: only escape TEXT inside tags, never the tags themselves.
|
||||
# "&" must be replaced first to avoid double-escaping "<" etc.
|
||||
_XML_ESCAPE_MAP = {
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
'"': """,
|
||||
}
|
||||
|
||||
|
||||
def xml_escape(text: Any) -> str:
|
||||
"""Escape XML text content. Converts newlines to <br/>."""
|
||||
if text is None:
|
||||
return ""
|
||||
s = str(text)
|
||||
for src, dst in _XML_ESCAPE_MAP.items():
|
||||
s = s.replace(src, dst)
|
||||
s = s.replace("\r\n", "\n").replace("\n", "<br/>")
|
||||
return s
|
||||
|
||||
|
||||
def safe_div(numerator: Any, denominator: Any) -> float:
|
||||
try:
|
||||
n = float(numerator)
|
||||
d = float(denominator)
|
||||
if d == 0:
|
||||
return 0.0
|
||||
return n / d
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _pct(v: float) -> str:
|
||||
return f"{v * 100:.2f}%" if v else "—"
|
||||
|
||||
|
||||
def _num(v: Any) -> str:
|
||||
if v in ("", None):
|
||||
return "—"
|
||||
try:
|
||||
return f"{int(float(v)):,}"
|
||||
except (TypeError, ValueError):
|
||||
return str(v)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# lark-cli subprocess wrapper
|
||||
# ---------------------------------------------------------------------------
|
||||
def run_lark(args: list[str], cwd: Path | None = None) -> dict[str, Any]:
|
||||
"""Invoke lark-cli via node + run.js, return parsed JSON dict."""
|
||||
run_js = (
|
||||
Path(os.environ.get("APPDATA", str(Path.home())))
|
||||
/ "npm"
|
||||
/ "node_modules"
|
||||
/ "@larksuite"
|
||||
/ "cli"
|
||||
/ "scripts"
|
||||
/ "run.js"
|
||||
)
|
||||
# Scheduled tasks run as SYSTEM; %APPDATA% is not Administrator's dir
|
||||
if not run_js.exists():
|
||||
admin_run_js = Path(os.path.expandvars(r"%APPDATA%\npm\node_modules\@larksuite\cli\scripts\run.js"))
|
||||
if admin_run_js.exists():
|
||||
run_js = admin_run_js
|
||||
if not run_js.exists():
|
||||
raise RuntimeError(f"找不到 lark-cli 入口: {run_js}")
|
||||
|
||||
cmd = [
|
||||
"node", str(run_js),
|
||||
"--profile", PROFILE,
|
||||
"--as", IDENTITY,
|
||||
"--format", "json",
|
||||
*args,
|
||||
]
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(cwd) if cwd else None,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=180,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise RuntimeError(f"lark-cli 调用超时: {exc}") from exc
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"lark-cli 调用失败 (exit {proc.returncode})\n"
|
||||
f"cmd head: {' '.join(args[:4])}\n"
|
||||
f"stderr: {proc.stderr.strip()[:500]}\n"
|
||||
f"stdout: {proc.stdout[:500]}"
|
||||
)
|
||||
|
||||
out = proc.stdout.strip()
|
||||
if not out:
|
||||
return {}
|
||||
try:
|
||||
result = json.loads(out)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"lark-cli 返回非 JSON: {out[:500]}") from exc
|
||||
|
||||
if isinstance(result, dict) and result.get("ok") is False:
|
||||
err = result.get("error") or {}
|
||||
raise RuntimeError(
|
||||
f"lark-cli API error: code={err.get('code')} type={err.get('type')} "
|
||||
f"msg={err.get('message')}"
|
||||
)
|
||||
|
||||
# Some API errors surface as ok=True + result="failed" + warnings. Detect those.
|
||||
data = (result.get("data") or {}) if isinstance(result, dict) else {}
|
||||
op_result = data.get("result") if isinstance(data, dict) else None
|
||||
if op_result == "failed":
|
||||
warnings = data.get("warnings") or []
|
||||
raise RuntimeError(
|
||||
f"lark-cli 操作失败 (result=failed): warnings={warnings}"
|
||||
)
|
||||
return result if isinstance(result, dict) else {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State persistence
|
||||
# ---------------------------------------------------------------------------
|
||||
def load_state() -> dict[str, Any]:
|
||||
if not STATE_FILE.exists():
|
||||
return {"docs": {}, "written_notes": {}}
|
||||
try:
|
||||
data = json.loads(STATE_FILE.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
return {"docs": {}, "written_notes": {}}
|
||||
data.setdefault("docs", {})
|
||||
data.setdefault("written_notes", {})
|
||||
return data
|
||||
except json.JSONDecodeError:
|
||||
backup = STATE_FILE.with_suffix(f".corrupt-{int(time.time())}.json")
|
||||
shutil.copy2(STATE_FILE, backup)
|
||||
return {"docs": {}, "written_notes": {}, "_corrupt_backup": str(backup)}
|
||||
|
||||
|
||||
def save_state(state: dict[str, Any]) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
tmp = STATE_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(STATE_FILE)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Doc API wrappers
|
||||
# ---------------------------------------------------------------------------
|
||||
def create_doc(xml_content: str) -> dict[str, Any]:
|
||||
"""Create a new Docx. Returns {token, url, revision_id}."""
|
||||
result = run_lark([
|
||||
"docs", "+create",
|
||||
"--api-version", "v2",
|
||||
"--content", xml_content,
|
||||
])
|
||||
doc = (result.get("data") or {}).get("document") or {}
|
||||
return {
|
||||
"token": doc.get("document_id") or "",
|
||||
"url": doc.get("url") or "",
|
||||
"revision_id": doc.get("revision_id"),
|
||||
}
|
||||
|
||||
|
||||
def _invoke_update_with_content(
|
||||
doc: str, command: str, xml_content: str,
|
||||
*, anchor: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run docs +update with either inline --content or @file content."""
|
||||
args = [
|
||||
"docs", "+update",
|
||||
"--api-version", "v2",
|
||||
"--doc", doc,
|
||||
"--command", command,
|
||||
]
|
||||
if command.startswith("block_"):
|
||||
if not anchor:
|
||||
raise ValueError(f"command {command} requires --block-id")
|
||||
args.extend(["--block-id", anchor])
|
||||
|
||||
if len(xml_content.encode("utf-8")) <= INLINE_CONTENT_LIMIT:
|
||||
args.extend(["--content", xml_content])
|
||||
return run_lark(args)
|
||||
|
||||
TMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"_docwriter_{int(time.time() * 1000)}.xml"
|
||||
fpath = TMP_DIR / fname
|
||||
fpath.write_text(xml_content, encoding="utf-8")
|
||||
try:
|
||||
# lark-cli resolves `@file` relative to cwd. We cwd=STATE_DIR but the
|
||||
# file lives in STATE_DIR/tmp — pass the relative path from STATE_DIR.
|
||||
rel_path = f"tmp/{fname}"
|
||||
args.extend(["--content", f"@{rel_path}"])
|
||||
return run_lark(args, cwd=STATE_DIR)
|
||||
finally:
|
||||
try:
|
||||
fpath.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def append_blocks(doc: str, xml_content: str) -> dict[str, Any]:
|
||||
"""Append XML blocks to end of doc. Returns {revision_id}."""
|
||||
return _invoke_update_with_content(doc, "append", xml_content)
|
||||
|
||||
|
||||
def insert_blocks_after(doc: str, anchor_block_id: str, xml_content: str) -> dict[str, Any]:
|
||||
"""Insert XML blocks after a given block. Returns {revision_id}."""
|
||||
return _invoke_update_with_content(
|
||||
doc, "block_insert_after", xml_content, anchor=anchor_block_id,
|
||||
)
|
||||
|
||||
|
||||
def delete_blocks(doc: str, block_ids: list[str]) -> dict[str, Any]:
|
||||
"""Batch-delete blocks (comma-separated --block-id)."""
|
||||
if not block_ids:
|
||||
return {}
|
||||
args = [
|
||||
"docs", "+update",
|
||||
"--api-version", "v2",
|
||||
"--doc", doc,
|
||||
"--command", "block_delete",
|
||||
"--block-id", ",".join(block_ids),
|
||||
]
|
||||
return run_lark(args)
|
||||
|
||||
|
||||
def fetch_doc_blocks(doc: str, *, scope: str = "all") -> list[dict[str, Any]]:
|
||||
"""Fetch block IDs in a doc. Returns list of {block_id, block_type} dicts."""
|
||||
args = [
|
||||
"docs", "+fetch",
|
||||
"--api-version", "v2",
|
||||
"--doc", doc,
|
||||
"--detail", "with-ids",
|
||||
]
|
||||
if scope and scope != "all":
|
||||
args.extend(["--scope", scope])
|
||||
|
||||
result = run_lark(args)
|
||||
# The fetch returns content as XML string; parse block IDs from `id="..."` attrs.
|
||||
content = ((result.get("data") or {}).get("document") or {}).get("content") or ""
|
||||
out: list[dict[str, Any]] = []
|
||||
import re
|
||||
# Extract <tagname id="..."> in document order
|
||||
for m in re.finditer(r"<(\w+)\s+id=\"([^\"]+)\"", content):
|
||||
out.append({"block_id": m.group(2), "block_type": m.group(1)})
|
||||
return out
|
||||
|
||||
|
||||
def find_block_after(doc: str, anchor_text: str) -> str | None:
|
||||
"""Find the block_id of the first <h*> block whose text contains anchor_text.
|
||||
|
||||
Used to discover the h1 anchor after doc creation, since the API response
|
||||
doesn't include new_blocks.
|
||||
"""
|
||||
blocks = fetch_doc_blocks(doc, scope="all")
|
||||
for b in blocks:
|
||||
if b["block_type"] in ("h1", "h2", "h3", "h4"):
|
||||
# We need the actual text; refetch with content
|
||||
pass
|
||||
# The fetch returned content embedded; do a full re-fetch with text
|
||||
result = run_lark([
|
||||
"docs", "+fetch",
|
||||
"--api-version", "v2",
|
||||
"--doc", doc,
|
||||
"--detail", "with-ids",
|
||||
])
|
||||
content = ((result.get("data") or {}).get("document") or {}).get("content") or ""
|
||||
import re
|
||||
# Find all <h1 ...>...</h1> or <h2 ...>...</h2>
|
||||
pattern = re.compile(r"<(h\d)\s+id=\"([^\"]+)\"[^>]*>(.*?)</\1>", re.DOTALL)
|
||||
for m in pattern.finditer(content):
|
||||
if anchor_text in m.group(3):
|
||||
return m.group(2)
|
||||
return None
|
||||
|
||||
|
||||
def find_blocks_after_anchor(doc: str, anchor_block_id: str) -> list[str]:
|
||||
"""Return block_ids of every block that appears AFTER anchor_block_id, in order.
|
||||
|
||||
Used by populate_master_index to delete stale index content before re-inserting.
|
||||
"""
|
||||
blocks = fetch_doc_blocks(doc, scope="all")
|
||||
out: list[str] = []
|
||||
found = False
|
||||
for b in blocks:
|
||||
if found:
|
||||
out.append(b["block_id"])
|
||||
elif b["block_id"] == anchor_block_id:
|
||||
found = True
|
||||
return out
|
||||
|
||||
|
||||
def find_index_blocks(doc: str, anchor_block_id: str) -> list[str]:
|
||||
"""Return block_ids of the index content (blocks between anchor h1 and the next style section).
|
||||
|
||||
The index lives immediately after the anchor h1 and ends at the first
|
||||
subsequent style section marker — either an H1 (the style title) or a
|
||||
bookmark that precedes it. This avoids deleting style sections.
|
||||
"""
|
||||
blocks = fetch_doc_blocks(doc, scope="all")
|
||||
out: list[str] = []
|
||||
found_anchor = False
|
||||
for b in blocks:
|
||||
if not found_anchor:
|
||||
if b["block_id"] == anchor_block_id:
|
||||
found_anchor = True
|
||||
continue
|
||||
# Stop at the first style-section marker after the anchor.
|
||||
if b["block_type"] in ("h1", "bookmark"):
|
||||
break
|
||||
out.append(b["block_id"])
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# XML builders
|
||||
# ---------------------------------------------------------------------------
|
||||
CATEGORY_LABELS = {
|
||||
"purchase_intent": "购买意向",
|
||||
"positive_feature": "正向功能反馈",
|
||||
"negative_feature": "负向功能反馈",
|
||||
"user_scenario": "用户场景",
|
||||
"aesthetic": "审美与风格",
|
||||
"brand_emotion": "品牌情感",
|
||||
}
|
||||
|
||||
|
||||
def build_basic_info_xml(info: dict[str, Any]) -> str:
|
||||
title = info.get("title") or "未获取"
|
||||
platform = info.get("platform") or "unknown"
|
||||
creator = info.get("creator_name") or "未指定"
|
||||
style_name = info.get("style_name") or ""
|
||||
source_url = info.get("source_url") or ""
|
||||
brand = info.get("brand") or ""
|
||||
|
||||
target_label = " / ".join(x for x in (brand, style_name) if x) if (brand or style_name) else "未指定"
|
||||
|
||||
lines = [
|
||||
f"<p><b>标题</b> {xml_escape(title)}</p>",
|
||||
f"<p><b>平台</b> {platform} · <b>达人</b> {xml_escape(creator)}</p>",
|
||||
f"<p><b>品牌 / 产品</b> {xml_escape(target_label)}</p>",
|
||||
]
|
||||
if source_url:
|
||||
lines.append(f'<p><b>链接</b> <a href="{xml_escape(source_url)}">{xml_escape(source_url)}</a></p>')
|
||||
|
||||
return "<callout emoji=\"📌\" background-color=\"light-blue\" border-color=\"blue\">" + "".join(lines) + "</callout>"
|
||||
|
||||
|
||||
def build_data_table_xml(
|
||||
info: dict[str, Any], metrics: dict[str, Any], raw_total: int, filtered_total: int,
|
||||
) -> str:
|
||||
"""XML table mirroring analyze_note._data_table_markdown."""
|
||||
has_view = bool(metrics.get("view_count"))
|
||||
likes = info.get("like_count")
|
||||
favorites = info.get("favorite_count")
|
||||
comments = info.get("comment_count_metric") or info.get("comment_count_total")
|
||||
shares = info.get("share_count")
|
||||
valid_ratio = safe_div(filtered_total, raw_total)
|
||||
|
||||
rows: list[str] = []
|
||||
rows.append(_tr(("维度", "指标", "数值"), header=True))
|
||||
rows.append(_tr(("基础", "点赞", _num(likes))))
|
||||
rows.append(_tr(("基础", "收藏", _num(favorites))))
|
||||
rows.append(_tr(("基础", "评论", _num(comments))))
|
||||
rows.append(_tr(("基础", "分享", _num(shares))))
|
||||
rows.append(_tr(("传播", "曝光量", _num(metrics.get("view_count")) if has_view else "未采集")))
|
||||
rows.append(_tr(("互动率", "点赞率", _pct(metrics.get("like_rate", 0)))))
|
||||
rows.append(_tr(("互动率", "评论率", _pct(metrics.get("comment_rate", 0)))))
|
||||
rows.append(_tr(("互动率", "收藏率", _pct(metrics.get("favorite_rate", 0)))))
|
||||
rows.append(_tr(("互动率", "分享率", _pct(metrics.get("share_rate", 0)))))
|
||||
rows.append(_tr(("互动率", "综合互动率", _pct(metrics.get("engagement_rate", 0)))))
|
||||
fav_to_cmt = metrics.get("favorite_to_comment", 0) or 0
|
||||
rows.append(_tr(("互动率", "收藏评论比", f"{fav_to_cmt:.2f}")))
|
||||
rows.append(_tr(("质量", "有效率", _pct(valid_ratio))))
|
||||
|
||||
return (
|
||||
'<callout emoji="📊" background-color="light-blue" border-color="blue">'
|
||||
"<table>"
|
||||
"<colgroup><col span=\"3\" width=\"120\"/></colgroup>"
|
||||
"<thead>" + rows[0] + "</thead>"
|
||||
"<tbody>" + "".join(rows[1:]) + "</tbody>"
|
||||
"</table>"
|
||||
"</callout>"
|
||||
)
|
||||
|
||||
|
||||
def _tr(cells: tuple[str, str, str], *, header: bool = False) -> str:
|
||||
tag = "th" if header else "td"
|
||||
bg = ' background-color="light-gray"' if header else ""
|
||||
return (
|
||||
f"<tr>"
|
||||
+ "".join(f"<{tag}{bg}>{xml_escape(c)}</{tag}>" for c in cells)
|
||||
+ "</tr>"
|
||||
)
|
||||
|
||||
|
||||
def build_keyword_distribution_xml(
|
||||
distribution: dict[str, Any], total_filtered: int,
|
||||
) -> str:
|
||||
overall = distribution.get("overall", {}) if isinstance(distribution, dict) else {}
|
||||
relevant = distribution.get("relevant", {}) if isinstance(distribution, dict) else {}
|
||||
relevant_total = distribution.get("relevant_total", 0) if isinstance(distribution, dict) else 0
|
||||
|
||||
if not overall:
|
||||
return "<p>（无关键词分布数据）</p>"
|
||||
|
||||
def fmt(d: dict[str, int]) -> str:
|
||||
return "".join(
|
||||
f"<li>{xml_escape(CATEGORY_LABELS[k])}：{d.get(k, 0)} 条</li>"
|
||||
for k in CATEGORY_LABELS
|
||||
)
|
||||
|
||||
overall_block = (
|
||||
f"<p><b>全评论（{total_filtered} 条）</b></p>"
|
||||
f"<ul>{fmt(overall)}</ul>"
|
||||
)
|
||||
if relevant_total:
|
||||
relevant_block = (
|
||||
f"<p><b>我方产品相关（{relevant_total} 条）</b></p>"
|
||||
f"<ul>{fmt(relevant)}</ul>"
|
||||
)
|
||||
else:
|
||||
relevant_block = "<p><b>我方产品相关（0 条）</b></p><p><i>（无评论明确提及我方产品/品牌）</i></p>"
|
||||
|
||||
return (
|
||||
'<callout emoji="🔑" background-color="light-cyan" border-color="cyan">'
|
||||
+ overall_block + relevant_block
|
||||
+ "</callout>"
|
||||
)
|
||||
|
||||
|
||||
# 模块 emoji 和颜色映射
|
||||
_MODULE_STYLES = {
|
||||
"模块1": ("🎯", "light-blue", "blue"), # 笔记表现归因
|
||||
"模块2": ("📊", "light-green", "green"), # 评论质量分层
|
||||
"模块3": ("🛡️", "light-red", "red"), # 品牌安全与控评
|
||||
"模块4": ("🎬", "light-purple", "purple"), # 脚本有效性拆解
|
||||
"模块5": ("🤝", "light-yellow", "orange"), # 达人匹配度评估
|
||||
"模块6": ("💡", "light-cyan", "cyan"), # 可沉淀的选题资产
|
||||
"模块7": ("✅", "light-green", "green"), # 下一篇可执行优化清单
|
||||
}
|
||||
|
||||
|
||||
def _split_analysis_modules(analysis_text: str) -> list[str]:
|
||||
"""将 LLM 输出的 7 模块分析拆分为独立 callout 块。
|
||||
|
||||
每个模块用不同颜色的 callout 渲染，视觉层次更分明。
|
||||
"""
|
||||
if not analysis_text or analysis_text == "（无分析结果）":
|
||||
return ['<callout emoji="💡" background-color="light-gray" border-color="gray">'
|
||||
'<p>（无分析结果）</p></callout>']
|
||||
|
||||
# 按模块标记拆分
|
||||
import re as _re
|
||||
parts = _re.split(r'(模块\d+[｜|])', analysis_text)
|
||||
# parts: [前缀, "模块1｜", 内容1, "模块2｜", 内容2, ...]
|
||||
|
||||
blocks: list[str] = []
|
||||
|
||||
# 处理模块前的引言文字（如有）
|
||||
if parts and parts[0].strip():
|
||||
intro = parts[0].strip()
|
||||
if len(intro) > 20:
|
||||
blocks.append(f'<callout emoji="📋" background-color="light-gray" border-color="gray">'
|
||||
f'<p>{xml_escape(intro)}</p></callout>')
|
||||
|
||||
# 逐模块渲染
|
||||
i = 1
|
||||
while i < len(parts) - 1:
|
||||
module_header = parts[i] # "模块1｜"
|
||||
module_body = parts[i + 1] # 该模块的内容
|
||||
i += 2
|
||||
|
||||
# 提取模块编号
|
||||
m = _re.match(r'模块(\d+)', module_header)
|
||||
if not m:
|
||||
continue
|
||||
mod_num = m.group(1)
|
||||
mod_key = f"模块{mod_num}"
|
||||
emoji, bg, border = _MODULE_STYLES.get(mod_key, ("📌", "light-gray", "gray"))
|
||||
|
||||
# 模块标题行 + 正文
|
||||
body_text = module_body.strip()
|
||||
if not body_text:
|
||||
continue
|
||||
|
||||
# 截断过长内容
|
||||
if len(body_text) > 800:
|
||||
body_text = body_text[:800] + "\n...（已截断）"
|
||||
|
||||
blocks.append(
|
||||
f'<callout emoji="{emoji}" background-color="{bg}" border-color="{border}">'
|
||||
f'<p>{xml_escape(body_text)}</p></callout>'
|
||||
)
|
||||
|
||||
# 如果没有拆出模块（LLM 没按格式输出），整体兜底
|
||||
if not blocks:
|
||||
text = analysis_text
|
||||
if len(text) > DOC_ANALYSIS_LIMIT:
|
||||
text = text[:DOC_ANALYSIS_LIMIT] + "\n...（已截断）"
|
||||
blocks.append(f'<callout emoji="💡" background-color="light-blue" border-color="blue">'
|
||||
f'<p>{xml_escape(text)}</p></callout>')
|
||||
|
||||
return blocks
|
||||
|
||||
|
||||
def build_note_section_xml(
|
||||
info: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
distribution: dict[str, Any],
|
||||
raw_comments: list[str],
|
||||
filtered_comments: list[str],
|
||||
analysis: str,
|
||||
*,
|
||||
bookmark_name: str,
|
||||
doc_token: str,
|
||||
) -> str:
|
||||
title = info.get("title") or info.get("source_url") or "未获取"
|
||||
platform = info.get("platform") or "unknown"
|
||||
# 截断标题用于 H3，避免超长
|
||||
short_title = title[:30] + "…" if len(title) > 30 else title
|
||||
h_text = f"📝 {short_title}（{platform}）"
|
||||
|
||||
analysis_text = analysis or "（无分析结果）"
|
||||
if len(analysis_text) > DOC_ANALYSIS_LIMIT:
|
||||
analysis_text = analysis_text[:DOC_ANALYSIS_LIMIT] + "<br/>...（内容过长，已截断）"
|
||||
|
||||
# Bookmark requires href attribute; use the doc's own URL (target for incoming #bookmark links).
|
||||
bookmark_href = f"{FEISHU_BASE_URL}/docx/{doc_token}#{bookmark_name}"
|
||||
|
||||
# 将分析文本按模块拆分，逐段渲染
|
||||
analysis_parts = _split_analysis_modules(analysis_text)
|
||||
|
||||
return (
|
||||
f"<bookmark name=\"{xml_escape(bookmark_name)}\" href=\"{xml_escape(bookmark_href)}\"></bookmark>"
|
||||
f"<h3>{xml_escape(h_text)}</h3>"
|
||||
+ build_basic_info_xml(info)
|
||||
+ build_data_table_xml(info, metrics, len(raw_comments), len(filtered_comments))
|
||||
+ build_keyword_distribution_xml(distribution, len(filtered_comments))
|
||||
+ "".join(analysis_parts)
|
||||
)
|
||||
|
||||
|
||||
def build_master_summary_table_xml(
|
||||
per_platform: dict[str, dict[str, Any]],
|
||||
) -> str:
|
||||
"""Build the 4-row master summary table (平台/笔记数/详情)."""
|
||||
rows = []
|
||||
for p, info in per_platform.items():
|
||||
url = info.get("url") or ""
|
||||
n = info.get("count", 0)
|
||||
link_cell = f'<a href="{xml_escape(url)}">打开 →</a>' if url else "—"
|
||||
rows.append(
|
||||
f"<tr><td>{xml_escape(p)}</td><td>{n}</td><td>{link_cell}</td></tr>"
|
||||
)
|
||||
return (
|
||||
"<table>"
|
||||
"<colgroup><col width=\"120\"/><col width=\"80\"/><col width=\"300\"/></colgroup>"
|
||||
"<thead>" + _tr(("平台", "笔记数", "详情"), header=True) + "</thead>"
|
||||
"<tbody>" + "".join(rows) + "</tbody>"
|
||||
"</table>"
|
||||
)
|
||||
|
||||
|
||||
def build_per_platform_index_table_xml(
|
||||
platform: str, state: dict[str, Any], platform_doc_url: str,
|
||||
) -> tuple[str, int]:
|
||||
"""Build index table XML for one platform section in the master doc.
|
||||
|
||||
Returns (xml_string, row_count).
|
||||
"""
|
||||
rows: list[str] = []
|
||||
for nid_str, entry in state.get("written_notes", {}).items():
|
||||
if entry.get("platform") != platform:
|
||||
continue
|
||||
bookmark = entry.get("bookmark_name") or f"note-{nid_str}"
|
||||
link = f"{platform_doc_url}#{bookmark}"
|
||||
rows.append(
|
||||
f'<tr><td>{nid_str}</td>'
|
||||
f'<td><a href="{xml_escape(link)}">查看 →</a></td></tr>'
|
||||
)
|
||||
rows.sort(key=lambda r: int(re.search(r">(\d+)<", r).group(1)) if re.search(r">(\d+)<", r) else 0)
|
||||
body = "".join(rows)
|
||||
return (
|
||||
f"<h2>📑 {platform} 索引（{len(rows)} 条）</h2>"
|
||||
"<table>"
|
||||
"<colgroup><col width=\"100\"/><col width=\"180\"/></colgroup>"
|
||||
"<thead>" + _tr(("note_id", "详情"), header=True) + "</thead>"
|
||||
"<tbody>" + body + "</tbody>"
|
||||
"</table>"
|
||||
), len(rows)
|
||||
|
||||
|
||||
# Local re for sort
|
||||
import re # noqa: E402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-style doc builders (used by single-archive layout)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _doc_skeleton() -> str:
|
||||
"""Initial skeleton for the single archive doc.
|
||||
|
||||
Title + intro + H1 anchor placeholder for the 款式索引. The actual index
|
||||
table is populated later via populate_index_table() (replace-on-update).
|
||||
"""
|
||||
return (
|
||||
"<title>👜 笔记分析报告 — 按款式汇总</title>"
|
||||
"<p>本汇总由 write_notes_to_doc.py 自动生成。点击款式名称进入详情。</p>"
|
||||
+ "<h1>⚠️ 款式索引（点击跳转）</h1>"
|
||||
)
|
||||
|
||||
|
||||
def build_style_overview_xml(style_name: str, aggregated: dict[str, Any]) -> str:
|
||||
"""Callout block with style-level rollup metrics."""
|
||||
plat = aggregated.get("platform_breakdown") or {}
|
||||
plat_str = " / ".join(f"{k} {v}" for k, v in plat.items() if v) or "—"
|
||||
creator = aggregated.get("creator_breakdown") or {}
|
||||
creator_str = " / ".join(f"{k}×{v}" for k, v in list(creator.items())[:3]) or "—"
|
||||
|
||||
lines = [
|
||||
f"<p><b>款式</b> {xml_escape(style_name)}</p>",
|
||||
f"<p><b>笔记总数</b> {aggregated.get('note_count', 0)} 篇 · "
|
||||
f"<b>平台分布</b> {xml_escape(plat_str)}</p>",
|
||||
f"<p><b>头部达人</b> {xml_escape(creator_str)}</p>",
|
||||
f"<p><b>总互动</b> "
|
||||
f"点赞 {_num(aggregated.get('total_likes'))} · "
|
||||
f"收藏 {_num(aggregated.get('total_favorites'))} · "
|
||||
f"评论 {_num(aggregated.get('total_comments'))} · "
|
||||
f"分享 {_num(aggregated.get('total_shares'))}</p>",
|
||||
f"<p><b>总曝光量</b> {_num(aggregated.get('total_views')) if aggregated.get('total_views') else '未采集'} · "
|
||||
f"<b>综合互动率</b> {_pct(aggregated.get('engagement_rate') or 0)}</p>",
|
||||
]
|
||||
return (
|
||||
'<callout emoji="👜" background-color="light-yellow" border-color="orange">'
|
||||
+ "".join(lines)
|
||||
+ "</callout>"
|
||||
)
|
||||
|
||||
|
||||
def build_style_section_xml(
|
||||
style_id: int,
|
||||
style_name: str,
|
||||
aggregated: dict[str, Any],
|
||||
rollup_text: str,
|
||||
note_sections_xml: str,
|
||||
*,
|
||||
doc_token: str,
|
||||
) -> str:
|
||||
"""Assemble one full style section: bookmark + H1 + overview + rollup H2 + per-note sections.
|
||||
|
||||
`note_sections_xml` should be the concatenation of `build_note_section_xml(...)`
|
||||
outputs for each note in this style (each already has its own bookmark + H3).
|
||||
"""
|
||||
bookmark_name = f"style-{style_id}"
|
||||
bookmark_href = f"{FEISHU_BASE_URL}/docx/{doc_token}#{bookmark_name}"
|
||||
h1_text = f"👜 {style_name}（{aggregated.get('note_count', 0)} 篇笔记）"
|
||||
|
||||
note_count = aggregated.get("note_count", 0)
|
||||
|
||||
rollup_in_callout = (
|
||||
'<callout emoji="💼" background-color="light-purple" border-color="purple">'
|
||||
f'<p>{xml_escape(rollup_text or "（无整体分析结果）")}</p></callout>'
|
||||
)
|
||||
|
||||
parts: list[str] = [
|
||||
f'<bookmark name="{xml_escape(bookmark_name)}" href="{xml_escape(bookmark_href)}"></bookmark>',
|
||||
f"<h1>{xml_escape(h1_text)}</h1>",
|
||||
build_style_overview_xml(style_name, aggregated),
|
||||
"<h2>💼 款式整体分析</h2>",
|
||||
rollup_in_callout,
|
||||
f"<h2>📚 单篇笔记分析（{note_count} 篇）</h2>",
|
||||
]
|
||||
|
||||
if note_sections_xml:
|
||||
parts.append(note_sections_xml)
|
||||
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def build_style_index_table_xml(
|
||||
written_styles: dict[str, Any],
|
||||
*,
|
||||
archive_url: str,
|
||||
) -> str:
|
||||
"""Build the 款式索引 table at the top of the archive doc.
|
||||
|
||||
Rows are sorted by note count DESC (driven by caller-supplied dict order,
|
||||
which write_notes_to_doc.py maintains from db.get_all_styles_with_notes).
|
||||
Each row: 款式名 | 笔记数 | 跳转链接.
|
||||
"""
|
||||
rows: list[str] = []
|
||||
for sid_str, entry in written_styles.items():
|
||||
if not entry.get("doc_token"):
|
||||
continue
|
||||
# Find note count from entry (carried over by writer)
|
||||
n = entry.get("note_count", 0)
|
||||
bookmark = entry.get("bookmark_name") or f"style-{sid_str}"
|
||||
link = f"{archive_url}#{bookmark}"
|
||||
rows.append(
|
||||
f"<tr><td>{xml_escape(str(entry.get('style_name', sid_str)))}</td>"
|
||||
f"<td>{n}</td>"
|
||||
f'<td><a href="{xml_escape(link)}">跳转 →</a></td></tr>'
|
||||
)
|
||||
if not rows:
|
||||
rows.append("<tr><td>—</td><td>0</td><td>—</td></tr>")
|
||||
return (
|
||||
"<table>"
|
||||
"<colgroup><col width=\"160\"/><col width=\"80\"/><col width=\"300\"/></colgroup>"
|
||||
"<thead>" + _tr(("款式", "笔记数", "详情"), header=True) + "</thead>"
|
||||
"<tbody>" + "".join(rows) + "</tbody>"
|
||||
"</table>"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Permission smoke test
|
||||
# ---------------------------------------------------------------------------
|
||||
def permission_smoke_test() -> bool:
|
||||
print("→ Running permission smoke test (--as bot, docs:document:create)...")
|
||||
try:
|
||||
run_lark([
|
||||
"docs", "+create",
|
||||
"--api-version", "v2",
|
||||
"--content", "<title>__smoke_test__</title><p>please ignore</p>",
|
||||
])
|
||||
except RuntimeError as exc:
|
||||
print(f" ✗ Smoke test failed: {exc}")
|
||||
print()
|
||||
print(" The hermes-analyzer bot identity lacks the required scopes.")
|
||||
print(" Run: lark-cli auth login --profile hermes-analyzer")
|
||||
print(" And ensure the app is granted these scopes:")
|
||||
print(" - docs:document:create")
|
||||
print(" - docs:document:update")
|
||||
print(" - docs:document:read")
|
||||
print(" Then retry.")
|
||||
return False
|
||||
|
||||
print(f" ✓ Smoke test passed. (A throwaway doc was created — delete it manually from Drive.)")
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(0 if permission_smoke_test() else 1)
|
||||
@@ -0,0 +1,39 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Friday relogin: 4 QR scans — B站, 蒲公英, 星图(+同步到抖音评论), 小红书评论
|
||||
REM 星图登录完成后自动把 cookie 同步到抖音评论 scraper,不需要单独再扫抖音码
|
||||
REM Register: schtasks /Create /SC WEEKLY /D FRI /TN YingxiaoYunying_FridayRelogin /TR %PROJECT_DIR%\data\tools\friday_relogin.bat /ST 10:00 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing\scheduled
|
||||
set PYTHON=%GYXX_PYTHON%
|
||||
where %PYTHON% >nul 2>&1 || set PYTHON=python
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
|
||||
set TS=%date:~0,4%%date:~5,2%%date:~8,2%_%time:~0,2%%time:~3,2%%time:~6,2%
|
||||
set TS=%TS: =0%
|
||||
|
||||
set LOG_FILE=%LOG_DIR%\friday_relogin_%TS%.log
|
||||
|
||||
echo === Friday relogin started at %date% %time% === > "%LOG_FILE%"
|
||||
echo PROJECT_DIR=%PROJECT_DIR% >> "%LOG_FILE%"
|
||||
|
||||
cd /d "%PROJECT_DIR%"
|
||||
|
||||
echo --- Launch 5 QR relogins in parallel; send screenshots; retry up to 3 rounds --- >> "%LOG_FILE%"
|
||||
call %PYTHON% data\tools\friday_relogin_parallel.py --max-attempts 3 --round-timeout 300 --screenshot-delay 60 >> "%LOG_FILE%" 2>&1
|
||||
set FINAL_RC=%ERRORLEVEL%
|
||||
|
||||
echo === Friday relogin finished at %date% %time% (exit=%FINAL_RC%) === >> "%LOG_FILE%"
|
||||
|
||||
endlocal & exit /b %FINAL_RC%
|
||||
+408
@@ -0,0 +1,408 @@
|
||||
"""Launch five QR relogins in parallel, notify via analyzer Lark, retry 3 times."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
import psutil
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
PROJECT_DIR = PATHS.module_root
|
||||
ANALYZER_DIR = PATHS.tmp_root / "relogin"
|
||||
MARKETING_RECIPIENT = "ou_24cc944d6e43c69c59d6560ad4e2ae6e"
|
||||
TECHNICAL_RECIPIENT = "ou_7ad5fc8012e2f741afc5346e05ffd447"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Platform:
|
||||
label: str
|
||||
command: tuple[str, ...]
|
||||
cookie_file: Path
|
||||
required_cookie: str
|
||||
|
||||
|
||||
PLATFORMS = {
|
||||
"douyin": Platform(
|
||||
"\u6296\u97f3", ("data/tools/relogin_douyin.py",),
|
||||
PATHS.state_root / "cookies/douyin_cookies.json", "sessionid",
|
||||
),
|
||||
"bilibili": Platform(
|
||||
"B\u7ad9", ("data/tools/relogin_bilibili.py",),
|
||||
PATHS.state_root / "cookies/bilibili_cookies.json", "SESSDATA",
|
||||
),
|
||||
"pgy": Platform(
|
||||
"\u84b2\u516c\u82f1", ("data/tools/relogin_pgy.py",),
|
||||
PATHS.state_root / "cookies/pgy_cookies.json", "",
|
||||
),
|
||||
"xingtu": Platform(
|
||||
"\u661f\u56fe", ("data/tools/relogin_xingtu.py", "--force"),
|
||||
PATHS.state_root / "cookies/xingtu_cookies.json", "sessionid",
|
||||
),
|
||||
"xiaohongshu": Platform(
|
||||
"\u5c0f\u7ea2\u4e66", ("data/tools/relogin_xiaohongshu.py",),
|
||||
PATHS.state_root / "cookies/xiaohongshu_cookies.json", "web_session",
|
||||
),
|
||||
}
|
||||
|
||||
DEFAULT_RECIPIENTS = {
|
||||
"pgy": MARKETING_RECIPIENT,
|
||||
"xingtu": MARKETING_RECIPIENT,
|
||||
"douyin": TECHNICAL_RECIPIENT,
|
||||
"bilibili": TECHNICAL_RECIPIENT,
|
||||
"xiaohongshu": TECHNICAL_RECIPIENT,
|
||||
}
|
||||
|
||||
|
||||
def recipient_for_platform(platform_name: str, override: str | None = None) -> str:
|
||||
"""Return the responsible recipient, unless a manual all-platform override is set."""
|
||||
if override:
|
||||
return override
|
||||
try:
|
||||
return DEFAULT_RECIPIENTS[platform_name]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"no Lark recipient configured for platform: {platform_name}") from exc
|
||||
|
||||
|
||||
def group_platforms_by_recipient(
|
||||
platform_names: list[str], override: str | None = None,
|
||||
) -> dict[str, list[str]]:
|
||||
grouped: dict[str, list[str]] = {}
|
||||
for name in platform_names:
|
||||
grouped.setdefault(recipient_for_platform(name, override), []).append(name)
|
||||
return grouped
|
||||
|
||||
|
||||
def run_with_retries(
|
||||
platform_names: list[str],
|
||||
*,
|
||||
max_attempts: int,
|
||||
run_round: Callable[[list[str], int], dict[str, bool]],
|
||||
) -> dict[str, bool]:
|
||||
status = {name: False for name in platform_names}
|
||||
pending = list(platform_names)
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
if not pending:
|
||||
break
|
||||
round_result = run_round(pending, attempt)
|
||||
for name in pending:
|
||||
status[name] = bool(round_result.get(name))
|
||||
pending = [name for name in pending if not status[name]]
|
||||
return status
|
||||
|
||||
|
||||
def _valid_cookie(platform: Platform) -> bool:
|
||||
if not platform.cookie_file.is_file():
|
||||
return False
|
||||
if not platform.required_cookie:
|
||||
return platform.cookie_file.stat().st_size > 10
|
||||
try:
|
||||
import json
|
||||
values = json.loads(platform.cookie_file.read_text(encoding="utf-8"))
|
||||
return any(
|
||||
item.get("name") == platform.required_cookie and item.get("value")
|
||||
for item in values if isinstance(item, dict)
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _descendant_pids(pid: int) -> set[int]:
|
||||
try:
|
||||
return {child.pid for child in psutil.Process(pid).children(recursive=True)}
|
||||
except (psutil.Error, OSError):
|
||||
return set()
|
||||
|
||||
|
||||
def _terminate_process_tree(process: subprocess.Popen) -> None:
|
||||
"""Terminate the whole Windows subprocess tree, including Chrome/Playwright."""
|
||||
try:
|
||||
subprocess.run(
|
||||
["taskkill", "/PID", str(process.pid), "/T", "/F"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
|
||||
|
||||
def _window_for_process(pid: int) -> int | None:
|
||||
pids = _descendant_pids(pid) | {pid}
|
||||
handles: list[int] = []
|
||||
user32 = ctypes.windll.user32
|
||||
callback_type = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p)
|
||||
|
||||
def callback(hwnd, _lparam):
|
||||
process_id = ctypes.c_ulong()
|
||||
user32.GetWindowThreadProcessId(hwnd, ctypes.byref(process_id))
|
||||
if process_id.value in pids and user32.IsWindowVisible(hwnd):
|
||||
handles.append(int(hwnd))
|
||||
return True
|
||||
|
||||
user32.EnumWindows(callback_type(callback), 0)
|
||||
return handles[0] if handles else None
|
||||
|
||||
|
||||
def _platform_from_window_title(title: str) -> str | None:
|
||||
"""Identify a login window without relying on Chrome's process tree."""
|
||||
if "\u84b2\u516c\u82f1" in title:
|
||||
return "pgy"
|
||||
if "\u6296\u97f3\u6388\u6743" in title or "\u5de8\u91cf\u661f\u56fe" in title:
|
||||
return "xingtu"
|
||||
if "\u6296\u97f3" in title:
|
||||
return "douyin"
|
||||
if "bilibili" in title.lower() or "\u8d26\u53f7\u767b\u5f55" in title:
|
||||
return "bilibili"
|
||||
if "\u5c0f\u7ea2\u4e66" in title:
|
||||
return "xiaohongshu"
|
||||
return None
|
||||
|
||||
|
||||
def _login_windows() -> dict[str, int]:
|
||||
"""Map visible Chrome login windows to platforms by their page titles."""
|
||||
user32 = ctypes.windll.user32
|
||||
found: dict[str, int] = {}
|
||||
callback_type = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p)
|
||||
|
||||
def callback(hwnd, _lparam):
|
||||
if not user32.IsWindowVisible(hwnd):
|
||||
return True
|
||||
length = user32.GetWindowTextLengthW(hwnd)
|
||||
if length <= 0:
|
||||
return True
|
||||
buffer = ctypes.create_unicode_buffer(length + 1)
|
||||
user32.GetWindowTextW(hwnd, buffer, length + 1)
|
||||
platform = _platform_from_window_title(buffer.value)
|
||||
if platform:
|
||||
found[platform] = int(hwnd)
|
||||
return True
|
||||
|
||||
user32.EnumWindows(callback_type(callback), 0)
|
||||
return found
|
||||
|
||||
|
||||
def _capture_window(hwnd: int, output: Path) -> None:
|
||||
"""Maximize and render one window directly, even when it is not foreground."""
|
||||
from PIL import Image
|
||||
|
||||
user32 = ctypes.windll.user32
|
||||
gdi32 = ctypes.windll.gdi32
|
||||
user32.ShowWindow(hwnd, 3) # SW_MAXIMIZE
|
||||
time.sleep(1)
|
||||
rect = wintypes.RECT()
|
||||
user32.GetWindowRect(hwnd, ctypes.byref(rect))
|
||||
width, height = rect.right - rect.left, rect.bottom - rect.top
|
||||
if width <= 0 or height <= 0:
|
||||
raise RuntimeError(f"invalid window size: {width}x{height}")
|
||||
|
||||
window_dc = user32.GetWindowDC(hwnd)
|
||||
memory_dc = gdi32.CreateCompatibleDC(window_dc)
|
||||
bitmap = gdi32.CreateCompatibleBitmap(window_dc, width, height)
|
||||
previous = gdi32.SelectObject(memory_dc, bitmap)
|
||||
try:
|
||||
if not user32.PrintWindow(hwnd, memory_dc, 2): # PW_RENDERFULLCONTENT
|
||||
raise RuntimeError("PrintWindow failed")
|
||||
|
||||
class BitmapInfoHeader(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("biSize", wintypes.DWORD), ("biWidth", wintypes.LONG),
|
||||
("biHeight", wintypes.LONG), ("biPlanes", wintypes.WORD),
|
||||
("biBitCount", wintypes.WORD), ("biCompression", wintypes.DWORD),
|
||||
("biSizeImage", wintypes.DWORD),
|
||||
("biXPelsPerMeter", wintypes.LONG),
|
||||
("biYPelsPerMeter", wintypes.LONG),
|
||||
("biClrUsed", wintypes.DWORD),
|
||||
("biClrImportant", wintypes.DWORD),
|
||||
]
|
||||
|
||||
byte_count = width * height * 4
|
||||
pixels = ctypes.create_string_buffer(byte_count)
|
||||
header = BitmapInfoHeader(
|
||||
ctypes.sizeof(BitmapInfoHeader), width, -height, 1, 32, 0,
|
||||
byte_count, 0, 0, 0, 0,
|
||||
)
|
||||
rows = gdi32.GetDIBits(
|
||||
memory_dc, bitmap, 0, height, pixels, ctypes.byref(header), 0
|
||||
)
|
||||
if rows != height:
|
||||
raise RuntimeError(f"GetDIBits returned {rows}/{height} rows")
|
||||
image = Image.frombuffer(
|
||||
"RGB", (width, height), pixels, "raw", "BGRX", 0, 1
|
||||
).copy()
|
||||
finally:
|
||||
gdi32.SelectObject(memory_dc, previous)
|
||||
gdi32.DeleteObject(bitmap)
|
||||
gdi32.DeleteDC(memory_dc)
|
||||
user32.ReleaseDC(hwnd, window_dc)
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.save(output, "PNG")
|
||||
|
||||
|
||||
def _tile_windows(handles: list[int]) -> None:
|
||||
"""Place four login windows in a 2x2 grid so screenshots cannot overlap."""
|
||||
user32 = ctypes.windll.user32
|
||||
screen_w = user32.GetSystemMetrics(0)
|
||||
screen_h = user32.GetSystemMetrics(1)
|
||||
cell_w = max(640, screen_w // 2)
|
||||
cell_h = max(450, screen_h // 2)
|
||||
for index, hwnd in enumerate(handles):
|
||||
x = (index % 2) * cell_w
|
||||
y = (index // 2) * cell_h
|
||||
user32.ShowWindow(hwnd, 9) # SW_RESTORE
|
||||
user32.MoveWindow(hwnd, x, y, cell_w, cell_h, True)
|
||||
|
||||
|
||||
def _send_lark(recipient: str, text: str | None = None, image: Path | None = None) -> None:
|
||||
"""Send a notification through the locally installed Lark CLI."""
|
||||
lark_env = {
|
||||
key: value for key, value in os.environ.items()
|
||||
if not key.upper().startswith("HERMES_")
|
||||
}
|
||||
|
||||
def run_lark_cli(args: list[str], cwd: Path) -> None:
|
||||
cwd.mkdir(parents=True, exist_ok=True)
|
||||
command = shutil.which("lark-cli.cmd") or shutil.which("lark-cli") or "lark-cli.cmd"
|
||||
completed = subprocess.run(
|
||||
[command, *args],
|
||||
cwd=cwd,
|
||||
env=lark_env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
check=False,
|
||||
)
|
||||
if completed.returncode:
|
||||
detail = (completed.stderr or completed.stdout).strip()
|
||||
raise RuntimeError(f"lark-cli failed ({completed.returncode}): {detail}")
|
||||
|
||||
if text:
|
||||
# ensure_ascii keeps the Windows subprocess argv ASCII-only while the
|
||||
# JSON parser restores the original Chinese text for Feishu.
|
||||
content = json.dumps({"text": text}, ensure_ascii=True)
|
||||
run_lark_cli(
|
||||
["im", "+messages-send", "--user-id", recipient,
|
||||
"--as", "bot", "--content", content],
|
||||
cwd=ANALYZER_DIR,
|
||||
)
|
||||
if image:
|
||||
run_lark_cli(
|
||||
["im", "+messages-send", "--user-id", recipient,
|
||||
"--as", "bot", "--image", f"./{image.name}"], cwd=image.parent,
|
||||
)
|
||||
|
||||
|
||||
def _run_round_factory(args):
|
||||
python = sys.executable
|
||||
screenshot_root = PATHS.tmp_root / "qrcode/friday_relogin"
|
||||
|
||||
def run_round(names: list[str], attempt: int) -> dict[str, bool]:
|
||||
started: dict[str, subprocess.Popen] = {}
|
||||
stamp = dt.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
for name in names:
|
||||
platform = PLATFORMS[name]
|
||||
cmd = [python, *platform.command, "--login-timeout", str(args.round_timeout)]
|
||||
started[name] = subprocess.Popen(cmd, cwd=PROJECT_DIR)
|
||||
|
||||
time.sleep(args.screenshot_delay)
|
||||
title_windows = _login_windows()
|
||||
windows = {
|
||||
name: title_windows.get(name) or _window_for_process(process.pid)
|
||||
for name, process in started.items()
|
||||
}
|
||||
_tile_windows([hwnd for hwnd in windows.values() if hwnd])
|
||||
time.sleep(2)
|
||||
for name, process in started.items():
|
||||
platform = PLATFORMS[name]
|
||||
hwnd = windows[name]
|
||||
if not hwnd:
|
||||
print(f"[{name}] no visible browser window for QR screenshot", flush=True)
|
||||
continue
|
||||
output = screenshot_root / f"{stamp}_{name}_round{attempt}.png"
|
||||
try:
|
||||
_capture_window(hwnd, output)
|
||||
if not args.no_send:
|
||||
recipient = recipient_for_platform(name, args.recipient)
|
||||
_send_lark(
|
||||
recipient,
|
||||
text=(
|
||||
f"{platform.label} \u767b\u5f55\u4e8c\u7ef4\u7801"
|
||||
f"\uff08\u7b2c {attempt}/3 \u6b21\uff09\uff0c"
|
||||
"\u8bf7\u5728 3 \u5206\u949f\u5185\u626b\u7801\u3002"
|
||||
),
|
||||
)
|
||||
_send_lark(recipient, image=output)
|
||||
print(f"[{name}] QR screenshot: {output}", flush=True)
|
||||
except Exception as exc:
|
||||
print(f"[{name}] screenshot/send failed: {exc}", flush=True)
|
||||
|
||||
# Put the still-open login windows back into a compact layout after
|
||||
# taking full-size screenshots one at a time.
|
||||
_tile_windows([hwnd for hwnd in windows.values() if hwnd])
|
||||
|
||||
results: dict[str, bool] = {}
|
||||
for name, process in started.items():
|
||||
try:
|
||||
rc = process.wait(timeout=args.round_timeout + 90)
|
||||
except subprocess.TimeoutExpired:
|
||||
_terminate_process_tree(process)
|
||||
rc = 1
|
||||
results[name] = rc == 0 and _valid_cookie(PLATFORMS[name])
|
||||
print(f"[{name}] round={attempt} exit={rc} success={results[name]}", flush=True)
|
||||
return results
|
||||
|
||||
return run_round
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Friday five-platform parallel QR relogin")
|
||||
parser.add_argument(
|
||||
"--recipient", default=None,
|
||||
help="Override the configured recipients and send every platform to one open_id",
|
||||
)
|
||||
parser.add_argument("--max-attempts", type=int, default=3)
|
||||
parser.add_argument("--round-timeout", type=int, default=300)
|
||||
parser.add_argument("--screenshot-delay", type=int, default=60)
|
||||
parser.add_argument("--no-send", action="store_true", help="Do not send Lark messages")
|
||||
args = parser.parse_args()
|
||||
|
||||
status = run_with_retries(
|
||||
list(PLATFORMS), max_attempts=args.max_attempts,
|
||||
run_round=_run_round_factory(args),
|
||||
)
|
||||
failed = [name for name, ok in status.items() if not ok]
|
||||
if failed and not args.no_send:
|
||||
for recipient, names in group_platforms_by_recipient(failed, args.recipient).items():
|
||||
labels = "\u3001".join(PLATFORMS[name].label for name in names)
|
||||
_send_lark(
|
||||
recipient,
|
||||
text=(
|
||||
"\u4ee5\u4e0b\u5e73\u53f0\u8fde\u7eed 3 \u6b21\u672a\u626b\u7801\uff0c"
|
||||
"\u5df2\u6062\u590d\u5e76\u7ee7\u4f7f\u7528\u65e7 Cookie\uff1a"
|
||||
f"{labels}"
|
||||
),
|
||||
)
|
||||
print(f"Final status: {status}", flush=True)
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+910
@@ -0,0 +1,910 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
generate_creator_report.py — 达人合作数据筛选与报价分析报告
|
||||
=====================================================
|
||||
从 PostgreSQL 读取 cmt_cooperations + cmt_creators + cmt_styles，
|
||||
按指令要求生成完整的9章分析报告，输出为 Markdown 文件。
|
||||
|
||||
用法:
|
||||
python generate_creator_report.py
|
||||
python generate_creator_report.py --style 宙斯 # 只分析指定款式
|
||||
python generate_creator_report.py --output report.md
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import (
|
||||
PATHS,
|
||||
resolve_layer_output,
|
||||
)
|
||||
|
||||
import psycopg
|
||||
from dotenv import load_dotenv
|
||||
|
||||
BASE_DIR = PATHS.tools_root
|
||||
DATA_DIR = PATHS.exports_root
|
||||
load_dotenv(PATHS.state_root / "config/db.env")
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_db_config # noqa: E402
|
||||
|
||||
DB_CONFIG = get_db_config()
|
||||
|
||||
LARK = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd")
|
||||
if not os.path.exists(LARK):
|
||||
LARK = "lark-cli.cmd"
|
||||
|
||||
PLATFORM_CN = {
|
||||
"xiaohongshu": "小红书",
|
||||
"douyin": "抖音",
|
||||
"bilibili": "B站",
|
||||
}
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
safe = str(msg).encode("gbk", errors="replace").decode("gbk", errors="replace")
|
||||
print(safe, flush=True)
|
||||
|
||||
|
||||
def infer_level(follower_num: int | None, platform: str) -> str:
|
||||
if not follower_num:
|
||||
return "未知"
|
||||
if follower_num >= 500000:
|
||||
return "头部"
|
||||
elif follower_num >= 100000:
|
||||
return "腰部"
|
||||
elif follower_num >= 10000:
|
||||
return "KOC"
|
||||
else:
|
||||
return "素人"
|
||||
|
||||
|
||||
def score_creator(stats: dict) -> str:
|
||||
if not stats.get("has_data"):
|
||||
return "C"
|
||||
avg_eng_rate = stats.get("avg_engagement_rate", 0) or 0
|
||||
avg_cpe = stats.get("avg_cpe", 0) or 0
|
||||
avg_roi = stats.get("avg_roi", 0) or 0
|
||||
exposure = stats.get("total_exposure", 0) or 0
|
||||
|
||||
score = 0
|
||||
if avg_eng_rate >= 5: score += 3
|
||||
elif avg_eng_rate >= 3: score += 2
|
||||
elif avg_eng_rate >= 1: score += 1
|
||||
|
||||
if 0 < avg_cpe <= 2: score += 3
|
||||
elif 0 < avg_cpe <= 5: score += 2
|
||||
elif avg_cpe > 0: score += 1
|
||||
|
||||
if avg_roi >= 2: score += 3
|
||||
elif avg_roi >= 1: score += 2
|
||||
elif avg_roi > 0: score += 1
|
||||
|
||||
if exposure >= 500000: score += 2
|
||||
elif exposure >= 100000: score += 1
|
||||
|
||||
if score >= 8: return "S"
|
||||
elif score >= 5: return "A"
|
||||
elif score >= 3: return "B"
|
||||
else: return "C"
|
||||
|
||||
|
||||
def one_line_profile(stats: dict) -> str:
|
||||
level = stats.get("level", "未知")
|
||||
platform = stats.get("platform", "")
|
||||
coop_count = stats.get("coop_count", 0)
|
||||
total_exp = stats.get("total_exposure", 0) or 0
|
||||
account_type = stats.get("account_type", "")
|
||||
|
||||
if total_exp >= 500000:
|
||||
exp_desc = "爆款制造者"
|
||||
elif total_exp >= 100000:
|
||||
exp_desc = "稳定产出"
|
||||
else:
|
||||
exp_desc = "基础曝光"
|
||||
|
||||
type_desc = f"{account_type}类" if account_type else ""
|
||||
return f"{level}{type_desc}达人，{PLATFORM_CN.get(platform, platform)}{coop_count}次合作，{exp_desc}"
|
||||
|
||||
|
||||
def generate_report(style_filter: str | None = None) -> str:
|
||||
conn = psycopg.connect(**DB_CONFIG)
|
||||
cur = conn.cursor()
|
||||
|
||||
where = "WHERE c.name IS NOT NULL AND c.name != ''"
|
||||
if style_filter:
|
||||
where += f" AND s.name = '{style_filter}'"
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT
|
||||
c.name AS creator_name,
|
||||
c.platform,
|
||||
c.follower_count,
|
||||
c.follower_count_num,
|
||||
c.platform_id,
|
||||
c.account_type,
|
||||
c.creator_level,
|
||||
c.homepage_url,
|
||||
s.name AS style_name,
|
||||
co.feishu_record_id,
|
||||
co.cooperation_date,
|
||||
co.cooperation_cost,
|
||||
co.ad_spend,
|
||||
co.content_direction,
|
||||
co.note_title,
|
||||
co.exposure_count,
|
||||
co.engagement_count,
|
||||
co.engagement_count_num,
|
||||
co.data_performance,
|
||||
co.cpm,
|
||||
co.is_paid,
|
||||
co.publish_time,
|
||||
n.like_count AS note_like_count,
|
||||
n.collect_count AS note_collect_count,
|
||||
n.comment_count AS note_comment_count,
|
||||
n.favorite_count AS note_favorite_count
|
||||
FROM cmt_cooperations co
|
||||
JOIN cmt_creators c ON c.id = co.creator_id
|
||||
JOIN cmt_styles s ON s.id = co.style_id
|
||||
LEFT JOIN cmt_notes n ON n.feishu_record_id = co.feishu_record_id
|
||||
{where}
|
||||
ORDER BY s.name, c.name
|
||||
""")
|
||||
|
||||
cols = [desc[0] for desc in cur.description]
|
||||
rows = [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
if not rows:
|
||||
return "# 达人合作数据筛选与报价分析报告\n\n无数据。"
|
||||
|
||||
# ─── Build per-creator stats ───
|
||||
creator_stats = defaultdict(lambda: {
|
||||
"coop_count": 0,
|
||||
"platform": "",
|
||||
"follower_count_num": None,
|
||||
"follower_count": "",
|
||||
"account_type": "",
|
||||
"creator_level": "",
|
||||
"styles": [],
|
||||
"costs": [],
|
||||
"cost_dates": [], # (date, cost, style, content_direction) per coop
|
||||
"exposures": [],
|
||||
"engagements": [],
|
||||
"engagement_nums": [],
|
||||
"cpms": [],
|
||||
"data_perfs": [],
|
||||
"dates": [],
|
||||
"has_data": False,
|
||||
"note_engagements": [],
|
||||
"note_exposures": [],
|
||||
"content_directions": [],
|
||||
})
|
||||
|
||||
for r in rows:
|
||||
key = r["creator_name"]
|
||||
s = creator_stats[key]
|
||||
s["coop_count"] += 1
|
||||
s["platform"] = r["platform"] or s["platform"]
|
||||
if r["follower_count_num"] and not s["follower_count_num"]:
|
||||
s["follower_count_num"] = r["follower_count_num"]
|
||||
if r["follower_count"] and not s["follower_count"]:
|
||||
s["follower_count"] = r["follower_count"]
|
||||
if r["account_type"] and not s["account_type"]:
|
||||
s["account_type"] = r["account_type"]
|
||||
if r["creator_level"] and not s["creator_level"]:
|
||||
s["creator_level"] = r["creator_level"]
|
||||
if r["style_name"]:
|
||||
s["styles"].append(r["style_name"])
|
||||
if r["content_direction"]:
|
||||
s["content_directions"].append(r["content_direction"])
|
||||
|
||||
# Per-cooperation detail for pricing analysis
|
||||
cost = float(r["cooperation_cost"]) if r["cooperation_cost"] is not None else None
|
||||
s["cost_dates"].append({
|
||||
"date": str(r["cooperation_date"]) if r["cooperation_date"] else "",
|
||||
"cost": cost,
|
||||
"style": r["style_name"] or "",
|
||||
"content_direction": r["content_direction"] or "",
|
||||
"exposure": int(r["exposure_count"]) if r["exposure_count"] is not None else None,
|
||||
})
|
||||
|
||||
if cost is not None:
|
||||
s["costs"].append(cost)
|
||||
if r["exposure_count"] is not None:
|
||||
s["exposures"].append(int(r["exposure_count"]))
|
||||
if r["engagement_count_num"] is not None:
|
||||
s["engagement_nums"].append(int(r["engagement_count_num"]))
|
||||
if r["cpm"] is not None:
|
||||
s["cpms"].append(float(r["cpm"]))
|
||||
if r["data_performance"]:
|
||||
s["data_perfs"].append(r["data_performance"])
|
||||
if r["cooperation_date"]:
|
||||
s["dates"].append(str(r["cooperation_date"]))
|
||||
|
||||
if r["cooperation_cost"] is not None and r["exposure_count"] is not None:
|
||||
s["has_data"] = True
|
||||
|
||||
note_eng = (r.get("note_like_count") or 0) + (r.get("note_collect_count") or 0) + (r.get("note_comment_count") or 0) + (r.get("note_favorite_count") or 0)
|
||||
if note_eng > 0:
|
||||
s["note_engagements"].append(note_eng)
|
||||
note_exp = r["exposure_count"] if r["exposure_count"] else None
|
||||
if note_exp:
|
||||
s["note_exposures"].append(note_exp)
|
||||
|
||||
# Compute derived metrics
|
||||
for key, s in creator_stats.items():
|
||||
if not s["creator_level"] and s["follower_count_num"]:
|
||||
s["creator_level"] = infer_level(s["follower_count_num"], s["platform"])
|
||||
elif not s["creator_level"]:
|
||||
s["creator_level"] = "未知"
|
||||
s["level"] = s["creator_level"]
|
||||
|
||||
if s["note_engagements"] and s["note_exposures"] and len(s["note_engagements"]) == len(s["note_exposures"]):
|
||||
rates = [e / max(x, 1) * 100 for e, x in zip(s["note_engagements"], s["note_exposures"])]
|
||||
s["avg_engagement_rate"] = round(sum(rates) / len(rates), 2)
|
||||
elif s["note_engagements"] and s["exposures"]:
|
||||
total_eng = sum(s["note_engagements"])
|
||||
total_exp = sum(s["exposures"])
|
||||
s["avg_engagement_rate"] = round(total_eng / max(total_exp, 1) * 100, 2)
|
||||
else:
|
||||
s["avg_engagement_rate"] = None
|
||||
|
||||
if s["costs"] and s["note_engagements"]:
|
||||
total_cost = sum(s["costs"])
|
||||
total_eng = sum(s["note_engagements"])
|
||||
s["avg_cpe"] = round(total_cost / max(total_eng, 1), 2)
|
||||
elif s["costs"] and s["engagement_nums"]:
|
||||
total_cost = sum(s["costs"])
|
||||
total_eng = sum(s["engagement_nums"])
|
||||
s["avg_cpe"] = round(total_cost / max(total_eng, 1), 2)
|
||||
else:
|
||||
s["avg_cpe"] = None
|
||||
|
||||
s["total_exposure"] = sum(s["exposures"]) if s["exposures"] else 0
|
||||
s["avg_roi"] = None
|
||||
s["score"] = score_creator(s)
|
||||
|
||||
# ─── Build per-style stats ───
|
||||
style_stats = defaultdict(lambda: {"coop_count": 0, "creators": set(), "exposures": [], "costs": [], "top_creators": []})
|
||||
for r in rows:
|
||||
sn = r["style_name"]
|
||||
ss = style_stats[sn]
|
||||
ss["coop_count"] += 1
|
||||
ss["creators"].add(r["creator_name"])
|
||||
if r["exposure_count"] is not None:
|
||||
ss["exposures"].append((r["creator_name"], int(r["exposure_count"])))
|
||||
if r["cooperation_cost"] is not None:
|
||||
ss["costs"].append(float(r["cooperation_cost"]))
|
||||
|
||||
# ─── Level average cost (for premium/discount analysis) ───
|
||||
level_costs = defaultdict(list)
|
||||
for name, s in creator_stats.items():
|
||||
if s["costs"] and s["creator_level"] != "未知":
|
||||
level_costs[s["creator_level"]].append(sum(s["costs"]) / len(s["costs"]))
|
||||
level_avg_cost = {lv: round(sum(v)/len(v), 0) for lv, v in level_costs.items() if v}
|
||||
|
||||
# ─── Generate Report ───
|
||||
lines = []
|
||||
L = lines.append
|
||||
|
||||
L("# 达人合作数据筛选与报价分析报告（自动生成）")
|
||||
L(f"\n> 生成时间：{datetime.now().strftime('%Y-%m-%d %H:%M')}")
|
||||
if style_filter:
|
||||
L(f"> 筛选款式：{style_filter}")
|
||||
L("")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 1. 数据质量说明
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
L("## 1. 数据质量说明")
|
||||
L(f"- **总记录数**：{len(rows)}")
|
||||
L(f"- **涉及达人总数**：{len(creator_stats)}")
|
||||
L(f"- **涉及款式数**：{len(style_stats)}")
|
||||
|
||||
total = len(rows)
|
||||
has_cost = sum(1 for r in rows if r["cooperation_cost"] is not None)
|
||||
has_exposure = sum(1 for r in rows if r["exposure_count"] is not None)
|
||||
has_engagement = sum(1 for r in rows if (r.get("note_like_count") or r.get("note_collect_count") or r.get("note_comment_count") or r.get("note_favorite_count")))
|
||||
has_follower = sum(1 for r in rows if r["follower_count_num"] is not None)
|
||||
has_platform = sum(1 for r in rows if r["platform"])
|
||||
has_level = sum(1 for r in rows if r["creator_level"])
|
||||
has_content_dir = sum(1 for r in rows if r["content_direction"])
|
||||
|
||||
L(f"\n| 字段 | 填充数 | 填充率 |")
|
||||
L(f"|---|---|---|")
|
||||
L(f"| 平台 | {has_platform} | {has_platform/total*100:.1f}% |")
|
||||
L(f"| 粉丝数 | {has_follower} | {has_follower/total*100:.1f}% |")
|
||||
L(f"| 达人层级 | {has_level} | {has_level/total*100:.1f}% |")
|
||||
L(f"| 合作花费 | {has_cost} | {has_cost/total*100:.1f}% |")
|
||||
L(f"| 曝光量 | {has_exposure} | {has_exposure/total*100:.1f}% |")
|
||||
L(f"| 互动量 | {has_engagement} | {has_engagement/total*100:.1f}% |")
|
||||
L(f"| 内容方向 | {has_content_dir} | {has_content_dir/total*100:.1f}% |")
|
||||
L(f"| 进店UV | 0 | 0.0% |")
|
||||
L(f"| 成交金额 | 0 | 0.0% |")
|
||||
L(f"| ROI | 0 | 0.0% |")
|
||||
L(f"| 评论品牌提及率 | 0 | 0.0% |")
|
||||
L(f"| 差评或舆情风险 | 0 | 0.0% |")
|
||||
L("")
|
||||
L('> **说明**：电商转化数据（进店UV、成交金额、ROI）和评论分析数据（品牌提及率、舆情风险）暂未采集，对应分析章节将标注"数据不足"。达人层级根据粉丝数自动推断：>=50万=头部、>=10万=腰部、>=1万=KOC、<1万=素人。')
|
||||
L("")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 2. 整体概览
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
L("## 2. 整体概览")
|
||||
|
||||
plat_dist = defaultdict(int)
|
||||
for r in rows:
|
||||
p = r["platform"] or "未知"
|
||||
plat_dist[p] += 1
|
||||
L("\n### 各平台达人分布")
|
||||
L("| 平台 | 合作记录数 | 达人数 |")
|
||||
L("|---|---|---|")
|
||||
plat_creators = defaultdict(set)
|
||||
for r in rows:
|
||||
plat_creators[r["platform"] or "未知"].add(r["creator_name"])
|
||||
for p in sorted(plat_dist, key=plat_dist.get, reverse=True):
|
||||
cn = PLATFORM_CN.get(p, p)
|
||||
L(f"| {cn} | {plat_dist[p]} | {len(plat_creators[p])} |")
|
||||
L("")
|
||||
|
||||
level_dist = defaultdict(int)
|
||||
for s in creator_stats.values():
|
||||
level_dist[s["creator_level"]] += 1
|
||||
L("### 各层级分布")
|
||||
L("| 层级 | 达人数 |")
|
||||
L("|---|---|")
|
||||
for lv in ["头部", "腰部", "KOC", "素人", "未知"]:
|
||||
L(f"| {lv} | {level_dist.get(lv, 0)} |")
|
||||
L("")
|
||||
|
||||
all_costs = [float(r["cooperation_cost"]) for r in rows if r["cooperation_cost"] is not None]
|
||||
all_exposures = [int(r["exposure_count"]) for r in rows if r["exposure_count"] is not None]
|
||||
all_engagements_note = []
|
||||
for r in rows:
|
||||
ne = (r.get("note_like_count") or 0) + (r.get("note_collect_count") or 0) + (r.get("note_comment_count") or 0) + (r.get("note_favorite_count") or 0)
|
||||
if ne > 0:
|
||||
all_engagements_note.append(ne)
|
||||
|
||||
total_cost = sum(all_costs) if all_costs else 0
|
||||
total_exp = sum(all_exposures) if all_exposures else 0
|
||||
avg_cpe = total_cost / sum(all_engagements_note) if all_costs and all_engagements_note else None
|
||||
|
||||
L("### 核心指标汇总")
|
||||
L("| 指标 | 数值 |")
|
||||
L("|---|---|")
|
||||
L(f"| 总合作花费 | ¥{total_cost:,.0f} |")
|
||||
L(f"| 总曝光量 | {total_exp:,} |")
|
||||
L(f"| 总互动量 | {sum(all_engagements_note):,} |" if all_engagements_note else "| 总互动量 | 数据不足 |")
|
||||
L(f"| 平均CPE | ¥{avg_cpe:.2f} |" if avg_cpe else "| 平均CPE | 数据不足 |")
|
||||
L(f"| 平均ROI | 数据不足 |")
|
||||
L(f"| 总成交金额 | 数据不足 |")
|
||||
L("")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 3. 达人效能画像（按达人分组汇总）
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
L("## 3. 达人效能画像（按达人分组汇总）")
|
||||
L("")
|
||||
L("> 按 S/A/B/C 层级分表展示，每个子表不超过飞书 2000 格限制。")
|
||||
L("")
|
||||
|
||||
sorted_creators = sorted(creator_stats.items(),
|
||||
key=lambda x: ({"S":0,"A":1,"B":2,"C":3}.get(x[1]["score"], 4), -(x[1]["total_exposure"] or 0)))
|
||||
|
||||
for tier in ["S", "A", "B", "C"]:
|
||||
tier_rows = [(n, s) for n, s in sorted_creators if s["score"] == tier]
|
||||
if not tier_rows:
|
||||
continue
|
||||
|
||||
# Split large tiers to stay under Feishu 2000-cell limit
|
||||
# 10 cols per row → max 200 rows per table
|
||||
MAX_ROWS = 200
|
||||
chunks = [tier_rows[i:i+MAX_ROWS] for i in range(0, len(tier_rows), MAX_ROWS)]
|
||||
for ci, chunk in enumerate(chunks):
|
||||
chunk_label = f"（{len(tier_rows)}人）" if len(chunks) == 1 else f"（{len(tier_rows)}人，第{ci+1}/{len(chunks)}批）"
|
||||
L(f"### {tier}级达人{chunk_label}")
|
||||
L("")
|
||||
L("| 达人昵称 | 平台 | 粉丝数 | 层级 | 合作次数 | 合作单品 | 每次金额 | 平均CPE | 平均互动率 | 综合评分 |")
|
||||
L("|---|---|---|---|---|---|---|---|---|---|")
|
||||
|
||||
for name, s in chunk:
|
||||
plat_cn = PLATFORM_CN.get(s["platform"], s["platform"] or "未知")
|
||||
follower_str = f"{s['follower_count_num']:,}" if s["follower_count_num"] else s.get("follower_count", "未知")
|
||||
unique_styles = list(dict.fromkeys(s["styles"]))
|
||||
styles_str = ", ".join(unique_styles[:3])
|
||||
if len(unique_styles) > 3:
|
||||
styles_str += f"等{len(unique_styles)}款"
|
||||
cost_entries = [e for e in s["cost_dates"] if e["cost"] is not None]
|
||||
cost_entries.sort(key=lambda e: e["date"] or "")
|
||||
costs_str = "→".join(f"¥{e['cost']:.0f}" for e in cost_entries) if cost_entries else "-"
|
||||
avg_cpe_str = f"¥{s['avg_cpe']:.2f}" if s.get("avg_cpe") else "-"
|
||||
avg_er_str = f"{s['avg_engagement_rate']:.1f}%" if s.get("avg_engagement_rate") else "-"
|
||||
L(f"| {name} | {plat_cn} | {follower_str} | {s['creator_level']} | {s['coop_count']} | {styles_str} | {costs_str} | {avg_cpe_str} | {avg_er_str} | {s['score']} |")
|
||||
L("")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 4. 达人报价差异分析
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
L("## 4. 达人报价差异分析")
|
||||
L("")
|
||||
|
||||
# 4a. 个体报价波动分析
|
||||
L("### 个体报价波动分析")
|
||||
L("")
|
||||
L("> 针对同一达人不同次合作花费不同的情况，结合合作单品、内容形式、合作日期分析报价变化原因。")
|
||||
L("")
|
||||
|
||||
multi_cost_creators = []
|
||||
for name, s in creator_stats.items():
|
||||
cost_entries = [e for e in s["cost_dates"] if e["cost"] is not None]
|
||||
if len(cost_entries) >= 2:
|
||||
costs = [e["cost"] for e in cost_entries]
|
||||
if max(costs) != min(costs):
|
||||
multi_cost_creators.append((name, s, cost_entries))
|
||||
|
||||
if multi_cost_creators:
|
||||
L("| 达人昵称 | 层级 | 合作次数 | 报价区间 | 报价变化 | 波动归因分析 |")
|
||||
L("|---|---|---|---|---|---|")
|
||||
for name, s, entries in multi_cost_creators[:30]:
|
||||
entries.sort(key=lambda e: e["date"] or "")
|
||||
costs = [e["cost"] for e in entries]
|
||||
cost_range = f"¥{min(costs):.0f}~¥{max(costs):.0f}"
|
||||
cost_trend = "→".join(f"¥{e['cost']:.0f}" for e in entries)
|
||||
# Analyze reasons
|
||||
reasons = []
|
||||
# Check content_direction variation
|
||||
dirs = [e["content_direction"] for e in entries if e["content_direction"]]
|
||||
if len(set(dirs)) > 1:
|
||||
reasons.append(f"内容形式不同({', '.join(set(dirs))})")
|
||||
# Check style variation
|
||||
styles = [e["style"] for e in entries if e["style"]]
|
||||
if len(set(styles)) > 1:
|
||||
reasons.append(f"合作不同款式({len(set(styles))}款)")
|
||||
# Check date proximity to promotion periods
|
||||
dates = [e["date"] for e in entries if e["date"]]
|
||||
promo_months = ["06", "11", "12"] # 618, 双11, 双12
|
||||
if any(d[5:7] in promo_months for d in dates):
|
||||
reasons.append("含大促节点")
|
||||
if not reasons:
|
||||
reasons.append("报价自然波动")
|
||||
L(f"| {name} | {s['creator_level']} | {len(entries)} | {cost_range} | {cost_trend} | {'；'.join(reasons)} |")
|
||||
L("")
|
||||
else:
|
||||
L("数据不足，无法分析个体报价波动。")
|
||||
L("")
|
||||
|
||||
# 4b. 同层级溢价/折价分析
|
||||
L("### 同层级溢价/折价分析")
|
||||
L("")
|
||||
L("> 对比同层级、同平台达人的平均合作花费，识别溢价（性价比低）和折价（高性价比洼地）达人。")
|
||||
L("")
|
||||
|
||||
if level_avg_cost:
|
||||
L("| 达人昵称 | 层级 | 平台 | 个人均价 | 层级均价 | 溢折率 | 判定 |")
|
||||
L("|---|---|---|---|---|---|---|")
|
||||
premium_list = []
|
||||
for name, s in creator_stats.items():
|
||||
if not s["costs"] or s["creator_level"] == "未知":
|
||||
continue
|
||||
personal_avg = sum(s["costs"]) / len(s["costs"])
|
||||
lv_avg = level_avg_cost.get(s["creator_level"])
|
||||
if not lv_avg or lv_avg == 0:
|
||||
continue
|
||||
ratio = (personal_avg - lv_avg) / lv_avg * 100
|
||||
if ratio > 30:
|
||||
tag = "溢价"
|
||||
elif ratio < -30:
|
||||
tag = "折价洼地"
|
||||
else:
|
||||
tag = "合理"
|
||||
premium_list.append((name, s, personal_avg, lv_avg, ratio, tag))
|
||||
|
||||
premium_list.sort(key=lambda x: -x[4])
|
||||
for name, s, p_avg, l_avg, ratio, tag in premium_list[:20]:
|
||||
plat_cn = PLATFORM_CN.get(s["platform"], s["platform"])
|
||||
L(f"| {name} | {s['creator_level']} | {plat_cn} | ¥{p_avg:.0f} | ¥{l_avg:.0f} | {ratio:+.0f}% | {tag} |")
|
||||
# Also show top discount
|
||||
discount_list = [x for x in premium_list if x[5] == "折价洼地"]
|
||||
for name, s, p_avg, l_avg, ratio, tag in discount_list[:10]:
|
||||
plat_cn = PLATFORM_CN.get(s["platform"], s["platform"])
|
||||
L(f"| {name} | {s['creator_level']} | {plat_cn} | ¥{p_avg:.0f} | ¥{l_avg:.0f} | {ratio:+.0f}% | {tag} |")
|
||||
L("")
|
||||
else:
|
||||
L("数据不足，无法进行同层级溢价/折价分析。")
|
||||
L("")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 5. 达人分层与分类
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
L("## 5. 达人分层与分类")
|
||||
|
||||
tiers = {"S": [], "A": [], "B": [], "C": []}
|
||||
for name, s in creator_stats.items():
|
||||
tiers[s["score"]].append(name)
|
||||
|
||||
L("\n### 分层矩阵")
|
||||
for tier in ["S", "A", "B", "C"]:
|
||||
names = tiers[tier]
|
||||
if names:
|
||||
names_str = ', '.join(names[:20]) + ('...' if len(names) > 20 else '')
|
||||
L(f"**{tier}级**（{len(names)}人）：{names_str}")
|
||||
else:
|
||||
L(f"**{tier}级**（0人）：无")
|
||||
|
||||
L("\n### 角色分类")
|
||||
voice_type = sorted(creator_stats.items(), key=lambda x: -(x[1]["total_exposure"] or 0))[:10]
|
||||
voice_names = [n for n, s in voice_type if s["total_exposure"]]
|
||||
L(f"**声量型**（高曝光）：{', '.join(voice_names)}")
|
||||
|
||||
harvest_type = sorted(creator_stats.items(), key=lambda x: x[1].get("avg_cpe", 9999) or 9999)[:10]
|
||||
L(f"**收割型**（低成本高效）：{', '.join(n for n, s in harvest_type if s.get('avg_cpe') and s['avg_cpe'] > 0)}")
|
||||
|
||||
volume_type = [(n, s) for n, s in creator_stats.items() if s["costs"]]
|
||||
volume_type.sort(key=lambda x: sum(x[1]["costs"]) / len(x[1]["costs"]))
|
||||
L(f"**铺量型**（低成本）：{', '.join(n for n, _ in volume_type[:10])}")
|
||||
|
||||
# 共创型: high engagement rate + content variety
|
||||
creative_type = [(n, s) for n, s in creator_stats.items() if s.get("avg_engagement_rate") and s["avg_engagement_rate"] > 0 and len(set(s["content_directions"])) >= 2]
|
||||
creative_type.sort(key=lambda x: -x[1].get("avg_engagement_rate", 0))
|
||||
L(f"**共创型**（内容创意强，多方向+高互动）：{', '.join(n for n, _ in creative_type[:10])}")
|
||||
|
||||
L("")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 6. 合作单品交叉分析
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
L("## 6. 合作单品交叉分析")
|
||||
L("")
|
||||
L("| 款式 | 合作达人数 | 平均曝光 | 平均花费 | Top3达人 | 单品最佳搭档 |")
|
||||
L("|---|---|---|---|---|---|")
|
||||
|
||||
for sn in sorted(style_stats.keys()):
|
||||
ss = style_stats[sn]
|
||||
avg_exp = sum(e for _, e in ss["exposures"]) / len(ss["exposures"]) if ss["exposures"] else 0
|
||||
avg_cost = sum(ss["costs"]) / len(ss["costs"]) if ss["costs"] else 0
|
||||
top3 = sorted(ss["exposures"], key=lambda x: -x[1])[:3]
|
||||
top3_str = ", ".join(f"{n}({e:,})" for n, e in top3)
|
||||
# Best partner: highest exposure creator
|
||||
best = top3[0][0] if top3 else "-"
|
||||
L(f"| {sn} | {len(ss['creators'])} | {avg_exp:,.0f} | ¥{avg_cost:,.0f} | {top3_str} | {best} |")
|
||||
L("")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 7. 趋势与风险评估
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
L("## 7. 趋势与风险评估")
|
||||
L("")
|
||||
|
||||
multi_coop = {n: s for n, s in creator_stats.items() if s["coop_count"] >= 2 and len(s["exposures"]) >= 2}
|
||||
|
||||
rising = []
|
||||
declining = []
|
||||
for n, s in multi_coop.items():
|
||||
if len(s["exposures"]) >= 2:
|
||||
first_half = s["exposures"][:len(s["exposures"])//2]
|
||||
second_half = s["exposures"][len(s["exposures"])//2:]
|
||||
avg_first = sum(first_half) / len(first_half)
|
||||
avg_second = sum(second_half) / len(second_half)
|
||||
if avg_second > avg_first * 1.2:
|
||||
rising.append(n)
|
||||
elif avg_second < avg_first * 0.8:
|
||||
declining.append(n)
|
||||
|
||||
L(f"### 表现持续上升（{len(rising)}人）")
|
||||
if rising:
|
||||
L("建议加大投入。")
|
||||
for n in rising[:20]:
|
||||
s = creator_stats[n]
|
||||
L(f"- **{n}**：曝光量从前期上升至后期，{s['coop_count']}次合作数据持续改善")
|
||||
else:
|
||||
L("数据不足以判断趋势")
|
||||
L("")
|
||||
|
||||
L(f"### 表现下滑（{len(declining)}人）")
|
||||
if declining:
|
||||
L("建议减少合作或调整Brief。")
|
||||
for n in declining[:20]:
|
||||
s = creator_stats[n]
|
||||
L(f"- **{n}**：曝光量从前期下降至后期，建议关注内容质量或平台算法变化")
|
||||
else:
|
||||
L("数据不足以判断趋势")
|
||||
L("")
|
||||
|
||||
L("### 风险达人")
|
||||
risk_tags = {"曝光限流": [], "效果一般": [], "挂车不出单": [], "内容卡审": []}
|
||||
for n, s in creator_stats.items():
|
||||
for dp in s["data_perfs"]:
|
||||
for tag in dp.split(","):
|
||||
tag = tag.strip()
|
||||
if tag in risk_tags:
|
||||
risk_tags[tag].append(n)
|
||||
has_risk = False
|
||||
for tag, names in risk_tags.items():
|
||||
if names:
|
||||
has_risk = True
|
||||
L(f"- **{tag}**：{', '.join(set(names))}")
|
||||
if not has_risk:
|
||||
L("- 暂无风险标记")
|
||||
L("")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 8. 筛选推荐（根据目标场景）
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
L("## 8. 筛选推荐（根据目标场景）")
|
||||
L("")
|
||||
|
||||
L("### 场景一：新品上市（重点曝光+种草）")
|
||||
s_a_creators = [(n, s) for n, s in creator_stats.items() if s["score"] in ("S", "A") and s["total_exposure"]]
|
||||
s_a_creators.sort(key=lambda x: -x[1]["total_exposure"])
|
||||
if s_a_creators:
|
||||
L("| 推荐达人 | 平台 | 粉丝数 | 累计曝光 | 综合评分 | 推荐理由 | 预估CPE |")
|
||||
L("|---|---|---|---|---|---|---|")
|
||||
for n, s in s_a_creators[:5]:
|
||||
plat_cn = PLATFORM_CN.get(s["platform"], s["platform"])
|
||||
follower_str = f"{s['follower_count_num']:,}" if s["follower_count_num"] else "未知"
|
||||
reason = f"曝光量{s['total_exposure']:,}，{s['coop_count']}次合作经验"
|
||||
est_cpe = f"¥{s['avg_cpe']:.2f}" if s.get("avg_cpe") else "待估"
|
||||
L(f"| {n} | {plat_cn} | {follower_str} | {s['total_exposure']:,} | {s['score']} | {reason} | {est_cpe} |")
|
||||
else:
|
||||
L("数据不足，无法推荐")
|
||||
L("")
|
||||
|
||||
L("### 场景二：大促收割（重点转化）")
|
||||
L("> 进店UV、成交金额、ROI 数据暂未采集，以下基于低成本高效（低CPE+高互动率）推荐：")
|
||||
low_cpe = [(n, s) for n, s in creator_stats.items() if s.get("avg_cpe") and s["avg_cpe"] > 0]
|
||||
low_cpe.sort(key=lambda x: x[1]["avg_cpe"])
|
||||
if low_cpe:
|
||||
L("| 推荐达人 | 平台 | 平均CPE | 平均互动率 | 综合评分 | 推荐理由 |")
|
||||
L("|---|---|---|---|---|---|")
|
||||
for n, s in low_cpe[:5]:
|
||||
plat_cn = PLATFORM_CN.get(s["platform"], s["platform"])
|
||||
er = f"{s['avg_engagement_rate']:.1f}%" if s.get("avg_engagement_rate") else "-"
|
||||
L(f"| {n} | {plat_cn} | ¥{s['avg_cpe']:.2f} | {er} | {s['score']} | CPE低，互动率高 |")
|
||||
else:
|
||||
L("数据不足，无法推荐")
|
||||
L("")
|
||||
|
||||
L("### 场景三：日常口碑维护")
|
||||
koc_creators = [(n, s) for n, s in creator_stats.items() if s["creator_level"] in ("KOC", "素人") and s["total_exposure"]]
|
||||
koc_creators.sort(key=lambda x: -(x[1].get("avg_engagement_rate") or 0))
|
||||
if koc_creators:
|
||||
L("| 推荐达人 | 平台 | 粉丝数 | 平均互动率 | 合作次数 | 推荐理由 |")
|
||||
L("|---|---|---|---|---|---|")
|
||||
for n, s in koc_creators[:5]:
|
||||
plat_cn = PLATFORM_CN.get(s["platform"], s["platform"])
|
||||
follower_str = f"{s['follower_count_num']:,}" if s["follower_count_num"] else "未知"
|
||||
er = f"{s['avg_engagement_rate']:.1f}%" if s.get("avg_engagement_rate") else "-"
|
||||
L(f"| {n} | {plat_cn} | {follower_str} | {er} | {s['coop_count']} | 低成本高互动KOC |")
|
||||
else:
|
||||
L("数据不足，无法推荐")
|
||||
L("")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 9. 优化建议与下一步行动
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
L("## 9. 优化建议与下一步行动")
|
||||
L("")
|
||||
|
||||
for tier, action in [
|
||||
("S", "建议签长期协议或独家合作，锁定优质达人资源"),
|
||||
("A", "增加合作频次或尝试新款式，挖掘更大潜力"),
|
||||
("B", "优化Brief或换内容形式测试，寻找突破口"),
|
||||
("C", "暂停合作或淘汰，将预算转移至更高层级达人"),
|
||||
]:
|
||||
names = tiers[tier]
|
||||
L(f"- **{tier}级达人**（{len(names)}人）：{action}")
|
||||
L("")
|
||||
|
||||
# 报价优化建议
|
||||
L("### 报价优化建议")
|
||||
L("")
|
||||
if multi_cost_creators:
|
||||
L("基于第4章报价差异分析，给出以下议价策略：")
|
||||
L("")
|
||||
# Find creators with rising costs
|
||||
for name, s, entries in multi_cost_creators[:10]:
|
||||
entries.sort(key=lambda e: e["date"] or "")
|
||||
costs = [e["cost"] for e in entries]
|
||||
if costs[-1] > costs[0]:
|
||||
L(f"- **{name}**：报价呈上升趋势（¥{costs[0]:.0f}→¥{costs[-1]:.0f}），建议锁定当前报价或签长期协议避免继续涨价")
|
||||
elif costs[-1] < costs[0]:
|
||||
L(f"- **{name}**：报价有下降空间（¥{costs[0]:.0f}→¥{costs[-1]:.0f}），可尝试进一步议价")
|
||||
if level_avg_cost:
|
||||
L("")
|
||||
L("**层级预算分配建议**：")
|
||||
for lv in ["头部", "腰部", "KOC", "素人"]:
|
||||
avg = level_avg_cost.get(lv)
|
||||
if avg:
|
||||
L(f"- {lv}达人均价¥{avg:.0f}，建议单次合作预算控制在¥{avg*0.8:.0f}~¥{avg*1.2:.0f}")
|
||||
L("")
|
||||
|
||||
L("---")
|
||||
L(f"*报告由 generate_creator_report.py 自动生成 · {datetime.now().strftime('%Y-%m-%d %H:%M')}*")
|
||||
|
||||
return "\n".join(lines), rows, creator_stats, style_stats
|
||||
|
||||
|
||||
def _compute_last_month():
|
||||
today = datetime.now().date()
|
||||
first_of_this_month = today.replace(day=1)
|
||||
month_end = first_of_this_month - timedelta(days=1)
|
||||
month_start = month_end.replace(day=1)
|
||||
return month_start, month_end
|
||||
|
||||
|
||||
def create_feishu_doc(report_md: str, month_start: str, month_end: str) -> str | None:
|
||||
"""创建飞书文档,返回 doc URL;失败返回 None"""
|
||||
doc_title = f"达人合作数据筛选与报价分析报告（{month_start}~{month_end[5:]}）"
|
||||
md_file = PATHS.tmp_root / "creator_report_doc.md"
|
||||
md_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_file.write_text(report_md, encoding="utf-8")
|
||||
|
||||
cmd = [LARK, "docs", "+create",
|
||||
"--as", "user", "--format", "json",
|
||||
"--title", doc_title,
|
||||
"--doc-format", "markdown",
|
||||
"--content", f"@{md_file.name}"]
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace")
|
||||
text = proc.stdout or proc.stderr
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
_log(f" [ERR] 创建飞书文档失败: {text[:300]}")
|
||||
return None
|
||||
resp = json.loads(text[start:])
|
||||
if not resp.get("ok"):
|
||||
_log(f" [ERR] 创建飞书文档失败: {resp.get('error')}")
|
||||
return None
|
||||
url = resp.get("data", {}).get("document", {}).get("url", "")
|
||||
if resp.get("warnings"):
|
||||
_log(f" [WARN] {resp['warnings']}")
|
||||
return url
|
||||
|
||||
|
||||
def save_to_db(report_md: str, rows: list, creator_stats: dict,
|
||||
style_stats: dict, month_start: str, month_end: str,
|
||||
doc_url: str | None, status: str) -> bool:
|
||||
"""把月度达人报告结果写入 cmt_creator_report"""
|
||||
time_label = f"{month_start}~{month_end[5:]}"
|
||||
|
||||
total_cooperations = len(rows)
|
||||
total_creators = len(creator_stats)
|
||||
total_styles = len(style_stats)
|
||||
|
||||
all_costs = [float(r["cooperation_cost"]) for r in rows if r["cooperation_cost"] is not None]
|
||||
all_exposures = [int(r["exposure_count"]) for r in rows if r["exposure_count"] is not None]
|
||||
all_engagements = []
|
||||
for r in rows:
|
||||
ne = (r.get("note_like_count") or 0) + (r.get("note_collect_count") or 0) + (r.get("note_comment_count") or 0) + (r.get("note_favorite_count") or 0)
|
||||
if ne > 0:
|
||||
all_engagements.append(ne)
|
||||
|
||||
total_cost = sum(all_costs) if all_costs else None
|
||||
total_exposure = sum(all_exposures) if all_exposures else None
|
||||
total_engagement = sum(all_engagements) if all_engagements else None
|
||||
avg_cpe = (total_cost / total_engagement) if total_cost and total_engagement else None
|
||||
|
||||
# Level distribution
|
||||
level_dist = defaultdict(int)
|
||||
for s in creator_stats.values():
|
||||
level_dist[s["creator_level"]] += 1
|
||||
# Platform distribution
|
||||
plat_dist = defaultdict(int)
|
||||
for r in rows:
|
||||
p = r["platform"] or "未知"
|
||||
plat_dist[p] += 1
|
||||
plat_dist_creators = defaultdict(set)
|
||||
for r in rows:
|
||||
plat_dist_creators[r["platform"] or "未知"].add(r["creator_name"])
|
||||
platform_dist = {p: len(plat_dist_creators[p]) for p in plat_dist_creators}
|
||||
# Score distribution
|
||||
score_dist = defaultdict(int)
|
||||
for s in creator_stats.values():
|
||||
score_dist[s["score"]] += 1
|
||||
|
||||
doc_title = f"达人合作数据筛选与报价分析报告（{time_label}）"
|
||||
|
||||
try:
|
||||
conn = psycopg.connect(**DB_CONFIG)
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
INSERT INTO cmt_creator_report
|
||||
(month_start, month_end, time_label,
|
||||
total_cooperations, total_creators, total_styles,
|
||||
total_cost, total_exposure, total_engagement, avg_cpe,
|
||||
level_distribution, platform_distribution, score_distribution,
|
||||
doc_url, doc_title, report_chars, status)
|
||||
VALUES
|
||||
(%s, %s, %s,
|
||||
%s, %s, %s,
|
||||
%s, %s, %s, %s,
|
||||
%s, %s, %s,
|
||||
%s, %s, %s, %s)
|
||||
ON CONFLICT (month_start) DO UPDATE SET
|
||||
total_cooperations = EXCLUDED.total_cooperations,
|
||||
total_creators = EXCLUDED.total_creators,
|
||||
total_styles = EXCLUDED.total_styles,
|
||||
total_cost = EXCLUDED.total_cost,
|
||||
total_exposure = EXCLUDED.total_exposure,
|
||||
total_engagement = EXCLUDED.total_engagement,
|
||||
avg_cpe = EXCLUDED.avg_cpe,
|
||||
level_distribution = EXCLUDED.level_distribution,
|
||||
platform_distribution = EXCLUDED.platform_distribution,
|
||||
score_distribution = EXCLUDED.score_distribution,
|
||||
doc_url = EXCLUDED.doc_url,
|
||||
doc_title = EXCLUDED.doc_title,
|
||||
report_chars = EXCLUDED.report_chars,
|
||||
status = EXCLUDED.status,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""", (month_start, month_end, time_label,
|
||||
total_cooperations, total_creators, total_styles,
|
||||
total_cost, total_exposure, total_engagement, avg_cpe,
|
||||
json.dumps(dict(level_dist), ensure_ascii=False),
|
||||
json.dumps(platform_dist, ensure_ascii=False),
|
||||
json.dumps(dict(score_dist), ensure_ascii=False),
|
||||
doc_url, doc_title, len(report_md), status))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
_log(f" 已入库 cmt_creator_report (month={month_start})")
|
||||
return True
|
||||
except Exception as exc:
|
||||
_log(f" [DB ERR] 入库失败: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="生成达人合作数据筛选与报价分析报告")
|
||||
parser.add_argument("--style", type=str, help="只分析指定款式 (如 '宙斯')")
|
||||
parser.add_argument("--output", type=str, default="", help="输出文件路径 (默认 data/reports/ 下自动命名)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="不写飞书/PG")
|
||||
args = parser.parse_args()
|
||||
|
||||
month_start, month_end = _compute_last_month()
|
||||
month_start_str = month_start.strftime("%Y-%m-%d")
|
||||
month_end_str = month_end.strftime("%Y-%m-%d")
|
||||
|
||||
_log(f"达人月度分析报告: {month_start_str}~{month_end_str}")
|
||||
|
||||
# generate_report returns (report_text, rows, creator_stats, style_stats)
|
||||
report, rows, creator_stats, style_stats = generate_report(style_filter=args.style)
|
||||
|
||||
if args.output:
|
||||
out_path = resolve_layer_output(
|
||||
args.output, layer_root=PATHS.exports_root, field="--output"
|
||||
)
|
||||
else:
|
||||
out_dir = DATA_DIR / "reports"
|
||||
out_dir.mkdir(exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
style_tag = f"_{args.style}" if args.style else "_all"
|
||||
out_path = out_dir / f"creator_report{style_tag}_{ts}.md"
|
||||
|
||||
out_path.write_text(report, encoding="utf-8")
|
||||
_log(f"报告已生成: {out_path} ({len(report)} 字符)")
|
||||
|
||||
if args.dry_run:
|
||||
_log("[DRY-RUN] 跳过飞书文档创建和入库")
|
||||
return
|
||||
|
||||
# 创建飞书文档
|
||||
doc_url = create_feishu_doc(report, month_start_str, month_end_str)
|
||||
if doc_url:
|
||||
_log(f"飞书文档: {doc_url}")
|
||||
else:
|
||||
_log("飞书文档创建失败")
|
||||
|
||||
# 入库
|
||||
status = "ok" if doc_url else "doc_failed"
|
||||
save_to_db(report, rows, creator_stats, style_stats,
|
||||
month_start_str, month_end_str, doc_url, status)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Initialize gyxx_super_data tables from schema_gyxx_super_data.sql."""
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Path setup
|
||||
_TOOLS_DIR = Path(__file__).resolve().parent
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import db
|
||||
|
||||
|
||||
def main() -> int:
|
||||
sql_path = _TOOLS_DIR / "schema_gyxx_super_data.sql"
|
||||
sql = sql_path.read_text(encoding="utf-8")
|
||||
|
||||
# Remove psql meta-commands
|
||||
sql = re.sub(r"^\\c .*?$", "", sql, flags=re.MULTILINE)
|
||||
sql = re.sub(r"^\\.*$", "", sql, flags=re.MULTILINE)
|
||||
|
||||
with db.get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql)
|
||||
conn.commit()
|
||||
print("Schema initialized successfully.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,7 @@
|
||||
$moduleRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path
|
||||
$dataRoot = if ($env:GYXX_DATA_ROOT) { $env:GYXX_DATA_ROOT } else { Join-Path $moduleRoot 'var' }
|
||||
$profileRoot = Join-Path $dataRoot 'state\content_marketing\browser-profiles'
|
||||
Get-WmiObject Win32_Process | Where-Object { $_.Name -eq 'chrome.exe' -and $_.CommandLine -like "*$profileRoot*" } | ForEach-Object { Write-Host "Killing PID $($_.ProcessId)"; Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }
|
||||
Start-Sleep -Seconds 3
|
||||
$procs = Get-WmiObject Win32_Process | Where-Object { $_.Name -eq 'chrome.exe' -and $_.CommandLine -like "*$profileRoot*" }
|
||||
if ($procs) { Write-Host 'Remaining project chrome processes:'; $procs | Select-Object ProcessId,CommandLine } else { Write-Host 'No project chrome processes remaining' }
|
||||
@@ -0,0 +1,4 @@
|
||||
$moduleRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path
|
||||
$dataRoot = if ($env:GYXX_DATA_ROOT) { $env:GYXX_DATA_ROOT } else { Join-Path $moduleRoot 'var' }
|
||||
$profileRoot = Join-Path $dataRoot 'state\content_marketing\browser-profiles'
|
||||
Get-WmiObject Win32_Process | Where-Object { $_.Name -eq 'chrome.exe' -and $_.CommandLine -like "*$profileRoot*" } | Select-Object ProcessId, ParentProcessId, @{Name='CmdStart';Expression={$_.CommandLine.Substring(0, [Math]::Min(200, $_.CommandLine.Length))}}
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
-- Migration 001: 扩展 cmt_creators 达人属性 + 新建 cmt_cooperations 合作记录表
|
||||
-- 日期: 2026-07-10
|
||||
-- 说明: 从飞书款式合作表同步达人信息和合作数据到数据库
|
||||
|
||||
-- ============================================================
|
||||
-- 1. 扩展 cmt_creators：加达人属性字段
|
||||
-- ============================================================
|
||||
|
||||
-- 加平台字段
|
||||
ALTER TABLE cmt_creators ADD COLUMN IF NOT EXISTS platform VARCHAR(32) DEFAULT '';
|
||||
COMMENT ON COLUMN cmt_creators.platform IS '主平台: xiaohongshu/douyin/bilibili';
|
||||
|
||||
-- 加平台达人ID
|
||||
ALTER TABLE cmt_creators ADD COLUMN IF NOT EXISTS platform_id VARCHAR(128) DEFAULT '';
|
||||
COMMENT ON COLUMN cmt_creators.platform_id IS '平台达人ID (如B站UID、抖音sec_uid等)';
|
||||
|
||||
-- 加粉丝数 (保留原始文本，因为飞书数据是"3.7万"这种格式)
|
||||
ALTER TABLE cmt_creators ADD COLUMN IF NOT EXISTS follower_count VARCHAR(64) DEFAULT '';
|
||||
COMMENT ON COLUMN cmt_creators.follower_count IS '达人粉丝数(原始文本如3.7万/12.4w)';
|
||||
|
||||
-- 加粉丝数(数字解析后)
|
||||
ALTER TABLE cmt_creators ADD COLUMN IF NOT EXISTS follower_count_num BIGINT DEFAULT NULL;
|
||||
COMMENT ON COLUMN cmt_creators.follower_count_num IS '达人粉丝数(数字解析后，3.7万=37000)';
|
||||
|
||||
-- 加达人主页链接
|
||||
ALTER TABLE cmt_creators ADD COLUMN IF NOT EXISTS homepage_url TEXT DEFAULT '';
|
||||
COMMENT ON COLUMN cmt_creators.homepage_url IS '达人主页链接';
|
||||
|
||||
-- 加微信号
|
||||
ALTER TABLE cmt_creators ADD COLUMN IF NOT EXISTS wechat VARCHAR(128) DEFAULT '';
|
||||
COMMENT ON COLUMN cmt_creators.wechat IS '微信号';
|
||||
|
||||
-- 加达人类型 (对应飞书"投放账号类型")
|
||||
ALTER TABLE cmt_creators ADD COLUMN IF NOT EXISTS account_type VARCHAR(64) DEFAULT '';
|
||||
COMMENT ON COLUMN cmt_creators.account_type IS '达人类型: 数码/vlog/好物/穿搭/摄影等';
|
||||
|
||||
-- 加达人层级
|
||||
ALTER TABLE cmt_creators ADD COLUMN IF NOT EXISTS creator_level VARCHAR(32) DEFAULT '';
|
||||
COMMENT ON COLUMN cmt_creators.creator_level IS '达人层级: 头部/腰部/KOC/素人';
|
||||
|
||||
-- 修改唯一约束: name 单列唯一 → (name, platform) 联合唯一
|
||||
-- 因为同一达人可能跨平台存在(如BeiBei贝贝同时在小红书和抖音)
|
||||
-- 先删除旧约束
|
||||
ALTER TABLE cmt_creators DROP CONSTRAINT IF EXISTS cmt_creators_name_key;
|
||||
-- 添加新约束
|
||||
ALTER TABLE cmt_creators ADD CONSTRAINT cmt_creators_name_platform_key UNIQUE (name, platform);
|
||||
|
||||
-- 添加索引
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_creators_platform ON cmt_creators (platform);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_creators_platform_id ON cmt_creators (platform_id);
|
||||
|
||||
-- ============================================================
|
||||
-- 2. 新建 cmt_cooperations：合作记录表
|
||||
-- ============================================================
|
||||
-- 每条记录 = 一个达人在一个款式下的一次合作
|
||||
-- 飞书款式合作表的每一行 → 此表一条记录
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cmt_cooperations (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
style_id BIGINT NOT NULL REFERENCES cmt_styles(id) ON DELETE CASCADE,
|
||||
creator_id BIGINT NOT NULL REFERENCES cmt_creators(id) ON DELETE CASCADE,
|
||||
feishu_record_id VARCHAR(64) DEFAULT '', -- 飞书行 record_id, 用于回写
|
||||
-- 合作基本信息
|
||||
cooperation_date DATE DEFAULT NULL, -- 合作日期 (飞书"制单日期")
|
||||
cooperation_cost NUMERIC(12,2) DEFAULT NULL, -- 合作花费 (飞书"合作费用")
|
||||
ad_spend NUMERIC(12,2) DEFAULT NULL, -- 投流金额 (飞书"投流金额")
|
||||
-- 内容信息
|
||||
content_direction VARCHAR(255) DEFAULT '', -- 内容方向 (飞书"发布笔记内容方向")
|
||||
content_format VARCHAR(64) DEFAULT '', -- 内容形式: 单品种草/测评合集/场景植入/开箱Vlog/穿搭变装等
|
||||
note_title VARCHAR(512) DEFAULT '', -- 发布笔记标题
|
||||
note_url TEXT DEFAULT '', -- 发布笔记链接
|
||||
publish_time TIMESTAMPTZ DEFAULT NULL, -- 发布时间
|
||||
-- 数据表现
|
||||
exposure_count BIGINT DEFAULT NULL, -- 曝光量 (飞书"曝光量(自动抓取)")
|
||||
engagement_count VARCHAR(64) DEFAULT '', -- 互动赞藏数 (原始文本如"68.5万")
|
||||
engagement_count_num BIGINT DEFAULT NULL, -- 互动赞藏数(数字解析后)
|
||||
data_performance VARCHAR(255) DEFAULT '', -- 数据表现 (逗号分隔: 曝光限流,数据良好)
|
||||
cpm NUMERIC(10,4) DEFAULT NULL, -- CPM (飞书公式字段)
|
||||
-- 物流/财务
|
||||
tracking_number VARCHAR(128) DEFAULT '', -- 快递单号
|
||||
is_paid BOOLEAN DEFAULT FALSE, -- 是否结款
|
||||
is_new_direction BOOLEAN DEFAULT FALSE, -- 是否新方向验证
|
||||
-- 电商效果 (飞书暂无,预留)
|
||||
shop_uv BIGINT DEFAULT NULL, -- 进店UV
|
||||
transaction_amount NUMERIC(12,2) DEFAULT NULL, -- 成交金额(元)
|
||||
roi NUMERIC(10,4) DEFAULT NULL, -- ROI
|
||||
cpe NUMERIC(10,4) DEFAULT NULL, -- CPE (花费/互动量)
|
||||
engagement_rate NUMERIC(8,4) DEFAULT NULL, -- 互动率 (互动量/曝光量)
|
||||
-- 评论分析 (飞书暂无,预留)
|
||||
brand_mention_rate NUMERIC(8,4) DEFAULT NULL, -- 评论品牌提及率
|
||||
risk_flag VARCHAR(255) DEFAULT '', -- 差评或舆情风险
|
||||
--
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT uniq_cooperation_feishu_record UNIQUE (feishu_record_id)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE cmt_cooperations IS '合作记录表(每个达人+款式的一次合作)';
|
||||
COMMENT ON COLUMN cmt_cooperations.cooperation_date IS '合作日期(飞书制单日期)';
|
||||
COMMENT ON COLUMN cmt_cooperations.cooperation_cost IS '合作花费(元)';
|
||||
COMMENT ON COLUMN cmt_cooperations.ad_spend IS '投流金额(元)';
|
||||
COMMENT ON COLUMN cmt_cooperations.content_direction IS '发布笔记内容方向(飞书原始值)';
|
||||
COMMENT ON COLUMN cmt_cooperations.content_format IS '内容形式: 单品种草/测评合集/场景植入/开箱Vlog等';
|
||||
COMMENT ON COLUMN cmt_cooperations.exposure_count IS '曝光量(飞书自动抓取)';
|
||||
COMMENT ON COLUMN cmt_cooperations.engagement_count IS '互动赞藏数(原始文本如68.5万)';
|
||||
COMMENT ON COLUMN cmt_cooperations.data_performance IS '数据表现(逗号分隔: 曝光限流,数据良好,表现优秀等)';
|
||||
COMMENT ON COLUMN cmt_cooperations.shop_uv IS '进店UV(预留,飞书暂无)';
|
||||
COMMENT ON COLUMN cmt_cooperations.transaction_amount IS '成交金额(元,预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.roi IS 'ROI=成交金额/花费(预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.cpe IS 'CPE=花费/互动量(预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.engagement_rate IS '互动率=互动量/曝光量(预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.brand_mention_rate IS '评论品牌提及率(预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.risk_flag IS '差评或舆情风险(预留)';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_cooperations_style ON cmt_cooperations (style_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_cooperations_creator ON cmt_cooperations (creator_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_cooperations_date ON cmt_cooperations (cooperation_date);
|
||||
|
||||
-- ============================================================
|
||||
-- 3. 给新表加 updated_at 触发器
|
||||
-- ============================================================
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_trigger WHERE tgname = 'trg_cmt_cooperations_updated_at'
|
||||
) THEN
|
||||
CREATE TRIGGER trg_cmt_cooperations_updated_at
|
||||
BEFORE UPDATE ON cmt_cooperations
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
-- 达人月度分析报告表
|
||||
-- 每月1号自动跑 generate_creator_report.py，结果入库 + 飞书文档
|
||||
-- UNIQUE(month_start) — 每月一份全量报告
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cmt_creator_report (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
month_start DATE NOT NULL,
|
||||
month_end DATE NOT NULL,
|
||||
time_label VARCHAR(32) NOT NULL,
|
||||
-- 统计快照
|
||||
total_cooperations INTEGER DEFAULT 0, -- 合作记录总数
|
||||
total_creators INTEGER DEFAULT 0, -- 达人总数
|
||||
total_styles INTEGER DEFAULT 0, -- 款式数
|
||||
total_cost NUMERIC(14,2), -- 总合作花费
|
||||
total_exposure BIGINT, -- 总曝光量
|
||||
total_engagement BIGINT, -- 总互动量
|
||||
avg_cpe NUMERIC(10,2), -- 平均CPE
|
||||
-- 层级分布快照 (JSON)
|
||||
level_distribution JSONB DEFAULT '{}', -- {"头部":2,"腰部":36,"KOC":131,"素人":68,"未知":1}
|
||||
-- 平台分布快照 (JSON)
|
||||
platform_distribution JSONB DEFAULT '{}', -- {"douyin":196,"xiaohongshu":164,"bilibili":38}
|
||||
-- 评分分布快照 (JSON)
|
||||
score_distribution JSONB DEFAULT '{}', -- {"S":2,"A":62,"B":31,"C":143}
|
||||
-- 飞书文档
|
||||
doc_url TEXT,
|
||||
doc_title VARCHAR(200),
|
||||
report_chars INTEGER, -- 报告字符数
|
||||
-- 状态
|
||||
status VARCHAR(32), -- ok/llm_failed/doc_failed/exception
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT uniq_creator_report_month UNIQUE (month_start)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_creator_report_month ON cmt_creator_report (month_start);
|
||||
|
||||
COMMENT ON TABLE cmt_creator_report IS '达人月度分析报告表(每月1号自动生成,全量达人筛选与报价分析)';
|
||||
COMMENT ON COLUMN cmt_creator_report.time_label IS '月时间范围标签 如 2026-06-01~06-30';
|
||||
COMMENT ON COLUMN cmt_creator_report.level_distribution IS '层级分布快照 JSON {"头部":N,"腰部":N,...}';
|
||||
COMMENT ON COLUMN cmt_creator_report.platform_distribution IS '平台分布快照 JSON {"douyin":N,...}';
|
||||
COMMENT ON COLUMN cmt_creator_report.score_distribution IS '评分分布快照 JSON {"S":N,"A":N,...}';
|
||||
COMMENT ON COLUMN cmt_creator_report.doc_url IS '飞书文档链接(达人分析报告)';
|
||||
COMMENT ON COLUMN cmt_creator_report.status IS 'ok/llm_failed/doc_failed/exception';
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE cmt_styles
|
||||
ADD COLUMN IF NOT EXISTS style_category VARCHAR(64) DEFAULT '';
|
||||
|
||||
COMMENT ON COLUMN cmt_styles.style_category IS '款式风格，来源于飞书全域事项表的风格单选字段';
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
-- Product profile fields for cmt_styles.
|
||||
ALTER TABLE cmt_styles
|
||||
ADD COLUMN IF NOT EXISTS brand VARCHAR(128) DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS erp_style_codes TEXT[] DEFAULT '{}',
|
||||
ADD COLUMN IF NOT EXISTS platform_product_ids JSONB DEFAULT '{}'::jsonb,
|
||||
ADD COLUMN IF NOT EXISTS product_type VARCHAR(128) DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS dimensions VARCHAR(255) DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS colors TEXT[] DEFAULT '{}',
|
||||
ADD COLUMN IF NOT EXISTS material TEXT DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS capacity VARCHAR(128) DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS weight VARCHAR(128) DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS compatible_devices TEXT DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS features TEXT DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS selling_points TEXT DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS product_source_urls TEXT[] DEFAULT '{}',
|
||||
ADD COLUMN IF NOT EXISTS product_research_status VARCHAR(32) DEFAULT 'pending',
|
||||
ADD COLUMN IF NOT EXISTS product_researched_at TIMESTAMPTZ;
|
||||
|
||||
COMMENT ON COLUMN cmt_styles.platform_product_ids IS 'Product IDs grouped by Feishu platform';
|
||||
COMMENT ON COLUMN cmt_styles.product_source_urls IS 'Public pages used to verify product facts';
|
||||
COMMENT ON COLUMN cmt_styles.product_research_status IS 'verified, partial, or pending';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_styles_brand ON cmt_styles (brand);
|
||||
@@ -0,0 +1,55 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Monday 13:00 self-operated pipeline: Bilibili + Chanmama -> sync to cmt_notes (with INSERT)
|
||||
REM Feishu write-back happens inside each scraper; sync step writes PostgreSQL.
|
||||
REM Register: schtasks /Create /SC WEEKLY /D MON /TN YingxiaoYunying_MondaySelf /TR %PROJECT_DIR%\data\tools\monday_self_run.bat /ST 13:00 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing\scheduled
|
||||
set PYTHON=%GYXX_PYTHON%
|
||||
if not exist "%PYTHON%" set PYTHON=python
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
|
||||
set TS=%date:~0,4%%date:~5,2%%date:~8,2%_%time:~0,2%%time:~3,2%%time:~6,2%
|
||||
set TS=%TS: =0%
|
||||
|
||||
set LOG_FILE=%LOG_DIR%\monday_self_%TS%.log
|
||||
|
||||
echo === Monday self-operated run started at %date% %time% === > "%LOG_FILE%"
|
||||
echo PROJECT_DIR=%PROJECT_DIR% >> "%LOG_FILE%"
|
||||
|
||||
cd /d "%PROJECT_DIR%"
|
||||
|
||||
echo --- [step 1] self_bilibili_scraper.py --- >> "%LOG_FILE%"
|
||||
call %PYTHON% self_bilibili_scraper.py >> "%LOG_FILE%" 2>&1
|
||||
set RC_BILI=%ERRORLEVEL%
|
||||
echo self_bilibili exit=%RC_BILI% >> "%LOG_FILE%"
|
||||
|
||||
echo --- [step 2] chanmama_scraper.py --- >> "%LOG_FILE%"
|
||||
call %PYTHON% chanmama_scraper.py >> "%LOG_FILE%" 2>&1
|
||||
set RC_CM=%ERRORLEVEL%
|
||||
echo chanmama exit=%RC_CM% >> "%LOG_FILE%"
|
||||
|
||||
echo --- [step 3] sync_metrics_to_cmt_notes.py (with INSERT) --- >> "%LOG_FILE%"
|
||||
call %PYTHON% data\tools\sync_metrics_to_cmt_notes.py >> "%LOG_FILE%" 2>&1
|
||||
set RC_SYNC=%ERRORLEVEL%
|
||||
echo sync exit=%RC_SYNC% >> "%LOG_FILE%"
|
||||
|
||||
set FINAL_RC=0
|
||||
if not "%RC_BILI%"=="0" set FINAL_RC=1
|
||||
if not "%RC_CM%"=="0" set FINAL_RC=1
|
||||
if not "%RC_SYNC%"=="0" set FINAL_RC=1
|
||||
|
||||
echo === Monday self-operated run finished at %date% %time% (bili=%RC_BILI%, cm=%RC_CM%, sync=%RC_SYNC%, final=%FINAL_RC%) === >> "%LOG_FILE%"
|
||||
|
||||
endlocal & exit /b %FINAL_RC%
|
||||
@@ -0,0 +1,41 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Monthly creator report: run on 1st of each month at 08:30
|
||||
REM 1. Read cmt_cooperations + cmt_creators + cmt_styles from PG
|
||||
REM 2. Generate 9-section analysis report (Markdown)
|
||||
REM 3. Create Feishu doc + save to cmt_creator_report
|
||||
REM Register: schtasks /Create /SC MONTHLY /D 1 /TN YingxiaoYunying_MonthlyCreatorReport /TR %PROJECT_DIR%\data\tools\monthly_creator_report.bat /ST 08:30 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing\scheduled
|
||||
set PYTHON=%GYXX_PYTHON%
|
||||
where %PYTHON% >nul 2>&1 || set PYTHON=python
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
|
||||
set TS=%date:~0,4%%date:~5,2%%date:~8,2%_%time:~0,2%%time:~3,2%%time:~6,2%
|
||||
set TS=%TS: =0%
|
||||
|
||||
set LOG_FILE=%LOG_DIR%\monthly_creator_report_%TS%.log
|
||||
|
||||
echo === Monthly creator report started at %date% %time% === > "%LOG_FILE%"
|
||||
echo PROJECT_DIR=%PROJECT_DIR% >> "%LOG_FILE%"
|
||||
|
||||
cd /d "%PROJECT_DIR%"
|
||||
|
||||
call %PYTHON% -u -X utf8 data/tools/generate_creator_report.py >> "%LOG_FILE%" 2>&1
|
||||
set RC=%ERRORLEVEL%
|
||||
echo monthly_creator_report exit=%RC% >> "%LOG_FILE%"
|
||||
|
||||
echo === Monthly creator report finished at %date% %time% (rc=%RC%) === >> "%LOG_FILE%"
|
||||
|
||||
endlocal
|
||||
@@ -0,0 +1,41 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Monthly summary: run on 1st of each month at 08:00
|
||||
REM 1. Compute last month date range
|
||||
REM 2. Query PG for notes per style
|
||||
REM 3. For each style: note analysis + LLM cross-compare + Feishu doc + write back + save to DB
|
||||
REM Register: schtasks /Create /SC MONTHLY /D 1 /TN YingxiaoYunying_MonthlySummary /TR %PROJECT_DIR%\data\tools\monthly_summary.bat /ST 08:00 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing\scheduled
|
||||
set PYTHON=%GYXX_PYTHON%
|
||||
where %PYTHON% >nul 2>&1 || set PYTHON=python
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
|
||||
set TS=%date:~0,4%%date:~5,2%%date:~8,2%_%time:~0,2%%time:~3,2%%time:~6,2%
|
||||
set TS=%TS: =0%
|
||||
|
||||
set LOG_FILE=%LOG_DIR%\monthly_summary_%TS%.log
|
||||
|
||||
echo === Monthly summary started at %date% %time% === > "%LOG_FILE%"
|
||||
echo PROJECT_DIR=%PROJECT_DIR% >> "%LOG_FILE%"
|
||||
|
||||
cd /d "%PROJECT_DIR%"
|
||||
|
||||
call %PYTHON% -u -X utf8 monthly_summary_all.py --max-workers 4 >> "%LOG_FILE%" 2>&1
|
||||
set RC=%ERRORLEVEL%
|
||||
echo monthly_summary exit=%RC% >> "%LOG_FILE%"
|
||||
|
||||
echo === Monthly summary finished at %date% %time% (rc=%RC%) === >> "%LOG_FILE%"
|
||||
|
||||
endlocal
|
||||
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
拉取 24 张多维表格的全量记录,保存为 JSON。
|
||||
每张表输出: {base_token, table_id, name, fields, total_records, records}
|
||||
其中 records 元素为 {record_id, 字段名: 值, ...}
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
TABLES = [
|
||||
(1, "UkbabpqRYanmD7sk7ksceVuhnwf", "tbl2BdhIaohZNkOD", "盖亚斜挎"),
|
||||
(2, "OsYvbUe1iaJYeKs5WvLcU4rVnHc", "tblASZDx24q5Bjwy", "极星pro"),
|
||||
(3, "Qs4mbzyAaaQ8SasnDpkc9lA5npe", "tblMok3p5E6QydPq", "盖亚微单"),
|
||||
(4, "KnBRbH9iQaH3PAsAUk5cHLxNnMg", "tblcZ8y5126gTHe7", "盖世m1"),
|
||||
(5, "Ssp8bXmrwahRBbsko2BcuiLanad", "tblqNsVvKj4EPyzA", "星迹2"),
|
||||
(6, "BJaWbLEOKaHONMshOidcxvsKngh", "tblR0tCfPNk34ULj", "星云2"),
|
||||
(7, "N43abi7IlaS8XosnUXec6gH2n7g", "tblasQwwbXwpGbpL", "宙斯"),
|
||||
(8, "W80vbtPDoaaHTOs5Tipc6CTLn6c", "tblUaRl444G5IhVY", "阿波罗2"),
|
||||
(9, "CtW5bUcc9a5rqRsZkwucUWDVnHb", "tbl8WgoFrOjkqOD5", "阿波罗x1"),
|
||||
(10, "JsVLbso8Ba6RjbslOl1cJX8lnyb", "tblwqyhbymn9Aour", "波塞冬edge"),
|
||||
(11, "IQrbbxAHpa9oiWs631GcPkHznob", "tblmWRjVp0wQiVeG", "逐星"),
|
||||
(12, "MAEZbKrNaabJbDs5ZY7caq1ynrd", "tblcT5gqVRsPBuOE", "逐星GT"),
|
||||
(13, "RKfkb5JkIa9FttsgHvNc2nJYnbf", "tblu1yXKkQqwje0d", "星云mini"),
|
||||
(14, "VrLtbkldsaggOms3F4Bcu3DinCg", "tblO5hzLek6Nk43J", "晨星2"),
|
||||
(15, "GubZbTrMSahLg5sdonIc8kR8nZe", "tbl1tl5CI0LhKeuk", "极星双肩"),
|
||||
(16, "VwpibmlakaqHuLsGChccMb3BnNd", "tbl7fPd2dMFKQdih", "凌云air"),
|
||||
(17, "CQcibpRjgaUCg7sUTMwcnrhunMh", "tblFHe5uONCWxbEh", "觅光"),
|
||||
(18, "XtHjbqpwcab3J4sWP6Fcfgh5nYt", "tblqRgK6MI5C1H4K", "凌云3"),
|
||||
(19, "GNOkbKNpna8pDJsLMfzcuZt3ncf", "tbljwMiRxb4SlpGr", "瑞白"),
|
||||
(20, "U2JGbbWcIal6WvsxURRcTMwEnhf", "tblVEdURLsfRZykJ", "黑曜石吐司包"),
|
||||
(21, "U9S8bLYsNa8Ul8sPffAcrJ02nHI", "tblhpEmlXGyqCGuh", "黑曜石托特包"),
|
||||
(22, "UPk3bR9MOaTkgrsG1qLc7IUvnQe", "tbllXFWRI59dLCM2", "曜影"),
|
||||
(23, "PwrGbXcpRaxJnOsd9cacFou8nAf", "tbludhnt5lkmdvSF", "黑曜石电脑包"),
|
||||
(24, "CNgPbjuXiaPoxPsShROcYbRDnbb", "tblEymEPk7lVaQN1", "拓界2"),
|
||||
]
|
||||
|
||||
OUT_DIR = PATHS.raw_root / "feishu_tables"
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
import os
|
||||
LARK_CLI = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd")
|
||||
if not os.path.exists(LARK_CLI):
|
||||
LARK_CLI = "lark-cli.cmd"
|
||||
|
||||
|
||||
def call_lark(token: str, table_id: str, offset: int, limit: int) -> dict:
|
||||
"""调用 lark-cli base +record-list,返回 JSON"""
|
||||
# LARK_CLI_NO_PROXY=1 避免代理警告污染 stderr
|
||||
cmd = [
|
||||
LARK_CLI, "base", "+record-list",
|
||||
"--base-token", token,
|
||||
"--table-id", table_id,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--limit", str(limit),
|
||||
"--offset", str(offset),
|
||||
]
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True, env=env, encoding="utf-8", shell=False)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"lark-cli 调用失败(return={proc.returncode}): {proc.stderr}\nstdout: {proc.stdout[:500]}")
|
||||
# 找到第一个 { 开始
|
||||
text = proc.stdout
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
raise RuntimeError(f"lark-cli 返回无 JSON: {text[:500]}")
|
||||
# 用 strict=False 容忍控制字符(GBK 字符串残留)
|
||||
return json.loads(text[start:], strict=False)
|
||||
|
||||
|
||||
def pull_table(idx: int, token: str, table_id: str, name: str) -> dict:
|
||||
if not table_id:
|
||||
return {"index": idx, "name": name, "token": token, "table_id": table_id,
|
||||
"status": "SKIP_NO_TABLE", "total_records": 0}
|
||||
|
||||
page_size = 200
|
||||
offset = 0
|
||||
fields = None
|
||||
field_ids = None
|
||||
records = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
resp = call_lark(token, table_id, offset, page_size)
|
||||
except Exception as exc:
|
||||
return {"index": idx, "name": name, "token": token, "table_id": table_id,
|
||||
"status": "ERROR", "error": str(exc), "partial_records": len(records)}
|
||||
|
||||
if not resp.get("ok"):
|
||||
return {"index": idx, "name": name, "token": token, "table_id": table_id,
|
||||
"status": "API_ERROR", "error": resp.get("error", resp),
|
||||
"partial_records": len(records)}
|
||||
|
||||
data = resp["data"]
|
||||
if fields is None:
|
||||
fields = data["fields"]
|
||||
field_ids = data["field_id_list"]
|
||||
|
||||
rid_list = data["record_id_list"]
|
||||
row_data = data["data"]
|
||||
|
||||
for i, row in enumerate(row_data):
|
||||
rec = {"record_id": rid_list[i]}
|
||||
for j, fid in enumerate(field_ids):
|
||||
rec[fid] = row[j] if j < len(row) else None
|
||||
records.append(rec)
|
||||
|
||||
offset += len(row_data)
|
||||
has_more = data.get("has_more", False)
|
||||
print(f" [{idx:02d}] {name} pulled={len(row_data)} total={len(records)} has_more={has_more}", flush=True)
|
||||
if not has_more or len(row_data) == 0:
|
||||
break
|
||||
time.sleep(0.3)
|
||||
|
||||
return {
|
||||
"index": idx,
|
||||
"base_token": token,
|
||||
"table_id": table_id,
|
||||
"name": name,
|
||||
"fields": fields,
|
||||
"field_id_to_name": dict(zip(field_ids, fields)),
|
||||
"total_records": len(records),
|
||||
"records": records,
|
||||
"status": "OK",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
summary = []
|
||||
for idx, token, table_id, name in TABLES:
|
||||
print(f"[{idx:02d}/24] {name} (table={table_id}) START", flush=True)
|
||||
result = pull_table(idx, token, table_id, name)
|
||||
summary.append({
|
||||
"index": idx, "name": name, "token": token, "table_id": table_id,
|
||||
"status": result["status"],
|
||||
"total_records": result.get("total_records") or result.get("partial_records", 0),
|
||||
})
|
||||
if result["status"] == "OK":
|
||||
safe = str(idx) + "-" + name.translate(str.maketrans({
|
||||
"/": "_", "\\": "_", ":": "_", "*": "_", "?": "_",
|
||||
"\"": "_", "<": "_", ">": "_", "|": "_", " ": "_"
|
||||
}))
|
||||
out_path = OUT_DIR / f"{safe}.json"
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
print(f" -> saved {out_path} ({result['total_records']} records)", flush=True)
|
||||
else:
|
||||
print(f" -> FAILED: {result.get('error', result.get('status'))}", flush=True)
|
||||
time.sleep(0.5)
|
||||
|
||||
with open(OUT_DIR / "_summary.json", "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, ensure_ascii=False, indent=2)
|
||||
print("\n========== 汇总 ==========", flush=True)
|
||||
for s in summary:
|
||||
print(f" {s['index']:02d} {s['name']:>16s} {s['status']:>12s} {s['total_records']:>6d}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""
|
||||
rebuild_mapping.py — 动态重建 款式_多维表格_对照.json
|
||||
|
||||
工作原理:调用 feishu_mapping.refresh_mapping() 动态读取「合作达人」地址表,
|
||||
自动发现所有款式(当前 34 款,新增款式自动往后追加 index)。
|
||||
|
||||
用法:
|
||||
python rebuild_mapping.py # 动态重建 + 写盘 + 打印 diff
|
||||
python rebuild_mapping.py --diff # 只构建内存 mapping,对比当前 JSON,打印 diff,不写盘
|
||||
python rebuild_mapping.py --in-place # 等价于默认(直接覆盖 JSON)
|
||||
python rebuild_mapping.py --print # 打印最新 mapping 到 stdout(不写盘)
|
||||
|
||||
依赖: feishu_mapping.py (内部使用), lark-cli (动态读取)
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
PROJECT_DIR = PATHS.module_root
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping # noqa: E402
|
||||
|
||||
MAPPING_PATH = feishu_mapping.MAPPING_PATH
|
||||
|
||||
|
||||
def diff_entry(new: dict, old: dict | None) -> list[str]:
|
||||
"""对比单张表的新旧条目"""
|
||||
if not old:
|
||||
return [f" + 新增表 [{new['index']:02d}] {new['name']}"]
|
||||
diffs = []
|
||||
new_fm = new.get("field_map", {})
|
||||
old_fm = old.get("field_map", {})
|
||||
for k, v in new_fm.items():
|
||||
if k not in old_fm:
|
||||
diffs.append(f" + [{new['index']:02d}] {new['name']}: 新增字段 {k} = {v['field_name']}({v['field_id']})")
|
||||
elif old_fm[k].get("field_id") != v["field_id"]:
|
||||
diffs.append(f" ~ [{new['index']:02d}] {new['name']}: {k} field_id 变更 {old_fm[k]['field_id']} -> {v['field_id']}")
|
||||
for k in old_fm:
|
||||
if k not in new_fm:
|
||||
diffs.append(f" - [{new['index']:02d}] {new['name']}: 字段 {k}({old_fm[k]['field_name']}) 不存在了!")
|
||||
if new.get("total_records", 0) != old.get("total_records", 0):
|
||||
diffs.append(f" · [{new['index']:02d}] {new['name']}: total_records {old.get('total_records', 0)} -> {new.get('total_records', 0)}")
|
||||
return diffs
|
||||
|
||||
|
||||
def diff_mapping(new: dict, old: dict | None) -> list[str]:
|
||||
"""对比新旧 mapping"""
|
||||
if old is None:
|
||||
return [f" + 全新 mapping,共 {len(new.get('tables', []))} 款"]
|
||||
new_by_name = {t["name"]: t for t in new.get("tables", [])}
|
||||
old_by_name = {t["name"]: t for t in old.get("tables", [])}
|
||||
diffs = []
|
||||
for name in new_by_name:
|
||||
diffs.extend(diff_entry(new_by_name[name], old_by_name.get(name)))
|
||||
for name in old_by_name:
|
||||
if name not in new_by_name:
|
||||
diffs.append(f" - 表 [{old_by_name[name]['index']:02d}] {name}: 不再出现在地址表!")
|
||||
return diffs
|
||||
|
||||
|
||||
def print_core_field_check(tables: list[dict]) -> None:
|
||||
"""打印每张表的核心字段映射情况"""
|
||||
print(f"\n核心字段映射检查:")
|
||||
for t in tables:
|
||||
missing = [k for k in ["creator_name", "note_title", "read_count_7d"] if k not in t["field_map"]]
|
||||
if missing:
|
||||
print(f" [WARN] [{t['index']:02d}] {t['name']}: 缺 {missing}")
|
||||
else:
|
||||
cn = t['field_map']['creator_name']['field_name']
|
||||
nt = t['field_map']['note_title']['field_name']
|
||||
r7 = t['field_map']['read_count_7d']['field_name']
|
||||
print(f" [OK] [{t['index']:02d}] {t['name']}: creator='{cn}' title='{nt}' read7='{r7}'")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="动态重建 款式_多维表格_对照.json")
|
||||
ap.add_argument("--diff", action="store_true", help="只构建内存 mapping,对比当前 JSON,打印 diff,不写盘")
|
||||
ap.add_argument("--in-place", action="store_true", help="直接覆盖现有 JSON(等价于默认行为)")
|
||||
ap.add_argument("--print", action="store_true", help="只打印最新 mapping 到 stdout(不写盘)")
|
||||
args = ap.parse_args()
|
||||
|
||||
# 读旧 JSON
|
||||
old_mapping = None
|
||||
if MAPPING_PATH.exists():
|
||||
try:
|
||||
old_mapping = json.loads(MAPPING_PATH.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
print(f"[WARN] 旧 JSON 解析失败: {exc}")
|
||||
|
||||
# 决定是否写盘
|
||||
write = not (args.diff or args.print)
|
||||
|
||||
# 动态重建
|
||||
print(f"动态读取「合作达人」地址表 + 各款式字段定义...\n")
|
||||
new_mapping = feishu_mapping.refresh_mapping(force=True, write=write)
|
||||
if new_mapping is None:
|
||||
print("[FATAL] refresh_mapping 失败")
|
||||
sys.exit(1)
|
||||
|
||||
tables = new_mapping.get("tables", [])
|
||||
print(f"\n========== 重建结果 ==========")
|
||||
print(f"表数: {len(tables)}")
|
||||
if old_mapping:
|
||||
old_n = len(old_mapping.get("tables", []))
|
||||
if old_n != len(tables):
|
||||
print(f" (旧 {old_n} 款 -> 新 {len(tables)} 款)")
|
||||
|
||||
# diff
|
||||
diffs = diff_mapping(new_mapping, old_mapping)
|
||||
if diffs:
|
||||
print(f"\n变更 ({len(diffs)} 项):")
|
||||
for d in diffs:
|
||||
print(d)
|
||||
else:
|
||||
print(f"\n无字段变化")
|
||||
|
||||
print_core_field_check(tables)
|
||||
|
||||
# 落盘 / 打印
|
||||
if args.print:
|
||||
print(f"\n{json.dumps(new_mapping, ensure_ascii=False, indent=2)}")
|
||||
elif args.diff:
|
||||
print(f"\n(--diff 模式,未写文件)")
|
||||
elif write:
|
||||
print(f"\nsaved (in-place): {MAPPING_PATH}")
|
||||
else:
|
||||
print(f"\n(saved): {MAPPING_PATH}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
"""每周一重登 B 站。
|
||||
|
||||
强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。
|
||||
任务计划: 每周一 09:00
|
||||
|
||||
跑法:
|
||||
python data/tools/relogin_bilibili.py # 实际跑(会弹二维码)
|
||||
python data/tools/relogin_bilibili.py --dry-run # 只看会做什么
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.relogin_transaction import begin, commit, rollback
|
||||
|
||||
PROJECT_DIR = PATHS.module_root
|
||||
COOKIE_FILE = PATHS.state_root / "cookies/bilibili_cookies.json"
|
||||
STATE_FILE = PATHS.state_root / "cookies/bilibili_storage_state.json"
|
||||
PROFILE_DIR = PATHS.state_root / "browser-profiles/bilibili"
|
||||
SCRAPER = PROJECT_DIR / "bilibili_comment_scraper.py"
|
||||
|
||||
|
||||
def reset_state():
|
||||
"""删 cookie JSON + storage_state + 清浏览器 profile。"""
|
||||
state = begin((COOKIE_FILE, STATE_FILE), PROFILE_DIR)
|
||||
print(" [OK] 旧 Cookie/Profile 已暂存,失败时将自动恢复")
|
||||
return state
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="每周一重登 B 站")
|
||||
ap.add_argument("--login-timeout", type=int, default=300, help="扫码超时(秒)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不删不跑")
|
||||
args = ap.parse_args()
|
||||
|
||||
print("== Relogin: B 站 ==")
|
||||
print(f" 今天: {dt.date.today()} ({dt.date.today().strftime('%A')})")
|
||||
print(f" cookie: {COOKIE_FILE}")
|
||||
print(f" profile: {PROFILE_DIR}")
|
||||
print()
|
||||
|
||||
if args.dry_run:
|
||||
print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码")
|
||||
return 0
|
||||
|
||||
old_state = reset_state()
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(SCRAPER),
|
||||
"login", # dummy url (login-only 模式不需要真实 url)
|
||||
"--login-only",
|
||||
"--login-timeout",
|
||||
str(args.login_timeout),
|
||||
]
|
||||
print(f"\n [RUN] {' '.join(cmd)}")
|
||||
try:
|
||||
rc = subprocess.call(cmd, cwd=str(PROJECT_DIR))
|
||||
except BaseException:
|
||||
rollback(old_state)
|
||||
raise
|
||||
print(f"\n exit={rc}")
|
||||
login_ok = False
|
||||
if rc == 0:
|
||||
# 验证 SESSDATA 是否真写入了
|
||||
try:
|
||||
import json
|
||||
cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
sessdata = any(c.get("name") == "SESSDATA" and c.get("value") for c in cookies)
|
||||
login_ok = bool(sessdata)
|
||||
print(f" SESSDATA 写入: {'OK' if sessdata else 'FAIL — 没有看到 SESSDATA,登录可能没成功'}")
|
||||
except Exception as exc:
|
||||
print(f" [WARN] 验证 cookie 时出错: {exc}")
|
||||
if rc != 0 or not login_ok:
|
||||
rollback(old_state)
|
||||
print(" [ROLLBACK] 未完成扫码,已恢复旧登录状态")
|
||||
return rc or 1
|
||||
commit(old_state)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Force a fresh Douyin login while preserving the previous usable session."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
try:
|
||||
from .relogin_transaction import begin, commit, rollback
|
||||
except ImportError: # Direct script execution from data/tools.
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.relogin_transaction import begin, commit, rollback
|
||||
|
||||
PROJECT_DIR = PATHS.module_root
|
||||
COOKIE_FILE = PATHS.state_root / "cookies/douyin_cookies.json"
|
||||
STATE_FILE = PATHS.state_root / "cookies/douyin_storage_state.json"
|
||||
PROFILE_DIR = PATHS.state_root / "browser-profiles/douyin"
|
||||
SCRAPER = PROJECT_DIR / "douyin_comment_scraper.py"
|
||||
|
||||
LOGIN_COOKIE = "sessionid"
|
||||
|
||||
|
||||
def reset_state():
|
||||
state = begin((COOKIE_FILE, STATE_FILE), PROFILE_DIR)
|
||||
print(" [OK] Old Cookie/Profile staged; failures will roll back automatically")
|
||||
return state
|
||||
|
||||
|
||||
def is_cookie_valid() -> bool:
|
||||
if not COOKIE_FILE.exists():
|
||||
return False
|
||||
try:
|
||||
cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
return any(
|
||||
isinstance(cookie, dict)
|
||||
and cookie.get("name") == LOGIN_COOKIE
|
||||
and cookie.get("value")
|
||||
for cookie in cookies
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Relogin Douyin")
|
||||
parser.add_argument("--login-timeout", type=int, default=300)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("== Relogin: Douyin ==")
|
||||
print(f" date: {dt.date.today()}")
|
||||
print(f" cookie: {COOKIE_FILE}")
|
||||
print(f" profile: {PROFILE_DIR}")
|
||||
print(f" current cookie: {'present' if is_cookie_valid() else 'missing/invalid'}")
|
||||
|
||||
if args.dry_run:
|
||||
print("[DRY-RUN] Would stage Cookie/Profile and open the QR login window")
|
||||
return 0
|
||||
|
||||
old_state = reset_state()
|
||||
command = [
|
||||
sys.executable,
|
||||
str(SCRAPER),
|
||||
"login",
|
||||
"--login-only",
|
||||
"--login-timeout",
|
||||
str(args.login_timeout),
|
||||
]
|
||||
print(f"\n [RUN] {' '.join(command)}")
|
||||
try:
|
||||
return_code = subprocess.call(command, cwd=str(PROJECT_DIR))
|
||||
except BaseException:
|
||||
rollback(old_state)
|
||||
raise
|
||||
|
||||
login_ok = return_code == 0 and is_cookie_valid()
|
||||
print(f"\n exit={return_code}")
|
||||
print(f" {LOGIN_COOKIE} written: {'OK' if login_ok else 'FAIL'}")
|
||||
if not login_ok:
|
||||
rollback(old_state)
|
||||
print(" [ROLLBACK] Login incomplete; restored previous Cookie/Profile")
|
||||
return return_code or 1
|
||||
|
||||
commit(old_state)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,106 @@
|
||||
"""每周一重登小红书蒲公英。
|
||||
|
||||
强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。
|
||||
任务计划: 每周一 09:00
|
||||
|
||||
跑法:
|
||||
python data/tools/relogin_pgy.py # 实际跑(会弹二维码)
|
||||
python data/tools/relogin_pgy.py --dry-run # 只看会做什么
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
PROJECT_DIR = PATHS.module_root
|
||||
COOKIE_FILE = PATHS.state_root / "cookies/pgy_cookies.json"
|
||||
PROFILE_DIR = PATHS.state_root / "browser-profiles/pgy"
|
||||
SCRAPER = PROJECT_DIR / "pgy_xhs_scraper_v2.py"
|
||||
|
||||
|
||||
def reset_state() -> tuple[Path | None, Path | None]:
|
||||
"""Temporarily move the old login state aside so it can be restored."""
|
||||
ts = dt.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
||||
cookie_backup = None
|
||||
profile_backup = None
|
||||
if COOKIE_FILE.exists():
|
||||
backup = COOKIE_FILE.with_suffix(f".json.bak.{ts}")
|
||||
COOKIE_FILE.rename(backup)
|
||||
cookie_backup = backup
|
||||
print(f" [OK] 备份 cookie -> {backup.name}")
|
||||
else:
|
||||
print(f" [INFO] {COOKIE_FILE.name} 不存在,跳过")
|
||||
if PROFILE_DIR.exists():
|
||||
profile_backup = PROFILE_DIR.with_name(f"{PROFILE_DIR.name}.bak.{ts}")
|
||||
PROFILE_DIR.rename(profile_backup)
|
||||
print(f" [OK] 暂存 {PROFILE_DIR.name}/ -> {profile_backup.name}/")
|
||||
else:
|
||||
print(f" [INFO] {PROFILE_DIR.name}/ 不存在,跳过")
|
||||
return cookie_backup, profile_backup
|
||||
|
||||
|
||||
def restore_previous_state(cookie_backup: Path | None, profile_backup: Path | None) -> None:
|
||||
"""Discard an incomplete login and restore the last known-good state."""
|
||||
if COOKIE_FILE.exists():
|
||||
COOKIE_FILE.unlink()
|
||||
if cookie_backup and cookie_backup.exists():
|
||||
cookie_backup.rename(COOKIE_FILE)
|
||||
print(f" [ROLLBACK] 已恢复旧 cookie: {COOKIE_FILE.name}")
|
||||
if PROFILE_DIR.exists():
|
||||
shutil.rmtree(PROFILE_DIR)
|
||||
if profile_backup and profile_backup.exists():
|
||||
profile_backup.rename(PROFILE_DIR)
|
||||
print(f" [ROLLBACK] 已恢复旧浏览器状态: {PROFILE_DIR.name}/")
|
||||
|
||||
|
||||
def finish_successful_relogin(profile_backup: Path | None) -> None:
|
||||
"""Keep the new state and remove the bulky old profile backup."""
|
||||
if profile_backup and profile_backup.exists():
|
||||
shutil.rmtree(profile_backup)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="每周一重登小红书蒲公英")
|
||||
ap.add_argument("--login-timeout", type=int, default=300, help="扫码超时(秒)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不删不跑")
|
||||
args = ap.parse_args()
|
||||
|
||||
print("== Relogin: 小红书蒲公英 ==")
|
||||
print(f" 今天: {dt.date.today()} ({dt.date.today().strftime('%A')})")
|
||||
print(f" cookie: {COOKIE_FILE}")
|
||||
print(f" profile: {PROFILE_DIR}")
|
||||
print()
|
||||
|
||||
if args.dry_run:
|
||||
print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码")
|
||||
return 0
|
||||
|
||||
cookie_backup, profile_backup = reset_state()
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(SCRAPER),
|
||||
"--login-only",
|
||||
"--login-timeout",
|
||||
str(args.login_timeout),
|
||||
]
|
||||
print(f"\n [RUN] {' '.join(cmd)}")
|
||||
try:
|
||||
rc = subprocess.call(cmd, cwd=str(PROJECT_DIR))
|
||||
except BaseException:
|
||||
restore_previous_state(cookie_backup, profile_backup)
|
||||
raise
|
||||
print(f"\n exit={rc}")
|
||||
if rc != 0 or not COOKIE_FILE.exists():
|
||||
print(" [WARN] 新登录未完成,恢复旧登录状态")
|
||||
restore_previous_state(cookie_backup, profile_backup)
|
||||
return rc or 1
|
||||
finish_successful_relogin(profile_backup)
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Transactional backup/rollback helpers for interactive relogin scripts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoginStateBackup:
|
||||
files: dict[Path, Path]
|
||||
profile: Path
|
||||
profile_backup: Path | None
|
||||
|
||||
|
||||
def begin(files: tuple[Path, ...], profile: Path) -> LoginStateBackup:
|
||||
stamp = dt.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
||||
backups: dict[Path, Path] = {}
|
||||
for source in files:
|
||||
if source.exists():
|
||||
backup = source.with_name(f"{source.name}.bak.{stamp}")
|
||||
source.rename(backup)
|
||||
backups[source] = backup
|
||||
profile_backup = None
|
||||
if profile.exists():
|
||||
profile_backup = profile.with_name(f"{profile.name}.bak.{stamp}")
|
||||
profile.rename(profile_backup)
|
||||
return LoginStateBackup(backups, profile, profile_backup)
|
||||
|
||||
|
||||
def rollback(state: LoginStateBackup) -> None:
|
||||
for source, backup in state.files.items():
|
||||
if source.exists():
|
||||
source.unlink()
|
||||
if backup.exists():
|
||||
backup.rename(source)
|
||||
if state.profile.exists():
|
||||
shutil.rmtree(state.profile)
|
||||
if state.profile_backup and state.profile_backup.exists():
|
||||
state.profile_backup.rename(state.profile)
|
||||
|
||||
|
||||
def commit(state: LoginStateBackup) -> None:
|
||||
if state.profile_backup and state.profile_backup.exists():
|
||||
shutil.rmtree(state.profile_backup)
|
||||
@@ -0,0 +1,90 @@
|
||||
"""每周重登小红书。
|
||||
|
||||
强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。
|
||||
跑法:
|
||||
python data/tools/relogin_xiaohongshu.py # 实际跑(会弹二维码)
|
||||
python data/tools/relogin_xiaohongshu.py --dry-run # 只看会做什么
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.relogin_transaction import begin, commit, rollback
|
||||
|
||||
PROJECT_DIR = PATHS.module_root
|
||||
COOKIE_FILE = PATHS.state_root / "cookies/xiaohongshu_cookies.json"
|
||||
STATE_FILE = PATHS.state_root / "cookies/xiaohongshu_storage_state.json"
|
||||
PROFILE_DIR = PATHS.state_root / "browser-profiles/xiaohongshu"
|
||||
SCRAPER = PROJECT_DIR / "xiaohongshu_comment_scraper.py"
|
||||
|
||||
LOGIN_COOKIE = "web_session"
|
||||
|
||||
|
||||
def reset_state():
|
||||
state = begin((COOKIE_FILE, STATE_FILE), PROFILE_DIR)
|
||||
print(" [OK] 旧 Cookie/Profile 已暂存,失败时将自动恢复")
|
||||
return state
|
||||
|
||||
|
||||
def is_cookie_valid() -> bool:
|
||||
if not COOKIE_FILE.exists():
|
||||
return False
|
||||
try:
|
||||
cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
return any(c.get("name") == LOGIN_COOKIE and c.get("value") for c in cookies)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="重登小红书")
|
||||
ap.add_argument("--login-timeout", type=int, default=300, help="扫码超时(秒)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不删不跑")
|
||||
args = ap.parse_args()
|
||||
|
||||
print("== Relogin: 小红书 ==")
|
||||
print(f" 今天: {dt.date.today()}")
|
||||
print(f" cookie: {COOKIE_FILE}")
|
||||
print(f" profile: {PROFILE_DIR}")
|
||||
print(f" 当前 cookie 状态: {'有效' if is_cookie_valid() else '无效/不存在'}")
|
||||
print()
|
||||
|
||||
if args.dry_run:
|
||||
print("[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码")
|
||||
return 0
|
||||
|
||||
old_state = reset_state()
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(SCRAPER),
|
||||
"login",
|
||||
"--login-only",
|
||||
"--login-timeout",
|
||||
str(args.login_timeout),
|
||||
]
|
||||
print(f"\n [RUN] {' '.join(cmd)}")
|
||||
try:
|
||||
rc = subprocess.call(cmd, cwd=str(PROJECT_DIR))
|
||||
except BaseException:
|
||||
rollback(old_state)
|
||||
raise
|
||||
print(f"\n exit={rc}")
|
||||
login_ok = rc == 0 and is_cookie_valid()
|
||||
if rc == 0:
|
||||
print(f" {LOGIN_COOKIE} 写入: {'OK' if login_ok else 'FAIL — 没有看到 web_session,登录可能没成功'}")
|
||||
if not login_ok:
|
||||
rollback(old_state)
|
||||
print(" [ROLLBACK] 未完成扫码,已恢复旧登录状态")
|
||||
return rc or 1
|
||||
commit(old_state)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,299 @@
|
||||
"""每 3 个周一重登一次抖音/星图。
|
||||
|
||||
强制清掉 cookie JSON + 浏览器 profile,确保下次启动时必须扫码。
|
||||
星图登录完成后,自动把登录态 cookie 同步到抖音评论 scraper,避免再扫一次码。
|
||||
|
||||
任务计划: 每周一 09:00 (脚本内部判断是否到了该重登的周一,不到就退出)
|
||||
|
||||
3 周循环基准: 任意一个周一,后续 (今天 - BASE_DATE) // 7 % 3 == 0 才是星图周。
|
||||
改 BASE_DATE 就改了重登周期。
|
||||
|
||||
跑法:
|
||||
python data/tools/relogin_xingtu.py # 实际跑
|
||||
python data/tools/relogin_xingtu.py --dry-run # 看今天是不是星图周
|
||||
python data/tools/relogin_xingtu.py --force # 忽略 3 周判断,强制重登
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
import psutil
|
||||
|
||||
PROJECT_DIR = PATHS.module_root
|
||||
COOKIE_FILE = PATHS.state_root / "cookies/xingtu_cookies.json"
|
||||
PROFILE_DIR = PATHS.state_root / "browser-profiles/xingtu"
|
||||
SCRAPER = PROJECT_DIR / "xingtu_scraper_v2.py"
|
||||
|
||||
# 抖音评论 scraper 的 cookie 文件 (登录态共享目标)
|
||||
DY_COOKIE_FILE = PATHS.state_root / "cookies/douyin_cookies.json"
|
||||
DY_STATE_FILE = PATHS.state_root / "cookies/douyin_storage_state.json"
|
||||
DY_PROFILE_DIR = PATHS.state_root / "browser-profiles/douyin"
|
||||
|
||||
# 需要同步的抖音登录 cookie 名
|
||||
DY_LOGIN_COOKIES = {"sessionid", "sessionid_ss", "sid_guard", "sid_tt", "passport_csrf_token"}
|
||||
|
||||
# 3 周循环起点(必须是周一)。星图 cookie ~22 天,每 3 周(21 天)重登正好。
|
||||
BASE_DATE = dt.date(2026, 6, 22)
|
||||
LOGIN_COOKIE_NAMES = {"sessionid", "sessionid_ss", "sid_tt", "sid_guard"}
|
||||
|
||||
|
||||
def transaction_file() -> Path:
|
||||
return COOKIE_FILE.with_name(".xingtu_relogin_transaction.json")
|
||||
|
||||
|
||||
def _write_transaction(cookie_backup: Path | None, profile_backup: Path | None) -> None:
|
||||
marker = transaction_file()
|
||||
marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp = marker.with_name(f"{marker.name}.tmp")
|
||||
temp.write_text(json.dumps({
|
||||
"cookie_backup": str(cookie_backup) if cookie_backup else "",
|
||||
"profile_backup": str(profile_backup) if profile_backup else "",
|
||||
}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
os.replace(temp, marker)
|
||||
|
||||
|
||||
def _clear_transaction() -> None:
|
||||
marker = transaction_file()
|
||||
if marker.exists():
|
||||
marker.unlink()
|
||||
|
||||
|
||||
def _remove_path(path: Path) -> None:
|
||||
if path.is_dir():
|
||||
shutil.rmtree(path)
|
||||
elif path.exists():
|
||||
path.unlink()
|
||||
|
||||
|
||||
def recover_interrupted_state() -> bool:
|
||||
"""Restore a relogin transaction that was killed after moving old state."""
|
||||
marker = transaction_file()
|
||||
if not marker.exists():
|
||||
return False
|
||||
try:
|
||||
state = json.loads(marker.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
cookie_backup = Path(state["cookie_backup"]) if state.get("cookie_backup") else None
|
||||
profile_backup = Path(state["profile_backup"]) if state.get("profile_backup") else None
|
||||
terminate_profile_processes()
|
||||
if cookie_backup and cookie_backup.exists():
|
||||
_remove_path(COOKIE_FILE)
|
||||
cookie_backup.rename(COOKIE_FILE)
|
||||
if profile_backup and profile_backup.exists():
|
||||
_remove_path(PROFILE_DIR)
|
||||
profile_backup.rename(PROFILE_DIR)
|
||||
_clear_transaction()
|
||||
print(" [RECOVER] 已恢复上次被中断的星图登录状态")
|
||||
return True
|
||||
|
||||
|
||||
def has_valid_login_cookie_file(path: Path | None = None) -> bool:
|
||||
path = path or COOKIE_FILE
|
||||
try:
|
||||
cookies = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return False
|
||||
return any(
|
||||
isinstance(cookie, dict)
|
||||
and cookie.get("name") in LOGIN_COOKIE_NAMES
|
||||
and cookie.get("value")
|
||||
for cookie in cookies
|
||||
)
|
||||
|
||||
|
||||
def terminate_profile_processes() -> None:
|
||||
"""Stop only browser processes using Xingtu's dedicated profile."""
|
||||
profile_text = str(PROFILE_DIR.resolve()).casefold()
|
||||
matches = []
|
||||
for process in psutil.process_iter(["pid", "cmdline"]):
|
||||
try:
|
||||
command = " ".join(process.info.get("cmdline") or []).casefold()
|
||||
if process.pid != os.getpid() and profile_text in command:
|
||||
matches.append(process)
|
||||
except (psutil.Error, OSError):
|
||||
continue
|
||||
for process in matches:
|
||||
try:
|
||||
process.terminate()
|
||||
except (psutil.Error, OSError):
|
||||
pass
|
||||
_gone, alive = psutil.wait_procs(matches, timeout=5)
|
||||
for process in alive:
|
||||
try:
|
||||
process.kill()
|
||||
except (psutil.Error, OSError):
|
||||
pass
|
||||
|
||||
|
||||
def is_xingtu_week(today: dt.date, base: dt.date) -> bool:
|
||||
days = (today - base).days
|
||||
if days < 0:
|
||||
return False
|
||||
return (days // 7) % 3 == 0
|
||||
|
||||
|
||||
def reset_state() -> tuple[Path | None, Path | None]:
|
||||
"""Temporarily move the old login state aside so it can be restored."""
|
||||
ts = dt.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
||||
cookie_backup = None
|
||||
profile_backup = None
|
||||
if COOKIE_FILE.exists():
|
||||
cookie_backup = COOKIE_FILE.with_suffix(f".json.bak.{ts}")
|
||||
if PROFILE_DIR.exists():
|
||||
profile_backup = PROFILE_DIR.with_name(f"{PROFILE_DIR.name}.bak.{ts}")
|
||||
_write_transaction(cookie_backup, profile_backup)
|
||||
|
||||
if cookie_backup:
|
||||
COOKIE_FILE.rename(cookie_backup)
|
||||
print(f" [OK] 备份 cookie -> {cookie_backup.name}")
|
||||
else:
|
||||
print(f" [INFO] {COOKIE_FILE.name} 不存在,跳过")
|
||||
if profile_backup:
|
||||
PROFILE_DIR.rename(profile_backup)
|
||||
print(f" [OK] 暂存 {PROFILE_DIR.name}/ -> {profile_backup.name}/")
|
||||
else:
|
||||
print(f" [INFO] {PROFILE_DIR.name}/ 不存在,跳过")
|
||||
return cookie_backup, profile_backup
|
||||
|
||||
|
||||
def restore_previous_state(cookie_backup: Path | None, profile_backup: Path | None) -> None:
|
||||
"""Discard an incomplete login and restore the last known-good state."""
|
||||
terminate_profile_processes()
|
||||
if COOKIE_FILE.exists():
|
||||
COOKIE_FILE.unlink()
|
||||
if cookie_backup and cookie_backup.exists():
|
||||
cookie_backup.rename(COOKIE_FILE)
|
||||
print(f" [ROLLBACK] 已恢复旧 cookie: {COOKIE_FILE.name}")
|
||||
if PROFILE_DIR.exists():
|
||||
shutil.rmtree(PROFILE_DIR)
|
||||
if profile_backup and profile_backup.exists():
|
||||
profile_backup.rename(PROFILE_DIR)
|
||||
print(f" [ROLLBACK] 已恢复旧浏览器状态: {PROFILE_DIR.name}/")
|
||||
_clear_transaction()
|
||||
|
||||
|
||||
def finish_successful_relogin(profile_backup: Path | None) -> None:
|
||||
"""Keep the new state and remove the bulky old profile backup."""
|
||||
if profile_backup and profile_backup.exists():
|
||||
shutil.rmtree(profile_backup)
|
||||
_clear_transaction()
|
||||
|
||||
|
||||
def sync_to_douyin_scraper() -> None:
|
||||
"""把星图的登录态 cookie 同步到抖音评论 scraper,避免再扫一次码。"""
|
||||
if not COOKIE_FILE.exists():
|
||||
print(" [SKIP] 星图 cookie 不存在,无法同步")
|
||||
return
|
||||
|
||||
try:
|
||||
xingtu_cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
print(f" [SKIP] 读取星图 cookie 失败: {exc}")
|
||||
return
|
||||
|
||||
# 提取登录态 cookie
|
||||
login_cookies = [c for c in xingtu_cookies if isinstance(c, dict) and c.get("name") in DY_LOGIN_COOKIES]
|
||||
if not login_cookies:
|
||||
print(" [SKIP] 星图 cookie 中没有找到登录态,跳过同步")
|
||||
return
|
||||
|
||||
# 清除抖音评论 scraper 的旧 profile (强制刷新)
|
||||
if DY_PROFILE_DIR.exists():
|
||||
shutil.rmtree(DY_PROFILE_DIR)
|
||||
print(f" [OK] 清空 {DY_PROFILE_DIR.name}/")
|
||||
|
||||
# 读取或创建抖音评论 cookie 文件
|
||||
dy_cookies = []
|
||||
if DY_COOKIE_FILE.exists():
|
||||
try:
|
||||
dy_cookies = json.loads(DY_COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
dy_cookies = []
|
||||
|
||||
# 合并: 同名 cookie 覆盖,新 cookie 追加
|
||||
dy_by_name = {c.get("name"): c for c in dy_cookies if isinstance(c, dict)}
|
||||
for c in login_cookies:
|
||||
# 修正 domain: 星图的 cookie domain 可能是 .douyin.com 子域,评论采集需要 .douyin.com
|
||||
domain = c.get("domain", "")
|
||||
if "xingtu" in domain or "oceanengine" in domain:
|
||||
c["domain"] = ".douyin.com"
|
||||
dy_by_name[c["name"]] = c
|
||||
|
||||
DY_COOKIE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
DY_COOKIE_FILE.write_text(
|
||||
json.dumps(list(dy_by_name.values()), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f" [OK] 同步 {len(login_cookies)} 个登录 cookie 到 {DY_COOKIE_FILE.name}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="每 3 周周一重登星图")
|
||||
ap.add_argument("--base-date", default=BASE_DATE.isoformat(),
|
||||
help="3 周循环起点(必须是周一),格式 YYYY-MM-DD")
|
||||
ap.add_argument("--login-timeout", type=int, default=300)
|
||||
ap.add_argument("--force", action="store_true", help="忽略 3 周判断,强制重登")
|
||||
ap.add_argument("--dry-run", action="store_true", help="只打印计划")
|
||||
args = ap.parse_args()
|
||||
|
||||
today = dt.date.today()
|
||||
base = dt.date.fromisoformat(args.base_date)
|
||||
is_xw = is_xingtu_week(today, base)
|
||||
|
||||
print("== Relogin: 抖音/星图 ==")
|
||||
print(f" 今天: {today} ({today.strftime('%A')})")
|
||||
print(f" 3 周基准: {base}")
|
||||
print(f" 星图周? {'是' if is_xw else '否'}")
|
||||
print()
|
||||
|
||||
if not is_xw and not args.force:
|
||||
print("今天不是星图周,跳过 (传 --force 强制重登)")
|
||||
return 0
|
||||
|
||||
if args.dry_run:
|
||||
print(f"[DRY-RUN] 实际跑会: 删 cookie + 清 profile + 弹二维码")
|
||||
return 0
|
||||
|
||||
recover_interrupted_state()
|
||||
terminate_profile_processes()
|
||||
cookie_backup, profile_backup = reset_state()
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(SCRAPER),
|
||||
"--login-only",
|
||||
"--login-timeout",
|
||||
str(args.login_timeout),
|
||||
]
|
||||
print(f"\n [RUN] {' '.join(cmd)}")
|
||||
try:
|
||||
rc = subprocess.call(cmd, cwd=str(PROJECT_DIR))
|
||||
except BaseException:
|
||||
restore_previous_state(cookie_backup, profile_backup)
|
||||
raise
|
||||
finally:
|
||||
terminate_profile_processes()
|
||||
print(f"\n exit={rc}")
|
||||
|
||||
if rc != 0 or not has_valid_login_cookie_file():
|
||||
print(" [WARN] 新登录未完成,恢复旧登录状态")
|
||||
restore_previous_state(cookie_backup, profile_backup)
|
||||
return rc or 1
|
||||
|
||||
finish_successful_relogin(profile_backup)
|
||||
if rc == 0:
|
||||
sync_to_douyin_scraper()
|
||||
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,507 @@
|
||||
"""补跑缺失 / 陈旧 / 部分失败的款式(自动检测 + 重试,粒度细到 record_id)。
|
||||
|
||||
定义"需要补跑":
|
||||
MISSING - v2 JSON 文件不存在(进程崩了 / 中途中断) -> 整款重跑
|
||||
STALE - v2 JSON 文件存在,但 mtime 超过 --stale-hours 小时 -> 整款重跑
|
||||
ALL_FAILED - v2 JSON 文件新鲜,但 matched=0 且 filled=0 -> 整款重跑
|
||||
HALF - v2 JSON 文件新鲜,部分 record 失败(matched>0 但有 failed) -> 只补失败的 record_id
|
||||
|
||||
HALF 默认开启(因为只补失败的单条,不再昂贵);
|
||||
可用 --no-include-half 关掉。
|
||||
|
||||
用法:
|
||||
python data/tools/retry_failed.py # 三个平台都补
|
||||
python data/tools/retry_failed.py --platform xt # 只补星图
|
||||
python data/tools/retry_failed.py --platform pgy,xt
|
||||
python data/tools/retry_failed.py --dry-run # 只看不跑
|
||||
python data/tools/retry_failed.py --style 8 # 强制补指定款式(忽略检测)
|
||||
python data/tools/retry_failed.py --style 8,15,20
|
||||
python data/tools/retry_failed.py --max-attempts 3 # 每款式最多重试 3 次
|
||||
python data/tools/retry_failed.py --stale-hours 4 # 4h 以上的算陈旧
|
||||
python data/tools/retry_failed.py --no-include-half # 不补 HALF
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
PROJECT_DIR = PATHS.module_root
|
||||
V2_DIR = PATHS.normalized_root / "v2_results"
|
||||
|
||||
# 复用 run_all.py 的 SCRIPTS 字典,保持单一事实
|
||||
SCRIPTS = {
|
||||
"bili": {
|
||||
"name": "B 站",
|
||||
"script": "bilibili_scraper.py",
|
||||
"label": "bili",
|
||||
},
|
||||
"pgy": {
|
||||
"name": "小红书蒲公英",
|
||||
"script": "pgy_xhs_scraper_v2.py",
|
||||
"label": "xhs",
|
||||
},
|
||||
"xt": {
|
||||
"name": "星图",
|
||||
"script": "xingtu_scraper_v2.py",
|
||||
"label": "xt",
|
||||
},
|
||||
}
|
||||
|
||||
SELF_SCRIPTS = {
|
||||
"bili": {**SCRIPTS["bili"], "extra_args": ["--self-operated"]},
|
||||
"pgy": {**SCRIPTS["pgy"], "extra_args": ["--self-operated"]},
|
||||
"xt": {
|
||||
"name": "抖音(自营)",
|
||||
"script": "self_douyin_scraper.py",
|
||||
"label": "xt",
|
||||
"extra_args": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _mapping_cache_path(self_operated: bool = False) -> Path:
|
||||
name = "款式_多维表格_对照_自营.json" if self_operated else "款式_多维表格_对照.json"
|
||||
cache = PATHS.normalized_root / "mappings" / name
|
||||
return cache if cache.exists() else PATHS.config_root / name
|
||||
|
||||
|
||||
def load_current_styles(self_operated: bool = False) -> list[dict]:
|
||||
"""只读加载当前款式缓存。
|
||||
|
||||
补跑属于结果审计阶段，不能在 import 或 ``--dry-run`` 时触发飞书刷新、
|
||||
更不能把自营缓存作为副作用写回。映射刷新由 run_all 在采集开始前负责。
|
||||
"""
|
||||
path = _mapping_cache_path(self_operated)
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
log(f"[ERROR] 读取款式缓存失败 {path}: {exc}")
|
||||
return []
|
||||
|
||||
by_index: dict[int, dict] = {}
|
||||
for raw in data.get("tables", []):
|
||||
if not isinstance(raw, dict) or not raw.get("index") or not raw.get("name"):
|
||||
continue
|
||||
try:
|
||||
idx = int(raw["index"])
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
style = dict(raw)
|
||||
style["index"] = idx
|
||||
style["name"] = str(raw["name"])
|
||||
by_index[idx] = style
|
||||
return [by_index[idx] for idx in sorted(by_index)]
|
||||
|
||||
|
||||
def canonical_result_filename(style: dict, platform_key: str,
|
||||
self_operated: bool = False) -> str:
|
||||
"""返回当前款式唯一允许参与完整度判断的规范结果文件名。"""
|
||||
if platform_key not in SCRIPTS:
|
||||
raise ValueError(f"unknown platform: {platform_key}")
|
||||
idx = int(style["index"])
|
||||
name = str(style["name"])
|
||||
self_sfx = "_self" if self_operated else ""
|
||||
if platform_key == "pgy":
|
||||
suffix = f"{self_sfx}_v2.json"
|
||||
elif platform_key == "xt":
|
||||
suffix = f"{self_sfx}_xingtu_v2.json"
|
||||
else:
|
||||
suffix = f"{self_sfx}_bilibili_v2.json"
|
||||
return f"{idx:02d}-{name}{suffix}"
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
ts = dt.datetime.now().strftime("%H:%M:%S")
|
||||
safe = msg.encode("gbk", errors="replace").decode("gbk", errors="replace")
|
||||
print(f"[{ts}] {safe}", flush=True)
|
||||
|
||||
|
||||
def _result_rows(data: dict) -> list[dict]:
|
||||
raw = data.get("results")
|
||||
if raw is None:
|
||||
raw = data.get("details")
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
return [row for row in raw if isinstance(row, dict)]
|
||||
|
||||
|
||||
def _result_is_unresolved(row: dict) -> bool:
|
||||
"""兼容新状态契约与旧版 matched/ok/error/reason 字段。"""
|
||||
status = row.get("status")
|
||||
if status == "blocked_input":
|
||||
return False
|
||||
explicit_failure = bool(
|
||||
row.get("matched") is False
|
||||
or row.get("ok") is False
|
||||
or row.get("write_ok") is False
|
||||
or row.get("error")
|
||||
or row.get("reason")
|
||||
)
|
||||
if status == "success":
|
||||
return explicit_failure
|
||||
if status:
|
||||
return True
|
||||
return explicit_failure
|
||||
|
||||
|
||||
def _result_is_success(row: dict) -> bool:
|
||||
status = row.get("status")
|
||||
if status is not None:
|
||||
return status == "success" and not _result_is_unresolved(row)
|
||||
if _result_is_unresolved(row):
|
||||
return False
|
||||
# 旧版 B 站成功行有 ok=True；旧版 pgy/xt 成功行有命中标题或曝光值。
|
||||
return bool(
|
||||
row.get("ok") is True
|
||||
or row.get("matched") is True
|
||||
or row.get("matched_title")
|
||||
or row.get("read_count") is not None
|
||||
or row.get("play_count") is not None
|
||||
)
|
||||
|
||||
|
||||
def requested_records_succeeded(data: dict,
|
||||
requested_record_ids: list[str]) -> tuple[bool, list[str]]:
|
||||
"""核验局部补跑请求的每个 record_id 都恰好有一条 success 终态。"""
|
||||
rows_by_id: dict[str, list[dict]] = {}
|
||||
for row in _result_rows(data):
|
||||
rid = row.get("record_id")
|
||||
if rid is not None:
|
||||
rows_by_id.setdefault(str(rid), []).append(row)
|
||||
|
||||
failed: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw_rid in requested_record_ids:
|
||||
rid = str(raw_rid)
|
||||
if rid in seen:
|
||||
continue
|
||||
seen.add(rid)
|
||||
candidates = rows_by_id.get(rid, [])
|
||||
if len(candidates) != 1 or not _result_is_success(candidates[0]):
|
||||
failed.append(rid)
|
||||
return not failed, failed
|
||||
|
||||
|
||||
def _summary_completed(data: dict) -> tuple[bool, str]:
|
||||
"""核验整款结果覆盖、唯一性和终态；blocked_input 是已解释终态。"""
|
||||
total = data.get("total")
|
||||
if total is None:
|
||||
total = data.get("total_b_records")
|
||||
if total is None:
|
||||
total = data.get("total_tasks")
|
||||
try:
|
||||
total = int(total or 0)
|
||||
except (TypeError, ValueError):
|
||||
return False, "invalid total"
|
||||
if total < 0:
|
||||
return False, "invalid total"
|
||||
|
||||
rows = _result_rows(data)
|
||||
record_ids = [str(row["record_id"]) for row in rows if row.get("record_id")]
|
||||
if len(rows) != total:
|
||||
return False, f"result coverage {len(rows)}/{total}"
|
||||
if len(record_ids) != len(rows) or len(set(record_ids)) != len(record_ids):
|
||||
return False, "missing or duplicate record_id"
|
||||
unresolved = [rid for rid, row in zip(record_ids, rows) if _result_is_unresolved(row)]
|
||||
if unresolved:
|
||||
return False, f"unresolved records: {','.join(unresolved[:5])}"
|
||||
if _summary_declares_incomplete(data):
|
||||
return False, "summary declares unresolved failures"
|
||||
return True, ""
|
||||
|
||||
|
||||
def _summary_declares_incomplete(data: dict) -> bool:
|
||||
if data.get("complete") is False:
|
||||
return True
|
||||
for key in (
|
||||
"unresolved", "retryable_failures", "write_failures",
|
||||
"missing_results", "duplicate_results",
|
||||
):
|
||||
try:
|
||||
if int(data.get(key) or 0) > 0:
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def extract_failed_records(data: dict, platform_key: str) -> list[str]:
|
||||
"""从 v2 JSON 内容里提取失败的 record_id 列表。
|
||||
|
||||
xingtu/pgy: results 数组,失败标志 matched: false(可能加 reason/error)
|
||||
bilibili: details 数组,失败标志 ok: false(可能加 action)
|
||||
"""
|
||||
rids: list[str] = []
|
||||
for row in _result_rows(data):
|
||||
rid = row.get("record_id")
|
||||
if not rid:
|
||||
continue
|
||||
# blocked_input 是可解释但不可重试的终态，不应反复补跑。
|
||||
if _result_is_unresolved(row):
|
||||
rids.append(str(rid))
|
||||
return list(dict.fromkeys(rids))
|
||||
|
||||
|
||||
def find_failed(platform_key: str, stale_hours: float, include_half: bool,
|
||||
self_operated: bool = False, *,
|
||||
styles: list[dict] | None = None,
|
||||
v2_dir: Path | None = None,
|
||||
now: dt.datetime | None = None) -> dict[int, dict]:
|
||||
"""扫 v2_results,返回 {idx: {reason, failed_rids}} 字典。
|
||||
|
||||
reason 取值: MISSING / STALE / ALL_FAILED / HALF
|
||||
failed_rids: HALF 时给失败 record_id 列表;其他场景为空(整款重跑)。
|
||||
self_operated=True 时只扫自营达人结果文件(*_self_*.json)
|
||||
"""
|
||||
if platform_key not in SCRIPTS:
|
||||
raise ValueError(f"unknown platform: {platform_key}")
|
||||
current_styles = styles if styles is not None else load_current_styles(self_operated)
|
||||
result_dir = v2_dir or V2_DIR
|
||||
current_time = now or dt.datetime.now()
|
||||
out: dict[int, dict] = {}
|
||||
|
||||
for style in current_styles:
|
||||
idx = int(style["index"])
|
||||
path = result_dir / canonical_result_filename(style, platform_key, self_operated)
|
||||
if not path.exists():
|
||||
out[idx] = {"reason": "MISSING", "failed_rids": []}
|
||||
continue
|
||||
|
||||
mtime = dt.datetime.fromtimestamp(path.stat().st_mtime)
|
||||
age_h = (current_time - mtime).total_seconds() / 3600.0
|
||||
if age_h > stale_hours:
|
||||
out[idx] = {"reason": "STALE", "failed_rids": []}
|
||||
continue
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
out[idx] = {"reason": "INVALID", "failed_rids": []}
|
||||
continue
|
||||
|
||||
total = data.get("total")
|
||||
if total is None:
|
||||
total = data.get("total_b_records")
|
||||
if total is None:
|
||||
total = data.get("total_tasks")
|
||||
try:
|
||||
total = int(total or 0)
|
||||
except (TypeError, ValueError):
|
||||
out[idx] = {"reason": "INVALID", "failed_rids": []}
|
||||
continue
|
||||
|
||||
rows = _result_rows(data)
|
||||
failed_rids = extract_failed_records(data, platform_key)
|
||||
resolved = sum(1 for row in rows if not _result_is_unresolved(row))
|
||||
if total > 0 and resolved == 0:
|
||||
out[idx] = {"reason": "ALL_FAILED", "failed_rids": []}
|
||||
continue
|
||||
record_ids = [str(row["record_id"]) for row in rows if row.get("record_id")]
|
||||
coverage_invalid = (
|
||||
len(rows) != total
|
||||
or len(record_ids) != len(rows)
|
||||
or len(set(record_ids)) != len(record_ids)
|
||||
)
|
||||
if coverage_invalid:
|
||||
# 缺失记录无法从结果中推导 record_id，只能整款重跑。
|
||||
out[idx] = {"reason": "INCOMPLETE", "failed_rids": []}
|
||||
continue
|
||||
if include_half and failed_rids:
|
||||
out[idx] = {"reason": "HALF", "failed_rids": failed_rids}
|
||||
continue
|
||||
if _summary_declares_incomplete(data):
|
||||
out[idx] = {"reason": "INCOMPLETE", "failed_rids": []}
|
||||
return out
|
||||
|
||||
|
||||
def run_one_style(platform_key: str, idx: int, max_attempts: int,
|
||||
self_operated: bool = False,
|
||||
record_ids: list[str] | None = None, *,
|
||||
style: dict | None = None,
|
||||
v2_dir: Path | None = None) -> bool:
|
||||
"""跑单款式,失败重试。返回最终是否成功。
|
||||
|
||||
record_ids 非空时,只跑这些 record_id(--record 参数,粒度细到单条笔记)。
|
||||
"""
|
||||
info = SELF_SCRIPTS[platform_key] if self_operated else SCRIPTS[platform_key]
|
||||
if style is None:
|
||||
style = next(
|
||||
(item for item in load_current_styles(self_operated)
|
||||
if int(item["index"]) == int(idx)),
|
||||
None,
|
||||
)
|
||||
if style is None:
|
||||
log(f" [ERROR] 当前款式映射中不存在编号 {idx}")
|
||||
return False
|
||||
result_path = (v2_dir or V2_DIR) / canonical_result_filename(
|
||||
style, platform_key, self_operated,
|
||||
)
|
||||
cmd = [sys.executable, str(PROJECT_DIR / info["script"]), "--style", str(idx)]
|
||||
cmd.extend(info.get("extra_args", []))
|
||||
for rid in record_ids or []:
|
||||
cmd += ["--record", rid]
|
||||
label = f"[{platform_key}{'-self' if self_operated else ''}] 款式 {idx:>2}"
|
||||
desc = f"整款" if not record_ids else f"{len(record_ids)} 条"
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
log(f" {label} 尝试 {attempt}/{max_attempts} ({desc}): {info['script']} --style {idx}"
|
||||
+ (f" --record x{len(record_ids)}" if record_ids else ""))
|
||||
before_mtime_ns = result_path.stat().st_mtime_ns if result_path.exists() else None
|
||||
try:
|
||||
rc = subprocess.call(cmd, cwd=str(PROJECT_DIR))
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception as exc:
|
||||
log(f" [ERROR] 进程异常: {exc}")
|
||||
rc = -1
|
||||
if rc != 0:
|
||||
log(f" [FAIL] exit={rc}")
|
||||
else:
|
||||
try:
|
||||
after_mtime_ns = result_path.stat().st_mtime_ns
|
||||
if before_mtime_ns is not None and after_mtime_ns == before_mtime_ns:
|
||||
raise RuntimeError("规范结果文件未在本次补跑中更新")
|
||||
payload = json.loads(result_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("规范结果不是 JSON object")
|
||||
if record_ids:
|
||||
verified, failed_rids = requested_records_succeeded(payload, record_ids)
|
||||
reason = f"仍失败/缺失 record: {','.join(failed_rids[:5])}"
|
||||
else:
|
||||
verified, reason = _summary_completed(payload)
|
||||
if verified:
|
||||
log(" [OK] exit=0 且结果完整度校验通过")
|
||||
return True
|
||||
log(f" [INCOMPLETE] exit=0 但{reason}")
|
||||
except Exception as exc:
|
||||
log(f" [INCOMPLETE] exit=0 但结果校验失败: {exc}")
|
||||
if attempt < max_attempts:
|
||||
time.sleep(2)
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="补跑缺失 / 陈旧的款式")
|
||||
ap.add_argument("--platform", type=str, default="bili,pgy,xt",
|
||||
help="逗号分隔 bili/pgy/xt,默认全跑")
|
||||
ap.add_argument("--style", type=str, default="",
|
||||
help="强制指定要补的款式(逗号分隔),忽略检测")
|
||||
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不跑")
|
||||
ap.add_argument("--max-attempts", type=int, default=1,
|
||||
help="每个款式最多重试次数(默认 1)")
|
||||
ap.add_argument("--stale-hours", type=float, default=26.0,
|
||||
help="超过多少小时算陈旧(默认 26h)")
|
||||
ap.add_argument("--no-include-half", action="store_true",
|
||||
help="不补 HALF 款式(部分 record 失败的款);默认会补,只跑失败的 record_id")
|
||||
ap.add_argument("--include-self-operated", action="store_true",
|
||||
help="同时补自营达人(默认只补合作达人)")
|
||||
ap.add_argument("--self-operated-only", action="store_true",
|
||||
help="只补自营达人")
|
||||
args = ap.parse_args()
|
||||
include_half = not args.no_include_half
|
||||
|
||||
keys = [k.strip() for k in args.platform.split(",") if k.strip()]
|
||||
for k in keys:
|
||||
if k not in SCRIPTS:
|
||||
print(f"[ERROR] 未知平台: {k} 可选: {','.join(SCRIPTS.keys())}")
|
||||
return 1
|
||||
|
||||
forced: list[int] = []
|
||||
if args.style:
|
||||
for s in args.style.split(","):
|
||||
s = s.strip()
|
||||
if not s:
|
||||
continue
|
||||
try:
|
||||
idx = int(s)
|
||||
except ValueError:
|
||||
print(f"[ERROR] --style 解析失败: {s}")
|
||||
return 1
|
||||
forced.append(idx)
|
||||
|
||||
# 决定要跑哪些轮: 合作达人 + 自营达人
|
||||
rounds: list[tuple[bool, str]] = [] # (self_operated, label)
|
||||
if not args.self_operated_only:
|
||||
rounds.append((False, "合作达人"))
|
||||
if args.include_self_operated or args.self_operated_only:
|
||||
rounds.append((True, "自营达人"))
|
||||
|
||||
any_failed = False
|
||||
for self_op, round_label in rounds:
|
||||
styles = load_current_styles(self_op)
|
||||
if not styles:
|
||||
log(f"[ERROR] {round_label} 当前款式缓存为空，无法安全补跑")
|
||||
any_failed = True
|
||||
continue
|
||||
styles_by_index = {int(style["index"]): style for style in styles}
|
||||
invalid_forced = [idx for idx in forced if idx not in styles_by_index]
|
||||
if invalid_forced:
|
||||
log(f"[ERROR] {round_label} 不存在款式编号: {invalid_forced}")
|
||||
any_failed = True
|
||||
|
||||
print(f"{'=' * 78}")
|
||||
print(f" Retry Failed Styles — {round_label}")
|
||||
print(f" 平台: {','.join(SCRIPTS[k]['name'] for k in keys)}")
|
||||
print(f" stale-hours: {args.stale_hours} max-attempts: {args.max_attempts}")
|
||||
print(f" include-half: {include_half} dry-run: {args.dry_run}")
|
||||
if forced:
|
||||
print(f" 强制指定: {forced}")
|
||||
print(f"{'=' * 78}")
|
||||
|
||||
grand_total = grand_ok = grand_fail = 0
|
||||
grand_failed_list: list[tuple[str, int]] = []
|
||||
|
||||
for k in keys:
|
||||
info = SCRIPTS[k]
|
||||
if forced:
|
||||
targets = {idx: {"reason": "强制", "failed_rids": []}
|
||||
for idx in forced if idx in styles_by_index}
|
||||
else:
|
||||
targets = find_failed(
|
||||
k, args.stale_hours, include_half,
|
||||
self_operated=self_op, styles=styles,
|
||||
)
|
||||
log(f"[{k}{'-self' if self_op else ''}] {info['name']} 待补: {len(targets)} 个 -> {sorted(targets.keys())}")
|
||||
if not targets:
|
||||
continue
|
||||
if args.dry_run:
|
||||
for idx in sorted(targets.keys()):
|
||||
info_i = targets[idx]
|
||||
n = len(info_i["failed_rids"])
|
||||
detail = f", {n} 条 record" if n else ""
|
||||
log(f" [DRY-RUN] 款式 {idx:>2} ({info_i['reason']}{detail})")
|
||||
grand_total += len(targets)
|
||||
continue
|
||||
for idx in sorted(targets.keys()):
|
||||
info_i = targets[idx]
|
||||
grand_total += 1
|
||||
ok = run_one_style(k, idx, args.max_attempts,
|
||||
self_operated=self_op,
|
||||
record_ids=info_i["failed_rids"] or None,
|
||||
style=styles_by_index[idx])
|
||||
if ok:
|
||||
grand_ok += 1
|
||||
else:
|
||||
grand_fail += 1
|
||||
grand_failed_list.append((k, idx))
|
||||
any_failed = True
|
||||
|
||||
print(f"\n{'=' * 78}")
|
||||
print(f" {round_label} 补跑汇总: 总 {grand_total} 成功 {grand_ok} 失败 {grand_fail}")
|
||||
if grand_failed_list:
|
||||
print(f" 仍失败的:")
|
||||
for k, idx in grand_failed_list:
|
||||
print(f" - [{k}] 款式 {idx}")
|
||||
print(f"{'=' * 78}\n")
|
||||
|
||||
return 1 if any_failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\n[中断]", file=sys.stderr)
|
||||
sys.exit(130)
|
||||
+342
@@ -0,0 +1,342 @@
|
||||
-- 数据库：gyxx_super_data
|
||||
-- 表结构：cmt_styles / cmt_creators / cmt_style_creators / cmt_notes / cmt_comments
|
||||
-- 适用于 PostgreSQL 14+
|
||||
|
||||
-- 如果数据库还没创建，取消下面一行的注释并执行：
|
||||
-- CREATE DATABASE gyxx_super_data WITH ENCODING = 'UTF8' LC_COLLATE = 'zh_CN.UTF-8' LC_CTYPE = 'zh_CN.UTF-8';
|
||||
|
||||
\c gyxx_super_data;
|
||||
|
||||
-- 款式表
|
||||
CREATE TABLE IF NOT EXISTS cmt_styles (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
style_category VARCHAR(64) DEFAULT '',
|
||||
brand VARCHAR(128) DEFAULT '',
|
||||
erp_style_codes TEXT[] DEFAULT '{}',
|
||||
platform_product_ids JSONB DEFAULT '{}'::jsonb,
|
||||
product_type VARCHAR(128) DEFAULT '',
|
||||
dimensions VARCHAR(255) DEFAULT '',
|
||||
colors TEXT[] DEFAULT '{}',
|
||||
material TEXT DEFAULT '',
|
||||
capacity VARCHAR(128) DEFAULT '',
|
||||
weight VARCHAR(128) DEFAULT '',
|
||||
compatible_devices TEXT DEFAULT '',
|
||||
features TEXT DEFAULT '',
|
||||
selling_points TEXT DEFAULT '',
|
||||
product_source_urls TEXT[] DEFAULT '{}',
|
||||
product_research_status VARCHAR(32) DEFAULT 'pending',
|
||||
product_researched_at TIMESTAMPTZ,
|
||||
platform VARCHAR(64) DEFAULT '',
|
||||
feishu_base_url TEXT DEFAULT '',
|
||||
feishu_base_token VARCHAR(64) DEFAULT '',
|
||||
feishu_table_id VARCHAR(64) DEFAULT '',
|
||||
feishu_view_id VARCHAR(64) DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
COMMENT ON TABLE cmt_styles IS '款式表';
|
||||
COMMENT ON COLUMN cmt_styles.name IS '款式名称';
|
||||
COMMENT ON COLUMN cmt_styles.style_category IS '款式风格';
|
||||
COMMENT ON COLUMN cmt_styles.platform IS '平台分组，如天猫';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_styles_platform ON cmt_styles (platform);
|
||||
|
||||
-- 博主/达人表
|
||||
CREATE TABLE IF NOT EXISTS cmt_creators (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
platform VARCHAR(32) DEFAULT '', -- 主平台: xiaohongshu/douyin/bilibili
|
||||
platform_id VARCHAR(128) DEFAULT '', -- 平台达人ID (B站UID/抖音sec_uid等)
|
||||
follower_count VARCHAR(64) DEFAULT '', -- 达人粉丝数(原始文本如3.7万/12.4w)
|
||||
follower_count_num BIGINT DEFAULT NULL, -- 达人粉丝数(数字解析后,3.7万=37000)
|
||||
homepage_url TEXT DEFAULT '', -- 达人主页链接
|
||||
wechat VARCHAR(128) DEFAULT '', -- 微信号
|
||||
account_type VARCHAR(64) DEFAULT '', -- 达人类型: 数码/vlog/好物/穿搭/摄影等
|
||||
creator_level VARCHAR(32) DEFAULT '', -- 达人层级: 头部/腰部/KOC/素人
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT cmt_creators_name_platform_key UNIQUE (name, platform)
|
||||
);
|
||||
COMMENT ON TABLE cmt_creators IS '博主/达人表';
|
||||
COMMENT ON COLUMN cmt_creators.name IS '博主/达人名称';
|
||||
COMMENT ON COLUMN cmt_creators.platform IS '主平台: xiaohongshu/douyin/bilibili';
|
||||
COMMENT ON COLUMN cmt_creators.platform_id IS '平台达人ID (B站UID/抖音sec_uid等)';
|
||||
COMMENT ON COLUMN cmt_creators.follower_count IS '达人粉丝数(原始文本如3.7万/12.4w)';
|
||||
COMMENT ON COLUMN cmt_creators.follower_count_num IS '达人粉丝数(数字解析后,3.7万=37000)';
|
||||
COMMENT ON COLUMN cmt_creators.homepage_url IS '达人主页链接';
|
||||
COMMENT ON COLUMN cmt_creators.wechat IS '微信号';
|
||||
COMMENT ON COLUMN cmt_creators.account_type IS '达人类型: 数码/vlog/好物/穿搭/摄影等';
|
||||
COMMENT ON COLUMN cmt_creators.creator_level IS '达人层级: 头部/腰部/KOC/素人';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_creators_name ON cmt_creators (name);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_creators_platform ON cmt_creators (platform);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_creators_platform_id ON cmt_creators (platform_id);
|
||||
|
||||
-- 款式-博主关系表（多对多）
|
||||
CREATE TABLE IF NOT EXISTS cmt_style_creators (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
style_id BIGINT NOT NULL,
|
||||
creator_id BIGINT NOT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE (style_id, creator_id),
|
||||
CONSTRAINT fk_cmt_sc_style FOREIGN KEY (style_id) REFERENCES cmt_styles (id) ON DELETE CASCADE,
|
||||
CONSTRAINT fk_cmt_sc_creator FOREIGN KEY (creator_id) REFERENCES cmt_creators (id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE cmt_style_creators IS '款式-博主关系表';
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_style_creators_creator ON cmt_style_creators (creator_id);
|
||||
|
||||
-- 笔记表
|
||||
CREATE TABLE IF NOT EXISTS cmt_notes (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
style_id BIGINT NOT NULL,
|
||||
creator_id BIGINT NOT NULL,
|
||||
platform VARCHAR(32) NOT NULL,
|
||||
title VARCHAR(512) DEFAULT '',
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
feishu_record_id VARCHAR(64) DEFAULT '',
|
||||
view_count BIGINT DEFAULT NULL, -- 曝光量 / 播放量
|
||||
like_count BIGINT DEFAULT NULL,
|
||||
collect_count BIGINT DEFAULT NULL, -- 小红书 / 抖音收藏
|
||||
favorite_count BIGINT DEFAULT NULL, -- B 站收藏
|
||||
comment_count BIGINT DEFAULT NULL,
|
||||
share_count BIGINT DEFAULT NULL,
|
||||
scraped_at TIMESTAMPTZ DEFAULT NULL,
|
||||
self_operated BOOLEAN DEFAULT FALSE, -- TRUE=自营达人笔记, FALSE=合作达人笔记
|
||||
publish_time TIMESTAMPTZ DEFAULT NULL, -- 笔记发布时间(从飞书 publish_time 字段同步)
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT fk_cmt_notes_style FOREIGN KEY (style_id) REFERENCES cmt_styles (id) ON DELETE CASCADE,
|
||||
CONSTRAINT fk_cmt_notes_creator FOREIGN KEY (creator_id) REFERENCES cmt_creators (id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON COLUMN cmt_notes.view_count IS '曝光量 / 播放量 (来自 yingxiaoyunying 每日采集)';
|
||||
COMMENT ON COLUMN cmt_notes.collect_count IS '收藏量: 小红书/抖音 (来自 yingxiaoyunying 每日采集)';
|
||||
COMMENT ON COLUMN cmt_notes.favorite_count IS '收藏量: B 站 (来自 comment-data-collector)';
|
||||
COMMENT ON TABLE cmt_notes IS '笔记表';
|
||||
COMMENT ON COLUMN cmt_notes.platform IS '平台：xiaohongshu / douyin / bilibili';
|
||||
COMMENT ON COLUMN cmt_notes.url IS '笔记链接';
|
||||
COMMENT ON COLUMN cmt_notes.scraped_at IS '最近一次抓取时间';
|
||||
COMMENT ON COLUMN cmt_notes.self_operated IS 'TRUE=自营达人笔记, FALSE=合作达人笔记';
|
||||
COMMENT ON COLUMN cmt_notes.publish_time IS '笔记发布时间(从飞书 publish_time 字段同步)';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_notes_style ON cmt_notes (style_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_notes_creator ON cmt_notes (creator_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_notes_platform ON cmt_notes (platform);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_notes_scraped_at ON cmt_notes (scraped_at);
|
||||
|
||||
-- 评论表
|
||||
CREATE TABLE IF NOT EXISTS cmt_comments (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
note_id BIGINT NOT NULL,
|
||||
platform VARCHAR(32) NOT NULL,
|
||||
level VARCHAR(16) NOT NULL,
|
||||
platform_comment_id VARCHAR(64) DEFAULT '',
|
||||
parent_comment_id VARCHAR(64) DEFAULT '',
|
||||
reply_to_comment_id VARCHAR(64) DEFAULT '',
|
||||
user_id VARCHAR(128) DEFAULT '',
|
||||
nickname VARCHAR(255) DEFAULT '',
|
||||
content TEXT DEFAULT NULL,
|
||||
like_count BIGINT DEFAULT 0,
|
||||
reply_count BIGINT DEFAULT 0,
|
||||
ip_location VARCHAR(128) DEFAULT '',
|
||||
comment_created_at TIMESTAMPTZ DEFAULT NULL,
|
||||
raw_create_time VARCHAR(64) DEFAULT '',
|
||||
scraped_at TIMESTAMPTZ DEFAULT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE (note_id, platform_comment_id),
|
||||
CONSTRAINT fk_cmt_comments_note FOREIGN KEY (note_id) REFERENCES cmt_notes (id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE cmt_comments IS '评论表';
|
||||
COMMENT ON COLUMN cmt_comments.level IS '层级：comment 一级评论 / reply 回复';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_comments_note_id ON cmt_comments (note_id);
|
||||
|
||||
-- ============================================================
|
||||
-- 合作记录表 (达人 + 款式维度)
|
||||
-- ============================================================
|
||||
-- 每条记录 = 一个达人在一个款式下的一次合作
|
||||
-- 飞书款式合作表的每一行 → 此表一条记录
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cmt_cooperations (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
style_id BIGINT NOT NULL REFERENCES cmt_styles(id) ON DELETE CASCADE,
|
||||
creator_id BIGINT NOT NULL REFERENCES cmt_creators(id) ON DELETE CASCADE,
|
||||
feishu_record_id VARCHAR(64) DEFAULT '', -- 飞书行 record_id, 用于回写
|
||||
-- 合作基本信息
|
||||
cooperation_date DATE DEFAULT NULL, -- 合作日期 (飞书"制单日期")
|
||||
cooperation_cost NUMERIC(12,2) DEFAULT NULL, -- 合作花费 (飞书"合作费用")
|
||||
ad_spend NUMERIC(12,2) DEFAULT NULL, -- 投流金额 (飞书"投流金额")
|
||||
-- 内容信息
|
||||
content_direction VARCHAR(255) DEFAULT '', -- 内容方向 (飞书"发布笔记内容方向")
|
||||
content_format VARCHAR(64) DEFAULT '', -- 内容形式: 单品种草/测评合集/场景植入/开箱Vlog/穿搭变装等
|
||||
note_title VARCHAR(512) DEFAULT '', -- 发布笔记标题
|
||||
note_url TEXT DEFAULT '', -- 发布笔记链接
|
||||
publish_time TIMESTAMPTZ DEFAULT NULL, -- 发布时间
|
||||
-- 数据表现
|
||||
exposure_count BIGINT DEFAULT NULL, -- 曝光量 (飞书"曝光量(自动抓取)")
|
||||
engagement_count VARCHAR(64) DEFAULT '', -- 互动赞藏数 (原始文本如"68.5万")
|
||||
engagement_count_num BIGINT DEFAULT NULL, -- 互动赞藏数(数字解析后)
|
||||
data_performance VARCHAR(255) DEFAULT '', -- 数据表现 (逗号分隔: 曝光限流,数据良好)
|
||||
cpm NUMERIC(10,4) DEFAULT NULL, -- CPM (飞书公式字段)
|
||||
-- 物流/财务
|
||||
tracking_number VARCHAR(128) DEFAULT '', -- 快递单号
|
||||
is_paid BOOLEAN DEFAULT FALSE, -- 是否结款
|
||||
is_new_direction BOOLEAN DEFAULT FALSE, -- 是否新方向验证
|
||||
-- 电商效果 (飞书暂无,预留)
|
||||
shop_uv BIGINT DEFAULT NULL, -- 进店UV
|
||||
transaction_amount NUMERIC(12,2) DEFAULT NULL, -- 成交金额(元)
|
||||
roi NUMERIC(10,4) DEFAULT NULL, -- ROI (成交金额/花费)
|
||||
cpe NUMERIC(10,4) DEFAULT NULL, -- CPE (花费/互动量)
|
||||
engagement_rate NUMERIC(8,4) DEFAULT NULL, -- 互动率 (互动量/曝光量)
|
||||
-- 评论分析 (飞书暂无,预留)
|
||||
brand_mention_rate NUMERIC(8,4) DEFAULT NULL, -- 评论品牌提及率
|
||||
risk_flag VARCHAR(255) DEFAULT '', -- 差评或舆情风险
|
||||
--
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT uniq_cooperation_feishu_record UNIQUE (feishu_record_id)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE cmt_cooperations IS '合作记录表(每个达人+款式的一次合作)';
|
||||
COMMENT ON COLUMN cmt_cooperations.cooperation_date IS '合作日期(飞书制单日期)';
|
||||
COMMENT ON COLUMN cmt_cooperations.cooperation_cost IS '合作花费(元)';
|
||||
COMMENT ON COLUMN cmt_cooperations.ad_spend IS '投流金额(元)';
|
||||
COMMENT ON COLUMN cmt_cooperations.content_direction IS '发布笔记内容方向(飞书原始值)';
|
||||
COMMENT ON COLUMN cmt_cooperations.content_format IS '内容形式: 单品种草/测评合集/场景植入/开箱Vlog等';
|
||||
COMMENT ON COLUMN cmt_cooperations.exposure_count IS '曝光量(飞书自动抓取)';
|
||||
COMMENT ON COLUMN cmt_cooperations.engagement_count IS '互动赞藏数(原始文本如68.5万)';
|
||||
COMMENT ON COLUMN cmt_cooperations.data_performance IS '数据表现(逗号分隔: 曝光限流,数据良好,表现优秀等)';
|
||||
COMMENT ON COLUMN cmt_cooperations.shop_uv IS '进店UV(预留,飞书暂无)';
|
||||
COMMENT ON COLUMN cmt_cooperations.transaction_amount IS '成交金额(元,预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.roi IS 'ROI=成交金额/花费(预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.cpe IS 'CPE=花费/互动量(预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.engagement_rate IS '互动率=互动量/曝光量(预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.brand_mention_rate IS '评论品牌提及率(预留)';
|
||||
COMMENT ON COLUMN cmt_cooperations.risk_flag IS '差评或舆情风险(预留)';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_cooperations_style ON cmt_cooperations (style_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_cooperations_creator ON cmt_cooperations (creator_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_cooperations_date ON cmt_cooperations (cooperation_date);
|
||||
|
||||
|
||||
-- ============================================================
|
||||
-- 周笔记汇总表 (款式 + 周维度)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS cmt_weekly_summary (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
style_id BIGINT NOT NULL REFERENCES cmt_styles(id) ON DELETE CASCADE,
|
||||
week_start DATE NOT NULL,
|
||||
week_end DATE NOT NULL,
|
||||
time_label VARCHAR(32) NOT NULL,
|
||||
note_count INTEGER DEFAULT 0,
|
||||
total_comments INTEGER DEFAULT 0,
|
||||
total_views BIGINT,
|
||||
total_engagement BIGINT,
|
||||
doc_url TEXT,
|
||||
doc_title VARCHAR(200),
|
||||
summary_chars INTEGER,
|
||||
status VARCHAR(32),
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT uniq_weekly_style_week UNIQUE (style_id, week_start)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_weekly_summary_week ON cmt_weekly_summary (week_start);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_weekly_summary_style ON cmt_weekly_summary (style_id);
|
||||
COMMENT ON TABLE cmt_weekly_summary IS '周笔记汇总表(款式+周维度,LLM 分析报告飞书文档链接)';
|
||||
COMMENT ON COLUMN cmt_weekly_summary.time_label IS '周时间范围标签 如 2026-06-29~07-05';
|
||||
COMMENT ON COLUMN cmt_weekly_summary.doc_url IS '飞书文档链接(每周笔记汇总报告)';
|
||||
COMMENT ON COLUMN cmt_weekly_summary.status IS 'ok/no_reports/no_notes/exception';
|
||||
|
||||
-- ============================================================
|
||||
-- 月笔记汇总表 (款式 + 月维度,横向对比分析)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS cmt_monthly_summary (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
style_id BIGINT NOT NULL REFERENCES cmt_styles(id) ON DELETE CASCADE,
|
||||
month_start DATE NOT NULL,
|
||||
month_end DATE NOT NULL,
|
||||
time_label VARCHAR(32) NOT NULL,
|
||||
note_count INTEGER DEFAULT 0,
|
||||
total_comments INTEGER DEFAULT 0,
|
||||
total_views BIGINT,
|
||||
total_engagement BIGINT,
|
||||
doc_url TEXT,
|
||||
doc_title VARCHAR(200),
|
||||
summary_chars INTEGER,
|
||||
status VARCHAR(32),
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT uniq_monthly_style_month UNIQUE (style_id, month_start)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_monthly_summary_month ON cmt_monthly_summary (month_start);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_monthly_summary_style ON cmt_monthly_summary (style_id);
|
||||
COMMENT ON TABLE cmt_monthly_summary IS '月笔记汇总表(款式+月维度,LLM 横向对比分析报告)';
|
||||
COMMENT ON COLUMN cmt_monthly_summary.time_label IS '月时间范围标签 如 2026-06-01~06-30';
|
||||
COMMENT ON COLUMN cmt_monthly_summary.doc_url IS '飞书文档链接(月度汇总报告)';
|
||||
|
||||
-- ============================================================
|
||||
-- 达人月度分析报告表 (每月1号自动生成)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS cmt_creator_report (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
month_start DATE NOT NULL,
|
||||
month_end DATE NOT NULL,
|
||||
time_label VARCHAR(32) NOT NULL,
|
||||
total_cooperations INTEGER DEFAULT 0,
|
||||
total_creators INTEGER DEFAULT 0,
|
||||
total_styles INTEGER DEFAULT 0,
|
||||
total_cost NUMERIC(14,2),
|
||||
total_exposure BIGINT,
|
||||
total_engagement BIGINT,
|
||||
avg_cpe NUMERIC(10,2),
|
||||
level_distribution JSONB DEFAULT '{}',
|
||||
platform_distribution JSONB DEFAULT '{}',
|
||||
score_distribution JSONB DEFAULT '{}',
|
||||
doc_url TEXT,
|
||||
doc_title VARCHAR(200),
|
||||
report_chars INTEGER,
|
||||
status VARCHAR(32),
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT uniq_creator_report_month UNIQUE (month_start)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_creator_report_month ON cmt_creator_report (month_start);
|
||||
|
||||
COMMENT ON TABLE cmt_creator_report IS '达人月度分析报告表(每月1号自动生成,全量达人筛选与报价分析)';
|
||||
COMMENT ON COLUMN cmt_creator_report.time_label IS '月时间范围标签 如 2026-06-01~06-30';
|
||||
COMMENT ON COLUMN cmt_creator_report.level_distribution IS '层级分布快照 JSON {"头部":N,"腰部":N,...}';
|
||||
COMMENT ON COLUMN cmt_creator_report.platform_distribution IS '平台分布快照 JSON {"douyin":N,...}';
|
||||
COMMENT ON COLUMN cmt_creator_report.score_distribution IS '评分分布快照 JSON {"S":N,"A":N,...}';
|
||||
COMMENT ON COLUMN cmt_creator_report.doc_url IS '飞书文档链接(达人分析报告)';
|
||||
COMMENT ON COLUMN cmt_creator_report.status IS 'ok/doc_failed/exception';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_comments_platform ON cmt_comments (platform);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_comments_level ON cmt_comments (level);
|
||||
CREATE INDEX IF NOT EXISTS idx_cmt_comments_comment_created_at ON cmt_comments (comment_created_at);
|
||||
|
||||
-- 自动更新 updated_at 的触发器函数
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- 为需要自动更新 updated_at 的表创建触发器
|
||||
DO $$
|
||||
DECLARE
|
||||
tbl TEXT;
|
||||
BEGIN
|
||||
FOR tbl IN SELECT unnest(ARRAY['cmt_styles', 'cmt_creators', 'cmt_style_creators', 'cmt_notes', 'cmt_comments', 'cmt_cooperations', 'cmt_weekly_summary', 'cmt_monthly_summary', 'cmt_creator_report'])
|
||||
LOOP
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_trigger WHERE tgname = 'trg_' || tbl || '_updated_at'
|
||||
) THEN
|
||||
EXECUTE format('CREATE TRIGGER trg_%I_updated_at BEFORE UPDATE ON %I FOR EACH ROW EXECUTE FUNCTION update_updated_at_column()', tbl, tbl);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
@@ -0,0 +1,362 @@
|
||||
"""Style-level rollup analyzer for the Feishu doc backfill.
|
||||
|
||||
Aggregates metrics + keyword distributions across all notes of one style,
|
||||
then calls Hermes for a cross-note synthesis analysis that becomes the
|
||||
"💼 款式整体分析" section in the doc.
|
||||
|
||||
Sibling of analyze_note.py — same import surface (call_hermes_analyzer,
|
||||
ANALYSIS_SYSTEM_PROMPT, code_filter_comments) but operates at style scope.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
# Path setup
|
||||
_TOOLS_DIR = PATHS.tools_root
|
||||
_PROJECT_ROOT = PATHS.module_root
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.analyze_comments import (
|
||||
ANALYSIS_SYSTEM_PROMPT,
|
||||
call_hermes_analyzer,
|
||||
)
|
||||
|
||||
|
||||
CATEGORY_LABELS = {
|
||||
"purchase_intent": "购买意向",
|
||||
"positive_feature": "正向功能反馈",
|
||||
"negative_feature": "负向功能反馈",
|
||||
"user_scenario": "用户场景",
|
||||
"aesthetic": "审美与风格",
|
||||
"brand_emotion": "品牌情感",
|
||||
}
|
||||
|
||||
|
||||
# 5-section deep prompt at style scope (mirrors SINGLE_NOTE_DEEP_PROMPT_TEMPLATE
|
||||
# but across all notes of one style).
|
||||
STYLE_ROLLUP_DEEP_PROMPT = """你是一位品牌内容分析师。本次任务是把同一款式的所有笔记的标题、平台、点赞/收藏/评论/分享数据和有效评论内容，汇总成一份**款式整体分析报告**，用于投放团队对这款产品的内容策略做一次通盘复盘。
|
||||
|
||||
输出结构（严格 5 大节，总长 800-1200 字）：
|
||||
|
||||
### 一、执行摘要（不超过 150 字）
|
||||
用一段话点明这款产品的整体定位和投放现状：覆盖多少篇笔记、整体互动表现、主要用户反馈倾向、是否存在明显短板。
|
||||
|
||||
### 二、数据表现
|
||||
- 该款式总笔记数 / 有效评论总数
|
||||
- 总点赞 / 总收藏 / 总评论 / 总分享（如曝光量汇总可用则一并列出）
|
||||
- 综合互动率（基于汇总数据）
|
||||
- 跨笔记表现差异：高/低表现笔记的标题特征（如"通勤""扫街""旅行"等场景标签）
|
||||
|
||||
### 三、受众洞察
|
||||
基于所有有效评论的合并视角：
|
||||
- 正向反馈汇总（被夸的核心卖点 / 场景 / 审美点）
|
||||
- 负向问题/风险点（被吐槽的尺寸 / 材质 / 价格 / 功能）
|
||||
- 购买意向强弱（直接询问价格/链接 vs 围观闲聊）
|
||||
- 典型使用场景（被反复提及的目标人群和场景）
|
||||
- 跨平台差异（如果 bilibili / douyin / xiaohongshu 反馈侧重不同，请分别说明）
|
||||
|
||||
### 四、内容策略
|
||||
基于跨笔记样本，提炼以下三类规律：
|
||||
- 哪种标题/封面/切入点最受欢迎
|
||||
- 哪种内容结构（测评 / 街拍 / Vlog / 教程）种草效率最高
|
||||
- 评论区的高频问题/梗，可作为后续内容选题/卖点提炼的素材
|
||||
|
||||
### 五、下一步（按 P0/P1/P2 排序，最多 5 条）
|
||||
- P0（立即）：【问题】→【行动】→【责任方】
|
||||
- P1（本周）：...
|
||||
- P2（下月）：...（如无可省略）
|
||||
|
||||
## 硬性约束
|
||||
|
||||
1. 引用评论必须直接引述原文（用「」包裹），禁止使用"#N""第N条"等编号引用。
|
||||
2. 禁止"根据数据/数据显示/数据来源/通过 XX 可知"等出处类表述。
|
||||
3. 若有效评论 < 30 条，必须在执行摘要后注明"样本量较小，结论仅供参考"。
|
||||
4. P0/P1/P2 必须明确标注【问题】→【行动】→【责任方】三段式。
|
||||
5. 若款式涉及多品牌联合推广，请仅聚焦我方品牌/产品相关反馈。
|
||||
6. 不要重复列出指标定义，专注解读与行动。
|
||||
"""
|
||||
|
||||
|
||||
def _to_number(value: Any) -> float:
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return 0.0
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def safe_div(num: Any, denom: Any) -> float:
|
||||
try:
|
||||
n = float(num)
|
||||
d = float(denom)
|
||||
if d == 0:
|
||||
return 0.0
|
||||
return n / d
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Aggregation
|
||||
# ---------------------------------------------------------------------------
|
||||
def aggregate_style_metrics(notes: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Sum raw engagement counts across all notes of a style.
|
||||
|
||||
Returns:
|
||||
{
|
||||
"note_count": int,
|
||||
"total_likes": int, "total_favorites": int, "total_comments": int,
|
||||
"total_shares": int, "total_views": int,
|
||||
"engagement_rate": float, # (likes+comments+favorites+shares) / views
|
||||
"platform_breakdown": {"bilibili": n, "douyin": n, "xiaohongshu": n},
|
||||
"creator_breakdown": {creator_name: n}, # top creators by note count
|
||||
"notes_meta": [{"id", "platform", "title", "like_count", "comment_count"}, ...]
|
||||
}
|
||||
"""
|
||||
totals = {"likes": 0.0, "favorites": 0.0, "comments": 0.0, "shares": 0.0, "views": 0.0}
|
||||
platforms = {"bilibili": 0, "douyin": 0, "xiaohongshu": 0}
|
||||
creators: dict[str, int] = {}
|
||||
notes_meta: list[dict[str, Any]] = []
|
||||
|
||||
for n in notes:
|
||||
likes = _to_number(n.get("like_count"))
|
||||
favs = _to_number(n.get("favorite_count"))
|
||||
cmts = _to_number(n.get("comment_count")) or _to_number(n.get("comment_count_total"))
|
||||
shares = _to_number(n.get("share_count"))
|
||||
views = _to_number(n.get("view_count"))
|
||||
totals["likes"] += likes
|
||||
totals["favorites"] += favs
|
||||
totals["comments"] += cmts
|
||||
totals["shares"] += shares
|
||||
totals["views"] += views
|
||||
|
||||
plat = n.get("platform") or "unknown"
|
||||
platforms[plat] = platforms.get(plat, 0) + 1
|
||||
|
||||
creator = n.get("creator_name") or "未指定"
|
||||
creators[creator] = creators.get(creator, 0) + 1
|
||||
|
||||
notes_meta.append({
|
||||
"id": n.get("id"),
|
||||
"platform": plat,
|
||||
"title": (n.get("title") or "")[:60],
|
||||
"like_count": int(likes) if likes else 0,
|
||||
"comment_count": int(cmts) if cmts else 0,
|
||||
"creator_name": creator,
|
||||
})
|
||||
|
||||
engagement = safe_div(
|
||||
totals["likes"] + totals["comments"] + totals["favorites"] + totals["shares"],
|
||||
totals["views"],
|
||||
)
|
||||
|
||||
# Top creators by note count
|
||||
top_creators = dict(sorted(creators.items(), key=lambda kv: -kv[1])[:5])
|
||||
|
||||
return {
|
||||
"note_count": len(notes),
|
||||
"total_likes": int(totals["likes"]),
|
||||
"total_favorites": int(totals["favorites"]),
|
||||
"total_comments": int(totals["comments"]),
|
||||
"total_shares": int(totals["shares"]),
|
||||
"total_views": int(totals["views"]),
|
||||
"engagement_rate": engagement,
|
||||
"platform_breakdown": platforms,
|
||||
"creator_breakdown": top_creators,
|
||||
"notes_meta": notes_meta,
|
||||
}
|
||||
|
||||
|
||||
def merge_style_distribution(per_note_distributions: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Element-wise sum of overall + relevant dicts across notes."""
|
||||
overall: dict[str, int] = {k: 0 for k in CATEGORY_LABELS}
|
||||
relevant: dict[str, int] = {k: 0 for k in CATEGORY_LABELS}
|
||||
relevant_total = 0
|
||||
for d in per_note_distributions:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
for k, v in (d.get("overall") or {}).items():
|
||||
overall[k] = overall.get(k, 0) + int(v or 0)
|
||||
for k, v in (d.get("relevant") or {}).items():
|
||||
relevant[k] = relevant.get(k, 0) + int(v or 0)
|
||||
relevant_total += int(d.get("relevant_total") or 0)
|
||||
return {"overall": overall, "relevant": relevant, "relevant_total": relevant_total}
|
||||
|
||||
|
||||
def _pct(v: float) -> str:
|
||||
return f"{v * 100:.2f}%" if v else "—"
|
||||
|
||||
|
||||
def _num(v: float) -> str:
|
||||
try:
|
||||
return f"{int(v):,}"
|
||||
except (TypeError, ValueError):
|
||||
return "—"
|
||||
|
||||
|
||||
def _format_distribution_md(d: dict[str, Any], total_filtered: int) -> str:
|
||||
"""Mirror analyze_note._keyword_distribution_markdown."""
|
||||
overall = d.get("overall", {})
|
||||
relevant = d.get("relevant", {})
|
||||
relevant_total = d.get("relevant_total", 0)
|
||||
if not overall:
|
||||
return "（无关键词分布数据）"
|
||||
|
||||
def fmt(m: dict[str, int]) -> str:
|
||||
return "\n".join(f"- {CATEGORY_LABELS[k]}：{m.get(k, 0)} 条" for k in CATEGORY_LABELS)
|
||||
|
||||
return "\n".join([
|
||||
f"### 全评论（{total_filtered} 条）",
|
||||
fmt(overall),
|
||||
"",
|
||||
f"### 我方产品相关（{relevant_total} 条）",
|
||||
fmt(relevant) if relevant_total else "_（无评论明确提及我方产品/品牌）_",
|
||||
])
|
||||
|
||||
|
||||
def _format_metrics_md(agg: dict[str, Any]) -> str:
|
||||
plat = agg["platform_breakdown"]
|
||||
plat_str = " / ".join(f"{k} {v}" for k, v in plat.items() if v) or "—"
|
||||
creator_str = " / ".join(f"{k} {v}" for k, v in agg["creator_breakdown"].items()) or "—"
|
||||
|
||||
return "\n".join([
|
||||
"| 维度 | 指标 | 数值 |",
|
||||
"| --- | --- | --- |",
|
||||
f"| 规模 | 笔记总数 | {agg['note_count']} |",
|
||||
f"| 规模 | 平台分布 | {plat_str} |",
|
||||
f"| 规模 | 主要达人 | {creator_str} |",
|
||||
f"| 基础 | 总点赞 | {_num(agg['total_likes'])} |",
|
||||
f"| 基础 | 总收藏 | {_num(agg['total_favorites'])} |",
|
||||
f"| 基础 | 总评论 | {_num(agg['total_comments'])} |",
|
||||
f"| 基础 | 总分享 | {_num(agg['total_shares'])} |",
|
||||
f"| 传播 | 总曝光量 | {_num(agg['total_views']) if agg['total_views'] else '未采集'} |",
|
||||
f"| 互动率 | 综合互动率 | {_pct(agg['engagement_rate'])} |",
|
||||
])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt building
|
||||
# ---------------------------------------------------------------------------
|
||||
def sample_representative_comments(
|
||||
per_note_comments: list[list[str]],
|
||||
per_note_meta: list[dict[str, Any]],
|
||||
*,
|
||||
max_total: int = 30,
|
||||
per_note_cap: int = 2,
|
||||
) -> list[str]:
|
||||
"""Pick up to `max_total` filtered comments across notes.
|
||||
|
||||
Tries to take `per_note_cap` from each note, prioritizing longer comments
|
||||
(proxy for substance). If still under `max_total`, fills with remaining
|
||||
longer comments.
|
||||
"""
|
||||
all_comments: list[tuple[str, dict[str, Any]]] = []
|
||||
for comments, meta in zip(per_note_comments, per_note_meta):
|
||||
# Sort by length desc; take up to per_note_cap
|
||||
sorted_c = sorted(comments, key=len, reverse=True)[:per_note_cap]
|
||||
for c in sorted_c:
|
||||
all_comments.append((c, meta))
|
||||
|
||||
# Final sort: longer first, then by note id asc for stability
|
||||
all_comments.sort(key=lambda cm: (-len(cm[0]), cm[1].get("id") or 0))
|
||||
return [c for c, _ in all_comments[:max_total]]
|
||||
|
||||
|
||||
def build_style_rollup_prompt(
|
||||
style_name: str,
|
||||
brand: str,
|
||||
aggregated: dict[str, Any],
|
||||
merged_distribution: dict[str, Any],
|
||||
sample_comments: list[str],
|
||||
note_summaries: list[str],
|
||||
) -> str:
|
||||
"""Build user_content for the style-level LLM call.
|
||||
|
||||
`note_summaries` are 1-2 line "title | platform | likes | comments" snippets
|
||||
to give the LLM a sense of which notes exist.
|
||||
"""
|
||||
target_label = brand or style_name
|
||||
lines: list[str] = [
|
||||
f"本款式「{style_name}」共采集 {aggregated['note_count']} 篇笔记，跨 {sum(1 for v in aggregated['platform_breakdown'].values() if v)} 个平台。",
|
||||
f"以下是基于这 {aggregated['note_count']} 篇笔记的合并数据 + 代表性评论样本，请输出款式整体分析报告。",
|
||||
"",
|
||||
"## 款式汇总数据",
|
||||
_format_metrics_md(aggregated),
|
||||
"",
|
||||
"## 关键词分布（合并所有笔记）",
|
||||
_format_distribution_md(
|
||||
merged_distribution,
|
||||
sum((m or {}).get("overall", {}).get(k, 0)
|
||||
for m in [merged_distribution]
|
||||
for k in CATEGORY_LABELS),
|
||||
),
|
||||
"",
|
||||
f"## {aggregated['note_count']} 篇笔记概览（标题 | 平台 | 点赞 | 评论）",
|
||||
]
|
||||
for meta in aggregated["notes_meta"]:
|
||||
lines.append(
|
||||
f"- #{meta['id']} | {meta['platform']} | 点赞 {meta['like_count']} | 评论 {meta['comment_count']} | {meta['title']}"
|
||||
)
|
||||
|
||||
if sample_comments:
|
||||
lines.extend([
|
||||
"",
|
||||
f"## 代表性评论样本（共 {len(sample_comments)} 条，来自多篇笔记）",
|
||||
])
|
||||
for c in sample_comments:
|
||||
lines.append(f"- {c}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM call
|
||||
# ---------------------------------------------------------------------------
|
||||
def analyze_style(
|
||||
style_name: str,
|
||||
brand: str,
|
||||
aggregated: dict[str, Any],
|
||||
merged_distribution: dict[str, Any],
|
||||
sample_comments: list[str],
|
||||
note_summaries: list[str],
|
||||
*,
|
||||
doc_text_limit: int = 6000,
|
||||
) -> str:
|
||||
"""Call Hermes for the style-level rollup analysis."""
|
||||
if not aggregated["note_count"]:
|
||||
return "（该款式暂无笔记可分析）"
|
||||
|
||||
log(
|
||||
f"调用 Hermes 生成款式整体分析（{style_name}，{aggregated['note_count']} 篇笔记，"
|
||||
f"{aggregated['total_comments']} 条总评论）..."
|
||||
)
|
||||
user_content = build_style_rollup_prompt(
|
||||
style_name, brand, aggregated, merged_distribution, sample_comments, note_summaries,
|
||||
)
|
||||
|
||||
target_term = brand or style_name
|
||||
brand_block = ""
|
||||
if target_term:
|
||||
brand_block = f"""
|
||||
|
||||
【实体隔离】
|
||||
本次分析的目标：品牌「{brand or '未指定'}」、产品「{style_name}」。
|
||||
本款式可能涉及多品牌联合推广，请严格将分析对象限定为我方品牌/产品：
|
||||
- 仅将「{target_term}」相关的信息计入种草指标
|
||||
- 评论中涉及的其他品牌或产品，仅作为场景搭配或联合推广的陪衬
|
||||
"""
|
||||
|
||||
system_prompt = ANALYSIS_SYSTEM_PROMPT + brand_block + "\n\n" + STYLE_ROLLUP_DEEP_PROMPT
|
||||
|
||||
result = call_hermes_analyzer(system_prompt, user_content)
|
||||
if len(result) > doc_text_limit:
|
||||
result = result[:doc_text_limit] + "\n\n...（内容过长，已截断）"
|
||||
return result
|
||||
@@ -0,0 +1,37 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Sync creator attributes and cooperation records from Feishu to PostgreSQL
|
||||
REM Runs after daily scrape to pick up new creators/notes/cooperations
|
||||
REM Register: schtasks /Create /SC DAILY /TN YingxiaoYunying_SyncCooperations /TR %PROJECT_DIR%\data\tools\sync_cooperations.bat /ST 09:00 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing\scheduled
|
||||
set PYTHON=%GYXX_PYTHON%
|
||||
where %PYTHON% >nul 2>&1 || set PYTHON=python
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
|
||||
set TS=%date:~0,4%%date:~5,2%%date:~8,2%_%time:~0,2%%time:~3,2%%time:~6,2%
|
||||
set TS=%TS: =0%
|
||||
|
||||
set LOG_FILE=%LOG_DIR%\sync_cooperations_%TS%.log
|
||||
|
||||
echo === sync_cooperations started at %date% %time% === > "%LOG_FILE%"
|
||||
|
||||
cd /d "%PROJECT_DIR%"
|
||||
|
||||
call %PYTHON% data\tools\sync_cooperations.py --refresh-mapping >> "%LOG_FILE%" 2>&1
|
||||
set RC=%ERRORLEVEL%
|
||||
|
||||
echo === sync_cooperations finished at %date% %time% exit=%RC% === >> "%LOG_FILE%"
|
||||
|
||||
endlocal & exit /b %RC%
|
||||
@@ -0,0 +1,463 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
sync_cooperations.py — 从飞书款式合作表同步达人属性和合作记录到数据库
|
||||
====================================================================
|
||||
|
||||
遍历 35 个款式的飞书多维表格,对每一行:
|
||||
1. upsert cmt_creators (达人属性: 名称/平台/粉丝数/主页链接/微信号/类型)
|
||||
2. upsert cmt_cooperations (合作记录: 费用/内容方向/曝光量/数据表现等)
|
||||
|
||||
幂等: feishu_record_id 为唯一键,重复运行只更新不重复插入。
|
||||
|
||||
用法:
|
||||
python sync_cooperations.py # 正式同步
|
||||
python sync_cooperations.py --dry-run # 仅预览
|
||||
python sync_cooperations.py --style 7 # 只同步指定款式
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
import psycopg
|
||||
from dotenv import load_dotenv
|
||||
|
||||
BASE_DIR = PATHS.tools_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
load_dotenv(PATHS.state_root / "config/db.env")
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping # noqa: E402
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_db_config # noqa: E402
|
||||
|
||||
DB_CONFIG = get_db_config()
|
||||
|
||||
PLATFORM_MAP = {
|
||||
"小红书": "xiaohongshu",
|
||||
"抖音": "douyin",
|
||||
"B站": "bilibili",
|
||||
}
|
||||
|
||||
# 飞书逻辑字段 → cmt_creators 列
|
||||
CREATOR_FIELD_MAP = {
|
||||
"creator_name": "name",
|
||||
"platform": "platform",
|
||||
"creator_id": "platform_id",
|
||||
"follower_count": "follower_count",
|
||||
"profile_url": "homepage_url",
|
||||
"wechat": "wechat",
|
||||
"account_type": "account_type",
|
||||
}
|
||||
|
||||
# 飞书逻辑字段 → cmt_cooperations 列
|
||||
COOP_FIELD_MAP = {
|
||||
"cooperation_date": "cooperation_date",
|
||||
"cost": "cooperation_cost",
|
||||
"ad_spend": "ad_spend",
|
||||
"content_direction": "content_direction",
|
||||
"note_title": "note_title",
|
||||
"note_url": "note_url",
|
||||
"publish_time": "publish_time",
|
||||
"read_count_auto": "exposure_count",
|
||||
"engagement_count": "engagement_count",
|
||||
"data_performance": "data_performance",
|
||||
"cpm": "cpm",
|
||||
"tracking_no": "tracking_number",
|
||||
"is_paid": "is_paid",
|
||||
"is_new_direction": "is_new_direction",
|
||||
}
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
safe = str(msg).encode("gbk", errors="replace").decode("gbk", errors="replace")
|
||||
print(safe, flush=True)
|
||||
|
||||
|
||||
def parse_follower_count(text: str) -> int | None:
|
||||
"""解析粉丝数文本: '3.7万' → 37000, '12.4w' → 124000, '5600' → 5600"""
|
||||
if not text or not text.strip():
|
||||
return None
|
||||
text = text.strip()
|
||||
m = re.match(r"([\d.]+)\s*万", text)
|
||||
if m:
|
||||
return int(float(m.group(1)) * 10000)
|
||||
m = re.match(r"([\d.]+)\s*w", text, re.IGNORECASE)
|
||||
if m:
|
||||
return int(float(m.group(1)) * 10000)
|
||||
m = re.match(r"([\d,]+)", text)
|
||||
if m:
|
||||
return int(m.group(1).replace(",", ""))
|
||||
return None
|
||||
|
||||
|
||||
def parse_engagement_count(text: str) -> int | None:
|
||||
"""解析互动赞藏数: '68.5万' → 685000"""
|
||||
return parse_follower_count(text)
|
||||
|
||||
|
||||
def cell_text(cell) -> str:
|
||||
"""从飞书单元格提取纯文本"""
|
||||
if cell is None:
|
||||
return ""
|
||||
if isinstance(cell, str):
|
||||
return cell.strip()
|
||||
if isinstance(cell, list):
|
||||
# select/multiselect/link 等返回数组
|
||||
parts = []
|
||||
for item in cell:
|
||||
if isinstance(item, str):
|
||||
parts.append(item)
|
||||
elif isinstance(item, dict):
|
||||
parts.append(item.get("text", item.get("name", str(item))))
|
||||
else:
|
||||
parts.append(str(item))
|
||||
return ",".join(parts)
|
||||
if isinstance(cell, dict):
|
||||
return cell.get("text", str(cell))
|
||||
return str(cell).strip()
|
||||
|
||||
|
||||
def cell_number(cell) -> float | None:
|
||||
"""从飞书单元格提取数字"""
|
||||
if cell is None:
|
||||
return None
|
||||
if isinstance(cell, (int, float)):
|
||||
return float(cell)
|
||||
if isinstance(cell, str):
|
||||
try:
|
||||
return float(cell.replace(",", ""))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def cell_bool(cell) -> bool:
|
||||
"""从飞书单元格提取布尔值"""
|
||||
if cell is None:
|
||||
return False
|
||||
if isinstance(cell, bool):
|
||||
return cell
|
||||
if isinstance(cell, str):
|
||||
return cell.lower() in ("true", "1", "yes")
|
||||
return bool(cell)
|
||||
|
||||
|
||||
def cell_date(cell) -> str | None:
|
||||
"""从飞书单元格提取日期 (返回 YYYY-MM-DD 或 None)"""
|
||||
text = cell_text(cell)
|
||||
if not text:
|
||||
return None
|
||||
# 飞书 datetime 格式: "2026-05-06 00:00:00"
|
||||
m = re.match(r"(\d{4}-\d{2}-\d{2})", text)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def resolve_platform(raw: str) -> str:
|
||||
"""飞书投放平台 → 数据库 platform 值"""
|
||||
for cn, en in PLATFORM_MAP.items():
|
||||
if cn in raw:
|
||||
return en
|
||||
return raw.lower() if raw else ""
|
||||
|
||||
|
||||
def get_style_id(cur, style_name: str) -> int | None:
|
||||
"""按款式名查 style_id"""
|
||||
cur.execute("SELECT id FROM cmt_styles WHERE name = %s", (style_name,))
|
||||
row = cur.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def ensure_style(cur, style_name: str) -> int:
|
||||
"""确保款式存在,不存在则插入"""
|
||||
sid = get_style_id(cur, style_name)
|
||||
if sid:
|
||||
return sid
|
||||
cur.execute(
|
||||
"INSERT INTO cmt_styles (name) VALUES (%s) ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id",
|
||||
(style_name,),
|
||||
)
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def upsert_creator(cur, name: str, platform: str, attrs: dict) -> int:
|
||||
"""upsert 达人,返回 creator_id"""
|
||||
|
||||
# 尝试按 (name, platform) 查找
|
||||
cur.execute(
|
||||
"SELECT id FROM cmt_creators WHERE name = %s AND platform = %s",
|
||||
(name, platform),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
creator_id = row[0]
|
||||
# 更新属性(只更新非空值)
|
||||
updates = {}
|
||||
for col, val in attrs.items():
|
||||
if val and val != "":
|
||||
updates[col] = val
|
||||
if updates:
|
||||
set_clause = ", ".join(f"{k} = %s" for k in updates)
|
||||
vals = list(updates.values()) + [creator_id]
|
||||
cur.execute(f"UPDATE cmt_creators SET {set_clause} WHERE id = %s", vals)
|
||||
return creator_id
|
||||
|
||||
# 插入新达人
|
||||
cols = ["name", "platform"] + list(attrs.keys())
|
||||
placeholders = ", ".join(["%s"] * len(cols))
|
||||
col_str = ", ".join(cols)
|
||||
vals = [name, platform] + list(attrs.values())
|
||||
cur.execute(
|
||||
f"INSERT INTO cmt_creators ({col_str}) VALUES ({placeholders}) RETURNING id",
|
||||
vals,
|
||||
)
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def upsert_cooperation(cur, style_id: int, creator_id: int,
|
||||
feishu_record_id: str, attrs: dict) -> bool:
|
||||
"""upsert 合作记录,返回是否插入(True)或更新(False)"""
|
||||
if not feishu_record_id:
|
||||
return False
|
||||
|
||||
# 检查是否已存在
|
||||
cur.execute(
|
||||
"SELECT id FROM cmt_cooperations WHERE feishu_record_id = %s",
|
||||
(feishu_record_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
# 更新
|
||||
updates = {"style_id": style_id, "creator_id": creator_id}
|
||||
for col, val in attrs.items():
|
||||
updates[col] = val
|
||||
set_clause = ", ".join(f"{k} = %s" for k in updates)
|
||||
vals = list(updates.values()) + [row[0]]
|
||||
cur.execute(f"UPDATE cmt_cooperations SET {set_clause} WHERE id = %s", vals)
|
||||
return False
|
||||
|
||||
# 插入
|
||||
cols = ["style_id", "creator_id", "feishu_record_id"] + list(attrs.keys())
|
||||
placeholders = ", ".join(["%s"] * len(cols))
|
||||
col_str = ", ".join(cols)
|
||||
vals = [style_id, creator_id, feishu_record_id] + list(attrs.values())
|
||||
cur.execute(
|
||||
f"INSERT INTO cmt_cooperations ({col_str}) VALUES ({placeholders})",
|
||||
vals,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def sync_table(cur, table_info: dict, dry_run: bool = False) -> dict:
|
||||
"""同步一个款式表的所有记录"""
|
||||
style_name = table_info.get("name", "")
|
||||
base_token = table_info.get("base_token", "")
|
||||
table_id = table_info.get("table_id", "")
|
||||
field_map = table_info.get("field_map", {})
|
||||
|
||||
if not base_token or not table_id:
|
||||
return {"style": style_name, "error": "missing base_token/table_id"}
|
||||
|
||||
style_id = ensure_style(cur, style_name)
|
||||
stats = {"style": style_name, "creators": 0, "cooperations_new": 0, "cooperations_updated": 0, "skipped": 0, "skipped_no_creator": 0}
|
||||
|
||||
# 拉取全部记录
|
||||
records = feishu_mapping._record_list_all(base_token, table_id)
|
||||
_log(f" [{style_name}] 拉到 {len(records)} 条记录")
|
||||
|
||||
# 构建 field_id → logical_key 反向映射
|
||||
fid_to_lkey = {}
|
||||
for lkey, finfo in field_map.items():
|
||||
fid_to_lkey[finfo["field_id"]] = lkey
|
||||
|
||||
for rec in records:
|
||||
record_id = rec.get("record_id", "")
|
||||
|
||||
# _record_list_all 返回扁平结构: record_id + 各 field_id 直接在顶层
|
||||
# 提取逻辑字段值
|
||||
row = {}
|
||||
for fid, val in rec.items():
|
||||
if fid == "record_id":
|
||||
continue
|
||||
lkey = fid_to_lkey.get(fid)
|
||||
if lkey:
|
||||
row[lkey] = val
|
||||
|
||||
# 达人名称
|
||||
creator_name = cell_text(row.get("creator_name", ""))
|
||||
if not creator_name:
|
||||
stats["skipped_no_creator"] += 1
|
||||
continue
|
||||
|
||||
# 平台
|
||||
raw_platform = cell_text(row.get("platform", ""))
|
||||
platform = resolve_platform(raw_platform)
|
||||
|
||||
# 达人属性
|
||||
creator_attrs = {}
|
||||
raw_follower = cell_text(row.get("follower_count", ""))
|
||||
if raw_follower:
|
||||
creator_attrs["follower_count"] = raw_follower
|
||||
follower_num = parse_follower_count(raw_follower)
|
||||
if follower_num is not None:
|
||||
creator_attrs["follower_count_num"] = follower_num
|
||||
|
||||
platform_id = cell_text(row.get("creator_id", ""))
|
||||
if platform_id:
|
||||
creator_attrs["platform_id"] = platform_id
|
||||
|
||||
homepage = cell_text(row.get("profile_url", ""))
|
||||
if homepage:
|
||||
creator_attrs["homepage_url"] = homepage
|
||||
|
||||
wechat = cell_text(row.get("wechat", ""))
|
||||
if wechat:
|
||||
creator_attrs["wechat"] = wechat
|
||||
|
||||
account_type = cell_text(row.get("account_type", ""))
|
||||
if account_type:
|
||||
creator_attrs["account_type"] = account_type
|
||||
|
||||
if not dry_run:
|
||||
creator_id = upsert_creator(cur, creator_name, platform, creator_attrs)
|
||||
stats["creators"] += 1
|
||||
else:
|
||||
stats["creators"] += 1
|
||||
creator_id = -1 # dry-run placeholder
|
||||
|
||||
# 合作记录属性
|
||||
coop_attrs = {}
|
||||
|
||||
coop_date = cell_date(row.get("cooperation_date", ""))
|
||||
if coop_date:
|
||||
coop_attrs["cooperation_date"] = coop_date
|
||||
|
||||
cost = cell_number(row.get("cost", ""))
|
||||
if cost is not None:
|
||||
coop_attrs["cooperation_cost"] = cost
|
||||
|
||||
ad_spend = cell_number(row.get("ad_spend", ""))
|
||||
if ad_spend is not None:
|
||||
coop_attrs["ad_spend"] = ad_spend
|
||||
|
||||
content_dir = cell_text(row.get("content_direction", ""))
|
||||
if content_dir:
|
||||
coop_attrs["content_direction"] = content_dir
|
||||
|
||||
note_title = cell_text(row.get("note_title", ""))
|
||||
if note_title:
|
||||
coop_attrs["note_title"] = note_title
|
||||
|
||||
note_url = cell_text(row.get("note_url", ""))
|
||||
if note_url:
|
||||
coop_attrs["note_url"] = note_url
|
||||
|
||||
pub_time = cell_text(row.get("publish_time", ""))
|
||||
if pub_time:
|
||||
# 飞书格式 "2026-06-09 00:00:00"
|
||||
coop_attrs["publish_time"] = pub_time
|
||||
|
||||
exposure = cell_number(row.get("read_count_auto", ""))
|
||||
if exposure is not None:
|
||||
coop_attrs["exposure_count"] = int(exposure)
|
||||
|
||||
raw_engagement = cell_text(row.get("engagement_count", ""))
|
||||
if raw_engagement:
|
||||
coop_attrs["engagement_count"] = raw_engagement
|
||||
eng_num = parse_engagement_count(raw_engagement)
|
||||
if eng_num is not None:
|
||||
coop_attrs["engagement_count_num"] = eng_num
|
||||
|
||||
data_perf = cell_text(row.get("data_performance", ""))
|
||||
if data_perf:
|
||||
coop_attrs["data_performance"] = data_perf
|
||||
|
||||
cpm_val = cell_number(row.get("cpm", ""))
|
||||
if cpm_val is not None:
|
||||
coop_attrs["cpm"] = cpm_val
|
||||
|
||||
tracking = cell_text(row.get("tracking_no", ""))
|
||||
if tracking:
|
||||
coop_attrs["tracking_number"] = tracking
|
||||
|
||||
is_paid = cell_bool(row.get("is_paid", False))
|
||||
coop_attrs["is_paid"] = is_paid
|
||||
|
||||
is_new_dir = cell_bool(row.get("is_new_direction", False))
|
||||
coop_attrs["is_new_direction"] = is_new_dir
|
||||
|
||||
if not dry_run:
|
||||
is_new = upsert_cooperation(cur, style_id, creator_id, record_id, coop_attrs)
|
||||
if is_new:
|
||||
stats["cooperations_new"] += 1
|
||||
else:
|
||||
stats["cooperations_updated"] += 1
|
||||
else:
|
||||
stats["cooperations_new"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="从飞书款式合作表同步达人属性和合作记录到数据库")
|
||||
parser.add_argument("--dry-run", action="store_true", help="仅预览,不写入数据库")
|
||||
parser.add_argument("--style", type=int, help="只同步指定款式编号 (如 7=宙斯)")
|
||||
parser.add_argument("--refresh-mapping", action="store_true",
|
||||
help="强制刷新 mapping (发现新款/新表/新字段); 不加则用缓存")
|
||||
args = parser.parse_args()
|
||||
|
||||
_log("=== sync_cooperations.py 开始 ===")
|
||||
if args.dry_run:
|
||||
_log("[DRY-RUN] 仅预览模式")
|
||||
|
||||
# 加载 mapping; 加 --refresh-mapping 时强制刷新以发现新款/新表/新字段
|
||||
if args.refresh_mapping:
|
||||
_log("[step 0] 刷新 feishu_mapping (发现新款/新表/新字段)...")
|
||||
mapping = feishu_mapping.refresh_mapping(force=True)
|
||||
else:
|
||||
mapping = feishu_mapping.load_mapping()
|
||||
tables = mapping.get("tables", [])
|
||||
|
||||
if args.style:
|
||||
tables = [t for t in tables if t.get("index") == args.style]
|
||||
if not tables:
|
||||
_log(f"未找到款式编号 {args.style}")
|
||||
return
|
||||
|
||||
_log(f"共 {len(tables)} 个款式待同步")
|
||||
|
||||
conn = psycopg.connect(**DB_CONFIG)
|
||||
cur = conn.cursor()
|
||||
|
||||
total_creators = 0
|
||||
total_coop_new = 0
|
||||
total_coop_upd = 0
|
||||
|
||||
for t in tables:
|
||||
try:
|
||||
stats = sync_table(cur, t, dry_run=args.dry_run)
|
||||
_log(f" [{stats['style']}] 达人={stats['creators']}, "
|
||||
f"合作新增={stats['cooperations_new']}, 更新={stats['cooperations_updated']}, "
|
||||
f"跳过={stats['skipped']}")
|
||||
total_creators += stats["creators"]
|
||||
total_coop_new += stats["cooperations_new"]
|
||||
total_coop_upd += stats["cooperations_updated"]
|
||||
except Exception as e:
|
||||
_log(f" [ERROR] {t.get('name', '?')}: {e}")
|
||||
conn.rollback()
|
||||
continue
|
||||
|
||||
if not args.dry_run:
|
||||
conn.commit()
|
||||
_log(f"=== 同步完成: 达人={total_creators}, 合作新增={total_coop_new}, 更新={total_coop_upd} ===")
|
||||
else:
|
||||
_log(f"=== DRY-RUN 预览: 达人={total_creators}, 合作={total_coop_new} ===")
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+472
@@ -0,0 +1,472 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
把 yingxiaoyunying 每日采集的曝光量(view_count)同步到
|
||||
comment-data-collector 的 cmt_notes 表。
|
||||
|
||||
数据源:data/v2_results/*_v2.json / *_xingtu_v2.json / *_bilibili_v2.json
|
||||
(由 xingtu_scraper_v2.py / pgy_xhs_scraper_v2.py / bilibili_scraper.py 每天产出)
|
||||
目标表:cmt_notes(由 D:\\comment-data-collector 维护)
|
||||
|
||||
本脚本只写 view_count 一个指标。like_count / comment_count / favorite_count /
|
||||
share_count / scraped_at 全部由 comment-data-collector 维护,yingxiaoyunying 不碰,
|
||||
避免双写冲突 / 列名歧义(收藏在 xhs 里叫 collect_count,在 cmt_notes 里对应
|
||||
favorite_count,跨表语义容易错位)。
|
||||
|
||||
平台映射:
|
||||
xingtu_v2 -> douyin
|
||||
_v2(无后缀) -> xiaohongshu
|
||||
_bilibili_v2 -> bilibili
|
||||
|
||||
INSERT 路径:对 v2_results 有但 cmt_notes 没的 record_id,先调 feishu +record-list
|
||||
回填 url / title / creator_name(用款式对应多维表格的 base_token),resolve
|
||||
style_id / creator_id 后 INSERT cmt_notes 行(只写必要字段)。同一物理笔记不同
|
||||
record_id(URL 已存在)走 merge-by-url:把这次的 view_count 写进已存在的行。
|
||||
|
||||
用法:
|
||||
python sync_metrics_to_cmt_notes.py # 正式同步(含 INSERT)
|
||||
python sync_metrics_to_cmt_notes.py --dry-run # 仅预览
|
||||
python sync_metrics_to_cmt_notes.py --no-insert-missing # 只 UPDATE 不 INSERT
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
import psycopg
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
def configure_console_output(stdout, stderr) -> None:
|
||||
"""Keep Windows GBK logs alive when creator names contain Unicode symbols."""
|
||||
for stream in (stdout, stderr):
|
||||
if stream is None:
|
||||
continue
|
||||
try:
|
||||
stream.reconfigure(errors="replace")
|
||||
except (AttributeError, OSError, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
configure_console_output(sys.stdout, sys.stderr)
|
||||
|
||||
BASE_DIR = PATHS.tools_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
V2_DIR = PATHS.normalized_root / "v2_results"
|
||||
load_dotenv(PATHS.state_root / "config/db.env")
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.v2_filename import parse_v2_filename # noqa: E402
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping # noqa: E402
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_db_config # noqa: E402
|
||||
|
||||
|
||||
DB_CONFIG = get_db_config()
|
||||
|
||||
|
||||
def parse_int(v) -> int | None:
|
||||
"""'1.2万' / '6,592' / '2422' / 2422 -> int"""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, (int, float)):
|
||||
return int(v)
|
||||
txt = str(v).strip()
|
||||
if not txt:
|
||||
return None
|
||||
m = re.match(r"^([\d,.]+)\s*(万|w|W)?", txt)
|
||||
if not m:
|
||||
return None
|
||||
num = float(m.group(1).replace(",", ""))
|
||||
if m.group(2):
|
||||
num *= 10000
|
||||
return int(num)
|
||||
|
||||
|
||||
def get_conn():
|
||||
return psycopg.connect(**DB_CONFIG)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# INSERT 路径辅助:回填 feishu + 解析 FK
|
||||
# ============================================================
|
||||
_URL_RE = re.compile(r"https?://[^\s)>\]]+")
|
||||
|
||||
|
||||
def _extract_url(text) -> str | None:
|
||||
"""从飞书链接单元格 '[url](url)' 或裸 URL 中提取纯 URL"""
|
||||
if not text:
|
||||
return None
|
||||
s = str(text).strip()
|
||||
if not s:
|
||||
return None
|
||||
m = _URL_RE.search(s)
|
||||
return m.group(0) if m else s
|
||||
|
||||
|
||||
def _parse_publish_time(text) -> dt.datetime | None:
|
||||
"""飞书发布时间 '2026-06-09 00:00:00' / '2026-06-09' / 毫秒时间戳 -> datetime"""
|
||||
if text is None:
|
||||
return None
|
||||
if isinstance(text, (int, float)):
|
||||
try:
|
||||
return dt.datetime.fromtimestamp(int(text) / 1000)
|
||||
except Exception:
|
||||
return None
|
||||
s = str(text).strip()
|
||||
if not s:
|
||||
return None
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d"):
|
||||
try:
|
||||
return dt.datetime.strptime(s, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
# (base_token, table_id) -> {record_id: record_dict}
|
||||
_BACKFILL_CACHE: dict[tuple[str, str], dict[str, dict]] = {}
|
||||
|
||||
|
||||
def _backfill_record(mapping: dict, style_name: str, rid: str) -> dict | None:
|
||||
"""从飞书合作达人多维表格拉指定 record_id 的完整字段。
|
||||
返回 {title, url, publish_time, creator_name} 或 None(找不到/拉取失败)。
|
||||
同一 (base_token, table_id) 在本进程内只拉一次。
|
||||
"""
|
||||
style = next((t for t in mapping.get("tables", []) if t["name"] == style_name), None)
|
||||
if not style:
|
||||
return None
|
||||
bt, tid = style["base_token"], style["table_id"]
|
||||
fm = style.get("field_map", {})
|
||||
# note_url 必须有; creator_name 自营表可能没有,用款式名兜底
|
||||
if not fm.get("note_url"):
|
||||
return None
|
||||
|
||||
cache_key = (bt, tid)
|
||||
if cache_key not in _BACKFILL_CACHE:
|
||||
try:
|
||||
rows = feishu_mapping._record_list_all(bt, tid)
|
||||
except Exception as exc:
|
||||
print(f" [backfill-err] {style_name}({bt}/{tid}): {exc}")
|
||||
return None
|
||||
_BACKFILL_CACHE[cache_key] = {r.get("record_id"): r for r in rows}
|
||||
|
||||
rec = _BACKFILL_CACHE[cache_key].get(rid)
|
||||
if not rec:
|
||||
return None
|
||||
|
||||
def gv(key: str):
|
||||
fid = fm.get(key, {}).get("field_id")
|
||||
return rec.get(fid) if fid else None
|
||||
|
||||
creator_name = gv("creator_name") or f"自营-{style_name}"
|
||||
# title 剥掉 markdown 链接 [text](url) -> 只留 text,再剥裸 URL,避免超 varchar(512)
|
||||
raw_title = gv("note_title")
|
||||
if isinstance(raw_title, list):
|
||||
raw_title = " ".join(str(x) for x in raw_title if x)
|
||||
title_clean = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", str(raw_title or ""))
|
||||
title_clean = re.sub(r"https?://[^\s）)\]]+", "", title_clean).strip()
|
||||
return {
|
||||
"title": title_clean[:500],
|
||||
"url": _extract_url(gv("note_url")),
|
||||
"creator_name": creator_name,
|
||||
"publish_time": _parse_publish_time(gv("publish_time")),
|
||||
}
|
||||
|
||||
|
||||
def _style_id_lookup(cur) -> dict[str, int]:
|
||||
"""name -> cmt_styles.id 映射(一次查全表)"""
|
||||
cur.execute("SELECT id, name FROM cmt_styles")
|
||||
return {name: id for id, name in cur.fetchall()}
|
||||
|
||||
|
||||
def _ensure_creator(cur, name: str | None) -> int | None:
|
||||
"""按 name 查找或新建 cmt_creators 行,返回 id。name 为空返回 None。"""
|
||||
if not name:
|
||||
return None
|
||||
cur.execute("SELECT id FROM cmt_creators WHERE name = %s", (name,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
cur.execute(
|
||||
"INSERT INTO cmt_creators (name, created_at, updated_at) "
|
||||
"VALUES (%s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id",
|
||||
(name,),
|
||||
)
|
||||
return cur.fetchone()[0] if cur.rowcount else None
|
||||
|
||||
|
||||
def detect_platform(path: Path) -> tuple[str | None, bool]:
|
||||
"""文件名后缀 -> (cmt_notes.platform, self_operated)"""
|
||||
_, _, plat, self_op = parse_v2_filename(path.name)
|
||||
return {"xhs": "xiaohongshu", "xt": "douyin", "bili": "bilibili"}.get(plat), self_op
|
||||
|
||||
|
||||
def _is_syncable_item(item: dict) -> bool:
|
||||
"""Only sync rows that represent a real matched scrape result."""
|
||||
if not isinstance(item, dict):
|
||||
return False
|
||||
if item.get("error"):
|
||||
return False
|
||||
if item.get("matched") is False:
|
||||
return False
|
||||
if item.get("ok") is False:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def collect_metrics_from_v2() -> list[dict]:
|
||||
"""扫一遍 v2_results,聚合每个 record_id 的最新指标"""
|
||||
out: list[dict] = []
|
||||
for path in sorted(V2_DIR.glob("*.json")):
|
||||
result = detect_platform(path)
|
||||
if not result or result[0] is None:
|
||||
continue
|
||||
platform, self_operated = result
|
||||
idx, style_name, _, _ = parse_v2_filename(path.name)
|
||||
if idx is None:
|
||||
continue
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if platform == "bilibili":
|
||||
items = data.get("details") or []
|
||||
for item in items:
|
||||
if not _is_syncable_item(item):
|
||||
continue
|
||||
rid = item.get("record_id")
|
||||
view_count = parse_int(item.get("play_count"))
|
||||
if not rid or view_count is None:
|
||||
continue
|
||||
out.append({
|
||||
"src_platform": "bili",
|
||||
"dst_platform": platform,
|
||||
"style_idx": idx,
|
||||
"style_name": style_name,
|
||||
"record_id": rid,
|
||||
"creator_name": item.get("creator"),
|
||||
"view_count": view_count,
|
||||
"like_count": None,
|
||||
"collect_count": None,
|
||||
"comment_count": None,
|
||||
"self_operated": self_operated,
|
||||
})
|
||||
else:
|
||||
items = data.get("results") or []
|
||||
for item in items:
|
||||
if not _is_syncable_item(item):
|
||||
continue
|
||||
rid = item.get("record_id")
|
||||
view_count = parse_int(item.get("play_count") or item.get("read_count"))
|
||||
if not rid or view_count is None:
|
||||
continue
|
||||
out.append({
|
||||
"src_platform": "xt" if platform == "douyin" else "xhs",
|
||||
"dst_platform": platform,
|
||||
"style_idx": idx,
|
||||
"style_name": style_name,
|
||||
"record_id": rid,
|
||||
"creator_name": item.get("creator"),
|
||||
"view_count": view_count,
|
||||
"like_count": parse_int(item.get("like_count")),
|
||||
"collect_count": parse_int(item.get("collect_count")),
|
||||
"comment_count": parse_int(item.get("comment_count")),
|
||||
"self_operated": self_operated,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def sync(dry_run: bool = False, insert_missing: bool = True) -> dict:
|
||||
metrics = collect_metrics_from_v2()
|
||||
print(f"[sync] v2_results 中可同步的命中记录: {len(metrics)} 条")
|
||||
|
||||
matched = updated = skipped_not_found = 0
|
||||
inserted = skipped_insert = 0
|
||||
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
style_name_to_id: dict[str, int] = _style_id_lookup(cur)
|
||||
creator_id_cache: dict[str, int | None] = {}
|
||||
mapping = None
|
||||
self_mapping = None
|
||||
if insert_missing:
|
||||
try:
|
||||
mapping = feishu_mapping.load_mapping(force_refresh=False)
|
||||
except Exception as exc:
|
||||
print(f"[WARN] feishu_mapping 加载失败,INSERT 路径禁用: {exc}")
|
||||
insert_missing = False
|
||||
try:
|
||||
self_mapping = feishu_mapping.load_mapping(force_refresh=False, self_operated=True)
|
||||
except Exception as exc:
|
||||
print(f"[WARN] feishu_mapping(自营) 加载失败: {exc}")
|
||||
|
||||
for row in metrics:
|
||||
record_id = row["record_id"]
|
||||
dst_platform = row["dst_platform"]
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, view_count FROM cmt_notes
|
||||
WHERE feishu_record_id = %s AND platform = %s
|
||||
""",
|
||||
(record_id, dst_platform),
|
||||
)
|
||||
note = cur.fetchone()
|
||||
if not note:
|
||||
skipped_not_found += 1
|
||||
if not insert_missing:
|
||||
continue
|
||||
# 选择正确的 mapping: 自营用 self_mapping, 合作用 mapping
|
||||
active_mapping = self_mapping if row.get("self_operated") else mapping
|
||||
if not active_mapping:
|
||||
continue
|
||||
# ---- INSERT path ----
|
||||
style_name = row.get("style_name")
|
||||
style_id = style_name_to_id.get(style_name or "")
|
||||
if not style_id:
|
||||
print(f" [skip-insert] rid={record_id}: style '{style_name}' 不在 cmt_styles")
|
||||
skipped_insert += 1
|
||||
continue
|
||||
bf = _backfill_record(active_mapping, style_name, record_id)
|
||||
if not bf or not bf.get("url"):
|
||||
print(f" [skip-insert] rid={record_id}: feishu 回填失败或无 URL")
|
||||
skipped_insert += 1
|
||||
continue
|
||||
creator_name = (bf.get("creator_name") or row.get("creator_name") or "").strip()
|
||||
if not creator_name:
|
||||
print(f" [skip-insert] rid={record_id}: creator 名为空")
|
||||
skipped_insert += 1
|
||||
continue
|
||||
if dry_run:
|
||||
cur.execute("SELECT id FROM cmt_creators WHERE name = %s", (creator_name,))
|
||||
creator_row = cur.fetchone()
|
||||
creator_id = creator_row[0] if creator_row else None
|
||||
else:
|
||||
if creator_name not in creator_id_cache:
|
||||
creator_id_cache[creator_name] = _ensure_creator(cur, creator_name)
|
||||
creator_id = creator_id_cache[creator_name]
|
||||
if not creator_id:
|
||||
reason = "creator 不存在(dry-run 不新建)" if dry_run else "creator 名为空"
|
||||
print(f" [skip-insert] rid={record_id}: {reason}")
|
||||
skipped_insert += 1
|
||||
continue
|
||||
# 先查 URL 是否已存在(避免 INSERT 触发 UNIQUE 整事务回滚)
|
||||
cur.execute(
|
||||
"SELECT id, view_count FROM cmt_notes WHERE url = %s LIMIT 1",
|
||||
(bf["url"],),
|
||||
)
|
||||
url_row = cur.fetchone()
|
||||
if url_row:
|
||||
old_id, old_view = url_row
|
||||
new_view = row.get("view_count")
|
||||
if new_view is not None and new_view != old_view:
|
||||
if dry_run:
|
||||
print(f" [dry-merge-by-url] note_id={old_id} view {old_view}->{new_view} (新 rid={record_id})")
|
||||
else:
|
||||
cur.execute(
|
||||
"UPDATE cmt_notes SET view_count = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
|
||||
(new_view, old_id),
|
||||
)
|
||||
print(f" [merge-by-url] note_id={old_id} view {old_view}->{new_view} (新 rid={record_id})")
|
||||
updated += 1
|
||||
else:
|
||||
print(f" [skip-merge] rid={record_id}: url 已存在(note_id={old_id}),view 无变化")
|
||||
skipped_insert += 1
|
||||
continue
|
||||
if dry_run:
|
||||
print(f" [dry-insert] rid={record_id} style={style_name} creator={creator_name} "
|
||||
f"view={row.get('view_count')} title={str(bf.get('title') or '')[:32]}")
|
||||
inserted += 1
|
||||
continue
|
||||
# 只写必要字段:FK + title + url + feishu_record_id + view_count
|
||||
# like/comment/favorite/share/scraped_at 全留给 comment-data-collector
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO cmt_notes
|
||||
(style_id, creator_id, platform, title, url,
|
||||
feishu_record_id, view_count, self_operated,
|
||||
publish_time,
|
||||
created_at, updated_at)
|
||||
VALUES
|
||||
(%s, %s, %s, %s, %s,
|
||||
%s, %s, %s,
|
||||
%s,
|
||||
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
(style_id, creator_id, dst_platform,
|
||||
bf.get("title"), bf["url"], record_id,
|
||||
row.get("view_count"), row.get("self_operated", False),
|
||||
bf.get("publish_time")),
|
||||
)
|
||||
print(f" [insert] rid={record_id} style={style_name} creator={creator_name} view={row.get('view_count')}")
|
||||
inserted += 1
|
||||
continue
|
||||
|
||||
# ---- UPDATE path: 只写 view_count,其它字段(like/comment/favorite/scraped)留给 cdc ----
|
||||
note_id, old_view = note
|
||||
matched += 1
|
||||
new_view = row["view_count"]
|
||||
|
||||
# 缺 publish_time 时从飞书补
|
||||
cur.execute("SELECT publish_time FROM cmt_notes WHERE id = %s", (note_id,))
|
||||
old_pub_row = cur.fetchone()
|
||||
if old_pub_row and old_pub_row[0] is None:
|
||||
try:
|
||||
bf_update = _backfill_record(mapping, style_name, record_id) or \
|
||||
(self_mapping and _backfill_record(self_mapping, style_name, record_id)) or {}
|
||||
new_pub = bf_update.get("publish_time")
|
||||
except Exception:
|
||||
new_pub = None
|
||||
if new_pub is not None:
|
||||
cur.execute(
|
||||
"UPDATE cmt_notes SET publish_time = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
|
||||
(new_pub, note_id),
|
||||
)
|
||||
|
||||
if new_view is not None and new_view != old_view:
|
||||
if dry_run:
|
||||
print(f" [dry-run] note_id={note_id} view {old_view}->{new_view}")
|
||||
else:
|
||||
cur.execute(
|
||||
"UPDATE cmt_notes SET view_count = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
|
||||
(new_view, note_id),
|
||||
)
|
||||
print(f" [update] note_id={note_id} view={new_view}")
|
||||
updated += 1
|
||||
|
||||
if dry_run:
|
||||
conn.rollback()
|
||||
else:
|
||||
conn.commit()
|
||||
|
||||
mode = "DRY-RUN" if dry_run else "WRITE"
|
||||
insert_mode = "ON" if insert_missing else "OFF"
|
||||
print(f"\n[sync] 完成 ({mode}, insert={insert_mode}): "
|
||||
f"匹配={matched}, 更新={updated}, 新增={inserted}, "
|
||||
f"cmt_notes 未找到={skipped_not_found}, 跳过 INSERT={skipped_insert}")
|
||||
return {
|
||||
"total": len(metrics),
|
||||
"matched": matched,
|
||||
"updated": updated,
|
||||
"inserted": inserted,
|
||||
"skipped_not_found": skipped_not_found,
|
||||
"skipped_insert": skipped_insert,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="同步 v2_results 曝光/收藏量到 cmt_notes")
|
||||
parser.add_argument("--dry-run", action="store_true", help="仅预览,不写入")
|
||||
parser.add_argument("--no-insert-missing", action="store_true",
|
||||
help="关闭 INSERT 路径(默认开启:对 v2_results 有但 cmt_notes 缺的行,回填 feishu 后 INSERT)")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"[sync] 模式: {'DRY-RUN' if args.dry_run else '正式写入'} "
|
||||
f"insert-missing: {'OFF' if args.no_insert_missing else 'ON'}")
|
||||
sync(dry_run=args.dry_run, insert_missing=not args.no_insert_missing)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Sync the Feishu product-style classification into ``cmt_styles``.
|
||||
|
||||
The source Base is read with the authenticated lark-cli user identity because
|
||||
the Hermes bot is not a member of that table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
except ModuleNotFoundError: # direct execution: python data/tools/sync_style_categories.py
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
|
||||
|
||||
BASE_TOKEN = "TH1JbxhfUaxetis6F4pcoqvWncd"
|
||||
TABLE_ID = "tbl8q0hbIPMNn0k8"
|
||||
VIEW_ID = "vew7j57rli"
|
||||
NAME_FIELD_ID = "fldRqiyAJW"
|
||||
CATEGORY_FIELD_ID = "fldsEuUtYb"
|
||||
PROFILE = "hermes-analyzer"
|
||||
|
||||
NAME_ALIASES = {
|
||||
"盖亚微单Pro": "盖亚微单",
|
||||
"星迹&星迹2": "星迹2",
|
||||
"宙斯双肩包": "宙斯",
|
||||
"极星Pro": "极星pro",
|
||||
"瑞白双肩包": "瑞白",
|
||||
"逐星gt": "逐星GT",
|
||||
"觅光双肩包": "觅光",
|
||||
"云栖相机双肩包": "云栖",
|
||||
"拾影双肩电脑包": "拾影双肩",
|
||||
"拾影相机包": "拾影斜挎相机包",
|
||||
"极星单肩包": "极星单肩",
|
||||
}
|
||||
|
||||
|
||||
def normalize_style_name(value: str) -> str:
|
||||
"""Convert the Feishu lifecycle-document label to the DB canonical name."""
|
||||
label = re.sub(r"^\[|\]\(.*\)$", "", (value or "").strip())
|
||||
label = re.sub(r"\s*生命进程\s*", "", label)
|
||||
label = re.sub(r"\s*[（(]\s*新\s*[）)]\s*", "", label)
|
||||
label = re.sub(r"\s+", "", label).strip()
|
||||
return NAME_ALIASES.get(label, label)
|
||||
|
||||
|
||||
def extract_style_mapping(payload: dict[str, Any]) -> dict[str, str]:
|
||||
"""Extract ``{canonical product name: style category}`` from CLI JSON."""
|
||||
mapping: dict[str, str] = {}
|
||||
for row in payload.get("data", {}).get("data", []):
|
||||
if not isinstance(row, list) or len(row) < 2 or not row[0] or not row[1]:
|
||||
continue
|
||||
name = normalize_style_name(str(row[0]))
|
||||
category_value = row[1]
|
||||
category = (
|
||||
str(category_value[0]).strip()
|
||||
if isinstance(category_value, list) and category_value
|
||||
else str(category_value).strip()
|
||||
)
|
||||
if not name or not category:
|
||||
continue
|
||||
previous = mapping.get(name)
|
||||
if previous and previous != category:
|
||||
raise ValueError(f"款式 {name} 同时出现多个风格：{previous} / {category}")
|
||||
mapping[name] = category
|
||||
return mapping
|
||||
|
||||
|
||||
def _lark_cli_path() -> str:
|
||||
appdata = os.environ.get("APPDATA")
|
||||
if not appdata:
|
||||
raise RuntimeError("APPDATA 未设置，无法定位 lark-cli.cmd")
|
||||
return str(Path(appdata) / "npm" / "lark-cli.cmd")
|
||||
|
||||
|
||||
def load_mapping_from_feishu() -> dict[str, str]:
|
||||
command = [
|
||||
_lark_cli_path(), "--profile", PROFILE, "base", "+record-list",
|
||||
"--as", "user", "--base-token", BASE_TOKEN, "--table-id", TABLE_ID,
|
||||
"--view-id", VIEW_ID, "--field-id", NAME_FIELD_ID,
|
||||
"--field-id", CATEGORY_FIELD_ID, "--limit", "200", "--format", "json",
|
||||
]
|
||||
result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", check=True)
|
||||
payload = json.loads(result.stdout)
|
||||
if not payload.get("ok"):
|
||||
raise RuntimeError(f"读取飞书款式风格失败：{payload}")
|
||||
mapping = extract_style_mapping(payload)
|
||||
if not mapping:
|
||||
raise RuntimeError("飞书表未返回任何有效的款式—风格记录")
|
||||
return mapping
|
||||
|
||||
|
||||
def sync_mapping(mapping: dict[str, str], *, dry_run: bool = False) -> dict[str, Any]:
|
||||
"""Update existing styles and insert new source styles without overwriting Base links."""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT name FROM cmt_styles")
|
||||
existing = {row[0] for row in cur.fetchall()}
|
||||
inserted = sorted(set(mapping) - existing)
|
||||
updated = sorted(set(mapping) & existing)
|
||||
if not dry_run:
|
||||
cur.execute(
|
||||
"ALTER TABLE cmt_styles ADD COLUMN IF NOT EXISTS style_category VARCHAR(64) DEFAULT ''"
|
||||
)
|
||||
cur.execute(
|
||||
"COMMENT ON COLUMN cmt_styles.style_category IS "
|
||||
"'款式风格，来源于飞书全域事项表的风格单选字段'"
|
||||
)
|
||||
for name, category in sorted(mapping.items()):
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO cmt_styles (name, style_category)
|
||||
VALUES (%s, %s)
|
||||
ON CONFLICT (name) DO UPDATE SET
|
||||
style_category = EXCLUDED.style_category,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(name, category),
|
||||
)
|
||||
conn.commit()
|
||||
return {"source_count": len(mapping), "updated": updated, "inserted": inserted}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="同步飞书款式风格到 cmt_styles")
|
||||
parser.add_argument("--dry-run", action="store_true", help="只核对，不写数据库")
|
||||
args = parser.parse_args()
|
||||
mapping = load_mapping_from_feishu()
|
||||
result = sync_mapping(mapping, dry_run=args.dry_run)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,50 @@
|
||||
"""v2_results/*.json 文件名解析(共享给 check_status / retry_failed / sync_metrics_to_cmt_notes)"""
|
||||
import re
|
||||
|
||||
# 后缀越长优先级越高(避免 _bilibili_v2 被 _v2 误吃)
|
||||
# 自营达人文件带 _self_ 前缀: *_self_bilibili_v2.json, *_self_xingtu_v2.json, *_self_v2.json
|
||||
RE_V2_SELF_BILI = re.compile(r"^(?P<idx>\d+)-.+_self_bilibili_v2\.json$")
|
||||
RE_V2_SELF_XT = re.compile(r"^(?P<idx>\d+)-.+_self_xingtu_v2\.json$")
|
||||
RE_V2_SELF_PGY = re.compile(r"^(?P<idx>\d+)-.+_self_v2\.json$")
|
||||
RE_V2_BILI = re.compile(r"^(?P<idx>\d+)-.+_bilibili_v2\.json$")
|
||||
RE_V2_XT = re.compile(r"^(?P<idx>\d+)-.+_xingtu_v2\.json$")
|
||||
RE_V2_PGY = re.compile(r"^(?P<idx>\d+)-.+_v2\.json$") # 必须放最后兜底
|
||||
|
||||
SUFFIX_LEN = {
|
||||
"self_bili": len("_self_bilibili_v2.json"),
|
||||
"self_xt": len("_self_xingtu_v2.json"),
|
||||
"self_xhs": len("_self_v2.json"),
|
||||
"bili": len("_bilibili_v2.json"),
|
||||
"xt": len("_xingtu_v2.json"),
|
||||
"xhs": len("_v2.json"),
|
||||
}
|
||||
|
||||
|
||||
def parse_v2_filename(name: str) -> tuple[int | None, str, str | None, bool]:
|
||||
"""01-盖亚斜挎_v2.json -> (1, '盖亚斜挎', 'xhs', False)
|
||||
02-极星pro_self_xingtu_v2.json -> (2, '极星pro', 'xt', True)
|
||||
03-盖亚微单_bilibili_v2.json -> (3, '盖亚微单', 'bili', False)
|
||||
不匹配返回 (None, '', None, False)
|
||||
第 4 个返回值: self_operated (True=自营达人)
|
||||
"""
|
||||
if RE_V2_SELF_BILI.match(name):
|
||||
m, plat, self_op = RE_V2_SELF_BILI.match(name), "bili", True
|
||||
elif RE_V2_SELF_XT.match(name):
|
||||
m, plat, self_op = RE_V2_SELF_XT.match(name), "xt", True
|
||||
elif RE_V2_SELF_PGY.match(name):
|
||||
m, plat, self_op = RE_V2_SELF_PGY.match(name), "xhs", True
|
||||
elif RE_V2_BILI.match(name):
|
||||
m, plat, self_op = RE_V2_BILI.match(name), "bili", False
|
||||
elif RE_V2_XT.match(name):
|
||||
m, plat, self_op = RE_V2_XT.match(name), "xt", False
|
||||
elif RE_V2_PGY.match(name):
|
||||
m, plat, self_op = RE_V2_PGY.match(name), "xhs", False
|
||||
else:
|
||||
return None, "", None, False
|
||||
|
||||
idx = int(m.group("idx"))
|
||||
suffix_key = f"self_{plat}" if self_op else plat
|
||||
stem = name[: -SUFFIX_LEN[suffix_key]]
|
||||
parts = stem.split("-", 1)
|
||||
name_only = parts[1] if len(parts) == 2 else ""
|
||||
return idx, name_only, plat, self_op
|
||||
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
validate_mapping.py
|
||||
-------------------
|
||||
校验 data/config/款式_多维表格_对照.json 中的 field_id 跟飞书实际字段是否匹配。
|
||||
不匹配报警(但允许差异,因为有些表故意没字段)。
|
||||
|
||||
用法:
|
||||
python validate_mapping.py # 检查 + 打印结果
|
||||
python validate_mapping.py --strict # 严格模式:任何差异都返回非 0
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
LARK = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd")
|
||||
if not os.path.exists(LARK):
|
||||
LARK = "lark-cli.cmd"
|
||||
MAPPING_CACHE_PATH = PATHS.normalized_root / "mappings" / "款式_多维表格_对照.json"
|
||||
MAPPING_SEED_PATH = PATHS.config_root / "款式_多维表格_对照.json"
|
||||
MAPPING_PATH = MAPPING_CACHE_PATH if MAPPING_CACHE_PATH.exists() else MAPPING_SEED_PATH
|
||||
ENV = {**os.environ, "LARK_CLI_NO_PROXY": "1", "PYTHONIOENCODING": "utf-8"}
|
||||
|
||||
|
||||
def fetch_fields(base_token: str, table_id: str) -> list[dict]:
|
||||
r = subprocess.run(
|
||||
[LARK, "base", "+field-list",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--format", "json"],
|
||||
capture_output=True, env=ENV, timeout=60
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return []
|
||||
raw = r.stdout
|
||||
if isinstance(raw, bytes):
|
||||
raw = raw.decode("utf-8", errors="replace")
|
||||
try:
|
||||
d = json.loads(raw)
|
||||
return d.get("data", {}).get("fields", []) if d.get("ok") else []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--strict", action="store_true", help="任何差异都返回非 0 退出码")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not MAPPING_PATH.exists():
|
||||
print(f"[FATAL] 找不到 {MAPPING_PATH}")
|
||||
sys.exit(1)
|
||||
|
||||
mapping = json.loads(MAPPING_PATH.read_text(encoding="utf-8"))
|
||||
tables = mapping.get("tables", [])
|
||||
|
||||
print(f"校验 {len(tables)} 张表...\n")
|
||||
issues = []
|
||||
for t in tables:
|
||||
idx = t["index"]
|
||||
name = t["name"]
|
||||
fmap = t.get("field_map", {})
|
||||
|
||||
# 拉取实际字段
|
||||
actual_fields = fetch_fields(t["base_token"], t["table_id"])
|
||||
if not actual_fields:
|
||||
issues.append(f"[{idx:02d}] {name}: 拉取失败,跳过")
|
||||
continue
|
||||
actual_by_id = {f["id"]: f["name"] for f in actual_fields}
|
||||
actual_by_name = {f["name"]: f["id"] for f in actual_fields}
|
||||
|
||||
# 1) 校验 JSON 里的 field_id 仍然存在
|
||||
for logical, info in fmap.items():
|
||||
fid = info.get("field_id")
|
||||
fname = info.get("field_name")
|
||||
if fid not in actual_by_id:
|
||||
issues.append(
|
||||
f"[{idx:02d}] {name}: 字段 {logical}({fname}) 的 field_id={fid} 不存在了!"
|
||||
)
|
||||
elif actual_by_id[fid] != fname:
|
||||
issues.append(
|
||||
f"[{idx:02d}] {name}: 字段 {logical}: JSON 说 {fname}({fid}) 但飞书是 {actual_by_id[fid]}"
|
||||
)
|
||||
# 2) 校验核心字段都映射上了
|
||||
for key in ["creator_name", "read_count_7d"]:
|
||||
if key not in fmap:
|
||||
issues.append(f"[{idx:02d}] {name}: 缺核心字段 {key}")
|
||||
|
||||
# 报告
|
||||
if not issues:
|
||||
print("✓ 全部 24 张表字段映射正确,核心字段都存在")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(f"发现 {len(issues)} 个问题:\n")
|
||||
for i in issues:
|
||||
print(f" ✗ {i}")
|
||||
print(f"\n建议: python rebuild_mapping.py --in-place")
|
||||
sys.exit(1 if args.strict else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,80 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Weekly three-platform comment scraper wrapper.
|
||||
REM Runs bilibili, xiaohongshu, douyin in parallel via PowerShell.
|
||||
REM Register: schtasks /Create /SC WEEKLY /D SUN /TN YingxiaoYunying_WeeklyCommentScrape /TR %PROJECT_DIR%\data\tools\weekly_comment_scrape.bat /ST 12:00 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set "PYTHON_EXE=%GYXX_PYTHON%"
|
||||
set "LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing"
|
||||
set "TMP_DIR=%GYXX_DATA_ROOT%\tmp\content_marketing"
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
if not exist "%TMP_DIR%" mkdir "%TMP_DIR%"
|
||||
|
||||
for /f "tokens=2 delims==" %%I in ('wmic os get localdatetime /value') do set "NOW=%%I"
|
||||
set "STAMP=%NOW:~0,4%-%NOW:~4,2%-%NOW:~6,2%_%NOW:~8,2%-%NOW:~10,2%-%NOW:~12,2%"
|
||||
|
||||
set "MASTER_LOG=%LOG_DIR%\weekly_%STAMP%.log"
|
||||
echo === weekly scrape started %STAMP% > "%MASTER_LOG%"
|
||||
|
||||
set "DOUYIN_LOG=%LOG_DIR%\weekly_%STAMP%_douyin.log"
|
||||
set "XHS_LOG=%LOG_DIR%\weekly_%STAMP%_xiaohongshu.log"
|
||||
set "BILI_LOG=%LOG_DIR%\weekly_%STAMP%_bilibili.log"
|
||||
|
||||
echo [%STAMP%] launching douyin, xiaohongshu, bilibili in parallel >> "%MASTER_LOG%"
|
||||
|
||||
set "PS_SCRIPT=%GYXX_DATA_ROOT%\tmp\content_marketing\weekly_scrape_%STAMP%.ps1"
|
||||
> "%PS_SCRIPT%" echo $ErrorActionPreference = 'Continue'
|
||||
>> "%PS_SCRIPT%" echo $env:PYTHONIOENCODING = 'utf-8'
|
||||
>> "%PS_SCRIPT%" echo $env:PYTHONUTF8 = '1'
|
||||
>> "%PS_SCRIPT%" echo Set-Location -LiteralPath '%PROJECT_DIR%'
|
||||
>> "%PS_SCRIPT%" echo $py = '%GYXX_PYTHON%'
|
||||
>> "%PS_SCRIPT%" echo $stamp = '%STAMP%'
|
||||
>> "%PS_SCRIPT%" echo $logDir = '%GYXX_DATA_ROOT%\logs\content_marketing'
|
||||
>> "%PS_SCRIPT%" echo $jobs = @(
|
||||
>> "%PS_SCRIPT%" echo [pscustomobject]@{ Name='bilibili'; Script='%PROJECT_DIR%\data\tools\batch_rescrape_bilibili.py'; Log=Join-Path $logDir ("weekly_${stamp}_bilibili.log") },
|
||||
>> "%PS_SCRIPT%" echo [pscustomobject]@{ Name='xiaohongshu'; Script='%PROJECT_DIR%\data\tools\batch_rescrape_xiaohongshu.py'; Log=Join-Path $logDir ("weekly_${stamp}_xiaohongshu.log") },
|
||||
>> "%PS_SCRIPT%" echo [pscustomobject]@{ Name='douyin'; Script='%PROJECT_DIR%\data\tools\batch_rescrape_douyin.py'; Log=Join-Path $logDir ("weekly_${stamp}_douyin.log") }
|
||||
>> "%PS_SCRIPT%" echo )
|
||||
>> "%PS_SCRIPT%" echo $started = @{}
|
||||
>> "%PS_SCRIPT%" echo foreach ($job in $jobs) {
|
||||
>> "%PS_SCRIPT%" echo $logPath = $job.Log
|
||||
>> "%PS_SCRIPT%" echo $header = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === $($job.Name) launch ==="
|
||||
>> "%PS_SCRIPT%" echo $cmd = "`"$py`" -X utf8 `"$($job.Script)`" >> `"$logPath`" 2>&1"
|
||||
>> "%PS_SCRIPT%" echo Add-Content -LiteralPath $logPath -Value $header -Encoding utf8
|
||||
>> "%PS_SCRIPT%" echo $proc = Start-Process -FilePath cmd.exe -ArgumentList '/c', "`"$cmd`"" -PassThru -WindowStyle Hidden
|
||||
>> "%PS_SCRIPT%" echo if ($null -eq $proc) {
|
||||
>> "%PS_SCRIPT%" echo Add-Content -LiteralPath $logPath -Value "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] !!! Start-Process returned null for $($job.Name) !!!" -Encoding utf8
|
||||
>> "%PS_SCRIPT%" echo continue
|
||||
>> "%PS_SCRIPT%" echo }
|
||||
>> "%PS_SCRIPT%" echo $started[$job.Name] = @{ Proc=$proc; Log=$logPath }
|
||||
>> "%PS_SCRIPT%" echo }
|
||||
>> "%PS_SCRIPT%" echo $exitCodes = @{}
|
||||
>> "%PS_SCRIPT%" echo foreach ($job in $jobs) {
|
||||
>> "%PS_SCRIPT%" echo $entry = $started[$job.Name]
|
||||
>> "%PS_SCRIPT%" echo $entry.Proc.WaitForExit()
|
||||
>> "%PS_SCRIPT%" echo $rc = $entry.Proc.ExitCode
|
||||
>> "%PS_SCRIPT%" echo $exitCodes[$job.Name] = $rc
|
||||
>> "%PS_SCRIPT%" echo Add-Content -LiteralPath $entry.Log -Value "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === $($job.Name) done, exit=$rc ===" -Encoding utf8
|
||||
>> "%PS_SCRIPT%" echo }
|
||||
>> "%PS_SCRIPT%" echo foreach ($k in $exitCodes.Keys) { Write-Output ("{0}={1}" -f $k, $exitCodes[$k]) }
|
||||
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%PS_SCRIPT%" >> "%MASTER_LOG%" 2>&1
|
||||
set "RC=%ERRORLEVEL%"
|
||||
|
||||
del "%PS_SCRIPT%" 2>nul
|
||||
|
||||
echo [%STAMP%] all three scrapers finished, ps exit=%RC% >> "%MASTER_LOG%"
|
||||
echo === weekly scrape done %STAMP% >> "%MASTER_LOG%"
|
||||
|
||||
endlocal & exit /b %RC%
|
||||
@@ -0,0 +1,42 @@
|
||||
@echo off
|
||||
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
|
||||
REM Weekly summary: 每周一 10:00 跑上周各款周笔记汇总
|
||||
REM 1. 动态算上周一到上周日时间范围
|
||||
REM 2. PG 查上周有笔记的款
|
||||
REM 3. 对每个款:跑单品分析 + LLM 4 维度综合 + 创建飞书文档 + 写入生命进程表
|
||||
REM Register: schtasks /Create /SC WEEKLY /D TUE /TN YingxiaoYunying_WeeklySummary /TR %PROJECT_DIR%\data\tools\weekly_summary.bat /ST 10:00 /F
|
||||
|
||||
setlocal
|
||||
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
|
||||
if not defined GYXX_DATA_ROOT (
|
||||
if not defined GYXX_PROJECT_ROOT (
|
||||
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
|
||||
exit /b 3
|
||||
)
|
||||
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
|
||||
)
|
||||
set LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing\scheduled
|
||||
set PYTHON=%GYXX_PYTHON%
|
||||
where %PYTHON% >nul 2>&1 || set PYTHON=python
|
||||
|
||||
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
|
||||
|
||||
set TS=%date:~0,4%%date:~5,2%%date:~8,2%_%time:~0,2%%time:~3,2%%time:~6,2%
|
||||
set TS=%TS: =0%
|
||||
|
||||
set LOG_FILE=%LOG_DIR%\weekly_summary_%TS%.log
|
||||
|
||||
echo === Weekly summary started at %date% %time% === > "%LOG_FILE%"
|
||||
echo PROJECT_DIR=%PROJECT_DIR% >> "%LOG_FILE%"
|
||||
|
||||
cd /d "%PROJECT_DIR%"
|
||||
|
||||
REM 跑周汇总(4 路并发,动态时间范围)
|
||||
call %PYTHON% -u -X utf8 weekly_summary_all.py --max-workers 4 >> "%LOG_FILE%" 2>&1
|
||||
set RC=%ERRORLEVEL%
|
||||
echo weekly_summary exit=%RC% >> "%LOG_FILE%"
|
||||
|
||||
echo === Weekly summary finished at %date% %time% (rc=%RC%) === >> "%LOG_FILE%"
|
||||
|
||||
endlocal
|
||||
@@ -0,0 +1,437 @@
|
||||
"""Backfill per-note + per-style analyses into a single Feishu Docx archive.
|
||||
|
||||
Layout:
|
||||
- ONE archive doc (token stored under state.docs.archive)
|
||||
- Top: H1 款式索引 + table (sorted by note count DESC, intra-doc #style-N anchors)
|
||||
- Body: per-style H1 sections, each with:
|
||||
- H2 款式整体分析 (LLM rollup across all notes of the style)
|
||||
- H2 单篇笔记分析 (N 篇) containing per-note H3 sections
|
||||
|
||||
Idempotent: re-runs read state/feishu_doc.json and skip already-written notes + styles.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
# Path setup
|
||||
_TOOLS_DIR = PATHS.tools_root
|
||||
_PROJECT_ROOT = PATHS.module_root
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace") # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import db # noqa: E402
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import feishu_doc_writer as fdw # noqa: E402
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import analyze_note # noqa: E402
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import analyze_comments # noqa: E402
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import style_analyzer # noqa: E402
|
||||
|
||||
|
||||
# Serializes state mutation + state.json writes across worker threads.
|
||||
_state_lock = threading.Lock()
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Archive doc lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
def get_or_create_archive(state: dict[str, Any]) -> dict[str, Any]:
|
||||
docs = state.setdefault("docs", {})
|
||||
existing = docs.get("archive")
|
||||
if existing and existing.get("token"):
|
||||
return existing
|
||||
log(" Creating archive doc ...")
|
||||
info = fdw.create_doc(fdw._doc_skeleton())
|
||||
docs["archive"] = {
|
||||
"token": info["token"],
|
||||
"url": info["url"],
|
||||
"revision_id": info.get("revision_id"),
|
||||
}
|
||||
return docs["archive"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-note processing (within a style)
|
||||
# ---------------------------------------------------------------------------
|
||||
def process_note(
|
||||
note_row: dict[str, Any], *, archive_token: str, force: bool,
|
||||
state: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Per-note pipeline: load → filter → metrics → LLM analysis.
|
||||
|
||||
Returns the data needed for `process_style` to assemble the full style
|
||||
section XML. Does NOT append anything to the doc — per-note sections are
|
||||
appended only as part of the style section to keep doc order tidy.
|
||||
"""
|
||||
note_id = note_row["id"]
|
||||
with _state_lock:
|
||||
written = state.setdefault("written_notes", {}).get(str(note_id))
|
||||
if written and not force:
|
||||
return {"note_id": note_id, "status": "skipped", "reason": "already-written"}
|
||||
|
||||
try:
|
||||
info, raw_comments = analyze_note.load_note_from_db(note_id=note_id)
|
||||
except Exception as exc:
|
||||
return {"note_id": note_id, "status": "error", "stage": "load", "error": str(exc)}
|
||||
|
||||
info["note_id"] = note_id
|
||||
|
||||
try:
|
||||
filtered_comments = analyze_comments.code_filter_comments(raw_comments)
|
||||
except Exception as exc:
|
||||
return {"note_id": note_id, "status": "error", "stage": "filter", "error": str(exc)}
|
||||
|
||||
if not filtered_comments:
|
||||
return {"note_id": note_id, "status": "skipped", "reason": "no-filtered-comments"}
|
||||
|
||||
metrics = analyze_note.compute_metrics(info)
|
||||
target_terms = [t for t in (info.get("brand"), info.get("style_name")) if t]
|
||||
distribution = analyze_note.keyword_distribution(filtered_comments, target_terms=target_terms)
|
||||
|
||||
try:
|
||||
analysis = analyze_note.analyze_note(
|
||||
info, metrics, distribution, filtered_comments,
|
||||
)
|
||||
except Exception as exc:
|
||||
return {"note_id": note_id, "status": "error", "stage": "llm", "error": str(exc)}
|
||||
|
||||
# Persist the analysis text in state so re-runs can rebuild the doc without
|
||||
# re-calling the LLM (idempotency + crash recovery).
|
||||
with _state_lock:
|
||||
state["written_notes"][str(note_id)] = {
|
||||
"platform": info.get("platform"),
|
||||
"doc_token": archive_token,
|
||||
"bookmark_name": f"note-{note_id}",
|
||||
"style_id": note_row.get("style_id"),
|
||||
"analysis_text": analysis,
|
||||
"written_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
}
|
||||
fdw.save_state(state)
|
||||
return {
|
||||
"note_id": note_id,
|
||||
"status": "ok",
|
||||
"title": info.get("title", ""),
|
||||
"filtered_comments": filtered_comments,
|
||||
"info": info,
|
||||
"metrics": metrics,
|
||||
"distribution": distribution,
|
||||
"analysis": analysis,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-style processing
|
||||
# ---------------------------------------------------------------------------
|
||||
def process_style(
|
||||
style_row: dict[str, Any], state: dict[str, Any], archive_token: str, *,
|
||||
max_notes: int | None, force: bool, force_rollup: bool, workers: int,
|
||||
) -> tuple[int, int, int, bool]:
|
||||
style_id = style_row["id"]
|
||||
style_name = style_row["name"]
|
||||
|
||||
notes = db.get_notes_by_style(style_id, only_with_comments=True)
|
||||
if max_notes is not None:
|
||||
notes = notes[:max_notes]
|
||||
log(f"[{style_name}] notes to process: {len(notes)} (workers={workers})")
|
||||
|
||||
# ---- Phase 1: per-note parallel ----
|
||||
note_results: list[dict[str, Any]] = []
|
||||
ok = fail = skipped = 0
|
||||
with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
|
||||
future_to_note = {
|
||||
pool.submit(
|
||||
process_note,
|
||||
note, archive_token=archive_token, force=force,
|
||||
state=state,
|
||||
): note
|
||||
for note in notes
|
||||
}
|
||||
for future in as_completed(future_to_note):
|
||||
result = future.result()
|
||||
status = result.get("status")
|
||||
nid = result.get("note_id")
|
||||
if status == "ok":
|
||||
ok += 1
|
||||
log(f" [{style_name}] ok note_id={nid} title={result.get('title', '')[:30]!r}")
|
||||
note_results.append(result)
|
||||
elif status == "skipped":
|
||||
skipped += 1
|
||||
reason = result.get("reason")
|
||||
log(f" [{style_name}] skip note_id={nid} reason={reason}")
|
||||
# If it was previously written, we still want its data for the
|
||||
# rollup + style section rebuild. Re-derive from DB and use the
|
||||
# saved analysis_text from state (no LLM re-call).
|
||||
if reason == "already-written":
|
||||
try:
|
||||
with _state_lock:
|
||||
prev_entry = state.get("written_notes", {}).get(str(nid)) or {}
|
||||
prev_analysis = prev_entry.get("analysis_text")
|
||||
prev_info, prev_raw = analyze_note.load_note_from_db(note_id=nid)
|
||||
prev_info["note_id"] = nid
|
||||
prev_filtered = analyze_comments.code_filter_comments(prev_raw)
|
||||
if prev_filtered and prev_analysis:
|
||||
prev_metrics = analyze_note.compute_metrics(prev_info)
|
||||
prev_target = [t for t in (prev_info.get("brand"), prev_info.get("style_name")) if t]
|
||||
prev_dist = analyze_note.keyword_distribution(prev_filtered, target_terms=prev_target)
|
||||
note_results.append({
|
||||
"note_id": nid,
|
||||
"status": "ok",
|
||||
"title": prev_info.get("title", ""),
|
||||
"filtered_comments": prev_filtered,
|
||||
"info": prev_info,
|
||||
"metrics": prev_metrics,
|
||||
"distribution": prev_dist,
|
||||
"analysis": prev_analysis,
|
||||
})
|
||||
except Exception as exc:
|
||||
log(f" [{style_name}] WARN: could not re-derive note_id={nid} for rollup: {exc}")
|
||||
else:
|
||||
fail += 1
|
||||
log(f" [{style_name}] FAIL note_id={nid} stage={result.get('stage')} error={result.get('error')}")
|
||||
|
||||
if not note_results:
|
||||
log(f"[{style_name}] no successful notes — skipping rollup + style section")
|
||||
with _state_lock:
|
||||
state.setdefault("written_styles", {})[str(style_id)] = {
|
||||
"style_name": style_name,
|
||||
"doc_token": archive_token,
|
||||
"note_count": 0,
|
||||
"bookmark_name": f"style-{style_id}",
|
||||
"skipped": True,
|
||||
"written_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
}
|
||||
fdw.save_state(state)
|
||||
return ok, fail, skipped, True
|
||||
|
||||
# ---- Phase 2: style-level rollup ----
|
||||
notes_meta = note_results # already has {id, platform, title, like_count, comment_count, ...} via info
|
||||
# Reuse info rows (which already have id/platform/title/etc) for aggregation.
|
||||
notes_as_rows = [
|
||||
{
|
||||
"id": r["info"].get("note_id"),
|
||||
"platform": r["info"].get("platform"),
|
||||
"title": r["info"].get("title", ""),
|
||||
"like_count": r["info"].get("like_count"),
|
||||
"favorite_count": r["info"].get("favorite_count"),
|
||||
"comment_count": r["info"].get("comment_count_metric") or r["info"].get("comment_count_total"),
|
||||
"share_count": r["info"].get("share_count"),
|
||||
"view_count": r["info"].get("view_count"),
|
||||
"creator_name": r["info"].get("creator_name", ""),
|
||||
}
|
||||
for r in note_results
|
||||
]
|
||||
aggregated = style_analyzer.aggregate_style_metrics(notes_as_rows)
|
||||
merged_distribution = style_analyzer.merge_style_distribution(
|
||||
[r.get("distribution") or {} for r in note_results]
|
||||
)
|
||||
|
||||
# ---- Phase 3: decide if rollup needed ----
|
||||
skip_rollup = False
|
||||
with _state_lock:
|
||||
existing_style = state.setdefault("written_styles", {}).get(str(style_id))
|
||||
if existing_style and not force_rollup and not force:
|
||||
log(f"[{style_name}] rollup already written; skipping LLM call")
|
||||
rollup_text = existing_style.get("rollup_text", "（已有整体分析）")
|
||||
skip_rollup = True
|
||||
else:
|
||||
sample_comments = style_analyzer.sample_representative_comments(
|
||||
[r.get("filtered_comments") or [] for r in note_results],
|
||||
notes_as_rows,
|
||||
max_total=30,
|
||||
per_note_cap=2,
|
||||
)
|
||||
try:
|
||||
rollup_text = style_analyzer.analyze_style(
|
||||
style_name,
|
||||
brand="", # brand not currently carried in style_row; left empty
|
||||
aggregated=aggregated,
|
||||
merged_distribution=merged_distribution,
|
||||
sample_comments=sample_comments,
|
||||
note_summaries=[],
|
||||
)
|
||||
except Exception as exc:
|
||||
log(f"[{style_name}] FAIL rollup LLM: {exc}")
|
||||
rollup_text = f"（整体分析生成失败：{exc}）"
|
||||
|
||||
# ---- Phase 4: build style section XML and append to archive ----
|
||||
# The per-note XML sections must be built here (not in worker) because we
|
||||
# need the XML to assemble into the style section in one append call.
|
||||
# We can however reuse the `analysis` text already produced.
|
||||
note_sections_xml_parts: list[str] = []
|
||||
for r in note_results:
|
||||
info = r["info"]
|
||||
if not r.get("analysis"):
|
||||
log(f" [{style_name}] WARN: note_id={r['note_id']} has no analysis text; skipping in doc")
|
||||
continue
|
||||
note_xml = fdw.build_note_section_xml(
|
||||
info,
|
||||
r["metrics"],
|
||||
r["distribution"],
|
||||
# raw_comments not preserved across runs; pass filtered as fallback
|
||||
r.get("filtered_comments") or [],
|
||||
r.get("filtered_comments") or [],
|
||||
r["analysis"],
|
||||
bookmark_name=f"note-{r['note_id']}",
|
||||
doc_token=archive_token,
|
||||
)
|
||||
note_sections_xml_parts.append(note_xml)
|
||||
|
||||
style_section_xml = fdw.build_style_section_xml(
|
||||
style_id=style_id,
|
||||
style_name=style_name,
|
||||
aggregated=aggregated,
|
||||
rollup_text=rollup_text,
|
||||
note_sections_xml="".join(note_sections_xml_parts),
|
||||
doc_token=archive_token,
|
||||
)
|
||||
|
||||
try:
|
||||
fdw.append_blocks(archive_token, style_section_xml)
|
||||
except Exception as exc:
|
||||
log(f"[{style_name}] FAIL append style section: {exc}")
|
||||
return ok, fail, skipped, skip_rollup
|
||||
|
||||
with _state_lock:
|
||||
state["written_styles"][str(style_id)] = {
|
||||
"style_name": style_name,
|
||||
"doc_token": archive_token,
|
||||
"note_count": aggregated["note_count"],
|
||||
"bookmark_name": f"style-{style_id}",
|
||||
"rollup_written_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"rollup_text": rollup_text,
|
||||
}
|
||||
fdw.save_state(state)
|
||||
log(f"[{style_name}] DONE: notes={aggregated['note_count']} rollup={'skipped' if skip_rollup else 'new'}")
|
||||
return ok, fail, skipped, skip_rollup
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Index table (replace-on-update)
|
||||
# ---------------------------------------------------------------------------
|
||||
def populate_index_table(state: dict[str, Any], archive_token: str) -> None:
|
||||
archive = state.get("docs", {}).get("archive")
|
||||
if not archive:
|
||||
log(" WARN: no archive doc in state; skipping index population")
|
||||
return
|
||||
|
||||
written_styles = state.get("written_styles", {})
|
||||
# Sort by note_count DESC; alphabetical fallback for ties (already sorted in db.get_all_styles_with_notes)
|
||||
sorted_styles = dict(
|
||||
sorted(
|
||||
written_styles.items(),
|
||||
key=lambda kv: (-(kv[1].get("note_count") or 0), kv[1].get("style_name", "")),
|
||||
)
|
||||
)
|
||||
index_table_xml = fdw.build_style_index_table_xml(sorted_styles, archive_url=archive["url"])
|
||||
|
||||
# Anchor: H1 "款式索引" — we'll insert AFTER it.
|
||||
anchor = fdw.find_block_after(archive_token, "款式索引")
|
||||
if not anchor:
|
||||
log(" WARN: could not find anchor h1 '款式索引'; appending to end of archive doc")
|
||||
fdw.append_blocks(archive_token, index_table_xml)
|
||||
return
|
||||
|
||||
# Replace-on-update: delete ONLY the index blocks (between anchor and the
|
||||
# first H1 after it). Deleting everything after the anchor would nuke the
|
||||
# style sections we just wrote.
|
||||
stale_block_ids = fdw.find_index_blocks(archive_token, anchor)
|
||||
if stale_block_ids:
|
||||
log(f" Clearing {len(stale_block_ids)} stale index blocks ...")
|
||||
try:
|
||||
fdw.delete_blocks(archive_token, stale_block_ids)
|
||||
except Exception as exc:
|
||||
log(f" WARN: stale-block delete failed: {exc}; new index will be appended (may produce duplicates)")
|
||||
|
||||
fdw.insert_blocks_after(archive_token, anchor, index_table_xml)
|
||||
log(f" Inserted index table with {len(sorted_styles)} style rows")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
def run_all(args: argparse.Namespace) -> int:
|
||||
state = fdw.load_state()
|
||||
log(f"Loaded state: docs={list(state.get('docs', {}).keys())} "
|
||||
f"written_notes={len(state.get('written_notes', {}))} "
|
||||
f"written_styles={len(state.get('written_styles', {}))}")
|
||||
|
||||
archive = get_or_create_archive(state)
|
||||
fdw.save_state(state)
|
||||
log(f"Archive doc: {archive['url']}")
|
||||
|
||||
# Decide which styles to process.
|
||||
if args.style == "all":
|
||||
styles = db.get_all_styles_with_notes()
|
||||
else:
|
||||
# Single style by id
|
||||
style_id = int(args.style)
|
||||
styles = [s for s in db.get_all_styles_with_notes() if s["id"] == style_id]
|
||||
if not styles:
|
||||
log(f" WARN: style_id={style_id} not found or has no notes")
|
||||
return 1
|
||||
|
||||
total_ok = total_fail = total_skip = 0
|
||||
for style_row in styles:
|
||||
ok, fail, skipped, _ = process_style(
|
||||
style_row, state, archive["token"],
|
||||
max_notes=args.max_notes, force=args.force,
|
||||
force_rollup=args.force_rollup, workers=args.workers,
|
||||
)
|
||||
total_ok += ok
|
||||
total_fail += fail
|
||||
total_skip += skipped
|
||||
|
||||
log("Populating index table ...")
|
||||
try:
|
||||
populate_index_table(state, archive["token"])
|
||||
except Exception as exc:
|
||||
log(f" WARN: index table population failed: {exc}")
|
||||
fdw.save_state(state)
|
||||
|
||||
log(f"DONE: notes ok={total_ok} fail={total_fail} skipped={total_skip} "
|
||||
f"styles={len(state.get('written_styles', {}))}")
|
||||
log(f"Archive: {archive['url']}")
|
||||
return 0 if total_fail == 0 else 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Backfill note + style analyses into one Feishu Docx")
|
||||
parser.add_argument("--style", default="all",
|
||||
help="款式 id（如 8 = 宙斯），或 'all' 处理所有款式（默认）")
|
||||
parser.add_argument("--max-notes", type=int, default=None,
|
||||
help="每个款式最多处理 N 条（测试用）")
|
||||
parser.add_argument("--force", action="store_true",
|
||||
help="忽略 state 中已写入标记，强制重写所有笔记 + 整体分析")
|
||||
parser.add_argument("--force-rollup", action="store_true",
|
||||
help="仅重写款式整体分析（保留单篇笔记）")
|
||||
parser.add_argument("--workers", type=int, default=3,
|
||||
help="并行 worker 数（默认 3；设为 1 退化为串行）")
|
||||
parser.add_argument("--smoke-test", action="store_true",
|
||||
help="只跑权限冒烟测试，不写任何笔记")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.smoke_test:
|
||||
return 0 if fdw.permission_smoke_test() else 1
|
||||
|
||||
return run_all(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,766 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
from playwright.sync_api import Page
|
||||
from scrapling.fetchers import DynamicSession
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
STATE_DIR = PATHS.state_root
|
||||
NOTES_DIR = PATHS.raw_root / "notes" / "douyin"
|
||||
COOKIE_FILE = PATHS.browser_cookie_file
|
||||
PROFILE_DIR = PATHS.browser_profile_dir
|
||||
|
||||
# Douyin comment API endpoints to intercept
|
||||
COMMENT_API_MARKS = (
|
||||
"/aweme/v1/web/comment/list/",
|
||||
"/aweme/v1/web/comment/list/reply/",
|
||||
)
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {message}", flush=True)
|
||||
|
||||
|
||||
def load_cookies() -> list[dict[str, Any]]:
|
||||
if not COOKIE_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except json.JSONDecodeError:
|
||||
log(f"Cookie file is invalid, ignoring: {COOKIE_FILE}")
|
||||
return []
|
||||
|
||||
|
||||
def restore_cookies(page: Page) -> None:
|
||||
cookies = load_cookies()
|
||||
if not cookies:
|
||||
return
|
||||
try:
|
||||
page.context.add_cookies(cookies)
|
||||
log(f"Loaded cookies: {COOKIE_FILE}")
|
||||
except Exception as exc:
|
||||
log(f"Failed to load cookies, will continue with browser profile: {exc}")
|
||||
|
||||
|
||||
def save_state(page: Page) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
COOKIE_FILE.write_text(
|
||||
json.dumps(page.context.cookies(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
PATHS.browser_storage_state_file.write_text(
|
||||
json.dumps(page.context.storage_state(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
log(f"Saved cookies: {COOKIE_FILE}")
|
||||
|
||||
|
||||
def normalize_douyin_url(url: str) -> str:
|
||||
# Strip trailing Chinese text that sometimes gets appended to share URLs
|
||||
url = re.sub(r'(https?://v\.douyin\.com/[A-Za-z0-9_-]+/).*', r'\1', url)
|
||||
parts = urlsplit(url)
|
||||
params = dict(parse_qsl(parts.query, keep_blank_values=True))
|
||||
modal_id = params.get("modal_id")
|
||||
if modal_id and modal_id.isdigit():
|
||||
return f"https://www.douyin.com/video/{modal_id}"
|
||||
return url
|
||||
|
||||
|
||||
def resolve_aweme_id(url: str, *, page: Page | None = None) -> tuple[str, bool]:
|
||||
"""Extract aweme_id and is_note from a Douyin URL.
|
||||
|
||||
Returns (aweme_id, is_note). If *page* is provided and httpx redirect
|
||||
fails, falls back to navigating the short link inside the browser.
|
||||
"""
|
||||
parts = urlsplit(url)
|
||||
netloc = parts.netloc.lower()
|
||||
is_note = "/note/" in parts.path
|
||||
|
||||
m = re.search(r"/(?:video|note)/(\d+)", parts.path)
|
||||
if m:
|
||||
is_note = "/note/" in parts.path
|
||||
return m.group(1), is_note
|
||||
|
||||
if netloc.startswith("v.douyin.com") or netloc.startswith("www.douyin.com"):
|
||||
try:
|
||||
r = httpx.get(url, follow_redirects=True, timeout=15)
|
||||
final_url = str(r.url)
|
||||
m = re.search(r"/(?:video|note)/(\d+)", final_url)
|
||||
if m:
|
||||
is_note = "/note/" in final_url
|
||||
return m.group(1), is_note
|
||||
except Exception as exc:
|
||||
log(f"resolve_aweme_id: httpx redirect failed: {exc}")
|
||||
|
||||
# Fallback: resolve inside the browser
|
||||
if page is not None:
|
||||
try:
|
||||
log("resolve_aweme_id: trying browser redirect fallback...")
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
final_url = page.url
|
||||
m = re.search(r"/(?:video|note)/(\d+)", final_url)
|
||||
if m:
|
||||
is_note = "/note/" in final_url
|
||||
log(f"resolve_aweme_id: browser fallback ok, aweme_id={m.group(1)}, is_note={is_note}")
|
||||
return m.group(1), is_note
|
||||
except Exception as exc:
|
||||
log(f"resolve_aweme_id: browser redirect also failed: {exc}")
|
||||
|
||||
modal_id = dict(parse_qsl(parts.query, keep_blank_values=True)).get("modal_id")
|
||||
if modal_id and modal_id.isdigit():
|
||||
return modal_id, False
|
||||
|
||||
raise ValueError(f"Cannot extract aweme_id from URL: {url}")
|
||||
|
||||
|
||||
def parse_comment(comment: dict[str, Any], source_url: str, level: str, parent_id: str = "") -> dict[str, Any]:
|
||||
user = comment.get("user") or {}
|
||||
create_time = comment.get("create_time")
|
||||
created_at = ""
|
||||
if isinstance(create_time, (int, float)) and create_time:
|
||||
try:
|
||||
created_at = datetime.fromtimestamp(create_time).isoformat(timespec="seconds")
|
||||
except Exception:
|
||||
created_at = str(create_time)
|
||||
|
||||
return {
|
||||
"source_url": source_url,
|
||||
"level": level,
|
||||
"comment_id": comment.get("cid") or comment.get("comment_id") or "",
|
||||
"reply_to_comment_id": parent_id or comment.get("reply_id") or comment.get("reply_to_reply_id") or "",
|
||||
"nickname": user.get("nickname") or "",
|
||||
"user_id": user.get("uid") or "",
|
||||
"sec_uid": user.get("sec_uid") or "",
|
||||
"text": comment.get("text") or "",
|
||||
"digg_count": comment.get("digg_count") or 0,
|
||||
"reply_count": comment.get("reply_comment_total") or comment.get("reply_comment_count") or 0,
|
||||
"ip_label": comment.get("ip_label") or "",
|
||||
"created_at": created_at,
|
||||
"raw_create_time": create_time or "",
|
||||
}
|
||||
|
||||
|
||||
def merge_comment(
|
||||
row: dict[str, Any],
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
comment_id = str(row.get("comment_id") or f"{row.get('level')}:{len(comments_by_id)}")
|
||||
comments_by_id[comment_id] = row
|
||||
|
||||
|
||||
def collect_from_payload(
|
||||
payload: dict[str, Any],
|
||||
source_url: str,
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
stats: dict[str, Any],
|
||||
) -> None:
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
api_url = str(payload.get("__source_url", ""))
|
||||
is_reply = "/comment/list/reply/" in api_url
|
||||
|
||||
if not is_reply:
|
||||
stats["api_total"] = payload.get("total", stats.get("api_total"))
|
||||
stats["last_has_more"] = payload.get("has_more", stats.get("last_has_more"))
|
||||
stats["last_cursor"] = payload.get("cursor", stats.get("last_cursor"))
|
||||
|
||||
comments = payload.get("comments") or []
|
||||
if not isinstance(comments, list):
|
||||
return
|
||||
|
||||
parent_id = ""
|
||||
if is_reply:
|
||||
params = dict(parse_qsl(urlsplit(api_url).query))
|
||||
parent_id = params.get("comment_id") or ""
|
||||
|
||||
for comment in comments:
|
||||
if not isinstance(comment, dict):
|
||||
continue
|
||||
row = parse_comment(comment, source_url, "reply" if is_reply else "comment", parent_id)
|
||||
merge_comment(row, comments_by_id)
|
||||
|
||||
# Inline replies embedded in top-level comment responses
|
||||
inline_replies = comment.get("reply_comment") or []
|
||||
if isinstance(inline_replies, list):
|
||||
parent_cid = str(row.get("comment_id") or "")
|
||||
for reply in inline_replies:
|
||||
if isinstance(reply, dict):
|
||||
sub_row = parse_comment(reply, source_url, "reply", parent_cid)
|
||||
merge_comment(sub_row, comments_by_id)
|
||||
|
||||
|
||||
def setup_response_capture(page: Page, source_url: str) -> tuple[dict[str, dict[str, Any]], dict[str, Any]]:
|
||||
comments_by_id: dict[str, dict[str, Any]] = {}
|
||||
stats: dict[str, Any] = {"api_hits": 0, "api_total": None, "last_has_more": None, "last_cursor": None}
|
||||
|
||||
def handle_response(response: Any) -> None:
|
||||
url = response.url
|
||||
if not any(mark in url for mark in COMMENT_API_MARKS):
|
||||
return
|
||||
try:
|
||||
payload = response.json()
|
||||
except Exception:
|
||||
return
|
||||
if isinstance(payload, dict):
|
||||
payload["__source_url"] = url
|
||||
stats["api_hits"] += 1
|
||||
# Save first-seen URL as template for direct reply fetching
|
||||
if "/comment/list/" in url and "/reply/" not in url and not stats.get("comment_url_template"):
|
||||
stats["comment_url_template"] = url
|
||||
if "/comment/list/reply/" in url and not stats.get("reply_url_template"):
|
||||
stats["reply_url_template"] = url
|
||||
collect_from_payload(payload, source_url, comments_by_id, stats)
|
||||
|
||||
page.on("response", handle_response)
|
||||
return comments_by_id, stats
|
||||
|
||||
|
||||
def click_if_visible(page: Page, text: str, timeout: int = 1200) -> bool:
|
||||
try:
|
||||
locator = page.get_by_text(text, exact=False).first
|
||||
if locator.count():
|
||||
locator.click(timeout=timeout)
|
||||
page.wait_for_timeout(500)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def dismiss_popups(page: Page) -> None:
|
||||
for text in ("我知道了", "稍后再说", "暂不登录", "取消", "跳过"):
|
||||
click_if_visible(page, text)
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const close = [...document.querySelectorAll('button, div, span')]
|
||||
.filter((el) => {
|
||||
const t = (el.innerText || '').trim();
|
||||
const r = el.getBoundingClientRect();
|
||||
return ['×', 'x', 'X'].includes(t) &&
|
||||
r.width > 0 && r.width <= 52 &&
|
||||
r.height > 0 && r.height <= 52 &&
|
||||
r.top >= 0 && r.top < window.innerHeight * 0.75;
|
||||
})[0];
|
||||
close?.click();
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def maybe_wait_for_login(page: Page, login_timeout: int) -> None:
|
||||
deadline = time.time() + login_timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
body = page.locator("body").inner_text(timeout=5000)
|
||||
except Exception:
|
||||
body = ""
|
||||
if "扫码登录" not in body and "登录后" not in body:
|
||||
return
|
||||
log("Douyin may require login. Please finish login in the opened Chrome window.")
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
|
||||
def is_logged_in(page: Page) -> bool:
|
||||
"""抖音登录态靠 sessionid cookie,有它才能采全量评论。"""
|
||||
for c in page.context.cookies():
|
||||
if c.get("name") == "sessionid" and c.get("value"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def expand_visible_replies(page: Page) -> int:
|
||||
try:
|
||||
clicked = page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const visible = (el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(el);
|
||||
return r.width > 0 && r.height > 0 &&
|
||||
r.bottom > 0 && r.top < window.innerHeight &&
|
||||
style.visibility !== 'hidden' &&
|
||||
style.display !== 'none' &&
|
||||
style.opacity !== '0';
|
||||
};
|
||||
const candidates = [...document.querySelectorAll('button, div, span')]
|
||||
.filter((el) => {
|
||||
const text = (el.innerText || '').trim();
|
||||
return visible(el) &&
|
||||
text.length <= 40 &&
|
||||
(/展开.*回复/.test(text) || /查看.*回复/.test(text) || /共.*条回复/.test(text) || /还有.*回复/.test(text));
|
||||
});
|
||||
let count = 0;
|
||||
for (const el of candidates.slice(0, 8)) {
|
||||
el.click();
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
"""
|
||||
)
|
||||
if clicked:
|
||||
page.wait_for_timeout(1200)
|
||||
return int(clicked or 0)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def scroll_comments(page: Page, max_scrolls: int, idle_rounds: int, comments_by_id: dict[str, Any]) -> None:
|
||||
last_count = len(comments_by_id)
|
||||
idle = 0
|
||||
page.mouse.move(1030, 620)
|
||||
|
||||
for index in range(max_scrolls):
|
||||
dismiss_popups(page)
|
||||
expand_visible_replies(page)
|
||||
page.mouse.wheel(0, 1000)
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const candidates = [...document.querySelectorAll('div, section, aside')]
|
||||
.filter((el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
return r.height > 160 &&
|
||||
r.width > 220 &&
|
||||
el.scrollHeight > el.clientHeight + 80 &&
|
||||
r.right > window.innerWidth * 0.35;
|
||||
})
|
||||
.sort((a, b) => (b.scrollHeight - b.clientHeight) - (a.scrollHeight - a.clientHeight));
|
||||
for (const el of candidates.slice(0, 6)) {
|
||||
el.scrollTop += 1000;
|
||||
}
|
||||
}
|
||||
"""
|
||||
)
|
||||
page.wait_for_timeout(1800)
|
||||
|
||||
count = len(comments_by_id)
|
||||
if count == last_count:
|
||||
idle += 1
|
||||
else:
|
||||
log(f"Loaded comments: {count}")
|
||||
idle = 0
|
||||
last_count = count
|
||||
if idle >= idle_rounds:
|
||||
log(f"No new comments after {idle_rounds} rounds, stopping scroll.")
|
||||
break
|
||||
|
||||
log(f"Scroll finished after {index + 1} rounds.")
|
||||
|
||||
|
||||
def with_query_params(url: str, updates: dict[str, Any]) -> str:
|
||||
parts = urlsplit(url)
|
||||
params = dict(parse_qsl(parts.query, keep_blank_values=True))
|
||||
for key, value in updates.items():
|
||||
params[key] = str(value)
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(params), parts.fragment))
|
||||
|
||||
|
||||
def fetch_missing_top_comments(
|
||||
page: Page,
|
||||
source_url: str,
|
||||
aweme_id: str,
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
stats: dict[str, Any],
|
||||
) -> None:
|
||||
"""After scrolling, proactively paginate remaining top-level comments via API."""
|
||||
template = stats.get("comment_url_template")
|
||||
if not template:
|
||||
log("No comment API template captured; skipped direct top-level fetch.")
|
||||
return
|
||||
|
||||
api_total = stats.get("api_total")
|
||||
# Fall back to note_metrics.comment_count if api_total is missing
|
||||
if not api_total:
|
||||
metrics = stats.get("note_metrics") or {}
|
||||
api_total = metrics.get("comment_count")
|
||||
if api_total and len(comments_by_id) >= int(api_total):
|
||||
return
|
||||
|
||||
last_cursor = stats.get("last_cursor")
|
||||
if not last_cursor:
|
||||
return
|
||||
|
||||
log(f"Top-level: collected {len(comments_by_id)} / api_total {api_total}; fetching remainder.")
|
||||
fetched = 0
|
||||
cursor = last_cursor
|
||||
for _ in range(100):
|
||||
top_url = with_query_params(template, {"cursor": cursor, "aweme_id": aweme_id})
|
||||
try:
|
||||
payload = page.evaluate(
|
||||
"""
|
||||
async (url) => {
|
||||
const res = await fetch(url, { credentials: 'include' });
|
||||
return await res.json();
|
||||
}
|
||||
""",
|
||||
top_url,
|
||||
)
|
||||
except Exception:
|
||||
break
|
||||
if not isinstance(payload, dict):
|
||||
break
|
||||
payload["__source_url"] = top_url
|
||||
before = len(comments_by_id)
|
||||
collect_from_payload(payload, source_url, comments_by_id, stats)
|
||||
fetched += max(0, len(comments_by_id) - before)
|
||||
|
||||
has_more = payload.get("has_more")
|
||||
next_cursor = payload.get("cursor")
|
||||
if not has_more or next_cursor in (None, cursor):
|
||||
break
|
||||
cursor = next_cursor
|
||||
page.wait_for_timeout(350)
|
||||
|
||||
log(f"Top-level direct fetch: {fetched} new comments; total now: {len(comments_by_id)}")
|
||||
|
||||
|
||||
def fetch_missing_replies(
|
||||
page: Page,
|
||||
source_url: str,
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
stats: dict[str, Any],
|
||||
) -> None:
|
||||
template = stats.get("reply_url_template")
|
||||
if not template:
|
||||
log("No reply API template captured; skipped direct reply fetch.")
|
||||
return
|
||||
|
||||
parents = [
|
||||
row for row in list(comments_by_id.values())
|
||||
if row.get("level") == "comment" and int(row.get("reply_count") or 0) > 0
|
||||
]
|
||||
if not parents:
|
||||
return
|
||||
|
||||
log(f"Fetching replies directly for {len(parents)} comments with replies.")
|
||||
fetched = 0
|
||||
for row in parents:
|
||||
parent_id = str(row.get("comment_id") or "")
|
||||
if not parent_id:
|
||||
continue
|
||||
cursor = 0
|
||||
for _ in range(30):
|
||||
reply_url = with_query_params(template, {
|
||||
"item_id": stats.get("aweme_id", ""),
|
||||
"comment_id": parent_id,
|
||||
"cursor": cursor,
|
||||
})
|
||||
try:
|
||||
payload = page.evaluate(
|
||||
"""
|
||||
async (url) => {
|
||||
const res = await fetch(url, { credentials: 'include' });
|
||||
return await res.json();
|
||||
}
|
||||
""",
|
||||
reply_url,
|
||||
)
|
||||
except Exception:
|
||||
break
|
||||
if not isinstance(payload, dict):
|
||||
break
|
||||
payload["__source_url"] = reply_url
|
||||
before = len(comments_by_id)
|
||||
collect_from_payload(payload, source_url, comments_by_id, stats)
|
||||
fetched += max(0, len(comments_by_id) - before)
|
||||
has_more = payload.get("has_more")
|
||||
next_cursor = payload.get("cursor")
|
||||
if not has_more or next_cursor in (None, cursor):
|
||||
break
|
||||
cursor = next_cursor
|
||||
page.wait_for_timeout(350)
|
||||
|
||||
log(f"Direct replies fetched/merged: {fetched}; total comments now: {len(comments_by_id)}")
|
||||
|
||||
|
||||
def extract_note_metrics(page: Page) -> dict[str, Any]:
|
||||
"""Extract video metrics from the Douyin page via DOM data-e2e attributes."""
|
||||
try:
|
||||
metrics = page.evaluate(
|
||||
r"""
|
||||
() => {
|
||||
const compactNumber = (text) => {
|
||||
text = String(text || '').replace(/,/g, '').trim();
|
||||
const match = text.match(/(\d+(?:\.\d+)?)(\s*[万wW])?/);
|
||||
if (!match) return 0;
|
||||
const value = parseFloat(match[1]);
|
||||
if (!Number.isFinite(value)) return 0;
|
||||
return match[2] ? Math.round(value * 10000) : Math.round(value);
|
||||
};
|
||||
|
||||
const titleEl = document.querySelector('[data-e2e="video-title"]') || document.querySelector('h1');
|
||||
const title = titleEl ? titleEl.innerText.trim() : '';
|
||||
|
||||
const likeEl = document.querySelector('[data-e2e="video-player-digg"]');
|
||||
const commentEl = document.querySelector('[data-e2e="feed-comment-icon"]');
|
||||
const collectEl = document.querySelector('[data-e2e="video-player-collect"]');
|
||||
const shareEl = document.querySelector('[data-e2e="video-player-share"]');
|
||||
|
||||
const like_count = likeEl ? compactNumber(likeEl.innerText) : 0;
|
||||
const comment_count = commentEl ? compactNumber(commentEl.innerText) : 0;
|
||||
const favorite_count = collectEl ? compactNumber(collectEl.innerText) : 0;
|
||||
const share_count = shareEl ? compactNumber(shareEl.innerText) : 0;
|
||||
|
||||
if (!like_count && !comment_count) return {};
|
||||
return {title, like_count, comment_count, favorite_count, share_count, view_count: 0};
|
||||
}
|
||||
"""
|
||||
)
|
||||
return metrics if isinstance(metrics, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def safe_filename(value: str) -> str:
|
||||
value = re.sub(r"[<>:\"/\\|?*\s]+", "_", value).strip("_")
|
||||
return value[:80] or "douyin_comments"
|
||||
|
||||
|
||||
def scrape_comments(url: str, login_timeout: int, max_scrolls: int, idle_rounds: int, headless: bool) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
url = normalize_douyin_url(url)
|
||||
result: dict[str, Any] = {"comments": [], "stats": {}}
|
||||
|
||||
# Try httpx first (fast); if it fails, we'll retry inside the browser
|
||||
aweme_id: str | None = None
|
||||
is_note: bool = False
|
||||
try:
|
||||
aweme_id, is_note = resolve_aweme_id(url)
|
||||
except ValueError as exc:
|
||||
log(str(exc))
|
||||
|
||||
if aweme_id:
|
||||
log(f"aweme_id: {aweme_id}{' (note)' if is_note else ''}")
|
||||
|
||||
def action(page: Page) -> None:
|
||||
nonlocal aweme_id, is_note
|
||||
|
||||
# If httpx couldn't resolve the short link, use the browser as fallback
|
||||
if aweme_id is None:
|
||||
try:
|
||||
aweme_id, is_note = resolve_aweme_id(url, page=page)
|
||||
log(f"aweme_id (browser fallback): {aweme_id}{' (note)' if is_note else ''}")
|
||||
except ValueError as exc:
|
||||
log(str(exc))
|
||||
return # Cannot proceed without aweme_id
|
||||
|
||||
# Always set up response capture first to intercept API calls
|
||||
comments_by_id, stats = setup_response_capture(page, url)
|
||||
stats["aweme_id"] = aweme_id
|
||||
|
||||
# Use the resolved full URL instead of the original short link
|
||||
resolved_url = f"https://www.douyin.com/note/{aweme_id}" if is_note else f"https://www.douyin.com/video/{aweme_id}"
|
||||
page.goto(resolved_url, wait_until="domcontentloaded", timeout=90000)
|
||||
page.wait_for_timeout(6000)
|
||||
maybe_wait_for_login(page, login_timeout)
|
||||
dismiss_popups(page)
|
||||
save_state(page)
|
||||
|
||||
# Wait briefly for the first comment API response to capture template URLs
|
||||
for _ in range(10):
|
||||
if stats.get("comment_url_template") and stats.get("api_total") is not None:
|
||||
break
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
total_comments = stats.get("api_total")
|
||||
if total_comments is not None:
|
||||
log(f"Comment count from API: {total_comments}")
|
||||
else:
|
||||
log("Could not determine comment count from API response")
|
||||
|
||||
# Scroll to collect comments (same pattern as xiaohongshu scraper)
|
||||
scroll_comments(page, max_scrolls, idle_rounds, comments_by_id)
|
||||
|
||||
# After scrolling, extract metrics and use comment_count as fallback for api_total
|
||||
stats["note_metrics"] = extract_note_metrics(page)
|
||||
result["title"] = stats["note_metrics"].get("title", "")
|
||||
|
||||
# Proactively paginate remaining top-level comments via API
|
||||
fetch_missing_top_comments(page, url, aweme_id, comments_by_id, stats)
|
||||
|
||||
# Fetch missing sub-replies via API
|
||||
fetch_missing_replies(page, url, comments_by_id, stats)
|
||||
|
||||
page.wait_for_timeout(1000)
|
||||
result["comments"] = list(comments_by_id.values())
|
||||
result["stats"] = stats
|
||||
# logged_in 判定:cookie 实际有效但抖音反爬可能在抓取过程中清掉 sessionid,
|
||||
# 改成"采集率 >= 90% 就算登录有效",避免误报噪音。
|
||||
actual_count = stats.get("api_total")
|
||||
if actual_count is None:
|
||||
note_metrics = stats.get("note_metrics") or {}
|
||||
actual_count = note_metrics.get("comment_count")
|
||||
collected_count = len(result["comments"])
|
||||
if actual_count and collected_count >= int(actual_count * 0.9):
|
||||
result["stats"]["logged_in"] = True
|
||||
else:
|
||||
result["stats"]["logged_in"] = is_logged_in(page)
|
||||
|
||||
result["final_url"] = page.url
|
||||
save_state(page)
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=headless,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
session.fetch(url, page_action=action, wait=1000)
|
||||
|
||||
# aweme_id 解析完全失败 → 抛异常让 batch 层处理
|
||||
if aweme_id is None:
|
||||
raise RuntimeError(f"Cannot extract aweme_id from URL: {url}")
|
||||
|
||||
# 静默失败检测:0 评论 + 空 note_metrics → 页面没加载成功
|
||||
stats = result.get("stats", {})
|
||||
note_metrics = stats.get("note_metrics", {})
|
||||
if not result["comments"] and not note_metrics:
|
||||
raise RuntimeError(
|
||||
"page not loaded: 0 comments and no metrics "
|
||||
"(likely cookie expired or rate-limited)"
|
||||
)
|
||||
|
||||
return result["comments"], result
|
||||
|
||||
|
||||
def write_outputs(url: str, comments: list[dict[str, Any]], result: dict[str, Any]) -> tuple[Path, Path]:
|
||||
NOTES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
m = re.search(r"/(?:video|note)/(\d+)", urlsplit(url).path)
|
||||
aweme_id = m.group(1) if m else ""
|
||||
modal = re.search(r"modal_id=(\d+)", url)
|
||||
if aweme_id:
|
||||
name = f"douyin_comments_{aweme_id}"
|
||||
elif modal:
|
||||
name = f"douyin_comments_{modal.group(1)}"
|
||||
else:
|
||||
name = safe_filename(url)
|
||||
json_path = NOTES_DIR / f"{name}.json"
|
||||
csv_path = NOTES_DIR / f"{name}.csv"
|
||||
if not comments and json_path.exists() and csv_path.exists() and not result.get("title"):
|
||||
log(f"No comments and no new title; keeping existing output files: {json_path}, {csv_path}")
|
||||
return json_path, csv_path
|
||||
|
||||
stats = result.get("stats", {})
|
||||
public_stats = {key: value for key, value in stats.items() if not key.endswith("_template")}
|
||||
payload = {
|
||||
"source_url": url,
|
||||
"title": result.get("title", ""),
|
||||
"scraped_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"count": len(comments),
|
||||
"stats": public_stats,
|
||||
"comments": comments,
|
||||
}
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
fieldnames = [
|
||||
"source_url",
|
||||
"level",
|
||||
"comment_id",
|
||||
"reply_to_comment_id",
|
||||
"nickname",
|
||||
"user_id",
|
||||
"sec_uid",
|
||||
"text",
|
||||
"digg_count",
|
||||
"reply_count",
|
||||
"ip_label",
|
||||
"created_at",
|
||||
"raw_create_time",
|
||||
]
|
||||
with csv_path.open("w", newline="", encoding="utf-8-sig") as file:
|
||||
writer = csv.DictWriter(file, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(comments)
|
||||
|
||||
return json_path, csv_path
|
||||
|
||||
|
||||
def login_only(login_timeout: int) -> int:
|
||||
"""只触发登录(headed),登录完成立刻退出。给 relogin_douyin.py 用。"""
|
||||
def action(page: Page) -> None:
|
||||
page.goto("https://www.douyin.com/", wait_until="domcontentloaded", timeout=90000)
|
||||
page.wait_for_timeout(2500)
|
||||
# 主动触发登录弹窗(点登录按钮)
|
||||
try:
|
||||
login_btn = page.get_by_text("登录", exact=True).first
|
||||
if login_btn.count():
|
||||
login_btn.click(timeout=3000)
|
||||
page.wait_for_timeout(1500)
|
||||
except Exception:
|
||||
pass
|
||||
maybe_wait_for_login(page, login_timeout)
|
||||
save_state(page)
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=False,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
session.fetch("https://www.douyin.com/", page_action=action, wait=1000)
|
||||
log("[login-only] 完成")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Scrape Douyin video comments via browser (Playwright + XHR interception).")
|
||||
parser.add_argument("url", help="Douyin video/user URL (传 'login' 配合 --login-only)")
|
||||
parser.add_argument("--login-timeout", type=int, default=300, help="等待扫码/登录秒数")
|
||||
parser.add_argument("--max-scrolls", type=int, default=120, help="最多滚动加载次数")
|
||||
parser.add_argument("--idle-rounds", type=int, default=8, help="连续几轮无新评论后停止")
|
||||
parser.add_argument("--headless", action="store_true", help="无头模式；首次登录不要使用")
|
||||
parser.add_argument("--login-only", action="store_true",
|
||||
help="只触发登录(headed)然后退出,给 relogin_douyin 用")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.login_only:
|
||||
return login_only(args.login_timeout)
|
||||
|
||||
comments, result = scrape_comments(args.url, args.login_timeout, args.max_scrolls, args.idle_rounds, args.headless)
|
||||
json_path, csv_path = write_outputs(args.url, comments, result)
|
||||
log(f"Final URL: {result.get('final_url', '')}")
|
||||
log(f"API hits: {result.get('stats', {}).get('api_hits', 0)}")
|
||||
log(f"Scraped {len(comments)} comments")
|
||||
log(f"JSON: {json_path}")
|
||||
log(f"CSV: {csv_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,427 @@
|
||||
"""
|
||||
feishu_mapping.py — 动态读取「合作达人」/「自营达人」多维表格地址,重建款式对照表
|
||||
=================================================================================
|
||||
|
||||
之前每个款对应的飞书表格是写死在 data/config/款式_多维表格_对照.json 里的,
|
||||
现在改成动态读取下面这张"合作达人"多维表格地址表:
|
||||
|
||||
https://bu0zgpibak.feishu.cn/base/TtoCb1NuQaDy3NsZWTpc0GIvnph?table=tblKCjplVAFrRwMC
|
||||
|
||||
这张表里每个款式一行,包含「款式」和「合作达人多维表格地址」两个字段。
|
||||
凡是款式 和 合作达人多维表格地址 都不为空的一行,就去解析地址里的 base_token + table_id,
|
||||
再拉该表的字段定义 (+field-list) 构建 field_map,最终拼出和原 款式_多维表格_对照.json
|
||||
完全相同结构的 mapping:
|
||||
{ "_comment":..., "platform":..., "tables":[ {index,name,base_token,table_id,url,field_map,all_field_names,total_records}, ... ] }
|
||||
|
||||
自营达人: 同一张索引表新增了「自营合作达人表格地址」列,结构完全相同,
|
||||
输出到 data/config/款式_多维表格_对照_自营.json。
|
||||
|
||||
策略:
|
||||
- 拉取后写回 data/config/款式_多维表格_对照.json (合作) 或 款式_多维表格_对照_自营.json (自营)
|
||||
- 带 TTL 缓存(默认 6 小时):缓存新鲜就直接读缓存,避免每次跑都打几十个 lark-cli 请求
|
||||
- lark-cli 拉取失败时退回缓存文件,保证可运行
|
||||
- index 编号:已知款式沿用旧 JSON 里的编号,新款式按出现顺序往后追加(保证历史文件名稳定)
|
||||
|
||||
被三方脚本 (bilibili / pgy / xingtu) 通过 load_mapping() 调用;
|
||||
run_all.py 启动前可调 refresh_mapping(force=True) 统一刷新一次,避免 3 个并行进程重复刷新打架。
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import (
|
||||
PATHS,
|
||||
resolve_layer_output,
|
||||
)
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
MAPPING_DIR = PATHS.normalized_root / "mappings"
|
||||
MAPPING_PATH = MAPPING_DIR / "款式_多维表格_对照.json"
|
||||
SELF_MAPPING_PATH = MAPPING_DIR / "款式_多维表格_对照_自营.json"
|
||||
MAPPING_SEED_PATH = PATHS.config_root / "款式_多维表格_对照.json"
|
||||
SELF_MAPPING_SEED_PATH = PATHS.config_root / "款式_多维表格_对照_自营.json"
|
||||
|
||||
LARK_CLI = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd")
|
||||
if not os.path.exists(LARK_CLI):
|
||||
LARK_CLI = "lark-cli.cmd"
|
||||
|
||||
# 「合作达人」多维表格地址表(款式 → 各自合作达人多维表格 URL)
|
||||
INDEX_BASE_TOKEN = "TtoCb1NuQaDy3NsZWTpc0GIvnph"
|
||||
INDEX_TABLE_ID = "tblKCjplVAFrRwMC"
|
||||
# 这张地址表里的关键字段名
|
||||
INDEX_STYLE_FIELD = "款式"
|
||||
INDEX_URL_FIELD = "合作达人多维表格地址"
|
||||
INDEX_SELF_URL_FIELD = "自营合作达人表格地址"
|
||||
|
||||
# 缓存新鲜阈值(秒),默认 6 小时
|
||||
CACHE_TTL = 6 * 3600
|
||||
|
||||
# 标准字段名 → 逻辑名 (与 rebuild_mapping.py 保持一致)
|
||||
STD = {
|
||||
"达人名称": "creator_name",
|
||||
"达人昵称": "creator_name",
|
||||
"达人ID": "creator_id",
|
||||
"达人id": "creator_id",
|
||||
"达人id号": "creator_id",
|
||||
"星图ID": "creator_id",
|
||||
"小红书号": "xhs_id",
|
||||
"发布笔记标题": "note_title",
|
||||
"笔记标题": "note_title",
|
||||
"标题": "note_title",
|
||||
"主页链接": "profile_url",
|
||||
"发布笔记链接": "note_url",
|
||||
"发布链接": "note_url",
|
||||
"笔记链接": "note_url",
|
||||
"发布时间": "publish_time",
|
||||
"投放平台": "platform",
|
||||
"合作费用": "cost",
|
||||
"快递单号": "tracking_no",
|
||||
"7天曝光量": "read_count_7d",
|
||||
"14天曝光量": "read_count_14d",
|
||||
"21天曝光量": "read_count_21d",
|
||||
"28天曝光量": "read_count_28d",
|
||||
"曝光量（自动抓取）": "read_count_auto",
|
||||
"60天曝光量": "read_count_60d",
|
||||
"90天曝光量": "read_count_90d",
|
||||
"120天曝光量": "read_count_120d",
|
||||
"150天曝光量": "read_count_150d",
|
||||
"180天曝光量": "read_count_180d",
|
||||
"月底曝光量": "month_end",
|
||||
"达人粉丝": "follower_count",
|
||||
"微信号": "wechat",
|
||||
"投放账号类型": "account_type",
|
||||
"观看画像": "audience_profile",
|
||||
"数据表现": "data_performance",
|
||||
"发布笔记内容方向": "content_direction",
|
||||
"互动赞藏数": "engagement_count",
|
||||
"投流金额": "ad_spend",
|
||||
"制单日期": "cooperation_date",
|
||||
"审稿时间": "review_time",
|
||||
"是否结款": "is_paid",
|
||||
"是否新方向验证": "is_new_direction",
|
||||
"cpm": "cpm",
|
||||
}
|
||||
|
||||
_URL_RE = re.compile(r"https?://[^\s)>\]]+")
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
safe = str(msg).encode("gbk", errors="replace").decode("gbk", errors="replace")
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] [feishu_mapping] {safe}", flush=True)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# lark-cli 调用
|
||||
# ============================================================
|
||||
def call_lark_json(args: list[str]) -> dict:
|
||||
cmd = [LARK_CLI] + args
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace")
|
||||
text = proc.stdout if proc.stdout else (proc.stderr or "")
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
return {"ok": False, "error": f"no json: {text[:300]}"}
|
||||
try:
|
||||
return json.loads(text[start:], strict=False)
|
||||
except json.JSONDecodeError as e:
|
||||
return {"ok": False, "error": f"json decode: {e}"}
|
||||
|
||||
|
||||
def _field_list(base_token: str, table_id: str) -> list[dict]:
|
||||
resp = call_lark_json([
|
||||
"base", "+field-list",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--format", "json",
|
||||
])
|
||||
if not resp.get("ok"):
|
||||
_log(f" [WARN] +field-list 失败 {base_token}/{table_id}: {resp.get('error')}")
|
||||
return []
|
||||
return resp.get("data", {}).get("fields", [])
|
||||
|
||||
|
||||
def _record_list_all(base_token: str, table_id: str,
|
||||
field_ids: list[str] | None = None) -> list[dict]:
|
||||
"""拉一张表全部记录,自动分页。field_ids 不传=全字段。"""
|
||||
all_records = []
|
||||
offset = 0
|
||||
base_args = [
|
||||
"base", "+record-list",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--limit", "200",
|
||||
]
|
||||
while True:
|
||||
args = base_args + ["--offset", str(offset)]
|
||||
if field_ids:
|
||||
for fid in field_ids:
|
||||
args += ["--field-id", fid]
|
||||
resp = call_lark_json(args)
|
||||
if not resp.get("ok"):
|
||||
raise RuntimeError(f"record-list 失败: {resp.get('error')}")
|
||||
d = resp["data"]
|
||||
ids = d["field_id_list"]
|
||||
rids = d["record_id_list"]
|
||||
rows = d["data"]
|
||||
for i, row in enumerate(rows):
|
||||
rec = {"record_id": rids[i]}
|
||||
for j, fid in enumerate(ids):
|
||||
rec[fid] = row[j] if j < len(row) else None
|
||||
all_records.append(rec)
|
||||
offset += len(rows)
|
||||
if not d.get("has_more", False) or len(rows) == 0:
|
||||
break
|
||||
return all_records
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 地址解析 & field_map 构建
|
||||
# ============================================================
|
||||
def parse_style_url(raw) -> tuple[str | None, str | None, str]:
|
||||
"""从飞书多维表格地址单元格(可能是 markdown 链接或纯 URL)解析出
|
||||
(base_token, table_id, url)。没有 table 参数时返回 (bt, None, url)。"""
|
||||
if raw is None:
|
||||
return None, None, ""
|
||||
if isinstance(raw, list):
|
||||
raw = " ".join(str(x) for x in raw if x)
|
||||
s = str(raw).strip()
|
||||
if not s:
|
||||
return None, None, ""
|
||||
m = _URL_RE.search(s)
|
||||
url = m.group(0) if m else s
|
||||
bt = None
|
||||
mb = re.search(r"/base/([A-Za-z0-9]+)", url)
|
||||
if mb:
|
||||
bt = mb.group(1)
|
||||
tid = None
|
||||
mt = re.search(r"[?&]table=([A-Za-z0-9]+)", url)
|
||||
if mt:
|
||||
tid = mt.group(1)
|
||||
return bt, tid, url
|
||||
|
||||
|
||||
def build_field_map(fields: list[dict]) -> tuple[dict, list[str]]:
|
||||
field_map = {}
|
||||
all_field_names = []
|
||||
for f in fields:
|
||||
fid = f.get("id")
|
||||
fname = f.get("name")
|
||||
all_field_names.append(fname)
|
||||
clean_name = fname.rstrip("?").rstrip() if fname else ""
|
||||
logical = STD.get(fname) or STD.get(clean_name)
|
||||
if not logical and clean_name:
|
||||
for k, v in STD.items():
|
||||
# 只做正向匹配: 字段名以已知 key 开头(如 "7天曝光量(自动)" 以 "7天曝光量" 开头)
|
||||
# 不做反向匹配(k.startswith(clean_name)): 单字如"曝"会误匹配多个字段
|
||||
if clean_name.startswith(k):
|
||||
logical = v
|
||||
break
|
||||
if logical and logical not in field_map:
|
||||
field_map[logical] = {"field_id": fid, "field_name": fname}
|
||||
return field_map, all_field_names
|
||||
|
||||
|
||||
def _norm_url(base_token: str, table_id: str, url: str) -> str:
|
||||
if url:
|
||||
return url
|
||||
return (f"https://bu0zgpibak.feishu.cn/base/{base_token}"
|
||||
f"?table={table_id}&view=vewStRNML4")
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 重建 mapping
|
||||
# ============================================================
|
||||
def refresh_mapping(force: bool = True, ttl: float = CACHE_TTL,
|
||||
mapping_path: Path | None = None,
|
||||
write: bool = True,
|
||||
url_field: str = INDEX_URL_FIELD) -> dict | None:
|
||||
"""动态读取地址表,重建 mapping 并写缓存。
|
||||
返回新的 mapping;失败返回 None(缓存仍可用时下游自行读缓存)。
|
||||
write=False 时只构建内存中的 mapping,不落盘(供 rebuild_mapping.py --diff 用)。
|
||||
url_field: 从索引表读哪一列 URL — 默认 INDEX_URL_FIELD(合作达人),
|
||||
传 INDEX_SELF_URL_FIELD 则读自营达人列。
|
||||
"""
|
||||
is_self = url_field == INDEX_SELF_URL_FIELD
|
||||
path = resolve_layer_output(
|
||||
mapping_path or (SELF_MAPPING_PATH if is_self else MAPPING_PATH),
|
||||
layer_root=PATHS.normalized_root,
|
||||
field="mapping_path",
|
||||
)
|
||||
seed_path = SELF_MAPPING_SEED_PATH if is_self else MAPPING_SEED_PATH
|
||||
if write:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not force and path.exists():
|
||||
age = time.time() - path.stat().st_mtime
|
||||
if age < ttl:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
# 旧缓存用于:已知款式的 index 编号沿用 + platform 字段保留
|
||||
old_by_name: dict[str, dict] = {}
|
||||
old_platform = "xhs_pgy"
|
||||
previous_path = path if path.exists() else seed_path
|
||||
if previous_path.exists():
|
||||
try:
|
||||
old = json.loads(previous_path.read_text(encoding="utf-8"))
|
||||
old_platform = old.get("platform", "xhs_pgy")
|
||||
for t in old.get("tables", []):
|
||||
if t.get("name"):
|
||||
old_by_name[t["name"]] = t
|
||||
except Exception as exc:
|
||||
_log(f"读旧缓存失败(忽略): {exc}")
|
||||
|
||||
_log(f"动态读取地址表 {INDEX_BASE_TOKEN}/{INDEX_TABLE_ID} ...")
|
||||
try:
|
||||
index_records = _record_list_all(INDEX_BASE_TOKEN, INDEX_TABLE_ID)
|
||||
except Exception as exc:
|
||||
_log(f"读取地址表失败: {exc}")
|
||||
return None
|
||||
_log(f"地址表共 {len(index_records)} 行")
|
||||
|
||||
# 需要先知道字段名→field_id 映射,取「款式」和「合作达人多维表格地址」两列
|
||||
fields = _field_list(INDEX_BASE_TOKEN, INDEX_TABLE_ID)
|
||||
name2id = {}
|
||||
for f in fields:
|
||||
name2id[f.get("name")] = f.get("id")
|
||||
style_fid = name2id.get(INDEX_STYLE_FIELD)
|
||||
url_fid = name2id.get(url_field)
|
||||
# 兜底: 合作达人列历史曾叫「飞书多维表格地址」
|
||||
if not url_fid and url_field == INDEX_URL_FIELD:
|
||||
url_fid = name2id.get("飞书多维表格地址")
|
||||
if not style_fid or not url_fid:
|
||||
_log(f" [ERR] 地址表找不到字段 '{INDEX_STYLE_FIELD}'/'{url_field}'")
|
||||
return None
|
||||
|
||||
# 去重:同一款式只保留第一条 款式+地址 都有且地址里能解析出 table_id 的行
|
||||
seen_styles: set[str] = set()
|
||||
raw_styles: list[dict] = [] # {name, base_token, table_id, url}
|
||||
for rec in index_records:
|
||||
name = rec.get(style_fid)
|
||||
if isinstance(name, list):
|
||||
name = " ".join(str(x) for x in name if x).strip()
|
||||
if isinstance(name, str):
|
||||
name = name.strip()
|
||||
if not name:
|
||||
continue
|
||||
bt, tid, url = parse_style_url(rec.get(url_fid))
|
||||
if not bt:
|
||||
continue
|
||||
# 没 table_id 的行先跳过;若有同款式的后续行带 table_id 会补上
|
||||
if not tid:
|
||||
continue
|
||||
if name in seen_styles:
|
||||
continue
|
||||
seen_styles.add(name)
|
||||
raw_styles.append({"name": name, "base_token": bt,
|
||||
"table_id": tid, "url": _norm_url(bt, tid, url)})
|
||||
|
||||
_log(f"去重后款式 {len(raw_styles)} 个,开始拉各表字段定义...")
|
||||
|
||||
# index 编号:已知款式用旧编号,新款式往后追加
|
||||
used_idx: set[int] = set()
|
||||
for s in old_by_name.values():
|
||||
idx = s.get("index")
|
||||
if isinstance(idx, int):
|
||||
used_idx.add(idx)
|
||||
|
||||
def next_idx() -> int:
|
||||
n = 1
|
||||
while n in used_idx:
|
||||
n += 1
|
||||
used_idx.add(n)
|
||||
return n
|
||||
|
||||
tables = []
|
||||
for s in raw_styles:
|
||||
fields = _field_list(s["base_token"], s["table_id"])
|
||||
if not fields:
|
||||
_log(f" [WARN] [{s['name']}] 拉不到字段,跳过")
|
||||
continue
|
||||
field_map, all_field_names = build_field_map(fields)
|
||||
old_entry = old_by_name.get(s["name"])
|
||||
if old_entry and isinstance(old_entry.get("index"), int):
|
||||
idx = old_entry["index"]
|
||||
used_idx.add(idx)
|
||||
else:
|
||||
idx = next_idx()
|
||||
total_records = old_entry.get("total_records", 0) if old_entry else 0
|
||||
tables.append({
|
||||
"index": idx,
|
||||
"name": s["name"],
|
||||
"base_token": s["base_token"],
|
||||
"table_id": s["table_id"],
|
||||
"url": s["url"],
|
||||
"field_map": field_map,
|
||||
"all_field_names": all_field_names,
|
||||
"total_records": total_records,
|
||||
})
|
||||
_log(f" [{idx:02d}] {s['name']}: {len(field_map)} 逻辑字段 / {len(all_field_names)} 列")
|
||||
|
||||
tables.sort(key=lambda t: t["index"])
|
||||
label = "自营" if is_self else "合作达人"
|
||||
mapping = {
|
||||
"_comment": (f"款式 → 多维表格 对照表({label}); 由 feishu_mapping.py 从{label}地址表动态生成。"
|
||||
" field_map 给出本表核心字段的 field_id (实时从飞书拉取)"),
|
||||
"platform": old_platform,
|
||||
"tables": tables,
|
||||
}
|
||||
|
||||
# 原子写:先写临时文件再 rename,避免并行进程读到半截 JSON
|
||||
if write:
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(mapping, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
_log(f"已写回缓存 ({len(tables)} 款式): {path}")
|
||||
else:
|
||||
_log(f"[--diff] 仅构建内存 mapping ({len(tables)} 款式),未落盘")
|
||||
return mapping
|
||||
|
||||
|
||||
def load_mapping(data_dir: Path | None = None,
|
||||
force_refresh: bool = False,
|
||||
ttl: float = CACHE_TTL,
|
||||
self_operated: bool = False) -> dict:
|
||||
"""供三个 scraper 调用:保证有 mapping 可用。
|
||||
优先:缓存新鲜就用缓存;否则动态刷新;刷新失败退回缓存;再不行报错退出。
|
||||
self_operated=True 时读自营达人 mapping。"""
|
||||
del data_dir # Legacy compatibility: runtime mappings have one canonical cache.
|
||||
path = SELF_MAPPING_PATH if self_operated else MAPPING_PATH
|
||||
seed_path = SELF_MAPPING_SEED_PATH if self_operated else MAPPING_SEED_PATH
|
||||
# 缓存是否新鲜
|
||||
fresh = path.exists() and (time.time() - path.stat().st_mtime) < ttl
|
||||
if not force_refresh and fresh:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
url_field = INDEX_SELF_URL_FIELD if self_operated else INDEX_URL_FIELD
|
||||
new = refresh_mapping(force=True, ttl=0, mapping_path=path, url_field=url_field)
|
||||
if new is not None:
|
||||
return new
|
||||
if not path.exists() and seed_path.exists():
|
||||
_log(f"dynamic refresh failed; using read-only seed: {seed_path}")
|
||||
return json.loads(seed_path.read_text(encoding="utf-8"))
|
||||
# 刷新失败 → 退回缓存(无论新旧)
|
||||
if path.exists():
|
||||
_log(f"动态刷新失败,使用缓存: {path}")
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
_log("[FATAL] 动态刷新失败且无缓存可用")
|
||||
raise RuntimeError("无法获取款式对照表(动态读取失败且无缓存)")
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 手动跑一次:python feishu_mapping.py
|
||||
m = refresh_mapping(force=True)
|
||||
if m:
|
||||
ts = m.get("tables", [])
|
||||
print(f"\n款式数: {len(ts)}")
|
||||
for t in ts:
|
||||
print(f" [{t['index']:02d}] {t['name']} bt={t['base_token']} tid={t['table_id']}")
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Open Xiaohongshu and Douyin in real Chrome so you can log in manually.
|
||||
|
||||
Cookies and browser profiles are saved to the same locations used by the scrapers.
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Page
|
||||
from scrapling.fetchers import DynamicSession
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime import bilibili_comment_scraper
|
||||
from gyxx_flow.modules.content_marketing.runtime import douyin_comment_scraper
|
||||
from gyxx_flow.modules.content_marketing.runtime import xiaohongshu_comment_scraper
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {message}", flush=True)
|
||||
|
||||
|
||||
def wait_for_login(page: Page, platform: str, timeout: int = 300) -> None:
|
||||
"""Wait until login prompts disappear from the page."""
|
||||
login_markers = {
|
||||
"xiaohongshu": ("扫码登录", "登录后", "验证码"),
|
||||
"douyin": ("扫码登录", "登录后"),
|
||||
"bilibili": ("扫码登录", "短信登录", "密码登录"),
|
||||
}
|
||||
markers = login_markers.get(platform, ())
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
body = page.locator("body").inner_text(timeout=5000)
|
||||
except Exception:
|
||||
body = ""
|
||||
if not any(marker in body for marker in markers):
|
||||
log(f"{platform}: login state detected (no login prompt)")
|
||||
return
|
||||
log(f"{platform}: waiting for login... ({int(deadline - time.time())}s left)")
|
||||
page.wait_for_timeout(3000)
|
||||
log(f"{platform}: timed out waiting for login")
|
||||
|
||||
|
||||
def login_platform(platform: str, url: str, profile_dir: Path, cookie_file: Path, save_state_fn: Any, restore_cookies_fn: Any, timeout: int = 300) -> None:
|
||||
log(f"Opening {platform} for manual login: {url}")
|
||||
profile_dir.mkdir(exist_ok=True)
|
||||
|
||||
def action(page: Page) -> None:
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=90000)
|
||||
page.wait_for_timeout(4000)
|
||||
wait_for_login(page, platform, timeout)
|
||||
save_state_fn(page)
|
||||
log(f"Saved {platform} cookies to {cookie_file}")
|
||||
|
||||
with DynamicSession(
|
||||
headless=False,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(profile_dir),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies_fn,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
session.fetch(url, page_action=action, wait=1000)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# 1. Xiaohongshu
|
||||
login_platform(
|
||||
platform="xiaohongshu",
|
||||
url="https://www.xiaohongshu.com/explore",
|
||||
profile_dir=xiaohongshu_comment_scraper.PROFILE_DIR,
|
||||
cookie_file=xiaohongshu_comment_scraper.COOKIE_FILE,
|
||||
save_state_fn=xiaohongshu_comment_scraper.save_state,
|
||||
restore_cookies_fn=xiaohongshu_comment_scraper.restore_cookies,
|
||||
)
|
||||
|
||||
# 2. Douyin
|
||||
login_platform(
|
||||
platform="douyin",
|
||||
url="https://www.douyin.com/",
|
||||
profile_dir=douyin_comment_scraper.PROFILE_DIR,
|
||||
cookie_file=douyin_comment_scraper.COOKIE_FILE,
|
||||
save_state_fn=douyin_comment_scraper.save_state,
|
||||
restore_cookies_fn=douyin_comment_scraper.restore_cookies,
|
||||
)
|
||||
|
||||
# 3. Bilibili
|
||||
login_platform(
|
||||
platform="bilibili",
|
||||
url="https://passport.bilibili.com/login",
|
||||
profile_dir=bilibili_comment_scraper.PROFILE_DIR,
|
||||
cookie_file=bilibili_comment_scraper.COOKIE_FILE,
|
||||
save_state_fn=bilibili_comment_scraper.save_state,
|
||||
restore_cookies_fn=bilibili_comment_scraper.restore_cookies,
|
||||
)
|
||||
|
||||
log("All platforms logged in and cookies saved.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,450 @@
|
||||
"""月度笔记汇总:对上月各款所有笔记做横向对比分析(曝光分层 × 平台特性)。
|
||||
|
||||
与 weekly_summary_all.py 的区别:
|
||||
- 时间范围:上月 1 号到月底(动态算)
|
||||
- Prompt:曝光分层 × 平台特性修正效率评级(S/A/B/C + 平台价值定位)
|
||||
- 入库:cmt_monthly_summary 表(UNIQUE(style_id, month_start))
|
||||
- 飞书写入:生命进程表的"笔记分析汇总"字段,时间标月度范围
|
||||
|
||||
流程:
|
||||
1. 读索引表拿每款生命进程表地址(复用 weekly 的 load_index_styles)
|
||||
2. PG 查上月该款所有笔记(按 publish_time 过滤)
|
||||
3. 对每款:跑单品分析(复用 note_<id>.txt)+ LLM 横向对比 + 创建飞书文档 + 写入生命进程表 + 入库
|
||||
|
||||
用法:
|
||||
python monthly_summary_all.py # 跑所有上月有笔记的款
|
||||
python monthly_summary_all.py --style 宙斯 # 只跑某款
|
||||
python monthly_summary_all.py --dry-run # 不写飞书/PG
|
||||
python monthly_summary_all.py --max-workers 4 # 并发数
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
# 复用 weekly_summary_all 的工具函数
|
||||
from gyxx_flow.modules.content_marketing.runtime.weekly_summary_all import (
|
||||
LARK, DATA_DIR, REPORTS_DIR, dry_run_global,
|
||||
URL_RE, log, call_lark_json, parse_url, parse_bt_tid,
|
||||
load_index_styles, find_target_fields, read_report,
|
||||
run_single_note_analysis, create_feishu_doc, write_to_target_table,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.analyze_comments import call_hermes_analyzer
|
||||
|
||||
# ============================================================
|
||||
# 月时间范围:动态算上月 1 号到月底
|
||||
# ============================================================
|
||||
def _compute_last_month():
|
||||
"""返回 (month_start, month_end) 都是 date 对象"""
|
||||
today = datetime.now().date()
|
||||
# 上月 1 号 = 本月 1 号 - 1 天
|
||||
first_of_this_month = today.replace(day=1)
|
||||
month_end = first_of_this_month - timedelta(days=1)
|
||||
month_start = month_end.replace(day=1)
|
||||
return month_start, month_end
|
||||
|
||||
_DEFAULT_START, _DEFAULT_END = _compute_last_month()
|
||||
MONTH_START = os.environ.get("MONTH_SINCE", _DEFAULT_START.strftime("%Y-%m-%d"))
|
||||
MONTH_END = os.environ.get("MONTH_UNTIL", _DEFAULT_END.strftime("%Y-%m-%d"))
|
||||
TIME_LABEL = f"{MONTH_START}~{MONTH_END[5:]}"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# PG 查上月笔记
|
||||
# ============================================================
|
||||
def get_last_month_notes(style_name):
|
||||
"""从 PG 拿上月该款的所有笔记(按 publish_time 过滤)"""
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
with get_conn() as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT n.id, n.platform, n.title, n.url, n.view_count,
|
||||
n.comment_count, n.like_count, n.favorite_count, n.share_count,
|
||||
c.name AS creator_name
|
||||
FROM cmt_notes n
|
||||
JOIN cmt_styles s ON n.style_id = s.id
|
||||
LEFT JOIN cmt_creators c ON n.creator_id = c.id
|
||||
WHERE s.name = %s AND n.self_operated = FALSE
|
||||
AND n.publish_time IS NOT NULL
|
||||
AND n.publish_time >= %s AND n.publish_time < %s
|
||||
ORDER BY n.id
|
||||
""", (style_name, MONTH_START, MONTH_END + " 23:59:59"))
|
||||
cols = [d[0] for d in cur.description]
|
||||
return [dict(zip(cols, r)) for r in cur.fetchall()]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# LLM 横向对比 prompt(曝光分层 × 平台特性)
|
||||
# ============================================================
|
||||
MONTHLY_PROMPT = """你是光影行星品牌「{style}」款式的批量内容横向分析师。
|
||||
请对以下【{n} 篇笔记清单】进行横向对比分析,严格遵循"先按曝光量级分层,再按平台特性修正效率评级"的双维逻辑。
|
||||
|
||||
【本月笔记清单】
|
||||
{notes}
|
||||
|
||||
【第一步:曝光量级分层(硬性门槛)】
|
||||
|
||||
| 平台 | 曝光"高"阈值 | 曝光"低"阈值 |
|
||||
| :--- | :--- | :--- |
|
||||
| 抖音 | ≥ 50万 | < 50万 |
|
||||
| 小红书 | ≥ 10万 | < 10万 |
|
||||
| B站 | ≥ 30万 | < 30万 |
|
||||
|
||||
- 高曝光 + 高效率 = **S级(明星爆款)**
|
||||
- 低曝光 + 高效率 = **A级(潜力黑马)**
|
||||
- 高曝光 + 低效率 = **B级(虚假繁荣)**
|
||||
- 低曝光 + 低效率 = **C级(失败品)**
|
||||
|
||||
【第二步:平台特性修正系数(判断"效率高/低"时,标准各不同)】
|
||||
|
||||
### 1. 抖音(公域推荐 × 情绪爆发)
|
||||
- 互动率 ≥ 5% 为高;完播率 ≥ 40% 为高;品牌提及率 ≥ 12% 为高
|
||||
- 核心观察窗口:发布后 24 小时
|
||||
- 主要价值定位:爆款驱动、即时冲动、快速拉新
|
||||
- 抖音特有分析维度:前 3 秒留存率 / 评论区情绪词 vs 理性词占比
|
||||
|
||||
### 2. 小红书(搜索心智 × 长尾效应)
|
||||
- 互动率 ≥ 6% 为高;收藏率 ≥ 2% 为高;品牌提及率 ≥ 20% 为高
|
||||
- 核心观察窗口:发布后 3-7 天
|
||||
- 主要价值定位:深度种草、搜索拦截、心智沉淀
|
||||
- 小红书特有分析维度:搜索词关联度(品类词+场景词) / 评论区"求链接"比例
|
||||
|
||||
### 3. B站(深度信任 × 长决策周期)
|
||||
- 互动率 ≥ 4% 为高;平均观看时长 ≥ 3 分钟为高;品牌提及率 ≥ 25% 为高
|
||||
- 核心观察窗口:发布后 7-14 天
|
||||
- 主要价值定位:深度测评、信任建立、高客单价转化
|
||||
- B站特有分析维度:评论区提问深度(竞品对比 vs 简单咨询) / 弹幕密度曲线
|
||||
|
||||
【输出要求】
|
||||
按以下结构输出(Markdown 格式,适合贴到飞书文本字段):
|
||||
|
||||
# {style} 月度笔记汇总（{time_range}）
|
||||
|
||||
## 一、本月投放概览
|
||||
- 投放笔记数：{n} 篇
|
||||
- 涉及平台：[列出各平台笔记数]
|
||||
- 总曝光 / 总互动：[汇总数据]
|
||||
- 整体表现：[一句话定性]
|
||||
|
||||
## 二、分层排序总表
|
||||
按 S -> A -> B -> C 输出,同一层级内按"曝光量×互动率"降序排列。
|
||||
|
||||
| 层级 | 笔记标题 | 平台 | 达人 | 曝光 | 互动率 | 品牌提及率 | 平台价值定位 |
|
||||
| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
||||
| S级 | ... | ... | ... | ... | ... | ... | 爆款拉新/深度种草/信任转化 |
|
||||
| A级 | ... | ... | ... | ... | ... | ... | ... |
|
||||
| B级 | ... | ... | ... | ... | ... | ... | ... |
|
||||
| C级 | ... | ... | ... | ... | ... | ... | ... |
|
||||
|
||||
## 三、跨平台对比洞察
|
||||
- 哪个平台的高效笔记占比最高? -> 重点倾斜预算的平台
|
||||
- 哪个平台的 S 级笔记内容公式最清晰? -> 优先复制到其他平台进行跨平台移植测试
|
||||
- 各平台笔记数分布是否均衡
|
||||
|
||||
## 四、各平台 Top3 笔记内容特征总结
|
||||
### 抖音 Top3(如有)
|
||||
- 是否都是"前3秒强钩子+情绪共鸣"型?
|
||||
- 列出 3 篇标题 + 共性特征
|
||||
|
||||
### 小红书 Top3(如有)
|
||||
- 是否都包含"搜索关键词+高收藏价值"?
|
||||
- 列出 3 篇标题 + 共性特征
|
||||
|
||||
### B站 Top3(如有)
|
||||
- 是否都是"深度信息密度+竞品对比"型?
|
||||
- 列出 3 篇标题 + 共性特征
|
||||
|
||||
若发现某平台 Top3 不符合该平台典型特征(如小红书 Top3 全是情绪段子而非实用内容),说明该平台选号或 Brief 方向已跑偏,需明确指出。
|
||||
|
||||
## 五、预算再分配建议
|
||||
结合平台特性,给出下月预算比例建议:
|
||||
- 抖音：X%(理由)
|
||||
- 小红书：X%(理由)
|
||||
- B站：X%(理由)
|
||||
|
||||
## 六、下轮 Brief 平台差异化修改指令
|
||||
针对各平台分别输出 1 条 Brief 修改指令:
|
||||
- 抖音 Brief 补什么:强调"前3秒留存"和"品牌名口播重复2次"
|
||||
- 小红书 Brief 补什么:强调"标题含品类搜索词"和"收藏率优化"
|
||||
- B站 Brief 补什么:强调"深度对比实测"和"评论区答疑准备"
|
||||
|
||||
【约束】
|
||||
- 分层必须严格按曝光阈值(抖音50万/小红书10万/B站30万)判定,不放宽
|
||||
- 效率判定必须按各平台特性标准(抖音5%/小红书6%/B站4%),不混用
|
||||
- 表格必须填充完整数据,不要留空或写"无"
|
||||
- 每条建议须具体可执行,避免"加强""优化"等空泛词汇
|
||||
- 整体篇幅 2500-4000 字
|
||||
- **必须把完整的 Markdown 报告作为响应内容直接返回,不要写到本地文件,不要只给摘要,不要提及"文件已保存"或"月报已生成"等元信息。响应正文就是最终报告本身。**"""
|
||||
|
||||
|
||||
def generate_monthly_summary(style, notes):
|
||||
"""调 LLM 生成月度横向对比汇总"""
|
||||
notes_block = []
|
||||
for i, n in enumerate(notes, 1):
|
||||
views = n.get('view_count')
|
||||
views_str = f"{views:,}" if views else "未采集"
|
||||
notes_block.append(
|
||||
f"### 笔记{i}: {n['title']}\n"
|
||||
f"- 平台: {n['platform']}\n- 达人: {n.get('creator_name', '?')}\n"
|
||||
f"- 曝光: {views_str} / 点赞: {n.get('like_count') or 0} / "
|
||||
f"评论: {n.get('comment_count') or 0} / 收藏: {n.get('favorite_count') or 0} / "
|
||||
f"分享: {n.get('share_count') or 0}\n"
|
||||
)
|
||||
prompt = MONTHLY_PROMPT.format(
|
||||
style=style,
|
||||
notes="\n---\n".join(notes_block),
|
||||
time_range=TIME_LABEL,
|
||||
n=len(notes),
|
||||
)
|
||||
system = "你是光影行星品牌的批量内容横向分析师,擅长按曝光分层 × 平台特性双维逻辑做笔记横向对比分析。"
|
||||
return call_hermes_analyzer(system, prompt)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 入库 cmt_monthly_summary
|
||||
# ============================================================
|
||||
def save_to_db_monthly(style, notes, summary, doc_url, doc_title, status):
|
||||
"""把月度汇总结果写入 PG cmt_monthly_summary(UNIQUE(style_id, month_start))"""
|
||||
if dry_run_global:
|
||||
return False
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
try:
|
||||
with get_conn() as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT id FROM cmt_styles WHERE name = %s", (style,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
log(f" [DB] 款式 {style} 不在 cmt_styles,跳过入库")
|
||||
return False
|
||||
style_id = row[0]
|
||||
|
||||
total_views = sum((n.get('view_count') or 0) for n in notes) or None
|
||||
total_comments = sum((n.get('comment_count') or 0) for n in notes)
|
||||
total_engagement = sum(
|
||||
((n.get('like_count') or 0) + (n.get('favorite_count') or 0) +
|
||||
(n.get('comment_count') or 0) + (n.get('share_count') or 0))
|
||||
for n in notes) or None
|
||||
|
||||
cur.execute("""
|
||||
INSERT INTO cmt_monthly_summary
|
||||
(style_id, month_start, month_end, time_label,
|
||||
note_count, total_comments, total_views, total_engagement,
|
||||
doc_url, doc_title, summary_chars, status,
|
||||
created_at, updated_at)
|
||||
VALUES
|
||||
(%s, %s, %s, %s,
|
||||
%s, %s, %s, %s,
|
||||
%s, %s, %s, %s,
|
||||
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (style_id, month_start) DO UPDATE SET
|
||||
note_count = EXCLUDED.note_count,
|
||||
total_comments = EXCLUDED.total_comments,
|
||||
total_views = EXCLUDED.total_views,
|
||||
total_engagement = EXCLUDED.total_engagement,
|
||||
doc_url = EXCLUDED.doc_url,
|
||||
doc_title = EXCLUDED.doc_title,
|
||||
summary_chars = EXCLUDED.summary_chars,
|
||||
status = EXCLUDED.status,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""", (style_id, MONTH_START, MONTH_END, TIME_LABEL,
|
||||
len(notes), total_comments, total_views, total_engagement,
|
||||
doc_url, doc_title, len(summary) if summary else None, status))
|
||||
conn.commit()
|
||||
log(f" ✅ 已入库 cmt_monthly_summary (style_id={style_id}, month={MONTH_START})")
|
||||
return True
|
||||
except Exception as exc:
|
||||
log(f" [DB ERR] {style} 入库失败: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 处理一个款
|
||||
# ============================================================
|
||||
def process_one_style_monthly(style, index_info):
|
||||
"""处理一个款的月度汇总"""
|
||||
bt, tid, url = index_info
|
||||
log(f"\n========== {style} (月度) ==========")
|
||||
log(f" 生命进程表: {bt}/{tid}")
|
||||
|
||||
def _save(notes_list, summary, doc_url, doc_title, status):
|
||||
save_to_db_monthly(style, notes_list, summary, doc_url, doc_title, status)
|
||||
|
||||
time_fid, summary_fid = find_target_fields(bt, tid)
|
||||
if not time_fid or not summary_fid:
|
||||
log(f" [ERR] 找不到 时间/笔记分析汇总 字段,跳过")
|
||||
_save([], None, None, None, "no_fields")
|
||||
return {"style": style, "status": "no_fields", "notes": 0}
|
||||
|
||||
notes = get_last_month_notes(style)
|
||||
log(f" 上月笔记: {len(notes)} 篇")
|
||||
if not notes:
|
||||
log(f" [SKIP] 上月无笔记")
|
||||
_save([], None, None, None, "no_notes")
|
||||
return {"style": style, "status": "no_notes", "notes": 0}
|
||||
|
||||
for n in notes:
|
||||
n["report_path"] = run_single_note_analysis(n["id"])
|
||||
|
||||
valid_notes = [n for n in notes if n.get("report_path")]
|
||||
if not valid_notes:
|
||||
log(f" [ERR] 没有有效报告,跳过")
|
||||
_save(notes, None, None, None, "no_reports")
|
||||
return {"style": style, "status": "no_reports", "notes": len(notes)}
|
||||
|
||||
log(f" 调 LLM 横向对比 {len(valid_notes)} 篇报告...")
|
||||
summary = generate_monthly_summary(style, valid_notes)
|
||||
if not summary:
|
||||
log(f" [ERR] LLM 综合失败")
|
||||
_save(notes, None, None, None, "llm_failed")
|
||||
return {"style": style, "status": "llm_failed", "notes": len(notes)}
|
||||
|
||||
out_path = REPORTS_DIR / f"{style}_月度汇总_{TIME_LABEL}.txt"
|
||||
out_path.write_text(summary, encoding="utf-8")
|
||||
log(f" 本地副本: {out_path.name} ({len(summary)} 字符)")
|
||||
|
||||
if dry_run_global:
|
||||
log(f" [DRY] 跳过飞书文档创建和写入")
|
||||
_save(notes, summary, None, None, "dry_run")
|
||||
return {"style": style, "status": "dry_run", "notes": len(notes), "chars": len(summary)}
|
||||
|
||||
doc_url, err = create_feishu_doc_monthly(style, summary)
|
||||
if not doc_url:
|
||||
log(f" [ERR] {err}")
|
||||
_save(notes, summary, None, None, "doc_failed")
|
||||
return {"style": style, "status": "doc_failed", "notes": len(notes)}
|
||||
|
||||
log(f" 飞书文档: {doc_url}")
|
||||
|
||||
ok = write_to_target_table(bt, tid, time_fid, summary_fid, style, doc_url, dry_run_global)
|
||||
doc_title = f"{style} 月度汇总（{TIME_LABEL}）"
|
||||
_save(notes, summary, doc_url, doc_title, "ok" if ok else "feishu_write_failed")
|
||||
|
||||
if ok:
|
||||
log(f" ✅ 已写入生命进程表")
|
||||
return {"style": style, "status": "ok", "notes": len(notes),
|
||||
"doc_url": doc_url, "chars": len(summary)}
|
||||
log(f" [ERR] 写入表失败")
|
||||
return {"style": style, "status": "write_failed", "notes": len(notes)}
|
||||
|
||||
|
||||
def create_feishu_doc_monthly(style, summary):
|
||||
"""创建飞书文档(月度),返回 url"""
|
||||
doc_title = f"{style} 月度汇总（{TIME_LABEL}）"
|
||||
md_file = PATHS.tmp_root / "monthly_summary_doc.md"
|
||||
md_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_file.write_text(summary, encoding="utf-8")
|
||||
|
||||
cmd = [LARK, "docs", "+create",
|
||||
"--as", "user", "--format", "json",
|
||||
"--title", doc_title,
|
||||
"--doc-format", "markdown",
|
||||
"--content", f"@{md_file.name}"]
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace")
|
||||
text = proc.stdout or proc.stderr
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
return None, f"创建失败: {text[:300]}"
|
||||
resp = json.loads(text[start:])
|
||||
if not resp.get("ok"):
|
||||
return None, f"创建失败: {resp.get('error')}"
|
||||
url = resp.get("data", {}).get("document", {}).get("url", "")
|
||||
return url, None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# main
|
||||
# ============================================================
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="月度笔记汇总:对上月各款所有笔记做横向对比分析")
|
||||
parser.add_argument("--style", help="只跑指定款式名")
|
||||
parser.add_argument("--dry-run", action="store_true", help="不写飞书/PG")
|
||||
parser.add_argument("--max-workers", type=int, default=4, help="款之间并发数(默认 4)")
|
||||
args = parser.parse_args()
|
||||
|
||||
global dry_run_global
|
||||
# 引用 weekly 的全局变量(改它会影响 weekly 模块,但 dry_run_global 在 weekly 顶部定义)
|
||||
from gyxx_flow.modules.content_marketing.runtime import weekly_summary_all
|
||||
weekly_summary_all.dry_run_global = args.dry_run
|
||||
# 本模块的 dry_run_global 是 import 时的副本,需要同步
|
||||
globals()['dry_run_global'] = args.dry_run
|
||||
|
||||
log(f"时间范围: {TIME_LABEL} (月度)")
|
||||
|
||||
index_styles = load_index_styles()
|
||||
if not index_styles:
|
||||
log("索引表无有效数据,退出")
|
||||
return 1
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
with get_conn() as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT s.name, COUNT(n.id)
|
||||
FROM cmt_styles s
|
||||
JOIN cmt_notes n ON n.style_id = s.id AND n.self_operated = FALSE
|
||||
WHERE n.publish_time IS NOT NULL
|
||||
AND n.publish_time >= %s AND n.publish_time < %s
|
||||
GROUP BY s.name
|
||||
ORDER BY s.name
|
||||
""", (MONTH_START, MONTH_END + " 23:59:59"))
|
||||
pg_styles = [(r[0], r[1]) for r in cur.fetchall()]
|
||||
|
||||
log(f"PG 上月有笔记的款: {len(pg_styles)} 个")
|
||||
|
||||
to_run = []
|
||||
for name, note_count in pg_styles:
|
||||
if args.style and name != args.style:
|
||||
continue
|
||||
if name in index_styles:
|
||||
to_run.append((name, index_styles[name], note_count))
|
||||
else:
|
||||
log(f" [SKIP] {name}: 索引表无'每周笔记分析'地址")
|
||||
|
||||
log(f"待跑款式: {len(to_run)} 个")
|
||||
if not to_run:
|
||||
return 0
|
||||
|
||||
import concurrent.futures
|
||||
log(f"并发跑 {len(to_run)} 款, max_workers={args.max_workers}")
|
||||
results = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as executor:
|
||||
future_to_style = {
|
||||
executor.submit(process_one_style_monthly, name, info): (name, ncount)
|
||||
for name, info, ncount in to_run
|
||||
}
|
||||
for future in concurrent.futures.as_completed(future_to_style):
|
||||
name, ncount = future_to_style[future]
|
||||
try:
|
||||
r = future.result()
|
||||
r["note_count"] = ncount
|
||||
results.append(r)
|
||||
log(f" ✅ 完成: {name} -> {r['status']}")
|
||||
except Exception as exc:
|
||||
log(f" [ERR] {name} 异常: {exc}")
|
||||
results.append({"style": name, "status": "exception",
|
||||
"error": str(exc), "note_count": ncount})
|
||||
|
||||
results.sort(key=lambda r: r.get("style", ""))
|
||||
|
||||
log(f"\n{'=' * 50}")
|
||||
log(f"全部完成,汇总:")
|
||||
for r in results:
|
||||
log(f" {r['style']}: {r['status']} (笔记 {r.get('note_count', 0)} 篇)")
|
||||
log(f"{'=' * 50}")
|
||||
|
||||
out = REPORTS_DIR / f"_monthly_summary_all_{TIME_LABEL}.json"
|
||||
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
log(f"结果已存: {out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,424 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Page, TimeoutError as PlaywrightTimeoutError
|
||||
from scrapling.fetchers import DynamicSession
|
||||
|
||||
|
||||
LOGIN_URL = "https://pgy.xiaohongshu.com/"
|
||||
HOME_URL = "https://pgy.xiaohongshu.com/solar/pre-trade/home"
|
||||
DETAIL_URL_MARK = "/solar/pre-trade/blogger-detail/"
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
COOKIE_FILE = PATHS.browser_cookie_file
|
||||
PROFILE_DIR = PATHS.browser_profile_dir
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {message}", flush=True)
|
||||
|
||||
|
||||
def load_cookies() -> list[dict[str, Any]]:
|
||||
if not COOKIE_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except json.JSONDecodeError:
|
||||
log(f"Cookie file is invalid, ignoring: {COOKIE_FILE}")
|
||||
return []
|
||||
|
||||
|
||||
def save_state(page: Page) -> None:
|
||||
DATA_DIR.mkdir(exist_ok=True)
|
||||
cookies = page.context.cookies()
|
||||
COOKIE_FILE.write_text(
|
||||
json.dumps(cookies, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
storage_state = page.context.storage_state()
|
||||
PATHS.browser_storage_state_file.write_text(
|
||||
json.dumps(storage_state, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
log(f"Saved cookies: {COOKIE_FILE}")
|
||||
|
||||
|
||||
def restore_cookies(page: Page) -> None:
|
||||
cookies = load_cookies()
|
||||
if not cookies:
|
||||
return
|
||||
try:
|
||||
page.context.add_cookies(cookies)
|
||||
log(f"Loaded cookies: {COOKIE_FILE}")
|
||||
except Exception as exc:
|
||||
log(f"Failed to load cookies, will continue with browser profile: {exc}")
|
||||
|
||||
|
||||
def is_logged_in(page: Page) -> bool:
|
||||
if "/solar/pre-trade/" in page.url:
|
||||
return True
|
||||
try:
|
||||
page.get_by_text("找博主", exact=True).wait_for(timeout=2500)
|
||||
return True
|
||||
except PlaywrightTimeoutError:
|
||||
return False
|
||||
|
||||
|
||||
def wait_for_scan_login(page: Page, timeout_seconds: int) -> None:
|
||||
log("Not logged in. Please scan the QR code or finish login in the opened Chrome window.")
|
||||
deadline = time.time() + timeout_seconds
|
||||
while time.time() < deadline:
|
||||
if is_logged_in(page):
|
||||
save_state(page)
|
||||
return
|
||||
try:
|
||||
login_button = page.get_by_text(re.compile("登录|账号登录")).first
|
||||
if login_button.count():
|
||||
login_button.click(timeout=1000)
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
raise TimeoutError(f"Login timed out after {timeout_seconds} seconds.")
|
||||
|
||||
|
||||
def ensure_login(page: Page, login_timeout: int) -> None:
|
||||
page.goto(HOME_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_timeout(2500)
|
||||
if is_logged_in(page):
|
||||
log("Already logged in, using saved browser session.")
|
||||
dismiss_onboarding(page)
|
||||
save_state(page)
|
||||
return
|
||||
|
||||
page.goto(LOGIN_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_timeout(1500)
|
||||
wait_for_scan_login(page, login_timeout)
|
||||
page.goto(HOME_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
dismiss_onboarding(page)
|
||||
|
||||
|
||||
def dismiss_onboarding(page: Page) -> None:
|
||||
for text in ("跳过", "不再提示", "我知道了"):
|
||||
try:
|
||||
button = page.get_by_text(text, exact=True).first
|
||||
if button.count():
|
||||
button.click(timeout=1500)
|
||||
page.wait_for_timeout(800)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def fill_search_box(page: Page, blogger: str) -> None:
|
||||
dismiss_onboarding(page)
|
||||
switch_search_type_to_blogger(page)
|
||||
input_locator = page.locator("input").filter(
|
||||
has_not_text=re.compile("有什么问题尽管问我")
|
||||
).first
|
||||
input_locator.wait_for(state="visible", timeout=30000)
|
||||
input_locator.click()
|
||||
input_locator.fill(blogger)
|
||||
input_locator.press("Enter")
|
||||
|
||||
try:
|
||||
page.wait_for_url(re.compile(r".*/solar/pre-trade/note/kol.*"), timeout=15000)
|
||||
except PlaywrightTimeoutError:
|
||||
# Some versions require clicking the red search icon after entering text.
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const input = [...document.querySelectorAll('input')]
|
||||
.find((el) => /博主昵称|小红书号/.test(el.placeholder || '') || el.value);
|
||||
const box = input?.closest('div');
|
||||
const candidates = [...document.querySelectorAll('button, [role="button"], div')]
|
||||
.filter((el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
return r.width >= 24 && r.width <= 80 && r.height >= 24 && r.height <= 80;
|
||||
});
|
||||
const near = candidates.find((el) => {
|
||||
const er = el.getBoundingClientRect();
|
||||
const br = box?.getBoundingClientRect();
|
||||
return br && er.left > br.right - 20 && Math.abs(er.top - br.top) < 80;
|
||||
});
|
||||
near?.click();
|
||||
}
|
||||
"""
|
||||
)
|
||||
page.wait_for_url(re.compile(r".*/solar/pre-trade/note/kol.*"), timeout=30000)
|
||||
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
|
||||
def switch_search_type_to_blogger(page: Page) -> None:
|
||||
current_type = page.get_by_text(re.compile("笔记关键词|博主昵称"), exact=False).first
|
||||
try:
|
||||
current_type.click(timeout=3000)
|
||||
page.wait_for_timeout(600)
|
||||
option = page.get_by_text("博主昵称", exact=True).last
|
||||
if option.count():
|
||||
option.click(timeout=3000)
|
||||
page.wait_for_timeout(600)
|
||||
except Exception:
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const nodes = [...document.querySelectorAll('div, span, button')];
|
||||
const current = nodes.find((el) => /笔记关键词|博主昵称/.test(el.innerText || ''));
|
||||
current?.click();
|
||||
setTimeout(() => {
|
||||
const option = [...document.querySelectorAll('div, span, button')]
|
||||
.reverse()
|
||||
.find((el) => (el.innerText || '').trim() === '博主昵称');
|
||||
option?.click();
|
||||
}, 300);
|
||||
}
|
||||
"""
|
||||
)
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
|
||||
def open_blogger_detail(page: Page, blogger: str) -> Page:
|
||||
log(f"Opening blogger detail: {blogger}")
|
||||
pages_before = list(page.context.pages)
|
||||
clicked = page.evaluate(
|
||||
"""
|
||||
(name) => {
|
||||
const textHasName = (el) => (el.innerText || '').includes(name);
|
||||
const rows = [...document.querySelectorAll(
|
||||
'tr, [class*="row"], [class*="table"], [class*="card"], [class*="item"]'
|
||||
)].filter(textHasName);
|
||||
const row = rows.find((el) => /粉丝|阅读|报价|合作/.test(el.innerText || '')) || rows[0];
|
||||
if (!row) return false;
|
||||
const target = row.querySelector('img, a, [class*="avatar"], [class*="name"]') || row;
|
||||
target.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
target.click();
|
||||
return true;
|
||||
}
|
||||
""",
|
||||
blogger,
|
||||
)
|
||||
if not clicked:
|
||||
page.get_by_text(blogger, exact=False).first.click(timeout=10000)
|
||||
|
||||
detail_page = wait_for_detail_page(page, pages_before, timeout_ms=30000)
|
||||
if detail_page is None:
|
||||
# If the first click only focused the row, try the visible name once more.
|
||||
page.get_by_text(blogger, exact=False).first.click(timeout=10000)
|
||||
detail_page = wait_for_detail_page(page, pages_before, timeout_ms=30000)
|
||||
|
||||
if detail_page is None:
|
||||
raise TimeoutError("Clicked blogger result, but no blogger-detail page was detected.")
|
||||
|
||||
detail_page.bring_to_front()
|
||||
detail_page.wait_for_load_state("domcontentloaded")
|
||||
detail_page.get_by_text("笔记数据", exact=False).first.wait_for(timeout=30000)
|
||||
detail_page.get_by_text("笔记数据", exact=False).first.click(timeout=5000)
|
||||
detail_page.get_by_text("笔记数据", exact=False).first.scroll_into_view_if_needed()
|
||||
detail_page.wait_for_timeout(2500)
|
||||
return detail_page
|
||||
|
||||
|
||||
def wait_for_detail_page(page: Page, pages_before: list[Page], timeout_ms: int) -> Page | None:
|
||||
deadline = time.time() + timeout_ms / 1000
|
||||
known = set(pages_before)
|
||||
while time.time() < deadline:
|
||||
all_pages = list(page.context.pages)
|
||||
candidates = [p for p in all_pages if p not in known] + [page] + all_pages
|
||||
for candidate in candidates:
|
||||
try:
|
||||
if DETAIL_URL_MARK in candidate.url:
|
||||
return candidate
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(500)
|
||||
return None
|
||||
|
||||
|
||||
def parse_notes_from_dom(page: Page) -> list[dict[str, str]]:
|
||||
return page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim();
|
||||
const parseCard = (card) => {
|
||||
const rawLines = (card.innerText || '')
|
||||
.split('\\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
const text = rawLines.join('\\n');
|
||||
if (!/阅读/.test(text) || !/点赞/.test(text) || !/收藏/.test(text) || !/发布时间/.test(text)) return null;
|
||||
|
||||
const readAfter = (label) => {
|
||||
const index = rawLines.findIndex((line) => line === label || line.includes(label));
|
||||
if (index < 0) return '';
|
||||
const sameLine = rawLines[index].replace(label, '').trim();
|
||||
if (sameLine) return sameLine;
|
||||
return rawLines[index + 1] || '';
|
||||
};
|
||||
|
||||
const dateMatch = text.match(/20\\d{2}[./-]\\d{1,2}[./-]\\d{1,2}/);
|
||||
const readIndex = rawLines.findIndex((line) => line === '阅读' || line.includes('阅读'));
|
||||
const titleCandidates = rawLines.slice(0, readIndex >= 0 ? readIndex : rawLines.length).filter((line) => {
|
||||
if (['阅读', '点赞', '收藏', '发布时间'].includes(line)) return false;
|
||||
if (/^20\\d{2}[./-]/.test(line)) return false;
|
||||
if (/^(合作|流量|图文笔记|视频笔记|全部类型|首页|笔记主页|数据概览|笔记数据|粉丝分析|笔记案例)$/.test(line)) return false;
|
||||
if (/^[\\d,.wW万]+$/.test(line)) return false;
|
||||
if (/中位数|获赞与|粉丝数|报价|筛选|全部流量|最新发布/.test(line)) return false;
|
||||
return line.length >= 2;
|
||||
});
|
||||
const title = titleCandidates.slice(-2).join(' ').trim();
|
||||
|
||||
return {
|
||||
title: clean(title),
|
||||
read_count: clean(readAfter('阅读')),
|
||||
like_count: clean(readAfter('点赞')),
|
||||
collect_count: clean(readAfter('收藏')),
|
||||
publish_time: clean(readAfter('发布时间') || (dateMatch ? dateMatch[0] : '')),
|
||||
};
|
||||
};
|
||||
|
||||
const all = [...document.querySelectorAll('div')];
|
||||
const cards = all.filter((el) => {
|
||||
const text = el.innerText || '';
|
||||
const rect = el.getBoundingClientRect();
|
||||
return rect.width >= 120 &&
|
||||
rect.width <= 280 &&
|
||||
rect.height >= 220 &&
|
||||
rect.height <= 520 &&
|
||||
/阅读/.test(text) &&
|
||||
/点赞/.test(text) &&
|
||||
/收藏/.test(text) &&
|
||||
/发布时间/.test(text) &&
|
||||
el.querySelector('img');
|
||||
});
|
||||
|
||||
const parsed = cards.map(parseCard).filter(Boolean);
|
||||
const seen = new Set();
|
||||
return parsed.filter((item) => {
|
||||
const key = [item.title, item.read_count, item.like_count, item.collect_count, item.publish_time].join('|');
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return item.title || item.publish_time;
|
||||
});
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def write_outputs(blogger: str, notes: list[dict[str, str]]) -> tuple[Path, Path]:
|
||||
DATA_DIR.mkdir(exist_ok=True)
|
||||
safe_name = re.sub(r'[<>:"/\\\\|?*\\s]+', "_", blogger).strip("_") or "blogger"
|
||||
json_path = DATA_DIR / f"{safe_name}_notes.json"
|
||||
csv_path = DATA_DIR / f"{safe_name}_notes.csv"
|
||||
|
||||
payload = {
|
||||
"blogger": blogger,
|
||||
"scraped_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"count": len(notes),
|
||||
"notes": notes,
|
||||
}
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
with csv_path.open("w", newline="", encoding="utf-8-sig") as file:
|
||||
writer = csv.DictWriter(
|
||||
file,
|
||||
fieldnames=["title", "read_count", "like_count", "collect_count", "publish_time"],
|
||||
)
|
||||
writer.writeheader()
|
||||
writer.writerows(notes)
|
||||
|
||||
return json_path, csv_path
|
||||
|
||||
|
||||
def load_names(cli_names: list[str], names_file: str | None) -> list[str]:
|
||||
names: list[str] = []
|
||||
names.extend(name.strip() for name in cli_names if name.strip())
|
||||
|
||||
if names_file:
|
||||
path = Path(names_file)
|
||||
for line in path.read_text(encoding="utf-8-sig").splitlines():
|
||||
name = line.strip()
|
||||
if name and not name.startswith("#"):
|
||||
names.append(name)
|
||||
|
||||
deduped: list[str] = []
|
||||
seen = set()
|
||||
for name in names:
|
||||
if name not in seen:
|
||||
seen.add(name)
|
||||
deduped.append(name)
|
||||
return deduped
|
||||
|
||||
|
||||
def scrape(blogger: str, login_timeout: int, headless: bool) -> list[dict[str, str]]:
|
||||
result: dict[str, Any] = {"notes": []}
|
||||
|
||||
def action(page: Page) -> None:
|
||||
ensure_login(page, login_timeout)
|
||||
fill_search_box(page, blogger)
|
||||
detail_page = open_blogger_detail(page, blogger)
|
||||
result["detail_url"] = detail_page.url
|
||||
result["notes"] = parse_notes_from_dom(detail_page)
|
||||
save_state(detail_page)
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=headless,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=60000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
session.fetch(HOME_URL, page_action=action, wait=1000)
|
||||
|
||||
log(f"Detail URL: {result.get('detail_url', '')}")
|
||||
return result["notes"]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Scrape XiaoHongShu Pgy blogger note metrics with Scrapling DynamicSession.",
|
||||
)
|
||||
parser.add_argument("bloggers", nargs="*", help="博主昵称，可传多个，例如：博主A 博主B")
|
||||
parser.add_argument("--names-file", help="批量博主名单文件，一行一个昵称")
|
||||
parser.add_argument("--login-timeout", type=int, default=300, help="首次扫码登录等待秒数")
|
||||
parser.add_argument("--headless", action="store_true", help="无头模式；首次登录不要使用")
|
||||
args = parser.parse_args()
|
||||
|
||||
bloggers = load_names(args.bloggers, args.names_file)
|
||||
if not bloggers:
|
||||
parser.error("Please provide at least one blogger name or --names-file.")
|
||||
|
||||
for index, blogger in enumerate(bloggers, start=1):
|
||||
log(f"=== Blogger {index}/{len(bloggers)}: {blogger} ===")
|
||||
notes = scrape(blogger, args.login_timeout, args.headless)
|
||||
json_path, csv_path = write_outputs(blogger, notes)
|
||||
log(f"Scraped {len(notes)} notes")
|
||||
log(f"JSON: {json_path}")
|
||||
log(f"CSV: {csv_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,4 @@
|
||||
scrapling[fetchers]>=0.4.8
|
||||
playwright>=1.59.0
|
||||
psycopg[binary]>=3.1
|
||||
python-dotenv>=1.0
|
||||
@@ -0,0 +1,622 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
一键跑 3 个平台 (B 站 + 小红书蒲公英 + 星图) 的 V2 抓取,真并行。
|
||||
|
||||
并行原理:
|
||||
- bilibili_scraper.py: 纯 HTTP (requests.Session),独立进程无冲突
|
||||
- pgy_xhs_scraper_v2.py: Scrapling 启独立 Chrome (profile=.pgy_chrome_profile)
|
||||
- xingtu_scraper_v2.py: Scrapling 启独立 Chrome (profile=.xingtu_chrome_profile)
|
||||
|
||||
三个进程 stdin/stdout 各自走独立文件,互不干扰。
|
||||
汇总报告 = 3 个 _all_summaries*.json 聚合。
|
||||
v2_results/*.json 由 daily_run.bat step4 的 sync_metrics_to_cmt_notes.py 同步到 PG。
|
||||
|
||||
用法:
|
||||
python run_all.py # 只跑合作达人(实写)
|
||||
python run_all.py --dry-run # 只抓不写
|
||||
python run_all.py --skip pgy # 跳过小红书
|
||||
python run_all.py --skip bili,xt # 跳过 B 站和星图
|
||||
python run_all.py --style 7 # 只跑款式 7
|
||||
python run_all.py --style 7,8,15 # 跑多款
|
||||
python run_all.py --include-self-operated # 合作+自营都跑
|
||||
python run_all.py --self-operated-only # 只跑自营达人
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
V2_DIR = PATHS.normalized_root / "v2_results"
|
||||
RUN_REPORT_DIR = PATHS.curated_root / "run_all"
|
||||
LOG_DIR = PATHS.logs_root / "run_all"
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 三个脚本定义
|
||||
SCRIPTS = {
|
||||
"bili": {
|
||||
"name": "B 站",
|
||||
"script": "bilibili_scraper.py",
|
||||
"summary_file": "_all_summaries_bilibili.json",
|
||||
"result_prefix": "_bilibili_v2",
|
||||
},
|
||||
"pgy": {
|
||||
"name": "小红书蒲公英",
|
||||
"script": "pgy_xhs_scraper_v2.py",
|
||||
"summary_file": "_all_summaries.json",
|
||||
"result_prefix": "_v2",
|
||||
},
|
||||
"xt": {
|
||||
"name": "星图",
|
||||
"script": "xingtu_scraper_v2.py",
|
||||
"summary_file": "_all_summaries_xingtu.json",
|
||||
"result_prefix": "_xingtu_v2",
|
||||
},
|
||||
}
|
||||
|
||||
# 自营达人脚本: 抖音用 self_douyin_scraper.py, B站/小红书复用现有脚本加 --self-operated
|
||||
SELF_SCRIPTS = {
|
||||
"bili": {
|
||||
"name": "B 站(自营)",
|
||||
"script": "bilibili_scraper.py",
|
||||
"summary_file": "_all_summaries_self_bilibili.json",
|
||||
"extra_args": ["--self-operated"],
|
||||
},
|
||||
"pgy": {
|
||||
"name": "小红书(自营)",
|
||||
"script": "pgy_xhs_scraper_v2.py",
|
||||
"summary_file": "_all_summaries_self.json",
|
||||
"extra_args": ["--self-operated"],
|
||||
},
|
||||
"xt": {
|
||||
"name": "抖音(自营)",
|
||||
"script": "self_douyin_scraper.py",
|
||||
"summary_file": "_all_summaries_self_xingtu.json",
|
||||
"extra_args": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
ts = datetime.now().strftime("%H:%M:%S")
|
||||
safe = msg.encode("gbk", errors="replace").decode("gbk", errors="replace")
|
||||
print(f"[{ts}] [run_all] {safe}", flush=True)
|
||||
|
||||
|
||||
def build_cmd(key: str, args: argparse.Namespace) -> list[str]:
|
||||
"""构造单个脚本的命令行"""
|
||||
info = SCRIPTS[key]
|
||||
cmd = [sys.executable, str(BASE_DIR / info["script"])]
|
||||
if args.dry_run:
|
||||
cmd.append("--dry-run")
|
||||
if args.style:
|
||||
# 多个款式循环传
|
||||
pass # 脚本自身支持 --style N
|
||||
return cmd
|
||||
|
||||
|
||||
def start_process(key: str, args: argparse.Namespace,
|
||||
self_operated: bool = False) -> tuple[subprocess.Popen, Path, Path]:
|
||||
"""启动一个抓取进程,返回 (process, stdout_log_path, stderr_log_path)"""
|
||||
info = SELF_SCRIPTS[key] if self_operated else SCRIPTS[key]
|
||||
cmd = [sys.executable, str(BASE_DIR / info["script"])]
|
||||
if args.dry_run:
|
||||
cmd.append("--dry-run")
|
||||
if args.style:
|
||||
for s in args.style:
|
||||
cmd += ["--style", str(s)]
|
||||
cmd.extend(info.get("extra_args", []))
|
||||
|
||||
stdout_path = LOG_DIR / f"{key}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.stdout.log"
|
||||
stderr_path = LOG_DIR / f"{key}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.stderr.log"
|
||||
stdout_f = open(stdout_path, "w", encoding="utf-8")
|
||||
stderr_f = open(stderr_path, "w", encoding="utf-8")
|
||||
|
||||
# Windows 用 CREATE_NEW_PROCESS_GROUP 方便 kill
|
||||
creationflags = 0
|
||||
if sys.platform == "win32":
|
||||
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
"PYTHONIOENCODING": "utf-8",
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
}
|
||||
|
||||
log(f"启动 [{key}] ({info['name']}): {info['script']} 日志: {stdout_path.name}")
|
||||
p = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=str(BASE_DIR),
|
||||
stdout=stdout_f,
|
||||
stderr=stderr_f,
|
||||
env=env,
|
||||
creationflags=creationflags,
|
||||
)
|
||||
return p, stdout_path, stderr_path
|
||||
|
||||
|
||||
def kill_all(procs: dict) -> None:
|
||||
"""Ctrl+C 或异常时,杀掉所有子进程"""
|
||||
for key, p in procs.items():
|
||||
if p.poll() is None:
|
||||
log(f" kill [{key}] (pid={p.pid})")
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
p.terminate() # Windows: CTRL_BREAK_EVENT
|
||||
else:
|
||||
p.terminate()
|
||||
except Exception as exc:
|
||||
log(f" kill [{key}] 失败: {exc}")
|
||||
|
||||
|
||||
def wait_all(procs: dict) -> dict[str, int]:
|
||||
"""等待所有进程结束,返回 {key: returncode}"""
|
||||
rc = {}
|
||||
while procs:
|
||||
for key in list(procs.keys()):
|
||||
p = procs[key]
|
||||
ret = p.poll()
|
||||
if ret is not None:
|
||||
rc[key] = ret
|
||||
log(f" [{key}] 完成, exit={ret}")
|
||||
del procs[key]
|
||||
if procs:
|
||||
time.sleep(2)
|
||||
return rc
|
||||
|
||||
|
||||
def _summary_path(key: str, self_operated: bool = False) -> Path:
|
||||
info = SCRIPTS[key]
|
||||
name = info["summary_file"]
|
||||
if self_operated:
|
||||
name = name.replace("_all_summaries", "_all_summaries_self", 1)
|
||||
return V2_DIR / name
|
||||
|
||||
|
||||
def _item_total(item: dict) -> int | None:
|
||||
raw = item.get("total")
|
||||
if raw is None:
|
||||
raw = item.get("total_b_records")
|
||||
if raw is None:
|
||||
raw = item.get("total_tasks")
|
||||
try:
|
||||
value = int(raw or 0)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return value if value >= 0 else None
|
||||
|
||||
|
||||
def _item_rows(item: dict) -> list[dict]:
|
||||
raw = item.get("results")
|
||||
if raw is None:
|
||||
raw = item.get("details")
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
return [row for row in raw if isinstance(row, dict)]
|
||||
|
||||
|
||||
def _row_is_unresolved(row: dict) -> bool:
|
||||
status = row.get("status")
|
||||
if status == "blocked_input":
|
||||
return False
|
||||
explicit_failure = bool(
|
||||
row.get("matched") is False
|
||||
or row.get("ok") is False
|
||||
or row.get("write_ok") is False
|
||||
or row.get("error")
|
||||
or row.get("reason")
|
||||
)
|
||||
if status == "success":
|
||||
return explicit_failure
|
||||
if status:
|
||||
return True
|
||||
return explicit_failure
|
||||
|
||||
|
||||
def _item_unresolved_count(item: dict) -> int:
|
||||
total = _item_total(item) or 0
|
||||
rows = _item_rows(item)
|
||||
record_ids = [str(row["record_id"]) for row in rows if row.get("record_id")]
|
||||
missing = max(0, total - len(rows))
|
||||
missing_ids = max(0, len(rows) - len(record_ids))
|
||||
duplicates = max(0, len(record_ids) - len(set(record_ids)))
|
||||
row_failures = sum(1 for row in rows if _row_is_unresolved(row))
|
||||
derived = missing + missing_ids + duplicates + row_failures
|
||||
declared_parts = 0
|
||||
for key in (
|
||||
"retryable_failures", "write_failures", "missing_results",
|
||||
"duplicate_results",
|
||||
):
|
||||
try:
|
||||
declared_parts += max(0, int(item.get(key) or 0))
|
||||
except (TypeError, ValueError):
|
||||
declared_parts += 1
|
||||
try:
|
||||
declared_total = max(0, int(item.get("unresolved") or 0))
|
||||
except (TypeError, ValueError):
|
||||
declared_total = 1
|
||||
declared = max(declared_parts, declared_total)
|
||||
if item.get("complete") is False:
|
||||
declared = max(declared, 1)
|
||||
return max(derived, declared)
|
||||
|
||||
|
||||
def validate_summary_payload(payload, expected_indices: set[int], path: Path,
|
||||
started_at: float) -> tuple[bool, str]:
|
||||
"""验证汇总确实属于本轮且完整覆盖所选款式。"""
|
||||
try:
|
||||
if not path.exists():
|
||||
return False, "summary file missing"
|
||||
if path.stat().st_mtime < started_at:
|
||||
return False, "summary file is stale"
|
||||
except OSError as exc:
|
||||
return False, f"summary stat failed: {exc}"
|
||||
if not isinstance(payload, list):
|
||||
return False, "summary payload is not a list"
|
||||
|
||||
indexed_items: list[tuple[int, dict]] = []
|
||||
for item in payload:
|
||||
if not isinstance(item, dict):
|
||||
return False, "summary contains non-object item"
|
||||
try:
|
||||
idx = int(item.get("index"))
|
||||
except (TypeError, ValueError):
|
||||
return False, "summary item has invalid index"
|
||||
indexed_items.append((idx, item))
|
||||
|
||||
indices = [idx for idx, _item in indexed_items]
|
||||
if len(indices) != len(set(indices)):
|
||||
return False, "summary has duplicate style index"
|
||||
actual_indices = set(indices)
|
||||
missing_indices = set(expected_indices) - actual_indices
|
||||
if missing_indices:
|
||||
return False, (
|
||||
f"style coverage mismatch expected={sorted(expected_indices)} "
|
||||
f"actual={sorted(actual_indices)}"
|
||||
)
|
||||
|
||||
# 采集器会按款式合并全局汇总；单款运行后文件中可能保留其他款式的
|
||||
# 历史结果。这里只校验本轮明确选择的款式，完整运行时 expected_indices
|
||||
# 本身就是全部当前款式，仍会严格校验全量。
|
||||
for idx, item in indexed_items:
|
||||
if idx not in expected_indices:
|
||||
continue
|
||||
if item.get("error"):
|
||||
return False, f"style {idx} reports error: {item.get('error')}"
|
||||
if started_at > 0:
|
||||
try:
|
||||
item_started_at = float(item.get("run_started_at"))
|
||||
except (TypeError, ValueError):
|
||||
return False, f"style {idx} has no current run marker"
|
||||
if item_started_at < started_at:
|
||||
return False, f"style {idx} was not generated in current run"
|
||||
total = _item_total(item)
|
||||
if total is None:
|
||||
return False, f"style {idx} has invalid total"
|
||||
rows = _item_rows(item)
|
||||
record_ids = [str(row["record_id"]) for row in rows if row.get("record_id")]
|
||||
if len(rows) != total:
|
||||
return False, f"style {idx} result coverage {len(rows)}/{total}"
|
||||
if len(record_ids) != len(rows):
|
||||
return False, f"style {idx} has result without record_id"
|
||||
if len(set(record_ids)) != len(record_ids):
|
||||
return False, f"style {idx} has duplicate record_id"
|
||||
unresolved = _item_unresolved_count(item)
|
||||
if unresolved:
|
||||
return False, f"style {idx} has {unresolved} unresolved result(s)"
|
||||
return True, ""
|
||||
|
||||
|
||||
def load_summary(key: str, self_operated: bool = False, *,
|
||||
started_at: float | None = None,
|
||||
expected_indices: set[int] | None = None) -> dict | list | None:
|
||||
"""加载并可选地验证单个平台本轮汇总 JSON。"""
|
||||
p = _summary_path(key, self_operated)
|
||||
if not p.exists():
|
||||
return None
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
except Exception as exc:
|
||||
return {"error": f"load summary failed: {exc}"}
|
||||
if started_at is not None and expected_indices is not None:
|
||||
valid, reason = validate_summary_payload(payload, expected_indices, p, started_at)
|
||||
if not valid:
|
||||
return {"error": f"invalid summary: {reason}", "payload": payload}
|
||||
return payload
|
||||
|
||||
|
||||
def aggregate(summaries: dict[str, dict]) -> dict:
|
||||
"""聚合三个平台的汇总"""
|
||||
out = {
|
||||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"platforms": {},
|
||||
"totals": {
|
||||
"styles": 0, "tasks": 0, "updated": 0, "skipped": 0,
|
||||
"unresolved": 0, "errors": 0, "hit_rate_pct": 0.0,
|
||||
},
|
||||
}
|
||||
for key, s in summaries.items():
|
||||
if s is None:
|
||||
out["platforms"][key] = {"name": SCRIPTS[key]["name"], "loaded": False}
|
||||
continue
|
||||
validation_error = None
|
||||
if isinstance(s, dict) and "error" in s:
|
||||
validation_error = s["error"]
|
||||
if isinstance(s.get("payload"), list):
|
||||
s = s["payload"]
|
||||
else:
|
||||
out["platforms"][key] = {
|
||||
"name": SCRIPTS[key]["name"], "error": validation_error
|
||||
}
|
||||
continue
|
||||
# 三个脚本的 summary 结构略有不同,做归一化
|
||||
# pgy: [{style, results, total, matched, filled, ...}]
|
||||
# xt: 同 pgy
|
||||
# bili: [{style, total_b_records, updated, skipped, details}]
|
||||
platform = {
|
||||
"name": SCRIPTS[key]["name"],
|
||||
"loaded": True,
|
||||
"styles": len(s) if isinstance(s, list) else 0,
|
||||
"tasks": 0,
|
||||
"updated": 0,
|
||||
"skipped": 0,
|
||||
"unresolved": 0,
|
||||
"errors": 0,
|
||||
"hit_rate_pct": 0.0,
|
||||
}
|
||||
if validation_error:
|
||||
platform["validation_error"] = validation_error
|
||||
if isinstance(s, list):
|
||||
for item in s:
|
||||
t = _item_total(item) or 0
|
||||
# 命中:pgy/xt 用 "matched",bili 用 "updated"
|
||||
u = (item.get("matched") if item.get("matched") is not None
|
||||
else item.get("updated") if item.get("updated") is not None
|
||||
else item.get("hits") if item.get("hits") is not None
|
||||
else 0)
|
||||
# 跳过:只有 bili 用 "skipped",pgy/xt 没用
|
||||
sk = (item.get("skipped") if item.get("skipped") is not None
|
||||
else item.get("misses") if item.get("misses") is not None
|
||||
else 0)
|
||||
declared_errors = (item.get("error_count") if item.get("error_count") is not None
|
||||
else item.get("errors") if item.get("errors") is not None
|
||||
else 0)
|
||||
try:
|
||||
declared_errors = int(declared_errors or 0)
|
||||
except (TypeError, ValueError):
|
||||
declared_errors = 1
|
||||
unresolved = _item_unresolved_count(item)
|
||||
err = max(declared_errors, unresolved)
|
||||
platform["tasks"] += t
|
||||
platform["updated"] += u
|
||||
platform["skipped"] += sk
|
||||
platform["unresolved"] += unresolved
|
||||
platform["errors"] += err
|
||||
if platform["tasks"] > 0:
|
||||
platform["hit_rate_pct"] = round(100.0 * platform["updated"] / platform["tasks"], 1)
|
||||
out["platforms"][key] = platform
|
||||
out["totals"]["styles"] += platform["styles"]
|
||||
out["totals"]["tasks"] += platform["tasks"]
|
||||
out["totals"]["updated"] += platform["updated"]
|
||||
out["totals"]["skipped"] += platform["skipped"]
|
||||
out["totals"]["unresolved"] += platform["unresolved"]
|
||||
out["totals"]["errors"] += platform["errors"]
|
||||
if out["totals"]["tasks"] > 0:
|
||||
out["totals"]["hit_rate_pct"] = round(
|
||||
100.0 * out["totals"]["updated"] / out["totals"]["tasks"], 1
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def print_report(agg: dict, dur_sec: float) -> None:
|
||||
"""打印汇总报告"""
|
||||
print()
|
||||
print("=" * 70)
|
||||
print(f" V2 全平台汇总报告 (总耗时 {dur_sec:.1f}s)")
|
||||
print("=" * 70)
|
||||
print(f" {'平台':<14} {'款式':>5} {'任务':>6} {'命中':>6} {'跳过':>6} {'未完成':>6} {'错误':>5} {'命中率':>7}")
|
||||
print(" " + "-" * 72)
|
||||
for key, p in agg["platforms"].items():
|
||||
if p.get("loaded"):
|
||||
print(f" {p['name']:<14} {p['styles']:>5} {p['tasks']:>6} "
|
||||
f"{p['updated']:>6} {p['skipped']:>6} {p['unresolved']:>6} {p['errors']:>5} "
|
||||
f"{p['hit_rate_pct']:>6.1f}%")
|
||||
if p.get("validation_error"):
|
||||
print(f" [不完整] {p['validation_error']}")
|
||||
else:
|
||||
err = p.get('error', 'no summary file')
|
||||
print(f" {SCRIPTS[key]['name']:<14} -- ({err})")
|
||||
t = agg["totals"]
|
||||
print(" " + "-" * 72)
|
||||
print(f" {'合计':<14} {t['styles']:>5} {t['tasks']:>6} "
|
||||
f"{t['updated']:>6} {t['skipped']:>6} {t['unresolved']:>6} {t['errors']:>5} "
|
||||
f"{t['hit_rate_pct']:>6.1f}%")
|
||||
print("=" * 70)
|
||||
print(f" 详细日志: {LOG_DIR}")
|
||||
print(f" 详细结果: {V2_DIR}/<款式>{['_v2', '_xingtu_v2', '_bilibili_v2'][0]}.json 等")
|
||||
print()
|
||||
|
||||
|
||||
def _expected_style_indices(args: argparse.Namespace,
|
||||
self_operated: bool = False) -> set[int]:
|
||||
if args.style:
|
||||
return {int(idx) for idx in args.style}
|
||||
try:
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping
|
||||
mapping = feishu_mapping.load_mapping(self_operated=self_operated)
|
||||
return {
|
||||
int(style["index"])
|
||||
for style in mapping.get("tables", [])
|
||||
if isinstance(style, dict) and style.get("index")
|
||||
}
|
||||
except Exception as exc:
|
||||
log(f"[ERROR] 无法读取期望款式列表: {exc}")
|
||||
return set()
|
||||
|
||||
|
||||
def _atomic_write_json(path: Path, payload) -> None:
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def run_round(args: argparse.Namespace, active: list[str],
|
||||
self_operated: bool = False) -> dict:
|
||||
"""跑一轮(合作达人或自营达人),返回聚合报告 dict。"""
|
||||
label = "自营达人" if self_operated else "合作达人"
|
||||
log(f"===== 开始 {label} 抓取 =====")
|
||||
|
||||
print("=" * 70)
|
||||
print(f" V2 三平台并行抓取 — {label} ({len(active)}/{len(SCRIPTS)} 个平台)")
|
||||
print(f" 模式: {'DRY-RUN (不写)' if args.dry_run else '正式跑 (写回多维表格)'}")
|
||||
if args.style:
|
||||
print(f" 款式: {','.join(str(s) for s in args.style)}")
|
||||
else:
|
||||
try:
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping as _fm
|
||||
n = len(_fm.load_mapping(self_operated=self_operated).get("tables", []))
|
||||
print(f" 款式: 全部 {n} 款")
|
||||
except Exception:
|
||||
print(f" 款式: 全部")
|
||||
print(f" 平台: {','.join(SCRIPTS[k]['name'] for k in active)}")
|
||||
print("=" * 70)
|
||||
|
||||
expected_indices = _expected_style_indices(args, self_operated)
|
||||
if not expected_indices:
|
||||
reason = "expected style mapping is empty"
|
||||
log(f"[ERROR] {reason}")
|
||||
return {"error": reason, "_non_zero_platforms": list(active)}
|
||||
|
||||
procs = {}
|
||||
log_files = {}
|
||||
start = time.time()
|
||||
try:
|
||||
for key in active:
|
||||
p, out_path, err_path = start_process(key, args, self_operated=self_operated)
|
||||
procs[key] = p
|
||||
log_files[key] = (out_path, err_path)
|
||||
except Exception as exc:
|
||||
log(f"启动失败: {exc}")
|
||||
kill_all(procs)
|
||||
return {"error": str(exc), "_non_zero_platforms": list(active)}
|
||||
|
||||
try:
|
||||
rc = wait_all(procs)
|
||||
except KeyboardInterrupt:
|
||||
log("Ctrl+C - 终止所有子进程...")
|
||||
kill_all(procs)
|
||||
return {"error": "interrupted", "_non_zero_platforms": list(active)}
|
||||
|
||||
dur = time.time() - start
|
||||
log(f"{label} 完成,耗时 {dur:.1f}s exit codes: {rc}")
|
||||
|
||||
summaries = {
|
||||
k: load_summary(
|
||||
k, self_operated=self_operated,
|
||||
started_at=start, expected_indices=expected_indices,
|
||||
)
|
||||
for k in active
|
||||
}
|
||||
agg = aggregate(summaries)
|
||||
print_report(agg, dur)
|
||||
|
||||
report_suffix = "_self" if self_operated else ""
|
||||
report_path = RUN_REPORT_DIR / f"_run_all_report{report_suffix}.json"
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
agg["duration_sec"] = round(dur, 1)
|
||||
agg["exit_codes"] = rc
|
||||
agg["self_operated"] = self_operated
|
||||
agg["log_files"] = {
|
||||
k: {"stdout": str(v[0].name), "stderr": str(v[1].name)}
|
||||
for k, v in log_files.items()
|
||||
}
|
||||
non_zero = {k for k, v in rc.items() if v != 0}
|
||||
non_zero.update(
|
||||
k for k, summary in summaries.items()
|
||||
if summary is None or (isinstance(summary, dict) and summary.get("error"))
|
||||
)
|
||||
agg["_non_zero_platforms"] = sorted(non_zero)
|
||||
_atomic_write_json(report_path, agg)
|
||||
log(f"{label} 汇总报告: {report_path}")
|
||||
return agg
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="一键跑 3 个平台 V2 抓取(真并行,带汇总报告)"
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="三个脚本都只抓不写")
|
||||
parser.add_argument("--style", type=int, action="append",
|
||||
help="只跑指定款式编号(可多次)。不传 = 全跑")
|
||||
parser.add_argument("--skip", type=str, default="",
|
||||
help="跳过平台,逗号分隔。可选: bili,pgy,xt。例如 --skip xt,xt")
|
||||
parser.add_argument("--include-self-operated", action="store_true",
|
||||
help="同时跑自营达人(默认只跑合作达人)")
|
||||
parser.add_argument("--self-operated-only", action="store_true",
|
||||
help="只跑自营达人")
|
||||
args = parser.parse_args()
|
||||
|
||||
skip_set = set(s.strip() for s in args.skip.split(",") if s.strip())
|
||||
for s in skip_set:
|
||||
if s not in SCRIPTS:
|
||||
print(f"[ERROR] --skip 未知平台: {s} (可选: {','.join(SCRIPTS.keys())})")
|
||||
return 1
|
||||
|
||||
active = [k for k in SCRIPTS.keys() if k not in skip_set]
|
||||
if not active:
|
||||
print("[ERROR] --skip 全部平台,没东西可跑")
|
||||
return 1
|
||||
|
||||
# 启动并行子进程前,先刷新款式对照表,
|
||||
# 避免 3 个子进程各自刷新时重复打 lark-cli 并抢写缓存文件。
|
||||
run_self = args.include_self_operated or args.self_operated_only
|
||||
try:
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping
|
||||
m = feishu_mapping.refresh_mapping(force=True)
|
||||
if m:
|
||||
log(f"合作达人款式对照表已动态刷新: {len(m.get('tables', []))} 个款式")
|
||||
except Exception as exc:
|
||||
log(f"[WARN] 动态刷新合作达人款式对照表失败(将退回缓存): {exc}")
|
||||
if run_self:
|
||||
try:
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping
|
||||
m = feishu_mapping.refresh_mapping(force=True, url_field=feishu_mapping.INDEX_SELF_URL_FIELD)
|
||||
if m:
|
||||
log(f"自营达人款式对照表已动态刷新: {len(m.get('tables', []))} 个款式")
|
||||
except Exception as exc:
|
||||
log(f"[WARN] 动态刷新自营达人款式对照表失败(将退回缓存): {exc}")
|
||||
|
||||
any_error = False
|
||||
|
||||
# 第一轮: 合作达人(除非 --self-operated-only)
|
||||
if not args.self_operated_only:
|
||||
agg = run_round(args, active, self_operated=False)
|
||||
if agg.get("_non_zero_platforms"):
|
||||
log(f"[WARN] 合作达人: 以下平台 exit != 0: {agg['_non_zero_platforms']}")
|
||||
any_error = True
|
||||
|
||||
# 第二轮: 自营达人(仅 --include-self-operated 或 --self-operated-only 时跑)
|
||||
if args.include_self_operated or args.self_operated_only:
|
||||
agg = run_round(args, active, self_operated=True)
|
||||
if agg.get("_non_zero_platforms"):
|
||||
log(f"[WARN] 自营达人: 以下平台 exit != 0: {agg['_non_zero_platforms']}")
|
||||
any_error = True
|
||||
|
||||
return 1 if any_error else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Portable path boundary for the migrated content-marketing runtime.
|
||||
|
||||
Code and immutable resources live below ``GYXX_MODULE_ROOT``. All mutable
|
||||
state is kept below ``GYXX_DATA_ROOT`` so a deployment can move either tree
|
||||
without editing business scripts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.core import DataLayout, Settings
|
||||
|
||||
|
||||
def _path_from_env(name: str, default: Path) -> Path:
|
||||
value = os.environ.get(name, "").strip()
|
||||
return Path(value).expanduser().resolve() if value else default.resolve()
|
||||
|
||||
|
||||
module_root = _path_from_env("GYXX_MODULE_ROOT", Path(__file__).resolve().parent)
|
||||
data_root = _path_from_env("GYXX_DATA_ROOT", Settings.from_env().data_root)
|
||||
_DATA_PATHS = DataLayout(data_root).for_module("content_marketing")
|
||||
|
||||
state_root = _DATA_PATHS.state_root
|
||||
raw_root = _DATA_PATHS.raw_root
|
||||
normalized_root = _DATA_PATHS.normalized_root
|
||||
curated_root = _DATA_PATHS.curated_root
|
||||
exports_root = _DATA_PATHS.exports_root
|
||||
evidence_root = _DATA_PATHS.evidence_root
|
||||
logs_root = _DATA_PATHS.logs_root
|
||||
tmp_root = _DATA_PATHS.tmp_root
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimePaths:
|
||||
module_root: Path = module_root
|
||||
data_root: Path = data_root
|
||||
state_root: Path = state_root
|
||||
raw_root: Path = raw_root
|
||||
normalized_root: Path = normalized_root
|
||||
curated_root: Path = curated_root
|
||||
exports_root: Path = exports_root
|
||||
evidence_root: Path = evidence_root
|
||||
logs_root: Path = logs_root
|
||||
tmp_root: Path = tmp_root
|
||||
config_root: Path = module_root / "data" / "config"
|
||||
tools_root: Path = module_root / "data" / "tools"
|
||||
browser_profile_dir: Path = _path_from_env(
|
||||
"GYXX_BROWSER_PROFILE_DIR", state_root / "browser-profiles" / "default"
|
||||
)
|
||||
browser_cookie_file: Path = _path_from_env(
|
||||
"GYXX_BROWSER_COOKIE_FILE", state_root / "cookies" / "default.json"
|
||||
)
|
||||
browser_storage_state_file: Path = _path_from_env(
|
||||
"GYXX_BROWSER_STORAGE_STATE_FILE",
|
||||
state_root / "cookies" / "default_storage_state.json",
|
||||
)
|
||||
|
||||
|
||||
PATHS = RuntimePaths()
|
||||
|
||||
|
||||
def resolve_layer_output(
|
||||
configured: str | Path,
|
||||
*,
|
||||
layer_root: Path,
|
||||
field: str = "output path",
|
||||
) -> Path:
|
||||
"""Resolve a caller-supplied output without allowing a data-layer escape."""
|
||||
|
||||
if not isinstance(configured, (str, Path)) or not str(configured).strip():
|
||||
raise ValueError(f"{field} must be a non-empty path")
|
||||
root = Path(layer_root).expanduser().resolve()
|
||||
requested = Path(configured).expanduser()
|
||||
candidate = (
|
||||
requested.resolve()
|
||||
if requested.is_absolute()
|
||||
else (root / requested).resolve()
|
||||
)
|
||||
if not candidate.is_relative_to(root):
|
||||
raise ValueError(f"{field} must stay inside {root}")
|
||||
return candidate
|
||||
|
||||
|
||||
__all__ = ["PATHS", "RuntimePaths", "resolve_layer_output"]
|
||||
@@ -0,0 +1,455 @@
|
||||
"""
|
||||
自营 B 站采集: B 站公开 API (api.bilibili.com/x/web-interface/view) 抓播放量
|
||||
============================================================================
|
||||
- 不用登录,不用 cookie
|
||||
- 动态读取自营 mapping (feishu_mapping.load_mapping self_operated=True)
|
||||
- 遍历每个有 base_token+table_id 的款式表
|
||||
- 过滤 platform=B站 的记录,必须有 note_url
|
||||
- 从 note_url 抽 BV/AV 号,调 API 拿 stat.view + data.pubdate
|
||||
- 飞书无发布时间 → 用 API pubdate (毫秒时间戳) 填进去,再按时槽回填播放量
|
||||
- 时槽规则跟 chanmama_scraper.py / bilibili_scraper.py 一致:
|
||||
0-7 天 → 7天曝光量
|
||||
8-14 天 → 14天曝光量
|
||||
15-21 天 → 21天曝光量
|
||||
22-28 天 → 28天曝光量
|
||||
29+ 天 → 月底曝光量
|
||||
|
||||
用法:
|
||||
python self_bilibili_scraper.py # 全跑
|
||||
python self_bilibili_scraper.py --dry-run # 只抓不写
|
||||
python self_bilibili_scraper.py --style 1 # 只跑某款
|
||||
python self_bilibili_scraper.py --delay 0.5 # 请求间隔
|
||||
"""
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
# Windows 控制台输出用 UTF-8,避免 emoji 编码失败
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")
|
||||
|
||||
import requests
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
|
||||
LARK_CLI = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd")
|
||||
if not os.path.exists(LARK_CLI):
|
||||
LARK_CLI = "lark-cli.cmd"
|
||||
|
||||
_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
|
||||
_BV_RE = re.compile(r"BV[0-9A-Za-z]{10}")
|
||||
_AV_RE = re.compile(r"av(\d+)", re.IGNORECASE)
|
||||
_B23_RE = re.compile(r"https?://b23\.tv/\S+", re.IGNORECASE)
|
||||
_MD_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
|
||||
_URL_RE = re.compile(r"https?://[^\s)>\]]+")
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
safe = str(msg).encode("gbk", errors="replace").decode("gbk", errors="replace")
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {safe}", flush=True)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 飞书多维表格
|
||||
# ============================================================
|
||||
def call_lark_json(args: list[str]) -> dict:
|
||||
cmd = [LARK_CLI] + args
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace")
|
||||
text = proc.stdout if proc.stdout else (proc.stderr or "")
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
return {"ok": False, "error": f"no json: {text[:300]}"}
|
||||
try:
|
||||
return json.loads(text[start:], strict=False)
|
||||
except json.JSONDecodeError as e:
|
||||
return {"ok": False, "error": f"json decode: {e}"}
|
||||
|
||||
|
||||
def list_records_by_table(base_token: str, table_id: str) -> list[dict]:
|
||||
all_records = []
|
||||
offset = 0
|
||||
while True:
|
||||
resp = call_lark_json([
|
||||
"base", "+record-list",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--limit", "200",
|
||||
"--offset", str(offset),
|
||||
])
|
||||
if not resp.get("ok"):
|
||||
raise RuntimeError(f"record-list 失败: {resp.get('error')}")
|
||||
d = resp["data"]
|
||||
ids = d["field_id_list"]
|
||||
rids = d["record_id_list"]
|
||||
rows = d["data"]
|
||||
for i, row in enumerate(rows):
|
||||
rec = {"record_id": rids[i]}
|
||||
for j, fid in enumerate(ids):
|
||||
rec[fid] = row[j] if j < len(row) else None
|
||||
all_records.append(rec)
|
||||
offset += len(rows)
|
||||
if not d.get("has_more", False) or len(rows) == 0:
|
||||
break
|
||||
return all_records
|
||||
|
||||
|
||||
def write_back(base_token: str, table_id: str, record_id: str,
|
||||
field_id: str, value) -> bool:
|
||||
payload = {field_id: value}
|
||||
resp = call_lark_json([
|
||||
"base", "+record-upsert",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--record-id", record_id,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--json", json.dumps(payload, ensure_ascii=False),
|
||||
])
|
||||
if not resp.get("ok"):
|
||||
log(f" ❌ 回填失败 record_id={record_id}: {resp.get('error')}")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ============================================================
|
||||
# URL / B 站 API
|
||||
# ============================================================
|
||||
def extract_url(text) -> str | None:
|
||||
"""从 note_url 字段抽真实 URL,处理 markdown / 纯文本 / b23 短链"""
|
||||
if text is None:
|
||||
return None
|
||||
if isinstance(text, list):
|
||||
text = text[0] if text else None
|
||||
if not text:
|
||||
return None
|
||||
s = str(text).strip()
|
||||
m = _MD_LINK_RE.search(s)
|
||||
if m:
|
||||
return m.group(2).strip()
|
||||
m = _URL_RE.search(s)
|
||||
if m:
|
||||
return m.group(0).strip()
|
||||
return None
|
||||
|
||||
|
||||
def resolve_b23(url: str, session: requests.Session) -> str | None:
|
||||
try:
|
||||
r = session.get(url, allow_redirects=True, timeout=10,
|
||||
headers={"User-Agent": _UA})
|
||||
return r.url
|
||||
except Exception as exc:
|
||||
log(f" [WARN] b23 解析失败: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_bili_stat(url: str, session: requests.Session) -> tuple[int | None, int | None]:
|
||||
"""调 B 站 API,返回 (play_count, pubdate_unix_seconds)。
|
||||
pubdate 是视频发布时间(Unix 秒),用于补飞书的发布时间字段。
|
||||
"""
|
||||
real = url
|
||||
if _B23_RE.match(url):
|
||||
resolved = resolve_b23(url, session)
|
||||
if not resolved:
|
||||
return None, None
|
||||
real = resolved
|
||||
bv = _BV_RE.search(real)
|
||||
av = _AV_RE.search(real)
|
||||
if not bv and not av:
|
||||
return None, None
|
||||
params = {"bvid": bv.group(0)} if bv else {"aid": av.group(1)}
|
||||
try:
|
||||
r = session.get("https://api.bilibili.com/x/web-interface/view",
|
||||
params=params, timeout=10,
|
||||
headers={"User-Agent": _UA,
|
||||
"Referer": "https://www.bilibili.com/"})
|
||||
j = r.json()
|
||||
if j.get("code") != 0:
|
||||
return None, None
|
||||
data = j.get("data", {})
|
||||
stat = data.get("stat", {})
|
||||
view = stat.get("view")
|
||||
pubdate = data.get("pubdate")
|
||||
return (int(view) if view is not None else None,
|
||||
int(pubdate) if pubdate else None)
|
||||
except Exception as exc:
|
||||
log(f" [WARN] B 站 API: {exc}")
|
||||
return None, None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 时间 / 槽位
|
||||
# ============================================================
|
||||
def parse_publish_time(value) -> datetime | None:
|
||||
"""飞书发布时间字段可能是毫秒时间戳 / 字符串 / datetime。"""
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
try:
|
||||
if value > 1e12:
|
||||
return datetime.fromtimestamp(value / 1000)
|
||||
return datetime.fromtimestamp(value)
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
v = value.strip()
|
||||
if not v:
|
||||
return None
|
||||
if v.isdigit():
|
||||
try:
|
||||
iv = int(v)
|
||||
if iv > 1e12:
|
||||
return datetime.fromtimestamp(iv / 1000)
|
||||
return datetime.fromtimestamp(iv)
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d", "%Y/%m/%d"):
|
||||
try:
|
||||
return datetime.strptime(v[:len(fmt) + 2], fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def pick_read_field(fmap: dict, pub_time: datetime | None,
|
||||
today: datetime | None = None) -> tuple[str, str, str] | None:
|
||||
if pub_time is None:
|
||||
return None
|
||||
today = today or datetime.now()
|
||||
days = (today.date() - pub_time.date()).days
|
||||
if days < 0:
|
||||
return "FUTURE"
|
||||
if days <= 7:
|
||||
logical = "read_count_7d"
|
||||
elif days <= 14:
|
||||
logical = "read_count_14d"
|
||||
elif days <= 21:
|
||||
logical = "read_count_21d"
|
||||
elif days <= 28:
|
||||
logical = "read_count_28d"
|
||||
else:
|
||||
logical = "month_end"
|
||||
info = fmap.get(logical)
|
||||
if info:
|
||||
return (logical, info["field_id"], info["field_name"])
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 采集一个款式
|
||||
# ============================================================
|
||||
def process_style(style: dict, dry_run: bool, delay: float,
|
||||
session: requests.Session) -> dict:
|
||||
base_token = style["base_token"]
|
||||
table_id = style["table_id"]
|
||||
fmap = style.get("field_map", {})
|
||||
|
||||
platform_fid = (fmap.get("platform") or {}).get("field_id")
|
||||
url_fid = (fmap.get("note_url") or {}).get("field_id")
|
||||
title_fid = (fmap.get("note_title") or {}).get("field_id")
|
||||
pub_fid = (fmap.get("publish_time") or {}).get("field_id")
|
||||
|
||||
summary = {
|
||||
"style": style["name"], "index": style["index"],
|
||||
"total": 0, "matched": 0, "filled": 0,
|
||||
"skipped_no_url": 0, "skipped_no_pubtime": 0,
|
||||
"details": [],
|
||||
}
|
||||
|
||||
if not platform_fid or not url_fid or not title_fid:
|
||||
log(f" [WARN] platform/note_url/note_title 字段缺失,跳过")
|
||||
return summary
|
||||
|
||||
try:
|
||||
all_records = list_records_by_table(base_token, table_id)
|
||||
except Exception as exc:
|
||||
log(f" [ERR] 拉记录失败: {exc}")
|
||||
return summary
|
||||
|
||||
def is_bili(v) -> bool:
|
||||
if not v:
|
||||
return False
|
||||
if isinstance(v, list):
|
||||
vals = [str(x).lower() for x in v if x]
|
||||
if not vals:
|
||||
return False
|
||||
return any("b站" in s or "b 站" in s or "bilibili" in s for s in vals)
|
||||
s = str(v).lower()
|
||||
return "b站" in s or "b 站" in s or "bilibili" in s
|
||||
|
||||
bili_records = [r for r in all_records if is_bili(r.get(platform_fid))]
|
||||
log(f" 全表 {len(all_records)} 条,B 站记录 {len(bili_records)} 条")
|
||||
|
||||
today = datetime.now()
|
||||
|
||||
for rec in bili_records:
|
||||
title = rec.get(title_fid)
|
||||
if isinstance(title, list):
|
||||
title = " ".join(str(x) for x in title if x)
|
||||
title = str(title or "").strip()
|
||||
if not title:
|
||||
continue
|
||||
summary["total"] += 1
|
||||
|
||||
url = extract_url(rec.get(url_fid))
|
||||
if not url:
|
||||
log(f" [SKIP] 无 URL: '{title[:20]}'")
|
||||
summary["skipped_no_url"] += 1
|
||||
continue
|
||||
|
||||
# 抓 B 站 API:播放量 + pubdate
|
||||
play, pubdate_unix = fetch_bili_stat(url, session)
|
||||
if play is None:
|
||||
log(f" [SKIP] API 抓不到: '{title[:20]}' (url={url[:50]})")
|
||||
continue
|
||||
summary["matched"] += 1
|
||||
|
||||
# 飞书发布时间
|
||||
pub_time = parse_publish_time(rec.get(pub_fid))
|
||||
|
||||
# 飞书无发布时间 → 用 API pubdate 补,再做时槽判断
|
||||
if pub_time is None and pubdate_unix is not None:
|
||||
api_pub = datetime.fromtimestamp(pubdate_unix)
|
||||
pub_str = api_pub.strftime("%Y-%m-%d %H:%M:%S")
|
||||
if dry_run:
|
||||
log(f" [DRY] '{title[:20]}' → 先填发布时间={pub_str}")
|
||||
else:
|
||||
ts_ms = int(api_pub.timestamp() * 1000)
|
||||
ok = write_back(base_token, table_id, rec["record_id"],
|
||||
pub_fid, ts_ms)
|
||||
if ok:
|
||||
log(f" ✅ '{title[:20]}' → 先填发布时间={pub_str}")
|
||||
pub_time = api_pub
|
||||
|
||||
if pub_time is None:
|
||||
log(f" [SKIP] 飞书+API 都无发布时间: '{title[:20]}'")
|
||||
summary["skipped_no_pubtime"] += 1
|
||||
continue
|
||||
|
||||
pick = pick_read_field(fmap, pub_time, today)
|
||||
if pick == "FUTURE":
|
||||
log(f" [SKIP] 发布时间在未来: '{title[:20]}'")
|
||||
continue
|
||||
if not pick:
|
||||
log(f" [SKIP] 无对应曝光槽位字段: '{title[:20]}'")
|
||||
continue
|
||||
logical, fill_fid, fill_fname = pick
|
||||
|
||||
if dry_run:
|
||||
log(f" [DRY] '{title[:20]}' → {fill_fname}={play}")
|
||||
summary["filled"] += 1
|
||||
summary["details"].append({
|
||||
"record_id": rec["record_id"],
|
||||
"title": title[:50],
|
||||
"play_count": play,
|
||||
"slot": logical,
|
||||
"pub_time": pub_time.isoformat() if pub_time else None,
|
||||
"matched": True,
|
||||
})
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
continue
|
||||
|
||||
ok = write_back(base_token, table_id, rec["record_id"], fill_fid, play)
|
||||
if ok:
|
||||
summary["filled"] += 1
|
||||
summary["details"].append({
|
||||
"record_id": rec["record_id"],
|
||||
"title": title[:50],
|
||||
"play_count": play,
|
||||
"slot": logical,
|
||||
"pub_time": pub_time.isoformat() if pub_time else None,
|
||||
"matched": True,
|
||||
"ok": True,
|
||||
})
|
||||
log(f" ✅ '{title[:20]}' → {fill_fname}={play}")
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
# ============================================================
|
||||
# main
|
||||
# ============================================================
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="自营 B 站采集: B 站公开 API 抓播放量,飞书无发布时间用 API pubdate 补")
|
||||
parser.add_argument("--style", type=int, help="只跑指定款式编号")
|
||||
parser.add_argument("--dry-run", action="store_true", help="不写回飞书")
|
||||
parser.add_argument("--delay", type=float, default=0.3,
|
||||
help="每条请求间隔秒数(防风控)")
|
||||
args = parser.parse_args()
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping
|
||||
try:
|
||||
mapping = feishu_mapping.load_mapping(self_operated=True)
|
||||
except Exception as exc:
|
||||
log(f"❌ 加载自营 mapping 失败: {exc}")
|
||||
return 1
|
||||
|
||||
styles = mapping.get("tables", [])
|
||||
if args.style:
|
||||
styles = [s for s in styles if s.get("index") == args.style]
|
||||
if not styles:
|
||||
log(f"⚠️ 未找到款式 {args.style}")
|
||||
return 1
|
||||
|
||||
out_dir = PATHS.normalized_root / "v2_results"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": _UA})
|
||||
|
||||
print("=" * 50)
|
||||
print(f"🔄 自营 B 站采集 (dry_run={args.dry_run})")
|
||||
print(f" 款式数: {len(styles)}")
|
||||
print("=" * 50)
|
||||
|
||||
all_summaries = []
|
||||
for s in styles:
|
||||
log(f"\n--- [{s['index']:02d}] {s['name']} ---")
|
||||
summary = process_style(s, args.dry_run, args.delay, session)
|
||||
all_summaries.append(summary)
|
||||
log(f" 小计: 任务={summary['total']} 命中={summary['matched']} 回填={summary['filled']}")
|
||||
out_file = out_dir / f"{s['index']:02d}-{s['name']}_self_bilibili_v2.json"
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, ensure_ascii=False, indent=2)
|
||||
|
||||
final_path = out_dir / "_all_summaries_self_bilibili.json"
|
||||
with open(final_path, "w", encoding="utf-8") as f:
|
||||
json.dump(all_summaries, f, ensure_ascii=False, indent=2)
|
||||
|
||||
total = sum(s["total"] for s in all_summaries)
|
||||
matched = sum(s["matched"] for s in all_summaries)
|
||||
filled = sum(s["filled"] for s in all_summaries)
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f"📊 完成: 任务={total} 命中={matched} 回填={filled}")
|
||||
if args.dry_run:
|
||||
print(" (dry-run,未写飞书)")
|
||||
print(f"{'=' * 50}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,692 @@
|
||||
"""
|
||||
自营抖音采集器: 登录 douyin.com, 打开自己账号作品页, 按标题匹配飞书记录, 回填曝光量
|
||||
|
||||
与星图采集器(xingtu_scraper_v2.py)的核心区别:
|
||||
- 星图: 搜索达人 → 打开详情页 → 找视频
|
||||
- 自营抖音: 直接打开自己账号的作品列表 → 滚动加载 → 匹配标题
|
||||
|
||||
Cookie/登录: 复用 douyin_comment_scraper 的 .douyin_chrome_profile + douyin_cookies.json
|
||||
飞书写回: 复用 lark-cli + field_map 体系, 结果文件用 _self_xingtu_v2.json 后缀
|
||||
|
||||
用法:
|
||||
python self_douyin_scraper.py # 全跑
|
||||
python self_douyin_scraper.py --dry-run # 只抓不写
|
||||
python self_douyin_scraper.py --style 3 # 只跑款式3
|
||||
python self_douyin_scraper.py --login-only # 只触发登录
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
from playwright.sync_api import Page
|
||||
from scrapling.fetchers import DynamicSession
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.collection_completeness import (
|
||||
BLOCKED_INPUT,
|
||||
RETRYABLE_FAILURE,
|
||||
SUCCESS,
|
||||
WRITE_FAILURE,
|
||||
add_repeatable_style_argument,
|
||||
atomic_write_json,
|
||||
finalize_summary,
|
||||
match_tasks_to_cards,
|
||||
merge_global_summaries,
|
||||
merge_style_summary,
|
||||
select_requested_styles,
|
||||
)
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
STATE_DIR = PATHS.state_root
|
||||
COOKIE_FILE = PATHS.browser_cookie_file
|
||||
PROFILE_DIR = PATHS.browser_profile_dir
|
||||
|
||||
SELF_POST_URL = "https://www.douyin.com/user/self?showTab=post"
|
||||
|
||||
LARK_CLI = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd")
|
||||
if not os.path.exists(LARK_CLI):
|
||||
LARK_CLI = "lark-cli.cmd"
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
safe = str(msg).encode("gbk", errors="replace").decode("gbk", errors="replace")
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {safe}", flush=True)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 飞书多维表格
|
||||
# ============================================================
|
||||
def call_lark_json(args: list[str]) -> dict:
|
||||
cmd = [LARK_CLI] + args
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env, encoding="utf-8", errors="replace")
|
||||
text = proc.stdout if proc.stdout else (proc.stderr or "")
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
return {"ok": False, "error": f"no json: {text[:300]}"}
|
||||
try:
|
||||
return json.loads(text[start:], strict=False)
|
||||
except json.JSONDecodeError as e:
|
||||
return {"ok": False, "error": f"json decode: {e}"}
|
||||
|
||||
|
||||
def load_mapping() -> dict:
|
||||
from gyxx_flow.modules.content_marketing.runtime import feishu_mapping
|
||||
return feishu_mapping.load_mapping(DATA_DIR, self_operated=True)
|
||||
|
||||
|
||||
def list_records_by_table(base_token: str, table_id: str) -> list[dict]:
|
||||
all_records = []
|
||||
offset = 0
|
||||
while True:
|
||||
resp = call_lark_json([
|
||||
"base", "+record-list",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--limit", "200",
|
||||
"--offset", str(offset),
|
||||
])
|
||||
if not resp.get("ok"):
|
||||
raise RuntimeError(f"record-list 失败: {resp.get('error')}")
|
||||
d = resp["data"]
|
||||
ids = d["field_id_list"]
|
||||
rids = d["record_id_list"]
|
||||
rows = d["data"]
|
||||
for i, row in enumerate(rows):
|
||||
rec = {"record_id": rids[i]}
|
||||
for j, fid in enumerate(ids):
|
||||
rec[fid] = row[j] if j < len(row) else None
|
||||
all_records.append(rec)
|
||||
offset += len(rows)
|
||||
if not d.get("has_more", False) or len(rows) == 0:
|
||||
break
|
||||
return all_records
|
||||
|
||||
|
||||
def write_back(base_token: str, table_id: str, record_id: str,
|
||||
field_id: str, value) -> bool:
|
||||
payload = {field_id: value}
|
||||
resp = call_lark_json([
|
||||
"base", "+record-upsert",
|
||||
"--base-token", base_token,
|
||||
"--table-id", table_id,
|
||||
"--record-id", record_id,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--json", json.dumps(payload, ensure_ascii=False),
|
||||
])
|
||||
if not resp.get("ok"):
|
||||
log(f" 回填失败 record_id={record_id}: {resp.get('error')}")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 时间/槽位
|
||||
# ============================================================
|
||||
def parse_publish_time(value) -> datetime | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
try:
|
||||
return datetime.fromtimestamp(value / 1000)
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
v = value.strip()
|
||||
if not v:
|
||||
return None
|
||||
if v.isdigit():
|
||||
try:
|
||||
return datetime.fromtimestamp(int(v) / 1000)
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d"):
|
||||
try:
|
||||
return datetime.strptime(v[:len(fmt) + 2], fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def pick_read_field(fmap: dict, pub_time: datetime | None,
|
||||
today: datetime | None = None) -> tuple[str, str, str] | None:
|
||||
if pub_time is None:
|
||||
return None
|
||||
today = today or datetime.now()
|
||||
days = (today.date() - pub_time.date()).days
|
||||
if days < 0:
|
||||
return "FUTURE"
|
||||
if days <= 7:
|
||||
logical = "read_count_7d"
|
||||
elif days <= 14:
|
||||
logical = "read_count_14d"
|
||||
elif days <= 21:
|
||||
logical = "read_count_21d"
|
||||
elif days <= 28:
|
||||
logical = "read_count_28d"
|
||||
else:
|
||||
logical = "month_end"
|
||||
info = fmap.get(logical)
|
||||
if info:
|
||||
return (logical, info["field_id"], info["field_name"])
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 数字解析
|
||||
# ============================================================
|
||||
def parse_int(v) -> int | None:
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, (int, float)):
|
||||
return int(v)
|
||||
txt = str(v).strip()
|
||||
if not txt:
|
||||
return None
|
||||
m = re.match(r"^([\d,.]+)\s*(万|w|W)?", txt)
|
||||
if not m:
|
||||
return None
|
||||
num = float(m.group(1).replace(",", ""))
|
||||
if m.group(2):
|
||||
num *= 10000
|
||||
return int(num)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Cookie / 登录
|
||||
# ============================================================
|
||||
def load_cookies() -> list[dict[str, Any]]:
|
||||
if not COOKIE_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
|
||||
|
||||
def restore_cookies(page: Page) -> None:
|
||||
cookies = load_cookies()
|
||||
if not cookies:
|
||||
return
|
||||
try:
|
||||
page.context.add_cookies(cookies)
|
||||
log(f"Loaded cookies: {COOKIE_FILE}")
|
||||
except Exception as exc:
|
||||
log(f"Failed to load cookies: {exc}")
|
||||
|
||||
|
||||
def save_state(page: Page) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
COOKIE_FILE.write_text(
|
||||
json.dumps(page.context.cookies(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
PATHS.browser_storage_state_file.write_text(
|
||||
json.dumps(page.context.storage_state(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
log(f"Saved cookies: {COOKIE_FILE}")
|
||||
|
||||
|
||||
def maybe_wait_for_login(page: Page, login_timeout: int) -> None:
|
||||
deadline = time.time() + login_timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
body = page.locator("body").inner_text(timeout=5000)
|
||||
except Exception:
|
||||
body = ""
|
||||
if "扫码登录" not in body and "登录后" not in body:
|
||||
return
|
||||
log("抖音需要登录,请在打开的 Chrome 窗口中完成登录")
|
||||
page.wait_for_timeout(3000)
|
||||
log("[WARN] 登录等待超时")
|
||||
|
||||
|
||||
def dismiss_popups(page: Page) -> None:
|
||||
for text in ("我知道了", "稍后再说", "暂不登录", "取消", "跳过"):
|
||||
try:
|
||||
btn = page.locator(f"text={text}").first
|
||||
if btn.is_visible(timeout=1000):
|
||||
btn.click(timeout=1000)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
page.evaluate("""
|
||||
() => {
|
||||
const close = [...document.querySelectorAll('button, div, span')]
|
||||
.filter((el) => {
|
||||
const t = (el.innerText || '').trim();
|
||||
const r = el.getBoundingClientRect();
|
||||
return ['×', 'x', 'X'].includes(t) &&
|
||||
r.width > 0 && r.width <= 52 &&
|
||||
r.height > 0 && r.height <= 52 &&
|
||||
r.top >= 0 && r.top < window.innerHeight * 0.75;
|
||||
})[0];
|
||||
close?.click();
|
||||
}
|
||||
""")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 标题匹配
|
||||
# ============================================================
|
||||
def title_match(haystack: str, needle: str) -> bool:
|
||||
"""模糊标题匹配:去掉空格/标点后比较,任一方包含另一方即命中"""
|
||||
def norm(s: str) -> str:
|
||||
return re.sub(r"[\s \-_.,;:!?！？，。；：、/\\()（）【】《》''\"\"'']+", "", str(s))
|
||||
nh = norm(haystack)
|
||||
nn = norm(needle)
|
||||
if not nh or not nn:
|
||||
return False
|
||||
return nn in nh or nh in nn
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 抖音作品页抓取
|
||||
# ============================================================
|
||||
def scroll_and_collect_posts(page: Page, max_scrolls: int = 50) -> list[dict]:
|
||||
"""在 /user/self?showTab=post 页面滚动加载,收集所有作品的标题和播放量。
|
||||
返回 [{"title": str, "play_count": int | None}, ...]
|
||||
"""
|
||||
posts = []
|
||||
seen_titles = set()
|
||||
no_new_count = 0
|
||||
|
||||
for scroll_idx in range(max_scrolls):
|
||||
# 解析当前页面上的作品卡片
|
||||
items = page.evaluate(r"""
|
||||
() => {
|
||||
const cards = document.querySelectorAll('li[data-e2e="user-post-item"], li[class*="user-post"], div[data-e2e="user-post-item"]');
|
||||
const results = [];
|
||||
for (const card of cards) {
|
||||
// 标题: 通常在 <a> 或 <p> 里
|
||||
let title = '';
|
||||
const links = card.querySelectorAll('a');
|
||||
for (const a of links) {
|
||||
const t = (a.textContent || '').trim();
|
||||
if (t.length > title.length) title = t;
|
||||
}
|
||||
if (!title) {
|
||||
const p = card.querySelector('p, span[class*="title"], div[class*="title"]');
|
||||
if (p) title = (p.textContent || '').trim();
|
||||
}
|
||||
// 播放量: 通常在卡片底部,格式如 "4.5万次播放" 或 "1234"
|
||||
let playText = '';
|
||||
const spans = card.querySelectorAll('span');
|
||||
for (const sp of spans) {
|
||||
const t = (sp.textContent || '').trim();
|
||||
if (t.includes('播放') || t.includes('次') || /^[\d,.]+$/.test(t) ||
|
||||
/[\d,.]+\s*(万|w|W)/.test(t)) {
|
||||
if (t.length > playText.length) playText = t;
|
||||
}
|
||||
}
|
||||
if (!playText) {
|
||||
const lastSpan = card.querySelector('span:last-of-type');
|
||||
if (lastSpan) {
|
||||
const t = (lastSpan.textContent || '').trim();
|
||||
if (/[\d]/.test(t)) playText = t;
|
||||
}
|
||||
}
|
||||
if (title) {
|
||||
results.push({title: title, playText: playText});
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
""")
|
||||
|
||||
new_count = 0
|
||||
for item in items:
|
||||
t = item.get("title", "").strip()
|
||||
if not t or t in seen_titles:
|
||||
continue
|
||||
seen_titles.add(t)
|
||||
play = parse_play_text(item.get("playText", ""))
|
||||
posts.append({"title": t, "play_count": play})
|
||||
new_count += 1
|
||||
|
||||
if new_count == 0:
|
||||
no_new_count += 1
|
||||
if no_new_count >= 3:
|
||||
break
|
||||
else:
|
||||
no_new_count = 0
|
||||
|
||||
# 向下滚动
|
||||
page.evaluate("window.scrollBy(0, window.innerHeight * 1.5)")
|
||||
page.wait_for_timeout(800)
|
||||
|
||||
return posts
|
||||
|
||||
|
||||
def parse_play_text(text: str) -> int | None:
|
||||
"""从 '4.5万次播放' / '1234' / '1.2w' 等文本中提取播放量数字"""
|
||||
if not text:
|
||||
return None
|
||||
# 先尝试提取 "X次播放" 格式
|
||||
m = re.search(r"([\d,.]+)\s*(万|w|W)?\s*次播放", text)
|
||||
if m:
|
||||
return parse_int(m.group(1) + (m.group(2) or ""))
|
||||
# 直接 parse_int
|
||||
return parse_int(text)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 采集一个款式
|
||||
# ============================================================
|
||||
def scrape_one_style(style: dict, login_timeout: int, headless: bool,
|
||||
only_record_ids: set[str] | None, dry_run: bool) -> dict:
|
||||
base_token = style["base_token"]
|
||||
table_id = style["table_id"]
|
||||
fmap = style.get("field_map", {})
|
||||
|
||||
summary = {"style": style["name"], "index": style["index"],
|
||||
"run_started_at": time.time(),
|
||||
"total": 0, "matched": 0, "filled": 0,
|
||||
"skipped_no_pubtime": 0, "results": []}
|
||||
|
||||
# 从飞书读取自营达人表记录,只看抖音
|
||||
platform_fid = fmap.get("platform", {}).get("field_id")
|
||||
name_fid = fmap.get("creator_name", {}).get("field_id")
|
||||
id_fid = fmap.get("creator_id", {}).get("field_id")
|
||||
title_fid = fmap.get("note_title", {}).get("field_id")
|
||||
url_fid = fmap.get("note_url", {}).get("field_id")
|
||||
pub_fid = fmap.get("publish_time", {}).get("field_id")
|
||||
|
||||
if not platform_fid or not title_fid or not url_fid or not pub_fid:
|
||||
summary["error"] = "missing platform/note_title/note_url/publish_time field mapping"
|
||||
log(f" [ERROR] platform/note_title/note_url/publish_time 字段缺失,跳过")
|
||||
return summary
|
||||
|
||||
all_records = list_records_by_table(base_token, table_id)
|
||||
|
||||
# 只看抖音记录
|
||||
def is_douyin(v) -> bool:
|
||||
if not v:
|
||||
return False
|
||||
s = str(v).strip()
|
||||
return s in ("抖音", "douyin")
|
||||
|
||||
def is_nonempty(v) -> bool:
|
||||
if v is None:
|
||||
return False
|
||||
if isinstance(v, list):
|
||||
return len(v) > 0
|
||||
if isinstance(v, str):
|
||||
return v.strip() != ""
|
||||
return True
|
||||
|
||||
douyin_records = [r for r in all_records if is_douyin(r.get(platform_fid))]
|
||||
log(f" 全表 {len(all_records)} 条,抖音记录 {len(douyin_records)} 条")
|
||||
|
||||
# 构建任务列表:需要标题+链接+发布时间
|
||||
tasks = []
|
||||
for rec in douyin_records:
|
||||
rid = rec["record_id"]
|
||||
if only_record_ids and rid not in only_record_ids:
|
||||
continue
|
||||
title = rec.get(title_fid)
|
||||
if isinstance(title, list):
|
||||
title = " ".join(str(x) for x in title if x)
|
||||
title = str(title or "").strip()
|
||||
if not title:
|
||||
continue
|
||||
url_raw = rec.get(url_fid)
|
||||
pub_raw = rec.get(pub_fid)
|
||||
creator = rec.get(name_fid) or rec.get(id_fid) or "?"
|
||||
pub_time = parse_publish_time(pub_raw)
|
||||
tasks.append({
|
||||
"record_id": rid,
|
||||
"creator_name": str(creator),
|
||||
"target_title": title,
|
||||
"note_url": str(url_raw or ""),
|
||||
"publish_time": pub_time,
|
||||
})
|
||||
|
||||
summary["total"] = len(tasks)
|
||||
if not tasks:
|
||||
log(f" 无抖音待抓任务,跳过")
|
||||
return summary
|
||||
|
||||
# 启动浏览器:所有浏览器操作都在 page_action 里完成
|
||||
collected_posts: list[dict] = []
|
||||
login_ok = False
|
||||
|
||||
def page_action(page: Page):
|
||||
nonlocal login_ok
|
||||
dismiss_popups(page)
|
||||
restore_cookies(page)
|
||||
page.goto(SELF_POST_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_timeout(3000)
|
||||
dismiss_popups(page)
|
||||
maybe_wait_for_login(page, login_timeout)
|
||||
save_state(page)
|
||||
login_ok = True
|
||||
|
||||
log(" 等待作品列表加载...")
|
||||
page.wait_for_timeout(3000)
|
||||
dismiss_popups(page)
|
||||
|
||||
# 滚动加载所有作品
|
||||
log(" 滚动加载作品列表...")
|
||||
posts = scroll_and_collect_posts(page, max_scrolls=50)
|
||||
log(f" 共加载 {len(posts)} 个作品")
|
||||
collected_posts.extend(posts)
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=headless,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=60000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies,
|
||||
max_pages=3,
|
||||
) as session:
|
||||
session.fetch("https://www.douyin.com/", page_action=page_action, wait=1000)
|
||||
|
||||
if not login_ok:
|
||||
log(f" [ERROR] 登录失败,跳过该款式")
|
||||
return summary
|
||||
|
||||
# 用收集到的作品匹配飞书记录并回填
|
||||
today = datetime.now()
|
||||
posts = collected_posts
|
||||
matched_posts = match_tasks_to_cards(tasks, posts, "douyin")
|
||||
for task in tasks:
|
||||
matched_post = matched_posts.get(str(task["record_id"]))
|
||||
|
||||
if matched_post is None:
|
||||
log(f" [MISS] {task['creator_name']}: '{task['target_title'][:20]}' 未匹配到作品")
|
||||
summary["results"].append({
|
||||
"record_id": task["record_id"],
|
||||
"creator": task["creator_name"],
|
||||
"target": task["target_title"],
|
||||
"status": RETRYABLE_FAILURE,
|
||||
"matched": False,
|
||||
"reason": "title_not_found",
|
||||
})
|
||||
continue
|
||||
|
||||
summary["matched"] += 1
|
||||
play_count = matched_post.get("play_count")
|
||||
log(f" {task['creator_name']}: '{task['target_title'][:20]}' → 播放={play_count}")
|
||||
|
||||
pick = pick_read_field(fmap, task.get("publish_time"), today)
|
||||
outcome: dict[str, Any]
|
||||
if play_count is None:
|
||||
outcome = {
|
||||
"status": RETRYABLE_FAILURE,
|
||||
"write_ok": False,
|
||||
"reason": "metric_parse_failed",
|
||||
}
|
||||
elif dry_run:
|
||||
outcome = {"status": SUCCESS, "write_ok": None, "dry_run": True}
|
||||
elif pick and pick != "FUTURE":
|
||||
logical, fill_fid, fill_fname = pick
|
||||
ok = write_back(base_token, table_id, task["record_id"], fill_fid, play_count)
|
||||
if ok:
|
||||
log(f" 回填 {fill_fname}={play_count} (key={logical})")
|
||||
summary["filled"] += 1
|
||||
outcome = {"status": SUCCESS, "write_ok": True}
|
||||
else:
|
||||
outcome = {
|
||||
"status": WRITE_FAILURE,
|
||||
"write_ok": False,
|
||||
"reason": "write_back_failed",
|
||||
}
|
||||
elif pick == "FUTURE":
|
||||
log(f" [SKIP] record={task['record_id']} 发布时间在未来,跳过回填")
|
||||
outcome = {
|
||||
"status": BLOCKED_INPUT,
|
||||
"write_ok": None,
|
||||
"reason": "publish_time_in_future",
|
||||
}
|
||||
elif not dry_run and not task.get("publish_time"):
|
||||
log(f" [SKIP] record={task['record_id']} 发布时间缺失,无法判断槽位")
|
||||
summary["skipped_no_pubtime"] += 1
|
||||
outcome = {
|
||||
"status": BLOCKED_INPUT,
|
||||
"write_ok": None,
|
||||
"reason": "publish_time_missing",
|
||||
}
|
||||
else:
|
||||
outcome = {
|
||||
"status": WRITE_FAILURE,
|
||||
"write_ok": False,
|
||||
"reason": "write_field_missing",
|
||||
}
|
||||
|
||||
summary["results"].append({
|
||||
"record_id": task["record_id"],
|
||||
"creator": task["creator_name"],
|
||||
"target": task["target_title"],
|
||||
"matched_title": matched_post["title"],
|
||||
"play_count": play_count,
|
||||
"matched": True,
|
||||
**outcome,
|
||||
})
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def login_only(login_timeout: int) -> int:
|
||||
"""只触发登录"""
|
||||
login_done = False
|
||||
|
||||
def page_action(page: Page):
|
||||
restore_cookies(page)
|
||||
page.goto("https://www.douyin.com/", wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_timeout(2000)
|
||||
maybe_wait_for_login(page, login_timeout)
|
||||
save_state(page)
|
||||
login_done = True
|
||||
log("登录完成")
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=False,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=60000,
|
||||
page_setup=restore_cookies,
|
||||
) as session:
|
||||
session.fetch("https://www.douyin.com/", page_action=page_action, wait=1000)
|
||||
|
||||
return 0 if login_done else 1
|
||||
|
||||
|
||||
# ============================================================
|
||||
# main
|
||||
# ============================================================
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="自营抖音采集: douyin.com 自己账号作品曝光量")
|
||||
add_repeatable_style_argument(parser, "只跑指定款式编号(可多次)")
|
||||
parser.add_argument("--record", action="append", help="只跑指定 record_id")
|
||||
parser.add_argument("--headless", action="store_true")
|
||||
parser.add_argument("--login-timeout", type=int, default=300)
|
||||
parser.add_argument("--dry-run", action="store_true", help="不写回多维表格")
|
||||
parser.add_argument("--login-only", action="store_true", help="只触发登录")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.login_only:
|
||||
return login_only(args.login_timeout)
|
||||
|
||||
mapping = load_mapping()
|
||||
styles = mapping["tables"]
|
||||
if args.style:
|
||||
try:
|
||||
styles = select_requested_styles(styles, args.style)
|
||||
except ValueError as exc:
|
||||
print(str(exc))
|
||||
return 1
|
||||
|
||||
only_rids = set(args.record) if args.record else None
|
||||
|
||||
out_dir = PATHS.normalized_root / "v2_results"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self_sfx = "_self"
|
||||
all_summaries = []
|
||||
for s in styles:
|
||||
log(f"\n========== [{s['index']:02d}] {s['name']} (自营抖音) ==========")
|
||||
summary = finalize_summary(
|
||||
scrape_one_style(s, args.login_timeout, args.headless, only_rids, args.dry_run),
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
out_file = out_dir / f"{s['index']:02d}-{s['name']}{self_sfx}_xingtu_v2.json"
|
||||
if only_rids and out_file.exists():
|
||||
try:
|
||||
previous = json.loads(out_file.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
previous = None
|
||||
summary = merge_style_summary(previous, summary, dry_run=args.dry_run)
|
||||
all_summaries.append(summary)
|
||||
atomic_write_json(out_file, summary)
|
||||
log(f" 落盘: {out_file}")
|
||||
log(
|
||||
f" 统计: 总任务={summary['total']} 命中={summary['matched']} "
|
||||
f"回填={summary['filled']} 未解决={summary.get('unresolved', 0)}"
|
||||
)
|
||||
|
||||
final_path = out_dir / f"_all_summaries{self_sfx}_xingtu.json"
|
||||
try:
|
||||
existing_global = json.loads(final_path.read_text(encoding="utf-8")) if final_path.exists() else []
|
||||
except Exception:
|
||||
existing_global = []
|
||||
atomic_write_json(final_path, merge_global_summaries(existing_global, all_summaries))
|
||||
log(f"\n=== 全部完成 ===")
|
||||
log(f"汇总: {final_path}")
|
||||
incomplete = [summary.get("index") for summary in all_summaries if not summary.get("complete")]
|
||||
if incomplete:
|
||||
log(f"[ERROR] 以下款式仍有未解决采集/写回结果: {incomplete}")
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Apply the unified per-script runtime binding to nested Python launches."""
|
||||
|
||||
from gyxx_flow.adapters.bootstrap import safe_bootstrap_current_process
|
||||
|
||||
safe_bootstrap_current_process()
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
TOOLS_DIR = PROJECT_ROOT / "data" / "tools"
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import analyze_note # noqa: E402
|
||||
|
||||
|
||||
class AnalyzeNoteWithoutCommentsTest(unittest.TestCase):
|
||||
def test_no_comments_still_writes_metrics_report(self):
|
||||
info = {
|
||||
"title": "无评论测试笔记",
|
||||
"platform": "xiaohongshu",
|
||||
"source_url": "https://example.com/note",
|
||||
"scraped_at": "",
|
||||
"comment_count_total": 0,
|
||||
"like_count": 12,
|
||||
"favorite_count": 3,
|
||||
"comment_count_metric": 0,
|
||||
"share_count": 1,
|
||||
"view_count": 1000,
|
||||
"collect_count_yxyy": 0,
|
||||
"style_name": "测试款",
|
||||
"creator_name": "测试达人",
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output = Path(tmpdir) / "note.txt"
|
||||
argv = [
|
||||
"analyze_note.py",
|
||||
"--note-id",
|
||||
"1",
|
||||
"--no-send",
|
||||
"--brand",
|
||||
"光影行星",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
with patch.object(analyze_note, "load_note_from_db", return_value=(info, [])), patch.object(
|
||||
sys, "argv", argv
|
||||
):
|
||||
result = analyze_note.main()
|
||||
|
||||
self.assertEqual(result, 0)
|
||||
self.assertTrue(output.exists())
|
||||
report = output.read_text(encoding="utf-8-sig")
|
||||
self.assertIn("无有效评论可分析", report)
|
||||
self.assertIn("1,000", report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,9 @@
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import batch_rescrape_douyin as batch
|
||||
|
||||
|
||||
def test_result_logged_in_reads_nested_scraper_stats():
|
||||
result = {"stats": {"logged_in": True}}
|
||||
|
||||
assert batch.result_logged_in(result) is True
|
||||
assert batch.result_logged_in({"stats": {"logged_in": False}}) is False
|
||||
assert batch.result_logged_in({}) is False
|
||||
+840
@@ -0,0 +1,840 @@
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime import collection_completeness as cc
|
||||
import bilibili_scraper as bili
|
||||
import pgy_xhs_scraper_v2 as pgy
|
||||
import run_all
|
||||
import self_douyin_scraper as self_dy
|
||||
import xingtu_scraper_v2 as xingtu
|
||||
|
||||
|
||||
def test_normalize_title_handles_nfkc_case_and_zero_width():
|
||||
assert cc.normalize_title("ＭａｃＢｏｏｋ\u200b 通勤包！") == "macbook通勤包"
|
||||
|
||||
|
||||
def test_coerce_url_unwraps_feishu_markdown_links():
|
||||
value = "[查看笔记](https://www.xiaohongshu.com/discovery/item/abc123?x=1)"
|
||||
assert cc.coerce_url(value) == "https://www.xiaohongshu.com/discovery/item/abc123?x=1"
|
||||
|
||||
|
||||
def test_match_tasks_prefers_note_id_over_changed_title():
|
||||
tasks = [{
|
||||
"record_id": "r1",
|
||||
"target_title": "飞书中的旧标题",
|
||||
"note_url": "https://www.xiaohongshu.com/explore/abc123",
|
||||
"note_id": "abc123",
|
||||
}]
|
||||
cards = [
|
||||
{"title": "完全不同的新标题", "href": "https://www.xiaohongshu.com/explore/abc123", "note_id": "abc123"},
|
||||
{"title": "飞书中的旧标题", "href": "https://www.xiaohongshu.com/explore/other", "note_id": "other"},
|
||||
]
|
||||
|
||||
matched = cc.match_tasks_to_cards(tasks, cards, platform="xhs")
|
||||
|
||||
assert matched["r1"]["note_id"] == "abc123"
|
||||
assert matched["r1"]["match_method"] == "content_id"
|
||||
|
||||
|
||||
def test_match_tasks_accepts_unique_truncated_title_but_rejects_ambiguous_title():
|
||||
task = {"record_id": "r1", "target_title": "男生长期主义通勤双肩包分享"}
|
||||
unique = [
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
{"title": "夏日轻量斜挎包"},
|
||||
]
|
||||
ambiguous = [
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
]
|
||||
|
||||
assert cc.match_tasks_to_cards([task], unique, platform="douyin")["r1"]
|
||||
assert "r1" not in cc.match_tasks_to_cards([task], ambiguous, platform="douyin")
|
||||
|
||||
|
||||
def test_title_similarity_ignores_episode_prefix_and_hashtag_suffix():
|
||||
target = "第20集：Pocket4拍旋焦，无后期也能出片？"
|
||||
platform_title = (
|
||||
"Pocket4拍旋焦，无后期也能出片？ #旋焦 #pocket4 #摄影装备 "
|
||||
"无论Pocket3还是Pocket4都可以轻松拍出旋焦效果！"
|
||||
)
|
||||
|
||||
assert cc.title_similarity(platform_title, target) >= 0.98
|
||||
|
||||
|
||||
def test_title_similarity_rejects_different_episode_numbers():
|
||||
assert cc.title_similarity(
|
||||
"第19集：Pocket4拍旋焦，无后期也能出片？",
|
||||
"第20集：Pocket4拍旋焦，无后期也能出片？",
|
||||
) == 0.0
|
||||
|
||||
|
||||
def test_match_tasks_rejects_duplicate_exact_titles_with_different_ids():
|
||||
task = {"record_id": "r1", "target_title": "完全相同的目标标题"}
|
||||
cards = [
|
||||
{"title": "完全相同的目标标题", "note_id": "video-1"},
|
||||
{"title": "完全相同的目标标题", "note_id": "video-2"},
|
||||
]
|
||||
|
||||
assert "r1" not in cc.match_tasks_to_cards([task], cards, platform="douyin")
|
||||
|
||||
|
||||
def test_one_card_is_not_reused_for_two_different_creator_tasks():
|
||||
tasks = [
|
||||
{
|
||||
"record_id": "loose",
|
||||
"target_title": "commuter backpack review today",
|
||||
"note_url": "https://v.douyin.com/loose-source/",
|
||||
},
|
||||
{
|
||||
"record_id": "exact",
|
||||
"target_title": "commuter backpack review",
|
||||
"note_url": "https://v.douyin.com/exact-source/",
|
||||
},
|
||||
]
|
||||
cards = [{"title": "commuter backpack review", "note_id": "observed-1"}]
|
||||
|
||||
matched = cc.match_tasks_to_cards(tasks, cards, platform="douyin")
|
||||
|
||||
assert set(matched) == {"exact"}
|
||||
|
||||
|
||||
def test_same_source_url_can_share_one_card_across_styles():
|
||||
tasks = [
|
||||
{
|
||||
"record_id": "style-1",
|
||||
"target_title": "同一篇跨款式笔记",
|
||||
"note_url": "https://v.douyin.com/same-short-link/",
|
||||
},
|
||||
{
|
||||
"record_id": "style-2",
|
||||
"target_title": "同一篇跨款式笔记",
|
||||
"note_url": "https://v.douyin.com/same-short-link/",
|
||||
},
|
||||
]
|
||||
cards = [{"title": "同一篇跨款式笔记", "note_id": "observed-1"}]
|
||||
|
||||
assert set(cc.match_tasks_to_cards(tasks, cards, platform="douyin")) == {
|
||||
"style-1", "style-2",
|
||||
}
|
||||
|
||||
|
||||
def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
|
||||
task = {
|
||||
"record_id": "r1",
|
||||
"target_title": "完全相同标题",
|
||||
"note_id": "video-a",
|
||||
}
|
||||
card = {"title": "完全相同标题", "note_id": "video-b"}
|
||||
|
||||
assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
|
||||
|
||||
|
||||
def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
|
||||
cards = [
|
||||
{
|
||||
"title": "同一条视频",
|
||||
"play_count": 19481,
|
||||
"note_id": "video-1",
|
||||
"source": "show_items_api",
|
||||
},
|
||||
{
|
||||
"title": "同一条视频",
|
||||
"play_count": "1.9w",
|
||||
"note_id": "",
|
||||
"page": 1,
|
||||
},
|
||||
]
|
||||
|
||||
deduped = cc.deduplicate_observed_cards(cards)
|
||||
matched = cc.match_tasks_to_cards(
|
||||
[{"record_id": "r1", "target_title": "同一条视频"}],
|
||||
deduped,
|
||||
platform="douyin",
|
||||
)
|
||||
|
||||
assert len(deduped) == 1
|
||||
assert matched["r1"]["note_id"] == "video-1"
|
||||
|
||||
|
||||
def test_merge_partial_summary_keeps_existing_success_rows():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total": 2, "matched": 1, "filled": 1,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
|
||||
{"record_id": "retry", "status": "retryable_failure", "matched": False},
|
||||
],
|
||||
}
|
||||
partial = {
|
||||
"style": "款式A", "index": 1, "total": 1, "matched": 1, "filled": 1,
|
||||
"results": [
|
||||
{"record_id": "retry", "status": "success", "matched": True, "write_ok": True},
|
||||
],
|
||||
}
|
||||
|
||||
merged = cc.merge_style_summary(existing, partial)
|
||||
|
||||
assert merged["total"] == 2
|
||||
assert {row["record_id"] for row in merged["results"]} == {"ok", "retry"}
|
||||
assert merged["matched"] == 2
|
||||
assert merged["filled"] == 2
|
||||
assert merged["complete"] is True
|
||||
|
||||
|
||||
def test_summary_contract_detects_missing_or_write_failure():
|
||||
summary = {
|
||||
"total": 2,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
|
||||
{"record_id": "bad", "status": "write_failure", "matched": True, "write_ok": False},
|
||||
],
|
||||
}
|
||||
finalized = cc.finalize_summary(summary, dry_run=False)
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] == 1
|
||||
|
||||
missing = cc.finalize_summary({"total": 2, "results": summary["results"][:1]}, dry_run=False)
|
||||
assert missing["complete"] is False
|
||||
assert missing["missing_results"] == 1
|
||||
|
||||
|
||||
def _style_for_platform(platform: str):
|
||||
return {
|
||||
"index": 1,
|
||||
"name": "款式A",
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"creator_name": {"field_id": "name"},
|
||||
"creator_id": {"field_id": "creator_id"},
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"platform": {"field_id": "platform"},
|
||||
"read_count_7d": {"field_id": "read7"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_extractors_preserve_note_url_and_content_id(monkeypatch):
|
||||
xhs_row = {
|
||||
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
|
||||
"title": "标题", "pub": "2026-07-20", "platform": "小红书",
|
||||
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
|
||||
}
|
||||
dy_row = {
|
||||
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
|
||||
"title": "标题", "pub": "2026-07-20", "platform": "抖音",
|
||||
"url": "https://www.douyin.com/video/7654321", "read7": None,
|
||||
}
|
||||
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
|
||||
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
|
||||
|
||||
xhs_task = pgy.extract_target_tasks(_style_for_platform("小红书"))[0]
|
||||
dy_task = xingtu.extract_target_tasks(_style_for_platform("抖音"))[0]
|
||||
|
||||
assert xhs_task["note_url"].endswith("/abc123")
|
||||
assert xhs_task["note_id"] == "abc123"
|
||||
assert dy_task["note_url"].endswith("/7654321")
|
||||
assert dy_task["note_id"] == "7654321"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "filled_url"),
|
||||
[
|
||||
(pgy, "小红书", "https://www.douyin.com/video/7654321"),
|
||||
(xingtu, "抖音", "http://8.99 分享文案 https://v.douyin.com/abc123/"),
|
||||
],
|
||||
)
|
||||
def test_filled_publish_link_triggers_collection_without_platform_validation(
|
||||
monkeypatch, module, platform, filled_url,
|
||||
):
|
||||
row = {
|
||||
"record_id": "r1",
|
||||
"name": "达人",
|
||||
"creator_id": "creator-id",
|
||||
"title": "已发布标题",
|
||||
"pub": "2026-07-20",
|
||||
"platform": platform,
|
||||
"url": filled_url,
|
||||
"read7": None,
|
||||
}
|
||||
monkeypatch.setattr(module, "list_records_by_table", lambda *_: [row])
|
||||
|
||||
task = module.extract_target_tasks(_style_for_platform(platform))[0]
|
||||
|
||||
assert task["input_error"] is None
|
||||
|
||||
|
||||
def test_extractors_can_collect_missing_title_when_direct_url_has_content_id(monkeypatch):
|
||||
xhs_style = _style_for_platform("小红书")
|
||||
dy_style = _style_for_platform("抖音")
|
||||
xhs_row = {
|
||||
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
|
||||
"title": None, "pub": "2026-07-20", "platform": "小红书",
|
||||
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
|
||||
}
|
||||
dy_row = {
|
||||
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
|
||||
"title": None, "pub": "2026-07-20", "platform": "抖音",
|
||||
"url": "https://www.douyin.com/video/7654321", "read7": None,
|
||||
}
|
||||
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
|
||||
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
|
||||
|
||||
assert pgy.extract_target_tasks(xhs_style)[0]["note_id"] == "abc123"
|
||||
assert xingtu.extract_target_tasks(dy_style)[0]["note_id"] == "7654321"
|
||||
assert xhs_style["_extract_stats"]["skipped_incomplete"] == 0
|
||||
assert dy_style["_extract_stats"]["skipped_incomplete"] == 0
|
||||
|
||||
|
||||
def test_missing_critical_mapping_is_an_incomplete_summary(monkeypatch):
|
||||
style = _style_for_platform("抖音")
|
||||
del style["field_map"]["note_url"]
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"list_records_by_table",
|
||||
lambda *_: (_ for _ in ()).throw(AssertionError("mapping should fail first")),
|
||||
)
|
||||
|
||||
_, summaries = xingtu.collect_tasks_across_styles([style], None)
|
||||
finalized = cc.finalize_summary(summaries[1], dry_run=True)
|
||||
|
||||
assert "note_url" in finalized["error"]
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] >= 1
|
||||
|
||||
|
||||
class _FakeInput:
|
||||
def __init__(self):
|
||||
self.filled = []
|
||||
|
||||
def click(self):
|
||||
return None
|
||||
|
||||
def fill(self, value):
|
||||
self.filled.append(value)
|
||||
|
||||
def press(self, _key):
|
||||
return None
|
||||
|
||||
|
||||
class _FakeLocator:
|
||||
def __init__(self, input_):
|
||||
self.first = input_
|
||||
|
||||
def filter(self, **_kwargs):
|
||||
return self
|
||||
|
||||
|
||||
class _FakeSearchPage:
|
||||
def __init__(self):
|
||||
self.input = _FakeInput()
|
||||
self.has_result_calls = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
def evaluate(self, script, arg=None):
|
||||
if "const rows" in script:
|
||||
self.has_result_calls += 1
|
||||
return self.has_result_calls >= 2
|
||||
if "const inputs" in script:
|
||||
return ""
|
||||
raise AssertionError(script)
|
||||
|
||||
def locator(self, _selector):
|
||||
return _FakeLocator(self.input)
|
||||
|
||||
def wait_for_url(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def test_pgy_search_retry_keeps_creator_id_query():
|
||||
page = _FakeSearchPage()
|
||||
|
||||
pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
|
||||
|
||||
assert page.input.filled == ["xhs-id-123"]
|
||||
|
||||
|
||||
def test_apply_matched_task_reports_write_failure(monkeypatch):
|
||||
task = {
|
||||
"record_id": "r1", "publish_time": object(),
|
||||
"style_context": {
|
||||
"index": 1, "base_token": "base", "table_id": "table", "field_map": {},
|
||||
},
|
||||
}
|
||||
summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []}
|
||||
monkeypatch.setattr(pgy, "pick_read_field", lambda *_: ("read_count_7d", "fid", "7天曝光量"))
|
||||
monkeypatch.setattr(pgy, "write_back", lambda *_: False)
|
||||
|
||||
outcome = pgy.apply_matched_task(task, {"read_count": "123", "title": "标题"}, summary, dry_run=False)
|
||||
|
||||
assert outcome["status"] == "write_failure"
|
||||
assert outcome["write_ok"] is False
|
||||
|
||||
|
||||
def test_default_creator_batch_has_no_forced_hour_pause():
|
||||
assert pgy.DEFAULT_BATCH_SIZE == 0
|
||||
assert xingtu.DEFAULT_BATCH_SIZE == 0
|
||||
|
||||
|
||||
def test_repeatable_style_argument_and_selection_keep_every_requested_style():
|
||||
parser = argparse.ArgumentParser()
|
||||
cc.add_repeatable_style_argument(parser, "styles")
|
||||
args = parser.parse_args(["--style", "1", "--style", "3"])
|
||||
styles = [{"index": 1}, {"index": 2}, {"index": 3}]
|
||||
|
||||
assert args.style == [1, 3]
|
||||
assert cc.select_requested_styles(styles, args.style) == [styles[0], styles[2]]
|
||||
with pytest.raises(ValueError, match="99"):
|
||||
cc.select_requested_styles(styles, [1, 99])
|
||||
|
||||
|
||||
def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
|
||||
sequence = [[], [{"title": "标题", "read_count": "1"}], [{"title": "标题", "read_count": "1"}]]
|
||||
monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
|
||||
|
||||
class Page:
|
||||
waits = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
self.waits += 1
|
||||
|
||||
page = Page()
|
||||
assert pgy.parse_cards_stable(page) == [{"title": "标题", "read_count": "1"}]
|
||||
assert page.waits == 2
|
||||
|
||||
|
||||
def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
|
||||
pages = [
|
||||
[{"title": "无关视频", "play_count": "10"}],
|
||||
[{"title": "目标视频标题", "play_count": "20"}],
|
||||
]
|
||||
calls = {"next": 0}
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "parse_videos_stable", lambda _page: pages.pop(0))
|
||||
|
||||
def next_page(_page):
|
||||
calls["next"] += 1
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(xingtu, "go_next_page", next_page)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
|
||||
|
||||
found, _, _ = xingtu.find_videos_for_tasks(
|
||||
object(), [{"record_id": "r1", "target_title": "目标视频标题"}], max_pages=2,
|
||||
)
|
||||
|
||||
assert found["r1"]["play_count"] == "20"
|
||||
assert calls["next"] == 1
|
||||
|
||||
|
||||
def test_xingtu_normalizes_show_items_api_video():
|
||||
card = xingtu.normalize_xingtu_api_item({
|
||||
"item_id": 7654321,
|
||||
"item_title": "目标视频标题",
|
||||
"play": 168000,
|
||||
"like": 1200,
|
||||
"comment": 34,
|
||||
"share": 56,
|
||||
"url": "https://www.douyin.com/video/7654321",
|
||||
})
|
||||
|
||||
assert card == {
|
||||
"title": "目标视频标题",
|
||||
"play_count": 168000,
|
||||
"like_count": 1200,
|
||||
"comment_count": 34,
|
||||
"share_count": 56,
|
||||
"href": "https://www.douyin.com/video/7654321",
|
||||
"note_id": "7654321",
|
||||
"source": "show_items_api",
|
||||
}
|
||||
|
||||
|
||||
def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
|
||||
class Response:
|
||||
url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
|
||||
|
||||
def json(self):
|
||||
return {
|
||||
"latest_item_info": [
|
||||
{"item_id": "1", "title": "视频一", "play": 10},
|
||||
{"item_id": "2", "item_title": "视频二", "play": 20},
|
||||
],
|
||||
"latest_star_item_info": [
|
||||
{"item_id": "2", "item_title": "视频二", "play": 20},
|
||||
],
|
||||
}
|
||||
|
||||
class Page:
|
||||
def on(self, event, callback):
|
||||
assert event == "response"
|
||||
self.callback = callback
|
||||
|
||||
page = Page()
|
||||
cards = xingtu.setup_show_items_capture(page)
|
||||
page.callback(Response())
|
||||
|
||||
assert [card["note_id"] for card in cards] == ["1", "2"]
|
||||
|
||||
|
||||
def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: (_ for _ in ()).throw(AssertionError("DOM should not be needed")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"is_blocked",
|
||||
lambda _page: (_ for _ in ()).throw(AssertionError("page should not be needed")),
|
||||
)
|
||||
tasks = [{
|
||||
"record_id": "r1",
|
||||
"target_title": "飞书旧标题",
|
||||
"note_url": "https://www.douyin.com/video/7654321",
|
||||
"note_id": "7654321",
|
||||
}]
|
||||
api_cards = [{
|
||||
"title": "平台上的新标题",
|
||||
"play_count": 168000,
|
||||
"note_id": "7654321",
|
||||
"source": "show_items_api",
|
||||
}]
|
||||
|
||||
found, _, page_limit_hit = xingtu.find_videos_for_tasks(
|
||||
object(), tasks, max_pages=5, api_cards=api_cards,
|
||||
)
|
||||
|
||||
assert found["r1"]["play_count"] == 168000
|
||||
assert found["r1"]["match_method"] == "content_id"
|
||||
assert page_limit_hit is False
|
||||
|
||||
|
||||
def test_xingtu_api_fuzzy_candidate_does_not_preempt_later_exact_dom(monkeypatch):
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: [{"title": "commuter backpack review", "note_id": "exact-card"}],
|
||||
)
|
||||
monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
|
||||
task = {"record_id": "r1", "target_title": "commuter backpack review"}
|
||||
api_cards = [{
|
||||
"title": "commuter backpack review today",
|
||||
"note_id": "fuzzy-card",
|
||||
"source": "show_items_api",
|
||||
}]
|
||||
|
||||
found, _, _ = xingtu.find_videos_for_tasks(
|
||||
object(), [task], max_pages=2, api_cards=api_cards,
|
||||
)
|
||||
|
||||
assert found["r1"]["note_id"] == "exact-card"
|
||||
|
||||
|
||||
def test_browser_session_loss_detection_is_specific():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
assert pgy.is_browser_session_lost(closed)
|
||||
assert xingtu.is_browser_session_lost(closed)
|
||||
assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
|
||||
|
||||
|
||||
def test_xingtu_nested_session_loss_is_reraised():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.reraise_if_browser_session_lost(closed)
|
||||
xingtu.reraise_if_browser_session_lost(RuntimeError("ordinary lookup miss"))
|
||||
|
||||
|
||||
def test_xingtu_detail_and_video_search_propagate_closed_session():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
class Detail:
|
||||
def goto(self, *_args, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
def new_page(self):
|
||||
return Detail()
|
||||
|
||||
class SearchPage:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, _script, _arg=None):
|
||||
return {"found": True, "url": "https://www.xingtu.cn/ad/creator/author-homepage/douyin-video/1"}
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.open_creator_detail(SearchPage(), "达人", "creator-id")
|
||||
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class VideoPage:
|
||||
def locator(self, _selector):
|
||||
return Locator()
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.submit_video_search(VideoPage(), "标题")
|
||||
|
||||
|
||||
def test_pgy_fallback_detail_click_propagates_closed_session():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
class Clicker:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def click(self, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
class Page:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, *_args, **_kwargs):
|
||||
return {"ok": False, "reason": "no-row"}
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Clicker()
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
pgy.open_blogger_detail(Page(), "达人", None)
|
||||
|
||||
|
||||
def test_bilibili_triggered_blocked_inputs_are_visible_terminal_rows():
|
||||
summary = bili.finalize_bili_summary({
|
||||
"style": "款式A", "index": 1, "total_b_records": 1,
|
||||
"details": [{
|
||||
"record_id": "r1", "status": "blocked_input",
|
||||
"matched": False, "reason": "publish_time_missing", "ok": False,
|
||||
}],
|
||||
})
|
||||
|
||||
assert summary["blocked_input"] == 1
|
||||
assert summary["unresolved"] == 0
|
||||
assert summary["complete"] is True
|
||||
|
||||
|
||||
def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
|
||||
style = {
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"read_count_7d": {"field_id": "s7"},
|
||||
"read_count_14d": {"field_id": "s14"},
|
||||
"read_count_21d": {"field_id": "s21"},
|
||||
"read_count_28d": {"field_id": "s28"},
|
||||
"month_end": {"field_id": "sm"},
|
||||
},
|
||||
}
|
||||
rows = [
|
||||
{
|
||||
"record_id": "no-link",
|
||||
"platform": "B站",
|
||||
"creator": "未发布达人",
|
||||
"url": "",
|
||||
"pub": "2026-07-20",
|
||||
},
|
||||
{
|
||||
"record_id": "published",
|
||||
"platform": "B站",
|
||||
"creator": "已发布达人",
|
||||
"url": "https://www.bilibili.com/video/BV1234567890",
|
||||
"pub": "2026-07-20",
|
||||
},
|
||||
]
|
||||
monkeypatch.setattr(bili, "list_all_records", lambda *_args: rows)
|
||||
monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: 321)
|
||||
monkeypatch.setattr(bili, "write_record", lambda *_args, **_kwargs: True)
|
||||
|
||||
summary = bili.process_style(
|
||||
style,
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=bili.date(2026, 7, 23),
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
assert summary["source_b_records"] == 2
|
||||
assert summary["skipped_no_url"] == 1
|
||||
assert summary["total_b_records"] == 1
|
||||
assert summary["success"] == 1
|
||||
assert summary["blocked_input"] == 0
|
||||
assert [row["record_id"] for row in summary["details"]] == ["published"]
|
||||
|
||||
|
||||
def test_bilibili_partial_retry_preserves_old_success():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 2,
|
||||
"details": [
|
||||
{"record_id": "ok", "status": "success", "ok": True},
|
||||
{"record_id": "retry", "status": "retryable_failure", "ok": False},
|
||||
],
|
||||
}
|
||||
partial = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 1,
|
||||
"details": [{"record_id": "retry", "status": "success", "ok": True}],
|
||||
}
|
||||
|
||||
merged = bili.merge_bili_summary(existing, partial)
|
||||
|
||||
assert merged["total_b_records"] == 2
|
||||
assert {row["record_id"] for row in merged["details"]} == {"ok", "retry"}
|
||||
assert merged["complete"] is True
|
||||
|
||||
|
||||
def test_bilibili_top_level_error_cannot_finalize_as_complete():
|
||||
finalized = bili.finalize_bili_summary({
|
||||
"style": "款式A",
|
||||
"index": 1,
|
||||
"total_b_records": 0,
|
||||
"details": [],
|
||||
"error": "missing platform/url field_id",
|
||||
"unresolved": 1,
|
||||
"retryable_failures": 1,
|
||||
"complete": False,
|
||||
})
|
||||
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] >= 1
|
||||
|
||||
|
||||
def test_bilibili_missing_slot_mapping_is_system_error_not_blocked_input():
|
||||
summary = bili.process_style(
|
||||
{
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
},
|
||||
},
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=None,
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
finalized = bili.finalize_bili_summary(summary, dry_run=True)
|
||||
assert "publish_time" in finalized["error"]
|
||||
assert "read_count_7d" in finalized["error"]
|
||||
assert finalized["complete"] is False
|
||||
|
||||
|
||||
def test_self_douyin_missing_mapping_and_write_failure_are_not_complete(monkeypatch):
|
||||
missing = self_dy.scrape_one_style(
|
||||
{"name": "款式A", "index": 1, "base_token": "base", "table_id": "table", "field_map": {}},
|
||||
login_timeout=1,
|
||||
headless=True,
|
||||
only_record_ids=None,
|
||||
dry_run=False,
|
||||
)
|
||||
assert cc.finalize_summary(missing)["complete"] is False
|
||||
|
||||
style = _style_for_platform("抖音")
|
||||
row = {
|
||||
"record_id": "r1",
|
||||
"platform": "抖音",
|
||||
"name": "自营达人",
|
||||
"title": "目标作品",
|
||||
"url": "https://www.douyin.com/video/1",
|
||||
"pub": "2026-07-20",
|
||||
}
|
||||
|
||||
class Page:
|
||||
def goto(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
class Session:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def fetch(self, _url, page_action, wait):
|
||||
page_action(Page())
|
||||
|
||||
monkeypatch.setattr(self_dy, "DynamicSession", Session)
|
||||
monkeypatch.setattr(self_dy, "list_records_by_table", lambda *_: [row])
|
||||
monkeypatch.setattr(self_dy, "dismiss_popups", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "restore_cookies", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "maybe_wait_for_login", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "save_state", lambda *_: None)
|
||||
monkeypatch.setattr(
|
||||
self_dy,
|
||||
"scroll_and_collect_posts",
|
||||
lambda *_args, **_kwargs: [{"title": "目标作品", "play_count": 123}],
|
||||
)
|
||||
monkeypatch.setattr(self_dy, "pick_read_field", lambda *_: ("read_count_7d", "read7", "7天曝光量"))
|
||||
monkeypatch.setattr(self_dy, "write_back", lambda *_: False)
|
||||
|
||||
failed = cc.finalize_summary(
|
||||
self_dy.scrape_one_style(
|
||||
style,
|
||||
login_timeout=1,
|
||||
headless=True,
|
||||
only_record_ids=None,
|
||||
dry_run=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert failed["results"][0]["status"] == "write_failure"
|
||||
assert failed["complete"] is False
|
||||
|
||||
|
||||
def test_run_all_rejects_stale_or_incomplete_summary(tmp_path: Path):
|
||||
path = tmp_path / "summary.json"
|
||||
path.write_text("[]", encoding="utf-8")
|
||||
started_at = path.stat().st_mtime + 1
|
||||
|
||||
assert run_all.validate_summary_payload([], {1}, path, started_at)[0] is False
|
||||
|
||||
path.write_text('[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}]', encoding="utf-8")
|
||||
assert run_all.validate_summary_payload(
|
||||
[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}],
|
||||
{1}, path, path.stat().st_mtime - 1,
|
||||
)[0] is False
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.daily_dashboard_analytics import build_dashboard_facts
|
||||
|
||||
|
||||
class DailyDashboardAnalyticsTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tracking = [
|
||||
{"style_name": "大流量", "visitors": 1000, "cart_users": 100, "sales": 20, "refund_orders": 2, "light": "🟡"},
|
||||
{"style_name": "高转化", "visitors": 100, "cart_users": 30, "sales": 20, "refund_orders": 8, "light": "🔴"},
|
||||
{"style_name": "极小样本", "visitors": 10, "cart_users": 5, "sales": 10, "refund_orders": 0, "light": "🔴"},
|
||||
{"style_name": "缺数据", "visitors": None, "cart_users": None, "sales": None, "refund_orders": None, "light": "无判断"},
|
||||
]
|
||||
self.platform = [
|
||||
{"style_name": "大流量", "platform": "天猫", "visitors": 900, "cart_users": 90, "sales": 18, "refund_orders": 2},
|
||||
{"style_name": "高转化", "platform": "天猫", "visitors": 100, "cart_users": 30, "sales": 20, "refund_orders": 8},
|
||||
{"style_name": "极小样本", "platform": "天猫", "visitors": 10, "cart_users": 5, "sales": 10, "refund_orders": 0},
|
||||
]
|
||||
self.personas = [
|
||||
{
|
||||
"style_name": "大流量", "platform": "天猫", "data_date": date(2026, 7, 15),
|
||||
"gender_top": "女性 60%", "age_top": "26-35岁 45%",
|
||||
"city_top": "一线城市 35%", "buying_power_top": "高级白领 40%",
|
||||
}
|
||||
]
|
||||
|
||||
def test_all_styles_receive_same_metrics_and_diagnosis(self):
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas,
|
||||
{"大流量": {"creator_count": 3, "directions": ["通勤"]}},
|
||||
)
|
||||
self.assertEqual(len(facts["styles"]), 4)
|
||||
high = next(row for row in facts["styles"] if row["style_name"] == "高转化")
|
||||
self.assertEqual(high["conversion_rate"], 0.2)
|
||||
self.assertEqual(high["cart_rate"], 0.3)
|
||||
self.assertEqual(high["refund_rate"], 0.4)
|
||||
missing = next(row for row in facts["styles"] if row["style_name"] == "缺数据")
|
||||
self.assertEqual(missing["diagnosis"], "无数据")
|
||||
|
||||
def test_top_conversion_excludes_tiny_visitor_base(self):
|
||||
facts = build_dashboard_facts("2026-07-15", self.tracking, self.platform, self.personas, {})
|
||||
self.assertEqual(facts["rankings"]["conversion"][0]["style_name"], "高转化")
|
||||
self.assertEqual(facts["rankings"]["visitors"][0]["style_name"], "大流量")
|
||||
self.assertEqual(facts["rankings"]["refund_rate"][0]["style_name"], "高转化")
|
||||
|
||||
def test_platform_winners_keep_missing_persona_explicit(self):
|
||||
facts = build_dashboard_facts("2026-07-15", self.tracking, self.platform, self.personas, {})
|
||||
card = facts["platform_personas"][0]
|
||||
self.assertEqual(card["platform"], "天猫")
|
||||
self.assertEqual(card["top_conversion"]["style_name"], "高转化")
|
||||
self.assertEqual(card["top_conversion"]["persona_text"], "无数据")
|
||||
self.assertEqual(card["top_visitors"]["style_name"], "大流量")
|
||||
self.assertIn("推测", card["top_visitors"]["interest_hypothesis"])
|
||||
self.assertIn("提转化", card["top_visitors"]["optimization"])
|
||||
|
||||
def test_daily_new_notes_overview_replaces_view_delta_and_creator_ranking(self):
|
||||
notes = [{"note_id": 7, "style_name": "大流量", "creator": "达人A", "platform": "小红书", "view_count": 500}]
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas, {}, daily_notes=notes
|
||||
)
|
||||
self.assertEqual(facts["daily_new_notes"], notes)
|
||||
self.assertNotIn("creator_connections", facts["rankings"])
|
||||
self.assertNotIn("daily_views", facts["rankings"])
|
||||
|
||||
def test_daily_view_ranking_and_platform_cards_receive_stored_images(self):
|
||||
images = {
|
||||
("高转化", "天猫"): {"local_image_path": "same.jpg", "image_platform": "天猫"},
|
||||
("大流量", "天猫"): {"local_image_path": "visitor.jpg", "image_platform": "天猫"},
|
||||
}
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas, {},
|
||||
style_images=images,
|
||||
)
|
||||
card = facts["platform_personas"][0]
|
||||
self.assertEqual(card["top_conversion"]["local_image_path"], "same.jpg")
|
||||
self.assertEqual(card["top_visitors"]["local_image_path"], "visitor.jpg")
|
||||
|
||||
def test_matrix_groups_by_style_category_without_total_exposure(self):
|
||||
signals = {
|
||||
"高转化": {"persona": True, "note_count": 3, "recent_note_exposure": 43000, "comment_count": 80,
|
||||
"review_count": 10, "review_negative_rate": 0.3, "creative_ctr": 0.08},
|
||||
}
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas, {},
|
||||
style_categories={
|
||||
"大流量": "都市机能", "高转化": "智性通勤",
|
||||
"极小样本": "智性通勤", "缺数据": "都市机能",
|
||||
},
|
||||
style_signals=signals,
|
||||
)
|
||||
self.assertEqual([group["style_category"] for group in facts["style_groups"]], ["都市机能", "智性通勤"])
|
||||
self.assertEqual([row["style_name"] for row in facts["style_groups"][0]["styles"]], ["大流量", "缺数据"])
|
||||
high = next(row for row in facts["styles"] if row["style_name"] == "高转化")
|
||||
self.assertNotIn("total_note_exposure", high)
|
||||
self.assertEqual(high["recent_note_exposure"], 43000)
|
||||
self.assertNotIn("total_note_exposure", facts["definitions"])
|
||||
self.assertIn("评价风险", high["diagnosis"])
|
||||
self.assertIn("画像", high["evidence_summary"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,429 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
from unittest.mock import patch
|
||||
|
||||
import daily_marketing_report as daily
|
||||
|
||||
|
||||
class DailyMarketingReportTests(unittest.TestCase):
|
||||
def test_sales_forecast_uses_seven_day_total_and_recent_three_day_velocity(self):
|
||||
forecast = daily.estimate_sales_forecast([10, 12, 14, 16, 18, 20, 22])
|
||||
|
||||
self.assertEqual(forecast["sales_7d_total"], 112)
|
||||
self.assertEqual(forecast["sales_forecast_7d"], 140)
|
||||
self.assertAlmostEqual(forecast["sales_forecast_change"], 7 / 13, places=4)
|
||||
|
||||
def test_sales_forecast_requires_complete_seven_day_history(self):
|
||||
self.assertEqual(
|
||||
daily.estimate_sales_forecast([10, 12, 14]),
|
||||
{
|
||||
"sales_7d_total": None,
|
||||
"sales_forecast_7d": None,
|
||||
"sales_forecast_change": None,
|
||||
},
|
||||
)
|
||||
|
||||
def test_default_recipients_include_all_four_daily_report_owners(self):
|
||||
self.assertEqual(
|
||||
daily.DEFAULT_RECIPIENT_OPEN_IDS,
|
||||
(
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
"ou_fa8d81a16527ad06352dbecc575285b8",
|
||||
"ou_2eda5eec112109ae6d19a1f6813eadcb",
|
||||
"ou_89bcff110ccbb09a23548dc0fb3d880c",
|
||||
),
|
||||
)
|
||||
|
||||
def test_hermes_transport_error_is_not_accepted_as_report(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "Hermes 分析失败"):
|
||||
daily.require_valid_hermes_report("API call failed after 3 retries: Connection error.")
|
||||
self.assertEqual(daily.require_valid_hermes_report("整体经营分析\n销量保持稳定"), "整体经营分析\n销量保持稳定")
|
||||
|
||||
def test_light_uses_highest_absolute_visitor_or_cart_change(self):
|
||||
self.assertEqual(daily.classify_light(0.05, -0.08), "🟢")
|
||||
self.assertEqual(daily.classify_light(0.12, 0.02), "🟡")
|
||||
self.assertEqual(daily.classify_light(-0.31, 0.01), "🔴")
|
||||
self.assertEqual(daily.classify_light(None, None), "无判断")
|
||||
|
||||
def test_comment_signal_requires_minimum_sample(self):
|
||||
comments = ["光影行星宙斯怎么买"] * 49
|
||||
self.assertIsNone(daily.calculate_comment_signal("宙斯", comments))
|
||||
|
||||
comments.append("好看")
|
||||
signal = daily.calculate_comment_signal("宙斯", comments)
|
||||
self.assertEqual(signal["sample_size"], 50)
|
||||
self.assertEqual(signal["brand_mention_rate"], 0.98)
|
||||
self.assertEqual(signal["purchase_intent_rate"], 0.98)
|
||||
|
||||
def test_prompt_uses_only_the_fresh_user_instruction_and_database_facts(self):
|
||||
prompt = daily.build_daily_prompt(
|
||||
report_date="2026-07-15",
|
||||
today_notes=[],
|
||||
tracking_rows=[],
|
||||
next_day_rows=[],
|
||||
comment_rows=[],
|
||||
)
|
||||
self.assertIn("以下为数据库查询结果", prompt)
|
||||
self.assertIn('"报告日期": "2026-07-15"', prompt)
|
||||
self.assertIn("增长型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("流量机会型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("潜力型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("风险型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertNotIn("A类：爆款增长SKU", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_prompt_uses_fashion_bag_data_analyst_role(self):
|
||||
self.assertIn("专业的时尚包袋品牌电商运营数据分析师", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("销售、电商运营及达人营销数据", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("发现增长机会和风险", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("SKU营销运营日报", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("展示关键数据", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_fresh_prompt_contains_the_complete_requested_analysis_structure(self):
|
||||
prompt = daily.SYSTEM_PROMPT
|
||||
for phrase in (
|
||||
"今日整体经营判断",
|
||||
"SKU经营表现分析",
|
||||
"达人营销效果分析",
|
||||
"产品节奏判断",
|
||||
"蓄水期",
|
||||
"放量期",
|
||||
"稳定期",
|
||||
"衰退期",
|
||||
"增长SKU TOP3",
|
||||
"风险SKU TOP3",
|
||||
"比较昨日、七日基准",
|
||||
"用图表展示",
|
||||
"曝光量",
|
||||
"建议动作",
|
||||
"对销量影响",
|
||||
"所有建议必须具体到SKU",
|
||||
):
|
||||
self.assertIn(phrase, prompt)
|
||||
for retired in ("A类：爆款增长SKU", "B类：潜力增长SKU", "D类：需要优化SKU"):
|
||||
self.assertNotIn(retired, prompt)
|
||||
|
||||
def test_database_payload_keeps_all_available_sources(self):
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], {"reviews": [{"sample_size": 3}]})
|
||||
for phrase in ("所有SKU电商运营数据", "达人营销内容数据", "评论与补充经营数据", "reviews"):
|
||||
self.assertIn(phrase, prompt)
|
||||
|
||||
def test_note_label_ignores_placeholder_creator_name(self):
|
||||
note = {"id": 7, "creator_name": "/", "title": "瑞白通勤包实测"}
|
||||
self.assertEqual(daily.note_label(note), "瑞白通勤包实测")
|
||||
|
||||
def test_cooperation_metrics_hide_unverifiable_engagement(self):
|
||||
invalid = daily._sanitize_cooperation_row({
|
||||
"publish_time": None,
|
||||
"exposure_count": 38000,
|
||||
"engagement_count_num": 21430000,
|
||||
})
|
||||
self.assertIsNone(invalid["engagement_count_num"])
|
||||
self.assertEqual(invalid["publish_status"], "发布时间无数据")
|
||||
self.assertIn("口径异常", invalid["engagement_data_status"])
|
||||
|
||||
valid = daily._sanitize_cooperation_row({
|
||||
"publish_time": "2026-07-14",
|
||||
"exposure_count": 38000,
|
||||
"engagement_count_num": 2143,
|
||||
})
|
||||
self.assertEqual(valid["engagement_count_num"], 2143)
|
||||
self.assertEqual(valid["publish_status"], "已记录发布时间")
|
||||
|
||||
def test_load_report_facts_tracks_all_styles_for_report_date(self):
|
||||
metrics = {
|
||||
"宙斯": {"light": "🟢"},
|
||||
}
|
||||
with patch.object(daily, "_note_rows", return_value=[]), \
|
||||
patch.object(daily, "_all_style_names", return_value=["宙斯", "蓝鹊"]) as all_styles, \
|
||||
patch.object(daily, "_style_metrics", return_value=metrics) as style_metrics:
|
||||
with patch.object(daily, "_load_enrichment", return_value={"platform_metrics": []}), \
|
||||
patch.object(daily, "_platform_metrics", return_value=[]), \
|
||||
patch.object(daily, "_persona_context", return_value=[]), \
|
||||
patch.object(daily, "load_creator_connections", return_value={}), \
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
patch.object(daily, "build_dashboard_facts", return_value={"styles": ["宙斯", "蓝鹊"]}):
|
||||
_, tracking, _, _, enrichment = daily.load_report_facts(date(2026, 7, 15))
|
||||
|
||||
all_styles.assert_called_once_with(date(2026, 7, 15))
|
||||
self.assertEqual([row["style_name"] for row in tracking], ["宙斯", "蓝鹊"])
|
||||
self.assertEqual(tracking[1]["light"], "无判断")
|
||||
self.assertEqual(style_metrics.call_args_list[0].args[1], ["宙斯", "蓝鹊"])
|
||||
self.assertEqual(enrichment["platform_metrics"], [])
|
||||
self.assertEqual(enrichment["candidate_styles"], [])
|
||||
self.assertEqual(enrichment["dashboard"]["styles"], ["宙斯", "蓝鹊"])
|
||||
|
||||
def test_report_date_uses_yesterday_notes_for_daily_new_notes(self):
|
||||
notes = [
|
||||
{"id": 1, "publish_date": "2026-07-15", "style_name": "宙斯", "creator_name": "达人甲", "platform": "小红书", "publish_time": "2026-07-15 10:00", "title": "今天", "view_count": 10, "engagement_rate": None},
|
||||
{"id": 2, "publish_date": "2026-07-14", "style_name": "宙斯", "creator_name": "达人乙", "platform": "抖音", "publish_time": "2026-07-14 11:00", "title": "昨天", "view_count": 20, "engagement_rate": None},
|
||||
]
|
||||
with patch.object(daily, "_note_rows", return_value=notes), \
|
||||
patch.object(daily, "_all_style_names", return_value=["宙斯"]), \
|
||||
patch.object(daily, "_style_metrics", return_value={}), \
|
||||
patch.object(daily, "_load_enrichment", return_value={}), \
|
||||
patch.object(daily, "_platform_metrics", return_value=[]), \
|
||||
patch.object(daily, "_persona_context", return_value=[]), \
|
||||
patch.object(daily, "load_creator_connections", return_value={}), \
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
patch.object(daily, "_comment_rows", return_value=[]), \
|
||||
patch.object(daily, "build_dashboard_facts", return_value={}) as dashboard:
|
||||
daily_notes, _, _, _, _ = daily.load_report_facts(date(2026, 7, 15))
|
||||
|
||||
self.assertEqual([row["note_id"] for row in daily_notes], [2])
|
||||
self.assertEqual([row["note_id"] for row in dashboard.call_args.kwargs["daily_notes"]], [2])
|
||||
|
||||
def test_prompt_requires_all_styles_without_top_ten_truncation(self):
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [])
|
||||
self.assertIn("所有SKU", prompt)
|
||||
self.assertNotIn("只保留异常程度最高的10个", prompt)
|
||||
self.assertIn("SKU经营表现分析", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("加购率", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_prompt_uses_the_new_daily_output_contract(self):
|
||||
self.assertIn("电商运营数据", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("达人营销数据", daily.SYSTEM_PROMPT)
|
||||
self.assertNotIn("TOP20", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("一页A4", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("↑↓展示趋势", daily.SYSTEM_PROMPT)
|
||||
self.assertNotIn("→稳定", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("电商销量表现 + 访客数 + 转化率 + 达人笔记曝光 + 用户反馈", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_prompt_payload_keeps_yesterday_and_seven_day_comparisons(self):
|
||||
rows = [{
|
||||
"style_name": "蓝鹊",
|
||||
"sales": 10,
|
||||
"sales_yesterday": 8,
|
||||
"avg_sales_7d": 7.5,
|
||||
"sales_change_yesterday": 0.25,
|
||||
"sales_change": 0.3333,
|
||||
}]
|
||||
prompt = daily.build_daily_prompt("2026-07-16", [], rows, [], [])
|
||||
self.assertIn('"sales_yesterday": 8', prompt)
|
||||
self.assertIn('"avg_sales_7d": 7.5', prompt)
|
||||
self.assertIn('"sales_change_yesterday": 0.25', prompt)
|
||||
|
||||
def test_style_metrics_calculates_yesterday_and_seven_day_changes(self):
|
||||
class FakeCursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def execute(self, query, params):
|
||||
self.query = query
|
||||
self.params = params
|
||||
|
||||
def fetchall(self):
|
||||
return [(
|
||||
"蓝鹊",
|
||||
120, 24, 10, 2,
|
||||
100, 20, 8, 1,
|
||||
90, 18, 7.5, 1.2,
|
||||
[4, 5, 6, 7, 8, 9, 10],
|
||||
)]
|
||||
|
||||
class FakeConnection:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return FakeCursor()
|
||||
|
||||
with patch.object(daily, "get_conn", return_value=FakeConnection()):
|
||||
row = daily._style_metrics(date(2026, 7, 16), ["蓝鹊"])["蓝鹊"]
|
||||
|
||||
self.assertEqual(row["sales_yesterday"], 8)
|
||||
self.assertEqual(row["sales_change_yesterday"], 0.25)
|
||||
self.assertEqual(row["sales_change"], 0.3333)
|
||||
self.assertEqual(row["visitor_change_yesterday"], 0.2)
|
||||
self.assertEqual(row["conversion_rate"], round(10 / 120, 4))
|
||||
self.assertEqual(row["conversion_rate_yesterday"], round(8 / 100, 4))
|
||||
self.assertEqual(row["sales_7d_total"], 49)
|
||||
self.assertEqual(row["sales_forecast_7d"], 63)
|
||||
|
||||
def test_prompt_does_not_reuse_retired_dashboard_sections(self):
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], {"dashboard": {
|
||||
"rankings": {"sales": []},
|
||||
"platform_personas": [{"platform": "天猫"}],
|
||||
"daily_new_notes": [],
|
||||
"styles": [{"style_name": "蓝鹊", "total_note_exposure": 1000, "evidence_summary": "经营、画像、营销笔记"}],
|
||||
"definitions": {"conversion_rate": "销量/访客"},
|
||||
}})
|
||||
self.assertNotIn("按风格分组的综合经营矩阵", prompt)
|
||||
self.assertNotIn("平台高转化与高访客画像", prompt)
|
||||
self.assertNotIn("全款式经营看板摘要", prompt)
|
||||
|
||||
def test_report_normalization_and_all_style_validation(self):
|
||||
report = daily.normalize_report(
|
||||
"健康度：偏弱。36 个款式中\n蓝鹊同比增长79%\n宙斯环比下降16%",
|
||||
35,
|
||||
)
|
||||
self.assertNotIn("同比", report)
|
||||
self.assertIn("环比下降16%", report)
|
||||
self.assertIn("35个款式中", report)
|
||||
self.assertEqual(daily.normalize_report("销量 0 / 退款 0（无数据）"), "销量 0 / 退款 0（当日均为0）")
|
||||
cleaned = daily.normalize_report("投放尚未发布，带动效应快速衰减；待发布后核对")
|
||||
self.assertNotIn("未发布", cleaned)
|
||||
self.assertNotIn("带动效应", cleaned)
|
||||
self.assertIn("发布时间无数据", cleaned)
|
||||
self.assertNotIn("驱动访客", daily.normalize_report("该内容未驱动访客回升"))
|
||||
daily.validate_all_styles(report, [{"style_name": "蓝鹊"}, {"style_name": "宙斯"}])
|
||||
with self.assertRaisesRegex(RuntimeError, "缺少款式"):
|
||||
daily.validate_all_styles(report, [{"style_name": "白鹭"}])
|
||||
|
||||
def test_report_normalization_removes_database_and_code_style_language(self):
|
||||
raw = """数据已收到，我先用脚本做一遍聚合。
|
||||
# 品牌SKU电商运营日报
|
||||
|
||||
• 最佳渠道：无法判定。今日 `昨日新发布内容` 为空，cooperations 仅1条，publish_time=null、engagement_data_status=无数据。
|
||||
|
||||
5. 数据治理与内容补位：（a）对齐 publish_time；（b）修复互动数据；（c）补齐数据。
|
||||
|
||||
报告字数提示：如需压缩请告知。
|
||||
完整数据索引：如需导出 CSV 请告诉我。
|
||||
"""
|
||||
report = daily.normalize_report(raw)
|
||||
|
||||
self.assertTrue(report.startswith("# 品牌SKU电商运营日报"))
|
||||
for forbidden in ("我先用脚本", "cooperations", "publish_time", "engagement_data_status", "null", "`", "（a）", "报告字数提示", "请告诉我"):
|
||||
self.assertNotIn(forbidden, report)
|
||||
self.assertIn("达人合作记录", report)
|
||||
self.assertIn("发布时间无数据", report)
|
||||
self.assertIn("互动数据无数据", report)
|
||||
self.assertIn("（1）", report)
|
||||
|
||||
def test_system_prompt_forbids_database_fields_and_conversational_tail(self):
|
||||
for phrase in (
|
||||
"禁止输出数据库表名、字段名、JSON键名",
|
||||
"禁止出现反引号、null",
|
||||
"禁止描述生成过程",
|
||||
"禁止在结尾追问",
|
||||
):
|
||||
self.assertIn(phrase, daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_status_groups_are_replaced_with_complete_deterministic_lists(self):
|
||||
report = "二、种草追踪池\n\nB. 全量状态（固定4行）\n🔴（1）：蓝鹊\n\n三、昨日笔记次日追踪"
|
||||
rows = [
|
||||
{"style_name": "蓝鹊", "light": "🔴"},
|
||||
{"style_name": "宙斯", "light": "🟡"},
|
||||
{"style_name": "白鹭", "light": "🟢"},
|
||||
{"style_name": "云卷2", "light": "无判断"},
|
||||
]
|
||||
result = daily.replace_status_groups(report, rows)
|
||||
self.assertIn("B. 全量状态（4款）", result)
|
||||
self.assertIn("🔴（1）：蓝鹊", result)
|
||||
self.assertIn("🟡（1）：宙斯", result)
|
||||
self.assertIn("🟢（1）：白鹭", result)
|
||||
self.assertIn("无判断（1）：云卷2", result)
|
||||
self.assertIn("三、昨日笔记次日追踪", result)
|
||||
|
||||
def test_status_groups_are_inserted_when_model_omits_marker(self):
|
||||
report = "一、核心经营结论\n正常\n\n三、达人投放与电商同期关联\n无数据"
|
||||
rows = [{"style_name": "蓝鹊", "light": "🔴"}, {"style_name": "白鹭", "light": "🟢"}]
|
||||
result = daily.replace_status_groups(report, rows, "C. 补充经营信号\n- 店铺大盘：无数据")
|
||||
self.assertIn("B. 全量状态（2款）", result)
|
||||
self.assertIn("🔴（1）：蓝鹊", result)
|
||||
self.assertIn("🟢（1）：白鹭", result)
|
||||
self.assertLess(result.index("B. 全量状态"), result.index("三、达人投放"))
|
||||
|
||||
def test_prompt_passes_available_enrichment_without_restoring_old_sections(self):
|
||||
enrichment = {
|
||||
"platform_metrics": [{"style_name": "蓝鹊", "platform": "天猫", "sales": 14}],
|
||||
"cooperations": [{"style_name": "蓝鹊", "content_direction": "通勤"}],
|
||||
"personas": [{"style_name": "蓝鹊", "gender_top": "女性用户 60%"}],
|
||||
"creatives": [{"style_name": "蓝鹊", "ctr": 4.2}],
|
||||
"reviews": [{"style_name": "蓝鹊", "sample_size": 6}],
|
||||
"shop_overview": [{"platform": "天猫", "week_end": "2026-07-12"}],
|
||||
}
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], enrichment)
|
||||
for source_key in ("platform_metrics", "cooperations", "personas", "creatives", "reviews", "shop_overview"):
|
||||
self.assertIn(source_key, prompt)
|
||||
self.assertIn("评论与补充经营数据", prompt)
|
||||
self.assertNotIn("补充经营信号", prompt)
|
||||
self.assertNotIn("全量状态", prompt)
|
||||
|
||||
def test_enrichment_section_validation_requires_available_categories(self):
|
||||
enrichment = {"personas": [{}], "creatives": [{}], "reviews": [{}], "shop_overview": [{}]}
|
||||
complete = "条件信号：人物画像；主图；商品评价；店铺大盘"
|
||||
daily.validate_enrichment_sections(complete, enrichment)
|
||||
with self.assertRaisesRegex(RuntimeError, "主图"):
|
||||
daily.validate_enrichment_sections("人物画像 商品评价 店铺大盘", enrichment)
|
||||
|
||||
def test_build_condition_signals_covers_all_available_sources(self):
|
||||
enrichment = {
|
||||
"candidate_styles": ["蓝鹊"],
|
||||
"personas": [{"style_name": "蓝鹊", "platform": "天猫", "data_date": "2026-07-15", "gender_top": "女性 60%"}],
|
||||
"creatives": [{"style_name": "蓝鹊", "platform": "天猫", "data_date": "2026-07-12", "impressions": 1000, "clicks": 50, "ctr": 5.0}],
|
||||
"reviews": [{"style_name": "蓝鹊", "sample_size": 6, "negative_count": 1, "negative_rate": 0.1667, "window_start": "2026-07-09", "window_end": "2026-07-15"}],
|
||||
"shop_overview": [{"platform": "天猫", "week_end": "2026-07-12", "visitors": 10000, "deal_amount": 20000}],
|
||||
}
|
||||
text = daily.build_condition_signals(enrichment)
|
||||
for phrase in ("C. 补充经营信号", "人物画像", "主图", "商品评价", "店铺大盘", "2026-07-12"):
|
||||
self.assertIn(phrase, text)
|
||||
|
||||
def test_persona_top_value_ignores_invalid_percentages(self):
|
||||
payload = {"buying_power": [{"name": "异常", "value": 133}, {"name": "L4", "value": 42}]}
|
||||
self.assertEqual(daily._top_persona_value(payload, "buying_power"), "L4 42.0%")
|
||||
|
||||
def test_enriched_report_rejects_missing_platform_claim_and_invented_new_product(self):
|
||||
daily.validate_enriched_report("平台拆分完整，均为同期合作")
|
||||
with self.assertRaisesRegex(RuntimeError, "平台拆分"):
|
||||
daily.validate_enriched_report("无平台拆分数据")
|
||||
with self.assertRaisesRegex(RuntimeError, "新品"):
|
||||
daily.validate_enriched_report("同期出现多个新品合作")
|
||||
|
||||
def test_platform_persona_section_uses_database_winners_not_model_text(self):
|
||||
report = ("### 三、平台人群与营销机会\n天猫高转化款为错误款式\n\n"
|
||||
"### 四、明日业务动作\n复查数据\n\n"
|
||||
"三、平台人群与营销机会\n重复错误款式")
|
||||
enrichment = {"dashboard": {
|
||||
"daily_views_status": "无数据：未保存每日增量快照",
|
||||
"platform_personas": [{
|
||||
"platform": "天猫",
|
||||
"top_conversion": {
|
||||
"style_name": "布谷", "conversion_rate": 0.0259, "visitors": 617,
|
||||
"persona_text": "无数据", "interest_hypothesis": "兴趣推测：无数据",
|
||||
"optimization": "先补采人群画像",
|
||||
},
|
||||
"top_visitors": {
|
||||
"style_name": "星云2", "conversion_rate": 0.0114, "visitors": 3504,
|
||||
"persona_text": "男性用户 62.7%", "interest_hypothesis": "兴趣推测：数码3C",
|
||||
"optimization": "优化详情页",
|
||||
},
|
||||
}],
|
||||
}}
|
||||
result = daily.replace_platform_persona_section(report, enrichment)
|
||||
self.assertIn("转化最高为布谷（转化率2.6%", result)
|
||||
self.assertIn("访客最高为星云2（访客3,504", result)
|
||||
self.assertIn("兴趣推测：数码3C", result)
|
||||
self.assertNotIn("错误款式", result)
|
||||
self.assertNotIn("重复错误款式", result)
|
||||
self.assertIn("四、明日业务动作", result)
|
||||
self.assertEqual(result.count("三、平台人群与营销机会"), 1)
|
||||
|
||||
def test_platform_persona_section_supports_new_daily_report_boundary(self):
|
||||
report = ("### 三、平台人群与营销机会\n模型内容\n\n"
|
||||
"### 四、今日重点SKU分析\n重点款内容\n")
|
||||
result = daily.replace_platform_persona_section(report, {"dashboard": {}})
|
||||
self.assertIn("三、平台人群与营销机会", result)
|
||||
self.assertIn("四、今日重点SKU分析", result)
|
||||
self.assertNotIn("模型内容", result)
|
||||
self.assertEqual(result.count("三、平台人群与营销机会"), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class DailyMarketingReportScheduleTests(unittest.TestCase):
|
||||
def test_batch_runs_latest_daily_report_and_propagates_exit_code(self):
|
||||
path = Path(__file__).resolve().parents[1] / "data" / "tools" / "daily_marketing_report.bat"
|
||||
content = path.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("daily_marketing_report.py --send", content)
|
||||
self.assertNotIn("--date", content)
|
||||
self.assertIn("daily_marketing_report_%TS%.log", content)
|
||||
self.assertIn("LARK_CLI_NO_PROXY=1", content)
|
||||
self.assertIn("exit /b %RC%", content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,189 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.daily_report_card import (
|
||||
build_analysis_card,
|
||||
build_dashboard_card,
|
||||
build_daily_report_card,
|
||||
prepare_feishu_image,
|
||||
send_card_to_recipients,
|
||||
send_daily_report_cards,
|
||||
upload_dashboard_image,
|
||||
)
|
||||
|
||||
|
||||
class DailyReportCardTests(unittest.TestCase):
|
||||
def test_dashboard_and_analysis_are_separate_cards_with_full_text(self):
|
||||
report = """
|
||||
|
||||
第一部分 整体经营分析
|
||||
总销量下降15%。
|
||||
整体趋势判断：下滑状态。
|
||||
|
||||
第二部分 SKU分析排序
|
||||
1. 蓝鹊：销量第一。
|
||||
2. 星云2：转化领先。
|
||||
3. 宙斯：访客增长。
|
||||
4. 南斯：退款偏高。
|
||||
5. 凌云3：需要优化。
|
||||
6. 波塞冬：继续观察。
|
||||
|
||||
第三部分 关键预警与建议
|
||||
退款订单比上升，48小时内排查。
|
||||
"""
|
||||
rows = [{"style_name": "蓝鹊", "light": "🔴"}]
|
||||
|
||||
dashboard = build_dashboard_card("2026-07-17", rows, "img_v3_test")
|
||||
analysis = build_analysis_card("2026-07-17", report)
|
||||
|
||||
dashboard_text = json.dumps(dashboard, ensure_ascii=False)
|
||||
analysis_text = json.dumps(analysis, ensure_ascii=False)
|
||||
self.assertIn("img_v3_test", dashboard_text)
|
||||
self.assertNotIn("总销量下降15%", dashboard_text)
|
||||
self.assertNotIn("img_v3_test", analysis_text)
|
||||
self.assertNotIn("![全域种草数据看板]", analysis_text)
|
||||
self.assertIn("波塞冬：继续观察", analysis_text)
|
||||
self.assertIn("退款订单比上升", analysis_text)
|
||||
|
||||
def test_build_card_uses_card_2_schema_and_dashboard(self):
|
||||
report = """一、核心经营结论
|
||||
1. 波塞冬edge访客明显下滑。
|
||||
2. 蓝鹊销量同期上行。
|
||||
|
||||
二、产品经营分析
|
||||
略
|
||||
|
||||
四、明日业务动作
|
||||
1. 复查波塞冬edge天猫流量。
|
||||
2. 跟进蓝鹊投放承接。
|
||||
"""
|
||||
rows = [
|
||||
{"style_name": "波塞冬edge", "light": "🔴"},
|
||||
{"style_name": "宙斯", "light": "🟡"},
|
||||
{"style_name": "白鹭", "light": "🟢"},
|
||||
]
|
||||
card = build_daily_report_card("2026-07-15", report, rows, "img_v3_test")
|
||||
|
||||
self.assertEqual(card["schema"], "2.0")
|
||||
self.assertEqual(card["config"]["width_mode"], "fill")
|
||||
self.assertIn("Hermes", card["header"]["subtitle"]["content"])
|
||||
elements = card["body"]["elements"]
|
||||
self.assertEqual([item["tag"] for item in elements], ["column_set", "img", "markdown", "markdown"])
|
||||
self.assertEqual(elements[1]["img_key"], "img_v3_test")
|
||||
metrics_text = json.dumps(elements[0], ensure_ascii=False)
|
||||
self.assertIn("红灯款式", metrics_text)
|
||||
self.assertIn("1", metrics_text)
|
||||
self.assertNotIn("二、产品经营分析", json.dumps(card, ensure_ascii=False))
|
||||
|
||||
def test_prepare_feishu_image_resizes_to_supported_width(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
source = Path(tmp) / "source.png"
|
||||
target = Path(tmp) / "target.png"
|
||||
Image.new("RGB", (1600, 1000), "white").save(source)
|
||||
prepare_feishu_image(source, target)
|
||||
with Image.open(target) as image:
|
||||
self.assertEqual(image.width, 1500)
|
||||
self.assertLessEqual(image.height / image.width, 16 / 9)
|
||||
|
||||
def test_build_card_reads_new_management_report_headings(self):
|
||||
report = """## 1. 今日经营概览
|
||||
销售：↑12%，蓝鹊贡献主要增量。
|
||||
整体判断：增长状态。
|
||||
|
||||
## 2. SKU健康排名 TOP20
|
||||
略
|
||||
|
||||
## 6. 今日运营建议
|
||||
1. 商品动作：复核蓝鹊库存。
|
||||
2. 电商动作：优化宙斯详情页。
|
||||
"""
|
||||
card = build_daily_report_card("2026-07-17", report, [], "img_v3_test")
|
||||
card_text = json.dumps(card, ensure_ascii=False)
|
||||
self.assertIn("蓝鹊贡献主要增量", card_text)
|
||||
self.assertIn("复核蓝鹊库存", card_text)
|
||||
|
||||
def test_build_card_reads_fresh_partial_prompt_report_headings(self):
|
||||
report = """第一部分 整体经营分析
|
||||
总销量下降15%。
|
||||
整体趋势判断：下滑状态。
|
||||
|
||||
第二部分 SKU分析排序
|
||||
略
|
||||
|
||||
第三部分 关键预警与建议
|
||||
退款订单比上升，48小时内排查。
|
||||
追加增长型SKU预算。
|
||||
"""
|
||||
card = build_daily_report_card("2026-07-17", report, [], "img_v3_test")
|
||||
card_text = json.dumps(card, ensure_ascii=False)
|
||||
self.assertIn("总销量下降15%", card_text)
|
||||
self.assertIn("48小时内排查", card_text)
|
||||
|
||||
@patch("data.tools.daily_report_card.subprocess.run")
|
||||
def test_upload_dashboard_uses_named_image_file_field(self, run):
|
||||
run.return_value = type("Result", (), {
|
||||
"returncode": 0, "stdout": json.dumps({"image_key": "img_v3_test"}), "stderr": ""
|
||||
})()
|
||||
key = upload_dashboard_image(Path("charts/dashboard.png"))
|
||||
self.assertEqual(key, "img_v3_test")
|
||||
command = run.call_args.args[0]
|
||||
self.assertEqual(command[command.index("--file") + 1], "image=./dashboard.png")
|
||||
|
||||
@patch("data.tools.daily_report_card.subprocess.run")
|
||||
def test_send_card_to_all_recipients(self, run):
|
||||
run.side_effect = [
|
||||
type("Result", (), {"returncode": 0, "stdout": json.dumps({"message_id": f"om_{i}"}), "stderr": ""})()
|
||||
for i in range(3)
|
||||
]
|
||||
recipients = ["ou_one", "ou_two", "ou_three"]
|
||||
results = send_card_to_recipients({"schema": "2.0"}, recipients, "2026-07-15")
|
||||
|
||||
self.assertEqual([item["message_id"] for item in results], ["om_0", "om_1", "om_2"])
|
||||
self.assertEqual(run.call_count, 3)
|
||||
commands = [call.args[0] for call in run.call_args_list]
|
||||
self.assertEqual([cmd[cmd.index("--user-id") + 1] for cmd in commands], recipients)
|
||||
self.assertTrue(all("--msg-type" in cmd and "interactive" in cmd for cmd in commands))
|
||||
keys = [cmd[cmd.index("--idempotency-key") + 1] for cmd in commands]
|
||||
self.assertTrue(all(key.startswith("daily-2026-07-15-") for key in keys))
|
||||
self.assertTrue(all(len(key) <= 50 for key in keys))
|
||||
|
||||
@patch("data.tools.daily_report_card.send_card_to_recipients")
|
||||
@patch("data.tools.daily_report_card.upload_dashboard_image", return_value="img_v3_test")
|
||||
@patch("data.tools.daily_report_card.prepare_feishu_image")
|
||||
def test_daily_report_sends_dashboard_then_analysis_as_two_messages_per_recipient(
|
||||
self, prepare, upload, send
|
||||
):
|
||||
send.side_effect = [
|
||||
[{"message_id": "om_board_1"}, {"message_id": "om_board_2"}],
|
||||
[{"message_id": "om_text_1"}, {"message_id": "om_text_2"}],
|
||||
]
|
||||
recipients = ["ou_one", "ou_two"]
|
||||
|
||||
results = send_daily_report_cards(
|
||||
"2026-07-17",
|
||||
"第一部分 整体经营分析\n完整分析内容",
|
||||
[{"light": "🟢"}],
|
||||
Path("charts/dashboard.png"),
|
||||
recipients,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[item["message_id"] for item in results],
|
||||
["om_board_1", "om_board_2", "om_text_1", "om_text_2"],
|
||||
)
|
||||
self.assertEqual(send.call_count, 2)
|
||||
first_card = send.call_args_list[0].args[0]
|
||||
second_card = send.call_args_list[1].args[0]
|
||||
self.assertIn("img_v3_test", json.dumps(first_card, ensure_ascii=False))
|
||||
self.assertIn("完整分析内容", json.dumps(second_card, ensure_ascii=False))
|
||||
self.assertEqual(send.call_args_list[0].kwargs["message_kind"], "dashboard")
|
||||
self.assertEqual(send.call_args_list[1].kwargs["message_kind"], "analysis")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,104 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.daily_report_charts import (
|
||||
_split_style_groups,
|
||||
build_overview_cards,
|
||||
classify_style_action,
|
||||
generate_dashboard,
|
||||
select_chart_styles,
|
||||
)
|
||||
|
||||
|
||||
class DailyReportChartTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.rows = [
|
||||
{"style_name": "大盘下滑", "light": "🔴", "visitors": 1000, "avg_visitors_7d": 2000, "cart_users": 80, "avg_cart_users_7d": 100, "visitor_change": -0.5, "cart_change": -0.2, "sales_change": -0.4, "refund_change": 0.1},
|
||||
{"style_name": "正向放量", "light": "🔴", "visitors": 1500, "avg_visitors_7d": 1000, "cart_users": 180, "avg_cart_users_7d": 100, "visitor_change": 0.5, "cart_change": 0.8, "sales_change": 0.3, "refund_change": -0.1},
|
||||
{"style_name": "小基数", "light": "🔴", "visitors": 20, "avg_visitors_7d": 100, "cart_users": 1, "avg_cart_users_7d": 2, "visitor_change": -0.8, "cart_change": -0.5, "sales_change": None, "refund_change": None},
|
||||
{"style_name": "黄灯A", "light": "🟡", "visitors": 800, "avg_visitors_7d": 1000, "cart_users": 70, "avg_cart_users_7d": 80, "visitor_change": -0.2, "cart_change": -0.125, "sales_change": -0.1, "refund_change": 0.2},
|
||||
{"style_name": "绿灯", "light": "🟢", "visitors": 500, "avg_visitors_7d": 510, "cart_users": 40, "avg_cart_users_7d": 41, "visitor_change": -0.02, "cart_change": -0.02, "sales_change": 0.0, "refund_change": 0.0},
|
||||
]
|
||||
|
||||
def test_selection_covers_negative_positive_and_small_base(self):
|
||||
names = [row["style_name"] for row in select_chart_styles(self.rows, limit=4)]
|
||||
self.assertIn("大盘下滑", names)
|
||||
self.assertIn("正向放量", names)
|
||||
self.assertIn("小基数", names)
|
||||
|
||||
def test_generate_dashboard_writes_png(self):
|
||||
enrichment = {"platform_metrics": [
|
||||
{"style_name": "大盘下滑", "platform": "天猫", "visitors": 700},
|
||||
{"style_name": "大盘下滑", "platform": "京东", "visitors": 300},
|
||||
]}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "dashboard.png"
|
||||
generate_dashboard("2026-07-15", self.rows, enrichment, path)
|
||||
self.assertTrue(path.exists())
|
||||
self.assertGreater(path.stat().st_size, 10_000)
|
||||
with Image.open(path) as image:
|
||||
self.assertEqual(image.size, (1600, 2500))
|
||||
|
||||
def test_style_groups_are_split_into_balanced_category_columns(self):
|
||||
groups = [
|
||||
{"style_category": "都市机能", "styles": [{}] * 12},
|
||||
{"style_category": "户外机能", "styles": [{}] * 3},
|
||||
{"style_category": "都市新贵", "styles": [{}] * 3},
|
||||
{"style_category": "智性通勤", "styles": [{}] * 14},
|
||||
{"style_category": "都市新旅", "styles": [{}]},
|
||||
{"style_category": "都市运动", "styles": [{}] * 3},
|
||||
]
|
||||
left, right = _split_style_groups(groups)
|
||||
self.assertEqual([group["style_category"] for group in left], ["都市机能", "户外机能", "都市新贵"])
|
||||
self.assertEqual([group["style_category"] for group in right], ["智性通勤", "都市新旅", "都市运动"])
|
||||
|
||||
groups.append({"style_category": "未分类", "styles": [{}] * 2})
|
||||
left, right = _split_style_groups(groups)
|
||||
self.assertEqual(sum(len(group["styles"]) for group in left), 18)
|
||||
self.assertEqual(sum(len(group["styles"]) for group in right), 20)
|
||||
|
||||
def test_overview_compares_today_yesterday_and_seven_day_average(self):
|
||||
rows = [
|
||||
{
|
||||
"style_name": "蓝鹊", "sales": 12, "sales_yesterday": 10, "avg_sales_7d": 8,
|
||||
"visitors": 120, "visitors_yesterday": 100, "avg_visitors_7d": 90,
|
||||
},
|
||||
{
|
||||
"style_name": "宙斯", "sales": 18, "sales_yesterday": 20, "avg_sales_7d": 15,
|
||||
"visitors": 180, "visitors_yesterday": 200, "avg_visitors_7d": 150,
|
||||
},
|
||||
]
|
||||
cards = build_overview_cards(rows, {"daily_new_notes": [{"view_count": 515}]})
|
||||
sales = next(card for card in cards if card["key"] == "sales")
|
||||
conversion = next(card for card in cards if card["key"] == "conversion")
|
||||
content = next(card for card in cards if card["key"] == "content")
|
||||
|
||||
self.assertEqual((sales["today"], sales["yesterday"], sales["baseline_7d"]), (30, 30, 23))
|
||||
self.assertEqual(conversion["today"], 0.1)
|
||||
self.assertEqual(content["today"], 515)
|
||||
self.assertIsNone(content["baseline_7d"])
|
||||
|
||||
def test_style_action_matches_prompt_growth_opportunity_and_risk_rules(self):
|
||||
growth = classify_style_action({
|
||||
"sales_change_yesterday": 0.2, "visitor_change_yesterday": 0.1,
|
||||
"conversion_change_yesterday": 0.001, "refund_change_yesterday": -0.1,
|
||||
})
|
||||
opportunity = classify_style_action({
|
||||
"sales_change_yesterday": -0.1, "visitor_change_yesterday": 0.2,
|
||||
"conversion_change_yesterday": -0.01, "refund_change_yesterday": 0,
|
||||
})
|
||||
risk = classify_style_action({
|
||||
"sales_change_yesterday": -0.2, "visitor_change_yesterday": -0.1,
|
||||
"conversion_change_yesterday": -0.01, "refund_change_yesterday": 0.4,
|
||||
})
|
||||
|
||||
self.assertEqual(growth, ("增长型", "加达人/备货"))
|
||||
self.assertEqual(opportunity, ("流量机会", "优化详情页"))
|
||||
self.assertEqual(risk, ("风险型", "查退款/控投放"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import friday_relogin_parallel as relogin
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import relogin_transaction
|
||||
|
||||
|
||||
def test_five_expected_platforms_are_configured():
|
||||
assert set(relogin.PLATFORMS) == {
|
||||
"bilibili", "douyin", "pgy", "xingtu", "xiaohongshu"
|
||||
}
|
||||
assert relogin.PLATFORMS["xingtu"].required_cookie == "sessionid"
|
||||
assert relogin.PLATFORMS["douyin"].required_cookie == "sessionid"
|
||||
|
||||
|
||||
def test_default_screenshot_recipients_are_routed_by_platform():
|
||||
assert relogin.DEFAULT_RECIPIENTS == {
|
||||
"pgy": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
"xingtu": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
"douyin": "ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
"bilibili": "ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
"xiaohongshu": "ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
}
|
||||
assert relogin.recipient_for_platform("pgy") == "ou_24cc944d6e43c69c59d6560ad4e2ae6e"
|
||||
assert relogin.recipient_for_platform("douyin") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
|
||||
assert relogin.recipient_for_platform("bilibili") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
|
||||
assert relogin.recipient_for_platform("pgy", "ou_override") == "ou_override"
|
||||
|
||||
|
||||
def test_failed_platforms_are_grouped_by_their_responsible_recipient():
|
||||
grouped = relogin.group_platforms_by_recipient(
|
||||
["pgy", "douyin", "bilibili", "xingtu", "xiaohongshu"]
|
||||
)
|
||||
assert grouped == {
|
||||
"ou_24cc944d6e43c69c59d6560ad4e2ae6e": ["pgy", "xingtu"],
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447": [
|
||||
"douyin", "bilibili", "xiaohongshu"
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_login_windows_are_classified_by_title():
|
||||
assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili"
|
||||
assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy"
|
||||
assert relogin._platform_from_window_title("抖音授权 - Google Chrome") == "xingtu"
|
||||
assert relogin._platform_from_window_title("抖音 - 记录美好生活") == "douyin"
|
||||
assert relogin._platform_from_window_title("小红书 - 你的生活兴趣社区") == "xiaohongshu"
|
||||
assert relogin._platform_from_window_title("Codex") is None
|
||||
|
||||
|
||||
def test_retry_only_failed_platforms_for_at_most_three_rounds():
|
||||
calls = []
|
||||
|
||||
def fake_round(names, attempt):
|
||||
calls.append((tuple(names), attempt))
|
||||
if attempt == 1:
|
||||
return {name: name != "xingtu" for name in names}
|
||||
if attempt == 2:
|
||||
return {name: False for name in names}
|
||||
return {name: True for name in names}
|
||||
|
||||
result = relogin.run_with_retries(
|
||||
["bilibili", "pgy", "xingtu", "xiaohongshu"],
|
||||
max_attempts=3,
|
||||
run_round=fake_round,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(("bilibili", "pgy", "xingtu", "xiaohongshu"), 1),
|
||||
(("xingtu",), 2),
|
||||
(("xingtu",), 3),
|
||||
]
|
||||
assert all(result.values())
|
||||
|
||||
|
||||
def test_stops_after_third_failed_round():
|
||||
calls = []
|
||||
|
||||
def always_fail(names, attempt):
|
||||
calls.append(attempt)
|
||||
return {name: False for name in names}
|
||||
|
||||
result = relogin.run_with_retries(
|
||||
["pgy"], max_attempts=3, run_round=always_fail
|
||||
)
|
||||
|
||||
assert calls == [1, 2, 3]
|
||||
assert result == {"pgy": False}
|
||||
|
||||
|
||||
def test_timeout_terminates_the_complete_process_tree(monkeypatch):
|
||||
calls = []
|
||||
|
||||
class Process:
|
||||
pid = 1234
|
||||
|
||||
def terminate(self):
|
||||
calls.append("fallback-terminate")
|
||||
|
||||
def wait(self, timeout):
|
||||
calls.append(("wait", timeout))
|
||||
|
||||
monkeypatch.setattr(
|
||||
relogin.subprocess,
|
||||
"run",
|
||||
lambda command, **kwargs: calls.append((command, kwargs)),
|
||||
)
|
||||
|
||||
relogin._terminate_process_tree(Process())
|
||||
|
||||
assert calls[0][0] == ["taskkill", "/PID", "1234", "/T", "/F"]
|
||||
assert calls[1] == ("wait", 10)
|
||||
|
||||
|
||||
def test_transaction_rolls_back_files_and_profile(tmp_path):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
state = relogin_transaction.begin((cookie, storage), profile)
|
||||
cookie.write_text("partial", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
relogin_transaction.rollback(state)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
+305
@@ -0,0 +1,305 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.creator_task_grouping import chunk_creator_groups, group_tasks_by_creator
|
||||
|
||||
import pgy_xhs_scraper_v2 as pgy
|
||||
import xingtu_scraper_v2 as xingtu
|
||||
|
||||
|
||||
def test_group_tasks_uses_creator_id_and_merges_missing_id_by_unique_name():
|
||||
tasks = [
|
||||
{"creator_name": "同一达人", "creator_id": " 123 ", "record_id": "a"},
|
||||
{"creator_name": "同一达人 ", "creator_id": None, "record_id": "b"},
|
||||
{"creator_name": "另一个达人", "creator_id": None, "record_id": "c"},
|
||||
]
|
||||
|
||||
groups = group_tasks_by_creator(tasks)
|
||||
|
||||
assert len(groups) == 2
|
||||
assert groups[0]["creator_id"] == "123"
|
||||
assert [task["record_id"] for task in groups[0]["tasks"]] == ["a", "b"]
|
||||
|
||||
|
||||
def test_group_tasks_does_not_guess_when_same_name_has_conflicting_ids():
|
||||
tasks = [
|
||||
{"creator_name": "重名达人", "creator_id": "1", "record_id": "a"},
|
||||
{"creator_name": "重名达人", "creator_id": "2", "record_id": "b"},
|
||||
{"creator_name": "重名达人", "creator_id": None, "record_id": "c"},
|
||||
]
|
||||
|
||||
groups = group_tasks_by_creator(tasks)
|
||||
|
||||
assert len(groups) == 3
|
||||
|
||||
|
||||
def test_zero_batch_size_means_one_unlimited_batch():
|
||||
groups = [{"creator_name": str(i), "tasks": []} for i in range(3)]
|
||||
assert chunk_creator_groups(groups, 0) == [groups]
|
||||
|
||||
|
||||
def _assert_cross_style_collection(module, monkeypatch):
|
||||
styles = [
|
||||
{"index": 1, "name": "款式A", "base_token": "base-a", "table_id": "table-a", "field_map": {"a": 1}},
|
||||
{"index": 2, "name": "款式B", "base_token": "base-b", "table_id": "table-b", "field_map": {"b": 2}},
|
||||
]
|
||||
|
||||
def fake_extract(style, only_record_ids):
|
||||
return [{
|
||||
"record_id": f"record-{style['index']}",
|
||||
"creator_name": "同一达人",
|
||||
"creator_id": "creator-1",
|
||||
"target_title": f"标题{style['index']}",
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(module, "extract_target_tasks", fake_extract)
|
||||
tasks, summaries = module.collect_tasks_across_styles(styles, None)
|
||||
|
||||
assert len(tasks) == 2
|
||||
assert len(group_tasks_by_creator(tasks)) == 1
|
||||
assert tasks[0]["style_context"]["table_id"] == "table-a"
|
||||
assert tasks[1]["style_context"]["table_id"] == "table-b"
|
||||
assert summaries[1]["total"] == 1
|
||||
assert summaries[2]["total"] == 1
|
||||
|
||||
|
||||
def test_pgy_collects_tasks_across_styles(monkeypatch):
|
||||
_assert_cross_style_collection(pgy, monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_collects_tasks_across_styles(monkeypatch):
|
||||
_assert_cross_style_collection(xingtu, monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_record_scope_is_applied_per_style(monkeypatch):
|
||||
styles = [
|
||||
{"index": 1, "name": "款式A", "base_token": "a", "table_id": "a", "field_map": {}},
|
||||
{"index": 2, "name": "款式B", "base_token": "b", "table_id": "b", "field_map": {}},
|
||||
]
|
||||
observed = {}
|
||||
|
||||
def fake_extract(style, only_record_ids):
|
||||
observed[style["index"]] = only_record_ids
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(xingtu, "extract_target_tasks", fake_extract)
|
||||
|
||||
xingtu.collect_tasks_across_styles(styles, {1: {"same-id"}, 2: {"other-id"}})
|
||||
|
||||
assert observed == {1: {"same-id"}, 2: {"other-id"}}
|
||||
|
||||
|
||||
def test_xingtu_load_retry_scope_keeps_style_and_record_pair(tmp_path):
|
||||
summary_path = tmp_path / "summary.json"
|
||||
summary_path.write_text(json.dumps([
|
||||
{"index": 1, "results": [
|
||||
{"record_id": "same-id", "reason": "creator_search_circuit_open"},
|
||||
]},
|
||||
{"index": 2, "results": [
|
||||
{"record_id": "same-id", "reason": "title_unmatched"},
|
||||
{"record_id": "other-id", "reason": "creator_search_circuit_open"},
|
||||
]},
|
||||
]), encoding="utf-8")
|
||||
|
||||
scope = xingtu.load_retry_scope(summary_path, "creator_search_circuit_open")
|
||||
|
||||
assert scope == {1: {"same-id"}, 2: {"other-id"}}
|
||||
|
||||
|
||||
def _assert_matched_task_uses_own_writeback_context(module, metric_key, monkeypatch):
|
||||
calls = []
|
||||
task = {
|
||||
"record_id": "record-b",
|
||||
"publish_time": object(),
|
||||
"style_context": {
|
||||
"index": 2,
|
||||
"base_token": "base-b",
|
||||
"table_id": "table-b",
|
||||
"field_map": {"slot": "style-b-slot"},
|
||||
},
|
||||
}
|
||||
summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []}
|
||||
monkeypatch.setattr(module, "pick_read_field", lambda fmap, publish_time: ("read_count_7d", "field-b", "7天曝光量"))
|
||||
monkeypatch.setattr(module, "write_back", lambda *args: calls.append(args) or True)
|
||||
|
||||
module.apply_matched_task(task, {metric_key: "1.2万", "title": "命中标题"}, summary, dry_run=False)
|
||||
|
||||
assert calls == [("base-b", "table-b", "record-b", "field-b", 12000)]
|
||||
assert summary["matched"] == 1
|
||||
assert summary["filled"] == 1
|
||||
|
||||
|
||||
def test_pgy_match_writes_to_task_style_context(monkeypatch):
|
||||
_assert_matched_task_uses_own_writeback_context(pgy, "read_count", monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_match_writes_to_task_style_context(monkeypatch):
|
||||
_assert_matched_task_uses_own_writeback_context(xingtu, "play_count", monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_search_and_open_creator_submits_only_one_search(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fail_search(_page, creator_name, creator_id):
|
||||
calls.append((creator_name, creator_id))
|
||||
raise TimeoutError("no result")
|
||||
|
||||
monkeypatch.setattr(xingtu, "search_creator", fail_search)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"open_creator_detail",
|
||||
lambda *_args, **_kwargs: pytest.fail("detail must not open after search failure"),
|
||||
)
|
||||
|
||||
with pytest.raises(TimeoutError, match="no result"):
|
||||
xingtu.search_and_open_creator_once(object(), "同一达人", "creator-1")
|
||||
|
||||
assert calls == [("同一达人", "creator-1")]
|
||||
|
||||
|
||||
def test_xingtu_search_budget_blocks_duplicate_without_failure_circuit():
|
||||
budget = xingtu.CreatorSearchBudget(max_consecutive_failures=3)
|
||||
|
||||
assert budget.begin(("id", "1")) is True
|
||||
assert budget.begin(("id", "1")) is False
|
||||
budget.failed("first")
|
||||
assert budget.begin(("id", "2")) is True
|
||||
budget.failed("second")
|
||||
assert budget.begin(("id", "3")) is True
|
||||
assert budget.failed("third") is False
|
||||
assert budget.begin(("id", "4")) is True
|
||||
assert budget.calls == 4
|
||||
assert budget.circuit_reason is None
|
||||
|
||||
budget.exhaust("search_quota_exhausted")
|
||||
assert budget.begin(("id", "5")) is False
|
||||
|
||||
|
||||
def test_xingtu_wait_for_results_detects_search_quota_exhausted():
|
||||
class Page:
|
||||
def evaluate(self, *_args):
|
||||
return "quota"
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pytest.fail("quota detection must return immediately")
|
||||
|
||||
with pytest.raises(xingtu.SearchQuotaExhausted):
|
||||
xingtu.wait_for_visible_creator_result(Page(), "达人", timeout_ms=1000)
|
||||
|
||||
|
||||
def test_xingtu_missing_creation_tab_is_not_treated_as_global_block():
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return "no_creation_tab"
|
||||
|
||||
assert xingtu.is_blocked(Page()) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reason", ["verify_text", "verify_iframe", "verify_div"])
|
||||
def test_xingtu_explicit_verification_still_opens_global_block(reason):
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return reason
|
||||
|
||||
assert xingtu.is_blocked(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_missing_creation_tab_falls_back_to_current_creator_page():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def click(self, **_kwargs):
|
||||
raise TimeoutError("missing tab")
|
||||
|
||||
class Page:
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
def evaluate(self, _script):
|
||||
return False
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
assert xingtu.open_creation_ability(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_current_creator_page_cards_are_used_as_fallback(monkeypatch):
|
||||
class Page:
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: [{
|
||||
"title": "MacBook党必看！提升幸福感的开工好物",
|
||||
"play_count": "1.2万",
|
||||
"note_id": "",
|
||||
}],
|
||||
)
|
||||
monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda _page, _title: False)
|
||||
|
||||
found, _candidates, page_limit_hit = xingtu.find_videos_for_tasks(
|
||||
Page(),
|
||||
[{
|
||||
"record_id": "record-1",
|
||||
"target_title": "MacBook党必看！提升幸福感的开工好物",
|
||||
}],
|
||||
)
|
||||
|
||||
assert found["record-1"]["play_count"] == "1.2万"
|
||||
assert found["record-1"]["match_method"] == "title"
|
||||
assert page_limit_hit is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
"BrowserContext.new_page: Protocol error (Target.createTarget): Failed to open a new tab",
|
||||
"Protocol error (Target.createTarget)",
|
||||
],
|
||||
)
|
||||
def test_xingtu_new_tab_creation_failure_rebuilds_browser_session(message):
|
||||
assert xingtu.is_browser_session_lost(RuntimeError(message)) is True
|
||||
|
||||
|
||||
def test_daily_retry_does_not_search_xingtu_again_same_day():
|
||||
batch = Path("data/tools/daily_run.bat").read_text(encoding="utf-8")
|
||||
|
||||
retry_lines = [line for line in batch.splitlines() if "retry_failed.py" in line and "call" in line]
|
||||
|
||||
assert len(retry_lines) == 1
|
||||
assert "--platform bili,pgy" in retry_lines[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch_path",
|
||||
[
|
||||
Path("data/tools/daily_run.bat"),
|
||||
Path("data/tools/daily_run_with_backfill.bat"),
|
||||
],
|
||||
)
|
||||
def test_scheduled_collection_syncs_pg_before_and_after_retry(batch_path):
|
||||
batch = batch_path.read_text(encoding="utf-8")
|
||||
calls = [
|
||||
line.strip().lower()
|
||||
for line in batch.splitlines()
|
||||
if line.strip().lower().startswith("call %python%")
|
||||
]
|
||||
|
||||
run_index = next(i for i, line in enumerate(calls) if "run_all.py" in line)
|
||||
retry_index = next(i for i, line in enumerate(calls) if "retry_failed.py" in line)
|
||||
sync_indices = [
|
||||
i for i, line in enumerate(calls)
|
||||
if "sync_metrics_to_cmt_notes.py" in line
|
||||
]
|
||||
|
||||
assert len(sync_indices) == 2
|
||||
assert run_index < sync_indices[0] < retry_index < sync_indices[1]
|
||||
@@ -0,0 +1,240 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import relogin_douyin, relogin_pgy, relogin_xingtu
|
||||
import xingtu_scraper_v2
|
||||
import pgy_xhs_scraper_v2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "state.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
assert not cookie.exists()
|
||||
assert not profile.exists()
|
||||
|
||||
cookie.write_text("partial-new-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
module.restore_previous_state(cookie_backup, profile_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
cookie.write_text("new-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
module.finish_successful_relogin(profile_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "new-cookie"
|
||||
assert cookie_backup.exists()
|
||||
assert not profile_backup.exists()
|
||||
|
||||
|
||||
def test_xingtu_scan_login_opens_customer_sso_and_clicks_douyin():
|
||||
class FakeLocator:
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
self.clicked = False
|
||||
self.evaluated = False
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def is_visible(self):
|
||||
return True
|
||||
|
||||
def click(self, timeout):
|
||||
self.clicked = True
|
||||
|
||||
def evaluate(self, script):
|
||||
assert script == "e => e.click()"
|
||||
self.evaluated = True
|
||||
|
||||
class FakePage:
|
||||
url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market"
|
||||
|
||||
def __init__(self):
|
||||
self.goto_url = None
|
||||
self.selectors = []
|
||||
self.douyin = FakeLocator()
|
||||
|
||||
def goto(self, url, **kwargs):
|
||||
self.goto_url = url
|
||||
self.url = url
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
def locator(self, selector):
|
||||
self.selectors.append(selector)
|
||||
return self.douyin
|
||||
|
||||
def get_by_text(self, _pattern):
|
||||
return FakeLocator()
|
||||
|
||||
page = FakePage()
|
||||
xingtu_scraper_v2.open_scan_login(page)
|
||||
|
||||
assert page.goto_url == xingtu_scraper_v2.CUSTOMER_LOGIN_URL
|
||||
assert 'img[src*="aweme.png"]' in page.selectors
|
||||
assert page.douyin.clicked
|
||||
|
||||
|
||||
def test_xingtu_index_page_is_recognized_as_logged_in():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def wait_for(self, timeout):
|
||||
assert timeout == 1200
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/ad/creator/index"
|
||||
|
||||
def get_by_text(self, _text, exact=False):
|
||||
return Locator()
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_cookie_validation_requires_real_session_cookie(tmp_path):
|
||||
cookie = tmp_path / "xingtu.json"
|
||||
cookie.write_text('[{"name":"other","value":"1"}]', encoding="utf-8")
|
||||
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is False
|
||||
|
||||
cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8")
|
||||
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is True
|
||||
|
||||
|
||||
def test_xingtu_login_only_does_not_report_success_when_page_action_was_swallowed(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
cookie = tmp_path / "xingtu.json"
|
||||
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
|
||||
|
||||
class Session:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def fetch(self, *_args, **_kwargs):
|
||||
# Scrapling can log a page_action error and still return a response.
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session())
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None)
|
||||
|
||||
assert xingtu_scraper_v2.login_only(1) == 1
|
||||
|
||||
|
||||
def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch):
|
||||
cookie = tmp_path / "xingtu_cookies.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = relogin_xingtu.reset_state()
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
assert relogin_xingtu.recover_interrupted_state() is True
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
assert cookie_backup is not None
|
||||
assert profile_backup is not None
|
||||
|
||||
|
||||
def test_pgy_scan_login_clicks_qr_switch_image():
|
||||
class FakeLocator:
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
self.clicked = False
|
||||
self.evaluated = False
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def is_visible(self):
|
||||
return True
|
||||
|
||||
def click(self, timeout):
|
||||
self.clicked = True
|
||||
|
||||
def evaluate(self, script):
|
||||
assert script == "e => e.click()"
|
||||
self.evaluated = True
|
||||
|
||||
class FakePage:
|
||||
def __init__(self):
|
||||
self.selectors = []
|
||||
self.qr_switch = FakeLocator()
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
def get_by_text(self, _pattern):
|
||||
return FakeLocator()
|
||||
|
||||
def locator(self, selector):
|
||||
self.selectors.append(selector)
|
||||
return self.qr_switch
|
||||
|
||||
page = FakePage()
|
||||
pgy_xhs_scraper_v2.open_scan_login(page)
|
||||
|
||||
assert 'img[src*="qr_code"]' in page.selectors
|
||||
assert page.qr_switch.evaluated
|
||||
|
||||
|
||||
def test_failed_douyin_relogin_restores_cookie_storage_and_profile(tmp_path, monkeypatch):
|
||||
cookie = tmp_path / "douyin_cookies.json"
|
||||
storage = tmp_path / "douyin_storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(relogin_douyin, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(relogin_douyin, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(relogin_douyin, "PROFILE_DIR", profile)
|
||||
monkeypatch.setattr(relogin_douyin.subprocess, "call", lambda *_args, **_kwargs: 1)
|
||||
monkeypatch.setattr(relogin_douyin.sys, "argv", ["relogin_douyin.py"])
|
||||
|
||||
assert relogin_douyin.main() == 1
|
||||
assert cookie.read_text(encoding="utf-8") == '[{"name":"sessionid","value":"old"}]'
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
@@ -0,0 +1,72 @@
|
||||
import io
|
||||
import os
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import db
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools import sync_metrics_to_cmt_notes as sync_metrics
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class RemoteDatabaseConfigTests(unittest.TestCase):
|
||||
def test_config_requires_every_connection_value_without_local_fallback(self):
|
||||
complete = {
|
||||
"PG_HOST": "db.example.com",
|
||||
"PG_PORT": "5432",
|
||||
"PG_DB": "analytics",
|
||||
"PG_USER": "reporter",
|
||||
"PG_PASSWORD": "secret",
|
||||
}
|
||||
self.assertEqual(
|
||||
db.database_config_from_env(complete),
|
||||
{
|
||||
"host": "db.example.com",
|
||||
"port": 5432,
|
||||
"dbname": "analytics",
|
||||
"user": "reporter",
|
||||
"password": "secret",
|
||||
},
|
||||
)
|
||||
for missing in complete:
|
||||
values = {key: value for key, value in complete.items() if key != missing}
|
||||
with self.subTest(missing=missing), self.assertRaisesRegex(RuntimeError, missing):
|
||||
db.database_config_from_env(values)
|
||||
|
||||
def test_runtime_config_points_to_remote_data_hub(self):
|
||||
config = db.get_db_config()
|
||||
self.assertEqual(config["host"], "8.148.185.119")
|
||||
self.assertEqual(config["dbname"], "data_hub")
|
||||
self.assertEqual(config["user"], "data_hub")
|
||||
self.assertNotIn(config["host"], {"localhost", "127.0.0.1"})
|
||||
|
||||
def test_database_scripts_do_not_define_local_connection_defaults(self):
|
||||
paths = (
|
||||
ROOT / "data" / "tools" / "db.py",
|
||||
ROOT / "data" / "tools" / "sync_metrics_to_cmt_notes.py",
|
||||
ROOT / "data" / "tools" / "sync_cooperations.py",
|
||||
ROOT / "data" / "tools" / "generate_creator_report.py",
|
||||
)
|
||||
for path in paths:
|
||||
source = path.read_text(encoding="utf-8-sig")
|
||||
with self.subTest(path=path.name):
|
||||
self.assertNotIn('"localhost"', source)
|
||||
self.assertNotIn("'localhost'", source)
|
||||
self.assertNotIn('"gyxx_super_data"', source)
|
||||
self.assertNotIn("'gyxx_super_data'", source)
|
||||
|
||||
def test_sync_console_replaces_characters_unsupported_by_gbk(self):
|
||||
raw = io.BytesIO()
|
||||
stream = io.TextIOWrapper(raw, encoding="gbk", errors="strict")
|
||||
|
||||
sync_metrics.configure_console_output(stream, None)
|
||||
stream.write("𝑻")
|
||||
stream.flush()
|
||||
|
||||
self.assertTrue(raw.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+385
@@ -0,0 +1,385 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import gyxx_flow.modules.content_marketing.runtime.data.tools.retry_failed as retry
|
||||
import run_all
|
||||
|
||||
|
||||
def test_requested_records_must_be_successful_even_when_process_exits_zero():
|
||||
data = {
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success", "matched": True},
|
||||
{"record_id": "b", "status": "retryable_failure", "matched": False},
|
||||
]
|
||||
}
|
||||
|
||||
ok, failed = retry.requested_records_succeeded(data, ["a", "b"])
|
||||
|
||||
assert ok is False
|
||||
assert failed == ["b"]
|
||||
|
||||
|
||||
def test_success_status_cannot_hide_write_failure_and_blocked_input_is_terminal():
|
||||
contradictory = {
|
||||
"results": [{
|
||||
"record_id": "a", "status": "success", "matched": True,
|
||||
"write_ok": False,
|
||||
}]
|
||||
}
|
||||
assert retry.requested_records_succeeded(contradictory, ["a"]) == (False, ["a"])
|
||||
|
||||
blocked = {
|
||||
"total": 1,
|
||||
"details": [{
|
||||
"record_id": "b", "status": "blocked_input", "ok": False,
|
||||
"reason": "url_missing",
|
||||
}],
|
||||
}
|
||||
assert retry.extract_failed_records(blocked, "bili") == []
|
||||
|
||||
|
||||
def test_canonical_filename_requires_current_style_name():
|
||||
style = {"index": 2, "name": "极星pro"}
|
||||
|
||||
assert retry.canonical_result_filename(style, "pgy", False) == "02-极星pro_v2.json"
|
||||
assert retry.canonical_result_filename(style, "xt", False) == "02-极星pro_xingtu_v2.json"
|
||||
assert retry.canonical_result_filename(style, "bili", True) == "02-极星pro_self_bilibili_v2.json"
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict | list) -> None:
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def test_find_failed_uses_exact_current_style_file_and_ignores_legacy_name(tmp_path):
|
||||
style = {"index": 2, "name": "极星pro"}
|
||||
canonical = tmp_path / "02-极星pro_v2.json"
|
||||
legacy = tmp_path / "02-盖亚微单_v2.json"
|
||||
payload = {
|
||||
"index": 2,
|
||||
"total": 1,
|
||||
"matched": 1,
|
||||
"filled": 1,
|
||||
"results": [{"record_id": "ok", "status": "success", "matched": True}],
|
||||
}
|
||||
_write_json(canonical, payload)
|
||||
_write_json(legacy, {"index": 2, "total": 1, "matched": 0, "results": []})
|
||||
old = time.time() - 48 * 3600
|
||||
os.utime(legacy, (old, old))
|
||||
|
||||
failed = retry.find_failed(
|
||||
"pgy", stale_hours=4, include_half=True,
|
||||
styles=[style], v2_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert failed == {}
|
||||
|
||||
|
||||
def test_find_failed_treats_missing_result_rows_as_incomplete(tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
path = tmp_path / retry.canonical_result_filename(style, "xt", False)
|
||||
_write_json(path, {"index": 1, "total": 2, "matched": 0, "filled": 0, "results": []})
|
||||
|
||||
failed = retry.find_failed(
|
||||
"xt", stale_hours=4, include_half=True,
|
||||
styles=[style], v2_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert failed[1]["reason"] == "ALL_FAILED"
|
||||
assert failed[1]["failed_rids"] == []
|
||||
|
||||
|
||||
def test_run_one_style_rechecks_payload_and_retries_exit_zero(monkeypatch, tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False)
|
||||
calls = []
|
||||
|
||||
def fake_call(_cmd, cwd):
|
||||
calls.append(cwd)
|
||||
_write_json(result_path, {
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"matched": 0,
|
||||
"filled": 0,
|
||||
"results": [{
|
||||
"record_id": "bad",
|
||||
"status": "retryable_failure",
|
||||
"matched": False,
|
||||
}],
|
||||
})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "call", fake_call)
|
||||
monkeypatch.setattr(retry.time, "sleep", lambda _seconds: None)
|
||||
|
||||
ok = retry.run_one_style(
|
||||
"pgy", 1, max_attempts=2, record_ids=["bad"],
|
||||
style=style, v2_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert ok is False
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
def test_run_one_style_accepts_exit_zero_only_after_requested_record_succeeds(monkeypatch, tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False)
|
||||
|
||||
def fake_call(_cmd, cwd):
|
||||
_write_json(result_path, {
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"matched": 1,
|
||||
"filled": 1,
|
||||
"results": [{
|
||||
"record_id": "ok",
|
||||
"status": "success",
|
||||
"matched": True,
|
||||
"write_ok": True,
|
||||
}],
|
||||
})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "call", fake_call)
|
||||
|
||||
assert retry.run_one_style(
|
||||
"pgy", 1, max_attempts=1, record_ids=["ok"],
|
||||
style=style, v2_dir=tmp_path,
|
||||
) is True
|
||||
|
||||
|
||||
def test_self_operated_xingtu_retry_uses_self_douyin_scraper(monkeypatch, tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
result_path = tmp_path / retry.canonical_result_filename(style, "xt", True)
|
||||
observed = {}
|
||||
|
||||
def fake_call(cmd, cwd):
|
||||
observed["cmd"] = cmd
|
||||
_write_json(result_path, {
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "ok", "status": "success", "matched": True}],
|
||||
})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "call", fake_call)
|
||||
|
||||
assert retry.run_one_style(
|
||||
"xt", 1, max_attempts=1, self_operated=True,
|
||||
record_ids=["ok"], style=style, v2_dir=tmp_path,
|
||||
) is True
|
||||
assert observed["cmd"][1].endswith("self_douyin_scraper.py")
|
||||
assert "--self-operated" not in observed["cmd"]
|
||||
|
||||
|
||||
def test_full_summary_declared_failure_cannot_be_treated_as_complete():
|
||||
complete, reason = retry._summary_completed({
|
||||
"total": 1,
|
||||
"results": [{
|
||||
"record_id": "ok",
|
||||
"status": "success",
|
||||
"matched": True,
|
||||
"write_ok": True,
|
||||
}],
|
||||
"write_failures": 1,
|
||||
})
|
||||
|
||||
assert complete is False
|
||||
assert reason == "summary declares unresolved failures"
|
||||
|
||||
|
||||
def test_retry_main_returns_nonzero_when_verified_retry_still_fails(monkeypatch):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
monkeypatch.setattr(retry, "load_current_styles", lambda self_operated=False: [style])
|
||||
monkeypatch.setattr(
|
||||
retry, "find_failed",
|
||||
lambda *_args, **_kwargs: {1: {"reason": "HALF", "failed_rids": ["bad"]}},
|
||||
)
|
||||
monkeypatch.setattr(retry, "run_one_style", lambda *_args, **_kwargs: False)
|
||||
monkeypatch.setattr(retry.sys, "argv", ["retry_failed.py", "--platform", "pgy"])
|
||||
|
||||
assert retry.main() == 1
|
||||
|
||||
|
||||
def test_validate_summary_payload_rejects_stale_missing_duplicate_and_unresolved(tmp_path):
|
||||
path = tmp_path / "summary.json"
|
||||
valid = [{
|
||||
"index": 1,
|
||||
"run_started_at": time.time() + 10,
|
||||
"total": 2,
|
||||
"matched": 2,
|
||||
"filled": 2,
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success", "matched": True},
|
||||
{"record_id": "b", "status": "success", "matched": True},
|
||||
],
|
||||
}]
|
||||
_write_json(path, valid)
|
||||
|
||||
assert run_all.validate_summary_payload(
|
||||
valid, {1}, path, path.stat().st_mtime - 1,
|
||||
)[0] is True
|
||||
assert run_all.validate_summary_payload(
|
||||
valid, {1}, path, path.stat().st_mtime + 1,
|
||||
)[0] is False
|
||||
assert run_all.validate_summary_payload(valid, {1, 2}, path, 0)[0] is False
|
||||
|
||||
duplicate = [{
|
||||
"index": 1, "total": 2,
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success"},
|
||||
{"record_id": "a", "status": "success"},
|
||||
],
|
||||
}]
|
||||
assert run_all.validate_summary_payload(duplicate, {1}, path, 0)[0] is False
|
||||
|
||||
unresolved = [{
|
||||
"index": 1, "total": 1,
|
||||
"results": [{
|
||||
"record_id": "a", "status": "retryable_failure", "matched": False,
|
||||
}],
|
||||
}]
|
||||
assert run_all.validate_summary_payload(unresolved, {1}, path, 0)[0] is False
|
||||
|
||||
|
||||
def test_validate_selected_style_allows_unrelated_rows_in_merged_global_summary(tmp_path):
|
||||
path = tmp_path / "summary.json"
|
||||
payload = [
|
||||
{
|
||||
"index": 1,
|
||||
"run_started_at": time.time() + 10,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "ok", "status": "success"}],
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"total": 1,
|
||||
"results": [{
|
||||
"record_id": "old-failure",
|
||||
"status": "retryable_failure",
|
||||
}],
|
||||
},
|
||||
]
|
||||
_write_json(path, payload)
|
||||
|
||||
valid, reason = run_all.validate_summary_payload(
|
||||
payload, {1}, path, path.stat().st_mtime - 1,
|
||||
)
|
||||
|
||||
assert valid is True, reason
|
||||
|
||||
|
||||
def test_validate_summary_rejects_expected_style_left_over_from_old_run(tmp_path):
|
||||
path = tmp_path / "summary.json"
|
||||
started_at = 100.0
|
||||
payload = [
|
||||
{
|
||||
"index": 1,
|
||||
"run_started_at": 101.0,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "new", "status": "success"}],
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"run_started_at": 90.0,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "old", "status": "success"}],
|
||||
},
|
||||
]
|
||||
_write_json(path, payload)
|
||||
|
||||
valid, reason = run_all.validate_summary_payload(
|
||||
payload, {1, 2}, path, started_at,
|
||||
)
|
||||
|
||||
assert valid is False
|
||||
assert "style 2" in reason
|
||||
assert "current run" in reason
|
||||
|
||||
|
||||
def test_aggregate_counts_all_unresolved_rows_as_errors():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": [{
|
||||
"index": 1,
|
||||
"total": 3,
|
||||
"matched": 1,
|
||||
"filled": 1,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success"},
|
||||
{"record_id": "miss", "matched": False},
|
||||
{"record_id": "detail", "reason": "no_detail_page"},
|
||||
],
|
||||
}],
|
||||
})
|
||||
|
||||
assert aggregate["platforms"]["pgy"]["unresolved"] == 2
|
||||
assert aggregate["platforms"]["pgy"]["errors"] == 2
|
||||
|
||||
|
||||
def test_aggregate_sums_declared_failure_categories_when_unresolved_is_absent():
|
||||
aggregate = run_all.aggregate({
|
||||
"xt": [{
|
||||
"index": 1,
|
||||
"total": 2,
|
||||
"matched": 2,
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success"},
|
||||
{"record_id": "b", "status": "success"},
|
||||
],
|
||||
"retryable_failures": 1,
|
||||
"write_failures": 1,
|
||||
}],
|
||||
})
|
||||
|
||||
assert aggregate["platforms"]["xt"]["unresolved"] == 2
|
||||
|
||||
|
||||
def test_aggregate_keeps_partial_counts_when_validation_marks_payload_invalid():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": {
|
||||
"error": "invalid summary: unresolved",
|
||||
"payload": [{
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"matched": 0,
|
||||
"results": [{
|
||||
"record_id": "bad", "status": "retryable_failure",
|
||||
}],
|
||||
}],
|
||||
},
|
||||
})
|
||||
|
||||
platform = aggregate["platforms"]["pgy"]
|
||||
assert platform["loaded"] is True
|
||||
assert platform["unresolved"] == 1
|
||||
assert platform["validation_error"] == "invalid summary: unresolved"
|
||||
|
||||
|
||||
def test_run_round_marks_exit_zero_platform_failed_when_summary_is_invalid(monkeypatch, tmp_path):
|
||||
args = SimpleNamespace(dry_run=True, style=[1])
|
||||
stdout = tmp_path / "stdout.log"
|
||||
stderr = tmp_path / "stderr.log"
|
||||
observed = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
run_all, "start_process",
|
||||
lambda *_args, **_kwargs: (object(), stdout, stderr),
|
||||
)
|
||||
monkeypatch.setattr(run_all, "wait_all", lambda _procs: {"pgy": 0})
|
||||
|
||||
def fake_load(key, self_operated=False, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return {"error": "invalid summary: stale"}
|
||||
|
||||
monkeypatch.setattr(run_all, "load_summary", fake_load)
|
||||
monkeypatch.setattr(run_all, "print_report", lambda *_args: None)
|
||||
monkeypatch.setattr(run_all, "_atomic_write_json", lambda *_args: None)
|
||||
|
||||
result = run_all.run_round(args, ["pgy"])
|
||||
|
||||
assert result["exit_codes"] == {"pgy": 0}
|
||||
assert result["_non_zero_platforms"] == ["pgy"]
|
||||
assert observed["expected_indices"] == {1}
|
||||
assert isinstance(observed["started_at"], float)
|
||||
@@ -0,0 +1,55 @@
|
||||
import unittest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.sync_style_categories import extract_style_mapping, normalize_style_name
|
||||
|
||||
|
||||
class SyncStyleCategoriesTests(unittest.TestCase):
|
||||
def test_normalizes_feishu_lifecycle_names_to_database_names(self):
|
||||
cases = {
|
||||
"盖亚微单Pro生命进程（新）": "盖亚微单",
|
||||
"星迹&星迹2生命进程（新）": "星迹2",
|
||||
"宙斯双肩包生命进程（新）": "宙斯",
|
||||
"极星Pro生命进程（新）": "极星pro",
|
||||
"瑞白双肩包生命进程（新）": "瑞白",
|
||||
"逐星gt生命进程（新）": "逐星GT",
|
||||
"拾影相机包生命进程 （新）": "拾影斜挎相机包",
|
||||
"极星托特生命进程（新）": "极星托特",
|
||||
}
|
||||
for source, expected in cases.items():
|
||||
with self.subTest(source=source):
|
||||
self.assertEqual(normalize_style_name(source), expected)
|
||||
|
||||
def test_extracts_mapping_and_ignores_blank_rows(self):
|
||||
payload = {
|
||||
"data": {
|
||||
"data": [
|
||||
[
|
||||
"[盖亚斜挎生命进程（新）](https://example.feishu.cn/base/TokenA)",
|
||||
["都市机能"],
|
||||
],
|
||||
[None, ["都市运动"]],
|
||||
["[极星托特生命进程（新）](https://example.feishu.cn/base/TokenB)", ["智性通勤"]],
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
extract_style_mapping(payload),
|
||||
{"盖亚斜挎": "都市机能", "极星托特": "智性通勤"},
|
||||
)
|
||||
|
||||
def test_rejects_conflicting_categories_for_same_style(self):
|
||||
payload = {
|
||||
"data": {
|
||||
"data": [
|
||||
["[布谷生命进程（新）](https://example/a)", ["都市运动"]],
|
||||
["[布谷生命进程（新）](https://example/b)", ["都市机能"]],
|
||||
]
|
||||
}
|
||||
}
|
||||
with self.assertRaisesRegex(ValueError, "布谷"):
|
||||
extract_style_mapping(payload)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,70 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime import weekly_summary_all as weekly
|
||||
|
||||
|
||||
class WeeklyTmallPersonaTests(unittest.TestCase):
|
||||
def test_missing_persona_forbids_invented_percentages(self):
|
||||
text = weekly.format_tmall_persona(None)
|
||||
self.assertIn("不得虚构具体比例", text)
|
||||
|
||||
def test_format_tmall_persona_includes_date_and_dimensions(self):
|
||||
persona = {
|
||||
"collect_date": "2026-07-15",
|
||||
"item_id": "900176117674",
|
||||
"payload": {
|
||||
"gender": [
|
||||
{"name": "男性用户", "value": 66.97},
|
||||
{"name": "女性用户", "value": 28.26},
|
||||
],
|
||||
"age": [{"name": "25-29岁", "value": 26.97}],
|
||||
"city_tier": [{"name": "准一线城市", "value": 31.14}],
|
||||
"buying_power": [{"name": "购买力L5", "value": 30.65}],
|
||||
},
|
||||
}
|
||||
|
||||
text = weekly.format_tmall_persona(persona)
|
||||
|
||||
self.assertIn("2026-07-15", text)
|
||||
self.assertIn("男性用户 66.97%", text)
|
||||
self.assertIn("25-29岁 26.97%", text)
|
||||
self.assertIn("准一线城市 31.14%", text)
|
||||
self.assertIn("购买力L5 30.65%", text)
|
||||
|
||||
def test_generate_summary_injects_latest_tmall_persona_into_single_note_prompt(self):
|
||||
persona = {
|
||||
"collect_date": "2026-07-15",
|
||||
"item_id": "900176117674",
|
||||
"payload": {
|
||||
"gender": [{"name": "男性用户", "value": 66.97}],
|
||||
"age": [{"name": "25-29岁", "value": 26.97}],
|
||||
"city_tier": [],
|
||||
"buying_power": [],
|
||||
},
|
||||
}
|
||||
notes = [{
|
||||
"title": "测试笔记",
|
||||
"platform": "douyin",
|
||||
"creator_name": "达人甲",
|
||||
"view_count": 100,
|
||||
"like_count": 10,
|
||||
"comment_count": 2,
|
||||
"favorite_count": 3,
|
||||
"share_count": 1,
|
||||
"report_path": None,
|
||||
}]
|
||||
|
||||
with patch.object(weekly, "get_latest_tmall_persona", return_value=persona) as get_persona, \
|
||||
patch.object(weekly, "call_hermes_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
|
||||
weekly.generate_summary("星云2", notes)
|
||||
|
||||
get_persona.assert_called_once_with("星云2")
|
||||
single_prompt = call.call_args_list[0].args[1]
|
||||
self.assertIn("天猫该款最新人物画像", single_prompt)
|
||||
self.assertIn("男性用户 66.97%", single_prompt)
|
||||
self.assertIn("内容触达人群", single_prompt)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,766 @@
|
||||
"""通用周笔记汇总脚本:对所有上周有笔记的款,生成周汇总飞书文档并写入对应生命进程表。
|
||||
|
||||
流程:
|
||||
1. 读索引表(TtoCb1NuQaDy3NsZWTpc0GIvnph / tblKCjplVAFrRwMC)拿"每周笔记分析"字段(每款生命进程表地址)
|
||||
2. PG 查上周(2026-06-29~07-05)新增笔记,按款分组
|
||||
3. 对每个款:
|
||||
a. 跑单品分析(调用 analyze_note.py --note-id N --output ...)
|
||||
b. LLM 综合评级(用 weekly_summary 的 prompt)
|
||||
c. 创建飞书文档(markdown)
|
||||
d. 写入该款生命进程表的"笔记分析汇总"字段(时间=上周范围)
|
||||
|
||||
用法:
|
||||
python weekly_summary_all.py # 跑所有上周有笔记的款
|
||||
python weekly_summary_all.py --style 宙斯 # 只跑某个款
|
||||
python weekly_summary_all.py --dry-run # 不写飞书,只生成报告
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.analyze_comments import call_hermes_analyzer
|
||||
|
||||
LARK = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd")
|
||||
if not os.path.exists(LARK):
|
||||
LARK = "lark-cli.cmd"
|
||||
|
||||
# 索引表
|
||||
INDEX_BT = "TtoCb1NuQaDy3NsZWTpc0GIvnph"
|
||||
INDEX_TID = "tblKCjplVAFrRwMC"
|
||||
INDEX_STYLE_FIELD = "fldr8T4ynL"
|
||||
INDEX_WEEKLY_FIELD = "fldgfS6I4a"
|
||||
|
||||
# 周时间范围:动态算上周一到上周日
|
||||
# 支持 --week-since/--week-until 参数覆盖(调试用,格式 YYYY-MM-DD)
|
||||
def _compute_last_week():
|
||||
"""返回 (week_start, week_end, time_label) 都是字符串"""
|
||||
today = datetime.now().date()
|
||||
# 上周一 = 今天 - (今天weekday + 7) 天;周一 weekday=0
|
||||
last_monday = today - timedelta(days=today.weekday() + 7)
|
||||
last_sunday = last_monday + timedelta(days=6)
|
||||
return last_monday, last_sunday
|
||||
|
||||
_DEFAULT_START, _DEFAULT_END = _compute_last_week()
|
||||
WEEK_START = os.environ.get("WEEK_SINCE", _DEFAULT_START.strftime("%Y-%m-%d"))
|
||||
WEEK_END = os.environ.get("WEEK_UNTIL", _DEFAULT_END.strftime("%Y-%m-%d"))
|
||||
TIME_LABEL = f"{WEEK_START}~{WEEK_END[5:]}"
|
||||
|
||||
DATA_DIR = PATHS.raw_root
|
||||
REPORTS_DIR = PATHS.exports_root / "summary"
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# dry_run 全局开关(在 main 设置,save_to_db 读取)
|
||||
dry_run_global = False
|
||||
|
||||
URL_RE = re.compile(r"https?://[^\s)>\]]+")
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True)
|
||||
|
||||
|
||||
def call_lark_json(args):
|
||||
cmd = [LARK] + args
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace")
|
||||
text = proc.stdout if proc.stdout else (proc.stderr or "")
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
return {"ok": False, "error": f"no json: {text[:300]}"}
|
||||
try:
|
||||
return json.loads(text[start:], strict=False)
|
||||
except json.JSONDecodeError as e:
|
||||
return {"ok": False, "error": f"json decode: {e}"}
|
||||
|
||||
|
||||
def parse_url(raw):
|
||||
if not raw:
|
||||
return None
|
||||
if isinstance(raw, list):
|
||||
raw = " ".join(str(x) for x in raw if x)
|
||||
s = str(raw).strip()
|
||||
m = URL_RE.search(s)
|
||||
return m.group(0) if m else None
|
||||
|
||||
|
||||
def parse_bt_tid(url):
|
||||
if not url:
|
||||
return None, None
|
||||
bt = re.search(r'/base/([A-Za-z0-9]+)', url)
|
||||
tid = re.search(r'[?&]table=([A-Za-z0-9]+)', url)
|
||||
return (bt.group(1) if bt else None,
|
||||
tid.group(1) if tid else None)
|
||||
|
||||
|
||||
def load_index_styles():
|
||||
"""读索引表,返回 {款式名: (base_token, table_id, url)}"""
|
||||
cmd = ["base", "+record-list",
|
||||
"--base-token", INDEX_BT, "--table-id", INDEX_TID,
|
||||
"--as", "user", "--format", "json", "--limit", "200",
|
||||
"--field-id", INDEX_STYLE_FIELD, "--field-id", INDEX_WEEKLY_FIELD]
|
||||
resp = call_lark_json(cmd)
|
||||
if not resp.get("ok"):
|
||||
log(f"读索引表失败: {resp.get('error')}")
|
||||
return {}
|
||||
|
||||
styles = {}
|
||||
for row in resp["data"]["data"]:
|
||||
name = row[0] if row else ""
|
||||
weekly = row[1] if len(row) > 1 else ""
|
||||
if isinstance(name, list):
|
||||
name = " ".join(str(x) for x in name if x)
|
||||
name = str(name).strip()
|
||||
if not name or not weekly:
|
||||
continue
|
||||
url = parse_url(weekly)
|
||||
bt, tid = parse_bt_tid(url)
|
||||
if bt and tid:
|
||||
styles[name] = (bt, tid, url)
|
||||
log(f"索引表读到 {len(styles)} 个款有'每周笔记分析'字段")
|
||||
return styles
|
||||
|
||||
|
||||
def find_target_fields(bt, tid):
|
||||
"""动态读生命进程表的字段,返回 (时间字段_id, 笔记分析汇总字段_id)"""
|
||||
cmd = ["base", "+field-list",
|
||||
"--base-token", bt, "--table-id", tid, "--format", "json"]
|
||||
resp = call_lark_json(cmd)
|
||||
if not resp.get("ok"):
|
||||
return None, None
|
||||
time_fid = summary_fid = None
|
||||
for f in resp["data"]["fields"]:
|
||||
if f.get("name") == "时间":
|
||||
time_fid = f.get("id")
|
||||
elif f.get("name") == "笔记分析汇总":
|
||||
summary_fid = f.get("id")
|
||||
return time_fid, summary_fid
|
||||
|
||||
|
||||
def get_last_week_notes(style_name):
|
||||
"""从 PG 拿上周该款的新增笔记(按 publish_time 过滤),返回 [{id, platform, title, ...}]"""
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
with get_conn() as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT n.id, n.platform, n.title, n.url, n.view_count,
|
||||
n.comment_count, n.like_count, n.favorite_count, n.share_count,
|
||||
c.name AS creator_name
|
||||
FROM cmt_notes n
|
||||
JOIN cmt_styles s ON n.style_id = s.id
|
||||
LEFT JOIN cmt_creators c ON n.creator_id = c.id
|
||||
WHERE s.name = %s AND n.self_operated = FALSE
|
||||
AND n.publish_time IS NOT NULL
|
||||
AND n.publish_time >= %s AND n.publish_time < %s
|
||||
ORDER BY n.id
|
||||
""", (style_name, WEEK_START, WEEK_END + " 23:59:59"))
|
||||
cols = [d[0] for d in cur.description]
|
||||
return [dict(zip(cols, r)) for r in cur.fetchall()]
|
||||
|
||||
|
||||
def get_latest_tmall_persona(style_name):
|
||||
"""读取不晚于本周结束日的最新天猫画像，失败或无数据时返回 None。"""
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
try:
|
||||
with get_conn() as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT collect_date, item_id, payload
|
||||
FROM daily_persona_metrics
|
||||
WHERE platform = 'tm' AND style_name = %s AND collect_date <= %s
|
||||
ORDER BY collect_date DESC, collected_at DESC
|
||||
LIMIT 1
|
||||
""", (style_name, WEEK_END))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"collect_date": row[0],
|
||||
"item_id": row[1],
|
||||
"payload": row[2] or {},
|
||||
}
|
||||
except Exception as exc:
|
||||
log(f" [WARN] 读取 {style_name} 天猫人物画像失败: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def _format_persona_dimension(rows):
|
||||
"""把画像维度格式化为适合提示词引用的百分比分布。"""
|
||||
parts = []
|
||||
for row in rows or []:
|
||||
if not isinstance(row, dict) or not row.get("name"):
|
||||
continue
|
||||
value = row.get("value")
|
||||
try:
|
||||
value_text = f"{float(value):.2f}".rstrip("0").rstrip(".")
|
||||
except (TypeError, ValueError):
|
||||
value_text = str(value or "未知")
|
||||
parts.append(f"{row['name']} {value_text}%")
|
||||
return "；".join(parts) if parts else "暂无"
|
||||
|
||||
|
||||
def format_tmall_persona(persona):
|
||||
"""将天猫画像记录转成结构化提示词文本。"""
|
||||
if not persona:
|
||||
return (
|
||||
"未找到该款式的天猫人物画像。受众部分只能结合内容和评论做谨慎的定性判断，"
|
||||
"不得虚构具体比例。"
|
||||
)
|
||||
payload = persona.get("payload") or {}
|
||||
return "\n".join([
|
||||
f"- 数据日期：{persona.get('collect_date') or '未知'}",
|
||||
f"- 天猫商品ID：{persona.get('item_id') or '未知'}",
|
||||
f"- 性别：{_format_persona_dimension(payload.get('gender'))}",
|
||||
f"- 年龄：{_format_persona_dimension(payload.get('age'))}",
|
||||
f"- 城市等级：{_format_persona_dimension(payload.get('city_tier'))}",
|
||||
f"- 消费能力：{_format_persona_dimension(payload.get('buying_power'))}",
|
||||
])
|
||||
|
||||
|
||||
def run_single_note_analysis(note_id):
|
||||
"""跑单品分析,输出到 data/summary/note_<id>.txt。已存在则跳过。"""
|
||||
out_path = REPORTS_DIR / f"note_{note_id}.txt"
|
||||
if out_path.exists() and out_path.stat().st_size > 100:
|
||||
log(f" 复用已有报告: {out_path.name}")
|
||||
return out_path
|
||||
|
||||
log(f" 跑单品分析 note_id={note_id} ...")
|
||||
cmd = [sys.executable, "-X", "utf8",
|
||||
str(Path(__file__).parent / "data" / "tools" / "analyze_note.py"),
|
||||
"--note-id", str(note_id), "--no-send", "--brand", "光影行星",
|
||||
"--output", str(out_path)]
|
||||
env = {**os.environ, "PYTHONIOENCODING": "utf-8"}
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace", timeout=600)
|
||||
if proc.returncode != 0:
|
||||
log(f" [WARN] note_id={note_id} 分析失败: {proc.stderr[:200]}")
|
||||
return None
|
||||
if out_path.exists() and out_path.stat().st_size > 100:
|
||||
return out_path
|
||||
log(f" [WARN] note_id={note_id} 未生成报告")
|
||||
return None
|
||||
|
||||
|
||||
def read_report(path):
|
||||
if not path or not path.exists():
|
||||
return ""
|
||||
for enc in ("utf-8-sig", "utf-8"):
|
||||
try:
|
||||
return path.read_text(encoding=enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
# LLM 综合 prompt - 单篇笔记四维度分析(逐篇调用,保证深度不打折)
|
||||
SINGLE_NOTE_PROMPT = """你是光影行星品牌「{style}」款式的内容种草分析师,聚焦前端表现。
|
||||
请对以下笔记从"内容质量 -> 受众匹配 -> 互动效果 -> 评论区信号"四个维度进行深度分析。
|
||||
|
||||
【笔记信息】
|
||||
{note_info}
|
||||
|
||||
【单品分析报告全文】
|
||||
{report}
|
||||
|
||||
【天猫该款最新人物画像（受众推算基准）】
|
||||
{tmall_persona}
|
||||
|
||||
说明：天猫画像代表该款实际购买/浏览人群，笔记标题、达人和评论反映内容触达人群。
|
||||
请比较两类人群是否吻合，不要把天猫购买人群直接等同于内容触达人群。
|
||||
|
||||
【输出要求】
|
||||
严格按以下结构输出(Markdown 格式),每项都必须填充具体数据和判断:
|
||||
|
||||
### 笔记{idx}：{title}
|
||||
|
||||
#### 基础信息
|
||||
- 平台/达人：{platform} / {creator}
|
||||
- 曝光：{views} | 互动总量：{engagement} | 互动率：{rate}%
|
||||
|
||||
#### 1. 标题与内容拆解
|
||||
(详细分析,不只列结论)
|
||||
- 标题钩子类型(痛点/价值观/利益点/悬念):判断类型 + 解释为什么 + 评估有效性
|
||||
- 内容结构:前3秒抓人能力 / 中间痛点铺陈 / 产品品牌植入方式 / 结尾引导,逐项给具体判断
|
||||
- 品牌/SKU记忆度:用户看完能否复述品牌或产品名,结合评论区信号佐证
|
||||
|
||||
#### 2. 数据表现评级
|
||||
(算多个比值,不只一个)
|
||||
- 互动率对比平台均值(抖音3-5% / 小红书3-8% / B站3-5%),判断优劣
|
||||
- 互动结构:算赞藏比/评赞比/分享点赞比/收藏评论比,每个比值都解读含义,识别认知留存/争议共鸣/实用收藏型
|
||||
|
||||
#### 3. 受众画像推算
|
||||
- 先引用天猫画像的真实性别、年龄、城市等级和消费力分布，并注明数据日期
|
||||
- 再结合标题词、达人人设和评论区用词，判断本篇内容触达人群与天猫购买人群是吻合还是偏离
|
||||
- 核心受众标签:年龄/消费场景/兴趣圈层/消费力,每项给判断和依据
|
||||
- 天猫画像没有覆盖的维度才允许推断，并明确标注为推断
|
||||
|
||||
#### 4. 评论区关键信号
|
||||
(每项都给具体词/数字,不要泛泛)
|
||||
- 高频实义词TOP5(排除语气词):列出 5 个具体词
|
||||
- 购买意向信号("求链接""已下单"等)占比估算:给百分比 + 条数
|
||||
- 品牌提及率(主动提到品牌/产品名的比例):给百分比
|
||||
- 用户顾虑点TOP3(尺寸/价格/质量等):每条顾虑解释为什么
|
||||
- 有无竞品提及:有则列出,无则说明
|
||||
|
||||
#### 5. 总结评级与优化建议
|
||||
- 最终评级(S/A/B/C):给评级 + 判定依据(哪几条指标支撑这个评级)
|
||||
- 是否值得继续投放或放大:给结论 + 理由
|
||||
- 针对内容本身的修改建议(非电商承接):至少 4-5 条具体可执行建议,涵盖标题/前3秒/卖点顺序/品牌植入/结尾引导
|
||||
|
||||
【评级标准】
|
||||
- S:互动率显著高于平台均值,品牌记忆度高,购买意向信号强,受众精准
|
||||
- A:互动率达平台均值以上,品牌有记忆,有购买意向信号
|
||||
- B:互动率接近均值,品牌记忆模糊,购买意向弱
|
||||
- C:互动率低于均值,品牌无记忆,无购买信号,受众错位
|
||||
|
||||
【约束】
|
||||
- 本篇分析必须详尽(800-1200 字),数据驱动,直接给结论
|
||||
- 有天猫画像时，男女比例必须原样引用画像数据；无画像时不得虚构具体比例
|
||||
- 高频词TOP5必须给具体词,不要写"无有效数据"
|
||||
- 评级严格按标准,不放宽
|
||||
- **只输出本篇笔记的分析,不要输出周度总结,不要提及"文件已保存"等元信息。响应正文就是本篇分析本身。**"""
|
||||
|
||||
# LLM 综合 prompt - 周度总结(输入所有单篇分析摘要 + 数据)
|
||||
WEEKLY_SUMMARY_PROMPT = """你是光影行星品牌「{style}」款式的内容种草分析师。
|
||||
请基于以下 {n} 篇笔记的单篇分析结果,输出周度总结。
|
||||
|
||||
【本周单篇分析汇总】
|
||||
{all_analyses}
|
||||
|
||||
【本周投放概览数据】
|
||||
- 投放笔记数：{n} 篇
|
||||
- 涉及平台：{platforms}
|
||||
- 总曝光：{total_views} / 总互动：{total_engagement}
|
||||
|
||||
【输出要求】
|
||||
严格按以下结构输出(Markdown 格式):
|
||||
|
||||
## 三、周度总结与建议
|
||||
|
||||
### 1. 整体评级分布
|
||||
- 各评级(S/A/B/C)笔记数及占比
|
||||
- 本周整体内容种草效率判断
|
||||
|
||||
### 2. 值得继续投放/放大的笔记
|
||||
[列出 S/A 级笔记,给出可复用经验,每篇 2-3 句]
|
||||
|
||||
### 3. 需要规避的问题
|
||||
[列出 B/C 级笔记的问题,每篇 2-3 句,给规避建议]
|
||||
|
||||
### 4. 下周内容优化清单
|
||||
- 标题钩子方向:[基于本周表现给具体建议]
|
||||
- 内容结构:[前3秒/中间铺陈/品牌植入/结尾引导]
|
||||
- 达人选择:[优先/规避类型]
|
||||
- 评论区运营:[具体话术]
|
||||
|
||||
【约束】
|
||||
- 周度总结 800-1500 字
|
||||
- 每条建议须具体可执行,避免"加强""优化"等空泛词汇
|
||||
- **只输出周度总结部分(从"## 三"开始),不要重复输出单篇分析,不要提及"文件已保存"等元信息。响应正文就是周度总结本身。**"""
|
||||
|
||||
|
||||
def generate_summary(style, notes):
|
||||
"""调 LLM 综合评级,返回汇总文本(两步:逐篇分析 + 周度总结)"""
|
||||
system = "你是光影行星品牌的内容种草分析师,擅长从内容质量、受众匹配、互动效果、评论区信号四个维度分析笔记表现,输出可落地的优化建议。"
|
||||
|
||||
tmall_persona = get_latest_tmall_persona(style)
|
||||
tmall_persona_text = format_tmall_persona(tmall_persona)
|
||||
if tmall_persona:
|
||||
log(f" 天猫人物画像: {tmall_persona.get('collect_date')} / item_id={tmall_persona.get('item_id')}")
|
||||
else:
|
||||
log(f" [WARN] {style} 暂无天猫人物画像")
|
||||
|
||||
# ---- 第一步:逐篇生成单篇四维度分析 ----
|
||||
single_analyses = []
|
||||
for i, n in enumerate(notes, 1):
|
||||
report = read_report(n.get("report_path"))
|
||||
views = n.get('view_count') or '无'
|
||||
likes = n.get('like_count') or 0
|
||||
cmts = n.get('comment_count') or 0
|
||||
favs = n.get('favorite_count') or 0
|
||||
shares = n.get('share_count') or 0
|
||||
engagement = (likes or 0) + (favs or 0) + (cmts or 0) + (shares or 0)
|
||||
if isinstance(views, (int, float)) and views > 0:
|
||||
rate = round(engagement * 100 / views, 2)
|
||||
else:
|
||||
rate = "无法计算(曝光未采集)"
|
||||
|
||||
note_info = (
|
||||
f"标题: {n['title']}\n"
|
||||
f"平台: {n['platform']} / 达人: {n.get('creator_name', '?')}\n"
|
||||
f"曝光: {views} / 点赞: {likes} / 评论: {cmts} / 收藏: {favs} / 分享: {shares}\n"
|
||||
f"互动总量: {engagement} / 互动率: {rate}%"
|
||||
)
|
||||
prompt = SINGLE_NOTE_PROMPT.format(
|
||||
style=style, idx=i, title=n['title'],
|
||||
note_info=note_info, report=report, tmall_persona=tmall_persona_text,
|
||||
platform=n['platform'], creator=n.get('creator_name', '?'),
|
||||
views=views, engagement=engagement, rate=rate,
|
||||
)
|
||||
log(f" 调 LLM 单篇分析 {i}/{len(notes)}: {n['title'][:30]}...")
|
||||
analysis = call_hermes_analyzer(system, prompt)
|
||||
if analysis:
|
||||
single_analyses.append(analysis)
|
||||
else:
|
||||
single_analyses.append(
|
||||
f"### 笔记{i}：{n['title']}\n**基础信息**\n"
|
||||
f"- 平台/达人：{n['platform']} / {n.get('creator_name', '?')}\n"
|
||||
f"- 曝光：{views} | 互动总量：{engagement} | 互动率：{rate}%\n\n"
|
||||
f"*(LLM 分析失败,数据仅供参考)*"
|
||||
)
|
||||
|
||||
# ---- 拼接投放概览 ----
|
||||
platform_counts = {}
|
||||
total_views = 0
|
||||
total_engagement = 0
|
||||
for n in notes:
|
||||
p = n['platform']
|
||||
platform_counts[p] = platform_counts.get(p, 0) + 1
|
||||
v = n.get('view_count')
|
||||
if isinstance(v, (int, float)) and v > 0:
|
||||
total_views += v
|
||||
likes = n.get('like_count') or 0
|
||||
cmts = n.get('comment_count') or 0
|
||||
favs = n.get('favorite_count') or 0
|
||||
shares = n.get('share_count') or 0
|
||||
total_engagement += (likes + cmts + favs + shares)
|
||||
platforms_str = " / ".join(f"{p} {c} 篇" for p, c in sorted(platform_counts.items()))
|
||||
views_str = f"{total_views:,}" if total_views else "部分未采集"
|
||||
|
||||
overview = (
|
||||
f"# {style} 周笔记汇总（{TIME_LABEL}）\n\n"
|
||||
f"## 一、本周投放概览\n\n"
|
||||
f"- 投放笔记数：{len(notes)} 篇\n"
|
||||
f"- 涉及平台：{platforms_str}\n"
|
||||
f"- 总曝光 / 总互动：{views_str} / {total_engagement:,}\n"
|
||||
f"- 整体表现：[见下方周度总结]\n\n"
|
||||
f"---\n\n"
|
||||
f"## 二、单篇笔记四维度分析\n\n"
|
||||
)
|
||||
|
||||
# ---- 第二步:生成周度总结 ----
|
||||
all_analyses_text = "\n\n---\n\n".join(single_analyses)
|
||||
summary_prompt = WEEKLY_SUMMARY_PROMPT.format(
|
||||
style=style, n=len(notes),
|
||||
all_analyses=all_analyses_text,
|
||||
platforms=platforms_str,
|
||||
total_views=views_str,
|
||||
total_engagement=f"{total_engagement:,}",
|
||||
)
|
||||
log(f" 调 LLM 周度总结...")
|
||||
weekly_summary = call_hermes_analyzer(system, summary_prompt)
|
||||
if not weekly_summary:
|
||||
weekly_summary = "## 三、周度总结与建议\n\n*(LLM 周度总结生成失败)*"
|
||||
|
||||
return overview + all_analyses_text + "\n\n---\n\n" + weekly_summary
|
||||
|
||||
|
||||
def create_feishu_doc(style, summary):
|
||||
"""创建飞书文档,返回 url"""
|
||||
doc_title = f"{style} 周笔记汇总（{TIME_LABEL}）"
|
||||
md_file = PATHS.tmp_root / "weekly_summary_doc.md"
|
||||
md_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_file.write_text(summary, encoding="utf-8")
|
||||
|
||||
cmd = [LARK, "docs", "+create",
|
||||
"--as", "user", "--format", "json",
|
||||
"--title", doc_title,
|
||||
"--doc-format", "markdown",
|
||||
"--content", f"@{md_file.name}"]
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env,
|
||||
encoding="utf-8", errors="replace")
|
||||
text = proc.stdout or proc.stderr
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
return None, f"创建失败: {text[:300]}"
|
||||
resp = json.loads(text[start:])
|
||||
if not resp.get("ok"):
|
||||
return None, f"创建失败: {resp.get('error')}"
|
||||
url = resp.get("data", {}).get("document", {}).get("url", "")
|
||||
return url, None
|
||||
|
||||
|
||||
def save_to_db(style, notes, summary, doc_url, doc_title, status):
|
||||
"""把周汇总结果写入 PG cmt_weekly_summary(UNIQUE(style_id, week_start),同款同周重跑时 UPDATE)"""
|
||||
if dry_run_global:
|
||||
return False
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
try:
|
||||
with get_conn() as conn:
|
||||
cur = conn.cursor()
|
||||
# 查 style_id
|
||||
cur.execute("SELECT id FROM cmt_styles WHERE name = %s", (style,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
log(f" [DB] 款式 {style} 不在 cmt_styles,跳过入库")
|
||||
return False
|
||||
style_id = row[0]
|
||||
|
||||
# 算聚合数据
|
||||
total_views = sum((n.get('view_count') or 0) for n in notes) or None
|
||||
total_comments = sum((n.get('comment_count') or 0) for n in notes)
|
||||
total_engagement = sum(
|
||||
((n.get('like_count') or 0) + (n.get('favorite_count') or 0) +
|
||||
(n.get('comment_count') or 0) + (n.get('share_count') or 0))
|
||||
for n in notes) or None
|
||||
|
||||
# UPSERT (style_id, week_start)
|
||||
cur.execute("""
|
||||
INSERT INTO cmt_weekly_summary
|
||||
(style_id, week_start, week_end, time_label,
|
||||
note_count, total_comments, total_views, total_engagement,
|
||||
doc_url, doc_title, summary_chars, status,
|
||||
created_at, updated_at)
|
||||
VALUES
|
||||
(%s, %s, %s, %s,
|
||||
%s, %s, %s, %s,
|
||||
%s, %s, %s, %s,
|
||||
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (style_id, week_start) DO UPDATE SET
|
||||
note_count = EXCLUDED.note_count,
|
||||
total_comments = EXCLUDED.total_comments,
|
||||
total_views = EXCLUDED.total_views,
|
||||
total_engagement = EXCLUDED.total_engagement,
|
||||
doc_url = EXCLUDED.doc_url,
|
||||
doc_title = EXCLUDED.doc_title,
|
||||
summary_chars = EXCLUDED.summary_chars,
|
||||
status = EXCLUDED.status,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""", (style_id, WEEK_START, WEEK_END, TIME_LABEL,
|
||||
len(notes), total_comments, total_views, total_engagement,
|
||||
doc_url, doc_title, len(summary) if summary else None, status))
|
||||
conn.commit()
|
||||
log(f" ✅ 已入库 cmt_weekly_summary (style_id={style_id}, week={WEEK_START})")
|
||||
return True
|
||||
except Exception as exc:
|
||||
log(f" [DB ERR] {style} 入库失败: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def write_to_target_table(bt, tid, time_fid, summary_fid, style, doc_url, dry_run):
|
||||
"""把文档链接写入生命进程表(已有同时间记录则更新,否则新建)"""
|
||||
md_link = f"[{style} 周笔记汇总（{TIME_LABEL}）]({doc_url})"
|
||||
payload = {time_fid: TIME_LABEL, summary_fid: md_link}
|
||||
pf = PATHS.tmp_root / "weekly_payload.json"
|
||||
pf.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
if dry_run:
|
||||
log(f" [DRY] 跳过写入 {bt}/{tid}")
|
||||
return True
|
||||
|
||||
# 先查是否已有同时间的记录
|
||||
existing_rid = None
|
||||
list_cmd = ["base", "+record-list",
|
||||
"--base-token", bt, "--table-id", tid,
|
||||
"--as", "user", "--format", "json", "--page-size", "100"]
|
||||
list_resp = call_lark_json(list_cmd)
|
||||
if list_resp.get("ok"):
|
||||
rid_list = list_resp.get("data", {}).get("record_id_list", [])
|
||||
data_rows = list_resp.get("data", {}).get("data", [])
|
||||
# 找字段索引: 用 field_id_list 定位 time_fid 的列号
|
||||
fid_list = list_resp.get("data", {}).get("field_id_list", [])
|
||||
time_col = fid_list.index(time_fid) if time_fid in fid_list else 0
|
||||
for rid, row in zip(rid_list, data_rows):
|
||||
row_time = row[time_col] if len(row) > time_col else ""
|
||||
# 时间字段是 select, 返回值可能是 ["2026-06-29~07-05"]
|
||||
if isinstance(row_time, list):
|
||||
row_time = row_time[0] if row_time else ""
|
||||
if str(row_time) == TIME_LABEL:
|
||||
existing_rid = rid
|
||||
break
|
||||
|
||||
if existing_rid:
|
||||
cmd = ["base", "+record-upsert",
|
||||
"--base-token", bt, "--table-id", tid,
|
||||
"--as", "user", "--format", "json",
|
||||
"--record-id", existing_rid,
|
||||
"--json", f"@{pf.name}"]
|
||||
log(f" 更新已有记录 {existing_rid}")
|
||||
else:
|
||||
cmd = ["base", "+record-upsert",
|
||||
"--base-token", bt, "--table-id", tid,
|
||||
"--as", "user", "--format", "json",
|
||||
"--json", f"@{pf.name}"]
|
||||
log(f" 新建记录")
|
||||
|
||||
resp = call_lark_json(cmd)
|
||||
return resp.get("ok", False)
|
||||
|
||||
|
||||
def process_one_style(style, index_info, dry_run):
|
||||
"""处理一个款:跑单品分析+综合+创建文档+写表"""
|
||||
bt, tid, url = index_info
|
||||
log(f"\n========== {style} ==========")
|
||||
log(f" 生命进程表: {bt}/{tid}")
|
||||
|
||||
def _save(notes_list, summary, doc_url, doc_title, status):
|
||||
"""统一入库辅助"""
|
||||
save_to_db(style, notes_list, summary, doc_url, doc_title, status)
|
||||
|
||||
# 动态读字段
|
||||
time_fid, summary_fid = find_target_fields(bt, tid)
|
||||
if not time_fid or not summary_fid:
|
||||
log(f" [ERR] 找不到 时间/笔记分析汇总 字段,跳过")
|
||||
_save([], None, None, None, "no_fields")
|
||||
return {"style": style, "status": "no_fields", "notes": 0}
|
||||
|
||||
# 拿上周笔记
|
||||
notes = get_last_week_notes(style)
|
||||
log(f" 上周笔记: {len(notes)} 篇")
|
||||
if not notes:
|
||||
log(f" [SKIP] 上周无笔记")
|
||||
_save([], None, None, None, "no_notes")
|
||||
return {"style": style, "status": "no_notes", "notes": 0}
|
||||
|
||||
# 跑单品分析
|
||||
for n in notes:
|
||||
n["report_path"] = run_single_note_analysis(n["id"])
|
||||
|
||||
# 过滤掉没生成报告的
|
||||
valid_notes = [n for n in notes if n.get("report_path")]
|
||||
if not valid_notes:
|
||||
log(f" [ERR] 没有有效报告,跳过")
|
||||
_save(notes, None, None, None, "no_reports")
|
||||
return {"style": style, "status": "no_reports", "notes": len(notes)}
|
||||
|
||||
# LLM 综合
|
||||
log(f" 调 LLM 综合 {len(valid_notes)} 篇报告...")
|
||||
summary = generate_summary(style, valid_notes)
|
||||
if not summary:
|
||||
log(f" [ERR] LLM 综合失败")
|
||||
_save(notes, None, None, None, "llm_failed")
|
||||
return {"style": style, "status": "llm_failed", "notes": len(notes)}
|
||||
|
||||
# 保存本地
|
||||
out_path = REPORTS_DIR / f"{style}_周笔记汇总_{TIME_LABEL}.txt"
|
||||
out_path.write_text(summary, encoding="utf-8")
|
||||
log(f" 本地副本: {out_path.name} ({len(summary)} 字符)")
|
||||
|
||||
if dry_run_global:
|
||||
log(f" [DRY] 跳过飞书文档创建和写入")
|
||||
_save(notes, summary, None, None, "dry_run")
|
||||
return {"style": style, "status": "dry_run", "notes": len(notes), "chars": len(summary)}
|
||||
|
||||
# 创建飞书文档
|
||||
doc_url, err = create_feishu_doc(style, summary)
|
||||
if not doc_url:
|
||||
log(f" [ERR] {err}")
|
||||
_save(notes, summary, None, None, "doc_failed")
|
||||
return {"style": style, "status": "doc_failed", "notes": len(notes)}
|
||||
|
||||
log(f" 飞书文档: {doc_url}")
|
||||
|
||||
# 写入生命进程表
|
||||
ok = write_to_target_table(bt, tid, time_fid, summary_fid, style, doc_url, dry_run_global)
|
||||
|
||||
# 写入 PG cmt_weekly_summary
|
||||
doc_title = f"{style} 周笔记汇总（{TIME_LABEL}）"
|
||||
_save(notes, summary, doc_url, doc_title, "ok" if ok else "feishu_write_failed")
|
||||
|
||||
if ok:
|
||||
log(f" ✅ 已写入生命进程表")
|
||||
return {"style": style, "status": "ok", "notes": len(notes),
|
||||
"doc_url": doc_url, "chars": len(summary)}
|
||||
log(f" [ERR] 写入表失败")
|
||||
return {"style": style, "status": "write_failed", "notes": len(notes)}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="通用周笔记汇总:对所有上周有笔记的款生成飞书文档并写入生命进程表")
|
||||
parser.add_argument("--style", help="只跑指定款式名")
|
||||
parser.add_argument("--dry-run", action="store_true", help="不写飞书,只生成本地报告")
|
||||
parser.add_argument("--max-workers", type=int, default=4,
|
||||
help="款之间并发数(默认 4)")
|
||||
args = parser.parse_args()
|
||||
|
||||
global dry_run_global
|
||||
dry_run_global = args.dry_run
|
||||
|
||||
log(f"时间范围: {TIME_LABEL}")
|
||||
|
||||
# 读索引表
|
||||
index_styles = load_index_styles()
|
||||
if not index_styles:
|
||||
log("索引表无有效数据,退出")
|
||||
return 1
|
||||
|
||||
# PG 查上周有笔记的款
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.db import get_conn
|
||||
with get_conn() as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT s.name, COUNT(n.id)
|
||||
FROM cmt_styles s
|
||||
JOIN cmt_notes n ON n.style_id = s.id AND n.self_operated = FALSE
|
||||
WHERE n.publish_time IS NOT NULL
|
||||
AND n.publish_time >= %s AND n.publish_time < %s
|
||||
GROUP BY s.name
|
||||
ORDER BY s.name
|
||||
""", (WEEK_START, WEEK_END + " 23:59:59"))
|
||||
pg_styles = [(r[0], r[1]) for r in cur.fetchall()]
|
||||
|
||||
log(f"PG 上周有笔记的款: {len(pg_styles)} 个")
|
||||
|
||||
# 取交集:PG 里有 + 索引表里有飞书表地址
|
||||
to_run = []
|
||||
for name, note_count in pg_styles:
|
||||
if args.style and name != args.style:
|
||||
continue
|
||||
if name in index_styles:
|
||||
to_run.append((name, index_styles[name], note_count))
|
||||
else:
|
||||
log(f" [SKIP] {name}: 索引表无'每周笔记分析'地址")
|
||||
|
||||
log(f"待跑款式: {len(to_run)} 个")
|
||||
if not to_run:
|
||||
return 0
|
||||
|
||||
# 并发跑款
|
||||
import concurrent.futures
|
||||
log(f"并发跑 {len(to_run)} 款, max_workers={args.max_workers}")
|
||||
results = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as executor:
|
||||
future_to_style = {
|
||||
executor.submit(process_one_style, name, info, args.dry_run): (name, ncount)
|
||||
for name, info, ncount in to_run
|
||||
}
|
||||
for future in concurrent.futures.as_completed(future_to_style):
|
||||
name, ncount = future_to_style[future]
|
||||
try:
|
||||
r = future.result()
|
||||
r["note_count"] = ncount
|
||||
results.append(r)
|
||||
log(f" ✅ 完成: {name} -> {r['status']}")
|
||||
except Exception as exc:
|
||||
log(f" [ERR] {name} 异常: {exc}")
|
||||
results.append({"style": name, "status": "exception",
|
||||
"error": str(exc), "note_count": ncount})
|
||||
|
||||
# 按款式名排序输出
|
||||
results.sort(key=lambda r: r.get("style", ""))
|
||||
|
||||
# 汇总
|
||||
log(f"\n{'=' * 50}")
|
||||
log(f"全部完成,汇总:")
|
||||
for r in results:
|
||||
log(f" {r['style']}: {r['status']} (笔记 {r.get('note_count', 0)} 篇)")
|
||||
log(f"{'=' * 50}")
|
||||
|
||||
# 保存结果
|
||||
out = REPORTS_DIR / f"_weekly_summary_all_{TIME_LABEL}.json"
|
||||
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
log(f"结果已存: {out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,244 @@
|
||||
"""综合 3 篇笔记分析报告,生成星云2 周笔记汇总 -> 写入飞书表"""
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.data.tools.analyze_comments import call_hermes_analyzer
|
||||
|
||||
LARK = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd")
|
||||
if not os.path.exists(LARK):
|
||||
LARK = "lark-cli.cmd"
|
||||
|
||||
TARGET_BT = "BJaWbLEOKaHONMshOidcxvsKngh"
|
||||
TARGET_TID = "tbltzBjbdiW4GrUd"
|
||||
TIME_FIELD = "fldpKSfCCC"
|
||||
SUMMARY_FIELD = "fldro5fLrb"
|
||||
|
||||
SUMMARY_PROMPT = """你是光影行星品牌「星云2」款式的周投放复盘分析师。
|
||||
请基于本周 3 篇笔记的单品分析报告,生成周笔记汇总报告。
|
||||
|
||||
【本周笔记清单】
|
||||
{notes}
|
||||
|
||||
【输出要求】
|
||||
按以下结构输出(Markdown 格式,适合贴到飞书文本字段):
|
||||
|
||||
# 星云2 周笔记汇总（{time_range}）
|
||||
|
||||
## 一、本周投放概览
|
||||
- 投放笔记数：{n} 篇
|
||||
- 涉及平台：[列出]
|
||||
- 总曝光 / 总互动：[汇总数据]
|
||||
- 整体表现：[一句话定性]
|
||||
|
||||
## 二、投放效果多维分析
|
||||
|
||||
### 1. 流量效率
|
||||
- 各平台曝光表现:[逐平台分析,抖音/小红书/B站]
|
||||
- 互动率结构:[点赞率/收藏率/评论率/分享率,识别认知留存型/社交裂变型/搜索截流型]
|
||||
- 流量是否匹配投放预期:[判断 + 理由]
|
||||
|
||||
### 2. 互动质量
|
||||
- 评论区有效评论占比:[统计]
|
||||
- 评论质量分层:[有效种草/连带兴趣/无效互动 占比]
|
||||
- 用户决策阶段:[规格确认/价值判断/竞品对比/冲动下单/售后反馈,哪个阶段为主]
|
||||
|
||||
### 3. 品牌词沉淀
|
||||
- 品牌词"光影行星"提及情况:[条数 + 上下文]
|
||||
- 款式词"星云2"提及情况
|
||||
- 我方产品相关评论占比
|
||||
- 品牌词是否被自然讨论:[是/否 + 理由]
|
||||
|
||||
### 4. 卖点传达
|
||||
- 笔记主推卖点 vs 评论区讨论卖点:[一致性分析]
|
||||
- 卖点偏离方向:[若偏离,说明偏向哪里]
|
||||
- 三大核心卖点(磁吸扣/分层收纳/容量/颜值/重量)在评论区的渗透情况
|
||||
|
||||
### 5. 达人匹配度
|
||||
- 各达人与星云2 目标客群重合度
|
||||
- 内容风格是否适合承载星云2 核心卖点
|
||||
- 哪些达人值得复用,哪些应淘汰
|
||||
|
||||
### 6. 负面舆情
|
||||
- 评论区负面集中点:[如自重/价格/做工]
|
||||
- 影响面大小
|
||||
- 是否需要控评
|
||||
|
||||
### 7. 平台差异
|
||||
- 同主题不同平台表现对比(如有)
|
||||
- 各平台用户关注点差异
|
||||
- 平台化适配建议
|
||||
|
||||
## 三、单篇笔记提炼（精简）
|
||||
|
||||
### 笔记1：{title1}
|
||||
- 平台/达人：{platform1} / {creator1}
|
||||
- 核心数据：曝光 {views1} | 互动率 [X%]
|
||||
- 评级：[优秀/良好/不好]
|
||||
- 一句话提炼：[20-30 字,点出成败关键]
|
||||
- 值得推荐：[是/否]
|
||||
|
||||
### 笔记2：{title2}
|
||||
[同上格式]
|
||||
|
||||
### 笔记3：{title3}
|
||||
[同上格式]
|
||||
|
||||
## 四、值得推荐的笔记
|
||||
[列出评级良好及以上的笔记,给出可复用经验,每篇 2-3 句]
|
||||
|
||||
## 五、需要规避的问题
|
||||
[列出"不好"评级笔记的问题,每篇 2-3 句,给规避建议]
|
||||
|
||||
## 六、下周投放优化清单
|
||||
- 内容方向：[基于本周反馈的具体建议]
|
||||
- 达人选择：[优先 / 规避类型]
|
||||
- 评论区运营：[具体话术]
|
||||
- 投放策略：[预算 / 流量助推 / 平台侧重]
|
||||
|
||||
【评级标准】
|
||||
- 优秀：互动率高于行业基准(抖音≥6% / 小红书≥8% / B站≥5%)、评论中有产品讨论、品牌词被自然提及
|
||||
- 良好：互动率达标但评论质量一般,或评论质量好但互动率刚达标
|
||||
- 不好：互动率不达标、评论区无产品讨论、品牌词未沉淀
|
||||
|
||||
【约束】
|
||||
- 第二部分"投放效果多维分析"必须详尽,每子模块给具体数据和判断
|
||||
- 第三部分"单篇笔记提炼"精简,每篇 5-6 行内
|
||||
- 直接给结论,不写"根据数据显示"
|
||||
- 每条建议须具体可执行
|
||||
- 评级严格按标准,不放宽
|
||||
- 整体篇幅 1500-2500 字(单篇精简,多维分析详尽)"""
|
||||
|
||||
|
||||
def read_report(path: Path) -> str:
|
||||
if not path.exists():
|
||||
return ""
|
||||
for enc in ("utf-8-sig", "utf-8"):
|
||||
try:
|
||||
return path.read_text(encoding=enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
def main():
|
||||
reports_dir = PATHS.exports_root / "summary"
|
||||
notes = [
|
||||
{"id": 1662, "platform": "bilibili", "creator": "裴念南",
|
||||
"title": "SwitchLite开箱验机教程📦➕全套配件分享✨",
|
||||
"views": 1318, "likes": 64, "cmts": 2, "favs": 21,
|
||||
"report": read_report(reports_dir / "note_1662.txt")},
|
||||
{"id": 1663, "platform": "xiaohongshu", "creator": "不爱出门的阿宅",
|
||||
"title": "六月switch配件购买分享～实用又好看",
|
||||
"views": 2224, "likes": 140, "cmts": 6, "favs": 59,
|
||||
"report": read_report(reports_dir / "note_1663.txt")},
|
||||
{"id": 1664, "platform": "douyin", "creator": "不爱出门的阿宅",
|
||||
"title": "六月switch配件购买分享～实用又好看",
|
||||
"views": None, "likes": 63, "cmts": 7, "favs": 16,
|
||||
"report": read_report(reports_dir / "note_1664.txt")},
|
||||
]
|
||||
|
||||
# 拼笔记清单
|
||||
notes_block = []
|
||||
for i, n in enumerate(notes, 1):
|
||||
notes_block.append(
|
||||
f"### 笔记{i}: {n['title']}\n"
|
||||
f"- 平台: {n['platform']}\n- 达人: {n['creator']}\n"
|
||||
f"- 曝光: {n['views'] or '无'} / 点赞: {n['likes']} / 评论: {n['cmts']} / 收藏: {n['favs']}\n\n"
|
||||
f"单品分析报告全文:\n{n['report']}\n"
|
||||
)
|
||||
notes_text = "\n---\n".join(notes_block)
|
||||
|
||||
# 填 prompt
|
||||
prompt = SUMMARY_PROMPT.format(notes=notes_text,
|
||||
time_range="2026-06-29~07-05",
|
||||
n=len(notes),
|
||||
title1=notes[0]['title'], platform1=notes[0]['platform'],
|
||||
creator1=notes[0]['creator'], views1=notes[0]['views'] or '无',
|
||||
likes1=notes[0]['likes'], cmts1=notes[0]['cmts'], favs1=notes[0]['favs'],
|
||||
title2=notes[1]['title'], platform2=notes[1]['platform'],
|
||||
creator2=notes[1]['creator'], views2=notes[1]['views'] or '无',
|
||||
likes2=notes[1]['likes'], cmts2=notes[1]['cmts'], favs2=notes[1]['favs'],
|
||||
title3=notes[2]['title'], platform3=notes[2]['platform'],
|
||||
creator3=notes[2]['creator'], views3=notes[2]['views'] or '无',
|
||||
likes3=notes[2]['likes'], cmts3=notes[2]['cmts'], favs3=notes[2]['favs'])
|
||||
|
||||
print("调用 LLM 综合评级...")
|
||||
summary = call_hermes_analyzer("你是光影行星品牌的周投放复盘分析师,擅长综合多篇笔记分析报告,生成款式级别的多维投放复盘。", prompt)
|
||||
print(f"汇总报告生成完成,长度 {len(summary)} 字符")
|
||||
|
||||
# 保存本地副本
|
||||
time_label = "2026-06-29~07-05"
|
||||
style_label = "星云2"
|
||||
out_path = reports_dir / f"{style_label}_周笔记汇总_{time_label}.txt"
|
||||
out_path.write_text(summary, encoding="utf-8")
|
||||
print(f"本地副本: {out_path}")
|
||||
|
||||
# 创建飞书文档(markdown 格式)
|
||||
# LLM 输出的 summary 自身已以 "# 标题" 开头,不需要额外加;--title 会被 lark-cli 自动设为文档标题
|
||||
print("创建飞书文档...")
|
||||
doc_title = f"{style_label} 周笔记汇总（{time_label}）"
|
||||
md_file = PATHS.tmp_root / "weekly_summary_doc.md"
|
||||
md_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_file.write_text(summary, encoding="utf-8")
|
||||
|
||||
cmd = [LARK, "docs", "+create",
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--title", doc_title,
|
||||
"--doc-format", "markdown",
|
||||
"--content", f"@{md_file.name}"]
|
||||
env = {**os.environ, "LARK_CLI_NO_PROXY": "1"}
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env, encoding="utf-8", errors="replace")
|
||||
text = proc.stdout or proc.stderr
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
print(f"创建文档失败: {text[:500]}")
|
||||
return 1
|
||||
import json as _json
|
||||
resp = _json.loads(text[start:])
|
||||
if not resp.get("ok"):
|
||||
print(f"创建文档失败: {resp.get('error')}")
|
||||
return 1
|
||||
doc_url = resp.get("data", {}).get("document", {}).get("url", "")
|
||||
doc_token = resp.get("data", {}).get("document", {}).get("document_id", "")
|
||||
print(f"飞书文档已创建: {doc_url}")
|
||||
print(f" doc_token: {doc_token}")
|
||||
|
||||
# 在飞书表里写 markdown 链接(标题+URL)
|
||||
md_link = f"[{doc_title}]({doc_url})"
|
||||
payload = {
|
||||
TIME_FIELD: time_label,
|
||||
SUMMARY_FIELD: md_link,
|
||||
}
|
||||
pf = PATHS.tmp_root / "weekly_summary_payload.json"
|
||||
pf.write_text(_json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
cmd = [LARK, "base", "+record-upsert",
|
||||
"--base-token", TARGET_BT,
|
||||
"--table-id", TARGET_TID,
|
||||
"--as", "user",
|
||||
"--format", "json",
|
||||
"--json", f"@{pf.name}"]
|
||||
proc = subprocess.run(cmd, capture_output=True, env=env, encoding="utf-8", errors="replace")
|
||||
text = proc.stdout or proc.stderr
|
||||
start = text.find("{")
|
||||
if start < 0:
|
||||
print(f"插入表失败: {text[:300]}")
|
||||
return 1
|
||||
resp = _json.loads(text[start:])
|
||||
if resp.get("ok"):
|
||||
print(f"已写入飞书表: 时间={time_label}, 笔记分析汇总=文档链接")
|
||||
return 0
|
||||
print(f"插入表失败: {resp.get('error')}")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,695 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
from playwright.sync_api import Page, TimeoutError as PlaywrightTimeoutError
|
||||
from scrapling.fetchers import DynamicSession
|
||||
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
STATE_DIR = PATHS.state_root
|
||||
NOTES_DIR = PATHS.raw_root / "notes" / "xiaohongshu"
|
||||
COOKIE_FILE = PATHS.browser_cookie_file
|
||||
PROFILE_DIR = PATHS.browser_profile_dir
|
||||
|
||||
COMMENT_API_MARKS = (
|
||||
"/api/sns/web/v2/comment/page",
|
||||
"/api/sns/web/v2/comment/sub/page",
|
||||
)
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {message}", flush=True)
|
||||
|
||||
|
||||
def load_cookies() -> list[dict[str, Any]]:
|
||||
if not COOKIE_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except json.JSONDecodeError:
|
||||
log(f"Cookie file is invalid, ignoring: {COOKIE_FILE}")
|
||||
return []
|
||||
|
||||
|
||||
def restore_cookies(page: Page) -> None:
|
||||
cookies = load_cookies()
|
||||
if not cookies:
|
||||
return
|
||||
try:
|
||||
page.context.add_cookies(cookies)
|
||||
log(f"Loaded cookies: {COOKIE_FILE}")
|
||||
except Exception as exc:
|
||||
log(f"Failed to load cookies, will continue with browser profile: {exc}")
|
||||
|
||||
|
||||
def save_state(page: Page) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
COOKIE_FILE.write_text(
|
||||
json.dumps(page.context.cookies(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
PATHS.browser_storage_state_file.write_text(
|
||||
json.dumps(page.context.storage_state(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
log(f"Saved cookies: {COOKIE_FILE}")
|
||||
|
||||
|
||||
def note_id_from_url(url: str) -> str:
|
||||
match = re.search(r"/explore/([0-9a-fA-F]+)", url)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def safe_filename(value: str) -> str:
|
||||
value = re.sub(r"[<>:\"/\\\\|?*\\s]+", "_", value).strip("_")
|
||||
return value[:90] or "xiaohongshu_comments"
|
||||
|
||||
|
||||
def ts_to_iso(value: Any) -> str:
|
||||
if not isinstance(value, (int, float)) or not value:
|
||||
return ""
|
||||
timestamp = value / 1000 if value > 10_000_000_000 else value
|
||||
try:
|
||||
return datetime.fromtimestamp(timestamp).isoformat(timespec="seconds")
|
||||
except Exception:
|
||||
return str(value)
|
||||
|
||||
|
||||
def parse_comment(comment: dict[str, Any], source_url: str, level: str, parent_id: str = "") -> dict[str, Any]:
|
||||
user = comment.get("user_info") or comment.get("user") or {}
|
||||
comment_id = comment.get("id") or comment.get("comment_id") or ""
|
||||
create_time = comment.get("create_time") or comment.get("time")
|
||||
return {
|
||||
"source_url": source_url,
|
||||
"level": level,
|
||||
"comment_id": comment_id,
|
||||
"reply_to_comment_id": parent_id or comment.get("target_comment", {}).get("id", ""),
|
||||
"nickname": user.get("nickname") or "",
|
||||
"user_id": user.get("user_id") or user.get("id") or "",
|
||||
"content": comment.get("content") or comment.get("text") or "",
|
||||
"like_count": comment.get("like_count") or 0,
|
||||
"reply_count": comment.get("sub_comment_count") or comment.get("sub_comment_total") or 0,
|
||||
"ip_location": comment.get("ip_location") or comment.get("ip_label") or "",
|
||||
"created_at": ts_to_iso(create_time),
|
||||
"raw_create_time": create_time or "",
|
||||
}
|
||||
|
||||
|
||||
def merge_comment(
|
||||
row: dict[str, Any],
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
comment_id = str(row.get("comment_id") or f"{row.get('level')}:{len(comments_by_id)}")
|
||||
comments_by_id[comment_id] = row
|
||||
|
||||
|
||||
def collect_from_payload(
|
||||
payload: dict[str, Any],
|
||||
source_url: str,
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
stats: dict[str, Any],
|
||||
) -> None:
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
data = payload.get("data") if isinstance(payload.get("data"), dict) else payload
|
||||
api_url = str(payload.get("__source_url", ""))
|
||||
is_sub = "/comment/sub/page" in api_url
|
||||
|
||||
if not is_sub:
|
||||
stats["api_total"] = data.get("total", stats.get("api_total"))
|
||||
stats["last_has_more"] = data.get("has_more", stats.get("last_has_more"))
|
||||
stats["last_cursor"] = data.get("cursor", stats.get("last_cursor"))
|
||||
|
||||
comments = data.get("comments") or data.get("comment_list") or []
|
||||
if not isinstance(comments, list):
|
||||
return
|
||||
|
||||
parent_id = ""
|
||||
if is_sub:
|
||||
params = dict(parse_qsl(urlsplit(api_url).query))
|
||||
parent_id = params.get("root_comment_id") or params.get("comment_id") or params.get("target_comment_id") or ""
|
||||
|
||||
for comment in comments:
|
||||
if not isinstance(comment, dict):
|
||||
continue
|
||||
row = parse_comment(comment, source_url, "reply" if is_sub else "comment", parent_id)
|
||||
merge_comment(row, comments_by_id)
|
||||
|
||||
sub_comments = comment.get("sub_comments") or []
|
||||
if isinstance(sub_comments, list):
|
||||
for sub in sub_comments:
|
||||
if isinstance(sub, dict):
|
||||
sub_row = parse_comment(sub, source_url, "reply", str(row.get("comment_id") or ""))
|
||||
merge_comment(sub_row, comments_by_id)
|
||||
|
||||
|
||||
def setup_response_capture(page: Page, source_url: str) -> tuple[dict[str, dict[str, Any]], dict[str, Any]]:
|
||||
comments_by_id: dict[str, dict[str, Any]] = {}
|
||||
stats: dict[str, Any] = {"api_hits": 0, "api_total": None, "last_has_more": None, "last_cursor": None}
|
||||
|
||||
def handle_response(response: Any) -> None:
|
||||
url = response.url
|
||||
if not any(mark in url for mark in COMMENT_API_MARKS):
|
||||
return
|
||||
try:
|
||||
payload = response.json()
|
||||
except Exception:
|
||||
return
|
||||
if isinstance(payload, dict):
|
||||
payload["__source_url"] = url
|
||||
stats["api_hits"] += 1
|
||||
if "/comment/page" in url and not stats.get("comment_url_template"):
|
||||
stats["comment_url_template"] = url
|
||||
if "/comment/sub/page" in url and not stats.get("sub_url_template"):
|
||||
stats["sub_url_template"] = url
|
||||
collect_from_payload(payload, source_url, comments_by_id, stats)
|
||||
|
||||
page.on("response", handle_response)
|
||||
return comments_by_id, stats
|
||||
|
||||
|
||||
def click_if_visible(page: Page, text: str, timeout: int = 1200) -> bool:
|
||||
try:
|
||||
locator = page.get_by_text(text, exact=False).first
|
||||
if locator.count():
|
||||
locator.click(timeout=timeout)
|
||||
page.wait_for_timeout(500)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def dismiss_popups(page: Page) -> None:
|
||||
for text in ("我知道了", "稍后再说", "暂不登录", "取消", "跳过"):
|
||||
click_if_visible(page, text)
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const close = [...document.querySelectorAll('button, div, span')]
|
||||
.filter((el) => {
|
||||
const t = (el.innerText || '').trim();
|
||||
const r = el.getBoundingClientRect();
|
||||
return ['×', 'x', 'X'].includes(t) &&
|
||||
r.width > 0 && r.width <= 52 &&
|
||||
r.height > 0 && r.height <= 52 &&
|
||||
r.top >= 0 && r.top < window.innerHeight * 0.75;
|
||||
})[0];
|
||||
close?.click();
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def maybe_wait_for_login(page: Page, login_timeout: int) -> None:
|
||||
deadline = time.time() + login_timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
body = page.locator("body").inner_text(timeout=5000)
|
||||
except Exception:
|
||||
body = ""
|
||||
if "扫码登录" not in body and "登录后" not in body and "验证码" not in body:
|
||||
return
|
||||
log("Xiaohongshu may require login. Please finish login in the opened Chrome window.")
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
|
||||
def is_logged_in(page: Page) -> bool:
|
||||
"""小红书登录态靠 web_session cookie,有它才能采全量评论。"""
|
||||
for c in page.context.cookies():
|
||||
if c.get("name") == "web_session" and c.get("value"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def expand_visible_replies(page: Page) -> int:
|
||||
try:
|
||||
clicked = page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const visible = (el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(el);
|
||||
return r.width > 0 && r.height > 0 &&
|
||||
r.bottom > 0 && r.top < window.innerHeight &&
|
||||
style.visibility !== 'hidden' &&
|
||||
style.display !== 'none' &&
|
||||
style.opacity !== '0';
|
||||
};
|
||||
const candidates = [...document.querySelectorAll('button, div, span')]
|
||||
.filter((el) => {
|
||||
const text = (el.innerText || '').trim();
|
||||
return visible(el) &&
|
||||
text.length <= 40 &&
|
||||
(/展开.*回复/.test(text) || /查看.*回复/.test(text) || /共.*条回复/.test(text) || /还有.*回复/.test(text));
|
||||
});
|
||||
let count = 0;
|
||||
for (const el of candidates.slice(0, 8)) {
|
||||
el.click();
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
"""
|
||||
)
|
||||
if clicked:
|
||||
page.wait_for_timeout(1200)
|
||||
return int(clicked or 0)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def scroll_comments(page: Page, max_scrolls: int, idle_rounds: int, comments_by_id: dict[str, Any]) -> None:
|
||||
last_count = len(comments_by_id)
|
||||
idle = 0
|
||||
page.mouse.move(1030, 620)
|
||||
|
||||
for index in range(max_scrolls):
|
||||
dismiss_popups(page)
|
||||
expand_visible_replies(page)
|
||||
page.mouse.wheel(0, 1000)
|
||||
# Scroll the comment panel from the inside — the right-side panel
|
||||
# is a fixed-height container with its own scroll; wheel events on
|
||||
# the page body don't propagate into it.
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
// First try the dedicated comment/interaction panel
|
||||
const commentPanel = document.querySelector('.comment-inner-container')
|
||||
|| document.querySelector('[class*="commentListContainer"]')
|
||||
|| document.querySelector('[class*="comments-container"]');
|
||||
if (commentPanel) {
|
||||
commentPanel.scrollTop += 1200;
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback: find tall scrollable containers on the right half
|
||||
const candidates = [...document.querySelectorAll('div, section, aside')]
|
||||
.filter((el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
return r.height > 160 &&
|
||||
r.width > 220 &&
|
||||
el.scrollHeight > el.clientHeight + 80 &&
|
||||
r.right > window.innerWidth * 0.35;
|
||||
})
|
||||
.sort((a, b) => (b.scrollHeight - b.clientHeight) - (a.scrollHeight - a.clientHeight));
|
||||
for (const el of candidates.slice(0, 6)) {
|
||||
el.scrollTop += 1000;
|
||||
}
|
||||
}
|
||||
"""
|
||||
)
|
||||
page.wait_for_timeout(1800)
|
||||
|
||||
count = len(comments_by_id)
|
||||
if count == last_count:
|
||||
idle += 1
|
||||
else:
|
||||
log(f"Loaded comments: {count}")
|
||||
idle = 0
|
||||
last_count = count
|
||||
if idle >= idle_rounds:
|
||||
log(f"No new comments after {idle_rounds} rounds, stopping scroll.")
|
||||
break
|
||||
|
||||
log(f"Scroll finished after {index + 1} rounds.")
|
||||
|
||||
|
||||
def with_query_params(url: str, updates: dict[str, Any]) -> str:
|
||||
parts = urlsplit(url)
|
||||
params = dict(parse_qsl(parts.query, keep_blank_values=True))
|
||||
for key, value in updates.items():
|
||||
params[key] = str(value)
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(params), parts.fragment))
|
||||
|
||||
|
||||
def fetch_missing_top_comments(
|
||||
page: Page,
|
||||
source_url: str,
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
stats: dict[str, Any],
|
||||
) -> None:
|
||||
"""After scrolling, proactively paginate remaining top-level comments via page.evaluate."""
|
||||
template = stats.get("comment_url_template")
|
||||
if not template:
|
||||
log("No top-level comment API template captured; skipped direct fetch.")
|
||||
return
|
||||
|
||||
api_total = stats.get("api_total")
|
||||
# api_total may be None if the first XHR response didn't include it;
|
||||
# fall back to note_metrics.comment_count from the page
|
||||
if not api_total:
|
||||
metrics = stats.get("note_metrics") or {}
|
||||
api_total = metrics.get("comment_count")
|
||||
if api_total and len(comments_by_id) >= int(api_total):
|
||||
return
|
||||
|
||||
last_cursor = stats.get("last_cursor")
|
||||
# Always attempt pagination if we have a cursor — the scroll may have
|
||||
# stopped early even though has_more was true in an earlier response.
|
||||
if not last_cursor:
|
||||
return
|
||||
|
||||
log(f"Top-level comments: collected {len(comments_by_id)} / api_total {api_total}; fetching remainder.")
|
||||
fetched = 0
|
||||
cursor = last_cursor
|
||||
for _ in range(100):
|
||||
top_url = with_query_params(template, {"cursor": cursor})
|
||||
try:
|
||||
payload = page.evaluate(
|
||||
"""
|
||||
async (url) => {
|
||||
const res = await fetch(url, { credentials: 'include' });
|
||||
return await res.json();
|
||||
}
|
||||
""",
|
||||
top_url,
|
||||
)
|
||||
except Exception:
|
||||
break
|
||||
if not isinstance(payload, dict):
|
||||
break
|
||||
payload["__source_url"] = top_url
|
||||
before = len(comments_by_id)
|
||||
collect_from_payload(payload, source_url, comments_by_id, stats)
|
||||
fetched += max(0, len(comments_by_id) - before)
|
||||
data = payload.get("data") if isinstance(payload.get("data"), dict) else payload
|
||||
next_has_more = data.get("has_more")
|
||||
next_cursor = data.get("cursor")
|
||||
if not next_has_more or next_cursor in (None, cursor):
|
||||
break
|
||||
cursor = next_cursor
|
||||
page.wait_for_timeout(350)
|
||||
|
||||
log(f"Top-level direct fetch: {fetched} new comments; total now: {len(comments_by_id)}")
|
||||
|
||||
|
||||
def fetch_missing_replies(
|
||||
page: Page,
|
||||
source_url: str,
|
||||
comments_by_id: dict[str, dict[str, Any]],
|
||||
stats: dict[str, Any],
|
||||
) -> None:
|
||||
template = stats.get("sub_url_template")
|
||||
if not template:
|
||||
log("No sub-comment API template captured; skipped direct reply fetch.")
|
||||
return
|
||||
|
||||
parents = [
|
||||
row for row in list(comments_by_id.values())
|
||||
if row.get("level") == "comment" and int(row.get("reply_count") or 0) > 0
|
||||
]
|
||||
if not parents:
|
||||
return
|
||||
|
||||
log(f"Fetching replies directly for {len(parents)} comments with replies.")
|
||||
fetched = 0
|
||||
for row in parents:
|
||||
parent_id = str(row.get("comment_id") or "")
|
||||
if not parent_id:
|
||||
continue
|
||||
cursor = ""
|
||||
for _ in range(30):
|
||||
reply_url = with_query_params(template, {"root_comment_id": parent_id, "comment_id": parent_id, "cursor": cursor})
|
||||
try:
|
||||
payload = page.evaluate(
|
||||
"""
|
||||
async (url) => {
|
||||
const res = await fetch(url, { credentials: 'include' });
|
||||
return await res.json();
|
||||
}
|
||||
""",
|
||||
reply_url,
|
||||
)
|
||||
except Exception:
|
||||
break
|
||||
if not isinstance(payload, dict):
|
||||
break
|
||||
payload["__source_url"] = reply_url
|
||||
before = len(comments_by_id)
|
||||
collect_from_payload(payload, source_url, comments_by_id, stats)
|
||||
fetched += max(0, len(comments_by_id) - before)
|
||||
data = payload.get("data") if isinstance(payload.get("data"), dict) else payload
|
||||
has_more = data.get("has_more")
|
||||
next_cursor = data.get("cursor")
|
||||
if not has_more or next_cursor in (None, cursor):
|
||||
break
|
||||
cursor = next_cursor
|
||||
page.wait_for_timeout(350)
|
||||
|
||||
log(f"Direct replies fetched/merged: {fetched}; total comments now: {len(comments_by_id)}")
|
||||
|
||||
|
||||
def extract_note_metrics(page: Page) -> dict[str, Any]:
|
||||
try:
|
||||
metrics = page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const compactNumber = (text) => {
|
||||
text = String(text || '').replace(/,/g, '').trim();
|
||||
const match = text.match(/(\d+(?:\.\d+)?)(\s*[\u4e07wW])?/);
|
||||
if (!match) return '';
|
||||
const value = parseFloat(match[1]);
|
||||
if (!Number.isFinite(value)) return '';
|
||||
return match[2] ? Math.round(value * 10000) : Math.round(value);
|
||||
};
|
||||
const metricFromState = () => {
|
||||
const noteId = location.href.match(/(?:explore|discovery\/item|profile\/[^/]+)\/(\w+)/)?.[1];
|
||||
const detailMap = window.__INITIAL_STATE__?.note?.noteDetailMap || {};
|
||||
const detail = noteId ? detailMap[noteId] : Object.values(detailMap)[0];
|
||||
const note = detail?.note || {};
|
||||
const info = note.interactInfo;
|
||||
if (!info) return null;
|
||||
return {
|
||||
title: note.title || note.desc || '',
|
||||
like_count: compactNumber(info.likedCount),
|
||||
favorite_count: compactNumber(info.collectedCount),
|
||||
comment_count: compactNumber(info.commentCount),
|
||||
share_count: '',
|
||||
};
|
||||
};
|
||||
const stateMetrics = metricFromState();
|
||||
if (stateMetrics) return stateMetrics;
|
||||
|
||||
const visible = (el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(el);
|
||||
return r.width > 0 && r.height > 0 &&
|
||||
r.bottom > 0 && r.top < window.innerHeight &&
|
||||
style.visibility !== 'hidden' &&
|
||||
style.display !== 'none' &&
|
||||
style.opacity !== '0';
|
||||
};
|
||||
const center = (r) => ({ x: (r.left + r.right) / 2, y: (r.top + r.bottom) / 2 });
|
||||
const dedupe = (items) => {
|
||||
const kept = [];
|
||||
for (const item of items.sort((a, b) => a.r.left - b.r.left || a.r.top - b.r.top)) {
|
||||
const c = center(item.r);
|
||||
const duplicate = kept.some((prev) => {
|
||||
const pc = center(prev.r);
|
||||
return item.value === prev.value &&
|
||||
Math.abs(c.x - pc.x) < 80 &&
|
||||
Math.abs(c.y - pc.y) < 40;
|
||||
});
|
||||
if (!duplicate) kept.push(item);
|
||||
}
|
||||
return kept;
|
||||
};
|
||||
const textOf = (el) => (el.innerText || el.textContent || '').trim();
|
||||
const nodes = [...document.querySelectorAll('button, div, span, p')]
|
||||
.filter(visible)
|
||||
.map((el) => ({ el, text: textOf(el), r: el.getBoundingClientRect() }))
|
||||
.filter(({ text }) => text && text.length <= 120);
|
||||
|
||||
const commentText = nodes.find(({ text }) =>
|
||||
/\u5171\s*\d+(?:\.\d+)?\s*[\u4e07wW]?\s*\u6761\u8bc4\u8bba/.test(text)
|
||||
)?.text || '';
|
||||
const commentByText = compactNumber(commentText);
|
||||
const bottomNums = dedupe(nodes
|
||||
.map((item) => ({ ...item, value: compactNumber(item.text) }))
|
||||
.filter(({ text, r, value }) => /^(\d+(?:\.\d+)?\s*[\u4e07wW]?)$/.test(text) &&
|
||||
value !== '' &&
|
||||
r.top > window.innerHeight - 140 &&
|
||||
r.left > window.innerWidth * 0.55 &&
|
||||
r.width < 180 &&
|
||||
r.height < 110))
|
||||
.map(({ value }) => value);
|
||||
|
||||
return {
|
||||
like_count: bottomNums.length >= 1 ? bottomNums[0] : '',
|
||||
favorite_count: bottomNums.length >= 2 ? bottomNums[1] : '',
|
||||
comment_count: commentByText || (bottomNums.length >= 3 ? bottomNums[2] : ''),
|
||||
share_count: '',
|
||||
};
|
||||
}
|
||||
"""
|
||||
)
|
||||
return metrics if isinstance(metrics, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def scrape_comments(url: str, login_timeout: int, max_scrolls: int, idle_rounds: int, headless: bool) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
result: dict[str, Any] = {"comments": [], "stats": {}, "final_url": ""}
|
||||
|
||||
def action(page: Page) -> None:
|
||||
comments_by_id, stats = setup_response_capture(page, url)
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=90000)
|
||||
page.wait_for_timeout(6000)
|
||||
maybe_wait_for_login(page, login_timeout)
|
||||
dismiss_popups(page)
|
||||
save_state(page)
|
||||
|
||||
scroll_comments(page, max_scrolls, idle_rounds, comments_by_id)
|
||||
# Extract metrics early so fetch_missing_top_comments can use comment_count as fallback
|
||||
stats["note_metrics"] = extract_note_metrics(page)
|
||||
result["title"] = stats["note_metrics"].get("title", "")
|
||||
fetch_missing_top_comments(page, url, comments_by_id, stats)
|
||||
fetch_missing_replies(page, url, comments_by_id, stats)
|
||||
page.wait_for_timeout(1000)
|
||||
save_state(page)
|
||||
|
||||
result["comments"] = list(comments_by_id.values())
|
||||
result["stats"] = stats
|
||||
result["stats"]["logged_in"] = is_logged_in(page)
|
||||
result["final_url"] = page.url
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=headless,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
session.fetch(url, page_action=action, wait=1000)
|
||||
|
||||
return result["comments"], result
|
||||
|
||||
|
||||
def write_outputs(url: str, comments: list[dict[str, Any]], result: dict[str, Any]) -> tuple[Path, Path]:
|
||||
NOTES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
note_id = note_id_from_url(url)
|
||||
name = f"xiaohongshu_comments_{note_id}" if note_id else safe_filename(url)
|
||||
json_path = NOTES_DIR / f"{name}.json"
|
||||
csv_path = NOTES_DIR / f"{name}.csv"
|
||||
if not comments and json_path.exists() and csv_path.exists():
|
||||
log(f"No comments scraped; keeping existing output files: {json_path}, {csv_path}")
|
||||
return json_path, csv_path
|
||||
|
||||
stats = result.get("stats", {})
|
||||
public_stats = {key: value for key, value in stats.items() if not key.endswith("_template")}
|
||||
payload = {
|
||||
"source_url": url,
|
||||
"title": result.get("title", ""),
|
||||
"scraped_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"count": len(comments),
|
||||
"stats": public_stats,
|
||||
"comments": comments,
|
||||
}
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
fieldnames = [
|
||||
"source_url",
|
||||
"level",
|
||||
"comment_id",
|
||||
"reply_to_comment_id",
|
||||
"nickname",
|
||||
"user_id",
|
||||
"content",
|
||||
"like_count",
|
||||
"reply_count",
|
||||
"ip_location",
|
||||
"created_at",
|
||||
"raw_create_time",
|
||||
]
|
||||
with csv_path.open("w", newline="", encoding="utf-8-sig") as file:
|
||||
writer = csv.DictWriter(file, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(comments)
|
||||
|
||||
return json_path, csv_path
|
||||
|
||||
|
||||
def login_only(login_timeout: int) -> int:
|
||||
"""只触发登录(headed),登录完成立刻退出。给 relogin_xiaohongshu.py 用。"""
|
||||
def action(page: Page) -> None:
|
||||
page.goto("https://www.xiaohongshu.com/", wait_until="domcontentloaded", timeout=90000)
|
||||
page.wait_for_timeout(2500)
|
||||
# 主动点登录按钮触发二维码
|
||||
try:
|
||||
login_btn = page.get_by_text("登录", exact=True).first
|
||||
if login_btn.count():
|
||||
login_btn.click(timeout=3000)
|
||||
page.wait_for_timeout(1500)
|
||||
except Exception:
|
||||
pass
|
||||
maybe_wait_for_login(page, login_timeout)
|
||||
save_state(page)
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=False,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=90000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
session.fetch("https://www.xiaohongshu.com/", page_action=action, wait=1000)
|
||||
log("[login-only] 完成")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Scrape Xiaohongshu note comments with a real browser.")
|
||||
parser.add_argument("url", help="Xiaohongshu explore/note URL (传 'login' 配合 --login-only)")
|
||||
parser.add_argument("--login-timeout", type=int, default=300, help="等待扫码/登录秒数")
|
||||
parser.add_argument("--max-scrolls", type=int, default=120, help="最多滚动加载次数")
|
||||
parser.add_argument("--idle-rounds", type=int, default=8, help="连续几轮无新评论后停止")
|
||||
parser.add_argument("--headless", action="store_true", help="无头模式；首次登录不要使用")
|
||||
parser.add_argument("--login-only", action="store_true",
|
||||
help="只触发登录(headed)然后退出,给 relogin_xiaohongshu 用")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.login_only:
|
||||
return login_only(args.login_timeout)
|
||||
|
||||
comments, result = scrape_comments(args.url, args.login_timeout, args.max_scrolls, args.idle_rounds, args.headless)
|
||||
json_path, csv_path = write_outputs(args.url, comments, result)
|
||||
log(f"Final URL: {result.get('final_url', '')}")
|
||||
log(f"API hits: {result.get('stats', {}).get('api_hits', 0)}")
|
||||
log(f"Scraped {len(comments)} comments")
|
||||
log(f"JSON: {json_path}")
|
||||
log(f"CSV: {csv_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
@@ -0,0 +1,479 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.content_marketing.runtime.runtime_paths import PATHS
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Page, TimeoutError as PlaywrightTimeoutError
|
||||
from scrapling.fetchers import DynamicSession
|
||||
|
||||
|
||||
MARKET_URL = "https://www.xingtu.cn/ad/creator/market"
|
||||
DETAIL_URL_MARK = "/ad/creator/author-homepage/"
|
||||
|
||||
BASE_DIR = PATHS.module_root
|
||||
DATA_DIR = PATHS.raw_root
|
||||
COOKIE_FILE = PATHS.browser_cookie_file
|
||||
PROFILE_DIR = PATHS.browser_profile_dir
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[{datetime.now().strftime('%H:%M:%S')}] {message}", flush=True)
|
||||
|
||||
|
||||
def load_cookies() -> list[dict[str, Any]]:
|
||||
if not COOKIE_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except json.JSONDecodeError:
|
||||
log(f"Cookie file is invalid, ignoring: {COOKIE_FILE}")
|
||||
return []
|
||||
|
||||
|
||||
def restore_cookies(page: Page) -> None:
|
||||
cookies = load_cookies()
|
||||
if not cookies:
|
||||
return
|
||||
try:
|
||||
page.context.add_cookies(cookies)
|
||||
log(f"Loaded cookies: {COOKIE_FILE}")
|
||||
except Exception as exc:
|
||||
log(f"Failed to load cookies, will continue with browser profile: {exc}")
|
||||
|
||||
|
||||
def save_state(page: Page) -> None:
|
||||
DATA_DIR.mkdir(exist_ok=True)
|
||||
cookies = page.context.cookies()
|
||||
COOKIE_FILE.write_text(
|
||||
json.dumps(cookies, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
storage_state = page.context.storage_state()
|
||||
PATHS.browser_storage_state_file.write_text(
|
||||
json.dumps(storage_state, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
log(f"Saved cookies: {COOKIE_FILE}")
|
||||
|
||||
|
||||
def is_logged_in(page: Page) -> bool:
|
||||
if "/ad/creator/market" not in page.url:
|
||||
return False
|
||||
for text in ("内容找人", "昵称找人", "达人信息", "达人清单", "发布任务"):
|
||||
try:
|
||||
page.get_by_text(text, exact=False).first.wait_for(timeout=1200)
|
||||
return True
|
||||
except PlaywrightTimeoutError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def wait_for_scan_login(page: Page, timeout_seconds: int) -> None:
|
||||
log("Not logged in. Please scan/login in the opened Chrome window, then make sure it enters the creator market page.")
|
||||
deadline = time.time() + timeout_seconds
|
||||
while time.time() < deadline:
|
||||
if "redirect_uri=/ad/creator/market" in page.url:
|
||||
try:
|
||||
page.goto(MARKET_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_timeout(2000)
|
||||
except Exception:
|
||||
pass
|
||||
if is_logged_in(page):
|
||||
save_state(page)
|
||||
return
|
||||
page.wait_for_timeout(2000)
|
||||
raise TimeoutError(f"Login timed out after {timeout_seconds} seconds.")
|
||||
|
||||
|
||||
def ensure_login(page: Page, login_timeout: int) -> None:
|
||||
page.goto(MARKET_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_timeout(3000)
|
||||
if is_logged_in(page):
|
||||
log("Already logged in, using saved browser session.")
|
||||
save_state(page)
|
||||
return
|
||||
|
||||
wait_for_scan_login(page, login_timeout)
|
||||
page.goto(MARKET_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
page.wait_for_timeout(3000)
|
||||
if not is_logged_in(page):
|
||||
wait_for_scan_login(page, login_timeout)
|
||||
|
||||
|
||||
def dismiss_popups(page: Page) -> None:
|
||||
for text in ("跳过", "我知道了", "稍后再说"):
|
||||
click_by_text(page, text, exact=True, timeout=1000)
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const close = [...document.querySelectorAll('button, span, div')]
|
||||
.filter((el) => {
|
||||
const text = (el.innerText || '').trim();
|
||||
const r = el.getBoundingClientRect();
|
||||
return (text === '×' || text === 'x' || text === 'X') &&
|
||||
r.width <= 40 && r.height <= 40 &&
|
||||
r.left > window.innerWidth * 0.45 &&
|
||||
r.top < window.innerHeight * 0.45;
|
||||
})[0];
|
||||
close?.click();
|
||||
}
|
||||
"""
|
||||
)
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
|
||||
def click_by_text(page: Page, text: str, exact: bool = True, timeout: int = 5000) -> bool:
|
||||
try:
|
||||
locator = page.get_by_text(text, exact=exact).first
|
||||
if locator.count():
|
||||
locator.click(timeout=timeout)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def select_nickname_search(page: Page) -> None:
|
||||
log("Switching to nickname search.")
|
||||
if click_by_text(page, "昵称找人", exact=True, timeout=5000):
|
||||
page.wait_for_timeout(800)
|
||||
return
|
||||
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const target = [...document.querySelectorAll('button, div, span')]
|
||||
.find((el) => (el.innerText || '').trim() === '昵称找人');
|
||||
target?.click();
|
||||
}
|
||||
"""
|
||||
)
|
||||
page.wait_for_timeout(800)
|
||||
|
||||
|
||||
def search_creator(page: Page, creator_name: str) -> None:
|
||||
dismiss_popups(page)
|
||||
select_nickname_search(page)
|
||||
input_locator = page.locator("input:visible").first
|
||||
input_locator.wait_for(state="visible", timeout=30000)
|
||||
input_locator.click()
|
||||
input_locator.fill(creator_name)
|
||||
input_locator.press("Enter")
|
||||
|
||||
try:
|
||||
page.wait_for_timeout(2000)
|
||||
wait_for_visible_creator_result(page, creator_name, timeout_ms=8000)
|
||||
except PlaywrightTimeoutError:
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const input = [...document.querySelectorAll('input')].find((el) => el.value);
|
||||
const box = input?.closest('div');
|
||||
const candidates = [...document.querySelectorAll('button, [role="button"], div, span')]
|
||||
.filter((el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
return r.width >= 24 && r.width <= 80 && r.height >= 24 && r.height <= 80;
|
||||
});
|
||||
const near = candidates.find((el) => {
|
||||
const er = el.getBoundingClientRect();
|
||||
const br = box?.getBoundingClientRect();
|
||||
return br && er.left > br.right - 30 && Math.abs(er.top - br.top) < 60;
|
||||
});
|
||||
near?.click();
|
||||
}
|
||||
"""
|
||||
)
|
||||
wait_for_visible_creator_result(page, creator_name, timeout_ms=30000)
|
||||
|
||||
page.wait_for_timeout(2500)
|
||||
|
||||
|
||||
def wait_for_visible_creator_result(page: Page, creator_name: str, timeout_ms: int) -> None:
|
||||
deadline = time.time() + timeout_ms / 1000
|
||||
while time.time() < deadline:
|
||||
found = page.evaluate(
|
||||
"""
|
||||
(name) => {
|
||||
const visible = (el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(el);
|
||||
return r.width > 0 && r.height > 0 &&
|
||||
r.bottom > 0 && r.top < window.innerHeight &&
|
||||
style.visibility !== 'hidden' &&
|
||||
style.display !== 'none' &&
|
||||
style.opacity !== '0';
|
||||
};
|
||||
return [...document.querySelectorAll('tr, [class*="row"], [class*="table"], [class*="item"], div')]
|
||||
.some((el) => visible(el) && (el.innerText || '').includes(name) && /达人|粉丝|播放|报价|下单|内容主题/.test(el.innerText || ''));
|
||||
}
|
||||
""",
|
||||
creator_name,
|
||||
)
|
||||
if found:
|
||||
return
|
||||
page.wait_for_timeout(500)
|
||||
raise PlaywrightTimeoutError(f"Timed out waiting for visible creator result: {creator_name}")
|
||||
|
||||
|
||||
def open_creator_detail(page: Page, creator_name: str) -> Page:
|
||||
log(f"Opening creator detail: {creator_name}")
|
||||
pages_before = list(page.context.pages)
|
||||
clicked = page.evaluate(
|
||||
"""
|
||||
(name) => {
|
||||
const textHasName = (el) => (el.innerText || '').includes(name);
|
||||
const rows = [...document.querySelectorAll('tr, [class*="row"], [class*="table"], [class*="item"]')]
|
||||
.filter(textHasName);
|
||||
const exactRow = rows.find((el) => {
|
||||
const lines = (el.innerText || '').split('\\n').map((line) => line.trim()).filter(Boolean);
|
||||
return lines.includes(name);
|
||||
});
|
||||
const row = exactRow || rows[0];
|
||||
if (!row) return false;
|
||||
const target = row.querySelector('img, a, [class*="avatar"], [class*="name"]') || row;
|
||||
target.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
target.click();
|
||||
return true;
|
||||
}
|
||||
""",
|
||||
creator_name,
|
||||
)
|
||||
if not clicked:
|
||||
page.get_by_text(creator_name, exact=True).first.click(timeout=10000)
|
||||
|
||||
detail_page = wait_for_detail_page(page, pages_before, timeout_ms=30000)
|
||||
if detail_page is None:
|
||||
page.get_by_text(creator_name, exact=False).first.click(timeout=10000)
|
||||
detail_page = wait_for_detail_page(page, pages_before, timeout_ms=30000)
|
||||
|
||||
if detail_page is None:
|
||||
raise TimeoutError("Clicked creator result, but no author-homepage page was detected.")
|
||||
|
||||
detail_page.bring_to_front()
|
||||
detail_page.wait_for_load_state("domcontentloaded")
|
||||
page.wait_for_timeout(1000)
|
||||
return detail_page
|
||||
|
||||
|
||||
def wait_for_detail_page(page: Page, pages_before: list[Page], timeout_ms: int) -> Page | None:
|
||||
deadline = time.time() + timeout_ms / 1000
|
||||
known = set(pages_before)
|
||||
while time.time() < deadline:
|
||||
all_pages = list(page.context.pages)
|
||||
candidates = [p for p in all_pages if p not in known] + [page] + all_pages
|
||||
for candidate in candidates:
|
||||
try:
|
||||
if DETAIL_URL_MARK in candidate.url or "/ad/creator/author-homepage/" in candidate.url:
|
||||
return candidate
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(500)
|
||||
return None
|
||||
|
||||
|
||||
def open_creation_ability(page: Page) -> None:
|
||||
log("Opening creation ability tab.")
|
||||
try:
|
||||
page.get_by_text("创作能力", exact=True).first.click(timeout=10000)
|
||||
except Exception:
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const target = [...document.querySelectorAll('button, div, span, a')]
|
||||
.find((el) => (el.innerText || '').trim() === '创作能力');
|
||||
target?.click();
|
||||
}
|
||||
"""
|
||||
)
|
||||
page.wait_for_timeout(3000)
|
||||
try:
|
||||
all_video = page.get_by_text("全部视频", exact=False).first
|
||||
all_video.wait_for(timeout=30000)
|
||||
all_video.scroll_into_view_if_needed()
|
||||
page.wait_for_timeout(1200)
|
||||
page.mouse.wheel(0, 600)
|
||||
page.wait_for_timeout(1500)
|
||||
except PlaywrightTimeoutError:
|
||||
pass
|
||||
|
||||
|
||||
def parse_video_cards(page: Page) -> list[dict[str, str]]:
|
||||
return page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim();
|
||||
|
||||
const valueAfter = (lines, label) => {
|
||||
const index = lines.findIndex((line) => line === label || line.includes(label));
|
||||
if (index < 0) return '';
|
||||
const sameLine = lines[index].replace(label, '').trim();
|
||||
if (sameLine) return sameLine;
|
||||
return lines[index + 1] || '';
|
||||
};
|
||||
|
||||
const parseCard = (card) => {
|
||||
const lines = (card.innerText || '')
|
||||
.split('\\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
const text = lines.join('\\n');
|
||||
if (!/播放量/.test(text) || !/点赞量/.test(text) || !/评论量/.test(text)) return null;
|
||||
|
||||
const firstMetric = lines.findIndex((line) => /播放量/.test(line));
|
||||
const titleCandidates = lines.slice(0, firstMetric >= 0 ? firstMetric : lines.length).filter((line) => {
|
||||
if (/^(全部视频|全部|个人视频|星图视频|播放量|点赞量|评论量)$/.test(line)) return false;
|
||||
if (/请输入关键词搜索|开始日期|结束日期|展开/.test(line)) return false;
|
||||
if (/^[\\d,.wW万]+$/.test(line)) return false;
|
||||
return line.length >= 2;
|
||||
});
|
||||
const title = titleCandidates.slice(-2).join(' ').trim();
|
||||
|
||||
return {
|
||||
title: clean(title),
|
||||
play_count: clean(valueAfter(lines, '播放量')),
|
||||
like_count: clean(valueAfter(lines, '点赞量')),
|
||||
comment_count: clean(valueAfter(lines, '评论量')),
|
||||
};
|
||||
};
|
||||
|
||||
const cards = [...document.querySelectorAll('div')].filter((el) => {
|
||||
const text = el.innerText || '';
|
||||
const rect = el.getBoundingClientRect();
|
||||
return rect.width >= 140 &&
|
||||
rect.width <= 340 &&
|
||||
rect.height >= 260 &&
|
||||
rect.height <= 620 &&
|
||||
/播放量/.test(text) &&
|
||||
/点赞量/.test(text) &&
|
||||
/评论量/.test(text) &&
|
||||
el.querySelector('img');
|
||||
});
|
||||
|
||||
const parsed = cards.map(parseCard).filter(Boolean);
|
||||
const seen = new Set();
|
||||
return parsed.filter((item) => {
|
||||
const key = [item.title, item.play_count, item.like_count, item.comment_count].join('|');
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return item.title && (item.play_count || item.like_count || item.comment_count);
|
||||
});
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def write_outputs(creator_name: str, videos: list[dict[str, str]]) -> tuple[Path, Path]:
|
||||
DATA_DIR.mkdir(exist_ok=True)
|
||||
safe_name = re.sub(r'[<>:"/\\\\|?*\\s]+', "_", creator_name).strip("_") or "creator"
|
||||
json_path = DATA_DIR / f"xingtu_{safe_name}_videos.json"
|
||||
csv_path = DATA_DIR / f"xingtu_{safe_name}_videos.csv"
|
||||
|
||||
payload = {
|
||||
"creator": creator_name,
|
||||
"scraped_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"count": len(videos),
|
||||
"videos": videos,
|
||||
}
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
with csv_path.open("w", newline="", encoding="utf-8-sig") as file:
|
||||
writer = csv.DictWriter(
|
||||
file,
|
||||
fieldnames=["title", "play_count", "like_count", "comment_count"],
|
||||
)
|
||||
writer.writeheader()
|
||||
writer.writerows(videos)
|
||||
|
||||
return json_path, csv_path
|
||||
|
||||
|
||||
def load_names(cli_names: list[str], names_file: str | None) -> list[str]:
|
||||
names: list[str] = []
|
||||
names.extend(name.strip() for name in cli_names if name.strip())
|
||||
|
||||
if names_file:
|
||||
path = Path(names_file)
|
||||
for line in path.read_text(encoding="utf-8-sig").splitlines():
|
||||
name = line.strip()
|
||||
if name and not name.startswith("#"):
|
||||
names.append(name)
|
||||
|
||||
deduped: list[str] = []
|
||||
seen = set()
|
||||
for name in names:
|
||||
if name not in seen:
|
||||
seen.add(name)
|
||||
deduped.append(name)
|
||||
return deduped
|
||||
|
||||
|
||||
def scrape(creator_name: str, login_timeout: int, headless: bool) -> list[dict[str, str]]:
|
||||
result: dict[str, Any] = {"videos": [], "detail_url": ""}
|
||||
|
||||
def action(page: Page) -> None:
|
||||
ensure_login(page, login_timeout)
|
||||
search_creator(page, creator_name)
|
||||
detail_page = open_creator_detail(page, creator_name)
|
||||
open_creation_ability(detail_page)
|
||||
result["detail_url"] = detail_page.url
|
||||
result["videos"] = parse_video_cards(detail_page)
|
||||
save_state(detail_page)
|
||||
|
||||
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with DynamicSession(
|
||||
headless=headless,
|
||||
real_chrome=True,
|
||||
user_data_dir=str(PROFILE_DIR),
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
timeout=60000,
|
||||
network_idle=False,
|
||||
disable_resources=False,
|
||||
google_search=False,
|
||||
page_setup=restore_cookies,
|
||||
max_pages=1,
|
||||
) as session:
|
||||
session.fetch(MARKET_URL, page_action=action, wait=1000)
|
||||
|
||||
log(f"Detail URL: {result.get('detail_url', '')}")
|
||||
return result["videos"]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Scrape Xingtu creator video metrics with Scrapling DynamicSession.",
|
||||
)
|
||||
parser.add_argument("creator_names", nargs="*", help="达人昵称，可传多个，例如：达人A 达人B")
|
||||
parser.add_argument("--names-file", help="批量达人名单文件，一行一个昵称")
|
||||
parser.add_argument("--login-timeout", type=int, default=300, help="首次扫码登录等待秒数")
|
||||
parser.add_argument("--headless", action="store_true", help="无头模式；首次登录不要使用")
|
||||
args = parser.parse_args()
|
||||
|
||||
creator_names = load_names(args.creator_names, args.names_file)
|
||||
if not creator_names:
|
||||
parser.error("Please provide at least one creator name or --names-file.")
|
||||
|
||||
for index, creator_name in enumerate(creator_names, start=1):
|
||||
log(f"=== Creator {index}/{len(creator_names)}: {creator_name} ===")
|
||||
videos = scrape(creator_name, args.login_timeout, args.headless)
|
||||
json_path, csv_path = write_outputs(creator_name, videos)
|
||||
log(f"Scraped {len(videos)} videos")
|
||||
log(f"JSON: {json_path}")
|
||||
log(f"CSV: {csv_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted.", file=sys.stderr)
|
||||
raise SystemExit(130)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,20 @@
|
||||
"""Stable contract implemented by every business module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from gyxx_flow.workflow import WorkflowDefinition
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class BusinessModule(Protocol):
|
||||
"""A business capability that contributes workflows to the application."""
|
||||
|
||||
module_id: str
|
||||
|
||||
def workflow_definitions(self) -> tuple[WorkflowDefinition, ...]:
|
||||
"""Return immutable workflow definitions owned by this module."""
|
||||
|
||||
|
||||
__all__ = ["BusinessModule"]
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Product, ERP, analysis, main-image, and alert workflows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
from gyxx_flow.adapters.native import DeferredModuleCommandStep, ModuleCommandFactory
|
||||
from gyxx_flow.workflow import StepDefinition, WorkflowDefinition
|
||||
|
||||
PRODUCT_SCHEDULED_WORKFLOW_IDS = (
|
||||
"product.persona.daily",
|
||||
"product.daily",
|
||||
"product.alert.daily",
|
||||
"product.import.daily",
|
||||
"product.style_analysis.interval",
|
||||
"product.main_image.jd.weekly",
|
||||
"product.main_image.weekly",
|
||||
)
|
||||
PRODUCT_MANUAL_WORKFLOW_IDS = (
|
||||
"product.backfill",
|
||||
"product.market_rank",
|
||||
"product.review_collection",
|
||||
)
|
||||
PRODUCT_WORKFLOW_IDS = (
|
||||
*PRODUCT_SCHEDULED_WORKFLOW_IDS,
|
||||
*PRODUCT_MANUAL_WORKFLOW_IDS,
|
||||
)
|
||||
PRODUCT_TIMEOUT_SECONDS = 6 * 60 * 60
|
||||
PRODUCT_RESOURCE = "module:product_commerce"
|
||||
|
||||
|
||||
class ProductCommerceModule:
|
||||
"""Product-commerce business module composition boundary."""
|
||||
|
||||
module_id = "product_commerce"
|
||||
|
||||
def __init__(self, definitions: Iterable[WorkflowDefinition] = ()) -> None:
|
||||
self._definitions = tuple(definitions)
|
||||
|
||||
@classmethod
|
||||
def from_catalog(
|
||||
cls,
|
||||
catalog: Any,
|
||||
*,
|
||||
command_factory: ModuleCommandFactory | None = None,
|
||||
) -> "ProductCommerceModule":
|
||||
"""Build all available product workflows, excluding unavailable metadata."""
|
||||
|
||||
entries = {
|
||||
entry.workflow_id: entry
|
||||
for entry in catalog.workflows
|
||||
if entry.module == cls.module_id and entry.trigger in {"scheduled", "manual"}
|
||||
}
|
||||
expected = set(PRODUCT_WORKFLOW_IDS)
|
||||
if set(entries) != expected:
|
||||
raise ValueError(
|
||||
"product catalog workflows do not match the migration contract: "
|
||||
f"expected={sorted(expected)}, actual={sorted(entries)}"
|
||||
)
|
||||
definitions: list[WorkflowDefinition] = []
|
||||
for workflow_id in PRODUCT_WORKFLOW_IDS:
|
||||
entry = entries[workflow_id]
|
||||
action = (
|
||||
DeferredModuleCommandStep(entry)
|
||||
if command_factory is None
|
||||
else DeferredModuleCommandStep(entry, command_factory=command_factory)
|
||||
)
|
||||
definitions.append(
|
||||
WorkflowDefinition(
|
||||
workflow_id,
|
||||
(
|
||||
StepDefinition(
|
||||
"module_execute",
|
||||
action,
|
||||
timeout_seconds=PRODUCT_TIMEOUT_SECONDS,
|
||||
max_attempts=1,
|
||||
resources=(PRODUCT_RESOURCE,),
|
||||
production_sink=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
return cls(definitions)
|
||||
|
||||
def workflow_definitions(self) -> tuple[WorkflowDefinition, ...]:
|
||||
return self._definitions
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PRODUCT_RESOURCE",
|
||||
"PRODUCT_TIMEOUT_SECONDS",
|
||||
"PRODUCT_MANUAL_WORKFLOW_IDS",
|
||||
"PRODUCT_SCHEDULED_WORKFLOW_IDS",
|
||||
"PRODUCT_WORKFLOW_IDS",
|
||||
"ProductCommerceModule",
|
||||
]
|
||||
@@ -0,0 +1,19 @@
|
||||
# ── PostgreSQL 连接 (data_hub 库) ───────────────────────────────────
|
||||
# 从 .env 文件加载；不要把真实密码提交到 git
|
||||
PG_HOST=8.148.185.119
|
||||
PG_PORT=5432
|
||||
PG_DB=data_hub
|
||||
PG_USER=data_hub
|
||||
PG_PASSWORD=
|
||||
|
||||
# ── 京东采集账号 (jd_self_inventory_sales_collector 等) ─────────────────
|
||||
JD_SELF_SHOP=光影自营研发
|
||||
JD_SELF_PASSWORD=
|
||||
|
||||
# ── 淘宝光合平台视频上传账号 ─────────────────────────────────────────
|
||||
GUANGHE_USERNAME=
|
||||
GUANGHE_PASSWORD=
|
||||
GUANGHE_USER_DATA_DIR=${GYXX_DATA_ROOT}\profiles\product_commerce\guanghe
|
||||
GUANGHE_LUGGAGE_USERNAME=
|
||||
GUANGHE_LUGGAGE_PASSWORD=
|
||||
GUANGHE_LUGGAGE_USER_DATA_DIR=${GYXX_DATA_ROOT}\profiles\product_commerce\guanghe-luggage
|
||||
@@ -0,0 +1,20 @@
|
||||
# Product-commerce source snapshot
|
||||
|
||||
This directory contains the reviewed working-tree snapshot of the legacy
|
||||
product-commerce runtime. It is now physically owned by `gyxx-flow`; no file in
|
||||
this directory needs the legacy checkout to exist.
|
||||
|
||||
The snapshot is not yet the public workflow entry point. Native packaging must
|
||||
first replace root-relative imports, subprocess script names, data-root
|
||||
assumptions, browser-profile defaults, and dynamic vendor-root injection.
|
||||
|
||||
`launchers_reference/` is provenance only. Those files document the old task
|
||||
arguments and must never be registered as final scheduled-task actions.
|
||||
|
||||
`runtime_tests/` preserves the 21 legacy regression tests without adding them to
|
||||
the current repository's automatic pytest collection. They will be moved and
|
||||
adapted as each native runtime slice becomes importable.
|
||||
|
||||
The authoritative source/target hashes, transformations, and six intentionally
|
||||
excluded diagnostic entries are recorded in
|
||||
`config/source-manifests/product_commerce.json`.
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Vendored product-commerce source snapshot awaiting native package refactoring.
|
||||
|
||||
Files in this package are copied into ``gyxx-flow`` so migration work no longer
|
||||
depends on the legacy checkout. They are provenance-preserving runtime sources;
|
||||
the public workflow registry is switched to native entries in a separate phase.
|
||||
"""
|
||||
@@ -0,0 +1,280 @@
|
||||
"""按日期优先汇总三平台 + ERP 的每日款式数据。
|
||||
|
||||
输出:
|
||||
data/<日期>/<款>/final_summary.json
|
||||
data/<日期>/<款>/final_summary.md
|
||||
|
||||
不计算转化率/加购率/退款率（这三个比率只在周汇总里算）。
|
||||
|
||||
每平台字段: sales / refund_orders / visitors / cart_users（来自平台层原始值）
|
||||
ERP 字段: style_codes / sales / refund_orders / source
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from erp_metric_overrides import DATA_ROOT, PLATFORM_DIRS, apply_erp_override, parse_number
|
||||
from runtime_paths import CURATED_DATA_ROOT, managed_data_path
|
||||
|
||||
|
||||
PLATFORMS = list(PLATFORM_DIRS)
|
||||
|
||||
PLATFORM_STYLE_FILENAME = {
|
||||
"jd": lambda style, date: f"{safe_name(style)}_{date}.json",
|
||||
"dy": lambda style, date: f"{safe_name(style)}_{date}.json",
|
||||
"tm": lambda style, date: f"业务款式_{date}.json",
|
||||
}
|
||||
|
||||
|
||||
def safe_name(value: str) -> str:
|
||||
return "".join("_" if ch in '\\/:*?"<>|' else ch for ch in str(value)).strip(" ._") or "未命名"
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def _extract_platform_metrics(platform: str, payload: dict) -> dict:
|
||||
"""从平台采集器产出的 <款>_<日期>.json 提取 4 个原始值。"""
|
||||
if platform == "jd":
|
||||
metrics = payload.get("指标汇总") or {}
|
||||
return {
|
||||
"visitors": parse_number(metrics.get("访客数")),
|
||||
"cart_users": parse_number(metrics.get("加购客户数") or metrics.get("加购人数")),
|
||||
"sales": parse_number(metrics.get("销量")),
|
||||
"refund_orders": parse_number(metrics.get("退款订单数")),
|
||||
}
|
||||
if platform == "dy":
|
||||
raw = payload.get("raw_totals") or {}
|
||||
platform_raw = payload.get("platform_raw_totals") or raw
|
||||
return {
|
||||
"visitors": parse_number(platform_raw.get("访客数") or payload.get("访客数")),
|
||||
"cart_users": parse_number(platform_raw.get("加购人数") or payload.get("加购人数")),
|
||||
"sales": parse_number(platform_raw.get("销量") or payload.get("销量")),
|
||||
"refund_orders": parse_number(platform_raw.get("退款订单数")),
|
||||
}
|
||||
if platform == "tm":
|
||||
style = payload.get("style") or payload
|
||||
return {
|
||||
"visitors": parse_number(style.get("访客量") or style.get("访客数")),
|
||||
"cart_users": parse_number(style.get("加购人数")),
|
||||
"sales": parse_number(style.get("销量")),
|
||||
"refund_orders": parse_number(style.get("退款订单数")),
|
||||
}
|
||||
raise ValueError(f"unknown platform: {platform}")
|
||||
|
||||
|
||||
def _extract_erp_metadata(payload: dict) -> dict:
|
||||
"""ERP 层只携带元信息（编码 + 源），销量/退货量已在各平台 sales/refund_orders 字段中（ERP 覆盖后）。"""
|
||||
return {
|
||||
"style_codes": payload.get("erp_style_codes") or [],
|
||||
}
|
||||
|
||||
|
||||
def collect_style_names(target_date: str, data_root: Path) -> list[str]:
|
||||
"""扫描三平台数据目录，得到 (日期, 款) 全部去重款式名。"""
|
||||
names: set[str] = set()
|
||||
for platform in PLATFORMS:
|
||||
base = data_root / PLATFORM_DIRS[platform]
|
||||
if not base.exists():
|
||||
continue
|
||||
for style_dir in base.glob(f"*_{target_date}"):
|
||||
if not style_dir.is_dir():
|
||||
continue
|
||||
for json_path in style_dir.glob("*.json"):
|
||||
if json_path.name in {"data.json"}:
|
||||
continue
|
||||
if json_path.name == "final_summary.json":
|
||||
continue
|
||||
if not (json_path.name.endswith(f"_{target_date}.json")
|
||||
or json_path.name.startswith("业务款式_")):
|
||||
continue
|
||||
try:
|
||||
payload = read_json(json_path)
|
||||
except Exception:
|
||||
continue
|
||||
if platform == "jd":
|
||||
name = payload.get("款式")
|
||||
elif platform == "dy":
|
||||
name = payload.get("style_name")
|
||||
else:
|
||||
style = payload.get("style") or {}
|
||||
name = style.get("款式") or payload.get("款式")
|
||||
if name:
|
||||
names.add(name)
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def load_platform_block(platform: str, style_name: str, target_date: str, data_root: Path) -> dict | None:
|
||||
filename = PLATFORM_STYLE_FILENAME[platform](style_name, target_date)
|
||||
style_dir = data_root / PLATFORM_DIRS[platform] / f"{safe_name(style_name)}_{target_date}"
|
||||
json_path = style_dir / filename
|
||||
if not json_path.exists():
|
||||
return None
|
||||
try:
|
||||
payload = read_json(json_path)
|
||||
except Exception:
|
||||
return None
|
||||
metrics = _extract_platform_metrics(platform, payload)
|
||||
# 不要相信平台文件里"已应用 ERP 覆盖"的旧值 —— 平台采集在 ERP 写 data.json 之前
|
||||
# 跑的话,summarize_styles_from_excel 拿到 all_failed 标记, 销量/退货量回退到平台原值
|
||||
# (TM 0/99 / DY 138/833) 并被冻进平台 JSON。这里用最新的 data.json 再覆盖一次。
|
||||
try:
|
||||
override = apply_erp_override(
|
||||
platform, style_name,
|
||||
metrics["visitors"], metrics["cart_users"],
|
||||
target_date=target_date,
|
||||
platform_sales=metrics.get("sales"),
|
||||
platform_refunds=metrics.get("refund_orders"),
|
||||
)
|
||||
metrics["sales"] = override["销量"]
|
||||
metrics["refund_orders"] = override["退款订单数"]
|
||||
metrics["erp_unavailable"] = override.get("erp_unavailable", False)
|
||||
except Exception as exc:
|
||||
metrics["erp_override_error"] = f"{type(exc).__name__}: {exc}"
|
||||
metrics["source"] = str(json_path)
|
||||
return metrics
|
||||
|
||||
|
||||
def load_erp_block(platform: str, style_name: str, target_date: str, data_root: Path) -> dict | None:
|
||||
style_dir = data_root / PLATFORM_DIRS[platform] / f"{safe_name(style_name)}_{target_date}"
|
||||
json_path = style_dir / "data.json"
|
||||
if not json_path.exists():
|
||||
return None
|
||||
try:
|
||||
payload = read_json(json_path)
|
||||
except Exception:
|
||||
return None
|
||||
block = _extract_erp_metadata(payload)
|
||||
block["source"] = str(json_path)
|
||||
return block
|
||||
|
||||
|
||||
def build_payload(date: str, style_name: str, data_root: Path) -> dict:
|
||||
payload = {
|
||||
"date": date,
|
||||
"style_name": style_name,
|
||||
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"metrics_note": {
|
||||
"platform.sales": "汇总时实时应用 ERP 覆盖后的销量（不要看平台 JSON 里的旧值）",
|
||||
"platform.refund_orders": "汇总时实时应用 ERP 覆盖后的退货量（不要看平台 JSON 里的旧值）",
|
||||
"platform.visitors": "平台商品访客数之和",
|
||||
"platform.cart_users": "平台商品加购人数之和",
|
||||
"erp.style_codes": "ERP 款式编码",
|
||||
"erp_unavailable": "True 表示本平台 ERP data.json 不存在或全部查询失败, 此时销量/退货量用平台原值",
|
||||
},
|
||||
"erp": {},
|
||||
"jd": {},
|
||||
"tm": {},
|
||||
"dy": {},
|
||||
}
|
||||
erp_seen: dict | None = None
|
||||
for platform in PLATFORMS:
|
||||
platform_block = load_platform_block(platform, style_name, date, data_root)
|
||||
if platform_block:
|
||||
payload[platform] = platform_block
|
||||
erp_block = load_erp_block(platform, style_name, date, data_root)
|
||||
if erp_block and erp_seen is None:
|
||||
erp_seen = erp_block
|
||||
if erp_seen is not None:
|
||||
payload["erp"] = erp_seen
|
||||
return payload
|
||||
|
||||
|
||||
def render_markdown(payload: dict) -> str:
|
||||
style = payload["style_name"]
|
||||
date = payload["date"]
|
||||
erp = payload.get("erp") or {}
|
||||
|
||||
lines = [
|
||||
f"# {style} - 每日最终汇总 - {date}",
|
||||
"",
|
||||
f"- 日期: {date}",
|
||||
f"- 款式: {style}",
|
||||
f"- ERP款式编码: {', '.join(erp.get('style_codes') or []) or '-'}",
|
||||
"",
|
||||
"| 平台 | 销量 | 退货量 | 访客数 | 加购人数 |",
|
||||
"|---|---:|---:|---:|---:|",
|
||||
]
|
||||
for platform in PLATFORMS:
|
||||
block = payload.get(platform) or {}
|
||||
if not block:
|
||||
lines.append(f"| {platform} | - | - | - | - |")
|
||||
continue
|
||||
lines.append(
|
||||
f"| {platform} | {int(parse_number(block.get('sales')))} | "
|
||||
f"{int(parse_number(block.get('refund_orders')))} | "
|
||||
f"{int(parse_number(block.get('visitors')))} | "
|
||||
f"{int(parse_number(block.get('cart_users')))} |"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append(f"_生成时间: {payload['generated_at']}_")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_outputs(payload: dict, data_root: Path) -> tuple[Path, Path]:
|
||||
style_dir = data_root / payload["date"] / safe_name(payload["style_name"])
|
||||
style_dir.mkdir(parents=True, exist_ok=True)
|
||||
json_path = style_dir / "final_summary.json"
|
||||
md_path = style_dir / "final_summary.md"
|
||||
json_path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
md_path.write_text(render_markdown(payload), encoding="utf-8-sig")
|
||||
return json_path, md_path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="按日期+款式汇总三平台 + ERP 每日数据（无比率字段）")
|
||||
parser.add_argument("--date", default=datetime.now().strftime("%Y-%m-%d"))
|
||||
parser.add_argument("--data-root", default=str(DATA_ROOT))
|
||||
args = parser.parse_args()
|
||||
|
||||
data_root = managed_data_path(args.data_root)
|
||||
style_names = collect_style_names(args.date, data_root)
|
||||
print(f"[INFO] {args.date} 共发现 {len(style_names)} 个款")
|
||||
|
||||
written = 0
|
||||
payloads = []
|
||||
for style_name in style_names:
|
||||
payload = build_payload(args.date, style_name, data_root)
|
||||
json_path, md_path = write_outputs(payload, CURATED_DATA_ROOT)
|
||||
written += 1
|
||||
payloads.append(payload)
|
||||
|
||||
print(f"[OK] {args.date} 写入 {written} 个款 -> {data_root / args.date}")
|
||||
|
||||
# 写库（库不可用时 WARN，不阻塞主流程）
|
||||
try:
|
||||
from db import get_conn, upsert_daily_metrics
|
||||
rows = []
|
||||
for p in payloads:
|
||||
erp_codes = (p.get("erp") or {}).get("style_codes") or []
|
||||
for platform in ("jd", "dy", "tm"):
|
||||
block = p.get(platform) or {}
|
||||
if not block:
|
||||
continue
|
||||
rows.append({
|
||||
"metric_date": args.date,
|
||||
"style_name": p.get("style_name"),
|
||||
"platform": platform,
|
||||
"sales": int(parse_number(block.get("sales"))),
|
||||
"refund_orders": int(parse_number(block.get("refund_orders"))),
|
||||
"visitors": int(parse_number(block.get("visitors"))),
|
||||
"cart_users": int(parse_number(block.get("cart_users"))),
|
||||
"erp_style_codes": erp_codes,
|
||||
"source": block.get("source"),
|
||||
})
|
||||
if rows:
|
||||
with get_conn() as conn:
|
||||
n = upsert_daily_metrics(conn, rows)
|
||||
print(f"[PG] daily_style_metrics upsert: {n} 行")
|
||||
except Exception as exc:
|
||||
print(f"[WARN] PG upsert 失败 (库未配/不可达)，主流程不受影响: {exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loaded 100 of 374 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user