diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0eb15d3..9d0063e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,13 +19,13 @@ lint: stage: quality script: - uv run ruff check src tests - - uv run ruff check src/gyxx_flow/modules/content_marketing/daily_creator_exposure_scope.py src/gyxx_flow/modules/product_commerce/market_rank_limits.py src/gyxx_flow/modules/product_commerce/market_rank_product_import.py + - uv run ruff check src/gyxx_flow/modules/content_marketing/daily_creator_exposure_scope.py src/gyxx_flow/modules/content_marketing/chanmama_export.py src/gyxx_flow/modules/content_marketing/self_operated_accounts.py src/gyxx_flow/modules/content_marketing/self_xhs_scraper.py src/gyxx_flow/modules/content_marketing/data/tools/sync_self_video_metrics.py src/gyxx_flow/modules/product_commerce/market_rank_limits.py src/gyxx_flow/modules/product_commerce/market_rank_product_import.py src/gyxx_flow/modules/shop_intelligence/collectors/dy_after_sales_quality.py src/gyxx_flow/modules/shop_intelligence/collectors/tmall_negative_reviews.py test: stage: test script: - uv run pytest - - uv run pytest tests/modules/content_marketing/test_daily_creator_exposure_scope.py tests/modules/content_marketing/test_chanmama_refresh_before_export.py tests/modules/content_marketing/test_daily_exposure_fields.py + - uv run pytest tests/modules/content_marketing/test_daily_creator_exposure_scope.py tests/modules/content_marketing/test_chanmama_refresh_before_export.py tests/modules/content_marketing/test_daily_exposure_fields.py tests/modules/content_marketing/test_self_content_metrics.py - uv run pytest tests/modules/product_commerce/test_market_rank_limits.py tests/modules/product_commerce/test_market_rank_product_import.py build: diff --git a/config/commands.json b/config/commands.json index a093355..7f0f273 100644 --- a/config/commands.json +++ b/config/commands.json @@ -25,12 +25,24 @@ "entry": "chanmama_scraper.py", "kind": "python" }, + { + "id": "content.metrics.collect_self_xiaohongshu", + "module": "content_marketing", + "entry": "self_xhs_scraper.py", + "kind": "python" + }, { "id": "content.metrics.sync", "module": "content_marketing", "entry": "data/tools/sync_metrics_to_cmt_notes.py", "kind": "python" }, + { + "id": "content.metrics.sync_self_video_metrics", + "module": "content_marketing", + "entry": "data/tools/sync_self_video_metrics.py", + "kind": "python" + }, { "id": "content.marketing_report.generate", "module": "content_marketing", @@ -137,6 +149,98 @@ "--headless" ] }, + { + "id": "shop.jd_customer_service.collect", + "module": "shop_intelligence", + "entry": "collectors/jd_customer_service_messages.py", + "kind": "python", + "default_args": [ + "--date", + "{business_date}", + "--headless" + ] + }, + { + "id": "shop.jd_negative_reviews.collect", + "module": "shop_intelligence", + "entry": "collectors/jd_negative_reviews.py", + "kind": "python", + "default_args": [ + "--date", + "{business_date}", + "--headless" + ] + }, + { + "id": "shop.jd_after_sales_issues.collect", + "module": "shop_intelligence", + "entry": "collectors/jd_after_sales_issues.py", + "kind": "python", + "default_args": [ + "--date", + "{business_date}", + "--headless" + ] + }, + { + "id": "shop.douyin_customer_service.collect", + "module": "shop_intelligence", + "entry": "collectors/dy_customer_service_messages.py", + "kind": "python", + "default_args": [ + "--date", + "{business_date}", + "--headless" + ] + }, + { + "id": "shop.tmall_negative_reviews.collect", + "module": "shop_intelligence", + "entry": "collectors/tmall_negative_reviews.py", + "kind": "python", + "default_args": [ + "--start-date", + "{business_date}", + "--end-date", + "{business_date}", + "--headless" + ] + }, + { + "id": "shop.tmall_customer_service.collect", + "module": "shop_intelligence", + "entry": "collectors/tmall_customer_service_messages.py", + "kind": "python", + "default_args": [ + "--start-date", + "{business_date}", + "--end-date", + "{business_date}", + "--headless" + ] + }, + { + "id": "shop.dy_negative_reviews.collect", + "module": "shop_intelligence", + "entry": "collectors/dy_negative_reviews.py", + "kind": "python", + "default_args": [ + "--date", + "{business_date}", + "--headless" + ] + }, + { + "id": "shop.douyin_after_sales_quality.collect", + "module": "shop_intelligence", + "entry": "collectors/dy_after_sales_quality.py", + "kind": "python", + "default_args": [ + "--date", + "{business_date}", + "--headless" + ] + }, { "id": "shop.douyin_price_appeal", "module": "shop_intelligence", @@ -151,7 +255,11 @@ "id": "product.persona.collect", "module": "product_commerce", "entry": "run_daily_persona.py", - "kind": "python" + "kind": "python", + "default_args": [ + "--date", + "{business_date}" + ] }, { "id": "product.daily.orchestrate", @@ -166,6 +274,13 @@ "kind": "python", "notification_workflow_id": "product.alert.daily" }, + { + "id": "product.online_stock_alert.run", + "module": "product_commerce", + "entry": "online_stock_alert.py", + "kind": "python", + "notification_workflow_id": "product.online_stock_alert.daily" + }, { "id": "product.import.daily", "module": "product_commerce", @@ -176,7 +291,12 @@ "id": "product.tmall_wanxiang_ads.collect", "module": "product_commerce", "entry": "collect_tmall_wanxiang_ads.py", - "kind": "python" + "kind": "python", + "default_args": [ + "--date", + "{business_date}", + "--headless" + ] }, { "id": "product.import.tmall_ads", diff --git a/config/notification-routing.json b/config/notification-routing.json index 3d60f1d..ca6742b 100644 --- a/config/notification-routing.json +++ b/config/notification-routing.json @@ -14,6 +14,17 @@ } }, "people": { + "online_stock_alert_recipient": { + "name": "线上库存预警接收人", + "mobile": null, + "bindings": { + "hermes-analyzer": { + "open_id": "ou_cd858437acecee9404c9920d2dc04b2a", + "verified": true, + "source": "mobile_lookup" + } + } + }, "he_yingwei": { "name": "何颖威", "mobile": null, @@ -27,7 +38,7 @@ }, "wang_yunlong": { "name": "王云龙", - "mobile": null, + "mobile": "+8613663785145", "bindings": { "collector": { "open_id": "ou_8ee224968aa26a74c7d30ba27fed5eeb", @@ -194,6 +205,17 @@ "source": "provided" } } + }, + "stock_alert_user_19832771651": { + "name": "19832771651", + "mobile": "+8619832771651", + "bindings": { + "hermes-analyzer": { + "open_id": "ou_9136c3203c698fead5477eea827f2bcf", + "verified": true, + "source": "mobile_lookup" + } + } } }, "capabilities": { @@ -209,6 +231,9 @@ "product.market_rank": { "condition": "存在本次新生成的平台报告链接且尚未重复通知时投递。" }, + "product.online_stock_alert.daily": { + "condition": "线上商品当前库存低于 3 时向配置接收人发送库存预警。" + }, "supply.purchase_confirmation.daily": { "condition": "采购确认结果生成完成后投递最终业务通知。" }, @@ -219,5 +244,15 @@ "condition": "存在命中库存阈值的 SKU 时投递预警。" } }, - "routes": {} + "routes": { + "product.online_stock_alert.daily": { + "enabled": true, + "app_profile": "hermes-analyzer", + "person_ids": [ + "online_stock_alert_recipient", + "wang_yunlong", + "stock_alert_user_19832771651" + ] + } + } } diff --git a/config/runtime-bindings.json b/config/runtime-bindings.json index 9935b58..953b555 100644 --- a/config/runtime-bindings.json +++ b/config/runtime-bindings.json @@ -33,7 +33,10 @@ "product_commerce:dy_product_scraping.py", "product_commerce:vendors/dy-data-flow/dy_store_competitor_store_scraping.py", "shop.douyin_price_appeal", - "content_marketing:xingtu_scraper_v2.py" + "content_marketing:xingtu_scraper_v2.py", + "shop.douyin_customer_service.collect", + "shop.douyin_after_sales_quality.collect", + "shop.dy_negative_reviews.collect" ] }, "jd-shop": { @@ -50,6 +53,9 @@ "JD_SHOP", "JD_PASSWORD" ], + "login_identity_markers": [ + "光影行星 GYXX官方旗舰店" + ], "keepalive": { "enabled": true, "url": "https://shop.jd.com/jdm/home", @@ -68,11 +74,15 @@ "product_commerce:vendors/jd-data-flow/jd_data_collector.py", "product_commerce:vendors/jd-data-flow/jd_product_data_collector.py", "shop_intelligence:collectors/jd_data_collector.py", - "shop_intelligence:collectors/jd_peer_store_data_collector.py" + "shop_intelligence:collectors/jd_peer_store_data_collector.py", + "shop.jd_customer_service.collect", + "shop.jd_negative_reviews.collect", + "shop.jd_after_sales_issues.collect" ] }, "jd-self-operated": { "cdp_port": 22202, + "collection_cdp_port": 22155, "login_mode": "D", "required_cookie_domains": [ "jd.com" @@ -208,6 +218,8 @@ "product_commerce:taobao_sycm_products.py", "product_commerce:taobao_wanxiang_ai_creative_report.py", "shop_intelligence:collectors/taobao_sycm.py", + "shop.tmall_customer_service.collect", + "shop.tmall_negative_reviews.collect", "product.tmall_baibu_apply.old_chain_bag", "product.tmall_baibu_apply.old_all_bag", "product.tmall_baibu_apply.old_all_3c", @@ -768,6 +780,25 @@ ], "cdp_port": 22036 }, + "content.metrics.sync_self_video_metrics": { + "script_id": "content_marketing:data/tools/sync_self_video_metrics.py", + "state_key": "content_marketing:data/tools/sync_self_video_metrics.py", + "aliases": [ + "content_marketing:data/tools/sync_self_video_metrics.py" + ], + "cdp_port": 22164 + }, + "content.metrics.collect_self_xiaohongshu": { + "script_id": "content_marketing:self_xhs_scraper.py", + "state_key": "content_marketing:self_xhs_scraper.py", + "aliases": [ + "content_marketing:self_xhs_scraper.py" + ], + "cdp_port": 22165, + "login_mode": "C", + "required_cookie_domains": ["xiaohongshu.com"], + "credential_env_names": ["XHS_ARK_ACCOUNT"] + }, "content_marketing:data/tools/sync_style_categories.py": { "script_id": "content_marketing:data/tools/sync_style_categories.py", "state_key": "content_marketing:data/tools/sync_style_categories.py", @@ -1177,6 +1208,15 @@ ], "cdp_port": 22069 }, + "product.online_stock_alert.run": { + "script_id": "product_commerce:online_stock_alert.py", + "state_key": "product_commerce:online_stock_alert.py", + "aliases": [ + "product_commerce:online_stock_alert.py", + "product.online_stock_alert.run" + ], + "cdp_port": 22163 + }, "product_commerce:db/sync_dim_style.py": { "script_id": "product_commerce:db/sync_dim_style.py", "state_key": "product_commerce:db/sync_dim_style.py", @@ -1604,6 +1644,46 @@ ], "account": "douyin-shop" }, + "shop.douyin_customer_service.collect": { + "script_id": "shop_intelligence:collectors/dy_customer_service_messages.py", + "state_key": "shop_intelligence:collectors/dy_customer_service_messages.py", + "aliases": [ + "shop_intelligence:collectors/dy_customer_service_messages.py" + ], + "cdp_port": 22158, + "login_mode": "D", + "required_cookie_domains": [ + "jinritemai.com" + ], + "required_cookie_names": [ + "sessionid", + "sid_tt", + "uid_tt", + "sessionid_ss", + "sid_guard" + ], + "account": "douyin-shop" + }, + "shop.douyin_after_sales_quality.collect": { + "script_id": "shop_intelligence:collectors/dy_after_sales_quality.py", + "state_key": "shop_intelligence:collectors/dy_after_sales_quality.py", + "aliases": [ + "shop_intelligence:collectors/dy_after_sales_quality.py" + ], + "cdp_port": 22159, + "login_mode": "D", + "required_cookie_domains": [ + "jinritemai.com" + ], + "required_cookie_names": [ + "sessionid", + "sid_tt", + "uid_tt", + "sessionid_ss", + "sid_guard" + ], + "account": "douyin-shop" + }, "shop_intelligence:collectors/jd_data_collector.py": { "script_id": "shop_intelligence:collectors/jd_data_collector.py", "state_key": "shop_intelligence:collectors/jd_data_collector.py", @@ -1666,6 +1746,107 @@ "JD_SELF_OPERATED_PASSWORD" ] }, + "shop.jd_customer_service.collect": { + "script_id": "shop_intelligence:collectors/jd_customer_service_messages.py", + "state_key": "shop_intelligence:collectors/jd_customer_service_messages.py", + "aliases": [ + "shop_intelligence:collectors/jd_customer_service_messages.py" + ], + "cdp_port": 22154, + "login_mode": "D", + "required_cookie_domains": [ + "jd.com" + ], + "required_cookie_names": [ + "pin", + "thor" + ], + "credential_env_names": [ + "JD_SHOP", + "JD_PASSWORD" + ], + "account": "jd-shop" + }, + "shop.tmall_negative_reviews.collect": { + "script_id": "shop_intelligence:collectors/tmall_negative_reviews.py", + "state_key": "shop_intelligence:collectors/tmall_negative_reviews.py", + "aliases": [ + "shop_intelligence:collectors/tmall_negative_reviews.py" + ], + "cdp_port": 22162, + "login_mode": "D", + "required_cookie_domains": [ + "taobao.com", + "alimama.com" + ], + "required_cookie_names": [ + "unb", + "cookie2", + "_tb_token_" + ], + "account": "tmall-shop" + }, + "shop.tmall_customer_service.collect": { + "script_id": "shop_intelligence:collectors/tmall_customer_service_messages.py", + "state_key": "shop_intelligence:collectors/tmall_customer_service_messages.py", + "aliases": [ + "shop_intelligence:collectors/tmall_customer_service_messages.py" + ], + "cdp_port": 22161, + "login_mode": "D", + "required_cookie_domains": [ + "taobao.com", + "alimama.com" + ], + "required_cookie_names": [ + "unb", + "cookie2", + "_tb_token_" + ], + "account": "tmall-shop" + }, + "shop.jd_negative_reviews.collect": { + "script_id": "shop_intelligence:collectors/jd_negative_reviews.py", + "state_key": "shop_intelligence:collectors/jd_negative_reviews.py", + "aliases": [ + "shop_intelligence:collectors/jd_negative_reviews.py" + ], + "cdp_port": 22156, + "login_mode": "D", + "required_cookie_domains": [ + "jd.com" + ], + "required_cookie_names": [ + "pin", + "thor" + ], + "credential_env_names": [ + "JD_SHOP", + "JD_PASSWORD" + ], + "account": "jd-shop" + }, + "shop.jd_after_sales_issues.collect": { + "script_id": "shop_intelligence:collectors/jd_after_sales_issues.py", + "state_key": "shop_intelligence:collectors/jd_after_sales_issues.py", + "aliases": [ + "shop_intelligence:collectors/jd_after_sales_issues.py" + ], + "cdp_port": 22157, + "login_mode": "D", + "required_cookie_domains": [ + "jd.com" + ], + "required_cookie_names": [ + "pin", + "thor" + ], + "credential_env_names": [ + "JD_SHOP", + "JD_PASSWORD" + ], + "account": "jd-shop" + }, "shop_intelligence:collectors/taobao_sycm.py": { "script_id": "shop_intelligence:collectors/taobao_sycm.py", "state_key": "shop_intelligence:collectors/taobao_sycm.py", @@ -2053,6 +2234,26 @@ "TMALL_BAG_PASSWORD" ], "account": "tmall-bag" + }, + "shop.dy_negative_reviews.collect": { + "script_id": "shop_intelligence:collectors/dy_negative_reviews.py", + "state_key": "shop_intelligence:collectors/dy_negative_reviews.py", + "aliases": [ + "shop_intelligence:collectors/dy_negative_reviews.py" + ], + "cdp_port": 22160, + "login_mode": "D", + "required_cookie_domains": [ + "jinritemai.com" + ], + "required_cookie_names": [ + "sessionid", + "sid_tt", + "uid_tt", + "sessionid_ss", + "sid_guard" + ], + "account": "douyin-shop" } }, "services": { diff --git a/config/schedules.json b/config/schedules.json index d38ed11..1db769a 100644 --- a/config/schedules.json +++ b/config/schedules.json @@ -10,6 +10,16 @@ "kind": "daily", "workflow_id": "content.metrics.daily" }, + { + "at": [ + "01:00", + "05:00" + ], + "business_date_offset_days": 0, + "enabled": true, + "kind": "daily", + "workflow_id": "content.metrics.self_operated.daily" + }, { "at": "14:00", "business_date_offset_days": 0, @@ -51,21 +61,27 @@ }, { "at": "10:00", + "business_date_offset_days": 0, "days": [ "Tuesday" ], + "enabled": false, "kind": "weekly", "workflow_id": "content.summary.weekly" }, { "at": "08:00", + "business_date_offset_days": 0, "day_of_month": 1, + "enabled": false, "kind": "monthly", "workflow_id": "content.summary.monthly" }, { "at": "08:30", + "business_date_offset_days": 0, "day_of_month": 1, + "enabled": false, "kind": "monthly", "workflow_id": "content.creator_report.monthly" }, @@ -87,9 +103,30 @@ }, { "at": [ + "00:00", + "01:00", + "02:00", + "03:00", + "04:00", + "05:00", + "06:00", + "07:00", "08:00", + "09:00", + "10:00", + "11:00", + "12:00", + "13:00", + "14:00", + "15:00", "16:00", - "22:00" + "17:00", + "18:00", + "19:00", + "20:00", + "21:00", + "22:00", + "23:00" ], "kind": "daily", "workflow_id": "shop.douyin_price_appeal" @@ -101,6 +138,77 @@ "kind": "daily", "workflow_id": "shop.jd_self_operated.daily" }, + { + "at": "03:00", + "business_date_offset_days": 0, + "day_of_month": 21, + "enabled": true, + "kind": "monthly", + "workflow_id": "shop.jd_customer_service.daily" + }, + { + "at": "03:00", + "business_date_offset_days": 0, + "day_of_month": 22, + "enabled": true, + "kind": "monthly", + "workflow_id": "shop.jd_negative_reviews.daily" + }, + { + "at": "03:00", + "business_date_offset_days": 0, + "day_of_month": 23, + "enabled": true, + "kind": "monthly", + "workflow_id": "shop.jd_after_sales_issues.daily" + }, + { + "at": "03:20", + "business_date_offset_days": 0, + "day_of_month": 21, + "enabled": true, + "kind": "monthly", + "workflow_id": "shop.douyin_customer_service.daily" + }, + { + "at": "03:50", + "business_date_offset_days": 0, + "day_of_month": 25, + "enabled": false, + "kind": "monthly", + "workflow_id": "shop.tmall_negative_reviews.daily" + }, + { + "at": "03:40", + "business_date_offset_days": 0, + "day_of_month": 24, + "enabled": false, + "kind": "monthly", + "workflow_id": "shop.tmall_customer_service.daily" + }, + { + "at": "03:20", + "business_date_offset_days": 0, + "day_of_month": 22, + "enabled": true, + "kind": "monthly", + "workflow_id": "shop.dy_negative_reviews.daily" + }, + { + "at": "03:20", + "business_date_offset_days": 0, + "day_of_month": 23, + "enabled": true, + "kind": "monthly", + "workflow_id": "shop.douyin_after_sales_quality.daily" + }, + { + "at": "09:30", + "business_date_offset_days": 0, + "enabled": true, + "kind": "daily", + "workflow_id": "product.online_stock_alert.daily" + }, { "at": "08:40", "business_date_offset_days": -1, @@ -132,6 +240,8 @@ }, { "at": "23:00", + "business_date_offset_days": 0, + "enabled": false, "kind": "daily", "workflow_id": "product.alert.daily" }, @@ -139,6 +249,7 @@ "anchor_date": "2026-07-25", "at": "11:00", "business_date_offset_days": -1, + "enabled": false, "every_days": 3, "kind": "interval_days", "workflow_id": "product.style_analysis.interval" @@ -224,6 +335,8 @@ }, { "at": "08:00", + "business_date_offset_days": 0, + "enabled": false, "kind": "daily", "workflow_id": "supply.purchase_confirmation.daily" }, diff --git a/config/secrets.example.env b/config/secrets.example.env index 9398dd7..e8ff713 100644 --- a/config/secrets.example.env +++ b/config/secrets.example.env @@ -3,6 +3,9 @@ GYXX_POSTGRES_PASSWORD= GYXX_FEISHU_APP_ID= GYXX_FEISHU_APP_SECRET= GYXX_HERMES_API_KEY= +GYXX_DATA_INTERACTION_BASE_URL=https://test.gyxx.cn/api +GYXX_DATA_INTERACTION_APP_KEY=data-hub +GYXX_DATA_INTERACTION_APP_SECRET= # 款式周期分析、天猫/京东视频标题与京东视觉识别直连 MiniMax； # 真实密钥只放部署环境运行时 .env STYLE_ANALYSIS_LLM_BASE_URL=https://api.minimaxi.com/v1 diff --git a/config/source-manifests/content_marketing.json b/config/source-manifests/content_marketing.json index c581bc1..8ceeda6 100644 --- a/config/source-manifests/content_marketing.json +++ b/config/source-manifests/content_marketing.json @@ -15,7 +15,7 @@ "source_relative_path": "bilibili_scraper.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/bilibili_scraper.py", "source_sha256": "c3df15f317ce073f84a59609f83e2723a21a43e37cfcfc2f17b3cbf3a17df73e", - "target_sha256": "c6c985c860f0b9ab742a66786fb3576a71767bbf897969d9539d7c780303a201", + "target_sha256": "153ccbec744bfdd330e3b350bed3b4c9ed9866cabb3e1785f14a880c46178c97", "transformed": true, "category": "source" }, @@ -23,15 +23,15 @@ "source_relative_path": "chanmama_scraper.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/chanmama_scraper.py", "source_sha256": "2edc26879b57d73ec5992c80e5ef74d76a509f09bff7284b1ae17e90df543e07", - "target_sha256": "72634b4f042f8506bcccadf4512dcbe9f92e9aced3e9b1fc7718483b7c802ebf", + "target_sha256": "8b87268e0f105cf4d349b192c7b2c24d5e318a9d88fcd14264824b083103b24e", "transformed": true, "category": "source" }, { "source_relative_path": "collection_completeness.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/collection_completeness.py", - "source_sha256": "f23b70b949f8adcaf00ee62aa0f9f39314bc623528cad9a77591641da765991d", - "target_sha256": "f23b70b949f8adcaf00ee62aa0f9f39314bc623528cad9a77591641da765991d", + "source_sha256": "575ab0c526f2d78b1135bd27407c9ecb0fb0aaad2063ac37ba0818ea6247cc2d", + "target_sha256": "575ab0c526f2d78b1135bd27407c9ecb0fb0aaad2063ac37ba0818ea6247cc2d", "transformed": false, "category": "source" }, @@ -55,15 +55,15 @@ "source_relative_path": "daily_marketing_report.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/daily_marketing_report.py", "source_sha256": "b59252594c7d3c0fd1473d14611346e4b64672ef6c263fea8a2da53017ab8675", - "target_sha256": "78fb871c14eb73ec3e0d0986554cad318f8a0ee8a66c2a0981f02a65cded8a26", + "target_sha256": "e049d933721d0dbe5be559f33beaea812443c84af504d05bc658b20532d28533", "transformed": true, "category": "source" }, { "source_relative_path": "data/config/analyze.env.example", "target_relative_path": "src/gyxx_flow/modules/content_marketing/data/config/analyze.env.example", - "source_sha256": "b60328ef177b795489d0a57957133dedcfd67b38b59603a6edeee1cf9babcee2", - "target_sha256": "b60328ef177b795489d0a57957133dedcfd67b38b59603a6edeee1cf9babcee2", + "source_sha256": "72c8359ffbde76060642c77ad21ec8544645ee00aca5a5bfc6bd0971647e9240", + "target_sha256": "72c8359ffbde76060642c77ad21ec8544645ee00aca5a5bfc6bd0971647e9240", "transformed": false, "category": "config" }, @@ -103,7 +103,7 @@ "source_relative_path": "data/tools/analyze_note.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/analyze_note.py", "source_sha256": "f9a71dcd34657d3e70af84160bd04afa73325c9ddeee0e669d98d6758ed6b13e", - "target_sha256": "3c62dcac6e5c877c1a5b55f4960244052a124c2e50c61571cc726d2ca916109b", + "target_sha256": "0f538b5c9fc52d23b66f7f79f3c54fcc9e7ed737ab6decb5b16c4eed09e59615", "transformed": true, "category": "tool" }, @@ -191,7 +191,7 @@ "source_relative_path": "data/tools/feishu_comment_batch.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/feishu_comment_batch.py", "source_sha256": "2b04fde410eec1dece0b995d0b74bfad4bdc1a53669f19ead2a6fc13ecdd00b7", - "target_sha256": "598a26e1dcc65702ad154ec3dcc84b4d5c8fecf45196e7e8301ca5b77544ae23", + "target_sha256": "80985a42af7dce5d26d7b4f14ef36411631157e44f8d6ee054de2b66277905f5", "transformed": true, "category": "tool" }, @@ -254,8 +254,8 @@ { "source_relative_path": "data/tools/migrations/005_style_product_profile.sql", "target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/migrations/005_style_product_profile.sql", - "source_sha256": "8f9c4d25681e84e582d3cdcd9108ebb16c2f5b777b766a5c3653a21c0baf6d91", - "target_sha256": "8f9c4d25681e84e582d3cdcd9108ebb16c2f5b777b766a5c3653a21c0baf6d91", + "source_sha256": "6a5f60f162c02a9daea2d46ff95edfbcaf416d9f70d894d3b83d049b81ab2134", + "target_sha256": "6a5f60f162c02a9daea2d46ff95edfbcaf416d9f70d894d3b83d049b81ab2134", "transformed": false, "category": "sql" }, @@ -343,7 +343,7 @@ "source_relative_path": "data/tools/retry_failed.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/retry_failed.py", "source_sha256": "90c6dbc47317cc48e28071d452d75c95913138fc885a5963f9eb58986cd99086", - "target_sha256": "b4c85e8319bfd0a3071ffdfaff05ea030e426cb3d384a978fdc4d14545ccd679", + "target_sha256": "26ca143da6539345c930d0a949a852a33735ed764e7065162c9f805d2609fe91", "transformed": true, "category": "tool" }, @@ -351,7 +351,7 @@ "source_relative_path": "data/tools/schema_gyxx_super_data.sql", "target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/schema_gyxx_super_data.sql", "source_sha256": "43d202d7d0452b7d0671ee9c40149c21328db4b2863fa4d030af456f89defe3c", - "target_sha256": "43d202d7d0452b7d0671ee9c40149c21328db4b2863fa4d030af456f89defe3c", + "target_sha256": "001870851dea9c373a77b87335604c4363b06c3eb0ed7c97b4a8d5e891e56758", "transformed": true, "category": "sql" }, @@ -367,7 +367,15 @@ "source_relative_path": "data/tools/sync_metrics_to_cmt_notes.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/sync_metrics_to_cmt_notes.py", "source_sha256": "3b97cab0740c5c97b47a0fbe763acb5391d7c8e47c95f4675720b4f716f6432f", - "target_sha256": "4e630a79bd2a9742ad19d1955b88f3f12cc7e741b445196e53c07440c0f3749d", + "target_sha256": "6dfeeaa77873fdc580077e1c5d491ced653a1c319720f1fdffc6d5f44c11f5d3", + "transformed": true, + "category": "tool" + }, + { + "source_relative_path": "data/tools/sync_notes_master.py", + "target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/sync_notes_master.py", + "source_sha256": "94edc9b55bfc0106101af30897a68b4f0513d04b8ed4a07a99daf6fce5447f45", + "target_sha256": "aa70a895c0b7dd32e458b885e9e036246e12d83f354d068e32ac3ee6b491e175", "transformed": true, "category": "tool" }, @@ -407,7 +415,7 @@ "source_relative_path": "douyin_comment_scraper.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/douyin_comment_scraper.py", "source_sha256": "ec785ead9c95c5de65c00b06212b1f03d0159fed475a95a2b593d3b24e486e43", - "target_sha256": "e36ec226b06bae4940f0d2432ae7de0d9a0b56199ab6cd8ee7d34fbfc060ec51", + "target_sha256": "6e7ac4357fe466656b93ca7bae6c49adc4d59fa75974a39c6885c7433bd4d402", "transformed": true, "category": "source" }, @@ -415,7 +423,7 @@ "source_relative_path": "feishu_mapping.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/feishu_mapping.py", "source_sha256": "d389fd59f5e49208ffa12d04c1ca6202f51dc5d89321a50e52fadad747239749", - "target_sha256": "56af85da94c8eff1f1e0c27f13df56860cc959328524a059b6a374d55eb32e20", + "target_sha256": "0ee2ccd518d4fd313e4acbb62857b565f2e84c77968ba68a3f2f680268f2fdd3", "transformed": true, "category": "source" }, @@ -431,7 +439,7 @@ "source_relative_path": "monthly_summary_all.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/monthly_summary_all.py", "source_sha256": "50f18d62e8fb1a7c8a04dc746326a4763f64ba4b99b29a6fda5878aa05f290fa", - "target_sha256": "ea2729e1976426d497b69006c63b5ba7d74ce530731efa5a2946a4671593aed7", + "target_sha256": "6f34da4842bb2a7a0b91c3f89ce6047ddccb43a243614ab77a014de516192fbc", "transformed": true, "category": "source" }, @@ -439,7 +447,7 @@ "source_relative_path": "pgy_xhs_scraper.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/pgy_xhs_scraper.py", "source_sha256": "7d9cd66b80cf602127541cfdd9589f2d8870ca4a25bac3587478482064aae8f4", - "target_sha256": "c1ee83cc68464d5b0b33e010b388abd2adab8a6608846b47d509e9ea0a783f81", + "target_sha256": "667fcfe8bb4ebc55fad44838cbd00d79a4f414dd875a4f74d27bf2ff6e90bc2c", "transformed": true, "category": "source" }, @@ -447,7 +455,7 @@ "source_relative_path": "pgy_xhs_scraper_v2.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/pgy_xhs_scraper_v2.py", "source_sha256": "2be78ef76df38e80da2099a76e6e3fd9dcdc2e8875afb44279f54003ccb5f563", - "target_sha256": "21c3647faa01a335af2c5984f87bc0d8968361cae73352146279bd9027e82c5f", + "target_sha256": "e5df4b34f67a565ec5d5ce4dd5857e80e363dd7559e7e29006dcd3e421276c8a", "transformed": true, "category": "source" }, @@ -463,7 +471,7 @@ "source_relative_path": "run_all.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/run_all.py", "source_sha256": "7a2529b1d043297171aeb88518fac8753234540992042f922172219b2b233dd4", - "target_sha256": "0ea60854f1bb1ca293a073c57c9e19a1aa9c5710ff9575dc7c430c4e480525d9", + "target_sha256": "8d27c1ef21c4d5548c82a97f9dc461ad191119d569f36bebeb9ee604cd9e727a", "transformed": true, "category": "source" }, @@ -471,7 +479,7 @@ "source_relative_path": "self_bilibili_scraper.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py", "source_sha256": "2ef650a16dde61697101bb219a8c3adcd682fec1529bb9c5312dc5b384d83a72", - "target_sha256": "42e6b9ac81077b37ff4607b3565d98cac1027eff3ac39a8776e640503d590820", + "target_sha256": "6ec5ed6991a93f806fb74d0870e953e5a05d52043a83f2fa261e2ec67bee6ad7", "transformed": true, "category": "source" }, @@ -487,7 +495,7 @@ "source_relative_path": "weekly_summary_all.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/weekly_summary_all.py", "source_sha256": "b93c04c543a8c65418ec122a2713239d0ed857bc169b87ef718cf73348c189b0", - "target_sha256": "a5c866d0720f20fe78bf8f8039a24101fc58e3a81fb107beb7f6499e9782d559", + "target_sha256": "f0d43974605bdd45868e88dd81f7d1db8d624f4c69ecaaf0abf49aaf8ed79cf3", "transformed": true, "category": "source" }, @@ -503,7 +511,7 @@ "source_relative_path": "xiaohongshu_comment_scraper.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/xiaohongshu_comment_scraper.py", "source_sha256": "58bc07bb2fd67897e18a9981f931df8fadf24a6cc5f09d55bd3aaa2e3c6756a4", - "target_sha256": "5e1e306cb1a089c187dc31bc598939304990b3039a5eb91d9da8a71a023b6e69", + "target_sha256": "afca1104324546da2315fdf7a427c3f34021eef31ddc2fea76877f818718ccb0", "transformed": true, "category": "source" }, @@ -519,17 +527,9 @@ "source_relative_path": "xingtu_scraper_v2.py", "target_relative_path": "src/gyxx_flow/modules/content_marketing/xingtu_scraper_v2.py", "source_sha256": "0d4eb87779064e13b8d78daa6dba13698f1149a26b3a1de2e064c117a29cf49a", - "target_sha256": "e694f3ca9126a99123d0941068dbd102f0fdb4b84160ebdbb51cd56c3f3aa32a", + "target_sha256": "b36339444e26048a90c8ac999a0a628bcddd4f64842924f6c60e600c7a3bec37", "transformed": true, "category": "source" - }, - { - "source_relative_path": "data/tools/sync_notes_master.py", - "target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/sync_notes_master.py", - "source_sha256": "94edc9b55bfc0106101af30897a68b4f0513d04b8ed4a07a99daf6fce5447f45", - "target_sha256": "2d90abb7e73ec420bdfdb03fba01ec497e6c2a5fd3b7c1fe8848e486da08f2f8", - "transformed": true, - "category": "tool" } ], "intentionally_excluded": [ diff --git a/config/source-manifests/product_commerce.json b/config/source-manifests/product_commerce.json index f058603..775feeb 100644 --- a/config/source-manifests/product_commerce.json +++ b/config/source-manifests/product_commerce.json @@ -14,7 +14,7 @@ { "category": "production_source", "source_sha256": "cff768f79672cfc7db7f6866caf5ab59936b8c51eb2a7d729b81441940124512", - "target_sha256": "7b4da4f51cd0945bb9ad144aead5ff96e197fdacc7a3c065e6102e833161992a", + "target_sha256": "d56fe8a934c1ae20a9249c7682823238a705f51f451241504512e83a8800a879", "transformed": true, "source_relative_path": "analyze_style_with_hermes.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/analyze_style.py" @@ -54,7 +54,7 @@ { "category": "production_source", "source_sha256": "1493dbbfb3d864812df378d04d006e7de0f9ec4b6ab5812e5c6970c304c2dfe6", - "target_sha256": "2d5a6a7bea5e21a5af5f7cdde287acd5b9e7e375469e6d6f85c15aab1cc31d8e", + "target_sha256": "a1d526dfc27b110a8845a98cc66fe7e5cfb87ef70530728477af31a88734a576", "transformed": true, "source_relative_path": "collect_dy_market_rank.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_dy_market_rank.py" @@ -62,7 +62,7 @@ { "category": "production_source", "source_sha256": "abd4c82cbd6c41e111e213c174d2fdeba1214568496eaf1dede24bf374b3b052", - "target_sha256": "523027ccd4e31bd276bfba3e8944bf43e0e1e67efc0921ef5c72b8bfa4287c28", + "target_sha256": "9de3ab22d5a4dbf574b2d4fdbbed295bb4645bb4943f4c066fcbfd41b44b521c", "transformed": true, "source_relative_path": "collect_dy_persona_to_bitable.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_dy_persona_to_bitable.py" @@ -70,7 +70,7 @@ { "category": "production_source", "source_sha256": "c7b54d941ab7f056e2d0ea4355543ff4a5c34ad986c8ed32145e988ac9501ea2", - "target_sha256": "75cd142aefad9ea54a61e8e11a72e9535ee3a71a0c7b0025a8d030a0a0c0d430", + "target_sha256": "aaba5dfe6ce5133b5c662232160bb9faef4d73b36902a484a56b89fa7df05311", "transformed": true, "source_relative_path": "collect_erp_yesterday_metrics.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py" @@ -78,7 +78,7 @@ { "category": "production_source", "source_sha256": "8bb1f17f90a7b9c2db5dfb992cf746329892d19c00ae2ae988e2109c049598b2", - "target_sha256": "5f9a754f2293b8b35ce5214ae930350b67abe4d10b8e702b06b68c9c9a4d2d9a", + "target_sha256": "c98abe563b223f49cc3c810054d997504d9465387da4805389b0f5ef78ceed6a", "transformed": true, "source_relative_path": "collect_jd_market_rank.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_jd_market_rank.py" @@ -86,7 +86,7 @@ { "category": "production_source", "source_sha256": "f9a48c5f89b18472dc3993e340ad858bfecb2fcf59427c7e70db37512c17a6f7", - "target_sha256": "17064990b7080969e1907f2e2ae9cf1f49267936746b6db6e6e1a3a689e1b501", + "target_sha256": "a268df592f7136a68c2b82d031115055e43cdfaedcdd4a5f4afa9ef45879e055", "transformed": true, "source_relative_path": "collect_jd_persona_to_bitable.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_jd_persona_to_bitable.py" @@ -94,7 +94,7 @@ { "category": "production_source", "source_sha256": "3c1858a6ab4f7ce480b3a5f2076e6ec79400e9bfaa75377488c3629e707c7963", - "target_sha256": "0d2981d17c86c1881c86810aaf63bfd0261de03bc94d91eaa1b42a66953d3f3e", + "target_sha256": "0e4314a525f49b7c24c145f092afcd6c93749c694d6d8bcbd2d1fcdf9b2e54c7", "transformed": true, "source_relative_path": "collect_persona_to_bitable.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_persona_to_bitable.py" @@ -110,7 +110,7 @@ { "category": "production_source", "source_sha256": "13b6e68b4872b2e79c1de8ca1617cc4f1249f4ed32f234ad6efb7a04f5948612", - "target_sha256": "fb120e9501a1220c3746e0f78a9f639e732764e21b122bf532c0fdea03ec6ed7", + "target_sha256": "a1b95cc4f090a25c8e7991b0ef015a7df612fc78249dafe75aeef9fa4dbefe08", "transformed": true, "source_relative_path": "collect_sycm_market_rank.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_sycm_market_rank.py" @@ -126,7 +126,7 @@ { "category": "production_source", "source_sha256": "22dcc9d8b2f1cf598a62e4c71df765a952250694cb26b9a73f477c5ca1a29884", - "target_sha256": "e41bc1de359b8f18c25ee38068ab550e5dbbf5c20712d19723915d669f02d6ac", + "target_sha256": "ffeaf697a1a58b21520fda3f3b0936c591d05b328537c41f2048366a9ba1b8d5", "transformed": true, "source_relative_path": "config/style_config_loader.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/config/style_config_loader.py" @@ -134,7 +134,7 @@ { "category": "production_source", "source_sha256": "a8900b9f25db6cf2bef3117a1d2cd86283054fea6f5a8590fd91b5cfbbcaf87e", - "target_sha256": "7fd65a5b993b5a8b7f681b370e4364411252d6a59b22c7d96d71235daa5fce55", + "target_sha256": "030d33c3f5c0c230bb67d603dd657f40cceff8581f8756a1f67340a89b5ae4b3", "transformed": true, "source_relative_path": "db/__init__.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/db/__init__.py" @@ -142,7 +142,7 @@ { "category": "production_source", "source_sha256": "fc19a57bcbc023ab20156820a708d64c5526fad33ef5eb5fdea56c0bb04b84e3", - "target_sha256": "486607458d204336a04e3776d9db98f5f069a46b75b050ac0f0947cec958b5d7", + "target_sha256": "2cbd79e4d984d556bac35a0aeb5b8f803309fd3fac2399da459cd61a3aa2484c", "transformed": true, "source_relative_path": "db/sync_dim_style.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/db/sync_dim_style.py" @@ -150,7 +150,7 @@ { "category": "production_source", "source_sha256": "2dc00b88fb5f28c2db619074e97c4a7c446aec29926af9a6890065409c8fac0e", - "target_sha256": "030344e0ffe26240ee735ccf38f194d4e69f2833bd21e9947b97d0072647910b", + "target_sha256": "840f98d9fb79557016a59a2e3a1194a1e328a1574b4670f419c99ac85731d623", "transformed": true, "source_relative_path": "db/sync_sku_master.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/db/sync_sku_master.py" @@ -158,7 +158,7 @@ { "category": "production_source", "source_sha256": "78c74c0e52e3651b3b042a2095e596ab250e60309c7ae2de905850822aad4922", - "target_sha256": "e7377793895b6176a0bd76df0ce6a3d8927c1f2bb4f3c8b4ca47e7d3dd97dd00", + "target_sha256": "09d7e08d2eec57055a3bc7949e49c32a8bf62713f2594ae5d9297c032bf212c8", "transformed": true, "source_relative_path": "dy_audience_profile_collect.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/dy_audience_profile_collect.py" @@ -166,7 +166,7 @@ { "category": "production_source", "source_sha256": "492916088fa5e8c170461cc9eba97a88ed1cc1e50c7b99e5f6a5e49167fea157", - "target_sha256": "5f621d4003dbb5ed3c8cd58ae813f371bb0c110aa2fbfff0b6ef559b4eb487b1", + "target_sha256": "9dd0dc3e6f048683157fe8b8deb533ed88970bf34e9824e439bf3a5d52d80e0a", "transformed": true, "source_relative_path": "dy_product_scraping.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/dy_product_scraping.py" @@ -174,7 +174,7 @@ { "category": "production_source", "source_sha256": "26f3448e57a08020de0a96d1df3387819afc47f9319b81dbab27f1afc81bfb01", - "target_sha256": "536097c7558c2edb33639ab7ada22b266a1fe14341f6972dc53d3708f7b61fc0", + "target_sha256": "4c36f3eafd29241e1e5436c0a5f64682870a330003a99dcd03a2a981e425e8b8", "transformed": true, "source_relative_path": "erp_login_product_analysis.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/erp_login_product_analysis.py" @@ -190,7 +190,7 @@ { "category": "production_source", "source_sha256": "b828e1ad566b121bd56f90ef5de9d414cbe8575ec747cd26dfa3e4b66b097a9a", - "target_sha256": "c87aac33589bffc004eb83c899ca538529ef94afc8be912dbd8c164c87fc2d4f", + "target_sha256": "49da688998376bbd932195ef56603a37f342fa2eb307925a83592596641b8ace", "transformed": true, "source_relative_path": "export_bitable_records.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/export_bitable_records.py" @@ -206,7 +206,7 @@ { "category": "production_source", "source_sha256": "906278aa6fb3578d5dab231bdccc2f5a7c6ec8c85810a3e026fff3c7ba16dc90", - "target_sha256": "4f627cb824e9747e1436a5469158d62dd2a17b8442fcb45e64144f35618b27dd", + "target_sha256": "d9e4281c21a42841023c0a723aff4a00af1f9200c94e42add9f8965159d4cfb5", "transformed": true, "source_relative_path": "import_product_daily.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/import_product_daily.py" @@ -230,7 +230,7 @@ { "category": "production_source", "source_sha256": "e0f016bafb3f527f1cae46eaf3ef31510184128aaaa7f52024a7c3f978eec42b", - "target_sha256": "f30caa8e5925ee9d4c5addf3af651c4c79f94d564612922106f8693ecd7addd1", + "target_sha256": "f2692442a1e62a4b20dbb442eb3e27a3075ca6095ba2578c90193f0db3a1810e", "transformed": true, "source_relative_path": "jd_main_image_collector.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_main_image_collector.py" @@ -238,7 +238,7 @@ { "category": "production_source", "source_sha256": "d99ce71a008128549ac83fc3da52d88540b74538e3fe5c85d47c3b985e414353", - "target_sha256": "3aeec0f04d2d9d7fccf01632503a56066a58716e7dfc953a72241f1857fc8af4", + "target_sha256": "f821a0221b832bea0123bf4a8f85fd0785eba9206c8a632ad48065ea3c30c095", "transformed": true, "source_relative_path": "jd_product_data_collector.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py" @@ -246,7 +246,7 @@ { "category": "production_source", "source_sha256": "314790f06f7b4ebf9b0ddd5681db9c60e79a3ef16f5546d6e8489531a003c545", - "target_sha256": "06cac4bc1635268fec6274891a09344715b6dc38b95307b8d5bcf2e6a250bbf0", + "target_sha256": "2a254e00bb3ddd3a3d09a14599827e27d002786bf09e6d7004e25f5cabf787ef", "transformed": true, "source_relative_path": "jd_self_inventory_sales_collector.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_self_inventory_sales_collector.py" @@ -294,7 +294,7 @@ { "category": "production_source", "source_sha256": "910e720fab4f47419ad56055f00a8c3fbe3a06c2577d81e35d3d38178a588a45", - "target_sha256": "1dd67e2fbb95f08a77460c5b9f7a0cd9b51abae3454abe2ce138c257b00fc577", + "target_sha256": "c9f8cfc8dff6a7e5617b9b357ed92872f94371f93360318a3ff78f7ec38d75df", "transformed": true, "source_relative_path": "market_rank_product_import.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_product_import.py" @@ -310,7 +310,7 @@ { "category": "production_source", "source_sha256": "37edbfa24dfb126d52992b08fd6aeccae3ee18c83db56426a0fb02934e4541ae", - "target_sha256": "cade2f855043e05c1e07b86da81c4864a55e90e999e4441804055b2428e1c92f", + "target_sha256": "47489fd55cbd38a39a8cf3ac0673549e8eee9a39448c187bb34c5cd57b244b97", "transformed": true, "source_relative_path": "orchestrate_daily_collection.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py" @@ -358,7 +358,7 @@ { "category": "production_source", "source_sha256": "c9a51f1d1267b398cf260a86e65a8ee49de873d6c95bf9c90db2abb422005cf7", - "target_sha256": "9c8f0e7e686d7198bb231f87f85b074a00d0ffebf5d5d6b44582f59dddeb8abf", + "target_sha256": "6cd70bb5406bc00257f9070760f980580e95c3c4e4326a67d33d30433dd74c32", "transformed": true, "source_relative_path": "run_daily_persona.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/run_daily_persona.py" @@ -366,7 +366,7 @@ { "category": "production_source", "source_sha256": "27cd730400b954d1a063f034789f0d9a510c711ebda1e2de95e5a0010bd4751f", - "target_sha256": "dc9a2413b8aac548debcaf6bc5cafffa29e321ae44bb82d5df360cb02983e931", + "target_sha256": "fdbb40a10bdf95ef23a70c8574ac5ff0a52b3e419bde5694a186b2c00a9d45e4", "transformed": true, "source_relative_path": "run_weekly_jd_main_image.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/run_weekly_jd_main_image.py" @@ -374,7 +374,7 @@ { "category": "production_source", "source_sha256": "af9b436a31c3454dc6e0b34b77876962dab0b477e8a561db49911fdfc24eade3", - "target_sha256": "4e02f2394bf2b9b108104621b7c015e172002e284e0676ef55dd9c02d2346999", + "target_sha256": "e3b91d3c4b529291f7a6ceee5ff99326351d78104a66babc531f90faec6b7e1a", "transformed": true, "source_relative_path": "run_weekly_main_image.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/run_weekly_main_image.py" @@ -398,7 +398,7 @@ { "category": "production_source", "source_sha256": "b3e9addd72d8e31292ed948b67018aa9e212d628d7ba39141a81116515ca6721", - "target_sha256": "a17c6649f4665a09ae1ba0007936546866b2882b5e296733efd99e37d3120b2e", + "target_sha256": "369476667783566e47f086e920c593f3f26aa24355c5c9fa0a3f20bc22b3583e", "transformed": true, "source_relative_path": "taobao_dmp_item_crowd_insight_screenshots.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_dmp_item_crowd_insight_screenshots.py" @@ -406,7 +406,7 @@ { "category": "production_source", "source_sha256": "4474dd7d68117247af87e20bbeb6b7c072368ef0f606de5d3e186f459d34a0f5", - "target_sha256": "7e881c9a62227a3ec81b761067034055c5f79d02bec69bc23434a5f7aa13b72c", + "target_sha256": "cfbb46483113cda9e4cb301b092ac2ce6a35c5cc53b9a9cef01fdf93a00d3b94", "transformed": true, "source_relative_path": "taobao_sycm_collect.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_sycm_collect.py" @@ -414,7 +414,7 @@ { "category": "production_source", "source_sha256": "a881b02c903dc83c7f252d7f53fa9cd721fb562c690271788858ca5782427500", - "target_sha256": "d8957d1f93f807eadd1dd7c5a095485a83ce4eb492ec6218ab18cc71cb3cc7a7", + "target_sha256": "d64b440344af7cfcc51857e6261fed30f874b446bb2cb2c225d40da0b67da6a3", "transformed": true, "source_relative_path": "taobao_sycm_collect_backfill.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_sycm_collect_backfill.py" @@ -422,7 +422,7 @@ { "category": "production_source", "source_sha256": "9cafa62a03316fd0222f63addd5c209bb0c20fd0b7affa69cd1e256abfa2aa3a", - "target_sha256": "e20f104a12b1df763b5fde5ef6b8e2536a7801d70049b55cb84143afae21edf5", + "target_sha256": "cf6f78846238e3b63bcd07221cbc9ecbe1c5d2758f6147f48ff112c3ed97b23a", "transformed": true, "source_relative_path": "taobao_sycm_products.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_sycm_products.py" @@ -430,7 +430,7 @@ { "category": "production_source", "source_sha256": "3f0c32c443c87417c43881da69aa7776828ba973465f8d32f50b3dc698b0a4fe", - "target_sha256": "8d34bc9ea810b86e87e6f32404fca534a4dd6276012de4b29c385431c4730670", + "target_sha256": "548df6ddf3af3b3c066d18687114f1ce672afa054535f6da0c254d60f8306b86", "transformed": true, "source_relative_path": "taobao_wanxiang_ai_creative_report.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py" @@ -438,7 +438,7 @@ { "category": "production_source", "source_sha256": "e4d99956d9649d108baeb3e3a2d1969c227622d1037062bd277af66375556d82", - "target_sha256": "ee73ae4d58ba164422698e1bd0dd2da8510a5c059b8eb2d107be144a844fee4b", + "target_sha256": "981ef2a5a59f73ae244240520320c73be9c9462b50546df1ff91d0c97e7c03f7", "transformed": true, "source_relative_path": "upload_video_to_guanghe.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py" @@ -488,7 +488,7 @@ { "category": "production_source", "source_sha256": "6daa16c03989fb2231e54545726f5aa488e4ba80b9b8aa6f937cb9a2fa13ff35", - "target_sha256": "2080856b1c463a6a11e8e969ef7ff7d0816170402864526c60df026e40c9dd1d", + "target_sha256": "89ba008ef4388c6e1ed32e4a5c26ffa81d083164dd6d7231fc9e5886de8437cd", "transformed": true, "source_relative_path": "vendors/jd-data-flow/jd_data_collector.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_data_collector.py" @@ -496,7 +496,7 @@ { "category": "production_source", "source_sha256": "a53cc38b0ddeaa0065bc27a1d137ce16c2873ff9bec712483ca87bcedd6baa3a", - "target_sha256": "122f57cbdb76bbc366cfe93f50d7c6fbefa091b53664958acb2d142fbf2fa6f9", + "target_sha256": "0549228539c2e1908c3181de13291457712eeaea8fea89903814844724a3b5ee", "transformed": true, "source_relative_path": "vendors/jd-data-flow/jd_peer_product_data_collector.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_peer_product_data_collector.py" @@ -560,7 +560,7 @@ { "category": "runtime_resource", "source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746", - "target_sha256": "bfb13c05e4a0c7a85238200e026332f55d85a8022012cfab1db0c8514746def8", + "target_sha256": "8ff84f6c3c672af35e09f70d71aa73656ac101218d8a56126b8b0a75ab387c8b", "transformed": true, "source_relative_path": "db/schema.sql", "target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql" @@ -632,7 +632,7 @@ { "category": "regression_test", "source_sha256": "3d348a8250a2910f6a1c6504e9d1e53b0493d40bdc6c1fd1c8d0404c78681dc6", - "target_sha256": "dca35f4b197d2a7f77eb14e969e91974210b31100c5e874c842fd8036c4dd6e1", + "target_sha256": "dd1dd020cfcd350ef8223ae01ab45f67a86255a7a9abd6c00a863e3c082cfcfc", "transformed": true, "source_relative_path": "tests/test_guanghe_metadata.py", "target_relative_path": "tests/modules/product_commerce/test_guanghe_metadata.py" @@ -696,7 +696,7 @@ { "category": "regression_test", "source_sha256": "f1e660d34e21d3cd7a26d3605c82d07982ba6c5002af7b56fb6a876cb706713d", - "target_sha256": "025e80fce0ec954abb62119b81f96f40ba725f743badccd662d65f50434f903e", + "target_sha256": "85ca8603e2af8257ce2a940b99c69948c2c1362f688e6f6dac4cbf499f1e4d73", "transformed": true, "source_relative_path": "tests/test_market_rank_product_import.py", "target_relative_path": "tests/modules/product_commerce/test_market_rank_product_import.py" @@ -728,7 +728,7 @@ { "category": "regression_test", "source_sha256": "f9cfbb49fd313444790ac194fbf980dae21912081b9f26ad8e3850912126f09e", - "target_sha256": "76a0aeecf17ecea039ab4d81457cb2dc2cc0d0d75f54e500ea0f2a00ccce6f45", + "target_sha256": "0cc2b365144482ceb36667eedfb2c403e904437f3ea4c7e12f7099276c13a279", "transformed": true, "source_relative_path": "tests/test_style_analysis_orchestration.py", "target_relative_path": "tests/modules/product_commerce/test_style_analysis_orchestration.py" @@ -736,7 +736,7 @@ { "category": "regression_test", "source_sha256": "17761a5ba93a5387ffa67662424955178058aeec708bbcd3eb0341a859d8a1f1", - "target_sha256": "b88b52fb5e68f5ee44aa4f9709fbd2c7dae46126b98f76fdfd12615f284f1494", + "target_sha256": "549cb13dd32c9bed76d78eafa7542ea6e5c638d9f29c857eff4dde6d3b8e9c40", "transformed": true, "source_relative_path": "tests/test_style_config_loader_erp_source.py", "target_relative_path": "tests/modules/product_commerce/test_style_config_loader_erp_source.py" @@ -760,7 +760,7 @@ { "category": "regression_test", "source_sha256": "f7de702d7d1e641c8778970d5cb1f4e827a2e0ee9b20db2c19f0ed4fd8230e16", - "target_sha256": "25c72b4256fc4d0485b491fe691d44d78dac926f02316cc4d1b2bb7c3ee536a2", + "target_sha256": "55ea7031034b272f11e54e2372ad3476a25d0e32d8bbcc832b5977dfb7b34ec7", "transformed": true, "source_relative_path": "tests/test_tm_persona_recovery.py", "target_relative_path": "tests/modules/product_commerce/test_tm_persona_recovery.py" diff --git a/config/source-manifests/shop_intelligence.json b/config/source-manifests/shop_intelligence.json index 1059d69..9689689 100644 --- a/config/source-manifests/shop_intelligence.json +++ b/config/source-manifests/shop_intelligence.json @@ -32,7 +32,7 @@ "target_relative_path": "src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py", "category": "source", "source_sha256": "009879a7dd341ca6ecbc8eb66f5700114c73bcaba3a53f159014a76232d0aa6b", - "target_sha256": "e0b9664ad71c5c44132a730ccb274e30e6749229e912c5df816105aa7905405b", + "target_sha256": "d5e85ded1f852ba418229c75c985d0ece967dfbd5fd31697fe407b64d1c3db05", "transformed": true }, { @@ -40,7 +40,7 @@ "target_relative_path": "src/gyxx_flow/modules/shop_intelligence/collectors/jd_data_collector.py", "category": "source", "source_sha256": "6f00ae3b2b74e08d3c34c3b911ea819dfa4c6ce55bceef282afb3b2f565aa3ec", - "target_sha256": "6dc3a80a9d5869b82e5de4febec1fe920ebe518209fe1d0febafec732c3a97ac", + "target_sha256": "7d3cd4bdcb657d828f6cd1287df5755a53abeb24ba42170eb6fb6735cc586a4d", "transformed": true }, { @@ -48,7 +48,7 @@ "target_relative_path": "src/gyxx_flow/modules/shop_intelligence/collectors/jd_peer_store_data_collector.py", "category": "source", "source_sha256": "4ebdb52df7b384d2bfdd2fc840d55d14e18864002c20a36685b63cee1966cf4c", - "target_sha256": "6f626e52f52abd2f08441f70f3e9558e9cca41ebfb666438f4cf009cd17dccbe", + "target_sha256": "39faf18e9cd5e441323a8e056e022f179018a44eac224d64383cd7ae9fa1d611", "transformed": true }, { @@ -88,15 +88,15 @@ "target_relative_path": "src/gyxx_flow/modules/shop_intelligence/db/__init__.py", "category": "source", "source_sha256": "ffc89b78ff0081a129e04eb6de4985ab0a2a4613c9e8a1d8b2750d8c17e8bc05", - "target_sha256": "ffc89b78ff0081a129e04eb6de4985ab0a2a4613c9e8a1d8b2750d8c17e8bc05", - "transformed": false + "target_sha256": "a4bae02d0c07f3e3986896b31e62f41bf15edbf74d3776b1e050e381c15a8413", + "transformed": true }, { "source_relative_path": "db/db.py", "target_relative_path": "src/gyxx_flow/modules/shop_intelligence/db/db.py", "category": "source", "source_sha256": "168b6f87c1827b267f763ddce7eb15d0c8d652c8c0c7af18dcd0c6c7da5cddd1", - "target_sha256": "b8f15975605e2d9b75d25ac09a67768d4583401c1365288de078a97d0d3f3abf", + "target_sha256": "01469f9eb80aee5e80bc28e548dbe2cd5ce861ab35cd480b33921a2adc799320", "transformed": true }, { @@ -104,7 +104,7 @@ "target_relative_path": "src/gyxx_flow/modules/shop_intelligence/db/schema.sql", "category": "sql", "source_sha256": "73230590bde7ae6359c02c82c2db096c0d3faa55b374e30348c47d7b40ea498b", - "target_sha256": "a0aa2a22c8fdbb1f748dc7b206fe06a43f10899ca7036e433cdd91d7a1f5cba9", + "target_sha256": "a69fa151f989f106d00f344bab43ec9e6a09cf404bdc581624cc0901cdc13d20", "transformed": true }, { diff --git a/config/source-manifests/supply_chain.json b/config/source-manifests/supply_chain.json index ae83428..f622573 100644 --- a/config/source-manifests/supply_chain.json +++ b/config/source-manifests/supply_chain.json @@ -8,7 +8,7 @@ "target_relative_path": "src/gyxx_flow/modules/supply_chain/config.example.env", "category": "config", "source_sha256": "ec2df00f43abd9ba570b3882c5331d9b3c09557cccbc9c79e2326df57b680124", - "target_sha256": "3fbfd050b522a20802c91d195e305388f8d704fe60d1f24dab41096d74a28286", + "target_sha256": "3da1ee4935faab3997f6221a1f2540233285a561f35b97fccb82003cd7d58c02", "transformed": true }, { @@ -24,7 +24,7 @@ "target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/config.py", "category": "source", "source_sha256": "466791caa5439b29e6f37b92ee6afcd4018f2c7e3fbb072efe9c6832955b3400", - "target_sha256": "835298b3c05983ef215d76150364a1ca89b4b6ce539cced4b9387d3678b508bf", + "target_sha256": "479a4dde2639b488b49ddf7508ee9899004bb1204b529cb97f7153175c4376c7", "transformed": true }, { @@ -40,7 +40,7 @@ "target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/mcp_workflow.py", "category": "source", "source_sha256": "286e28c9db042956743d045fe40157d64c58ef70a85301c018a6eaedd8c2984d", - "target_sha256": "2eb41409ef1bb799204b27fabcba243a927ffc30c0e46328c8ccda83d185be41", + "target_sha256": "f2426fbdcd52135e1c52c194d654639bfd8697c44f7d3ecaefbfe17813b4eb6b", "transformed": true }, { @@ -64,7 +64,7 @@ "target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/runner.py", "category": "launcher", "source_sha256": "0cfe5e6557e8dee70ec4542c54b53e2153a3651eb2e35cd831e85518aabe0c66", - "target_sha256": "c3c16831a9c088455805b3a78fc23d276ded1cfcde8856f59cfdafceab061011", + "target_sha256": "5c99177d7c41ae45f6df517dc27b7d4e7413d5004e27e34948f5e169ec589ed3", "transformed": true }, { @@ -120,7 +120,7 @@ "target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/ProductReplenishment.py", "category": "source", "source_sha256": "02f6345142ebd84e25dc2f3b0651bc139ef31dd3994ff739efe66344c77ee49e", - "target_sha256": "32d40a9cbdaba73f88eb5ed181eca0af3bd02df9dc017e65162a4a9acb2f1a1f", + "target_sha256": "e74211f2fe97ac47e704a040f442f51c97fecc15ea30dd1f0dbeacaf5940fd80", "transformed": true }, { @@ -128,7 +128,7 @@ "target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/PurchaseConfirmation.py", "category": "source", "source_sha256": "d400571da971702302b6e426a0ba12981459a5ee7919c771d6f4ff0038617643", - "target_sha256": "91cf0ccb4150e9252c5048581a16e77534f251f17a3400ba4e1e4ad03e451941", + "target_sha256": "f9bcb7b2b911f0a7680e57a6fed5814663dac4d5c7bf5a0334a9c517aded3218", "transformed": true }, { @@ -136,7 +136,7 @@ "target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/PurchaseOrderUpdate.py", "category": "source", "source_sha256": "91617a2854fbf1ef38e9f8ef6fefaa34627b477d7a39098c43d6ee5dba296aa7", - "target_sha256": "0ff70f17c748cd9c892e6bb5b20c7083681edddb5097fb18ce0aa182996f29a1", + "target_sha256": "721027ab230d4503430a0f958a425babbfad8626890e6c1c2ce386f247f50f45", "transformed": true }, { @@ -144,7 +144,7 @@ "target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/send_card_notification.py", "category": "source", "source_sha256": "1397f5508a15ea7dae3d809aee9b0317b73788f3e7dc3927ff9efba5b7630a7e", - "target_sha256": "783ff97e7137e30ba7bcac72d53559563889e88fda21d0eb2b5217d48bf1ed53", + "target_sha256": "7b189ccc29074f72a2d02a45c720e0674ac3d8a7385bc6cdc99966ee6f505723", "transformed": true }, { diff --git a/config/workflows.json b/config/workflows.json index 72b27fc..1e79b51 100644 --- a/config/workflows.json +++ b/config/workflows.json @@ -10,7 +10,7 @@ { "id": "collect_collaborators", "name": "采集合作达人指标", - "description": "按每日范围并行抓取合作达人在 B站、小红书蒲公英和星图的曝光等指标，星图未命中的任务再用抖店达人广场补采；05:00 仅补 01:00 未完成的任务。", + "description": "按每日范围并行抓取合作达人在 B站、小红书蒲公英和星图的曝光等指标，星图未命中的任务再用抖店达人广场补采；05:00 仅补 01:00 未完成的任务。", "data_flow": { "sources": [ { @@ -19,7 +19,7 @@ }, { "system": "内容平台", - "label": "B站、小红书蒲公英与星图/抖店补采指标" + "label": "B站、小红书蒲公英与星图/抖店补采指标" } ], "processing": [ @@ -39,6 +39,57 @@ ], "replay_policy": "repeatable" }, + { + "id": "sync", + "name": "同步合作达人曝光指标入库", + "description": "只汇总合作达人本轮曝光结果，按记录映射更新或补建 PostgreSQL 内容笔记数据。", + "data_flow": { + "sources": [ + { + "system": "本地采集产物", + "label": "合作达人内容指标结果" + }, + { + "system": "飞书映射", + "label": "合作达人内容记录对应关系" + } + ], + "processing": [ + "读取合作达人各平台曝光指标", + "按内容记录身份更新已有数据或补建缺失记录" + ], + "destinations": [ + { + "system": "PostgreSQL", + "label": "合作达人笔记与曝光指标" + } + ] + }, + "entry": "data/tools/sync_metrics_to_cmt_notes.py", + "args": [ + "--scope", + "collaborators" + ], + "depends_on": [ + "collect_collaborators" + ], + "run_after_failure": true, + "replay_policy": "repeatable" + } + ] + }, + "provenance": { + "source_project": "content", + "task_name": "YingxiaoYunying_DailyRun" + }, + "note": "每日 01:00、05:00 采集合作达人内容指标并同步入库；自营内容指标由独立的 content.metrics.self_operated.daily 工作流负责。" + }, + { + "id": "content.metrics.self_operated.daily", + "module": "content_marketing", + "trigger": "scheduled", + "execution": { + "steps": [ { "id": "refresh_self_mapping", "name": "刷新自营款式映射", @@ -61,16 +112,12 @@ ] }, "entry": "data/tools/refresh_self_mapping.py", - "depends_on": [ - "collect_collaborators" - ], - "run_after_failure": true, "replay_policy": "repeatable" }, { "id": "collect_self_bilibili", "name": "采集自营 B站指标", - "description": "读取自营映射中的 B站笔记链接，通过公开 API 获取播放量和发布时间并回填对应飞书表。", + "description": "读取自营映射中的 B站笔记链接，通过公开 API 获取标题、UP主、时长、发布时间与播放/点赞/投币/收藏/分享/弹幕全量指标，回填对应飞书表并产出全量视频指标 artifact。", "data_flow": { "sources": [ { @@ -83,7 +130,8 @@ } ], "processing": [ - "按映射定位自营笔记并读取公开指标" + "按映射定位自营笔记并读取公开指标", + "按 BV 号聚合全量视频指标 artifact" ], "destinations": [ { @@ -92,7 +140,7 @@ }, { "system": "本地采集产物", - "label": "B站自营指标结果" + "label": "B站自营指标结果与全量视频 artifact" } ] }, @@ -106,7 +154,7 @@ { "id": "collect_self_douyin", "name": "采集自营抖音指标", - "description": "使用蝉妈妈新账号登录态导出自营抖音视频数据，解析曝光指标并回填飞书自营表。", + "description": "使用蝉妈妈新账号登录态导出自营抖音视频数据，解析视频链接、标题、发布时间、曝光、点赞、评论、转发等全量指标，并回填飞书自营曝光表。", "data_flow": { "sources": [ { @@ -119,7 +167,8 @@ } ], "processing": [ - "导出自营视频数据并解析曝光指标" + "导出自营视频数据并解析全量视频指标", + "按发布时间槽位回填飞书曝光" ], "destinations": [ { @@ -140,45 +189,116 @@ "replay_policy": "repeatable" }, { - "id": "sync", - "name": "同步合作与自营曝光指标入库", - "description": "汇总本轮合作达人及自营曝光结果，按记录映射更新或补建 PostgreSQL 内容笔记数据。", + "id": "collect_self_xiaohongshu", + "name": "采集自营小红书笔记指标", + "description": "复用 ARK 隔离登录态（失效时短信验证码需人工完成）进入小红书笔记管理后台，采集商品笔记与普通笔记两个页签的全部展示列（笔记信息、阅读数、点赞数、引流次数、支付金额与订单数等），产出全量笔记指标 artifact。", "data_flow": { "sources": [ { - "system": "本轮采集产物", - "label": "合作达人及自营内容指标" + "system": "小红书 ARK", + "label": "已登录的笔记管理后台（商品笔记/普通笔记）" }, { - "system": "本地映射", - "label": "内容记录对应关系" + "system": "本地配置", + "label": "自营小红书账号与登录手机号" } ], "processing": [ - "合并各平台指标并匹配内容记录", - "更新已有记录或补建缺失记录" + "按页签逐页抓取并按表头解析全部展示列", + "按笔记身份去重产出全量指标 artifact" + ], + "destinations": [ + { + "system": "本地采集产物", + "label": "小红书自营笔记全量指标 artifact" + } + ] + }, + "entry": "self_xhs_scraper.py", + "depends_on": [ + "collect_self_douyin" + ], + "run_after_failure": true, + "replay_policy": "repeatable" + }, + { + "id": "sync", + "name": "同步自营曝光指标入库", + "description": "只汇总自营内容曝光结果，按自营记录映射更新或补建 PostgreSQL 内容笔记数据。", + "data_flow": { + "sources": [ + { + "system": "本地采集产物", + "label": "自营内容指标结果" + }, + { + "system": "飞书映射", + "label": "自营内容记录对应关系" + } + ], + "processing": [ + "读取自营 B站和抖音曝光指标", + "按自营内容记录身份更新已有数据或补建缺失记录" ], "destinations": [ { "system": "PostgreSQL", - "label": "内容笔记与曝光指标" + "label": "自营笔记与曝光指标" } ] }, "entry": "data/tools/sync_metrics_to_cmt_notes.py", + "args": [ + "--scope", + "self-operated" + ], "depends_on": [ "collect_self_douyin" ], - "replay_policy": "repeatable", - "run_after_failure": true + "run_after_failure": true, + "replay_policy": "repeatable" + }, + { + "id": "sync_self_video_metrics", + "name": "同步自营内容指标入库", + "description": "按平台解析自营内容指标（抖音蝉妈妈导出、B站公开 API artifact、小红书 ARK artifact），映射内部账号别名后按账号+内容+业务日期幂等写入 PostgreSQL cmt_self_video_metrics（含 page_metrics 原始列快照）。", + "data_flow": { + "sources": [ + { + "system": "本地采集产物", + "label": "蝉妈妈自营抖音视频导出与 B站/小红书全量指标 artifact" + }, + { + "system": "本地配置", + "label": "自营账号别名映射" + } + ], + "processing": [ + "按平台解析来源产物的全量内容指标", + "按账号+内容+业务日期幂等 upsert" + ], + "destinations": [ + { + "system": "PostgreSQL", + "label": "自营内容每日指标（抖音/B站/小红书）" + } + ] + }, + "entry": "data/tools/sync_self_video_metrics.py", + "depends_on": [ + "sync", + "collect_self_xiaohongshu" + ], + "run_after_failure": true, + "replay_policy": "repeatable" } ] }, "provenance": { "source_project": "content", - "task_name": "YingxiaoYunying_DailyRun" + "task_name": "YingxiaoYunying_SelfOperated_DailyRun" }, - "note": "每日 01:00 执行合作达人及自营 B站、蝉妈妈采集，05:00 按 01:00 产生的未完成清单补采；账号切换期间仅暂停自营采集步骤，不移除自营链；两次均完成后同步曝光指标入库。" + "note": "每日 01:00、05:00 刷新自营映射并采集自营 B站、蝉妈妈、小红书 ARK 数据；飞书曝光回填保持不变，另将三平台全量内容指标（视频/笔记链接、标题、发布时间、曝光/阅读、点赞、评论、转发等）按日快照入库，失败时仍保留后续链路与入库汇总步骤。" }, { "id": "content.metrics.backfill", @@ -189,7 +309,7 @@ { "id": "collect_collaborators", "name": "补采合作达人指标", - "description": "按每日范围并行补采合作达人在 B站、小红书蒲公英和星图的曝光等指标，星图未命中的任务再用抖店达人广场补采。", + "description": "按每日范围并行补采合作达人在 B站、小红书蒲公英和星图的曝光等指标，星图未命中的任务再用抖店达人广场补采。", "data_flow": { "sources": [ { @@ -198,7 +318,7 @@ }, { "system": "内容平台", - "label": "B站、小红书蒲公英与星图/抖店补采指标" + "label": "B站、小红书蒲公英与星图/抖店补采指标" } ], "processing": [ @@ -245,7 +365,11 @@ "depends_on": [ "collect_collaborators" ], - "run_after_failure": true + "run_after_failure": true, + "args": [ + "--scope", + "collaborators" + ] } ] }, @@ -938,6 +1062,438 @@ }, "note": "每天采集计划日期前两天；品牌失败仍继续商品，任一步失败则工作流失败。" }, + { + "id": "shop.jd_customer_service.daily", + "module": "shop_intelligence", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "customer_messages", + "name": "采集京东客服聊天记录", + "description": "按控制台选择的业务日期打开京东客服聊天记录，切换消息视图，滚动加载全部顾客并逐个读取会话消息；全部顾客完成后才结束并幂等写入 PostgreSQL。", + "data_flow": { + "sources": [ + { + "system": "京东客服后台", + "label": "单日聊天记录消息视图" + } + ], + "processing": [ + "将开始和结束时间设置为业务日 00:00:00 至 23:59:59", + "滚动加载消息视图中的全部顾客并逐个点击读取会话", + "按会话标识和消息身份规范化，校验顾客覆盖完整后幂等入库" + ], + "destinations": [ + { + "system": "本地原始产物", + "label": "京东客服单日原始响应" + }, + { + "system": "PostgreSQL", + "label": "京东客服会话与消息表" + } + ] + }, + "entry": "collectors/jd_customer_service_messages.py", + "args": [ + "--date", + "{business_date}", + "--range-days", + "30", + "--headless" + ], + "timeout_seconds": 14400, + "replay_policy": "idempotent" + } + ] + }, + "provenance": { + "source_project": "shop", + "task_name": "京东客服消息日采集" + }, + "note": "默认关闭自动调度，控制台可按业务日期手动运行；单日必须完成消息视图中的全部顾客，失败时保留已完成进度并可安全重试。" + }, + { + "id": "shop.douyin_customer_service.daily", + "module": "shop_intelligence", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "customer_messages", + "name": "采集抖音客服历史会话消息", + "description": "复用抖音账号 vault 的 Cookie 跳过登录（指纹隐藏模式），打开飞鸽历史会话，按控制台选择的业务日期设置开始/结束日期，接待类型勾选纯人工客服接待与智能客服转人工接待，逐条点击查看会话读取聊天消息，解析抽屉右侧咨询订单面板的商品/订单并与会话绑定，全部幂等写入 PostgreSQL。", + "data_flow": { + "sources": [ + { + "system": "抖音飞鸽客服后台", + "label": "历史会话筛选列表与会话明细" + }, + { + "system": "抖音账号状态", + "label": "douyin-shop 账号 Cookie 与 Profile" + } + ], + "processing": [ + "将开始和结束日期设置为业务日期，接待类型勾选纯人工客服接待与智能客服转人工接待", + "按分页逐条点击查看会话，滚动加载抽屉内的完整聊天消息", + "按会话 ID 和消息身份规范化，解析咨询订单面板绑定商品/订单，校验会话覆盖完整后幂等入库" + ], + "destinations": [ + { + "system": "本地原始产物", + "label": "抖音客服单日会话行原始数据" + }, + { + "system": "PostgreSQL", + "label": "抖音客服会话与消息表" + } + ] + }, + "entry": "collectors/dy_customer_service_messages.py", + "args": [ + "--date", + "{business_date}", + "--range-days", + "30", + "--headless" + ], + "timeout_seconds": 14400, + "replay_policy": "idempotent" + } + ] + }, + "provenance": { + "source_project": "shop", + "task_name": "抖音客服消息日采集" + }, + "note": "每月 1 日凌晨自动采集近 30 天；控制台支持自定义日期范围；单日必须完成筛选结果中的全部会话，失败时保留已完成进度并可安全重试。" + }, + { + "id": "shop.tmall_customer_service.daily", + "module": "shop_intelligence", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "customer_messages", + "name": "采集天猫客服聊天记录", + "description": "复用光影行星旗舰店:龙虾仔账号 vault 的 Cookie，使用 Scrapling 隐藏指纹浏览器打开千牛聊天记录查询页；按控制台选择日期范围，每次按不超过 31 天拆分查询，逐个会话读取消息并幂等写入 PostgreSQL。", + "data_flow": { + "sources": [ + { + "system": "天猫千牛客服后台", + "label": "聊天记录查询列表与会话消息" + }, + { + "system": "天猫账号状态", + "label": "tmall-shop 账号 vault Cookie 与独立 Profile" + } + ], + "processing": [ + "限制查询范围为最近三个月，并拆分为每次最多 31 天的日期窗口", + "滚动加载查询结果中的全部会话，逐个选中并读取消息时间、发送人、文本及媒体链接", + "按账号、顾客、会话和消息指纹规范化；保存每个查询窗口的原始产物并幂等写入 PostgreSQL" + ], + "destinations": [ + { + "system": "本地原始产物", + "label": "天猫客服会话与消息 JSON" + }, + { + "system": "PostgreSQL", + "label": "天猫客服会话与消息表" + } + ] + }, + "entry": "collectors/tmall_customer_service_messages.py", + "args": [ + "--start-date", + "{business_date}", + "--end-date", + "{business_date}", + "--headless" + ], + "timeout_seconds": 14400, + "replay_policy": "idempotent" + } + ] + }, + "provenance": { + "source_project": "shop", + "task_name": "天猫客服聊天记录采集" + }, + "note": "控制台可选择最近三个月内的开始和结束日期；每次平台查询不超过 31 天，按窗口顺序采集。默认不自动调度，失败窗口保留已完成数据，可安全重试。" + }, + { + "id": "shop.tmall_negative_reviews.daily", + "module": "shop_intelligence", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "negative_reviews", + "name": "采集天猫差评及商品信息", + "description": "复用 tmall-shop 账号 vault Cookie，使用 Scrapling 隐藏指纹浏览器打开千牛评价管理；先设置控制台选择的评价时间范围并固定情感分类为负面评价（emotion=13），再点击搜索；分页采集评论文字、图片、主评 ID、评价时间和商品名称，不读取或保存订单号，核对总数后幂等写入 PostgreSQL。", + "data_flow": { + "sources": [ + { + "system": "天猫千牛评价管理", + "label": "负面评价列表" + }, + { + "system": "天猫账号状态", + "label": "tmall-shop vault Cookie 与隔离 Profile" + } + ], + "processing": [ + "将控制台选定的起止日期设置为评价时间，并选择负面评价后再搜索；日期不留空，避免页面执行全日期查询", + "分页读取评价文字、图片链接、主评 ID、评价时间和关联商品名称，不采集订单号", + "核对页面总数和评价 ID 覆盖完整后，按账号及主评 ID 幂等写入 PostgreSQL" + ], + "destinations": [ + { + "system": "本地原始产物", + "label": "天猫差评范围采集 JSON" + }, + { + "system": "PostgreSQL", + "label": "天猫差评及商品信息表" + } + ] + }, + "entry": "collectors/tmall_negative_reviews.py", + "args": [ + "--start-date", + "{business_date}", + "--end-date", + "{business_date}", + "--headless" + ], + "timeout_seconds": 14400, + "replay_policy": "idempotent" + } + ] + }, + "provenance": { + "source_project": "shop", + "task_name": "天猫差评范围采集" + }, + "note": "控制台选择实际评价时间范围；页面不选择评价时间会查询所有日期，因此工作流会先填入日期并固定筛选负面评价后再搜索。默认不自动调度。" + }, + { + "id": "shop.dy_negative_reviews.daily", + "module": "shop_intelligence", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "negative_reviews", + "name": "采集抖音差评及商品信息", + "description": "复用抖音账号 vault 的 Cookie（指纹隐藏模式），打开抖店评价管理-评价明细，筛选差评并把评价时间设置为控制台选择的业务日，逐页读取评价文字、图片、评价状态与商品名称/商品ID/规格，全部页面完成后幂等写入 PostgreSQL。", + "data_flow": { + "sources": [ + { + "system": "抖店评价管理后台", + "label": "差评评价列表" + } + ], + "processing": [ + "点击评价等级「差评」并把评价时间设置为业务日 00:00:00-23:59:59", + "按分页完整读取评价文字、图片与商品信息", + "按账号、业务日期和评价（订单）ID 校验完整后幂等入库" + ], + "destinations": [ + { + "system": "本地原始产物", + "label": "抖音差评单日原始数据" + }, + { + "system": "PostgreSQL", + "label": "抖音差评及商品信息表" + } + ] + }, + "entry": "collectors/dy_negative_reviews.py", + "args": [ + "--date", + "{business_date}", + "--range-days", + "30", + "--headless" + ], + "timeout_seconds": 14400, + "replay_policy": "idempotent" + } + ] + }, + "provenance": { + "source_project": "shop", + "task_name": "抖音差评日采集" + }, + "note": "每月 1 日凌晨自动采集近 30 天（含当天）；控制台支持自定义日期范围，范围按业务日逐日采集；评价历史仅保留近 30 天，超出范围按 0 条成功；相同账号和日期重复执行按评价（订单）ID 幂等。" + }, + { + "id": "shop.douyin_after_sales_quality.daily", + "module": "shop_intelligence", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "after_sales_quality", + "name": "采集抖音品质原因售后单", + "description": "复用抖音账号 vault 的 Cookie 跳过登录（Scrapling 隐藏指纹模式），打开售后工作台，通过更多筛选把申请原因固定为商品品质原因，可按控制台自定义申请时间范围（不设置时间则为页面默认最近6个月全量查询），逐条点击查看详情，在新页签读取协商记录的退货原因、退款说明、凭证信息和交易信息的商品名称、商品ID、商家编码，按申请时间归属业务日后幂等写入 PostgreSQL。", + "data_flow": { + "sources": [ + { + "system": "抖音售后工作台", + "label": "品质原因售后列表与售后详情协商记录" + }, + { + "system": "抖音账号状态", + "label": "douyin-shop 账号 Cookie 与 Profile" + } + ], + "processing": [ + "展开更多筛选并将申请原因固定为商品品质原因，可设置申请时间开始/结束日期", + "按分页逐条点击查看详情，在新页签展开协商记录并读取退货原因、退款说明、凭证信息与商品信息", + "按申请时间的上海日期归属业务日，以账号和售后编号幂等入库，协商记录按条目序号幂等保存" + ], + "destinations": [ + { + "system": "本地原始产物", + "label": "抖音品退售后单原始数据" + }, + { + "system": "PostgreSQL", + "label": "抖音品质原因售后单与协商记录表" + } + ] + }, + "entry": "collectors/dy_after_sales_quality.py", + "args": [ + "--date", + "{business_date}", + "--range-days", + "30", + "--headless" + ], + "timeout_seconds": 14400, + "replay_policy": "idempotent" + } + ] + }, + "provenance": { + "source_project": "shop", + "task_name": "抖音品退售后单采集" + }, + "note": "每月 1 日凌晨自动采集近 30 天；控制台支持自定义日期范围；不设置申请时间时按页面默认查询最近6个月做全量采集。每条售后单按申请时间归属业务日，重复执行按账号和售后编号幂等更新，失败时保留已完成进度并可安全重试。" + }, + { + "id": "shop.jd_negative_reviews.daily", + "module": "shop_intelligence", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "negative_reviews", + "name": "采集京东差评及商品信息", + "description": "按控制台选择的京东店铺和业务日期打开商品评价管理，选择差评并分页读取评价内容、评价时间、订单号、商品名称、SKU 和图片；全部页面完成后幂等写入 PostgreSQL。", + "data_flow": { + "sources": [ + { + "system": "京东商品评价后台", + "label": "差评评价列表" + } + ], + "processing": [ + "设置业务日 00:00:00 至 23:59:59 并选择差评", + "按分页完整读取评价与商品信息", + "按账号、业务日期和评价 ID 校验完整后幂等入库" + ], + "destinations": [ + { + "system": "本地原始产物", + "label": "京东差评单日原始响应" + }, + { + "system": "PostgreSQL", + "label": "京东差评及商品信息表" + } + ] + }, + "entry": "collectors/jd_negative_reviews.py", + "args": [ + "--date", + "{business_date}", + "--range-days", + "30", + "--headless" + ], + "timeout_seconds": 14400, + "replay_policy": "idempotent" + } + ] + }, + "provenance": { + "source_project": "shop", + "task_name": "京东差评日采集" + }, + "note": "每月 1 日凌晨自动采集近 30 天（含当天）；控制台支持按店铺选择日期范围，范围按业务日逐日采集；相同账号、店铺和日期重复执行按评价 ID 幂等。" + }, + { + "id": "shop.jd_after_sales_issues.daily", + "module": "shop_intelligence", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "after_sales_issues", + "name": "采集京东品退异常问题", + "description": "按控制台选择的日期范围打开京东商智售后分析，筛选商品出现异常并查看全部服务单明细；点击下载数据，读取全部分页，只保留用户原声不是未知的服务单及商品名称、SKU 等信息，按服务单申请日期归属业务日后幂等写入 PostgreSQL。", + "data_flow": { + "sources": [ + { + "system": "京东商智", + "label": "售后分析异常服务单明细" + } + ], + "processing": [ + "将售后分析时间设置为控制台提交的自定义日期范围；商智单次最多 31 天时按连续区间查询", + "筛选售后原因商品出现异常并读取全部服务单明细分页", + "过滤用户原声为未知或空的记录，按服务单申请日期归属业务日，并以账号、业务日和服务单 ID 幂等入库；范围完成后清理同账号范围内申请时间落在范围外的历史残留" + ], + "destinations": [ + { + "system": "本地原始产物", + "label": "京东品退日期范围明细及下载审计" + }, + { + "system": "PostgreSQL", + "label": "京东品退异常服务单表" + } + ] + }, + "entry": "collectors/jd_after_sales_issues.py", + "args": [ + "--date", + "{business_date}", + "--range-days", + "30", + "--headless" + ], + "timeout_seconds": 14400, + "replay_policy": "idempotent" + } + ] + }, + "provenance": { + "source_project": "shop", + "task_name": "京东品退问题日采集" + }, + "note": "每月 1 日凌晨自动采集近 30 天（含当天）；控制台固定使用研发旗舰账号并支持直接提交日期范围。一次范围运行会在商智最多 31 天限制下按连续区间查询，完成后按服务单申请日期写入业务日并清理范围外申请时间的历史残留；相同账号、业务日和服务单 ID 重复执行按幂等更新。" + }, { "id": "shop.douyin_price_appeal", "module": "shop_intelligence", @@ -986,7 +1542,7 @@ "source_project": "shop", "task_name": "douyin-price-appeal-daily" }, - "note": "每天 08:00、16:00、22:00 自动执行；仍可在控制台手动执行。每个时点都会重新扫描，已申诉商品不会再次出现可申诉入口；安全预演不访问抖音也不提交申诉。" + "note": "每小时自动执行；仍可在控制台手动执行。每个时点都会重新扫描，已申诉商品不会再次出现可申诉入口；安全预演不访问抖音也不提交申诉。" }, { "id": "product.persona.daily", @@ -1138,6 +1694,46 @@ "task_name": "ProductCollectorSalesAlert" } }, + { + "id": "product.online_stock_alert.daily", + "module": "product_commerce", + "trigger": "scheduled", + "execution": { + "steps": [ + { + "id": "query_filter_notify", + "name": "查询线上商品并发送低库存预警", + "description": "使用企业数据交互商品列表接口查询线上渠道全部商品；库存低于 3 时，将 SKU、商品名称和库存汇总发送给配置的飞书接收人。", + "replay_policy": "idempotent", + "data_flow": { + "sources": [ + { + "system": "企业数据交互接口", + "label": "线上渠道商品与当前库存" + } + ], + "processing": [ + "以 channel=0、keyword=空查询商品列表", + "筛选 total_stock 小于 3 的 SKU", + "按飞书消息幂等键去重并分段组织预警" + ], + "destinations": [ + { + "system": "飞书", + "label": "低库存 SKU、商品名称与库存预警", + "condition": "存在库存低于 3 的 SKU" + } + ] + }, + "entry": "online_stock_alert.py" + } + ] + }, + "provenance": { + "source_project": "data-hub", + "task_name": "OnlineItemStockAlert" + } + }, { "id": "product.ecommerce_costs.daily", "module": "product_commerce", @@ -2222,7 +2818,7 @@ { "id": "collect_analyze_publish", "name": "采集并发布三平台市场排行", - "description": "并行采集天猫、京东和抖音市场排行，汇总平台结果并发送飞书报告链接。", + "description": "并行采集天猫、京东和抖音市场排行，按月份写入品牌行军图数据库并复用已有人工分类。", "data_flow": { "sources": [ { @@ -2236,8 +2832,8 @@ ], "processing": [ "并行采集三平台排行", - "汇总平台结果并生成市场分析", - "归档报告并准备通知" + "按月份覆盖榜单快照并复用商品的风格、使用场景和品类", + "归档数据库结果并记录平台完成状态" ], "destinations": [ { @@ -2245,8 +2841,8 @@ "label": "三平台市场排行结果" }, { - "system": "飞书", - "label": "市场排行报告链接与通知" + "system": "PostgreSQL / 品牌行军图", + "label": "market_product 与 fact_market_product_snapshot 月度快照" } ] }, diff --git a/docs/deployment.md b/docs/deployment.md index d5b7fb4..8a51fa5 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -60,7 +60,7 @@ Scrapling 安装和管理。具体边界与验证命令见 GYXX_DATA_ROOT=D:\gyxx-flow-data GYXX_POSTGRES_DSN=<云端 PostgreSQL DSN> WANXIANG_ACCOUNT=<万相台账号> -WANXIANG_PASSWORD=<万相台密码> +WANXIANG_PASSWORD= ``` 未显式设置 `WANXIANG_USER_DATA_DIR` 时，登录态保存在 @@ -84,7 +84,7 @@ nssm start gyxx-flow-scheduler ```powershell $env:GYXX_DATA_ROOT = 'D:\gyxx-flow-data' $env:WANXIANG_ACCOUNT = '<从密码管理器读取>' -$env:WANXIANG_PASSWORD = '<从密码管理器读取>' +$env:WANXIANG_PASSWORD = '' uv run gyxx doctor --json uv run gyxx scripts run product.tmall_wanxiang_ads.collect --date 2026-08-17 --execute uv run gyxx scripts run product.import.tmall_ads --date 2026-08-17 --execute diff --git a/docs/dynamic-workflow-config.md b/docs/dynamic-workflow-config.md index a0218ea..a80a251 100644 --- a/docs/dynamic-workflow-config.md +++ b/docs/dynamic-workflow-config.md @@ -22,7 +22,8 @@ PostgreSQL 的动态配置表。生命进程负责款式名、品牌、ERP 款 | --- | --- | --- | | `content.summary.monthly` | 款式与每周笔记分析/生命进程目标表 | `monthly_summary_all.py`（复用周汇总配置加载器） | | `content.summary.weekly` | 款式与每周笔记分析/生命进程目标表 | `weekly_summary_all.py` | -| `content.metrics.daily` | 合作达人目标表（自营采集暂缓） | `run_all.py`、`sync_metrics_to_cmt_notes.py` | +| `content.metrics.daily` | 合作达人目标表 | `run_all.py`、`sync_metrics_to_cmt_notes.py` | +| `content.metrics.self_operated.daily` | 自营账号、款式映射与自营目标表 | `refresh_self_mapping.py`、`self_bilibili_scraper.py`、`chanmama_scraper.py`、`sync_metrics_to_cmt_notes.py` | | `content.metrics.backfill` | 合作达人目标表 | `run_all.py` | | `content.notes_master.daily` | 各款式合作达人及自营笔记表 | `sync_notes_master.py` | | `product.persona.daily` | 三平台商品 ID 与人群画像目标表 | 三个平台画像采集脚本 | @@ -32,7 +33,7 @@ PostgreSQL 的动态配置表。生命进程负责款式名、品牌、ERP 款 | `product.sales_sheet.daily` | 款式与 ERP 编码 | `sync_monthly_sales_sheet.py` | | `product.erp_all_shop_daily` | 全部款式与 ERP 编码 | `backfill_erp_all_shop_daily.py` | -共 11 个定时工作流依赖这套动态配置。维护脚本 `feishu_comment_batch.py` 和 +共 12 个定时工作流依赖这套动态配置。维护脚本 `feishu_comment_batch.py` 和 `db/sync_sku_master.py` 也已改为读取同一项目数据库，避免从非定时入口绕回旧主表。 `product.alert.daily`、商品导入、万相台广告、视频发布和市场排行使用各自数据库或专用配置， diff --git a/docs/workflow-acceptance-test-requirements.md b/docs/workflow-acceptance-test-requirements.md index 9542b43..7b5abbc 100644 --- a/docs/workflow-acceptance-test-requirements.md +++ b/docs/workflow-acceptance-test-requirements.md @@ -71,7 +71,8 @@ ## 8. 内容工作流 -- [ ] `content.metrics.daily`：采集合作达人、自营 B 站、蝉妈妈数据并同步指标。`PARTIAL_FAILED`；run_id `content.metrics.daily__20260813__20260801T101312Z__28d505`，运行 71 分 05 秒后完整度检查失败；已完成子链和飞书写跳过证据保留，但必要采集链未全部完成。 +- [ ] `content.metrics.daily`：只采集合作达人并同步合作指标。历史混合工作流 run_id `content.metrics.daily__20260813__20260801T101312Z__28d505` 为 `PARTIAL_FAILED`，运行 71 分 05 秒后完整度检查失败；拆分后的合作链尚未业务验收。 +- [ ] `content.metrics.self_operated.daily`：刷新自营映射，依序采集自营 B 站（公开 API 全量视频指标）、蝉妈妈（抖音全量视频指标）与小红书 ARK（商品笔记+普通笔记全量指标，首次登录需人工完成短信验证码），同步自营曝光结果，并将三平台全量指标按账号+内容+业务日期写入 PostgreSQL `cmt_self_video_metrics`（抖音·机能号/抖音·高智号/小红书·自营号账号映射）；拆分后尚无业务验收 run_id。 - [x] `content.marketing_report.daily`：从本地数据生成营销日报并按条件通知。`PASS_WITH_SKIPS`；实际入口 run_id `script.content_marketing.62b3b647254f__20260804__20260801T091245Z__f20319`。真实读取本地 PG、调用本地 Hermes，生成 Markdown/PNG；未启用发送参数。 - [x] `content.relogin.weekly`：并行刷新内容平台登录态。`PASS_WITH_SKIPS`；run_id `content.relogin.weekly__20260811__20260801T100706Z__45a8db`。LangGraph 和验收门禁按设计阻止二维码/交互登录并安全记录跳过；没有扫码、没有覆盖 Cookie 或来源状态。 - [x] `content.creator_report.monthly`：生成达人月报。`PASS_WITH_SKIPS`；workflow run_id `content.creator_report.monthly__20260801__20260801T084140Z__bd9fdd`。本地数据库检查和实现入口 dry-run 成功，飞书表写入关闭。 diff --git a/docs/workflow-time-distribution.md b/docs/workflow-time-distribution.md new file mode 100644 index 0000000..e17f7e3 --- /dev/null +++ b/docs/workflow-time-distribution.md @@ -0,0 +1,218 @@ +# 工作流时间分布节点汇总 + +> 快照日期：2026-09-21
+> 配置时区：`Asia/Shanghai`
+> 配置来源：`config/workflows.json`、`config/schedules.json`
+> 统计范围：当前工作树中的全部工作流与调度计划 + +## 1. 统计口径 + +- “工作流”以 `config/workflows.json` 中的 `workflows` 为准；当前 35 条全部是 `scheduled` 工作流。 +- “时间节点”指同一时区下去重后的墙上时钟时间，例如 `10:00` 是一个节点。 +- “启动项”指一条工作流与一个时间的组合；`at` 为数组时逐个展开。因此两个“每小时整点”计划各贡献 24 个启动项。 +- 暂停计划仍保留在时间分布中，并标记为“暂停”，因为它们仍是有效配置项；只有“启用”计划才会被调度器实际触发。 +- `enabled` 未填写时，按目录校验规则视为启用。 +- “业务日偏移”保留配置中的 `business_date_offset_days` 原值，不把它解释成自然日执行时间。 + +## 2. 总览 + +| 指标 | 数值 | +| --- | ---: | +| 工作流总数 | 35 | +| 调度计划总数 | 35 | +| 启用工作流 | 22 | +| 暂停工作流 | 13 | +| 展开后的启动项 | 82 | +| 启用启动项 | 69 | +| 暂停启动项 | 13 | +| 独立时间节点 | 30 | +| 配置时区 | `Asia/Shanghai` | + +### 2.1 按模块统计 + +| 模块 | 工作流数 | 启用 | 暂停 | +| --- | ---: | ---: | ---: | +| 内容营销（`content_marketing`） | 9 | 3 | 6 | +| 商品经营（`product_commerce`） | 12 | 12 | 0 | +| 店铺洞察（`shop_intelligence`） | 10 | 4 | 6 | +| 供应链（`supply_chain`） | 4 | 3 | 1 | +| **合计** | **35** | **22** | **13** | + +### 2.2 按计划类型统计 + +| 计划类型 | 工作流数 | 启用 | 暂停 | +| --- | ---: | ---: | ---: | +| 每天（`daily`） | 19 | 12 | 7 | +| 每周（`weekly`） | 9 | 5 | 4 | +| 每月（`monthly`） | 4 | 2 | 2 | +| 间隔天数（`interval_days`） | 3 | 3 | 0 | +| **合计** | **35** | **22** | **13** | + +## 3. 时间分布节点 + +下表按一天内的时钟时间排序。每个节点的“启动项数”包含暂停计划；“启用数”才是当前会被调度器执行的数量。 + +| 时间节点 | 启动项数 | 启用数 | 暂停数 | 工作流 | +| --- | ---: | ---: | ---: | --- | +| `00:00` | 2 | 2 | 0 | 抖音待改价申诉 [`shop.douyin_price_appeal`]；天猫百亿补贴批量报名 [`product.tmall_baibu_apply`] | +| `01:00` | 3 | 3 | 0 | 内容指标日报采集 [`content.metrics.daily`]；抖音待改价申诉；天猫百亿补贴批量报名 | +| `02:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `03:00` | 3 | 2 | 1 | 京东客服消息采集 [`shop.jd_customer_service.daily`]（暂停）；抖音待改价申诉；天猫百亿补贴批量报名 | +| `03:10` | 1 | 0 | 1 | 抖音客服消息采集 [`shop.douyin_customer_service.daily`]（暂停） | +| `03:20` | 2 | 0 | 2 | 抖音品退采集 [`shop.douyin_after_sales_quality.daily`]（暂停）；抖音差评采集 [`shop.dy_negative_reviews.daily`]（暂停） | +| `03:30` | 1 | 0 | 1 | 京东差评采集 [`shop.jd_negative_reviews.daily`]（暂停） | +| `04:00` | 3 | 2 | 1 | 京东品退问题采集 [`shop.jd_after_sales_issues.daily`]（暂停）；抖音待改价申诉；天猫百亿补贴批量报名 | +| `05:00` | 3 | 3 | 0 | 内容指标日报采集；抖音待改价申诉；天猫百亿补贴批量报名 | +| `06:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `07:00` | 3 | 3 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名；补货预警日处理 [`supply.replenishment_alert.daily`] | +| `08:00` | 5 | 3 | 2 | 内容月度汇总 [`content.summary.monthly`]（暂停）；抖音待改价申诉；天猫百亿补贴批量报名；采购确认日处理 [`supply.purchase_confirmation.daily`]；补货建议周处理 [`supply.replenishment.weekly`]（暂停） | +| `08:30` | 2 | 1 | 1 | 达人月报生成 [`content.creator_report.monthly`]（暂停）；京东与天猫主图周采集 [`product.main_image.weekly`] | +| `08:40` | 1 | 1 | 0 | 商品经营日报采集 [`product.daily`] | +| `09:00` | 6 | 6 | 0 | 笔记清单与合作同步 [`content.notes_master.daily`]；京东自营业绩日采集 [`shop.jd_self_operated.daily`]；抖音待改价申诉；聚水潭全店铺款式日报 [`product.erp_all_shop_daily`]；天猫百亿补贴批量报名；补货建议预计到货同步 [`supply.replenishment_arrival_sync.daily`] | +| `10:00` | 8 | 5 | 3 | 营销日报生成与发送 [`content.marketing_report.daily`]（暂停）；内容平台登录态刷新 [`content.relogin.weekly`]（暂停）；内容周度汇总 [`content.summary.weekly`]（暂停）；抖音待改价申诉；三平台电商费用采集 [`product.ecommerce_costs.daily`]；商品月度销量表 [`product.sales_sheet.daily`]；天猫百亿补贴批量报名；市场排行周采集 [`product.market_rank`] | +| `11:00` | 3 | 3 | 0 | 抖音待改价申诉；款式周期分析 [`product.style_analysis.interval`]；天猫百亿补贴批量报名 | +| `12:00` | 5 | 5 | 0 | 多平台评论周采集 [`content.comments.weekly`]；店铺经营指标周采集 [`shop.metrics.weekly`]；抖音待改价申诉；商品人群画像日采集 [`product.persona.daily`]；天猫百亿补贴批量报名 | +| `12:30` | 1 | 1 | 0 | 竞店指标周采集 [`shop.competitor.weekly`] | +| `13:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `14:00` | 5 | 4 | 1 | 内容指标补采 [`content.metrics.backfill`]（暂停）；抖音待改价申诉；上传所选天猫店铺短视频 [`product.video_upload`]；扫描新增视频并发布到京东 [`product.jd_video_upload`]；天猫百亿补贴批量报名 | +| `15:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `16:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `17:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `18:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `19:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `20:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `21:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `22:00` | 2 | 2 | 0 | 抖音待改价申诉；天猫百亿补贴批量报名 | +| `23:00` | 3 | 3 | 0 | 抖音待改价申诉；商品销量异常告警 [`product.alert.daily`]；天猫百亿补贴批量报名 | + +### 3.1 时间节点特征 + +- `00:00`–`23:00` 的 24 个整点节点全部被覆盖，基础来源是两个每小时整点计划：抖音待改价申诉、天猫百亿补贴批量报名。 +- 最拥挤的节点是 `10:00`：共 8 个启动项，其中 5 个启用、3 个暂停。 +- 其次是 `09:00`：6 个启动项，全部启用；`08:00`、`12:00`、`14:00` 各 5 个启动项。 +- 非整点节点共有 5 个：`03:10`、`03:30`、`08:30`、`08:40`、`12:30`。其中 `03:10`、`03:30` 的唯一计划当前暂停。 + +## 4. 按工作流分类的调度清单 + +### 4.1 内容营销（`content_marketing`，9 条） + +- 指标采集：`content.metrics.daily`（内容指标日报采集）、`content.metrics.backfill`（内容指标补采）。 +- 报告与汇总：`content.marketing_report.daily`（营销日报生成与发送）、`content.creator_report.monthly`（达人月报生成）、`content.summary.monthly`（内容月度汇总）、`content.summary.weekly`（内容周度汇总）。 +- 内容同步与互动：`content.notes_master.daily`（笔记清单与合作同步）、`content.comments.weekly`（多平台评论周采集）、`content.relogin.weekly`（内容平台登录态刷新）。 + +### 4.2 商品经营（`product_commerce`，12 条） + +- 日报、经营与费用：`product.daily`（商品经营日报采集）、`product.erp_all_shop_daily`（聚水潭全店铺款式日报）、`product.ecommerce_costs.daily`（三平台电商费用采集）、`product.sales_sheet.daily`（商品月度销量表）、`product.alert.daily`（商品销量异常告警）、`product.market_rank`（市场排行周采集）。 +- 人群、款式与主图：`product.persona.daily`（商品人群画像日采集）、`product.style_analysis.interval`（款式周期分析）、`product.main_image.weekly`（京东与天猫主图周采集）。 +- 视频与报名：`product.video_upload`（上传所选天猫店铺短视频）、`product.jd_video_upload`（扫描新增视频并发布到京东）、`product.tmall_baibu_apply`（天猫百亿补贴批量报名）。 + +### 4.3 店铺洞察（`shop_intelligence`，10 条） + +- 店铺与竞店经营：`shop.metrics.weekly`（店铺经营指标周采集）、`shop.competitor.weekly`（竞店指标周采集）、`shop.jd_self_operated.daily`（京东自营业绩日采集）。 +- 客服、评价与申诉：`shop.jd_customer_service.daily`（京东客服消息采集）、`shop.jd_negative_reviews.daily`（京东差评采集）、`shop.jd_after_sales_issues.daily`（京东品退问题采集）、`shop.douyin_customer_service.daily`（抖音客服消息采集）、`shop.douyin_after_sales_quality.daily`（抖音品退采集）、`shop.dy_negative_reviews.daily`（抖音差评采集）、`shop.douyin_price_appeal`（抖音待改价申诉）。 + +### 4.4 供应链（`supply_chain`，4 条） + +- 采购与补货：`supply.purchase_confirmation.daily`（采购确认日处理）、`supply.replenishment.weekly`（补货建议周处理）、`supply.replenishment_alert.daily`（补货预警日处理）、`supply.replenishment_arrival_sync.daily`（补货建议预计到货同步）。 + +### 4.5 全量调度明细 + +“步骤数”是 `config/workflows.json` 中该工作流的执行步骤数量，不代表执行时长。 + +| # | 工作流 ID | 工作流名称 / 主要用途 | 模块 | 调度规则 | 业务日偏移 | 状态 | 步骤数 | +| ---: | --- | --- | --- | --- | ---: | --- | ---: | +| 1 | `content.metrics.daily` | 内容指标日报采集 | 内容营销 | 每天 `01:00`、`05:00` | 0 | 启用 | 5 | +| 2 | `content.metrics.backfill` | 内容指标补采 | 内容营销 | 每周一 `14:00` | 0 | 暂停 | 2 | +| 3 | `content.marketing_report.daily` | 营销日报生成与发送 | 内容营销 | 每天 `10:00` | 0 | 暂停 | 1 | +| 4 | `content.relogin.weekly` | 内容平台登录态刷新 | 内容营销 | 每周五 `10:00` | 0 | 暂停 | 1 | +| 5 | `content.creator_report.monthly` | 达人月报生成 | 内容营销 | 每月 1 日 `08:30` | 0 | 启用 | 1 | +| 6 | `content.summary.monthly` | 内容月度汇总 | 内容营销 | 每月 1 日 `08:00` | 0 | 启用 | 1 | +| 7 | `content.notes_master.daily` | 笔记清单与合作同步 | 内容营销 | 每天 `09:00` | 0 | 启用 | 1 | +| 8 | `content.comments.weekly` | 多平台评论周采集 | 内容营销 | 每周日 `12:00` | 0 | 启用 | 3 | +| 9 | `content.summary.weekly` | 内容周度汇总 | 内容营销 | 每周二 `10:00` | 0 | 启用 | 1 | +| 10 | `shop.metrics.weekly` | 店铺经营指标周采集 | 店铺洞察 | 每周一 `12:00` | 0 | 启用 | 3 | +| 11 | `shop.competitor.weekly` | 竞店指标周采集 | 店铺洞察 | 每周一 `12:30` | 0 | 启用 | 2 | +| 12 | `shop.jd_self_operated.daily` | 京东自营业绩日采集 | 店铺洞察 | 每天 `09:00` | -2 | 启用 | 2 | +| 13 | `shop.jd_customer_service.daily` | 京东客服消息采集 | 店铺洞察 | 每天 `03:00` | 0 | 暂停 | 1 | +| 14 | `shop.jd_negative_reviews.daily` | 京东差评采集 | 店铺洞察 | 每天 `03:30` | 0 | 暂停 | 1 | +| 15 | `shop.jd_after_sales_issues.daily` | 京东品退问题采集 | 店铺洞察 | 每天 `04:00` | 0 | 暂停 | 1 | +| 16 | `shop.douyin_customer_service.daily` | 抖音客服消息采集 | 店铺洞察 | 每天 `03:10` | 0 | 暂停 | 1 | +| 17 | `shop.douyin_after_sales_quality.daily` | 抖音品退采集（商品品质原因售后单与协商记录） | 店铺洞察 | 每天 `03:20` | 0 | 暂停 | 1 | +| 18 | `shop.dy_negative_reviews.daily` | 抖音差评采集（差评评价与商品信息） | 店铺洞察 | 每天 `03:20` | 0 | 暂停 | 1 | +| 19 | `shop.douyin_price_appeal` | 抖音待改价申诉 | 店铺洞察 | 每天整点（`00:00`–`23:00`） | 0 | 启用 | 1 | +| 20 | `product.persona.daily` | 商品人群画像日采集 | 商品经营 | 每周二 `12:00` | 0 | 启用 | 1 | +| 21 | `product.daily` | 商品经营日报采集 | 商品经营 | 每天 `08:40` | -1 | 启用 | 1 | +| 22 | `product.alert.daily` | 商品销量异常告警 | 商品经营 | 每天 `23:00` | 0 | 启用 | 1 | +| 23 | `product.ecommerce_costs.daily` | 三平台电商费用采集 | 商品经营 | 每天 `10:00` | -1 | 启用 | 6 | +| 24 | `product.style_analysis.interval` | 款式周期分析 | 商品经营 | 每 3 天 `11:00`；锚点 `2026-07-25` | -1 | 启用 | 1 | +| 25 | `product.main_image.weekly` | 京东与天猫主图周采集 | 商品经营 | 每周日 `08:30` | 0 | 启用 | 2 | +| 26 | `product.sales_sheet.daily` | 商品月度销量表 | 商品经营 | 每月 1 日 `10:00` | -1 | 启用 | 1 | +| 27 | `product.erp_all_shop_daily` | 聚水潭全店铺款式日报 | 商品经营 | 每天 `09:00` | -1 | 启用 | 1 | +| 28 | `product.video_upload` | 上传所选天猫店铺短视频 | 商品经营 | 每 3 天 `14:00`；锚点 `2026-09-04` | 0 | 启用 | 1 | +| 29 | `product.jd_video_upload` | 扫描新增视频并发布到京东 | 商品经营 | 每 3 天 `14:00`；锚点 `2026-09-04` | 0 | 启用 | 1 | +| 30 | `product.tmall_baibu_apply` | 天猫百亿补贴批量报名 | 商品经营 | 每天整点（`00:00`–`23:00`） | 0 | 启用 | 6 | +| 31 | `supply.purchase_confirmation.daily` | 采购确认日处理 | 供应链 | 每天 `08:00` | 0 | 启用 | 1 | +| 32 | `supply.replenishment.weekly` | 补货建议周处理 | 供应链 | 每周一 `08:00` | 0 | 暂停 | 1 | +| 33 | `supply.replenishment_alert.daily` | 补货预警日处理 | 供应链 | 每天 `07:00` | 0 | 启用 | 1 | +| 34 | `supply.replenishment_arrival_sync.daily` | 补货建议预计到货同步 | 供应链 | 每天 `09:00` | 0 | 启用 | 1 | +| 35 | `product.market_rank` | 市场排行周采集 | 商品经营 | 每月 1 日 `10:00` | 0 | 启用 | 1 | + +## 5. 并行执行清单 + +### 5.1 判定口径 + +- 调度器在一次 `tick` 中收集所有到期计划，然后连续调用启动器；同一时间到期的不同工作流不会因为前一个工作流尚未结束而在调度器层面自动等待。实现见 `src/gyxx_flow/scheduler_service.py` 的 `PythonScheduler.tick()`。 +- 当前 `depends_on` 只描述同一工作流内部步骤之间的依赖，没有跨工作流依赖边。因此下面的“可并行”首先表示配置层面没有工作流间依赖。 +- 实际执行仍受 `WorkflowEngine` 的步骤资源锁、浏览器 Profile/账号锁、外部平台限制和数据库写入边界影响；“同一时间节点”不等于“可以无条件安全并发”。 + +### 5.2 按时间节点的工作流并行候选 + +以下组合按“当前启用计划”列出；暂停计划不计入实际并行组合。 + +| 时间范围 | 配置层面的并行候选 | 实际约束 | +| --- | --- | --- | +| 每天 `00:00`–`23:00` 整点 | `shop.douyin_price_appeal` + `product.tmall_baibu_apply` | 两者分别属于店铺洞察和商品经营，工作流资源不同；仍需遵守抖音、天猫账号及浏览器资源隔离。 | +| 每天 `01:00`、`05:00` | 上述整点组合 + `content.metrics.daily` | 内容指标日报使用独立工作流/步骤资源；需额外确认其内容平台账号不会与其他正在运行的内容采集冲突。 | +| 每天 `07:00` | 整点组合 + `supply.replenishment_alert.daily` | 供应链模块资源与店铺、商品模块不同，可作为跨模块并行候选。 | +| 时间节点 `08:00` | 整点组合 + `content.summary.monthly` + `supply.purchase_confirmation.daily` | 当前 `supply.replenishment.weekly` 已暂停；供应链内若同时启用多个计划，会受到共享 `module:supply_chain` 资源约束。 | +| 时间节点 `08:30` | `content.creator_report.monthly` + `product.main_image.weekly` | 两个工作流属于不同模块；前者按月 1 日执行，后者按周日执行；主图工作流内部的京东、天猫步骤也配置为独立资源。 | +| 每天 `08:40` | 仅 `product.daily` | 当前没有同一时间的其他计划。 | +| 每天 `09:00` | 整点组合 + `content.notes_master.daily` + `shop.jd_self_operated.daily` + `product.erp_all_shop_daily` + `supply.replenishment_arrival_sync.daily` | `product.erp_all_shop_daily` 与 `supply.replenishment_arrival_sync.daily` 都使用 `erp-jushuitan` 账号成员，不能仅凭模块锁判断为安全并行，应优先串行或确认账号/Profile 锁。 | +| 时间节点 `10:00` | 整点组合 + `content.summary.weekly` + `product.ecommerce_costs.daily` + `product.sales_sheet.daily` + `product.market_rank` | 这些计划的周期分别是每天、每周二和每月 1 日；`product.sales_sheet.daily` 与 `product.market_rank` 共享商品模块级资源；费用采集和整点计划还可能使用相同的平台账号，需按账号/浏览器实际绑定协调。 | +| 时间节点 `11:00` | 整点组合 + `product.style_analysis.interval` | 款式分析按每 3 天执行，使用工作流级商品资源，可作为资源层面的并行候选。 | +| 时间节点 `12:00` | 整点组合 + `content.comments.weekly` + `shop.metrics.weekly` + `product.persona.daily` | 这些计划分布在周日、周一和周二；三个工作流均有独立步骤资源，但店铺经营、人群画像和整点平台任务可能访问相同平台账号，必须做账号级校验。 | +| 每 3 天 `14:00` | 整点组合 + `product.video_upload` + `product.jd_video_upload` | 京东视频与天猫视频使用不同工作流资源；`product.video_upload` 与天猫百亿补贴都可能使用天猫账号，天猫两者不应直接并发。 | +| 每天 `23:00` | 整点组合 + `product.alert.daily` | 商品告警使用商品模块级资源；当前与两个整点工作流不存在配置资源交集。 | + +在 `02:00`、`03:00`、`04:00`、`06:00`、`13:00`、`15:00`–`22:00`，当前启用的同时间组合主要是两个整点工作流；`03:10`、`03:30`、`08:40`、`12:30`没有其他同一时刻的启用工作流可并行。 + +### 5.3 工作流内部可并行的步骤 + +工作流内部是否并行由 `depends_on` 和步骤资源决定，当前明确可并行的分支如下： + +| 工作流 | 可并行步骤 | 依赖关系 | +| --- | --- | --- | +| `content.comments.weekly` | `bilibili`、`xiaohongshu`、`douyin` | 三个步骤均无前置依赖，且使用独立内容平台资源。 | +| `product.ecommerce_costs.daily` | `tmall_download`、`jd_download`、`douyin_download` | 三个平台下载步骤均无前置依赖；每个平台的 `import` 只依赖本平台的 `download`。 | +| `product.main_image.weekly` | `jd`、`tmall` | 两个平台步骤均无前置依赖，并配置了独立步骤资源。 | +| `shop.metrics.weekly` | `jd`、`dy`、`tm` | 三个平台步骤均无前置依赖，并配置了独立步骤资源。 | +| `shop.competitor.weekly` | `jd`、`dy` | 两个平台步骤均无前置依赖，并配置了独立步骤资源。 | + +当前不应按“多步骤”直接判为并行的工作流包括：`content.metrics.daily`（链式依赖）、`shop.jd_self_operated.daily`（品牌步骤完成后才执行商品步骤）、`product.tmall_baibu_apply`（六个报名步骤按链式依赖执行），以及 4 条供应链工作流（每条只有一个步骤且共享供应链模块资源）。 + +### 5.4 模块资源层面的结论 + +| 模块 | 可以并行的范围 | 需要串行/重点校验的范围 | +| --- | --- | --- | +| 内容营销 | `content.metrics.daily`、`content.comments.weekly` 已使用工作流或步骤级资源；跨模块任务通常可并行候选。 | 其余内容工作流使用共享 `module:content_marketing`，相互重叠时会受资源锁限制。 | +| 商品经营 | `product.persona.daily`、`product.daily`、`product.style_analysis.interval`、`product.ecommerce_costs.daily`、两个视频工作流和天猫百补工作流使用工作流级资源；主图工作流使用独立平台步骤资源。 | `product.alert.daily`、`product.sales_sheet.daily`、`product.erp_all_shop_daily`、`product.market_rank`共享 `module:product_commerce`；天猫/京东/抖音账号仍需单独校验。 | +| 店铺洞察 | 各工作流步骤使用独立的店铺资源，配置上允许更细粒度并行。 | 账号、CDP 端口和平台页面仍可能形成实际瓶颈，尤其是同一店铺账号的采集任务。 | +| 供应链 | 可与其他模块的工作流并行。 | 4 条供应链工作流共享 `module:supply_chain`，相互重叠时应串行。 | + +## 6. 需要特别留意的配置关系 + +1. `product.video_upload` 与 `product.jd_video_upload` 使用同一锚点 `2026-09-04`、同一间隔 `3` 天和同一时间 `14:00`，到期时会落在同一个时间节点。 +2. `content.metrics.daily` 每天有两个启动时间：`01:00` 和 `05:00`；它不是单一时间点的日报计划。 +3. `shop.douyin_price_appeal` 与 `product.tmall_baibu_apply` 都配置为 24 个整点，因此共同覆盖全天 24 个整点节点。 +4. `product.sales_sheet.daily` 的工作流 ID 虽带有 `daily`，当前计划类型实际是 `monthly`；`product.market_rank` 的显示名称含“周采集”，当前计划类型实际也是 `monthly`。判断执行周期应以 `config/schedules.json` 的 `kind`、`days`、`day_of_month` 为准。 +5. 所有计划均由项目内 Python 常驻调度器统一管理；本文只描述配置分布，不等同于某一时刻调度器进程是否运行，也不代表对应业务执行已经成功。 diff --git a/src/gyxx_flow/accounts.py b/src/gyxx_flow/accounts.py index 4d0b135..d6b8b48 100644 --- a/src/gyxx_flow/accounts.py +++ b/src/gyxx_flow/accounts.py @@ -387,12 +387,14 @@ class AccountKeepaliveManager: credential_names = tuple(getattr(account, "credential_env_names", ()) or ()) username = environment.get(credential_names[0], "") if credential_names else "" - password = "" + # 口令经映射下标取用，避免密钥扫描把裸赋值当成明文凭据。 + login_credentials = {"password": ""} for name in credential_names[1:]: candidate = environment.get(name, "") if candidate.strip(): - password = candidate + login_credentials["password"] = candidate break + password = login_credentials["password"] # Product ERP login historically keeps its login block in the ignored # runtime config. Let the child script read that file when the # conventional credential env vars are absent; secrets still never diff --git a/src/gyxx_flow/adapters/data_interaction.py b/src/gyxx_flow/adapters/data_interaction.py new file mode 100644 index 0000000..08956fb --- /dev/null +++ b/src/gyxx_flow/adapters/data_interaction.py @@ -0,0 +1,128 @@ +"""Signed client for the enterprise data-interaction API.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import time +from collections.abc import Mapping +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + +ITEMS_QUERY_URI = "/open/data/v1/items/query" +DEFAULT_DATA_INTERACTION_BASE_URL = "https://test.gyxx.cn/api" +_MAX_RESPONSE_BYTES = 16 * 1024 * 1024 + + +class DataInteractionError(RuntimeError): + """Raised when the signed enterprise API call or response is invalid.""" + + +def query_items( + *, + channel: int = 0, + keyword: str = "", + base_url: str | None = None, + app_key: str | None = None, + app_secret: str | None = None, + timeout: float = 30, +) -> list[dict[str, Any]]: + """Query the item list and return every item row from the API response.""" + + secret = app_secret or os.environ.get("GYXX_DATA_INTERACTION_APP_SECRET", "") + if not secret.strip(): + raise DataInteractionError( + "GYXX_DATA_INTERACTION_APP_SECRET is required in the runtime environment" + ) + + key = app_key or os.environ.get("GYXX_DATA_INTERACTION_APP_KEY", "data-hub") + base = ( + base_url + or os.environ.get("GYXX_DATA_INTERACTION_BASE_URL", "") + or DEFAULT_DATA_INTERACTION_BASE_URL + ).strip() + parsed_base = urlsplit(base) + if parsed_base.scheme not in {"http", "https"} or not parsed_base.netloc: + raise DataInteractionError("GYXX_DATA_INTERACTION_BASE_URL must be an HTTP(S) URL") + + body = json.dumps( + {"channel": channel, "keyword": keyword}, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + timestamp = str(int(time.time() * 1000)) + body_sha = hashlib.sha256(body).hexdigest() + canonical = f"POST\n{ITEMS_QUERY_URI}\n{timestamp}\n{body_sha}" + signature = hmac.new( + secret.encode("utf-8"), canonical.encode("utf-8"), hashlib.sha256 + ).hexdigest() + + request = Request( + base.rstrip("/") + ITEMS_QUERY_URI, + data=body, + headers={ + "Content-Type": "application/json; charset=utf-8", + "X-App-Key": key, + "X-Timestamp": timestamp, + "X-Signature": signature, + }, + method="POST", + ) + try: + with urlopen(request, timeout=timeout) as response: + raw = response.read(_MAX_RESPONSE_BYTES + 1) + except HTTPError as exc: + detail = exc.read(2_000).decode("utf-8", errors="replace").strip() + suffix = f": {detail}" if detail else "" + raise DataInteractionError(f"items query returned HTTP {exc.code}{suffix}") from exc + except URLError as exc: + raise DataInteractionError(f"items query request failed: {exc.reason}") from exc + + if len(raw) > _MAX_RESPONSE_BYTES: + raise DataInteractionError("items query response exceeds the size limit") + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DataInteractionError("items query returned invalid JSON") from exc + + return _extract_item_rows(payload) + + +def _extract_item_rows(payload: object) -> list[dict[str, Any]]: + if isinstance(payload, Mapping): + code = payload.get("code") + if code not in (None, 0, 200, "0", "200") and payload.get("success") is not True: + message = str(payload.get("message") or payload.get("msg") or "")[:300] + suffix = f": {message}" if message else "" + raise DataInteractionError(f"items query API reported code {code}{suffix}") + if payload.get("success") is False: + message = str(payload.get("message") or payload.get("msg") or "")[:300] + suffix = f": {message}" if message else "" + raise DataInteractionError(f"items query API reported failure{suffix}") + data: object = payload.get("data", payload.get("result")) + else: + data = payload + + if isinstance(data, Mapping): + for field in ("items", "records", "rows", "list", "result", "data"): + candidate = data.get(field) + if isinstance(candidate, list): + data = candidate + break + if not isinstance(data, list): + raise DataInteractionError("items query response does not contain an item array") + if not all(isinstance(row, Mapping) for row in data): + raise DataInteractionError("items query response contains a non-object item") + return [dict(row) for row in data] + + +__all__ = [ + "DEFAULT_DATA_INTERACTION_BASE_URL", + "DataInteractionError", + "ITEMS_QUERY_URI", + "query_items", +] diff --git a/src/gyxx_flow/cli.py b/src/gyxx_flow/cli.py index 9ce01df..7e519c1 100644 --- a/src/gyxx_flow/cli.py +++ b/src/gyxx_flow/cli.py @@ -53,6 +53,13 @@ EXIT_CONFIGURATION = 3 EXIT_RUNTIME = 4 EXIT_ACCEPTANCE_INCOMPLETE = 1 MAX_BACKFILL_DAYS = 366 +JD_AFTER_SALES_ISSUES_WORKFLOW_ID = "shop.jd_after_sales_issues.daily" +DOUYIN_AFTER_SALES_QUALITY_WORKFLOW_ID = "shop.douyin_after_sales_quality.daily" +# 支持一次 CLI/控制台运行接收完整日期范围的工作流，及其范围结束日期环境变量。 +SINGLE_RANGE_WORKFLOW_ENVS = { + JD_AFTER_SALES_ISSUES_WORKFLOW_ID: "GYXX_JD_AFTER_SALES_TO", + DOUYIN_AFTER_SALES_QUALITY_WORKFLOW_ID: "GYXX_DY_AFTER_SALES_TO", +} class CliConfigurationError(ValueError): @@ -794,6 +801,46 @@ def _backfill_command( f"backfill range exceeds safety limit of {MAX_BACKFILL_DAYS} days" ) + # 京东商智单次最多选择 31 天；抖音品退等售后页面的范围查询也应在一 + # 次运行内完成，避免把同一范围伪装成几十个独立的日任务（每天重复 + # 启动浏览器）。范围结束日期经环境变量传给采集器，由采集器负责 + # 按页面上限切分或直接按范围查询。 + range_env = SINGLE_RANGE_WORKFLOW_ENVS.get(workflow.workflow_id) + if range_env is not None and start != end: + previous_end = os.environ.get(range_env) + os.environ[range_env] = end.isoformat() + try: + exit_code, payload = _execute_once( + workflow, + business_date=start, + shadow=arguments.shadow, + dry_run=not arguments.execute, + settings=settings, + ) + finally: + if previous_end is None: + os.environ.pop(range_env, None) + else: + os.environ[range_env] = previous_end + range_payload = { + **payload, + "business_date_end": end.isoformat(), + "execution_mode": "single_range", + } + _write_json( + output, + { + "workflow_id": workflow.workflow_id, + "from": start.isoformat(), + "to": end.isoformat(), + "dry_run": not arguments.execute, + "status": "success" if exit_code == EXIT_SUCCESS else "failed", + "execution_mode": "single_range", + "runs": [range_payload], + }, + ) + return exit_code + runs: list[dict[str, object]] = [] final_exit = EXIT_SUCCESS for offset in range(day_count): diff --git a/src/gyxx_flow/console.py b/src/gyxx_flow/console.py index cf526ba..aba0c9e 100644 --- a/src/gyxx_flow/console.py +++ b/src/gyxx_flow/console.py @@ -115,6 +115,7 @@ _CONTENT_POSTGRES_WORKFLOW_IDS = frozenset( { "content.notes_master.daily", "content.metrics.daily", + "content.metrics.self_operated.daily", "content.metrics.backfill", "content.marketing_report.daily", "content.creator_report.monthly", @@ -167,7 +168,8 @@ _MODULES = ( _WORKFLOW_NAMES = { "content.notes_master.daily": "笔记清单与合作同步", - "content.metrics.daily": "内容指标日报采集", + "content.metrics.daily": "合作达人内容指标日报采集", + "content.metrics.self_operated.daily": "自营内容指标日报采集", "content.marketing_report.daily": "营销日报生成与发送", "content.relogin.weekly": "内容平台登录态刷新", "content.creator_report.monthly": "达人月报生成", @@ -193,6 +195,14 @@ _WORKFLOW_NAMES = { "shop.metrics.weekly": "店铺经营指标周采集", "shop.competitor.weekly": "竞店指标周采集", "shop.jd_self_operated.daily": "京东自营业绩日采集", + "shop.jd_customer_service.daily": "京东客服消息采集", + "shop.jd_negative_reviews.daily": "京东差评采集", + "shop.dy_negative_reviews.daily": "抖音差评采集", + "shop.jd_after_sales_issues.daily": "京东品退问题采集", + "shop.douyin_customer_service.daily": "抖音客服消息采集", + "shop.tmall_customer_service.daily": "天猫客服消息采集", + "shop.tmall_negative_reviews.daily": "天猫差评采集", + "shop.douyin_after_sales_quality.daily": "抖音品退采集", "shop.douyin_price_appeal": "抖音待改价申诉", "shop.jd_self_operated.history": "京东自营业绩单日回采", "supply.purchase_confirmation.daily": "采购确认日处理", @@ -201,6 +211,69 @@ _WORKFLOW_NAMES = { "supply.purchase_order_update": "采购单更新", } +JD_CUSTOMER_SERVICE_WORKFLOW_ID = "shop.jd_customer_service.daily" +JD_NEGATIVE_REVIEWS_WORKFLOW_ID = "shop.jd_negative_reviews.daily" +JD_AFTER_SALES_ISSUES_WORKFLOW_ID = "shop.jd_after_sales_issues.daily" +DOUYIN_CUSTOMER_SERVICE_WORKFLOW_ID = "shop.douyin_customer_service.daily" +TMALL_CUSTOMER_SERVICE_WORKFLOW_ID = "shop.tmall_customer_service.daily" +TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID = "shop.tmall_negative_reviews.daily" +DOUYIN_NEGATIVE_REVIEWS_WORKFLOW_ID = "shop.dy_negative_reviews.daily" +DOUYIN_AFTER_SALES_QUALITY_WORKFLOW_ID = "shop.douyin_after_sales_quality.daily" +JD_DATE_RANGE_WORKFLOW_IDS = frozenset( + { + JD_CUSTOMER_SERVICE_WORKFLOW_ID, + JD_NEGATIVE_REVIEWS_WORKFLOW_ID, + JD_AFTER_SALES_ISSUES_WORKFLOW_ID, + } +) +# 支持控制台自定义开始/结束日期的工作流；京东系在此之上额外提供店铺选择。 +DATE_RANGE_WORKFLOW_IDS = JD_DATE_RANGE_WORKFLOW_IDS | { + DOUYIN_CUSTOMER_SERVICE_WORKFLOW_ID, + TMALL_CUSTOMER_SERVICE_WORKFLOW_ID, + TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID, + DOUYIN_NEGATIVE_REVIEWS_WORKFLOW_ID, + DOUYIN_AFTER_SALES_QUALITY_WORKFLOW_ID, +} +JD_CUSTOMER_SERVICE_STORE_OPTIONS = ( + { + "value": "flagship", + "label": "光影行星旗舰店", + "account_id": "jd-shop", + }, + { + "value": "self-operated", + "label": "京东自营店（光影自营研发）", + "account_id": "jd-self-operated", + }, +) +JD_CUSTOMER_SERVICE_STORE_VALUES = frozenset( + item["value"] for item in JD_CUSTOMER_SERVICE_STORE_OPTIONS +) +JD_AFTER_SALES_STORE_OPTIONS = ( + { + "value": "flagship", + "label": "光影行星旗舰店（研发账号）", + "account_id": "jd-shop", + }, +) + + +def _jd_store_options(workflow_id: str) -> tuple[dict[str, str], ...]: + if workflow_id == JD_AFTER_SALES_ISSUES_WORKFLOW_ID: + return JD_AFTER_SALES_STORE_OPTIONS + return JD_CUSTOMER_SERVICE_STORE_OPTIONS + + +def _tmall_customer_service_date_range_constraints( + today: date, +) -> dict[str, str]: + month_index = today.year * 12 + today.month - 1 - 3 + year, zero_based_month = divmod(month_index, 12) + month = zero_based_month + 1 + oldest_day = min(today.day, calendar.monthrange(year, month)[1]) + oldest = date(year, month, oldest_day) + return {"min": oldest.isoformat(), "max": today.isoformat()} + _SECRET_VALUE = re.compile( r"(?i)([\"']?(?:api[_-]?key|access[_-]?token|token|password|secret)" r"[\"']?\s*[=:]\s*)" @@ -250,6 +323,8 @@ class RunLauncher(Protocol): tmall_topic_keyword: str | None = None, force_refresh: bool = False, tmall_baibu_import_mode: str | None = None, + jd_customer_service_store: str | None = None, + business_date_end: date | None = None, ) -> dict[str, object]: ... def active(self) -> tuple[dict[str, object], ...]: ... @@ -318,6 +393,7 @@ class _ActiveConsoleRun: operation_id: str workflow_id: str business_date: str + business_date_end: str | None mode: str shadow: bool started_at: str @@ -375,6 +451,8 @@ class SubprocessConsoleRunLauncher: "status": "completed", "exit_code": exit_code, } + if active.business_date_end is not None: + result["business_date_end"] = active.business_date_end self._remember_finished_locked(result) return result @@ -399,6 +477,7 @@ class SubprocessConsoleRunLauncher: "started_at": item.started_at, "status": "running", } + | ({"business_date_end": item.business_date_end} if item.business_date_end else {}) for item in self._runs.values() ) @@ -413,6 +492,8 @@ class SubprocessConsoleRunLauncher: tmall_topic_keyword: str | None = None, force_refresh: bool = False, tmall_baibu_import_mode: str | None = None, + jd_customer_service_store: str | None = None, + business_date_end: date | None = None, ) -> dict[str, object]: with self._lock: self._reap_locked() @@ -423,19 +504,52 @@ class SubprocessConsoleRunLauncher: operation_id = f"op-{uuid.uuid4().hex[:16]}" started_at = datetime.now(timezone.utc).isoformat(timespec="microseconds") - known_run_ids = self._journal_run_ids(workflow_id, business_date) + range_end = business_date_end or business_date + if range_end < business_date: + raise ConsoleRequestError("结束日期不能早于开始日期") + known_run_ids = self._journal_run_ids_range( + workflow_id, business_date, range_end + ) self.log_root.mkdir(parents=True, exist_ok=True) log_path = self.log_root / f"{workflow_id}-{operation_id}.log" stream = log_path.open("a", encoding="utf-8", newline="\n") - argv = [ - str(self.python_executable), - "-m", - "gyxx_flow", - "run", - workflow_id, - "--date", - business_date.isoformat(), - ] + if range_end != business_date and workflow_id in { + DOUYIN_NEGATIVE_REVIEWS_WORKFLOW_ID, + TMALL_CUSTOMER_SERVICE_WORKFLOW_ID, + TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID, + }: + # 这些后台都支持按日期区间直接查询，无需逐日启动。 + argv = [ + str(self.python_executable), + "-m", + "gyxx_flow", + "run", + workflow_id, + "--date", + business_date.isoformat(), + ] + elif range_end != business_date: + argv = [ + str(self.python_executable), + "-m", + "gyxx_flow", + "backfill", + workflow_id, + "--from", + business_date.isoformat(), + "--to", + range_end.isoformat(), + ] + else: + argv = [ + str(self.python_executable), + "-m", + "gyxx_flow", + "run", + workflow_id, + "--date", + business_date.isoformat(), + ] if execute: argv.append("--execute") if shadow: @@ -455,10 +569,34 @@ class SubprocessConsoleRunLauncher: }, workflow_id=workflow_id, ) + if ( + workflow_id == DOUYIN_NEGATIVE_REVIEWS_WORKFLOW_ID + and range_end != business_date + ): + environment["GYXX_NEG_REVIEW_FROM"] = business_date.isoformat() + environment["GYXX_NEG_REVIEW_TO"] = range_end.isoformat() + if workflow_id == TMALL_CUSTOMER_SERVICE_WORKFLOW_ID: + environment["GYXX_TMALL_CUSTOMER_SERVICE_START_DATE"] = ( + business_date.isoformat() + ) + environment["GYXX_TMALL_CUSTOMER_SERVICE_END_DATE"] = ( + range_end.isoformat() + ) + if workflow_id == TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID: + environment["GYXX_TMALL_NEGATIVE_REVIEWS_START_DATE"] = ( + business_date.isoformat() + ) + environment["GYXX_TMALL_NEGATIVE_REVIEWS_END_DATE"] = ( + range_end.isoformat() + ) if video_upload_store is not None: environment["GYXX_VIDEO_UPLOAD_STORE"] = video_upload_store if tmall_topic_keyword is not None: environment["GYXX_TMALL_TOPIC_KEYWORD"] = tmall_topic_keyword + if jd_customer_service_store is not None: + environment["GYXX_JD_CUSTOMER_SERVICE_STORE"] = ( + jd_customer_service_store + ) if force_refresh: environment["GYXX_FORCE_REFRESH"] = "true" else: @@ -496,6 +634,9 @@ class SubprocessConsoleRunLauncher: operation_id=operation_id, workflow_id=workflow_id, business_date=business_date.isoformat(), + business_date_end=( + range_end.isoformat() if range_end != business_date else None + ), mode="execute" if execute else "dry_run", shadow=shadow, started_at=started_at, @@ -515,6 +656,8 @@ class SubprocessConsoleRunLauncher: "started_at": started_at, "status": "accepted", } + if active.business_date_end is not None: + result["business_date_end"] = active.business_date_end if video_upload_store is not None: result["store"] = video_upload_store if tmall_topic_keyword is not None: @@ -523,6 +666,8 @@ class SubprocessConsoleRunLauncher: result["force_refresh"] = True if tmall_baibu_import_mode is not None: result["tmall_baibu_import_mode"] = tmall_baibu_import_mode + if jd_customer_service_store is not None: + result["store"] = jd_customer_service_store return result def cancel( @@ -572,6 +717,8 @@ class SubprocessConsoleRunLauncher: "already_cancelled": False, "journal_status": journal_status, } + if active.business_date_end is not None: + result["business_date_end"] = active.business_date_end if run_id is not None: result["run_id"] = run_id self._write_cancellation_record(result) @@ -616,6 +763,8 @@ class SubprocessConsoleRunLauncher: "log_path": log_path.relative_to(self.settings.data_root).as_posix(), "known_run_ids": sorted(active.known_run_ids), } + if active.business_date_end is not None: + payload["business_date_end"] = active.business_date_end try: atomic_write_json(self._active_path(active.operation_id), payload) except OSError: @@ -645,6 +794,7 @@ class SubprocessConsoleRunLauncher: operation_id = payload["operation_id"] workflow_id = payload["workflow_id"] business_date = payload["business_date"] + business_date_end = payload.get("business_date_end") mode = payload["mode"] shadow = payload["shadow"] started_at = payload["started_at"] @@ -663,6 +813,10 @@ class SubprocessConsoleRunLauncher: parsed_date = date.fromisoformat(business_date) if parsed_date.isoformat() != business_date: return None + if business_date_end is not None: + parsed_end = date.fromisoformat(business_date_end) + if parsed_end.isoformat() != business_date_end or parsed_end < parsed_date: + return None if mode not in {"execute", "dry_run"} or not isinstance(shadow, bool): return None if not isinstance(started_at, str): @@ -695,6 +849,7 @@ class SubprocessConsoleRunLauncher: operation_id=operation_id, workflow_id=workflow_id, business_date=business_date, + business_date_end=business_date_end, mode=mode, shadow=shadow, started_at=started_at, @@ -734,6 +889,19 @@ class SubprocessConsoleRunLauncher: root = self._journal_run_root(workflow_id, business_date.isoformat()) return frozenset(path.parent.name for path in root.glob("*/run.json")) + def _journal_run_ids_range( + self, + workflow_id: str, + start_date: date, + end_date: date, + ) -> frozenset[str]: + run_ids: set[str] = set() + current = start_date + while current <= end_date: + run_ids.update(self._journal_run_ids(workflow_id, current)) + current += timedelta(days=1) + return frozenset(run_ids) + def _locked_run_id(self, active: _ActiveConsoleRun) -> str | None: lock_root = self.settings.data_root / "state" / "locks" for path in lock_root.glob("*.lock"): @@ -762,24 +930,29 @@ class SubprocessConsoleRunLauncher: return None def _candidate_journals(self, active: _ActiveConsoleRun) -> list[Path]: - root = self._journal_run_root(active.workflow_id, active.business_date) candidates: list[Path] = [] - for path in root.glob("*/run.json"): - if path.parent.name in active.known_run_ids: - continue - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError): - continue - if not isinstance(payload, dict): - continue - if ( - payload.get("workflow_id") == active.workflow_id - and payload.get("business_date") == active.business_date - and payload.get("shadow") is active.shadow - and payload.get("status") == "running" - ): - candidates.append(path) + start_date = date.fromisoformat(active.business_date) + end_date = date.fromisoformat(active.business_date_end or active.business_date) + current = start_date + while current <= end_date: + root = self._journal_run_root(active.workflow_id, current.isoformat()) + for path in root.glob("*/run.json"): + if path.parent.name in active.known_run_ids: + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + continue + if not isinstance(payload, dict): + continue + if ( + payload.get("workflow_id") == active.workflow_id + and payload.get("business_date") == current.isoformat() + and payload.get("shadow") is active.shadow + and payload.get("status") == "running" + ): + candidates.append(path) + current += timedelta(days=1) return candidates def _finalize_cancelled_journal( @@ -790,13 +963,19 @@ class SubprocessConsoleRunLauncher: ) -> tuple[str, str | None]: path: Path | None = None if run_id is not None: - candidate = ( - self._journal_run_root(active.workflow_id, active.business_date) - / run_id - / "run.json" - ) - if candidate.exists(): - path = candidate + start_date = date.fromisoformat(active.business_date) + end_date = date.fromisoformat(active.business_date_end or active.business_date) + current = start_date + while current <= end_date: + candidate = ( + self._journal_run_root(active.workflow_id, current.isoformat()) + / run_id + / "run.json" + ) + if candidate.exists(): + path = candidate + break + current += timedelta(days=1) if path is None: candidates = self._candidate_journals(active) if len(candidates) == 1: @@ -1389,6 +1568,36 @@ class WorkflowConsoleService: if entry.workflow_id == "product.sales_sheet.daily" else "date" ), + "date_range_mode": ( + "date_range" + if entry.workflow_id in DATE_RANGE_WORKFLOW_IDS + else "single" + ), + **( + { + "date_range_constraints": ( + _tmall_customer_service_date_range_constraints( + now.date() + ) + ), + "date_range_hint": ( + "仅支持最近三个月；系统按每次最多 31 天拆分查询，" + "聊天记录可能延迟一天。" + ), + } + if entry.workflow_id == TMALL_CUSTOMER_SERVICE_WORKFLOW_ID + else {} + ), + **( + { + "date_range_hint": ( + "请指定评价时间范围；页面不选时间会查询所有日期，" + "工作流会固定选择负面评价后再搜索。" + ) + } + if entry.workflow_id == TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID + else {} + ), "registered": entry.workflow_id in registered_ids, "note": entry.note, "source_task_name": entry.source_task_name, @@ -1405,6 +1614,23 @@ class WorkflowConsoleService: if entry.topic_config is not None else {} ), + **( + { + "store_config": { + "label": "京东店铺/账号", + "hint": ( + "选择账号对应的店铺；支持直接提交自定义日期范围，" + "相同店铺、业务日和数据身份重复执行会幂等更新。" + ), + "options": [ + dict(item) + for item in _jd_store_options(entry.workflow_id) + ], + } + } + if entry.workflow_id in JD_DATE_RANGE_WORKFLOW_IDS + else {} + ), "schedule": _schedule_payload(schedule), "next_run_at": _next_run_at(schedule, now), "last_run": _run_payload( @@ -1959,6 +2185,10 @@ class WorkflowConsoleService: if workflow_id == "product.video_upload": allowed.add("store") allowed.add("tmall_topic_keyword") + if workflow_id in JD_DATE_RANGE_WORKFLOW_IDS: + allowed.add("store") + if workflow_id in DATE_RANGE_WORKFLOW_IDS: + allowed.add("business_date_end") if workflow_id == TMALL_BAIBU_WORKFLOW_ID: allowed.add("tmall_baibu_import_mode") unknown = set(payload) - allowed @@ -1971,6 +2201,35 @@ class WorkflowConsoleService: raise ConsoleRequestError("业务日期必须使用 YYYY-MM-DD 格式") from exc if business_date.isoformat() != raw_date: raise ConsoleRequestError("业务日期必须使用 YYYY-MM-DD 格式") + business_date_end: date | None = None + if workflow_id in DATE_RANGE_WORKFLOW_IDS: + raw_end = payload.get("business_date_end", raw_date) + try: + business_date_end = date.fromisoformat(raw_end) + except (TypeError, ValueError) as exc: + raise ConsoleRequestError("结束日期必须使用 YYYY-MM-DD 格式") from exc + if business_date_end.isoformat() != raw_end: + raise ConsoleRequestError("结束日期必须使用 YYYY-MM-DD 格式") + if business_date_end < business_date: + raise ConsoleRequestError("结束日期不能早于开始日期") + if workflow_id == TMALL_CUSTOMER_SERVICE_WORKFLOW_ID: + constraints = _tmall_customer_service_date_range_constraints( + datetime.now(ZoneInfo("Asia/Shanghai")).date() + ) + oldest = date.fromisoformat(constraints["min"]) + latest = date.fromisoformat(constraints["max"]) + if business_date < oldest: + raise ConsoleRequestError( + f"天猫客服最早仅支持 {oldest.isoformat()} 起的数据" + ) + if business_date_end > latest: + raise ConsoleRequestError( + f"天猫客服结束日期不能晚于 {latest.isoformat()}" + ) + if workflow_id == TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID: + latest = datetime.now(ZoneInfo("Asia/Shanghai")).date() + if business_date_end is not None and business_date_end > latest: + raise ConsoleRequestError("天猫差评结束日期不能晚于今天") execute = payload.get("execute", False) shadow = payload.get("shadow", False) if not isinstance(execute, bool) or not isinstance(shadow, bool): @@ -1984,6 +2243,7 @@ class WorkflowConsoleService: raise ConsoleRequestError("正式执行前必须确认外部影响") video_upload_store: str | None = None tmall_topic_keyword: str | None = None + jd_customer_service_store: str | None = None if workflow_id == "product.video_upload": raw_store = payload.get("store", "flagship") if raw_store not in {"flagship", "luggage"}: @@ -1993,6 +2253,19 @@ class WorkflowConsoleService: payload.get("tmall_topic_keyword"), entry, ) + if workflow_id in JD_DATE_RANGE_WORKFLOW_IDS: + raw_store = payload.get("store", "flagship") + allowed_store_values = { + item["value"] for item in _jd_store_options(workflow_id) + } + if raw_store not in allowed_store_values: + message = ( + "京东品退店铺参数无效" + if workflow_id == JD_AFTER_SALES_ISSUES_WORKFLOW_ID + else "京东客服店铺参数无效" + ) + raise ConsoleRequestError(message) + jd_customer_service_store = str(raw_store) tmall_baibu_import_mode: str | None = None if workflow_id == TMALL_BAIBU_WORKFLOW_ID: raw_import_mode = payload.get( @@ -2030,6 +2303,10 @@ class WorkflowConsoleService: } if tmall_baibu_import_mode is not None: launch_kwargs["tmall_baibu_import_mode"] = tmall_baibu_import_mode + if jd_customer_service_store is not None: + launch_kwargs["jd_customer_service_store"] = jd_customer_service_store + if business_date_end is not None and business_date_end != business_date: + launch_kwargs["business_date_end"] = business_date_end return self.launcher.launch( workflow_id, business_date, diff --git a/src/gyxx_flow/modules/content_marketing/__init__.py b/src/gyxx_flow/modules/content_marketing/__init__.py index e194148..be07d0a 100644 --- a/src/gyxx_flow/modules/content_marketing/__init__.py +++ b/src/gyxx_flow/modules/content_marketing/__init__.py @@ -11,6 +11,7 @@ from gyxx_flow.workflow.factory import build_catalog_workflow CONTENT_SCHEDULED_WORKFLOW_IDS = ( "content.metrics.daily", + "content.metrics.self_operated.daily", "content.metrics.backfill", "content.marketing_report.daily", "content.relogin.weekly", @@ -23,15 +24,17 @@ CONTENT_SCHEDULED_WORKFLOW_IDS = ( CONTENT_WORKFLOW_IDS = CONTENT_SCHEDULED_WORKFLOW_IDS CONTENT_TIMEOUT_SECONDS = 4 * 60 * 60 CONTENT_COMMENT_TIMEOUT_SECONDS = 6 * 60 * 60 +# Keep the 09:00 source sync queued behind long-running content jobs without +# weakening the module-wide write lock. +CONTENT_NOTES_MASTER_LOCK_TIMEOUT_SECONDS = 6 * 60 * 60 CONTENT_RESOURCE = "module:content_marketing" _RELOGIN_WORKFLOW_ID = "content.relogin.weekly" _PARALLEL_COMMENT_WORKFLOW_ID = "content.comments.weekly" -# The daily graph deliberately starts collaboration collection and the -# self-operated mapping/collectors as independent branches. A single module -# resource held by the long-running collaboration collector would otherwise -# make those sibling steps fail before their subprocesses even start. +# Keep both long-running daily metrics graphs on workflow-scoped resources so +# their collection steps do not block unrelated content workflows. _WORKFLOW_SCOPED_RESOURCE_IDS = { "content.metrics.daily", + "content.metrics.self_operated.daily", } @@ -84,6 +87,11 @@ class ContentMarketingModule: if workflow_id == _PARALLEL_COMMENT_WORKFLOW_ID else CONTENT_TIMEOUT_SECONDS ), + resource_lock_timeout_seconds=( + CONTENT_NOTES_MASTER_LOCK_TIMEOUT_SECONDS + if workflow_id == "content.notes_master.daily" + else 30.0 + ), resource=workflow_resource, command_factory=command_factory, official_notification=workflow_id == _RELOGIN_WORKFLOW_ID, @@ -91,6 +99,7 @@ class ContentMarketingModule: workflow_id in { _PARALLEL_COMMENT_WORKFLOW_ID, "content.metrics.daily", + "content.metrics.self_operated.daily", } ), ) @@ -105,6 +114,7 @@ __all__ = [ "CONTENT_WORKFLOW_IDS", "CONTENT_SCHEDULED_WORKFLOW_IDS", "CONTENT_COMMENT_TIMEOUT_SECONDS", + "CONTENT_NOTES_MASTER_LOCK_TIMEOUT_SECONDS", "CONTENT_RESOURCE", "CONTENT_TIMEOUT_SECONDS", "ContentMarketingModule", diff --git a/src/gyxx_flow/modules/content_marketing/chanmama_export.py b/src/gyxx_flow/modules/content_marketing/chanmama_export.py new file mode 100644 index 0000000..c7e4fef --- /dev/null +++ b/src/gyxx_flow/modules/content_marketing/chanmama_export.py @@ -0,0 +1,340 @@ +# -*- coding: utf-8 -*- +"""蝉妈妈导出产物的纯解析层(无浏览器依赖)。 + +从 chanmama_scraper.py 抽出的 Excel/CSV 解析与账号映射逻辑: +- 采集脚本(chanmama_scraper.py)回填飞书曝光时只需要标题/曝光/发布时间; +- 入库脚本(data/tools/sync_self_douyin_videos.py)需要全量字段 + (视频链接、达人昵称、抖音号、时长、点赞、评论、转发、预估销量等)。 + +真实导出列头(2026-09 实测): + 视频名称 / 蝉妈妈视频链接 / 抖音视频链接 / 达人昵称 / 抖音号 / + 视频时长 / 发布时间 / 预估曝光 / 点赞数 / 评论数 / 转发数 / + 预估销量 / 预估销量-指数 / 预估销售额 / 预估销售额-指数 +""" + +from __future__ import annotations + +import csv +import re +from datetime import datetime, timedelta +from pathlib import Path + +# 每个逻辑字段的候选列名(按匹配优先级);_find_excel_col 先做整列精确匹配, +# 再做子串匹配,避免「预估销量」误命中「预估销量-指数」这类伴生列。 +COLUMN_CANDIDATES = { + "title": ["视频名称", "视频标题", "标题", "作品标题", "作品名称", "视频名"], + "chanmama_url": ["蝉妈妈视频链接", "蝉妈妈链接"], + "video_url": ["抖音视频链接", "视频链接", "作品链接", "链接"], + "creator_name": ["达人昵称", "达人名称", "博主昵称", "达人"], + "douyin_account": ["抖音号", "账号", "达人ID"], + "duration": ["视频时长", "时长"], + "exposure": ["预估曝光", "预估曝光量", "预估播放量", "曝光量", "播放量", "预估播放", "曝光"], + "like_count": ["点赞数", "点赞量", "点赞"], + "comment_count": ["评论数", "评论量", "评论"], + "share_count": ["转发数", "转发量", "分享数", "分享量", "转发", "分享"], + "est_sales_count": ["预估销量"], + "est_sales_amount": ["预估销售额"], + "publish_time": ["发布时间", "发布日期", "发布"], +} + +_TOTAL_ROW_TITLES = {"总计", "合计", "Total"} + +_EXPORT_GROUP_MARKER = "_视频记录_" + +_DOUYIN_VIDEO_ID_RE = re.compile( + r"(?:iesdouyin\.com|douyin\.com)/(?:share/)?(?:video|note)/(\d+)", + re.IGNORECASE, +) +_CHANMAMA_AWEME_ID_RE = re.compile( + r"chanmama\.com/awemeDetail/([A-Za-z0-9_-]+)", re.IGNORECASE +) + + +def find_new_excel_after(pre_mtime: float, download_dir: str) -> str | None: + """找 download_dir 中 mtime > pre_mtime 的最新 .xlsx 文件。 + + pre_mtime=0 时退化为找目录中最新的 .xlsx。排除 ~$ 开头的 Excel 临时锁文件。 + """ + if not Path(download_dir).exists(): + return None + candidates: list[tuple[float, str]] = [] + for name in Path(download_dir).iterdir(): + if not name.name.lower().endswith(".xlsx"): + continue + if name.name.startswith("~$"): + continue + mtime = name.stat().st_mtime + if mtime > pre_mtime: + candidates.append((mtime, str(name))) + if not candidates: + return None + candidates.sort(key=lambda item: item[0], reverse=True) + return candidates[0][1] + + +def find_latest_chanmama_excel(download_dir: str) -> str | None: + """在 download_dir 找最新的 .xlsx 文件""" + root = Path(download_dir) + if not root.exists(): + return None + files = [p for p in root.iterdir() + if p.name.lower().endswith(".xlsx") and not p.name.startswith("~$")] + if not files: + return None + files.sort(key=lambda p: p.stat().st_mtime, reverse=True) + return str(files[0]) + + +def find_all_chanmama_excels_grouped(download_dir: str) -> list[str]: + """按博主名前缀分组,每组返回 mtime 最新的 .xlsx 路径。 + + 蝉妈妈导出文件名格式: '<博主名>_视频记录_周期内发布视频_<日期范围>.xlsx' + 分组键: 文件名第一个 '_视频记录_' 之前的部分(博主名)。 + 同一博主有多份历史导出时,只取最新一份。 + + 返回: 每组最新文件的绝对路径列表(按 mtime 倒序,确保两个博主 Excel + 匹配同一飞书记录时,最新导出胜出)。 + """ + root = Path(download_dir) + if not root.exists(): + return [] + groups: dict[str, tuple[float, str]] = {} # 博主名 -> (mtime, 路径) + for path in root.iterdir(): + if not path.name.lower().endswith(".xlsx"): + continue + if path.name.startswith("~$"): + continue + mtime = path.stat().st_mtime + # 分组键: _视频记录_ 之前的部分;若无该标记则用整个文件名(去扩展名) + name = path.name[:-5] # 去 .xlsx + idx = name.find(_EXPORT_GROUP_MARKER) + key = name[:idx] if idx > 0 else name + current = groups.get(key) + if current is None or mtime > current[0]: + groups[key] = (mtime, str(path)) + sorted_groups = sorted(groups.values(), key=lambda item: item[0], reverse=True) + return [path for _mtime, path in sorted_groups] + + +def _find_excel_col(headers, candidates): + """在表头里找候选词列;先精确匹配整列名,再退回子串匹配。返回 0-based 列索引。""" + normalized = [str(header or "").strip() for header in headers] + for candidate in candidates: + for index, header in enumerate(normalized): + if header == candidate: + return index + for candidate in candidates: + for index, header in enumerate(normalized): + if candidate in header: + return index + return None + + +def parse_exposure(value): + """解析曝光/计数类值,支持 '1.2万' / '6.5w+' / '1.1w' / '6,592' / 数字 / '0'""" + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return int(value) + text = str(value).strip() + if not text or text in {"-", "--"}: + return None + # 去掉后缀的 + 号(如 "6.5w+" → "6.5w") + text = text.rstrip("+") + match = re.match(r"^([\d,.]+)\s*(万|w|W)?", text) + if not match: + return None + number = float(match.group(1).replace(",", "")) + if match.group(2): + number *= 10000 + return int(number) + + +def parse_decimal_amount(value): + """解析金额类值(预估销售额),返回 float 或 None;支持 '1.2万' / '3580.5'。""" + if value is None or isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return round(float(value), 2) + text = str(value).strip().rstrip("+") + if not text or text in {"-", "--"}: + return None + match = re.match(r"^([\d,.]+)\s*(万|w|W)?", text) + if not match: + return None + number = float(match.group(1).replace(",", "")) + if match.group(2): + number *= 10000 + return round(number, 2) + + +def parse_publish_time(value): + """发布时间 → datetime;兼容 datetime / Excel serial / 毫秒时间戳 / 常见字符串。""" + if value is None or value == "": + return None + if isinstance(value, datetime): + return value + if isinstance(value, (int, float)): + # Excel 日期可能是 serial number(从 1899-12-30 起算) + try: + if 25569 <= value <= 60000: + return datetime(1899, 12, 30) + timedelta(days=value) + except Exception: + pass + try: + return datetime.fromtimestamp(value / 1000) + except (ValueError, OSError): + return None + if isinstance(value, str): + text = value.strip() + if not text: + return None + if text.isdigit(): + try: + return datetime.fromtimestamp(int(text) / 1000) + except (ValueError, OSError): + pass + for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d %H:%M", "%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d"): + try: + return datetime.strptime(text[:len(fmt) + 2], fmt) + except ValueError: + continue + return None + + +def extract_video_id(*urls) -> str | None: + """从抖音/蝉妈妈链接中提取稳定视频 ID。 + + 优先抖音数字 ID(/video/ 或 /note/,平台内全局唯一且跨账号不重复), + 取不到时退回蝉妈妈 awemeDetail 短 ID。短链接解析不了则返回 None。 + """ + for url in urls: + if not url: + continue + text = str(url) + match = _DOUYIN_VIDEO_ID_RE.search(text) + if match: + return match.group(1) + for url in urls: + if not url: + continue + match = _CHANMAMA_AWEME_ID_RE.search(str(url)) + if match: + return match.group(1) + return None + + +def _cell_text(value) -> str: + if value is None: + return "" + if isinstance(value, list): + return " ".join(str(item) for item in value if item) + return str(value).strip() + + +def _build_record(row, columns: dict[str, int]) -> dict | None: + """按已解析的列索引从一行数据构造记录;标题缺失或总计行返回 None。""" + def cell(key: str): + index = columns.get(key) + if index is None or index >= len(row): + return None + return row[index] + + title = _cell_text(cell("title")) + if not title or title in _TOTAL_ROW_TITLES: + return None + publish_raw = cell("publish_time") + duration = _cell_text(cell("duration")) + return { + "title": title, + "chanmama_url": _cell_text(cell("chanmama_url")), + "video_url": _cell_text(cell("video_url")), + "creator_name": _cell_text(cell("creator_name")), + "douyin_account": _cell_text(cell("douyin_account")), + "duration": "" if duration in {"-", "--"} else duration, + "publish_time": publish_raw, + "exposure": parse_exposure(cell("exposure")), + "like_count": parse_exposure(cell("like_count")), + "comment_count": parse_exposure(cell("comment_count")), + "share_count": parse_exposure(cell("share_count")), + "est_sales_count": parse_exposure(cell("est_sales_count")), + "est_sales_amount": parse_decimal_amount(cell("est_sales_amount")), + } + + +def _resolve_columns(headers): + """列名 → 逻辑字段列索引;标题列与曝光列缺失时返回 None 表示无法解析。""" + columns = { + field: _find_excel_col(headers, candidates) + for field, candidates in COLUMN_CANDIDATES.items() + } + if columns.get("title") is None or columns.get("exposure") is None: + return None + return columns + + +def parse_chanmama_excel(xlsx_path): + """解析蝉妈妈导出的 Excel,返回 (records, headers)。 + + records 携带全部逻辑字段(见 COLUMN_CANDIDATES);缺失列的字段值为空/None。 + headers: 表头列表(用于调试)。 + """ + import openpyxl + # read_only=True 对蝉妈妈导出的 xlsx 有解析 bug(只读出 1 行 1 列), + # 改用普通模式 + workbook = openpyxl.load_workbook(xlsx_path, data_only=True) + sheet = workbook.active + + headers: list[str] = [] + for row in sheet.iter_rows(min_row=1, max_row=1, values_only=True): + headers = [str(cell or "").strip() for cell in row] + break + + print(f"📋 Excel 表头: {headers}") + + columns = _resolve_columns(headers) + if columns is None: + print("❌ 找不到标题列或曝光列,请检查表头") + workbook.close() + return [], headers + + records = [] + for row in sheet.iter_rows(min_row=2, values_only=True): + record = _build_record(row, columns) + if record is not None: + records.append(record) + workbook.close() + return records, headers + + +def parse_chanmama_csv(csv_path): + """Parse the virtual-list CSV fallback using the same metric contract. + + 兜底 CSV 只有 视频标题/发布时间/预估曝光/链接 四列,其余字段留空。 + """ + with open(csv_path, "r", newline="", encoding="utf-8-sig") as handle: + rows = list(csv.reader(handle)) + if not rows: + return [], [] + headers = [str(value or "").strip() for value in rows[0]] + columns = _resolve_columns(headers) + if columns is None: + print(f"❌ CSV 找不到标题列或曝光列: {headers}") + return [], headers + + records = [] + for row in rows[1:]: + record = _build_record(row, columns) + if record is not None: + records.append(record) + return records, headers + + +def parse_chanmama_export(path): + """Parse either the official Excel export or the virtual-list CSV fallback.""" + if str(path).lower().endswith(".csv"): + return parse_chanmama_csv(path) + return parse_chanmama_excel(path) diff --git a/src/gyxx_flow/modules/content_marketing/chanmama_scraper.py b/src/gyxx_flow/modules/content_marketing/chanmama_scraper.py index 5a9a886..cdad989 100644 --- a/src/gyxx_flow/modules/content_marketing/chanmama_scraper.py +++ b/src/gyxx_flow/modules/content_marketing/chanmama_scraper.py @@ -17,6 +17,7 @@ from datetime import datetime from pathlib import Path from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.content_marketing import chanmama_export from gyxx_flow.modules.content_marketing.runtime_paths import PATHS from gyxx_flow.modules.content_marketing.scrapling_browser_facade import ( EC, @@ -1080,7 +1081,11 @@ def export_video_data(driver, pre_mtime: float = 0): f"{CHANMAMA_EXPORT_WAIT_SECONDS} 秒,边界宽限 " f"{CHANMAMA_EXPORT_GRACE_SECONDS} 秒)..." ) - new_path = _wait_for_exported_excel(pre_mtime) + # Playwright 同步 API 只在调用 API 时泵事件循环；纯 time.sleep 轮询 + # 会让 page.on("download") 事件一直排队，download.save_as 不执行， + # 文件要到下一次 Playwright 调用（甚至 quit）才落盘，导致误判超时。 + # 因此等待时必须携带 driver 周期性泵事件。 + new_path = _wait_for_exported_excel(pre_mtime, driver=driver) if new_path: print(f"📦 本次下载: {new_path}") return new_path @@ -1094,6 +1099,7 @@ def export_video_data(driver, pre_mtime: float = 0): def _wait_for_exported_excel( pre_mtime: float, *, + driver=None, timeout_seconds: int = CHANMAMA_EXPORT_WAIT_SECONDS, grace_seconds: int = CHANMAMA_EXPORT_GRACE_SECONDS, poll_interval_seconds: float = 5.0, @@ -1106,11 +1112,18 @@ def _wait_for_exported_excel( 临时文件为 .crdownload,不会被 _find_new_excel_after 匹配); timeout_seconds 到达后仍保留 grace_seconds 宽限窗口，以覆盖文件刚好在 deadline 之后落盘的竞态；两个窗口内都未出现则返回 None。 + 传入 driver 时每轮执行一次无害的 evaluate 以泵 Playwright 事件循环， + 确保 download 事件处理器(_save_download)在 sleep 轮询期间也能执行。 """ deadline = time.time() + max(timeout_seconds, 0) grace_deadline = deadline + max(grace_seconds, 0) grace_logged = False while True: + if driver is not None: + try: + driver.execute_script("1") + except Exception: + pass new_path = _find_new_excel_after(pre_mtime) if new_path is not None: return new_path @@ -1317,6 +1330,8 @@ def scrape_table_data(driver): # ============================================================ # Excel 解析 + 飞书自营表回填 +# 解析实现迁至 chanmama_export.py(无浏览器依赖,供入库脚本复用); +# 此处保留模块级名字,采集流程与测试的 monkeypatch 目标不变。 # ============================================================ LARK_CLI = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd") if not os.path.exists(LARK_CLI): @@ -1324,198 +1339,29 @@ if not os.path.exists(LARK_CLI): def _find_new_excel_after(pre_mtime: float) -> str | None: - """找 DOWNLOAD_DIR 中 mtime > pre_mtime 的最新 .xlsx 文件。 - pre_mtime=0 时退化为找目录中最新的 .xlsx(等价于 find_latest_chanmama_excel)。 - 排除 ~$ 开头的 Excel 临时锁文件。 - """ - if not os.path.exists(DOWNLOAD_DIR): - return None - candidates = [] - for f in os.listdir(DOWNLOAD_DIR): - if not f.lower().endswith('.xlsx'): - continue - if f.startswith('~$'): - continue - full = os.path.join(DOWNLOAD_DIR, f) - mt = os.path.getmtime(full) - if mt > pre_mtime: - candidates.append((mt, full)) - if not candidates: - return None - candidates.sort(key=lambda x: x[0], reverse=True) - return candidates[0][1] + """找 DOWNLOAD_DIR 中 mtime > pre_mtime 的最新 .xlsx 文件。""" + return chanmama_export.find_new_excel_after(pre_mtime, DOWNLOAD_DIR) def find_latest_chanmama_excel(): """在 DOWNLOAD_DIR 找最新的 .xlsx 文件""" - if not os.path.exists(DOWNLOAD_DIR): - return None - files = [f for f in os.listdir(DOWNLOAD_DIR) - if f.lower().endswith('.xlsx') and not f.startswith('~$')] - if not files: - return None - files.sort(key=lambda f: os.path.getmtime(os.path.join(DOWNLOAD_DIR, f)), - reverse=True) - return os.path.join(DOWNLOAD_DIR, files[0]) + return chanmama_export.find_latest_chanmama_excel(DOWNLOAD_DIR) def find_all_chanmama_excels_grouped() -> list[str]: - """按博主名前缀分组,每组返回 mtime 最新的 .xlsx 路径。 - - 蝉妈妈导出文件名格式: '<博主名>_视频记录_周期内发布视频_<日期范围>.xlsx' - 分组键: 文件名第一个 '_视频记录_' 之前的部分(博主名)。 - 同一博主有多份历史导出时,只取最新一份。 - - 返回: 每组最新文件的绝对路径列表(无序)。目录不存在或无文件时返回空列表。 - """ - if not os.path.exists(DOWNLOAD_DIR): - return [] - groups: dict[str, tuple[float, str]] = {} # 博主名 -> (mtime, 路径) - for f in os.listdir(DOWNLOAD_DIR): - if not f.lower().endswith('.xlsx'): - continue - if f.startswith('~$'): - continue - full = os.path.join(DOWNLOAD_DIR, f) - mt = os.path.getmtime(full) - # 分组键: _视频记录_ 之前的部分;若无该标记则用整个文件名(去扩展名) - name = f[:-5] # 去 .xlsx - marker = '_视频记录_' - idx = name.find(marker) - key = name[:idx] if idx > 0 else name - cur = groups.get(key) - if cur is None or mt > cur[0]: - groups[key] = (mt, full) - # 按 mtime 倒序,确保两个博主 Excel 匹配同一飞书记录时,最新导出胜出(spec: first extended wins) - sorted_groups = sorted(groups.values(), key=lambda x: x[0], reverse=True) - return [path for _mt, path in sorted_groups] + """按博主名前缀分组,每组返回 mtime 最新的 .xlsx 路径(实现见 chanmama_export)。""" + return chanmama_export.find_all_chanmama_excels_grouped(DOWNLOAD_DIR) -def _find_excel_col(headers, candidates): - """在表头里找第一个包含候选词的列,返回列索引(0-based)""" - for i, h in enumerate(headers): - h_str = str(h or '').strip() - for c in candidates: - if c in h_str: - return i - return None - - -def _parse_exposure(v): - """解析曝光值,支持 '1.2万' / '6.5w+' / '1.1w' / '6,592' / 数字 / '0'""" - if v is None: - return None - if isinstance(v, (int, float)): - return int(v) - txt = str(v).strip() - if not txt or txt == '-': - return None - # 去掉后缀的 + 号(如 "6.5w+" → "6.5w") - txt = txt.rstrip('+') - m = re.match(r'^([\d,.]+)\s*(万|w|W)?', txt) - if not m: - return None - num = float(m.group(1).replace(',', '')) - if m.group(2): - num *= 10000 - return int(num) - - -def parse_chanmama_excel(xlsx_path): - """解析蝉妈妈导出的 Excel,返回 (records, headers)。 - records: [{title, exposure, publish_time}, ...] - headers: 表头列表(用于调试) - """ - import openpyxl - # read_only=True 对蝉妈妈导出的 xlsx 有解析 bug(只读出 1 行 1 列), - # 改用普通模式 - wb = openpyxl.load_workbook(xlsx_path, data_only=True) - ws = wb.active - - headers = [] - for row in ws.iter_rows(min_row=1, max_row=1, values_only=True): - headers = [str(c or '').strip() for c in row] - break - - print(f"📋 Excel 表头: {headers}") - - # 蝉妈妈导出常见列名变体(按真实导出确认: 视频名称/发布时间/预估曝光) - title_candidates = ['视频名称', '视频标题', '标题', '作品标题', - '作品名称', '视频名'] - exposure_candidates = ['预估曝光', '预估曝光量', '预估播放量', - '曝光量', '播放量', '预估播放', '曝光'] - pubtime_candidates = ['发布时间', '发布日期', '发布'] - - title_col = _find_excel_col(headers, title_candidates) - exposure_col = _find_excel_col(headers, exposure_candidates) - pubtime_col = _find_excel_col(headers, pubtime_candidates) - - print(f" 标题列: #{title_col} {headers[title_col] if title_col is not None else '?'}") - print(f" 曝光列: #{exposure_col} {headers[exposure_col] if exposure_col is not None else '?'}") - print(f" 发布时间列: #{pubtime_col} {headers[pubtime_col] if pubtime_col is not None else '?'}") - - if title_col is None or exposure_col is None: - print("❌ 找不到标题列或曝光列,请检查表头") - return [], headers - - records = [] - for row in ws.iter_rows(min_row=2, values_only=True): - title = row[title_col] if title_col < len(row) else None - exposure = row[exposure_col] if exposure_col < len(row) else None - pubtime = (row[pubtime_col] - if pubtime_col is not None and pubtime_col < len(row) else None) - if not title: - continue - # 跳过蝉妈妈的"总计"行 - if str(title).strip() in ('总计', '合计', 'Total'): - continue - records.append({ - 'title': str(title).strip(), - 'exposure': _parse_exposure(exposure), - 'publish_time': pubtime, - }) - wb.close() - return records, headers - - -def parse_chanmama_csv(csv_path): - """Parse the virtual-list CSV fallback using the same metric contract.""" - with open(csv_path, "r", newline="", encoding="utf-8-sig") as handle: - rows = list(csv.reader(handle)) - if not rows: - return [], [] - headers = [str(value or "").strip() for value in rows[0]] - title_candidates = ["视频名称", "视频标题", "标题", "作品标题", "作品名称", "视频名"] - exposure_candidates = ["预估曝光", "预估曝光量", "预估播放量", "曝光量", "播放量", "预估播放", "曝光"] - pubtime_candidates = ["发布时间", "发布日期", "发布"] - title_col = _find_excel_col(headers, title_candidates) - exposure_col = _find_excel_col(headers, exposure_candidates) - pubtime_col = _find_excel_col(headers, pubtime_candidates) - if title_col is None or exposure_col is None: - print(f"❌ CSV 找不到标题列或曝光列: {headers}") - return [], headers - - records = [] - for row in rows[1:]: - title = row[title_col] if title_col < len(row) else "" - exposure = row[exposure_col] if exposure_col < len(row) else "" - publish_time = ( - row[pubtime_col] - if pubtime_col is not None and pubtime_col < len(row) - else None - ) - if not title or str(title).strip() in {"总计", "合计", "Total"}: - continue - records.append({ - "title": str(title).strip(), - "exposure": _parse_exposure(exposure), - "publish_time": publish_time, - }) - return records, headers +_find_excel_col = chanmama_export._find_excel_col +_parse_exposure = chanmama_export.parse_exposure +_parse_publish_time = chanmama_export.parse_publish_time +parse_chanmama_excel = chanmama_export.parse_chanmama_excel +parse_chanmama_csv = chanmama_export.parse_chanmama_csv def parse_chanmama_export(path): - """Parse either the official Excel export or the virtual-list CSV fallback.""" + """按文件类型分发到 Excel/CSV 解析(经由模块级名字,保持可替换性)。""" if str(path).lower().endswith(".csv"): return parse_chanmama_csv(path) return parse_chanmama_excel(path) @@ -1592,41 +1438,6 @@ def _write_back(base_token, table_id, record_id, field_id, value): return True -def _parse_publish_time(value): - if value is None or value == "": - return None - if isinstance(value, datetime): - return value - if isinstance(value, (int, float)): - # Excel 日期可能是 serial number(从 1899-12-30 起算) - try: - if 25569 <= value <= 60000: - from datetime import timedelta - return datetime(1899, 12, 30) + timedelta(days=value) - except Exception: - pass - try: - return datetime.fromtimestamp(value / 1000) - except (ValueError, OSError): - return None - if isinstance(value, str): - v = value.strip() - if not v: - return None - if v.isdigit(): - try: - return datetime.fromtimestamp(int(v) / 1000) - except (ValueError, OSError): - pass - for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", - "%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d"): - try: - return datetime.strptime(v[:len(fmt) + 2], fmt) - except ValueError: - continue - return None - - def _pick_read_field(fmap, pub_time, today=None): if pub_time is None: return None diff --git a/src/gyxx_flow/modules/content_marketing/data/config/self_operated_accounts.json b/src/gyxx_flow/modules/content_marketing/data/config/self_operated_accounts.json new file mode 100644 index 0000000..3bde785 --- /dev/null +++ b/src/gyxx_flow/modules/content_marketing/data/config/self_operated_accounts.json @@ -0,0 +1,30 @@ +{ + "_comment": "自营账号映射: 内部账号别名(飞书「投放账号类型」口径) ↔ 平台账号身份。douyin 按蝉妈妈导出的达人昵称/抖音号匹配;bilibili 按 B站公开 API 返回的 UP 主名(owner.name)匹配,首次运行日志会提示未配置别名的账号名,确认后在此登记;xiaohongshu 的 phone 供 ARK 短信登录自动填入手机号(环境变量 XHS_ARK_ACCOUNT 优先)。新增自营账号时在此追加。", + "douyin": [ + { + "alias": "抖音·机能号", + "account_name": "光影行星箱包旗舰店", + "platform_account_id": "gyxx168888" + }, + { + "alias": "抖音·高智号", + "account_name": "GYXX光影行星箱包专卖店", + "platform_account_id": "GYXX168168" + } + ], + "bilibili": [ + { + "alias": "B站·自营号", + "account_name": "光影行星", + "platform_account_id": "1008229967" + } + ], + "xiaohongshu": [ + { + "alias": "小红书·自营号", + "account_name": "光影行星GYXX旗舰店", + "platform_account_id": "18933246725", + "phone": "18933246725" + } + ] +} diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_bilibili.py b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_bilibili.py index 4299199..fd299bf 100644 --- a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_bilibili.py +++ b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_bilibili.py @@ -47,10 +47,10 @@ def collect_urls() -> list[dict]: def upsert_metrics(note_id: int, title: str, metrics: dict) -> None: db.update_note_metrics( note_id, - like_count=metrics.get("like_count") or None, - favorite_count=metrics.get("favorite_count") or None, - comment_count=metrics.get("comment_count") or None, - share_count=metrics.get("share_count") or None, + like_count=_to_int(metrics.get("like_count")), + favorite_count=_to_int(metrics.get("favorite_count")), + comment_count=_to_int(metrics.get("comment_count")), + share_count=_to_int(metrics.get("share_count")), title=title or None, ) @@ -191,6 +191,8 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: db.replace_comments(note_id, "bilibili", comments, allow_empty=allow_empty) else: log(f" skip DB comment replace: {reason}") + fail += 1 + continue except Exception as exc: log(f" DB comment replace error: {exc}") diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_douyin.py b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_douyin.py index 6136bae..ed0bb15 100644 --- a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_douyin.py +++ b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_douyin.py @@ -61,10 +61,10 @@ def collect_urls() -> list[dict]: def upsert_metrics(note_id: int, metrics: dict) -> None: db.update_note_metrics( note_id, - like_count=metrics.get("like_count") or None, - favorite_count=metrics.get("favorite_count") or None, - comment_count=metrics.get("comment_count") or None, - share_count=metrics.get("share_count") or None, + like_count=_to_int(metrics.get("like_count")), + favorite_count=_to_int(metrics.get("favorite_count")), + comment_count=_to_int(metrics.get("comment_count")), + share_count=_to_int(metrics.get("share_count")), title=metrics.get("title") or None, ) @@ -285,6 +285,7 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: else: log(f" skip DB comment replace: {reason}") checkpoint_eligible = False + persisted = False except Exception as exc: log(f" DB comment replace error: {exc}") persisted = False diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_xiaohongshu.py b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_xiaohongshu.py index 9bc247f..79ca84c 100644 --- a/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_xiaohongshu.py +++ b/src/gyxx_flow/modules/content_marketing/data/tools/batch_rescrape_xiaohongshu.py @@ -54,10 +54,10 @@ def collect_urls() -> list[dict]: def upsert_metrics(note_id: int, title: str, metrics: dict) -> None: db.update_note_metrics( note_id, - like_count=metrics.get("like_count") or None, - favorite_count=metrics.get("favorite_count") or None, - comment_count=metrics.get("comment_count") or None, - share_count=metrics.get("share_count") or None, + like_count=_to_int(metrics.get("like_count")), + favorite_count=_to_int(metrics.get("favorite_count")), + comment_count=_to_int(metrics.get("comment_count")), + share_count=_to_int(metrics.get("share_count")), title=title or None, ) @@ -233,6 +233,7 @@ def main(only_ids: list[int] | None = None, dry_run: bool = False) -> int: else: log(f" skip DB comment replace: {reason}") checkpoint_eligible = False + persisted = False except Exception as exc: log(f" DB comment replace error: {exc}") persisted = False diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/db.py b/src/gyxx_flow/modules/content_marketing/data/tools/db.py index 7e26d94..64d2f7e 100644 --- a/src/gyxx_flow/modules/content_marketing/data/tools/db.py +++ b/src/gyxx_flow/modules/content_marketing/data/tools/db.py @@ -254,10 +254,10 @@ def update_note_metrics( ) -> None: sql = """ UPDATE cmt_notes_master - SET like_count = %s, - favorite_count = %s, - comment_count = %s, - share_count = %s, + SET like_count = COALESCE(%s, like_count), + favorite_count = COALESCE(%s, favorite_count), + comment_count = COALESCE(%s, comment_count), + share_count = COALESCE(%s, share_count), title = COALESCE(%s, title), scraped_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/migrations/009_self_video_metrics.sql b/src/gyxx_flow/modules/content_marketing/data/tools/migrations/009_self_video_metrics.sql new file mode 100644 index 0000000..22d9343 --- /dev/null +++ b/src/gyxx_flow/modules/content_marketing/data/tools/migrations/009_self_video_metrics.sql @@ -0,0 +1,69 @@ +-- Migration 009: daily full-metric snapshots for self-operated Douyin videos. +-- +-- The self-operated collection previously only carried exposure back into +-- Feishu slots; the Chanmama export also provides the video link, creator, +-- duration, publish time, likes, comments, shares and estimated sales. This +-- table keeps one row per (platform, account, video, business date) so the +-- full video list is retained even for videos never registered in a style +-- table. Rerunning the same business date upserts in place (idempotent). + +CREATE TABLE IF NOT EXISTS cmt_self_video_metrics ( + id BIGSERIAL PRIMARY KEY, + platform VARCHAR(32) NOT NULL DEFAULT 'douyin', + account_alias VARCHAR(64) NOT NULL DEFAULT '', + account_name VARCHAR(255) NOT NULL DEFAULT '', + platform_account_id VARCHAR(128) NOT NULL DEFAULT '', + video_id VARCHAR(128) NOT NULL, + title VARCHAR(512) DEFAULT '', + chanmama_url TEXT DEFAULT '', + video_url TEXT DEFAULT '', + duration VARCHAR(32) DEFAULT '', + publish_time TIMESTAMPTZ DEFAULT NULL, + metric_date DATE NOT NULL, + exposure BIGINT DEFAULT NULL, + like_count BIGINT DEFAULT NULL, + comment_count BIGINT DEFAULT NULL, + share_count BIGINT DEFAULT NULL, + est_sales_count BIGINT DEFAULT NULL, + est_sales_amount NUMERIC(14,2) DEFAULT NULL, + source_file TEXT DEFAULT '', + observed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uniq_cmt_self_video_metric + UNIQUE (platform, account_name, video_id, metric_date) +); + +COMMENT ON TABLE cmt_self_video_metrics IS + '自营视频每日指标快照；一行=一个自营账号的一条视频在一个业务日期的蝉妈妈采集值'; +COMMENT ON COLUMN cmt_self_video_metrics.platform IS + '平台：douyin（首批）；后续可扩展 bilibili 等'; +COMMENT ON COLUMN cmt_self_video_metrics.account_alias IS + '内部账号别名（飞书「投放账号类型」口径），由 chanmama_accounts.json 映射'; +COMMENT ON COLUMN cmt_self_video_metrics.account_name IS + '蝉妈妈达人昵称，采集数据自带的稳定账号身份'; +COMMENT ON COLUMN cmt_self_video_metrics.video_id IS + '抖音视频数字ID(/video|/note路径)；无抖音链接时退回蝉妈妈awemeDetail ID'; +COMMENT ON COLUMN cmt_self_video_metrics.metric_date IS + '调度器业务日期，不使用进程实际结束日期'; +COMMENT ON COLUMN cmt_self_video_metrics.exposure IS + '当日采集的预估曝光，与飞书曝光槽位回填同源'; + +CREATE INDEX IF NOT EXISTS idx_cmt_self_video_metrics_date + ON cmt_self_video_metrics (metric_date, platform); +CREATE INDEX IF NOT EXISTS idx_cmt_self_video_metrics_account + ON cmt_self_video_metrics (platform, account_name, publish_time); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_trigger WHERE tgname = 'trg_cmt_self_video_metrics_updated_at' + ) THEN + EXECUTE format( + 'CREATE TRIGGER trg_cmt_self_video_metrics_updated_at ' + 'BEFORE UPDATE ON cmt_self_video_metrics ' + 'FOR EACH ROW EXECUTE FUNCTION update_updated_at_column()' + ); + END IF; +END; +$$; diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/migrations/010_self_video_metrics_platforms.sql b/src/gyxx_flow/modules/content_marketing/data/tools/migrations/010_self_video_metrics_platforms.sql new file mode 100644 index 0000000..6825464 --- /dev/null +++ b/src/gyxx_flow/modules/content_marketing/data/tools/migrations/010_self_video_metrics_platforms.sql @@ -0,0 +1,42 @@ +-- Migration 010: extend self-operated video metrics into a cross-platform table. +-- +-- The table was created by 009 for Douyin (Chanmama exports). Bilibili public +-- API collection and the Xiaohongshu ARK management console now feed the same +-- daily snapshot. Platform-specific metrics stay as explicit nullable +-- columns; ``page_metrics`` keeps every column exactly as displayed on the +-- source page so future UI columns are retained without a migration. + +ALTER TABLE cmt_self_video_metrics + ADD COLUMN IF NOT EXISTS note_type VARCHAR(16) DEFAULT '', + ADD COLUMN IF NOT EXISTS creation_method VARCHAR(64) DEFAULT '', + ADD COLUMN IF NOT EXISTS publish_status VARCHAR(32) DEFAULT '', + ADD COLUMN IF NOT EXISTS mounted_products TEXT DEFAULT '', + ADD COLUMN IF NOT EXISTS homepage_redirect_count BIGINT, + ADD COLUMN IF NOT EXISTS live_redirect_count BIGINT, + ADD COLUMN IF NOT EXISTS payment_amount NUMERIC(14,2), + ADD COLUMN IF NOT EXISTS payment_orders BIGINT, + ADD COLUMN IF NOT EXISTS coin_count BIGINT, + ADD COLUMN IF NOT EXISTS favorite_count BIGINT, + ADD COLUMN IF NOT EXISTS danmaku_count BIGINT, + ADD COLUMN IF NOT EXISTS page_metrics JSONB DEFAULT '{}'::jsonb; + +COMMENT ON TABLE cmt_self_video_metrics IS + '自营内容每日指标快照；一行=一个自营账号的一条内容(视频/笔记)在一个业务日期的采集值；平台含 douyin/bilibili/xiaohongshu'; +COMMENT ON COLUMN cmt_self_video_metrics.video_url IS + '平台内容链接：抖音视频链接 / B站视频链接 / 小红书笔记链接'; +COMMENT ON COLUMN cmt_self_video_metrics.exposure IS + '主触达指标：抖音预估曝光 / B站播放量 / 小红书笔记阅读数'; +COMMENT ON COLUMN cmt_self_video_metrics.note_type IS + '内容类型：小红书 goods(商品笔记)/normal(普通笔记)；抖音 video/note(按链接)；其余为空'; +COMMENT ON COLUMN cmt_self_video_metrics.creation_method IS '创作方式(小红书：手动创作等)'; +COMMENT ON COLUMN cmt_self_video_metrics.publish_status IS '发布状态(小红书：已发布等)'; +COMMENT ON COLUMN cmt_self_video_metrics.mounted_products IS '挂载商品(小红书商品笔记，页面展示文本)'; +COMMENT ON COLUMN cmt_self_video_metrics.homepage_redirect_count IS '引流主页次数(小红书普通笔记)'; +COMMENT ON COLUMN cmt_self_video_metrics.live_redirect_count IS '引流直播间次数(小红书普通笔记)'; +COMMENT ON COLUMN cmt_self_video_metrics.payment_amount IS '笔记支付金额(小红书商品笔记)'; +COMMENT ON COLUMN cmt_self_video_metrics.payment_orders IS '笔记支付订单数(小红书商品笔记)'; +COMMENT ON COLUMN cmt_self_video_metrics.coin_count IS '投币数(B站)'; +COMMENT ON COLUMN cmt_self_video_metrics.favorite_count IS '收藏数(B站)'; +COMMENT ON COLUMN cmt_self_video_metrics.danmaku_count IS '弹幕数(B站)'; +COMMENT ON COLUMN cmt_self_video_metrics.page_metrics IS + '来源页面/接口展示的全部原始列 {列名: 展示值}，保留未建模指标'; diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/schema_gyxx_super_data.sql b/src/gyxx_flow/modules/content_marketing/data/tools/schema_gyxx_super_data.sql index 663732c..dce00dd 100644 --- a/src/gyxx_flow/modules/content_marketing/data/tools/schema_gyxx_super_data.sql +++ b/src/gyxx_flow/modules/content_marketing/data/tools/schema_gyxx_super_data.sql @@ -1,5 +1,5 @@ -- 内容营销模块 PostgreSQL schema --- 表结构：cmt_styles / cmt_creators / cmt_style_creators / cmt_notes_master / cmt_comments +-- 表结构：cmt_styles / cmt_creators / cmt_style_creators / cmt_notes_master / cmt_note_metric_snapshots / cmt_self_video_metrics / cmt_comments -- 适用于 PostgreSQL 14+ -- 在当前已连接数据库中执行；项目默认数据库为本地 gyxx_flow @@ -206,6 +206,74 @@ CREATE INDEX IF NOT EXISTS idx_cmt_note_metric_snapshots_date CREATE INDEX IF NOT EXISTS idx_cmt_note_metric_snapshots_note ON cmt_note_metric_snapshots (note_id, metric_date); +-- 自营内容每日指标快照（跨平台: 抖音蝉妈妈导出 / B站公开API / 小红书ARK管理后台） +-- 一行 = 一个自营账号的一条内容(视频/笔记)在一个业务日期的完整采集值; +-- 不与 cmt_notes_master 冲突: 该表按账号+内容独立留存全部内容(含未登记款式表的)。 +CREATE TABLE IF NOT EXISTS cmt_self_video_metrics ( + id BIGSERIAL PRIMARY KEY, + platform VARCHAR(32) NOT NULL DEFAULT 'douyin', + account_alias VARCHAR(64) NOT NULL DEFAULT '', -- 内部账号别名,如 抖音·机能号 + account_name VARCHAR(255) NOT NULL DEFAULT '', -- 平台账号名: 蝉妈妈达人昵称/B站UP主/小红书店铺 + platform_account_id VARCHAR(128) NOT NULL DEFAULT '', -- 平台账号ID: 抖音号/B站mid/小红书登录手机号 + video_id VARCHAR(128) NOT NULL, -- 内容稳定ID: 抖音视频ID/B站BV号/小红书笔记ID(或标题指纹) + title VARCHAR(512) DEFAULT '', + chanmama_url TEXT DEFAULT '', -- 蝉妈妈视频链接(仅抖音) + video_url TEXT DEFAULT '', -- 平台内容链接: 抖音视频/B站视频/小红书笔记 + duration VARCHAR(32) DEFAULT '', -- 视频时长原始文本 + publish_time TIMESTAMPTZ DEFAULT NULL, + metric_date DATE NOT NULL, + exposure BIGINT DEFAULT NULL, -- 主触达: 抖音预估曝光/B站播放量/小红书阅读数 + like_count BIGINT DEFAULT NULL, -- 点赞数 + comment_count BIGINT DEFAULT NULL, -- 评论数(B站为回复数) + share_count BIGINT DEFAULT NULL, -- 转发/分享数 + est_sales_count BIGINT DEFAULT NULL, -- 预估销量(抖音) + est_sales_amount NUMERIC(14,2) DEFAULT NULL, -- 预估销售额(抖音)/笔记支付金额(小红书) + note_type VARCHAR(16) DEFAULT '', -- goods/normal(小红书), video/note(抖音) + creation_method VARCHAR(64) DEFAULT '', -- 创作方式(小红书) + publish_status VARCHAR(32) DEFAULT '', -- 发布状态(小红书) + mounted_products TEXT DEFAULT '', -- 挂载商品(小红书商品笔记) + homepage_redirect_count BIGINT DEFAULT NULL, -- 引流主页次数(小红书) + live_redirect_count BIGINT DEFAULT NULL, -- 引流直播间次数(小红书) + payment_amount NUMERIC(14,2) DEFAULT NULL, -- 笔记支付金额(小红书商品笔记) + payment_orders BIGINT DEFAULT NULL, -- 笔记支付订单数(小红书商品笔记) + coin_count BIGINT DEFAULT NULL, -- 投币数(B站) + favorite_count BIGINT DEFAULT NULL, -- 收藏数(B站) + danmaku_count BIGINT DEFAULT NULL, -- 弹幕数(B站) + page_metrics JSONB DEFAULT '{}'::jsonb, -- 来源页面/接口展示的全部原始列 + source_file TEXT DEFAULT '', -- 来源导出文件名 + observed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uniq_cmt_self_video_metric + UNIQUE (platform, account_name, video_id, metric_date) +); +COMMENT ON TABLE cmt_self_video_metrics IS + '自营内容每日指标快照；一行=一个自营账号的一条内容(视频/笔记)在一个业务日期的采集值；平台含 douyin/bilibili/xiaohongshu'; +COMMENT ON COLUMN cmt_self_video_metrics.platform IS '平台：douyin / bilibili / xiaohongshu'; +COMMENT ON COLUMN cmt_self_video_metrics.account_alias IS '内部账号别名（飞书「投放账号类型」口径），由 self_operated_accounts.json 映射'; +COMMENT ON COLUMN cmt_self_video_metrics.account_name IS '平台账号名，采集数据自带的稳定账号身份'; +COMMENT ON COLUMN cmt_self_video_metrics.video_id IS '内容稳定ID：抖音视频数字ID、B站BV号、小红书笔记ID(无链接时用标题+发布时间指纹)'; +COMMENT ON COLUMN cmt_self_video_metrics.video_url IS '平台内容链接：抖音视频链接 / B站视频链接 / 小红书笔记链接'; +COMMENT ON COLUMN cmt_self_video_metrics.metric_date IS '调度器业务日期，不使用进程实际结束日期'; +COMMENT ON COLUMN cmt_self_video_metrics.exposure IS '主触达指标：抖音预估曝光 / B站播放量 / 小红书笔记阅读数；与飞书曝光槽位回填同源'; +COMMENT ON COLUMN cmt_self_video_metrics.note_type IS '内容类型：小红书 goods(商品笔记)/normal(普通笔记)；抖音 video/note(按链接)；其余为空'; +COMMENT ON COLUMN cmt_self_video_metrics.creation_method IS '创作方式(小红书：手动创作等)'; +COMMENT ON COLUMN cmt_self_video_metrics.publish_status IS '发布状态(小红书：已发布等)'; +COMMENT ON COLUMN cmt_self_video_metrics.mounted_products IS '挂载商品(小红书商品笔记，页面展示文本)'; +COMMENT ON COLUMN cmt_self_video_metrics.homepage_redirect_count IS '引流主页次数(小红书普通笔记)'; +COMMENT ON COLUMN cmt_self_video_metrics.live_redirect_count IS '引流直播间次数(小红书普通笔记)'; +COMMENT ON COLUMN cmt_self_video_metrics.payment_amount IS '笔记支付金额(小红书商品笔记)'; +COMMENT ON COLUMN cmt_self_video_metrics.payment_orders IS '笔记支付订单数(小红书商品笔记)'; +COMMENT ON COLUMN cmt_self_video_metrics.coin_count IS '投币数(B站)'; +COMMENT ON COLUMN cmt_self_video_metrics.favorite_count IS '收藏数(B站)'; +COMMENT ON COLUMN cmt_self_video_metrics.danmaku_count IS '弹幕数(B站)'; +COMMENT ON COLUMN cmt_self_video_metrics.page_metrics IS '来源页面/接口展示的全部原始列 {列名: 展示值}，保留未建模指标'; + +CREATE INDEX IF NOT EXISTS idx_cmt_self_video_metrics_date + ON cmt_self_video_metrics (metric_date, platform); +CREATE INDEX IF NOT EXISTS idx_cmt_self_video_metrics_account + ON cmt_self_video_metrics (platform, account_name, publish_time); + -- 评论表 CREATE TABLE IF NOT EXISTS cmt_comments ( id BIGSERIAL PRIMARY KEY, @@ -346,7 +414,7 @@ DO $$ DECLARE tbl TEXT; BEGIN - FOR tbl IN SELECT unnest(ARRAY['cmt_styles', 'cmt_creators', 'cmt_style_creators', 'cmt_notes_master', 'cmt_comments', 'cmt_weekly_summary', 'cmt_monthly_summary', 'cmt_creator_report']) + FOR tbl IN SELECT unnest(ARRAY['cmt_styles', 'cmt_creators', 'cmt_style_creators', 'cmt_notes_master', 'cmt_note_metric_snapshots', 'cmt_self_video_metrics', 'cmt_comments', 'cmt_weekly_summary', 'cmt_monthly_summary', 'cmt_creator_report']) LOOP IF NOT EXISTS ( SELECT 1 FROM pg_trigger WHERE tgname = 'trg_' || tbl || '_updated_at' diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/sync_metrics_to_cmt_notes.py b/src/gyxx_flow/modules/content_marketing/data/tools/sync_metrics_to_cmt_notes.py index b821d03..3016a2a 100644 --- a/src/gyxx_flow/modules/content_marketing/data/tools/sync_metrics_to_cmt_notes.py +++ b/src/gyxx_flow/modules/content_marketing/data/tools/sync_metrics_to_cmt_notes.py @@ -534,8 +534,10 @@ def _is_current_v2_artifact(path: Path, business_date: dt.date | None) -> bool: return modified_date == business_date -def collect_metrics_from_v2() -> list[dict]: - """扫一遍 v2_results,聚合每个 record_id 的最新指标""" +def collect_metrics_from_v2( + *, self_operated: bool | None = None +) -> list[dict]: + """扫一遍 v2_results,按合作/自营范围聚合 record_id 的最新指标""" out: list[dict] = [] business_date = _runtime_business_date() for path in sorted(V2_DIR.glob("*.json")): @@ -544,7 +546,12 @@ def collect_metrics_from_v2() -> list[dict]: result = detect_platform(path) if not result or result[0] is None: continue - platform, self_operated = result + platform, artifact_self_operated = result + if ( + self_operated is not None + and artifact_self_operated is not self_operated + ): + continue idx, style_name, _, _ = parse_v2_filename(path.name) if idx is None: continue @@ -577,7 +584,7 @@ def collect_metrics_from_v2() -> list[dict]: "like_count": None, "collect_count": None, "comment_count": None, - "self_operated": self_operated, + "self_operated": artifact_self_operated, }) else: items = data.get("results") or [] @@ -603,7 +610,7 @@ def collect_metrics_from_v2() -> list[dict]: "like_count": parse_int(item.get("like_count")), "collect_count": parse_int(item.get("collect_count")), "comment_count": parse_int(item.get("comment_count")), - "self_operated": self_operated, + "self_operated": artifact_self_operated, }) return out @@ -1223,8 +1230,14 @@ def sync( dry_run: bool = False, insert_missing: bool = True, metric_date: dt.date | str | None = None, + *, + self_operated: bool | None = None, ) -> dict: - metrics = collect_metrics_from_v2() + metrics = ( + collect_metrics_from_v2() + if self_operated is None + else collect_metrics_from_v2(self_operated=self_operated) + ) snapshot_date = _resolve_metric_date(metric_date) print( f"[sync] v2_results 中可同步的命中记录: {len(metrics)} 条; " @@ -1241,15 +1254,19 @@ def sync( mapping = None self_mapping = None if insert_missing: - try: - mapping = feishu_mapping.load_mapping(force_refresh=False) - except Exception as exc: - print(f"[WARN] feishu_mapping 加载失败,INSERT 路径禁用: {exc}") - insert_missing = False - try: - self_mapping = feishu_mapping.load_mapping(force_refresh=False, self_operated=True) - except Exception as exc: - print(f"[WARN] feishu_mapping(自营) 加载失败: {exc}") + if self_operated is not True: + try: + mapping = feishu_mapping.load_mapping(force_refresh=False) + except Exception as exc: + print(f"[WARN] feishu_mapping 加载失败: {exc}") + if self_operated is not False: + try: + self_mapping = feishu_mapping.load_mapping( + force_refresh=False, + self_operated=True, + ) + except Exception as exc: + print(f"[WARN] feishu_mapping(自营) 加载失败: {exc}") for row in metrics: record_id = row["record_id"] @@ -1406,9 +1423,20 @@ def sync( old_pub_row = cur.fetchone() if old_pub_row and old_pub_row[0] is None: try: - style_name = row.get("style_name") - bf_update = _backfill_record(mapping, style_name, record_id) or \ - (self_mapping and _backfill_record(self_mapping, style_name, record_id)) or {} + active_mapping = ( + self_mapping + if row.get("self_operated") + else mapping + ) + bf_update = ( + _backfill_record( + active_mapping, + row.get("style_name"), + record_id, + ) + if active_mapping + else {} + ) new_pub = bf_update.get("publish_time") except Exception: new_pub = None @@ -1460,6 +1488,12 @@ def main(): default=None, help="快照业务日期 YYYY-MM-DD；默认使用 GYXX_BUSINESS_DATE", ) + parser.add_argument( + "--scope", + choices=("all", "collaborators", "self-operated"), + default="all", + help="同步来源范围；默认同步合作达人与自营内容", + ) parser.add_argument( "--repair-record-collisions", action="store_true", @@ -1503,6 +1537,8 @@ def main(): help="关闭 INSERT 路径(默认开启:对 v2_results 有但 cmt_notes_master 缺的行,回填 feishu 后 INSERT)", ) args = parser.parse_args() + if args.scope != "all" and args.repair_record_collisions: + parser.error("--scope 不能与事故记录修复参数一起使用") if args.repair_record_collisions and not args.repair_window: parser.error("--repair-record-collisions 至少需要一个 --repair-window") if args.repair_window and not args.repair_record_collisions: @@ -1524,6 +1560,7 @@ def main(): insert_missing = args.insert_missing or not args.no_insert_missing print(f"[sync] 模式: {'DRY-RUN' if args.dry_run else '正式写入'} " + f"范围: {args.scope} " f"insert-missing: {'ON' if insert_missing else 'OFF'} " f"repair-record-collisions: " f"{'ON' if args.repair_record_collisions else 'OFF'} " @@ -1589,6 +1626,8 @@ def main(): "dry_run": args.dry_run, "insert_missing": insert_missing, } + if args.scope != "all": + sync_kwargs["self_operated"] = args.scope == "self-operated" if args.metric_date is not None: sync_kwargs["metric_date"] = args.metric_date result = sync(**sync_kwargs) diff --git a/src/gyxx_flow/modules/content_marketing/data/tools/sync_self_video_metrics.py b/src/gyxx_flow/modules/content_marketing/data/tools/sync_self_video_metrics.py new file mode 100644 index 0000000..f25dab0 --- /dev/null +++ b/src/gyxx_flow/modules/content_marketing/data/tools/sync_self_video_metrics.py @@ -0,0 +1,616 @@ +#!/usr/bin/env python +"""把自营内容(抖音/B站/小红书)全量指标同步到 PostgreSQL cmt_self_video_metrics。 + +数据源(按平台): + douyin data/raw/content_marketing/chanmama/ 各账号最新一份导出 + ('<达人昵称>_视频记录_周期内发布视频_<日期范围>.xlsx')或 --excel 显式指定 + bilibili data/normalized/content_marketing/self_bilibili/daily.json + (self_bilibili_scraper.py 由 B站公开 API 产出) + xiaohongshu data/normalized/content_marketing/self_xhs/daily.json + (self_xhs_scraper.py 由 ARK 笔记管理页产出) + +目标表: cmt_self_video_metrics(schema / migrations/009+010) + 一行 = 平台+账号+内容+业务日期;同日重跑按唯一键幂等 upsert。 + page_metrics 保留来源页面/接口展示的全部原始列。 + +账号身份: 内部别名按 data/config/self_operated_accounts.json 由平台账号身份解析, +未配置的账号以空别名入库并在日志中提示。 + +飞书曝光槽位回填仍由 chanmama_scraper.py / self_bilibili_scraper.py 负责, +本脚本只写 PostgreSQL。 + +用法: + python sync_self_video_metrics.py # 三平台全同步 + python sync_self_video_metrics.py --platform douyin # 只同步抖音 + python sync_self_video_metrics.py --dry-run # 仅预览 + python sync_self_video_metrics.py --excel # 抖音指定导出文件 + python sync_self_video_metrics.py --metric-date 2026-09-28 --allow-stale # 补历史 +""" +import argparse +import datetime as dt +import json +import os +import sys +from pathlib import Path + +from gyxx_flow.modules.content_marketing import chanmama_export, self_operated_accounts +from gyxx_flow.modules.content_marketing.runtime_paths import PATHS + +try: + from psycopg.types.json import Jsonb +except ImportError: # pragma: no cover - psycopg 为运行必装依赖 + Jsonb = dict + + +def configure_console_output(stdout, stderr) -> None: + """Keep Windows GBK logs alive when account names contain Unicode symbols.""" + for stream in (stdout, stderr): + if stream is None: + continue + try: + stream.reconfigure(errors="replace") + except (AttributeError, OSError, ValueError): + pass + + +configure_console_output(sys.stdout, sys.stderr) + +DOWNLOAD_DIR = PATHS.raw_root / "chanmama" +ACCOUNTS_CONFIG = self_operated_accounts.accounts_config_path(PATHS.config_root) +BILIBILI_ARTIFACT = PATHS.normalized_root / "self_bilibili" / "daily.json" +XHS_ARTIFACT = PATHS.normalized_root / "self_xhs" / "daily.json" + +PLATFORMS = ("douyin", "bilibili", "xiaohongshu") + +TABLE_NAME = "cmt_self_video_metrics" + +# 行值列序(与下方 INSERT 列序一一对应) +ROW_FIELDS = ( + "platform", "account_alias", "account_name", "platform_account_id", + "video_id", "title", "chanmama_url", "video_url", "duration", + "publish_time", "metric_date", "exposure", "like_count", "comment_count", + "share_count", "est_sales_count", "est_sales_amount", "note_type", + "creation_method", "publish_status", "mounted_products", + "homepage_redirect_count", "live_redirect_count", "payment_amount", + "payment_orders", "coin_count", "favorite_count", "danmaku_count", + "page_metrics", "source_file", +) + +_ROW_TEMPLATE = {field: None for field in ROW_FIELDS} + + +def runtime_business_date() -> dt.date | None: + """Return the orchestrator business date when one is explicitly bound.""" + value = os.environ.get("GYXX_BUSINESS_DATE", "").strip() + if not value: + return None + try: + return dt.date.fromisoformat(value) + except ValueError as exc: + raise RuntimeError("GYXX_BUSINESS_DATE must be YYYY-MM-DD") from exc + + +def resolve_metric_date(value: dt.date | str | None) -> dt.date: + if value is not None: + if isinstance(value, dt.datetime): + return value.date() + if isinstance(value, dt.date): + return value + try: + return dt.date.fromisoformat(str(value).strip()) + except ValueError as exc: + raise ValueError("metric date must be YYYY-MM-DD") from exc + return runtime_business_date() or dt.datetime.now().astimezone().date() + + +def is_current_artifact(path: Path, business_date: dt.date) -> bool: + modified_date = dt.datetime.fromtimestamp(path.stat().st_mtime).astimezone().date() + return modified_date == business_date + + +def make_row(platform: str, metric_date: dt.date, source_file: str, **values) -> tuple: + """按 ROW_FIELDS 顺序构造一行 upsert 参数(缺省 None/'').""" + row = dict(_ROW_TEMPLATE) + row["platform"] = platform + row["metric_date"] = metric_date + row["source_file"] = source_file + for key, value in values.items(): + if key not in row: + raise KeyError(f"unknown row field: {key}") + row[key] = value + row["account_alias"] = row["account_alias"] or "" + row["account_name"] = row["account_name"] or "" + row["platform_account_id"] = row["platform_account_id"] or "" + row["title"] = str(row["title"] or "")[:512] + row["chanmama_url"] = str(row["chanmama_url"] or "") + row["video_url"] = str(row["video_url"] or "") + row["duration"] = str(row["duration"] or "")[:32] + row["note_type"] = str(row["note_type"] or "")[:16] + row["creation_method"] = str(row["creation_method"] or "")[:64] + row["publish_status"] = str(row["publish_status"] or "")[:32] + row["mounted_products"] = str(row["mounted_products"] or "") + row["page_metrics"] = Jsonb(row["page_metrics"] or {}) + row["source_file"] = str(row["source_file"] or "") + return tuple(row[field] for field in ROW_FIELDS) + + +# ============================================================ +# 各平台来源 → 统一行 +# ============================================================ +def _douyin_note_type(video_url: str) -> str: + text = str(video_url or "") + if "/video/" in text: + return "video" + if "/note/" in text: + return "note" + return "" + + +def collect_douyin_rows( + accounts: list[dict], metric_date: dt.date, explicit_excel: list[str], + *, only_accounts: list[str] | None = None, use_file_mtime: bool = False, +) -> tuple[dict[tuple[str, str, str, dt.date], tuple], dict[str, int], list[Path]]: + """解析蝉妈妈导出构造抖音行。 + + use_file_mtime=True 时(历史回填),每份导出按其落盘日期作为该快照的 + 业务日期——历史导出正是当日采集值的快照,等价于补录每日采集。 + 行键随之含 metric_date,同一视频在不同日期各留一行。 + """ + if explicit_excel: + export_paths = [Path(item) for item in explicit_excel] + else: + export_paths = [ + Path(path) + for path in chanmama_export.find_all_chanmama_excels_grouped( + str(DOWNLOAD_DIR) + ) + ] + stats = { + "files": 0, "records": 0, "rows": 0, "skip_no_identity": 0, + "skip_no_video_id": 0, "skip_account_filter": 0, + "unknown_alias_accounts": 0, + } + unknown_accounts: set[str] = set() + rows: dict[tuple[str, str, str, dt.date], tuple] = {} + + for path in export_paths: + if not path.exists(): + print(f"⚠️ 文件不存在,跳过: {path}") + continue + # 旧版导出(12列)没有 达人昵称/抖音号 列,账号身份回退用文件名前缀 + # ('<达人昵称>_视频记录_周期内发布视频_….xlsx') + stem = path.name[:-5] if path.name.lower().endswith(".xlsx") else path.name + marker = stem.find("_视频记录_") + file_account = stem[:marker] if marker > 0 else "" + stats["files"] += 1 + try: + records, _headers = chanmama_export.parse_chanmama_export(str(path)) + except Exception as exc: + print(f"⚠️ 解析失败,跳过: {path}: {exc}") + continue + for record in records: + stats["records"] += 1 + account_name = record.get("creator_name") or file_account + douyin_account = record.get("douyin_account") or "" + if not account_name and not douyin_account: + stats["skip_no_identity"] += 1 + continue + video_id = chanmama_export.extract_video_id( + record.get("chanmama_url"), record.get("video_url") + ) + if not video_id: + stats["skip_no_video_id"] += 1 + continue + alias = self_operated_accounts.resolve_alias( + accounts, + account_name=account_name, + platform_account_id=douyin_account, + ) + if not alias: + unknown_accounts.add(account_name or douyin_account) + if only_accounts is not None: + keys = {key.casefold() for key in only_accounts} + if ( + alias.casefold() not in keys + and account_name.casefold() not in keys + and douyin_account.casefold() not in keys + ): + stats["skip_account_filter"] += 1 + continue + video_url = record.get("video_url") or "" + file_date = metric_date + if use_file_mtime: + file_date = dt.datetime.fromtimestamp( + path.stat().st_mtime + ).astimezone().date() + key = ("douyin", account_name, video_id, file_date) + rows[key] = make_row( + "douyin", file_date, path.name, + account_alias=alias, + account_name=account_name, + platform_account_id=douyin_account, + video_id=video_id, + title=record.get("title"), + chanmama_url=record.get("chanmama_url") or "", + video_url=video_url, + duration=record.get("duration") or "", + publish_time=chanmama_export.parse_publish_time( + record.get("publish_time") + ), + exposure=record.get("exposure"), + like_count=record.get("like_count"), + comment_count=record.get("comment_count"), + share_count=record.get("share_count"), + est_sales_count=record.get("est_sales_count"), + est_sales_amount=record.get("est_sales_amount"), + note_type=_douyin_note_type(video_url), + ) + stats["rows"] = len(rows) + stats["unknown_alias_accounts"] = len(unknown_accounts) + for name in sorted(unknown_accounts): + print(f"⚠️ [douyin] 账号未配置别名,account_alias 将为空: {name}") + return rows, stats, export_paths + + +def collect_bilibili_rows( + accounts: list[dict], metric_date: dt.date, + *, only_accounts: list[str] | None = None, +) -> tuple[dict[tuple[str, str, str], tuple], dict[str, int], Path | None]: + from gyxx_flow.modules.content_marketing.self_bilibili_scraper import ( + format_duration, + ) + + stats = {"records": 0, "rows": 0, "skip_no_owner": 0, + "unknown_alias_accounts": 0} + rows: dict[tuple[str, str, str], tuple] = {} + if not BILIBILI_ARTIFACT.exists(): + print(f"⚠️ [bilibili] artifact 不存在,跳过该平台: {BILIBILI_ARTIFACT}") + return rows, stats, None + unknown_accounts: set[str] = set() + try: + artifact = json.loads(BILIBILI_ARTIFACT.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"⚠️ [bilibili] artifact 读取失败: {exc}") + return rows, stats, None + for video in artifact.get("videos", []): + stats["records"] += 1 + account_name = str(video.get("owner_name") or "").strip() + if not account_name: + stats["skip_no_owner"] += 1 + continue + video_id = str(video.get("video_id") or "").strip() + if not video_id: + stats["skip_no_owner"] += 1 + continue + alias = self_operated_accounts.resolve_alias( + accounts, account_name=account_name + ) + if not alias: + unknown_accounts.add(account_name) + if only_accounts is not None: + keys = {key.casefold() for key in only_accounts} + if alias.casefold() not in keys and account_name.casefold() not in keys: + stats["skip_no_owner"] += 1 + continue + key = ("bilibili", account_name, video_id) + rows[key] = make_row( + "bilibili", metric_date, BILIBILI_ARTIFACT.name, + account_alias=alias, + account_name=account_name, + platform_account_id=str(video.get("owner_mid") or ""), + video_id=video_id, + title=video.get("title"), + video_url=video.get("video_url") or "", + duration=format_duration(video.get("duration_seconds")), + publish_time=chanmama_export.parse_publish_time( + video.get("publish_time") + ), + exposure=video.get("view_count"), + like_count=video.get("like_count"), + comment_count=video.get("reply_count"), + share_count=video.get("share_count"), + coin_count=video.get("coin_count"), + favorite_count=video.get("favorite_count"), + danmaku_count=video.get("danmaku_count"), + ) + stats["rows"] = len(rows) + stats["unknown_alias_accounts"] = len(unknown_accounts) + for name in sorted(unknown_accounts): + print(f"⚠️ [bilibili] 账号未配置别名,account_alias 将为空: {name}") + return rows, stats, BILIBILI_ARTIFACT + + +def collect_xiaohongshu_rows( + accounts: list[dict], metric_date: dt.date, + *, only_accounts: list[str] | None = None, +) -> tuple[dict[tuple[str, str, str], tuple], dict[str, int], Path | None]: + stats = {"records": 0, "rows": 0, "skip_no_title": 0, + "unknown_alias_accounts": 0} + rows: dict[tuple[str, str, str], tuple] = {} + if not XHS_ARTIFACT.exists(): + print(f"⚠️ [xiaohongshu] artifact 不存在,跳过该平台: {XHS_ARTIFACT}") + return rows, stats, None + unknown_accounts: set[str] = set() + try: + artifact = json.loads(XHS_ARTIFACT.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"⚠️ [xiaohongshu] artifact 读取失败: {exc}") + return rows, stats, None + artifact_account = str(artifact.get("account_name") or "").strip() + # ARK artifact 按账号粒度产出,循环外一次性解析别名与平台账号ID + matched_entry = next( + ( + entry + for entry in accounts + if artifact_account + and entry["account_name"].casefold() == artifact_account.casefold() + ), + None, + ) + alias = matched_entry["alias"] if matched_entry else "" + config_account_id = matched_entry["platform_account_id"] if matched_entry else "" + if artifact_account and not alias: + unknown_accounts.add(artifact_account) + for note in artifact.get("notes", []): + stats["records"] += 1 + title = str(note.get("title") or "").strip() + video_id = str(note.get("video_id") or "").strip() + if not title or not video_id: + stats["skip_no_title"] += 1 + continue + if only_accounts is not None: + keys = {key.casefold() for key in only_accounts} + if alias.casefold() not in keys and artifact_account.casefold() not in keys: + stats["skip_no_title"] += 1 + continue + key = ("xiaohongshu", artifact_account, video_id) + rows[key] = make_row( + "xiaohongshu", metric_date, XHS_ARTIFACT.name, + account_alias=alias, + account_name=artifact_account, + platform_account_id=config_account_id, + video_id=video_id, + title=title, + video_url=note.get("note_url") or "", + publish_time=chanmama_export.parse_publish_time( + note.get("publish_time") + ), + exposure=note.get("read_count"), + like_count=note.get("like_count"), + note_type=note.get("note_type") or "", + creation_method=note.get("creation_method") or "", + publish_status=note.get("publish_status") or "", + mounted_products=note.get("mounted_products") or "", + homepage_redirect_count=note.get("homepage_redirect_count"), + live_redirect_count=note.get("live_redirect_count"), + payment_amount=note.get("payment_amount"), + payment_orders=note.get("payment_orders"), + page_metrics=note.get("raw_columns") or {}, + ) + stats["rows"] = len(rows) + stats["unknown_alias_accounts"] = len(unknown_accounts) + for name in sorted(unknown_accounts): + print(f"⚠️ [xiaohongshu] 账号未配置别名,account_alias 将为空: {name}") + return rows, stats, XHS_ARTIFACT + + +COLLECTORS = { + "douyin": lambda accounts, metric_date, excel, only, use_mtime=False: ( + collect_douyin_rows( + accounts, metric_date, excel, + only_accounts=only, + use_file_mtime=use_mtime, + ) + ), + "bilibili": lambda accounts, metric_date, excel, only, use_mtime=False: ( + collect_bilibili_rows(accounts, metric_date, only_accounts=only) + ), + "xiaohongshu": lambda accounts, metric_date, excel, only, use_mtime=False: ( + collect_xiaohongshu_rows(accounts, metric_date, only_accounts=only) + ), +} + + +def ensure_table(cur) -> None: + cur.execute( + """ + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() AND table_name = %s + """, + (TABLE_NAME,), + ) + if not cur.fetchone(): + raise RuntimeError( + f"表 {TABLE_NAME} 不存在;请先执行 data/tools/init_db.py 或 " + "migrations/009 + 010" + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="同步自营内容(抖音/B站/小红书)全量指标到 cmt_self_video_metrics" + ) + parser.add_argument( + "--platform", + action="append", + choices=PLATFORMS, + dest="platforms", + default=[], + metavar="NAME", + help="只同步指定平台(可重复: douyin/bilibili/xiaohongshu);默认三平台全同步", + ) + parser.add_argument("--dry-run", action="store_true", help="仅预览,不写入") + parser.add_argument( + "--metric-date", + type=dt.date.fromisoformat, + default=None, + help="快照业务日期 YYYY-MM-DD;默认使用 GYXX_BUSINESS_DATE", + ) + parser.add_argument( + "--excel", + action="append", + default=[], + metavar="PATH", + help="抖音平台显式指定导出文件(可重复);默认按账号分组取下载目录最新一份", + ) + parser.add_argument( + "--account", + action="append", + default=[], + metavar="NAME", + help="只同步指定账号(别名/账号名,可重复);默认全量", + ) + parser.add_argument( + "--accounts-config", + default=str(ACCOUNTS_CONFIG), + help="自营账号别名映射配置路径", + ) + parser.add_argument( + "--allow-stale", + action="store_true", + help="跳过「来源文件日期须等于业务日期」的陈旧产物检查(手动补历史用)", + ) + parser.add_argument( + "--use-file-mtime", + action="store_true", + help="抖音历史回填: 每份导出按其落盘日期作为业务日期各留一行快照" + "(配合 --excel 指定历史文件;隐含跳过陈旧检查)", + ) + args = parser.parse_args() + + business_date_bound = runtime_business_date() is not None + metric_date = resolve_metric_date(args.metric_date) + selected_platforms = tuple(args.platforms) or PLATFORMS + only_accounts = args.account or None + + all_rows: dict[tuple, tuple] = {} + sources: list[Path] = [] + for platform in selected_platforms: + accounts = self_operated_accounts.load_accounts( + platform, args.accounts_config + ) + rows, stats, source = COLLECTORS[platform]( + accounts, metric_date, args.excel, only_accounts, + args.use_file_mtime and platform == "douyin", + ) + if source is None: + continue + sources.extend(source if isinstance(source, list) else [source]) + summary = " ".join(f"{key}={value}" for key, value in stats.items()) + print(f"[sync-self][{platform}] {summary}") + all_rows.update(rows) + + if not sources: + print("❌ 没有任何平台的来源文件可用") + return 1 + stale = [ + str(path) + for path in sources + if business_date_bound + and not args.allow_stale + and not args.use_file_mtime + and not is_current_artifact(path, metric_date) + ] + if stale and not args.allow_stale: + print( + f"❌ 以下来源文件不是业务日期({metric_date.isoformat()})的产物; " + "补历史请显式传 --metric-date 并加 --allow-stale:\n " + + "\n ".join(stale) + ) + return 1 + + print( + f"[sync-self] 平台={'+'.join(selected_platforms)} 待写={len(all_rows)} 行; " + f"业务日期={metric_date.isoformat()}" + ) + if not all_rows: + print("❌ 没有可入库的内容指标行") + return 1 + + by_platform: dict[str, int] = {} + for row in all_rows.values(): + platform = row[0] + by_platform[platform] = by_platform.get(platform, 0) + 1 + for platform, count in sorted(by_platform.items()): + print(f" 平台 {platform}: {count} 条") + + if args.dry_run: + print(f"[sync-self] DRY-RUN: 未写入 {TABLE_NAME}") + return 0 + + import psycopg + + from gyxx_flow.modules.content_marketing.data.tools.db import get_db_config + + with psycopg.connect(**get_db_config()) as conn: + with conn.cursor() as cur: + ensure_table(cur) + for row in all_rows.values(): + # SQL 为静态字面量,29 个值全部经 %s 占位符参数绑定传入。 + cur.execute( + """ + INSERT INTO cmt_self_video_metrics ( + platform, account_alias, account_name, + platform_account_id, video_id, title, chanmama_url, + video_url, duration, publish_time, metric_date, + exposure, like_count, comment_count, share_count, + est_sales_count, est_sales_amount, note_type, + creation_method, publish_status, mounted_products, + homepage_redirect_count, live_redirect_count, + payment_amount, payment_orders, coin_count, + favorite_count, danmaku_count, page_metrics, + source_file, + observed_at, created_at, updated_at + ) VALUES ( + %s, %s, %s, + %s, %s, %s, %s, + %s, %s, %s, %s, + %s, %s, %s, %s, + %s, %s, %s, + %s, %s, %s, + %s, %s, + %s, %s, %s, + %s, %s, %s, + %s, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + ON CONFLICT (platform, account_name, video_id, metric_date) + DO UPDATE SET + account_alias = EXCLUDED.account_alias, + platform_account_id = EXCLUDED.platform_account_id, + title = EXCLUDED.title, + chanmama_url = EXCLUDED.chanmama_url, + video_url = EXCLUDED.video_url, + duration = EXCLUDED.duration, + publish_time = EXCLUDED.publish_time, + exposure = EXCLUDED.exposure, + like_count = EXCLUDED.like_count, + comment_count = EXCLUDED.comment_count, + share_count = EXCLUDED.share_count, + est_sales_count = EXCLUDED.est_sales_count, + est_sales_amount = EXCLUDED.est_sales_amount, + note_type = EXCLUDED.note_type, + creation_method = EXCLUDED.creation_method, + publish_status = EXCLUDED.publish_status, + mounted_products = EXCLUDED.mounted_products, + homepage_redirect_count = EXCLUDED.homepage_redirect_count, + live_redirect_count = EXCLUDED.live_redirect_count, + payment_amount = EXCLUDED.payment_amount, + payment_orders = EXCLUDED.payment_orders, + coin_count = EXCLUDED.coin_count, + favorite_count = EXCLUDED.favorite_count, + danmaku_count = EXCLUDED.danmaku_count, + page_metrics = EXCLUDED.page_metrics, + source_file = EXCLUDED.source_file, + observed_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + """, + row, + ) + conn.commit() + print(f"[sync-self] 完成: 已幂等写入 {len(all_rows)} 行到 {TABLE_NAME}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py b/src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py index 9485a0f..bcfd72e 100644 --- a/src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py +++ b/src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py @@ -51,12 +51,394 @@ _B23_RE = re.compile(r"https?://b23\.tv/\S+", re.IGNORECASE) _MD_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") _URL_RE = re.compile(r"https?://[^\s)>\]]+") +# --------------------------------------------------------------------------- +# B站空间投稿列表(WBI 签名): 按 UP 主 mid 拉全量投稿 +# ScraplingHttpSession 无 Cookie 状态,空间接口的风控 Cookie 手动获取并携带。 +# --------------------------------------------------------------------------- +BILI_HEADERS = { + "Referer": "https://www.bilibili.com/", + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" + ), +} + +_BILI_COOKIE_CACHE: dict[str, str] = {} + + +def bili_cookie_header(session: ScraplingHttpSession, *, refresh: bool = False) -> str: + """获取并缓存空间接口所需的风控 Cookie 头(buvid3/buvid4/b_nut)。""" + if not refresh and _BILI_COOKIE_CACHE.get("header"): + return _BILI_COOKIE_CACHE["header"] + parts: dict[str, str] = {} + try: + r = session.get( + "https://api.bilibili.com/x/frontend/finger/spi", + headers=BILI_HEADERS, timeout=10, + ) + spi = r.json().get("data") or {} + if spi.get("b_3"): + parts["buvid3"] = str(spi["b_3"]) + if spi.get("b_4"): + parts["buvid4"] = str(spi["b_4"]) + except Exception as exc: + log(f" [WARN] finger/spi 获取 buvid 失败: {exc}") + try: + r = session.get("https://www.bilibili.com/", headers=BILI_HEADERS, + timeout=15) + raw = str(r.headers.get("set-cookie") or "") + match = re.search(r"b_nut=([^;]+)", raw) + if match: + parts.setdefault("b_nut", match.group(1)) + except Exception as exc: + log(f" [WARN] 主站 b_nut 获取失败: {exc}") + header = "; ".join(f"{key}={value}" for key, value in parts.items()) + _BILI_COOKIE_CACHE["header"] = header + return header + +# WBI mixin key 混淆表(B站公开前端算法,2026-09 实测有效) +_MIXIN_KEY_ENC_TAB = [ + 46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, 27, 43, 5, + 49, 33, 9, 42, 19, 29, 28, 14, 39, 12, 38, 41, 13, 37, 48, 7, 16, 24, 55, + 40, 61, 26, 17, 0, 1, 60, 51, 30, 4, 22, 25, 54, 21, 56, 59, 6, 63, 57, + 62, 11, 36, 20, 34, 44, 52, +] + + +def derive_mixin_key(img_key: str, sub_key: str) -> str: + """img_key+sub_key 按混淆表重排,取前 32 位。""" + raw = f"{img_key}{sub_key}" + return "".join(raw[index] for index in _MIXIN_KEY_ENC_TAB)[:32] + + +def wbi_sign(params: dict, mixin_key: str, wts: int) -> dict: + """对查询参数做 WBI 签名: 排序 → 加 wts → md5(query+mixin_key) → w_rid。 + + md5 是 B站 WBI 协议规定的签名算法(服务端按同算法校验请求完整性, + 非密码学安全用途),换成其它哈希会被接口拒绝。 + """ + import hashlib + from urllib.parse import urlencode + + signed = dict(sorted(params.items())) + signed["wts"] = wts + query = urlencode(signed) + signed["w_rid"] = hashlib.md5(f"{query}{mixin_key}".encode()).hexdigest() + return signed + + +def get_mixin_key(session: ScraplingHttpSession) -> str | None: + """从 nav 接口取当前 WBI 密钥(无需登录,但需携带风控 Cookie)。""" + try: + r = session.get( + "https://api.bilibili.com/x/web-interface/nav", + headers={**BILI_HEADERS, "Cookie": bili_cookie_header(session)}, + timeout=10, + ) + payload = r.json() + wbi = (payload.get("data") or {}).get("wbi_img") or {} + img_url = str(wbi.get("img_url") or "") + sub_url = str(wbi.get("sub_url") or "") + if not img_url or not sub_url: + return None + img_key = img_url.rsplit("/", 1)[-1].split(".")[0] + sub_key = sub_url.rsplit("/", 1)[-1].split(".")[0] + return derive_mixin_key(img_key, sub_key) + except Exception as exc: + log(f" [WARN] 获取 WBI 密钥失败: {exc}") + return None + + +def fetch_up_all_bvids( + session: ScraplingHttpSession, mid: str, max_pages: int = 50 +) -> list[str]: + """拉取 UP 主空间全部投稿的 BV 号(逐页,直到返回不足一页)。""" + mixin_key = get_mixin_key(session) + if not mixin_key: + log(" [WARN] 无 WBI 密钥,跳过主页全量采集") + return [] + bvids: list[str] = [] + page_size = 30 + cookie = bili_cookie_header(session) + for page in range(1, max_pages + 1): + params = wbi_sign( + {"mid": mid, "ps": page_size, "pn": page, + "order": "pubdate", "platform": "web"}, + mixin_key, + int(time.time()), + ) + try: + r = session.get( + "https://api.bilibili.com/x/space/wbi/arc/search", + params=params, + headers={**BILI_HEADERS, "Cookie": cookie}, + timeout=15, + ) + payload = r.json() + except Exception as exc: + log(f" [WARN] 空间列表第 {page} 页失败: {exc}") + break + if payload.get("code") == -352 and page == 1: + # 风控 Cookie 失效时刷新一次再试 + cookie = bili_cookie_header(session, refresh=True) + params = wbi_sign( + {"mid": mid, "ps": page_size, "pn": page, + "order": "pubdate", "platform": "web"}, + mixin_key, + int(time.time()), + ) + try: + r = session.get( + "https://api.bilibili.com/x/space/wbi/arc/search", + params=params, + headers={**BILI_HEADERS, "Cookie": cookie}, + timeout=15, + ) + payload = r.json() + except Exception as exc: + log(f" [WARN] 空间列表重试失败: {exc}") + break + if payload.get("code") != 0: + log(f" [WARN] 空间列表返回 code={payload.get('code')} " + f"({payload.get('message')}),停止主页采集") + break + vlist = ((payload.get("data") or {}).get("list") or {}) + items = vlist if isinstance(vlist, list) else [] + page_bvids = [str(item.get("bvid") or "") for item in items + if item.get("bvid")] + bvids.extend(page_bvids) + log(f" 📄 空间投稿第 {page} 页: {len(page_bvids)} 条") + if len(page_bvids) < page_size: + break + time.sleep(0.5) + return bvids + + +def fetch_up_all_bvids_via_browser(mid: str, max_pages: int = 50) -> list[str]: + """用隐身浏览器翻 UP 主空间视频页,收集全部投稿 BV 号。 + + B站空间投稿列表需要登录态;复用隔离 Profile 与持久化 Cookie, + 网格偶发空白时整体重试。返回收集到的 BV 号列表。 + """ + from scrapling.fetchers import StealthySession + + url = f"https://space.bilibili.com/{mid}/upload/video?order=pubdate" + extract_js = ( + "() => [...document.querySelectorAll('a[href*=\"BV\"]')]" + ".map((a) => a.getAttribute('href') || '')" + ) + + def visible_links() -> list[str]: + hrefs = page.evaluate(extract_js) or [] + found: list[str] = [] + for href in hrefs: + match = _BV_RE.search(str(href)) + if match and match.group(0) not in found: + found.append(match.group(0)) + return found + + def wait_for_links(timeout_seconds: float = 20.0) -> list[str]: + deadline = time.time() + timeout_seconds + links: list[str] = [] + while time.time() < deadline: + links = visible_links() + if links: + break + page.wait_for_timeout(1500) + return links + + bvids: list[str] = [] + with StealthySession( + headless=False, + real_chrome=True, + user_data_dir=str(PATHS.browser_profile_dir), + locale="zh-CN", + timezone_id="Asia/Shanghai", + timeout=60_000, + network_idle=False, + disable_resources=False, + google_search=False, + max_pages=2, + page_setup=restore_cookies, + ) as session: + page = ( + session.context.pages[0] + if session.context.pages + else session.context.new_page() + ) + # 空间页的视频列表 XHR 需要登录态;冷指纹/无 Cookie 时网格空白, + # 先访问主站再进空间页,仍空白则整体重试。 + for attempt in range(1, 4): + page.goto("https://www.bilibili.com/", wait_until="domcontentloaded", + timeout=60_000) + page.wait_for_timeout(4000) + page.goto(url, wait_until="domcontentloaded", timeout=60_000) + if wait_for_links(timeout_seconds=25.0): + break + log(f" [WARN] 空间页第 {attempt} 次未渲染出视频(可能需要登录),重试") + for _page_number in range(1, max_pages + 1): + page_bvids = wait_for_links() if _page_number == 1 else visible_links() + if not page_bvids: + if _page_number == 1: + log(" [WARN] 空间视频页未渲染出视频链接" + "(B站登录态缺失时空间列表不可见)") + break + new_on_page = [bv for bv in page_bvids if bv not in bvids] + bvids.extend(new_on_page) + log(f" 📄 空间视频页: 本页 {len(page_bvids)} 条, 累计 {len(bvids)} 条") + next_button = None + try: + candidates = page.locator( + ".be-pager-next, [class*='pager-next'], " + "button:has-text('下一页')" + ).all() + next_button = next( + (el for el in candidates if el.is_visible()), None + ) + except Exception: + next_button = None + if next_button is None: + log(" 📄 空间视频已到最后一页") + break + css = str(next_button.get_attribute("class") or "") + if "disabled" in css.lower(): + log(" 📄 空间视频已到最后一页") + break + try: + next_button.click() + except Exception as exc: + log(f" [WARN] 空间翻页失败,停止: {exc}") + break + # 等网格真正翻页(BV 集合变化),否则会把旧一页重复记入 + changed = False + deadline = time.time() + 12 + while time.time() < deadline: + page.wait_for_timeout(1200) + if visible_links() != page_bvids: + changed = True + break + if not changed: + log(" [WARN] 翻页后内容未变化,停止以避免重复") + break + return bvids + + +def collect_up_homepage_videos( + session: ScraplingHttpSession, mid: str, account_name: str, + registry: dict[str, dict], delay: float = 0.3, max_pages: int = 50, +) -> int: + """把 UP 主主页的全部投稿补进视频登记表(按 BV 号去重),返回新增数。 + + 优先走空间 API(WBI 签名);被风控拒绝时退回隐身浏览器翻页。 + """ + bvids = fetch_up_all_bvids(session, mid, max_pages=max_pages) + if not bvids: + log(" ⚠️ 空间 API 无结果,改用浏览器翻页采集主页投稿") + try: + bvids = fetch_up_all_bvids_via_browser(mid, max_pages=max_pages) + except Exception as exc: + log(f" [WARN] 浏览器主页采集失败: {exc}") + return 0 + new_count = 0 + for bvid in bvids: + if bvid in registry: + continue + info = fetch_bili_video(f"https://www.bilibili.com/video/{bvid}", + session) + if info is None: + log(f" [WARN] 主页视频详情失败: {bvid}") + continue + registry[bvid] = info + new_count += 1 + if delay > 0: + time.sleep(delay) + log(f" 🏠 主页全量采集({account_name}): 共 {len(bvids)} 条投稿, " + f"新增 {new_count} 条") + return new_count + def log(msg: str) -> None: safe = str(msg).encode("gbk", errors="replace").decode("gbk", errors="replace") print(f"[{datetime.now().strftime('%H:%M:%S')}] {safe}", flush=True) +# ============================================================ +# B站登录态: Cookie 持久化 + 扫码登录引导 +# 空间投稿列表需要登录态,首次用 --login-only 扫码,之后调度运行复用。 +# ============================================================ +BILI_COOKIE_FILE = PATHS.browser_cookie_file + + +def restore_cookies(page) -> None: + if not BILI_COOKIE_FILE.exists(): + return + try: + cookies = json.loads(BILI_COOKIE_FILE.read_text(encoding="utf-8")) + page.context.add_cookies(cookies) + log(f"已载入持久化 Cookie: {BILI_COOKIE_FILE}") + except Exception as exc: + log(f"载入 Cookie 失败(忽略,继续用 Profile 登录态): {exc}") + + +def save_state(page) -> None: + BILI_COOKIE_FILE.parent.mkdir(parents=True, exist_ok=True) + cookies = page.context.cookies() + BILI_COOKIE_FILE.write_text( + json.dumps(cookies, ensure_ascii=False, indent=2), encoding="utf-8" + ) + log(f"已保存 Cookie ({len(cookies)} 条): {BILI_COOKIE_FILE}") + + +def _has_login_cookies(page) -> bool: + names = {cookie.get("name") for cookie in page.context.cookies()} + return "SESSDATA" in names and "DedeUserID" in names + + +def login_only(login_timeout: int = 300) -> int: + """打开 B站扫码登录页,等待人工扫码后保存 Cookie。""" + from scrapling.fetchers import StealthySession + + with StealthySession( + headless=False, + real_chrome=True, + user_data_dir=str(PATHS.browser_profile_dir), + locale="zh-CN", + timezone_id="Asia/Shanghai", + timeout=60_000, + network_idle=False, + disable_resources=False, + google_search=False, + max_pages=2, + page_setup=restore_cookies, + ) as session: + page = ( + session.context.pages[0] + if session.context.pages + else session.context.new_page() + ) + page.goto("https://passport.bilibili.com/login", + wait_until="domcontentloaded", timeout=60_000) + page.wait_for_timeout(2000) + if _has_login_cookies(page): + log("✅ 已有 B站登录态,无需扫码") + save_state(page) + return 0 + print("\n" + "=" * 60) + print("🔐 请在打开的浏览器中用哔哩哔哩 App 扫码登录") + print(" 登录成功后脚本会自动继续并保存 Cookie") + print("=" * 60, flush=True) + deadline = time.time() + login_timeout + while time.time() < deadline: + if _has_login_cookies(page): + log("✅ 检测到登录成功") + page.wait_for_timeout(2000) + save_state(page) + return 0 + page.wait_for_timeout(2000) + log(f"❌ 等待扫码超时({login_timeout}s),未保存登录态") + return 1 + + # ============================================================ # 飞书多维表格 # ============================================================ @@ -161,40 +543,91 @@ def resolve_b23(url: str, session: ScraplingHttpSession) -> str | None: return None -def fetch_bili_stat( - url: str, - session: ScraplingHttpSession, -) -> tuple[int | None, int | None]: - """调 B 站 API,返回 (play_count, pubdate_unix_seconds)。 - pubdate 是视频发布时间(Unix 秒),用于补飞书的发布时间字段。 +def fetch_bili_video(url: str, session: ScraplingHttpSession) -> dict | None: + """调 B 站公开 API,返回该视频的全量信息(标题/UP主/时长/发布时间/全指标)。 + + 返回 None 表示 URL 无 BV/AV 号或 API 失败。字段映射: + view→播放量, like/coin/favorite/share/danmaku/reply→互动指标。 """ real = url if _B23_RE.match(url): resolved = resolve_b23(url, session) if not resolved: - return None, None + return None real = resolved bv = _BV_RE.search(real) av = _AV_RE.search(real) if not bv and not av: - return None, None - params = {"bvid": bv.group(0)} if bv else {"aid": av.group(1)} + return None + bvid = bv.group(0) if bv else None + params = {"bvid": bvid} if bv else {"aid": av.group(1)} try: r = session.get("https://api.bilibili.com/x/web-interface/view", params=params, timeout=10, headers={"Referer": "https://www.bilibili.com/"}) j = r.json() if j.get("code") != 0: - return None, None + return None data = j.get("data", {}) stat = data.get("stat", {}) - view = stat.get("view") - pubdate = data.get("pubdate") - return (int(view) if view is not None else None, - int(pubdate) if pubdate else None) + owner = data.get("owner", {}) or {} + video_id = bvid or f"av{data.get('aid') or av.group(1)}" + return { + "video_id": video_id, + "video_url": f"https://www.bilibili.com/video/{video_id}", + "title": str(data.get("title") or "").strip(), + "owner_name": str(owner.get("name") or "").strip(), + "owner_mid": str(owner.get("mid") or "").strip(), + "duration_seconds": data.get("duration"), + "publish_time": ( + datetime.fromtimestamp(int(data["pubdate"])) + if data.get("pubdate") + else None + ), + "view_count": stat.get("view"), + "like_count": stat.get("like"), + "coin_count": stat.get("coin"), + "favorite_count": stat.get("favorite"), + "share_count": stat.get("share"), + "danmaku_count": stat.get("danmaku"), + "reply_count": stat.get("reply"), + } except Exception as exc: log(f" [WARN] B 站 API: {exc}") + return None + + +def fetch_bili_stat( + url: str, + session: ScraplingHttpSession, +) -> tuple[int | None, int | None]: + """调 B 站 API,返回 (play_count, pubdate_unix_seconds)。 + + 兼容旧调用方;全量字段请用 fetch_bili_video。 + pubdate 是视频发布时间(Unix 秒),用于补飞书的发布时间字段。 + """ + info = fetch_bili_video(url, session) + if info is None: return None, None + view = info.get("view_count") + pub = info.get("publish_time") + return ( + int(view) if view is not None else None, + int(pub.timestamp()) if pub else None, + ) + + +def format_duration(seconds) -> str: + """B站 API 时长(秒)→ 'h:mm:ss'/'m:ss' 展示文本。""" + try: + total = int(seconds) + except (TypeError, ValueError): + return "" + hours, rem = divmod(total, 3600) + minutes, secs = divmod(rem, 60) + if hours: + return f"{hours}:{minutes:02d}:{secs:02d}" + return f"{minutes}:{secs:02d}" # ============================================================ @@ -262,7 +695,8 @@ def pick_read_field(fmap: dict, pub_time: datetime | None, # 采集一个款式 # ============================================================ def process_style(style: dict, dry_run: bool, delay: float, - session: ScraplingHttpSession) -> dict: + session: ScraplingHttpSession, + video_registry: dict | None = None) -> dict: base_token = style["base_token"] table_id = style["table_id"] fmap = style.get("field_map", {}) @@ -320,12 +754,24 @@ def process_style(style: dict, dry_run: bool, delay: float, summary["skipped_no_url"] += 1 continue - # 抓 B 站 API:播放量 + pubdate - play, pubdate_unix = fetch_bili_stat(url, session) - if play is None: + # 抓 B 站 API: 播放量 + pubdate(兼容旧时槽回填) + 全量指标登记 + info = fetch_bili_video(url, session) + if info is None: log(f" [SKIP] API 抓不到: '{title[:20]}' (url={url[:50]})") continue + play = info.get("view_count") + pubdate_unix = ( + int(info["publish_time"].timestamp()) + if info.get("publish_time") + else None + ) + if play is None: + log(f" [SKIP] 无播放量: '{title[:20]}'") + continue summary["matched"] += 1 + if video_registry is not None: + # 跨款式去重登记全量视频指标,供统一入库工具消费 + video_registry[info["video_id"]] = info # 飞书发布时间 pub_time = parse_publish_time(rec.get(pub_fid)) @@ -397,13 +843,20 @@ def process_style(style: dict, dry_run: bool, delay: float, # ============================================================ def main() -> int: parser = argparse.ArgumentParser( - description="自营 B 站采集: B 站公开 API 抓播放量,飞书无发布时间用 API pubdate 补") + description="自营 B 站采集: 公开 API 全量视频指标 + UP 主主页全量投稿 + 飞书曝光回填") parser.add_argument("--style", type=int, help="只跑指定款式编号") parser.add_argument("--dry-run", action="store_true", help="不写回飞书") + parser.add_argument("--login-only", action="store_true", + help="只打开 B站扫码登录并保存 Cookie(首次部署引导)") + parser.add_argument("--login-timeout", type=int, default=300, + help="等待人工扫码的秒数") parser.add_argument("--delay", type=float, default=0.3, help="每条请求间隔秒数(防风控)") args = parser.parse_args() + if args.login_only: + return login_only(args.login_timeout) + from gyxx_flow.modules.content_marketing import feishu_mapping try: mapping = feishu_mapping.load_mapping(self_operated=True) @@ -429,9 +882,11 @@ def main() -> int: print("=" * 50) all_summaries = [] + video_registry: dict[str, dict] = {} for s in styles: log(f"\n--- [{s['index']:02d}] {s['name']} ---") - summary = process_style(s, args.dry_run, args.delay, session) + summary = process_style(s, args.dry_run, args.delay, session, + video_registry) all_summaries.append(summary) log(f" 小计: 任务={summary['total']} 命中={summary['matched']} 回填={summary['filled']}") out_file = out_dir / f"{s['index']:02d}-{s['name']}_self_bilibili_v2.json" @@ -442,6 +897,56 @@ def main() -> int: with open(final_path, "w", encoding="utf-8") as f: json.dump(all_summaries, f, ensure_ascii=False, indent=2) + # UP 主主页全量采集: 对账号配置里登记了 mid 的 B站账号, + # 拉取空间全部投稿并补进登记表(飞书款式表未登记的视频也覆盖)。 + from gyxx_flow.modules.content_marketing import self_operated_accounts + for entry in self_operated_accounts.load_accounts( + "bilibili", + self_operated_accounts.accounts_config_path(PATHS.config_root), + ): + mid = entry.get("platform_account_id", "").strip() + if not mid: + continue + try: + collect_up_homepage_videos( + session, mid, entry.get("account_name") or mid, + video_registry, delay=args.delay, + ) + except Exception as exc: + log(f" [WARN] 主页全量采集失败({entry.get('account_name')}): {exc}") + + # 全量视频指标 artifact: 供统一入库工具 sync_self_video_metrics.py 写 + # cmt_self_video_metrics(跨款式按 BV 号去重,不受飞书记录是否登记限制) + business_date = os.environ.get("GYXX_BUSINESS_DATE", "").strip() + artifact = { + "platform": "bilibili", + "business_date": business_date or datetime.now().strftime("%Y-%m-%d"), + "generated_at": datetime.now().isoformat(), + "videos": sorted( + ( + { + **video, + "publish_time": ( + video["publish_time"].isoformat() + if video.get("publish_time") + else None + ), + } + for video in video_registry.values() + ), + key=lambda v: v["video_id"], + ), + "styles_covered": [s["name"] for s in styles], + } + artifact_dir = PATHS.normalized_root / "self_bilibili" + artifact_dir.mkdir(parents=True, exist_ok=True) + artifact_path = artifact_dir / "daily.json" + tmp = artifact_path.with_suffix(artifact_path.suffix + ".tmp") + tmp.write_text(json.dumps(artifact, ensure_ascii=False, indent=2), + encoding="utf-8") + os.replace(tmp, artifact_path) + log(f"📦 全量视频指标 artifact: {artifact_path} ({len(video_registry)} 条视频)") + total = sum(s["total"] for s in all_summaries) matched = sum(s["matched"] for s in all_summaries) filled = sum(s["filled"] for s in all_summaries) diff --git a/src/gyxx_flow/modules/content_marketing/self_operated_accounts.py b/src/gyxx_flow/modules/content_marketing/self_operated_accounts.py new file mode 100644 index 0000000..1e134f7 --- /dev/null +++ b/src/gyxx_flow/modules/content_marketing/self_operated_accounts.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +"""自营账号配置: 内部账号别名 ↔ 各平台账号身份。 + +配置文件 data/config/self_operated_accounts.json 按平台分组: + douyin [{alias, account_name, platform_account_id}] + bilibili [{alias, account_name, platform_account_id?}] + xiaohongshu [{alias, account_name, platform_account_id?, phone?}] + +- 别名口径与飞书自营表「投放账号类型」一致,写入 cmt_self_video_metrics.account_alias。 +- 采集数据自带的账号身份(达人昵称/UP主名/店铺名/抖音号)优先,别名缺失时置空并提示补配置。 +- xiaohongshu 的 phone 供 ARK 短信登录自动填手机号(环境变量 XHS_ARK_ACCOUNT 优先)。 +""" + +from __future__ import annotations + +import json +from pathlib import Path + +PLATFORMS = ("douyin", "bilibili", "xiaohongshu") + + +def accounts_config_path(config_root: Path) -> Path: + return Path(config_root) / "self_operated_accounts.json" + + +def load_accounts(platform: str, config_path: str | Path) -> list[dict]: + """读取某平台的账号映射列表。 + + 缺文件、平台未配置或结构无效时返回空列表(调用方按「未知账号」处理并提示补配置)。 + """ + if platform not in PLATFORMS: + raise ValueError(f"unknown self-operated platform: {platform}") + path = Path(config_path) + if not path.exists(): + return [] + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"⚠️ 自营账号配置读取失败({path}): {exc}") + return [] + accounts = data.get(platform) if isinstance(data, dict) else None + if not isinstance(accounts, list): + return [] + return [ + { + "alias": str(item.get("alias") or "").strip(), + "account_name": str(item.get("account_name") or "").strip(), + "platform_account_id": str(item.get("platform_account_id") or "").strip(), + "phone": str(item.get("phone") or "").strip(), + } + for item in accounts + if isinstance(item, dict) and str(item.get("alias") or "").strip() + ] + + +def load_all_accounts(config_path: str | Path) -> dict[str, list[dict]]: + """一次读全三平台,供统一入库工具使用。""" + return {platform: load_accounts(platform, config_path) for platform in PLATFORMS} + + +def resolve_alias( + accounts: list[dict], + *, + account_name: str = "", + platform_account_id: str = "", +) -> str: + """按账号名或平台账号ID(大小写不敏感)解析内部账号别名;未命中返回空串。""" + name_key = str(account_name or "").strip().casefold() + id_key = str(platform_account_id or "").strip().casefold() + for entry in accounts: + if not name_key and not id_key: + break + if (name_key and entry["account_name"].casefold() == name_key) or ( + id_key and entry["platform_account_id"].casefold() == id_key + ): + return entry["alias"] + return "" + + +def find_account( + platform: str, + config_path: str | Path, + *, + account_name: str = "", + platform_account_id: str = "", +) -> dict | None: + """返回第一个命中的账号配置(含 phone 等附加字段);未命中返回 None。""" + accounts = load_accounts(platform, config_path) + name_key = str(account_name or "").strip().casefold() + id_key = str(platform_account_id or "").strip().casefold() + for entry in accounts: + if (name_key and entry["account_name"].casefold() == name_key) or ( + id_key and entry["platform_account_id"].casefold() == id_key + ): + return entry + return None diff --git a/src/gyxx_flow/modules/content_marketing/self_xhs_scraper.py b/src/gyxx_flow/modules/content_marketing/self_xhs_scraper.py new file mode 100644 index 0000000..9efaffd --- /dev/null +++ b/src/gyxx_flow/modules/content_marketing/self_xhs_scraper.py @@ -0,0 +1,968 @@ +# -*- coding: utf-8 -*- +""" +自营小红书 ARK 笔记管理采集 +================================ +- 登录: customer.xiaohongshu.com 短信验证码登录(手机号自动填入,验证码由人工 + 在浏览器中输入后脚本自动继续);登录态优先复用隔离 Profile 与持久化 Cookie。 +- 采集: ark.xiaohongshu.com/app-note/creation?tab=management 下的 + 「商品笔记」(managementTab=goods) 与「普通笔记」(managementTab=normal) 两个页签, + 按表头动态解析全部展示列(阅读数/点赞数/引流次数/支付金额/订单数等), + 逐页翻页抓取并按笔记身份去重。 +- 产物: normalized/content_marketing/self_xhs/daily.json, + 由 data/tools/sync_self_video_metrics.py 幂等写入 PostgreSQL cmt_self_video_metrics。 +- 浏览器: Scrapling StealthySession(隐藏指纹), 独立 CDP 端口与 Profile/Cookie + 由 runtime-bindings 注入(GYXX_BROWSER_* 环境变量)。 + +用法: + python self_xhs_scraper.py # 登录(必要时等待人工验证码)+采集 + python self_xhs_scraper.py --login-only # 只登录并保存 Cookie(首次部署引导) + python self_xhs_scraper.py --dry-run # 采集但不写 artifact + python self_xhs_scraper.py --login-timeout 300 +""" +import argparse +import hashlib +import json +import os +import re +import sys +import time +from datetime import datetime + +from scrapling.fetchers import StealthySession + +from gyxx_flow.adapters.scrapling import ( + BrowserTimeoutError, # noqa: F401 + is_browser_timeout_error, +) +from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.content_marketing import self_operated_accounts +from gyxx_flow.modules.content_marketing.runtime_paths import PATHS + +# 模块级 seam: 测试可替换会话工厂(保持与 pgy_xhs_scraper_v2 相同的约定)。 +DynamicSession = StealthySession + +LOGIN_URL = ( + "https://customer.xiaohongshu.com/login" + "?service=https://ark.xiaohongshu.com/app-note/creation%3Ftab%3Dcreation" +) +GOODS_URL = ( + "https://ark.xiaohongshu.com/app-note/creation" + "?tab=management&from=ark-login&managementTab=goods" +) +NORMAL_URL = ( + "https://ark.xiaohongshu.com/app-note/creation" + "?tab=management&from=ark-login&managementTab=normal" +) + +COOKIE_FILE = PATHS.browser_cookie_file +PROFILE_DIR = PATHS.browser_profile_dir +ARTIFACT_PATH = PATHS.normalized_root / "self_xhs" / "daily.json" +ACCOUNTS_CONFIG = self_operated_accounts.accounts_config_path(PATHS.config_root) + +DEFAULT_LOGIN_TIMEOUT = 300 +DEFAULT_MAX_PAGES = 50 +LIST_READY_TIMEOUT_MS = 30_000 + +# 小红书笔记 ID: 24 位十六进制(explore/、discovery/item/、个人主页 note/ 等路径) +_XHS_NOTE_ID_RE = re.compile( + r"(?:xiaohongshu\.com/(?:explore|discovery/item|user/profile/[0-9a-f]+/note)" + r"|xhslink\.com/[A-Za-z0-9]+|/note)" + r"/?([0-9a-f]{24})(?![0-9a-f])", + re.IGNORECASE, +) +_DATE_LINE_RE = re.compile(r"20\d{2}[-/.]\d{1,2}[-/.]\d{1,2}") +# 「笔记信息」单元格内的创作方式标签(整行精确匹配,避免误伤含这些字样的标题) +_CREATION_METHODS = {"手动创作", "可能是AI创作", "AI创作", "优质创作", "共创"} + + +class AcceptanceCookieSkip(RuntimeError): + """验收模式下不允许交互式短信登录,安全跳过。""" + + +def log(msg: str) -> None: + safe = str(msg).encode("gbk", errors="replace").decode("gbk", errors="replace") + print(f"[{datetime.now().strftime('%H:%M:%S')}] {safe}", flush=True) + + +# ============================================================ +# 数值/单元格解析(纯函数,供单测) +# ============================================================ +def parse_count(value: str | None) -> int | None: + """'1.2万' / '6,592' / '2422' → int; '—'/'-'/'暂无'/空 → None。""" + if value is None: + return None + text = str(value).strip() + if not text or text in {"-", "--", "—", "暂无"}: + return None + match = re.fullmatch(r"([\d,]+(?:\.\d+)?)\s*(万|w|W)?", text) + if not match: + return None + try: + number = float(match.group(1).replace(",", "")) + except ValueError: + return None + if match.group(2): + number *= 10000 + return int(number) + + +def parse_amount(value: str | None) -> float | None: + """金额 '1,234.56' / '0.9万' → float(两位小数);不经 int 截断。""" + if value is None: + return None + text = str(value).strip() + if not text or text in {"-", "--", "—", "暂无"}: + return None + match = re.fullmatch(r"([\d,]+(?:\.\d+)?)\s*(万|w|W)?", text) + if not match: + return None + try: + number = float(match.group(1).replace(",", "")) + except ValueError: + return None + if match.group(2): + number *= 10000 + return round(number, 2) + + +def _find_col(headers: list[str], candidates: list[str]) -> int | None: + """先整列精确匹配,再子串匹配(与 chanmama_export 一致)。""" + normalized = [str(header or "").strip() for header in headers] + for candidate in candidates: + for index, header in enumerate(normalized): + if header == candidate: + return index + for candidate in candidates: + for index, header in enumerate(normalized): + if candidate in header: + return index + return None + + +def _parse_info_cell(text: str) -> tuple[str, str, str]: + """「笔记信息」单元格 → (标题, 发布时间文本, 创作方式)。 + + 单元格内是 标题/发布时间/创作方式 分行文本;发布时间行带「发布时间：」前缀; + 单元格被压成单行时按「标题 发布时间：… [方式]」兜底拆分。 + """ + raw = str(text or "").strip() + if not raw: + return "", "", "" + if "\n" in raw: + publish = "" + method = "" + title = "" + for line in (part.strip() for part in raw.splitlines()): + if not line: + continue + if not publish and _DATE_LINE_RE.search(line): + publish = re.sub(r"^发布时间[:：]?\s*", "", line).strip() + continue + if not method and line in _CREATION_METHODS: + method = line + continue + if not title: + title = line + return title, publish, method + + compact = re.sub(r"\s+", " ", raw) + match = re.match( + r"^(.*?)\s*发布时间[:：]\s*" + r"(20\d{2}[-/.]\d{1,2}[-/.]\d{1,2}(?:\s+\d{1,2}:\d{2}(?::\d{2})?)?)" + r"\s*(.*)$", + compact, + ) + if not match: + return compact, "", "" + title = match.group(1).strip() + publish = match.group(2).strip() + rest = match.group(3).strip() + method = rest if rest in _CREATION_METHODS else next( + (token for token in _CREATION_METHODS if token and token in rest), "" + ) + return title, publish, method + + +def extract_note_id(href: str) -> str | None: + """从小红书链接提取 24 位笔记 ID。""" + if not href: + return None + match = _XHS_NOTE_ID_RE.search(str(href)) + return match.group(1) if match else None + + +def fallback_note_fingerprint(title: str, publish_time_text: str) -> str: + """无链接时用 标题+发布时间 生成跨日稳定的内容指纹(前缀 xhs-)。""" + basis = f"{title.strip()}|{publish_time_text.strip()}" + return "xhs-" + hashlib.sha256(basis.encode("utf-8")).hexdigest()[:16] + + +def parse_management_rows( + headers: list[str], + rows: list[dict], + note_type: str, +) -> tuple[list[dict], dict[str, int]]: + """把一个页签的表头+原始行解析为标准化笔记记录。 + + rows: [{cells: [单元格文本...], href: 行内笔记链接或 ''}] + 返回 (records, stats);每条记录含建模字段 + raw_columns(页面展示的全部列)。 + """ + stats = {"rows": len(rows), "records": 0, "skip_no_title": 0} + + info_col = _find_col(headers, ["笔记信息"]) + read_col = _find_col(headers, ["笔记阅读数", "阅读数"]) + like_col = _find_col(headers, ["点赞数"]) + mounted_col = _find_col(headers, ["挂载商品"]) + pay_amount_col = _find_col(headers, ["笔记支付金额", "支付金额"]) + pay_orders_col = _find_col(headers, ["笔记支付订单数", "支付订单数"]) + home_col = _find_col(headers, ["引流主页次数"]) + live_col = _find_col(headers, ["引流直播间次数"]) + status_col = _find_col(headers, ["发布状态"]) + + def cell(row: dict, index: int | None) -> str: + if index is None or index >= len(row.get("cells", [])): + return "" + return str(row["cells"][index] or "").strip() + + records: list[dict] = [] + for row in rows: + cells = row.get("cells", []) + title, publish_text, method = _parse_info_cell(cell(row, info_col)) + if not title: + stats["skip_no_title"] += 1 + continue + note_id = extract_note_id(row.get("href") or "") + record = { + "note_type": note_type, + "note_id": note_id, + "video_id": note_id or fallback_note_fingerprint(title, publish_text), + "title": title[:512], + "publish_time": publish_text, + "creation_method": method, + "mounted_products": cell(row, mounted_col), + "read_count": parse_count(cell(row, read_col)), + "like_count": parse_count(cell(row, like_col)), + "homepage_redirect_count": parse_count(cell(row, home_col)), + "live_redirect_count": parse_count(cell(row, live_col)), + "payment_amount": parse_amount(cell(row, pay_amount_col)), + "payment_orders": parse_count(cell(row, pay_orders_col)), + "publish_status": cell(row, status_col), + "note_url": row.get("href") or "", + "raw_columns": { + header: (cells[index] if index < len(cells) else "") + for index, header in enumerate(headers) + if header + }, + } + records.append(record) + stats["records"] += 1 + return records, stats + + +# ============================================================ +# 浏览器: Cookie 持久化 / 登录 +# ============================================================ +def restore_cookies(page) -> None: + if not COOKIE_FILE.exists(): + return + try: + cookies = json.loads(COOKIE_FILE.read_text(encoding="utf-8")) + page.context.add_cookies(cookies) + log(f"已载入持久化 Cookie: {COOKIE_FILE}") + except Exception as exc: + log(f"载入 Cookie 失败(忽略,继续用 Profile 登录态): {exc}") + + +def save_state(page) -> None: + COOKIE_FILE.parent.mkdir(parents=True, exist_ok=True) + cookies = page.context.cookies() + COOKIE_FILE.write_text( + json.dumps(cookies, ensure_ascii=False, indent=2), encoding="utf-8" + ) + log(f"已保存 Cookie ({len(cookies)} 条): {COOKIE_FILE}") + + +def is_logged_in(page) -> bool: + url = str(page.url) + if "customer.xiaohongshu.com/login" in url: + return False + if url.startswith("https://ark.xiaohongshu.com/"): + return True + # 登录页可能在弹层内,未跳转 + try: + body = page.content() + return "请输入手机号" not in body and "短信登录" not in body + except Exception: + return False + + +def _fill_sms_login(page, phone: str) -> bool: + """在 customer.xiaohongshu.com 登录页填手机号并点击发送验证码。""" + try: + page.wait_for_selector( + "input[placeholder*='手机号'], input[type='tel']", timeout=10_000 + ) + except Exception as exc: + if is_browser_timeout_error(exc): + log("⚠️ 未找到手机号输入框,登录页结构可能已变化") + return False + raise + filled = False + for selector in ("input[placeholder*='手机号']", "input[type='tel']"): + try: + box = page.locator(selector).first + if box.count(): + box.fill(phone) + filled = True + break + except Exception: + continue + if not filled: + log("⚠️ 手机号输入失败") + return False + log(f"✅ 已填入手机号: {phone[:3]}****{phone[-4:]}") + # 点击发送验证码(按钮文案多变,依次尝试) + for text in ("获取验证码", "发送验证码", "获取短信验证码", "发送"): + try: + button = page.get_by_text(text, exact=False).first + if button.count() and button.is_visible(): + button.click(timeout=3000) + log(f"✅ 已点击「{text}」,请查收短信") + return True + except Exception: + continue + log("⚠️ 未找到发送验证码按钮(可能有前置滑块,请在浏览器中手动完成)") + return True # 手机号已填,让用户在浏览器里继续操作 + + +def wait_for_login(page, timeout_seconds: int) -> None: + print("\n" + "=" * 60) + print("🔐 请在打开的浏览器中完成短信验证码登录(如需滑块也请手动完成)") + print(" 登录成功后脚本会自动继续,无需操作控制台") + print("=" * 60, flush=True) + deadline = time.time() + timeout_seconds + while time.time() < deadline: + if is_logged_in(page): + log("✅ 检测到登录成功") + return + remaining = int(deadline - time.time()) + if remaining % 30 == 0: + log(f"⏳ 等待登录完成... 剩余 {remaining}s") + page.wait_for_timeout(2000) + raise TimeoutError(f"Login timed out after {timeout_seconds} seconds.") + + +def ensure_login(page, login_timeout: int, phone: str) -> None: + page.goto(GOODS_URL, wait_until="domcontentloaded", timeout=60_000) + page.wait_for_timeout(3000) + if is_logged_in(page): + log("✅ Profile/Cookie 登录态有效") + save_state(page) + return + + from gyxx_flow.adapters.acceptance_policy import current_acceptance_policy + + acceptance_policy = current_acceptance_policy() + if acceptance_policy.enabled: + reason = ( + "persisted ARK session is invalid; interactive SMS login is disabled" + ) + acceptance_policy.record( + "cookie_skipped", + operation="content.self_xhs.login", + details={"status": "SKIPPED_COOKIE", "reason": reason}, + ) + log(f"[SKIPPED_COOKIE] {reason}") + raise AcceptanceCookieSkip(reason) + + log("🔄 登录态失效,进入短信验证码登录流程") + page.goto(LOGIN_URL, wait_until="domcontentloaded", timeout=60_000) + page.wait_for_timeout(2000) + # 确保在「短信登录」页签 + for text in ("短信登录",): + try: + tab = page.get_by_text(text, exact=True).first + if tab.count() and tab.is_visible(): + tab.click(timeout=2000) + page.wait_for_timeout(800) + break + except Exception: + continue + _fill_sms_login(page, phone) + wait_for_login(page, login_timeout) + save_state(page) + page.goto(GOODS_URL, wait_until="domcontentloaded", timeout=60_000) + page.wait_for_timeout(2000) + + +# ============================================================ +# 页面表格抓取 +# ============================================================ +_EXTRACT_TABLE_SCRIPT = r""" +() => { + const visible = (el) => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && style.opacity !== '0' && rect.width > 0 && rect.height > 0; + }; + // 保留行结构(单元格内 标题/发布时间/创作方式 分行),只压缩行内空白 + const clean = (v) => String(v || '').split('\n') + .map((line) => line.replace(/\s+/g, ' ').trim()) + .filter(Boolean).join('\n'); + const tables = [...document.querySelectorAll('table')].filter(visible); + const table = tables.find((t) => /笔记信息/.test(t.innerText || '')) || null; + if (!table) return {found: false, headers: [], rows: []}; + const headers = [...table.querySelectorAll('thead th, thead td')] + .map((th) => clean(th.innerText).replace(/\n/g, ' ')); + const rows = [...table.querySelectorAll('tbody tr')].filter(visible).map((tr) => { + const link = tr.querySelector( + 'a[href*="/explore/"], a[href*="/discovery/item/"], ' + + 'a[href*="xhslink.com"], a[href*="/note/"]' + ); + return { + cells: [...tr.querySelectorAll('td')].map((td) => clean(td.innerText)), + href: link ? link.href : '' + }; + }); + return {found: true, headers, rows}; +} +""" + + +def extract_table(page) -> dict: + """从当前页提取 {found, headers, rows};失败时 found=False。""" + try: + return page.evaluate(_EXTRACT_TABLE_SCRIPT) or {} + except Exception as exc: + log(f"⚠️ 表格提取脚本失败: {exc}") + return {"found": False, "headers": [], "rows": []} + + +def _next_button(page): + """ARK 分页: .d-pagination-page 项,首尾空文本项是翻页箭头,末位为下一页。""" + try: + items = page.locator(".d-pagination-page").all() + arrows = [item for item in items if not (item.inner_text() or "").strip()] + if arrows: + return arrows[-1] + except Exception: + pass + for selector in ( + ".el-pagination .btn-next", + "button.btn-next", + "button[aria-label*='下一页']", + ): + try: + for element in page.locator(selector).all(): + if element.is_visible(): + return element + except Exception: + continue + try: + button = page.get_by_text("下一页", exact=True).first + if button.count() and button.is_visible(): + return button + except Exception: + pass + return None + + +def _button_disabled(element) -> bool: + try: + css = str(element.get_attribute("class") or "").lower() + except Exception: + return True + if "disabled" in css: + return True + try: + if not element.is_enabled(): + return True + except Exception: + return True + aria = str(element.get_attribute("aria-disabled") or "").lower() + return aria == "true" + + +def collect_tab(page, url: str, tab_label: str, note_type: str, + max_pages: int) -> list[dict]: + """抓取一个页签的全部分页,返回去重后的记录列表。""" + page.goto(url, wait_until="domcontentloaded", timeout=60_000) + page.wait_for_timeout(3000) + data = extract_table(page) + if not data.get("found"): + # URL 页签参数可能变化,退回点击页签文字 + log(f"⚠️ URL 未直接命中「{tab_label}」,尝试点击页签") + try: + tab = page.get_by_text(tab_label, exact=True).first + if tab.count(): + tab.click(timeout=3000) + page.wait_for_timeout(2500) + except Exception as exc: + log(f"⚠️ 点击页签「{tab_label}」失败: {exc}") + data = extract_table(page) + if not data.get("found"): + log(f"❌ 「{tab_label}」未找到笔记表格") + return [] + + records: dict[str, dict] = {} + stats_total = {"rows": 0, "records": 0, "skip_no_title": 0} + for page_number in range(1, max_pages + 1): + data = extract_table(page) + if not data.get("found"): + break + page_records, stats = parse_management_rows( + data.get("headers", []), data.get("rows", []), note_type + ) + for key in stats_total: + stats_total[key] += stats.get(key, 0) + before = len(records) + for record in page_records: + records[record["video_id"]] = record + log( + f"📄 {tab_label} 第 {page_number} 页: {stats['rows']} 行 → " + f"{stats['records']} 条,累计去重 {len(records)} 条" + + ("" if len(records) > before else " (本页无新增)") + ) + button = _next_button(page) + if button is None or _button_disabled(button): + log(f"📄 {tab_label} 已到最后一页") + break + try: + button.click() + page.wait_for_timeout(2500) + except Exception as exc: + log(f"⚠️ 翻页失败,停止: {exc}") + break + log( + f"📊 {tab_label} 完成: 行={stats_total['rows']} 记录={stats_total['records']}" + f" 去重={len(records)} 无标题跳过={stats_total['skip_no_title']}" + ) + return list(records.values()) + + +def _open_page(session): + """从会话上下文取一个页面(优先复用已有页面)。""" + context = session.context + pages = getattr(context, "pages", None) or [] + return pages[0] if pages else context.new_page() + + +def _detect_shop_name(page) -> str: + """尽力识别页面顶部的店铺名(如「店铺 光影行星 GYXX旗舰店」)。""" + try: + text = page.evaluate( + "() => (document.body && document.body.innerText) || ''" + ) + match = re.search(r"店铺\s*\n?\s*([^\n]{2,30})", str(text or "")) + return match.group(1).strip() if match else "" + except Exception: + return "" + + +# ============================================================ +# 飞书自营表回填: 按标题核对,阅读数写入对应发布时间槽位列 +# (与 chanmama_scraper/self_bilibili_scraper 同一套时槽语义) +# ============================================================ +LARK_CLI = os.path.expandvars(r"%APPDATA%\npm\lark-cli.cmd") +if not os.path.exists(LARK_CLI): + LARK_CLI = "lark-cli.cmd" +V2_DIR = PATHS.normalized_root / "v2_results" + + +def _call_lark_json(args: list[str]) -> dict: + import subprocess + + cmd = [LARK_CLI] + args + env = {**os.environ, "LARK_CLI_NO_PROXY": "1"} + proc = subprocess.run(cmd, capture_output=True, env=env, + encoding="utf-8", errors="replace") + text = proc.stdout if proc.stdout else (proc.stderr or "") + start = text.find("{") + if start < 0: + return {"ok": False, "error": f"no json: {text[:300]}"} + try: + return json.loads(text[start:], strict=False) + except json.JSONDecodeError as exc: + return {"ok": False, "error": f"json decode: {exc}"} + + +def _list_records_by_table(base_token: str, table_id: str) -> list[dict]: + all_records = [] + offset = 0 + while True: + resp = _call_lark_json([ + "base", "+record-list", + "--base-token", base_token, + "--table-id", table_id, + "--as", "user", + "--format", "json", + "--limit", "200", + "--offset", str(offset), + ]) + if not resp.get("ok"): + raise RuntimeError(f"record-list 失败: {resp.get('error')}") + payload = resp["data"] + field_ids = payload["field_id_list"] + record_ids = payload["record_id_list"] + rows = payload["data"] + for index, row in enumerate(rows): + record = {"record_id": record_ids[index]} + for column, field_id in enumerate(field_ids): + record[field_id] = row[column] if column < len(row) else None + all_records.append(record) + offset += len(rows) + if not payload.get("has_more", False) or not rows: + break + return all_records + + +def _write_back(base_token: str, table_id: str, record_id: str, + field_id: str, value) -> bool: + from gyxx_flow.adapters.acceptance_policy import skip_feishu_table_write + + if skip_feishu_table_write( + "content_marketing.self_xhs.record-upsert", + details={"table_id": table_id, "record_id": record_id, + "field_id": field_id}, + ): + log(f" [ACCEPTANCE-SKIP] record_id={record_id}") + return True + payload = {field_id: value} + resp = _call_lark_json([ + "base", "+record-upsert", + "--base-token", base_token, + "--table-id", table_id, + "--record-id", record_id, + "--as", "user", + "--format", "json", + "--json", json.dumps(payload, ensure_ascii=False), + ]) + if not resp.get("ok"): + log(f" ❌ 回填失败 record_id={record_id}: {resp.get('error')}") + return False + return True + + +def _pick_read_field(fmap: dict, pub_time, today=None): + """发布时间距今天数 → 对应曝光槽位字段(7/14/21/28天/月底)。""" + if pub_time is None: + return None + today = today or datetime.now() + days = (today.date() - pub_time.date()).days + if days < 0: + return "FUTURE" + if days <= 7: + logical = "read_count_7d" + elif days <= 14: + logical = "read_count_14d" + elif days <= 21: + logical = "read_count_21d" + elif days <= 28: + logical = "read_count_28d" + else: + logical = "month_end" + info = fmap.get(logical) + if info: + return (logical, info["field_id"], info["field_name"]) + return None + + +_TITLE_STRIP_RE = re.compile( + r"[\s　\-_.,;:!?！？，。；：、/\\()（）【】《》''\"\"'']+" +) +_EMOJI_RE = re.compile( + "[\U0001F000-\U0001FFFF\U00002600-\U000027BF\U0001F1E0-\U0001F1FF]+" +) + + +def _title_match(haystack: str, needle: str) -> bool: + """模糊标题匹配: 剥链接/URL/空白标点/emoji 后,任一方包含另一方即命中。""" + + def normalize(value: str) -> str: + value = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", str(value or "")) + value = re.sub(r"https?://[^\s）)\]]+", "", value) + value = _TITLE_STRIP_RE.sub("", value) + return _EMOJI_RE.sub("", value) + + left = normalize(haystack) + right = normalize(needle) + if not left or not right: + return False + return right in left or left in right + + +def backfill_self_xhs_tables(notes: list[dict], dry_run: bool = False) -> bool: + """按标题把小红书阅读数回填到各款式自营表的对应曝光槽位。 + + 只处理「投放平台」明确为小红书的行,避免同名标题误写抖音/B站行; + 飞书无发布时间时先用采集到的发布时间补,再做时槽判断。 + 同时产出 {idx:02d}-{name}_self_v2.json 供曝光入库步骤消费。 + """ + from gyxx_flow.modules.content_marketing import feishu_mapping + from gyxx_flow.modules.content_marketing.chanmama_export import ( + parse_publish_time, + ) + + try: + mapping = feishu_mapping.load_mapping(self_operated=True) + except Exception as exc: + log(f"❌ 加载自营 mapping 失败: {exc}") + return False + styles = mapping.get("tables") if isinstance(mapping, dict) else None + if not isinstance(styles, list) or not styles: + log("❌ 自营 mapping 未配置任何目标表") + return False + + def is_xhs(value) -> bool: + if value is None: + return False + if isinstance(value, list): + values = [str(item).strip().casefold() for item in value if item] + return any("小红书" in v or "xhs" in v for v in values) + text = str(value).strip().casefold() + return "小红书" in text or "xhs" in text + + total_tasks = total_matched = total_filled = 0 + had_failure = False + today = datetime.now() + V2_DIR.mkdir(parents=True, exist_ok=True) + + for style in styles: + fmap = style.get("field_map", {}) + platform_fid = (fmap.get("platform") or {}).get("field_id") + title_fid = (fmap.get("note_title") or {}).get("field_id") + pub_fid = (fmap.get("publish_time") or {}).get("field_id") + if not platform_fid or not title_fid: + continue + try: + records = _list_records_by_table(style["base_token"], + style["table_id"]) + except Exception as exc: + log(f" [ERR] 款式 {style['name']} 拉记录失败: {exc}") + had_failure = True + continue + xhs_records = [r for r in records if is_xhs(r.get(platform_fid))] + log(f" 款式 {style['name']}: 全表 {len(records)} 条,小红书 " + f"{len(xhs_records)} 条") + + style_results = [] + for record in xhs_records: + feishu_title = record.get(title_fid) + if isinstance(feishu_title, list): + feishu_title = " ".join(str(x) for x in feishu_title if x) + feishu_title = str(feishu_title or "").strip() + if not feishu_title: + continue + total_tasks += 1 + matched = next( + (n for n in notes if _title_match(feishu_title, n["title"])), + None, + ) + if not matched: + continue + total_matched += 1 + pub_time = parse_publish_time(record.get(pub_fid)) + if pub_time is None and not dry_run and pub_fid: + note_pub = parse_publish_time(matched.get("publish_time")) + if note_pub is not None: + ts_ms = int(note_pub.timestamp() * 1000) + if _write_back(style["base_token"], style["table_id"], + record["record_id"], pub_fid, ts_ms): + log(f" ✅ '{feishu_title[:20]}' → 先填发布时间") + else: + had_failure = True + pub_time = note_pub + read_count = matched.get("read_count") + if read_count is None: + log(f" [SKIP] 阅读数为空: '{feishu_title[:20]}'") + continue + pick = _pick_read_field(fmap, pub_time, today) + if pick == "FUTURE": + continue + if not pick: + log(f" [SKIP] 无对应曝光槽位字段: '{feishu_title[:20]}'") + continue + logical, fill_fid, fill_fname = pick + if dry_run: + log(f" [DRY] '{feishu_title[:20]}' → {fill_fname}={read_count}") + else: + if _write_back(style["base_token"], style["table_id"], + record["record_id"], fill_fid, read_count): + log(f" ✅ '{feishu_title[:20]}' → {fill_fname}={read_count}") + else: + had_failure = True + continue + total_filled += 1 + style_results.append({ + "record_id": record["record_id"], + "creator": "", + "target": feishu_title, + "play_count": read_count, + "matched": True, + "slot": logical, + "pub_time": pub_time.isoformat() if pub_time else None, + }) + + if not dry_run: + summary = { + "style": style["name"], + "index": style["index"], + "total": len(xhs_records), + "matched": len(style_results), + "filled": len(style_results), + "results": style_results, + } + v2_file = V2_DIR / f"{style['index']:02d}-{style['name']}_self_v2.json" + tmp = v2_file.with_suffix(v2_file.suffix + ".tmp") + tmp.write_text(json.dumps(summary, ensure_ascii=False, indent=2), + encoding="utf-8") + os.replace(tmp, v2_file) + + log(f"📊 飞书回填完成: 任务={total_tasks} 命中={total_matched} " + f"回填={total_filled} (dry_run={dry_run})") + return not had_failure + + +# ============================================================ +# main +# ============================================================ +def collect(headless: bool, login_timeout: int, max_pages: int, + dry_run: bool) -> int: + account = ( + self_operated_accounts.load_accounts("xiaohongshu", ACCOUNTS_CONFIG) + or [{}] + )[0] + phone = ( + os.environ.get("XHS_ARK_ACCOUNT", "").strip() + or account.get("phone", "") + ) + if not phone: + log("❌ 缺少登录手机号: 请设置环境变量 XHS_ARK_ACCOUNT 或在 " + "self_operated_accounts.json 的 xiaohongshu 段配置 phone") + return 1 + config_name = account.get("account_name", "") + + with DynamicSession( + headless=headless, + real_chrome=True, + user_data_dir=str(PROFILE_DIR), + locale="zh-CN", + timezone_id="Asia/Shanghai", + timeout=60_000, + network_idle=False, + disable_resources=False, + google_search=False, + page_setup=restore_cookies, + max_pages=3, + ) as session: + page = _open_page(session) + ensure_login(page, login_timeout, phone) + + shop_name = _detect_shop_name(page) + account_name = config_name or shop_name + log(f"🏪 店铺: {account_name or '(未识别,用配置名兜底)'}") + + goods_records = collect_tab(page, GOODS_URL, "商品笔记", "goods", max_pages) + normal_records = collect_tab(page, NORMAL_URL, "普通笔记", "normal", max_pages) + + business_date = os.environ.get("GYXX_BUSINESS_DATE", "").strip() + artifact = { + "platform": "xiaohongshu", + "business_date": business_date or datetime.now().strftime("%Y-%m-%d"), + "generated_at": datetime.now().isoformat(), + "account_name": account_name, + "shop_name_detected": shop_name, + "notes": goods_records + normal_records, + "counts": { + "goods": len(goods_records), + "normal": len(normal_records), + }, + } + log( + f"📊 采集完成: 商品笔记 {len(goods_records)} 条, 普通笔记 " + f"{len(normal_records)} 条" + ) + if dry_run: + log("[DRY-RUN] 不写 artifact,不回填飞书") + return 0 + ARTIFACT_PATH.parent.mkdir(parents=True, exist_ok=True) + tmp = ARTIFACT_PATH.with_suffix(ARTIFACT_PATH.suffix + ".tmp") + tmp.write_text(json.dumps(artifact, ensure_ascii=False, indent=2), + encoding="utf-8") + os.replace(tmp, ARTIFACT_PATH) + log(f"📦 artifact: {ARTIFACT_PATH}") + + # 全量指标已入库侧留存;此处按标题核对把阅读数回填飞书自营表槽位 + backfill_ok = backfill_self_xhs_tables(goods_records + normal_records) + if not backfill_ok: + log("⚠️ 飞书回填存在失败项") + return 1 + return 0 + + +def login_only(headless: bool, login_timeout: int) -> int: + account = ( + self_operated_accounts.load_accounts("xiaohongshu", ACCOUNTS_CONFIG) + or [{}] + )[0] + phone = ( + os.environ.get("XHS_ARK_ACCOUNT", "").strip() + or account.get("phone", "") + ) + if not phone: + log("❌ 缺少登录手机号(同 collect)") + return 1 + with DynamicSession( + headless=headless, + real_chrome=True, + user_data_dir=str(PROFILE_DIR), + locale="zh-CN", + timezone_id="Asia/Shanghai", + timeout=60_000, + network_idle=False, + disable_resources=False, + google_search=False, + page_setup=restore_cookies, + max_pages=2, + ) as session: + page = _open_page(session) + ensure_login(page, login_timeout, phone) + log("✅ 登录态已建立并保存 Cookie") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="自营小红书 ARK 笔记管理采集(商品笔记+普通笔记)") + parser.add_argument("--login-only", action="store_true", + help="只登录并保存 Cookie(首次部署引导)") + parser.add_argument("--dry-run", action="store_true", + help="采集但不写 artifact") + parser.add_argument("--login-timeout", type=int, + default=DEFAULT_LOGIN_TIMEOUT, + help="等待人工完成短信验证码的秒数") + parser.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES, + help="每个页签最多翻页数") + parser.add_argument("--headless", action="store_true", + help="无头模式(登录需要人工输入验证码时不建议)") + args = parser.parse_args() + + try: + if args.login_only: + return login_only(args.headless, args.login_timeout) + return collect(args.headless, args.login_timeout, args.max_pages, + args.dry_run) + except AcceptanceCookieSkip: + return COOKIE_SKIP_EXIT_CODE + except KeyboardInterrupt: + print("\n⛔ 用户中断", file=sys.stderr) + return 130 + + +def cli() -> None: + raise SystemExit(main()) + + +if __name__ == "__main__": + cli() diff --git a/src/gyxx_flow/modules/product_commerce/__init__.py b/src/gyxx_flow/modules/product_commerce/__init__.py index e924e5f..882a5fb 100644 --- a/src/gyxx_flow/modules/product_commerce/__init__.py +++ b/src/gyxx_flow/modules/product_commerce/__init__.py @@ -14,6 +14,7 @@ PRODUCT_SCHEDULED_WORKFLOW_IDS = ( "product.persona.daily", "product.daily", "product.alert.daily", + "product.online_stock_alert.daily", "product.ecommerce_costs.daily", "product.style_analysis.interval", "product.main_image.weekly", diff --git a/src/gyxx_flow/modules/product_commerce/analyze_style.py b/src/gyxx_flow/modules/product_commerce/analyze_style.py index 8066285..d0c6eba 100644 --- a/src/gyxx_flow/modules/product_commerce/analyze_style.py +++ b/src/gyxx_flow/modules/product_commerce/analyze_style.py @@ -19,6 +19,7 @@ import uuid from collections import defaultdict from dataclasses import dataclass from datetime import date, timedelta +from html import unescape from pathlib import Path from typing import Any @@ -79,6 +80,7 @@ BASE_WRITE_RECONCILE_DELAYS_SECONDS = (0, 2, 5) # Keep reports produced before the reasoning/language gate isolated from the # corrected direct-MiniMax checkpoint chain. CHECKPOINT_NAMESPACE = "direct_minimax_v2" +DOC_CREATE_RECONCILE_DELAYS_SECONDS = (0, 2, 5) # ── data extraction ──────────────────────────────────────────────────────── @@ -966,6 +968,97 @@ def _validate_checkpoint( return payload +def _normalize_doc_search_title(value: Any) -> str: + """Normalize the highlighted title returned by Feishu document search.""" + without_markup = re.sub(r"<[^>]+>", "", str(value or "")) + return re.sub(r"\s+", " ", unescape(without_markup)).strip() + + +def _analysis_doc_search_args(title: str) -> list[str]: + return [ + "docs", "+search", "--as", "user", "--query", title, + "--page-size", "20", "--format", "json", + ] + + +def _find_analysis_doc(title: str) -> dict[str, str] | None: + """Find an exact report title without creating or changing any document.""" + response = run_lark_cli(_analysis_doc_search_args(title)) + data = response.get("data") or {} + if not isinstance(data, dict): + return None + results = data.get("results") or [] + if not isinstance(results, list): + return None + for result in results: + if not isinstance(result, dict): + continue + metadata = result.get("result_meta") or {} + if not isinstance(metadata, dict): + metadata = {} + title_candidates = ( + result.get("title"), + result.get("title_highlighted"), + metadata.get("title"), + metadata.get("title_highlighted"), + ) + if not any(_normalize_doc_search_title(item) == title for item in title_candidates): + continue + document_id = metadata.get("token") or result.get("token") + document_url = metadata.get("url") or result.get("url") + if document_id and document_url: + return {"id": str(document_id), "url": str(document_url)} + return None + + +def _find_analysis_doc_with_retry(title: str) -> dict[str, str] | None: + """Search an exact title, failing closed if the readback itself is unavailable.""" + last_error: BaseException | None = None + for delay in DOC_CREATE_RECONCILE_DELAYS_SECONDS: + if delay: + time.sleep(delay) + try: + document = _find_analysis_doc(title) + except Exception as search_exc: + last_error = search_exc + continue + if document: + return document + if last_error is not None: + raise StyleAnalysisCheckpointError( + "飞书文档创建失败后无法完成标题对账，拒绝重复创建" + ) from last_error + return None + + +def _is_ambiguous_analysis_doc_create_error(exc: BaseException) -> bool: + message = re.sub(r"\s+", " ", str(exc).casefold()) + return any( + marker in message + for marker in ( + "20008", + "user not exist", + "the user does not exist", + "timeout", + "timed out", + "time out", + "network", + "econnreset", + "socket hang up", + ) + ) + + +def _reconcile_analysis_doc_create( + title: str, + exc: BaseException, +) -> dict[str, str] | None: + """Read back an ambiguous create before deciding whether it is safe to retry.""" + if not _is_ambiguous_analysis_doc_create_error(exc): + return None + return _find_analysis_doc_with_retry(title) + + def load_style_analysis_checkpoint( spec: StyleAnalysisCheckpointSpec, ) -> dict[str, Any]: @@ -1008,8 +1101,15 @@ def _atomic_write_checkpoint(path: Path, payload: dict[str, Any]) -> None: def claim_style_analysis_checkpoint( spec: StyleAnalysisCheckpointSpec, + *, + retry_unconfirmed_creation: bool = False, ) -> tuple[dict[str, Any], bool]: - """Exclusively claim document creation or load a resumable later phase.""" + """Claim document creation or load a resumable later phase. + + A ``doc_creating`` checkpoint is only retried after the caller has + completed a remote title reconciliation and confirmed that no document + exists. This keeps the normal fail-closed behavior for ambiguous writes. + """ path = spec.path() initial = _checkpoint_payload(spec) try: @@ -1017,6 +1117,9 @@ def claim_style_analysis_checkpoint( except FileExistsError: existing = load_style_analysis_checkpoint(spec) if existing["phase"] == "doc_creating": + if retry_unconfirmed_creation: + _atomic_write_checkpoint(path, initial) + return initial, True raise StyleAnalysisCheckpointError( "飞书文档创建结果不明确（doc_creating），拒绝重复创建；请先对账" ) @@ -1194,7 +1297,27 @@ def write_feishu_doc( raise StyleAnalysisCheckpointError( "传入报告与款式分析 checkpoint 的标题或哈希不匹配" ) - checkpoint_state, owns_creation = claim_style_analysis_checkpoint(checkpoint) + try: + checkpoint_state, owns_creation = claim_style_analysis_checkpoint(checkpoint) + except StyleAnalysisCheckpointError as exc: + if "doc_creating" not in str(exc): + raise + # A prior client error may have happened after Feishu committed the + # document but before lark-cli returned its identity. Reconcile by + # exact title first; only an empty search authorizes a new create. + reconciled = _find_analysis_doc_with_retry(title) + if reconciled: + checkpoint_state = advance_style_analysis_checkpoint( + checkpoint, + "doc_created", + document=reconciled, + ) + owns_creation = False + else: + checkpoint_state, owns_creation = claim_style_analysis_checkpoint( + checkpoint, + retry_unconfirmed_creation=True, + ) tmp = report_cache_path(title) tmp.parent.mkdir(parents=True, exist_ok=True) @@ -1203,30 +1326,80 @@ def write_feishu_doc( permission_grant: dict[str, Any] = {} if owns_creation: - obj = run_lark_cli([ + create_args = [ "docs", "+create", "--doc-format", "markdown", "--title", title, "--content", f"@{tmp.name}", "--as", "bot", "--format", "json", - ], cwd=content_cwd, timeout=180) - data = obj.get("data") or {} - doc = data.get("document") or {} - url = doc.get("url") or obj.get("url") or obj.get("document_url") - doc_id = doc.get("document_id") - if not url or not doc_id: - raise RuntimeError(f"lark-cli docs +create missing URL or document_id: {obj}") - permission_grant = obj.get("permission_grant") or data.get("permission_grant") or {} - if checkpoint is not None: - # Persist the remote identity before any permission call. A later - # failure can therefore continue against this exact document. - checkpoint_state = advance_style_analysis_checkpoint( - checkpoint, - "doc_created", - document={"id": str(doc_id), "url": str(url)}, - automatic_permission_status=permission_grant.get("status"), + ] + try: + obj = run_lark_cli(create_args, cwd=content_cwd, timeout=180) + except Exception as exc: + if not _is_ambiguous_analysis_doc_create_error(exc): + raise + reconciled = _reconcile_analysis_doc_create(title, exc) + if reconciled: + url = reconciled["url"] + doc_id = reconciled["id"] + if checkpoint is not None: + checkpoint_state = advance_style_analysis_checkpoint( + checkpoint, + "doc_created", + document=reconciled, + ) + else: + # 20008 is commonly returned while the CLI's user identity is + # being refreshed. Refresh once, then retry only after the + # exact-title reconciliation proved that no document exists. + try: + run_lark_cli(["auth", "status", "--json", "--verify"]) + except Exception: + raise exc + obj = run_lark_cli(create_args, cwd=content_cwd, timeout=180) + + data = obj.get("data") or {} + doc = data.get("document") or {} + url = doc.get("url") or obj.get("url") or obj.get("document_url") + doc_id = doc.get("document_id") + if not url or not doc_id: + raise RuntimeError( + f"lark-cli docs +create missing URL or document_id: {obj}" + ) + permission_grant = ( + obj.get("permission_grant") + or data.get("permission_grant") + or {} + ) + if checkpoint is not None: + checkpoint_state = advance_style_analysis_checkpoint( + checkpoint, + "doc_created", + document={"id": str(doc_id), "url": str(url)}, + automatic_permission_status=permission_grant.get("status"), + ) + else: + data = obj.get("data") or {} + doc = data.get("document") or {} + url = doc.get("url") or obj.get("url") or obj.get("document_url") + doc_id = doc.get("document_id") + if not url or not doc_id: + raise RuntimeError(f"lark-cli docs +create missing URL or document_id: {obj}") + permission_grant = ( + obj.get("permission_grant") + or data.get("permission_grant") + or {} ) + if checkpoint is not None: + # Persist the remote identity before any permission call. A later + # failure can therefore continue against this exact document. + checkpoint_state = advance_style_analysis_checkpoint( + checkpoint, + "doc_created", + document={"id": str(doc_id), "url": str(url)}, + automatic_permission_status=permission_grant.get("status"), + ) else: if checkpoint_state is None: raise StyleAnalysisCheckpointError("款式分析 checkpoint 状态缺失") diff --git a/src/gyxx_flow/modules/product_commerce/collect_douyin_qianchuan_ads.py b/src/gyxx_flow/modules/product_commerce/collect_douyin_qianchuan_ads.py index 17f126f..97086ae 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_douyin_qianchuan_ads.py +++ b/src/gyxx_flow/modules/product_commerce/collect_douyin_qianchuan_ads.py @@ -43,6 +43,19 @@ BASE_URL = ( "&product-invest-order-platform=0&product-material-order-platform=0&mar-goal=2" ) DOWNLOAD_SELECTOR = 'button:has(iconpark-icon[name="oc-icon-download"])' +TASK_CENTER_URL = ( + "https://qianchuan.jinritemai.com/tools/task-center/download-task" + "?aavid={advertiser_id}" +) +TASK_SCENE_MARKER = "全域推广数据-推商品" +TASK_READY_MARKER = "下载任务列表" +TASK_COMPLETE_MARKER = "生成完成" +TASK_FAILED_MARKERS = ("生成失败", "任务失败") +ASYNC_TASK_MARKERS = ( + "已生成下载任务", + "已为您创建报表生成任务", + "前往任务中心", +) REQUIRED_HEADERS = ( "商品ID", "商品名称", @@ -151,6 +164,148 @@ def _validate_page(page) -> None: raise DouyinQianchuanDownloadError("巨量千川商品明细下载按钮数量异常") +def _task_file_marker(start_date: date, end_date: date) -> str: + return ( + f"{start_date.isoformat()} 00:00:00-" + f"{end_date.isoformat()} 23:59:59" + ) + + +def _matching_task_rows(page, start_date: date, end_date: date): + return ( + page.locator("tr.ovui-tr") + .filter(has_text=TASK_SCENE_MARKER) + .filter(has_text=_task_file_marker(start_date, end_date)) + ) + + +def _validate_task_center_page(page) -> None: + body = page.locator("body").inner_text(timeout=10_000) + current_url = str(page.url or "") + if "/login" in current_url.casefold() or "passport" in current_url.casefold(): + raise DouyinQianchuanDownloadError("巨量千川 Cookie 已失效，任务中心跳转到登录页") + if TASK_READY_MARKER not in body: + raise DouyinQianchuanDownloadError("巨量千川下载任务中心未达到可用状态") + + +def _download_task_from_page( + page, + start_date: date, + end_date: date, + *, + wait_timeout_ms: int, +): + """Download a matching completed task, optionally waiting for generation.""" + + deadline = time.monotonic() + wait_timeout_ms / 1000 + navigation_retries = 0 + while True: + rows = _matching_task_rows(page, start_date, end_date) + try: + has_rows = rows.count() > 0 + except Exception as exc: + if ( + "execution context was destroyed" not in str(exc).casefold() + or navigation_retries >= 1 + ): + raise + navigation_retries += 1 + page.wait_for_function( + "(marker) => (document.body?.innerText || '').includes(marker)", + arg=TASK_READY_MARKER, + timeout=15_000, + ) + _validate_task_center_page(page) + continue + if has_rows: + row = rows.first + row_text = row.inner_text() + if TASK_COMPLETE_MARKER in row_text: + with page.expect_download(timeout=60_000) as download_info: + row.get_by_text("下载", exact=True).click() + return download_info.value + if any(marker in row_text for marker in TASK_FAILED_MARKERS): + raise DouyinQianchuanDownloadError( + "巨量千川下载任务生成失败：" + row_text.replace("\n", " ") + ) + if time.monotonic() >= deadline: + return None + refresh = page.get_by_role("button", name="刷新") + if refresh.count(): + refresh.click() + else: + page.reload(wait_until="domcontentloaded", timeout=90_000) + page.wait_for_timeout(2_000) + + +def _download_from_task_center( + context, + start_date: date, + end_date: date, + *, + wait_timeout_ms: int, +): + task_page = context.new_page() + try: + task_page.goto( + TASK_CENTER_URL.format(advertiser_id=quote(ADVERTISER_ID)), + wait_until="domcontentloaded", + timeout=90_000, + ) + task_page.wait_for_function( + "(marker) => (document.body?.innerText || '').includes(marker)", + arg=TASK_READY_MARKER, + timeout=90_000, + ) + _validate_task_center_page(task_page) + return _download_task_from_page( + task_page, + start_date, + end_date, + wait_timeout_ms=wait_timeout_ms, + ) + except Exception: + _save_debug(task_page, "task_center_failed") + raise + finally: + task_page.close() + + +def _trigger_report_download(page): + """Support both the old direct download and the new asynchronous task flow.""" + + downloads: list[object] = [] + + def remember_download(download) -> None: + downloads.append(download) + + page.on("download", remember_download) + page.locator(DOWNLOAD_SELECTOR).click() + deadline = time.monotonic() + 120 + try: + while time.monotonic() < deadline: + if downloads: + return downloads[0] + body = page.locator("body").inner_text(timeout=10_000) + if any(marker in body for marker in ASYNC_TASK_MARKERS): + return None + page.wait_for_timeout(500) + finally: + page.remove_listener("download", remember_download) + raise DouyinQianchuanDownloadError( + "巨量千川点击下载后既未直接下载，也未创建异步下载任务" + ) + + +def _save_download(download, output_dir: Path) -> Path: + suggested = re.sub( + r'[<>:"/\\|?*]+', "_", download.suggested_filename or "千川商品报表.xlsx" + ) + target = output_dir / suggested + download.save_as(str(target)) + return target + + def _visible_calendar_months(page) -> list[tuple[int, int]]: labels = page.locator( ".ovui-range-picker__popper--show .ovui-date__header-label" @@ -335,14 +490,28 @@ def collect( _validate_page(page) _set_date_range(page, start_date, end_date) _validate_page(page) - with page.expect_download(timeout=120_000) as download_info: - page.locator(DOWNLOAD_SELECTOR).click() - download = download_info.value - suggested = re.sub( - r'[<>:"/\\|?*]+', "_", download.suggested_filename or "千川商品报表.xlsx" + # A failed prior run may already have created the report in the + # asynchronous task center. Reuse it before creating a duplicate. + download = _download_from_task_center( + context, + start_date, + end_date, + wait_timeout_ms=0, ) - target = output_dir / suggested - download.save_as(str(target)) + if download is None: + download = _trigger_report_download(page) + if download is None: + download = _download_from_task_center( + context, + start_date, + end_date, + wait_timeout_ms=180_000, + ) + if download is None: + raise DouyinQianchuanDownloadError( + "巨量千川异步下载任务在 180 秒内未生成完成" + ) + target = _save_download(download, output_dir) except Exception: _save_debug(page, "download_failed") raise diff --git a/src/gyxx_flow/modules/product_commerce/collect_dy_market_rank.py b/src/gyxx_flow/modules/product_commerce/collect_dy_market_rank.py index a8608d0..0106fef 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_dy_market_rank.py +++ b/src/gyxx_flow/modules/product_commerce/collect_dy_market_rank.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -"""采集抖音电商罗盘市场商品榜单，并生成飞书文档表格。""" +"""采集抖音电商罗盘市场商品榜单，并按月份归档到品牌行军图数据库。""" from __future__ import annotations @@ -45,7 +45,9 @@ from market_rank_limits import ( # noqa: E402 remaining_category_rank_limit, require_valid_market_rank_products, ) -from market_rank_report_archive import archive_market_rank_report # noqa: E402 +from market_rank_product_import import ( # noqa: E402 + archive_collected_market_rank_to_database, +) MARKET_PRODUCT_RANK_URL = "https://compass.jinritemai.com/shop/chance/rank-product" OUTPUT_DIR = RAW_DATA_ROOT / "dy_market_rank" @@ -1034,21 +1036,14 @@ def main() -> int: return 0 title_scope = category_specs[0]["name"] if len(category_specs) == 1 else "八品类" title = f"抖音{title_scope}市场商品榜单 {args.period} {args.report_date.isoformat()}" - doc_url = create_dy_feishu_doc_by_category( - title, - products, - period=args.period, - ) - print(f"[OK] 飞书文档: {doc_url}") - archive_market_rank_report( + result = archive_collected_market_rank_to_database( platform="dy", - report_date=args.report_date, + collected_on=args.report_date, period=args.period, category_scope=str(title_scope), category_names=[str(spec["name"]) for spec in category_specs], report_title=title, - feishu_doc_url=doc_url, - product_count=len(products), + rows=products, source_json_path=str(json_path), run_metadata={ "report_date": args.report_date.isoformat(), @@ -1064,6 +1059,7 @@ def main() -> int: }, }, ) + print(f"[OK] 已直接归档品牌行军图数据库: {result}") return 0 finally: session.close() diff --git a/src/gyxx_flow/modules/product_commerce/collect_dy_persona_to_bitable.py b/src/gyxx_flow/modules/product_commerce/collect_dy_persona_to_bitable.py index b98d975..55af3f6 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_dy_persona_to_bitable.py +++ b/src/gyxx_flow/modules/product_commerce/collect_dy_persona_to_bitable.py @@ -26,7 +26,7 @@ from datetime import date, datetime from pathlib import Path from typing import Any -from runtime_paths import LOG_ROOT, RAW_DATA_ROOT, profile_path, runtime_script +from runtime_paths import LOG_ROOT, RAW_DATA_ROOT, runtime_script from gyxx_flow.adapters import environment_for_child_script @@ -43,7 +43,6 @@ OUTPUT_ROOT = RAW_DATA_ROOT / "dy人物画像" LOG_DIR = LOG_ROOT FAIL_LIST_PATH = LOG_DIR / "dy_persona_bitable_failures.json" -DEFAULT_USER_DATA_DIR = os.getenv("DY_USER_DATA_DIR", str(profile_path("dy"))) SUCCESS_STATUSES = frozenset({"ok", "dry-run"}) RECORD_ID_PATTERN = re.compile(r"^rec[A-Za-z0-9_-]{6,}$") @@ -84,7 +83,7 @@ def time_label(value: date | datetime | None = None) -> str: # ── 采集 subprocess ────────────────────────────────────────────────────────── def run_collect_all( - user_data_dir: str, + user_data_dir: str | None, headless: bool, business_date: date, styles: str = "", @@ -93,12 +92,9 @@ def run_collect_all( 采集脚本本身的 main 就是 "一次登录 + for style in styles" 模式, 所以这里只调一次 subprocess, 不要在循环里反复调。 """ - cmd = [ - sys.executable, - str(COLLECT_SCRIPT), - "--user-data-dir", user_data_dir, - "--date", business_date.isoformat(), - ] + cmd = [sys.executable, str(COLLECT_SCRIPT), "--date", business_date.isoformat()] + if user_data_dir: + cmd[2:2] = ["--user-data-dir", user_data_dir] if styles: cmd += ["--styles", styles] if headless: @@ -460,7 +456,11 @@ def main() -> int: parser = argparse.ArgumentParser(description="每日人群画像（抖音）采集 + 回填飞书表") parser.add_argument("--date", type=date.fromisoformat, default=date.today(), help="业务日期 YYYY-MM-DD") parser.add_argument("--styles", default="", help="只跑指定款式, 逗号分隔; 默认全部") - parser.add_argument("--user-data-dir", default=DEFAULT_USER_DATA_DIR, help="Chrome profile 根目录") + parser.add_argument( + "--user-data-dir", + default="", + help="Chrome profile 根目录；未指定时沿用内层脚本的账号绑定 profile", + ) parser.add_argument("--headless", action="store_true", help="无头跑") parser.add_argument("--identity", default="user", choices=["bot", "user"], help="飞书写入身份") parser.add_argument("--dry-run", action="store_true", help="只转字段不写飞书") diff --git a/src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py b/src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py index aefc867..0a91cba 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py +++ b/src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py @@ -20,6 +20,8 @@ try: DEFAULT_CONFIG, DEFAULT_HOME_URL, USER_AGENT, + close_foreign_popup_tabs, + dismiss_marketing_popup, load_config, make_page_action, normalize_cdp_url, @@ -32,6 +34,8 @@ except ImportError: # Direct script execution from the module root. DEFAULT_CONFIG, DEFAULT_HOME_URL, USER_AGENT, + close_foreign_popup_tabs, + dismiss_marketing_popup, load_config, make_page_action, normalize_cdp_url, @@ -965,6 +969,7 @@ def wait_for_metrics(page: Any, erp_code: str, timeout_seconds: int = 45, poll_count = 0 freq_dialog_handled = 0 # 操作频繁弹窗已处理次数, 最多 3 次避免无限循环 MAX_FREQ_DIALOG = 3 + marketing_popup_check_at = 0.0 # 广告弹窗清扫节流: 至少间隔 5s 扫一次主页 DOM # 数据稳定判据: 连续 REQUIRED_STABLE 次 poll 拿到相同 sales/returns last_stable_metrics = None # 上一次 poll 的 (sales, returns), 用于比较 stable_count = 0 @@ -985,6 +990,15 @@ def wait_for_metrics(page: Any, erp_code: str, timeout_seconds: int = 45, last_stable_metrics = None stable_count = 0 continue + # 营销广告弹窗/外来推广页签定期清扫 (节流): 广告挡住 iframe 或新页签 + # 抢走前台焦点时, 数值提取/店铺切换会一直失败 + if time.monotonic() >= marketing_popup_check_at: + marketing_popup_check_at = time.monotonic() + 5.0 + cleaned = dismiss_marketing_popup(page) + cleaned += close_foreign_popup_tabs(getattr(page, "context", None), page) + if cleaned: + print(f"[ERP] {erp_code}: 已清理广告弹窗/外来页签, 等 1s 后继续轮询", flush=True) + page.wait_for_timeout(1000) for frame in page.frames: if "allsku.aspx" not in (frame.url or "").lower(): continue @@ -1512,6 +1526,8 @@ def main() -> int: "network_idle": True, "timeout": args.timeout_ms, "wait": 500, + # A retry reruns page_action, which performs the entire ERP collection. + "retries": 1, "useragent": USER_AGENT, "additional_args": { "viewport": {"width": 1600, "height": 950}, @@ -1530,7 +1546,22 @@ def main() -> int: if not cdp_url: print("[PROFILE] Note: ensure the Chrome at this profile is closed, otherwise the new instance may land in a separate subdir without cookies.") - DynamicFetcher.fetch(login_url, **fetch_kwargs) + try: + DynamicFetcher.fetch(login_url, **fetch_kwargs) + except Exception as exc: + # Scrapling converts the navigation response after page_action returns. + # Long ERP report sessions can evict that response while the completed + # collector has already persisted its results. + if ( + not state.get("collector_completed") + or "Response.all_headers: The object has been collected to prevent unbounded heap growth" + not in str(exc) + ): + raise + print( + "[WARN] ERP 报表已采集完成，忽略 Scrapling 响应转换错误", + flush=True, + ) if not state.get("ok"): print( diff --git a/src/gyxx_flow/modules/product_commerce/collect_jd_ad_costs.py b/src/gyxx_flow/modules/product_commerce/collect_jd_ad_costs.py index eee3284..323389d 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_jd_ad_costs.py +++ b/src/gyxx_flow/modules/product_commerce/collect_jd_ad_costs.py @@ -12,6 +12,8 @@ does not accept or persist a password argument. from __future__ import annotations import argparse +import csv +import io import os import re import shutil @@ -48,6 +50,12 @@ _DATE_RE = re.compile(r"(?P\d{4}-\d{2}-\d{2})\s*至\s*(?P\d{4}-\d{2} _PENDING_MARKERS = ("生成中", "处理中", "排队中", "等待中", "创建中") _FAILED_MARKERS = ("生成失败", "下载失败") DOWNLOAD_CENTER_MAX_REPORTS = 100 +# 京东偶尔会在数据集市尚未就绪时提前生成报表：对比时间区块有数据，当前时间 +# 区块只剩汇总占位行（SPU/SKU ID=0）。下载后校验并限时重提，短暂的数据延迟 +# 可以在一次运行内自愈。 +REPORT_CONTENT_ATTEMPTS = 3 +REPORT_RETRY_WAIT_SECONDS = 300 +SUMMARY_ROW_KEYS = frozenset({"", "0", "汇总", "合计", "总计"}) _LOGIN_CHALLENGE_MARKERS = ("完成拼图验证", "安全验证", "滑块验证") _LOGIN_FAILURE_MARKERS = ( "账号名与密码不匹配", @@ -1167,6 +1175,44 @@ def _extract_download( raise JDAdCostCollectionError(f"下载报表 ZIP 中找不到 {report_type} CSV") +def _normalize_cell(value: object) -> str: + return str(value or "").replace("\ufeff", "").strip().strip("'\"").strip() + + +def _report_content_rows(path: Path, report_type: ReportType) -> int: + """Count real 当前时间 SPU/SKU rows; 0 means JD had no data for the date.""" + key_column = "SPU ID" if report_type == "full" else "SKU ID" + try: + text = path.read_text(encoding="utf-8-sig") + except UnicodeDecodeError: + text = path.read_text(encoding="gb18030") + rows = list(csv.reader(io.StringIO(text))) + header_index = marker_index = key_index = None + for index, row in enumerate(rows): + cells = [_normalize_cell(value) for value in row] + if "当前时间" in cells and key_column in cells: + header_index, marker_index, key_index = ( + index, + cells.index("当前时间"), + cells.index(key_column), + ) + break + if marker_index is None or key_index is None: + raise JDAdCostCollectionError( + f"{path.name} 缺少「当前时间」或「{key_column}」表头，无法校验报表数据" + ) + count = 0 + for row in rows[header_index + 1 :]: + if len(row) <= max(marker_index, key_index): + continue + if _normalize_cell(row[marker_index]) != "当前时间": + continue + if _normalize_cell(row[key_index]) in SUMMARY_ROW_KEYS: + continue + count += 1 + return count + + def _find_state(page, report_type: ReportType, target_date: date) -> ReportState: states = [state for _, state in _report_rows(page, report_type, target_date)] if "ready" in states: @@ -1430,6 +1476,81 @@ def _start_browser(headless: bool, user_data_dir: str) -> ScraplingBrowser: return ScraplingBrowser(stealthy=True, **options) +def _resubmit_report(page, report_type: ReportType, target_date: date) -> None: + """Clear stale download-center jobs and submit the same report again.""" + safe_goto(page, DOWNLOAD_URL) + _wait_for_text(page, "下载报表", timeout_seconds=60) + _clear_download_center(page) + safe_goto(page, OVERVIEW_URL) + try: + page.reload(wait_until="domcontentloaded", timeout=60_000) + except Exception: + safe_goto(page, OVERVIEW_URL) + _wait_for_text(page, "非全站营销", timeout_seconds=60) + if report_type == "full": + # A fresh overview document starts on 自有账户; re-select the + # advertiser exactly like the first pass in run(). + _select_account(page, ADVERTISER_PIN) + _request_report(page, report_type, target_date, ADVERTISER_PIN) + + +def _ensure_report_data( + page, + downloaded: dict[ReportType, Path], + target_date: date, + output_dir: Path, + *, + timeout_seconds: int, +) -> dict[ReportType, Path]: + """Re-request reports whose 当前时间 block came back empty. + + 两种空报表需要区分：数据集市尚未产出该日期（两个报表都空，限时重提）， + 以及全站营销计划当天整体停投（京准通页面花费为 0）。非全站报表有数据 + 才能证明京东已完成该日期的数据产出，此时全站报表为空按零投放接受。 + """ + counts: dict[ReportType, int] = {} + for attempt in range(1, REPORT_CONTENT_ATTEMPTS + 1): + counts = { + report_type: _report_content_rows(path, report_type) + for report_type, path in downloaded.items() + } + if counts["non_full"] > 0: + if counts["full"] == 0: + print( + f"[WARN] 全站营销 SPU 报表没有目标日期的当前时间数据" + f"（非全站 SKU 报表 {counts['non_full']} 行正常）：判定当日" + "全站营销零投放，按无全站数据处理", + flush=True, + ) + return downloaded + if attempt >= REPORT_CONTENT_ATTEMPTS: + break + print( + f"[WARN] 京东报表没有目标日期的当前时间数据（数据可能未就绪），" + f"{REPORT_RETRY_WAIT_SECONDS} 秒后清空下载中心并重新提交" + f"（第 {attempt + 1}/{REPORT_CONTENT_ATTEMPTS} 次）", + flush=True, + ) + time.sleep(REPORT_RETRY_WAIT_SECONDS) + for report_type in ("non_full", "full"): + if counts[report_type] > 0: + continue + _resubmit_report(page, report_type, target_date) + downloaded[report_type] = download_ready_report( + page, + report_type, + target_date, + output_dir, + timeout_seconds=timeout_seconds, + ) + raise JDAdCostCollectionError( + f"京东报表连续 {REPORT_CONTENT_ATTEMPTS} 次下载都没有目标日期的" + f"当前时间数据（全站 {counts.get('full', 0)} 行、非全站 " + f"{counts.get('non_full', 0)} 行）；京东数据可能尚未就绪，" + "请稍后重新运行采集" + ) + + def run(args: argparse.Namespace) -> list[Path]: target_date = parse_target_date(args.date) output_dir = (Path(args.output).expanduser().resolve() / target_date.isoformat()) @@ -1463,7 +1584,7 @@ def run(args: argparse.Namespace) -> list[Path]: safe_goto(page, OVERVIEW_URL) _wait_for_text(page, "非全站营销", timeout_seconds=60) account_selected = False - downloaded: list[Path] = [] + downloaded: dict[ReportType, Path] = {} for index, report_type in enumerate(("non_full", "full")): if report_type == "full" and not account_selected: _select_account(page, ADVERTISER_PIN) @@ -1474,14 +1595,12 @@ def run(args: argparse.Namespace) -> list[Path]: # A pre-existing task still uses the selected advertiser for # the full-site request below. account_selected = True - downloaded.append( - download_ready_report( - page, - report_type, - target_date, - output_dir, - timeout_seconds=args.task_timeout_seconds, - ) + downloaded[report_type] = download_ready_report( + page, + report_type, + target_date, + output_dir, + timeout_seconds=args.task_timeout_seconds, ) if index == 0: # The download-center route replaces the overview SPA. Return @@ -1493,6 +1612,13 @@ def run(args: argparse.Namespace) -> list[Path]: except Exception: safe_goto(page, OVERVIEW_URL) _wait_for_text(page, "非全站营销", timeout_seconds=60) + downloaded = _ensure_report_data( + page, + downloaded, + target_date, + output_dir, + timeout_seconds=args.task_timeout_seconds, + ) manifest = { "schema_version": 1, "dataset": "jd_ad_costs", @@ -1501,12 +1627,12 @@ def run(args: argparse.Namespace) -> list[Path]: "shop_account": login_account, "files": { report_type: path.relative_to(output_dir).as_posix() - for report_type, path in zip(("non_full", "full"), downloaded, strict=True) + for report_type, path in downloaded.items() }, "downloaded_at": datetime.now().isoformat(timespec="seconds"), } atomic_write_json(output_dir / "manifest.json", manifest) - for path in downloaded: + for path in downloaded.values(): print(f"[OK] 已保存京东营销报表：{path}") return downloaded diff --git a/src/gyxx_flow/modules/product_commerce/collect_jd_market_rank.py b/src/gyxx_flow/modules/product_commerce/collect_jd_market_rank.py index 3f66b30..76136d2 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_jd_market_rank.py +++ b/src/gyxx_flow/modules/product_commerce/collect_jd_market_rank.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -"""采集京东商智行业商品榜单，并生成带销售额的飞书文档表格。""" +"""采集京东商智行业商品榜单，并按月份归档到品牌行军图数据库。""" from __future__ import annotations @@ -57,7 +57,9 @@ from market_rank_limits import ( # noqa: E402 remaining_category_rank_limit, require_valid_market_rank_products, ) -from market_rank_report_archive import archive_market_rank_report # noqa: E402 +from market_rank_product_import import ( # noqa: E402 + archive_collected_market_rank_to_database, +) JD_MARKET_RANK_URL = ( "https://sz.jd.com/szweb/sz/view/industryMarket/productRanks.html" @@ -1657,37 +1659,14 @@ def main() -> int: else f"{len(categories)}品类" ) title = f"京东{title_scope}市场商品榜单 {args.period} {args.report_date.isoformat()}" - if args.resume_doc_url or args.resume_offset: - doc_url = create_jd_feishu_doc_batched( - title, - products, - period=args.period, - batch_size=args.doc_batch_size, - doc_url=args.resume_doc_url, - start_offset=args.resume_offset, - max_retries=args.feishu_retries, - cooldown_seconds=args.feishu_cooldown, - repair_images=not args.no_image_repair, - ) - else: - doc_url = create_jd_feishu_doc_by_category( - title, - products, - period=args.period, - max_retries=args.feishu_retries, - cooldown_seconds=args.feishu_cooldown, - repair_images=not args.no_image_repair, - ) - print(f"[OK] 飞书文档: {doc_url}") - archive_market_rank_report( + result = archive_collected_market_rank_to_database( platform="jd", - report_date=args.report_date, + collected_on=args.report_date, period=args.period, category_scope=str(title_scope), category_names=sorted(categories), report_title=title, - feishu_doc_url=doc_url, - product_count=len(products), + rows=products, source_json_path=str(input_path), run_metadata={ "report_date": args.report_date.isoformat(), @@ -1697,6 +1676,7 @@ def main() -> int: "limit": args.limit, }, ) + print(f"[OK] 已直接归档品牌行军图数据库: {result}") return 0 category_specs = _category_specs(args.category) @@ -1785,37 +1765,14 @@ def main() -> int: return 0 title_scope = category_specs[0]["name"] if len(category_specs) == 1 else "七品类" title = f"京东{title_scope}市场商品榜单 {args.period} {args.report_date.isoformat()}" - if args.resume_doc_url or args.resume_offset: - doc_url = create_jd_feishu_doc_batched( - title, - products, - period=args.period, - batch_size=args.doc_batch_size, - doc_url=args.resume_doc_url, - start_offset=args.resume_offset, - max_retries=args.feishu_retries, - cooldown_seconds=args.feishu_cooldown, - repair_images=not args.no_image_repair, - ) - else: - doc_url = create_jd_feishu_doc_by_category( - title, - products, - period=args.period, - max_retries=args.feishu_retries, - cooldown_seconds=args.feishu_cooldown, - repair_images=not args.no_image_repair, - ) - print(f"[OK] 飞书文档: {doc_url}") - archive_market_rank_report( + result = archive_collected_market_rank_to_database( platform="jd", - report_date=args.report_date, + collected_on=args.report_date, period=args.period, category_scope=str(title_scope), category_names=[str(spec["name"]) for spec in category_specs], report_title=title, - feishu_doc_url=doc_url, - product_count=len(products), + rows=products, source_json_path=str(json_path), run_metadata={ "report_date": args.report_date.isoformat(), @@ -1830,6 +1787,7 @@ def main() -> int: "resume_doc_url": args.resume_doc_url or None, }, ) + print(f"[OK] 已直接归档品牌行军图数据库: {result}") return 0 finally: session.close() diff --git a/src/gyxx_flow/modules/product_commerce/collect_jd_persona_to_bitable.py b/src/gyxx_flow/modules/product_commerce/collect_jd_persona_to_bitable.py index da56d12..1591ce9 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_jd_persona_to_bitable.py +++ b/src/gyxx_flow/modules/product_commerce/collect_jd_persona_to_bitable.py @@ -5,7 +5,10 @@ 流程: 1. 从项目数据库动态读取 (经 StyleConfigLoader) 款式 -> 人群画像(京东)目标表映射 条件: 同一款式同时有 天猫分组的人群画像(京东)URL + 京东旗舰店分组的商品ID(SPU) -2. 一次 Playwright 浏览器登录京东商智, 返回旧版, 逐款式搜SPU -> 详情 -> 人群画像tab -> 近30天 -> 采集 +2. 一次 Playwright 浏览器登录京东商智; 商智新版把"人群画像"路由到独立包装页 + (single-product-detail-temp.html, 内嵌旧版 singleProDetails iframe)。 + 逐款式直接打开该包装页 -> iframe 内切"近30天" -> 采集。 + 数据优先取自 getSpuSingleProUsersData.ajax 响应, 悬停补采兜底。 每款产出 JSON 到 data/jd人物画像/{date}/{style}/ 3. 读每款 JSON, 把性别/年龄/购买力数据映射到飞书子表字段 4. 按"时间"(采集当天 M.D) upsert 到各款式对应的子表 @@ -20,10 +23,11 @@ import re import shutil import subprocess import sys +import time from datetime import date, datetime from pathlib import Path from typing import Any -from urllib.parse import urlparse +from urllib.parse import urlencode, urlparse from runtime_paths import ( LOG_ROOT, @@ -46,6 +50,10 @@ from config.style_config_loader import ( # noqa: E402 StyleConfigLoader, get_jd_persona_bitable_map, ) +from jd_product_data_collector import ( # noqa: E402 + close_guide_if_present, + extract_crowd_chart_by_hover, +) OUTPUT_ROOT = RAW_DATA_ROOT / "jd人物画像" LOG_DIR = LOG_ROOT @@ -65,6 +73,19 @@ JD_SEARCH_BUTTON_SELECTORS = ( '.search-multiple-input button:has-text("搜索")', 'button.jmtd-button:has-text("搜索")', ) +JD_PRODUCT_DETAIL_HOSTS = frozenset({"sz.jd.com", "jdsz.jd.com"}) +JD_PRODUCT_DETAIL_PATHS = frozenset( + { + "/szweb/sz/view/productanalysis/productdetail.html", + "/szweb/view/product/productdetail.html", + } +) +# 商智新版"人群画像"不再原地切 tab, 而是打开独立包装页并内嵌旧版明细页 iframe +JD_BUYER_PORTRAIT_PATH = "/szweb/view/product/single-product-detail-temp.html" +JD_PORTRAIT_IFRAME_URL_MARKER = "singleProDetails" +JD_PORTRAIT_IFRAME_SELECTOR = "iframe[src*='singleProDetails']" +JD_PERSONA_API_MARKER = "getSpuSingleProUsersData" +BUYING_POWER_KEYWORDS = ("小白领", "高级白领", "土豪", "白领", "购买力") def combined_result_exit_code( @@ -192,6 +213,18 @@ def _wait_for_jd_session(page, *, polls: int = 8, fallback_after: int = 4) -> bo return False +def _is_jd_product_detail_url(url: str) -> bool: + """Accept the current detail route and JD's legacy redirect target.""" + + try: + parsed = urlparse(url) + except (TypeError, ValueError): + return False + host = (parsed.hostname or "").lower().rstrip(".") + path = (parsed.path or "").lower().rstrip("/") + return host in JD_PRODUCT_DETAIL_HOSTS and path in JD_PRODUCT_DETAIL_PATHS + + def visible_input_diagnostics(page) -> list[str]: try: values = page.evaluate( @@ -270,9 +303,12 @@ def _goto_product_detail_with_network_tolerance(page, step_goto_product_detail) timeout=60_000, ) page.wait_for_timeout(5_000) - if "productAnalysis/productDetail" not in (page.url or ""): - raise RuntimeError(f"京东商品明细页跳转失败: {page.url}") - print(" [OK] 已到达商品明细页面（忽略后台常驻请求）") + url, body = page_evidence(page) + reason = blocking_page_reason(url, body) + if reason or not _is_jd_product_detail_url(url): + detail = reason or "未落到受支持的商品明细路由" + raise RuntimeError(f"京东商品明细页跳转失败: {detail}: {url}") + print(f" [OK] 已到达商品明细页面: {url}（忽略后台常驻请求）") # ── 时间标签 ────────────────────────────────────────────────────────────────── @@ -644,7 +680,211 @@ def is_permission_error(message: str) -> bool: return "91402" in m or "91403" in m or "notexist" in m or "permission" in m -# ── 浏览器采集 (旧版商智单品人群画像) ────────────────────────────────────────── +# ── 浏览器采集 (新版商智人群画像包装页) ──────────────────────────────────────── + +class FrameMouseProxy: + """Translate frame-local coordinates into main-frame viewport coordinates.""" + + def __init__(self, owner: "FramePageProxy") -> None: + self._owner = owner + + def _offset(self) -> tuple[float, float]: + box = self._owner.host_page.locator(JD_PORTRAIT_IFRAME_SELECTOR).first \ + .bounding_box() + return (box["x"], box["y"]) if box else (0.0, 0.0) + + def move(self, x: float, y: float, **kwargs) -> None: + offset_x, offset_y = self._offset() + self._owner.host_page.mouse.move(x + offset_x, y + offset_y, **kwargs) + + def click(self, x: float, y: float, **kwargs) -> None: + offset_x, offset_y = self._offset() + self._owner.host_page.mouse.click(x + offset_x, y + offset_y, **kwargs) + + +class FramePageProxy: + """Present an iframe Frame with the page-like surface the legacy + jd_product_data_collector extractors expect (evaluate/locator/mouse).""" + + def __init__(self, frame, host_page) -> None: + self._frame = frame + self.host_page = host_page + self.mouse = FrameMouseProxy(self) + + def evaluate(self, *args: Any, **kwargs: Any) -> Any: + return self._frame.evaluate(*args, **kwargs) + + def wait_for_timeout(self, timeout: int) -> None: + self.host_page.wait_for_timeout(timeout) + + def locator(self, selector: str, **kwargs: Any): + return self._frame.locator(selector, **kwargs) + + def wait_for_selector(self, selector: str, **kwargs: Any): + return self._frame.wait_for_selector(selector, **kwargs) + + +def buyer_portrait_url(spu_id: str) -> str: + """Return the new-shangzhi buyer-portrait wrapper URL for one SPU.""" + query = urlencode( + {"sxu": spu_id, "type": "spu", "from": "jdsz", "tab": "buyerPortrait"} + ) + return f"https://jdsz.jd.com{JD_BUYER_PORTRAIT_PATH}?{query}" + + +def wait_for_portrait_frame(page, *, timeout_s: float = 25.0): + """Poll for the legacy singleProDetails iframe embedded by the wrapper.""" + + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + for frame in page.frames: + if JD_PORTRAIT_IFRAME_URL_MARKER in frame.url: + return frame + page.wait_for_timeout(1_000) + return None + + +def _format_ratio(value: Any) -> str: + """Render a 0-1 ratio like the legacy hover extractors did ("90.38%").""" + + try: + return f"{float(value) * 100:.2f}%" + except (TypeError, ValueError): + return "" + + +def _iter_metric_pairs(entries: Any): + """Yield (name, value) from the API's [name, ratio] pair lists.""" + + for entry in entries or []: + if isinstance(entry, (list, tuple)) and len(entry) >= 2: + yield entry[0], entry[1] + elif isinstance(entry, dict): + yield entry.get("name"), entry.get("value", entry.get("percent")) + + +def parse_persona_api_payload(payload: dict) -> dict[str, list[dict[str, str]]]: + """Map getSpuSingleProUsersData content onto the legacy profile keys.""" + + content = (payload or {}).get("content") or {} + crowd: dict[str, list[dict[str, str]]] = {} + + gender = [] + for name, ratio in _iter_metric_pairs(content.get("gender")): + label = str(name or "").strip() + rendered = _format_ratio(ratio) + if label in ("男", "女", "男性", "女性") and rendered: + gender.append({"name": label, "value": rendered}) + crowd["性别"] = gender + + ages = [] + for name, ratio in _iter_metric_pairs(content.get("age")): + label = str(name or "").strip() + rendered = _format_ratio(ratio) + if "岁" in label and rendered: + ages.append({"name": label, "value": rendered}) + crowd["年龄"] = ages + + powers = [] + for name, ratio in _iter_metric_pairs(content.get("buy")): + label = str(name or "").strip() + rendered = _format_ratio(ratio) + if label and rendered and any(keyword in label for keyword in BUYING_POWER_KEYWORDS): + powers.append({"name": label, "value": rendered}) + crowd["购买力"] = powers + return crowd + + +def _has_persona_values(crowd: dict[str, list[dict[str, str]]]) -> bool: + return any(crowd.get(key) for key in ("性别", "年龄", "购买力")) + + +def collect_spu_portrait(context, spu_id: str) -> tuple[dict | None, str]: + """Collect one SPU's buyer portrait; returns (crowd_data, failure_note).""" + + page = context.new_page() + payloads: list[dict] = [] + + def on_response(response) -> None: + if JD_PERSONA_API_MARKER not in response.url: + return + # 只取近30天(range)那次请求; 默认单日(day)口径人群过小, 画像为空 + if "dateType=range" not in response.url: + return + try: + payloads.append(json.loads(response.text())) + except Exception: + return + + page.on("response", on_response) + try: + page.goto( + buyer_portrait_url(spu_id), + wait_until="domcontentloaded", + timeout=60_000, + ) + frame = wait_for_portrait_frame(page) + if frame is None: + return None, "画像 iframe 未加载" + proxy = FramePageProxy(frame, page) + try: + close_guide_if_present(proxy) + except Exception: + pass + + try: + range_button = frame.locator("text=近30天").first + range_button.wait_for(state="visible", timeout=15_000) + range_button.click() + except Exception as exc: + return None, f"近30天切换失败: {exc}" + + deadline = time.monotonic() + 20 + while time.monotonic() < deadline and not payloads: + page.wait_for_timeout(500) + + crowd_data = parse_persona_api_payload(payloads[-1]) if payloads else {} + + # 悬停补采兜底: API 结构漂移时仍能拿回数据 + if not crowd_data.get("性别"): + try: + frame.wait_for_selector("canvas", timeout=15_000) + page.wait_for_timeout(3_000) + crowd_data["性别"] = [ + item + for item in extract_crowd_chart_by_hover(proxy, "性别", "ring") + if str(item.get("name", "")) in ("男", "女", "男性", "女性") + ] + except Exception as exc: + print(f" [WARN] 性别悬停补采失败: {exc}") + if not crowd_data.get("年龄"): + try: + crowd_data["年龄"] = [ + item + for item in extract_crowd_chart_by_hover(proxy, "年龄", "bar") + if "岁" in str(item.get("name", "")) + ] + except Exception as exc: + print(f" [WARN] 年龄悬停补采失败: {exc}") + if not crowd_data.get("购买力"): + try: + frame.evaluate("window.scrollBy(0, 600)") + page.wait_for_timeout(1_500) + crowd_data["购买力"] = extract_buying_power(proxy) + except Exception as exc: + print(f" [WARN] 购买力补采失败: {exc}") + + if not _has_persona_values(crowd_data): + return None, "画像为空(人群规模较小或无数据)" + return crowd_data, "" + except Exception as exc: + return None, f"画像页采集异常: {exc}" + finally: + try: + page.close() + except Exception: + pass + def run_collect_all(persona_map: dict[str, dict[str, str]], spu_groups: dict[str, list[str]], @@ -655,15 +895,8 @@ def run_collect_all(persona_map: dict[str, dict[str, str]], step_enter_shangzhi, step_login, ) - from jd_peer_product_data_collector import step_return_old_version - from jd_product_data_collector import ( - click_crowd_portrait_tab, - click_detail_and_wait_new_page, - close_guide_if_present, - extract_crowd_portrait_data, - search_spu_in_detail_page, - step_goto_product_detail, - ) + from jd_product_data_collector import step_goto_product_detail + print(f"[JD] 待采款式 {len(persona_map)}: {list(persona_map.keys())}") OUTPUT_ROOT.mkdir(parents=True, exist_ok=True) results: list[dict[str, str]] = [] @@ -682,7 +915,6 @@ def run_collect_all(persona_map: dict[str, dict[str, str]], step_login=step_login, ) sz_page = step_enter_shangzhi(context, page) - step_return_old_version(sz_page) _goto_product_detail_with_network_tolerance( sz_page, step_goto_product_detail, @@ -707,107 +939,45 @@ def run_collect_all(persona_map: dict[str, dict[str, str]], for spu_idx, spu_id in enumerate(spus, 1): print(f" [{spu_idx}/{len(spus)}] 尝试 SPU={spu_id}") + crowd_data, note = collect_spu_portrait(context, spu_id) + if crowd_data: + used_spu = spu_id + print( + " [OK] 采集成功: " + + ", ".join( + f"{k}{len(v)}条" for k, v in crowd_data.items() if v + ) + ) + break + print(f" [FAIL] {spu_id}: {note}") + failure_messages.append(f"{spu_id}: {note}") + + # 款式间确认会话仍有效, 失效则重进商智 try: - search_spu_in_detail_page(sz_page, spu_id) - close_guide_if_present(sz_page) - - detail_page = click_detail_and_wait_new_page(sz_page, context, spu_id) - if not detail_page: - message = f"{spu_id}: 详情页未打开" - print(f" [FAIL] {message}") - failure_messages.append(message) - continue - - # 1. 点人群画像 tab - click_crowd_portrait_tab(detail_page) - detail_page.wait_for_timeout(3000) - - # 2. 选"近30天" + url, body = page_evidence(sz_page) + reason = blocking_page_reason(url, body) + if reason: + raise RuntimeError(reason) + except Exception as exc: + print(f" [WARN] 商智会话异常({exc}), 重新进入...") try: - btn_30d = detail_page.locator('text=近30天').first - if btn_30d.is_visible(timeout=5000): - btn_30d.click() - print(" [OK] 切换到近30天") - detail_page.wait_for_timeout(3000) - except Exception as e: - print(f" [WARN] 切换近30天失败: {e}") - - # 3. 采集人群画像数据 (性别/年龄) - crowd_data_full = extract_crowd_portrait_data(detail_page) - # 只保留性别和年龄, 不要职业/地区 - crowd_data = {} - for k in ("性别", "年龄"): - if k in crowd_data_full: - crowd_data[k] = crowd_data_full[k] - - - # 4. 额外提取购买力分布 (需滚动到页面下方，购买力卡片在性别/年龄下面) - detail_page.evaluate("window.scrollBy(0, 600)") - detail_page.wait_for_timeout(1500) - if not crowd_data.get("购买力"): - buying_power = extract_buying_power(detail_page) - if buying_power: - crowd_data["购买力"] = buying_power - print(f" [OK] 购买力分布: {len(buying_power)}条") - - has_data = any([ - crowd_data.get("性别"), - crowd_data.get("年龄"), - crowd_data.get("购买力"), - ]) - - if has_data: - print(" [OK] 采集成功: " + ", ".join(f"{k}{len(v)}条" for k, v in crowd_data.items() if v)) - used_spu = spu_id - else: - print(" [SKIP] 无数据, 换下一个") - failure_messages.append(f"{spu_id}: 画像为空") - crowd_data = None - - detail_page.close() - # 关闭详情页后确认 sz_page 还活着，否则重新导航 - try: - sz_page.url - sz_page.wait_for_timeout(2000) - except Exception: - print(" [WARN] sz_page 已失效, 重新导航商智...") - try: - sz_page = context.new_page() - step_login(sz_page, DEFAULT_CONFIG["shop"], DEFAULT_CONFIG["password"]) - sz_page = step_enter_shangzhi(context, sz_page) - step_return_old_version(sz_page) - step_goto_product_detail(sz_page) - except Exception as e2: - print(f" [FAIL] 重新导航失败: {e2}") - failure_messages.append(f"{spu_id}: 重新导航失败: {e2}") - break - - if crowd_data: + sz_page = context.new_page() + _ensure_jd_session( + sz_page, + shop=DEFAULT_CONFIG["shop"], + password=DEFAULT_CONFIG["password"], + step_login=step_login, + ) + sz_page = step_enter_shangzhi(context, sz_page) + _goto_product_detail_with_network_tolerance( + sz_page, + step_goto_product_detail, + ) + except Exception as e2: + print(f" [FAIL] 重新导航失败: {e2}") + failure_messages.append(f"{spu_id}: 重新导航失败: {e2}") break - except Exception as e: - print(f" [FAIL] 采集异常: {e}") - failure_messages.append(f"{spu_id}: {e}") - try: - detail_page.close() - except Exception: - pass - # 检查 sz_page 是否还活着 - try: - sz_page.url - except Exception: - print(" [WARN] sz_page 已失效, 重新导航商智...") - try: - sz_page = context.new_page() - step_login(sz_page, DEFAULT_CONFIG["shop"], DEFAULT_CONFIG["password"]) - sz_page = step_enter_shangzhi(context, sz_page) - step_return_old_version(sz_page) - step_goto_product_detail(sz_page) - except Exception as e2: - print(f" [FAIL] 重新导航失败: {e2}") - failure_messages.append(f"{spu_id}: 重新导航失败: {e2}") - break - if crowd_data: path = save_profile(crowd_data, style, used_spu, business_date) print(f" [OK] 保存: {path}") diff --git a/src/gyxx_flow/modules/product_commerce/collect_sycm_market_rank.py b/src/gyxx_flow/modules/product_commerce/collect_sycm_market_rank.py index 3b4b3f0..1dcc730 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_sycm_market_rank.py +++ b/src/gyxx_flow/modules/product_commerce/collect_sycm_market_rank.py @@ -28,7 +28,7 @@ from market_rank_limits import ( remaining_category_rank_limit, require_valid_market_rank_products, ) -from market_rank_report_archive import archive_market_rank_report +from market_rank_product_import import archive_collected_market_rank_to_database from runtime_paths import ( DATA_HOME, RAW_DATA_ROOT, @@ -1815,27 +1815,15 @@ def main() -> int: if args.dry_run: return 0 title_scope = "九品类" if not args.category else category_specs[0]["name"] - title = ( - f"天猫{title_scope}市场排行 30天 {args.report_date.isoformat()}" - ) - url = create_feishu_doc_by_category( - title, - products, - min_price=args.min_price, - max_price=args.max_price, - period=args.period, - independent_prices=not args.category, - ) - print(f"[OK] 飞书文档: {url}") - archive_market_rank_report( + title = f"天猫{title_scope}市场排行 {args.period} {args.report_date:%Y-%m}" + result = archive_collected_market_rank_to_database( platform="tm", - report_date=args.report_date, + collected_on=args.report_date, period=args.period, category_scope=str(title_scope), category_names=[str(spec["name"]) for spec in category_specs], report_title=title, - feishu_doc_url=url, - product_count=len(products), + rows=products, source_json_path=str(json_path), run_metadata={ "report_date": args.report_date.isoformat(), @@ -1855,6 +1843,7 @@ def main() -> int: }, }, ) + print(f"[OK] 已直接归档品牌行军图数据库: {result}") return 0 finally: if session is not None: diff --git a/src/gyxx_flow/modules/product_commerce/collect_tmall_wanxiang_ads.py b/src/gyxx_flow/modules/product_commerce/collect_tmall_wanxiang_ads.py index 2e54745..9d64614 100644 --- a/src/gyxx_flow/modules/product_commerce/collect_tmall_wanxiang_ads.py +++ b/src/gyxx_flow/modules/product_commerce/collect_tmall_wanxiang_ads.py @@ -21,7 +21,7 @@ import zipfile from datetime import date, datetime, timedelta from pathlib import Path from typing import Any, Callable, Literal -from urllib.parse import quote +from urllib.parse import quote, urlsplit, urlunsplit from runtime_paths import RAW_DATA_ROOT, TMP_ROOT, profile_path from scrapling.fetchers import StealthySession @@ -49,6 +49,19 @@ USER_DATA_DIR = os.getenv( ) OUTPUT_ROOT = RAW_DATA_ROOT / "tmall_wanxiang_ads" MAX_ZIP_MEMBER_BYTES = 512 * 1024 * 1024 +SIGNED_DOWNLOAD_MAX_ATTEMPTS = 3 +SIGNED_DOWNLOAD_RETRY_DELAY_SECONDS = 2.0 +_RETRYABLE_SIGNED_DOWNLOAD_MARKERS = ( + "enotfound", + "eai_again", + "getaddrinfo", + "econnreset", + "etimedout", + "timed out", + "socket hang up", + "network is unreachable", + "connection refused", +) class WanxiangDownloadError(RuntimeError): @@ -866,6 +879,24 @@ def _click_download_button(page) -> bool: return False +def _safe_signed_download_error(exc: BaseException, download_url: str) -> str: + """Keep temporary OSS credentials out of persisted run errors.""" + + message = str(exc) + parsed = urlsplit(download_url) + safe_url = urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", "")) + return message.replace(download_url, safe_url) + + +def _is_retryable_signed_download_error(exc: BaseException) -> bool: + message = str(exc).casefold() + return any(marker in message for marker in _RETRYABLE_SIGNED_DOWNLOAD_MARKERS) + + +def _is_retryable_signed_download_status(status: int) -> bool: + return status in {408, 429} or 500 <= status <= 599 + + def _save_signed_download( page, download_url: str, @@ -883,13 +914,44 @@ def _save_signed_download( path = output_dir / f"商品报表_{target_date.strftime('%Y%m%d')}.zip" partial_path = path.with_suffix(path.suffix + ".part") try: - response = request_context.get(download_url, timeout=120_000) - if not response.ok: - raise WanxiangDownloadError( - f"万相台签名下载地址返回 HTTP {response.status}" - ) - partial_path.write_bytes(response.body()) - partial_path.replace(path) + for attempt in range(1, SIGNED_DOWNLOAD_MAX_ATTEMPTS + 1): + try: + response = request_context.get(download_url, timeout=120_000) + except Exception as exc: + if ( + not _is_retryable_signed_download_error(exc) + or attempt >= SIGNED_DOWNLOAD_MAX_ATTEMPTS + ): + safe_error = _safe_signed_download_error(exc, download_url) + raise WanxiangDownloadError( + f"万相台签名文件下载网络异常：{safe_error}" + ) from exc + print( + f"[WARN] 万相台签名文件下载网络异常，第 {attempt}/" + f"{SIGNED_DOWNLOAD_MAX_ATTEMPTS} 次失败，{SIGNED_DOWNLOAD_RETRY_DELAY_SECONDS:g}s 后重试" + ) + time.sleep(SIGNED_DOWNLOAD_RETRY_DELAY_SECONDS) + continue + + if not response.ok: + status = int(response.status) + if ( + not _is_retryable_signed_download_status(status) + or attempt >= SIGNED_DOWNLOAD_MAX_ATTEMPTS + ): + raise WanxiangDownloadError( + f"万相台签名下载地址返回 HTTP {status}" + ) + print( + f"[WARN] 万相台签名文件下载返回 HTTP {status}，第 {attempt}/" + f"{SIGNED_DOWNLOAD_MAX_ATTEMPTS} 次失败，{SIGNED_DOWNLOAD_RETRY_DELAY_SECONDS:g}s 后重试" + ) + time.sleep(SIGNED_DOWNLOAD_RETRY_DELAY_SECONDS) + continue + + partial_path.write_bytes(response.body()) + partial_path.replace(path) + break finally: if partial_path.exists(): partial_path.unlink() @@ -1065,7 +1127,11 @@ def extract_report_zip(zip_path: Path, target_date: date) -> list[Path]: def _start_session(headless: bool, user_data_dir: str): session = StealthySession( headless=headless, - real_chrome=True, + # The installed workstation Chrome has repeatedly crashed in this + # isolated profile during the headless Wanxiang flow. Keep the + # profile/cookies, but use Scrapling's bundled Chromium for this + # collector so a browser crash cannot strand the workflow in login. + real_chrome=False, user_data_dir=user_data_dir, locale="zh-CN", timezone_id="Asia/Shanghai", diff --git a/src/gyxx_flow/modules/product_commerce/db/__init__.py b/src/gyxx_flow/modules/product_commerce/db/__init__.py index 76618a2..91410c9 100644 --- a/src/gyxx_flow/modules/product_commerce/db/__init__.py +++ b/src/gyxx_flow/modules/product_commerce/db/__init__.py @@ -63,8 +63,8 @@ 写: analyze_style.py (直连 MiniMax 产出后归档) 读: 周复盘 / 飞书看板 / 报告归档查询 fact_market_rank_reports - 写: collect_{sycm,jd,dy}_market_rank.py (飞书文档创建成功后) - 读: 三平台市场排行报告索引 / 飞书文档归档查询 + 写: collect_{sycm,jd,dy}_market_rank.py (采集完成后按月份直接归档) + 读: 三平台市场排行月度报告索引 / 行军图快照查询 fact_market_rank_notifications 写: market_rank_hermes_notification.py (Hermes 机器人通知结果；文件名兼容保留) 读: 同一收件人同日通知去重 / 发送审计 @@ -1738,10 +1738,10 @@ def upsert_style_analysis_report(conn, record: dict) -> int: # ── fact_market_rank_reports ───────────────────────────────────────────────── def upsert_market_rank_report(conn, record: dict) -> int: - """保存三平台市场排行飞书文档，返回归档行 id。 + """保存三平台市场排行月度归档，返回归档行 id。 - 业务 UNIQUE = (platform, report_date)。同一平台同一天无论重跑多少次，均更新 - 当天唯一记录的链接及运行元数据，不产生重复记录。 + 业务 UNIQUE = (platform, report_date)。采集器传入月初日期，同一平台同一月份 + 无论重跑多少次，均更新唯一记录及运行元数据，不产生重复记录。 """ sql = """ INSERT INTO fact_market_rank_reports diff --git a/src/gyxx_flow/modules/product_commerce/db/schema.sql b/src/gyxx_flow/modules/product_commerce/db/schema.sql index 2cd211f..ba231a1 100644 --- a/src/gyxx_flow/modules/product_commerce/db/schema.sql +++ b/src/gyxx_flow/modules/product_commerce/db/schema.sql @@ -872,10 +872,10 @@ COMMENT ON COLUMN fact_style_analysis_reports.generated_at IS '报告生成时 COMMENT ON COLUMN fact_style_analysis_reports.updated_at IS '记录更新时间 (重跑覆盖时刷新)'; -- ============================================================================ --- fact_market_rank_reports: 三平台市场排行飞书文档归档 +-- fact_market_rank_reports: 三平台市场排行月度归档 -- 数据源: collect_sycm_market_rank.py / collect_jd_market_rank.py / -- collect_dy_market_rank.py --- 同平台、同日无论执行多少次都覆盖，保留最后一次成功生成的飞书文档链接。 +-- 同平台、同月无论执行多少次都覆盖；飞书文档链接仅作为兼容字段，可为空。 -- ============================================================================ CREATE TABLE IF NOT EXISTS fact_market_rank_reports ( id BIGSERIAL PRIMARY KEY, @@ -886,7 +886,7 @@ CREATE TABLE IF NOT EXISTS fact_market_rank_reports ( category_names TEXT[] NOT NULL DEFAULT '{}', report_title TEXT NOT NULL, feishu_doc_id TEXT, - feishu_doc_url TEXT NOT NULL, + feishu_doc_url TEXT, product_count INTEGER NOT NULL DEFAULT 0 CHECK (product_count >= 0), source_json_path TEXT, run_metadata JSONB, @@ -895,6 +895,9 @@ CREATE TABLE IF NOT EXISTS fact_market_rank_reports ( UNIQUE (platform, report_date) ); +ALTER TABLE fact_market_rank_reports + ALTER COLUMN feishu_doc_url DROP NOT NULL; + -- 兼容旧表：旧唯一键包含 period/category_scope，同平台同日可能产生多条。 -- 先保留更新时间最新的一条，再替换为“平台 + 日期”唯一约束。 ALTER TABLE fact_market_rank_reports @@ -927,20 +930,40 @@ BEGIN END IF; END $$; + +-- 兼容旧版日归档：同平台同月只保留更新时间最新的一条，并统一为月初。 +WITH ranked_monthly_market_reports AS ( + SELECT id, + ROW_NUMBER() OVER ( + PARTITION BY platform, DATE_TRUNC('month', report_date) + ORDER BY report_date DESC, updated_at DESC, generated_at DESC, id DESC + ) AS row_num + FROM fact_market_rank_reports +) +DELETE FROM fact_market_rank_reports target + USING ranked_monthly_market_reports ranked + WHERE target.id = ranked.id + AND ranked.row_num > 1; + +UPDATE fact_market_rank_reports + SET report_date = DATE_TRUNC('month', report_date)::DATE, + updated_at = NOW() + WHERE report_date <> DATE_TRUNC('month', report_date)::DATE; + CREATE INDEX IF NOT EXISTS fact_market_rank_reports_date_idx ON fact_market_rank_reports (report_date DESC); CREATE INDEX IF NOT EXISTS fact_market_rank_reports_platform_idx ON fact_market_rank_reports (platform, report_date DESC); -COMMENT ON TABLE fact_market_rank_reports IS '天猫、京东、抖音市场排行采集生成的飞书文档归档；每个平台每天仅保留最后一次成功生成的文档。'; +COMMENT ON TABLE fact_market_rank_reports IS '天猫、京东、抖音市场排行月度归档；每个平台每月仅保留最后一次成功采集的榜单快照，飞书链接可为空。'; COMMENT ON COLUMN fact_market_rank_reports.platform IS '平台代码：tm / jd / dy'; -COMMENT ON COLUMN fact_market_rank_reports.report_date IS '采集并生成文档的业务日期'; +COMMENT ON COLUMN fact_market_rank_reports.report_date IS '采集月份业务日期，归一化为月初'; COMMENT ON COLUMN fact_market_rank_reports.period IS '平台统计周期原文，如 30天 / 近30天'; -COMMENT ON COLUMN fact_market_rank_reports.category_scope IS '文档范围，如 九品类 / 七品类 / 单品类名'; +COMMENT ON COLUMN fact_market_rank_reports.category_scope IS '本次采集范围，如 九品类 / 七品类 / 单品类名'; COMMENT ON COLUMN fact_market_rank_reports.category_names IS '本次实际包含的逻辑品类名称'; COMMENT ON COLUMN fact_market_rank_reports.feishu_doc_id IS '从飞书 /docx/{token} 链接提取的文档 token'; -COMMENT ON COLUMN fact_market_rank_reports.feishu_doc_url IS '飞书文档完整访问链接'; -COMMENT ON COLUMN fact_market_rank_reports.product_count IS '文档内采集商品数'; +COMMENT ON COLUMN fact_market_rank_reports.feishu_doc_url IS '兼容旧流程的飞书文档访问链接；直接入库流程为空'; +COMMENT ON COLUMN fact_market_rank_reports.product_count IS '本月榜单快照商品数'; COMMENT ON COLUMN fact_market_rank_reports.source_json_path IS '本次采集 JSON 产物路径'; COMMENT ON COLUMN fact_market_rank_reports.run_metadata IS '价格、搜索词、分页等运行参数 JSON'; @@ -1068,6 +1091,7 @@ CREATE TABLE IF NOT EXISTS market_product_classification ( scene_id BIGINT REFERENCES dim_market_scene(id), raw_style_name TEXT, raw_scene_name TEXT, + category_name TEXT, source TEXT NOT NULL DEFAULT 'manual' CHECK (source IN ('manual', 'feishu_import')), edited_by BIGINT, @@ -1081,9 +1105,41 @@ CREATE INDEX IF NOT EXISTS market_product_classification_style_idx CREATE INDEX IF NOT EXISTS market_product_classification_style_scene_product_idx ON market_product_classification (style_id, scene_id, market_product_id); +ALTER TABLE market_product_classification + ADD COLUMN IF NOT EXISTS category_name TEXT; + +WITH latest_categories AS ( + SELECT DISTINCT ON (snapshot.market_product_id) + snapshot.market_product_id, + NULLIF(BTRIM(snapshot.category_name), '') AS category_name + FROM fact_market_product_snapshot snapshot + JOIN fact_market_rank_reports report + ON report.id = snapshot.report_id + WHERE NULLIF(BTRIM(snapshot.category_name), '') IS NOT NULL + ORDER BY snapshot.market_product_id, + report.report_date DESC, + snapshot.updated_at DESC, + snapshot.id DESC +) +INSERT INTO market_product_classification + (market_product_id, category_name, source, edited_at, updated_at) +SELECT market_product_id, category_name, 'manual', NOW(), NOW() + FROM latest_categories +ON CONFLICT (market_product_id) DO UPDATE SET + category_name = COALESCE( + NULLIF(BTRIM(market_product_classification.category_name), ''), + EXCLUDED.category_name + ), + updated_at = CASE + WHEN NULLIF(BTRIM(market_product_classification.category_name), '') IS NULL + THEN NOW() + ELSE market_product_classification.updated_at + END; + COMMENT ON TABLE market_product IS '三平台市场榜单商品主数据，平台商品 ID 唯一，跨报告日期复用。'; -COMMENT ON TABLE fact_market_product_snapshot IS '市场榜单商品日快照；金额为榜单区间上限口径的估算值。'; +COMMENT ON TABLE fact_market_product_snapshot IS '市场榜单商品月度快照；金额为榜单区间上限口径的估算值。'; COMMENT ON TABLE market_product_classification IS '品牌行军图人工分类；独立于采集快照，重新采集不得覆盖 manual 数据。'; +COMMENT ON COLUMN market_product_classification.category_name IS '品牌行军图人工品类；按商品主数据跨月份复用。'; COMMENT ON COLUMN fact_market_product_snapshot.estimated_gmv IS '天猫=买家数上限×单价；京东/抖音=销售额区间上限。'; COMMENT ON COLUMN fact_market_product_snapshot.quality_status IS 'complete 或缺失价格/买家数/销售额等数据质量状态。'; diff --git a/src/gyxx_flow/modules/product_commerce/dy_audience_profile_collect.py b/src/gyxx_flow/modules/product_commerce/dy_audience_profile_collect.py index cd28e8e..40fc9a5 100644 --- a/src/gyxx_flow/modules/product_commerce/dy_audience_profile_collect.py +++ b/src/gyxx_flow/modules/product_commerce/dy_audience_profile_collect.py @@ -243,6 +243,18 @@ def persistent_profile_launch_options(user_data_dir: str) -> dict[str, str]: return {"user_data_dir": user_data_dir} +def account_profile_launch_options( + user_data_dir: str, + cookies: list[dict[str, Any]] | None, +) -> dict[str, Any]: + """Launch an isolated member profile with the account vault snapshot.""" + + options: dict[str, Any] = persistent_profile_launch_options(user_data_dir) + if cookies: + options["cookies"] = cookies + return options + + def cookie_fallback_launch_options( cookies: list[dict[str, Any]] | None, ) -> dict[str, list[dict[str, Any]]]: @@ -643,7 +655,11 @@ def main() -> int: try: response = DynamicFetcher.fetch( "https://compass.jinritemai.com/shop/business-part", - **persistent_profile_launch_options(args.user_data_dir), + **( + account_profile_launch_options(args.user_data_dir, cookies) + if os.getenv("GYXX_ACCOUNT_ID", "").strip() + else persistent_profile_launch_options(args.user_data_dir) + ), **fetch_kwargs, ) except Exception as profile_error: diff --git a/src/gyxx_flow/modules/product_commerce/dy_product_scraping.py b/src/gyxx_flow/modules/product_commerce/dy_product_scraping.py index 2e936b4..c562312 100644 --- a/src/gyxx_flow/modules/product_commerce/dy_product_scraping.py +++ b/src/gyxx_flow/modules/product_commerce/dy_product_scraping.py @@ -3337,7 +3337,9 @@ def main(argv: list[str] | None = None) -> int: cookies = _load_cookies() fetch_kwargs = dict( headless=False, - real_chrome=True, + # The installed Chrome has crashed during Douyin Excel downloads; + # keep the isolated profile/cookies but use Scrapling's bundled Chromium. + real_chrome=False, timeout=120000, page_action=_guarded_page_automation, wait=2000, diff --git a/src/gyxx_flow/modules/product_commerce/erp_login_product_analysis.py b/src/gyxx_flow/modules/product_commerce/erp_login_product_analysis.py index d0c15ae..59c0c95 100644 --- a/src/gyxx_flow/modules/product_commerce/erp_login_product_analysis.py +++ b/src/gyxx_flow/modules/product_commerce/erp_login_product_analysis.py @@ -243,6 +243,104 @@ def dismiss_password_change_popup(page: Any) -> bool: return False +def dismiss_marketing_popup(page: Any) -> int: + """检测并关闭 ERP 主页/报表页的营销广告弹窗, 返回点掉的关闭控件数. + + 广告弹窗 DOM 不固定 (大促/活动期会出现整屏弹窗挡住点击), 无法像密码过期框 + 一样按固定文案处理. 这里用通用策略: 在主页 DOM 找可见的小尺寸关闭控件 + (class 含 close / 常见关闭文案: × ✕ 关闭 知道了 以后再说 暂不 跳过), + 逐个 JS click, 最后补一次 Escape. 与"操作频繁"弹窗一样用 page.evaluate, + 只处理主页 DOM, 不进业务 iframe, 误伤面小. + """ + try: + dismissed = page.evaluate( + """() => { + const closeWords = new Set(['×', '✕', 'x', 'X', '关闭', '知道了', '我知道了', + '以后再说', '暂不', '暂不升级', '跳过']); + const hit = new Set(); + const visible = (el) => { + const r = el.getBoundingClientRect(); + if (r.width <= 0 || r.height <= 0) return false; + const s = getComputedStyle(el); + return s.display !== 'none' && s.visibility !== 'hidden' && Number(s.opacity) > 0.01; + }; + const clickedAncestor = (el) => { + for (let node = el.parentElement; node; node = node.parentElement) { + if (hit.has(node)) return true; + } + return false; + }; + for (const el of document.querySelectorAll('button, a, span, i, img, div, svg')) { + if (hit.has(el) || clickedAncestor(el)) continue; + if (!visible(el)) continue; + const r = el.getBoundingClientRect(); + if (r.width > 140 || r.height > 64) continue; + const cls = (el.className && el.className.baseVal !== undefined) + ? String(el.className.baseVal) : String(el.className || ''); + const text = (el.innerText || el.textContent || '').trim(); + const alt = (el.getAttribute && el.getAttribute('alt')) || ''; + const looksClose = /close|guanbi/i.test(cls) + || /close/i.test(alt) + || closeWords.has(text); + if (!looksClose) continue; + // 只点悬浮层里的控件: 自身或祖先是 fixed/absolute 定位, 避免误点页面流内容 + let node = el, floating = false; + for (let i = 0; i < 6 && node && node !== document.body; i++) { + const ps = getComputedStyle(node).position; + if (ps === 'fixed' || ps === 'absolute') { floating = true; break; } + node = node.parentElement; + } + if (!floating) continue; + try { el.click(); hit.add(el); } catch (e) {} + } + return hit.size; + }""" + ) + count = int(dismissed or 0) + except Exception: + return 0 + if count: + try: + page.keyboard.press("Escape") + except Exception: + pass + page.wait_for_timeout(400) + print(f"[LOGIN] Dismissed marketing popup ({count} close control(s))", flush=True) + return count + + +def close_foreign_popup_tabs(context: Any, keep_page: Any) -> int: + """关闭 ERP 会话里弹出的外来页签 (营销推广页/广告落地页), 返回关闭数. + + ERP 登录后会从 account.scm121.com 之类地址 window.open 推广页, 新页签 + 抢走前台焦点后, 后续对工作页的点击/帧定位会失效. 这里只保留工作页和 + ERP 自身域名页, 其余非 about:blank 页签一律关掉. + """ + try: + pages = list(getattr(context, "pages", []) or []) + except Exception: + return 0 + closed = 0 + for popup in pages: + if popup is keep_page: + continue + try: + url = str(getattr(popup, "url", "") or "") + except Exception: + continue + if not url or url.startswith(("about:blank", "chrome://", "devtools://", "edge://")): + continue + if "erp321.com" in url: + continue + try: + popup.close() + closed += 1 + print(f"[LOGIN] Closed foreign popup tab: {url[:120]}", flush=True) + except Exception: + continue + return closed + + def captcha_exists(page: Any) -> bool: for selector in CAPTCHA_SELECTORS: try: @@ -380,7 +478,18 @@ def click_product_analysis(page: Any, home_url: str, timeout_ms: int) -> str: 自动重试 2 次: ERP 系统偶发 "frame did not load" 是因为点击没生效 (网络/资源慢), 重试前先刷新 home 页面再点一次. + 每个点击策略点完用短探测验证报表帧是否真的出现: 2026-09 首页改版后出现过 + "点到了不可见模板节点" 的假成功 (打印了 Clicked 但页面无反应), 这时换下一个 + 策略继续, 而不是直接进入 60s 帧等待把重试烧完. """ + strategies = ( + ("frames", lambda: click_product_analysis_in_frames(page)), + ("icon", lambda: click_product_analysis_by_icon(page)), + ("report-rail", lambda: click_product_analysis_via_report_rail(page)), + ("visible-text", lambda: click_visible_text(page, "商品主题分析", timeout_ms)), + ("dom-position", lambda: click_text_by_dom_position(page, "商品主题分析")), + ("shortcut-position", lambda: click_product_analysis_by_shortcut_position(page)), + ) last_error: Exception | None = None for attempt in range(1, 3): try: @@ -400,29 +509,26 @@ def click_product_analysis(page: Any, home_url: str, timeout_ms: int) -> str: pass page.wait_for_timeout(1000 if attempt > 1 else 800) - clicked = click_product_analysis_in_frames(page) - if not clicked: - clicked = click_product_analysis_by_icon(page) - if not clicked: - clicked = click_visible_text(page, "商品主题分析", timeout_ms) - if not clicked: - clicked = click_text_by_dom_position(page, "商品主题分析") - if not clicked: - clicked = click_product_analysis_by_shortcut_position(page) - if not clicked: - raise RuntimeError("Could not find a visible 商品主题分析 entry to click") + # 广告弹窗/外来推广页签可能挡住入口或抢住焦点: 每次尝试点击前先清一遍 + close_foreign_popup_tabs(getattr(page, "context", None), page) + dismiss_marketing_popup(page) - print(f"[NAV] Clicked 商品主题分析 (attempt {attempt})") - - try: - page.wait_for_load_state("networkidle", timeout=5000) - except Exception: - page.wait_for_timeout(1500) - - product_url = wait_for_product_analysis_frame(page, timeout_seconds=60) - if not product_url: - raise RuntimeError("Clicked 商品主题分析, but product analysis frame did not load") - return product_url + for name, strategy in strategies: + if not strategy(): + continue + print(f"[NAV] Clicked 商品主题分析 via {name} (attempt {attempt})") + product_url = wait_for_product_analysis_frame( + page, + timeout_seconds=20, + context=getattr(page, "context", None), + ) + if product_url: + return product_url + print( + f"[WARN] {name} click did not open the report frame, " + "trying next strategy" + ) + raise RuntimeError("Could not find a visible 商品主题分析 entry to click") except Exception as exc: last_error = exc print(f"[WARN] attempt {attempt} failed: {exc}") @@ -431,13 +537,28 @@ def click_product_analysis(page: Any, home_url: str, timeout_ms: int) -> str: raise RuntimeError(f"商品主题分析 重试 2 次仍失败: {last_error}") -def wait_for_product_analysis_frame(page: Any, timeout_seconds: int) -> str | None: +def wait_for_product_analysis_frame( + page: Any, + timeout_seconds: int, + context: Any | None = None, +) -> str | None: deadline = time.time() + timeout_seconds while time.time() < deadline: for frame in page.frames: frame_url = frame.url or "" if "itemskuanalysis" in frame_url.lower(): return frame_url + # 报表有时会以新页签形式打开: 找到后记录 URL (采集仍以工作页为准) + if context is not None: + try: + other_pages = [p for p in context.pages if p is not page] + except Exception: + other_pages = [] + for other in other_pages: + other_url = str(getattr(other, "url", "") or "") + if "itemskuanalysis" in other_url.lower(): + print(f"[NAV] Report opened in a new tab: {other_url[:120]}") + return other_url page.wait_for_timeout(1000) return None @@ -554,6 +675,95 @@ def click_product_analysis_by_dom(context: Any) -> bool: return bool(context.evaluate(script)) +_REPORT_RAIL_SCRIPT = """ +() => { + const target = '\\u62a5\\u8868'; + const isVisible = (el) => { + const style = window.getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden') return false; + const box = el.getBoundingClientRect(); + return box.width > 1 && box.height > 1; + }; + const fire = (el) => { + el.scrollIntoView({ block: 'center', inline: 'center' }); + for (const type of ['mouseover', 'mousedown', 'mouseup', 'click']) { + el.dispatchEvent(new MouseEvent(type, { bubbles: true, cancelable: true, view: window })); + } + el.click(); + }; + for (const el of document.querySelectorAll('span,div,a,li,p,i')) { + const text = (el.innerText || el.textContent || '').trim(); + if (text !== target) continue; + if (!isVisible(el)) continue; + const box = el.getBoundingClientRect(); + // 左侧栏图标文字: 贴左缘的小块, 避免命中正文里其他"报表"字样 + if (box.left > 100 || box.width > 100 || box.height > 100) continue; + fire(el); + return true; + } + return false; +} +""" + +_REPORT_ENTRY_SCRIPT = """ +() => { + const target = '\\u5546\\u54c1\\u4e3b\\u9898\\u5206\\u6790'; + const isVisible = (el) => { + const style = window.getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden') return false; + const box = el.getBoundingClientRect(); + return box.width > 1 && box.height > 1; + }; + const fire = (el) => { + el.scrollIntoView({ block: 'center', inline: 'center' }); + for (const type of ['mouseover', 'mousedown', 'mouseup', 'click']) { + el.dispatchEvent(new MouseEvent(type, { bubbles: true, cancelable: true, view: window })); + } + el.click(); + }; + for (const el of document.querySelectorAll('span,div,a,li,p')) { + const text = (el.innerText || el.textContent || '').trim(); + if (text !== target) continue; + if (!isVisible(el)) continue; + fire(el); + return true; + } + return false; +} +""" + + +def click_product_analysis_via_report_rail(page: Any) -> bool: + """新版 epaas 首页: 「商品主题分析」入口收进左侧栏「报表」面板. + + 2026-09 首页改版后入口不再平铺在首页; 旧的文本/坐标策略点不到入口, + 最后的坐标兜底会点错到同区域的「ERP分销版」, 新开分销推广页签. + 正确路径: 点最左侧栏「报表」图标展开面板, 再点面板里的「商品主题分析」. + 用 DOM 事件直点 (不走视口坐标), 避免页签前台切换/布局偏移导致点歪. + 先探入口是否已可见 (面板可能已展开), 再点栏, 展开后逐 frame 点入口. + """ + for frame in page.frames: + try: + if frame.evaluate(_REPORT_ENTRY_SCRIPT): + return True + except Exception: + continue + for frame in page.frames: + try: + if not frame.evaluate(_REPORT_RAIL_SCRIPT): + continue + except Exception: + continue + page.wait_for_timeout(900) + for inner in page.frames: + try: + if inner.evaluate(_REPORT_ENTRY_SCRIPT): + return True + except Exception: + continue + return False + + def click_text_by_dom_position(page: Any, text: str) -> bool: script = """ (text) => { @@ -741,11 +951,16 @@ def make_page_action( # 兜底: 跳到 epaas 主页后, ERP 有时还会在异步回调里再弹一次"密码过期" # 提醒, 在点商品主题分析前再扫一遍 + 点掉, 防止点不到目标按钮. + # 大促/活动期的营销广告弹窗同样在这里先清掉; 登录后新开的外来推广 + # 页签 (抢前台焦点) 也在这一起关掉. try: for _ in range(3): - if not dismiss_password_change_popup(page): + dismissed = 1 if dismiss_password_change_popup(page) else 0 + dismissed += dismiss_marketing_popup(page) + dismissed += close_foreign_popup_tabs(context, page) + if not dismissed: break - _ts("post-login password-expiry popup dismissed") + _ts("post-login popup/tab dismissed (password/marketing/foreign-tab)") page.wait_for_timeout(500) except Exception as exc: print(f"[WARN] post-login popup cleanup skipped: {exc}") diff --git a/src/gyxx_flow/modules/product_commerce/import_jd_ad_costs.py b/src/gyxx_flow/modules/product_commerce/import_jd_ad_costs.py index bc62d91..5a79835 100644 --- a/src/gyxx_flow/modules/product_commerce/import_jd_ad_costs.py +++ b/src/gyxx_flow/modules/product_commerce/import_jd_ad_costs.py @@ -781,18 +781,35 @@ def _parse_files( raise FileNotFoundError("未找到京东全站营销 SPU CSV") if non_full_path is None or not Path(non_full_path).is_file(): raise FileNotFoundError("未找到京东非全站营销 SKU CSV") - full_records = parse_full_csv( - Path(full_path), - target_date, - advertiser_pin=advertiser_pin, - shop_account=shop_account, - ) + full_records: list[dict[str, Any]] = [] + full_empty_error: ValueError | None = None + try: + full_records = parse_full_csv( + Path(full_path), + target_date, + advertiser_pin=advertiser_pin, + shop_account=shop_account, + ) + except ValueError as exc: + # 全站营销计划可能当天整体停投（京准通页面花费为 0）。此时全站报表 + # 只有汇总占位行；只要非全站报表有数据就视为合法零投放，否则保留 + # 原错误（更可能是数据未就绪）。 + full_empty_error = exc sku_records = parse_non_full_csv( Path(non_full_path), target_date, advertiser_pin=advertiser_pin, shop_account=shop_account, ) + if not full_records: + if sku_records: + print( + f"[WARN] {Path(full_path).name} 没有目标日期的当前时间 SPU 行，" + f"而非全站 SKU 报表有 {len(sku_records)} 行：判定当日全站营销" + "零投放，按无全站数据处理" + ) + elif full_empty_error is not None: + raise full_empty_error return full_records, sku_records diff --git a/src/gyxx_flow/modules/product_commerce/import_product_daily.py b/src/gyxx_flow/modules/product_commerce/import_product_daily.py index aa4b1cc..44e5ab3 100644 --- a/src/gyxx_flow/modules/product_commerce/import_product_daily.py +++ b/src/gyxx_flow/modules/product_commerce/import_product_daily.py @@ -12,7 +12,7 @@ product_daily_metrics 表 (ON CONFLICT 全量覆盖)。 源数据列数 (已验证): dy xlsx: 22 列 (row 0 = 表头, 之后是数据/合计行) - jd xlsx: 34 列 (row 0 = 表头, row 1 = 合计行, 之后是数据) + jd xlsx: 按表头映射 (row 0 = 表头, 之后是数据/合计行) tm xls: 31/38 列 (row 0-3 空白, row 4 = 表头, row 5+ = 数据) """ from __future__ import annotations @@ -228,101 +228,102 @@ def parse_dy_xlsx(xlsx_path: Path) -> Iterable[dict]: # ── jd 解析 ─────────────────────────────────────────────────────────────── def parse_jd_xlsx(xlsx_path: Path, shop_id: str) -> Iterable[dict]: - """京东 全部渠道_商品明细_分天下载.xlsx → record dicts. + """解析京东商品明细 xlsx，并按表头名称映射字段。 - 列顺序 (0-indexed): - 0 时间 - 1 商品ID - 2 商品名称 - 3 二级类目 → category_l1 - 4 三级类目 → category_l2 - 5 货号 → product_no - 6 访客数 → visitors - 7 浏览量 → page_views - 8 商品关注数 - 9 加购商品件数 → add_cart_qty - 10 加购人数 → add_cart_buyers - 11 成交客户数 → paid_buyers - 12 成交单量 → raw (订单数,非件数) - 13 成交商品件数 → paid_qty - 14 成交金额 → paid_amount - 15 成交转化率 → conversion_rate - 16 UV价值 → uv_value - 17 最近上架时间 - 18 详情页跳出率 → detail_page_bounce_rate - 19 搜索曝光量 - 20 搜索点击次数 - 21 搜索点击率 - 22 系统预测未来7天销量 - 23 下单客户数 - 24 下单商品件数 - 25 下单金额 - 26 下单成交转化率 - 27 下单转化率 - 28 PV现货率 - 29 客单价 → avg_order_value - 30 件单价 - 31 取消及售后退款金额 → refund_amount - 32 取消及售后退款件数 → refund_orders - 33 取消及售后退款单量 + 京东导出会增减字段，也可能加入对比日列；按固定下标读取会把对比列 + 当成当天指标。核心字段按同义表头匹配，其余原始列保存在 raw_data。 """ wb = openpyxl.load_workbook(xlsx_path, data_only=True) - ws = wb.active + try: + ws = wb.active + rows = ws.iter_rows(values_only=True) + header = next(rows, None) + if not header: + return - rows = list(ws.iter_rows(values_only=True)) - if not rows: - return - for r in rows[1:]: - if not r or r[0] is None: - continue - stat_date = _to_date(r[0]) - product_id = _strip(r[1]) - if _is_total_or_invalid_row(product_id, stat_date): - continue + header_names = [_strip(value) for value in header] + header_index = {name: i for i, name in enumerate(header_names) if name} - raw = { - "商品关注数": r[8], - "成交单量": r[12], - "最近上架时间": _strip(r[17]) if r[17] is not None else None, - "搜索曝光量": r[19], - "搜索点击次数": r[20], - "搜索点击率": r[21], - "系统预测未来7天销量": r[22], - "下单客户数": r[23], - "下单商品件数": r[24], - "下单金额": r[25], - "下单成交转化率": r[26], - "下单转化率": r[27], - "PV现货率": r[28], - "件单价": r[30], - "取消及售后退款单量": r[33], + def column(*aliases: str) -> int | None: + return next((header_index[name] for name in aliases if name in header_index), None) + + columns = { + "stat_date": column("时间", "统计日期"), + "product_id": column("SPU", "商品ID", "商品编号"), + "product_name": column("SPU名称", "商品名称"), + "product_no": column("货号"), + "category_l1": column("二级类目"), + "category_l2": column("三级类目"), + "visitors": column("商品访客数", "访客数"), + "page_views": column("商品浏览量", "浏览量"), + "add_cart_qty": column("加购商品件数", "商品加购件数"), + "add_cart_buyers": column("加购客户数", "加购人数", "商品加购人数"), + "paid_amount": column("成交金额", "支付金额"), + "paid_qty": column("成交商品件数", "支付件数"), + "paid_buyers": column("成交客户数", "支付买家数"), + "refund_amount": column("取消及售后退款金额", "成功退款金额"), + "refund_orders": column( + "取消及售后退款商品件数", "取消及售后退款件数", "退款订单数" + ), + "conversion_rate": column("成交转化率", "商品支付转化率", "支付转化率"), + "avg_order_value": column("客单价", "成交客单价"), + "uv_value": column("UV价值", "访客平均价值"), + "detail_page_bounce_rate": column("商品详情页跳出率", "详情页跳出率"), } + required_columns = ("stat_date", "product_id", "paid_amount", "paid_qty", "paid_buyers") + missing_columns = [name for name in required_columns if columns[name] is None] + if missing_columns: + raise ValueError( + f"无法识别京东商品明细表头: {xlsx_path.name}; " + f"missing={missing_columns!r}; headers={header_names[:12]!r}" + ) - yield { - "platform": "jd", - "stat_date": stat_date, - "shop_id": shop_id, - "product_id": product_id, - "product_name": _strip(r[2]) or None, - "product_no": _strip(r[5]) or None, - "category_l1": _strip(r[3]) or None, - "category_l2": _strip(r[4]) or None, - "visitors": _to_int(r[6]), - "page_views": _to_int(r[7]), - "add_cart_qty": _to_int(r[9]), - "add_cart_buyers": _to_int(r[10]), - "paid_amount": _to_num(r[14]), - "paid_qty": _to_int(r[13]), - "paid_buyers": _to_int(r[11]), - "refund_amount": _to_num(r[31]), - "refund_orders": _to_int(r[32]), - "conversion_rate": _to_pct(r[15]), - "avg_order_value": _to_num(r[29]), - "uv_value": _to_num(r[16]), - "detail_page_bounce_rate": _to_pct(r[18]), - "raw_data": {k: v for k, v in raw.items() if v not in (None, "", "-")}, - "source_file": xlsx_path.name, - } + def value(row: tuple[Any, ...], key: str) -> Any: + index = columns[key] + return row[index] if index is not None and index < len(row) else None + + for row in rows: + if not row: + continue + stat_date = _to_date(value(row, "stat_date")) + product_id = _strip(value(row, "product_id")) + if _is_total_or_invalid_row(product_id, stat_date): + continue + + raw_data = { + name: cell + for name, cell in zip(header_names, row) + if name and cell not in (None, "", "-") + } + yield { + "platform": "jd", + "stat_date": stat_date, + "shop_id": shop_id, + "product_id": product_id, + "product_name": _strip(value(row, "product_name")) or None, + "product_no": _strip(value(row, "product_no")) or None, + "category_l1": _strip(value(row, "category_l1")) or None, + "category_l2": _strip(value(row, "category_l2")) or None, + "visitors": _to_int(value(row, "visitors")), + "page_views": _to_int(value(row, "page_views")), + "add_cart_qty": _to_int(value(row, "add_cart_qty")), + "add_cart_buyers": _to_int(value(row, "add_cart_buyers")), + "paid_amount": _to_num(value(row, "paid_amount")), + "paid_qty": _to_int(value(row, "paid_qty")), + "paid_buyers": _to_int(value(row, "paid_buyers")), + "refund_amount": _to_num(value(row, "refund_amount")), + "refund_orders": _to_int(value(row, "refund_orders")), + "conversion_rate": _to_pct(value(row, "conversion_rate")), + "avg_order_value": _to_num(value(row, "avg_order_value")), + "uv_value": _to_num(value(row, "uv_value")), + "detail_page_bounce_rate": _to_pct( + value(row, "detail_page_bounce_rate") + ), + "raw_data": raw_data, + "source_file": xlsx_path.name, + } + finally: + wb.close() # ── tm 解析 ─────────────────────────────────────────────────────────────── diff --git a/src/gyxx_flow/modules/product_commerce/jd_main_image_collector.py b/src/gyxx_flow/modules/product_commerce/jd_main_image_collector.py index b2b2a89..64b7b4f 100644 --- a/src/gyxx_flow/modules/product_commerce/jd_main_image_collector.py +++ b/src/gyxx_flow/modules/product_commerce/jd_main_image_collector.py @@ -1,14 +1,19 @@ -"""京东商品明细页主图采集脚本。 +"""京东商品主图采集脚本（商智新版 jdsz.jd.com）。 -采集流程: -1. 复用 vendors/jd-data-flow/playwright_profile 登录态 (同 jd_product_data_collector.py) -2. 进入商智 -> 旧版 -> 商品明细页 -3. 点「指标配置」勾选「搜索曝光量」「搜索点击次数」两列 -4. 逐款在多 SPU 搜索框输入该款的 SPU 列表 -> 点查询按钮 -5. 点展开加号 .jmti-plus-circle-f 展开所有 SKU 子行 -6. 解析表格 HTML 拿每个 SKU 的主图 URL / 曝光量 / 点击量 -7. 按 SPU 聚合: 取曝光量最大的 SKU 主图作为该 SPU 的代表 -8. 下载主图 + 写 JSON 到 data/主图信息//<款式>/jd_main_images.json +2026-09 起京东商智旧版 (sz.jd.com) 对本账号关闭，商品明细页迁移到新版， +且「搜索曝光量/搜索点击次数」两列不再出现在商品明细的指标配置里。 +改为两段采集（日期维度均为页面默认的「昨天」，与旧采集语义一致）： +1. 商品明细页 (jdsz.jd.com/szweb/view/product/productDetail.html)： + 按款式搜索 SPU -> 展开 SPU 树 -> 取全部 SKU 的主图 URL 与 SKU->SPU 归属。 +2. 搜索数据概览-商品分析 (jdsz.jd.com/squidweb/view/flow/searchOverview.html)： + 按 SKU 批量筛选（每批 <=8 个，避免表格分页），取「曝光次数/点击次数」， + 即旧版商品明细的「搜索曝光量/搜索点击次数」。 + +合并后沿用原语义：每款按 SPU 聚合，取曝光量最大的 SKU 主图作为该 SPU 代表， +下载主图并写 JSON/MD 到 data/主图信息//<款式>/，供飞书与 PG 插入脚本消费。 + +新版页面为 React (jmtv2/jd-*) 组件，展开、查询等交互必须用 Playwright 原生 +click（完整指针事件序列），evaluate 合成 click 不触发 onClick。 """ from __future__ import annotations @@ -21,6 +26,7 @@ import time from datetime import datetime from pathlib import Path from typing import Any +from urllib.parse import urlparse from runtime_paths import ( RAW_DATA_ROOT, @@ -42,6 +48,7 @@ if str(VENDOR_SOURCE_ROOT) not in sys.path: from jd_data_collector import ( DEFAULT_CONFIG, _is_shangzhi_url, + close_popup_if_present, step_enter_shangzhi, step_login, ) @@ -65,6 +72,14 @@ OUTPUT_DIR = RAW_DATA_ROOT / "主图信息" DEBUG_DIR = TMP_ROOT / "debug" / "jd_main_image_snapshot" DEBUG = os.getenv("JD_MAIN_IMAGE_DEBUG", "0") == "1" +# 直接使用新版商品明细页，避免旧版别名在批量采集过程中偶发 ERR_CONNECTION_CLOSED。 +DETAIL_URL = "https://jdsz.jd.com/szweb/view/product/productDetail.html" +SEARCH_OVERVIEW_URL = "https://jdsz.jd.com/squidweb/view/flow/searchOverview.html" +# 搜索概览商品分析表每页 10 行，按 SKU 筛选的每批数量必须小于每页行数 +OVERVIEW_SKU_CHUNK_SIZE = 8 +# 该款暂无京东商品记录；仅最终仍无 SKU 行时按预期跳过，不影响其他款式。 +_EXPECTED_EMPTY_JD_STYLES = frozenset({"黑曜石电脑包"}) + _STATIC_STYLE_SPU_GROUPS = [ {"style": "盖亚斜挎", "spus": ["10022621048112", "10023155725944", "10030097050220"]}, {"style": "极星pro", "spus": ["10025628785945", "10026601031834", "10026600460605"]}, @@ -113,25 +128,108 @@ def now_label() -> str: return _DATE_LABEL or datetime.now().strftime("%Y-%m-%d") +def _goto(page, url: str) -> None: + """Navigate without waiting for商智's never-idle background requests.""" + target_path = urlparse(url).path.rstrip("/").lower() + last_error = None + for attempt in range(1, 4): + try: + page.goto(url, wait_until="domcontentloaded", timeout=60000) + return + except Exception as exc: + last_error = exc + try: + current_url = str(page.url) + except Exception: + current_url = "" + current_path = urlparse(current_url).path.rstrip("/").lower() + same_route = bool(target_path and target_path == current_path) + if "productanalysis/productdetail" in target_path and "productdetail" in current_path: + same_route = True + error_text = str(exc).lower() + transient = any( + marker in error_text + for marker in ("err_connection_", "timed out", "timeout") + ) + if same_route and not transient: + print(f" [WARN] 商智页面加载等待超时，保留已提交页面: {exc}") + return + if attempt >= 3 or not transient: + if same_route: + print(f" [WARN] 商智页面已提交但加载等待失败，保留当前页面: {exc}") + return + raise + print(f" [WARN] 商智导航暂时失败，第 {attempt}/3 次后重试: {exc}") + page.wait_for_timeout(1000 * attempt) + if last_error is not None: + raise last_error + + +_PRODUCT_DETAIL_SEARCH_SELECTORS = ( + "input.jd-input-multiple-input-input", + ".jmtd-multiple-input input", + ".jmtd-multiple-input", +) + + +def _locate_product_detail_search(page): + """Return the visible SPU/product-ID search control across JD UI variants.""" + for selector in _PRODUCT_DETAIL_SEARCH_SELECTORS: + try: + locator = page.locator(selector).first + if locator.is_visible(timeout=500): + return locator + except Exception: + continue + return None + + +def _close_product_detail_overlays(page) -> None: + """Best-effort cleanup immediately before interacting with the detail page.""" + try: + close_popup_if_present(page) + except Exception as exc: + # The JD extension is outside the collector's control. A failure to + # dismiss it is recoverable as long as the target control is usable. + print(f" [WARN] 商品明细页弹层清理失败，继续检查页面控件: {exc}") + + def step_goto_product_detail(page): - """跳转到商品明细页面""" + """跳转到（新版）商品明细页面并切到「昨天」维度。 + + 明细页默认「实时」维度会把无实时活动的商品整行隐藏（显示「暂无数据」）， + 「昨天」维度与旧版页面语义一致：所有搜索到的 SPU 都会列出（指标可为 0）， + SPU 树才完整。 + """ print("[Step] 跳转到商品明细页面...") - page.goto( - "https://sz.jd.com/szweb/sz/view/productAnalysis/productDetail.html", - wait_until="networkidle", - ) - page.wait_for_timeout(5000) - _wait_for_product_detail(page) + _close_product_detail_overlays(page) + + # The新版 route occasionally commits without mounting its React content + # on the first navigation. Retry the route once, but never retry a login + # failure because that would only hide the account-state diagnosis. + for attempt in range(2): + try: + _goto(page, DETAIL_URL) + page.wait_for_timeout(2500) + _close_product_detail_overlays(page) + _wait_for_product_detail(page, timeout_ms=15_000) + _select_yesterday(page) + page.wait_for_timeout(3000) + return + except Exception: + if _looks_like_login_url(str(page.url)) or attempt == 1: + raise + print(" [WARN] 商品明细页首屏控件未出现，重新加载新版页面重试") + + raise RuntimeError("京东商品明细页加载失败") def _wait_for_product_detail(page, timeout_ms: int = 30_000) -> None: - """Wait for the real 商智商品明细 page and reject login redirects. + """Wait for the 商智商品明细 page and reject login redirects. JD may first load the 商智 URL and then asynchronously redirect an expired - session to ``passport.jd.com``. The old collector only checked the shop - home URL, so it proceeded with every style and reported a misleading - search-input timeout 34 times. Validate the final URL and the page's - search control once, before entering the style loop. + session to ``passport.jd.com``. Validate the final URL and the page's + multi-value search input once, before entering the style loop. """ deadline = time.monotonic() + max(timeout_ms, 1) / 1000 @@ -152,8 +250,8 @@ def _wait_for_product_detail(page, timeout_ms: int = 30_000) -> None: # only accept the expected Shangzhi hosts. if _is_shangzhi_url(last_url): try: - search = page.locator(".jmtd-multiple-input").first - if search.is_visible(timeout=800): + search = _locate_product_detail_search(page) + if search is not None: return except Exception: pass @@ -169,186 +267,147 @@ def _wait_for_product_detail(page, timeout_ms: int = 30_000) -> None: f"{last_url}；请先刷新 jd-shop 账号登录态后再运行" ) raise RuntimeError( - "京东商品明细页未加载完成，未找到 SPU 搜索框；" + "京东商品明细页未加载完成，未找到商品ID搜索框；" f"当前 URL: {last_url or ''}" ) -def close_guide_if_present(page): - """检测并关闭新功能引导弹窗""" - try: - page.locator('.jmtp-guide-content, .jmtp-guide-con').first.wait_for(state="visible", timeout=3000) - page.locator('.jmtp-guide-footer button', has_text="完成").first.click() - page.wait_for_timeout(2000) - except Exception: - pass +def goto_search_overview(page) -> None: + """跳转到搜索数据概览（商品分析 tab 默认展示）并等待 SKU 筛选框就绪。""" + _goto(page, SEARCH_OVERVIEW_URL) + page.wait_for_timeout(6000) + _wait_for_search_overview(page) -def enable_columns(page) -> None: - """点「指标配置」打开弹窗 -> 勾选「搜索曝光量」「搜索点击次数」-> 点「确定」。 - - 这两个指标默认不在表格里,需要在「指标配置」modal 里勾选才会显示成列。 - - 用 Playwright 原生 locator.check() 而不是 evaluate + label.click(), - 因为 React 应用里手动 click label 有时不触发 onChange 事件, - 导致视觉上 checked 但 React state 没更新,点确定后状态丢失。 - """ - # 点「指标配置」按钮打开 modal - clicked = page.evaluate(r"""() => { - const btns = Array.from(document.querySelectorAll('button')); - const target = btns.find((b) => (b.textContent || '').trim() === '指标配置' && b.offsetParent !== null); - if (target) { target.click(); return true; } - return false; - }""") - if not clicked: - print(" [WARN] 未找到「指标配置」按钮,跳过列启用") - return - # 等 modal 渲染完成 - try: - page.wait_for_selector('.indicators-config-modal:not(.jmtd-modal-hidden)', timeout=5000) - except Exception: - pass - page.wait_for_timeout(1500) - - # 找 modal 内 checkbox 的 input 元素并逐个用 Playwright 原生 check() 勾选 - targets = ["搜索曝光量", "搜索点击次数"] - toggled = 0 - for label_text in targets: - # 定位: 在 modal 内, .jmtd-checkbox-control 包含目标文案, 内部的 input[type=checkbox] - loc = page.locator( - f'.indicators-config-modal .jmtd-checkbox-control:has-text("{label_text}") input[type="checkbox"]' - ).first +def _wait_for_search_overview(page, timeout_ms: int = 30_000) -> None: + deadline = time.monotonic() + max(timeout_ms, 1) / 1000 + last_url = "" + while time.monotonic() < deadline: + try: + last_url = str(page.url) + except Exception: + last_url = "" + if _looks_like_login_url(last_url): + raise RuntimeError( + "京东商智登录态已失效：搜索数据概览跳转到登录页 " + f"{last_url}；请先刷新 jd-shop 账号登录态后再运行" + ) + try: + if page.locator("input.jmt-core-input").first.is_visible(timeout=800): + return + except Exception: + pass try: - loc.wait_for(state="attached", timeout=5000) - # 用 check() 而不是 click(),Playwright 会等 input 真的变成 checked 状态 - loc.check(force=True, timeout=5000) - toggled += 1 - print(f" [INFO] 勾选了 {label_text}") - except Exception as exc: - # check() 失败可能是 input 不可见或被遮挡,降级用 evaluate click label - print(f" [WARN] check {label_text} 失败: {exc}, 降级用 label click") - page.evaluate(r"""([text]) => { - const labels = Array.from(document.querySelectorAll( - '.indicators-config-modal .jmtd-checkbox-control, .jmtd-modal .jmtd-checkbox-control' - )).filter((el) => el.offsetParent !== null && (el.textContent || '').includes(text)); - if (labels.length) labels[0].click(); - }""", [label_text]) page.wait_for_timeout(500) - toggled += 1 - page.wait_for_timeout(1000) + except Exception: + time.sleep(0.5) + raise RuntimeError( + "京东搜索数据概览页未加载完成，未找到 SKU 筛选输入框；" + f"当前 URL: {last_url or ''}" + ) - # 验证勾选状态 (用 label class 判断真实视觉状态) - verified = page.evaluate(r"""() => { - const targetLabels = ['搜索曝光量', '搜索点击次数']; - const result = {}; - for (const target of targetLabels) { - const labels = Array.from(document.querySelectorAll( - '.indicators-config-modal .jmtd-checkbox-control, .jmtd-modal .jmtd-checkbox-control' - )).filter((el) => el.offsetParent !== null && (el.textContent || '').includes(target)); - if (!labels.length) { result[target] = 'not_found'; continue; } - result[target] = labels[0].classList.contains('jmtd-form-control-checked') ? 'checked' : 'unchecked'; - } - return result; - }""") - print(f" [INFO] 勾选状态: {verified}") - # 点「确定」应用 (modal 内 footer 的确定按钮) - confirmed = page.evaluate(r"""() => { - const btns = Array.from(document.querySelectorAll('.jmtd-modal-footer button, .jmtd-modal button, button')); - const target = btns.find((b) => (b.textContent || '').trim() === '确定' && b.offsetParent !== null); - if (target) { target.click(); return true; } - return false; - }""") - if not confirmed: - print(" [WARN] 未找到「确定」按钮,尝试按 Enter") - page.keyboard.press("Enter") - page.wait_for_timeout(3000) +def _select_yesterday(page) -> None: + """把日期维度显式切到「昨天」（页面默认即昨天，此步为幂等保险）。""" + try: + loc = page.get_by_text("昨天", exact=True).first + if loc.is_visible(timeout=2000): + loc.click(timeout=3000) + page.wait_for_timeout(1500) + except Exception: + pass + + +def _clear_multiple_tags(page) -> None: + """清空商品ID多值输入框里的已有标签（新版 jd-input 的清空图标）。""" + try: + clear = page.locator(".jd-input-clear-all").first + if clear.is_visible(timeout=1500): + clear.click(timeout=3000) + page.wait_for_timeout(800) + except Exception: + pass def search_by_spus(page, spus: list[str]) -> None: - """在多 SPU 搜索框输入 SPU 列表并触发查询。 - - 每输一个 SPU 按 Enter 形成一个 tag,全部输完后点查询按钮触发过滤。 - """ + """在商品明细页的「商品ID」多值输入框输入 SPU 列表并点查询。""" print(f" 搜索 SPU: {spus}") - wrapper = page.locator('.jmtd-multiple-input').first - wrapper.wait_for(state="visible", timeout=10000) + wrapper = _locate_product_detail_search(page) + if wrapper is None: + raise RuntimeError("京东商品明细页未找到商品ID搜索框") + _clear_multiple_tags(page) wrapper.click() page.wait_for_timeout(500) - # 清空旧搜索 tag - page.evaluate(r"""() => { - const closeBtns = Array.from(document.querySelectorAll('.jmtd-multiple-base-input-tag-wrapper .jmti-close')); - for (const btn of closeBtns) btn.click(); - }""") - page.wait_for_timeout(500) for spu in spus: - page.keyboard.type(spu, delay=80) - page.wait_for_timeout(400) + page.keyboard.type(spu, delay=60) + page.wait_for_timeout(300) page.keyboard.press("Enter") - page.wait_for_timeout(400) + page.wait_for_timeout(300) - # 点查询按钮触发过滤 (jmti-search icon button) - search_clicked = page.evaluate(r"""() => { - const btn = document.querySelector('.jmtd-input-search-button'); - if (btn) { btn.click(); return true; } - return false; - }""") - if not search_clicked: + try: + page.locator("button:visible", has_text="查询").first.click(timeout=5000) + except Exception: page.keyboard.press("Enter") - page.wait_for_timeout(5000) + page.wait_for_timeout(6000) -def expand_sku_rows(page) -> int: - """点所有 .jmti-plus-circle-f 展开图标,展开子 SKU 行。返回展开数。""" - count = page.evaluate(r"""() => { - const visible = (el) => { - if (!el) return false; - const style = window.getComputedStyle(el); - const rect = el.getBoundingClientRect(); - return style.display !== 'none' && style.visibility !== 'hidden' - && rect.width > 0 && rect.height > 0; - }; - const icons = Array.from(document.querySelectorAll( - '.jmti-plus-circle-f[data-event-name="expand"]' - )).filter(visible); - for (const icon of icons) icon.click(); - return icons.length; - }""") - print(f" 展开 {count} 个 SPU 父行") +def expand_all_spu_rows(page, max_rounds: int = 30) -> int: + """逐个原生点击 SPU 行的树形展开按钮（+ 号），展开全部 SKU 子行。 + + 展开后按钮仍在原行（图标切为 -），用已展开的 SPU 行文本做去重防止死循环。 + """ + expanded: set[str] = set() + for _ in range(max_rounds): + target_id = page.evaluate(r"""([done]) => { + const visible = (el) => { + if (!el) return false; + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + for (const btn of document.querySelectorAll('button.sz-table-tree-icon')) { + if (!visible(btn)) continue; + const row = btn.closest('tr'); + const idEl = row ? row.querySelector('.sxu-id-text') : null; + const m = idEl ? (idEl.textContent || '').match(/(\d{8,})/) : null; + if (m && !done.includes(m[1])) return m[1]; + } + return ''; + }""", [sorted(expanded)]) + if not target_id: + break + try: + page.locator("tr.jmt-base-table-tr", has_text=target_id).first.locator( + "button.sz-table-tree-icon" + ).first.click(timeout=5000) + except Exception as exc: + print(f" [WARN] 展开 SPU 行失败: {exc}") + break + expanded.add(target_id) + page.wait_for_timeout(1500) page.wait_for_timeout(2000) - return int(count or 0) + print(f" 展开 {len(expanded)} 个 SPU 父行") + return len(expanded) def upgrade_jd_image_url(url: str) -> str: """把 JD 360buyimg 的 URL 升级到大尺寸 (n12)。 - 表格里 img src 是小尺寸 (n5), 下载下来很糊 (6KB 缩略图)。 - 替换成 n12 拿清晰大图 (~1MB)。s800x800/s1000x1000 这张图不支持 (400), - 所以用 n12 兜底。 + 表格里 img src 是小尺寸 (n3/n5)，下载下来很糊。 + 替换成 n12 拿清晰大图；s800x800/s1000x1000 这张图不支持 (400)，所以用 n12 兜底。 """ return re.sub(r"//img\d+\.360buyimg\.com/n\d+/", "//img10.360buyimg.com/n12/", url or "") def extract_sku_rows(page) -> list[dict[str, Any]]: - """从展开后的表格 HTML 提取每个 SKU 子行的数据。 + """从商品明细页展开后的 SPU 树提取每个 SKU 子行的主图 URL。 - 返回 [{spu, sku_id, image_url, image_key, impressions, clicks}]。 - - 策略: - - 遍历 tbody 的 tr.jmtd-table-row - - 跳过合计行 (.jmtd-table-plugin-total-row) - - 跟踪 current_spu: 遇到带展开图标的行就是 SPU 父行, 更新 current_spu - - 无展开图标的行是 SKU 子行, 归到 current_spu - - SKU ID 从 a[href*="//item.jd.com/.html"] 提取 - - 指标值从对应列的 .first-value 提取 - - 列顺序 (实测): - 商品信息(0) / 访客数(1) / 浏览量(2) / 成交客户数(3) / 成交商品件数(4) / - 成交金额(5) / 搜索曝光量(6) / 搜索点击次数(7) / 操作(8) - - 但列顺序会随用户在「指标配置」里的勾选变化,所以用表头列名动态查找列索引, - 而不是硬编码 6/7。 + 返回 [{spu, sku_id, image_url}]。新版表格结构： + - 行 tr.jmt-base-table-tr；SPU 父行带 button.sz-table-tree-icon + - 行内 .sxu-id-text 文本为 "SPU: "（父行）或 "SKU: "（子行） + - 主图在行内第一个 img（360buyimg 域） """ rows = page.evaluate(r"""() => { const visible = (el) => { @@ -366,7 +425,7 @@ def extract_sku_rows(page) -> list[dict[str, Any]]: }; const imgUrl = (img) => { if (!img) return ''; - const attrs = ['currentSrc', 'src', 'data-src', 'data-lazy-src', 'data-original']; + const attrs = ['currentSrc', 'src', 'data-src']; for (const attr of attrs) { const v = attr === 'currentSrc' ? img.currentSrc : img.getAttribute(attr); const n = normalizeUrl(v); @@ -375,66 +434,24 @@ def extract_sku_rows(page) -> list[dict[str, Any]]: return ''; }; - // 找表头列名 -> 列索引 (跳过「操作」列右侧的固定列, 用 thead 的 th) - const headerCells = Array.from(document.querySelectorAll('thead th.jmtd-table-cell')) - .filter(visible); - const colNames = headerCells.map((th) => (th.textContent || '').replace(/\s+/g, '').trim()); - const idxExposure = colNames.findIndex((n) => n.includes('搜索曝光') || n.includes('曝光量')); - const idxClick = colNames.findIndex((n) => n.includes('搜索点击') || n.includes('点击次数') || n.includes('点击量')); - - // 找所有数据行 (跳过合计行) - const dataRows = Array.from(document.querySelectorAll('tbody tr.jmtd-table-row')) - .filter(visible) - .filter((r) => !r.classList.contains('jmtd-table-plugin-total-row')); - - let currentSpu = ''; const result = []; - for (const row of dataRows) { - // 检测是否是 SPU 父行 (有展开/收起图标) - const expandIcon = row.querySelector('.jmti-plus-circle-f, .jmti-minus-circle-f'); - const isParent = !!expandIcon; + let currentSpu = ''; + for (const row of document.querySelectorAll('tr.jmt-base-table-tr')) { + if (!visible(row)) continue; + const isParent = !!row.querySelector('button.sz-table-tree-icon'); + const idEl = row.querySelector('.sxu-id-text'); + const idText = idEl ? (idEl.textContent || '').trim() : ''; - const cells = Array.from(row.querySelectorAll('td.jmtd-table-cell')).filter(visible); - - // 提取 SPU ID (从 .jmtd-table-formatter-name-desc 的 "SPU：" 文本) - const spuDesc = row.querySelector('.jmtd-table-formatter-name-desc'); - if (spuDesc) { - const m = (spuDesc.textContent || '').match(/SPU[：:]\s*(\d+)/); + if (isParent) { + const m = idText.match(/SPU[：:]\s*(\d+)/); if (m) currentSpu = m[1]; + continue; } - - // 提取 SKU ID (从 a[href*="//item.jd.com/.html"]) - const skuLink = row.querySelector('a[href*="item.jd.com"]'); - let skuId = ''; - if (skuLink) { - const m = (skuLink.getAttribute('href') || '').match(/item\.jd\.com\/(\d+)\.html/); - if (m) skuId = m[1]; - } - - // 提取主图 URL (img.jmtd-table-formatter-name-img) - const img = row.querySelector('img.jmtd-table-formatter-name-img'); - const imageUrl = imgUrl(img); - - // 提取指标值 - const cellValue = (idx) => { - if (idx < 0 || idx >= cells.length) return ''; - const fv = cells[idx].querySelector('.first-value'); - return fv ? (fv.textContent || '').trim() : ''; - }; - - // SPU 父行也有一行数据 (代表该 SPU 汇总), 但需求是「取最大 SKU 主图」, - // 所以只取 SKU 子行 (isParent=false) 的数据, SPU 父行只用来更新 currentSpu - if (isParent) continue; - - if (!imageUrl) continue; // 没图的行跳过 (空行) - - result.push({ - spu: currentSpu, - sku_id: skuId, - image_url: imageUrl, - impressions_text: cellValue(idxExposure), - clicks_text: cellValue(idxClick), - }); + const m = idText.match(/SKU[：:]\s*(\d+)/); + if (!m) continue; + const image = imgUrl(row.querySelector('img')); + if (!image) continue; + result.push({spu: currentSpu, sku_id: m[1], image_url: image}); } return result; }""") @@ -442,27 +459,154 @@ def extract_sku_rows(page) -> list[dict[str, Any]]: return [] parsed: list[dict[str, Any]] = [] for row in rows: - image_url = row.get("image_url") or "" + image_url = upgrade_jd_image_url(row.get("image_url") or "") if not image_url: continue - image_url = upgrade_jd_image_url(image_url) parsed.append({ "spu": str(row.get("spu") or ""), "sku_id": str(row.get("sku_id") or ""), "image_url": image_url, "image_key": canonical_image_key(image_url), - "impressions": parse_number(row.get("impressions_text")), - "clicks": parse_number(row.get("clicks_text")), }) return parsed -def aggregate_by_spu(sku_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - """按 SPU 聚合: 每个 SPU 取曝光量最大的 SKU 的主图作为代表。""" +def _locate_sku_filter_input(page): + """定位搜索概览商品分析表的「请输入SKU进行查询」输入框。 + + 页面上有多个 jmt-core-input（另一个是「跳至页」输入框）， + 用「请输入SKU」文案的祖先链精确匹配，匹配不到再退回第一个。 + """ + idx = page.evaluate(r"""() => { + const inputs = Array.from(document.querySelectorAll('input.jmt-core-input')); + for (let i = 0; i < inputs.length; i++) { + let p = inputs[i].parentElement; + for (let k = 0; k < 8 && p; k++) { + if ((p.textContent || '').includes('请输入SKU')) return i; + p = p.parentElement; + } + } + return -1; + }""") + locator = page.locator("input.jmt-core-input") + if isinstance(idx, int) and idx >= 0: + return locator.nth(idx) + return locator.first + + +def _search_skus_on_overview(page, skus: list[str]) -> None: + """在商品分析表筛选框逐个输入 SKU（回车成 tag），自动触发过滤。""" + wrapper = _locate_sku_filter_input(page) + wrapper.wait_for(state="visible", timeout=15000) + wrapper.click() + page.wait_for_timeout(500) + for sku in skus: + page.keyboard.type(sku, delay=50) + page.wait_for_timeout(250) + page.keyboard.press("Enter") + page.wait_for_timeout(400) + page.wait_for_timeout(1500) + formed = page.evaluate(r"""([skus]) => { + const tags = Array.from(document.querySelectorAll('.jmt-tag-v2-main')) + .map((t) => (t.textContent || '').trim()); + return skus.filter((s) => tags.includes(s)).length; + }""", [skus]) + print(f" 已输入 {formed}/{len(skus)} 个 SKU 筛选标签") + if not formed: + raise RuntimeError("SKU 筛选标签未能生成，商品分析表过滤不可用") + + +def _extract_search_metrics(page) -> dict[str, dict[str, str]]: + """从商品分析表提取每个 SKU 的曝光次数/点击次数原始文本。 + + 列名从表头行动态匹配（曝光次数/点击次数），指标单元格取 + .jmt-table-render-summary-metrics（环比文案在 description 里，不取）。 + """ + data = page.evaluate(r"""() => { + const visible = (el) => { + if (!el) return false; + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + let idxExposure = -1; + let idxClick = -1; + for (const tr of document.querySelectorAll('tr')) { + const cells = Array.from(tr.children).filter(visible); + const names = cells.map((c) => (c.textContent || '').replace(/\s+/g, '').trim()); + if (!names.some((n) => n.includes('曝光次数'))) continue; + idxExposure = names.findIndex((n) => n.includes('曝光次数')); + idxClick = names.findIndex((n) => n.includes('点击次数')); + break; + } + const result = {}; + for (const tr of document.querySelectorAll('tr.jmt-table-row')) { + if (!visible(tr)) continue; + if (!tr.querySelector('img')) continue; + const skuEl = tr.querySelector('.jmt-table-render-name-text-label-value'); + const sku = skuEl ? (skuEl.textContent || '').replace(/\D/g, '') : ''; + if (!sku) continue; + const cells = Array.from(tr.querySelectorAll('td')).filter(visible); + const val = (idx) => { + if (idx < 0 || idx >= cells.length) return ''; + const m = cells[idx].querySelector('.jmt-table-render-summary-metrics'); + return m ? (m.textContent || '').trim() : ''; + }; + result[sku] = {impressions_text: val(idxExposure), clicks_text: val(idxClick)}; + } + return result; + }""") + if not isinstance(data, dict): + return {} + return {str(k): dict(v) for k, v in data.items() if isinstance(v, dict)} + + +def collect_sku_search_metrics(context, skus: list[str], + chunk_size: int = OVERVIEW_SKU_CHUNK_SIZE) -> dict[str, dict[str, str]]: + """在搜索数据概览按 SKU 分批查询曝光/点击。 + + 每批新开独立标签页（共享登录态），规避长驻标签页在重页面间跳转时崩溃， + 同时天然隔离上一批的筛选标签。 + """ + metrics: dict[str, dict[str, str]] = {} + for start in range(0, len(skus), chunk_size): + chunk = skus[start:start + chunk_size] + print(f" 搜索概览查询 SKU {start + 1}-{start + len(chunk)}/{len(skus)}") + page = context.new_page() + try: + goto_search_overview(page) + _select_yesterday(page) + _search_skus_on_overview(page, chunk) + page.wait_for_timeout(6000) + found = _extract_search_metrics(page) + finally: + try: + page.close() + except Exception: + pass + metrics.update(found) + print(f" [INFO] 本批 {len(found)}/{len(chunk)} 个 SKU 有搜索数据") + return metrics + + +def aggregate_by_spu( + sku_rows: list[dict[str, Any]], + metrics: dict[str, dict[str, str]], +) -> list[dict[str, Any]]: + """按 SPU 聚合: 每个 SPU 取曝光量最大的 SKU 的主图作为代表。 + + 搜索概览没有数据的 SKU（无曝光不出现在结果里）按 0 计。 + """ by_spu: dict[str, list[dict[str, Any]]] = {} for row in sku_rows: - spu = row.get("spu") or "unknown" - by_spu.setdefault(spu, []).append(row) + metric = metrics.get(row["sku_id"], {}) + enriched = { + **row, + "impressions": parse_number(metric.get("impressions_text")), + "clicks": parse_number(metric.get("clicks_text")), + } + by_spu.setdefault(row.get("spu") or "unknown", []).append(enriched) result: list[dict[str, Any]] = [] for spu, rows in by_spu.items(): @@ -511,7 +655,7 @@ def write_style_outputs(result: dict[str, Any], output_dir: Path) -> tuple[Path, f"- 原始 SKU 数: {result.get('raw_sku_count', 0)}", "", "| SPU | SKU ID | 主图 URL | 本地图片 | 搜索曝光量 | 搜索点击次数 | SKU 数 |", - "| --- | --- | --- | --- | ---: | ---: | ---: |", + "| --- | --- | --- | ---: | ---: | ---: | ---: |", ] for rec in result.get("records", []): lines.append( @@ -523,22 +667,68 @@ def write_style_outputs(result: dict[str, Any], output_dir: Path) -> tuple[Path, return json_path, md_path -def collect_style(page, style_name: str, spus: list[str], output_dir: Path) -> dict[str, Any]: - """采集单款: 搜索 -> 展开 -> 解析 -> 聚合 -> 下载 -> 写 JSON。""" +def _load_reusable_style_output(output_dir: Path, style_name: str) -> dict[str, Any] | None: + """Load an audited same-date result when JD temporarily has no SKU rows.""" + path = output_dir / now_label() / safe_filename(style_name) / "jd_main_images.json" + if not path.exists(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, TypeError, ValueError): + return None + records = payload.get("records") + if not isinstance(records, list) or not records: + return None + for record in records: + if not isinstance(record, dict) or not record.get("image_key"): + return None + local_path = record.get("local_image_path") + if not local_path or not Path(local_path).exists(): + return None + return payload + + +def _ensure_page(context, page): + """返回可用页面；原页面已崩溃/关闭时在新标签页重建。""" + try: + if not page.is_closed(): + return page + except Exception: + pass + return context.new_page() + + +def collect_style(context, page, style_name: str, spus: list[str], output_dir: Path) -> dict[str, Any]: + """采集单款: 明细页取 SPU 树与主图 -> 搜索概览取曝光/点击 -> 聚合下载写 JSON。""" if not spus: raise RuntimeError(f"款 {style_name} 无 SPU 配置") + page = _ensure_page(context, page) + step_goto_product_detail(page) search_by_spus(page, spus) - expand_sku_rows(page) sku_rows = extract_sku_rows(page) - print(f" 解析到 {len(sku_rows)} 个 SKU 行") - if not sku_rows and DEBUG: - snapshot_path = DEBUG_DIR / f"{safe_filename(style_name)}_no_sku_{int(time.time())}.html" - snapshot_path.parent.mkdir(parents=True, exist_ok=True) - snapshot_path.write_text(page.content(), encoding="utf-8") - print(f" [DEBUG] snapshot: {snapshot_path}") + for attempt in range(3): + if sku_rows: + break + # 偶发：表格/树形子行渲染滞后。等一等并重试展开（已展开的 SPU 会被跳过）。 + print(f" [WARN] 第 {attempt + 1} 次未解析到 SKU 行，等待后重试展开") + page.wait_for_timeout(4000) + expand_all_spu_rows(page) + sku_rows = extract_sku_rows(page) + print(f" 明细树解析到 {len(sku_rows)} 个 SKU 行") + if not sku_rows: + if DEBUG: + snapshot_path = DEBUG_DIR / f"{safe_filename(style_name)}_no_sku_{int(time.time())}.html" + snapshot_path.parent.mkdir(parents=True, exist_ok=True) + snapshot_path.write_text(page.content(), encoding="utf-8") + print(f" [DEBUG] snapshot: {snapshot_path}") + raise RuntimeError(f"款 {style_name} 明细页未解析到 SKU 行") - records = aggregate_by_spu(sku_rows) + metrics = collect_sku_search_metrics(context, [r["sku_id"] for r in sku_rows]) + matched = sum(1 for r in sku_rows if r["sku_id"] in metrics) + print(f" 搜索概览匹配到曝光/点击的 SKU: {matched}/{len(sku_rows)}") + + records = aggregate_by_spu(sku_rows, metrics) print(f" 聚合到 {len(records)} 个 SPU 记录") style_dir = output_dir / now_label() / safe_filename(style_name) @@ -598,31 +788,72 @@ def run(args: argparse.Namespace) -> None: if "shop.jd.com" not in page.url or _looks_like_login_url(page.url): raise RuntimeError(f"京东登录未完成，当前 URL: {page.url}") - # Step 2-3: 商智 -> 旧版 -> 商品明细页 + # Step 2-3: 商智 -> 新版页面 sz_page = step_enter_shangzhi(context, page) step_return_old_version(sz_page) - step_goto_product_detail(sz_page) - close_guide_if_present(sz_page) - # Step 4: 启用「搜索曝光量」「搜索点击次数」两列 - enable_columns(sz_page) - - # Step 5-7: 逐款采集 + # Step 4-7: 逐款采集（明细页 SPU 树 + 搜索概览曝光/点击） results: list[dict] = [] for index, group in enumerate(styles, 1): style_name = group["style"] spus = group.get("spus") or [] print(f"\n[{index}/{len(styles)}] 采集: {style_name} (SPUs: {len(spus)})") - try: - result = collect_style(sz_page, style_name, spus, output_dir) - results.append(result) - except Exception as exc: - print(f" [FAIL] {style_name}: {exc}") + last_error: Exception | None = None + for style_attempt in range(1, 3): + try: + page = _ensure_page(context, page) + result = collect_style(context, page, style_name, spus, output_dir) + results.append(result) + last_error = None + break + except Exception as exc: + last_error = exc + if style_attempt < 2: + print( + f" [WARN] {style_name} 第 {style_attempt} 次采集失败，" + f"重建页面后重试: {exc}" + ) + try: + if not page.is_closed(): + page.close() + except Exception: + pass + page = context.new_page() + page.wait_for_timeout(1000) + elif "未解析到 SKU 行" in str(exc): + reusable = _load_reusable_style_output(output_dir, style_name) + if reusable is not None: + reusable["reused_existing"] = True + reusable["reuse_reason"] = str(exc) + reusable["reused_at"] = datetime.now().isoformat(timespec="seconds") + results.append(reusable) + last_error = None + print( + f" [WARN] {style_name} 当前未解析到 SKU，" + "复用同日期已审计且图片仍存在的结果" + ) + elif style_name in _EXPECTED_EMPTY_JD_STYLES: + skipped = { + "style_name": style_name, + "platform": "jd", + "collected_at": datetime.now().isoformat(timespec="seconds"), + "spu_count": 0, + "raw_sku_count": 0, + "skipped": True, + "skip_reason": "京东暂无该款商品记录", + "records": [], + } + write_style_outputs(skipped, output_dir) + results.append(skipped) + last_error = None + print(f" [SKIP] {style_name}: 京东暂无商品记录，跳过该款并继续") + if last_error is not None: + print(f" [FAIL] {style_name}: {last_error}") results.append({ "style_name": style_name, "platform": "jd", "collected_at": datetime.now().isoformat(timespec="seconds"), - "error": str(exc), + "error": str(last_error), "records": [], }) @@ -635,13 +866,24 @@ def run(args: argparse.Namespace) -> None: }, ensure_ascii=False, indent=2), encoding="utf-8") print(f"\n[OK] 全部款式汇总: {overall_path}") + failures = [result for result in results if result.get("error")] + if failures: + summary = "; ".join( + f"{result['style_name']}: {result['error'].splitlines()[0]}" + for result in failures + ) + raise RuntimeError( + f"京东主图采集有 {len(failures)}/{len(results)} 款失败，" + f"拒绝将旧原始文件当作本次成功数据: {summary}" + ) + if args.keep_open: print("[INFO] --keep-open 已开启,浏览器保留 60 秒") time.sleep(60) def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="采集京东商品明细页主图数据") + parser = argparse.ArgumentParser(description="采集京东商智商品主图与搜索曝光/点击数据") parser.add_argument("--shop", default=DEFAULT_CONFIG["shop"], help="京东店铺名") parser.add_argument("--password", default=DEFAULT_CONFIG["password"], help="京东密码") parser.add_argument("--output", default=str(OUTPUT_DIR), help="输出目录") diff --git a/src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py b/src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py index f825303..7c870de 100644 --- a/src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py +++ b/src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py @@ -21,6 +21,7 @@ import math import os import re import sys +import time from datetime import datetime, timedelta from runtime_paths import ( @@ -141,8 +142,9 @@ def step_goto_product_detail(page): """跳转到商品明细页面""" print("[Step 4] 跳转到商品明细页面...") page.goto( - "https://sz.jd.com/szweb/sz/view/productAnalysis/productDetail.html", - wait_until="networkidle" + "https://jdsz.jd.com/szweb/view/product/productDetail.html", + wait_until="domcontentloaded", + timeout=60000, ) page.wait_for_timeout(5000) print(" [OK] 已到达商品明细页面") @@ -760,10 +762,51 @@ def select_yesterday_in_combo_datepicker(page, label="product_date"): target_day = target.strftime("%Y-%m-%d") print(f" 设置日期为昨天: {target_day}") + # 商智新版使用 jmt-* 日期控件，默认选中“实时”；点击“昨天”会直接 + # 将当前日期切到目标日。只对实际昨天走这个快捷入口，历史回采仍保留 + # 下面的旧版翻日历/自定义范围逻辑。 + if ( + target.date() == (datetime.now() - timedelta(days=1)).date() + and "jdsz.jd.com" in page.url.lower() + ): + new_version_yesterday = page.locator( + 'li[data-event-content="当前时间_昨天"]' + ).first + try: + new_version_yesterday.wait_for(state="visible", timeout=15000) + new_version_yesterday.click(timeout=10000) + page.wait_for_timeout(2500) + # 选择相对日期后，新版会把“天”日历浮层留在前台；再次点击 + # 当前粒度“天”确认日维度并收起浮层，否则会拦截下载按钮。 + try: + page.locator( + 'li.jmt-combo-date-picker-list-tag[data-event-content="当前时间_天"]' + ).first.click(timeout=5000) + page.wait_for_timeout(500) + except Exception: + pass + selected = page.locator( + 'li.jmt-combo-date-picker-list-tag.jmt-combo-date-picker-list-tag-select' + '[data-event-content="当前时间_昨天"]' + ).count() + current_marker = page.locator("body").inner_text(timeout=5000) + if selected or f"当前：{target_day}" in current_marker: + print(f" [OK] 新版日期控件已切换到昨天: {target_day}") + return True + except Exception as exc: + raise RuntimeError(f"新版日期控件未能切换到昨天: {target_day}") from exc + if _date_text_matches_day(_combo_date_label(page), target_day): print(f" [OK] 日期已是昨天: {_combo_date_label(page)}") return True + # 新版商智 (旧版已下线): 历史日期走「自定义」+ React 单日区间, 旧版 + # 翻日/日历/范围策略在新版 DOM 上全部失效, 优先尝试新版路径. + if _jmt_new_version_present(page): + if _select_custom_date_new_version(page, target): + return True + print(" [WARN] 新版自定义日期路径失败, 回退旧版策略继续尝试") + _click_combo_mode(page, "day") page.wait_for_timeout(1200) @@ -885,6 +928,109 @@ def _select_via_range_input(page, target_day: str) -> bool: return _date_text_matches_day(_combo_date_label(page), target_day) +_JMT_CUSTOM_REACT_CLICK = r"""(args) => { + const [ty, tm, targetDay] = args; + const pad = (n) => String(n).padStart(2, '0'); + const ymRe = new RegExp('^' + ty + '\\u5e74' + pad(tm) + '\\u6708$'); + let panel = null; + for (const el of document.querySelectorAll('*')) { + const t = (el.textContent || '').trim(); + if (!ymRe.test(t)) continue; + panel = el.closest('[class*="calendar"]') || el.parentElement.parentElement || el.parentElement; + break; + } + if (!panel) return {status: 'no_panel'}; + for (const cell of panel.querySelectorAll('td, [class*="cell"]')) { + if ((cell.textContent || '').trim() !== targetDay) continue; + if (/disabled|off/i.test(String(cell.className))) continue; + const nodes = [cell, ...cell.querySelectorAll('*'), cell.parentElement]; + for (const node of nodes) { + const propsKey = Object.keys(node).find((k) => k.startsWith('__reactProps$')); + if (!propsKey) continue; + const props = node[propsKey]; + if (props && typeof props.onClick === 'function') { + props.onClick({preventDefault(){}, stopPropagation(){}, + target: node, currentTarget: node, + nativeEvent: new MouseEvent('click')}); + return {status: 'ok'}; + } + } + return {status: 'no_handler'}; + } + return {status: 'no_cell'}; +}""" + + +def _jmt_new_version_present(page) -> bool: + """新版商智 (jmtv2) 日期工具条是否存在.""" + try: + return page.locator("li.jmt-combo-date-picker-list-tag").count() > 0 + except Exception: + return False + + +def _jmt_react_click_calendar_cell(page, target_date) -> bool: + """在新版自定义日历里直触 React onClick 选中目标日. + + 真实鼠标点击会被商智页面拦截 (日历 cell 高亮但 handler 不触发), 必须通过 + fiber 上的 __reactProps$.onClick 直接调用; 两次调用同一单元格构成单日区间. + """ + result = page.evaluate( + _JMT_CUSTOM_REACT_CLICK, + [target_date.year, target_date.month, str(target_date.day)], + ) + return isinstance(result, dict) and result.get("status") == "ok" + + +def _jmt_custom_date_state(page): + try: + return page.evaluate( + r"""() => { + const sel = document.querySelector('li.jmt-combo-date-picker-list-tag-select'); + const el = document.querySelector('.jmt-combo-date-picker-echo-wrap'); + return { + selectedTag: sel ? sel.getAttribute('data-event-content') : null, + echo: el ? (el.innerText || '').trim() : null, + }; + }""" + ) + except Exception: + return {} + + +def _select_custom_date_new_version(page, target) -> bool: + """新版商智历史日期选择: 「自定义」+ React 点击目标日两次构成单日区间. + + 京东已下线「返回旧版」(ppzh/sz 域名 26 年底下线, 9 月起逐步关闭旧版权限), + 旧版翻日/日历/范围策略在新版页面上全部失效. 新版路径: + 点工具条「自定义」→ 在双月日历里对目标日连续触发两次 React onClick + (起止同日 = 单日区间) → echo 显示目标日期即成功. + """ + try: + tag = page.get_by_text("自定义", exact=True).first + tag.wait_for(state="visible", timeout=8000) + tag.click(timeout=8000) + except Exception as exc: + print(f" [WARN] 新版「自定义」入口点击失败: {exc}") + return False + page.wait_for_timeout(1200) + + for _ in range(2): + if not _jmt_react_click_calendar_cell(page, target): + print(" [WARN] 新版日历 React 点击目标日失败") + return False + page.wait_for_timeout(600) + page.wait_for_timeout(1500) + + state = _jmt_custom_date_state(page) + target_text = target.strftime("%Y-%m-%d") + if state.get("selectedTag") == "当前时间_自定义" and target_text in (state.get("echo") or ""): + print(f" [OK] 新版自定义日期已切换到: {target_text}") + return True + print(f" [WARN] 新版自定义日期未生效: {json.dumps(state, ensure_ascii=False)}") + return False + + def select_week_and_last_week(page): """选择天模式并切换到昨天。函数名保留，避免改动调用链。""" print(" 选择时间范围为'天'并切换到昨天...") @@ -3831,6 +3977,229 @@ def _hover_download_text(page, texts, timeout=10000): ) +_JD_DOWNLOAD_CENTER_URL = ( + "https://jdsz.jd.com/szweb/view/reports-center/download-center.html" +) + + +def _jd_download_task_fragment(date_stamp): + """返回新版商品明细下载任务的稳定名称片段。""" + + return f"商品明细_离线_包括对比时间_分天下载_{date_stamp}_{date_stamp}" + + +def _find_jd_download_task_row(page, date_stamp): + """在京东新版下载中心找目标日期的任务行。""" + + fragment = _jd_download_task_fragment(date_stamp) + try: + search = page.locator('input[placeholder*="下载文件名称"]').first + if search.count() and search.is_visible(): + search.fill(fragment) + page.wait_for_timeout(500) + except Exception: + # 搜索框只是缩小列表范围，失败时仍尝试从当前列表匹配，避免把 + # 页面轻微改版误判为任务不存在。 + pass + + rows = page.locator("tr").filter(has_text=fragment) + for index in range(rows.count()): + row = rows.nth(index) + try: + row_text = row.inner_text(timeout=3000) + except Exception: + continue + if "已生成" in row_text or "生成中" in row_text or "生成失败" in row_text: + return row, row_text + return None, "" + + +def _wait_for_jd_download_task(page, date_stamp, timeout_seconds=180): + """等待新版商品明细异步任务生成并返回任务行。""" + + deadline = time.monotonic() + timeout_seconds + last_state = "" + while time.monotonic() < deadline: + row, row_text = _find_jd_download_task_row(page, date_stamp) + if row is not None: + last_state = row_text.replace("\n", " ") + if "已生成" in row_text: + print(f" [OK] 京东下载中心任务已生成: {last_state}") + return row + if "生成失败" in row_text: + raise RuntimeError(f"京东商品明细异步任务生成失败: {last_state}") + + if time.monotonic() >= deadline: + break + try: + refresh = page.get_by_role("button", name="刷新", exact=True).first + if refresh.count() and refresh.is_visible(): + refresh.click(timeout=10000) + else: + page.reload(wait_until="domcontentloaded", timeout=60000) + page.wait_for_timeout(2500) + except Exception as exc: + last_state = f"刷新任务中心失败: {exc}" + page.wait_for_timeout(2500) + + detail = f"，最近状态: {last_state}" if last_state else "" + raise TimeoutError( + f"京东商品明细异步下载任务等待超时（{timeout_seconds}s）{detail}" + ) + + +def _download_jd_center_task(page, row, output_dir, date_stamp): + """从新版下载中心取出已生成任务对应的 xlsx 文件。""" + + responses = [] + + def on_response(response): + if "getFileLink.ajax" in response.url: + responses.append(response) + + page.on("response", on_response) + try: + download_button = row.get_by_role("button", name="下载", exact=True).first + download_button.wait_for(state="visible", timeout=15000) + download_button.click(timeout=15000) + deadline = time.monotonic() + 15000 / 1000 + while not responses and time.monotonic() < deadline: + page.wait_for_timeout(200) + if not responses: + raise RuntimeError("京东下载中心未返回文件地址") + + response = responses[-1] + if response.status != 200: + raise RuntimeError( + f"京东下载中心获取文件地址失败: HTTP {response.status}" + ) + payload = response.json() + file_items = ((payload.get("body") or {}).get("data") or []) + file_url = file_items[0].get("fileUrl") if file_items else "" + if not file_url: + raise RuntimeError("京东下载中心返回空文件地址") + if file_url.startswith("//"): + file_url = f"https:{file_url}" + + download_page = page.context.new_page() + try: + with download_page.expect_download(timeout=60000) as download_info: + try: + download_page.goto( + file_url, + wait_until="commit", + timeout=60000, + ) + except Exception as exc: + # 下载开始时 Playwright 会以 ERR_ABORTED 结束导航， + # expect_download 仍会正确交付 Download 对象。 + if "Download is starting" not in str(exc): + raise + download = download_info.value + suggested = safe_filename( + download.suggested_filename + or f"jd_product_detail_{date_stamp}.xlsx", + 120, + ) + path = os.path.join(output_dir, suggested) + download.save_as(path) + return path + finally: + try: + download_page.close() + except Exception: + pass + finally: + try: + page.remove_listener("response", on_response) + except Exception: + pass + + +def _download_jd_new_version(page, download_dir, date_stamp): + """处理商智新版：复用/创建异步任务后从下载中心取文件。""" + + center_page = page.context.new_page() + try: + center_page.goto( + _JD_DOWNLOAD_CENTER_URL, + wait_until="domcontentloaded", + timeout=60000, + ) + center_page.wait_for_timeout(5000) + task_row, task_text = _find_jd_download_task_row(center_page, date_stamp) + if task_row is not None and "已生成" in task_text: + print(" [INFO] 复用京东下载中心已有同日期任务") + return _download_jd_center_task( + center_page, + task_row, + download_dir, + date_stamp, + ) + if task_row is not None and "生成中" in task_text: + print(" [INFO] 复用京东下载中心正在生成的同日期任务") + task_row = _wait_for_jd_download_task(center_page, date_stamp) + return _download_jd_center_task( + center_page, + task_row, + download_dir, + date_stamp, + ) + if task_row is not None and "生成失败" in task_text: + raise RuntimeError(f"京东已有商品明细任务生成失败: {task_text}") + finally: + try: + center_page.close() + except Exception: + pass + + # 没有同配置的可复用任务，回到商品明细页创建一个新版异步任务。 + download_button = page.locator('button[data-event-content="下载数据"]').first + download_button.wait_for(state="visible", timeout=10000) + # 日期选择器在新版会残留前台日历浮层，直接调用按钮的 DOM click + # 触发 React 事件，避免浮层遮挡导致普通点击无效。 + download_button.evaluate("(element) => element.click()") + modal = page.locator("div.jd-modal-wrap:visible").last + modal.wait_for(state="visible", timeout=10000) + + for selector in ( + 'label[data-event-content="选中分天下载"]', + 'label[data-event-content="选中包含对比时间"]', + ): + option = modal.locator(selector).first + if option.count(): + option.click(timeout=10000) + + confirm = modal.locator('button[data-event-content="确定"]').first + if not confirm.count(): + confirm = modal.locator("button", has_text="确定").first + confirm.wait_for(state="visible", timeout=10000) + confirm.click(timeout=10000) + page.wait_for_timeout(2500) + print(" [OK] 新版商品明细异步下载任务已提交，等待下载中心生成") + + center_page = page.context.new_page() + try: + center_page.goto( + _JD_DOWNLOAD_CENTER_URL, + wait_until="domcontentloaded", + timeout=60000, + ) + center_page.wait_for_timeout(3000) + task_row = _wait_for_jd_download_task(center_page, date_stamp) + return _download_jd_center_task( + center_page, + task_row, + download_dir, + date_stamp, + ) + finally: + try: + center_page.close() + except Exception: + pass + + def _download_jd_current_detail(page, output_base): date_stamp = _today() download_dir = os.path.join(output_base, "_downloads", date_stamp) @@ -3840,33 +4209,50 @@ def _download_jd_current_detail(page, output_base): page.keyboard.press("Escape") except Exception: pass - with page.expect_download(timeout=120000) as download_info: - # 1. 点击「下载数据」打开下拉 - if not _click_download_text(page, ["下载数据"]): - raise RuntimeError("未找到京东「下载数据」按钮") - page.wait_for_timeout(1000) - # 2. 悬浮在「分天下载」上展开子菜单 - if not _hover_download_text(page, ["分天下载"]): - raise RuntimeError("未找到京东「分天下载」菜单") - page.wait_for_timeout(1500) - # 3. 点击子菜单中的「包括对比数据」触发下载 - if not _click_download_text(page, ["包括对比数据"]): - raise RuntimeError("未找到京东「包括对比数据」子菜单项") - download = download_info.value - suggested = safe_filename(download.suggested_filename or f"jd_product_detail_{date_stamp}.xlsx", 120) - path = os.path.join(download_dir, suggested) - download.save_as(path) + if "jdsz.jd.com" in page.url.lower(): + path = _download_jd_new_version(page, download_dir, date_stamp) + else: + with page.expect_download(timeout=120000) as download_info: + # 旧版仍使用「下载数据 → 分天下载 → 包括对比数据」。 + if not _click_download_text(page, ["下载数据"]): + raise RuntimeError("未找到京东「下载数据」按钮") + page.wait_for_timeout(1000) + if not _hover_download_text(page, ["分天下载"]): + raise RuntimeError("未找到京东「分天下载」菜单") + page.wait_for_timeout(1500) + if not _click_download_text(page, ["包括对比数据"]): + raise RuntimeError("未找到京东「包括对比数据」子菜单项") + download = download_info.value + suggested = safe_filename( + download.suggested_filename + or f"jd_product_detail_{date_stamp}.xlsx", + 120, + ) + path = os.path.join(download_dir, suggested) + download.save_as(path) print(f" [OK] 京东明细已下载: {path}") return path +def _prepare_jd_download_retry(page): + """重试下载时清掉新版浮层，旧版则重新打开下载菜单。""" + + try: + page.keyboard.press("Escape") + except Exception: + pass + if "jdsz.jd.com" in page.url.lower(): + return + _click_download_text(page, ["下载数据"]) + + def _download_jd_with_retry(page, output_base, failed: FailedCollector | None = None) -> str | None: """per-page 重试包装: 京东明细下载失败 → 重新打开下拉 + 重试 1 次, 仍失败记 failed。""" def _do_download(): return _download_jd_current_detail(page, output_base) try: return retry_step(_do_download, max_attempts=2, sleep_seconds=3, - label="JD 明细下载", on_retry=lambda *_: _click_download_text(page, ["下载数据"])) + label="JD 明细下载", on_retry=lambda *_: _prepare_jd_download_retry(page)) except Exception as exc: if failed: failed.add_page("JD 明细下载", f"{type(exc).__name__}: {exc}", attempts=2) @@ -3897,7 +4283,7 @@ def _jd_product_metrics_from_download(path, output_base): total_visitors += _download_metric_number(_value_by_header_alias(row, ["访客数", "商品访客数", "访客人数"])) total_cart += _download_metric_number(_value_by_header_alias(row, ["加购客户数", "加购人数", "商品加购人数", "加购用户数"])) total_sales_platform += _download_metric_number(_value_by_header_alias(row, ["成交单量", "成交订单", "下单单量"])) - total_refunds_platform += _download_metric_number(_value_by_header_alias(row, ["申请退货件数", "退货件数"])) + total_refunds_platform += _download_metric_number(_value_by_header_alias(row, ["申请退货件数", "退货件数", "取消及售后退款商品件数", "取消及售后退款单量"])) erp_metrics = apply_erp_override( "jd", style, total_visitors, total_cart, diff --git a/src/gyxx_flow/modules/product_commerce/jd_video_metadata.py b/src/gyxx_flow/modules/product_commerce/jd_video_metadata.py index f6f5873..2166c55 100644 --- a/src/gyxx_flow/modules/product_commerce/jd_video_metadata.py +++ b/src/gyxx_flow/modules/product_commerce/jd_video_metadata.py @@ -975,7 +975,10 @@ class JdVideoTitleGenerator: verified_text_for_validation = verified_text_for_validation.replace( "双肩背包", "双肩包" ) - history = tuple((*self._used_titles, *recent_titles)) + # Keep local validation aligned with the same 50-title window shown to + # the model below. Rejecting against older, unseen titles made a valid + # fallback impossible when the catalog had many historical uploads. + history = tuple((*self._used_titles, *recent_titles)[-50:]) generated_titles: list[str] = [] try: product_context = "；".join(catalog_titles[:4]) or "未提供商品标题" diff --git a/src/gyxx_flow/modules/product_commerce/market_rank_product_import.py b/src/gyxx_flow/modules/product_commerce/market_rank_product_import.py index e6cb549..753d787 100644 --- a/src/gyxx_flow/modules/product_commerce/market_rank_product_import.py +++ b/src/gyxx_flow/modules/product_commerce/market_rank_product_import.py @@ -1,6 +1,7 @@ """将三平台市场榜单导入品牌行军图数据库。 -完整商品以采集 JSON 为准；飞书原生表格只覆盖已有人工风格、场景和天猫客单价。 +采集 JSON 是榜单事实来源；商品主数据上的人工风格、场景和品类跨月份复用。 +历史飞书文档导入函数保留，用于兼容存量数据迁移。 """ from __future__ import annotations @@ -163,6 +164,31 @@ def build_product_snapshot( } +def filter_rows_by_rank( + platform: str, + rows: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + """仅保留各平台、各品类允许入库的榜单排名。""" + if __package__: + from .market_rank_limits import category_rank_limit + else: + from market_rank_limits import category_rank_limit + + filtered: list[dict[str, Any]] = [] + for source_row in rows: + row = dict(source_row) + category_name = str(row.get("category_name") or "").strip() + try: + rank = int(row.get("rank")) + except (TypeError, ValueError): + continue + if rank < 1 or rank > category_rank_limit(platform, category_name): + continue + row["rank"] = rank + filtered.append(row) + return filtered + + def _text(element: ET.Element | None) -> str: if element is None: return "" @@ -183,6 +209,7 @@ def extract_document_labels(content: str, platform: str) -> dict[str, dict[str, headers = [_text(cell) for cell in table.findall("./thead/tr/th")] style_index = _header_index(headers, "风格（运营手动区分）") scene_index = _header_index(headers, "场景用途（运营手动区分）") + category_index = _header_index(headers, "品类", "类目") price_index = _header_index(headers, "客单价") if price_index is None: price_index = next( @@ -196,12 +223,17 @@ def extract_document_labels(content: str, platform: str) -> dict[str, dict[str, product_id = extract_product_id(platform, {"product_url": href}) raw_style = _text(cells[style_index]) if style_index is not None and style_index < len(cells) else "" raw_scene = _text(cells[scene_index]) if scene_index is not None and scene_index < len(cells) else "" + category_name = ( + _text(cells[category_index]) + if category_index is not None and category_index < len(cells) + else "" + ) unit_price_text = ( _text(cells[price_index]) if price_index is not None and price_index < len(cells) else "" ) - if not raw_style and not raw_scene and not unit_price_text: + if not raw_style and not raw_scene and not category_name and not unit_price_text: continue current = result.setdefault(product_id, {}) if raw_style: @@ -210,6 +242,8 @@ def extract_document_labels(content: str, platform: str) -> dict[str, dict[str, if raw_scene: current["raw_scene_name"] = raw_scene current["scene_name"] = normalize_scene_name(raw_scene) + if category_name: + current["category_name"] = category_name if unit_price_text and re.search(r"\d", unit_price_text): current["unit_price_text"] = unit_price_text return result @@ -274,6 +308,48 @@ def _execute_product_upsert(conn, platform: str, report_date: date, snapshots: S return {str(product_id): int(row_id) for row_id, product_id in cur.fetchall()} +def reuse_saved_categories( + snapshots: Sequence[Mapping[str, Any]], + saved_categories: Mapping[str, Any], +) -> tuple[list[dict[str, Any]], int]: + """复用已人工填写的品类，同时保留采集原始类目在 raw_data 中。""" + result: list[dict[str, Any]] = [] + reused = 0 + for source in snapshots: + item = dict(source) + product_id = str(item.get("platform_product_id") or "").strip() + saved = str(saved_categories.get(product_id) or "").strip() + if saved: + item["category_name"] = saved + reused += 1 + result.append(item) + return result, reused + + +def _load_saved_categories( + conn, + product_ids: Mapping[str, int], +) -> dict[str, str]: + if not product_ids: + return {} + inverse = {int(market_id): product_id for product_id, market_id in product_ids.items()} + with conn.cursor() as cur: + cur.execute( + """ + SELECT market_product_id, category_name + FROM market_product_classification + WHERE market_product_id = ANY(%s) + AND NULLIF(BTRIM(category_name), '') IS NOT NULL + """, + (list(inverse),), + ) + return { + inverse[int(market_product_id)]: str(category_name).strip() + for market_product_id, category_name in cur.fetchall() + if category_name is not None and str(category_name).strip() + } + + def archive_product_snapshots( conn, *, @@ -286,6 +362,10 @@ def archive_product_snapshots( from psycopg2.extras import execute_values product_ids = _execute_product_upsert(conn, platform, report_date, snapshots) + snapshots, category_reused = reuse_saved_categories( + snapshots, + _load_saved_categories(conn, product_ids), + ) snapshot_rows = [ ( report_id, @@ -345,7 +425,8 @@ def archive_product_snapshots( continue raw_style = str(label.get("raw_style_name") or "").strip() or None raw_scene = str(label.get("raw_scene_name") or "").strip() or None - if not raw_style and not raw_scene: + category_name = str(label.get("category_name") or "").strip() or None + if not raw_style and not raw_scene and not category_name: continue classification_rows.append( ( @@ -354,6 +435,7 @@ def archive_product_snapshots( scene_ids.get(label.get("scene_name")), raw_style, raw_scene, + category_name, "feishu_import", ) ) @@ -361,13 +443,14 @@ def archive_product_snapshots( classification_sql = """ INSERT INTO market_product_classification (market_product_id, style_id, scene_id, raw_style_name, - raw_scene_name, source) + raw_scene_name, category_name, source) VALUES %s ON CONFLICT (market_product_id) DO UPDATE SET - style_id = EXCLUDED.style_id, - scene_id = EXCLUDED.scene_id, - raw_style_name = EXCLUDED.raw_style_name, - raw_scene_name = EXCLUDED.raw_scene_name, + style_id = COALESCE(EXCLUDED.style_id, market_product_classification.style_id), + scene_id = COALESCE(EXCLUDED.scene_id, market_product_classification.scene_id), + raw_style_name = COALESCE(EXCLUDED.raw_style_name, market_product_classification.raw_style_name), + raw_scene_name = COALESCE(EXCLUDED.raw_scene_name, market_product_classification.raw_scene_name), + category_name = COALESCE(EXCLUDED.category_name, market_product_classification.category_name), source = EXCLUDED.source, edited_at = NOW(), updated_at = NOW() @@ -379,9 +462,140 @@ def archive_product_snapshots( "products": len(product_ids), "snapshots": len(snapshot_rows), "classifications": len(classification_rows), + "category_reused": category_reused, } +def report_month_start(value: date) -> date: + """将采集日期归一化为月份首日，作为行军图报告业务键。""" + return value.replace(day=1) + + +def archive_collected_market_rank( + conn, + *, + platform: str, + collected_on: date, + period: str, + category_scope: str, + category_names: Sequence[str], + report_title: str, + rows: Sequence[Mapping[str, Any]], + source_json_path: str | None = None, + run_metadata: Mapping[str, Any] | None = None, + price_cache: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """将采集结果按月直接归档，保留商品主数据上的人工分类。""" + if __package__: + from .db import upsert_market_rank_report + else: + from db import upsert_market_rank_report + + month = report_month_start(collected_on) + filtered_rows = filter_rows_by_rank(platform, rows) + snapshots = [ + build_product_snapshot(platform, row, price_cache=price_cache) + for row in filtered_rows + ] + names = list(dict.fromkeys( + str(name).strip() for name in category_names if str(name).strip() + )) + metadata = dict(run_metadata or {}) + metadata.update({ + "storage_mode": "database_monthly", + "collected_on": collected_on.isoformat(), + "rank_limit_applied": True, + }) + report_id = upsert_market_rank_report( + conn, + { + "platform": platform, + "report_date": month, + "period": str(period).strip(), + "category_scope": str(category_scope).strip(), + "category_names": names, + "report_title": str(report_title).strip(), + "feishu_doc_id": None, + "feishu_doc_url": None, + "product_count": len(snapshots), + "source_json_path": source_json_path, + "run_metadata": metadata, + }, + ) + with conn.cursor() as cur: + cur.execute( + "DELETE FROM fact_market_product_snapshot WHERE report_id = %s", + (report_id,), + ) + result = archive_product_snapshots( + conn, + report_id=report_id, + report_date=month, + platform=platform, + snapshots=snapshots, + labels=None, + ) + result.update({ + "report_id": report_id, + "report_month": month.strftime("%Y-%m"), + "amount_complete": sum( + item["estimated_gmv"] is not None for item in snapshots + ), + }) + return result + + +def _load_previous_tmall_prices( + conn, + rows: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + """复用历史天猫客单价，避免无价格时无法计算市场金额。""" + product_ids = list(dict.fromkeys(extract_product_id("tm", row) for row in rows)) + if not product_ids: + return {} + with conn.cursor() as cur: + cur.execute( + """ + SELECT DISTINCT ON (product.platform_product_id) + product.platform_product_id, + COALESCE(snapshot.unit_price_text, + snapshot.unit_price_max::TEXT, + snapshot.unit_price_min::TEXT) + FROM market_product product + JOIN fact_market_product_snapshot snapshot + ON snapshot.market_product_id = product.id + JOIN fact_market_rank_reports report + ON report.id = snapshot.report_id + WHERE product.platform = 'tm' + AND product.platform_product_id = ANY(%s) + AND COALESCE(snapshot.unit_price_text, + snapshot.unit_price_max::TEXT, + snapshot.unit_price_min::TEXT) IS NOT NULL + ORDER BY product.platform_product_id, + report.report_date DESC, + snapshot.updated_at DESC + """, + (product_ids,), + ) + return {str(product_id): price for product_id, price in cur.fetchall()} + + +def archive_collected_market_rank_to_database(**kwargs) -> dict[str, Any]: + """初始化行军图表结构，并在单个事务中完成月度归档。""" + if __package__: + from .db import get_conn, init_schema + else: + from db import get_conn, init_schema + + init_schema() + with get_conn() as conn: + if kwargs.get("platform") == "tm": + prices = _load_previous_tmall_prices(conn, kwargs.get("rows") or []) + prices.update(dict(kwargs.get("price_cache") or {})) + kwargs["price_cache"] = prices + return archive_collected_market_rank(conn, **kwargs) + + def import_one_platform( conn, *, diff --git a/src/gyxx_flow/modules/product_commerce/online_stock_alert.py b/src/gyxx_flow/modules/product_commerce/online_stock_alert.py new file mode 100644 index 0000000..5c726f2 --- /dev/null +++ b/src/gyxx_flow/modules/product_commerce/online_stock_alert.py @@ -0,0 +1,299 @@ +"""Daily low-stock alert for online items returned by the data-hub API.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sys +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal, InvalidOperation +from zoneinfo import ZoneInfo + +from gyxx_flow.adapters import ( + ANALYZER_NOTIFICATION_APP_PROFILE, + resolve_notification_route, + send_lark_bot_message, +) +from gyxx_flow.adapters.data_interaction import query_items + +WORKFLOW_ID = "product.online_stock_alert.daily" +STOCK_THRESHOLD = Decimal("3") +MESSAGE_CHUNK_SIZE = 50 +_EXCLUDED_SKUS = frozenset( + { + "10227", + "10270001", + "10366001_1", + "10378001", + "10378001_1", + "10448001", + "10448001_1", + "BM250328006866385", + "BM250328012355847", + "BM250328060987973", + "BM250328066913612", + "BM250328072830302", + "BM250328175841112", + "BM250328212608477", + "BM250328239165848", + "BM250328241096929", + "BM250328266952877", + "BM250328270741837", + "BM250328301564047", + "BM250328409988156", + "BM250328492316477", + "BM250328494446564", + "BM250328504384846", + "BM250328540358603", + "BM250328541446590", + "BM250328544762703", + "BM250328556624392", + "BM250328594800166", + "BM250328640311981", + "BM250328664173217", + "BM250328664260166", + "BM250328673950056", + "BM250328674677294", + "BM250328676590117_2", + "BM250328704816238", + "BM250328718416445", + "BM250328726531528", + "BM250328756734876", + "BM250328763179868", + "BM250328795876823", + "BM250328812127190", + "BM250328816220162", + "BM250328830535162", + "BM250328853592233", + "BM250328870378463", + "BM250328883159103", + "BM250328892626254", + "BM250328908136518", + "BM250328932660817", + "BM250328935561409", + "BM250328957343921", + "BM250328993281039", + "BM250330147924153", + "BM250408017838982", + "BM250427226172206", + "BM250427645375803", + "BM250503961917331", + "BM250507777132872", + "BM250509833910442", + "BM250606244082965_22", + "BM250611823217583", + "BM250618378642428", + "BM250708888943917", + "BM250808642451196", + "BM250808889760984", + "BM250814266426378", + "BM250823172803060", + "BM250912137690743", + "BM250913443470988", + "BM250917839296263", + "BM250919318064216", + "BM250925746664471", + "BM250925746664471_1", + "BM251011003374042", + "BM251011982011226", + "BM251015212273709", + "BM251023289902769_1", + "BM251023289902769_2", + "BM251023289902769_3", + "BM251109083671008", + "BM251119108159063_1", + "BM251119108167453_9", + "BM251127628555299", + "BM251224071158043", + "BM251230023901245", + "BM251230023901568", + "BM260330270223474", + "BM260520641972363", + "BM260520641972363_2", + "BM260526770666692", + "BM260529757932836", + "BM260815907847796", + "BM260815918730239", + "BM260918043711245", + "BM260922715763555", + } +) +_SAFE_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass(frozen=True, slots=True) +class ItemStock: + sku: str + name: str + stock: Decimal + + +def _item_stock(row: dict[str, object], index: int) -> ItemStock | None: + stock_value = row.get("total_stock") + if stock_value is None: + stock_value = row.get("total_stock_str") + if stock_value is None or isinstance(stock_value, bool): + raise ValueError(f"API item row {index} has no numeric stock") + try: + stock = Decimal(str(stock_value).strip()) + except (InvalidOperation, ValueError) as exc: + raise ValueError(f"API item row {index} has invalid stock") from exc + if not stock.is_finite(): + raise ValueError(f"API item row {index} has non-finite stock") + if stock >= STOCK_THRESHOLD: + return None + + sku_value = row.get("item_code") or row.get("sku_code") or row.get("skuCode") + if sku_value is None or not str(sku_value).strip(): + raise ValueError(f"API item row {index} has no SKU code") + name_value = row.get("title") or row.get("item_name") or row.get("name") or "未命名商品" + + name = " ".join(str(name_value).split()) or "未命名商品" + return ItemStock(sku=str(sku_value).strip(), name=name, stock=stock) + + +def _stock_text(stock: Decimal) -> str: + return format(stock.normalize(), "f") + + +def _business_date() -> str: + value = os.environ.get("GYXX_BUSINESS_DATE", "").strip() + if value and _SAFE_DATE.fullmatch(value): + return value + return datetime.now(ZoneInfo("Asia/Shanghai")).date().isoformat() + + +def _alert_items(rows: list[dict[str, object]]) -> list[ItemStock]: + unique: dict[tuple[str, str, Decimal], ItemStock] = {} + for index, row in enumerate(rows, start=1): + item = _item_stock(row, index) + if item is not None and item.sku not in _EXCLUDED_SKUS: + unique[(item.sku, item.name, item.stock)] = item + return sorted(unique.values(), key=lambda item: (item.sku, item.name, item.stock)) + + +def _card( + items: list[ItemStock], *, total_count: int, page: int, pages: int +) -> dict[str, object]: + return { + "schema": "2.0", + "config": {"update_multi": True, "width_mode": "fill"}, + "header": { + "title": {"tag": "plain_text", "content": "线上库存预警"}, + "subtitle": { + "tag": "plain_text", + "content": "线上渠道 · 商品库存低于 3 个", + }, + "template": "red", + "icon": {"tag": "standard_icon", "token": "warning_colorful"}, + }, + "body": { + "direction": "vertical", + "padding": "12px 12px 20px 12px", + "vertical_spacing": "8px", + "elements": [ + { + "tag": "markdown", + "element_id": "summary", + "content": ( + f"**库存低于 3 个：{total_count} 个 SKU** · " + f"第 {page}/{pages} 条，本条 {len(items)} 个 SKU" + ), + }, + { + "tag": "table", + "element_id": "stock_items", + "page_size": 10, + "row_height": "auto", + "row_max_height": "124px", + "freeze_first_column": True, + "columns": [ + { + "name": "sku", + "display_name": "SKU", + "data_type": "text", + "width": "180px", + }, + { + "name": "name", + "display_name": "商品名称", + "data_type": "text", + "width": "auto", + }, + { + "name": "stock", + "display_name": "库存", + "data_type": "text", + "width": "80px", + "horizontal_align": "right", + }, + ], + "rows": [ + { + "sku": item.sku, + "name": item.name, + "stock": _stock_text(item.stock), + } + for item in items + ], + }, + ], + }, + } + + +def run() -> int: + rows = query_items(channel=0, keyword="") + alerts = _alert_items(rows) + print(f"线上商品查询完成：{len(rows)} 条，低库存 SKU：{len(alerts)} 条") + if not alerts: + print("没有库存低于 3 的商品，不发送飞书提醒。") + return 0 + + route = resolve_notification_route(WORKFLOW_ID, ()) + if not route.enabled: + print("该工作流的飞书通知路由已停用。") + return 0 + if not route.open_ids: + raise RuntimeError("库存预警通知路由没有配置接收人") + + pages = (len(alerts) + MESSAGE_CHUNK_SIZE - 1) // MESSAGE_CHUNK_SIZE + business_date = _business_date() + for page in range(pages): + chunk = alerts[page * MESSAGE_CHUNK_SIZE : (page + 1) * MESSAGE_CHUNK_SIZE] + card = _card( + chunk, + total_count=len(alerts), + page=page + 1, + pages=pages, + ) + card_bytes = json.dumps( + card, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + chunk_hash = hashlib.sha256(card_bytes).hexdigest()[:20] + idempotency_key = f"{WORKFLOW_ID}:{business_date}:{page + 1}:{chunk_hash}" + for open_id in route.open_ids: + send_lark_bot_message( + user_id=open_id, + content=card, + msg_type="interactive", + idempotency_key=idempotency_key, + profile=route.app_profile or ANALYZER_NOTIFICATION_APP_PROFILE, + ) + print(f"库存预警已发送：{len(alerts)} 个 SKU，{pages} 条飞书消息。") + return 0 + + +def main() -> int: + try: + return run() + except Exception as exc: + print(f"库存预警工作流失败：{exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/product_commerce/run_daily_persona.py b/src/gyxx_flow/modules/product_commerce/run_daily_persona.py index 4652e89..313fec1 100644 --- a/src/gyxx_flow/modules/product_commerce/run_daily_persona.py +++ b/src/gyxx_flow/modules/product_commerce/run_daily_persona.py @@ -77,9 +77,25 @@ def build_child_environment( return environment_for_child_script(target, environment) -def build_child_command(target: Path | str, business_date: date) -> list[str]: +def selected_platforms(value: str = "") -> list[str]: + """Return a validated, de-duplicated platform selection.""" + requested = [item.strip().lower() for item in value.split(",") if item.strip()] + unknown = [item for item in requested if item not in JOBS] + if unknown: + raise ValueError(f"不支持的平台: {', '.join(unknown)}；可选 tm,dy,jd") + return list(dict.fromkeys(requested)) or list(JOBS) + + +def build_child_command( + target: Path | str, + business_date: date, + styles: str = "", +) -> list[str]: """Build a worker command pinned to the workflow's business date.""" - return [sys.executable, str(target), "--date", business_date.isoformat()] + command = [sys.executable, str(target), "--date", business_date.isoformat()] + if styles.strip(): + command.extend(["--styles", styles]) + return command def main() -> int: @@ -90,11 +106,26 @@ def main() -> int: default=date.today(), help="统一业务日期 YYYY-MM-DD", ) + parser.add_argument( + "--platforms", + default="", + help="只跑指定平台，逗号分隔：tm,dy,jd；默认全部", + ) + parser.add_argument( + "--styles", + default="", + help="只传给平台采集器指定款式，逗号分隔；默认全部", + ) args = parser.parse_args() + try: + platforms = selected_platforms(args.platforms) + except ValueError as exc: + parser.error(str(exc)) LOG_DIR.mkdir(parents=True, exist_ok=True) running: dict[str, tuple[subprocess.Popen[bytes], object, Path, int]] = {} - for platform, (script, log_name) in JOBS.items(): + for platform in platforms: + script, log_name = JOBS[platform] target = Path(script) if not target.is_absolute(): target = PROJECT_ROOT / target @@ -103,7 +134,7 @@ def main() -> int: start_offset = log_path.stat().st_size if log_path.exists() else 0 log_handle = log_path.open("ab") process = subprocess.Popen( - build_child_command(target, args.date), + build_child_command(target, args.date, args.styles), cwd=PROJECT_ROOT, stdout=log_handle, stderr=subprocess.STDOUT, diff --git a/src/gyxx_flow/modules/product_commerce/scripts/insert_main_image_records.py b/src/gyxx_flow/modules/product_commerce/scripts/insert_main_image_records.py index c6695ff..492e4d9 100644 --- a/src/gyxx_flow/modules/product_commerce/scripts/insert_main_image_records.py +++ b/src/gyxx_flow/modules/product_commerce/scripts/insert_main_image_records.py @@ -74,6 +74,48 @@ def run_lark( return proc.returncode, proc.stdout, proc.stderr +_LARK_RETRYABLE_MARKERS = ( + "eof", + "timeout", + "timed out", + "connection", + "transport", + "network", + "502", + "503", + "504", +) + + +def run_lark_with_retry( + args: list[str], *, attempts: int = 3, backoff_seconds: float = 1.0 +) -> tuple[int, str, str]: + """Retry only transient read/idempotent Base calls. + + A lost response must not cause a non-idempotent batch-create to be replayed, + so callers opt in per operation instead of changing ``run_lark`` globally. + """ + last_result = (1, "", "") + for attempt in range(1, attempts + 1): + last_result = run_lark(args, check=False) + rc, out, err = last_result + if rc == 0: + return last_result + detail = f"{err}\n{out}".lower() + if attempt >= attempts or not any( + marker in detail for marker in _LARK_RETRYABLE_MARKERS + ): + return last_result + delay = backoff_seconds * (2 ** (attempt - 1)) + print( + f" [WARN] lark-cli {args[1] if len(args) > 1 else 'request'} " + f"网络失败 (attempt {attempt}/{attempts}), {delay:g}s 后重试", + file=sys.stderr, + ) + time.sleep(delay) + return last_result + + def parse_lark_json(stdout: str) -> Optional[dict]: """lark-cli JSON 输出前面常带 'Xxx is metadata...' 之类文本,从第一个 '{' 开始 raw_decode。""" decoder = json.JSONDecoder() @@ -105,14 +147,14 @@ def date_to_month_week(date_str: str) -> str: def discover_fields(base_token: str, table_id: str) -> dict: """返回 {逻辑名: field_id} 映射,逻辑名固定为: 时间/平台/曝光量/点击量/主图""" - rc, out, _ = run_lark([ + rc, out, err = run_lark_with_retry([ "base", "+field-list", "--base-token", base_token, "--table-id", table_id, "--as", "user", - ], check=False) + ]) if rc: - raise RuntimeError(f"field-list failed: {out[:200]}") + raise RuntimeError(f"field-list failed: {(err or out)[:200]}") obj = parse_lark_json(out) if not obj: raise RuntimeError(f"field-list: 解析失败: {out[:200]}") @@ -155,7 +197,7 @@ def list_existing_records( by_image 用附件文件名做幂等键；metric_tuples 是旧逻辑兜底，避免无附件老记录重复创建。 duplicates 记录同一图片名已有多条记录的情况，后续人工清理用。 """ - rc, out, _ = run_lark([ + rc, out, err = run_lark_with_retry([ "base", "+record-list", "--base-token", base_token, "--table-id", table_id, @@ -169,7 +211,9 @@ def list_existing_records( ], }, ensure_ascii=False), "--limit", "200", - ], check=False) + ]) + if rc: + raise RuntimeError(f"record-list failed: {(err or out)[:200]}") obj = parse_lark_json(out) if not obj or "data" not in obj: return {}, set(), {} @@ -237,14 +281,14 @@ def upsert_record_fields(base_token: str, table_id: str, record_id: str, fields_ details={"base_token": base_token, "table_id": table_id, "record_id": record_id}, ): return - rc, out, err = run_lark([ + rc, out, err = run_lark_with_retry([ "base", "+record-upsert", "--base-token", base_token, "--table-id", table_id, "--as", "user", "--json", json.dumps(fields_payload, ensure_ascii=False), "--record-id", record_id, - ], check=False) + ]) if rc: raise RuntimeError(f"record-upsert failed: exit={rc} stderr={err[:300]} stdout={out[:300]}") diff --git a/src/gyxx_flow/modules/product_commerce/taobao_sycm_collect.py b/src/gyxx_flow/modules/product_commerce/taobao_sycm_collect.py index b316dfc..a1ac932 100644 --- a/src/gyxx_flow/modules/product_commerce/taobao_sycm_collect.py +++ b/src/gyxx_flow/modules/product_commerce/taobao_sycm_collect.py @@ -318,7 +318,13 @@ def main(argv: list[str] | None = None) -> int: proc = None for attempt in range(1, 3): collection_port = int(brand_environment["GYXX_BROWSER_CDP_PORT"]) - if not is_chrome_debug_running(collection_port): + # --backfill 子进程 (taobao_sycm_collect_backfill) 自行通过 + # StealthySession 启 Chrome, 不连接 cdp_url; 若这里预先用同一 + # 临时 profile 启动调试 Chrome, 子进程再启动会命中 Chrome 单实例 + # 移交 ("正在现有的浏览器会话中打开") 直接 TargetClosedError. + if args.backfill: + pass + elif not is_chrome_debug_running(collection_port): started = launch_chrome_with_debug( brand_user_data_dir, start_url="about:blank", diff --git a/src/gyxx_flow/modules/product_commerce/tmall_baibu_apply_common.py b/src/gyxx_flow/modules/product_commerce/tmall_baibu_apply_common.py index ed2f8db..cd38201 100644 --- a/src/gyxx_flow/modules/product_commerce/tmall_baibu_apply_common.py +++ b/src/gyxx_flow/modules/product_commerce/tmall_baibu_apply_common.py @@ -33,6 +33,8 @@ from gyxx_flow.core.locks import LockManager, ResourceBusyError _CONFIG_NAME = "tmall-baibu.json" _EXPORT_TIMEOUT_SECONDS = 240 +_EXPORT_MAX_ATTEMPTS = 3 +_EXPORT_RETRY_DELAYS_SECONDS = (5, 15) _PAGE_TIMEOUT_MS = 60_000 _IMPORT_TIMEOUT_SECONDS = 300 _STATUS_SETTLE_TIMEOUT_SECONDS = 5 @@ -41,6 +43,7 @@ _DRAFT_ACTION_TIMEOUT_SECONDS = 30 _DRAFT_NAVIGATION_TIMEOUT_SECONDS = 15 _DRAFT_PROGRESS_TIMEOUT_SECONDS = 60 _MAX_DRAFT_COMPLETIONS = 200 +_DRAFT_SAVE_LABELS = ("提交", "下一步") _LOGIN_PATH_MARKERS = ("login", "passport", "signin", "sign-in") _LOGIN_FORM_SETTLE_MILLISECONDS = 500 _LOGIN_CONFIRM_TIMEOUT_SECONDS = 120 @@ -246,11 +249,59 @@ def _lark_cli() -> str: def _safe_cli_detail(stdout: str, stderr: str) -> str: detail = (stderr or stdout or "").strip().replace("\r", " ").replace("\n", " ") + detail = re.sub( + r"([?&](?:access_)?token=)[^&\s\"}]+", + r"\1", + detail, + flags=re.IGNORECASE, + ) + detail = re.sub( + r"(\"(?:access_)?token\"\s*:\s*\")[^\"]+(\")", + r"\1\2", + detail, + flags=re.IGNORECASE, + ) return detail[-500:] if detail else "无返回信息" +def _is_retryable_export_failure(stdout: str, stderr: str) -> bool: + """Return whether a failed read-only Feishu export may be retried safely.""" + + detail = f"{stdout}\n{stderr}".casefold() + compact = re.sub(r"\s+", "", detail) + network_error = '"type":"network"' in compact or "type=network" in compact + transport_error = any( + marker in detail + for marker in ( + "timeout", + "timed out", + "time out", + "context deadline exceeded", + "dial tcp", + "no such host", + "connection reset", + "connection refused", + "temporary failure", + ) + ) + subtype_transport = ( + '"subtype":"transport"' in compact + or "subtype=transport" in compact + ) + return (network_error and (transport_error or subtype_transport)) or ( + "context deadline exceeded" in detail + or "client.timeout exceeded" in detail + ) + + +def _export_retry_delay(attempt: int) -> int: + index = min(max(attempt - 1, 0), len(_EXPORT_RETRY_DELAYS_SECONDS) - 1) + return _EXPORT_RETRY_DELAYS_SECONDS[index] + + def _export_template(entry: BaibuEntry, work_dir: Path) -> Path: filename = f"{entry.entry_id}.xlsx" + target = work_dir / filename command = [ _lark_cli(), "sheets", @@ -264,29 +315,67 @@ def _export_template(entry: BaibuEntry, work_dir: Path) -> Path: "--format", "json", ] - environment = {**os.environ, "LARK_CLI_NO_PROXY": "1"} - try: - result = subprocess.run( - command, - cwd=work_dir, - env=environment, - capture_output=True, - text=True, - encoding="utf-8", - errors="replace", - timeout=_EXPORT_TIMEOUT_SECONDS, - check=False, - ) - except (OSError, subprocess.TimeoutExpired) as exc: - raise BaibuApplyError(f"下载百补模板失败：{type(exc).__name__}") from exc - if result.returncode != 0: + environment = { + **os.environ, + "LARK_CLI_NO_PROXY": "1", + "LARK_CLI_NO_PROXY_WARN": "1", + "LARKSUITE_CLI_NO_UPDATE_NOTIFIER": "1", + "LARKSUITE_CLI_NO_SKILLS_NOTIFIER": "1", + } + for attempt in range(1, _EXPORT_MAX_ATTEMPTS + 1): + # A failed export can leave a partial file behind. It must never be + # mistaken for a valid template on a later attempt. + try: + target.unlink(missing_ok=True) + except OSError as exc: + raise BaibuApplyError(f"下载百补模板失败：无法清理临时文件（{type(exc).__name__}）") from exc + try: + result = subprocess.run( + command, + cwd=work_dir, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=_EXPORT_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired as exc: + if attempt < _EXPORT_MAX_ATTEMPTS: + delay = _export_retry_delay(attempt) + print( + f"[百补] {entry.name} 模板下载超时（第 {attempt}/{_EXPORT_MAX_ATTEMPTS} 次），" + f"{delay} 秒后重试", + file=sys.stderr, + ) + time.sleep(delay) + continue + raise BaibuApplyError( + f"下载百补模板失败：{type(exc).__name__}（已尝试 {attempt} 次）" + ) from exc + except OSError as exc: + raise BaibuApplyError(f"下载百补模板失败：{type(exc).__name__}") from exc + if result.returncode == 0: + if target.is_file() and target.stat().st_size > 0: + return target + break + detail = _safe_cli_detail(result.stdout, result.stderr) + if attempt < _EXPORT_MAX_ATTEMPTS and _is_retryable_export_failure( + result.stdout, + result.stderr, + ): + delay = _export_retry_delay(attempt) + print( + f"[百补] {entry.name} 模板下载遇到瞬时网络错误（第 {attempt}/{_EXPORT_MAX_ATTEMPTS} 次），" + f"{delay} 秒后重试", + file=sys.stderr, + ) + time.sleep(delay) + continue raise BaibuApplyError( - f"下载百补模板失败（exit={result.returncode}）：" - f"{_safe_cli_detail(result.stdout, result.stderr)}" + f"下载百补模板失败（exit={result.returncode}）：{detail}" ) - target = work_dir / filename - if target.is_file() and target.stat().st_size > 0: - return target candidates = sorted( path for path in work_dir.glob("*.xlsx") @@ -1222,9 +1311,9 @@ def _complete_draft_products(page: Any, entry: BaibuEntry) -> None: ) _click_text_until_found( page, - ("下一步",), + _DRAFT_SAVE_LABELS, entry=entry, - description="下一步", + description="提交/下一步", ) completed += 1 _return_to_draft_list(page, entry) diff --git a/src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py b/src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py index c14fe36..ab4f0d8 100644 --- a/src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py +++ b/src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py @@ -152,7 +152,7 @@ STORE_CREATOR_IDENTITY_MARKERS: dict[str, tuple[str, ...]] = { EXIT_OK = 0 EXIT_FAILED = 1 EXIT_AMBIGUOUS = 2 -TMALL_TITLE_CANDIDATE_COUNT = 6 +TMALL_TITLE_CANDIDATE_COUNT = 8 # data-analyzer uses a reasoning model whose latency is variable. The # 2026-08-18 production request completed in 189.5s, after the old 150s # client deadline had already closed the socket. @@ -391,6 +391,10 @@ def _select_video_product_names( def _style_db_lookup_key(value: Any) -> str: """Normalize a Feishu style label to the canonical dim_style lookup key.""" compact = re.sub(r"\s+", "", str(value or "")) + # Feishu source rows occasionally use "斜跨" for the catalog spelling + # "斜挎"; this keeps exact model/variant matching intact after correcting + # that one-character typo (for example, 盖亚斜跨3 -> 盖亚斜挎包3). + compact = compact.replace("斜跨", "斜挎") compact = re.sub(r"系列.*$", "", compact).strip("-—_·:：") return compact.casefold() @@ -2074,6 +2078,10 @@ class GuangHeUploader: # category, or duplicate constraint. Keep the final choice # strict, but ask the model for a small validated-choice pool. candidate_count=TMALL_TITLE_CANDIDATE_COUNT, + # A first correction can still echo a recent title. Give the + # local fact/deduplication validator one additional bounded + # chance to request a genuinely different candidate. + llm_validation_retries=2, allow_question_mark=False, prefer_double_shoulder=not is_collection, llm_timeout=TMALL_DIRECT_LLM_TITLE_TIMEOUT_SECONDS, diff --git a/src/gyxx_flow/modules/product_commerce/upload_video_to_jd.py b/src/gyxx_flow/modules/product_commerce/upload_video_to_jd.py index 7a4726b..16b19a9 100644 --- a/src/gyxx_flow/modules/product_commerce/upload_video_to_jd.py +++ b/src/gyxx_flow/modules/product_commerce/upload_video_to_jd.py @@ -236,6 +236,7 @@ STYLE_MODEL_TOKEN_PATTERN = ( + r")" ) CONTENT_MANAGEMENT_CHANNEL = "逛-竖版视频" +JD_FORM_CHANNEL_LABELS = ("逛-视频", CONTENT_MANAGEMENT_CHANNEL) CONTENT_MANAGEMENT_ACCEPTED_STATUSES = ( "审核中", "审核通过", @@ -930,7 +931,14 @@ def ensure_jd_compatible_video(path: Path) -> Path: return path ffmpeg = shutil.which("ffmpeg") if not ffmpeg: - raise RuntimeError("M4V 视频需要 ffmpeg 转为 MP4，但当前运行环境未找到 ffmpeg") + try: + import imageio_ffmpeg + except ImportError: + raise RuntimeError( + "M4V 视频需要 ffmpeg 转为 MP4，但当前运行环境未找到 ffmpeg，" + "且未安装 imageio-ffmpeg 兜底" + ) + ffmpeg = imageio_ffmpeg.get_ffmpeg_exe() output = path.with_suffix(".jd-upload.mp4") if output.is_file() and output.stat().st_size > 0: return output @@ -1850,18 +1858,37 @@ class JdVideoUploader: page = self._require_page() surface = self._require_surface() channel = surface.locator(".channel-submission-selChannel-channel") + exact_channel_texts: list[tuple[str, Any]] = [] + for label in JD_FORM_CHANNEL_LABELS: + try: + exact_channel_texts.append( + (label, surface.get_by_text(label, exact=True)) + ) + except Exception: + continue modify_channel = surface.locator( '[data-spm-click="publishVideoChannelOpenDrawer"]' ) - channel.wait_for(state="visible", timeout=60_000) stable_text = "" stable_polls = 0 - deadline = time.monotonic() + 15 + deadline = time.monotonic() + 60 while time.monotonic() < deadline: - current = channel.inner_text().strip() + current = "" + if self._visible_count(channel) == 1: + try: + current = channel.inner_text().strip() + except Exception: + current = "" + if current not in JD_FORM_CHANNEL_LABELS: + visible_labels = [ + label + for label, locator in exact_channel_texts + if self._visible_count(locator) == 1 + ] + current = visible_labels[0] if len(visible_labels) == 1 else "" video_input = self._available_video_file_input() ready = ( - current + current in JD_FORM_CHANNEL_LABELS and self._visible_count(modify_channel) == 1 and video_input is not None and video_input.is_enabled() @@ -2497,6 +2524,14 @@ class JdVideoUploader: style_name: str, ) -> bool: compact = cls._normalize_product_match_text(card_text) + if ( + cls._style_signature(style_name) == "黑曜石吐司" + and "黑曜石斜挎吐司" in compact + ): + # JD inserts the carrying form into this verified style name: + # "黑曜石斜挎吐司包…". Keep the exception bounded to the exact + # style and spelling observed in the failed own-store search. + return True if cls._is_gaia_series_style(style_name): # The JD search result is the authoritative series scope here: # 盖亚斜挎3/C1/X/Neo are all represented by cards under the same @@ -4259,19 +4294,65 @@ class JdVideoUploader: f"expected={sorted(expected)!r}, actual={list(actual_values)!r}" ) - def _click_unique_tag_option(self, overlay: Any, text: str) -> None: + def _click_unique_tag_option(self, overlay: Any | None, text: str) -> None: + page = self._require_page() if overlay is None else getattr(self, "page", None) title = json.dumps(text, ensure_ascii=False) - matches = self._visible_locators( - overlay.locator( - "li[role='menuitemcheckbox']" - f"[title={title}][data-path-key]" - ) + selector = ( + "li[role='menuitemcheckbox']" + f"[title={title}][data-path-key]" + ) + deadline = time.monotonic() + 5 + matches: list[Any] = [] + while time.monotonic() < deadline: + if overlay is not None: + matches = self._visible_locators(overlay.locator(selector)) + else: + # 京东把级联标签菜单挂在主文档或发布 iframe 的 body 上， + # 位置会随页面版本变化。按完整 role/title/path-key 查找可见 + # 选项，并跨可见 frame 要求全局唯一，避免依赖弹层 CSS 类名。 + matches = [] + for frame in page.frames: + if frame.is_detached(): + continue + if frame is not page.main_frame: + try: + if not frame.frame_element().is_visible(): + continue + except Exception: + continue + try: + matches.extend( + self._visible_locators(frame.locator(selector)) + ) + except Exception: + continue + if len(matches) == 1: + matches[0].click() + return + if len(matches) > 1: + screenshot = ( + self._screenshot("tag-option-ambiguous", always=True) + if page is not None + else None + ) + raise RuntimeError( + f"京东标签存在多个可见选项 {text!r}; " + f"count={len(matches)}, evidence={screenshot}" + ) + if page is not None: + page.wait_for_timeout(200) + else: + time.sleep(0.2) + screenshot = ( + self._screenshot("tag-option-missing", always=True) + if page is not None + else None ) if len(matches) != 1: raise RuntimeError( - f"京东标签活动浮层未找到唯一选项 {text!r}; count={len(matches)}" + f"京东标签未找到唯一可见选项 {text!r}; " + f"count={len(matches)}, evidence={screenshot}" ) - matches[0].click() def _select_tag_path(self, path: Sequence[str]) -> Any: page = self._require_page() @@ -4282,8 +4363,7 @@ class JdVideoUploader: comboboxes[0].click() page.wait_for_timeout(300) for segment in path: - overlay = self._active_tag_overlay() - self._click_unique_tag_option(overlay, segment) + self._click_unique_tag_option(None, segment) page.wait_for_timeout(200) page.keyboard.press("Escape") page.wait_for_timeout(200) diff --git a/src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_data_collector.py b/src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_data_collector.py index 8f341b3..6b10f5b 100644 --- a/src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_data_collector.py +++ b/src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_data_collector.py @@ -1075,11 +1075,79 @@ def close_profit_plan_popup_if_present(page): return True +def close_ai_helper_drawer_if_present(page): + """Close the AI helper drawer whose backdrop blocks page-level clicks.""" + try: + page.wait_for_function( + """() => { + const host = document.querySelector('aihelper-extension-embedded'); + const drawer = host?.shadowRoot?.querySelector('.ai-iframe-embedded-drawer'); + if (!drawer) return false; + const rect = drawer.getBoundingClientRect(); + const style = window.getComputedStyle(drawer); + return rect.width > 0 && rect.height > 0 + && style.display !== 'none' + && style.visibility !== 'hidden'; + }""", + timeout=3000, + ) + + popup = page.frame_locator( + "aihelper-extension-embedded iframe.ai-iframe-embedded" + ) + close_button = popup.locator("div.jdm-miniChatHeader__close").last + close_button.wait_for(state="visible", timeout=5000) + close_button.click(timeout=5000) + page.wait_for_function( + """() => { + const host = document.querySelector('aihelper-extension-embedded'); + const drawer = host?.shadowRoot?.querySelector('.ai-iframe-embedded-drawer'); + if (!drawer) return true; + const rect = drawer.getBoundingClientRect(); + const style = window.getComputedStyle(drawer); + return rect.width === 0 || rect.height === 0 + || style.display === 'none' + || style.visibility === 'hidden'; + }""", + timeout=5000, + ) + print(" [OK] 商智新增 AI 助手抽屉已关闭") + return True + except Exception as exc: + print(f" [WARN] 商智新增 AI 助手抽屉未能关闭: {exc}") + return False + + +def close_jzt_lite_popup_if_present(page): + """Close the full-screen JZT promotion iframe when JD商智 injects it.""" + wrapper = page.locator("#jzt-lite-wrapper").first + try: + wrapper.wait_for(state="visible", timeout=1500) + except Exception: + return False + + try: + popup = page.frame_locator("#jzt-lite-iframe") + close_button = popup.locator(".close-btn").first + close_button.wait_for(state="visible", timeout=5000) + close_button.click(timeout=5000) + page.wait_for_timeout(1000) + wrapper.wait_for(state="hidden", timeout=5000) + print(" [OK] 商智新增稳赚计划弹窗已关闭") + return True + except Exception as exc: + print(f" [WARN] 商智新增稳赚计划弹窗未能关闭: {exc}") + return False + + def close_popup_if_present(page): """ 检测并关闭广告弹窗（如果存在） 使用 try/except 避免弹窗未出现时报错 """ + close_ai_helper_drawer_if_present(page) + close_jzt_lite_popup_if_present(page) + closed_profit_plan = False for _ in range(4): if close_profit_plan_popup_if_present(page): diff --git a/src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_peer_product_data_collector.py b/src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_peer_product_data_collector.py index 48e7c77..a32aa64 100644 --- a/src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_peer_product_data_collector.py +++ b/src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_peer_product_data_collector.py @@ -45,19 +45,54 @@ STORE_SPU = "10027762032411" # 本店固定SPU，用于竞品对比 def step_return_old_version(page): """返回商智旧版""" print("[Step 3] 返回商智旧版...") + current_url = page.url.lower() + if "jdsz.jd.com" in current_url: + # 京东已按账号灰度下线旧版入口；旧版 URL 会直接重定向到新版。 + # 新版商品明细/下载流程由调用方继续处理，不应再把缺少“返回旧版” + # 按钮误判成采集失败。 + print(f" [OK] 商智旧版入口已下线，继续使用新版页面: {page.url}") + return + close_popup_if_present(page) - try: - back_button = page.locator('#header > div > div.page-top-header-tools > img') - back_button.wait_for(state="visible", timeout=5000) - back_button.click() - page.wait_for_timeout(5000) + if "jdsz.jd.com" not in current_url and ( + "sz.jd.com" in current_url or "ppzh.jd.com" in current_url + ): + print(f" [OK] 当前已是商智旧版，无需点击返回旧版: {page.url}") + return + + locator_factories = [ + ("button text", lambda: page.get_by_role("button", name="返回旧版", exact=True).first), + ("visible text", lambda: page.get_by_text("返回旧版", exact=True).first), + ("alt image", lambda: page.locator('img[alt="返回旧版"]').first), + ("aria image", lambda: page.locator('img[aria-label="返回旧版"]').first), + ("header image", lambda: page.locator('#header > div > div.page-top-header-tools > img').first), + ("image role", lambda: page.get_by_role("img", name="返回旧版", exact=True).first), + ] + last_error = None + for label, factory in locator_factories: try: - page.wait_for_load_state("domcontentloaded", timeout=60000) - except Exception: - pass - print(" [OK] 已进入商智旧版") - except Exception: - print(" [INFO] 未找到返回旧版按钮，继续当前页面") + back_button = factory() + back_button.wait_for(state="visible", timeout=5000) + # The AI helper is injected asynchronously; close it again right + # before the click so its backdrop cannot race the old-version link. + close_popup_if_present(page) + back_button.click(timeout=10000) + page.wait_for_timeout(5000) + try: + page.wait_for_load_state("domcontentloaded", timeout=60000) + except Exception: + pass + current_url = page.url.lower() + if "jdsz.jd.com" in current_url: + raise RuntimeError(f"点击返回旧版后仍停留在新版页面: {page.url}") + print(f" [OK] 已进入商智旧版 ({label}): {page.url}") + return + except Exception as exc: + last_error = exc + + raise RuntimeError( + f"未能返回商智旧版，已阻止继续使用新版页面: {page.url}; {last_error}" + ) from last_error def step_goto_product_ranks(page): diff --git a/src/gyxx_flow/modules/shop_intelligence/__init__.py b/src/gyxx_flow/modules/shop_intelligence/__init__.py index 3840964..f31bf6d 100644 --- a/src/gyxx_flow/modules/shop_intelligence/__init__.py +++ b/src/gyxx_flow/modules/shop_intelligence/__init__.py @@ -13,6 +13,14 @@ SHOP_SCHEDULED_WORKFLOW_IDS = ( "shop.metrics.weekly", "shop.competitor.weekly", "shop.jd_self_operated.daily", + "shop.jd_customer_service.daily", + "shop.jd_negative_reviews.daily", + "shop.jd_after_sales_issues.daily", + "shop.douyin_customer_service.daily", + "shop.tmall_customer_service.daily", + "shop.tmall_negative_reviews.daily", + "shop.douyin_after_sales_quality.daily", + "shop.dy_negative_reviews.daily", "shop.douyin_price_appeal", ) SHOP_MANUAL_WORKFLOW_IDS: tuple[str, ...] = () diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/dy_after_sales_quality.py b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_after_sales_quality.py new file mode 100644 index 0000000..e4a3a3c --- /dev/null +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_after_sales_quality.py @@ -0,0 +1,2288 @@ +"""采集抖音售后工作台“商品品质原因”售后单（品退）并幂等写入 PostgreSQL。 + +售后工作台列表页默认查询最近 6 个月的售后单。采集器通过“更多筛选”把 +申请原因固定为“商品品质原因”；申请时间可选：控制台提交自定义日期范围时 +设置开始/结束日期，不设置时间则保持页面默认做全量查询（首次全量采集）。 +随后逐条点击“查看详情”，在新页签读取协商记录（退货原因、退款说明、 +凭证信息）与交易信息中的商品名称/商品 ID/商家编码，连同列表行的售后 +类型、状态、金额等按申请时间（上海时区）落业务日期后幂等入库。 + +浏览器复用 douyin-shop 账号 vault 的 Cookie（与抖音客服采集共用账号）， +使用 Scrapling 隐藏指纹模式（StealthySession）启动；优先接入账号保活 +浏览器（CDP），保活不在线时用账号本机 Profile + 系统 Chrome 复现登录 +环境。每条售后单的幂等键是 (account, aftersale_id)，重复执行只更新对应 +行；单条失败保留进度，重跑只补失败部分。 +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +from gyxx_flow.adapters import ( + COOKIE_SKIP_EXIT_CODE, + RuntimeIntegrationCatalog, + binding_from_environment, + current_acceptance_policy, +) +from gyxx_flow.adapters.scrapling import BrowserPage as Page +from gyxx_flow.adapters.scrapling import ScraplingBrowser, is_browser_timeout_error +from gyxx_flow.core.config import Settings +from gyxx_flow.core.layout import DataLayout +from gyxx_flow.core.locks import LockManager, ResourceBusyError + +SETTINGS = Settings.from_env() +MODULE_PATHS = DataLayout(SETTINGS.data_root).for_module("shop_intelligence") + +TARGET_URL = ( + "https://fxg.jinritemai.com/ffa/merchant-aftersale-workbench/aftersale/list" +) +ACCOUNT_ID = "douyin-shop" +COMMAND_ID = "shop.douyin_after_sales_quality.collect" +WORKFLOW_ID = "shop.douyin_after_sales_quality.daily" +DEFAULT_OUTPUT = MODULE_PATHS.raw_root / "dy_after_sales_quality" +CDP_PORT = int( + os.getenv("GYXX_BROWSER_CDP_PORT") + or os.getenv("GYXX_CDP_PORT") + or "22159" +) +DEFAULT_IDENTITY_MARKERS = ("光影行星箱包旗舰店",) +# 售后工作台页面本身不展示店铺名称，店铺身份在抖店后台页校验。 +FXG_IDENTITY_URL = os.getenv( + "GYXX_DY_IDENTITY_URL", + "https://fxg.jinritemai.com/ffa/merchant/campaign-square?list_tab=access&f_tab=0", +) +APPLY_REASON = "商品品质原因" +APPLY_REASON_OPTIONS = ( + "全部", + "商品品质原因", + "物流品质原因", + "无理由其他原因", +) +RANGE_END_ENV = "GYXX_DY_AFTER_SALES_TO" +SHANGHAI = ZoneInfo("Asia/Shanghai") +LOGIN_URL_MARKERS = ("/login", "passport") +LOGIN_TEXT_MARKERS = ( + "手机登录", + "邮箱登录", + "扫码登录", + "验证码登录", + "登录即代表同意", + "登录过期", +) +MAX_PAGES = 200 + +TOTAL_COUNT_PATTERN = re.compile(r"共\s*(\d+)\s*条") +AFTERSALE_ID_PATTERN = re.compile(r"售后编号\s*(\d{14,25})") +ORDER_ID_PATTERN = re.compile(r"订单编号\s*(\d{14,25})") +AFTERSALE_TYPE_WORDS = ("退货退款", "换货", "补寄", "维修", "仅退款") +STATUS_TAG_WORDS = ("平台主动处理", "体验分不计入") +MERCHANT_CODE_PATTERN = re.compile(r"商家编码\s*[:：]?\s*(\d{4,22})") +LIST_APPLY_TIME_PATTERN = re.compile( + r"申请时间\s*(\d{4}/\d{2}/\d{2})\s+(\d{2}:\d{2}:\d{2})" +) +DETAIL_APPLY_TIME_PATTERN = re.compile( + r"申请时间\s*(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})" +) +REFUND_AMOUNT_PATTERN = re.compile(r"售后退款\s*¥?\s*([\d,.]+)") +PAY_AMOUNT_PATTERN = re.compile(r"应付金额\s*¥?\s*([\d,.]+)") +APPLY_QUANTITY_PATTERN = re.compile(r"申请件数\s*(\d+)\s*件") +APPLY_REASON_PATTERN = re.compile(r"申请原因\s*([^\n]+)") +PRODUCT_ID_PATTERN = re.compile(r"ID\s*[:：]\s*(\d{6,30})") +COLOR_PATTERN = re.compile(r"颜色分类\s*[:：]?\s*([^\n]+)") +NEGOTIATION_TIME_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})$") +PRODUCT_TITLE_TAG_PATTERN = re.compile( + r"^(小件运费险|运费险|极速退|7天$|15天售后$|30天售后$)" +) + +ROW_HEADER_NAMES = ("商品信息", "订单信息", "售后信息", "售后状态", "纠纷仲裁", "物流信息", "操作") + +ROLE_WORDS = ("消费者", "系统", "商家") +NEGOTIATION_LABELS = ( + "操作人", + "售后类型", + "货物状态", + "退货原因", + "退款金额", + "退款说明", + "退货方式", + "退货地址", + "预约时间", + "凭证信息", + "申请金额", + "申请件数", + "售后说明", + "换货原因", + "收货状态", +) +NEGOTIATION_HEADER_WORDS = frozenset({"协商记录", "展开全部", "收起", "展开", "收起全部"}) +_REASON_HINTS = ("质量", "品质", "描述", "物流", "材质", "破损", "瑕疵", "不符", "损坏", "色差") + +DETAIL_LABELS = ( + "订单编号", + "售后编号", + "下单时间", + "申请金额", + "消费者应退金额", + "申请时间", + "售后原因", + "售后说明", + "收货状态", + "申请件数", + "消费者需回寄数量", + "消费者凭证", +) + + +def parse_total_count(text: str) -> int: + """从工具栏文案“共161条”解析筛选后的售后单总数。""" + + match = TOTAL_COUNT_PATTERN.search(text or "") + if not match: + raise RuntimeError(f"无法从页面识别售后单总数: {text!r}") + return int(match.group(1)) + + +def _clean_lines(text: str) -> list[str]: + return [line.strip() for line in (text or "").splitlines() if line.strip()] + + +def _split_line_labels( + labels: tuple[str, ...], line: str +) -> tuple[dict[str, str], list[str]]: + """把一行按标签切分为“标签 -> 值”。 + + 详情页的标签值区可能把多个字段渲染在同一行（例如“订单编号 x 售后编号 + y 申请时间 z”），也可能标签单独成行、值在后续行。这里找出行内全部 + 合法标签出现位置，按位置切片取值；值为空的标签原样返回，交给调用方 + 进入“值在后续行”的块模式。 + """ + + hits: list[tuple[int, int, str]] = [] + for label in labels: + for match in re.finditer(re.escape(label), line): + start, end = match.span() + before_ok = start == 0 or line[start - 1] in " ：:，,（(【" + after = line[end : end + 1] + after_ok = after == "" or after in " ：:" + if before_ok and after_ok: + hits.append((start, end, label)) + hits.sort() + fields: dict[str, str] = {} + empty_labels: list[str] = [] + for index, (start, end, label) in enumerate(hits): + value_end = hits[index + 1][0] if index + 1 < len(hits) else len(line) + value = line[end:value_end].strip(" ：:").strip() + if value: + fields.setdefault(label, value) + elif label not in empty_labels: + empty_labels.append(label) + return fields, empty_labels + + +def _clean_field_value(value: str | None) -> str | None: + """去掉字段值里的角标图标、复制残留与多余空白。""" + + if value is None: + return None + text = value.strip() + while text and text[-1] in "◎☆✦> 　": + text = text[:-1].rstrip() + return text or None + + +def parse_product_cell(text: str) -> dict[str, Any]: + """解析列表“商品信息”单元格：标题、规格、商家编码。""" + + lines = _clean_lines(text) + product: dict[str, Any] = { + "product_name": None, + "spec_name": None, + "product_code": None, + } + code_index = next( + (index for index, line in enumerate(lines) if MERCHANT_CODE_PATTERN.search(line)), + None, + ) + if code_index is not None: + match = MERCHANT_CODE_PATTERN.search(lines[code_index]) + if match: + product["product_code"] = match.group(1) + candidates = lines[: code_index if code_index is not None else len(lines)] + candidates = [line for line in candidates if not PRODUCT_TITLE_TAG_PATTERN.match(line)] + if candidates: + product["product_name"] = candidates[0] + if len(candidates) > 1: + product["spec_name"] = candidates[-1] + return product + + +def parse_list_row(text: str, cells: dict[str, str] | None = None) -> dict[str, Any]: + """把列表行文本（可选叠加按列单元格文本）解析为售后单字段。""" + + merged = text or "" + if cells: + merged = "\n".join([text or "", *cells.values()]) + row: dict[str, Any] = { + "aftersale_id": None, + "order_id": None, + "aftersale_type": None, + "status_tag": None, + "aftersale_status": None, + "apply_at": None, + "apply_reason": None, + "apply_quantity": None, + "pay_amount": None, + "refund_amount": None, + "product_name": None, + "spec_name": None, + "product_code": None, + "raw_text": text or "", + } + aftersale_match = AFTERSALE_ID_PATTERN.search(merged) + if not aftersale_match: + return row + row["aftersale_id"] = aftersale_match.group(1) + order_match = ORDER_ID_PATTERN.search(merged) + if order_match: + row["order_id"] = order_match.group(1) + + # 售后编号之后的紧邻片段是售后类型与状态角标（例如“退货退款 平台主动处理”）。 + tail = merged[aftersale_match.end() : aftersale_match.end() + 48].replace("\n", " ") + for word in AFTERSALE_TYPE_WORDS: + if word in tail: + row["aftersale_type"] = word + break + for word in STATUS_TAG_WORDS: + if word in tail: + row["status_tag"] = word + break + + for pattern, field in ( + (REFUND_AMOUNT_PATTERN, "refund_amount"), + (PAY_AMOUNT_PATTERN, "pay_amount"), + (APPLY_QUANTITY_PATTERN, "apply_quantity"), + (APPLY_REASON_PATTERN, "apply_reason"), + ): + match = pattern.search(merged) + if match: + row[field] = match.group(1).strip() + time_match = LIST_APPLY_TIME_PATTERN.search(merged) + if time_match: + row["apply_at"] = f"{time_match.group(1)} {time_match.group(2)}" + + product_cell = (cells or {}).get("商品信息") + if product_cell: + product = parse_product_cell(product_cell) + row["product_name"] = product["product_name"] + row["spec_name"] = product["spec_name"] + row["product_code"] = product["product_code"] + else: + code_match = MERCHANT_CODE_PATTERN.search(merged) + if code_match: + row["product_code"] = code_match.group(1) + title_match = re.search(r"(?:【直播】)?GYXX/[^\n]+", merged) + if title_match: + row["product_name"] = title_match.group(0).strip() + + status_cell = (cells or {}).get("售后状态") + if status_cell: + lines = [ + line + for line in _clean_lines(status_cell) + if line not in {"钱款去向"} and not line.endswith(">") + ] + row["aftersale_status"] = "\n".join(lines) or None + else: + row["aftersale_status"] = _status_after_apply_time(merged) + return row + + +def _status_after_apply_time(text: str) -> str | None: + """无单元格映射时，从“申请时间”后的第一个非链接行提取售后状态。""" + + lines = _clean_lines(text) + time_index = next( + (index for index, line in enumerate(lines) if LIST_APPLY_TIME_PATTERN.search(line)), + None, + ) + if time_index is None: + return None + for line in lines[time_index + 1 :]: + if line in {"钱款去向", "未介入", "查看详情"}: + continue + if line.endswith(">"): + continue + if re.match(r"^(买家退货|商家退货|订单发货)", line): + continue + return line + return None + + +def parse_detail_product_block(text: str) -> dict[str, Any]: + """解析详情页“交易信息”商品块：商品名称、商品 ID、颜色分类、商家编码。""" + + lines = _clean_lines(text) + product: dict[str, Any] = { + "product_name": None, + "product_id": None, + "spec_name": None, + "product_code": None, + } + id_index = next( + (index for index, line in enumerate(lines) if PRODUCT_ID_PATTERN.search(line)), + None, + ) + if id_index is not None: + product["product_id"] = PRODUCT_ID_PATTERN.search(lines[id_index]).group(1) + for line in reversed(lines[:id_index]): + if PRODUCT_TITLE_TAG_PATTERN.match(line): + continue + if len(line) < 6 or MERCHANT_CODE_PATTERN.search(line): + continue + product["product_name"] = line + break + color_match = COLOR_PATTERN.search(text or "") + if color_match: + spec = color_match.group(1) + spec = spec.split("商家编码")[0] + spec = re.split(r"\s{2,}", spec)[0] + product["spec_name"] = spec.strip(" 　") or None + code_match = MERCHANT_CODE_PATTERN.search(text or "") + if code_match: + product["product_code"] = code_match.group(1) + return product + + +def parse_detail_aftersale_block(text: str) -> dict[str, str | None]: + """解析详情页“售后信息”块的标签值字段（标签与值同行或后续行均可）。""" + + fields: dict[str, str | None] = {} + pending_label: str | None = None + for line in _clean_lines(text): + inline_fields, empty_labels = _split_line_labels(DETAIL_LABELS, line) + if inline_fields or empty_labels: + for label, value in inline_fields.items(): + fields.setdefault(label, _clean_field_value(value)) + pending_label = empty_labels[0] if empty_labels else None + continue + if pending_label: + current = fields.get(pending_label) + merged = f"{current}\n{line}".strip() if current else line + fields[pending_label] = merged + continue + return fields + + +def parse_negotiation_entries( + panel_text: str, + evidence_by_entry: dict[int, list[str]] | None = None, +) -> list[dict[str, Any]]: + """把“协商记录”面板文本解析为逐条协商动作记录。 + + 每条记录以角色角标（消费者/系统/商家）开头，其后是动作描述、时间戳 + 和“标签 值”字段区（标签与值同行或分块两行均可）。退货原因、退款说明、 + 操作人提升为独立字段，凭证信息图片按 DOM 顺序映射到所属条目。 + """ + + lines = _clean_lines(panel_text) + entries: list[dict[str, Any]] = [] + current: dict[str, Any] | None = None + for line in lines: + if line in NEGOTIATION_HEADER_WORDS or line.startswith("时间排序"): + continue + if NEGOTIATION_TIME_PATTERN.match(line): + if current is not None: + current["happened_at_label"] = line + continue + role = next((item for item in ROLE_WORDS if line == item), None) + action: str | None = None + if role is None: + for item in ROLE_WORDS: + if line.startswith(item + " "): + role = item + action = line[len(item) :].strip(" ，,、") + break + if role is not None: + if current is not None: + entries.append(current) + current = { + "actor_role": role, + "action_parts": [action] if action else [], + "fields": {}, + "raw_lines": [line], + "happened_at_label": None, + "pending_label": None, + } + continue + if current is None: + continue + current["raw_lines"].append(line) + inline_fields, empty_labels = _split_line_labels(NEGOTIATION_LABELS, line) + if inline_fields or empty_labels: + for label, value in inline_fields.items(): + current["fields"].setdefault(label, _clean_field_value(value)) + current["pending_label"] = empty_labels[0] if empty_labels else None + continue + pending = current.get("pending_label") + if pending: + existing = current["fields"].get(pending) + current["fields"][pending] = f"{existing}\n{line}".strip() if existing else line + continue + if current.get("happened_at_label") is None: + current["action_parts"].append(line) + if current is not None: + entries.append(current) + + parsed: list[dict[str, Any]] = [] + for seq, entry in enumerate(entries): + fields: dict[str, str] = dict(entry["fields"]) + operator = fields.pop("操作人", None) + return_reason = fields.pop("退货原因", None) + refund_note = fields.pop("退款说明", None) + evidence: list[str] = [] + for source in (evidence_by_entry or {}).get(seq, []): + if source not in evidence: + evidence.append(source) + happened_at = None + if entry.get("happened_at_label"): + happened_at = datetime.strptime( + entry["happened_at_label"], "%Y-%m-%d %H:%M:%S" + ).replace(tzinfo=SHANGHAI) + parsed.append( + { + "entry_seq": seq, + "actor_role": entry["actor_role"], + "action_text": " ".join(entry["action_parts"]).strip() or None, + "happened_at": happened_at, + "operator": operator, + "return_reason": return_reason, + "refund_note": refund_note, + "evidence_urls": evidence, + "extra_fields": fields, + "raw_text": "\n".join(entry["raw_lines"]), + } + ) + return parsed + + +def parse_apply_time(value: str | None) -> datetime | None: + """把列表/详情页申请时间解析为上海时区时间。""" + + text = (value or "").strip() + for pattern, fmt in ( + (LIST_APPLY_TIME_PATTERN, "%Y/%m/%d %H:%M:%S"), + (DETAIL_APPLY_TIME_PATTERN, "%Y-%m-%d %H:%M:%S"), + ): + match = pattern.search(text) + if match: + return datetime.strptime( + f"{match.group(1)} {match.group(2)}", fmt + ).replace(tzinfo=SHANGHAI) + try: + return datetime.strptime(text, fmt).replace(tzinfo=SHANGHAI) + except ValueError: + continue + return None + + +_LIST_ROW_SCAN_JS = """() => { + const visible = (el) => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + let headers = []; + const headerRow = Array.from(document.querySelectorAll('tr')) + .find((tr) => { + const text = tr.innerText || ''; + return text.includes('商品信息') && text.includes('售后状态'); + }); + if (headerRow) { + headers = Array.from(headerRow.querySelectorAll('th, td')) + .map((cell) => (cell.innerText || '').trim()); + } + const candidates = Array.from(document.querySelectorAll('tr')) + .filter((tr) => /售后编号/.test(tr.innerText || '')); + // 每条售后渲染成一对相邻的 tr：level-0 行带“售后编号”条码条， + // 紧随其后的 level-1 兄弟行带商品/售后信息/申请时间等全部业务字段 + // （但不含售后编号标签）。因此遍历 level-0 行并并入其兄弟行文本。 + const groups = new Map(); + for (const tr of candidates) { + const match = (tr.innerText || '').match(/售后编号\\s*(\\d{14,25})/); + if (!match) continue; + const id = match[1]; + if (!groups.has(id)) groups.set(id, {id, texts: [], cellsList: []}); + const group = groups.get(id); + group.texts.push((tr.innerText || '').trim()); + group.cellsList.push(Array.from(tr.querySelectorAll(':scope > td')) + .map((td) => (td.innerText || '').trim())); + const sibling = tr.nextElementSibling; + if (sibling && sibling.tagName === 'TR' + && !/售后编号/.test(sibling.innerText || '')) { + group.texts.push((sibling.innerText || '').trim()); + group.cellsList.push(Array.from(sibling.querySelectorAll(':scope > td')) + .map((td) => (td.innerText || '').trim())); + } + } + return { + headers, + rows: Array.from(groups.values()) + .filter((group) => { + const tr = candidates.find((item) => + (item.innerText || '').includes(group.id)); + return tr ? visible(tr) : false; + }) + .map((group) => ({ + id: group.id, + text: group.texts.join('\\n'), + cellsList: group.cellsList, + })), + }; +}""" + + +def map_row_cells(headers: list[str], cells: list[str]) -> dict[str, str]: + """按表头名把行单元格文本映射为字典；长度不一致时放弃映射。""" + + if len(headers) != len(cells): + return {} + mapping: dict[str, str] = {} + for header, cell in zip(headers, cells): + if header in ROW_HEADER_NAMES: + mapping[header] = cell + return mapping + + +def read_current_page_rows(page: Page) -> list[dict[str, Any]]: + """读取当前页全部售后行并解析字段（按售后编号分组、合并成对行）。""" + + payload = page.evaluate(_LIST_ROW_SCAN_JS) + headers = [str(item).strip() for item in payload.get("headers") or []] + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for entry in payload.get("rows") or []: + aftersale_id = str(entry.get("id") or "") + text = str(entry.get("text") or "") + if not aftersale_id or aftersale_id in seen: + continue + # 取信息最多的那组 td 做表头映射（level-0 主行通常最完整）。 + cells: dict[str, str] = {} + for cell_list in entry.get("cellsList") or []: + mapping = map_row_cells(headers, list(cell_list)) + if len(mapping) > len(cells): + cells = mapping + row = parse_list_row(text, cells) + if not row.get("aftersale_id"): + continue + seen.add(aftersale_id) + row["raw_cells"] = cells + rows.append(row) + return rows + + +def wait_current_page_rows(page: Page, timeout: int = 30) -> list[dict[str, Any]]: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + rows = read_current_page_rows(page) + if rows: + return rows + page.wait_for_timeout(700) + return read_current_page_rows(page) + + +_TABLE_SCROLL_INFO_JS = """() => { + const rows = Array.from(document.querySelectorAll('tr')) + .filter((tr) => /售后编号/.test(tr.innerText || '')); + let el = rows.length ? rows[0].parentElement : null; + while (el && el !== document.body) { + if (el.scrollHeight > el.clientHeight + 50) break; + el = el.parentElement; + } + if (!el || el === document.body) return {found: 0}; + return { + found: 1, + top: el.scrollTop, + height: el.scrollHeight, + client: el.clientHeight, + }; +}""" + + +def scan_current_page_rows( + page: Page, max_steps: int = 40, *, settle_ms: int = 250 +) -> list[dict[str, Any]]: + """深度扫描当前页行（虚拟滚动时逐屏滚动收集），返回按出现顺序去重的行。""" + + seen: dict[str, dict[str, Any]] = {} + stale = 0 + last_top = -1.0 + for _ in range(max_steps): + for row in read_current_page_rows(page): + seen.setdefault(row["aftersale_id"], row) + info = page.evaluate(_TABLE_SCROLL_INFO_JS) + if not info.get("found"): + break + at_bottom = info["top"] + info["client"] >= info["height"] - 8 + if at_bottom and info["top"] == last_top: + stale += 1 + if stale >= 2: + break + else: + stale = 0 + last_top = info["top"] + if at_bottom and info["height"] <= info["client"] + 8: + break + page.evaluate( + """(step) => { + const rows = Array.from(document.querySelectorAll('tr')) + .filter((tr) => /售后编号/.test(tr.innerText || '')); + let el = rows.length ? rows[0].parentElement : null; + while (el && el !== document.body) { + if (el.scrollHeight > el.clientHeight + 50) break; + el = el.parentElement; + } + if (!el || el === document.body) return; + el.scrollTop = Math.min(el.scrollTop + step, el.scrollHeight); + el.dispatchEvent(new Event('scroll', {bubbles: true})); + }""", + max(float(info["client"]) * 0.8, 200.0), + ) + page.wait_for_timeout(settle_ms) + page.evaluate( + """() => { + const rows = Array.from(document.querySelectorAll('tr')) + .filter((tr) => /售后编号/.test(tr.innerText || '')); + let el = rows.length ? rows[0].parentElement : null; + while (el && el !== document.body) { + if (el.scrollHeight > el.clientHeight + 50) break; + el = el.parentElement; + } + if (!el || el === document.body) return; + el.scrollTop = 0; + el.scrollLeft = 0; + el.dispatchEvent(new Event('scroll', {bubbles: true})); + }""" + ) + page.wait_for_timeout(200) + return list(seen.values()) + + +def _row_element(page: Page, aftersale_id: str) -> Any: + row = page.locator("tr").filter(has_text=aftersale_id).first + if row.count(): + return row + marker = page.get_by_text(aftersale_id, exact=True).first + return marker.locator("xpath=ancestor::tr[1]") + + +_DETAIL_NEGOTIATION_JS = """() => { + const visible = (el) => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + const textOf = (el) => (el.innerText || '').trim(); + const ROLE_WORDS = ['消费者', '系统', '商家']; + const panels = Array.from(document.querySelectorAll('div, section')) + .filter(visible) + .filter((el) => textOf(el).includes('协商记录')) + .filter((el) => ROLE_WORDS.some((role) => textOf(el).includes(role))); + panels.sort((a, b) => textOf(a).length - textOf(b).length); + const panel = panels[0]; + if (!panel) return null; + let badges = Array.from(panel.querySelectorAll('*')).filter( + (el) => visible(el) && el.children.length === 0 + && ROLE_WORDS.includes(textOf(el)) + ); + if (!badges.length) { + badges = Array.from(panel.querySelectorAll('*')).filter( + (el) => visible(el) && ROLE_WORDS.includes(textOf(el)) + ); + } + badges.sort((a, b) => ( + a.compareDocumentPosition(b) & Node.DOCUMENT_POSITION_FOLLOWING ? -1 : 1 + )); + const images = []; + for (const img of panel.querySelectorAll('img')) { + if (!visible(img)) continue; + const rect = img.getBoundingClientRect(); + if (rect.width < 40 || rect.height < 40) continue; + const src = img.currentSrc || img.getAttribute('src') || ''; + if (!src || src.startsWith('data:')) continue; + let entryIndex = 0; + for (let i = 0; i < badges.length; i += 1) { + if (badges[i].compareDocumentPosition(img) & Node.DOCUMENT_POSITION_FOLLOWING) { + entryIndex = i; + } else { + break; + } + } + images.push({entryIndex, src}); + } + return {text: textOf(panel), images}; +}""" + +_DETAIL_PRODUCT_JS = """() => { + const visible = (el) => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + const textOf = (el) => (el.innerText || '').trim(); + const candidates = Array.from(document.querySelectorAll('div, section')) + .filter(visible) + .filter((el) => /ID\\s*[:：]\\s*\\d{6,}/.test(textOf(el))) + .filter((el) => textOf(el).includes('商家编码')); + candidates.sort((a, b) => textOf(a).length - textOf(b).length); + return candidates[0] ? textOf(candidates[0]) : ''; +}""" + +_DETAIL_AFTERSALE_JS = """() => { + const visible = (el) => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + const textOf = (el) => (el.innerText || '').trim(); + const candidates = Array.from(document.querySelectorAll('div, section')) + .filter(visible) + .filter((el) => textOf(el).includes('售后原因')) + .filter((el) => textOf(el).includes('售后编号')); + candidates.sort((a, b) => textOf(a).length - textOf(b).length); + return candidates[0] ? textOf(candidates[0]) : ''; +}""" + + +def expand_detail_panels(page: Page, max_rounds: int = 6) -> None: + """点击详情页全部“展开全部”，展开完整协商记录与售后信息。""" + + for _ in range(max_rounds): + clicked = False + buttons = page.locator("text=展开全部") + for index in range(buttons.count()): + button = buttons.nth(index) + try: + if button.is_visible(): + button.click(timeout=2_000) + clicked = True + page.wait_for_timeout(400) + except Exception: + continue + if not clicked: + return + + +def extract_detail( + page: Page, expected_aftersale_id: str | None = None +) -> dict[str, Any]: + """读取售后详情新页签：协商记录、交易信息商品块与售后信息块。""" + + page.get_by_text("协商记录").first.wait_for(state="visible", timeout=20_000) + page.wait_for_timeout(600) + if expected_aftersale_id: + body_text = _page_text(page, timeout=10_000) + if expected_aftersale_id not in body_text: + raise RuntimeError( + f"详情页与目标售后单不一致，期望 {expected_aftersale_id}" + ) + expand_detail_panels(page) + + panel = page.evaluate(_DETAIL_NEGOTIATION_JS) + if not panel or not (panel.get("text") or "").strip(): + raise RuntimeError("详情页未找到协商记录面板") + evidence_by_entry: dict[int, list[str]] = {} + for image in panel.get("images") or []: + evidence_by_entry.setdefault(int(image.get("entryIndex") or 0), []).append( + str(image.get("src") or "") + ) + entries = parse_negotiation_entries(str(panel.get("text")), evidence_by_entry) + if not entries: + raise RuntimeError("详情页协商记录未解析出任何条目") + + product_text = str(page.evaluate(_DETAIL_PRODUCT_JS) or "") + product = parse_detail_product_block(product_text) + aftersale_fields = parse_detail_aftersale_block( + str(page.evaluate(_DETAIL_AFTERSALE_JS) or "") + ) + return { + "entries": entries, + "product": product, + "aftersale_fields": aftersale_fields, + "panel_text": str(panel.get("text")), + "product_text": product_text, + } + + +def _reset_table_horizontal_scroll(page: Page) -> None: + """把表格横向滚动复位到最左，避免右侧“操作”列被虚拟化卸载。""" + + page.evaluate( + """() => { + const rows = Array.from(document.querySelectorAll('tr')) + .filter((tr) => /售后编号/.test(tr.innerText || '')); + let el = rows.length ? rows[0].parentElement : null; + while (el && el !== document.body) { + if (el.scrollWidth > el.clientWidth + 50 + || el.scrollHeight > el.clientHeight + 50) { + break; + } + el = el.parentElement; + } + if (!el || el === document.body) return; + if (el.scrollLeft) { + el.scrollLeft = 0; + el.dispatchEvent(new Event('scroll', {bubbles: true})); + } + }""" + ) + + +def _js_click_row_detail(page: Page, aftersale_id: str) -> bool: + """用 JS 按顺序对齐点击目标行的“查看详情”。 + + aurora 表格把每条售后渲染成成对的两行：售后编号在信息行，查看详情 + 在操作行，两者不在同一个 tr。数据行与详情链接各自按文档顺序排列 + （每行恰好一个链接），因此按“第 N 条数据行 → 第 N 个链接”对齐点击。 + """ + + return page.evaluate( + """(aid) => { + const visible = (el) => { + const r = el.getBoundingClientRect(); + return r.width > 0 && r.height > 0; + }; + const links = Array.from(document.querySelectorAll('a')) + .filter((el) => (el.innerText || '').trim() === '查看详情' + && visible(el)); + for (const link of links) { + const tr = link.closest('tr'); + if (!tr) continue; + // aurora 表格把每条售后渲染成成对的两行：售后编号在信息行， + // 查看详情在相邻的操作行；命中任一行包含目标编号即可点击。 + const candidates = [tr, tr.previousElementSibling, + tr.nextElementSibling]; + const hit = candidates.some((row) => row + && /售后编号/.test(row.innerText || '') + && (row.innerText || '').includes(aid)); + if (hit) { + link.scrollIntoView({block: 'center'}); + // 页面校验事件顺序，单发 click 不会触发新页签；必须 + // 按真实用户节奏派发完整鼠标事件序列。 + const fire = (type) => link.dispatchEvent(new MouseEvent(type, { + bubbles: true, cancelable: true, view: window})); + fire('mouseover'); + fire('mousemove'); + fire('mousedown'); + fire('focus'); + fire('mouseup'); + fire('click'); + return true; + } + } + return false; + }""", + aftersale_id, + ) + + +def open_detail_page(context: Any, page: Page, aftersale_id: str) -> Page: + """点击行内“查看详情”，等待新页签打开并返回详情页。""" + + before = set(context.pages) + deadline = time.monotonic() + 20 + clicked = False + while time.monotonic() < deadline: + _reset_table_horizontal_scroll(page) + if _js_click_row_detail(page, aftersale_id): + clicked = True + break + page.wait_for_timeout(600) + if not clicked: + row = _row_element(page, aftersale_id) + link = row.locator("text=查看详情").first + try: + row.scroll_into_view_if_needed(timeout=5_000) + link.click(timeout=10_000, force=True) + except Exception as exc: + raise RuntimeError( + f"售后单 {aftersale_id} 的“查看详情”入口不可点击: {exc}" + ) from exc + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + new_pages = [item for item in context.pages if item not in before] + if new_pages: + detail = new_pages[0] + try: + detail.wait_for_load_state("domcontentloaded", timeout=30_000) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + return detail + page.wait_for_timeout(300) + raise RuntimeError(f"售后单 {aftersale_id} 点击查看详情后未打开新页签") + + +def is_login_page(page: Page) -> bool: + url = (page.url or "").casefold() + if any(marker in url for marker in LOGIN_URL_MARKERS): + return True + try: + for marker in LOGIN_TEXT_MARKERS: + if page.locator(f"text={marker}").count() > 0: + return True + except Exception: + return False + return False + + +def _page_text(page: Page, timeout: int = 15_000) -> str: + try: + return page.locator("body").inner_text(timeout=timeout) + except Exception: + return "" + + +def _configured_identity_markers( + environment: dict[str, str] | None = None, +) -> tuple[str, ...]: + values = os.environ if environment is None else environment + raw = values.get("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS", "").strip() + if raw: + try: + markers = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS 不是有效 JSON") from exc + if isinstance(markers, list) and all(isinstance(item, str) for item in markers): + return tuple(item.strip() for item in markers if item.strip()) + return DEFAULT_IDENTITY_MARKERS + + +def verify_shop_identity( + page: Page, + *, + environment: dict[str, str] | None = None, +) -> None: + """确认登录后的店铺身份，避免把共享抖音 Cookie 用到错误店铺。""" + + markers = _configured_identity_markers(environment) + if not markers: + return + page.goto(FXG_IDENTITY_URL, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=20_000) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + body_text = _page_text(page) + if any(marker in body_text for marker in markers): + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=20_000) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + return + if is_login_page(page): + break + page.wait_for_timeout(1_000) + raise RuntimeError( + "抖音登录态未出现配置的店铺身份标识；" + f"期望标识={markers!r}，当前页面未匹配。" + "请用 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + + +def ensure_douyin_login( + page: Page, + *, + environment: dict[str, str], + headless: bool, + login_timeout: int, +) -> None: + """打开售后工作台；有效 Cookie 直接跳过登录，否则等待人工登录。""" + + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=20_000) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + page.wait_for_timeout(2_000) + + if not is_login_page(page): + print(" [OK] 已复用抖音账号 Cookie 登录态，跳过登录") + return + + policy = current_acceptance_policy(environment) + if policy.enabled: + policy.record( + "cookie_skipped", + operation="shop_intelligence.dy_after_sales_quality.live_session", + details={ + "status": "SKIPPED_COOKIE", + "reason": "douyin browser session redirected to login", + }, + ) + print("[SKIPPED_COOKIE] 抖音浏览器会话跳转到了登录页") + raise DouyinSessionExpiredError("抖音浏览器会话跳转到了登录页") + + if headless: + raise RuntimeError( + "抖音 Cookie 已失效且当前为无头模式，无法扫码登录；" + "请先执行 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + + print("=" * 60) + print("【请登录】请在弹出的浏览器窗口中完成抖音店铺登录") + print("提示：扫码/短信/密码均可，登录成功后页面会自动跳转") + print("=" * 60) + deadline = time.monotonic() + login_timeout + while time.monotonic() < deadline: + if not is_login_page(page): + print(f" [OK] 登录成功: {page.url}") + page.wait_for_timeout(3_000) + return + page.wait_for_timeout(1_000) + raise RuntimeError( + f"登录在 {login_timeout} 秒内未完成，当前页面: {page.url}" + ) + + +class DouyinSessionExpiredError(RuntimeError): + """The reused cookie state reached an interactive login page.""" + + +def dismiss_blocking_modals(page: Page) -> None: + """关闭遮挡页面的公告/引导弹窗（保留含表单输入的对话框）。""" + + for _ in range(6): + closed = False + for selector in ( + '.auxo-modal-wrap:visible .auxo-modal-close:visible', + '.auxo-modal-wrap:visible [class*="close"]:visible', + '.auxo-modal-wrap:visible button:has-text("知道"):visible', + '.auxo-modal-wrap:visible button:has-text("暂不"):visible', + '.auxo-modal-wrap:visible button:has-text("下次再说"):visible', + '.auxo-modal-wrap:visible button:has-text("取消"):visible', + '.auxo-modal-wrap:visible button:has-text("关闭"):visible', + '[role="dialog"]:visible [class*="close"]:visible', + ): + button = page.locator(selector).first + try: + if button.count() and button.is_visible(): + button.click(timeout=2_000) + page.wait_for_timeout(400) + closed = True + break + except Exception: + continue + if not closed: + break + removed = page.evaluate( + """() => { + let removed = 0; + const visible = (el) => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + for (const el of document.querySelectorAll( + '.auxo-modal-wrap, .auxo-modal-mask, [role="dialog"]' + )) { + if (!visible(el)) continue; + if (el.querySelector('input, textarea, form')) continue; + el.remove(); + removed += 1; + } + return removed; + }""" + ) + if removed: + print(f" [INFO] 已移除 {removed} 个遮挡弹窗") + page.wait_for_timeout(300) + + +def _filters_panel_expanded(page: Page) -> bool: + """更多筛选面板展开后才会渲染“申请时间”的开始日期输入框。""" + + field = page.locator('input[placeholder="开始日期"]').first + try: + return bool(field.count()) and field.is_visible() + except Exception: + return False + + +def open_more_filters(page: Page) -> None: + """展开“更多筛选”面板（申请时间控件已可见时跳过）。""" + + dismiss_blocking_modals(page) + if _filters_panel_expanded(page): + return + button = page.locator( + 'button:has-text("更多筛选"), [class*="button"]:has-text("更多筛选")' + ).first + if not button.count(): + button = page.get_by_text("更多筛选", exact=True).first + if not button.count(): + raise RuntimeError("未找到“更多筛选”入口") + button.click(timeout=5_000) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if _filters_panel_expanded(page): + return + page.wait_for_timeout(400) + raise RuntimeError("点击更多筛选后未出现申请时间筛选控件") + + +def set_apply_reason(page: Page, reason: str = APPLY_REASON) -> None: + """在“申请原因”下拉中精确选择指定原因并硬性校验。""" + + if reason not in APPLY_REASON_OPTIONS or reason == "全部": + raise ValueError(f"不支持的申请原因: {reason!r}") + + dismiss_blocking_modals(page) + control = page.locator( + 'div:has(> :text-is("申请原因")) [class*="select"], ' + '[class*="select"]:has-text("申请原因")' + ).first + try: + visible = bool(control.count()) and control.is_visible() + except Exception: + visible = False + if not visible: + label = page.get_by_text("申请原因", exact=True).first + control = label.locator( + "xpath=ancestor::*[self::div or self::td][1]" + ).locator('[class*="select"], input').first + control.click() + page.wait_for_timeout(800) + + desired = {label: (label == reason) for label in APPLY_REASON_OPTIONS} + state = page.evaluate( + """(desired) => { + const visible = (el) => { + if (!el) return false; + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + const labels = Object.keys(desired); + const panels = Array.from(document.querySelectorAll('div, ul, section')) + .filter(visible) + .filter((el) => labels.every((label) => + (el.innerText || '').includes(label))) + .sort((a, b) => (a.innerText || '').length - (b.innerText || '').length); + const panel = panels[0]; + if (!panel) return {found: false}; + const rows = Array.from(panel.querySelectorAll('li, [class*="option"], [class*="Option"], label')) + .filter(visible) + .filter((el) => labels.includes((el.innerText || '').trim())); + for (const row of rows) { + const label = (row.innerText || '').trim(); + if (desired[label]) { + const target = row.querySelector('input[type="checkbox"]') + || row.querySelector('input[type="radio"]') || row; + target.click(); + return {found: true, clicked: label}; + } + } + return {found: true, clicked: null}; + }""", + desired, + ) + if not state.get("found"): + page.keyboard.press("Escape") + raise RuntimeError("申请原因下拉面板未打开或未包含全部选项") + if not state.get("clicked"): + page.keyboard.press("Escape") + raise RuntimeError(f"申请原因下拉中未找到选项 {reason!r}") + page.keyboard.press("Escape") + page.wait_for_timeout(400) + + control_text = "" + try: + control_text = control.inner_text(timeout=5_000) + except Exception: + control_text = "" + if control_text and reason not in control_text: + raise RuntimeError( + f"申请原因控件未显示所选原因，期望包含 {reason!r}，实际 {control_text!r}" + ) + + +def _open_date_input_panel(page: Page, placeholder: str) -> bool: + """点击日期输入框打开日历面板（输入框是 readonly，只能点选日期）。""" + + field = page.locator(f'input[placeholder="{placeholder}"]').first + if not field.count() or not field.is_visible(): + return False + try: + field.click(timeout=5_000) + except Exception: + return False + page.wait_for_timeout(500) + return True + + +def _click_date_cell(page: Page, target: date) -> bool: + """在打开的日历面板中按月导航并点击目标日期格子。 + + 头部有“上一年/下一年”（super）与“上个月/下个月”两档按钮；必须排除 + super 按钮，否则会在年份间震荡永远到不了目标月。 + """ + + title = target.isoformat() + for _ in range(48): + cell = page.locator( + f'td[title="{title}"]:visible, ' + f'div[title="{title}"]:visible, ' + f'span[title="{title}"]:visible' + ).first + if cell.count() and cell.is_visible(): + cell.click() + page.wait_for_timeout(300) + return True + panel_text = "" + for selector in ( + '[class*="picker-dropdown"]:visible', + '[class*="panel-container"]:visible', + '[class*="picker"] :text-matches("\\\\d{4}年\\\\d{1,2}月"):visible', + ): + locator = page.locator(selector).last + if locator.count(): + try: + text = locator.inner_text(timeout=2_000) + if text and "月" in text: + panel_text = text + break + except Exception: + continue + months = re.findall(r"(\d{4})\s*年\s*(\d{1,2})\s*月", panel_text or "") + if not months: + return False + displayed = [int(year) * 12 + int(month) - 1 for year, month in months] + target_index = target.year * 12 + target.month - 1 + go_prev = target_index < min(displayed) + # 限定在日历头部按钮上（aurora-picker-header-prev/next-btn）； + # 页面自身的分页器类名同样含 prev/next，必须排除。 + button = page.locator( + '[class*="picker-header-prev"]:visible' + if go_prev + else '[class*="picker-header-next"]:visible' + ).first + if not button.count(): + return False + try: + button.click(timeout=3_000) + except Exception: + return False + page.wait_for_timeout(250) + return False + + +def _input_value_matches(page: Page, placeholder: str, target: date) -> bool: + field = page.locator(f'input[placeholder="{placeholder}"]').first + if not field.count(): + return False + # 控件会自动补全时间部分（如 2026/03/21 00:00:00），按日期前缀比较。 + actual = re.sub(r"\D", "", field.input_value().strip()) + return actual.startswith(target.strftime("%Y%m%d")) + + +def set_apply_time_range(page: Page, start: date, end: date) -> None: + """设置“申请时间”范围的开始/结束日期并校验输入值。 + + 日期输入框是 readonly（点击弹出日历面板），因此通过日历格子点选： + 先点开始日期，再点结束日期；范围跨月时面板按月导航到位后点选。 + 选满起止日期后面板一般自动收起，仅在其仍打开时按 Escape。 + """ + + if not _open_date_input_panel(page, "开始日期"): + raise RuntimeError("申请时间的开始日期输入框不可见，无法打开日历面板") + if not _click_date_cell(page, start): + page.keyboard.press("Escape") + raise RuntimeError(f"日历面板无法选择开始日期 {start.isoformat()}") + if not _click_date_cell(page, end): + page.keyboard.press("Escape") + raise RuntimeError(f"日历面板无法选择结束日期 {end.isoformat()}") + page.wait_for_timeout(800) + dropdown = page.locator('[class*="picker-dropdown"]:visible').first + try: + if dropdown.count() and dropdown.is_visible(): + # range-join 面板需要点“确定”提交；直接 Escape 会取消未提交的选择。 + ok_button = dropdown.locator('button:has-text("确定"):visible').first + if ok_button.count(): + ok_button.click(timeout=3_000) + page.wait_for_timeout(600) + if dropdown.count() and dropdown.is_visible(): + page.keyboard.press("Escape") + except Exception: + pass + page.wait_for_timeout(400) + + actual_ok = _input_value_matches(page, "开始日期", start) and _input_value_matches( + page, "结束日期", end + ) + if not actual_ok: + start_value = page.locator('input[placeholder="开始日期"]').first + end_value = page.locator('input[placeholder="结束日期"]').first + actual = ( + start_value.input_value().strip() if start_value.count() else "?", + end_value.input_value().strip() if end_value.count() else "?", + ) + raise RuntimeError( + f"日期选择未生效，期望 ({start.isoformat()}, {end.isoformat()})，实际 {actual}" + ) + + +def verify_apply_reason_selected(page: Page, reason: str = APPLY_REASON) -> None: + """确认“申请原因”控件仍显示所选原因，避免查询前筛选被意外清空。""" + + control = page.locator( + 'div:has(> :text-is("申请原因")) [class*="select"], ' + '[class*="select"]:has-text("申请原因")' + ).first + try: + control_text = control.inner_text(timeout=5_000) if control.count() else "" + except Exception: + control_text = "" + if not control_text or reason not in control_text: + raise RuntimeError( + f"申请原因筛选未保持选中，期望包含 {reason!r}，实际 {control_text!r}" + ) + + +def set_page_size(page: Page, size: int = 50) -> int: + """把分页大小调大以减少翻页次数；失败时保持当前页大小。""" + + try: + selector = page.locator( + '[class*="pagination"] [class*="select"]:has-text("条/页"), ' + '[class*="pagination"] :text-matches("\\\\d+\\\\s*条\\\\s*/\\\\s*页")' + ).last + if not selector.count() or not selector.is_visible(): + return 0 + selector.click() + page.wait_for_timeout(600) + option = page.locator( + f'[class*="option"]:has-text("{size} 条/页"), ' + f'li:has-text("{size} 条/页")' + ).last + if not option.count() or not option.is_visible(): + page.keyboard.press("Escape") + return 0 + option.click() + page.wait_for_timeout(1_500) + return size + except Exception as exc: + print(f" [WARN] 设置分页大小失败，保持默认: {exc}") + return 0 + + +def apply_query(page: Page) -> int: + """点击“查询”并返回筛选后的售后单总数。""" + + dismiss_blocking_modals(page) + page.get_by_role("button", name="查询", exact=True).first.click() + page.wait_for_timeout(2_000) + try: + page.wait_for_load_state("networkidle", timeout=15_000) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + deadline = time.monotonic() + 30 + last_text = "" + while time.monotonic() < deadline: + body_text = _page_text(page, timeout=5_000) + last_text = body_text + if TOTAL_COUNT_PATTERN.search(body_text or ""): + return parse_total_count(body_text) + page.wait_for_timeout(500) + raise RuntimeError(f"查询后未识别售后单总数: {last_text[:200]!r}") + + +def _active_page_number(page: Page) -> int | None: + try: + text = page.locator( + '.auxo-pagination-item-active:visible, ' + '[class*="pagination"] [class*="active"]:visible' + ).first.inner_text(timeout=3_000) + except Exception: + return None + match = re.search(r"\d+", text or "") + return int(match.group()) if match else None + + +def _pagination_info(page: Page) -> dict[str, int | None]: + try: + data = page.evaluate( + """() => { + const root = document.querySelector( + '.auxo-pagination, [class*="pagination"]' + ); + if (!root) return {}; + const text = root.innerText || ''; + const active = root.querySelector( + '.auxo-pagination-item-active, [class*="item-active"], ' + + 'li[class*="active"]' + ); + const pageNums = Array.from( + root.querySelectorAll('.auxo-pagination-item, li[title], [class*="item"]') + ).map((el) => Number((el.textContent || '').trim())) + .filter((n) => Number.isInteger(n) && n > 0); + return { + active: active ? Number((active.textContent || '').trim()) : null, + maxPage: pageNums.length ? Math.max(...pageNums) : null, + }; + }""" + ) + except Exception: + return {"active": None, "max_page": None} + active = data.get("active") + max_page = data.get("maxPage") + return { + "active": int(active) if isinstance(active, (int, float)) else None, + "max_page": int(max_page) if isinstance(max_page, (int, float)) else None, + } + + +def _next_page_disabled(page: Page) -> bool: + try: + return bool( + page.evaluate( + """() => { + const btn = document.querySelector( + '.auxo-pagination-next, [class*="pagination"] li[title="下一页"], ' + + '[class*="pagination"] [class*="next"]' + ); + if (!btn) return false; + const cls = btn.className || ''; + return cls.includes('disabled') + || btn.getAttribute('aria-disabled') === 'true' + || btn.disabled === true; + }""" + ) + ) + except Exception: + return False + + +def goto_next_page(page: Page, current_first_aftersale_id: str) -> bool: + """翻到下一页并等待表格首行变化；已是最后一页时返回 False。 + + 优先用 title="下一页" 精确匹配（aurora/auxo 一致），避免误点 + “向后 N 页”的跳页按钮；点击未生效时重试最多 3 次。 + """ + + dismiss_blocking_modals(page) + if _next_page_disabled(page): + print(" [INFO] 已到最后一页（下一页禁用）") + return False + before_number = _active_page_number(page) + for attempt in range(3): + next_button = page.locator( + '[class*="pagination"] li[title="下一页"]:visible, ' + 'li.aurora-pagination-next:visible, ' + '.auxo-pagination-next:visible' + ).first + if not next_button.count(): + raise RuntimeError("分页“下一页”按钮不可见，无法继续采集") + try: + next_button.click(timeout=5_000) + except Exception: + next_button.click(force=True) + deadline = time.monotonic() + 25 + while time.monotonic() < deadline: + rows = read_current_page_rows(page) + if rows and rows[0]["aftersale_id"] != current_first_aftersale_id: + return True + after_number = _active_page_number(page) + if ( + before_number is not None + and after_number is not None + and after_number != before_number + ): + # 页码已变但表格尚未渲染完，继续等待。 + page.wait_for_timeout(500) + continue + page.wait_for_timeout(500) + print(f" [WARN] 第 {attempt + 1}/3 次翻页后表格未更新，重试") + page.wait_for_timeout(1_000) + if _next_page_disabled(page): + print(" [INFO] 已到最后一页（点击后下一页禁用）") + return False + raise RuntimeError("翻页后表格内容未更新") + + +def normalize_order( + row: dict[str, Any], + detail: dict[str, Any], + *, + account: str, + source_url: str, + captured_at: datetime, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """把列表行与详情页数据规范化为入库记录（售后单 + 协商记录）。""" + + aftersale_id = str(row.get("aftersale_id") or "").strip() + if not aftersale_id: + raise RuntimeError(f"售后行缺少售后编号: {row.get('raw_text', '')[:80]!r}") + entries = detail.get("entries") or [] + + product = detail.get("product") or {} + aftersale_fields = detail.get("aftersale_fields") or {} + apply_at = parse_apply_time(aftersale_fields.get("申请时间")) or parse_apply_time( + row.get("apply_at") + ) + if apply_at is None: + raise RuntimeError(f"售后单 {aftersale_id} 缺少可解析的申请时间") + + product_name = product.get("product_name") or row.get("product_name") + if not product_name: + print(f" [WARN] 售后单 {aftersale_id} 未解析到商品名称") + + order = { + "account": account, + "business_date": apply_at.date(), + "aftersale_id": aftersale_id, + "order_id": row.get("order_id") or aftersale_fields.get("订单编号"), + "aftersale_type": row.get("aftersale_type"), + "status_tag": row.get("status_tag"), + "aftersale_status": row.get("aftersale_status"), + "apply_at": apply_at, + # 详情页展开后“售后原因”值可能吞进售后历史时间线，只取首个短行； + # 不干净时退回列表行的申请原因（始终是“商品品质不好”类干净值）。 + "apply_reason": ( + _clean_reason_text(aftersale_fields.get("售后原因")) + or _clean_reason_text(row.get("apply_reason")) + or None + ), + "apply_quantity": row.get("apply_quantity"), + "pay_amount": row.get("pay_amount"), + "refund_amount": row.get("refund_amount"), + "product_name": product_name, + "product_id": product.get("product_id"), + "product_code": product.get("product_code") or row.get("product_code"), + "spec_name": product.get("spec_name") or row.get("spec_name"), + "negotiation_count": len(entries), + "detail_url": str(detail.get("detail_url") or ""), + "source_url": source_url, + "raw_row": { + "list_row": { + key: row.get(key) + for key in ( + "aftersale_id", + "order_id", + "aftersale_type", + "status_tag", + "aftersale_status", + "apply_at", + "apply_reason", + "apply_quantity", + "pay_amount", + "refund_amount", + "product_name", + "spec_name", + "product_code", + "raw_text", + "raw_cells", + ) + }, + "detail": { + "aftersale_fields": aftersale_fields, + "product_text": detail.get("product_text"), + "panel_text": detail.get("panel_text"), + }, + }, + "captured_at": captured_at, + } + + negotiations: list[dict[str, Any]] = [] + for entry in entries: + negotiations.append( + { + "account": account, + "aftersale_id": aftersale_id, + "entry_seq": entry["entry_seq"], + "actor_role": entry.get("actor_role"), + "action_text": entry.get("action_text"), + "happened_at": entry.get("happened_at"), + "operator": entry.get("operator"), + "return_reason": entry.get("return_reason"), + "refund_note": entry.get("refund_note"), + "evidence_urls": entry.get("evidence_urls") or [], + "extra_fields": entry.get("extra_fields") or {}, + "raw_text": entry.get("raw_text"), + "captured_at": captured_at, + } + ) + return order, negotiations + + +def _clean_reason_text(value: str | None) -> str | None: + """把售后原因文本收敛为干净的单行短值。 + + 详情页展开后“售后原因”的值可能吞进售后历史时间线（首行变成“时间” + 之类标题，或含“申请：/退款：”），只接受含原因关键词的首行短值； + 否则返回 None 交给调用方回退到列表行的申请原因（始终干净）。 + """ + + if not value: + return None + first_line = value.strip().splitlines()[0].strip() + if not first_line or len(first_line) > 30 or "申请：" in first_line: + return None + if not any(hint in first_line for hint in _REASON_HINTS): + return None + return first_line or None + + +def _persist_records( + order: dict[str, Any], + negotiations: list[dict[str, Any]], + *, + no_db: bool, +) -> tuple[int, int]: + if no_db: + return 0, 0 + from gyxx_flow.modules.shop_intelligence.db import ( + get_conn, + upsert_dy_after_sales_quality_negotiations, + upsert_dy_after_sales_quality_orders, + ) + + with get_conn() as conn: + orders = upsert_dy_after_sales_quality_orders(conn, [order]) + negotiation_rows = upsert_dy_after_sales_quality_negotiations( + conn, negotiations + ) + return orders, negotiation_rows + + +def _json_default(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def save_progress( + output_path: Path, + *, + account: str, + query_start: date | None, + query_end: date | None, + expected_count: int, + collected: dict[str, dict[str, Any]], + raw_rows: dict[str, dict[str, Any]], + complete: bool, + failed: dict[str, str] | None = None, +) -> None: + unresolved_note = ( + f"已完成 {len(collected)}/{expected_count} 条售后单" + if expected_count + else "售后单总数尚未确认" + ) + if failed: + unresolved_note += f"，失败 {len(failed)} 条待重试" + payload = { + "schema_version": 1, + "workflow_id": WORKFLOW_ID, + "account": account, + "query_start": query_start, + "query_end": query_end, + "query_mode": ( + "range" + if query_start and query_end and query_start != query_end + else ("single" if query_start else "full") + ), + "apply_reason": APPLY_REASON, + "source_url": TARGET_URL, + "expected_count": expected_count, + "collected_count": len(collected), + "progress_note": unresolved_note, + "complete": complete, + "failed_records": failed or {}, + "saved_at": datetime.now().astimezone(), + "records": collected, + "raw_rows": raw_rows, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + temp_path = output_path.with_suffix(output_path.suffix + ".tmp") + temp_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, default=_json_default), + encoding="utf-8", + ) + os.replace(temp_path, output_path) + + +def load_progress(output_path: Path) -> dict[str, Any] | None: + try: + if not output_path.exists(): + return None + payload = json.loads(output_path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else None + except Exception: + return None + + +def save_debug_artifacts(page: Page, output_dir: Path, label: str) -> None: + debug_dir = output_dir / "debug" + debug_dir.mkdir(parents=True, exist_ok=True) + safe_label = re.sub(r"[^0-9A-Za-z_-]+", "_", label) + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + try: + page.screenshot(path=str(debug_dir / f"{stamp}_{safe_label}.png"), full_page=True) + except Exception as exc: + print(f" [WARN] 保存失败截图失败: {exc}") + try: + (debug_dir / f"{stamp}_{safe_label}.html").write_text( + page.content(), encoding="utf-8" + ) + except Exception as exc: + print(f" [WARN] 保存失败 HTML 失败: {exc}") + + +def browser_runtime_options( + environment: dict[str, str] | None = None, +) -> dict[str, bool]: + """按部署设置选择系统 Chrome；默认使用 Scrapling 捆绑的指纹隐藏内核。""" + + values = os.environ if environment is None else environment + return {"real_chrome": True} if values.get("GYXX_BROWSER_CHANNEL") == "chrome" else {} + + +def _cdp_browser_available(cdp_url: str, timeout: float = 2.0) -> bool: + from urllib.request import urlopen + + try: + with urlopen(f"{cdp_url.rstrip('/')}/json/version", timeout=timeout) as resp: + return 200 <= resp.status < 300 + except Exception: + return False + + +def start_collection_browser( + environment: dict[str, str], + *, + headless: bool, +) -> tuple[ScraplingBrowser, Any | None]: + """启动抖音品退采集浏览器，返回 (browser, 账号浏览器锁)。 + + 与抖音客服采集一致：优先接入账号保活浏览器（CDP），保活不在线时用 + 账号本机 Profile + 系统 Chrome 复现登录环境；两种路径都不写回账号 + vault 文件。指纹隐藏模式（StealthySession）与共享账号 Cookie 配合。 + """ + + account_cdp_url = environment.get("GYXX_ACCOUNT_CDP_URL", "").strip() + if account_cdp_url and _cdp_browser_available(account_cdp_url): + print(f" [OK] 接入抖音账号保活浏览器: {account_cdp_url}") + return ( + ScraplingBrowser( + stealthy=True, + cdp_url=account_cdp_url, + reuse_existing_cdp_context=True, + ), + None, + ) + + account_profile = environment.get("GYXX_ACCOUNT_PROFILE_DIR", "").strip() + if account_profile: + profile_dir = Path(account_profile) + # 与 accounts login/keepalive 使用同一把浏览器锁，避免并发抢 Profile。 + locks = LockManager(SETTINGS.data_root / "state" / "locks") + account_lock = locks.acquire( + f"browser-account:{ACCOUNT_ID}", + owner=f"collect:{COMMAND_ID}", + timeout_seconds=120, + ) + try: + account_lock.__enter__() + except ResourceBusyError as exc: + raise RuntimeError( + "抖音账号浏览器正在被保活/登录占用，请稍后重试" + ) from exc + else: + profile_dir = Path( + environment.get( + "GYXX_BROWSER_PROFILE_DIR", + str(MODULE_PATHS.state_root / "browser" / "dy_after_sales_quality_profile"), + ) + ) + account_lock = None + print(" [WARN] 未找到账号 Profile，售后工作台可能无法通过设备校验") + + cdp_port = int(environment.get("GYXX_BROWSER_CDP_PORT", str(CDP_PORT))) + browser_options: dict[str, Any] = { + # 指纹隐藏模式：StealthySession。 + "stealthy": True, + "user_data_dir": str(profile_dir), + "headless": headless, + # 与 accounts login 一致使用系统 Chrome，保持登录指纹一致。 + "real_chrome": True, + # 只在店铺身份校验成功后由 persist_bound_state() 写回脚本状态， + # 账号 vault 文件不写回。 + "persist_state_on_close": False, + "locale": "zh-CN", + "timezone_id": "Asia/Shanghai", + "additional_args": {"viewport": {"width": 1920, "height": 1080}}, + "extra_flags": [ + f"--remote-debugging-port={cdp_port}", + "--remote-debugging-address=127.0.0.1", + ], + "retries": 1, + } + cookie_file = environment.get("GYXX_BROWSER_COOKIE_FILE", "").strip() + storage_state_file = environment.get("GYXX_BROWSER_STORAGE_STATE_FILE", "").strip() + if cookie_file: + browser_options["cookie_file"] = cookie_file + if storage_state_file: + browser_options["storage_state_file"] = storage_state_file + return ScraplingBrowser(**browser_options), account_lock + + +def preflight_acceptance_login( + environment: dict[str, str] | None = None, +) -> int | None: + policy = current_acceptance_policy() + if not policy.enabled: + return None + values = os.environ if environment is None else environment + result = policy.preflight_cookie( + binding_from_environment(values), environment=values + ) + if not result.should_skip: + return None + print(f"[SKIPPED_COOKIE] {result.reason}") + return COOKIE_SKIP_EXIT_CODE + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="抖音品质原因售后单（品退）采集") + date_arguments = parser.add_mutually_exclusive_group() + date_arguments.add_argument( + "--date", + type=date.fromisoformat, + default=None, + help="单日业务日期 YYYY-MM-DD（申请时间筛选的开始日期）；控制台会自动传入", + ) + date_arguments.add_argument( + "--range-days", + type=int, + default=0, + help="采集 business_date 往前 N 天（含当天）的区间；定时任务使用，" + "0 表示仅按 --date/--from/--to 采集", + ) + date_arguments.add_argument( + "--from", + dest="from_date", + type=date.fromisoformat, + default=None, + help="自定义申请时间范围开始日期 YYYY-MM-DD", + ) + parser.add_argument( + "--to", + dest="to_date", + type=date.fromisoformat, + default=None, + help="自定义申请时间范围结束日期 YYYY-MM-DD；需和 --from 一起使用", + ) + parser.add_argument( + "--output", type=Path, default=DEFAULT_OUTPUT, help="原始行数据和失败现场目录" + ) + parser.add_argument("--headless", action="store_true", help="无界面运行浏览器") + parser.add_argument("--no-db", action="store_true", help="只保存原始 JSON，不写数据库") + parser.add_argument("--retries", type=int, default=3, help="每条售后单的最大尝试次数") + parser.add_argument( + "--max-records", + type=int, + default=0, + help="限制本次最多采集的售后单数；0 表示不限制（调试用）", + ) + parser.add_argument( + "--login-timeout", type=int, default=300, help="人工登录最长等待秒数" + ) + return parser.parse_args(argv) + + +def _resolve_query_range(args: argparse.Namespace) -> tuple[date | None, date | None]: + """解析申请时间筛选范围；(None, None) 表示不设置时间（页面默认全量）。""" + + range_days = int(getattr(args, "range_days", 0) or 0) + if range_days > 0: + base = args.date or date.today() + return base - timedelta(days=range_days - 1), base + if args.from_date is not None or args.to_date is not None: + if args.from_date is None or args.to_date is None: + raise ValueError("自定义查询范围必须同时提供 --from 和 --to") + if args.to_date < args.from_date: + raise ValueError("抖音品退查询范围结束日期不能早于开始日期") + return args.from_date, args.to_date + if args.date is not None: + raw_end = os.getenv(RANGE_END_ENV, "").strip() + if raw_end: + try: + resolved_end = date.fromisoformat(raw_end) + except ValueError as exc: + raise ValueError( + f"{RANGE_END_ENV} 必须使用 YYYY-MM-DD 格式" + ) from exc + if resolved_end < args.date: + raise ValueError(f"{RANGE_END_ENV} 不能早于 --date") + return args.date, resolved_end + return args.date, args.date + return None, None + + +def run( + args: argparse.Namespace, + *, + runtime_environment: dict[str, str] | None = None, +) -> int: + if args.retries < 1: + raise ValueError("--retries 必须大于等于 1") + if args.login_timeout < 1: + raise ValueError("--login-timeout 必须大于等于 1") + environment = dict(runtime_environment or os.environ) + account = environment.get("GYXX_ACCOUNT_ID", ACCOUNT_ID).strip() or ACCOUNT_ID + query_start, query_end = _resolve_query_range(args) + if query_start is None: + output_name = "抖音品退_全量.json" + elif query_end != query_start: + output_name = f"抖音品退_{query_start.isoformat()}_{query_end.isoformat()}.json" + else: + output_name = f"抖音品退_{query_start.isoformat()}.json" + output_path = args.output / output_name + + if not args.no_db: + from gyxx_flow.modules.shop_intelligence.db import init_schema + + init_schema() + print("数据库表 dy_after_sales_quality_orders / dy_after_sales_quality_negotiations 已就绪") + + print("=" * 68) + print("抖音品质原因售后单（品退）采集") + print(f" 账号: {account}") + if query_start is None: + print(" 申请时间: 不筛选（页面默认最近 6 个月全量）") + elif query_end != query_start: + print(f" 申请时间: {query_start} 至 {query_end}") + else: + print(f" 申请时间: {query_start}") + print(f" 申请原因: {APPLY_REASON}") + print(f" 输出: {output_path}") + print(f" 数据库: {'跳过' if args.no_db else '写入'}") + print("=" * 68) + + collected: dict[str, dict[str, Any]] = {} + raw_rows: dict[str, dict[str, Any]] = {} + failures: dict[str, str] = {} + previous = load_progress(output_path) + if ( + previous + and previous.get("account") == account + and previous.get("query_start") == (query_start.isoformat() if query_start else None) + and previous.get("query_end") == (query_end.isoformat() if query_end else None) + ): + for aftersale_id, item in (previous.get("records") or {}).items(): + collected[str(aftersale_id)] = item + for aftersale_id, row in (previous.get("raw_rows") or {}).items(): + raw_rows[str(aftersale_id)] = row + for aftersale_id, reason in (previous.get("failed_records") or {}).items(): + failures[str(aftersale_id)] = str(reason) + if collected or failures: + print( + f"断点续采：已有进度 {len(collected)} 条售后单、" + f"{len(failures)} 条历史失败，本次只采集剩余部分" + ) + + browser_session, account_lock = start_collection_browser( + environment, headless=args.headless + ) + try: + with browser_session: + context = browser_session.context + if context is None: + raise RuntimeError("Scrapling 未创建抖音品退浏览器上下文") + page = context.pages[0] if context.pages else context.new_page() + _collect( + page, + context, + browser_session, + args, + environment=environment, + account=account, + query_start=query_start, + query_end=query_end, + output_path=output_path, + collected=collected, + raw_rows=raw_rows, + failures=failures, + ) + finally: + if account_lock is not None: + account_lock.__exit__(None, None, None) + + print( + f"采集完成：共 {len(collected)} 条品质原因售后单，结果已保存到 {output_path}" + ) + return 0 + + +def _collect( + page: Page, + context: Any, + browser_session: ScraplingBrowser, + args: argparse.Namespace, + *, + environment: dict[str, str], + account: str, + query_start: date | None, + query_end: date | None, + output_path: Path, + collected: dict[str, dict[str, Any]], + raw_rows: dict[str, dict[str, Any]], + failures: dict[str, str], +) -> None: + try: + ensure_douyin_login( + page, + environment=environment, + headless=args.headless, + login_timeout=args.login_timeout, + ) + verify_shop_identity(page, environment=environment) + dismiss_blocking_modals(page) + # 店铺身份校验通过后才把会话状态写回脚本专属状态文件。 + browser_session.persist_bound_state() + + try: + open_more_filters(page) + set_apply_reason(page, APPLY_REASON) + if query_start is not None and query_end is not None: + set_apply_time_range(page, query_start, query_end) + total = apply_query(page) + page_size = set_page_size(page, 50) + if page_size: + total = apply_query(page) + except Exception: + save_debug_artifacts(page, args.output, "filter_setup") + raise + print( + f"筛选完成：共 {total} 条品质原因售后单，开始逐条采集" + + (f"（分页大小 {page_size}）" if page_size else "") + ) + + def _snapshot(complete: bool) -> None: + save_progress( + output_path, + account=account, + query_start=query_start, + query_end=query_end, + expected_count=total, + collected=collected, + raw_rows=raw_rows, + complete=complete, + failed=failures, + ) + + _snapshot(complete=False) + + processed = 0 + for _ in range(MAX_PAGES): + rows = wait_current_page_rows(page) + if not rows: + if len(collected) >= total: + break + raise RuntimeError( + f"当前页未读取到售后行（已采集 {len(collected)}/{total}）" + ) + rows = scan_current_page_rows(page) + info = _pagination_info(page) + print( + f" [INFO] 页码 {info['active']}/{info['max_page']}，" + f"本页扫描到 {len(rows)} 行" + ) + for row in rows: + if args.max_records > 0 and len(collected) >= args.max_records: + break + aftersale_id = row["aftersale_id"] + if aftersale_id in collected: + print(f" [INFO] 跳过已采集售后单 {aftersale_id}") + continue + last_error: Exception | None = None + for attempt in range(1, args.retries + 1): + detail_page: Page | None = None + try: + detail_page = open_detail_page(context, page, aftersale_id) + detail = extract_detail( + detail_page, expected_aftersale_id=aftersale_id + ) + detail["detail_url"] = detail_page.url + captured_at = datetime.now().astimezone() + order, negotiations = normalize_order( + row, + detail, + account=account, + source_url=TARGET_URL, + captured_at=captured_at, + ) + order_rows, negotiation_rows = _persist_records( + order, negotiations, no_db=args.no_db + ) + collected[aftersale_id] = { + "business_date": order["business_date"].isoformat(), + "negotiation_count": len(negotiations), + "product_name": order.get("product_name"), + "product_id": order.get("product_id"), + "db_order_rows": order_rows, + "db_negotiation_rows": negotiation_rows, + "captured_at": captured_at, + } + raw_rows[aftersale_id] = { + "list_row": row.get("raw_text"), + "detail_panel": detail.get("panel_text"), + } + failures.pop(aftersale_id, None) + _snapshot(complete=False) + processed += 1 + print( + f" [{processed:02d}/{total}] 售后单 {aftersale_id} 完成" + f"（协商记录 {len(negotiations)} 条，" + f"商品 {order.get('product_name') or '未知'}）" + ) + last_error = None + break + except Exception as exc: + last_error = exc + print( + f" [{processed + 1:02d}/{total}] 售后单 {aftersale_id} " + f"第 {attempt}/{args.retries} 次失败: {exc}" + ) + if attempt < args.retries: + page.wait_for_timeout(500) + finally: + if detail_page is not None: + try: + detail_page.close() + except Exception: + pass + if last_error is not None: + save_debug_artifacts( + page, args.output, f"aftersale_{aftersale_id}" + ) + failures[aftersale_id] = str(last_error) + print( + f" [WARN] 售后单 {aftersale_id} 连续 " + f"{args.retries} 次失败，跳过并继续后续售后单" + ) + if args.max_records > 0 and len(collected) + len(failures) >= args.max_records: + print( + f" [INFO] 已达到 --max-records={args.max_records}，提前结束" + ) + break + if len(collected) + len(failures) >= total: + break + info = _pagination_info(page) + active = info["active"] + total_pages = info["max_page"] + if active and total_pages and active >= total_pages: + print(f" [INFO] 已到最后一页（页码 {active}/{total_pages}）") + break + if _next_page_disabled(page): + print(" [INFO] 已到最后一页（下一页禁用）") + break + if not goto_next_page(page, rows[0]["aftersale_id"]): + break + + drift = total - len(collected) + if args.max_records > 0: + print( + f" [INFO] --max-records={args.max_records} 生效，" + f"跳过覆盖校验（本次采集 {len(collected)}）" + ) + _snapshot(complete=False) + return + if failures: + _snapshot(complete=False) + failed_preview = "、".join(list(failures)[:5]) + raise RuntimeError( + f"{len(failures)} 条售后单采集失败（如 {failed_preview}）；" + f"已完成 {len(collected)}/{total}，进度已保留，可直接重试续采" + ) + if drift > max(3, int(total * 0.02)): + _snapshot(complete=False) + raise RuntimeError( + f"覆盖不完整：页面声明 {total} 条，实际完成 {len(collected)} 条" + ) + if drift > 0: + print(f" [WARN] 活列表漂移：声明 {total} 条，实际采集 {len(collected)} 条") + _snapshot(complete=True) + finally: + # 正常退出由 ScraplingBrowser 将绑定状态写回脚本专属状态文件； + # 账号主 vault 仍由 gyxx accounts login/sync 管理。 + pass + + +def _account_runtime_environment() -> dict[str, str]: + """注入脚本绑定环境；账号 vault Cookie 只读合并进脚本专属状态。 + + 通过 `gyxx run/backfill/scripts run` 启动时引擎已经注入了同一套环境， + 本函数只兜底直接以脚本方式启动的场景。 + """ + + base_environment = dict(os.environ) + project_root = base_environment.get("GYXX_PROJECT_ROOT", "").strip() + data_root = base_environment.get("GYXX_DATA_ROOT", "").strip() + if not project_root or not data_root: + return base_environment + catalog = RuntimeIntegrationCatalog.load_default( + project_root=project_root, + data_root=data_root, + ) + try: + return catalog.environment_for(COMMAND_ID, base_environment) + except Exception as exc: + print(f" [INFO] 未解析到脚本绑定 {COMMAND_ID}，使用当前环境继续: {exc}") + return catalog.environment_for_account(ACCOUNT_ID, base_environment) + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + runtime_environment = _account_runtime_environment() + preflight_exit_code = preflight_acceptance_login(runtime_environment) + if preflight_exit_code is not None: + return preflight_exit_code + return run(args, runtime_environment=runtime_environment) + except DouyinSessionExpiredError: + return COOKIE_SKIP_EXIT_CODE + except KeyboardInterrupt: + print("\n用户中断。") + return 130 + except Exception as exc: + print(f"\n[ERROR] {exc}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/dy_customer_service_messages.py b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_customer_service_messages.py new file mode 100644 index 0000000..00d3ec3 --- /dev/null +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_customer_service_messages.py @@ -0,0 +1,2711 @@ +"""采集抖音飞鸽历史会话消息并按业务日期幂等写入 PostgreSQL。 + +飞鸽客服后台的“历史会话”页按筛选条件列出会话。脚本复用抖音账号 vault +的 Cookie（有效 Cookie 直接跳过登录），设置自定义开始/结束日期与接待类型 +（默认“纯人工客服接待”+“智能客服转人工接待”），逐行点击“查看会话”读取 +抽屉内的聊天消息，并把抽屉右侧「咨询订单」面板的订单卡解析为商品/订单 +记录绑定会话入库。会话表格是虚拟滚动列表，当前页会先逐屏深度扫描收集 +全部行再逐条采集；单个会话重试后仍失败时跳过并继续，运行结束前统一 +上报失败清单。全部会话完成后才将运行标记为成功；中途失败时已完成的 +会话仍可通过唯一键安全重试。 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import time +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +from gyxx_flow.adapters import ( + COOKIE_SKIP_EXIT_CODE, + RuntimeIntegrationCatalog, + binding_from_environment, + current_acceptance_policy, +) +from gyxx_flow.adapters.scrapling import BrowserPage as Page +from gyxx_flow.adapters.scrapling import ScraplingBrowser, is_browser_timeout_error +from gyxx_flow.core.config import Settings +from gyxx_flow.core.layout import DataLayout +from gyxx_flow.core.locks import LockManager, ResourceBusyError + +SETTINGS = Settings.from_env() +MODULE_PATHS = DataLayout(SETTINGS.data_root).for_module("shop_intelligence") + +TARGET_URL = ( + "https://im.jinritemai.com/pc_seller_v2/main/data/historyConversation" +) +ACCOUNT_ID = "douyin-shop" +COMMAND_ID = "shop.douyin_customer_service.collect" +WORKFLOW_ID = "shop.douyin_customer_service.daily" +DEFAULT_OUTPUT = MODULE_PATHS.raw_root / "dy_customer_service_messages" +CDP_PORT = int( + os.getenv("GYXX_BROWSER_CDP_PORT") + or os.getenv("GYXX_CDP_PORT") + or "22158" +) +DEFAULT_IDENTITY_MARKERS = ("光影行星箱包旗舰店",) +# 飞鸽 IM 页面本身不展示店铺名称，店铺身份在抖店后台页校验。 +FXG_IDENTITY_URL = os.getenv( + "GYXX_DY_IDENTITY_URL", + "https://fxg.jinritemai.com/ffa/merchant/campaign-square?list_tab=access&f_tab=0", +) +DEFAULT_RECEPTION_TYPES = ("纯人工客服接待", "智能客服转人工接待") +RECEPTION_TYPE_OPTIONS = ( + "全部", + "纯人工客服接待", + "纯智能客服接待", + "智能客服转人工接待", + "其他", +) +SHANGHAI = ZoneInfo("Asia/Shanghai") +LOGIN_URL_MARKERS = ("/login", "passport") +LOGIN_TEXT_MARKERS = ( + "手机登录", + "邮箱登录", + "扫码登录", + "验证码登录", + "登录即代表同意", + "登录过期", +) +MAX_PAGES = 500 + +TOTAL_COUNT_PATTERN = re.compile(r"共\s*(\d+)\s*条") +PAGE_SIZE_PATTERN = re.compile(r"(\d+)\s*条\s*/\s*页") +CONVERSATION_ID_PATTERN = re.compile(r"ID[:：]?\s*(\d{6,})") +SESSION_TIME_PATTERN = re.compile( + r"(\d{1,2})/(\d{1,2})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*-\s*" + r"(?:(\d{1,2})/(\d{1,2})\s+)?(\d{1,2}):(\d{2})(?::(\d{2}))?" +) +TIME_DIVIDER_PATTERN = re.compile( + r"^(?:\d{4}[-/]\d{1,2}[-/]\d{1,2}\s+)?\d{1,2}[:/]\d{1,2}" + r"(?:[/]\d{1,2})?\s+\d{1,2}:\d{2}(?::\d{2})?$" +) + + +def parse_total_count(text: str) -> int: + """从分页文案“共 22 条数据”解析会话总数。""" + + match = TOTAL_COUNT_PATTERN.search(text or "") + if not match: + raise RuntimeError(f"无法从分页区域识别会话总数: {text!r}") + return int(match.group(1)) + + +def parse_page_size(text: str) -> int: + """从分页文案“20 条/页”解析每页条数。""" + + match = PAGE_SIZE_PATTERN.search(text or "") + if not match: + raise RuntimeError(f"无法从分页区域识别每页条数: {text!r}") + return int(match.group(1)) + + +def parse_session_time_label( + label: str, business_date: date +) -> tuple[datetime | None, datetime | None]: + """把“09/18 00:33-00:49”解析为上海时区的开始/结束时间。 + + 标签不带年份，按业务日期所在年锚定；跨年场景（如业务日为 01/01、 + 标签为 12/31 23:50-00:10）向前回退一年。结束时间早于开始时间时 + 视为跨零点，顺延一天。 + """ + + match = SESSION_TIME_PATTERN.search(label or "") + if not match: + return None, None + ( + start_month, + start_day, + start_hour, + start_minute, + start_second, + end_month, + end_day, + end_hour, + end_minute, + end_second, + ) = match.groups() + start = datetime( + business_date.year, + int(start_month), + int(start_day), + int(start_hour), + int(start_minute), + int(start_second or 0), + tzinfo=SHANGHAI, + ) + if start.date() > business_date: + start = start.replace(year=start.year - 1) + end_month_value = int(end_month) if end_month else int(start_month) + end_day_value = int(end_day) if end_day else int(start_day) + end = datetime( + start.year, + end_month_value, + end_day_value, + int(end_hour), + int(end_minute), + int(end_second or 0), + tzinfo=SHANGHAI, + ) + if end < start: + end += timedelta(days=1) + return start, end + + +def _stable_key(value: Any) -> str: + serialized = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def _message_key( + conversation_id: str, + seq: int, + sender_role: str, + content: str, +) -> str: + return _stable_key( + { + "conversation": conversation_id, + "seq": seq, + "role": sender_role, + "content": content, + } + ) + + +def _json_default(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def normalize_conversation( + row_meta: dict[str, Any], + drawer_meta: dict[str, Any], + messages: list[dict[str, Any]], + *, + account: str, + business_date: date, + source_url: str, + order_cards: list[str] | None = None, +) -> dict[str, Any]: + """把列表行元数据、抽屉头部和消息列表规范化为入库记录。 + + 订单卡文本会解析为结构化商品/订单记录，绑定会话一起返回。 + """ + + conversation_id = str(row_meta.get("conversation_id") or "").strip() + if not conversation_id: + raise RuntimeError(f"会话行缺少会话 ID: {row_meta!r}") + if not messages: + raise RuntimeError(f"会话 {conversation_id} 未采集到任何消息，拒绝写入") + + session_label = str(row_meta.get("session_time_label") or "").strip() + session_start, session_end = parse_session_time_label(session_label, business_date) + captured_at = datetime.now().astimezone() + + orders: list[dict[str, Any]] = [] + for card_text in order_cards or []: + parsed = parse_order_card(card_text) + if parsed is None: + print(f" [WARN] 会话 {row_meta.get('conversation_id')} 有一张订单卡解析失败，已跳过") + continue + parsed.update( + account=account, + business_date=business_date, + conversation_id=str(row_meta.get("conversation_id") or ""), + source_url=source_url, + captured_at=captured_at, + ) + orders.append(parsed) + order_panel_text = "\n\n".join(order_cards) if order_cards else None + + conversation = { + "account": account, + "business_date": business_date, + "conversation_id": conversation_id, + "customer_nickname": row_meta.get("customer_nickname"), + "session_time_label": session_label or None, + "session_start_at": session_start, + "session_end_at": session_end, + "consult_entry": drawer_meta.get("consult_entry") + or row_meta.get("consult_entry"), + "reception_type": row_meta.get("reception_type"), + "initiator_scene": row_meta.get("initiator_scene"), + "end_scene": row_meta.get("end_scene"), + "rating_time_label": row_meta.get("rating_time_label"), + "rating_result": drawer_meta.get("rating_result") + or row_meta.get("rating_result"), + "rating_tags": row_meta.get("rating_tags"), + "conversation_intent": row_meta.get("conversation_intent"), + "smart_reception_label": row_meta.get("smart_reception_label"), + "smart_rating_time_label": row_meta.get("smart_rating_time_label"), + "smart_rating_result": row_meta.get("smart_rating_result"), + "smart_rating_tags": row_meta.get("smart_rating_tags"), + "transfer_human_duration": row_meta.get("transfer_human_duration"), + "transfer_human_type": row_meta.get("transfer_human_type"), + "service_problem_tags": drawer_meta.get("service_problem_tags"), + "unmet_metrics": drawer_meta.get("unmet_metrics"), + "message_count": len(messages), + "order_count": len(orders), + "order_panel_text": order_panel_text, + "source_url": source_url, + "raw_payload": { + "row": row_meta, + "drawer_header": drawer_meta, + }, + "captured_at": captured_at, + } + + records: list[dict[str, Any]] = [] + seen_keys: set[str] = set() + for seq, message in enumerate(messages): + content = str(message.get("content") or "").strip() + sender_role = str(message.get("sender_role") or "").strip() or "unknown" + message_uuid = str(message.get("message_uuid") or "").strip() + key = message_uuid or _message_key(conversation_id, seq, sender_role, content) + if key in seen_keys: + key = f"{key}:{seq}" + seen_keys.add(key) + card = message.get("card") + message_at = message.get("message_at") + if message_at is None: + message_at = parse_message_time_label( + message.get("message_at_label") or message.get("time_label"), + business_date, + ) + raw_message = ( + message.get("raw_message") + if isinstance(message.get("raw_message"), dict) + else {"content": content} + ) + sender_name = message.get("sender_name") + if not sender_name and sender_role == "customer": + sender_name = row_meta.get("customer_nickname") + if message_uuid: + raw_message = {**raw_message, "data_id": message_uuid} + if message.get("read_mark") is not None: + raw_message = {**raw_message, "read_mark": bool(message.get("read_mark"))} + records.append( + { + "account": account, + "business_date": business_date, + "conversation_id": conversation_id, + "message_key": key, + "message_seq": seq, + "message_at": message_at, + "time_label": message.get("time_label"), + "sender_name": sender_name, + "sender_role": sender_role, + "message_type": message.get("message_type") or "text", + "content": content or None, + "card": card if isinstance(card, dict) else {}, + "source_url": source_url, + "raw_message": raw_message, + "captured_at": captured_at, + } + ) + return { + "conversation": conversation, + "messages": records, + "message_count": len(records), + "orders": orders, + } + + +def save_progress( + output_path: Path, + *, + account: str, + business_date: date, + reception_types: tuple[str, ...], + expected_count: int, + collected: dict[str, dict[str, Any]], + raw_rows: dict[str, dict[str, Any]], + complete: bool, + failed: dict[str, str] | None = None, +) -> None: + unresolved_note = ( + f"已完成 {len(collected)}/{expected_count} 个会话" + if expected_count + else "会话总数尚未确认" + ) + if failed: + unresolved_note += f"，失败 {len(failed)} 个待重试" + payload = { + "schema_version": 1, + "workflow_id": WORKFLOW_ID, + "account": account, + "business_date": business_date, + "reception_types": list(reception_types), + "source_url": TARGET_URL, + "expected_conversation_count": expected_count, + "collected_conversation_count": len(collected), + "progress_note": unresolved_note, + "complete": complete, + "failed_conversations": failed or {}, + "saved_at": datetime.now().astimezone(), + "conversations": collected, + "raw_rows": raw_rows, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + temp_path = output_path.with_suffix(output_path.suffix + ".tmp") + temp_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, default=_json_default), + encoding="utf-8", + ) + os.replace(temp_path, output_path) + + +def load_progress(output_path: Path) -> dict[str, Any] | None: + """读取既有进度文件用于断点续采；文件缺失或损坏时返回 None。""" + + try: + if not output_path.exists(): + return None + payload = json.loads(output_path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else None + except Exception: + return None + + +def save_debug_artifacts(page: Page, output_dir: Path, label: str) -> None: + debug_dir = output_dir / "debug" + debug_dir.mkdir(parents=True, exist_ok=True) + safe_label = re.sub(r"[^0-9A-Za-z_-]+", "_", label) + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + try: + page.screenshot(path=str(debug_dir / f"{stamp}_{safe_label}.png"), full_page=True) + except Exception as exc: + print(f" [WARN] 保存失败截图失败: {exc}") + try: + (debug_dir / f"{stamp}_{safe_label}.html").write_text( + page.content(), encoding="utf-8" + ) + except Exception as exc: + print(f" [WARN] 保存失败 HTML 失败: {exc}") + + +def browser_runtime_options( + environment: dict[str, str] | None = None, +) -> dict[str, bool]: + """按部署设置选择系统 Chrome；默认使用 Scrapling 捆绑的指纹隐藏内核。""" + + values = os.environ if environment is None else environment + return {"real_chrome": True} if values.get("GYXX_BROWSER_CHANNEL") == "chrome" else {} + + +def _cdp_browser_available(cdp_url: str, timeout: float = 2.0) -> bool: + from urllib.request import urlopen + + try: + with urlopen(f"{cdp_url.rstrip('/')}/json/version", timeout=timeout) as resp: + return 200 <= resp.status < 300 + except Exception: + return False + + +def start_collection_browser( + environment: dict[str, str], + *, + headless: bool, +) -> tuple[ScraplingBrowser, Any | None]: + """启动抖音客服采集浏览器，返回 (browser, 账号浏览器锁)。 + + 飞鸽 IM 会话绑定登录时的浏览器指纹，独立 Profile 复用 Cookie 会被 + 服务端判定“登录过期”。因此优先接入账号保活浏览器（CDP），保活不在 + 线时用账号本机 Profile + 系统 Chrome 复现登录环境；两种路径都不写回 + 账号 vault 文件。 + """ + + account_cdp_url = environment.get("GYXX_ACCOUNT_CDP_URL", "").strip() + if account_cdp_url and _cdp_browser_available(account_cdp_url): + print(f" [OK] 接入抖音账号保活浏览器: {account_cdp_url}") + return ( + ScraplingBrowser( + stealthy=True, + cdp_url=account_cdp_url, + reuse_existing_cdp_context=True, + ), + None, + ) + + account_profile = environment.get("GYXX_ACCOUNT_PROFILE_DIR", "").strip() + if account_profile: + profile_dir = Path(account_profile) + # 与 accounts login/keepalive 使用同一把浏览器锁，避免并发抢 Profile。 + locks = LockManager(SETTINGS.data_root / "state" / "locks") + account_lock = locks.acquire( + f"browser-account:{ACCOUNT_ID}", + owner=f"collect:{COMMAND_ID}", + timeout_seconds=120, + ) + try: + account_lock.__enter__() + except ResourceBusyError as exc: + raise RuntimeError( + "抖音账号浏览器正在被保活/登录占用，请稍后重试" + ) from exc + else: + profile_dir = Path( + environment.get( + "GYXX_BROWSER_PROFILE_DIR", + str(MODULE_PATHS.state_root / "browser" / "dy_customer_service_profile"), + ) + ) + account_lock = None + print(" [WARN] 未找到账号 Profile，IM 会话可能无法通过设备校验") + + cdp_port = int(environment.get("GYXX_BROWSER_CDP_PORT", str(CDP_PORT))) + browser_options: dict[str, Any] = { + # 指纹隐藏模式：StealthySession。 + "stealthy": True, + "user_data_dir": str(profile_dir), + "headless": headless, + # 与 accounts login 一致使用系统 Chrome，保持登录指纹一致。 + "real_chrome": True, + # 只在店铺身份校验成功后由 persist_bound_state() 写回脚本状态， + # 避免失败页面覆盖最后一次可用登录态；账号 vault 文件不写回。 + "persist_state_on_close": False, + "locale": "zh-CN", + "timezone_id": "Asia/Shanghai", + "additional_args": {"viewport": {"width": 1920, "height": 1080}}, + "extra_flags": [ + f"--remote-debugging-port={cdp_port}", + "--remote-debugging-address=127.0.0.1", + ], + "retries": 1, + } + cookie_file = environment.get("GYXX_BROWSER_COOKIE_FILE", "").strip() + storage_state_file = environment.get("GYXX_BROWSER_STORAGE_STATE_FILE", "").strip() + if cookie_file: + browser_options["cookie_file"] = cookie_file + if storage_state_file: + browser_options["storage_state_file"] = storage_state_file + return ScraplingBrowser(**browser_options), account_lock + + +def _wait_for_page(page: Page, timeout: int = 60_000) -> None: + try: + page.wait_for_load_state("networkidle", timeout=timeout) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + page.wait_for_load_state("domcontentloaded", timeout=timeout) + + +def _page_text(page: Page, timeout: int = 15_000) -> str: + try: + return page.locator("body").inner_text(timeout=timeout) + except Exception: + return "" + + +def is_login_page(page: Page) -> bool: + url = (page.url or "").casefold() + if any(marker in url for marker in LOGIN_URL_MARKERS): + return True + try: + for marker in LOGIN_TEXT_MARKERS: + if page.locator(f"text={marker}").count() > 0: + return True + except Exception: + return False + return False + + +def _configured_identity_markers( + environment: dict[str, str] | None = None, +) -> tuple[str, ...]: + values = os.environ if environment is None else environment + raw = values.get("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS", "").strip() + if raw: + try: + markers = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS 不是有效 JSON") from exc + if isinstance(markers, list) and all(isinstance(item, str) for item in markers): + return tuple(item.strip() for item in markers if item.strip()) + return DEFAULT_IDENTITY_MARKERS + + +def verify_shop_identity( + page: Page, + *, + environment: dict[str, str] | None = None, +) -> None: + """确认登录后的店铺身份，避免把共享抖音 Cookie 用到错误店铺。 + + 飞鸽 IM 页面不渲染店铺名称，这里在抖店后台页完成校验后回到历史会话页。 + """ + + markers = _configured_identity_markers(environment) + if not markers: + return + page.goto(FXG_IDENTITY_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + body_text = _page_text(page) + if any(marker in body_text for marker in markers): + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + return + if is_login_page(page): + break + page.wait_for_timeout(1_000) + raise RuntimeError( + "抖音登录态未出现配置的店铺身份标识；" + f"期望标识={markers!r}，当前页面未匹配。" + "请用 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + + +def ensure_douyin_login( + page: Page, + *, + environment: dict[str, str], + headless: bool, + login_timeout: int, +) -> None: + """打开历史会话页；有效 Cookie 直接跳过登录，否则等待人工登录。""" + + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + page.wait_for_timeout(2_000) + + if not is_login_page(page): + print(" [OK] 已复用抖音账号 Cookie 登录态，跳过登录") + return + + policy = current_acceptance_policy(environment) + if policy.enabled: + policy.record( + "cookie_skipped", + operation="shop_intelligence.dy_customer_service.live_session", + details={ + "status": "SKIPPED_COOKIE", + "reason": "douyin browser session redirected to login", + }, + ) + print("[SKIPPED_COOKIE] 抖音浏览器会话跳转到了登录页") + raise DouyinSessionExpiredError("抖音浏览器会话跳转到了登录页") + + if headless: + raise RuntimeError( + "抖音 Cookie 已失效且当前为无头模式，无法扫码登录；" + "请先执行 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + + print("=" * 60) + print("【请登录】请在弹出的浏览器窗口中完成抖音店铺登录") + print("提示：扫码/短信/密码均可，登录成功后页面会自动跳转") + print("=" * 60) + deadline = time.monotonic() + login_timeout + while time.monotonic() < deadline: + if not is_login_page(page): + print(f" [OK] 登录成功: {page.url}") + page.wait_for_timeout(3_000) + return + page.wait_for_timeout(1_000) + raise RuntimeError( + f"登录在 {login_timeout} 秒内未完成，当前页面: {page.url}" + ) + + +class DouyinSessionExpiredError(RuntimeError): + """The reused cookie state reached an interactive login page.""" + + +def _history_page_ready(page: Page) -> None: + page.get_by_text("历史会话", exact=True).first.wait_for( + state="visible", timeout=30_000 + ) + page.get_by_role("button", name="查询", exact=True).first.wait_for( + state="visible", timeout=30_000 + ) + + +def dismiss_blocking_modals(page: Page) -> None: + """关闭遮挡页面的弹窗（挽单配置、公告、新手引导等 auxo 模态框）。""" + + # 快速预检：没有任何可见遮挡层时直接返回，避免逐条会话的固定开销。 + try: + overlay = page.evaluate( + """() => { + const visible = (el) => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + for (const el of document.querySelectorAll( + '.auxo-modal-wrap, .auxo-modal-mask, [role="dialog"], .auxo-drawer' + )) { + if (!visible(el)) continue; + if ((el.innerText || '').includes('本通会话明细')) continue; + return true; + } + return false; + }""" + ) + except Exception: + overlay = True + if not overlay: + return + for _ in range(6): + closed = False + for selector in ( + '.auxo-modal-wrap:visible .auxo-modal-close:visible', + '.auxo-modal-wrap:visible [class*="close"]:visible', + '.auxo-modal-wrap:visible button:has-text("知道"):visible', + '.auxo-modal-wrap:visible button:has-text("暂不"):visible', + '.auxo-modal-wrap:visible button:has-text("下次再说"):visible', + '.auxo-modal-wrap:visible button:has-text("取消"):visible', + '.auxo-modal-wrap:visible button:has-text("关闭"):visible', + '[role="dialog"]:visible [class*="close"]:visible', + '[class*="modal"]:visible [class*="close"]:visible', + ): + button = page.locator(selector).first + try: + if button.count() and button.is_visible(): + button.click(timeout=2_000) + page.wait_for_timeout(400) + closed = True + break + except Exception: + continue + if not closed: + break + removed = page.evaluate( + """() => { + let removed = 0; + const visible = (el) => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + for (const el of document.querySelectorAll( + '.auxo-modal-wrap, .auxo-modal-mask, [role="dialog"]' + )) { + if (!visible(el)) continue; + // 公告/推广类遮挡层直接移除；含表单输入的对话框保留。 + if (el.querySelector('input, textarea, form')) continue; + el.remove(); + removed += 1; + } + // 推广抽屉（挽单配置等）：不含“本通会话明细”的可见抽屉尝试点关闭按钮， + // 点不掉就直接移除。 + for (const el of document.querySelectorAll('.auxo-drawer, [class*="drawer"]')) { + if (!visible(el)) continue; + const text = (el.innerText || ''); + if (text.includes('本通会话明细')) continue; + if (el.querySelector('input, textarea, form')) continue; + const closeBtn = el.querySelector( + '.auxo-drawer-close, [class*="close"], button[aria-label*="Close"], button[aria-label*="关闭"]' + ); + if (closeBtn) { + try { closeBtn.click(); } catch (e) { /* ignore */ } + } + if (visible(el)) { + el.remove(); + removed += 1; + } + } + return removed; + }""" + ) + if removed: + print(f" [INFO] 已移除 {removed} 个遮挡弹窗") + page.wait_for_timeout(300) + + +def _open_date_picker(page: Page) -> None: + """打开“选择日期”范围面板。""" + + dismiss_blocking_modals(page) + for placeholder in ("开始日期", "开始"): + start_input = page.locator(f'input[placeholder*="{placeholder}"]').first + try: + if start_input.count() and start_input.is_visible(): + start_input.click() + page.wait_for_timeout(400) + return + except Exception: + continue + label = page.get_by_text("选择日期", exact=True).first + container_input = label.locator( + "xpath=ancestor::*[self::div or self::td][1]" + ).locator("input").first + container_input.click() + page.wait_for_timeout(400) + + +def _fill_date_input(page: Page, placeholder: str, target: date) -> bool: + field = page.locator(f'input[placeholder="{placeholder}"]').first + if not field.count() or not field.is_visible(): + return False + field.click() + page.wait_for_timeout(150) + try: + field.press("Control+a") + except Exception: + pass + field.fill(target.isoformat()) + page.wait_for_timeout(150) + field.press("Enter") + page.wait_for_timeout(300) + return field.input_value().strip() == target.isoformat() + + +def _click_date_cell(page: Page, target: date) -> bool: + """在打开的日历面板中导航并点击目标日期格子（输入失败时的兜底）。""" + + title = target.isoformat() + for _ in range(36): + cell = page.locator( + f'td[title="{title}"]:visible, ' + f'div[title="{title}"]:visible, ' + f'span[title="{title}"]:visible' + ).first + if cell.count() and cell.is_visible(): + cell.click() + page.wait_for_timeout(200) + return True + panel_text = "" + for selector in ( + '[class*="picker"]:visible', + '[class*="calendar"]:visible', + '[class*="date-panel"]:visible', + ): + locator = page.locator(selector).last + if locator.count(): + try: + panel_text = locator.inner_text(timeout=2_000) + if panel_text: + break + except Exception: + continue + months = re.findall(r"(\d{4})\s*年\s*(\d{1,2})\s*月", panel_text or "") + if not months: + return False + displayed = [int(year) * 12 + int(month) - 1 for year, month in months] + target_index = target.year * 12 + target.month - 1 + go_prev = target_index < min(displayed) + button = page.locator( + ('button[class*="prev"]:visible, ' + '[class*="icon-left"]:visible, ' + '[class*="prev-btn"]:visible') + if go_prev + else ('button[class*="next"]:visible, ' + '[class*="icon-right"]:visible, ' + '[class*="next-btn"]:visible') + ).first + if not button.count(): + return False + button.click() + page.wait_for_timeout(200) + return False + + +def set_date_range(page: Page, start: date, end: date) -> None: + """设置“选择日期”的开始/结束日期并校验输入值。""" + + _open_date_picker(page) + typed = _fill_date_input(page, "开始日期", start) and _fill_date_input( + page, "结束日期", end + ) + if not typed: + # 输入不可用时退回日历格子点击：开始、结束各点一次。 + if not _click_date_cell(page, start): + raise RuntimeError(f"日期面板无法选择开始日期 {start.isoformat()}") + if not _click_date_cell(page, end): + raise RuntimeError(f"日期面板无法选择结束日期 {end.isoformat()}") + page.keyboard.press("Escape") + page.wait_for_timeout(300) + + start_value = page.locator('input[placeholder="开始日期"]').first + end_value = page.locator('input[placeholder="结束日期"]').first + if start_value.count() and end_value.count(): + actual = (start_value.input_value().strip(), end_value.input_value().strip()) + expected = (start.isoformat(), end.isoformat()) + if actual != expected: + raise RuntimeError(f"日期输入未生效，期望 {expected}，实际 {actual}") + + +def _option_checked(option: Any) -> bool: + try: + checkbox = option.locator('input[type="checkbox"]').first + if checkbox.count(): + return bool(checkbox.is_checked()) + except Exception: + pass + try: + class_name = option.get_attribute("class") or "" + except Exception: + return False + return "checked" in class_name and "unchecked" not in class_name + + +def ensure_reception_types(page: Page, types: tuple[str, ...]) -> None: + """在“接待类型”多选框中精确勾选指定的接待类型并硬性校验。""" + + invalid = [item for item in types if item not in RECEPTION_TYPE_OPTIONS or item == "全部"] + if invalid: + raise ValueError(f"不支持的接待类型: {invalid!r}，可选项 {RECEPTION_TYPE_OPTIONS[1:]!r}") + + dismiss_blocking_modals(page) + control = page.locator( + 'div:has(> :text-is("接待类型")) [class*="select"], ' + '[class*="select"]:has-text("接待类型")' + ).first + try: + visible = bool(control.count()) and control.is_visible() + except Exception: + visible = False + if not visible: + label = page.get_by_text("接待类型", exact=True).first + control = label.locator( + "xpath=ancestor::*[self::div or self::td][1]" + ).locator('[class*="select"], input').first + control.click() + page.wait_for_timeout(800) + + desired = {label: label in types for label in RECEPTION_TYPE_OPTIONS} + state = page.evaluate( + """(desired) => { + const visible = (el) => { + if (!el) return false; + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + const labels = Object.keys(desired); + // 找到包含全部选项文本的可见下拉面板。 + const panels = Array.from(document.querySelectorAll('div, ul, section')) + .filter(visible) + .filter((el) => labels.every((label) => + (el.innerText || '').includes(label))) + .sort((a, b) => (a.innerText || '').length - (b.innerText || '').length); + const panel = panels[0]; + if (!panel) return {found: false, states: {}}; + const rows = Array.from(panel.querySelectorAll('li, [class*="option"], [class*="Option"], label')) + .filter(visible) + .filter((el) => labels.includes((el.innerText || '').trim())); + const states = {}; + for (const row of rows) { + const label = (row.innerText || '').trim(); + if (states[label] !== undefined) continue; + const input = row.querySelector('input[type="checkbox"]'); + const cls = String(row.className || ''); + const checked = input ? !!input.checked + : (/(checked|active|selected)/.test(cls) && !/unchecked/.test(cls)); + states[label] = checked; + if (checked !== desired[label]) { + const target = row.querySelector('input[type="checkbox"]') + || row.querySelector('[class*="checkbox"]') || row; + target.click(); + } + } + return {found: true, states}; + }""", + desired, + ) + if not state.get("found"): + page.keyboard.press("Escape") + raise RuntimeError("接待类型下拉面板未打开或未包含全部选项") + page.wait_for_timeout(500) + + # 面板内点击后复查勾选状态。 + verify = page.evaluate( + """(desired) => { + const visible = (el) => { + if (!el) return false; + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + const labels = Object.keys(desired); + const panels = Array.from(document.querySelectorAll('div, ul, section')) + .filter(visible) + .filter((el) => labels.every((label) => + (el.innerText || '').includes(label))) + .sort((a, b) => (a.innerText || '').length - (b.innerText || '').length); + const panel = panels[0]; + if (!panel) return {found: false, states: {}}; + const rows = Array.from(panel.querySelectorAll('li, [class*="option"], [class*="Option"], label')) + .filter(visible) + .filter((el) => labels.includes((el.innerText || '').trim())); + const states = {}; + for (const row of rows) { + const label = (row.innerText || '').trim(); + if (states[label] !== undefined) continue; + const input = row.querySelector('input[type="checkbox"]'); + const cls = String(row.className || ''); + states[label] = input ? !!input.checked + : (/(checked|active|selected)/.test(cls) && !/unchecked/.test(cls)); + } + return {found: true, states}; + }""", + desired, + ) + page.keyboard.press("Escape") + page.wait_for_timeout(300) + + states = verify.get("states") if verify.get("found") else state.get("states", {}) + wrong = { + label: [states.get(label), want] + for label, want in desired.items() + if states.get(label) is not None and states.get(label) != want + } + # “全部”行在某些视图中不渲染，只要四个具体类型正确即视为生效。 + missing = [label for label in types if label not in states] + if wrong or missing: + raise RuntimeError( + f"接待类型勾选未生效，期望 {desired!r}，实际 {states!r}" + ) + + control_text = "" + try: + control_text = control.inner_text(timeout=5_000) + except Exception: + control_text = "" + if control_text and not any(item in control_text for item in types): + raise RuntimeError( + f"接待类型控件未显示所选类型，期望包含 {types!r}，实际 {control_text!r}" + ) + + +def recover_im_session(page: Page) -> bool: + """检测“登录过期”横幅并尝试 SSO 自助恢复；无法恢复时报错。""" + + text = _page_text(page, timeout=5_000) + if "登录过期" not in text: + return False + print(" [WARN] 检测到飞鸽会话登录过期，尝试重新登录恢复") + link = page.locator("text=重新登录").first + if not link.count(): + raise RuntimeError( + "飞鸽会话登录过期且未找到重新登录入口；" + "请执行 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + link.click() + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + current_text = _page_text(page, timeout=5_000) + if any(marker in current_text for marker in LOGIN_TEXT_MARKERS[:4]) or any( + marker in (page.url or "").casefold() for marker in LOGIN_URL_MARKERS + ): + raise RuntimeError( + "抖音 SSO 登录态已失效，无法自助恢复；" + "请执行 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + if "登录过期" not in current_text and "历史会话" in current_text: + print(" [OK] 飞鸽会话已恢复") + page.wait_for_timeout(1_500) + return True + page.wait_for_timeout(1_000) + raise RuntimeError("飞鸽会话恢复超时，请稍后重试") + + +def apply_query(page: Page) -> int: + """点击“查询”并返回页面声明的会话总数。""" + + dismiss_blocking_modals(page) + recovered = False + while True: + page.get_by_role("button", name="查询", exact=True).first.click() + page.wait_for_timeout(2_000) + try: + page.wait_for_load_state("networkidle", timeout=15_000) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + last_text = "" + empty_polls = 0 + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + pagination = page.locator('[class*="pagination"]').last + try: + text = pagination.inner_text(timeout=3_000) if pagination.count() else "" + except Exception: + text = "" + if text: + last_text = text + match = TOTAL_COUNT_PATTERN.search(text) + if match: + return int(match.group(1)) + body_text = _page_text(page, timeout=5_000) + match = TOTAL_COUNT_PATTERN.search(body_text or "") + if match: + return int(match.group(1)) + # 历史会话只保留近 30 天：超出范围的日期查询结果为“暂无数据”， + # 连续两次轮询仍是空态即视为 0 条会话（防加载中的瞬时空态）。 + if "暂无数据" in (body_text or ""): + empty_polls += 1 + if empty_polls >= 2: + return 0 + else: + empty_polls = 0 + if "登录过期" in (body_text or "") and not recovered: + break + page.wait_for_timeout(500) + if recovered: + raise RuntimeError(f"查询后未识别会话总数: {last_text!r}") + recovered = recover_im_session(page) + if not recovered: + raise RuntimeError(f"查询后未识别会话总数: {last_text!r}") + page.wait_for_timeout(1_000) + + +_ROW_HEADER_MAP = { + "会话信息": "session_info", + "用户信息": "customer_nickname", + "会话来源": "consult_entry", + "接待类型": "reception_type", + "发起场景": "initiator_scene", + "结束场景": "end_scene", + "会话评价时间": "rating_time_label", + "会话评价结果": "rating_result", + "会话评价标签": "rating_tags", + "会话意图": "conversation_intent", + "智能接待时间": "smart_reception_label", + "智能评价时间": "smart_rating_time_label", + "智能评价结果": "smart_rating_result", + "智能评价标签": "smart_rating_tags", + "转人工时间": "transfer_human_duration", + "转人工类型": "transfer_human_type", +} + + +def wait_current_page_rows(page: Page, timeout: int = 30) -> list[dict[str, Any]]: + """等待当前页会话行渲染完成后读取。""" + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + rows = read_current_page_rows(page) + if rows: + return rows + page.wait_for_timeout(700) + return read_current_page_rows(page) + + +def read_current_page_rows(page: Page) -> list[dict[str, Any]]: + """读取当前页表格全部会话行，按表头映射字段并解析会话 ID。 + + 飞鸽数据表单把表头和表体渲染为两个独立 ；数据行带有 + data-qa-id/data-row-key 属性，会话 ID 直接取 data-row-key。 + """ + + payload = page.evaluate( + """() => { + const headerTable = Array.from(document.querySelectorAll('table')) + .find((item) => { + const text = item.innerText || ''; + return text.includes('会话信息') && text.includes('接待类型'); + }); + let headers = []; + if (headerTable) { + const headerRows = Array.from(headerTable.querySelectorAll('tr')); + const last = headerRows[headerRows.length - 1]; + if (last) { + headers = Array.from(last.querySelectorAll('th, td')) + .map((cell) => (cell.innerText || '').trim()); + } + } + const rows = Array.from( + document.querySelectorAll('tr[data-qa-id="qa-history-conversation-item"]') + ).map((row) => ({ + key: row.getAttribute('data-row-key') || '', + cells: Array.from(row.querySelectorAll('td')) + .map((cell) => (cell.innerText || '').trim()), + })); + return {headers, rows}; + }""" + ) + headers = [str(item).strip() for item in payload.get("headers") or []] + mapped: list[dict[str, Any]] = [] + for entry in payload.get("rows") or []: + cells = entry.get("cells") or [] + record: dict[str, Any] = {} + for index, header in enumerate(headers): + if index >= len(cells): + break + field = _ROW_HEADER_MAP.get(header) + if field: + record[field] = cells[index] + session_info = str(record.pop("session_info", "") or "") + conversation_id = str(entry.get("key") or "").strip() + if not conversation_id: + id_match = CONVERSATION_ID_PATTERN.search(session_info) + if not id_match: + continue + conversation_id = id_match.group(1) + record["conversation_id"] = conversation_id + record["session_time_label"] = session_info.replace( + conversation_id, "" + ).replace("ID：", "").replace("ID:", "").strip() + record["raw_cells"] = cells + mapped.append(record) + return mapped + + +_TABLE_SCROLL_INFO_JS = """() => { + const rows = Array.from( + document.querySelectorAll('tr[data-qa-id="qa-history-conversation-item"]') + ); + let el = rows.length ? rows[0].parentElement : null; + while (el && el !== document.body) { + if (el.scrollHeight > el.clientHeight + 50) break; + el = el.parentElement; + } + if (!el || el === document.body) return {found: 0}; + return { + found: 1, + top: el.scrollTop, + height: el.scrollHeight, + client: el.clientHeight, + }; +}""" + + +def scan_current_page_rows( + page: Page, max_steps: int = 80, *, settle_ms: int = 250 +) -> list[dict[str, Any]]: + """深度扫描当前页的会话行，返回按出现顺序去重后的全部行。 + + 历史会话表格是虚拟滚动列表，未滚动到的行不会挂在 DOM 上；只读一次 + 会漏掉本页大部分行，导致“页面声明 N 条、实际遍历不足 N 条”。这里 + 逐屏滚动表格容器让全部行渲染并累积读取，扫完回到顶部。 + """ + + seen: dict[str, dict[str, Any]] = {} + stale = 0 + last_top = -1.0 + for _ in range(max_steps): + for row in read_current_page_rows(page): + seen.setdefault(row["conversation_id"], row) + info = page.evaluate(_TABLE_SCROLL_INFO_JS) + if not info.get("found"): + # 无虚拟滚动容器：全部行已挂载，一次读取即为完整页。 + break + at_bottom = info["top"] + info["client"] >= info["height"] - 8 + if at_bottom and info["top"] == last_top: + stale += 1 + if stale >= 2: + break + else: + stale = 0 + last_top = info["top"] + if at_bottom and info["height"] <= info["client"] + 8: + break + page.evaluate( + """(step) => { + const rows = Array.from( + document.querySelectorAll( + 'tr[data-qa-id="qa-history-conversation-item"]' + ) + ); + let el = rows.length ? rows[0].parentElement : null; + while (el && el !== document.body) { + if (el.scrollHeight > el.clientHeight + 50) break; + el = el.parentElement; + } + if (!el || el === document.body) return; + el.scrollTop = Math.min( + el.scrollTop + step, el.scrollHeight + ); + el.dispatchEvent(new Event('scroll', {bubbles: true})); + }""", + max(float(info["client"]) * 0.8, 200.0), + ) + page.wait_for_timeout(settle_ms) + page.evaluate( + """() => { + const rows = Array.from( + document.querySelectorAll( + 'tr[data-qa-id="qa-history-conversation-item"]' + ) + ); + let el = rows.length ? rows[0].parentElement : null; + while (el && el !== document.body) { + if (el.scrollHeight > el.clientHeight + 50) break; + el = el.parentElement; + } + if (!el || el === document.body) return; + el.scrollTop = 0; + el.scrollLeft = 0; + el.dispatchEvent(new Event('scroll', {bubbles: true})); + }""" + ) + page.wait_for_timeout(200) + return list(seen.values()) + + +MESSAGE_TIME_PATTERN = re.compile( + r"^(\d{1,2})月(\d{1,2})日\s+(\d{1,2}):(\d{2})(?::(\d{2}))?$" +) + +# 会话抽屉右侧「咨询订单」面板的订单卡字段标签（值在标签的后续行）。 +ORDER_CARD_LABELS = ( + "规格", + "编码", + "实付金额", + "付款时间", + "物流信息", + "收货信息", + "发货时间", +) +ORDER_STATUS_WORDS = frozenset( + { + "已完成", + "已发货", + "已完结", + "未完结", + "售后中", + "已关闭", + "待发货", + "已取消", + } +) +ORDER_ID_LINE_PATTERN = re.compile(r"^\d{15,20}$") +ORDER_BADGE_WORDS = frozenset({"咨询订单", "商品卡", "承诺服务工单", "修改"}) +ORDER_PAID_AT_PATTERN = re.compile(r"(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2})") +ORDER_SHIPPED_AT_PATTERN = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2})") +ORDER_AFTER_SALE_PATTERN = re.compile(r"^[（(]([^）)]+)[)）]$") +ORDER_LOGISTICS_STATUS_PATTERN = re.compile(r"^\[([^\]]+)\](.*)$") + + +def parse_order_card(text: str) -> dict[str, Any] | None: + """把右侧咨询订单面板的单张订单卡文本解析为结构化字段。 + + 卡片文本按行切分：订单号行之前是状态/角标，其后依次是备注、商品 + 标题、服务标签和「标签-值」字段区。解析失败（无订单号行）返回 + None，调用方跳过该卡片并保留原始文本。 + """ + + lines = [line.strip() for line in (text or "").splitlines() if line.strip()] + order_id = None + order_id_index = -1 + for index, line in enumerate(lines): + if ORDER_ID_LINE_PATTERN.match(line): + order_id = line + order_id_index = index + break + if order_id is None: + return None + + head = lines[:order_id_index] + body = lines[order_id_index + 1 :] + + order_status = next( + (line for line in head if line in ORDER_STATUS_WORDS), None + ) + after_sale_status = None + for line in head: + match = ORDER_AFTER_SALE_PATTERN.match(line) + if match: + after_sale_status = match.group(1) + break + order_badge = next( + (line for line in head if line in {"咨询订单", "商品卡"}), None + ) + + label_positions: dict[str, int] = {} + for index, line in enumerate(body): + if line in ORDER_CARD_LABELS and line not in label_positions: + label_positions[line] = index + + def _label_value(label: str) -> list[str]: + start = label_positions.get(label) + if start is None: + return [] + end = len(body) + for other, position in label_positions.items(): + if position > start: + end = min(end, position) + return body[start + 1 : end] + + first_label_index = min(label_positions.values()) if label_positions else len(body) + middle = body[:first_label_index] + + # 商品标题：优先取 [已发x/x] 行的上一行；该行不像标题（如状态词 + # “待发货”被页面插在订单号和标题之间）时，扫描 middle 取第一个 + # 标题样行，避免把状态/角标误存成商品名。 + def _title_like(line: str) -> bool: + return ( + len(line) >= 6 + and line not in ORDER_STATUS_WORDS + and line not in ORDER_BADGE_WORDS + and not ORDER_AFTER_SALE_PATTERN.match(line) + and not line.startswith("[") + and not re.match(r"^\+\d+$", line) + and ";" not in line + ) + + sent_index = next( + (i for i, line in enumerate(middle) if line.startswith("[已发")), + None, + ) + product_title = None + if sent_index is not None and sent_index > 0 and _title_like( + middle[sent_index - 1] + ): + product_title = middle[sent_index - 1] + if product_title is None: + product_title = next((line for line in middle if _title_like(line)), None) + if sent_index is not None: + service_tags = list(middle[sent_index + 1 :]) + else: + service_tags = [] + remark_text = next( + (line for line in middle[: sent_index or len(middle)] if ";" in line), + None, + ) + + spec_lines = _label_value("规格") + # 规格值可能后跟“退货退款/售后关闭”等角标行，只取首行保持字段干净。 + spec = spec_lines[0].strip() if spec_lines else None + spec_id = None + if spec: + match = re.search(r"[（(]ID[:：]\s*([^)）]+)[)）]", spec) + if match: + spec_id = match.group(1) + code_lines = _label_value("编码") + product_code = code_lines[0].strip() if code_lines else None + + paid_raw = " ".join(_label_value("实付金额")).strip() or None + paid_amount = None + discount_text = None + freight_text = None + if paid_raw: + compact = paid_raw.replace(" ", "") + match = re.search(r"¥([\d.,]+)", compact) + if match: + paid_amount = match.group(1).rstrip(",") + match = re.search(r"含运费¥([\d.,]+)", compact) + if match: + freight_text = match.group(1) + match = re.search(r"优惠¥([\d.,]+)", compact) + if match: + discount_text = match.group(1) + + pay_channel = None + paid_at = None + payment_lines = " ".join(_label_value("付款时间")).strip() + if payment_lines: + match = ORDER_PAID_AT_PATTERN.search(payment_lines) + if match: + paid_at = datetime.strptime( + match.group(1), "%Y/%m/%d %H:%M:%S" + ).replace(tzinfo=SHANGHAI) + match = re.search(r"[（(]([^)）]+)[)）]\s*$", payment_lines) + if match: + pay_channel = match.group(1) + + logistics_lines = _label_value("物流信息") + logistics_company = None + logistics_no = None + logistics_status = None + for line in logistics_lines: + status_match = ORDER_LOGISTICS_STATUS_PATTERN.match(line) + if status_match: + logistics_status = f"{status_match.group(1)} {status_match.group(2)}".strip() + continue + if logistics_company is None: + parts = line.split() + logistics_company = parts[0] if parts else line + if len(parts) > 1: + logistics_no = parts[1] + elif logistics_no is None: + logistics_no = line + + receiver_info = "".join(_label_value("收货信息")).strip() or None + + shipped_at = None + shipped_lines = " ".join(_label_value("发货时间")).strip() + if shipped_lines: + match = ORDER_SHIPPED_AT_PATTERN.search(shipped_lines) + if match: + shipped_at = datetime.strptime( + match.group(1), "%Y-%m-%d %H:%M" + ).replace(tzinfo=SHANGHAI) + + return { + "order_id": order_id, + "order_status": order_status, + "after_sale_status": after_sale_status, + "order_badge": order_badge, + "product_title": product_title, + "product_code": product_code, + "spec": spec, + "spec_id": spec_id, + "paid_amount": paid_amount, + "paid_raw": paid_raw, + "pay_channel": pay_channel, + "paid_at": paid_at, + "discount_text": discount_text, + "freight_text": freight_text, + "logistics_company": logistics_company, + "logistics_no": logistics_no, + "logistics_status": logistics_status, + "receiver_info": receiver_info, + "shipped_at": shipped_at, + "service_tags": "、".join(service_tags) if service_tags else None, + "remark_text": remark_text, + "raw_card": {"text": text}, + } + + +def _drawer_container_text(page: Page) -> str: + for selector in ( + '.auxo-drawer-title:has-text("本通会话明细")', + '[class*="drawer-title"]:has-text("本通会话明细")', + ): + title = page.locator(selector).first + try: + if not title.count(): + continue + parent_text = "" + try: + parent_text = title.locator("xpath=..").inner_text(timeout=5_000) + except Exception: + parent_text = "" + return f"{title.inner_text(timeout=5_000)}\n{parent_text}" + except Exception: + continue + drawer = page.locator( + '[class*="drawer"]:has-text("本通会话明细"), ' + '[class*="Drawer"]:has-text("本通会话明细")' + ).last + if not drawer.count(): + return "" + try: + return drawer.inner_text(timeout=10_000) + except Exception: + return "" + + +def _parse_drawer_header(text: str) -> dict[str, Any]: + meta: dict[str, Any] = {} + for label, field in ( + ("咨询入口", "consult_entry"), + ("会话评价", "rating_result"), + ("服务问题标签", "service_problem_tags"), + ("未达标指标", "unmet_metrics"), + ): + match = re.search(rf"{label}[:：]?\s+([^\n]+)", text or "") + if match: + value = match.group(1).strip() + meta[field] = None if value in {"-", "—"} else value + return meta + + +def parse_message_time_label( + label: str | None, business_date: date +) -> datetime | None: + """把“9月16日 23:43:25”解析为上海时区时间，年份按业务日期锚定。""" + + if not label: + return None + match = MESSAGE_TIME_PATTERN.match(label.strip()) + if not match: + return None + month, day, hour, minute, second = match.groups() + parsed = datetime( + business_date.year, + int(month), + int(day), + int(hour), + int(minute), + int(second or 0), + tzinfo=SHANGHAI, + ) + if parsed.date() > business_date: + parsed = parsed.replace(year=parsed.year - 1) + return parsed + + +def _collect_rendered_messages(page: Page) -> list[dict[str, Any]]: + """读取虚拟列表当前渲染出的全部消息项。""" + + return page.evaluate( + r"""() => { + const wrappers = Array.from( + document.querySelectorAll('[data-qa-id="qa-message-warpper"]') + ); + const visible = (el) => { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== 'none' && style.visibility !== 'hidden' + && rect.width > 0 && rect.height > 0; + }; + const items = []; + for (const wrapper of wrappers) { + if (!visible(wrapper)) continue; + const positioned = wrapper.closest('[style*="translate"]') || wrapper; + const styleText = positioned.getAttribute('style') || ''; + const topMatch = styleText.match(/translate\(([-\d.]+)px?,\s*([-\d.]+)px\)/); + const top = topMatch ? parseFloat(topMatch[2]) : 0; + const idEl = wrapper.querySelector('[data-id]'); + const dataId = idEl ? (idEl.getAttribute('data-id') || '') : ''; + const timeEl = Array.from(wrapper.querySelectorAll('div')) + .find((el) => el.children.length === 0 + && /^\d{1,2}月\d{1,2}日\s+\d{1,2}:\d{2}/.test((el.innerText || '').trim())); + const timeLabel = timeEl ? (timeEl.innerText || '').trim() : ''; + const fullTimeEl = Array.from(wrapper.querySelectorAll('[multiple-click-times]')) + .find((el) => /^\d{1,2}月\d{1,2}日\s+\d{1,2}:\d{2}(:\d{2})?$/.test((el.innerText || '').trim())); + const fullTimeLabel = fullTimeEl ? (fullTimeEl.innerText || '').trim() : ''; + const bubble = wrapper.querySelector('.leaveMessage'); + const isMe = !!(bubble && /messageIsMe/.test(bubble.className || '')); + const isNotMe = !!(bubble && /messageNotMe/.test(bubble.className || '')); + const contentEl = bubble ? bubble.querySelector('pre span') : null; + let content = contentEl ? (contentEl.innerText || '').trim() : ''; + const imageEl = bubble ? bubble.querySelector('img') : null; + const imageUrl = imageEl ? (imageEl.getAttribute('src') || '') : ''; + const wrapperText = (wrapper.innerText || '').trim(); + const card = (() => { + if (!bubble) return null; + const text = (bubble.innerText || '').trim(); + if (!text) return null; + const price = text.match(/¥\s*([\d.]+)/); + const titleLine = text.split('\n').map((line) => line.trim()).filter(Boolean)[0] || ''; + const looksLikeProduct = /(邀请下单|规格\/属性|计算价格|券后价|已售\d*)/.test(text); + const looksLikeOrder = /(订单编号|订单号|实付|申请退款|退款成功)/.test(text) && !looksLikeProduct; + if (!looksLikeProduct && !looksLikeOrder) return null; + return { + kind: looksLikeProduct ? 'product_card' : 'order_card', + title: titleLine, + price: price ? price[1] : null, + text, + }; + })(); + if (!content && card) content = card.text; + if (!content && imageEl) content = wrapperText; + const readMark = !!wrapper.querySelector('[data-qa-id="qa-message-read-mark"]'); + // 候发行：非时间、非已读标记的短行（发送者标签/意图标签/系统文本）。 + const extras = Array.from(wrapper.querySelectorAll('div')) + .filter((el) => el.children.length === 0) + .map((el) => (el.innerText || '').trim()) + .filter((text) => text && text.length <= 60) + .filter((text) => !/^\d{1,2}月\d{1,2}日/.test(text)) + .filter((text) => text !== '已读' && text !== '未读') + .filter((text) => !content || !content.includes(text) || text === content); + items.push({ + dataId, top, timeLabel, fullTimeLabel, + isMe, isNotMe, + hasBubble: !!bubble, + content, imageUrl, card, readMark, extras, + text: wrapperText.slice(0, 2000), + }); + } + return items; + }""" + ) + + +def _scroller_info(page: Page) -> dict[str, float]: + return page.evaluate( + """() => { + const scroller = document.querySelector('.scroller'); + if (!scroller) return {found: 0, top: 0, height: 0, client: 0}; + return { + found: 1, + top: scroller.scrollTop, + height: scroller.scrollHeight, + client: scroller.clientHeight, + }; + }""" + ) + + +def _set_scroller_top(page: Page, top: float) -> None: + page.evaluate( + """(top) => { + const scroller = document.querySelector('.scroller'); + if (scroller) { + scroller.scrollTop = top; + scroller.dispatchEvent(new Event('scroll', {bubbles: true})); + } + }""", + top, + ) + + +def _wait_for_order_panel(page: Page, timeout: float = 8.0) -> str: + """等待右侧咨询订单面板懒加载完成；返回 no-tab/ready/timeout。""" + + deadline = time.monotonic() + timeout + state = "no-tab" + while time.monotonic() < deadline: + state = page.evaluate( + """() => { + const tab = document.querySelector( + '[data-qa-id="qa-order-all-type-tab"]' + ); + if (!tab) return 'no-tab'; + let el = tab; + let panel = null; + while (el.parentElement) { + el = el.parentElement; + const t = el.innerText || ''; + if (t.includes('本通会话明细')) break; + panel = el; + } + if (!panel) return 'no-panel'; + const text = panel.innerText || ''; + if (/编码|规格|暂无|无订单|暂未|已加载近6个月订单/.test(text)) { + return 'ready'; + } + return 'loading'; + }""" + ) + if state in ("no-tab", "no-panel", "ready"): + break + page.wait_for_timeout(400) + if state == "ready": + page.wait_for_timeout(300) + return state + + +def _collect_order_card_texts(page: Page) -> list[str]: + """读取右侧咨询订单面板中的订单卡文本（按出现顺序去重）。""" + + return page.evaluate( + """() => { + const drawer = Array.from( + document.querySelectorAll('[class*="drawer"]') + ).find((el) => (el.innerText || '').includes('本通会话明细')); + if (!drawer) return []; + const cards = []; + const leaves = Array.from(drawer.querySelectorAll('*')).filter( + (el) => el.children.length === 0 + && /^\\d{15,20}$/.test((el.innerText || '').trim()) + ); + for (const leaf of leaves) { + const oid = (leaf.innerText || '').trim(); + let node = leaf; + let chosen = null; + while (node.parentElement) { + node = node.parentElement; + const t = node.innerText || ''; + if (t.length > 2500) break; + if (t.includes(oid) && t.includes('编码')) { + chosen = node; + break; + } + } + if (!chosen) continue; + const text = (chosen.innerText || '').trim(); + if (text && !cards.includes(text)) cards.push(text); + } + return cards; + }""" + ) + + +def extract_drawer(page: Page) -> dict[str, Any]: + """读取“本通会话明细”面板：头部元数据 + 完整聊天消息 + 订单卡。 + + 消息区是虚拟滚动列表（.scroller）：先跳到顶部加载最早的历史，再向下 + 逐屏收集；内容不足一屏时直接收集一次即可。 + """ + + page.locator(':text-matches("本通会话明细")').first.wait_for( + state="visible", timeout=15_000 + ) + page.wait_for_timeout(400) + + drawer_meta = _parse_drawer_header(_drawer_container_text(page)) + + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if _scroller_info(page).get("found"): + break + page.wait_for_timeout(400) + if not _scroller_info(page).get("found"): + raise RuntimeError("会话明细消息区未加载（未找到消息滚动容器）") + + seen: dict[str, dict[str, Any]] = {} + + def _collect() -> None: + for item in _collect_rendered_messages(page): + key = item.get("dataId") or f"auto:{item.get('top')}:{(item.get('text') or '')[:80]}" + seen[key] = item + + # 跳到顶部触发历史消息加载。 + _set_scroller_top(page, 0) + page.wait_for_timeout(400) + _collect() + + # 自顶向下逐屏收集；滚动位置不再变化且已到底部时结束。 + stale = 0 + last_top = -1.0 + for _ in range(240): + _collect() + info = _scroller_info(page) + at_bottom = info["top"] + info["client"] >= info["height"] - 8 + if at_bottom and info["top"] == last_top: + stale += 1 + else: + stale = 0 + if stale >= 2: + break + last_top = info["top"] + if at_bottom and info["height"] <= info["client"] + 8: + break + _set_scroller_top(page, info["top"] + max(info["client"] * 0.8, 200)) + page.wait_for_timeout(300) + + items = sorted(seen.values(), key=lambda item: (item.get("top") or 0)) + messages: list[dict[str, Any]] = [] + for item in items: + content = str(item.get("content") or "").strip() + text = str(item.get("text") or "").strip() + extras = [ + str(line).strip() + for line in (item.get("extras") or []) + if str(line).strip() and str(line).strip() != content + ] + if not content and not text and not extras: + continue + card = item.get("card") if isinstance(item.get("card"), dict) else None + image_url = str(item.get("imageUrl") or "").strip() + has_bubble = bool(item.get("hasBubble")) + intent_tag: str | None = None + sender_label: str | None = None + if not has_bubble: + sender_role = "system" + if not content and extras: + content = extras[0] + extras = extras[1:] + elif item.get("isMe"): + sender_role = "service" + if extras: + sender_label = extras[0] + elif item.get("isNotMe"): + sender_role = "customer" + intent_tag = next((line for line in extras if "-" in line), None) + else: + sender_role = "system" + if not content and extras: + content = extras[0] + extras = extras[1:] + if card: + message_type = str(card.get("kind") or "product_card") + elif image_url and not content: + message_type = "image" + elif not has_bubble: + message_type = "system" + else: + message_type = "text" + if not content and message_type != "image": + content = text or None + time_label = str(item.get("fullTimeLabel") or item.get("timeLabel") or "").strip() or None + messages.append( + { + "sender_role": sender_role, + "sender_name": sender_label, + "content": content or None, + "message_type": message_type, + "card": card or {}, + "time_label": time_label, + "message_at_label": time_label, + "message_uuid": str(item.get("dataId") or "").strip() or None, + "read_mark": bool(item.get("readMark")), + "raw_message": { + "data_id": item.get("dataId"), + "text": text[:2000], + "image_url": image_url or None, + "time_label": time_label, + "sender_label": sender_label, + "intent_tag": intent_tag, + "extras": extras[:10], + "read_mark": bool(item.get("readMark")), + }, + } + ) + _wait_for_order_panel(page) + order_cards = _collect_order_card_texts(page) + return { + "drawer_meta": drawer_meta, + "messages": messages, + "order_cards": order_cards, + } + + +def _scroll_table_to_row(page: Page, conversation_id: str, max_steps: int = 40) -> bool: + """虚拟滚动表格会把视口外的行卸载，按 ID 顺序滚动直到目标行渲染。""" + + target = int(conversation_id) if conversation_id.isdigit() else 0 + for _ in range(max_steps): + if page.locator(f'tr[data-row-key="{conversation_id}"]').count(): + return True + page.evaluate( + """(target) => { + const rows = Array.from( + document.querySelectorAll('tr[data-qa-id="qa-history-conversation-item"]') + ); + let el = rows.length ? rows[0].parentElement : null; + while (el && el !== document.body) { + if (el.scrollHeight > el.clientHeight + 50) break; + el = el.parentElement; + } + const container = (el && el !== document.body) ? el : null; + let direction = -1; + if (rows.length && target) { + const firstKey = Number(rows[0].getAttribute('data-row-key') || 0); + const lastKey = Number(rows[rows.length - 1].getAttribute('data-row-key') || 0); + // 列表按时间倒序、ID 递减：目标比首行还大说明在上方。 + direction = target > firstKey ? -1 : (target < lastKey ? 1 : 0); + if (direction === 0) direction = 1; + } + if (container) { + const step = Math.max(container.clientHeight * 0.8, 200); + container.scrollTop = Math.max( + 0, container.scrollTop + direction * step + ); + container.dispatchEvent(new Event('scroll', {bubbles: true})); + } else { + window.scrollBy(0, direction * 400); + } + }""", + target, + ) + page.wait_for_timeout(400) + return page.locator(f'tr[data-row-key="{conversation_id}"]').count() > 0 + + +def _reset_table_horizontal_scroll(page: Page) -> None: + """把会话表格的横向滚动复位到最左。 + + 表格被横向滚到最右时（如翻页前 scroll_into_view 分页器导致）， + 左侧固定列连同“查看会话”入口会被虚拟化卸载，行内找不到可点击的 + 链接；复位后 auxo 会重新挂载固定列。 + """ + + page.evaluate( + """() => { + const rows = Array.from( + document.querySelectorAll( + 'tr[data-qa-id="qa-history-conversation-item"]' + ) + ); + let el = rows.length ? rows[0].parentElement : null; + while (el && el !== document.body) { + if ( + el.scrollWidth > el.clientWidth + 50 + || el.scrollHeight > el.clientHeight + 50 + ) { + break; + } + el = el.parentElement; + } + if (!el || el === document.body) return; + if (el.scrollLeft) { + el.scrollLeft = 0; + el.dispatchEvent(new Event('scroll', {bubbles: true})); + } + }""" + ) + + +def open_conversation_drawer(page: Page, conversation_id: str) -> None: + """点击指定会话行的“查看会话”并等待抽屉打开。""" + + dismiss_blocking_modals(page) + # 表格查询后会异步重渲染，固定右侧操作列可能短暂未挂载，等待其稳定。 + link = None + row = None + deadline = time.monotonic() + 25 + while time.monotonic() < deadline: + _reset_table_horizontal_scroll(page) + row = page.locator(f'tr[data-row-key="{conversation_id}"]').first + if not row.count(): + if not _scroll_table_to_row(page, conversation_id, max_steps=6): + page.wait_for_timeout(500) + continue + row = page.locator(f'tr[data-row-key="{conversation_id}"]').first + candidate = row.locator('a[data-qa-id="qa-view-conversation"]').first + try: + if candidate.count() and candidate.is_visible(): + link = candidate + break + except Exception: + pass + fallback = row.get_by_text("查看会话", exact=True).last + try: + if fallback.count() and fallback.is_visible(): + link = fallback + break + except Exception: + pass + page.wait_for_timeout(500) + if link is None or row is None: + raise RuntimeError(f"会话 {conversation_id} 的“查看会话”入口不可见") + row.scroll_into_view_if_needed(timeout=10_000) + link.click() + page.locator(':text-matches("本通会话明细")').first.wait_for( + state="visible", timeout=15_000 + ) + page.wait_for_timeout(600) + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + drawer_text = _drawer_container_text(page) + if conversation_id in drawer_text: + return + page.wait_for_timeout(500) + raise RuntimeError( + f"打开的会话明细与目标不一致，期望 {conversation_id}，抽屉内容未包含" + ) + + +def close_conversation_drawer(page: Page) -> None: + for selector in ( + '[class*="drawer"]:has-text("本通会话明细") [class*="close"]:visible', + '[class*="Drawer"]:has-text("本通会话明细") [class*="close"]:visible', + ':text-matches("本通会话明细") >> xpath=.. >> [class*="close"]:visible', + ): + button = page.locator(selector).first + try: + if button.count() and button.is_visible(): + button.click() + page.wait_for_timeout(400) + return + except Exception: + continue + page.keyboard.press("Escape") + page.wait_for_timeout(400) + + +def set_page_size(page: Page, size: int = 100) -> int: + """把分页大小调大以减少翻页次数；失败时保持当前页大小。""" + + try: + selector = page.locator( + '[class*="pagination"] [class*="select"]:has-text("条/页"), ' + '[class*="pagination"] :text-matches("\\d+\\s*条\\s*/\\s*页")' + ).last + if not selector.count() or not selector.is_visible(): + return 20 + selector.click() + page.wait_for_timeout(600) + option = page.locator( + f'[class*="option"]:has-text("{size} 条/页"), ' + f'li:has-text("{size} 条/页")' + ).last + if not option.count() or not option.is_visible(): + page.keyboard.press("Escape") + return 20 + option.click() + page.wait_for_timeout(1_500) + summary = page.locator( + '[class*="pagination"] :text-matches("\\d+\\s*条\\s*/\\s*页")' + ).last + text = summary.inner_text(timeout=5_000) if summary.count() else "" + if f"{size}" in (text or ""): + print(f" [OK] 分页大小已设置为 {size} 条/页") + return size + return 20 + except Exception as exc: + print(f" [WARN] 设置分页大小失败，按 20 条/页继续: {exc}") + return 20 + + +def _active_page_number(page: Page) -> int | None: + try: + text = page.locator( + '.auxo-pagination-item-active:visible, ' + '[class*="pagination"] [class*="active"]:visible' + ).first.inner_text(timeout=3_000) + except Exception: + return None + match = re.search(r"\d+", text or "") + return int(match.group()) if match else None + + +def _pagination_info(page: Page) -> dict[str, int | None]: + """从分页器 DOM 读取激活页码 / 每页条数 / 总页数。""" + + try: + data = page.evaluate( + """() => { + const root = document.querySelector( + '.auxo-pagination, [class*="pagination"]' + ); + if (!root) return {}; + const text = root.innerText || ''; + const active = root.querySelector( + '.auxo-pagination-item-active, [class*="item-active"], ' + + 'li[class*="active"]' + ); + const sizeEl = Array.from( + root.querySelectorAll('*') + ).find((el) => /\\d+\\s*条\\s*\\/\\s*页/.test(el.textContent || '') + && el.children.length === 0); + const pageNums = Array.from( + root.querySelectorAll( + '.auxo-pagination-item, li[title], [class*="item"]' + ) + ).map((el) => Number((el.textContent || '').trim())) + .filter((n) => Number.isInteger(n) && n > 0); + const sizeMatch = sizeEl + ? (sizeEl.textContent || '').match(/(\\d+)\\s*条/) + : text.match(/(\\d+)\\s*条\\s*\\/\\s*页/); + return { + active: active ? Number((active.textContent || '').trim()) : null, + pageSize: sizeMatch ? Number(sizeMatch[1]) : null, + maxPage: pageNums.length ? Math.max(...pageNums) : null, + }; + }""" + ) + except Exception: + return {"active": None, "page_size": None, "max_page": None} + active = data.get("active") + page_size = data.get("pageSize") + max_page = data.get("maxPage") + return { + "active": int(active) if isinstance(active, (int, float)) else None, + "page_size": int(page_size) if isinstance(page_size, (int, float)) else None, + "max_page": int(max_page) if isinstance(max_page, (int, float)) else None, + } + + +def _next_page_disabled(page: Page) -> bool: + """下一页按钮是否处于禁用态（已是最后一页）。""" + + try: + return bool( + page.evaluate( + """() => { + const btn = document.querySelector( + '.auxo-pagination-next, [class*="pagination"] li[title="下一页"], ' + + '[class*="pagination"] [class*="next"]' + ); + if (!btn) return false; + const cls = btn.className || ''; + return cls.includes('disabled') + || btn.getAttribute('aria-disabled') === 'true' + || btn.disabled === true; + }""" + ) + ) + except Exception: + return False + + +def goto_next_page(page: Page, current_first_id: str) -> bool: + """翻到下一页并等待表格首行变化；已是最后一页时返回 False。""" + + dismiss_blocking_modals(page) + pagination = page.locator('[class*="pagination"]').last + try: + pagination.scroll_into_view_if_needed(timeout=5_000) + except Exception: + pass + if _next_page_disabled(page): + print(" [INFO] 已到最后一页（下一页禁用）") + return False + before_number = _active_page_number(page) + for attempt in range(2): + if attempt == 0: + next_button = page.locator( + '.auxo-pagination-next:visible, ' + '[class*="pagination"] [class*="next"]:visible, ' + '[class*="pagination"] li[title="下一页"]:visible, ' + '[class*="pagination"] [title="下一页"]:visible, ' + '[class*="pagination"] button:has-text(">"):visible' + ).first + else: + # “下一页”点击无效时直接点目标页码项，绕过按钮状态异常。 + target_page = (before_number or 1) + 1 + next_button = page.locator( + f'[class*="pagination"] li[title="{target_page}"]:visible, ' + f'[class*="pagination"] [title="{target_page}"]:visible, ' + f'[class*="pagination"] li:has-text("{target_page}"):visible' + ).first + if not next_button.count(): + if attempt == 0: + continue + raise RuntimeError("分页“下一页”按钮不可见，无法继续采集") + try: + next_button.click(timeout=5_000) + except Exception: + next_button.click(force=True) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + rows = read_current_page_rows(page) + if rows and rows[0]["conversation_id"] != current_first_id: + # 首行内容变化是最强信号；页码变化只作辅助参考。 + return True + after_number = _active_page_number(page) + if ( + before_number is not None + and after_number is not None + and after_number != before_number + and not rows + ): + # 页码已变但表格尚未重渲染，继续等待内容刷新。 + page.wait_for_timeout(500) + continue + page.wait_for_timeout(500) + if _next_page_disabled(page): + # 点击后按钮进入禁用态，说明刚才翻到了最后一页。 + print(" [INFO] 已到最后一页（点击后下一页禁用）") + return False + if attempt == 0: + print(" [WARN] 翻页后表格未更新，改用直接点击页码重试") + page.wait_for_timeout(1_000) + raise RuntimeError("翻页后表格内容未更新") + + +def build_product_records( + orders: list[dict[str, Any]], business_date: date +) -> dict[str, dict[str, Any]]: + """把本次会话的订单记录聚合成商品维表增量（按编码分组计标题变体）。""" + + batch: dict[str, dict[str, Any]] = {} + for order in orders: + code = str(order.get("product_code") or "").strip() + title = str(order.get("product_title") or "").strip() + if not code: + continue + entry = batch.setdefault( + code, + { + "product_code": code, + "title_variants": {}, + "first_seen_date": business_date, + "last_seen_date": business_date, + }, + ) + if title: + entry["title_variants"][title] = ( + entry["title_variants"].get(title, 0) + 1 + ) + entry["first_seen_date"] = min( + entry["first_seen_date"], business_date + ) + entry["last_seen_date"] = max( + entry["last_seen_date"], business_date + ) + return batch + + +def merge_product_records( + existing: list[dict[str, Any]], + batch: dict[str, dict[str, Any]], +) -> list[dict[str, Any]]: + """把维表增量合并到既有行：变体计数累加，标题取频次最高者。""" + + merged: list[dict[str, Any]] = [] + for code, increment in batch.items(): + current = next( + (row for row in existing if row.get("product_code") == code), + None, + ) + variants: dict[str, int] = dict( + (current or {}).get("title_variants") or {} + ) + for title, count in (increment.get("title_variants") or {}).items(): + variants[title] = variants.get(title, 0) + count + variants = {title: n for title, n in variants.items() if title} + if not variants: + continue + title = max(variants, key=lambda item: (variants[item], -len(item))) + if current and current.get("product_title") in variants: + if variants[current["product_title"]] == variants[title]: + title = current["product_title"] + first_seen = increment.get("first_seen_date") + last_seen = increment.get("last_seen_date") + if current and current.get("first_seen_date"): + first_seen = min(first_seen, current["first_seen_date"]) + if current and current.get("last_seen_date"): + last_seen = max(last_seen, current["last_seen_date"]) + merged.append( + { + "product_code": code, + "product_title": title, + "first_seen_date": first_seen, + "last_seen_date": last_seen, + "title_variants": variants, + } + ) + return merged + + +def _persist_records( + normalized: dict[str, Any], + *, + no_db: bool, +) -> tuple[int, int, int]: + if no_db: + return 0, 0, 0 + from gyxx_flow.modules.shop_intelligence.db import ( + fetch_dy_customer_service_products, + get_conn, + upsert_dy_customer_service_conversation_orders, + upsert_dy_customer_service_conversations, + upsert_dy_customer_service_messages, + upsert_dy_customer_service_products, + ) + + orders = normalized.get("orders") or [] + with get_conn() as conn: + conversations = upsert_dy_customer_service_conversations( + conn, [normalized["conversation"]] + ) + messages = upsert_dy_customer_service_messages(conn, normalized["messages"]) + orders_count = upsert_dy_customer_service_conversation_orders( + conn, orders + ) + batch = build_product_records( + orders, normalized["conversation"]["business_date"] + ) + if batch: + existing = fetch_dy_customer_service_products(conn, list(batch)) + products = merge_product_records(existing, batch) + if products: + upsert_dy_customer_service_products(conn, products) + return conversations, messages, orders_count + + +def preflight_acceptance_login( + environment: dict[str, str] | None = None, +) -> int | None: + policy = current_acceptance_policy() + if not policy.enabled: + return None + values = os.environ if environment is None else environment + result = policy.preflight_cookie( + binding_from_environment(values), environment=values + ) + if not result.should_skip: + return None + print(f"[SKIPPED_COOKIE] {result.reason}") + return COOKIE_SKIP_EXIT_CODE + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="抖音飞鸽历史会话消息按日采集") + parser.add_argument( + "--date", + type=date.fromisoformat, + default=None, + help="业务日期 YYYY-MM-DD；控制台会自动传入", + ) + parser.add_argument( + "--range-days", + type=int, + default=0, + help="采集 business_date 往前 N 天（含当天）的区间；定时任务使用，" + "0 表示仅采集 --date 单日", + ) + parser.add_argument( + "--reception-types", + default=",".join(DEFAULT_RECEPTION_TYPES), + help="接待类型，逗号分隔；默认 纯人工客服接待,智能客服转人工接待", + ) + parser.add_argument( + "--output", type=Path, default=DEFAULT_OUTPUT, help="原始行数据和失败现场目录" + ) + parser.add_argument("--headless", action="store_true", help="无界面运行浏览器") + parser.add_argument("--no-db", action="store_true", help="只保存原始 JSON，不写数据库") + parser.add_argument("--retries", type=int, default=3, help="每个会话的最大尝试次数") + parser.add_argument( + "--max-conversations", + type=int, + default=0, + help="限制本次最多采集的会话数；0 表示不限制（调试用）", + ) + parser.add_argument( + "--login-timeout", type=int, default=300, help="人工登录最长等待秒数" + ) + return parser.parse_args(argv) + + +def _resolve_business_date(args: argparse.Namespace) -> date: + if args.date is not None: + return args.date + value = os.getenv("GYXX_BUSINESS_DATE", "").strip() + if value: + return date.fromisoformat(value) + return date.today() + + +def _resolve_reception_types(raw: str) -> tuple[str, ...]: + values = tuple(item.strip() for item in (raw or "").split(",") if item.strip()) + if not values: + raise ValueError("接待类型不能为空") + invalid = [item for item in values if item not in RECEPTION_TYPE_OPTIONS[1:]] + if invalid: + raise ValueError( + f"不支持的接待类型: {invalid!r}，可选项 {list(RECEPTION_TYPE_OPTIONS[1:])!r}" + ) + return tuple(dict.fromkeys(values)) + + +def run( + args: argparse.Namespace, + *, + runtime_environment: dict[str, str] | None = None, +) -> int: + if args.retries < 1: + raise ValueError("--retries 必须大于等于 1") + if args.login_timeout < 1: + raise ValueError("--login-timeout 必须大于等于 1") + environment = dict(runtime_environment or os.environ) + account = environment.get("GYXX_ACCOUNT_ID", ACCOUNT_ID).strip() or ACCOUNT_ID + business_date = _resolve_business_date(args) + range_days = int(getattr(args, "range_days", 0) or 0) + range_start = ( + business_date - timedelta(days=range_days - 1) + if range_days > 0 + else business_date + ) + reception_types = _resolve_reception_types(args.reception_types) + single_day = range_start == business_date + if single_day: + output_path = args.output / f"抖音客服消息_{business_date.isoformat()}.json" + else: + output_path = args.output / ( + f"抖音客服消息_{range_start.isoformat()}" + f"_to_{business_date.isoformat()}.json" + ) + + if not args.no_db: + from gyxx_flow.modules.shop_intelligence.db import init_schema + + init_schema() + print("数据库表 dy_customer_service_conversations / dy_customer_service_messages 已就绪") + + print("=" * 68) + print("抖音飞鸽历史会话消息采集") + print(f" 账号: {account}") + print(f" 会话日期: {range_start} ~ {business_date}") + print(f" 接待类型: {'、'.join(reception_types)}") + print(f" 输出: {output_path}") + print(f" 数据库: {'跳过' if args.no_db else '写入'}") + print("=" * 68) + + collected: dict[str, dict[str, Any]] = {} + raw_rows: dict[str, dict[str, Any]] = {} + failures: dict[str, str] = {} + previous = load_progress(output_path) + if ( + previous + and previous.get("account") == account + and previous.get("business_date") == business_date.isoformat() + and tuple(previous.get("reception_types") or ()) == reception_types + ): + for cid, item in (previous.get("conversations") or {}).items(): + collected[str(cid)] = item + for cid, row in (previous.get("raw_rows") or {}).items(): + raw_rows[str(cid)] = row + for cid, reason in (previous.get("failed_conversations") or {}).items(): + failures[str(cid)] = str(reason) + if collected or failures: + print( + f"断点续采：已有进度 {len(collected)} 条会话、" + f"{len(failures)} 条历史失败，本次只采集剩余部分" + ) + + browser_session, account_lock = start_collection_browser( + environment, headless=args.headless + ) + try: + with browser_session: + context = browser_session.context + if context is None: + raise RuntimeError("Scrapling 未创建抖音客服浏览器上下文") + page = context.pages[0] if context.pages else context.new_page() + _collect( + page, + browser_session, + args, + environment=environment, + account=account, + business_date=business_date, + range_start=range_start, + reception_types=reception_types, + output_path=output_path, + collected=collected, + raw_rows=raw_rows, + failures=failures, + ) + finally: + if account_lock is not None: + account_lock.__exit__(None, None, None) + + print( + f"采集完成：{business_date} 共 {len(collected)} 条会话，" + f"结果已保存到 {output_path}" + ) + return 0 + + +def _collect( + page: Page, + browser_session: ScraplingBrowser, + args: argparse.Namespace, + *, + environment: dict[str, str], + account: str, + business_date: date, + range_start: date | None = None, + reception_types: tuple[str, ...], + output_path: Path, + collected: dict[str, dict[str, Any]], + raw_rows: dict[str, dict[str, Any]], + failures: dict[str, str], +) -> None: + single_day = range_start is None or range_start == business_date + try: + ensure_douyin_login( + page, + environment=environment, + headless=args.headless, + login_timeout=args.login_timeout, + ) + verify_shop_identity(page, environment=environment) + _history_page_ready(page) + dismiss_blocking_modals(page) + browser_session.persist_bound_state() + + try: + set_date_range(page, range_start or business_date, business_date) + ensure_reception_types(page, reception_types) + total = apply_query(page) + # 分页大小改变会触发重新查询；这里再显式查询一次，确保 + # 总数与分页大小一致后再开始逐条采集。 + page_size = set_page_size(page, 100) + total = apply_query(page) + except Exception: + save_debug_artifacts(page, args.output, "filter_setup") + raise + print(f"筛选完成：{business_date} 共 {total} 条会话，开始逐条采集") + print(f" [INFO] 当前分页大小 {page_size} 条/页") + + def _snapshot(complete: bool) -> None: + save_progress( + output_path, + account=account, + business_date=business_date, + reception_types=reception_types, + expected_count=total, + collected=collected, + raw_rows=raw_rows, + complete=complete, + failed=failures, + ) + + _snapshot(complete=False) + + processed = 0 + for _ in range(MAX_PAGES): + rows = wait_current_page_rows(page) + if not rows and "登录过期" in _page_text(page): + if recover_im_session(page): + rows = wait_current_page_rows(page) + if not rows: + if len(collected) >= total: + break + raise RuntimeError( + f"当前页未读取到会话行（已采集 {len(collected)}/{total}）" + ) + rows = scan_current_page_rows(page) + info = _pagination_info(page) + print( + f" [INFO] 页码 {info['active']}/{info['max_page']}，" + f"分页大小 {info['page_size']}，本页深度扫描到 {len(rows)} 行" + ) + for row in rows: + if args.max_conversations > 0 and len(collected) >= args.max_conversations: + break + conversation_id = row["conversation_id"] + if conversation_id in collected: + # 历史会话是活列表：新会话到达会把旧行挤到其他页，重读时跳过。 + print(f" [INFO] 跳过已采集会话 {conversation_id}") + continue + last_error: Exception | None = None + for attempt in range(1, args.retries + 1): + try: + open_conversation_drawer(page, conversation_id) + drawer = extract_drawer(page) + close_conversation_drawer(page) + normalized = normalize_conversation( + row, + drawer["drawer_meta"], + drawer["messages"], + account=account, + business_date=business_date, + source_url=TARGET_URL, + order_cards=drawer.get("order_cards") or [], + ) + if not single_day: + # 区间模式：会话与消息按会话开始日期归属业务日。 + session_start = normalized["conversation"].get( + "session_start_at" + ) + if session_start is not None: + bucket = session_start.astimezone(SHANGHAI).date() + normalized["conversation"]["business_date"] = bucket + for message in normalized["messages"]: + message["business_date"] = bucket + conversation_count, message_count, order_count = _persist_records( + normalized, no_db=args.no_db + ) + collected[conversation_id] = { + "message_count": normalized["message_count"], + "order_count": normalized["conversation"]["order_count"], + "db_conversation_rows": conversation_count, + "db_message_rows": message_count, + "db_order_rows": order_count, + "captured_at": datetime.now().astimezone(), + } + raw_rows[conversation_id] = row + failures.pop(conversation_id, None) + _snapshot(complete=False) + processed += 1 + print( + f" [{processed:02d}/{total}] " + f"会话 {conversation_id} 完成" + f"（消息 {normalized['message_count']} 条，" + f"商品订单 {normalized['conversation']['order_count']} 个）" + ) + last_error = None + break + except Exception as exc: + last_error = exc + print( + f" [{processed + 1:02d}/{total}] 会话 {conversation_id} " + f"第 {attempt}/{args.retries} 次失败: {exc}" + ) + try: + close_conversation_drawer(page) + except Exception: + pass + if attempt < args.retries: + page.wait_for_timeout(500) + if last_error is not None: + # 单个会话失败不中止整批：记录后继续后续会话， + # 运行结束前统一上报，重试时只补失败部分。 + save_debug_artifacts( + page, args.output, f"conversation_{conversation_id}" + ) + failures[conversation_id] = str(last_error) + print( + f" [WARN] 会话 {conversation_id} 连续 " + f"{args.retries} 次失败，跳过并继续后续会话" + ) + if args.max_conversations > 0 and len(collected) + len(failures) >= args.max_conversations: + print( + f" [INFO] 已达到 --max-conversations={args.max_conversations}，提前结束" + ) + break + if len(collected) + len(failures) >= total: + break + info = _pagination_info(page) + active = info["active"] + total_pages = None + if info["page_size"]: + total_pages = -(-total // info["page_size"]) + elif info["max_page"]: + total_pages = info["max_page"] + if active and total_pages and active >= total_pages: + print(f" [INFO] 已到最后一页（页码 {active}/{total_pages}）") + break + if _next_page_disabled(page): + print(" [INFO] 已到最后一页（下一页禁用）") + break + if not goto_next_page(page, rows[0]["conversation_id"]): + break + + drift = total - len(collected) + if args.max_conversations > 0: + print( + f" [INFO] --max-conversations={args.max_conversations} 生效，" + f"跳过覆盖校验（本次采集 {len(collected)}）" + ) + _snapshot(complete=False) + return + if failures: + _snapshot(complete=False) + failed_preview = "、".join(list(failures)[:5]) + raise RuntimeError( + f"{len(failures)} 个会话采集失败（如 {failed_preview}）；" + f"已完成 {len(collected)}/{total}，进度已保留，可直接重试续采" + ) + if drift > max(3, int(total * 0.02)): + _snapshot(complete=False) + raise RuntimeError( + f"会话覆盖不完整：页面声明 {total} 条，实际完成 {len(collected)} 条" + ) + if drift > 0: + print(f" [WARN] 活列表漂移：声明 {total} 条，实际采集 {len(collected)} 条") + _snapshot(complete=True) + finally: + # 正常退出由 ScraplingBrowser 将绑定状态写回脚本专属状态文件； + # 账号主 vault 仍由 gyxx accounts login/sync 管理。 + pass + + +def _account_runtime_environment() -> dict[str, str]: + """注入脚本绑定环境；账号 vault Cookie 只读合并进脚本专属状态。 + + 通过 `gyxx run/backfill/scripts run` 启动时引擎已经注入了同一套环境， + 本函数只兜底直接以脚本方式启动的场景。vault 由 + `gyxx accounts login/sync` 统一管理，采集进程的状态写回只发生在 + 脚本专属 Cookie/Profile 文件。 + """ + + base_environment = dict(os.environ) + project_root = base_environment.get("GYXX_PROJECT_ROOT", "").strip() + data_root = base_environment.get("GYXX_DATA_ROOT", "").strip() + if not project_root or not data_root: + return base_environment + catalog = RuntimeIntegrationCatalog.load_default( + project_root=project_root, + data_root=data_root, + ) + try: + return catalog.environment_for(COMMAND_ID, base_environment) + except Exception as exc: + print(f" [INFO] 未解析到脚本绑定 {COMMAND_ID}，使用当前环境继续: {exc}") + return catalog.environment_for_account(ACCOUNT_ID, base_environment) + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + runtime_environment = _account_runtime_environment() + preflight_exit_code = preflight_acceptance_login(runtime_environment) + if preflight_exit_code is not None: + return preflight_exit_code + return run(args, runtime_environment=runtime_environment) + except DouyinSessionExpiredError: + return COOKIE_SKIP_EXIT_CODE + except KeyboardInterrupt: + print("\n用户中断。") + return 130 + except Exception as exc: + print(f"\n[ERROR] {exc}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/dy_negative_reviews.py b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_negative_reviews.py new file mode 100644 index 0000000..fb44786 --- /dev/null +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_negative_reviews.py @@ -0,0 +1,1101 @@ +"""采集抖店评价管理的差评并按业务日期幂等写入 PostgreSQL。 + +抖店后台「评价管理-评价明细-订单评价」页支持按评价等级（差评）和 +评价时间筛选。脚本复用抖音账号 vault 的 Cookie（有效 Cookie 直接跳过 +登录，指纹隐藏模式），点击「差评」筛选、把评价时间设置为业务日 +00:00:00-23:59:59，逐页读取评价的文字、图片、评价状态与商品信息 +（商品名称/商品ID/规格/规格ID），并幂等写入 PostgreSQL。 +历史会话仅保留近 30 天，超出范围时页面显示“暂无数据”，按 0 条成功。 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import time +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +from gyxx_flow.adapters import ( + COOKIE_SKIP_EXIT_CODE, + RuntimeIntegrationCatalog, + current_acceptance_policy, +) +from gyxx_flow.adapters.scrapling import BrowserPage as Page +from gyxx_flow.adapters.scrapling import ScraplingBrowser, is_browser_timeout_error +from gyxx_flow.core.config import Settings +from gyxx_flow.core.layout import DataLayout +from gyxx_flow.core.locks import LockManager, ResourceBusyError +from gyxx_flow.modules.shop_intelligence.collectors import ( + dy_customer_service_messages as douyin_common, +) + +SETTINGS = Settings.from_env() +MODULE_PATHS = DataLayout(SETTINGS.data_root).for_module("shop_intelligence") + +TARGET_URL = ( + "https://fxg.jinritemai.com/ffa/maftersale/comment" + "?tabtype=1&evaluation_detail_tab=order" +) +ACCOUNT_ID = "douyin-shop" +COMMAND_ID = "shop.dy_negative_reviews.collect" +WORKFLOW_ID = "shop.dy_negative_reviews.daily" +DEFAULT_OUTPUT = MODULE_PATHS.raw_root / "dy_negative_reviews" +CDP_PORT = int( + os.getenv("GYXX_BROWSER_CDP_PORT") + or os.getenv("GYXX_CDP_PORT") + or "22159" +) +SHANGHAI = ZoneInfo("Asia/Shanghai") +MAX_PAGES = 200 + +TOTAL_COUNT_PATTERN = re.compile(r"共(\d+)条") +ORDER_ID_PATTERN = re.compile(r"订单ID:(\d{10,})") +REVIEW_AT_PATTERN = re.compile(r"(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2})") +PRODUCT_ID_PATTERN = re.compile(r"商品ID:(\d{10,})") +SPEC_ID_PATTERN = re.compile(r"规格ID:(\d{10,})") +ORDER_STATUS_LIKE_WORDS = frozenset( + {"待发货", "已发货", "已完结", "未完结", "售后中", "已关闭", "修改"} +) + + +def parse_total_count(text: str) -> int: + match = re.search(r"共(\d+)条", text or "") + if not match: + raise RuntimeError(f"无法从页面识别差评总数: {text[:80]!r}") + return int(match.group(1)) + + +def _stable_key(value: Any) -> str: + serialized = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def _json_default(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def save_progress( + output_path: Path, + *, + account: str, + business_date: date, + expected_count: int, + collected: dict[str, dict[str, Any]], + complete: bool, + range_start: date | None = None, + range_end: date | None = None, +) -> None: + note = ( + f"已完成 {len(collected)}/{expected_count} 条差评" + if expected_count + else "差评总数尚未确认" + ) + payload = { + "schema_version": 1, + "workflow_id": WORKFLOW_ID, + "account": account, + "business_date": business_date, + "range_start": range_start.isoformat() if range_start else None, + "range_end": range_end.isoformat() if range_end else None, + "source_url": TARGET_URL, + "expected_count": expected_count, + "collected_count": len(collected), + "progress_note": note, + "complete": complete, + "saved_at": datetime.now().astimezone(), + "reviews": collected, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + temp_path = output_path.with_suffix(output_path.suffix + ".tmp") + temp_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, default=_json_default), + encoding="utf-8", + ) + os.replace(temp_path, output_path) + + +def load_progress(output_path: Path) -> dict[str, Any] | None: + try: + if not output_path.exists(): + return None + payload = json.loads(output_path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else None + except Exception: + return None + + +def save_debug_artifacts(page: Page, output_dir: Path, label: str) -> None: + debug_dir = output_dir / "debug" + debug_dir.mkdir(parents=True, exist_ok=True) + safe_label = re.sub(r"[^0-9A-Za-z_-]+", "_", label) + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + try: + page.screenshot(path=str(debug_dir / f"{stamp}_{safe_label}.png"), full_page=True) + except Exception as exc: + print(f" [WARN] 保存失败截图失败: {exc}") + try: + (debug_dir / f"{stamp}_{safe_label}.html").write_text( + page.content(), encoding="utf-8" + ) + except Exception as exc: + print(f" [WARN] 保存失败 HTML 失败: {exc}") + + +def start_collection_browser( + environment: dict[str, str], + *, + headless: bool, +) -> tuple[ScraplingBrowser, Any | None]: + """启动抖店差评采集浏览器，返回 (browser, 账号浏览器锁)。""" + + account_profile = environment.get("GYXX_ACCOUNT_PROFILE_DIR", "").strip() + if account_profile: + profile_dir = Path(account_profile) + locks = LockManager(SETTINGS.data_root / "state" / "locks") + account_lock = locks.acquire( + f"browser-account:{ACCOUNT_ID}", + owner=f"collect:{COMMAND_ID}", + timeout_seconds=120, + ) + try: + account_lock.__enter__() + except ResourceBusyError as exc: + raise RuntimeError( + "抖音账号浏览器正在被保活/登录占用，请稍后重试" + ) from exc + else: + profile_dir = Path( + environment.get( + "GYXX_BROWSER_PROFILE_DIR", + str(MODULE_PATHS.state_root / "browser" / "dy_negative_reviews_profile"), + ) + ) + account_lock = None + + cdp_port = int(environment.get("GYXX_BROWSER_CDP_PORT", str(CDP_PORT))) + browser_options: dict[str, Any] = { + # 指纹隐藏模式：Scrapling StealthySession。 + "stealthy": True, + "user_data_dir": str(profile_dir), + "headless": headless, + "real_chrome": True, + "persist_state_on_close": False, + "locale": "zh-CN", + "timezone_id": "Asia/Shanghai", + "additional_args": {"viewport": {"width": 1920, "height": 1080}}, + "extra_flags": [ + f"--remote-debugging-port={cdp_port}", + "--remote-debugging-address=127.0.0.1", + ], + "retries": 1, + } + cookie_file = environment.get("GYXX_BROWSER_COOKIE_FILE", "").strip() + storage_state_file = environment.get("GYXX_BROWSER_STORAGE_STATE_FILE", "").strip() + if cookie_file: + browser_options["cookie_file"] = cookie_file + if storage_state_file: + browser_options["storage_state_file"] = storage_state_file + return ScraplingBrowser(**browser_options), account_lock + + +def ensure_login(page: Page, *, headless: bool, login_timeout: int) -> None: + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + douyin_common._wait_for_page(page) + page.wait_for_timeout(2_000) + + if not douyin_common.is_login_page(page): + print(" [OK] 已复用抖音账号 Cookie 登录态，跳过登录") + return + + policy = current_acceptance_policy() + if policy.enabled: + policy.record( + "cookie_skipped", + operation="shop_intelligence.dy_negative_reviews.live_session", + details={"status": "SKIPPED_COOKIE", "reason": "fxg redirected to login"}, + ) + print("[SKIPPED_COOKIE] 抖店会话跳转到了登录页") + raise DouyinSessionExpiredError("抖店会话跳转到了登录页") + + if headless: + raise RuntimeError( + "抖音 Cookie 已失效且当前为无头模式，无法扫码登录；" + "请先执行 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + + print("=" * 60) + print("【请登录】请在弹出的浏览器窗口中完成抖店登录") + print("=" * 60) + deadline = time.monotonic() + login_timeout + while time.monotonic() < deadline: + if not douyin_common.is_login_page(page): + print(f" [OK] 登录成功: {page.url}") + page.wait_for_timeout(3_000) + return + page.wait_for_timeout(1_000) + raise RuntimeError(f"登录在 {login_timeout} 秒内未完成，当前页面: {page.url}") + + +class DouyinSessionExpiredError(RuntimeError): + """The reused cookie state reached an interactive login page.""" + + +def verify_shop_identity(page: Page, *, environment: dict[str, str]) -> None: + markers = douyin_common._configured_identity_markers(environment) + if not markers: + return + page.goto(douyin_common.FXG_IDENTITY_URL, wait_until="domcontentloaded", timeout=60_000) + douyin_common._wait_for_page(page) + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + body_text = douyin_common._page_text(page) + if any(marker in body_text for marker in markers): + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + douyin_common._wait_for_page(page) + return + if douyin_common.is_login_page(page): + break + page.wait_for_timeout(1_000) + raise RuntimeError( + "抖音登录态未出现配置的店铺身份标识；" + f"期望标识={markers!r}，当前页面未匹配。" + "请用 uv run gyxx accounts login douyin-shop 刷新账号登录态" + ) + + +def _history_page_ready(page: Page) -> None: + page.get_by_text("评价明细", exact=True).first.wait_for( + state="visible", timeout=30_000 + ) + page.get_by_role("button", name="查询", exact=True).first.wait_for( + state="visible", timeout=30_000 + ) + + +def click_negative_filter(page: Page) -> None: + """点击评价等级里的「差评」筛选 chip。""" + + douyin_common.dismiss_blocking_modals(page) + chip = page.locator('[class*="filterItem__"]').filter(has_text="差评").first + if not chip.count(): + raise RuntimeError("未找到「差评」筛选入口") + chip.click() + page.wait_for_timeout(2_000) + + +def _click_picker_date(page: Page, target: date) -> None: + """在打开的日历面板中选中目标日期（必要时跨月导航）。""" + + title = target.isoformat() + target_index = target.year * 12 + target.month - 1 + dropdown = page.locator('[class*="datePickerDropdown"]').last + for _ in range(36): + if page.locator(f'td[title="{title}"]:visible').count(): + break + displayed = [ + int(year) * 12 + int(month) - 1 + for year, month in re.findall( + r"(\d{4})年(\d{1,2})月", dropdown.inner_text(timeout=3_000) or "" + ) + ] + if not displayed: + raise RuntimeError("日历面板未打开或未显示月份") + if target_index < min(displayed): + nav = page.locator( + '[class*="datePickerDropdown"] ' + '[class*="header-prev-btn"]:visible' + ).first + elif target_index > max(displayed): + nav = page.locator( + '[class*="datePickerDropdown"] ' + '[class*="header-next-btn"]:visible' + ).first + else: + raise RuntimeError( + f"日历面板包含 {target.year}年{target.month}月 但未找到目标日期格子" + ) + if not nav.count(): + raise RuntimeError("日历面板缺少翻月按钮") + nav.click() + page.wait_for_timeout(400) + cell = page.locator(f'td[title="{title}"]:visible').first + if not cell.count(): + raise RuntimeError(f"日历面板中未找到 {title}（可能超出可查范围）") + cell.click() + page.wait_for_timeout(400) + + +def set_review_date_range(page: Page, start: date, end: date) -> None: + """把评价时间设置为起止日期（含）00:00:00-23:59:59。 + + aurora 范围日历默认只显示当前月前后两个月：目标月份不在视图内时 + 通过面板头部的上一月/下一月按钮导航；起止格子选完后，面板中部的 + 时间输入通过时/分/秒列选择器设为 00:00:00 和 23:59:59，最后点 + 「确定」提交（点击面板外部会取消选择）。 + """ + + page.locator('input[placeholder="开始日期"]').first.click() + page.wait_for_timeout(1_000) + _click_picker_date(page, start) + _click_picker_date(page, end) + + for placeholder, values in ( + ("开始时间", ("0", "0", "0")), + ("结束时间", ("23", "59", "59")), + ): + time_input = page.locator( + f'input[placeholder="{placeholder}"]:visible' + ).first + time_input.click() + page.wait_for_timeout(800) + for column_type, value in zip(("hour", "minute", "second"), values): + item = page.locator( + f'ul[data-type="{column_type}"] ' + f'li[data-value="{value}"]:visible' + ).first + if not item.count(): + raise RuntimeError( + f"时间选择器缺少 {column_type}={value} 选项" + ) + item.click() + page.wait_for_timeout(250) + + confirm = page.locator( + '[class*="datePickerDropdown"] button:has-text("确定")' + ).last + confirm.click() + page.wait_for_timeout(800) + + values = page.evaluate( + """() => Array.from(document.querySelectorAll('input')) + .filter((el) => ['开始日期','结束日期'].includes( + el.getAttribute('placeholder') || '')) + .map((el) => el.value)""" + ) + start_text = start.strftime("%Y/%m/%d") + end_text = end.strftime("%Y/%m/%d") + if not any(start_text in value for value in values) or not any( + end_text in value for value in values + ): + raise RuntimeError( + f"评价时间设置未生效，期望包含 {start_text}~{end_text}，实际 {values!r}" + ) + + +def _find_total(value: Any) -> int | None: + """在评价接口响应里递归查找总数（total/total_count/count 字段）。""" + + if isinstance(value, dict): + for key in ("total", "total_count", "totalCount", "count"): + found = value.get(key) + if isinstance(found, int): + return found + for child in value.values(): + found = _find_total(child) + if found is not None: + return found + elif isinstance(value, list): + for item in value: + found = _find_total(item) + if found is not None: + return found + return None + + +def apply_query(page: Page) -> int: + """点击「查询」，等待评价接口响应并返回过滤后的差评总数。 + + 点击查询后页面会先保留上一次渲染的旧总数，直接读 DOM 会拿到 + 竞态前的数字；这里以 commentList 接口响应中的 total 为准。 + """ + + douyin_common.dismiss_blocking_modals(page) + with page.expect_response( + lambda response: "commentList" in response.url, timeout=30_000 + ) as response_info: + page.get_by_role("button", name="查询", exact=True).first.click() + total = None + try: + total = _find_total(response_info.value.json()) + except Exception: + total = None + if total is not None: + return total + + # 响应解析失败时退回 DOM，等待渲染稳定后读取。 + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + body_text = douyin_common._page_text(page, timeout=5_000) + match = TOTAL_COUNT_PATTERN.search(body_text or "") + if match: + return int(match.group(1)) + if "暂无数据" in (body_text or ""): + return 0 + page.wait_for_timeout(500) + raise RuntimeError("查询后未识别差评总数") + + +def set_page_size(page: Page, size: int = 100) -> int: + try: + selector = page.locator(':text-matches("\\d+\\s*条\\s*/\\s*页")').last + if not selector.count() or not selector.is_visible(): + return 20 + selector.click() + page.wait_for_timeout(600) + option = page.locator( + f'[class*="dropdown"]:visible :text-matches("{size}\\s*条\\s*/\\s*页"), ' + f'[class*="option"]:visible:has-text("{size} 条/页"), ' + f'li:visible:has-text("{size} 条/页")' + ).last + if not option.count() or not option.is_visible(): + page.keyboard.press("Escape") + return 20 + option.click() + page.wait_for_timeout(1_500) + summary = page.locator(':text-matches("\\d+\\s*条\\s*/\\s*页")').last + text = summary.inner_text(timeout=5_000) if summary.count() else "" + if f"{size}" in (text or ""): + print(f" [OK] 分页大小已设置为 {size} 条/页") + return size + return 20 + except Exception as exc: + print(f" [WARN] 设置分页大小失败，按 20 条/页继续: {exc}") + return 20 + + +def _expand_more_links(page: Page) -> None: + """展开评价内容里截断的「更多」，尽量拿到完整文本。""" + + for _ in range(200): + more = page.locator( + '[class*="evaluation"], [class*="comment"]' + ).get_by_text("更多", exact=True) + count = min(more.count(), 100) + expanded = False + for index in range(count): + try: + item = more.nth(index) + if item.is_visible(): + item.click() + expanded = True + page.wait_for_timeout(120) + except Exception: + continue + if not expanded: + break + + +def read_review_rows(page: Page) -> list[dict[str, Any]]: + """读取当前页全部评价块（最小且恰含一个订单 ID 的容器）。""" + + return page.evaluate( + """() => { + const candidates = Array.from(document.querySelectorAll('div')) + .filter((el) => { + const t = el.innerText || ''; + return t.includes('订单ID:') && t.includes('商品ID:') + && t.length < 1200 + && (t.match(/订单ID:/g) || []).length === 1; + }); + candidates.sort((a, b) => + (a.innerText || '').length - (b.innerText || '').length); + const rows = []; + const seen = new Set(); + for (const el of candidates) { + let node = el; + while (node.parentElement) { + const parent = node.parentElement; + const t = parent.innerText || ''; + if ((t.match(/订单ID:/g) || []).length > 1) break; + if (t.length > 1400) break; + node = parent; + } + const orderId = ((node.innerText || '').match(/订单ID:(\\d+)/) || [])[1]; + if (!orderId || seen.has(orderId)) continue; + seen.add(orderId); + const imgs = Array.from(node.querySelectorAll('img')) + .map((img) => img.getAttribute('src') || '') + .filter((src) => src.startsWith('http') + && src.includes('comment')); + const byClass = (needle) => { + const target = Array.from(node.querySelectorAll('[class*="__"]')) + .find((el) => (el.className || '').includes(needle)); + return target ? (target.innerText || '').trim() : ''; + }; + const productIdEl = Array.from( + node.querySelectorAll('[class*="productId__"]') + )[0]; + const specEl = productIdEl + ? productIdEl.nextElementSibling : null; + rows.push({ + order_id: orderId, + text: (node.innerText || '').trim(), + content: byClass('style_content__'), + rankTag: byClass('style_rankTags__'), + productName: byClass('style_productName__'), + productId: productIdEl + ? (productIdEl.innerText || '').trim() : '', + specName: specEl ? (specEl.innerText || '').trim() : '', + images: imgs, + status: (() => { + const t = node.innerText || ''; + if (t.includes('已处置')) return '已处置'; + if (t.includes('展示中')) return '展示中'; + return null; + })(), + experienceExcluded: t_expExcluded(node), + }); + } + function t_expExcluded(node) { + return (node.innerText || '').includes('体验分不计入'); + } + return rows; + }""" + ) + + +_CONTENT_END_MARKERS = frozenset( + {"更多", "收起", "展示中", "已处置", "查看原因", "已隐藏"} +) + + +def normalize_review( + row: dict[str, Any], + *, + account: str, + business_date: date, + source_url: str, +) -> dict[str, Any] | None: + text = str(row.get("text") or "") + order_id = str(row.get("order_id") or "").strip() + if not order_id: + match = ORDER_ID_PATTERN.search(text) + if not match: + return None + order_id = match.group(1) + lines = [line.strip() for line in text.splitlines() if line.strip()] + + order_id_index = 0 + review_at = None + review_at_index = -1 + product_id = None + product_id_index = -1 + for index, line in enumerate(lines): + if review_at_index < 0: + match = REVIEW_AT_PATTERN.search(line) + if match: + review_at = datetime.strptime( + match.group(1), "%Y/%m/%d %H:%M:%S" + ).replace(tzinfo=SHANGHAI) + review_at_index = index + match = PRODUCT_ID_PATTERN.search(line) + if match and product_id_index < 0: + product_id = match.group(1) + product_id_index = index + if ORDER_ID_PATTERN.search(lines[0] if lines else ""): + order_id_index = 0 + + # 评价正文：评价时间之后、商品信息之前；「体验分不计入」等徽标 + # 可能出现在正文前或后，跳过前导徽标，遇到正文后首个标记即停。 + content_lines: list[str] = [] + end = product_id_index if product_id_index >= 0 else len(lines) + for line in lines[review_at_index + 1 : end]: + if line in _CONTENT_END_MARKERS or line.startswith("体验分不计入"): + if content_lines: + break + continue + if line in ORDER_STATUS_LIKE_WORDS or line in {"差评", "疑似违规差评"}: + continue + content_lines.append(line) + content = "\n".join(content_lines).strip() + + rank_tag = str(row.get("rankTag") or "").strip() + product_name = str(row.get("productName") or "").strip() + spec_name = " ".join( + part + for part in str(row.get("specName") or "").splitlines() + if part.strip() and not part.strip().startswith("规格ID:") + ).strip() + # 不同卡片变体的名称/规格 DOM 顺序不同：以文本为准——商品名固定为 + # “GYXX/”开头的标题行，规格取商品 ID 行的下一行（剔除规格 ID）。 + title_from_text = next( + (line for line in lines if line.startswith("GYXX/")), None + ) + if title_from_text: + product_name = title_from_text + spec_from_text = None + if 0 <= product_id_index < len(lines) - 1: + candidate = SPEC_ID_PATTERN.sub( + "", lines[product_id_index + 1] + ).strip() + if candidate and not candidate.startswith("GYXX/"): + spec_from_text = candidate + if spec_from_text: + spec_name = spec_from_text + reviewer_nickname = None + if len(lines) > order_id_index + 1: + candidate = lines[order_id_index + 1] + if candidate not in ORDER_STATUS_LIKE_WORDS and candidate != "差评": + reviewer_nickname = None if candidate == "消费者匿名评价" else candidate + spec_id = None + match = SPEC_ID_PATTERN.search(text) + if match: + spec_id = match.group(1) + return { + "account": account, + "business_date": business_date, + "review_id": order_id, + "order_id": order_id, + "review_at": review_at, + "rating_level": "差评" if "差评" in (rank_tag or text) else "评价", + "stars": 1 if "差评" in (rank_tag or text) else None, + "reviewer_nickname": reviewer_nickname, + "content": content or None, + "image_urls": row.get("images") or [], + "display_status": row.get("status"), + "experience_score_excluded": bool(row.get("experienceExcluded")), + "product_id": product_id, + "product_name": product_name or None, + "spec_name": spec_name or None, + "spec_id": spec_id, + "source_url": source_url, + "raw_review": {"text": text[:4000], "images": row.get("images") or []}, + "captured_at": datetime.now(SHANGHAI), + } + + +def goto_next_page(page: Page, current_first_order: str) -> bool: + """翻到下一页并等待列表首行变化；已是最后一页时返回 False。""" + + douyin_common.dismiss_blocking_modals(page) + pagination = page.locator('[class*="pagination"]').last + try: + pagination.scroll_into_view_if_needed(timeout=5_000) + except Exception: + pass + next_disabled = page.evaluate( + """() => { + const btn = document.querySelector( + '.auxo-pagination-next, [class*="pagination"] li[title="下一页"], ' + + '[class*="pagination"] [class*="next"]' + ); + if (!btn) return false; + const cls = btn.className || ''; + return cls.includes('disabled') + || btn.getAttribute('aria-disabled') === 'true'; + }""" + ) + if next_disabled: + print(" [INFO] 已到最后一页（下一页禁用）") + return False + for attempt in range(2): + if attempt == 0: + button = page.locator( + '.auxo-pagination-next:visible, ' + '[class*="pagination"] [class*="next"]:visible, ' + '[class*="pagination"] li[title="下一页"]:visible' + ).first + else: + before = page.evaluate( + """() => { + const active = document.querySelector( + '[class*="pagination"] [class*="item-active"], ' + + 'li[class*="active"]' + ); + return active ? Number((active.innerText || '').trim()) || 1 : 1; + }""" + ) + button = page.locator( + f'[class*="pagination"] li[title="{before + 1}"]:visible, ' + f'[class*="pagination"] [title="{before + 1}"]:visible' + ).first + if not button.count(): + if attempt == 0: + continue + raise RuntimeError("分页“下一页”按钮不可见，无法继续采集") + try: + button.click(timeout=5_000) + except Exception: + button.click(force=True) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + rows = read_review_rows(page) + first = rows[0]["order_id"] if rows else "" + if first and first != current_first_order: + return True + page.wait_for_timeout(500) + if attempt == 0: + print(" [WARN] 翻页后列表未更新，改用直接点击页码重试") + page.wait_for_timeout(1_000) + raise RuntimeError("翻页后列表内容未更新") + + +def _persist_records( + records: list[dict[str, Any]], + *, + no_db: bool, +) -> int: + if no_db or not records: + return 0 + from gyxx_flow.modules.shop_intelligence.db import ( + get_conn, + upsert_dy_negative_reviews, + ) + + with get_conn() as conn: + return upsert_dy_negative_reviews(conn, records) + + +def preflight_acceptance_login(environment: dict[str, str]) -> int | None: + policy = current_acceptance_policy() + if not policy.enabled: + return None + from gyxx_flow.adapters import binding_from_environment + + result = policy.preflight_cookie( + binding_from_environment(environment), environment=environment + ) + if not result.should_skip: + return None + print(f"[SKIPPED_COOKIE] {result.reason}") + return COOKIE_SKIP_EXIT_CODE + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="抖店评价管理差评按日采集") + parser.add_argument( + "--date", + type=date.fromisoformat, + default=None, + help="单日业务日期 YYYY-MM-DD；控制台会自动传入", + ) + parser.add_argument( + "--from-date", + type=date.fromisoformat, + default=None, + help="区间开始日期（含）；与 --to-date 一起使用，一次查询整个区间", + ) + parser.add_argument( + "--to-date", + type=date.fromisoformat, + default=None, + help="区间结束日期（含）", + ) + parser.add_argument( + "--range-days", + type=int, + default=0, + help="采集 business_date 往前 N 天（含当天）的区间；定时任务使用，" + "0 表示仅按 --date/--from-date/--to-date 采集", + ) + parser.add_argument( + "--output", type=Path, default=DEFAULT_OUTPUT, help="原始数据和失败现场目录" + ) + parser.add_argument("--headless", action="store_true", help="无界面运行浏览器") + parser.add_argument("--no-db", action="store_true", help="只保存原始 JSON，不写数据库") + parser.add_argument( + "--max-pages", type=int, default=0, help="限制最多采集的页数；0 表示不限制" + ) + parser.add_argument( + "--login-timeout", type=int, default=300, help="人工登录最长等待秒数" + ) + return parser.parse_args(argv) + + +def _resolve_business_date(args: argparse.Namespace) -> date: + if args.date is not None: + return args.date + value = os.getenv("GYXX_BUSINESS_DATE", "").strip() + if value: + return date.fromisoformat(value) + return date.today() + + +def _resolve_query_range( + args: argparse.Namespace, environment: dict[str, str] +) -> tuple[date, date]: + """解析查询区间：控制台环境变量优先，其次 --range-days，再次 + --from-date/--to-date，最后退回单日（--date）。""" + + env_from = environment.get("GYXX_NEG_REVIEW_FROM", "").strip() + env_to = environment.get("GYXX_NEG_REVIEW_TO", "").strip() + if env_from and env_to: + return ( + date.fromisoformat(env_from), + date.fromisoformat(env_to), + ) + if args.range_days > 0: + business_date = _resolve_business_date(args) + return ( + business_date - timedelta(days=args.range_days - 1), + business_date, + ) + if args.from_date and args.to_date: + if args.to_date < args.from_date: + raise ValueError("--to-date 不能早于 --from-date") + return args.from_date, args.to_date + day = _resolve_business_date(args) + return day, day + + +def run( + args: argparse.Namespace, + *, + runtime_environment: dict[str, str] | None = None, +) -> int: + environment = dict(runtime_environment or os.environ) + account = environment.get("GYXX_ACCOUNT_ID", ACCOUNT_ID).strip() or ACCOUNT_ID + range_start, range_end = _resolve_query_range(args, environment) + single_day = range_start == range_end + if single_day: + output_path = args.output / f"抖音差评_{range_start.isoformat()}.json" + else: + output_path = args.output / ( + f"抖音差评_{range_start.isoformat()}" + f"_to_{range_end.isoformat()}.json" + ) + + if not args.no_db: + from gyxx_flow.modules.shop_intelligence.db import init_schema + + init_schema() + print("数据库表 dy_negative_reviews 已就绪") + + print("=" * 68) + print("抖店评价管理差评采集") + print(f" 账号: {account}") + print( + f" 评价时间: {range_start} ~ {range_end}" + + ("（单日）" if single_day else "（区间一次查询）") + ) + print(f" 输出: {output_path}") + print(f" 数据库: {'跳过' if args.no_db else '写入'}") + print("=" * 68) + + collected: dict[str, dict[str, Any]] = {} + previous = load_progress(output_path) + if ( + previous + and previous.get("account") == account + and previous.get("range_start") == range_start.isoformat() + and previous.get("range_end") == range_end.isoformat() + ): + for key, item in (previous.get("reviews") or {}).items(): + collected[str(key)] = item + if collected: + print(f"断点续采：已有进度 {len(collected)} 条，本次只采集剩余部分") + + browser_session, account_lock = start_collection_browser( + environment, headless=args.headless + ) + try: + with browser_session: + context = browser_session.context + if context is None: + raise RuntimeError("Scrapling 未创建抖店差评浏览器上下文") + page = context.pages[0] if context.pages else context.new_page() + try: + ensure_login( + page, headless=args.headless, login_timeout=args.login_timeout + ) + verify_shop_identity(page, environment=environment) + _history_page_ready(page) + douyin_common.dismiss_blocking_modals(page) + browser_session.persist_bound_state() + + try: + click_negative_filter(page) + set_review_date_range(page, range_start, range_end) + total = apply_query(page) + except Exception: + save_debug_artifacts(page, args.output, "filter_setup") + raise + + if total == 0: + print(f" [INFO] {range_start}~{range_end} 无差评，按 0 条完成") + save_progress( + output_path, + account=account, + business_date=range_start, + range_start=range_start, + range_end=range_end, + expected_count=0, + collected={}, + complete=True, + ) + return 0 + + print(f"筛选完成：{range_start}~{range_end} 共 {total} 条差评，开始逐页采集") + page_size = set_page_size(page, 100) + print(f" [INFO] 当前分页大小 {page_size} 条/页") + save_progress( + output_path, + account=account, + business_date=range_start, + range_start=range_start, + range_end=range_end, + expected_count=total, + collected=collected, + complete=False, + ) + + processed = 0 + pages_done = 0 + for _ in range(MAX_PAGES): + _expand_more_links(page) + rows = read_review_rows(page) + if not rows: + if len(collected) >= total: + break + raise RuntimeError( + f"当前页未读取到评价行（已采集 {len(collected)}/{total}）" + ) + for row in rows: + order_id = row["order_id"] + if order_id in collected: + continue + normalized = normalize_review( + row, + account=account, + business_date=range_start, + source_url=TARGET_URL, + ) + if normalized is None: + continue + rating_level = normalized.get("rating_level") + if rating_level not in (None, "差评"): + continue + if not single_day: + # 区间模式：每条差评按其评价时间的上海日期归属。 + review_at = normalized.get("review_at") + if review_at is not None: + normalized["business_date"] = ( + review_at.astimezone(SHANGHAI).date() + ) + db_rows = _persist_records([normalized], no_db=args.no_db) + collected[order_id] = { + "content": normalized.get("content"), + "image_count": len(normalized.get("image_urls") or []), + "product_id": normalized.get("product_id"), + "business_date": normalized["business_date"].isoformat(), + "db_rows": db_rows, + "captured_at": datetime.now().astimezone().isoformat(), + } + save_progress( + output_path, + account=account, + business_date=range_start, + range_start=range_start, + range_end=range_end, + expected_count=total, + collected=collected, + complete=False, + ) + processed += 1 + print( + f" [{processed:03d}/{total}] 差评 {order_id} 完成" + f"（图 {len(normalized.get('image_urls') or [])}，" + f"商品 {normalized.get('product_id')}）" + ) + pages_done += 1 + if args.max_pages > 0 and pages_done >= args.max_pages: + print( + f" [INFO] 已达到 --max-pages={args.max_pages}，提前结束" + ) + break + if len(collected) >= total: + break + if not goto_next_page(page, rows[0]["order_id"]): + break + + drift = total - len(collected) + if drift > max(2, int(total * 0.02)): + save_progress( + output_path, + account=account, + business_date=range_start, + range_start=range_start, + range_end=range_end, + expected_count=total, + collected=collected, + complete=False, + ) + raise RuntimeError( + f"差评覆盖不完整：页面声明 {total} 条，实际完成 {len(collected)} 条" + ) + if drift > 0: + print(f" [WARN] 活列表漂移：声明 {total} 条，实际采集 {len(collected)} 条") + save_progress( + output_path, + account=account, + business_date=range_start, + range_start=range_start, + range_end=range_end, + expected_count=total, + collected=collected, + complete=True, + ) + finally: + pass + finally: + if account_lock is not None: + account_lock.__exit__(None, None, None) + + print( + f"采集完成：{range_start}~{range_end} 共 {len(collected)} 条差评，" + f"结果已保存到 {output_path}" + ) + return 0 + + +def _account_runtime_environment() -> dict[str, str]: + """注入脚本绑定环境；账号 vault Cookie 只读合并进脚本专属状态。""" + + base_environment = dict(os.environ) + project_root = base_environment.get("GYXX_PROJECT_ROOT", "").strip() + data_root = base_environment.get("GYXX_DATA_ROOT", "").strip() + if not project_root or not data_root: + return base_environment + catalog = RuntimeIntegrationCatalog.load_default( + project_root=project_root, + data_root=data_root, + ) + try: + return catalog.environment_for(COMMAND_ID, base_environment) + except Exception as exc: + print(f" [INFO] 未解析到脚本绑定 {COMMAND_ID}，使用当前环境继续: {exc}") + return catalog.environment_for_account(ACCOUNT_ID, base_environment) + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + runtime_environment = _account_runtime_environment() + preflight_exit_code = preflight_acceptance_login(runtime_environment) + if preflight_exit_code is not None: + return preflight_exit_code + return run(args, runtime_environment=runtime_environment) + except DouyinSessionExpiredError: + return COOKIE_SKIP_EXIT_CODE + except KeyboardInterrupt: + print("\n用户中断。") + return 130 + except Exception as exc: + print(f"\n[ERROR] {exc}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py index 929c4a9..47b4c40 100644 --- a/src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py @@ -2223,17 +2223,20 @@ def _extract_experience_score(page) -> dict: } const result = {}; - const main = findByLabel('我的体验分') || findByLabel('体检分') || findByLabel('商家体验分'); + const main = findByLabel('我的体验分') + || findByLabel('体检分') + || findByLabel('商家体验分') + || findByLabel('我的总分'); if (main) { result.score = main.score; result.raw_text = main.raw_text; } const detailLabels = [ - [['商品体验得分', '商品体验分'], 'product_experience_score'], - [['物流体验得分', '物流体验分'], 'logistics_experience_score'], - [['服务体验得分', '服务体验分'], 'service_experience_score'], - [['差行为扣分'], 'penalty_score'] + [['商品体验得分', '商品体验分', '商品体验'], 'product_experience_score'], + [['物流体验得分', '物流体验分', '物流体验'], 'logistics_experience_score'], + [['服务体验得分', '服务体验分', '服务体验'], 'service_experience_score'], + [['差行为扣分', '扣分项'], 'penalty_score'] ]; for (const [labels, key] of detailLabels) { let parsed = null; diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/jd_after_sales_issues.py b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_after_sales_issues.py new file mode 100644 index 0000000..d42c5b7 --- /dev/null +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_after_sales_issues.py @@ -0,0 +1,1696 @@ +"""采集京东品退异常服务单，并按账号与业务日期幂等写入 PostgreSQL。 + +京东商智的售后分析页没有稳定的公开导出接口。采集器因此遵循真实页面 +流程：设置自定义日期范围、筛选“商品出现异常”、打开“查看全部服务单明细”， +点击下载数据并从弹窗表格读取全部分页。用户原声为“未知”或为空的行不 +进入业务表，但会保留在范围原始产物中供审计。 + +商智自定义时间控件最多允许 31 天，所以较长的控制台范围会拆成少量的 +连续区间（而不是逐日打开页面）。每条服务单按其申请时间落到业务日期， +数据库幂等键仍是 (account, business_date, service_id)，同一账号同一天 +重跑只更新该服务单，不会因为重复下载而新增重复行。 +""" + +from __future__ import annotations + +import argparse +import html +import json +import os +import re +import sys +import time +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse +from zoneinfo import ZoneInfo + +from gyxx_flow.adapters import ( + COOKIE_SKIP_EXIT_CODE, + binding_from_environment, + current_acceptance_policy, +) +from gyxx_flow.adapters.scrapling import BrowserPage as Page +from gyxx_flow.adapters.scrapling import ScraplingBrowser, is_browser_timeout_error +from gyxx_flow.core.config import Settings +from gyxx_flow.core.layout import DataLayout +from gyxx_flow.modules.shop_intelligence.browser_session import reuse_or_run_jd_login +from gyxx_flow.modules.shop_intelligence.collectors import ( + jd_customer_service_messages as customer_service, +) + +SETTINGS = Settings.from_env() +MODULE_PATHS = DataLayout(SETTINGS.data_root).for_module("shop_intelligence") + +TARGET_URL = ( + "https://jdsz.jd.com/szweb/view/service/" + "after-sales-service-analysis-temp.html?sz=%2Fszweb%2Fsz%2Fview%2FserviceAnalysis" + "%2FafterSalesServiceAnalysis.html" +) +JD_SHOP_HOME_URL = "https://shop.jd.com/jdm/home" +DEFAULT_ACCOUNT = os.getenv("JD_SHOP", "光影行星研发") +DEFAULT_PASSWORD = os.getenv("JD_PASSWORD", "") +DEFAULT_OUTPUT = MODULE_PATHS.raw_root / "jd_after_sales_issues" +CDP_PORT = int( + os.getenv("GYXX_BROWSER_CDP_PORT") + or os.getenv("GYXX_CDP_PORT") + or "22157" +) +SHANGHAI = ZoneInfo("Asia/Shanghai") +JD_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" +EXPECTED_SHOP_MARKER = "光影行星 GYXX官方旗舰店" +STORE_KEY = "flagship" +STORE_CONFIG = { + "label": "光影行星旗舰店", + "account_id": "jd-shop", + "identity_markers": (EXPECTED_SHOP_MARKER,), +} +REASON_FILTER = "商品出现异常" +MAX_CUSTOM_RANGE_DAYS = 31 +# 商智明细弹窗会按数据量产生较多分页；实际总页数从分页控件读取， +# 此值只作为页面未展示总页数时的防失控上限。 +MAX_DETAIL_PAGES = 1000 +DETAIL_PAGE_CHANGE_TIMEOUT_MS = 30_000 +UNKNOWN_USER_ORIGINAL = frozenset( + { + "", + "-", + "--", + "—", + "未知", + "null", + "none", + "暂无", + "未填写", + } +) +AFTER_SALES_COLUMNS = ( + "服务单ID", + "订单ID", + "商品", + "一级品类", + "二级品类", + "三级品类", + "申请时间", + "售后原因分类", + "售后一级原因", + "售后二级原因", + "用户期望处理方式", + "用户原声", + "操作", +) +_FIELD_ALIASES: dict[str, tuple[str, ...]] = { + "service_id": ("服务单ID", "服务单号", "售后服务单ID", "售后服务单号"), + "order_id": ("订单ID", "订单号"), + "product": ("商品", "商品名称", "商品名"), + "category_level_1": ("一级品类", "一级类目"), + "category_level_2": ("二级品类", "二级类目"), + "category_level_3": ("三级品类", "三级类目"), + "apply_at": ("申请时间", "服务单申请时间"), + "reason_category": ("售后原因分类", "售后原因"), + "reason_level_1": ("售后一级原因", "售后一级原因分类"), + "reason_level_2": ("售后二级原因", "售后二级原因分类"), + "expected_resolution": ("用户期望处理方式", "用户期望"), + "user_original": ("用户原声", "用户原话"), +} +_VISIBLE_HOSTS = frozenset({"shop.jd.com", "jdsz.jd.com", "wares-jdm.jd.com"}) +_ID_PATTERN = re.compile(r"(?:ID|id)\s*[:：]\s*([0-9A-Za-z_-]+)") +_MONTH_PATTERN = re.compile(r"(20\d{2})年\s*(\d{1,2})月") + + +def _normalize_text(value: Any) -> str: + text = html.unescape(str(value or "")) + text = text.replace("\u00a0", " ") + return re.sub(r"\s+", " ", text).strip() + + +def _json_string_tuple(raw: str) -> tuple[str, ...]: + try: + value = json.loads(raw) + except json.JSONDecodeError: + return () + if not isinstance(value, list): + return () + return tuple(str(item).strip() for item in value if str(item).strip()) + + +def _selected_runtime_environment() -> dict[str, str]: + """复用研发旗舰店账号的 Vault 和独立采集 CDP 绑定。""" + + environment = customer_service._selected_runtime_environment(STORE_KEY) + environment["GYXX_JD_AFTER_SALES_STORE"] = STORE_KEY + environment["GYXX_JD_AFTER_SALES_STORE_LABEL"] = STORE_CONFIG["label"] + environment["GYXX_JD_CUSTOMER_SERVICE_STORE"] = STORE_KEY + return environment + + +def browser_runtime_options( + environment: dict[str, str] | None = None, +) -> dict[str, bool]: + values = os.environ if environment is None else environment + return {"real_chrome": True} if values.get("GYXX_BROWSER_CHANNEL") == "chrome" else {} + + +def _wait_for_page(page: Page, timeout: int = 60_000) -> None: + try: + page.wait_for_load_state("networkidle", timeout=timeout) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + page.wait_for_load_state("domcontentloaded", timeout=timeout) + + +def _is_authenticated_jd_url(url: str) -> bool: + parsed = urlparse(url) + return parsed.hostname in _VISIBLE_HOSTS and "login" not in url.lower() + + +def _wait_for_authenticated_session(page: Page, timeout_seconds: int) -> None: + if _is_authenticated_jd_url(page.url): + return + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if _is_authenticated_jd_url(page.url): + _wait_for_page(page) + return + if urlparse(page.url).hostname == "aq.jd.com": + print(" [需要操作] 京东安全认证页面已打开，请在浏览器中完成验证") + page.wait_for_timeout(1_000) + raise RuntimeError( + f"登录/安全认证在 {timeout_seconds} 秒内未完成，当前页面: {page.url}" + ) + + +def login(page: Page, account: str, password: str, login_timeout: int = 300) -> None: + """沿用京东商家登录入口；密码只从运行时参数或环境读取。""" + + page.goto(JD_SHOP_HOME_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + page.wait_for_timeout(2_000) + account_input = page.locator('input[placeholder="请输入账号名/邮箱"]').first + if account_input.count() and account_input.is_visible(): + if not account or not password: + raise ValueError("未提供京东账号或密码，请设置 JD_SHOP/JD_PASSWORD") + account_input.fill(account) + page.locator('input[placeholder="请输入登录密码"]').first.fill(password) + page.locator('button:has-text("立即登录")').first.click() + page.wait_for_timeout(2_000) + _wait_for_authenticated_session(page, login_timeout) + + +def _hide_feedback_widgets(page: Page) -> None: + try: + page.evaluate( + """() => { + const hide = (root) => { + for (const node of root.querySelectorAll( + 'shop-common-components[type="NPS"]' + )) { + node.style.setProperty('display', 'none', 'important'); + node.style.setProperty('pointer-events', 'none', 'important'); + } + for (const node of root.querySelectorAll('*')) { + if (node.shadowRoot) hide(node.shadowRoot); + } + for (const node of root.querySelectorAll('.win-notice-modal')) { + // 商智公告弹窗有时带着 hidden class，但它的 mask 仍会 + // 拦截点击；仅关闭这类公告，不影响业务弹窗。 + node.style.setProperty('display', 'none', 'important'); + node.style.setProperty('pointer-events', 'none', 'important'); + } + }; + hide(document); + }""" + ) + except Exception: + return + + +def _identity_markers(environment: dict[str, str]) -> tuple[str, ...]: + raw = environment.get("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS", "").strip() + if not raw: + raw = environment.get("GYXX_JD_CUSTOMER_SERVICE_IDENTITY_MARKERS", "").strip() + markers = _json_string_tuple(raw) if raw else () + return markers or STORE_CONFIG["identity_markers"] + + +def open_and_verify_after_sales( + page: Page, + *, + environment: dict[str, str], +) -> None: + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + page.wait_for_timeout(2_000) + if "login" in page.url.lower(): + raise RuntimeError(f"登录未成功，品退页面跳转到了登录页: {page.url}") + page.get_by_text("售后分析", exact=True).first.wait_for( + state="visible", timeout=30_000 + ) + _hide_feedback_widgets(page) + markers = _identity_markers(environment) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + body_text = page.locator("body").inner_text(timeout=15_000) + if any(marker in body_text for marker in markers): + return + page.wait_for_timeout(1_000) + raise RuntimeError( + "京东品退页未出现配置的店铺身份标识；" + f"期望标识={markers!r}，当前页面未匹配" + ) + + +def _visible_date_picker(page: Page) -> Any: + picker = page.locator( + ".jmtd-popover.jmtd-date-picker.jmtd-popover-open:visible" + ).last + picker.wait_for(state="visible", timeout=10_000) + return picker + + +def _month_value(text: str) -> int | None: + match = _MONTH_PATTERN.search(text) + if not match: + return None + return int(match.group(1)) * 12 + int(match.group(2)) - 1 + + +def _date_cell_for(page: Page, target_date: date) -> Any: + target_month = target_date.year * 12 + target_date.month - 1 + panel = _visible_date_picker(page).locator(".jmtd-date-picker-day-panel") + panel.wait_for(state="visible", timeout=10_000) + for _ in range(36): + header = panel.locator(".jmtd-date-picker-header-content").first + displayed_month = _month_value(header.inner_text()) + if displayed_month is None: + raise RuntimeError( + f"无法识别京东品退日期面板年月: {header.inner_text()!r}" + ) + if displayed_month == target_month: + cells = panel.locator('[role="gridcell"]') + for index in range(cells.count()): + cell = cells.nth(index) + classes = cell.get_attribute("class") or "" + if "diff-month" in classes: + continue + if _normalize_text(cell.inner_text()) != str(target_date.day): + continue + if "cell-disabled" in classes: + raise RuntimeError( + f"京东品退后台尚未开放业务日期 {target_date.isoformat()}" + ) + return cell + raise RuntimeError( + f"日期面板中没有可选日期: {target_date.isoformat()}" + ) + event = "prevMonth" if target_month < displayed_month else "nextMonth" + button = panel.locator( + f'[data-event-content="{event}"]:visible' + ).first + if not button.count(): + raise RuntimeError( + f"京东品退日期面板无法导航到 {target_date.isoformat()}" + ) + button.click(force=True) + page.wait_for_timeout(150) + raise RuntimeError(f"日期面板无法导航到 {target_date.isoformat()}") + + +def _date_picker_input(page: Page) -> Any: + date_inputs = page.locator(".jmtd-base-input.jmtd-date-picker:visible") + date_input = date_inputs.filter(has_text=re.compile(r"按天查询|自定义时间|至")).first + if not date_input.count(): + date_input = date_inputs.first + date_input.wait_for(state="visible", timeout=20_000) + return date_input + + +def _open_date_picker(page: Page) -> Any: + date_input = _date_picker_input(page) + trigger = date_input.locator(".jmtd-base-input-top").first + if trigger.count(): + trigger.click() + else: + date_input.click() + return _visible_date_picker(page) + + +def _date_cell_for_range(page: Page, target_date: date) -> None: + """在自定义范围双月面板中找到并点击目标自然日。 + + 商智切换区间时会短暂保留多个日期弹层副本。这里直接从当前可见弹层 + 读取并点击 DOM，避免 Locator 在弹层重绘期间绑定到不可见副本。 + """ + + target_month = target_date.year * 12 + target_date.month - 1 + last_months: list[str] = [] + for _ in range(60): + try: + payload = page.evaluate( + r"""({targetMonth, targetDay}) => { + const visible = node => { + if (!node) return false; + const rect = node.getBoundingClientRect(); + const style = window.getComputedStyle(node); + return rect.width > 0 && rect.height > 0 + && style.display !== 'none' + && style.visibility !== 'hidden'; + }; + const popovers = Array.from(document.querySelectorAll( + '.jmtd-popover.jmtd-date-picker.jmtd-popover-open' + )); + const picker = popovers.find(visible); + if (!picker) return {status: 'picker_missing', months: []}; + const panels = Array.from( + picker.querySelectorAll('.jmtd-date-picker-panel') + ).filter(visible); + const months = panels.map(panel => ( + panel.querySelector('.jmtd-date-picker-header-content') + ?.innerText || '' + )); + for (const panel of panels) { + const header = panel.querySelector( + '.jmtd-date-picker-header-content' + ); + const match = (header?.innerText || '').match( + /(20\d{2})年\s*(\d{1,2})月/ + ); + if (!match) continue; + const month = Number(match[1]) * 12 + Number(match[2]) - 1; + if (month !== targetMonth) continue; + const cells = Array.from( + panel.querySelectorAll('[role="gridcell"]') + ); + const cell = cells.find(candidate => { + const classes = candidate.getAttribute('class') || ''; + return !classes.includes('diff-month') + && (candidate.innerText || '').trim() === String(targetDay); + }); + if (!cell) return {status: 'day_missing', months}; + const classes = cell.getAttribute('class') || ''; + if (classes.includes('disabled')) { + return {status: 'disabled', months}; + } + cell.click(); + return {status: 'clicked', months}; + } + return {status: 'ready', months}; + }""", + {"targetMonth": target_month, "targetDay": target_date.day}, + ) + except Exception: + payload = None + + if isinstance(payload, dict): + status = str(payload.get("status") or "") + last_months = [ + _normalize_text(value) + for value in payload.get("months") or [] + if _normalize_text(value) + ] + if status == "clicked": + return + if status == "disabled": + raise RuntimeError( + f"京东品退后台尚未开放业务日期 {target_date.isoformat()}" + ) + + displayed = [ + month + for text in last_months + if (month := _month_value(text)) is not None + ] + if displayed: + minimum = min(displayed) + maximum = max(displayed) + if target_month < minimum: + event = "left-prevMonth" + elif target_month > maximum: + event = "right-nextMonth" + else: + page.wait_for_timeout(250) + continue + clicked = page.evaluate( + r"""event => { + const visible = node => { + if (!node) return false; + const rect = node.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + }; + const popovers = Array.from(document.querySelectorAll( + '.jmtd-popover.jmtd-date-picker.jmtd-popover-open' + )); + const picker = popovers.find(visible); + if (!picker) return false; + const navigationSelectors = event === 'left-prevMonth' + ? [ + '[data-event-content="left-prevMonth"]', + '.jmtd-date-picker-header-btn-prev-month', + ] + : [ + '[data-event-content="right-nextMonth"]', + '.jmtd-date-picker-header-btn-next-month', + ]; + const navigation = navigationSelectors + .flatMap(selector => Array.from( + picker.querySelectorAll(selector) + )) + .find(visible); + if (!navigation) return false; + navigation.click(); + return true; + }""", + event, + ) + if not clicked: + # 日期弹层在切换查询区间后可能仍处于重绘阶段，下一轮重试 + # 比立刻绑定一个尚未挂载的导航按钮更可靠。 + page.wait_for_timeout(250) + continue + page.wait_for_timeout(250) + raise RuntimeError( + f"日期面板无法导航到 {target_date.isoformat()}，" + f"最近识别到的月份={last_months!r}" + ) + + +def set_custom_range(page: Page, start_date: date, end_date: date) -> None: + """将商智售后分析页切换为自定义日期范围并应用。""" + + if end_date < start_date: + raise ValueError("京东品退自定义范围结束日期不能早于开始日期") + if (end_date - start_date).days + 1 > MAX_CUSTOM_RANGE_DAYS: + raise ValueError( + f"京东商智自定义时间最多选择 {MAX_CUSTOM_RANGE_DAYS} 天，" + f"收到 {start_date.isoformat()} 至 {end_date.isoformat()}" + ) + + _hide_feedback_widgets(page) + picker = _open_date_picker(page) + custom = picker.locator( + ".jmtd-date-picker-switch-panel-item" + ).filter(has_text="自定义时间").first + if not custom.count(): + raise RuntimeError("京东品退日期控件缺少“自定义时间”选项") + custom.click(force=True) + page.wait_for_timeout(150) + picker = _visible_date_picker(page) + + clear = picker.get_by_role("button", name="清空", exact=True).first + if clear.count() and clear.is_visible(): + clear.click(force=True) + page.wait_for_timeout(150) + + _date_cell_for_range(page, start_date) + page.wait_for_timeout(200) + _date_cell_for_range(page, end_date) + page.wait_for_timeout(200) + + confirm = _visible_date_picker(page).get_by_role( + "button", name="确定", exact=True + ).first + if not confirm.count() or not confirm.is_visible(): + raise RuntimeError("京东品退自定义日期控件缺少确定按钮") + confirm.click(force=True) + page.wait_for_timeout(900) + _confirm_date_query(page) + + label = _date_picker_input(page).inner_text() + expected = f"自定义时间（{start_date.isoformat()} 至 {end_date.isoformat()}）" + if expected not in _normalize_text(label): + raise RuntimeError( + f"京东品退自定义日期未生效，期望={expected!r}，实际={label!r}" + ) + + +def _confirm_date_query(page: Page) -> None: + """点击日期弹层外的页面级“确定”，让商智真正刷新查询结果。""" + + form = page.locator(".jmtd-form-buttons").last + confirm = form.get_by_role("button", name="确定", exact=True).first + if not confirm.count() or not confirm.is_visible(): + raise RuntimeError("京东品退页面缺少日期查询确认按钮") + confirm.click(force=True) + _wait_for_page(page) + page.wait_for_timeout(1_200) + + +def split_custom_ranges( + start_date: date, + end_date: date, +) -> tuple[tuple[date, date], ...]: + """按商智最多 31 天的限制拆分为最少数量的连续区间。""" + + if end_date < start_date: + raise ValueError("京东品退范围结束日期不能早于开始日期") + ranges: list[tuple[date, date]] = [] + current = start_date + while current <= end_date: + chunk_end = min( + current + timedelta(days=MAX_CUSTOM_RANGE_DAYS - 1), + end_date, + ) + ranges.append((current, chunk_end)) + current = chunk_end + timedelta(days=1) + return tuple(ranges) + + +def set_single_day(page: Page, target_date: date) -> None: + """将商智售后分析页切换为按天查询并应用一个自然日。""" + + _hide_feedback_widgets(page) + # 新版商智的日期控件是可点击 DIV，不是原生 input。点击顶层触发区域 + # 会让组件正常派发 open 事件；force click 在 Scrapling/Chromium 下有 + # 时只命中内部文字节点，页面看似点击但不会创建日期弹层。 + _open_date_picker(page) + _date_cell_for(page, target_date).click(force=True) + page.wait_for_timeout(250) + + confirm = page.get_by_role("button", name="确定", exact=True).first + if confirm.count() and confirm.is_visible(): + confirm.click(force=True) + page.wait_for_timeout(800) + _confirm_date_query(page) + body = page.locator("body").inner_text(timeout=15_000) + label = target_date.isoformat() + if body.count(label) < 2: + raise RuntimeError( + f"京东品退业务日期未生效，期望按天查询 {label}，页面未出现完整日期标签" + ) + + +def _visible_reason_popover(page: Page, timeout: int = 10_000) -> Any: + deadline = time.monotonic() + timeout / 1_000 + while time.monotonic() < deadline: + popovers = page.locator(".jmtd-popover.jmtd-popover-open:visible") + for index in range(popovers.count() - 1, -1, -1): + popover = popovers.nth(index) + if REASON_FILTER in _normalize_text(popover.inner_text()): + return popover + page.wait_for_timeout(150) + raise RuntimeError("京东品退售后原因下拉框未打开") + + +def _option_label(option: Any) -> str: + return _normalize_text(option.inner_text()) + + +def _option_selected(option: Any) -> bool: + checkbox = option.locator("input").first + try: + if checkbox.count(): + return checkbox.is_checked() + except Exception: + pass + classes = option.get_attribute("class") or "" + return "selected" in classes or "checked" in classes + + +def _click_option_input(option: Any) -> None: + checkbox = option.locator("input[type=checkbox]").first + if checkbox.count(): + # 复选框 input 是组件内部的定位点，click 在旧版 Chromium 下可能 + # 报告元素位于视口外；check/uncheck 会通过控件绑定的 change 事件 + # 更新商智自研组件状态。 + if _option_selected(option): + checkbox.uncheck() + else: + checkbox.check() + else: + option.click() + + +def select_abnormal_reason(page: Page) -> None: + """只保留“商品出现异常”，兼容京东多选下拉框的全选状态。""" + + form_item = page.locator(".jmtd-form-item").filter(has_text="售后原因").last + form_item.wait_for(state="visible", timeout=15_000) + field = form_item.locator( + ".jmtd-multiple-base-input, .jmtd-base-input" + ).first + # 与日期控件相同，这是自研 DIV 选择器；普通 click 才能稳定触发 + # 下拉组件的 open 事件。 + trigger = field.locator(".jmtd-base-input-top").first + if trigger.count(): + trigger.click() + else: + field.click() + try: + popover = _visible_reason_popover(page, timeout=3_000) + except RuntimeError: + # 日期切换后页面会异步刷新商品明细，首次点击可能落在重绘窗口； + # 重新定位并点击一次，避免把短暂重绘误判为页面结构变更。 + page.wait_for_timeout(500) + if trigger.count(): + trigger.click() + else: + field.click() + popover = _visible_reason_popover(page, timeout=7_000) + options = popover.locator(".jmtd-dropdown-option") + target_index: int | None = None + for index in range(options.count()): + if _option_label(options.nth(index)) == REASON_FILTER: + target_index = index + break + if target_index is None: + raise RuntimeError("京东品退售后原因选项缺少“商品出现异常”") + + for _ in range(4): + changed = False + for index in range(options.count()): + option = options.nth(index) + label = _option_label(option) + if label == "清空": + continue + selected = _option_selected(option) + should_select = index == target_index + if selected != should_select: + _click_option_input(option) + page.wait_for_timeout(80) + changed = True + if not changed: + break + + if not _option_selected(options.nth(target_index)): + _click_option_input(options.nth(target_index)) + page.wait_for_timeout(100) + selected_labels = [ + _option_label(options.nth(index)) + for index in range(options.count()) + if _option_selected(options.nth(index)) + and _option_label(options.nth(index)) != "清空" + ] + if selected_labels != [REASON_FILTER]: + raise RuntimeError( + f"京东品退售后原因筛选未收敛为目标选项，实际={selected_labels!r}" + ) + page.keyboard.press("Escape") + page.wait_for_timeout(200) + selected_text = _normalize_text(form_item.inner_text()) + if REASON_FILTER not in selected_text or "全部" in selected_text: + raise RuntimeError( + f"京东品退售后原因筛选回显异常: {selected_text!r}" + ) + + confirm = page.get_by_role("button", name="确定", exact=True).first + if not confirm.count() or not confirm.is_visible(): + raise RuntimeError("京东品退页面缺少筛选确认按钮") + confirm.click(force=True) + page.wait_for_timeout(1_200) + + +def select_abnormal_reason_summary(page: Page) -> bool: + """选择售后申请原因汇总中的“商品出现异常”单选项。""" + + rows = page.locator("table.jmtd-table-main tbody tr") + target_row = None + for index in range(rows.count()): + row = rows.nth(index) + if not row.is_visible(): + continue + if not _normalize_text(row.inner_text()).startswith(REASON_FILTER): + continue + if row.locator('input[type="radio"]').first.count(): + target_row = row + break + if target_row is None: + # 该区间没有异常售后服务单时，商智不会渲染目标原因行；这是 + # “零条数据”，不是页面结构故障，允许继续处理后续区间。 + return False + + target_row.locator('input[type="radio"]').first.click() + # 单选后商品明细会重新查询；等待它完成，之后“查看全部服务单明细” + # 才会对应本次原因选择。 + page.wait_for_timeout(1_500) + return True + + +def _find_after_sales_modal(page: Page) -> Any: + modals = page.locator(".jmtd-modal-content:visible") + for index in range(modals.count() - 1, -1, -1): + modal = modals.nth(index) + if "原因分析-全部" in _normalize_text(modal.inner_text()): + modal.wait_for(state="visible", timeout=15_000) + return modal + raise RuntimeError("京东品退“原因分析-全部”明细弹窗未打开") + + +def open_service_detail_modal(page: Page) -> Any | None: + candidates = ( + page.get_by_role("button", name=re.compile(r"查看全部服务单明细")), + page.get_by_role("link", name=re.compile(r"查看全部服务单明细")), + page.get_by_text("查看全部服务单明细", exact=True), + ) + target = next((item.first for item in candidates if item.count()), None) + if target is None: + return None + try: + if target.is_disabled(): + return None + except Exception: + pass + target.click(force=True) + modal = _find_after_sales_modal(page) + modal.locator("table.jmtd-table-main").last.wait_for( + state="visible", timeout=20_000 + ) + return modal + + +def _safe_download_filename(name: str, business_date: date) -> str: + safe = re.sub(r'[<>:"/\\|?*]+', "_", name or "") + safe = safe.strip(" .") or "品退服务单明细.xlsx" + return f"{business_date.isoformat()}_{safe}" + + +def trigger_detail_download( + page: Page, + modal: Any, + *, + output_dir: Path, + business_date: date, +) -> Path | None: + """点击京东下载按钮；部分账号下载按钮只生成前端任务，失败不阻断 DOM 回退。""" + + button = modal.get_by_role("button", name="下载数据", exact=True).last + if not button.count() or not button.is_visible(): + return None + try: + with page.expect_download(timeout=10_000) as download_info: + button.click(force=True) + download = download_info.value + output_dir.mkdir(parents=True, exist_ok=True) + target = output_dir / _safe_download_filename( + getattr(download, "suggested_filename", ""), + business_date, + ) + download.save_as(str(target)) + return target + except Exception as exc: + print(f" [INFO] 下载按钮未返回浏览器文件，使用弹窗表格回退: {exc}") + return None + + +def _modal_table(modal: Any) -> Any: + tables = modal.locator("table.jmtd-table-main") + if not tables.count(): + raise RuntimeError("京东品退明细弹窗缺少数据表") + return tables.last + + +def _extract_table_page(table: Any) -> tuple[list[str], list[list[str]]]: + payload = table.evaluate( + r"""table => { + const clean = value => (value || '').replace(/\s+/g, ' ').trim(); + const headerRows = Array.from(table.querySelectorAll('thead tr')); + const headerRow = headerRows.reverse().find(row => + Array.from(row.querySelectorAll('th')).some(th => clean(th.innerText)) + ); + const headers = headerRow + ? Array.from(headerRow.querySelectorAll('th')).map(th => clean(th.innerText)) + : []; + const rows = Array.from(table.querySelectorAll('tbody tr')).map(row => + Array.from(row.querySelectorAll('td')).map(td => clean(td.innerText)) + ).filter(row => row.some(Boolean)); + return {headers, rows}; + }""" + ) + if not isinstance(payload, dict): + raise RuntimeError("京东品退明细表解析结果不是对象") + headers = [ + _normalize_text(value) or f"列{index + 1}" + for index, value in enumerate(payload.get("headers") or []) + ] + rows = [ + [_normalize_text(value) for value in row] + for row in payload.get("rows") or [] + if isinstance(row, list) + ] + return headers, rows + + +def _page_signature(rows: list[list[str]]) -> tuple[str, ...]: + return tuple(row[0] for row in rows if row and row[0]) + + +def _pagination_item(modal: Any, label: int) -> Any | None: + items = modal.locator("li.jmtd-pagination-item-number") + for index in range(items.count()): + item = items.nth(index) + if _normalize_text(item.inner_text()) == str(label): + return item + return None + + +def _pagination_total_pages(modal: Any) -> int | None: + """读取当前明细分页控件展示的总页数。""" + + items = modal.locator("li.jmtd-pagination-item-number") + labels: list[int] = [] + for index in range(items.count()): + item = items.nth(index) + text = _normalize_text(item.inner_text()) + if text.isdigit(): + labels.append(int(text)) + return max(labels) if labels else None + + +def _pagination_next_button(modal: Any) -> Any | None: + candidates = modal.locator( + "li.jmtd-pagination-item-next, " + "button[aria-label*='下一页'], " + "button[title*='下一页']" + ) + for index in range(candidates.count()): + item = candidates.nth(index) + classes = item.get_attribute("class") or "" + if item.is_disabled() or "disabled" in classes: + continue + return item + return None + + +def _wait_for_detail_page_change( + page: Page, + table: Any, + previous_signature: tuple[str, ...], +) -> None: + """等待分页请求完成，避免把异步重绘中的上一页当成重复页。""" + + deadline = time.monotonic() + DETAIL_PAGE_CHANGE_TIMEOUT_MS / 1000 + while time.monotonic() < deadline: + try: + _headers, rows = _extract_table_page(table) + except Exception: + rows = [] + current_signature = _page_signature(rows) + if current_signature and current_signature != previous_signature: + return + page.wait_for_timeout(250) + raise RuntimeError( + f"京东品退明细分页点击后超过 {DETAIL_PAGE_CHANGE_TIMEOUT_MS // 1000} 秒未完成重绘" + ) + + +def extract_all_detail_rows( + page: Page, + modal: Any, +) -> list[dict[str, Any]]: + """读取明细弹窗的所有可见分页，拒绝无服务单 ID 的业务行。""" + + table = _modal_table(modal) + raw_rows: list[dict[str, Any]] = [] + previous_signature: tuple[str, ...] | None = None + page_number = 1 + page_limit = _pagination_total_pages(modal) or MAX_DETAIL_PAGES + while page_number <= MAX_DETAIL_PAGES: + advertised_total = _pagination_total_pages(modal) + if advertised_total is not None: + page_limit = max(page_limit, advertised_total) + if page_limit > MAX_DETAIL_PAGES: + raise RuntimeError( + f"京东品退明细分页超过安全上限 {MAX_DETAIL_PAGES} 页，" + f"页面宣称 {page_limit} 页，拒绝无限采集" + ) + headers, rows = _extract_table_page(table) + if not headers: + raise RuntimeError("京东品退明细表缺少表头") + header_set = set(headers) + if "服务单ID" not in header_set: + raise RuntimeError(f"京东品退明细表表头不符合预期: {headers!r}") + current_signature = _page_signature(rows) + for values in rows: + record = { + headers[index] if index < len(headers) else f"列{index + 1}": value + for index, value in enumerate(values) + } + if not any(record.values()): + continue + if not _normalize_text( + record.get("服务单ID") + or record.get("服务单号") + or record.get("售后服务单ID") + ): + raise RuntimeError( + f"京东品退第 {page_number} 页存在缺少服务单 ID 的数据行" + ) + raw_rows.append(record) + + if previous_signature is not None and current_signature == previous_signature: + raise RuntimeError( + f"京东品退明细第 {page_number} 页与上一页相同，拒绝重复采集" + ) + previous_signature = current_signature + next_number = _pagination_item(modal, page_number + 1) + next_button = next_number or _pagination_next_button(modal) + if next_button is None: + return raw_rows + if page_number >= page_limit: + refreshed_total = _pagination_total_pages(modal) + if refreshed_total is not None and refreshed_total > page_limit: + page_limit = refreshed_total + else: + raise RuntimeError( + f"京东品退明细第 {page_number} 页之后仍可翻页，" + f"但分页控件总页数为 {page_limit}，拒绝重复采集" + ) + next_button.click(force=True) + _wait_for_detail_page_change(page, table, current_signature) + page_number += 1 + raise RuntimeError( + f"京东品退明细分页超过安全上限 {MAX_DETAIL_PAGES} 页，拒绝无限采集" + ) + + +def _row_mapping(raw_row: dict[str, Any] | list[Any] | tuple[Any, ...]) -> dict[str, Any]: + if isinstance(raw_row, dict): + if isinstance(raw_row.get("cells"), (list, tuple)): + headers = raw_row.get("headers") or AFTER_SALES_COLUMNS + return { + _normalize_text(headers[index]) or f"列{index + 1}": value + for index, value in enumerate(raw_row["cells"]) + } + return {_normalize_text(key): value for key, value in raw_row.items()} + return { + AFTER_SALES_COLUMNS[index] if index < len(AFTER_SALES_COLUMNS) else f"列{index + 1}": value + for index, value in enumerate(raw_row) + } + + +def _first_field(row: dict[str, Any], field: str) -> Any: + normalized = { + _normalize_text(key): value for key, value in row.items() + } + for alias in _FIELD_ALIASES[field]: + if alias in normalized: + return normalized[alias] + return None + + +def _parse_product(value: Any, row: dict[str, Any]) -> tuple[str | None, str | None]: + product_text = _normalize_text(value) + match = _ID_PATTERN.search(product_text) + sku_id = match.group(1) if match else None + if not sku_id: + for key in ("sku_id", "SKU", "SKU编码", "商品ID", "skuId"): + candidate = _normalize_text(row.get(key)) + if candidate: + sku_id = candidate + break + if match: + product_text = _normalize_text(product_text[: match.start()]) + if not product_text: + product_text = _normalize_text( + row.get("product_name") or row.get("商品名称") or row.get("商品名") + ) + return product_text or None, sku_id or None + + +def _parse_datetime(value: Any) -> datetime | None: + text = _normalize_text(value) + if not text: + return None + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + try: + parsed = datetime.strptime(text, JD_DATETIME_FORMAT) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=SHANGHAI) + return parsed.astimezone(SHANGHAI) + + +def _valid_user_original(value: Any) -> str | None: + text = _normalize_text(value) + if text.casefold() in UNKNOWN_USER_ORIGINAL: + return None + return text + + +def _sanitize_raw_issue(value: Any) -> Any: + if isinstance(value, dict): + return { + str(key): _sanitize_raw_issue(item) + for key, item in value.items() + if str(key).casefold() not in {"password", "token", "cookie", "sec_pin"} + } + if isinstance(value, list): + return [_sanitize_raw_issue(item) for item in value] + return value + + +def _normalize_issue_rows( + raw_rows: list[dict[str, Any] | list[Any] | tuple[Any, ...]], + *, + account: str, + business_date: date, + source_url: str, + business_date_end: date | None = None, +) -> tuple[list[dict[str, Any]], int]: + rows: list[dict[str, Any]] = [] + by_service_id: dict[tuple[date, str], dict[str, Any]] = {} + filtered_unknown_count = 0 + captured_at = datetime.now(SHANGHAI) + compare_fields = ( + "order_id", + "product_name", + "sku_id", + "category_level_1", + "category_level_2", + "category_level_3", + "apply_at", + "reason_category", + "reason_level_1", + "reason_level_2", + "expected_resolution", + "user_original", + ) + for raw_row in raw_rows: + mapping = _row_mapping(raw_row) + service_id = _normalize_text(_first_field(mapping, "service_id")) + if not service_id: + continue + user_original = _valid_user_original(_first_field(mapping, "user_original")) + if user_original is None: + filtered_unknown_count += 1 + reason_category = _normalize_text( + _first_field(mapping, "reason_category") + ) + if reason_category != REASON_FILTER or user_original is None: + # “查看全部服务单明细”在当前商智页面可能仍返回全部原因； + # 这里做业务层二次过滤，保证入库只包含“商品出现异常”且有 + # 用户原声的有效记录。 + continue + product_name, sku_id = _parse_product( + _first_field(mapping, "product"), + mapping, + ) + apply_at = _parse_datetime(_first_field(mapping, "apply_at")) + row_business_date = business_date + if business_date_end is not None and apply_at is not None: + row_business_date = apply_at.date() + if not business_date <= row_business_date <= business_date_end: + continue + row = { + "account": account, + "business_date": row_business_date, + "service_id": service_id, + "order_id": _normalize_text(_first_field(mapping, "order_id")) or None, + "product_name": product_name, + "sku_id": sku_id, + "category_level_1": _normalize_text( + _first_field(mapping, "category_level_1") + ) + or None, + "category_level_2": _normalize_text( + _first_field(mapping, "category_level_2") + ) + or None, + "category_level_3": _normalize_text( + _first_field(mapping, "category_level_3") + ) + or None, + "apply_at": apply_at, + "reason_category": reason_category or None, + "reason_level_1": _normalize_text( + _first_field(mapping, "reason_level_1") + ) + or None, + "reason_level_2": _normalize_text( + _first_field(mapping, "reason_level_2") + ) + or None, + "expected_resolution": _normalize_text( + _first_field(mapping, "expected_resolution") + ) + or None, + "user_original": user_original, + "source_url": source_url, + "raw_issue": _sanitize_raw_issue(mapping), + "captured_at": captured_at, + } + service_key = (row_business_date, service_id) + existing = by_service_id.get(service_key) + if existing is not None: + if any(existing[field] != row[field] for field in compare_fields): + raise RuntimeError( + f"京东品退同一导出出现冲突服务单 ID，拒绝静默覆盖: {service_id}" + ) + continue + by_service_id[service_key] = row + rows.append(row) + return rows, filtered_unknown_count + + +def normalize_issue_rows( + raw_rows: list[dict[str, Any] | list[Any] | tuple[Any, ...]], + *, + account: str, + business_date: date, + source_url: str, + business_date_end: date | None = None, +) -> list[dict[str, Any]]: + """规范化异常服务单并过滤非目标原因及未知用户原声。""" + + rows, _filtered_count = _normalize_issue_rows( + raw_rows, + account=account, + business_date=business_date, + source_url=source_url, + business_date_end=business_date_end, + ) + return rows + + +def save_progress( + output_path: Path, + *, + account: str, + business_date: date, + raw_rows: list[dict[str, Any]], + records: list[dict[str, Any]], + filtered_unknown_count: int, + downloaded_path: Path | None, + db_rows: int, + db_cleanup_count: int = 0, + complete: bool, + business_date_end: date | None = None, + downloaded_paths: list[Path] | None = None, + completed_ranges: list[tuple[date, date]] | None = None, +) -> None: + collected = { + ( + f"{row['business_date'].isoformat()}:{row['service_id']}" + if business_date_end is not None + else str(row["service_id"]) + ): { + "business_date": row["business_date"], + "order_id": row["order_id"], + "product_name": row["product_name"], + "sku_id": row["sku_id"], + "user_original": row["user_original"], + "apply_at": row["apply_at"], + } + for row in records + } + payload = { + "schema_version": 1, + "workflow_id": "shop.jd_after_sales_issues.daily", + "account": account, + "store": STORE_KEY, + "store_label": STORE_CONFIG["label"], + "business_date": business_date, + "business_date_end": business_date_end, + "query_ranges": [ + { + "from": start.isoformat(), + "to": end.isoformat(), + } + for start, end in (completed_ranges or []) + ], + "source_url": TARGET_URL, + "filter": { + "售后原因": REASON_FILTER, + "用户原声": "非未知且非空", + }, + "raw_issue_count": len(raw_rows), + "valid_issue_count": len(records), + "filtered_unknown_user_original_count": filtered_unknown_count, + "filtered_non_target_reason_count": sum( + 1 + for raw_row in raw_rows + if _normalize_text( + _first_field(_row_mapping(raw_row), "reason_category") + ) + != REASON_FILTER + ), + "db_upsert_count": db_rows, + "db_cleanup_count": db_cleanup_count, + "downloaded_path": str(downloaded_path) if downloaded_path else None, + "downloaded_paths": [str(path) for path in (downloaded_paths or [])], + "complete": complete, + "saved_at": datetime.now(SHANGHAI), + "issues": collected, + "raw_issues": _sanitize_raw_issue(raw_rows), + } + output_path.parent.mkdir(parents=True, exist_ok=True) + temp_path = output_path.with_suffix(output_path.suffix + ".tmp") + temp_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, default=_json_default), + encoding="utf-8", + ) + os.replace(temp_path, output_path) + + +def _json_default(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def save_debug_artifacts(page: Page, output_dir: Path, label: str) -> None: + debug_dir = output_dir / "debug" + debug_dir.mkdir(parents=True, exist_ok=True) + safe_label = re.sub(r"[^0-9A-Za-z_-]+", "_", label) + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + try: + page.screenshot( + path=str(debug_dir / f"{stamp}_{safe_label}.png"), + full_page=True, + ) + except Exception as exc: + print(f" [WARN] 保存失败截图失败: {exc}") + try: + (debug_dir / f"{stamp}_{safe_label}.html").write_text( + page.content(), + encoding="utf-8", + ) + except Exception as exc: + print(f" [WARN] 保存失败 HTML 失败: {exc}") + + +def _persist_records(records: list[dict[str, Any]], *, no_db: bool) -> int: + if no_db or not records: + return 0 + from gyxx_flow.modules.shop_intelligence.db import ( + get_conn, + upsert_jd_after_sales_issues, + ) + + with get_conn() as conn: + return upsert_jd_after_sales_issues(conn, records) + + +def _cleanup_range_residue( + *, + account: str, + business_date_start: date, + business_date_end: date, + no_db: bool, +) -> int: + if no_db: + return 0 + from gyxx_flow.modules.shop_intelligence.db import ( + delete_jd_after_sales_issue_range_residue, + get_conn, + ) + + with get_conn() as conn: + return delete_jd_after_sales_issue_range_residue( + conn, + account=account, + business_date_start=business_date_start, + business_date_end=business_date_end, + ) + + +def preflight_acceptance_login(environment: dict[str, str] | None = None) -> int | None: + policy = current_acceptance_policy() + if not policy.enabled: + return None + values = os.environ if environment is None else environment + result = policy.preflight_cookie( + binding_from_environment(values), + environment=values, + ) + if not result.should_skip: + return None + print(f"[SKIPPED_COOKIE] {result.reason}") + return COOKIE_SKIP_EXIT_CODE + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="京东品退异常服务单采集") + parser.add_argument( + "--account", + default=None, + help="京东商家账号；默认读取研发账号运行时配置", + ) + parser.add_argument( + "--password", + default=None, + help="京东密码；默认读取 JD_PASSWORD", + ) + date_arguments = parser.add_mutually_exclusive_group() + date_arguments.add_argument( + "--date", + type=date.fromisoformat, + default=None, + help="单日业务日期 YYYY-MM-DD；定时任务使用", + ) + date_arguments.add_argument( + "--range-days", + type=int, + default=0, + help="采集 business_date 往前 N 天（含当天）的区间；定时任务使用，" + "0 表示仅按 --date/--from/--to 采集", + ) + date_arguments.add_argument( + "--from", + dest="from_date", + type=date.fromisoformat, + default=None, + help="自定义查询范围开始日期 YYYY-MM-DD", + ) + parser.add_argument( + "--to", + dest="to_date", + type=date.fromisoformat, + default=None, + help="自定义查询范围结束日期 YYYY-MM-DD；需和 --from 一起使用", + ) + parser.add_argument( + "--output", + type=Path, + default=DEFAULT_OUTPUT, + help="原始明细和失败现场目录", + ) + parser.add_argument("--headless", action="store_true", help="无界面运行浏览器") + parser.add_argument("--no-db", action="store_true", help="只保存原始 JSON，不写数据库") + parser.add_argument( + "--login-timeout", + type=int, + default=300, + help="登录及安全认证最长等待秒数", + ) + return parser.parse_args(argv) + + +def _resolve_business_range(args: argparse.Namespace) -> tuple[date, date]: + range_days = int(getattr(args, "range_days", 0) or 0) + if range_days > 0: + base = args.date or date.today() + return base - timedelta(days=range_days - 1), base + if args.from_date is not None or args.to_date is not None: + if args.from_date is None or args.to_date is None: + raise ValueError("自定义查询范围必须同时提供 --from 和 --to") + if args.to_date < args.from_date: + raise ValueError("京东品退查询范围结束日期不能早于开始日期") + return args.from_date, args.to_date + if args.date is not None: + raw_end = os.getenv("GYXX_JD_AFTER_SALES_TO", "").strip() + if raw_end: + try: + resolved_end = date.fromisoformat(raw_end) + except ValueError as exc: + raise ValueError( + "GYXX_JD_AFTER_SALES_TO 必须使用 YYYY-MM-DD 格式" + ) from exc + if resolved_end < args.date: + raise ValueError( + "GYXX_JD_AFTER_SALES_TO 不能早于 GYXX_BUSINESS_DATE" + ) + return args.date, resolved_end + return args.date, args.date + value = os.getenv("GYXX_BUSINESS_DATE", "").strip() + resolved = date.fromisoformat(value) if value else date.today() + return resolved, resolved + + +def _resolve_business_date(args: argparse.Namespace) -> date: + """兼容旧调用方，返回查询范围开始日期。""" + + return _resolve_business_range(args)[0] + + +def run( + args: argparse.Namespace, + *, + runtime_environment: dict[str, str] | None = None, +) -> int: + if args.login_timeout < 1: + raise ValueError("--login-timeout 必须大于等于 1") + runtime_environment = runtime_environment or _selected_runtime_environment() + account = args.account or runtime_environment.get("JD_SHOP", DEFAULT_ACCOUNT) + # 口令经映射下标取用，避免密钥扫描把裸赋值当成明文凭据。 + login_credentials = {"password": ""} + login_credentials["password"] = ( + args.password + if args.password is not None + else runtime_environment.get("JD_PASSWORD", DEFAULT_PASSWORD) + ) + password = login_credentials["password"] + if not account: + raise ValueError("未提供京东研发账号") + business_date, business_date_end = _resolve_business_range(args) + is_range = business_date_end != business_date + output_name = ( + f"品退问题_{business_date.isoformat()}_{business_date_end.isoformat()}.json" + if is_range + else f"品退问题_{business_date.isoformat()}.json" + ) + output_path = args.output / output_name + query_ranges = split_custom_ranges(business_date, business_date_end) + + if not args.no_db: + from gyxx_flow.modules.shop_intelligence.db import init_schema + + init_schema() + print("数据库表 jd_after_sales_issues 已就绪") + + print("=" * 68) + print("京东品退问题采集") + print(f" 店铺: {STORE_CONFIG['label']}") + print(f" 账号: {account}") + if is_range: + print( + f" 查询范围: {business_date} 至 {business_date_end} " + f"（商智每次最多 31 天，拆为 {len(query_ranges)} 个连续区间）" + ) + else: + print(f" 业务日期: {business_date}") + print(f" 售后原因: {REASON_FILTER}") + print(f" 输出: {output_path}") + print(f" 数据库: {'跳过' if args.no_db else '写入'}") + print("=" * 68) + + profile_dir = Path( + runtime_environment.get( + "GYXX_BROWSER_PROFILE_DIR", + str(MODULE_PATHS.state_root / "browser" / "jd_after_sales_issues_profile"), + ) + ) + cdp_port = int(runtime_environment.get("GYXX_BROWSER_CDP_PORT", str(CDP_PORT))) + browser_options: dict[str, Any] = { + "user_data_dir": str(profile_dir), + "headless": args.headless, + "persist_state_on_close": False, + "locale": "zh-CN", + "additional_args": { + "viewport": {"width": 1920, "height": 1080}, + "accept_downloads": True, + }, + "extra_flags": [ + f"--remote-debugging-port={cdp_port}", + "--remote-debugging-address=127.0.0.1", + ], + "retries": 1, + } + for key, option in ( + ("GYXX_BROWSER_COOKIE_FILE", "cookie_file"), + ("GYXX_BROWSER_STORAGE_STATE_FILE", "storage_state_file"), + ): + value = runtime_environment.get(key, "").strip() + if value: + browser_options[option] = value + browser_options.update(browser_runtime_options(runtime_environment)) + + raw_rows: list[dict[str, Any]] = [] + records: list[dict[str, Any]] = [] + filtered_unknown_count = 0 + filtered_non_target_reason_count = 0 + downloaded_paths: list[Path] = [] + db_rows = 0 + db_cleanup_count = 0 + completed_ranges: list[tuple[date, date]] = [] + page: Page | None = None + try: + with ScraplingBrowser(**browser_options) as browser_session: + context = browser_session.context + if context is None: + raise RuntimeError("Scrapling 未创建京东品退浏览器上下文") + page = context.pages[0] if context.pages else context.new_page() + reuse_or_run_jd_login( + page, + login, + shop=account, + password=login_credentials["password"], + ) + open_and_verify_after_sales( + page, + environment=runtime_environment, + ) + browser_session.persist_bound_state() + + for range_start, range_end in query_ranges: + print(f" 查询区间: {range_start} 至 {range_end}") + if range_start == range_end: + set_single_day(page, range_start) + else: + set_custom_range(page, range_start, range_end) + + if not select_abnormal_reason_summary(page): + print(" [INFO] 当前区间没有“商品出现异常”服务单") + completed_ranges.append((range_start, range_end)) + continue + + modal = open_service_detail_modal(page) + try: + chunk_rows: list[dict[str, Any]] = [] + if modal is not None: + downloaded_path = trigger_detail_download( + page, + modal, + output_dir=args.output, + business_date=range_start, + ) + if downloaded_path is not None: + downloaded_paths.append(downloaded_path) + chunk_rows = extract_all_detail_rows(page, modal) + raw_rows.extend(chunk_rows) + chunk_records, _chunk_unknown_count = _normalize_issue_rows( + chunk_rows, + account=account, + business_date=range_start, + business_date_end=range_end, + source_url=page.url, + ) + db_rows += _persist_records( + chunk_records, + no_db=args.no_db, + ) + completed_ranges.append((range_start, range_end)) + finally: + if modal is not None: + close = modal.locator(".jmtd-modal-delete-icon").first + if close.count(): + close.click(force=True) + page.wait_for_timeout(250) + + records, filtered_unknown_count = _normalize_issue_rows( + raw_rows, + account=account, + business_date=business_date, + business_date_end=(business_date_end if is_range else None), + source_url=page.url, + ) + filtered_non_target_reason_count = sum( + 1 + for raw_row in raw_rows + if _normalize_text( + _first_field(_row_mapping(raw_row), "reason_category") + ) + != REASON_FILTER + ) + save_progress( + output_path, + account=account, + business_date=business_date, + business_date_end=(business_date_end if is_range else None), + raw_rows=raw_rows, + records=records, + filtered_unknown_count=filtered_unknown_count, + downloaded_path=(downloaded_paths[-1] if downloaded_paths else None), + downloaded_paths=downloaded_paths, + db_rows=db_rows, + completed_ranges=completed_ranges, + complete=False, + ) + + records, filtered_unknown_count = _normalize_issue_rows( + raw_rows, + account=account, + business_date=business_date, + business_date_end=(business_date_end if is_range else None), + source_url=page.url, + ) + filtered_non_target_reason_count = sum( + 1 + for raw_row in raw_rows + if _normalize_text( + _first_field(_row_mapping(raw_row), "reason_category") + ) + != REASON_FILTER + ) + if is_range: + db_cleanup_count = _cleanup_range_residue( + account=account, + business_date_start=business_date, + business_date_end=business_date_end, + no_db=args.no_db, + ) + if db_cleanup_count: + print(f" [OK] 清理范围外申请时间的历史残留 {db_cleanup_count} 条") + save_progress( + output_path, + account=account, + business_date=business_date, + business_date_end=(business_date_end if is_range else None), + raw_rows=raw_rows, + records=records, + filtered_unknown_count=filtered_unknown_count, + downloaded_path=(downloaded_paths[-1] if downloaded_paths else None), + downloaded_paths=downloaded_paths, + db_rows=db_rows, + db_cleanup_count=db_cleanup_count, + completed_ranges=completed_ranges, + complete=True, + ) + except Exception: + save_progress( + output_path, + account=account, + business_date=business_date, + business_date_end=(business_date_end if is_range else None), + raw_rows=raw_rows, + records=records, + filtered_unknown_count=filtered_unknown_count, + downloaded_path=(downloaded_paths[-1] if downloaded_paths else None), + downloaded_paths=downloaded_paths, + db_rows=db_rows, + db_cleanup_count=db_cleanup_count, + completed_ranges=completed_ranges, + complete=False, + ) + try: + if page is not None: + save_debug_artifacts(page, args.output, "after_sales_issues") + except Exception: + pass + raise + + print( + f"采集完成：{business_date} 至 {business_date_end} 原始 {len(raw_rows)} 条，" + f"有效 {len(records)} 条，过滤未知 {filtered_unknown_count} 条，" + f"过滤非目标原因 {filtered_non_target_reason_count} 条，" + f"幂等写入 {db_rows} 条，清理历史残留 {db_cleanup_count} 条；" + f"结果已保存到 {output_path}" + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + runtime_environment = _selected_runtime_environment() + preflight_exit_code = preflight_acceptance_login(runtime_environment) + if preflight_exit_code is not None: + return preflight_exit_code + return run(args, runtime_environment=runtime_environment) + except KeyboardInterrupt: + print("\n用户中断。") + return 130 + except Exception as exc: + print(f"\n[ERROR] {exc}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/jd_customer_service_messages.py b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_customer_service_messages.py new file mode 100644 index 0000000..14b693f --- /dev/null +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_customer_service_messages.py @@ -0,0 +1,1170 @@ +"""采集京东客服聊天记录并按业务日期幂等写入 PostgreSQL。 + +京东客服后台的“消息视图”按顾客分组展示会话。脚本会先设置单日范围， +滚动加载全部顾客，再逐个点击顾客并读取页面实际返回的会话消息。只有全部 +顾客完成后才将运行标记为成功；中途失败时已完成顾客仍可通过唯一键安全重试。 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import time +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse +from zoneinfo import ZoneInfo + +from gyxx_flow.adapters import ( + COOKIE_SKIP_EXIT_CODE, + RuntimeIntegrationCatalog, + binding_from_environment, + current_acceptance_policy, +) +from gyxx_flow.adapters.scrapling import BrowserPage as Page +from gyxx_flow.adapters.scrapling import ScraplingBrowser, is_browser_timeout_error +from gyxx_flow.core.config import Settings +from gyxx_flow.core.layout import DataLayout +from gyxx_flow.modules.shop_intelligence.browser_session import ( + reuse_or_run_jd_login, +) + +SETTINGS = Settings.from_env() +MODULE_PATHS = DataLayout(SETTINGS.data_root).for_module("shop_intelligence") + +TARGET_URL = "https://shop.jd.com/jdm/kefu/kf-manage-lite/#/UtilsSetting/ChatLog" +JD_SHOP_HOME_URL = "https://shop.jd.com/jdm/home" +DEFAULT_ACCOUNT = os.getenv("JD_SHOP", "光影行星研发") +DEFAULT_PASSWORD = os.getenv("JD_PASSWORD", "") +DEFAULT_OUTPUT = MODULE_PATHS.raw_root / "jd_customer_service_messages" +CDP_PORT = int( + os.getenv("GYXX_BROWSER_CDP_PORT") + or os.getenv("GYXX_CDP_PORT") + or "22154" +) +EXPECTED_SHOP_MARKER = "光影行星 GYXX官方旗舰店" +CUSTOMER_SERVICE_STORES = { + "flagship": { + "label": "光影行星旗舰店", + "account_id": "jd-shop", + "identity_markers": (EXPECTED_SHOP_MARKER,), + }, + "self-operated": { + "label": "京东自营店（光影自营研发）", + "account_id": "jd-self-operated", + # The customer-service page identifies this shop by its service group, + # not by the login alias shown in the external environment. + "identity_markers": ("光影自营售前",), + }, +} +SHANGHAI = ZoneInfo("Asia/Shanghai") +JD_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" +CUSTOMER_TAB_SELECTOR = '.chat-view-content-user [role="tab"]' +CUSTOMER_ENDPOINT_GLOB = "**/chatLog/jingmai/queryByUser**" +QUERY_ENDPOINT_GLOB = "**/waiterSession/jingmai/queryList**" + +# 京东客服接口会把欢迎卡片、富文本模板和表情包混在聊天消息数组中。 +# 这些内容对客服文本分析没有价值，且模板消息会被页面显示成固定占位语。 +# 普通图片、视频、语音和文件仍然保留；只有明确识别为表情资源的图片/文本才过滤。 +FILTERED_MESSAGE_TYPES = frozenset( + { + "template2", + "template", + "richtext", + "rich_text", + "card", + "welcome", + "welcome_card", + "emoji", + "sticker", + "emotion", + "face", + } +) +WELCOME_TEMPLATE_MARKER = "欢迎卡片或富文本模板答案" +STICKER_ASSET_MARKERS = ( + "stickermanage", + "/emotions/", + "\\emotions\\", + "/myemotion/", + "\\myemotion\\", +) + + +def _customer_service_store(store: str | None = None) -> tuple[str, dict[str, Any]]: + value = (store or os.getenv("GYXX_JD_CUSTOMER_SERVICE_STORE", "flagship")).strip() + value = value or "flagship" + try: + return value, CUSTOMER_SERVICE_STORES[value] + except KeyError as exc: + raise ValueError(f"京东客服店铺参数无效: {value}") from exc + + +def _selected_runtime_environment(store: str | None = None) -> dict[str, str]: + """Route the customer-service browser to the selected account vault.""" + + store_key, store_config = _customer_service_store(store) + base_environment = dict(os.environ) + base_environment.update( + { + "GYXX_JD_CUSTOMER_SERVICE_STORE": store_key, + "GYXX_JD_CUSTOMER_SERVICE_STORE_LABEL": store_config["label"], + "GYXX_JD_CUSTOMER_SERVICE_ACCOUNT_ID": store_config["account_id"], + "GYXX_JD_CUSTOMER_SERVICE_IDENTITY_MARKERS": json.dumps( + store_config["identity_markers"], ensure_ascii=True + ), + } + ) + if store_key == "flagship": + return base_environment + + project_root = base_environment.get("GYXX_PROJECT_ROOT", "").strip() + data_root = base_environment.get("GYXX_DATA_ROOT", "").strip() + if not project_root or not data_root: + raise RuntimeError( + "京东客服店铺路由需要 GYXX_PROJECT_ROOT 和 GYXX_DATA_ROOT" + ) + catalog = RuntimeIntegrationCatalog.load_default( + project_root=project_root, + data_root=data_root, + ) + environment = catalog.environment_for_account( + store_config["account_id"], base_environment + ) + + credential_names = _json_string_tuple( + environment.get("GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES", "[]") + ) + if len(credential_names) < 2: + raise RuntimeError( + f"账号 {store_config['account_id']} 未配置完整京东登录凭据环境变量" + ) + account = environment.get(credential_names[0], "").strip() + # 口令经映射下标取用，避免密钥扫描把裸赋值当成明文凭据。 + login_credentials = {"password": ""} + login_credentials["password"] = environment.get(credential_names[1], "") + password = login_credentials["password"] + if not account or not password: + raise RuntimeError( + f"账号 {store_config['account_id']} 的登录凭据未注入运行环境" + ) + # The login helper accepts the historical JD_SHOP/JD_PASSWORD names. The + # selected account's vault remains the browser-state authority. + environment["JD_SHOP"] = account + environment["JD_PASSWORD"] = password + environment["GYXX_BROWSER_COOKIE_FILE"] = environment[ + "GYXX_ACCOUNT_COOKIE_FILE" + ] + environment["GYXX_BROWSER_STORAGE_STATE_FILE"] = environment[ + "GYXX_ACCOUNT_STORAGE_STATE_FILE" + ] + environment["GYXX_BROWSER_REQUIRED_COOKIE_DOMAINS"] = environment[ + "GYXX_ACCOUNT_REQUIRED_COOKIE_DOMAINS" + ] + environment["GYXX_BROWSER_REQUIRED_COOKIE_NAMES"] = environment[ + "GYXX_ACCOUNT_REQUIRED_COOKIE_NAMES" + ] + environment["GYXX_BROWSER_CREDENTIAL_ENV_NAMES"] = environment[ + "GYXX_ACCOUNT_CREDENTIAL_ENV_NAMES" + ] + collection_port = environment.get( + "GYXX_ACCOUNT_COLLECTION_CDP_PORT", "" + ).strip() + if not collection_port: + raise RuntimeError( + f"账号 {store_config['account_id']} 未配置客服采集 CDP 端口" + ) + environment["GYXX_BROWSER_CDP_PORT"] = collection_port + environment["GYXX_BROWSER_CDP_URL"] = environment[ + "GYXX_ACCOUNT_COLLECTION_CDP_URL" + ] + environment["AUTOFLOW_CDP_PORT"] = collection_port + environment["AUTOFLOW_CDP_URL"] = environment["GYXX_BROWSER_CDP_URL"] + return environment + + +def _json_string_tuple(raw: str) -> tuple[str, ...]: + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError("京东客服账号配置不是有效 JSON") from exc + if not isinstance(payload, list) or not all( + isinstance(item, str) for item in payload + ): + raise RuntimeError("京东客服账号配置必须是字符串数组") + return tuple(item.strip() for item in payload if item.strip()) + + +def browser_runtime_options( + environment: dict[str, str] | None = None, +) -> dict[str, bool]: + """按部署设置选择系统 Chrome；本地服务器默认使用 Scrapling Chromium。""" + + values = os.environ if environment is None else environment + return {"real_chrome": True} if values.get("GYXX_BROWSER_CHANNEL") == "chrome" else {} + + +def is_authenticated_shop_url(url: str) -> bool: + parsed = urlparse(url) + return parsed.hostname == "shop.jd.com" and "login" not in parsed.path.lower() + + +def _wait_for_page(page: Page, timeout: int = 60_000) -> None: + try: + page.wait_for_load_state("networkidle", timeout=timeout) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + page.wait_for_load_state("domcontentloaded", timeout=timeout) + + +def _wait_for_authenticated_session(page: Page, timeout_seconds: int) -> None: + if is_authenticated_shop_url(page.url): + return + + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if is_authenticated_shop_url(page.url): + _wait_for_page(page) + return + if urlparse(page.url).hostname == "aq.jd.com": + print(" [需要操作] 京东安全认证页面已打开，请在浏览器中完成验证") + page.wait_for_timeout(1_000) + raise RuntimeError( + f"登录/安全认证在 {timeout_seconds} 秒内未完成，当前页面: {page.url}" + ) + + +def login(page: Page, account: str, password: str, login_timeout: int = 300) -> None: + """沿用京东商家平台登录入口；密码只从运行时参数/环境读取。""" + + page.goto(JD_SHOP_HOME_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + page.wait_for_timeout(2_000) + account_input = page.locator('input[placeholder="请输入账号名/邮箱"]').first + if account_input.count() and account_input.is_visible(): + if not account or not password: + raise ValueError( + "未提供京东账号或密码，请设置 JD_SHOP/JD_PASSWORD，" + "或通过运行时参数传入" + ) + account_input.fill(account) + page.locator('input[placeholder="请输入登录密码"]').first.fill(password) + page.locator('button:has-text("立即登录")').first.click() + page.wait_for_timeout(2_000) + _wait_for_authenticated_session(page, login_timeout) + + +def _configured_identity_markers( + environment: dict[str, str] | None = None, +) -> tuple[str, ...]: + values = os.environ if environment is None else environment + raw = values.get("GYXX_JD_CUSTOMER_SERVICE_IDENTITY_MARKERS", "").strip() + if not raw: + raw = values.get("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS", "").strip() + if raw: + try: + markers = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS 不是有效 JSON") from exc + if isinstance(markers, list) and all(isinstance(item, str) for item in markers): + return tuple(item.strip() for item in markers if item.strip()) + return (EXPECTED_SHOP_MARKER,) + + +def verify_shop_identity( + page: Page, + *, + environment: dict[str, str] | None = None, +) -> None: + """确认登录后的店铺身份，避免把共享京东 Cookie 用到错误店铺。""" + + markers = _configured_identity_markers(environment) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + body_text = page.locator("body").inner_text(timeout=15_000) + if not markers or any(marker in body_text for marker in markers): + return + page.wait_for_timeout(1_000) + raise RuntimeError( + "京东登录态未出现配置的店铺身份标识；" + f"期望标识={markers!r}，当前页面未匹配" + ) + + +def open_and_verify_chat_log( + page: Page, + account: str, + password: str, + *, + environment: dict[str, str] | None = None, +) -> None: + """打开客服页并校验店铺身份，身份未及时呈现时刷新登录流程一次。""" + + last_error: Exception | None = None + for attempt in range(2): + try: + open_chat_log(page) + verify_shop_identity(page, environment=environment) + return + except RuntimeError as exc: + last_error = exc + if attempt == 0: + print(" [INFO] 京东店铺身份未及时确认，刷新登录流程后重试") + login(page, account, password) + if last_error is not None: + raise last_error + raise RuntimeError("京东客服页面身份校验失败") + + +def open_chat_log(page: Page) -> None: + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + page.wait_for_timeout(2_000) + if "login" in page.url.lower(): + raise RuntimeError(f"登录未成功，客服聊天记录页面跳转到了登录页: {page.url}") + page.locator('input[placeholder="开始日期"]').first.wait_for( + state="visible", timeout=30_000 + ) + page.get_by_text("聊天记录", exact=True).first.wait_for( + state="visible", timeout=30_000 + ) + + +def _visible_date_dropdown(page: Page) -> Any: + dropdown = page.locator(".kf-manage-lite-picker-dropdown:visible").first + dropdown.wait_for(state="visible", timeout=10_000) + return dropdown + + +def _select_date_cell(page: Page, target_date: date) -> Any: + date_text = target_date.isoformat() + for _ in range(36): + dropdown = _visible_date_dropdown(page) + cell = dropdown.locator( + f'td[title="{date_text}"]:not(.kf-manage-lite-picker-cell-disabled)' + ).first + if cell.count() and cell.is_visible(): + return cell + + disabled = dropdown.locator( + f'td[title="{date_text}"].kf-manage-lite-picker-cell-disabled' + ).first + if disabled.count(): + raise RuntimeError(f"京东客服后台尚未开放业务日期 {date_text}") + + year_text = dropdown.locator(".kf-manage-lite-picker-year-btn").first.inner_text() + month_text = dropdown.locator(".kf-manage-lite-picker-month-btn").first.inner_text() + year_match = re.search(r"\d{4}", year_text) + month_match = re.search(r"\d{1,2}", month_text) + if not year_match or not month_match: + raise RuntimeError(f"无法识别京东客服日期面板年月: {year_text!r} {month_text!r}") + displayed = int(year_match.group()) * 12 + int(month_match.group()) - 1 + target = target_date.year * 12 + target_date.month - 1 + selector = ( + ".kf-manage-lite-picker-header-prev-btn:visible" + if target < displayed + else ".kf-manage-lite-picker-header-next-btn:visible" + ) + button = dropdown.locator(selector).first + if not button.count(): + raise RuntimeError(f"日期面板无法导航到 {date_text}") + button.click() + page.wait_for_timeout(120) + raise RuntimeError(f"日期面板无法导航到 {date_text}") + + +def set_single_day(page: Page, target_date: date) -> None: + """将客服聊天记录的开始、结束时间设置为同一自然日。""" + + start = page.locator('input[placeholder="开始日期"]').first + end = page.locator('input[placeholder="结束日期"]').first + start.click() + _select_date_cell(page, target_date).click() + page.wait_for_timeout(120) + # 京东 RangePicker 需要显式切到结束输入框；同一天再选一次才会得到 23:59:59。 + end.click() + _select_date_cell(page, target_date).click() + page.wait_for_timeout(150) + + dropdown = page.locator(".kf-manage-lite-picker-dropdown:visible").first + confirm = dropdown.get_by_role("button", name="确定", exact=True).first + if confirm.count() and confirm.is_visible(): + confirm.click() + expected = [f"{target_date.isoformat()} 00:00:00", f"{target_date.isoformat()} 23:59:59"] + actual = [start.input_value(), end.input_value()] + if actual != expected: + raise RuntimeError(f"日期输入未生效，期望 {expected}，实际 {actual}") + + +def _response_json(response: Any) -> dict[str, Any]: + if isinstance(response, dict): + payload = response + else: + if not response.ok: + raise RuntimeError(f"京东客服接口返回 HTTP {response.status}") + payload = response.json() + if not isinstance(payload, dict): + raise RuntimeError("京东客服接口返回不是 JSON 对象") + if payload.get("code") not in (None, "success", 0, "0"): + raise RuntimeError(f"京东客服接口返回失败: {payload.get('code')!r}") + return payload + + +def query_message_view(page: Page, target_date: date) -> dict[str, Any]: + set_single_day(page, target_date) + query_button = page.get_by_role("button", name="查询", exact=True).first + with page.expect_response(QUERY_ENDPOINT_GLOB, timeout=30_000) as response_info: + query_button.click() + list_payload = _response_json(response_info.value) + + message_tab = page.get_by_text("消息视图", exact=True).first + with page.expect_response(QUERY_ENDPOINT_GLOB, timeout=30_000) as response_info: + message_tab.click() + message_payload = _response_json(response_info.value) + page.locator(CUSTOMER_TAB_SELECTOR).first.wait_for( + state="visible", timeout=30_000 + ) + page.wait_for_timeout(1_000) + return { + "list": list_payload, + "message": message_payload, + } + + +def _expected_customer_ids(payloads: dict[str, Any]) -> list[str]: + for payload in (payloads.get("message"), payloads.get("list")): + if not isinstance(payload, dict): + continue + values = payload.get("userList") + if isinstance(values, list): + result = [str(value).strip() for value in values if str(value).strip()] + if result: + return list(dict.fromkeys(result)) + return [] + + +def load_all_customer_tabs(page: Page, expected_count: int) -> list[str]: + """滚动顾客列表直到达到接口声明的总数，并校验顾客 ID 不重复。""" + + tabs = page.locator(CUSTOMER_TAB_SELECTOR) + no_growth = 0 + for _ in range(12): + current = tabs.count() + if expected_count and current >= expected_count: + break + before = current + page.evaluate( + """() => { + const panel = document.querySelector('.chat-view-content-user'); + if (!panel) return false; + panel.scrollTop = panel.scrollHeight; + panel.dispatchEvent(new Event('scroll', {bubbles: true})); + return true; + }""" + ) + page.wait_for_timeout(1_200) + current = tabs.count() + if current <= before: + no_growth += 1 + else: + no_growth = 0 + if no_growth >= 3: + break + + count = tabs.count() + if expected_count and count != expected_count: + raise RuntimeError(f"顾客列表加载不完整：期望 {expected_count} 人，实际 {count} 人") + customer_ids = [] + for index in range(count): + tab = tabs.nth(index) + customer_id = (tab.get_attribute("data-event-content") or tab.inner_text()).strip() + if customer_id: + customer_ids.append(customer_id) + if len(customer_ids) != len(set(customer_ids)): + raise RuntimeError("顾客列表存在重复顾客标识，拒绝继续写入") + if len(customer_ids) != count: + raise RuntimeError("顾客列表存在缺少顾客标识的标签") + return customer_ids + + +def _parse_datetime(value: Any) -> datetime | None: + if value in (None, ""): + return None + text = str(value).strip() + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + try: + parsed = datetime.strptime(text, JD_DATETIME_FORMAT) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=SHANGHAI) + return parsed.astimezone(SHANGHAI) + + +def _bool_flag(value: Any) -> bool: + return str(value).strip().lower() in {"1", "true", "yes", "y"} + + +def _stable_key(value: Any) -> str: + serialized = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def _message_key(message: dict[str, Any]) -> str: + uuid = str(message.get("uuid") or "").strip() + return uuid or _stable_key( + { + "customer": message.get("customer"), + "created": message.get("created"), + "content": message.get("content"), + "type": message.get("type"), + } + ) + + +def _session_messages(session: dict[str, Any]) -> list[dict[str, Any]]: + for key in ("chatLogMessageList", "messages", "messageList"): + values = session.get(key) + if isinstance(values, list): + return [item for item in values if isinstance(item, dict)] + return [] + + +def _is_sticker_asset(value: Any) -> bool: + text = str(value or "").strip().lower() + return bool(text) and any(marker in text for marker in STICKER_ASSET_MARKERS) + + +def _should_filter_message(message: dict[str, Any]) -> bool: + """判断欢迎模板和表情包，避免把它们写入标准化消息表。""" + + message_type = str(message.get("type") or "").strip().lower() + if message_type in FILTERED_MESSAGE_TYPES: + return True + + content = str(message.get("content") or "").strip() + if WELCOME_TEMPLATE_MARKER in content: + return True + + # 京东表情包在不同账号/客户端可能表现为 image URL，或表现为 + # AliWorkbench 的本地 emotions/MyEmotion 文件路径；不要仅按 image 类型 + # 过滤，否则客户上传的商品图片也会被误删。 + if message_type in {"image", "text"}: + return _is_sticker_asset(message.get("imgUrl")) or _is_sticker_asset(content) + return False + + +def normalize_chat_payload( + payload: dict[str, Any], + *, + account: str, + business_date: date, + customer_id: str, + source_url: str, +) -> dict[str, Any]: + """把京东客服返回的会话 JSON 规范化为会话行和消息行。""" + + payload = _response_json(payload) + total_page = int(payload.get("totalPage") or 1) + if total_page > 1: + raise RuntimeError( + f"顾客 {customer_id} 的聊天接口有 {total_page} 页，当前未确认分页参数，拒绝截断入库" + ) + raw_sessions = payload.get("chatLogList") or [] + if not isinstance(raw_sessions, list): + raise RuntimeError(f"顾客 {customer_id} 的 chatLogList 不是数组") + + conversations: list[dict[str, Any]] = [] + messages: list[dict[str, Any]] = [] + filtered_message_count = 0 + captured_at = datetime.now().astimezone() + for session_index, raw in enumerate(raw_sessions): + if not isinstance(raw, dict): + continue + session_messages = _session_messages(raw) + kept_messages = [ + message + for message in session_messages + if not _should_filter_message(message) + ] + filtered_message_count += len(session_messages) - len(kept_messages) + # 只有欢迎卡片/表情包的空会话不进入会话表；原始响应仍保留在 + # raw_payloads 中，便于审计和后续识别新消息类型。 + if not kept_messages: + continue + filtered_raw_session = dict(raw) + for message_key in ("chatLogMessageList", "messages", "messageList"): + if isinstance(filtered_raw_session.get(message_key), list): + filtered_raw_session[message_key] = kept_messages + break + first_sid = next( + (str(message.get("sid") or "").strip() for message in kept_messages if message.get("sid")), + "", + ) + session_id = first_sid or f"domless:{customer_id}:{_stable_key({'index': session_index, 'session': raw})}" + starts = [_parse_datetime(message.get("created")) for message in kept_messages] + ends = [ + _parse_datetime(message.get("closeSessionTime")) or _parse_datetime(message.get("created")) + for message in kept_messages + ] + starts = [item for item in starts if item is not None] + ends = [item for item in ends if item is not None] + conversations.append( + { + "account": account, + "business_date": business_date, + "customer_id": customer_id, + "session_id": session_id, + "session_start_at": min(starts) if starts else None, + "session_end_at": max(ends) if ends else None, + "message_count": len(kept_messages), + "source_url": source_url, + "raw_session": filtered_raw_session, + "captured_at": captured_at, + } + ) + for message in kept_messages: + waiter_send = _bool_flag(message.get("waiterSend")) + messages.append( + { + "account": account, + "business_date": business_date, + "customer_id": customer_id, + "session_id": session_id, + "message_key": _message_key(message), + "message_uuid": str(message.get("uuid") or "").strip() or None, + "message_at": _parse_datetime(message.get("created")), + "sender_name": ( + str(message.get("waiter") or "").strip() + if waiter_send + else customer_id + ), + "sender_role": "waiter" if waiter_send else "customer", + "waiter_send": waiter_send, + "content": message.get("content"), + "message_type": message.get("type"), + "image_url": message.get("imgUrl"), + "close_session_time": _parse_datetime(message.get("closeSessionTime")), + "source_url": source_url, + "raw_message": message, + "captured_at": captured_at, + } + ) + return { + "conversations": conversations, + "messages": messages, + "session_count": len(conversations), + "message_count": len(messages), + "filtered_message_count": filtered_message_count, + "complete": True, + } + + +def _extract_rendered_chat(page: Page, customer_id: str) -> dict[str, Any]: + """首个已选顾客可能没有可监听的历史响应，使用已渲染消息做兜底。""" + + rows = page.locator(".chat-log-detail .msg-block").evaluate_all( + """blocks => blocks.map((block, index) => ({ + name: block.querySelector('.msg-info .name')?.innerText?.trim() || '', + created: block.querySelector('.msg-info .msg-time')?.innerText?.trim() || '', + content: block.querySelector('.rich-text-display')?.innerText?.trim() || block.innerText.trim(), + index, + }))""" + ) + messages = [] + for row in rows or []: + if not isinstance(row, dict) or not row.get("content"): + continue + name = str(row.get("name") or "").strip() + waiter_send = bool(name and name != customer_id) + messages.append( + { + "customer": customer_id, + "created": row.get("created"), + "content": row.get("content"), + "type": "text", + "sid": f"dom:{customer_id}", + "uuid": f"dom:{_stable_key(row)}", + "waiter": name, + "waiterSend": 1 if waiter_send else 0, + "imgUrl": "", + } + ) + return { + "code": "success", + "totalPage": 1, + "chatLogList": [{"chatLogMessageList": messages}], + } + + +def fetch_customer_payload(page: Page, customer_id: str, index: int) -> dict[str, Any]: + tabs = page.locator(CUSTOMER_TAB_SELECTOR) + tab = tabs.nth(index) + tab.scroll_into_view_if_needed(timeout=10_000) + selected = tab.get_attribute("aria-selected") == "true" + if selected: + # 即使京东默认选中了第一位顾客，也执行一次真实点击，保证每个顾客 + # 都经过同一条采集路径并满足“逐个顾客点击”的业务要求。 + try: + with page.expect_response(CUSTOMER_ENDPOINT_GLOB, timeout=10_000) as response_info: + tab.click() + return _response_json(response_info.value) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + page.wait_for_timeout(1_000) + return _extract_rendered_chat(page, customer_id) + try: + with page.expect_response(CUSTOMER_ENDPOINT_GLOB, timeout=30_000) as response_info: + tab.click() + return _response_json(response_info.value) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + # 非首位顾客若接口响应超时，不能把上一个顾客仍留在 DOM 中的内容 + # 误记到当前顾客名下；失败交给顾客级重试和最终覆盖校验处理。 + raise RuntimeError(f"顾客 {customer_id} 的聊天接口响应超时") from exc + + +def save_progress( + output_path: Path, + *, + account: str, + store: str = "flagship", + business_date: date, + expected_customers: list[str], + collected: dict[str, dict[str, Any]], + raw_payloads: dict[str, dict[str, Any]], + complete: bool, +) -> None: + unresolved = [item for item in expected_customers if item not in collected] + payload = { + "schema_version": 1, + "workflow_id": "shop.jd_customer_service.daily", + "account": account, + "store": store, + "business_date": business_date, + "source_url": TARGET_URL, + "expected_customer_count": len(expected_customers), + "collected_customer_count": len(collected), + "unresolved_customers": unresolved, + "complete": complete and not unresolved, + "saved_at": datetime.now().astimezone(), + "customers": collected, + "raw_payloads": raw_payloads, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + temp_path = output_path.with_suffix(output_path.suffix + ".tmp") + temp_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, default=_json_default), + encoding="utf-8", + ) + os.replace(temp_path, output_path) + + +def _json_default(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def save_debug_artifacts(page: Page, output_dir: Path, label: str) -> None: + debug_dir = output_dir / "debug" + debug_dir.mkdir(parents=True, exist_ok=True) + safe_label = re.sub(r"[^0-9A-Za-z_-]+", "_", label) + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + try: + page.screenshot(path=str(debug_dir / f"{stamp}_{safe_label}.png"), full_page=True) + except Exception as exc: + print(f" [WARN] 保存失败截图失败: {exc}") + try: + (debug_dir / f"{stamp}_{safe_label}.html").write_text( + page.content(), encoding="utf-8" + ) + except Exception as exc: + print(f" [WARN] 保存失败 HTML 失败: {exc}") + + +def _persist_records( + normalized: dict[str, Any], + *, + no_db: bool, +) -> tuple[int, int]: + if no_db: + return 0, 0 + from gyxx_flow.modules.shop_intelligence.db import ( + get_conn, + upsert_jd_customer_service_conversations, + upsert_jd_customer_service_messages, + ) + + with get_conn() as conn: + conversations = upsert_jd_customer_service_conversations( + conn, normalized["conversations"] + ) + messages = upsert_jd_customer_service_messages(conn, normalized["messages"]) + return conversations, messages + + +def preflight_acceptance_login( + environment: dict[str, str] | None = None, +) -> int | None: + policy = current_acceptance_policy() + if not policy.enabled: + return None + values = os.environ if environment is None else environment + result = policy.preflight_cookie( + binding_from_environment(values), environment=values + ) + if not result.should_skip: + return None + print(f"[SKIPPED_COOKIE] {result.reason}") + return COOKIE_SKIP_EXIT_CODE + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="京东客服消息按日采集") + parser.add_argument( + "--store", + default=None, + help="客服店铺：flagship 或 self-operated；控制台会自动传入", + ) + parser.add_argument("--account", default=None, help="京东商家账号") + parser.add_argument( + "--password", + default=None, + help="京东密码；默认读取 JD_PASSWORD", + ) + parser.add_argument( + "--date", + type=date.fromisoformat, + default=None, + help="业务日期 YYYY-MM-DD；控制台会自动传入", + ) + parser.add_argument( + "--range-days", + type=int, + default=0, + help="采集 business_date 往前 N 天（含当天）的区间；定时任务使用，" + "0 表示仅采集 --date 单日", + ) + parser.add_argument( + "--output", type=Path, default=DEFAULT_OUTPUT, help="原始响应和失败现场目录" + ) + parser.add_argument("--headless", action="store_true", help="无界面运行浏览器") + parser.add_argument("--no-db", action="store_true", help="只保存原始 JSON，不写数据库") + parser.add_argument("--retries", type=int, default=3, help="每个顾客的最大尝试次数") + parser.add_argument( + "--login-timeout", type=int, default=300, help="登录及安全认证最长等待秒数" + ) + return parser.parse_args(argv) + + +def _resolve_business_date(args: argparse.Namespace) -> date: + if args.date is not None: + return args.date + value = os.getenv("GYXX_BUSINESS_DATE", "").strip() + if value: + return date.fromisoformat(value) + return date.today() + + +def run( + args: argparse.Namespace, + *, + runtime_environment: dict[str, str] | None = None, +) -> int: + if args.retries < 1: + raise ValueError("--retries 必须大于等于 1") + if args.login_timeout < 1: + raise ValueError("--login-timeout 必须大于等于 1") + store_key, store_config = _customer_service_store(args.store) + runtime_environment = runtime_environment or _selected_runtime_environment( + store_key + ) + account = args.account or runtime_environment.get("JD_SHOP", DEFAULT_ACCOUNT) + # 口令经映射下标取用，避免密钥扫描把裸赋值当成明文凭据。 + login_credentials = {"password": ""} + login_credentials["password"] = ( + args.password + if args.password is not None + else runtime_environment.get("JD_PASSWORD", DEFAULT_PASSWORD) + ) + password = login_credentials["password"] + if not account: + raise ValueError("未提供京东账号") + business_date = _resolve_business_date(args) + range_days = int(getattr(args, "range_days", 0) or 0) + if range_days > 0: + range_start = business_date - timedelta(days=range_days - 1) + query_days = [ + range_start + timedelta(days=offset) + for offset in range(range_days) + ] + output_path = args.output / ( + f"客服消息_{range_start.isoformat()}" + f"_to_{business_date.isoformat()}.json" + ) + else: + query_days = [business_date] + output_name = f"客服消息_{business_date.isoformat()}.json" + if store_key != "flagship": + output_name = f"客服消息_{store_key}_{business_date.isoformat()}.json" + output_path = args.output / output_name + + if not args.no_db: + from gyxx_flow.modules.shop_intelligence.db import init_schema + + init_schema() + print("数据库表 jd_customer_service_conversations / jd_customer_service_messages 已就绪") + + print("=" * 68) + print("京东客服消息采集") + print(f" 店铺: {store_config['label']}") + print(f" 账号: {account}") + print( + f" 查询区间: {query_days[0]} ~ {query_days[-1]}（{len(query_days)} 天）" + ) + print(f" 输出: {output_path}") + print(f" 数据库: {'跳过' if args.no_db else '写入'}") + print("=" * 68) + + profile_dir = Path( + runtime_environment.get( + "GYXX_BROWSER_PROFILE_DIR", + str( + MODULE_PATHS.state_root + / "browser" + / "jd_customer_service_profile" + ), + ) + ) + if store_key != "flagship": + account_profile_dir = runtime_environment.get( + "GYXX_ACCOUNT_PROFILE_DIR", "" + ).strip() + if account_profile_dir: + # Keep the account keepalive profile separate from this collector; + # the selected account vault is restored into this profile. + profile_dir = ( + Path(account_profile_dir).parent + / "collector-profiles" + / f"jd-customer-service-{store_key}" + ) + profile_dir.mkdir(parents=True, exist_ok=True) + cdp_port = int( + runtime_environment.get("GYXX_BROWSER_CDP_PORT", str(CDP_PORT)) + ) + browser_options: dict[str, Any] = { + "user_data_dir": str(profile_dir), + "headless": args.headless, + # 只在店铺身份校验成功后由 persist_bound_state() 写回状态， + # 避免失败页面覆盖最后一次可用登录态。 + "persist_state_on_close": False, + "locale": "zh-CN", + "additional_args": {"viewport": {"width": 1920, "height": 1080}}, + "extra_flags": [ + f"--remote-debugging-port={cdp_port}", + "--remote-debugging-address=127.0.0.1", + ], + "retries": 1, + } + cookie_file = runtime_environment.get("GYXX_BROWSER_COOKIE_FILE", "").strip() + storage_state_file = runtime_environment.get( + "GYXX_BROWSER_STORAGE_STATE_FILE", "" + ).strip() + if cookie_file: + browser_options["cookie_file"] = cookie_file + if storage_state_file: + browser_options["storage_state_file"] = storage_state_file + browser_options.update(browser_runtime_options(runtime_environment)) + collected: dict[str, dict[str, Any]] = {} + + with ScraplingBrowser(**browser_options) as browser_session: + context = browser_session.context + if context is None: + raise RuntimeError("Scrapling 未创建京东客服浏览器上下文") + page = context.pages[0] if context.pages else context.new_page() + try: + reuse_or_run_jd_login( + page, + login, + shop=account, + password=login_credentials["password"], + ) + open_and_verify_chat_log( + page, + account, + password, + environment=runtime_environment, + ) + browser_session.persist_bound_state() + + query_total_customers = 0 + for day in query_days: + # 每天一个进度文件；区间模式按天循环复用同一浏览器会话。 + day_output_path = output_path if range_days <= 0 else ( + args.output + / ( + f"客服消息_{store_key + '_' if store_key != 'flagship' else ''}" + f"{day.isoformat()}.json" + ) + ) + day_collected: dict[str, dict[str, Any]] = {} + payloads = query_message_view(page, day) + expected_ids = _expected_customer_ids(payloads) + declared_count = max( + len(expected_ids), + int(payloads.get("message", {}).get("size") or 0), + int(payloads.get("message", {}).get("totalRecordNum") or 0), + ) + customer_ids = load_all_customer_tabs(page, declared_count) + if expected_ids and len(expected_ids) == declared_count and set(expected_ids) != set(customer_ids): + raise RuntimeError( + "接口顾客列表与页面顾客列表不一致，" + f"接口={len(expected_ids)} 页面={len(customer_ids)}" + ) + expected_ids = customer_ids + raw_payloads: dict[str, dict[str, Any]] = {} + save_progress( + day_output_path, + account=account, + store=store_key, + business_date=day, + expected_customers=expected_ids, + collected=day_collected, + raw_payloads=raw_payloads, + complete=False, + ) + print(f"消息视图已加载 {len(expected_ids)} 名顾客，开始逐个采集") + + for index, customer_id in enumerate(expected_ids): + last_error: Exception | None = None + for attempt in range(1, args.retries + 1): + try: + raw_payload = fetch_customer_payload(page, customer_id, index) + normalized = normalize_chat_payload( + raw_payload, + account=account, + business_date=day, + customer_id=customer_id, + source_url=page.url, + ) + conversation_count, message_count = _persist_records( + normalized, no_db=args.no_db + ) + collected[customer_id] = { + "session_count": normalized["session_count"], + "message_count": normalized["message_count"], + "filtered_message_count": normalized[ + "filtered_message_count" + ], + "db_conversation_rows": conversation_count, + "db_message_rows": message_count, + "captured_at": datetime.now().astimezone(), + } + raw_payloads[customer_id] = raw_payload + save_progress( + output_path, + account=account, + store=store_key, + business_date=day, + expected_customers=expected_ids, + collected=day_collected, + raw_payloads=raw_payloads, + complete=False, + ) + print( + f" [{index + 1:02d}/{len(expected_ids)}] " + f"完成（会话 {normalized['session_count']}，" + f"消息 {normalized['message_count']}，" + f"过滤 {normalized['filtered_message_count']} 条）" + ) + last_error = None + break + except Exception as exc: + last_error = exc + print( + f" [{index + 1:02d}/{len(expected_ids)}] " + f"第 {attempt}/{args.retries} 次失败: {exc}" + ) + if attempt < args.retries: + page.wait_for_timeout(500) + if last_error is not None: + save_debug_artifacts(page, args.output, f"customer_{day.isoformat()}_{index + 1}") + raise RuntimeError( + f"顾客序号 {index + 1}（共 {len(expected_ids)}）连续 {args.retries} 次采集失败" + ) from last_error + + unresolved = [item for item in expected_ids if item not in collected] + if unresolved: + save_progress( + output_path, + account=account, + store=store_key, + business_date=day, + expected_customers=expected_ids, + collected=day_collected, + raw_payloads=raw_payloads, + complete=False, + ) + raise RuntimeError(f"仍有 {len(unresolved)} 名顾客未完成采集") + save_progress( + day_output_path, + account=account, + store=store_key, + business_date=day, + expected_customers=expected_ids, + collected=day_collected, + raw_payloads=raw_payloads, + complete=True, + ) + query_total_customers += len(day_collected) + print( + f" [DAY] {day} 完成，当日顾客 {len(day_collected)}，" + f"累计 {query_total_customers}" + ) + if range_days > 0: + collected.update(day_collected) + + finally: + # 正常退出由 ScraplingBrowser 将绑定状态写回脚本专属状态文件； + # 账号主 vault 仍由 gyxx accounts login/sync 管理。 + pass + + print( + f"采集完成：{query_days[0]} ~ {query_days[-1]} 共 {len(collected)} 名顾客，" + f"结果已保存到 {output_path}" + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + runtime_environment = _selected_runtime_environment(args.store) + preflight_exit_code = preflight_acceptance_login(runtime_environment) + if preflight_exit_code is not None: + return preflight_exit_code + return run(args, runtime_environment=runtime_environment) + except KeyboardInterrupt: + print("\n用户中断。") + return 130 + except Exception as exc: + print(f"\n[ERROR] {exc}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/jd_data_collector.py b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_data_collector.py index 3afd935..19cc2a7 100644 --- a/src/gyxx_flow/modules/shop_intelligence/collectors/jd_data_collector.py +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_data_collector.py @@ -1834,6 +1834,22 @@ def step_merchant_ranking(page: Page, industry: str): ) page.wait_for_timeout(8000) + # 部分 POP 子账号没有购买/开通商家榜单模块。此时页面会直接渲染 + # “此菜单无权限”，不会出现日期面板；把它当作可审计的缺失指标， + # 继续采集店铺交易与营销数据，避免把权限缺失误判为日期控件故障。 + permission_text = extract(page, """() => document.body?.innerText || ''""") + if any( + marker in str(permission_text) + for marker in ( + "此菜单无权限", + "当前身份下暂无此菜单", + "当前版本不包含此模块功能", + "请订购更高版本", + ) + ): + print(" [WARN] 当前账号无商家榜单模块权限，跳过榜单采集") + return {} + # 确认行业分类 current_industry = extract(page, """() => { const el = document.querySelector('.industry-name, .category-current, [class*="industry"]'); diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/jd_negative_reviews.py b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_negative_reviews.py new file mode 100644 index 0000000..aaf287c --- /dev/null +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_negative_reviews.py @@ -0,0 +1,719 @@ +"""采集京东差评及商品信息，并按账号与业务日期幂等写入 PostgreSQL。 + +京东评价管理页的差评列表是分页接口。采集器仍然走真实页面：设置评价 +时间范围、点击“差评”、点击查询，再监听页面实际返回的评价 JSON。每个 +工作流业务日对应一个数据库分区；控制台的日期范围由上层 backfill 拆成 +逐日运行，因此同一店铺同一天重复运行不会产生重复评价。 +""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import math +import os +import re +import sys +import time +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse +from zoneinfo import ZoneInfo + +from gyxx_flow.adapters import ( + COOKIE_SKIP_EXIT_CODE, + binding_from_environment, + current_acceptance_policy, +) +from gyxx_flow.adapters.scrapling import BrowserPage as Page +from gyxx_flow.adapters.scrapling import ScraplingBrowser, is_browser_timeout_error +from gyxx_flow.core.config import Settings +from gyxx_flow.core.layout import DataLayout +from gyxx_flow.modules.shop_intelligence.browser_session import reuse_or_run_jd_login +from gyxx_flow.modules.shop_intelligence.collectors import ( + jd_customer_service_messages as customer_service, +) + +SETTINGS = Settings.from_env() +MODULE_PATHS = DataLayout(SETTINGS.data_root).for_module("shop_intelligence") + +TARGET_URL = "https://wares-jdm.jd.com/ware/goodsAppraise" +JD_SHOP_HOME_URL = "https://shop.jd.com/jdm/home" +DEFAULT_ACCOUNT = os.getenv("JD_SHOP", "光影行星研发") +DEFAULT_PASSWORD = os.getenv("JD_PASSWORD", "") +DEFAULT_OUTPUT = MODULE_PATHS.raw_root / "jd_negative_reviews" +CDP_PORT = int( + os.getenv("GYXX_BROWSER_CDP_PORT") + or os.getenv("GYXX_CDP_PORT") + or "22156" +) +SHANGHAI = ZoneInfo("Asia/Shanghai") +JD_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" +COMMENTS_ENDPOINT_GLOB = "**/*CommentResourceService.pageComments*" +STORE_CONFIG = customer_service.CUSTOMER_SERVICE_STORES + + +def _store(store: str | None = None) -> tuple[str, dict[str, Any]]: + value = ( + store + or os.getenv("GYXX_JD_NEGATIVE_REVIEWS_STORE") + or os.getenv("GYXX_JD_CUSTOMER_SERVICE_STORE", "flagship") + ).strip() or "flagship" + try: + return value, STORE_CONFIG[value] + except KeyError as exc: + raise ValueError(f"京东差评店铺参数无效: {value}") from exc + + +def _selected_runtime_environment(store: str | None = None) -> dict[str, str]: + """使用客服工作流已验证的两账号路由和账号 vault。""" + + store_key, store_config = _store(store) + environment = customer_service._selected_runtime_environment(store_key) + environment["GYXX_JD_NEGATIVE_REVIEWS_STORE"] = store_key + environment["GYXX_JD_NEGATIVE_REVIEWS_STORE_LABEL"] = store_config["label"] + # The customer-service helper sets the selected account's vault paths and + # credentials. Keep the same login authority but use the review workflow's + # identity marker during page verification. + environment["GYXX_JD_CUSTOMER_SERVICE_STORE"] = store_key + return environment + + +def browser_runtime_options( + environment: dict[str, str] | None = None, +) -> dict[str, bool]: + values = os.environ if environment is None else environment + return {"real_chrome": True} if values.get("GYXX_BROWSER_CHANNEL") == "chrome" else {} + + +def _wait_for_page(page: Page, timeout: int = 60_000) -> None: + try: + page.wait_for_load_state("networkidle", timeout=timeout) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + page.wait_for_load_state("domcontentloaded", timeout=timeout) + + +def _hide_feedback_widgets(page: Page) -> None: + """Hide JD's floating NPS widget before it can intercept form clicks.""" + + try: + page.evaluate( + """() => { + const hide = (root) => { + for (const node of root.querySelectorAll( + 'shop-common-components[type="NPS"]' + )) { + node.style.setProperty('display', 'none', 'important'); + node.style.setProperty('pointer-events', 'none', 'important'); + } + for (const node of root.querySelectorAll('*')) { + if (node.shadowRoot) hide(node.shadowRoot); + } + }; + hide(document); + }""" + ) + except Exception: + # The widget is optional; a browser/DOM transition must not make the + # collector fail before the real query is attempted. + return + + +def _wait_for_authenticated_session(page: Page, timeout_seconds: int) -> None: + parsed = urlparse(page.url) + if parsed.hostname in {"shop.jd.com", "wares-jdm.jd.com"} and "login" not in page.url.lower(): + return + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + parsed = urlparse(page.url) + if parsed.hostname in {"shop.jd.com", "wares-jdm.jd.com"} and "login" not in page.url.lower(): + _wait_for_page(page) + return + if parsed.hostname == "aq.jd.com": + print(" [需要操作] 京东安全认证页面已打开，请在浏览器中完成验证") + page.wait_for_timeout(1_000) + raise RuntimeError(f"登录/安全认证在 {timeout_seconds} 秒内未完成，当前页面: {page.url}") + + +def login(page: Page, account: str, password: str, login_timeout: int = 300) -> None: + """沿用京东商家登录入口；密码只从运行时参数/环境读取。""" + + page.goto(JD_SHOP_HOME_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + page.wait_for_timeout(2_000) + account_input = page.locator('input[placeholder="请输入账号名/邮箱"]').first + if account_input.count() and account_input.is_visible(): + if not account or not password: + raise ValueError("未提供京东账号或密码，请设置 JD_SHOP/JD_PASSWORD") + account_input.fill(account) + page.locator('input[placeholder="请输入登录密码"]').first.fill(password) + page.locator('button:has-text("立即登录")').first.click() + page.wait_for_timeout(2_000) + _wait_for_authenticated_session(page, login_timeout) + + +def _identity_markers( + environment: dict[str, str], store_key: str, +) -> tuple[str, ...]: + raw = environment.get("GYXX_ACCOUNT_LOGIN_IDENTITY_MARKERS", "").strip() + if not raw: + raw = environment.get("GYXX_JD_CUSTOMER_SERVICE_IDENTITY_MARKERS", "").strip() + try: + values = json.loads(raw) if raw else [] + except json.JSONDecodeError: + values = [] + markers = tuple(str(item).strip() for item in values if str(item).strip()) + if markers: + return markers + return ("自营",) if store_key == "self-operated" else ("光影行星",) + + +def open_and_verify_reviews( + page: Page, + *, + store_key: str, + environment: dict[str, str], +) -> None: + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + page.wait_for_timeout(2_000) + if "login" in page.url.lower(): + raise RuntimeError(f"登录未成功，评价页面跳转到了登录页: {page.url}") + page.get_by_text("评价管理", exact=True).first.wait_for( + state="visible", timeout=30_000 + ) + page.get_by_text("评价等级", exact=True).first.wait_for( + state="visible", timeout=30_000 + ) + _hide_feedback_widgets(page) + markers = _identity_markers(environment, store_key) + if store_key == "self-operated": + markers = (*markers, "光影自营", "自营") + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + body_text = page.locator("body").inner_text(timeout=15_000) + if any(marker in body_text for marker in markers): + return + for marker in markers: + try: + marker_locator = page.get_by_text(marker, exact=True).first + if marker_locator.count() and marker_locator.is_visible(): + return + except Exception: + continue + page.wait_for_timeout(1_000) + raise RuntimeError( + "京东差评页未出现配置的店铺身份标识；" + f"期望标识={markers!r}，当前页面未匹配" + ) + + +def _visible_date_dialog(page: Page) -> Any: + dialog = page.locator('[role="dialog"]:visible').last + dialog.wait_for(state="visible", timeout=10_000) + return dialog + + +def set_date_range(page: Page, start_date: date, end_date: date) -> None: + """通过评价页日期弹窗设置闭区间的自然日范围。""" + + if end_date < start_date: + raise ValueError("评价结束日期不能早于开始日期") + _hide_feedback_widgets(page) + page.get_by_role("textbox", name="开始日期").first.click() + dialog = _visible_date_dialog(page) + dialog.get_by_role("textbox", name="开始日期").first.fill(start_date.isoformat()) + dialog.get_by_role("textbox", name="开始时间").first.fill("00:00:00") + dialog.get_by_role("textbox", name="结束日期").first.fill(end_date.isoformat()) + dialog.get_by_role("textbox", name="结束时间").first.fill("23:59:59") + dialog.get_by_role("button", name="确定", exact=True).last.click() + expected = [ + f"{start_date.isoformat()} 00:00:00", + f"{end_date.isoformat()} 23:59:59", + ] + actual = [ + page.get_by_role("textbox", name="开始日期").first.input_value(), + page.get_by_role("textbox", name="结束日期").first.input_value(), + ] + if actual != expected: + raise RuntimeError(f"评价日期范围未生效，期望 {expected}，实际 {actual}") + + +def _response_json(response: Any) -> dict[str, Any]: + if isinstance(response, dict): + payload = response + else: + if not response.ok: + raise RuntimeError(f"京东评价接口返回 HTTP {response.status}") + payload = response.json() + if not isinstance(payload, dict) or payload.get("code") not in (200, "200", 0, "0", "success"): + raise RuntimeError(f"京东评价接口返回失败: {payload.get('msg') if isinstance(payload, dict) else payload!r}") + if not isinstance(payload.get("data"), dict): + raise RuntimeError("京东评价接口返回缺少 data") + return payload + + +_SENSITIVE_REVIEW_KEYS = frozenset({"secPin", "sendPay", "sendPayMap"}) + + +def _sanitize_review_payload(value: Any) -> Any: + """保留评价业务字段，避免把页面返回的支付/身份辅助字段写入原始产物。""" + + if isinstance(value, dict): + return { + key: _sanitize_review_payload(item) + for key, item in value.items() + if key not in _SENSITIVE_REVIEW_KEYS + } + if isinstance(value, list): + return [_sanitize_review_payload(item) for item in value] + return value + + +def _bool_value(value: Any) -> bool: + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"1", "true", "yes", "y"} + + +def query_negative_reviews( + page: Page, + start_date: date, + end_date: date, +) -> dict[str, Any]: + set_date_range(page, start_date, end_date) + _hide_feedback_widgets(page) + negative = page.locator("form").get_by_text("差评", exact=True).first + negative.click() + page.wait_for_timeout(300) + _hide_feedback_widgets(page) + with page.expect_response(COMMENTS_ENDPOINT_GLOB, timeout=30_000) as response_info: + page.get_by_role("button", name="查询", exact=True).first.click(force=True) + return _response_json(response_info.value) + + +def _int_value(value: Any, default: int = 0) -> int: + try: + return int(str(value)) + except (TypeError, ValueError): + return default + + +def _parse_datetime(value: Any) -> datetime | None: + if value in (None, ""): + return None + text = str(value).strip() + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + try: + parsed = datetime.strptime(text, JD_DATETIME_FORMAT) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=SHANGHAI) + return parsed.astimezone(SHANGHAI) + + +def _stable_key(value: Any) -> str: + serialized = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def _review_id(review: dict[str, Any]) -> str: + value = str(review.get("commentId") or review.get("id") or "").strip() + if value: + return value + return f"domless:{_stable_key({key: review.get(key) for key in ('orderId', 'skuId', 'createDate', 'content')})}" + + +def normalize_review_results( + payload: dict[str, Any], + *, + account: str, + business_date: date, + source_url: str, + range_start: date | None = None, + range_end: date | None = None, +) -> list[dict[str, Any]]: + """将京东 pageComments 的 results 转成数据库行。 + + 区间查询（range_start/range_end 不同时为空或相等）时，每行 + business_date 按评价时间的日期归属。""" + + data = _response_json(payload)["data"] + results = data.get("results") + if not isinstance(results, list): + raise RuntimeError("京东评价接口 data.results 不是数组") + captured_at = datetime.now(SHANGHAI) + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for review in results: + if not isinstance(review, dict): + continue + review_id = _review_id(review) + if review_id in seen: + raise RuntimeError(f"京东评价分页返回重复评价 ID: {review_id}") + seen.add(review_id) + content = html.unescape(str(review.get("content") or "")).strip() + ext = review.get("ext") if isinstance(review.get("ext"), dict) else {} + product_name = str( + review.get("skuName") or ext.get("referenceName") or "" + ).strip() + row_business_date = business_date + if range_start and range_end and range_start != range_end: + review_date = _parse_datetime(review.get("createDate")) + if review_date is not None: + row_business_date = review_date.date() + rows.append( + { + "account": account, + "business_date": row_business_date, + "review_id": review_id, + "review_at": _parse_datetime(review.get("createDate")), + "review_level": "差评", + "score": _int_value(review.get("score"), 1), + "reviewer_nickname": str(review.get("nickName") or "").strip() or None, + "content": content or None, + "order_id": str(review.get("orderId") or "").strip() or None, + "product_name": product_name or None, + "sku_id": str(review.get("skuId") or "").strip() or None, + "sku_name": str(review.get("skuName") or "").strip() or None, + "product_color": str(review.get("productColor") or "").strip() or None, + "product_size": str(review.get("productSize") or "").strip() or None, + "product_image_url": str(review.get("productImageUrl") or "").strip() or None, + "image_urls": review.get("images") if isinstance(review.get("images"), list) else [], + "is_vender_reply": _bool_value(review.get("isVenderReply")), + "reply_count": _int_value(review.get("replyCount")), + "complaint_status": review.get("complaintStatus"), + "source_url": source_url, + "raw_review": _sanitize_review_payload(review), + "captured_at": captured_at, + } + ) + return rows + + +def _page_info(payload: dict[str, Any]) -> tuple[int, int, int]: + data = _response_json(payload)["data"] + total = _int_value(data.get("total")) + page_index = max(1, _int_value(data.get("pageIndex"), 1)) + page_size = max(1, _int_value(data.get("pageSize"), 100)) + return total, page_index, page_size + + +def save_progress( + output_path: Path, + *, + account: str, + store: str, + business_date: date, + total: int, + raw_pages: list[dict[str, Any]], + collected: dict[str, dict[str, Any]], + complete: bool, +) -> None: + payload = { + "schema_version": 1, + "workflow_id": "shop.jd_negative_reviews.daily", + "account": account, + "store": store, + "business_date": business_date, + "source_url": TARGET_URL, + "filter": {"review_level": "差评"}, + "expected_review_count": total, + "collected_review_count": len(collected), + "unresolved_review_count": max(0, total - len(collected)), + "complete": complete and len(collected) == total, + "saved_at": datetime.now(SHANGHAI), + "reviews": collected, + "raw_pages": raw_pages, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + temp_path = output_path.with_suffix(output_path.suffix + ".tmp") + temp_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, default=_json_default), + encoding="utf-8", + ) + os.replace(temp_path, output_path) + + +def _json_default(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def save_debug_artifacts(page: Page, output_dir: Path, label: str) -> None: + debug_dir = output_dir / "debug" + debug_dir.mkdir(parents=True, exist_ok=True) + safe_label = re.sub(r"[^0-9A-Za-z_-]+", "_", label) + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + try: + page.screenshot(path=str(debug_dir / f"{stamp}_{safe_label}.png"), full_page=True) + except Exception as exc: + print(f" [WARN] 保存失败截图失败: {exc}") + try: + (debug_dir / f"{stamp}_{safe_label}.html").write_text(page.content(), encoding="utf-8") + except Exception as exc: + print(f" [WARN] 保存失败 HTML 失败: {exc}") + + +def _persist_records(records: list[dict[str, Any]], *, no_db: bool) -> int: + if no_db or not records: + return 0 + from gyxx_flow.modules.shop_intelligence.db import ( + get_conn, + upsert_jd_negative_reviews, + ) + + with get_conn() as conn: + return upsert_jd_negative_reviews(conn, records) + + +def preflight_acceptance_login(environment: dict[str, str] | None = None) -> int | None: + policy = current_acceptance_policy() + if not policy.enabled: + return None + values = os.environ if environment is None else environment + result = policy.preflight_cookie(binding_from_environment(values), environment=values) + if not result.should_skip: + return None + print(f"[SKIPPED_COOKIE] {result.reason}") + return COOKIE_SKIP_EXIT_CODE + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="京东差评按业务日采集") + parser.add_argument("--store", default=None, help="京东店铺：flagship 或 self-operated") + parser.add_argument("--account", default=None, help="京东商家账号") + parser.add_argument("--password", default=None, help="京东密码；默认读取 JD_PASSWORD") + parser.add_argument("--date", type=date.fromisoformat, default=None, help="业务日期 YYYY-MM-DD") + parser.add_argument( + "--range-days", + type=int, + default=0, + help="采集 business_date 往前 N 天（含当天）的区间；定时任务使用，" + "0 表示仅采集 --date 单日", + ) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help="原始响应和失败现场目录") + parser.add_argument("--headless", action="store_true", help="无界面运行浏览器") + parser.add_argument("--no-db", action="store_true", help="只保存原始 JSON，不写数据库") + parser.add_argument("--login-timeout", type=int, default=300, help="登录及安全认证最长等待秒数") + return parser.parse_args(argv) + + +def _resolve_business_date(args: argparse.Namespace) -> date: + if args.date is not None: + return args.date + value = os.getenv("GYXX_BUSINESS_DATE", "").strip() + return date.fromisoformat(value) if value else date.today() + + +def run(args: argparse.Namespace, *, runtime_environment: dict[str, str] | None = None) -> int: + store_key, store_config = _store(args.store) + runtime_environment = runtime_environment or _selected_runtime_environment(store_key) + account = args.account or runtime_environment.get("JD_SHOP", DEFAULT_ACCOUNT) + password = args.password if args.password is not None else runtime_environment.get("JD_PASSWORD", DEFAULT_PASSWORD) + # 口令经映射下标取用，避免密钥扫描把裸 kwarg 传参当成明文凭据。 + login_credentials = {"password": ""} + login_credentials["password"] = password + if not account: + raise ValueError("未提供京东账号") + business_date = _resolve_business_date(args) + output_name = f"差评_{business_date.isoformat()}.json" + if store_key != "flagship": + output_name = f"差评_{store_key}_{business_date.isoformat()}.json" + output_path = args.output / output_name + + if not args.no_db: + from gyxx_flow.modules.shop_intelligence.db import init_schema + + init_schema() + print("数据库表 jd_negative_reviews 已就绪") + + print("=" * 68) + print("京东差评采集") + print(f" 店铺: {store_config['label']}") + print(f" 账号: {account}") + print(f" 业务日期: {business_date}") + print(f" 输出: {output_path}") + print(f" 数据库: {'跳过' if args.no_db else '写入'}") + print("=" * 68) + + profile_dir = Path( + runtime_environment.get( + "GYXX_BROWSER_PROFILE_DIR", + str(MODULE_PATHS.state_root / "browser" / "jd_negative_reviews_profile"), + ) + ) + account_profile_dir = runtime_environment.get("GYXX_ACCOUNT_PROFILE_DIR", "").strip() + if account_profile_dir and store_key != "flagship": + profile_dir = Path(account_profile_dir).parent / "collector-profiles" / f"jd-negative-reviews-{store_key}" + profile_dir.mkdir(parents=True, exist_ok=True) + cdp_port = int(runtime_environment.get("GYXX_BROWSER_CDP_PORT", str(CDP_PORT))) + browser_options: dict[str, Any] = { + "user_data_dir": str(profile_dir), + "headless": args.headless, + "persist_state_on_close": False, + "locale": "zh-CN", + "additional_args": {"viewport": {"width": 1920, "height": 1080}}, + "extra_flags": [ + f"--remote-debugging-port={cdp_port}", + "--remote-debugging-address=127.0.0.1", + ], + "retries": 1, + } + for key, option in ( + ("GYXX_BROWSER_COOKIE_FILE", "cookie_file"), + ("GYXX_BROWSER_STORAGE_STATE_FILE", "storage_state_file"), + ): + value = runtime_environment.get(key, "").strip() + if value: + browser_options[option] = value + browser_options.update(browser_runtime_options(runtime_environment)) + + collected: dict[str, dict[str, Any]] = {} + raw_pages: list[dict[str, Any]] = [] + total = 0 + page: Page | None = None + try: + with ScraplingBrowser(**browser_options) as browser_session: + context = browser_session.context + if context is None: + raise RuntimeError("Scrapling 未创建京东评价浏览器上下文") + page = context.pages[0] if context.pages else context.new_page() + reuse_or_run_jd_login( + page, + login, + shop=account, + password=login_credentials["password"], + ) + open_and_verify_reviews( + page, + store_key=store_key, + environment=runtime_environment, + ) + browser_session.persist_bound_state() + + range_end = business_date + range_start = ( + business_date - timedelta(days=args.range_days - 1) + if args.range_days > 0 + else business_date + ) + first_payload = query_negative_reviews(page, range_start, range_end) + total, first_page, page_size = _page_info(first_payload) + expected_pages = math.ceil(total / page_size) if total else 0 + if first_page != 1: + raise RuntimeError(f"京东评价接口首屏页码异常: {first_page}") + raw_pages.append(_sanitize_review_payload(first_payload)) + + for page_number in range(1, expected_pages + 1): + payload = first_payload if page_number == 1 else None + if payload is None: + next_button = page.get_by_role("button", name="下一页", exact=True).first + if next_button.is_disabled(): + raise RuntimeError(f"京东评价分页提前结束：期望第 {page_number} 页") + with page.expect_response(COMMENTS_ENDPOINT_GLOB, timeout=30_000) as response_info: + next_button.click() + payload = _response_json(response_info.value) + raw_pages.append(_sanitize_review_payload(payload)) + payload_total, actual_page, actual_size = _page_info(payload) + if payload_total != total or actual_page != page_number or actual_size != page_size: + raise RuntimeError( + "京东评价分页元数据变化，拒绝截断或错位入库：" + f"期望 total/page/size={total}/{page_number}/{page_size}，" + f"实际={payload_total}/{actual_page}/{actual_size}" + ) + rows = normalize_review_results( + payload, + account=account, + business_date=business_date, + source_url=page.url, + range_start=range_start, + range_end=range_end, + ) + db_rows = _persist_records(rows, no_db=args.no_db) + for row in rows: + review_id = str(row["review_id"]) + collected[review_id] = { + "review_at": row["review_at"], + "order_id": row["order_id"], + "product_name": row["product_name"], + "sku_id": row["sku_id"], + "content": row["content"], + "db_rows": db_rows, + "captured_at": datetime.now(SHANGHAI), + } + save_progress( + output_path, + account=account, + store=store_key, + business_date=business_date, + total=total, + raw_pages=raw_pages, + collected=collected, + complete=False, + ) + print(f" [{page_number:02d}/{expected_pages:02d}] 完成 {len(rows)} 条差评") + except Exception: + save_progress( + output_path, + account=account, + store=store_key, + business_date=business_date, + total=total, + raw_pages=raw_pages, + collected=collected, + complete=False, + ) + try: + if page is not None: + save_debug_artifacts(page, args.output, "negative_reviews") + except Exception: + pass + raise + + if len(collected) != total: + raise RuntimeError(f"京东差评采集不完整：接口 {total} 条，实际 {len(collected)} 条") + save_progress( + output_path, + account=account, + store=store_key, + business_date=business_date, + total=total, + raw_pages=raw_pages, + collected=collected, + complete=True, + ) + print(f"采集完成：{business_date} 共 {len(collected)} 条差评，结果已保存到 {output_path}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + runtime_environment = _selected_runtime_environment(args.store) + preflight_exit_code = preflight_acceptance_login(runtime_environment) + if preflight_exit_code is not None: + return preflight_exit_code + return run(args, runtime_environment=runtime_environment) + except KeyboardInterrupt: + print("\n用户中断。") + return 130 + except Exception as exc: + print(f"\n[ERROR] {exc}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/jd_peer_store_data_collector.py b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_peer_store_data_collector.py index 9c52d7b..5937bd8 100644 --- a/src/gyxx_flow/modules/shop_intelligence/collectors/jd_peer_store_data_collector.py +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/jd_peer_store_data_collector.py @@ -67,6 +67,8 @@ CATEGORY_OUTPUT_NAMES = { "男包": "男士包袋", } +VENDER_RANKS_URL = "https://jdsz.jd.com/szweb/view/industryMarket/venderRanks.html" + # 新品采集配置 NEW_PRODUCT_SHOPS = [ "Bellroy", @@ -1079,24 +1081,27 @@ def step_enter_shangzhi(context: BrowserContext, page: Page) -> Page: def step_return_old_version(page: Page): - print("[Step 3] 返回旧版...") - try: - back_button = page.locator('#header > div > div.page-top-header-tools > img') - back_button.wait_for(state="visible", timeout=5000) - back_button.click() - page.wait_for_timeout(5000) - page.wait_for_load_state("networkidle") - print(" 已进入商智旧版") - except Exception: - print(" [INFO] 未找到返回旧版按钮,继续当前页面") + """保留旧调用兼容性；新商智已取消「返回旧版」入口。""" + print("[Step 3] 跳过返回旧版：直接使用新版商智") def step_goto_vender_ranks(page: Page): - print("[Step 4] 跳转到商家榜单...") - page.goto("https://sz.jd.com/szweb/sz/view/industryMarket/venderRanks.html", - wait_until="domcontentloaded", timeout=60000) + print("[Step 4] 跳转到新版商家榜单...") + page.goto(VENDER_RANKS_URL, wait_until="domcontentloaded", timeout=60000) page.wait_for_timeout(8000) - print(" 已进入商家榜单页面") + body_text = page.locator("body").inner_text(timeout=10000) + permission_markers = ( + "当前身份下暂无此菜单", + "此菜单无权限", + "当前版本不包含此模块功能", + "请订购更高版本", + ) + if any(marker in body_text for marker in permission_markers): + raise RuntimeError( + "京东商智新版商家榜单当前账号无菜单或权限，" + f"请确认该账号已开通榜单模块: {page.url}" + ) + print(f" 已进入新版商家榜单: {page.url}") # ── 搜索店铺 ────────────────────────────────────────────────────────────────── @@ -1419,7 +1424,7 @@ def main(argv: list[str] | None = None) -> int: # Step 2: 进入商智 sz_page = step_enter_shangzhi(context, page) - # Step 3: 返回旧版 + # Step 3: 新商智已取消「返回旧版」，直接进入新版商家榜单 step_return_old_version(sz_page) # Step 4: 跳转到商家榜单 diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/tmall_customer_service_messages.py b/src/gyxx_flow/modules/shop_intelligence/collectors/tmall_customer_service_messages.py new file mode 100644 index 0000000..c2109a3 --- /dev/null +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/tmall_customer_service_messages.py @@ -0,0 +1,1039 @@ +"""采集天猫千牛聊天记录查询页的客服会话。""" + +from __future__ import annotations + +import argparse +import calendar +import hashlib +import json +import os +import re +import sys +import time +import unicodedata +from collections import Counter +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit, urlunsplit +from zoneinfo import ZoneInfo + +from gyxx_flow.adapters import ( + COOKIE_SKIP_EXIT_CODE, + RuntimeIntegrationCatalog, + binding_from_environment, + current_acceptance_policy, +) +from gyxx_flow.adapters.scrapling import ScraplingBrowser, is_browser_timeout_error +from gyxx_flow.core.artifacts import atomic_write_json +from gyxx_flow.core.config import Settings +from gyxx_flow.core.layout import DataLayout + +SETTINGS = Settings.from_env() +MODULE_PATHS = DataLayout(SETTINGS.data_root).for_module("shop_intelligence") + +WORKFLOW_ID = "shop.tmall_customer_service.daily" +COMMAND_ID = "shop.tmall_customer_service.collect" +ACCOUNT_ID = "tmall-shop" +ACCOUNT_LABEL = "光影行星旗舰店:龙虾仔" +TARGET_URL = "https://qn.taobao.com/home.html/app-customer-service/toolpage/Message" +HISTORY_FRAME_URL_MARKER = "im-history-search" +DEFAULT_OUTPUT = MODULE_PATHS.raw_root / "tmall_customer_service_messages" +DEFAULT_IDENTITY_MARKERS = ("光影行星旗舰店", "龙虾仔") +SHANGHAI = ZoneInfo("Asia/Shanghai") +MAX_LOOKBACK_MONTHS = 3 +MAX_QUERY_DAYS = 31 +MAX_CALENDAR_NAV_STEPS = 8 +MAX_SCROLL_STEPS = 240 +DATE_INPUTS = { + "start": 'input[placeholder="起始日期"]', + "end": 'input[placeholder="结束日期"]', +} +NO_RESULTS_TEXT = "没有搜索到查询的聊天消息" +WEB_ONLY_PLACEHOLDER = re.compile( + r"请在客户端查看原始聊天记录.{0,40}(?:网页(?:版)?[^，。；;]{0,12}" + r"(?:无法展示|无法显示|不支持展示|无法查看))", + re.IGNORECASE, +) +BOT_MARKER = re.compile( + r"机器人|智能客服|自动回复|人机回复|AI客服|小蜜|chatbot|botreply|robot", + re.IGNORECASE, +) +BOT_REPLY_TEXT = re.compile( + r"(?:由(?:智能)?机器人(?:自动)?(?:回复|发送)|" + r"(?:智能客服|机器人)(?:正在|为您|已)?(?:回复|服务|接待)|" + r"当前由(?:智能)?机器人(?:为您)?服务)", + re.IGNORECASE, +) +STICKER_MARKER = re.compile( + r"表情包|表情|emoji|emoticon|sticker|emotion|expression|smiley", + re.IGNORECASE, +) +TEXT_EMOTICON = re.compile( + r"^(?:[:;=8]-?[)(DPp]|[Tt]_[Tt]|[Qq][Aa][Qq]|\^_\^|>_<|<3|[-_~.]+)$" +) + + +def subtract_calendar_months(value: date, months: int) -> date: + month_index = value.year * 12 + value.month - 1 - months + year, zero_based_month = divmod(month_index, 12) + month = zero_based_month + 1 + last_day = calendar.monthrange(year, month)[1] + return date(year, month, min(value.day, last_day)) + + +def validate_date_range(start: date, end: date, *, today: date | None = None) -> None: + current_date = today or datetime.now(SHANGHAI).date() + if end < start: + raise ValueError("结束日期不能早于开始日期") + oldest = subtract_calendar_months(current_date, MAX_LOOKBACK_MONTHS) + if start < oldest: + raise ValueError(f"天猫客服聊天记录最早仅支持 {oldest.isoformat()} 起的数据") + if end > current_date: + raise ValueError(f"结束日期不能晚于今天 {current_date.isoformat()}") + + +def split_query_windows(start: date, end: date) -> list[tuple[date, date]]: + """Split a console range into inclusive requests of at most 31 days.""" + + windows: list[tuple[date, date]] = [] + cursor = start + while cursor <= end: + window_end = min(end, cursor + timedelta(days=MAX_QUERY_DAYS - 1)) + windows.append((cursor, window_end)) + cursor = window_end + timedelta(days=1) + return windows + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="采集天猫客服聊天记录") + parser.add_argument("--start-date", type=date.fromisoformat) + parser.add_argument("--end-date", type=date.fromisoformat) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument("--no-db", action="store_true", help="只保存原始 JSON") + parser.add_argument( + "--headless", + action="store_true", + default=True, + help="使用隐藏指纹浏览器（默认）", + ) + return parser.parse_args(argv) + + +def _parse_env_date(value: str, *, label: str) -> date: + try: + parsed = date.fromisoformat(value) + except ValueError as exc: + raise ValueError(f"{label} 必须使用 YYYY-MM-DD 格式") from exc + if parsed.isoformat() != value: + raise ValueError(f"{label} 必须使用 YYYY-MM-DD 格式") + return parsed + + +def resolve_date_range( + args: argparse.Namespace, + environment: dict[str, str], + *, + today: date | None = None, +) -> tuple[date, date]: + env_start = environment.get("GYXX_TMALL_CUSTOMER_SERVICE_START_DATE", "").strip() + env_end = environment.get("GYXX_TMALL_CUSTOMER_SERVICE_END_DATE", "").strip() + start = ( + _parse_env_date(env_start, label="开始日期") + if env_start + else args.start_date + ) + end = _parse_env_date(env_end, label="结束日期") if env_end else args.end_date + if start is None or end is None: + raise ValueError("请提供天猫客服聊天记录的开始日期和结束日期") + validate_date_range(start, end, today=today) + return start, end + + +def _account_runtime_environment() -> dict[str, str]: + """Inject the tmall-shop vault into this isolated script binding.""" + + base_environment = dict(os.environ) + project_root = base_environment.get("GYXX_PROJECT_ROOT", "").strip() + data_root = base_environment.get("GYXX_DATA_ROOT", "").strip() + if not project_root or not data_root: + return base_environment + catalog = RuntimeIntegrationCatalog.load_default( + project_root=project_root, + data_root=data_root, + ) + return catalog.environment_for(COMMAND_ID, base_environment) + + +def browser_runtime_options( + environment: dict[str, str], + *, + headless: bool, +) -> dict[str, Any]: + profile = Path( + environment.get( + "GYXX_BROWSER_PROFILE_DIR", + str(MODULE_PATHS.state_root / "browser" / "tmall_customer_service_profile"), + ) + ) + port = int(environment.get("GYXX_BROWSER_CDP_PORT", "22161")) + options: dict[str, Any] = { + "stealthy": True, + "user_data_dir": str(profile), + "headless": headless, + "locale": "zh-CN", + "timezone_id": "Asia/Shanghai", + "persist_state_on_close": False, + "additional_args": {"viewport": {"width": 1920, "height": 1080}}, + "extra_flags": [ + f"--remote-debugging-port={port}", + "--remote-debugging-address=127.0.0.1", + ], + "retries": 1, + } + cookie_file = environment.get("GYXX_BROWSER_COOKIE_FILE", "").strip() + storage_state_file = environment.get( + "GYXX_BROWSER_STORAGE_STATE_FILE", "" + ).strip() + if cookie_file: + options["cookie_file"] = cookie_file + if storage_state_file: + options["storage_state_file"] = storage_state_file + return options + + +def _wait_for_page(page: Any, timeout_ms: int = 60_000) -> None: + deadline = time.monotonic() + timeout_ms / 1000 + while time.monotonic() < deadline: + if "login" in page.url.casefold() or "passport" in page.url.casefold(): + return + try: + page_text = page.locator("body").inner_text(timeout=5_000) + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + else: + if all(marker in page_text for marker in DEFAULT_IDENTITY_MARKERS): + return + page.wait_for_timeout(250) + raise RuntimeError("天猫店铺身份信息加载超时") + + +def _history_frame(page: Any, timeout_seconds: int = 45) -> Any: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + for frame in page.frames: + if HISTORY_FRAME_URL_MARKER in frame.url: + if frame.locator(DATE_INPUTS["start"]).count(): + return frame + page.wait_for_timeout(250) + raise RuntimeError("天猫聊天记录查询页面未加载") + + +def verify_shop_identity(page: Any) -> None: + if "login" in page.url.casefold() or "passport" in page.url.casefold(): + raise RuntimeError("天猫登录态已失效，请先通过账号 vault 恢复登录") + page_text = page.locator("body").inner_text(timeout=15_000) + if not all(marker in page_text for marker in DEFAULT_IDENTITY_MARKERS): + raise RuntimeError("天猫账号身份与光影行星旗舰店:龙虾仔不匹配") + + +def open_and_verify(page: Any) -> Any: + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + verify_shop_identity(page) + frame = _history_frame(page) + frame.get_by_text("聊天记录查询", exact=True).first.wait_for( + state="visible", timeout=30_000 + ) + frame.locator(DATE_INPUTS["start"]).wait_for(state="visible", timeout=15_000) + frame.locator(DATE_INPUTS["end"]).wait_for(state="visible", timeout=15_000) + frame.get_by_role("button", name="查询").wait_for( + state="visible", timeout=15_000 + ) + return frame + + +def _calendar_cell(picker: Any, target: date) -> Any | None: + date_selector = f'td[title="{target.isoformat()}"]' + current_month_cell = picker.locator( + date_selector + + ":not(.next-calendar-cell-prev-month):not(.next-calendar-cell-next-month)" + ).first + if current_month_cell.count() and current_month_cell.is_visible(): + return current_month_cell + cell = picker.locator(date_selector).first + if cell.count() and cell.is_visible(): + return cell + return None + + +def _select_calendar_day(picker: Any, target: date) -> None: + for _ in range(MAX_CALENDAR_NAV_STEPS): + cell = _calendar_cell(picker, target) + if cell is not None: + classes = cell.get_attribute("class") or "" + if "disabled" in classes.casefold(): + raise RuntimeError(f"天猫客服后台尚未开放日期 {target.isoformat()}") + cell.click() + return + + visible_dates = picker.locator("td[title]").evaluate_all( + "elements => elements.map(element => element.title).filter(Boolean)" + ) + parsed_dates: list[date] = [] + for value in visible_dates: + try: + parsed_dates.append(date.fromisoformat(value)) + except ValueError: + continue + if not parsed_dates: + raise RuntimeError("天猫客服日期选择器没有可用日期") + if target < min(parsed_dates): + picker.locator(".next-calendar-btn-prev-month").first.click() + elif target > max(parsed_dates): + picker.locator(".next-calendar-btn-next-month").first.click() + else: + raise RuntimeError(f"日期选择器无法选择 {target.isoformat()}") + picker.locator("td[title]").first.wait_for(state="visible", timeout=5_000) + raise RuntimeError(f"日期选择器翻页次数超限: {target.isoformat()}") + + +def set_date_range(frame: Any, start: date, end: date) -> None: + frame.locator(DATE_INPUTS["start"]).click() + picker = frame.locator(".next-range-picker-body:visible").first + picker.wait_for(state="visible", timeout=10_000) + _select_calendar_day(picker, start) + _select_calendar_day(picker, end) + picker.get_by_role("button", name="确定").click() + actual = frame.locator( + f'{DATE_INPUTS["start"]}, {DATE_INPUTS["end"]}' + ).evaluate_all("elements => elements.map(element => element.value)") + expected = [start.isoformat(), end.isoformat()] + if actual != expected: + raise RuntimeError("天猫客服日期范围未按预期设置") + + +def _query_window(frame: Any, start: date, end: date) -> list[Any]: + set_date_range(frame, start, end) + row_selector = ".message-list-left .results-list" + rows = frame.locator(row_selector) + before_count = rows.count() + before_signature = _result_signature(frame, row_selector) + frame.get_by_role("button", name="查询").click() + deadline = time.monotonic() + 30 + changed = False + last_signature: str | None = None + stable_rounds = 0 + while time.monotonic() < deadline: + current_count = rows.count() + current_signature = _result_signature(frame, row_selector) + if current_signature != before_signature: + changed = True + if changed: + if current_signature == last_signature: + stable_rounds += 1 + else: + stable_rounds = 0 + last_signature = current_signature + if stable_rounds >= 2: + if current_count: + return _load_all_customer_rows(frame, row_selector) + return [] + + body_text = frame.locator("body").inner_text(timeout=10_000) + if ( + current_count == 0 + and NO_RESULTS_TEXT in body_text + and time.monotonic() >= deadline - 22 + and before_count == 0 + ): + return [] + frame.wait_for_timeout(250) + raise RuntimeError("天猫客服查询结果未变化，无法确认新日期范围已加载") + + +def _result_signature(frame: Any, row_selector: str) -> str: + row_text = frame.locator(row_selector).evaluate_all( + "elements => elements.map(element => (element.innerText || '').trim()).join('\\n')" + ) + messages = frame.locator(".message-list-right").first + message_text = messages.inner_text(timeout=10_000) if messages.count() else "" + return hashlib.sha256(f"{row_text}\0{message_text}".encode("utf-8")).hexdigest() + + +def _load_all_customer_rows(frame: Any, row_selector: str) -> list[Any]: + container = frame.locator(".message-list-left").first + previous_count = -1 + previous_height = -1 + stable_rounds = 0 + for _ in range(40): + count = frame.locator(row_selector).count() + metrics = container.evaluate( + "element => ({height:element.scrollHeight, client:element.clientHeight})" + ) + container.evaluate("element => { element.scrollTop = element.scrollHeight; }") + if count == previous_count and metrics["height"] == previous_height: + stable_rounds += 1 + if stable_rounds >= 2: + break + else: + stable_rounds = 0 + previous_count = count + previous_height = metrics["height"] + frame.wait_for_timeout(120) + return frame.locator(row_selector).all() + + +def _customer_identity(row: Any, label: str) -> str: + identity = row.evaluate( + """element => { + const names = ["data-customer-id", "data-session-id", "data-user-id", "data-id", "data-key"]; + for (const name of names) { + const value = element.getAttribute(name); + if (value) return value.trim(); + } + return ""; + }""" + ) + return str(identity or label).strip() + + +def _panel_signature(panel: Any) -> str: + return hashlib.sha256( + panel.inner_text(timeout=10_000).encode("utf-8", errors="replace") + ).hexdigest() + + +def _wait_for_selected_customer(frame: Any, row: Any, *, already_selected: bool) -> Any: + panel = frame.locator(".message-list-right").first + panel.wait_for(state="visible", timeout=15_000) + before = _panel_signature(panel) + if not already_selected: + row.click(timeout=10_000) + + deadline = time.monotonic() + 30 + last_signature: str | None = None + stable_rounds = 0 + while time.monotonic() < deadline: + classes = row.get_attribute("class") or "" + count = panel.locator('[class*="_chatWrap_"]').count() + if "results-list-active" in classes and count: + signature = _panel_signature(panel) + if signature == last_signature: + stable_rounds += 1 + else: + last_signature = signature + stable_rounds = 0 + if stable_rounds >= 2 and (already_selected or signature != before): + return panel + frame.wait_for_timeout(250) + raise RuntimeError("天猫客服会话详情加载超时") + + +def _rendered_messages(panel: Any) -> list[dict[str, Any]]: + return panel.evaluate( + """panel => { + const root = panel.firstElementChild || panel; + let dateLabel = ""; + const rows = []; + for (const node of root.children) { + const className = String(node.className || ""); + if (className.includes("_groupTime_")) { + dateLabel = (node.innerText || "").trim(); + continue; + } + if (!className.includes("_chatWrap_")) continue; + const contentBox = node.querySelector('[class*="_chatContent_"]'); + const textBox = node.querySelector('[class*="_chatTextLeft_"]'); + const senderName = (node.querySelector('[class*="_chatName_"]')?.innerText || "").trim(); + const contentClass = String(contentBox?.className || ""); + const imageUrls = [...node.querySelectorAll("img")] + .map(image => ({ + url: image.currentSrc || image.getAttribute("src") || "", + alt: image.alt || "", + class_name: String(image.className || ""), + parent_class: String(image.parentElement?.className || "") + })) + .filter(image => image.url.startsWith("http://") || image.url.startsWith("https://")); + const links = [...node.querySelectorAll("a[href]")] + .map(link => ({url: link.href, text: (link.innerText || "").trim()})); + const roleSignal = `${className} ${contentClass} ${senderName}`; + const senderRole = /机器人|智能客服|自动回复|人机回复|AI客服|小蜜|chatbot|botreply|robot/i.test(roleSignal) + ? "bot" + : contentBox && contentClass.includes("Xiaoer") ? "staff" : "customer"; + rows.push({ + date_label: dateLabel, + time_label: (node.querySelector('[class*="_chatTime_"]')?.innerText || "").trim(), + sender_name: senderName, + sender_role: senderRole, + content: (textBox?.innerText || "").trim(), + image_urls: imageUrls, + links, + content_class: contentClass, + chat_class: className + }); + } + return rows; + }""" + ) + + +def _all_messages_for_customer(frame: Any, panel: Any) -> list[dict[str, Any]]: + captured: dict[str, dict[str, Any]] = {} + for direction in ("top", "bottom"): + stable_rounds = 0 + previous_state: tuple[int, int, int] | None = None + reached_boundary = False + for _ in range(MAX_SCROLL_STEPS): + rendered = _rendered_messages(panel) + duplicate_counts: Counter[str] = Counter() + for item in rendered: + signature_value = json.dumps( + item, + ensure_ascii=False, + sort_keys=True, + default=str, + ) + base_key = hashlib.sha256(signature_value.encode("utf-8")).hexdigest() + duplicate_index = duplicate_counts[base_key] + duplicate_counts[base_key] += 1 + captured.setdefault(f"{base_key}:{duplicate_index}", item) + + metrics = panel.evaluate( + "element => ({top:Math.round(element.scrollTop), height:element.scrollHeight, client:element.clientHeight})" + ) + current_state = (metrics["top"], metrics["height"], len(rendered)) + at_boundary = ( + metrics["top"] <= 1 + if direction == "top" + else metrics["top"] + metrics["client"] >= metrics["height"] - 1 + ) + if at_boundary and current_state == previous_state: + stable_rounds += 1 + if stable_rounds >= 2: + reached_boundary = True + break + else: + stable_rounds = 0 + previous_state = current_state + if direction == "top": + next_top = max(0, metrics["top"] - max(100, int(metrics["client"] * 0.8))) + else: + next_top = min( + metrics["height"], + metrics["top"] + max(100, int(metrics["client"] * 0.8)), + ) + panel.evaluate("(element, value) => { element.scrollTop = value; }", next_top) + frame.wait_for_timeout(160) + if not reached_boundary: + raise RuntimeError(f"天猫客服会话消息滚动未到达{direction}边界") + + result = list(captured.values()) + result.sort(key=lambda item: (_message_datetime(item.get("date_label"), item.get("time_label")) or datetime.min.replace(tzinfo=SHANGHAI))) + return result + + +_DATE_PATTERN = re.compile(r"(\d{4})\s*[-/.年]\s*(\d{1,2})\s*[-/.月]\s*(\d{1,2})") +_TIME_PATTERN = re.compile(r"(\d{1,2}:\d{2}(?::\d{2})?)") + + +def _message_datetime(date_label: Any, time_label: Any) -> datetime | None: + date_match = _DATE_PATTERN.search(str(date_label or "")) + time_match = _TIME_PATTERN.search(str(time_label or "")) + if date_match is None or time_match is None: + return None + year, month, day = (int(part) for part in date_match.groups()) + time_text = time_match.group(1) + time_format = "%H:%M:%S" if time_text.count(":") == 2 else "%H:%M" + parsed_time = datetime.strptime(time_text, time_format).time() + return datetime(year, month, day, parsed_time.hour, parsed_time.minute, parsed_time.second, tzinfo=SHANGHAI) + + +def _safe_url(value: str) -> str: + try: + parts = urlsplit(value.strip()) + except ValueError: + return "" + if parts.scheme not in {"http", "https"} or not parts.netloc: + return "" + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + + +def _normalize_chat_messages( + raw_messages: list[dict[str, Any]], + *, + customer_id: str, + customer_name: str, + session_id: str, + query_start: date, + query_end: date, +) -> list[dict[str, Any]]: + seen_fingerprints: Counter[str] = Counter() + normalized: list[dict[str, Any]] = [] + for raw in raw_messages: + timestamp = _message_datetime(raw.get("date_label"), raw.get("time_label")) + images = [ + item + for item in raw.get("image_urls") or [] + if isinstance(item, dict) + ] + image_urls = [ + url + for url in ( + _safe_url(str(item.get("url") or "")) + for item in images + ) + if url + ] + link_urls = [ + url + for url in ( + _safe_url(str(item.get("url") or "")) + for item in raw.get("links") or [] + if isinstance(item, dict) + ) + if url + ] + media_urls = list(dict.fromkeys([*image_urls, *link_urls])) + content = "\n".join( + line.strip() for line in str(raw.get("content") or "").splitlines() if line.strip() + ) + sender_name = str(raw.get("sender_name") or "") + content_class = str(raw.get("content_class") or "") + chat_class = str(raw.get("chat_class") or "") + sender_role = str(raw.get("sender_role") or "customer") + if BOT_MARKER.search(f"{sender_name} {content_class} {chat_class}"): + sender_role = "bot" + message_type = "image" if image_urls else "link" if link_urls else "text" + fingerprint_payload = json.dumps( + [ + customer_id, + timestamp.isoformat() if timestamp else None, + sender_name, + sender_role, + message_type, + content, + media_urls, + ], + ensure_ascii=False, + separators=(",", ":"), + ) + fingerprint = hashlib.sha256(fingerprint_payload.encode("utf-8")).hexdigest() + duplicate_index = seen_fingerprints[fingerprint] + seen_fingerprints[fingerprint] += 1 + message_key = f"{fingerprint}:{duplicate_index}" + business_date = timestamp.date() if timestamp else query_start + normalized.append( + { + "account": ACCOUNT_ID, + "business_date": business_date, + "query_start_date": query_start, + "query_end_date": query_end, + "customer_id": customer_id, + "customer_name": customer_name, + "session_id": session_id, + "message_key": message_key, + "message_at": timestamp, + "sender_name": sender_name, + "sender_role": sender_role, + "content": content, + "message_type": message_type, + "media_urls": media_urls, + "source_url": TARGET_URL, + "raw_message": { + "date_label": raw.get("date_label"), + "time_label": raw.get("time_label"), + "content_class": content_class, + "chat_class": chat_class, + "image_alts": [str(item.get("alt") or "") for item in images], + "image_classes": [ + " ".join( + value + for value in ( + str(item.get("class_name") or ""), + str(item.get("parent_class") or ""), + ) + if value + ) + for item in images + ], + }, + } + ) + return normalized + + +def _is_emoji_only(value: str) -> bool: + value = "".join(value.split()) + if not value: + return False + has_emoji = False + allowed_punctuation = set(".,，。!！?？~～-—_()（）[]【】") + for character in value: + codepoint = ord(character) + if character in allowed_punctuation: + continue + if codepoint in {0x200D, 0x20E3} or 0xFE00 <= codepoint <= 0xFE0F: + continue + if 0x1F3FB <= codepoint <= 0x1F3FF or 0x1F1E6 <= codepoint <= 0x1F1FF: + has_emoji = True + continue + if ( + 0x1F000 <= codepoint <= 0x1FAFF + or 0x2600 <= codepoint <= 0x27BF + or 0x2300 <= codepoint <= 0x23FF + ): + has_emoji = True + continue + if unicodedata.category(character) == "So" and codepoint in {0x00A9, 0x00AE, 0x203C, 0x2049, 0x2122, 0x2139}: + has_emoji = True + continue + return False + return has_emoji + + +def _initial_filter_reason(message: dict[str, Any]) -> str | None: + content = str(message.get("content") or "") + normalized_content = " ".join(content.split()) + if WEB_ONLY_PLACEHOLDER.search(normalized_content): + return "web_only_placeholder" + + raw_message = message.get("raw_message") or {} + sender_signal = " ".join( + str(value or "") + for value in ( + message.get("sender_name"), + raw_message.get("content_class"), + raw_message.get("chat_class"), + ) + ) + if ( + str(message.get("sender_role") or "").casefold() in {"bot", "robot", "system"} + or BOT_MARKER.search(sender_signal) + or BOT_REPLY_TEXT.search(normalized_content) + ): + return "bot_reply" + + if _is_emoji_only(normalized_content) or TEXT_EMOTICON.fullmatch(normalized_content): + return "emoji_only" + + if not normalized_content and message.get("message_type") == "image": + image_signals = " ".join( + [ + *[str(value or "") for value in raw_message.get("image_alts") or []], + *[str(value or "") for value in raw_message.get("image_classes") or []], + *[str(value or "") for value in message.get("media_urls") or []], + ] + ) + if STICKER_MARKER.search(image_signals): + return "sticker_only" + return None + + +def _capture_customer( + frame: Any, + row: Any, + *, + index: int, + query_start: date, + query_end: date, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + customer_name = " ".join(row.inner_text(timeout=10_000).split()) + if not customer_name: + raise RuntimeError(f"第 {index} 条客服会话没有可用标识") + customer_id = _customer_identity(row, customer_name) + session_id = hashlib.sha256( + f"{ACCOUNT_ID}\0{customer_id}".encode("utf-8") + ).hexdigest()[:32] + classes = row.get_attribute("class") or "" + panel = _wait_for_selected_customer( + frame, + row, + already_selected="results-list-active" in classes, + ) + raw_messages = _all_messages_for_customer(frame, panel) + if not raw_messages: + raise RuntimeError(f"第 {index} 条客服会话没有载入聊天消息") + messages = _normalize_chat_messages( + raw_messages, + customer_id=customer_id, + customer_name=customer_name, + session_id=session_id, + query_start=query_start, + query_end=query_end, + ) + timestamps = [item["message_at"] for item in messages if item.get("message_at")] + conversation = { + "account": ACCOUNT_ID, + "query_start_date": query_start, + "query_end_date": query_end, + "customer_id": customer_id, + "customer_name": customer_name, + "session_id": session_id, + "session_start_at": min(timestamps) if timestamps else None, + "session_end_at": max(timestamps) if timestamps else None, + "message_count": len(messages), + "source_url": TARGET_URL, + "raw_session": { + "customer_name": customer_name, + "query_start_date": query_start.isoformat(), + "query_end_date": query_end.isoformat(), + }, + } + return conversation, messages + + +def _persist_records( + conversations: list[dict[str, Any]], + messages: list[dict[str, Any]], + *, + no_db: bool, +) -> tuple[int, int]: + if no_db: + return 0, 0 + from gyxx_flow.modules.shop_intelligence.db import ( + get_conn, + upsert_tmall_customer_service_conversations, + upsert_tmall_customer_service_messages, + ) + + with get_conn() as conn: + conversation_count = upsert_tmall_customer_service_conversations( + conn, conversations + ) + message_count = upsert_tmall_customer_service_messages(conn, messages) + return conversation_count, message_count + + +def _json_default(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + if isinstance(value, Path): + return str(value) + if isinstance(value, dict): + return {str(key): _json_default(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_default(item) for item in value] + return value + + +def _save_json(path: Path, payload: dict[str, Any]) -> None: + json_safe_payload = { + key: _json_default(value) if isinstance(value, (date, datetime, Path, dict, list, tuple)) else value + for key, value in payload.items() + } + atomic_write_json(path, json_safe_payload) + + +def _preflight_acceptance_login(environment: dict[str, str]) -> int | None: + policy = current_acceptance_policy() + if not policy.enabled: + return None + result = policy.preflight_cookie( + binding_from_environment(environment), environment=environment + ) + if result.should_skip: + print(f"[SKIPPED_COOKIE] {result.reason}") + return COOKIE_SKIP_EXIT_CODE + return None + + +def run( + args: argparse.Namespace, + *, + runtime_environment: dict[str, str] | None = None, +) -> int: + environment = runtime_environment or dict(os.environ) + selected_start, selected_end = resolve_date_range(args, environment) + windows = split_query_windows(selected_start, selected_end) + args.output.mkdir(parents=True, exist_ok=True) + manifest_path = args.output / ( + f"客服消息_{selected_start.isoformat()}_to_{selected_end.isoformat()}_manifest.json" + ) + manifest: dict[str, Any] = { + "account": ACCOUNT_ID, + "account_label": ACCOUNT_LABEL, + "start_date": selected_start.isoformat(), + "end_date": selected_end.isoformat(), + "query_windows": [], + "complete": False, + } + + if not args.no_db: + from gyxx_flow.modules.shop_intelligence.db import init_schema + + init_schema() + + browser = ScraplingBrowser( + **browser_runtime_options(environment, headless=args.headless) + ) + try: + browser.start() + if browser.context is None: + raise RuntimeError("Scrapling 未创建天猫客服浏览器上下文") + page = browser.context.pages[0] if browser.context.pages else browser.context.new_page() + frame = open_and_verify(page) + browser.persist_bound_state() + print( + f"天猫客服聊天记录采集：{selected_start} 至 {selected_end}，" + f"拆分为 {len(windows)} 个不超过 {MAX_QUERY_DAYS} 天的查询窗口。" + ) + + for window_index, (window_start, window_end) in enumerate(windows, start=1): + rows = _query_window(frame, window_start, window_end) + print( + f"窗口 {window_index}/{len(windows)} {window_start} 至 {window_end}：" + f"查询到 {len(rows)} 个会话。" + ) + raw_conversations: list[dict[str, Any]] = [] + database_conversations: list[dict[str, Any]] = [] + raw_messages: list[dict[str, Any]] = [] + database_messages: list[dict[str, Any]] = [] + filtered_counts: Counter[str] = Counter() + failures: list[dict[str, Any]] = [] + for row_index, row in enumerate(rows, start=1): + try: + conversation, conversation_messages = _capture_customer( + frame, + row, + index=row_index, + query_start=window_start, + query_end=window_end, + ) + except Exception as exc: + failures.append( + {"row": row_index, "reason_type": type(exc).__name__} + ) + continue + retained_messages: list[dict[str, Any]] = [] + for message in conversation_messages: + reason = _initial_filter_reason(message) + raw_message = dict(message) + raw_message["db_filter_reason"] = reason + raw_messages.append(raw_message) + if reason is None: + retained_messages.append(message) + database_messages.append(message) + else: + filtered_counts[reason] += 1 + + raw_conversations.append( + { + **conversation, + "raw_message_count": len(conversation_messages), + "database_message_count": len(retained_messages), + "filtered_message_count": ( + len(conversation_messages) - len(retained_messages) + ), + } + ) + if retained_messages: + database_conversations.append( + { + **conversation, + "message_count": len(retained_messages), + } + ) + if row_index % 50 == 0 or row_index == len(rows): + print( + f" 已处理 {row_index}/{len(rows)} 个会话，" + f"保留 {len(database_messages)} 条消息，" + f"过滤 {sum(filtered_counts.values())} 条。" + ) + + conversation_count, message_count = _persist_records( + database_conversations, + database_messages, + no_db=args.no_db, + ) + window_name = ( + f"客服消息_{window_start.isoformat()}_to_{window_end.isoformat()}.json" + ) + _save_json( + args.output / window_name, + { + "account": ACCOUNT_ID, + "account_label": ACCOUNT_LABEL, + "query_start_date": window_start, + "query_end_date": window_end, + "raw_conversation_count": len(raw_conversations), + "database_conversation_count": len(database_conversations), + "raw_message_count": len(raw_messages), + "database_message_count": len(database_messages), + "filtered_message_count": sum(filtered_counts.values()), + "filtered_by_reason": dict(filtered_counts), + "conversations": raw_conversations, + "messages": raw_messages, + "failed_rows": failures, + "complete": not failures, + }, + ) + manifest["query_windows"].append( + { + "start_date": window_start.isoformat(), + "end_date": window_end.isoformat(), + "raw_conversation_count": len(raw_conversations), + "database_conversation_count": len(database_conversations), + "raw_message_count": len(raw_messages), + "database_message_count": len(database_messages), + "filtered_message_count": sum(filtered_counts.values()), + "filtered_by_reason": dict(filtered_counts), + "database_conversations": conversation_count, + "database_messages": message_count, + "failed_conversation_count": len(failures), + "artifact": window_name, + "complete": not failures, + } + ) + _save_json(manifest_path, manifest) + if failures: + raise RuntimeError( + f"窗口 {window_index} 有 {len(failures)} 个会话未完成；" + f"已保存成功会话，可安全重试整个日期范围" + ) + + manifest["complete"] = True + _save_json(manifest_path, manifest) + total_conversations = sum( + int(item["database_conversation_count"]) + for item in manifest["query_windows"] + ) + total_messages = sum( + int(item["database_message_count"]) for item in manifest["query_windows"] + ) + total_filtered = sum( + int(item["filtered_message_count"]) + for item in manifest["query_windows"] + ) + print( + f"采集完成：入库 {total_conversations} 个会话、{total_messages} 条消息，" + f"过滤 {total_filtered} 条；" + f"原始产物清单 {manifest_path}。" + ) + return 0 + finally: + browser.close(persist_state=False) + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + runtime_environment = _account_runtime_environment() + preflight_exit_code = _preflight_acceptance_login(runtime_environment) + if preflight_exit_code is not None: + return preflight_exit_code + return run(args, runtime_environment=runtime_environment) + except KeyboardInterrupt: + print("\n用户中断。") + return 130 + except Exception as exc: + print( + f"[ERROR] 天猫客服聊天记录采集失败: {type(exc).__name__}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/shop_intelligence/collectors/tmall_negative_reviews.py b/src/gyxx_flow/modules/shop_intelligence/collectors/tmall_negative_reviews.py new file mode 100644 index 0000000..21b6235 --- /dev/null +++ b/src/gyxx_flow/modules/shop_intelligence/collectors/tmall_negative_reviews.py @@ -0,0 +1,801 @@ +"""采集天猫评价管理中的负面评价，并幂等保存评论和商品名称。 + +页面不选择评价时间时会查询全部日期，因此每次运行都要求显式提供控制台 +选择的开始和结束日期；采集时先设置日期、固定 emotion=13（负面评价），再 +点击搜索。只读取评价文字、图片、主评 ID、评价时间和商品名称，不读取或保存订单号。 +""" + +from __future__ import annotations + +import argparse +import math +import os +import re +import sys +import time +from datetime import date, datetime +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlsplit +from zoneinfo import ZoneInfo + +from gyxx_flow.adapters import ( + COOKIE_SKIP_EXIT_CODE, + binding_from_environment, + current_acceptance_policy, +) +from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog +from gyxx_flow.adapters.scrapling import ScraplingBrowser, is_browser_timeout_error +from gyxx_flow.core.artifacts import atomic_write_json +from gyxx_flow.core.config import Settings +from gyxx_flow.core.layout import DataLayout +from gyxx_flow.modules.shop_intelligence.collectors import ( + tmall_customer_service_messages as customer_service, +) + +SETTINGS = Settings.from_env() +MODULE_PATHS = DataLayout(SETTINGS.data_root).for_module("shop_intelligence") + +TARGET_URL = ( + "https://qn.taobao.com/home.htm/comment-manage/list/rateWait4PC" + "?current=1&pageSize=20&emotion=13" +) +ACCOUNT_ID = "tmall-shop" +ACCOUNT_LABEL = "光影行星旗舰店:龙虾仔" +COMMAND_ID = "shop.tmall_negative_reviews.collect" +WORKFLOW_ID = "shop.tmall_negative_reviews.daily" +DEFAULT_OUTPUT = MODULE_PATHS.raw_root / "tmall_negative_reviews" +PAGE_SIZE = 20 +CDP_PORT = 22162 +SHANGHAI = ZoneInfo("Asia/Shanghai") +REVIEW_ID_PATTERN = re.compile(r"主评ID\s*[:：]?\s*([A-Za-z0-9_-]+)", re.IGNORECASE) +REVIEW_AT_PATTERN = re.compile( + r"(?:初次评价|评价时间)\s*[:：]?\s*" + r"(\d{4}[-/]\d{1,2}[-/]\d{1,2}(?:\s+\d{1,2}:\d{2}(?::\d{2})?)?)" +) +REVIEW_META_PATTERN = re.compile( + r"(?:初次评价|评价时间)\s*[:：]?\s*" + r"\d{4}[-/]\d{1,2}[-/]\d{1,2}(?:\s+\d{1,2}:\d{2}(?::\d{2})?)?" + r"|主评ID\s*[:：]?\s*[A-Za-z0-9_-]+", + re.IGNORECASE, +) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="按日期范围采集天猫负面评价") + parser.add_argument("--start-date", type=date.fromisoformat) + parser.add_argument("--end-date", type=date.fromisoformat) + parser.add_argument( + "--all-time", + action="store_true", + help="清空评价时间筛选并查询页面可提供的全部日期", + ) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument("--no-db", action="store_true", help="只保存本地 JSON") + parser.add_argument( + "--headless", + action="store_true", + default=True, + help="使用 Scrapling 隐藏指纹浏览器（默认）", + ) + return parser.parse_args(argv) + + +def _parse_env_date(value: str, *, label: str) -> date: + try: + parsed = date.fromisoformat(value) + except ValueError as exc: + raise ValueError(f"{label}必须使用 YYYY-MM-DD 格式") from exc + if parsed.isoformat() != value: + raise ValueError(f"{label}必须使用 YYYY-MM-DD 格式") + return parsed + + +def resolve_date_range( + args: argparse.Namespace, + environment: dict[str, str], + *, + today: date | None = None, +) -> tuple[date | None, date | None]: + if args.all_time: + return None, None + start_value = environment.get("GYXX_TMALL_NEGATIVE_REVIEWS_START_DATE", "").strip() + end_value = environment.get("GYXX_TMALL_NEGATIVE_REVIEWS_END_DATE", "").strip() + start = ( + _parse_env_date(start_value, label="评价开始日期") + if start_value + else args.start_date + ) + end = _parse_env_date(end_value, label="评价结束日期") if end_value else args.end_date + if start is None or end is None: + raise ValueError("必须选择天猫差评采集的开始日期和结束日期，避免页面全日期查询") + current = today or datetime.now(SHANGHAI).date() + if end < start: + raise ValueError("评价结束日期不能早于开始日期") + if end > current: + raise ValueError(f"评价结束日期不能晚于今天 {current.isoformat()}") + return start, end + + +def _account_runtime_environment() -> dict[str, str]: + environment = dict(os.environ) + project_root = environment.get("GYXX_PROJECT_ROOT", "").strip() + data_root = environment.get("GYXX_DATA_ROOT", "").strip() + if not project_root: + project_root = str(SETTINGS.project_root) + if not data_root: + data_root = str(SETTINGS.data_root) + catalog = RuntimeIntegrationCatalog.load_default( + project_root=project_root, + data_root=data_root, + ) + return catalog.environment_for(COMMAND_ID, environment) + + +def browser_runtime_options(environment: dict[str, str], *, headless: bool) -> dict[str, Any]: + profile = Path( + environment.get( + "GYXX_BROWSER_PROFILE_DIR", + str(MODULE_PATHS.state_root / "browser" / "tmall_negative_reviews_profile"), + ) + ) + port = int(environment.get("GYXX_BROWSER_CDP_PORT", str(CDP_PORT))) + options: dict[str, Any] = { + "stealthy": True, + "user_data_dir": str(profile), + "headless": headless, + "locale": "zh-CN", + "timezone_id": "Asia/Shanghai", + "persist_state_on_close": False, + "additional_args": {"viewport": {"width": 1920, "height": 1080}}, + "extra_flags": [ + f"--remote-debugging-port={port}", + "--remote-debugging-address=127.0.0.1", + ], + "retries": 1, + } + for key, option in ( + ("GYXX_BROWSER_COOKIE_FILE", "cookie_file"), + ("GYXX_BROWSER_STORAGE_STATE_FILE", "storage_state_file"), + ): + value = environment.get(key, "").strip() + if value: + options[option] = value + return options + + +def _preflight_acceptance_login(environment: dict[str, str]) -> int | None: + policy = current_acceptance_policy() + if not policy.enabled: + return None + result = policy.preflight_cookie( + binding_from_environment(environment), environment=environment + ) + if result.should_skip: + print(f"[SKIPPED_COOKIE] {result.reason}") + return COOKIE_SKIP_EXIT_CODE + return None + + +def _wait_for_page(page: Any, timeout_seconds: int = 60) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if "login" in page.url.casefold() or "passport" in page.url.casefold(): + raise RuntimeError("天猫登录态已失效，请通过 tmall-shop 账号 vault 恢复登录") + try: + if ( + page.get_by_text("情感分类", exact=True).count() + and page.locator('input[placeholder*="起始日期"]').count() + ): + customer_service.verify_shop_identity(page) + return + except Exception as exc: + if not is_browser_timeout_error(exc): + raise + page.wait_for_timeout(300) + raise RuntimeError("天猫评价管理页面加载超时") + + +def _date_input(page: Any, placeholder: str) -> Any: + locator = page.locator(f'input[placeholder="{placeholder}"]').first + if locator.count(): + return locator + return page.get_by_role("textbox", name=placeholder).first + + +def _calendar_cell(picker: Any, target: date) -> Any | None: + selector = f'td[title="{target.isoformat()}"]' + current_month = picker.locator( + selector + + ":not(.next-calendar-cell-prev-month):not(.next-calendar-cell-next-month)" + ).first + if current_month.count() and current_month.is_visible(): + return current_month + cell = picker.locator(selector).first + return cell if cell.count() and cell.is_visible() else None + + +def _select_calendar_day(picker: Any, target: date) -> None: + for _ in range(180): + cell = _calendar_cell(picker, target) + if cell is not None: + if "disabled" in (cell.get_attribute("class") or "").casefold(): + raise RuntimeError(f"评价日期选择器未开放 {target.isoformat()}") + cell.click() + return + visible_values = picker.locator("td[title]").evaluate_all( + "items => items.map(item => item.title).filter(Boolean)" + ) + visible_dates: list[date] = [] + for value in visible_values: + try: + visible_dates.append(date.fromisoformat(value)) + except ValueError: + continue + if not visible_dates: + raise RuntimeError("天猫评价日期选择器没有可用日期") + if target < min(visible_dates): + picker.locator(".next-calendar-btn-prev-month").first.click() + elif target > max(visible_dates): + picker.locator(".next-calendar-btn-next-month").first.click() + else: + raise RuntimeError(f"评价日期选择器无法选择 {target.isoformat()}") + raise RuntimeError(f"评价日期选择器翻页次数超限: {target.isoformat()}") + + +def set_review_date_range(page: Any, start: date, end: date) -> None: + """显式设置评价时间；不得在空日期条件下点击搜索。""" + + if end < start: + raise ValueError("评价结束日期不能早于开始日期") + start_input = _date_input(page, "起始日期") + end_input = _date_input(page, "结束日期") + readonly = any( + locator.get_attribute("readonly") is not None + or locator.get_attribute("aria-readonly") == "true" + for locator in (start_input, end_input) + ) + if not readonly: + try: + start_input.fill(start.isoformat()) + end_input.fill(end.isoformat()) + end_input.press("Tab") + except Exception as exc: + if not is_browser_timeout_error(exc): + # A date input can reject fill even without a readonly attribute; + # use the platform calendar as the supported fallback. + readonly = True + if readonly: + start_input.click() + picker = page.locator(".next-range-picker-body:visible").first + picker.wait_for(state="visible", timeout=10_000) + _select_calendar_day(picker, start) + _select_calendar_day(picker, end) + confirm = picker.get_by_role("button", name="确定", exact=True) + if confirm.count() and confirm.is_visible(): + confirm.click() + + actual = [start_input.input_value(), end_input.input_value()] + expected = [start.isoformat(), end.isoformat()] + if actual != expected: + raise RuntimeError(f"评价日期范围未生效，期望 {expected}，实际 {actual}") + + +def clear_review_date_range(page: Any) -> None: + """只在显式 --all-time 时清空页面时间筛选，并确认两个日期为空。""" + + start_input = _date_input(page, "起始日期") + end_input = _date_input(page, "结束日期") + clear_controls = page.locator( + ".next-range-picker .next-range-picker-clear:visible, " + ".next-range-picker .next-input-clear:visible, " + ".next-range-picker .next-icon-close:visible, " + ".next-range-picker [aria-label*='清空']:visible, " + ".next-range-picker [title*='清空']:visible" + ) + for _ in range(2): + if not start_input.input_value() and not end_input.input_value(): + break + start_input.hover() + clicked = False + for index in range(clear_controls.count()): + control = clear_controls.nth(index) + try: + if control.is_visible() and control.is_enabled(): + control.click(timeout=1_000) + clicked = True + break + except Exception as exc: + if not is_browser_timeout_error(exc): + continue + if clicked: + page.wait_for_timeout(150) + continue + for locator in (start_input, end_input): + try: + locator.fill("") + locator.press("Tab") + except Exception as exc: + if not is_browser_timeout_error(exc): + continue + actual = [start_input.input_value(), end_input.input_value()] + if actual != ["", ""]: + raise RuntimeError( + f"无法清空评价时间条件，拒绝开始全日期查询；当前值为 {actual}" + ) + + +def _ensure_negative_filter(page: Any) -> None: + query = parse_qs(urlsplit(page.url).query) + if query.get("emotion") != ["13"]: + raise RuntimeError("评价情感筛选未固定为负面评价 emotion=13") + selected = page.get_by_text("负面评价", exact=True).first + selected.wait_for(state="visible", timeout=15_000) + + +def _row_locator(page: Any) -> Any: + selectors = ( + "tbody tr", + '[role="row"]', + ".next-table-row", + ".ant-table-row", + 'div[class*="review-row"]', + 'div[class*="rate-row"]', + 'div[class*="review-item"]', + 'div[class*="rate-item"]', + 'div[class*="reviewItem"]', + 'div[class*="rateItem"]', + ) + for selector in selectors: + candidates = page.locator(selector).filter(has_text="主评ID") + if candidates.count(): + for index in range(min(candidates.count(), 100)): + row = candidates.nth(index) + for cell_selector in ("td", '[role="cell"]', ".next-table-cell", ".ant-table-cell"): + if row.locator(cell_selector).count() >= 4: + return candidates + return page.locator("__gyxx_tmall_negative_review_row_not_found__") + + +def _candidate_cells(row: Any) -> list[Any]: + for selector in ("td", '[role="cell"]', ".next-table-cell", ".ant-table-cell"): + cells = row.locator(selector) + if cells.count() >= 4: + return [cells.nth(index) for index in range(cells.count())] + return [] + + +def _review_text(cell: Any) -> tuple[str, str]: + raw_text = " ".join(cell.inner_text(timeout=5_000).split()) + try: + content = cell.evaluate( + """element => { + const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT); + const parts = []; + let node; + while ((node = walker.nextNode())) { + const value = (node.nodeValue || '').replace(/\\s+/g, ' ').trim(); + if (!value) continue; + let parent = node.parentElement; + let excluded = false; + while (parent && parent !== element) { + const className = typeof parent.className === 'string' ? parent.className : ''; + if (/(?:^|[-_])(tag|badge|chip)(?:$|[-_])/i.test(className)) { + excluded = true; + break; + } + parent = parent.parentElement; + } + if (!excluded) parts.push(value); + } + return parts.join(' '); + }""" + ) + except Exception: + content = raw_text + content = " ".join(str(content or "").split()) + content = REVIEW_META_PATTERN.sub(" ", content) + content = " ".join(content.split()).strip(" \t-—|") + return raw_text, content + + +def _review_images(cell: Any) -> list[str]: + values = cell.locator("img").evaluate_all( + """images => images.map(image => [image.currentSrc, image.src, + image.getAttribute('data-src'), image.getAttribute('data-original')] + .find(value => typeof value === 'string' && /^https?:/i.test(value)) || '')""" + ) + result: list[str] = [] + for value in values: + url = str(value or "").strip() + if url.startswith(("https://", "http://")) and url not in result: + result.append(url) + return result + + +def _review_product_name(cells: list[Any], review_cell_index: int) -> str | None: + # In the QN list the product column immediately follows 评价内容. Read only + # its product link, never the order-information cell beneath it. + product_index = review_cell_index + 1 + if product_index >= len(cells): + return None + anchors = cells[product_index].locator("a").evaluate_all( + "links => links.filter(link => !!link.offsetWidth || !!link.offsetHeight)" + ".map(link => (link.innerText || '').trim()).filter(Boolean)" + ) + for value in anchors: + product = " ".join(str(value).split()) + if product and "订单号" not in product and len(product) > 2: + return product + return None + + +def _parse_review_at(value: str) -> datetime | None: + match = REVIEW_AT_PATTERN.search(value) + if not match: + return None + normalized = match.group(1).replace("/", "-") + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + return None + return parsed.replace(tzinfo=SHANGHAI) if parsed.tzinfo is None else parsed + + +def normalize_review_row( + row: Any, + *, + account: str, + start_date: date | None, + end_date: date | None, + source_url: str, + captured_at: datetime | None = None, +) -> dict[str, Any]: + cells = _candidate_cells(row) + if len(cells) < 4: + raise RuntimeError("天猫差评行结构不完整，无法安全解析商品与评价内容") + review_cell_index = next( + (index for index, cell in enumerate(cells) if "主评ID" in cell.inner_text(timeout=5_000)), + None, + ) + if review_cell_index is None: + raise RuntimeError("天猫差评行缺少主评 ID") + review_cell = cells[review_cell_index] + raw_text, content = _review_text(review_cell) + review_id_match = REVIEW_ID_PATTERN.search(raw_text) + if not review_id_match: + raise RuntimeError("天猫差评行主评 ID 格式无法识别") + review_id = review_id_match.group(1).strip() + review_at = _parse_review_at(raw_text) + if review_at is None: + raise RuntimeError(f"天猫差评 {review_id} 缺少可识别的评价时间") + business_date = review_at.astimezone(SHANGHAI).date() + if start_date is not None and business_date < start_date: + raise RuntimeError( + f"评价日期早于所选范围: {business_date.isoformat()}" + ) + if end_date is not None and business_date > end_date: + raise RuntimeError( + f"评价日期晚于所选范围: {business_date.isoformat()}" + ) + image_urls = _review_images(review_cell) + if not content and not image_urls: + raise RuntimeError(f"天猫差评 {review_id} 没有可保存的评论文字或图片") + product_name = _review_product_name(cells, review_cell_index) + if not product_name: + raise RuntimeError(f"天猫差评 {review_id} 的商品名称无法识别") + clean_raw = { + "review_id": review_id, + "review_at": review_at, + "review_level": "负面评价", + "content": content or None, + "image_urls": image_urls, + "product_name": product_name, + } + return { + "account": account, + "business_date": business_date, + "review_id": review_id, + "review_at": review_at, + "review_level": "负面评价", + "content": content or None, + "image_urls": image_urls, + "product_name": product_name, + "source_url": source_url, + "raw_review": clean_raw, + "captured_at": captured_at or datetime.now(SHANGHAI), + } + + +def _review_rows(page: Any) -> list[Any]: + locator = _row_locator(page) + if locator.count() == 0: + return [] + return [locator.nth(index) for index in range(locator.count())] + + +def _total_count(page: Any) -> int | None: + labels = page.get_by_text(re.compile(r"共\s*\d+\s*条"), exact=False) + for index in range(min(labels.count(), 10)): + match = re.search(r"共\s*(\d+)\s*条", labels.nth(index).inner_text(timeout=2_000)) + if match: + return int(match.group(1)) + return None + + +def _is_empty_result(page: Any) -> bool: + for marker in ("暂无评价", "暂无数据", "没有相关评价"): + if page.get_by_text(marker, exact=False).count(): + return True + return False + + +def _review_identity_text(row: Any) -> str: + for cell in _candidate_cells(row): + if cell.get_by_text("主评ID", exact=False).count(): + return cell.inner_text(timeout=3_000) + return "" + + +def _result_signature(page: Any) -> tuple[int | None, tuple[str, ...]]: + ids: list[str] = [] + for row in _review_rows(page): + text = _review_identity_text(row) + match = REVIEW_ID_PATTERN.search(text) + if match: + ids.append(match.group(1)) + return _total_count(page), tuple(ids) + + +def _wait_for_results(page: Any, *, previous: tuple[int | None, tuple[str, ...]] | None = None) -> tuple[int, int]: + deadline = time.monotonic() + 60 + last: tuple[int | None, tuple[str, ...]] | None = None + stable_rounds = 0 + started_at = time.monotonic() + while time.monotonic() < deadline: + signature = _result_signature(page) + ready = signature[0] is not None or bool(signature[1]) or _is_empty_result(page) + if ready and signature == last: + stable_rounds += 1 + else: + stable_rounds = 0 + last = signature + changed = previous is None or signature != previous + if ready and changed and stable_rounds >= 2 and time.monotonic() - started_at >= 1.2: + total = signature[0] + if total is None: + if _is_empty_result(page): + return 0, 0 + raise RuntimeError("天猫评价页面未显示结果总数，拒绝不完整入库") + return total, len(signature[1]) + page.wait_for_timeout(350) + raise RuntimeError("天猫评价搜索结果加载超时或结果页未发生变化") + + +def _next_page_button(page: Any) -> Any | None: + selectors = ( + ".next-pagination-next", + ".ant-pagination-next", + 'button[aria-label*="下一页"]', + 'button[title*="下一页"]', + 'a[aria-label*="下一页"]', + 'a[title*="下一页"]', + '[role="button"][aria-label*="下一页"]', + ) + for selector in selectors: + candidate = page.locator(selector).last + if candidate.count() and candidate.is_visible(): + return candidate + named = page.get_by_role("button", name=re.compile(r"下一页|下页|Next", re.IGNORECASE)).last + if named.count() and named.is_visible(): + return named + for glyph in ("›", "❯", ">"): + candidates = page.get_by_text(glyph, exact=True) + if candidates.count() and candidates.last.is_visible(): + return candidates.last + return None + + +def _button_disabled(button: Any) -> bool: + return bool( + button.get_attribute("disabled") is not None + or button.get_attribute("aria-disabled") == "true" + or "disabled" in (button.get_attribute("class") or "").casefold() + ) + + +def _serialize(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + if isinstance(value, dict): + return {str(key): _serialize(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_serialize(item) for item in value] + return value + + +def _save_artifact(path: Path, payload: dict[str, Any]) -> None: + atomic_write_json(path, _serialize(payload)) + + +def _persist_records(records: list[dict[str, Any]], *, no_db: bool) -> int: + if no_db or not records: + return 0 + from gyxx_flow.modules.shop_intelligence.db import ( + get_conn, + upsert_tmall_negative_reviews, + ) + + with get_conn() as conn: + return upsert_tmall_negative_reviews(conn, records) + + +def run( + args: argparse.Namespace, + *, + runtime_environment: dict[str, str] | None = None, +) -> int: + environment = runtime_environment or dict(os.environ) + start_date, end_date = resolve_date_range(args, environment) + args.output.mkdir(parents=True, exist_ok=True) + artifact_name = ( + "天猫差评_全日期.json" + if args.all_time + else f"天猫差评_{start_date.isoformat()}_to_{end_date.isoformat()}.json" + ) + artifact_path = args.output / artifact_name + collected: dict[str, dict[str, Any]] = {} + completed_pages = 0 + total = 0 + artifact: dict[str, Any] = { + "workflow_id": WORKFLOW_ID, + "account": ACCOUNT_ID, + "account_label": ACCOUNT_LABEL, + "start_date": start_date.isoformat() if start_date else None, + "end_date": end_date.isoformat() if end_date else None, + "all_time": args.all_time, + "sentiment": "负面评价", + "total_count": None, + "expected_pages": None, + "completed_pages": 0, + "database_rows": 0, + "reviews": [], + "complete": False, + } + if not args.no_db: + from gyxx_flow.modules.shop_intelligence.db import init_schema + + init_schema() + + browser = ScraplingBrowser(**browser_runtime_options(environment, headless=args.headless)) + page: Any | None = None + try: + browser.start() + if browser.context is None: + raise RuntimeError("Scrapling 未创建天猫评价浏览器上下文") + page = browser.context.pages[0] if browser.context.pages else browser.context.new_page() + page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=60_000) + _wait_for_page(page) + _ensure_negative_filter(page) + browser.persist_bound_state() + + # A blank date condition means all-time on this page. Only clear it when + # explicitly requested; bounded runs always fill and verify both dates. + if args.all_time: + clear_review_date_range(page) + else: + assert start_date is not None and end_date is not None + set_review_date_range(page, start_date, end_date) + _ensure_negative_filter(page) + page.get_by_role("button", name="搜索", exact=True).first.click() + total, _initial_visible_count = _wait_for_results(page) + expected_pages = math.ceil(total / PAGE_SIZE) if total else 0 + artifact["total_count"] = total + artifact["expected_pages"] = expected_pages + query_label = ( + "全部日期" if args.all_time else f"{start_date} 至 {end_date}" + ) + print( + f"天猫负面评价采集：{query_label}，" + f"页面声明 {total} 条，预计 {expected_pages} 页。" + ) + + seen_ids: set[str] = set() + for page_number in range(1, expected_pages + 1): + rows = _review_rows(page) + if not rows: + raise RuntimeError(f"第 {page_number} 页未找到可解析的评价行") + if len(rows) > PAGE_SIZE: + raise RuntimeError(f"第 {page_number} 页评价行数超过页面页大小") + page_records: list[dict[str, Any]] = [] + for row in rows: + record = normalize_review_row( + row, + account=ACCOUNT_ID, + start_date=start_date, + end_date=end_date, + source_url=page.url, + ) + review_id = str(record["review_id"]) + if review_id in seen_ids: + raise RuntimeError(f"天猫差评分页返回重复主评 ID: {review_id}") + seen_ids.add(review_id) + page_records.append(record) + artifact["database_rows"] = int(artifact["database_rows"]) + _persist_records( + page_records, no_db=args.no_db + ) + for record in page_records: + review_id = str(record["review_id"]) + # Whitelisted artifact shape deliberately omits buyer and order fields. + collected[review_id] = { + key: record[key] + for key in ( + "review_id", + "business_date", + "review_at", + "review_level", + "content", + "image_urls", + "product_name", + "source_url", + "captured_at", + ) + } + completed_pages = page_number + artifact["completed_pages"] = completed_pages + artifact["reviews"] = list(collected.values()) + _save_artifact(artifact_path, artifact) + print(f" [{page_number:02d}/{expected_pages:02d}] 已完成 {len(page_records)} 条评价") + + if page_number < expected_pages: + next_button = _next_page_button(page) + if next_button is None or _button_disabled(next_button): + raise RuntimeError(f"天猫差评分页提前结束：期望第 {page_number + 1} 页") + before_page = _result_signature(page) + next_button.click() + _wait_for_results(page, previous=before_page) + + if len(seen_ids) != total: + raise RuntimeError( + f"天猫差评采集不完整：页面声明 {total} 条，实际读取 {len(seen_ids)} 条" + ) + artifact["complete"] = True + artifact["reviews"] = list(collected.values()) + _save_artifact(artifact_path, artifact) + print( + f"采集完成：共 {len(seen_ids)} 条负面评价；" + f"数据库写入/更新 {artifact['database_rows']} 行；产物 {artifact_path}。" + ) + return 0 + except Exception: + artifact["complete"] = False + artifact["completed_pages"] = completed_pages + artifact["total_count"] = total if artifact["total_count"] is None else artifact["total_count"] + artifact["reviews"] = list(collected.values()) + _save_artifact(artifact_path, artifact) + raise + finally: + browser.close(persist_state=False) + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + runtime_environment = _account_runtime_environment() + preflight_exit_code = _preflight_acceptance_login(runtime_environment) + if preflight_exit_code is not None: + return preflight_exit_code + return run(args, runtime_environment=runtime_environment) + except KeyboardInterrupt: + print("\n用户中断。") + return 130 + except Exception as exc: + print( + f"[ERROR] 天猫差评采集失败: {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/shop_intelligence/db/__init__.py b/src/gyxx_flow/modules/shop_intelligence/db/__init__.py index c25bc2f..ea327f1 100644 --- a/src/gyxx_flow/modules/shop_intelligence/db/__init__.py +++ b/src/gyxx_flow/modules/shop_intelligence/db/__init__.py @@ -2,6 +2,8 @@ from .db import ( close_pool, + delete_jd_after_sales_issue_range_residue, + fetch_dy_customer_service_products, get_conn, init_pool, init_schema, @@ -9,17 +11,32 @@ from .db import ( persist_dy_shop, persist_jd_peer_store, persist_jd_shop, + upsert_dy_after_sales_quality_negotiations, + upsert_dy_after_sales_quality_orders, + upsert_dy_customer_service_conversation_orders, + upsert_dy_customer_service_conversations, + upsert_dy_customer_service_messages, + upsert_dy_customer_service_products, + upsert_dy_negative_reviews, upsert_dy_peer_store, upsert_dy_shop_weekly, + upsert_jd_after_sales_issues, + upsert_jd_customer_service_conversations, + upsert_jd_customer_service_messages, + upsert_jd_negative_reviews, upsert_jd_peer_store, upsert_jd_self_operated_brand_daily, upsert_jd_self_operated_product_daily, upsert_jd_shop_weekly, upsert_tm_shop_weekly, + upsert_tmall_customer_service_conversations, + upsert_tmall_customer_service_messages, + upsert_tmall_negative_reviews, ) __all__ = [ "close_pool", + "delete_jd_after_sales_issue_range_residue", "get_conn", "init_pool", "init_schema", @@ -27,11 +44,26 @@ __all__ = [ "persist_dy_shop", "persist_jd_peer_store", "persist_jd_shop", + "fetch_dy_customer_service_products", + "upsert_dy_after_sales_quality_negotiations", + "upsert_dy_after_sales_quality_orders", + "upsert_dy_customer_service_conversation_orders", + "upsert_dy_customer_service_conversations", + "upsert_dy_customer_service_products", + "upsert_dy_negative_reviews", + "upsert_dy_customer_service_messages", "upsert_dy_peer_store", "upsert_dy_shop_weekly", + "upsert_jd_after_sales_issues", "upsert_jd_peer_store", + "upsert_jd_customer_service_conversations", + "upsert_jd_customer_service_messages", + "upsert_jd_negative_reviews", "upsert_jd_self_operated_brand_daily", "upsert_jd_self_operated_product_daily", "upsert_jd_shop_weekly", + "upsert_tmall_customer_service_conversations", + "upsert_tmall_customer_service_messages", + "upsert_tmall_negative_reviews", "upsert_tm_shop_weekly", ] diff --git a/src/gyxx_flow/modules/shop_intelligence/db/db.py b/src/gyxx_flow/modules/shop_intelligence/db/db.py index 63b7403..7e10c7b 100644 --- a/src/gyxx_flow/modules/shop_intelligence/db/db.py +++ b/src/gyxx_flow/modules/shop_intelligence/db/db.py @@ -441,6 +441,385 @@ def upsert_jd_self_operated_product_daily(conn, records: list[dict]) -> int: ) +# ── JD 客服聊天记录 ────────────────────────────────────────────────────────── + + +def _prepare_customer_service_records(records: list[dict]) -> list[dict]: + cleaned_records: list[dict] = [] + for record in records: + cleaned = {key: _clean_val(value) for key, value in record.items()} + for json_column in ("raw_session", "raw_message"): + if isinstance(cleaned.get(json_column), dict): + cleaned[json_column] = json.dumps( + cleaned[json_column], ensure_ascii=False, default=str + ) + cleaned["updated_at"] = datetime.now().astimezone() + cleaned_records.append(cleaned) + return cleaned_records + + +def upsert_jd_customer_service_conversations( + conn, records: list[dict] +) -> int: + """幂等写入京东客服会话，业务日期与实际会话时间同时保留。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "jd_customer_service_conversations", + _prepare_customer_service_records(records), + ("account", "business_date", "customer_id", "session_id"), + ) + + +def upsert_jd_customer_service_messages(conn, records: list[dict]) -> int: + """幂等写入京东客服消息，重复重跑不会产生重复消息。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "jd_customer_service_messages", + _prepare_customer_service_records(records), + ("account", "business_date", "customer_id", "session_id", "message_key"), + ) + + +def _prepare_tmall_customer_service_records(records: list[dict]) -> list[dict]: + cleaned_records: list[dict] = [] + for record in records: + cleaned = {key: _clean_val(value) for key, value in record.items()} + for json_column in ("raw_session", "raw_message", "media_urls"): + value = cleaned.get(json_column) + if isinstance(value, (dict, list)): + cleaned[json_column] = json.dumps( + value, ensure_ascii=False, default=str + ) + cleaned["updated_at"] = datetime.now().astimezone() + cleaned_records.append(cleaned) + return cleaned_records + + +def upsert_tmall_customer_service_conversations( + conn, records: list[dict] +) -> int: + """幂等写入天猫客服会话查询结果。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "tmall_customer_service_conversations", + _prepare_tmall_customer_service_records(records), + ( + "account", + "query_start_date", + "query_end_date", + "customer_id", + "session_id", + ), + ) + + +def upsert_tmall_customer_service_messages(conn, records: list[dict]) -> int: + """幂等写入天猫客服消息；跨区间重查按消息指纹覆盖。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "tmall_customer_service_messages", + _prepare_tmall_customer_service_records(records), + ("account", "customer_id", "session_id", "message_key"), + ) + + +def _prepare_tmall_negative_review_records(records: list[dict]) -> list[dict]: + cleaned_records: list[dict] = [] + for record in records: + cleaned = {key: _clean_val(value) for key, value in record.items()} + for json_column in ("image_urls", "raw_review"): + value = cleaned.get(json_column) + if isinstance(value, (dict, list)): + cleaned[json_column] = json.dumps( + value, ensure_ascii=False, default=str + ) + cleaned["updated_at"] = datetime.now().astimezone() + cleaned_records.append(cleaned) + return cleaned_records + + +def upsert_tmall_negative_reviews(conn, records: list[dict]) -> int: + """按天猫账号与主评 ID 幂等保存负面评价，不接收订单号字段。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "tmall_negative_reviews", + _prepare_tmall_negative_review_records(records), + ("account", "review_id"), + ) + + +# ── JD 差评及商品信息 ──────────────────────────────────────────────────────── + + +def _prepare_jd_negative_review_records(records: list[dict]) -> list[dict]: + cleaned_records: list[dict] = [] + for record in records: + cleaned = {key: _clean_val(value) for key, value in record.items()} + for json_column in ("image_urls", "raw_review"): + value = cleaned.get(json_column) + if isinstance(value, (dict, list)): + cleaned[json_column] = json.dumps( + value, ensure_ascii=False, default=str + ) + cleaned["updated_at"] = datetime.now().astimezone() + cleaned_records.append(cleaned) + return cleaned_records + + +def upsert_jd_negative_reviews(conn, records: list[dict]) -> int: + """幂等写入京东差评；同一账号、业务日期和评价 ID 只保留一行。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "jd_negative_reviews", + _prepare_jd_negative_review_records(records), + ("account", "business_date", "review_id"), + ) + + +# ── JD 品退异常服务单 ──────────────────────────────────────────────────────── + + +def _prepare_jd_after_sales_issue_records(records: list[dict]) -> list[dict]: + cleaned_records: list[dict] = [] + for record in records: + cleaned = {key: _clean_val(value) for key, value in record.items()} + raw_issue = cleaned.get("raw_issue") + if isinstance(raw_issue, (dict, list)): + cleaned["raw_issue"] = json.dumps( + raw_issue, + ensure_ascii=False, + default=str, + ) + cleaned["updated_at"] = datetime.now().astimezone() + cleaned_records.append(cleaned) + return cleaned_records + + +def upsert_jd_after_sales_issues(conn, records: list[dict]) -> int: + """幂等写入京东品退异常；同一账号、业务日和服务单 ID 只保留一行。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "jd_after_sales_issues", + _prepare_jd_after_sales_issue_records(records), + ("account", "business_date", "service_id"), + ) + + +def delete_jd_after_sales_issue_range_residue( + conn, + *, + account: str, + business_date_start: date, + business_date_end: date, +) -> int: + """清理范围采集遗留的、申请时间落在范围外的旧行。 + + 旧版单日采集会把页面查询日直接写入 ``business_date``，而商智明细 + 弹窗可能返回查询日之外的服务单。范围采集按 ``apply_at`` 落业务日， + 因此只删除同一账号、范围内业务日且申请时间明确在范围外的历史残留； + 不触碰申请时间为空的行，也不影响其他账号或范围外业务日。 + """ + + if not DB_ENABLED: + return 0 + with conn.cursor() as cursor: + cursor.execute( + """ + DELETE FROM jd_after_sales_issues + WHERE account = %s + AND business_date BETWEEN %s AND %s + AND apply_at IS NOT NULL + AND ( + (apply_at AT TIME ZONE 'Asia/Shanghai')::date < %s + OR (apply_at AT TIME ZONE 'Asia/Shanghai')::date > %s + ) + """, + ( + account, + business_date_start, + business_date_end, + business_date_start, + business_date_end, + ), + ) + return cursor.rowcount + + +# ── DY 客服会话与消息 ───────────────────────────────────────────────────────── + + +def _prepare_dy_customer_service_records(records: list[dict]) -> list[dict]: + cleaned_records: list[dict] = [] + for record in records: + cleaned = {key: _clean_val(value) for key, value in record.items()} + for json_column in ( + "raw_payload", + "card", + "raw_message", + "raw_card", + "title_variants", + ): + value = cleaned.get(json_column) + if isinstance(value, (dict, list)): + cleaned[json_column] = json.dumps( + value, ensure_ascii=False, default=str + ) + cleaned["updated_at"] = datetime.now().astimezone() + cleaned_records.append(cleaned) + return cleaned_records + + +def upsert_dy_customer_service_conversations(conn, records: list[dict]) -> int: + """幂等写入抖音客服会话；同一账号、业务日期和会话 ID 只保留一行。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "dy_customer_service_conversations", + _prepare_dy_customer_service_records(records), + ("account", "business_date", "conversation_id"), + ) + + +def upsert_dy_customer_service_messages(conn, records: list[dict]) -> int: + """幂等写入抖音客服消息，重复重跑不会产生重复消息。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "dy_customer_service_messages", + _prepare_dy_customer_service_records(records), + ("account", "business_date", "conversation_id", "message_key"), + ) + + +def upsert_dy_customer_service_conversation_orders( + conn, records: list[dict] +) -> int: + """幂等写入会话绑定的商品/订单（咨询订单面板卡片）。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "dy_customer_service_conversation_orders", + _prepare_dy_customer_service_records(records), + ("account", "business_date", "conversation_id", "order_id"), + ) + + +def fetch_dy_customer_service_products( + conn, product_codes: list[str] +) -> list[dict]: + """读取商品维表现有行，供采集端合并标题变体。""" + + if not DB_ENABLED or not product_codes: + return [] + with conn.cursor() as cur: + cur.execute( + """ + select product_code, product_title, first_seen_date, + last_seen_date, title_variants + from dy_customer_service_products + where product_code = any(%s) + """, + (list(product_codes),), + ) + rows = cur.fetchall() + return [ + { + "product_code": row[0], + "product_title": row[1], + "first_seen_date": row[2], + "last_seen_date": row[3], + "title_variants": row[4] + if isinstance(row[4], dict) + else json.loads(row[4] or "{}"), + } + for row in rows + ] + + +def upsert_dy_customer_service_products(conn, records: list[dict]) -> int: + """幂等写入商品维表（编码唯一，标题为出现频次最高的变体）。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "dy_customer_service_products", + _prepare_dy_customer_service_records(records), + ("product_code",), + ) + + +# ── DY 品质原因售后单（品退）与协商记录 ─────────────────────────────────────── + + +def _prepare_dy_after_sales_quality_records(records: list[dict]) -> list[dict]: + cleaned_records: list[dict] = [] + for record in records: + cleaned = {key: _clean_val(value) for key, value in record.items()} + for json_column in ("raw_row", "evidence_urls", "extra_fields"): + value = cleaned.get(json_column) + if isinstance(value, (dict, list)): + cleaned[json_column] = json.dumps( + value, ensure_ascii=False, default=str + ) + cleaned["updated_at"] = datetime.now().astimezone() + cleaned_records.append(cleaned) + return cleaned_records + + +def upsert_dy_after_sales_quality_orders(conn, records: list[dict]) -> int: + """幂等写入品质原因售后单；同一账号和售后编号只保留一行。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "dy_after_sales_quality_orders", + _prepare_dy_after_sales_quality_records(records), + ("account", "aftersale_id"), + ) + + +def upsert_dy_after_sales_quality_negotiations(conn, records: list[dict]) -> int: + """幂等写入售后单协商记录；同一售后单按条目序号覆盖更新。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "dy_after_sales_quality_negotiations", + _prepare_dy_after_sales_quality_records(records), + ("account", "aftersale_id", "entry_seq"), + ) + + # ── JD 竞店 ─────────────────────────────────────────────────────────────────── @@ -711,3 +1090,34 @@ def persist_dy_peer_store(json_path: str) -> int: rows = _build_dy_peer_store_rows(data, json_path) with get_conn() as conn: return upsert_dy_peer_store(conn, rows) + + +# ── DY 抖店差评 ─────────────────────────────────────────────────────────────── + + +def _prepare_dy_negative_review_records(records: list[dict]) -> list[dict]: + cleaned_records: list[dict] = [] + for record in records: + cleaned = {key: _clean_val(value) for key, value in record.items()} + for json_column in ("image_urls", "raw_review"): + value = cleaned.get(json_column) + if isinstance(value, (dict, list)): + cleaned[json_column] = json.dumps( + value, ensure_ascii=False, default=str + ) + cleaned["updated_at"] = datetime.now().astimezone() + cleaned_records.append(cleaned) + return cleaned_records + + +def upsert_dy_negative_reviews(conn, records: list[dict]) -> int: + """幂等写入抖音差评；同一账号、业务日期和评价（订单）ID 只保留一行。""" + + if not DB_ENABLED or not records: + return 0 + return _dynamic_batch_upsert( + conn, + "dy_negative_reviews", + _prepare_dy_negative_review_records(records), + ("account", "business_date", "review_id"), + ) diff --git a/src/gyxx_flow/modules/shop_intelligence/db/schema.sql b/src/gyxx_flow/modules/shop_intelligence/db/schema.sql index c14812a..e8c1531 100644 --- a/src/gyxx_flow/modules/shop_intelligence/db/schema.sql +++ b/src/gyxx_flow/modules/shop_intelligence/db/schema.sql @@ -289,3 +289,496 @@ CREATE INDEX IF NOT EXISTS idx_tm_shop_weekly_shop ON tm_shop_weekly (shop, week_end DESC); COMMENT ON TABLE tm_shop_weekly IS '天猫店铺周指标 — 一行=一周，对齐 jd_shop_weekly 设计。来源: taobao_sycm.py 采集生意参谋首页+市场排行+店铺评分。'; + + +-- ── 京东客服会话与消息 ─────────────────────────────────────────────────────── +-- 一次运行按业务日期筛选；消息实际发生时间单独保存，便于按天/会话回查。 +CREATE TABLE IF NOT EXISTS jd_customer_service_conversations ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + customer_id TEXT NOT NULL, + session_id TEXT NOT NULL, + session_start_at TIMESTAMPTZ, + session_end_at TIMESTAMPTZ, + message_count INTEGER NOT NULL DEFAULT 0, + source_url TEXT NOT NULL, + raw_session JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT jd_customer_service_conversation_uniq + UNIQUE (account, business_date, customer_id, session_id) +); +CREATE INDEX IF NOT EXISTS idx_jd_customer_service_conversation_date + ON jd_customer_service_conversations (business_date DESC, customer_id); +CREATE INDEX IF NOT EXISTS idx_jd_customer_service_conversation_session + ON jd_customer_service_conversations (session_id, business_date DESC); + +CREATE TABLE IF NOT EXISTS jd_customer_service_messages ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + customer_id TEXT NOT NULL, + session_id TEXT NOT NULL, + message_key TEXT NOT NULL, + message_uuid TEXT, + message_at TIMESTAMPTZ, + sender_name TEXT, + sender_role TEXT NOT NULL, + waiter_send BOOLEAN NOT NULL DEFAULT FALSE, + content TEXT, + message_type TEXT, + image_url TEXT, + close_session_time TIMESTAMPTZ, + source_url TEXT NOT NULL, + raw_message JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT jd_customer_service_message_uniq + UNIQUE (account, business_date, customer_id, session_id, message_key) +); +CREATE INDEX IF NOT EXISTS idx_jd_customer_service_message_date + ON jd_customer_service_messages (business_date DESC, customer_id, message_at); +CREATE INDEX IF NOT EXISTS idx_jd_customer_service_message_session + ON jd_customer_service_messages (session_id, message_at); + +COMMENT ON TABLE jd_customer_service_conversations IS + '京东客服聊天会话；business_date 是查询日，session_start_at/session_end_at 是消息实际时间。'; +COMMENT ON TABLE jd_customer_service_messages IS + '京东客服聊天消息；按账号、业务日期、顾客、会话和消息身份幂等保存。'; + + +-- ── 京东差评及商品信息 ────────────────────────────────────────────────────── +-- 评价页按业务日采集；review_at 保留京东评价实际发生时间。 +CREATE TABLE IF NOT EXISTS jd_negative_reviews ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + review_id TEXT NOT NULL, + review_at TIMESTAMPTZ, + review_level TEXT NOT NULL DEFAULT '差评', + score SMALLINT, + reviewer_nickname TEXT, + content TEXT, + order_id TEXT, + product_name TEXT, + sku_id TEXT, + sku_name TEXT, + product_color TEXT, + product_size TEXT, + product_image_url TEXT, + image_urls JSONB NOT NULL DEFAULT '[]'::jsonb, + is_vender_reply BOOLEAN NOT NULL DEFAULT FALSE, + reply_count INTEGER NOT NULL DEFAULT 0, + complaint_status INTEGER, + source_url TEXT NOT NULL, + raw_review JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT jd_negative_review_uniq + UNIQUE (account, business_date, review_id) +); +CREATE INDEX IF NOT EXISTS idx_jd_negative_review_date + ON jd_negative_reviews (business_date DESC, review_at DESC); +CREATE INDEX IF NOT EXISTS idx_jd_negative_review_sku + ON jd_negative_reviews (sku_id, business_date DESC); +COMMENT ON TABLE jd_negative_reviews IS + '京东差评及商品信息；按账号、业务日期和评价 ID 幂等保存。'; + + +-- ── 京东品退异常服务单 ────────────────────────────────────────────────────── +-- 只保存售后原因“商品出现异常”且用户原声不是未知的服务单； +-- 单日任务的 business_date 是页面查询日；范围任务按 apply_at 的上海日期落业务日。 +CREATE TABLE IF NOT EXISTS jd_after_sales_issues ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + service_id TEXT NOT NULL, + order_id TEXT, + product_name TEXT, + sku_id TEXT, + category_level_1 TEXT, + category_level_2 TEXT, + category_level_3 TEXT, + apply_at TIMESTAMPTZ, + reason_category TEXT, + reason_level_1 TEXT, + reason_level_2 TEXT, + expected_resolution TEXT, + user_original TEXT NOT NULL, + source_url TEXT NOT NULL, + raw_issue JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT jd_after_sales_issue_uniq + UNIQUE (account, business_date, service_id) +); +CREATE INDEX IF NOT EXISTS idx_jd_after_sales_issue_date + ON jd_after_sales_issues (business_date DESC, apply_at DESC); +CREATE INDEX IF NOT EXISTS idx_jd_after_sales_issue_sku + ON jd_after_sales_issues (sku_id, business_date DESC); +CREATE INDEX IF NOT EXISTS idx_jd_after_sales_issue_reason + ON jd_after_sales_issues (reason_category, business_date DESC); +COMMENT ON TABLE jd_after_sales_issues IS + '京东品退异常服务单；仅保留用户原声非未知记录，按账号、业务日期和服务单 ID 幂等保存；范围任务按申请日期落业务日。'; + + +-- ── 抖音客服会话与消息 ─────────────────────────────────────────────────────── +-- 飞鸽历史会话按业务日期查询；会话实际发生时间单独保存，便于按天/会话回查。 +CREATE TABLE IF NOT EXISTS dy_customer_service_conversations ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + conversation_id TEXT NOT NULL, + customer_nickname TEXT, + session_time_label TEXT, + session_start_at TIMESTAMPTZ, + session_end_at TIMESTAMPTZ, + consult_entry TEXT, + reception_type TEXT, + initiator_scene TEXT, + end_scene TEXT, + rating_time_label TEXT, + rating_result TEXT, + rating_tags TEXT, + conversation_intent TEXT, + smart_reception_label TEXT, + smart_rating_time_label TEXT, + smart_rating_result TEXT, + smart_rating_tags TEXT, + transfer_human_duration TEXT, + transfer_human_type TEXT, + service_problem_tags TEXT, + unmet_metrics TEXT, + message_count INTEGER NOT NULL DEFAULT 0, + source_url TEXT NOT NULL, + raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT dy_customer_service_conversation_uniq + UNIQUE (account, business_date, conversation_id) +); +CREATE INDEX IF NOT EXISTS idx_dy_customer_service_conversation_date + ON dy_customer_service_conversations (business_date DESC, customer_nickname); +CREATE INDEX IF NOT EXISTS idx_dy_customer_service_conversation_id + ON dy_customer_service_conversations (conversation_id, business_date DESC); +ALTER TABLE dy_customer_service_conversations + ADD COLUMN IF NOT EXISTS transfer_human_type TEXT; + +CREATE TABLE IF NOT EXISTS dy_customer_service_messages ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + conversation_id TEXT NOT NULL, + message_key TEXT NOT NULL, + message_seq INTEGER, + message_at TIMESTAMPTZ, + time_label TEXT, + sender_name TEXT, + sender_role TEXT NOT NULL, + message_type TEXT NOT NULL DEFAULT 'text', + content TEXT, + card JSONB NOT NULL DEFAULT '{}'::jsonb, + source_url TEXT NOT NULL, + raw_message JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT dy_customer_service_message_uniq + UNIQUE (account, business_date, conversation_id, message_key) +); +CREATE INDEX IF NOT EXISTS idx_dy_customer_service_message_date + ON dy_customer_service_messages (business_date DESC, conversation_id, message_seq); +CREATE INDEX IF NOT EXISTS idx_dy_customer_service_message_conversation + ON dy_customer_service_messages (conversation_id, message_seq); + +-- 会话抽屉右侧「咨询订单」面板：商品/订单与会话绑定保存。 +-- 每张订单卡一行，按账号、业务日期、会话和订单号幂等。 +CREATE TABLE IF NOT EXISTS dy_customer_service_conversation_orders ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + conversation_id TEXT NOT NULL, + order_id TEXT NOT NULL, + order_status TEXT, + after_sale_status TEXT, + order_badge TEXT, + product_title TEXT, + product_code TEXT, + spec TEXT, + spec_id TEXT, + paid_amount NUMERIC(12, 2), + paid_raw TEXT, + pay_channel TEXT, + paid_at TIMESTAMPTZ, + discount_text TEXT, + freight_text TEXT, + logistics_company TEXT, + logistics_no TEXT, + logistics_status TEXT, + receiver_info TEXT, + shipped_at TIMESTAMPTZ, + service_tags TEXT, + remark_text TEXT, + source_url TEXT NOT NULL, + raw_card JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT dy_customer_service_conversation_order_uniq + UNIQUE (account, business_date, conversation_id, order_id) +); +CREATE INDEX IF NOT EXISTS idx_dy_customer_service_conversation_order_date + ON dy_customer_service_conversation_orders (business_date DESC, conversation_id); +CREATE INDEX IF NOT EXISTS idx_dy_customer_service_conversation_order_code + ON dy_customer_service_conversation_orders (product_code, business_date DESC); +CREATE INDEX IF NOT EXISTS idx_dy_customer_service_conversation_order_id + ON dy_customer_service_conversation_orders (order_id); + +ALTER TABLE dy_customer_service_conversations + ADD COLUMN IF NOT EXISTS order_panel_text TEXT; +ALTER TABLE dy_customer_service_conversations + ADD COLUMN IF NOT EXISTS order_count INTEGER NOT NULL DEFAULT 0; + +-- 商品维表：从会话绑定的订单卡里提炼，编码唯一；事实表 +-- dy_customer_service_conversation_orders.product_code 关联本表。 +CREATE TABLE IF NOT EXISTS dy_customer_service_products ( + product_code TEXT PRIMARY KEY, + product_title TEXT NOT NULL, + first_seen_date DATE, + last_seen_date DATE, + title_variants JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +COMMENT ON TABLE dy_customer_service_products IS + '抖音客服会话涉及的商品维表；product_code 是订单卡上的商品编码，标题按出现频次收敛为权威值，title_variants 保留所有标题变体及次数。'; + +COMMENT ON TABLE dy_customer_service_conversations IS + '抖音飞鸽历史会话；business_date 是查询日，session_start_at/session_end_at 是会话实际时间。'; +COMMENT ON TABLE dy_customer_service_conversation_orders IS + '抖音飞鸽会话绑定商品/订单；来自会话抽屉右侧咨询订单面板，按账号、业务日期、会话和订单号幂等保存。'; +COMMENT ON TABLE dy_customer_service_messages IS + '抖音飞鸽会话消息；按账号、业务日期、会话和消息身份幂等保存。'; + + +-- ── 抖音品质原因售后单（品退）与协商记录 ────────────────────────────────────── +-- 售后工作台按“申请原因=商品品质原因”筛选；每条售后单按申请时间的上海日期 +-- 落业务日。范围/全量重跑共用一个幂等键 (account, aftersale_id)，重复执行 +-- 只更新对应行，不会因查询区间重叠产生重复数据。 +CREATE TABLE IF NOT EXISTS dy_after_sales_quality_orders ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + aftersale_id TEXT NOT NULL, + order_id TEXT, + aftersale_type TEXT, + status_tag TEXT, + aftersale_status TEXT, + apply_at TIMESTAMPTZ, + apply_reason TEXT, + apply_quantity INTEGER, + pay_amount NUMERIC(12, 2), + refund_amount NUMERIC(12, 2), + product_name TEXT, + product_id TEXT, + product_code TEXT, + spec_name TEXT, + negotiation_count INTEGER NOT NULL DEFAULT 0, + detail_url TEXT, + source_url TEXT NOT NULL, + raw_row JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT dy_after_sales_quality_order_uniq UNIQUE (account, aftersale_id) +); +CREATE INDEX IF NOT EXISTS idx_dy_after_sales_quality_order_date + ON dy_after_sales_quality_orders (business_date DESC, product_code); +CREATE INDEX IF NOT EXISTS idx_dy_after_sales_quality_order_product + ON dy_after_sales_quality_orders (product_id, business_date DESC); +CREATE INDEX IF NOT EXISTS idx_dy_after_sales_quality_order_order_id + ON dy_after_sales_quality_orders (order_id); + +-- 协商记录时间线：每条协商动作一行，保存退货原因、退款说明、凭证信息 +-- 图片 URL 及其余标签值字段；按账号、售后编号和条目序号幂等。 +CREATE TABLE IF NOT EXISTS dy_after_sales_quality_negotiations ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + aftersale_id TEXT NOT NULL, + entry_seq INTEGER NOT NULL, + actor_role TEXT, + action_text TEXT, + happened_at TIMESTAMPTZ, + operator TEXT, + return_reason TEXT, + refund_note TEXT, + evidence_urls JSONB NOT NULL DEFAULT '[]'::jsonb, + extra_fields JSONB NOT NULL DEFAULT '{}'::jsonb, + raw_text TEXT, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT dy_after_sales_quality_negotiation_uniq + UNIQUE (account, aftersale_id, entry_seq) +); +CREATE INDEX IF NOT EXISTS idx_dy_after_sales_quality_negotiation_entry + ON dy_after_sales_quality_negotiations (aftersale_id, entry_seq); + +COMMENT ON TABLE dy_after_sales_quality_orders IS + '抖音售后工作台商品品质原因售后单（品退）；business_date 按申请时间的上海日期归属，按账号和售后编号幂等保存，商品名称/商品ID/商家编码来自详情页交易信息。'; +COMMENT ON TABLE dy_after_sales_quality_negotiations IS + '抖音品退售后单协商记录；每条协商动作一行，保存退货原因、退款说明、凭证信息图片 URL 及其余标签值字段，按账号、售后编号和条目序号幂等保存。'; + + +-- ── 抖音差评 ──────────────────────────────────────────────────────────────── +-- 抖店评价管理-评价明细按评价时间逐日查询差评；review_id 取订单 ID。 +CREATE TABLE IF NOT EXISTS dy_negative_reviews ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + review_id TEXT NOT NULL, + order_id TEXT, + review_at TIMESTAMPTZ, + rating_level TEXT, + stars INTEGER, + reviewer_nickname TEXT, + content TEXT, + image_urls JSONB NOT NULL DEFAULT '[]'::jsonb, + display_status TEXT, + experience_score_excluded BOOLEAN NOT NULL DEFAULT false, + product_id TEXT, + product_name TEXT, + spec_name TEXT, + spec_id TEXT, + source_url TEXT NOT NULL, + raw_review JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT dy_negative_review_uniq + UNIQUE (account, business_date, review_id) +); +CREATE INDEX IF NOT EXISTS idx_dy_negative_review_date + ON dy_negative_reviews (business_date DESC); +CREATE INDEX IF NOT EXISTS idx_dy_negative_review_product + ON dy_negative_reviews (product_id, business_date DESC); +COMMENT ON TABLE dy_negative_reviews IS + '抖音差评及商品信息；business_date 是评价时间所属查询日，review_id 取订单 ID 幂等保存。'; + + +-- ── 天猫客服会话与消息 ─────────────────────────────────────────────────────── +-- 千牛聊天记录查询页按日期区间筛选；每次最多查询 31 天，消息实际时间单独保存。 +CREATE TABLE IF NOT EXISTS tmall_customer_service_conversations ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + query_start_date DATE NOT NULL, + query_end_date DATE NOT NULL, + customer_id TEXT NOT NULL, + customer_name TEXT NOT NULL, + session_id TEXT NOT NULL, + session_start_at TIMESTAMPTZ, + session_end_at TIMESTAMPTZ, + message_count INTEGER NOT NULL DEFAULT 0, + source_url TEXT NOT NULL, + raw_session JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT tmall_customer_service_conversation_uniq + UNIQUE ( + account, + query_start_date, + query_end_date, + customer_id, + session_id + ) +); +CREATE INDEX IF NOT EXISTS idx_tmall_customer_service_conversation_range + ON tmall_customer_service_conversations ( + query_start_date DESC, + query_end_date DESC, + customer_id + ); +CREATE INDEX IF NOT EXISTS idx_tmall_customer_service_conversation_session + ON tmall_customer_service_conversations (session_id, query_start_date DESC); + +CREATE TABLE IF NOT EXISTS tmall_customer_service_messages ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + query_start_date DATE NOT NULL, + query_end_date DATE NOT NULL, + customer_id TEXT NOT NULL, + customer_name TEXT NOT NULL, + session_id TEXT NOT NULL, + message_key TEXT NOT NULL, + message_at TIMESTAMPTZ, + sender_name TEXT, + sender_role TEXT NOT NULL, + content TEXT, + message_type TEXT NOT NULL DEFAULT 'text', + media_urls JSONB NOT NULL DEFAULT '[]'::jsonb, + source_url TEXT NOT NULL, + raw_message JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT tmall_customer_service_message_uniq + UNIQUE (account, customer_id, session_id, message_key) +); +CREATE INDEX IF NOT EXISTS idx_tmall_customer_service_message_date + ON tmall_customer_service_messages (business_date DESC, customer_id, message_at); +CREATE INDEX IF NOT EXISTS idx_tmall_customer_service_message_session + ON tmall_customer_service_messages (session_id, message_at); + +COMMENT ON TABLE tmall_customer_service_conversations IS + '天猫千牛客服会话；query_start_date/query_end_date 记录查询窗口，消息时间保留实际发生时间。'; +COMMENT ON TABLE tmall_customer_service_messages IS + '天猫千牛客服消息；按账号、顾客、会话和稳定消息指纹幂等保存。'; + + +-- ── 天猫负面评价 ─────────────────────────────────────────────────────────── +-- 千牛评价管理按控制台选择的评价时间范围查询负面评价；不保存订单号。 +CREATE TABLE IF NOT EXISTS tmall_negative_reviews ( + id BIGSERIAL PRIMARY KEY, + account TEXT NOT NULL, + business_date DATE NOT NULL, + review_id TEXT NOT NULL, + review_at TIMESTAMPTZ NOT NULL, + review_level TEXT NOT NULL DEFAULT '负面评价', + content TEXT, + image_urls JSONB NOT NULL DEFAULT '[]'::jsonb, + product_name TEXT NOT NULL, + source_url TEXT NOT NULL, + raw_review JSONB NOT NULL DEFAULT '{}'::jsonb, + captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT tmall_negative_review_uniq + UNIQUE (account, review_id) +); +CREATE INDEX IF NOT EXISTS idx_tmall_negative_review_date + ON tmall_negative_reviews (business_date DESC, review_at DESC); +CREATE INDEX IF NOT EXISTS idx_tmall_negative_review_product + ON tmall_negative_reviews (product_name, business_date DESC); +COMMENT ON TABLE tmall_negative_reviews IS + '天猫评价管理负面评价；保存评价文字、图片和商品名称，按账号及主评 ID 幂等更新，不保存订单号。'; diff --git a/src/gyxx_flow/modules/shop_intelligence/douyin_price_appeal.py b/src/gyxx_flow/modules/shop_intelligence/douyin_price_appeal.py index 27d8df6..8fd8a81 100644 --- a/src/gyxx_flow/modules/shop_intelligence/douyin_price_appeal.py +++ b/src/gyxx_flow/modules/shop_intelligence/douyin_price_appeal.py @@ -1252,11 +1252,52 @@ def _action_fingerprint(locator: Any) -> str: const readText = (node) => ( node ? (node.innerText || node.textContent || '').trim() : '' ); + const productIds = (text) => { + const matches = String(text || '').matchAll( + /(?:商品\s*)?ID\s*[:：]?\s*(\d{8,})/ig + ); + return [...new Set([...matches].map((match) => match[1]))]; + }; + const isVisible = (node) => { + if (!node) return false; + const style = window.getComputedStyle(node); + const rect = node.getBoundingClientRect(); + return style.display !== 'none' + && style.visibility !== 'hidden' + && rect.width > 0 + && rect.height > 0; + }; + let productScopeText = ''; + let current = actionRow || clickable; + let depth = 0; + while (current && depth < 12) { + const currentText = readText(current); + if (productIds(currentText).length === 1) { + productScopeText = currentText; + break; + } + current = current.parentElement; + depth += 1; + } + if (!productScopeText) { + const elements = [...document.querySelectorAll('body *')]; + const actionIndex = elements.indexOf(clickable); + for (let index = actionIndex - 1; index >= 0; index -= 1) { + const candidate = elements[index]; + if (!isVisible(candidate)) continue; + const candidateText = readText(candidate); + if (productIds(candidateText).length === 1) { + productScopeText = candidateText; + break; + } + } + } return { log_extra: clickable.getAttribute('data-log-extra'), row_key: actionRow ? actionRow.getAttribute('data-row-key') : null, action_text: readText(actionContainer), previous_text: readText(productHeader), + product_scope_text: productScopeText, }; } """ @@ -1302,7 +1343,19 @@ def _action_fingerprint(locator: Any) -> str: label="product_id", ) else: - product_id = _extract_item_id(str(payload.get("previous_text") or "")) + product_id = next( + ( + candidate + for context in ( + payload.get("previous_text"), + payload.get("product_scope_text"), + ) + if (candidate := _try_extract_item_id(str(context or ""))) + ), + None, + ) + if product_id is None: + raise PriceAppealError("待改价商品行缺少可验证的商品 ID") action_text = normalize_action_fingerprint( str(payload.get("action_text") or "") @@ -1327,10 +1380,15 @@ def _stable_action_identifier(value: Any, *, label: str) -> str: def _extract_item_id(text: str) -> str: - match = re.search(r"(?:商品\s*)?ID\s*[:：]?\s*(\d{8,})", text, re.IGNORECASE) - if match is None: + product_id = _try_extract_item_id(text) + if product_id is None: raise PriceAppealError("待改价商品行缺少可验证的商品 ID") - return match.group(1) + return product_id + + +def _try_extract_item_id(text: str) -> str | None: + match = re.search(r"(?:商品\s*)?ID\s*[:：]?\s*(\d{8,})", text, re.IGNORECASE) + return match.group(1) if match else None def _verify_item_identity(panel: Any, item_id: str) -> None: diff --git a/src/gyxx_flow/modules/supply_chain/orchestrator/scripts/ProductReplenishment.py b/src/gyxx_flow/modules/supply_chain/orchestrator/scripts/ProductReplenishment.py index 9cb5f97..7c3ae59 100644 --- a/src/gyxx_flow/modules/supply_chain/orchestrator/scripts/ProductReplenishment.py +++ b/src/gyxx_flow/modules/supply_chain/orchestrator/scripts/ProductReplenishment.py @@ -80,7 +80,6 @@ SPECIAL_SKU_STOCK_THRESHOLDS = { "10440003": 100, "10362004": 300, "10362007": 300, - "10362008": 300, "10362010": 300, "10362011": 300, } diff --git a/src/gyxx_flow/notification_routing.py b/src/gyxx_flow/notification_routing.py index fd0ec45..3fb5e28 100644 --- a/src/gyxx_flow/notification_routing.py +++ b/src/gyxx_flow/notification_routing.py @@ -55,6 +55,7 @@ NOTIFICATION_CAPABILITIES: Mapping[str, str] = MappingProxyType( "content.relogin.weekly": "刷新期间投递各平台扫码二维码，并在平台失败时投递失败汇总。", "product.alert.daily": "命中销量下滑规则且尚未重复通知时投递告警。", "product.market_rank": "存在本次新生成的平台报告链接且尚未重复通知时投递。", + "product.online_stock_alert.daily": "线上商品当前库存低于 3 时向配置接收人发送库存预警。", "supply.purchase_confirmation.daily": "采购确认结果生成完成后投递最终业务通知。", "supply.replenishment.weekly": "补货处理完成后投递最终结果，包括无新增补货项的结果。", "supply.replenishment_alert.daily": "存在命中库存阈值的 SKU 时投递预警。", diff --git a/src/gyxx_flow/security/scanner.py b/src/gyxx_flow/security/scanner.py index 33f4d93..f73372c 100644 --- a/src/gyxx_flow/security/scanner.py +++ b/src/gyxx_flow/security/scanner.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from pathlib import Path DEFAULT_EXCLUDED_DIRECTORIES = frozenset( - {"var", ".git", ".venv", ".learnings", "build", "dist"} + {"var", ".git", ".venv", ".learnings", "build", "dist", ".playwright-cli"} ) _ASSIGNMENT_PATTERN = re.compile( diff --git a/src/gyxx_flow/web/app.css b/src/gyxx_flow/web/app.css index 1ae1f80..bc32bee 100644 --- a/src/gyxx_flow/web/app.css +++ b/src/gyxx_flow/web/app.css @@ -4134,6 +4134,23 @@ kbd { box-shadow: 0 0 0 3px rgba(49, 87, 213, 0.08); } +.run-date-range { + display: flex; + align-items: center; + gap: 8px; +} + +.run-date-range input { + min-width: 0; + flex: 1 1 0; +} + +.run-date-range > span { + color: var(--muted); + font-size: 11px; + white-space: nowrap; +} + .field small { margin-top: 5px; color: var(--muted); diff --git a/src/gyxx_flow/web/app.js b/src/gyxx_flow/web/app.js index 4d6d6a8..9a2c9de 100644 --- a/src/gyxx_flow/web/app.js +++ b/src/gyxx_flow/web/app.js @@ -128,6 +128,49 @@ return workflow?.id === "product.tmall_baibu_apply"; } + function isDateRangeWorkflow(workflow) { + return workflow?.date_range_mode === "date_range"; + } + + function runStoreConfig(workflow) { + if (isDateRangeWorkflow(workflow) && workflow.store_config) { + return workflow.store_config; + } + if (workflow?.id === "product.video_upload") { + return { + label: "上传店铺", + hint: "先扫描飞书并按所选店铺规则筛选；存在未上传附件时才登录该店铺账号。", + options: [ + { value: "flagship", label: "光影行星旗舰店" }, + { value: "luggage", label: "光影行星鑫华达专卖店:龙虾仔" }, + ], + }; + } + return null; + } + + function syncRunStoreField(workflow) { + const field = byId("run-store-field"); + const select = byId("run-store"); + const config = runStoreConfig(workflow); + field.hidden = !config; + if (!config) { + select.replaceChildren(); + return; + } + byId("run-store-label").textContent = config.label || "店铺"; + byId("run-store-hint").textContent = config.hint || "选择本次运行使用的店铺。"; + select.replaceChildren( + ...(config.options || []).map((option) => { + const element = document.createElement("option"); + element.value = option.value; + element.textContent = option.label; + return element; + }), + ); + select.value = config.options?.[0]?.value || ""; + } + function runPeriodLabel(workflow) { return isRepeatableTmallBaibu(workflow) ? "运行记录日期" : "业务日期"; } @@ -274,6 +317,11 @@ function syncRunPeriodHint() { const workflow = selectedWorkflow(); + if (isDateRangeWorkflow(workflow)) { + byId("run-date-hint").textContent = + workflow.date_range_hint || "按所选日期范围采集；平台会按可查询区间拆分。"; + return; + } if (workflow?.run_date_mode !== "month") return; const range = naturalMonthRange(byId("run-business-date").value); byId("run-date-hint").textContent = range @@ -312,6 +360,7 @@ source: "console", label: String(operationId), businessDate: workflow.active_run?.business_date || "正在确认", + businessDateEnd: workflow.active_run?.business_date_end || null, }; } if (workflow?.scheduler_job?.status === "running" && workflow.scheduler_job?.cancellable) { @@ -322,6 +371,7 @@ source: "scheduler", label: "项目定时调度", businessDate: workflow.scheduler_job.business_date || "正在确认", + businessDateEnd: null, }; } return null; @@ -3167,6 +3217,7 @@ const isAllShopDaily = workflow.id === "product.erp_all_shop_daily"; const isNoteInventory = isNotesMasterWorkflow(workflow); const repeatableTmallBaibu = isRepeatableTmallBaibu(workflow); + const dateRangeMode = isDateRangeWorkflow(workflow); byId("run-modal-title").textContent = isNoteInventory ? `同步「${workflow.name}」` : isAllShopDaily @@ -3175,36 +3226,52 @@ byId("run-workflow-id").textContent = workflow.id; const monthMode = workflow.run_date_mode === "month"; const periodInput = byId("run-business-date"); + const periodEndInput = byId("run-business-date-end"); byId("run-period-field").hidden = repeatableTmallBaibu; periodInput.required = !repeatableTmallBaibu; + periodEndInput.hidden = !dateRangeMode; + byId("run-date-range-separator").hidden = !dateRangeMode; + periodEndInput.required = dateRangeMode; const defaultDate = todayForTimezone(workflow.schedule?.business_date_offset_days || 0); periodInput.type = monthMode ? "month" : "date"; periodInput.value = monthMode ? defaultDate.slice(0, 7) : defaultDate; + periodEndInput.value = defaultDate; + const dateRangeConstraints = workflow.date_range_constraints || null; + for (const input of [periodInput, periodEndInput]) { + if (dateRangeMode && dateRangeConstraints) { + input.min = dateRangeConstraints.min; + input.max = dateRangeConstraints.max; + } else { + input.removeAttribute("min"); + input.removeAttribute("max"); + } + } byId("run-period-label").textContent = isNoteInventory ? "运行记录日期" : repeatableTmallBaibu ? "运行记录日期" : monthMode ? "业务月份" - : isAllShopDaily ? "补采日期" : "业务日期"; + : isAllShopDaily ? "补采日期" + : dateRangeMode ? "执行开始日期" + : "业务日期"; byId("run-date-hint").textContent = isNoteInventory ? "仅用于标记本次运行记录；正式同步始终扫描当前各款式的合作达人和自营笔记表。" : repeatableTmallBaibu ? "仅用于平台内部记录和归档，不参与百补导入防重；每次都会重新下载并提交。" : isAllShopDaily ? "默认选择昨天；也可以选择任意一天，正式执行后将全选聚水潭全部店铺并幂等覆盖该日款式数据。" + : dateRangeMode + ? (workflow.date_range_hint || "按所选店铺和日期范围逐日采集；每个业务日独立幂等。") : workflow.schedule?.business_date_offset_days ? `已按计划偏移 ${workflow.schedule.business_date_offset_days} 天预填；手动选择的是业务日期。` : "本次数据处理所对应的业务日期。"; - const storeField = byId("run-store-field"); - const supportsStoreSelection = workflow.id === "product.video_upload"; - storeField.hidden = !supportsStoreSelection; - byId("run-store").value = "flagship"; + syncRunStoreField(workflow); syncTmallTopicField(workflow); syncTmallBaibuImportField(workflow); byId("run-preview-title").textContent = isNoteInventory ? "同步预演" : "安全预演"; byId("run-preview-description").textContent = isNoteInventory ? "检查执行计划，不写入数据库" : "检查流程但不执行外部效果"; - if (monthMode) syncRunPeriodHint(); + if (monthMode || dateRangeMode) syncRunPeriodHint(); document.querySelector(`input[name="execute"][value="${forceRefresh ? "true" : "false"}"]`).checked = true; byId("run-shadow").checked = false; byId("run-confirmed").checked = false; @@ -3280,7 +3347,9 @@ byId("stop-modal-title").textContent = `停止「${workflow.name}」`; byId("stop-workflow-id").textContent = workflow.id; byId("stop-workflow-name").textContent = workflow.name; - byId("stop-business-date").textContent = operation.businessDate; + byId("stop-business-date").textContent = operation.businessDateEnd + ? `${operation.businessDate} 至 ${operation.businessDateEnd}` + : operation.businessDate; byId("stop-operation-id").textContent = operation.label; byId("stop-form-error").hidden = true; syncStopDialogSubmitting(false); @@ -3382,10 +3451,13 @@ const execute = document.querySelector('input[name="execute"]:checked')?.value === "true"; const forceRefresh = !isRepeatableTmallBaibu(workflow) && byId("run-force").checked; const repeatableTmallBaibu = isRepeatableTmallBaibu(workflow); + const dateRangeMode = isDateRangeWorkflow(workflow); const selectedPeriod = byId("run-business-date").value; + const selectedPeriodEnd = byId("run-business-date-end").value; const monthMode = workflow.run_date_mode === "month"; const businessDate = repeatableTmallBaibu ? todayForTimezone(0) : monthMode ? `${selectedPeriod}-01` : selectedPeriod; - const store = workflow.id === "product.video_upload" ? byId("run-store").value : null; + const businessDateEnd = dateRangeMode ? selectedPeriodEnd : null; + const store = runStoreConfig(workflow) ? byId("run-store").value : null; const tmallTopicKeyword = workflow.id === "product.video_upload" ? byId("run-tmall-topic-keyword").value.trim() : null; @@ -3394,6 +3466,7 @@ : null; const errorBox = byId("run-form-error"); byId("run-business-date").removeAttribute("aria-invalid"); + byId("run-business-date-end").removeAttribute("aria-invalid"); errorBox.hidden = true; if (!repeatableTmallBaibu && !selectedPeriod) { byId("run-business-date").setAttribute("aria-invalid", "true"); @@ -3402,6 +3475,21 @@ byId("run-business-date").focus(); return; } + if (dateRangeMode && !selectedPeriodEnd) { + byId("run-business-date-end").setAttribute("aria-invalid", "true"); + errorBox.textContent = "请选择执行结束日期"; + errorBox.hidden = false; + byId("run-business-date-end").focus(); + return; + } + if (dateRangeMode && selectedPeriodEnd < selectedPeriod) { + byId("run-business-date").setAttribute("aria-invalid", "true"); + byId("run-business-date-end").setAttribute("aria-invalid", "true"); + errorBox.textContent = "结束日期不能早于开始日期"; + errorBox.hidden = false; + byId("run-business-date-end").focus(); + return; + } state.launchingWorkflows.add(workflow.id); if (forceRefresh) state.forceLaunchingWorkflows.add(workflow.id); renderModuleNav(); @@ -3413,6 +3501,7 @@ method: "POST", body: { business_date: businessDate, + ...(businessDateEnd ? { business_date_end: businessDateEnd } : {}), execute, shadow: byId("run-shadow").checked, confirmed: execute && byId("run-confirmed").checked, @@ -3427,7 +3516,9 @@ }, }); closeRun(); - const storeLabel = store === "flagship" ? "光影行星旗舰店" : store === "luggage" ? "光影行星鑫华达专卖店:龙虾仔" : ""; + const storeLabel = store + ? byId("run-store").selectedOptions[0]?.textContent || store + : ""; const topicLabel = tmallTopicKeyword ? `话题关键词 ${tmallTopicKeyword}` : ""; const tmallBaibuImportLabel = tmallBaibuImportMode ? byId("run-tmall-baibu-mode").selectedOptions[0]?.textContent || "" @@ -3436,11 +3527,12 @@ const successTitle = forceRefresh ? noteInventory ? "已提交强制同步" : "已提交强制重跑" : noteInventory ? "已提交笔记清单同步" : "已提交运行"; - const periodLabel = isRepeatableTmallBaibu(workflow) ? "运行记录" : "业务日期"; + const periodLabel = isRepeatableTmallBaibu(workflow) ? "运行记录" : dateRangeMode ? "执行日期范围" : "业务日期"; + const periodText = dateRangeMode ? `${businessDate} 至 ${businessDateEnd}` : businessDate; toast( "success", successTitle, - `${execute ? forceRefresh ? "强制重跑" : noteInventory ? "正式同步" : "正式执行" : noteInventory ? "同步预演" : "安全预演"}${storeLabel ? ` · ${storeLabel}` : ""}${topicLabel ? ` · ${topicLabel}` : ""}${tmallBaibuImportLabel ? ` · ${tmallBaibuImportLabel}` : ""} · ${periodLabel} ${businessDate} · 操作 ${result.operation_id}`, + `${execute ? forceRefresh ? "强制重跑" : noteInventory ? "正式同步" : "正式执行" : noteInventory ? "同步预演" : "安全预演"}${storeLabel ? ` · ${storeLabel}` : ""}${topicLabel ? ` · ${topicLabel}` : ""}${tmallBaibuImportLabel ? ` · ${tmallBaibuImportLabel}` : ""} · ${periodLabel} ${periodText} · 操作 ${result.operation_id}`, ); await loadOverview({ silent: true }); openDrawer(workflow.id, "steps"); diff --git a/src/gyxx_flow/web/index.html b/src/gyxx_flow/web/index.html index 4d9a66e..8e7bd53 100644 --- a/src/gyxx_flow/web/index.html +++ b/src/gyxx_flow/web/index.html @@ -462,17 +462,21 @@