wangyunlong/gyxx-flow
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: [workflow] 提交工作区代码

Browse Source
This commit is contained in:
wangyunlong committed 2026-09-30 10:46:15 +08:00
1 parent 2ff588a65d
commit 01f3d7aaf7
131 files changed
+24306 -1320
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

config/commands.json
+122 -2
View File
@@ -25,12 +25,24 @@
"entry": "chanmama_scraper.py",
"kind": "python"
},
{
"id": "content.metrics.collect_self_xiaohongshu",
"module": "content_marketing",
"entry": "self_xhs_scraper.py",
"kind": "python"
},
{
"id": "content.metrics.sync",
"module": "content_marketing",
"entry": "data/tools/sync_metrics_to_cmt_notes.py",
"kind": "python"
},
{
"id": "content.metrics.sync_self_video_metrics",
"module": "content_marketing",
"entry": "data/tools/sync_self_video_metrics.py",
"kind": "python"
},
{
"id": "content.marketing_report.generate",
"module": "content_marketing",
@@ -137,6 +149,98 @@
"--headless"
]
},
{
"id": "shop.jd_customer_service.collect",
"module": "shop_intelligence",
"entry": "collectors/jd_customer_service_messages.py",
"kind": "python",
"default_args": [
"--date",
"{business_date}",
"--headless"
]
},
{
"id": "shop.jd_negative_reviews.collect",
"module": "shop_intelligence",
"entry": "collectors/jd_negative_reviews.py",
"kind": "python",
"default_args": [
"--date",
"{business_date}",
"--headless"
]
},
{
"id": "shop.jd_after_sales_issues.collect",
"module": "shop_intelligence",
"entry": "collectors/jd_after_sales_issues.py",
"kind": "python",
"default_args": [
"--date",
"{business_date}",
"--headless"
]
},
{
"id": "shop.douyin_customer_service.collect",
"module": "shop_intelligence",
"entry": "collectors/dy_customer_service_messages.py",
"kind": "python",
"default_args": [
"--date",
"{business_date}",
"--headless"
]
},
{
"id": "shop.tmall_negative_reviews.collect",
"module": "shop_intelligence",
"entry": "collectors/tmall_negative_reviews.py",
"kind": "python",
"default_args": [
"--start-date",
"{business_date}",
"--end-date",
"{business_date}",
"--headless"
]
},
{
"id": "shop.tmall_customer_service.collect",
"module": "shop_intelligence",
"entry": "collectors/tmall_customer_service_messages.py",
"kind": "python",
"default_args": [
"--start-date",
"{business_date}",
"--end-date",
"{business_date}",
"--headless"
]
},
{
"id": "shop.dy_negative_reviews.collect",
"module": "shop_intelligence",
"entry": "collectors/dy_negative_reviews.py",
"kind": "python",
"default_args": [
"--date",
"{business_date}",
"--headless"
]
},
{
"id": "shop.douyin_after_sales_quality.collect",
"module": "shop_intelligence",
"entry": "collectors/dy_after_sales_quality.py",
"kind": "python",
"default_args": [
"--date",
"{business_date}",
"--headless"
]
},
{
"id": "shop.douyin_price_appeal",
"module": "shop_intelligence",
@@ -151,7 +255,11 @@
"id": "product.persona.collect",
"module": "product_commerce",
"entry": "run_daily_persona.py",
"kind": "python"
"kind": "python",
"default_args": [
"--date",
"{business_date}"
]
},
{
"id": "product.daily.orchestrate",
@@ -166,6 +274,13 @@
"kind": "python",
"notification_workflow_id": "product.alert.daily"
},
{
"id": "product.online_stock_alert.run",
"module": "product_commerce",
"entry": "online_stock_alert.py",
"kind": "python",
"notification_workflow_id": "product.online_stock_alert.daily"
},
{
"id": "product.import.daily",
"module": "product_commerce",
@@ -176,7 +291,12 @@
"id": "product.tmall_wanxiang_ads.collect",
"module": "product_commerce",
"entry": "collect_tmall_wanxiang_ads.py",
"kind": "python"
"kind": "python",
"default_args": [
"--date",
"{business_date}",
"--headless"
]
},
{
"id": "product.import.tmall_ads",
config/notification-routing.json
+37 -2
View File
@@ -14,6 +14,17 @@
}
},
"people": {
"online_stock_alert_recipient": {
"name": "线上库存预警接收人",
"mobile": null,
"bindings": {
"hermes-analyzer": {
"open_id": "ou_cd858437acecee9404c9920d2dc04b2a",
"verified": true,
"source": "mobile_lookup"
}
}
},
"he_yingwei": {
"name": "何颖威",
"mobile": null,
@@ -27,7 +38,7 @@
},
"wang_yunlong": {
"name": "王云龙",
"mobile": null,
"mobile": "+8613663785145",
"bindings": {
"collector": {
"open_id": "ou_8ee224968aa26a74c7d30ba27fed5eeb",
@@ -194,6 +205,17 @@
"source": "provided"
}
}
},
"stock_alert_user_19832771651": {
"name": "19832771651",
"mobile": "+8619832771651",
"bindings": {
"hermes-analyzer": {
"open_id": "ou_9136c3203c698fead5477eea827f2bcf",
"verified": true,
"source": "mobile_lookup"
}
}
}
},
"capabilities": {
@@ -209,6 +231,9 @@
"product.market_rank": {
"condition": "存在本次新生成的平台报告链接且尚未重复通知时投递。"
},
"product.online_stock_alert.daily": {
"condition": "线上商品当前库存低于 3 时向配置接收人发送库存预警。"
},
"supply.purchase_confirmation.daily": {
"condition": "采购确认结果生成完成后投递最终业务通知。"
},
@@ -219,5 +244,15 @@
"condition": "存在命中库存阈值的 SKU 时投递预警。"
}
},
"routes": {}
"routes": {
"product.online_stock_alert.daily": {
"enabled": true,
"app_profile": "hermes-analyzer",
"person_ids": [
"online_stock_alert_recipient",
"wang_yunlong",
"stock_alert_user_19832771651"
]
}
}
}
config/runtime-bindings.json
+203 -2
View File
@@ -33,7 +33,10 @@
"product_commerce:dy_product_scraping.py",
"product_commerce:vendors/dy-data-flow/dy_store_competitor_store_scraping.py",
"shop.douyin_price_appeal",
"content_marketing:xingtu_scraper_v2.py"
"content_marketing:xingtu_scraper_v2.py",
"shop.douyin_customer_service.collect",
"shop.douyin_after_sales_quality.collect",
"shop.dy_negative_reviews.collect"
]
},
"jd-shop": {
@@ -50,6 +53,9 @@
"JD_SHOP",
"JD_PASSWORD"
],
"login_identity_markers": [
"光影行星 GYXX官方旗舰店"
],
"keepalive": {
"enabled": true,
"url": "https://shop.jd.com/jdm/home",
@@ -68,11 +74,15 @@
"product_commerce:vendors/jd-data-flow/jd_data_collector.py",
"product_commerce:vendors/jd-data-flow/jd_product_data_collector.py",
"shop_intelligence:collectors/jd_data_collector.py",
"shop_intelligence:collectors/jd_peer_store_data_collector.py"
"shop_intelligence:collectors/jd_peer_store_data_collector.py",
"shop.jd_customer_service.collect",
"shop.jd_negative_reviews.collect",
"shop.jd_after_sales_issues.collect"
]
},
"jd-self-operated": {
"cdp_port": 22202,
"collection_cdp_port": 22155,
"login_mode": "D",
"required_cookie_domains": [
"jd.com"
@@ -208,6 +218,8 @@
"product_commerce:taobao_sycm_products.py",
"product_commerce:taobao_wanxiang_ai_creative_report.py",
"shop_intelligence:collectors/taobao_sycm.py",
"shop.tmall_customer_service.collect",
"shop.tmall_negative_reviews.collect",
"product.tmall_baibu_apply.old_chain_bag",
"product.tmall_baibu_apply.old_all_bag",
"product.tmall_baibu_apply.old_all_3c",
@@ -768,6 +780,25 @@
],
"cdp_port": 22036
},
"content.metrics.sync_self_video_metrics": {
"script_id": "content_marketing:data/tools/sync_self_video_metrics.py",
"state_key": "content_marketing:data/tools/sync_self_video_metrics.py",
"aliases": [
"content_marketing:data/tools/sync_self_video_metrics.py"
],
"cdp_port": 22164
},
"content.metrics.collect_self_xiaohongshu": {
"script_id": "content_marketing:self_xhs_scraper.py",
"state_key": "content_marketing:self_xhs_scraper.py",
"aliases": [
"content_marketing:self_xhs_scraper.py"
],
"cdp_port": 22165,
"login_mode": "C",
"required_cookie_domains": ["xiaohongshu.com"],
"credential_env_names": ["XHS_ARK_ACCOUNT"]
},
"content_marketing:data/tools/sync_style_categories.py": {
"script_id": "content_marketing:data/tools/sync_style_categories.py",
"state_key": "content_marketing:data/tools/sync_style_categories.py",
@@ -1177,6 +1208,15 @@
],
"cdp_port": 22069
},
"product.online_stock_alert.run": {
"script_id": "product_commerce:online_stock_alert.py",
"state_key": "product_commerce:online_stock_alert.py",
"aliases": [
"product_commerce:online_stock_alert.py",
"product.online_stock_alert.run"
],
"cdp_port": 22163
},
"product_commerce:db/sync_dim_style.py": {
"script_id": "product_commerce:db/sync_dim_style.py",
"state_key": "product_commerce:db/sync_dim_style.py",
@@ -1604,6 +1644,46 @@
],
"account": "douyin-shop"
},
"shop.douyin_customer_service.collect": {
"script_id": "shop_intelligence:collectors/dy_customer_service_messages.py",
"state_key": "shop_intelligence:collectors/dy_customer_service_messages.py",
"aliases": [
"shop_intelligence:collectors/dy_customer_service_messages.py"
],
"cdp_port": 22158,
"login_mode": "D",
"required_cookie_domains": [
"jinritemai.com"
],
"required_cookie_names": [
"sessionid",
"sid_tt",
"uid_tt",
"sessionid_ss",
"sid_guard"
],
"account": "douyin-shop"
},
"shop.douyin_after_sales_quality.collect": {
"script_id": "shop_intelligence:collectors/dy_after_sales_quality.py",
"state_key": "shop_intelligence:collectors/dy_after_sales_quality.py",
"aliases": [
"shop_intelligence:collectors/dy_after_sales_quality.py"
],
"cdp_port": 22159,
"login_mode": "D",
"required_cookie_domains": [
"jinritemai.com"
],
"required_cookie_names": [
"sessionid",
"sid_tt",
"uid_tt",
"sessionid_ss",
"sid_guard"
],
"account": "douyin-shop"
},
"shop_intelligence:collectors/jd_data_collector.py": {
"script_id": "shop_intelligence:collectors/jd_data_collector.py",
"state_key": "shop_intelligence:collectors/jd_data_collector.py",
@@ -1666,6 +1746,107 @@
"JD_SELF_OPERATED_PASSWORD"
]
},
"shop.jd_customer_service.collect": {
"script_id": "shop_intelligence:collectors/jd_customer_service_messages.py",
"state_key": "shop_intelligence:collectors/jd_customer_service_messages.py",
"aliases": [
"shop_intelligence:collectors/jd_customer_service_messages.py"
],
"cdp_port": 22154,
"login_mode": "D",
"required_cookie_domains": [
"jd.com"
],
"required_cookie_names": [
"pin",
"thor"
],
"credential_env_names": [
"JD_SHOP",
"JD_PASSWORD"
],
"account": "jd-shop"
},
"shop.tmall_negative_reviews.collect": {
"script_id": "shop_intelligence:collectors/tmall_negative_reviews.py",
"state_key": "shop_intelligence:collectors/tmall_negative_reviews.py",
"aliases": [
"shop_intelligence:collectors/tmall_negative_reviews.py"
],
"cdp_port": 22162,
"login_mode": "D",
"required_cookie_domains": [
"taobao.com",
"alimama.com"
],
"required_cookie_names": [
"unb",
"cookie2",
"_tb_token_"
],
"account": "tmall-shop"
},
"shop.tmall_customer_service.collect": {
"script_id": "shop_intelligence:collectors/tmall_customer_service_messages.py",
"state_key": "shop_intelligence:collectors/tmall_customer_service_messages.py",
"aliases": [
"shop_intelligence:collectors/tmall_customer_service_messages.py"
],
"cdp_port": 22161,
"login_mode": "D",
"required_cookie_domains": [
"taobao.com",
"alimama.com"
],
"required_cookie_names": [
"unb",
"cookie2",
"_tb_token_"
],
"account": "tmall-shop"
},
"shop.jd_negative_reviews.collect": {
"script_id": "shop_intelligence:collectors/jd_negative_reviews.py",
"state_key": "shop_intelligence:collectors/jd_negative_reviews.py",
"aliases": [
"shop_intelligence:collectors/jd_negative_reviews.py"
],
"cdp_port": 22156,
"login_mode": "D",
"required_cookie_domains": [
"jd.com"
],
"required_cookie_names": [
"pin",
"thor"
],
"credential_env_names": [
"JD_SHOP",
"JD_PASSWORD"
],
"account": "jd-shop"
},
"shop.jd_after_sales_issues.collect": {
"script_id": "shop_intelligence:collectors/jd_after_sales_issues.py",
"state_key": "shop_intelligence:collectors/jd_after_sales_issues.py",
"aliases": [
"shop_intelligence:collectors/jd_after_sales_issues.py"
],
"cdp_port": 22157,
"login_mode": "D",
"required_cookie_domains": [
"jd.com"
],
"required_cookie_names": [
"pin",
"thor"
],
"credential_env_names": [
"JD_SHOP",
"JD_PASSWORD"
],
"account": "jd-shop"
},
"shop_intelligence:collectors/taobao_sycm.py": {
"script_id": "shop_intelligence:collectors/taobao_sycm.py",
"state_key": "shop_intelligence:collectors/taobao_sycm.py",
@@ -2053,6 +2234,26 @@
"TMALL_BAG_PASSWORD"
],
"account": "tmall-bag"
},
"shop.dy_negative_reviews.collect": {
"script_id": "shop_intelligence:collectors/dy_negative_reviews.py",
"state_key": "shop_intelligence:collectors/dy_negative_reviews.py",
"aliases": [
"shop_intelligence:collectors/dy_negative_reviews.py"
],
"cdp_port": 22160,
"login_mode": "D",
"required_cookie_domains": [
"jinritemai.com"
],
"required_cookie_names": [
"sessionid",
"sid_tt",
"uid_tt",
"sessionid_ss",
"sid_guard"
],
"account": "douyin-shop"
}
},
"services": {
config/schedules.json
+114 -1
View File
@@ -10,6 +10,16 @@
"kind": "daily",
"workflow_id": "content.metrics.daily"
},
{
"at": [
"01:00",
"05:00"
],
"business_date_offset_days": 0,
"enabled": true,
"kind": "daily",
"workflow_id": "content.metrics.self_operated.daily"
},
{
"at": "14:00",
"business_date_offset_days": 0,
@@ -51,21 +61,27 @@
},
{
"at": "10:00",
"business_date_offset_days": 0,
"days": [
"Tuesday"
],
"enabled": false,
"kind": "weekly",
"workflow_id": "content.summary.weekly"
},
{
"at": "08:00",
"business_date_offset_days": 0,
"day_of_month": 1,
"enabled": false,
"kind": "monthly",
"workflow_id": "content.summary.monthly"
},
{
"at": "08:30",
"business_date_offset_days": 0,
"day_of_month": 1,
"enabled": false,
"kind": "monthly",
"workflow_id": "content.creator_report.monthly"
},
@@ -87,9 +103,30 @@
},
{
"at": [
"00:00",
"01:00",
"02:00",
"03:00",
"04:00",
"05:00",
"06:00",
"07:00",
"08:00",
"09:00",
"10:00",
"11:00",
"12:00",
"13:00",
"14:00",
"15:00",
"16:00",
"22:00"
"17:00",
"18:00",
"19:00",
"20:00",
"21:00",
"22:00",
"23:00"
],
"kind": "daily",
"workflow_id": "shop.douyin_price_appeal"
@@ -101,6 +138,77 @@
"kind": "daily",
"workflow_id": "shop.jd_self_operated.daily"
},
{
"at": "03:00",
"business_date_offset_days": 0,
"day_of_month": 21,
"enabled": true,
"kind": "monthly",
"workflow_id": "shop.jd_customer_service.daily"
},
{
"at": "03:00",
"business_date_offset_days": 0,
"day_of_month": 22,
"enabled": true,
"kind": "monthly",
"workflow_id": "shop.jd_negative_reviews.daily"
},
{
"at": "03:00",
"business_date_offset_days": 0,
"day_of_month": 23,
"enabled": true,
"kind": "monthly",
"workflow_id": "shop.jd_after_sales_issues.daily"
},
{
"at": "03:20",
"business_date_offset_days": 0,
"day_of_month": 21,
"enabled": true,
"kind": "monthly",
"workflow_id": "shop.douyin_customer_service.daily"
},
{
"at": "03:50",
"business_date_offset_days": 0,
"day_of_month": 25,
"enabled": false,
"kind": "monthly",
"workflow_id": "shop.tmall_negative_reviews.daily"
},
{
"at": "03:40",
"business_date_offset_days": 0,
"day_of_month": 24,
"enabled": false,
"kind": "monthly",
"workflow_id": "shop.tmall_customer_service.daily"
},
{
"at": "03:20",
"business_date_offset_days": 0,
"day_of_month": 22,
"enabled": true,
"kind": "monthly",
"workflow_id": "shop.dy_negative_reviews.daily"
},
{
"at": "03:20",
"business_date_offset_days": 0,
"day_of_month": 23,
"enabled": true,
"kind": "monthly",
"workflow_id": "shop.douyin_after_sales_quality.daily"
},
{
"at": "09:30",
"business_date_offset_days": 0,
"enabled": true,
"kind": "daily",
"workflow_id": "product.online_stock_alert.daily"
},
{
"at": "08:40",
"business_date_offset_days": -1,
@@ -132,6 +240,8 @@
},
{
"at": "23:00",
"business_date_offset_days": 0,
"enabled": false,
"kind": "daily",
"workflow_id": "product.alert.daily"
},
@@ -139,6 +249,7 @@
"anchor_date": "2026-07-25",
"at": "11:00",
"business_date_offset_days": -1,
"enabled": false,
"every_days": 3,
"kind": "interval_days",
"workflow_id": "product.style_analysis.interval"
@@ -224,6 +335,8 @@
},
{
"at": "08:00",
"business_date_offset_days": 0,
"enabled": false,
"kind": "daily",
"workflow_id": "supply.purchase_confirmation.daily"
},
config/secrets.example.env
+3
View File
@@ -3,6 +3,9 @@ GYXX_POSTGRES_PASSWORD=
GYXX_FEISHU_APP_ID=
GYXX_FEISHU_APP_SECRET=
GYXX_HERMES_API_KEY=
GYXX_DATA_INTERACTION_BASE_URL=https://test.gyxx.cn/api
GYXX_DATA_INTERACTION_APP_KEY=data-hub
GYXX_DATA_INTERACTION_APP_SECRET=
# 款式周期分析、天猫/京东视频标题与京东视觉识别直连 MiniMax；
# 真实密钥只放部署环境运行时 .env
STYLE_ANALYSIS_LLM_BASE_URL=https://api.minimaxi.com/v1
config/source-manifests/content_marketing.json
+32 -32
View File
@@ -15,7 +15,7 @@
"source_relative_path": "bilibili_scraper.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/bilibili_scraper.py",
"source_sha256": "c3df15f317ce073f84a59609f83e2723a21a43e37cfcfc2f17b3cbf3a17df73e",
"target_sha256": "c6c985c860f0b9ab742a66786fb3576a71767bbf897969d9539d7c780303a201",
"target_sha256": "153ccbec744bfdd330e3b350bed3b4c9ed9866cabb3e1785f14a880c46178c97",
"transformed": true,
"category": "source"
},
@@ -23,15 +23,15 @@
"source_relative_path": "chanmama_scraper.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/chanmama_scraper.py",
"source_sha256": "2edc26879b57d73ec5992c80e5ef74d76a509f09bff7284b1ae17e90df543e07",
"target_sha256": "72634b4f042f8506bcccadf4512dcbe9f92e9aced3e9b1fc7718483b7c802ebf",
"target_sha256": "8b87268e0f105cf4d349b192c7b2c24d5e318a9d88fcd14264824b083103b24e",
"transformed": true,
"category": "source"
},
{
"source_relative_path": "collection_completeness.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/collection_completeness.py",
"source_sha256": "f23b70b949f8adcaf00ee62aa0f9f39314bc623528cad9a77591641da765991d",
"target_sha256": "f23b70b949f8adcaf00ee62aa0f9f39314bc623528cad9a77591641da765991d",
"source_sha256": "575ab0c526f2d78b1135bd27407c9ecb0fb0aaad2063ac37ba0818ea6247cc2d",
"target_sha256": "575ab0c526f2d78b1135bd27407c9ecb0fb0aaad2063ac37ba0818ea6247cc2d",
"transformed": false,
"category": "source"
},
@@ -55,15 +55,15 @@
"source_relative_path": "daily_marketing_report.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/daily_marketing_report.py",
"source_sha256": "b59252594c7d3c0fd1473d14611346e4b64672ef6c263fea8a2da53017ab8675",
"target_sha256": "78fb871c14eb73ec3e0d0986554cad318f8a0ee8a66c2a0981f02a65cded8a26",
"target_sha256": "e049d933721d0dbe5be559f33beaea812443c84af504d05bc658b20532d28533",
"transformed": true,
"category": "source"
},
{
"source_relative_path": "data/config/analyze.env.example",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/data/config/analyze.env.example",
"source_sha256": "b60328ef177b795489d0a57957133dedcfd67b38b59603a6edeee1cf9babcee2",
"target_sha256": "b60328ef177b795489d0a57957133dedcfd67b38b59603a6edeee1cf9babcee2",
"source_sha256": "72c8359ffbde76060642c77ad21ec8544645ee00aca5a5bfc6bd0971647e9240",
"target_sha256": "72c8359ffbde76060642c77ad21ec8544645ee00aca5a5bfc6bd0971647e9240",
"transformed": false,
"category": "config"
},
@@ -103,7 +103,7 @@
"source_relative_path": "data/tools/analyze_note.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/analyze_note.py",
"source_sha256": "f9a71dcd34657d3e70af84160bd04afa73325c9ddeee0e669d98d6758ed6b13e",
"target_sha256": "3c62dcac6e5c877c1a5b55f4960244052a124c2e50c61571cc726d2ca916109b",
"target_sha256": "0f538b5c9fc52d23b66f7f79f3c54fcc9e7ed737ab6decb5b16c4eed09e59615",
"transformed": true,
"category": "tool"
},
@@ -191,7 +191,7 @@
"source_relative_path": "data/tools/feishu_comment_batch.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/feishu_comment_batch.py",
"source_sha256": "2b04fde410eec1dece0b995d0b74bfad4bdc1a53669f19ead2a6fc13ecdd00b7",
"target_sha256": "598a26e1dcc65702ad154ec3dcc84b4d5c8fecf45196e7e8301ca5b77544ae23",
"target_sha256": "80985a42af7dce5d26d7b4f14ef36411631157e44f8d6ee054de2b66277905f5",
"transformed": true,
"category": "tool"
},
@@ -254,8 +254,8 @@
{
"source_relative_path": "data/tools/migrations/005_style_product_profile.sql",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/migrations/005_style_product_profile.sql",
"source_sha256": "8f9c4d25681e84e582d3cdcd9108ebb16c2f5b777b766a5c3653a21c0baf6d91",
"target_sha256": "8f9c4d25681e84e582d3cdcd9108ebb16c2f5b777b766a5c3653a21c0baf6d91",
"source_sha256": "6a5f60f162c02a9daea2d46ff95edfbcaf416d9f70d894d3b83d049b81ab2134",
"target_sha256": "6a5f60f162c02a9daea2d46ff95edfbcaf416d9f70d894d3b83d049b81ab2134",
"transformed": false,
"category": "sql"
},
@@ -343,7 +343,7 @@
"source_relative_path": "data/tools/retry_failed.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/retry_failed.py",
"source_sha256": "90c6dbc47317cc48e28071d452d75c95913138fc885a5963f9eb58986cd99086",
"target_sha256": "b4c85e8319bfd0a3071ffdfaff05ea030e426cb3d384a978fdc4d14545ccd679",
"target_sha256": "26ca143da6539345c930d0a949a852a33735ed764e7065162c9f805d2609fe91",
"transformed": true,
"category": "tool"
},
@@ -351,7 +351,7 @@
"source_relative_path": "data/tools/schema_gyxx_super_data.sql",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/schema_gyxx_super_data.sql",
"source_sha256": "43d202d7d0452b7d0671ee9c40149c21328db4b2863fa4d030af456f89defe3c",
"target_sha256": "43d202d7d0452b7d0671ee9c40149c21328db4b2863fa4d030af456f89defe3c",
"target_sha256": "001870851dea9c373a77b87335604c4363b06c3eb0ed7c97b4a8d5e891e56758",
"transformed": true,
"category": "sql"
},
@@ -367,7 +367,15 @@
"source_relative_path": "data/tools/sync_metrics_to_cmt_notes.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/sync_metrics_to_cmt_notes.py",
"source_sha256": "3b97cab0740c5c97b47a0fbe763acb5391d7c8e47c95f4675720b4f716f6432f",
"target_sha256": "4e630a79bd2a9742ad19d1955b88f3f12cc7e741b445196e53c07440c0f3749d",
"target_sha256": "6dfeeaa77873fdc580077e1c5d491ced653a1c319720f1fdffc6d5f44c11f5d3",
"transformed": true,
"category": "tool"
},
{
"source_relative_path": "data/tools/sync_notes_master.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/sync_notes_master.py",
"source_sha256": "94edc9b55bfc0106101af30897a68b4f0513d04b8ed4a07a99daf6fce5447f45",
"target_sha256": "aa70a895c0b7dd32e458b885e9e036246e12d83f354d068e32ac3ee6b491e175",
"transformed": true,
"category": "tool"
},
@@ -407,7 +415,7 @@
"source_relative_path": "douyin_comment_scraper.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/douyin_comment_scraper.py",
"source_sha256": "ec785ead9c95c5de65c00b06212b1f03d0159fed475a95a2b593d3b24e486e43",
"target_sha256": "e36ec226b06bae4940f0d2432ae7de0d9a0b56199ab6cd8ee7d34fbfc060ec51",
"target_sha256": "6e7ac4357fe466656b93ca7bae6c49adc4d59fa75974a39c6885c7433bd4d402",
"transformed": true,
"category": "source"
},
@@ -415,7 +423,7 @@
"source_relative_path": "feishu_mapping.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/feishu_mapping.py",
"source_sha256": "d389fd59f5e49208ffa12d04c1ca6202f51dc5d89321a50e52fadad747239749",
"target_sha256": "56af85da94c8eff1f1e0c27f13df56860cc959328524a059b6a374d55eb32e20",
"target_sha256": "0ee2ccd518d4fd313e4acbb62857b565f2e84c77968ba68a3f2f680268f2fdd3",
"transformed": true,
"category": "source"
},
@@ -431,7 +439,7 @@
"source_relative_path": "monthly_summary_all.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/monthly_summary_all.py",
"source_sha256": "50f18d62e8fb1a7c8a04dc746326a4763f64ba4b99b29a6fda5878aa05f290fa",
"target_sha256": "ea2729e1976426d497b69006c63b5ba7d74ce530731efa5a2946a4671593aed7",
"target_sha256": "6f34da4842bb2a7a0b91c3f89ce6047ddccb43a243614ab77a014de516192fbc",
"transformed": true,
"category": "source"
},
@@ -439,7 +447,7 @@
"source_relative_path": "pgy_xhs_scraper.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/pgy_xhs_scraper.py",
"source_sha256": "7d9cd66b80cf602127541cfdd9589f2d8870ca4a25bac3587478482064aae8f4",
"target_sha256": "c1ee83cc68464d5b0b33e010b388abd2adab8a6608846b47d509e9ea0a783f81",
"target_sha256": "667fcfe8bb4ebc55fad44838cbd00d79a4f414dd875a4f74d27bf2ff6e90bc2c",
"transformed": true,
"category": "source"
},
@@ -447,7 +455,7 @@
"source_relative_path": "pgy_xhs_scraper_v2.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/pgy_xhs_scraper_v2.py",
"source_sha256": "2be78ef76df38e80da2099a76e6e3fd9dcdc2e8875afb44279f54003ccb5f563",
"target_sha256": "21c3647faa01a335af2c5984f87bc0d8968361cae73352146279bd9027e82c5f",
"target_sha256": "e5df4b34f67a565ec5d5ce4dd5857e80e363dd7559e7e29006dcd3e421276c8a",
"transformed": true,
"category": "source"
},
@@ -463,7 +471,7 @@
"source_relative_path": "run_all.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/run_all.py",
"source_sha256": "7a2529b1d043297171aeb88518fac8753234540992042f922172219b2b233dd4",
"target_sha256": "0ea60854f1bb1ca293a073c57c9e19a1aa9c5710ff9575dc7c430c4e480525d9",
"target_sha256": "8d27c1ef21c4d5548c82a97f9dc461ad191119d569f36bebeb9ee604cd9e727a",
"transformed": true,
"category": "source"
},
@@ -471,7 +479,7 @@
"source_relative_path": "self_bilibili_scraper.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/self_bilibili_scraper.py",
"source_sha256": "2ef650a16dde61697101bb219a8c3adcd682fec1529bb9c5312dc5b384d83a72",
"target_sha256": "42e6b9ac81077b37ff4607b3565d98cac1027eff3ac39a8776e640503d590820",
"target_sha256": "6ec5ed6991a93f806fb74d0870e953e5a05d52043a83f2fa261e2ec67bee6ad7",
"transformed": true,
"category": "source"
},
@@ -487,7 +495,7 @@
"source_relative_path": "weekly_summary_all.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/weekly_summary_all.py",
"source_sha256": "b93c04c543a8c65418ec122a2713239d0ed857bc169b87ef718cf73348c189b0",
"target_sha256": "a5c866d0720f20fe78bf8f8039a24101fc58e3a81fb107beb7f6499e9782d559",
"target_sha256": "f0d43974605bdd45868e88dd81f7d1db8d624f4c69ecaaf0abf49aaf8ed79cf3",
"transformed": true,
"category": "source"
},
@@ -503,7 +511,7 @@
"source_relative_path": "xiaohongshu_comment_scraper.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/xiaohongshu_comment_scraper.py",
"source_sha256": "58bc07bb2fd67897e18a9981f931df8fadf24a6cc5f09d55bd3aaa2e3c6756a4",
"target_sha256": "5e1e306cb1a089c187dc31bc598939304990b3039a5eb91d9da8a71a023b6e69",
"target_sha256": "afca1104324546da2315fdf7a427c3f34021eef31ddc2fea76877f818718ccb0",
"transformed": true,
"category": "source"
},
@@ -519,17 +527,9 @@
"source_relative_path": "xingtu_scraper_v2.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/xingtu_scraper_v2.py",
"source_sha256": "0d4eb87779064e13b8d78daa6dba13698f1149a26b3a1de2e064c117a29cf49a",
"target_sha256": "e694f3ca9126a99123d0941068dbd102f0fdb4b84160ebdbb51cd56c3f3aa32a",
"target_sha256": "b36339444e26048a90c8ac999a0a628bcddd4f64842924f6c60e600c7a3bec37",
"transformed": true,
"category": "source"
},
{
"source_relative_path": "data/tools/sync_notes_master.py",
"target_relative_path": "src/gyxx_flow/modules/content_marketing/data/tools/sync_notes_master.py",
"source_sha256": "94edc9b55bfc0106101af30897a68b4f0513d04b8ed4a07a99daf6fce5447f45",
"target_sha256": "2d90abb7e73ec420bdfdb03fba01ec497e6c2a5fd3b7c1fe8848e486da08f2f8",
"transformed": true,
"category": "tool"
}
],
"intentionally_excluded": [
config/source-manifests/product_commerce.json
+39 -39
View File
@@ -14,7 +14,7 @@
{
"category": "production_source",
"source_sha256": "cff768f79672cfc7db7f6866caf5ab59936b8c51eb2a7d729b81441940124512",
"target_sha256": "7b4da4f51cd0945bb9ad144aead5ff96e197fdacc7a3c065e6102e833161992a",
"target_sha256": "d56fe8a934c1ae20a9249c7682823238a705f51f451241504512e83a8800a879",
"transformed": true,
"source_relative_path": "analyze_style_with_hermes.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/analyze_style.py"
@@ -54,7 +54,7 @@
{
"category": "production_source",
"source_sha256": "1493dbbfb3d864812df378d04d006e7de0f9ec4b6ab5812e5c6970c304c2dfe6",
"target_sha256": "2d5a6a7bea5e21a5af5f7cdde287acd5b9e7e375469e6d6f85c15aab1cc31d8e",
"target_sha256": "a1d526dfc27b110a8845a98cc66fe7e5cfb87ef70530728477af31a88734a576",
"transformed": true,
"source_relative_path": "collect_dy_market_rank.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_dy_market_rank.py"
@@ -62,7 +62,7 @@
{
"category": "production_source",
"source_sha256": "abd4c82cbd6c41e111e213c174d2fdeba1214568496eaf1dede24bf374b3b052",
"target_sha256": "523027ccd4e31bd276bfba3e8944bf43e0e1e67efc0921ef5c72b8bfa4287c28",
"target_sha256": "9de3ab22d5a4dbf574b2d4fdbbed295bb4645bb4943f4c066fcbfd41b44b521c",
"transformed": true,
"source_relative_path": "collect_dy_persona_to_bitable.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_dy_persona_to_bitable.py"
@@ -70,7 +70,7 @@
{
"category": "production_source",
"source_sha256": "c7b54d941ab7f056e2d0ea4355543ff4a5c34ad986c8ed32145e988ac9501ea2",
"target_sha256": "75cd142aefad9ea54a61e8e11a72e9535ee3a71a0c7b0025a8d030a0a0c0d430",
"target_sha256": "aaba5dfe6ce5133b5c662232160bb9faef4d73b36902a484a56b89fa7df05311",
"transformed": true,
"source_relative_path": "collect_erp_yesterday_metrics.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_erp_yesterday_metrics.py"
@@ -78,7 +78,7 @@
{
"category": "production_source",
"source_sha256": "8bb1f17f90a7b9c2db5dfb992cf746329892d19c00ae2ae988e2109c049598b2",
"target_sha256": "5f9a754f2293b8b35ce5214ae930350b67abe4d10b8e702b06b68c9c9a4d2d9a",
"target_sha256": "c98abe563b223f49cc3c810054d997504d9465387da4805389b0f5ef78ceed6a",
"transformed": true,
"source_relative_path": "collect_jd_market_rank.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_jd_market_rank.py"
@@ -86,7 +86,7 @@
{
"category": "production_source",
"source_sha256": "f9a48c5f89b18472dc3993e340ad858bfecb2fcf59427c7e70db37512c17a6f7",
"target_sha256": "17064990b7080969e1907f2e2ae9cf1f49267936746b6db6e6e1a3a689e1b501",
"target_sha256": "a268df592f7136a68c2b82d031115055e43cdfaedcdd4a5f4afa9ef45879e055",
"transformed": true,
"source_relative_path": "collect_jd_persona_to_bitable.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_jd_persona_to_bitable.py"
@@ -94,7 +94,7 @@
{
"category": "production_source",
"source_sha256": "3c1858a6ab4f7ce480b3a5f2076e6ec79400e9bfaa75377488c3629e707c7963",
"target_sha256": "0d2981d17c86c1881c86810aaf63bfd0261de03bc94d91eaa1b42a66953d3f3e",
"target_sha256": "0e4314a525f49b7c24c145f092afcd6c93749c694d6d8bcbd2d1fcdf9b2e54c7",
"transformed": true,
"source_relative_path": "collect_persona_to_bitable.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_persona_to_bitable.py"
@@ -110,7 +110,7 @@
{
"category": "production_source",
"source_sha256": "13b6e68b4872b2e79c1de8ca1617cc4f1249f4ed32f234ad6efb7a04f5948612",
"target_sha256": "fb120e9501a1220c3746e0f78a9f639e732764e21b122bf532c0fdea03ec6ed7",
"target_sha256": "a1b95cc4f090a25c8e7991b0ef015a7df612fc78249dafe75aeef9fa4dbefe08",
"transformed": true,
"source_relative_path": "collect_sycm_market_rank.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/collect_sycm_market_rank.py"
@@ -126,7 +126,7 @@
{
"category": "production_source",
"source_sha256": "22dcc9d8b2f1cf598a62e4c71df765a952250694cb26b9a73f477c5ca1a29884",
"target_sha256": "e41bc1de359b8f18c25ee38068ab550e5dbbf5c20712d19723915d669f02d6ac",
"target_sha256": "ffeaf697a1a58b21520fda3f3b0936c591d05b328537c41f2048366a9ba1b8d5",
"transformed": true,
"source_relative_path": "config/style_config_loader.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/config/style_config_loader.py"
@@ -134,7 +134,7 @@
{
"category": "production_source",
"source_sha256": "a8900b9f25db6cf2bef3117a1d2cd86283054fea6f5a8590fd91b5cfbbcaf87e",
"target_sha256": "7fd65a5b993b5a8b7f681b370e4364411252d6a59b22c7d96d71235daa5fce55",
"target_sha256": "030d33c3f5c0c230bb67d603dd657f40cceff8581f8756a1f67340a89b5ae4b3",
"transformed": true,
"source_relative_path": "db/__init__.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/__init__.py"
@@ -142,7 +142,7 @@
{
"category": "production_source",
"source_sha256": "fc19a57bcbc023ab20156820a708d64c5526fad33ef5eb5fdea56c0bb04b84e3",
"target_sha256": "486607458d204336a04e3776d9db98f5f069a46b75b050ac0f0947cec958b5d7",
"target_sha256": "2cbd79e4d984d556bac35a0aeb5b8f803309fd3fac2399da459cd61a3aa2484c",
"transformed": true,
"source_relative_path": "db/sync_dim_style.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/sync_dim_style.py"
@@ -150,7 +150,7 @@
{
"category": "production_source",
"source_sha256": "2dc00b88fb5f28c2db619074e97c4a7c446aec29926af9a6890065409c8fac0e",
"target_sha256": "030344e0ffe26240ee735ccf38f194d4e69f2833bd21e9947b97d0072647910b",
"target_sha256": "840f98d9fb79557016a59a2e3a1194a1e328a1574b4670f419c99ac85731d623",
"transformed": true,
"source_relative_path": "db/sync_sku_master.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/sync_sku_master.py"
@@ -158,7 +158,7 @@
{
"category": "production_source",
"source_sha256": "78c74c0e52e3651b3b042a2095e596ab250e60309c7ae2de905850822aad4922",
"target_sha256": "e7377793895b6176a0bd76df0ce6a3d8927c1f2bb4f3c8b4ca47e7d3dd97dd00",
"target_sha256": "09d7e08d2eec57055a3bc7949e49c32a8bf62713f2594ae5d9297c032bf212c8",
"transformed": true,
"source_relative_path": "dy_audience_profile_collect.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/dy_audience_profile_collect.py"
@@ -166,7 +166,7 @@
{
"category": "production_source",
"source_sha256": "492916088fa5e8c170461cc9eba97a88ed1cc1e50c7b99e5f6a5e49167fea157",
"target_sha256": "5f621d4003dbb5ed3c8cd58ae813f371bb0c110aa2fbfff0b6ef559b4eb487b1",
"target_sha256": "9dd0dc3e6f048683157fe8b8deb533ed88970bf34e9824e439bf3a5d52d80e0a",
"transformed": true,
"source_relative_path": "dy_product_scraping.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/dy_product_scraping.py"
@@ -174,7 +174,7 @@
{
"category": "production_source",
"source_sha256": "26f3448e57a08020de0a96d1df3387819afc47f9319b81dbab27f1afc81bfb01",
"target_sha256": "536097c7558c2edb33639ab7ada22b266a1fe14341f6972dc53d3708f7b61fc0",
"target_sha256": "4c36f3eafd29241e1e5436c0a5f64682870a330003a99dcd03a2a981e425e8b8",
"transformed": true,
"source_relative_path": "erp_login_product_analysis.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/erp_login_product_analysis.py"
@@ -190,7 +190,7 @@
{
"category": "production_source",
"source_sha256": "b828e1ad566b121bd56f90ef5de9d414cbe8575ec747cd26dfa3e4b66b097a9a",
"target_sha256": "c87aac33589bffc004eb83c899ca538529ef94afc8be912dbd8c164c87fc2d4f",
"target_sha256": "49da688998376bbd932195ef56603a37f342fa2eb307925a83592596641b8ace",
"transformed": true,
"source_relative_path": "export_bitable_records.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/export_bitable_records.py"
@@ -206,7 +206,7 @@
{
"category": "production_source",
"source_sha256": "906278aa6fb3578d5dab231bdccc2f5a7c6ec8c85810a3e026fff3c7ba16dc90",
"target_sha256": "4f627cb824e9747e1436a5469158d62dd2a17b8442fcb45e64144f35618b27dd",
"target_sha256": "d9e4281c21a42841023c0a723aff4a00af1f9200c94e42add9f8965159d4cfb5",
"transformed": true,
"source_relative_path": "import_product_daily.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/import_product_daily.py"
@@ -230,7 +230,7 @@
{
"category": "production_source",
"source_sha256": "e0f016bafb3f527f1cae46eaf3ef31510184128aaaa7f52024a7c3f978eec42b",
"target_sha256": "f30caa8e5925ee9d4c5addf3af651c4c79f94d564612922106f8693ecd7addd1",
"target_sha256": "f2692442a1e62a4b20dbb442eb3e27a3075ca6095ba2578c90193f0db3a1810e",
"transformed": true,
"source_relative_path": "jd_main_image_collector.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_main_image_collector.py"
@@ -238,7 +238,7 @@
{
"category": "production_source",
"source_sha256": "d99ce71a008128549ac83fc3da52d88540b74538e3fe5c85d47c3b985e414353",
"target_sha256": "3aeec0f04d2d9d7fccf01632503a56066a58716e7dfc953a72241f1857fc8af4",
"target_sha256": "f821a0221b832bea0123bf4a8f85fd0785eba9206c8a632ad48065ea3c30c095",
"transformed": true,
"source_relative_path": "jd_product_data_collector.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py"
@@ -246,7 +246,7 @@
{
"category": "production_source",
"source_sha256": "314790f06f7b4ebf9b0ddd5681db9c60e79a3ef16f5546d6e8489531a003c545",
"target_sha256": "06cac4bc1635268fec6274891a09344715b6dc38b95307b8d5bcf2e6a250bbf0",
"target_sha256": "2a254e00bb3ddd3a3d09a14599827e27d002786bf09e6d7004e25f5cabf787ef",
"transformed": true,
"source_relative_path": "jd_self_inventory_sales_collector.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_self_inventory_sales_collector.py"
@@ -294,7 +294,7 @@
{
"category": "production_source",
"source_sha256": "910e720fab4f47419ad56055f00a8c3fbe3a06c2577d81e35d3d38178a588a45",
"target_sha256": "1dd67e2fbb95f08a77460c5b9f7a0cd9b51abae3454abe2ce138c257b00fc577",
"target_sha256": "c9f8cfc8dff6a7e5617b9b357ed92872f94371f93360318a3ff78f7ec38d75df",
"transformed": true,
"source_relative_path": "market_rank_product_import.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_product_import.py"
@@ -310,7 +310,7 @@
{
"category": "production_source",
"source_sha256": "37edbfa24dfb126d52992b08fd6aeccae3ee18c83db56426a0fb02934e4541ae",
"target_sha256": "cade2f855043e05c1e07b86da81c4864a55e90e999e4441804055b2428e1c92f",
"target_sha256": "47489fd55cbd38a39a8cf3ac0673549e8eee9a39448c187bb34c5cd57b244b97",
"transformed": true,
"source_relative_path": "orchestrate_daily_collection.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py"
@@ -358,7 +358,7 @@
{
"category": "production_source",
"source_sha256": "c9a51f1d1267b398cf260a86e65a8ee49de873d6c95bf9c90db2abb422005cf7",
"target_sha256": "9c8f0e7e686d7198bb231f87f85b074a00d0ffebf5d5d6b44582f59dddeb8abf",
"target_sha256": "6cd70bb5406bc00257f9070760f980580e95c3c4e4326a67d33d30433dd74c32",
"transformed": true,
"source_relative_path": "run_daily_persona.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/run_daily_persona.py"
@@ -366,7 +366,7 @@
{
"category": "production_source",
"source_sha256": "27cd730400b954d1a063f034789f0d9a510c711ebda1e2de95e5a0010bd4751f",
"target_sha256": "dc9a2413b8aac548debcaf6bc5cafffa29e321ae44bb82d5df360cb02983e931",
"target_sha256": "fdbb40a10bdf95ef23a70c8574ac5ff0a52b3e419bde5694a186b2c00a9d45e4",
"transformed": true,
"source_relative_path": "run_weekly_jd_main_image.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/run_weekly_jd_main_image.py"
@@ -374,7 +374,7 @@
{
"category": "production_source",
"source_sha256": "af9b436a31c3454dc6e0b34b77876962dab0b477e8a561db49911fdfc24eade3",
"target_sha256": "4e02f2394bf2b9b108104621b7c015e172002e284e0676ef55dd9c02d2346999",
"target_sha256": "e3b91d3c4b529291f7a6ceee5ff99326351d78104a66babc531f90faec6b7e1a",
"transformed": true,
"source_relative_path": "run_weekly_main_image.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/run_weekly_main_image.py"
@@ -398,7 +398,7 @@
{
"category": "production_source",
"source_sha256": "b3e9addd72d8e31292ed948b67018aa9e212d628d7ba39141a81116515ca6721",
"target_sha256": "a17c6649f4665a09ae1ba0007936546866b2882b5e296733efd99e37d3120b2e",
"target_sha256": "369476667783566e47f086e920c593f3f26aa24355c5c9fa0a3f20bc22b3583e",
"transformed": true,
"source_relative_path": "taobao_dmp_item_crowd_insight_screenshots.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_dmp_item_crowd_insight_screenshots.py"
@@ -406,7 +406,7 @@
{
"category": "production_source",
"source_sha256": "4474dd7d68117247af87e20bbeb6b7c072368ef0f606de5d3e186f459d34a0f5",
"target_sha256": "7e881c9a62227a3ec81b761067034055c5f79d02bec69bc23434a5f7aa13b72c",
"target_sha256": "cfbb46483113cda9e4cb301b092ac2ce6a35c5cc53b9a9cef01fdf93a00d3b94",
"transformed": true,
"source_relative_path": "taobao_sycm_collect.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_sycm_collect.py"
@@ -414,7 +414,7 @@
{
"category": "production_source",
"source_sha256": "a881b02c903dc83c7f252d7f53fa9cd721fb562c690271788858ca5782427500",
"target_sha256": "d8957d1f93f807eadd1dd7c5a095485a83ce4eb492ec6218ab18cc71cb3cc7a7",
"target_sha256": "d64b440344af7cfcc51857e6261fed30f874b446bb2cb2c225d40da0b67da6a3",
"transformed": true,
"source_relative_path": "taobao_sycm_collect_backfill.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_sycm_collect_backfill.py"
@@ -422,7 +422,7 @@
{
"category": "production_source",
"source_sha256": "9cafa62a03316fd0222f63addd5c209bb0c20fd0b7affa69cd1e256abfa2aa3a",
"target_sha256": "e20f104a12b1df763b5fde5ef6b8e2536a7801d70049b55cb84143afae21edf5",
"target_sha256": "cf6f78846238e3b63bcd07221cbc9ecbe1c5d2758f6147f48ff112c3ed97b23a",
"transformed": true,
"source_relative_path": "taobao_sycm_products.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_sycm_products.py"
@@ -430,7 +430,7 @@
{
"category": "production_source",
"source_sha256": "3f0c32c443c87417c43881da69aa7776828ba973465f8d32f50b3dc698b0a4fe",
"target_sha256": "8d34bc9ea810b86e87e6f32404fca534a4dd6276012de4b29c385431c4730670",
"target_sha256": "548df6ddf3af3b3c066d18687114f1ce672afa054535f6da0c254d60f8306b86",
"transformed": true,
"source_relative_path": "taobao_wanxiang_ai_creative_report.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py"
@@ -438,7 +438,7 @@
{
"category": "production_source",
"source_sha256": "e4d99956d9649d108baeb3e3a2d1969c227622d1037062bd277af66375556d82",
"target_sha256": "ee73ae4d58ba164422698e1bd0dd2da8510a5c059b8eb2d107be144a844fee4b",
"target_sha256": "981ef2a5a59f73ae244240520320c73be9c9462b50546df1ff91d0c97e7c03f7",
"transformed": true,
"source_relative_path": "upload_video_to_guanghe.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py"
@@ -488,7 +488,7 @@
{
"category": "production_source",
"source_sha256": "6daa16c03989fb2231e54545726f5aa488e4ba80b9b8aa6f937cb9a2fa13ff35",
"target_sha256": "2080856b1c463a6a11e8e969ef7ff7d0816170402864526c60df026e40c9dd1d",
"target_sha256": "89ba008ef4388c6e1ed32e4a5c26ffa81d083164dd6d7231fc9e5886de8437cd",
"transformed": true,
"source_relative_path": "vendors/jd-data-flow/jd_data_collector.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_data_collector.py"
@@ -496,7 +496,7 @@
{
"category": "production_source",
"source_sha256": "a53cc38b0ddeaa0065bc27a1d137ce16c2873ff9bec712483ca87bcedd6baa3a",
"target_sha256": "122f57cbdb76bbc366cfe93f50d7c6fbefa091b53664958acb2d142fbf2fa6f9",
"target_sha256": "0549228539c2e1908c3181de13291457712eeaea8fea89903814844724a3b5ee",
"transformed": true,
"source_relative_path": "vendors/jd-data-flow/jd_peer_product_data_collector.py",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/vendors/jd-data-flow/jd_peer_product_data_collector.py"
@@ -560,7 +560,7 @@
{
"category": "runtime_resource",
"source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746",
"target_sha256": "bfb13c05e4a0c7a85238200e026332f55d85a8022012cfab1db0c8514746def8",
"target_sha256": "8ff84f6c3c672af35e09f70d71aa73656ac101218d8a56126b8b0a75ab387c8b",
"transformed": true,
"source_relative_path": "db/schema.sql",
"target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql"
@@ -632,7 +632,7 @@
{
"category": "regression_test",
"source_sha256": "3d348a8250a2910f6a1c6504e9d1e53b0493d40bdc6c1fd1c8d0404c78681dc6",
"target_sha256": "dca35f4b197d2a7f77eb14e969e91974210b31100c5e874c842fd8036c4dd6e1",
"target_sha256": "dd1dd020cfcd350ef8223ae01ab45f67a86255a7a9abd6c00a863e3c082cfcfc",
"transformed": true,
"source_relative_path": "tests/test_guanghe_metadata.py",
"target_relative_path": "tests/modules/product_commerce/test_guanghe_metadata.py"
@@ -696,7 +696,7 @@
{
"category": "regression_test",
"source_sha256": "f1e660d34e21d3cd7a26d3605c82d07982ba6c5002af7b56fb6a876cb706713d",
"target_sha256": "025e80fce0ec954abb62119b81f96f40ba725f743badccd662d65f50434f903e",
"target_sha256": "85ca8603e2af8257ce2a940b99c69948c2c1362f688e6f6dac4cbf499f1e4d73",
"transformed": true,
"source_relative_path": "tests/test_market_rank_product_import.py",
"target_relative_path": "tests/modules/product_commerce/test_market_rank_product_import.py"
@@ -728,7 +728,7 @@
{
"category": "regression_test",
"source_sha256": "f9cfbb49fd313444790ac194fbf980dae21912081b9f26ad8e3850912126f09e",
"target_sha256": "76a0aeecf17ecea039ab4d81457cb2dc2cc0d0d75f54e500ea0f2a00ccce6f45",
"target_sha256": "0cc2b365144482ceb36667eedfb2c403e904437f3ea4c7e12f7099276c13a279",
"transformed": true,
"source_relative_path": "tests/test_style_analysis_orchestration.py",
"target_relative_path": "tests/modules/product_commerce/test_style_analysis_orchestration.py"
@@ -736,7 +736,7 @@
{
"category": "regression_test",
"source_sha256": "17761a5ba93a5387ffa67662424955178058aeec708bbcd3eb0341a859d8a1f1",
"target_sha256": "b88b52fb5e68f5ee44aa4f9709fbd2c7dae46126b98f76fdfd12615f284f1494",
"target_sha256": "549cb13dd32c9bed76d78eafa7542ea6e5c638d9f29c857eff4dde6d3b8e9c40",
"transformed": true,
"source_relative_path": "tests/test_style_config_loader_erp_source.py",
"target_relative_path": "tests/modules/product_commerce/test_style_config_loader_erp_source.py"
@@ -760,7 +760,7 @@
{
"category": "regression_test",
"source_sha256": "f7de702d7d1e641c8778970d5cb1f4e827a2e0ee9b20db2c19f0ed4fd8230e16",
"target_sha256": "25c72b4256fc4d0485b491fe691d44d78dac926f02316cc4d1b2bb7c3ee536a2",
"target_sha256": "55ea7031034b272f11e54e2372ad3476a25d0e32d8bbcc832b5977dfb7b34ec7",
"transformed": true,
"source_relative_path": "tests/test_tm_persona_recovery.py",
"target_relative_path": "tests/modules/product_commerce/test_tm_persona_recovery.py"
config/source-manifests/shop_intelligence.json
+7 -7
View File
@@ -32,7 +32,7 @@
"target_relative_path": "src/gyxx_flow/modules/shop_intelligence/collectors/dy_store_competitor_store_scraping.py",
"category": "source",
"source_sha256": "009879a7dd341ca6ecbc8eb66f5700114c73bcaba3a53f159014a76232d0aa6b",
"target_sha256": "e0b9664ad71c5c44132a730ccb274e30e6749229e912c5df816105aa7905405b",
"target_sha256": "d5e85ded1f852ba418229c75c985d0ece967dfbd5fd31697fe407b64d1c3db05",
"transformed": true
},
{
@@ -40,7 +40,7 @@
"target_relative_path": "src/gyxx_flow/modules/shop_intelligence/collectors/jd_data_collector.py",
"category": "source",
"source_sha256": "6f00ae3b2b74e08d3c34c3b911ea819dfa4c6ce55bceef282afb3b2f565aa3ec",
"target_sha256": "6dc3a80a9d5869b82e5de4febec1fe920ebe518209fe1d0febafec732c3a97ac",
"target_sha256": "7d3cd4bdcb657d828f6cd1287df5755a53abeb24ba42170eb6fb6735cc586a4d",
"transformed": true
},
{
@@ -48,7 +48,7 @@
"target_relative_path": "src/gyxx_flow/modules/shop_intelligence/collectors/jd_peer_store_data_collector.py",
"category": "source",
"source_sha256": "4ebdb52df7b384d2bfdd2fc840d55d14e18864002c20a36685b63cee1966cf4c",
"target_sha256": "6f626e52f52abd2f08441f70f3e9558e9cca41ebfb666438f4cf009cd17dccbe",
"target_sha256": "39faf18e9cd5e441323a8e056e022f179018a44eac224d64383cd7ae9fa1d611",
"transformed": true
},
{
@@ -88,15 +88,15 @@
"target_relative_path": "src/gyxx_flow/modules/shop_intelligence/db/__init__.py",
"category": "source",
"source_sha256": "ffc89b78ff0081a129e04eb6de4985ab0a2a4613c9e8a1d8b2750d8c17e8bc05",
"target_sha256": "ffc89b78ff0081a129e04eb6de4985ab0a2a4613c9e8a1d8b2750d8c17e8bc05",
"transformed": false
"target_sha256": "a4bae02d0c07f3e3986896b31e62f41bf15edbf74d3776b1e050e381c15a8413",
"transformed": true
},
{
"source_relative_path": "db/db.py",
"target_relative_path": "src/gyxx_flow/modules/shop_intelligence/db/db.py",
"category": "source",
"source_sha256": "168b6f87c1827b267f763ddce7eb15d0c8d652c8c0c7af18dcd0c6c7da5cddd1",
"target_sha256": "b8f15975605e2d9b75d25ac09a67768d4583401c1365288de078a97d0d3f3abf",
"target_sha256": "01469f9eb80aee5e80bc28e548dbe2cd5ce861ab35cd480b33921a2adc799320",
"transformed": true
},
{
@@ -104,7 +104,7 @@
"target_relative_path": "src/gyxx_flow/modules/shop_intelligence/db/schema.sql",
"category": "sql",
"source_sha256": "73230590bde7ae6359c02c82c2db096c0d3faa55b374e30348c47d7b40ea498b",
"target_sha256": "a0aa2a22c8fdbb1f748dc7b206fe06a43f10899ca7036e433cdd91d7a1f5cba9",
"target_sha256": "a69fa151f989f106d00f344bab43ec9e6a09cf404bdc581624cc0901cdc13d20",
"transformed": true
},
{
config/source-manifests/supply_chain.json
+8 -8
View File
@@ -8,7 +8,7 @@
"target_relative_path": "src/gyxx_flow/modules/supply_chain/config.example.env",
"category": "config",
"source_sha256": "ec2df00f43abd9ba570b3882c5331d9b3c09557cccbc9c79e2326df57b680124",
"target_sha256": "3fbfd050b522a20802c91d195e305388f8d704fe60d1f24dab41096d74a28286",
"target_sha256": "3da1ee4935faab3997f6221a1f2540233285a561f35b97fccb82003cd7d58c02",
"transformed": true
},
{
@@ -24,7 +24,7 @@
"target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/config.py",
"category": "source",
"source_sha256": "466791caa5439b29e6f37b92ee6afcd4018f2c7e3fbb072efe9c6832955b3400",
"target_sha256": "835298b3c05983ef215d76150364a1ca89b4b6ce539cced4b9387d3678b508bf",
"target_sha256": "479a4dde2639b488b49ddf7508ee9899004bb1204b529cb97f7153175c4376c7",
"transformed": true
},
{
@@ -40,7 +40,7 @@
"target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/mcp_workflow.py",
"category": "source",
"source_sha256": "286e28c9db042956743d045fe40157d64c58ef70a85301c018a6eaedd8c2984d",
"target_sha256": "2eb41409ef1bb799204b27fabcba243a927ffc30c0e46328c8ccda83d185be41",
"target_sha256": "f2426fbdcd52135e1c52c194d654639bfd8697c44f7d3ecaefbfe17813b4eb6b",
"transformed": true
},
{
@@ -64,7 +64,7 @@
"target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/runner.py",
"category": "launcher",
"source_sha256": "0cfe5e6557e8dee70ec4542c54b53e2153a3651eb2e35cd831e85518aabe0c66",
"target_sha256": "c3c16831a9c088455805b3a78fc23d276ded1cfcde8856f59cfdafceab061011",
"target_sha256": "5c99177d7c41ae45f6df517dc27b7d4e7413d5004e27e34948f5e169ec589ed3",
"transformed": true
},
{
@@ -120,7 +120,7 @@
"target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/ProductReplenishment.py",
"category": "source",
"source_sha256": "02f6345142ebd84e25dc2f3b0651bc139ef31dd3994ff739efe66344c77ee49e",
"target_sha256": "32d40a9cbdaba73f88eb5ed181eca0af3bd02df9dc017e65162a4a9acb2f1a1f",
"target_sha256": "e74211f2fe97ac47e704a040f442f51c97fecc15ea30dd1f0dbeacaf5940fd80",
"transformed": true
},
{
@@ -128,7 +128,7 @@
"target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/PurchaseConfirmation.py",
"category": "source",
"source_sha256": "d400571da971702302b6e426a0ba12981459a5ee7919c771d6f4ff0038617643",
"target_sha256": "91cf0ccb4150e9252c5048581a16e77534f251f17a3400ba4e1e4ad03e451941",
"target_sha256": "f9bcb7b2b911f0a7680e57a6fed5814663dac4d5c7bf5a0334a9c517aded3218",
"transformed": true
},
{
@@ -136,7 +136,7 @@
"target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/PurchaseOrderUpdate.py",
"category": "source",
"source_sha256": "91617a2854fbf1ef38e9f8ef6fefaa34627b477d7a39098c43d6ee5dba296aa7",
"target_sha256": "0ff70f17c748cd9c892e6bb5b20c7083681edddb5097fb18ce0aa182996f29a1",
"target_sha256": "721027ab230d4503430a0f958a425babbfad8626890e6c1c2ce386f247f50f45",
"transformed": true
},
{
@@ -144,7 +144,7 @@
"target_relative_path": "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/send_card_notification.py",
"category": "source",
"source_sha256": "1397f5508a15ea7dae3d809aee9b0317b73788f3e7dc3927ff9efba5b7630a7e",
"target_sha256": "783ff97e7137e30ba7bcac72d53559563889e88fda21d0eb2b5217d48bf1ed53",
"target_sha256": "7b189ccc29074f72a2d02a45c720e0674ac3d8a7385bc6cdc99966ee6f505723",
"transformed": true
},
{
config/workflows.json
+630 -34
View File
@@ -10,7 +10,7 @@
{
"id": "collect_collaborators",
"name": "采集合作达人指标",
"description": "按每日范围并行抓取合作达人在 B站、小红书蒲公英和星图的曝光等指标，星图未命中的任务再用抖店达人广场补采；05:00 仅补 01:00 未完成的任务。",
"description": "按每日范围并行抓取合作达人在 B站、小红书蒲公英和星图的曝光等指标，星图未命中的任务再用抖店达人广场补采；05:00 仅补 01:00 未完成的任务。",
"data_flow": {
"sources": [
{
@@ -19,7 +19,7 @@
},
{
"system": "内容平台",
"label": "B站、小红书蒲公英与星图/抖店补采指标"
"label": "B站、小红书蒲公英与星图/抖店补采指标"
}
],
"processing": [
@@ -39,6 +39,57 @@
],
"replay_policy": "repeatable"
},
{
"id": "sync",
"name": "同步合作达人曝光指标入库",
"description": "只汇总合作达人本轮曝光结果，按记录映射更新或补建 PostgreSQL 内容笔记数据。",
"data_flow": {
"sources": [
{
"system": "本地采集产物",
"label": "合作达人内容指标结果"
},
{
"system": "飞书映射",
"label": "合作达人内容记录对应关系"
}
],
"processing": [
"读取合作达人各平台曝光指标",
"按内容记录身份更新已有数据或补建缺失记录"
],
"destinations": [
{
"system": "PostgreSQL",
"label": "合作达人笔记与曝光指标"
}
]
},
"entry": "data/tools/sync_metrics_to_cmt_notes.py",
"args": [
"--scope",
"collaborators"
],
"depends_on": [
"collect_collaborators"
],
"run_after_failure": true,
"replay_policy": "repeatable"
}
]
},
"provenance": {
"source_project": "content",
"task_name": "YingxiaoYunying_DailyRun"
},
"note": "每日 01:00、05:00 采集合作达人内容指标并同步入库；自营内容指标由独立的 content.metrics.self_operated.daily 工作流负责。"
},
{
"id": "content.metrics.self_operated.daily",
"module": "content_marketing",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "refresh_self_mapping",
"name": "刷新自营款式映射",
@@ -61,16 +112,12 @@
]
},
"entry": "data/tools/refresh_self_mapping.py",
"depends_on": [
"collect_collaborators"
],
"run_after_failure": true,
"replay_policy": "repeatable"
},
{
"id": "collect_self_bilibili",
"name": "采集自营 B站指标",
"description": "读取自营映射中的 B站笔记链接，通过公开 API 获取播放量和发布时间并回填对应飞书表。",
"description": "读取自营映射中的 B站笔记链接，通过公开 API 获取标题、UP主、时长、发布时间与播放/点赞/投币/收藏/分享/弹幕全量指标，回填对应飞书表并产出全量视频指标 artifact。",
"data_flow": {
"sources": [
{
@@ -83,7 +130,8 @@
}
],
"processing": [
"按映射定位自营笔记并读取公开指标"
"按映射定位自营笔记并读取公开指标",
"按 BV 号聚合全量视频指标 artifact"
],
"destinations": [
{
@@ -92,7 +140,7 @@
},
{
"system": "本地采集产物",
"label": "B站自营指标结果"
"label": "B站自营指标结果与全量视频 artifact"
}
]
},
@@ -106,7 +154,7 @@
{
"id": "collect_self_douyin",
"name": "采集自营抖音指标",
"description": "使用蝉妈妈新账号登录态导出自营抖音视频数据，解析曝光指标并回填飞书自营表。",
"description": "使用蝉妈妈新账号登录态导出自营抖音视频数据，解析视频链接、标题、发布时间、曝光、点赞、评论、转发等全量指标，并回填飞书自营曝光表。",
"data_flow": {
"sources": [
{
@@ -119,7 +167,8 @@
}
],
"processing": [
"导出自营视频数据并解析曝光指标"
"导出自营视频数据并解析全量视频指标",
"按发布时间槽位回填飞书曝光"
],
"destinations": [
{
@@ -140,45 +189,116 @@
"replay_policy": "repeatable"
},
{
"id": "sync",
"name": "同步合作与自营曝光指标入库",
"description": "汇总本轮合作达人及自营曝光结果，按记录映射更新或补建 PostgreSQL 内容笔记数据。",
"id": "collect_self_xiaohongshu",
"name": "采集自营小红书笔记指标",
"description": "复用 ARK 隔离登录态（失效时短信验证码需人工完成）进入小红书笔记管理后台，采集商品笔记与普通笔记两个页签的全部展示列（笔记信息、阅读数、点赞数、引流次数、支付金额与订单数等），产出全量笔记指标 artifact。",
"data_flow": {
"sources": [
{
"system": "本轮采集产物",
"label": "合作达人及自营内容指标"
"system": "小红书 ARK",
"label": "已登录的笔记管理后台（商品笔记/普通笔记）"
},
{
"system": "本地映射",
"label": "内容记录对应关系"
"system": "本地配置",
"label": "自营小红书账号与登录手机号"
}
],
"processing": [
"合并各平台指标并匹配内容记录",
"更新已有记录或补建缺失记录"
"按页签逐页抓取并按表头解析全部展示列",
"按笔记身份去重产出全量指标 artifact"
],
"destinations": [
{
"system": "本地采集产物",
"label": "小红书自营笔记全量指标 artifact"
}
]
},
"entry": "self_xhs_scraper.py",
"depends_on": [
"collect_self_douyin"
],
"run_after_failure": true,
"replay_policy": "repeatable"
},
{
"id": "sync",
"name": "同步自营曝光指标入库",
"description": "只汇总自营内容曝光结果，按自营记录映射更新或补建 PostgreSQL 内容笔记数据。",
"data_flow": {
"sources": [
{
"system": "本地采集产物",
"label": "自营内容指标结果"
},
{
"system": "飞书映射",
"label": "自营内容记录对应关系"
}
],
"processing": [
"读取自营 B站和抖音曝光指标",
"按自营内容记录身份更新已有数据或补建缺失记录"
],
"destinations": [
{
"system": "PostgreSQL",
"label": "内容笔记与曝光指标"
"label": "自营笔记与曝光指标"
}
]
},
"entry": "data/tools/sync_metrics_to_cmt_notes.py",
"args": [
"--scope",
"self-operated"
],
"depends_on": [
"collect_self_douyin"
],
"replay_policy": "repeatable",
"run_after_failure": true
"run_after_failure": true,
"replay_policy": "repeatable"
},
{
"id": "sync_self_video_metrics",
"name": "同步自营内容指标入库",
"description": "按平台解析自营内容指标（抖音蝉妈妈导出、B站公开 API artifact、小红书 ARK artifact），映射内部账号别名后按账号+内容+业务日期幂等写入 PostgreSQL cmt_self_video_metrics（含 page_metrics 原始列快照）。",
"data_flow": {
"sources": [
{
"system": "本地采集产物",
"label": "蝉妈妈自营抖音视频导出与 B站/小红书全量指标 artifact"
},
{
"system": "本地配置",
"label": "自营账号别名映射"
}
],
"processing": [
"按平台解析来源产物的全量内容指标",
"按账号+内容+业务日期幂等 upsert"
],
"destinations": [
{
"system": "PostgreSQL",
"label": "自营内容每日指标（抖音/B站/小红书）"
}
]
},
"entry": "data/tools/sync_self_video_metrics.py",
"depends_on": [
"sync",
"collect_self_xiaohongshu"
],
"run_after_failure": true,
"replay_policy": "repeatable"
}
]
},
"provenance": {
"source_project": "content",
"task_name": "YingxiaoYunying_DailyRun"
"task_name": "YingxiaoYunying_SelfOperated_DailyRun"
},
"note": "每日 01:00 执行合作达人及自营 B站、蝉妈妈采集，05:00 按 01:00 产生的未完成清单补采；账号切换期间仅暂停自营采集步骤，不移除自营链；两次均完成后同步曝光指标入库。"
"note": "每日 01:00、05:00 刷新自营映射并采集自营 B站、蝉妈妈、小红书 ARK 数据；飞书曝光回填保持不变，另将三平台全量内容指标（视频/笔记链接、标题、发布时间、曝光/阅读、点赞、评论、转发等）按日快照入库，失败时仍保留后续链路与入库汇总步骤。"
},
{
"id": "content.metrics.backfill",
@@ -189,7 +309,7 @@
{
"id": "collect_collaborators",
"name": "补采合作达人指标",
"description": "按每日范围并行补采合作达人在 B站、小红书蒲公英和星图的曝光等指标，星图未命中的任务再用抖店达人广场补采。",
"description": "按每日范围并行补采合作达人在 B站、小红书蒲公英和星图的曝光等指标，星图未命中的任务再用抖店达人广场补采。",
"data_flow": {
"sources": [
{
@@ -198,7 +318,7 @@
},
{
"system": "内容平台",
"label": "B站、小红书蒲公英与星图/抖店补采指标"
"label": "B站、小红书蒲公英与星图/抖店补采指标"
}
],
"processing": [
@@ -245,7 +365,11 @@
"depends_on": [
"collect_collaborators"
],
"run_after_failure": true
"run_after_failure": true,
"args": [
"--scope",
"collaborators"
]
}
]
},
@@ -938,6 +1062,438 @@
},
"note": "每天采集计划日期前两天；品牌失败仍继续商品，任一步失败则工作流失败。"
},
{
"id": "shop.jd_customer_service.daily",
"module": "shop_intelligence",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "customer_messages",
"name": "采集京东客服聊天记录",
"description": "按控制台选择的业务日期打开京东客服聊天记录，切换消息视图，滚动加载全部顾客并逐个读取会话消息；全部顾客完成后才结束并幂等写入 PostgreSQL。",
"data_flow": {
"sources": [
{
"system": "京东客服后台",
"label": "单日聊天记录消息视图"
}
],
"processing": [
"将开始和结束时间设置为业务日 00:00:00 至 23:59:59",
"滚动加载消息视图中的全部顾客并逐个点击读取会话",
"按会话标识和消息身份规范化，校验顾客覆盖完整后幂等入库"
],
"destinations": [
{
"system": "本地原始产物",
"label": "京东客服单日原始响应"
},
{
"system": "PostgreSQL",
"label": "京东客服会话与消息表"
}
]
},
"entry": "collectors/jd_customer_service_messages.py",
"args": [
"--date",
"{business_date}",
"--range-days",
"30",
"--headless"
],
"timeout_seconds": 14400,
"replay_policy": "idempotent"
}
]
},
"provenance": {
"source_project": "shop",
"task_name": "京东客服消息日采集"
},
"note": "默认关闭自动调度，控制台可按业务日期手动运行；单日必须完成消息视图中的全部顾客，失败时保留已完成进度并可安全重试。"
},
{
"id": "shop.douyin_customer_service.daily",
"module": "shop_intelligence",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "customer_messages",
"name": "采集抖音客服历史会话消息",
"description": "复用抖音账号 vault 的 Cookie 跳过登录（指纹隐藏模式），打开飞鸽历史会话，按控制台选择的业务日期设置开始/结束日期，接待类型勾选纯人工客服接待与智能客服转人工接待，逐条点击查看会话读取聊天消息，解析抽屉右侧咨询订单面板的商品/订单并与会话绑定，全部幂等写入 PostgreSQL。",
"data_flow": {
"sources": [
{
"system": "抖音飞鸽客服后台",
"label": "历史会话筛选列表与会话明细"
},
{
"system": "抖音账号状态",
"label": "douyin-shop 账号 Cookie 与 Profile"
}
],
"processing": [
"将开始和结束日期设置为业务日期，接待类型勾选纯人工客服接待与智能客服转人工接待",
"按分页逐条点击查看会话，滚动加载抽屉内的完整聊天消息",
"按会话 ID 和消息身份规范化，解析咨询订单面板绑定商品/订单，校验会话覆盖完整后幂等入库"
],
"destinations": [
{
"system": "本地原始产物",
"label": "抖音客服单日会话行原始数据"
},
{
"system": "PostgreSQL",
"label": "抖音客服会话与消息表"
}
]
},
"entry": "collectors/dy_customer_service_messages.py",
"args": [
"--date",
"{business_date}",
"--range-days",
"30",
"--headless"
],
"timeout_seconds": 14400,
"replay_policy": "idempotent"
}
]
},
"provenance": {
"source_project": "shop",
"task_name": "抖音客服消息日采集"
},
"note": "每月 1 日凌晨自动采集近 30 天；控制台支持自定义日期范围；单日必须完成筛选结果中的全部会话，失败时保留已完成进度并可安全重试。"
},
{
"id": "shop.tmall_customer_service.daily",
"module": "shop_intelligence",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "customer_messages",
"name": "采集天猫客服聊天记录",
"description": "复用光影行星旗舰店:龙虾仔账号 vault 的 Cookie，使用 Scrapling 隐藏指纹浏览器打开千牛聊天记录查询页；按控制台选择日期范围，每次按不超过 31 天拆分查询，逐个会话读取消息并幂等写入 PostgreSQL。",
"data_flow": {
"sources": [
{
"system": "天猫千牛客服后台",
"label": "聊天记录查询列表与会话消息"
},
{
"system": "天猫账号状态",
"label": "tmall-shop 账号 vault Cookie 与独立 Profile"
}
],
"processing": [
"限制查询范围为最近三个月，并拆分为每次最多 31 天的日期窗口",
"滚动加载查询结果中的全部会话，逐个选中并读取消息时间、发送人、文本及媒体链接",
"按账号、顾客、会话和消息指纹规范化；保存每个查询窗口的原始产物并幂等写入 PostgreSQL"
],
"destinations": [
{
"system": "本地原始产物",
"label": "天猫客服会话与消息 JSON"
},
{
"system": "PostgreSQL",
"label": "天猫客服会话与消息表"
}
]
},
"entry": "collectors/tmall_customer_service_messages.py",
"args": [
"--start-date",
"{business_date}",
"--end-date",
"{business_date}",
"--headless"
],
"timeout_seconds": 14400,
"replay_policy": "idempotent"
}
]
},
"provenance": {
"source_project": "shop",
"task_name": "天猫客服聊天记录采集"
},
"note": "控制台可选择最近三个月内的开始和结束日期；每次平台查询不超过 31 天，按窗口顺序采集。默认不自动调度，失败窗口保留已完成数据，可安全重试。"
},
{
"id": "shop.tmall_negative_reviews.daily",
"module": "shop_intelligence",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "negative_reviews",
"name": "采集天猫差评及商品信息",
"description": "复用 tmall-shop 账号 vault Cookie，使用 Scrapling 隐藏指纹浏览器打开千牛评价管理；先设置控制台选择的评价时间范围并固定情感分类为负面评价（emotion=13），再点击搜索；分页采集评论文字、图片、主评 ID、评价时间和商品名称，不读取或保存订单号，核对总数后幂等写入 PostgreSQL。",
"data_flow": {
"sources": [
{
"system": "天猫千牛评价管理",
"label": "负面评价列表"
},
{
"system": "天猫账号状态",
"label": "tmall-shop vault Cookie 与隔离 Profile"
}
],
"processing": [
"将控制台选定的起止日期设置为评价时间，并选择负面评价后再搜索；日期不留空，避免页面执行全日期查询",
"分页读取评价文字、图片链接、主评 ID、评价时间和关联商品名称，不采集订单号",
"核对页面总数和评价 ID 覆盖完整后，按账号及主评 ID 幂等写入 PostgreSQL"
],
"destinations": [
{
"system": "本地原始产物",
"label": "天猫差评范围采集 JSON"
},
{
"system": "PostgreSQL",
"label": "天猫差评及商品信息表"
}
]
},
"entry": "collectors/tmall_negative_reviews.py",
"args": [
"--start-date",
"{business_date}",
"--end-date",
"{business_date}",
"--headless"
],
"timeout_seconds": 14400,
"replay_policy": "idempotent"
}
]
},
"provenance": {
"source_project": "shop",
"task_name": "天猫差评范围采集"
},
"note": "控制台选择实际评价时间范围；页面不选择评价时间会查询所有日期，因此工作流会先填入日期并固定筛选负面评价后再搜索。默认不自动调度。"
},
{
"id": "shop.dy_negative_reviews.daily",
"module": "shop_intelligence",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "negative_reviews",
"name": "采集抖音差评及商品信息",
"description": "复用抖音账号 vault 的 Cookie（指纹隐藏模式），打开抖店评价管理-评价明细，筛选差评并把评价时间设置为控制台选择的业务日，逐页读取评价文字、图片、评价状态与商品名称/商品ID/规格，全部页面完成后幂等写入 PostgreSQL。",
"data_flow": {
"sources": [
{
"system": "抖店评价管理后台",
"label": "差评评价列表"
}
],
"processing": [
"点击评价等级「差评」并把评价时间设置为业务日 00:00:00-23:59:59",
"按分页完整读取评价文字、图片与商品信息",
"按账号、业务日期和评价（订单）ID 校验完整后幂等入库"
],
"destinations": [
{
"system": "本地原始产物",
"label": "抖音差评单日原始数据"
},
{
"system": "PostgreSQL",
"label": "抖音差评及商品信息表"
}
]
},
"entry": "collectors/dy_negative_reviews.py",
"args": [
"--date",
"{business_date}",
"--range-days",
"30",
"--headless"
],
"timeout_seconds": 14400,
"replay_policy": "idempotent"
}
]
},
"provenance": {
"source_project": "shop",
"task_name": "抖音差评日采集"
},
"note": "每月 1 日凌晨自动采集近 30 天（含当天）；控制台支持自定义日期范围，范围按业务日逐日采集；评价历史仅保留近 30 天，超出范围按 0 条成功；相同账号和日期重复执行按评价（订单）ID 幂等。"
},
{
"id": "shop.douyin_after_sales_quality.daily",
"module": "shop_intelligence",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "after_sales_quality",
"name": "采集抖音品质原因售后单",
"description": "复用抖音账号 vault 的 Cookie 跳过登录（Scrapling 隐藏指纹模式），打开售后工作台，通过更多筛选把申请原因固定为商品品质原因，可按控制台自定义申请时间范围（不设置时间则为页面默认最近6个月全量查询），逐条点击查看详情，在新页签读取协商记录的退货原因、退款说明、凭证信息和交易信息的商品名称、商品ID、商家编码，按申请时间归属业务日后幂等写入 PostgreSQL。",
"data_flow": {
"sources": [
{
"system": "抖音售后工作台",
"label": "品质原因售后列表与售后详情协商记录"
},
{
"system": "抖音账号状态",
"label": "douyin-shop 账号 Cookie 与 Profile"
}
],
"processing": [
"展开更多筛选并将申请原因固定为商品品质原因，可设置申请时间开始/结束日期",
"按分页逐条点击查看详情，在新页签展开协商记录并读取退货原因、退款说明、凭证信息与商品信息",
"按申请时间的上海日期归属业务日，以账号和售后编号幂等入库，协商记录按条目序号幂等保存"
],
"destinations": [
{
"system": "本地原始产物",
"label": "抖音品退售后单原始数据"
},
{
"system": "PostgreSQL",
"label": "抖音品质原因售后单与协商记录表"
}
]
},
"entry": "collectors/dy_after_sales_quality.py",
"args": [
"--date",
"{business_date}",
"--range-days",
"30",
"--headless"
],
"timeout_seconds": 14400,
"replay_policy": "idempotent"
}
]
},
"provenance": {
"source_project": "shop",
"task_name": "抖音品退售后单采集"
},
"note": "每月 1 日凌晨自动采集近 30 天；控制台支持自定义日期范围；不设置申请时间时按页面默认查询最近6个月做全量采集。每条售后单按申请时间归属业务日，重复执行按账号和售后编号幂等更新，失败时保留已完成进度并可安全重试。"
},
{
"id": "shop.jd_negative_reviews.daily",
"module": "shop_intelligence",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "negative_reviews",
"name": "采集京东差评及商品信息",
"description": "按控制台选择的京东店铺和业务日期打开商品评价管理，选择差评并分页读取评价内容、评价时间、订单号、商品名称、SKU 和图片；全部页面完成后幂等写入 PostgreSQL。",
"data_flow": {
"sources": [
{
"system": "京东商品评价后台",
"label": "差评评价列表"
}
],
"processing": [
"设置业务日 00:00:00 至 23:59:59 并选择差评",
"按分页完整读取评价与商品信息",
"按账号、业务日期和评价 ID 校验完整后幂等入库"
],
"destinations": [
{
"system": "本地原始产物",
"label": "京东差评单日原始响应"
},
{
"system": "PostgreSQL",
"label": "京东差评及商品信息表"
}
]
},
"entry": "collectors/jd_negative_reviews.py",
"args": [
"--date",
"{business_date}",
"--range-days",
"30",
"--headless"
],
"timeout_seconds": 14400,
"replay_policy": "idempotent"
}
]
},
"provenance": {
"source_project": "shop",
"task_name": "京东差评日采集"
},
"note": "每月 1 日凌晨自动采集近 30 天（含当天）；控制台支持按店铺选择日期范围，范围按业务日逐日采集；相同账号、店铺和日期重复执行按评价 ID 幂等。"
},
{
"id": "shop.jd_after_sales_issues.daily",
"module": "shop_intelligence",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "after_sales_issues",
"name": "采集京东品退异常问题",
"description": "按控制台选择的日期范围打开京东商智售后分析，筛选商品出现异常并查看全部服务单明细；点击下载数据，读取全部分页，只保留用户原声不是未知的服务单及商品名称、SKU 等信息，按服务单申请日期归属业务日后幂等写入 PostgreSQL。",
"data_flow": {
"sources": [
{
"system": "京东商智",
"label": "售后分析异常服务单明细"
}
],
"processing": [
"将售后分析时间设置为控制台提交的自定义日期范围；商智单次最多 31 天时按连续区间查询",
"筛选售后原因商品出现异常并读取全部服务单明细分页",
"过滤用户原声为未知或空的记录，按服务单申请日期归属业务日，并以账号、业务日和服务单 ID 幂等入库；范围完成后清理同账号范围内申请时间落在范围外的历史残留"
],
"destinations": [
{
"system": "本地原始产物",
"label": "京东品退日期范围明细及下载审计"
},
{
"system": "PostgreSQL",
"label": "京东品退异常服务单表"
}
]
},
"entry": "collectors/jd_after_sales_issues.py",
"args": [
"--date",
"{business_date}",
"--range-days",
"30",
"--headless"
],
"timeout_seconds": 14400,
"replay_policy": "idempotent"
}
]
},
"provenance": {
"source_project": "shop",
"task_name": "京东品退问题日采集"
},
"note": "每月 1 日凌晨自动采集近 30 天（含当天）；控制台固定使用研发旗舰账号并支持直接提交日期范围。一次范围运行会在商智最多 31 天限制下按连续区间查询，完成后按服务单申请日期写入业务日并清理范围外申请时间的历史残留；相同账号、业务日和服务单 ID 重复执行按幂等更新。"
},
{
"id": "shop.douyin_price_appeal",
"module": "shop_intelligence",
@@ -986,7 +1542,7 @@
"source_project": "shop",
"task_name": "douyin-price-appeal-daily"
},
"note": "每天 08:00、16:00、22:00 自动执行；仍可在控制台手动执行。每个时点都会重新扫描，已申诉商品不会再次出现可申诉入口；安全预演不访问抖音也不提交申诉。"
"note": "每小时自动执行；仍可在控制台手动执行。每个时点都会重新扫描，已申诉商品不会再次出现可申诉入口；安全预演不访问抖音也不提交申诉。"
},
{
"id": "product.persona.daily",
@@ -1138,6 +1694,46 @@
"task_name": "ProductCollectorSalesAlert"
}
},
{
"id": "product.online_stock_alert.daily",
"module": "product_commerce",
"trigger": "scheduled",
"execution": {
"steps": [
{
"id": "query_filter_notify",
"name": "查询线上商品并发送低库存预警",
"description": "使用企业数据交互商品列表接口查询线上渠道全部商品；库存低于 3 时，将 SKU、商品名称和库存汇总发送给配置的飞书接收人。",
"replay_policy": "idempotent",
"data_flow": {
"sources": [
{
"system": "企业数据交互接口",
"label": "线上渠道商品与当前库存"
}
],
"processing": [
"以 channel=0、keyword=空查询商品列表",
"筛选 total_stock 小于 3 的 SKU",
"按飞书消息幂等键去重并分段组织预警"
],
"destinations": [
{
"system": "飞书",
"label": "低库存 SKU、商品名称与库存预警",
"condition": "存在库存低于 3 的 SKU"
}
]
},
"entry": "online_stock_alert.py"
}
]
},
"provenance": {
"source_project": "data-hub",
"task_name": "OnlineItemStockAlert"
}
},
{
"id": "product.ecommerce_costs.daily",
"module": "product_commerce",
@@ -2222,7 +2818,7 @@
{
"id": "collect_analyze_publish",
"name": "采集并发布三平台市场排行",
"description": "并行采集天猫、京东和抖音市场排行，汇总平台结果并发送飞书报告链接。",
"description": "并行采集天猫、京东和抖音市场排行，按月份写入品牌行军图数据库并复用已有人工分类。",
"data_flow": {
"sources": [
{
@@ -2236,8 +2832,8 @@
],
"processing": [
"并行采集三平台排行",
"汇总平台结果并生成市场分析",
"归档报告并准备通知"
"按月份覆盖榜单快照并复用商品的风格、使用场景和品类",
"归档数据库结果并记录平台完成状态"
],
"destinations": [
{
@@ -2245,8 +2841,8 @@
"label": "三平台市场排行结果"
},
{
"system": "飞书",
"label": "市场排行报告链接与通知"
"system": "PostgreSQL / 品牌行军图",
"label": "market_product 与 fact_market_product_snapshot 月度快照"
}
]
},