diff --git a/config/commands.json b/config/commands.json index f9ed91f..41edf8e 100644 --- a/config/commands.json +++ b/config/commands.json @@ -512,16 +512,6 @@ "kind": "python", "notification_workflow_id": "product.market_rank" }, - { - "id": "product.review.orchestrate", - "module": "product_commerce", - "entry": "orchestrate_review_collection.py", - "kind": "python", - "default_args": [ - "--target-date", - "{business_date}" - ] - }, { "id": "supply.replenishment.arrival_sync", "module": "supply_chain", diff --git a/config/runtime-bindings.json b/config/runtime-bindings.json index fca16b3..1e84f78 100644 --- a/config/runtime-bindings.json +++ b/config/runtime-bindings.json @@ -1178,14 +1178,6 @@ ], "cdp_port": 22076 }, - "product_commerce:import_product_reviews.py": { - "script_id": "product_commerce:import_product_reviews.py", - "state_key": "product_commerce:import_product_reviews.py", - "aliases": [ - "product_commerce:import_product_reviews.py" - ], - "cdp_port": 22077 - }, "product_commerce:insert_bitable_records.py": { "script_id": "product_commerce:insert_bitable_records.py", "state_key": "product_commerce:insert_bitable_records.py", @@ -1256,14 +1248,6 @@ ], "cdp_port": 22083 }, - "product.review.orchestrate": { - "script_id": "product_commerce:orchestrate_review_collection.py", - "state_key": "product_commerce:orchestrate_review_collection.py", - "aliases": [ - "product_commerce:orchestrate_review_collection.py" - ], - "cdp_port": 22084 - }, "product_commerce:reapply_erp_override.py": { "script_id": "product_commerce:reapply_erp_override.py", "state_key": "product_commerce:reapply_erp_override.py", @@ -1468,7 +1452,9 @@ "GUANGHE_USERNAME", "GUANGHE_PASSWORD", "GUANGHE_LUGGAGE_USERNAME", - "GUANGHE_LUGGAGE_PASSWORD" + "GUANGHE_LUGGAGE_PASSWORD", + "GUANGHE_NEW_BAG_USERNAME", + "GUANGHE_NEW_BAG_PASSWORD" ] }, "product.jd_video_upload.run": { diff --git a/config/source-manifests/product_commerce.json b/config/source-manifests/product_commerce.json index 7bceb55..990bcc3 100644 --- a/config/source-manifests/product_commerce.json +++ b/config/source-manifests/product_commerce.json @@ -3,6 +3,86 @@ "module": "product_commerce", "snapshot": "current-filesystem", "files": [ + { + "category": "launcher_provenance", + "source_sha256": "c7ad2a359730d8c4d449b0e37e5a4fb2940b4e8b5a4293a1b0a3767c185ee2cc", + "target_sha256": "106a6a15ac2226e4db4d4bcdeda09ba8eca9facbd6e80f2ade6eea8821be5c5f", + "transformed": true, + "source_relative_path": "run_alerts.bat", + "target_relative_path": "docs/history/product-commerce/launchers_reference/run_alerts.bat" + }, + { + "category": "launcher_provenance", + "source_sha256": "a96d18ccf79ea43d2152f2ab7de8de5e77c5a45e1cbbb2ec4495eb92e69bbe1b", + "target_sha256": "1f286ed25cdebb4315aa64b878a7529ae74514b2def33d021834645116d3b48e", + "transformed": true, + "source_relative_path": "run_daily_collect.bat", + "target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_collect.bat" + }, + { + "category": "launcher_provenance", + "source_sha256": "a7c8f6a777515f53160996b61804a93aedd4f3b37fa6374dc3eb64ff1588046a", + "target_sha256": "4211341edcf8be34feef7b0be1e7779cb28707c4af5cc8c77d36d85835c28037", + "transformed": true, + "source_relative_path": "run_daily_import.bat", + "target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.bat" + }, + { + "category": "launcher_provenance", + "source_sha256": "3df6a5bdda8448c342f2aa37992f471f572df7d5e51d5b0ad50911b4805cbb67", + "target_sha256": "aa44824a204ee6b05a97fd58dd1964ea6d1b6f477823f10adc5c681384caeced", + "transformed": true, + "source_relative_path": "run_daily_import.ps1", + "target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.ps1" + }, + { + "category": "launcher_provenance", + "source_sha256": "0c64ed81bed07ecdcd522549d0dcaccc97ed966cbb21a2ae60b37c634234feef", + "target_sha256": "8a4af678695c849d5f49b295edb656155e967796229a8bb580a5d8955c6ba6a3", + "transformed": true, + "source_relative_path": "run_daily_persona.bat", + "target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_persona.bat" + }, + { + "category": "launcher_provenance", + "source_sha256": "e461e25912cb3dafe3d6b0a092e599d121a756e11e564f1d33706d0709611cf1", + "target_sha256": "c05295617b0361337192b21a76557c99dae14aad0d549c5980535f0f0090a41b", + "transformed": true, + "source_relative_path": "run_style_analysis_3d.bat", + "target_relative_path": "docs/history/product-commerce/launchers_reference/run_style_analysis_3d.bat" + }, + { + "category": "launcher_provenance", + "source_sha256": "7edfb8cdbfdb88bbaca364b1ad213caef1818eb654b21a73131d67044ef356c0", + "target_sha256": "42613ed10f083c803657dd6609c3b388fca88f7e0ee9dc105807ad8cab288689", + "transformed": true, + "source_relative_path": "run_weekly_jd_main_image.bat", + "target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_jd_main_image.bat" + }, + { + "category": "launcher_provenance", + "source_sha256": "1beadfdb5a0183b6e4a59307827ee4891eab985b1a1b17d0765bd02414bbd12c", + "target_sha256": "c853990ca8857ef10b46ba71ba2177d4b6897dab4e420bce23bb7f2b22a23fae", + "transformed": true, + "source_relative_path": "run_weekly_main_image.bat", + "target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_main_image.bat" + }, + { + "category": "launcher_provenance", + "source_sha256": "656e72dc9a2999960b96d1921125e6649f0a204066664622cb2d6b1d45b290b5", + "target_sha256": "be206938f49d9d5343dfb49e04514d6cdd9c7850a02e43f8d8c790b08b24c6fa", + "transformed": true, + "source_relative_path": "run_weekly_market_rank.bat", + "target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_market_rank.bat" + }, + { + "category": "runtime_resource", + "source_sha256": "589679874b75643d69ea44522e2fe40914a396506ff8e3232150d7e2a96ae2f9", + "target_sha256": "dbe5760e9cbcaedadf82a2cfd84733a56a12c21e77d00327e95a8a013a5779ad", + "transformed": true, + "source_relative_path": ".env.example", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/.env.example" + }, { "category": "production_source", "source_sha256": "071286d2fb8b0516f0f22f0d088eda9e4a85ae4227619eb1217087ad0dbe3db6", @@ -14,19 +94,11 @@ { "category": "production_source", "source_sha256": "cff768f79672cfc7db7f6866caf5ab59936b8c51eb2a7d729b81441940124512", - "target_sha256": "c1c0167b96c818a4e7d44aa5c860b5510c9925cb096b7fb4510b8319b91592fc", + "target_sha256": "01ed46fbbf089f2c46bd67853d64774831ff2c63e2b3792e397bd01df7d9e847", "transformed": true, "source_relative_path": "analyze_style_with_hermes.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/analyze_style.py" }, - { - "category": "production_source", - "source_sha256": "524bff755d6921d64857b7ca6adfe1cf3a64730ddb66ae318751e028f5cf6f10", - "target_sha256": "dd230b802fc88e16c67abcc8fd08a64c4dbfb68a58b74ea90ee45bad2d4d7870", - "transformed": true, - "source_relative_path": "tmp_backfill_poseidon_sales.py", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_poseidon_sales.py" - }, { "category": "production_source", "source_sha256": "4d24f0e0cc65be9c2f6ce4047817794a520801f4e6563fbae2c3174c3d411d72", @@ -43,6 +115,38 @@ "source_relative_path": "backfill_one_day.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_one_day.py" }, + { + "category": "production_source", + "source_sha256": "524bff755d6921d64857b7ca6adfe1cf3a64730ddb66ae318751e028f5cf6f10", + "target_sha256": "dd230b802fc88e16c67abcc8fd08a64c4dbfb68a58b74ea90ee45bad2d4d7870", + "transformed": true, + "source_relative_path": "tmp_backfill_poseidon_sales.py", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_poseidon_sales.py" + }, + { + "category": "production_source", + "source_sha256": "8a2b09f8e5d4d49239c99a28fbc70bb5e44c8f510b6a1eca86b928eb45f21564", + "target_sha256": "8a2b09f8e5d4d49239c99a28fbc70bb5e44c8f510b6a1eca86b928eb45f21564", + "transformed": false, + "source_relative_path": "backfill_style_id.py", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/backfill_style_id.py" + }, + { + "category": "runtime_resource", + "source_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a", + "target_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a", + "transformed": false, + "source_relative_path": "bitable_main_image_map.json", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_main_image_map.json" + }, + { + "category": "runtime_resource", + "source_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2", + "target_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2", + "transformed": false, + "source_relative_path": "bitable_style_map.json", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_style_map.json" + }, { "category": "production_source", "source_sha256": "6fb7cd48665c8cb7cf40dd69693408047a85a6abba3cbd410b4e5e9a9a55e9ba", @@ -123,10 +227,18 @@ "source_relative_path": "config/__init__.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/config/__init__.py" }, + { + "category": "runtime_resource", + "source_sha256": "277f80393a7ead72b260b56173471e924e139db0ba60e1ae534d85cc1ef5f9cc", + "target_sha256": "bd0f4087fc061555cb8d2bab485bc0d98417dc1816d210bd58bf7ff6801cff39", + "transformed": true, + "source_relative_path": "config/auto-flow-config.example.json", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/config/auto-flow-config.example.json" + }, { "category": "production_source", "source_sha256": "22dcc9d8b2f1cf598a62e4c71df765a952250694cb26b9a73f477c5ca1a29884", - "target_sha256": "ce328f87c883257c91025708209338ba6ee34d3c284a149c1969aeae85ffbdb7", + "target_sha256": "5af46cd70a75930ca34142775d145416c073699ecec265dab6a34f27a59794fe", "transformed": true, "source_relative_path": "config/style_config_loader.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/config/style_config_loader.py" @@ -134,11 +246,19 @@ { "category": "production_source", "source_sha256": "a8900b9f25db6cf2bef3117a1d2cd86283054fea6f5a8590fd91b5cfbbcaf87e", - "target_sha256": "a34efe965b312fe02509a4955cba835f7cdc081daa8a1624d571a8f46bf9380e", + "target_sha256": "61cb3cc7e5ecbe76dea8f7557327c371971e9b301a62f99727708ed342caeaf4", "transformed": true, "source_relative_path": "db/__init__.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/db/__init__.py" }, + { + "category": "runtime_resource", + "source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746", + "target_sha256": "51893cc8b9cb241820fcdccc84729bc2303e5c705d47cef7f76db911fb6984cb", + "transformed": true, + "source_relative_path": "db/schema.sql", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql" + }, { "category": "production_source", "source_sha256": "78c74c0e52e3651b3b042a2095e596ab250e60309c7ae2de905850822aad4922", @@ -150,7 +270,7 @@ { "category": "production_source", "source_sha256": "492916088fa5e8c170461cc9eba97a88ed1cc1e50c7b99e5f6a5e49167fea157", - "target_sha256": "855feba2cfd51bbbb70b2fc912a7b5b517f91c3fa47ff5129ba05e2f7ffa329c", + "target_sha256": "1ab238bf134a4e639c81d60f3e2cff63ec07040b3330c90289fb2e661f082573", "transformed": true, "source_relative_path": "dy_product_scraping.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/dy_product_scraping.py" @@ -166,7 +286,7 @@ { "category": "production_source", "source_sha256": "fcd8db904a14cd5239a5e075e1a888af18c5b24c928608576f6decf4c48c1d8f", - "target_sha256": "282e9eaee483fde3394d330a62887fb4832cbd876b67a002e27651cd49b8730a", + "target_sha256": "18cfd51b7f6df00a4a4834ccce59acc83035177b39a40e1cb4b0f3639545a2fc", "transformed": true, "source_relative_path": "erp_metric_overrides.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/erp_metric_overrides.py" @@ -195,14 +315,6 @@ "source_relative_path": "import_product_daily.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/import_product_daily.py" }, - { - "category": "production_source", - "source_sha256": "003a8969da4c50bd500f48fd6b3dcd6378322d87893cfcaf755b3890579e7a52", - "target_sha256": "50d757be4bfdf8849a43471f419ee460cb844921cf567454de35fcaadbae737f", - "transformed": true, - "source_relative_path": "import_product_reviews.py", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/import_product_reviews.py" - }, { "category": "production_source", "source_sha256": "057a01e8a22cdd2d1806d290da207209954af89f7da6339e9cdcef3bc0297e63", @@ -222,7 +334,7 @@ { "category": "production_source", "source_sha256": "d99ce71a008128549ac83fc3da52d88540b74538e3fe5c85d47c3b985e414353", - "target_sha256": "a597578741795926844b93fa6b74bd8316e1c72cac5196f9dbc2e492e76d88e3", + "target_sha256": "a24cc909d319c801fa63372dfcb6e3a0bd1cb6326e28008db7ada7293802a743", "transformed": true, "source_relative_path": "jd_product_data_collector.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py" @@ -269,11 +381,11 @@ }, { "category": "production_source", - "source_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba", - "target_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba", - "transformed": false, - "source_relative_path": "market_rank_report_archive.py", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_report_archive.py" + "source_sha256": "f200d890e62bb9e387949264f5b518ed22c2bb620a60fcd28b673ff3e513c398", + "target_sha256": "b6a3ad04efbe9b42a598caeb67f7408171ef1862f146fce4dc3eeb57fb2ce24c", + "transformed": true, + "source_relative_path": "market_rank_limits.py", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_limits.py" }, { "category": "production_source", @@ -285,16 +397,16 @@ }, { "category": "production_source", - "source_sha256": "f200d890e62bb9e387949264f5b518ed22c2bb620a60fcd28b673ff3e513c398", - "target_sha256": "b6a3ad04efbe9b42a598caeb67f7408171ef1862f146fce4dc3eeb57fb2ce24c", - "transformed": true, - "source_relative_path": "market_rank_limits.py", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_limits.py" + "source_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba", + "target_sha256": "4f2621d855dde386d2764dd925bb88a8aa225f819d1d04944a752a0addd7e9ba", + "transformed": false, + "source_relative_path": "market_rank_report_archive.py", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/market_rank_report_archive.py" }, { "category": "production_source", "source_sha256": "37edbfa24dfb126d52992b08fd6aeccae3ee18c83db56426a0fb02934e4541ae", - "target_sha256": "47489fd55cbd38a39a8cf3ac0673549e8eee9a39448c187bb34c5cd57b244b97", + "target_sha256": "b52b75e188906de1cabf55eb8791976f740c4bc9f1192971a5a4a5e7a082498c", "transformed": true, "source_relative_path": "orchestrate_daily_collection.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py" @@ -307,14 +419,6 @@ "source_relative_path": "orchestrate_market_rank_collection.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_market_rank_collection.py" }, - { - "category": "production_source", - "source_sha256": "0d0f2c0a1533a09e698cd23aca42662b309c865659b86f9b9522de8207c3fb8b", - "target_sha256": "47b2943790486ae6a1ca36a7e972467b89547f0ef6c37aa673a3733316277ec8", - "transformed": true, - "source_relative_path": "orchestrate_review_collection.py", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/orchestrate_review_collection.py" - }, { "category": "production_source", "source_sha256": "69f376f2c209d8e2df8f154aceb501eab93d7e077ab0278f5ff35c97356ba296", @@ -379,6 +483,14 @@ "source_relative_path": "scripts/insert_main_image_records.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/scripts/insert_main_image_records.py" }, + { + "category": "runtime_resource", + "source_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3", + "target_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3", + "transformed": false, + "source_relative_path": "styles_input.json", + "target_relative_path": "src/gyxx_flow/modules/product_commerce/styles_input.json" + }, { "category": "production_source", "source_sha256": "b3e9addd72d8e31292ed948b67018aa9e212d628d7ba39141a81116515ca6721", @@ -414,7 +526,7 @@ { "category": "production_source", "source_sha256": "3f0c32c443c87417c43881da69aa7776828ba973465f8d32f50b3dc698b0a4fe", - "target_sha256": "6363ec2a70a22b078af23a70fcdda6e3cce9f95eb4aa7ebc54f4cb85a293cb6d", + "target_sha256": "ed86f1b7d4fe6d8e68f4d17abc6bb299df8d3083a0f54c005470491e753c1755", "transformed": true, "source_relative_path": "taobao_wanxiang_ai_creative_report.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py" @@ -422,7 +534,7 @@ { "category": "production_source", "source_sha256": "e4d99956d9649d108baeb3e3a2d1969c227622d1037062bd277af66375556d82", - "target_sha256": "2ccb359ab943c185b17854f0f00cfb76b8a1235d7b031209a1c07e9386f8d93f", + "target_sha256": "c5ed343c698efa35b76f35d95e6deeb19be113f0107d170bbd91596bab2f8e3b", "transformed": true, "source_relative_path": "upload_video_to_guanghe.py", "target_relative_path": "src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py" @@ -510,52 +622,12 @@ "target_relative_path": "src/gyxx_flow/modules/product_commerce/weekly_aggregate.py" }, { - "category": "runtime_resource", - "source_sha256": "589679874b75643d69ea44522e2fe40914a396506ff8e3232150d7e2a96ae2f9", - "target_sha256": "dbe5760e9cbcaedadf82a2cfd84733a56a12c21e77d00327e95a8a013a5779ad", + "category": "regression_test", + "source_sha256": "55f55a9658453e6751ed055c6b2d1a43538012ce715835bbef59d88203211696", + "target_sha256": "5fc5e15e0356b1438ea158ed1d64b769bfaf8c94bfd9f6b3a8d828a15d53c231", "transformed": true, - "source_relative_path": ".env.example", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/.env.example" - }, - { - "category": "runtime_resource", - "source_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a", - "target_sha256": "af4df439f109549a2d287514d24193cb046800d23a39cabcc070b8c10d76a52a", - "transformed": false, - "source_relative_path": "bitable_main_image_map.json", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_main_image_map.json" - }, - { - "category": "runtime_resource", - "source_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2", - "target_sha256": "51102b78d1f44b9dfb60f8204a0d5b2403cd8651061e7607c264c6ef730defa2", - "transformed": false, - "source_relative_path": "bitable_style_map.json", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/bitable_style_map.json" - }, - { - "category": "runtime_resource", - "source_sha256": "277f80393a7ead72b260b56173471e924e139db0ba60e1ae534d85cc1ef5f9cc", - "target_sha256": "bd0f4087fc061555cb8d2bab485bc0d98417dc1816d210bd58bf7ff6801cff39", - "transformed": true, - "source_relative_path": "config/auto-flow-config.example.json", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/config/auto-flow-config.example.json" - }, - { - "category": "runtime_resource", - "source_sha256": "9739407b61842c598491d3189a32f5f15aa944aed79a164a5cd5b5a34f20c746", - "target_sha256": "0ea747eb7e40875338bebe7fc5b8d2524869e55ba1b503a322b37c857b274d8e", - "transformed": true, - "source_relative_path": "db/schema.sql", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/db/schema.sql" - }, - { - "category": "runtime_resource", - "source_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3", - "target_sha256": "b8acd08b97283a1163210d26c2a35b182e1920d2edbad6864226ad24e73ed5b3", - "transformed": false, - "source_relative_path": "styles_input.json", - "target_relative_path": "src/gyxx_flow/modules/product_commerce/styles_input.json" + "source_relative_path": "tests/test_tmp_backfill_poseidon_sales.py", + "target_relative_path": "tests/modules/product_commerce/test_backfill_poseidon_sales.py" }, { "category": "regression_test", @@ -583,8 +655,8 @@ }, { "category": "regression_test", - "source_sha256": "4041e1104d5adbb92703b88e457795ac221bc7bc4fe4aa2cb72dff08f0c4ab6b", - "target_sha256": "4041e1104d5adbb92703b88e457795ac221bc7bc4fe4aa2cb72dff08f0c4ab6b", + "source_sha256": "a9b397cbc0a04cf3ba0caec54c0194ab97a96a4e8bacc3207cd0a0b7b8cbee8b", + "target_sha256": "a9b397cbc0a04cf3ba0caec54c0194ab97a96a4e8bacc3207cd0a0b7b8cbee8b", "transformed": false, "source_relative_path": "tests/test_erp_metric_overrides.py", "target_relative_path": "tests/modules/product_commerce/test_erp_metric_overrides.py" @@ -623,8 +695,8 @@ }, { "category": "regression_test", - "source_sha256": "e8848b95c28a822b9fc92b5cd8ad75716b80d00fbac1844bfad9b7f877ea0173", - "target_sha256": "e8848b95c28a822b9fc92b5cd8ad75716b80d00fbac1844bfad9b7f877ea0173", + "source_sha256": "6ba0ead03f6889cb66ea77799d4416bd4d9ccc81ccaa1df6ee79c1ff4c5dd1bc", + "target_sha256": "6ba0ead03f6889cb66ea77799d4416bd4d9ccc81ccaa1df6ee79c1ff4c5dd1bc", "transformed": false, "source_relative_path": "tests/test_guanghe_store_routing.py", "target_relative_path": "tests/modules/product_commerce/test_guanghe_store_routing.py" @@ -656,7 +728,7 @@ { "category": "regression_test", "source_sha256": "820e17b910b62a213f0bebafcff72998c2c978307c97f976c074276150ce7c6a", - "target_sha256": "77221bd2bfb9e2314585ba9bbaea770093a86cfe5918ffe9c67fc1fadba3b04a", + "target_sha256": "88ff084edc73a011a4c29ed61fbd88e12d9771c3e7aed38805f152add09a64d1", "transformed": true, "source_relative_path": "tests/test_main_image_concurrency.py", "target_relative_path": "tests/modules/product_commerce/test_main_image_concurrency.py" @@ -671,11 +743,11 @@ }, { "category": "regression_test", - "source_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69", - "target_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69", - "transformed": false, - "source_relative_path": "tests/test_market_rank_report_archive.py", - "target_relative_path": "tests/modules/product_commerce/test_market_rank_report_archive.py" + "source_sha256": "1e0f8a519ad3ce0079240fa2b6b1d2f2df1b2c0091da89049f7df9aeec660196", + "target_sha256": "72b85f901d43f836cdc5e8eabe9d83f67a89f8bbd49ef8c5267f92e3114d28b1", + "transformed": true, + "source_relative_path": "tests/test_market_rank_limits.py", + "target_relative_path": "tests/modules/product_commerce/test_market_rank_limits.py" }, { "category": "regression_test", @@ -687,11 +759,11 @@ }, { "category": "regression_test", - "source_sha256": "1e0f8a519ad3ce0079240fa2b6b1d2f2df1b2c0091da89049f7df9aeec660196", - "target_sha256": "72b85f901d43f836cdc5e8eabe9d83f67a89f8bbd49ef8c5267f92e3114d28b1", - "transformed": true, - "source_relative_path": "tests/test_market_rank_limits.py", - "target_relative_path": "tests/modules/product_commerce/test_market_rank_limits.py" + "source_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69", + "target_sha256": "f21d28753f929daf0808fb8147a6654f6b52efc599b8b13248328b9452c37b69", + "transformed": false, + "source_relative_path": "tests/test_market_rank_report_archive.py", + "target_relative_path": "tests/modules/product_commerce/test_market_rank_report_archive.py" }, { "category": "regression_test", @@ -712,7 +784,7 @@ { "category": "regression_test", "source_sha256": "f9cfbb49fd313444790ac194fbf980dae21912081b9f26ad8e3850912126f09e", - "target_sha256": "043b2dd69172bf5ea1b324bf07f8bba4c43f4bca9817c30a2c805ffe8ce70525", + "target_sha256": "857b87a507807d522136199ad4acc54a2bbb4c7667514cf264f136e2cc6a28c9", "transformed": true, "source_relative_path": "tests/test_style_analysis_orchestration.py", "target_relative_path": "tests/modules/product_commerce/test_style_analysis_orchestration.py" @@ -720,18 +792,18 @@ { "category": "regression_test", "source_sha256": "17761a5ba93a5387ffa67662424955178058aeec708bbcd3eb0341a859d8a1f1", - "target_sha256": "edc906fdf2ab2fab731fdcf56dd8ad7cf272949a8b4f66fb71c1693de5822f1f", + "target_sha256": "b1725d401b59a2a19ef4824438d8362d2c68e4ccc0c06d75a2143dbc3d62d1bf", "transformed": true, "source_relative_path": "tests/test_style_config_loader_erp_source.py", "target_relative_path": "tests/modules/product_commerce/test_style_config_loader_erp_source.py" }, { "category": "regression_test", - "source_sha256": "55f55a9658453e6751ed055c6b2d1a43538012ce715835bbef59d88203211696", - "target_sha256": "5fc5e15e0356b1438ea158ed1d64b769bfaf8c94bfd9f6b3a8d828a15d53c231", - "transformed": true, - "source_relative_path": "tests/test_tmp_backfill_poseidon_sales.py", - "target_relative_path": "tests/modules/product_commerce/test_backfill_poseidon_sales.py" + "source_sha256": "895e9e2f149d7ef01a712246f0a5c3c641c9103908b6216571cf5764f6bff0dd", + "target_sha256": "895e9e2f149d7ef01a712246f0a5c3c641c9103908b6216571cf5764f6bff0dd", + "transformed": false, + "source_relative_path": "tests/test_style_id_identity.py", + "target_relative_path": "tests/modules/product_commerce/test_style_id_identity.py" }, { "category": "regression_test", @@ -752,82 +824,10 @@ { "category": "regression_test", "source_sha256": "5598dcbf6d9a30fb3dc4133e983ed78c2860d90162c6f7750509d4533466917a", - "target_sha256": "198cb3979b355a81863683dc357f4152afaf331cca963259d4010ba63778b40b", + "target_sha256": "09981f868a2f1b2b26ce14cc6bd96f1bb4cb2b8f53e2d668ae157b81966b9b36", "transformed": true, "source_relative_path": "tests/test_wanxiang_report_template.py", "target_relative_path": "tests/modules/product_commerce/test_wanxiang_report_template.py" - }, - { - "category": "launcher_provenance", - "source_sha256": "c7ad2a359730d8c4d449b0e37e5a4fb2940b4e8b5a4293a1b0a3767c185ee2cc", - "target_sha256": "106a6a15ac2226e4db4d4bcdeda09ba8eca9facbd6e80f2ade6eea8821be5c5f", - "transformed": true, - "source_relative_path": "run_alerts.bat", - "target_relative_path": "docs/history/product-commerce/launchers_reference/run_alerts.bat" - }, - { - "category": "launcher_provenance", - "source_sha256": "a96d18ccf79ea43d2152f2ab7de8de5e77c5a45e1cbbb2ec4495eb92e69bbe1b", - "target_sha256": "1f286ed25cdebb4315aa64b878a7529ae74514b2def33d021834645116d3b48e", - "transformed": true, - "source_relative_path": "run_daily_collect.bat", - "target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_collect.bat" - }, - { - "category": "launcher_provenance", - "source_sha256": "a7c8f6a777515f53160996b61804a93aedd4f3b37fa6374dc3eb64ff1588046a", - "target_sha256": "4211341edcf8be34feef7b0be1e7779cb28707c4af5cc8c77d36d85835c28037", - "transformed": true, - "source_relative_path": "run_daily_import.bat", - "target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.bat" - }, - { - "category": "launcher_provenance", - "source_sha256": "3df6a5bdda8448c342f2aa37992f471f572df7d5e51d5b0ad50911b4805cbb67", - "target_sha256": "aa44824a204ee6b05a97fd58dd1964ea6d1b6f477823f10adc5c681384caeced", - "transformed": true, - "source_relative_path": "run_daily_import.ps1", - "target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_import.ps1" - }, - { - "category": "launcher_provenance", - "source_sha256": "0c64ed81bed07ecdcd522549d0dcaccc97ed966cbb21a2ae60b37c634234feef", - "target_sha256": "8a4af678695c849d5f49b295edb656155e967796229a8bb580a5d8955c6ba6a3", - "transformed": true, - "source_relative_path": "run_daily_persona.bat", - "target_relative_path": "docs/history/product-commerce/launchers_reference/run_daily_persona.bat" - }, - { - "category": "launcher_provenance", - "source_sha256": "e461e25912cb3dafe3d6b0a092e599d121a756e11e564f1d33706d0709611cf1", - "target_sha256": "c05295617b0361337192b21a76557c99dae14aad0d549c5980535f0f0090a41b", - "transformed": true, - "source_relative_path": "run_style_analysis_3d.bat", - "target_relative_path": "docs/history/product-commerce/launchers_reference/run_style_analysis_3d.bat" - }, - { - "category": "launcher_provenance", - "source_sha256": "7edfb8cdbfdb88bbaca364b1ad213caef1818eb654b21a73131d67044ef356c0", - "target_sha256": "42613ed10f083c803657dd6609c3b388fca88f7e0ee9dc105807ad8cab288689", - "transformed": true, - "source_relative_path": "run_weekly_jd_main_image.bat", - "target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_jd_main_image.bat" - }, - { - "category": "launcher_provenance", - "source_sha256": "1beadfdb5a0183b6e4a59307827ee4891eab985b1a1b17d0765bd02414bbd12c", - "target_sha256": "c853990ca8857ef10b46ba71ba2177d4b6897dab4e420bce23bb7f2b22a23fae", - "transformed": true, - "source_relative_path": "run_weekly_main_image.bat", - "target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_main_image.bat" - }, - { - "category": "launcher_provenance", - "source_sha256": "656e72dc9a2999960b96d1921125e6649f0a204066664622cb2d6b1d45b290b5", - "target_sha256": "be206938f49d9d5343dfb49e04514d6cdd9c7850a02e43f8d8c790b08b24c6fa", - "transformed": true, - "source_relative_path": "run_weekly_market_rank.bat", - "target_relative_path": "docs/history/product-commerce/launchers_reference/run_weekly_market_rank.bat" } ], "intentionally_excluded": [ diff --git a/config/workflows.json b/config/workflows.json index b127681..0bc6a02 100644 --- a/config/workflows.json +++ b/config/workflows.json @@ -1796,7 +1796,7 @@ { "id": "tmall_import", "name": "导入万相台商品广告指标", - "description": "解析解压后的商品 CSV，将花费、广告归因成交金额和投产比写入 product_daily_metrics.raw_data.tm_ad_metrics；款式按商品 ID 优先、商品名称精确包含兜底，并记录匹配方式，不覆盖经营日报字段。", + "description": "解析解压后的商品 CSV，将花费、广告归因成交金额和投产比写入独立的 tm_product_ad_daily_metrics（每日每商品一行），报表其余指标存 metrics JSON；款式按商品 ID 优先、商品名称精确包含兜底，并记录匹配方式。不再写 product_daily_metrics.raw_data。", "entry": "import_tmall_product_ads.py", "depends_on": [ "tmall_download" @@ -1818,13 +1818,13 @@ "processing": [ "校验 CSV 编码、日期、商品主体和广告指标", "合并同商品同日多计划行", - "按平台/日期/商品幂等更新广告 JSON 节点" + "按日期/商品幂等写入天猫商品投放日报表" ], "destinations": [ { "system": "PostgreSQL", - "label": "product_daily_metrics.raw_data.tm_ad_metrics", - "detail": "由 data-hub 产品生命进程读取并计算广告成交、投产比、电商费比；节点同时保存 style_name/style_match_method" + "label": "tm_product_ad_daily_metrics", + "detail": "由 data-hub 产品生命进程读取 spend/attributed_deal_amount 计算广告成交、投产比、电商费比；style_name/style_match_method 记录款式匹配，metrics 保留报表全部指标" } ] } @@ -2305,7 +2305,7 @@ { "id": "upload", "name": "上传所选天猫店铺短视频", - "description": "先按前端所选店铺扫描飞书并筛选符合路由的视频附件，再按视频 file_token、目标账号和飞书记录写入 PostgreSQL 附件级发布账本；仅在存在可领取附件时登录对应光合账号上传，已发布附件自动跳过。", + "description": "先按前端所选店铺扫描飞书并筛选符合路由的视频附件，再按视频 file_token、目标账号和飞书记录写入 PostgreSQL 附件级发布账本；仅在存在可领取附件时登录对应光合账号上传，已发布附件自动跳过。支持旗舰店、鑫华达专卖店和天猫新店（天猫新店只发黑曜石系列，视频源与其他店铺为同一张飞书表）。", "replay_policy": "idempotent", "data_flow": { "sources": [ @@ -2319,11 +2319,11 @@ }, { "system": "淘宝光合浏览器状态", - "label": "旗舰店与箱包店独立账号和 Profile" + "label": "旗舰店、箱包店与天猫新店独立账号和 Profile" } ], "processing": [ - "按前端所选旗舰店或箱包店规则筛选飞书记录并拆分视频附件", + "按前端所选旗舰店、箱包店或天猫新店规则筛选飞书记录并拆分视频附件；天猫新店仅保留全部款式都属于黑曜石系列的记录", "按视频 file_token、目标账号和飞书记录原子领取未上传附件", "确认存在可上传附件后登录所选店铺账号", "生成标题、关联天猫商品并发布；明确成功后写 published，提交后结果不明写 ambiguous" @@ -2364,7 +2364,7 @@ "label": "天猫话题关键词", "hint": "输入完整话题或关键词；例如输入“秋上新”也会匹配并选中“秋上新开拍了”。" }, - "note": "默认由常驻调度器按 config/schedules.json 每 3 天 14:00 执行；前端仍可手动选择旗舰店或箱包店后执行。脚本先扫描飞书并按所选店铺规则筛选，只有数据库存在可领取附件时才登录对应账号上传。允许同一业务日期重复扫描新增附件，PostgreSQL 附件级账本阻止已发布或结果不明的附件自动重发；框架安全预演不会启动上传脚本。" + "note": "默认由常驻调度器按 config/schedules.json 每 3 天 14:00 执行；前端仍可手动选择旗舰店、箱包店或天猫新店后执行。天猫新店账号为gyxx箱包旗舰店:龙虾仔（凭据 GUANGHE_NEW_BAG_USERNAME/GUANGHE_NEW_BAG_PASSWORD），只上传黑曜石系列，视频来源与其他店铺相同。脚本先扫描飞书并按所选店铺规则筛选，只有数据库存在可领取附件时才登录对应账号上传。允许同一业务日期重复扫描新增附件，PostgreSQL 附件级账本阻止已发布或结果不明的附件自动重发；框架安全预演不会启动上传脚本。" }, { "id": "product.jd_video_upload", diff --git a/docs/runbook.md b/docs/runbook.md index 055bb2a..7edd50b 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -43,7 +43,7 @@ uv run gyxx scripts run --date 2026-08-01 --execute | 京东自营品牌单日回采 | `shop.jd_self_operated.collect_brand` | `--date` 自动成为起止日期 | | 商品历史补采 | `product.backfill.run` | `--date` 自动成为 `--from/--to` | | 商品评价补采 | `product.review.orchestrate` | `--date` 自动成为目标日期 | -| 天猫旗舰店和箱包店视频上传 | `product.video_upload.run` | 业务日期用于运行审计；默认顺序扫描两店，由 PostgreSQL 附件级账本防重 | +| 天猫旗舰店、箱包店与天猫新店视频上传 | `product.video_upload.run` | 业务日期用于运行审计；`--store all` 顺序扫描三家，由 PostgreSQL 附件级账本防重 | | 京东旗舰店视频上传 | `product.jd_video_upload.run` | 业务日期只用于运行审计；从固定凌云air记录锚点按飞书视图顺序扫描，日期可为空 | | 采购单更新 | `supply.workflow.run` | 自动选择 `mcp-run purchase-order-update` | @@ -51,16 +51,16 @@ uv run gyxx scripts run --date 2026-08-01 --execute ### 天猫视频上传工作流 -天猫视频上传注册为幂等定时工作流 `product.video_upload`，按 `config/schedules.json` 以 2026-09-04 为起始日每 3 天 14:00（Asia/Shanghai）执行；前端仍可手动选择“光影行星旗舰店 / 光影行星鑫华达专卖店:龙虾仔”并立即运行，也可填写天猫话题完整名称或关键词。正式执行只处理所选店铺，话题搜索结果按包含关系选中。默认 dry-run 只生成运行记录并跳过浏览器节点；确认飞书待传记录、对应淘宝光合店铺的登录态和 PostgreSQL 后，再显式正式执行： +天猫视频上传注册为幂等定时工作流 `product.video_upload`，按 `config/schedules.json` 以 2026-09-04 为起始日每 3 天 14:00（Asia/Shanghai）执行；前端仍可手动选择“光影行星旗舰店 / 光影行星鑫华达专卖店:龙虾仔 / 天猫新店（gyxx箱包旗舰店:龙虾仔）”并立即运行，也可填写天猫话题完整名称或关键词。正式执行只处理所选店铺，话题搜索结果按包含关系选中。默认 dry-run 只生成运行记录并跳过浏览器节点；确认飞书待传记录、对应淘宝光合店铺的登录态和 PostgreSQL 后，再显式正式执行： ```bash uv run gyxx run product.video_upload --date 2026-08-07 uv run gyxx run product.video_upload --date 2026-08-07 --execute ``` -前端把店铺选择作为受限运行参数传给脚本，只允许 `flagship` 或 `luggage`。脚本先读取飞书并应用所选店铺路由，再写入/读取附件级账本；只有存在可领取的未上传附件时才启动有头 Chrome、登录对应账号并上传。旗舰店只选择“天猫上传”未勾选且有附件的光影行星男/女记录；箱包店只选择女性、全部款式命中箱包店白名单且有附件的记录。两店分别使用 `GUANGHE_USERNAME` / `GUANGHE_PASSWORD` 和 `GUANGHE_LUGGAGE_USERNAME` / `GUANGHE_LUGGAGE_PASSWORD`，箱包店当前账号为 `光影行星鑫华达专卖店:龙虾仔`，密码配置保持不变，并保留独立浏览器 Profile。发生登录或页面错误时保留窗口供人工定位，排查完成后手动关闭。 +前端把店铺选择作为受限运行参数传给脚本，只允许 `flagship`、`luggage` 或 `new_bag`。脚本先读取飞书并应用所选店铺路由，再写入/读取附件级账本；只有存在可领取的未上传附件时才启动有头 Chrome、登录对应账号并上传。旗舰店只选择“天猫上传”未勾选且有附件的光影行星男/女记录；箱包店只选择女性、全部款式命中箱包店白名单且有附件的记录；天猫新店不区分性别，只要求一条记录的**全部**「代拍产品」款式都属黑曜石系列（关键词 `黑曜石`，如黑曜石托特、黑曜石吐司、黑曜石电脑包、黑曜石饺子），组合款里掺了其它系列整条记录都不进新店，视频源与其它店铺为同一张飞书表。三店分别使用 `GUANGHE_USERNAME` / `GUANGHE_PASSWORD`、`GUANGHE_LUGGAGE_USERNAME` / `GUANGHE_LUGGAGE_PASSWORD` 和 `GUANGHE_NEW_BAG_USERNAME` / `GUANGHE_NEW_BAG_PASSWORD`，箱包店当前账号为 `光影行星鑫华达专卖店:龙虾仔`，天猫新店当前账号为 `gyxx箱包旗舰店:龙虾仔`，密码配置保持不变，各自保留独立浏览器 Profile（`-luggage` / `-new-bag`）。发生登录或页面错误时保留窗口供人工定位，排查完成后手动关闭。 -每次正式扫描都会把视频附件写入 PostgreSQL 附件级发布账本，防重身份由视频 `file_token + 目标账号 + 飞书记录` 构成。账本状态为 `published` 的附件在下次执行时自动跳过，因此同一业务日期可以安全重跑并发现新增附件。旗舰店仍只在一条飞书记录的全部附件发布成功后回写“天猫上传”；箱包店不借用该勾选状态，是否已上传完全以自己的目标账号账本为准。 +每次正式扫描都会把视频附件写入 PostgreSQL 附件级发布账本，防重身份由视频 `file_token + 目标账号 + 飞书记录` 构成。账本状态为 `published` 的附件在下次执行时自动跳过，因此同一业务日期可以安全重跑并发现新增附件。旗舰店仍只在一条飞书记录的全部附件发布成功后回写“天猫上传”；箱包店和天猫新店不借用该勾选状态，是否已上传完全以自己的目标账号账本（`guanghe-luggage` / `guanghe-new-bag`）为准，同一视频可以分别发到多家店铺。 首次正式执行会读取 `config/tmall-video-publication-backfill.json`，将 2026-07-26 箱包店旧日志中可按 `record_id + file_token + 文件名` 再次核验的 25 个明确成功附件回填为 `published`。配置与当前飞书任一身份不一致都会停止，不会静默猜测。该回填不依赖部署机保留旧 `var/` 日志文件；证据路径与成功行号已固化在配置中供审计。 @@ -103,11 +103,17 @@ uv run gyxx scripts run product.jd_video_upload.run --date 2026-08-07 --arg=--re 唯一工作流 ID 是 `product.ecommerce_costs.daily`，每天 10:00 由项目内 Python 调度器运行， 业务日期为前一天；天猫、京东、抖音三个下载分支并行启动，各自完成下载后再导入。天猫流程先进入万相台商品报表选择昨日，提交下载任务；任务在下载列表显示“生成成功” -后下载 ZIP、做路径安全校验并解压 CSV，随后将商品级广告指标幂等写入 -`product_daily_metrics.raw_data.tm_ad_metrics`。商品款式先按 `dim_style.tm_spus` 的商品 ID 匹配， -匹配不到再按商品名称做精确包含匹配，并在节点内保留 `style_name` 与 -`style_match_method`（`product_id`/`product_name`/`ambiguous`/`unmatched`）。它不会覆盖同一商品日期 -已有的访客、成交等经营日报字段；data-hub 产品生命进程从该节点读取广告成交金额和花费并计算投产比、电商费比。 +后下载 ZIP、做路径安全校验并解压 CSV，随后将商品级广告指标幂等写入独立表 +`tm_product_ad_daily_metrics`（每日每商品一行，主键 `(stat_date, product_id)`）。花费、直接/间接/归因 +成交金额和投产比走标量列，报表其余指标（展现、点击、点击率、转化率、预售成交等）原样存在 `metrics` +JSONB。商品款式先按 `dim_style.tm_spus` 的商品 ID 匹配，匹配不到再按商品名称做精确包含匹配，结果 +写进 `style_name` 与 `style_match_method`（`product_id`/`product_name`/`ambiguous`/`unmatched`）。 +天猫广告数据**不再写入** `product_daily_metrics.raw_data.tm_ad_metrics`，投放与经营日报分表，经营数据 +重新同步不会覆盖投放数据；data-hub 产品生命进程从新表读取花费和广告归因成交金额并计算投产比、电商费比。 + +历史数据迁移（2026-11-01 一次性执行）：`src/gyxx_flow/modules/product_commerce/scripts/ +migrate_tm_ad_metrics_to_table.py`，默认 dry-run，`--execute` 先备份旧节点到运行时 TMP_ROOT 再回填， +校验行数与花费合计一致后才允许 `--clear-source` 清除 `raw_data` 里的旧节点。 正式服务必须注入 `WANXIANG_ACCOUNT`、`WANXIANG_PASSWORD`、`GYXX_DATA_ROOT` 和 `GYXX_POSTGRES_DSN`。默认浏览器 Profile 是 @@ -139,8 +145,9 @@ uv run gyxx scripts run product.import.tmall_ads --date 2026-08-17 --execute 1. 检查 `gyxx schedule status`、服务日志和对应 `run_id`，确认失败发生在申请、下载、解压还是入库。 2. 检查同一 Profile 是否仍能打开万相台商品报表；登录失效时在有头命令中人工完成验证。 3. 检查日期目录中的 `manifest.json`、CSV 表头和 `日期/主体ID/主体名称`；ZIP 存在不代表数据库已导入。 -4. 检查 PostgreSQL 的 `product_daily_metrics` 是否存在 `platform='tm'`、目标日期和商品 ID，且 - `raw_data->'tm_ad_metrics'` 已更新；确认经营日报其他 JSON 节点仍保留。 +4. 检查 PostgreSQL 的 `tm_product_ad_daily_metrics` 是否存在目标日期与商品 ID 的行，`spend`、 + `attributed_deal_amount` 是否为目标值，`style_name` 是否已匹配；确认 `product_daily_metrics` + 同日期经营数据（访客、成交）仍完整。 5. 只有在外部状态核对清楚后才按同一业务日期重跑，不能用换日期或重复申请掩盖下载结果不明。 ## 调度操作 @@ -385,6 +392,116 @@ ORDER BY style_name; 必须把它拆成独立系列（单独的 `series_name` + 锚点标记）并把地址写在自己那行；只写地址 而不拆系列的话，聚合会把它并入原系列，只在日志里留一条告警，不会真的生效。 +## 款式主键统一（style_id = dim_style.id，2026-11-01） + +采集端不再只用 `style_name` 关联款式。**唯一关联键是 `style_id`**（= `dim_style.id`， +与 data-hub 的 22 张表同一套主键），`style_name` 降级为显示冗余。这样在 dim_style +改款名时，历史指标表不会变成孤儿。 + +### 数据从哪里来 + +| 环节 | 机制 | +|---|---| +| loader 下发 | `merge_lifecycle_style_records()` 输出 `style_id` + `style_match_method`，经 `style_config_loader` 进入每条款式配置 | +| db 层兜底 | `db.attach_style_ids()`（按款名）/ `db.resolve_style_ids_by_product()`（按平台商品 ID）在 upsert 内补齐 | +| 历史回填 | `backfill_style_id.py` | + +### 回填与验收 + +```powershell +# 只报数，不写库 +.venv\Scripts\python.exe backfill_style_id.py --dry-run +# 执行（幂等，只补 style_id IS NULL 的行） +.venv\Scripts\python.exe backfill_style_id.py +# 只补款名表，跳过按商品 ID 的两步 +.venv\Scripts\python.exe backfill_style_id.py --skip-product-table --skip-ad-tables +``` + +回填分三段，缺一段覆盖率就上不去： + +| 段 | 表 | 依据 | +|---|---|---| +| 按款名 | 13 张（`daily_style_metrics` / `weekly_style_metrics` / `erp_all_shop_style_daily_metrics` 等） | `t.style_name = dim_style.style_name` | +| 按商品 ID | `product_daily_metrics` | `platform + product_id` 反查 `*_spus` | +| 按商品 ID（二次） | 三张广告表 | 同上；**顺带把空的 `style_name` 补成 dim_style 的权威款名** | + +广告表必须走第三段：平台的报表不给款式名，`jd_product_ad_daily_metrics` 有 27%、 +`dy_product_ad_daily_metrics` 有 40% 的行 `style_name` 为空，只按款名回填完全无效。 +补完后 data-hub 现有按款名的 join 立刻多认回一批行，不必等主键切换。 + +```sql +-- 各表回填进度；除 product_daily_metrics 与三张广告表外都应 100% +SELECT * FROM v_style_id_coverage ORDER BY tbl; +-- 未建档的平台商品 ID，仅统计经营日报口径 +SELECT * FROM v_style_unmatched_product ORDER BY platform, rows DESC; +-- 同上，但把三张广告表也算进来（纯投流商品只在这里出现） +SELECT * FROM v_style_unmatched_source ORDER BY platform, rows DESC; +``` + +`style_match_method` 取值：`dim_style`（按款名精确）、`product_id`（按平台商品 ID）、 +`product_name`（按商品名包含）、`unmatched`（确实没建档）。 + +### 补 spus 的正确做法 + +`product_daily_metrics.product_no` 存的是**款名**（不是 ERP 编码），天猫填充率 100%、 +京东 97.8%、抖音 0%。补 spus 时按这个优先级，命中率远高于猜商品标题： + +1. `product_no` 精确等于 `dim_style.style_name`； +2. `product_no` 与款名互为前后缀（长度差 ≤ 3，如「蓝鹊pro」→「蓝鹊」、「知更大号」→「知更」）； +3. 商品标题包含完整款名（抖音只能用这一条），并用「命中位置后的版本后缀」排除串款 + —— 例如标题「盖亚斜挎包**3**」不能归到「盖亚斜挎」； +4. 编码与标题互相矛盾时**不要自动写**，列出来人工定（实测有天猫单品编码写「理岩」、 + 标题写「阿波罗 air」，支付额 50 万，猜错代价极大）。 + +物料类（数据线、内胆包、补邮费、配件包、礼品袋、定制服务）不该进 `spus`，应留 +`unmatched`。`dim_style` 里本身建成款的物料（`配件包`、`飞机盒`、`电泵` 等）也不写。 + +### 必须知道的边界 + +- `product_daily_metrics` 表里**没有款名列**，只能拿 `platform + product_id` 去 + `tm_spus / jd_spus / dy_spus` 反查。未建档的商品 ID 回填不了，要靠建档页补 `spus` + 才能涨，不是脚本 bug。 +- 2026-11-01 补过一轮：`product_daily_metrics` 49.1% → **86.0%（jd）/ 77.3%（tm）/ + 76.7%（dy）**，整体 79.0%；广告表 tm 99.4% / jd 92.4% / dy 83.2%。 + 剩余 259 个商品 ID（dy 107 / jd 52 / tm 100）未建档，清单见 `v_style_unmatched_source`。 +- **不建物理外键**：14 张表的 `style_id` 都只做**逻辑外键**（与 data-hub 侧一致）。 + 库里没有任何指向 `dim_style(id)` 的 FK 约束，完整性靠写入端（loader 下发 + + `attach_style_ids` 兜底）、`backfill_style_id.py --dry-run`、`v_style_id_coverage` + 三道保证。理由：采集语义是「宁可留 NULL 待回填，也不要整批写入失败」， + 物理外键会在传错 id 时直接中断整天采集；此外拆系列/合并同款要改数据时， + 还得先删约束。早期 P4 建过的 10 个 `<表名>_style_id_fk` 已在 schema.sql 里 + 幂等 `DROP CONSTRAINT IF EXISTS` 清掉，老库重跑 schema 即自动回到无 FK 状态。 +- 主键一律没动：`daily_style_metrics` 仍是 `(metric_date, style_name, platform)`。 + data-hub 有 6 处按 `style_name` join，改主键收益小、回归面大。 + +## 商品评价采集下线（product_reviews，2026-11-01） + +`product_reviews` 已**停止写入并归档**为 `product_reviews_archived_20261101`。 +原因是 JD 侧 `style_name` 全靠 `sku_id` 反查，而映射表不存在，214 行全部为 NULL， +两个消费方都是 `WHERE style_name = ?`，等于入库了读不到。 + +差评统一由三个独立工作流采集：`shop.tmall_negative_reviews.daily`、 +`shop.jd_negative_reviews.daily`、`shop.dy_negative_reviews.daily`。 + +### 消费方改读视图 + +```sql +SELECT platform, comment_date, rating, content, style_id, style_name, style_match_method +FROM v_negative_reviews; -- tm + jd + dy 三张差评表 UNION，款式按 ID 优先、商品名兜底 +``` + +改动点：`analyze_style.py:pull_reviews`、`content_marketing/daily_marketing_report.py:_review_context`。 + +### 口径变化（必须知道） + +- 数据源**只有差评一档**，不再有好评/中评。风格分析的「评价等级分布」因此只剩差评。 +- 营销日报的 `sample_size` = 差评条数，`negative_rate` 恒为 1.0；门槛从「至少 5 条评价」 + 改成「至少 1 条差评」（旧门槛在纯差评源上会导致永不产出）。 +- 覆盖面反而更全：天猫差评 97 条（旧表天猫为 0）、京东 32 条（旧表 8 月断更）。 +- 若产品侧确实需要好评样本，应把差评采集器扩成「按等级采集」，**不要恢复 product_reviews**。 +- `dy_product_scraping.py` 默认流程里的评论采集已摘除（只产出无人读取的本地 JSON）； + 需要历史样本时用 `--reviews-only` 显式触发。 + ## 业务源码同步 外部业务源码目录只用于发现更新，GYXX Flow 运行时不导入或挂载这些目录。只读检查： diff --git a/docs/style-identity-refactor-plan.md b/docs/style-identity-refactor-plan.md new file mode 100644 index 0000000..c9db18d --- /dev/null +++ b/docs/style-identity-refactor-plan.md @@ -0,0 +1,386 @@ +# 款式主键统一 + 评论采集下线 方案 + +> 状态：**已实施（2026-11-01）**，见文末「八、实施记录」 +> 涉及项目：`gyxx-flow`（采集端）、`data-hub`（消费端，共用同一个 PostgreSQL） +> 编写日期：2026-11-01 + +## 一、三条结论 + +1. **`product_reviews` 可以下线**，但不能直接删——它有两个消费方（风格分析、营销日报）要改成读三张差评表。 +2. **`style_id` 就是 `dim_style.id`**，与 data-hub 现有 `style_id` 同一体系，不会产生第二套 ID。已实测：`erp_product.style_id` 的 135 个值 **135/135** 命中 `dim_style.id`，`own_brand_product.style_id` 31/34 命中（3 条为历史脏数据）。 +3. **改造入口只有一处**：`DIM_STYLE_RUNTIME_SQL` 早已 `SELECT s.id`，`merge_lifecycle_style_records()` 也已经拿到这个 id——只是被编码进 `source_record_id = "lifecycle_style:"` 字符串里，没有作为独立字段下发给写入脚本。把它提成正式字段，全链路就都有了。 + +## 二、现状盘点 + +### 2.1 两套引用方式并存 + +| 侧 | 关联方式 | 表 | +|---|---|---| +| data-hub（产品生命、营销中台、客服、内容） | `style_id BIGINT` → `dim_style.id` | `erp_product`、`own_brand_product`、`erp_marketing_*`、`erp_cs_*`、`cmt_*`、`market_product_classification` 等 22 张 | +| gyxx-flow（采集端） | `style_name TEXT` | 15 张，见 2.3 | + +### 2.2 实测匹配率（2026-11-01） + +| 表 | 总行数 | 按 style_name 命中 dim_style | +|---|---|---| +| `daily_style_metrics` | 17,352 | 17,352（100%） | +| `erp_all_shop_style_daily_metrics` | 9,399 | 9,399（100%） | +| `product_reviews` | 1,936 | 1,434（74%，全部来自抖音；京东 214 行 style_name 全 NULL） | +| `product_daily_metrics` | 75,568 | **无款式列**，只能靠 `tm_spus/jd_spus/dy_spus @> to_jsonb(product_id)` 现算 | + +`product_daily_metrics` 按商品 ID 反查的现状： + +| 平台 | 行数 | 可匹配 | 去重商品 ID | 可匹配 ID | +|---|---|---|---|---| +| dy | 28,972 | 13,615（47%） | 253 | 83 | +| jd | 17,044 | 9,394（55%） | 160 | 61 | +| tm | 29,552 | 14,089（48%） | 281 | 92 | + +一对多歧义 **0 条**。约一半行匹配不上，原因是 **458 个商品 ID 从未写进 dim_style 的 spus 数组**（dy 170 / jd 99 / tm 189），不是字段设计问题。 + +### 2.3 采集端 15 张只存 style_name 的表 + +`daily_style_metrics`、`weekly_style_metrics`、`erp_all_shop_style_daily_metrics`、`daily_persona_metrics`、`main_image_creatives`、`fact_style_analysis_reports`、`decline_events`、`fact_alert_history`、`jd_self_weekly_sales`、`persona_screenshots`、`product_reviews`、`jd_product_ad_daily_metrics`、`dy_product_ad_daily_metrics`、`workflow_dynamic_styles`、`product_daily_metrics`（无列，靠 ID 反查）。 + +## 三、目标模型 + +- **唯一关联键 = `style_id`（`dim_style.id`）**，所有采集表落库时必须带上。 +- `style_name` 保留为**显示冗余**，不再作为 join 键；等下游切完 join 后可以只在视图里出现。 +- 匹配不上的行（未建档商品、广告未匹配 SKU、差评）**允许 `style_id IS NULL`**，但必须同时写 `style_match_method`（`product_id` / `product_name` / `anchor` / `unmatched`），便于统计漏维护。 +- **不动现有主键**。`daily_style_metrics` 的 PK 仍是 `(metric_date, style_name, platform)`——data-hub 有 6 处 `metric.style_name = style.name` 的 join，改 PK 收益小、回归面大。加列、加索引、切 join 即可。 + +## 四、实施分期 + +### P0 观测层（只读，不动数据） + +新增两个视图，先把问题量化出来，方案落地后靠它们验收： + +```sql +-- 未建档的平台商品 ID（推动在 dim_style 里补 spus） +CREATE OR REPLACE VIEW v_style_unmatched_product AS +SELECT m.platform, m.product_id, + MAX(m.product_name) AS product_name, + MIN(m.stat_date) AS first_date, MAX(m.stat_date) AS last_date, + COUNT(*) AS rows +FROM product_daily_metrics m +WHERE NOT EXISTS ( + SELECT 1 FROM dim_style d + WHERE (m.platform = 'tm' AND d.tm_spus @> to_jsonb(m.product_id)) + OR (m.platform = 'jd' AND d.jd_spus @> to_jsonb(m.product_id)) + OR (m.platform = 'dy' AND d.dy_spus @> to_jsonb(m.product_id))) +GROUP BY m.platform, m.product_id; + +-- 各表 style_id 回填进度 +CREATE OR REPLACE VIEW v_style_id_coverage AS +SELECT 'daily_style_metrics' AS tbl, COUNT(*) AS rows, + COUNT(style_id) AS with_style_id FROM daily_style_metrics +UNION ALL SELECT 'erp_all_shop_style_daily_metrics', COUNT(*), COUNT(style_id) + FROM erp_all_shop_style_daily_metrics +UNION ALL SELECT 'product_daily_metrics', COUNT(*), COUNT(style_id) + FROM product_daily_metrics; +``` + +验收锚点：`v_style_id_coverage` 里前三张表 `with_style_id` 开始上涨到预期比例；`v_style_unmatched_product` 的条数随建档推进下降。 + +### P1 评论采集链路下线 + +**要删的东西** + +| 位置 | 动作 | +|---|---| +| `orchestrate_daily_collection.py:437-452` | 移除 `import_product_reviews` 步骤 | +| `import_product_reviews.py` | 删除 | +| `orchestrate_review_collection.py` | 删除 | +| `config/commands.json:516` `product.review.orchestrate` | 移除命令 | +| `db/__init__.py` `upsert_product_reviews` | 删除 | +| `db/schema.sql:813-870` `product_reviews` 建表段 | 移除（库里的表归档不删） | + +**要补的东西** + +1. 建统一差评视图，字段对齐两个消费方现在的用法（`platform` / `rating` / `content` / `style_id` / `comment_date`）： + +```sql +CREATE OR REPLACE VIEW v_negative_reviews AS +SELECT 'tm' AS platform, n.business_date AS comment_date, n.review_level AS rating, + n.content, NULL::text AS order_id, NULL::text AS product_id, + n.product_name, n.review_id, n.review_at, n.raw_review +FROM tmall_negative_reviews n +UNION ALL +SELECT 'jd', n.business_date, n.review_level, n.content, n.order_id, NULL, + n.product_name, n.review_id, n.review_at, n.raw_review +FROM jd_negative_reviews n +UNION ALL +SELECT 'dy', n.business_date, n.rating_level, n.content, n.order_id, n.product_id, + n.product_name, n.review_id, n.review_at, n.raw_review +FROM dy_negative_reviews n; +``` + +2. 改两个消费方： + - `analyze_style.py:112 pull_reviews()` → 改查 `v_negative_reviews`（按 `style_id` 或 `product_id` 关联） + - `content_marketing/daily_marketing_report.py:1020 _review_context()` → 同上 + +3. `product_reviews` 归档而不是删除： + +```sql +ALTER TABLE product_reviews RENAME TO product_reviews_archived_20261101; +``` + +**必须明确的功能退化**：`product_reviews` 存的是好/中/差全量，差评三表只有差评。下线后： +- 风格分析里的「评价等级分布」从三档变成只有差评一档（差评样本反而更完整：天猫 97 + 京东 32 + 抖音 14，而 `product_reviews` 里天猫是 0、京东 8 月就断了）。 +- 营销日报的「近 7 天差评归因」不受影响，反而多了天猫来源。 +- 若产品侧确实需要好评样本沉淀，需要把三个差评采集器扩成「按等级采集」，不要恢复 `product_reviews`。 + +另外 `dy_product_scraping.py` 主流程里的评论采集动作只产出本地 JSON，`product_reviews` 停写后就没有消费方，建议同步关掉（或保留但不再进 `product.daily`）。 + +### P2 loader 下发 style_id（1 处改动，收益最大） + +`src/gyxx_flow/adapters/workflow_config.py:322 merge_lifecycle_style_records()`： + +```python +output["style_id"] = lifecycle.get("id") # dim_style.id +output["style_match_method"] = "dim_style" +``` + +同时 `source_record_id` 保留（控制台/账本还在用），只是不再承担传 ID 的职责。 + +影响面：所有通过 `runtime_workflow_configs()` 取款的脚本自动获得 `style_id`，包括日报聚合、周报、ERP 全店铺、人像、主图、声明式分析。这一步不改表、不改写入逻辑，可以先合入验证 loader 输出。 + +### P3 采集表加列 + 回填 + 写入端填充 + +**统一 DDL 模板**（以 `daily_style_metrics` 为例，其余同构）： + +```sql +ALTER TABLE daily_style_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT; +UPDATE daily_style_metrics d SET style_id = s.id +FROM dim_style s WHERE s.style_name = d.style_name AND d.style_id IS NULL; +CREATE INDEX IF NOT EXISTS daily_style_metrics_style_date_idx + ON daily_style_metrics (style_id, metric_date DESC); +``` + +按表处理（**必须匹配**的加 `NOT NULL`，**允许未匹配**的保持可空）： + +| 表 | 款式来源 | style_id | 写入端改动 | +|---|---|---|---| +| `daily_style_metrics` | loader 款式清单 | NOT NULL | `aggregate_daily_final.py` + `db.upsert_daily_metrics` | +| `weekly_style_metrics` | 从 daily 聚合 | NOT NULL | `weekly_aggregate.py` | +| `erp_all_shop_style_daily_metrics` | loader ERP 款 | NOT NULL | `backfill_erp_all_shop_daily.py` | +| `daily_persona_metrics` | loader | NOT NULL | `collect_*_persona_to_bitable.py` | +| `main_image_creatives` | loader | NOT NULL | `insert_main_image_records.py` / `insert_jd_main_image_records.py` | +| `fact_style_analysis_reports` | loader | NOT NULL | `analyze_style.py` | +| `decline_events` / `fact_alert_history` | daily 派生 | NOT NULL | `check_nine_day_decline.py` | +| `jd_self_weekly_sales` | loader | NOT NULL | `jd_self_inventory_sales_collector.py` | +| `persona_screenshots` | loader | NOT NULL | `taobao_dmp_item_crowd_insight_screenshots.py` | +| `product_daily_metrics` | 平台商品 ID 反查 | **可空** | `import_product_daily.py` | +| `jd_product_ad_daily_metrics` / `dy_product_ad_daily_metrics` | 名称/ID 反查 | **可空**（已有 `match_status`） | 两个 `import_*_ad_*.py` | +| 差评三表 | 反查 | **可空** | `collectors/*_negative_reviews.py` 入库侧 | + +**`product_daily_metrics` 的回填要处理系列聚合带来的歧义**（当前 0 条歧义，但要按锚点优先写死优先级）： + +```sql +UPDATE product_daily_metrics m +SET style_id = pick.style_id +FROM LATERAL ( + SELECT d.id AS style_id + FROM dim_style d + WHERE (m.platform = 'tm' AND d.tm_spus @> to_jsonb(m.product_id)) + OR (m.platform = 'jd' AND d.jd_spus @> to_jsonb(m.product_id)) + OR (m.platform = 'dy' AND d.dy_spus @> to_jsonb(m.product_id)) + ORDER BY d.is_series_anchor DESC, LENGTH(d.style_name), d.id + LIMIT 1 +) pick +WHERE m.style_id IS NULL; +``` + +（真实脚本里用 Python 侧 `_resolve_style_match()` 复用同一套优先级，SQL 只用于一次性回填。） + +回填后预期的非空比例 = 2.2 节的实测值：**dy 47% / jd 55% / tm 48%**。这不是 bug——剩下的是 458 个未建档商品 ID，要靠 P5 的人补 spus 才能涨。 + +### P4 约束 + 下游切 join + +1. ~~对「必须匹配」的表加外键~~ → **改为不建物理外键（2026-11-01 定案）**。 + +原始设想是给 10 张「必须匹配」的表加 `FOREIGN KEY (style_id) REFERENCES dim_style(id) ON UPDATE CASCADE`， +实施后又按业主意见拆掉了。理由是采集链路的语义与硬约束冲突： + +- 采集要求 **fail-soft**：一个商品 ID 没建档，正确行为是把该行 `style_id` 留 NULL 并标 + `unmatched`，继续写其它行。物理外键会让整批 INSERT/UPDATE 直接报错，等于「一个商品 + 未建档 → 当天数据全丢」，比留 NULL 更糟。 +- 款式会动：拆系列、合并同款、重建 dim_style 行都要先卸约束再动数据，运维成本不小。 +- data-hub 侧 22 张表本来就用 `style_id` 关联、**不带物理外键**，两边保持一致才好维护。 + +改为**逻辑外键**，完整性由三处保证： + +| 环节 | 手段 | +|---|---| +| 写入 | loader 下发 `style_id`（源自 dim_style 本身）；缺失时 `attach_style_ids` / `resolve_style_ids_by_product` 批量兜底 | +| 校验 | `backfill_style_id.py --dry-run` 预演；`v_style_id_coverage` 报每张表覆盖率 | +| 观测 | `v_style_unmatched_product`（日报口径）/ `v_style_unmatched_source`（全源口径）列未建档清单 | + +老库里早期建过的 10 个 `<表名>_style_id_fk` 由 `schema.sql` 里的 `DROP CONSTRAINT IF EXISTS` +幂等清理；只保留 `*_style_id_idx` 索引（join 性能靠索引，不靠约束）。 + +2. data-hub 侧把 `metric.style_name = style.name` 换成 `metric.style_id = style.id`。涉及 `ProductLifecycleMapper.xml` 第 120、155 行（`daily_style_metrics`），以及 `JOIN dim_style dim ON dim.style_name = style.name`（第 15 行）。改完后 text join 变 bigint join，是纯性能收益；`style_name` 列仍在，改错了可以立即回切。 + +3. 加测试： + - `tests/.../test_daily_style_metrics_style_id.py`：写入必须带 `style_id`，且等于 loader 输出的 id + - `tests/.../test_style_id_coverage.py`：`v_style_id_coverage` 里「必须匹配」的表 `with_style_id == rows` + - 现有 `test_import_tmall_product_ads.py` 的风格（断言 SQL 形状）继续用在新增字段上 + +### P5 数据治理（不是代码活，但决定 P3 的收益） + +`v_style_unmatched_product` 出来的 458 个商品 ID 需要在 dim_style 建档页补 `tm_spus/jd_spus/dy_spus`。分批推进：先补最近 90 天有销量的，历史行允许永久 NULL。 + +## 五、风险与取舍 + +| 风险 | 影响 | 应对 | +|---|---|---| +| 款名在 dim_style 被改，历史表 `style_name` 变孤儿 | 现在 join 直接丢数据，这正是要 style_id 的原因 | P3 之后 join 走 id，款名改动不再影响 | +| 加 `NOT NULL` 时某天有未建档款写入 | 写入失败，工作流中断 | 先在写入端做「未匹配记 NULL + 打 WARN」，P5 补齐后再收紧为 NOT NULL | +| `product_reviews` 下线后没有好评样本 | 风格分析评价分布退化 | 明确接受；需要时扩差评采集器等级，不恢复旧表 | +| 改 PK（有人会想顺手做） | data-hub 6 处 join + 飞书脚本全回归 | **本次明确不做**，PK 保持 style_name | +| 差评表天猫侧没有 product_id | 只能靠商品名反查款式 | 采集器补 product_id（页面有商品链接可解析），或退化为名称匹配并标记 `unmatched` | + +## 六、验收标准 + +1. `v_style_id_coverage`：`daily_style_metrics`、`erp_all_shop_style_daily_metrics`、`weekly_style_metrics` 的 `with_style_id = rows`（100%）。 +2. `product_daily_metrics` 的 `style_id` 非空比例不低于 2.2 节的实测基线（47%/55%/48%），且逐月上升。 +3. `product_reviews` 无新增行；两个消费方在 `v_negative_reviews` 上跑通，风格分析与营销日报输出结构不变。 +4. data-hub 产品生命页、营销中台预算页的款式指标与改造前同月数值一致（对账 2026-10 全月）。 + +## 七、建议执行顺序 + +P0 → P1（下线评论链路，独立可交付）→ P2（loader 一行改动，验证输出）→ P3（分表灰度，先做 `daily_style_metrics`）→ P4（约束 + 下游切换）→ P5（补建档）。 + +P1 与 P2/P3 没有依赖关系，可以先做 P1 拿到收益（少一条采集链路、少一张表），再从容做款式主键统一。 + +## 八、实施记录（2026-11-01） + +### 已落地 + +| 阶段 | 成果 | +|---|---| +| P0 | `v_style_unmatched_product`、`v_style_id_coverage` 两个观测视图已建并验证 | +| P1 | 删 `import_product_reviews.py` / `orchestrate_review_collection.py`；移除 `product.review.orchestrate` 命令与 runtime binding；`run_collect` 不再调评论入库；建 `v_negative_reviews`；两个消费方改读视图；`product_reviews` 归档为 `product_reviews_archived_20261101`（1936 行已备份到 `var/tmp/product_reviews_archive_20261101.json`） | +| P2 | `merge_lifecycle_style_records` 输出 `style_id` + `style_match_method`；loader 透传；实库验证 **204/204 款全部拿到 style_id** | +| P3 | 14 张采集表加 `style_id`（+ `style_match_method`）+ 索引；db 层 `attach_style_ids` / `resolve_style_ids_by_product` 统一兜底；`backfill_style_id.py` 回填历史 | +| P4 | ~~10 张「必须匹配」表加 `style_id → dim_style(id)` 物理外键~~ → 已按要求**全部拆除**，改逻辑外键（库里 0 个 FK，只留 14 个 `style_id` 索引）；新增 `tests/modules/product_commerce/test_style_id_identity.py`（28 条契约测试） | +| P5 | 提供 `v_style_unmatched_product` 清单（dy 170 / jd 99 / tm 189），需业务侧补 spus | + +### 回填结果（实库） + +| 表 | 行数 | 有 style_id | 比例 | +|---|---|---|---| +| daily_style_metrics | 17,352 | 17,352 | 100% | +| weekly_style_metrics | 532 | 532 | 100% | +| erp_all_shop_style_daily_metrics | 9,399 | 9,399 | 100% | +| daily_persona_metrics | 2,357 | 2,357 | 100% | +| main_image_creatives | 9,094 | 9,094 | 100% | +| fact_style_analysis_reports | 465 | 465 | 100% | +| decline_events / fact_alert_history | 14 / 14 | 14 / 14 | 100% | +| persona_screenshots | 25 | 25 | 100% | +| jd_self_weekly_sales | 117 | 111 | 94.9%（6 款已不在 dim_style） | +| tm_product_ad_daily_metrics | 11,727 | 11,108 | 94.7% | +| jd_product_ad_daily_metrics | 33,808 | 24,707 | 73.1% | +| dy_product_ad_daily_metrics | 18,202 | 10,875 | 59.7% | +| product_daily_metrics | 75,568 | 37,098 | 49.1%（未建档商品 ID 所致） | + +### 与方案不同之处 + +1. **天猫广告表已先行迁到独立表**：方案编写时天猫指标还写在 + `product_daily_metrics.raw_data.tm_ad_metrics`，实施期间该迁移已完成，因此改为给 + `tm_product_ad_daily_metrics` 加 `style_id`（并纳入 `v_style_id_coverage`）， + 并未改动 `raw_data` 节点。 +2. **写入端没有逐个脚本改**：方案原计划在 13 个写入脚本里各自填充，实际收敛到 db 层 + 的 `attach_style_ids` / `resolve_style_ids_by_product`（一次批量查询），再加 loader + 下发。少改 13 处、不会漏，代价是 upsert 多一次按款名/商品 ID 的查询。 +3. **`dy_product_scraping.py` 默认流程的评论采集直接摘除**（仍保留 `--reviews-only` + 显式入口），比方案建议的「保留但不再进 product.daily」更彻底。 +4. **物理外键最终没有保留**：P4 建过的 10 个 `<表名>_style_id_fk` 已全部 DROP， + `schema.sql` 改为「幂等清理 + 只建索引」。`style_id` 与 `dim_style.id` 是纯逻辑 + 外键关系，理由见 P4 段（fail-soft 采集语义、款式可重建、与 data-hub 对齐）。 + +### 仍待处理 + +- **data-hub 侧 join 未切**（`ProductLifecycleMapper.xml` 6 处仍是 `metric.style_name = style.name`）。 + 现在两边都有 id，可以随时切；切换后是 bigint join 替代 text join。 +- P5 补建档：458 个商品 ID（dy 170 / jd 99 / tm 189）写进 `dim_style.spus`。 +- 既有红灯（与本次无关）：`test_tm_brand_profiles` 的 backfill 用例、`shop_intelligence` + 的 12 个 jd 竞店用例，改动前即失败。 + +## 九、P5 补 spus 实施记录（2026-11-01） + +### 依据的权威字段 + +补 spus 的匹配依据不是商品标题，而是 **`product_daily_metrics.product_no` —— 它存的就是款名**： + +| 平台 | `product_no` 填充率 | 可用性 | +|---|---:|---| +| 天猫 | 100%（29552/29552） | 权威 | +| 京东 | 97.8%（16664/17044） | 权威 | +| 抖音 | 0%（0/28972） | 只能靠标题 | + +这一条此前被忽略，导致「458 个未建档」被误判为只能猜标题。实际天猫/京东 136 个商品 +靠 `product_no` 就精确命中，另有 31 个靠前后缀近似命中（「蓝鹊pro」→「蓝鹊」、 +「知更大号」→「知更」、「星迹」→「星迹2」）。 + +注意 `product_no` **不是** ERP 编码：`dim_style.erp_style_codes` 存的是 `["10510"]` +这种数字码，两者用 `product_no = erp_style_codes` 只能对上 2 个，别混淆。 + +### 匹配算法（四路，按优先级） + +1. `product_no` 精确 = `style_name` → 直接写； +2. `product_no` 与款名互为前后缀且长度差 ≤ 3，且候选唯一 → 直接写； +3. 标题包含完整款名（抖音只能走这条）→ 用「命中位置后的版本后缀」防串款： + 标题「盖亚斜挎包**3**」、`dim_style` 有「盖亚斜挎3」时必须指后者而非「盖亚斜挎」； + 若该版本款没建档，判「版本款未建档」而不是硬归到基础款； +4. 编码与标题互相矛盾 → 进人工清单，**不自动写**。 + +实测第 4 类里最贵的一条：天猫 `649545919334` 商品编码写「理岩」、标题写「阿波罗air」， +支付额 **50.2 万**。猜错就是 50 万挂错款，只能人工定。 + +### 写入结果 + +| 项 | 数值 | +|---|---:| +| 写入商品 ID | 199 个 | +| 涉及款式 | 67 个 | +| 覆盖 `product_daily_metrics` 行 | 22,636 | +| 未写（物料 / 冲突待定） | 87 个 | + +合并策略：`tm_spus / jd_spus / dy_spus` 取**并集追加**，不覆盖已有值；写前备份到 +`var/tmp/dim_style_spus_backup_20261101.json`（204 行）。 + +### 覆盖率变化 + +| 表 | 补前 | 补后 | +|---|---:|---:| +| product_daily_metrics | 49.1% | **79.0%** | +| tm_product_ad_daily_metrics | 94.7% | **99.4%** | +| jd_product_ad_daily_metrics | 73.1% | **92.4%** | +| dy_product_ad_daily_metrics | 59.7% | **83.2%** | + +广告表的提升来自新增的第三段回填（按商品 ID + 补空 `style_name`）：京东广告里 +32 个商品 ID 就占了 9,101 行空值，补完 `style_name` 空白从 9,095 降到 2,554， +抖音从 7,327 降到 3,062。这意味着 **data-hub 侧不改任何 SQL，按款名的 join 已经多认回 +11,359 行**。 + +### 剩余 259 个未建档 ID 的构成 + +| 分类 | 商品数 | 支付额 | 处置 | +|---|---:|---:|---| +| 待建档-真款 | 128 | 6.9 万 | 需在 dim_style 新建款（星云系列链接、拾影单肩包、幻月、纪元、极地冰冠、阿瑞斯、盖亚S1、漫游者、光/暗系列等） | +| 待建档-ozko 线 | 20 | 0.2 万 | ozko 子品牌款（逐云、微笑系列、甜酷系列、银鸥、独角兽、HB、cdbb、761X…） | +| 白名单-物料配件 | 30 | 5.6 万 | 不该进 spus（内胆包、数据线、风扇、防丢器…） | +| 白名单-服务/券 | 4 | 0 | 补邮费、兑换券、补贴 | +| 冲突待定 | 10 | 62.9 万 | 编码与标题矛盾，需人工定 | + +清单：`var/tmp/spus_todo.csv`（按分档 + 金额排序）。 + +### 新增的长期机制 + +- `backfill_style_id.py` 增加 `AD_PRODUCT_TABLES` 与 `_backfill_ad_tables_by_product`， + 广告表按 `spu_id / product_id` 二次回填，并只在 `style_name` 为空时补权威款名 + （`COALESCE(NULLIF(BTRIM(style_name), ''), %s)`，不覆盖平台给出的值）。 +- 新增视图 `v_style_unmatched_source`：`product_daily_metrics` + 三张广告表统一口径， + 补建档只看这一个视图就够（实测与原 `v_style_unmatched_product` 数值一致， + 说明没有纯投流商品只出现在广告表）。 diff --git a/src/gyxx_flow/adapters/integration.py b/src/gyxx_flow/adapters/integration.py index 2dafec4..d691fba 100644 --- a/src/gyxx_flow/adapters/integration.py +++ b/src/gyxx_flow/adapters/integration.py @@ -766,6 +766,11 @@ class RuntimeIntegrationCatalog: f"{binding.profile_dir.name}-luggage" ) ), + "GUANGHE_NEW_BAG_USER_DATA_DIR": str( + binding.profile_dir.with_name( + f"{binding.profile_dir.name}-new-bag" + ) + ), "DY_COOKIES_FILE": str(binding.cookie_file), "DY_STORAGE_STATE_FILE": str(binding.storage_state_file), } diff --git a/src/gyxx_flow/adapters/workflow_config.py b/src/gyxx_flow/adapters/workflow_config.py index a090024..7619d53 100644 --- a/src/gyxx_flow/adapters/workflow_config.py +++ b/src/gyxx_flow/adapters/workflow_config.py @@ -360,6 +360,10 @@ def merge_lifecycle_style_records( record = dict(row) for field in URL_FIELDS: record[field] = "" + # 已不在 dim_style 的残留款：没有主键可下发，显式标记未匹配， + # 避免下游把 None 当成「还没回填」而不是「确实没建档」。 + record["style_id"] = None + record["style_match_method"] = "unmatched" records.append(record) continue @@ -423,6 +427,17 @@ def merge_lifecycle_style_records( template["sales_bitable_url"] = sales_bitable_url template["series_name"] = series_name template["is_series_anchor"] = is_series_anchor + # 款式主键统一 (2026-11-01): dim_style.id 作为正式字段下发。 + # 原先它只被编码进 source_record_id="lifecycle_style:" 字符串，写入脚本 + # 拿不到，只能靠 style_name 关联——改款名历史行就成孤儿。这里提成字段后， + # 所有经 runtime_workflow_configs() 取款的脚本自动获得 style_id。 + style_id_value = lifecycle.get("id") + template["style_id"] = ( + int(style_id_value) if style_id_value is not None else None + ) + template["style_match_method"] = ( + "dim_style" if style_id_value is not None else "unmatched" + ) for field in URL_FIELDS: template[field] = str(lifecycle.get(field) or "").strip() # 采集开关由产品生命进程在 dim_style 上维护（系列级，读锚点那行）。 diff --git a/src/gyxx_flow/console.py b/src/gyxx_flow/console.py index a0055ef..da9d9b7 100644 --- a/src/gyxx_flow/console.py +++ b/src/gyxx_flow/console.py @@ -191,7 +191,6 @@ _WORKFLOW_NAMES = { "product.backfill": "商品数据补采", "product.market_rank": "市场排行周采集", "product.tmall_baibu_apply": "天猫百亿补贴批量报名", - "product.review_collection": "商品评价采集", "product.weekly_aggregate.documented_missing": "商品周聚合（待接入）", "shop.metrics.weekly": "店铺经营指标周采集", "shop.competitor.weekly": "竞店指标周采集", @@ -2247,7 +2246,7 @@ class WorkflowConsoleService: jd_customer_service_store: str | None = None if workflow_id == "product.video_upload": raw_store = payload.get("store", "flagship") - if raw_store not in {"flagship", "luggage"}: + if raw_store not in {"flagship", "luggage", "new_bag"}: raise ConsoleRequestError("天猫视频上传店铺参数无效") video_upload_store = str(raw_store) tmall_topic_keyword = _normalize_tmall_topic_keyword( diff --git a/src/gyxx_flow/modules/content_marketing/daily_marketing_report.py b/src/gyxx_flow/modules/content_marketing/daily_marketing_report.py index 2770077..93238ef 100644 --- a/src/gyxx_flow/modules/content_marketing/daily_marketing_report.py +++ b/src/gyxx_flow/modules/content_marketing/daily_marketing_report.py @@ -1019,18 +1019,28 @@ def _creative_context(metric_date: date, style_names: list[str]) -> list[dict[st def _review_context(metric_date: date, style_names: list[str]) -> list[dict[str, Any]]: + """近 7 天差评归因（读 v_negative_reviews）。 + + 2026-11-01 起 ``product_reviews`` 下线，改读三平台差评归一视图。口径随之变化： + + * 数据源只剩差评，因此 ``sample_size`` = 差评条数、``negative_rate`` 恒为 1.0， + 不再表示「差评/全部评价」；真正要看比例的场合请取平台评价总数另算。 + * 输出门槛从「窗口内至少 5 条评价」改为「至少 1 条差评」——旧门槛在只有差评 + 的数据源上会导致几乎永不产出。 + * 覆盖天猫（旧表天猫为 0 行）、京东（旧表 8 月断更），负向样本反而更全。 + """ + if not style_names: return [] - negative_terms = ("差", "坏", "退", "重", "小", "掉色", "线头", "不值", "失望") with get_conn() as conn: with conn.cursor() as cur: cur.execute( """ - SELECT style_name, platform, content, score, tags - FROM product_reviews + SELECT style_name, platform, content + FROM v_negative_reviews WHERE style_name = ANY(%s) AND comment_date BETWEEN %s AND %s AND content IS NOT NULL AND BTRIM(content) <> '' - ORDER BY style_name, comment_date DESC, id DESC + ORDER BY style_name, comment_date DESC, review_at DESC """, (style_names, metric_date - timedelta(days=6), metric_date), ) @@ -1040,17 +1050,15 @@ def _review_context(metric_date: date, style_names: list[str]) -> list[dict[str, grouped.setdefault(row[0], []).append(row) output = [] for style_name, items in grouped.items(): - if len(items) < 5: - continue - negative = [item for item in items if (item[3] is not None and item[3] <= 2) or any(term in item[2] for term in negative_terms)] output.append({ "style_name": style_name, "window_start": metric_date - timedelta(days=6), "window_end": metric_date, "sample_size": len(items), - "negative_count": len(negative), - "negative_rate": round(len(negative) / len(items), 4), - "negative_examples": [item[2][:80] for item in negative[:2]], + "negative_count": len(items), + "negative_rate": 1.0, + "negative_examples": [item[2][:80] for item in items[:2]], + "source": "v_negative_reviews", }) return output diff --git a/src/gyxx_flow/modules/product_commerce/analyze_style.py b/src/gyxx_flow/modules/product_commerce/analyze_style.py index d0c6eba..1732107 100644 --- a/src/gyxx_flow/modules/product_commerce/analyze_style.py +++ b/src/gyxx_flow/modules/product_commerce/analyze_style.py @@ -109,12 +109,26 @@ def missing_window_dates(rows: list[dict], start: date, end: date) -> list[date] def pull_reviews(style: str, start: date, end: date) -> dict: - """仅返回当前三日窗口内的评论分布与样本。""" + """仅返回当前三日窗口内的差评分布与样本。 + + 2026-11-01 起 ``product_reviews`` 下线，改读三平台差评归一视图 + ``v_negative_reviews``（tm + jd + dy 三张差评表 UNION）。 + + 口径变化（必须知道）：旧表存好/中/差全量，视图只有差评一档，所以这里 + 返回的 key 会集中在 ``<平台>|负面评价``。换来的是覆盖率更高——天猫差评 + 旧表根本没有（0 行），京东旧表 8 月就断更。 + + 款式归因走 ``style_id``（dim_style.id），款名只作显示冗余。 + """ + with get_conn() as c, c.cursor() as cur: cur.execute( - """SELECT platform, rating, COUNT(*) n, array_agg(content ORDER BY comment_date DESC) samples - FROM product_reviews - WHERE style_name=%s AND comment_date BETWEEN %s AND %s + """SELECT platform, rating, COUNT(*) n, + array_agg(content ORDER BY comment_date DESC) samples + FROM v_negative_reviews + WHERE style_id = (SELECT id FROM dim_style WHERE style_name = %s) + AND comment_date BETWEEN %s AND %s + AND content IS NOT NULL AND BTRIM(content) <> '' GROUP BY platform, rating""", (style, start, end), ) @@ -380,7 +394,7 @@ def aggregate_style_total(platforms: dict[str, dict]) -> dict: LEGACY_SYSTEM = """你是光影行星 (GYXX) 的资深电商数据分析师, 擅长把数据现象挖到根因并给出可执行方案. 用户会给定一个款式近 3 天的真实业务数据: - 站内 (店铺自身): 天猫/京东/抖音三平台当前 3 天的 daily_style_metrics、 - product_reviews、daily_persona_metrics、main_image_creatives + v_negative_reviews(三平台差评)、daily_persona_metrics、main_image_creatives - 站外 (种草营销): cmt_notes_master (笔记+合作投入), cmt_comments (笔记评论), cmt_creators (创作者静态属性) @@ -787,7 +801,7 @@ def build_user_prompt(style: str, start: date, end: date, payload: dict) -> str: # 人物画像 {json.dumps(payload['persona'], ensure_ascii=False, indent=2)} -# 商品评价 +# 商品差评 (仅负向评价, 覆盖天猫/京东/抖音; 无好评与中评样本) {json.dumps(payload['product_reviews'], ensure_ascii=False, indent=2)} # 主图表现 diff --git a/src/gyxx_flow/modules/product_commerce/backfill_style_id.py b/src/gyxx_flow/modules/product_commerce/backfill_style_id.py new file mode 100644 index 0000000..6e02786 --- /dev/null +++ b/src/gyxx_flow/modules/product_commerce/backfill_style_id.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +"""回填采集表的 style_id (= dim_style.id)，完成款式主键统一。 + +背景 +---- +2026-11-01 之前，采集端所有表只用 ``style_name`` 关联款式，而 data-hub 侧 22 张 +表用的是 ``style_id``（= dim_style.id）。同一库里两种引用方式的后果是：款名在 +dim_style 被改一次，历史指标表就整体变孤儿，join 静默丢数据。 + +本脚本负责把**历史数据**补上 style_id；**新数据**由写入端负责： + * loader 下发（``merge_lifecycle_style_records`` → ``style_config_loader``） + * db 层兜底（``db.attach_style_ids`` / ``db.resolve_style_ids_by_product``） + +两类表，两种回填口径 +-------------------- +1. **按款名**（13 张）：款式清单本来就来自 dim_style loader，理论上 100% 命中。 +2. **按平台商品 ID**（``product_daily_metrics`` + 三张广告表）：表里没有可用的 + 款名列（``product_daily_metrics`` 完全没有；广告表的 ``style_name`` 由平台 + 报表决定，京东 27%、抖音 40% 是空的），只能拿 ``platform + product_id`` 去 + ``tm_spus / jd_spus / dy_spus`` 里反查。未建档的商品 ID 回填不了 —— 这不是 + 脚本的问题，要用 ``v_style_unmatched_product`` 推动补建档。 + 广告表回填时会顺带把空的 ``style_name`` 补成 dim_style 的权威款名，让下游 + 现有按款名的 join 立即生效。 + +用法 +---- + .venv/Scripts/python.exe backfill_style_id.py --dry-run # 只报数 + .venv/Scripts/python.exe backfill_style_id.py # 执行 + .venv/Scripts/python.exe backfill_style_id.py --tables daily_style_metrics + .venv/Scripts/python.exe backfill_style_id.py --skip-schema # 不跑建表 + .venv/Scripts/python.exe backfill_style_id.py --skip-ad-tables # 只补款名表 + +幂等：所有 UPDATE 都带 ``style_id IS NULL`` 条件，重复执行不会覆盖已有归属。 +""" + +from __future__ import annotations + +import argparse +import sys +from typing import Any, Sequence + +try: + from .db import ( + STYLE_MATCH_BY_NAME, + STYLE_MATCH_BY_PRODUCT, + STYLE_MATCH_UNMATCHED, + get_conn, + init_schema, + resolve_style_ids_by_product, + ) +except ImportError: # Direct script execution from the module root. + from db import ( # type: ignore[no-redef] + STYLE_MATCH_BY_NAME, + STYLE_MATCH_BY_PRODUCT, + STYLE_MATCH_UNMATCHED, + get_conn, + init_schema, + resolve_style_ids_by_product, + ) + + +# 按款名回填：这些表的 style_name 来自 dim_style 款式清单，属于「必须匹配」。 +NAME_TABLES: tuple[str, ...] = ( + "daily_style_metrics", + "weekly_style_metrics", + "erp_all_shop_style_daily_metrics", + "daily_persona_metrics", + "main_image_creatives", + "fact_style_analysis_reports", + "decline_events", + "fact_alert_history", + "jd_self_weekly_sales", + "persona_screenshots", + "jd_product_ad_daily_metrics", + "dy_product_ad_daily_metrics", + "tm_product_ad_daily_metrics", +) + +# 按平台商品 ID 回填：允许未匹配（商品 ID 还没建档到 dim_style.spus）。 +PRODUCT_TABLE = "product_daily_metrics" +PLATFORM_COLUMNS = ("tm", "jd", "dy") + +# 广告表：表里带平台商品 ID，但 style_name 常为空（平台报表不给款式名）。 +# 只按款名的回填对它们无效，必须再走一遍商品 ID 反查。 +# {表名: (商品 ID 列, 平台编码)} +AD_PRODUCT_TABLES: dict[str, tuple[str, str]] = { + "tm_product_ad_daily_metrics": ("product_id", "tm"), + "jd_product_ad_daily_metrics": ("spu_id", "jd"), + "dy_product_ad_daily_metrics": ("product_id", "dy"), +} + + +def _coverage(conn, table: str) -> tuple[int, int]: + with conn.cursor() as cur: + cur.execute( + f"SELECT COUNT(*), COUNT(style_id) FROM {table}" + ) + rows, with_id = cur.fetchone() + return int(rows or 0), int(with_id or 0) + + +def _backfill_by_name(conn, table: str, *, dry_run: bool) -> dict[str, Any]: + """按款名精确回填。返回 {before, after, updated}。""" + + before_rows, before_with_id = _coverage(conn, table) + if dry_run: + with conn.cursor() as cur: + cur.execute( + f"""SELECT COUNT(*) + FROM {table} t + JOIN dim_style s ON s.style_name = t.style_name + WHERE t.style_id IS NULL""" + ) + pending = int(cur.fetchone()[0] or 0) + return { + "table": table, + "rows": before_rows, + "with_style_id_before": before_with_id, + "updated": pending, + "dry_run": True, + } + + with conn.cursor() as cur: + cur.execute( + f"""UPDATE {table} t + SET style_id = s.id, + style_match_method = COALESCE(t.style_match_method, %s) + FROM dim_style s + WHERE t.style_id IS NULL + AND t.style_name = s.style_name""", + (STYLE_MATCH_BY_NAME,), + ) + updated = cur.rowcount or 0 + after_rows, after_with_id = _coverage(conn, table) + return { + "table": table, + "rows": after_rows, + "with_style_id_before": before_with_id, + "with_style_id_after": after_with_id, + "updated": updated, + } + + +def _backfill_product_table(conn, *, dry_run: bool) -> dict[str, Any]: + """按 platform + product_id 反查回填 product_daily_metrics。 + + 逐平台取去重商品 ID → 用 db.resolve_style_ids_by_product 解析（锚点优先、 + 款名最短优先，与写入端同一套优先级）→ 按 (platform, product_id) 批量更新。 + """ + + before_rows, before_with_id = _coverage(conn, PRODUCT_TABLE) + planned: list[tuple[int, str, str]] = [] + unmatched_total = 0 + with conn.cursor() as cur: + for platform in PLATFORM_COLUMNS: + cur.execute( + f"""SELECT DISTINCT product_id FROM {PRODUCT_TABLE} + WHERE platform = %s AND style_id IS NULL AND product_id IS NOT NULL""", + (platform,), + ) + product_ids = [str(row[0]) for row in cur.fetchall() if row[0] is not None] + if not product_ids: + continue + matched = resolve_style_ids_by_product(conn, platform, product_ids) + unmatched_total += len(product_ids) - len(matched) + planned.extend( + (style_id, platform, product_id) + for product_id, style_id in matched.items() + ) + + if dry_run: + return { + "table": PRODUCT_TABLE, + "rows": before_rows, + "with_style_id_before": before_with_id, + "planned_keys": len(planned), + "unmatched_product_ids": unmatched_total, + "dry_run": True, + } + + updated = 0 + with conn.cursor() as cur: + for style_id, platform, product_id in planned: + cur.execute( + f"""UPDATE {PRODUCT_TABLE} + SET style_id = %s, style_match_method = %s + WHERE platform = %s AND product_id = %s AND style_id IS NULL""", + (style_id, STYLE_MATCH_BY_PRODUCT, platform, product_id), + ) + updated += cur.rowcount or 0 + # 回填不了的显式标记 unmatched，供 v_style_unmatched_product 统计 + cur.execute( + f"""UPDATE {PRODUCT_TABLE} + SET style_match_method = %s + WHERE style_id IS NULL AND style_match_method IS NULL""", + (STYLE_MATCH_UNMATCHED,), + ) + + after_rows, after_with_id = _coverage(conn, PRODUCT_TABLE) + return { + "table": PRODUCT_TABLE, + "rows": after_rows, + "with_style_id_before": before_with_id, + "with_style_id_after": after_with_id, + "updated": updated, + "unmatched_product_ids": unmatched_total, + } + + +def _backfill_ad_tables_by_product( + conn, tables: Sequence[str], *, dry_run: bool +) -> list[dict[str, Any]]: + """按平台商品 ID 反查回填广告表的 style_id（顺带补空的 style_name）。 + + 广告表的 ``style_name`` 由平台报表决定，经常为空（京东 27%、抖音 40%）， + 只按款名回填对它们完全无效。这里改为用 ``spu_id`` / ``product_id`` 去 + ``dim_style._spus`` 反查，与写入端 ``resolve_style_ids_by_product`` + 共用同一套优先级。 + + ``style_name`` 只在原来为空时补写（``NULLIF(BTRIM(...), '')``），不覆盖 + 平台已经给出的值 —— 这样 data-hub 侧现有按款名 join 的口径立刻能多认回 + 一批行，不必等主键切换。 + """ + + results: list[dict[str, Any]] = [] + for table in tables: + id_column, platform = AD_PRODUCT_TABLES[table] + before_rows, before_with_id = _coverage(conn, table) + + with conn.cursor() as cur: + cur.execute( + f"""SELECT DISTINCT {id_column} FROM {table} + WHERE style_id IS NULL AND {id_column} IS NOT NULL + AND BTRIM({id_column}) <> ''""" + ) + product_ids = [str(row[0]) for row in cur.fetchall() if row[0] is not None] + + matched: dict[str, int] = {} + if product_ids: + matched = resolve_style_ids_by_product(conn, platform, product_ids) + + if dry_run: + results.append( + { + "table": table, + "rows": before_rows, + "with_style_id_before": before_with_id, + "planned_keys": len(matched), + "unmatched_product_ids": len(product_ids) - len(matched), + "dry_run": True, + } + ) + continue + + name_by_id: dict[int, str] = {} + if matched: + with conn.cursor() as cur: + cur.execute( + "SELECT id, style_name FROM dim_style WHERE id = ANY(%s)", + (list({v for v in matched.values()}),), + ) + name_by_id = {int(i): n for i, n in cur.fetchall()} + + updated = 0 + with conn.cursor() as cur: + for product_id, style_id in matched.items(): + cur.execute( + f"""UPDATE {table} + SET style_id = %s, + style_match_method = %s, + style_name = COALESCE(NULLIF(BTRIM(style_name), ''), %s) + WHERE {id_column} = %s AND style_id IS NULL""", + (style_id, STYLE_MATCH_BY_PRODUCT, name_by_id.get(style_id), product_id), + ) + updated += cur.rowcount or 0 + cur.execute( + f"""UPDATE {table} SET style_match_method = %s + WHERE style_id IS NULL AND style_match_method IS NULL""", + (STYLE_MATCH_UNMATCHED,), + ) + + after_rows, after_with_id = _coverage(conn, table) + results.append( + { + "table": table, + "rows": after_rows, + "with_style_id_before": before_with_id, + "with_style_id_after": after_with_id, + "updated": updated, + "unmatched_product_ids": len(product_ids) - len(matched), + } + ) + return results + + +def _print_coverage(conn) -> None: + with conn.cursor() as cur: + cur.execute("SELECT tbl, rows, with_style_id FROM v_style_id_coverage ORDER BY tbl") + rows = cur.fetchall() + cur.execute("SELECT platform, COUNT(*) FROM v_style_unmatched_product GROUP BY platform ORDER BY platform") + unmatched = cur.fetchall() + + print("\n[覆盖率] v_style_id_coverage") + print(f" {'表':<38}{'行数':>9}{'有 style_id':>12}{'比例':>9}") + for table, total, with_id in rows: + ratio = (with_id / total * 100) if total else 0.0 + print(f" {table:<38}{total:>9}{with_id:>12}{ratio:>8.1f}%") + + if unmatched: + print("\n[未建档] v_style_unmatched_product") + for platform, count in unmatched: + print(f" {platform}: {count} 个商品 ID 未写进 dim_style.spus") + print(" -> 这些行的 style_id 只能保持 NULL，需要在建档页补 spus") + else: + print("\n[未建档] v_style_unmatched_product: 无，全部商品 ID 已建档") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="回填采集表 style_id (= dim_style.id)，完成款式主键统一" + ) + parser.add_argument( + "--tables", + default="", + help="逗号分隔，只回填指定表；留空表示全部", + ) + parser.add_argument("--dry-run", action="store_true", help="只统计不写库") + parser.add_argument( + "--skip-schema", + action="store_true", + help="跳过 init_schema（列已存在时可用，节省时间）", + ) + parser.add_argument( + "--skip-product-table", + action="store_true", + help="跳过 product_daily_metrics（按商品 ID 反查，最慢的一步）", + ) + parser.add_argument( + "--skip-ad-tables", + action="store_true", + help="跳过三张广告表的按商品 ID 二次回填", + ) + args = parser.parse_args(argv) + + if not args.skip_schema and not args.dry_run: + init_schema() + + requested = { + item.strip() for item in str(args.tables or "").split(",") if item.strip() + } + name_tables = [t for t in NAME_TABLES if not requested or t in requested] + run_product = ( + not args.skip_product_table + and (not requested or PRODUCT_TABLE in requested) + ) + ad_tables = ( + [] + if args.skip_ad_tables + else [t for t in AD_PRODUCT_TABLES if not requested or t in requested] + ) + + results: list[dict[str, Any]] = [] + with get_conn() as conn: + try: + for table in name_tables: + result = _backfill_by_name(conn, table, dry_run=args.dry_run) + results.append(result) + print( + f"[{'DRY' if args.dry_run else 'OK '}] {table:<38} " + f"{result['with_style_id_before']}/{result['rows']} -> " + f"{result.get('updated', 0)} 行可填" + ) + if run_product: + result = _backfill_product_table(conn, dry_run=args.dry_run) + results.append(result) + if args.dry_run: + detail = ( + f"{result['planned_keys']} 个商品 ID 可解析, " + f"{result['unmatched_product_ids']} 个未建档" + ) + else: + detail = f"{result.get('updated', 0)} 行已填" + print( + f"[{'DRY' if args.dry_run else 'OK '}] {PRODUCT_TABLE:<38} " + f"{result['with_style_id_before']}/{result['rows']} -> {detail}" + ) + if args.dry_run: + conn.rollback() + else: + conn.commit() + except Exception: + conn.rollback() + raise + + if ad_tables: + # 广告表的 style_name 大量为空，按款名回填无效，必须再按商品 ID 走一遍。 + # 单独开事务，避免和上面的款名回填混在一个长事务里。 + with get_conn() as conn: + try: + ad_results = _backfill_ad_tables_by_product( + conn, ad_tables, dry_run=args.dry_run + ) + results.extend(ad_results) + for result in ad_results: + if args.dry_run: + detail = ( + f"{result['planned_keys']} 个商品 ID 可解析, " + f"{result['unmatched_product_ids']} 个未建档" + ) + else: + detail = f"{result.get('updated', 0)} 行已填" + print( + f"[{'DRY' if args.dry_run else 'OK '}] {result['table']:<38} " + f"{result['with_style_id_before']}/{result['rows']} -> {detail} (按商品ID)" + ) + if args.dry_run: + conn.rollback() + else: + conn.commit() + except Exception: + conn.rollback() + raise + + total_updated = sum(int(r.get("updated", 0)) for r in results) + print(f"\n{'[DRY]' if args.dry_run else '[DONE]'} 合计更新 {total_updated} 行") + + if not args.dry_run: + with get_conn() as conn: + _print_coverage(conn) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/gyxx_flow/modules/product_commerce/config/style_config_loader.py b/src/gyxx_flow/modules/product_commerce/config/style_config_loader.py index 0e1507e..8263c36 100644 --- a/src/gyxx_flow/modules/product_commerce/config/style_config_loader.py +++ b/src/gyxx_flow/modules/product_commerce/config/style_config_loader.py @@ -32,7 +32,7 @@ except ImportError: # Direct script execution from the module root. from runtime_paths import NORMALIZED_DATA_ROOT, STATE_ROOT, managed_data_path -CACHE_SCHEMA_VERSION = 8 +CACHE_SCHEMA_VERSION = 9 # 9: 款式主键统一, 每条 style 增加 style_id / style_match_method # 系列级飞书地址：约定只维护在锚点行，子款误填也会被整系列聚合去重后统一下发。 # 顺序即写入目标表的优先级说明顺序，不含 bitable_sku_master（人工维护，不参与采集）。 @@ -307,6 +307,10 @@ class StyleConfigLoader: "destinations": row.get("destinations", []), "series_name": row.get("series_name", ""), "is_series_anchor": bool(row.get("is_series_anchor", False)), + # 款式主键统一 (2026-11-01): dim_style.id 正式下发, 写入脚本 + # 直接用它落库, style_name 只作显示冗余。 + "style_id": row.get("style_id"), + "style_match_method": row.get("style_match_method", ""), # 采集开关随记录下发；缺字段（旧 cmt_styles 回退）按启用处理 "collect_enabled": True if row.get("collect_enabled") is None @@ -347,6 +351,8 @@ class StyleConfigLoader: def _empty(style_name: str) -> dict: return { "name": style_name, + "style_id": None, + "style_match_method": "", "brand": "", "erp_codes": [], "tm": {"item_ids": []}, @@ -379,6 +385,13 @@ class StyleConfigLoader: style = styles.setdefault(style_name, _empty(style_name)) + # 款式主键: dim_style.id, 与 data-hub 的 22 张表同一套主键。 + # merge 已把同款展开成多条平台记录, 取值一致, 取第一个非空即可。 + if style.get("style_id") is None and record.get("style_id") is not None: + style["style_id"] = int(record["style_id"]) + if not style.get("style_match_method") and record.get("style_match_method"): + style["style_match_method"] = str(record["style_match_method"]) + # 品牌（款式级别,取第一个非空） brand = str(record.get("brand") or "").strip() if brand and not style["brand"]: diff --git a/src/gyxx_flow/modules/product_commerce/db/__init__.py b/src/gyxx_flow/modules/product_commerce/db/__init__.py index 0316e1b..011d2ff 100644 --- a/src/gyxx_flow/modules/product_commerce/db/__init__.py +++ b/src/gyxx_flow/modules/product_commerce/db/__init__.py @@ -38,10 +38,12 @@ 写: import_product_daily.py (扫 data/单品数据/_downloads/* 入库) 读: 三平台单品日维度查询 / 跨平台对账 product_daily_metrics.raw_data.tm_ad_metrics - 写: import_tmall_product_ads.py (万相台商品投放 CSV) - 读: data-hub 产品生命进程广告成交金额 / 投产比 / 电商费比；节点内保存款式匹配结果 + 已废弃（2026-11-01）：天猫广告指标改为写 tm_product_ad_daily_metrics； + 历史节点由 scripts/migrate_tm_ad_metrics_to_table.py 迁移后清除 tm_product_ad_daily_metrics - 兼容旧版独立广告表；新流程不再写入该表 + 写: import_tmall_product_ads.py (万相台商品投放 CSV) + 读: data-hub 产品生命进程广告花费 / 广告归因成交 / 投产比 / 电商费比 + 注: 标量列供直接聚合，metrics 保留报表全部指标；款式按 tm_spus 反查 jd_product_ad_daily_metrics 写: import_jd_ad_costs.py (京东营销概况全站 SPU 与非全站 SKU) 读: 京东电商费用 / 款式广告成本分析 @@ -49,8 +51,9 @@ 写: import_douyin_qianchuan_ads.py (巨量千川商品推广日报) 读: 抖音电商费用 / 款式广告成本分析；款式按 ID 优先、名称兜底反查 product_reviews - 写: import_product_reviews.py (扫 data/{jd,dy}单品数据/评价*.json 入库) - 读: 评价分析 / 差评归因 / 标签挖掘 + 已下线 (2026-11-01): 差评统一由 shop_intelligence 三张差评表采集, + 消费方改读视图 v_negative_reviews; 存量表归档为 + product_reviews_archived_20261101, 不再写入。 persona_screenshots 写: taobao_dmp_item_crowd_insight_screenshots.py (v2 改造后只入库, 不推飞书; insert_persona_to_bitable_v2.py 已删) 读: 人物画像审计 / 飞书附件溯源 @@ -84,6 +87,7 @@ import os import time import unicodedata from contextlib import contextmanager +from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any, Iterable, Mapping, Sequence @@ -216,21 +220,36 @@ def ensure_tmall_product_ad_daily_table(conn) -> None: stat_date DATE NOT NULL, product_id TEXT NOT NULL, product_name TEXT, + style_name TEXT, + style_match_method TEXT NOT NULL DEFAULT 'unmatched', spend NUMERIC(18, 2) NOT NULL DEFAULT 0, direct_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0, indirect_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0, attributed_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0, reported_roas NUMERIC(18, 6), calculated_roas NUMERIC(18, 6), + metrics JSONB NOT NULL DEFAULT '{}'::jsonb, duplicate_row_count INTEGER NOT NULL DEFAULT 1, source_file TEXT, + source_row_number INTEGER, imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), UNIQUE (stat_date, product_id) ); + ALTER TABLE tm_product_ad_daily_metrics + ADD COLUMN IF NOT EXISTS style_name TEXT; + ALTER TABLE tm_product_ad_daily_metrics + ADD COLUMN IF NOT EXISTS style_match_method TEXT + NOT NULL DEFAULT 'unmatched'; + ALTER TABLE tm_product_ad_daily_metrics + ADD COLUMN IF NOT EXISTS metrics JSONB NOT NULL DEFAULT '{}'::jsonb; + ALTER TABLE tm_product_ad_daily_metrics + ADD COLUMN IF NOT EXISTS source_row_number INTEGER; CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_date_idx ON tm_product_ad_daily_metrics (stat_date DESC); CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_product_date_idx ON tm_product_ad_daily_metrics (product_id, stat_date DESC); + CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_style_date_idx + ON tm_product_ad_daily_metrics (style_name, stat_date DESC); COMMENT ON TABLE tm_product_ad_daily_metrics IS '天猫商品级投放日报，每日每商品一行；广告归因成交额仅用于计算投产比。'; """ @@ -332,6 +351,103 @@ def ensure_douyin_product_ad_daily_table(conn) -> None: cur.execute(sql) +# ── 款式主键 (style_id = dim_style.id) 解析 ────────────────────────────────── +# +# 采集端 2026-11-01 起统一用 style_id 关联款式, style_name 降级为显示冗余。 +# 好处: 与 data-hub 的 22 张表同一套主键; 改款名不再让历史行变孤儿。 +# +# 解析优先复用调用方传入的 style_id (loader 已下发), 缺失时在 db 层批量兜底, +# 避免为 13 张表的每个写入脚本单独加查询。 + +STYLE_MATCH_BY_NAME = "dim_style" +STYLE_MATCH_BY_PRODUCT = "product_id" +STYLE_MATCH_BY_ANCHOR = "anchor" +STYLE_MATCH_UNMATCHED = "unmatched" + +_PLATFORM_SPU_COLUMN = {"tm": "tm_spus", "jd": "jd_spus", "dy": "dy_spus"} + + +def resolve_style_ids(conn, style_names: Sequence[Any]) -> dict[str, int]: + """按款名批量解析 dim_style.id。查不到的款名不出现在返回值里。""" + + names = sorted( + {" ".join(str(name or "").split()) for name in style_names if str(name or "").strip()} + ) + if not names: + return {} + with conn.cursor() as cur: + cur.execute( + "SELECT style_name, id FROM dim_style WHERE style_name = ANY(%s)", + (names,), + ) + return {str(name): int(style_id) for name, style_id in cur.fetchall()} + + +def resolve_style_ids_by_product( + conn, platform: str, product_ids: Sequence[Any] +) -> dict[str, int]: + """按平台商品 ID 反查 dim_style.id。 + + 优先级: 系列锚点 > 款名最短 > id 最小。锚点优先是因为平台编码在 loader 里 + 已经整系列聚合到锚点, 同一商品 ID 归属锚点才与采集口径一致; 款名最短是为了 + 在「极星」与「极星双肩」同时命中时选更具体的那个。 + """ + + column = _PLATFORM_SPU_COLUMN.get(str(platform or "").strip()) + ids = sorted({str(item).strip() for item in product_ids if str(item or "").strip()}) + if not column or not ids: + return {} + sql = f""" + SELECT p.product_id, pick.id + FROM unnest(%s::text[]) AS p(product_id) + JOIN LATERAL ( + SELECT d.id + FROM dim_style d + WHERE d.{column} @> to_jsonb(p.product_id) + ORDER BY COALESCE(d.is_series_anchor, FALSE) DESC, + LENGTH(d.style_name), + d.id + LIMIT 1 + ) pick ON TRUE + """ + with conn.cursor() as cur: + cur.execute(sql, (ids,)) + return {str(pid): int(style_id) for pid, style_id in cur.fetchall()} + + +def attach_style_ids( + conn, + records: Sequence[dict], + *, + name_key: str = "style_name", +) -> list[dict]: + """就地补齐 records 的 style_id / style_match_method, 返回同一批 dict 副本。 + + 调用方已带 style_id 的（loader 下发）不覆盖, 只补缺失的。 + """ + + copied = [dict(record) for record in records] + pending = [ + str(record.get(name_key) or "").strip() + for record in copied + if record.get("style_id") is None + ] + mapping = resolve_style_ids(conn, pending) if pending else {} + for record in copied: + if record.get("style_id") is not None: + if not record.get("style_match_method"): + record["style_match_method"] = STYLE_MATCH_BY_NAME + continue + name = str(record.get(name_key) or "").strip() + style_id = mapping.get(name) + record["style_id"] = style_id + if not record.get("style_match_method"): + record["style_match_method"] = ( + STYLE_MATCH_BY_NAME if style_id is not None else STYLE_MATCH_UNMATCHED + ) + return copied + + # ── daily_style_metrics ────────────────────────────────────────────────────── def upsert_daily_metrics(conn, records: Sequence[dict]) -> int: @@ -351,6 +467,7 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int: "metric_date": r["metric_date"], "style_name": r["style_name"], "platform": r["platform"], + "style_id": r.get("style_id"), "sales": 0, "refund_orders": 0, "visitors": 0, @@ -359,6 +476,8 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int: "source": r.get("source"), } cur = dedup[key] + if cur.get("style_id") is None and r.get("style_id") is not None: + cur["style_id"] = r["style_id"] for fld in ("sales", "refund_orders", "visitors", "cart_users"): cur[fld] = int(cur.get(fld, 0) or 0) + int(r.get(fld, 0) or 0) existing_codes = set(cur.get("erp_style_codes") or []) @@ -367,7 +486,7 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int: cur["erp_style_codes"] = sorted(existing_codes) if r.get("source"): cur["source"] = r["source"] - records = list(dedup.values()) + records = attach_style_ids(conn, list(dedup.values())) rows = [ ( @@ -380,12 +499,15 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int: int(r.get("cart_users", 0) or 0), r.get("erp_style_codes") or [], r.get("source"), + r.get("style_id"), + r.get("style_match_method"), ) for r in records ] sql = """ INSERT INTO daily_style_metrics - (metric_date, style_name, platform, sales, refund_orders, visitors, cart_users, erp_style_codes, source) + (metric_date, style_name, platform, sales, refund_orders, visitors, cart_users, + erp_style_codes, source, style_id, style_match_method) VALUES %s ON CONFLICT (metric_date, style_name, platform) DO UPDATE SET sales = EXCLUDED.sales, @@ -394,6 +516,9 @@ def upsert_daily_metrics(conn, records: Sequence[dict]) -> int: cart_users = EXCLUDED.cart_users, erp_style_codes = EXCLUDED.erp_style_codes, source = EXCLUDED.source, + style_id = COALESCE(EXCLUDED.style_id, daily_style_metrics.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, daily_style_metrics.style_match_method), updated_at = NOW() """ with conn.cursor() as cur: @@ -414,6 +539,7 @@ def upsert_erp_all_shop_style_daily_metrics(conn, records: Sequence[dict]) -> in dedup[key] = dict(record) if not dedup: return 0 + records = attach_style_ids(conn, list(dedup.values())) rows = [ ( @@ -431,13 +557,16 @@ def upsert_erp_all_shop_style_daily_metrics(conn, records: Sequence[dict]) -> in int(record["selected_shop_count"]), int(record.get("matched_product_rows", 0) or 0), str(record["source"]), + record.get("style_id"), + record.get("style_match_method"), ) - for record in dedup.values() + for record in records ] sql = """ INSERT INTO erp_all_shop_style_daily_metrics (metric_date, style_name, sales, return_qty, erp_style_codes, - selected_shop_count, matched_product_rows, source) + selected_shop_count, matched_product_rows, source, + style_id, style_match_method) VALUES %s ON CONFLICT (metric_date, style_name) DO UPDATE SET sales = EXCLUDED.sales, @@ -446,6 +575,9 @@ def upsert_erp_all_shop_style_daily_metrics(conn, records: Sequence[dict]) -> in selected_shop_count = EXCLUDED.selected_shop_count, matched_product_rows = EXCLUDED.matched_product_rows, source = EXCLUDED.source, + style_id = COALESCE(EXCLUDED.style_id, erp_all_shop_style_daily_metrics.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, erp_all_shop_style_daily_metrics.style_match_method), updated_at = NOW() """ with conn.cursor() as cur: @@ -491,8 +623,11 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int: "present_days": 0, "erp_style_codes": [], "erp_source": r.get("erp_source"), + "style_id": r.get("style_id"), } cur = dedup[key] + if cur.get("style_id") is None and r.get("style_id") is not None: + cur["style_id"] = r["style_id"] # 数字字段累加 for fld in ("sales", "pop_sales", "jd_self_sales", "refund_orders", "visitors", "cart_users", "present_days"): @@ -505,7 +640,7 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int: # 保留最后一行的 erp_source 标记 if r.get("erp_source"): cur["erp_source"] = r["erp_source"] - records = list(dedup.values()) + records = attach_style_ids(conn, list(dedup.values())) # ── dedup 完毕 ────────────────────────────────────────────────────── rows = [ @@ -527,6 +662,8 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int: int(r.get("present_days", 0) or 0), r.get("erp_style_codes") or [], r.get("erp_source"), + r.get("style_id"), + r.get("style_match_method"), ) for r in records ] @@ -534,7 +671,8 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int: INSERT INTO weekly_style_metrics (week_end, week_start, time_period, style_name, platform, sales, pop_sales, jd_self_sales, refund_orders, visitors, cart_users, conversion_rate, - cart_rate, refund_rate, present_days, erp_style_codes, erp_source) + cart_rate, refund_rate, present_days, erp_style_codes, erp_source, + style_id, style_match_method) VALUES %s ON CONFLICT (week_end, style_name, platform) DO UPDATE SET week_start = EXCLUDED.week_start, @@ -551,6 +689,9 @@ def upsert_weekly_metrics(conn, records: Sequence[dict]) -> int: present_days = EXCLUDED.present_days, erp_style_codes = EXCLUDED.erp_style_codes, erp_source = EXCLUDED.erp_source, + style_id = COALESCE(EXCLUDED.style_id, weekly_style_metrics.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, weekly_style_metrics.style_match_method), updated_at = NOW() """ with conn.cursor() as cur: @@ -583,6 +724,7 @@ def upsert_jd_self_sales(conn, rows: Sequence[dict]) -> int: """rows: [{'week_end', 'week_start', 'style_name', 'sku', 'outbound_qty', 'found', 'collected_at'}, ...]""" if not rows: return 0 + rows = attach_style_ids(conn, rows) payload = [ ( r["week_end"], @@ -592,17 +734,23 @@ def upsert_jd_self_sales(conn, rows: Sequence[dict]) -> int: int(r.get("outbound_qty", 0) or 0), bool(r.get("found", False)), r.get("collected_at"), + r.get("style_id"), + r.get("style_match_method"), ) for r in rows ] sql = """ INSERT INTO jd_self_weekly_sales - (week_end, week_start, style_name, sku, outbound_qty, found, collected_at) + (week_end, week_start, style_name, sku, outbound_qty, found, collected_at, + style_id, style_match_method) VALUES %s ON CONFLICT (week_end, style_name, sku) DO UPDATE SET outbound_qty = EXCLUDED.outbound_qty, found = EXCLUDED.found, collected_at = EXCLUDED.collected_at, + style_id = COALESCE(EXCLUDED.style_id, jd_self_weekly_sales.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, jd_self_weekly_sales.style_match_method), updated_at = NOW() """ with conn.cursor() as cur: @@ -696,11 +844,13 @@ def fetch_daily_for_range(conn, start_date: str, end_date: str) -> dict[tuple[st # ── decline_events ────────────────────────────────────────────────────────── def upsert_decline_event(conn, event: dict, hermes_response: Any = None) -> None: + event = attach_style_ids(conn, [event])[0] sql = """ INSERT INTO decline_events (end_date, window_days, style_name, platform, window_start, seg1_sales, seg2_sales, seg3_sales, drop_pct_1to2, drop_pct_2to3, - drop_pct_1to3, erp_style_codes, hermes_notified, hermes_response) + drop_pct_1to3, erp_style_codes, hermes_notified, hermes_response, + style_id, style_match_method) VALUES %s ON CONFLICT (end_date, style_name, platform) DO UPDATE SET window_days = EXCLUDED.window_days, @@ -712,6 +862,9 @@ def upsert_decline_event(conn, event: dict, hermes_response: Any = None) -> None drop_pct_2to3 = EXCLUDED.drop_pct_2to3, drop_pct_1to3 = EXCLUDED.drop_pct_1to3, erp_style_codes = EXCLUDED.erp_style_codes, + style_id = COALESCE(EXCLUDED.style_id, decline_events.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, decline_events.style_match_method), hermes_notified = decline_events.hermes_notified OR EXCLUDED.hermes_notified, hermes_response = COALESCE( @@ -735,6 +888,8 @@ def upsert_decline_event(conn, event: dict, hermes_response: Any = None) -> None event.get("erp_style_codes") or [], bool(hermes_response is not None and not (isinstance(hermes_response, dict) and "error" in hermes_response)), json.dumps(hermes_response, ensure_ascii=False) if hermes_response is not None else None, + event.get("style_id"), + event.get("style_match_method"), ) with conn.cursor() as cur: execute_values(cur, sql, [payload]) @@ -863,11 +1018,13 @@ def insert_fact_alert_history(conn, record: dict) -> int: sent_at, recipient, send_channel, hermes_message_id, hermes_response, decline_payload} 唯一键: (style_name, trigger_date, recipient) — 同款同日每位收件人只发一次 """ + record = attach_style_ids(conn, [record])[0] sql = """ INSERT INTO fact_alert_history (decline_event_key, style_name, trigger_date, consecutive_days, sales_drop_pct, decline_window_start, decline_window_end, baseline_window_start, baseline_window_end, - sent_at, recipient, send_channel, hermes_message_id, hermes_response, decline_payload) + sent_at, recipient, send_channel, hermes_message_id, hermes_response, decline_payload, + style_id, style_match_method) VALUES %s ON CONFLICT (style_name, trigger_date, recipient) DO UPDATE SET decline_event_key = EXCLUDED.decline_event_key, @@ -881,7 +1038,10 @@ def insert_fact_alert_history(conn, record: dict) -> int: send_channel = EXCLUDED.send_channel, hermes_message_id = EXCLUDED.hermes_message_id, hermes_response = EXCLUDED.hermes_response, - decline_payload = EXCLUDED.decline_payload + decline_payload = EXCLUDED.decline_payload, + style_id = COALESCE(EXCLUDED.style_id, fact_alert_history.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, fact_alert_history.style_match_method) RETURNING id """ payload = ( @@ -900,6 +1060,8 @@ def insert_fact_alert_history(conn, record: dict) -> int: record.get("hermes_message_id"), json.dumps(record["hermes_response"], ensure_ascii=False) if record.get("hermes_response") is not None else None, json.dumps(record["decline_payload"], ensure_ascii=False, default=str) if record.get("decline_payload") is not None else None, + record.get("style_id"), + record.get("style_match_method"), ) with conn.cursor() as cur: cur.execute(sql, [payload]) @@ -924,8 +1086,31 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int: if not records: return 0 + # 款式归因: 本表只有平台商品 ID, 没有款名列, 按 spus 数组反查 dim_style.id。 + # 未建档 (product_id 没写进 dim_style) 的行 style_id 留 NULL 并标 unmatched, + # 规模用 v_style_unmatched_product 监控。 + pending: dict[str, list[str]] = {} + for r in records: + if r.get("style_id") is None: + pending.setdefault(str(r.get("platform") or ""), []).append(r.get("product_id")) + resolved: dict[tuple[str, str], int] = {} + for platform, product_ids in pending.items(): + for product_id, style_id in resolve_style_ids_by_product( + conn, platform, product_ids + ).items(): + resolved[(platform, product_id)] = style_id + rows = [] for r in records: + platform = str(r.get("platform") or "") + product_id = str(r.get("product_id") or "") + style_id = r.get("style_id") + match_method = r.get("style_match_method") + if style_id is None: + style_id = resolved.get((platform, product_id)) + match_method = match_method or ( + STYLE_MATCH_BY_PRODUCT if style_id is not None else STYLE_MATCH_UNMATCHED + ) rows.append(( r["platform"], r["stat_date"], @@ -950,6 +1135,8 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int: r.get("detail_page_bounce_rate"), json.dumps(r.get("raw_data") or {}, ensure_ascii=False, default=str), r.get("source_file"), + style_id, + match_method or STYLE_MATCH_UNMATCHED, )) sql = """ @@ -960,7 +1147,7 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int: paid_amount, paid_qty, paid_buyers, refund_amount, refund_orders, conversion_rate, avg_order_value, uv_value, detail_page_bounce_rate, - raw_data, source_file) + raw_data, source_file, style_id, style_match_method) VALUES %s ON CONFLICT (platform, stat_date, product_id) DO UPDATE SET shop_id = EXCLUDED.shop_id, @@ -985,6 +1172,14 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int: raw_data = COALESCE(product_daily_metrics.raw_data, '{}'::jsonb) || EXCLUDED.raw_data, source_file = EXCLUDED.source_file, + -- 款式归因只在新值命中时覆盖，避免 spus 临时缺维护时把历史归属抹掉 + style_id = COALESCE(EXCLUDED.style_id, product_daily_metrics.style_id), + style_match_method = CASE + WHEN EXCLUDED.style_id IS NOT NULL + THEN EXCLUDED.style_match_method + ELSE COALESCE(product_daily_metrics.style_match_method, + EXCLUDED.style_match_method) + END, imported_at = NOW() """ with conn.cursor() as cur: @@ -992,122 +1187,154 @@ def upsert_product_daily_metrics(conn, records: Sequence[dict]) -> int: return len(rows) -def upsert_tmall_product_ad_metrics(conn, records: Sequence[dict]) -> int: - """兼容旧调用：写入历史独立广告表。 +def _tm_ad_number(value: Any) -> Decimal | None: + """把报表里的金额/比率单元格转成 Decimal；缺失值返回 None。""" + if value is None: + return None + text = str(value).strip().replace(",", "") + if not text: + return None + try: + return Decimal(text) + except InvalidOperation: + return None - 新的万相台工作流使用 :func:`upsert_tmall_product_ad_raw_metrics`，因为 - data-hub 产品生命进程读取的是 ``product_daily_metrics.raw_data``。 + +def _tm_ad_metrics_payload(record: Mapping[str, Any]) -> dict[str, Any]: + """取报表指标字典，兼容调用方传 ``ad_metrics`` 或扁平标量键两种写法。""" + + metrics = dict(record.get("ad_metrics") or {}) + if metrics: + return metrics + for key in ( + "spend", + "attributed_deal_amount", + "direct_deal_amount", + "indirect_deal_amount", + "reported_roas", + "calculated_roas", + ): + if record.get(key) is not None: + metrics[key] = record[key] + return metrics + + +def upsert_tmall_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int: + """把万相台商品广告指标幂等写入 ``tm_product_ad_daily_metrics``。 + + 天猫广告数据不再写入 ``product_daily_metrics.raw_data.tm_ad_metrics``：广告 + 与经营日报分表后，经营数据重新同步不会覆盖投放数据。花费/直接/间接/归因成交 + 额和投产比走标量列（下游可直接 SUM），报表其余指标（展现、点击、点击率、 + 转化率、预售成交等）原样存在 ``metrics`` JSONB。款式按 ``dim_style.tm_spus`` + 的商品 ID 精确匹配，未命中再按商品名称兜底，结果写进 ``style_name`` / + ``style_match_method``。 """ + if not records: return 0 - rows = [ - ( - record["stat_date"], - record["product_id"], - record.get("product_name"), - record.get("spend") or 0, - record.get("direct_deal_amount") or 0, - record.get("indirect_deal_amount") or 0, - record.get("attributed_deal_amount") or 0, - record.get("reported_roas"), - record.get("calculated_roas"), - int(record.get("duplicate_row_count") or 1), - record.get("source_file"), + # 同一 (stat_date, product_id) 在一次批量里出现两次会让 ON CONFLICT 报 + # "cannot affect row a second time"，先按输入顺序去重（后者覆盖前者）。 + dedup: dict[tuple[Any, str], dict] = {} + for record in records: + dedup[(record["stat_date"], str(record["product_id"]))] = dict(record) + + style_by_product_id, style_names = _load_platform_style_catalog(conn, "tm_spus") + # 先解析款式名与匹配方式，再统一补 style_id（款式主键统一 2026-11-01）。 + for record in dedup.values(): + style_name, match_method = _resolve_style_match( + record, style_by_product_id, style_names + ) + record["style_name"] = style_name + record["style_match_method"] = match_method + enriched = iter(attach_style_ids(conn, list(dedup.values()))) + + rows = [] + for (stat_date, product_id), record in dedup.items(): + style_id_record = next(enriched) + metrics = _tm_ad_metrics_payload(record) + metrics.pop("style_name", None) + metrics.pop("style_match_method", None) + style_name = record["style_name"] + match_method = record["style_match_method"] + spend = _tm_ad_number(metrics.get("spend")) or Decimal("0") + attributed = _tm_ad_number(metrics.get("attributed_deal_amount")) or Decimal("0") + calculated_roas = ( + (attributed / spend).quantize(Decimal("0.000001")) if spend else None + ) + rows.append( + ( + stat_date, + product_id, + record.get("product_name"), + style_name, + match_method, + spend, + _tm_ad_number(metrics.get("direct_deal_amount")) or Decimal("0"), + _tm_ad_number(metrics.get("indirect_deal_amount")) or Decimal("0"), + attributed, + _tm_ad_number(metrics.get("roi")) + or _tm_ad_number(record.get("reported_roas")), + calculated_roas, + json.dumps(metrics, ensure_ascii=False, default=str), + int(record.get("duplicate_row_count") or 1), + record.get("source_file"), + record.get("source_row_number"), + style_id_record.get("style_id"), + ) ) - for record in records - ] sql = """ INSERT INTO tm_product_ad_daily_metrics - (stat_date, product_id, product_name, spend, - direct_deal_amount, indirect_deal_amount, attributed_deal_amount, - reported_roas, calculated_roas, duplicate_row_count, source_file) + (stat_date, product_id, product_name, style_name, style_match_method, + spend, direct_deal_amount, indirect_deal_amount, attributed_deal_amount, + reported_roas, calculated_roas, metrics, duplicate_row_count, + source_file, source_row_number, style_id) VALUES %s ON CONFLICT (stat_date, product_id) DO UPDATE SET - product_name = EXCLUDED.product_name, + product_name = COALESCE( + NULLIF(BTRIM(EXCLUDED.product_name), ''), + tm_product_ad_daily_metrics.product_name + ), + -- 款式只有本次能匹配到才覆盖，避免后续导入用 NULL 清掉已维护的映射。 + style_name = COALESCE( + EXCLUDED.style_name, tm_product_ad_daily_metrics.style_name + ), + style_match_method = CASE + WHEN EXCLUDED.style_name IS NULL + THEN tm_product_ad_daily_metrics.style_match_method + ELSE EXCLUDED.style_match_method + END, + style_id = COALESCE(EXCLUDED.style_id, + tm_product_ad_daily_metrics.style_id), spend = EXCLUDED.spend, direct_deal_amount = EXCLUDED.direct_deal_amount, indirect_deal_amount = EXCLUDED.indirect_deal_amount, attributed_deal_amount = EXCLUDED.attributed_deal_amount, reported_roas = EXCLUDED.reported_roas, calculated_roas = EXCLUDED.calculated_roas, + metrics = EXCLUDED.metrics, duplicate_row_count = EXCLUDED.duplicate_row_count, source_file = EXCLUDED.source_file, - imported_at = NOW() + source_row_number = EXCLUDED.source_row_number, + imported_at = NOW() """ + template = "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s, %s)" with conn.cursor() as cur: - execute_values(cur, sql, rows) + execute_values(cur, sql, rows, template=template) return len(rows) def upsert_tmall_product_ad_raw_metrics(conn, records: Sequence[dict]) -> int: - """只更新 ``product_daily_metrics.raw_data.tm_ad_metrics``。 + """兼容历史调用名；实际写入 ``tm_product_ad_daily_metrics``。""" - ``product_daily_metrics`` 可能已经由三平台经营日报导入过；这里不能用 - ``upsert_product_daily_metrics``，否则会把 visitors/paid_amount 等经营 - 字段清成 NULL。JSONB 节点级更新同时满足重复导入幂等和字段保留；广告 - 节点内的款式按 ``dim_style.tm_spus`` 精确匹配，未命中再按商品名称兜底。 - """ + return upsert_tmall_product_ad_daily_metrics(conn, records) - if not records: - return 0 - style_by_product_id, style_names = _load_platform_style_catalog(conn, "tm_spus") - rows = [] - for record in records: - metrics = dict(record.get("ad_metrics") or {}) - if not metrics: - # Keep callers written against the first draft of this importer - # compatible while making the canonical storage shape explicit. - for key in ( - "spend", - "attributed_deal_amount", - "direct_deal_amount", - "indirect_deal_amount", - "reported_roas", - "calculated_roas", - ): - if record.get(key) is not None: - metrics[key] = record[key] - style_name, match_method = _resolve_style_match( - record, style_by_product_id, style_names - ) - metrics["style_name"] = style_name - metrics["style_match_method"] = match_method - rows.append( - ( - record["stat_date"], - str(record["product_id"]), - record.get("product_name"), - json.dumps({"tm_ad_metrics": metrics}, ensure_ascii=False, default=str), - record.get("source_file"), - ) - ) +def upsert_tmall_product_ad_metrics(conn, records: Sequence[dict]) -> int: + """兼容历史调用名；实际写入 ``tm_product_ad_daily_metrics``。""" - sql = """ - INSERT INTO product_daily_metrics - (platform, stat_date, product_id, product_name, raw_data, source_file, imported_at) - VALUES %s - ON CONFLICT (platform, stat_date, product_id) DO UPDATE SET - product_name = CASE - WHEN COALESCE(BTRIM(product_daily_metrics.product_name), '') = '' - THEN EXCLUDED.product_name - ELSE product_daily_metrics.product_name - END, - raw_data = jsonb_set( - COALESCE(product_daily_metrics.raw_data, '{}'::jsonb), - '{tm_ad_metrics}', - EXCLUDED.raw_data -> 'tm_ad_metrics', - true - ), - source_file = EXCLUDED.source_file, - imported_at = NOW() - """ - template = "('tm', %s, %s, %s, %s::jsonb, %s, NOW())" - with conn.cursor() as cur: - execute_values(cur, sql, rows, template=template) - return len(rows) + return upsert_tmall_product_ad_daily_metrics(conn, records) def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int: @@ -1129,6 +1356,7 @@ def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int: ) dedup[key] = dict(record) + records = attach_style_ids(conn, list(dedup.values())) rows = [ ( record["stat_date"], @@ -1147,14 +1375,17 @@ def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int: json.dumps(record.get("metrics") or {}, ensure_ascii=False, default=str), record.get("source_file"), record.get("source_row_number"), + record.get("style_id"), + record.get("style_match_method"), ) - for record in dedup.values() + for record in records ] sql = """ INSERT INTO jd_product_ad_daily_metrics (stat_date, advertiser_pin, shop_account, report_type, row_key, spu_id, spu_name, sku_id, sku_name, style_name, match_status, - spend, reported_roi, metrics, source_file, source_row_number) + spend, reported_roi, metrics, source_file, source_row_number, + style_id, style_match_method) VALUES %s ON CONFLICT (stat_date, advertiser_pin, report_type, row_key) DO UPDATE SET shop_account = EXCLUDED.shop_account, @@ -1169,6 +1400,9 @@ def upsert_jd_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int: metrics = EXCLUDED.metrics, source_file = EXCLUDED.source_file, source_row_number = EXCLUDED.source_row_number, + style_id = COALESCE(EXCLUDED.style_id, jd_product_ad_daily_metrics.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, jd_product_ad_daily_metrics.style_match_method), imported_at = NOW() """ with conn.cursor() as cur: @@ -1289,6 +1523,7 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int ) record["style_name"] = style_name record["style_match_method"] = match_method + records = attach_style_ids(conn, list(dedup.values())) fields = ( "stat_date", "advertiser_id", @@ -1313,8 +1548,9 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int "refund_rate_1h", "source_file", "source_row_number", + "style_id", ) - rows = [tuple(record.get(field) for field in fields) for record in dedup.values()] + rows = [tuple(record.get(field) for field in fields) for record in records] sql = """ INSERT INTO dy_product_ad_daily_metrics (stat_date, advertiser_id, product_id, product_name, @@ -1322,7 +1558,8 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int overall_conversion_rate, spend, gross_deal_amount, gross_payment_roi, gross_order_cost, user_actual_paid_amount, platform_subsidy_amount, net_roi, net_deal_amount, net_order_cost, - net_settlement_rate, refund_rate_1h, source_file, source_row_number) + net_settlement_rate, refund_rate_1h, source_file, source_row_number, + style_id) VALUES %s ON CONFLICT (stat_date, advertiser_id, product_id) DO UPDATE SET product_name = EXCLUDED.product_name, @@ -1345,6 +1582,7 @@ def upsert_douyin_product_ad_daily_metrics(conn, records: Sequence[dict]) -> int refund_rate_1h = EXCLUDED.refund_rate_1h, source_file = EXCLUDED.source_file, source_row_number = EXCLUDED.source_row_number, + style_id = COALESCE(EXCLUDED.style_id, dy_product_ad_daily_metrics.style_id), imported_at = NOW() """ with conn.cursor() as cur: @@ -1364,11 +1602,12 @@ def upsert_persona_screenshot(conn, record: dict) -> int: } 主键 = (collect_date, style_name, platform)。同款同采集日期幂等覆盖。 """ + record = attach_style_ids(conn, [record])[0] sql = """ INSERT INTO persona_screenshots (collect_date, style_name, platform, item_id, screenshot_path, image_data, file_size, bitable_base, bitable_table, bitable_record_id, bitable_file_token, - uploaded_to_bitable) + uploaded_to_bitable, style_id, style_match_method) VALUES %s ON CONFLICT (collect_date, style_name, platform) DO UPDATE SET item_id = EXCLUDED.item_id, @@ -1380,6 +1619,9 @@ def upsert_persona_screenshot(conn, record: dict) -> int: bitable_record_id = EXCLUDED.bitable_record_id, bitable_file_token = EXCLUDED.bitable_file_token, uploaded_to_bitable = EXCLUDED.uploaded_to_bitable, + style_id = COALESCE(EXCLUDED.style_id, persona_screenshots.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, persona_screenshots.style_match_method), updated_at = NOW() """ payload = ( @@ -1395,6 +1637,8 @@ def upsert_persona_screenshot(conn, record: dict) -> int: record.get("bitable_record_id"), record.get("bitable_file_token"), bool(record.get("uploaded_to_bitable", False)), + record.get("style_id"), + record.get("style_match_method"), ) with conn.cursor() as cur: execute_values(cur, sql, [payload]) @@ -1417,13 +1661,14 @@ def upsert_main_image_creatives(conn, records: Sequence[dict]) -> int: """ if not records: return 0 + records = attach_style_ids(conn, records) sql = """ INSERT INTO main_image_creatives (collect_date, style_name, platform, image_key, creative_ids, creative_image_url, impressions, clicks, cost, click_rate, row_count, local_image_path, image_downloaded, image_fingerprint, bitable_base, bitable_table, bitable_record_id, bitable_file_token, - uploaded_to_bitable) + uploaded_to_bitable, style_id, style_match_method) VALUES %s ON CONFLICT (collect_date, style_name, platform, image_key) DO UPDATE SET creative_ids = EXCLUDED.creative_ids, @@ -1441,6 +1686,9 @@ def upsert_main_image_creatives(conn, records: Sequence[dict]) -> int: bitable_record_id = EXCLUDED.bitable_record_id, bitable_file_token = EXCLUDED.bitable_file_token, uploaded_to_bitable = EXCLUDED.uploaded_to_bitable, + style_id = COALESCE(EXCLUDED.style_id, main_image_creatives.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, main_image_creatives.style_match_method), updated_at = NOW() """ rows = [ @@ -1464,6 +1712,8 @@ def upsert_main_image_creatives(conn, records: Sequence[dict]) -> int: r.get("bitable_record_id"), r.get("bitable_file_token"), bool(r.get("uploaded_to_bitable", False)), + r.get("style_id"), + r.get("style_match_method"), ) for r in records ] @@ -1575,10 +1825,12 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int: """ if not records: return 0 + records = attach_style_ids(conn, records) sql = """ INSERT INTO daily_persona_metrics (collect_date, style_name, platform, item_id, - payload, raw_payload, source_file, collected_at) + payload, raw_payload, source_file, collected_at, + style_id, style_match_method) VALUES %s ON CONFLICT (collect_date, style_name, platform) DO UPDATE SET item_id = EXCLUDED.item_id, @@ -1586,6 +1838,9 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int: raw_payload = EXCLUDED.raw_payload, source_file = EXCLUDED.source_file, collected_at = EXCLUDED.collected_at, + style_id = COALESCE(EXCLUDED.style_id, daily_persona_metrics.style_id), + style_match_method = COALESCE( + EXCLUDED.style_match_method, daily_persona_metrics.style_match_method), updated_at = NOW() """ rows = [ @@ -1598,6 +1853,8 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int: json.dumps(r.get("raw_payload"), ensure_ascii=False) if r.get("raw_payload") is not None else None, r.get("source_file"), r.get("collected_at"), + r.get("style_id"), + r.get("style_match_method"), ) for r in records ] @@ -1606,72 +1863,6 @@ def upsert_persona_metrics(conn, records: Sequence[dict]) -> int: return len(rows) -# ── product_reviews ─────────────────────────────────────────────────────────── - -def upsert_product_reviews(conn, records: Sequence[dict]) -> int: - """records: [{ - 'platform' ('jd'/'dy'), 'collect_date', 'comment_date', - 'style_name' (可空, DY 通过 product_id 反查 dim_style.dy_spus 填充), - 'rating' ('好评'/'中评'/'差评'), 'score', 'content', 'tags' (list), - 'user_name', 'order_id', 'product_id', 'product_name', - 'sku_id', 'sku_name', 'sku_size', - 'raw_data' (dict), 'source_file' - }, ...] - UNIQUE (platform, order_id) -> 同订单重抓覆盖。 - """ - if not records: - return 0 - rows = [ - ( - r["platform"], - r["collect_date"], - r.get("comment_date"), - r.get("style_name"), - r["rating"], - r.get("score"), - r.get("content"), - r.get("tags") or None, - r.get("user_name"), - r["order_id"], - r.get("product_id"), - r.get("product_name"), - r.get("sku_id"), - r.get("sku_name"), - r.get("sku_size"), - json.dumps(r.get("raw_data") or {}, ensure_ascii=False, default=str), - r.get("source_file"), - ) - for r in records - ] - sql = """ - INSERT INTO product_reviews - (platform, collect_date, comment_date, style_name, rating, score, - content, tags, user_name, order_id, product_id, product_name, - sku_id, sku_name, sku_size, raw_data, source_file) - VALUES %s - ON CONFLICT (platform, order_id) DO UPDATE SET - collect_date = EXCLUDED.collect_date, - comment_date = EXCLUDED.comment_date, - style_name = EXCLUDED.style_name, - rating = EXCLUDED.rating, - score = EXCLUDED.score, - content = EXCLUDED.content, - tags = EXCLUDED.tags, - user_name = EXCLUDED.user_name, - product_id = EXCLUDED.product_id, - product_name = EXCLUDED.product_name, - sku_id = EXCLUDED.sku_id, - sku_name = EXCLUDED.sku_name, - sku_size = EXCLUDED.sku_size, - raw_data = EXCLUDED.raw_data, - source_file = EXCLUDED.source_file, - imported_at = NOW() - """ - with conn.cursor() as cur: - execute_values(cur, sql, rows) - return len(rows) - - # ── fact_style_analysis_reports ────────────────────────────────────────────── def upsert_style_analysis_report(conn, record: dict) -> int: @@ -1687,12 +1878,13 @@ def upsert_style_analysis_report(conn, record: dict) -> int: 业务 UNIQUE = (style_name, analysis_date, window_days). 同款同日同窗口重跑覆盖. 返回写入行的 id. """ + record = attach_style_ids(conn, [record])[0] sql = """ INSERT INTO fact_style_analysis_reports (style_name, analysis_date, window_days, window_start, window_end, platforms_included, feishu_doc_id, feishu_doc_url, report_title, report_markdown, hermes_model, prompt_tokens, completion_tokens, - source_payload) + source_payload, style_id, style_match_method) VALUES %s ON CONFLICT (style_name, analysis_date, window_days) DO UPDATE SET window_start = EXCLUDED.window_start, @@ -1706,6 +1898,10 @@ def upsert_style_analysis_report(conn, record: dict) -> int: prompt_tokens = EXCLUDED.prompt_tokens, completion_tokens = EXCLUDED.completion_tokens, source_payload = EXCLUDED.source_payload, + style_id = COALESCE(EXCLUDED.style_id, + fact_style_analysis_reports.style_id), + style_match_method = COALESCE(EXCLUDED.style_match_method, + fact_style_analysis_reports.style_match_method), updated_at = NOW() RETURNING id """ @@ -1728,6 +1924,8 @@ def upsert_style_analysis_report(conn, record: dict) -> int: record.get("completion_tokens"), json.dumps(record["source_payload"], ensure_ascii=False, default=str) if record.get("source_payload") is not None else None, + record.get("style_id"), + record.get("style_match_method"), ) with conn.cursor() as cur: execute_values(cur, sql, [payload]) diff --git a/src/gyxx_flow/modules/product_commerce/db/schema.sql b/src/gyxx_flow/modules/product_commerce/db/schema.sql index 3a14362..0de77bb 100644 --- a/src/gyxx_flow/modules/product_commerce/db/schema.sql +++ b/src/gyxx_flow/modules/product_commerce/db/schema.sql @@ -490,7 +490,8 @@ COMMENT ON COLUMN product_daily_metrics.detail_page_bounce_rate IS '详情页跳 -- ============================================================================ -- tm_product_ad_daily_metrics: 天猫商品投放日报 --- 数据源: 天猫投放平台导出的商品报表 CSV。 +-- 数据源: 万相台商品报表 CSV (import_tmall_product_ads.py 解析后直接写本表)。 +-- 每日每商品一行；花费/成交/投产比走标量列供下游直接聚合，报表其余指标留在 metrics。 -- 与 product_daily_metrics 分表，避免经营数据重新同步时覆盖投放数据。 -- ============================================================================ CREATE TABLE IF NOT EXISTS tm_product_ad_daily_metrics ( @@ -498,27 +499,47 @@ CREATE TABLE IF NOT EXISTS tm_product_ad_daily_metrics ( stat_date DATE NOT NULL, product_id TEXT NOT NULL, product_name TEXT, + style_name TEXT, + style_match_method TEXT NOT NULL DEFAULT 'unmatched', spend NUMERIC(18, 2) NOT NULL DEFAULT 0, direct_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0, indirect_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0, attributed_deal_amount NUMERIC(18, 2) NOT NULL DEFAULT 0, reported_roas NUMERIC(18, 6), calculated_roas NUMERIC(18, 6), + metrics JSONB NOT NULL DEFAULT '{}'::jsonb, duplicate_row_count INTEGER NOT NULL DEFAULT 1, source_file TEXT, + source_row_number INTEGER, imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), UNIQUE (stat_date, product_id) ); +-- 兼容已创建的窄表：补齐款式匹配结果与完整指标快照列。 +ALTER TABLE tm_product_ad_daily_metrics + ADD COLUMN IF NOT EXISTS style_name TEXT; +ALTER TABLE tm_product_ad_daily_metrics + ADD COLUMN IF NOT EXISTS style_match_method TEXT + NOT NULL DEFAULT 'unmatched'; +ALTER TABLE tm_product_ad_daily_metrics + ADD COLUMN IF NOT EXISTS metrics JSONB NOT NULL DEFAULT '{}'::jsonb; +ALTER TABLE tm_product_ad_daily_metrics + ADD COLUMN IF NOT EXISTS source_row_number INTEGER; CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_date_idx ON tm_product_ad_daily_metrics (stat_date DESC); CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_product_date_idx ON tm_product_ad_daily_metrics (product_id, stat_date DESC); +CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_style_date_idx + ON tm_product_ad_daily_metrics (style_name, stat_date DESC); COMMENT ON TABLE tm_product_ad_daily_metrics IS '天猫商品级投放日报，每日每商品一行；广告归因成交额仅用于计算投产比。'; +COMMENT ON COLUMN tm_product_ad_daily_metrics.product_id IS '万相台主体ID（商品），按文本保存'; +COMMENT ON COLUMN tm_product_ad_daily_metrics.style_name IS '款式名；先按商品 ID 反查 dim_style.tm_spus，未命中再按商品名称精确包含匹配，无法确认时为 NULL'; +COMMENT ON COLUMN tm_product_ad_daily_metrics.style_match_method IS '款式匹配方式：product_id=商品 ID，product_name=商品名称，ambiguous=名称歧义，unmatched=未匹配'; COMMENT ON COLUMN tm_product_ad_daily_metrics.spend IS '天猫广告花费（页面电商支付金额）'; COMMENT ON COLUMN tm_product_ad_daily_metrics.attributed_deal_amount IS '广告归因总成交金额，仅用于计算投产比'; COMMENT ON COLUMN tm_product_ad_daily_metrics.reported_roas IS '源报表投入产出比'; COMMENT ON COLUMN tm_product_ad_daily_metrics.calculated_roas IS '归因成交金额 / 花费'; +COMMENT ON COLUMN tm_product_ad_daily_metrics.metrics IS '万相台报表指标原样快照 (展现/点击/点击率/转化率/预售成交等)，键名与源报表中文列一一对应'; -- ============================================================================ -- jd_product_ad_daily_metrics: 京东营销概况广告日报 @@ -615,7 +636,7 @@ COMMENT ON COLUMN dy_product_ad_daily_metrics.overall_ctr IS '报表整体点击 COMMENT ON COLUMN dy_product_ad_daily_metrics.overall_conversion_rate IS '报表整体转化率，单位为百分点'; COMMENT ON COLUMN dy_product_ad_daily_metrics.net_settlement_rate IS '净成交金额结算率，单位为百分点'; COMMENT ON COLUMN dy_product_ad_daily_metrics.refund_rate_1h IS '1 小时内退款率，单位为百分点'; -COMMENT ON COLUMN product_daily_metrics.raw_data IS '平台特有原始列 (下单金额/件数/买家数、聚划算、竞争力评分、月累计、搜索引导...) 原样保留；天猫 tm_ad_metrics 同时含 style_name/style_match_method'; +COMMENT ON COLUMN product_daily_metrics.raw_data IS '平台特有原始列 (下单金额/件数/买家数、聚划算、竞争力评分、月累计、搜索引导...) 原样保留；天猫广告指标已迁至 tm_product_ad_daily_metrics，本列不再写 tm_ad_metrics'; COMMENT ON COLUMN product_daily_metrics.source_file IS '原始文件名 (溯源用)'; COMMENT ON COLUMN product_daily_metrics.imported_at IS '入库时间'; @@ -810,69 +831,19 @@ COMMENT ON COLUMN main_image_creatives.collected_at IS '采集时间 (脚本写 COMMENT ON COLUMN main_image_creatives.updated_at IS '最近更新时间'; -- ============================================================================ --- product_reviews: 三平台商品评价 (JD + DY, TM 暂未接入) --- 数据源: --- JD: jd_product_data_collector.py --reviews-only -> data/jd单品数据/评价汇总_.json --- DY: dy_product_scraping.py (主流程已挂载) -> data/dy单品数据/评论汇总_.json --- 设计: 三平台字段归一到独立列, 平台特有字段进 raw_data JSONB。 --- 写入策略: ON CONFLICT (platform, order_id) DO UPDATE 全量覆盖 (order_id 是评价业务键)。 +-- product_reviews: 已下线 (2026-11-01) +-- +-- 原设计是 JD + DY 商品评价合一表, 由 import_product_reviews.py 从平台 JSON 入库。 +-- 下线原因: +-- 1. JD 侧 style_name 全靠 sku_id 反查, 而 dim_jd_sku_spu 映射表不存在, +-- 214 行 style_name 全为 NULL, 两个消费方 (analyze_style / 营销日报) 都是 +-- WHERE style_name = ?, 等于入库了但没人能读。 +-- 2. 差评全量已由 shop_intelligence 的三张差评表 (tmall/jd/dy_negative_reviews) +-- 独立采集, 覆盖面更全 (天猫 97 条, 而本表天猫为 0)。 +-- +-- 现消费方统一读视图 v_negative_reviews (见文件末尾「款式主键统一」段)。 +-- 存量表已归档为 product_reviews_archived_20261101, 不再建表、不再写入。 -- ============================================================================ -CREATE TABLE IF NOT EXISTS product_reviews ( - id BIGSERIAL PRIMARY KEY, - platform TEXT NOT NULL CHECK (platform IN ('jd','dy')), - collect_date DATE NOT NULL, -- 采集日期 (脚本运行日) - comment_date DATE, -- 评价时间 (JD 取评价时间日期部分, DY 取 comment_time 日期部分) - style_name TEXT, -- 款式名 (DY: 通过 product_id 反查 dim_style.dy_spus 填充; 未匹配时为 NULL) - rating TEXT NOT NULL, -- '好评' / '中评' / '差评' - score SMALLINT, -- JD 评分 (1-5 星), DY 留空 - content TEXT, - tags TEXT[], -- JD 评价标签 (做工精细/容量大 等), DY 留空 - user_name TEXT, - order_id TEXT NOT NULL, -- 订单 ID (业务键, 用于去重) - product_id TEXT, -- 平台商品 ID (DY product_id / JD sku_name 对应的 SPU) - product_name TEXT, - sku_id TEXT, -- JD SKU 编码, DY 留空 - sku_name TEXT, -- JD SKU 名称, DY 留空 - sku_size TEXT, -- JD 规格, DY 留空 - raw_data JSONB NOT NULL DEFAULT '{}'::jsonb, - source_file TEXT, -- 原始 JSON 文件路径 - imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - UNIQUE (platform, order_id) -); -CREATE INDEX IF NOT EXISTS product_reviews_platform_date_idx - ON product_reviews (platform, comment_date DESC); -CREATE INDEX IF NOT EXISTS product_reviews_rating_idx - ON product_reviews (platform, rating, comment_date DESC); -CREATE INDEX IF NOT EXISTS product_reviews_product_idx - ON product_reviews (platform, product_id, comment_date DESC); --- product_reviews_style_idx 在文件末尾 ALTER TABLE 加 style_name 列之后建 (老表兼容) - -COMMENT ON TABLE product_reviews IS '商品评价 - JD + DY 合一表。来源: jd_product_data_collector.py --reviews-only / dy_product_scraping.py。UNIQUE (platform, order_id) -> 同订单评价重抓覆盖。'; -COMMENT ON COLUMN product_reviews.platform IS '平台代码 (''jd'' / ''dy'')'; -COMMENT ON COLUMN product_reviews.collect_date IS '采集日期 (脚本运行日, 不等于评价发生日)'; -COMMENT ON COLUMN product_reviews.comment_date IS '评价时间 (从评价发生时间戳抽日期部分)'; --- style_name 列的 COMMENT 在文件末尾 ALTER TABLE 加列之后 (老表兼容) -COMMENT ON COLUMN product_reviews.rating IS '评价等级: 好评 / 中评 / 差评'; -COMMENT ON COLUMN product_reviews.score IS 'JD 评分 (1-5 星, aria-valuenow), DY 留空'; -COMMENT ON COLUMN product_reviews.content IS '评价正文'; -COMMENT ON COLUMN product_reviews.tags IS 'JD 评价标签数组 (如 [做工精细, 容量大]), DY 留空'; -COMMENT ON COLUMN product_reviews.user_name IS '用户昵称'; -COMMENT ON COLUMN product_reviews.order_id IS '订单 ID (业务键, 同订单只保留一条, 重抓覆盖)'; -COMMENT ON COLUMN product_reviews.product_id IS '平台商品 ID (DY product_id / JD SPU)'; -COMMENT ON COLUMN product_reviews.product_name IS '商品名称'; -COMMENT ON COLUMN product_reviews.sku_id IS 'JD SKU 编码, DY 留空'; -COMMENT ON COLUMN product_reviews.sku_name IS 'JD SKU 名称, DY 留空'; -COMMENT ON COLUMN product_reviews.sku_size IS 'JD 规格文本, DY 留空'; -COMMENT ON COLUMN product_reviews.raw_data IS '平台特有原始字段 (JD 完整评价对象 / DY comment_rows 行)'; -COMMENT ON COLUMN product_reviews.source_file IS '源 JSON 文件相对路径 (溯源用)'; -COMMENT ON COLUMN product_reviews.imported_at IS '入库时间'; - --- 兼容已有库: product_reviews 加 style_name 列 (CREATE TABLE IF NOT EXISTS 不会改已有表) -ALTER TABLE product_reviews ADD COLUMN IF NOT EXISTS style_name TEXT; -CREATE INDEX IF NOT EXISTS product_reviews_style_idx - ON product_reviews (style_name, comment_date DESC) - WHERE style_name IS NOT NULL; -COMMENT ON COLUMN product_reviews.style_name IS '款式名 (DY: 通过 product_id 反查 dim_style.dy_spus 填充; 未匹配时为 NULL, 表示 dim_style 缺该商品 ID 维护)'; -- ============================================================================ -- fact_style_analysis_reports: 单款式多维度分析报告 (直连 MiniMax 产出) @@ -1364,3 +1335,276 @@ CREATE TABLE IF NOT EXISTS erp_sales_theme_daily_metrics ( ); CREATE INDEX IF NOT EXISTS erp_sales_theme_daily_metrics_date_idx ON erp_sales_theme_daily_metrics (stat_date DESC); + +-- ============================================================================ +-- 款式主键统一 (2026-11-01) +-- +-- 目标: 采集端所有带款式的表统一用 style_id (= dim_style.id) 关联, style_name +-- 只作为显示冗余。data-hub 侧 22 张表本来就用 style_id, 两端从此同一套主键, +-- 不再出现「同一库两种引用方式」。已实测 erp_product.style_id 135/135 命中。 +-- +-- 约定: +-- * style_id 可空的表 = 允许「未建档」的行 (平台商品 ID 未写进 dim_style 的 +-- spus 数组), 这类行必须同时写 style_match_method='unmatched', 便于统计 +-- 漏维护的规模。 +-- * style_id 非空的表 = 款式清单来自 dim_style loader, 理论上必然命中; +-- 但**不加 NOT NULL**, 由 backfill_style_id.py 与写入端负责填充, +-- 避免历史脏款名在 ALTER 阶段中断建表。 +-- * 主键一律不动 (daily_style_metrics 仍是 metric_date+style_name+platform), +-- data-hub 有 6 处按名字 join, 改 PK 收益小、回归面大。 +-- ============================================================================ + +ALTER TABLE daily_style_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE weekly_style_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE erp_all_shop_style_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE daily_persona_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE main_image_creatives ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE fact_style_analysis_reports ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE decline_events ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE fact_alert_history ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE jd_self_weekly_sales ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE persona_screenshots ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE product_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE jd_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE dy_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT; +ALTER TABLE tm_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_id BIGINT; + +-- 匹配方式留痕: dim_style / product_id / product_name / anchor / unmatched +ALTER TABLE daily_style_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE weekly_style_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE erp_all_shop_style_daily_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE daily_persona_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE main_image_creatives ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE fact_style_analysis_reports ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE decline_events ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE fact_alert_history ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE jd_self_weekly_sales ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE persona_screenshots ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE product_daily_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT; +ALTER TABLE jd_product_ad_daily_metrics ADD COLUMN IF NOT EXISTS style_match_method TEXT; + +COMMENT ON COLUMN daily_style_metrics.style_id IS 'dim_style.id; 采集端统一款式主键, style_name 仅作显示冗余'; +COMMENT ON COLUMN product_daily_metrics.style_id IS 'dim_style.id; 按 platform+product_id 反查 spus 得到, 未建档为 NULL 且 style_match_method=unmatched'; + +-- 查询索引: 按款式跨日期取数 (产品生命页、风格分析、预警) +CREATE INDEX IF NOT EXISTS daily_style_metrics_style_id_idx + ON daily_style_metrics (style_id, metric_date DESC); +CREATE INDEX IF NOT EXISTS weekly_style_metrics_style_id_idx + ON weekly_style_metrics (style_id, week_end DESC); +CREATE INDEX IF NOT EXISTS erp_all_shop_style_daily_metrics_style_id_idx + ON erp_all_shop_style_daily_metrics (style_id, metric_date DESC); +CREATE INDEX IF NOT EXISTS daily_persona_metrics_style_id_idx + ON daily_persona_metrics (style_id, collect_date DESC); +CREATE INDEX IF NOT EXISTS main_image_creatives_style_id_idx + ON main_image_creatives (style_id, collect_date DESC); +CREATE INDEX IF NOT EXISTS fact_style_analysis_reports_style_id_idx + ON fact_style_analysis_reports (style_id, analysis_date DESC); +CREATE INDEX IF NOT EXISTS decline_events_style_id_idx + ON decline_events (style_id, end_date DESC); +CREATE INDEX IF NOT EXISTS fact_alert_history_style_id_idx + ON fact_alert_history (style_id, trigger_date DESC); +CREATE INDEX IF NOT EXISTS jd_self_weekly_sales_style_id_idx + ON jd_self_weekly_sales (style_id, week_end DESC); +CREATE INDEX IF NOT EXISTS persona_screenshots_style_id_idx + ON persona_screenshots (style_id, collect_date DESC); +CREATE INDEX IF NOT EXISTS product_daily_metrics_style_id_idx + ON product_daily_metrics (style_id, stat_date DESC); +CREATE INDEX IF NOT EXISTS jd_product_ad_daily_metrics_style_id_idx + ON jd_product_ad_daily_metrics (style_id, stat_date DESC); +CREATE INDEX IF NOT EXISTS dy_product_ad_daily_metrics_style_id_idx + ON dy_product_ad_daily_metrics (style_id, stat_date DESC); +CREATE INDEX IF NOT EXISTS tm_product_ad_daily_metrics_style_id_idx + ON tm_product_ad_daily_metrics (style_id, stat_date DESC); + +-- ── P0 观测视图 ───────────────────────────────────────────────────────────── + +-- 未建档的平台商品 ID: 推动在 dim_style 里补 spus, 是 style_id 覆盖率的天花板。 +CREATE OR REPLACE VIEW v_style_unmatched_product AS +SELECT m.platform, + m.product_id, + MAX(m.product_name) AS product_name, + MIN(m.stat_date) AS first_date, + MAX(m.stat_date) AS last_date, + COUNT(*) AS rows +FROM product_daily_metrics m +WHERE NOT EXISTS ( + SELECT 1 FROM dim_style d + WHERE (m.platform = 'tm' AND d.tm_spus @> to_jsonb(m.product_id)) + OR (m.platform = 'jd' AND d.jd_spus @> to_jsonb(m.product_id)) + OR (m.platform = 'dy' AND d.dy_spus @> to_jsonb(m.product_id))) +GROUP BY m.platform, m.product_id; + +COMMENT ON VIEW v_style_unmatched_product IS + '未写进 dim_style.spus 的平台商品 ID 清单; 这些行的 product_daily_metrics.style_id 必然为 NULL。'; + +-- 同上, 但把三张广告表也算进来。广告投的商品不一定有经营日报 (纯投流款), 只看 +-- v_style_unmatched_product 会漏掉这部分待建档 ID。列结构保持一致, 便于并排检索。 +CREATE OR REPLACE VIEW v_style_unmatched_source AS +WITH all_rows AS ( + SELECT platform, product_id, product_name, stat_date FROM product_daily_metrics + UNION ALL + SELECT 'tm', product_id, product_name, stat_date FROM tm_product_ad_daily_metrics + UNION ALL + SELECT 'jd', spu_id, spu_name, stat_date FROM jd_product_ad_daily_metrics + UNION ALL + SELECT 'dy', product_id, product_name, stat_date FROM dy_product_ad_daily_metrics +) +SELECT s.platform, + s.product_id, + MAX(s.product_name) AS product_name, + MIN(s.stat_date) AS first_date, + MAX(s.stat_date) AS last_date, + COUNT(*) AS rows +FROM all_rows s +WHERE s.product_id IS NOT NULL + AND BTRIM(s.product_id) <> '' + AND NOT EXISTS ( + SELECT 1 FROM dim_style d + WHERE (s.platform = 'tm' AND d.tm_spus @> to_jsonb(s.product_id)) + OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id)) + OR (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id))) +GROUP BY s.platform, s.product_id; + +COMMENT ON VIEW v_style_unmatched_source IS + '全部采集表 (经营日报 + 三张广告表) 里未写进 dim_style.spus 的商品 ID; 补建档的完整待办清单。'; + +-- 各表 style_id 回填进度 (验收用)。 +CREATE OR REPLACE VIEW v_style_id_coverage AS +SELECT 'daily_style_metrics' AS tbl, COUNT(*) AS rows, COUNT(style_id) AS with_style_id FROM daily_style_metrics +UNION ALL SELECT 'weekly_style_metrics', COUNT(*), COUNT(style_id) FROM weekly_style_metrics +UNION ALL SELECT 'erp_all_shop_style_daily_metrics', COUNT(*), COUNT(style_id) FROM erp_all_shop_style_daily_metrics +UNION ALL SELECT 'daily_persona_metrics', COUNT(*), COUNT(style_id) FROM daily_persona_metrics +UNION ALL SELECT 'main_image_creatives', COUNT(*), COUNT(style_id) FROM main_image_creatives +UNION ALL SELECT 'fact_style_analysis_reports', COUNT(*), COUNT(style_id) FROM fact_style_analysis_reports +UNION ALL SELECT 'decline_events', COUNT(*), COUNT(style_id) FROM decline_events +UNION ALL SELECT 'fact_alert_history', COUNT(*), COUNT(style_id) FROM fact_alert_history +UNION ALL SELECT 'jd_self_weekly_sales', COUNT(*), COUNT(style_id) FROM jd_self_weekly_sales +UNION ALL SELECT 'persona_screenshots', COUNT(*), COUNT(style_id) FROM persona_screenshots +UNION ALL SELECT 'product_daily_metrics', COUNT(*), COUNT(style_id) FROM product_daily_metrics +UNION ALL SELECT 'jd_product_ad_daily_metrics', COUNT(*), COUNT(style_id) FROM jd_product_ad_daily_metrics +UNION ALL SELECT 'dy_product_ad_daily_metrics', COUNT(*), COUNT(style_id) FROM dy_product_ad_daily_metrics +UNION ALL SELECT 'tm_product_ad_daily_metrics', COUNT(*), COUNT(style_id) FROM tm_product_ad_daily_metrics; + +COMMENT ON VIEW v_style_id_coverage IS + '各采集表 style_id 非空比例; 除 product_daily_metrics / 广告表外应为 100%。'; + +-- ── P1 统一差评视图 ──────────────────────────────────────────────────────── + +-- product_reviews 下线后, 风格分析与营销日报统一读这张视图。 +-- 与旧表的口径差异: 只有差评一档 (旧表含好/中/差), 但覆盖三个平台。 +-- 款式归因优先级: 平台商品 ID 精确命中 > 商品名称包含款名 (取最长款名, 更具体)。 +-- 差评三表由 shop_intelligence 模块建表, 这里用守卫避免全新库初始化时视图先于源表。 +DO $$ +BEGIN + IF to_regclass('public.tmall_negative_reviews') IS NOT NULL + AND to_regclass('public.jd_negative_reviews') IS NOT NULL + AND to_regclass('public.dy_negative_reviews') IS NOT NULL THEN + EXECUTE $view$ +CREATE OR REPLACE VIEW v_negative_reviews AS +WITH src AS ( + SELECT 'tm'::text AS platform, + n.business_date AS comment_date, + COALESCE(n.review_level, '负面评价') AS rating, + n.content, + NULL::text AS order_id, + NULL::text AS product_id, + n.product_name, + n.review_id, + n.review_at, + n.raw_review + FROM tmall_negative_reviews n + UNION ALL + SELECT 'jd', n.business_date, COALESCE(n.review_level, '负面评价'), + n.content, n.order_id, NULL::text, n.product_name, + n.review_id, n.review_at, n.raw_review + FROM jd_negative_reviews n + UNION ALL + SELECT 'dy', n.business_date, COALESCE(n.rating_level, '负面评价'), + n.content, n.order_id, n.product_id, n.product_name, + n.review_id, n.review_at, n.raw_review + FROM dy_negative_reviews n +) +SELECT s.platform, s.comment_date, s.rating, s.content, + s.order_id, s.product_id, s.product_name, + s.review_id, s.review_at, s.raw_review, + m.style_id, m.style_name, m.style_match_method +FROM src s +LEFT JOIN LATERAL ( + SELECT d.id AS style_id, + d.style_name, + CASE + WHEN (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id)) + OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id)) + THEN 'product_id' + ELSE 'product_name' + END AS style_match_method + FROM dim_style d + WHERE COALESCE(d.style_name, '') <> '' + AND ( + (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id)) + OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id)) + OR (s.product_name IS NOT NULL AND s.product_name LIKE '%' || d.style_name || '%') + ) + ORDER BY ( + CASE + WHEN (s.platform = 'dy' AND d.dy_spus @> to_jsonb(s.product_id)) + OR (s.platform = 'jd' AND d.jd_spus @> to_jsonb(s.product_id)) + THEN 0 ELSE 1 + END + ), + COALESCE(d.is_series_anchor, FALSE) DESC, + LENGTH(d.style_name) DESC, + d.id + LIMIT 1 +) m ON TRUE +$view$; + EXECUTE $c$COMMENT ON VIEW v_negative_reviews IS + '三平台差评归一视图 (tm/jd/dy_negative_reviews); 取代已下线的 product_reviews。style_id 为 dim_style.id, 未匹配为 NULL。'$c$; + ELSE + RAISE NOTICE 'skip v_negative_reviews: 差评表尚未创建 (先运行 shop_intelligence init_schema)'; + END IF; +END +$$; + +-- ── style_id 一律走「逻辑外键」, 不建物理外键 ─────────────────────────────── +-- +-- 2026-11-01 定案: style_id 与 dim_style.id 的关联由应用层保证, 库里**不加** +-- FOREIGN KEY 约束。原因: +-- * 物理外键会在写入端传错/传旧 id 时直接报错中断整批采集, 而采集是「宁可 +-- 留 NULL 待回填, 也不要整日落空」的语义, 硬约束与这个语义冲突; +-- * 款式被清理/重建 (拆系列、合并同款) 时要先删约束才能动数据, 运维成本高; +-- * data-hub 侧 22 张表同样只用 style_id 做关联, 不带物理外键, 两边保持一致。 +-- +-- 完整性靠三件事保证, 不需要数据库兜底: +-- 1. 写入端: loader 下发的 style_id 来自 dim_style 本身; +-- 按商品 ID / 款名兜底的路径由 attach_style_ids 统一查询后填充; +-- 2. 校验: backfill_style_id.py --dry-run 与 v_style_id_coverage 报覆盖率; +-- 3. 观火: v_style_unmatched_product / v_style_unmatched_source 列未建档清单。 +-- +-- 下面这段只做**历史清理**: 早期 P4 曾给 10 张表建过 <表名>_style_id_fk, +-- 这里幂等删除, 保证老库重跑 schema 后回到「只有索引、没有物理外键」的状态。 +DO $$ +DECLARE + target_table TEXT; +BEGIN + FOREACH target_table IN ARRAY ARRAY[ + 'daily_style_metrics', + 'weekly_style_metrics', + 'erp_all_shop_style_daily_metrics', + 'daily_persona_metrics', + 'main_image_creatives', + 'fact_style_analysis_reports', + 'decline_events', + 'fact_alert_history', + 'jd_self_weekly_sales', + 'persona_screenshots' + ] LOOP + IF to_regclass('public.' || target_table) IS NOT NULL THEN + EXECUTE format( + 'ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I', + target_table, target_table || '_style_id_fk' + ); + END IF; + END LOOP; +END +$$; diff --git a/src/gyxx_flow/modules/product_commerce/dy_product_scraping.py b/src/gyxx_flow/modules/product_commerce/dy_product_scraping.py index 4b9f943..89d7419 100644 --- a/src/gyxx_flow/modules/product_commerce/dy_product_scraping.py +++ b/src/gyxx_flow/modules/product_commerce/dy_product_scraping.py @@ -3258,15 +3258,13 @@ def _page_automation(page) -> None: for product in summary.get("products", []) ] - # 评论采集挂在默认流程末尾, 让 daily --target-date 也能采到评价. - # _collect_comment_summary 内部会 _open_comment_detail_page 跳转到评论页, - # 采完好/中/差后覆盖 评论汇总_.json. 幂等: 同 order_id 重采覆盖. - try: - COMMENTS_SUMMARY = _collect_comment_summary(page) - _save_comment_summary(COMMENTS_SUMMARY) - except Exception as exc: - print(f"[WARN] 评论采集失败 (不影响单品数据已落盘): {exc}") - COMMENTS_SUMMARY = {} + # 评论采集已于 2026-11-01 从默认流程摘除： + # 它只产出本地 评论汇总_.json，而唯一消费方 product_reviews 已下线 + # （差评改由 shop.dy_negative_reviews.daily 独立采集，消费方读 v_negative_reviews）。 + # 继续跑等于每天多一次浏览器跳转 + 一份无人读取的文件。 + # 需要历史评价样本时用 --reviews-only 显式触发，不要恢复默认流程。 + print("[SKIP] 评论采集已在默认流程下线（如需补采请用 --reviews-only）") + COMMENTS_SUMMARY = {} _results_saved = True diff --git a/src/gyxx_flow/modules/product_commerce/erp_metric_overrides.py b/src/gyxx_flow/modules/product_commerce/erp_metric_overrides.py index c64da92..5bc1b76 100644 --- a/src/gyxx_flow/modules/product_commerce/erp_metric_overrides.py +++ b/src/gyxx_flow/modules/product_commerce/erp_metric_overrides.py @@ -75,9 +75,20 @@ def load_erp_metrics(platform, style_name, target_date=None): failed = [c for c in code_results if not c.get("ok") or c.get("error")] if code_results and failed: # 任一编码失败都会造成 ERP 合计不完整，整款回退到平台原值。 + # 但 TM/DY 没有平台销量字段，整款回退会被兜底成 0，把整款销量抹掉 + # （如 10244 allsku iframe 偶发 timeout → 宙斯天猫销量变 0）。 + # 所以同时给出"已成功编码合计"，供无平台原值时使用。 return { "sales": None, "refund_orders": None, + "partial_sales": ( + sum(int(parse_number(c.get("yesterday_sales"))) for c in ok_results) + if ok_results else None + ), + "partial_refunds": ( + sum(int(parse_number(c.get("yesterday_returns"))) for c in ok_results) + if ok_results else None + ), "source": str(json_path), "erp_style_codes": payload.get("erp_style_codes", []), "all_failed": True, @@ -114,9 +125,17 @@ def apply_erp_override(platform, style_name, visitors, cart_users, target_date=N c = int(parse_number(cart_users)) if not erp or erp.get("all_failed"): - # ERP 拿不到数据 → 用平台原值（如果有），否则 0 - sales = int(parse_number(platform_sales)) if platform_sales is not None else 0 - refunds = int(parse_number(platform_refunds)) if platform_refunds is not None else 0 + if platform_sales is not None: + # 平台自己有销量字段（JD/DY）→ 用平台原值，最接近真相 + sales = int(parse_number(platform_sales)) + refunds = int(parse_number(platform_refunds)) if platform_refunds is not None else 0 + elif erp and erp.get("partial_sales") is not None: + # 平台没有销量字段（TM）→ 用已成功编码的合计，避免整款归零 + sales = int(erp["partial_sales"]) + refunds = int(erp.get("partial_refunds") or 0) + else: + sales = 0 + refunds = 0 return { "销量": sales, "访客数": v, @@ -127,6 +146,7 @@ def apply_erp_override(platform, style_name, visitors, cart_users, target_date=N "退款率": format_percent(refunds, sales), "erp_override": erp, "erp_unavailable": True, + "erp_partial": bool(erp and erp.get("partial_sales") is not None and platform_sales is None), "failed_codes": (erp or {}).get("failed_codes", []), } diff --git a/src/gyxx_flow/modules/product_commerce/import_product_reviews.py b/src/gyxx_flow/modules/product_commerce/import_product_reviews.py deleted file mode 100644 index 5d930df..0000000 --- a/src/gyxx_flow/modules/product_commerce/import_product_reviews.py +++ /dev/null @@ -1,331 +0,0 @@ -""" -扫描 data/{jd,dy}单品数据/评价*.json (或 评论汇总_*.json), -归一化后写入 product_reviews 表 (ON CONFLICT 全量覆盖)。 - -DY 评价入库时会反查 dim_style.dy_spus 填 style_name; -JD 评价缺 SPU->款式映射, style_name 留 NULL (待 dim_jd_sku_spu 映射表补齐后再做)。 - -用法: - .venv\\Scripts\\python.exe import_product_reviews.py # 全量 (默认只 dy) - .venv/Scripts/python.exe import_product_reviews.py --date 2026-07-13 # 单日 - .venv/Scripts/python.exe import_product_reviews.py --from 2026-06-01 --to 2026-07-13 - .venv/Scripts/python.exe import_product_reviews.py --platforms dy - .venv/Scripts/python.exe import_product_reviews.py --dry-run # 只解析不入库 - -源 JSON 文件: - JD: data/jd单品数据/评价汇总_.json - 结构: {platform, shop, collect_date, comment_date_range, captured_at, - total_count, ratings: {好评/中评/差评: {collected_count, comments:[{ - user_name, score, tags, content, time, order_id, - sku_name, sku_id, sku_size, rating}]}}} - DY: data/dy单品数据/评论汇总_.json - 结构: {shop, period, captured_at, url, total_collected_count, - ratings: {好评/中评/差评: {expected_count, collected_count, comments:[{ - rating, order_id, comment_time, comment_text, product_name, - product_id, spec_name, spec_id, raw_text}]}}} -""" -from __future__ import annotations - -import argparse -import json -import os -import re -import sys -from datetime import date, datetime -from pathlib import Path - -from runtime_paths import RAW_DATA_ROOT, artifact_relative_path -from typing import Any, Iterable - -PROJECT_ROOT = Path(__file__).resolve().parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from db import get_conn, upsert_product_reviews # noqa: E402 - - -DATA_ROOT = RAW_DATA_ROOT - -PLATFORM_DIRS = { - "jd": "jd单品数据", - "dy": "dy单品数据", -} - -# 文件名模式: JD 用 评价汇总_2026-07-14.json, DY 用 评论汇总_20260604.json -FILE_PATTERNS = { - "jd": re.compile(r"^评价汇总_(\d{4}-\d{2}-\d{2})\.json$"), - "dy": re.compile(r"^评论汇总_(\d{8})\.json$"), -} - -VALID_PLATFORMS = tuple(PLATFORM_DIRS.keys()) - -RATING_VALUES = ("好评", "中评", "差评") - - -def _parse_jd_date(s: str) -> date: - return datetime.strptime(s, "%Y-%m-%d").date() - - -def _parse_dy_date(s: str) -> date: - return datetime.strptime(s, "%Y%m%d").date() - - -def _parse_comment_date(s: str) -> date | None: - """从评价时间字符串抽日期部分。 - - JD time 格式: '2026-07-13 12:34:56' 或 '2026-07-13' - DY comment_time 格式: '2026/06/05 16:39:18' 或 '2026/06/05' - """ - if not s: - return None - s = s.strip() - for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d %H:%M:%S", "%Y/%m/%d"): - try: - return datetime.strptime(s, fmt).date() - except ValueError: - continue - m = re.match(r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})", s) - if m: - try: - return date(int(m.group(1)), int(m.group(2)), int(m.group(3))) - except ValueError: - return None - return None - - -def _to_smallint(v: Any) -> int | None: - if v is None or v == "": - return None - try: - return int(float(str(v))) - except (ValueError, TypeError): - return None - - -def _normalize_jd(payload: dict, source_file: str, collect_date: date, - style_map: dict[str, str] | None = None) -> list[dict]: - """JD JSON -> product_reviews 记录列表 - - JD 评价表只有 sku_id (无 SPU), 暂不反查 style_name, 留 NULL。 - """ - records = [] - for rating in RATING_VALUES: - bucket = (payload.get("ratings") or {}).get(rating) or {} - for c in bucket.get("comments") or []: - comment_date = _parse_comment_date(c.get("time", "")) - records.append({ - "platform": "jd", - "collect_date": collect_date, - "comment_date": comment_date, - "style_name": None, # JD sku_id -> SPU 映射暂缺, 待 dim_jd_sku_spu 补齐 - "rating": rating, - "score": _to_smallint(c.get("score")), - "content": (c.get("content") or "").strip() or None, - "tags": [t for t in (c.get("tags") or []) if t] or None, - "user_name": c.get("user_name") or None, - "order_id": str(c.get("order_id") or "").strip(), - "product_id": None, # JD 评价表只有 sku_id, SPU 留空 - "product_name": c.get("sku_name") or None, - "sku_id": c.get("sku_id") or None, - "sku_name": c.get("sku_name") or None, - "sku_size": c.get("sku_size") or None, - "raw_data": c, - "source_file": source_file, - }) - return records - - -def _normalize_dy(payload: dict, source_file: str, collect_date: date, - style_map: dict[str, str] | None = None) -> list[dict]: - """DY JSON -> product_reviews 记录列表 - - style_map: {product_id: style_name}, 从 dim_style.dy_spus 构建。 - 匹配不上时 style_name = None (表示 dim_style 缺该商品 ID 维护)。 - """ - style_map = style_map or {} - records = [] - for rating in RATING_VALUES: - bucket = (payload.get("ratings") or {}).get(rating) or {} - for c in bucket.get("comments") or []: - comment_date = _parse_comment_date(c.get("comment_time", "")) - product_id = str(c.get("product_id") or "").strip() - records.append({ - "platform": "dy", - "collect_date": collect_date, - "comment_date": comment_date, - "style_name": style_map.get(product_id), - "rating": rating, - "score": None, # DY 无星级 - "content": (c.get("comment_text") or "").strip() or None, - "tags": None, - "user_name": None, # DY 评论无独立用户名字段 - "order_id": str(c.get("order_id") or "").strip(), - "product_id": product_id or None, - "product_name": c.get("product_name") or None, - "sku_id": c.get("spec_id") or None, - "sku_name": c.get("spec_name") or None, - "sku_size": None, - "raw_data": c, - "source_file": source_file, - }) - return records - - -NORMALIZERS = { - "jd": _normalize_jd, - "dy": _normalize_dy, -} - - -def _load_dy_style_map() -> dict[str, str]: - """从 dim_style.dy_spus 构建 {product_id: style_name} 映射。""" - mapping: dict[str, str] = {} - with get_conn() as conn: - with conn.cursor() as cur: - cur.execute("SELECT style_name, dy_spus FROM dim_style WHERE dy_spus IS NOT NULL") - for style_name, spus in cur.fetchall(): - if not spus: - continue - for pid in spus: - mapping[str(pid)] = style_name - return mapping - - -def _discover_files(platforms: list[str], date_from: date | None, date_to: date | None) -> list[tuple[str, date, Path]]: - """返回 [(platform, collect_date, file_path), ...]""" - out = [] - for plat in platforms: - plat_dir = DATA_ROOT / PLATFORM_DIRS[plat] - if not plat_dir.exists(): - continue - pattern = FILE_PATTERNS[plat] - for path in sorted(plat_dir.iterdir()): - m = pattern.match(path.name) - if not m: - continue - try: - collect_date = (_parse_jd_date if plat == "jd" else _parse_dy_date)(m.group(1)) - except ValueError: - continue - if date_from and collect_date < date_from: - continue - if date_to and collect_date > date_to: - continue - out.append((plat, collect_date, path)) - return out - - -def _load_and_normalize(plat: str, collect_date: date, path: Path, - style_map: dict[str, str] | None = None) -> list[dict]: - with path.open(encoding="utf-8") as f: - payload = json.load(f) - rel = artifact_relative_path(path) - return NORMALIZERS[plat](payload, rel, collect_date, style_map=style_map) - - -def main() -> int: - parser = argparse.ArgumentParser(description="扫描 JD/DY 评价 JSON 入库 product_reviews") - parser.add_argument("--date", help="只导入指定采集日期 (YYYY-MM-DD); JD 用 - 分隔, DY 无分隔也可") - parser.add_argument("--from", dest="date_from", help="起始采集日期 (YYYY-MM-DD)") - parser.add_argument("--to", dest="date_to", help="结束采集日期 (YYYY-MM-DD)") - parser.add_argument("--platforms", default="dy", help="平台, 逗号分隔 (默认 dy; JD 待 SPU 映射补齐)") - parser.add_argument("--dry-run", action="store_true", help="只解析不入库") - args = parser.parse_args() - - platforms = [p.strip() for p in args.platforms.split(",") if p.strip()] - for p in platforms: - if p not in VALID_PLATFORMS: - print(f"[ERR] 未知平台: {p} (允许: {VALID_PLATFORMS})") - return 2 - - date_from = _parse_single(args.date_from) if args.date_from else None - date_to = _parse_single(args.date_to) if args.date_to else None - if args.date: - d = _parse_single(args.date) - date_from = d - date_to = d - - files = _discover_files(platforms, date_from, date_to) - if not files: - print(f"[WARN] 未找到匹配的 JSON 文件 (platforms={platforms}, from={date_from}, to={date_to})") - return 0 - - # 预加载 DY 款式映射 (dim_style.dy_spus -> {product_id: style_name}) - style_map: dict[str, str] = {} - if "dy" in platforms and not args.dry_run: - style_map = _load_dy_style_map() - print(f"[INFO] dim_style.dy_spus 映射: {len(style_map)} 个 product_id -> style_name") - elif "dy" in platforms and args.dry_run: - # dry-run 也加载, 便于预览匹配情况 - style_map = _load_dy_style_map() - print(f"[INFO] (dry-run) dim_style.dy_spus 映射: {len(style_map)} 个 product_id -> style_name") - - total = 0 - skipped = 0 - by_plat: dict[str, int] = {p: 0 for p in platforms} - by_rating: dict[str, int] = {"好评": 0, "中评": 0, "差评": 0} - matched_styles: set[str] = set() - - for plat, collect_date, path in files: - try: - records = _load_and_normalize(plat, collect_date, path, style_map=style_map) - except Exception as e: - print(f"[ERR] 解析 {path}: {e}") - continue - # 去重: 同 order_id 多条 (理论上 UNIQUE 会处理, 这里先在内存去重避免执行 batch fail) - seen = set() - deduped = [] - for r in records: - if not r["order_id"]: - skipped += 1 - continue - if r["order_id"] in seen: - skipped += 1 - continue - seen.add(r["order_id"]) - deduped.append(r) - by_rating[r["rating"]] = by_rating.get(r["rating"], 0) + 1 - if r.get("style_name"): - matched_styles.add(r["style_name"]) - - plat_style_matched = sum(1 for r in deduped if r.get("style_name")) - print(f"[{plat}] {collect_date} {path.name} -> {len(deduped)} 条 (款式匹配 {plat_style_matched}/{len(deduped)})") - - if args.dry_run: - if deduped: - sample = deduped[0].copy() - sample["raw_data"] = "(省略)" - print(f" (dry-run, 不入库) 样本: {sample}") - else: - with get_conn() as conn: - n = upsert_product_reviews(conn, deduped) - print(f" -> upsert {n} rows") - - total += len(deduped) - by_plat[plat] = by_plat.get(plat, 0) + len(deduped) - - print() - print("=" * 60) - print(f" 总计: {total} 条 (跳过 {skipped} 条无 order_id)") - for p, n in by_plat.items(): - print(f" {p}: {n} 条") - print(" 按评级:") - for r, n in by_rating.items(): - print(f" {r}: {n} 条") - if matched_styles: - print(f" 匹配到款式 ({len(matched_styles)} 款): {', '.join(sorted(matched_styles))}") - print("=" * 60) - return 0 - - -def _parse_single(s: str) -> date: - s = s.strip() - for fmt in ("%Y-%m-%d", "%Y%m%d"): - try: - return datetime.strptime(s, fmt).date() - except ValueError: - continue - raise SystemExit(f"无法解析日期: {s}") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/gyxx_flow/modules/product_commerce/import_tmall_product_ads.py b/src/gyxx_flow/modules/product_commerce/import_tmall_product_ads.py index e5baa7c..432751b 100644 --- a/src/gyxx_flow/modules/product_commerce/import_tmall_product_ads.py +++ b/src/gyxx_flow/modules/product_commerce/import_tmall_product_ads.py @@ -1,9 +1,11 @@ -"""将万相台商品广告 CSV 幂等写入产品生命进程读取的字段。 +"""将万相台商品广告 CSV 幂等写入天猫商品投放日报表。 -真实数据契约是 ``product_daily_metrics.raw_data.tm_ad_metrics``，不是一张 -独立的广告表。导入时只替换该 JSON 节点，保留同一商品日期上已经存在的 -经营日报字段；同时把款式匹配结果写入该节点的 ``style_name`` / -``style_match_method``，数据-hub 后端据此计算广告成交金额、投产比和电商费比。 +数据契约是独立表 ``tm_product_ad_daily_metrics``，每日每商品一行：花费、直接/ +间接/归因成交金额与投产比走标量列，报表其余指标原样存 ``metrics`` JSONB。 +款式按 ``dim_style.tm_spus`` 的商品 ID 精确匹配，未命中再按商品名称精确包含 +兜底，结果写进 ``style_name`` / ``style_match_method``，data-hub 后端据此计算 +广告成交金额、投产比和电商费比。广告与经营日报分表存储，经营数据重新同步不会 +覆盖投放数据。 默认只解析校验；显式传入 ``--execute`` 才写 PostgreSQL。 """ @@ -25,8 +27,9 @@ PROJECT_ROOT = Path(__file__).resolve().parent sys.path.insert(0, str(PROJECT_ROOT)) from db import ( # noqa: E402 + ensure_tmall_product_ad_daily_table, get_conn, - upsert_tmall_product_ad_raw_metrics, + upsert_tmall_product_ad_daily_metrics, ) AMOUNT_COLUMNS = { @@ -327,8 +330,9 @@ def main(argv: list[str] | None = None) -> int: return 0 with get_conn() as conn: - written = upsert_tmall_product_ad_raw_metrics(conn, records) - print(f"[OK] 已写入 product_daily_metrics.raw_data.tm_ad_metrics：{written} 行") + ensure_tmall_product_ad_daily_table(conn) + written = upsert_tmall_product_ad_daily_metrics(conn, records) + print(f"[OK] 已写入 tm_product_ad_daily_metrics：{written} 行") return 0 diff --git a/src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py b/src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py index e629210..c5d469b 100644 --- a/src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py +++ b/src/gyxx_flow/modules/product_commerce/jd_product_data_collector.py @@ -45,6 +45,7 @@ from collection_progress import SpuProgress from jd_data_collector import ( DEFAULT_CONFIG, + close_ai_helper_drawer_if_present as close_jd_ai_helper_if_present, step_login, step_enter_shangzhi, extract, @@ -932,19 +933,40 @@ _JMT_CUSTOM_REACT_CLICK = r"""(args) => { const [ty, tm, targetDay] = args; const pad = (n) => String(n).padStart(2, '0'); const ymRe = new RegExp('^' + ty + '\\u5e74' + pad(tm) + '\\u6708$'); + // 商智新版「自定义」是双月日历: 每个月的面板里除了本月日期, 还会渲染 + // 上下月的补位日期 (class 带 -diff-month)。同一个月面板里因此可能出现 + // 两个同号日期 (如 10 月面板里的 9/30 补位格与 10/30 真身), 之前按 + // 「第一个匹配」点击会落到补位格, 于是 echo 变成 2026-09-30。 + // 这里先定位目标月份自己的面板, 再排除补位格与禁用格。 let panel = null; for (const el of document.querySelectorAll('*')) { const t = (el.textContent || '').trim(); if (!ymRe.test(t)) continue; - panel = el.closest('[class*="calendar"]') || el.parentElement.parentElement || el.parentElement; - break; + let cur = el.parentElement; + let depth = 0; + while (cur && depth < 6) { + if (cur.querySelector && cur.querySelector('td')) break; + cur = cur.parentElement; + depth++; + } + if (cur && cur.querySelector('td')) { + panel = cur; + break; + } } if (!panel) return {status: 'no_panel'}; - for (const cell of panel.querySelectorAll('td, [class*="cell"]')) { - if ((cell.textContent || '').trim() !== targetDay) continue; - if (/disabled|off/i.test(String(cell.className))) continue; + const cells = Array.from(panel.querySelectorAll('td, [class*="cell"]')).filter((cell) => { + if ((cell.textContent || '').trim() !== targetDay) return false; + const cls = String(cell.className || ''); + if (/diff-month|disabled|-off\b|other-month|prev-month|next-month/i.test(cls)) return false; + const rect = cell.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + }); + if (!cells.length) return {status: 'no_cell'}; + for (const cell of cells) { const nodes = [cell, ...cell.querySelectorAll('*'), cell.parentElement]; for (const node of nodes) { + if (!node) continue; const propsKey = Object.keys(node).find((k) => k.startsWith('__reactProps$')); if (!propsKey) continue; const props = node[propsKey]; @@ -955,9 +977,8 @@ _JMT_CUSTOM_REACT_CLICK = r"""(args) => { return {status: 'ok'}; } } - return {status: 'no_handler'}; } - return {status: 'no_cell'}; + return {status: 'no_handler'}; }""" @@ -1006,13 +1027,33 @@ def _select_custom_date_new_version(page, target) -> bool: 点工具条「自定义」→ 在双月日历里对目标日连续触发两次 React onClick (起止同日 = 单日区间) → echo 显示目标日期即成功. """ + # 商智会随机弹出 AI 助手抽屉 / 引导层, 全屏 jd-modal-wrap 会拦截 + # 日期工具条的点击; 先尝试收起再点「自定义」。 + try: + page.keyboard.press("Escape") + page.wait_for_timeout(300) + except Exception: + pass + try: + close_jd_ai_helper_if_present(page) + except Exception: + pass try: tag = page.get_by_text("自定义", exact=True).first tag.wait_for(state="visible", timeout=8000) tag.click(timeout=8000) except Exception as exc: - print(f" [WARN] 新版「自定义」入口点击失败: {exc}") - return False + print(f" [WARN] 新版「自定义」入口点击失败({str(exc)[:120]}), 改用 DOM 事件点击") + clicked = page.evaluate( + r"""() => { + const el = document.querySelector('li[data-event-content="当前时间_自定义"]'); + if (!el) return false; + el.dispatchEvent(new MouseEvent('click', {bubbles: true, cancelable: true})); + return true; + }""" + ) + if not clicked: + return False page.wait_for_timeout(1200) for _ in range(2): diff --git a/src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py b/src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py index ed595ed..87962fd 100644 --- a/src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py +++ b/src/gyxx_flow/modules/product_commerce/orchestrate_daily_collection.py @@ -433,20 +433,9 @@ def run_collect(args, target_date, platforms, failed): failed=failed, historical=is_historical_target(target_date), ) - # 评论入库 (DY 默认分支已采评论 -> 评论汇总_.json; JD 单独 --reviews-only 采) - # 幂等: product_reviews UNIQUE (platform, order_id) ON CONFLICT DO UPDATE, 重跑覆盖不双写. - # critical=False: 评论入库失败不影响后续 analyze/export/insert. - if not args.skip_platforms and not args.skip_reviews_import: - review_plats = [p for p in platforms if p in ("jd", "dy")] - if review_plats: - run_step_with_retry( - "import_product_reviews", - [sys.executable, str(runtime_script("import_product_reviews.py")), - "--platforms", ",".join(review_plats)], - dry_run=args.dry_run, - critical=False, - failed=failed, - ) + # 评论入库已下线 (2026-11-01): 差评由 shop.jd/dy/tmall_negative_reviews.daily + # 三个独立工作流采集, 消费方统一读视图 v_negative_reviews。 + # product_reviews 停写并归档为 product_reviews_archived_20261101。 def collect_daily_skip_report(date_str: str) -> Path | None: @@ -675,8 +664,6 @@ def main() -> int: parser.add_argument("--platforms", default="jd,dy,tm", help="逗号分隔: jd,dy,tm") parser.add_argument("--serial-platforms", action="store_true", help="三平台串行跑") parser.add_argument("--skip-insert", action="store_true", help="只导出，不插入飞书") - parser.add_argument("--skip-reviews-import", action="store_true", - help="跳过评论入库 (默认 daily collect 后跑 import_product_reviews)") parser.add_argument("--dry-run", action="store_true") parser.add_argument( "--erp-mode", diff --git a/src/gyxx_flow/modules/product_commerce/orchestrate_review_collection.py b/src/gyxx_flow/modules/product_commerce/orchestrate_review_collection.py deleted file mode 100644 index 0f66fe2..0000000 --- a/src/gyxx_flow/modules/product_commerce/orchestrate_review_collection.py +++ /dev/null @@ -1,192 +0,0 @@ -"""评论采集编排器: 默认只跑 DY 采集 + 入库 PG。 - -JD 评价采集代码已就绪 (jd_product_data_collector.py --reviews-only), -但 JD 评价表只有 sku_id 缺 SPU, 暂时无法反查到款式, 所以编排器默认不跑 JD。 -等 dim_jd_sku_spu 映射表补齐后可加 --platforms jd,dy。 - -DY 主流程 (dy_product_scraping.py 默认分支) 已挂载 _collect_comment_summary, -daily --target-date 会顺带采评论; 本编排器用 --reviews-only 只跑评论 (跳过单品列表/ -下载/人群画像), 用于历史补采或重跑。 -入库调 import_product_reviews.py, ON CONFLICT (platform, order_id) 幂等覆盖。 - -用法: - .venv/Scripts/python.exe orchestrate_review_collection.py - .venv/Scripts/python.exe orchestrate_review_collection.py --target-date 2026-07-13 - .venv/Scripts/python.exe orchestrate_review_collection.py --platforms jd,dy # JD 待 SPU 映射补齐 - .venv/Scripts/python.exe orchestrate_review_collection.py --skip-collect # 只入库 - .venv/Scripts/python.exe orchestrate_review_collection.py --dry-run -""" -from __future__ import annotations - -import argparse -import os -import subprocess -import sys -from datetime import datetime, timedelta -from pathlib import Path - -from collect_retry_utils import FailedCollector -from runtime_paths import runtime_script - -from gyxx_flow.adapters import environment_for_child_script - - -PROJECT_ROOT = Path(__file__).resolve().parent -MAX_RUN_RETRIES = 2 -RUN_SLEEP_SECONDS = 5 - - -def default_target_date() -> str: - return (datetime.now().date() - timedelta(days=1)).strftime("%Y-%m-%d") - - -def validate_date(value: str) -> str: - try: - datetime.strptime(value, "%Y-%m-%d") - except ValueError as exc: - raise argparse.ArgumentTypeError("date must be YYYY-MM-DD") from exc - return value - - -def run_step(name, command, dry_run=False, critical=True): - """执行子命令。critical=True 失败抛 RuntimeError, False 仅 [WARN] 继续。""" - printable = " ".join(str(c) for c in command) - print(f"[RUN] {name}: {printable}") - if dry_run: - return {"name": name, "code": 0, "dry_run": True} - try: - proc = subprocess.run( - command, - cwd=PROJECT_ROOT, - text=True, - encoding="utf-8", - errors="replace", - env=environment_for_child_script(command[1], os.environ), - ) - except (FileNotFoundError, OSError) as exc: - msg = f"{name} {type(exc).__name__}: {exc}" - if critical: - raise RuntimeError(msg) from exc - print(f"[WARN] {msg}") - return {"name": name, "code": 126, "error": msg} - - if proc.returncode: - msg = f"{name} failed with exit code {proc.returncode}" - if critical: - raise RuntimeError(msg) - print(f"[WARN] {msg}") - return {"name": name, "code": proc.returncode, "error": msg} - print(f"[OK] {name}") - return {"name": name, "code": proc.returncode} - - -def run_step_with_retry(name, command, dry_run=False, critical=True, - failed: FailedCollector | None = None, - max_retries: int = MAX_RUN_RETRIES, - sleep_seconds: float = RUN_SLEEP_SECONDS) -> dict: - if dry_run: - return run_step(name, command, dry_run=True, critical=critical) - last_result: dict = {"name": name, "code": 0} - total = max_retries + 1 - for attempt in range(1, total + 1): - result = run_step(name, command, dry_run=False, critical=False) - last_result = result - if result.get("code") == 0: - if attempt > 1: - print(f"[OK] {name} 第 {attempt}/{total} 次重试成功", flush=True) - return result - err = result.get("error", "unknown") - if attempt >= total: - if failed is not None: - failed.add_page(name, f"{err} (attempts={attempt})", attempts=attempt) - if critical: - raise RuntimeError(f"{name} failed after {attempt} attempts: {err}") - print(f"[WARN] {name} 失败 (已重试 {max_retries} 次, {err}), 继续", flush=True) - return result - print(f"[RETRY] {name} 第 {attempt}/{total} 次失败, {sleep_seconds}s 后重试...", flush=True) - import time - time.sleep(sleep_seconds) - return last_result - - -def build_collect_command(plat: str, target_date: str) -> list[str]: - if plat == "jd": - # JD --reviews-only: 只跑评价采集 (好/中/差全采), 跳过商智单品明细 - return [sys.executable, str(runtime_script("jd_product_data_collector.py")), - "--reviews-only", "--target-date", target_date] - if plat == "dy": - # DY --reviews-only: 登录后直接跳评论页, 跳过单品列表/下载/人群画像 - return [sys.executable, str(runtime_script("dy_product_scraping.py")), - "--reviews-only", "--target-date", target_date] - raise ValueError(f"unknown platform: {plat}") - - -def main() -> int: - parser = argparse.ArgumentParser(description="评论采集编排器 (JD + DY)") - parser.add_argument("--target-date", type=validate_date, default=None, - help="采集指定日期 YYYY-MM-DD (默认昨天)") - parser.add_argument("--platforms", default="dy", - help="平台, 逗号分隔 (默认 dy; JD 待 SPU 映射补齐, 可手动 jd,dy)") - parser.add_argument("--skip-collect", action="store_true", - help="跳过采集, 只跑 import_product_reviews.py") - parser.add_argument("--skip-import", action="store_true", - help="跳过入库, 只跑采集") - parser.add_argument("--dry-run", action="store_true", - help="只打印命令不执行") - args = parser.parse_args() - - target_date = args.target_date or default_target_date() - platforms = [p.strip() for p in args.platforms.split(",") if p.strip()] - for p in platforms: - if p not in ("jd", "dy"): - print(f"[ERR] 未知平台: {p} (当前评论采集仅支持 jd, dy)") - return 2 - - today_str = datetime.now().strftime("%Y-%m-%d") - failed = FailedCollector("reviews", today_str) - - print("=" * 60) - print(" 评论采集编排器 (默认 DY)") - print("=" * 60) - print(f" Target date: {target_date}") - print(f" Platforms: {platforms}") - print(f" Skip collect: {args.skip_collect}") - print(f" Skip import: {args.skip_import}") - print(f" Dry run: {args.dry_run}") - print("=" * 60 + "\n") - - # Stage 1: 采集 (串行, Chrome profile 互锁) - if not args.skip_collect: - for plat in platforms: - cmd = build_collect_command(plat, target_date) - run_step_with_retry( - f"collect_{plat}", cmd, - dry_run=args.dry_run, critical=False, failed=failed, - ) - else: - print("[INFO] --skip-collect: 跳过采集阶段\n") - - # Stage 2: 入库 - if not args.skip_import: - import_cmd = [sys.executable, str(runtime_script("import_product_reviews.py")), - "--platforms", ",".join(platforms)] - if args.dry_run: - import_cmd.append("--dry-run") - run_step_with_retry( - "import_reviews", import_cmd, - dry_run=args.dry_run, critical=False, failed=failed, - ) - else: - print("[INFO] --skip-import: 跳过入库阶段\n") - - # 失败清单 - failed_path = failed.write() - if failed_path: - print(f"\n[FAIL_LOG] {failed.summary_line()} -> {failed_path}") - return 1 - print("\n[OK] 评论采集编排完成, 无失败") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/gyxx_flow/modules/product_commerce/scripts/migrate_tm_ad_metrics_to_table.py b/src/gyxx_flow/modules/product_commerce/scripts/migrate_tm_ad_metrics_to_table.py new file mode 100644 index 0000000..9c2bcb2 --- /dev/null +++ b/src/gyxx_flow/modules/product_commerce/scripts/migrate_tm_ad_metrics_to_table.py @@ -0,0 +1,349 @@ +"""一次性迁移：把天猫广告指标从 ``product_daily_metrics.raw_data`` 搬到独立表。 + +背景：天猫广告指标原先写在 ``product_daily_metrics.raw_data.tm_ad_metrics`` +JSONB 节点里，与经营日报共用一行；经营数据重新同步时需要靠 JSONB 节点级合并 +才能保住投放数据。2026-11-01 起改为写 ``tm_product_ad_daily_metrics``（与京东、 +抖音同构），本脚本负责把历史节点回填进新表，并在校验通过后清除旧节点。 + +行为： +- 默认 dry-run：只统计源数据规模、金额合计、款式匹配情况，不写库。 +- ``--execute``：先备份源节点到 JSON（``--backup-dir``，默认运行时 TMP_ROOT）， + 再按 ``(stat_date, product_id)`` 幂等 upsert 到新表。 +- ``--clear-source``：仅在行数/金额校验一致、备份写盘成功后，才执行 + ``raw_data - 'tm_ad_metrics'`` 清除节点；任何一项不通过都不清理并返回非零。 + +用法:: + + python scripts/migrate_tm_ad_metrics_to_table.py + python scripts/migrate_tm_ad_metrics_to_table.py --execute + python scripts/migrate_tm_ad_metrics_to_table.py --execute --clear-source + +连接信息来自 PG_* 环境变量（与 db/__init__.py 同一套）。 +""" +# ruff: noqa: E402 + +from __future__ import annotations + +import argparse +import gzip +import json +import sys +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Iterator + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +import db # noqa: E402 +from runtime_paths import TMP_ROOT # noqa: E402 + +SOURCE_PREDICATE = ( + "platform = 'tm' AND jsonb_typeof(raw_data -> 'tm_ad_metrics') = 'object'" +) +BATCH_SIZE = 500 + + +def _decimal(value: Any) -> Decimal: + if value is None: + return Decimal("0") + text = str(value).strip().replace(",", "") + if not text: + return Decimal("0") + try: + return Decimal(text) + except InvalidOperation: + return Decimal("0") + + +def _iter_source_rows( + conn, date_from: date | None, date_to: date | None, batch_size: int +) -> Iterator[list[dict[str, Any]]]: + """按 (stat_date, product_id) keyset 分页读源节点，避免一次性载入全部行。""" + + clauses = [SOURCE_PREDICATE] + params: list[Any] = [] + if date_from: + clauses.append("stat_date >= %s") + params.append(date_from) + if date_to: + clauses.append("stat_date <= %s") + params.append(date_to) + where = " AND ".join(clauses) + + last: tuple[date, str] | None = None + while True: + page_where = where + page_params = list(params) + if last is not None: + page_where += " AND (stat_date, product_id) > (%s, %s)" + page_params.extend([last[0], last[1]]) + sql = f""" + SELECT stat_date, product_id, product_name, + raw_data -> 'tm_ad_metrics' AS node, + source_file + FROM product_daily_metrics + WHERE {page_where} + ORDER BY stat_date, product_id + LIMIT {batch_size} + """ + with conn.cursor() as cur: + cur.execute(sql, page_params) + page = cur.fetchall() + if not page: + return + yield [ + { + "stat_date": row[0], + "product_id": str(row[1]), + "product_name": row[2], + "node": row[3] or {}, + "source_file": row[4], + } + for row in page + ] + last = (page[-1][0], str(page[-1][1])) + if len(page) < batch_size: + return + + +def _to_records(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """把 JSONB 节点转成 upsert 函数接受的记录形态。""" + + records = [] + for row in rows: + node = dict(row["node"]) + metrics = { + key: value + for key, value in node.items() + if key not in {"style_name", "style_match_method"} + } + records.append( + { + "stat_date": row["stat_date"], + "product_id": row["product_id"], + "product_name": row["product_name"], + "ad_metrics": metrics, + # 节点里已有的款式映射作为兜底；能按 tm_spus/名称重新匹配的以重算为准。 + "style_name": node.get("style_name"), + "style_match_method": node.get("style_match_method"), + "source_file": row["source_file"], + } + ) + return records + + +def _write_backup(rows: list[dict[str, Any]], backup_dir: Path) -> Path: + backup_dir.mkdir(parents=True, exist_ok=True) + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + path = backup_dir / f"tm_ad_metrics_backup_{stamp}.json.gz" + payload = [ + { + "stat_date": row["stat_date"].isoformat(), + "product_id": row["product_id"], + "product_name": row["product_name"], + "source_file": row["source_file"], + "tm_ad_metrics": row["node"], + } + for row in rows + ] + with gzip.open(path, "wt", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False, default=str) + return path + + +def _target_totals(conn, date_from: date | None, date_to: date | None) -> tuple[int, Decimal]: + clauses = ["TRUE"] + params: list[Any] = [] + if date_from: + clauses.append("stat_date >= %s") + params.append(date_from) + if date_to: + clauses.append("stat_date <= %s") + params.append(date_to) + with conn.cursor() as cur: + cur.execute( + f""" + SELECT COUNT(*), COALESCE(SUM(spend), 0) + FROM tm_product_ad_daily_metrics + WHERE {' AND '.join(clauses)} + """, + params, + ) + count, spend = cur.fetchone() + return int(count), Decimal(str(spend)) + + +def _source_count(conn, date_from: date | None, date_to: date | None) -> int: + clauses = [SOURCE_PREDICATE] + params: list[Any] = [] + if date_from: + clauses.append("stat_date >= %s") + params.append(date_from) + if date_to: + clauses.append("stat_date <= %s") + params.append(date_to) + with conn.cursor() as cur: + cur.execute( + f"SELECT COUNT(*) FROM product_daily_metrics WHERE {' AND '.join(clauses)}", + params, + ) + return int(cur.fetchone()[0]) + + +def _parse_date(value: str | None) -> date | None: + return date.fromisoformat(value) if value else None + + +def _missing_target_columns(conn) -> list[str]: + """dry-run 只读检查目标表是否已具备款式/指标列。""" + + required = {"style_name", "style_match_method", "metrics", "source_row_number"} + with conn.cursor() as cur: + cur.execute( + """ + SELECT column_name FROM information_schema.columns + WHERE table_name = 'tm_product_ad_daily_metrics' + """ + ) + existing = {row[0] for row in cur.fetchall()} + if not existing: + return sorted(required | {"<表不存在>"}) + return sorted(required - existing) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="迁移天猫广告指标到独立表") + parser.add_argument("--date-from", help="只迁移该日期起 YYYY-MM-DD") + parser.add_argument("--date-to", help="只迁移到该日期 YYYY-MM-DD") + parser.add_argument("--execute", action="store_true", help="确认写入 PostgreSQL") + parser.add_argument( + "--clear-source", + action="store_true", + help="校验通过后清除 product_daily_metrics 里的旧 JSONB 节点", + ) + parser.add_argument( + "--backup-dir", type=Path, help="旧节点备份目录（默认运行时 TMP_ROOT）" + ) + args = parser.parse_args(argv) + + date_from = _parse_date(args.date_from) + date_to = _parse_date(args.date_to) + backup_dir = args.backup_dir or TMP_ROOT + + with db.get_conn() as conn: + total_rows = _source_count(conn, date_from, date_to) + if total_rows == 0: + print("[INFO] 没有可迁移的天猫广告节点，退出") + return 0 + + tally = { + "rows": 0, + "spend": Decimal("0"), + "deal": Decimal("0"), + "missing_style": 0, + "unknown_style": 0, + "min_date": None, + "max_date": None, + } + all_rows: list[dict[str, Any]] = [] + resolved_style: dict[tuple[str, str], str | None] = {} + + style_by_product_id, style_names = db._load_platform_style_catalog(conn, "tm_spus") + for batch in _iter_source_rows(conn, date_from, date_to, BATCH_SIZE): + all_rows.extend(batch) + for record in _to_records(batch): + node_style = str(record.get("style_name") or "").strip() + if not node_style: + tally["missing_style"] += 1 + key = (record["product_id"], str(record.get("product_name") or "")) + if key not in resolved_style: + resolved_style[key] = db._resolve_style_match( + record, style_by_product_id, style_names + )[0] + if not (resolved_style[key] or node_style): + tally["unknown_style"] += 1 + tally["rows"] += 1 + tally["spend"] += _decimal(record["ad_metrics"].get("spend")) + tally["deal"] += _decimal( + record["ad_metrics"].get("attributed_deal_amount") + ) + stat_date = record["stat_date"] + tally["min_date"] = ( + stat_date if tally["min_date"] is None else min(tally["min_date"], stat_date) + ) + tally["max_date"] = ( + stat_date if tally["max_date"] is None else max(tally["max_date"], stat_date) + ) + + print( + f"[INFO] 源节点 {tally['rows']} 行（{tally['min_date']} ~ {tally['max_date']}），" + f"花费 {tally['spend']:.2f}，广告归因成交 {tally['deal']:.2f}" + ) + print( + f"[INFO] 款式：源节点缺 style_name {tally['missing_style']} 行；" + f"按 tm_spus/名称仍无法确定 {tally['unknown_style']} 行" + ) + + if not args.execute: + existing_count, _ = _target_totals(conn, date_from, date_to) + missing = _missing_target_columns(conn) + print( + f"[INFO] 目标表现有 {existing_count} 行（同日期区间）；" + "DRY-RUN 未写库，确认后添加 --execute" + ) + print( + "[INFO] 目标表待补列：" + + (", ".join(missing) if missing else "无（结构已就绪）") + ) + return 0 + + db.ensure_tmall_product_ad_daily_table(conn) + + backup_path = _write_backup(all_rows, backup_dir) + print(f"[OK] 旧节点已备份：{backup_path}") + + written = 0 + for offset in range(0, len(all_rows), BATCH_SIZE): + records = _to_records(all_rows[offset : offset + BATCH_SIZE]) + written += db.upsert_tmall_product_ad_daily_metrics(conn, records) + print(f"[OK] 已写入 tm_product_ad_daily_metrics：{written} 行") + + target_count, target_spend = _target_totals(conn, date_from, date_to) + ok = target_count == tally["rows"] and target_spend == tally["spend"] + print( + f"[CHECK] 行数 {target_count}/{tally['rows']}，" + f"花费 {target_spend:.2f}/{tally['spend']:.2f} -> " + f"{'一致' if ok else '不一致'}" + ) + if not ok: + print("[FAIL] 校验不一致，保留旧 JSONB 节点，请人工核对后重跑", file=sys.stderr) + return 1 + + if not args.clear_source: + print("[INFO] 未指定 --clear-source，旧节点保留") + return 0 + + with conn.cursor() as cur: + cur.execute( + f""" + UPDATE product_daily_metrics + SET raw_data = COALESCE(raw_data, '{{}}'::jsonb) - 'tm_ad_metrics' + WHERE {SOURCE_PREDICATE} + {"AND stat_date >= %s" if date_from else ""} + {"AND stat_date <= %s" if date_to else ""} + """, + [value for value in (date_from, date_to) if value is not None], + ) + cleared = cur.rowcount + remaining = _source_count(conn, date_from, date_to) + print(f"[OK] 已清除旧节点：{cleared} 行，剩余 {remaining} 行") + if remaining: + print("[FAIL] 仍有残留节点，请重跑 --clear-source", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py b/src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py index 182e6dd..629cf0f 100644 --- a/src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py +++ b/src/gyxx_flow/modules/product_commerce/taobao_wanxiang_ai_creative_report.py @@ -86,6 +86,23 @@ _STATIC_STYLE_NAMES = [ ] +# 万相「主体名称/ID」搜索词覆盖（2026-10-30）： +# 款式身份仍然只用 style_name —— 输出目录、飞书主图表、PG 记录都以款名为键； +# 只有搜索框里填的词可以换，用于投放主体名与内部款名不一致的款。 +# 主搜索词无结果时按「原始款名 → 天猫商品ID」逐级兜底，不会因为改名少采集。 +STYLE_SEARCH_KEYWORDS = { + "黑曜石吐司包": "黑曜石吐司", +} + + +def search_keywords_for(style_name: str) -> tuple[str, ...]: + """返回按优先级排列的万相搜索词；首个元素是主搜索词。""" + keyword = STYLE_SEARCH_KEYWORDS.get(style_name, "").strip() + if not keyword or keyword == style_name: + return (style_name,) + return (keyword, style_name) + + def _get_style_names() -> list[str]: """采集目标=身份齐全的天猫款式；主图飞书表地址只影响写表，不影响采集。""" try: @@ -1656,17 +1673,37 @@ def aggregate_rows(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: return aggregated -def collect_style(page, style_name: str, max_pages: int, item_ids: list[str] | None = None) -> dict[str, Any]: - search_style_by_subject(page, style_name) +def collect_style( + page, + style_name: str, + max_pages: int, + item_ids: list[str] | None = None, + search_keywords: tuple[str, ...] | list[str] | None = None, +) -> dict[str, Any]: + # 搜索词可与款名不同（见 STYLE_SEARCH_KEYWORDS），但 style_name 始终是结果身份， + # 输出目录/飞书/PG 都用它，不受搜索词影响。 + keywords = tuple(search_keywords) if search_keywords else search_keywords_for(style_name) + used_keyword = keywords[0] + if used_keyword != style_name: + print(f" [SEARCH] 搜索词覆盖: {style_name} -> {used_keyword}") + search_style_by_subject(page, used_keyword) all_rows: list[dict[str, Any]] = [] visited_signatures: set[str] = set() - # 先检查款式名搜索是否有结果，无结果时用商品ID回退 + # 先检查主搜索词是否有结果，无结果时按「原始款名 → 商品ID」逐级回退 first_check = extract_current_page_rows(page) + for fallback in keywords[1:]: + if first_check: + break + print(f" [FALLBACK] 搜索词“{used_keyword}”无结果，改用“{fallback}”重搜") + search_style_by_subject(page, fallback) + used_keyword = fallback + first_check = extract_current_page_rows(page) if not first_check and item_ids: first_id = item_ids[0] print(f" [FALLBACK] 款式名搜不到结果，用商品ID {first_id} 重搜") search_style_by_subject(page, first_id) + used_keyword = first_id for page_no in range(1, max_pages + 1): image_map = collect_current_page_image_map(page) @@ -1692,6 +1729,7 @@ def collect_style(page, style_name: str, max_pages: int, item_ids: list[str] | N aggregated = aggregate_rows(all_rows) return { "style_name": style_name, + "search_keyword": used_keyword, "collected_at": datetime.now().isoformat(timespec="seconds"), "page_count": len(visited_signatures), "raw_row_count": len(all_rows), @@ -1731,6 +1769,7 @@ def write_style_outputs(result: dict[str, Any], output_dir: Path) -> tuple[Path, f"# {style_name} 万相创意主图汇总", "", f"- 采集时间: {result.get('collected_at', '')}", + f"- 搜索词: {result.get('search_keyword') or style_name}", f"- 翻页数: {result.get('page_count', 0)}", f"- 原始行数: {result.get('raw_row_count', 0)}", f"- 聚合主图数: {result.get('creative_count', 0)}", diff --git a/src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py b/src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py index 92e8d85..ecdc66d 100644 --- a/src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py +++ b/src/gyxx_flow/modules/product_commerce/upload_video_to_guanghe.py @@ -114,6 +114,10 @@ LUGGAGE_USER_DATA_DIR = os.getenv( "GUANGHE_LUGGAGE_USER_DATA_DIR", str(profile_path("guanghe-luggage")), ) +NEW_BAG_USER_DATA_DIR = os.getenv( + "GUANGHE_NEW_BAG_USER_DATA_DIR", + str(profile_path("guanghe-new-bag")), +) # ─────────────────────────── 配置 ─────────────────────────── BASE_TOKEN = "TH1JbxhfUaxetis6F4pcoqvWncd" @@ -134,10 +138,24 @@ TABLE_FIELDS = [ STORE_FLAGSHIP = "flagship" STORE_LUGGAGE = "luggage" +# 「天猫新店」（淘宝账号 gyxx箱包旗舰店:龙虾仔）只接黑曜石系列，视频源与其他店铺同一张飞书表。 +STORE_NEW_BAG = "new_bag" STORE_ALL = "all" STORE_ACCOUNT_KEYS = { STORE_FLAGSHIP: "guanghe-flagship", STORE_LUGGAGE: "guanghe-luggage", + STORE_NEW_BAG: "guanghe-new-bag", +} +# 每个店铺独立的运行时凭据前缀与浏览器 Profile 目录环境变量；新增店铺只需在此登记。 +STORE_CREDENTIAL_PREFIXES = { + STORE_FLAGSHIP: "GUANGHE", + STORE_LUGGAGE: "GUANGHE_LUGGAGE", + STORE_NEW_BAG: "GUANGHE_NEW_BAG", +} +# 天猫新店没有独立的款式白名单表，按「代拍产品」里的系列关键词过滤： +# 一条记录的全部款式都必须落在该系列内才会上传，避免把非黑曜石视频发到新店。 +STORE_STYLE_KEYWORDS = { + STORE_NEW_BAG: "黑曜石", } # Guanghe exposes a creator account, not the merchant-store name used by the # source routing and publication ledger. The flagship creator card currently @@ -147,6 +165,9 @@ STORE_ACCOUNT_KEYS = { # the platform no longer renders the merchant-store name on its home page. STORE_CREATOR_IDENTITY_MARKERS: dict[str, tuple[str, ...]] = { STORE_FLAGSHIP: ("光影行星天猫官方", "逛逛号：3301283916"), + # 天猫新店的逛逛内容号由店铺资料自动生成，创作者后台只显示这个名字， + # 淘宝登录名 gyxx箱包旗舰店:龙虾仔 不会出现在光合页面上。 + STORE_NEW_BAG: ("GYXX箱包天猫官方",), } EXIT_OK = 0 @@ -2630,24 +2651,13 @@ class GuangHeUploader: # 一次性打开弹窗，复用同一搜索框 self._click_by_text("添加商品", timeout=10_000) print("[OK] 已点开关联商品弹窗") - time.sleep(1.5) - # 搜索框（占位符含「搜索」） - search = None - for sel in [ - "input[placeholder*='搜索']", - "input[type='search']", - "input[placeholder*='商品']", - ]: - try: - loc = self._loc(sel).first - if loc.count() > 0: - search = loc - break - except Exception: - continue + # 天猫新店的商品选择弹窗渲染比旗舰店慢，且占位文案不固定（有的店铺是 + # 「请输入商品ID/名称」），因此轮询多个候选选择器，超时后再打诊断。 + search = self._find_dialog_search_input(timeout_seconds=12) if search is None: print("[ERR] 关联商品弹窗内找不到搜索框 → 跳过本次发布") + self._dump_dialog_diagnostics() for label in ["取消", "关 闭", "关闭"]: try: self._click_by_text(label, timeout=2000) @@ -3183,6 +3193,75 @@ class GuangHeUploader: print("[WARN] 未明确检测到发布成功提示，请人工确认") return published + DIALOG_SEARCH_SELECTORS = ( + "input[placeholder*='搜索']", + "input[placeholder*='请输入']", + "input[placeholder*='商品']", + "input[type='search']", + ".next-dialog input[type='text']", + ".next-dialog input:not([type])", + ) + + def _dialog_contexts(self) -> list: + """发布页 iframe、主页面以及其中所有子 frame，弹窗可能落在任意一层。""" + contexts: list = [] + if self.pub_frame is not None: + contexts.append(self.pub_frame) + contexts.append(self.page) + try: + for frame in self.page.frames: + if frame not in contexts: + contexts.append(frame) + except Exception: + pass + return contexts + + def _find_dialog_search_input(self, timeout_seconds: float = 12): + """轮询各 frame 找关联商品弹窗的搜索框，找不到返回 None。""" + deadline = time.time() + timeout_seconds + while True: + for ctx in self._dialog_contexts(): + for selector in self.DIALOG_SEARCH_SELECTORS: + try: + loc = ctx.locator(selector).first + if loc.count() > 0: + return loc + except Exception: + continue + if time.time() >= deadline: + return None + time.sleep(0.8) + + def _dump_dialog_diagnostics(self) -> None: + """找不到搜索框时把弹窗真实结构写进日志，避免只能靠复现定位。""" + for idx, ctx in enumerate(self._dialog_contexts()): + try: + url = str(getattr(ctx, "url", "") or "") + except Exception: + url = "" + print(f" [诊断 ctx#{idx}] url={url[:120]}") + try: + text = ctx.locator("body").inner_text(timeout=2000) + print(f" text={re.sub(r'\\s+', ' ', text)[:300]}") + except Exception: + pass + try: + handles = ctx.locator("input").all() + for handle in handles[:12]: + try: + placeholder = handle.get_attribute("placeholder", timeout=1000) + input_type = handle.get_attribute("type", timeout=1000) + class_name = handle.get_attribute("class", timeout=1000) + print( + f" input type={input_type} placeholder={placeholder} " + f"class={class_name}" + ) + except Exception: + continue + except Exception: + pass + self._snapshot("07_link_product_no_search") + def _click_by_text(self, text: str, timeout: int = 10_000): """根据文本点击元素，支持逗号分隔多个候选文本。优先在 pub_frame(iframe)里找，fallback 到主页面。""" candidates = [t.strip() for t in text.split(",")] @@ -3255,8 +3334,10 @@ class GuangHeUploader: def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description="将飞书 Base 视频上传到淘宝光合平台") runtime_store = os.getenv("GYXX_VIDEO_UPLOAD_STORE", "").strip() or STORE_FLAGSHIP - if runtime_store not in {STORE_FLAGSHIP, STORE_LUGGAGE, STORE_ALL}: - raise ValueError("GYXX_VIDEO_UPLOAD_STORE 仅支持 flagship、luggage 或 all") + if runtime_store not in {STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG, STORE_ALL}: + raise ValueError( + "GYXX_VIDEO_UPLOAD_STORE 仅支持 flagship、luggage、new_bag 或 all" + ) p.add_argument("--limit", type=int, default=None, help="最多处理多少条记录") p.add_argument("--debug", action="store_true", help="启用调试截图") mode = p.add_mutually_exclusive_group() @@ -3305,9 +3386,12 @@ def parse_args() -> argparse.Namespace: ) p.add_argument( "--store", - choices=[STORE_FLAGSHIP, STORE_LUGGAGE, STORE_ALL], + choices=[STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG, STORE_ALL], default=runtime_store, - help="上传店铺：flagship=旗舰店，luggage=箱包店，all=顺序运行两家", + help=( + "上传店铺：flagship=旗舰店，luggage=箱包店，" + "new_bag=天猫新店（只发黑曜石系列），all=顺序运行三家" + ), ) p.add_argument( "--topic-keyword", @@ -3354,7 +3438,7 @@ def select_pending_records( republish_from_date: Optional[str] = None, ) -> List[Dict[str, Any]]: """Apply the explicit brand/gender/status route for one Guanghe store.""" - if store not in {STORE_FLAGSHIP, STORE_LUGGAGE}: + if store not in {STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG}: raise ValueError(f"未知光合店铺: {store}") if store == STORE_LUGGAGE and not luggage_style_whitelist: raise ValueError("箱包店款式白名单未加载或为空") @@ -3363,6 +3447,7 @@ def select_pending_records( normalized_whitelist = { _normalize_style_name(name) for name in (luggage_style_whitelist or set()) } + style_keyword = _normalize_style_name(STORE_STYLE_KEYWORDS.get(store, "")) result = [] for rec in records: @@ -3390,6 +3475,23 @@ def select_pending_records( or not normalized_products.issubset(normalized_whitelist) ): continue + if store == STORE_NEW_BAG: + # 天猫新店按系列关键词过滤：全部款式都必须属于黑曜石系列， + # 组合款里只要掺了别的系列整条记录都不进新店。 + product_names = GuangHeUploader._parse_product_names( + rec.get("代拍产品") + ) + if "__SKIP__" in product_names: + continue + normalized_products = [ + _normalize_style_name(name) + for name in product_names + if name and name != "__SKIP__" + ] + if not normalized_products or not all( + style_keyword in name for name in normalized_products + ): + continue if store == STORE_FLAGSHIP: if republish_from_date: record_date = str(rec.get("日期") or "")[:10] @@ -3680,6 +3782,13 @@ def _store_runtime(store: str) -> Tuple[str, str, str, str]: os.getenv("GUANGHE_LUGGAGE_PASSWORD", ""), LUGGAGE_USER_DATA_DIR, ) + if store == STORE_NEW_BAG: + return ( + "gyxx箱包旗舰店:龙虾仔", + os.getenv("GUANGHE_NEW_BAG_USERNAME", ""), + os.getenv("GUANGHE_NEW_BAG_PASSWORD", ""), + NEW_BAG_USER_DATA_DIR, + ) raise ValueError(f"未知光合店铺: {store}") @@ -3870,7 +3979,7 @@ def _run_store( print(f"[OK] {store_name} 没有可入账的视频附件") return EXIT_OK if not username or not password: - prefix = "GUANGHE" if store == STORE_FLAGSHIP else "GUANGHE_LUGGAGE" + prefix = STORE_CREDENTIAL_PREFIXES[store] raise RuntimeError(f"请在运行时配置 {prefix}_USERNAME / {prefix}_PASSWORD") if not direct_llm_configured(): raise RuntimeError( @@ -4186,7 +4295,7 @@ def main() -> int: print(f"[OK] 读取到 {len(records)} 条记录") stores = ( - [STORE_FLAGSHIP, STORE_LUGGAGE] + [STORE_FLAGSHIP, STORE_LUGGAGE, STORE_NEW_BAG] if args.store == STORE_ALL else [args.store] ) diff --git a/src/gyxx_flow/web/app.js b/src/gyxx_flow/web/app.js index 9a2c9de..2fd69df 100644 --- a/src/gyxx_flow/web/app.js +++ b/src/gyxx_flow/web/app.js @@ -143,6 +143,7 @@ options: [ { value: "flagship", label: "光影行星旗舰店" }, { value: "luggage", label: "光影行星鑫华达专卖店:龙虾仔" }, + { value: "new_bag", label: "天猫新店 · gyxx箱包旗舰店:龙虾仔（只发黑曜石系列）" }, ], }; } diff --git a/src/gyxx_flow/web/index.html b/src/gyxx_flow/web/index.html index 8e7bd53..c401cff 100644 --- a/src/gyxx_flow/web/index.html +++ b/src/gyxx_flow/web/index.html @@ -7,7 +7,7 @@ GYXX Flow · 工作流控制台 - + @@ -475,6 +475,7 @@ 先扫描飞书并按所选店铺规则筛选；存在未上传附件时才登录该店铺账号。 @@ -636,6 +637,6 @@
- + diff --git a/tests/modules/product_commerce/test_daily_result_integrity.py b/tests/modules/product_commerce/test_daily_result_integrity.py index cbba330..911f6bd 100644 --- a/tests/modules/product_commerce/test_daily_result_integrity.py +++ b/tests/modules/product_commerce/test_daily_result_integrity.py @@ -426,27 +426,31 @@ def test_platform_artifact_verifier_rejects_an_export_with_no_product_matches( daily.verify_platform_artifacts("dy", "2026-08-03", marker_ns) -def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None: +def test_daily_collect_no_longer_runs_review_import(monkeypatch) -> None: + """product_reviews 已下线 (2026-11-01)。 + + collect 阶段不得再调评论入库——差评由 shop.jd/dy/tmall_negative_reviews.daily + 三个独立工作流采集，消费方读 v_negative_reviews。这条断言守住下线状态， + 防止有人顺手把旧链路加回来。 + """ + args = SimpleNamespace( skip_erp=True, skip_platforms=False, serial_platforms=False, dry_run=False, - skip_reviews_import=False, ) monkeypatch.setattr(daily, "run_platforms", lambda **_kwargs: []) calls = [] monkeypatch.setattr( daily, "run_step_with_retry", - lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 1}, + lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0}, ) daily.run_collect(args, "2026-08-03", ["jd"], failed=None) - assert len(calls) == 1 - assert calls[0][0][0] == "import_product_reviews" - assert calls[0][1]["critical"] is False + assert calls == [] def test_daily_import_stage_uses_the_collected_business_date(monkeypatch) -> None: diff --git a/tests/modules/product_commerce/test_erp_all_shop_daily_backfill.py b/tests/modules/product_commerce/test_erp_all_shop_daily_backfill.py index 0a9e335..7b8c623 100644 --- a/tests/modules/product_commerce/test_erp_all_shop_daily_backfill.py +++ b/tests/modules/product_commerce/test_erp_all_shop_daily_backfill.py @@ -310,6 +310,14 @@ def test_database_upsert_uses_date_and_style_conflict_key(monkeypatch) -> None: def __exit__(self, *_args): return None + # db.attach_style_ids 会按款名查 dim_style；这里返回空映射， + # 断言的是 SQL 形状与业务列，不依赖真实款式表。 + def execute(self, *_args, **_kwargs): + return None + + def fetchall(self): + return [] + class Connection: def cursor(self): return Cursor() diff --git a/tests/modules/product_commerce/test_erp_metric_overrides.py b/tests/modules/product_commerce/test_erp_metric_overrides.py index 83913a2..1df564d 100644 --- a/tests/modules/product_commerce/test_erp_metric_overrides.py +++ b/tests/modules/product_commerce/test_erp_metric_overrides.py @@ -15,8 +15,8 @@ class ErpMetricOverridesTests(unittest.TestCase): def tearDown(self): self.temp_dir.cleanup() - def _write_erp(self, code_results): - data_dir = self.data_root / "dy" / "apollo_2026-07-22" + def _write_erp(self, code_results, platform="dy"): + data_dir = self.data_root / platform / "apollo_2026-07-22" data_dir.mkdir(parents=True) (data_dir / "data.json").write_text( json.dumps( @@ -33,12 +33,12 @@ class ErpMetricOverridesTests(unittest.TestCase): encoding="utf-8", ) - def _apply(self, platform_sales, platform_refunds): + def _apply(self, platform_sales, platform_refunds, platform="dy"): with patch.object(overrides, "DATA_ROOT", self.data_root), patch.dict( - overrides.PLATFORM_DIRS, {"dy": "dy"} + overrides.PLATFORM_DIRS, {platform: platform} ): return overrides.apply_erp_override( - "dy", "阿波罗x1", 394, 46, + platform, "阿波罗x1", 394, 46, target_date="2026-07-22", platform_sales=platform_sales, platform_refunds=platform_refunds, @@ -87,6 +87,21 @@ class ErpMetricOverridesTests(unittest.TestCase): self.assertEqual(result["销量"], 0) self.assertEqual(result["退款订单数"], 0) + def test_partial_codes_used_when_platform_has_no_sales(self): + """TM 没有平台销量字段：部分编码失败时不能整款归零，用成功编码合计。""" + self._write_erp([ + {"erp_style_code": "1", "ok": True, "yesterday_sales": 3, "yesterday_returns": 1}, + {"erp_style_code": "2", "ok": False, "error": "timeout", "yesterday_sales": 0, "yesterday_returns": 0}, + ], platform="tm") + + result = self._apply(platform_sales=None, platform_refunds=None, platform="tm") + + self.assertTrue(result["erp_unavailable"]) + self.assertTrue(result["erp_partial"]) + self.assertEqual(result["销量"], 3) + self.assertEqual(result["退款订单数"], 1) + self.assertEqual(result["failed_codes"], ["2"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/modules/product_commerce/test_guanghe_store_routing.py b/tests/modules/product_commerce/test_guanghe_store_routing.py index 2e13e41..ac0da01 100644 --- a/tests/modules/product_commerce/test_guanghe_store_routing.py +++ b/tests/modules/product_commerce/test_guanghe_store_routing.py @@ -3,6 +3,7 @@ import unittest from upload_video_to_guanghe import ( STORE_FLAGSHIP, STORE_LUGGAGE, + STORE_NEW_BAG, TM_UPLOAD_FIELD, select_pending_records, ) @@ -113,6 +114,53 @@ class GuangHeStoreRoutingTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "未知光合店铺"): select_pending_records([record()], "unknown") + def test_new_bag_uploads_only_obsidian_series_regardless_of_tm_status(self): + unchecked = record(gender="女", uploaded=False) + unchecked["代拍产品"] = "黑曜石托特包" + checked = record(gender="女", uploaded=True) + checked["代拍产品"] = "黑曜石吐司包" + records = [ + unchecked, + checked, + record(gender="男"), + ] + + selected = select_pending_records(records, STORE_NEW_BAG) + + self.assertEqual(selected, [unchecked, checked]) + + def test_new_bag_skips_mixed_and_non_obsidian_styles(self): + obsidian = record() + obsidian["代拍产品"] = " 黑曜石电脑包 " + mixed = record() + mixed["代拍产品"] = "黑曜石饺子 / 极星双肩" + other = record() + other["代拍产品"] = "极星双肩" + skipped = record() + skipped["代拍产品"] = "黑曜石饺子 / 未到货" + no_product = record() + other_brand = record(brand="OZKO") + other_brand["代拍产品"] = "黑曜石托特包" + no_attachment = record(attachments=False) + no_attachment["代拍产品"] = "黑曜石托特包" + + selected = select_pending_records( + [obsidian, mixed, other, skipped, no_product, other_brand, no_attachment], + STORE_NEW_BAG, + ) + + self.assertEqual(selected, [obsidian]) + + def test_new_bag_rejects_republish_from_date(self): + rec = record(date="2026-06-03 00:00:00") + rec["代拍产品"] = "黑曜石托特包" + with self.assertRaisesRegex(ValueError, "旗舰店"): + select_pending_records( + [rec], + STORE_NEW_BAG, + republish_from_date="2026-06-03", + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/modules/product_commerce/test_import_douyin_qianchuan_ads.py b/tests/modules/product_commerce/test_import_douyin_qianchuan_ads.py index 4fc2916..863676a 100644 --- a/tests/modules/product_commerce/test_import_douyin_qianchuan_ads.py +++ b/tests/modules/product_commerce/test_import_douyin_qianchuan_ads.py @@ -119,6 +119,9 @@ def test_upsert_uses_advertiser_date_product_business_key(monkeypatch) -> None: executed.update(sql=sql, rows=rows) monkeypatch.setattr(product_db, "execute_values", fake_execute_values) + # resolve_style_ids 走独立查询；本测试只验证 SQL 形状与款式匹配列， + # 用空映射避免和上面的 style catalog 假数据串台。 + monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {}) record = { "stat_date": date(2026, 8, 23), "advertiser_id": importer.ADVERTISER_ID, @@ -162,6 +165,7 @@ def test_upsert_falls_back_to_exact_product_name_style_match(monkeypatch) -> Non executed.update(sql=sql, rows=rows) monkeypatch.setattr(product_db, "execute_values", fake_execute_values) + monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {}) record = { "stat_date": date(2026, 8, 23), "advertiser_id": importer.ADVERTISER_ID, diff --git a/tests/modules/product_commerce/test_import_tmall_product_ads.py b/tests/modules/product_commerce/test_import_tmall_product_ads.py index fee8972..217210d 100644 --- a/tests/modules/product_commerce/test_import_tmall_product_ads.py +++ b/tests/modules/product_commerce/test_import_tmall_product_ads.py @@ -99,7 +99,7 @@ def test_find_report_files_uses_current_manifest_csv(monkeypatch, tmp_path: Path assert importer.find_report_files(target_date) == [current_csv.resolve()] -def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None: +def test_upsert_writes_dedicated_table_with_style_and_metrics(monkeypatch) -> None: captured: dict[str, object] = {} def fake_execute_values(_cursor, sql, rows, *, template=None): @@ -108,7 +108,9 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None: captured["template"] = template monkeypatch.setattr(product_db, "execute_values", fake_execute_values) - written = product_db.upsert_tmall_product_ad_raw_metrics( + # 款式主键由 resolve_style_ids 单独查询；本测试只验证列顺序与款式匹配结果。 + monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {}) + written = product_db.upsert_tmall_product_ad_daily_metrics( _Connection(), [ { @@ -119,6 +121,7 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None: "spend": "90.25", "attributed_deal_amount": "730.02", "roi": "8.09", + "impressions": 1000, }, "source_file": "商品报表.csv", } @@ -127,16 +130,22 @@ def test_upsert_updates_only_tmall_ad_json_node(monkeypatch) -> None: assert written == 1 sql = str(captured["sql"]) - assert "INSERT INTO product_daily_metrics" in sql - assert "jsonb_set" in sql - assert "tm_product_ad_daily_metrics" not in sql - assert captured["template"] == "('tm', %s, %s, %s, %s::jsonb, %s, NOW())" + assert "INSERT INTO tm_product_ad_daily_metrics" in sql + assert "product_daily_metrics" not in sql + assert "jsonb_set" not in sql row = captured["rows"][0] - payload = json.loads(row[3]) - assert payload["tm_ad_metrics"]["spend"] == "90.25" - assert payload["tm_ad_metrics"]["style_name"] == "极星" - assert payload["tm_ad_metrics"]["style_match_method"] == "product_id" assert row[1] == "1001" + assert row[3] == "极星" + assert row[4] == "product_id" + assert str(row[5]) == "90.25" + assert str(row[8]) == "730.02" + assert str(row[9]) == "8.09" + # 15 个业务列 + style_id（款式主键统一 2026-11-01） + assert captured["template"].count("%s") == 16 + assert row[-1] is None # resolve_style_ids 被 mock 成空映射 + metrics = json.loads(row[11]) + assert metrics["impressions"] == 1000 + assert "style_name" not in metrics def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> None: @@ -154,7 +163,9 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> captured["rows"] = rows monkeypatch.setattr(product_db, "execute_values", fake_execute_values) - written = product_db.upsert_tmall_product_ad_raw_metrics( + # 款式主键由 resolve_style_ids 单独查询；本测试只验证列顺序与款式匹配结果。 + monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {}) + written = product_db.upsert_tmall_product_ad_daily_metrics( _NameFallbackConnection(), [ { @@ -169,9 +180,41 @@ def test_upsert_tmall_ad_uses_product_name_when_id_is_not_mapped(monkeypatch) -> assert written == 1 row = captured["rows"][0] - payload = json.loads(row[3]) - assert payload["tm_ad_metrics"]["style_name"] == "极星" - assert payload["tm_ad_metrics"]["style_match_method"] == "product_name" + assert row[3] == "极星" + assert row[4] == "product_name" + + +def test_upsert_dedupes_repeated_product_day(monkeypatch) -> None: + captured: dict[str, object] = {} + + def fake_execute_values(_cursor, _sql, rows, *, template=None): + captured["rows"] = rows + + monkeypatch.setattr(product_db, "execute_values", fake_execute_values) + # 款式主键由 resolve_style_ids 单独查询；本测试只验证列顺序与款式匹配结果。 + monkeypatch.setattr(product_db, "resolve_style_ids", lambda *a, **k: {}) + written = product_db.upsert_tmall_product_ad_daily_metrics( + _Connection(), + [ + { + "stat_date": importer.date(2026, 8, 17), + "product_id": "1001", + "product_name": "older", + "ad_metrics": {"spend": "1.00"}, + }, + { + "stat_date": importer.date(2026, 8, 17), + "product_id": "1001", + "product_name": "latest", + "ad_metrics": {"spend": "2.00"}, + }, + ], + ) + + assert written == 1 + row = captured["rows"][0] + assert row[2] == "latest" + assert str(row[5]) == "2.00" def test_import_date_filter_and_dry_run(tmp_path: Path, capsys) -> None: diff --git a/tests/modules/product_commerce/test_main_image_concurrency.py b/tests/modules/product_commerce/test_main_image_concurrency.py index ea3deeb..442a660 100644 --- a/tests/modules/product_commerce/test_main_image_concurrency.py +++ b/tests/modules/product_commerce/test_main_image_concurrency.py @@ -312,7 +312,9 @@ class MainImageDatabaseTests(unittest.TestCase): } for platform in ("jd", "tm") ] - with patch.object(product_db, "execute_values") as execute_values: + with patch.object(product_db, "execute_values") as execute_values, patch.object( + product_db, "resolve_style_ids", return_value={} + ): count = product_db.upsert_main_image_creatives(Connection(), records) self.assertEqual(count, 2) diff --git a/tests/modules/product_commerce/test_product_daily_raw_data_protection.py b/tests/modules/product_commerce/test_product_daily_raw_data_protection.py index 17f9517..1f0cae6 100644 --- a/tests/modules/product_commerce/test_product_daily_raw_data_protection.py +++ b/tests/modules/product_commerce/test_product_daily_raw_data_protection.py @@ -12,6 +12,13 @@ class _Cursor: def __exit__(self, *_args): return False + # db 层按商品 ID 反查 dim_style 时用；空映射表示这些商品都没建档。 + def execute(self, *_args, **_kwargs): + return None + + def fetchall(self): + return [] + class _Connection: def cursor(self): diff --git a/tests/modules/product_commerce/test_product_nested_child_bindings.py b/tests/modules/product_commerce/test_product_nested_child_bindings.py index 6667b82..4bd0141 100644 --- a/tests/modules/product_commerce/test_product_nested_child_bindings.py +++ b/tests/modules/product_commerce/test_product_nested_child_bindings.py @@ -11,7 +11,6 @@ import collect_persona_to_bitable as tm_persona_worker import dy_product_scraping as dy_product import orchestrate_daily_collection as daily import orchestrate_market_rank_collection as market_rank -import orchestrate_review_collection as reviews import run_alerts_with_retry as alerts import run_daily_persona as persona import run_weekly_jd_main_image as jd_main_image @@ -191,24 +190,6 @@ def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None: _assert_child_binding(environment) -def test_review_run_step_rebinds_to_command_target(monkeypatch) -> None: - observed: dict[str, object] = {} - monkeypatch.setattr(reviews, "environment_for_child_script", _fake_rebind(observed)) - - def fake_run(command, **kwargs): - observed["environment"] = kwargs["env"] - return subprocess.CompletedProcess(command, 0) - - monkeypatch.setattr(reviews.subprocess, "run", fake_run) - target = reviews.PROJECT_ROOT / "dy_product_scraping.py" - - result = reviews.run_step("dy", ["python", str(target)], critical=True) - - assert result["code"] == 0 - assert observed["target"] == target - _assert_child_binding(observed["environment"]) - - def test_main_image_launchers_rebind_to_command_target(monkeypatch) -> None: for launcher in (tm_main_image, jd_main_image): observed: dict[str, object] = {} diff --git a/tests/modules/product_commerce/test_style_analysis_orchestration.py b/tests/modules/product_commerce/test_style_analysis_orchestration.py index 1c6906c..15a783d 100644 --- a/tests/modules/product_commerce/test_style_analysis_orchestration.py +++ b/tests/modules/product_commerce/test_style_analysis_orchestration.py @@ -116,7 +116,8 @@ class StyleAnalysisWindowTests(unittest.TestCase): self.assertNotIn("近 60d", prompt) self.assertIn("逐日经营数据", prompt) self.assertIn("人物画像", prompt) - self.assertIn("商品评价", prompt) + # product_reviews 下线后数据源改为差评视图，标题同步改成「商品差评」 + self.assertIn("商品差评", prompt) self.assertIn("主图表现", prompt) self.assertIn("营销笔记、曝光互动与评论", prompt) self.assertIn("肩带偏硬", prompt) diff --git a/tests/modules/product_commerce/test_style_config_loader_erp_source.py b/tests/modules/product_commerce/test_style_config_loader_erp_source.py index 9b34dd6..37baf69 100644 --- a/tests/modules/product_commerce/test_style_config_loader_erp_source.py +++ b/tests/modules/product_commerce/test_style_config_loader_erp_source.py @@ -73,6 +73,9 @@ def test_database_row_uses_explicit_style_name_not_style_content() -> None: "series_name": "", "is_series_anchor": False, "collect_enabled": True, + # 款式主键统一 (2026-11-01): loader 把 dim_style.id 作为正式字段下发 + "style_id": None, + "style_match_method": "", } ] diff --git a/tests/modules/product_commerce/test_style_id_identity.py b/tests/modules/product_commerce/test_style_id_identity.py new file mode 100644 index 0000000..72beef0 --- /dev/null +++ b/tests/modules/product_commerce/test_style_id_identity.py @@ -0,0 +1,470 @@ +"""款式主键统一 (style_id = dim_style.id) 的契约测试。 + +守住四件事： +1. 写入端必须把 style_id 落库（13 张采集表）； +2. 调用方已带 style_id 时不覆盖，缺省时由 db 层按款名 / 平台商品 ID 兜底； +3. 匹配不上要留痕 ``unmatched``，不能静默写成 NULL 了事； +4. schema.sql 里必须有列、索引、观测视图，且**不得**建物理外键（style_id 只做逻辑外键）， + 同时不再建已下线的 product_reviews。 + +全部测试离线运行（Fake connection），不依赖 PostgreSQL。 +""" + +from __future__ import annotations + +from pathlib import Path + +import db +import pytest + + +MODULE_ROOT = ( + Path(__file__).resolve().parents[3] + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" +) +SCHEMA_PATH = MODULE_ROOT / "db" / "schema.sql" + + +class RecordingCursor: + def __init__(self, conn: "RecordingConnection") -> None: + self._conn = conn + self._rows: list[tuple] = [] + + def __enter__(self) -> "RecordingCursor": + return self + + def __exit__(self, *_args) -> bool: + return False + + def execute(self, sql: str, params=None) -> None: + self._conn.sqls.append((sql, params)) + if "unnest" in sql and params: + self._rows = [ + (item, self._conn.product_lookup[item]) + for item in params[0] + if item in self._conn.product_lookup + ] + elif "FROM dim_style" in sql and "ANY" in sql and params: + self._rows = [ + (name, self._conn.name_lookup[name]) + for name in params[0] + if name in self._conn.name_lookup + ] + else: + self._rows = [] + + def fetchall(self) -> list[tuple]: + return self._rows + + def fetchone(self): + return None + + @property + def rowcount(self) -> int: + return 0 + + +class RecordingConnection: + def __init__(self, name_lookup=None, product_lookup=None) -> None: + self.name_lookup = dict(name_lookup or {}) + self.product_lookup = dict(product_lookup or {}) + self.sqls: list[tuple] = [] + + def cursor(self) -> RecordingCursor: + return RecordingCursor(self) + + +@pytest.fixture() +def captured(monkeypatch): + """拦截 execute_values，暴露最终写入的 SQL 与 rows。""" + + box: dict[str, object] = {} + + def fake_execute_values(_cursor, sql, rows): + box["sql"] = sql + box["rows"] = list(rows) + + monkeypatch.setattr(db, "execute_values", fake_execute_values) + return box + + +def test_attach_style_ids_keeps_supplied_identity() -> None: + conn = RecordingConnection(name_lookup={"盖亚斜挎": 7}) + + records = db.attach_style_ids( + conn, + [{"style_name": "盖亚斜挎", "style_id": 42, "style_match_method": "dim_style"}], + ) + + assert records[0]["style_id"] == 42 + assert records[0]["style_match_method"] == "dim_style" + # 已有 style_id 时不应再查库 + assert conn.sqls == [] + + +def test_attach_style_ids_fills_missing_and_marks_unmatched() -> None: + conn = RecordingConnection(name_lookup={"盖亚斜挎": 7}) + + records = db.attach_style_ids( + conn, + [{"style_name": "盖亚斜挎"}, {"style_name": "未建档款"}], + ) + + assert records[0]["style_id"] == 7 + assert records[0]["style_match_method"] == db.STYLE_MATCH_BY_NAME + assert records[1]["style_id"] is None + assert records[1]["style_match_method"] == db.STYLE_MATCH_UNMATCHED + + +def test_attach_style_ids_does_not_mutate_caller_records() -> None: + conn = RecordingConnection(name_lookup={"盖亚斜挎": 7}) + original = [{"style_name": "盖亚斜挎"}] + + db.attach_style_ids(conn, original) + + assert "style_id" not in original[0] + + +def test_daily_style_metrics_upsert_writes_style_id(captured) -> None: + conn = RecordingConnection(name_lookup={"盖亚斜挎": 7}) + + written = db.upsert_daily_metrics( + conn, + [ + { + "metric_date": "2026-10-31", + "style_name": "盖亚斜挎", + "platform": "tm", + "sales": 3, + } + ], + ) + + assert written == 1 + sql = captured["sql"] + assert "style_id" in sql + assert "style_match_method" in sql + # 冲突时不能把已填的 id 抹成 NULL + assert "COALESCE(EXCLUDED.style_id" in sql + row = captured["rows"][0] + assert row[-2] == 7 + assert row[-1] == db.STYLE_MATCH_BY_NAME + + +def test_erp_all_shop_upsert_writes_style_id(captured) -> None: + conn = RecordingConnection(name_lookup={"极星双肩": 11}) + + db.upsert_erp_all_shop_style_daily_metrics( + conn, + [ + { + "metric_date": "2026-10-31", + "style_name": "极星双肩", + "sales": 10, + "selected_shop_count": 2, + "source": "jushuitan", + } + ], + ) + + row = captured["rows"][0] + assert row[-2] == 11 + assert row[-1] == db.STYLE_MATCH_BY_NAME + + +def test_weekly_metrics_upsert_writes_style_id(captured) -> None: + conn = RecordingConnection(name_lookup={"极星双肩": 11}) + + db.upsert_weekly_metrics( + conn, + [ + { + "week_end": "2026-10-31", + "week_start": "2026-10-25", + "style_name": "极星双肩", + "platform": "tm", + "sales": 5, + } + ], + ) + + assert "style_id" in captured["sql"] + assert captured["rows"][0][-2] == 11 + + +def test_product_daily_metrics_resolves_by_platform_product_id(captured, monkeypatch) -> None: + conn = RecordingConnection(product_lookup={"700001": 33}) + + written = db.upsert_product_daily_metrics( + conn, + [ + { + "platform": "tm", + "stat_date": "2026-10-31", + "product_id": "700001", + "product_name": "GYXX 极星双肩包", + "paid_amount": 100, + }, + { + "platform": "tm", + "stat_date": "2026-10-31", + "product_id": "999999", + "product_name": "未建档商品", + "paid_amount": 5, + }, + ], + ) + + assert written == 2 + rows = captured["rows"] + assert rows[0][-2] == 33 + assert rows[0][-1] == db.STYLE_MATCH_BY_PRODUCT + assert rows[1][-2] is None + assert rows[1][-1] == db.STYLE_MATCH_UNMATCHED + + +def test_product_daily_metrics_keeps_existing_identity_on_conflict(captured) -> None: + conn = RecordingConnection(product_lookup={}) + + db.upsert_product_daily_metrics( + conn, + [ + { + "platform": "jd", + "stat_date": "2026-10-31", + "product_id": "abc", + "paid_amount": 1, + } + ], + ) + + sql = captured["sql"] + assert "COALESCE(EXCLUDED.style_id, product_daily_metrics.style_id)" in sql + + +def test_ad_tables_write_style_id(captured) -> None: + conn = RecordingConnection( + name_lookup={"盖亚斜挎": 7}, + product_lookup={"5001": 7}, + ) + + db.upsert_jd_product_ad_daily_metrics( + conn, + [ + { + "stat_date": "2026-10-31", + "advertiser_pin": "pin", + "shop_account": "shop", + "report_type": "spu", + "row_key": "r1", + "style_name": "盖亚斜挎", + "spend": 12, + } + ], + ) + assert "style_id" in captured["sql"] + assert captured["rows"][0][-2] == 7 + + db.upsert_douyin_product_ad_daily_metrics( + conn, + [ + { + "stat_date": "2026-10-31", + "advertiser_id": "adv", + "product_id": "5001", + "product_name": "盖亚斜挎包", + "style_name": "盖亚斜挎", + "style_match_method": "product_id", + "spend": 8, + } + ], + ) + assert "style_id" in captured["sql"] + assert captured["rows"][0][-1] == 7 + + +def test_persona_and_main_image_write_style_id(captured) -> None: + conn = RecordingConnection(name_lookup={"盖亚斜挎": 7}) + + db.upsert_persona_metrics( + conn, + [ + { + "collect_date": "2026-10-31", + "style_name": "盖亚斜挎", + "platform": "tm", + "payload": {}, + } + ], + ) + assert captured["rows"][0][-2] == 7 + + db.upsert_main_image_creatives( + conn, + [ + { + "collect_date": "2026-10-31", + "style_name": "盖亚斜挎", + "image_key": "k1", + } + ], + ) + assert captured["rows"][0][-2] == 7 + + +@pytest.mark.parametrize( + "table", + [ + "daily_style_metrics", + "weekly_style_metrics", + "erp_all_shop_style_daily_metrics", + "daily_persona_metrics", + "main_image_creatives", + "fact_style_analysis_reports", + "decline_events", + "fact_alert_history", + "jd_self_weekly_sales", + "persona_screenshots", + "product_daily_metrics", + "jd_product_ad_daily_metrics", + "dy_product_ad_daily_metrics", + ], +) +def test_schema_adds_style_id_column(table: str) -> None: + text = SCHEMA_PATH.read_text(encoding="utf-8") + assert f"ALTER TABLE {table}" in text + assert f"ADD COLUMN IF NOT EXISTS style_id BIGINT" in text + + +def test_schema_defines_observation_views_and_no_physical_foreign_keys() -> None: + text = SCHEMA_PATH.read_text(encoding="utf-8") + + assert "CREATE OR REPLACE VIEW v_style_unmatched_product" in text + assert "CREATE OR REPLACE VIEW v_style_id_coverage" in text + assert "CREATE OR REPLACE VIEW v_negative_reviews" in text + + # style_id 只做逻辑外键：库里不得出现指向 dim_style 的物理外键。 + # 采集语义是「宁可留 NULL 待回填，也不要整批写入失败」，硬约束与之冲突。 + assert "FOREIGN KEY (style_id) REFERENCES dim_style(id)" not in text + assert "REFERENCES dim_style(id)" not in text + # 老库重跑 schema 时要能自己清掉早期建的约束（幂等 DROP） + assert "DROP CONSTRAINT IF EXISTS" in text + assert "_style_id_fk" in text + + +def test_schema_no_longer_creates_product_reviews() -> None: + text = SCHEMA_PATH.read_text(encoding="utf-8") + + assert "CREATE TABLE IF NOT EXISTS product_reviews " not in text + assert "ALTER TABLE product_reviews " not in text + assert "product_reviews_archived_20261101" in text + + +# ---------------------------------------------------------------- 回填脚本 + +def test_backfill_covers_three_ad_tables_with_right_id_column() -> None: + """广告表的 style_name 大量为空，必须按商品 ID 再回填一遍。""" + + import backfill_style_id as bf + + assert set(bf.AD_PRODUCT_TABLES) == { + "tm_product_ad_daily_metrics", + "jd_product_ad_daily_metrics", + "dy_product_ad_daily_metrics", + } + assert bf.AD_PRODUCT_TABLES["tm_product_ad_daily_metrics"] == ("product_id", "tm") + # 京东广告表的商品列叫 spu_id，不是 product_id + assert bf.AD_PRODUCT_TABLES["jd_product_ad_daily_metrics"] == ("spu_id", "jd") + assert bf.AD_PRODUCT_TABLES["dy_product_ad_daily_metrics"] == ("product_id", "dy") + + # 款名表里的广告表项不能删，否则 style_name 有值的那部分就回填不了 + for table in bf.AD_PRODUCT_TABLES: + assert table in bf.NAME_TABLES + + +class _BackfillCursor: + def __init__(self, log: list) -> None: + self._log = log + self._rows: list[tuple] = [] + self.rowcount = 0 + + def __enter__(self) -> "_BackfillCursor": + return self + + def __exit__(self, *_args) -> bool: + return False + + def execute(self, sql: str, params=None) -> None: + self._log.append((" ".join(sql.split()), params)) + if "SELECT DISTINCT" in sql: + self._rows = [("1001",), ("9999",)] + elif "SELECT id, style_name FROM dim_style" in sql: + self._rows = [(7, "赤潮")] + elif sql.lstrip().startswith("UPDATE"): + self.rowcount = 5 if "style_id IS NULL AND style_match_method IS NULL" not in sql else 0 + elif "COUNT(*)" in sql: + self._rows = [(10, 2)] + else: + self._rows = [] + + def fetchall(self) -> list[tuple]: + return self._rows + + def fetchone(self) -> tuple: + return self._rows[0] if self._rows else (0, 0) + + +class _BackfillConn: + def __init__(self) -> None: + self.log: list = [] + + def cursor(self) -> _BackfillCursor: + return _BackfillCursor(self.log) + + +def test_backfill_ad_tables_fills_style_id_and_keeps_platform_style_name(monkeypatch) -> None: + """回填广告表时：style_id 照写，style_name 只在原值为空时补。""" + + import backfill_style_id as bf + + monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7}) + conn = _BackfillConn() + result = bf._backfill_ad_tables_by_product( + conn, ["jd_product_ad_daily_metrics"], dry_run=False + )[0] + + updates = [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")] + payload = next(sql for sql in updates if "NULLIF(BTRIM(style_name)" in sql) + assert "COALESCE(NULLIF(BTRIM(style_name), ''), %s)" in payload, ( + "style_name 只能补空，不能覆盖平台已给出的值" + ) + assert "AND style_id IS NULL" in payload, "幂等：已归属的行不能被改写" + assert "spu_id = %s" in payload, "京东广告表要按 spu_id 反查" + + # 匹配上的 1 个 ID 写 style_id，1 个未建档 + assert result["updated"] == 5 + assert result["unmatched_product_ids"] == 1 + + # 未建档的显式标 unmatched，供观测视图统计 + assert any( + params and params[0] == "unmatched" + for sql, params in conn.log + if sql.lstrip().startswith("UPDATE") + ) + + +def test_backfill_ad_tables_dry_run_does_not_update(monkeypatch) -> None: + import backfill_style_id as bf + + monkeypatch.setattr(bf, "resolve_style_ids_by_product", lambda *a, **k: {"1001": 7}) + conn = _BackfillConn() + result = bf._backfill_ad_tables_by_product( + conn, ["dy_product_ad_daily_metrics"], dry_run=True + )[0] + + assert result["dry_run"] is True + assert result["planned_keys"] == 1 + assert not [sql for sql, _ in conn.log if sql.lstrip().startswith("UPDATE")] diff --git a/tests/modules/product_commerce/test_tmall_video_attachment_idempotency.py b/tests/modules/product_commerce/test_tmall_video_attachment_idempotency.py index e336045..f481676 100644 --- a/tests/modules/product_commerce/test_tmall_video_attachment_idempotency.py +++ b/tests/modules/product_commerce/test_tmall_video_attachment_idempotency.py @@ -176,13 +176,22 @@ def test_discovery_creates_one_candidate_per_exact_video_attachment() -> None: def test_flagship_and_luggage_use_distinct_account_scopes() -> None: flagship = _one_candidate(store=tmall.STORE_FLAGSHIP) luggage = _one_candidate(store=tmall.STORE_LUGGAGE) + new_bag = _one_candidate(store=tmall.STORE_NEW_BAG) assert flagship.target_account_key == "guanghe-flagship" assert luggage.target_account_key == "guanghe-luggage" - assert flagship.target_account_key != luggage.target_account_key + assert new_bag.target_account_key == "guanghe-new-bag" + assert len( + { + flagship.target_account_key, + luggage.target_account_key, + new_bag.target_account_key, + } + ) == 3 assert tmall.STORE_ACCOUNT_KEYS == { tmall.STORE_FLAGSHIP: flagship.target_account_key, tmall.STORE_LUGGAGE: luggage.target_account_key, + tmall.STORE_NEW_BAG: new_bag.target_account_key, } @@ -199,6 +208,25 @@ def test_luggage_runtime_uses_new_store_account_label_without_changing_password_ assert password == "same-runtime-secret" +def test_new_bag_runtime_uses_its_own_credentials_and_profile( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GUANGHE_NEW_BAG_USERNAME", "gyxx箱包旗舰店:龙虾仔") + monkeypatch.setenv("GUANGHE_NEW_BAG_PASSWORD", "new-bag-runtime-secret") + + store_name, username, password, profile = tmall._store_runtime(tmall.STORE_NEW_BAG) + + assert store_name == "gyxx箱包旗舰店:龙虾仔" + assert username == "gyxx箱包旗舰店:龙虾仔" + assert password == "new-bag-runtime-secret" + assert str(profile).endswith("guanghe-new-bag") + assert profile != tmall._store_runtime(tmall.STORE_LUGGAGE)[3] + assert profile != tmall._store_runtime(tmall.STORE_FLAGSHIP)[3] + assert tmall._expected_creator_identity_markers( + tmall.STORE_NEW_BAG, store_name + ) == ("GYXX箱包天猫官方",) + + def test_account_identity_markers_accept_store_name_with_login_nickname() -> None: assert tmall.GuangHeUploader._account_identity_markers( "光影行星鑫华达专卖店:龙虾仔" diff --git a/tests/modules/product_commerce/test_wanxiang_report_template.py b/tests/modules/product_commerce/test_wanxiang_report_template.py index 6793d9c..cda570e 100644 --- a/tests/modules/product_commerce/test_wanxiang_report_template.py +++ b/tests/modules/product_commerce/test_wanxiang_report_template.py @@ -148,5 +148,80 @@ class WanxiangReportTemplateTests(unittest.TestCase): session.close.assert_called_once() +class WanxiangSearchKeywordTests(unittest.TestCase): + """搜索词可与款名不同，但款式身份始终是 style_name。""" + + def test_obsidian_toast_style_searches_shorter_keyword_first(self): + self.assertEqual( + wanxiang.search_keywords_for("黑曜石吐司包"), + ("黑曜石吐司", "黑曜石吐司包"), + ) + + def test_styles_without_override_keep_their_own_name(self): + self.assertEqual(wanxiang.search_keywords_for("星云2"), ("星云2",)) + self.assertEqual( + wanxiang.search_keywords_for("黑曜石托特包"), ("黑曜石托特包",) + ) + + def test_collect_style_searches_keyword_but_keeps_style_identity(self): + searched: list[str] = [] + row = { + "creative_id": "c1", + "impressions": 10, + "clicks": 2, + "cost": 0.5, + "creative_image_url": "https://img/1.jpg", + } + with ( + patch.object( + wanxiang, + "search_style_by_subject", + side_effect=lambda _page, keyword: searched.append(keyword), + ), + patch.object(wanxiang, "extract_current_page_rows", return_value=[row]), + patch.object(wanxiang, "collect_current_page_image_map", return_value={}), + patch.object(wanxiang, "click_next_page", return_value=False), + patch.object(wanxiang, "aggregate_rows", return_value={}), + ): + result = wanxiang.collect_style( + Mock(), "黑曜石吐司包", 1, item_ids=["1022486093557"] + ) + + self.assertEqual(searched, ["黑曜石吐司"]) + self.assertEqual(result["style_name"], "黑曜石吐司包") + self.assertEqual(result["search_keyword"], "黑曜石吐司") + + def test_empty_keyword_falls_back_to_original_style_name(self): + state = {"searched": [], "checks": 0} + row = { + "creative_id": "c1", + "impressions": 10, + "clicks": 2, + "cost": 0.5, + "creative_image_url": "https://img/1.jpg", + } + + def fake_extract(_page): + state["checks"] += 1 + return [] if state["checks"] == 1 else [row] + + with ( + patch.object( + wanxiang, + "search_style_by_subject", + side_effect=lambda _page, keyword: state["searched"].append(keyword), + ), + patch.object(wanxiang, "extract_current_page_rows", side_effect=fake_extract), + patch.object(wanxiang, "collect_current_page_image_map", return_value={}), + patch.object(wanxiang, "click_next_page", return_value=False), + patch.object(wanxiang, "aggregate_rows", return_value={}), + ): + result = wanxiang.collect_style(Mock(), "黑曜石吐司包", 1, item_ids=["1022"]) + + self.assertEqual(state["searched"], ["黑曜石吐司", "黑曜石吐司包"]) + self.assertEqual(result["style_name"], "黑曜石吐司包") + self.assertEqual(result["search_keyword"], "黑曜石吐司包") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_console.py b/tests/test_console.py index 1b138ed..cda5878 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -1709,6 +1709,28 @@ def test_tmall_video_trigger_routes_only_the_selected_store( assert flagship["store"] == "flagship" assert flagship["topic_keyword"] == "我的夏日焕新清单" assert launcher.calls[-1]["store"] == "flagship" + launcher.complete("product.video_upload") + + new_bag = service.trigger( + "product.video_upload", + {"business_date": "2026-08-11", "store": "new_bag"}, + ) + + assert new_bag["store"] == "new_bag" + assert launcher.calls[-1]["store"] == "new_bag" + + +def test_tmall_video_trigger_rejects_unknown_store(console_settings: Settings) -> None: + service = WorkflowConsoleService( + console_settings, + launcher=FakeConsoleLauncher(), + ) + + with pytest.raises(ConsoleRequestError, match="店铺参数无效"): + service.trigger( + "product.video_upload", + {"business_date": "2026-08-11", "store": "obsidian"}, + ) @pytest.mark.parametrize("value", [123, True, "x" * 121]) @@ -2824,6 +2846,8 @@ def test_http_home_assets_and_overview_have_security_headers( assert "强制重跑：忽略幂等跳过与已有结果，重新采集并正式执行" in page assert '' in page assert '' in page + assert '